diff --git a/.mailmap b/.mailmap
index 8a99d77389b24..4c787a4560399 100644
--- a/.mailmap
+++ b/.mailmap
@@ -36,6 +36,8 @@ Jon Roelofs <jonathan_roelofs@apple.com> <jonathan@codesourcery.com>
 Jon Roelofs <jonathan_roelofs@apple.com> <jroelofs@jroelofs.com>
 LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
 Martin Storsjö <martin@martin.st>
+Med Ismail Bennani <ismail@bennani.ma> <m.i.b@apple.com>
+Med Ismail Bennani <ismail@bennani.ma> <medismail.bennani@gmail.com>
 Ramkumar Ramachandra <r@artagnon.com> <artagnon@gmail.com>
 Saleem Abdulrasool <compnerd@compnerd.org>
 Tommy Chiang <ty1208chiang@gmail.com> <oToToT@users.noreply.github.com>
diff --git a/bolt/docs/doxygen.cfg.in b/bolt/docs/doxygen.cfg.in
index acdf6f66842b6..421ef4760aa37 100644
--- a/bolt/docs/doxygen.cfg.in
+++ b/bolt/docs/doxygen.cfg.in
@@ -2107,7 +2107,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2152,7 +2152,7 @@ TEMPLATE_RELATIONS     = NO
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2161,7 +2161,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h
index 8c044597b5fd0..c9694ab29d440 100644
--- a/bolt/include/bolt/Core/BinaryBasicBlock.h
+++ b/bolt/include/bolt/Core/BinaryBasicBlock.h
@@ -144,6 +144,9 @@ class BinaryBasicBlock {
   /// blocks may contain out of date or incorrect information.
   bool IsValid{true};
 
+  /// Last computed hash value.
+  mutable uint64_t Hash{0};
+
 private:
   BinaryBasicBlock() = delete;
   BinaryBasicBlock(const BinaryBasicBlock &) = delete;
@@ -939,6 +942,9 @@ class BinaryBasicBlock {
   /// Check if the block has a jump table instruction.
   bool hasJumpTable() const { return getJumpTable() != nullptr; }
 
+  /// Returns the last computed hash value of the block.
+  uint64_t getHash() const { return Hash; }
+
 private:
   void adjustNumPseudos(const MCInst &Inst, int Sign);
 
@@ -963,6 +969,9 @@ class BinaryBasicBlock {
   /// Set the index of this basic block.
   void setIndex(unsigned I) { Index = I; }
 
+  /// Sets the hash value of the basic block.
+  void setHash(uint64_t Value) const { Hash = Value; }
+
   template <typename T> void clearList(T &List) {
     T TempList;
     TempList.swap(List);
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index a96243c579eb0..a54492c290f00 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -2205,6 +2205,9 @@ class BinaryFunction {
         return std::string();
       }) const;
 
+  /// Compute hash values for each block of the function.
+  void computeBlockHashes() const;
+
   void setDWARFUnit(DWARFUnit *Unit) { DwarfUnit = Unit; }
 
   /// Return DWARF compile unit for this function.
diff --git a/bolt/include/bolt/Core/HashUtilities.h b/bolt/include/bolt/Core/HashUtilities.h
new file mode 100644
index 0000000000000..8d445ff837564
--- /dev/null
+++ b/bolt/include/bolt/Core/HashUtilities.h
@@ -0,0 +1,41 @@
+//===- bolt/Core/HashUtilities.h - Misc hash utilities --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains functions for computing hash values over BinaryFunction
+// and BinaryBasicBlock.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BOLT_CORE_HASH_UTILITIES_H
+#define BOLT_CORE_HASH_UTILITIES_H
+
+#include "bolt/Core/BinaryBasicBlock.h"
+#include "bolt/Core/BinaryContext.h"
+
+namespace llvm {
+namespace bolt {
+
+uint16_t hash_64_to_16(const uint64_t Hash);
+
+std::string hashInteger(uint64_t Value);
+
+std::string hashSymbol(BinaryContext &BC, const MCSymbol &Symbol);
+
+std::string hashExpr(BinaryContext &BC, const MCExpr &Expr);
+
+std::string hashInstOperand(BinaryContext &BC, const MCOperand &Operand);
+
+using OperandHashFuncTy = function_ref<typename std::string(const MCOperand &)>;
+
+std::string hashBlock(BinaryContext &BC, const BinaryBasicBlock &BB,
+                      OperandHashFuncTy OperandHashFunc);
+
+} // namespace bolt
+} // namespace llvm
+
+#endif
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index e6eb1429d3a18..86756a4434ed9 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -466,6 +466,11 @@ class MCPlusBuilder {
 
   virtual MCPhysReg getX86R11() const { llvm_unreachable("not implemented"); }
 
+  virtual unsigned getShortBranchOpcode(unsigned Opcode) const {
+    llvm_unreachable("not implemented");
+    return 0;
+  }
+
   /// Create increment contents of target by 1 for Instrumentation
   virtual InstructionListType createInstrIncMemory(const MCSymbol *Target,
                                                    MCContext *Ctx,
diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h
index e20d449e0b93e..f02f2197c469a 100644
--- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h
+++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h
@@ -126,6 +126,7 @@ template <> struct MappingTraits<bolt::BinaryBasicBlockProfile> {
   static void mapping(IO &YamlIO, bolt::BinaryBasicBlockProfile &BBP) {
     YamlIO.mapRequired("bid", BBP.Index);
     YamlIO.mapRequired("insns", BBP.NumInstructions);
+    YamlIO.mapOptional("hash", BBP.Hash, (llvm::yaml::Hex64)0);
     YamlIO.mapOptional("exec", BBP.ExecCount, (uint64_t)0);
     YamlIO.mapOptional("events", BBP.EventCount, (uint64_t)0);
     YamlIO.mapOptional("calls", BBP.CallSites,
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 89ba3627cdd7d..47c6b7eddaf83 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -14,6 +14,7 @@
 #include "bolt/Core/BinaryBasicBlock.h"
 #include "bolt/Core/BinaryDomTree.h"
 #include "bolt/Core/DynoStats.h"
+#include "bolt/Core/HashUtilities.h"
 #include "bolt/Core/MCPlusBuilder.h"
 #include "bolt/Utils/NameResolver.h"
 #include "bolt/Utils/NameShortener.h"
@@ -3604,34 +3605,18 @@ size_t BinaryFunction::computeHash(bool UseDFS,
   // The hash is computed by creating a string of all instruction opcodes and
   // possibly their operands and then hashing that string with std::hash.
   std::string HashString;
-  for (const BinaryBasicBlock *BB : Order) {
-    for (const MCInst &Inst : *BB) {
-      unsigned Opcode = Inst.getOpcode();
-
-      if (BC.MIB->isPseudo(Inst))
-        continue;
+  for (const BinaryBasicBlock *BB : Order)
+    HashString.append(hashBlock(BC, *BB, OperandHashFunc));
 
-      // Ignore unconditional jumps since we check CFG consistency by processing
-      // basic blocks in order and do not rely on branches to be in-sync with
-      // CFG. Note that we still use condition code of conditional jumps.
-      if (BC.MIB->isUnconditionalBranch(Inst))
-        continue;
-
-      if (Opcode == 0)
-        HashString.push_back(0);
-
-      while (Opcode) {
-        uint8_t LSB = Opcode & 0xff;
-        HashString.push_back(LSB);
-        Opcode = Opcode >> 8;
-      }
+  return Hash = std::hash<std::string>{}(HashString);
+}
 
-      for (const MCOperand &Op : MCPlus::primeOperands(Inst))
-        HashString.append(OperandHashFunc(Op));
-    }
+void BinaryFunction::computeBlockHashes() const {
+  for (const BinaryBasicBlock *BB : BasicBlocks) {
+    std::string Hash =
+        hashBlock(BC, *BB, [](const MCOperand &Op) { return std::string(); });
+    BB->setHash(std::hash<std::string>{}(Hash));
   }
-
-  return Hash = std::hash<std::string>{}(HashString);
 }
 
 void BinaryFunction::insertBasicBlocks(
diff --git a/bolt/lib/Core/CMakeLists.txt b/bolt/lib/Core/CMakeLists.txt
index e7215b2babd65..7c633e37eeb77 100644
--- a/bolt/lib/Core/CMakeLists.txt
+++ b/bolt/lib/Core/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_library(LLVMBOLTCore
   DynoStats.cpp
   Exceptions.cpp
   FunctionLayout.cpp
+  HashUtilities.cpp
   JumpTable.cpp
   MCPlusBuilder.cpp
   ParallelUtilities.cpp
diff --git a/bolt/lib/Core/DebugData.cpp b/bolt/lib/Core/DebugData.cpp
index 2b56357df9c03..f1204925a98a4 100644
--- a/bolt/lib/Core/DebugData.cpp
+++ b/bolt/lib/Core/DebugData.cpp
@@ -732,8 +732,16 @@ void DebugLoclistWriter::addList(AttrInfo &AttrVal,
 uint32_t DebugLoclistWriter::LoclistBaseOffset = 0;
 void DebugLoclistWriter::finalizeDWARF5(
     DebugInfoBinaryPatcher &DebugInfoPatcher, DebugAbbrevWriter &AbbrevWriter) {
-  if (LocBodyBuffer->empty())
+  if (LocBodyBuffer->empty()) {
+    std::optional<AttrInfo> AttrInfoVal =
+        findAttributeInfo(CU.getUnitDIE(), dwarf::DW_AT_loclists_base);
+    // Pointing to first one, because it doesn't matter. There are no uses of it
+    // in this CU.
+    if (!isSplitDwarf() && AttrInfoVal)
+      DebugInfoPatcher.addLE32Patch(AttrInfoVal->Offset,
+                                    getDWARF5RngListLocListHeaderSize());
     return;
+  }
 
   std::unique_ptr<DebugBufferVector> LocArrayBuffer =
       std::make_unique<DebugBufferVector>();
diff --git a/bolt/lib/Core/HashUtilities.cpp b/bolt/lib/Core/HashUtilities.cpp
new file mode 100644
index 0000000000000..1c980e003f884
--- /dev/null
+++ b/bolt/lib/Core/HashUtilities.cpp
@@ -0,0 +1,135 @@
+//===- bolt/Core/HashUtilities.cpp - Misc hash utilities ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Computation of hash values over BinaryFunction and BinaryBasicBlock.
+//
+//===----------------------------------------------------------------------===//
+
+#include "bolt/Core/HashUtilities.h"
+#include "bolt/Core/BinaryContext.h"
+#include "bolt/Core/BinaryFunction.h"
+
+namespace llvm {
+namespace bolt {
+
+/// Hashing a 64-bit integer to a 16-bit one.
+uint16_t hash_64_to_16(const uint64_t Hash) {
+  uint16_t Res = (uint16_t)(Hash & 0xFFFF);
+  Res ^= (uint16_t)((Hash >> 16) & 0xFFFF);
+  Res ^= (uint16_t)((Hash >> 32) & 0xFFFF);
+  Res ^= (uint16_t)((Hash >> 48) & 0xFFFF);
+  return Res;
+}
+
+std::string hashInteger(uint64_t Value) {
+  std::string HashString;
+  if (Value == 0)
+    HashString.push_back(0);
+
+  while (Value) {
+    uint8_t LSB = Value & 0xff;
+    HashString.push_back(LSB);
+    Value >>= 8;
+  }
+
+  return HashString;
+}
+
+std::string hashSymbol(BinaryContext &BC, const MCSymbol &Symbol) {
+  std::string HashString;
+
+  // Ignore function references.
+  if (BC.getFunctionForSymbol(&Symbol))
+    return HashString;
+
+  llvm::ErrorOr<uint64_t> ErrorOrValue = BC.getSymbolValue(Symbol);
+  if (!ErrorOrValue)
+    return HashString;
+
+  // Ignore jump table references.
+  if (BC.getJumpTableContainingAddress(*ErrorOrValue))
+    return HashString;
+
+  return HashString.append(hashInteger(*ErrorOrValue));
+}
+
+std::string hashExpr(BinaryContext &BC, const MCExpr &Expr) {
+  switch (Expr.getKind()) {
+  case MCExpr::Constant:
+    return hashInteger(cast<MCConstantExpr>(Expr).getValue());
+  case MCExpr::SymbolRef:
+    return hashSymbol(BC, cast<MCSymbolRefExpr>(Expr).getSymbol());
+  case MCExpr::Unary: {
+    const auto &UnaryExpr = cast<MCUnaryExpr>(Expr);
+    return hashInteger(UnaryExpr.getOpcode())
+        .append(hashExpr(BC, *UnaryExpr.getSubExpr()));
+  }
+  case MCExpr::Binary: {
+    const auto &BinaryExpr = cast<MCBinaryExpr>(Expr);
+    return hashExpr(BC, *BinaryExpr.getLHS())
+        .append(hashInteger(BinaryExpr.getOpcode()))
+        .append(hashExpr(BC, *BinaryExpr.getRHS()));
+  }
+  case MCExpr::Target:
+    return std::string();
+  }
+
+  llvm_unreachable("invalid expression kind");
+}
+
+std::string hashInstOperand(BinaryContext &BC, const MCOperand &Operand) {
+  if (Operand.isImm())
+    return hashInteger(Operand.getImm());
+  if (Operand.isReg())
+    return hashInteger(Operand.getReg());
+  if (Operand.isExpr())
+    return hashExpr(BC, *Operand.getExpr());
+
+  return std::string();
+}
+
+std::string hashBlock(BinaryContext &BC, const BinaryBasicBlock &BB,
+                      OperandHashFuncTy OperandHashFunc) {
+  const bool IsX86 = BC.isX86();
+
+  // The hash is computed by creating a string of all instruction opcodes and
+  // possibly their operands and then hashing that string with std::hash.
+  std::string HashString;
+
+  for (const MCInst &Inst : BB) {
+    if (BC.MIB->isPseudo(Inst))
+      continue;
+
+    unsigned Opcode = Inst.getOpcode();
+
+    // Ignore unconditional jumps since we check CFG consistency by processing
+    // basic blocks in order and do not rely on branches to be in-sync with
+    // CFG. Note that we still use condition code of conditional jumps.
+    if (BC.MIB->isUnconditionalBranch(Inst))
+      continue;
+
+    if (IsX86 && BC.MIB->isConditionalBranch(Inst))
+      Opcode = BC.MIB->getShortBranchOpcode(Opcode);
+
+    if (Opcode == 0)
+      HashString.push_back(0);
+
+    while (Opcode) {
+      uint8_t LSB = Opcode & 0xff;
+      HashString.push_back(LSB);
+      Opcode = Opcode >> 8;
+    }
+
+    for (const MCOperand &Op : MCPlus::primeOperands(Inst))
+      HashString.append(OperandHashFunc(Op));
+  }
+  return HashString;
+}
+
+} // namespace bolt
+} // namespace llvm
diff --git a/bolt/lib/Passes/AsmDump.cpp b/bolt/lib/Passes/AsmDump.cpp
index c163345fe7f2e..18d0395cbc4ad 100644
--- a/bolt/lib/Passes/AsmDump.cpp
+++ b/bolt/lib/Passes/AsmDump.cpp
@@ -55,34 +55,6 @@ void dumpCFI(const BinaryFunction &BF, const MCInst &Instr, AsmPrinter &MAP) {
   }
 }
 
-void dumpJumpTableFdata(raw_ostream &OS, const BinaryFunction &BF,
-                        const MCInst &Instr, const std::string &BranchLabel) {
-  StringRef FunctionName = BF.getOneName();
-  const JumpTable *JT = BF.getJumpTable(Instr);
-  for (uint32_t i = 0; i < JT->Entries.size(); ++i) {
-    StringRef TargetName = JT->Entries[i]->getName();
-    const uint64_t Mispreds = JT->Counts[i].Mispreds;
-    const uint64_t Count = JT->Counts[i].Count;
-    OS << "# FDATA: 1 " << FunctionName << " #" << BranchLabel << "# "
-       << "1 " << FunctionName << " #" << TargetName << "# " << Mispreds << " "
-       << Count << '\n';
-  }
-}
-
-void dumpTailCallFdata(raw_ostream &OS, const BinaryFunction &BF,
-                       const MCInst &Instr, const std::string &BranchLabel) {
-  const BinaryContext &BC = BF.getBinaryContext();
-  StringRef FunctionName = BF.getOneName();
-  auto CallFreq = BC.MIB->getAnnotationWithDefault<uint64_t>(Instr, "Count");
-  const MCSymbol *Target = BC.MIB->getTargetSymbol(Instr);
-  const BinaryFunction *TargetBF = BC.getFunctionForSymbol(Target);
-  if (!TargetBF)
-    return;
-  OS << "# FDATA: 1 " << FunctionName << " #" << BranchLabel << "# "
-     << "1 " << TargetBF->getPrintName() << " 0 "
-     << "0 " << CallFreq << '\n';
-}
-
 void dumpTargetFunctionStub(raw_ostream &OS, const BinaryContext &BC,
                             const MCSymbol *CalleeSymb,
                             const BinarySection *&LastCS) {
@@ -230,12 +202,6 @@ void dumpFunction(const BinaryFunction &BF) {
 
       BC.InstPrinter->printInst(&Instr, 0, "", *BC.STI, OS);
       OS << '\n';
-
-      // Dump profile data in FDATA format (as parsed by link_fdata).
-      if (BC.MIB->getJumpTable(Instr))
-        dumpJumpTableFdata(OS, BF, Instr, BranchLabel);
-      else if (BC.MIB->isTailCall(Instr))
-        dumpTailCallFdata(OS, BF, Instr, BranchLabel);
     }
 
     // Dump profile data in FDATA format (as parsed by link_fdata).
diff --git a/bolt/lib/Passes/IdenticalCodeFolding.cpp b/bolt/lib/Passes/IdenticalCodeFolding.cpp
index ccf1749c47bf3..e96666b37bd93 100644
--- a/bolt/lib/Passes/IdenticalCodeFolding.cpp
+++ b/bolt/lib/Passes/IdenticalCodeFolding.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "bolt/Passes/IdenticalCodeFolding.h"
+#include "bolt/Core/HashUtilities.h"
 #include "bolt/Core/ParallelUtilities.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/CommandLine.h"
@@ -337,74 +338,6 @@ typedef std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>,
                            KeyHash, KeyEqual>
     IdenticalBucketsMap;
 
-static std::string hashInteger(uint64_t Value) {
-  std::string HashString;
-  if (Value == 0)
-    HashString.push_back(0);
-
-  while (Value) {
-    uint8_t LSB = Value & 0xff;
-    HashString.push_back(LSB);
-    Value >>= 8;
-  }
-
-  return HashString;
-}
-
-static std::string hashSymbol(BinaryContext &BC, const MCSymbol &Symbol) {
-  std::string HashString;
-
-  // Ignore function references.
-  if (BC.getFunctionForSymbol(&Symbol))
-    return HashString;
-
-  llvm::ErrorOr<uint64_t> ErrorOrValue = BC.getSymbolValue(Symbol);
-  if (!ErrorOrValue)
-    return HashString;
-
-  // Ignore jump table references.
-  if (BC.getJumpTableContainingAddress(*ErrorOrValue))
-    return HashString;
-
-  return HashString.append(hashInteger(*ErrorOrValue));
-}
-
-static std::string hashExpr(BinaryContext &BC, const MCExpr &Expr) {
-  switch (Expr.getKind()) {
-  case MCExpr::Constant:
-    return hashInteger(cast<MCConstantExpr>(Expr).getValue());
-  case MCExpr::SymbolRef:
-    return hashSymbol(BC, cast<MCSymbolRefExpr>(Expr).getSymbol());
-  case MCExpr::Unary: {
-    const auto &UnaryExpr = cast<MCUnaryExpr>(Expr);
-    return hashInteger(UnaryExpr.getOpcode())
-        .append(hashExpr(BC, *UnaryExpr.getSubExpr()));
-  }
-  case MCExpr::Binary: {
-    const auto &BinaryExpr = cast<MCBinaryExpr>(Expr);
-    return hashExpr(BC, *BinaryExpr.getLHS())
-        .append(hashInteger(BinaryExpr.getOpcode()))
-        .append(hashExpr(BC, *BinaryExpr.getRHS()));
-  }
-  case MCExpr::Target:
-    return std::string();
-  }
-
-  llvm_unreachable("invalid expression kind");
-}
-
-static std::string hashInstOperand(BinaryContext &BC,
-                                   const MCOperand &Operand) {
-  if (Operand.isImm())
-    return hashInteger(Operand.getImm());
-  if (Operand.isReg())
-    return hashInteger(Operand.getReg());
-  if (Operand.isExpr())
-    return hashExpr(BC, *Operand.getExpr());
-
-  return std::string();
-}
-
 namespace llvm {
 namespace bolt {
 
diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp
index bee18a6c1627e..01ff9e2c26dee 100644
--- a/bolt/lib/Profile/YAMLProfileWriter.cpp
+++ b/bolt/lib/Profile/YAMLProfileWriter.cpp
@@ -28,9 +28,13 @@ void convert(const BinaryFunction &BF,
 
   const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR;
 
+  // Prepare function and block hashes
+  BF.computeHash(/*UseDFS=*/true);
+  BF.computeBlockHashes();
+
   YamlBF.Name = BF.getPrintName();
   YamlBF.Id = BF.getFunctionNumber();
-  YamlBF.Hash = BF.computeHash(/*UseDFS=*/true);
+  YamlBF.Hash = BF.getHash();
   YamlBF.NumBasicBlocks = BF.size();
   YamlBF.ExecCount = BF.getKnownExecutionCount();
 
@@ -38,6 +42,7 @@ void convert(const BinaryFunction &BF,
     yaml::bolt::BinaryBasicBlockProfile YamlBB;
     YamlBB.Index = BB->getLayoutIndex();
     YamlBB.NumInstructions = BB->getNumNonPseudos();
+    YamlBB.Hash = BB->getHash();
 
     if (!LBRProfile) {
       YamlBB.EventCount = BB->getKnownExecutionCount();
@@ -112,11 +117,15 @@ void convert(const BinaryFunction &BF,
     // Include landing pads with non-zero execution count.
     if (YamlBB.CallSites.empty() && !BB->isEntryPoint() &&
         !(BB->isLandingPad() && BB->getKnownExecutionCount() != 0)) {
+      // Include blocks having successors or predecessors with positive counts.
       uint64_t SuccessorExecCount = 0;
       for (const BinaryBasicBlock::BinaryBranchInfo &BranchInfo :
            BB->branch_info())
         SuccessorExecCount += BranchInfo.Count;
-      if (!SuccessorExecCount)
+      uint64_t PredecessorExecCount = 0;
+      for (auto Pred : BB->predecessors())
+        PredecessorExecCount += Pred->getBranchInfo(*BB).Count;
+      if (!SuccessorExecCount && !PredecessorExecCount)
         continue;
     }
 
diff --git a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp
index 8c41ca97fd353..bc0dd2faa694c 100644
--- a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp
+++ b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp
@@ -27,6 +27,12 @@ uint8_t *ExecutableFileMemoryManager::allocateSection(
   uint8_t *Ret = static_cast<uint8_t *>(llvm::allocate_buffer(Size, Alignment));
   AllocatedSections.push_back(AllocInfo{Ret, Size, Alignment});
 
+  // A Size of 1 might mean an empty section for which RuntimeDyld decided to
+  // allocate 1 byte. In this case, the allocation will never be initialized
+  // causing non-deterministic output section contents.
+  if (Size == 1)
+    *Ret = 0;
+
   // Register a debug section as a note section.
   if (!ObjectsLoaded && RewriteInstance::isDebugSection(SectionName)) {
     BinarySection &Section =
diff --git a/bolt/lib/Rewrite/MachORewriteInstance.cpp b/bolt/lib/Rewrite/MachORewriteInstance.cpp
index f1f20f278b112..8078dc4e3fd24 100644
--- a/bolt/lib/Rewrite/MachORewriteInstance.cpp
+++ b/bolt/lib/Rewrite/MachORewriteInstance.cpp
@@ -527,6 +527,7 @@ void MachORewriteInstance::emitAndLink() {
   mapCodeSections();
   mapInstrumentationSection("__counters");
   mapInstrumentationSection("__tables");
+  RTDyld->finalizeWithMemoryManagerLocking();
 
           // TODO: Refactor addRuntimeLibSections to work properly on Mach-O
           // and use it here.
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index ad80255dcf35e..5a45f14db5979 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -50,17 +50,6 @@ static cl::opt<bool> X86StripRedundantAddressSize(
 
 namespace {
 
-unsigned getShortBranchOpcode(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    return Opcode;
-  case X86::JMP_2: return X86::JMP_1;
-  case X86::JMP_4: return X86::JMP_1;
-  case X86::JCC_2: return X86::JCC_1;
-  case X86::JCC_4: return X86::JCC_1;
-  }
-}
-
 unsigned getShortArithOpcode(unsigned Opcode) {
   return X86::getShortOpcodeArith(Opcode);
 }
@@ -2867,6 +2856,21 @@ class X86MCPlusBuilder : public MCPlusBuilder {
 
   MCPhysReg getX86R11() const override { return X86::R11; }
 
+  unsigned getShortBranchOpcode(unsigned Opcode) const override {
+    switch (Opcode) {
+    default:
+      return Opcode;
+    case X86::JMP_2:
+      return X86::JMP_1;
+    case X86::JMP_4:
+      return X86::JMP_1;
+    case X86::JCC_2:
+      return X86::JCC_1;
+    case X86::JCC_4:
+      return X86::JCC_1;
+    }
+  }
+
   MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
     // FIXME: this should depend on the calling convention.
     switch (ArgNo) {
diff --git a/bolt/test/X86/Inputs/dwarf5-loc-base-no-loc-accesshelper.s b/bolt/test/X86/Inputs/dwarf5-loc-base-no-loc-accesshelper.s
new file mode 100644
index 0000000000000..dcf9f6a23a07b
--- /dev/null
+++ b/bolt/test/X86/Inputs/dwarf5-loc-base-no-loc-accesshelper.s
@@ -0,0 +1,410 @@
+# clang++ helper.cpp -g -O2 -S
+# int fooVar = 0;
+# void useFoo(int * x) {
+# *x += 4;
+# }
+#
+# int  foo(int argc) {
+#    int x = argc;
+#    useFoo(&x);
+#    return x;
+# }
+# Manually removed DW_AT_location
+	.text
+	.file	"foo.cpp"
+	.file	0 "/testLocListMultiple" "foo.cpp" md5 0x9410f31145d031fcc1f2464b809e409a
+	.globl	_Z6useFooPi                     # -- Begin function _Z6useFooPi
+	.p2align	4, 0x90
+	.type	_Z6useFooPi,@function
+_Z6useFooPi:                            # @_Z6useFooPi
+.Lfunc_begin0:
+	.loc	0 2 0                           # foo.cpp:2:0
+	.cfi_startproc
+# %bb.0:                                # %entry
+	#DEBUG_VALUE: useFoo:x <- $rdi
+	.loc	0 3 4 prologue_end              # foo.cpp:3:4
+	addl	$4, (%rdi)
+	.loc	0 4 1                           # foo.cpp:4:1
+	retq
+.Ltmp0:
+.Lfunc_end0:
+	.size	_Z6useFooPi, .Lfunc_end0-_Z6useFooPi
+	.cfi_endproc
+                                        # -- End function
+	.globl	_Z3fooi                         # -- Begin function _Z3fooi
+	.p2align	4, 0x90
+	.type	_Z3fooi,@function
+_Z3fooi:                                # @_Z3fooi
+.Lfunc_begin1:
+	.loc	0 6 0                           # foo.cpp:6:0
+	.cfi_startproc
+# %bb.0:                                # %entry
+	#DEBUG_VALUE: foo:argc <- $edi
+	#DEBUG_VALUE: foo:x <- $edi
+                                        # kill: def $edi killed $edi def $rdi
+	#DEBUG_VALUE: useFoo:x <- undef
+	.loc	0 3 4 prologue_end              # foo.cpp:3:4
+	leal	4(%rdi), %eax
+.Ltmp1:
+	#DEBUG_VALUE: foo:x <- $eax
+	.loc	0 9 4                           # foo.cpp:9:4
+	retq
+.Ltmp2:
+.Lfunc_end1:
+	.size	_Z3fooi, .Lfunc_end1-_Z3fooi
+	.cfi_endproc
+                                        # -- End function
+	.type	fooVar,@object                  # @fooVar
+	.bss
+	.globl	fooVar
+	.p2align	2
+fooVar:
+	.long	0                               # 0x0
+	.size	fooVar, 4
+
+	.section	.debug_loclists,"",@progbits
+	.long	.Ldebug_list_header_end0-.Ldebug_list_header_start0 # Length
+.Ldebug_list_header_start0:
+	.short	5                               # Version
+	.byte	8                               # Address size
+	.byte	0                               # Segment selector size
+	.long	1                               # Offset entry count
+.Lloclists_table_base0:
+	.long	.Ldebug_loc0-.Lloclists_table_base0
+.Ldebug_loc0:
+	.byte	4                               # DW_LLE_offset_pair
+	.uleb128 .Lfunc_begin1-.Lfunc_begin0    #   starting offset
+	.uleb128 .Ltmp1-.Lfunc_begin0           #   ending offset
+	.byte	1                               # Loc expr size
+	.byte	85                              # super-register DW_OP_reg5
+	.byte	4                               # DW_LLE_offset_pair
+	.uleb128 .Ltmp1-.Lfunc_begin0           #   starting offset
+	.uleb128 .Lfunc_end1-.Lfunc_begin0      #   ending offset
+	.byte	1                               # Loc expr size
+	.byte	80                              # super-register DW_OP_reg0
+	.byte	0                               # DW_LLE_end_of_list
+.Ldebug_list_header_end0:
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                               # Abbreviation Code
+	.byte	17                              # DW_TAG_compile_unit
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	37                              # DW_AT_producer
+	.byte	37                              # DW_FORM_strx1
+	.byte	19                              # DW_AT_language
+	.byte	5                               # DW_FORM_data2
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	114                             # DW_AT_str_offsets_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	16                              # DW_AT_stmt_list
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	27                              # DW_AT_comp_dir
+	.byte	37                              # DW_FORM_strx1
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	115                             # DW_AT_addr_base
+	.byte	23                              # DW_FORM_sec_offset
+	.ascii	"\214\001"                      # DW_AT_loclists_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	2                               # Abbreviation Code
+	.byte	52                              # DW_TAG_variable
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	3                               # Abbreviation Code
+	.byte	36                              # DW_TAG_base_type
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	62                              # DW_AT_encoding
+	.byte	11                              # DW_FORM_data1
+	.byte	11                              # DW_AT_byte_size
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	4                               # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	64                              # DW_AT_frame_base
+	.byte	24                              # DW_FORM_exprloc
+	.byte	122                             # DW_AT_call_all_calls
+	.byte	25                              # DW_FORM_flag_present
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	5                               # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	6                               # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	110                             # DW_AT_linkage_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	32                              # DW_AT_inline
+	.byte	33                              # DW_FORM_implicit_const
+	.byte	1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	7                               # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	8                               # Abbreviation Code
+	.byte	15                              # DW_TAG_pointer_type
+	.byte	0                               # DW_CHILDREN_no
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	9                               # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	64                              # DW_AT_frame_base
+	.byte	24                              # DW_FORM_exprloc
+	.byte	122                             # DW_AT_call_all_calls
+	.byte	25                              # DW_FORM_flag_present
+	.byte	110                             # DW_AT_linkage_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	10                              # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	11                              # Abbreviation Code
+	.byte	52                              # DW_TAG_variable
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	12                              # Abbreviation Code
+	.byte	29                              # DW_TAG_inlined_subroutine
+	.byte	0                               # DW_CHILDREN_no
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	19                              # DW_FORM_ref4
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	88                              # DW_AT_call_file
+	.byte	11                              # DW_FORM_data1
+	.byte	89                              # DW_AT_call_line
+	.byte	11                              # DW_FORM_data1
+	.byte	87                              # DW_AT_call_column
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	0                               # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	5                               # DWARF version number
+	.byte	1                               # DWARF Unit Type
+	.byte	8                               # Address Size (in bytes)
+	.long	.debug_abbrev                   # Offset Into Abbrev. Section
+	.byte	1                               # Abbrev [1] 0xc:0x83 DW_TAG_compile_unit
+	.byte	0                               # DW_AT_producer
+	.short	33                              # DW_AT_language
+	.byte	1                               # DW_AT_name
+	.long	.Lstr_offsets_base0             # DW_AT_str_offsets_base
+	.long	.Lline_table_start0             # DW_AT_stmt_list
+	.byte	2                               # DW_AT_comp_dir
+	.byte	1                               # DW_AT_low_pc
+	.long	.Lfunc_end1-.Lfunc_begin0       # DW_AT_high_pc
+	.long	.Laddr_table_base0              # DW_AT_addr_base
+	.long	.Lloclists_table_base0          # DW_AT_loclists_base
+	.byte	2                               # Abbrev [2] 0x27:0xb DW_TAG_variable
+	.byte	3                               # DW_AT_name
+	.long	50                              # DW_AT_type
+                                        # DW_AT_external
+	.byte	0                               # DW_AT_decl_file
+	.byte	1                               # DW_AT_decl_line
+	.byte	0
+	.byte	3                               # Abbrev [3] 0x32:0x4 DW_TAG_base_type
+	.byte	4                               # DW_AT_name
+	.byte	5                               # DW_AT_encoding
+	.byte	4                               # DW_AT_byte_size
+	.byte	4                               # Abbrev [4] 0x36:0x14 DW_TAG_subprogram
+	.byte	1                               # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.byte	1                               # DW_AT_frame_base
+	.byte	87
+                                        # DW_AT_call_all_calls
+	.long	74                              # DW_AT_abstract_origin
+	.byte	5                               # Abbrev [5] 0x42:0x7 DW_TAG_formal_parameter
+	.long	79                              # DW_AT_abstract_origin
+	.byte	0                               # End Of Children Mark
+	.byte	6                               # Abbrev [6] 0x4a:0xe DW_TAG_subprogram
+	.byte	5                               # DW_AT_linkage_name
+	.byte	6                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+                                        # DW_AT_external
+                                        # DW_AT_inline
+	.byte	7                               # Abbrev [7] 0x4f:0x8 DW_TAG_formal_parameter
+	.byte	7                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+	.long	88                              # DW_AT_type
+	.byte	0                               # End Of Children Mark
+	.byte	8                               # Abbrev [8] 0x58:0x5 DW_TAG_pointer_type
+	.long	50                              # DW_AT_type
+	.byte	9                               # Abbrev [9] 0x5d:0x31 DW_TAG_subprogram
+	.byte	2                               # DW_AT_low_pc
+	.long	.Lfunc_end1-.Lfunc_begin1       # DW_AT_high_pc
+	.byte	1                               # DW_AT_frame_base
+	.byte	87
+                                        # DW_AT_call_all_calls
+	.byte	8                               # DW_AT_linkage_name
+	.byte	9                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	6                               # DW_AT_decl_line
+	.long	50                              # DW_AT_type
+                                        # DW_AT_external
+	.byte	10                              # Abbrev [10] 0x6d:0xa DW_TAG_formal_parameter
+	.byte	10                              # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	6                               # DW_AT_decl_line
+	.long	50                              # DW_AT_type
+	.byte	11                              # Abbrev [11] 0x77:0x9 DW_TAG_variable
+	.byte	7                               # DW_AT_name
+	.byte	0                               # DW_AT_decl_file
+	.byte	7                               # DW_AT_decl_line
+	.long	50                              # DW_AT_type
+	.byte	12                              # Abbrev [12] 0x80:0xd DW_TAG_inlined_subroutine
+	.long	74                              # DW_AT_abstract_origin
+	.byte	2                               # DW_AT_low_pc
+	.long	.Ltmp1-.Lfunc_begin1            # DW_AT_high_pc
+	.byte	0                               # DW_AT_call_file
+	.byte	8                               # DW_AT_call_line
+	.byte	4                               # DW_AT_call_column
+	.byte	0                               # End Of Children Mark
+	.byte	0                               # End Of Children Mark
+.Ldebug_info_end0:
+	.section	.debug_str_offsets,"",@progbits
+	.long	48                              # Length of String Offsets Set
+	.short	5
+	.short	0
+.Lstr_offsets_base0:
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"clang version 15.0.0)" # string offset=0
+.Linfo_string1:
+	.asciz	"foo.cpp"                       # string offset=134
+.Linfo_string2:
+	.asciz	"/testLocListMultiple" # string offset=142
+.Linfo_string3:
+	.asciz	"fooVar"                        # string offset=199
+.Linfo_string4:
+	.asciz	"int"                           # string offset=206
+.Linfo_string5:
+	.asciz	"_Z6useFooPi"                   # string offset=210
+.Linfo_string6:
+	.asciz	"useFoo"                        # string offset=222
+.Linfo_string7:
+	.asciz	"x"                             # string offset=229
+.Linfo_string8:
+	.asciz	"_Z3fooi"                       # string offset=231
+.Linfo_string9:
+	.asciz	"foo"                           # string offset=239
+.Linfo_string10:
+	.asciz	"argc"                          # string offset=243
+	.section	.debug_str_offsets,"",@progbits
+	.long	.Linfo_string0
+	.long	.Linfo_string1
+	.long	.Linfo_string2
+	.long	.Linfo_string3
+	.long	.Linfo_string4
+	.long	.Linfo_string5
+	.long	.Linfo_string6
+	.long	.Linfo_string7
+	.long	.Linfo_string8
+	.long	.Linfo_string9
+	.long	.Linfo_string10
+	.section	.debug_addr,"",@progbits
+	.long	.Ldebug_addr_end0-.Ldebug_addr_start0 # Length of contribution
+.Ldebug_addr_start0:
+	.short	5                               # DWARF version number
+	.byte	8                               # Address size
+	.byte	0                               # Segment selector size
+.Laddr_table_base0:
+	.quad	fooVar
+	.quad	.Lfunc_begin0
+	.quad	.Lfunc_begin1
+.Ldebug_addr_end0:
+	.ident	"clang version 15.0.0"
+	.section	".note.GNU-stack","",@progbits
+	.addrsig
+	.section	.debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/bolt/test/X86/MachO/Inputs/external_symbol.yaml b/bolt/test/X86/MachO/Inputs/external_symbol.yaml
new file mode 100644
index 0000000000000..049c39d0f7ba2
--- /dev/null
+++ b/bolt/test/X86/MachO/Inputs/external_symbol.yaml
@@ -0,0 +1,357 @@
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x1000007
+  cpusubtype:      0x80000003
+  filetype:        0x2
+  ncmds:           15
+  sizeofcmds:      1216
+  flags:           0x200085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         392
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          4096
+    fileoff:         0
+    filesize:        4096
+    maxprot:         5
+    initprot:        5
+    nsects:          4
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x100000F50
+        size:            70
+        offset:          0xF50
+        align:           4
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         554889E54883EC20C745FC00000000897DF8488975F0837DF8010F8E0D000000E8210000008945FCE910000000C745EC010000008B45EC83C0028945FC8B45FC4883C4205DC3
+      - sectname:        __stubs
+        segname:         __TEXT
+        addr:            0x100000F96
+        size:            6
+        offset:          0xF96
+        align:           1
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000408
+        reserved1:       0x0
+        reserved2:       0x6
+        reserved3:       0x0
+        content:         FF2574000000
+      - sectname:        __stub_helper
+        segname:         __TEXT
+        addr:            0x100000F9C
+        size:            28
+        offset:          0xF9C
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         6800000000E90200000000004C8D1D510000004153FF255100000090
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x100000FB8
+        size:            72
+        offset:          0xFB8
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000000000001C000000000000001C00000002000000500F00003400000034000000970F00000000000034000000030000000C000100100001000000000000000001
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         312
+    segname:         __DATA
+    vmaddr:          4294971392
+    vmsize:          4096
+    fileoff:         4096
+    filesize:        4096
+    maxprot:         3
+    initprot:        3
+    nsects:          3
+    flags:           0
+    Sections:
+      - sectname:        __nl_symbol_ptr
+        segname:         __DATA
+        addr:            0x100001000
+        size:            8
+        offset:          0x1000
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x6
+        reserved1:       0x1
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000000000000'
+      - sectname:        __got
+        segname:         __DATA
+        addr:            0x100001008
+        size:            8
+        offset:          0x1008
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x6
+        reserved1:       0x2
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000000000000'
+      - sectname:        __la_symbol_ptr
+        segname:         __DATA
+        addr:            0x100001010
+        size:            8
+        offset:          0x1010
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x7
+        reserved1:       0x3
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         9C0F000001000000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4294975488
+    vmsize:          4096
+    fileoff:         8192
+    filesize:        232
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      8192
+    rebase_size:     8
+    bind_off:        8200
+    bind_size:       24
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   8224
+    lazy_bind_size:  16
+    export_off:      8240
+    export_size:     48
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          8296
+    nsyms:           4
+    stroff:          8376
+    strsize:         48
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       2
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  8360
+    nindirectsyms:   4
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    Content:         '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            4A03D302-35D2-3E3B-93AC-22375185ACFB
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           658944
+    sdk:             658944
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         34734080
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        3920
+    stacksize:       0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 82115073
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         8288
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         8296
+    datasize:        0
+LinkEditData:
+  RebaseOpcodes:
+    - Opcode:          REBASE_OPCODE_SET_TYPE_IMM
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             2
+      ExtraData:       [ 0x10 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DONE
+      Imm:             0
+  BindOpcodes:
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          dyld_stub_binder
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             2
+      ULEBExtraData:   [ 0x8 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+  LazyBindOpcodes:
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             2
+      ULEBExtraData:   [ 0x10 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_SPECIAL_IMM
+      Imm:             14
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          _f
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      33
+            Name:            _mh_execute_header
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      37
+            Name:            main
+            Flags:           0x0
+            Address:         0xF50
+            Other:           0x0
+            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294971216
+    - n_strx:          28
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          65024
+      n_value:         0
+    - n_strx:          31
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          256
+      n_value:         0
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - _f
+    - dyld_stub_binder
+  IndirectSymbols: [ 0x2, 0x40000000, 0x3, 0x2 ]
+  FunctionStarts:  [ 0xF50 ]
+...
diff --git a/bolt/test/X86/MachO/emit_new_binary_with_external_symbol.test b/bolt/test/X86/MachO/emit_new_binary_with_external_symbol.test
new file mode 100644
index 0000000000000..8d9c545b3cfa3
--- /dev/null
+++ b/bolt/test/X86/MachO/emit_new_binary_with_external_symbol.test
@@ -0,0 +1,55 @@
+# RUN: yaml2obj -o %t.exe %p/Inputs/external_symbol.yaml
+# RUN: llvm-bolt -o %t.original %t.exe
+# RUN: llvm-objdump --macho --disassemble-all --no-print-imm-hex %t.original | FileCheck --check-prefix=ORIGINAL %s
+
+# ORIGINAL:       _main:
+# ORIGINAL-NEXT:  100000f50:	55	pushq	%rbp
+# ORIGINAL-NEXT:  100000f51:	48 89 e5	movq	%rsp, %rbp
+# ORIGINAL-NEXT:  100000f54:	48 83 ec 20	subq	$32, %rsp
+# ORIGINAL-NEXT:  100000f58:	c7 45 fc 00 00 00 00	movl	$0, -4(%rbp)
+# ORIGINAL-NEXT:  100000f5f:	89 7d f8	movl	%edi, -8(%rbp)
+# ORIGINAL-NEXT:  100000f62:	48 89 75 f0	movq	%rsi, -16(%rbp)
+# ORIGINAL-NEXT:  100000f66:	83 7d f8 01	cmpl	$1, -8(%rbp)
+# ORIGINAL-NEXT:  100000f6a:	7e 0a	jle	0x100000f76
+# ORIGINAL-NEXT:  100000f6c:	e8 25 00 00 00	callq	0x100000f96
+# ORIGINAL-NEXT:  100000f71:	89 45 fc	movl	%eax, -4(%rbp)
+# ORIGINAL-NEXT:  100000f74:	eb 10	jmp	0x100000f86
+# ORIGINAL-NEXT:  100000f76:	c7 45 ec 01 00 00 00	movl	$1, -20(%rbp)
+# ORIGINAL-NEXT:  100000f7d:	8b 45 ec	movl	-20(%rbp), %eax
+# ORIGINAL-NEXT:  100000f80:	83 c0 02	addl	$2, %eax
+# ORIGINAL-NEXT:  100000f83:	89 45 fc	movl	%eax, -4(%rbp)
+# ORIGINAL-NEXT:  100000f86:	8b 45 fc	movl	-4(%rbp), %eax
+# ORIGINAL-NEXT:  100000f89:	48 83 c4 20	addq	$32, %rsp
+# ORIGINAL-NEXT:  100000f8d:	5d	popq	%rbp
+# ORIGINAL-NEXT:  100000f8e:	c3	retq
+# ORIGINAL-NEXT:  100000f8f:	fc	cld
+# ORIGINAL-NEXT:  100000f90:	48 83 c4 20	addq	$32, %rsp
+# ORIGINAL-NEXT:  100000f94:	5d	popq	%rbp
+# ORIGINAL-NEXT:  100000f95:	c3	retq
+
+# RUN: llvm-bolt -o %t.reversed -reorder-blocks=reverse %t.exe
+# RUN: llvm-objdump --macho --disassemble-all --no-print-imm-hex %t.reversed | FileCheck --check-prefix=REVERSED %s
+
+# REVERSED:       _main:
+# REVERSED-NEXT: 100000f50:	55	pushq	%rbp
+# REVERSED-NEXT: 100000f51:	48 89 e5	movq	%rsp, %rbp
+# REVERSED-NEXT: 100000f54:	48 83 ec 20	subq	$32, %rsp
+# REVERSED-NEXT: 100000f58:	c7 45 fc 00 00 00 00	movl	$0, -4(%rbp)
+# REVERSED-NEXT: 100000f5f:	89 7d f8	movl	%edi, -8(%rbp)
+# REVERSED-NEXT: 100000f62:	48 89 75 f0	movq	%rsi, -16(%rbp)
+# REVERSED-NEXT: 100000f66:	83 7d f8 01	cmpl	$1, -8(%rbp)
+# REVERSED-NEXT: 100000f6a:	7e 0b	jle	0x100000f77
+# REVERSED-NEXT: 100000f6c:	eb 1b	jmp	0x100000f89
+# REVERSED-NEXT: 100000f6e:	8b 45 fc	movl	-4(%rbp), %eax
+# REVERSED-NEXT: 100000f71:	48 83 c4 20	addq	$32, %rsp
+# REVERSED-NEXT: 100000f75:	5d	popq	%rbp
+# REVERSED-NEXT: 100000f76:	c3	retq
+# REVERSED-NEXT: 100000f77:	c7 45 ec 01 00 00 00	movl	$1, -20(%rbp)
+# REVERSED-NEXT: 100000f7e:	8b 45 ec	movl	-20(%rbp), %eax
+# REVERSED-NEXT: 100000f81:	83 c0 02	addl	$2, %eax
+# REVERSED-NEXT: 100000f84:	89 45 fc	movl	%eax, -4(%rbp)
+# REVERSED-NEXT: 100000f87:	eb e5	jmp	0x100000f6e
+# REVERSED-NEXT: 100000f89:	e8 08 00 00 00	callq	0x100000f96
+# REVERSED-NEXT: 100000f8e:	89 45 fc	movl	%eax, -4(%rbp)
+# REVERSED-NEXT: 100000f91:	eb db	jmp	0x100000f6e
+# REVERSED-NEXT: 100000f93:	20 5d c3	andb	%bl, -61(%rbp)
diff --git a/bolt/test/X86/dwarf5-one-loclists-two-bases.test b/bolt/test/X86/dwarf5-one-loclists-two-bases.test
new file mode 100644
index 0000000000000..d85eca18b08ac
--- /dev/null
+++ b/bolt/test/X86/dwarf5-one-loclists-two-bases.test
@@ -0,0 +1,57 @@
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5_main.s -o %tmain.o
+# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-loc-base-no-loc-accesshelper.s -o %thelper.o
+# RUN: %clang %cflags -dwarf-5 %tmain.o %thelper.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections
+# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.exe | FileCheck --check-prefix=PRECHECK %s
+# RUN: llvm-dwarfdump --show-form --verbose --debug-addr %t.bolt > %t.txt
+# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt >> %t.txt
+# RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s
+
+# This tests checks that re-writing of .debug_loclists is handled correctly when one of the CUs
+# doesn't have any DW_AT_location accesses.
+
+# PRECHECK: version = 0x0005
+# PRECHECK: DW_AT_loclists_base [DW_FORM_sec_offset] (0x0000000c)
+# PRECHECK-EMPTY:
+# PRECHECK: DW_TAG_variable
+# PRECHECK: DW_AT_location [DW_FORM_loclistx]
+# PRECHECK-SAME: indexed (0x0)
+# PRECHECK-SAME: loclist = 0x00000014
+# PRECHECK: DW_AT_location [DW_FORM_loclistx]
+# PRECHECK-SAME: indexed (0x1)
+# PRECHECK-SAME: loclist = 0x00000028
+# PRECHECK: DW_AT_loclists_base [DW_FORM_sec_offset]  (0x00000043)
+# PRECHECK-NOT: DW_AT_location
+
+# POSTCHECK: Addrs: [
+# POSTCHECK-NEXT: 0x
+# POSTCHECK-NEXT: 0x
+# POSTCHECK-NEXT: 0x[[#%.16x,ADDR:]]
+# POSTCHECK-NEXT: 0x[[#%.16x,ADDR2:]]
+
+# POSTCHECK: version = 0x0005
+# POSTCHECK: DW_AT_loclists_base [DW_FORM_sec_offset]	(0x0000000c)
+# POSTCHECK: DW_AT_rnglists_base [DW_FORM_sec_offset]	(0x0000000c)
+# POSTCHECK-EMPTY
+# POSTCHECK: DW_TAG_variable
+# POSTCHECK: DW_AT_location [DW_FORM_loclistx]
+# POSTCHECK-SAME: indexed (0x0)
+# POSTCHECK-SAME: loclist = 0x00000014
+# POSTCHECK-NEXT: [0x[[#ADDR]]
+# POSTCHECK-SAME: 0x[[#ADDR + 0x6]]
+# POSTCHECK-NEXT: [0x[[#ADDR + 0x6]]
+# POSTCHECK-SAME: 0x[[#ADDR + 0xc]]
+# POSTCHECK: DW_TAG_variable
+# POSTCHECK: DW_AT_location [DW_FORM_loclistx]
+# POSTCHECK-SAME: indexed (0x1)
+# POSTCHECK-SAME: loclist = 0x0000002a
+# POSTCHECK-NEXT: [0x[[#ADDR2]]
+# POSTCHECK-SAME: 0x[[#ADDR2 + 0x2]]
+
+# Checking second CU
+# POSTCHECK: version = 0x0005
+# POSTCHECK: DW_AT_loclists_base [DW_FORM_sec_offset]	(0x0000000c)
+# POSTCHECK: DW_AT_rnglists_base [DW_FORM_sec_offset]	(0x00000035)
+# POSTCHECK-NOT: DW_AT_location
diff --git a/bolt/test/lit.cfg.py b/bolt/test/lit.cfg.py
index b5e15e20b1fe0..5838699157db7 100644
--- a/bolt/test/lit.cfg.py
+++ b/bolt/test/lit.cfg.py
@@ -110,4 +110,4 @@ def calculate_arch_features(arch_string):
         ('--targets-built', calculate_arch_features)
      ])
 
-config.targets = frozenset(config.targets_to_build.split())
+config.targets = frozenset(config.targets_to_build.split(';'))
diff --git a/bolt/utils/llvm-bolt-wrapper.py b/bolt/utils/llvm-bolt-wrapper.py
index 652cc6074462e..a6d863ce7caa6 100755
--- a/bolt/utils/llvm-bolt-wrapper.py
+++ b/bolt/utils/llvm-bolt-wrapper.py
@@ -206,7 +206,8 @@ def parse_cmp_offset(cmp_out):
     Extracts byte number from cmp output:
     file1 file2 differ: byte X, line Y
     '''
-    return int(re.search(r'byte (\d+),', cmp_out).groups()[0])
+    # NOTE: cmp counts bytes starting from 1!
+    return int(re.search(r'byte (\d+),', cmp_out).groups()[0]) - 1
 
 def report_real_time(binary, main_err, cmp_err, cfg):
     '''
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.h b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
index d0df474bc38e0..b3b714353642b 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
@@ -108,15 +108,15 @@ struct ClangTidyOptions {
 
   /// Helper structure for storing option value with priority of the value.
   struct ClangTidyValue {
-    ClangTidyValue() : Value(), Priority(0) {}
-    ClangTidyValue(const char *Value) : Value(Value), Priority(0) {}
+    ClangTidyValue() = default;
+    ClangTidyValue(const char *Value) : Value(Value) {}
     ClangTidyValue(llvm::StringRef Value, unsigned Priority = 0)
         : Value(Value), Priority(Priority) {}
 
     std::string Value;
     /// Priority stores relative precedence of the value loaded from config
     /// files to disambiguate local vs global value from different levels.
-    unsigned Priority;
+    unsigned Priority = 0;
   };
   typedef std::pair<std::string, std::string> StringPair;
   typedef llvm::StringMap<ClangTidyValue> OptionMap;
diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
index 60666287cf307..5e9c7d0add4f8 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
@@ -36,6 +36,7 @@
 #include "MisplacedPointerArithmeticInAllocCheck.h"
 #include "MisplacedWideningCastCheck.h"
 #include "MoveForwardingReferenceCheck.h"
+#include "MultipleNewInOneExpressionCheck.h"
 #include "MultipleStatementMacroCheck.h"
 #include "NoEscapeCheck.h"
 #include "NonZeroEnumToBoolConversionCheck.h"
@@ -133,6 +134,8 @@ class BugproneModule : public ClangTidyModule {
         "bugprone-misplaced-widening-cast");
     CheckFactories.registerCheck<MoveForwardingReferenceCheck>(
         "bugprone-move-forwarding-reference");
+    CheckFactories.registerCheck<MultipleNewInOneExpressionCheck>(
+        "bugprone-multiple-new-in-one-expression");
     CheckFactories.registerCheck<MultipleStatementMacroCheck>(
         "bugprone-multiple-statement-macro");
     CheckFactories.registerCheck<RedundantBranchConditionCheck>(
diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
index 5af5a59e5340f..e70d1b426a1c6 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
@@ -31,6 +31,7 @@ add_clang_library(clangTidyBugproneModule
   MisplacedPointerArithmeticInAllocCheck.cpp
   MisplacedWideningCastCheck.cpp
   MoveForwardingReferenceCheck.cpp
+  MultipleNewInOneExpressionCheck.cpp
   MultipleStatementMacroCheck.cpp
   NoEscapeCheck.cpp
   NonZeroEnumToBoolConversionCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp
index 276c46ca9fe90..635cc2ca05d12 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp
@@ -52,7 +52,8 @@ void ExceptionEscapeCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 
 void ExceptionEscapeCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
-      functionDecl(anyOf(isNoThrow(), cxxDestructorDecl(),
+      functionDecl(isDefinition(),
+                   anyOf(isNoThrow(), cxxDestructorDecl(),
                          cxxConstructorDecl(isMoveConstructor()),
                          cxxMethodDecl(isMoveAssignmentOperator()),
                          hasName("main"), hasName("swap"),
diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
new file mode 100644
index 0000000000000..47b9667859903
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
@@ -0,0 +1,162 @@
+//===--- MultipleNewInOneExpressionCheck.cpp - clang-tidy------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MultipleNewInOneExpressionCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Lex/Lexer.h"
+
+using namespace clang::ast_matchers;
+using namespace clang;
+
+namespace {
+
+// Determine if the result of an expression is "stored" in some way.
+// It is true if the value is stored into a variable or used as initialization
+// or passed to a function or constructor.
+// For this use case compound assignments are not counted as a "store" (the 'E'
+// expression should have pointer type).
+bool isExprValueStored(const Expr *E, ASTContext &C) {
+  E = E->IgnoreParenCasts();
+  // Get first non-paren, non-cast parent.
+  ParentMapContext &PMap = C.getParentMapContext();
+  DynTypedNodeList P = PMap.getParents(*E);
+  if (P.size() != 1)
+    return false;
+  const Expr *ParentE;
+  while ((ParentE = P[0].get<Expr>()) && ParentE->IgnoreParenCasts() == E) {
+    P = PMap.getParents(P[0]);
+    if (P.size() != 1)
+      return false;
+  }
+
+  if (const auto *ParentVarD = P[0].get<VarDecl>())
+    return ParentVarD->getInit()->IgnoreParenCasts() == E;
+
+  if (!ParentE)
+    return false;
+
+  if (const auto *BinOp = dyn_cast<BinaryOperator>(ParentE))
+    return BinOp->getOpcode() == BO_Assign &&
+           BinOp->getRHS()->IgnoreParenCasts() == E;
+
+  return isa<CallExpr, CXXConstructExpr>(ParentE);
+}
+
+} // namespace
+
+namespace clang {
+namespace tidy {
+namespace bugprone {
+
+AST_MATCHER_P(CXXTryStmt, hasHandlerFor,
+              ast_matchers::internal::Matcher<QualType>, InnerMatcher) {
+  for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) {
+    const CXXCatchStmt *CatchS = Node.getHandler(I);
+    // Check for generic catch handler (match anything).
+    if (CatchS->getCaughtType().isNull())
+      return true;
+    ast_matchers::internal::BoundNodesTreeBuilder Result(*Builder);
+    if (InnerMatcher.matches(CatchS->getCaughtType(), Finder, &Result)) {
+      *Builder = std::move(Result);
+      return true;
+    }
+  }
+  return false;
+}
+
+AST_MATCHER(CXXNewExpr, mayThrow) {
+  FunctionDecl *OperatorNew = Node.getOperatorNew();
+  if (!OperatorNew)
+    return false;
+  return !OperatorNew->getType()->castAs<FunctionProtoType>()->isNothrow();
+}
+
+void MultipleNewInOneExpressionCheck::registerMatchers(MatchFinder *Finder) {
+  auto BadAllocType =
+      recordType(hasDeclaration(cxxRecordDecl(hasName("::std::bad_alloc"))));
+  auto ExceptionType =
+      recordType(hasDeclaration(cxxRecordDecl(hasName("::std::exception"))));
+  auto BadAllocReferenceType = referenceType(pointee(BadAllocType));
+  auto ExceptionReferenceType = referenceType(pointee(ExceptionType));
+
+  auto CatchBadAllocType =
+      qualType(hasCanonicalType(anyOf(BadAllocType, BadAllocReferenceType,
+                                      ExceptionType, ExceptionReferenceType)));
+  auto BadAllocCatchingTryBlock = cxxTryStmt(hasHandlerFor(CatchBadAllocType));
+
+  auto NewExprMayThrow = cxxNewExpr(mayThrow());
+  auto HasNewExpr1 = expr(anyOf(NewExprMayThrow.bind("new1"),
+                                hasDescendant(NewExprMayThrow.bind("new1"))));
+  auto HasNewExpr2 = expr(anyOf(NewExprMayThrow.bind("new2"),
+                                hasDescendant(NewExprMayThrow.bind("new2"))));
+
+  Finder->addMatcher(
+      callExpr(
+          hasAnyArgument(
+              expr(HasNewExpr1).bind("arg1")),
+          hasAnyArgument(
+              expr(HasNewExpr2, unless(equalsBoundNode("arg1"))).bind("arg2")),
+          hasAncestor(BadAllocCatchingTryBlock)),
+      this);
+  Finder->addMatcher(
+      cxxConstructExpr(
+          hasAnyArgument(
+              expr(HasNewExpr1).bind("arg1")),
+          hasAnyArgument(
+              expr(HasNewExpr2, unless(equalsBoundNode("arg1"))).bind("arg2")),
+          unless(isListInitialization()),
+          hasAncestor(BadAllocCatchingTryBlock)),
+      this);
+  Finder->addMatcher(binaryOperator(hasLHS(HasNewExpr1), hasRHS(HasNewExpr2),
+                                    unless(hasAnyOperatorName("&&", "||", ",")),
+                                    hasAncestor(BadAllocCatchingTryBlock)),
+                     this);
+  Finder->addMatcher(
+      cxxNewExpr(mayThrow(),
+                 hasDescendant(NewExprMayThrow.bind("new2_in_new1")),
+                 hasAncestor(BadAllocCatchingTryBlock))
+          .bind("new1"),
+      this);
+}
+
+void MultipleNewInOneExpressionCheck::check(
+    const MatchFinder::MatchResult &Result) {
+  const auto *NewExpr1 = Result.Nodes.getNodeAs<CXXNewExpr>("new1");
+  const auto *NewExpr2 = Result.Nodes.getNodeAs<CXXNewExpr>("new2");
+  const auto *NewExpr2InNewExpr1 =
+      Result.Nodes.getNodeAs<CXXNewExpr>("new2_in_new1");
+  if (!NewExpr2)
+    NewExpr2 = NewExpr2InNewExpr1;
+  assert(NewExpr1 && NewExpr2 && "Bound nodes not found.");
+
+  // No warning if both allocations are not stored.
+  // The value may be intentionally not stored (no deallocations needed or
+  // self-destructing object).
+  if (!isExprValueStored(NewExpr1, *Result.Context) &&
+      !isExprValueStored(NewExpr2, *Result.Context))
+    return;
+
+  // In C++17 sequencing of a 'new' inside constructor arguments of another
+  // 'new' is fixed. Still a leak can happen if the returned value from the
+  // first 'new' is not saved (yet) and the second fails.
+  if (getLangOpts().CPlusPlus17 && NewExpr2InNewExpr1)
+    diag(NewExpr1->getBeginLoc(),
+         "memory allocation may leak if an other allocation is sequenced after "
+         "it and throws an exception")
+        << NewExpr1->getSourceRange() << NewExpr2->getSourceRange();
+  else
+    diag(NewExpr1->getBeginLoc(),
+         "memory allocation may leak if an other allocation is sequenced after "
+         "it and throws an exception; order of these allocations is undefined")
+        << NewExpr1->getSourceRange() << NewExpr2->getSourceRange();
+}
+
+} // namespace bugprone
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.h b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.h
new file mode 100644
index 0000000000000..eba6c84d818ca
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.h
@@ -0,0 +1,35 @@
+//===--- MultipleNewInOneExpressionCheck.h - clang-tidy----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_MULTIPLENEWINONEEXPRESSIONCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_MULTIPLENEWINONEEXPRESSIONCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace bugprone {
+
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/multiple-new-in-one-expression.html
+class MultipleNewInOneExpressionCheck : public ClangTidyCheck {
+public:
+  MultipleNewInOneExpressionCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace bugprone
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_MULTIPLENEWINONEEXPRESSIONCHECK_H
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
index dfcc82c270540..27fa07ed3a385 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
@@ -78,7 +78,7 @@ RedundantStringCStrCheck::RedundantStringCStrCheck(StringRef Name,
           Options.get("StringParameterFunctions", ""))) {
   if (getLangOpts().CPlusPlus20)
     StringParameterFunctions.push_back("::std::format");
-  if (getLangOpts().CPlusPlus2b)
+  if (getLangOpts().CPlusPlus23)
     StringParameterFunctions.push_back("::std::print");
 }
 
diff --git a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
index f9555be57e738..50df451ecfa26 100644
--- a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
@@ -131,6 +131,16 @@ const Stmt *ExprSequence::getSequenceSuccessor(const Stmt *S) const {
           }
         }
       }
+    } else if (const auto *ConstructExpr = dyn_cast<CXXConstructExpr>(Parent)) {
+      // Constructor arguments are sequenced if the constructor call is written
+      // as list-initialization.
+      if (ConstructExpr->isListInitialization()) {
+        for (unsigned I = 1; I < ConstructExpr->getNumArgs(); ++I) {
+          if (ConstructExpr->getArg(I - 1) == S) {
+            return ConstructExpr->getArg(I);
+          }
+        }
+      }
     } else if (const auto *Compound = dyn_cast<CompoundStmt>(Parent)) {
       // Compound statement: Each sub-statement is sequenced after the
       // statements that precede it.
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index 6ead59a7ec90e..26e239fa7bed6 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -45,6 +45,7 @@
 #include <mutex>
 #include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
 namespace clang {
@@ -95,6 +96,36 @@ CodeAction toCodeAction(const ClangdServer::TweakRef &T, const URIForFile &File,
   return CA;
 }
 
+/// Convert from Fix to LSP CodeAction.
+CodeAction toCodeAction(const Fix &F, const URIForFile &File,
+                        const std::optional<int64_t> &Version,
+                        bool SupportsDocumentChanges,
+                        bool SupportChangeAnnotation) {
+  CodeAction Action;
+  Action.title = F.Message;
+  Action.kind = std::string(CodeAction::QUICKFIX_KIND);
+  Action.edit.emplace();
+  if (!SupportsDocumentChanges) {
+    Action.edit->changes.emplace();
+    auto &Changes = (*Action.edit->changes)[File.uri()];
+    for (const auto &E : F.Edits)
+      Changes.push_back({E.range, E.newText, /*annotationId=*/""});
+  } else {
+    Action.edit->documentChanges.emplace();
+    TextDocumentEdit &Edit = Action.edit->documentChanges->emplace_back();
+    Edit.textDocument = VersionedTextDocumentIdentifier{{File}, Version};
+    for (const auto &E : F.Edits)
+      Edit.edits.push_back(
+          {E.range, E.newText,
+           SupportChangeAnnotation ? E.annotationId : ""});
+    if (SupportChangeAnnotation) {
+      for (const auto &[AID, Annotation]: F.Annotations)
+        Action.edit->changeAnnotations[AID] = Annotation;
+    }
+  }
+  return Action;
+}
+
 void adjustSymbolKinds(llvm::MutableArrayRef<DocumentSymbol> Syms,
                        SymbolKindBitset Kinds) {
   for (auto &S : Syms) {
@@ -487,6 +518,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
       Params.capabilities.HierarchicalDocumentSymbol;
   SupportsReferenceContainer = Params.capabilities.ReferenceContainer;
   SupportFileStatus = Params.initializationOptions.FileStatus;
+  SupportsDocumentChanges = Params.capabilities.DocumentChanges;
+  SupportsChangeAnnotation = Params.capabilities.ChangeAnnotation;
   HoverContentFormat = Params.capabilities.HoverContentFormat;
   Opts.LineFoldingOnly = Params.capabilities.LineFoldingOnly;
   SupportsOffsetsInSignatureHelp = Params.capabilities.OffsetsInSignatureHelp;
@@ -761,8 +794,10 @@ void ClangdLSPServer::onCommandApplyTweak(const TweakArgs &Args,
       return Reply(std::move(Err));
 
     WorkspaceEdit WE;
+    // FIXME: use documentChanges when SupportDocumentChanges is true.
+    WE.changes.emplace();
     for (const auto &It : R->ApplyEdits) {
-      WE.changes[URI::createFile(It.first()).toString()] =
+      (*WE.changes)[URI::createFile(It.first()).toString()] =
           It.second.asTextEdits();
     }
     // ApplyEdit will take care of calling Reply().
@@ -833,8 +868,12 @@ void ClangdLSPServer::onRename(const RenameParams &Params,
                    if (auto Err = validateEdits(*Server, R->GlobalChanges))
                      return Reply(std::move(Err));
                    WorkspaceEdit Result;
+                   // FIXME: use documentChanges if SupportDocumentChanges is
+                   // true.
+                   Result.changes.emplace();
                    for (const auto &Rep : R->GlobalChanges) {
-                     Result.changes[URI::createFile(Rep.first()).toString()] =
+                     (*Result
+                           .changes)[URI::createFile(Rep.first()).toString()] =
                          Rep.second.asTextEdits();
                    }
                    Reply(Result);
@@ -984,8 +1023,8 @@ void ClangdLSPServer::onCodeAction(const CodeActionParams &Params,
   std::vector<CodeAction> FixIts;
   if (KindAllowed(CodeAction::QUICKFIX_KIND)) {
     for (const Diagnostic &D : Params.context.diagnostics) {
-      for (auto &F : getFixes(File.file(), D)) {
-        FixIts.push_back(toCodeAction(F, Params.textDocument.uri));
+      for (auto &CA : getFixes(File.file(), D)) {
+        FixIts.push_back(CA);
         FixIts.back().diagnostics = {D};
       }
     }
@@ -1663,8 +1702,8 @@ void ClangdLSPServer::profile(MemoryTree &MT) const {
     Server->profile(MT.child("clangd_server"));
 }
 
-std::vector<Fix> ClangdLSPServer::getFixes(llvm::StringRef File,
-                                           const clangd::Diagnostic &D) {
+std::vector<CodeAction> ClangdLSPServer::getFixes(llvm::StringRef File,
+                                                  const clangd::Diagnostic &D) {
   std::lock_guard<std::mutex> Lock(FixItsMutex);
   auto DiagToFixItsIter = FixItsMap.find(File);
   if (DiagToFixItsIter == FixItsMap.end())
@@ -1710,8 +1749,19 @@ void ClangdLSPServer::onDiagnosticsReady(PathRef File, llvm::StringRef Version,
   for (auto &Diag : Diagnostics) {
     toLSPDiags(Diag, Notification.uri, DiagOpts,
                [&](clangd::Diagnostic Diag, llvm::ArrayRef<Fix> Fixes) {
-                 auto &FixItsForDiagnostic = LocalFixIts[Diag];
-                 llvm::copy(Fixes, std::back_inserter(FixItsForDiagnostic));
+                 std::vector<CodeAction> CodeActions;
+                 for (const auto &Fix : Fixes)
+                   CodeActions.push_back(toCodeAction(Fix, Notification.uri,
+                                                      Notification.version,
+                                                      SupportsDocumentChanges,
+                                                      SupportsChangeAnnotation));
+
+                 if (DiagOpts.EmbedFixesInDiagnostics) {
+                   Diag.codeActions.emplace(CodeActions);
+                   if (Diag.codeActions->size() == 1)
+                     Diag.codeActions->front().isPreferred = true;
+                 }
+                 LocalFixIts[Diag] = std::move(CodeActions);
                  Notification.diagnostics.push_back(std::move(Diag));
                });
   }
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index cd5bb662c3931..33c8d221f9bd4 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -21,6 +21,7 @@
 #include "llvm/Support/JSON.h"
 #include <chrono>
 #include <cstddef>
+#include <cstdint>
 #include <memory>
 #include <optional>
 #include <vector>
@@ -197,7 +198,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
                  Callback<llvm::json::Value> Reply);
 
   void bindMethods(LSPBinder &, const ClientCapabilities &Caps);
-  std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
+  std::vector<CodeAction> getFixes(StringRef File, const clangd::Diagnostic &D);
+
 
   /// Checks if completion request should be ignored. We need this due to the
   /// limitation of the LSP. Per LSP, a client sends requests for all "trigger
@@ -231,10 +233,12 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
   std::atomic<bool> IsBeingDestroyed = {false};
 
   std::mutex FixItsMutex;
-  typedef std::map<clangd::Diagnostic, std::vector<Fix>, LSPDiagnosticCompare>
+  typedef std::map<clangd::Diagnostic, std::vector<CodeAction>,
+                   LSPDiagnosticCompare>
       DiagnosticToReplacementMap;
   /// Caches FixIts per file and diagnostics
-  llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
+  llvm::StringMap<DiagnosticToReplacementMap>
+      FixItsMap;
   // Last semantic-tokens response, for incremental requests.
   std::mutex SemanticTokensMutex;
   llvm::StringMap<SemanticTokens> LastSemanticTokens;
@@ -271,6 +275,11 @@ class ClangdLSPServer : private ClangdServer::Callbacks,
   MarkupKind HoverContentFormat = MarkupKind::PlainText;
   /// Whether the client supports offsets for parameter info labels.
   bool SupportsOffsetsInSignatureHelp = false;
+  /// Whether the client supports the versioned document changes.
+  bool SupportsDocumentChanges = false;
+  /// Whether the client supports change annotations on text edits.
+  bool SupportsChangeAnnotation = false;
+
   std::mutex BackgroundIndexProgressMutex;
   enum class BackgroundIndexProgress {
     // Client doesn't support reporting progress. No transitions possible.
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 260c44b2064b5..bb01fa421b498 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -2242,7 +2242,7 @@ CompletionItem CodeCompletion::render(const CodeCompleteOptions &Opts) const {
   }
   LSP.sortText = sortText(Score.Total, FilterText);
   LSP.filterText = FilterText;
-  LSP.textEdit = {CompletionTokenRange, RequiredQualifier + Name};
+  LSP.textEdit = {CompletionTokenRange, RequiredQualifier + Name, ""};
   // Merge continuous additionalTextEdits into main edit. The main motivation
   // behind this is to help LSP clients, it seems most of them are confused when
   // they are provided with additionalTextEdits that are consecutive to main
diff --git a/clang-tools-extra/clangd/Compiler.cpp b/clang-tools-extra/clangd/Compiler.cpp
index f242db44b4317..7b93353cab1f3 100644
--- a/clang-tools-extra/clangd/Compiler.cpp
+++ b/clang-tools-extra/clangd/Compiler.cpp
@@ -73,7 +73,7 @@ void disableUnsupportedOptions(CompilerInvocation &CI) {
   // Always default to raw container format as clangd doesn't registry any other
   // and clang dies when faced with unknown formats.
   CI.getHeaderSearchOpts().ModuleFormat =
-      PCHContainerOperations().getRawReader().getFormat().str();
+      PCHContainerOperations().getRawReader().getFormats().front().str();
 
   CI.getFrontendOpts().Plugins.clear();
   CI.getFrontendOpts().AddPluginActions.clear();
diff --git a/clang-tools-extra/clangd/Diagnostics.cpp b/clang-tools-extra/clangd/Diagnostics.cpp
index 963bf02e21848..d1726142c5ded 100644
--- a/clang-tools-extra/clangd/Diagnostics.cpp
+++ b/clang-tools-extra/clangd/Diagnostics.cpp
@@ -423,15 +423,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Diag &D) {
   return OS;
 }
 
-CodeAction toCodeAction(const Fix &F, const URIForFile &File) {
-  CodeAction Action;
-  Action.title = F.Message;
-  Action.kind = std::string(CodeAction::QUICKFIX_KIND);
-  Action.edit.emplace();
-  Action.edit->changes[File.uri()] = {F.Edits.begin(), F.Edits.end()};
-  return Action;
-}
-
 Diag toDiag(const llvm::SMDiagnostic &D, Diag::DiagSource Source) {
   Diag Result;
   Result.Message = D.getMessage().str();
@@ -499,13 +490,6 @@ void toLSPDiags(
   case Diag::Unknown:
     break;
   }
-  if (Opts.EmbedFixesInDiagnostics) {
-    Main.codeActions.emplace();
-    for (const auto &Fix : D.Fixes)
-      Main.codeActions->push_back(toCodeAction(Fix, File));
-    if (Main.codeActions->size() == 1)
-      Main.codeActions->front().isPreferred = true;
-  }
   if (Opts.SendDiagnosticCategory && !D.Category.empty())
     Main.category = D.Category;
 
@@ -786,7 +770,7 @@ void StoreDiags::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
     if (Message.empty()) // either !SyntheticMessage, or we failed to make one.
       Info.FormatDiagnostic(Message);
     LastDiag->Fixes.push_back(
-        Fix{std::string(Message.str()), std::move(Edits)});
+        Fix{std::string(Message.str()), std::move(Edits), {}});
     return true;
   };
 
diff --git a/clang-tools-extra/clangd/Diagnostics.h b/clang-tools-extra/clangd/Diagnostics.h
index b20b82fcb3298..d4c0478c63a5c 100644
--- a/clang-tools-extra/clangd/Diagnostics.h
+++ b/clang-tools-extra/clangd/Diagnostics.h
@@ -83,6 +83,10 @@ struct Fix {
   std::string Message;
   /// TextEdits from clang's fix-its. Must be non-empty.
   llvm::SmallVector<TextEdit, 1> Edits;
+
+  /// Annotations for the Edits.
+  llvm::SmallVector<std::pair<ChangeAnnotationIdentifier, ChangeAnnotation>>
+      Annotations;
 };
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Fix &F);
 
@@ -120,9 +124,6 @@ void toLSPDiags(
     const Diag &D, const URIForFile &File, const ClangdDiagnosticOptions &Opts,
     llvm::function_ref<void(clangd::Diagnostic, llvm::ArrayRef<Fix>)> OutFn);
 
-/// Convert from Fix to LSP CodeAction.
-CodeAction toCodeAction(const Fix &D, const URIForFile &File);
-
 /// Convert from clang diagnostic level to LSP severity.
 int getSeverity(DiagnosticsEngine::Level L);
 
diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp
index 3eb0900715744..558769f209860 100644
--- a/clang-tools-extra/clangd/Hover.cpp
+++ b/clang-tools-extra/clangd/Hover.cpp
@@ -981,12 +981,23 @@ void maybeAddCalleeArgInfo(const SelectionTree::Node *N, HoverInfo &HI,
   const auto &OuterNode = N->outerImplicit();
   if (!OuterNode.Parent)
     return;
-  const auto *CE = OuterNode.Parent->ASTNode.get<CallExpr>();
-  if (!CE)
+
+  const FunctionDecl *FD = nullptr;
+  llvm::ArrayRef<const Expr *> Args;
+
+  if (const auto *CE = OuterNode.Parent->ASTNode.get<CallExpr>()) {
+    FD = CE->getDirectCallee();
+    Args = {CE->getArgs(), CE->getNumArgs()};
+  } else if (const auto *CE =
+                 OuterNode.Parent->ASTNode.get<CXXConstructExpr>()) {
+    FD = CE->getConstructor();
+    Args = {CE->getArgs(), CE->getNumArgs()};
+  }
+  if (!FD)
     return;
-  const FunctionDecl *FD = CE->getDirectCallee();
-  // For non-function-call-like operatators (e.g. operator+, operator<<) it's
-  // not immediattely obvious what the "passed as" would refer to and, given
+
+  // For non-function-call-like operators (e.g. operator+, operator<<) it's
+  // not immediately obvious what the "passed as" would refer to and, given
   // fixed function signature, the value would be very low anyway, so we choose
   // to not support that.
   // Both variadic functions and operator() (especially relevant for lambdas)
@@ -996,13 +1007,15 @@ void maybeAddCalleeArgInfo(const SelectionTree::Node *N, HoverInfo &HI,
 
   HoverInfo::PassType PassType;
 
+  auto Parameters = resolveForwardingParameters(FD);
+
   // Find argument index for N.
-  for (unsigned I = 0; I < CE->getNumArgs() && I < FD->getNumParams(); ++I) {
-    if (CE->getArg(I) != OuterNode.ASTNode.get<Expr>())
+  for (unsigned I = 0; I < Args.size() && I < Parameters.size(); ++I) {
+    if (Args[I] != OuterNode.ASTNode.get<Expr>())
       continue;
 
     // Extract matching argument from function declaration.
-    if (const ParmVarDecl *PVD = FD->getParamDecl(I)) {
+    if (const ParmVarDecl *PVD = Parameters[I]) {
       HI.CalleeArgInfo.emplace(toHoverInfoParam(PVD, PP));
       if (N == &OuterNode)
         PassType.PassBy = getPassMode(PVD->getType());
diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp
index d15dd70efdb70..3b913d851abe2 100644
--- a/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -40,10 +40,12 @@
 #include "clang/Tooling/Syntax/Tokens.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/GenericUniformityImpl.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Casting.h"
@@ -415,11 +417,138 @@ IncludeCleanerFindings computeIncludeCleanerFindings(ParsedAST &AST) {
             Ref.Target, TouchingTokens.back().range(SM), Providers};
         MissingIncludes.push_back(std::move(DiagInfo));
       });
+  // Put possibly equal diagnostics together for deduplication.
+  // The duplicates might be from macro arguments that get expanded multiple
+  // times.
+  llvm::stable_sort(MissingIncludes, [](const MissingIncludeDiagInfo &LHS,
+                                        const MissingIncludeDiagInfo &RHS) {
+    // First sort by reference location.
+    if (LHS.SymRefRange != RHS.SymRefRange) {
+      // We can get away just by comparing the offsets as all the ranges are in
+      // main file.
+      return LHS.SymRefRange.beginOffset() < RHS.SymRefRange.beginOffset();
+    }
+    // For the same location, break ties using the symbol. Note that this won't
+    // be stable across runs.
+    using MapInfo = llvm::DenseMapInfo<include_cleaner::Symbol>;
+    return MapInfo::getHashValue(LHS.Symbol) <
+           MapInfo::getHashValue(RHS.Symbol);
+  });
+  MissingIncludes.erase(llvm::unique(MissingIncludes), MissingIncludes.end());
   std::vector<const Inclusion *> UnusedIncludes =
       getUnused(AST, Used, /*ReferencedPublicHeaders*/ {});
   return {std::move(UnusedIncludes), std::move(MissingIncludes)};
 }
 
+Fix removeAllUnusedIncludes(llvm::ArrayRef<Diag> UnusedIncludes) {
+  assert(!UnusedIncludes.empty());
+
+  Fix RemoveAll;
+  RemoveAll.Message = "remove all unused includes";
+  for (const auto &Diag : UnusedIncludes) {
+    assert(Diag.Fixes.size() == 1 && "Expected exactly one fix.");
+    RemoveAll.Edits.insert(RemoveAll.Edits.end(),
+         Diag.Fixes.front().Edits.begin(),
+         Diag.Fixes.front().Edits.end());
+  }
+
+  // TODO(hokein): emit a suitable text for the label.
+  ChangeAnnotation Annotation = {/*label=*/"",
+                                 /*needsConfirmation=*/true,
+                                 /*description=*/""};
+  static const ChangeAnnotationIdentifier RemoveAllUnusedID =
+      "RemoveAllUnusedIncludes";
+  for (unsigned I = 0; I < RemoveAll.Edits.size(); ++I) {
+    ChangeAnnotationIdentifier ID = RemoveAllUnusedID + std::to_string(I);
+    RemoveAll.Edits[I].annotationId = ID;
+    RemoveAll.Annotations.push_back({ID, Annotation});
+  }
+  return RemoveAll;
+}
+Fix addAllMissingIncludes(llvm::ArrayRef<Diag> MissingIncludeDiags) {
+  assert(!MissingIncludeDiags.empty());
+
+  Fix AddAllMissing;
+  AddAllMissing.Message = "add all missing includes";
+  // A map to deduplicate the edits with the same new text.
+  // newText (#include "my_missing_header.h") -> TextEdit.
+  llvm::StringMap<TextEdit> Edits;
+  for (const auto &Diag : MissingIncludeDiags) {
+    assert(Diag.Fixes.size() == 1 && "Expected exactly one fix.");
+    for (const auto& Edit : Diag.Fixes.front().Edits) {
+      Edits.try_emplace(Edit.newText, Edit);
+    }
+  }
+  // FIXME(hokein): emit used symbol reference in the annotation.
+  ChangeAnnotation Annotation = {/*label=*/"",
+                                 /*needsConfirmation=*/true,
+                                 /*description=*/""};
+  static const ChangeAnnotationIdentifier AddAllMissingID =
+      "AddAllMissingIncludes";
+  unsigned I = 0;
+  for (auto &It : Edits) {
+    ChangeAnnotationIdentifier ID = AddAllMissingID + std::to_string(I++);
+    AddAllMissing.Edits.push_back(std::move(It.getValue()));
+    AddAllMissing.Edits.back().annotationId = ID;
+
+    AddAllMissing.Annotations.push_back({ID, Annotation});
+  }
+  return AddAllMissing;
+}
+Fix fixAll(const Fix& RemoveAllUnused, const Fix& AddAllMissing) {
+  Fix FixAll;
+  FixAll.Message = "fix all includes";
+
+  for (const auto &F : RemoveAllUnused.Edits)
+    FixAll.Edits.push_back(F);
+  for (const auto &F : AddAllMissing.Edits)
+    FixAll.Edits.push_back(F);
+
+  for (const auto& A : RemoveAllUnused.Annotations)
+    FixAll.Annotations.push_back(A);
+  for (const auto& A : AddAllMissing.Annotations)
+    FixAll.Annotations.push_back(A);
+  return FixAll;
+}
+
+std::vector<Diag> generateIncludeCleanerDiagnostic(
+    ParsedAST &AST, const IncludeCleanerFindings &Findings,
+    llvm::StringRef Code) {
+  std::vector<Diag> UnusedIncludes = generateUnusedIncludeDiagnostics(
+      AST.tuPath(), Findings.UnusedIncludes, Code);
+  std::optional<Fix> RemoveAllUnused;;
+  if (UnusedIncludes.size() > 1)
+    RemoveAllUnused = removeAllUnusedIncludes(UnusedIncludes);
+
+  std::vector<Diag> MissingIncludeDiags = generateMissingIncludeDiagnostics(
+      AST, Findings.MissingIncludes, Code);
+  std::optional<Fix> AddAllMissing;
+  if (MissingIncludeDiags.size() > 1)
+    AddAllMissing = addAllMissingIncludes(MissingIncludeDiags);
+
+  std::optional<Fix> FixAll;
+  if (RemoveAllUnused && AddAllMissing)
+    FixAll = fixAll(*RemoveAllUnused, *AddAllMissing);
+
+  auto AddBatchFix = [](const std::optional<Fix> &F, clang::clangd::Diag *Out) {
+    if (!F) return;
+    Out->Fixes.push_back(*F);
+  };
+  for (auto &Diag : MissingIncludeDiags) {
+    AddBatchFix(AddAllMissing, &Diag);
+    AddBatchFix(FixAll, &Diag);
+  }
+  for (auto &Diag : UnusedIncludes) {
+    AddBatchFix(RemoveAllUnused, &Diag);
+    AddBatchFix(FixAll, &Diag);
+  }
+
+  auto Result = std::move(MissingIncludeDiags);
+  llvm::move(UnusedIncludes,
+             std::back_inserter(Result));
+  return Result;
+}
+
 std::vector<Diag> issueIncludeCleanerDiagnostics(ParsedAST &AST,
                                                  llvm::StringRef Code) {
   // Interaction is only polished for C/CPP.
@@ -435,13 +564,7 @@ std::vector<Diag> issueIncludeCleanerDiagnostics(ParsedAST &AST,
     // will need include-cleaner results, call it once
     Findings = computeIncludeCleanerFindings(AST);
   }
-
-  std::vector<Diag> Result = generateUnusedIncludeDiagnostics(
-      AST.tuPath(), Findings.UnusedIncludes, Code);
-  llvm::move(
-      generateMissingIncludeDiagnostics(AST, Findings.MissingIncludes, Code),
-      std::back_inserter(Result));
-  return Result;
+  return generateIncludeCleanerDiagnostic(AST, Findings, Code);
 }
 
 std::optional<include_cleaner::Header>
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index 4b2472ad4be04..f613624cd25d8 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -187,14 +187,45 @@ bool fromJSON(const llvm::json::Value &Params, TextDocumentItem &R,
 bool fromJSON(const llvm::json::Value &Params, TextEdit &R,
               llvm::json::Path P) {
   llvm::json::ObjectMapper O(Params, P);
-  return O && O.map("range", R.range) && O.map("newText", R.newText);
+  return O && O.map("range", R.range) && O.map("newText", R.newText) &&
+         O.mapOptional("annotationId", R.annotationId);
 }
 
 llvm::json::Value toJSON(const TextEdit &P) {
-  return llvm::json::Object{
+  llvm::json::Object Result{
       {"range", P.range},
       {"newText", P.newText},
   };
+  if (!P.annotationId.empty())
+    Result["annotationId"] = P.annotationId;
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, ChangeAnnotation &R,
+              llvm::json::Path P) {
+  llvm::json::ObjectMapper O(Params, P);
+  return O && O.map("label", R.label) &&
+         O.map("needsConfirmation", R.needsConfirmation) &&
+         O.mapOptional("description", R.description);
+}
+llvm::json::Value toJSON(const ChangeAnnotation & CA) {
+  llvm::json::Object Result{{"label", CA.label}};
+  if (CA.needsConfirmation)
+    Result["needsConfirmation"] = *CA.needsConfirmation;
+  if (!CA.description.empty())
+    Result["description"] = CA.description;
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, TextDocumentEdit &R,
+              llvm::json::Path P) {
+  llvm::json::ObjectMapper O(Params, P);
+  return O && O.map("textDocument", R.textDocument) && O.map("edits", R.edits);
+}
+llvm::json::Value toJSON(const TextDocumentEdit &P) {
+  llvm::json::Object Result{{"textDocument", P.textDocument},
+                            {"edits", P.edits}};
+  return Result;
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TextEdit &TE) {
@@ -444,6 +475,13 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R,
       if (auto RefreshSupport = SemanticTokens->getBoolean("refreshSupport"))
         R.SemanticTokenRefreshSupport = *RefreshSupport;
     }
+    if (auto *WorkspaceEdit = Workspace->getObject("workspaceEdit")) {
+      if (auto DocumentChanges = WorkspaceEdit->getBoolean("documentChanges"))
+        R.DocumentChanges = *DocumentChanges;
+      if (WorkspaceEdit->getObject("changeAnnotationSupport")) {
+        R.ChangeAnnotation = true;
+      }
+    }
   }
   if (auto *Window = O->getObject("window")) {
     if (auto WorkDoneProgress = Window->getBoolean("workDoneProgress"))
@@ -717,7 +755,9 @@ bool fromJSON(const llvm::json::Value &Params, CodeActionParams &R,
 bool fromJSON(const llvm::json::Value &Params, WorkspaceEdit &R,
               llvm::json::Path P) {
   llvm::json::ObjectMapper O(Params, P);
-  return O && O.map("changes", R.changes);
+  return O && O.map("changes", R.changes) &&
+         O.map("documentChanges", R.documentChanges) &&
+         O.mapOptional("changeAnnotations", R.changeAnnotations);
 }
 
 bool fromJSON(const llvm::json::Value &Params, ExecuteCommandParams &R,
@@ -863,10 +903,22 @@ llvm::json::Value toJSON(const DocumentSymbol &S) {
 }
 
 llvm::json::Value toJSON(const WorkspaceEdit &WE) {
-  llvm::json::Object FileChanges;
-  for (auto &Change : WE.changes)
-    FileChanges[Change.first] = llvm::json::Array(Change.second);
-  return llvm::json::Object{{"changes", std::move(FileChanges)}};
+  llvm::json::Object Result;
+  if (WE.changes) {
+    llvm::json::Object FileChanges;
+    for (auto &Change : *WE.changes)
+      FileChanges[Change.first] = llvm::json::Array(Change.second);
+    Result["changes"] = std::move(FileChanges);
+  }
+  if (WE.documentChanges)
+    Result["documentChanges"] = *WE.documentChanges;
+  if (!WE.changeAnnotations.empty()) {
+    llvm::json::Object ChangeAnnotations;
+    for (auto &Annotation : WE.changeAnnotations)
+      ChangeAnnotations[Annotation.first] = Annotation.second;
+    Result["changeAnnotations"] = std::move(ChangeAnnotations);
+  }
+  return Result;
 }
 
 bool fromJSON(const llvm::json::Value &Params, TweakArgs &A,
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index eb271676e651f..1d6fccff6ea4d 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -238,6 +238,8 @@ struct ReferenceLocation : Location {
 llvm::json::Value toJSON(const ReferenceLocation &);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const ReferenceLocation &);
 
+using ChangeAnnotationIdentifier = std::string;
+// A combination of a LSP standard TextEdit and AnnotatedTextEdit.
 struct TextEdit {
   /// The range of the text document to be manipulated. To insert
   /// text into a document create a range where start === end.
@@ -246,14 +248,46 @@ struct TextEdit {
   /// The string to be inserted. For delete operations use an
   /// empty string.
   std::string newText;
+
+  /// The actual annotation identifier (optional)
+  /// If empty, then this field is nullopt.
+  ChangeAnnotationIdentifier annotationId = "";
 };
 inline bool operator==(const TextEdit &L, const TextEdit &R) {
-  return std::tie(L.newText, L.range) == std::tie(R.newText, R.range);
+  return std::tie(L.newText, L.range, L.annotationId) ==
+         std::tie(R.newText, R.range, L.annotationId);
 }
 bool fromJSON(const llvm::json::Value &, TextEdit &, llvm::json::Path);
 llvm::json::Value toJSON(const TextEdit &);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const TextEdit &);
 
+struct ChangeAnnotation {
+  /// A human-readable string describing the actual change. The string
+  /// is rendered prominent in the user interface.
+  std::string label;
+
+  /// A flag which indicates that user confirmation is needed
+  /// before applying the change.
+  std::optional<bool> needsConfirmation;
+
+  /// A human-readable string which is rendered less prominent in
+  /// the user interface.
+  std::string description;
+};
+bool fromJSON(const llvm::json::Value &, ChangeAnnotation &, llvm::json::Path);
+llvm::json::Value toJSON(const ChangeAnnotation &);
+
+struct TextDocumentEdit {
+  /// The text document to change.
+  VersionedTextDocumentIdentifier textDocument;
+
+	/// The edits to be applied.
+  /// FIXME: support the AnnotatedTextEdit variant.
+  std::vector<TextEdit> edits;
+};
+bool fromJSON(const llvm::json::Value &, TextDocumentEdit &, llvm::json::Path);
+llvm::json::Value toJSON(const TextDocumentEdit &);
+
 struct TextDocumentItem {
   /// The text document's URI.
   URIForFile uri;
@@ -517,6 +551,12 @@ struct ClientCapabilities {
   /// server to the client.
   bool SemanticTokenRefreshSupport = false;
 
+  /// The client supports versioned document changes for WorkspaceEdit.
+  bool DocumentChanges = false;
+  
+  /// The client supports change annotations on text edits,
+  bool ChangeAnnotation = false;
+
   /// Whether the client supports the textDocument/inactiveRegions
   /// notification. This is a clangd extension.
   bool InactiveRegions = false;
@@ -970,12 +1010,22 @@ struct CodeActionParams {
 };
 bool fromJSON(const llvm::json::Value &, CodeActionParams &, llvm::json::Path);
 
+/// The edit should either provide changes or documentChanges. If the client
+/// can handle versioned document edits and if documentChanges are present,
+/// the latter are preferred over changes.
 struct WorkspaceEdit {
   /// Holds changes to existing resources.
-  std::map<std::string, std::vector<TextEdit>> changes;
-
-  /// Note: "documentChanges" is not currently used because currently there is
-  /// no support for versioned edits.
+  std::optional<std::map<std::string, std::vector<TextEdit>>> changes;
+  /// Versioned document edits.
+  ///
+  /// If a client neither supports `documentChanges` nor
+	/// `workspace.workspaceEdit.resourceOperations` then only plain `TextEdit`s
+	/// using the `changes` property are supported.
+  std::optional<std::vector<TextDocumentEdit>> documentChanges;
+  
+  /// A map of change annotations that can be referenced in
+	/// AnnotatedTextEdit.
+  std::map<std::string, ChangeAnnotation> changeAnnotations;
 };
 bool fromJSON(const llvm::json::Value &, WorkspaceEdit &, llvm::json::Path);
 llvm::json::Value toJSON(const WorkspaceEdit &WE);
diff --git a/clang-tools-extra/clangd/index/StdLib.cpp b/clang-tools-extra/clangd/index/StdLib.cpp
index 1133d984efbae..381d599e3d2aa 100644
--- a/clang-tools-extra/clangd/index/StdLib.cpp
+++ b/clang-tools-extra/clangd/index/StdLib.cpp
@@ -47,8 +47,8 @@ llvm::StringLiteral mandatoryHeader(Lang L) {
 
 LangStandard::Kind standardFromOpts(const LangOptions &LO) {
   if (LO.CPlusPlus) {
-    if (LO.CPlusPlus2b)
-      return LangStandard::lang_cxx2b;
+    if (LO.CPlusPlus23)
+      return LangStandard::lang_cxx23;
     if (LO.CPlusPlus20)
       return LangStandard::lang_cxx20;
     if (LO.CPlusPlus17)
diff --git a/clang-tools-extra/clangd/test/Inputs/include-cleaner/all1.h b/clang-tools-extra/clangd/test/Inputs/include-cleaner/all1.h
new file mode 100644
index 0000000000000..c1ec51b8f24b9
--- /dev/null
+++ b/clang-tools-extra/clangd/test/Inputs/include-cleaner/all1.h
@@ -0,0 +1,4 @@
+#pragma once
+
+#include "bar.h"
+#include "foo.h"
diff --git a/clang-tools-extra/clangd/test/Inputs/include-cleaner/all2.h b/clang-tools-extra/clangd/test/Inputs/include-cleaner/all2.h
new file mode 100644
index 0000000000000..c1ec51b8f24b9
--- /dev/null
+++ b/clang-tools-extra/clangd/test/Inputs/include-cleaner/all2.h
@@ -0,0 +1,4 @@
+#pragma once
+
+#include "bar.h"
+#include "foo.h"
diff --git a/clang-tools-extra/clangd/test/Inputs/include-cleaner/bar.h b/clang-tools-extra/clangd/test/Inputs/include-cleaner/bar.h
new file mode 100644
index 0000000000000..f70dd9565fd48
--- /dev/null
+++ b/clang-tools-extra/clangd/test/Inputs/include-cleaner/bar.h
@@ -0,0 +1,2 @@
+#pragma once
+class Bar {};
diff --git a/clang-tools-extra/clangd/test/Inputs/include-cleaner/foo.h b/clang-tools-extra/clangd/test/Inputs/include-cleaner/foo.h
new file mode 100644
index 0000000000000..f35930c9fe80d
--- /dev/null
+++ b/clang-tools-extra/clangd/test/Inputs/include-cleaner/foo.h
@@ -0,0 +1,2 @@
+#pragma once
+class Foo {};
diff --git a/clang-tools-extra/clangd/test/fixits-codeaction-documentchanges.test b/clang-tools-extra/clangd/test/fixits-codeaction-documentchanges.test
new file mode 100644
index 0000000000000..7a591319a7405
--- /dev/null
+++ b/clang-tools-extra/clangd/test/fixits-codeaction-documentchanges.test
@@ -0,0 +1,145 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"codeAction":{"codeActionLiteralSupport":{}}},"workspace":{"workspaceEdit":{"documentChanges":true}}},"trace":"off"}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.c","languageId":"c","version":1,"text":"int main(int i, char **a) { if (i = 2) {}}"}}}
+#      CHECK:    "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "code": "-Wparentheses",
+# CHECK-NEXT:        "message": "Using the result of an assignment as a condition without parentheses (fixes available)",
+# CHECK-NEXT:        "range": {
+# CHECK-NEXT:          "end": {
+# CHECK-NEXT:            "character": 37,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "start": {
+# CHECK-NEXT:            "character": 32,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          }
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "severity": 2,
+# CHECK-NEXT:        "source": "clang"
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:    "version": 1
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","id":2,"method":"textDocument/codeAction","params":{"textDocument":{"uri":"test:///foo.c"},"range":{"start":{"line":0,"character":13},"end":{"line":0,"character":35}},"context":{"diagnostics":[{"range":{"start": {"line": 0, "character": 32}, "end": {"line": 0, "character": 37}},"severity":2,"message":"Using the result of an assignment as a condition without parentheses (fixes available)", "code": "-Wparentheses", "source": "clang"}]}}}
+#      CHECK:  "id": 2,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "diagnostics": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "code": "-Wparentheses",
+# CHECK-NEXT:          "message": "Using the result of an assignment as a condition without parentheses (fixes available)",
+# CHECK-NEXT:          "range": {
+# CHECK-NEXT:            "end": {
+# CHECK-NEXT:              "character": 37,
+# CHECK-NEXT:              "line": 0
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "start": {
+# CHECK-NEXT:              "character": 32,
+# CHECK-NEXT:              "line": 0
+# CHECK-NEXT:            }
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "severity": 2,
+# CHECK-NEXT:          "source": "clang"
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "edit": {
+# CHECK-NEXT:        "documentChanges": [
+# CHECK-NEXT:          {
+# CHECK-NEXT:            "edits": [
+# CHECK-NEXT:              {
+# CHECK-NEXT:                "newText": "(",
+# CHECK-NEXT:                "range": {
+# CHECK-NEXT:                  "end": {
+# CHECK-NEXT:                    "character": 32,
+# CHECK-NEXT:                    "line": 0
+# CHECK-NEXT:                  },
+# CHECK-NEXT:                  "start": {
+# CHECK-NEXT:                    "character": 32,
+# CHECK-NEXT:                    "line": 0
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              },
+# CHECK-NEXT:              {
+# CHECK-NEXT:                "newText": ")",
+# CHECK-NEXT:                "range": {
+# CHECK-NEXT:                  "end": {
+# CHECK-NEXT:                    "character": 37,
+# CHECK-NEXT:                    "line": 0
+# CHECK-NEXT:                  },
+# CHECK-NEXT:                  "start": {
+# CHECK-NEXT:                    "character": 37,
+# CHECK-NEXT:                    "line": 0
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              }
+# CHECK-NEXT:            ],
+# CHECK-NEXT:            "textDocument": {
+# CHECK-NEXT:              "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:              "version": 1
+# CHECK-NEXT:            }
+# CHECK-NEXT:          }
+# CHECK-NEXT:        ]
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "kind": "quickfix",
+# CHECK-NEXT:      "title": "place parentheses around the assignment to silence this warning"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "diagnostics": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "code": "-Wparentheses",
+# CHECK-NEXT:          "message": "Using the result of an assignment as a condition without parentheses (fixes available)",
+# CHECK-NEXT:          "range": {
+# CHECK-NEXT:            "end": {
+# CHECK-NEXT:              "character": 37,
+# CHECK-NEXT:              "line": 0
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "start": {
+# CHECK-NEXT:              "character": 32,
+# CHECK-NEXT:              "line": 0
+# CHECK-NEXT:            }
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "severity": 2,
+# CHECK-NEXT:          "source": "clang"
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "edit": {
+# CHECK-NEXT:        "documentChanges": [
+# CHECK-NEXT:          {
+# CHECK-NEXT:            "edits": [
+# CHECK-NEXT:              {
+# CHECK-NEXT:                "newText": "==",
+# CHECK-NEXT:                "range": {
+# CHECK-NEXT:                  "end": {
+# CHECK-NEXT:                    "character": 35,
+# CHECK-NEXT:                    "line": 0
+# CHECK-NEXT:                  },
+# CHECK-NEXT:                  "start": {
+# CHECK-NEXT:                    "character": 34,
+# CHECK-NEXT:                    "line": 0
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              }
+# CHECK-NEXT:            ],
+# CHECK-NEXT:            "textDocument": {
+# CHECK-NEXT:              "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:              "version": 1
+# CHECK-NEXT:            }
+# CHECK-NEXT:          }
+# CHECK-NEXT:        ]
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "kind": "quickfix",
+# CHECK-NEXT:      "title": "use '==' to turn this assignment into an equality comparison"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+---
+{"jsonrpc":"2.0","id":4,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
+
diff --git a/clang-tools-extra/clangd/test/fixits-command-documentchanges.test b/clang-tools-extra/clangd/test/fixits-command-documentchanges.test
new file mode 100644
index 0000000000000..cd636c4df387a
--- /dev/null
+++ b/clang-tools-extra/clangd/test/fixits-command-documentchanges.test
@@ -0,0 +1,194 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"workspace":{"workspaceEdit":{"documentChanges":true}}},"trace":"off"}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.c","languageId":"c","text":"int main(int i, char **a) { if (i = 2) {}}"}}}
+#      CHECK:    "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "code": "-Wparentheses",
+# CHECK-NEXT:        "message": "Using the result of an assignment as a condition without parentheses (fixes available)",
+# CHECK-NEXT:        "range": {
+# CHECK-NEXT:          "end": {
+# CHECK-NEXT:            "character": 37,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "start": {
+# CHECK-NEXT:            "character": 32,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          }
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "severity": 2,
+# CHECK-NEXT:        "source": "clang"
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:    "version": 0
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","id":2,"method":"textDocument/codeAction","params":{"textDocument":{"uri":"test:///foo.c"},"range":{"start":{"line":0,"character":13},"end":{"line":0,"character":35}},"context":{"diagnostics":[{"range":{"start": {"line": 0, "character": 32}, "end": {"line": 0, "character": 37}},"severity":2,"message":"Using the result of an assignment as a condition without parentheses (fixes available)"}]}}}
+#      CHECK:  "id": 2,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": "(",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 32,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 32,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": ")",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 37,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 37,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: place parentheses around the assignment to silence this warning"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": "==",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 35,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 34,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: use '==' to turn this assignment into an equality comparison"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+---
+{"jsonrpc":"2.0","id":3,"method":"textDocument/codeAction","params":{"textDocument":{"uri":"test:///foo.c"},"range":{"start":{"line":0,"character":13},"end":{"line":0,"character":35}},"context":{"diagnostics":[{"range":{"start": {"line": 0, "character": 32}, "end": {"line": 0, "character": 37}},"severity":2,"message":"Using the result of an assignment as a condition without parentheses (fixes available)"}]}}}
+# Make sure unused "code" and "source" fields ignored gracefully
+#      CHECK:  "id": 3,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": "(",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 32,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 32,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": ")",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 37,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 37,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: place parentheses around the assignment to silence this warning"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": "==",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 35,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 34,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: use '==' to turn this assignment into an equality comparison"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+---
+{"jsonrpc":"2.0","id":4,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/test/fixits-embed-in-diagnostic.test b/clang-tools-extra/clangd/test/fixits-embed-in-diagnostic.test
index debe4dfa5e789..26d3820d04322 100644
--- a/clang-tools-extra/clangd/test/fixits-embed-in-diagnostic.test
+++ b/clang-tools-extra/clangd/test/fixits-embed-in-diagnostic.test
@@ -48,6 +48,7 @@
 # CHECK-NEXT:        "source": "clang"
 # CHECK-NEXT:      },
 # CHECK-NEXT:      {
+# CHECK-NEXT:        "codeActions": [],
 # CHECK-NEXT:        "message": "Previous use is here\n\nfoo.c:1:18: error: use of 'Point' with tag type that does not match previous declaration",
 # CHECK-NEXT:        "range": {
 # CHECK-NEXT:          "end": {
diff --git a/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test b/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test
new file mode 100644
index 0000000000000..b7e9661b79b08
--- /dev/null
+++ b/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test
@@ -0,0 +1,487 @@
+# We specify a custom path in XDG_CONFIG_HOME, which only works on some systems.
+# UNSUPPORTED: system-windows
+# UNSUPPORTED: system-darwin
+
+# RUN: rm -rf %t
+# RUN: mkdir -p %t/clangd
+# RUN: cp -r %S/Inputs/include-cleaner %t/include
+# Create a config file enabling include-cleaner features.
+# RUN: echo $'Diagnostics:\n  UnusedIncludes: Strict\n  MissingIncludes: Strict' >> %t/clangd/config.yaml
+
+# RUN: env XDG_CONFIG_HOME=%t clangd -lit-test -enable-config --resource-dir=%t < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"workspace":{"workspaceEdit":{"documentChanges":true, "changeAnnotationSupport":{"groupsOnLabel":true}}}},"trace":"off"}}
+---
+{
+  "jsonrpc": "2.0",
+  "method": "textDocument/didOpen",
+  "params": {
+    "textDocument": {
+      "uri": "test:///simple.cpp",
+      "languageId": "cpp",
+      "text": "#include \"all1.h\"\n#include \"all2.h\"\n Foo* foo; Bar* bar;"
+    }
+  }
+}
+# First, the diagnostic from the config file.
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [],
+
+# Then, diagnostic from the main cpp file.
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:     "diagnostics": [
+# CHECK-NEXT:       {
+# CHECK-NEXT:         "code": "missing-includes",
+# CHECK-NEXT:         "message": "No header providing \"Foo\" is directly included (fixes available)",
+# CHECK-NEXT:         "range": {
+# CHECK-NEXT:           "end": {
+# CHECK-NEXT:             "character": 4,
+# CHECK-NEXT:             "line": 2
+# CHECK-NEXT:           },
+# CHECK-NEXT:           "start": {
+# CHECK-NEXT:             "character": 1,
+# CHECK-NEXT:             "line": 2
+# CHECK-NEXT:           }
+# CHECK-NEXT:         },
+# CHECK-NEXT:         "severity": 2,
+# CHECK-NEXT:         "source": "clangd"
+# CHECK-NEXT:       },
+# CHECK-NEXT:       {
+# CHECK-NEXT:         "code": "missing-includes",
+# CHECK-NEXT:         "message": "No header providing \"Bar\" is directly included (fixes available)",
+# CHECK-NEXT:         "range": {
+# CHECK-NEXT:           "end": {
+# CHECK-NEXT:             "character": 14,
+# CHECK-NEXT:             "line": 2
+# CHECK-NEXT:           },
+# CHECK-NEXT:           "start": {
+# CHECK-NEXT:             "character": 11,
+# CHECK-NEXT:             "line": 2
+# CHECK-NEXT:           }
+# CHECK-NEXT:         },
+# CHECK-NEXT:         "severity": 2,
+# CHECK-NEXT:         "source": "clangd"
+# CHECK-NEXT:       },
+# CHECK-NEXT:       {
+# CHECK-NEXT:         "code": "unused-includes",
+# CHECK-NEXT:         "codeDescription": {
+# CHECK-NEXT:           "href": "https://clangd.llvm.org/guides/include-cleaner"
+# CHECK-NEXT:         },
+# CHECK-NEXT:         "message": "Included header all1.h is not used directly (fixes available)",
+# CHECK-NEXT:         "range": {
+# CHECK-NEXT:           "end": {
+# CHECK-NEXT:             "character": 17,
+# CHECK-NEXT:             "line": 0
+# CHECK-NEXT:           },
+# CHECK-NEXT:           "start": {
+# CHECK-NEXT:             "character": 0,
+# CHECK-NEXT:             "line": 0
+# CHECK-NEXT:           }
+# CHECK-NEXT:         },
+# CHECK-NEXT:         "severity": 2,
+# CHECK-NEXT:         "source": "clangd",
+# CHECK-NEXT:         "tags": [
+# CHECK-NEXT:           1
+# CHECK-NEXT:         ]
+# CHECK-NEXT:       },
+# CHECK-NEXT:       {
+# CHECK-NEXT:         "code": "unused-includes",
+# CHECK-NEXT:         "codeDescription": {
+# CHECK-NEXT:           "href": "https://clangd.llvm.org/guides/include-cleaner"
+# CHECK-NEXT:         },
+# CHECK-NEXT:         "message": "Included header all2.h is not used directly (fixes available)",
+# CHECK-NEXT:         "range": {
+# CHECK-NEXT:           "end": {
+# CHECK-NEXT:             "character": 17,
+# CHECK-NEXT:             "line": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           "start": {
+# CHECK-NEXT:             "character": 0,
+# CHECK-NEXT:             "line": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         },
+# CHECK-NEXT:         "severity": 2,
+# CHECK-NEXT:         "source": "clangd",
+# CHECK-NEXT:         "tags": [
+# CHECK-NEXT:           1
+# CHECK-NEXT:         ]
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ],
+# CHECK-NEXT:     "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:     "version": 0
+# CHECK-NEXT:   }
+---
+{"jsonrpc":"2.0","id":2,"method":"textDocument/codeAction","params":{"textDocument":{"uri":"test:///simple.cpp"},"range":{"start":{"line":2,"character":1},"end":{"line":2,"character":4}},"context":{"diagnostics":[{"range":{"start": {"line": 2, "character": 1}, "end": {"line": 2, "character": 4}},"severity":2,"message":"No header providing \"Foo\" is directly included (fixes available)", "code": "missing-includes", "source": "clangd"}]}}}
+#      CHECK:  "id": 2,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": "#include {{.*}}foo.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: #include {{.*}}foo.h{{.*}}"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "changeAnnotations": {
+# CHECK-NEXT:            "AddAllMissingIncludes0": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "AddAllMissingIncludes1": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            }
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "AddAllMissingIncludes0",
+# CHECK-NEXT:                  "newText": "#include {{.*}}bar.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "AddAllMissingIncludes1",
+# CHECK-NEXT:                  "newText": "#include {{.*}}foo.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: add all missing includes"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "changeAnnotations": {
+# CHECK-NEXT:            "AddAllMissingIncludes0": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "AddAllMissingIncludes1": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "RemoveAllUnusedIncludes0": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "RemoveAllUnusedIncludes1": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            }
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "RemoveAllUnusedIncludes0",
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "RemoveAllUnusedIncludes1",
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "AddAllMissingIncludes0",
+# CHECK-NEXT:                  "newText": "#include {{.*}}bar.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "AddAllMissingIncludes1",
+# CHECK-NEXT:                  "newText": "#include {{.*}}foo.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: fix all includes"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+---
+{"jsonrpc":"2.0","id":3,"method":"textDocument/codeAction","params":{"textDocument":{"uri":"test:///simple.cpp"},"range":{"start":{"line":0,"character":0},"end":{"line":0,"character":17}},"context":{"diagnostics":[{"range":{"start": {"line": 0, "character": 0}, "end": {"line": 0, "character": 17}},"severity":2,"message":"Included header all1.h is not used directly (fixes available)", "code": "unused-includes", "source": "clangd"}]}}}
+#      CHECK:  "id": 3,
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: remove #include directive"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "changeAnnotations": {
+# CHECK-NEXT:            "RemoveAllUnusedIncludes0": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "RemoveAllUnusedIncludes1": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            }
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "RemoveAllUnusedIncludes0",
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "RemoveAllUnusedIncludes1",
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: remove all unused includes"
+# CHECK-NEXT:    },
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "arguments": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "changeAnnotations": {
+# CHECK-NEXT:            "AddAllMissingIncludes0": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "AddAllMissingIncludes1": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "RemoveAllUnusedIncludes0": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            },
+# CHECK-NEXT:            "RemoveAllUnusedIncludes1": {
+# CHECK-NEXT:              "label": "",
+# CHECK-NEXT:              "needsConfirmation": true
+# CHECK-NEXT:            }
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "documentChanges": [
+# CHECK-NEXT:            {
+# CHECK-NEXT:              "edits": [
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "RemoveAllUnusedIncludes0",
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 0
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "RemoveAllUnusedIncludes1",
+# CHECK-NEXT:                  "newText": "",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 1
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "AddAllMissingIncludes0",
+# CHECK-NEXT:                  "newText": "#include {{.*}}bar.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                },
+# CHECK-NEXT:                {
+# CHECK-NEXT:                  "annotationId": "AddAllMissingIncludes1",
+# CHECK-NEXT:                  "newText": "#include {{.*}}foo.h{{.*}}",
+# CHECK-NEXT:                  "range": {
+# CHECK-NEXT:                    "end": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    },
+# CHECK-NEXT:                    "start": {
+# CHECK-NEXT:                      "character": 0,
+# CHECK-NEXT:                      "line": 2
+# CHECK-NEXT:                    }
+# CHECK-NEXT:                  }
+# CHECK-NEXT:                }
+# CHECK-NEXT:              ],
+# CHECK-NEXT:              "textDocument": {
+# CHECK-NEXT:                "uri": "file://{{.*}}/simple.cpp",
+# CHECK-NEXT:                "version": 0
+# CHECK-NEXT:              }
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
+# CHECK-NEXT:      "command": "clangd.applyFix",
+# CHECK-NEXT:      "title": "Apply fix: fix all includes"
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+---
+{"jsonrpc":"2.0","id":4,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index 289767b0dec0f..552f4f4ecf886 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -1908,11 +1908,11 @@ TEST(DiagnosticsTest, IncludeCleaner) {
   auto AST = TU.build();
   EXPECT_THAT(
       *AST.getDiagnostics(),
-      UnorderedElementsAre(AllOf(
-          Diag(Test.range("diag"),
-               "included header unused.h is not used directly"),
-          withTag(DiagnosticTag::Unnecessary), diagSource(Diag::Clangd),
-          withFix(Fix(Test.range("fix"), "", "remove #include directive")))));
+      UnorderedElementsAre(
+          AllOf(Diag(Test.range("diag"),
+                     "included header unused.h is not used directly"),
+                withTag(DiagnosticTag::Unnecessary), diagSource(Diag::Clangd),
+                withFix(Fix(Test.range("fix"), "", "remove #include directive")))));
   auto &Diag = AST.getDiagnostics()->front();
   EXPECT_EQ(getDiagnosticDocURI(Diag.Source, Diag.ID, Diag.Name),
             std::string("https://clangd.llvm.org/guides/include-cleaner"));
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 5ad9d69cb0dc2..ce546f54dd75f 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -906,6 +906,35 @@ class Foo final {})cpp";
          HI.CalleeArgInfo->Type = "int &";
          HI.CallPassType = HoverInfo::PassType{PassMode::Ref, false};
        }},
+      {// make_unique-like function call
+       R"cpp(
+          struct Foo {
+            explicit Foo(int arg_a) {}
+          };
+          template<class T, class... Args>
+          T make(Args&&... args)
+          {
+              return T(args...);
+          }
+
+          void code() {
+            int a = 1;
+            auto foo = make<Foo>([[^a]]);
+          }
+          )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "a";
+         HI.Kind = index::SymbolKind::Variable;
+         HI.NamespaceScope = "";
+         HI.Definition = "int a = 1";
+         HI.LocalScope = "code::";
+         HI.Value = "1";
+         HI.Type = "int";
+         HI.CalleeArgInfo.emplace();
+         HI.CalleeArgInfo->Name = "arg_a";
+         HI.CalleeArgInfo->Type = "int";
+         HI.CallPassType = HoverInfo::PassType{PassMode::Value, false};
+       }},
       {
           R"cpp(
           void foobar(const float &arg);
@@ -927,6 +956,29 @@ class Foo final {})cpp";
             HI.CalleeArgInfo->Type = "const float &";
             HI.CallPassType = HoverInfo::PassType{PassMode::Value, true};
           }},
+      {
+          R"cpp(
+          struct Foo {
+            explicit Foo(const float& arg) {}
+          };
+          int main() {
+            int a = 0;
+            Foo foo([[^a]]);
+          }
+          )cpp",
+          [](HoverInfo &HI) {
+            HI.Name = "a";
+            HI.Kind = index::SymbolKind::Variable;
+            HI.NamespaceScope = "";
+            HI.Definition = "int a = 0";
+            HI.LocalScope = "main::";
+            HI.Value = "0";
+            HI.Type = "int";
+            HI.CalleeArgInfo.emplace();
+            HI.CalleeArgInfo->Name = "arg";
+            HI.CalleeArgInfo->Type = "const float &";
+            HI.CallPassType = HoverInfo::PassType{PassMode::Value, true};
+          }},
       {// Literal passed to function call
        R"cpp(
           void fun(int arg_a, const int &arg_b) {};
@@ -1313,6 +1365,7 @@ class CustomClass {
   CustomClass(const Base &x) {}
   CustomClass(int &x) {}
   CustomClass(float x) {}
+  CustomClass(int x, int y) {}
 };
 
 void int_by_ref(int &x) {}
@@ -1359,6 +1412,11 @@ void fun() {
       {"base_by_ref([[^derived]]);", PassMode::Ref, false},
       {"base_by_const_ref([[^derived]]);", PassMode::ConstRef, false},
       {"base_by_value([[^derived]]);", PassMode::Value, false},
+      // Custom class constructor tests
+      {"CustomClass c1([[^base]]);", PassMode::ConstRef, false},
+      {"auto c2 = new CustomClass([[^base]]);", PassMode::ConstRef, false},
+      {"CustomClass c3([[^int_x]]);", PassMode::Ref, false},
+      {"CustomClass c3(int_x, [[^int_x]]);", PassMode::Value, false},
       // Converted tests
       {"float_by_value([[^int_x]]);", PassMode::Value, true},
       {"float_by_value([[^int_ref]]);", PassMode::Value, true},
diff --git a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
index 39901fda31656..a38c01b43270f 100644
--- a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp
@@ -45,8 +45,8 @@ using ::testing::Matcher;
 using ::testing::Pointee;
 using ::testing::UnorderedElementsAre;
 
-Matcher<const Diag &> withFix(::testing::Matcher<Fix> FixMatcher) {
-  return Field(&Diag::Fixes, ElementsAre(FixMatcher));
+Matcher<const Diag &> withFix(std::vector<::testing::Matcher<Fix>> FixMatcheres) {
+  return Field(&Diag::Fixes, testing::UnorderedElementsAreArray(FixMatcheres));
 }
 
 MATCHER_P2(Diag, Range, Message,
@@ -60,6 +60,8 @@ MATCHER_P3(Fix, Range, Replacement, Message,
   return arg.Message == Message && arg.Edits.size() == 1 &&
          arg.Edits[0].range == Range && arg.Edits[0].newText == Replacement;
 }
+MATCHER_P(FixMessage, Message, "") { return arg.Message == Message; }
+
 
 std::string guard(llvm::StringRef Code) {
   return "#pragma once\n" + Code.str();
@@ -255,42 +257,51 @@ TEST(IncludeCleaner, GenerateMissingHeaderDiags) {
       UnorderedElementsAre(
           AllOf(Diag(MainFile.range("b"),
                      "No header providing \"b\" is directly included"),
-                withFix(Fix(MainFile.range("insert_b"), "#include \"b.h\"\n",
-                            "#include \"b.h\""))),
+                withFix({Fix(MainFile.range("insert_b"), "#include \"b.h\"\n",
+                             "#include \"b.h\""),
+                         FixMessage("add all missing includes")})),
           AllOf(Diag(MainFile.range("bar"),
                      "No header providing \"ns::Bar\" is directly included"),
-                withFix(Fix(MainFile.range("insert_d"),
-                            "#include \"dir/d.h\"\n", "#include \"dir/d.h\""))),
+                withFix({Fix(MainFile.range("insert_d"),
+                             "#include \"dir/d.h\"\n", "#include \"dir/d.h\""),
+                         FixMessage("add all missing includes")})),
           AllOf(Diag(MainFile.range("f"),
                      "No header providing \"f\" is directly included"),
-                withFix(Fix(MainFile.range("insert_f"), "#include <f.h>\n",
-                            "#include <f.h>"))),
+                withFix({Fix(MainFile.range("insert_f"), "#include <f.h>\n",
+                             "#include <f.h>"),
+                         FixMessage("add all missing includes")})),
           AllOf(
               Diag(MainFile.range("foobar"),
                    "No header providing \"foobar\" is directly included"),
-              withFix(Fix(MainFile.range("insert_foobar"),
-                          "#include \"public.h\"\n", "#include \"public.h\""))),
+              withFix({Fix(MainFile.range("insert_foobar"),
+                           "#include \"public.h\"\n", "#include \"public.h\""),
+                       FixMessage("add all missing includes")})),
           AllOf(
               Diag(MainFile.range("vector"),
                    "No header providing \"std::vector\" is directly included"),
-              withFix(Fix(MainFile.range("insert_vector"),
-                          "#include <vector>\n", "#include <vector>"))),
+              withFix({Fix(MainFile.range("insert_vector"),
+                           "#include <vector>\n", "#include <vector>"),
+                       FixMessage("add all missing includes"),})),
           AllOf(Diag(MainFile.range("FOO"),
                      "No header providing \"FOO\" is directly included"),
-                withFix(Fix(MainFile.range("insert_foo"),
-                            "#include \"foo.h\"\n", "#include \"foo.h\""))),
+                withFix({Fix(MainFile.range("insert_foo"),
+                             "#include \"foo.h\"\n", "#include \"foo.h\""),
+                         FixMessage("add all missing includes")})),
           AllOf(Diag(MainFile.range("DEF"),
                      "No header providing \"Foo\" is directly included"),
-                withFix(Fix(MainFile.range("insert_foo"),
-                            "#include \"foo.h\"\n", "#include \"foo.h\""))),
+                withFix({Fix(MainFile.range("insert_foo"),
+                             "#include \"foo.h\"\n", "#include \"foo.h\""),
+                         FixMessage("add all missing includes")})),
           AllOf(Diag(MainFile.range("BAR"),
                      "No header providing \"BAR\" is directly included"),
-                withFix(Fix(MainFile.range("insert_foo"),
-                            "#include \"foo.h\"\n", "#include \"foo.h\""))),
+                withFix({Fix(MainFile.range("insert_foo"),
+                             "#include \"foo.h\"\n", "#include \"foo.h\""),
+                         FixMessage("add all missing includes")})),
           AllOf(Diag(MainFile.range("Foo"),
                      "No header providing \"Foo\" is directly included"),
-                withFix(Fix(MainFile.range("insert_foo"),
-                            "#include \"foo.h\"\n", "#include \"foo.h\"")))));
+                withFix({Fix(MainFile.range("insert_foo"),
+                             "#include \"foo.h\"\n", "#include \"foo.h\""),
+                         FixMessage("add all missing includes")}))));
 }
 
 TEST(IncludeCleaner, IWYUPragmas) {
@@ -403,15 +414,36 @@ TEST(IncludeCleaner, MacroExpandedThroughIncludes) {
   ParsedAST AST = TU.build();
 
   auto Findings = computeIncludeCleanerFindings(AST).MissingIncludes;
-  // FIXME: Deduplicate references resulting from expansion of the same macro in
-  // multiple places.
-  EXPECT_THAT(Findings, testing::SizeIs(2));
+  EXPECT_THAT(Findings, testing::SizeIs(1));
   auto RefRange = Findings.front().SymRefRange;
   auto &SM = AST.getSourceManager();
   EXPECT_EQ(RefRange.file(), SM.getMainFileID());
   // FIXME: Point at the spelling location, rather than the include.
   EXPECT_EQ(halfOpenToRange(SM, RefRange.toCharRange(SM)), MainFile.range());
-  EXPECT_EQ(RefRange, Findings[1].SymRefRange);
+}
+
+TEST(IncludeCleaner, MissingIncludesAreUnique) {
+  Annotations MainFile(R"cpp(
+    #include "all.h"
+    FOO([[Foo]]);
+  )cpp");
+
+  TestTU TU;
+  TU.AdditionalFiles["foo.h"] = guard("struct Foo {};");
+  TU.AdditionalFiles["all.h"] = guard(R"cpp(
+    #include "foo.h"
+    #define FOO(X) X y; X z
+  )cpp");
+
+  TU.Code = MainFile.code();
+  ParsedAST AST = TU.build();
+
+  auto Findings = computeIncludeCleanerFindings(AST).MissingIncludes;
+  EXPECT_THAT(Findings, testing::SizeIs(1));
+  auto RefRange = Findings.front().SymRefRange;
+  auto &SM = AST.getSourceManager();
+  EXPECT_EQ(RefRange.file(), SM.getMainFileID());
+  EXPECT_EQ(halfOpenToRange(SM, RefRange.toCharRange(SM)), MainFile.range());
 }
 
 TEST(IncludeCleaner, NoCrash) {
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 49f880c971a34..98f18cc539faf 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -106,6 +106,12 @@ Improvements to clang-tidy
 New checks
 ^^^^^^^^^^
 
+- New :doc:`bugprone-multiple-new-in-one-expression
+  <clang-tidy/checks/bugprone/multiple-new-in-one-expression>` check.
+
+  Finds multiple ``new`` operator calls in a single expression, where the allocated
+  memory by the first ``new`` may leak if the second allocation fails and throws exception.
+
 - New :doc:`bugprone-non-zero-enum-to-bool-conversion
   <clang-tidy/checks/bugprone/non-zero-enum-to-bool-conversion>` check.
 
@@ -201,6 +207,10 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/dynamic-static-initializers>` check.
   Global options of the same name should be used instead.
 
+- Improved :doc:`bugprone-exception-escape
+  <clang-tidy/checks/bugprone/exception-escape>` to not emit warnings for
+  forward declarations of functions.
+
 - Improved :doc:`bugprone-fold-init-type
   <clang-tidy/checks/bugprone/fold-init-type>` to handle iterators that do not
   define `value_type` type aliases.
@@ -219,8 +229,9 @@ Changes in existing checks
   to ``std::forward``.
 
 - Improved :doc:`bugprone-use-after-move
-  <clang-tidy/checks/bugprone/use-after-move>` check to also cover constructor
-  initializers.
+  <clang-tidy/checks/bugprone/use-after-move>` check. Detect uses and moves in
+  constructor initializers. Correctly handle constructor arguments as being
+  sequenced when constructor call is written as list-initialization.
 
 - Deprecated :doc:`cert-dcl21-cpp
   <clang-tidy/checks/cert/dcl21-cpp>` check.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/multiple-new-in-one-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/multiple-new-in-one-expression.rst
new file mode 100644
index 0000000000000..b9b8984ef6584
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/multiple-new-in-one-expression.rst
@@ -0,0 +1,99 @@
+.. title:: clang-tidy - bugprone-multiple-new-in-one-expression
+
+bugprone-multiple-new-in-one-expression
+=======================================
+
+Finds multiple ``new`` operator calls in a single expression, where the
+allocated memory by the first ``new`` may leak if the second allocation fails
+and throws exception.
+
+C++ does often not specify the exact order of evaluation of the operands of an
+operator or arguments of a function. Therefore if a first allocation succeeds
+and a second fails, in an exception handler it is not possible to tell which
+allocation has failed and free the memory. Even if the order is fixed the result
+of a first ``new`` may be stored in a temporary location that is not reachable
+at the time when a second allocation fails. It is best to avoid any expression
+that contains more than one ``operator new`` call, if exception handling is
+used to check for allocation errors.
+
+Different rules apply for are the short-circuit operators ``||`` and ``&&`` and
+the ``,`` operator, where evaluation of one side must be completed before the
+other starts. Expressions of a list-initialization (initialization or
+construction using ``{`` and ``}`` characters) are evaluated in fixed order.
+Similarly, condition of a ``?`` operator is evaluated before the branches are
+evaluated.
+
+The check reports warning if two ``new`` calls appear in one expression at
+different sides of an operator, or if ``new`` calls appear in different
+arguments of a function call (that can be an object construction with ``()``
+syntax). These ``new`` calls can be nested at any level.
+For any warning to be emitted the ``new`` calls should be in a code block where
+exception handling is used with catch for ``std::bad_alloc`` or
+``std::exception``. At ``||``, ``&&``, ``,``, ``?`` (condition and one branch)
+operators no warning is emitted. No warning is emitted if both of the memory
+allocations are not assigned to a variable or not passed directly to a function.
+The reason is that in this case the memory may be intentionally not freed or the
+allocated objects can be self-destructing objects.
+
+Examples:
+
+.. code-block:: c++
+
+  struct A {
+    int Var;
+  };
+  struct B {
+    B();
+    B(A *);
+    int Var;
+  };
+  struct C {
+    int *X1;
+    int *X2;
+  };
+
+  void f(A *, B *);
+  int f1(A *);
+  int f1(B *);
+  bool f2(A *);
+
+  void foo() {
+    A *PtrA;
+    B *PtrB;
+    try {
+      // Allocation of 'B'/'A' may fail after memory for 'A'/'B' was allocated.
+      f(new A, new B); // warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception; order of these allocations is undefined
+
+      // List (aggregate) initialization is used.
+      C C1{new int, new int}; // no warning
+
+      // Allocation of 'B'/'A' may fail after memory for 'A'/'B' was allocated but not yet passed to function 'f1'.
+      int X = f1(new A) + f1(new B); // warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception; order of these allocations is undefined
+
+      // Allocation of 'B' may fail after memory for 'A' was allocated.
+      // From C++17 on memory for 'B' is allocated first but still may leak if allocation of 'A' fails.
+      PtrB = new B(new A); // warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception
+
+      // 'new A' and 'new B' may be performed in any order.
+      // 'new B'/'new A' may fail after memory for 'A'/'B' was allocated but not assigned to 'PtrA'/'PtrB'.
+      (PtrA = new A)->Var = (PtrB = new B)->Var; // warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception; order of these allocations is undefined
+
+      // Evaluation of 'f2(new A)' must be finished before 'f1(new B)' starts.
+      // If 'new B' fails the allocated memory for 'A' is supposedly handled correctly because function 'f2' could take the ownership.
+      bool Z = f2(new A) || f1(new B); // no warning
+
+      X = (f2(new A) ? f1(new A) : f1(new B)); // no warning
+
+      // No warning if the result of both allocations is not passed to a function
+      // or stored in a variable.
+      (new A)->Var = (new B)->Var; // no warning
+
+      // No warning if at least one non-throwing allocation is used.
+      f(new(std::nothrow) A, new B); // no warning
+    } catch(std::bad_alloc) {
+    }
+
+    // No warning if the allocation is outside a try block (or no catch handler exists for std::bad_alloc).
+    // (The fact if exceptions can escape from 'foo' is not taken into account.)
+    f(new A, new B); // no warning
+  }
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 3ddd257e70de6..6a58bb932e9ec 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -102,6 +102,7 @@ Clang-Tidy Checks
    `bugprone-misplaced-pointer-arithmetic-in-alloc <bugprone/misplaced-pointer-arithmetic-in-alloc.html>`_, "Yes"
    `bugprone-misplaced-widening-cast <bugprone/misplaced-widening-cast.html>`_,
    `bugprone-move-forwarding-reference <bugprone/move-forwarding-reference.html>`_, "Yes"
+   `bugprone-multiple-new-in-one-expression <bugprone/multiple-new-in-one-expression.html>`_,
    `bugprone-multiple-statement-macro <bugprone/multiple-statement-macro.html>`_,
    `bugprone-no-escape <bugprone/no-escape.html>`_,
    `bugprone-non-zero-enum-to-bool-conversion <bugprone/non-zero-enum-to-bool-conversion.html>`_,
diff --git a/clang-tools-extra/docs/doxygen.cfg.in b/clang-tools-extra/docs/doxygen.cfg.in
index 7e1d47a7a95a5..df33a63906389 100644
--- a/clang-tools-extra/docs/doxygen.cfg.in
+++ b/clang-tools-extra/docs/doxygen.cfg.in
@@ -2102,7 +2102,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2147,7 +2147,7 @@ TEMPLATE_RELATIONS     = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2156,7 +2156,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp
index 142c8b114b7c8..78d434854b095 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp
@@ -655,7 +655,6 @@ int directly_recursive(int n) noexcept {
 }
 
 int indirectly_recursive(int n) noexcept;
-  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: an exception may be thrown in function 'indirectly_recursive' which should not throw exceptions
 
 int recursion_helper(int n) {
   indirectly_recursive(n);
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/multiple-new-in-one-expression.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/multiple-new-in-one-expression.cpp
new file mode 100644
index 0000000000000..3bf8544d599f3
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/multiple-new-in-one-expression.cpp
@@ -0,0 +1,197 @@
+// RUN: %check_clang_tidy -std=c++11 -check-suffixes=ALL,CPP11 %s bugprone-multiple-new-in-one-expression %t -- -- -target x86_64-unknown-unknown
+// RUN: %check_clang_tidy -std=c++17 -check-suffixes=ALL,CPP17 %s bugprone-multiple-new-in-one-expression %t -- -- -target x86_64-unknown-unknown
+
+namespace std {
+typedef __typeof__(sizeof(0)) size_t;
+enum class align_val_t : std::size_t {};
+class exception {};
+class bad_alloc : public exception {};
+struct nothrow_t {};
+extern const nothrow_t nothrow;
+} // namespace std
+
+void *operator new(std::size_t, const std::nothrow_t &) noexcept;
+void *operator new(std::size_t, std::align_val_t, const std::nothrow_t &) noexcept;
+void *operator new(std::size_t, void *) noexcept;
+void *operator new(std::size_t, char);
+
+struct B;
+
+struct A { int VarI; int *PtrI; B *PtrB; };
+
+struct B { int VarI; };
+
+struct G {
+  G(A*, B*) {}
+  int operator+=(A *) { return 3; };
+};
+
+struct H {
+  int *a;
+  int *b;
+};
+
+int f(int);
+int f(A*);
+int f(A*, B*);
+int f(int, B*);
+int f(G, G);
+int f(B*);
+int f(const H &);
+void f1(void *, void *);
+A *g(A *);
+
+G operator+(const G&, const G&);
+
+void test_function_parameter(A *XA, B *XB) {
+  (void)f(new A, new B);
+  try {
+    (void)f(new A, new B);
+  }
+  catch (A) {};
+  try {
+    (void)f(new A, new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception; order of these allocations is undefined [
+    (void)f(f(new A, new B));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    int X = f(new A, new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    X = f(new A, new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:11: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    X = 1 + f(new A, new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)f(g(new A), new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)f(1 + f(new A), new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:19: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (void)f(XA = new A, new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:18: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (void)f(1 + f(new A), XB = new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:19: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+  }
+  catch (std::exception) {}
+}
+
+void test_operator(G *G1) {
+  (void)(f(new A) + f(new B));
+  try {
+    (void)(f(new A) + f(new B));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (void)f(f(new A) + f(new B));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    int X = f(new A) + f(new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    X = f(new A) + f(new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:11: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    X = 1 + f(new A) + 1 + f(new B);
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:15: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)(f(g(new A)) + f(new B));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:16: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)(f(1 + f(new A)) + f(new B));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:20: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (void)(f(1 + f(new A)) + f(1 + f(new B)));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:20: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)((new A)->VarI + (new A)->VarI);
+
+    (void)(f(new A) + ((*G1) += new A));
+     // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+  }
+  catch (std::bad_alloc) {}
+}
+
+void test_construct() {
+  (void)(G(new A, new B));
+  try {
+    (void)(G(new A, new B));
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (void)(G(new A, nullptr) + G(nullptr, new B));
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    f(G(new A, nullptr), G(new A, nullptr));
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:9: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)new G(new A, nullptr);
+    // CHECK-MESSAGES-CPP11: :[[@LINE-1]]:11: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    // CHECK-MESSAGES-CPP17: :[[@LINE-2]]:11: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception [
+    (void)new G(nullptr, (new A)->PtrB);
+    G *Z = new G(new A, nullptr);
+    // CHECK-MESSAGES-CPP11: :[[@LINE-1]]:12: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    // CHECK-MESSAGES-CPP17: :[[@LINE-2]]:12: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception [
+    Z = new G(g(new A), nullptr);
+    // CHECK-MESSAGES-CPP11: :[[@LINE-1]]:9: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    // CHECK-MESSAGES-CPP17: :[[@LINE-2]]:9: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception [
+    G *Z1, *Z2 = new G(nullptr, (new A)->PtrB), *Z3;
+    // CHECK-MESSAGES-CPP11: :[[@LINE-1]]:18: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    // CHECK-MESSAGES-CPP17: :[[@LINE-2]]:18: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception [
+ }
+  catch (const std::bad_alloc &) {}
+}
+
+void test_new_assign() {
+  A *X, *Y;
+  (X = new A)->VarI = (Y = new A)->VarI;
+  try {
+    (X = new A)->VarI = (Y = new A)->VarI;
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:10: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (new A)->VarI = (Y = new A)->VarI;
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:6: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (X = new A)->VarI = (new A)->VarI;
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:10: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (new A)->VarI = (new A)->VarI;
+    (new A)->PtrI = new int;
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:6: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+    (X = new A)->VarI += (new A)->VarI;
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:10: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+  }
+  catch (...) {}
+}
+
+void test_operator_fixed_order(unsigned int L) {
+  (void)(f((f(new A) || f(0)) + f(new B[L])));
+  try {
+    (void)(f(new A) || f(new B));
+    (void)(f(new A) && f(new B));
+    (void)(f(new A) || f(new B) || f(new A));
+
+    (void)(f(new A), f(new B));
+
+    int Y = f(0, new B) ? f(new A) : f(new B);
+    Y = f(new A) ? 1 : f(new B);
+    Y = f(new A) ? f(new B) : 1;
+
+    G g{new A, new B};
+    H h{new int, new int};
+    f({new int, new int});
+    (void)f({new A, new B}, {nullptr, nullptr});
+    (void)f({new A, new B}, {new A, nullptr});
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:14: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+
+    (void)(f((f(new A) || f(0)) + f(new B[L])));
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:17: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+  }
+  catch (std::bad_alloc) {}
+}
+
+void test_cast() {
+  try {
+    f1(static_cast<void *>(new A), new B);
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:28: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+  }
+  catch (std::bad_alloc &) {}
+}
+
+void test_nothrow(void *P) {
+  try {
+    (void)f(new(std::nothrow) A, new B);
+    (void)f(new A, new(std::nothrow) B);
+    (void)f(new(static_cast<std::align_val_t>(8), std::nothrow) A, new B);
+    (void)f(new(P) A, new B);
+    (void)f(new('a') A, new B);
+    // CHECK-MESSAGES-ALL: :[[@LINE-1]]:13: warning: memory allocation may leak if an other allocation is sequenced after it and throws an exception;
+  }
+  catch (std::exception) {}
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp
index 1e0831048dbd4..00b1da1e727e4 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/use-after-move.cpp
@@ -1160,42 +1160,63 @@ void commaOperatorSequences() {
   }
 }
 
+namespace InitializerListSequences {
+
+struct S1 {
+  int i;
+  A a;
+};
+
+struct S2 {
+  A a;
+  int i;
+};
+
+struct S3 {
+  S3() {}
+  S3(int, A) {}
+  S3(A, int) {}
+};
+
 // An initializer list sequences its initialization clauses.
 void initializerListSequences() {
   {
-    struct S1 {
-      int i;
-      A a;
-    };
-    {
-      A a;
-      S1 s1{a.getInt(), std::move(a)};
-    }
-    {
-      A a;
-      S1 s1{.i = a.getInt(), .a = std::move(a)};
-    }
+    A a;
+    S1 s1{a.getInt(), std::move(a)};
   }
   {
-    struct S2 {
-      A a;
-      int i;
-    };
-    {
-      A a;
-      S2 s2{std::move(a), a.getInt()};
-      // CHECK-NOTES: [[@LINE-1]]:27: warning: 'a' used after it was moved
-      // CHECK-NOTES: [[@LINE-2]]:13: note: move occurred here
-    }
-    {
-      A a;
-      S2 s2{.a = std::move(a), .i = a.getInt()};
-      // CHECK-NOTES: [[@LINE-1]]:37: warning: 'a' used after it was moved
-      // CHECK-NOTES: [[@LINE-2]]:13: note: move occurred here
-    }
+    A a;
+    S1 s1{.i = a.getInt(), .a = std::move(a)};
+  }
+  {
+    A a;
+    S2 s2{std::move(a), a.getInt()};
+    // CHECK-NOTES: [[@LINE-1]]:25: warning: 'a' used after it was moved
+    // CHECK-NOTES: [[@LINE-2]]:11: note: move occurred here
+  }
+  {
+    A a;
+    S2 s2{.a = std::move(a), .i = a.getInt()};
+    // CHECK-NOTES: [[@LINE-1]]:35: warning: 'a' used after it was moved
+    // CHECK-NOTES: [[@LINE-2]]:11: note: move occurred here
+  }
+  {
+    // Check the case where the constructed type has a constructor and the
+    // initializer list therefore manifests as a `CXXConstructExpr` instead of
+    // an `InitListExpr`.
+    A a;
+    S3 s3{a.getInt(), std::move(a)};
+  }
+  {
+    A a;
+    S3 s3{std::move(a), a.getInt()};
+    // CHECK-NOTES: [[@LINE-1]]:25: warning: 'a' used after it was moved
+    // CHECK-NOTES: [[@LINE-2]]:11: note: move occurred here
   }
 }
 
+} // namespace InitializerListSequences
+
 // A declaration statement containing multiple declarations sequences the
 // initializer expressions.
 void declarationSequences() {
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 4508ea4c77aa5..fd8558a16f81d 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -660,8 +660,6 @@ if (CLANG_ENABLE_BOOTSTRAP)
     LLVM_VERSION_SUFFIX
     LLVM_BINUTILS_INCDIR
     CLANG_REPOSITORY_STRING
-    CMAKE_C_COMPILER_LAUNCHER
-    CMAKE_CXX_COMPILER_LAUNCHER
     CMAKE_MAKE_PROGRAM
     CMAKE_OSX_ARCHITECTURES
     CMAKE_BUILD_TYPE
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 2e9038933c814..bdcd1e88d2404 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -1829,6 +1829,41 @@ the configuration (without a prefix: ``Auto``).
                            }
 
 
+.. _BracedInitializerIndentWidth:
+
+**BracedInitializerIndentWidth** (``Unsigned``) :versionbadge:`clang-format 17` :ref:`¶ <BracedInitializerIndentWidth>`
+  The number of columns to use to indent the contents of braced init lists.
+  If unset, ``ContinuationIndentWidth`` is used.
+
+  .. code-block:: c++
+
+    AlignAfterOpenBracket: AlwaysBreak
+    BracedInitializerIndentWidth: 2
+
+    void f() {
+      SomeClass c{
+        "foo",
+        "bar",
+        "baz",
+      };
+      auto s = SomeStruct{
+        .foo = "foo",
+        .bar = "bar",
+        .baz = "baz",
+      };
+      SomeArrayT a[3] = {
+        {
+          foo,
+          bar,
+        },
+        {
+          foo,
+          bar,
+        },
+        SomeArrayT{},
+      };
+    }
+
 .. _BreakAfterAttributes:
 
 **BreakAfterAttributes** (``AttributeBreakingStyle``) :versionbadge:`clang-format 16` :ref:`¶ <BreakAfterAttributes>`
diff --git a/clang/docs/CommandGuide/clang.rst b/clang/docs/CommandGuide/clang.rst
index 0722979885afb..6c59ffd2ab120 100644
--- a/clang/docs/CommandGuide/clang.rst
+++ b/clang/docs/CommandGuide/clang.rst
@@ -193,13 +193,13 @@ Language Selection and Mode Options
 
    ISO C++ 2020 with amendments and GNU extensions
 
-  | ``c++2b``
+  | ``c++23``
 
-   Working draft for ISO C++ 2023
+   ISO C++ 2023 with amendments
 
-  | ``gnu++2b``
+  | ``gnu++23``
 
-   Working draft for ISO C++ 2023 with GNU extensions
+   ISO C++ 2023 with amendments and GNU extensions
 
  The default C++ language standard is ``gnu++17``.
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index a1bb925e8ae24..e0ad5f5e3ae53 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -93,13 +93,19 @@ C++20 Feature Support
   error again in the future once there is a less fragile way to mark a module
   as being part of the implementation rather than a user module.
 
-C++2b Feature Support
+C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
 - Implemented `P2036R3: Change scope of lambda trailing-return-type <https://wg21.link/P2036R3>`_
   and `P2579R0 Mitigation strategies for P2036 <https://wg21.link/P2579R0>`_.
   These proposals modify how variables captured in lambdas can appear in trailing return type
   expressions and how their types are deduced therein, in all C++ language versions.
+- Implemented partial support for `P2448R2: Relaxing some constexpr restrictions <https://wg21.link/p2448r2>`_
+  Explicitly defaulted functions no longer have to be constexpr-compatible but merely constexpr suitable.
+  We do not support outside of defaulted special memeber functions the change that constexpr functions no
+  longer have to be constexpr compatible but rather support a less restricted requirements for constexpr
+  functions. Which include allowing non-literal types as return values and paremeters, allow calling of
+  non-constexpr functions and constructors.
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -139,6 +145,25 @@ C2x Feature Support
   removed, as this is no longer a GNU extension but a C2x extension. You can
   use ``-Wno-c2x-extensions`` to silence the extension warning instead.
 
+- Updated the implementation of
+  `WG14 N3042 <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3042.htm>`_
+  based on decisions reached during the WG14 CD Ballot Resolution meetings held
+  in Jan and Feb 2023. This should complete the implementation of ``nullptr``
+  and ``nullptr_t`` in C. The specific changes are:
+
+  .. code-block:: c
+
+    void func(nullptr_t);
+    func(0); // Previously required to be rejected, is now accepted.
+    func((void *)0); // Previously required to be rejected, is now accepted.
+
+    nullptr_t val;
+    val = 0; // Previously required to be rejected, is now accepted.
+    val = (void *)0; // Previously required to be rejected, is now accepted.
+
+    bool b = nullptr; // Was incorrectly rejected by Clang, is now accepted.
+
+
 Non-comprehensive list of changes in this release
 -------------------------------------------------
 - Clang now saves the address of ABI-indirect function parameters on the stack,
@@ -157,9 +182,13 @@ Non-comprehensive list of changes in this release
 - A new builtin type trait ``__is_trivially_equaltiy_comparable`` has been added,
   which checks whether comparing two instances of a type is equivalent to
   ``memcmp(&lhs, &rhs, sizeof(T)) == 0``.
+- Clang now ignores null directives outside of the include guard when deciding
+  whether a file can be enabled for the multiple-include optimization.
 
 New Compiler Flags
 ------------------
+- The flag ``-std=c++23`` has been added. This behaves the same as the existing
+  flag ``-std=c++2b``.
 
 Deprecated Compiler Flags
 -------------------------
@@ -231,10 +260,14 @@ Improvements to Clang's diagnostics
 - ``-Wformat`` now recognizes ``%lb`` for the ``printf``/``scanf`` family of
   functions.
   (`#62247: <https://github.com/llvm/llvm-project/issues/62247>`_).
+- Clang now diagnoses shadowing of lambda's template parameter by a capture.
+  (`#61105: <https://github.com/llvm/llvm-project/issues/61105>`_).
 
 Bug Fixes in This Version
 -------------------------
 
+- Fix segfault while running clang-rename on a non existing file.
+  (`#36471 <https://github.com/llvm/llvm-project/issues/36471>`_)
 - Fix crash when diagnosing incorrect usage of ``_Nullable`` involving alias
   templates.
   (`#60344 <https://github.com/llvm/llvm-project/issues/60344>`_)
@@ -307,8 +340,8 @@ Bug Fixes in This Version
   not a type concept.
 - Fix crash when a doc comment contains a line splicing.
   (`#62054 <https://github.com/llvm/llvm-project/issues/62054>`_)
-- Work around with a clang coverage crash which happens when visiting 
-  expressions/statements with invalid source locations in non-assert builds. 
+- Work around with a clang coverage crash which happens when visiting
+  expressions/statements with invalid source locations in non-assert builds.
   Assert builds may still see assertions triggered from this.
 - Fix a failed assertion due to an invalid source location when trying to form
   a coverage report for an unresolved constructor expression.
@@ -321,12 +354,31 @@ Bug Fixes in This Version
   (`#61885 <https://github.com/llvm/llvm-project/issues/61885>`_)
 - Clang constexpr evaluator now treats comparison of [[gnu::weak]]-attributed
   member pointer as an invalid expression.
+- Fix crash when member function contains invalid default argument.
+  (`#62122 <https://github.com/llvm/llvm-project/issues/62122>`_)
+- Fix crash when handling undefined template partial specialization
+  (`#61356 <https://github.com/llvm/llvm-project/issues/61356>`_)
+- Fix premature substitution into the constraints of an inherited constructor.
+- Fix crash when attempting to perform parenthesized initialization of an
+  aggregate with a base class with only non-public constructors.
+  (`#62296 <https://github.com/llvm/llvm-project/issues/62296>`_)
+- Fix a stack overflow issue when evaluating ``consteval`` default arguments.
+  (`#60082` <https://github.com/llvm/llvm-project/issues/60082>`_)
+- Fix the assertion hit when generating code for global variable initializer of
+  _BitInt(1) type.
+  (`#62207 <https://github.com/llvm/llvm-project/issues/62207>`_)
+- Fix lambdas and other anonymous function names not respecting ``-fdebug-prefix-map``
+  (`#62192 <https://github.com/llvm/llvm-project/issues/62192>`_)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Bug Fixes to Attribute Support
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- Fixed a bug where attribute annotations on type specifiers (enums, classes,
+  structs, unions, and scoped enums) were not properly ignored, resulting in
+  misleading warning messages. Now, such attribute annotations are correctly
+  ignored. (`#61660 <https://github.com/llvm/llvm-project/issues/61660>`_)
 
 Bug Fixes to C++ Support
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -342,7 +394,7 @@ Bug Fixes to C++ Support
   (`#58674 <https://github.com/llvm/llvm-project/issues/58674>`_)
 - Fix incorrect deletion of the default constructor of unions in some
   cases. (`#48416 <https://github.com/llvm/llvm-project/issues/48416>`_)
-- No longer issue a pre-C++2b compatibility warning in ``-pedantic`` mode
+- No longer issue a pre-C++23 compatibility warning in ``-pedantic`` mode
   regading overloaded `operator[]` with more than one parmeter or for static
   lambdas. (`#61582 <https://github.com/llvm/llvm-project/issues/61582>`_)
 - Stop stripping CV qualifiers from the type of ``this`` when capturing it by value in
@@ -355,6 +407,13 @@ Bug Fixes to C++ Support
 - Fix bug in the computation of the ``__has_unique_object_representations``
   builtin for types with unnamed bitfields.
   (`#61336 <https://github.com/llvm/llvm-project/issues/61336>`_)
+- Fix default member initializers sometimes being ignored when performing
+  parenthesized aggregate initialization of templated types.
+  (`#62266 <https://github.com/llvm/llvm-project/issues/62266>`_)
+- Fix overly aggressive lifetime checks for parenthesized aggregate
+  initialization.
+  (`#61567 <https://github.com/llvm/llvm-project/issues/61567>`_)
+- Fix a crash when expanding a pack as the index of a subscript expression.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -402,6 +461,18 @@ Arm and AArch64 Support
 - Clang builtin ``__arithmetic_fence`` and the command line option ``-fprotect-parens``
   are now enabled for AArch64.
 
+- Clang supports flag output operands by which conditions in the NZCV could be outputs
+  of inline assembly for AArch64. This change is more consistent with the behavior of
+  GCC.
+
+   .. code-block:: c
+
+     // int a = foo(); int* b = bar();
+     asm("ands %w[a], %w[a], #3" : [a] "+r"(a), "=@cceq"(*b));
+
+- Fix a crash when ``preserve_all`` calling convention is used on AArch64.
+  `Issue 58145 <https://github.com/llvm/llvm-project/issues/58145>`_
+
 Windows Support
 ^^^^^^^^^^^^^^^
 
@@ -420,6 +491,11 @@ RISC-V Support
 - Fixed incorrect ABI lowering of ``_Float16`` in the case of structs
   containing ``_Float16`` that are eligible for passing via GPR+FPR or
   FPR+FPR.
+- Removed support for ``__attribute__((interrupt("user")))``. User-level
+  interrupts are not in version 1.12 of the privileged specification.
+- Added ``attribute(riscv_rvv_vector_bits(__riscv_v_fixed_vlen))`` to allow
+  the size of a RVV (RISC-V Vector) scalable type to be specified. This allows
+  RVV scalable vector types to be used in structs or in global variables.
 
 CUDA/HIP Language Changes
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -475,6 +551,8 @@ clang-format
 - Add additional Qualifier Ordering support for special cases such
   as templates, requires clauses, long qualified names.
 - Fix all known issues associated with ``LambdaBodyIndentation: OuterScope``.
+- Add ``BracedInitializerIndentWidth`` which can be used to configure
+  the indentation level of the contents of braced init lists.
 
 libclang
 --------
diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index 0b44265374a8b..a59a3edbbbac2 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -434,7 +434,7 @@ The following example is not allowed:
 .. code-block:: console
 
   $ clang++ -std=c++20 M.cppm --precompile -o M.pcm
-  $ clang++ -std=c++2b Use.cpp -fprebuilt-module-path=.
+  $ clang++ -std=c++23 Use.cpp -fprebuilt-module-path=.
 
 The compiler would reject the example due to the inconsistent language options.
 Not all options are language options.
@@ -611,7 +611,7 @@ The following describes issues in the current implementation of modules.
 Please see https://github.com/llvm/llvm-project/labels/clang%3Amodules for more issues
 or file a new issue if you don't find an existing one.
 If you're going to create a new issue for standard C++ modules,
-please start the title with ``[C++20] [Modules]`` (or ``[C++2b] [Modules]``, etc)
+please start the title with ``[C++20] [Modules]`` (or ``[C++23] [Modules]``, etc)
 and add the label ``clang:modules`` (if you have permissions for that).
 
 For higher level support for proposals, you could visit https://clang.llvm.org/cxx_status.html.
diff --git a/clang/docs/doxygen.cfg.in b/clang/docs/doxygen.cfg.in
index 39a346409b935..251afb179b205 100644
--- a/clang/docs/doxygen.cfg.in
+++ b/clang/docs/doxygen.cfg.in
@@ -2090,7 +2090,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2135,7 +2135,7 @@ TEMPLATE_RELATIONS     = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2144,7 +2144,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/clang/docs/tools/dump_format_style.py b/clang/docs/tools/dump_format_style.py
index fe6e9147ee94f..7b032894ce62c 100755
--- a/clang/docs/tools/dump_format_style.py
+++ b/clang/docs/tools/dump_format_style.py
@@ -69,9 +69,13 @@ def to_yaml_type(typestr: str):
   elif typestr == 'std::string':
     return 'String'
 
-  subtype, napplied = re.subn(r'^std::vector<(.*)>$', r'\1', typestr)
-  if napplied == 1:
-    return 'List of ' + pluralize(to_yaml_type(subtype))
+  match = re.match(r'std::vector<(.*)>$', typestr)
+  if match:
+    return 'List of ' + pluralize(to_yaml_type(match.group(1)))
+
+  match = re.match(r'std::optional<(.*)>$', typestr)
+  if match:
+    return to_yaml_type(match.group(1))
 
   return typestr
 
@@ -331,7 +335,8 @@ class State:
       if option.type not in ['bool', 'unsigned', 'int', 'std::string',
                              'std::vector<std::string>',
                              'std::vector<IncludeCategory>',
-                             'std::vector<RawStringFormat>']:
+                             'std::vector<RawStringFormat>',
+                             'std::optional<unsigned>']:
         if option.type in enums:
           option.enum = enums[option.type]
         elif option.type in nested_structs:
diff --git a/clang/include/clang-c/module.modulemap b/clang/include/clang-c/module.modulemap
deleted file mode 100644
index 95a59d62344cc..0000000000000
--- a/clang/include/clang-c/module.modulemap
+++ /dev/null
@@ -1,4 +0,0 @@
-module Clang_C {
-  umbrella "."
-  module * { export * }
-}
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 54b41d248a341..b08eb525602b6 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -2253,6 +2253,17 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// false otherwise.
   bool areLaxCompatibleSveTypes(QualType FirstType, QualType SecondType);
 
+  /// Return true if the given types are an RISC-V vector builtin type and a
+  /// VectorType that is a fixed-length representation of the RISC-V vector
+  /// builtin type for a specific vector-length.
+  bool areCompatibleRVVTypes(QualType FirstType, QualType SecondType);
+
+  /// Return true if the given vector types are lax-compatible RISC-V vector
+  /// types as defined by -flax-vector-conversions=, which permits implicit
+  /// conversions between vectors with different number of elements and/or
+  /// incompatible element types, false otherwise.
+  bool areLaxCompatibleRVVTypes(QualType FirstType, QualType SecondType);
+
   /// Return true if the type has been explicitly qualified with ObjC ownership.
   /// A type may be implicitly qualified with ownership under ObjC ARC, and in
   /// some cases the compiler treats these differently.
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index d9b17f8eaf936..0ab778e5d8cd3 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -2233,14 +2233,14 @@ class UnaryOperator final
   bool canOverflow() const { return UnaryOperatorBits.CanOverflow; }
   void setCanOverflow(bool C) { UnaryOperatorBits.CanOverflow = C; }
 
-  // Get the FP contractability status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FP contractability status of this operator. Only meaningful for
+  /// operations on floating point types.
   bool isFPContractableWithinStatement(const LangOptions &LO) const {
     return getFPFeaturesInEffect(LO).allowFPContractWithinStatement();
   }
 
-  // Get the FENV_ACCESS status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FENV_ACCESS status of this operator. Only meaningful for
+  /// operations on floating point types.
   bool isFEnvAccessOn(const LangOptions &LO) const {
     return getFPFeaturesInEffect(LO).getAllowFEnvAccess();
   }
@@ -2325,8 +2325,8 @@ class UnaryOperator final
   void setStoredFPFeatures(FPOptionsOverride F) { getTrailingFPFeatures() = F; }
 
 public:
-  // Get the FP features status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FP features status of this operator. Only meaningful for
+  /// operations on floating point types.
   FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
     if (UnaryOperatorBits.HasFPFeatures)
       return getStoredFPFeatures().applyOverrides(LO);
@@ -3082,8 +3082,8 @@ class CallExpr : public Expr {
     *getTrailingFPFeatures() = F;
   }
 
-  // Get the FP features status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FP features status of this operator. Only meaningful for
+  /// operations on floating point types.
   FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
     if (hasStoredFPFeatures())
       return getStoredFPFeatures().applyOverrides(LO);
@@ -3573,8 +3573,8 @@ class CastExpr : public Expr {
     return *getTrailingFPFeatures();
   }
 
-  // Get the FP features status of this operation. Only meaningful for
-  // operations on floating point types.
+  /// Get the FP features status of this operation. Only meaningful for
+  /// operations on floating point types.
   FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
     if (hasStoredFPFeatures())
       return getStoredFPFeatures().applyOverrides(LO);
@@ -3971,9 +3971,9 @@ class BinaryOperator : public Expr {
     return isShiftAssignOp(getOpcode());
   }
 
-  // Return true if a binary operator using the specified opcode and operands
-  // would match the 'p = (i8*)nullptr + n' idiom for casting a pointer-sized
-  // integer to a pointer.
+  /// Return true if a binary operator using the specified opcode and operands
+  /// would match the 'p = (i8*)nullptr + n' idiom for casting a pointer-sized
+  /// integer to a pointer.
   static bool isNullPointerArithmeticExtension(ASTContext &Ctx, Opcode Opc,
                                                const Expr *LHS,
                                                const Expr *RHS);
@@ -4007,8 +4007,8 @@ class BinaryOperator : public Expr {
     *getTrailingFPFeatures() = F;
   }
 
-  // Get the FP features status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FP features status of this operator. Only meaningful for
+  /// operations on floating point types.
   FPOptions getFPFeaturesInEffect(const LangOptions &LO) const {
     if (BinaryOperatorBits.HasFPFeatures)
       return getStoredFPFeatures().applyOverrides(LO);
@@ -4022,14 +4022,14 @@ class BinaryOperator : public Expr {
     return FPOptionsOverride();
   }
 
-  // Get the FP contractability status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FP contractability status of this operator. Only meaningful for
+  /// operations on floating point types.
   bool isFPContractableWithinStatement(const LangOptions &LO) const {
     return getFPFeaturesInEffect(LO).allowFPContractWithinStatement();
   }
 
-  // Get the FENV_ACCESS status of this operator. Only meaningful for
-  // operations on floating point types.
+  /// Get the FENV_ACCESS status of this operator. Only meaningful for
+  /// operations on floating point types.
   bool isFEnvAccessOn(const LangOptions &LO) const {
     return getFPFeaturesInEffect(LO).getAllowFEnvAccess();
   }
@@ -4125,17 +4125,17 @@ class AbstractConditionalOperator : public Expr {
     : Expr(SC, Empty) { }
 
 public:
-  // getCond - Return the expression representing the condition for
-  //   the ?: operator.
+  /// getCond - Return the expression representing the condition for
+  ///   the ?: operator.
   Expr *getCond() const;
 
-  // getTrueExpr - Return the subexpression representing the value of
-  //   the expression if the condition evaluates to true.
+  /// getTrueExpr - Return the subexpression representing the value of
+  ///   the expression if the condition evaluates to true.
   Expr *getTrueExpr() const;
 
-  // getFalseExpr - Return the subexpression representing the value of
-  //   the expression if the condition evaluates to false.  This is
-  //   the same as getRHS.
+  /// getFalseExpr - Return the subexpression representing the value of
+  ///   the expression if the condition evaluates to false.  This is
+  ///   the same as getRHS.
   Expr *getFalseExpr() const;
 
   SourceLocation getQuestionLoc() const { return QuestionLoc; }
@@ -4170,17 +4170,17 @@ class ConditionalOperator : public AbstractConditionalOperator {
   explicit ConditionalOperator(EmptyShell Empty)
     : AbstractConditionalOperator(ConditionalOperatorClass, Empty) { }
 
-  // getCond - Return the expression representing the condition for
-  //   the ?: operator.
+  /// getCond - Return the expression representing the condition for
+  ///   the ?: operator.
   Expr *getCond() const { return cast<Expr>(SubExprs[COND]); }
 
-  // getTrueExpr - Return the subexpression representing the value of
-  //   the expression if the condition evaluates to true.
+  /// getTrueExpr - Return the subexpression representing the value of
+  ///   the expression if the condition evaluates to true.
   Expr *getTrueExpr() const { return cast<Expr>(SubExprs[LHS]); }
 
-  // getFalseExpr - Return the subexpression representing the value of
-  //   the expression if the condition evaluates to false.  This is
-  //   the same as getRHS.
+  /// getFalseExpr - Return the subexpression representing the value of
+  ///   the expression if the condition evaluates to false.  This is
+  ///   the same as getRHS.
   Expr *getFalseExpr() const { return cast<Expr>(SubExprs[RHS]); }
 
   Expr *getLHS() const { return cast<Expr>(SubExprs[LHS]); }
@@ -4903,7 +4903,7 @@ class InitListExpr : public Expr {
   /// has been set.
   bool hasArrayFiller() const { return getArrayFiller(); }
 
-  // Determine whether this initializer list contains a designated initializer.
+  /// Determine whether this initializer list contains a designated initializer.
   bool hasDesignatedInit() const {
     return std::any_of(begin(), end(), [](const Stmt *S) {
       return isa<DesignatedInitExpr>(S);
@@ -4938,8 +4938,8 @@ class InitListExpr : public Expr {
     return LBraceLoc.isValid() && RBraceLoc.isValid();
   }
 
-  // Is this an initializer for an array of characters, initialized by a string
-  // literal or an @encode?
+  /// Is this an initializer for an array of characters, initialized by a string
+  /// literal or an @encode?
   bool isStringLiteralInit() const;
 
   /// Is this a transparent initializer list (that is, an InitListExpr that is
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 724904b4d2041..16f54d753d42f 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -2321,7 +2321,7 @@ class CXXNewExpr final
 
   /// This might return std::nullopt even if isArray() returns true,
   /// since there might not be an array size expression.
-  /// If the result is not-None, it will never wrap a nullptr.
+  /// If the result is not std::nullopt, it will never wrap a nullptr.
   std::optional<Expr *> getArraySize() {
     if (!isArray())
       return std::nullopt;
@@ -2335,7 +2335,7 @@ class CXXNewExpr final
 
   /// This might return std::nullopt even if isArray() returns true,
   /// since there might not be an array size expression.
-  /// If the result is not-None, it will never wrap a nullptr.
+  /// If the result is not std::nullopt, it will never wrap a nullptr.
   std::optional<const Expr *> getArraySize() const {
     if (!isArray())
       return std::nullopt;
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index af0e724311017..7ce1ad9b90998 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1770,7 +1770,7 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
 
     /// The kind of vector, either a generic vector type or some
     /// target-specific vector type such as for AltiVec or Neon.
-    unsigned VecKind : 3;
+    unsigned VecKind : 4;
     /// The number of elements in the vector.
     uint32_t NumElements;
   };
@@ -2049,6 +2049,16 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
   /// 'arm_sve_vector_bits' type attribute as VectorType.
   QualType getSveEltType(const ASTContext &Ctx) const;
 
+  /// Determines if this is a sizeless type supported by the
+  /// 'riscv_rvv_vector_bits' type attribute, which can be applied to a single
+  /// RVV vector or mask.
+  bool isRVVVLSBuiltinType() const;
+
+  /// Returns the representative type for the element of an RVV builtin type.
+  /// This is used to represent fixed-length RVV vectors created with the
+  /// 'riscv_rvv_vector_bits' type attribute as VectorType.
+  QualType getRVVEltType(const ASTContext &Ctx) const;
+
   /// Types are partitioned into 3 broad categories (C99 6.2.5p1):
   /// object types, function types, and incomplete types.
 
@@ -3402,7 +3412,10 @@ class VectorType : public Type, public llvm::FoldingSetNode {
     SveFixedLengthDataVector,
 
     /// is AArch64 SVE fixed-length predicate vector
-    SveFixedLengthPredicateVector
+    SveFixedLengthPredicateVector,
+
+    /// is RISC-V RVV fixed-length data vector
+    RVVFixedLengthDataVector,
   };
 
 protected:
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index 8690616411db9..cd1703be0507b 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -27,6 +27,7 @@
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <memory>
 #include <type_traits>
@@ -178,12 +179,16 @@ class Environment {
   /// with a symbolic representation of the `this` pointee.
   Environment(DataflowAnalysisContext &DACtx, const DeclContext &DeclCtx);
 
+  LLVM_DEPRECATED("Use getDataflowAnalysisContext().getOptions() instead.", "")
   const DataflowAnalysisContext::Options &getAnalysisOptions() const {
     return DACtx->getOptions();
   }
 
+  LLVM_DEPRECATED("Use getDataflowAnalysisContext().arena() instead.", "")
   Arena &arena() const { return DACtx->arena(); }
 
+  LLVM_DEPRECATED("Use getDataflowAnalysisContext().getOptions().Log instead.",
+                  "")
   Logger &logger() const { return *DACtx->getOptions().Log; }
 
   /// Creates and returns an environment to use for an inline analysis  of the
@@ -264,13 +269,24 @@ class Environment {
   ///
   /// Requirements:
   ///
-  ///  `D` must not be assigned a storage location in the environment.
+  ///  `D` must not already have a storage location in the environment.
+  ///
+  ///  If `D` has reference type, `Loc` must refer directly to the referenced
+  ///  object (if any), not to a `ReferenceValue`, and it is not permitted to
+  ///  later change `Loc` to refer to a `ReferenceValue.`
   void setStorageLocation(const ValueDecl &D, StorageLocation &Loc);
 
-  /// Returns the storage location assigned to `D` in the environment, applying
-  /// the `SP` policy for skipping past indirections, or null if `D` isn't
-  /// assigned a storage location in the environment.
-  StorageLocation *getStorageLocation(const ValueDecl &D, SkipPast SP) const;
+  /// Returns the storage location assigned to `D` in the environment, or null
+  /// if `D` isn't assigned a storage location in the environment.
+  ///
+  /// Note that if `D` has reference type, the storage location that is returned
+  /// refers directly to the referenced object, not a `ReferenceValue`.
+  ///
+  /// The `SP` parameter is deprecated and has no effect. In addition, it is
+  /// not permitted to pass `SkipPast::ReferenceThenPointer` for this parameter.
+  /// New uses of this function should use the default argument for `SP`.
+  StorageLocation *getStorageLocation(const ValueDecl &D,
+                                      SkipPast SP = SkipPast::None) const;
 
   /// Assigns `Loc` as the storage location of `E` in the environment.
   ///
@@ -315,7 +331,11 @@ class Environment {
 
   /// Equivalent to `getValue(getStorageLocation(D, SP), SkipPast::None)` if `D`
   /// is assigned a storage location in the environment, otherwise returns null.
-  Value *getValue(const ValueDecl &D, SkipPast SP) const;
+  ///
+  /// The `SP` parameter is deprecated and has no effect. In addition, it is
+  /// not permitted to pass `SkipPast::ReferenceThenPointer` for this parameter.
+  /// New uses of this function should use the default argument for `SP`.
+  Value *getValue(const ValueDecl &D, SkipPast SP = SkipPast::None) const;
 
   /// Equivalent to `getValue(getStorageLocation(E, SP), SkipPast::None)` if `E`
   /// is assigned a storage location in the environment, otherwise returns null.
@@ -331,23 +351,23 @@ class Environment {
   template <typename T, typename... Args>
   std::enable_if_t<std::is_base_of<Value, T>::value, T &>
   create(Args &&...args) {
-    return arena().create<T>(std::forward<Args>(args)...);
+    return DACtx->arena().create<T>(std::forward<Args>(args)...);
   }
 
   /// Returns a symbolic boolean value that models a boolean literal equal to
   /// `Value`
   AtomicBoolValue &getBoolLiteralValue(bool Value) const {
-    return arena().makeLiteral(Value);
+    return DACtx->arena().makeLiteral(Value);
   }
 
   /// Returns an atomic boolean value.
   BoolValue &makeAtomicBoolValue() const {
-    return arena().create<AtomicBoolValue>();
+    return DACtx->arena().create<AtomicBoolValue>();
   }
 
   /// Returns a unique instance of boolean Top.
   BoolValue &makeTopBoolValue() const {
-    return arena().create<TopBoolValue>();
+    return DACtx->arena().create<TopBoolValue>();
   }
 
   /// Returns a boolean value that represents the conjunction of `LHS` and
@@ -355,7 +375,7 @@ class Environment {
   /// order, will return the same result. If the given boolean values represent
   /// the same value, the result will be the value itself.
   BoolValue &makeAnd(BoolValue &LHS, BoolValue &RHS) const {
-    return arena().makeAnd(LHS, RHS);
+    return DACtx->arena().makeAnd(LHS, RHS);
   }
 
   /// Returns a boolean value that represents the disjunction of `LHS` and
@@ -363,13 +383,13 @@ class Environment {
   /// order, will return the same result. If the given boolean values represent
   /// the same value, the result will be the value itself.
   BoolValue &makeOr(BoolValue &LHS, BoolValue &RHS) const {
-    return arena().makeOr(LHS, RHS);
+    return DACtx->arena().makeOr(LHS, RHS);
   }
 
   /// Returns a boolean value that represents the negation of `Val`. Subsequent
   /// calls with the same argument will return the same result.
   BoolValue &makeNot(BoolValue &Val) const {
-    return arena().makeNot(Val);
+    return DACtx->arena().makeNot(Val);
   }
 
   /// Returns a boolean value represents `LHS` => `RHS`. Subsequent calls with
@@ -377,7 +397,7 @@ class Environment {
   /// values represent the same value, the result will be a value that
   /// represents the true boolean literal.
   BoolValue &makeImplication(BoolValue &LHS, BoolValue &RHS) const {
-    return arena().makeImplies(LHS, RHS);
+    return DACtx->arena().makeImplies(LHS, RHS);
   }
 
   /// Returns a boolean value represents `LHS` <=> `RHS`. Subsequent calls with
@@ -385,7 +405,7 @@ class Environment {
   /// result. If the given boolean values represent the same value, the result
   /// will be a value that represents the true boolean literal.
   BoolValue &makeIff(BoolValue &LHS, BoolValue &RHS) const {
-    return arena().makeEquals(LHS, RHS);
+    return DACtx->arena().makeEquals(LHS, RHS);
   }
 
   /// Returns the token that identifies the flow condition of the environment.
@@ -409,10 +429,15 @@ class Environment {
 
   /// Returns the `ControlFlowContext` registered for `F`, if any. Otherwise,
   /// returns null.
+  LLVM_DEPRECATED(
+      "Use getDataflowAnalysisContext().getControlFlowContext(F) instead.", "")
   const ControlFlowContext *getControlFlowContext(const FunctionDecl *F) {
     return DACtx->getControlFlowContext(F);
   }
 
+  /// Returns the `DataflowAnalysisContext` used by the environment.
+  DataflowAnalysisContext &getDataflowAnalysisContext() const { return *DACtx; }
+
   LLVM_DUMP_METHOD void dump() const;
   LLVM_DUMP_METHOD void dump(raw_ostream &OS) const;
 
diff --git a/clang/include/clang/Analysis/FlowSensitive/Value.h b/clang/include/clang/Analysis/FlowSensitive/Value.h
index 32d10a3489483..861a9963e6689 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Value.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Value.h
@@ -73,6 +73,11 @@ class Value {
     Properties.insert_or_assign(Name, &Val);
   }
 
+  llvm::iterator_range<llvm::StringMap<Value *>::const_iterator>
+  properties() const {
+    return {Properties.begin(), Properties.end()};
+  }
+
 private:
   Kind ValKind;
   llvm::StringMap<Value *> Properties;
@@ -307,6 +312,12 @@ class StructValue final : public Value {
   /// Assigns `Val` as the child value for `D`.
   void setChild(const ValueDecl &D, Value &Val) { Children[&D] = &Val; }
 
+  llvm::iterator_range<
+      llvm::DenseMap<const ValueDecl *, Value *>::const_iterator>
+  children() const {
+    return {Children.begin(), Children.end()};
+  }
+
 private:
   llvm::DenseMap<const ValueDecl *, Value *> Children;
 };
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index a23dd5ce3634b..8507e65137697 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1864,13 +1864,23 @@ def RISCVInterrupt : InheritableAttr, TargetSpecificAttr<TargetRISCV> {
   let Spellings = [GCC<"interrupt">];
   let Subjects = SubjectList<[Function]>;
   let Args = [EnumArgument<"Interrupt", "InterruptType",
-                           ["user", "supervisor", "machine"],
-                           ["user", "supervisor", "machine"],
+                           ["supervisor", "machine"],
+                           ["supervisor", "machine"],
                            1>];
   let ParseKind = "Interrupt";
   let Documentation = [RISCVInterruptDocs];
 }
 
+def RISCVRVVVectorBits : TypeAttr {
+  let Spellings = [GNU<"riscv_rvv_vector_bits">];
+  let Subjects = SubjectList<[TypedefName], ErrorDiag>;
+  let Args = [UnsignedArgument<"NumBits">];
+  let Documentation = [RISCVRVVVectorBitsDocs];
+  let PragmaAttributeSupport = 0;
+  // Represented as VectorType instead.
+  let ASTNode = 0;
+}
+
 // This is not a TargetSpecificAttr so that is silently accepted and
 // ignored on other targets as encouraged by the OpenCL spec.
 //
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index dae12624a822b..f62350f33013c 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -2317,6 +2317,37 @@ Version 1.10.
   }];
 }
 
+def RISCVRVVVectorBitsDocs : Documentation {
+  let Category = DocCatType;
+  let Content = [{
+On RISC-V targets, the ``riscv_rvv_vector_bits(N)`` attribute is used to define
+fixed-length variants of sizeless types.
+
+For example:
+
+.. code-block:: c
+
+  #include <riscv_vector.h>
+
+  #if defined(__riscv_v_fixed_vlen)
+  typedef vint8m1_t fixed_vint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+  #endif
+
+Creates a type ``fixed_vint8m1_t_t`` that is a fixed-length variant of
+``vint8m1_t`` that contains exactly 512 bits. Unlike ``vint8m1_t``, this type
+can be used in globals, structs, unions, and arrays, all of which are
+unsupported for sizeless types.
+
+The attribute can be attached to a single RVV vector (such as ``vint8m1_t``).
+The attribute will be rejected unless
+``N==__riscv_v_fixed_vlen``, the implementation defined feature macro that
+is enabled under the ``-mrvv-vector-bits`` flag. ``__riscv_v_fixed_vlen`` can
+only be a power of 2 between 64 and 65536.
+
+Only ``*m1_t`` (LMUL=1) types are supported at this time.
+}];
+}
+
 def AVRInterruptDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "interrupt (AVR)";
@@ -5182,6 +5213,9 @@ apply for values returned in callee-saved registers.
   R11. R11 can be used as a scratch register. Floating-point registers
   (XMMs/YMMs) are not preserved and need to be saved by the caller.
 
+- On AArch64 the callee preserve all general purpose registers, except X0-X8 and
+  X16-X18.
+
 The idea behind this convention is to support calls to runtime functions
 that have a hot path and a cold path. The hot path is usually a small piece
 of code that doesn't use many registers. The cold path might need to call out to
@@ -5222,6 +5256,10 @@ returned in callee-saved registers.
   R11. R11 can be used as a scratch register. Furthermore it also preserves
   all floating-point registers (XMMs/YMMs).
 
+- On AArch64 the callee preserve all general purpose registers, except X0-X8 and
+  X16-X18. Furthermore it also preserves lower 128 bits of V8-V31 SIMD - floating
+  point registers.
+
 The idea behind this convention is to support calls to runtime functions
 that don't need to call out to any other functions.
 
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.def b/clang/include/clang/Basic/BuiltinsRISCV.def
index 3ca7654a32adc..370ef0af8f9a5 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.def
+++ b/clang/include/clang/Basic/BuiltinsRISCV.def
@@ -79,5 +79,9 @@ TARGET_BUILTIN(__builtin_riscv_sm4ks, "LiLiLiIUc", "nc", "zksed")
 TARGET_BUILTIN(__builtin_riscv_sm3p0, "LiLi", "nc", "zksh")
 TARGET_BUILTIN(__builtin_riscv_sm3p1, "LiLi", "nc", "zksh")
 
+// Zihintntl extension
+TARGET_BUILTIN(__builtin_riscv_ntl_load, "v.", "t", "experimental-zihintntl")
+TARGET_BUILTIN(__builtin_riscv_ntl_store, "v.", "t", "experimental-zihintntl")
+
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsRISCVVector.def b/clang/include/clang/Basic/BuiltinsRISCVVector.def
index 008cb939a30bc..6dfa87a1a1d31 100644
--- a/clang/include/clang/Basic/BuiltinsRISCVVector.def
+++ b/clang/include/clang/Basic/BuiltinsRISCVVector.def
@@ -16,6 +16,7 @@
 #endif
 
 #include "clang/Basic/riscv_vector_builtins.inc"
+#include "clang/Basic/riscv_sifive_vector_builtins.inc"
 
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt
index b930842ae8cfd..53a713b13ea39 100644
--- a/clang/include/clang/Basic/CMakeLists.txt
+++ b/clang/include/clang/Basic/CMakeLists.txt
@@ -93,3 +93,12 @@ clang_tablegen(riscv_vector_builtin_cg.inc -gen-riscv-vector-builtin-codegen
 clang_tablegen(riscv_vector_builtin_sema.inc -gen-riscv-vector-builtin-sema
   SOURCE riscv_vector.td
   TARGET ClangRISCVVectorBuiltinSema)
+clang_tablegen(riscv_sifive_vector_builtins.inc -gen-riscv-sifive-vector-builtins
+  SOURCE riscv_sifive_vector.td
+  TARGET ClangRISCVSiFiveVectorBuiltins)
+clang_tablegen(riscv_sifive_vector_builtin_cg.inc -gen-riscv-sifive-vector-builtin-codegen
+  SOURCE riscv_sifive_vector.td
+  TARGET ClangRISCVSiFiveVectorBuiltinCG)
+clang_tablegen(riscv_sifive_vector_builtin_sema.inc -gen-riscv-sifive-vector-builtin-sema
+  SOURCE riscv_sifive_vector.td
+  TARGET ClangRISCVSiFiveVectorBuiltinSema)
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index e1817b990cdf9..b0f22411e1ad2 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -206,8 +206,11 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// if non-empty.
   std::string RecordCommandLine;
 
-  std::map<std::string, std::string> DebugPrefixMap;
-  std::map<std::string, std::string> CoveragePrefixMap;
+  llvm::SmallVector<std::pair<std::string, std::string>, 0> DebugPrefixMap;
+
+  /// Prefix replacement map for source-based code coverage to remap source
+  /// file paths in coverage mapping.
+  llvm::SmallVector<std::pair<std::string, std::string>, 0> CoveragePrefixMap;
 
   /// The ABI to use for passing floating point arguments.
   std::string FloatABI;
@@ -368,9 +371,6 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// transformation.
   OptRemark OptimizationRemarkAnalysis;
 
-  /// Set of files defining the rules for the symbol rewriting.
-  std::vector<std::string> RewriteMapFiles;
-
   /// Set of sanitizer checks that are non-fatal (i.e. execution should be
   /// continued when possible).
   SanitizerSet SanitizeRecover;
diff --git a/clang/include/clang/Basic/Diagnostic.td b/clang/include/clang/Basic/Diagnostic.td
index 503b833e63bbf..8d66e265fbaef 100644
--- a/clang/include/clang/Basic/Diagnostic.td
+++ b/clang/include/clang/Basic/Diagnostic.td
@@ -157,7 +157,6 @@ class DefaultRemark { Severity DefaultSeverity = SEV_Remark; }
 
 // Definitions for Diagnostics.
 include "DiagnosticASTKinds.td"
-include "DiagnosticAnalysisKinds.td"
 include "DiagnosticCommentKinds.td"
 include "DiagnosticCommonKinds.td"
 include "DiagnosticCrossTUKinds.td"
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index bac77299671c5..42d8a0abfbceb 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -210,14 +210,14 @@ def ext_cxx11_longlong : Extension<
 def warn_cxx98_compat_longlong : Warning<
   "'long long' is incompatible with C++98">,
   InGroup<CXX98CompatPedantic>, DefaultIgnore;
-def ext_cxx2b_size_t_suffix : ExtWarn<
-  "'size_t' suffix for literals is a C++2b extension">,
-  InGroup<CXX2b>;
+def ext_cxx23_size_t_suffix : ExtWarn<
+  "'size_t' suffix for literals is a C++23 extension">,
+  InGroup<CXX23>;
 def warn_cxx20_compat_size_t_suffix : Warning<
   "'size_t' suffix for literals is incompatible with C++ standards before "
-  "C++2b">, InGroup<CXXPre2bCompat>, DefaultIgnore;
-def err_cxx2b_size_t_suffix: Error<
-  "'size_t' suffix for literals is a C++2b feature">;
+  "C++23">, InGroup<CXXPre23Compat>, DefaultIgnore;
+def err_cxx23_size_t_suffix: Error<
+  "'size_t' suffix for literals is a C++23 feature">;
 def err_size_t_literal_too_large: Error<
   "%select{signed |}0'size_t' literal is out of range of possible "
   "%select{signed |}0'size_t' values">;
@@ -308,7 +308,7 @@ def warn_dup_category_def : Warning<
 // Targets
 
 def err_target_unknown_triple : Error<
-  "unknown target triple '%0', please use -triple or -arch">;
+  "unknown target triple '%0'">;
 def err_target_unknown_cpu : Error<"unknown target CPU '%0'">;
 def note_valid_options : Note<"valid target CPU values are: %0">;
 def err_target_unsupported_cpu_for_micromips : Error<
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 1efe7573028ef..4c7d9e80338ec 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -255,6 +255,7 @@ def warn_drv_unknown_argument_clang_cl : Warning<
 def warn_drv_unknown_argument_clang_cl_with_suggestion : Warning<
   "unknown argument ignored in clang-cl '%0'; did you mean '%1'?">,
   InGroup<UnknownArgument>;
+def err_drv_unknown_target_triple : Error<"unknown target triple '%0'">;
 
 def warn_drv_ycyu_different_arg_clang_cl : Warning<
   "support for '/Yc' and '/Yu' with different filenames not implemented yet; flags ignored">,
diff --git a/clang/include/clang/Basic/DiagnosticError.h b/clang/include/clang/Basic/DiagnosticError.h
index 3660bd1b3b3d8..744f7fe19db79 100644
--- a/clang/include/clang/Basic/DiagnosticError.h
+++ b/clang/include/clang/Basic/DiagnosticError.h
@@ -35,8 +35,8 @@ class DiagnosticError : public llvm::ErrorInfo<DiagnosticError> {
   }
 
   /// Extracts and returns the diagnostic payload from the given \c Error if
-  /// the error is a \c DiagnosticError. Returns none if the given error is not
-  /// a \c DiagnosticError.
+  /// the error is a \c DiagnosticError. Returns std::nullopt if the given error
+  /// is not a \c DiagnosticError.
   static std::optional<PartialDiagnosticAt> take(llvm::Error &Err) {
     std::optional<PartialDiagnosticAt> Result;
     Err = llvm::handleErrors(std::move(Err), [&](DiagnosticError &E) {
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 0ee43fb8837a1..87e72f000d494 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -304,9 +304,9 @@ def CXXPre20CompatPedantic : DiagGroup<"pre-c++20-compat-pedantic",
                                        [CXXPre20Compat]>;
 def : DiagGroup<"c++98-c++11-c++14-c++17-compat-pedantic",
                 [CXXPre20CompatPedantic]>;
-def CXXPre2bCompat : DiagGroup<"pre-c++2b-compat">;
-def CXXPre2bCompatPedantic :
-  DiagGroup<"pre-c++2b-compat-pedantic", [CXXPre2bCompat]>;
+def CXXPre23Compat : DiagGroup<"pre-c++23-compat">;
+def CXXPre23CompatPedantic :
+  DiagGroup<"pre-c++23-compat-pedantic", [CXXPre23Compat]>;
 
 def CXX98CompatBindToTemporaryCopy :
   DiagGroup<"c++98-compat-bind-to-temporary-copy">;
@@ -321,7 +321,7 @@ def CXX98Compat : DiagGroup<"c++98-compat",
                              CXXPre14Compat,
                              CXXPre17Compat,
                              CXXPre20Compat,
-                             CXXPre2bCompat]>;
+                             CXXPre23Compat]>;
 // Warnings for C++11 features which are Extensions in C++98 mode.
 def CXX98CompatPedantic : DiagGroup<"c++98-compat-pedantic",
                                     [CXX98Compat,
@@ -330,7 +330,7 @@ def CXX98CompatPedantic : DiagGroup<"c++98-compat-pedantic",
                                      CXXPre14CompatPedantic,
                                      CXXPre17CompatPedantic,
                                      CXXPre20CompatPedantic,
-                                     CXXPre2bCompatPedantic]>;
+                                     CXXPre23CompatPedantic]>;
 
 def CXX11Narrowing : DiagGroup<"c++11-narrowing">;
 
@@ -360,39 +360,39 @@ def CXX11Compat : DiagGroup<"c++11-compat",
                              CXXPre14Compat,
                              CXXPre17Compat,
                              CXXPre20Compat,
-                             CXXPre2bCompat]>;
+                             CXXPre23Compat]>;
 def : DiagGroup<"c++0x-compat", [CXX11Compat]>;
 def CXX11CompatPedantic : DiagGroup<"c++11-compat-pedantic",
                                     [CXX11Compat,
                                      CXXPre14CompatPedantic,
                                      CXXPre17CompatPedantic,
                                      CXXPre20CompatPedantic,
-                                     CXXPre2bCompatPedantic]>;
+                                     CXXPre23CompatPedantic]>;
 
 def CXX14Compat : DiagGroup<"c++14-compat", [CXXPre17Compat,
                                              CXXPre20Compat,
-                                             CXXPre2bCompat]>;
+                                             CXXPre23Compat]>;
 def CXX14CompatPedantic : DiagGroup<"c++14-compat-pedantic",
                                     [CXX14Compat,
                                      CXXPre17CompatPedantic,
                                      CXXPre20CompatPedantic,
-                                     CXXPre2bCompatPedantic]>;
+                                     CXXPre23CompatPedantic]>;
 
 def CXX17Compat : DiagGroup<"c++17-compat", [DeprecatedRegister,
                                              DeprecatedIncrementBool,
                                              CXX17CompatMangling,
                                              CXXPre20Compat,
-                                             CXXPre2bCompat]>;
+                                             CXXPre23Compat]>;
 def CXX17CompatPedantic : DiagGroup<"c++17-compat-pedantic",
                                     [CXX17Compat,
                                      CXXPre20CompatPedantic,
-                                     CXXPre2bCompatPedantic]>;
+                                     CXXPre23CompatPedantic]>;
 def : DiagGroup<"c++1z-compat", [CXX17Compat]>;
 
-def CXX20Compat : DiagGroup<"c++20-compat", [CXXPre2bCompat]>;
+def CXX20Compat : DiagGroup<"c++20-compat", [CXXPre23Compat]>;
 def CXX20CompatPedantic : DiagGroup<"c++20-compat-pedantic",
                                     [CXX20Compat,
-                                     CXXPre2bCompatPedantic]>;
+                                     CXXPre23CompatPedantic]>;
 def : DiagGroup<"c++2a-compat", [CXX20Compat]>;
 def : DiagGroup<"c++2a-compat-pedantic", [CXX20CompatPedantic]>;
 
@@ -1108,9 +1108,9 @@ def CXX17 : DiagGroup<"c++17-extensions", [CXX17Attrs]>;
 // earlier C++ versions.
 def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>;
 
-// A warning group for warnings about using C++2b features as extensions in
+// A warning group for warnings about using C++23 features as extensions in
 // earlier C++ versions.
-def CXX2b : DiagGroup<"c++2b-extensions">;
+def CXX23 : DiagGroup<"c++23-extensions">;
 
 def : DiagGroup<"c++0x-extensions", [CXX11]>;
 def : DiagGroup<"c++1y-extensions", [CXX14]>;
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index b77c19b816d86..ef5b1d8be0fd5 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -138,13 +138,13 @@ def ext_mathematical_notation : ExtWarn<
 
 def ext_delimited_escape_sequence : Extension<
   "%select{delimited|named}0 escape sequences are a "
-  "%select{Clang|C++2b}1 extension">,
+  "%select{Clang|C++23}1 extension">,
   InGroup<DiagGroup<"delimited-escape-sequence-extension">>;
 
-def warn_cxx2b_delimited_escape_sequence : Warning<
+def warn_cxx23_delimited_escape_sequence : Warning<
   "%select{delimited|named}0 escape sequences are "
-  "incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 
 def err_delimited_escape_empty : Error<
   "delimited escape sequence cannot be empty">;
@@ -396,10 +396,10 @@ def ext_pp_include_next_directive : Extension<
   "#include_next is a language extension">, InGroup<GNUIncludeNext>;
 
 def ext_pp_warning_directive : Extension<
-  "#warning is a %select{C2x|C++2b}0 extension">;
-def warn_cxx2b_compat_warning_directive : Warning<
-  "#warning is incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "#warning is a %select{C2x|C++23}0 extension">;
+def warn_cxx23_compat_warning_directive : Warning<
+  "#warning is incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 def warn_c2x_compat_warning_directive : Warning<
   "#warning is incompatible with C standards before C2x">,
   InGroup<CPre2xCompat>, DefaultIgnore;
@@ -739,14 +739,14 @@ def ext_c2x_pp_directive : ExtWarn<
   "use of a '#%select{<BUG IF SEEN>|elifdef|elifndef}0' directive "
   "is a C2x extension">,
   InGroup<C2x>;
-def warn_cxx2b_compat_pp_directive : Warning<
+def warn_cxx23_compat_pp_directive : Warning<
   "use of a '#%select{<BUG IF SEEN>|elifdef|elifndef}0' directive "
-  "is incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
-def ext_cxx2b_pp_directive : ExtWarn<
+  "is incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
+def ext_cxx23_pp_directive : ExtWarn<
   "use of a '#%select{<BUG IF SEEN>|elifdef|elifndef}0' directive "
-  "is a C++2b extension">,
-  InGroup<CXX2b>;
+  "is a C++23 extension">,
+  InGroup<CXX23>;
 
 def err_pp_visibility_non_macro : Error<"no macro named %0">;
 
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index d8ad04728b32d..5d5048a7d2c04 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -300,14 +300,14 @@ def ext_c_label_end_of_compound_statement : ExtWarn<
   "label at end of compound statement is a C2x extension">,
    InGroup<C2x>;
 def ext_cxx_label_end_of_compound_statement : ExtWarn<
-  "label at end of compound statement is a C++2b extension">,
-   InGroup<CXX2b>;
+  "label at end of compound statement is a C++23 extension">,
+   InGroup<CXX23>;
 def warn_c2x_compat_label_end_of_compound_statement : Warning<
   "label at end of compound statement is incompatible with C standards before C2x">,
   InGroup<CPre2xCompat>, DefaultIgnore;
 def warn_cxx20_compat_label_end_of_compound_statement : Warning<
-  "label at end of compound statement is incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "label at end of compound statement is incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 def err_address_of_label_outside_fn : Error<
   "use of address-of-label extension outside of a function body">;
 def err_asm_operand_wide_string_literal : Error<
@@ -566,11 +566,11 @@ def err_expected_init_in_condition_lparen : Error<
 def err_extraneous_rparen_in_condition : Error<
   "extraneous ')' after condition, expected a statement">;
 def ext_alias_in_init_statement : ExtWarn<
-  "alias declaration in this context is a C++2b extension">,
-  InGroup<CXX2b>;
+  "alias declaration in this context is a C++23 extension">,
+  InGroup<CXX23>;
 def warn_cxx20_alias_in_init_statement  : Warning<
-  "alias declaration in this context is incompatible with C++ standards before C++2b">,
-  DefaultIgnore, InGroup<CXXPre2bCompat>;
+  "alias declaration in this context is incompatible with C++ standards before C++23">,
+  DefaultIgnore, InGroup<CXXPre23Compat>;
 def warn_dangling_else : Warning<
   "add explicit braces to avoid dangling else">,
   InGroup<DanglingElse>;
@@ -652,11 +652,11 @@ def warn_cxx14_compat_constexpr_if : Warning<
   "constexpr if is incompatible with C++ standards before C++17">,
   DefaultIgnore, InGroup<CXXPre17Compat>;
 def ext_consteval_if : ExtWarn<
-  "consteval if is a C++2b extension">,
-   InGroup<CXX2b>;
+  "consteval if is a C++23 extension">,
+   InGroup<CXX23>;
 def warn_cxx20_compat_consteval_if : Warning<
-  "consteval if is incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "consteval if is incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 
 def ext_init_statement : ExtWarn<
   "'%select{if|switch}0' initialization statements are a C++17 extension">,
@@ -1014,14 +1014,14 @@ def err_lambda_capture_multiple_ellipses : Error<
 def err_capture_default_first : Error<
   "capture default must be first">;
 def ext_decl_attrs_on_lambda : ExtWarn<
-  "an attribute specifier sequence in this position is a C++2b extension">,
-  InGroup<CXX2b>;
+  "an attribute specifier sequence in this position is a C++23 extension">,
+  InGroup<CXX23>;
 def ext_lambda_missing_parens : ExtWarn<
-  "lambda without a parameter clause is a C++2b extension">,
-  InGroup<CXX2b>;
+  "lambda without a parameter clause is a C++23 extension">,
+  InGroup<CXX23>;
 def warn_cxx20_compat_decl_attrs_on_lambda : Warning<
   "an attribute specifier sequence in this position is incompatible with C++ "
-  "standards before C++2b">, InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "standards before C++23">, InGroup<CXXPre23Compat>, DefaultIgnore;
 
 // C++17 lambda expressions
 def err_expected_star_this_capture : Error<
@@ -1044,12 +1044,12 @@ def warn_cxx17_compat_lambda_template_parameter_list: Warning<
 def err_lambda_template_parameter_list_empty : Error<
   "lambda template parameter list cannot be empty">;
 
-// C++2b static lambdas
+// C++23 static lambdas
 def err_static_lambda: ExtWarn<
-  "static lambdas are a C++2b extension">, InGroup<CXX2b>;
+  "static lambdas are a C++23 extension">, InGroup<CXX23>;
 def warn_cxx20_compat_static_lambda : Warning<
-  "static lambdas are incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "static lambdas are incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 def err_static_mutable_lambda : Error<
   "lambda cannot be both mutable and static">;
 def err_static_lambda_captures : Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 18a0154b00411..b496900fac06e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2126,7 +2126,8 @@ def err_init_conversion_failed : Error<
   "exception object|a member subobject|an array element|a new value|a value|a "
   "base class|a constructor delegation|a vector element|a block element|a "
   "block element|a complex element|a lambda capture|a compound literal "
-  "initializer|a related result|a parameter of CF audited function}0 "
+  "initializer|a related result|a parameter of CF audited function|a "
+  "structured binding|a member subobject}0 "
   "%diff{of type $ with an %select{rvalue|lvalue}2 of type $|"
   "with an %select{rvalue|lvalue}2 of incompatible type}1,3"
   "%select{|: different classes%diff{ ($ vs $)|}5,6"
@@ -2369,7 +2370,7 @@ def err_auto_expr_init_paren_braces : Error<
   "%select{parenthesized|nested}0 initializer list">;
 def warn_cxx20_compat_auto_expr : Warning<
   "'auto' as a functional-style cast is incompatible with C++ standards "
-  "before C++2b">, InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "before C++23">, InGroup<CXXPre23Compat>, DefaultIgnore;
 def err_auto_missing_trailing_return : Error<
   "'auto' return without trailing return type; deduced return types are a "
   "C++14 extension">;
@@ -2666,7 +2667,7 @@ def err_invalid_constexpr : Error<
 def err_invalid_constexpr_member : Error<"non-static data member cannot be "
   "constexpr%select{; did you intend to make it %select{const|static}0?|}1">;
 def err_constexpr_tag : Error<
-  "%select{class|struct|interface|union|enum}0 "
+  "%select{class|struct|interface|union|enum|enum class|enum struct}0 "
   "cannot be marked %sub{select_constexpr_spec_kind}1">;
 def err_constexpr_dtor : Error<
   "destructor cannot be declared %sub{select_constexpr_spec_kind}0">;
@@ -2726,13 +2727,13 @@ def warn_cxx17_compat_constexpr_body_invalid_stmt : Warning<
   "use of this statement in a constexpr %select{function|constructor}0 "
   "is incompatible with C++ standards before C++20">,
   InGroup<CXXPre20Compat>, DefaultIgnore;
-def ext_constexpr_body_invalid_stmt_cxx2b : ExtWarn<
+def ext_constexpr_body_invalid_stmt_cxx23 : ExtWarn<
   "use of this statement in a constexpr %select{function|constructor}0 "
-  "is a C++2b extension">, InGroup<CXX2b>;
+  "is a C++23 extension">, InGroup<CXX23>;
 def warn_cxx20_compat_constexpr_body_invalid_stmt : Warning<
   "use of this statement in a constexpr %select{function|constructor}0 "
-  "is incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "is incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 def ext_constexpr_type_definition : ExtWarn<
   "type definition in a constexpr %select{function|constructor}0 "
   "is a C++14 extension">, InGroup<CXX14>;
@@ -2753,15 +2754,15 @@ def warn_cxx11_compat_constexpr_local_var : Warning<
 def ext_constexpr_static_var : ExtWarn<
   "definition of a %select{static|thread_local}1 variable "
   "in a constexpr %select{function|constructor}0 "
-  "is a C++2b extension">, InGroup<CXX2b>;
+  "is a C++23 extension">, InGroup<CXX23>;
 def warn_cxx20_compat_constexpr_var : Warning<
   "definition of a %select{static variable|thread_local variable|variable "
   "of non-literal type}1 in a constexpr %select{function|constructor}0 "
-  "is incompatible with C++ standards before C++2b">,
-  InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "is incompatible with C++ standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 def err_constexpr_local_var_non_literal_type : Error<
   "variable of non-literal type %1 cannot be defined in a constexpr "
-  "%select{function|constructor}0 before C++2b">;
+  "%select{function|constructor}0 before C++23">;
 def ext_constexpr_local_var_no_init : ExtWarn<
   "uninitialized variable in a constexpr %select{function|constructor}0 "
   "is a C++20 extension">, InGroup<CXX20>;
@@ -3055,6 +3056,14 @@ def err_attribute_arm_feature_sve_bits_unsupported : Error<
   "value of 128, 256, 512, 1024 or 2048.">;
 def err_sve_vector_in_non_sve_target : Error<
   "SVE vector type %0 cannot be used in a target without sve">;
+def err_attribute_riscv_rvv_bits_unsupported : Error<
+  "%0 is only supported when '-mrvv-vector-bits=<bits>' is specified with a "
+  "value of \"zvl\" or a power 2 in the range [64,65536]">;
+def err_attribute_bad_rvv_vector_size : Error<
+  "invalid RVV vector size '%0', must match value set by "
+  "'-mrvv-vector-bits' ('%1')">;
+def err_attribute_invalid_rvv_type : Error<
+  "%0 attribute applied to non-RVV type %1">;
 def err_attribute_requires_positive_integer : Error<
   "%0 attribute requires a %select{positive|non-negative}1 "
   "integral compile time constant expression">;
@@ -3165,8 +3174,9 @@ def err_attribute_invalid_size : Error<
   "vector size not an integral multiple of component size">;
 def err_attribute_zero_size : Error<"zero %0 size">;
 def err_attribute_size_too_large : Error<"%0 size too large">;
-def err_typecheck_sve_ambiguous : Error<
-  "cannot combine fixed-length and sizeless SVE vectors in expression, result is ambiguous (%0 and %1)">;
+def err_typecheck_sve_rvv_ambiguous : Error<
+  "cannot combine fixed-length and sizeless %select{SVE|RVV}0 vectors "
+  "in expression, result is ambiguous (%1 and %2)">;
 def err_typecheck_sve_rvv_gnu_ambiguous : Error<
   "cannot combine GNU and %select{SVE|RVV}0 vectors in expression, result is ambiguous (%1 and %2)">;
 def err_typecheck_vector_not_convertable_implict_truncation : Error<
@@ -3423,7 +3433,7 @@ def warn_type_attribute_deprecated_on_decl : Warning<
   InGroup<DeprecatedAttributes>;
 def warn_declspec_attribute_ignored : Warning<
   "attribute %0 is ignored, place it after "
-  "\"%select{class|struct|interface|union|enum}1\" to apply attribute to "
+  "\"%select{class|struct|interface|union|enum|enum class|enum struct}1\" to apply attribute to "
   "type declaration">, InGroup<IgnoredAttributes>;
 def warn_attribute_precede_definition : Warning<
   "attribute declaration must precede definition">,
@@ -6734,7 +6744,7 @@ def err_arithmetic_nonfragile_interface : Error<
 
 def warn_deprecated_comma_subscript : Warning<
   "top-level comma expression in array subscript is deprecated "
-  "in C++20 and unsupported in C++2b">,
+  "in C++20 and unsupported in C++23">,
   InGroup<DeprecatedCommaSubscript>;
 
 def ext_subscript_non_lvalue : Extension<
@@ -6928,7 +6938,7 @@ def warn_standalone_specifier : Warning<"'%0' ignored on this declaration">,
 def ext_standalone_specifier : ExtWarn<"'%0' is not permitted on a declaration "
   "of a type">, InGroup<MissingDeclarations>;
 def err_standalone_class_nested_name_specifier : Error<
-  "forward declaration of %select{class|struct|interface|union|enum}0 cannot "
+  "forward declaration of %select{class|struct|interface|union|enum|enum class|enum struct}0 cannot "
   "have a nested name specifier">;
 def err_typecheck_sclass_func : Error<"illegal storage class on function">;
 def err_static_block_func : Error<
@@ -9138,9 +9148,9 @@ def err_operator_overload_needs_class_or_enum : Error<
 def err_operator_overload_variadic : Error<"overloaded %0 cannot be variadic">;
 def warn_cxx20_compat_operator_overload_static : Warning<
   "declaring overloaded %0 as 'static' is incompatible with C++ standards "
-  "before C++2b">, InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "before C++23">, InGroup<CXXPre23Compat>, DefaultIgnore;
 def ext_operator_overload_static : ExtWarn<
-  "declaring overloaded %0 as 'static' is a C++2b extension">, InGroup<CXX2b>;
+  "declaring overloaded %0 as 'static' is a C++23 extension">, InGroup<CXX23>;
 def err_operator_overload_static : Error<
   "overloaded %0 cannot be a static member function">;
 def err_operator_overload_default_arg : Error<
@@ -9148,9 +9158,9 @@ def err_operator_overload_default_arg : Error<
 
 def ext_subscript_overload : Warning<
   "overloaded %0 with %select{no|a defaulted|more than one}1 parameter is a "
-  "C++2b extension">, InGroup<CXXPre2bCompat>, DefaultIgnore;
+  "C++23 extension">, InGroup<CXXPre23Compat>, DefaultIgnore;
 def error_subscript_overload : Error<
-  "overloaded %0 cannot have %select{no|a defaulted|more than one}1 parameter before C++2b">;
+  "overloaded %0 cannot have %select{no|a defaulted|more than one}1 parameter before C++23">;
 
 def err_operator_overload_must_be : Error<
   "overloaded %0 must be a %select{unary|binary|unary or binary}2 operator "
@@ -9416,12 +9426,21 @@ def note_defaulted_comparison_cannot_deduce_undeduced_auto : Note<
   "%select{|member|base class}0 %1 declared here">;
 def note_defaulted_comparison_cannot_deduce_callee : Note<
   "selected 'operator<=>' for %select{|member|base class}0 %1 declared here">;
-def err_incorrect_defaulted_comparison_constexpr : Error<
+def ext_defaulted_comparison_constexpr_mismatch : Extension<
   "defaulted definition of %select{%sub{select_defaulted_comparison_kind}1|"
-  "three-way comparison operator}0 "
-  "cannot be declared %select{constexpr|consteval}2 because "
-  "%select{it|the corresponding implicit 'operator=='}0 "
-  "invokes a non-constexpr comparison function">;
+  "three-way comparison operator}0 that is "
+  "declared %select{constexpr|consteval}2 but"
+  "%select{|for which the corresponding implicit 'operator==' }0 "
+  "invokes a non-constexpr comparison function is a C++23 extension">,
+  InGroup<DiagGroup<"c++23-default-comp-relaxed-constexpr">>;
+def warn_cxx23_compat_defaulted_comparison_constexpr_mismatch : Warning<
+  "defaulted definition of %select{%sub{select_defaulted_comparison_kind}1|"
+  "three-way comparison operator}0 that is "
+  "declared %select{constexpr|consteval}2 but"
+  "%select{|for which the corresponding implicit 'operator==' }0 "
+  "invokes a non-constexpr comparison function is incompatible with C++ "
+  "standards before C++23">,
+  InGroup<CXXPre23Compat>, DefaultIgnore;
 def note_defaulted_comparison_not_constexpr : Note<
   "non-constexpr comparison function would be used to compare "
   "%select{|member %1|base class %1}0">;
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 20bddca48cba1..495b17e769e5d 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -97,7 +97,7 @@ LANGOPT(CPlusPlus11       , 1, 0, "C++11")
 LANGOPT(CPlusPlus14       , 1, 0, "C++14")
 LANGOPT(CPlusPlus17       , 1, 0, "C++17")
 LANGOPT(CPlusPlus20       , 1, 0, "C++20")
-LANGOPT(CPlusPlus2b       , 1, 0, "C++2b")
+LANGOPT(CPlusPlus23       , 1, 0, "C++23")
 LANGOPT(ObjC              , 1, 0, "Objective-C")
 BENIGN_LANGOPT(ObjCDefaultSynthProperties , 1, 0,
                "Objective-C auto-synthesized properties")
diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h
index 4e78570ff733b..ebe14c8a3f8fd 100644
--- a/clang/include/clang/Basic/LangStandard.h
+++ b/clang/include/clang/Basic/LangStandard.h
@@ -55,7 +55,7 @@ enum LangFeatures {
   CPlusPlus14 = (1 << 7),
   CPlusPlus17 = (1 << 8),
   CPlusPlus20 = (1 << 9),
-  CPlusPlus2b = (1 << 10),
+  CPlusPlus23 = (1 << 10),
   Digraphs = (1 << 11),
   GNUMode = (1 << 12),
   HexFloat = (1 << 13),
@@ -118,8 +118,8 @@ struct LangStandard {
   /// isCPlusPlus20 - Language is a C++20 variant (or later).
   bool isCPlusPlus20() const { return Flags & CPlusPlus20; }
 
-  /// isCPlusPlus2b - Language is a post-C++20 variant (or later).
-  bool isCPlusPlus2b() const { return Flags & CPlusPlus2b; }
+  /// isCPlusPlus23 - Language is a post-C++23 variant (or later).
+  bool isCPlusPlus23() const { return Flags & CPlusPlus23; }
 
   /// hasDigraphs - Language supports digraphs.
   bool hasDigraphs() const { return Flags & Digraphs; }
diff --git a/clang/include/clang/Basic/LangStandards.def b/clang/include/clang/Basic/LangStandards.def
index c5d4da1cb2f90..8e4d3759f33b7 100644
--- a/clang/include/clang/Basic/LangStandards.def
+++ b/clang/include/clang/Basic/LangStandards.def
@@ -151,15 +151,17 @@ LANGSTANDARD(gnucxx20, "gnu++20",
              CPlusPlus20 | Digraphs | HexFloat | GNUMode)
 LANGSTANDARD_ALIAS_DEPR(gnucxx20, "gnu++2a")
 
-LANGSTANDARD(cxx2b, "c++2b",
-             CXX, "Working draft for ISO C++ 2023 DIS",
+LANGSTANDARD(cxx23, "c++23",
+             CXX, "ISO C++ 2023 DIS",
              LineComment | CPlusPlus | CPlusPlus11 | CPlusPlus14 | CPlusPlus17 |
-             CPlusPlus20 | CPlusPlus2b | Digraphs | HexFloat)
+             CPlusPlus20 | CPlusPlus23 | Digraphs | HexFloat)
+LANGSTANDARD_ALIAS_DEPR(cxx23, "c++2b")
 
-LANGSTANDARD(gnucxx2b, "gnu++2b",
-             CXX, "Working draft for ISO C++ 2023 DIS with GNU extensions",
+LANGSTANDARD(gnucxx23, "gnu++23",
+             CXX, "ISO C++ 2023 DIS with GNU extensions",
              LineComment | CPlusPlus | CPlusPlus11 | CPlusPlus14 | CPlusPlus17 |
-             CPlusPlus20 | CPlusPlus2b | Digraphs | HexFloat | GNUMode)
+             CPlusPlus20 | CPlusPlus23 | Digraphs | HexFloat | GNUMode)
+LANGSTANDARD_ALIAS_DEPR(gnucxx23, "gnu++2b")
 
 // OpenCL
 LANGSTANDARD(opencl10, "cl1.0",
diff --git a/clang/include/clang/Basic/RISCVVTypes.def b/clang/include/clang/Basic/RISCVVTypes.def
index 1d4024dfb20d3..b21a3bb4d5a6b 100644
--- a/clang/include/clang/Basic/RISCVVTypes.def
+++ b/clang/include/clang/Basic/RISCVVTypes.def
@@ -40,6 +40,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef RVV_TYPE
+#define RVV_TYPE(Name, Id, SingletonId)
+#endif
+
 #ifndef RVV_VECTOR_TYPE
 #define RVV_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, NF, IsSigned, IsFP)\
   RVV_TYPE(Name, Id, SingletonId)
diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h
index a8c35fed9997e..06279a016a507 100644
--- a/clang/include/clang/Basic/Specifiers.h
+++ b/clang/include/clang/Basic/Specifiers.h
@@ -19,6 +19,9 @@
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
 
+namespace llvm {
+class raw_ostream;
+} // namespace llvm
 namespace clang {
 
   /// Define the meaning of possible values of the kind in ExplicitSpecifier.
@@ -333,6 +336,8 @@ namespace clang {
     // parameters are assumed to only get null on error.
     NullableResult,
   };
+  /// Prints human-readable debug representation.
+  llvm::raw_ostream &operator<<(llvm::raw_ostream&, NullabilityKind);
 
   /// Return true if \p L has a weaker nullability annotation than \p R. The
   /// ordering is: Unspecified < Nullable < NonNull.
diff --git a/clang/include/clang/Basic/riscv_sifive_vector.td b/clang/include/clang/Basic/riscv_sifive_vector.td
new file mode 100644
index 0000000000000..0d390be711c83
--- /dev/null
+++ b/clang/include/clang/Basic/riscv_sifive_vector.td
@@ -0,0 +1,105 @@
+//==--- riscv_sifive_vector.td - RISC-V SiFive VCIX function list ---------===//
+//
+//  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//  See https://llvm.org/LICENSE.txt for license information.
+//  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the builtins for RISC-V SiFive VCIX. See:
+//
+//     https://sifive.cdn.prismic.io/sifive/c3829e36-8552-41f0-a841-79945784241b_vcix-spec-software.pdf
+//
+//===----------------------------------------------------------------------===//
+
+include "riscv_vector_common.td"
+
+//===----------------------------------------------------------------------===//
+// Instruction definitions
+//===----------------------------------------------------------------------===//
+
+class VCIXSuffix<string range> {
+  list<string> suffix = !cond(!eq(range, "c"): ["8mf8", "8mf4", "8mf2", "8m1", "8m2", "8m4", "8m8"],
+                              !eq(range, "s"): ["16mf4", "16mf2", "16m1", "16m2", "16m4", "16m8"],
+                              !eq(range, "i"): ["32mf2", "32m1", "32m2", "32m4", "32m8"],
+                              !eq(range, "l"): ["64m1", "64m2", "64m4", "64m8"]);
+}
+
+class VCIXBuiltinSet<string name, string IR_name, string suffix,
+                     string prototype, string type_range,
+                     list<int> intrinsic_types>
+    : RVVBuiltin<suffix, prototype, type_range> {
+  let Name = name;
+  let OverloadedName = name;
+  let IRName = IR_name;
+  let HasMasked = false;
+  let IntrinsicTypes = intrinsic_types;
+}
+
+multiclass VCIXBuiltinSet<string name, string IR_name, string suffix,
+                          string prototype, string type_range,
+                          list<int> intrinsic_types> {
+  if !find(prototype, "0") then {
+    def : VCIXBuiltinSet<name, IR_name, suffix, prototype, type_range, intrinsic_types>;
+  }
+  def : VCIXBuiltinSet<name # "_se", IR_name # "_se", suffix, prototype, type_range, intrinsic_types>;
+}
+
+multiclass RVVVCIXBuiltinSet<list<string> range, string prototype,
+                             list<int> intrinsic_types, bit UseGPR> {
+  foreach r = range in
+    let RequiredFeatures = !if(!and(UseGPR, !eq(r, "l")),
+                               ["Xsfvcp", "RV64"], ["Xsfvcp"]) in
+      defm : VCIXBuiltinSet<NAME, NAME, "Uv", prototype, r, intrinsic_types>;
+}
+
+multiclass RVVVCIXBuiltinSetWVType<list<string> range, string prototype,
+                             list<int> intrinsic_types, bit UseGPR> {
+  foreach r = range in
+    let RequiredFeatures = !if(!and(UseGPR, !eq(r, "l")),
+                               ["Xsfvcp", "RV64"], ["Xsfvcp"]) in
+      // These intrinsics don't have any vector types in the output and inputs,
+      // but we still need to add vetvli for them. So we encode different
+      // VTYPE into the intrinsic names, and then will know which vsetvli is
+      // correct.
+      foreach s = VCIXSuffix<r>.suffix in
+        // Since we already encode the Vtype into the name, so just set
+        // Log2LMUL to zero.  Otherwise the RISCVVEmitter will expand
+        // lots of redundant intrinsic but have same names.
+        let Log2LMUL = [0] in
+          def : VCIXBuiltinSet<NAME # "_u" # s, NAME # "_e" # s,
+                               "", prototype, r, intrinsic_types>;
+}
+
+let SupportOverloading = false in {
+  defm sf_vc_x_se  : RVVVCIXBuiltinSetWVType<["c", "s", "i", "l"], "0KzKzKzUe", [0, 3], /*UseGPR*/1>;
+  defm sf_vc_i_se  : RVVVCIXBuiltinSetWVType<["c", "s", "i", "l"], "0KzKzKzKz", [2, 3], /*UseGPR*/0>;
+  defm sf_vc_xv    : RVVVCIXBuiltinSet<["csi", "l"], "0KzKzUvUe",  [0, 2, 3],  /*UseGPR*/1>;
+  defm sf_vc_iv    : RVVVCIXBuiltinSet<["csi", "l"], "0KzKzUvKz",  [0, 2, 3],  /*UseGPR*/0>;
+  defm sf_vc_vv    : RVVVCIXBuiltinSet<["csi", "l"], "0KzKzUvUv",  [0, 2, 3],  /*UseGPR*/0>;
+  defm sf_vc_fv    : RVVVCIXBuiltinSet<["si",  "l"], "0KzKzUvFe",  [0, 2, 3],  /*UseGPR*/0>;
+  defm sf_vc_xvv   : RVVVCIXBuiltinSet<["csi", "l"], "0KzUvUvUe",  [0, 1, 3],  /*UseGPR*/1>;
+  defm sf_vc_ivv   : RVVVCIXBuiltinSet<["csi", "l"], "0KzUvUvKz",  [0, 1, 3],  /*UseGPR*/0>;
+  defm sf_vc_vvv   : RVVVCIXBuiltinSet<["csi", "l"], "0KzUvUvUv",  [0, 1, 3],  /*UseGPR*/0>;
+  defm sf_vc_fvv   : RVVVCIXBuiltinSet<["si",  "l"], "0KzUvUvFe",  [0, 1, 3],  /*UseGPR*/0>;
+  defm sf_vc_v_x   : RVVVCIXBuiltinSet<["csi", "l"], "UvKzKzUe",   [-1, 1, 2], /*UseGPR*/1>;
+  defm sf_vc_v_i   : RVVVCIXBuiltinSet<["csi", "l"], "UvKzKzKz",   [-1, 1, 2], /*UseGPR*/0>;
+  defm sf_vc_v_xv  : RVVVCIXBuiltinSet<["csi", "l"], "UvKzUvUe",   [-1, 0, 2], /*UseGPR*/1>;
+  defm sf_vc_v_iv  : RVVVCIXBuiltinSet<["csi", "l"], "UvKzUvKz",   [-1, 0, 2], /*UseGPR*/0>;
+  defm sf_vc_v_vv  : RVVVCIXBuiltinSet<["csi", "l"], "UvKzUvUv",   [-1, 0, 2], /*UseGPR*/0>;
+  defm sf_vc_v_fv  : RVVVCIXBuiltinSet<["si",  "l"], "UvKzUvFe",   [-1, 0, 2], /*UseGPR*/0>;
+  defm sf_vc_v_xvv : RVVVCIXBuiltinSet<["csi", "l"], "UvKzUvUvUe", [-1, 0, 3], /*UseGPR*/1>;
+  defm sf_vc_v_ivv : RVVVCIXBuiltinSet<["csi", "l"], "UvKzUvUvKz", [-1, 0, 3], /*UseGPR*/0>;
+  defm sf_vc_v_vvv : RVVVCIXBuiltinSet<["csi", "l"], "UvKzUvUvUv", [-1, 0, 3], /*UseGPR*/0>;
+  defm sf_vc_v_fvv : RVVVCIXBuiltinSet<["si",  "l"], "UvKzUvUvFe", [-1, 0, 3], /*UseGPR*/0>;
+  let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
+    defm sf_vc_xvw   : RVVVCIXBuiltinSet<["csi"], "0KzUwUvUe",  [0, 1, 2, 3],  /*UseGPR*/1>;
+    defm sf_vc_ivw   : RVVVCIXBuiltinSet<["csi"], "0KzUwUvKz",  [0, 1, 2, 3],  /*UseGPR*/0>;
+    defm sf_vc_vvw   : RVVVCIXBuiltinSet<["csi"], "0KzUwUvUv",  [0, 1, 2, 3],  /*UseGPR*/0>;
+    defm sf_vc_fvw   : RVVVCIXBuiltinSet<["si"],  "0KzUwUvFe",  [0, 1, 2, 3],  /*UseGPR*/0>;
+    defm sf_vc_v_xvw : RVVVCIXBuiltinSet<["csi"], "UwKzUwUvUe", [-1, 0, 2, 3], /*UseGPR*/1>;
+    defm sf_vc_v_ivw : RVVVCIXBuiltinSet<["csi"], "UwKzUwUvKz", [-1, 0, 2, 3], /*UseGPR*/0>;
+    defm sf_vc_v_vvw : RVVVCIXBuiltinSet<["csi"], "UwKzUwUvUv", [-1, 0, 2, 3], /*UseGPR*/0>;
+    defm sf_vc_v_fvw : RVVVCIXBuiltinSet<["si"],  "UwKzUwUvFe", [-1, 0, 2, 3], /*UseGPR*/0>;
+  }
+}
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index b23e26ecaa579..8eb873dfafc79 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -12,233 +12,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-//===----------------------------------------------------------------------===//
-// Instruction definitions
-//===----------------------------------------------------------------------===//
-// Each record of the class RVVBuiltin defines a collection of builtins (i.e.
-// "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1",
-// "vadd_vv_i32m2", etc).
-//
-// The elements of this collection are defined by an instantiation process the
-// range of which is specified by the cross product of the LMUL attribute and
-// every element in the attribute TypeRange. By default builtins have LMUL = [1,
-// 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we
-// use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL.
-//
-// LMUL represents the fact that the types of values used by that builtin are
-// values generated by instructions that are executed under that LMUL. However,
-// this does not mean the builtin is necessarily lowered into an instruction
-// that executes under the specified LMUL. An example where this happens are
-// loads and stores of masks. A mask like `vbool8_t` can be generated, for
-// instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two
-// `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will
-// be performed under LMUL=1 because mask registers are not grouped.
-//
-// TypeRange is a non-empty sequence of basic types:
-//
-//   c: int8_t (i8)
-//   s: int16_t (i16)
-//   i: int32_t (i32)
-//   l: int64_t (i64)
-//   x: float16_t (half)
-//   f: float32_t (float)
-//   d: float64_t (double)
-//
-// This way, given an LMUL, a record with a TypeRange "sil" will cause the
-// definition of 3 builtins. Each type "t" in the TypeRange (in this example
-// they are int16_t, int32_t, int64_t) is used as a parameter that drives the
-// definition of that particular builtin (for the given LMUL).
-//
-// During the instantiation, types can be transformed or modified using type
-// transformers. Given a type "t" the following primitive type transformers can
-// be applied to it to yield another type.
-//
-//   e: type of "t" as is (identity)
-//   v: computes a vector type whose element type is "t" for the current LMUL
-//   w: computes a vector type identical to what 'v' computes except for the
-//      element type which is twice as wide as the element type of 'v'
-//   q: computes a vector type identical to what 'v' computes except for the
-//      element type which is four times as wide as the element type of 'v'
-//   o: computes a vector type identical to what 'v' computes except for the
-//      element type which is eight times as wide as the element type of 'v'
-//   m: computes a vector type identical to what 'v' computes except for the
-//      element type which is bool
-//   0: void type, ignores "t"
-//   z: size_t, ignores "t"
-//   t: ptrdiff_t, ignores "t"
-//   u: unsigned long, ignores "t"
-//   l: long, ignores "t"
-//
-// So for instance if t is "i", i.e. int, then "e" will yield int again. "v"
-// will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t.
-// Accordingly "w" would yield __rvv_int64m2_t.
-//
-// A type transformer can be prefixed by other non-primitive type transformers.
-//
-//   P: constructs a pointer to the current type
-//   C: adds const to the type
-//   K: requires the integer type to be a constant expression
-//   U: given an integer type or vector type, computes its unsigned variant
-//   I: given a vector type, compute the vector type with integer type
-//      elements of the same width
-//   F: given a vector type, compute the vector type with floating-point type
-//      elements of the same width
-//   S: given a vector type, computes its equivalent one for LMUL=1. This is a
-//      no-op if the vector was already LMUL=1
-//   (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a
-//      vector type (SEW and LMUL) and EEW (8/16/32/64), computes its
-//      equivalent integer vector type with EEW and corresponding ELMUL (elmul =
-//      (eew/sew) * lmul). For example, vector type is __rvv_float16m4
-//      (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector
-//      type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new
-//      builtins if its equivalent type has illegal lmul.
-//   (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another
-//      vector type which only changed SEW as given value. Ignore to define a new
-//      builtin if its equivalent type has illegal lmul or the SEW does not changed.
-//   (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW
-//      and LMUL), and computes another vector type which only changed LMUL as
-//      given value. The new LMUL should be smaller than the old one. Ignore to
-//      define a new builtin if its equivalent type has illegal lmul.
-//   (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW
-//      and LMUL), and computes another vector type which only changed LMUL as
-//      given value. The new LMUL should be larger than the old one. Ignore to
-//      define a new builtin if its equivalent type has illegal lmul.
-//
-// Following with the example above, if t is "i", then "Ue" will yield unsigned
-// int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would
-// yield __rvv_float64m2_t, etc.
-//
-// Each builtin is then defined by applying each type in TypeRange against the
-// sequence of type transformers described in Suffix and Prototype.
-//
-// The name of the builtin is defined by the Name attribute (which defaults to
-// the name of the class) appended (separated with an underscore) the Suffix
-// attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il",
-// the builtin generated will be __builtin_rvv_foo_i32m1 and
-// __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one
-// type transformer (say "vv") each of the types is separated with an
-// underscore as in "__builtin_rvv_foo_i32m1_i32m1".
-//
-// The C/C++ prototype of the builtin is defined by the Prototype attribute.
-// Prototype is a non-empty sequence of type transformers, the first of which
-// is the return type of the builtin and the rest are the parameters of the
-// builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si"
-// a first builtin will have type
-// __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin
-// will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again
-// under LMUL=1).
-//
-// There are a number of attributes that are used to constraint the number and
-// shape of the builtins generated. Refer to the comments below for them.
-
-class PolicyScheme<int val>{
-  int Value = val;
-}
-def NonePolicy : PolicyScheme<0>;
-def HasPassthruOperand : PolicyScheme<1>;
-def HasPolicyOperand : PolicyScheme<2>;
-
-class RVVBuiltin<string suffix, string prototype, string type_range,
-                 string overloaded_suffix = ""> {
-  // Base name that will be prepended in __builtin_rvv_ and appended the
-  // computed Suffix.
-  string Name = NAME;
-
-  // If not empty, each instantiated builtin will have this appended after an
-  // underscore (_). It is instantiated like Prototype.
-  string Suffix = suffix;
-
-  // If empty, default OverloadedName is sub string of `Name` which end of first
-  // '_'. For example, the default overloaded name  is `vadd` for Name `vadd_vv`.
-  // It's used for describe some special naming cases.
-  string OverloadedName = "";
-
-  // If not empty, each OverloadedName will have this appended after an
-  // underscore (_). It is instantiated like Prototype.
-  string OverloadedSuffix = overloaded_suffix;
-
-  // The different variants of the builtin, parameterised with a type.
-  string TypeRange = type_range;
-
-  // We use each type described in TypeRange and LMUL with prototype to
-  // instantiate a specific element of the set of builtins being defined.
-  // Prototype attribute defines the C/C++ prototype of the builtin. It is a
-  // non-empty sequence of type transformers, the first of which is the return
-  // type of the builtin and the rest are the parameters of the builtin, in
-  // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a
-  // first builtin will have type
-  // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin
-  // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t).
-  string Prototype = prototype;
-
-  // This builtin has a masked form.
-  bit HasMasked = true;
-
-  // If HasMasked, this flag states that this builtin has a maskedoff operand. It
-  // is always the first operand in builtin and IR intrinsic.
-  bit HasMaskedOffOperand = true;
-
-  // This builtin has a granted vector length parameter.
-  bit HasVL = true;
-
-  // The policy scheme for masked intrinsic IR.
-  // It could be NonePolicy or HasPolicyOperand.
-  // HasPolicyOperand: Has a policy operand. 0 is tail and mask undisturbed, 1 is
-  // tail agnostic, 2 is mask undisturbed, and 3 is tail and mask agnostic. The
-  // policy operand is located at the last position.
-  PolicyScheme MaskedPolicyScheme = HasPolicyOperand;
-
-  // The policy scheme for unmasked intrinsic IR.
-  // It could be NonePolicy, HasPassthruOperand or HasPolicyOperand.
-  // HasPassthruOperand: Has a passthru operand to decide tail policy. If it is
-  // poison, tail policy is tail agnostic, otherwise policy is tail undisturbed.
-  // HasPolicyOperand: Has a policy operand. 1 is tail agnostic and 0 is tail
-  // undisturbed.
-  PolicyScheme UnMaskedPolicyScheme = NonePolicy;
-
-  // This builtin support tail agnostic and undisturbed policy.
-  bit HasTailPolicy = true;
-  // This builtin support mask agnostic and undisturbed policy.
-  bit HasMaskPolicy = true;
-
-  // This builtin prototype with TA or TAMA policy could not support overloading
-  // API. Other policy intrinsic functions would support overloading API with
-  // suffix `_tu`, `tumu`, `tuma`, `tamu` and `tama`.
-  bit SupportOverloading = true;
-
-  // This builtin is valid for the given Log2LMULs.
-  list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3];
-
-  // Manual code in clang codegen riscv_vector_builtin_cg.inc
-  code ManualCodegen = [{}];
-
-  // When emit the automatic clang codegen, it describes what types we have to use
-  // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise,
-  // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd
-  // parameter of the unmasked version. k can't be the mask operand's position.
-  list<int> IntrinsicTypes = [];
-
-  // If these names are not empty, this is the ID of the LLVM intrinsic
-  // we want to lower to.
-  string IRName = NAME;
-
-  // If HasMasked, this is the ID of the LLVM intrinsic we want to lower to.
-  string MaskedIRName = NAME #"_mask";
-
-  // Use clang_builtin_alias to save the number of builtins.
-  bit HasBuiltinAlias = true;
-
-  // Features required to enable for this builtin.
-  list<string> RequiredFeatures = [];
-
-  // Number of fields for Load/Store Segment instructions.
-  int NF = 1;
-}
-
-// This is the code emitted in the header.
-class RVVHeader {
-  code HeaderCode;
-}
+include "riscv_vector_common.td"
 
 //===----------------------------------------------------------------------===//
 // Basic classes with automatic codegen.
diff --git a/clang/include/clang/Basic/riscv_vector_common.td b/clang/include/clang/Basic/riscv_vector_common.td
new file mode 100644
index 0000000000000..6fd1dbde52be4
--- /dev/null
+++ b/clang/include/clang/Basic/riscv_vector_common.td
@@ -0,0 +1,239 @@
+//==------ riscv_vector_common.td - RISC-V V-ext builtin class ------------===//
+//
+//  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+//  See https://llvm.org/LICENSE.txt for license information.
+//  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines RVV builtin base class for RISC-V V-extension.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Instruction definitions
+//===----------------------------------------------------------------------===//
+// Each record of the class RVVBuiltin defines a collection of builtins (i.e.
+// "def vadd : RVVBuiltin" will be used to define things like "vadd_vv_i32m1",
+// "vadd_vv_i32m2", etc).
+//
+// The elements of this collection are defined by an instantiation process the
+// range of which is specified by the cross product of the LMUL attribute and
+// every element in the attribute TypeRange. By default builtins have LMUL = [1,
+// 2, 4, 8, 1/2, 1/4, 1/8] so the process is repeated 7 times. In tablegen we
+// use the Log2LMUL [0, 1, 2, 3, -1, -2, -3] to represent the LMUL.
+//
+// LMUL represents the fact that the types of values used by that builtin are
+// values generated by instructions that are executed under that LMUL. However,
+// this does not mean the builtin is necessarily lowered into an instruction
+// that executes under the specified LMUL. An example where this happens are
+// loads and stores of masks. A mask like `vbool8_t` can be generated, for
+// instance, by comparing two `__rvv_int8m1_t` (this is LMUL=1) or comparing two
+// `__rvv_int16m2_t` (this is LMUL=2). The actual load or store, however, will
+// be performed under LMUL=1 because mask registers are not grouped.
+//
+// TypeRange is a non-empty sequence of basic types:
+//
+//   c: int8_t (i8)
+//   s: int16_t (i16)
+//   i: int32_t (i32)
+//   l: int64_t (i64)
+//   x: float16_t (half)
+//   f: float32_t (float)
+//   d: float64_t (double)
+//
+// This way, given an LMUL, a record with a TypeRange "sil" will cause the
+// definition of 3 builtins. Each type "t" in the TypeRange (in this example
+// they are int16_t, int32_t, int64_t) is used as a parameter that drives the
+// definition of that particular builtin (for the given LMUL).
+//
+// During the instantiation, types can be transformed or modified using type
+// transformers. Given a type "t" the following primitive type transformers can
+// be applied to it to yield another type.
+//
+//   e: type of "t" as is (identity)
+//   v: computes a vector type whose element type is "t" for the current LMUL
+//   w: computes a vector type identical to what 'v' computes except for the
+//      element type which is twice as wide as the element type of 'v'
+//   q: computes a vector type identical to what 'v' computes except for the
+//      element type which is four times as wide as the element type of 'v'
+//   o: computes a vector type identical to what 'v' computes except for the
+//      element type which is eight times as wide as the element type of 'v'
+//   m: computes a vector type identical to what 'v' computes except for the
+//      element type which is bool
+//   0: void type, ignores "t"
+//   z: size_t, ignores "t"
+//   t: ptrdiff_t, ignores "t"
+//   u: unsigned long, ignores "t"
+//   l: long, ignores "t"
+//
+// So for instance if t is "i", i.e. int, then "e" will yield int again. "v"
+// will yield an RVV vector type (assume LMUL=1), so __rvv_int32m1_t.
+// Accordingly "w" would yield __rvv_int64m2_t.
+//
+// A type transformer can be prefixed by other non-primitive type transformers.
+//
+//   P: constructs a pointer to the current type
+//   C: adds const to the type
+//   K: requires the integer type to be a constant expression
+//   U: given an integer type or vector type, computes its unsigned variant
+//   I: given a vector type, compute the vector type with integer type
+//      elements of the same width
+//   F: given a vector type, compute the vector type with floating-point type
+//      elements of the same width
+//   S: given a vector type, computes its equivalent one for LMUL=1. This is a
+//      no-op if the vector was already LMUL=1
+//   (Log2EEW:Value): Log2EEW value could be 3/4/5/6 (8/16/32/64), given a
+//      vector type (SEW and LMUL) and EEW (8/16/32/64), computes its
+//      equivalent integer vector type with EEW and corresponding ELMUL (elmul =
+//      (eew/sew) * lmul). For example, vector type is __rvv_float16m4
+//      (SEW=16, LMUL=4) and Log2EEW is 3 (EEW=8), and then equivalent vector
+//      type is __rvv_uint8m2_t (elmul=(8/16)*4 = 2). Ignore to define a new
+//      builtins if its equivalent type has illegal lmul.
+//   (FixedSEW:Value): Given a vector type (SEW and LMUL), and computes another
+//      vector type which only changed SEW as given value. Ignore to define a new
+//      builtin if its equivalent type has illegal lmul or the SEW does not changed.
+//   (SFixedLog2LMUL:Value): Smaller Fixed Log2LMUL. Given a vector type (SEW
+//      and LMUL), and computes another vector type which only changed LMUL as
+//      given value. The new LMUL should be smaller than the old one. Ignore to
+//      define a new builtin if its equivalent type has illegal lmul.
+//   (LFixedLog2LMUL:Value): Larger Fixed Log2LMUL. Given a vector type (SEW
+//      and LMUL), and computes another vector type which only changed LMUL as
+//      given value. The new LMUL should be larger than the old one. Ignore to
+//      define a new builtin if its equivalent type has illegal lmul.
+//
+// Following with the example above, if t is "i", then "Ue" will yield unsigned
+// int and "Fv" will yield __rvv_float32m1_t (again assuming LMUL=1), Fw would
+// yield __rvv_float64m2_t, etc.
+//
+// Each builtin is then defined by applying each type in TypeRange against the
+// sequence of type transformers described in Suffix and Prototype.
+//
+// The name of the builtin is defined by the Name attribute (which defaults to
+// the name of the class) appended (separated with an underscore) the Suffix
+// attribute. For instance with Name="foo", Suffix = "v" and TypeRange = "il",
+// the builtin generated will be __builtin_rvv_foo_i32m1 and
+// __builtin_rvv_foo_i64m1 (under LMUL=1). If Suffix contains more than one
+// type transformer (say "vv") each of the types is separated with an
+// underscore as in "__builtin_rvv_foo_i32m1_i32m1".
+//
+// The C/C++ prototype of the builtin is defined by the Prototype attribute.
+// Prototype is a non-empty sequence of type transformers, the first of which
+// is the return type of the builtin and the rest are the parameters of the
+// builtin, in order. For instance if Prototype is "wvv" and TypeRange is "si"
+// a first builtin will have type
+// __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t) and the second builtin
+// will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t) (again
+// under LMUL=1).
+//
+// There are a number of attributes that are used to constraint the number and
+// shape of the builtins generated. Refer to the comments below for them.
+
+class PolicyScheme<int val>{
+  int Value = val;
+}
+def NonePolicy : PolicyScheme<0>;
+def HasPassthruOperand : PolicyScheme<1>;
+def HasPolicyOperand : PolicyScheme<2>;
+
+class RVVBuiltin<string suffix, string prototype, string type_range,
+                 string overloaded_suffix = ""> {
+  // Base name that will be prepended in __builtin_rvv_ and appended the
+  // computed Suffix.
+  string Name = NAME;
+
+  // If not empty, each instantiated builtin will have this appended after an
+  // underscore (_). It is instantiated like Prototype.
+  string Suffix = suffix;
+
+  // If empty, default OverloadedName is sub string of `Name` which end of first
+  // '_'. For example, the default overloaded name  is `vadd` for Name `vadd_vv`.
+  // It's used for describe some special naming cases.
+  string OverloadedName = "";
+
+  // If not empty, each OverloadedName will have this appended after an
+  // underscore (_). It is instantiated like Prototype.
+  string OverloadedSuffix = overloaded_suffix;
+
+  // The different variants of the builtin, parameterised with a type.
+  string TypeRange = type_range;
+
+  // We use each type described in TypeRange and LMUL with prototype to
+  // instantiate a specific element of the set of builtins being defined.
+  // Prototype attribute defines the C/C++ prototype of the builtin. It is a
+  // non-empty sequence of type transformers, the first of which is the return
+  // type of the builtin and the rest are the parameters of the builtin, in
+  // order. For instance if Prototype is "wvv", TypeRange is "si" and LMUL=1, a
+  // first builtin will have type
+  // __rvv_int32m2_t (__rvv_int16m1_t, __rvv_int16m1_t), and the second builtin
+  // will have type __rvv_int64m2_t (__rvv_int32m1_t, __rvv_int32m1_t).
+  string Prototype = prototype;
+
+  // This builtin has a masked form.
+  bit HasMasked = true;
+
+  // If HasMasked, this flag states that this builtin has a maskedoff operand. It
+  // is always the first operand in builtin and IR intrinsic.
+  bit HasMaskedOffOperand = true;
+
+  // This builtin has a granted vector length parameter.
+  bit HasVL = true;
+
+  // The policy scheme for masked intrinsic IR.
+  // It could be NonePolicy or HasPolicyOperand.
+  // HasPolicyOperand: Has a policy operand. 0 is tail and mask undisturbed, 1 is
+  // tail agnostic, 2 is mask undisturbed, and 3 is tail and mask agnostic. The
+  // policy operand is located at the last position.
+  PolicyScheme MaskedPolicyScheme = HasPolicyOperand;
+
+  // The policy scheme for unmasked intrinsic IR.
+  // It could be NonePolicy, HasPassthruOperand or HasPolicyOperand.
+  // HasPassthruOperand: Has a passthru operand to decide tail policy. If it is
+  // poison, tail policy is tail agnostic, otherwise policy is tail undisturbed.
+  // HasPolicyOperand: Has a policy operand. 1 is tail agnostic and 0 is tail
+  // undisturbed.
+  PolicyScheme UnMaskedPolicyScheme = NonePolicy;
+
+  // This builtin support tail agnostic and undisturbed policy.
+  bit HasTailPolicy = true;
+  // This builtin support mask agnostic and undisturbed policy.
+  bit HasMaskPolicy = true;
+
+  // This builtin prototype with TA or TAMA policy could not support overloading
+  // API. Other policy intrinsic functions would support overloading API with
+  // suffix `_tu`, `tumu`, `tuma`, `tamu` and `tama`.
+  bit SupportOverloading = true;
+
+  // This builtin is valid for the given Log2LMULs.
+  list<int> Log2LMUL = [0, 1, 2, 3, -1, -2, -3];
+
+  // Manual code in clang codegen riscv_vector_builtin_cg.inc
+  code ManualCodegen = [{}];
+
+  // When emit the automatic clang codegen, it describes what types we have to use
+  // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise,
+  // k >= 0 meaning the k-th operand (counting from zero) of the codegen'd
+  // parameter of the unmasked version. k can't be the mask operand's position.
+  list<int> IntrinsicTypes = [];
+
+  // If these names are not empty, this is the ID of the LLVM intrinsic
+  // we want to lower to.
+  string IRName = NAME;
+
+  // If HasMasked, this is the ID of the LLVM intrinsic we want to lower to.
+  string MaskedIRName = NAME #"_mask";
+
+  // Use clang_builtin_alias to save the number of builtins.
+  bit HasBuiltinAlias = true;
+
+  // Features required to enable for this builtin.
+  list<string> RequiredFeatures = [];
+
+  // Number of fields for Load/Store Segment instructions.
+  int NF = 1;
+}
+
+// This is the code emitted in the header.
+class RVVHeader {
+  code HeaderCode;
+}
diff --git a/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h b/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h
index c13e052149d95..7a02d8725885a 100644
--- a/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h
+++ b/clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h
@@ -32,7 +32,7 @@ class ObjectFilePCHContainerWriter : public PCHContainerWriter {
 /// A PCHContainerReader implementation that uses LLVM to
 /// wraps Clang modules inside a COFF, ELF, or Mach-O container.
 class ObjectFilePCHContainerReader : public PCHContainerReader {
-  StringRef getFormat() const override { return "obj"; }
+  ArrayRef<StringRef> getFormats() const override;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 2abec84271356..182f0290736d8 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -967,8 +967,10 @@ def cxx_isystem : JoinedOrSeparate<["-"], "cxx-isystem">, Group<clang_i_Group>,
   MetaVarName<"<directory>">;
 def c : Flag<["-"], "c">, Flags<[NoXarchOption, FlangOption]>, Group<Action_Group>,
   HelpText<"Only run preprocess, compile, and assemble steps">;
-def fconvergent_functions : Flag<["-"], "fconvergent-functions">, Group<f_Group>, Flags<[CC1Option]>,
-  HelpText<"Assume functions may be convergent">;
+defm convergent_functions : BoolFOption<"convergent-functions",
+  LangOpts<"ConvergentFunctions">, DefaultFalse,
+  NegFlag<SetFalse, [], "Assume all functions may be convergent.">,
+  PosFlag<SetTrue, [CC1Option]>>;
 
 def gpu_use_aux_triple_only : Flag<["--"], "gpu-use-aux-triple-only">,
   InternalDriverOpt, HelpText<"Prepare '-aux-triple' only without populating "
@@ -2034,11 +2036,6 @@ defm delete_null_pointer_checks : BoolFOption<"delete-null-pointer-checks",
   PosFlag<SetFalse, [], "Treat usage of null pointers as undefined behavior (default)">,
   BothFlags<[CoreOption]>>;
 
-def frewrite_map_file_EQ : Joined<["-"], "frewrite-map-file=">,
-                           Group<f_Group>,
-                           Flags<[NoXarchOption, CC1Option]>,
-                           MarshallingInfoStringVector<CodeGenOpts<"RewriteMapFiles">>;
-
 defm use_line_directives : BoolFOption<"use-line-directives",
   PreprocessorOutputOpts<"UseLineDirectives">, DefaultFalse,
   PosFlag<SetTrue, [CC1Option], "Use #line in preprocessed output">, NegFlag<SetFalse>>;
@@ -3221,11 +3218,13 @@ def fdebug_default_version: Joined<["-"], "fdebug-default-version=">, Group<f_Gr
 def fdebug_prefix_map_EQ
   : Joined<["-"], "fdebug-prefix-map=">, Group<f_Group>,
     Flags<[CC1Option,CC1AsOption]>,
-    HelpText<"remap file source paths in debug info">;
+    MetaVarName<"<old>=<new>">,
+    HelpText<"For paths in debug info, remap directory <old> to <new>. If multiple options match a path, the last option wins">;
 def fcoverage_prefix_map_EQ
   : Joined<["-"], "fcoverage-prefix-map=">, Group<f_Group>,
     Flags<[CC1Option]>,
-    HelpText<"remap file source paths in coverage mapping">;
+    MetaVarName<"<old>=<new>">,
+    HelpText<"remap file source paths <old> to <new> in coverage mapping. If there are multiple options, prefix replacement is applied in reverse order starting from the last one">;
 def ffile_prefix_map_EQ
   : Joined<["-"], "ffile-prefix-map=">, Group<f_Group>,
     HelpText<"remap file source paths in debug info, predefined preprocessor "
@@ -3647,9 +3646,11 @@ def mcmodel_EQ_medany : Flag<["-"], "mcmodel=medany">, Group<m_Group>,
 def menable_experimental_extensions : Flag<["-"], "menable-experimental-extensions">, Group<m_Group>,
   HelpText<"Enable use of experimental RISC-V extensions.">;
 def mrvv_vector_bits_EQ : Joined<["-"], "mrvv-vector-bits=">, Group<m_Group>,
-  HelpText<"Specify the size in bits of an RVV vector register. Defaults to the"
-           " vector length agnostic value of \"scalable\". Also accepts \"zvl\""
-           " to use the value implied by -march/-mcpu (RISC-V only)">;
+  HelpText<"Specify the size in bits of an RVV vector register. Defaults to "
+           "the vector length agnostic value of \"scalable\". Accepts power of "
+           "2 values between 64 and 65536. Also accepts \"zvl\" "
+           "to use the value implied by -march/-mcpu. Value will be reflected "
+           "in __riscv_v_fixed_vlen preprocessor define (RISC-V only)">;
 
 def munaligned_access : Flag<["-"], "munaligned-access">, Group<m_arm_Features_Group>,
   HelpText<"Allow memory accesses to be unaligned (AArch32/AArch64 only)">;
@@ -3830,11 +3831,6 @@ def mno_prefixed: Flag<["-"], "mno-prefixed">, Group<m_ppc_Features_Group>;
 def mspe : Flag<["-"], "mspe">, Group<m_ppc_Features_Group>;
 def mno_spe : Flag<["-"], "mno-spe">, Group<m_ppc_Features_Group>;
 def mefpu2 : Flag<["-"], "mefpu2">, Group<m_ppc_Features_Group>;
-def mabi_EQ_vec_extabi : Flag<["-"], "mabi=vec-extabi">, Group<m_Group>, Flags<[CC1Option]>,
-  HelpText<"Enable the extended Altivec ABI on AIX (AIX only). Uses volatile and nonvolatile vector registers">,
-  MarshallingInfoFlag<LangOpts<"EnableAIXExtendedAltivecABI">>;
-def mabi_EQ_vec_default : Flag<["-"], "mabi=vec-default">, Group<m_Group>, Flags<[CC1Option]>,
-  HelpText<"Enable the default Altivec ABI on AIX (AIX only). Uses only volatile vector registers.">;
 def mabi_EQ_quadword_atomics : Flag<["-"], "mabi=quadword-atomics">,
   Group<m_Group>, Flags<[CC1Option]>,
   HelpText<"Enable quadword atomics ABI on AIX (AIX PPC64 only). Uses lqarx/stqcx. instructions.">,
@@ -3963,8 +3959,12 @@ def mno_outline_atomics : Flag<["-"], "mno-outline-atomics">, Group<f_clang_Grou
 def mno_implicit_float : Flag<["-"], "mno-implicit-float">, Group<m_Group>,
   HelpText<"Don't generate implicit floating point or vector instructions">;
 def mimplicit_float : Flag<["-"], "mimplicit-float">, Group<m_Group>;
-def mrecip : Flag<["-"], "mrecip">, Group<m_Group>;
+def mrecip : Flag<["-"], "mrecip">, Group<m_Group>,
+  HelpText<"Equivalent to '-mrecip=all'">;
 def mrecip_EQ : CommaJoined<["-"], "mrecip=">, Group<m_Group>, Flags<[CC1Option]>,
+  HelpText<"Control use of approximate reciprocal and reciprocal square root instructions followed by <n> iterations of "
+           "Newton-Raphson refinement. "
+           "<value> = ( ['!'] ['vec-'] ('rcp'|'sqrt') [('h'|'s'|'d')] [':'<n>] ) | 'all' | 'default' | 'none'">,
   MarshallingInfoStringVector<CodeGenOpts<"Reciprocals">>;
 def mprefer_vector_width_EQ : Joined<["-"], "mprefer-vector-width=">, Group<m_Group>, Flags<[CC1Option]>,
   HelpText<"Specifies preferred vector width for auto-vectorization. Defaults to 'none' which allows target specific decisions.">,
@@ -4648,6 +4648,8 @@ def m68030 : Flag<["-"], "m68030">, Group<m_m68k_Features_Group>;
 def m68040 : Flag<["-"], "m68040">, Group<m_m68k_Features_Group>;
 def m68060 : Flag<["-"], "m68060">, Group<m_m68k_Features_Group>;
 
+def m68881 : Flag<["-"], "m68881">, Group<m_m68k_Features_Group>;
+
 foreach i = {0-6} in
   def ffixed_a#i : Flag<["-"], "ffixed-a"#i>, Group<m_m68k_Features_Group>,
     HelpText<"Reserve the a"#i#" register (M68k only)">;
@@ -5629,6 +5631,9 @@ def mframe_pointer_EQ : Joined<["-"], "mframe-pointer=">,
 def mabi_EQ_ieeelongdouble : Flag<["-"], "mabi=ieeelongdouble">,
   HelpText<"Use IEEE 754 quadruple-precision for long double">,
   MarshallingInfoFlag<LangOpts<"PPCIEEELongDouble">>;
+def mabi_EQ_vec_extabi : Flag<["-"], "mabi=vec-extabi">,
+  HelpText<"Enable the extended Altivec ABI on AIX. Use volatile and nonvolatile vector registers">,
+  MarshallingInfoFlag<LangOpts<"EnableAIXExtendedAltivecABI">>;
 def mfloat_abi : Separate<["-"], "mfloat-abi">,
   HelpText<"The float ABI to use">,
   MarshallingInfoString<CodeGenOpts<"FloatABI">>;
@@ -5817,7 +5822,7 @@ def fexperimental_assignment_tracking_EQ : Joined<["-"], "fexperimental-assignme
   Group<f_Group>, CodeGenOpts<"EnableAssignmentTracking">,
   NormalizedValuesScope<"CodeGenOptions::AssignmentTrackingOpts">,
   Values<"disabled,enabled,forced">, NormalizedValues<["Disabled","Enabled","Forced"]>,
-  MarshallingInfoEnum<CodeGenOpts<"AssignmentTrackingMode">, "Disabled">;
+  MarshallingInfoEnum<CodeGenOpts<"AssignmentTrackingMode">, "Enabled">;
 
 } // let Flags = [CC1Option, NoDriverOption]
 
@@ -5830,6 +5835,9 @@ let Flags = [CC1Option, NoDriverOption] in {
 def sys_header_deps : Flag<["-"], "sys-header-deps">,
   HelpText<"Include system headers in dependency output">,
   MarshallingInfoFlag<DependencyOutputOpts<"IncludeSystemHeaders">>;
+def canonical_system_headers : Flag<["-"], "canonical-system-headers">,
+  HelpText<"Canonicalize system headers in dependency output">,
+  MarshallingInfoFlag<DependencyOutputOpts<"CanonicalSystemHeaders">>;
 def module_file_deps : Flag<["-"], "module-file-deps">,
   HelpText<"Include module files in dependency output">,
   MarshallingInfoFlag<DependencyOutputOpts<"IncludeModuleFiles">>;
@@ -6552,12 +6560,14 @@ def fno_cuda_host_device_constexpr : Flag<["-"], "fno-cuda-host-device-constexpr
 // OpenMP Options
 //===----------------------------------------------------------------------===//
 
+let Flags = [CC1Option, FC1Option, NoDriverOption] in {
+
 def fopenmp_is_device : Flag<["-"], "fopenmp-is-device">,
-  HelpText<"Generate code only for an OpenMP target device.">,
-  Flags<[CC1Option, FC1Option, NoDriverOption]>;
+  HelpText<"Generate code only for an OpenMP target device.">;
 def fopenmp_host_ir_file_path : Separate<["-"], "fopenmp-host-ir-file-path">,
-  HelpText<"Path to the IR file produced by the frontend for the host.">,
-  Flags<[CC1Option, NoDriverOption]>;
+  HelpText<"Path to the IR file produced by the frontend for the host.">;
+
+} // let Flags = [CC1Option, FC1Option, NoDriverOption]
 
 //===----------------------------------------------------------------------===//
 // SYCL Options
diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
index 57f2c413545c0..8b3721a4d7adb 100644
--- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
+++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
@@ -179,7 +179,7 @@ bool ExtractAPIVisitorBase<Derived>::VisitFunctionDecl(
       return true;
 
     // Skip methods in records.
-    for (auto P : Context.getParents(*Method)) {
+    for (const auto &P : Context.getParents(*Method)) {
       if (P.template get<CXXRecordDecl>())
         return true;
     }
diff --git a/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h b/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h
index 55c7bb32054bc..6639082bbf332 100644
--- a/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h
+++ b/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h
@@ -67,7 +67,7 @@ class SymbolGraphSerializer : public APISerializer {
   ///
   /// \returns an optional JSON Object representing the payload that libclang
   /// expects for providing symbol information for a single symbol. If this is
-  /// not a known symbol returns \c None.
+  /// not a known symbol returns \c std::nullopt.
   static std::optional<Object> serializeSingleSymbolSGF(StringRef USR,
                                                         const APISet &API);
 
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 980e6297fd6ac..40f1ae0b49d46 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -944,6 +944,39 @@ struct FormatStyle {
   /// \version 12
   BitFieldColonSpacingStyle BitFieldColonSpacing;
 
+  /// The number of columns to use to indent the contents of braced init lists.
+  /// If unset, ``ContinuationIndentWidth`` is used.
+  /// \code
+  ///   AlignAfterOpenBracket: AlwaysBreak
+  ///   BracedInitializerIndentWidth: 2
+  ///
+  ///   void f() {
+  ///     SomeClass c{
+  ///       "foo",
+  ///       "bar",
+  ///       "baz",
+  ///     };
+  ///     auto s = SomeStruct{
+  ///       .foo = "foo",
+  ///       .bar = "bar",
+  ///       .baz = "baz",
+  ///     };
+  ///     SomeArrayT a[3] = {
+  ///       {
+  ///         foo,
+  ///         bar,
+  ///       },
+  ///       {
+  ///         foo,
+  ///         bar,
+  ///       },
+  ///       SomeArrayT{},
+  ///     };
+  ///   }
+  /// \endcode
+  /// \version 17
+  std::optional<unsigned> BracedInitializerIndentWidth;
+
   /// Different ways to wrap braces after control statements.
   enum BraceWrappingAfterControlStatementStyle : int8_t {
     /// Never wrap braces after a control statement.
@@ -4286,6 +4319,7 @@ struct FormatStyle {
            BinPackArguments == R.BinPackArguments &&
            BinPackParameters == R.BinPackParameters &&
            BitFieldColonSpacing == R.BitFieldColonSpacing &&
+           BracedInitializerIndentWidth == R.BracedInitializerIndentWidth &&
            BreakAfterAttributes == R.BreakAfterAttributes &&
            BreakAfterJavaFieldAnnotations == R.BreakAfterJavaFieldAnnotations &&
            BreakArrays == R.BreakArrays &&
diff --git a/clang/include/clang/Frontend/DependencyOutputOptions.h b/clang/include/clang/Frontend/DependencyOutputOptions.h
index e0f445bb5970c..e140ff9baa117 100644
--- a/clang/include/clang/Frontend/DependencyOutputOptions.h
+++ b/clang/include/clang/Frontend/DependencyOutputOptions.h
@@ -34,6 +34,8 @@ enum ExtraDepKind {
 class DependencyOutputOptions {
 public:
   unsigned IncludeSystemHeaders : 1; ///< Include system header dependencies.
+  unsigned
+      CanonicalSystemHeaders : 1; ///< canonicalize system header dependencies.
   unsigned ShowHeaderIncludes : 1;   ///< Show header inclusions (-H).
   unsigned UsePhonyTargets : 1;      ///< Include phony targets for each
                                      /// dependency, which can avoid some 'make'
@@ -85,10 +87,11 @@ class DependencyOutputOptions {
 
 public:
   DependencyOutputOptions()
-      : IncludeSystemHeaders(0), ShowHeaderIncludes(0), UsePhonyTargets(0),
-        AddMissingHeaderDeps(0), IncludeModuleFiles(0),
-        ShowSkippedHeaderIncludes(0), HeaderIncludeFormat(HIFMT_Textual),
-        HeaderIncludeFiltering(HIFIL_None) {}
+      : IncludeSystemHeaders(0), CanonicalSystemHeaders(0),
+        ShowHeaderIncludes(0), UsePhonyTargets(0), AddMissingHeaderDeps(0),
+        IncludeModuleFiles(0), ShowSkippedHeaderIncludes(0),
+        HeaderIncludeFormat(HIFMT_Textual), HeaderIncludeFiltering(HIFIL_None) {
+  }
 };
 
 }  // end namespace clang
diff --git a/clang/include/clang/Frontend/Utils.h b/clang/include/clang/Frontend/Utils.h
index 143cf4359f00b..8300e45d15fe5 100644
--- a/clang/include/clang/Frontend/Utils.h
+++ b/clang/include/clang/Frontend/Utils.h
@@ -41,6 +41,7 @@ class ExternalSemaSource;
 class FrontendOptions;
 class PCHContainerReader;
 class Preprocessor;
+class FileManager;
 class PreprocessorOptions;
 class PreprocessorOutputOptions;
 
@@ -79,11 +80,14 @@ class DependencyCollector {
   /// Return true if system files should be passed to sawDependency().
   virtual bool needSystemDependencies() { return false; }
 
+  /// Return true if system files should be canonicalized.
+  virtual bool shouldCanonicalizeSystemDependencies() { return false; }
+
   /// Add a dependency \p Filename if it has not been seen before and
   /// sawDependency() returns true.
   virtual void maybeAddDependency(StringRef Filename, bool FromModule,
                                   bool IsSystem, bool IsModuleFile,
-                                  bool IsMissing);
+                                  FileManager *FileMgr, bool IsMissing);
 
 protected:
   /// Return true if the filename was added to the list of dependencies, false
@@ -112,6 +116,10 @@ class DependencyFileGenerator : public DependencyCollector {
   bool sawDependency(StringRef Filename, bool FromModule, bool IsSystem,
                      bool IsModuleFile, bool IsMissing) final;
 
+  bool shouldCanonicalizeSystemDependencies() override {
+    return CanonicalSystemHeaders;
+  }
+
 protected:
   void outputDependencyFile(llvm::raw_ostream &OS);
 
@@ -121,6 +129,7 @@ class DependencyFileGenerator : public DependencyCollector {
   std::string OutputFile;
   std::vector<std::string> Targets;
   bool IncludeSystemHeaders;
+  bool CanonicalSystemHeaders;
   bool PhonyTarget;
   bool AddMissingHeaderDeps;
   bool SeenMissingHeader;
diff --git a/clang/include/clang/Lex/DependencyDirectivesScanner.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h
index 529b93aa0ffbf..0e115906fbfe5 100644
--- a/clang/include/clang/Lex/DependencyDirectivesScanner.h
+++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h
@@ -68,6 +68,7 @@ enum DirectiveKind : uint8_t {
   pp_pragma_push_macro,
   pp_pragma_pop_macro,
   pp_pragma_include_alias,
+  pp_pragma_system_header,
   pp_include_next,
   pp_if,
   pp_ifdef,
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index 8c2923b0150a0..98d34b783f084 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -551,7 +551,7 @@ class Lexer : public PreprocessorLexer {
 
   /// Finds the token that comes right after the given location.
   ///
-  /// Returns the next token, or none if the location is inside a macro.
+  /// Returns the next token, or std::nullopt if the location is inside a macro.
   static std::optional<Token> findNextToken(SourceLocation Loc,
                                             const SourceManager &SM,
                                             const LangOptions &LangOpts);
diff --git a/clang/include/clang/Lex/LiteralSupport.h b/clang/include/clang/Lex/LiteralSupport.h
index fd237c2c9cd87..313af292af372 100644
--- a/clang/include/clang/Lex/LiteralSupport.h
+++ b/clang/include/clang/Lex/LiteralSupport.h
@@ -63,7 +63,7 @@ class NumericLiteralParser {
   bool isUnsigned : 1;
   bool isLong : 1;          // This is *not* set for long long.
   bool isLongLong : 1;
-  bool isSizeT : 1;         // 1z, 1uz (C++2b)
+  bool isSizeT : 1;         // 1z, 1uz (C++23)
   bool isHalf : 1;          // 1.0h
   bool isFloat : 1;         // 1.0f
   bool isImaginary : 1;     // 1.0i
diff --git a/clang/include/clang/Lex/MultipleIncludeOpt.h b/clang/include/clang/Lex/MultipleIncludeOpt.h
index 7ceb7e53c75d7..8e570226c4b27 100644
--- a/clang/include/clang/Lex/MultipleIncludeOpt.h
+++ b/clang/include/clang/Lex/MultipleIncludeOpt.h
@@ -108,6 +108,12 @@ class MultipleIncludeOpt {
     ImmediatelyAfterTopLevelIfndef = false;
   }
 
+  /// SetReadToken - Set whether the value of 'ReadAnyTokens'.  Called to
+  /// override when encountering tokens outside of the include guard that have
+  /// no effect if the file in question is is included multiple times (e.g. the
+  /// null directive).
+  void SetReadToken(bool Value) { ReadAnyTokens = Value; }
+
   /// ExpandedMacro - When a macro is expanded with this lexer as the current
   /// buffer, this method is called to disable the MIOpt if needed.
   void ExpandedMacro() { DidMacroExpansion = true; }
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index a34a451f4bad1..8bdaf25e9b870 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -729,7 +729,7 @@ class Preprocessor {
   /// Only one of CurLexer, or CurTokenLexer will be non-null.
   std::unique_ptr<Lexer> CurLexer;
 
-  /// The current top of the stack what we're lexing from
+  /// The current top of the stack that we're lexing from
   /// if not expanding a macro.
   ///
   /// This is an alias for CurLexer.
diff --git a/clang/include/clang/Sema/EnterExpressionEvaluationContext.h b/clang/include/clang/Sema/EnterExpressionEvaluationContext.h
new file mode 100644
index 0000000000000..5eca797b8842b
--- /dev/null
+++ b/clang/include/clang/Sema/EnterExpressionEvaluationContext.h
@@ -0,0 +1,69 @@
+//===--- EnterExpressionEvaluationContext.h ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_ENTEREXPRESSIONEVALUATIONCONTEXT_H
+#define LLVM_CLANG_SEMA_ENTEREXPRESSIONEVALUATIONCONTEXT_H
+
+#include "clang/Sema/Sema.h"
+
+namespace clang {
+
+class Decl;
+
+/// RAII object that enters a new expression evaluation context.
+class EnterExpressionEvaluationContext {
+  Sema &Actions;
+  bool Entered = true;
+
+public:
+  EnterExpressionEvaluationContext(
+      Sema &Actions, Sema::ExpressionEvaluationContext NewContext,
+      Decl *LambdaContextDecl = nullptr,
+      Sema::ExpressionEvaluationContextRecord::ExpressionKind ExprContext =
+          Sema::ExpressionEvaluationContextRecord::EK_Other,
+      bool ShouldEnter = true)
+      : Actions(Actions), Entered(ShouldEnter) {
+    if (Entered)
+      Actions.PushExpressionEvaluationContext(NewContext, LambdaContextDecl,
+                                              ExprContext);
+  }
+  EnterExpressionEvaluationContext(
+      Sema &Actions, Sema::ExpressionEvaluationContext NewContext,
+      Sema::ReuseLambdaContextDecl_t,
+      Sema::ExpressionEvaluationContextRecord::ExpressionKind ExprContext =
+          Sema::ExpressionEvaluationContextRecord::EK_Other)
+      : Actions(Actions) {
+    Actions.PushExpressionEvaluationContext(
+        NewContext, Sema::ReuseLambdaContextDecl, ExprContext);
+  }
+
+  enum InitListTag { InitList };
+  EnterExpressionEvaluationContext(Sema &Actions, InitListTag,
+                                   bool ShouldEnter = true)
+      : Actions(Actions), Entered(false) {
+    // In C++11 onwards, narrowing checks are performed on the contents of
+    // braced-init-lists, even when they occur within unevaluated operands.
+    // Therefore we still need to instantiate constexpr functions used in such
+    // a context.
+    if (ShouldEnter && Actions.isUnevaluatedContext() &&
+        Actions.getLangOpts().CPlusPlus11) {
+      Actions.PushExpressionEvaluationContext(
+          Sema::ExpressionEvaluationContext::UnevaluatedList);
+      Entered = true;
+    }
+  }
+
+  ~EnterExpressionEvaluationContext() {
+    if (Entered)
+      Actions.PopExpressionEvaluationContext();
+  }
+};
+
+} // namespace clang
+
+#endif
diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h
index fcfb56f9731ef..2072cd8d1c3ef 100644
--- a/clang/include/clang/Sema/Initialization.h
+++ b/clang/include/clang/Sema/Initialization.h
@@ -123,6 +123,10 @@ class alignas(8) InitializedEntity {
     /// decomposition declaration.
     EK_Binding,
 
+    /// The entity being initialized is a non-static data member subobject of an
+    /// object initialized via parenthesized aggregate initialization.
+    EK_ParenAggInitMember,
+
     // Note: err_init_conversion_failed in DiagnosticSemaKinds.td uses this
     // enum as an index for its first %select.  When modifying this list,
     // that diagnostic text needs to be updated as well.
@@ -227,8 +231,10 @@ class alignas(8) InitializedEntity {
 
   /// Create the initialization entity for a member subobject.
   InitializedEntity(FieldDecl *Member, const InitializedEntity *Parent,
-                    bool Implicit, bool DefaultMemberInit)
-      : Kind(EK_Member), Parent(Parent), Type(Member->getType()),
+                    bool Implicit, bool DefaultMemberInit,
+                    bool IsParenAggInit = false)
+      : Kind(IsParenAggInit ? EK_ParenAggInitMember : EK_Member),
+        Parent(Parent), Type(Member->getType()),
         Variable{Member, Implicit, DefaultMemberInit} {}
 
   /// Create the initialization entity for an array element.
@@ -388,6 +394,14 @@ class alignas(8) InitializedEntity {
     return InitializedEntity(Member->getAnonField(), Parent, Implicit, false);
   }
 
+  /// Create the initialization entity for a member subobject initialized via
+  /// parenthesized aggregate init.
+  static InitializedEntity InitializeMemberFromParenAggInit(FieldDecl *Member) {
+    return InitializedEntity(Member, /*Parent=*/nullptr, /*Implicit=*/false,
+                             /*DefaultMemberInit=*/false,
+                             /*IsParenAggInit=*/true);
+  }
+
   /// Create the initialization entity for a default member initializer.
   static InitializedEntity
   InitializeMemberFromDefaultMemberInitializer(FieldDecl *Member) {
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 1827ea5d1e546..a97968dc7b209 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -162,6 +162,9 @@ class Sema;
     /// Arm SVE Vector conversions
     ICK_SVE_Vector_Conversion,
 
+    /// RISC-V RVV Vector conversions
+    ICK_RVV_Vector_Conversion,
+
     /// A vector splat from an arithmetic type
     ICK_Vector_Splat,
 
diff --git a/clang/include/clang/Sema/RISCVIntrinsicManager.h b/clang/include/clang/Sema/RISCVIntrinsicManager.h
index 128858bb43019..731bc3ae1c9d2 100644
--- a/clang/include/clang/Sema/RISCVIntrinsicManager.h
+++ b/clang/include/clang/Sema/RISCVIntrinsicManager.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_CLANG_SEMA_RISCVINTRINSICMANAGER_H
 #define LLVM_CLANG_SEMA_RISCVINTRINSICMANAGER_H
 
+#include <cstdint>
+
 namespace clang {
 class LookupResult;
 class IdentifierInfo;
@@ -22,6 +24,8 @@ class Preprocessor;
 namespace sema {
 class RISCVIntrinsicManager {
 public:
+  enum class IntrinsicKind : uint8_t { RVV, SIFIVE_VECTOR };
+
   virtual ~RISCVIntrinsicManager() = default;
 
   // Create RISC-V intrinsic and insert into symbol table and return true if
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 0ab5dd9e8bb64..92f3e6f0ff848 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -946,10 +946,10 @@ class Sema final {
   class DelayedDiagnostics {
     /// The current pool of diagnostics into which delayed
     /// diagnostics should go.
-    sema::DelayedDiagnosticPool *CurPool;
+    sema::DelayedDiagnosticPool *CurPool = nullptr;
 
   public:
-    DelayedDiagnostics() : CurPool(nullptr) {}
+    DelayedDiagnostics() = default;
 
     /// Adds a delayed diagnostic.
     void add(const sema::DelayedDiagnostic &diag); // in DelayedDiagnostic.h
@@ -1370,7 +1370,7 @@ class Sema final {
       return Context == ExpressionEvaluationContext::ImmediateFunctionContext ||
              (Context == ExpressionEvaluationContext::DiscardedStatement &&
               InImmediateFunctionContext) ||
-             // C++2b [expr.const]p14:
+             // C++23 [expr.const]p14:
              // An expression or conversion is in an immediate function
              // context if it is potentially evaluated and either:
              //   * its innermost enclosing non-block scope is a function
@@ -1621,6 +1621,9 @@ class Sema final {
   /// Indicate RISC-V vector builtin functions enabled or not.
   bool DeclareRISCVVBuiltins = false;
 
+  /// Indicate RISC-V Sifive vector builtin functions enabled or not.
+  bool DeclareRISCVVectorBuiltins = false;
+
 private:
   std::unique_ptr<sema::RISCVIntrinsicManager> RVIntrinsicManager;
 
@@ -2588,13 +2591,11 @@ class Sema final {
   //
 
   struct SkipBodyInfo {
-    SkipBodyInfo()
-        : ShouldSkip(false), CheckSameAsPrevious(false), Previous(nullptr),
-          New(nullptr) {}
-    bool ShouldSkip;
-    bool CheckSameAsPrevious;
-    NamedDecl *Previous;
-    NamedDecl *New;
+    SkipBodyInfo() = default;
+    bool ShouldSkip = false;
+    bool CheckSameAsPrevious = false;
+    NamedDecl *Previous = nullptr;
+    NamedDecl *New = nullptr;
   };
 
   DeclGroupPtrTy ConvertDeclToDeclGroup(Decl *Ptr, Decl *OwnedType = nullptr);
@@ -12685,6 +12686,7 @@ class Sema final {
                                        SourceLocation Loc, bool IsCompAssign);
 
   bool isValidSveBitcast(QualType srcType, QualType destType);
+  bool isValidRVVBitcast(QualType srcType, QualType destType);
 
   bool areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy);
 
@@ -13971,55 +13973,6 @@ class Sema final {
                                   ValueDecl *DeclToCheck);
 };
 
-/// RAII object that enters a new expression evaluation context.
-class EnterExpressionEvaluationContext {
-  Sema &Actions;
-  bool Entered = true;
-
-public:
-  EnterExpressionEvaluationContext(
-      Sema &Actions, Sema::ExpressionEvaluationContext NewContext,
-      Decl *LambdaContextDecl = nullptr,
-      Sema::ExpressionEvaluationContextRecord::ExpressionKind ExprContext =
-          Sema::ExpressionEvaluationContextRecord::EK_Other,
-      bool ShouldEnter = true)
-      : Actions(Actions), Entered(ShouldEnter) {
-    if (Entered)
-      Actions.PushExpressionEvaluationContext(NewContext, LambdaContextDecl,
-                                              ExprContext);
-  }
-  EnterExpressionEvaluationContext(
-      Sema &Actions, Sema::ExpressionEvaluationContext NewContext,
-      Sema::ReuseLambdaContextDecl_t,
-      Sema::ExpressionEvaluationContextRecord::ExpressionKind ExprContext =
-          Sema::ExpressionEvaluationContextRecord::EK_Other)
-      : Actions(Actions) {
-    Actions.PushExpressionEvaluationContext(
-        NewContext, Sema::ReuseLambdaContextDecl, ExprContext);
-  }
-
-  enum InitListTag { InitList };
-  EnterExpressionEvaluationContext(Sema &Actions, InitListTag,
-                                   bool ShouldEnter = true)
-      : Actions(Actions), Entered(false) {
-    // In C++11 onwards, narrowing checks are performed on the contents of
-    // braced-init-lists, even when they occur within unevaluated operands.
-    // Therefore we still need to instantiate constexpr functions used in such
-    // a context.
-    if (ShouldEnter && Actions.isUnevaluatedContext() &&
-        Actions.getLangOpts().CPlusPlus11) {
-      Actions.PushExpressionEvaluationContext(
-          Sema::ExpressionEvaluationContext::UnevaluatedList);
-      Entered = true;
-    }
-  }
-
-  ~EnterExpressionEvaluationContext() {
-    if (Entered)
-      Actions.PopExpressionEvaluationContext();
-  }
-};
-
 DeductionFailureInfo
 MakeDeductionFailureInfo(ASTContext &Context, Sema::TemplateDeductionResult TDK,
                          sema::TemplateDeductionInfo &Info);
diff --git a/clang/include/clang/Serialization/PCHContainerOperations.h b/clang/include/clang/Serialization/PCHContainerOperations.h
index 9f9700a418a9c..be10feb5e351c 100644
--- a/clang/include/clang/Serialization/PCHContainerOperations.h
+++ b/clang/include/clang/Serialization/PCHContainerOperations.h
@@ -56,7 +56,7 @@ class PCHContainerReader {
 public:
   virtual ~PCHContainerReader() = 0;
   /// Equivalent to the format passed to -fmodule-format=
-  virtual llvm::StringRef getFormat() const = 0;
+  virtual llvm::ArrayRef<llvm::StringRef> getFormats() const = 0;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   virtual llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const = 0;
@@ -78,8 +78,7 @@ class RawPCHContainerWriter : public PCHContainerWriter {
 
 /// Implements read operations for a raw pass-through PCH container.
 class RawPCHContainerReader : public PCHContainerReader {
-  llvm::StringRef getFormat() const override { return "raw"; }
-
+  llvm::ArrayRef<llvm::StringRef> getFormats() const override;
   /// Simply returns the buffer contained in Buffer.
   llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;
 };
@@ -87,7 +86,9 @@ class RawPCHContainerReader : public PCHContainerReader {
 /// A registry of PCHContainerWriter and -Reader objects for different formats.
 class PCHContainerOperations {
   llvm::StringMap<std::unique_ptr<PCHContainerWriter>> Writers;
-  llvm::StringMap<std::unique_ptr<PCHContainerReader>> Readers;
+  llvm::StringMap<PCHContainerReader *> Readers;
+  llvm::SmallVector<std::unique_ptr<PCHContainerReader>> OwnedReaders;
+
 public:
   /// Automatically registers a RawPCHContainerWriter and
   /// RawPCHContainerReader.
@@ -96,13 +97,17 @@ class PCHContainerOperations {
     Writers[Writer->getFormat()] = std::move(Writer);
   }
   void registerReader(std::unique_ptr<PCHContainerReader> Reader) {
-    Readers[Reader->getFormat()] = std::move(Reader);
+    assert(!Reader->getFormats().empty() &&
+           "PCHContainerReader must handle >=1 format");
+    for (llvm::StringRef Fmt : Reader->getFormats())
+      Readers[Fmt] = Reader.get();
+    OwnedReaders.push_back(std::move(Reader));
   }
   const PCHContainerWriter *getWriterOrNull(llvm::StringRef Format) {
     return Writers[Format].get();
   }
   const PCHContainerReader *getReaderOrNull(llvm::StringRef Format) {
-    return Readers[Format].get();
+    return Readers[Format];
   }
   const PCHContainerReader &getRawReader() {
     return *getReaderOrNull("raw");
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Taint.h b/clang/include/clang/StaticAnalyzer/Checkers/Taint.h
index df863a2495413..3ec8dbfb09ee3 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Taint.h
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Taint.h
@@ -79,26 +79,48 @@ bool isTainted(ProgramStateRef State, SymbolRef Sym,
 bool isTainted(ProgramStateRef State, const MemRegion *Reg,
                TaintTagType Kind = TaintTagGeneric);
 
+/// Returns the tainted Symbols for a given Statement and state.
+std::vector<SymbolRef> getTaintedSymbols(ProgramStateRef State, const Stmt *S,
+                                         const LocationContext *LCtx,
+                                         TaintTagType Kind = TaintTagGeneric);
+
+/// Returns the tainted Symbols for a given SVal and state.
+std::vector<SymbolRef> getTaintedSymbols(ProgramStateRef State, SVal V,
+                                         TaintTagType Kind = TaintTagGeneric);
+
+/// Returns the tainted Symbols for a SymbolRef and state.
+std::vector<SymbolRef> getTaintedSymbols(ProgramStateRef State, SymbolRef Sym,
+                                         TaintTagType Kind = TaintTagGeneric);
+
+/// Returns the tainted (index, super/sub region, symbolic region) symbols
+/// for a given memory region.
+std::vector<SymbolRef> getTaintedSymbols(ProgramStateRef State,
+                                         const MemRegion *Reg,
+                                         TaintTagType Kind = TaintTagGeneric);
+
+std::vector<SymbolRef> getTaintedSymbolsImpl(ProgramStateRef State,
+                                             const Stmt *S,
+                                             const LocationContext *LCtx,
+                                             TaintTagType Kind,
+                                             bool returnFirstOnly);
+
+std::vector<SymbolRef> getTaintedSymbolsImpl(ProgramStateRef State, SVal V,
+                                             TaintTagType Kind,
+                                             bool returnFirstOnly);
+
+std::vector<SymbolRef> getTaintedSymbolsImpl(ProgramStateRef State,
+                                             SymbolRef Sym, TaintTagType Kind,
+                                             bool returnFirstOnly);
+
+std::vector<SymbolRef> getTaintedSymbolsImpl(ProgramStateRef State,
+                                             const MemRegion *Reg,
+                                             TaintTagType Kind,
+                                             bool returnFirstOnly);
+
 void printTaint(ProgramStateRef State, raw_ostream &Out, const char *nl = "\n",
                 const char *sep = "");
 
 LLVM_DUMP_METHOD void dumpTaint(ProgramStateRef State);
-
-/// The bug visitor prints a diagnostic message at the location where a given
-/// variable was tainted.
-class TaintBugVisitor final : public BugReporterVisitor {
-private:
-  const SVal V;
-
-public:
-  TaintBugVisitor(const SVal V) : V(V) {}
-  void Profile(llvm::FoldingSetNodeID &ID) const override { ID.Add(V); }
-
-  PathDiagnosticPieceRef VisitNode(const ExplodedNode *N,
-                                   BugReporterContext &BRC,
-                                   PathSensitiveBugReport &BR) override;
-};
-
 } // namespace taint
 } // namespace ento
 } // namespace clang
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
index 392bc484bf62c..5d2c96e5bc9de 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
@@ -22,6 +22,7 @@ extern const char *const CXXObjectLifecycle;
 extern const char *const CXXMoveSemantics;
 extern const char *const SecurityError;
 extern const char *const UnusedCode;
+extern const char *const TaintedData;
 } // namespace categories
 } // namespace ento
 } // namespace clang
diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap
deleted file mode 100644
index 624045dc4c57b..0000000000000
--- a/clang/include/clang/module.modulemap
+++ /dev/null
@@ -1,199 +0,0 @@
-module Clang_Analysis {
-  requires cplusplus
-  umbrella "Analysis"
-
-  textual header "Analysis/Analyses/ThreadSafetyOps.def"
-
-  module * { export * }
-
-  // FIXME: Exclude these headers to avoid pulling all of the AST matchers
-  // library into clang. Due to inline key functions in the headers,
-  // importing the AST matchers library gives a link dependency on the AST
-  // matchers (and thus the AST), which clang-format should not have.
-  exclude header "Analysis/Analyses/ExprMutationAnalyzer.h"
-}
-
-module Clang_AST {
-  requires cplusplus
-  umbrella "AST"
-
-  textual header "AST/BuiltinTypes.def"
-  textual header "AST/CXXRecordDeclDefinitionBits.def"
-  textual header "AST/OperationKinds.def"
-  textual header "AST/TypeLocNodes.def"
-
-  module * { export * }
-}
-
-module Clang_ASTMatchers { requires cplusplus umbrella "ASTMatchers" module * { export * } }
-
-module Clang_Basic {
-  requires cplusplus
-  umbrella "Basic"
-
-  textual header "Basic/AArch64SVEACLETypes.def"
-  textual header "Basic/BuiltinsAArch64.def"
-  textual header "Basic/BuiltinsAMDGPU.def"
-  textual header "Basic/BuiltinsAArch64NeonSVEBridge.def"
-  textual header "Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
-  textual header "Basic/BuiltinsARM.def"
-  textual header "Basic/BuiltinsBPF.def"
-  textual header "Basic/Builtins.def"
-  textual header "Basic/BuiltinHeaders.def"
-  textual header "Basic/BuiltinsHexagon.def"
-  textual header "Basic/BuiltinsHexagonDep.def"
-  textual header "Basic/BuiltinsHexagonMapCustomDep.def"
-  textual header "Basic/BuiltinsLoongArch.def"
-  textual header "Basic/BuiltinsMips.def"
-  textual header "Basic/BuiltinsNEON.def"
-  textual header "Basic/BuiltinsNVPTX.def"
-  textual header "Basic/BuiltinsPPC.def"
-  textual header "Basic/BuiltinsRISCV.def"
-  textual header "Basic/BuiltinsRISCVVector.def"
-  textual header "Basic/BuiltinsSVE.def"
-  textual header "Basic/BuiltinsSystemZ.def"
-  textual header "Basic/BuiltinsVE.def"
-  textual header "Basic/BuiltinsVEVL.gen.def"
-  textual header "Basic/BuiltinsWebAssembly.def"
-  textual header "Basic/BuiltinsX86.def"
-  textual header "Basic/BuiltinsX86_64.def"
-  textual header "Basic/BuiltinsXCore.def"
-  textual header "Basic/CodeGenOptions.def"
-  textual header "Basic/DiagnosticOptions.def"
-  textual header "Basic/Features.def"
-  textual header "Basic/FPOptions.def"
-  textual header "Basic/MSP430Target.def"
-  textual header "Basic/LangOptions.def"
-  textual header "Basic/OpenCLExtensions.def"
-  textual header "Basic/OpenCLImageTypes.def"
-  textual header "Basic/OpenCLExtensionTypes.def"
-  textual header "Basic/OpenMPKinds.def"
-  textual header "Basic/OperatorKinds.def"
-  textual header "Basic/PPCTypes.def"
-  textual header "Basic/RISCVVTypes.def"
-  textual header "Basic/Sanitizers.def"
-  textual header "Basic/TargetCXXABI.def"
-  textual header "Basic/TransformTypeTraits.def"
-  textual header "Basic/TokenKinds.def"
-  textual header "Basic/WebAssemblyReferenceTypes.def"
-
-  module * { export * }
-}
-module Clang_Basic_TokenKinds {
-  requires cplusplus
-
-  header "Basic/TokenKinds.h"
-  textual header "Basic/TokenKinds.def"
-
-  export *
-}
-
-module Clang_CodeGen { requires cplusplus umbrella "CodeGen" module * { export * } }
-module Clang_Config { requires cplusplus umbrella "Config" module * { export * } }
-
-// Files for diagnostic groups are spread all over the include/clang/ tree, but
-// logically form a single module.
-module Clang_Diagnostics {
-  requires cplusplus
-
-  module All { header "Basic/AllDiagnostics.h" export * }
-  module Analysis { textual header "Analysis/Analyses/UnsafeBufferUsageGadgets.def" }
-  module AST { header "AST/ASTDiagnostic.h" export * }
-  module Comment { header "AST/CommentDiagnostic.h" export * }
-  module Driver { header "Driver/DriverDiagnostic.h" export * }
-  module Frontend { header "Frontend/FrontendDiagnostic.h" export * }
-  module Lex { header "Lex/LexDiagnostic.h" export * }
-  module Parse { header "Parse/ParseDiagnostic.h" export * }
-  module Sema { header "Sema/SemaDiagnostic.h" export * }
-  module Serialization { header "Serialization/SerializationDiagnostic.h" export * }
-  module Refactoring { header "Tooling/Refactoring/RefactoringDiagnostic.h" export * }
-}
-
-module Clang_Driver {
-  requires cplusplus
-  umbrella "Driver"
-
-  textual header "Driver/Types.def"
-
-  module * { export * }
-}
-
-module Clang_Edit { requires cplusplus umbrella "Edit" module * { export * } }
-module Clang_Format { requires cplusplus umbrella "Format" module * { export * } }
-
-module Clang_Frontend {
-  requires cplusplus
-  umbrella "Frontend"
-
-  textual header "Basic/LangStandards.def"
-
-  module * { export * }
-}
-
-module Clang_FrontendTool { requires cplusplus umbrella "FrontendTool" module * { export * } }
-module Clang_Index { requires cplusplus umbrella "Index" module * { export * } }
-module Clang_Lex { requires cplusplus umbrella "Lex" module * { export * } }
-module Clang_Parse { requires cplusplus umbrella "Parse" module * { export * } }
-module Clang_Rewrite { requires cplusplus umbrella "Rewrite/Core" module * { export * } }
-module Clang_RewriteFrontend { requires cplusplus umbrella "Rewrite/Frontend" module * { export * } }
-module Clang_Sema { requires cplusplus umbrella "Sema" module * { export * } }
-
-module Clang_Serialization {
-  requires cplusplus
-  umbrella "Serialization"
-
-  textual header "Serialization/TypeBitCodes.def"
-
-  module * { export * }
-}
-
-module Clang_StaticAnalyzer_Core {
-  requires cplusplus
-  umbrella "StaticAnalyzer/Core"
-
-  textual header "StaticAnalyzer/Core/Analyses.def"
-  textual header "StaticAnalyzer/Core/AnalyzerOptions.def"
-  textual header "StaticAnalyzer/Core/PathSensitive/SVals.def"
-  textual header "StaticAnalyzer/Core/PathSensitive/Symbols.def"
-  textual header "StaticAnalyzer/Core/PathSensitive/Regions.def"
-
-  module * { export * }
-}
-
-module Clang_StaticAnalyzer_Checkers {
-  requires cplusplus
-  umbrella "StaticAnalyzer/Checkers"
-  module * { export * }
-}
-
-module Clang_StaticAnalyzer_Frontend {
-  requires cplusplus
-  umbrella "StaticAnalyzer/Frontend"
-  module * { export * }
-}
-
-module Clang_Testing {
-  requires cplusplus
-  umbrella "Testing"
-  module * { export * }
-}
-
-module Clang_Tooling {
-  requires cplusplus umbrella "Tooling" module * { export * }
-  // FIXME: Exclude these headers to avoid pulling all of the AST matchers
-  // library into clang-format. Due to inline key functions in the headers,
-  // importing the AST matchers library gives a link dependency on the AST
-  // matchers (and thus the AST), which clang-format should not have.
-  exclude header "Tooling/RefactoringCallbacks.h"
-}
-
-module Clang_ToolingCore {
-  requires cplusplus
-  umbrella "Tooling/Core" module * { export * }
-}
-
-module Clang_ToolingInclusions {
-  requires cplusplus
-  umbrella "Tooling/Inclusions"
-  module * { export * }
-}
diff --git a/clang/include/module.modulemap b/clang/include/module.modulemap
new file mode 100644
index 0000000000000..9fea3620100f4
--- /dev/null
+++ b/clang/include/module.modulemap
@@ -0,0 +1,204 @@
+module Clang_C {
+  umbrella "clang-c"
+  module * { export * }
+}
+
+module Clang_Analysis {
+  requires cplusplus
+  umbrella "clang/Analysis"
+
+  textual header "clang/Analysis/Analyses/ThreadSafetyOps.def"
+
+  module * { export * }
+
+  // FIXME: Exclude these headers to avoid pulling all of the AST matchers
+  // library into clang. Due to inline key functions in the headers,
+  // importing the AST matchers library gives a link dependency on the AST
+  // matchers (and thus the AST), which clang-format should not have.
+  exclude header "clang/Analysis/Analyses/ExprMutationAnalyzer.h"
+}
+
+module Clang_AST {
+  requires cplusplus
+  umbrella "clang/AST"
+
+  textual header "clang/AST/BuiltinTypes.def"
+  textual header "clang/AST/CXXRecordDeclDefinitionBits.def"
+  textual header "clang/AST/OperationKinds.def"
+  textual header "clang/AST/TypeLocNodes.def"
+
+  module * { export * }
+}
+
+module Clang_ASTMatchers { requires cplusplus umbrella "clang/ASTMatchers" module * { export * } }
+
+module Clang_Basic {
+  requires cplusplus
+  umbrella "clang/Basic"
+
+  textual header "clang/Basic/AArch64SVEACLETypes.def"
+  textual header "clang/Basic/BuiltinsAArch64.def"
+  textual header "clang/Basic/BuiltinsAMDGPU.def"
+  textual header "clang/Basic/BuiltinsAArch64NeonSVEBridge.def"
+  textual header "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
+  textual header "clang/Basic/BuiltinsARM.def"
+  textual header "clang/Basic/BuiltinsBPF.def"
+  textual header "clang/Basic/Builtins.def"
+  textual header "clang/Basic/BuiltinHeaders.def"
+  textual header "clang/Basic/BuiltinsHexagon.def"
+  textual header "clang/Basic/BuiltinsHexagonDep.def"
+  textual header "clang/Basic/BuiltinsHexagonMapCustomDep.def"
+  textual header "clang/Basic/BuiltinsLoongArch.def"
+  textual header "clang/Basic/BuiltinsMips.def"
+  textual header "clang/Basic/BuiltinsNEON.def"
+  textual header "clang/Basic/BuiltinsNVPTX.def"
+  textual header "clang/Basic/BuiltinsPPC.def"
+  textual header "clang/Basic/BuiltinsRISCV.def"
+  textual header "clang/Basic/BuiltinsRISCVVector.def"
+  textual header "clang/Basic/BuiltinsSVE.def"
+  textual header "clang/Basic/BuiltinsSystemZ.def"
+  textual header "clang/Basic/BuiltinsVE.def"
+  textual header "clang/Basic/BuiltinsVEVL.gen.def"
+  textual header "clang/Basic/BuiltinsWebAssembly.def"
+  textual header "clang/Basic/BuiltinsX86.def"
+  textual header "clang/Basic/BuiltinsX86_64.def"
+  textual header "clang/Basic/BuiltinsXCore.def"
+  textual header "clang/Basic/CodeGenOptions.def"
+  textual header "clang/Basic/DiagnosticOptions.def"
+  textual header "clang/Basic/Features.def"
+  textual header "clang/Basic/FPOptions.def"
+  textual header "clang/Basic/MSP430Target.def"
+  textual header "clang/Basic/LangOptions.def"
+  textual header "clang/Basic/OpenCLExtensions.def"
+  textual header "clang/Basic/OpenCLImageTypes.def"
+  textual header "clang/Basic/OpenCLExtensionTypes.def"
+  textual header "clang/Basic/OpenMPKinds.def"
+  textual header "clang/Basic/OperatorKinds.def"
+  textual header "clang/Basic/PPCTypes.def"
+  textual header "clang/Basic/RISCVVTypes.def"
+  textual header "clang/Basic/Sanitizers.def"
+  textual header "clang/Basic/TargetCXXABI.def"
+  textual header "clang/Basic/TransformTypeTraits.def"
+  textual header "clang/Basic/TokenKinds.def"
+  textual header "clang/Basic/WebAssemblyReferenceTypes.def"
+
+  module * { export * }
+}
+module Clang_Basic_TokenKinds {
+  requires cplusplus
+
+  header "clang/Basic/TokenKinds.h"
+  textual header "clang/Basic/TokenKinds.def"
+
+  export *
+}
+
+module Clang_CodeGen { requires cplusplus umbrella "clang/CodeGen" module * { export * } }
+module Clang_Config { requires cplusplus umbrella "clang/Config" module * { export * } }
+
+// Files for diagnostic groups are spread all over the include/clang/ tree, but
+// logically form a single module.
+module Clang_Diagnostics {
+  requires cplusplus
+
+  module All { header "clang/Basic/AllDiagnostics.h" export * }
+  module Analysis { textual header "clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def" }
+  module AST { header "clang/AST/ASTDiagnostic.h" export * }
+  module Comment { header "clang/AST/CommentDiagnostic.h" export * }
+  module Driver { header "clang/Driver/DriverDiagnostic.h" export * }
+  module Frontend { header "clang/Frontend/FrontendDiagnostic.h" export * }
+  module Lex { header "clang/Lex/LexDiagnostic.h" export * }
+  module Parse { header "clang/Parse/ParseDiagnostic.h" export * }
+  module Sema { header "clang/Sema/SemaDiagnostic.h" export * }
+  module Serialization { header "clang/Serialization/SerializationDiagnostic.h" export * }
+  module Refactoring { header "clang/Tooling/Refactoring/RefactoringDiagnostic.h" export * }
+}
+
+module Clang_Driver {
+  requires cplusplus
+  umbrella "clang/Driver"
+
+  textual header "clang/Driver/Types.def"
+
+  module * { export * }
+}
+
+module Clang_Edit { requires cplusplus umbrella "clang/Edit" module * { export * } }
+module Clang_Format { requires cplusplus umbrella "clang/Format" module * { export * } }
+
+module Clang_Frontend {
+  requires cplusplus
+  umbrella "clang/Frontend"
+
+  textual header "clang/Basic/LangStandards.def"
+
+  module * { export * }
+}
+
+module Clang_FrontendTool { requires cplusplus umbrella "clang/FrontendTool" module * { export * } }
+module Clang_Index { requires cplusplus umbrella "clang/Index" module * { export * } }
+module Clang_Lex { requires cplusplus umbrella "clang/Lex" module * { export * } }
+module Clang_Parse { requires cplusplus umbrella "clang/Parse" module * { export * } }
+module Clang_Rewrite { requires cplusplus umbrella "clang/Rewrite/Core" module * { export * } }
+module Clang_RewriteFrontend { requires cplusplus umbrella "clang/Rewrite/Frontend" module * { export * } }
+module Clang_Sema { requires cplusplus umbrella "clang/Sema" module * { export * } }
+
+module Clang_Serialization {
+  requires cplusplus
+  umbrella "clang/Serialization"
+
+  textual header "clang/Serialization/TypeBitCodes.def"
+
+  module * { export * }
+}
+
+module Clang_StaticAnalyzer_Core {
+  requires cplusplus
+  umbrella "clang/StaticAnalyzer/Core"
+
+  textual header "clang/StaticAnalyzer/Core/Analyses.def"
+  textual header "clang/StaticAnalyzer/Core/AnalyzerOptions.def"
+  textual header "clang/StaticAnalyzer/Core/PathSensitive/SVals.def"
+  textual header "clang/StaticAnalyzer/Core/PathSensitive/Symbols.def"
+  textual header "clang/StaticAnalyzer/Core/PathSensitive/Regions.def"
+
+  module * { export * }
+}
+
+module Clang_StaticAnalyzer_Checkers {
+  requires cplusplus
+  umbrella "clang/StaticAnalyzer/Checkers"
+  module * { export * }
+}
+
+module Clang_StaticAnalyzer_Frontend {
+  requires cplusplus
+  umbrella "clang/StaticAnalyzer/Frontend"
+  module * { export * }
+}
+
+module Clang_Testing {
+  requires cplusplus
+  umbrella "clang/Testing"
+  module * { export * }
+}
+
+module Clang_Tooling {
+  requires cplusplus umbrella "clang/Tooling" module * { export * }
+  // FIXME: Exclude these headers to avoid pulling all of the AST matchers
+  // library into clang-format. Due to inline key functions in the headers,
+  // importing the AST matchers library gives a link dependency on the AST
+  // matchers (and thus the AST), which clang-format should not have.
+  exclude header "clang/Tooling/RefactoringCallbacks.h"
+}
+
+module Clang_ToolingCore {
+  requires cplusplus
+  umbrella "clang/Tooling/Core" module * { export * }
+}
+
+module Clang_ToolingInclusions {
+  requires cplusplus
+  umbrella "clang/Tooling/Inclusions"
+  module * { export * }
+}
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index e331df86235b2..3a72c3d257944 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -85,6 +85,7 @@
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include <algorithm>
 #include <cassert>
@@ -2010,6 +2011,9 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     else if (VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
       // Adjust the alignment for fixed-length SVE predicates.
       Align = 16;
+    else if (VT->getVectorKind() == VectorType::RVVFixedLengthDataVector)
+      // Adjust the alignment for fixed-length RVV vectors.
+      Align = 64;
     break;
   }
 
@@ -9468,7 +9472,9 @@ bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
       First->getVectorKind() != VectorType::SveFixedLengthDataVector &&
       First->getVectorKind() != VectorType::SveFixedLengthPredicateVector &&
       Second->getVectorKind() != VectorType::SveFixedLengthDataVector &&
-      Second->getVectorKind() != VectorType::SveFixedLengthPredicateVector)
+      Second->getVectorKind() != VectorType::SveFixedLengthPredicateVector &&
+      First->getVectorKind() != VectorType::RVVFixedLengthDataVector &&
+      Second->getVectorKind() != VectorType::RVVFixedLengthDataVector)
     return true;
 
   return false;
@@ -9566,6 +9572,85 @@ bool ASTContext::areLaxCompatibleSveTypes(QualType FirstType,
          IsLaxCompatible(SecondType, FirstType);
 }
 
+/// getRVVTypeSize - Return RVV vector register size.
+static uint64_t getRVVTypeSize(ASTContext &Context, const BuiltinType *Ty) {
+  assert(Ty->isRVVVLSBuiltinType() && "Invalid RVV Type");
+  auto VScale = Context.getTargetInfo().getVScaleRange(Context.getLangOpts());
+  return VScale ? VScale->first * llvm::RISCV::RVVBitsPerBlock : 0;
+}
+
+bool ASTContext::areCompatibleRVVTypes(QualType FirstType,
+                                       QualType SecondType) {
+  assert(
+      ((FirstType->isRVVSizelessBuiltinType() && SecondType->isVectorType()) ||
+       (FirstType->isVectorType() && SecondType->isRVVSizelessBuiltinType())) &&
+      "Expected RVV builtin type and vector type!");
+
+  auto IsValidCast = [this](QualType FirstType, QualType SecondType) {
+    if (const auto *BT = FirstType->getAs<BuiltinType>()) {
+      if (const auto *VT = SecondType->getAs<VectorType>()) {
+        // Predicates have the same representation as uint8 so we also have to
+        // check the kind to make these types incompatible.
+        if (VT->getVectorKind() == VectorType::RVVFixedLengthDataVector)
+          return FirstType->isRVVVLSBuiltinType() &&
+                 VT->getElementType().getCanonicalType() ==
+                     FirstType->getRVVEltType(*this);
+        if (VT->getVectorKind() == VectorType::GenericVector)
+          return getTypeSize(SecondType) == getRVVTypeSize(*this, BT) &&
+                 hasSameType(VT->getElementType(),
+                             getBuiltinVectorTypeInfo(BT).ElementType);
+      }
+    }
+    return false;
+  };
+
+  return IsValidCast(FirstType, SecondType) ||
+         IsValidCast(SecondType, FirstType);
+}
+
+bool ASTContext::areLaxCompatibleRVVTypes(QualType FirstType,
+                                          QualType SecondType) {
+  assert(
+      ((FirstType->isRVVSizelessBuiltinType() && SecondType->isVectorType()) ||
+       (FirstType->isVectorType() && SecondType->isRVVSizelessBuiltinType())) &&
+      "Expected RVV builtin type and vector type!");
+
+  auto IsLaxCompatible = [this](QualType FirstType, QualType SecondType) {
+    const auto *BT = FirstType->getAs<BuiltinType>();
+    if (!BT)
+      return false;
+
+    const auto *VecTy = SecondType->getAs<VectorType>();
+    if (VecTy &&
+        (VecTy->getVectorKind() == VectorType::RVVFixedLengthDataVector ||
+         VecTy->getVectorKind() == VectorType::GenericVector)) {
+      const LangOptions::LaxVectorConversionKind LVCKind =
+          getLangOpts().getLaxVectorConversions();
+
+      // If __riscv_v_fixed_vlen != N do not allow GNU vector lax conversion.
+      if (VecTy->getVectorKind() == VectorType::GenericVector &&
+          getTypeSize(SecondType) != getRVVTypeSize(*this, BT))
+        return false;
+
+      // If -flax-vector-conversions=all is specified, the types are
+      // certainly compatible.
+      if (LVCKind == LangOptions::LaxVectorConversionKind::All)
+        return true;
+
+      // If -flax-vector-conversions=integer is specified, the types are
+      // compatible if the elements are integer types.
+      if (LVCKind == LangOptions::LaxVectorConversionKind::Integer)
+        return VecTy->getElementType().getCanonicalType()->isIntegerType() &&
+               FirstType->getRVVEltType(*this)->isIntegerType();
+    }
+
+    return false;
+  };
+
+  return IsLaxCompatible(FirstType, SecondType) ||
+         IsLaxCompatible(SecondType, FirstType);
+}
+
 bool ASTContext::hasDirectOwnershipQualifier(QualType Ty) const {
   while (true) {
     // __strong id
@@ -12982,8 +13067,10 @@ static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X,
     SmallVector<TemplateArgument, 8> As;
     if (CD &&
         getCommonTemplateArguments(Ctx, As, AX->getTypeConstraintArguments(),
-                                   AY->getTypeConstraintArguments()))
+                                   AY->getTypeConstraintArguments())) {
       CD = nullptr; // The arguments differ, so make it unconstrained.
+      As.clear();
+    }
 
     // Both auto types can't be dependent, otherwise they wouldn't have been
     // sugar. This implies they can't contain unexpanded packs either.
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index c561f0462ed05..5a301c10aca6d 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -663,7 +663,7 @@ ExprDependence clang::computeDependence(GenericSelectionExpr *E,
 
 ExprDependence clang::computeDependence(DesignatedInitExpr *E) {
   auto Deps = E->getInit()->getDependence();
-  for (auto D : E->designators()) {
+  for (const auto &D : E->designators()) {
     auto DesignatorDeps = ExprDependence::None;
     if (D.isArrayDesignator())
       DesignatorDeps |= E->getArrayIndex(D)->getDependence();
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index b728917fdda6d..cf00db9c948bd 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1635,8 +1635,8 @@ Module *Decl::getOwningModuleForLinkage(bool IgnoreLinkage) const {
   llvm_unreachable("unknown module kind");
 }
 
-void NamedDecl::printName(raw_ostream &OS, const PrintingPolicy&) const {
-  OS << Name;
+void NamedDecl::printName(raw_ostream &OS, const PrintingPolicy &Policy) const {
+  Name.print(OS, Policy);
 }
 
 void NamedDecl::printName(raw_ostream &OS) const {
@@ -3245,7 +3245,7 @@ bool FunctionDecl::isReplaceableGlobalAllocationFunction(
     return false;
 
   const auto *FPT = getType()->castAs<FunctionProtoType>();
-  if (FPT->getNumParams() == 0 || FPT->getNumParams() > 3 || FPT->isVariadic())
+  if (FPT->getNumParams() == 0 || FPT->getNumParams() > 4 || FPT->isVariadic())
     return false;
 
   // If this is a single-parameter function, it must be a replaceable global
@@ -3280,8 +3280,8 @@ bool FunctionDecl::isReplaceableGlobalAllocationFunction(
       *AlignmentParam = Params;
   }
 
-  // Finally, if this is not a sized delete, the final parameter can
-  // be a 'const std::nothrow_t&'.
+  // If this is not a sized delete, the next parameter can be a
+  // 'const std::nothrow_t&'.
   if (!IsSizedDelete && !Ty.isNull() && Ty->isReferenceType()) {
     Ty = Ty->getPointeeType();
     if (Ty.getCVRQualifiers() != Qualifiers::Const)
@@ -3293,6 +3293,19 @@ bool FunctionDecl::isReplaceableGlobalAllocationFunction(
     }
   }
 
+  // Finally, recognize the not yet standard versions of new that take a
+  // hot/cold allocation hint (__hot_cold_t). These are currently supported by
+  // tcmalloc (see
+  // https://github.com/google/tcmalloc/blob/220043886d4e2efff7a5702d5172cb8065253664/tcmalloc/malloc_extension.h#L53).
+  if (!IsSizedDelete && !Ty.isNull() && Ty->isEnumeralType()) {
+    QualType T = Ty;
+    while (const auto *TD = T->getAs<TypedefType>())
+      T = TD->getDecl()->getUnderlyingType();
+    IdentifierInfo *II = T->getAs<EnumType>()->getDecl()->getIdentifier();
+    if (II && II->isStr("__hot_cold_t"))
+      Consume();
+  }
+
   return Params == FPT->getNumParams();
 }
 
diff --git a/clang/lib/AST/DeclarationName.cpp b/clang/lib/AST/DeclarationName.cpp
index 73c56e6610ee0..a3ac5551e0cc2 100644
--- a/clang/lib/AST/DeclarationName.cpp
+++ b/clang/lib/AST/DeclarationName.cpp
@@ -117,12 +117,12 @@ static void printCXXConstructorDestructorName(QualType ClassType,
   Policy.adjustForCPlusPlus();
 
   if (const RecordType *ClassRec = ClassType->getAs<RecordType>()) {
-    OS << *ClassRec->getDecl();
+    ClassRec->getDecl()->printName(OS, Policy);
     return;
   }
   if (Policy.SuppressTemplateArgsInCXXConstructors) {
     if (auto *InjTy = ClassType->getAs<InjectedClassNameType>()) {
-      OS << *InjTy->getDecl();
+      InjTy->getDecl()->printName(OS, Policy);
       return;
     }
   }
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 6bfb3a37b2433..03eb5dfe6ff60 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -5043,7 +5043,7 @@ static EvalStmtResult EvaluateSwitch(StmtResult &Result, EvalInfo &Info,
 
 static bool CheckLocalVariableDeclaration(EvalInfo &Info, const VarDecl *VD) {
   // An expression E is a core constant expression unless the evaluation of E
-  // would evaluate one of the following: [C++2b] - a control flow that passes
+  // would evaluate one of the following: [C++23] - a control flow that passes
   // through a declaration of a variable with static or thread storage duration
   // unless that variable is usable in constant expressions.
   if (VD->isLocalVarDecl() && VD->isStaticLocal() &&
@@ -14839,6 +14839,7 @@ class AtomicExprEvaluator :
     switch (E->getCastKind()) {
     default:
       return ExprEvaluatorBaseTy::VisitCastExpr(E);
+    case CK_NullToPointer:
     case CK_NonAtomicToAtomic:
       return This ? EvaluateInPlace(Result, Info, *This, E->getSubExpr())
                   : Evaluate(Result, Info, E->getSubExpr());
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index a8e8b2997ddef..df7c4a72f21a7 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -40,7 +40,7 @@ template <class Emitter> class DeclScope final : public VariableScope<Emitter> {
 };
 
 /// Scope used to handle initialization methods.
-template <class Emitter> class OptionScope {
+template <class Emitter> class OptionScope final {
 public:
   /// Root constructor, compiling or discarding primitives.
   OptionScope(ByteCodeExprGen<Emitter> *Ctx, bool NewDiscardResult)
@@ -146,10 +146,23 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     if (!this->visit(SubExpr))
       return false;
 
-    // TODO: Emit this only if FromT != ToT.
+    if (FromT == ToT)
+      return true;
+
     return this->emitCast(*FromT, *ToT, CE);
   }
 
+  case CK_PointerToBoolean: {
+    // Just emit p != nullptr for this.
+    if (!this->visit(SubExpr))
+      return false;
+
+    if (!this->emitNullPtr(CE))
+      return false;
+
+    return this->emitNEPtr(CE);
+  }
+
   case CK_ToVoid:
     return discard(SubExpr);
 
@@ -177,7 +190,12 @@ bool ByteCodeExprGen<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
 
 template <class Emitter>
 bool ByteCodeExprGen<Emitter>::VisitParenExpr(const ParenExpr *PE) {
-  return this->visit(PE->getSubExpr());
+  const Expr *SubExpr = PE->getSubExpr();
+
+  if (DiscardResult)
+    return this->discard(SubExpr);
+
+  return this->visit(SubExpr);
 }
 
 template <class Emitter>
@@ -390,7 +408,7 @@ bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueIni
   if (!T)
     return false;
 
-  return this->visitZeroInitializer(*T, E);
+  return this->visitZeroInitializer(E->getType(), E);
 }
 
 template <class Emitter>
@@ -782,12 +800,10 @@ bool ByteCodeExprGen<Emitter>::VisitExprWithCleanups(
   const Expr *SubExpr = E->getSubExpr();
 
   assert(E->getNumObjects() == 0 && "TODO: Implement cleanups");
-  if (!this->visit(SubExpr))
-    return false;
-
   if (DiscardResult)
-    return this->emitPopPtr(E);
-  return true;
+    return this->discard(SubExpr);
+
+  return this->visit(SubExpr);
 }
 
 template <class Emitter>
@@ -815,7 +831,7 @@ bool ByteCodeExprGen<Emitter>::VisitMaterializeTemporaryExpr(
   // For everyhing else, use local variables.
   if (SubExprT) {
     if (std::optional<unsigned> LocalIndex = allocateLocalPrimitive(
-            SubExpr, *SubExprT, /*IsMutable=*/true, /*IsExtended=*/true)) {
+            SubExpr, *SubExprT, /*IsConst=*/true, /*IsExtended=*/true)) {
       if (!this->visitInitializer(SubExpr))
         return false;
       this->emitSetLocal(*SubExprT, *LocalIndex, E);
@@ -929,7 +945,12 @@ bool ByteCodeExprGen<Emitter>::visitConditional(
 }
 
 template <class Emitter>
-bool ByteCodeExprGen<Emitter>::visitZeroInitializer(PrimType T, const Expr *E) {
+bool ByteCodeExprGen<Emitter>::visitZeroInitializer(QualType QT,
+                                                    const Expr *E) {
+  // FIXME: We need the QualType to get the float semantics, but that means we
+  //   classify it over and over again in array situations.
+  PrimType T = classifyPrim(QT);
+
   switch (T) {
   case PT_Bool:
     return this->emitZeroBool(E);
@@ -954,8 +975,7 @@ bool ByteCodeExprGen<Emitter>::visitZeroInitializer(PrimType T, const Expr *E) {
   case PT_FnPtr:
     return this->emitNullFnPtr(E);
   case PT_Float: {
-    return this->emitConstFloat(
-        APFloat::getZero(Ctx.getFloatSemantics(E->getType())), E);
+    return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
   }
   }
   llvm_unreachable("unknown primitive type");
@@ -1276,7 +1296,7 @@ bool ByteCodeExprGen<Emitter>::visitArrayInitializer(const Expr *Initializer) {
       //   since we memset our Block*s to 0 and so we have the desired value
       //   without this.
       for (size_t I = 0; I != NumElems; ++I) {
-        if (!this->emitZero(*ElemT, Initializer))
+        if (!this->visitZeroInitializer(CAT->getElementType(), Initializer))
           return false;
         if (!this->emitInitElem(*ElemT, I, Initializer))
           return false;
@@ -1758,6 +1778,11 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       return DiscardResult ? this->emitPopPtr(E) : true;
     }
 
+    if (T == PT_Float) {
+      return DiscardResult ? this->emitIncfPop(getRoundingMode(E), E)
+                           : this->emitIncf(getRoundingMode(E), E);
+    }
+
     return DiscardResult ? this->emitIncPop(*T, E) : this->emitInc(*T, E);
   }
   case UO_PostDec: { // x--
@@ -1771,6 +1796,11 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       return DiscardResult ? this->emitPopPtr(E) : true;
     }
 
+    if (T == PT_Float) {
+      return DiscardResult ? this->emitDecfPop(getRoundingMode(E), E)
+                           : this->emitDecf(getRoundingMode(E), E);
+    }
+
     return DiscardResult ? this->emitDecPop(*T, E) : this->emitDec(*T, E);
   }
   case UO_PreInc: { // ++x
@@ -1785,9 +1815,19 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     }
 
     // Post-inc and pre-inc are the same if the value is to be discarded.
-    if (DiscardResult)
+    if (DiscardResult) {
+      if (T == PT_Float)
+        return this->emitIncfPop(getRoundingMode(E), E);
       return this->emitIncPop(*T, E);
+    }
 
+    if (T == PT_Float) {
+      const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
+      this->emitLoadFloat(E);
+      this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E);
+      this->emitAddf(getRoundingMode(E), E);
+      return this->emitStoreFloat(E);
+    }
     this->emitLoad(*T, E);
     this->emitConst(1, E);
     this->emitAdd(*T, E);
@@ -1805,9 +1845,19 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     }
 
     // Post-dec and pre-dec are the same if the value is to be discarded.
-    if (DiscardResult)
+    if (DiscardResult) {
+      if (T == PT_Float)
+        return this->emitDecfPop(getRoundingMode(E), E);
       return this->emitDecPop(*T, E);
+    }
 
+    if (T == PT_Float) {
+      const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
+      this->emitLoadFloat(E);
+      this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E);
+      this->emitSubf(getRoundingMode(E), E);
+      return this->emitStoreFloat(E);
+    }
     this->emitLoad(*T, E);
     this->emitConst(1, E);
     this->emitSub(*T, E);
@@ -1857,6 +1907,9 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
 
 template <class Emitter>
 bool ByteCodeExprGen<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
+  if (DiscardResult)
+    return true;
+
   const auto *D = E->getDecl();
 
   if (const auto *ECD = dyn_cast<EnumConstantDecl>(D)) {
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index 85588c6ecd3c1..52a8eca593c2b 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -185,7 +185,7 @@ class ByteCodeExprGen : public ConstStmtVisitor<ByteCodeExprGen<Emitter>, bool>,
                         llvm::function_ref<bool(const Expr *)> V);
 
   /// Creates a local primitive value.
-  unsigned allocateLocalPrimitive(DeclTy &&Decl, PrimType Ty, bool IsMutable,
+  unsigned allocateLocalPrimitive(DeclTy &&Decl, PrimType Ty, bool IsConst,
                                   bool IsExtended = false);
 
   /// Allocates a space storing a local given its type.
@@ -201,7 +201,7 @@ class ByteCodeExprGen : public ConstStmtVisitor<ByteCodeExprGen<Emitter>, bool>,
   friend class ArrayIndexScope<Emitter>;
 
   /// Emits a zero initializer.
-  bool visitZeroInitializer(PrimType T, const Expr *E);
+  bool visitZeroInitializer(QualType QT, const Expr *E);
 
   enum class DerefKind {
     /// Value is read and pushed to stack.
diff --git a/clang/lib/AST/Interp/EvalEmitter.h b/clang/lib/AST/Interp/EvalEmitter.h
index 22d7b7c68d104..99933900f2921 100644
--- a/clang/lib/AST/Interp/EvalEmitter.h
+++ b/clang/lib/AST/Interp/EvalEmitter.h
@@ -71,7 +71,7 @@ class EvalEmitter : public SourceMapper {
 
   /// Returns the source location of the current opcode.
   SourceInfo getSource(const Function *F, CodePtr PC) const override {
-    return F ? F->getSource(PC) : CurrentSource;
+    return (F && F->hasBody()) ? F->getSource(PC) : CurrentSource;
   }
 
   /// Parameter indices.
diff --git a/clang/lib/AST/Interp/Floating.h b/clang/lib/AST/Interp/Floating.h
index fb0884bb3b605..85876236a9998 100644
--- a/clang/lib/AST/Interp/Floating.h
+++ b/clang/lib/AST/Interp/Floating.h
@@ -105,31 +105,45 @@ class Floating final {
 
   // -------
 
-  static APFloat::opStatus add(Floating A, Floating B, llvm::RoundingMode RM,
-                               Floating *R) {
+  static APFloat::opStatus add(const Floating &A, const Floating &B,
+                               llvm::RoundingMode RM, Floating *R) {
     *R = Floating(A.F);
     return R->F.add(B.F, RM);
   }
 
-  static APFloat::opStatus sub(Floating A, Floating B, llvm::RoundingMode RM,
-                               Floating *R) {
+  static APFloat::opStatus increment(const Floating &A, llvm::RoundingMode RM,
+                                     Floating *R) {
+    APFloat One(A.F.getSemantics(), 1);
+    *R = Floating(A.F);
+    return R->F.add(One, RM);
+  }
+
+  static APFloat::opStatus sub(const Floating &A, const Floating &B,
+                               llvm::RoundingMode RM, Floating *R) {
     *R = Floating(A.F);
     return R->F.subtract(B.F, RM);
   }
 
-  static APFloat::opStatus mul(Floating A, Floating B, llvm::RoundingMode RM,
-                               Floating *R) {
+  static APFloat::opStatus decrement(const Floating &A, llvm::RoundingMode RM,
+                                     Floating *R) {
+    APFloat One(A.F.getSemantics(), 1);
+    *R = Floating(A.F);
+    return R->F.subtract(One, RM);
+  }
+
+  static APFloat::opStatus mul(const Floating &A, const Floating &B,
+                               llvm::RoundingMode RM, Floating *R) {
     *R = Floating(A.F);
     return R->F.multiply(B.F, RM);
   }
 
-  static APFloat::opStatus div(Floating A, Floating B, llvm::RoundingMode RM,
-                               Floating *R) {
+  static APFloat::opStatus div(const Floating &A, const Floating &B,
+                               llvm::RoundingMode RM, Floating *R) {
     *R = Floating(A.F);
     return R->F.divide(B.F, RM);
   }
 
-  static bool neg(Floating A, Floating *R) {
+  static bool neg(const Floating &A, Floating *R) {
     *R = -A;
     return false;
   }
diff --git a/clang/lib/AST/Interp/Function.cpp b/clang/lib/AST/Interp/Function.cpp
index 40001faad4116..4e6d175c41b26 100644
--- a/clang/lib/AST/Interp/Function.cpp
+++ b/clang/lib/AST/Interp/Function.cpp
@@ -32,6 +32,7 @@ Function::ParamDescriptor Function::getParamDescriptor(unsigned Offset) const {
 SourceInfo Function::getSource(CodePtr PC) const {
   assert(PC >= getCodeBegin() && "PC does not belong to this function");
   assert(PC <= getCodeEnd() && "PC Does not belong to this function");
+  assert(hasBody() && "Function has no body");
   unsigned Offset = PC - getCodeBegin();
   using Elem = std::pair<unsigned, SourceInfo>;
   auto It = llvm::lower_bound(SrcMap, Elem{Offset, {}}, llvm::less_first());
diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/Interp/FunctionPointer.h
index 2d449bdb031d8..20d4d7793185c 100644
--- a/clang/lib/AST/Interp/FunctionPointer.h
+++ b/clang/lib/AST/Interp/FunctionPointer.h
@@ -8,6 +8,7 @@
 #include "clang/AST/APValue.h"
 
 namespace clang {
+class ASTContext;
 namespace interp {
 
 class FunctionPointer final {
@@ -38,6 +39,13 @@ class FunctionPointer final {
     OS << ")";
   }
 
+  std::string toDiagnosticString(const ASTContext &Ctx) const {
+    if (!Func)
+      return "nullptr";
+
+    return toAPValue().getAsString(Ctx, Func->getDecl()->getType());
+  }
+
   ComparisonCategoryResult compare(const FunctionPointer &RHS) const {
     if (Func == RHS.Func)
       return ComparisonCategoryResult::Equal;
diff --git a/clang/lib/AST/Interp/Integral.h b/clang/lib/AST/Interp/Integral.h
index 932caca29b9f0..cc7c7a9d2430e 100644
--- a/clang/lib/AST/Interp/Integral.h
+++ b/clang/lib/AST/Interp/Integral.h
@@ -223,6 +223,9 @@ template <unsigned Bits, bool Signed> class Integral final {
   }
 
   static bool neg(Integral A, Integral *R) {
+    if (Signed && A.isMin())
+      return true;
+
     *R = -A;
     return false;
   }
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 1eda38e9fa5b1..64bdd872221ac 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -413,12 +413,32 @@ bool Inv(InterpState &S, CodePtr OpPC) {
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Neg(InterpState &S, CodePtr OpPC) {
-  const T &Val = S.Stk.pop<T>();
+  const T &Value = S.Stk.pop<T>();
   T Result;
-  T::neg(Val, &Result);
 
+  if (!T::neg(Value, &Result)) {
+    S.Stk.push<T>(Result);
+    return true;
+  }
+
+  assert(isIntegralType(Name) &&
+         "don't expect other types to fail at constexpr negation");
   S.Stk.push<T>(Result);
-  return true;
+
+  APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
+  const Expr *E = S.Current->getExpr(OpPC);
+  QualType Type = E->getType();
+
+  if (S.checkingForUndefinedBehavior()) {
+    SmallString<32> Trunc;
+    NegatedValue.trunc(Result.bitWidth()).toString(Trunc, 10);
+    auto Loc = E->getExprLoc();
+    S.report(Loc, diag::warn_integer_constant_overflow) << Trunc << Type;
+    return true;
+  }
+
+  S.CCEDiag(E, diag::note_constexpr_overflow) << NegatedValue << Type;
+  return S.noteUndefinedBehavior();
 }
 
 enum class PushVal : bool {
@@ -436,7 +456,7 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   T Result;
 
   if constexpr (DoPush == PushVal::Yes)
-    S.Stk.push<T>(Result);
+    S.Stk.push<T>(Value);
 
   if constexpr (Op == IncDecOp::Inc) {
     if (!T::increment(Value, &Result)) {
@@ -520,6 +540,50 @@ bool DecPop(InterpState &S, CodePtr OpPC) {
   return IncDecHelper<T, IncDecOp::Dec, PushVal::No>(S, OpPC, Ptr);
 }
 
+template <IncDecOp Op, PushVal DoPush>
+bool IncDecFloatHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                       llvm::RoundingMode RM) {
+  Floating Value = Ptr.deref<Floating>();
+  Floating Result;
+
+  if constexpr (DoPush == PushVal::Yes)
+    S.Stk.push<Floating>(Value);
+
+  llvm::APFloat::opStatus Status;
+  if constexpr (Op == IncDecOp::Inc)
+    Status = Floating::increment(Value, RM, &Result);
+  else
+    Status = Floating::decrement(Value, RM, &Result);
+
+  Ptr.deref<Floating>() = Result;
+
+  return CheckFloatResult(S, OpPC, Status);
+}
+
+inline bool Incf(InterpState &S, CodePtr OpPC, llvm::RoundingMode RM) {
+  // FIXME: Check initialization of Ptr
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+  return IncDecFloatHelper<IncDecOp::Inc, PushVal::Yes>(S, OpPC, Ptr, RM);
+}
+
+inline bool IncfPop(InterpState &S, CodePtr OpPC, llvm::RoundingMode RM) {
+  // FIXME: Check initialization of Ptr
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+  return IncDecFloatHelper<IncDecOp::Inc, PushVal::No>(S, OpPC, Ptr, RM);
+}
+
+inline bool Decf(InterpState &S, CodePtr OpPC, llvm::RoundingMode RM) {
+  // FIXME: Check initialization of Ptr
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+  return IncDecFloatHelper<IncDecOp::Dec, PushVal::Yes>(S, OpPC, Ptr, RM);
+}
+
+inline bool DecfPop(InterpState &S, CodePtr OpPC, llvm::RoundingMode RM) {
+  // FIXME: Check initialization of Ptr
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+  return IncDecFloatHelper<IncDecOp::Dec, PushVal::No>(S, OpPC, Ptr, RM);
+}
+
 /// 1) Pops the value from the stack.
 /// 2) Pushes the bitwise complemented value on the stack (~V).
 template <PrimType Name, class T = typename PrimConv<Name>::T>
@@ -554,6 +618,29 @@ bool CmpHelperEQ(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   return CmpHelper<T>(S, OpPC, Fn);
 }
 
+/// Function pointers cannot be compared in an ordered way.
+template <>
+inline bool CmpHelper<FunctionPointer>(InterpState &S, CodePtr OpPC,
+                                       CompareFn Fn) {
+  const auto &RHS = S.Stk.pop<FunctionPointer>();
+  const auto &LHS = S.Stk.pop<FunctionPointer>();
+
+  const SourceInfo &Loc = S.Current->getSource(OpPC);
+  S.FFDiag(Loc, diag::note_constexpr_pointer_comparison_unspecified)
+      << LHS.toDiagnosticString(S.getCtx())
+      << RHS.toDiagnosticString(S.getCtx());
+  return false;
+}
+
+template <>
+inline bool CmpHelperEQ<FunctionPointer>(InterpState &S, CodePtr OpPC,
+                                         CompareFn Fn) {
+  const auto &RHS = S.Stk.pop<FunctionPointer>();
+  const auto &LHS = S.Stk.pop<FunctionPointer>();
+  S.Stk.push<Boolean>(Boolean::from(Fn(LHS.compare(RHS))));
+  return true;
+}
+
 template <>
 inline bool CmpHelper<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   using BoolT = PrimConv<PT_Bool>::T;
@@ -701,6 +788,9 @@ bool GetLocal(InterpState &S, CodePtr OpPC, uint32_t I) {
   return true;
 }
 
+/// 1) Pops the value from the stack.
+/// 2) Writes the value to the local variable with the
+///    given offset.
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool SetLocal(InterpState &S, CodePtr OpPC, uint32_t I) {
   S.Current->setLocal<T>(I, S.Stk.pop<T>());
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td
index ed0774a788333..15f7312ad00e6 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/Interp/Opcodes.td
@@ -94,7 +94,7 @@ def AllTypeClass : TypeClass {
 }
 
 def ComparableTypeClass : TypeClass {
-  let Types = !listconcat(AluTypeClass.Types, [Ptr], [Float]);
+  let Types = !listconcat(AluTypeClass.Types, [Ptr], [Float], [FnPtr]);
 }
 
 class SingletonTypeClass<Type Ty> : TypeClass {
@@ -499,11 +499,18 @@ def Inv: Opcode {
   let HasGroup = 1;
 }
 
+// Increment and decrement.
 def Inc: IntegerOpcode;
 def IncPop : IntegerOpcode;
 def Dec: IntegerOpcode;
 def DecPop: IntegerOpcode;
 
+// Float increment and decrement.
+def Incf: FloatOpcode;
+def IncfPop : FloatOpcode;
+def Decf: FloatOpcode;
+def DecfPop : FloatOpcode;
+
 // [Real] -> [Real]
 def Neg: Opcode {
   let Types = [NonPtrTypeClass];
diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index 8f1dfa346c632..7f127143a99d8 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -147,7 +147,7 @@ APValue Pointer::toAPValue() const {
 
 bool Pointer::isInitialized() const {
   assert(Pointee && "Cannot check if null pointer was initialized");
-  Descriptor *Desc = getFieldDesc();
+  const Descriptor *Desc = getFieldDesc();
   assert(Desc);
   if (Desc->isPrimitiveArray()) {
     if (isStatic() && Base == 0)
@@ -167,7 +167,7 @@ bool Pointer::isInitialized() const {
 
 void Pointer::initialize() const {
   assert(Pointee && "Cannot initialize null pointer");
-  Descriptor *Desc = getFieldDesc();
+  const Descriptor *Desc = getFieldDesc();
 
   assert(Desc);
   if (Desc->isPrimitiveArray()) {
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index 5895c61295e02..10d21a27167f0 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -290,7 +290,7 @@ class Pointer {
   /// Returns the number of elements.
   unsigned getNumElems() const { return getSize() / elemSize(); }
 
-  Block *block() const { return Pointee; }
+  const Block *block() const { return Pointee; }
 
   /// Returns the index into an array.
   int64_t getIndex() const {
diff --git a/clang/lib/AST/Interp/PrimType.h b/clang/lib/AST/Interp/PrimType.h
index 30bec3f2a17cf..693e57210608d 100644
--- a/clang/lib/AST/Interp/PrimType.h
+++ b/clang/lib/AST/Interp/PrimType.h
@@ -42,6 +42,8 @@ enum PrimType : unsigned {
   PT_FnPtr,
 };
 
+constexpr bool isIntegralType(PrimType T) { return T <= PT_Uint64; }
+
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
 template <> struct PrimConv<PT_Sint8> { using T = Integral<8, true>; };
diff --git a/clang/lib/AST/Interp/State.cpp b/clang/lib/AST/Interp/State.cpp
index 2512979257605..f67bde1082fa0 100644
--- a/clang/lib/AST/Interp/State.cpp
+++ b/clang/lib/AST/Interp/State.cpp
@@ -11,6 +11,7 @@
 #include "Program.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/OptionalDiagnostic.h"
 
 using namespace clang;
 using namespace clang::interp;
diff --git a/clang/lib/AST/Interp/State.h b/clang/lib/AST/Interp/State.h
index 131fbcf3cffc4..d897b7c202756 100644
--- a/clang/lib/AST/Interp/State.h
+++ b/clang/lib/AST/Interp/State.h
@@ -15,9 +15,9 @@
 
 #include "clang/AST/ASTDiagnostic.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/OptionalDiagnostic.h"
 
 namespace clang {
+class OptionalDiagnostic;
 
 /// Kinds of access we can perform on an object, for diagnostics. Note that
 /// we consider a member function call to be a kind of access, even though
@@ -36,7 +36,7 @@ enum AccessKinds {
   AK_Destroy,
 };
 
-// The order of this enum is important for diagnostics.
+/// The order of this enum is important for diagnostics.
 enum CheckSubobjectKind {
   CSK_Base,
   CSK_Derived,
@@ -72,7 +72,7 @@ class State {
 
 public:
   State() : InConstantContext(false) {}
-  // Diagnose that the evaluation could not be folded (FF => FoldFailure)
+  /// Diagnose that the evaluation could not be folded (FF => FoldFailure)
   OptionalDiagnostic
   FFDiag(SourceLocation Loc,
          diag::kind DiagId = diag::note_invalid_subexpr_in_const_expr,
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 53371394730a2..806bd30972312 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -563,6 +563,8 @@ class CXXNameMangler {
   void mangleAArch64NeonVectorType(const DependentVectorType *T);
   void mangleAArch64FixedSveVectorType(const VectorType *T);
   void mangleAArch64FixedSveVectorType(const DependentVectorType *T);
+  void mangleRISCVFixedRVVVectorType(const VectorType *T);
+  void mangleRISCVFixedRVVVectorType(const DependentVectorType *T);
 
   void mangleIntegerLiteral(QualType T, const llvm::APSInt &Value);
   void mangleFloatLiteral(QualType T, const llvm::APFloat &V);
@@ -3812,6 +3814,68 @@ void CXXNameMangler::mangleAArch64FixedSveVectorType(
   Diags.Report(T->getAttributeLoc(), DiagID);
 }
 
+void CXXNameMangler::mangleRISCVFixedRVVVectorType(const VectorType *T) {
+  assert(T->getVectorKind() == VectorType::RVVFixedLengthDataVector &&
+         "expected fixed-length RVV vector!");
+
+  QualType EltType = T->getElementType();
+  assert(EltType->isBuiltinType() &&
+         "expected builtin type for fixed-length RVV vector!");
+
+  StringRef TypeName;
+  switch (cast<BuiltinType>(EltType)->getKind()) {
+  case BuiltinType::SChar:
+    TypeName = "__rvv_int8m1_t";
+    break;
+  case BuiltinType::UChar:
+    TypeName = "__rvv_uint8m1_t";
+    break;
+  case BuiltinType::Short:
+    TypeName = "__rvv_int16m1_t";
+    break;
+  case BuiltinType::UShort:
+    TypeName = "__rvv_uint16m1_t";
+    break;
+  case BuiltinType::Int:
+    TypeName = "__rvv_int32m1_t";
+    break;
+  case BuiltinType::UInt:
+    TypeName = "__rvv_uint32m1_t";
+    break;
+  case BuiltinType::Long:
+    TypeName = "__rvv_int64m1_t";
+    break;
+  case BuiltinType::ULong:
+    TypeName = "__rvv_uint64m1_t";
+    break;
+  case BuiltinType::Half:
+    TypeName = "__rvv_float16m1_t";
+    break;
+  case BuiltinType::Float:
+    TypeName = "__rvv_float32m1_t";
+    break;
+  case BuiltinType::Double:
+    TypeName = "__rvv_float64m1_t";
+    break;
+  default:
+    llvm_unreachable("unexpected element type for fixed-length RVV vector!");
+  }
+
+  unsigned VecSizeInBits = getASTContext().getTypeInfo(T).Width;
+
+  Out << "9__RVV_VLSI" << 'u' << TypeName.size() << TypeName << "Lj"
+      << VecSizeInBits << "EE";
+}
+
+void CXXNameMangler::mangleRISCVFixedRVVVectorType(
+    const DependentVectorType *T) {
+  DiagnosticsEngine &Diags = Context.getDiags();
+  unsigned DiagID = Diags.getCustomDiagID(
+      DiagnosticsEngine::Error,
+      "cannot mangle this dependent fixed-length RVV vector type yet");
+  Diags.Report(T->getAttributeLoc(), DiagID);
+}
+
 // GNU extension: vector types
 // <type>                  ::= <vector-type>
 // <vector-type>           ::= Dv <positive dimension number> _
@@ -3836,6 +3900,9 @@ void CXXNameMangler::mangleType(const VectorType *T) {
              T->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
     mangleAArch64FixedSveVectorType(T);
     return;
+  } else if (T->getVectorKind() == VectorType::RVVFixedLengthDataVector) {
+    mangleRISCVFixedRVVVectorType(T);
+    return;
   }
   Out << "Dv" << T->getNumElements() << '_';
   if (T->getVectorKind() == VectorType::AltiVecPixel)
@@ -3862,6 +3929,9 @@ void CXXNameMangler::mangleType(const DependentVectorType *T) {
              T->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
     mangleAArch64FixedSveVectorType(T);
     return;
+  } else if (T->getVectorKind() == VectorType::RVVFixedLengthDataVector) {
+    mangleRISCVFixedRVVVectorType(T);
+    return;
   }
 
   Out << "Dv";
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 95692e1e7f6ec..43b7b710d7f45 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -662,6 +662,9 @@ void JSONNodeDumper::VisitVectorType(const VectorType *VT) {
   case VectorType::SveFixedLengthPredicateVector:
     JOS.attribute("vectorKind", "fixed-length sve predicate vector");
     break;
+  case VectorType::RVVFixedLengthDataVector:
+    JOS.attribute("vectorKind", "fixed-length rvv data vector");
+    break;
   }
 }
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index e0fd8abe5e3b8..d70f1a19acbbd 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -29,12 +29,14 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CRC.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/xxhash.h"
+#include <functional>
 #include <optional>
 
 using namespace clang;
@@ -368,9 +370,13 @@ class MicrosoftCXXNameMangler {
   void mangleVariableEncoding(const VarDecl *VD);
   void mangleMemberDataPointer(const CXXRecordDecl *RD, const ValueDecl *VD,
                                StringRef Prefix = "$");
+  void mangleMemberDataPointerInClassNTTP(const CXXRecordDecl *,
+                                          const ValueDecl *);
   void mangleMemberFunctionPointer(const CXXRecordDecl *RD,
                                    const CXXMethodDecl *MD,
                                    StringRef Prefix = "$");
+  void mangleMemberFunctionPointerInClassNTTP(const CXXRecordDecl *RD,
+                                              const CXXMethodDecl *MD);
   void mangleVirtualMemPtrThunk(const CXXMethodDecl *MD,
                                 const MethodVFTableLocation &ML);
   void mangleNumber(int64_t Number);
@@ -711,6 +717,28 @@ void MicrosoftCXXNameMangler::mangleMemberDataPointer(const CXXRecordDecl *RD,
     mangleNumber(VBTableOffset);
 }
 
+void MicrosoftCXXNameMangler::mangleMemberDataPointerInClassNTTP(
+    const CXXRecordDecl *RD, const ValueDecl *VD) {
+  MSInheritanceModel IM = RD->getMSInheritanceModel();
+  // <nttp-class-member-data-pointer> ::= <member-data-pointer>
+  //                                  ::= N
+  //                                  ::= 8 <postfix> @ <unqualified-name> @
+
+  if (IM != MSInheritanceModel::Single && IM != MSInheritanceModel::Multiple)
+    return mangleMemberDataPointer(RD, VD, "");
+
+  if (!VD) {
+    Out << 'N';
+    return;
+  }
+
+  Out << '8';
+  mangleNestedName(VD);
+  Out << '@';
+  mangleUnqualifiedName(VD);
+  Out << '@';
+}
+
 void
 MicrosoftCXXNameMangler::mangleMemberFunctionPointer(const CXXRecordDecl *RD,
                                                      const CXXMethodDecl *MD,
@@ -775,6 +803,34 @@ MicrosoftCXXNameMangler::mangleMemberFunctionPointer(const CXXRecordDecl *RD,
     mangleNumber(VBTableOffset);
 }
 
+void MicrosoftCXXNameMangler::mangleMemberFunctionPointerInClassNTTP(
+    const CXXRecordDecl *RD, const CXXMethodDecl *MD) {
+  // <nttp-class-member-function-pointer> ::= <member-function-pointer>
+  //                           ::= N
+  //                           ::= E? <virtual-mem-ptr-thunk>
+  //                           ::= E? <mangled-name> <type-encoding>
+
+  if (!MD) {
+    if (RD->getMSInheritanceModel() != MSInheritanceModel::Single)
+      return mangleMemberFunctionPointer(RD, MD, "");
+
+    Out << 'N';
+    return;
+  }
+
+  Out << "E?";
+  if (MD->isVirtual()) {
+    MicrosoftVTableContext *VTContext =
+        cast<MicrosoftVTableContext>(getASTContext().getVTableContext());
+    MethodVFTableLocation ML =
+        VTContext->getMethodVFTableLocation(GlobalDecl(MD));
+    mangleVirtualMemPtrThunk(MD, ML);
+  } else {
+    mangleName(MD);
+    mangleFunctionEncoding(MD, /*ShouldMangle=*/true);
+  }
+}
+
 void MicrosoftCXXNameMangler::mangleVirtualMemPtrThunk(
     const CXXMethodDecl *MD, const MethodVFTableLocation &ML) {
   // Get the vftable offset.
@@ -1188,6 +1244,11 @@ void MicrosoftCXXNameMangler::mangleUnqualifiedName(GlobalDecl GD,
 //           ::= <substitution> [<postfix>]
 void MicrosoftCXXNameMangler::mangleNestedName(GlobalDecl GD) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
+
+  if (const auto *ID = dyn_cast<IndirectFieldDecl>(ND))
+    for (unsigned I = 1, IE = ID->getChainingSize(); I < IE; ++I)
+      mangleSourceName("<unnamed-tag>");
+
   const DeclContext *DC = getEffectiveDeclContext(ND);
   while (!DC->isTranslationUnit()) {
     if (isa<TagDecl>(ND) || isa<VarDecl>(ND)) {
@@ -1570,7 +1631,6 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
   //                  ::= 8 <class> <unqualified-name> @
   //                  ::= A <type> <non-negative integer>  # float
   //                  ::= B <type> <non-negative integer>  # double
-  //                  ::= E <mangled-name>             # reference to D
   //                  # pointer to member, by component value
   //                  ::= F <number> <number>
   //                  ::= G <number> <number> <number>
@@ -1615,7 +1675,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
       mangleTemplateArgValue(TPO->getType().getUnqualifiedType(),
                              TPO->getValue());
     } else {
-      mangle(ND, TA.getParamTypeForDecl()->isReferenceType() ? "$E?" : "$1?");
+      mangle(ND, "$1?");
     }
     break;
   }
@@ -1744,46 +1804,62 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
         // FIXME: This can only happen as an extension. Invent a mangling.
         break;
       } else if (auto *VD = Base.dyn_cast<const ValueDecl*>()) {
-        Out << (T->isReferenceType() ? "E" : "1");
+        Out << "E";
         mangle(VD);
       } else {
         break;
       }
     } else {
-      unsigned NumAts = 0;
-      if (T->isPointerType()) {
+      if (T->isPointerType())
         Out << "5";
-        ++NumAts;
-      }
 
-      QualType T = Base.getType();
+      SmallVector<char, 2> EntryTypes;
+      SmallVector<std::function<void()>, 2> EntryManglers;
+      QualType ET = Base.getType();
       for (APValue::LValuePathEntry E : V.getLValuePath()) {
-        // We don't know how to mangle array subscripting yet.
-        if (T->isArrayType())
-          goto mangling_unknown;
+        if (auto *AT = ET->getAsArrayTypeUnsafe()) {
+          EntryTypes.push_back('C');
+          EntryManglers.push_back([this, I = E.getAsArrayIndex()] {
+            Out << '0';
+            mangleNumber(I);
+            Out << '@';
+          });
+          ET = AT->getElementType();
+          continue;
+        }
 
         const Decl *D = E.getAsBaseOrMember().getPointer();
-        auto *FD = dyn_cast<FieldDecl>(D);
-        // We don't know how to mangle derived-to-base conversions yet.
-        if (!FD)
-          goto mangling_unknown;
-
-        Out << "6";
-        ++NumAts;
-        T = FD->getType();
+        if (auto *FD = dyn_cast<FieldDecl>(D)) {
+          ET = FD->getType();
+          if (const auto *RD = ET->getAsRecordDecl())
+            if (RD->isAnonymousStructOrUnion())
+              continue;
+        } else {
+          ET = getASTContext().getRecordType(cast<CXXRecordDecl>(D));
+          // Bug in MSVC: fully qualified name of base class should be used for
+          // mangling to prevent collisions e.g. on base classes with same names
+          // in different namespaces.
+        }
+
+        EntryTypes.push_back('6');
+        EntryManglers.push_back([this, D] {
+          mangleUnqualifiedName(cast<NamedDecl>(D));
+          Out << '@';
+        });
       }
 
+      for (auto I = EntryTypes.rbegin(), E = EntryTypes.rend(); I != E; ++I)
+        Out << *I;
+
       auto *VD = Base.dyn_cast<const ValueDecl*>();
       if (!VD)
         break;
       Out << "E";
       mangle(VD);
 
-      for (APValue::LValuePathEntry E : V.getLValuePath()) {
-        const Decl *D = E.getAsBaseOrMember().getPointer();
-        mangleUnqualifiedName(cast<FieldDecl>(D));
-      }
-      for (unsigned I = 0; I != NumAts; ++I)
+      for (const std::function<void()> &Mangler : EntryManglers)
+        Mangler();
+      if (T->isPointerType())
         Out << '@';
     }
 
@@ -1794,20 +1870,14 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     if (WithScalarType)
       mangleType(T, SourceRange(), QMM_Escape);
 
-    // FIXME: The below manglings don't include a conversion, so bail if there
-    // would be one. MSVC mangles the (possibly converted) value of the
-    // pointer-to-member object as if it were a struct, leading to collisions
-    // in some cases.
-    if (!V.getMemberPointerPath().empty())
-      break;
-
     const CXXRecordDecl *RD =
         T->castAs<MemberPointerType>()->getMostRecentCXXRecordDecl();
     const ValueDecl *D = V.getMemberPointerDecl();
     if (T->isMemberDataPointerType())
-      mangleMemberDataPointer(RD, D, "");
+      mangleMemberDataPointerInClassNTTP(RD, D);
     else
-      mangleMemberFunctionPointer(RD, cast_or_null<CXXMethodDecl>(D), "");
+      mangleMemberFunctionPointerInClassNTTP(RD,
+                                             cast_or_null<CXXMethodDecl>(D));
     return;
   }
 
@@ -1895,7 +1965,6 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     break;
   }
 
-mangling_unknown:
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
       DiagnosticsEngine::Error, "cannot mangle this template argument yet");
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index 3374b49f5d8e2..0ead0940479d8 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -594,7 +594,7 @@ void ODRHash::AddCXXRecordDecl(const CXXRecordDecl *Record) {
 
   ID.AddInteger(Record->getNumBases());
   auto Bases = Record->bases();
-  for (auto Base : Bases) {
+  for (const auto &Base : Bases) {
     AddQualType(Base.getType());
     ID.AddInteger(Base.isVirtual());
     ID.AddInteger(Base.getAccessSpecifierAsWritten());
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index a5573c117e629..3817025c051c5 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1495,6 +1495,9 @@ void TextNodeDumper::VisitVectorType(const VectorType *T) {
   case VectorType::SveFixedLengthPredicateVector:
     OS << " fixed-length sve predicate vector";
     break;
+  case VectorType::RVVFixedLengthDataVector:
+    OS << " fixed-length rvv data vector";
+    break;
   }
   OS << " " << T->getNumElements();
 }
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index e1686a7c69d52..ed481a4c06962 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -46,6 +46,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -1928,6 +1929,11 @@ bool Type::hasIntegerRepresentation() const {
            (VT->getKind() >= BuiltinType::SveInt8 &&
             VT->getKind() <= BuiltinType::SveUint64);
   }
+  if (CanonicalType->isRVVVLSBuiltinType()) {
+    const auto *VT = cast<BuiltinType>(CanonicalType);
+    return (VT->getKind() >= BuiltinType::RvvInt8mf8 &&
+            VT->getKind() <= BuiltinType::RvvUint64m8);
+  }
 
   return isIntegerType();
 }
@@ -2425,6 +2431,28 @@ QualType Type::getSveEltType(const ASTContext &Ctx) const {
     return Ctx.getBuiltinVectorTypeInfo(BTy).ElementType;
 }
 
+bool Type::isRVVVLSBuiltinType() const {
+  if (const BuiltinType *BT = getAs<BuiltinType>()) {
+    switch (BT->getKind()) {
+    // FIXME: Support more than LMUL 1.
+#define RVV_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, NF, IsSigned, IsFP) \
+    case BuiltinType::Id: \
+      return NF == 1 && (NumEls * ElBits) == llvm::RISCV::RVVBitsPerBlock;
+#include "clang/Basic/RISCVVTypes.def"
+    default:
+      return false;
+    }
+  }
+  return false;
+}
+
+QualType Type::getRVVEltType(const ASTContext &Ctx) const {
+  assert(isRVVVLSBuiltinType() && "unsupported type!");
+
+  const BuiltinType *BTy = getAs<BuiltinType>();
+  return Ctx.getBuiltinVectorTypeInfo(BTy).ElementType;
+}
+
 bool QualType::isPODType(const ASTContext &Context) const {
   // C++11 has a more relaxed definition of POD.
   if (Context.getLangOpts().CPlusPlus11)
@@ -4630,6 +4658,7 @@ AutoType::AutoType(QualType DeducedAsType, AutoTypeKeyword Keyword,
   AutoTypeBits.Keyword = (unsigned)Keyword;
   AutoTypeBits.NumArgs = TypeConstraintArgs.size();
   this->TypeConstraintConcept = TypeConstraintConcept;
+  assert(TypeConstraintConcept || AutoTypeBits.NumArgs == 0);
   if (TypeConstraintConcept) {
     auto *ArgBuffer =
         const_cast<TemplateArgument *>(getTypeConstraintArguments().data());
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 02a354da56a42..408c7caa16c93 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -687,6 +687,19 @@ void TypePrinter::printVectorBefore(const VectorType *T, raw_ostream &OS) {
     else
       OS << T->getNumElements();
 
+    OS << " * sizeof(";
+    print(T->getElementType(), OS, StringRef());
+    // Multiply by 8 for the number of bits.
+    OS << ") * 8))) ";
+    printBefore(T->getElementType(), OS);
+    break;
+  case VectorType::RVVFixedLengthDataVector:
+    // FIXME: We prefer to print the size directly here, but have no way
+    // to get the size of the type.
+    OS << "__attribute__((__riscv_rvv_vector_bits__(";
+
+    OS << T->getNumElements();
+
     OS << " * sizeof(";
     print(T->getElementType(), OS, StringRef());
     // Multiply by 8 for the number of bits.
@@ -759,6 +772,20 @@ void TypePrinter::printDependentVectorBefore(
     OS << "))) ";
     printBefore(T->getElementType(), OS);
     break;
+  case VectorType::RVVFixedLengthDataVector:
+    // FIXME: We prefer to print the size directly here, but have no way
+    // to get the size of the type.
+    OS << "__attribute__((__riscv_rvv_vector_bits__(";
+    if (T->getSizeExpr()) {
+      T->getSizeExpr()->printPretty(OS, nullptr, Policy);
+      OS << " * sizeof(";
+      print(T->getElementType(), OS, StringRef());
+      // Multiply by 8 for the number of bits.
+      OS << ") * 8";
+    }
+    OS << "))) ";
+    printBefore(T->getElementType(), OS);
+    break;
   }
 }
 
diff --git a/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp b/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp
index 6699a0fc9d79e..5520633da68ae 100644
--- a/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp
@@ -70,7 +70,7 @@ static llvm::BitVector findReachableBlocks(const CFG &Cfg) {
 llvm::Expected<ControlFlowContext>
 ControlFlowContext::build(const Decl *D, Stmt &S, ASTContext &C) {
   CFG::BuildOptions Options;
-  Options.PruneTriviallyFalseEdges = false;
+  Options.PruneTriviallyFalseEdges = true;
   Options.AddImplicitDtors = true;
   Options.AddTemporaryDtors = true;
   Options.AddInitializers = true;
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index 35988af6d5bc6..a4d3768b8b189 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -247,9 +247,9 @@ void Environment::initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl) {
   for (const VarDecl *D : Vars) {
     if (getStorageLocation(*D, SkipPast::None) != nullptr)
       continue;
-    auto &Loc = createStorageLocation(*D);
+    auto &Loc = createStorageLocation(D->getType().getNonReferenceType());
     setStorageLocation(*D, Loc);
-    if (auto *Val = createValue(D->getType()))
+    if (auto *Val = createValue(D->getType().getNonReferenceType()))
       setValue(Loc, *Val);
   }
 
@@ -291,10 +291,16 @@ Environment::Environment(DataflowAnalysisContext &DACtx,
 
     for (const auto *ParamDecl : FuncDecl->parameters()) {
       assert(ParamDecl != nullptr);
-      auto &ParamLoc = createStorageLocation(*ParamDecl);
+      // References aren't objects, so the reference itself doesn't have a
+      // storage location. Instead, the storage location for a reference refers
+      // directly to an object of the referenced type -- so strip off any
+      // reference from the type.
+      auto &ParamLoc =
+          createStorageLocation(ParamDecl->getType().getNonReferenceType());
       setStorageLocation(*ParamDecl, ParamLoc);
-      if (Value *ParamVal = createValue(ParamDecl->getType()))
-        setValue(ParamLoc, *ParamVal);
+      if (Value *ParamVal =
+              createValue(ParamDecl->getType().getNonReferenceType()))
+          setValue(ParamLoc, *ParamVal);
     }
 
     QualType ReturnType = FuncDecl->getReturnType();
@@ -376,17 +382,19 @@ void Environment::pushCallInternal(const FunctionDecl *FuncDecl,
       continue;
 
     const VarDecl *Param = *ParamIt;
-    auto &Loc = createStorageLocation(*Param);
-    setStorageLocation(*Param, Loc);
 
     QualType ParamType = Param->getType();
     if (ParamType->isReferenceType()) {
-      auto &Val = arena().create<ReferenceValue>(*ArgLoc);
-      setValue(Loc, Val);
-    } else if (auto *ArgVal = getValue(*ArgLoc)) {
-      setValue(Loc, *ArgVal);
-    } else if (Value *Val = createValue(ParamType)) {
-      setValue(Loc, *Val);
+      setStorageLocation(*Param, *ArgLoc);
+    } else {
+      auto &Loc = createStorageLocation(*Param);
+      setStorageLocation(*Param, Loc);
+
+      if (auto *ArgVal = getValue(*ArgLoc)) {
+        setValue(Loc, *ArgVal);
+      } else if (Value *Val = createValue(ParamType)) {
+        setValue(Loc, *Val);
+      }
     }
   }
 }
@@ -518,6 +526,10 @@ LatticeJoinEffect Environment::join(const Environment &Other,
   JoinedEnv.ReturnLoc = ReturnLoc;
   JoinedEnv.ThisPointeeLoc = ThisPointeeLoc;
 
+  // FIXME: Once we're able to remove declarations from `DeclToLoc` when their
+  // lifetime ends, add an assertion that there aren't any entries in
+  // `DeclToLoc` and `Other.DeclToLoc` that map the same declaration to
+  // different storage locations.
   JoinedEnv.DeclToLoc = intersectDenseMaps(DeclToLoc, Other.DeclToLoc);
   if (DeclToLoc.size() != JoinedEnv.DeclToLoc.size())
     Effect = LatticeJoinEffect::Changed;
@@ -589,13 +601,23 @@ StorageLocation &Environment::createStorageLocation(const Expr &E) {
 
 void Environment::setStorageLocation(const ValueDecl &D, StorageLocation &Loc) {
   assert(!DeclToLoc.contains(&D));
+  assert(!isa_and_nonnull<ReferenceValue>(getValue(Loc)));
   DeclToLoc[&D] = &Loc;
 }
 
 StorageLocation *Environment::getStorageLocation(const ValueDecl &D,
                                                  SkipPast SP) const {
+  assert(SP != SkipPast::ReferenceThenPointer);
+
   auto It = DeclToLoc.find(&D);
-  return It == DeclToLoc.end() ? nullptr : &skip(*It->second, SP);
+  if (It == DeclToLoc.end())
+    return nullptr;
+
+  StorageLocation *Loc = It->second;
+
+  assert(!isa_and_nonnull<ReferenceValue>(getValue(*Loc)));
+
+  return Loc;
 }
 
 void Environment::setStorageLocation(const Expr &E, StorageLocation &Loc) {
@@ -662,6 +684,8 @@ Value *Environment::getValue(const StorageLocation &Loc) const {
 }
 
 Value *Environment::getValue(const ValueDecl &D, SkipPast SP) const {
+  assert(SP != SkipPast::ReferenceThenPointer);
+
   auto *Loc = getStorageLocation(D, SP);
   if (Loc == nullptr)
     return nullptr;
@@ -707,7 +731,7 @@ Value *Environment::createValueUnlessSelfReferential(
     // with integers, and so distinguishing them serves no purpose, but could
     // prevent convergence.
     CreatedValuesCount++;
-    return &arena().create<IntegerValue>();
+    return &DACtx->arena().create<IntegerValue>();
   }
 
   if (Type->isReferenceType() || Type->isPointerType()) {
@@ -725,9 +749,9 @@ Value *Environment::createValueUnlessSelfReferential(
     }
 
     if (Type->isReferenceType())
-      return &arena().create<ReferenceValue>(PointeeLoc);
+      return &DACtx->arena().create<ReferenceValue>(PointeeLoc);
     else
-      return &arena().create<PointerValue>(PointeeLoc);
+      return &DACtx->arena().create<PointerValue>(PointeeLoc);
   }
 
   if (Type->isRecordType()) {
@@ -747,7 +771,7 @@ Value *Environment::createValueUnlessSelfReferential(
       Visited.erase(FieldType.getCanonicalType());
     }
 
-    return &arena().create<StructValue>(std::move(FieldValues));
+    return &DACtx->arena().create<StructValue>(std::move(FieldValues));
   }
 
   return nullptr;
diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp
index b229194bc8f44..14293a3043f98 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp
@@ -67,6 +67,7 @@
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
 // Defines assets: HTMLLogger_{html_js,css}
 #include "HTMLLogger.inc"
@@ -79,6 +80,96 @@ llvm::Expected<std::string> renderSVG(llvm::StringRef DotGraph);
 
 using StreamFactory = std::function<std::unique_ptr<llvm::raw_ostream>()>;
 
+// Recursively dumps Values/StorageLocations as JSON
+class ModelDumper {
+public:
+  ModelDumper(llvm::json::OStream &JOS, const Environment &Env)
+      : JOS(JOS), Env(Env) {}
+
+  void dump(Value &V) {
+    JOS.attribute("value_id", llvm::to_string(&V));
+    if (!Visited.insert(&V).second)
+      return;
+
+    JOS.attribute("kind", debugString(V.getKind()));
+
+    switch (V.getKind()) {
+    case Value::Kind::Integer:
+    case Value::Kind::TopBool:
+    case Value::Kind::AtomicBool:
+      break;
+    case Value::Kind::Reference:
+      JOS.attributeObject(
+          "referent", [&] { dump(cast<ReferenceValue>(V).getReferentLoc()); });
+      break;
+    case Value::Kind::Pointer:
+      JOS.attributeObject(
+          "pointee", [&] { dump(cast<PointerValue>(V).getPointeeLoc()); });
+      break;
+    case Value::Kind::Struct:
+      for (const auto &Child : cast<StructValue>(V).children())
+        JOS.attributeObject("f:" + Child.first->getNameAsString(),
+                            [&] { dump(*Child.second); });
+      break;
+    case Value::Kind::Disjunction: {
+      auto &VV = cast<DisjunctionValue>(V);
+      JOS.attributeObject("lhs", [&] { dump(VV.getLeftSubValue()); });
+      JOS.attributeObject("rhs", [&] { dump(VV.getRightSubValue()); });
+      break;
+    }
+    case Value::Kind::Conjunction: {
+      auto &VV = cast<ConjunctionValue>(V);
+      JOS.attributeObject("lhs", [&] { dump(VV.getLeftSubValue()); });
+      JOS.attributeObject("rhs", [&] { dump(VV.getRightSubValue()); });
+      break;
+    }
+    case Value::Kind::Negation: {
+      auto &VV = cast<NegationValue>(V);
+      JOS.attributeObject("not", [&] { dump(VV.getSubVal()); });
+      break;
+    }
+    case Value::Kind::Implication: {
+      auto &VV = cast<ImplicationValue>(V);
+      JOS.attributeObject("if", [&] { dump(VV.getLeftSubValue()); });
+      JOS.attributeObject("then", [&] { dump(VV.getRightSubValue()); });
+      break;
+    }
+    case Value::Kind::Biconditional: {
+      auto &VV = cast<BiconditionalValue>(V);
+      JOS.attributeObject("lhs", [&] { dump(VV.getLeftSubValue()); });
+      JOS.attributeObject("rhs", [&] { dump(VV.getRightSubValue()); });
+      break;
+    }
+    }
+
+    for (const auto& Prop : V.properties())
+      JOS.attributeObject(("p:" + Prop.first()).str(),
+                          [&] { dump(*Prop.second); });
+
+    // Running the SAT solver is expensive, but knowing which booleans are
+    // guaranteed true/false here is valuable and hard to determine by hand.
+    if (auto *B = llvm::dyn_cast<BoolValue>(&V)) {
+      JOS.attribute("truth", Env.flowConditionImplies(*B) ? "true"
+                             : Env.flowConditionImplies(Env.makeNot(*B))
+                                 ? "false"
+                                 : "unknown");
+    }
+  }
+  void dump(const StorageLocation &L) {
+    JOS.attribute("location", llvm::to_string(&L));
+    if (!Visited.insert(&L).second)
+      return;
+
+    JOS.attribute("type", L.getType().getAsString());
+    if (auto *V = Env.getValue(L))
+      dump(*V);
+  }
+
+  llvm::DenseSet<const void*> Visited;
+  llvm::json::OStream &JOS;
+  const Environment &Env;
+};
+
 class HTMLLogger : public Logger {
   StreamFactory Streams;
   std::unique_ptr<llvm::raw_ostream> OS;
@@ -184,6 +275,16 @@ class HTMLLogger : public Logger {
       JOS->attribute("block", blockID(Block));
       JOS->attribute("iter", Iter);
       JOS->attribute("element", ElementIndex);
+
+      // If this state immediately follows an Expr, show its built-in model.
+      if (ElementIndex > 0) {
+        auto S =
+            Iters.back().first->Elements[ElementIndex - 1].getAs<CFGStmt>();
+        if (const Expr *E = S ? llvm::dyn_cast<Expr>(S->getStmt()) : nullptr)
+          if (auto *Loc = State.Env.getStorageLocation(*E, SkipPast::None))
+            JOS->attributeObject(
+                "value", [&] { ModelDumper(*JOS, State.Env).dump(*Loc); });
+      }
       if (!ContextLogs.empty()) {
         JOS->attribute("logs", ContextLogs);
         ContextLogs.clear();
diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.css b/clang/lib/Analysis/FlowSensitive/HTMLLogger.css
index 4877c1264c83b..c8212df1f94b8 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.css
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.css
@@ -116,3 +116,27 @@ code.line:has(.bb-select):before {
   position: relative;
   margin-left: 0.5em;
 }
+
+.value {
+  border: 1px solid #888;
+  font-size: x-small;
+  flex-grow: 1;
+}
+.value summary {
+  background-color: #ace;
+  display: flex;
+  justify-content: space-between;
+}
+.value .address {
+  font-size: xx-small;
+  font-family: monospace;
+  color: #888;
+}
+.value .property {
+  display: flex;
+  margin-top: 0.5em;
+}
+.value .property .key {
+  font-weight: bold;
+  min-width: 5em;
+}
diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.html b/clang/lib/Analysis/FlowSensitive/HTMLLogger.html
index c97f3ea8ac7d4..a60259a99cce0 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.html
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.html
@@ -10,6 +10,30 @@
 
 <head>
 <?INJECT?>
+
+<template id="value-template">
+  <details class="value" open>
+    <summary>
+      <span>{{v.kind}}
+        <template data-if="v.value_id"><span class="address">#{{v.value_id}}</span></template>
+      </span>
+      <template data-if="v.location">
+        <span>{{v.type}} <span class="address">@{{v.location}}</span></span>
+      </template>
+    </summary>
+    <template
+        data-for="kv in Object.entries(v)"
+        data-if="['kind', 'value_id', 'type', 'location'].indexOf(kv[0]) < 0">
+      <div class="property"><span class="key">{{kv[0]}}</span>
+        <template data-if="typeof(kv[1]) != 'object'">{{kv[1]}}</template>
+        <template data-if="typeof(kv[1]) == 'object'" data-let="v = kv[1]">
+          <template data-use="value-template"></template>
+        </template>
+      </div>
+    </template>
+  </details>
+</template>
+
 </head>
 
 <body>
@@ -35,9 +59,15 @@
   </template>
 </div>
 <table id="bb-elements">
+<template>
+  <tr id="{{selection.bb}}.0">
+    <td class="{{selection.bb}}">{{selection.bb}}.0</td>
+    <td>(initial state)</td>
+  </tr>
+</template>
 <template data-for="elt in cfg[selection.bb].elements">
-  <tr id="{{selection.bb}}.{{elt_index}}">
-    <td class="{{selection.bb}}">{{selection.bb}}.{{elt_index}}</td>
+  <tr id="{{selection.bb}}.{{elt_index+1}}">
+    <td class="{{selection.bb}}">{{selection.bb}}.{{elt_index+1}}</td>
     <td>{{elt}}</td>
   </tr>
 </template>
@@ -50,6 +80,10 @@
   <template data-if="state.element == 0">{{state.block}} (iteration {{state.iter}}) initial state</template>
   <template data-if="state.element != 0">Element {{selection.elt}} (iteration {{state.iter}})</template>
 </header>
+<template data-if="state.value" data-let="v = state.value">
+  <h2>Value</h2>
+  <template data-use="value-template"></template>
+</template>
 <template data-if="state.logs">
   <h2>Logs</h2>
   <pre>{{state.logs}}</pre>
diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.js b/clang/lib/Analysis/FlowSensitive/HTMLLogger.js
index 5958418bc1e93..6e04bc00f6631 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.js
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.js
@@ -73,6 +73,9 @@ function inflate(tmpl, data, parent, next) {
     }
     return parent.insertBefore(clone, next);
   }
+  // data-use="xyz": use <template id="xyz"> instead. (Allows recursion.)
+  if ('use' in tmpl.dataset)
+    return inflate(document.getElementById(tmpl.dataset.use), data, parent, next);
   // <template> tag handling. Base case: recursively inflate.
   function handle(data) {
     for (c of tmpl.content.childNodes)
diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
index 9745cd8a48311..e306b55753752 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
@@ -276,12 +276,6 @@ BoolValue *getHasValue(Environment &Env, Value *OptionalVal) {
   return nullptr;
 }
 
-/// If `Type` is a reference type, returns the type of its pointee. Otherwise,
-/// returns `Type` itself.
-QualType stripReference(QualType Type) {
-  return Type->isReferenceType() ? Type->getPointeeType() : Type;
-}
-
 /// Returns true if and only if `Type` is an optional type.
 bool isOptionalType(QualType Type) {
   if (!Type->isRecordType())
@@ -339,7 +333,7 @@ StorageLocation *maybeInitializeOptionalValueMember(QualType Q,
     return &ValueLoc;
   }
 
-  auto Ty = stripReference(Q);
+  auto Ty = Q.getNonReferenceType();
   auto *ValueVal = Env.createValue(Ty);
   if (ValueVal == nullptr)
     return nullptr;
@@ -493,11 +487,13 @@ BoolValue &valueOrConversionHasValue(const FunctionDecl &F, const Expr &E,
   assert(F.getTemplateSpecializationArgs() != nullptr);
   assert(F.getTemplateSpecializationArgs()->size() > 0);
 
-  const int TemplateParamOptionalWrappersCount = countOptionalWrappers(
-      *MatchRes.Context,
-      stripReference(F.getTemplateSpecializationArgs()->get(0).getAsType()));
-  const int ArgTypeOptionalWrappersCount =
-      countOptionalWrappers(*MatchRes.Context, stripReference(E.getType()));
+  const int TemplateParamOptionalWrappersCount =
+      countOptionalWrappers(*MatchRes.Context, F.getTemplateSpecializationArgs()
+                                                   ->get(0)
+                                                   .getAsType()
+                                                   .getNonReferenceType());
+  const int ArgTypeOptionalWrappersCount = countOptionalWrappers(
+      *MatchRes.Context, E.getType().getNonReferenceType());
 
   // Check if this is a constructor/assignment call for `optional<T>` with
   // argument of type `U` such that `T` is constructible from `U`.
diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
index 2d85e7b90f73e..ddabf352fd111 100644
--- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
@@ -223,24 +223,7 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
     if (DeclLoc == nullptr)
       return;
 
-    // If the value is already an lvalue, don't double-wrap it.
-    if (isa_and_nonnull<ReferenceValue>(Env.getValue(*DeclLoc))) {
-      // We only expect to encounter a `ReferenceValue` for a reference type
-      // (always) or for `BindingDecl` (sometimes). For the latter, we can't
-      // rely on type, because their type does not indicate whether they are a
-      // reference type. The assert is not strictly necessary, since we don't
-      // depend on its truth to proceed. But, it verifies our assumptions,
-      // which, if violated, probably indicate a problem elsewhere.
-      assert((VD->getType()->isReferenceType() || isa<BindingDecl>(VD)) &&
-             "Only reference-typed declarations or `BindingDecl`s should map "
-             "to `ReferenceValue`s");
-      Env.setStorageLocation(*S, *DeclLoc);
-    } else {
-      auto &Loc = Env.createStorageLocation(*S);
-      auto &Val = Env.create<ReferenceValue>(*DeclLoc);
-      Env.setStorageLocation(*S, Loc);
-      Env.setValue(Loc, Val);
-    }
+    Env.setStorageLocation(*S, *DeclLoc);
   }
 
   void VisitDeclStmt(const DeclStmt *S) {
@@ -248,55 +231,69 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
     // is safe.
     const auto &D = *cast<VarDecl>(S->getSingleDecl());
 
+    ProcessVarDecl(D);
+  }
+
+  void ProcessVarDecl(const VarDecl &D) {
     // Static local vars are already initialized in `Environment`.
     if (D.hasGlobalStorage())
       return;
 
-    // The storage location for `D` could have been created earlier, before the
-    // variable's declaration statement (for example, in the case of
-    // BindingDecls).
-    auto *MaybeLoc = Env.getStorageLocation(D, SkipPast::None);
-    if (MaybeLoc == nullptr) {
-      MaybeLoc = &Env.createStorageLocation(D);
-      Env.setStorageLocation(D, *MaybeLoc);
-    }
-    auto &Loc = *MaybeLoc;
+    if (D.getType()->isReferenceType()) {
+      // If this is the holding variable for a `BindingDecl`, we may already
+      // have a storage location set up -- so check. (See also explanation below
+      // where we process the `BindingDecl`.)
+      if (Env.getStorageLocation(D) == nullptr) {
+        const Expr *InitExpr = D.getInit();
+        assert(InitExpr != nullptr);
+        if (auto *InitExprLoc =
+                Env.getStorageLocation(*InitExpr, SkipPast::Reference)) {
+          Env.setStorageLocation(D, *InitExprLoc);
+        } else {
+          // Even though we have an initializer, we might not get an
+          // InitExprLoc, for example if the InitExpr is a CallExpr for which we
+          // don't have a function body. In this case, we just invent a storage
+          // location and value -- it's the best we can do.
+          StorageLocation &Loc =
+              Env.createStorageLocation(D.getType().getNonReferenceType());
+          Env.setStorageLocation(D, Loc);
+          if (Value *Val = Env.createValue(D.getType().getNonReferenceType()))
+            Env.setValue(Loc, *Val);
+        }
+      }
+    } else {
+      // Not a reference type.
 
-    const Expr *InitExpr = D.getInit();
-    if (InitExpr == nullptr) {
-      // No initializer expression - associate `Loc` with a new value.
-      if (Value *Val = Env.createValue(D.getType()))
-        Env.setValue(Loc, *Val);
-      return;
-    }
+      assert(Env.getStorageLocation(D) == nullptr);
+      StorageLocation &Loc = Env.createStorageLocation(D);
+      Env.setStorageLocation(D, Loc);
 
-    if (D.getType()->isReferenceType()) {
-      // Initializing a reference variable - do not create a reference to
-      // reference.
-      // FIXME: reuse the ReferenceValue instead of creating a new one.
-      if (auto *InitExprLoc =
-              Env.getStorageLocation(*InitExpr, SkipPast::Reference)) {
-        auto &Val = Env.create<ReferenceValue>(*InitExprLoc);
-        Env.setValue(Loc, Val);
+      const Expr *InitExpr = D.getInit();
+      if (InitExpr == nullptr) {
+        // No initializer expression - associate `Loc` with a new value.
+        if (Value *Val = Env.createValue(D.getType()))
+          Env.setValue(Loc, *Val);
+        return;
       }
-    } else if (auto *InitExprVal = Env.getValue(*InitExpr, SkipPast::None)) {
-      Env.setValue(Loc, *InitExprVal);
-    }
 
-    if (Env.getValue(Loc) == nullptr) {
-      // We arrive here in (the few) cases where an expression is intentionally
-      // "uninterpreted". There are two ways to handle this situation: propagate
-      // the status, so that uninterpreted initializers result in uninterpreted
-      // variables, or provide a default value. We choose the latter so that
-      // later refinements of the variable can be used for reasoning about the
-      // surrounding code.
-      //
-      // FIXME. If and when we interpret all language cases, change this to
-      // assert that `InitExpr` is interpreted, rather than supplying a default
-      // value (assuming we don't update the environment API to return
-      // references).
-      if (Value *Val = Env.createValue(D.getType()))
-        Env.setValue(Loc, *Val);
+      if (auto *InitExprVal = Env.getValue(*InitExpr, SkipPast::None))
+        Env.setValue(Loc, *InitExprVal);
+
+      if (Env.getValue(Loc) == nullptr) {
+        // We arrive here in (the few) cases where an expression is
+        // intentionally "uninterpreted". There are two ways to handle this
+        // situation: propagate the status, so that uninterpreted initializers
+        // result in uninterpreted variables, or provide a default value. We
+        // choose the latter so that later refinements of the variable can be
+        // used for reasoning about the surrounding code.
+        //
+        // FIXME. If and when we interpret all language cases, change this to
+        // assert that `InitExpr` is interpreted, rather than supplying a
+        // default value (assuming we don't update the environment API to return
+        // references).
+        if (Value *Val = Env.createValue(D.getType()))
+          Env.setValue(Loc, *Val);
+      }
     }
 
     // `DecompositionDecl` must be handled after we've interpreted the loc
@@ -322,15 +319,16 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
           if (auto *Loc = Env.getStorageLocation(*ME, SkipPast::Reference))
             Env.setStorageLocation(*B, *Loc);
         } else if (auto *VD = B->getHoldingVar()) {
-          // Holding vars are used to back the BindingDecls of tuple-like
-          // types. The holding var declarations appear *after* this statement,
-          // so we have to create a location for them here to share with `B`. We
-          // don't visit the binding, because we know it will be a DeclRefExpr
-          // to `VD`. Note that, by construction of the AST, `VD` will always be
-          // a reference -- either lvalue or rvalue.
-          auto &VDLoc = Env.createStorageLocation(*VD);
-          Env.setStorageLocation(*VD, VDLoc);
-          Env.setStorageLocation(*B, VDLoc);
+          // Holding vars are used to back the `BindingDecl`s of tuple-like
+          // types. The holding var declarations appear after the
+          // `DecompositionDecl`, so we have to explicitly process them here
+          // to know their storage location. They will be processed a second
+          // time when we visit their `VarDecl`s, so we have code that protects
+          // against this above.
+          ProcessVarDecl(*VD);
+          auto *VDLoc = Env.getStorageLocation(*VD);
+          assert(VDLoc != nullptr);
+          Env.setStorageLocation(*B, *VDLoc);
         }
       }
     }
@@ -496,7 +494,7 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
   }
 
   void VisitReturnStmt(const ReturnStmt *S) {
-    if (!Env.getAnalysisOptions().ContextSensitiveOpts)
+    if (!Env.getDataflowAnalysisContext().getOptions().ContextSensitiveOpts)
       return;
 
     auto *Ret = S->getRetValue();
@@ -539,15 +537,7 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
         if (VarDeclLoc == nullptr)
           return;
 
-        if (VarDeclLoc->getType()->isReferenceType()) {
-          assert(isa_and_nonnull<ReferenceValue>(Env.getValue((*VarDeclLoc))) &&
-                 "reference-typed declarations map to `ReferenceValue`s");
-          Env.setStorageLocation(*S, *VarDeclLoc);
-        } else {
-          auto &Loc = Env.createStorageLocation(*S);
-          Env.setStorageLocation(*S, Loc);
-          Env.setValue(Loc, Env.create<ReferenceValue>(*VarDeclLoc));
-        }
+        Env.setStorageLocation(*S, *VarDeclLoc);
         return;
       }
     }
@@ -646,9 +636,12 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
       assert(Arg1 != nullptr);
 
       // Evaluate only copy and move assignment operators.
-      auto *Arg0Type = Arg0->getType()->getUnqualifiedDesugaredType();
-      auto *Arg1Type = Arg1->getType()->getUnqualifiedDesugaredType();
-      if (Arg0Type != Arg1Type)
+      const auto *Method =
+          dyn_cast_or_null<CXXMethodDecl>(S->getDirectCallee());
+      if (!Method)
+        return;
+      if (!Method->isCopyAssignmentOperator() &&
+          !Method->isMoveAssignmentOperator())
         return;
 
       auto *ObjectLoc = Env.getStorageLocation(*Arg0, SkipPast::Reference);
@@ -860,12 +853,13 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
   // `F` of `S`. The type `E` must be either `CallExpr` or `CXXConstructExpr`.
   template <typename E>
   void transferInlineCall(const E *S, const FunctionDecl *F) {
-    const auto &Options = Env.getAnalysisOptions();
+    const auto &Options = Env.getDataflowAnalysisContext().getOptions();
     if (!(Options.ContextSensitiveOpts &&
           Env.canDescend(Options.ContextSensitiveOpts->Depth, F)))
       return;
 
-    const ControlFlowContext *CFCtx = Env.getControlFlowContext(F);
+    const ControlFlowContext *CFCtx =
+        Env.getDataflowAnalysisContext().getControlFlowContext(F);
     if (!CFCtx)
       return;
 
diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
index ad250c432e2c5..418a6cff21cc1 100644
--- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
+++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
@@ -64,6 +64,8 @@ static bool isLoopHead(const CFGBlock &B) {
   return false;
 }
 
+namespace {
+
 // The return type of the visit functions in TerminatorVisitor. The first
 // element represents the terminator expression (that is the conditional
 // expression in case of a path split in the CFG). The second element
@@ -168,7 +170,8 @@ struct AnalysisContext {
                   llvm::ArrayRef<std::optional<TypeErasedDataflowAnalysisState>>
                       BlockStates)
       : CFCtx(CFCtx), Analysis(Analysis), InitEnv(InitEnv),
-        Log(InitEnv.logger()), BlockStates(BlockStates) {
+        Log(*InitEnv.getDataflowAnalysisContext().getOptions().Log),
+        BlockStates(BlockStates) {
     Log.beginAnalysis(CFCtx, Analysis);
   }
   ~AnalysisContext() { Log.endAnalysis(); }
@@ -185,6 +188,8 @@ struct AnalysisContext {
   llvm::ArrayRef<std::optional<TypeErasedDataflowAnalysisState>> BlockStates;
 };
 
+} // namespace
+
 /// Computes the input state for a given basic block by joining the output
 /// states of its predecessors.
 ///
@@ -276,17 +281,19 @@ computeBlockInputState(const CFGBlock &Block, AnalysisContext &AC) {
 }
 
 /// Built-in transfer function for `CFGStmt`.
-void builtinTransferStatement(const CFGStmt &Elt,
-                              TypeErasedDataflowAnalysisState &InputState,
-                              AnalysisContext &AC) {
+static void
+builtinTransferStatement(const CFGStmt &Elt,
+                         TypeErasedDataflowAnalysisState &InputState,
+                         AnalysisContext &AC) {
   const Stmt *S = Elt.getStmt();
   assert(S != nullptr);
   transfer(StmtToEnvMap(AC.CFCtx, AC.BlockStates), *S, InputState.Env);
 }
 
 /// Built-in transfer function for `CFGInitializer`.
-void builtinTransferInitializer(const CFGInitializer &Elt,
-                                TypeErasedDataflowAnalysisState &InputState) {
+static void
+builtinTransferInitializer(const CFGInitializer &Elt,
+                           TypeErasedDataflowAnalysisState &InputState) {
   const CXXCtorInitializer *Init = Elt.getInitializer();
   assert(Init != nullptr);
 
@@ -319,9 +326,9 @@ void builtinTransferInitializer(const CFGInitializer &Elt,
   }
 }
 
-void builtinTransfer(const CFGElement &Elt,
-                     TypeErasedDataflowAnalysisState &State,
-                     AnalysisContext &AC) {
+static void builtinTransfer(const CFGElement &Elt,
+                            TypeErasedDataflowAnalysisState &State,
+                            AnalysisContext &AC) {
   switch (Elt.getKind()) {
   case CFGElement::Statement:
     builtinTransferStatement(Elt.castAs<CFGStmt>(), State, AC);
@@ -330,7 +337,18 @@ void builtinTransfer(const CFGElement &Elt,
     builtinTransferInitializer(Elt.castAs<CFGInitializer>(), State);
     break;
   default:
-    // FIXME: Evaluate other kinds of `CFGElement`.
+    // FIXME: Evaluate other kinds of `CFGElement`, including:
+    // - When encountering `CFGLifetimeEnds`, remove the declaration from
+    //   `Environment::DeclToLoc`. This would serve two purposes:
+    //   a) Eliminate unnecessary clutter from `Environment::DeclToLoc`
+    //   b) Allow us to implement an assertion that, when joining two
+    //      `Environments`, the two `DeclToLoc` maps never contain entries that
+    //      map the same declaration to different storage locations.
+    //   Unfortunately, however, we can't currently process `CFGLifetimeEnds`
+    //   because the corresponding CFG option `AddLifetime` is incompatible with
+    //   the option 'AddImplicitDtors`, which we already use. We will first
+    //   need to modify the CFG implementation to make these two options
+    //   compatible before we can process `CFGLifetimeEnds`.
     break;
   }
 }
@@ -343,7 +361,7 @@ void builtinTransfer(const CFGElement &Elt,
 /// user-specified analysis.
 /// `PostVisitCFG` (if provided) will be applied to the element after evaluation
 /// by the user-specified analysis.
-TypeErasedDataflowAnalysisState
+static TypeErasedDataflowAnalysisState
 transferCFGBlock(const CFGBlock &Block, AnalysisContext &AC,
                  std::function<void(const CFGElement &,
                                     const TypeErasedDataflowAnalysisState &)>
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 7a432ea5323a4..7871fed519b98 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -332,7 +332,7 @@ class FixableGadget : public Gadget {
   bool isWarningGadget() const final { return false; }
 
   /// Returns a fixit that would fix the current gadget according to
-  /// the current strategy. Returns None if the fix cannot be produced;
+  /// the current strategy. Returns std::nullopt if the fix cannot be produced;
   /// returns an empty list if no fixes are necessary.
   virtual std::optional<FixItList> getFixits(const Strategy &) const {
     return std::nullopt;
@@ -1117,42 +1117,44 @@ static std::string getAPIntText(APInt Val) {
 
 // Return the source location of the last character of the AST `Node`.
 template <typename NodeTy>
-static SourceLocation getEndCharLoc(const NodeTy *Node, const SourceManager &SM,
-                                    const LangOptions &LangOpts) {
+static std::optional<SourceLocation>
+getEndCharLoc(const NodeTy *Node, const SourceManager &SM,
+              const LangOptions &LangOpts) {
   unsigned TkLen = Lexer::MeasureTokenLength(Node->getEndLoc(), SM, LangOpts);
   SourceLocation Loc = Node->getEndLoc().getLocWithOffset(TkLen - 1);
 
-  // We expect `Loc` to be valid. The location is obtained by moving forward
-  // from the beginning of the token 'len(token)-1' characters. The file ID of
-  // the locations within a token must be consistent.
-  assert(Loc.isValid() && "Expected the source location of the last"
-                          "character of an AST Node to be always valid");
-  return Loc;
+  if (Loc.isValid())
+    return Loc;
+
+  return std::nullopt;
 }
 
 // Return the source location just past the last character of the AST `Node`.
 template <typename NodeTy>
-static SourceLocation getPastLoc(const NodeTy *Node, const SourceManager &SM,
-                                 const LangOptions &LangOpts) {
+static std::optional<SourceLocation> getPastLoc(const NodeTy *Node,
+                                                const SourceManager &SM,
+                                                const LangOptions &LangOpts) {
   SourceLocation Loc =
       Lexer::getLocForEndOfToken(Node->getEndLoc(), 0, SM, LangOpts);
 
-  // We expect `Loc` to be valid as it is either associated to a file ID or
-  //   it must be the end of a macro expansion. (see
-  //   `Lexer::getLocForEndOfToken`)
-  assert(Loc.isValid() && "Expected the source location just past the last "
-                          "character of an AST Node to be always valid");
-  return Loc;
+  if (Loc.isValid())
+    return Loc;
+
+  return std::nullopt;
 }
 
 // Return text representation of an `Expr`.
-static StringRef getExprText(const Expr *E, const SourceManager &SM,
-                             const LangOptions &LangOpts) {
-  SourceLocation LastCharLoc = getPastLoc(E, SM, LangOpts);
+static std::optional<StringRef> getExprText(const Expr *E,
+                                            const SourceManager &SM,
+                                            const LangOptions &LangOpts) {
+  std::optional<SourceLocation> LastCharLoc = getPastLoc(E, SM, LangOpts);
+
+  if (LastCharLoc)
+    return Lexer::getSourceText(
+        CharSourceRange::getCharRange(E->getBeginLoc(), *LastCharLoc), SM,
+        LangOpts);
 
-  return Lexer::getSourceText(
-      CharSourceRange::getCharRange(E->getBeginLoc(), LastCharLoc), SM,
-      LangOpts);
+  return std::nullopt;
 }
 
 std::optional<FixItList>
@@ -1191,12 +1193,24 @@ DerefSimplePtrArithFixableGadget::getFixits(const Strategy &s) const {
     CharSourceRange StarWithTrailWhitespace =
         clang::CharSourceRange::getCharRange(DerefOp->getOperatorLoc(),
                                              LHS->getBeginLoc());
+
+    std::optional<SourceLocation> LHSLocation = getPastLoc(LHS, SM, LangOpts);
+    if (!LHSLocation)
+      return std::nullopt;
+
     CharSourceRange PlusWithSurroundingWhitespace =
-        clang::CharSourceRange::getCharRange(getPastLoc(LHS, SM, LangOpts),
-                                             RHS->getBeginLoc());
+        clang::CharSourceRange::getCharRange(*LHSLocation, RHS->getBeginLoc());
+
+    std::optional<SourceLocation> AddOpLocation =
+        getPastLoc(AddOp, SM, LangOpts);
+    std::optional<SourceLocation> DerefOpLocation =
+        getPastLoc(DerefOp, SM, LangOpts);
+
+    if (!AddOpLocation || !DerefOpLocation)
+      return std::nullopt;
+
     CharSourceRange ClosingParenWithPrecWhitespace =
-        clang::CharSourceRange::getCharRange(getPastLoc(AddOp, SM, LangOpts),
-                                             getPastLoc(DerefOp, SM, LangOpts));
+        clang::CharSourceRange::getCharRange(*AddOpLocation, *DerefOpLocation);
 
     return FixItList{
         {FixItHint::CreateRemoval(StarWithTrailWhitespace),
@@ -1218,12 +1232,12 @@ PointerDereferenceGadget::getFixits(const Strategy &S) const {
     CharSourceRange derefRange = clang::CharSourceRange::getCharRange(
         Op->getBeginLoc(), Op->getBeginLoc().getLocWithOffset(1));
     // Inserts the [0]
-    std::optional<SourceLocation> endOfOperand =
+    std::optional<SourceLocation> EndOfOperand =
         getEndCharLoc(BaseDeclRefExpr, SM, Ctx.getLangOpts());
-    if (endOfOperand) {
+    if (EndOfOperand) {
       return FixItList{{FixItHint::CreateRemoval(derefRange),
                         FixItHint::CreateInsertion(
-                            endOfOperand.value().getLocWithOffset(1), "[0]")}};
+                            (*EndOfOperand).getLocWithOffset(1), "[0]")}};
     }
   }
     [[fallthrough]];
@@ -1248,11 +1262,14 @@ std::optional<FixItList> UPCStandalonePointerGadget::getFixits(const Strategy &S
       ASTContext &Ctx = VD->getASTContext();
       SourceManager &SM = Ctx.getSourceManager();
       // Inserts the .data() after the DRE
-      SourceLocation endOfOperand = getEndCharLoc(Node, SM, Ctx.getLangOpts());
+      std::optional<SourceLocation> EndOfOperand =
+          getPastLoc(Node, SM, Ctx.getLangOpts());
 
-      return FixItList{{FixItHint::CreateInsertion(
-          endOfOperand.getLocWithOffset(1), ".data()")}};
+      if (EndOfOperand)
+        return FixItList{{FixItHint::CreateInsertion(
+            *EndOfOperand, ".data()")}};
     }
+      [[fallthrough]];
     case Strategy::Kind::Wontfix:
     case Strategy::Kind::Iterator:
     case Strategy::Kind::Array:
@@ -1281,12 +1298,20 @@ fixUPCAddressofArraySubscriptWithSpan(const UnaryOperator *Node) {
   if (auto ICE = Idx->getIntegerConstantExpr(Ctx))
     if ((*ICE).isZero())
       IdxIsLitZero = true;
+  std::optional<StringRef> DreString = getExprText(DRE, SM, LangOpts);
+  if (!DreString)
+    return std::nullopt;
+
   if (IdxIsLitZero) {
     // If the index is literal zero, we produce the most concise fix-it:
-    SS << getExprText(DRE, SM, LangOpts).str() << ".data()";
+    SS << (*DreString).str() << ".data()";
   } else {
-    SS << "&" << getExprText(DRE, SM, LangOpts).str() << ".data()"
-       << "[" << getExprText(Idx, SM, LangOpts).str() << "]";
+    std::optional<StringRef> IndexString = getExprText(Idx, SM, LangOpts);
+    if (!IndexString)
+      return std::nullopt;
+
+    SS << "&" << (*DreString).str() << ".data()"
+       << "[" << (*IndexString).str() << "]";
   }
   return FixItList{
       FixItHint::CreateReplacement(Node->getSourceRange(), SS.str())};
@@ -1310,11 +1335,13 @@ std::optional<FixItList> UPCPreIncrementGadget::getFixits(const Strategy &S) con
       // To transform UPC(++p) to UPC((p = p.subspan(1)).data()):
       SS << "(" << varName.data() << " = " << varName.data()
          << ".subspan(1)).data()";
+      std::optional<SourceLocation> PreIncLocation =
+          getEndCharLoc(PreIncNode, Ctx.getSourceManager(), Ctx.getLangOpts());
+      if (!PreIncLocation)
+        return std::nullopt;
+
       Fixes.push_back(FixItHint::CreateReplacement(
-          SourceRange(PreIncNode->getBeginLoc(),
-                      getEndCharLoc(PreIncNode, Ctx.getSourceManager(),
-                                    Ctx.getLangOpts())),
-          SS.str()));
+          SourceRange(PreIncNode->getBeginLoc(), *PreIncLocation), SS.str()));
       return Fixes;
     }
   }
@@ -1350,7 +1377,12 @@ populateInitializerFixItWithSpan(const Expr *Init, const ASTContext &Ctx,
           // &`? It should. Maybe worth an NFC patch later.
           Expr::NullPointerConstantValueDependence::
               NPC_ValueDependentIsNotNull)) {
-    SourceRange SR(Init->getBeginLoc(), getEndCharLoc(Init, SM, LangOpts));
+    std::optional<SourceLocation> InitLocation =
+        getEndCharLoc(Init, SM, LangOpts);
+    if (!InitLocation)
+      return {};
+
+    SourceRange SR(Init->getBeginLoc(), *InitLocation);
 
     return {FixItHint::CreateRemoval(SR)};
   }
@@ -1368,8 +1400,12 @@ populateInitializerFixItWithSpan(const Expr *Init, const ASTContext &Ctx,
     // of `T`. So the extent is `n` unless `n` has side effects.  Similar but
     // simpler for the case where `Init` is `new T`.
     if (const Expr *Ext = CxxNew->getArraySize().value_or(nullptr)) {
-      if (!Ext->HasSideEffects(Ctx))
-        ExtentText = getExprText(Ext, SM, LangOpts);
+      if (!Ext->HasSideEffects(Ctx)) {
+        std::optional<StringRef> ExtentString = getExprText(Ext, SM, LangOpts);
+        if (!ExtentString)
+          return {};
+        ExtentText = *ExtentString;
+      }
     } else if (!CxxNew->isArray())
       // Although the initializer is not allocating a buffer, the pointer
       // variable could still be used in buffer access operations.
@@ -1392,12 +1428,15 @@ populateInitializerFixItWithSpan(const Expr *Init, const ASTContext &Ctx,
   }
 
   SmallString<32> StrBuffer{};
-  SourceLocation LocPassInit = getPastLoc(Init, SM, LangOpts);
+  std::optional<SourceLocation> LocPassInit = getPastLoc(Init, SM, LangOpts);
+
+  if (!LocPassInit)
+    return {};
 
   StrBuffer.append(", ");
   StrBuffer.append(ExtentText);
   StrBuffer.append("}");
-  FixIts.push_back(FixItHint::CreateInsertion(LocPassInit, StrBuffer.str()));
+  FixIts.push_back(FixItHint::CreateInsertion(*LocPassInit, StrBuffer.str()));
   return FixIts;
 }
 
@@ -1420,7 +1459,7 @@ static FixItList fixVarDeclWithSpan(const VarDecl *D, const ASTContext &Ctx,
   assert(!SpanEltT.isNull() && "Trying to fix a non-pointer type variable!");
 
   FixItList FixIts{};
-  SourceLocation
+  std::optional<SourceLocation>
       ReplacementLastLoc; // the inclusive end location of the replacement
   const SourceManager &SM = Ctx.getSourceManager();
 
@@ -1428,8 +1467,9 @@ static FixItList fixVarDeclWithSpan(const VarDecl *D, const ASTContext &Ctx,
     FixItList InitFixIts =
         populateInitializerFixItWithSpan(Init, Ctx, UserFillPlaceHolder);
 
-    assert(!InitFixIts.empty() &&
-           "Unable to fix initializer of unsafe variable declaration");
+    if (InitFixIts.empty())
+      return {};
+
     // The loc right before the initializer:
     ReplacementLastLoc = Init->getBeginLoc().getLocWithOffset(-1);
     FixIts.insert(FixIts.end(), std::make_move_iterator(InitFixIts.begin()),
@@ -1442,8 +1482,12 @@ static FixItList fixVarDeclWithSpan(const VarDecl *D, const ASTContext &Ctx,
   raw_svector_ostream OS(Replacement);
 
   OS << "std::span<" << SpanEltT.getAsString() << "> " << D->getName();
+
+  if (!ReplacementLastLoc)
+    return {};
+
   FixIts.push_back(FixItHint::CreateReplacement(
-      SourceRange{D->getBeginLoc(), ReplacementLastLoc}, OS.str()));
+      SourceRange{D->getBeginLoc(), *ReplacementLastLoc}, OS.str()));
   return FixIts;
 }
 
diff --git a/clang/lib/Basic/CMakeLists.txt b/clang/lib/Basic/CMakeLists.txt
index c05036a216c74..caa1b6002e6f1 100644
--- a/clang/lib/Basic/CMakeLists.txt
+++ b/clang/lib/Basic/CMakeLists.txt
@@ -125,7 +125,6 @@ add_clang_library(clangBasic
 
   DEPENDS
   omp_gen
-  RISCVTargetParserTableGen
   )
 
 target_link_libraries(clangBasic
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index 3474acbced195..b2106b0eb6611 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -43,28 +43,12 @@ using namespace clang;
 
 const StreamingDiagnostic &clang::operator<<(const StreamingDiagnostic &DB,
                                              DiagNullabilityKind nullability) {
-  StringRef string;
-  switch (nullability.first) {
-  case NullabilityKind::NonNull:
-    string = nullability.second ? "'nonnull'" : "'_Nonnull'";
-    break;
-
-  case NullabilityKind::Nullable:
-    string = nullability.second ? "'nullable'" : "'_Nullable'";
-    break;
-
-  case NullabilityKind::Unspecified:
-    string = nullability.second ? "'null_unspecified'" : "'_Null_unspecified'";
-    break;
-
-  case NullabilityKind::NullableResult:
-    assert(!nullability.second &&
-           "_Nullable_result isn't supported as context-sensitive keyword");
-    string = "_Nullable_result";
-    break;
-  }
-
-  DB.AddString(string);
+  DB.AddString(
+      ("'" +
+       getNullabilitySpelling(nullability.first,
+                              /*isContextSensitive=*/nullability.second) +
+       "'")
+          .str());
   return DB;
 }
 
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 97dc5cd2d9a6d..9be42c0f95ddc 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -849,6 +849,20 @@ StringRef clang::getNullabilitySpelling(NullabilityKind kind,
   llvm_unreachable("Unknown nullability kind.");
 }
 
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, NullabilityKind NK) {
+  switch (NK) {
+  case NullabilityKind::NonNull:
+    return OS << "NonNull";
+  case NullabilityKind::Nullable:
+    return OS << "Nullable";
+  case NullabilityKind::NullableResult:
+    return OS << "NullableResult";
+  case NullabilityKind::Unspecified:
+    return OS << "Unspecified";
+  }
+  llvm_unreachable("Unknown nullability kind.");
+}
+
 diag::kind
 IdentifierTable::getFutureCompatDiagKind(const IdentifierInfo &II,
                                          const LangOptions &LangOpts) {
diff --git a/clang/lib/Basic/LangOptions.cpp b/clang/lib/Basic/LangOptions.cpp
index f22fe9a82bda4..1abbaa9e4b2b4 100644
--- a/clang/lib/Basic/LangOptions.cpp
+++ b/clang/lib/Basic/LangOptions.cpp
@@ -117,7 +117,7 @@ void LangOptions::setLangDefaults(LangOptions &Opts, Language Lang,
   Opts.CPlusPlus14 = Std.isCPlusPlus14();
   Opts.CPlusPlus17 = Std.isCPlusPlus17();
   Opts.CPlusPlus20 = Std.isCPlusPlus20();
-  Opts.CPlusPlus2b = Std.isCPlusPlus2b();
+  Opts.CPlusPlus23 = Std.isCPlusPlus23();
   Opts.GNUMode = Std.isGNUMode();
   Opts.GNUCVersion = 0;
   Opts.HexFloats = Std.hasHexFloats();
diff --git a/clang/lib/Basic/TargetID.cpp b/clang/lib/Basic/TargetID.cpp
index 5d5708d274a09..3c06d9bad1dc0 100644
--- a/clang/lib/Basic/TargetID.cpp
+++ b/clang/lib/Basic/TargetID.cpp
@@ -133,7 +133,7 @@ std::string getCanonicalTargetID(llvm::StringRef Processor,
   std::map<const llvm::StringRef, bool> OrderedMap;
   for (const auto &F : Features)
     OrderedMap[F.first()] = F.second;
-  for (auto F : OrderedMap)
+  for (const auto &F : OrderedMap)
     TargetID = TargetID + ':' + F.first.str() + (F.second ? "+" : "-");
   return TargetID;
 }
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 67ba5f6b35145..3840139d27434 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -333,6 +333,8 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
                                          MacroBuilder &Builder) const {
   // Target identification.
   Builder.defineMacro("__aarch64__");
+  // Inline assembly supports AArch64 flag outputs.
+  Builder.defineMacro("__GCC_ASM_FLAG_OUTPUTS__");
   // For bare-metal.
   if (getTriple().getOS() == llvm::Triple::UnknownOS &&
       getTriple().isOSBinFormatELF())
@@ -1210,6 +1212,52 @@ ArrayRef<TargetInfo::GCCRegAlias> AArch64TargetInfo::getGCCRegAliases() const {
   return llvm::ArrayRef(GCCRegAliases);
 }
 
+// Returns the length of cc constraint.
+static unsigned matchAsmCCConstraint(const char *Name) {
+  constexpr unsigned len = 5;
+  auto RV = llvm::StringSwitch<unsigned>(Name)
+                .Case("@cceq", len)
+                .Case("@ccne", len)
+                .Case("@cchs", len)
+                .Case("@cccs", len)
+                .Case("@cccc", len)
+                .Case("@cclo", len)
+                .Case("@ccmi", len)
+                .Case("@ccpl", len)
+                .Case("@ccvs", len)
+                .Case("@ccvc", len)
+                .Case("@cchi", len)
+                .Case("@ccls", len)
+                .Case("@ccge", len)
+                .Case("@cclt", len)
+                .Case("@ccgt", len)
+                .Case("@ccle", len)
+                .Default(0);
+  return RV;
+}
+
+std::string
+AArch64TargetInfo::convertConstraint(const char *&Constraint) const {
+  std::string R;
+  switch (*Constraint) {
+  case 'U': // Three-character constraint; add "@3" hint for later parsing.
+    R = std::string("@3") + std::string(Constraint, 3);
+    Constraint += 2;
+    break;
+  case '@':
+    if (const unsigned Len = matchAsmCCConstraint(Constraint)) {
+      std::string Converted = "{" + std::string(Constraint, Len) + "}";
+      Constraint += Len - 1;
+      return Converted;
+    }
+    return std::string(1, *Constraint);
+  default:
+    R = TargetInfo::convertConstraint(Constraint);
+    break;
+  }
+  return R;
+}
+
 bool AArch64TargetInfo::validateAsmConstraint(
     const char *&Name, TargetInfo::ConstraintInfo &Info) const {
   switch (*Name) {
@@ -1257,6 +1305,13 @@ bool AArch64TargetInfo::validateAsmConstraint(
   case 'y': // SVE registers (V0-V7)
     Info.setAllowsRegister();
     return true;
+  case '@':
+    // CC condition
+    if (const unsigned Len = matchAsmCCConstraint(Name)) {
+      Name += Len - 1;
+      Info.setAllowsRegister();
+      return true;
+    }
   }
   return false;
 }
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 967a888f6b6f8..cce80e10f2bb1 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -173,19 +173,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   ArrayRef<const char *> getGCCRegNames() const override;
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override;
 
-  std::string convertConstraint(const char *&Constraint) const override {
-    std::string R;
-    switch (*Constraint) {
-    case 'U': // Three-character constraint; add "@3" hint for later parsing.
-      R = std::string("@3") + std::string(Constraint, 3);
-      Constraint += 2;
-      break;
-    default:
-      R = TargetInfo::convertConstraint(Constraint);
-      break;
-    }
-    return R;
-  }
+  std::string convertConstraint(const char *&Constraint) const override;
 
   bool validateAsmConstraint(const char *&Name,
                              TargetInfo::ConstraintInfo &Info) const override;
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index 5b99755c21e2b..0785b2b9c0edd 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -33,9 +33,9 @@ static const char *const DataLayoutStringR600 =
 
 static const char *const DataLayoutStringAMDGCN =
     "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
-    "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
+    "-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
     "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1"
-    "-ni:7";
+    "-ni:7:8";
 
 const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     Generic,  // Default
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index 437ad7253a31c..1b0cc4d0b13ff 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -27,8 +27,8 @@ namespace clang {
 namespace targets {
 
 M68kTargetInfo::M68kTargetInfo(const llvm::Triple &Triple,
-                               const TargetOptions &)
-    : TargetInfo(Triple) {
+                               const TargetOptions &Opts)
+    : TargetInfo(Triple), TargetOpts(Opts) {
 
   std::string Layout;
 
@@ -120,6 +120,11 @@ void M68kTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
     Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4");
   }
+
+  // Floating point
+  if (TargetOpts.FeatureMap.lookup("isa-68881") ||
+      TargetOpts.FeatureMap.lookup("isa-68882"))
+    Builder.defineMacro("__HAVE_68881__");
 }
 
 ArrayRef<Builtin::Info> M68kTargetInfo::getTargetBuiltins() const {
diff --git a/clang/lib/Basic/Targets/M68k.h b/clang/lib/Basic/Targets/M68k.h
index dea9b59334919..1af00115a5059 100644
--- a/clang/lib/Basic/Targets/M68k.h
+++ b/clang/lib/Basic/Targets/M68k.h
@@ -36,6 +36,8 @@ class LLVM_LIBRARY_VISIBILITY M68kTargetInfo : public TargetInfo {
     CK_68060
   } CPU = CK_Unknown;
 
+  const TargetOptions &TargetOpts;
+
 public:
   M68kTargetInfo(const llvm::Triple &Triple, const TargetOptions &);
 
diff --git a/clang/lib/Basic/Targets/OSTargets.cpp b/clang/lib/Basic/Targets/OSTargets.cpp
index 763235fb10a52..627bc912fa231 100644
--- a/clang/lib/Basic/Targets/OSTargets.cpp
+++ b/clang/lib/Basic/Targets/OSTargets.cpp
@@ -214,7 +214,8 @@ static void addVisualCDefines(const LangOptions &Opts, MacroBuilder &Builder) {
       Builder.defineMacro("_HAS_CHAR16_T_LANGUAGE_SUPPORT", Twine(1));
 
     if (Opts.isCompatibleWithMSVC(LangOptions::MSVC2015)) {
-      if (Opts.CPlusPlus2b)
+      if (Opts.CPlusPlus23)
+        // TODO update to the proper value.
         Builder.defineMacro("_MSVC_LANG", "202004L");
       else if (Opts.CPlusPlus20)
         Builder.defineMacro("_MSVC_LANG", "202002L");
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 5f764f7875361..6720fcd567ac8 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -200,6 +200,11 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
     // Currently we support the v0.11 RISC-V V intrinsics.
     Builder.defineMacro("__riscv_v_intrinsic", Twine(getVersionValue(0, 11)));
   }
+
+  auto VScale = getVScaleRange(Opts);
+  if (VScale && VScale->first && VScale->first == VScale->second)
+    Builder.defineMacro("__riscv_v_fixed_vlen",
+                        Twine(VScale->first * llvm::RISCV::RVVBitsPerBlock));
 }
 
 static constexpr Builtin::Info BuiltinInfo[] = {
@@ -320,7 +325,7 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
 
 bool RISCVTargetInfo::isValidCPUName(StringRef Name) const {
   bool Is64Bit = getTriple().isArch64Bit();
-  return llvm::RISCV::checkCPUKind(llvm::RISCV::parseCPUKind(Name), Is64Bit);
+  return llvm::RISCV::parseCPU(Name, Is64Bit);
 }
 
 void RISCVTargetInfo::fillValidCPUList(
@@ -331,8 +336,7 @@ void RISCVTargetInfo::fillValidCPUList(
 
 bool RISCVTargetInfo::isValidTuneCPUName(StringRef Name) const {
   bool Is64Bit = getTriple().isArch64Bit();
-  return llvm::RISCV::checkTuneCPUKind(
-      llvm::RISCV::parseTuneCPUKind(Name, Is64Bit), Is64Bit);
+  return llvm::RISCV::parseTuneCPU(Name, Is64Bit);
 }
 
 void RISCVTargetInfo::fillValidTuneCPUList(
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index f8240e7f09483..e4e39506bccf5 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -41,6 +41,7 @@ class RISCVTargetInfo : public TargetInfo {
     HasRISCVVTypes = true;
     MCountName = "_mcount";
     HasFloat16 = true;
+    HasStrictFP = true;
   }
 
   bool setCPU(const std::string &Name) override {
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index a997614a65a86..b88b6b5d1951c 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -1213,7 +1213,7 @@ bool X86TargetInfo::validateCpuIs(StringRef FeatureStr) const {
       .Default(false);
 }
 
-static unsigned matchAsmCCConstraint(const char *&Name) {
+static unsigned matchAsmCCConstraint(const char *Name) {
   auto RV = llvm::StringSwitch<unsigned>(Name)
                 .Case("@cca", 4)
                 .Case("@ccae", 5)
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index ea38cc402575a..a0cf26006d16f 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -428,7 +428,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.EmitAddrsig = CodeGenOpts.Addrsig;
   Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection;
   Options.EmitCallSiteInfo = CodeGenOpts.EmitCallSiteInfo;
-  Options.EnableAIXExtendedAltivecABI = CodeGenOpts.EnableAIXExtendedAltivecABI;
+  Options.EnableAIXExtendedAltivecABI = LangOpts.EnableAIXExtendedAltivecABI;
   Options.XRayOmitFunctionIndex = CodeGenOpts.XRayOmitFunctionIndex;
   Options.LoopAlignment = CodeGenOpts.LoopAlignment;
   Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf;
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 803ad398c449e..5952ac4ff5ac4 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19760,6 +19760,11 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
     assert(Error == ASTContext::GE_None && "Unexpected error");
   }
 
+  if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
+    ICEArguments |= (1 << 1);
+  if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
+    ICEArguments |= (1 << 2);
+
   for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
     // If this is a normal argument, just emit it as a scalar.
     if ((ICEArguments & (1 << i)) == 0) {
@@ -19962,8 +19967,60 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
     IntrinsicTypes = {ResultType};
     break;
 
+  // Zihintntl
+  case RISCV::BI__builtin_riscv_ntl_load: {
+    llvm::Type *ResTy = ConvertType(E->getType());
+    ConstantInt *Mode = cast<ConstantInt>(Ops[1]);
+
+    llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
+        getLLVMContext(),
+        llvm::ConstantAsMetadata::get(Builder.getInt32(Mode->getZExtValue())));
+    llvm::MDNode *NontemporalNode = llvm::MDNode::get(
+        getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
+
+    int Width;
+    if(ResTy->isScalableTy()) {
+      const ScalableVectorType *SVTy = cast<ScalableVectorType>(ResTy);
+      llvm::Type *ScalarTy = ResTy->getScalarType();
+      Width = ScalarTy->getPrimitiveSizeInBits() *
+              SVTy->getElementCount().getKnownMinValue();
+    } else
+      Width = ResTy->getPrimitiveSizeInBits();
+    LoadInst *Load = Builder.CreateLoad(
+        Address(Ops[0], ResTy, CharUnits::fromQuantity(Width / 8)));
+
+    Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"),
+                      NontemporalNode);
+    Load->setMetadata(CGM.getModule().getMDKindID("riscv-nontemporal-domain"),
+                      RISCVDomainNode);
+
+    return Load;
+  }
+  case RISCV::BI__builtin_riscv_ntl_store: {
+    ConstantInt *Mode = cast<ConstantInt>(Ops[2]);
+
+    llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
+        getLLVMContext(),
+        llvm::ConstantAsMetadata::get(Builder.getInt32(Mode->getZExtValue())));
+    llvm::MDNode *NontemporalNode = llvm::MDNode::get(
+        getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
+
+    Value *BC = Builder.CreateBitCast(
+        Ops[0], llvm::PointerType::getUnqual(Ops[1]->getType()), "cast");
+
+    StoreInst *Store = Builder.CreateDefaultAlignedStore(Ops[1], BC);
+    Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"),
+                       NontemporalNode);
+    Store->setMetadata(CGM.getModule().getMDKindID("riscv-nontemporal-domain"),
+                       RISCVDomainNode);
+
+    return Store;
+  }
+
   // Vector builtins are handled from here.
 #include "clang/Basic/riscv_vector_builtin_cg.inc"
+  // SiFive Vector builtins are handled from here.
+#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
   }
 
   assert(ID != Intrinsic::not_intrinsic);
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 63296aeec5b9f..9cdf689b74440 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1829,10 +1829,32 @@ static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
          Module.getLangOpts().Sanitize.has(SanitizerKind::Return);
 }
 
-void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
-                                                 bool HasOptnone,
-                                                 bool AttrOnCallSite,
-                                               llvm::AttrBuilder &FuncAttrs) {
+/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
+/// requested denormal behavior, accounting for the overriding behavior of the
+/// -f32 case.
+static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
+                                 llvm::DenormalMode FP32DenormalMode,
+                                 llvm::AttrBuilder &FuncAttrs) {
+  if (FPDenormalMode != llvm::DenormalMode::getDefault())
+    FuncAttrs.addAttribute("denormal-fp-math", FPDenormalMode.str());
+
+  if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
+    FuncAttrs.addAttribute("denormal-fp-math-f32", FP32DenormalMode.str());
+}
+
+/// Add default attributes to a function, which have merge semantics under
+/// -mlink-builtin-bitcode and should not simply overwrite any existing
+/// attributes in the linked library.
+static void
+addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
+                                     llvm::AttrBuilder &FuncAttrs) {
+  addDenormalModeAttrs(CodeGenOpts.FPDenormalMode, CodeGenOpts.FP32DenormalMode,
+                       FuncAttrs);
+}
+
+void CodeGenModule::getTrivialDefaultFunctionAttributes(
+    StringRef Name, bool HasOptnone, bool AttrOnCallSite,
+    llvm::AttrBuilder &FuncAttrs) {
   // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
   if (!HasOptnone) {
     if (CodeGenOpts.OptimizeSize)
@@ -1874,15 +1896,6 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
     if (CodeGenOpts.NullPointerIsValid)
       FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
 
-    if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
-      FuncAttrs.addAttribute("denormal-fp-math",
-                             CodeGenOpts.FPDenormalMode.str());
-    if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
-      FuncAttrs.addAttribute(
-          "denormal-fp-math-f32",
-          CodeGenOpts.FP32DenormalMode.str());
-    }
-
     if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
       FuncAttrs.addAttribute("no-trapping-math", "true");
 
@@ -1984,6 +1997,19 @@ void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
   }
 }
 
+void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
+                                                 bool HasOptnone,
+                                                 bool AttrOnCallSite,
+                                                 llvm::AttrBuilder &FuncAttrs) {
+  getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
+                                      FuncAttrs);
+  if (!AttrOnCallSite) {
+    // If we're just getting the default, get the default values for mergeable
+    // attributes.
+    addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
+  }
+}
+
 void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
   llvm::AttrBuilder FuncAttrs(F.getContext());
   getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
@@ -1992,8 +2018,60 @@ void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
   F.addFnAttrs(FuncAttrs);
 }
 
+/// Apply default attributes to \p F, accounting for merge semantics of
+/// attributes that should not overwrite existing attributes.
+void CodeGenModule::mergeDefaultFunctionDefinitionAttributes(
+    llvm::Function &F, bool WillInternalize) {
+  llvm::AttrBuilder FuncAttrs(F.getContext());
+  getTrivialDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
+                                      /*AttrOnCallSite=*/false, FuncAttrs);
+  GetCPUAndFeaturesAttributes(GlobalDecl(), FuncAttrs);
+
+  if (!WillInternalize && F.isInterposable()) {
+    // Do not promote "dynamic" denormal-fp-math to this translation unit's
+    // setting for weak functions that won't be internalized. The user has no
+    // real control for how builtin bitcode is linked, so we shouldn't assume
+    // later copies will use a consistent mode.
+    F.addFnAttrs(FuncAttrs);
+    return;
+  }
+
+  llvm::AttributeMask AttrsToRemove;
+
+  llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
+  llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
+  llvm::DenormalMode Merged =
+      CodeGenOpts.FPDenormalMode.mergeCalleeMode(DenormModeToMerge);
+  llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
+
+  if (DenormModeToMergeF32.isValid()) {
+    MergedF32 =
+        CodeGenOpts.FP32DenormalMode.mergeCalleeMode(DenormModeToMergeF32);
+  }
+
+  if (Merged == llvm::DenormalMode::getDefault()) {
+    AttrsToRemove.addAttribute("denormal-fp-math");
+  } else if (Merged != DenormModeToMerge) {
+    // Overwrite existing attribute
+    FuncAttrs.addAttribute("denormal-fp-math",
+                           CodeGenOpts.FPDenormalMode.str());
+  }
+
+  if (MergedF32 == llvm::DenormalMode::getDefault()) {
+    AttrsToRemove.addAttribute("denormal-fp-math-f32");
+  } else if (MergedF32 != DenormModeToMergeF32) {
+    // Overwrite existing attribute
+    FuncAttrs.addAttribute("denormal-fp-math-f32",
+                           CodeGenOpts.FP32DenormalMode.str());
+  }
+
+  F.removeFnAttrs(AttrsToRemove);
+  addDenormalModeAttrs(Merged, MergedF32, FuncAttrs);
+  F.addFnAttrs(FuncAttrs);
+}
+
 void CodeGenModule::addDefaultFunctionDefinitionAttributes(
-                                                   llvm::AttrBuilder &attrs) {
+    llvm::AttrBuilder &attrs) {
   getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
                                /*for call*/ false, attrs);
   GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 5100d6c65baa4..2cf7b025744ac 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -73,8 +73,6 @@ CGDebugInfo::CGDebugInfo(CodeGenModule &CGM)
     : CGM(CGM), DebugKind(CGM.getCodeGenOpts().getDebugInfo()),
       DebugTypeExtRefs(CGM.getCodeGenOpts().DebugTypeExtRefs),
       DBuilder(CGM.getModule()) {
-  for (const auto &KV : CGM.getCodeGenOpts().DebugPrefixMap)
-    DebugPrefixMap[KV.first] = KV.second;
   CreateCompileUnit();
 }
 
@@ -470,12 +468,9 @@ llvm::DIFile *CGDebugInfo::createFile(
 }
 
 std::string CGDebugInfo::remapDIPath(StringRef Path) const {
-  if (DebugPrefixMap.empty())
-    return Path.str();
-
   SmallString<256> P = Path;
-  for (const auto &Entry : DebugPrefixMap)
-    if (llvm::sys::path::replace_path_prefix(P, Entry.first, Entry.second))
+  for (auto &[From, To] : llvm::reverse(CGM.getCodeGenOpts().DebugPrefixMap))
+    if (llvm::sys::path::replace_path_prefix(P, From, To))
       break;
   return P.str().str();
 }
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 5a3839d7f3946..5c42d3620bdbd 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -86,9 +86,6 @@ class CGDebugInfo {
   /// Cache of previously constructed Types.
   llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache;
 
-  std::map<llvm::StringRef, llvm::StringRef, std::greater<llvm::StringRef>>
-      DebugPrefixMap;
-
   /// Cache that maps VLA types to size expressions for that type,
   /// represented by instantiated Metadata nodes.
   llvm::SmallDenseMap<QualType, llvm::Metadata *> SizeExprCache;
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 329e1a27d98fc..5d77e513da5eb 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -1730,7 +1730,7 @@ llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM,
   }
 
   // Zero-extend bool.
-  if (C->getType()->isIntegerTy(1)) {
+  if (C->getType()->isIntegerTy(1) && !destType->isBitIntType()) {
     llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(destType);
     return llvm::ConstantExpr::getZExt(C, boolTy);
   }
diff --git a/clang/lib/CodeGen/CGGPUBuiltin.cpp b/clang/lib/CodeGen/CGGPUBuiltin.cpp
index c39e0cc75f2d3..8451ebebfc0b7 100644
--- a/clang/lib/CodeGen/CGGPUBuiltin.cpp
+++ b/clang/lib/CodeGen/CGGPUBuiltin.cpp
@@ -125,7 +125,7 @@ packArgsIntoNVPTXFormatBuffer(CodeGenFunction *CGF, const CallArgList &Args) {
   }
 }
 
-bool containsNonScalarVarargs(CodeGenFunction *CGF, CallArgList Args) {
+bool containsNonScalarVarargs(CodeGenFunction *CGF, const CallArgList &Args) {
   return llvm::any_of(llvm::drop_begin(Args), [&](const CallArg &A) {
     return !A.getRValue(*CGF).isScalar();
   });
@@ -189,7 +189,7 @@ RValue CodeGenFunction::EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E) {
                /* ParamsToSkip = */ 0);
 
   SmallVector<llvm::Value *, 8> Args;
-  for (auto A : CallArgs) {
+  for (const auto &A : CallArgs) {
     // We don't know how to emit non-scalar varargs.
     if (!A.getRValue(*this).isScalar()) {
       CGM.ErrorUnsupported(E, "non-scalar arg to printf");
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index a10e51b8cb441..98378e1386414 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -323,7 +323,7 @@ static const CGFunctionInfo &getFunctionInfo(CodeGenModule &CGM,
 template <size_t N, size_t... Ints>
 static std::array<Address, N> getParamAddrs(std::index_sequence<Ints...> IntSeq,
                                             std::array<CharUnits, N> Alignments,
-                                            FunctionArgList Args,
+                                            const FunctionArgList &Args,
                                             CodeGenFunction *CGF) {
   return std::array<Address, N>{
       {Address(CGF->Builder.CreateLoad(CGF->GetAddrOfLocalVar(Args[Ints])),
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 1400afbb3a9e0..ce92aa21e0fc5 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -6831,67 +6831,30 @@ class MappableExprsHandler {
     const Expr *getMapExpr() const { return MapExpr; }
   };
 
-  /// Class that associates information with a base pointer to be passed to the
-  /// runtime library.
-  class BasePointerInfo {
-    /// The base pointer.
-    llvm::Value *Ptr = nullptr;
-    /// The base declaration that refers to this device pointer, or null if
-    /// there is none.
-    const ValueDecl *DevPtrDecl = nullptr;
-
-  public:
-    BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr)
-        : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
-    llvm::Value *operator*() const { return Ptr; }
-    const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; }
-    void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; }
-  };
-
+  using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
+  using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
+  using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
+  using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
+  using MapNonContiguousArrayTy =
+      llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
   using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
-  using MapBaseValuesArrayTy = SmallVector<BasePointerInfo, 4>;
-  using MapValuesArrayTy = SmallVector<llvm::Value *, 4>;
-  using MapFlagsArrayTy = SmallVector<OpenMPOffloadMappingFlags, 4>;
-  using MapMappersArrayTy = SmallVector<const ValueDecl *, 4>;
-  using MapDimArrayTy = SmallVector<uint64_t, 4>;
-  using MapNonContiguousArrayTy = SmallVector<MapValuesArrayTy, 4>;
+  using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
 
   /// This structure contains combined information generated for mappable
   /// clauses, including base pointers, pointers, sizes, map types, user-defined
   /// mappers, and non-contiguous information.
-  struct MapCombinedInfoTy {
-    struct StructNonContiguousInfo {
-      bool IsNonContiguous = false;
-      MapDimArrayTy Dims;
-      MapNonContiguousArrayTy Offsets;
-      MapNonContiguousArrayTy Counts;
-      MapNonContiguousArrayTy Strides;
-    };
+  struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
     MapExprsArrayTy Exprs;
-    MapBaseValuesArrayTy BasePointers;
-    MapValuesArrayTy Pointers;
-    MapValuesArrayTy Sizes;
-    MapFlagsArrayTy Types;
-    MapMappersArrayTy Mappers;
-    StructNonContiguousInfo NonContigInfo;
+    MapValueDeclsArrayTy Mappers;
+    MapValueDeclsArrayTy DevicePtrDecls;
 
     /// Append arrays in \a CurInfo.
     void append(MapCombinedInfoTy &CurInfo) {
       Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());
-      BasePointers.append(CurInfo.BasePointers.begin(),
-                          CurInfo.BasePointers.end());
-      Pointers.append(CurInfo.Pointers.begin(), CurInfo.Pointers.end());
-      Sizes.append(CurInfo.Sizes.begin(), CurInfo.Sizes.end());
-      Types.append(CurInfo.Types.begin(), CurInfo.Types.end());
+      DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),
+                            CurInfo.DevicePtrDecls.end());
       Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());
-      NonContigInfo.Dims.append(CurInfo.NonContigInfo.Dims.begin(),
-                                 CurInfo.NonContigInfo.Dims.end());
-      NonContigInfo.Offsets.append(CurInfo.NonContigInfo.Offsets.begin(),
-                                    CurInfo.NonContigInfo.Offsets.end());
-      NonContigInfo.Counts.append(CurInfo.NonContigInfo.Counts.begin(),
-                                   CurInfo.NonContigInfo.Counts.end());
-      NonContigInfo.Strides.append(CurInfo.NonContigInfo.Strides.begin(),
-                                    CurInfo.NonContigInfo.Strides.end());
+      llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
     }
   };
 
@@ -7638,6 +7601,7 @@ class MappableExprsHandler {
             assert(Size && "Failed to determine structure size");
             CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
             CombinedInfo.BasePointers.push_back(BP.getPointer());
+            CombinedInfo.DevicePtrDecls.push_back(nullptr);
             CombinedInfo.Pointers.push_back(LB.getPointer());
             CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
@@ -7649,6 +7613,7 @@ class MappableExprsHandler {
           }
           CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
           CombinedInfo.BasePointers.push_back(BP.getPointer());
+          CombinedInfo.DevicePtrDecls.push_back(nullptr);
           CombinedInfo.Pointers.push_back(LB.getPointer());
           Size = CGF.Builder.CreatePtrDiff(
               CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(),
@@ -7666,6 +7631,7 @@ class MappableExprsHandler {
             (Next == CE && MapType != OMPC_MAP_unknown)) {
           CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
           CombinedInfo.BasePointers.push_back(BP.getPointer());
+          CombinedInfo.DevicePtrDecls.push_back(nullptr);
           CombinedInfo.Pointers.push_back(LB.getPointer());
           CombinedInfo.Sizes.push_back(
               CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
@@ -8168,7 +8134,8 @@ class MappableExprsHandler {
         [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
                                      CodeGenFunction &CGF) {
           UseDeviceDataCombinedInfo.Exprs.push_back(VD);
-          UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr, VD);
+          UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);
+          UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);
           UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);
           UseDeviceDataCombinedInfo.Sizes.push_back(
               llvm::Constant::getNullValue(CGF.Int64Ty));
@@ -8337,8 +8304,7 @@ class MappableExprsHandler {
             assert(RelevantVD &&
                    "No relevant declaration related with device pointer??");
 
-            CurInfo.BasePointers[CurrentBasePointersIdx].setDevicePtrDecl(
-                RelevantVD);
+            CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;
             CurInfo.Types[CurrentBasePointersIdx] |=
                 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
           }
@@ -8377,7 +8343,8 @@ class MappableExprsHandler {
                 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
           }
           CurInfo.Exprs.push_back(L.VD);
-          CurInfo.BasePointers.emplace_back(BasePtr, L.VD);
+          CurInfo.BasePointers.emplace_back(BasePtr);
+          CurInfo.DevicePtrDecls.emplace_back(L.VD);
           CurInfo.Pointers.push_back(Ptr);
           CurInfo.Sizes.push_back(
               llvm::Constant::getNullValue(this->CGF.Int64Ty));
@@ -8472,6 +8439,7 @@ class MappableExprsHandler {
     CombinedInfo.Exprs.push_back(VD);
     // Base is the base of the struct
     CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer());
+    CombinedInfo.DevicePtrDecls.push_back(nullptr);
     // Pointer is the address of the lowest element
     llvm::Value *LB = LBAddr.getPointer();
     const CXXMethodDecl *MD =
@@ -8593,6 +8561,7 @@ class MappableExprsHandler {
                                  VDLVal.getPointer(CGF));
       CombinedInfo.Exprs.push_back(VD);
       CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF));
+      CombinedInfo.DevicePtrDecls.push_back(nullptr);
       CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF));
       CombinedInfo.Sizes.push_back(
           CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy),
@@ -8619,6 +8588,7 @@ class MappableExprsHandler {
                                    VDLVal.getPointer(CGF));
         CombinedInfo.Exprs.push_back(VD);
         CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
+        CombinedInfo.DevicePtrDecls.push_back(nullptr);
         CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF));
         CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
             CGF.getTypeSize(
@@ -8630,6 +8600,7 @@ class MappableExprsHandler {
                                    VDLVal.getPointer(CGF));
         CombinedInfo.Exprs.push_back(VD);
         CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF));
+        CombinedInfo.DevicePtrDecls.push_back(nullptr);
         CombinedInfo.Pointers.push_back(VarRVal.getScalarVal());
         CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0));
       }
@@ -8654,7 +8625,7 @@ class MappableExprsHandler {
                        OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
                        OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
         continue;
-      llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]);
+      llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);
       assert(BasePtr && "Unable to find base lambda address.");
       int TgtIdx = -1;
       for (unsigned J = I; J > 0; --J) {
@@ -8696,7 +8667,8 @@ class MappableExprsHandler {
     // pass its value.
     if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {
       CombinedInfo.Exprs.push_back(VD);
-      CombinedInfo.BasePointers.emplace_back(Arg, VD);
+      CombinedInfo.BasePointers.emplace_back(Arg);
+      CombinedInfo.DevicePtrDecls.emplace_back(VD);
       CombinedInfo.Pointers.push_back(Arg);
       CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
           CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty,
@@ -8938,6 +8910,7 @@ class MappableExprsHandler {
     if (CI.capturesThis()) {
       CombinedInfo.Exprs.push_back(nullptr);
       CombinedInfo.BasePointers.push_back(CV);
+      CombinedInfo.DevicePtrDecls.push_back(nullptr);
       CombinedInfo.Pointers.push_back(CV);
       const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr());
       CombinedInfo.Sizes.push_back(
@@ -8950,6 +8923,7 @@ class MappableExprsHandler {
       const VarDecl *VD = CI.getCapturedVar();
       CombinedInfo.Exprs.push_back(VD->getCanonicalDecl());
       CombinedInfo.BasePointers.push_back(CV);
+      CombinedInfo.DevicePtrDecls.push_back(nullptr);
       CombinedInfo.Pointers.push_back(CV);
       if (!RI.getType()->isAnyPointerType()) {
         // We have to signal to the runtime captures passed by value that are
@@ -8981,6 +8955,7 @@ class MappableExprsHandler {
       auto I = FirstPrivateDecls.find(VD);
       CombinedInfo.Exprs.push_back(VD->getCanonicalDecl());
       CombinedInfo.BasePointers.push_back(CV);
+      CombinedInfo.DevicePtrDecls.push_back(nullptr);
       if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) {
         Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
             CV, ElementType, CGF.getContext().getDeclAlign(VD),
@@ -9266,7 +9241,7 @@ static void emitOffloadingArrays(
     }
 
     for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
-      llvm::Value *BPVal = *CombinedInfo.BasePointers[I];
+      llvm::Value *BPVal = CombinedInfo.BasePointers[I];
       llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32(
           llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs),
           Info.RTArgs.BasePointersArray, 0, I);
@@ -9277,8 +9252,7 @@ static void emitOffloadingArrays(
       CGF.Builder.CreateStore(BPVal, BPAddr);
 
       if (Info.requiresDevicePointerInfo())
-        if (const ValueDecl *DevVD =
-                CombinedInfo.BasePointers[I].getDevicePtrDecl())
+        if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I])
           Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
 
       llvm::Value *PVal = CombinedInfo.Pointers[I];
@@ -9592,7 +9566,7 @@ void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
   // Fill up the runtime mapper handle for all components.
   for (unsigned I = 0; I < Info.BasePointers.size(); ++I) {
     llvm::Value *CurBaseArg = MapperCGF.Builder.CreateBitCast(
-        *Info.BasePointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
+        Info.BasePointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
     llvm::Value *CurBeginArg = MapperCGF.Builder.CreateBitCast(
         Info.Pointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy));
     llvm::Value *CurSizeArg = Info.Sizes[I];
@@ -10028,6 +10002,7 @@ void CGOpenMPRuntime::emitTargetCall(
       if (CI->capturesVariableArrayType()) {
         CurInfo.Exprs.push_back(nullptr);
         CurInfo.BasePointers.push_back(*CV);
+        CurInfo.DevicePtrDecls.push_back(nullptr);
         CurInfo.Pointers.push_back(*CV);
         CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
             CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true));
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index a257de859d4e8..32259d1e4cbff 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -1212,7 +1212,8 @@ void CodeGenModule::EmitDeferredVTables() {
 }
 
 bool CodeGenModule::AlwaysHasLTOVisibilityPublic(const CXXRecordDecl *RD) {
-  if (RD->hasAttr<LTOVisibilityPublicAttr>() || RD->hasAttr<UuidAttr>())
+  if (RD->hasAttr<LTOVisibilityPublicAttr>() || RD->hasAttr<UuidAttr>() ||
+      RD->hasAttr<DLLExportAttr>() || RD->hasAttr<DLLImportAttr>())
     return true;
 
   if (!getCodeGenOpts().LTOVisibilityPublicStd)
@@ -1239,13 +1240,9 @@ bool CodeGenModule::HasHiddenLTOVisibility(const CXXRecordDecl *RD) {
   if (!isExternallyVisible(LV.getLinkage()))
     return true;
 
-  if (getTriple().isOSBinFormatCOFF()) {
-    if (RD->hasAttr<DLLExportAttr>() || RD->hasAttr<DLLImportAttr>())
-      return false;
-  } else {
-    if (LV.getVisibility() != HiddenVisibility)
-      return false;
-  }
+  if (!getTriple().isOSBinFormatCOFF() &&
+      LV.getVisibility() != HiddenVisibility)
+    return false;
 
   return !AlwaysHasLTOVisibilityPublic(RD);
 }
diff --git a/clang/lib/CodeGen/CMakeLists.txt b/clang/lib/CodeGen/CMakeLists.txt
index 7c7403f239c82..ca71ad9e68e27 100644
--- a/clang/lib/CodeGen/CMakeLists.txt
+++ b/clang/lib/CodeGen/CMakeLists.txt
@@ -3,6 +3,7 @@ set(LLVM_LINK_COMPONENTS
   Analysis
   BitReader
   BitWriter
+  CodeGenTypes
   Core
   Coroutines
   Coverage
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 17e781181ee26..29adf88acd704 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -270,7 +270,8 @@ namespace clang {
             // in LLVM IR.
             if (F.isIntrinsic())
               continue;
-            Gen->CGM().addDefaultFunctionDefinitionAttributes(F);
+            Gen->CGM().mergeDefaultFunctionDefinitionAttributes(F,
+                                                                LM.Internalize);
           }
 
         CurLinkModule = LM.Module.get();
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index b8c45bc226ee1..367f802253e01 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -990,7 +990,7 @@ void CodeGenModule::EmitOpenCLMetadata() {
 }
 
 void CodeGenModule::EmitBackendOptionsMetadata(
-    const CodeGenOptions CodeGenOpts) {
+    const CodeGenOptions &CodeGenOpts) {
   if (getTriple().isRISCV()) {
     getModule().addModuleFlag(llvm::Module::Min, "SmallDataLimit",
                               CodeGenOpts.SmallDataLimit);
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 64c2902572376..e4ef209195275 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1272,6 +1272,8 @@ class CodeGenModule : public CodeGenTypeCache {
   /// function which relies on particular fast-math attributes for correctness.
   /// It's up to you to ensure that this is safe.
   void addDefaultFunctionDefinitionAttributes(llvm::Function &F);
+  void mergeDefaultFunctionDefinitionAttributes(llvm::Function &F,
+                                                bool WillInternalize);
 
   /// Like the overload taking a `Function &`, but intended specifically
   /// for frontends that want to build on Clang's target-configuration logic.
@@ -1711,7 +1713,7 @@ class CodeGenModule : public CodeGenTypeCache {
 
   /// Emit the module flag metadata used to pass options controlling the
   /// the backend to LLVM.
-  void EmitBackendOptionsMetadata(const CodeGenOptions CodeGenOpts);
+  void EmitBackendOptionsMetadata(const CodeGenOptions &CodeGenOpts);
 
   /// Emits OpenCL specific Metadata e.g. OpenCL version.
   void EmitOpenCLMetadata();
@@ -1734,6 +1736,12 @@ class CodeGenModule : public CodeGenTypeCache {
   /// function.
   void SimplifyPersonality();
 
+  /// Helper function for getDefaultFunctionAttributes. Builds a set of function
+  /// attributes which can be simply added to a function.
+  void getTrivialDefaultFunctionAttributes(StringRef Name, bool HasOptnone,
+                                           bool AttrOnCallSite,
+                                           llvm::AttrBuilder &FuncAttrs);
+
   /// Helper function for ConstructAttributeList and
   /// addDefaultFunctionDefinitionAttributes.  Builds a set of function
   /// attributes to add to a function with the given properties.
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index 15a3d74666ca2..b80317529b72b 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -1036,7 +1036,7 @@ void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
   llvm::Expected<llvm::InstrProfRecord> RecordExpected =
       PGOReader->getInstrProfRecord(FuncName, FunctionHash);
   if (auto E = RecordExpected.takeError()) {
-    auto IPE = llvm::InstrProfError::take(std::move(E));
+    auto IPE = std::get<0>(llvm::InstrProfError::take(std::move(E)));
     if (IPE == llvm::instrprof_error::unknown_function)
       CGM.getPGOStats().addMissing(IsInMainFile);
     else if (IPE == llvm::instrprof_error::hash_mismatch)
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index bac01c7ff67f2..bb4c6f5e0cdee 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -1634,9 +1634,7 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName,
 
 CoverageMappingModuleGen::CoverageMappingModuleGen(
     CodeGenModule &CGM, CoverageSourceInfo &SourceInfo)
-    : CGM(CGM), SourceInfo(SourceInfo) {
-  CoveragePrefixMap = CGM.getCodeGenOpts().CoveragePrefixMap;
-}
+    : CGM(CGM), SourceInfo(SourceInfo) {}
 
 std::string CoverageMappingModuleGen::getCurrentDirname() {
   if (!CGM.getCodeGenOpts().CoverageCompilationDir.empty())
@@ -1650,8 +1648,13 @@ std::string CoverageMappingModuleGen::getCurrentDirname() {
 std::string CoverageMappingModuleGen::normalizeFilename(StringRef Filename) {
   llvm::SmallString<256> Path(Filename);
   llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
-  for (const auto &Entry : CoveragePrefixMap) {
-    if (llvm::sys::path::replace_path_prefix(Path, Entry.first, Entry.second))
+
+  /// Traverse coverage prefix map in reverse order because prefix replacements
+  /// are applied in reverse order starting from the last one when multiple
+  /// prefix replacement options are provided.
+  for (const auto &[From, To] :
+       llvm::reverse(CGM.getCodeGenOpts().CoveragePrefixMap)) {
+    if (llvm::sys::path::replace_path_prefix(Path, From, To))
       break;
   }
   return Path.str().str();
diff --git a/clang/lib/CodeGen/CoverageMappingGen.h b/clang/lib/CodeGen/CoverageMappingGen.h
index f5282601b6406..eca68d9abd79a 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.h
+++ b/clang/lib/CodeGen/CoverageMappingGen.h
@@ -107,7 +107,6 @@ class CoverageMappingModuleGen {
   llvm::SmallDenseMap<const FileEntry *, unsigned, 8> FileEntries;
   std::vector<llvm::Constant *> FunctionNames;
   std::vector<FunctionInfo> FunctionRecords;
-  std::map<std::string, std::string> CoveragePrefixMap;
 
   std::string getCurrentDirname();
   std::string normalizeFilename(StringRef Filename);
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index c5218129b0fb7..e5a2b35c0b396 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -3757,7 +3757,7 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() {
   Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr);
   detectAmbiguousBases(Classes);
   int Flags = 0;
-  for (auto Class : Classes) {
+  for (const MSRTTIClass &Class : Classes) {
     if (Class.RD->getNumBases() > 1)
       Flags |= HasBranchingHierarchy;
     // Note: cl.exe does not calculate "HasAmbiguousBases" correctly.  We
diff --git a/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp b/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
index 07db4a63e2c10..114a9c1e2eac1 100644
--- a/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
+++ b/clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp
@@ -349,6 +349,11 @@ ObjectFilePCHContainerWriter::CreatePCHContainerGenerator(
       CI, MainFileName, OutputFileName, std::move(OS), Buffer);
 }
 
+ArrayRef<StringRef> ObjectFilePCHContainerReader::getFormats() const {
+  static StringRef Formats[] = {"obj", "raw"};
+  return Formats;
+}
+
 StringRef
 ObjectFilePCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const {
   StringRef PCH;
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index cc332aeee40d4..ff4d516090ea4 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -33,6 +33,7 @@
 #include "llvm/IR/Type.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include <algorithm>
 
@@ -11105,6 +11106,8 @@ class RISCVABIInfo : public DefaultABIInfo {
                                                CharUnits Field1Off,
                                                llvm::Type *Field2Ty,
                                                CharUnits Field2Off) const;
+
+  ABIArgInfo coerceVLSVector(QualType Ty) const;
 };
 } // end anonymous namespace
 
@@ -11348,6 +11351,25 @@ ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
   return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
 }
 
+// Fixed-length RVV vectors are represented as scalable vectors in function
+// args/return and must be coerced from fixed vectors.
+ABIArgInfo RISCVABIInfo::coerceVLSVector(QualType Ty) const {
+  assert(Ty->isVectorType() && "expected vector type!");
+
+  const auto *VT = Ty->castAs<VectorType>();
+  assert(VT->getVectorKind() == VectorType::RVVFixedLengthDataVector &&
+         "Unexpected vector kind");
+
+  assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
+
+  const auto *BT = VT->getElementType()->castAs<BuiltinType>();
+  unsigned EltSize = getContext().getTypeSize(BT);
+  llvm::ScalableVectorType *ResType =
+        llvm::ScalableVectorType::get(CGT.ConvertType(VT->getElementType()),
+                                      llvm::RISCV::RVVBitsPerBlock / EltSize);
+  return ABIArgInfo::getDirect(ResType);
+}
+
 ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
                                               int &ArgGPRsLeft,
                                               int &ArgFPRsLeft) const {
@@ -11443,6 +11465,10 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
     return ABIArgInfo::getDirect();
   }
 
+  if (const VectorType *VT = Ty->getAs<VectorType>())
+    if (VT->getVectorKind() == VectorType::RVVFixedLengthDataVector)
+      return coerceVLSVector(Ty);
+
   // Aggregates which are <= 2*XLen will be passed in registers if possible,
   // so coerce to integers.
   if (Size <= 2 * XLen) {
@@ -11524,7 +11550,6 @@ class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
 
     const char *Kind;
     switch (Attr->getInterrupt()) {
-    case RISCVInterruptAttr::user: Kind = "user"; break;
     case RISCVInterruptAttr::supervisor: Kind = "supervisor"; break;
     case RISCVInterruptAttr::machine: Kind = "machine"; break;
     }
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 3d83c86100bec..a6bd2d41e7975 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -95,7 +95,6 @@ add_clang_library(clangDriver
 
   DEPENDS
   ClangDriverOptions
-  RISCVTargetParserTableGen
 
   LINK_LIBS
   clangBasic
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 2e924960eda37..a2deb9b0dd9ad 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -12,6 +12,7 @@
 #include "ToolChains/AMDGPUOpenMP.h"
 #include "ToolChains/AVR.h"
 #include "ToolChains/Ananas.h"
+#include "ToolChains/Arch/RISCV.h"
 #include "ToolChains/BareMetal.h"
 #include "ToolChains/CSKYToolChain.h"
 #include "ToolChains/Clang.h"
@@ -689,8 +690,9 @@ static llvm::Triple computeTargetTriple(const Driver &D,
   // If target is RISC-V adjust the target triple according to
   // provided architecture name
   if (Target.isRISCV()) {
-    if ((A = Args.getLastArg(options::OPT_march_EQ))) {
-      StringRef ArchName = A->getValue();
+    if (Args.hasArg(options::OPT_march_EQ) ||
+        Args.hasArg(options::OPT_mcpu_EQ)) {
+      StringRef ArchName = tools::riscv::getRISCVArch(Args, Target);
       if (ArchName.startswith_insensitive("rv32"))
         Target.setArch(llvm::Triple::riscv32);
       else if (ArchName.startswith_insensitive("rv64"))
@@ -1393,8 +1395,9 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   const Arg *Std = Args.getLastArg(options::OPT_std_EQ);
   ModulesModeCXX20 =
       !Args.hasArg(options::OPT_fmodules) && Std &&
-      (Std->containsValue("c++20") || Std->containsValue("c++2b") ||
-       Std->containsValue("c++2a") || Std->containsValue("c++latest"));
+      (Std->containsValue("c++20") || Std->containsValue("c++2a") ||
+       Std->containsValue("c++23") || Std->containsValue("c++2b") ||
+       Std->containsValue("c++latest"));
 
   // Process -fmodule-header{=} flags.
   if (Arg *A = Args.getLastArg(options::OPT_fmodule_header_EQ,
@@ -1809,9 +1812,6 @@ void Driver::generateCompilationDiagnostics(
     }
   }
 
-  for (const auto &A : C.getArgs().filtered(options::OPT_frewrite_map_file_EQ))
-    Diag(clang::diag::note_drv_command_failed_diag_msg) << A->getValue();
-
   Diag(clang::diag::note_drv_command_failed_diag_msg)
       << "\n\n********************";
 }
diff --git a/clang/lib/Driver/ToolChains/Arch/M68k.cpp b/clang/lib/Driver/ToolChains/Arch/M68k.cpp
index 628c252e83864..963f7a187d636 100644
--- a/clang/lib/Driver/ToolChains/Arch/M68k.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/M68k.cpp
@@ -65,13 +65,35 @@ std::string m68k::getM68kTargetCPU(const ArgList &Args) {
   return "";
 }
 
+static void addFloatABIFeatures(const llvm::opt::ArgList &Args,
+                                std::vector<llvm::StringRef> &Features) {
+  Arg *A = Args.getLastArg(options::OPT_msoft_float, options::OPT_mhard_float,
+                           options::OPT_m68881);
+  // Opt out FPU even for newer CPUs.
+  if (A && A->getOption().matches(options::OPT_msoft_float)) {
+    Features.push_back("-isa-68881");
+    Features.push_back("-isa-68882");
+    return;
+  }
+
+  std::string CPU = m68k::getM68kTargetCPU(Args);
+  // Only enable M68881 for CPU < 68020 if the related flags are present.
+  if ((A && (CPU == "M68000" || CPU == "M68010")) ||
+      // Otherwise, by default we assume newer CPUs have M68881/2.
+      CPU == "M68020")
+    Features.push_back("+isa-68881");
+  else if (CPU == "M68030" || CPU == "M68040" || CPU == "M68060")
+    // Note that although CPU >= M68040 imply M68882, we still add `isa-68882`
+    // anyway so that it's easier to add or not add the corresponding macro
+    // definitions later, in case we want to disable 68881/2 in newer CPUs
+    // (with -msoft-float, for instance).
+    Features.push_back("+isa-68882");
+}
+
 void m68k::getM68kTargetFeatures(const Driver &D, const llvm::Triple &Triple,
                                  const ArgList &Args,
                                  std::vector<StringRef> &Features) {
-
-  m68k::FloatABI FloatABI = m68k::getM68kFloatABI(D, Args);
-  if (FloatABI == m68k::FloatABI::Soft)
-    Features.push_back("-hard-float");
+  addFloatABIFeatures(Args, Features);
 
   // Handle '-ffixed-<register>' flags
   if (Args.hasArg(options::OPT_ffixed_a0))
@@ -105,21 +127,3 @@ void m68k::getM68kTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   if (Args.hasArg(options::OPT_ffixed_d7))
     Features.push_back("+reserve-d7");
 }
-
-m68k::FloatABI m68k::getM68kFloatABI(const Driver &D, const ArgList &Args) {
-  m68k::FloatABI ABI = m68k::FloatABI::Invalid;
-  if (Arg *A =
-          Args.getLastArg(options::OPT_msoft_float, options::OPT_mhard_float)) {
-
-    if (A->getOption().matches(options::OPT_msoft_float))
-      ABI = m68k::FloatABI::Soft;
-    else if (A->getOption().matches(options::OPT_mhard_float))
-      ABI = m68k::FloatABI::Hard;
-  }
-
-  // If unspecified, choose the default based on the platform.
-  if (ABI == m68k::FloatABI::Invalid)
-    ABI = m68k::FloatABI::Hard;
-
-  return ABI;
-}
diff --git a/clang/lib/Driver/ToolChains/Arch/M68k.h b/clang/lib/Driver/ToolChains/Arch/M68k.h
index 41d53efb940ba..051e7e1af1032 100644
--- a/clang/lib/Driver/ToolChains/Arch/M68k.h
+++ b/clang/lib/Driver/ToolChains/Arch/M68k.h
@@ -20,14 +20,6 @@ namespace driver {
 namespace tools {
 namespace m68k {
 
-enum class FloatABI {
-  Invalid,
-  Soft,
-  Hard,
-};
-
-FloatABI getM68kFloatABI(const Driver &D, const llvm::opt::ArgList &Args);
-
 std::string getM68kTargetCPU(const llvm::opt::ArgList &Args);
 
 void getM68kTargetFeatures(const Driver &D, const llvm::Triple &Triple,
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 7cf01ad8a72b8..a26c9caf64c70 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -54,11 +54,11 @@ static void getRISCFeaturesFromMcpu(const Driver &D, const Arg *A,
                                     StringRef Mcpu,
                                     std::vector<StringRef> &Features) {
   bool Is64Bit = Triple.isRISCV64();
-  llvm::RISCV::CPUKind CPUKind = llvm::RISCV::parseCPUKind(Mcpu);
-  if (!llvm::RISCV::checkCPUKind(CPUKind, Is64Bit)) {
+  if (!llvm::RISCV::parseCPU(Mcpu, Is64Bit)) {
     // Try inverting Is64Bit in case the CPU is valid, but for the wrong target.
-    if (llvm::RISCV::checkCPUKind(CPUKind, !Is64Bit))
-      D.Diag(clang::diag::err_drv_invalid_riscv_cpu_name_for_target) << Mcpu << Is64Bit;
+    if (llvm::RISCV::parseCPU(Mcpu, !Is64Bit))
+      D.Diag(clang::diag::err_drv_invalid_riscv_cpu_name_for_target)
+          << Mcpu << Is64Bit;
     else
       D.Diag(clang::diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Mcpu;
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 38f26d0176471..3232394007bc3 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -342,6 +342,12 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   if (TC.getTriple().isRISCV())
     CmdArgs.push_back("-X");
 
+  // The R_ARM_TARGET2 relocation must be treated as R_ARM_REL32 on arm*-*-elf
+  // and arm*-*-eabi (the default is R_ARM_GOT_PREL, used on arm*-*-linux and
+  // arm*-*-*bsd).
+  if (isARMBareMetal(TC.getTriple()))
+    CmdArgs.push_back("--target2=rel");
+
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index edea4acfc3abd..32ba56066af58 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -740,6 +740,12 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
     PGOGenerateArg = nullptr;
   }
 
+  if (TC.getTriple().isOSAIX()) {
+    if (Arg *ProfileSampleUseArg = getLastProfileSampleUseArg(Args))
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << ProfileSampleUseArg->getSpelling() << TC.getTriple().str();
+  }
+
   if (ProfileGenerateArg) {
     if (ProfileGenerateArg->getOption().matches(
             options::OPT_fprofile_instr_generate_EQ))
@@ -1133,6 +1139,9 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
     if (ArgM->getOption().matches(options::OPT_M) ||
         ArgM->getOption().matches(options::OPT_MD))
       CmdArgs.push_back("-sys-header-deps");
+    if (Args.hasFlag(options::OPT_canonical_prefixes,
+                     options::OPT_no_canonical_prefixes, true))
+      CmdArgs.push_back("-canonical-system-headers");
     if ((isa<PrecompileJobAction>(JA) &&
          !Args.hasArg(options::OPT_fno_module_file_deps)) ||
         Args.hasArg(options::OPT_fmodule_file_deps))
@@ -1976,6 +1985,7 @@ void Clang::AddMIPSTargetArgs(const ArgList &Args,
 
 void Clang::AddPPCTargetArgs(const ArgList &Args,
                              ArgStringList &CmdArgs) const {
+  const Driver &D = getToolChain().getDriver();
   const llvm::Triple &T = getToolChain().getTriple();
   if (Args.getLastArg(options::OPT_mtune_EQ)) {
     CmdArgs.push_back("-tune-cpu");
@@ -2003,13 +2013,22 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   }
 
   bool IEEELongDouble = getToolChain().defaultToIEEELongDouble();
+  bool VecExtabi = false;
   for (const Arg *A : Args.filtered(options::OPT_mabi_EQ)) {
     StringRef V = A->getValue();
-    if (V == "ieeelongdouble")
+    if (V == "ieeelongdouble") {
       IEEELongDouble = true;
-    else if (V == "ibmlongdouble")
+      A->claim();
+    } else if (V == "ibmlongdouble") {
       IEEELongDouble = false;
-    else if (V != "altivec")
+      A->claim();
+    } else if (V == "vec-default") {
+      VecExtabi = false;
+      A->claim();
+    } else if (V == "vec-extabi") {
+      VecExtabi = true;
+      A->claim();
+    } else if (V != "altivec")
       // The ppc64 linux abis are all "altivec" abis by default. Accept and ignore
       // the option if given as we don't have backend support for any targets
       // that don't use the altivec abi.
@@ -2017,10 +2036,14 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   }
   if (IEEELongDouble)
     CmdArgs.push_back("-mabi=ieeelongdouble");
+  if (VecExtabi) {
+    if (!T.isOSAIX())
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << "-mabi=vec-extabi" << T.str();
+    CmdArgs.push_back("-mabi=vec-extabi");
+  }
 
-  ppc::FloatABI FloatABI =
-      ppc::getPPCFloatABI(getToolChain().getDriver(), Args);
-
+  ppc::FloatABI FloatABI = ppc::getPPCFloatABI(D, Args);
   if (FloatABI == ppc::FloatABI::Soft) {
     // Floating point operations and argument passing are soft.
     CmdArgs.push_back("-msoft-float");
@@ -3650,7 +3673,8 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
   bool HaveStdCXXModules =
       IsCXX && Std &&
       (Std->containsValue("c++2a") || Std->containsValue("c++20") ||
-       Std->containsValue("c++2b") || Std->containsValue("c++latest"));
+       Std->containsValue("c++2b") || Std->containsValue("c++23") ||
+       Std->containsValue("c++latest"));
   bool HaveModules = HaveStdCXXModules;
 
   // -fmodules enables the use of precompiled modules (off by default).
@@ -4013,6 +4037,13 @@ static void RenderDiagnosticsOptions(const Driver &D, const ArgList &Args,
     CmdArgs.push_back(Args.MakeArgString(Opt));
   }
 
+  if (const Arg *A =
+          Args.getLastArg(options::OPT_fdiagnostics_misexpect_tolerance_EQ)) {
+    std::string Opt =
+        std::string("-fdiagnostics-misexpect-tolerance=") + A->getValue();
+    CmdArgs.push_back(Args.MakeArgString(Opt));
+  }
+
   if (const Arg *A = Args.getLastArg(options::OPT_fdiagnostics_format_EQ)) {
     CmdArgs.push_back("-fdiagnostics-format");
     CmdArgs.push_back(A->getValue());
@@ -5193,27 +5224,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // LLVM Code Generator Options.
 
-  for (const Arg *A : Args.filtered(options::OPT_frewrite_map_file_EQ)) {
-    StringRef Map = A->getValue();
-    if (!llvm::sys::fs::exists(Map)) {
-      D.Diag(diag::err_drv_no_such_file) << Map;
-    } else {
-      A->render(Args, CmdArgs);
-      A->claim();
-    }
-  }
-
-  if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ_vec_extabi,
-                               options::OPT_mabi_EQ_vec_default)) {
-    if (!Triple.isOSAIX())
-      D.Diag(diag::err_drv_unsupported_opt_for_target)
-          << A->getSpelling() << RawTriple.str();
-    if (A->getOption().getID() == options::OPT_mabi_EQ_vec_extabi)
-      CmdArgs.push_back("-mabi=vec-extabi");
-    else
-      CmdArgs.push_back("-mabi=vec-default");
-  }
-
   if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ_quadword_atomics)) {
     if (!Triple.isOSAIX() || Triple.isPPC32())
       D.Diag(diag::err_drv_unsupported_opt_for_target)
@@ -5230,14 +5240,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_Wframe_larger_than_EQ)) {
-    StringRef v = A->getValue();
-    // FIXME: Validate the argument here so we don't produce meaningless errors
-    // about -fwarn-stack-size=.
-    if (v.empty())
-      D.Diag(diag::err_drv_missing_argument) << A->getSpelling() << 1;
+    StringRef V = A->getValue(), V1 = V;
+    unsigned Size;
+    if (V1.consumeInteger(10, Size) || !V1.empty())
+      D.Diag(diag::err_drv_invalid_argument_to_option)
+          << V << A->getOption().getName();
     else
-      CmdArgs.push_back(Args.MakeArgString("-fwarn-stack-size=" + v));
-    A->claim();
+      CmdArgs.push_back(Args.MakeArgString("-fwarn-stack-size=" + V));
   }
 
   Args.addOptOutFlag(CmdArgs, options::OPT_fjump_tables,
@@ -5700,6 +5709,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                     options::OPT_fno_unique_internal_linkage_names);
   Args.addOptInFlag(CmdArgs, options::OPT_funique_basic_block_section_names,
                     options::OPT_fno_unique_basic_block_section_names);
+  Args.addOptInFlag(CmdArgs, options::OPT_fconvergent_functions,
+                    options::OPT_fno_convergent_functions);
 
   if (Arg *A = Args.getLastArg(options::OPT_fsplit_machine_functions,
                                options::OPT_fno_split_machine_functions)) {
@@ -6627,7 +6638,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                              .Case("c++14", "-std=c++14")
                              .Case("c++17", "-std=c++17")
                              .Case("c++20", "-std=c++20")
-                             .Case("c++latest", "-std=c++2b")
+                             // TODO add c++23 when MSVC supports it.
+                             .Case("c++latest", "-std=c++23")
                              .Default("");
       if (LanguageStandard.empty())
         D.Diag(clang::diag::warn_drv_unused_argument)
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 315cf6325d396..cf21d6243b180 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -683,9 +683,16 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
       CmdArgs.push_back(
           Args.MakeArgString(Twine(PluginOptPrefix) + "-strict-dwarf=true"));
 
-    if (Args.getLastArg(options::OPT_mabi_EQ_vec_extabi))
-      CmdArgs.push_back(
-          Args.MakeArgString(Twine(PluginOptPrefix) + "-vec-extabi"));
+    for (const Arg *A : Args.filtered_reverse(options::OPT_mabi_EQ)) {
+      StringRef V = A->getValue();
+      if (V == "vec-default")
+        break;
+      if (V == "vec-extabi") {
+        CmdArgs.push_back(
+            Args.MakeArgString(Twine(PluginOptPrefix) + "-vec-extabi"));
+        break;
+      }
+    }
   }
 
   bool UseSeparateSections =
diff --git a/clang/lib/Driver/ToolChains/CrossWindows.cpp b/clang/lib/Driver/ToolChains/CrossWindows.cpp
index bc91449326a5d..3c5dfba329cf8 100644
--- a/clang/lib/Driver/ToolChains/CrossWindows.cpp
+++ b/clang/lib/Driver/ToolChains/CrossWindows.cpp
@@ -94,7 +94,8 @@ void tools::CrossWindows::Linker::ConstructJob(
   CmdArgs.push_back("-m");
   switch (TC.getArch()) {
   default:
-    llvm_unreachable("unsupported architecture");
+    D.Diag(diag::err_target_unknown_triple) << TC.getEffectiveTriple().str();
+    break;
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
     // FIXME: this is incorrect for WinCE
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index 8ff7807b54b73..c0fcdad861994 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -695,8 +695,9 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,
 /// toolchain.
 NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                                const llvm::Triple &HostTriple,
-                               const ArgList &Args)
-    : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args) {
+                               const ArgList &Args, bool Freestanding = false)
+    : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args),
+      Freestanding(Freestanding) {
   if (CudaInstallation.isValid()) {
     CudaInstallation.WarnIfUnsupportedVersion();
     getProgramPaths().push_back(std::string(CudaInstallation.getBinPath()));
@@ -711,7 +712,8 @@ NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
 NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                                const ArgList &Args)
     : NVPTXToolChain(D, Triple,
-                     llvm::Triple(llvm::sys::getDefaultTargetTriple()), Args) {}
+                     llvm::Triple(llvm::sys::getDefaultTargetTriple()), Args,
+                     /*Freestanding=*/true) {}
 
 llvm::opt::DerivedArgList *
 NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
@@ -735,6 +737,16 @@ NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,
   return DAL;
 }
 
+void NVPTXToolChain::addClangTargetOptions(
+    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
+    Action::OffloadKind DeviceOffloadingKind) const {
+  // If we are compiling with a standalone NVPTX toolchain we want to try to
+  // mimic a standard environment as much as possible. So we enable lowering
+  // ctor / dtor functions to global symbols that can be registered.
+  if (Freestanding)
+    CC1Args.append({"-mllvm", "--nvptx-lower-global-ctor-dtor"});
+}
+
 bool NVPTXToolChain::supportsDebugInfoOption(const llvm::opt::Arg *A) const {
   const Option &O = A->getOption();
   return (O.matches(options::OPT_gN_Group) &&
diff --git a/clang/lib/Driver/ToolChains/Cuda.h b/clang/lib/Driver/ToolChains/Cuda.h
index 9cafbddbf7ca0..39df6e06fb26d 100644
--- a/clang/lib/Driver/ToolChains/Cuda.h
+++ b/clang/lib/Driver/ToolChains/Cuda.h
@@ -132,8 +132,8 @@ namespace toolchains {
 class LLVM_LIBRARY_VISIBILITY NVPTXToolChain : public ToolChain {
 public:
   NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
-                 const llvm::Triple &HostTriple,
-                 const llvm::opt::ArgList &Args);
+                 const llvm::Triple &HostTriple, const llvm::opt::ArgList &Args,
+                 bool Freestanding);
 
   NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,
                  const llvm::opt::ArgList &Args);
@@ -142,6 +142,11 @@ class LLVM_LIBRARY_VISIBILITY NVPTXToolChain : public ToolChain {
   TranslateArgs(const llvm::opt::DerivedArgList &Args, StringRef BoundArch,
                 Action::OffloadKind DeviceOffloadKind) const override;
 
+  void
+  addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
+                        llvm::opt::ArgStringList &CC1Args,
+                        Action::OffloadKind DeviceOffloadKind) const override;
+
   // Never try to use the integrated assembler with CUDA; always fork out to
   // ptxas.
   bool useIntegratedAs() const override { return false; }
@@ -168,6 +173,9 @@ class LLVM_LIBRARY_VISIBILITY NVPTXToolChain : public ToolChain {
 protected:
   Tool *buildAssembler() const override; // ptxas.
   Tool *buildLinker() const override;    // nvlink.
+
+private:
+  bool Freestanding = false;
 };
 
 class LLVM_LIBRARY_VISIBILITY CudaToolChain : public NVPTXToolChain {
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 11e65358bec05..35b30e6044411 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -393,6 +393,13 @@ void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
     }
   }
 
+  if (Args.hasArg(options::OPT_mkernel) ||
+      Args.hasArg(options::OPT_fapple_kext) ||
+      Args.hasArg(options::OPT_ffreestanding)) {
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back("-disable-atexit-based-global-dtor-lowering");
+  }
+
   Args.AddLastArg(CmdArgs, options::OPT_prebind);
   Args.AddLastArg(CmdArgs, options::OPT_noprebind);
   Args.AddLastArg(CmdArgs, options::OPT_nofixprebinding);
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 300072db35edc..ffd30af08fd0d 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -194,15 +194,29 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
 
   // Skips the primary input file, which is the input file that the compilation
   // proccess will be executed upon (e.g. the host bitcode file) and
-  // adds the other secondary input (e.g. device bitcode files for embedding)
-  // to the embed offload object. This is condensed logic from the Clang driver
-  // for embedding offload objects during HostOffloading.
-  if (IsHostOffloadingAction) {
-    for (size_t i = 1; i < Inputs.size(); ++i) {
-      if (Inputs[i].getType() != types::TY_Nothing)
-        CmdArgs.push_back(
-            Args.MakeArgString("-fembed-offload-object=" +
-                               getToolChain().getInputFilename(Inputs[i])));
+  // adds other secondary input (e.g. device bitcode files for embedding to the
+  // -fembed-offload-object argument or the host IR file for proccessing
+  // during device compilation to the fopenmp-host-ir-file-path argument via
+  // OpenMPDeviceInput). This is condensed logic from the ConstructJob
+  // function inside of the Clang driver for pushing on further input arguments
+  // needed for offloading during various phases of compilation.
+  for (size_t i = 1; i < Inputs.size(); ++i) {
+    if (Inputs[i].getType() == types::TY_Nothing) {
+      // contains nothing, so it's skippable
+    } else if (IsHostOffloadingAction) {
+      CmdArgs.push_back(
+          Args.MakeArgString("-fembed-offload-object=" +
+                             getToolChain().getInputFilename(Inputs[i])));
+    } else if (IsOpenMPDevice) {
+      if (Inputs[i].getFilename()) {
+        CmdArgs.push_back("-fopenmp-host-ir-file-path");
+        CmdArgs.push_back(Args.MakeArgString(Inputs[i].getFilename()));
+      } else {
+        llvm_unreachable("missing openmp host-ir file for device offloading");
+      }
+    } else {
+      llvm_unreachable(
+          "unexpectedly given multiple inputs or given unknown input");
     }
   }
 
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 8ce5a16bfe0d8..5b1922970ce2b 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -332,13 +332,6 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
 
   Generic_GCC::AddMultiarchPaths(D, SysRoot, OSLibDir, Paths);
 
-  // The deprecated -DLLVM_ENABLE_PROJECTS=libcxx configuration installs
-  // libc++.so in D.Dir+"/../lib/". Detect this path.
-  // TODO Remove once LLVM_ENABLE_PROJECTS=libcxx is unsupported.
-  if (StringRef(D.Dir).startswith(SysRoot) &&
-      D.getVFS().exists(D.Dir + "/../lib/libc++.so"))
-    addPathIfExists(D, D.Dir + "/../lib", Paths);
-
   addPathIfExists(D, concat(SysRoot, "/lib"), Paths);
   addPathIfExists(D, concat(SysRoot, "/usr/lib"), Paths);
 }
@@ -802,7 +795,7 @@ SanitizerMask Linux::getSupportedSanitizers() const {
   if (IsX86_64 || IsMIPS64 || IsAArch64 || IsPowerPC64 || IsSystemZ ||
       IsLoongArch64)
     Res |= SanitizerKind::Thread;
-  if (IsX86_64)
+  if (IsX86_64 || IsSystemZ)
     Res |= SanitizerKind::KernelMemory;
   if (IsX86 || IsX86_64)
     Res |= SanitizerKind::Function;
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index bac486bab8857..947abb9b02b51 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -135,7 +135,7 @@ void tools::MinGW::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("arm64pe");
     break;
   default:
-    llvm_unreachable("Unsupported target architecture.");
+    D.Diag(diag::err_target_unknown_triple) << TC.getEffectiveTriple().str();
   }
 
   Arg *SubsysArg =
diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index da75a701b405e..1e52f221c7982 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -457,7 +457,7 @@ DeclarationFragmentsBuilder::getFragmentsForFunction(const FunctionDecl *Func) {
   Fragments.append(")", DeclarationFragments::FragmentKind::Text);
 
   // FIXME: Handle exception specifiers: throw, noexcept
-  return Fragments;
+  return Fragments.append(";", DeclarationFragments::FragmentKind::Text);
 }
 
 DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForEnumConstant(
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index f5748cfc50890..fbd11a4cb48c7 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1665,10 +1665,14 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
     if (Current.opensBlockOrBlockTypeList(Style)) {
       NewIndent = Style.IndentWidth +
                   std::min(State.Column, CurrentState.NestedBlockIndent);
+    } else if (Current.is(tok::l_brace)) {
+      NewIndent =
+          CurrentState.LastSpace + Style.BracedInitializerIndentWidth.value_or(
+                                       Style.ContinuationIndentWidth);
     } else {
       NewIndent = CurrentState.LastSpace + Style.ContinuationIndentWidth;
     }
-    const FormatToken *NextNoComment = Current.getNextNonComment();
+    const FormatToken *NextNonComment = Current.getNextNonComment();
     bool EndsInComma = Current.MatchingParen &&
                        Current.MatchingParen->Previous &&
                        Current.MatchingParen->Previous->is(tok::comma);
@@ -1676,9 +1680,9 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
                       Style.Language == FormatStyle::LK_Proto ||
                       Style.Language == FormatStyle::LK_TextProto ||
                       !Style.BinPackArguments ||
-                      (NextNoComment &&
-                       NextNoComment->isOneOf(TT_DesignatedInitializerPeriod,
-                                              TT_DesignatedInitializerLSquare));
+                      (NextNonComment && NextNonComment->isOneOf(
+                                             TT_DesignatedInitializerPeriod,
+                                             TT_DesignatedInitializerLSquare));
     BreakBeforeParameter = EndsInComma;
     if (Current.ParameterCount > 1)
       NestedBlockIndent = std::max(NestedBlockIndent, State.Column + 1);
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 087ac8f627331..f7c82c581bfc9 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -879,6 +879,8 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("BinPackArguments", Style.BinPackArguments);
     IO.mapOptional("BinPackParameters", Style.BinPackParameters);
     IO.mapOptional("BitFieldColonSpacing", Style.BitFieldColonSpacing);
+    IO.mapOptional("BracedInitializerIndentWidth",
+                   Style.BracedInitializerIndentWidth);
     IO.mapOptional("BraceWrapping", Style.BraceWrapping);
     IO.mapOptional("BreakAfterAttributes", Style.BreakAfterAttributes);
     IO.mapOptional("BreakAfterJavaFieldAnnotations",
@@ -1337,6 +1339,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.BitFieldColonSpacing = FormatStyle::BFCS_Both;
   LLVMStyle.BinPackArguments = true;
   LLVMStyle.BinPackParameters = true;
+  LLVMStyle.BracedInitializerIndentWidth = std::nullopt;
   LLVMStyle.BraceWrapping = {/*AfterCaseLabel=*/false,
                              /*AfterClass=*/false,
                              /*AfterControlStatement=*/FormatStyle::BWACS_Never,
@@ -2806,7 +2809,7 @@ struct JavaImportDirective {
 // Determines whether 'Ranges' intersects with ('Start', 'End').
 static bool affectsRange(ArrayRef<tooling::Range> Ranges, unsigned Start,
                          unsigned End) {
-  for (auto Range : Ranges) {
+  for (const auto &Range : Ranges) {
     if (Range.getOffset() < End &&
         Range.getOffset() + Range.getLength() > Start) {
       return true;
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 834ebf6f8b204..4672fac9d02ec 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -37,6 +37,8 @@ namespace format {
   TYPE(BitFieldColon)                                                          \
   TYPE(BlockComment)                                                           \
   TYPE(BracedListLBrace)                                                       \
+  /* The colon at the end of a case label. */                                  \
+  TYPE(CaseLabelColon)                                                         \
   TYPE(CastRParen)                                                             \
   TYPE(ClassLBrace)                                                            \
   TYPE(CompoundRequirementLBrace)                                              \
@@ -73,8 +75,7 @@ namespace format {
   TYPE(FunctionTypeLParen)                                                     \
   /* The colons as part of a C11 _Generic selection */                         \
   TYPE(GenericSelectionColon)                                                  \
-  /* The colon at the end of a goto label or a case label. Currently only used \
-   * for Verilog. */                                                           \
+  /* The colon at the end of a goto label. */                                  \
   TYPE(GotoLabelColon)                                                         \
   TYPE(IfMacro)                                                                \
   TYPE(ImplicitStringLiteral)                                                  \
@@ -1803,7 +1804,7 @@ struct AdditionalKeywords {
   bool isVerilogEndOfLabel(const FormatToken &Tok) const {
     const FormatToken *Next = Tok.getNextNonComment();
     // In Verilog the colon in a default label is optional.
-    return Tok.is(TT_GotoLabelColon) ||
+    return Tok.is(TT_CaseLabelColon) ||
            (Tok.is(tok::kw_default) &&
             !(Next && Next->isOneOf(tok::colon, tok::semi, kw_clocking, kw_iff,
                                     kw_input, kw_output, kw_sequence)));
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index 5142a83efdd45..ff54fb75b3dd9 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -587,7 +587,7 @@ LeftRightQualifierAlignmentFixer::analyze(
   assert(QualifierToken != tok::identifier && "Unrecognised Qualifier");
 
   for (AnnotatedLine *Line : AnnotatedLines) {
-    if (Line->InPPDirective)
+    if (!Line->Affected || Line->InPPDirective)
       continue;
     FormatToken *First = Line->First;
     assert(First);
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 06593355d4d5a..be734858b84ee 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -970,6 +970,10 @@ class AnnotatingParser {
     case tok::colon:
       if (!Tok->Previous)
         return false;
+      // Goto labels and case labels are already identified in
+      // UnwrappedLineParser.
+      if (Tok->isTypeFinalized())
+        break;
       // Colons from ?: are handled in parseConditional().
       if (Style.isJavaScript()) {
         if (Contexts.back().ColonIsForRangeExpr || // colon in for loop
@@ -1008,7 +1012,7 @@ class AnnotatingParser {
           // In Verilog a case label doesn't have the case keyword. We
           // assume a colon following an expression is a case label.
           // Colons from ?: are annotated in parseConditional().
-          Tok->setType(TT_GotoLabelColon);
+          Tok->setType(TT_CaseLabelColon);
           if (Line.Level > 1 || (!Line.InPPDirective && Line.Level > 0))
             --Line.Level;
         }
@@ -2161,6 +2165,10 @@ class AnnotatingParser {
   /// This is a heuristic based on whether \p Tok is an identifier following
   /// something that is likely a type.
   bool isStartOfName(const FormatToken &Tok) {
+    // Handled in ExpressionParser for Verilog.
+    if (Style.isVerilog())
+      return false;
+
     if (Tok.isNot(tok::identifier) || !Tok.Previous)
       return false;
 
@@ -4541,13 +4549,12 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
            Style.BitFieldColonSpacing == FormatStyle::BFCS_After;
   }
   if (Right.is(tok::colon)) {
-    if (Right.is(TT_GotoLabelColon) ||
-        (!Style.isVerilog() &&
-         Line.First->isOneOf(tok::kw_default, tok::kw_case))) {
+    if (Right.is(TT_CaseLabelColon))
       return Style.SpaceBeforeCaseColon;
-    }
-    const FormatToken *Next = Right.getNextNonComment();
-    if (!Next || Next->is(tok::semi))
+    if (Right.is(TT_GotoLabelColon))
+      return false;
+    // `private:` and `public:`.
+    if (!Right.getNextNonComment())
       return false;
     if (Right.is(TT_ObjCMethodExpr))
       return false;
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 5c6816d6df30f..2e5bdf96deb88 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -1493,6 +1493,7 @@ void UnwrappedLineParser::parseStructuralElement(
     }
     nextToken();
     if (FormatTok->is(tok::colon)) {
+      FormatTok->setFinalizedType(TT_CaseLabelColon);
       parseLabel();
       return;
     }
@@ -1925,6 +1926,7 @@ void UnwrappedLineParser::parseStructuralElement(
         if (!Style.isVerilog() && FormatTok->is(tok::colon) &&
             !Line->MustBeDeclaration) {
           Line->Tokens.begin()->Tok->MustBreakBefore = true;
+          FormatTok->setFinalizedType(TT_GotoLabelColon);
           parseLabel(!Style.IndentGotoLabels);
           if (HasLabel)
             *HasLabel = true;
@@ -3113,7 +3115,11 @@ void UnwrappedLineParser::parseCaseLabel() {
   // FIXME: fix handling of complex expressions here.
   do {
     nextToken();
-  } while (!eof() && !FormatTok->is(tok::colon));
+    if (FormatTok->is(tok::colon)) {
+      FormatTok->setFinalizedType(TT_CaseLabelColon);
+      break;
+    }
+  } while (!eof());
   parseLabel();
 }
 
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 3269c9d0f479f..bda49fb68e460 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -811,7 +811,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
                                      UserFilesAreVolatile);
   AST->ModuleCache = new InMemoryModuleCache;
   AST->HSOpts = std::make_shared<HeaderSearchOptions>();
-  AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormat());
+  AST->HSOpts->ModuleFormat = std::string(PCHContainerRdr.getFormats().front());
   AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
                                          AST->getSourceManager(),
                                          AST->getDiagnostics(),
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index cb29049d5a3dd..cbd2663a56278 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1696,14 +1696,12 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
 
   for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ)) {
     auto Split = StringRef(Arg).split('=');
-    Opts.DebugPrefixMap.insert(
-        {std::string(Split.first), std::string(Split.second)});
+    Opts.DebugPrefixMap.emplace_back(Split.first, Split.second);
   }
 
   for (const auto &Arg : Args.getAllArgValues(OPT_fcoverage_prefix_map_EQ)) {
     auto Split = StringRef(Arg).split('=');
-    Opts.CoveragePrefixMap.insert(
-        {std::string(Split.first), std::string(Split.second)});
+    Opts.CoveragePrefixMap.emplace_back(Split.first, Split.second);
   }
 
   const llvm::Triple::ArchType DebugEntryValueArchs[] = {
@@ -1935,16 +1933,6 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
     }
   }
 
-  if (Arg *A =
-          Args.getLastArg(OPT_mabi_EQ_vec_default, OPT_mabi_EQ_vec_extabi)) {
-    if (!T.isOSAIX())
-      Diags.Report(diag::err_drv_unsupported_opt_for_target)
-          << A->getSpelling() << T.str();
-
-    const Option &O = A->getOption();
-    Opts.EnableAIXExtendedAltivecABI = O.matches(OPT_mabi_EQ_vec_extabi);
-  }
-
   if (Arg *A = Args.getLastArg(OPT_mabi_EQ_quadword_atomics)) {
     if (!T.isOSAIX() || T.isPPC32())
       Diags.Report(diag::err_drv_unsupported_opt_for_target)
@@ -3731,9 +3719,9 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.Blocks = Args.hasArg(OPT_fblocks) || (Opts.OpenCL
     && Opts.OpenCLVersion == 200);
 
-  Opts.ConvergentFunctions = Opts.OpenCL || (Opts.CUDA && Opts.CUDAIsDevice) ||
-                             Opts.SYCLIsDevice ||
-                             Args.hasArg(OPT_fconvergent_functions);
+  Opts.ConvergentFunctions = Args.hasArg(OPT_fconvergent_functions) ||
+                             Opts.OpenCL || (Opts.CUDA && Opts.CUDAIsDevice) ||
+                             Opts.SYCLIsDevice;
 
   Opts.NoBuiltin = Args.hasArg(OPT_fno_builtin) || Opts.Freestanding;
   if (!Opts.NoBuiltin)
diff --git a/clang/lib/Frontend/DependencyFile.cpp b/clang/lib/Frontend/DependencyFile.cpp
index fe4218b6e672d..44268e71dc241 100644
--- a/clang/lib/Frontend/DependencyFile.cpp
+++ b/clang/lib/Frontend/DependencyFile.cpp
@@ -49,6 +49,7 @@ struct DepCollectorPPCallbacks : public PPCallbacks {
       DepCollector.maybeAddDependency(
           llvm::sys::path::remove_leading_dotslash(*Filename),
           /*FromModule*/ false, isSystem(FileType), /*IsModuleFile*/ false,
+          &PP.getFileManager(),
           /*IsMissing*/ false);
   }
 
@@ -56,9 +57,11 @@ struct DepCollectorPPCallbacks : public PPCallbacks {
                    SrcMgr::CharacteristicKind FileType) override {
     StringRef Filename =
         llvm::sys::path::remove_leading_dotslash(SkippedFile.getName());
-    DepCollector.maybeAddDependency(Filename, /*FromModule=*/false,
+    DepCollector.maybeAddDependency(Filename,
+                                    /*FromModule=*/false,
                                     /*IsSystem=*/isSystem(FileType),
                                     /*IsModuleFile=*/false,
+                                    &PP.getFileManager(),
                                     /*IsMissing=*/false);
   }
 
@@ -69,9 +72,12 @@ struct DepCollectorPPCallbacks : public PPCallbacks {
                           StringRef RelativePath, const Module *Imported,
                           SrcMgr::CharacteristicKind FileType) override {
     if (!File)
-      DepCollector.maybeAddDependency(FileName, /*FromModule*/false,
-                                     /*IsSystem*/false, /*IsModuleFile*/false,
-                                     /*IsMissing*/true);
+      DepCollector.maybeAddDependency(FileName,
+                                      /*FromModule*/ false,
+                                      /*IsSystem*/ false,
+                                      /*IsModuleFile*/ false,
+                                      &PP.getFileManager(),
+                                      /*IsMissing*/ true);
     // Files that actually exist are handled by FileChanged.
   }
 
@@ -82,9 +88,11 @@ struct DepCollectorPPCallbacks : public PPCallbacks {
       return;
     StringRef Filename =
         llvm::sys::path::remove_leading_dotslash(File->getName());
-    DepCollector.maybeAddDependency(Filename, /*FromModule=*/false,
+    DepCollector.maybeAddDependency(Filename,
+                                    /*FromModule=*/false,
                                     /*IsSystem=*/isSystem(FileType),
                                     /*IsModuleFile=*/false,
+                                    &PP.getFileManager(),
                                     /*IsMissing=*/false);
   }
 
@@ -100,10 +108,12 @@ struct DepCollectorMMCallbacks : public ModuleMapCallbacks {
   void moduleMapFileRead(SourceLocation Loc, const FileEntry &Entry,
                          bool IsSystem) override {
     StringRef Filename = Entry.getName();
-    DepCollector.maybeAddDependency(Filename, /*FromModule*/false,
-                                    /*IsSystem*/IsSystem,
-                                    /*IsModuleFile*/false,
-                                    /*IsMissing*/false);
+    DepCollector.maybeAddDependency(Filename,
+                                    /*FromModule*/ false,
+                                    /*IsSystem*/ IsSystem,
+                                    /*IsModuleFile*/ false,
+                                    /*FileMgr*/ nullptr,
+                                    /*IsMissing*/ false);
   }
 };
 
@@ -118,9 +128,11 @@ struct DepCollectorASTListener : public ASTReaderListener {
   }
   void visitModuleFile(StringRef Filename,
                        serialization::ModuleKind Kind) override {
-    DepCollector.maybeAddDependency(Filename, /*FromModule*/true,
-                                   /*IsSystem*/false, /*IsModuleFile*/true,
-                                   /*IsMissing*/false);
+    DepCollector.maybeAddDependency(Filename,
+                                    /*FromModule*/ true,
+                                    /*IsSystem*/ false, /*IsModuleFile*/ true,
+                                    /*FileMgr*/ nullptr,
+                                    /*IsMissing*/ false);
   }
   bool visitInputFile(StringRef Filename, bool IsSystem,
                       bool IsOverridden, bool IsExplicitModule) override {
@@ -132,8 +144,9 @@ struct DepCollectorASTListener : public ASTReaderListener {
     if (auto FE = FileMgr.getOptionalFileRef(Filename))
       Filename = FE->getName();
 
-    DepCollector.maybeAddDependency(Filename, /*FromModule*/true, IsSystem,
-                                   /*IsModuleFile*/false, /*IsMissing*/false);
+    DepCollector.maybeAddDependency(Filename, /*FromModule*/ true, IsSystem,
+                                    /*IsModuleFile*/ false, /*FileMgr*/ nullptr,
+                                    /*IsMissing*/ false);
     return true;
   }
 };
@@ -142,9 +155,15 @@ struct DepCollectorASTListener : public ASTReaderListener {
 void DependencyCollector::maybeAddDependency(StringRef Filename,
                                              bool FromModule, bool IsSystem,
                                              bool IsModuleFile,
+                                             FileManager *FileMgr,
                                              bool IsMissing) {
-  if (sawDependency(Filename, FromModule, IsSystem, IsModuleFile, IsMissing))
+  if (sawDependency(Filename, FromModule, IsSystem, IsModuleFile, IsMissing)) {
+    if (IsSystem && FileMgr && shouldCanonicalizeSystemDependencies()) {
+      if (auto F = FileMgr->getFile(Filename))
+        Filename = FileMgr->getCanonicalName(*F);
+    }
     addDependency(Filename);
+  }
 }
 
 bool DependencyCollector::addDependency(StringRef Filename) {
@@ -192,6 +211,7 @@ DependencyFileGenerator::DependencyFileGenerator(
     const DependencyOutputOptions &Opts)
     : OutputFile(Opts.OutputFile), Targets(Opts.Targets),
       IncludeSystemHeaders(Opts.IncludeSystemHeaders),
+      CanonicalSystemHeaders(Opts.CanonicalSystemHeaders),
       PhonyTarget(Opts.UsePhonyTargets),
       AddMissingHeaderDeps(Opts.AddMissingHeaderDeps), SeenMissingHeader(false),
       IncludeModuleFiles(Opts.IncludeModuleFiles),
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 0b39dd1c84f7d..0c4e73ed931c9 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -452,7 +452,7 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI,
   } else {
     //   -- __cplusplus
     // FIXME: Use correct value for C++23.
-    if (LangOpts.CPlusPlus2b)
+    if (LangOpts.CPlusPlus23)
       Builder.defineMacro("__cplusplus", "202101L");
     //      [C++20] The integer literal 202002L.
     else if (LangOpts.CPlusPlus20)
@@ -606,7 +606,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
     Builder.defineMacro("__cpp_unicode_literals", "200710L");
     Builder.defineMacro("__cpp_user_defined_literals", "200809L");
     Builder.defineMacro("__cpp_lambdas", "200907L");
-    Builder.defineMacro("__cpp_constexpr", LangOpts.CPlusPlus2b   ? "202211L"
+    Builder.defineMacro("__cpp_constexpr", LangOpts.CPlusPlus23   ? "202211L"
                                            : LangOpts.CPlusPlus20 ? "201907L"
                                            : LangOpts.CPlusPlus17 ? "201603L"
                                            : LangOpts.CPlusPlus14 ? "201304L"
@@ -690,15 +690,15 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
     //Builder.defineMacro("__cpp_modules", "201907L");
     Builder.defineMacro("__cpp_using_enum", "201907L");
   }
-  // C++2b features.
-  if (LangOpts.CPlusPlus2b) {
+  // C++23 features.
+  if (LangOpts.CPlusPlus23) {
     Builder.defineMacro("__cpp_implicit_move", "202011L");
     Builder.defineMacro("__cpp_size_t_suffix", "202011L");
     Builder.defineMacro("__cpp_if_consteval", "202106L");
     Builder.defineMacro("__cpp_multidimensional_subscript", "202211L");
   }
 
-  // We provide those C++2b features as extensions in earlier language modes, so
+  // We provide those C++23 features as extensions in earlier language modes, so
   // we also define their feature test macros.
   if (LangOpts.CPlusPlus11)
     Builder.defineMacro("__cpp_static_call_operator", "202207L");
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 52f0e23a265cb..33168d09f0f39 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -98,6 +98,14 @@ set(ppc_htm_files
   htmxlintrin.h
   )
 
+set(riscv_files
+  riscv_ntlh.h
+  )
+
+set(sifive_files
+  sifive_vector.h
+  )
+
 set(systemz_files
   s390intrin.h
   vecintrin.h
@@ -244,6 +252,8 @@ set(files
   ${opencl_files}
   ${ppc_files}
   ${ppc_htm_files}
+  ${riscv_files}
+  ${sifive_files}
   ${systemz_files}
   ${ve_files}
   ${x86_files}
@@ -257,6 +267,7 @@ set(cuda_wrapper_files
   cuda_wrappers/cmath
   cuda_wrappers/complex
   cuda_wrappers/new
+  cuda_wrappers/bits/shared_ptr_base.h
 )
 
 set(ppc_wrapper_files
@@ -425,7 +436,7 @@ add_header_target("loongarch-resource-headers" "${loongarch_files}")
 add_header_target("mips-resource-headers" "${mips_msa_files}")
 add_header_target("ppc-resource-headers" "${ppc_files};${ppc_wrapper_files}")
 add_header_target("ppc-htm-resource-headers" "${ppc_htm_files}")
-add_header_target("riscv-resource-headers" "${riscv_generated_files}")
+add_header_target("riscv-resource-headers" "${riscv_files};${riscv_generated_files}")
 add_header_target("systemz-resource-headers" "${systemz_files}")
 add_header_target("ve-resource-headers" "${ve_files}")
 add_header_target("webassembly-resource-headers" "${webassembly_files}")
@@ -548,6 +559,12 @@ install(
   EXCLUDE_FROM_ALL
   COMPONENT riscv-resource-headers)
 
+install(
+  FILES ${riscv_files}
+  DESTINATION ${header_install_dir}
+  EXCLUDE_FROM_ALL
+  COMPONENT riscv-resource-headers)
+  
 install(
   FILES ${systemz_files}
   DESTINATION ${header_install_dir}
diff --git a/clang/lib/Headers/__clang_hip_math.h b/clang/lib/Headers/__clang_hip_math.h
index 537dd0fca8702..a4e557e45e5a0 100644
--- a/clang/lib/Headers/__clang_hip_math.h
+++ b/clang/lib/Headers/__clang_hip_math.h
@@ -243,7 +243,7 @@ float floorf(float __x) { return __ocml_floor_f32(__x); }
 
 __DEVICE__
 float fmaf(float __x, float __y, float __z) {
-  return __ocml_fma_f32(__x, __y, __z);
+  return __builtin_fmaf(__x, __y, __z);
 }
 
 __DEVICE__
@@ -621,7 +621,7 @@ float __fmaf_rz(float __x, float __y, float __z) {
 #else
 __DEVICE__
 float __fmaf_rn(float __x, float __y, float __z) {
-  return __ocml_fma_f32(__x, __y, __z);
+  return __builtin_fmaf(__x, __y, __z);
 }
 #endif
 
@@ -799,7 +799,7 @@ double floor(double __x) { return __ocml_floor_f64(__x); }
 
 __DEVICE__
 double fma(double __x, double __y, double __z) {
-  return __ocml_fma_f64(__x, __y, __z);
+  return __builtin_fma(__x, __y, __z);
 }
 
 __DEVICE__
@@ -1258,7 +1258,7 @@ double __fma_rz(double __x, double __y, double __z) {
 #else
 __DEVICE__
 double __fma_rn(double __x, double __y, double __z) {
-  return __ocml_fma_f64(__x, __y, __z);
+  return __builtin_fma(__x, __y, __z);
 }
 #endif
 // END INTRINSICS
diff --git a/clang/lib/Headers/avx2intrin.h b/clang/lib/Headers/avx2intrin.h
index f8521e7d72b5e..33f24f2443b3a 100644
--- a/clang/lib/Headers/avx2intrin.h
+++ b/clang/lib/Headers/avx2intrin.h
@@ -935,102 +935,810 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
   return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y);
 }
 
+/// Conditionally gathers two 64-bit floating-point values, either from the
+///    128-bit vector of [2 x double] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
+///    of [2 x double] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*32
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128d _mm_mask_i32gather_pd(__m128d a, const double *m, __m128i i,
+///                               __m128d mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPD instruction.
+///
+/// \param a
+///    A 128-bit vector of [2 x double] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
+///    the first two elements are used.
+/// \param mask
+///    A 128-bit vector of [2 x double] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x double] containing the gathered values.
 #define _mm_mask_i32gather_pd(a, m, i, mask, s) \
   ((__m128d)__builtin_ia32_gatherd_pd((__v2df)(__m128i)(a), \
                                       (double const *)(m), \
                                       (__v4si)(__m128i)(i), \
                                       (__v2df)(__m128d)(mask), (s)))
 
+/// Conditionally gathers four 64-bit floating-point values, either from the
+///    256-bit vector of [4 x double] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i. The 256-bit vector
+///    of [4 x double] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*32
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256d _mm256_mask_i32gather_pd(__m256d a, const double *m, __m128i i,
+///                                  __m256d mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPD instruction.
+///
+/// \param a
+///    A 256-bit vector of [4 x double] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param mask
+///    A 256-bit vector of [4 x double] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x double] containing the gathered values.
 #define _mm256_mask_i32gather_pd(a, m, i, mask, s) \
   ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)(__m256d)(a), \
                                          (double const *)(m), \
                                          (__v4si)(__m128i)(i), \
                                          (__v4df)(__m256d)(mask), (s)))
 
+/// Conditionally gathers two 64-bit floating-point values, either from the
+///    128-bit vector of [2 x double] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
+///    of [2 x double] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*64
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128d _mm_mask_i64gather_pd(__m128d a, const double *m, __m128i i,
+///                               __m128d mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPD instruction.
+///
+/// \param a
+///    A 128-bit vector of [2 x double] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [2 x double] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x double] containing the gathered values.
 #define _mm_mask_i64gather_pd(a, m, i, mask, s) \
   ((__m128d)__builtin_ia32_gatherq_pd((__v2df)(__m128d)(a), \
                                       (double const *)(m), \
                                       (__v2di)(__m128i)(i), \
                                       (__v2df)(__m128d)(mask), (s)))
 
+/// Conditionally gathers four 64-bit floating-point values, either from the
+///    256-bit vector of [4 x double] in \a a, or from memory \a m using scaled
+///    indexes from the 256-bit vector of [4 x i64] in \a i. The 256-bit vector
+///    of [4 x double] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*64
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256d _mm256_mask_i64gather_pd(__m256d a, const double *m, __m256i i,
+///                                  __m256d mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPD instruction.
+///
+/// \param a
+///    A 256-bit vector of [4 x double] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 256-bit vector of [4 x double] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x double] containing the gathered values.
 #define _mm256_mask_i64gather_pd(a, m, i, mask, s) \
   ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)(__m256d)(a), \
                                          (double const *)(m), \
                                          (__v4di)(__m256i)(i), \
                                          (__v4df)(__m256d)(mask), (s)))
 
+/// Conditionally gathers four 32-bit floating-point values, either from the
+///    128-bit vector of [4 x float] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
+///    of [4 x float] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*32
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128 _mm_mask_i32gather_ps(__m128 a, const float *m, __m128i i,
+///                              __m128 mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPS instruction.
+///
+/// \param a
+///    A 128-bit vector of [4 x float] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [4 x float] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x float] containing the gathered values.
 #define _mm_mask_i32gather_ps(a, m, i, mask, s) \
   ((__m128)__builtin_ia32_gatherd_ps((__v4sf)(__m128)(a), \
                                      (float const *)(m), \
                                      (__v4si)(__m128i)(i), \
                                      (__v4sf)(__m128)(mask), (s)))
 
+/// Conditionally gathers eight 32-bit floating-point values, either from the
+///    256-bit vector of [8 x float] in \a a, or from memory \a m using scaled
+///    indexes from the 256-bit vector of [8 x i32] in \a i. The 256-bit vector
+///    of [8 x float] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 7
+///   j := element*32
+///   k := element*32
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256 _mm256_mask_i32gather_ps(__m256 a, const float *m, __m256i i,
+///                                 __m256 mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPS instruction.
+///
+/// \param a
+///    A 256-bit vector of [8 x float] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [8 x i32] containing signed indexes into \a m.
+/// \param mask
+///    A 256-bit vector of [8 x float] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [8 x float] containing the gathered values.
 #define _mm256_mask_i32gather_ps(a, m, i, mask, s) \
   ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)(__m256)(a), \
                                         (float const *)(m), \
                                         (__v8si)(__m256i)(i), \
                                         (__v8sf)(__m256)(mask), (s)))
 
+/// Conditionally gathers two 32-bit floating-point values, either from the
+///    128-bit vector of [4 x float] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
+///    of [4 x float] in \a mask determines the source for the lower two
+///    elements. The upper two elements of the result are zeroed.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*32
+///   k := element*64
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// result[127:64] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128 _mm_mask_i64gather_ps(__m128 a, const float *m, __m128i i,
+///                              __m128 mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPS instruction.
+///
+/// \param a
+///    A 128-bit vector of [4 x float] used as the source when a mask bit is
+///    zero. Only the first two elements are used.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [4 x float] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory. Only the first
+///    two elements are used.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x float] containing the gathered values.
 #define _mm_mask_i64gather_ps(a, m, i, mask, s) \
   ((__m128)__builtin_ia32_gatherq_ps((__v4sf)(__m128)(a), \
                                      (float const *)(m), \
                                      (__v2di)(__m128i)(i), \
                                      (__v4sf)(__m128)(mask), (s)))
 
+/// Conditionally gathers four 32-bit floating-point values, either from the
+///    128-bit vector of [4 x float] in \a a, or from memory \a m using scaled
+///    indexes from the 256-bit vector of [4 x i64] in \a i. The 128-bit vector
+///    of [4 x float] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*64
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128 _mm256_mask_i64gather_ps(__m128 a, const float *m, __m256i i,
+///                                 __m128 mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPS instruction.
+///
+/// \param a
+///    A 128-bit vector of [4 x float] used as the source when a mask bit is
+///   zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [4 x float] containing the mask. The most
+///    significant bit of each element in the mask vector represents the mask
+///    bits. If a mask bit is zero, the corresponding value from vector \a a
+///    is gathered; otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x float] containing the gathered values.
 #define _mm256_mask_i64gather_ps(a, m, i, mask, s) \
   ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)(__m128)(a), \
                                         (float const *)(m), \
                                         (__v4di)(__m256i)(i), \
                                         (__v4sf)(__m128)(mask), (s)))
 
+/// Conditionally gathers four 32-bit integer values, either from the
+///    128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
+///    of [4 x i32] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*32
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_mask_i32gather_epi32(__m128i a, const int *m, __m128i i,
+///                                  __m128i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDD instruction.
+///
+/// \param a
+///    A 128-bit vector of [4 x i32] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [4 x i32] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
 #define _mm_mask_i32gather_epi32(a, m, i, mask, s) \
   ((__m128i)__builtin_ia32_gatherd_d((__v4si)(__m128i)(a), \
                                      (int const *)(m), \
                                      (__v4si)(__m128i)(i), \
                                      (__v4si)(__m128i)(mask), (s)))
 
+/// Conditionally gathers eight 32-bit integer values, either from the
+///    256-bit vector of [8 x i32] in \a a, or from memory \a m using scaled
+///    indexes from the 256-bit vector of [8 x i32] in \a i. The 256-bit vector
+///    of [8 x i32] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 7
+///   j := element*32
+///   k := element*32
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_mask_i32gather_epi32(__m256i a, const int *m, __m256i i,
+///                                     __m256i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDD instruction.
+///
+/// \param a
+///    A 256-bit vector of [8 x i32] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [8 x i32] containing signed indexes into \a m.
+/// \param mask
+///    A 256-bit vector of [8 x i32] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [8 x i32] containing the gathered values.
 #define _mm256_mask_i32gather_epi32(a, m, i, mask, s) \
   ((__m256i)__builtin_ia32_gatherd_d256((__v8si)(__m256i)(a), \
                                         (int const *)(m), \
                                         (__v8si)(__m256i)(i), \
                                         (__v8si)(__m256i)(mask), (s)))
 
+/// Conditionally gathers two 32-bit integer values, either from the
+///    128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
+///    of [4 x i32] in \a mask determines the source for the lower two
+///    elements. The upper two elements of the result are zeroed.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*32
+///   k := element*64
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// result[127:64] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_mask_i64gather_epi32(__m128i a, const int *m, __m128i i,
+///                                  __m128i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQD instruction.
+///
+/// \param a
+///    A 128-bit vector of [4 x i32] used as the source when a mask bit is
+///   zero. Only the first two elements are used.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [4 x i32] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory. Only the first two elements
+///    are used.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
 #define _mm_mask_i64gather_epi32(a, m, i, mask, s) \
   ((__m128i)__builtin_ia32_gatherq_d((__v4si)(__m128i)(a), \
                                      (int const *)(m), \
                                      (__v2di)(__m128i)(i), \
                                      (__v4si)(__m128i)(mask), (s)))
 
+/// Conditionally gathers four 32-bit integer values, either from the
+///    128-bit vector of [4 x i32] in \a a, or from memory \a m using scaled
+///    indexes from the 256-bit vector of [4 x i64] in \a i. The 128-bit vector
+///    of [4 x i32] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*64
+///   IF mask[j+31] == 0
+///     result[j+31:j] := a[j+31:j]
+///   ELSE
+///     result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm256_mask_i64gather_epi32(__m128i a, const int *m, __m256i i,
+///                                     __m128i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQD instruction.
+///
+/// \param a
+///    A 128-bit vector of [4 x i32] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [4 x i32] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
 #define _mm256_mask_i64gather_epi32(a, m, i, mask, s) \
   ((__m128i)__builtin_ia32_gatherq_d256((__v4si)(__m128i)(a), \
                                         (int const *)(m), \
                                         (__v4di)(__m256i)(i), \
                                         (__v4si)(__m128i)(mask), (s)))
 
+/// Conditionally gathers two 64-bit integer values, either from the
+///    128-bit vector of [2 x i64] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i. The 128-bit vector
+///    of [2 x i64] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*32
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_mask_i32gather_epi64(__m128i a, const long long *m, __m128i i,
+///                                  __m128i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
+///
+/// \param a
+///    A 128-bit vector of [2 x i64] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
+///    the first two elements are used.
+/// \param mask
+///    A 128-bit vector of [2 x i64] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
 #define _mm_mask_i32gather_epi64(a, m, i, mask, s) \
   ((__m128i)__builtin_ia32_gatherd_q((__v2di)(__m128i)(a), \
                                      (long long const *)(m), \
                                      (__v4si)(__m128i)(i), \
                                      (__v2di)(__m128i)(mask), (s)))
 
+/// Conditionally gathers four 64-bit integer values, either from the
+///    256-bit vector of [4 x i64] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i. The 256-bit vector
+///    of [4 x i64] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*32
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_mask_i32gather_epi64(__m256i a, const long long *m,
+///                                     __m128i i, __m256i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
+///
+/// \param a
+///    A 256-bit vector of [4 x i64] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param mask
+///    A 256-bit vector of [4 x i64] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
 #define _mm256_mask_i32gather_epi64(a, m, i, mask, s) \
   ((__m256i)__builtin_ia32_gatherd_q256((__v4di)(__m256i)(a), \
                                         (long long const *)(m), \
                                         (__v4si)(__m128i)(i), \
                                         (__v4di)(__m256i)(mask), (s)))
 
+/// Conditionally gathers two 64-bit integer values, either from the
+///    128-bit vector of [2 x i64] in \a a, or from memory \a m using scaled
+///    indexes from the 128-bit vector of [2 x i64] in \a i. The 128-bit vector
+///    of [2 x i64] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*64
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_mask_i64gather_epi64(__m128i a, const long long *m, __m128i i,
+///                                  __m128i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
+///
+/// \param a
+///    A 128-bit vector of [2 x i64] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 128-bit vector of [2 x i64] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
 #define _mm_mask_i64gather_epi64(a, m, i, mask, s) \
   ((__m128i)__builtin_ia32_gatherq_q((__v2di)(__m128i)(a), \
                                      (long long const *)(m), \
                                      (__v2di)(__m128i)(i), \
                                      (__v2di)(__m128i)(mask), (s)))
 
+/// Conditionally gathers four 64-bit integer values, either from the
+///    256-bit vector of [4 x i64] in \a a, or from memory \a m using scaled
+///    indexes from the 256-bit vector of [4 x i64] in \a i. The 256-bit vector
+///    of [4 x i64] in \a mask determines the source for each element.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*64
+///   IF mask[j+63] == 0
+///     result[j+63:j] := a[j+63:j]
+///   ELSE
+///     result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+///   FI
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_mask_i64gather_epi64(__m256i a, const long long *m,
+///                                     __m256i i, __m256i mask, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
+///
+/// \param a
+///    A 256-bit vector of [4 x i64] used as the source when a mask bit is
+///    zero.
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param mask
+///    A 256-bit vector of [4 x i64] containing the mask. The most significant
+///    bit of each element in the mask vector represents the mask bits. If a
+///    mask bit is zero, the corresponding value from vector \a a is gathered;
+///    otherwise the value is loaded from memory.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
 #define _mm256_mask_i64gather_epi64(a, m, i, mask, s) \
   ((__m256i)__builtin_ia32_gatherq_q256((__v4di)(__m256i)(a), \
                                         (long long const *)(m), \
                                         (__v4di)(__m256i)(i), \
                                         (__v4di)(__m256i)(mask), (s)))
 
+/// Gathers two 64-bit floating-point values from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*32
+///   result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128d _mm_i32gather_pd(const double *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
+///    the first two elements are used.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x double] containing the gathered values.
 #define _mm_i32gather_pd(m, i, s) \
   ((__m128d)__builtin_ia32_gatherd_pd((__v2df)_mm_undefined_pd(), \
                                       (double const *)(m), \
@@ -1039,6 +1747,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                            _mm_setzero_pd()), \
                                       (s)))
 
+/// Gathers four 64-bit floating-point values from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*32
+///   result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256d _mm256_i32gather_pd(const double *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x double] containing the gathered values.
 #define _mm256_i32gather_pd(m, i, s) \
   ((__m256d)__builtin_ia32_gatherd_pd256((__v4df)_mm256_undefined_pd(), \
                                          (double const *)(m), \
@@ -1048,6 +1783,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                                _CMP_EQ_OQ), \
                                          (s)))
 
+/// Gathers two 64-bit floating-point values from memory \a m using scaled
+///    indexes from the 128-bit vector of [2 x i64] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*64
+///   result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128d _mm_i64gather_pd(const double *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x double] containing the gathered values.
 #define _mm_i64gather_pd(m, i, s) \
   ((__m128d)__builtin_ia32_gatherq_pd((__v2df)_mm_undefined_pd(), \
                                       (double const *)(m), \
@@ -1056,6 +1818,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                            _mm_setzero_pd()), \
                                       (s)))
 
+/// Gathers four 64-bit floating-point values from memory \a m using scaled
+///    indexes from the 256-bit vector of [4 x i64] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*64
+///   result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256d _mm256_i64gather_pd(const double *m, __m256i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x double] containing the gathered values.
 #define _mm256_i64gather_pd(m, i, s) \
   ((__m256d)__builtin_ia32_gatherq_pd256((__v4df)_mm256_undefined_pd(), \
                                          (double const *)(m), \
@@ -1065,6 +1854,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                                _CMP_EQ_OQ), \
                                          (s)))
 
+/// Gathers four 32-bit floating-point values from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*32
+///   result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128 _mm_i32gather_ps(const float *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPS instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x float] containing the gathered values.
 #define _mm_i32gather_ps(m, i, s) \
   ((__m128)__builtin_ia32_gatherd_ps((__v4sf)_mm_undefined_ps(), \
                                      (float const *)(m), \
@@ -1073,6 +1889,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                           _mm_setzero_ps()), \
                                      (s)))
 
+/// Gathers eight 32-bit floating-point values from memory \a m using scaled
+///    indexes from the 256-bit vector of [8 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 7
+///   j := element*32
+///   k := element*32
+///   result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256 _mm256_i32gather_ps(const float *m, __m256i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERDPS instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [8 x i32] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [8 x float] containing the gathered values.
 #define _mm256_i32gather_ps(m, i, s) \
   ((__m256)__builtin_ia32_gatherd_ps256((__v8sf)_mm256_undefined_ps(), \
                                         (float const *)(m), \
@@ -1082,6 +1925,35 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                               _CMP_EQ_OQ), \
                                         (s)))
 
+/// Gathers two 32-bit floating-point values from memory \a m using scaled
+///    indexes from the 128-bit vector of [2 x i64] in \a i. The upper two
+///    elements of the result are zeroed.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*32
+///   k := element*64
+///   result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// result[127:64] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128 _mm_i64gather_ps(const float *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPS instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x float] containing the gathered values.
 #define _mm_i64gather_ps(m, i, s) \
   ((__m128)__builtin_ia32_gatherq_ps((__v4sf)_mm_undefined_ps(), \
                                      (float const *)(m), \
@@ -1090,6 +1962,33 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                           _mm_setzero_ps()), \
                                      (s)))
 
+/// Gathers four 32-bit floating-point values from memory \a m using scaled
+///    indexes from the 256-bit vector of [4 x i64] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*64
+///   result[j+31:j] := Load32(m + SignExtend(i[k+64:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128 _mm256_i64gather_ps(const float *m, __m256i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VGATHERQPS instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x float] containing the gathered values.
 #define _mm256_i64gather_ps(m, i, s) \
   ((__m128)__builtin_ia32_gatherq_ps256((__v4sf)_mm_undefined_ps(), \
                                         (float const *)(m), \
@@ -1098,44 +1997,263 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
                                                              _mm_setzero_ps()), \
                                         (s)))
 
+/// Gathers four 32-bit floating-point values from memory \a m using scaled
+///    indexes from the 128-bit vector of [4 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*32
+///   result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_i32gather_epi32(const int *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
 #define _mm_i32gather_epi32(m, i, s) \
   ((__m128i)__builtin_ia32_gatherd_d((__v4si)_mm_undefined_si128(), \
                                      (int const *)(m), (__v4si)(__m128i)(i), \
                                      (__v4si)_mm_set1_epi32(-1), (s)))
 
+/// Gathers eight 32-bit floating-point values from memory \a m using scaled
+///    indexes from the 256-bit vector of [8 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 7
+///   j := element*32
+///   k := element*32
+///   result[j+31:j] := Load32(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_i32gather_epi32(const int *m, __m256i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [8 x i32] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [8 x i32] containing the gathered values.
 #define _mm256_i32gather_epi32(m, i, s) \
   ((__m256i)__builtin_ia32_gatherd_d256((__v8si)_mm256_undefined_si256(), \
                                         (int const *)(m), (__v8si)(__m256i)(i), \
                                         (__v8si)_mm256_set1_epi32(-1), (s)))
 
+/// Gathers two 32-bit integer values from memory \a m using scaled indexes
+///    from the 128-bit vector of [2 x i64] in \a i. The upper two elements
+///    of the result are zeroed.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*32
+///   k := element*64
+///   result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// result[127:64] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_i64gather_epi32(const int *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
 #define _mm_i64gather_epi32(m, i, s) \
   ((__m128i)__builtin_ia32_gatherq_d((__v4si)_mm_undefined_si128(), \
                                      (int const *)(m), (__v2di)(__m128i)(i), \
                                      (__v4si)_mm_set1_epi32(-1), (s)))
 
+/// Gathers four 32-bit integer values from memory \a m using scaled indexes
+///    from the 256-bit vector of [4 x i64] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*32
+///   k := element*64
+///   result[j+31:j] := Load32(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm256_i64gather_epi32(const int *m, __m256i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQD instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [4 x i32] containing the gathered values.
 #define _mm256_i64gather_epi32(m, i, s) \
   ((__m128i)__builtin_ia32_gatherq_d256((__v4si)_mm_undefined_si128(), \
                                         (int const *)(m), (__v4di)(__m256i)(i), \
                                         (__v4si)_mm_set1_epi32(-1), (s)))
 
+/// Gathers two 64-bit integer values from memory \a m using scaled indexes
+///    from the 128-bit vector of [4 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*32
+///   result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_i32gather_epi64(const long long *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m. Only
+///    the first two elements are used.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
 #define _mm_i32gather_epi64(m, i, s) \
   ((__m128i)__builtin_ia32_gatherd_q((__v2di)_mm_undefined_si128(), \
                                      (long long const *)(m), \
                                      (__v4si)(__m128i)(i), \
                                      (__v2di)_mm_set1_epi64x(-1), (s)))
 
+/// Gathers four 64-bit integer values from memory \a m using scaled indexes
+///    from the 128-bit vector of [4 x i32] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*32
+///   result[j+63:j] := Load64(m + SignExtend(i[k+31:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_i32gather_epi64(const long long *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERDQ instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [4 x i32] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
 #define _mm256_i32gather_epi64(m, i, s) \
   ((__m256i)__builtin_ia32_gatherd_q256((__v4di)_mm256_undefined_si256(), \
                                         (long long const *)(m), \
                                         (__v4si)(__m128i)(i), \
                                         (__v4di)_mm256_set1_epi64x(-1), (s)))
 
+/// Gathers two 64-bit integer values from memory \a m using scaled indexes
+///    from the 128-bit vector of [2 x i64] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 1
+///   j := element*64
+///   k := element*64
+///   result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_i64gather_epi64(const long long *m, __m128i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 128-bit vector of [2 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 128-bit vector of [2 x i64] containing the gathered values.
 #define _mm_i64gather_epi64(m, i, s) \
   ((__m128i)__builtin_ia32_gatherq_q((__v2di)_mm_undefined_si128(), \
                                      (long long const *)(m), \
                                      (__v2di)(__m128i)(i), \
                                      (__v2di)_mm_set1_epi64x(-1), (s)))
 
+/// Gathers four 64-bit integer values from memory \a m using scaled indexes
+///    from the 256-bit vector of [4 x i64] in \a i.
+///
+/// \code{.operation}
+/// FOR element := 0 to 3
+///   j := element*64
+///   k := element*64
+///   result[j+63:j] := Load64(m + SignExtend(i[k+63:k])*s)
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_i64gather_epi64(const long long *m, __m256i i, const int s);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPGATHERQQ instruction.
+///
+/// \param m
+///    A pointer to the memory used for loading values.
+/// \param i
+///    A 256-bit vector of [4 x i64] containing signed indexes into \a m.
+/// \param s
+///    A literal constant scale factor for the indexes in \a i. Must be
+///    1, 2, 4, or 8.
+/// \returns A 256-bit vector of [4 x i64] containing the gathered values.
 #define _mm256_i64gather_epi64(m, i, s) \
   ((__m256i)__builtin_ia32_gatherq_q256((__v4di)_mm256_undefined_si256(), \
                                         (long long const *)(m), \
diff --git a/clang/lib/Headers/cuda_wrappers/bits/shared_ptr_base.h b/clang/lib/Headers/cuda_wrappers/bits/shared_ptr_base.h
new file mode 100644
index 0000000000000..10028dd7bd9a0
--- /dev/null
+++ b/clang/lib/Headers/cuda_wrappers/bits/shared_ptr_base.h
@@ -0,0 +1,9 @@
+// CUDA headers define __noinline__ which interferes with libstdc++'s use of
+// `__attribute((__noinline__))`. In order to avoid compilation error,
+// temporarily unset __noinline__ when we include affected libstdc++ header.
+
+#pragma push_macro("__noinline__")
+#undef __noinline__
+#include_next "bits/shared_ptr_base.h"
+
+#pragma pop_macro("__noinline__")
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index dff4da2465d27..c5f84ae0286b7 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -323,18 +323,14 @@ _rdrand32_step(unsigned int *__p)
 /// \param __p
 ///    A pointer to a 64-bit memory location to place the random value.
 /// \returns 1 if the value was successfully generated, 0 otherwise.
-#ifdef __x86_64__
 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 _rdrand64_step(unsigned long long *__p)
 {
+#ifdef __x86_64__
   return (int)__builtin_ia32_rdrand64_step(__p);
-}
 #else
-// We need to emulate the functionality of 64-bit rdrand with 2 32-bit
-// rdrand instructions.
-static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
-_rdrand64_step(unsigned long long *__p)
-{
+  // We need to emulate the functionality of 64-bit rdrand with 2 32-bit
+  // rdrand instructions.
   unsigned int __lo, __hi;
   unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);
   unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);
@@ -345,8 +341,8 @@ _rdrand64_step(unsigned long long *__p)
     *__p = 0;
     return 0;
   }
-}
 #endif
+}
 #endif /* __RDRND__ */
 
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
diff --git a/clang/lib/Headers/riscv_ntlh.h b/clang/lib/Headers/riscv_ntlh.h
new file mode 100644
index 0000000000000..9ce1709205835
--- /dev/null
+++ b/clang/lib/Headers/riscv_ntlh.h
@@ -0,0 +1,28 @@
+/*===---- riscv_ntlh.h - RISC-V NTLH intrinsics ----------------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __RISCV_NTLH_H
+#define __RISCV_NTLH_H
+
+#ifndef __riscv_zihintntl
+#error "NTLH intrinsics require the NTLH extension."
+#endif
+
+enum {
+  __RISCV_NTLH_INNERMOST_PRIVATE = 2,
+  __RISCV_NTLH_ALL_PRIVATE,
+  __RISCV_NTLH_INNERMOST_SHARED,
+  __RISCV_NTLH_ALL
+};
+
+#define __riscv_ntl_load(PTR, DOMAIN) __builtin_riscv_ntl_load((PTR), (DOMAIN))
+#define __riscv_ntl_store(PTR, VAL, DOMAIN)                                    \
+  __builtin_riscv_ntl_store((PTR), (VAL), (DOMAIN))
+
+#endif
\ No newline at end of file
diff --git a/libcxx/include/__pstl_execution b/clang/lib/Headers/sifive_vector.h
similarity index 59%
rename from libcxx/include/__pstl_execution
rename to clang/lib/Headers/sifive_vector.h
index 0e2cd44561cdf..42d7224db6145 100644
--- a/libcxx/include/__pstl_execution
+++ b/clang/lib/Headers/sifive_vector.h
@@ -1,5 +1,4 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===----- sifive_vector.h - SiFive Vector definitions --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,9 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __PSTL_EXECUTION
-#define __PSTL_EXECUTION
+#ifndef _SIFIVE_VECTOR_H_
+#define _SIFIVE_VECTOR_H_
+
+#include "riscv_vector.h"
 
-#include <pstl/internal/glue_execution_defs.h>
+#pragma clang riscv intrinsic sifive_vector
 
-#endif /* __PSTL_EXECUTION */
+#endif //_SIFIVE_VECTOR_H_
diff --git a/clang/lib/Headers/stdatomic.h b/clang/lib/Headers/stdatomic.h
index 48b3ab98c640c..aed33d4333e84 100644
--- a/clang/lib/Headers/stdatomic.h
+++ b/clang/lib/Headers/stdatomic.h
@@ -49,7 +49,7 @@ extern "C" {
    in C2x mode; switch to the correct values once they've been published. */
 #if (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L) ||               \
     defined(__cplusplus)
-/* ATOMIC_VAR_INIT was removed in C2x, but still remains in C++2b. */
+/* ATOMIC_VAR_INIT was removed in C2x, but still remains in C++23. */
 #define ATOMIC_VAR_INIT(value) (value)
 #endif
 
diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp
index 0adbaa36bf7cd..a506b49176302 100644
--- a/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -652,9 +652,22 @@ bool Scanner::lexPragma(const char *&First, const char *const End) {
     return false;
   }
 
-  // #pragma clang.
-  if (!isNextIdentifierOrSkipLine("module", First, End))
+  FoundId = tryLexIdentifierOrSkipLine(First, End);
+  if (!FoundId)
     return false;
+  Id = *FoundId;
+
+  // #pragma clang system_header
+  if (Id == "system_header") {
+    lexPPDirectiveBody(First, End);
+    pushDirective(pp_pragma_system_header);
+    return false;
+  }
+
+  if (Id != "module") {
+    skipLine(First, End);
+    return false;
+  }
 
   // #pragma clang module.
   if (!isNextIdentifierOrSkipLine("import", First, End))
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index bc97ca18a2ebd..82f65ef3aaf8c 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -3348,8 +3348,8 @@ std::optional<uint32_t> Lexer::tryReadNumericUCN(const char *&StartPtr,
   }
 
   if (Delimited && PP) {
-    Diag(SlashLoc, PP->getLangOpts().CPlusPlus2b
-                       ? diag::warn_cxx2b_delimited_escape_sequence
+    Diag(SlashLoc, PP->getLangOpts().CPlusPlus23
+                       ? diag::warn_cxx23_delimited_escape_sequence
                        : diag::ext_delimited_escape_sequence)
         << /*delimited*/ 0 << (PP->getLangOpts().CPlusPlus ? 1 : 0);
   }
@@ -3436,8 +3436,8 @@ std::optional<uint32_t> Lexer::tryReadNamedUCN(const char *&StartPtr,
   }
 
   if (Diagnose && Match)
-    Diag(SlashLoc, PP->getLangOpts().CPlusPlus2b
-                       ? diag::warn_cxx2b_delimited_escape_sequence
+    Diag(SlashLoc, PP->getLangOpts().CPlusPlus23
+                       ? diag::warn_cxx23_delimited_escape_sequence
                        : diag::ext_delimited_escape_sequence)
         << /*named*/ 1 << (PP->getLangOpts().CPlusPlus ? 1 : 0);
 
@@ -4361,11 +4361,9 @@ bool Lexer::LexTokenInternal(Token &Result, bool TokAtPhysicalStartOfLine) {
   FormTokenWithChars(Result, CurPtr, tok::hash);
   PP->HandleDirective(Result);
 
-  if (PP->hadModuleLoaderFatalFailure()) {
+  if (PP->hadModuleLoaderFatalFailure())
     // With a fatal failure in the module loader, we abort parsing.
-    assert(Result.is(tok::eof) && "Preprocessor did not set tok:eof");
     return true;
-  }
 
   // We parsed the directive; lex a token with the new state.
   return false;
@@ -4482,6 +4480,7 @@ bool Lexer::LexDependencyDirectiveTokenWhileSkipping(Token &Result) {
     case pp_pragma_push_macro:
     case pp_pragma_pop_macro:
     case pp_pragma_include_alias:
+    case pp_pragma_system_header:
     case pp_include_next:
     case decl_at_import:
     case cxx_module_decl:
diff --git a/clang/lib/Lex/LiteralSupport.cpp b/clang/lib/Lex/LiteralSupport.cpp
index 9cc0215fb6250..c066e1daa618d 100644
--- a/clang/lib/Lex/LiteralSupport.cpp
+++ b/clang/lib/Lex/LiteralSupport.cpp
@@ -312,7 +312,7 @@ static unsigned ProcessCharEscape(const char *ThisTokBegin,
           << tok::r_brace;
     else if (!HadError) {
       Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
-           Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence
+           Features.CPlusPlus23 ? diag::warn_cxx23_delimited_escape_sequence
                                 : diag::ext_delimited_escape_sequence)
           << /*delimited*/ 0 << (Features.CPlusPlus ? 1 : 0);
     }
@@ -641,7 +641,7 @@ static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
 
   if ((IsDelimitedEscapeSequence || IsNamedEscapeSequence) && Diags)
     Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
-         Features.CPlusPlus2b ? diag::warn_cxx2b_delimited_escape_sequence
+         Features.CPlusPlus23 ? diag::warn_cxx23_delimited_escape_sequence
                               : diag::ext_delimited_escape_sequence)
         << (IsNamedEscapeSequence ? 1 : 0) << (Features.CPlusPlus ? 1 : 0);
 
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 3132d57a62d3f..62e51e133b3af 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -434,7 +434,7 @@ void Preprocessor::SuggestTypoedDirective(const Token &Tok,
   std::vector<StringRef> Candidates = {
       "if", "ifdef", "ifndef", "elif", "else", "endif"
   };
-  if (LangOpts.C2x || LangOpts.CPlusPlus2b)
+  if (LangOpts.C2x || LangOpts.CPlusPlus23)
     Candidates.insert(Candidates.end(), {"elifdef", "elifndef"});
 
   if (std::optional<StringRef> Sugg = findSimilarStr(Directive, Candidates)) {
@@ -745,12 +745,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
         if (!CondInfo.WasSkipping)
           SkippingRangeState.endLexPass(Hashptr);
 
-        // Warn if using `#elifdef` & `#elifndef` in not C2x & C++2b mode even
+        // Warn if using `#elifdef` & `#elifndef` in not C2x & C++23 mode even
         // if this branch is in a skipping block.
         unsigned DiagID;
         if (LangOpts.CPlusPlus)
-          DiagID = LangOpts.CPlusPlus2b ? diag::warn_cxx2b_compat_pp_directive
-                                        : diag::ext_cxx2b_pp_directive;
+          DiagID = LangOpts.CPlusPlus23 ? diag::warn_cxx23_compat_pp_directive
+                                        : diag::ext_cxx23_pp_directive;
         else
           DiagID = LangOpts.C2x ? diag::warn_c2x_compat_pp_directive
                                 : diag::ext_c2x_pp_directive;
@@ -1177,6 +1177,10 @@ void Preprocessor::HandleDirective(Token &Result) {
 
   switch (Result.getKind()) {
   case tok::eod:
+    // Ignore the null directive with regards to the multiple-include
+    // optimization, i.e. allow the null directive to appear outside of the
+    // include guard and still enable the multiple-include optimization.
+    CurPPLexer->MIOpt.SetReadToken(ReadAnyTokensBeforeDirective);
     return;   // null directive.
   case tok::code_completion:
     setCodeCompletionReached();
@@ -1252,10 +1256,10 @@ void Preprocessor::HandleDirective(Token &Result) {
 
     case tok::pp_warning:
       if (LangOpts.CPlusPlus)
-        Diag(Result, LangOpts.CPlusPlus2b
-                         ? diag::warn_cxx2b_compat_warning_directive
+        Diag(Result, LangOpts.CPlusPlus23
+                         ? diag::warn_cxx23_compat_warning_directive
                          : diag::ext_pp_warning_directive)
-            << /*C++2b*/ 1;
+            << /*C++23*/ 1;
       else
         Diag(Result, LangOpts.C2x ? diag::warn_c2x_compat_warning_directive
                                   : diag::ext_pp_warning_directive)
@@ -3419,14 +3423,14 @@ void Preprocessor::HandleElifFamilyDirective(Token &ElifToken,
                                                  : PED_Elifndef;
   ++NumElse;
 
-  // Warn if using `#elifdef` & `#elifndef` in not C2x & C++2b mode.
+  // Warn if using `#elifdef` & `#elifndef` in not C2x & C++23 mode.
   switch (DirKind) {
   case PED_Elifdef:
   case PED_Elifndef:
     unsigned DiagID;
     if (LangOpts.CPlusPlus)
-      DiagID = LangOpts.CPlusPlus2b ? diag::warn_cxx2b_compat_pp_directive
-                                    : diag::ext_cxx2b_pp_directive;
+      DiagID = LangOpts.CPlusPlus23 ? diag::warn_cxx23_compat_pp_directive
+                                    : diag::ext_cxx23_pp_directive;
     else
       DiagID = LangOpts.C2x ? diag::warn_c2x_compat_pp_directive
                             : diag::ext_c2x_pp_directive;
diff --git a/clang/lib/Lex/PPExpressions.cpp b/clang/lib/Lex/PPExpressions.cpp
index aa411cfc5f2c0..6c935148f4072 100644
--- a/clang/lib/Lex/PPExpressions.cpp
+++ b/clang/lib/Lex/PPExpressions.cpp
@@ -323,13 +323,13 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
         PP.Diag(PeekTok, diag::ext_c99_longlong);
     }
 
-    // 'z/uz' literals are a C++2b feature.
+    // 'z/uz' literals are a C++23 feature.
     if (Literal.isSizeT)
       PP.Diag(PeekTok, PP.getLangOpts().CPlusPlus
-                           ? PP.getLangOpts().CPlusPlus2b
+                           ? PP.getLangOpts().CPlusPlus23
                                  ? diag::warn_cxx20_compat_size_t_suffix
-                                 : diag::ext_cxx2b_size_t_suffix
-                           : diag::err_cxx2b_size_t_suffix);
+                                 : diag::ext_cxx23_size_t_suffix
+                           : diag::err_cxx23_size_t_suffix);
 
     // 'wb/uwb' literals are a C2x feature. We explicitly do not support the
     // suffix in C++ as an extension because a library-based UDL that resolves
diff --git a/clang/lib/Parse/ParseAST.cpp b/clang/lib/Parse/ParseAST.cpp
index 04b3f0460bf3e..77ab3b556da58 100644
--- a/clang/lib/Parse/ParseAST.cpp
+++ b/clang/lib/Parse/ParseAST.cpp
@@ -18,6 +18,7 @@
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/Sema/TemplateInstCallback.h"
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 930bb81a558fb..43a0ea4f0ed77 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -10,12 +10,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Parse/Parser.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/Parse/ParseDiagnostic.h"
+#include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+
 using namespace clang;
 
 /// ParseCXXInlineMethodDef - We parsed and verified that the specified
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 18897188828f1..e28d609464123 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -21,6 +21,7 @@
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
@@ -7381,13 +7382,9 @@ void Parser::ParseParameterDeclarationClause(
           DefArgToks.reset(new CachedTokens);
 
           SourceLocation ArgStartLoc = NextToken().getLocation();
-          if (!ConsumeAndStoreInitializer(*DefArgToks, CIK_DefaultArgument)) {
-            DefArgToks.reset();
-            Actions.ActOnParamDefaultArgumentError(Param, EqualLoc);
-          } else {
-            Actions.ActOnParamUnparsedDefaultArgument(Param, EqualLoc,
-                                                      ArgStartLoc);
-          }
+          ConsumeAndStoreInitializer(*DefArgToks, CIK_DefaultArgument);
+          Actions.ActOnParamUnparsedDefaultArgument(Param, EqualLoc,
+                                                    ArgStartLoc);
         } else {
           // Consume the '='.
           ConsumeToken();
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 45b0a1a5f8669..717c014520892 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -23,6 +23,7 @@
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/ADT/SmallString.h"
@@ -1075,7 +1076,7 @@ SourceLocation Parser::ParseDecltypeSpecifier(DeclSpec &DS) {
     // Check for C++1y 'decltype(auto)'.
     if (Tok.is(tok::kw_auto) && NextToken().is(tok::r_paren)) {
       // the typename-specifier in a function-style cast expression may
-      // be 'auto' since C++2b.
+      // be 'auto' since C++23.
       Diag(Tok.getLocation(),
            getLangOpts().CPlusPlus14
                ? diag::warn_cxx11_compat_decltype_auto_type_specifier
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 554113c5a27c2..b99baf476592f 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -20,12 +20,13 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "clang/Parse/Parser.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/Basic/PrettyStackTrace.h"
+#include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/TypoCorrection.h"
@@ -1870,7 +1871,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 ///         primary-expression
 ///         postfix-expression '[' expression ']'
 ///         postfix-expression '[' braced-init-list ']'
-///         postfix-expression '[' expression-list [opt] ']'  [C++2b 12.4.5]
+///         postfix-expression '[' expression-list [opt] ']'  [C++23 12.4.5]
 ///         postfix-expression '(' argument-expression-list[opt] ')'
 ///         postfix-expression '.' identifier
 ///         postfix-expression '->' identifier
@@ -1945,10 +1946,10 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
 
       // We try to parse a list of indexes in all language mode first
       // and, in we find 0 or one index, we try to parse an OpenMP array
-      // section. This allow us to support C++2b multi dimensional subscript and
+      // section. This allow us to support C++23 multi dimensional subscript and
       // OpenMp sections in the same language mode.
       if (!getLangOpts().OpenMP || Tok.isNot(tok::colon)) {
-        if (!getLangOpts().CPlusPlus2b) {
+        if (!getLangOpts().CPlusPlus23) {
           ExprResult Idx;
           if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace)) {
             Diag(Tok, diag::warn_cxx98_compat_generalized_initializer_lists);
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 7d1d1010455ca..8f56316cefcdf 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -20,6 +20,7 @@
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/Support/Compiler.h"
@@ -724,7 +725,7 @@ ExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
 ///         '&' identifier initializer
 ///
 ///       lambda-declarator:
-///         lambda-specifiers     [C++2b]
+///         lambda-specifiers     [C++23]
 ///         '(' parameter-declaration-clause ')' lambda-specifiers
 ///             requires-clause[opt]
 ///
@@ -1204,7 +1205,7 @@ static void tryConsumeLambdaSpecifierToken(Parser &P,
 static void addStaticToLambdaDeclSpecifier(Parser &P, SourceLocation StaticLoc,
                                            DeclSpec &DS) {
   if (StaticLoc.isValid()) {
-    P.Diag(StaticLoc, !P.getLangOpts().CPlusPlus2b
+    P.Diag(StaticLoc, !P.getLangOpts().CPlusPlus23
                           ? diag::err_static_lambda
                           : diag::warn_cxx20_compat_static_lambda);
     const char *PrevSpec = nullptr;
@@ -1361,7 +1362,7 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
   // or operator template declaration. We accept this as a conforming extension
   // in all language modes that support lambdas.
   if (isCXX11AttributeSpecifier()) {
-    Diag(Tok, getLangOpts().CPlusPlus2b
+    Diag(Tok, getLangOpts().CPlusPlus23
                   ? diag::warn_cxx20_compat_decl_attrs_on_lambda
                   : diag::ext_decl_attrs_on_lambda);
     MaybeParseCXX11Attributes(D);
@@ -1500,7 +1501,7 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
                   tok::kw_requires, tok::kw_noexcept) ||
       (Tok.is(tok::l_square) && NextToken().is(tok::l_square));
 
-  if (HasSpecifiers && !HasParentheses && !getLangOpts().CPlusPlus2b) {
+  if (HasSpecifiers && !HasParentheses && !getLangOpts().CPlusPlus23) {
     // It's common to forget that one needs '()' before 'mutable', an
     // attribute specifier, the result type, or the requires clause. Deal with
     // this.
@@ -1995,7 +1996,7 @@ Parser::ParseAliasDeclarationInInitStatement(DeclaratorContext Context,
   if (!DG)
     return DG;
 
-  Diag(DeclStart, !getLangOpts().CPlusPlus2b
+  Diag(DeclStart, !getLangOpts().CPlusPlus23
                       ? diag::ext_alias_in_init_statement
                       : diag::warn_cxx20_alias_in_init_statement)
       << SourceRange(DeclStart, DeclEnd);
diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp
index e4c2dc446000e..f52c04ba2c4d7 100644
--- a/clang/lib/Parse/ParseInit.cpp
+++ b/clang/lib/Parse/ParseInit.cpp
@@ -15,6 +15,7 @@
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/Designator.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 10f0b532ebf3c..a7db7a1c25822 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -19,6 +19,7 @@
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 31d597abde6bf..aad8c09729774 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -19,6 +19,7 @@
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -4024,6 +4025,7 @@ void PragmaMaxTokensTotalHandler::HandlePragma(Preprocessor &PP,
 }
 
 // Handle '#pragma clang riscv intrinsic vector'.
+//        '#pragma clang riscv intrinsic sifive_vector'.
 void PragmaRISCVHandler::HandlePragma(Preprocessor &PP,
                                       PragmaIntroducer Introducer,
                                       Token &FirstToken) {
@@ -4039,9 +4041,11 @@ void PragmaRISCVHandler::HandlePragma(Preprocessor &PP,
 
   PP.Lex(Tok);
   II = Tok.getIdentifierInfo();
-  if (!II || !II->isStr("vector")) {
+  StringRef IntrinsicClass = II->getName();
+  if (!II || !(II->isStr("vector") || II->isStr("sifive_vector"))) {
     PP.Diag(Tok.getLocation(), diag::warn_pragma_invalid_argument)
-        << PP.getSpelling(Tok) << "riscv" << /*Expected=*/true << "'vector'";
+        << PP.getSpelling(Tok) << "riscv" << /*Expected=*/true
+        << "'vector' or 'sifive_vector'";
     return;
   }
 
@@ -4052,5 +4056,8 @@ void PragmaRISCVHandler::HandlePragma(Preprocessor &PP,
     return;
   }
 
-  Actions.DeclareRISCVVBuiltins = true;
+  if (IntrinsicClass == "vector")
+    Actions.DeclareRISCVVBuiltins = true;
+  else if (IntrinsicClass == "sifive_vector")
+    Actions.DeclareRISCVVectorBuiltins = true;
 }
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index c02ad90a805b1..b423c5c47051f 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -19,6 +19,7 @@
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/STLExtras.h"
@@ -1052,7 +1053,7 @@ void Parser::ParseCompoundStatementLeadingPragmas() {
 
 void Parser::DiagnoseLabelAtEndOfCompoundStatement() {
   if (getLangOpts().CPlusPlus) {
-    Diag(Tok, getLangOpts().CPlusPlus2b
+    Diag(Tok, getLangOpts().CPlusPlus23
                   ? diag::warn_cxx20_compat_label_end_of_compound_statement
                   : diag::ext_cxx_label_end_of_compound_statement);
   } else {
@@ -1456,7 +1457,7 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) {
     }
 
     if (Tok.is(tok::kw_consteval)) {
-      Diag(Tok, getLangOpts().CPlusPlus2b ? diag::warn_cxx20_compat_consteval_if
+      Diag(Tok, getLangOpts().CPlusPlus23 ? diag::warn_cxx20_compat_consteval_if
                                           : diag::ext_consteval_if);
       IsConsteval = true;
       ConstevalLoc = ConsumeToken();
@@ -1928,7 +1929,7 @@ bool Parser::isForRangeIdentifier() {
 /// [C++] for-init-statement:
 /// [C++]   expression-statement
 /// [C++]   simple-declaration
-/// [C++2b] alias-declaration
+/// [C++23] alias-declaration
 ///
 /// [C++0x] for-range-declaration:
 /// [C++0x]   attribute-specifier-seq[opt] type-specifier-seq declarator
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index f4c45c86b0397..72df117c123e6 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -17,6 +17,7 @@
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaDiagnostic.h"
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index b26faaff7f9f3..934087e59b809 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -88,10 +88,8 @@ bool Parser::isCXXDeclarationStatement(
       }
       case tok::kw_operator:
         return true;
-      case tok::annot_cxxscope: // Check if this is a dtor.
-        if (NextToken().is(tok::tilde))
-          return true;
-        break;
+      case tok::tilde:
+        return true;
       default:
         break;
       }
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 7c138d489d1d4..07ece4078474e 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -33,6 +33,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/CXXFieldCollector.h"
 #include "clang/Sema/DelayedDiagnostic.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ExternalSemaSource.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/MultiplexExternalSemaSource.h"
@@ -2168,7 +2169,7 @@ static void checkEscapingByref(VarDecl *VD, Sema &S) {
       new (S.Context) DeclRefExpr(S.Context, VD, false, T, VK_LValue, Loc);
   ExprResult Result;
   auto IE = InitializedEntity::InitializeBlock(Loc, T);
-  if (S.getLangOpts().CPlusPlus2b) {
+  if (S.getLangOpts().CPlusPlus23) {
     auto *E = ImplicitCastExpr::Create(S.Context, T, CK_NoOp, VarRef, nullptr,
                                        VK_XValue, FPOptionsOverride());
     Result = S.PerformCopyInitialization(IE, SourceLocation(), E);
diff --git a/clang/lib/Sema/SemaAccess.cpp b/clang/lib/Sema/SemaAccess.cpp
index cbda62497e6a6..4a39c2d065e60 100644
--- a/clang/lib/Sema/SemaAccess.cpp
+++ b/clang/lib/Sema/SemaAccess.cpp
@@ -1651,7 +1651,8 @@ Sema::AccessResult Sema::CheckConstructorAccess(SourceLocation UseLoc,
        << Entity.getBaseSpecifier()->getType() << getSpecialMember(Constructor);
     break;
 
-  case InitializedEntity::EK_Member: {
+  case InitializedEntity::EK_Member:
+  case InitializedEntity::EK_ParenAggInitMember: {
     const FieldDecl *Field = cast<FieldDecl>(Entity.getDecl());
     PD = PDiag(diag::err_access_field_ctor);
     PD << Field->getType() << getSpecialMember(Constructor);
diff --git a/clang/lib/Sema/SemaCXXScopeSpec.cpp b/clang/lib/Sema/SemaCXXScopeSpec.cpp
index 759ed6f352c81..b11f4f1b7cdd9 100644
--- a/clang/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/clang/lib/Sema/SemaCXXScopeSpec.cpp
@@ -131,7 +131,8 @@ DeclContext *Sema::computeDeclContext(const CXXScopeSpec &SS,
             // entering the context, and that can't happen in a SFINAE context.
             assert(!isSFINAEContext() && "partial specialization scope "
                                          "specifier in SFINAE context?");
-            if (!hasReachableDefinition(PartialSpec))
+            if (PartialSpec->hasDefinition() &&
+                !hasReachableDefinition(PartialSpec))
               diagnoseMissingImport(SS.getLastQualifierNameLoc(), PartialSpec,
                                     MissingImportKind::PartialSpecialization,
                                     true);
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index cd71288476345..a411abd5b2a41 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -2354,6 +2354,12 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
       return TC_Success;
     }
 
+    // Allow bitcasting between SVE VLATs and VLSTs, and vice-versa.
+    if (Self.isValidRVVBitcast(SrcType, DestType)) {
+      Kind = CK_BitCast;
+      return TC_Success;
+    }
+
     // The non-vector type, if any, must have integral type.  This is
     // the same rule that C vector casts use; note, however, that enum
     // types are not integral in C++.
@@ -2941,6 +2947,13 @@ void CastOperation::CheckCStyleCast() {
     return;
   }
 
+  // Allow bitcasting between compatible RVV vector types.
+  if ((SrcType->isVectorType() || DestType->isVectorType()) &&
+      Self.isValidRVVBitcast(SrcType, DestType)) {
+    Kind = CK_BitCast;
+    return;
+  }
+
   if (!DestType->isScalarType() && !DestType->isVectorType() &&
       !DestType->isMatrixType()) {
     const RecordType *DestRecordTy = DestType->getAs<RecordType>();
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index f66eb9fcf13dc..63c5b25b27262 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -4641,6 +4641,127 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI,
         (VecInfo.EC.getKnownMinValue() * VecInfo.NumVectors);
     return SemaBuiltinConstantArgRange(TheCall, 1, 0, MaxIndex - 1);
   }
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8mf8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8mf4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8mf2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u8m8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u16mf4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u16mf2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u16m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u16m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u16m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u16m8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u32mf2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u32m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u32m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u32m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u32m8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u64m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u64m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u64m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_i_se_u64m8:
+    // bit_27_26, bit_24_20, bit_11_7, simm5
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 1, 0, 31) ||
+           SemaBuiltinConstantArgRange(TheCall, 2, 0, 31) ||
+           SemaBuiltinConstantArgRange(TheCall, 3, -16, 15);
+  case RISCVVector::BI__builtin_rvv_sf_vc_iv_se:
+    // bit_27_26, bit_11_7, vs2, simm5
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 1, 0, 31) ||
+           SemaBuiltinConstantArgRange(TheCall, 3, -16, 15);
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_i:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_i_se:
+    // bit_27_26, bit_24_20, simm5
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 1, 0, 31) ||
+           SemaBuiltinConstantArgRange(TheCall, 2, -16, 15);
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_iv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_iv_se:
+    // bit_27_26, vs2, simm5
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 2, -16, 15);
+  case RISCVVector::BI__builtin_rvv_sf_vc_ivv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_ivw_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_ivv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_ivw:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_ivv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_ivw_se:
+    // bit_27_26, vd, vs2, simm5
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 3, -16, 15);
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8mf8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8mf4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8mf2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u8m8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u16mf4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u16mf2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u16m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u16m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u16m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u16m8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u32mf2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u32m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u32m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u32m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u32m8:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u64m1:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u64m2:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u64m4:
+  case RISCVVector::BI__builtin_rvv_sf_vc_x_se_u64m8:
+    // bit_27_26, bit_24_20, bit_11_7, xs1
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 1, 0, 31) ||
+           SemaBuiltinConstantArgRange(TheCall, 2, 0, 31);
+  case RISCVVector::BI__builtin_rvv_sf_vc_xv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_vv_se:
+    // bit_27_26, bit_11_7, vs2, xs1/vs1
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_x:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_x_se:
+    // bit_27_26, bit_24-20, xs1
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3) ||
+           SemaBuiltinConstantArgRange(TheCall, 1, 0, 31);
+  case RISCVVector::BI__builtin_rvv_sf_vc_vvv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_xvv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_vvw_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_xvw_se:
+    // bit_27_26, vd, vs2, xs1
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_xv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_vv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_xv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_vv_se:
+    // bit_27_26, vs2, xs1/vs1
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_xvv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_vvv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_xvw:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_vvw:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_xvv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_vvv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_xvw_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_vvw_se:
+    // bit_27_26, vd, vs2, xs1/vs1
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 3);
+  case RISCVVector::BI__builtin_rvv_sf_vc_fv_se:
+    // bit_26, bit_11_7, vs2, fs1
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 1) ||
+           SemaBuiltinConstantArgRange(TheCall, 1, 0, 31);
+  case RISCVVector::BI__builtin_rvv_sf_vc_fvv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_fvw_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_fvv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_fvw:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_fvv_se:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_fvw_se:
+    // bit_26, vd, vs2, fs1
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_fv:
+  case RISCVVector::BI__builtin_rvv_sf_vc_v_fv_se:
+    // bit_26, vs2, fs1
+    return SemaBuiltinConstantArgRange(TheCall, 0, 0, 1);
   // Check if byteselect is in [0, 3]
   case RISCV::BI__builtin_riscv_aes32dsi_32:
   case RISCV::BI__builtin_riscv_aes32dsmi_32:
@@ -4652,6 +4773,65 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI,
   // Check if rnum is in [0, 10]
   case RISCV::BI__builtin_riscv_aes64ks1i_64:
     return SemaBuiltinConstantArgRange(TheCall, 1, 0, 10);
+  case RISCV::BI__builtin_riscv_ntl_load:
+  case RISCV::BI__builtin_riscv_ntl_store:
+    DeclRefExpr *DRE =
+        cast<DeclRefExpr>(TheCall->getCallee()->IgnoreParenCasts());
+    assert((BuiltinID == RISCV::BI__builtin_riscv_ntl_store ||
+            BuiltinID == RISCV::BI__builtin_riscv_ntl_load) &&
+           "Unexpected RISC-V nontemporal load/store builtin!");
+    bool IsStore = BuiltinID == RISCV::BI__builtin_riscv_ntl_store;
+    unsigned NumArgs = IsStore ? 3 : 2;
+
+    if (checkArgCount(*this, TheCall, NumArgs))
+      return true;
+
+    // Domain value should be compile-time constant.
+    // 2 <= domain <= 5
+    if (SemaBuiltinConstantArgRange(TheCall, NumArgs - 1, 2, 5))
+      return true;
+
+    Expr *PointerArg = TheCall->getArg(0);
+    ExprResult PointerArgResult =
+        DefaultFunctionArrayLvalueConversion(PointerArg);
+
+    if (PointerArgResult.isInvalid())
+      return true;
+    PointerArg = PointerArgResult.get();
+
+    const PointerType *PtrType = PointerArg->getType()->getAs<PointerType>();
+    if (!PtrType) {
+      Diag(DRE->getBeginLoc(), diag::err_nontemporal_builtin_must_be_pointer)
+          << PointerArg->getType() << PointerArg->getSourceRange();
+      return true;
+    }
+
+    QualType ValType = PtrType->getPointeeType();
+    ValType = ValType.getUnqualifiedType();
+    if (!ValType->isIntegerType() && !ValType->isAnyPointerType() &&
+        !ValType->isBlockPointerType() && !ValType->isFloatingType() &&
+        !ValType->isVectorType() && !ValType->isRVVType()) {
+      Diag(DRE->getBeginLoc(),
+           diag::err_nontemporal_builtin_must_be_pointer_intfltptr_or_vector)
+          << PointerArg->getType() << PointerArg->getSourceRange();
+      return true;
+    }
+
+    if (!IsStore) {
+      TheCall->setType(ValType);
+      return false;
+    }
+
+    ExprResult ValArg = TheCall->getArg(1);
+    InitializedEntity Entity = InitializedEntity::InitializeParameter(
+        Context, ValType, /*consume*/ false);
+    ValArg = PerformCopyInitialization(Entity, SourceLocation(), ValArg);
+    if (ValArg.isInvalid())
+      return true;
+
+    TheCall->setArg(1, ValArg.get());
+    TheCall->setType(Context.VoidTy);
+    return false;
   }
 
   return false;
@@ -13994,6 +14174,13 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
                                             QualType(Source, 0))))
       return;
 
+    if (Target->isRVVVLSBuiltinType() &&
+        (S.Context.areCompatibleRVVTypes(QualType(Target, 0),
+                                         QualType(Source, 0)) ||
+         S.Context.areLaxCompatibleRVVTypes(QualType(Target, 0),
+                                            QualType(Source, 0))))
+      return;
+
     if (!isa<VectorType>(Target)) {
       if (S.SourceMgr.isInSystemMacro(CC))
         return;
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 328d66bf33afa..7aa06b615fec2 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -10,19 +10,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "TreeTransform.h"
 #include "clang/Sema/SemaConcept.h"
-#include "clang/Sema/Sema.h"
-#include "clang/Sema/SemaInternal.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include "clang/Sema/TemplateDeduction.h"
-#include "clang/Sema/Template.h"
-#include "clang/Sema/Overload.h"
-#include "clang/Sema/Initialization.h"
+#include "TreeTransform.h"
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/OperatorPrecedence.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Overload.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaDiagnostic.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Template.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringExtras.h"
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 719215d3f38c4..4cdf2982b99d5 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -5033,7 +5033,8 @@ void Sema::setTagNameForLinkagePurposes(TagDecl *TagFromDeclSpec,
   TagFromDeclSpec->setTypedefNameForAnonDecl(NewTD);
 }
 
-static unsigned GetDiagnosticTypeSpecifierID(DeclSpec::TST T) {
+static unsigned GetDiagnosticTypeSpecifierID(const DeclSpec &DS) {
+  DeclSpec::TST T = DS.getTypeSpecType();
   switch (T) {
   case DeclSpec::TST_class:
     return 0;
@@ -5044,12 +5045,17 @@ static unsigned GetDiagnosticTypeSpecifierID(DeclSpec::TST T) {
   case DeclSpec::TST_union:
     return 3;
   case DeclSpec::TST_enum:
+    if (const auto *ED = dyn_cast<EnumDecl>(DS.getRepAsDecl())) {
+      if (ED->isScopedUsingClassTag())
+        return 5;
+      if (ED->isScoped())
+        return 6;
+    }
     return 4;
   default:
     llvm_unreachable("unexpected type specifier");
   }
 }
-
 /// ParsedFreeStandingDeclSpec - This method is invoked when a declspec with
 /// no declarator (e.g. "struct foo;") is parsed. It also accepts template
 /// parameters to cope with template friend declarations.
@@ -5107,7 +5113,7 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     // the declaration of a function or function template
     if (Tag)
       Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_tag)
-          << GetDiagnosticTypeSpecifierID(DS.getTypeSpecType())
+          << GetDiagnosticTypeSpecifierID(DS)
           << static_cast<int>(DS.getConstexprSpecifier());
     else
       Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_wrong_decl_kind)
@@ -5141,7 +5147,7 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     //
     // Per C++ [dcl.enum]p1, an opaque-enum-declaration can't either.
     Diag(SS.getBeginLoc(), diag::err_standalone_class_nested_name_specifier)
-        << GetDiagnosticTypeSpecifierID(DS.getTypeSpecType()) << SS.getRange();
+        << GetDiagnosticTypeSpecifierID(DS) << SS.getRange();
     return nullptr;
   }
 
@@ -5306,10 +5312,10 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
         TypeSpecType == DeclSpec::TST_enum) {
       for (const ParsedAttr &AL : DS.getAttributes())
         Diag(AL.getLoc(), diag::warn_declspec_attribute_ignored)
-            << AL << GetDiagnosticTypeSpecifierID(TypeSpecType);
+            << AL << GetDiagnosticTypeSpecifierID(DS);
       for (const ParsedAttr &AL : DeclAttrs)
         Diag(AL.getLoc(), diag::warn_declspec_attribute_ignored)
-            << AL << GetDiagnosticTypeSpecifierID(TypeSpecType);
+            << AL << GetDiagnosticTypeSpecifierID(DS);
     }
   }
 
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 27cfde2ed2161..3cb3250f67318 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -447,7 +447,7 @@ static bool threadSafetyCheckIsSmartPointer(Sema &S, const RecordType* RT) {
   if (!CXXRecord)
     return false;
 
-  for (auto BaseSpecifier : CXXRecord->bases()) {
+  for (const auto &BaseSpecifier : CXXRecord->bases()) {
     if (!foundStarOperator)
       foundStarOperator = IsOverloadedOperatorPresent(
           BaseSpecifier.getType()->getAsRecordDecl(), OO_Star);
@@ -8591,13 +8591,7 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
 
   // Ignore C++11 attributes on declarator chunks: they appertain to the type
   // instead.
-  // FIXME: We currently check the attribute syntax directly instead of using
-  // isCXX11Attribute(), which currently erroneously classifies the C11
-  // `_Alignas` attribute as a C++11 attribute. `_Alignas` can appear on the
-  // `DeclSpec`, so we need to let it through here to make sure it is processed
-  // appropriately. Once the behavior of isCXX11Attribute() is fixed, we can
-  // go back to using that here.
-  if (AL.getSyntax() == ParsedAttr::AS_CXX11 && !Options.IncludeCXX11Attributes)
+  if (AL.isCXX11Attribute() && !Options.IncludeCXX11Attributes)
     return;
 
   // Unknown attributes are automatically warned on. Target-specific attributes
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 70238a92d0a9e..dcb3e676b0ba4 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -34,6 +34,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/CXXFieldCollector.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
@@ -41,9 +42,9 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include <map>
 #include <optional>
@@ -761,7 +762,7 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D,
   // C++20 [dcl.dcl]/8:
   //   If decl-specifier-seq contains any decl-specifier other than static,
   //   thread_local, auto, or cv-qualifiers, the program is ill-formed.
-  // C++2b [dcl.pre]/6:
+  // C++23 [dcl.pre]/6:
   //   Each decl-specifier in the decl-specifier-seq shall be static,
   //   thread_local, auto (9.2.9.6 [dcl.spec.auto]), or a cv-qualifier.
   auto &DS = D.getDeclSpec();
@@ -1927,16 +1928,16 @@ static bool CheckConstexprDeclStmt(Sema &SemaRef, const FunctionDecl *Dcl,
         if (VD->isStaticLocal()) {
           if (Kind == Sema::CheckConstexprKind::Diagnose) {
             SemaRef.Diag(VD->getLocation(),
-                         SemaRef.getLangOpts().CPlusPlus2b
+                         SemaRef.getLangOpts().CPlusPlus23
                              ? diag::warn_cxx20_compat_constexpr_var
                              : diag::ext_constexpr_static_var)
                 << isa<CXXConstructorDecl>(Dcl)
                 << (VD->getTLSKind() == VarDecl::TLS_Dynamic);
-          } else if (!SemaRef.getLangOpts().CPlusPlus2b) {
+          } else if (!SemaRef.getLangOpts().CPlusPlus23) {
             return false;
           }
         }
-        if (SemaRef.LangOpts.CPlusPlus2b) {
+        if (SemaRef.LangOpts.CPlusPlus23) {
           CheckLiteralType(SemaRef, Kind, VD->getLocation(), VD->getType(),
                            diag::warn_cxx20_compat_constexpr_var,
                            isa<CXXConstructorDecl>(Dcl),
@@ -2275,15 +2276,15 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl,
   if (Kind == Sema::CheckConstexprKind::CheckValid) {
     // If this is only valid as an extension, report that we don't satisfy the
     // constraints of the current language.
-    if ((Cxx2bLoc.isValid() && !SemaRef.getLangOpts().CPlusPlus2b) ||
+    if ((Cxx2bLoc.isValid() && !SemaRef.getLangOpts().CPlusPlus23) ||
         (Cxx2aLoc.isValid() && !SemaRef.getLangOpts().CPlusPlus20) ||
         (Cxx1yLoc.isValid() && !SemaRef.getLangOpts().CPlusPlus17))
       return false;
   } else if (Cxx2bLoc.isValid()) {
     SemaRef.Diag(Cxx2bLoc,
-                 SemaRef.getLangOpts().CPlusPlus2b
+                 SemaRef.getLangOpts().CPlusPlus23
                      ? diag::warn_cxx20_compat_constexpr_body_invalid_stmt
-                     : diag::ext_constexpr_body_invalid_stmt_cxx2b)
+                     : diag::ext_constexpr_body_invalid_stmt_cxx23)
         << isa<CXXConstructorDecl>(Dcl);
   } else if (Cxx2aLoc.isValid()) {
     SemaRef.Diag(Cxx2aLoc,
@@ -8812,12 +8813,25 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
   //   the requirements for a constexpr function [...]
   // The only relevant requirements are that the parameter and return types are
   // literal types. The remaining conditions are checked by the analyzer.
+  //
+  // We support P2448R2 in language modes earlier than C++23 as an extension.
+  // The concept of constexpr-compatible was removed.
+  // C++23 [dcl.fct.def.default]p3 [P2448R2]
+  //  A function explicitly defaulted on its first declaration is implicitly
+  //  inline, and is implicitly constexpr if it is constexpr-suitable.
+  // C++23 [dcl.constexpr]p3
+  //   A function is constexpr-suitable if
+  //    - it is not a coroutine, and
+  //    - if the function is a constructor or destructor, its class does not
+  //      have any virtual base classes.
   if (FD->isConstexpr()) {
     if (CheckConstexprReturnType(*this, FD, CheckConstexprKind::Diagnose) &&
         CheckConstexprParameterTypes(*this, FD, CheckConstexprKind::Diagnose) &&
         !Info.Constexpr) {
       Diag(FD->getBeginLoc(),
-           diag::err_incorrect_defaulted_comparison_constexpr)
+           getLangOpts().CPlusPlus23
+               ? diag::warn_cxx23_compat_defaulted_comparison_constexpr_mismatch
+               : diag::ext_defaulted_comparison_constexpr_mismatch)
           << FD->isImplicit() << (int)DCK << FD->isConsteval();
       DefaultedComparisonAnalyzer(*this, RD, FD, DCK,
                                   DefaultedComparisonAnalyzer::ExplainConstexpr)
@@ -15975,7 +15989,7 @@ bool Sema::CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl) {
     if (MethodDecl->isStatic()) {
       if (Op == OO_Call || Op == OO_Subscript)
         Diag(FnDecl->getLocation(),
-             (LangOpts.CPlusPlus2b
+             (LangOpts.CPlusPlus23
                   ? diag::warn_cxx20_compat_operator_overload_static
                   : diag::ext_operator_overload_static))
             << FnDecl;
@@ -16016,7 +16030,7 @@ bool Sema::CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl) {
     }
     if (FirstDefaultedParam) {
       if (Op == OO_Subscript) {
-        Diag(FnDecl->getLocation(), LangOpts.CPlusPlus2b
+        Diag(FnDecl->getLocation(), LangOpts.CPlusPlus23
                                         ? diag::ext_subscript_overload
                                         : diag::error_subscript_overload)
             << FnDecl->getDeclName() << 1
@@ -16067,7 +16081,7 @@ bool Sema::CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl) {
   }
 
   if (Op == OO_Subscript && NumParams != 2) {
-    Diag(FnDecl->getLocation(), LangOpts.CPlusPlus2b
+    Diag(FnDecl->getLocation(), LangOpts.CPlusPlus23
                                     ? diag::ext_subscript_overload
                                     : diag::error_subscript_overload)
         << FnDecl->getDeclName() << (NumParams == 1 ? 0 : 2);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index fd64a79d164bb..622b36407495b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -41,6 +41,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/DelayedDiagnostic.h"
 #include "clang/Sema/Designator.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Overload.h"
@@ -3956,13 +3957,13 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
   } else {
     QualType Ty;
 
-    // 'z/uz' literals are a C++2b feature.
+    // 'z/uz' literals are a C++23 feature.
     if (Literal.isSizeT)
       Diag(Tok.getLocation(), getLangOpts().CPlusPlus
-                                  ? getLangOpts().CPlusPlus2b
+                                  ? getLangOpts().CPlusPlus23
                                         ? diag::warn_cxx20_compat_size_t_suffix
-                                        : diag::ext_cxx2b_size_t_suffix
-                                  : diag::err_cxx2b_size_t_suffix);
+                                        : diag::ext_cxx23_size_t_suffix
+                                  : diag::err_cxx23_size_t_suffix);
 
     // 'wb/uwb' literals are a C2x feature. We support _BitInt as a type in C++,
     // but we do not currently support the suffix in C++ mode because it's not
@@ -4042,7 +4043,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
         Ty = Context.getBitIntType(Literal.isUnsigned, Width);
       }
 
-      // Check C++2b size_t literals.
+      // Check C++23 size_t literals.
       if (Literal.isSizeT) {
         assert(!Literal.MicrosoftInteger &&
                "size_t literals can't be Microsoft literals");
@@ -4924,7 +4925,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
   // Build an unanalyzed expression if either operand is type-dependent.
   if (getLangOpts().CPlusPlus && ArgExprs.size() == 1 &&
       (base->isTypeDependent() ||
-       Expr::hasAnyTypeDependentArguments(ArgExprs))) {
+       Expr::hasAnyTypeDependentArguments(ArgExprs)) &&
+      !isa<PackExpansionExpr>(ArgExprs[0])) {
     return new (Context) ArraySubscriptExpr(
         base, ArgExprs.front(),
         getDependentArraySubscriptType(base, ArgExprs.front(), getASTContext()),
@@ -4958,7 +4960,8 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
   // to overload resolution and so should not take this path.
   if (getLangOpts().CPlusPlus && !base->getType()->isObjCObjectPointerType() &&
       ((base->getType()->isRecordType() ||
-        (ArgExprs.size() != 1 || ArgExprs[0]->getType()->isRecordType())))) {
+        (ArgExprs.size() != 1 || isa<PackExpansionExpr>(ArgExprs[0]) ||
+         ArgExprs[0]->getType()->isRecordType())))) {
     return CreateOverloadedArraySubscriptExpr(lbLoc, rbLoc, base, ArgExprs);
   }
 
@@ -5926,7 +5929,9 @@ bool Sema::CheckCXXDefaultArgExpr(SourceLocation CallLoc, FunctionDecl *FD,
       Param);
   ExprEvalContexts.back().IsCurrentlyCheckingDefaultArgumentOrInitializer =
       SkipImmediateInvocations;
-  MarkDeclarationsReferencedInExpr(Init, /*SkipLocalVariables*/ true);
+  runWithSufficientStackSpace(CallLoc, [&] {
+    MarkDeclarationsReferencedInExpr(Init, /*SkipLocalVariables=*/true);
+  });
   return false;
 }
 
@@ -6036,8 +6041,11 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
       ExprEvalContexts.back().DelayedDefaultInitializationContext = {
           CallLoc, Param, CurContext};
       EnsureImmediateInvocationInDefaultArgs Immediate(*this);
-      ExprResult Res = Immediate.TransformInitializer(Param->getInit(),
-                                                      /*NotCopy=*/false);
+      ExprResult Res;
+      runWithSufficientStackSpace(CallLoc, [&] {
+        Res = Immediate.TransformInitializer(Param->getInit(),
+                                             /*NotCopy=*/false);
+      });
       if (Res.isInvalid())
         return ExprError();
       Res = ConvertParamDefaultArgument(Param, Res.get(),
@@ -6117,10 +6125,11 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
         NestedDefaultChecking;
 
     EnsureImmediateInvocationInDefaultArgs Immediate(*this);
-
-    ExprResult Res =
-        Immediate.TransformInitializer(Field->getInClassInitializer(),
-                                       /*CXXDirectInit=*/false);
+    ExprResult Res;
+    runWithSufficientStackSpace(Loc, [&] {
+      Res = Immediate.TransformInitializer(Field->getInClassInitializer(),
+                                           /*CXXDirectInit=*/false);
+    });
     if (!Res.isInvalid())
       Res = ConvertMemberDefaultInitExpression(Field, Res.get(), Loc);
     if (Res.isInvalid()) {
@@ -6133,7 +6142,9 @@ ExprResult Sema::BuildCXXDefaultInitExpr(SourceLocation Loc, FieldDecl *Field) {
   if (Field->getInClassInitializer()) {
     Expr *E = Init ? Init : Field->getInClassInitializer();
     if (!NestedDefaultChecking)
-      MarkDeclarationsReferencedInExpr(E, /*SkipLocalVariables=*/false);
+      runWithSufficientStackSpace(Loc, [&] {
+        MarkDeclarationsReferencedInExpr(E, /*SkipLocalVariables=*/false);
+      });
     // C++11 [class.base.init]p7:
     //   The initialization of each base and member constitutes a
     //   full-expression.
@@ -7975,6 +7986,28 @@ bool Sema::isValidSveBitcast(QualType srcTy, QualType destTy) {
          ValidScalableConversion(destTy, srcTy);
 }
 
+/// Are the two types RVV-bitcast-compatible types? I.e. is bitcasting from the
+/// first RVV type (e.g. an RVV scalable type) to the second type (e.g. an RVV
+/// VLS type) allowed?
+///
+/// This will also return false if the two given types do not make sense from
+/// the perspective of RVV bitcasts.
+bool Sema::isValidRVVBitcast(QualType srcTy, QualType destTy) {
+  assert(srcTy->isVectorType() || destTy->isVectorType());
+
+  auto ValidScalableConversion = [](QualType FirstType, QualType SecondType) {
+    if (!FirstType->isRVVSizelessBuiltinType())
+      return false;
+
+    const auto *VecTy = SecondType->getAs<VectorType>();
+    return VecTy &&
+           VecTy->getVectorKind() == VectorType::RVVFixedLengthDataVector;
+  };
+
+  return ValidScalableConversion(srcTy, destTy) ||
+         ValidScalableConversion(destTy, srcTy);
+}
+
 /// Are the two types matrix types and do they have the same dimensions i.e.
 /// do they have the same number of rows and the same number of columns?
 bool Sema::areMatrixTypesOfTheSameDimension(QualType srcTy, QualType destTy) {
@@ -9907,6 +9940,16 @@ Sema::CheckAssignmentConstraints(QualType LHSType, ExprResult &RHS,
         return Compatible;
       }
 
+    // Allow assignments between fixed-length and sizeless RVV vectors.
+    if ((LHSType->isRVVSizelessBuiltinType() && RHSType->isVectorType()) ||
+        (LHSType->isVectorType() && RHSType->isRVVSizelessBuiltinType())) {
+      if (Context.areCompatibleRVVTypes(LHSType, RHSType) ||
+          Context.areLaxCompatibleRVVTypes(LHSType, RHSType)) {
+        Kind = CK_BitCast;
+        return Compatible;
+      }
+    }
+
     return Incompatible;
   }
 
@@ -10076,6 +10119,15 @@ Sema::CheckAssignmentConstraints(QualType LHSType, ExprResult &RHS,
     return Incompatible;
   }
 
+  // Conversion to nullptr_t (C2x only)
+  if (getLangOpts().C2x && LHSType->isNullPtrType() &&
+      RHS.get()->isNullPointerConstant(Context,
+                                       Expr::NPC_ValueDependentIsNull)) {
+    // null -> nullptr_t
+    Kind = CK_NullToPointer;
+    return Compatible;
+  }
+
   // Conversions from pointers that are not covered by the above.
   if (isa<PointerType>(RHSType)) {
     // T* -> _Bool
@@ -10293,12 +10345,13 @@ Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &CallerRHS,
   QualType LHSTypeAfterConversion = LHSType.getAtomicUnqualifiedType();
 
   // C99 6.5.16.1p1: the left operand is a pointer and the right is
-  // a null pointer constant.
+  // a null pointer constant <C2x>or its type is nullptr_t;</C2x>.
   if ((LHSTypeAfterConversion->isPointerType() ||
        LHSTypeAfterConversion->isObjCObjectPointerType() ||
        LHSTypeAfterConversion->isBlockPointerType()) &&
-      RHS.get()->isNullPointerConstant(Context,
-                                       Expr::NPC_ValueDependentIsNull)) {
+      ((getLangOpts().C2x && RHS.get()->getType()->isNullPtrType()) ||
+       RHS.get()->isNullPointerConstant(Context,
+                                        Expr::NPC_ValueDependentIsNull))) {
     if (Diagnose || ConvertRHS) {
       CastKind Kind;
       CXXCastPath Path;
@@ -10309,6 +10362,26 @@ Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &CallerRHS,
     }
     return Compatible;
   }
+  // C2x 6.5.16.1p1: the left operand has type atomic, qualified, or
+  // unqualified bool, and the right operand is a pointer or its type is
+  // nullptr_t.
+  if (getLangOpts().C2x && LHSType->isBooleanType() &&
+      RHS.get()->getType()->isNullPtrType()) {
+    // NB: T* -> _Bool is handled in CheckAssignmentConstraints, this only
+    // only handles nullptr -> _Bool due to needing an extra conversion
+    // step.
+    // We model this by converting from nullptr -> void * and then let the
+    // conversion from void * -> _Bool happen naturally.
+    if (Diagnose || ConvertRHS) {
+      CastKind Kind;
+      CXXCastPath Path;
+      CheckPointerConversion(RHS.get(), Context.VoidPtrTy, Kind, Path,
+                             /*IgnoreBaseAccess=*/false, Diagnose);
+      if (ConvertRHS)
+        RHS = ImpCastExprToType(RHS.get(), Context.VoidPtrTy, Kind, VK_PRValue,
+                                &Path);
+    }
+  }
 
   // OpenCL queue_t type assignment.
   if (LHSType->isQueueT() && RHS.get()->isNullPointerConstant(
@@ -10768,18 +10841,30 @@ QualType Sema::CheckVectorOperands(ExprResult &LHS, ExprResult &RHS,
     }
   }
 
-  // Expressions containing fixed-length and sizeless SVE vectors are invalid
-  // since the ambiguity can affect the ABI.
-  auto IsSveConversion = [](QualType FirstType, QualType SecondType) {
+  // Expressions containing fixed-length and sizeless SVE/RVV vectors are
+  // invalid since the ambiguity can affect the ABI.
+  auto IsSveRVVConversion = [](QualType FirstType, QualType SecondType,
+                               unsigned &SVEorRVV) {
     const VectorType *VecType = SecondType->getAs<VectorType>();
-    return FirstType->isSizelessBuiltinType() && VecType &&
-           (VecType->getVectorKind() == VectorType::SveFixedLengthDataVector ||
-            VecType->getVectorKind() ==
-                VectorType::SveFixedLengthPredicateVector);
+    SVEorRVV = 0;
+    if (FirstType->isSizelessBuiltinType() && VecType) {
+      if (VecType->getVectorKind() == VectorType::SveFixedLengthDataVector ||
+          VecType->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
+        return true;
+      if (VecType->getVectorKind() == VectorType::RVVFixedLengthDataVector) {
+        SVEorRVV = 1;
+        return true;
+      }
+    }
+
+    return false;
   };
 
-  if (IsSveConversion(LHSType, RHSType) || IsSveConversion(RHSType, LHSType)) {
-    Diag(Loc, diag::err_typecheck_sve_ambiguous) << LHSType << RHSType;
+  unsigned SVEorRVV;
+  if (IsSveRVVConversion(LHSType, RHSType, SVEorRVV) ||
+      IsSveRVVConversion(RHSType, LHSType, SVEorRVV)) {
+    Diag(Loc, diag::err_typecheck_sve_rvv_ambiguous)
+        << SVEorRVV << LHSType << RHSType;
     return QualType();
   }
 
@@ -10791,12 +10876,21 @@ QualType Sema::CheckVectorOperands(ExprResult &LHS, ExprResult &RHS,
     const VectorType *SecondVecType = SecondType->getAs<VectorType>();
 
     SVEorRVV = 0;
-    if (FirstVecType && SecondVecType)
-      return FirstVecType->getVectorKind() == VectorType::GenericVector &&
-             (SecondVecType->getVectorKind() ==
-                  VectorType::SveFixedLengthDataVector ||
-              SecondVecType->getVectorKind() ==
-                  VectorType::SveFixedLengthPredicateVector);
+    if (FirstVecType && SecondVecType) {
+      if (FirstVecType->getVectorKind() == VectorType::GenericVector) {
+        if (SecondVecType->getVectorKind() ==
+                VectorType::SveFixedLengthDataVector ||
+            SecondVecType->getVectorKind() ==
+                VectorType::SveFixedLengthPredicateVector)
+          return true;
+        if (SecondVecType->getVectorKind() ==
+            VectorType::RVVFixedLengthDataVector) {
+          SVEorRVV = 1;
+          return true;
+        }
+      }
+      return false;
+    }
 
     if (SecondVecType &&
         SecondVecType->getVectorKind() == VectorType::GenericVector) {
@@ -10811,7 +10905,6 @@ QualType Sema::CheckVectorOperands(ExprResult &LHS, ExprResult &RHS,
     return false;
   };
 
-  unsigned SVEorRVV;
   if (IsSveRVVGnuConversion(LHSType, RHSType, SVEorRVV) ||
       IsSveRVVGnuConversion(RHSType, LHSType, SVEorRVV)) {
     Diag(Loc, diag::err_typecheck_sve_rvv_gnu_ambiguous)
@@ -18493,7 +18586,9 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func,
   if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(Func)) {
     for (CXXCtorInitializer *Init : Constructor->inits()) {
       if (Init->isInClassMemberInitializer())
-        MarkDeclarationsReferencedInExpr(Init->getInit());
+        runWithSufficientStackSpace(Init->getSourceLocation(), [&]() {
+          MarkDeclarationsReferencedInExpr(Init->getInit());
+        });
     }
   }
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 69595c455b47a..2bf5e6a473e30 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -31,6 +31,7 @@
 #include "clang/Basic/TypeTraits.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
@@ -1466,10 +1467,10 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
           : InitializationKind::CreateValue(TyBeginLoc, LParenOrBraceLoc,
                                             RParenOrBraceLoc);
 
-  // C++1z [expr.type.conv]p1:
+  // C++17 [expr.type.conv]p1:
   //   If the type is a placeholder for a deduced class type, [...perform class
   //   template argument deduction...]
-  // C++2b:
+  // C++23:
   //   Otherwise, if the type contains a placeholder type, it is replaced by the
   //   type determined by placeholder type deduction.
   DeducedType *Deduced = Ty->getContainedDeducedType();
@@ -1496,7 +1497,7 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
                             diag::err_auto_expr_init_multiple_expressions)
                        << Ty << FullRange);
     }
-    if (getLangOpts().CPlusPlus2b) {
+    if (getLangOpts().CPlusPlus23) {
       if (Ty->getAs<AutoType>())
         Diag(TyBeginLoc, diag::warn_cxx20_compat_auto_expr) << FullRange;
     }
@@ -4021,7 +4022,7 @@ ExprResult Sema::CheckCXXBooleanCondition(Expr *CondExpr, bool IsConstexpr) {
   // The value of a condition that is an expression is the value of the
   // expression, implicitly converted to bool.
   //
-  // C++2b 8.5.2p2
+  // C++23 8.5.2p2
   // If the if statement is of the form if constexpr, the value of the condition
   // is contextually converted to bool and the converted expression shall be
   // a constant expression.
@@ -4565,6 +4566,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     break;
 
   case ICK_SVE_Vector_Conversion:
+  case ICK_RVV_Vector_Conversion:
     From = ImpCastExprToType(From, ToType, CK_BitCast, VK_PRValue,
                              /*BasePath=*/nullptr, CCK)
                .get();
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 018ba5ed1bce4..d27cd6b64c62a 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -20,6 +20,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Designator.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/SemaInternal.h"
@@ -1186,6 +1187,7 @@ static void warnBracedScalarInit(Sema &S, const InitializedEntity &Entity,
   case InitializedEntity::EK_LambdaToBlockConversionBlockElement:
   case InitializedEntity::EK_Binding:
   case InitializedEntity::EK_StmtExprResult:
+  case InitializedEntity::EK_ParenAggInitMember:
     llvm_unreachable("unexpected braced scalar init");
   }
 
@@ -3359,6 +3361,7 @@ DeclarationName InitializedEntity::getName() const {
 
   case EK_Variable:
   case EK_Member:
+  case EK_ParenAggInitMember:
   case EK_Binding:
   case EK_TemplateParameter:
     return Variable.VariableOrMember->getDeclName();
@@ -3390,6 +3393,7 @@ ValueDecl *InitializedEntity::getDecl() const {
   switch (getKind()) {
   case EK_Variable:
   case EK_Member:
+  case EK_ParenAggInitMember:
   case EK_Binding:
   case EK_TemplateParameter:
     return Variable.VariableOrMember;
@@ -3431,6 +3435,7 @@ bool InitializedEntity::allowsNRVO() const {
   case EK_Parameter_CF_Audited:
   case EK_TemplateParameter:
   case EK_Member:
+  case EK_ParenAggInitMember:
   case EK_Binding:
   case EK_New:
   case EK_Temporary:
@@ -3465,7 +3470,10 @@ unsigned InitializedEntity::dumpImpl(raw_ostream &OS) const {
   case EK_Result: OS << "Result"; break;
   case EK_StmtExprResult: OS << "StmtExprResult"; break;
   case EK_Exception: OS << "Exception"; break;
-  case EK_Member: OS << "Member"; break;
+  case EK_Member:
+  case EK_ParenAggInitMember:
+    OS << "Member";
+    break;
   case EK_Binding: OS << "Binding"; break;
   case EK_New: OS << "New"; break;
   case EK_Temporary: OS << "Temporary"; break;
@@ -5303,176 +5311,224 @@ static void TryOrBuildParenListInitialization(
     Sema &S, const InitializedEntity &Entity, const InitializationKind &Kind,
     ArrayRef<Expr *> Args, InitializationSequence &Sequence, bool VerifyOnly,
     ExprResult *Result = nullptr) {
-  unsigned ArgIndexToProcess = 0;
+  unsigned EntityIndexToProcess = 0;
   SmallVector<Expr *, 4> InitExprs;
   QualType ResultType;
   Expr *ArrayFiller = nullptr;
   FieldDecl *InitializedFieldInUnion = nullptr;
 
-  // Process entities (i.e. array members, base classes, or class fields) by
-  // adding an initialization expression to InitExprs for each entity to
-  // initialize.
-  auto ProcessEntities = [&](auto Range) -> bool {
-    bool IsUnionType = Entity.getType()->isUnionType();
-    for (InitializedEntity SubEntity : Range) {
-      // Unions should only have one initializer expression.
-      // If there are more initializers than it will be caught when we check
-      // whether Index equals Args.size().
-      if (ArgIndexToProcess == 1 && IsUnionType)
-        return true;
-
-      bool IsMember = SubEntity.getKind() == InitializedEntity::EK_Member;
-
-      // Unnamed bitfields should not be initialized at all, either with an arg
-      // or by default.
-      if (IsMember && cast<FieldDecl>(SubEntity.getDecl())->isUnnamedBitfield())
-        continue;
-
-      if (ArgIndexToProcess < Args.size()) {
-        // There are still expressions in Args that haven't been processed.
-        // Let's match them to the current entity to initialize.
-        Expr *E = Args[ArgIndexToProcess++];
-
-        // Incomplete array types indicate flexible array members. Do not allow
-        // paren list initializations of structs with these members, as GCC
-        // doesn't either.
-        if (IsMember) {
-          auto *FD = cast<FieldDecl>(SubEntity.getDecl());
-          if (FD->getType()->isIncompleteArrayType()) {
-            if (!VerifyOnly) {
-              S.Diag(E->getBeginLoc(), diag::err_flexible_array_init)
-                  << SourceRange(E->getBeginLoc(), E->getEndLoc());
-              S.Diag(FD->getLocation(), diag::note_flexible_array_member) << FD;
-            }
-            Sequence.SetFailed(
-                InitializationSequence::FK_ParenthesizedListInitFailed);
-            return false;
-          }
-        }
-
-        InitializationKind SubKind = InitializationKind::CreateForInit(
-            E->getExprLoc(), /*isDirectInit=*/false, E);
-        InitializationSequence SubSeq(S, SubEntity, SubKind, E);
+  auto HandleInitializedEntity = [&](const InitializedEntity &SubEntity,
+                                     const InitializationKind &SubKind,
+                                     Expr *Arg, Expr **InitExpr = nullptr) {
+    InitializationSequence IS = [&]() {
+      if (Arg)
+        return InitializationSequence(S, SubEntity, SubKind, Arg);
+      return InitializationSequence(S, SubEntity, SubKind, std::nullopt);
+    }();
 
-        if (SubSeq.Failed()) {
-          if (!VerifyOnly)
-            SubSeq.Diagnose(S, SubEntity, SubKind, E);
-          else
-            Sequence.SetFailed(
-                InitializationSequence::FK_ParenthesizedListInitFailed);
-
-          return false;
-        }
-        if (!VerifyOnly) {
-          ExprResult ER = SubSeq.Perform(S, SubEntity, SubKind, E);
-          InitExprs.push_back(ER.get());
-          if (IsMember && IsUnionType)
-            InitializedFieldInUnion = cast<FieldDecl>(SubEntity.getDecl());
-        }
+    if (IS.Failed()) {
+      if (!VerifyOnly) {
+        if (Arg)
+          IS.Diagnose(S, SubEntity, SubKind, Arg);
+        else
+          IS.Diagnose(S, SubEntity, SubKind, std::nullopt);
       } else {
-        // We've processed all of the args, but there are still entities that
-        // have to be initialized.
-        if (IsMember) {
-          // C++ [dcl.init]p17.6.2.2
-          //   The remaining elements are initialized with their default member
-          //   initializers, if any
-          auto *FD = cast<FieldDecl>(SubEntity.getDecl());
-          if (Expr *ICE = FD->getInClassInitializer(); ICE && !VerifyOnly) {
-            ExprResult DIE = S.BuildCXXDefaultInitExpr(FD->getLocation(), FD);
-            if (DIE.isInvalid())
-              return false;
-            S.checkInitializerLifetime(SubEntity, DIE.get());
-            InitExprs.push_back(DIE.get());
-            continue;
-          };
-        }
-        // Remaining class elements without default member initializers and
-        // array elements are value initialized:
-        //
-        // C++ [dcl.init]p17.6.2.2
-        //   The remaining elements...otherwise are value initialzed
-        //
-        // C++ [dcl.init]p17.5
-        //   if the destination type is an array, the object is initialized as
-        // . follows. Let x1, . . . , xk be the elements of the expression-list
-        //   ...Let n denote the array size...the ith array element is...value-
-        //   initialized for each k < i <= n.
-        InitializationKind SubKind = InitializationKind::CreateValue(
-            Kind.getLocation(), Kind.getLocation(), Kind.getLocation(), true);
-        InitializationSequence SubSeq(S, SubEntity, SubKind, std::nullopt);
-        if (SubSeq.Failed()) {
-          if (!VerifyOnly)
-            SubSeq.Diagnose(S, SubEntity, SubKind, std::nullopt);
-          return false;
-        }
-        if (!VerifyOnly) {
-          ExprResult ER = SubSeq.Perform(S, SubEntity, SubKind, std::nullopt);
-          if (SubEntity.getKind() == InitializedEntity::EK_ArrayElement) {
-            ArrayFiller = ER.get();
-            return true;
-          }
-          InitExprs.push_back(ER.get());
-        }
+        Sequence.SetFailed(
+            InitializationSequence::FK_ParenthesizedListInitFailed);
       }
+
+      return false;
+    }
+    if (!VerifyOnly) {
+      ExprResult ER;
+      if (Arg)
+        ER = IS.Perform(S, SubEntity, SubKind, Arg);
+      else
+        ER = IS.Perform(S, SubEntity, SubKind, std::nullopt);
+      if (InitExpr)
+        *InitExpr = ER.get();
+      else
+        InitExprs.push_back(ER.get());
     }
     return true;
   };
 
   if (const ArrayType *AT =
           S.getASTContext().getAsArrayType(Entity.getType())) {
-
     SmallVector<InitializedEntity, 4> ElementEntities;
     uint64_t ArrayLength;
-    // C++ [dcl.init]p17.5
+    // C++ [dcl.init]p16.5
     //   if the destination type is an array, the object is initialized as
     //   follows. Let x1, . . . , xk be the elements of the expression-list. If
-    //   the destination type is an array of unknown bound, it is define as
+    //   the destination type is an array of unknown bound, it is defined as
     //   having k elements.
     if (const ConstantArrayType *CAT =
-            S.getASTContext().getAsConstantArrayType(Entity.getType()))
+            S.getASTContext().getAsConstantArrayType(Entity.getType())) {
       ArrayLength = CAT->getSize().getZExtValue();
-    else
+      ResultType = Entity.getType();
+    } else if (const VariableArrayType *VAT =
+                   S.getASTContext().getAsVariableArrayType(Entity.getType())) {
+      // Braced-initialization of variable array types is not allowed, even if
+      // the size is greater than or equal to the number of args, so we don't
+      // allow them to be initialized via parenthesized aggregate initialization
+      // either.
+      const Expr *SE = VAT->getSizeExpr();
+      S.Diag(SE->getBeginLoc(), diag::err_variable_object_no_init)
+          << SE->getSourceRange();
+      return;
+    } else {
+      assert(isa<IncompleteArrayType>(Entity.getType()));
       ArrayLength = Args.size();
+    }
+    EntityIndexToProcess = ArrayLength;
 
-    if (ArrayLength >= Args.size()) {
-      for (uint64_t I = 0; I < ArrayLength; ++I)
-        ElementEntities.push_back(
-            InitializedEntity::InitializeElement(S.getASTContext(), I, Entity));
-
-      if (!ProcessEntities(ElementEntities))
+    //   ...the ith array element is copy-initialized with xi for each
+    //   1 <= i <= k
+    for (Expr *E : Args) {
+      InitializedEntity SubEntity = InitializedEntity::InitializeElement(
+          S.getASTContext(), EntityIndexToProcess, Entity);
+      InitializationKind SubKind = InitializationKind::CreateForInit(
+          E->getExprLoc(), /*isDirectInit=*/false, E);
+      if (!HandleInitializedEntity(SubEntity, SubKind, E))
         return;
+    }
+    //   ...and value-initialized for each k < i <= n;
+    if (ArrayLength > Args.size()) {
+      InitializedEntity SubEntity = InitializedEntity::InitializeElement(
+          S.getASTContext(), Args.size(), Entity);
+      InitializationKind SubKind = InitializationKind::CreateValue(
+          Kind.getLocation(), Kind.getLocation(), Kind.getLocation(), true);
+      if (!HandleInitializedEntity(SubEntity, SubKind, nullptr, &ArrayFiller))
+        return;
+    }
 
+    if (ResultType.isNull()) {
       ResultType = S.Context.getConstantArrayType(
           AT->getElementType(), llvm::APInt(/*numBits=*/32, ArrayLength),
-          nullptr, ArrayType::Normal, 0);
+          /*SizeExpr=*/nullptr, ArrayType::Normal, 0);
     }
   } else if (auto *RT = Entity.getType()->getAs<RecordType>()) {
+    bool IsUnion = RT->isUnionType();
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
 
-    auto BaseRange = map_range(RD->bases(), [&S](auto &base) {
-      return InitializedEntity::InitializeBase(S.getASTContext(), &base, false);
-    });
-    auto FieldRange = map_range(RD->fields(), [](auto *field) {
-      return InitializedEntity::InitializeMember(field);
-    });
+    if (!IsUnion) {
+      for (const CXXBaseSpecifier &Base : RD->bases()) {
+        InitializedEntity SubEntity = InitializedEntity::InitializeBase(
+            S.getASTContext(), &Base, false, &Entity);
+        if (EntityIndexToProcess < Args.size()) {
+          // C++ [dcl.init]p16.6.2.2.
+          //   ...the object is initialized is follows. Let e1, ..., en be the
+          //   elements of the aggregate([dcl.init.aggr]). Let x1, ..., xk be
+          //   the elements of the expression-list...The element ei is
+          //   copy-initialized with xi for 1 <= i <= k.
+          Expr *E = Args[EntityIndexToProcess];
+          InitializationKind SubKind = InitializationKind::CreateForInit(
+              E->getExprLoc(), /*isDirectInit=*/false, E);
+          if (!HandleInitializedEntity(SubEntity, SubKind, E))
+            return;
+        } else {
+          // We've processed all of the args, but there are still base classes
+          // that have to be initialized.
+          // C++ [dcl.init]p17.6.2.2
+          //   The remaining elements...otherwise are value initialzed
+          InitializationKind SubKind = InitializationKind::CreateValue(
+              Kind.getLocation(), Kind.getLocation(), Kind.getLocation(),
+              /*IsImplicit=*/true);
+          if (!HandleInitializedEntity(SubEntity, SubKind, nullptr))
+            return;
+        }
+        EntityIndexToProcess++;
+      }
+    }
+
+    for (FieldDecl *FD : RD->fields()) {
+      // Unnamed bitfields should not be initialized at all, either with an arg
+      // or by default.
+      if (FD->isUnnamedBitfield())
+        continue;
 
-    if (!ProcessEntities(BaseRange))
-      return;
+      InitializedEntity SubEntity =
+          InitializedEntity::InitializeMemberFromParenAggInit(FD);
 
-    if (!ProcessEntities(FieldRange))
-      return;
+      if (EntityIndexToProcess < Args.size()) {
+        //   ...The element ei is copy-initialized with xi for 1 <= i <= k.
+        Expr *E = Args[EntityIndexToProcess];
+
+        // Incomplete array types indicate flexible array members. Do not allow
+        // paren list initializations of structs with these members, as GCC
+        // doesn't either.
+        if (FD->getType()->isIncompleteArrayType()) {
+          if (!VerifyOnly) {
+            S.Diag(E->getBeginLoc(), diag::err_flexible_array_init)
+                << SourceRange(E->getBeginLoc(), E->getEndLoc());
+            S.Diag(FD->getLocation(), diag::note_flexible_array_member) << FD;
+          }
+          Sequence.SetFailed(
+              InitializationSequence::FK_ParenthesizedListInitFailed);
+          return;
+        }
+
+        InitializationKind SubKind = InitializationKind::CreateForInit(
+            E->getExprLoc(), /*isDirectInit=*/false, E);
+        if (!HandleInitializedEntity(SubEntity, SubKind, E))
+          return;
 
+        // Unions should have only one initializer expression, so we bail out
+        // after processing the first field. If there are more initializers then
+        // it will be caught when we later check whether EntityIndexToProcess is
+        // less than Args.size();
+        if (IsUnion) {
+          InitializedFieldInUnion = FD;
+          EntityIndexToProcess = 1;
+          break;
+        }
+      } else {
+        // We've processed all of the args, but there are still members that
+        // have to be initialized.
+        if (FD->hasInClassInitializer()) {
+          if (!VerifyOnly) {
+            // C++ [dcl.init]p16.6.2.2
+            //   The remaining elements are initialized with their default
+            //   member initializers, if any
+            ExprResult DIE = S.BuildCXXDefaultInitExpr(FD->getLocation(), FD);
+            if (DIE.isInvalid())
+              return;
+            S.checkInitializerLifetime(SubEntity, DIE.get());
+            InitExprs.push_back(DIE.get());
+          }
+        } else {
+          // C++ [dcl.init]p17.6.2.2
+          //   The remaining elements...otherwise are value initialzed
+          if (FD->getType()->isReferenceType()) {
+            Sequence.SetFailed(
+                InitializationSequence::FK_ParenthesizedListInitFailed);
+            if (!VerifyOnly) {
+              SourceRange SR = Kind.getParenOrBraceRange();
+              S.Diag(SR.getEnd(), diag::err_init_reference_member_uninitialized)
+                  << FD->getType() << SR;
+              S.Diag(FD->getLocation(), diag::note_uninit_reference_member);
+            }
+            return;
+          }
+          InitializationKind SubKind = InitializationKind::CreateValue(
+              Kind.getLocation(), Kind.getLocation(), Kind.getLocation(), true);
+          if (!HandleInitializedEntity(SubEntity, SubKind, nullptr))
+            return;
+        }
+      }
+      EntityIndexToProcess++;
+    }
     ResultType = Entity.getType();
   }
 
   // Not all of the args have been processed, so there must've been more args
   // than were required to initialize the element.
-  if (ArgIndexToProcess < Args.size()) {
+  if (EntityIndexToProcess < Args.size()) {
     Sequence.SetFailed(InitializationSequence::FK_ParenthesizedListInitFailed);
     if (!VerifyOnly) {
       QualType T = Entity.getType();
       int InitKind = T->isArrayType() ? 0 : T->isUnionType() ? 3 : 4;
-      SourceRange ExcessInitSR(Args[ArgIndexToProcess]->getBeginLoc(),
+      SourceRange ExcessInitSR(Args[EntityIndexToProcess]->getBeginLoc(),
                                Args.back()->getEndLoc());
       S.Diag(Kind.getLocation(), diag::err_excess_initializers)
           << InitKind << ExcessInitSR;
@@ -6438,6 +6494,7 @@ getAssignmentAction(const InitializedEntity &Entity, bool Diagnose = false) {
     return Sema::AA_Converting;
 
   case InitializedEntity::EK_Member:
+  case InitializedEntity::EK_ParenAggInitMember:
   case InitializedEntity::EK_Binding:
   case InitializedEntity::EK_ArrayElement:
   case InitializedEntity::EK_VectorElement:
@@ -6458,6 +6515,7 @@ static bool shouldBindAsTemporary(const InitializedEntity &Entity) {
   switch (Entity.getKind()) {
   case InitializedEntity::EK_ArrayElement:
   case InitializedEntity::EK_Member:
+  case InitializedEntity::EK_ParenAggInitMember:
   case InitializedEntity::EK_Result:
   case InitializedEntity::EK_StmtExprResult:
   case InitializedEntity::EK_New:
@@ -6502,6 +6560,7 @@ static bool shouldDestroyEntity(const InitializedEntity &Entity) {
       return false;
 
     case InitializedEntity::EK_Member:
+    case InitializedEntity::EK_ParenAggInitMember:
     case InitializedEntity::EK_Binding:
     case InitializedEntity::EK_Variable:
     case InitializedEntity::EK_Parameter:
@@ -6538,6 +6597,7 @@ static SourceLocation getInitializationLoc(const InitializedEntity &Entity,
 
   case InitializedEntity::EK_ArrayElement:
   case InitializedEntity::EK_Member:
+  case InitializedEntity::EK_ParenAggInitMember:
   case InitializedEntity::EK_Parameter:
   case InitializedEntity::EK_Parameter_CF_Audited:
   case InitializedEntity::EK_TemplateParameter:
@@ -6885,7 +6945,7 @@ PerformConstructorInitialization(Sema &S,
 
   if (isExplicitTemporary(Entity, Kind, NumArgs)) {
     // An explicitly-constructed temporary, e.g., X(1, 2).
-    if (S.DiagnoseUseOfDecl(Constructor, Loc))
+    if (S.DiagnoseUseOfDecl(Step.Function.FoundDecl, Loc))
       return ExprError();
 
     TypeSourceInfo *TSInfo = Entity.getTypeSourceInfo();
@@ -6900,8 +6960,6 @@ PerformConstructorInitialization(Sema &S,
     if (auto *Shadow = dyn_cast<ConstructorUsingShadowDecl>(
             Step.Function.FoundDecl.getDecl())) {
       CalleeDecl = S.findInheritingConstructor(Loc, Constructor, Shadow);
-      if (S.DiagnoseUseOfDecl(CalleeDecl, Loc))
-        return ExprError();
     }
     S.MarkFunctionReferenced(Loc, CalleeDecl);
 
@@ -7106,7 +7164,15 @@ static LifetimeResult getEntityLifetime(
   case InitializedEntity::EK_Exception:
     // FIXME: Can we diagnose lifetime problems with exceptions?
     return {nullptr, LK_FullExpression};
+
+  case InitializedEntity::EK_ParenAggInitMember:
+    //   -- A temporary object bound to a reference element of an aggregate of
+    //      class type initialized from a parenthesized expression-list
+    //      [dcl.init, 9.3] persists until the completion of the full-expression
+    //      containing the expression-list.
+    return {nullptr, LK_FullExpression};
   }
+
   llvm_unreachable("unknown entity kind");
 }
 
@@ -9222,7 +9288,9 @@ ExprResult InitializationSequence::Perform(Sema &S,
     S.checkInitializerLifetime(Entity, Init);
 
   // Diagnose non-fatal problems with the completed initialization.
-  if (Entity.getKind() == InitializedEntity::EK_Member &&
+  if (InitializedEntity::EntityKind EK = Entity.getKind();
+      (EK == InitializedEntity::EK_Member ||
+       EK == InitializedEntity::EK_ParenAggInitMember) &&
       cast<FieldDecl>(Entity.getDecl())->isBitField())
     S.CheckBitFieldInitialization(Kind.getLocation(),
                                   cast<FieldDecl>(Entity.getDecl()),
@@ -9676,7 +9744,8 @@ bool InitializationSequence::Diagnose(Sema &S,
       case OR_No_Viable_Function:
         if (Kind.getKind() == InitializationKind::IK_Default &&
             (Entity.getKind() == InitializedEntity::EK_Base ||
-             Entity.getKind() == InitializedEntity::EK_Member) &&
+             Entity.getKind() == InitializedEntity::EK_Member ||
+             Entity.getKind() == InitializedEntity::EK_ParenAggInitMember) &&
             isa<CXXConstructorDecl>(S.CurContext)) {
           // This is implicit default initialization of a member or
           // base within a constructor. If no viable function was
@@ -10608,7 +10677,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
 
     // Make sure we didn't select an unusable deduction guide, and mark it
     // as referenced.
-    DiagnoseUseOfDecl(Best->Function, Kind.getLocation());
+    DiagnoseUseOfDecl(Best->FoundDecl, Kind.getLocation());
     MarkFunctionReferenced(Kind.getLocation(), Best->Function);
     break;
   }
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 7f9bec2f7627e..730f70e732579 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -1365,6 +1365,26 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
     PushOnScopeChains(P, CurScope);
   }
 
+  // C++23 [expr.prim.lambda.capture]p5:
+  // If an identifier in a capture appears as the declarator-id of a parameter
+  // of the lambda-declarator's parameter-declaration-clause or as the name of a
+  // template parameter of the lambda-expression's template-parameter-list, the
+  // program is ill-formed.
+  TemplateParameterList *TemplateParams =
+      getGenericLambdaTemplateParameterList(LSI, *this);
+  if (TemplateParams) {
+    for (const auto *TP : TemplateParams->asArray()) {
+      if (!TP->getIdentifier())
+        continue;
+      for (const auto &Capture : Intro.Captures) {
+        if (Capture.Id == TP->getIdentifier()) {
+          Diag(Capture.Loc, diag::err_template_param_shadow) << Capture.Id;
+          Diag(TP->getLocation(), diag::note_template_param_here);
+        }
+      }
+    }
+  }
+
   // C++20: dcl.decl.general p4:
   // The optional requires-clause ([temp.pre]) in an init-declarator or
   // member-declarator shall be present only if the declarator declares a
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 767dfeeb3cd7b..e82c17cfe630f 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -933,7 +933,7 @@ bool Sema::LookupBuiltin(LookupResult &R) {
         }
       }
 
-      if (DeclareRISCVVBuiltins) {
+      if (DeclareRISCVVBuiltins || DeclareRISCVVectorBuiltins) {
         if (!RVIntrinsicManager)
           RVIntrinsicManager = CreateRISCVIntrinsicManager(*this);
 
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 84a1fd854d804..53e453b117e17 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -255,7 +255,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
     }
   }
 
-  // C++2b [module.unit]p1: ... The identifiers module and import shall not
+  // C++23 [module.unit]p1: ... The identifiers module and import shall not
   // appear as identifiers in a module-name or module-partition. All
   // module-names either beginning with an identifier consisting of std
   // followed by zero or more digits or containing a reserved identifier
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index be80e933ab1ac..4eb765a2220b5 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -27,6 +27,7 @@
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Scope.h"
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 5a8544e3739fe..29d6bd852ec67 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -26,6 +26,7 @@
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Overload.h"
@@ -141,6 +142,7 @@ ImplicitConversionRank clang::GetConversionRank(ImplicitConversionKind Kind) {
     ICR_Conversion,
     ICR_Conversion,
     ICR_Conversion,
+    ICR_Conversion,
     ICR_OCL_Scalar_Widening,
     ICR_Complex_Real_Conversion,
     ICR_Conversion,
@@ -183,6 +185,7 @@ static const char* GetImplicitConversionName(ImplicitConversionKind Kind) {
     "Derived-to-base conversion",
     "Vector conversion",
     "SVE Vector conversion",
+    "RVV Vector conversion",
     "Vector splat",
     "Complex-real conversion",
     "Block Pointer conversion",
@@ -1761,6 +1764,14 @@ static bool IsVectorConversion(Sema &S, QualType FromType, QualType ToType,
       return true;
     }
 
+  if (ToType->isRVVSizelessBuiltinType() ||
+      FromType->isRVVSizelessBuiltinType())
+    if (S.Context.areCompatibleRVVTypes(FromType, ToType) ||
+        S.Context.areLaxCompatibleRVVTypes(FromType, ToType)) {
+      ICK = ICK_RVV_Vector_Conversion;
+      return true;
+    }
+
   // We can perform the conversion between vector types in the following cases:
   // 1)vector types are equivalent AltiVec and GCC vector types
   // 2)lax vector conversions are permitted and the vector types are of the
@@ -4327,6 +4338,20 @@ CompareStandardConversionSequences(Sema &S, SourceLocation Loc,
                  : ImplicitConversionSequence::Worse;
   }
 
+  if (SCS1.Second == ICK_RVV_Vector_Conversion &&
+      SCS2.Second == ICK_RVV_Vector_Conversion) {
+    bool SCS1IsCompatibleRVVVectorConversion =
+        S.Context.areCompatibleRVVTypes(SCS1.getFromType(), SCS1.getToType(2));
+    bool SCS2IsCompatibleRVVVectorConversion =
+        S.Context.areCompatibleRVVTypes(SCS2.getFromType(), SCS2.getToType(2));
+
+    if (SCS1IsCompatibleRVVVectorConversion !=
+        SCS2IsCompatibleRVVVectorConversion)
+      return SCS1IsCompatibleRVVVectorConversion
+                 ? ImplicitConversionSequence::Better
+                 : ImplicitConversionSequence::Worse;
+  }
+
   return ImplicitConversionSequence::Indistinguishable;
 }
 
@@ -5765,6 +5790,7 @@ static bool CheckConvertedConstantConversions(Sema &S,
   case ICK_Derived_To_Base:
   case ICK_Vector_Conversion:
   case ICK_SVE_Vector_Conversion:
+  case ICK_RVV_Vector_Conversion:
   case ICK_Vector_Splat:
   case ICK_Complex_Real:
   case ICK_Block_Pointer_Conversion:
@@ -9903,7 +9929,7 @@ bool clang::isBetterOverloadCandidate(
     }
   }
 
-  // C++ [over.match.best]p1: (Changed in C++2b)
+  // C++ [over.match.best]p1: (Changed in C++23)
   //
   //   -- if F is a static member function, ICS1(F) is defined such
   //      that ICS1(F) is neither better nor worse than ICS1(G) for
@@ -15073,7 +15099,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   bool IsError = false;
 
   // Initialize the implicit object parameter if needed.
-  // Since C++2b, this could also be a call to a static call operator
+  // Since C++23, this could also be a call to a static call operator
   // which we emit as a regular CallExpr.
   if (Method->isInstance()) {
     ExprResult ObjRes = PerformObjectArgumentInitialization(
diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
index 3e98da5f8b8f4..981ab5f13716d 100644
--- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp
+++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
@@ -28,6 +28,8 @@ using namespace llvm;
 using namespace clang;
 using namespace clang::RISCV;
 
+using IntrinsicKind = sema::RISCVIntrinsicManager::IntrinsicKind;
+
 namespace {
 
 // Function definition of a RVV intrinsic.
@@ -58,16 +60,34 @@ static const PrototypeDescriptor RVVSignatureTable[] = {
 #undef DECL_SIGNATURE_TABLE
 };
 
+static const PrototypeDescriptor RVSiFiveVectorSignatureTable[] = {
+#define DECL_SIGNATURE_TABLE
+#include "clang/Basic/riscv_sifive_vector_builtin_sema.inc"
+#undef DECL_SIGNATURE_TABLE
+};
+
 static const RVVIntrinsicRecord RVVIntrinsicRecords[] = {
 #define DECL_INTRINSIC_RECORDS
 #include "clang/Basic/riscv_vector_builtin_sema.inc"
 #undef DECL_INTRINSIC_RECORDS
 };
 
+static const RVVIntrinsicRecord RVSiFiveVectorIntrinsicRecords[] = {
+#define DECL_INTRINSIC_RECORDS
+#include "clang/Basic/riscv_sifive_vector_builtin_sema.inc"
+#undef DECL_INTRINSIC_RECORDS
+};
+
 // Get subsequence of signature table.
-static ArrayRef<PrototypeDescriptor> ProtoSeq2ArrayRef(uint16_t Index,
-                                                       uint8_t Length) {
-  return ArrayRef(&RVVSignatureTable[Index], Length);
+static ArrayRef<PrototypeDescriptor>
+ProtoSeq2ArrayRef(IntrinsicKind K, uint16_t Index, uint8_t Length) {
+  switch (K) {
+  case IntrinsicKind::RVV:
+    return ArrayRef(&RVVSignatureTable[Index], Length);
+  case IntrinsicKind::SIFIVE_VECTOR:
+    return ArrayRef(&RVSiFiveVectorSignatureTable[Index], Length);
+  }
+  llvm_unreachable("Unhandled IntrinsicKind");
 }
 
 static QualType RVVType2Qual(ASTContext &Context, const RVVType *Type) {
@@ -172,123 +192,132 @@ void RISCVIntrinsicManagerImpl::InitIntrinsicList() {
   bool HasRV64 = TI.hasFeature("64bit");
   bool HasFullMultiply = TI.hasFeature("v");
 
-  // Construction of RVVIntrinsicRecords need to sync with createRVVIntrinsics
-  // in RISCVVEmitter.cpp.
-  for (auto &Record : RVVIntrinsicRecords) {
-    // Create Intrinsics for each type and LMUL.
-    BasicType BaseType = BasicType::Unknown;
-    ArrayRef<PrototypeDescriptor> BasicProtoSeq =
-        ProtoSeq2ArrayRef(Record.PrototypeIndex, Record.PrototypeLength);
-    ArrayRef<PrototypeDescriptor> SuffixProto =
-        ProtoSeq2ArrayRef(Record.SuffixIndex, Record.SuffixLength);
-    ArrayRef<PrototypeDescriptor> OverloadedSuffixProto = ProtoSeq2ArrayRef(
-        Record.OverloadedSuffixIndex, Record.OverloadedSuffixSize);
-
-    PolicyScheme UnMaskedPolicyScheme =
-        static_cast<PolicyScheme>(Record.UnMaskedPolicyScheme);
-    PolicyScheme MaskedPolicyScheme =
-        static_cast<PolicyScheme>(Record.MaskedPolicyScheme);
-
-    const Policy DefaultPolicy;
-
-    llvm::SmallVector<PrototypeDescriptor> ProtoSeq =
-        RVVIntrinsic::computeBuiltinTypes(BasicProtoSeq, /*IsMasked=*/false,
-                                          /*HasMaskedOffOperand=*/false,
-                                          Record.HasVL, Record.NF,
-                                          UnMaskedPolicyScheme, DefaultPolicy);
-
-    llvm::SmallVector<PrototypeDescriptor> ProtoMaskSeq =
-        RVVIntrinsic::computeBuiltinTypes(
-            BasicProtoSeq, /*IsMasked=*/true, Record.HasMaskedOffOperand,
-            Record.HasVL, Record.NF, MaskedPolicyScheme, DefaultPolicy);
-
-    bool UnMaskedHasPolicy = UnMaskedPolicyScheme != PolicyScheme::SchemeNone;
-    bool MaskedHasPolicy = MaskedPolicyScheme != PolicyScheme::SchemeNone;
-    SmallVector<Policy> SupportedUnMaskedPolicies =
-        RVVIntrinsic::getSupportedUnMaskedPolicies();
-    SmallVector<Policy> SupportedMaskedPolicies =
-        RVVIntrinsic::getSupportedMaskedPolicies(Record.HasTailPolicy,
-                                                 Record.HasMaskPolicy);
-
-    for (unsigned int TypeRangeMaskShift = 0;
-         TypeRangeMaskShift <= static_cast<unsigned int>(BasicType::MaxOffset);
-         ++TypeRangeMaskShift) {
-      unsigned int BaseTypeI = 1 << TypeRangeMaskShift;
-      BaseType = static_cast<BasicType>(BaseTypeI);
-
-      if ((BaseTypeI & Record.TypeRangeMask) != BaseTypeI)
-        continue;
-
-      // Check requirement.
-      if (((Record.RequiredExtensions & RVV_REQ_RV64) == RVV_REQ_RV64) &&
-          !HasRV64)
-        continue;
-
-      if ((BaseType == BasicType::Int64) &&
-          ((Record.RequiredExtensions & RVV_REQ_FullMultiply) ==
-           RVV_REQ_FullMultiply) &&
-          !HasFullMultiply)
-        continue;
-
-      // Expanded with different LMUL.
-      for (int Log2LMUL = -3; Log2LMUL <= 3; Log2LMUL++) {
-        if (!(Record.Log2LMULMask & (1 << (Log2LMUL + 3))))
+  auto ConstructRVVIntrinsics = [&](ArrayRef<RVVIntrinsicRecord> Recs,
+                                    IntrinsicKind K) {
+    // Construction of RVVIntrinsicRecords need to sync with createRVVIntrinsics
+    // in RISCVVEmitter.cpp.
+    for (auto &Record : Recs) {
+      // Create Intrinsics for each type and LMUL.
+      BasicType BaseType = BasicType::Unknown;
+      ArrayRef<PrototypeDescriptor> BasicProtoSeq =
+          ProtoSeq2ArrayRef(K, Record.PrototypeIndex, Record.PrototypeLength);
+      ArrayRef<PrototypeDescriptor> SuffixProto =
+          ProtoSeq2ArrayRef(K, Record.SuffixIndex, Record.SuffixLength);
+      ArrayRef<PrototypeDescriptor> OverloadedSuffixProto = ProtoSeq2ArrayRef(
+          K, Record.OverloadedSuffixIndex, Record.OverloadedSuffixSize);
+
+      PolicyScheme UnMaskedPolicyScheme =
+          static_cast<PolicyScheme>(Record.UnMaskedPolicyScheme);
+      PolicyScheme MaskedPolicyScheme =
+          static_cast<PolicyScheme>(Record.MaskedPolicyScheme);
+
+      const Policy DefaultPolicy;
+
+      llvm::SmallVector<PrototypeDescriptor> ProtoSeq =
+          RVVIntrinsic::computeBuiltinTypes(BasicProtoSeq, /*IsMasked=*/false,
+                                            /*HasMaskedOffOperand=*/false,
+                                            Record.HasVL, Record.NF,
+                                            UnMaskedPolicyScheme, DefaultPolicy);
+
+      llvm::SmallVector<PrototypeDescriptor> ProtoMaskSeq =
+          RVVIntrinsic::computeBuiltinTypes(
+              BasicProtoSeq, /*IsMasked=*/true, Record.HasMaskedOffOperand,
+              Record.HasVL, Record.NF, MaskedPolicyScheme, DefaultPolicy);
+
+      bool UnMaskedHasPolicy = UnMaskedPolicyScheme != PolicyScheme::SchemeNone;
+      bool MaskedHasPolicy = MaskedPolicyScheme != PolicyScheme::SchemeNone;
+      SmallVector<Policy> SupportedUnMaskedPolicies =
+          RVVIntrinsic::getSupportedUnMaskedPolicies();
+      SmallVector<Policy> SupportedMaskedPolicies =
+          RVVIntrinsic::getSupportedMaskedPolicies(Record.HasTailPolicy,
+                                                   Record.HasMaskPolicy);
+
+      for (unsigned int TypeRangeMaskShift = 0;
+           TypeRangeMaskShift <= static_cast<unsigned int>(BasicType::MaxOffset);
+           ++TypeRangeMaskShift) {
+        unsigned int BaseTypeI = 1 << TypeRangeMaskShift;
+        BaseType = static_cast<BasicType>(BaseTypeI);
+
+        if ((BaseTypeI & Record.TypeRangeMask) != BaseTypeI)
           continue;
 
-        std::optional<RVVTypes> Types =
-            TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoSeq);
-
-        // Ignored to create new intrinsic if there are any illegal types.
-        if (!Types.has_value())
+        // Check requirement.
+        if (((Record.RequiredExtensions & RVV_REQ_RV64) == RVV_REQ_RV64) &&
+            !HasRV64)
           continue;
 
-        std::string SuffixStr = RVVIntrinsic::getSuffixStr(
-            TypeCache, BaseType, Log2LMUL, SuffixProto);
-        std::string OverloadedSuffixStr = RVVIntrinsic::getSuffixStr(
-            TypeCache, BaseType, Log2LMUL, OverloadedSuffixProto);
-
-        // Create non-masked intrinsic.
-        InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, false, *Types,
-                         UnMaskedHasPolicy, DefaultPolicy);
+        if ((BaseType == BasicType::Int64) &&
+            ((Record.RequiredExtensions & RVV_REQ_FullMultiply) ==
+             RVV_REQ_FullMultiply) &&
+            !HasFullMultiply)
+          continue;
 
-        // Create non-masked policy intrinsic.
-        if (Record.UnMaskedPolicyScheme != PolicyScheme::SchemeNone) {
-          for (const auto &P : SupportedUnMaskedPolicies) {
+        // Expanded with different LMUL.
+        for (int Log2LMUL = -3; Log2LMUL <= 3; Log2LMUL++) {
+          if (!(Record.Log2LMULMask & (1 << (Log2LMUL + 3))))
+            continue;
+
+          std::optional<RVVTypes> Types =
+              TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoSeq);
+
+          // Ignored to create new intrinsic if there are any illegal types.
+          if (!Types.has_value())
+            continue;
+
+          std::string SuffixStr = RVVIntrinsic::getSuffixStr(
+              TypeCache, BaseType, Log2LMUL, SuffixProto);
+          std::string OverloadedSuffixStr = RVVIntrinsic::getSuffixStr(
+              TypeCache, BaseType, Log2LMUL, OverloadedSuffixProto);
+
+          // Create non-masked intrinsic.
+          InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, false, *Types,
+                           UnMaskedHasPolicy, DefaultPolicy);
+
+          // Create non-masked policy intrinsic.
+          if (Record.UnMaskedPolicyScheme != PolicyScheme::SchemeNone) {
+            for (auto P : SupportedUnMaskedPolicies) {
+              llvm::SmallVector<PrototypeDescriptor> PolicyPrototype =
+                  RVVIntrinsic::computeBuiltinTypes(
+                      BasicProtoSeq, /*IsMasked=*/false,
+                      /*HasMaskedOffOperand=*/false, Record.HasVL, Record.NF,
+                      UnMaskedPolicyScheme, P);
+              std::optional<RVVTypes> PolicyTypes = TypeCache.computeTypes(
+                  BaseType, Log2LMUL, Record.NF, PolicyPrototype);
+              InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr,
+                               /*IsMask=*/false, *PolicyTypes, UnMaskedHasPolicy,
+                               P);
+            }
+          }
+          if (!Record.HasMasked)
+            continue;
+          // Create masked intrinsic.
+          std::optional<RVVTypes> MaskTypes =
+              TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoMaskSeq);
+          InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, true,
+                           *MaskTypes, MaskedHasPolicy, DefaultPolicy);
+          if (Record.MaskedPolicyScheme == PolicyScheme::SchemeNone)
+            continue;
+          // Create masked policy intrinsic.
+          for (auto P : SupportedMaskedPolicies) {
             llvm::SmallVector<PrototypeDescriptor> PolicyPrototype =
                 RVVIntrinsic::computeBuiltinTypes(
-                    BasicProtoSeq, /*IsMasked=*/false,
-                    /*HasMaskedOffOperand=*/false, Record.HasVL, Record.NF,
-                    UnMaskedPolicyScheme, P);
+                    BasicProtoSeq, /*IsMasked=*/true, Record.HasMaskedOffOperand,
+                    Record.HasVL, Record.NF, MaskedPolicyScheme, P);
             std::optional<RVVTypes> PolicyTypes = TypeCache.computeTypes(
                 BaseType, Log2LMUL, Record.NF, PolicyPrototype);
             InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr,
-                             /*IsMask=*/false, *PolicyTypes, UnMaskedHasPolicy,
-                             P);
+                             /*IsMask=*/true, *PolicyTypes, MaskedHasPolicy, P);
           }
-        }
-        if (!Record.HasMasked)
-          continue;
-        // Create masked intrinsic.
-        std::optional<RVVTypes> MaskTypes =
-            TypeCache.computeTypes(BaseType, Log2LMUL, Record.NF, ProtoMaskSeq);
-        InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr, true,
-                         *MaskTypes, MaskedHasPolicy, DefaultPolicy);
-        if (Record.MaskedPolicyScheme == PolicyScheme::SchemeNone)
-          continue;
-        // Create masked policy intrinsic.
-        for (auto P : SupportedMaskedPolicies) {
-          llvm::SmallVector<PrototypeDescriptor> PolicyPrototype =
-              RVVIntrinsic::computeBuiltinTypes(
-                  BasicProtoSeq, /*IsMasked=*/true, Record.HasMaskedOffOperand,
-                  Record.HasVL, Record.NF, MaskedPolicyScheme, P);
-          std::optional<RVVTypes> PolicyTypes = TypeCache.computeTypes(
-              BaseType, Log2LMUL, Record.NF, PolicyPrototype);
-          InitRVVIntrinsic(Record, SuffixStr, OverloadedSuffixStr,
-                           /*IsMask=*/true, *PolicyTypes, MaskedHasPolicy, P);
-        }
-      } // End for different LMUL
-    } // End for different TypeRange
-  }
+        } // End for different LMUL
+      } // End for different TypeRange
+    }
+  };
+  if (S.DeclareRISCVVBuiltins)
+    ConstructRVVIntrinsics(RVVIntrinsicRecords,
+                           IntrinsicKind::RVV);
+  if (S.DeclareRISCVVectorBuiltins)
+    ConstructRVVIntrinsics(RVSiFiveVectorIntrinsicRecords,
+                           IntrinsicKind::SIFIVE_VECTOR);
 }
 
 // Compute name and signatures for intrinsic with practical types.
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 1af5f733edbfc..7daebbd914024 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -3364,7 +3364,7 @@ Sema::ActOnBreakStmt(SourceLocation BreakLoc, Scope *CurScope) {
 /// might be modified by the implementation.
 ///
 /// \param Mode Overrides detection of current language mode
-/// and uses the rules for C++2b.
+/// and uses the rules for C++23.
 ///
 /// \returns An aggregate which contains the Candidate and isMoveEligible
 /// and isCopyElidable methods. If Candidate is non-null, it means
@@ -3385,7 +3385,7 @@ Sema::NamedReturnInfo Sema::getNamedReturnInfo(Expr *&E,
   if (Res.Candidate && !E->isXValue() &&
       (Mode == SimplerImplicitMoveMode::ForceOn ||
        (Mode != SimplerImplicitMoveMode::ForceOff &&
-        getLangOpts().CPlusPlus2b))) {
+        getLangOpts().CPlusPlus23))) {
     E = ImplicitCastExpr::Create(Context, VD->getType().getNonReferenceType(),
                                  CK_NoOp, E, nullptr, VK_XValue,
                                  FPOptionsOverride());
@@ -3529,7 +3529,7 @@ ExprResult Sema::PerformMoveOrCopyInitialization(
     const InitializedEntity &Entity, const NamedReturnInfo &NRInfo, Expr *Value,
     bool SupressSimplerImplicitMoves) {
   if (getLangOpts().CPlusPlus &&
-      (!getLangOpts().CPlusPlus2b || SupressSimplerImplicitMoves) &&
+      (!getLangOpts().CPlusPlus23 || SupressSimplerImplicitMoves) &&
       NRInfo.isMoveEligible()) {
     ImplicitCastExpr AsRvalue(ImplicitCastExpr::OnStack, Value->getType(),
                               CK_NoOp, Value, VK_XValue, FPOptionsOverride());
@@ -3902,7 +3902,7 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
 
 static bool CheckSimplerImplicitMovesMSVCWorkaround(const Sema &S,
                                                     const Expr *E) {
-  if (!E || !S.getLangOpts().CPlusPlus2b || !S.getLangOpts().MSVCCompat)
+  if (!E || !S.getLangOpts().CPlusPlus23 || !S.getLangOpts().MSVCCompat)
     return false;
   const Decl *D = E->getReferencedDeclOfCallee();
   if (!D || !S.SourceMgr.isInSystemHeader(D->getLocation()))
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index d4f89cc1f0dfd..ca1b86c06097e 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -26,6 +26,7 @@
 #include "clang/Basic/Stack.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Overload.h"
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 2f13505d8ed71..b249d389bb677 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Sema/TemplateDeduction.h"
 #include "TreeTransform.h"
 #include "TypeLocBuilder.h"
 #include "clang/AST/ASTContext.h"
@@ -37,9 +36,11 @@
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/Template.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -3591,11 +3592,28 @@ Sema::TemplateDeductionResult Sema::FinishTemplateArgumentDeduction(
   DeclContext *Owner = FunctionTemplate->getDeclContext();
   if (FunctionTemplate->getFriendObjectKind())
     Owner = FunctionTemplate->getLexicalDeclContext();
+  FunctionDecl *FD = FunctionTemplate->getTemplatedDecl();
+  // additional check for inline friend, 
+  // ```
+  //   template <class F1> int foo(F1 X);
+  //   template <int A1> struct A {
+  //     template <class F1> friend int foo(F1 X) { return A1; }
+  //   };
+  //   template struct A<1>;
+  //   int a = foo(1.0);
+  // ```
+  const FunctionDecl *FDFriend;
+  if (FD->getFriendObjectKind() == Decl::FriendObjectKind::FOK_None &&
+      FD->isDefined(FDFriend, /*CheckForPendingFriendDefinition*/ true) &&
+      FDFriend->getFriendObjectKind() != Decl::FriendObjectKind::FOK_None) {
+    FD = const_cast<FunctionDecl *>(FDFriend);
+    Owner = FD->getLexicalDeclContext();
+  }
   MultiLevelTemplateArgumentList SubstArgs(
       FunctionTemplate, CanonicalDeducedArgumentList->asArray(),
       /*Final=*/false);
   Specialization = cast_or_null<FunctionDecl>(
-      SubstDecl(FunctionTemplate->getTemplatedDecl(), Owner, SubstArgs));
+      SubstDecl(FD, Owner, SubstArgs));
   if (!Specialization || Specialization->isInvalidDecl())
     return TDK_SubstitutionFailure;
 
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 526f19b8aedcf..1582a5c4b6e44 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -26,6 +26,7 @@
 #include "clang/Basic/Stack.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index e749c24e7f702..e74b2b2e868ee 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ScopeInfo.h"
@@ -4662,11 +4663,7 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
   ActiveInstType &ActiveInst = SemaRef.CodeSynthesisContexts.back();
   if (ActiveInst.Kind == ActiveInstType::ExplicitTemplateArgumentSubstitution ||
       ActiveInst.Kind == ActiveInstType::DeducedTemplateArgumentSubstitution) {
-    if (FunctionTemplateDecl *FunTmpl
-          = dyn_cast<FunctionTemplateDecl>(ActiveInst.Entity)) {
-      assert(FunTmpl->getTemplatedDecl() == Tmpl &&
-             "Deduction from the wrong function template?");
-      (void) FunTmpl;
+    if (isa<FunctionTemplateDecl>(ActiveInst.Entity)) {
       SemaRef.InstantiatingSpecializations.erase(
           {ActiveInst.Entity->getCanonicalDecl(), ActiveInst.Kind});
       atTemplateEnd(SemaRef.TemplateInstCallbacks, SemaRef, ActiveInst);
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index be44803947501..da105e3d6ad28 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -40,6 +40,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
 #include <bitset>
 #include <optional>
 
@@ -3644,7 +3645,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
     case DeclaratorContext::FunctionalCast:
       if (isa<DeducedTemplateSpecializationType>(Deduced))
         break;
-      if (SemaRef.getLangOpts().CPlusPlus2b && IsCXXAutoType &&
+      if (SemaRef.getLangOpts().CPlusPlus23 && IsCXXAutoType &&
           !Auto->isDecltypeAuto())
         break; // auto(x)
       [[fallthrough]];
@@ -4553,7 +4554,7 @@ static bool hasOuterPointerLikeChunk(const Declarator &D, unsigned endIndex) {
   return false;
 }
 
-static bool IsNoDerefableChunk(DeclaratorChunk Chunk) {
+static bool IsNoDerefableChunk(const DeclaratorChunk &Chunk) {
   return (Chunk.Kind == DeclaratorChunk::Pointer ||
           Chunk.Kind == DeclaratorChunk::Array);
 }
@@ -8285,6 +8286,67 @@ static void HandleArmMveStrictPolymorphismAttr(TypeProcessingState &State,
                               CurType, CurType);
 }
 
+/// HandleRISCVRVVVectorBitsTypeAttr - The "riscv_rvv_vector_bits" attribute is
+/// used to create fixed-length versions of sizeless RVV types such as
+/// vint8m1_t_t.
+static void HandleRISCVRVVVectorBitsTypeAttr(QualType &CurType,
+                                             ParsedAttr &Attr, Sema &S) {
+  // Target must have vector extension.
+  if (!S.Context.getTargetInfo().hasFeature("zve32x")) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_unsupported)
+        << Attr << "'zve32x'";
+    Attr.setInvalid();
+    return;
+  }
+
+  auto VScale = S.Context.getTargetInfo().getVScaleRange(S.getLangOpts());
+  if (!VScale || !VScale->first || VScale->first != VScale->second) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_riscv_rvv_bits_unsupported)
+        << Attr;
+    Attr.setInvalid();
+    return;
+  }
+
+  // Check the attribute arguments.
+  if (Attr.getNumArgs() != 1) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+        << Attr << 1;
+    Attr.setInvalid();
+    return;
+  }
+
+  // The vector size must be an integer constant expression.
+  llvm::APSInt RVVVectorSizeInBits(32);
+  if (!verifyValidIntegerConstantExpr(S, Attr, RVVVectorSizeInBits))
+    return;
+
+  unsigned VecSize = static_cast<unsigned>(RVVVectorSizeInBits.getZExtValue());
+
+  // The attribute vector size must match -mrvv-vector-bits.
+  // FIXME: Add support for types with LMUL!=1. Need to make sure size passed
+  // to attribute is equal to LMUL*VScaleMin*RVVBitsPerBlock.
+  if (VecSize != VScale->first * llvm::RISCV::RVVBitsPerBlock) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_bad_rvv_vector_size)
+        << VecSize << VScale->first * llvm::RISCV::RVVBitsPerBlock;
+    Attr.setInvalid();
+    return;
+  }
+
+  // Attribute can only be attached to a single RVV vector type.
+  if (!CurType->isRVVVLSBuiltinType()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_invalid_rvv_type)
+        << Attr << CurType;
+    Attr.setInvalid();
+    return;
+  }
+
+  QualType EltType = CurType->getRVVEltType(S.Context);
+  unsigned TypeSize = S.Context.getTypeSize(EltType);
+  VectorType::VectorKind VecKind = VectorType::RVVFixedLengthDataVector;
+  VecSize /= TypeSize;
+  CurType = S.Context.getVectorType(EltType, VecSize, VecKind);
+}
+
 /// Handle OpenCL Access Qualifier Attribute.
 static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
                                    Sema &S) {
@@ -8531,6 +8593,10 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       attr.setUsedAsTypeAttr();
       break;
     }
+    case ParsedAttr::AT_RISCVRVVVectorBits:
+      HandleRISCVRVVVectorBitsTypeAttr(type, attr, state.getSema());
+      attr.setUsedAsTypeAttr();
+      break;
     case ParsedAttr::AT_OpenCLAccess:
       HandleOpenCLAccessAttr(type, attr, state.getSema());
       attr.setUsedAsTypeAttr();
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 1d09450454262..d2b81fcacc6ae 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -19,8 +19,8 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprOpenMP.h"
 #include "clang/AST/OpenMPClause.h"
@@ -31,6 +31,7 @@
 #include "clang/Basic/DiagnosticParse.h"
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Sema/Designator.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/ParsedTemplate.h"
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 098ce5314ab17..6a7bc22d3b7b3 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -277,12 +277,17 @@ static bool checkLanguageOptions(const LangOptions &LangOpts,
                                  const LangOptions &ExistingLangOpts,
                                  DiagnosticsEngine *Diags,
                                  bool AllowCompatibleDifferences = true) {
-#define LANGOPT(Name, Bits, Default, Description)                 \
-  if (ExistingLangOpts.Name != LangOpts.Name) {                   \
-    if (Diags)                                                    \
-      Diags->Report(diag::err_pch_langopt_mismatch)               \
-        << Description << LangOpts.Name << ExistingLangOpts.Name; \
-    return true;                                                  \
+#define LANGOPT(Name, Bits, Default, Description)                   \
+  if (ExistingLangOpts.Name != LangOpts.Name) {                     \
+    if (Diags) {                                                    \
+      if (Bits == 1)                                                \
+        Diags->Report(diag::err_pch_langopt_mismatch)               \
+          << Description << LangOpts.Name << ExistingLangOpts.Name; \
+      else                                                          \
+        Diags->Report(diag::err_pch_langopt_value_mismatch)         \
+          << Description;                                           \
+    }                                                               \
+    return true;                                                    \
   }
 
 #define VALUE_LANGOPT(Name, Bits, Default, Description)   \
@@ -2523,7 +2528,8 @@ void ASTReader::ResolveImportedPath(ModuleFile &M, std::string &Filename) {
 }
 
 void ASTReader::ResolveImportedPath(std::string &Filename, StringRef Prefix) {
-  if (Filename.empty() || llvm::sys::path::is_absolute(Filename))
+  if (Filename.empty() || llvm::sys::path::is_absolute(Filename) ||
+      Filename == "<built-in>" || Filename == "<command line>")
     return;
 
   SmallString<128> Buffer;
diff --git a/clang/lib/Serialization/PCHContainerOperations.cpp b/clang/lib/Serialization/PCHContainerOperations.cpp
index d4990fce2d99d..56ca3394385b4 100644
--- a/clang/lib/Serialization/PCHContainerOperations.cpp
+++ b/clang/lib/Serialization/PCHContainerOperations.cpp
@@ -57,6 +57,11 @@ std::unique_ptr<ASTConsumer> RawPCHContainerWriter::CreatePCHContainerGenerator(
   return std::make_unique<RawPCHContainerGenerator>(std::move(OS), Buffer);
 }
 
+ArrayRef<llvm::StringRef> RawPCHContainerReader::getFormats() const {
+  static StringRef Raw("raw");
+  return ArrayRef(Raw);
+}
+
 StringRef
 RawPCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const {
   return Buffer.getBuffer();
diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
index 986b0add93dfb..269277aaf357f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
@@ -33,14 +33,19 @@ namespace {
 class ArrayBoundCheckerV2 :
     public Checker<check::Location> {
   mutable std::unique_ptr<BuiltinBug> BT;
+  mutable std::unique_ptr<BugType> TaintBT;
 
-  enum OOB_Kind { OOB_Precedes, OOB_Excedes, OOB_Tainted };
+  enum OOB_Kind { OOB_Precedes, OOB_Excedes };
 
-  void reportOOB(CheckerContext &C, ProgramStateRef errorState, OOB_Kind kind,
-                 std::unique_ptr<BugReporterVisitor> Visitor = nullptr) const;
+  void reportOOB(CheckerContext &C, ProgramStateRef errorState,
+                 OOB_Kind kind) const;
+  void reportTaintOOB(CheckerContext &C, ProgramStateRef errorState,
+                      SVal TaintedSVal) const;
+
+  static bool isFromCtypeMacro(const Stmt *S, ASTContext &AC);
 
 public:
-  void checkLocation(SVal l, bool isLoad, const Stmt*S,
+  void checkLocation(SVal l, bool isLoad, const Stmt *S,
                      CheckerContext &C) const;
 };
 
@@ -48,40 +53,29 @@ class ArrayBoundCheckerV2 :
 class RegionRawOffsetV2 {
 private:
   const SubRegion *baseRegion;
-  SVal byteOffset;
-
-  RegionRawOffsetV2()
-    : baseRegion(nullptr), byteOffset(UnknownVal()) {}
+  NonLoc byteOffset;
 
 public:
-  RegionRawOffsetV2(const SubRegion* base, SVal offset)
-    : baseRegion(base), byteOffset(offset) {}
+  RegionRawOffsetV2(const SubRegion *base, NonLoc offset)
+      : baseRegion(base), byteOffset(offset) { assert(base); }
 
-  NonLoc getByteOffset() const { return byteOffset.castAs<NonLoc>(); }
+  NonLoc getByteOffset() const { return byteOffset; }
   const SubRegion *getRegion() const { return baseRegion; }
 
-  static RegionRawOffsetV2 computeOffset(ProgramStateRef state,
-                                         SValBuilder &svalBuilder,
-                                         SVal location);
+  static std::optional<RegionRawOffsetV2>
+  computeOffset(ProgramStateRef State, SValBuilder &SVB, SVal Location);
 
   void dump() const;
   void dumpToStream(raw_ostream &os) const;
 };
 }
 
-static SVal computeExtentBegin(SValBuilder &svalBuilder,
-                               const MemRegion *region) {
-  const MemSpaceRegion *SR = region->getMemorySpace();
-  if (SR->getKind() == MemRegion::UnknownSpaceRegionKind)
-    return UnknownVal();
-  else
-    return svalBuilder.makeZeroArrayIndex();
-}
-
 // TODO: once the constraint manager is smart enough to handle non simplified
 // symbolic expressions remove this function. Note that this can not be used in
 // the constraint manager as is, since this does not handle overflows. It is
 // safe to assume, however, that memory offsets will not overflow.
+// NOTE: callers of this function need to be aware of the effects of overflows
+// and signed<->unsigned conversions!
 static std::pair<NonLoc, nonloc::ConcreteInt>
 getSimplifiedOffsets(NonLoc offset, nonloc::ConcreteInt extent,
                      SValBuilder &svalBuilder) {
@@ -114,6 +108,38 @@ getSimplifiedOffsets(NonLoc offset, nonloc::ConcreteInt extent,
   return std::pair<NonLoc, nonloc::ConcreteInt>(offset, extent);
 }
 
+// Evaluate the comparison Value < Threshold with the help of the custom
+// simplification algorithm defined for this checker. Return a pair of states,
+// where the first one corresponds to "value below threshold" and the second
+// corresponds to "value at or above threshold". Returns {nullptr, nullptr} in
+// the case when the evaluation fails.
+static std::pair<ProgramStateRef, ProgramStateRef>
+compareValueToThreshold(ProgramStateRef State, NonLoc Value, NonLoc Threshold,
+                        SValBuilder &SVB) {
+  if (auto ConcreteThreshold = Threshold.getAs<nonloc::ConcreteInt>()) {
+    std::tie(Value, Threshold) = getSimplifiedOffsets(Value, *ConcreteThreshold, SVB);
+  }
+  if (auto ConcreteThreshold = Threshold.getAs<nonloc::ConcreteInt>()) {
+    QualType T = Value.getType(SVB.getContext());
+    if (T->isUnsignedIntegerType() && ConcreteThreshold->getValue().isNegative()) {
+      // In this case we reduced the bound check to a comparison of the form
+      //   (symbol or value with unsigned type) < (negative number)
+      // which is always false. We are handling these cases separately because
+      // evalBinOpNN can perform a signed->unsigned conversion that turns the
+      // negative number into a huge positive value and leads to wildly
+      // inaccurate conclusions.
+      return {nullptr, State};
+    }
+  }
+  auto BelowThreshold =
+      SVB.evalBinOpNN(State, BO_LT, Value, Threshold, SVB.getConditionType()).getAs<NonLoc>();
+
+  if (BelowThreshold)
+    return State->assume(*BelowThreshold);
+
+  return {nullptr, nullptr};
+}
+
 void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
                                         const Stmt* LoadS,
                                         CheckerContext &checkerContext) const {
@@ -127,109 +153,101 @@ void ArrayBoundCheckerV2::checkLocation(SVal location, bool isLoad,
   // memory access is within the extent of the base region.  Since we
   // have some flexibility in defining the base region, we can achieve
   // various levels of conservatism in our buffer overflow checking.
-  ProgramStateRef state = checkerContext.getState();
 
-  SValBuilder &svalBuilder = checkerContext.getSValBuilder();
-  const RegionRawOffsetV2 &rawOffset =
-    RegionRawOffsetV2::computeOffset(state, svalBuilder, location);
-
-  if (!rawOffset.getRegion())
+  // The header ctype.h (from e.g. glibc) implements the isXXXXX() macros as
+  //   #define isXXXXX(arg) (LOOKUP_TABLE[arg] & BITMASK_FOR_XXXXX)
+  // and incomplete analysis of these leads to false positives. As even
+  // accurate reports would be confusing for the users, just disable reports
+  // from these macros:
+  if (isFromCtypeMacro(LoadS, checkerContext.getASTContext()))
     return;
 
-  NonLoc rawOffsetVal = rawOffset.getByteOffset();
-
-  // CHECK LOWER BOUND: Is byteOffset < extent begin?
-  //  If so, we are doing a load/store
-  //  before the first valid offset in the memory region.
+  ProgramStateRef state = checkerContext.getState();
 
-  SVal extentBegin = computeExtentBegin(svalBuilder, rawOffset.getRegion());
+  SValBuilder &svalBuilder = checkerContext.getSValBuilder();
+  const std::optional<RegionRawOffsetV2> &RawOffset =
+      RegionRawOffsetV2::computeOffset(state, svalBuilder, location);
 
-  if (std::optional<NonLoc> NV = extentBegin.getAs<NonLoc>()) {
-    if (auto ConcreteNV = NV->getAs<nonloc::ConcreteInt>()) {
-      std::pair<NonLoc, nonloc::ConcreteInt> simplifiedOffsets =
-          getSimplifiedOffsets(rawOffset.getByteOffset(), *ConcreteNV,
-                               svalBuilder);
-      rawOffsetVal = simplifiedOffsets.first;
-      *NV = simplifiedOffsets.second;
-    }
+  if (!RawOffset)
+    return;
 
-    SVal lowerBound = svalBuilder.evalBinOpNN(state, BO_LT, rawOffsetVal, *NV,
-                                              svalBuilder.getConditionType());
+  NonLoc ByteOffset = RawOffset->getByteOffset();
 
-    std::optional<NonLoc> lowerBoundToCheck = lowerBound.getAs<NonLoc>();
-    if (!lowerBoundToCheck)
-      return;
+  // CHECK LOWER BOUND
+  const MemSpaceRegion *SR = RawOffset->getRegion()->getMemorySpace();
+  if (!llvm::isa<UnknownSpaceRegion>(SR)) {
+    // A pointer to UnknownSpaceRegion may point to the middle of
+    // an allocated region.
 
-    ProgramStateRef state_precedesLowerBound, state_withinLowerBound;
-    std::tie(state_precedesLowerBound, state_withinLowerBound) =
-      state->assume(*lowerBoundToCheck);
+    auto [state_precedesLowerBound, state_withinLowerBound] =
+        compareValueToThreshold(state, ByteOffset,
+                                svalBuilder.makeZeroArrayIndex(), svalBuilder);
 
-    // Are we constrained enough to definitely precede the lower bound?
     if (state_precedesLowerBound && !state_withinLowerBound) {
+      // We know that the index definitely precedes the lower bound.
       reportOOB(checkerContext, state_precedesLowerBound, OOB_Precedes);
       return;
     }
 
-    // Otherwise, assume the constraint of the lower bound.
-    assert(state_withinLowerBound);
-    state = state_withinLowerBound;
+    if (state_withinLowerBound)
+      state = state_withinLowerBound;
   }
 
-  do {
-    // CHECK UPPER BOUND: Is byteOffset >= size(baseRegion)?  If so,
-    // we are doing a load/store after the last valid offset.
-    const MemRegion *MR = rawOffset.getRegion();
-    DefinedOrUnknownSVal Size = getDynamicExtent(state, MR, svalBuilder);
-    if (!isa<NonLoc>(Size))
-      break;
-
-    if (auto ConcreteSize = Size.getAs<nonloc::ConcreteInt>()) {
-      std::pair<NonLoc, nonloc::ConcreteInt> simplifiedOffsets =
-          getSimplifiedOffsets(rawOffset.getByteOffset(), *ConcreteSize,
-                               svalBuilder);
-      rawOffsetVal = simplifiedOffsets.first;
-      Size = simplifiedOffsets.second;
-    }
-
-    SVal upperbound = svalBuilder.evalBinOpNN(state, BO_GE, rawOffsetVal,
-                                              Size.castAs<NonLoc>(),
-                                              svalBuilder.getConditionType());
-
-    std::optional<NonLoc> upperboundToCheck = upperbound.getAs<NonLoc>();
-    if (!upperboundToCheck)
-      break;
-
-    ProgramStateRef state_exceedsUpperBound, state_withinUpperBound;
-    std::tie(state_exceedsUpperBound, state_withinUpperBound) =
-      state->assume(*upperboundToCheck);
-
-    // If we are under constrained and the index variables are tainted, report.
-    if (state_exceedsUpperBound && state_withinUpperBound) {
-      SVal ByteOffset = rawOffset.getByteOffset();
+  // CHECK UPPER BOUND
+  DefinedOrUnknownSVal Size =
+      getDynamicExtent(state, RawOffset->getRegion(), svalBuilder);
+  if (auto KnownSize = Size.getAs<NonLoc>()) {
+    auto [state_withinUpperBound, state_exceedsUpperBound] =
+        compareValueToThreshold(state, ByteOffset, *KnownSize, svalBuilder);
+
+    if (state_exceedsUpperBound) {
+      if (!state_withinUpperBound) {
+        // We know that the index definitely exceeds the upper bound.
+        reportOOB(checkerContext, state_exceedsUpperBound, OOB_Excedes);
+        return;
+      }
       if (isTainted(state, ByteOffset)) {
-        reportOOB(checkerContext, state_exceedsUpperBound, OOB_Tainted,
-                  std::make_unique<TaintBugVisitor>(ByteOffset));
+        // Both cases are possible, but the index is tainted, so report.
+        reportTaintOOB(checkerContext, state_exceedsUpperBound, ByteOffset);
         return;
       }
-    } else if (state_exceedsUpperBound) {
-      // If we are constrained enough to definitely exceed the upper bound,
-      // report.
-      assert(!state_withinUpperBound);
-      reportOOB(checkerContext, state_exceedsUpperBound, OOB_Excedes);
-      return;
     }
 
-    assert(state_withinUpperBound);
-    state = state_withinUpperBound;
+    if (state_withinUpperBound)
+      state = state_withinUpperBound;
   }
-  while (false);
 
   checkerContext.addTransition(state);
 }
 
-void ArrayBoundCheckerV2::reportOOB(
-    CheckerContext &checkerContext, ProgramStateRef errorState, OOB_Kind kind,
-    std::unique_ptr<BugReporterVisitor> Visitor) const {
+void ArrayBoundCheckerV2::reportTaintOOB(CheckerContext &checkerContext,
+                                         ProgramStateRef errorState,
+                                         SVal TaintedSVal) const {
+  ExplodedNode *errorNode = checkerContext.generateErrorNode(errorState);
+  if (!errorNode)
+    return;
+
+  if (!TaintBT)
+    TaintBT.reset(
+        new BugType(this, "Out-of-bound access", categories::TaintedData));
+
+  SmallString<256> buf;
+  llvm::raw_svector_ostream os(buf);
+  os << "Out of bound memory access (index is tainted)";
+  auto BR =
+      std::make_unique<PathSensitiveBugReport>(*TaintBT, os.str(), errorNode);
+
+  // Track back the propagation of taintedness.
+  for (SymbolRef Sym : getTaintedSymbols(errorState, TaintedSVal)) {
+    BR->markInteresting(Sym);
+  }
+
+  checkerContext.emitReport(std::move(BR));
+}
+
+void ArrayBoundCheckerV2::reportOOB(CheckerContext &checkerContext,
+                                    ProgramStateRef errorState,
+                                    OOB_Kind kind) const {
 
   ExplodedNode *errorNode = checkerContext.generateErrorNode(errorState);
   if (!errorNode)
@@ -251,16 +269,30 @@ void ArrayBoundCheckerV2::reportOOB(
   case OOB_Excedes:
     os << "(access exceeds upper limit of memory block)";
     break;
-  case OOB_Tainted:
-    os << "(index is tainted)";
-    break;
   }
-
   auto BR = std::make_unique<PathSensitiveBugReport>(*BT, os.str(), errorNode);
-  BR->addVisitor(std::move(Visitor));
   checkerContext.emitReport(std::move(BR));
 }
 
+bool ArrayBoundCheckerV2::isFromCtypeMacro(const Stmt *S, ASTContext &ACtx) {
+  SourceLocation Loc = S->getBeginLoc();
+  if (!Loc.isMacroID())
+    return false;
+
+  StringRef MacroName = Lexer::getImmediateMacroName(
+      Loc, ACtx.getSourceManager(), ACtx.getLangOpts());
+
+  if (MacroName.size() < 7 || MacroName[0] != 'i' || MacroName[1] != 's')
+    return false;
+
+  return ((MacroName == "isalnum") || (MacroName == "isalpha") ||
+          (MacroName == "isblank") || (MacroName == "isdigit") ||
+          (MacroName == "isgraph") || (MacroName == "islower") ||
+          (MacroName == "isnctrl") || (MacroName == "isprint") ||
+          (MacroName == "ispunct") || (MacroName == "isspace") ||
+          (MacroName == "isupper") || (MacroName == "isxdigit"));
+}
+
 #ifndef NDEBUG
 LLVM_DUMP_METHOD void RegionRawOffsetV2::dump() const {
   dumpToStream(llvm::errs());
@@ -271,85 +303,55 @@ void RegionRawOffsetV2::dumpToStream(raw_ostream &os) const {
 }
 #endif
 
-// Lazily computes a value to be used by 'computeOffset'.  If 'val'
-// is unknown or undefined, we lazily substitute '0'.  Otherwise,
-// return 'val'.
-static inline SVal getValue(SVal val, SValBuilder &svalBuilder) {
-  return val.isUndef() ? svalBuilder.makeZeroArrayIndex() : val;
-}
-
-// Scale a base value by a scaling factor, and return the scaled
-// value as an SVal.  Used by 'computeOffset'.
-static inline SVal scaleValue(ProgramStateRef state,
-                              NonLoc baseVal, CharUnits scaling,
-                              SValBuilder &sb) {
-  return sb.evalBinOpNN(state, BO_Mul, baseVal,
-                        sb.makeArrayIndex(scaling.getQuantity()),
-                        sb.getArrayIndexType());
-}
-
-// Add an SVal to another, treating unknown and undefined values as
-// summing to UnknownVal.  Used by 'computeOffset'.
-static SVal addValue(ProgramStateRef state, SVal x, SVal y,
-                     SValBuilder &svalBuilder) {
-  // We treat UnknownVals and UndefinedVals the same here because we
-  // only care about computing offsets.
-  if (x.isUnknownOrUndef() || y.isUnknownOrUndef())
-    return UnknownVal();
-
-  return svalBuilder.evalBinOpNN(state, BO_Add, x.castAs<NonLoc>(),
-                                 y.castAs<NonLoc>(),
-                                 svalBuilder.getArrayIndexType());
-}
-
-/// Compute a raw byte offset from a base region.  Used for array bounds
-/// checking.
-RegionRawOffsetV2 RegionRawOffsetV2::computeOffset(ProgramStateRef state,
-                                                   SValBuilder &svalBuilder,
-                                                   SVal location)
-{
-  const MemRegion *region = location.getAsRegion();
-  SVal offset = UndefinedVal();
-
-  while (region) {
-    switch (region->getKind()) {
-      default: {
-        if (const SubRegion *subReg = dyn_cast<SubRegion>(region)) {
-          offset = getValue(offset, svalBuilder);
-          if (!offset.isUnknownOrUndef())
-            return RegionRawOffsetV2(subReg, offset);
-        }
-        return RegionRawOffsetV2();
-      }
-      case MemRegion::ElementRegionKind: {
-        const ElementRegion *elemReg = cast<ElementRegion>(region);
-        SVal index = elemReg->getIndex();
-        if (!isa<NonLoc>(index))
-          return RegionRawOffsetV2();
-        QualType elemType = elemReg->getElementType();
+/// For a given Location that can be represented as a symbolic expression
+/// Arr[Idx] (or perhaps Arr[Idx1][Idx2] etc.), return the parent memory block
+/// Arr and the distance of Location from the beginning of Arr (expressed in a
+/// NonLoc that specifies the number of CharUnits). Returns nullopt when these
+/// cannot be determined.
+std::optional<RegionRawOffsetV2>
+RegionRawOffsetV2::computeOffset(ProgramStateRef State, SValBuilder &SVB,
+                                 SVal Location) {
+  QualType T = SVB.getArrayIndexType();
+  auto Calc = [&SVB, State, T](BinaryOperatorKind Op, NonLoc LHS, NonLoc RHS) {
+    // We will use this utility to add and multiply values.
+    return SVB.evalBinOpNN(State, Op, LHS, RHS, T).getAs<NonLoc>();
+  };
+
+  const MemRegion *Region = Location.getAsRegion();
+  NonLoc Offset = SVB.makeZeroArrayIndex();
+
+  while (Region) {
+    if (const auto *ERegion = dyn_cast<ElementRegion>(Region)) {
+      if (const auto Index = ERegion->getIndex().getAs<NonLoc>()) {
+        QualType ElemType = ERegion->getElementType();
         // If the element is an incomplete type, go no further.
-        ASTContext &astContext = svalBuilder.getContext();
-        if (elemType->isIncompleteType())
-          return RegionRawOffsetV2();
-
-        // Update the offset.
-        offset = addValue(state,
-                          getValue(offset, svalBuilder),
-                          scaleValue(state,
-                          index.castAs<NonLoc>(),
-                          astContext.getTypeSizeInChars(elemType),
-                          svalBuilder),
-                          svalBuilder);
-
-        if (offset.isUnknownOrUndef())
-          return RegionRawOffsetV2();
-
-        region = elemReg->getSuperRegion();
-        continue;
+        if (ElemType->isIncompleteType())
+          return std::nullopt;
+
+        // Perform Offset += Index * sizeof(ElemType); then continue the offset
+        // calculations with SuperRegion:
+        NonLoc Size = SVB.makeArrayIndex(
+            SVB.getContext().getTypeSizeInChars(ElemType).getQuantity());
+        if (auto Delta = Calc(BO_Mul, *Index, Size)) {
+          if (auto NewOffset = Calc(BO_Add, Offset, *Delta)) {
+            Offset = *NewOffset;
+            Region = ERegion->getSuperRegion();
+            continue;
+          }
+        }
       }
+    } else if (const auto *SRegion = dyn_cast<SubRegion>(Region)) {
+      // NOTE: The dyn_cast<>() is expected to succeed, it'd be very surprising
+      // to see a MemSpaceRegion at this point.
+      // FIXME: We may return with {<Region>, 0} even if we didn't handle any
+      // ElementRegion layers. I think that this behavior was introduced
+      // accidentally by 8a4c760c204546aba566e302f299f7ed2e00e287 in 2011, so
+      // it may be useful to review it in the future.
+      return RegionRawOffsetV2(SRegion, Offset);
     }
+    return std::nullopt;
   }
-  return RegionRawOffsetV2();
+  return std::nullopt;
 }
 
 void ento::registerArrayBoundCheckerV2(CheckerManager &mgr) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
index cc01e97d3fa20..5331d9574743f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
@@ -25,9 +25,13 @@ using namespace taint;
 
 namespace {
 class DivZeroChecker : public Checker< check::PreStmt<BinaryOperator> > {
-  mutable std::unique_ptr<BuiltinBug> BT;
-  void reportBug(const char *Msg, ProgramStateRef StateZero, CheckerContext &C,
-                 std::unique_ptr<BugReporterVisitor> Visitor = nullptr) const;
+  mutable std::unique_ptr<BugType> BT;
+  mutable std::unique_ptr<BugType> TaintBT;
+  void reportBug(StringRef Msg, ProgramStateRef StateZero,
+                 CheckerContext &C) const;
+  void reportTaintBug(StringRef Msg, ProgramStateRef StateZero,
+                      CheckerContext &C,
+                      llvm::ArrayRef<SymbolRef> TaintedSyms) const;
 
 public:
   void checkPreStmt(const BinaryOperator *B, CheckerContext &C) const;
@@ -41,20 +45,34 @@ static const Expr *getDenomExpr(const ExplodedNode *N) {
   return nullptr;
 }
 
-void DivZeroChecker::reportBug(
-    const char *Msg, ProgramStateRef StateZero, CheckerContext &C,
-    std::unique_ptr<BugReporterVisitor> Visitor) const {
+void DivZeroChecker::reportBug(StringRef Msg, ProgramStateRef StateZero,
+                               CheckerContext &C) const {
   if (ExplodedNode *N = C.generateErrorNode(StateZero)) {
     if (!BT)
-      BT.reset(new BuiltinBug(this, "Division by zero"));
+      BT.reset(new BugType(this, "Division by zero", categories::LogicError));
 
     auto R = std::make_unique<PathSensitiveBugReport>(*BT, Msg, N);
-    R->addVisitor(std::move(Visitor));
     bugreporter::trackExpressionValue(N, getDenomExpr(N), *R);
     C.emitReport(std::move(R));
   }
 }
 
+void DivZeroChecker::reportTaintBug(
+    StringRef Msg, ProgramStateRef StateZero, CheckerContext &C,
+    llvm::ArrayRef<SymbolRef> TaintedSyms) const {
+  if (ExplodedNode *N = C.generateErrorNode(StateZero)) {
+    if (!TaintBT)
+      TaintBT.reset(
+          new BugType(this, "Division by zero", categories::TaintedData));
+
+    auto R = std::make_unique<PathSensitiveBugReport>(*TaintBT, Msg, N);
+    bugreporter::trackExpressionValue(N, getDenomExpr(N), *R);
+    for (auto Sym : TaintedSyms)
+      R->markInteresting(Sym);
+    C.emitReport(std::move(R));
+  }
+}
+
 void DivZeroChecker::checkPreStmt(const BinaryOperator *B,
                                   CheckerContext &C) const {
   BinaryOperator::Opcode Op = B->getOpcode();
@@ -86,11 +104,13 @@ void DivZeroChecker::checkPreStmt(const BinaryOperator *B,
     return;
   }
 
-  bool TaintedD = isTainted(C.getState(), *DV);
-  if ((stateNotZero && stateZero && TaintedD)) {
-    reportBug("Division by a tainted value, possibly zero", stateZero, C,
-              std::make_unique<taint::TaintBugVisitor>(*DV));
-    return;
+  if ((stateNotZero && stateZero)) {
+    std::vector<SymbolRef> taintedSyms = getTaintedSymbols(C.getState(), *DV);
+    if (!taintedSyms.empty()) {
+      reportTaintBug("Division by a tainted value, possibly zero", stateZero, C,
+                     taintedSyms);
+      return;
+    }
   }
 
   // If we get here, then the denom should not be zero. We abandon the implicit
diff --git a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index f6e2f59d5697c..4bf8ce13e0572 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -26,12 +26,14 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/YAMLTraits.h"
 
 #include <limits>
 #include <memory>
 #include <optional>
 #include <utility>
+#include <vector>
 
 #define DEBUG_TYPE "taint-checker"
 
@@ -114,47 +116,110 @@ bool isStdin(SVal Val, const ASTContext &ACtx) {
   return false;
 }
 
-SVal getPointeeOf(const CheckerContext &C, Loc LValue) {
-  const QualType ArgTy = LValue.getType(C.getASTContext());
+SVal getPointeeOf(ProgramStateRef State, Loc LValue) {
+  const QualType ArgTy = LValue.getType(State->getStateManager().getContext());
   if (!ArgTy->isPointerType() || !ArgTy->getPointeeType()->isVoidType())
-    return C.getState()->getSVal(LValue);
+    return State->getSVal(LValue);
 
   // Do not dereference void pointers. Treat them as byte pointers instead.
   // FIXME: we might want to consider more than just the first byte.
-  return C.getState()->getSVal(LValue, C.getASTContext().CharTy);
+  return State->getSVal(LValue, State->getStateManager().getContext().CharTy);
 }
 
 /// Given a pointer/reference argument, return the value it refers to.
-std::optional<SVal> getPointeeOf(const CheckerContext &C, SVal Arg) {
+std::optional<SVal> getPointeeOf(ProgramStateRef State, SVal Arg) {
   if (auto LValue = Arg.getAs<Loc>())
-    return getPointeeOf(C, *LValue);
+    return getPointeeOf(State, *LValue);
   return std::nullopt;
 }
 
 /// Given a pointer, return the SVal of its pointee or if it is tainted,
 /// otherwise return the pointer's SVal if tainted.
 /// Also considers stdin as a taint source.
-std::optional<SVal> getTaintedPointeeOrPointer(const CheckerContext &C,
+std::optional<SVal> getTaintedPointeeOrPointer(ProgramStateRef State,
                                                SVal Arg) {
-  const ProgramStateRef State = C.getState();
-
-  if (auto Pointee = getPointeeOf(C, Arg))
+  if (auto Pointee = getPointeeOf(State, Arg))
     if (isTainted(State, *Pointee)) // FIXME: isTainted(...) ? Pointee : None;
       return Pointee;
 
   if (isTainted(State, Arg))
     return Arg;
+  return std::nullopt;
+}
 
-  // FIXME: This should be done by the isTainted() API.
-  if (isStdin(Arg, C.getASTContext()))
-    return Arg;
+bool isTaintedOrPointsToTainted(ProgramStateRef State, SVal ExprSVal) {
+  return getTaintedPointeeOrPointer(State, ExprSVal).has_value();
+}
 
-  return std::nullopt;
+/// Helps in printing taint diagnostics.
+/// Marks the incoming parameters of a function interesting (to be printed)
+/// when the return value, or the outgoing parameters are tainted.
+const NoteTag *taintOriginTrackerTag(CheckerContext &C,
+                                     std::vector<SymbolRef> TaintedSymbols,
+                                     std::vector<ArgIdxTy> TaintedArgs,
+                                     const LocationContext *CallLocation) {
+  return C.getNoteTag([TaintedSymbols = std::move(TaintedSymbols),
+                       TaintedArgs = std::move(TaintedArgs), CallLocation](
+                          PathSensitiveBugReport &BR) -> std::string {
+    SmallString<256> Msg;
+    // We give diagnostics only for taint related reports
+    if (!BR.isInteresting(CallLocation) ||
+        BR.getBugType().getCategory() != categories::TaintedData) {
+      return "";
+    }
+    if (TaintedSymbols.empty())
+      return "Taint originated here";
+
+    for (auto Sym : TaintedSymbols) {
+      BR.markInteresting(Sym);
+    }
+    LLVM_DEBUG(for (auto Arg
+                    : TaintedArgs) {
+      llvm::dbgs() << "Taint Propagated from argument " << Arg + 1 << "\n";
+    });
+    return "";
+  });
 }
 
-bool isTaintedOrPointsToTainted(const Expr *E, const ProgramStateRef &State,
-                                CheckerContext &C) {
-  return getTaintedPointeeOrPointer(C, C.getSVal(E)).has_value();
+/// Helps in printing taint diagnostics.
+/// Marks the function interesting (to be printed)
+/// when the return value, or the outgoing parameters are tainted.
+const NoteTag *taintPropagationExplainerTag(
+    CheckerContext &C, std::vector<SymbolRef> TaintedSymbols,
+    std::vector<ArgIdxTy> TaintedArgs, const LocationContext *CallLocation) {
+  assert(TaintedSymbols.size() == TaintedArgs.size());
+  return C.getNoteTag([TaintedSymbols = std::move(TaintedSymbols),
+                       TaintedArgs = std::move(TaintedArgs), CallLocation](
+                          PathSensitiveBugReport &BR) -> std::string {
+    SmallString<256> Msg;
+    llvm::raw_svector_ostream Out(Msg);
+    // We give diagnostics only for taint related reports
+    if (TaintedSymbols.empty() ||
+        BR.getBugType().getCategory() != categories::TaintedData) {
+      return "";
+    }
+    int nofTaintedArgs = 0;
+    for (auto [Idx, Sym] : llvm::enumerate(TaintedSymbols)) {
+      if (BR.isInteresting(Sym)) {
+        BR.markInteresting(CallLocation);
+        if (TaintedArgs[Idx] != ReturnValueIndex) {
+          LLVM_DEBUG(llvm::dbgs() << "Taint Propagated to argument "
+                                  << TaintedArgs[Idx] + 1 << "\n");
+          if (nofTaintedArgs == 0)
+            Out << "Taint propagated to the ";
+          else
+            Out << ", ";
+          Out << TaintedArgs[Idx] + 1
+              << llvm::getOrdinalSuffix(TaintedArgs[Idx] + 1) << " argument";
+          nofTaintedArgs++;
+        } else {
+          LLVM_DEBUG(llvm::dbgs() << "Taint Propagated to return value.\n");
+          Out << "Taint propagated to the return value";
+        }
+      }
+    }
+    return std::string(Out.str());
+  });
 }
 
 /// ArgSet is used to describe arguments relevant for taint detection or
@@ -193,7 +258,7 @@ class GenericTaintRule {
   ArgSet SinkArgs;
   /// Arguments which should be sanitized on function return.
   ArgSet FilterArgs;
-  /// Arguments which can participate in taint propagationa. If any of the
+  /// Arguments which can participate in taint propagation. If any of the
   /// arguments in PropSrcArgs is tainted, all arguments in  PropDstArgs should
   /// be tainted.
   ArgSet PropSrcArgs;
@@ -343,7 +408,7 @@ class GenericTaintChecker : public Checker<check::PreCall, check::PostCall> {
                                CheckerContext &C) const;
 
 private:
-  const BugType BT{this, "Use of Untrusted Data", "Untrusted Data"};
+  const BugType BT{this, "Use of Untrusted Data", categories::TaintedData};
 
   bool checkUncontrolledFormatString(const CallEvent &Call,
                                      CheckerContext &C) const;
@@ -351,7 +416,7 @@ class GenericTaintChecker : public Checker<check::PreCall, check::PostCall> {
   void taintUnsafeSocketProtocol(const CallEvent &Call,
                                  CheckerContext &C) const;
 
-  /// Default taint rules are initilized with the help of a CheckerContext to
+  /// Default taint rules are initalized with the help of a CheckerContext to
   /// access the names of built-in functions like memcpy.
   void initTaintRules(CheckerContext &C) const;
 
@@ -788,22 +853,39 @@ void GenericTaintChecker::checkPostCall(const CallEvent &Call,
     llvm::dbgs() << "> actually wants to taint arg index: " << I << '\n';
   });
 
+  const NoteTag *InjectionTag = nullptr;
+  std::vector<SymbolRef> TaintedSymbols;
+  std::vector<ArgIdxTy> TaintedIndexes;
   for (ArgIdxTy ArgNum : *TaintArgs) {
     // Special handling for the tainted return value.
     if (ArgNum == ReturnValueIndex) {
       State = addTaint(State, Call.getReturnValue());
+      std::vector<SymbolRef> TaintedSyms =
+          getTaintedSymbols(State, Call.getReturnValue());
+      if (!TaintedSyms.empty()) {
+        TaintedSymbols.push_back(TaintedSyms[0]);
+        TaintedIndexes.push_back(ArgNum);
+      }
       continue;
     }
-
     // The arguments are pointer arguments. The data they are pointing at is
     // tainted after the call.
-    if (auto V = getPointeeOf(C, Call.getArgSVal(ArgNum)))
+    if (auto V = getPointeeOf(State, Call.getArgSVal(ArgNum))) {
       State = addTaint(State, *V);
+      std::vector<SymbolRef> TaintedSyms = getTaintedSymbols(State, *V);
+      if (!TaintedSyms.empty()) {
+        TaintedSymbols.push_back(TaintedSyms[0]);
+        TaintedIndexes.push_back(ArgNum);
+      }
+    }
   }
-
+  // Create a NoteTag callback, which prints to the user where the taintedness
+  // was propagated to.
+  InjectionTag = taintPropagationExplainerTag(C, TaintedSymbols, TaintedIndexes,
+                                              Call.getCalleeStackFrame(0));
   // Clear up the taint info from the state.
   State = State->remove<TaintArgsOnPostVisit>(CurrentFrame);
-  C.addTransition(State);
+  C.addTransition(State, InjectionTag);
 }
 
 void GenericTaintChecker::printState(raw_ostream &Out, ProgramStateRef State,
@@ -826,15 +908,19 @@ void GenericTaintRule::process(const GenericTaintChecker &Checker,
 
   /// Check for taint sinks.
   ForEachCallArg([this, &Checker, &C, &State](ArgIdxTy I, const Expr *E, SVal) {
-    if (SinkArgs.contains(I) && isTaintedOrPointsToTainted(E, State, C))
+    // Add taintedness to stdin parameters
+    if (isStdin(C.getSVal(E), C.getASTContext())) {
+      State = addTaint(State, C.getSVal(E));
+    }
+    if (SinkArgs.contains(I) && isTaintedOrPointsToTainted(State, C.getSVal(E)))
       Checker.generateReportIfTainted(E, SinkMsg.value_or(MsgCustomSink), C);
   });
 
   /// Check for taint filters.
-  ForEachCallArg([this, &C, &State](ArgIdxTy I, const Expr *E, SVal S) {
+  ForEachCallArg([this, &State](ArgIdxTy I, const Expr *E, SVal S) {
     if (FilterArgs.contains(I)) {
       State = removeTaint(State, S);
-      if (auto P = getPointeeOf(C, S))
+      if (auto P = getPointeeOf(State, S))
         State = removeTaint(State, *P);
     }
   });
@@ -843,11 +929,25 @@ void GenericTaintRule::process(const GenericTaintChecker &Checker,
   /// A rule is relevant if PropSrcArgs is empty, or if any of its signified
   /// args are tainted in context of the current CallEvent.
   bool IsMatching = PropSrcArgs.isEmpty();
-  ForEachCallArg(
-      [this, &C, &IsMatching, &State](ArgIdxTy I, const Expr *E, SVal) {
-        IsMatching = IsMatching || (PropSrcArgs.contains(I) &&
-                                    isTaintedOrPointsToTainted(E, State, C));
-      });
+  std::vector<SymbolRef> TaintedSymbols;
+  std::vector<ArgIdxTy> TaintedIndexes;
+  ForEachCallArg([this, &C, &IsMatching, &State, &TaintedSymbols,
+                  &TaintedIndexes](ArgIdxTy I, const Expr *E, SVal) {
+    std::optional<SVal> TaintedSVal =
+        getTaintedPointeeOrPointer(State, C.getSVal(E));
+    IsMatching =
+        IsMatching || (PropSrcArgs.contains(I) && TaintedSVal.has_value());
+
+    // We track back tainted arguments except for stdin
+    if (TaintedSVal && !isStdin(*TaintedSVal, C.getASTContext())) {
+      std::vector<SymbolRef> TaintedArgSyms =
+          getTaintedSymbols(State, *TaintedSVal);
+      if (!TaintedArgSyms.empty()) {
+        llvm::append_range(TaintedSymbols, TaintedArgSyms);
+        TaintedIndexes.push_back(I);
+      }
+    }
+  });
 
   if (!IsMatching)
     return;
@@ -890,7 +990,10 @@ void GenericTaintRule::process(const GenericTaintChecker &Checker,
 
   if (!Result.isEmpty())
     State = State->set<TaintArgsOnPostVisit>(C.getStackFrame(), Result);
-  C.addTransition(State);
+  const NoteTag *InjectionTag = taintOriginTrackerTag(
+      C, std::move(TaintedSymbols), std::move(TaintedIndexes),
+      Call.getCalleeStackFrame(0));
+  C.addTransition(State, InjectionTag);
 }
 
 bool GenericTaintRule::UntrustedEnv(CheckerContext &C) {
@@ -902,7 +1005,8 @@ bool GenericTaintRule::UntrustedEnv(CheckerContext &C) {
 bool GenericTaintChecker::generateReportIfTainted(const Expr *E, StringRef Msg,
                                                   CheckerContext &C) const {
   assert(E);
-  std::optional<SVal> TaintedSVal{getTaintedPointeeOrPointer(C, C.getSVal(E))};
+  std::optional<SVal> TaintedSVal =
+      getTaintedPointeeOrPointer(C.getState(), C.getSVal(E));
 
   if (!TaintedSVal)
     return false;
@@ -911,7 +1015,10 @@ bool GenericTaintChecker::generateReportIfTainted(const Expr *E, StringRef Msg,
   if (ExplodedNode *N = C.generateNonFatalErrorNode()) {
     auto report = std::make_unique<PathSensitiveBugReport>(BT, Msg, N);
     report->addRange(E->getSourceRange());
-    report->addVisitor(std::make_unique<TaintBugVisitor>(*TaintedSVal));
+    for (auto TaintedSym : getTaintedSymbols(C.getState(), *TaintedSVal)) {
+      report->markInteresting(TaintedSym);
+    }
+
     C.emitReport(std::move(report));
     return true;
   }
diff --git a/clang/lib/StaticAnalyzer/Checkers/Taint.cpp b/clang/lib/StaticAnalyzer/Checkers/Taint.cpp
index a95c0e1832844..daf2e1851b3d7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/Taint.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/Taint.cpp
@@ -146,41 +146,117 @@ ProgramStateRef taint::addPartialTaint(ProgramStateRef State,
 
 bool taint::isTainted(ProgramStateRef State, const Stmt *S,
                       const LocationContext *LCtx, TaintTagType Kind) {
-  SVal val = State->getSVal(S, LCtx);
-  return isTainted(State, val, Kind);
+  return !getTaintedSymbolsImpl(State, S, LCtx, Kind, /*ReturnFirstOnly=*/true)
+              .empty();
 }
 
 bool taint::isTainted(ProgramStateRef State, SVal V, TaintTagType Kind) {
-  if (SymbolRef Sym = V.getAsSymbol())
-    return isTainted(State, Sym, Kind);
-  if (const MemRegion *Reg = V.getAsRegion())
-    return isTainted(State, Reg, Kind);
-  return false;
+  return !getTaintedSymbolsImpl(State, V, Kind, /*ReturnFirstOnly=*/true)
+              .empty();
 }
 
 bool taint::isTainted(ProgramStateRef State, const MemRegion *Reg,
                       TaintTagType K) {
-  if (!Reg)
-    return false;
+  return !getTaintedSymbolsImpl(State, Reg, K, /*ReturnFirstOnly=*/true)
+              .empty();
+}
+
+bool taint::isTainted(ProgramStateRef State, SymbolRef Sym, TaintTagType Kind) {
+  return !getTaintedSymbolsImpl(State, Sym, Kind, /*ReturnFirstOnly=*/true)
+              .empty();
+}
+
+std::vector<SymbolRef> taint::getTaintedSymbols(ProgramStateRef State,
+                                                const Stmt *S,
+                                                const LocationContext *LCtx,
+                                                TaintTagType Kind) {
+  return getTaintedSymbolsImpl(State, S, LCtx, Kind, /*ReturnFirstOnly=*/false);
+}
+
+std::vector<SymbolRef> taint::getTaintedSymbols(ProgramStateRef State, SVal V,
+                                                TaintTagType Kind) {
+  return getTaintedSymbolsImpl(State, V, Kind, /*ReturnFirstOnly=*/false);
+}
+
+std::vector<SymbolRef> taint::getTaintedSymbols(ProgramStateRef State,
+                                                SymbolRef Sym,
+                                                TaintTagType Kind) {
+  return getTaintedSymbolsImpl(State, Sym, Kind, /*ReturnFirstOnly=*/false);
+}
+
+std::vector<SymbolRef> taint::getTaintedSymbols(ProgramStateRef State,
+                                                const MemRegion *Reg,
+                                                TaintTagType Kind) {
+  return getTaintedSymbolsImpl(State, Reg, Kind, /*ReturnFirstOnly=*/false);
+}
+
+std::vector<SymbolRef> taint::getTaintedSymbolsImpl(ProgramStateRef State,
+                                                    const Stmt *S,
+                                                    const LocationContext *LCtx,
+                                                    TaintTagType Kind,
+                                                    bool returnFirstOnly) {
+  SVal val = State->getSVal(S, LCtx);
+  return getTaintedSymbolsImpl(State, val, Kind, returnFirstOnly);
+}
 
+std::vector<SymbolRef> taint::getTaintedSymbolsImpl(ProgramStateRef State,
+                                                    SVal V, TaintTagType Kind,
+                                                    bool returnFirstOnly) {
+  if (SymbolRef Sym = V.getAsSymbol())
+    return getTaintedSymbolsImpl(State, Sym, Kind, returnFirstOnly);
+  if (const MemRegion *Reg = V.getAsRegion())
+    return getTaintedSymbolsImpl(State, Reg, Kind, returnFirstOnly);
+  return {};
+}
+
+std::vector<SymbolRef> taint::getTaintedSymbolsImpl(ProgramStateRef State,
+                                                    const MemRegion *Reg,
+                                                    TaintTagType K,
+                                                    bool returnFirstOnly) {
+  std::vector<SymbolRef> TaintedSymbols;
+  if (!Reg)
+    return TaintedSymbols;
   // Element region (array element) is tainted if either the base or the offset
   // are tainted.
-  if (const ElementRegion *ER = dyn_cast<ElementRegion>(Reg))
-    return isTainted(State, ER->getSuperRegion(), K) ||
-           isTainted(State, ER->getIndex(), K);
+  if (const ElementRegion *ER = dyn_cast<ElementRegion>(Reg)) {
+    std::vector<SymbolRef> TaintedIndex =
+        getTaintedSymbolsImpl(State, ER->getIndex(), K, returnFirstOnly);
+    llvm::append_range(TaintedSymbols, TaintedIndex);
+    if (returnFirstOnly && !TaintedSymbols.empty())
+      return TaintedSymbols; // return early if needed
+    std::vector<SymbolRef> TaintedSuperRegion =
+        getTaintedSymbolsImpl(State, ER->getSuperRegion(), K, returnFirstOnly);
+    llvm::append_range(TaintedSymbols, TaintedSuperRegion);
+    if (returnFirstOnly && !TaintedSymbols.empty())
+      return TaintedSymbols; // return early if needed
+  }
 
-  if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(Reg))
-    return isTainted(State, SR->getSymbol(), K);
+  if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(Reg)) {
+    std::vector<SymbolRef> TaintedRegions =
+        getTaintedSymbolsImpl(State, SR->getSymbol(), K, returnFirstOnly);
+    llvm::append_range(TaintedSymbols, TaintedRegions);
+    if (returnFirstOnly && !TaintedSymbols.empty())
+      return TaintedSymbols; // return early if needed
+  }
 
-  if (const SubRegion *ER = dyn_cast<SubRegion>(Reg))
-    return isTainted(State, ER->getSuperRegion(), K);
+  if (const SubRegion *ER = dyn_cast<SubRegion>(Reg)) {
+    std::vector<SymbolRef> TaintedSubRegions =
+        getTaintedSymbolsImpl(State, ER->getSuperRegion(), K, returnFirstOnly);
+    llvm::append_range(TaintedSymbols, TaintedSubRegions);
+    if (returnFirstOnly && !TaintedSymbols.empty())
+      return TaintedSymbols; // return early if needed
+  }
 
-  return false;
+  return TaintedSymbols;
 }
 
-bool taint::isTainted(ProgramStateRef State, SymbolRef Sym, TaintTagType Kind) {
+std::vector<SymbolRef> taint::getTaintedSymbolsImpl(ProgramStateRef State,
+                                                    SymbolRef Sym,
+                                                    TaintTagType Kind,
+                                                    bool returnFirstOnly) {
+  std::vector<SymbolRef> TaintedSymbols;
   if (!Sym)
-    return false;
+    return TaintedSymbols;
 
   // Traverse all the symbols this symbol depends on to see if any are tainted.
   for (SymExpr::symbol_iterator SI = Sym->symbol_begin(),
@@ -190,18 +266,24 @@ bool taint::isTainted(ProgramStateRef State, SymbolRef Sym, TaintTagType Kind) {
       continue;
 
     if (const TaintTagType *Tag = State->get<TaintMap>(*SI)) {
-      if (*Tag == Kind)
-        return true;
+      if (*Tag == Kind) {
+        TaintedSymbols.push_back(*SI);
+        if (returnFirstOnly)
+          return TaintedSymbols; // return early if needed
+      }
     }
 
     if (const auto *SD = dyn_cast<SymbolDerived>(*SI)) {
       // If this is a SymbolDerived with a tainted parent, it's also tainted.
-      if (isTainted(State, SD->getParentSymbol(), Kind))
-        return true;
+      std::vector<SymbolRef> TaintedParents = getTaintedSymbolsImpl(
+          State, SD->getParentSymbol(), Kind, returnFirstOnly);
+      llvm::append_range(TaintedSymbols, TaintedParents);
+      if (returnFirstOnly && !TaintedSymbols.empty())
+        return TaintedSymbols; // return early if needed
 
       // If this is a SymbolDerived with the same parent symbol as another
-      // tainted SymbolDerived and a region that's a sub-region of that tainted
-      // symbol, it's also tainted.
+      // tainted SymbolDerived and a region that's a sub-region of that
+      // tainted symbol, it's also tainted.
       if (const TaintedSubRegions *Regs =
               State->get<DerivedSymTaint>(SD->getParentSymbol())) {
         const TypedValueRegion *R = SD->getRegion();
@@ -210,46 +292,32 @@ bool taint::isTainted(ProgramStateRef State, SymbolRef Sym, TaintTagType Kind) {
           // complete. For example, this would not currently identify
           // overlapping fields in a union as tainted. To identify this we can
           // check for overlapping/nested byte offsets.
-          if (Kind == I.second && R->isSubRegionOf(I.first))
-            return true;
+          if (Kind == I.second && R->isSubRegionOf(I.first)) {
+            TaintedSymbols.push_back(SD->getParentSymbol());
+            if (returnFirstOnly && !TaintedSymbols.empty())
+              return TaintedSymbols; // return early if needed
+          }
         }
       }
     }
 
     // If memory region is tainted, data is also tainted.
     if (const auto *SRV = dyn_cast<SymbolRegionValue>(*SI)) {
-      if (isTainted(State, SRV->getRegion(), Kind))
-        return true;
+      std::vector<SymbolRef> TaintedRegions =
+          getTaintedSymbolsImpl(State, SRV->getRegion(), Kind, returnFirstOnly);
+      llvm::append_range(TaintedSymbols, TaintedRegions);
+      if (returnFirstOnly && !TaintedSymbols.empty())
+        return TaintedSymbols; // return early if needed
     }
 
     // If this is a SymbolCast from a tainted value, it's also tainted.
     if (const auto *SC = dyn_cast<SymbolCast>(*SI)) {
-      if (isTainted(State, SC->getOperand(), Kind))
-        return true;
+      std::vector<SymbolRef> TaintedCasts =
+          getTaintedSymbolsImpl(State, SC->getOperand(), Kind, returnFirstOnly);
+      llvm::append_range(TaintedSymbols, TaintedCasts);
+      if (returnFirstOnly && !TaintedSymbols.empty())
+        return TaintedSymbols; // return early if needed
     }
   }
-
-  return false;
-}
-
-PathDiagnosticPieceRef TaintBugVisitor::VisitNode(const ExplodedNode *N,
-                                                  BugReporterContext &BRC,
-                                                  PathSensitiveBugReport &BR) {
-
-  // Find the ExplodedNode where the taint was first introduced
-  if (!isTainted(N->getState(), V) ||
-      isTainted(N->getFirstPred()->getState(), V))
-    return nullptr;
-
-  const Stmt *S = N->getStmtForDiagnostics();
-  if (!S)
-    return nullptr;
-
-  const LocationContext *NCtx = N->getLocationContext();
-  PathDiagnosticLocation L =
-      PathDiagnosticLocation::createBegin(S, BRC.getSourceManager(), NCtx);
-  if (!L.isValid() || !L.asLocation().isValid())
-    return nullptr;
-
-  return std::make_shared<PathDiagnosticEventPiece>(L, "Taint originated here");
-}
+  return TaintedSymbols;
+}
\ No newline at end of file
diff --git a/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
index fe910ce353021..b195d912cadfe 100644
--- a/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
@@ -35,13 +35,8 @@ class VLASizeChecker
     : public Checker<check::PreStmt<DeclStmt>,
                      check::PreStmt<UnaryExprOrTypeTraitExpr>> {
   mutable std::unique_ptr<BugType> BT;
-  enum VLASize_Kind {
-    VLA_Garbage,
-    VLA_Zero,
-    VLA_Tainted,
-    VLA_Negative,
-    VLA_Overflow
-  };
+  mutable std::unique_ptr<BugType> TaintBT;
+  enum VLASize_Kind { VLA_Garbage, VLA_Zero, VLA_Negative, VLA_Overflow };
 
   /// Check a VLA for validity.
   /// Every dimension of the array and the total size is checked for validity.
@@ -55,8 +50,10 @@ class VLASizeChecker
                                     const Expr *SizeE) const;
 
   void reportBug(VLASize_Kind Kind, const Expr *SizeE, ProgramStateRef State,
-                 CheckerContext &C,
-                 std::unique_ptr<BugReporterVisitor> Visitor = nullptr) const;
+                 CheckerContext &C) const;
+
+  void reportTaintBug(const Expr *SizeE, ProgramStateRef State,
+                      CheckerContext &C, SVal TaintedSVal) const;
 
 public:
   void checkPreStmt(const DeclStmt *DS, CheckerContext &C) const;
@@ -167,8 +164,7 @@ ProgramStateRef VLASizeChecker::checkVLAIndexSize(CheckerContext &C,
 
   // Check if the size is tainted.
   if (isTainted(State, SizeV)) {
-    reportBug(VLA_Tainted, SizeE, nullptr, C,
-              std::make_unique<TaintBugVisitor>(SizeV));
+    reportTaintBug(SizeE, State, C, SizeV);
     return nullptr;
   }
 
@@ -209,17 +205,44 @@ ProgramStateRef VLASizeChecker::checkVLAIndexSize(CheckerContext &C,
   return State;
 }
 
-void VLASizeChecker::reportBug(
-    VLASize_Kind Kind, const Expr *SizeE, ProgramStateRef State,
-    CheckerContext &C, std::unique_ptr<BugReporterVisitor> Visitor) const {
+void VLASizeChecker::reportTaintBug(const Expr *SizeE, ProgramStateRef State,
+                                    CheckerContext &C, SVal TaintedSVal) const {
+  // Generate an error node.
+  ExplodedNode *N = C.generateErrorNode(State);
+  if (!N)
+    return;
+
+  if (!TaintBT)
+    TaintBT.reset(
+        new BugType(this, "Dangerous variable-length array (VLA) declaration",
+                    categories::TaintedData));
+
+  SmallString<256> buf;
+  llvm::raw_svector_ostream os(buf);
+  os << "Declared variable-length array (VLA) ";
+  os << "has tainted size";
+
+  auto report = std::make_unique<PathSensitiveBugReport>(*TaintBT, os.str(), N);
+  report->addRange(SizeE->getSourceRange());
+  bugreporter::trackExpressionValue(N, SizeE, *report);
+  // The vla size may be a complex expression where multiple memory locations
+  // are tainted.
+  for (auto Sym : getTaintedSymbols(State, TaintedSVal))
+    report->markInteresting(Sym);
+  C.emitReport(std::move(report));
+}
+
+void VLASizeChecker::reportBug(VLASize_Kind Kind, const Expr *SizeE,
+                               ProgramStateRef State, CheckerContext &C) const {
   // Generate an error node.
   ExplodedNode *N = C.generateErrorNode(State);
   if (!N)
     return;
 
   if (!BT)
-    BT.reset(new BuiltinBug(
-        this, "Dangerous variable-length array (VLA) declaration"));
+    BT.reset(new BugType(this,
+                         "Dangerous variable-length array (VLA) declaration",
+                         categories::LogicError));
 
   SmallString<256> buf;
   llvm::raw_svector_ostream os(buf);
@@ -231,9 +254,6 @@ void VLASizeChecker::reportBug(
   case VLA_Zero:
     os << "has zero size";
     break;
-  case VLA_Tainted:
-    os << "has tainted size";
-    break;
   case VLA_Negative:
     os << "has negative size";
     break;
@@ -243,7 +263,6 @@ void VLASizeChecker::reportBug(
   }
 
   auto report = std::make_unique<PathSensitiveBugReport>(*BT, os.str(), N);
-  report->addVisitor(std::move(Visitor));
   report->addRange(SizeE->getSourceRange());
   bugreporter::trackExpressionValue(N, SizeE, *report);
   C.emitReport(std::move(report));
diff --git a/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp b/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
index d12c35ef156a0..e9cc080caf5f8 100644
--- a/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CommonBugCategories.cpp
@@ -23,6 +23,7 @@ const char *const CXXObjectLifecycle = "C++ object lifecycle";
 const char *const CXXMoveSemantics = "C++ move semantics";
 const char *const SecurityError = "Security error";
 const char *const UnusedCode = "Unused code";
+const char *const TaintedData = "Tainted data used";
 } // namespace categories
 } // namespace ento
 } // namespace clang
diff --git a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index a275d36286d3b..5de99384449a4 100644
--- a/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -1083,7 +1083,7 @@ areFeasible(ConstraintRangeTy Constraints) {
 ///
 /// \returns true if assuming this Sym to be true means equality of operands
 ///          false if it means disequality of operands
-///          None otherwise
+///          std::nullopt otherwise
 std::optional<bool> meansEquality(const SymSymExpr *Sym) {
   switch (Sym->getOpcode()) {
   case BO_Sub:
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 937f39ba3bffa..baf76f01cfb54 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -181,6 +181,7 @@ class DependencyScanningAction : public tooling::ToolAction {
     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
     ScanInstance.getFrontendOpts().ModulesShareFileManager = false;
+    ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw";
 
     ScanInstance.setFileManager(FileMgr);
     // Support for virtual file system overlays.
@@ -309,12 +310,11 @@ DependencyScanningWorker::DependencyScanningWorker(
     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()),
       EagerLoadModules(Service.shouldEagerLoadModules()) {
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
+  // We need to read object files from PCH built outside the scanner.
   PCHContainerOps->registerReader(
       std::make_unique<ObjectFilePCHContainerReader>());
-  // We don't need to write object files, but the current PCH implementation
-  // requires the writer to be registered as well.
-  PCHContainerOps->registerWriter(
-      std::make_unique<ObjectFilePCHContainerWriter>());
+  // The scanner itself writes only raw ast files.
+  PCHContainerOps->registerWriter(std::make_unique<RawPCHContainerWriter>());
 
   switch (Service.getMode()) {
   case ScanningMode::DependencyDirectivesScan:
diff --git a/clang/lib/Tooling/Syntax/Tokens.cpp b/clang/lib/Tooling/Syntax/Tokens.cpp
index b13dc9ef4aeed..64e6eee6b62f2 100644
--- a/clang/lib/Tooling/Syntax/Tokens.cpp
+++ b/clang/lib/Tooling/Syntax/Tokens.cpp
@@ -986,7 +986,7 @@ std::string TokenBuffer::dumpForTests() const {
   OS << "\n";
 
   std::vector<FileID> Keys;
-  for (auto F : Files)
+  for (const auto &F : Files)
     Keys.push_back(F.first);
   llvm::sort(Keys);
 
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp
index 413ab2fa45d84..22ccafd579241 100644
--- a/clang/test/AST/Interp/arrays.cpp
+++ b/clang/test/AST/Interp/arrays.cpp
@@ -334,3 +334,19 @@ namespace IncDec {
                                    // ref-error {{not an integral constant expression}} \
                                   // ref-note {{in call to}}
 };
+
+namespace ZeroInit {
+  struct A {
+    int *p[2];
+  };
+  constexpr A a = {};
+  static_assert(a.p[0] == nullptr, "");
+  static_assert(a.p[1] == nullptr, "");
+
+  struct B {
+    double f[2];
+  };
+  constexpr B b = {};
+  static_assert(b.f[0] == 0.0, "");
+  static_assert(b.f[1] == 0.0, "");
+}
diff --git a/clang/test/AST/Interp/floats.cpp b/clang/test/AST/Interp/floats.cpp
index 71f585ee70a9c..ab75a537d0ce3 100644
--- a/clang/test/AST/Interp/floats.cpp
+++ b/clang/test/AST/Interp/floats.cpp
@@ -1,6 +1,15 @@
 // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify %s
 // RUN: %clang_cc1 -verify=ref %s
 
+
+constexpr void assert(bool C) {
+  if (C)
+    return;
+  // Invalid in constexpr.
+  (void)(1 / 0); // expected-warning {{undefined}} \
+                 // ref-warning {{undefined}}
+}
+
 constexpr int i = 2;
 constexpr float f = 1.0f;
 static_assert(f == 1.0f, "");
@@ -79,6 +88,30 @@ namespace compound {
   static_assert(f2() == __FLT_MAX__, "");
 }
 
+namespace unary {
+  constexpr float a() {
+    float f = 0.0;
+    assert(++f == 1.0);
+    assert(f == 1.0);
+    ++f;
+    f++;
+    assert(f == 3.0);
+    --f;
+    f--;
+    assert(f == 1.0);
+    return 1.0;
+  }
+  static_assert(a() == 1.0, "");
+
+  constexpr float b() {
+    float f = __FLT_MAX__;
+    f++;
+    return f;
+  }
+  static_assert(b() == __FLT_MAX__, "");
+}
+
+
 namespace ZeroInit {
   template<typename FloatT>
   struct A {
diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/Interp/functions.cpp
index a8681aae0d582..5bb48ffc54dd5 100644
--- a/clang/test/AST/Interp/functions.cpp
+++ b/clang/test/AST/Interp/functions.cpp
@@ -178,6 +178,31 @@ namespace FunctionReturnType {
   static_assert(s.fp == nullptr, ""); // zero-initialized function pointer.
 }
 
+namespace Comparison {
+  void f(), g();
+  constexpr void (*pf)() = &f, (*pg)() = &g;
+
+  constexpr bool u13 = pf < pg; // ref-warning {{ordered comparison of function pointers}} \
+                                // ref-error {{must be initialized by a constant expression}} \
+                                // ref-note {{comparison between '&f' and '&g' has unspecified value}} \
+                                // expected-warning {{ordered comparison of function pointers}} \
+                                // expected-error {{must be initialized by a constant expression}} \
+                                // expected-note {{comparison between '&f' and '&g' has unspecified value}}
+
+  constexpr bool u14 = pf < (void(*)())nullptr; // ref-warning {{ordered comparison of function pointers}} \
+                                                // ref-error {{must be initialized by a constant expression}} \
+                                                // ref-note {{comparison between '&f' and 'nullptr' has unspecified value}} \
+                                                // expected-warning {{ordered comparison of function pointers}} \
+                                                // expected-error {{must be initialized by a constant expression}} \
+                                                // expected-note {{comparison between '&f' and 'nullptr' has unspecified value}}
+
+
+
+  static_assert(pf != pg, "");
+  static_assert(pf == &f, "");
+  static_assert(pg == &g, "");
+}
+
 }
 
 struct F {
diff --git a/clang/test/AST/Interp/if.cpp b/clang/test/AST/Interp/if.cpp
index 5d35868cd00ec..2449ace4dd6c6 100644
--- a/clang/test/AST/Interp/if.cpp
+++ b/clang/test/AST/Interp/if.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -fexperimental-new-constant-interpreter %s -verify
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only %s -verify=ref
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fexperimental-new-constant-interpreter %s -verify
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only %s -verify=ref
 
 // expected-no-diagnostics
 // ref-no-diagnostics
diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp
index f31a49088dc6c..5f4c0485ba772 100644
--- a/clang/test/AST/Interp/literals.cpp
+++ b/clang/test/AST/Interp/literals.cpp
@@ -80,6 +80,12 @@ static_assert(~255 == -256, "");
 static_assert(~INT_MIN == INT_MAX, "");
 static_assert(~INT_MAX == INT_MIN, "");
 
+static_assert(-(1 << 31), ""); // expected-error {{not an integral constant expression}} \
+                               // expected-note {{outside the range of representable values}} \
+                               // ref-error {{not an integral constant expression}} \
+                               // ref-note {{outside the range of representable values}} \
+
+
 enum E {};
 constexpr E e = static_cast<E>(0);
 static_assert(~e == -1, "");
@@ -101,6 +107,19 @@ constexpr int gimme(int k) {
 }
 static_assert(gimme(5) == 5, "");
 
+namespace PointerToBool {
+
+  constexpr void *N = nullptr;
+  constexpr bool B = N;
+  static_assert(!B, "");
+  static_assert(!N, "");
+
+  constexpr float F = 1.0;
+  constexpr const float *FP = &F;
+  static_assert(FP, "");
+  static_assert(!!FP, "");
+}
+
 namespace SizeOf {
   constexpr int soint = sizeof(int);
   constexpr int souint = sizeof(unsigned int);
@@ -706,6 +725,23 @@ namespace IncDec {
              // ref-note {{cannot refer to element -1 of array of 3 elements}}
     return *p;
   }
+
+  /// This used to leave a 0 on the stack instead of the previous
+  /// value of a.
+  constexpr int bug1Inc() {
+    int a = 3;
+    int b = a++;
+    return b;
+  }
+  static_assert(bug1Inc() == 3);
+
+  constexpr int bug1Dec() {
+    int a = 3;
+    int b = a--;
+    return b;
+  }
+  static_assert(bug1Dec() == 3);
+
 };
 #endif
 
@@ -778,4 +814,18 @@ constexpr int ignoredDecls() {
   return F{12}.a;
 }
 static_assert(ignoredDecls() == 12, "");
+
+struct A{};
+constexpr int ignoredExprs() {
+  (void)(1 / 2);
+  A a;
+  a; // expected-warning {{unused}} \
+     // ref-warning {{unused}}
+  (void)a;
+  (a); // expected-warning {{unused}} \
+       // ref-warning {{unused}}
+
+  return 0;
+}
+
 #endif
diff --git a/clang/test/AST/ast-dump-if-json.cpp b/clang/test/AST/ast-dump-if-json.cpp
index 9ec68ed6c66f6..fe8fc88cdb751 100644
--- a/clang/test/AST/ast-dump-if-json.cpp
+++ b/clang/test/AST/ast-dump-if-json.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple x86_64-pc-linux -std=c++2b -ast-dump=json %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux -std=c++23 -ast-dump=json %s | FileCheck %s
 
 void func(int val) {
   if (val)
diff --git a/clang/test/AST/ast-dump-stmt.cpp b/clang/test/AST/ast-dump-stmt.cpp
index 6a1b0d366eda6..4be66756e6c1e 100644
--- a/clang/test/AST/ast-dump-stmt.cpp
+++ b/clang/test/AST/ast-dump-stmt.cpp
@@ -1,10 +1,10 @@
 // Test without serialization:
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-linux-gnu -fcxx-exceptions -ast-dump %s \
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-linux-gnu -fcxx-exceptions -ast-dump %s \
 // RUN: | FileCheck -strict-whitespace %s
 //
 // Test with serialization:
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-linux-gnu -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -x c++ -std=c++2b -triple x86_64-linux-gnu -fcxx-exceptions -include-pch %t -ast-dump-all /dev/null \
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-linux-gnu -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -std=c++23 -triple x86_64-linux-gnu -fcxx-exceptions -include-pch %t -ast-dump-all /dev/null \
 // RUN: | sed -e "s/ <undeserialized declarations>//" -e "s/ imported//" \
 // RUN: | FileCheck -strict-whitespace %s
 
diff --git a/clang/test/Analysis/out-of-bounds-false-positive.c b/clang/test/Analysis/array-bound-v2-constraint-check.c
similarity index 94%
rename from clang/test/Analysis/out-of-bounds-false-positive.c
rename to clang/test/Analysis/array-bound-v2-constraint-check.c
index 2c63f0791ecb2..dc138196b56a9 100644
--- a/clang/test/Analysis/out-of-bounds-false-positive.c
+++ b/clang/test/Analysis/array-bound-v2-constraint-check.c
@@ -8,12 +8,11 @@ typedef unsigned long long size_t;
 const char a[] = "abcd"; // extent: 5 bytes
 
 void symbolic_size_t_and_int0(size_t len) {
-  // FIXME: Should not warn for this.
-  (void)a[len + 1]; // expected-warning {{Out of bound memory access}}
+  (void)a[len + 1]; // no-warning
   // We infered that the 'len' must be in a specific range to make the previous indexing valid.
   // len: [0,3]
-  clang_analyzer_eval(len <= 3); // expected - warning {{TRUE}}
-  clang_analyzer_eval(len <= 2); // expected - warning {{UNKNOWN}}
+  clang_analyzer_eval(len <= 3); // expected-warning {{TRUE}}
+  clang_analyzer_eval(len <= 2); // expected-warning {{UNKNOWN}}
 }
 
 void symbolic_size_t_and_int1(size_t len) {
diff --git a/clang/test/Analysis/taint-diagnostic-visitor.c b/clang/test/Analysis/taint-diagnostic-visitor.c
index 34414a209817a..663836836d3db 100644
--- a/clang/test/Analysis/taint-diagnostic-visitor.c
+++ b/clang/test/Analysis/taint-diagnostic-visitor.c
@@ -2,13 +2,24 @@
 
 // This file is for testing enhanced diagnostics produced by the GenericTaintChecker
 
+typedef __typeof(sizeof(int)) size_t;
+struct _IO_FILE;
+typedef struct _IO_FILE FILE;
+
 int scanf(const char *restrict format, ...);
 int system(const char *command);
+char* getenv( const char* env_var );
+size_t strlen( const char* str );
+void *malloc(size_t size );
+void free( void *ptr );
+char *fgets(char *str, int n, FILE *stream);
+FILE *stdin;
 
 void taintDiagnostic(void)
 {
   char buf[128];
   scanf("%s", buf); // expected-note {{Taint originated here}}
+                    // expected-note@-1 {{Taint propagated to the 2nd argument}}
   system(buf); // expected-warning {{Untrusted data is passed to a system call}} // expected-note {{Untrusted data is passed to a system call (CERT/STR02-C. Sanitize data passed to complex subsystems)}}
 }
 
@@ -16,6 +27,7 @@ int taintDiagnosticOutOfBound(void) {
   int index;
   int Array[] = {1, 2, 3, 4, 5};
   scanf("%d", &index); // expected-note {{Taint originated here}}
+                       // expected-note@-1 {{Taint propagated to the 2nd argument}}
   return Array[index]; // expected-warning {{Out of bound memory access (index is tainted)}}
                        // expected-note@-1 {{Out of bound memory access (index is tainted)}}
 }
@@ -23,6 +35,7 @@ int taintDiagnosticOutOfBound(void) {
 int taintDiagnosticDivZero(int operand) {
   scanf("%d", &operand); // expected-note {{Value assigned to 'operand'}}
                          // expected-note@-1 {{Taint originated here}}
+                         // expected-note@-2 {{Taint propagated to the 2nd argument}}
   return 10 / operand; // expected-warning {{Division by a tainted value, possibly zero}}
                        // expected-note@-1 {{Division by a tainted value, possibly zero}}
 }
@@ -31,6 +44,72 @@ void taintDiagnosticVLA(void) {
   int x;
   scanf("%d", &x); // expected-note {{Value assigned to 'x'}}
                    // expected-note@-1 {{Taint originated here}}
+                   // expected-note@-2 {{Taint propagated to the 2nd argument}}
   int vla[x]; // expected-warning {{Declared variable-length array (VLA) has tainted size}}
               // expected-note@-1 {{Declared variable-length array (VLA) has tainted size}}
 }
+
+
+// Tests if the originated note is correctly placed even if the path is
+// propagating through variables and expressions
+char *taintDiagnosticPropagation(){
+  char *pathbuf;
+  char *pathlist=getenv("PATH"); // expected-note {{Taint originated here}}
+                                 // expected-note@-1 {{Taint propagated to the return value}}
+  if (pathlist){ // expected-note {{Assuming 'pathlist' is non-null}}
+	               // expected-note@-1 {{Taking true branch}}
+    pathbuf=(char*) malloc(strlen(pathlist)+1); // expected-warning{{Untrusted data is used to specify the buffer size}}
+                                                // expected-note@-1{{Untrusted data is used to specify the buffer size}}
+                                                // expected-note@-2 {{Taint propagated to the return value}}
+    return pathbuf;
+  }
+  return 0;
+}
+
+// Taint origin should be marked correctly even if there are multiple taint
+// sources in the function
+char *taintDiagnosticPropagation2(){
+  char *pathbuf;
+  char *user_env2=getenv("USER_ENV_VAR2");//unrelated taint source
+  char *pathlist=getenv("PATH"); // expected-note {{Taint originated here}}
+                                 // expected-note@-1 {{Taint propagated to the return value}}
+  char *user_env=getenv("USER_ENV_VAR");//unrelated taint source
+  if (pathlist){ // expected-note {{Assuming 'pathlist' is non-null}}
+	               // expected-note@-1 {{Taking true branch}}
+    pathbuf=(char*) malloc(strlen(pathlist)+1); // expected-warning{{Untrusted data is used to specify the buffer size}}
+                                                // expected-note@-1{{Untrusted data is used to specify the buffer size}}
+                                                // expected-note@-2 {{Taint propagated to the return value}}
+    return pathbuf;
+  }
+  return 0;
+}
+
+void testReadStdIn(){
+  char buf[1024];
+  fgets(buf, sizeof(buf), stdin);// expected-note {{Taint originated here}}
+                                 // expected-note@-1 {{Taint propagated to the 1st argument}}
+  system(buf);// expected-warning {{Untrusted data is passed to a system call}}
+              // expected-note@-1 {{Untrusted data is passed to a system call (CERT/STR02-C. Sanitize data passed to complex subsystems)}}
+
+}
+
+void multipleTaintSources(void) {
+  int x,y,z;
+  scanf("%d", &x); // expected-note {{Taint originated here}}
+                   // expected-note@-1 {{Taint propagated to the 2nd argument}}
+  scanf("%d", &y); // expected-note {{Taint originated here}}
+                   // expected-note@-1 {{Taint propagated to the 2nd argument}}
+  scanf("%d", &z);
+  int* ptr = (int*) malloc(y + x); // expected-warning {{Untrusted data is used to specify the buffer size}}
+                                   // expected-note@-1{{Untrusted data is used to specify the buffer size}}
+  free (ptr);
+}
+
+void multipleTaintedArgs(void) {
+  int x,y;
+  scanf("%d %d", &x, &y); // expected-note {{Taint originated here}}
+                          // expected-note@-1 {{Taint propagated to the 2nd argument, 3rd argument}}
+  int* ptr = (int*) malloc(x + y); // expected-warning {{Untrusted data is used to specify the buffer size}}
+                                   // expected-note@-1{{Untrusted data is used to specify the buffer size}}
+  free (ptr);
+}
diff --git a/clang/test/Analysis/taint-generic.c b/clang/test/Analysis/taint-generic.c
index 626e01e39d158..62e1f570b6622 100644
--- a/clang/test/Analysis/taint-generic.c
+++ b/clang/test/Analysis/taint-generic.c
@@ -156,6 +156,28 @@ void bufferGetchar(int x) {
   Buffer[m] = 1;  //expected-warning {{Out of bound memory access (index is tainted)}}
 }
 
+extern const unsigned short int **__ctype_b_loc (void);
+enum { _ISdigit = 2048 };
+# define isdigit(c) ((*__ctype_b_loc ())[(int) (c)] & (unsigned short int) _ISdigit)
+
+int isdigitImplFalsePositive(void) {
+  // If this code no longer produces a bug report, then consider removing the
+  // special case that disables buffer overflow reports coming from the isXXXXX
+  // macros in ctypes.h.
+  int c = getchar();
+  return ((*__ctype_b_loc ())[(int) (c)] & (unsigned short int) _ISdigit);
+  //expected-warning@-1 {{Out of bound memory access (index is tainted)}}
+}
+
+int isdigitSuppressed(void) {
+  // Same code as above, but reports are suppressed based on macro name:
+  int c = getchar();
+  return isdigit(c); //no-warning
+}
+
+// Some later tests use isdigit as a function, so we need to undef it:
+#undef isdigit
+
 void testUncontrolledFormatString(char **p) {
   char s[80];
   fscanf(stdin, "%s", s);
diff --git a/clang/test/Analysis/taint-tester.c b/clang/test/Analysis/taint-tester.c
index 800a771452bef..ddfa91021825f 100644
--- a/clang/test/Analysis/taint-tester.c
+++ b/clang/test/Analysis/taint-tester.c
@@ -122,7 +122,7 @@ void stdinTest2(FILE *pIn) {
   fscanf(pp, "%d", &ii);
   int jj = ii;// expected-warning + {{tainted}}
 
-  fscanf(p, "%d", &ii);
+  fscanf(p, "%d", &ii);// expected-warning + {{tainted}}
   int jj2 = ii;// expected-warning + {{tainted}}
 
   ii = 3;
diff --git a/clang/test/C/C2x/n3042.c b/clang/test/C/C2x/n3042.c
index e949890bf82f1..3f869013af480 100644
--- a/clang/test/C/C2x/n3042.c
+++ b/clang/test/C/C2x/n3042.c
@@ -1,10 +1,7 @@
 // RUN: %clang_cc1 -verify -ffreestanding -Wno-unused -std=c2x %s
 
-/* WG14 N3042: partial
+/* WG14 N3042: full
  * Introduce the nullptr constant
- *
- * Claiming partial support for this feature until the WG14 NB comments can be
- * resolved to know what the correct behavior really should be.
  */
 
 #include <stddef.h>
@@ -21,25 +18,17 @@
 void questionable_behaviors() {
   nullptr_t val;
 
-  // FIXME: This code is intended to be rejected by C and is accepted by C++.
-  // We've filed an NB comment with WG14 about the incompatibility.
+  // This code is intended to be rejected by C and is accepted by C++. We filed
+  // an NB comment asking for this to be changed, but WG14 declined.
   (void)(1 ? val : 0);     // expected-error {{non-pointer operand type 'int' incompatible with nullptr}}
   (void)(1 ? nullptr : 0); // expected-error {{non-pointer operand type 'int' incompatible with nullptr}}
 
-  // FIXME: This code is intended to be accepted by C and is rejected by C++.
-  // We're following the C++ semantics until WG14 has resolved the NB comments
-  // we've filed about the incompatibility.
-  _Bool another = val;    // expected-error {{initializing 'bool' with an expression of incompatible type 'nullptr_t'}}
-  another = val;          // expected-error {{assigning to 'bool' from incompatible type 'nullptr_t'}}
-  _Bool again = nullptr;  // expected-error {{initializing 'bool' with an expression of incompatible type 'nullptr_t'}}
-  again = nullptr;        // expected-error {{assigning to 'bool' from incompatible type 'nullptr_t'}}
-
-  // FIXME: This code is intended to be rejected by C and is accepted by C++.
-  // We've filed an NB comment with WG14 about the incompatibility.
-  val = 0; // expected-error {{assigning to 'nullptr_t' from incompatible type 'int'}}
-
-  // Not accepted in C++ but might want to accept in C as a null pointer constant?
-  val = (void *)0; // expected-error {{assigning to 'nullptr_t' from incompatible type 'void *'}}
+  // This code is intended to be accepted by C and is rejected by C++. We filed
+  // an NB comment asking for this to be changed, but WG14 declined.
+  _Bool another = val;    // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+  another = val;          // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+  _Bool again = nullptr;  // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+  again = nullptr;        // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
 }
 
 void test() {
@@ -67,6 +56,14 @@ void test() {
   // How about the null pointer named constant?
   &nullptr; // expected-error {{cannot take the address of an rvalue of type 'nullptr_t'}}
 
+  // Assignment from a null pointer constant to a nullptr_t is valid.
+  null_val = 0;
+  null_val = (void *)0;
+
+  // Assignment from a nullptr_t to a pointer is also valid.
+  typed_ptr = null_val;
+  void *other_ptr = null_val;
+
   // Can it be used in all the places a scalar can be used?
   if (null_val) {}
   if (!null_val) {}
@@ -162,18 +159,15 @@ void test() {
 }
 
 // Can we use it as a function parameter?
-void null_param(nullptr_t); // expected-note 2 {{passing argument to parameter here}}
+void null_param(nullptr_t);
 
 void other_test() {
   // Can we call the function properly?
   null_param(nullptr);
 
-  // Do we get reasonable diagnostics when we can't call the function?
-  null_param((void *)0); // expected-error {{passing 'void *' to parameter of incompatible type 'nullptr_t'}}
-
-  // FIXME: The paper requires this to be rejected, but it is accepted in C++.
-  // This should be addressed after WG14 has processed national body comments.
-  null_param(0);         // expected-error {{passing 'int' to parameter of incompatible type 'nullptr_t'}}
+  // We can pass any kind of null pointer constant.
+  null_param((void *)0);
+  null_param(0);
 }
 
 
@@ -182,3 +176,7 @@ void format_specifiers() {
   // Don't warn when using nullptr with %p.
   printf("%p", nullptr);
 }
+
+// Ensure that conversion from a null pointer constant to nullptr_t is
+// valid in a constant expression.
+static_assert((nullptr_t){} == 0);
diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp
index 936ca7c5100a2..166bd97e2731c 100644
--- a/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp
+++ b/clang/test/CXX/class/class.compare/class.compare.default/p3.cpp
@@ -1,6 +1,8 @@
 // This test is for the [class.compare.default]p3 added by P2002R0
+// Also covers modifications made by P2448R2 and extension warnings
 
 // RUN: %clang_cc1 -std=c++2a -verify %s
+// RUN: %clang_cc1 -std=c++2a -Wc++23-default-comp-relaxed-constexpr -verify=expected,extension %s
 
 namespace std {
   struct strong_ordering {
@@ -80,10 +82,10 @@ struct TestB {
 };
 
 struct C {
-  friend bool operator==(const C&, const C&); // expected-note {{previous}} expected-note 2{{here}}
+  friend bool operator==(const C&, const C&); // expected-note {{previous}} extension-note 2{{non-constexpr comparison function declared here}}
   friend bool operator!=(const C&, const C&) = default; // expected-note {{previous}}
 
-  friend std::strong_ordering operator<=>(const C&, const C&); // expected-note {{previous}} expected-note 2{{here}}
+  friend std::strong_ordering operator<=>(const C&, const C&); // expected-note {{previous}} extension-note 2{{non-constexpr comparison function declared here}}
   friend bool operator<(const C&, const C&) = default; // expected-note {{previous}}
   friend bool operator<=(const C&, const C&) = default; // expected-note {{previous}}
   friend bool operator>(const C&, const C&) = default; // expected-note {{previous}}
@@ -127,23 +129,23 @@ struct TestD {
 
 struct E {
   A a;
-  C c; // expected-note 2{{non-constexpr comparison function would be used to compare member 'c'}}
+  C c; // extension-note 2{{non-constexpr comparison function would be used to compare member 'c'}}
   A b;
-  friend constexpr bool operator==(const E&, const E&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
+  friend constexpr bool operator==(const E&, const E&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
   friend constexpr bool operator!=(const E&, const E&) = default;
 
-  friend constexpr std::strong_ordering operator<=>(const E&, const E&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
+  friend constexpr std::strong_ordering operator<=>(const E&, const E&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
   friend constexpr bool operator<(const E&, const E&) = default;
   friend constexpr bool operator<=(const E&, const E&) = default;
   friend constexpr bool operator>(const E&, const E&) = default;
   friend constexpr bool operator>=(const E&, const E&) = default;
 };
 
-struct E2 : A, C { // expected-note 2{{non-constexpr comparison function would be used to compare base class 'C'}}
-  friend constexpr bool operator==(const E2&, const E2&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
+struct E2 : A, C { // extension-note 2{{non-constexpr comparison function would be used to compare base class 'C'}}
+  friend constexpr bool operator==(const E2&, const E2&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
   friend constexpr bool operator!=(const E2&, const E2&) = default;
 
-  friend constexpr std::strong_ordering operator<=>(const E2&, const E2&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
+  friend constexpr std::strong_ordering operator<=>(const E2&, const E2&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
   friend constexpr bool operator<(const E2&, const E2&) = default;
   friend constexpr bool operator<=(const E2&, const E2&) = default;
   friend constexpr bool operator>(const E2&, const E2&) = default;
@@ -151,14 +153,14 @@ struct E2 : A, C { // expected-note 2{{non-constexpr comparison function would b
 };
 
 struct F {
-  friend bool operator==(const F&, const F&); // expected-note {{here}}
-  friend constexpr bool operator!=(const F&, const F&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
-
-  friend std::strong_ordering operator<=>(const F&, const F&); // expected-note 4{{here}}
-  friend constexpr bool operator<(const F&, const F&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
-  friend constexpr bool operator<=(const F&, const F&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
-  friend constexpr bool operator>(const F&, const F&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
-  friend constexpr bool operator>=(const F&, const F&) = default; // expected-error {{cannot be declared constexpr because it invokes a non-constexpr comparison function}}
+  friend bool operator==(const F&, const F&); // extension-note {{non-constexpr comparison function declared here}}
+  friend constexpr bool operator!=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
+
+  friend std::strong_ordering operator<=>(const F&, const F&); // extension-note 4{{non-constexpr comparison function declared here}}
+  friend constexpr bool operator<(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
+  friend constexpr bool operator<=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
+  friend constexpr bool operator>(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
+  friend constexpr bool operator>=(const F&, const F&) = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
 };
 
 // No implicit 'constexpr' if it's not the first declaration.
diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp
index 02adf3d51dede..fab2abb7dee00 100644
--- a/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp
+++ b/clang/test/CXX/class/class.compare/class.compare.default/p4.cpp
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -std=c++2a -verify %s
+// RUN: %clang_cc1 -std=c++2a -Wc++23-default-comp-relaxed-constexpr -verify=expected,extension %s
 
 // This test is for [class.compare.default]p3 as modified and renumbered to p4
 // by P2002R0.
+// Also covers modifications made by P2448R2 and extension warnings
 
 namespace std {
   struct strong_ordering {
@@ -76,14 +78,13 @@ void use_g(G g) {
 }
 
 struct H {
-  bool operator==(const H&) const; // expected-note {{here}}
+  bool operator==(const H&) const; // extension-note {{non-constexpr comparison function declared here}}
   constexpr std::strong_ordering operator<=>(const H&) const { return std::strong_ordering::equal; }
 };
 
 struct I {
-  H h; // expected-note {{used to compare}}
-  // expected-error@+1 {{defaulted definition of three-way comparison operator cannot be declared constexpr because the corresponding implicit 'operator==' invokes a non-constexpr comparison function}}
-  constexpr std::strong_ordering operator<=>(const I&) const = default;
+  H h; // extension-note {{non-constexpr comparison function would be used to compare member 'h'}}
+  constexpr std::strong_ordering operator<=>(const I&) const = default; // extension-warning {{implicit 'operator=='  invokes a non-constexpr comparison function is a C++23 extension}}
 };
 
 struct J {
@@ -144,3 +145,19 @@ namespace NoInjectionIfOperatorEqualsDeclared {
   };
   bool test_d = D() == D();
 }
+
+namespace GH61238 {
+template <typename A> struct my_struct {
+    A value; // extension-note {{non-constexpr comparison function would be used to compare member 'value'}}
+
+    constexpr friend bool operator==(const my_struct &, const my_struct &) noexcept = default; // extension-warning {{declared constexpr but invokes a non-constexpr comparison function is a C++23 extension}}
+};
+
+struct non_constexpr_type {
+    friend bool operator==(non_constexpr_type, non_constexpr_type) noexcept { // extension-note {{non-constexpr comparison function declared here}}
+        return false;
+    }
+};
+
+my_struct<non_constexpr_type> obj; // extension-note {{in instantiation of template class 'GH61238::my_struct<GH61238::non_constexpr_type>' requested here}}
+}
diff --git a/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp b/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp
index e1a25c71c6ed1..24aef8a1a91c3 100644
--- a/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp
+++ b/clang/test/CXX/class/class.init/class.copy.elision/p3.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -fcxx-exceptions                       -verify=expected,cxx11_2b,cxx2b    %s
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions                       -verify=expected,cxx98_20,cxx11_2b,cxx11_20 %s
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fcxx-exceptions                       -verify=expected,cxx98_20,cxx11_2b,cxx11_20 %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions                       -verify=expected,cxx11_23,cxx23    %s
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions                       -verify=expected,cxx98_20,cxx11_23,cxx11_20 %s
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fcxx-exceptions                       -verify=expected,cxx98_20,cxx11_23,cxx11_20 %s
 // RUN: %clang_cc1 -std=c++98 -fsyntax-only -fcxx-exceptions -Wno-c++11-extensions -verify=expected,cxx98_20,cxx98 %s
 
 namespace test_delete_function {
@@ -98,7 +98,7 @@ struct A1 {
   A1(const A1 &);
   A1(A1 &&) = delete;
   // expected-note@-1 2{{'A1' has been explicitly marked deleted here}}
-  // cxx11_2b-note@-2 3{{'A1' has been explicitly marked deleted here}}
+  // cxx11_23-note@-2 3{{'A1' has been explicitly marked deleted here}}
 };
 void test1() {
   try {
@@ -208,7 +208,7 @@ struct NeedRvalueRef {
 struct NeedValue {
   NeedValue(A1); // cxx98-note 2 {{passing argument to parameter here}}
   NeedValue(A2);
-  NeedValue(B1); // cxx11_2b-note 2 {{passing argument to parameter here}}
+  NeedValue(B1); // cxx11_23-note 2 {{passing argument to parameter here}}
   NeedValue(B2);
 };
 
@@ -267,14 +267,14 @@ NeedValue test_2_3() {
 struct B1 {
   B1();
   B1(const B1 &);
-  B1(B1 &&) = delete; // cxx11_2b-note 3 {{'B1' has been explicitly marked deleted here}}
+  B1(B1 &&) = delete; // cxx11_23-note 3 {{'B1' has been explicitly marked deleted here}}
                       // cxx98-note@-1 {{'B1' has been explicitly marked deleted here}}
 };
 NeedValue test_3_1() {
   // not rvalue reference
   // same type
   B1 b;
-  return b; // cxx11_2b-error {{call to deleted constructor of 'B1'}}
+  return b; // cxx11_23-error {{call to deleted constructor of 'B1'}}
 }
 class DerivedB1 : public B1 {};
 B1 test_3_2() {
@@ -287,7 +287,7 @@ NeedValue test_3_3() {
   // not rvalue reference
   // not same type
   DerivedB1 b;
-  return b; // cxx11_2b-error {{call to deleted constructor of 'B1'}}
+  return b; // cxx11_23-error {{call to deleted constructor of 'B1'}}
 }
 
 struct B2 {
@@ -295,14 +295,14 @@ struct B2 {
   B2(const B2 &);
 
 private:
-  B2(B2 &&); // cxx11_2b-note 3 {{declared private here}}
+  B2(B2 &&); // cxx11_23-note 3 {{declared private here}}
              // cxx98-note@-1 {{declared private here}}
 };
 NeedValue test_4_1() {
   // not rvalue reference
   // same type
   B2 b;
-  return b; // cxx11_2b-error {{calling a private constructor of class 'test_ctor_param_rvalue_ref::B2'}}
+  return b; // cxx11_23-error {{calling a private constructor of class 'test_ctor_param_rvalue_ref::B2'}}
 }
 class DerivedB2 : public B2 {};
 B2 test_4_2() {
@@ -315,7 +315,7 @@ NeedValue test_4_3() {
   // not rvalue reference
   // not same type
   DerivedB2 b;
-  return b; // cxx11_2b-error {{calling a private constructor of class 'test_ctor_param_rvalue_ref::B2'}}
+  return b; // cxx11_23-error {{calling a private constructor of class 'test_ctor_param_rvalue_ref::B2'}}
 }
 } // namespace test_ctor_param_rvalue_ref
 
@@ -323,7 +323,7 @@ namespace test_lvalue_ref_is_not_moved_from {
 
 struct Target {};
 // expected-note@-1  {{candidate constructor (the implicit copy constructor) not viable}}
-// cxx11_2b-note@-2  {{candidate constructor (the implicit move constructor) not viable}}
+// cxx11_23-note@-2  {{candidate constructor (the implicit move constructor) not viable}}
 
 struct CopyOnly {
   CopyOnly(CopyOnly &&) = delete; // expected-note {{has been explicitly marked deleted here}}
@@ -333,7 +333,7 @@ struct CopyOnly {
 };
 
 struct MoveOnly {
-  MoveOnly(MoveOnly &&); // cxx11_2b-note {{copy constructor is implicitly deleted because}}
+  MoveOnly(MoveOnly &&); // cxx11_23-note {{copy constructor is implicitly deleted because}}
   operator Target() &&;  // expected-note {{candidate function not viable}}
 };
 
@@ -352,7 +352,7 @@ CopyOnly t2() {
 
 MoveOnly t3() {
     MoveOnly& r = moveonly;
-    return r; // cxx11_2b-error {{call to implicitly-deleted copy constructor}}
+    return r; // cxx11_23-error {{call to implicitly-deleted copy constructor}}
 }
 
 MoveOnly t4() {
@@ -437,11 +437,11 @@ ConstCopyOnly t1() {
 struct NonConstCopyOnly {
   NonConstCopyOnly();
   NonConstCopyOnly(NonConstCopyOnly &);
-  NonConstCopyOnly(const NonConstCopyOnly &) = delete; // cxx11_2b-note {{marked deleted here}}
+  NonConstCopyOnly(const NonConstCopyOnly &) = delete; // cxx11_23-note {{marked deleted here}}
 };
 NonConstCopyOnly t2() {
   NonConstCopyOnly x;
-  return x; // cxx11_2b-error {{call to deleted constructor}}
+  return x; // cxx11_23-error {{call to deleted constructor}}
 }
 
 } // namespace test_constandnonconstcopy
@@ -469,10 +469,10 @@ C test2(D x) { return x; } // expected-error {{invokes a deleted function}}
 namespace test_simpler_implicit_move {
 
 struct CopyOnly {
-  CopyOnly(); // cxx2b-note {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
-  // cxx2b-note@-1 {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
-  CopyOnly(CopyOnly &); // cxx2b-note {{candidate constructor not viable: expects an lvalue for 1st argument}}
-  // cxx2b-note@-1 {{candidate constructor not viable: expects an lvalue for 1st argument}}
+  CopyOnly(); // cxx23-note {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+  // cxx23-note@-1 {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+  CopyOnly(CopyOnly &); // cxx23-note {{candidate constructor not viable: expects an lvalue for 1st argument}}
+  // cxx23-note@-1 {{candidate constructor not viable: expects an lvalue for 1st argument}}
 };
 struct MoveOnly {
   MoveOnly();
@@ -490,7 +490,7 @@ CopyOnly test2(bool b) {
   if (b) {
     return w1;
   } else {
-    return w2; // cxx2b-error {{no matching constructor for initialization}}
+    return w2; // cxx23-error {{no matching constructor for initialization}}
   }
 }
 
@@ -505,7 +505,7 @@ MoveOnly &&test4() {
 
 void test5() try {
   CopyOnly x;
-  throw x; // cxx2b-error {{no matching constructor for initialization}}
+  throw x; // cxx23-error {{no matching constructor for initialization}}
 } catch (...) {
 }
 
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp
index 88334031ec456..7ad2e582a8126 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -std=c++2a -verify=expected,cxx2a %s
-// RUN: %clang_cc1 -std=c++2b -verify=expected %s
+// RUN: %clang_cc1 -std=c++23 -verify=expected %s
 
 // p3: if the function is a constructor or destructor, its class shall not have
 // any virtual base classes;
@@ -14,13 +14,13 @@ namespace contents {
   struct A {
     constexpr ~A() {
       return;
-      goto x; // cxx2a-warning {{use of this statement in a constexpr function is a C++2b extension}}
+      goto x; // cxx2a-warning {{use of this statement in a constexpr function is a C++23 extension}}
       x: ;
     }
   };
   struct B {
     constexpr ~B() {
-    x:; // cxx2a-warning {{use of this statement in a constexpr function is a C++2b extension}}
+    x:; // cxx2a-warning {{use of this statement in a constexpr function is a C++23 extension}}
     }
   };
   struct Nonlit { // cxx2a-note {{'Nonlit' is not literal because}}
@@ -29,19 +29,19 @@ namespace contents {
   struct C {
     constexpr ~C() {
       return;
-      Nonlit nl; // cxx2a-error {{variable of non-literal type 'Nonlit' cannot be defined in a constexpr function before C++2b}}
+      Nonlit nl; // cxx2a-error {{variable of non-literal type 'Nonlit' cannot be defined in a constexpr function before C++23}}
     }
   };
   struct D {
     constexpr ~D() {
       return;
-      static int a; // cxx2a-warning {{definition of a static variable in a constexpr function is a C++2b extension}}
+      static int a; // cxx2a-warning {{definition of a static variable in a constexpr function is a C++23 extension}}
     }
   };
   struct E {
     constexpr ~E() {
       return;
-      thread_local int e; // cxx2a-warning {{definition of a thread_local variable in a constexpr function is a C++2b extension}}
+      thread_local int e; // cxx2a-warning {{definition of a thread_local variable in a constexpr function is a C++23 extension}}
     }
   };
   struct F {
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp
index 3ba5cbba79ce2..c07502c0555b5 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3-2b.cpp
@@ -1,28 +1,28 @@
-// RUN: %clang_cc1 -verify -std=c++2b -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -verify -std=c++23 -Wpre-c++23-compat %s
 
 constexpr int h(int n) {
   if (!n)
     return 0;
-  static const int m = n; // expected-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  static const int m = n; // expected-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int i(int n) {
   if (!n)
     return 0;
-  thread_local const int m = n; // expected-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  thread_local const int m = n; // expected-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int g() { // expected-error {{constexpr function never produces a constant expression}}
   goto test;        // expected-note {{subexpression not valid in a constant expression}} \
-           // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++2b}}
+           // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}}
 test:
   return 0;
 }
 
 constexpr void h() {
-label:; // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++2b}}
+label:; // expected-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}}
 }
 
 struct NonLiteral { // expected-note 2 {{'NonLiteral' is not literal}}
@@ -31,18 +31,18 @@ struct NonLiteral { // expected-note 2 {{'NonLiteral' is not literal}}
 
 constexpr void non_literal() { // expected-error {{constexpr function never produces a constant expression}}
   NonLiteral n;                // expected-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}} \
-                               // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++2b}}
+                               // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}}
 }
 
 constexpr void non_literal2(bool b) {
   if (!b)
-    NonLiteral n; // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++2b}}
+    NonLiteral n; // expected-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}}
 }
 
 constexpr int c_thread_local(int n) {
   if (!n)
     return 0;
-  static _Thread_local int a; // expected-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  static _Thread_local int a; // expected-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   _Thread_local int b;        // // expected-error {{'_Thread_local' variables must have global storage}}
   return 0;
 }
@@ -50,7 +50,7 @@ constexpr int c_thread_local(int n) {
 constexpr int gnu_thread_local(int n) {
   if (!n)
     return 0;
-  static __thread int a; // expected-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  static __thread int a; // expected-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   __thread int b;        // expected-error {{'__thread' variables must have global storage}}
   return 0;
 }
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
index 2db0995127506..d545f49fa5db3 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,beforecxx14,beforecxx20,beforecxx2b -std=c++11 %s
-// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,beforecxx20,beforecxx2b -std=c++14 %s
-// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20,beforecxx2b -std=c++20  %s
-// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20 -std=c++2b %s
+// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,beforecxx14,beforecxx20,beforecxx23 -std=c++11 %s
+// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,beforecxx20,beforecxx23 -std=c++14 %s
+// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20,beforecxx23 -std=c++20  %s
+// RUN: %clang_cc1 -fcxx-exceptions -verify=expected,aftercxx14,aftercxx20 -std=c++23 %s
 
 namespace N {
   typedef char C;
@@ -41,6 +41,8 @@ struct T : SS, NonLiteral {
   virtual constexpr int OutOfLineVirtual() const; // beforecxx20-error {{virtual function cannot be constexpr}}
 
   //  - its return type shall be a literal type;
+  // Once we support P2448R2 constexpr functions will be allowd to return non-literal types
+  // The destructor will also be allowed
   constexpr NonLiteral NonLiteralReturn() const { return {}; } // expected-error {{constexpr function's return type 'NonLiteral' is not a literal type}}
   constexpr void VoidReturn() const { return; }                // beforecxx14-error {{constexpr function's return type 'void' is not a literal type}}
   constexpr ~T();                                              // beforecxx20-error {{destructor cannot be declared constexpr}}
@@ -49,6 +51,7 @@ struct T : SS, NonLiteral {
   constexpr F NonLiteralReturn2; // ok until definition
 
   //  - each of its parameter types shall be a literal type;
+  // Once we support P2448R2 constexpr functions will be allowd to have parameters of non-literal types
   constexpr int NonLiteralParam(NonLiteral) const { return 0; } // expected-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}}
   typedef int G(NonLiteral) const;
   constexpr G NonLiteralParam2; // ok until definition
@@ -124,12 +127,12 @@ constexpr int DisallowedStmtsCXX14_1(bool b) {
 constexpr int DisallowedStmtsCXX14_2() {
   return 0; // beforecxx14-note {{previous}}
   //  - a goto statement
-  goto x; // beforecxx2b-warning {{use of this statement in a constexpr function is a C++2b extension}}
+  goto x; // beforecxx23-warning {{use of this statement in a constexpr function is a C++23 extension}}
   x:;
     return 0; // beforecxx14-warning {{multiple return}}
 }
 constexpr int DisallowedStmtsCXX14_2_1() {
-merp: // beforecxx2b-warning {{use of this statement in a constexpr function is a C++2b extension}}
+merp: // beforecxx23-warning {{use of this statement in a constexpr function is a C++23 extension}}
   return 0;
 }
 constexpr int DisallowedStmtsCXX14_3() {
@@ -141,14 +144,14 @@ constexpr int DisallowedStmtsCXX14_3() {
 constexpr int DisallowedStmtsCXX14_4() {
   //  - a definition of a variable of non-literal type
   return 0;
-  NonLiteral nl; // beforecxx2b-error {{variable of non-literal type 'NonLiteral' cannot be defined in a constexpr function before C++2b}} \
-                 // beforecxx2b-note@14  {{'NonLiteral' is not literal}}
+  NonLiteral nl; // beforecxx23-error {{variable of non-literal type 'NonLiteral' cannot be defined in a constexpr function before C++23}} \
+                 // beforecxx23-note@14  {{'NonLiteral' is not literal}}
 }
 
 constexpr int DisallowedStmtsCXX14_5() {
   return 0;
   //  - a definition of a variable of static storage duration
-  static constexpr int n = 123; // beforecxx2b-warning {{definition of a static variable in a constexpr function is a C++2b extension}} \
+  static constexpr int n = 123; // beforecxx23-warning {{definition of a static variable in a constexpr function is a C++23 extension}} \
                                //  beforecxx14-warning {{variable declaration in a constexpr function is a C++14 extension}}
 }
 
@@ -156,7 +159,7 @@ constexpr int DisallowedStmtsCXX14_6() {
   //  - a definition of a variable of thread storage duration
   return 0;
   thread_local constexpr int n = 123; // beforecxx14-warning {{variable declaration in a constexpr function is a C++14 extension}} \
-                                      // beforecxx2b-warning {{definition of a thread_local variable in a constexpr function is a C++2b extension}}
+                                      // beforecxx23-warning {{definition of a thread_local variable in a constexpr function is a C++23 extension}}
 }
 constexpr int DisallowedStmtsCXX14_7() {
   //  - a definition of a variable for which no initialization is performed
@@ -262,7 +265,7 @@ namespace std_example {
   }
   constexpr int first(int n) {
     return 0;
-    static int value = n; // beforecxx2b-warning {{definition of a static variable in a constexpr function is a C++2b extension}} \
+    static int value = n; // beforecxx23-warning {{definition of a static variable in a constexpr function is a C++23 extension}} \
                           // beforecxx14-warning {{variable declaration in a constexpr function is a C++14 extension}}
   }
   constexpr int uninit() {
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp
index aed2f29f67c3a..b6f3e2e721f38 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp
@@ -242,6 +242,7 @@ constexpr int f(enable_shared_from_this<int>);
 
 // - every constructor involved in initializing non-static data members and base
 //   class sub-objects shall be a constexpr constructor.
+// This will no longer be the case once we support P2448R2
 struct ConstexprBaseMemberCtors : Literal {
   Literal l;
 
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.spec.auto/p7-cxx14.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.spec.auto/p7-cxx14.cpp
index 609ab8c999ff8..97e860f91dcd3 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.spec.auto/p7-cxx14.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.spec.auto/p7-cxx14.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify -std=c++2b -verify=expected,cxx2b    %s
+// RUN: %clang_cc1 -verify -std=c++23 -verify=expected,cxx23    %s
 // RUN: %clang_cc1 -verify -std=c++20 -verify=expected,cxx14_20 %s
 // RUN: %clang_cc1 -verify -std=c++14 -verify=expected,cxx14_20 %s
 
@@ -30,7 +30,7 @@ using Int = decltype(x3d);
 auto x4a = (i);
 decltype(auto) x4d = (i);
 using Int = decltype(x4a);
-using IntLRef = decltype(x4d); // cxx2b-note {{previous definition is here}}
+using IntLRef = decltype(x4d); // cxx23-note {{previous definition is here}}
 
 auto x5a = f();
 decltype(auto) x5d = f();
@@ -81,7 +81,7 @@ using Int = decltype(f2d(0));
 auto f3a(int n) { return (n); }
 decltype(auto) f3d(int n) { return (n); } // expected-warning {{reference to stack memory}}
 using Int = decltype(f3a(0));
-using IntLRef = decltype(f3d(0)); // cxx2b-error {{type alias redefinition with different types ('decltype(f3d(0))' (aka 'int &&') vs 'decltype(x4d)' (aka 'int &'))}}
+using IntLRef = decltype(f3d(0)); // cxx23-error {{type alias redefinition with different types ('decltype(f3d(0))' (aka 'int &&') vs 'decltype(x4d)' (aka 'int &'))}}
 
 auto f4a(int n) { return f(); }
 decltype(auto) f4d(int n) { return f(); }
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.auto.deduct/p2.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.auto.deduct/p2.cpp
index 49bd5e9168ac9..1655685f44808 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.auto.deduct/p2.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.auto.deduct/p2.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 // p2.3 allows only T = auto in T(x).
 
diff --git a/clang/test/CXX/drs/dr0xx.cpp b/clang/test/CXX/drs/dr0xx.cpp
index 6e7c5c557ec11..5a127ff4f4164 100644
--- a/clang/test/CXX/drs/dr0xx.cpp
+++ b/clang/test/CXX/drs/dr0xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 
 namespace dr1 { // dr1: no
   namespace X { extern "C" void dr1_f(int a = 1); }
diff --git a/clang/test/CXX/drs/dr10xx.cpp b/clang/test/CXX/drs/dr10xx.cpp
index 76b0e0876d7a0..f30ed1cb3e496 100644
--- a/clang/test/CXX/drs/dr10xx.cpp
+++ b/clang/test/CXX/drs/dr10xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 namespace std {
   __extension__ typedef __SIZE_TYPE__ size_t;
diff --git a/clang/test/CXX/drs/dr12xx.cpp b/clang/test/CXX/drs/dr12xx.cpp
index 863e5be9a5d16..ce8b0e370b420 100644
--- a/clang/test/CXX/drs/dr12xx.cpp
+++ b/clang/test/CXX/drs/dr12xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 // dr1200: na
 
diff --git a/clang/test/CXX/drs/dr18xx.cpp b/clang/test/CXX/drs/dr18xx.cpp
index 43db6e3a95a1f..3d57f5e5b74db 100644
--- a/clang/test/CXX/drs/dr18xx.cpp
+++ b/clang/test/CXX/drs/dr18xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
 
 #if __cplusplus < 201103L
 // expected-error@+1 {{variadic macro}}
@@ -51,6 +51,22 @@ namespace dr1815 { // dr1815: no
 #endif
 }
 
+namespace dr1821 { // dr1821: yes
+struct A {
+  template <typename> struct B {
+    void f();
+  };
+  template <typename T> void B<T>::f(){};
+  // expected-error@-1 {{non-friend class member 'f' cannot have a qualified name}}
+
+  struct C {
+    void f();
+  };
+  void C::f() {}
+  // expected-error@-1 {{non-friend class member 'f' cannot have a qualified name}}
+};
+} // namespace dr1821
+
 namespace dr1822 { // dr1822: yes
 #if __cplusplus >= 201103L
   int a;
diff --git a/clang/test/CXX/drs/dr1xx.cpp b/clang/test/CXX/drs/dr1xx.cpp
index f70cd20133ebc..60e80a4c0e1c4 100644
--- a/clang/test/CXX/drs/dr1xx.cpp
+++ b/clang/test/CXX/drs/dr1xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 namespace dr100 { // dr100: yes
   template<const char (*)[4]> struct A {}; // expected-note 0-1{{declared here}}
diff --git a/clang/test/CXX/drs/dr20xx.cpp b/clang/test/CXX/drs/dr20xx.cpp
index aea1e23527181..dd60af14bb6b7 100644
--- a/clang/test/CXX/drs/dr20xx.cpp
+++ b/clang/test/CXX/drs/dr20xx.cpp
@@ -4,7 +4,7 @@
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 #if __cplusplus < 201103L
 #define static_assert(...) _Static_assert(__VA_ARGS__)
diff --git a/clang/test/CXX/drs/dr21xx.cpp b/clang/test/CXX/drs/dr21xx.cpp
index 80db71bccc8db..bd6f8b7cb6e68 100644
--- a/clang/test/CXX/drs/dr21xx.cpp
+++ b/clang/test/CXX/drs/dr21xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors
 
 #if __cplusplus < 201103L
 // expected-error@+1 {{variadic macro}}
diff --git a/clang/test/CXX/drs/dr23xx.cpp b/clang/test/CXX/drs/dr23xx.cpp
index ac448994c6932..6cb10067739f8 100644
--- a/clang/test/CXX/drs/dr23xx.cpp
+++ b/clang/test/CXX/drs/dr23xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 
 #if __cplusplus >= 201103L
 namespace dr2303 { // dr2303: 12
diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/dr2xx.cpp
index d8fb8941121de..de8ce2b77da4d 100644
--- a/clang/test/CXX/drs/dr2xx.cpp
+++ b/clang/test/CXX/drs/dr2xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 // PR13819 -- __SIZE_TYPE__ is incompatible.
 typedef __SIZE_TYPE__ size_t; // expected-error 0-1 {{extension}}
diff --git a/clang/test/CXX/drs/dr3xx.cpp b/clang/test/CXX/drs/dr3xx.cpp
index e3fe8d932b30a..253fc5d07521e 100644
--- a/clang/test/CXX/drs/dr3xx.cpp
+++ b/clang/test/CXX/drs/dr3xx.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b -verify=expected,cxx20_2b,cxx2b    -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++20 -verify=expected,cxx98_20,cxx20_2b -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 -verify=expected,cxx20_23,cxx23    -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++20 -verify=expected,cxx98_20,cxx20_23 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 -verify=expected,cxx98_17,cxx98_20 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++14 -verify=expected,cxx98_17,cxx98_20 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++11 -verify=expected,cxx98_17,cxx98_20 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
@@ -20,8 +20,8 @@ namespace dr301 { // dr301: yes
   void f() {
     bool a = (void(*)(S, S))operator+<S> < // expected-warning {{ordered comparison of function pointers}}
              (void(*)(S, S))operator+<S>;
-    bool b = (void(*)(S, S))operator- < // cxx20_2b-note {{to match this '<'}} cxx98_17-warning {{ordered comparison of function pointers}}
-             (void(*)(S, S))operator-; // cxx20_2b-error {{expected '>'}}
+    bool b = (void(*)(S, S))operator- < // cxx20_23-note {{to match this '<'}} cxx98_17-warning {{ordered comparison of function pointers}}
+             (void(*)(S, S))operator-; // cxx20_23-error {{expected '>'}}
     bool c = (void(*)(S, S))operator+ < // expected-note {{to match this '<'}}
              (void(*)(S, S))operator-; // expected-error {{expected '>'}}
   }
@@ -459,10 +459,10 @@ namespace dr331 { // dr331: yes
 
 namespace dr332 { // dr332: dup 577
   void f(volatile void); // expected-error {{'void' as parameter must not have type qualifiers}}
-  // cxx20_2b-warning@-1 {{volatile-qualified parameter type 'volatile void' is deprecated}}
+  // cxx20_23-warning@-1 {{volatile-qualified parameter type 'volatile void' is deprecated}}
   void g(const void); // expected-error {{'void' as parameter must not have type qualifiers}}
   void h(int n, volatile void); // expected-error {{'void' must be the first and only parameter}}
-  // cxx20_2b-warning@-1 {{volatile-qualified parameter type 'volatile void' is deprecated}}
+  // cxx20_23-warning@-1 {{volatile-qualified parameter type 'volatile void' is deprecated}}
 }
 
 namespace dr333 { // dr333: yes
@@ -647,7 +647,7 @@ namespace dr349 { // dr349: no
     template <class T> operator T ***() {
       int ***p = 0;
       return p; // cxx98_20-error {{cannot initialize return object of type 'const int ***' with an lvalue of type 'int ***'}}
-      // cxx2b-error@-1 {{cannot initialize return object of type 'const int ***' with an rvalue of type 'int ***'}}
+      // cxx23-error@-1 {{cannot initialize return object of type 'const int ***' with an rvalue of type 'int ***'}}
     }
   };
 
@@ -961,12 +961,12 @@ namespace dr368 { // dr368: yes
   template<typename T, T> struct S {}; // expected-note {{here}}
   template<typename T> int f(S<T, T()> *); // expected-error {{function type}}
   template<typename T> int g(S<T, (T())> *); // cxx98_17-note {{type 'X'}}
-  // cxx20_2b-note@-1 {{candidate function [with T = dr368::X]}}
+  // cxx20_23-note@-1 {{candidate function [with T = dr368::X]}}
   template<typename T> int g(S<T, true ? T() : T()> *); // cxx98_17-note {{type 'X'}}
-  // cxx20_2b-note@-1 {{candidate function [with T = dr368::X]}}
+  // cxx20_23-note@-1 {{candidate function [with T = dr368::X]}}
   struct X {};
   int n = g<X>(0); // cxx98_17-error {{no matching}}
-  // cxx20_2b-error@-1 {{call to 'g' is ambiguous}}
+  // cxx20_23-error@-1 {{call to 'g' is ambiguous}}
 }
 
 // dr370: na
diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/dr4xx.cpp
index f1ba2a0471389..d8bdf49d0b2dd 100644
--- a/clang/test/CXX/drs/dr4xx.cpp
+++ b/clang/test/CXX/drs/dr4xx.cpp
@@ -3,7 +3,7 @@
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 // FIXME: __SIZE_TYPE__ expands to 'long long' on some targets.
 __extension__ typedef __SIZE_TYPE__ size_t;
diff --git a/clang/test/CXX/drs/dr5xx.cpp b/clang/test/CXX/drs/dr5xx.cpp
index 1477a40fd25d0..cba0c4e1241c5 100644
--- a/clang/test/CXX/drs/dr5xx.cpp
+++ b/clang/test/CXX/drs/dr5xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 // FIXME: This is included to avoid a diagnostic with no source location
 // pointing at the implicit operator new. We can't match such a diagnostic
diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp
index b74e466c62ee6..59331e7de7687 100644
--- a/clang/test/CXX/drs/dr6xx.cpp
+++ b/clang/test/CXX/drs/dr6xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 
 namespace dr600 { // dr600: yes
 struct S {
diff --git a/clang/test/CXX/drs/dr9xx.cpp b/clang/test/CXX/drs/dr9xx.cpp
index 2c1d620c6e4bf..3d45dc8e72d2d 100644
--- a/clang/test/CXX/drs/dr9xx.cpp
+++ b/clang/test/CXX/drs/dr9xx.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++2b %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 namespace std {
   __extension__ typedef __SIZE_TYPE__ size_t;
diff --git a/clang/test/CXX/expr/expr.post/expr.type.conv/p1-2b.cpp b/clang/test/CXX/expr/expr.post/expr.type.conv/p1-2b.cpp
index a7137447e506f..bbfce5e698d15 100644
--- a/clang/test/CXX/expr/expr.post/expr.type.conv/p1-2b.cpp
+++ b/clang/test/CXX/expr/expr.post/expr.type.conv/p1-2b.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 template <class T>
 void foo(T);
@@ -9,7 +9,7 @@ struct A {
   float g(double);
 } a{1};
 
-// C++2b [dcl.type.auto.deduct]p2.3
+// C++23 [dcl.type.auto.deduct]p2.3
 // For an explicit type conversion, T is the specified type, which shall be auto.
 void diagnostics() {
   foo(auto());   // expected-error {{initializer for functional-style cast to 'auto' is empty}}
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/expr.prim.lambda.capture/p5.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/expr.prim.lambda.capture/p5.cpp
new file mode 100644
index 0000000000000..f0d337e5dd565
--- /dev/null
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/expr.prim.lambda.capture/p5.cpp
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+
+void f() {
+  int x = 0;
+  auto g = [x](int x) { return 0; }; // expected-error {{a lambda parameter cannot shadow an explicitly captured entity}} \
+                                     // expected-note {{variable 'x' is explicitly captured here}}
+  auto h = [y = 0]<typename y>(y) { return 0; };  // expected-error {{declaration of 'y' shadows template parameter}} \
+                                                  // expected-note {{template parameter is declared here}}
+
+}
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-cxx14.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-cxx14.cpp
index 2933532627d4c..a5e4b6a2c3bbd 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-cxx14.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4-cxx14.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx2b    %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx23    %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20 %s
 // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,cxx14_20 %s
 
@@ -9,7 +9,7 @@ int &d = [] (int &r) -> auto & { return r; } (a);
 int &e = [] (int &r) -> auto { return r; } (a); // expected-error {{cannot bind to a temporary}}
 int &f = [] (int r) -> decltype(auto) { return r; } (a); // expected-error {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
 int &g = [] (int r) -> decltype(auto) { return (r); } (a); // expected-warning {{reference to stack}}
-// cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+// cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
 
 int test_explicit_auto_return()
 {
@@ -17,7 +17,7 @@ int test_explicit_auto_return()
     auto L = [](auto F, auto a) { return F(a); };
     auto M = [](auto a) -> auto { return a; }; // OK
     auto MRef = [](auto b) -> auto & { return b; }; //cxx14_20-warning{{reference to stack}}
-    // cxx2b-error@-1 {{non-const lvalue reference to type 'X' cannot bind to a temporary of type 'X'}}
+    // cxx23-error@-1 {{non-const lvalue reference to type 'X' cannot bind to a temporary of type 'X'}}
     auto MPtr = [](auto c) -> auto* { return &c; }; //expected-warning{{address of stack}}
     auto MDeclType = [](auto&& d) -> decltype(auto) { return static_cast<decltype(d)>(d); }; //OK
     M(3);
@@ -57,7 +57,7 @@ int test_no_parameter_list()
     auto M = [] { return 5; }; // OK
     auto M2 = [] -> auto && { return si; };
 #if __cplusplus <= 202002L
-      // expected-warning@-2{{is a C++2b extension}}
+      // expected-warning@-2{{is a C++23 extension}}
 #endif
     M();
 }
diff --git a/clang/test/CXX/expr/expr.unary/expr.new/p2-cxx1z.cpp b/clang/test/CXX/expr/expr.unary/expr.new/p2-cxx1z.cpp
index b8383b83a589a..633f46fdd621d 100644
--- a/clang/test/CXX/expr/expr.unary/expr.new/p2-cxx1z.cpp
+++ b/clang/test/CXX/expr/expr.unary/expr.new/p2-cxx1z.cpp
@@ -2,7 +2,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++17 -pedantic
 
 // [expr.new]p2 ... the invented declaration: T x init ;
-// C++2b [dcl.type.auto.deduct]p2.2
+// C++23 [dcl.type.auto.deduct]p2.2
 // For a variable declared with a type that contains a placeholder type, T is the declared type of the variable.
 void f() {
   // - If the initializer is a parenthesized expression-list, the expression-list shall be a single assignmentexpression and E is the assignment-expression.
diff --git a/clang/test/CXX/over/over.match/over.match.best/over.best.ics/p6.cpp b/clang/test/CXX/over/over.match/over.match.best/over.best.ics/p6.cpp
index d33b1a4b7438b..909ff47560d35 100644
--- a/clang/test/CXX/over/over.match/over.match.best/over.best.ics/p6.cpp
+++ b/clang/test/CXX/over/over.match/over.match.best/over.best.ics/p6.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=cxx2b -std=c++2b %s
+// RUN: %clang_cc1 -fsyntax-only -verify=cxx23 -std=c++23 %s
 // RUN: %clang_cc1 -fsyntax-only -verify=cxx20 -std=c++20 %s
-// cxx2b-no-diagnostics
+// cxx23-no-diagnostics
 
 struct __unique {
-    static constexpr auto operator()() { return 4; };  // cxx20-warning {{is a C++2b extension}}
+    static constexpr auto operator()() { return 4; };  // cxx20-warning {{is a C++23 extension}}
 
     using P = int();
     constexpr operator P*() { return operator(); }
diff --git a/clang/test/CXX/over/over.oper/p7.cpp b/clang/test/CXX/over/over.oper/p7.cpp
index 7d304762c57f8..9d1201a103426 100644
--- a/clang/test/CXX/over/over.oper/p7.cpp
+++ b/clang/test/CXX/over/over.oper/p7.cpp
@@ -1,15 +1,15 @@
 // RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11
-// RUN: %clang_cc1 -std=c++2b %s -verify=expected,cxx2b
-// RUN: %clang_cc1 -std=c++2b -Wpre-c++2b-compat %s -verify=expected,precxx2b
+// RUN: %clang_cc1 -std=c++23 %s -verify=expected,cxx23
+// RUN: %clang_cc1 -std=c++23 -Wpre-c++23-compat %s -verify=expected,precxx23
 
 
 struct Functor {
   static int operator()(int a, int b);
   static int operator[](int a1);
-  // cxx11-warning@-2 {{declaring overloaded 'operator()' as 'static' is a C++2b extension}}
-  // cxx11-warning@-2 {{declaring overloaded 'operator[]' as 'static' is a C++2b extension}}
-  // precxx2b-warning@-4 {{incompatible with C++ standards before C++2b}}
-  // precxx2b-warning@-4 {{incompatible with C++ standards before C++2b}}
+  // cxx11-warning@-2 {{declaring overloaded 'operator()' as 'static' is a C++23 extension}}
+  // cxx11-warning@-2 {{declaring overloaded 'operator[]' as 'static' is a C++23 extension}}
+  // precxx23-warning@-4 {{incompatible with C++ standards before C++23}}
+  // precxx23-warning@-4 {{incompatible with C++ standards before C++23}}
 };
 
 struct InvalidParsing1 {
diff --git a/clang/test/CXX/special/class.copy/p3-cxx11.cpp b/clang/test/CXX/special/class.copy/p3-cxx11.cpp
index bee2d02c96911..8d27f1c2eacb2 100644
--- a/clang/test/CXX/special/class.copy/p3-cxx11.cpp
+++ b/clang/test/CXX/special/class.copy/p3-cxx11.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++2b -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++23 -fsyntax-only -verify %s
 // RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++20 -fsyntax-only -verify %s
 // RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++11 -fsyntax-only -verify %s
 class X {
diff --git a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p4.cpp b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p4.cpp
index 661b7c5987351..8d43be6fc9047 100644
--- a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p4.cpp
+++ b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p4.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 void test_consteval() {
   if consteval ({(void)1;});  // expected-error {{expected { after consteval}}
diff --git a/clang/test/CXX/temp/temp.decls/temp.mem/p5.cpp b/clang/test/CXX/temp/temp.decls/temp.mem/p5.cpp
index f5cadc75e6d3d..668941b401419 100644
--- a/clang/test/CXX/temp/temp.decls/temp.mem/p5.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.mem/p5.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx2b    %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx23    %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx98_20 %s
 // RUN: %clang_cc1 -std=c++98 -fsyntax-only -verify=expected,cxx98_20 %s
 // RUN: %clang_cc1            -fsyntax-only -verify=expected,cxx98_20 %s
@@ -69,8 +69,8 @@ struct X0 {
     T x = T();
     return x; // cxx98_20-error{{cannot initialize return object of type 'const char *' with an lvalue of type 'char'}} \
     // cxx98_20-error{{cannot initialize return object of type 'const int *' with an lvalue of type 'int'}} \
-    // cxx2b-error{{cannot initialize return object of type 'const char *' with an rvalue of type 'char'}} \
-    // cxx2b-error{{cannot initialize return object of type 'const int *' with an rvalue of type 'int'}}
+    // cxx23-error{{cannot initialize return object of type 'const char *' with an rvalue of type 'char'}} \
+    // cxx23-error{{cannot initialize return object of type 'const int *' with an rvalue of type 'int'}}
   }
 };
 
diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp
index 7542186c22c71..10af03eeaf0a9 100644
--- a/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++11 -fblocks -fms-extensions -fsyntax-only -verify=expected,cxx11 %s
-// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++2b -fblocks -fms-extensions -fsyntax-only -verify=expected %s
+// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -std=c++23 -fblocks -fms-extensions -fsyntax-only -verify=expected %s
 
 template<typename T, typename U> struct pair;
 template<typename ...> struct tuple;
diff --git a/clang/test/ClangScanDeps/module-format.c b/clang/test/ClangScanDeps/module-format.c
new file mode 100644
index 0000000000000..001a011ae0b59
--- /dev/null
+++ b/clang/test/ClangScanDeps/module-format.c
@@ -0,0 +1,64 @@
+// Check that the scanner produces raw ast files, even when builds produce the
+// obj format, and that the scanner can read obj format from PCH and modules
+// imported by PCH.
+
+// Unsupported on AIX because we don't support the requisite "__clangast"
+// section in XCOFF yet.
+// UNSUPPORTED: target={{.*}}-aix{{.*}}
+
+// REQUIRES: shell
+
+// RUN: rm -rf %t && mkdir %t
+// RUN: cp %S/Inputs/modules-pch/* %t
+
+// Scan dependencies of the PCH:
+//
+// RUN: rm -f %t/cdb_pch.json
+// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb_pch.json
+// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-full \
+// RUN:   -module-files-dir %t/build > %t/result_pch.json
+
+// Explicitly build the PCH:
+//
+// RUN: %deps-to-rsp %t/result_pch.json --module-name=ModCommon1 > %t/mod_common_1.cc1.rsp
+// RUN: %deps-to-rsp %t/result_pch.json --module-name=ModCommon2 > %t/mod_common_2.cc1.rsp
+// RUN: %deps-to-rsp %t/result_pch.json --module-name=ModPCH > %t/mod_pch.cc1.rsp
+// RUN: %deps-to-rsp %t/result_pch.json --tu-index=0 > %t/pch.rsp
+//
+// RUN: %clang @%t/mod_common_1.cc1.rsp
+// RUN: %clang @%t/mod_common_2.cc1.rsp
+// RUN: %clang @%t/mod_pch.cc1.rsp
+// RUN: %clang @%t/pch.rsp
+
+// Scan dependencies of the TU:
+//
+// RUN: rm -f %t/cdb_tu.json
+// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb_tu.json
+// RUN: clang-scan-deps -compilation-database %t/cdb_tu.json -format experimental-full \
+// RUN:   -module-files-dir %t/build > %t/result_tu.json
+
+// Explicitly build the TU:
+//
+// RUN: %deps-to-rsp %t/result_tu.json --module-name=ModTU > %t/mod_tu.cc1.rsp
+// RUN: %deps-to-rsp %t/result_tu.json --tu-index=0 > %t/tu.rsp
+//
+// RUN: %clang @%t/mod_tu.cc1.rsp
+// RUN: %clang @%t/tu.rsp
+
+// Check the module format for scanner modules:
+//
+// RUN: find %t/cache -name "*.pcm" -exec %clang_cc1 -module-file-info "{}" ";" | FileCheck %s -check-prefix=SCAN
+// SCAN: Module format: raw
+// SCAN: Module format: raw
+// SCAN: Module format: raw
+// SCAN: Module format: raw
+
+// Check the module format for built modules:
+//
+// RUN: find %t/build -name "*.pcm" -exec %clang_cc1 -module-file-info "{}" ";" | FileCheck %s -check-prefix=BUILD
+// BUILD: Module format: obj
+// BUILD: Module format: obj
+// BUILD: Module format: obj
+// BUILD: Module format: obj
+
+// FIXME: check pch format as well; -module-file-info does not work with a PCH
diff --git a/clang/test/CodeGen/PowerPC/altivec.c b/clang/test/CodeGen/PowerPC/altivec.c
index 1d4980bd0d7ef..808135de00ea6 100644
--- a/clang/test/CodeGen/PowerPC/altivec.c
+++ b/clang/test/CodeGen/PowerPC/altivec.c
@@ -4,8 +4,8 @@
 // RUN: %clang_cc1 -target-feature +altivec -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LE
 // RUN: %clang_cc1 -target-feature +altivec -mabi=vec-extabi -target-cpu pwr8 -triple powerpc-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 // RUN: %clang_cc1 -target-feature +altivec -mabi=vec-extabi -target-cpu pwr8 -triple powerpc64-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
-// RUN: %clang_cc1 -target-feature +altivec -mabi=vec-default -target-cpu pwr8 -triple powerpc-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
-// RUN: %clang_cc1 -target-feature +altivec -mabi=vec-default -target-cpu pwr8 -triple powerpc64-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+// RUN: %clang_cc1 -target-feature +altivec -target-cpu pwr8 -triple powerpc-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+// RUN: %clang_cc1 -target-feature +altivec -target-cpu pwr8 -triple powerpc64-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 // RUN: %clang -S -emit-llvm -maltivec -mabi=vec-extabi -mcpu=pwr8 --target=powerpc-unknown-aix %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 // RUN: %clang -S -emit-llvm -maltivec -mabi=vec-extabi -mcpu=pwr8 --target=powerpc64-unknown-aix %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 // RUN: %clang -S -emit-llvm -maltivec -mabi=vec-default -mcpu=pwr8 --target=powerpc-unknown-aix -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BE
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-p10vector.c b/clang/test/CodeGen/PowerPC/builtins-ppc-p10vector.c
index 484f83aceb89c..151a29976d8ed 100644
--- a/clang/test/CodeGen/PowerPC/builtins-ppc-p10vector.c
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-p10vector.c
@@ -2160,8 +2160,8 @@ vector unsigned __int128 test_vec_rl_u128(void) {
 
 vector signed __int128 test_vec_rlnm_s128(void) {
   // CHECK-LABEL: @test_vec_rlnm_s128(
-  // CHECK-LE: %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  // CHECK-BE: %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 31, i32 30, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  // CHECK-LE: %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  // CHECK-BE: %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 31, i32 30, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   // CHECK: call <1 x i128> @llvm.ppc.altivec.vrlqnm(<1 x i128>
   // CHECK-NEXT: ret <1 x i128>
   return vec_rlnm(vsi128a, vsi128b, vsi128c);
@@ -2169,8 +2169,8 @@ vector signed __int128 test_vec_rlnm_s128(void) {
 
 vector unsigned __int128 test_vec_rlnm_u128(void) {
   // CHECK-LABEL: @test_vec_rlnm_u128(
-  // CHECK-LE:  %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  // CHECK-BE: %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 31, i32 30, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  // CHECK-LE:  %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  // CHECK-BE: %shuffle.i = shufflevector <16 x i8> %7, <16 x i8> %8, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 31, i32 30, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
   // CHECK: call <1 x i128> @llvm.ppc.altivec.vrlqnm(<1 x i128>
   // CHECK-NEXT: ret <1 x i128>
   return vec_rlnm(vui128a, vui128b, vui128c);
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c b/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c
index 39262faca812f..b55a522ed2608 100644
--- a/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-p9vector.c
@@ -1177,19 +1177,19 @@ void test113(void) {
   return vec_xst_len_r(vuca,uc,0);
 }
 vector float test114(void) {
-// CHECK-BE: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3>
+// CHECK-BE: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 poison, i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3>
 // CHECK-BE: @llvm.ppc.vsx.xvcvhpsp(<8 x i16> {{.+}})
 // CHECK-BE-NEXT: ret <4 x float>
-// CHECK: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 3, i32 undef>
+// CHECK: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 3, i32 poison>
 // CHECK: @llvm.ppc.vsx.xvcvhpsp(<8 x i16> {{.+}})
 // CHECK-NEXT: ret <4 x float>
   return vec_extract_fp32_from_shorth(vusa);
 }
 vector float test115(void) {
-// CHECK-BE: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 undef, i32 4, i32 undef, i32 5, i32 undef, i32 6, i32 undef, i32 7>
+// CHECK-BE: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 poison, i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7>
 // CHECK-BE: @llvm.ppc.vsx.xvcvhpsp(<8 x i16> {{.+}})
 // CHECK-BE-NEXT: ret <4 x float>
-// CHECK: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 4, i32 undef, i32 5, i32 undef, i32 6, i32 undef, i32 7, i32 undef>
+// CHECK: shufflevector <8 x i16> {{.+}}, <8 x i16> {{.+}}, <8 x i32> <i32 4, i32 poison, i32 5, i32 poison, i32 6, i32 poison, i32 7, i32 poison>
 // CHECK: @llvm.ppc.vsx.xvcvhpsp(<8 x i16> {{.+}})
 // CHECK-NEXT: ret <4 x float>
   return vec_extract_fp32_from_shortl(vusa);
diff --git a/clang/test/CodeGen/RISCV/fpconstrained.c b/clang/test/CodeGen/RISCV/fpconstrained.c
new file mode 100644
index 0000000000000..d5a7a4aab1556
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/fpconstrained.c
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -triple riscv64 -frounding-math -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=FPMODELSTRICT
+// RUN: %clang_cc1 -triple riscv64 -ffp-contract=fast -emit-llvm -o - %s | FileCheck %s -check-prefix=PRECISE
+// RUN: %clang_cc1 -triple riscv64 -ffast-math -ffp-contract=fast -emit-llvm -o - %s | FileCheck %s -check-prefix=FAST
+// RUN: %clang_cc1 -triple riscv64 -ffast-math -emit-llvm -o - %s | FileCheck %s -check-prefix=FASTNOCONTRACT
+// RUN: %clang_cc1 -triple riscv64 -ffast-math -ffp-contract=fast -ffp-exception-behavior=ignore -emit-llvm -o - %s | FileCheck %s -check-prefix=FAST
+// RUN: %clang_cc1 -triple riscv64 -ffast-math -ffp-contract=fast -ffp-exception-behavior=strict -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=EXCEPT
+// RUN: %clang_cc1 -triple riscv64 -ffast-math -ffp-contract=fast -ffp-exception-behavior=maytrap -fexperimental-strict-floating-point -emit-llvm -o - %s | FileCheck %s -check-prefix=MAYTRAP
+
+// Test strict-fp support in RISC-V.
+
+float f0, f1, f2;
+
+void foo(void) {
+  // CHECK-LABEL: define {{.*}}void @foo()
+
+  // MAYTRAP: llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // EXCEPT: llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // FPMODELSTRICT: llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  // STRICTEXCEPT: llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  // STRICTNOEXCEPT: llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.dynamic", metadata !"fpexcept.ignore")
+  // PRECISE: fadd contract float %{{.*}}, %{{.*}}
+  // FAST: fadd fast
+  // FASTNOCONTRACT: fadd reassoc nnan ninf nsz arcp afn float
+  f0 = f1 + f2;
+
+  // CHECK: ret
+}
diff --git a/clang/test/CodeGen/RISCV/ntlh-intrinsics/riscv32-zihintntl.c b/clang/test/CodeGen/RISCV/ntlh-intrinsics/riscv32-zihintntl.c
new file mode 100644
index 0000000000000..5276e486604de
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/ntlh-intrinsics/riscv32-zihintntl.c
@@ -0,0 +1,182 @@
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1  -triple riscv32 -target-feature +v -target-feature +experimental-zihintntl -emit-llvm %s -o - \
+// RUN:     | FileCheck %s
+
+#include <riscv_ntlh.h>
+#include <riscv_vector.h>
+
+signed char sc;
+unsigned char uc;
+signed short ss;
+unsigned short us;
+signed int si;
+unsigned int ui;
+signed long long sll;
+unsigned long long ull;
+_Float16 h1, h2;
+float f1, f2;
+double d1, d2;
+typedef int v4si __attribute__((vector_size(16)));
+typedef signed short v8ss __attribute__((vector_size(16)));
+typedef signed char v16sc __attribute__((vector_size(16)));
+v4si v4si1, v4si2;
+v8ss v8ss1, v8ss2;
+v16sc v16sc1, v16sc2;
+vint32m1_t *scvi1, *scvi2;
+vint16m1_t *scvs1, *scvs2;
+vint8m1_t *scvc1, *scvc2;
+
+// clang-format off
+void ntl_all_sizes() {                                       // CHECK-LABEL: ntl_all_sizes
+  uc = __riscv_ntl_load(&sc, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !5
+  sc = __riscv_ntl_load(&uc, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !5
+  us = __riscv_ntl_load(&ss, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !5
+  ss = __riscv_ntl_load(&us, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !5
+  ui = __riscv_ntl_load(&si, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !5
+  si = __riscv_ntl_load(&ui, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !5
+  ull = __riscv_ntl_load(&sll, __RISCV_NTLH_INNERMOST_PRIVATE); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  sll = __riscv_ntl_load(&ull, __RISCV_NTLH_INNERMOST_PRIVATE); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  h1 = __riscv_ntl_load(&h2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !5
+  f1 = __riscv_ntl_load(&f2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !5
+  d1 = __riscv_ntl_load(&d2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  v4si1 = __riscv_ntl_load(&v4si2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !5
+  v8ss1 = __riscv_ntl_load(&v8ss2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !5
+  v16sc1 = __riscv_ntl_load(&v16sc2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !5
+  *scvi1 = __riscv_ntl_load(scvi2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  *scvs1 = __riscv_ntl_load(scvs2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  *scvc1 = __riscv_ntl_load(scvc2, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: load <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+
+  uc = __riscv_ntl_load(&sc, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !6
+  sc = __riscv_ntl_load(&uc, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !6
+  us = __riscv_ntl_load(&ss, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !6
+  ss = __riscv_ntl_load(&us, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !6
+  ui = __riscv_ntl_load(&si, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !6
+  si = __riscv_ntl_load(&ui, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !6
+  ull = __riscv_ntl_load(&sll, __RISCV_NTLH_ALL_PRIVATE); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  sll = __riscv_ntl_load(&ull, __RISCV_NTLH_ALL_PRIVATE); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  h1 = __riscv_ntl_load(&h2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !6
+  f1 = __riscv_ntl_load(&f2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !6
+  d1 = __riscv_ntl_load(&d2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  v4si1 = __riscv_ntl_load(&v4si2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !6
+  v8ss1 = __riscv_ntl_load(&v8ss2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !6
+  v16sc1 = __riscv_ntl_load(&v16sc2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !6
+  *scvi1 = __riscv_ntl_load(scvi2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  *scvs1 = __riscv_ntl_load(scvs2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  *scvc1 = __riscv_ntl_load(scvc2, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: load <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+
+  uc = __riscv_ntl_load(&sc, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !7
+  sc = __riscv_ntl_load(&uc, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !7
+  us = __riscv_ntl_load(&ss, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !7
+  ss = __riscv_ntl_load(&us, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !7
+  ui = __riscv_ntl_load(&si, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !7
+  si = __riscv_ntl_load(&ui, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !7
+  ull = __riscv_ntl_load(&sll, __RISCV_NTLH_INNERMOST_SHARED); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  sll = __riscv_ntl_load(&ull, __RISCV_NTLH_INNERMOST_SHARED); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  h1 = __riscv_ntl_load(&h2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !7
+  f1 = __riscv_ntl_load(&f2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !7
+  d1 = __riscv_ntl_load(&d2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  v4si1 = __riscv_ntl_load(&v4si2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !7
+  v8ss1 = __riscv_ntl_load(&v8ss2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !7
+  v16sc1 = __riscv_ntl_load(&v16sc2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !7
+  *scvi1 = __riscv_ntl_load(scvi2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  *scvs1 = __riscv_ntl_load(scvs2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  *scvc1 = __riscv_ntl_load(scvc2, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: load <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+
+  uc = __riscv_ntl_load(&sc, __RISCV_NTLH_ALL);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !8
+  sc = __riscv_ntl_load(&uc, __RISCV_NTLH_ALL);   // CHECK: load i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !8
+  us = __riscv_ntl_load(&ss, __RISCV_NTLH_ALL);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !8
+  ss = __riscv_ntl_load(&us, __RISCV_NTLH_ALL);   // CHECK: load i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !8
+  ui = __riscv_ntl_load(&si, __RISCV_NTLH_ALL);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !8
+  si = __riscv_ntl_load(&ui, __RISCV_NTLH_ALL);   // CHECK: load i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !8
+  ull = __riscv_ntl_load(&sll, __RISCV_NTLH_ALL); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  sll = __riscv_ntl_load(&ull, __RISCV_NTLH_ALL); // CHECK: load i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  h1 = __riscv_ntl_load(&h2, __RISCV_NTLH_ALL);   // CHECK: load half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !8
+  f1 = __riscv_ntl_load(&f2, __RISCV_NTLH_ALL);   // CHECK: load float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !8
+  d1 = __riscv_ntl_load(&d2, __RISCV_NTLH_ALL);   // CHECK: load double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  v4si1 = __riscv_ntl_load(&v4si2, __RISCV_NTLH_ALL);   // CHECK: load <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !8
+  v8ss1 = __riscv_ntl_load(&v8ss2, __RISCV_NTLH_ALL);   // CHECK: load <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !8
+  v16sc1 = __riscv_ntl_load(&v16sc2, __RISCV_NTLH_ALL);   // CHECK: load <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !8
+  *scvi1 = __riscv_ntl_load(scvi2, __RISCV_NTLH_ALL);   // CHECK: load <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  *scvs1 = __riscv_ntl_load(scvs2, __RISCV_NTLH_ALL);   // CHECK: load <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  *scvc1 = __riscv_ntl_load(scvc2, __RISCV_NTLH_ALL);   // CHECK: load <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+
+  __riscv_ntl_store(&uc, 1, __RISCV_NTLH_INNERMOST_PRIVATE);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&sc, 1, __RISCV_NTLH_INNERMOST_PRIVATE);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&us, 1, __RISCV_NTLH_INNERMOST_PRIVATE);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&ss, 1, __RISCV_NTLH_INNERMOST_PRIVATE);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&ui, 1, __RISCV_NTLH_INNERMOST_PRIVATE);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&si, 1, __RISCV_NTLH_INNERMOST_PRIVATE);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&ull, 1, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&sll, 1, __RISCV_NTLH_INNERMOST_PRIVATE);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&h1, 1.0, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&f1, 1.0, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&d1, 1.0, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&v4si1, v4si2, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&v8ss1, v8ss2, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(&v16sc1, v16sc2, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(scvi2, *scvi1, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(scvs2, *scvs1, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+  __riscv_ntl_store(scvc2, *scvc1, __RISCV_NTLH_INNERMOST_PRIVATE);  // CHECK: store <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !5
+
+  __riscv_ntl_store(&uc, 1, __RISCV_NTLH_ALL_PRIVATE);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&sc, 1, __RISCV_NTLH_ALL_PRIVATE);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&us, 1, __RISCV_NTLH_ALL_PRIVATE);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&ss, 1, __RISCV_NTLH_ALL_PRIVATE);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&ui, 1, __RISCV_NTLH_ALL_PRIVATE);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&si, 1, __RISCV_NTLH_ALL_PRIVATE);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&ull, 1, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&sll, 1, __RISCV_NTLH_ALL_PRIVATE);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&h1, 1.0, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&f1, 1.0, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&d1, 1.0, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&v4si1, v4si2, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&v8ss1, v8ss2, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(&v16sc1, v16sc2, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(scvi2, *scvi1, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(scvs2, *scvs1, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+  __riscv_ntl_store(scvc2, *scvc1, __RISCV_NTLH_ALL_PRIVATE);  // CHECK: store <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !6
+
+  __riscv_ntl_store(&uc, 1, __RISCV_NTLH_INNERMOST_SHARED);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&sc, 1, __RISCV_NTLH_INNERMOST_SHARED);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&us, 1, __RISCV_NTLH_INNERMOST_SHARED);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&ss, 1, __RISCV_NTLH_INNERMOST_SHARED);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&ui, 1, __RISCV_NTLH_INNERMOST_SHARED);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&si, 1, __RISCV_NTLH_INNERMOST_SHARED);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&ull, 1, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&sll, 1, __RISCV_NTLH_INNERMOST_SHARED);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&h1, 1.0, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&f1, 1.0, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&d1, 1.0, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&v4si1, v4si2, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&v8ss1, v8ss2, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(&v16sc1, v16sc2, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(scvi2, *scvi1, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(scvs2, *scvs1, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+  __riscv_ntl_store(scvc2, *scvc1, __RISCV_NTLH_INNERMOST_SHARED);  // CHECK: store <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !7
+
+  __riscv_ntl_store(&uc, 1, __RISCV_NTLH_ALL);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&sc, 1, __RISCV_NTLH_ALL);    // CHECK: store i8{{.*}}align 1, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&us, 1, __RISCV_NTLH_ALL);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&ss, 1, __RISCV_NTLH_ALL);    // CHECK: store i16{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&ui, 1, __RISCV_NTLH_ALL);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&si, 1, __RISCV_NTLH_ALL);    // CHECK: store i32{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&ull, 1, __RISCV_NTLH_ALL);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&sll, 1, __RISCV_NTLH_ALL);   // CHECK: store i64{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&h1, 1.0, __RISCV_NTLH_ALL);  // CHECK: store half{{.*}}align 2, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&f1, 1.0, __RISCV_NTLH_ALL);  // CHECK: store float{{.*}}align 4, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&d1, 1.0, __RISCV_NTLH_ALL);  // CHECK: store double{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&v4si1, v4si2, __RISCV_NTLH_ALL);  // CHECK: store <4 x i32>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&v8ss1, v8ss2, __RISCV_NTLH_ALL);  // CHECK: store <8 x i16>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(&v16sc1, v16sc2, __RISCV_NTLH_ALL);  // CHECK: store <16 x i8>{{.*}}align 16, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(scvi2, *scvi1, __RISCV_NTLH_ALL);  // CHECK: store <vscale x 2 x i32>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(scvs2, *scvs1, __RISCV_NTLH_ALL);  // CHECK: store <vscale x 4 x i16>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+  __riscv_ntl_store(scvc2, *scvc1, __RISCV_NTLH_ALL);  // CHECK: store <vscale x 8 x i8>{{.*}}align 8, !nontemporal !4, !riscv-nontemporal-domain !8
+
+}
+// clang-format on
+
+// CHECK: !4 = !{i32 1}
+// CHECK: !5 = !{i32 2}
+// CHECK: !6 = !{i32 3}
+// CHECK: !7 = !{i32 4}
+// CHECK: !8 = !{i32 5}
\ No newline at end of file
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-x-rv64.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-x-rv64.c
new file mode 100644
index 0000000000000..fe6972dc7942f
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-x-rv64.c
@@ -0,0 +1,117 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p24_20 (0b11111)
+#define p11_7  (0b11111)
+
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e64m1.i64.i64.i64(i64 3, i64 31, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u64m1(uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u64m1(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e64m2.i64.i64.i64(i64 3, i64 31, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u64m2(uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u64m2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e64m4.i64.i64.i64(i64 3, i64 31, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u64m4(uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u64m4(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e64m8.i64.i64.i64(i64 3, i64 31, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u64m8(uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u64m8(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.x.se.nxv1i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_x_se_u64m1(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u64m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.x.se.nxv2i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_x_se_u64m2(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u64m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.x.se.nxv4i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_x_se_u64m4(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u64m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.x.se.nxv8i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_x_se_u64m8(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u64m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.x.nxv1i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_x_u64m1(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u64m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.x.nxv2i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_x_u64m2(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u64m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.x.nxv4i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_x_u64m4(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u64m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.x.nxv8i64.i64.i64.i64(i64 3, i64 31, i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_x_u64m8(uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u64m8(p27_26, p24_20, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-x.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-x.c
new file mode 100644
index 0000000000000..bac7e019c35b1
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-x.c
@@ -0,0 +1,1691 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +v -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p24_20 (0b11111)
+#define p11_7  (0b11111)
+#define simm5  (10)
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8mf8.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8mf8.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8mf8(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8mf8(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8mf4.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8mf4.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8mf4(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8mf4(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8mf2.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8mf2.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8mf2(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8mf2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m1.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m1.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8m1(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8m1(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m2.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m2.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8m2(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8m2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m4.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m4.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8m4(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8m4(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m8.i32.i8.i32(i32 3, i32 31, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e8m8.i64.i8.i64(i64 3, i64 31, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u8m8(uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u8m8(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16mf4.i32.i16.i32(i32 3, i32 31, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16mf4.i64.i16.i64(i64 3, i64 31, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u16mf4(uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u16mf4(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16mf2.i32.i16.i32(i32 3, i32 31, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16mf2.i64.i16.i64(i64 3, i64 31, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u16mf2(uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u16mf2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m1.i32.i16.i32(i32 3, i32 31, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m1.i64.i16.i64(i64 3, i64 31, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u16m1(uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u16m1(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m2.i32.i16.i32(i32 3, i32 31, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m2.i64.i16.i64(i64 3, i64 31, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u16m2(uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u16m2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m4.i32.i16.i32(i32 3, i32 31, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m4.i64.i16.i64(i64 3, i64 31, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u16m4(uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u16m4(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m8.i32.i16.i32(i32 3, i32 31, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e16m8.i64.i16.i64(i64 3, i64 31, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u16m8(uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u16m8(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32mf2.i32.i32.i32(i32 3, i32 31, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32mf2.i64.i32.i64(i64 3, i64 31, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u32mf2(uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u32mf2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m1.i32.i32.i32(i32 3, i32 31, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m1.i64.i32.i64(i64 3, i64 31, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u32m1(uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u32m1(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m2.i32.i32.i32(i32 3, i32 31, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m2.i64.i32.i64(i64 3, i64 31, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u32m2(uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u32m2(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m4.i32.i32.i32(i32 3, i32 31, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m4.i64.i32.i64(i64 3, i64 31, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u32m4(uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u32m4(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_x_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m8.i32.i32.i32(i32 3, i32 31, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_x_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.x.se.e32m8.i64.i32.i64(i64 3, i64 31, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_x_se_u32m8(uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_x_se_u32m8(p27_26, p24_20, p11_7, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.x.se.nxv1i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.x.se.nxv1i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_x_se_u8mf8(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8mf8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.x.se.nxv2i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.x.se.nxv2i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_x_se_u8mf4(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8mf4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.x.se.nxv4i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.x.se.nxv4i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_x_se_u8mf2(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8mf2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.x.se.nxv8i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.x.se.nxv8i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_x_se_u8m1(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.x.se.nxv16i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.x.se.nxv16i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_x_se_u8m2(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.x.se.nxv32i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.x.se.nxv32i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_x_se_u8m4(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.x.se.nxv64i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.x.se.nxv64i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_x_se_u8m8(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u8m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.x.se.nxv1i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.x.se.nxv1i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_x_se_u16mf4(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u16mf4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.x.se.nxv2i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.x.se.nxv2i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_x_se_u16mf2(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u16mf2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.x.se.nxv4i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.x.se.nxv4i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_x_se_u16m1(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u16m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.x.se.nxv8i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.x.se.nxv8i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_x_se_u16m2(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u16m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.x.se.nxv16i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.x.se.nxv16i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_x_se_u16m4(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u16m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.x.se.nxv32i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.x.se.nxv32i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_x_se_u16m8(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u16m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.x.se.nxv1i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.x.se.nxv1i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_x_se_u32mf2(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u32mf2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.x.se.nxv2i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.x.se.nxv2i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_x_se_u32m1(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u32m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.x.se.nxv4i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.x.se.nxv4i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_x_se_u32m2(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u32m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.x.se.nxv8i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.x.se.nxv8i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_x_se_u32m4(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u32m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.x.se.nxv16i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.x.se.nxv16i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_x_se_u32m8(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_se_u32m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.x.nxv1i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.x.nxv1i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_x_u8mf8(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8mf8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.x.nxv2i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.x.nxv2i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_x_u8mf4(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8mf4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.x.nxv4i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.x.nxv4i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_x_u8mf2(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8mf2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.x.nxv8i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.x.nxv8i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_x_u8m1(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.x.nxv16i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.x.nxv16i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_x_u8m2(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.x.nxv32i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.x.nxv32i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_x_u8m4(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.x.nxv64i8.i32.i8.i32(i32 3, i32 31, i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.x.nxv64i8.i64.i8.i64(i64 3, i64 31, i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_x_u8m8(uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u8m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.x.nxv1i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.x.nxv1i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_x_u16mf4(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u16mf4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.x.nxv2i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.x.nxv2i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_x_u16mf2(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u16mf2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.x.nxv4i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.x.nxv4i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_x_u16m1(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u16m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.x.nxv8i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.x.nxv8i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_x_u16m2(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u16m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.x.nxv16i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.x.nxv16i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_x_u16m4(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u16m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.x.nxv32i16.i32.i16.i32(i32 3, i32 31, i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.x.nxv32i16.i64.i16.i64(i64 3, i64 31, i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_x_u16m8(uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u16m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.x.nxv1i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.x.nxv1i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_x_u32mf2(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u32mf2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.x.nxv2i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.x.nxv2i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_x_u32m1(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u32m1(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.x.nxv4i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.x.nxv4i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_x_u32m2(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u32m2(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.x.nxv8i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.x.nxv8i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_x_u32m4(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u32m4(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_x_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.x.nxv16i32.i32.i32.i32(i32 3, i32 31, i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_x_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.x.nxv16i32.i64.i32.i64(i64 3, i64 31, i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_x_u32m8(uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_x_u32m8(p27_26, p24_20, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8mf8.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8mf8.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8mf8(size_t vl) {
+  __riscv_sf_vc_i_se_u8mf8(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8mf4.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8mf4.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8mf4(size_t vl) {
+  __riscv_sf_vc_i_se_u8mf4(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8mf2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8mf2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8mf2(size_t vl) {
+  __riscv_sf_vc_i_se_u8mf2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m1.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m1.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8m1(size_t vl) {
+  __riscv_sf_vc_i_se_u8m1(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8m2(size_t vl) {
+  __riscv_sf_vc_i_se_u8m2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m4.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m4.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8m4(size_t vl) {
+  __riscv_sf_vc_i_se_u8m4(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m8.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e8m8.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u8m8(size_t vl) {
+  __riscv_sf_vc_i_se_u8m8(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16mf4.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16mf4.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u16mf4(size_t vl) {
+  __riscv_sf_vc_i_se_u16mf4(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16mf2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16mf2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u16mf2(size_t vl) {
+  __riscv_sf_vc_i_se_u16mf2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m1.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m1.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u16m1(size_t vl) {
+  __riscv_sf_vc_i_se_u16m1(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u16m2(size_t vl) {
+  __riscv_sf_vc_i_se_u16m2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m4.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m4.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u16m4(size_t vl) {
+  __riscv_sf_vc_i_se_u16m4(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m8.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e16m8.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u16m8(size_t vl) {
+  __riscv_sf_vc_i_se_u16m8(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32mf2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32mf2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u32mf2(size_t vl) {
+  __riscv_sf_vc_i_se_u32mf2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m1.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m1.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u32m1(size_t vl) {
+  __riscv_sf_vc_i_se_u32m1(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u32m2(size_t vl) {
+  __riscv_sf_vc_i_se_u32m2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m4.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m4.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u32m4(size_t vl) {
+  __riscv_sf_vc_i_se_u32m4(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m8.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e32m8.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u32m8(size_t vl) {
+  __riscv_sf_vc_i_se_u32m8(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m1.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m1.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u64m1(size_t vl) {
+  __riscv_sf_vc_i_se_u64m1(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m2.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m2.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u64m2(size_t vl) {
+  __riscv_sf_vc_i_se_u64m2(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m4.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m4.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u64m4(size_t vl) {
+  __riscv_sf_vc_i_se_u64m4(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_i_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m8.i32.i32.i32(i32 3, i32 31, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_i_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.i.se.e64m8.i64.i64.i64(i64 3, i64 31, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_i_se_u64m8(size_t vl) {
+  __riscv_sf_vc_i_se_u64m8(p27_26, p24_20, p11_7, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.i.se.nxv1i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.i.se.nxv1i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8mf8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.i.se.nxv2i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.i.se.nxv2i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_i_se_u8mf4(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8mf4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.i.se.nxv4i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.i.se.nxv4i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_i_se_u8mf2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8mf2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.i.se.nxv8i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.i.se.nxv8i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_i_se_u8m1(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.i.se.nxv16i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.i.se.nxv16i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_i_se_u8m2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.i.se.nxv32i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.i.se.nxv32i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_i_se_u8m4(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.i.se.nxv64i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.i.se.nxv64i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_i_se_u8m8(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u8m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.i.se.nxv1i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.i.se.nxv1i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_i_se_u16mf4(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u16mf4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.i.se.nxv2i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.i.se.nxv2i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_i_se_u16mf2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u16mf2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.i.se.nxv4i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.i.se.nxv4i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_i_se_u16m1(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u16m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.i.se.nxv8i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.i.se.nxv8i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_i_se_u16m2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u16m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.i.se.nxv16i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.i.se.nxv16i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_i_se_u16m4(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u16m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.i.se.nxv32i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.i.se.nxv32i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_i_se_u16m8(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u16m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.i.se.nxv1i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.i.se.nxv1i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_i_se_u32mf2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u32mf2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.i.se.nxv2i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.i.se.nxv2i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_i_se_u32m1(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u32m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.i.se.nxv4i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.i.se.nxv4i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_i_se_u32m2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u32m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.i.se.nxv8i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.i.se.nxv8i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_i_se_u32m4(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u32m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.i.se.nxv16i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.i.se.nxv16i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_i_se_u32m8(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u32m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.i.se.nxv1i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.i.se.nxv1i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_i_se_u64m1(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u64m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.i.se.nxv2i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.i.se.nxv2i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_i_se_u64m2(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u64m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.i.se.nxv4i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.i.se.nxv4i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_i_se_u64m4(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u64m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.i.se.nxv8i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.i.se.nxv8i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_i_se_u64m8(size_t vl) {
+  return __riscv_sf_vc_v_i_se_u64m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.i.nxv1i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.i.nxv1i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_i_u8mf8(size_t vl) {
+  return __riscv_sf_vc_v_i_u8mf8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.i.nxv2i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.i.nxv2i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_i_u8mf4(size_t vl) {
+  return __riscv_sf_vc_v_i_u8mf4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.i.nxv4i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.i.nxv4i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_i_u8mf2(size_t vl) {
+  return __riscv_sf_vc_v_i_u8mf2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.i.nxv8i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.i.nxv8i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_i_u8m1(size_t vl) {
+  return __riscv_sf_vc_v_i_u8m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.i.nxv16i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.i.nxv16i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_i_u8m2(size_t vl) {
+  return __riscv_sf_vc_v_i_u8m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.i.nxv32i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.i.nxv32i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_i_u8m4(size_t vl) {
+  return __riscv_sf_vc_v_i_u8m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.i.nxv64i8.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.i.nxv64i8.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_i_u8m8(size_t vl) {
+  return __riscv_sf_vc_v_i_u8m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.i.nxv1i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.i.nxv1i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_i_u16mf4(size_t vl) {
+  return __riscv_sf_vc_v_i_u16mf4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.i.nxv2i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.i.nxv2i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_i_u16mf2(size_t vl) {
+  return __riscv_sf_vc_v_i_u16mf2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.i.nxv4i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.i.nxv4i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_i_u16m1(size_t vl) {
+  return __riscv_sf_vc_v_i_u16m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.i.nxv8i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.i.nxv8i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_i_u16m2(size_t vl) {
+  return __riscv_sf_vc_v_i_u16m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.i.nxv16i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.i.nxv16i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_i_u16m4(size_t vl) {
+  return __riscv_sf_vc_v_i_u16m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.i.nxv32i16.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.i.nxv32i16.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_i_u16m8(size_t vl) {
+  return __riscv_sf_vc_v_i_u16m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.i.nxv1i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.i.nxv1i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_i_u32mf2(size_t vl) {
+  return __riscv_sf_vc_v_i_u32mf2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.i.nxv2i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.i.nxv2i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_i_u32m1(size_t vl) {
+  return __riscv_sf_vc_v_i_u32m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.i.nxv4i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.i.nxv4i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_i_u32m2(size_t vl) {
+  return __riscv_sf_vc_v_i_u32m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.i.nxv8i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.i.nxv8i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_i_u32m4(size_t vl) {
+  return __riscv_sf_vc_v_i_u32m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.i.nxv16i32.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.i.nxv16i32.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_i_u32m8(size_t vl) {
+  return __riscv_sf_vc_v_i_u32m8(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.i.nxv1i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.i.nxv1i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_i_u64m1(size_t vl) {
+  return __riscv_sf_vc_v_i_u64m1(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.i.nxv2i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.i.nxv2i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_i_u64m2(size_t vl) {
+  return __riscv_sf_vc_v_i_u64m2(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.i.nxv4i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.i.nxv4i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_i_u64m4(size_t vl) {
+  return __riscv_sf_vc_v_i_u64m4(p27_26, p24_20, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_i_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.i.nxv8i64.i32.i32.i32(i32 3, i32 31, i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_i_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.i.nxv8i64.i64.i64.i64(i64 3, i64 31, i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_i_u64m8(size_t vl) {
+  return __riscv_sf_vc_v_i_u64m8(p27_26, p24_20, simm5, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xv-rv64.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xv-rv64.c
new file mode 100644
index 0000000000000..39704189ae45e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xv-rv64.c
@@ -0,0 +1,116 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p11_7  (0b11111)
+
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv1i64.i64.i64(i64 3, i64 31, <vscale x 1 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u64m1(vuint64m1_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u64m1(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv2i64.i64.i64(i64 3, i64 31, <vscale x 2 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u64m2(vuint64m2_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u64m2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv4i64.i64.i64(i64 3, i64 31, <vscale x 4 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u64m4(vuint64m4_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u64m4(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv8i64.i64.i64(i64 3, i64 31, <vscale x 8 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u64m8(vuint64m8_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u64m8(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xv.se.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_xv_se_u64m1(vuint64m1_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u64m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xv.se.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_xv_se_u64m2(vuint64m2_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u64m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xv.se.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_xv_se_u64m4(vuint64m4_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u64m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xv.se.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_xv_se_u64m8(vuint64m8_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u64m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xv.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_xv_u64m1(vuint64m1_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u64m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xv.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_xv_u64m2(vuint64m2_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u64m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xv.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_xv_u64m4(vuint64m4_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u64m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xv.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_xv_u64m8(vuint64m8_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u64m8(p27_26, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xv.c
new file mode 100644
index 0000000000000..f37748ac3020b
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xv.c
@@ -0,0 +1,3245 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p26    (0b1)
+#define p11_7  (0b11111)
+#define simm5  (10)
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv1i8.nxv1i8.i32(i32 3, i32 31, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv1i8.nxv1i8.i64(i64 3, i64 31, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8mf8(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8mf8(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv2i8.nxv2i8.i32(i32 3, i32 31, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv2i8.nxv2i8.i64(i64 3, i64 31, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8mf4(vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8mf4(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv4i8.nxv4i8.i32(i32 3, i32 31, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv4i8.nxv4i8.i64(i64 3, i64 31, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8mf2(vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8mf2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv8i8.nxv8i8.i32(i32 3, i32 31, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv8i8.nxv8i8.i64(i64 3, i64 31, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8m1(vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8m1(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv16i8.nxv16i8.i32(i32 3, i32 31, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv16i8.nxv16i8.i64(i64 3, i64 31, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8m2(vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8m2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv32i8.nxv32i8.i32(i32 3, i32 31, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv32i8.nxv32i8.i64(i64 3, i64 31, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8m4(vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8m4(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv64i8.nxv64i8.i32(i32 3, i32 31, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv64i8.nxv64i8.i64(i64 3, i64 31, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u8m8(vuint8m8_t vs2, vuint8m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u8m8(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv1i16.nxv1i16.i32(i32 3, i32 31, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv1i16.nxv1i16.i64(i64 3, i64 31, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u16mf4(vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u16mf4(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv2i16.nxv2i16.i32(i32 3, i32 31, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv2i16.nxv2i16.i64(i64 3, i64 31, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u16mf2(vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u16mf2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv4i16.nxv4i16.i32(i32 3, i32 31, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv4i16.nxv4i16.i64(i64 3, i64 31, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u16m1(vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u16m1(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv8i16.nxv8i16.i32(i32 3, i32 31, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv8i16.nxv8i16.i64(i64 3, i64 31, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u16m2(vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u16m2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv16i16.nxv16i16.i32(i32 3, i32 31, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv16i16.nxv16i16.i64(i64 3, i64 31, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u16m4(vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u16m4(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv32i16.nxv32i16.i32(i32 3, i32 31, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv32i16.nxv32i16.i64(i64 3, i64 31, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u16m8(vuint16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u16m8(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv1i32.nxv1i32.i32(i32 3, i32 31, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv1i32.nxv1i32.i64(i64 3, i64 31, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u32mf2(vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u32mf2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv2i32.nxv2i32.i32(i32 3, i32 31, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv2i32.nxv2i32.i64(i64 3, i64 31, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u32m1(vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u32m1(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv4i32.nxv4i32.i32(i32 3, i32 31, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv4i32.nxv4i32.i64(i64 3, i64 31, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u32m2(vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u32m2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv8i32.nxv8i32.i32(i32 3, i32 31, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv8i32.nxv8i32.i64(i64 3, i64 31, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u32m4(vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u32m4(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv16i32.nxv16i32.i32(i32 3, i32 31, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv16i32.nxv16i32.i64(i64 3, i64 31, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u32m8(vuint32m8_t vs2, vuint32m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u32m8(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv1i64.nxv1i64.i32(i32 3, i32 31, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv1i64.nxv1i64.i64(i64 3, i64 31, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u64m1(vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u64m1(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv2i64.nxv2i64.i32(i32 3, i32 31, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv2i64.nxv2i64.i64(i64 3, i64 31, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u64m2(vuint64m2_t vs2, vuint64m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u64m2(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv4i64.nxv4i64.i32(i32 3, i32 31, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv4i64.nxv4i64.i64(i64 3, i64 31, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u64m4(vuint64m4_t vs2, vuint64m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u64m4(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i32.nxv8i64.nxv8i64.i32(i32 3, i32 31, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vv.se.i64.nxv8i64.nxv8i64.i64(i64 3, i64 31, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vv_se_u64m8(vuint64m8_t vs2, vuint64m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vv_se_u64m8(p27_26, p11_7, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv1i8.i32.nxv1i8.i32(i32 3, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv1i8.i64.nxv1i8.i64(i64 3, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_vv_se_u8mf8(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8mf8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv2i8.i32.nxv2i8.i32(i32 3, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv2i8.i64.nxv2i8.i64(i64 3, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_vv_se_u8mf4(vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8mf4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv4i8.i32.nxv4i8.i32(i32 3, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv4i8.i64.nxv4i8.i64(i64 3, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_vv_se_u8mf2(vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8mf2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv8i8.i32.nxv8i8.i32(i32 3, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv8i8.i64.nxv8i8.i64(i64 3, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_vv_se_u8m1(vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv16i8.i32.nxv16i8.i32(i32 3, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv16i8.i64.nxv16i8.i64(i64 3, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_vv_se_u8m2(vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv32i8.i32.nxv32i8.i32(i32 3, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv32i8.i64.nxv32i8.i64(i64 3, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_vv_se_u8m4(vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv64i8.i32.nxv64i8.i32(i32 3, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vv.se.nxv64i8.i64.nxv64i8.i64(i64 3, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_vv_se_u8m8(vuint8m8_t vs2, vuint8m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u8m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv1i16.i32.nxv1i16.i32(i32 3, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv1i16.i64.nxv1i16.i64(i64 3, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_vv_se_u16mf4(vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u16mf4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv2i16.i32.nxv2i16.i32(i32 3, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv2i16.i64.nxv2i16.i64(i64 3, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_vv_se_u16mf2(vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u16mf2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv4i16.i32.nxv4i16.i32(i32 3, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv4i16.i64.nxv4i16.i64(i64 3, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_vv_se_u16m1(vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u16m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv8i16.i32.nxv8i16.i32(i32 3, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv8i16.i64.nxv8i16.i64(i64 3, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_vv_se_u16m2(vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u16m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv16i16.i32.nxv16i16.i32(i32 3, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv16i16.i64.nxv16i16.i64(i64 3, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_vv_se_u16m4(vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u16m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv32i16.i32.nxv32i16.i32(i32 3, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vv.se.nxv32i16.i64.nxv32i16.i64(i64 3, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_vv_se_u16m8(vuint16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u16m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv1i32.i32.nxv1i32.i32(i32 3, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv1i32.i64.nxv1i32.i64(i64 3, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_vv_se_u32mf2(vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u32mf2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv2i32.i32.nxv2i32.i32(i32 3, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv2i32.i64.nxv2i32.i64(i64 3, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_vv_se_u32m1(vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u32m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv4i32.i32.nxv4i32.i32(i32 3, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv4i32.i64.nxv4i32.i64(i64 3, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_vv_se_u32m2(vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u32m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv8i32.i32.nxv8i32.i32(i32 3, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv8i32.i64.nxv8i32.i64(i64 3, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_vv_se_u32m4(vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u32m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv16i32.i32.nxv16i32.i32(i32 3, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vv.se.nxv16i32.i64.nxv16i32.i64(i64 3, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_vv_se_u32m8(vuint32m8_t vs2, vuint32m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u32m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv1i64.i32.nxv1i64.i32(i32 3, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv1i64.i64.nxv1i64.i64(i64 3, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_vv_se_u64m1(vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u64m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv2i64.i32.nxv2i64.i32(i32 3, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv2i64.i64.nxv2i64.i64(i64 3, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_vv_se_u64m2(vuint64m2_t vs2, vuint64m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u64m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv4i64.i32.nxv4i64.i32(i32 3, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv4i64.i64.nxv4i64.i64(i64 3, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_vv_se_u64m4(vuint64m4_t vs2, vuint64m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u64m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv8i64.i32.nxv8i64.i32(i32 3, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vv.se.nxv8i64.i64.nxv8i64.i64(i64 3, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_vv_se_u64m8(vuint64m8_t vs2, vuint64m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_se_u64m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vv.nxv1i8.i32.nxv1i8.i32(i32 3, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vv.nxv1i8.i64.nxv1i8.i64(i64 3, <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_vv_u8mf8(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8mf8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vv.nxv2i8.i32.nxv2i8.i32(i32 3, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vv.nxv2i8.i64.nxv2i8.i64(i64 3, <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_vv_u8mf4(vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8mf4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vv.nxv4i8.i32.nxv4i8.i32(i32 3, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vv.nxv4i8.i64.nxv4i8.i64(i64 3, <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_vv_u8mf2(vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8mf2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vv.nxv8i8.i32.nxv8i8.i32(i32 3, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vv.nxv8i8.i64.nxv8i8.i64(i64 3, <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_vv_u8m1(vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vv.nxv16i8.i32.nxv16i8.i32(i32 3, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vv.nxv16i8.i64.nxv16i8.i64(i64 3, <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_vv_u8m2(vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vv.nxv32i8.i32.nxv32i8.i32(i32 3, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vv.nxv32i8.i64.nxv32i8.i64(i64 3, <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_vv_u8m4(vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vv.nxv64i8.i32.nxv64i8.i32(i32 3, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vv.nxv64i8.i64.nxv64i8.i64(i64 3, <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_vv_u8m8(vuint8m8_t vs2, vuint8m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u8m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vv.nxv1i16.i32.nxv1i16.i32(i32 3, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vv.nxv1i16.i64.nxv1i16.i64(i64 3, <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_vv_u16mf4(vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u16mf4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vv.nxv2i16.i32.nxv2i16.i32(i32 3, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vv.nxv2i16.i64.nxv2i16.i64(i64 3, <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_vv_u16mf2(vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u16mf2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vv.nxv4i16.i32.nxv4i16.i32(i32 3, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vv.nxv4i16.i64.nxv4i16.i64(i64 3, <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_vv_u16m1(vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u16m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vv.nxv8i16.i32.nxv8i16.i32(i32 3, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vv.nxv8i16.i64.nxv8i16.i64(i64 3, <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_vv_u16m2(vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u16m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vv.nxv16i16.i32.nxv16i16.i32(i32 3, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vv.nxv16i16.i64.nxv16i16.i64(i64 3, <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_vv_u16m4(vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u16m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vv.nxv32i16.i32.nxv32i16.i32(i32 3, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vv.nxv32i16.i64.nxv32i16.i64(i64 3, <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_vv_u16m8(vuint16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u16m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vv.nxv1i32.i32.nxv1i32.i32(i32 3, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vv.nxv1i32.i64.nxv1i32.i64(i64 3, <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_vv_u32mf2(vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u32mf2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vv.nxv2i32.i32.nxv2i32.i32(i32 3, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vv.nxv2i32.i64.nxv2i32.i64(i64 3, <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_vv_u32m1(vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u32m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vv.nxv4i32.i32.nxv4i32.i32(i32 3, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vv.nxv4i32.i64.nxv4i32.i64(i64 3, <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_vv_u32m2(vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u32m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vv.nxv8i32.i32.nxv8i32.i32(i32 3, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vv.nxv8i32.i64.nxv8i32.i64(i64 3, <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_vv_u32m4(vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u32m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vv.nxv16i32.i32.nxv16i32.i32(i32 3, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vv.nxv16i32.i64.nxv16i32.i64(i64 3, <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_vv_u32m8(vuint32m8_t vs2, vuint32m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u32m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vv.nxv1i64.i32.nxv1i64.i32(i32 3, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vv.nxv1i64.i64.nxv1i64.i64(i64 3, <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_vv_u64m1(vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u64m1(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vv.nxv2i64.i32.nxv2i64.i32(i32 3, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vv.nxv2i64.i64.nxv2i64.i64(i64 3, <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_vv_u64m2(vuint64m2_t vs2, vuint64m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u64m2(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vv.nxv4i64.i32.nxv4i64.i32(i32 3, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vv.nxv4i64.i64.nxv4i64.i64(i64 3, <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_vv_u64m4(vuint64m4_t vs2, vuint64m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u64m4(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vv_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vv.nxv8i64.i32.nxv8i64.i32(i32 3, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vv.nxv8i64.i64.nxv8i64.i64(i64 3, <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_vv_u64m8(vuint64m8_t vs2, vuint64m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vv_u64m8(p27_26, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv1i8.i8.i32(i32 3, i32 31, <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv1i8.i8.i64(i64 3, i64 31, <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8mf8(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8mf8(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv2i8.i8.i32(i32 3, i32 31, <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv2i8.i8.i64(i64 3, i64 31, <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8mf4(vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8mf4(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv4i8.i8.i32(i32 3, i32 31, <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv4i8.i8.i64(i64 3, i64 31, <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8mf2(vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8mf2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv8i8.i8.i32(i32 3, i32 31, <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv8i8.i8.i64(i64 3, i64 31, <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8m1(vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8m1(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv16i8.i8.i32(i32 3, i32 31, <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv16i8.i8.i64(i64 3, i64 31, <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8m2(vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8m2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv32i8.i8.i32(i32 3, i32 31, <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv32i8.i8.i64(i64 3, i64 31, <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8m4(vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8m4(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv64i8.i8.i32(i32 3, i32 31, <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv64i8.i8.i64(i64 3, i64 31, <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u8m8(vuint8m8_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u8m8(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv1i16.i16.i32(i32 3, i32 31, <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv1i16.i16.i64(i64 3, i64 31, <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u16mf4(vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u16mf4(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv2i16.i16.i32(i32 3, i32 31, <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv2i16.i16.i64(i64 3, i64 31, <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u16mf2(vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u16mf2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv4i16.i16.i32(i32 3, i32 31, <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv4i16.i16.i64(i64 3, i64 31, <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u16m1(vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u16m1(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv8i16.i16.i32(i32 3, i32 31, <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv8i16.i16.i64(i64 3, i64 31, <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u16m2(vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u16m2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv16i16.i16.i32(i32 3, i32 31, <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv16i16.i16.i64(i64 3, i64 31, <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u16m4(vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u16m4(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv32i16.i16.i32(i32 3, i32 31, <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv32i16.i16.i64(i64 3, i64 31, <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u16m8(vuint16m8_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u16m8(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv1i32.i32.i32(i32 3, i32 31, <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv1i32.i32.i64(i64 3, i64 31, <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u32mf2(vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u32mf2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv2i32.i32.i32(i32 3, i32 31, <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv2i32.i32.i64(i64 3, i64 31, <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u32m1(vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u32m1(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv4i32.i32.i32(i32 3, i32 31, <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv4i32.i32.i64(i64 3, i64 31, <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u32m2(vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u32m2(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv8i32.i32.i32(i32 3, i32 31, <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv8i32.i32.i64(i64 3, i64 31, <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u32m4(vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u32m4(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i32.nxv16i32.i32.i32(i32 3, i32 31, <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xv.se.i64.nxv16i32.i32.i64(i64 3, i64 31, <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xv_se_u32m8(vuint32m8_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xv_se_u32m8(p27_26, p11_7, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv1i8.i32.i8.i32(i32 3, <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv1i8.i64.i8.i64(i64 3, <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_xv_se_u8mf8(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8mf8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv2i8.i32.i8.i32(i32 3, <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv2i8.i64.i8.i64(i64 3, <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_xv_se_u8mf4(vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8mf4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv4i8.i32.i8.i32(i32 3, <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv4i8.i64.i8.i64(i64 3, <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_xv_se_u8mf2(vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8mf2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv8i8.i32.i8.i32(i32 3, <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv8i8.i64.i8.i64(i64 3, <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_xv_se_u8m1(vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv16i8.i32.i8.i32(i32 3, <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv16i8.i64.i8.i64(i64 3, <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_xv_se_u8m2(vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv32i8.i32.i8.i32(i32 3, <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv32i8.i64.i8.i64(i64 3, <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_xv_se_u8m4(vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv64i8.i32.i8.i32(i32 3, <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xv.se.nxv64i8.i64.i8.i64(i64 3, <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_xv_se_u8m8(vuint8m8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u8m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv1i16.i32.i16.i32(i32 3, <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv1i16.i64.i16.i64(i64 3, <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_xv_se_u16mf4(vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u16mf4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv2i16.i32.i16.i32(i32 3, <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv2i16.i64.i16.i64(i64 3, <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_xv_se_u16mf2(vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u16mf2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv4i16.i32.i16.i32(i32 3, <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv4i16.i64.i16.i64(i64 3, <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_xv_se_u16m1(vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u16m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv8i16.i32.i16.i32(i32 3, <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv8i16.i64.i16.i64(i64 3, <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_xv_se_u16m2(vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u16m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv16i16.i32.i16.i32(i32 3, <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv16i16.i64.i16.i64(i64 3, <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_xv_se_u16m4(vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u16m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv32i16.i32.i16.i32(i32 3, <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xv.se.nxv32i16.i64.i16.i64(i64 3, <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_xv_se_u16m8(vuint16m8_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u16m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv1i32.i64.i32.i64(i64 3, <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_xv_se_u32mf2(vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u32mf2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv2i32.i64.i32.i64(i64 3, <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_xv_se_u32m1(vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u32m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv4i32.i64.i32.i64(i64 3, <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_xv_se_u32m2(vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u32m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv8i32.i64.i32.i64(i64 3, <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_xv_se_u32m4(vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u32m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xv.se.nxv16i32.i64.i32.i64(i64 3, <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_xv_se_u32m8(vuint32m8_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_se_u32m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xv.nxv1i8.i32.i8.i32(i32 3, <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xv.nxv1i8.i64.i8.i64(i64 3, <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_xv_u8mf8(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8mf8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xv.nxv2i8.i32.i8.i32(i32 3, <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xv.nxv2i8.i64.i8.i64(i64 3, <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_xv_u8mf4(vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8mf4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xv.nxv4i8.i32.i8.i32(i32 3, <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xv.nxv4i8.i64.i8.i64(i64 3, <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_xv_u8mf2(vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8mf2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xv.nxv8i8.i32.i8.i32(i32 3, <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xv.nxv8i8.i64.i8.i64(i64 3, <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_xv_u8m1(vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xv.nxv16i8.i32.i8.i32(i32 3, <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xv.nxv16i8.i64.i8.i64(i64 3, <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_xv_u8m2(vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xv.nxv32i8.i32.i8.i32(i32 3, <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xv.nxv32i8.i64.i8.i64(i64 3, <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_xv_u8m4(vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xv.nxv64i8.i32.i8.i32(i32 3, <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xv.nxv64i8.i64.i8.i64(i64 3, <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_xv_u8m8(vuint8m8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u8m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xv.nxv1i16.i32.i16.i32(i32 3, <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xv.nxv1i16.i64.i16.i64(i64 3, <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_xv_u16mf4(vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u16mf4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xv.nxv2i16.i32.i16.i32(i32 3, <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xv.nxv2i16.i64.i16.i64(i64 3, <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_xv_u16mf2(vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u16mf2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xv.nxv4i16.i32.i16.i32(i32 3, <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xv.nxv4i16.i64.i16.i64(i64 3, <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_xv_u16m1(vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u16m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xv.nxv8i16.i32.i16.i32(i32 3, <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xv.nxv8i16.i64.i16.i64(i64 3, <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_xv_u16m2(vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u16m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xv.nxv16i16.i32.i16.i32(i32 3, <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xv.nxv16i16.i64.i16.i64(i64 3, <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_xv_u16m4(vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u16m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xv.nxv32i16.i32.i16.i32(i32 3, <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xv.nxv32i16.i64.i16.i64(i64 3, <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_xv_u16m8(vuint16m8_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u16m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xv.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xv.nxv1i32.i64.i32.i64(i64 3, <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_xv_u32mf2(vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u32mf2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xv.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xv.nxv2i32.i64.i32.i64(i64 3, <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_xv_u32m1(vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u32m1(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xv.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xv.nxv4i32.i64.i32.i64(i64 3, <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_xv_u32m2(vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u32m2(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xv.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xv.nxv8i32.i64.i32.i64(i64 3, <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_xv_u32m4(vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u32m4(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xv.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xv.nxv16i32.i64.i32.i64(i64 3, <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_xv_u32m8(vuint32m8_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xv_u32m8(p27_26, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv1i8.i32.i32(i32 3, i32 31, <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv1i8.i64.i64(i64 3, i64 31, <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8mf8(vuint8mf8_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8mf8(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv2i8.i32.i32(i32 3, i32 31, <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv2i8.i64.i64(i64 3, i64 31, <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8mf4(vuint8mf4_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8mf4(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv4i8.i32.i32(i32 3, i32 31, <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv4i8.i64.i64(i64 3, i64 31, <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8mf2(vuint8mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8mf2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv8i8.i32.i32(i32 3, i32 31, <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv8i8.i64.i64(i64 3, i64 31, <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8m1(vuint8m1_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8m1(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv16i8.i32.i32(i32 3, i32 31, <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv16i8.i64.i64(i64 3, i64 31, <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8m2(vuint8m2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8m2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv32i8.i32.i32(i32 3, i32 31, <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv32i8.i64.i64(i64 3, i64 31, <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8m4(vuint8m4_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8m4(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv64i8.i32.i32(i32 3, i32 31, <vscale x 64 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv64i8.i64.i64(i64 3, i64 31, <vscale x 64 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u8m8(vuint8m8_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u8m8(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv1i16.i32.i32(i32 3, i32 31, <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv1i16.i64.i64(i64 3, i64 31, <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u16mf4(vuint16mf4_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u16mf4(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv2i16.i32.i32(i32 3, i32 31, <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv2i16.i64.i64(i64 3, i64 31, <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u16mf2(vuint16mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u16mf2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv4i16.i32.i32(i32 3, i32 31, <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv4i16.i64.i64(i64 3, i64 31, <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u16m1(vuint16m1_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u16m1(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv8i16.i32.i32(i32 3, i32 31, <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv8i16.i64.i64(i64 3, i64 31, <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u16m2(vuint16m2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u16m2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv16i16.i32.i32(i32 3, i32 31, <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv16i16.i64.i64(i64 3, i64 31, <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u16m4(vuint16m4_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u16m4(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv32i16.i32.i32(i32 3, i32 31, <vscale x 32 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv32i16.i64.i64(i64 3, i64 31, <vscale x 32 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u16m8(vuint16m8_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u16m8(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv1i32.i32.i32(i32 3, i32 31, <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv1i32.i64.i64(i64 3, i64 31, <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u32mf2(vuint32mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u32mf2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv2i32.i32.i32(i32 3, i32 31, <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv2i32.i64.i64(i64 3, i64 31, <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u32m1(vuint32m1_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u32m1(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv4i32.i32.i32(i32 3, i32 31, <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv4i32.i64.i64(i64 3, i64 31, <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u32m2(vuint32m2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u32m2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv8i32.i32.i32(i32 3, i32 31, <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv8i32.i64.i64(i64 3, i64 31, <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u32m4(vuint32m4_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u32m4(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv16i32.i32.i32(i32 3, i32 31, <vscale x 16 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv16i32.i64.i64(i64 3, i64 31, <vscale x 16 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u32m8(vuint32m8_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u32m8(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv1i64.i32.i32(i32 3, i32 31, <vscale x 1 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv1i64.i64.i64(i64 3, i64 31, <vscale x 1 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u64m1(vuint64m1_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u64m1(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv2i64.i32.i32(i32 3, i32 31, <vscale x 2 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv2i64.i64.i64(i64 3, i64 31, <vscale x 2 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u64m2(vuint64m2_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u64m2(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv4i64.i32.i32(i32 3, i32 31, <vscale x 4 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv4i64.i64.i64(i64 3, i64 31, <vscale x 4 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u64m4(vuint64m4_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u64m4(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_iv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i32.nxv8i64.i32.i32(i32 3, i32 31, <vscale x 8 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_iv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.iv.se.i64.nxv8i64.i64.i64(i64 3, i64 31, <vscale x 8 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_iv_se_u64m8(vuint64m8_t vs2, size_t vl) {
+  __riscv_sf_vc_iv_se_u64m8(p27_26, p11_7, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv1i8.i32.i32.i32(i32 3, <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv1i8.i64.i64.i64(i64 3, <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_iv_se_u8mf8(vuint8mf8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8mf8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv2i8.i32.i32.i32(i32 3, <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv2i8.i64.i64.i64(i64 3, <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_iv_se_u8mf4(vuint8mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8mf4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv4i8.i32.i32.i32(i32 3, <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv4i8.i64.i64.i64(i64 3, <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_iv_se_u8mf2(vuint8mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8mf2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv8i8.i32.i32.i32(i32 3, <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv8i8.i64.i64.i64(i64 3, <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_iv_se_u8m1(vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv16i8.i32.i32.i32(i32 3, <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv16i8.i64.i64.i64(i64 3, <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_iv_se_u8m2(vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv32i8.i32.i32.i32(i32 3, <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv32i8.i64.i64.i64(i64 3, <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_iv_se_u8m4(vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv64i8.i32.i32.i32(i32 3, <vscale x 64 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.iv.se.nxv64i8.i64.i64.i64(i64 3, <vscale x 64 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_iv_se_u8m8(vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u8m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv1i16.i32.i32.i32(i32 3, <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv1i16.i64.i64.i64(i64 3, <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_iv_se_u16mf4(vuint16mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u16mf4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv2i16.i32.i32.i32(i32 3, <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv2i16.i64.i64.i64(i64 3, <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_iv_se_u16mf2(vuint16mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u16mf2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv4i16.i32.i32.i32(i32 3, <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv4i16.i64.i64.i64(i64 3, <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_iv_se_u16m1(vuint16m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u16m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv8i16.i32.i32.i32(i32 3, <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv8i16.i64.i64.i64(i64 3, <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_iv_se_u16m2(vuint16m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u16m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv16i16.i32.i32.i32(i32 3, <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv16i16.i64.i64.i64(i64 3, <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_iv_se_u16m4(vuint16m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u16m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv32i16.i32.i32.i32(i32 3, <vscale x 32 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.iv.se.nxv32i16.i64.i64.i64(i64 3, <vscale x 32 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_iv_se_u16m8(vuint16m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u16m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv1i32.i64.i64.i64(i64 3, <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_iv_se_u32mf2(vuint32mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u32mf2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv2i32.i64.i64.i64(i64 3, <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_iv_se_u32m1(vuint32m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u32m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv4i32.i64.i64.i64(i64 3, <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_iv_se_u32m2(vuint32m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u32m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv8i32.i64.i64.i64(i64 3, <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_iv_se_u32m4(vuint32m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u32m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.iv.se.nxv16i32.i64.i64.i64(i64 3, <vscale x 16 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_iv_se_u32m8(vuint32m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u32m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv1i64.i32.i32.i32(i32 3, <vscale x 1 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_iv_se_u64m1(vuint64m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u64m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv2i64.i32.i32.i32(i32 3, <vscale x 2 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_iv_se_u64m2(vuint64m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u64m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv4i64.i32.i32.i32(i32 3, <vscale x 4 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_iv_se_u64m4(vuint64m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u64m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv8i64.i32.i32.i32(i32 3, <vscale x 8 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.iv.se.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_iv_se_u64m8(vuint64m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_se_u64m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.iv.nxv1i8.i32.i32.i32(i32 3, <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.iv.nxv1i8.i64.i64.i64(i64 3, <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_iv_u8mf8(vuint8mf8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8mf8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.iv.nxv2i8.i32.i32.i32(i32 3, <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.iv.nxv2i8.i64.i64.i64(i64 3, <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_iv_u8mf4(vuint8mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8mf4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.iv.nxv4i8.i32.i32.i32(i32 3, <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.iv.nxv4i8.i64.i64.i64(i64 3, <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_iv_u8mf2(vuint8mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8mf2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.iv.nxv8i8.i32.i32.i32(i32 3, <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.iv.nxv8i8.i64.i64.i64(i64 3, <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_iv_u8m1(vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.iv.nxv16i8.i32.i32.i32(i32 3, <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.iv.nxv16i8.i64.i64.i64(i64 3, <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_iv_u8m2(vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.iv.nxv32i8.i32.i32.i32(i32 3, <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.iv.nxv32i8.i64.i64.i64(i64 3, <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_iv_u8m4(vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.iv.nxv64i8.i32.i32.i32(i32 3, <vscale x 64 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.iv.nxv64i8.i64.i64.i64(i64 3, <vscale x 64 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_iv_u8m8(vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u8m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.iv.nxv1i16.i32.i32.i32(i32 3, <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.iv.nxv1i16.i64.i64.i64(i64 3, <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_iv_u16mf4(vuint16mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u16mf4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.iv.nxv2i16.i32.i32.i32(i32 3, <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.iv.nxv2i16.i64.i64.i64(i64 3, <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_iv_u16mf2(vuint16mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u16mf2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.iv.nxv4i16.i32.i32.i32(i32 3, <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.iv.nxv4i16.i64.i64.i64(i64 3, <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_iv_u16m1(vuint16m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u16m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.iv.nxv8i16.i32.i32.i32(i32 3, <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.iv.nxv8i16.i64.i64.i64(i64 3, <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_iv_u16m2(vuint16m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u16m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.iv.nxv16i16.i32.i32.i32(i32 3, <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.iv.nxv16i16.i64.i64.i64(i64 3, <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_iv_u16m4(vuint16m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u16m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.iv.nxv32i16.i32.i32.i32(i32 3, <vscale x 32 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.iv.nxv32i16.i64.i64.i64(i64 3, <vscale x 32 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_iv_u16m8(vuint16m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u16m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.iv.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.iv.nxv1i32.i64.i64.i64(i64 3, <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_iv_u32mf2(vuint32mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u32mf2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.iv.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.iv.nxv2i32.i64.i64.i64(i64 3, <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_iv_u32m1(vuint32m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u32m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.iv.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.iv.nxv4i32.i64.i64.i64(i64 3, <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_iv_u32m2(vuint32m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u32m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.iv.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.iv.nxv8i32.i64.i64.i64(i64 3, <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_iv_u32m4(vuint32m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u32m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.iv.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.iv.nxv16i32.i64.i64.i64(i64 3, <vscale x 16 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_iv_u32m8(vuint32m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u32m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.iv.nxv1i64.i32.i32.i32(i32 3, <vscale x 1 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.iv.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_iv_u64m1(vuint64m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u64m1(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.iv.nxv2i64.i32.i32.i32(i32 3, <vscale x 2 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.iv.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_iv_u64m2(vuint64m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u64m2(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.iv.nxv4i64.i32.i32.i32(i32 3, <vscale x 4 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.iv.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_iv_u64m4(vuint64m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u64m4(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_iv_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.iv.nxv8i64.i32.i32.i32(i32 3, <vscale x 8 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_iv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.iv.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_iv_u64m8(vuint64m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_iv_u64m8(p27_26, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv1i16.f16.i32(i32 1, i32 31, <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv1i16.f16.i64(i64 1, i64 31, <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u16mf4(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u16mf4(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv2i16.f16.i32(i32 1, i32 31, <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv2i16.f16.i64(i64 1, i64 31, <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u16mf2(vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u16mf2(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv4i16.f16.i32(i32 1, i32 31, <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv4i16.f16.i64(i64 1, i64 31, <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u16m1(vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u16m1(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv8i16.f16.i32(i32 1, i32 31, <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv8i16.f16.i64(i64 1, i64 31, <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u16m2(vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u16m2(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv16i16.f16.i32(i32 1, i32 31, <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv16i16.f16.i64(i64 1, i64 31, <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u16m4(vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u16m4(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv32i16.f16.i32(i32 1, i32 31, <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv32i16.f16.i64(i64 1, i64 31, <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u16m8(vuint16m8_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u16m8(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv1i32.f32.i32(i32 1, i32 31, <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv1i32.f32.i64(i64 1, i64 31, <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u32mf2(vuint32mf2_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u32mf2(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv2i32.f32.i32(i32 1, i32 31, <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv2i32.f32.i64(i64 1, i64 31, <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u32m1(vuint32m1_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u32m1(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv4i32.f32.i32(i32 1, i32 31, <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv4i32.f32.i64(i64 1, i64 31, <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u32m2(vuint32m2_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u32m2(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv8i32.f32.i32(i32 1, i32 31, <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv8i32.f32.i64(i64 1, i64 31, <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u32m4(vuint32m4_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u32m4(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv16i32.f32.i32(i32 1, i32 31, <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv16i32.f32.i64(i64 1, i64 31, <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u32m8(vuint32m8_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u32m8(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv1i64.f64.i32(i32 1, i32 31, <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv1i64.f64.i64(i64 1, i64 31, <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u64m1(vuint64m1_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u64m1(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv2i64.f64.i32(i32 1, i32 31, <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv2i64.f64.i64(i64 1, i64 31, <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u64m2(vuint64m2_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u64m2(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv4i64.f64.i32(i32 1, i32 31, <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv4i64.f64.i64(i64 1, i64 31, <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u64m4(vuint64m4_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u64m4(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i32.nxv8i64.f64.i32(i32 1, i32 31, <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fv.se.i64.nxv8i64.f64.i64(i64 1, i64 31, <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fv_se_u64m8(vuint64m8_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fv_se_u64m8(p26, p11_7, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv1i16.i32.f16.i32(i32 1, <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv1i16.i64.f16.i64(i64 1, <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_fv_se_u16mf4(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u16mf4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv2i16.i32.f16.i32(i32 1, <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv2i16.i64.f16.i64(i64 1, <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_fv_se_u16mf2(vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u16mf2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv4i16.i32.f16.i32(i32 1, <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv4i16.i64.f16.i64(i64 1, <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_fv_se_u16m1(vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u16m1(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv8i16.i32.f16.i32(i32 1, <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv8i16.i64.f16.i64(i64 1, <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_fv_se_u16m2(vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u16m2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv16i16.i32.f16.i32(i32 1, <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv16i16.i64.f16.i64(i64 1, <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_fv_se_u16m4(vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u16m4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv32i16.i32.f16.i32(i32 1, <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fv.se.nxv32i16.i64.f16.i64(i64 1, <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_fv_se_u16m8(vuint16m8_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u16m8(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv1i32.i32.f32.i32(i32 1, <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv1i32.i64.f32.i64(i64 1, <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_fv_se_u32mf2(vuint32mf2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u32mf2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv2i32.i32.f32.i32(i32 1, <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv2i32.i64.f32.i64(i64 1, <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_fv_se_u32m1(vuint32m1_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u32m1(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv4i32.i32.f32.i32(i32 1, <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv4i32.i64.f32.i64(i64 1, <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_fv_se_u32m2(vuint32m2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u32m2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv8i32.i32.f32.i32(i32 1, <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv8i32.i64.f32.i64(i64 1, <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_fv_se_u32m4(vuint32m4_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u32m4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv16i32.i32.f32.i32(i32 1, <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fv.se.nxv16i32.i64.f32.i64(i64 1, <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_fv_se_u32m8(vuint32m8_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u32m8(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv1i64.i32.f64.i32(i32 1, <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv1i64.i64.f64.i64(i64 1, <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_fv_se_u64m1(vuint64m1_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u64m1(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv2i64.i32.f64.i32(i32 1, <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv2i64.i64.f64.i64(i64 1, <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_fv_se_u64m2(vuint64m2_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u64m2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv4i64.i32.f64.i32(i32 1, <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv4i64.i64.f64.i64(i64 1, <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_fv_se_u64m4(vuint64m4_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u64m4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv8i64.i32.f64.i32(i32 1, <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fv.se.nxv8i64.i64.f64.i64(i64 1, <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_fv_se_u64m8(vuint64m8_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_se_u64m8(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fv.nxv1i16.i32.f16.i32(i32 1, <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fv.nxv1i16.i64.f16.i64(i64 1, <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_fv_u16mf4(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u16mf4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fv.nxv2i16.i32.f16.i32(i32 1, <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fv.nxv2i16.i64.f16.i64(i64 1, <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_fv_u16mf2(vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u16mf2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fv.nxv4i16.i32.f16.i32(i32 1, <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fv.nxv4i16.i64.f16.i64(i64 1, <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_fv_u16m1(vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u16m1(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fv.nxv8i16.i32.f16.i32(i32 1, <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fv.nxv8i16.i64.f16.i64(i64 1, <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_fv_u16m2(vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u16m2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fv.nxv16i16.i32.f16.i32(i32 1, <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fv.nxv16i16.i64.f16.i64(i64 1, <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_fv_u16m4(vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u16m4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fv.nxv32i16.i32.f16.i32(i32 1, <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fv.nxv32i16.i64.f16.i64(i64 1, <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_fv_u16m8(vuint16m8_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u16m8(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fv.nxv1i32.i32.f32.i32(i32 1, <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fv.nxv1i32.i64.f32.i64(i64 1, <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_fv_u32mf2(vuint32mf2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u32mf2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fv.nxv2i32.i32.f32.i32(i32 1, <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fv.nxv2i32.i64.f32.i64(i64 1, <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_fv_u32m1(vuint32m1_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u32m1(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fv.nxv4i32.i32.f32.i32(i32 1, <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fv.nxv4i32.i64.f32.i64(i64 1, <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_fv_u32m2(vuint32m2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u32m2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fv.nxv8i32.i32.f32.i32(i32 1, <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fv.nxv8i32.i64.f32.i64(i64 1, <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_fv_u32m4(vuint32m4_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u32m4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fv.nxv16i32.i32.f32.i32(i32 1, <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fv.nxv16i32.i64.f32.i64(i64 1, <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_fv_u32m8(vuint32m8_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u32m8(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fv.nxv1i64.i32.f64.i32(i32 1, <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fv.nxv1i64.i64.f64.i64(i64 1, <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_fv_u64m1(vuint64m1_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u64m1(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fv.nxv2i64.i32.f64.i32(i32 1, <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fv.nxv2i64.i64.f64.i64(i64 1, <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_fv_u64m2(vuint64m2_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u64m2(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fv.nxv4i64.i32.f64.i32(i32 1, <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fv.nxv4i64.i64.f64.i64(i64 1, <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_fv_u64m4(vuint64m4_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u64m4(p26, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fv_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fv.nxv8i64.i32.f64.i32(i32 1, <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fv.nxv8i64.i64.f64.i64(i64 1, <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_fv_u64m8(vuint64m8_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fv_u64m8(p26, vs2, fs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvv-rv64.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvv-rv64.c
new file mode 100644
index 0000000000000..8b0c73776948e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvv-rv64.c
@@ -0,0 +1,115 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv1i64.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u64m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv2i64.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u64m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv4i64.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u64m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv8i64.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, uint64_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u64m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xvv.se.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_xvv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u64m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xvv.se.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_xvv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u64m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xvv.se.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_xvv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u64m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xvv.se.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_xvv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u64m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xvv.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_xvv_u64m1(vuint64m1_t vd, vuint64m1_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u64m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xvv.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_xvv_u64m2(vuint64m2_t vd, vuint64m2_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u64m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xvv.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_xvv_u64m4(vuint64m4_t vd, vuint64m4_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u64m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xvv.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_xvv_u64m8(vuint64m8_t vd, vuint64m8_t vs2, uint64_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u64m8(p27_26, vd, vs2, rs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvv.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvv.c
new file mode 100644
index 0000000000000..4efd7da81bac4
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvv.c
@@ -0,0 +1,3244 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p26    (0b1)
+#define simm5  (10)
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv1i8.nxv1i8.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv1i8.nxv1i8.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8mf8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv2i8.nxv2i8.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv2i8.nxv2i8.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv4i8.nxv4i8.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv4i8.nxv4i8.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv8i8.nxv8i8.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv8i8.nxv8i8.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8m1(vuint8m1_t vd, vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv16i8.nxv16i8.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv16i8.nxv16i8.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8m2(vuint8m2_t vd, vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv32i8.nxv32i8.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv32i8.nxv32i8.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8m4(vuint8m4_t vd, vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv64i8.nxv64i8.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv64i8.nxv64i8.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u8m8(vuint8m8_t vd, vuint8m8_t vs2, vuint8m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u8m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv1i16.nxv1i16.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv1i16.nxv1i16.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u16mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv2i16.nxv2i16.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv2i16.nxv2i16.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u16mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv4i16.nxv4i16.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv4i16.nxv4i16.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u16m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv8i16.nxv8i16.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv8i16.nxv8i16.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u16m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv16i16.nxv16i16.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv16i16.nxv16i16.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u16m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv32i16.nxv32i16.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv32i16.nxv32i16.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u16m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv1i32.nxv1i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv1i32.nxv1i32.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u32mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv2i32.nxv2i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv2i32.nxv2i32.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u32m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv4i32.nxv4i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv4i32.nxv4i32.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u32m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv8i32.nxv8i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv8i32.nxv8i32.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u32m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv16i32.nxv16i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv16i32.nxv16i32.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, vuint32m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u32m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv1i64.nxv1i64.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv1i64.nxv1i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u64m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv2i64.nxv2i64.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv2i64.nxv2i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, vuint64m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u64m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv4i64.nxv4i64.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv4i64.nxv4i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, vuint64m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u64m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i32.nxv8i64.nxv8i64.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvv.se.i64.nxv8i64.nxv8i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, vuint64m8_t vs1, size_t vl) {
+  __riscv_sf_vc_vvv_se_u64m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv1i8.i32.nxv1i8.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv1i8.i64.nxv1i8.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_vvv_se_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8mf8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv2i8.i32.nxv2i8.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv2i8.i64.nxv2i8.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_vvv_se_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv4i8.i32.nxv4i8.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv4i8.i64.nxv4i8.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_vvv_se_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv8i8.i32.nxv8i8.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv8i8.i64.nxv8i8.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_vvv_se_u8m1(vuint8m1_t vd, vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv16i8.i32.nxv16i8.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv16i8.i64.nxv16i8.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_vvv_se_u8m2(vuint8m2_t vd, vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv32i8.i32.nxv32i8.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv32i8.i64.nxv32i8.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_vvv_se_u8m4(vuint8m4_t vd, vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv64i8.i32.nxv64i8.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vvv.se.nxv64i8.i64.nxv64i8.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_vvv_se_u8m8(vuint8m8_t vd, vuint8m8_t vs2, vuint8m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u8m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv1i16.i32.nxv1i16.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv1i16.i64.nxv1i16.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_vvv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u16mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv2i16.i32.nxv2i16.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv2i16.i64.nxv2i16.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_vvv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u16mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv4i16.i32.nxv4i16.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv4i16.i64.nxv4i16.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_vvv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u16m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv8i16.i32.nxv8i16.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv8i16.i64.nxv8i16.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_vvv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u16m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv16i16.i32.nxv16i16.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv16i16.i64.nxv16i16.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_vvv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u16m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv32i16.i32.nxv32i16.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvv.se.nxv32i16.i64.nxv32i16.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_vvv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u16m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv1i32.i32.nxv1i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv1i32.i64.nxv1i32.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_vvv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u32mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv2i32.i32.nxv2i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv2i32.i64.nxv2i32.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_vvv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u32m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv4i32.i32.nxv4i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv4i32.i64.nxv4i32.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_vvv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u32m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv8i32.i32.nxv8i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv8i32.i64.nxv8i32.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_vvv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u32m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv16i32.i32.nxv16i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvv.se.nxv16i32.i64.nxv16i32.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_vvv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, vuint32m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u32m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv1i64.i32.nxv1i64.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv1i64.i64.nxv1i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_vvv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u64m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv2i64.i32.nxv2i64.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv2i64.i64.nxv2i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_vvv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, vuint64m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u64m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv4i64.i32.nxv4i64.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv4i64.i64.nxv4i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_vvv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, vuint64m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u64m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv8i64.i32.nxv8i64.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvv.se.nxv8i64.i64.nxv8i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_vvv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, vuint64m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_se_u64m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vvv.nxv1i8.i32.nxv1i8.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.vvv.nxv1i8.i64.nxv1i8.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_vvv_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8mf8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vvv.nxv2i8.i32.nxv2i8.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.vvv.nxv2i8.i64.nxv2i8.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_vvv_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vvv.nxv4i8.i32.nxv4i8.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.vvv.nxv4i8.i64.nxv4i8.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_vvv_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vvv.nxv8i8.i32.nxv8i8.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.vvv.nxv8i8.i64.nxv8i8.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_vvv_u8m1(vuint8m1_t vd, vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vvv.nxv16i8.i32.nxv16i8.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.vvv.nxv16i8.i64.nxv16i8.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_vvv_u8m2(vuint8m2_t vd, vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vvv.nxv32i8.i32.nxv32i8.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.vvv.nxv32i8.i64.nxv32i8.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_vvv_u8m4(vuint8m4_t vd, vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vvv.nxv64i8.i32.nxv64i8.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.vvv.nxv64i8.i64.nxv64i8.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], <vscale x 64 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_vvv_u8m8(vuint8m8_t vd, vuint8m8_t vs2, vuint8m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u8m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvv.nxv1i16.i32.nxv1i16.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvv.nxv1i16.i64.nxv1i16.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_vvv_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u16mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvv.nxv2i16.i32.nxv2i16.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvv.nxv2i16.i64.nxv2i16.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_vvv_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u16mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvv.nxv4i16.i32.nxv4i16.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvv.nxv4i16.i64.nxv4i16.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_vvv_u16m1(vuint16m1_t vd, vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u16m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvv.nxv8i16.i32.nxv8i16.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvv.nxv8i16.i64.nxv8i16.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_vvv_u16m2(vuint16m2_t vd, vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u16m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvv.nxv16i16.i32.nxv16i16.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvv.nxv16i16.i64.nxv16i16.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_vvv_u16m4(vuint16m4_t vd, vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u16m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvv.nxv32i16.i32.nxv32i16.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvv.nxv32i16.i64.nxv32i16.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], <vscale x 32 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_vvv_u16m8(vuint16m8_t vd, vuint16m8_t vs2, vuint16m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u16m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvv.nxv1i32.i32.nxv1i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvv.nxv1i32.i64.nxv1i32.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_vvv_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u32mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvv.nxv2i32.i32.nxv2i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvv.nxv2i32.i64.nxv2i32.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_vvv_u32m1(vuint32m1_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u32m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvv.nxv4i32.i32.nxv4i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvv.nxv4i32.i64.nxv4i32.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_vvv_u32m2(vuint32m2_t vd, vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u32m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvv.nxv8i32.i32.nxv8i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvv.nxv8i32.i64.nxv8i32.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_vvv_u32m4(vuint32m4_t vd, vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u32m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvv.nxv16i32.i32.nxv16i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvv.nxv16i32.i64.nxv16i32.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], <vscale x 16 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_vvv_u32m8(vuint32m8_t vd, vuint32m8_t vs2, vuint32m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u32m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvv.nxv1i64.i32.nxv1i64.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvv.nxv1i64.i64.nxv1i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], <vscale x 1 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_vvv_u64m1(vuint64m1_t vd, vuint64m1_t vs2, vuint64m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u64m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvv.nxv2i64.i32.nxv2i64.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvv.nxv2i64.i64.nxv2i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], <vscale x 2 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_vvv_u64m2(vuint64m2_t vd, vuint64m2_t vs2, vuint64m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u64m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvv.nxv4i64.i32.nxv4i64.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvv.nxv4i64.i64.nxv4i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], <vscale x 4 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_vvv_u64m4(vuint64m4_t vd, vuint64m4_t vs2, vuint64m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u64m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvv_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvv.nxv8i64.i32.nxv8i64.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvv.nxv8i64.i64.nxv8i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], <vscale x 8 x i64> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_vvv_u64m8(vuint64m8_t vd, vuint64m8_t vs2, vuint64m8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvv_u64m8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv1i8.i8.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv1i8.i8.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8mf8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv2i8.i8.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv2i8.i8.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv4i8.i8.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv4i8.i8.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv8i8.i8.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv8i8.i8.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8m1(vuint8m1_t vd, vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv16i8.i8.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv16i8.i8.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8m2(vuint8m2_t vd, vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv32i8.i8.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv32i8.i8.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8m4(vuint8m4_t vd, vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv64i8.i8.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv64i8.i8.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u8m8(vuint8m8_t vd, vuint8m8_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u8m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv1i16.i16.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv1i16.i16.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u16mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv2i16.i16.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv2i16.i16.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u16mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv4i16.i16.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv4i16.i16.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u16m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv8i16.i16.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv8i16.i16.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u16m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv16i16.i16.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv16i16.i16.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u16m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv32i16.i16.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv32i16.i16.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u16m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv1i32.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv1i32.i32.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u32mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv2i32.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv2i32.i32.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u32m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv4i32.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv4i32.i32.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u32m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv8i32.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv8i32.i32.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u32m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i32.nxv16i32.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvv.se.i64.nxv16i32.i32.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvv_se_u32m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv1i8.i32.i8.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv1i8.i64.i8.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_xvv_se_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8mf8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv2i8.i32.i8.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv2i8.i64.i8.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_xvv_se_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv4i8.i32.i8.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv4i8.i64.i8.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_xvv_se_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv8i8.i32.i8.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv8i8.i64.i8.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_xvv_se_u8m1(vuint8m1_t vd, vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv16i8.i32.i8.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv16i8.i64.i8.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_xvv_se_u8m2(vuint8m2_t vd, vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv32i8.i32.i8.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv32i8.i64.i8.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_xvv_se_u8m4(vuint8m4_t vd, vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv64i8.i32.i8.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xvv.se.nxv64i8.i64.i8.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_xvv_se_u8m8(vuint8m8_t vd, vuint8m8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u8m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv1i16.i32.i16.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv1i16.i64.i16.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_xvv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u16mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv2i16.i32.i16.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv2i16.i64.i16.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_xvv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u16mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv4i16.i32.i16.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv4i16.i64.i16.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_xvv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u16m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv8i16.i32.i16.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv8i16.i64.i16.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_xvv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u16m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv16i16.i32.i16.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv16i16.i64.i16.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_xvv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u16m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv32i16.i32.i16.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvv.se.nxv32i16.i64.i16.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_xvv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u16m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv1i32.i64.i32.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_xvv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u32mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv2i32.i64.i32.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_xvv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u32m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv4i32.i64.i32.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_xvv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u32m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv8i32.i64.i32.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_xvv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u32m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvv.se.nxv16i32.i64.i32.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_xvv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_se_u32m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xvv.nxv1i8.i32.i8.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.xvv.nxv1i8.i64.i8.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_xvv_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8mf8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xvv.nxv2i8.i32.i8.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.xvv.nxv2i8.i64.i8.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_xvv_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xvv.nxv4i8.i32.i8.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.xvv.nxv4i8.i64.i8.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_xvv_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xvv.nxv8i8.i32.i8.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.xvv.nxv8i8.i64.i8.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_xvv_u8m1(vuint8m1_t vd, vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xvv.nxv16i8.i32.i8.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.xvv.nxv16i8.i64.i8.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_xvv_u8m2(vuint8m2_t vd, vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xvv.nxv32i8.i32.i8.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.xvv.nxv32i8.i64.i8.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_xvv_u8m4(vuint8m4_t vd, vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xvv.nxv64i8.i32.i8.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.xvv.nxv64i8.i64.i8.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_xvv_u8m8(vuint8m8_t vd, vuint8m8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u8m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvv.nxv1i16.i32.i16.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvv.nxv1i16.i64.i16.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_xvv_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u16mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvv.nxv2i16.i32.i16.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvv.nxv2i16.i64.i16.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_xvv_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u16mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvv.nxv4i16.i32.i16.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvv.nxv4i16.i64.i16.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_xvv_u16m1(vuint16m1_t vd, vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u16m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvv.nxv8i16.i32.i16.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvv.nxv8i16.i64.i16.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_xvv_u16m2(vuint16m2_t vd, vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u16m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvv.nxv16i16.i32.i16.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvv.nxv16i16.i64.i16.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_xvv_u16m4(vuint16m4_t vd, vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u16m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvv.nxv32i16.i32.i16.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvv.nxv32i16.i64.i16.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_xvv_u16m8(vuint16m8_t vd, vuint16m8_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u16m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvv.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvv.nxv1i32.i64.i32.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_xvv_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u32mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvv.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvv.nxv2i32.i64.i32.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_xvv_u32m1(vuint32m1_t vd, vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u32m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvv.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvv.nxv4i32.i64.i32.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_xvv_u32m2(vuint32m2_t vd, vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u32m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvv.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvv.nxv8i32.i64.i32.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_xvv_u32m4(vuint32m4_t vd, vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u32m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvv.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvv.nxv16i32.i64.i32.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_xvv_u32m8(vuint32m8_t vd, vuint32m8_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvv_u32m8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv1i8.i32.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv1i8.i64.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8mf8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv2i8.i32.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv2i8.i64.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv4i8.i32.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv4i8.i64.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv8i8.i32.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv8i8.i64.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8m1(vuint8m1_t vd, vuint8m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv16i8.i32.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv16i8.i64.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8m2(vuint8m2_t vd, vuint8m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv32i8.i32.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv32i8.i64.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8m4(vuint8m4_t vd, vuint8m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv64i8.i32.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv64i8.i64.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u8m8(vuint8m8_t vd, vuint8m8_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u8m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv1i16.i32.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv1i16.i64.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u16mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv2i16.i32.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv2i16.i64.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u16mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv4i16.i32.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv4i16.i64.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u16m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv8i16.i32.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv8i16.i64.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u16m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv16i16.i32.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv16i16.i64.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u16m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv32i16.i32.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv32i16.i64.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u16m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv1i32.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv1i32.i64.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u32mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv2i32.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv2i32.i64.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u32m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv4i32.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv4i32.i64.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u32m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv8i32.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv8i32.i64.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u32m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv16i32.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv16i32.i64.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u32m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv1i64.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv1i64.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u64m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv2i64.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv2i64.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u64m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv4i64.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv4i64.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u64m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i32.nxv8i64.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivv.se.i64.nxv8i64.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, size_t vl) {
+  __riscv_sf_vc_ivv_se_u64m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv1i8.i32.i32.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv1i8.i64.i64.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_ivv_se_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8mf8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv2i8.i32.i32.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv2i8.i64.i64.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_ivv_se_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv4i8.i32.i32.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv4i8.i64.i64.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_ivv_se_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv8i8.i32.i32.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv8i8.i64.i64.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_ivv_se_u8m1(vuint8m1_t vd, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv16i8.i32.i32.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv16i8.i64.i64.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_ivv_se_u8m2(vuint8m2_t vd, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv32i8.i32.i32.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv32i8.i64.i64.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_ivv_se_u8m4(vuint8m4_t vd, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv64i8.i32.i32.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.ivv.se.nxv64i8.i64.i64.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_ivv_se_u8m8(vuint8m8_t vd, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u8m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv1i16.i32.i32.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv1i16.i64.i64.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_ivv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u16mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv2i16.i32.i32.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv2i16.i64.i64.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_ivv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u16mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv4i16.i32.i32.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv4i16.i64.i64.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_ivv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u16m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv8i16.i32.i32.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv8i16.i64.i64.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_ivv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u16m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv16i16.i32.i32.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv16i16.i64.i64.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_ivv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u16m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv32i16.i32.i32.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivv.se.nxv32i16.i64.i64.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_ivv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u16m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv1i32.i64.i64.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_ivv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u32mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv2i32.i64.i64.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_ivv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u32m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv4i32.i64.i64.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_ivv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u32m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv8i32.i64.i64.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_ivv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u32m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivv.se.nxv16i32.i64.i64.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_ivv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u32m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv1i64.i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_ivv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u64m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv2i64.i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_ivv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u64m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv4i64.i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_ivv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u64m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv8i64.i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivv.se.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_ivv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_se_u64m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.ivv.nxv1i8.i32.i32.i32(i32 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i8> @llvm.riscv.sf.vc.v.ivv.nxv1i8.i64.i64.i64(i64 3, <vscale x 1 x i8> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i8> [[TMP0]]
+//
+vuint8mf8_t test_sf_vc_v_ivv_u8mf8(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8mf8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.ivv.nxv2i8.i32.i32.i32(i32 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i8> @llvm.riscv.sf.vc.v.ivv.nxv2i8.i64.i64.i64(i64 3, <vscale x 2 x i8> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i8> [[TMP0]]
+//
+vuint8mf4_t test_sf_vc_v_ivv_u8mf4(vuint8mf4_t vd, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.ivv.nxv4i8.i32.i32.i32(i32 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i8> @llvm.riscv.sf.vc.v.ivv.nxv4i8.i64.i64.i64(i64 3, <vscale x 4 x i8> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i8> [[TMP0]]
+//
+vuint8mf2_t test_sf_vc_v_ivv_u8mf2(vuint8mf2_t vd, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.ivv.nxv8i8.i32.i32.i32(i32 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.sf.vc.v.ivv.nxv8i8.i64.i64.i64(i64 3, <vscale x 8 x i8> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i8> [[TMP0]]
+//
+vuint8m1_t test_sf_vc_v_ivv_u8m1(vuint8m1_t vd, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.ivv.nxv16i8.i32.i32.i32(i32 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i8> @llvm.riscv.sf.vc.v.ivv.nxv16i8.i64.i64.i64(i64 3, <vscale x 16 x i8> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+vuint8m2_t test_sf_vc_v_ivv_u8m2(vuint8m2_t vd, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.ivv.nxv32i8.i32.i32.i32(i32 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i8> @llvm.riscv.sf.vc.v.ivv.nxv32i8.i64.i64.i64(i64 3, <vscale x 32 x i8> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i8> [[TMP0]]
+//
+vuint8m4_t test_sf_vc_v_ivv_u8m4(vuint8m4_t vd, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u8m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.ivv.nxv64i8.i32.i32.i32(i32 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u8m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 64 x i8> @llvm.riscv.sf.vc.v.ivv.nxv64i8.i64.i64.i64(i64 3, <vscale x 64 x i8> [[VD:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 64 x i8> [[TMP0]]
+//
+vuint8m8_t test_sf_vc_v_ivv_u8m8(vuint8m8_t vd, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u8m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivv.nxv1i16.i32.i32.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivv.nxv1i16.i64.i64.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_ivv_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u16mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivv.nxv2i16.i32.i32.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivv.nxv2i16.i64.i64.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_ivv_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u16mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivv.nxv4i16.i32.i32.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivv.nxv4i16.i64.i64.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_ivv_u16m1(vuint16m1_t vd, vuint16m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u16m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivv.nxv8i16.i32.i32.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivv.nxv8i16.i64.i64.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_ivv_u16m2(vuint16m2_t vd, vuint16m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u16m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivv.nxv16i16.i32.i32.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivv.nxv16i16.i64.i64.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_ivv_u16m4(vuint16m4_t vd, vuint16m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u16m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivv.nxv32i16.i32.i32.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivv.nxv32i16.i64.i64.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_ivv_u16m8(vuint16m8_t vd, vuint16m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u16m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivv.nxv1i32.i32.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivv.nxv1i32.i64.i64.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_ivv_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u32mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivv.nxv2i32.i32.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivv.nxv2i32.i64.i64.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_ivv_u32m1(vuint32m1_t vd, vuint32m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u32m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivv.nxv4i32.i32.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivv.nxv4i32.i64.i64.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_ivv_u32m2(vuint32m2_t vd, vuint32m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u32m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivv.nxv8i32.i32.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivv.nxv8i32.i64.i64.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_ivv_u32m4(vuint32m4_t vd, vuint32m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u32m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivv.nxv16i32.i32.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivv.nxv16i32.i64.i64.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_ivv_u32m8(vuint32m8_t vd, vuint32m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u32m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivv.nxv1i64.i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivv.nxv1i64.i64.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_ivv_u64m1(vuint64m1_t vd, vuint64m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u64m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivv.nxv2i64.i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivv.nxv2i64.i64.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_ivv_u64m2(vuint64m2_t vd, vuint64m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u64m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivv.nxv4i64.i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivv.nxv4i64.i64.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_ivv_u64m4(vuint64m4_t vd, vuint64m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u64m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivv_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivv.nxv8i64.i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivv.nxv8i64.i64.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_ivv_u64m8(vuint64m8_t vd, vuint64m8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivv_u64m8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv1i16.f16.i32(i32 1, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv1i16.f16.i64(i64 1, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u16mf4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv2i16.f16.i32(i32 1, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv2i16.f16.i64(i64 1, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u16mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv4i16.f16.i32(i32 1, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv4i16.f16.i64(i64 1, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u16m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv8i16.f16.i32(i32 1, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv8i16.f16.i64(i64 1, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u16m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv16i16.f16.i32(i32 1, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv16i16.f16.i64(i64 1, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u16m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv32i16.f16.i32(i32 1, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv32i16.f16.i64(i64 1, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u16m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv1i32.f32.i32(i32 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv1i32.f32.i64(i64 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u32mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv2i32.f32.i32(i32 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv2i32.f32.i64(i64 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u32m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv4i32.f32.i32(i32 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv4i32.f32.i64(i64 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u32m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv8i32.f32.i32(i32 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv8i32.f32.i64(i64 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u32m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv16i32.f32.i32(i32 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv16i32.f32.i64(i64 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u32m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv1i64.f64.i32(i32 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv1i64.f64.i64(i64 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u64m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv2i64.f64.i32(i32 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv2i64.f64.i64(i64 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u64m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv4i64.f64.i32(i32 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv4i64.f64.i64(i64 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u64m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i32.nxv8i64.f64.i32(i32 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvv.se.i64.nxv8i64.f64.i64(i64 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, double fs1, size_t vl) {
+  __riscv_sf_vc_fvv_se_u64m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv1i16.i32.f16.i32(i32 1, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv1i16.i64.f16.i64(i64 1, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_fvv_se_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u16mf4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv2i16.i32.f16.i32(i32 1, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv2i16.i64.f16.i64(i64 1, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_fvv_se_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u16mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv4i16.i32.f16.i32(i32 1, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv4i16.i64.f16.i64(i64 1, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_fvv_se_u16m1(vuint16m1_t vd, vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u16m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv8i16.i32.f16.i32(i32 1, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv8i16.i64.f16.i64(i64 1, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_fvv_se_u16m2(vuint16m2_t vd, vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u16m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv16i16.i32.f16.i32(i32 1, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv16i16.i64.f16.i64(i64 1, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_fvv_se_u16m4(vuint16m4_t vd, vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u16m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv32i16.i32.f16.i32(i32 1, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fvv.se.nxv32i16.i64.f16.i64(i64 1, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_fvv_se_u16m8(vuint16m8_t vd, vuint16m8_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u16m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv1i32.i32.f32.i32(i32 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv1i32.i64.f32.i64(i64 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_fvv_se_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u32mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv2i32.i32.f32.i32(i32 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv2i32.i64.f32.i64(i64 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_fvv_se_u32m1(vuint32m1_t vd, vuint32m1_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u32m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv4i32.i32.f32.i32(i32 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv4i32.i64.f32.i64(i64 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_fvv_se_u32m2(vuint32m2_t vd, vuint32m2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u32m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv8i32.i32.f32.i32(i32 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv8i32.i64.f32.i64(i64 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_fvv_se_u32m4(vuint32m4_t vd, vuint32m4_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u32m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv16i32.i32.f32.i32(i32 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvv.se.nxv16i32.i64.f32.i64(i64 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_fvv_se_u32m8(vuint32m8_t vd, vuint32m8_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u32m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv1i64.i32.f64.i32(i32 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv1i64.i64.f64.i64(i64 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_fvv_se_u64m1(vuint64m1_t vd, vuint64m1_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u64m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv2i64.i32.f64.i32(i32 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv2i64.i64.f64.i64(i64 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_fvv_se_u64m2(vuint64m2_t vd, vuint64m2_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u64m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv4i64.i32.f64.i32(i32 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv4i64.i64.f64.i64(i64 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_fvv_se_u64m4(vuint64m4_t vd, vuint64m4_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u64m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_se_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv8i64.i32.f64.i32(i32 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_se_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvv.se.nxv8i64.i64.f64.i64(i64 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_fvv_se_u64m8(vuint64m8_t vd, vuint64m8_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_se_u64m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fvv.nxv1i16.i32.f16.i32(i32 1, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.fvv.nxv1i16.i64.f16.i64(i64 1, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_fvv_u16mf4(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u16mf4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fvv.nxv2i16.i32.f16.i32(i32 1, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.fvv.nxv2i16.i64.f16.i64(i64 1, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_fvv_u16mf2(vuint16mf2_t vd, vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u16mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fvv.nxv4i16.i32.f16.i32(i32 1, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.fvv.nxv4i16.i64.f16.i64(i64 1, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_fvv_u16m1(vuint16m1_t vd, vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u16m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fvv.nxv8i16.i32.f16.i32(i32 1, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.fvv.nxv8i16.i64.f16.i64(i64 1, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_fvv_u16m2(vuint16m2_t vd, vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u16m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fvv.nxv16i16.i32.f16.i32(i32 1, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.fvv.nxv16i16.i64.f16.i64(i64 1, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_fvv_u16m4(vuint16m4_t vd, vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u16m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u16m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fvv.nxv32i16.i32.f16.i32(i32 1, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u16m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.fvv.nxv32i16.i64.f16.i64(i64 1, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_fvv_u16m8(vuint16m8_t vd, vuint16m8_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u16m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvv.nxv1i32.i32.f32.i32(i32 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvv.nxv1i32.i64.f32.i64(i64 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_fvv_u32mf2(vuint32mf2_t vd, vuint32mf2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u32mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvv.nxv2i32.i32.f32.i32(i32 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvv.nxv2i32.i64.f32.i64(i64 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_fvv_u32m1(vuint32m1_t vd, vuint32m1_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u32m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvv.nxv4i32.i32.f32.i32(i32 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvv.nxv4i32.i64.f32.i64(i64 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_fvv_u32m2(vuint32m2_t vd, vuint32m2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u32m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvv.nxv8i32.i32.f32.i32(i32 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvv.nxv8i32.i64.f32.i64(i64 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_fvv_u32m4(vuint32m4_t vd, vuint32m4_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u32m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u32m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvv.nxv16i32.i32.f32.i32(i32 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u32m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvv.nxv16i32.i64.f32.i64(i64 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_fvv_u32m8(vuint32m8_t vd, vuint32m8_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u32m8(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u64m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvv.nxv1i64.i32.f64.i32(i32 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u64m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvv.nxv1i64.i64.f64.i64(i64 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_fvv_u64m1(vuint64m1_t vd, vuint64m1_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u64m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u64m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvv.nxv2i64.i32.f64.i32(i32 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u64m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvv.nxv2i64.i64.f64.i64(i64 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_fvv_u64m2(vuint64m2_t vd, vuint64m2_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u64m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u64m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvv.nxv4i64.i32.f64.i32(i32 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u64m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvv.nxv4i64.i64.f64.i64(i64 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_fvv_u64m4(vuint64m4_t vd, vuint64m4_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u64m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvv_u64m8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvv.nxv8i64.i32.f64.i32(i32 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvv_u64m8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvv.nxv8i64.i64.f64.i64(i64 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i64> [[VS2:%.*]], double [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_fvv_u64m8(vuint64m8_t vd, vuint64m8_t vs2, double fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvv_u64m8(p26, vd, vs2, fs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvw.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvw.c
new file mode 100644
index 0000000000000..23ee2b7bb0f4e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/xsfvcp-xvw.c
@@ -0,0 +1,2278 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV32 %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +zfh -target-feature +xsfvcp -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p26    (0b1)
+#define simm5  (10)
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv1i16.nxv1i8.nxv1i8.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv1i16.nxv1i8.nxv1i8.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u8mf8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv2i16.nxv2i8.nxv2i8.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv2i16.nxv2i8.nxv2i8.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u8mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv4i16.nxv4i8.nxv4i8.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv4i16.nxv4i8.nxv4i8.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u8mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv8i16.nxv8i8.nxv8i8.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv8i16.nxv8i8.nxv8i8.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u8m1(vuint16m2_t vd, vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u8m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv16i16.nxv16i8.nxv16i8.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv16i16.nxv16i8.nxv16i8.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u8m2(vuint16m4_t vd, vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u8m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv32i16.nxv32i8.nxv32i8.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv32i16.nxv32i8.nxv32i8.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u8m4(vuint16m8_t vd, vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u8m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv1i32.nxv1i16.nxv1i16.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv1i32.nxv1i16.nxv1i16.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u16mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv2i32.nxv2i16.nxv2i16.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv2i32.nxv2i16.nxv2i16.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u16mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv4i32.nxv4i16.nxv4i16.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv4i32.nxv4i16.nxv4i16.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u16m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv8i32.nxv8i16.nxv8i16.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv8i32.nxv8i16.nxv8i16.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u16m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv16i32.nxv16i16.nxv16i16.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv16i32.nxv16i16.nxv16i16.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u16m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv1i64.nxv1i32.nxv1i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv1i64.nxv1i32.nxv1i32.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u32mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv2i64.nxv2i32.nxv2i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv2i64.nxv2i32.nxv2i32.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u32m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv4i64.nxv4i32.nxv4i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv4i64.nxv4i32.nxv4i32.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u32m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_vvw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i32.nxv8i64.nxv8i32.nxv8i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_vvw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.vvw.se.i64.nxv8i64.nxv8i32.nxv8i32.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_vvw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  __riscv_sf_vc_vvw_se_u32m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv1i16.i32.nxv1i8.nxv1i8.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv1i16.i64.nxv1i8.nxv1i8.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_vvw_se_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u8mf8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv2i16.i32.nxv2i8.nxv2i8.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv2i16.i64.nxv2i8.nxv2i8.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_vvw_se_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u8mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv4i16.i32.nxv4i8.nxv4i8.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv4i16.i64.nxv4i8.nxv4i8.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_vvw_se_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u8mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv8i16.i32.nxv8i8.nxv8i8.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv8i16.i64.nxv8i8.nxv8i8.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_vvw_se_u8m1(vuint16m2_t vd, vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u8m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv16i16.i32.nxv16i8.nxv16i8.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv16i16.i64.nxv16i8.nxv16i8.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_vvw_se_u8m2(vuint16m4_t vd, vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u8m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv32i16.i32.nxv32i8.nxv32i8.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvw.se.nxv32i16.i64.nxv32i8.nxv32i8.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_vvw_se_u8m4(vuint16m8_t vd, vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u8m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv1i32.i32.nxv1i16.nxv1i16.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv1i32.i64.nxv1i16.nxv1i16.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_vvw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u16mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv2i32.i32.nxv2i16.nxv2i16.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv2i32.i64.nxv2i16.nxv2i16.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_vvw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u16mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv4i32.i32.nxv4i16.nxv4i16.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv4i32.i64.nxv4i16.nxv4i16.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_vvw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u16m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv8i32.i32.nxv8i16.nxv8i16.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv8i32.i64.nxv8i16.nxv8i16.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_vvw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u16m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv16i32.i32.nxv16i16.nxv16i16.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvw.se.nxv16i32.i64.nxv16i16.nxv16i16.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_vvw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u16m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv1i64.i32.nxv1i32.nxv1i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv1i64.i64.nxv1i32.nxv1i32.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_vvw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u32mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv2i64.i32.nxv2i32.nxv2i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv2i64.i64.nxv2i32.nxv2i32.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_vvw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u32m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv4i64.i32.nxv4i32.nxv4i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv4i64.i64.nxv4i32.nxv4i32.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_vvw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u32m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv8i64.i32.nxv8i32.nxv8i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvw.se.nxv8i64.i64.nxv8i32.nxv8i32.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_vvw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_se_u32m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvw.nxv1i16.i32.nxv1i8.nxv1i8.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.vvw.nxv1i16.i64.nxv1i8.nxv1i8.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], <vscale x 1 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_vvw_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u8mf8(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvw.nxv2i16.i32.nxv2i8.nxv2i8.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.vvw.nxv2i16.i64.nxv2i8.nxv2i8.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], <vscale x 2 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_vvw_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, vuint8mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u8mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvw.nxv4i16.i32.nxv4i8.nxv4i8.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.vvw.nxv4i16.i64.nxv4i8.nxv4i8.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], <vscale x 4 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_vvw_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, vuint8mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u8mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvw.nxv8i16.i32.nxv8i8.nxv8i8.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.vvw.nxv8i16.i64.nxv8i8.nxv8i8.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], <vscale x 8 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_vvw_u8m1(vuint16m2_t vd, vuint8m1_t vs2, vuint8m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u8m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvw.nxv16i16.i32.nxv16i8.nxv16i8.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.vvw.nxv16i16.i64.nxv16i8.nxv16i8.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], <vscale x 16 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_vvw_u8m2(vuint16m4_t vd, vuint8m2_t vs2, vuint8m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u8m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvw.nxv32i16.i32.nxv32i8.nxv32i8.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.vvw.nxv32i16.i64.nxv32i8.nxv32i8.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], <vscale x 32 x i8> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_vvw_u8m4(vuint16m8_t vd, vuint8m4_t vs2, vuint8m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u8m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvw.nxv1i32.i32.nxv1i16.nxv1i16.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.vvw.nxv1i32.i64.nxv1i16.nxv1i16.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], <vscale x 1 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_vvw_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, vuint16mf4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u16mf4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvw.nxv2i32.i32.nxv2i16.nxv2i16.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.vvw.nxv2i32.i64.nxv2i16.nxv2i16.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], <vscale x 2 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_vvw_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, vuint16mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u16mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvw.nxv4i32.i32.nxv4i16.nxv4i16.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.vvw.nxv4i32.i64.nxv4i16.nxv4i16.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], <vscale x 4 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_vvw_u16m1(vuint32m2_t vd, vuint16m1_t vs2, vuint16m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u16m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvw.nxv8i32.i32.nxv8i16.nxv8i16.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.vvw.nxv8i32.i64.nxv8i16.nxv8i16.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], <vscale x 8 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_vvw_u16m2(vuint32m4_t vd, vuint16m2_t vs2, vuint16m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u16m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvw.nxv16i32.i32.nxv16i16.nxv16i16.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.vvw.nxv16i32.i64.nxv16i16.nxv16i16.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], <vscale x 16 x i16> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_vvw_u16m4(vuint32m8_t vd, vuint16m4_t vs2, vuint16m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u16m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvw.nxv1i64.i32.nxv1i32.nxv1i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.vvw.nxv1i64.i64.nxv1i32.nxv1i32.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], <vscale x 1 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_vvw_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, vuint32mf2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u32mf2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvw.nxv2i64.i32.nxv2i32.nxv2i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.vvw.nxv2i64.i64.nxv2i32.nxv2i32.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], <vscale x 2 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_vvw_u32m1(vuint64m2_t vd, vuint32m1_t vs2, vuint32m1_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u32m1(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvw.nxv4i64.i32.nxv4i32.nxv4i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.vvw.nxv4i64.i64.nxv4i32.nxv4i32.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], <vscale x 4 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_vvw_u32m2(vuint64m4_t vd, vuint32m2_t vs2, vuint32m2_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u32m2(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_vvw_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvw.nxv8i64.i32.nxv8i32.nxv8i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_vvw_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.vvw.nxv8i64.i64.nxv8i32.nxv8i32.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], <vscale x 8 x i32> [[VS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_vvw_u32m4(vuint64m8_t vd, vuint32m4_t vs2, vuint32m4_t vs1, size_t vl) {
+  return __riscv_sf_vc_v_vvw_u32m4(p27_26, vd, vs2, vs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv1i16.nxv1i8.i8.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv1i16.nxv1i8.i8.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u8mf8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv2i16.nxv2i8.i8.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv2i16.nxv2i8.i8.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u8mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv4i16.nxv4i8.i8.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv4i16.nxv4i8.i8.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u8mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv8i16.nxv8i8.i8.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv8i16.nxv8i8.i8.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u8m1(vuint16m2_t vd, vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u8m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv16i16.nxv16i8.i8.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv16i16.nxv16i8.i8.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u8m2(vuint16m4_t vd, vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u8m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv32i16.nxv32i8.i8.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv32i16.nxv32i8.i8.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u8m4(vuint16m8_t vd, vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u8m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv1i32.nxv1i16.i16.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv1i32.nxv1i16.i16.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u16mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv2i32.nxv2i16.i16.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv2i32.nxv2i16.i16.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u16mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv4i32.nxv4i16.i16.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv4i32.nxv4i16.i16.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u16m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv8i32.nxv8i16.i16.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv8i32.nxv8i16.i16.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u16m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv16i32.nxv16i16.i16.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv16i32.nxv16i16.i16.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u16m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv1i64.nxv1i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv1i64.nxv1i32.i32.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u32mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv2i64.nxv2i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv2i64.nxv2i32.i32.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u32m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv4i64.nxv4i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv4i64.nxv4i32.i32.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u32m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_xvw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i32.nxv8i64.nxv8i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_xvw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.xvw.se.i64.nxv8i64.nxv8i32.i32.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_xvw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  __riscv_sf_vc_xvw_se_u32m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv1i16.i32.nxv1i8.i8.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv1i16.i64.nxv1i8.i8.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_xvw_se_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u8mf8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv2i16.i32.nxv2i8.i8.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv2i16.i64.nxv2i8.i8.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_xvw_se_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u8mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv4i16.i32.nxv4i8.i8.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv4i16.i64.nxv4i8.i8.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_xvw_se_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u8mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv8i16.i32.nxv8i8.i8.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv8i16.i64.nxv8i8.i8.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_xvw_se_u8m1(vuint16m2_t vd, vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u8m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv16i16.i32.nxv16i8.i8.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv16i16.i64.nxv16i8.i8.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_xvw_se_u8m2(vuint16m4_t vd, vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u8m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv32i16.i32.nxv32i8.i8.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvw.se.nxv32i16.i64.nxv32i8.i8.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_xvw_se_u8m4(vuint16m8_t vd, vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u8m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv1i32.i32.nxv1i16.i16.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv1i32.i64.nxv1i16.i16.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_xvw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u16mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv2i32.i32.nxv2i16.i16.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv2i32.i64.nxv2i16.i16.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_xvw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u16mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv4i32.i32.nxv4i16.i16.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv4i32.i64.nxv4i16.i16.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_xvw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u16m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv8i32.i32.nxv8i16.i16.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv8i32.i64.nxv8i16.i16.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_xvw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u16m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv16i32.i32.nxv16i16.i16.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvw.se.nxv16i32.i64.nxv16i16.i16.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_xvw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u16m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv1i64.i32.nxv1i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv1i64.i64.nxv1i32.i32.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_xvw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u32mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv2i64.i32.nxv2i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv2i64.i64.nxv2i32.i32.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_xvw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u32m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv4i64.i32.nxv4i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv4i64.i64.nxv4i32.i32.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_xvw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u32m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv8i64.i32.nxv8i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xvw.se.nxv8i64.i64.nxv8i32.i32.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_xvw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_se_u32m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvw.nxv1i16.i32.nxv1i8.i8.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.xvw.nxv1i16.i64.nxv1i8.i8.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_xvw_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u8mf8(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvw.nxv2i16.i32.nxv2i8.i8.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.xvw.nxv2i16.i64.nxv2i8.i8.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_xvw_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u8mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvw.nxv4i16.i32.nxv4i8.i8.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.xvw.nxv4i16.i64.nxv4i8.i8.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_xvw_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u8mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvw.nxv8i16.i32.nxv8i8.i8.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.xvw.nxv8i16.i64.nxv8i8.i8.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_xvw_u8m1(vuint16m2_t vd, vuint8m1_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u8m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvw.nxv16i16.i32.nxv16i8.i8.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.xvw.nxv16i16.i64.nxv16i8.i8.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_xvw_u8m2(vuint16m4_t vd, vuint8m2_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u8m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvw.nxv32i16.i32.nxv32i8.i8.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.xvw.nxv32i16.i64.nxv32i8.i8.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i8 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_xvw_u8m4(vuint16m8_t vd, vuint8m4_t vs2, uint8_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u8m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvw.nxv1i32.i32.nxv1i16.i16.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.xvw.nxv1i32.i64.nxv1i16.i16.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_xvw_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u16mf4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvw.nxv2i32.i32.nxv2i16.i16.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.xvw.nxv2i32.i64.nxv2i16.i16.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_xvw_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u16mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvw.nxv4i32.i32.nxv4i16.i16.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.xvw.nxv4i32.i64.nxv4i16.i16.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_xvw_u16m1(vuint32m2_t vd, vuint16m1_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u16m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvw.nxv8i32.i32.nxv8i16.i16.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.xvw.nxv8i32.i64.nxv8i16.i16.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_xvw_u16m2(vuint32m4_t vd, vuint16m2_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u16m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvw.nxv16i32.i32.nxv16i16.i16.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.xvw.nxv16i32.i64.nxv16i16.i16.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i16 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_xvw_u16m4(vuint32m8_t vd, vuint16m4_t vs2, uint16_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u16m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xvw.nxv1i64.i32.nxv1i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.xvw.nxv1i64.i64.nxv1i32.i32.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_xvw_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u32mf2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xvw.nxv2i64.i32.nxv2i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.xvw.nxv2i64.i64.nxv2i32.i32.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_xvw_u32m1(vuint64m2_t vd, vuint32m1_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u32m1(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xvw.nxv4i64.i32.nxv4i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.xvw.nxv4i64.i64.nxv4i32.i32.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_xvw_u32m2(vuint64m4_t vd, vuint32m2_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u32m2(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_xvw_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xvw.nxv8i64.i32.nxv8i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_xvw_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.xvw.nxv8i64.i64.nxv8i32.i32.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 [[RS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_xvw_u32m4(vuint64m8_t vd, vuint32m4_t vs2, uint32_t rs1, size_t vl) {
+  return __riscv_sf_vc_v_xvw_u32m4(p27_26, vd, vs2, rs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv1i16.nxv1i8.i32.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv1i16.nxv1i8.i64.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u8mf8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv2i16.nxv2i8.i32.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv2i16.nxv2i8.i64.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u8mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv4i16.nxv4i8.i32.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv4i16.nxv4i8.i64.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u8mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv8i16.nxv8i8.i32.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv8i16.nxv8i8.i64.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u8m1(vuint16m2_t vd, vuint8m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u8m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv16i16.nxv16i8.i32.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv16i16.nxv16i8.i64.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u8m2(vuint16m4_t vd, vuint8m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u8m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv32i16.nxv32i8.i32.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv32i16.nxv32i8.i64.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u8m4(vuint16m8_t vd, vuint8m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u8m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv1i32.nxv1i16.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv1i32.nxv1i16.i64.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u16mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv2i32.nxv2i16.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv2i32.nxv2i16.i64.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u16mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv4i32.nxv4i16.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv4i32.nxv4i16.i64.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u16m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv8i32.nxv8i16.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv8i32.nxv8i16.i64.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u16m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv16i32.nxv16i16.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv16i32.nxv16i16.i64.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u16m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv1i64.nxv1i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv1i64.nxv1i32.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u32mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv2i64.nxv2i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv2i64.nxv2i32.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u32m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv4i64.nxv4i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv4i64.nxv4i32.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u32m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_ivw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i32.nxv8i64.nxv8i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_ivw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.ivw.se.i64.nxv8i64.nxv8i32.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_ivw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, size_t vl) {
+  __riscv_sf_vc_ivw_se_u32m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv1i16.i32.nxv1i8.i32.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv1i16.i64.nxv1i8.i64.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_ivw_se_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u8mf8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv2i16.i32.nxv2i8.i32.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv2i16.i64.nxv2i8.i64.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_ivw_se_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u8mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv4i16.i32.nxv4i8.i32.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv4i16.i64.nxv4i8.i64.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_ivw_se_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u8mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv8i16.i32.nxv8i8.i32.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv8i16.i64.nxv8i8.i64.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_ivw_se_u8m1(vuint16m2_t vd, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u8m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv16i16.i32.nxv16i8.i32.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv16i16.i64.nxv16i8.i64.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_ivw_se_u8m2(vuint16m4_t vd, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u8m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv32i16.i32.nxv32i8.i32.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivw.se.nxv32i16.i64.nxv32i8.i64.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_ivw_se_u8m4(vuint16m8_t vd, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u8m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv1i32.i32.nxv1i16.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv1i32.i64.nxv1i16.i64.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_ivw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u16mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv2i32.i32.nxv2i16.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv2i32.i64.nxv2i16.i64.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_ivw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u16mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv4i32.i32.nxv4i16.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv4i32.i64.nxv4i16.i64.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_ivw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u16m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv8i32.i32.nxv8i16.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv8i32.i64.nxv8i16.i64.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_ivw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u16m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv16i32.i32.nxv16i16.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivw.se.nxv16i32.i64.nxv16i16.i64.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_ivw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u16m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv1i64.i32.nxv1i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv1i64.i64.nxv1i32.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_ivw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u32mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv2i64.i32.nxv2i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv2i64.i64.nxv2i32.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_ivw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u32m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv4i64.i32.nxv4i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv4i64.i64.nxv4i32.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_ivw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u32m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv8i64.i32.nxv8i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivw.se.nxv8i64.i64.nxv8i32.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_ivw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_se_u32m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u8mf8(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivw.nxv1i16.i32.nxv1i8.i32.i32(i32 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u8mf8(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i16> @llvm.riscv.sf.vc.v.ivw.nxv1i16.i64.nxv1i8.i64.i64(i64 3, <vscale x 1 x i16> [[VD:%.*]], <vscale x 1 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i16> [[TMP0]]
+//
+vuint16mf4_t test_sf_vc_v_ivw_u8mf8(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u8mf8(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u8mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivw.nxv2i16.i32.nxv2i8.i32.i32(i32 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u8mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i16> @llvm.riscv.sf.vc.v.ivw.nxv2i16.i64.nxv2i8.i64.i64(i64 3, <vscale x 2 x i16> [[VD:%.*]], <vscale x 2 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i16> [[TMP0]]
+//
+vuint16mf2_t test_sf_vc_v_ivw_u8mf4(vuint16mf2_t vd, vuint8mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u8mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u8mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivw.nxv4i16.i32.nxv4i8.i32.i32(i32 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u8mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i16> @llvm.riscv.sf.vc.v.ivw.nxv4i16.i64.nxv4i8.i64.i64(i64 3, <vscale x 4 x i16> [[VD:%.*]], <vscale x 4 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i16> [[TMP0]]
+//
+vuint16m1_t test_sf_vc_v_ivw_u8mf2(vuint16m1_t vd, vuint8mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u8mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u8m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivw.nxv8i16.i32.nxv8i8.i32.i32(i32 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u8m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i16> @llvm.riscv.sf.vc.v.ivw.nxv8i16.i64.nxv8i8.i64.i64(i64 3, <vscale x 8 x i16> [[VD:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i16> [[TMP0]]
+//
+vuint16m2_t test_sf_vc_v_ivw_u8m1(vuint16m2_t vd, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u8m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u8m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivw.nxv16i16.i32.nxv16i8.i32.i32(i32 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u8m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i16> @llvm.riscv.sf.vc.v.ivw.nxv16i16.i64.nxv16i8.i64.i64(i64 3, <vscale x 16 x i16> [[VD:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i16> [[TMP0]]
+//
+vuint16m4_t test_sf_vc_v_ivw_u8m2(vuint16m4_t vd, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u8m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u8m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivw.nxv32i16.i32.nxv32i8.i32.i32(i32 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u8m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 32 x i16> @llvm.riscv.sf.vc.v.ivw.nxv32i16.i64.nxv32i8.i64.i64(i64 3, <vscale x 32 x i16> [[VD:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 32 x i16> [[TMP0]]
+//
+vuint16m8_t test_sf_vc_v_ivw_u8m4(vuint16m8_t vd, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u8m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivw.nxv1i32.i32.nxv1i16.i32.i32(i32 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.ivw.nxv1i32.i64.nxv1i16.i64.i64(i64 3, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_ivw_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u16mf4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivw.nxv2i32.i32.nxv2i16.i32.i32(i32 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.ivw.nxv2i32.i64.nxv2i16.i64.i64(i64 3, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_ivw_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u16mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivw.nxv4i32.i32.nxv4i16.i32.i32(i32 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.ivw.nxv4i32.i64.nxv4i16.i64.i64(i64 3, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_ivw_u16m1(vuint32m2_t vd, vuint16m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u16m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivw.nxv8i32.i32.nxv8i16.i32.i32(i32 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.ivw.nxv8i32.i64.nxv8i16.i64.i64(i64 3, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_ivw_u16m2(vuint32m4_t vd, vuint16m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u16m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivw.nxv16i32.i32.nxv16i16.i32.i32(i32 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.ivw.nxv16i32.i64.nxv16i16.i64.i64(i64 3, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_ivw_u16m4(vuint32m8_t vd, vuint16m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u16m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivw.nxv1i64.i32.nxv1i32.i32.i32(i32 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.ivw.nxv1i64.i64.nxv1i32.i64.i64(i64 3, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_ivw_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u32mf2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivw.nxv2i64.i32.nxv2i32.i32.i32(i32 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.ivw.nxv2i64.i64.nxv2i32.i64.i64(i64 3, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_ivw_u32m1(vuint64m2_t vd, vuint32m1_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u32m1(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivw.nxv4i64.i32.nxv4i32.i32.i32(i32 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.ivw.nxv4i64.i64.nxv4i32.i64.i64(i64 3, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_ivw_u32m2(vuint64m4_t vd, vuint32m2_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u32m2(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_ivw_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivw.nxv8i64.i32.nxv8i32.i32.i32(i32 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i32 10, i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_ivw_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.ivw.nxv8i64.i64.nxv8i32.i64.i64(i64 3, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], i64 10, i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_ivw_u32m4(vuint64m8_t vd, vuint32m4_t vs2, size_t vl) {
+  return __riscv_sf_vc_v_ivw_u32m4(p27_26, vd, vs2, simm5, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv1i32.nxv1i16.f16.i32(i32 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv1i32.nxv1i16.f16.i64(i64 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u16mf4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv2i32.nxv2i16.f16.i32(i32 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv2i32.nxv2i16.f16.i64(i64 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u16mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv4i32.nxv4i16.f16.i32(i32 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv4i32.nxv4i16.f16.i64(i64 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u16m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv8i32.nxv8i16.f16.i32(i32 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv8i32.nxv8i16.f16.i64(i64 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u16m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv16i32.nxv16i16.f16.i32(i32 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv16i32.nxv16i16.f16.i64(i64 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u16m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv1i64.nxv1i32.f32.i32(i32 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv1i64.nxv1i32.f32.i64(i64 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u32mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv2i64.nxv2i32.f32.i32(i32 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv2i64.nxv2i32.f32.i64(i64 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u32m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv4i64.nxv4i32.f32.i32(i32 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv4i64.nxv4i32.f32.i64(i64 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u32m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_fvw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i32.nxv8i64.nxv8i32.f32.i32(i32 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret void
+//
+// CHECK-RV64-LABEL: @test_sf_vc_fvw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    call void @llvm.riscv.sf.vc.fvw.se.i64.nxv8i64.nxv8i32.f32.i64(i64 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret void
+//
+void test_sf_vc_fvw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, float fs1, size_t vl) {
+  __riscv_sf_vc_fvw_se_u32m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv1i32.i32.nxv1i16.f16.i32(i32 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv1i32.i64.nxv1i16.f16.i64(i64 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_fvw_se_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u16mf4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv2i32.i32.nxv2i16.f16.i32(i32 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv2i32.i64.nxv2i16.f16.i64(i64 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_fvw_se_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u16mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv4i32.i32.nxv4i16.f16.i32(i32 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv4i32.i64.nxv4i16.f16.i64(i64 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_fvw_se_u16m1(vuint32m2_t vd, vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u16m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv8i32.i32.nxv8i16.f16.i32(i32 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv8i32.i64.nxv8i16.f16.i64(i64 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_fvw_se_u16m2(vuint32m4_t vd, vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u16m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv16i32.i32.nxv16i16.f16.i32(i32 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvw.se.nxv16i32.i64.nxv16i16.f16.i64(i64 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_fvw_se_u16m4(vuint32m8_t vd, vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u16m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv1i64.i32.nxv1i32.f32.i32(i32 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv1i64.i64.nxv1i32.f32.i64(i64 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_fvw_se_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u32mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv2i64.i32.nxv2i32.f32.i32(i32 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv2i64.i64.nxv2i32.f32.i64(i64 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_fvw_se_u32m1(vuint64m2_t vd, vuint32m1_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u32m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv4i64.i32.nxv4i32.f32.i32(i32 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv4i64.i64.nxv4i32.f32.i64(i64 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_fvw_se_u32m2(vuint64m4_t vd, vuint32m2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u32m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_se_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv8i64.i32.nxv8i32.f32.i32(i32 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_se_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvw.se.nxv8i64.i64.nxv8i32.f32.i64(i64 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_fvw_se_u32m4(vuint64m8_t vd, vuint32m4_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_se_u32m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u16mf4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvw.nxv1i32.i32.nxv1i16.f16.i32(i32 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u16mf4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i32> @llvm.riscv.sf.vc.v.fvw.nxv1i32.i64.nxv1i16.f16.i64(i64 1, <vscale x 1 x i32> [[VD:%.*]], <vscale x 1 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i32> [[TMP0]]
+//
+vuint32mf2_t test_sf_vc_v_fvw_u16mf4(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u16mf4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u16mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvw.nxv2i32.i32.nxv2i16.f16.i32(i32 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u16mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vc.v.fvw.nxv2i32.i64.nxv2i16.f16.i64(i64 1, <vscale x 2 x i32> [[VD:%.*]], <vscale x 2 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vuint32m1_t test_sf_vc_v_fvw_u16mf2(vuint32m1_t vd, vuint16mf2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u16mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u16m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvw.nxv4i32.i32.nxv4i16.f16.i32(i32 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u16m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vc.v.fvw.nxv4i32.i64.nxv4i16.f16.i64(i64 1, <vscale x 4 x i32> [[VD:%.*]], <vscale x 4 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vuint32m2_t test_sf_vc_v_fvw_u16m1(vuint32m2_t vd, vuint16m1_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u16m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u16m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvw.nxv8i32.i32.nxv8i16.f16.i32(i32 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u16m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vc.v.fvw.nxv8i32.i64.nxv8i16.f16.i64(i64 1, <vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vuint32m4_t test_sf_vc_v_fvw_u16m2(vuint32m4_t vd, vuint16m2_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u16m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u16m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvw.nxv16i32.i32.nxv16i16.f16.i32(i32 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u16m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vc.v.fvw.nxv16i32.i64.nxv16i16.f16.i64(i64 1, <vscale x 16 x i32> [[VD:%.*]], <vscale x 16 x i16> [[VS2:%.*]], half [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vuint32m8_t test_sf_vc_v_fvw_u16m4(vuint32m8_t vd, vuint16m4_t vs2, _Float16 fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u16m4(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u32mf2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvw.nxv1i64.i32.nxv1i32.f32.i32(i32 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u32mf2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x i64> @llvm.riscv.sf.vc.v.fvw.nxv1i64.i64.nxv1i32.f32.i64(i64 1, <vscale x 1 x i64> [[VD:%.*]], <vscale x 1 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vuint64m1_t test_sf_vc_v_fvw_u32mf2(vuint64m1_t vd, vuint32mf2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u32mf2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u32m1(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvw.nxv2i64.i32.nxv2i32.f32.i32(i32 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u32m1(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.riscv.sf.vc.v.fvw.nxv2i64.i64.nxv2i32.f32.i64(i64 1, <vscale x 2 x i64> [[VD:%.*]], <vscale x 2 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i64> [[TMP0]]
+//
+vuint64m2_t test_sf_vc_v_fvw_u32m1(vuint64m2_t vd, vuint32m1_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u32m1(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u32m2(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvw.nxv4i64.i32.nxv4i32.f32.i32(i32 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u32m2(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i64> @llvm.riscv.sf.vc.v.fvw.nxv4i64.i64.nxv4i32.f32.i64(i64 1, <vscale x 4 x i64> [[VD:%.*]], <vscale x 4 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i64> [[TMP0]]
+//
+vuint64m4_t test_sf_vc_v_fvw_u32m2(vuint64m4_t vd, vuint32m2_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u32m2(p26, vd, vs2, fs1, vl);
+}
+
+// CHECK-RV32-LABEL: @test_sf_vc_v_fvw_u32m4(
+// CHECK-RV32-NEXT:  entry:
+// CHECK-RV32-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvw.nxv8i64.i32.nxv8i32.f32.i32(i32 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i32 [[VL:%.*]])
+// CHECK-RV32-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+// CHECK-RV64-LABEL: @test_sf_vc_v_fvw_u32m4(
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i64> @llvm.riscv.sf.vc.v.fvw.nxv8i64.i64.nxv8i32.f32.i64(i64 1, <vscale x 8 x i64> [[VD:%.*]], <vscale x 8 x i32> [[VS2:%.*]], float [[FS1:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i64> [[TMP0]]
+//
+vuint64m8_t test_sf_vc_v_fvw_u32m4(vuint64m8_t vd, vuint32m4_t vs2, float fs1, size_t vl) {
+  return __riscv_sf_vc_v_fvw_u32m4(p26, vd, vs2, fs1, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/xsfvcp-index-out-of-range.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/xsfvcp-index-out-of-range.c
new file mode 100644
index 0000000000000..259364f5e987a
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/xsfvcp-index-out-of-range.c
@@ -0,0 +1,764 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +f -target-feature +d \
+// RUN:   -target-feature +v -target-feature +zfh -target-feature +experimental-zvfh \
+// RUN:   -target-feature +xsfvcp \
+// RUN:   -fsyntax-only -verify %s
+
+#include <sifive_vector.h>
+
+#define p27_26 (0b11)
+#define p26    (0b1)
+#define p24_20 (0b11111)
+#define p11_7  (0b11111)
+#define simm5  (15)
+
+#define p27_26_overflow (0b100)
+#define p26_overflow    (0b10)
+#define p24_20_overflow (0b100000)
+#define p11_7_overflow  (0b100001)
+#define simm5_overflow  (16)
+
+// sf_vc_x_se
+
+void test_sf_vc_x_se_u8mf8_p27_26_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_x_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_x_se_u8mf8(index, p24_20, p11_7, rs1, vl);
+}
+
+void test_sf_vc_x_se_u8mf8_24_20_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_x_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_x_se_u8mf8(p27_26, index, p11_7, rs1, vl);
+}
+
+void test_sf_vc_x_se_u8mf8_11_7_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_x_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_x_se_u8mf8(p27_26, p24_20, index, rs1, vl);
+}
+
+void test_sf_vc_x_se_u8mf8_p27_26_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_x_se_u8mf8(p27_26_overflow, p24_20, p11_7, rs1, vl);
+}
+
+void test_sf_vc_x_se_u8mf8_p24_20_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 32 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_x_se_u8mf8(p27_26, p24_20_overflow, p11_7, rs1, vl);
+}
+
+void test_sf_vc_x_se_u8mf8_p11_7_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 33 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_x_se_u8mf8(p27_26, p24_20, p11_7_overflow, rs1, vl);
+}
+
+// sf_vc_i_se
+
+void test_sf_vc_i_se_u8mf8_p27_26_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_i_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_i_se_u8mf8(index, p24_20, p11_7, simm5, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_24_20_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_i_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26, index, p11_7, simm5, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_11_7_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_i_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26, p24_20, index, simm5, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_simm5_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_i_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26, p24_20, p11_7, index, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_p27_26_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26_overflow, p24_20, p11_7, simm5, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_p24_20_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 32 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26, p24_20_overflow, p11_7, simm5, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_p11_7_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 33 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26, p24_20, p11_7_overflow, simm5, vl);
+}
+
+void test_sf_vc_i_se_u8mf8_simm5_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  __riscv_sf_vc_i_se_u8mf8(p27_26, p24_20, p11_7, simm5_overflow, vl);
+}
+
+// sf_vc_v_x_se
+
+vuint8mf8_t test_sf_vc_v_x_se_u8mf8_p27_26_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_x_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_x_se_u8mf8(index, p24_20, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_x_se_u8mf8_p24_20_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_x_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_x_se_u8mf8(p27_26, index, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_x_se_u8mf8_p27_26_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_x_se_u8mf8(p27_26_overflow, p24_20, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_x_se_u8mf8_p24_20_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 32 is outside the valid range [0, 31]}}
+  return __riscv_sf_vc_v_x_se_u8mf8(p27_26, p24_20_overflow, rs1, vl);
+}
+
+// sf_vc_v_x
+
+vuint8mf8_t test_sf_vc_v_x_u8mf8_p27_26_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_x_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_x_u8mf8(index, p24_20, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_x_u8mf8_p24_20_not_constant(uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_x_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_x_u8mf8(p27_26, index, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_x_u8mf8_p27_26_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_x_u8mf8(p27_26_overflow, p24_20, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_x_u8mf8_p24_20_overflow(uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 32 is outside the valid range [0, 31]}}
+  return __riscv_sf_vc_v_x_u8mf8(p27_26, p24_20_overflow, rs1, vl);
+}
+
+// sf_vc_v_i_se
+
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8_p27_26_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_i_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_i_se_u8mf8(index, p24_20, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8_p24_20_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_i_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_i_se_u8mf8(p27_26, index, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8_simm5_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_i_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_i_se_u8mf8(p27_26, p24_20, index, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8_p27_26_overflow(size_t vl, int index) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_i_se_u8mf8(p27_26_overflow, p24_20, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8_p24_20_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 32 is outside the valid range [0, 31]}}
+  return __riscv_sf_vc_v_i_se_u8mf8(p27_26, p24_20_overflow, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_se_u8mf8_simm5_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_i_se_u8mf8(p27_26, p24_20, simm5_overflow, vl);
+}
+
+// sf_vc_v_i
+
+vuint8mf8_t test_sf_vc_v_i_u8mf8_p27_26_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_i_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_i_u8mf8(index, p24_20, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_u8mf8_p24_20_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_i_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_i_u8mf8(p27_26, index, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_u8mf8_simm5_not_constant(size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_i_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_i_u8mf8(p27_26, p24_20, index, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_u8mf8_p27_26_overflow(size_t vl, int index) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_i_u8mf8(p27_26_overflow, p24_20, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_u8mf8_p24_20_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 32 is outside the valid range [0, 31]}}
+  return __riscv_sf_vc_v_i_u8mf8(p27_26, p24_20_overflow, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_i_u8mf8_simm5_overflow(size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_i_u8mf8(p27_26, p24_20, simm5_overflow, vl);
+}
+
+// sf_vc_vv_se
+
+void test_sf_vc_vv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_vv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_vv_se_u8mf8(index, p11_7, vs2, vs1, vl);
+}
+
+void test_sf_vc_vv_se_u8mf8_p11_7_not_constant(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_vv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_vv_se_u8mf8(p27_26, index, vs2, vs1, vl);
+}
+
+void test_sf_vc_vv_se_u8mf8_p27_26_overflow(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_vv_se_u8mf8(p27_26_overflow, p11_7, vs2, vs1, vl);
+}
+
+void test_sf_vc_vv_se_u8mf8_p11_7_overflow(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 33 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_vv_se_u8mf8(p27_26, p11_7_overflow, vs2, vs1, vl);
+}
+
+// sf_vc_xv_se
+
+void test_sf_vc_xv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_xv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_xv_se_u8mf8(index, p11_7, vs2, rs1, vl);
+}
+
+void test_sf_vc_xv_se_u8mf8_p11_7_not_constant(vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_xv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_xv_se_u8mf8(p27_26, index, vs2, rs1, vl);
+}
+
+void test_sf_vc_xv_se_u8mf8_p27_26_overflow(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_xv_se_u8mf8(p27_26_overflow, p11_7, vs2, rs1, vl);
+}
+
+void test_sf_vc_xv_se_u8mf8_p11_7_overflow(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 33 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_xv_se_u8mf8(p27_26, p11_7_overflow, vs2, rs1, vl);
+}
+
+// sf_vc_iv_se
+
+void test_sf_vc_iv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_iv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_iv_se_u8mf8(index, p11_7, vs2, simm5, vl);
+}
+
+void test_sf_vc_iv_se_u8mf8_p11_7_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_iv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_iv_se_u8mf8(p27_26, index, vs2, simm5, vl);
+}
+
+void test_sf_vc_iv_se_u8mf8_simm5_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_iv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_iv_se_u8mf8(p27_26, p11_7, vs2, index, vl);
+}
+
+void test_sf_vc_iv_se_u8mf8_p27_26_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_iv_se_u8mf8(p27_26_overflow, p11_7, vs2, simm5, vl);
+}
+
+void test_sf_vc_iv_se_u8mf8_p11_7_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 33 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_iv_se_u8mf8(p27_26, p11_7_overflow, vs2, simm5, vl);
+}
+
+void test_sf_vc_iv_se_u8mf8_simm5_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  __riscv_sf_vc_iv_se_u8mf8(p27_26, p11_7, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_fv_se
+
+void test_sf_vc_fv_se_u16mf4_p26_not_constant(vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_fv_se_u16mf4' must be a constant integer}}
+  __riscv_sf_vc_fv_se_u16mf4(index, p11_7, vs2, fs1, vl);
+}
+
+void test_sf_vc_fv_se_u16mf4_p11_7_not_constant(vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_fv_se_u16mf4' must be a constant integer}}
+  __riscv_sf_vc_fv_se_u16mf4(p26, index, vs2, fs1, vl);
+}
+
+void test_sf_vc_fv_se_u16mf4_p26_overflow(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  __riscv_sf_vc_fv_se_u16mf4(p26_overflow, p11_7, vs2, fs1, vl);
+}
+
+void test_sf_vc_fv_se_u16mf4_p11_7_overflow(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 33 is outside the valid range [0, 31]}}
+  __riscv_sf_vc_fv_se_u16mf4(p26, p11_7_overflow, vs2, fs1, vl);
+}
+
+// sf_vc_v_vv_se
+
+vuint8mf8_t test_sf_vc_v_vv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_vv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_vv_se_u8mf8(index, vs2, vs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_vv_se_u8mf8_p27_26_overflow(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_vv_se_u8mf8(p27_26_overflow, vs2, vs1, vl);
+}
+
+// sf_vc_v_vv
+
+vuint8mf8_t test_sf_vc_v_vv_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_vv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_vv_u8mf8(index, vs2, vs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_vv_u8mf8_p27_26_overflow(vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_vv_u8mf8(p27_26_overflow, vs2, vs1, vl);
+}
+
+// sf_vc_v_xv_se
+
+vuint8mf8_t test_sf_vc_v_xv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_xv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_xv_se_u8mf8(index, vs2, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_xv_se_u8mf8_p27_26_overflow(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_xv_se_u8mf8(p27_26_overflow, vs2, rs1, vl);
+}
+
+// sf_vc_v_xv
+
+vuint8mf8_t test_sf_vc_v_xv_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_xv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_xv_u8mf8(index, vs2, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_xv_u8mf8_p27_26_overflow(vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_xv_u8mf8(p27_26_overflow, vs2, rs1, vl);
+}
+
+// sf_vc_v_iv_se
+
+vuint8mf8_t test_sf_vc_v_iv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_iv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_iv_se_u8mf8(index, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_iv_se_u8mf8_simm5_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_iv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_iv_se_u8mf8(p27_26, vs2, index, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_iv_se_u8mf8_p27_26_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_iv_se_u8mf8(p27_26_overflow, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_iv_se_u8mf8_simm5_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_iv_se_u8mf8(p27_26, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_v_iv
+
+vuint8mf8_t test_sf_vc_v_iv_u8mf8_p27_26_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_iv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_iv_u8mf8(index, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_iv_u8mf8_simm5_not_constant(vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_iv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_iv_u8mf8(p27_26, vs2, index, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_iv_u8mf8_p27_26_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_iv_u8mf8(p27_26_overflow, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_iv_u8mf8_simm5_overflow(vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_iv_u8mf8(p27_26, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_v_fv_se
+
+vuint16mf4_t test_sf_vc_v_fv_se_u16mf4_p26_not_constant(vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  //expected-error@+1 {{argument to '__riscv_sf_vc_v_fv_se_u16mf4' must be a constant integer}}
+  return __riscv_sf_vc_v_fv_se_u16mf4(index, vs2, fs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_fv_se_u16mf4_p26_overflow(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  return __riscv_sf_vc_v_fv_se_u16mf4(p26_overflow, vs2, fs1, vl);
+}
+
+// sf_vc_v_fv
+
+vuint16mf4_t test_sf_vc_v_fv_u16mf4_p26_not_constant(vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_fv_se_u16mf4' must be a constant integer}}
+  return __riscv_sf_vc_v_fv_se_u16mf4(index, vs2, fs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_fv_u16mf4_p26_overflow(vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  return __riscv_sf_vc_v_fv_u16mf4(p26_overflow, vs2, fs1, vl);
+}
+
+// sf_vc_vvv_se
+
+void test_sf_vc_vvv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_vvv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_vvv_se_u8mf8(index, vd, vs2, vs1, vl);
+}
+
+void test_sf_vc_vvv_se_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_vvv_se_u8mf8(p27_26_overflow, vd, vs2, vs1, vl);
+}
+
+// sf_vc_xvv_se
+
+void test_sf_vc_xvv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_xvv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_xvv_se_u8mf8(index, vd, vs2, rs1, vl);
+}
+
+void test_sf_vc_xvv_se_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_xvv_se_u8mf8(p27_26_overflow, vd, vs2, rs1, vl);
+}
+
+// sf_vc_ivv_se
+
+void test_sf_vc_ivv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_ivv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_ivv_se_u8mf8(index, vd, vs2, simm5, vl);
+}
+
+void test_sf_vc_ivv_se_u8mf8_simm5_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_ivv_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_ivv_se_u8mf8(p27_26, vd, vs2, index, vl);
+}
+
+void test_sf_vc_ivv_se_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_ivv_se_u8mf8(p27_26_overflow, vd, vs2, simm5, vl);
+}
+
+void test_sf_vc_ivv_se_u8mf8_simm5_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  __riscv_sf_vc_ivv_se_u8mf8(p27_26, vd, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_fvv_se
+
+void test_sf_vc_fvv_se_u16mf4_p26_not_constant(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_fvv_se_u16mf4' must be a constant integer}}
+  __riscv_sf_vc_fvv_se_u16mf4(index, vd, vs2, fs1, vl);
+}
+
+void test_sf_vc_fvv_se_u16mf4_p26_overflow(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  __riscv_sf_vc_fvv_se_u16mf4(p26_overflow, vd, vs2, fs1, vl);
+}
+
+// sf_vc_v_vvv_se
+
+vuint8mf8_t test_sf_vc_v_vvv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_vvv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_vvv_se_u8mf8(index, vd, vs2, vs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_vvv_se_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_vvv_se_u8mf8(p27_26_overflow, vd, vs2, vs1, vl);
+}
+
+// sf_vc_v_vvv
+
+vuint8mf8_t test_sf_vc_v_vvv_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_vvv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_vvv_u8mf8(index, vd, vs2, vs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_vvv_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_vvv_u8mf8(p27_26_overflow, vd, vs2, vs1, vl);
+}
+
+// sf_vc_v_xvv_se
+
+vuint8mf8_t test_sf_vc_v_xvv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_xvv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_xvv_se_u8mf8(index, vd, vs2, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_xvv_se_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_xvv_se_u8mf8(p27_26_overflow, vd, vs2, rs1, vl);
+}
+
+// sf_vc_v_xvv
+
+vuint8mf8_t test_sf_vc_v_xvv_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_xvv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_xvv_u8mf8(index, vd, vs2, rs1, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_xvv_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_xvv_u8mf8(p27_26_overflow, vd, vs2, rs1, vl);
+}
+
+// sf_vc_v_ivv_se
+
+vuint8mf8_t test_sf_vc_v_ivv_se_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivv_se_u8mf8(index, vd, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_ivv_se_u8mf8_simm5_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivv_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivv_se_u8mf8(p27_26, vd, vs2, index, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_ivv_se_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_ivv_se_u8mf8(p27_26_overflow, vd, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_ivv_se_u8mf8_simm5_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_ivv_se_u8mf8(p27_26, vd, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_v_ivv
+
+vuint8mf8_t test_sf_vc_v_ivv_u8mf8_p27_26_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivv_u8mf8(index, vd, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_ivv_u8mf8_simm5_not_constant(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivv_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivv_u8mf8(p27_26, vd, vs2, index, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_ivv_u8mf8_p27_26_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_ivv_u8mf8(p27_26_overflow, vd, vs2, simm5, vl);
+}
+
+vuint8mf8_t test_sf_vc_v_ivv_u8mf8_simm5_overflow(vuint8mf8_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_ivv_u8mf8(p27_26, vd, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_v_fvv_se
+
+vuint16mf4_t test_sf_vc_v_fvv_se_u16mf4_p26_not_constant(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_fvv_se_u16mf4' must be a constant integer}}
+  return __riscv_sf_vc_v_fvv_se_u16mf4(index, vd, vs2, fs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_fvv_se_u16mf4_p26_overflow(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  return __riscv_sf_vc_v_fvv_se_u16mf4(p26_overflow, vd, vs2, fs1, vl);
+}
+
+// sf_vc_v_fvv
+
+vuint16mf4_t test_sf_vc_v_fvv_u16mf4_p26_not_constant(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_fvv_u16mf4' must be a constant integer}}
+  return __riscv_sf_vc_v_fvv_u16mf4(index, vd, vs2, fs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_fvv_u16mf4_p26_overflow(vuint16mf4_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  return __riscv_sf_vc_v_fvv_u16mf4(p26_overflow, vd, vs2, fs1, vl);
+}
+
+// sf_vc_vvw_se
+
+void test_sf_vc_vvw_se_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_vvw_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_vvw_se_u8mf8(index, vd, vs2, vs1, vl);
+}
+
+void test_sf_vc_vvw_se_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_vvw_se_u8mf8(p27_26_overflow, vd, vs2, vs1, vl);
+}
+
+// sf_vc_xvw_se
+
+void test_sf_vc_xvw_se_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_xvw_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_xvw_se_u8mf8(index, vd, vs2, rs1, vl);
+}
+
+void test_sf_vc_xvw_se_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_xvw_se_u8mf8(p27_26_overflow, vd, vs2, rs1, vl);
+}
+
+// sf_vc_ivw_se
+
+void test_sf_vc_ivw_se_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_ivw_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_ivw_se_u8mf8(index, vd, vs2, simm5, vl);
+}
+
+void test_sf_vc_ivw_se_u8mf8_simm5_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_ivw_se_u8mf8' must be a constant integer}}
+  __riscv_sf_vc_ivw_se_u8mf8(p27_26, vd, vs2, index, vl);
+}
+
+void test_sf_vc_ivw_se_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  __riscv_sf_vc_ivw_se_u8mf8(p27_26_overflow, vd, vs2, simm5, vl);
+}
+
+void test_sf_vc_ivw_se_u8mf8_simm5_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  __riscv_sf_vc_ivw_se_u8mf8(p27_26, vd, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_fvw_se
+
+void test_sf_vc_fvw_se_u16mf4_p26_not_constant(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_fvw_se_u16mf4' must be a constant integer}}
+  __riscv_sf_vc_fvw_se_u16mf4(index, vd, vs2, fs1, vl);
+}
+
+void test_sf_vc_fvw_se_u16mf4_p26_overflow(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  __riscv_sf_vc_fvw_se_u16mf4(p26_overflow, vd, vs2, fs1, vl);
+}
+
+// sf_vc_v_vvw_se
+
+vuint16mf4_t test_sf_vc_v_vvw_se_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_vvw_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_vvw_se_u8mf8(index, vd, vs2, vs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_vvw_se_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_vvw_se_u8mf8(p27_26_overflow, vd, vs2, vs1, vl);
+}
+
+// sf_vc_v_vvw
+
+vuint16mf4_t test_sf_vc_v_vvw_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_vvw_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_vvw_u8mf8(index, vd, vs2, vs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_vvw_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, vuint8mf8_t vs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_vvw_u8mf8(p27_26_overflow, vd, vs2, vs1, vl);
+}
+
+// sf_vc_v_xvw_se
+
+vuint16mf4_t test_sf_vc_v_xvw_se_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_xvw_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_xvw_se_u8mf8(index, vd, vs2, rs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_xvw_se_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_xvw_se_u8mf8(p27_26_overflow, vd, vs2, rs1, vl);
+}
+
+// sf_vc_v_xvw
+
+vuint16mf4_t test_sf_vc_v_xvw_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_xvw_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_xvw_u8mf8(index, vd, vs2, rs1, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_xvw_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, uint8_t rs1, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_xvw_u8mf8(p27_26_overflow, vd, vs2, rs1, vl);
+}
+
+// sf_vc_v_ivw_se
+
+vuint16mf4_t test_sf_vc_v_ivw_se_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivw_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivw_se_u8mf8(index, vd, vs2, simm5, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_ivw_se_u8mf8_simm5_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivw_se_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivw_se_u8mf8(p27_26, vd, vs2, index, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_ivw_se_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_ivw_se_u8mf8(p27_26_overflow, vd, vs2, simm5, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_ivw_se_u8mf8_simm5_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_ivw_se_u8mf8(p27_26, vd, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_v_ivw
+
+vuint16mf4_t test_sf_vc_v_ivw_u8mf8_p27_26_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivw_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivw_u8mf8(index, vd, vs2, simm5, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_ivw_u8mf8_simm5_not_constant(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_ivw_u8mf8' must be a constant integer}}
+  return __riscv_sf_vc_v_ivw_u8mf8(p27_26, vd, vs2, index, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_ivw_u8mf8_p27_26_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 4 is outside the valid range [0, 3]}}
+  return __riscv_sf_vc_v_ivw_u8mf8(p27_26_overflow, vd, vs2, simm5, vl);
+}
+
+vuint16mf4_t test_sf_vc_v_ivw_u8mf8_simm5_overflow(vuint16mf4_t vd, vuint8mf8_t vs2, size_t vl) {
+  // expected-error@+1 {{argument value 16 is outside the valid range [-16, 15]}}
+  return __riscv_sf_vc_v_ivw_u8mf8(p27_26, vd, vs2, simm5_overflow, vl);
+}
+
+// sf_vc_v_fvw_se
+
+vuint32mf2_t test_sf_vc_v_fvw_se_u16mf4_p26_not_constant(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_fvw_se_u16mf4' must be a constant integer}}
+  return __riscv_sf_vc_v_fvw_se_u16mf4(index, vd, vs2, fs1, vl);
+}
+
+vuint32mf2_t test_sf_vc_v_fvw_se_u16mf4_p26_overflow(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  return __riscv_sf_vc_v_fvw_se_u16mf4(p26_overflow, vd, vs2, fs1, vl);
+}
+
+// sf_vc_v_fvw
+
+vuint32mf2_t test_sf_vc_v_fvw_u16mf4_p26_not_constant(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl, int index) {
+  // expected-error@+1 {{argument to '__riscv_sf_vc_v_fvw_u16mf4' must be a constant integer}}
+  return __riscv_sf_vc_v_fvw_u16mf4(index, vd, vs2, fs1, vl);
+}
+
+vuint32mf2_t test_sf_vc_v_fvw_u16mf4_p26_overflow(vuint32mf2_t vd, vuint16mf4_t vs2, _Float16 fs1, size_t vl) {
+  // expected-error@+1 {{argument value 2 is outside the valid range [0, 1]}}
+  return __riscv_sf_vc_v_fvw_u16mf4(p26_overflow, vd, vs2, fs1, vl);
+}
diff --git a/clang/test/CodeGen/assignment-tracking/flag.cpp b/clang/test/CodeGen/assignment-tracking/flag.cpp
index aa1f054dae4d7..3bd974fe07c6c 100644
--- a/clang/test/CodeGen/assignment-tracking/flag.cpp
+++ b/clang/test/CodeGen/assignment-tracking/flag.cpp
@@ -8,10 +8,10 @@
 // RUN:     -emit-llvm  %s -o - -fexperimental-assignment-tracking=disabled -O1\
 // RUN: | FileCheck %s --check-prefixes=DISABLE
 
-//// Disabled by default:
+//// Enabled by default:
 // RUN: %clang_cc1 -triple x86_64-none-linux-gnu -debug-info-kind=standalone   \
 // RUN:     -emit-llvm  %s -o - -O1                                            \
-// RUN: | FileCheck %s --check-prefixes=DISABLE
+// RUN: | FileCheck %s --check-prefixes=ENABLE
 
 //// Disabled at O0 unless forced.
 // RUN: %clang_cc1 -triple x86_64-none-linux-gnu -debug-info-kind=standalone   \
diff --git a/clang/test/CodeGen/atomic.c b/clang/test/CodeGen/atomic.c
index f232d5b63ac74..69f06eedb7caa 100644
--- a/clang/test/CodeGen/atomic.c
+++ b/clang/test/CodeGen/atomic.c
@@ -1,5 +1,9 @@
 // RUN: %clang_cc1 %s -emit-llvm -o - -triple=i686-apple-darwin9 | FileCheck %s
 
+// CHECK: @[[GLOB_POINTER:.+]] = internal global ptr null
+// CHECK: @[[GLOB_INT:.+]] = internal global i32 0
+// CHECK: @[[GLOB_FLT:.+]] = internal global float {{[0e\+-\.]+}}, align
+
 int atomic(void) {
   // non-sensical test for sync functions
   int old;
@@ -118,3 +122,19 @@ void addrspace(int  __attribute__((address_space(256))) * P) {
   __sync_xor_and_fetch(P, 123);
   // CHECK: atomicrmw xor ptr addrspace(256){{.*}}, i32 123 seq_cst, align 4
 }
+
+// Ensure that global initialization of atomics is correct.
+static _Atomic(int *) glob_pointer = (void *)0;
+static _Atomic int glob_int = 0;
+static _Atomic float glob_flt = 0.0f;
+
+void force_global_uses(void) {
+  (void)glob_pointer;
+  // CHECK: %[[LOCAL_INT:.+]] = load atomic i32, ptr @[[GLOB_POINTER]] seq_cst
+  // CHECK-NEXT: inttoptr i32 %[[LOCAL_INT]] to ptr
+  (void)glob_int;
+  // CHECK: load atomic i32, ptr @[[GLOB_INT]] seq_cst
+  (void)glob_flt;
+  // CHECK: %[[LOCAL_FLT:.+]] = load atomic i32, ptr @[[GLOB_FLT]] seq_cst
+  // CHECK-NEXT: bitcast i32 %[[LOCAL_FLT]] to float
+}
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c
new file mode 100644
index 0000000000000..c5a3cb0fcffa4
--- /dev/null
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c
@@ -0,0 +1,138 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 -S -O1 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=2 -mvscale-max=2 -S -O1 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-128
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -S -O1 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-256
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+#define DEFINE_STRUCT(ty)   \
+  struct struct_##ty {      \
+    fixed_##ty##_t x, y[3]; \
+  } struct_##ty;
+
+DEFINE_STRUCT(int64m1)
+DEFINE_STRUCT(float64m1)
+
+//===----------------------------------------------------------------------===//
+// int64
+//===----------------------------------------------------------------------===//
+
+// CHECK-64-LABEL: @read_int64m1(
+// CHECK-64-NEXT:  entry:
+// CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-64-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> undef, <1 x i64> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+// CHECK-128-LABEL: @read_int64m1(
+// CHECK-128-NEXT:  entry:
+// CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v2i64(<vscale x 1 x i64> undef, <2 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+// CHECK-256-LABEL: @read_int64m1(
+// CHECK-256-NEXT:  entry:
+// CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+vint64m1_t read_int64m1(struct struct_int64m1 *s) {
+  return s->y[0];
+}
+
+// CHECK-64-LABEL: @write_int64m1(
+// CHECK-64-NEXT:  entry:
+// CHECK-64-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <1 x i64> @llvm.vector.extract.v1i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
+// CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-64-NEXT:    store <1 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-64-NEXT:    ret void
+//
+// CHECK-128-LABEL: @write_int64m1(
+// CHECK-128-NEXT:  entry:
+// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x i64> @llvm.vector.extract.v2i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-128-NEXT:    store <2 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-128-NEXT:    ret void
+//
+// CHECK-256-LABEL: @write_int64m1(
+// CHECK-256-NEXT:  entry:
+// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-256-NEXT:    store <4 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-256-NEXT:    ret void
+//
+void write_int64m1(struct struct_int64m1 *s, vint64m1_t x) {
+  s->y[0] = x;
+}
+
+//===----------------------------------------------------------------------===//
+// float64
+//===----------------------------------------------------------------------===//
+
+// CHECK-64-LABEL: @read_float64m1(
+// CHECK-64-NEXT:  entry:
+// CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x double>, ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-64-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v1f64(<vscale x 1 x double> undef, <1 x double> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+// CHECK-128-LABEL: @read_float64m1(
+// CHECK-128-NEXT:  entry:
+// CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v2f64(<vscale x 1 x double> undef, <2 x double> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+// CHECK-256-LABEL: @read_float64m1(
+// CHECK-256-NEXT:  entry:
+// CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+vfloat64m1_t read_float64m1(struct struct_float64m1 *s) {
+  return s->y[0];
+}
+
+// CHECK-64-LABEL: @write_float64m1(
+// CHECK-64-NEXT:  entry:
+// CHECK-64-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <1 x double> @llvm.vector.extract.v1f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
+// CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-64-NEXT:    store <1 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-64-NEXT:    ret void
+//
+// CHECK-128-LABEL: @write_float64m1(
+// CHECK-128-NEXT:  entry:
+// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x double> @llvm.vector.extract.v2f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-128-NEXT:    store <2 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-128-NEXT:    ret void
+//
+// CHECK-256-LABEL: @write_float64m1(
+// CHECK-256-NEXT:  entry:
+// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
+// CHECK-256-NEXT:    store <4 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-256-NEXT:    ret void
+//
+void write_float64m1(struct struct_float64m1 *s, vfloat64m1_t x) {
+  s->y[0] = x;
+}
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-call.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-call.c
new file mode 100644
index 0000000000000..c9e0eeedbf26d
--- /dev/null
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-call.c
@@ -0,0 +1,115 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -S -O1 -emit-llvm -o - %s | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <riscv_vector.h>
+
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+//===----------------------------------------------------------------------===//
+// Test caller/callee with VLST <-> VLAT
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @sizeless_callee(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[X:%.*]]
+//
+vint32m1_t sizeless_callee(vint32m1_t x) {
+  return x;
+}
+
+// CHECK-LABEL: @fixed_caller(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[X_COERCE:%.*]]
+//
+fixed_int32m1_t fixed_caller(fixed_int32m1_t x) {
+  return sizeless_callee(x);
+}
+
+// CHECK-LABEL: @fixed_callee(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[X_COERCE:%.*]]
+//
+fixed_int32m1_t fixed_callee(fixed_int32m1_t x) {
+  return x;
+}
+
+// CHECK-LABEL: @sizeless_caller(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[COERCE1:%.*]] = alloca <8 x i32>, align 8
+// CHECK-NEXT:    store <vscale x 2 x i32> [[X:%.*]], ptr [[COERCE1]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[COERCE1]], align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-NEXT:    [[CASTSCALABLESVE2:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP0]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE2]]
+//
+vint32m1_t sizeless_caller(vint32m1_t x) {
+  return fixed_callee(x);
+}
+
+//===----------------------------------------------------------------------===//
+// fixed, fixed
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @call_int32_ff(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i32> @llvm.riscv.vadd.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[OP1_COERCE:%.*]], <vscale x 2 x i32> [[OP2_COERCE:%.*]], i64 8)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+fixed_int32m1_t call_int32_ff(fixed_int32m1_t op1, fixed_int32m1_t op2) {
+  return __riscv_vadd(op1, op2, __riscv_v_fixed_vlen/32);
+}
+
+// CHECK-LABEL: @call_float64_ff(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64.i64(<vscale x 1 x double> poison, <vscale x 1 x double> [[OP1_COERCE:%.*]], <vscale x 1 x double> [[OP2_COERCE:%.*]], i64 4)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[TMP0]]
+//
+fixed_float64m1_t call_float64_ff(fixed_float64m1_t op1, fixed_float64m1_t op2) {
+  return __riscv_vfadd(op1, op2, __riscv_v_fixed_vlen/64);
+}
+
+//===----------------------------------------------------------------------===//
+// fixed, scalable
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @call_int32_fs(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i32> @llvm.riscv.vadd.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[OP1_COERCE:%.*]], <vscale x 2 x i32> [[OP2:%.*]], i64 8)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+fixed_int32m1_t call_int32_fs(fixed_int32m1_t op1, vint32m1_t op2) {
+  return __riscv_vadd(op1, op2, __riscv_v_fixed_vlen/32);
+}
+
+// CHECK-LABEL: @call_float64_fs(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64.i64(<vscale x 1 x double> poison, <vscale x 1 x double> [[OP1_COERCE:%.*]], <vscale x 1 x double> [[OP2:%.*]], i64 4)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[TMP0]]
+//
+fixed_float64m1_t call_float64_fs(fixed_float64m1_t op1, vfloat64m1_t op2) {
+  return __riscv_vfadd(op1, op2, __riscv_v_fixed_vlen/64);
+}
+
+//===----------------------------------------------------------------------===//
+// scalable, scalable
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @call_int32_ss(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i32> @llvm.riscv.vadd.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[OP1:%.*]], <vscale x 2 x i32> [[OP2:%.*]], i64 8)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+fixed_int32m1_t call_int32_ss(vint32m1_t op1, vint32m1_t op2) {
+  return __riscv_vadd(op1, op2, __riscv_v_fixed_vlen/32);
+}
+
+// CHECK-LABEL: @call_float64_ss(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x double> @llvm.riscv.vfadd.nxv1f64.nxv1f64.i64(<vscale x 1 x double> poison, <vscale x 1 x double> [[OP1:%.*]], <vscale x 1 x double> [[OP2:%.*]], i64 4)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[TMP0]]
+//
+fixed_float64m1_t call_float64_ss(vfloat64m1_t op1, vfloat64m1_t op2) {
+  return __riscv_vfadd(op1, op2, __riscv_v_fixed_vlen/64);
+}
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-cast.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-cast.c
new file mode 100644
index 0000000000000..b324d0a2b3cbd
--- /dev/null
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-cast.c
@@ -0,0 +1,108 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -S -O1 -emit-llvm -o - %s | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef int32_t gnu_int32m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+
+// CHECK-LABEL: @to_vint32m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[TYPE_COERCE:%.*]]
+//
+vint32m1_t to_vint32m1_t(fixed_int32m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @from_vint32m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[TYPE:%.*]]
+//
+fixed_int32m1_t from_vint32m1_t(vint32m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @to_vfloat64m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 1 x double> [[TYPE_COERCE:%.*]]
+//
+vfloat64m1_t to_vfloat64m1_t(fixed_float64m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @from_vfloat64m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <vscale x 1 x double> [[TYPE:%.*]]
+//
+fixed_float64m1_t from_vfloat64m1_t(vfloat64m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @lax_cast(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[SAVED_VALUE:%.*]] = alloca <8 x i32>, align 32
+// CHECK-NEXT:    [[TYPE:%.*]] = tail call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[TYPE_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    store <8 x i32> [[TYPE]], ptr [[SAVED_VALUE]], align 32, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <vscale x 1 x i64>, ptr [[SAVED_VALUE]], align 32, !tbaa [[TBAA4]]
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[TMP0]]
+//
+vint64m1_t lax_cast(fixed_int32m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @to_vint32m1_t__from_gnu_int32m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TYPE:%.*]] = load <8 x i32>, ptr [[TMP0:%.*]], align 32, !tbaa [[TBAA4]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+vint32m1_t to_vint32m1_t__from_gnu_int32m1_t(gnu_int32m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @from_vint32m1_t__to_gnu_int32m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[TYPE:%.*]], i64 0)
+// CHECK-NEXT:    store <8 x i32> [[CASTFIXEDSVE]], ptr [[AGG_RESULT:%.*]], align 32, !tbaa [[TBAA4]]
+// CHECK-NEXT:    ret void
+//
+gnu_int32m1_t from_vint32m1_t__to_gnu_int32m1_t(vint32m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @to_fixed_int32m1_t__from_gnu_int32m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TYPE:%.*]] = load <8 x i32>, ptr [[TMP0:%.*]], align 32, !tbaa [[TBAA4]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TYPE]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t to_fixed_int32m1_t__from_gnu_int32m1_t(gnu_int32m1_t type) {
+  return type;
+}
+
+// CHECK-LABEL: @from_fixed_int32m1_t__to_gnu_int32m1_t(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TYPE:%.*]] = tail call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[TYPE_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    store <8 x i32> [[TYPE]], ptr [[AGG_RESULT:%.*]], align 32, !tbaa [[TBAA4]]
+// CHECK-NEXT:    ret void
+//
+gnu_int32m1_t from_fixed_int32m1_t__to_gnu_int32m1_t(fixed_int32m1_t type) {
+  return type;
+}
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-codegen.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-codegen.c
new file mode 100644
index 0000000000000..5268117c5a2a3
--- /dev/null
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-codegen.c
@@ -0,0 +1,77 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -S -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <riscv_vector.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+fixed_int32m1_t global_vec;
+
+// CHECK-LABEL: @test_ptr_to_global(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca <8 x i32>, align 8
+// CHECK-NEXT:    [[GLOBAL_VEC_PTR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr @global_vec, ptr [[GLOBAL_VEC_PTR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[GLOBAL_VEC_PTR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr [[TMP0]], align 8
+// CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr [[RETVAL]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t test_ptr_to_global() {
+  fixed_int32m1_t *global_vec_ptr;
+  global_vec_ptr = &global_vec;
+  return *global_vec_ptr;
+}
+
+//
+// Test casting pointer from fixed-length array to scalable vector.
+// CHECK-LABEL: @array_arg(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca <8 x i32>, align 8
+// CHECK-NEXT:    [[ARR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[ARR:%.*]], ptr [[ARR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARR_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds <8 x i32>, ptr [[TMP0]], i64 0
+// CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr [[ARRAYIDX]], align 8
+// CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr [[RETVAL]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t array_arg(fixed_int32m1_t arr[]) {
+  return arr[0];
+}
+
+// CHECK-LABEL: @test_cast(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca <8 x i32>, align 8
+// CHECK-NEXT:    [[VEC_ADDR:%.*]] = alloca <vscale x 2 x i32>, align 4
+// CHECK-NEXT:    store <vscale x 2 x i32> [[VEC:%.*]], ptr [[VEC_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr @global_vec, align 8
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP0]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = load <vscale x 2 x i32>, ptr [[VEC_ADDR]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i32> @llvm.riscv.vadd.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> poison, <vscale x 2 x i32> [[CASTSCALABLESVE]], <vscale x 2 x i32> [[TMP1]], i64 8)
+// CHECK-NEXT:    [[CASTFIXEDSVE:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    store <8 x i32> [[CASTFIXEDSVE]], ptr [[RETVAL]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr [[RETVAL]], align 8
+// CHECK-NEXT:    [[CASTSCALABLESVE1:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[TMP3]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE1]]
+//
+fixed_int32m1_t test_cast(vint32m1_t vec) {
+  return __riscv_vadd(global_vec, vec, __riscv_v_fixed_vlen/32);
+}
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-globals.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-globals.c
new file mode 100644
index 0000000000000..447c3803dec8e
--- /dev/null
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-globals.c
@@ -0,0 +1,58 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 -S -O1 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-128
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -S -O1 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-512
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+fixed_int64m1_t global_i64;
+
+//===----------------------------------------------------------------------===//
+// WRITES
+//===----------------------------------------------------------------------===//
+
+// CHECK-128-LABEL: @write_global_i64(
+// CHECK-128-NEXT:  entry:
+// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <1 x i64> @llvm.vector.extract.v1i64.nxv1i64(<vscale x 1 x i64> [[V:%.*]], i64 0)
+// CHECK-128-NEXT:    store <1 x i64> [[CASTFIXEDSVE]], ptr @global_i64, align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-128-NEXT:    ret void
+//
+// CHECK-512-LABEL: @write_global_i64(
+// CHECK-512-NEXT:  entry:
+// CHECK-512-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[V:%.*]], i64 0)
+// CHECK-512-NEXT:    store <4 x i64> [[CASTFIXEDSVE]], ptr @global_i64, align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-512-NEXT:    ret void
+//
+void write_global_i64(vint64m1_t v) { global_i64 = v; }
+
+//===----------------------------------------------------------------------===//
+// READS
+//===----------------------------------------------------------------------===//
+
+// CHECK-128-LABEL: @read_global_i64(
+// CHECK-128-NEXT:  entry:
+// CHECK-128-NEXT:    [[TMP0:%.*]] = load <1 x i64>, ptr @global_i64, align 8, !tbaa [[TBAA4]]
+// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> undef, <1 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+// CHECK-512-LABEL: @read_global_i64(
+// CHECK-512-NEXT:  entry:
+// CHECK-512-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr @global_i64, align 8, !tbaa [[TBAA4]]
+// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+vint64m1_t read_global_i64() { return global_i64; }
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-types.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-types.c
new file mode 100644
index 0000000000000..8ebbbf181fd36
--- /dev/null
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-types.c
@@ -0,0 +1,483 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=2 -mvscale-max=2 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-128
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-256
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-512
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=16 -mvscale-max=16 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-1024
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+// Define valid fixed-width RVV types
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+//===----------------------------------------------------------------------===//
+// Structs and unions
+//===----------------------------------------------------------------------===//
+#define DEFINE_STRUCT(ty) \
+  struct struct_##ty {    \
+    fixed_##ty##_t x;     \
+  } struct_##ty;
+
+#define DEFINE_UNION(ty) \
+  union union_##ty {     \
+    fixed_##ty##_t x;    \
+  } union_##ty;
+
+DEFINE_STRUCT(int8m1)
+DEFINE_STRUCT(int16m1)
+DEFINE_STRUCT(int32m1)
+DEFINE_STRUCT(int64m1)
+DEFINE_STRUCT(uint8m1)
+DEFINE_STRUCT(uint16m1)
+DEFINE_STRUCT(uint32m1)
+DEFINE_STRUCT(uint64m1)
+DEFINE_STRUCT(float32m1)
+DEFINE_STRUCT(float64m1)
+
+DEFINE_UNION(int8m1)
+DEFINE_UNION(int16m1)
+DEFINE_UNION(int32m1)
+DEFINE_UNION(int64m1)
+DEFINE_UNION(uint8m1)
+DEFINE_UNION(uint16m1)
+DEFINE_UNION(uint32m1)
+DEFINE_UNION(uint64m1)
+DEFINE_UNION(float32m1)
+DEFINE_UNION(float64m1)
+
+//===----------------------------------------------------------------------===//
+// Global variables
+//===----------------------------------------------------------------------===//
+fixed_int8m1_t global_i8;
+fixed_int16m1_t global_i16;
+fixed_int32m1_t global_i32;
+fixed_int64m1_t global_i64;
+
+fixed_uint8m1_t global_u8;
+fixed_uint16m1_t global_u16;
+fixed_uint32m1_t global_u32;
+fixed_uint64m1_t global_u64;
+
+fixed_float32m1_t global_f32;
+fixed_float64m1_t global_f64;
+
+//===----------------------------------------------------------------------===//
+// Global arrays
+//===----------------------------------------------------------------------===//
+fixed_int8m1_t global_arr_i8[3];
+fixed_int16m1_t global_arr_i16[3];
+fixed_int32m1_t global_arr_i32[3];
+fixed_int64m1_t global_arr_i64[3];
+
+fixed_uint8m1_t global_arr_u8[3];
+fixed_uint16m1_t global_arr_u16[3];
+fixed_uint32m1_t global_arr_u32[3];
+fixed_uint64m1_t global_arr_u64[3];
+
+fixed_float32m1_t global_arr_f32[3];
+fixed_float64m1_t global_arr_f64[3];
+
+//===----------------------------------------------------------------------===//
+// Locals
+//===----------------------------------------------------------------------===//
+void f() {
+  // Variables
+  fixed_int8m1_t local_i8;
+  fixed_int16m1_t local_i16;
+  fixed_int32m1_t local_i32;
+  fixed_int64m1_t local_i64;
+  fixed_uint8m1_t local_u8;
+  fixed_uint16m1_t local_u16;
+  fixed_uint32m1_t local_u32;
+  fixed_uint64m1_t local_u64;
+  fixed_float32m1_t local_f32;
+  fixed_float64m1_t local_f64;
+
+  // Arrays
+  fixed_int8m1_t local_arr_i8[3];
+  fixed_int16m1_t local_arr_i16[3];
+  fixed_int32m1_t local_arr_i32[3];
+  fixed_int64m1_t local_arr_i64[3];
+  fixed_uint8m1_t local_arr_u8[3];
+  fixed_uint16m1_t local_arr_u16[3];
+  fixed_uint32m1_t local_arr_u32[3];
+  fixed_uint64m1_t local_arr_u64[3];
+  fixed_float32m1_t local_arr_f32[3];
+  fixed_float64m1_t local_arr_f64[3];
+}
+
+//===----------------------------------------------------------------------===//
+// Structs and unions
+//===----------------------------------------------------------------------===//
+// CHECK-64:      %struct.struct_int8m1 = type { <8 x i8> }
+// CHECK-64-NEXT: %struct.struct_int16m1 = type { <4 x i16> }
+// CHECK-64-NEXT: %struct.struct_int32m1 = type { <2 x i32> }
+// CHECK-64-NEXT: %struct.struct_int64m1 = type { <1 x i64> }
+// CHECK-64-NEXT: %struct.struct_uint8m1 = type { <8 x i8> }
+// CHECK-64-NEXT: %struct.struct_uint16m1 = type { <4 x i16> }
+// CHECK-64-NEXT: %struct.struct_uint32m1 = type { <2 x i32> }
+// CHECK-64-NEXT: %struct.struct_uint64m1 = type { <1 x i64> }
+// CHECK-64-NEXT: %struct.struct_float32m1 = type { <2 x float> }
+// CHECK-64-NEXT: %struct.struct_float64m1 = type { <1 x double> }
+
+// CHECK-128:      %struct.struct_int8m1 = type { <16 x i8> }
+// CHECK-128-NEXT: %struct.struct_int16m1 = type { <8 x i16> }
+// CHECK-128-NEXT: %struct.struct_int32m1 = type { <4 x i32> }
+// CHECK-128-NEXT: %struct.struct_int64m1 = type { <2 x i64> }
+// CHECK-128-NEXT: %struct.struct_uint8m1 = type { <16 x i8> }
+// CHECK-128-NEXT: %struct.struct_uint16m1 = type { <8 x i16> }
+// CHECK-128-NEXT: %struct.struct_uint32m1 = type { <4 x i32> }
+// CHECK-128-NEXT: %struct.struct_uint64m1 = type { <2 x i64> }
+// CHECK-128-NEXT: %struct.struct_float32m1 = type { <4 x float> }
+// CHECK-128-NEXT: %struct.struct_float64m1 = type { <2 x double> }
+
+// CHECK-256:      %struct.struct_int8m1 = type { <32 x i8> }
+// CHECK-256-NEXT: %struct.struct_int16m1 = type { <16 x i16> }
+// CHECK-256-NEXT: %struct.struct_int32m1 = type { <8 x i32> }
+// CHECK-256-NEXT: %struct.struct_int64m1 = type { <4 x i64> }
+// CHECK-256-NEXT: %struct.struct_uint8m1 = type { <32 x i8> }
+// CHECK-256-NEXT: %struct.struct_uint16m1 = type { <16 x i16> }
+// CHECK-256-NEXT: %struct.struct_uint32m1 = type { <8 x i32> }
+// CHECK-256-NEXT: %struct.struct_uint64m1 = type { <4 x i64> }
+// CHECK-256-NEXT: %struct.struct_float32m1 = type { <8 x float> }
+// CHECK-256-NEXT: %struct.struct_float64m1 = type { <4 x double> }
+
+// CHECK-512:      %struct.struct_int8m1 = type { <64 x i8> }
+// CHECK-512-NEXT: %struct.struct_int16m1 = type { <32 x i16> }
+// CHECK-512-NEXT: %struct.struct_int32m1 = type { <16 x i32> }
+// CHECK-512-NEXT: %struct.struct_int64m1 = type { <8 x i64> }
+// CHECK-512-NEXT: %struct.struct_uint8m1 = type { <64 x i8> }
+// CHECK-512-NEXT: %struct.struct_uint16m1 = type { <32 x i16> }
+// CHECK-512-NEXT: %struct.struct_uint32m1 = type { <16 x i32> }
+// CHECK-512-NEXT: %struct.struct_uint64m1 = type { <8 x i64> }
+// CHECK-512-NEXT: %struct.struct_float32m1 = type { <16 x float> }
+// CHECK-512-NEXT: %struct.struct_float64m1 = type { <8 x double> }
+
+// CHECK-1024:      %struct.struct_int8m1 = type { <128 x i8> }
+// CHECK-1024-NEXT: %struct.struct_int16m1 = type { <64 x i16> }
+// CHECK-1024-NEXT: %struct.struct_int32m1 = type { <32 x i32> }
+// CHECK-1024-NEXT: %struct.struct_int64m1 = type { <16 x i64> }
+// CHECK-1024-NEXT: %struct.struct_uint8m1 = type { <128 x i8> }
+// CHECK-1024-NEXT: %struct.struct_uint16m1 = type { <64 x i16> }
+// CHECK-1024-NEXT: %struct.struct_uint32m1 = type { <32 x i32> }
+// CHECK-1024-NEXT: %struct.struct_uint64m1 = type { <16 x i64> }
+// CHECK-1024-NEXT: %struct.struct_float32m1 = type { <32 x float> }
+// CHECK-1024-NEXT: %struct.struct_float64m1 = type { <16 x double> }
+
+// CHECK-64:      %union.union_int8m1 = type { <8 x i8> }
+// CHECK-64-NEXT: %union.union_int16m1 = type { <4 x i16> }
+// CHECK-64-NEXT: %union.union_int32m1 = type { <2 x i32> }
+// CHECK-64-NEXT: %union.union_int64m1 = type { <1 x i64> }
+// CHECK-64-NEXT: %union.union_uint8m1 = type { <8 x i8> }
+// CHECK-64-NEXT: %union.union_uint16m1 = type { <4 x i16> }
+// CHECK-64-NEXT: %union.union_uint32m1 = type { <2 x i32> }
+// CHECK-64-NEXT: %union.union_uint64m1 = type { <1 x i64> }
+// CHECK-64-NEXT: %union.union_float32m1 = type { <2 x float> }
+// CHECK-64-NEXT: %union.union_float64m1 = type { <1 x double> }
+
+// CHECK-128:      %union.union_int8m1 = type { <16 x i8> }
+// CHECK-128-NEXT: %union.union_int16m1 = type { <8 x i16> }
+// CHECK-128-NEXT: %union.union_int32m1 = type { <4 x i32> }
+// CHECK-128-NEXT: %union.union_int64m1 = type { <2 x i64> }
+// CHECK-128-NEXT: %union.union_uint8m1 = type { <16 x i8> }
+// CHECK-128-NEXT: %union.union_uint16m1 = type { <8 x i16> }
+// CHECK-128-NEXT: %union.union_uint32m1 = type { <4 x i32> }
+// CHECK-128-NEXT: %union.union_uint64m1 = type { <2 x i64> }
+// CHECK-128-NEXT: %union.union_float32m1 = type { <4 x float> }
+// CHECK-128-NEXT: %union.union_float64m1 = type { <2 x double> }
+
+// CHECK-256:      %union.union_int8m1 = type { <32 x i8> }
+// CHECK-256-NEXT: %union.union_int16m1 = type { <16 x i16> }
+// CHECK-256-NEXT: %union.union_int32m1 = type { <8 x i32> }
+// CHECK-256-NEXT: %union.union_int64m1 = type { <4 x i64> }
+// CHECK-256-NEXT: %union.union_uint8m1 = type { <32 x i8> }
+// CHECK-256-NEXT: %union.union_uint16m1 = type { <16 x i16> }
+// CHECK-256-NEXT: %union.union_uint32m1 = type { <8 x i32> }
+// CHECK-256-NEXT: %union.union_uint64m1 = type { <4 x i64> }
+// CHECK-256-NEXT: %union.union_float32m1 = type { <8 x float> }
+// CHECK-256-NEXT: %union.union_float64m1 = type { <4 x double> }
+
+// CHECK-512:      %union.union_int8m1 = type { <64 x i8> }
+// CHECK-512-NEXT: %union.union_int16m1 = type { <32 x i16> }
+// CHECK-512-NEXT: %union.union_int32m1 = type { <16 x i32> }
+// CHECK-512-NEXT: %union.union_int64m1 = type { <8 x i64> }
+// CHECK-512-NEXT: %union.union_uint8m1 = type { <64 x i8> }
+// CHECK-512-NEXT: %union.union_uint16m1 = type { <32 x i16> }
+// CHECK-512-NEXT: %union.union_uint32m1 = type { <16 x i32> }
+// CHECK-512-NEXT: %union.union_uint64m1 = type { <8 x i64> }
+// CHECK-512-NEXT: %union.union_float32m1 = type { <16 x float> }
+// CHECK-512-NEXT: %union.union_float64m1 = type { <8 x double> }
+
+// CHECK-1024:      %union.union_int8m1 = type { <128 x i8> }
+// CHECK-1024-NEXT: %union.union_int16m1 = type { <64 x i16> }
+// CHECK-1024-NEXT: %union.union_int32m1 = type { <32 x i32> }
+// CHECK-1024-NEXT: %union.union_int64m1 = type { <16 x i64> }
+// CHECK-1024-NEXT: %union.union_uint8m1 = type { <128 x i8> }
+// CHECK-1024-NEXT: %union.union_uint16m1 = type { <64 x i16> }
+// CHECK-1024-NEXT: %union.union_uint32m1 = type { <32 x i32> }
+// CHECK-1024-NEXT: %union.union_uint64m1 = type { <16 x i64> }
+// CHECK-1024-NEXT: %union.union_float32m1 = type { <32 x float> }
+// CHECK-1024-NEXT: %union.union_float64m1 = type { <16 x double> }
+
+//===----------------------------------------------------------------------===//
+// Global variables
+//===----------------------------------------------------------------------===//
+// CHECK-64:      @global_i8 ={{.*}} global <8 x i8> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_i16 ={{.*}} global <4 x i16> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_i32 ={{.*}} global <2 x i32> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_i64 ={{.*}} global <1 x i64> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_u8 ={{.*}} global <8 x i8> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_u16 ={{.*}} global <4 x i16> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_u32 ={{.*}} global <2 x i32> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_u64 ={{.*}} global <1 x i64> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_f32 ={{.*}} global <2 x float> zeroinitializer, align 8
+// CHECK-64-NEXT: @global_f64 ={{.*}} global <1 x double> zeroinitializer, align 8
+
+// CHECK-128:      @global_i8 ={{.*}} global <16 x i8> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_i16 ={{.*}} global <8 x i16> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_i32 ={{.*}} global <4 x i32> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_i64 ={{.*}} global <2 x i64> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_u8 ={{.*}} global <16 x i8> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_u16 ={{.*}} global <8 x i16> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_u32 ={{.*}} global <4 x i32> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_u64 ={{.*}} global <2 x i64> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_f32 ={{.*}} global <4 x float> zeroinitializer, align 8
+// CHECK-128-NEXT: @global_f64 ={{.*}} global <2 x double> zeroinitializer, align 8
+
+// CHECK-256:      @global_i8 ={{.*}} global <32 x i8> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_i16 ={{.*}} global <16 x i16> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_i32 ={{.*}} global <8 x i32> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_i64 ={{.*}} global <4 x i64> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_u8 ={{.*}} global <32 x i8> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_u16 ={{.*}} global <16 x i16> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_u32 ={{.*}} global <8 x i32> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_u64 ={{.*}} global <4 x i64> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_f32 ={{.*}} global <8 x float> zeroinitializer, align 8
+// CHECK-NEXT-256: @global_f64 ={{.*}} global <4 x double> zeroinitializer, align 8
+
+// CHECK-512:      @global_i8 ={{.*}} global <64 x i8> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_i16 ={{.*}} global <32 x i16> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_i32 ={{.*}} global <16 x i32> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_i64 ={{.*}} global <8 x i64> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_u8 ={{.*}} global <64 x i8> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_u16 ={{.*}} global <32 x i16> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_u32 ={{.*}} global <16 x i32> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_u64 ={{.*}} global <8 x i64> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_f32 ={{.*}} global <16 x float> zeroinitializer, align 8
+// CHECK-NEXT-512: @global_f64 ={{.*}} global <8 x double> zeroinitializer, align 8
+
+// CHECK-1024:      @global_i8 ={{.*}} global <128 x i8> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_i16 ={{.*}} global <64 x i16> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_i32 ={{.*}} global <32 x i32> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_i64 ={{.*}} global <16 x i64> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_u8 ={{.*}} global <128 x i8> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_u16 ={{.*}} global <64 x i16> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_u32 ={{.*}} global <32 x i32> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_u64 ={{.*}} global <16 x i64> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_f32 ={{.*}} global <32 x float> zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_f64 ={{.*}} global <16 x double> zeroinitializer, align 8
+
+//===----------------------------------------------------------------------===//
+// Global arrays
+//===----------------------------------------------------------------------===//
+// CHECK-64:      @global_arr_i8 ={{.*}} global [3 x <8 x i8>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_i16 ={{.*}} global [3 x <4 x i16>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_i32 ={{.*}} global [3 x <2 x i32>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_i64 ={{.*}} global [3 x <1 x i64>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_u8 ={{.*}} global [3 x <8 x i8>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_u16 ={{.*}} global [3 x <4 x i16>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_u32 ={{.*}} global [3 x <2 x i32>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_u64 ={{.*}} global [3 x <1 x i64>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_f32 ={{.*}} global [3 x <2 x float>] zeroinitializer, align 8
+// CHECK-64-NEXT: @global_arr_f64 ={{.*}} global [3 x <1 x double>] zeroinitializer, align 8
+
+// CHECK-128:      @global_arr_i8 ={{.*}} global [3 x <16 x i8>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_i16 ={{.*}} global [3 x <8 x i16>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_i32 ={{.*}} global [3 x <4 x i32>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_i64 ={{.*}} global [3 x <2 x i64>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_u8 ={{.*}} global [3 x <16 x i8>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_u16 ={{.*}} global [3 x <8 x i16>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_u32 ={{.*}} global [3 x <4 x i32>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_u64 ={{.*}} global [3 x <2 x i64>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_f32 ={{.*}} global [3 x <4 x float>] zeroinitializer, align 8
+// CHECK-128-NEXT: @global_arr_f64 ={{.*}} global [3 x <2 x double>] zeroinitializer, align 8
+
+// CHECK-256:      @global_arr_i8 ={{.*}} global [3 x <32 x i8>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_i16 ={{.*}} global [3 x <16 x i16>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_i32 ={{.*}} global [3 x <8 x i32>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_i64 ={{.*}} global [3 x <4 x i64>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_u8 ={{.*}} global [3 x <32 x i8>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_u16 ={{.*}} global [3 x <16 x i16>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_u32 ={{.*}} global [3 x <8 x i32>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_u64 ={{.*}} global [3 x <4 x i64>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_f32 ={{.*}} global [3 x <8 x float>] zeroinitializer, align 8
+// CHECK-NEXT-256: @global_arr_f64 ={{.*}} global [3 x <4 x double>] zeroinitializer, align 8
+
+// CHECK-512:      @global_arr_i8 ={{.*}} global [3 x <64 x i8>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_i16 ={{.*}} global [3 x <32 x i16>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_i32 ={{.*}} global [3 x <16 x i32>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_i64 ={{.*}} global [3 x <8 x i64>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_u8 ={{.*}} global [3 x <64 x i8>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_u16 ={{.*}} global [3 x <32 x i16>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_u32 ={{.*}} global [3 x <16 x i32>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_u64 ={{.*}} global [3 x <8 x i64>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_f32 ={{.*}} global [3 x <16 x float>] zeroinitializer, align 8
+// CHECK-NEXT-512: @global_arr_f64 ={{.*}} global [3 x <8 x double>] zeroinitializer, align 8
+
+// CHECK-1024:      @global_arr_i8 ={{.*}} global [3 x <128 x i8>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_i16 ={{.*}} global [3 x <64 x i16>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_i32 ={{.*}} global [3 x <32 x i32>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_i64 ={{.*}} global [3 x <16 x i64>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_u8 ={{.*}} global [3 x <128 x i8>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_u16 ={{.*}} global [3 x <64 x i16>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_u32 ={{.*}} global [3 x <32 x i32>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_u64 ={{.*}} global [3 x <16 x i64>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_f32 ={{.*}} global [3 x <32 x float>] zeroinitializer, align 8
+// CHECK-NEXT-1024: @global_arr_f64 ={{.*}} global [3 x <16 x double>] zeroinitializer, align 8
+
+//===----------------------------------------------------------------------===//
+// Local variables
+//===----------------------------------------------------------------------===//
+// CHECK-64:      %local_i8 = alloca <8 x i8>, align 8
+// CHECK-64-NEXT: %local_i16 = alloca <4 x i16>, align 8
+// CHECK-64-NEXT: %local_i32 = alloca <2 x i32>, align 8
+// CHECK-64-NEXT: %local_i64 = alloca <1 x i64>, align 8
+// CHECK-64-NEXT: %local_u8 = alloca <8 x i8>, align 8
+// CHECK-64-NEXT: %local_u16 = alloca <4 x i16>, align 8
+// CHECK-64-NEXT: %local_u32 = alloca <2 x i32>, align 8
+// CHECK-64-NEXT: %local_u64 = alloca <1 x i64>, align 8
+// CHECK-64-NEXT: %local_f32 = alloca <2 x float>, align 8
+// CHECK-64-NEXT: %local_f64 = alloca <1 x double>, align 8
+
+// CHECK-128:      %local_i8 = alloca <16 x i8>, align 8
+// CHECK-128-NEXT: %local_i16 = alloca <8 x i16>, align 8
+// CHECK-128-NEXT: %local_i32 = alloca <4 x i32>, align 8
+// CHECK-128-NEXT: %local_i64 = alloca <2 x i64>, align 8
+// CHECK-128-NEXT: %local_u8 = alloca <16 x i8>, align 8
+// CHECK-128-NEXT: %local_u16 = alloca <8 x i16>, align 8
+// CHECK-128-NEXT: %local_u32 = alloca <4 x i32>, align 8
+// CHECK-128-NEXT: %local_u64 = alloca <2 x i64>, align 8
+// CHECK-128-NEXT: %local_f32 = alloca <4 x float>, align 8
+// CHECK-128-NEXT: %local_f64 = alloca <2 x double>, align 8
+
+// CHECK-256:      %local_i8 = alloca <32 x i8>, align 8
+// CHECK-256-NEXT: %local_i16 = alloca <16 x i16>, align 8
+// CHECK-256-NEXT: %local_i32 = alloca <8 x i32>, align 8
+// CHECK-256-NEXT: %local_i64 = alloca <4 x i64>, align 8
+// CHECK-256-NEXT: %local_u8 = alloca <32 x i8>, align 8
+// CHECK-256-NEXT: %local_u16 = alloca <16 x i16>, align 8
+// CHECK-256-NEXT: %local_u32 = alloca <8 x i32>, align 8
+// CHECK-256-NEXT: %local_u64 = alloca <4 x i64>, align 8
+// CHECK-256-NEXT: %local_f32 = alloca <8 x float>, align 8
+// CHECK-256-NEXT: %local_f64 = alloca <4 x double>, align 8
+
+// CHECK-512:      %local_i8 = alloca <64 x i8>, align 8
+// CHECK-512-NEXT: %local_i16 = alloca <32 x i16>, align 8
+// CHECK-512-NEXT: %local_i32 = alloca <16 x i32>, align 8
+// CHECK-512-NEXT: %local_i64 = alloca <8 x i64>, align 8
+// CHECK-512-NEXT: %local_u8 = alloca <64 x i8>, align 8
+// CHECK-512-NEXT: %local_u16 = alloca <32 x i16>, align 8
+// CHECK-512-NEXT: %local_u32 = alloca <16 x i32>, align 8
+// CHECK-512-NEXT: %local_u64 = alloca <8 x i64>, align 8
+// CHECK-512-NEXT: %local_f32 = alloca <16 x float>, align 8
+// CHECK-512-NEXT: %local_f64 = alloca <8 x double>, align 8
+
+// CHECK-1024:       %local_i8 = alloca <128 x i8>, align 8
+// CHECK-1024-NEXT:  %local_i16 = alloca <64 x i16>, align 8
+// CHECK-1024-NEXT:  %local_i32 = alloca <32 x i32>, align 8
+// CHECK-1024-NEXT:  %local_i64 = alloca <16 x i64>, align 8
+// CHECK-1024-NEXT:  %local_u8 = alloca <128 x i8>, align 8
+// CHECK-1024-NEXT:  %local_u16 = alloca <64 x i16>, align 8
+// CHECK-1024-NEXT:  %local_u32 = alloca <32 x i32>, align 8
+// CHECK-1024-NEXT:  %local_u64 = alloca <16 x i64>, align 8
+// CHECK-1024-NEXT:  %local_f32 = alloca <32 x float>, align 8
+// CHECK-1024-NEXT:  %local_f64 = alloca <16 x double>, align 8
+
+//===----------------------------------------------------------------------===//
+// Local arrays
+//===----------------------------------------------------------------------===//
+// CHECK-64:      %local_arr_i8 = alloca [3 x <8 x i8>], align 8
+// CHECK-64-NEXT: %local_arr_i16 = alloca [3 x <4 x i16>], align 8
+// CHECK-64-NEXT: %local_arr_i32 = alloca [3 x <2 x i32>], align 8
+// CHECK-64-NEXT: %local_arr_i64 = alloca [3 x <1 x i64>], align 8
+// CHECK-64-NEXT: %local_arr_u8 = alloca [3 x <8 x i8>], align 8
+// CHECK-64-NEXT: %local_arr_u16 = alloca [3 x <4 x i16>], align 8
+// CHECK-64-NEXT: %local_arr_u32 = alloca [3 x <2 x i32>], align 8
+// CHECK-64-NEXT: %local_arr_u64 = alloca [3 x <1 x i64>], align 8
+// CHECK-64-NEXT: %local_arr_f32 = alloca [3 x <2 x float>], align 8
+// CHECK-64-NEXT: %local_arr_f64 = alloca [3 x <1 x double>], align 8
+
+// CHECK-128:      %local_arr_i8 = alloca [3 x <16 x i8>], align 8
+// CHECK-128-NEXT: %local_arr_i16 = alloca [3 x <8 x i16>], align 8
+// CHECK-128-NEXT: %local_arr_i32 = alloca [3 x <4 x i32>], align 8
+// CHECK-128-NEXT: %local_arr_i64 = alloca [3 x <2 x i64>], align 8
+// CHECK-128-NEXT: %local_arr_u8 = alloca [3 x <16 x i8>], align 8
+// CHECK-128-NEXT: %local_arr_u16 = alloca [3 x <8 x i16>], align 8
+// CHECK-128-NEXT: %local_arr_u32 = alloca [3 x <4 x i32>], align 8
+// CHECK-128-NEXT: %local_arr_u64 = alloca [3 x <2 x i64>], align 8
+// CHECK-128-NEXT: %local_arr_f32 = alloca [3 x <4 x float>], align 8
+// CHECK-128-NEXT: %local_arr_f64 = alloca [3 x <2 x double>], align 8
+
+// CHECK-256:      %local_arr_i8 = alloca [3 x <32 x i8>], align 8
+// CHECK-256-NEXT: %local_arr_i16 = alloca [3 x <16 x i16>], align 8
+// CHECK-256-NEXT: %local_arr_i32 = alloca [3 x <8 x i32>], align 8
+// CHECK-256-NEXT: %local_arr_i64 = alloca [3 x <4 x i64>], align 8
+// CHECK-256-NEXT: %local_arr_u8 = alloca [3 x <32 x i8>], align 8
+// CHECK-256-NEXT: %local_arr_u16 = alloca [3 x <16 x i16>], align 8
+// CHECK-256-NEXT: %local_arr_u32 = alloca [3 x <8 x i32>], align 8
+// CHECK-256-NEXT: %local_arr_u64 = alloca [3 x <4 x i64>], align 8
+// CHECK-256-NEXT: %local_arr_f32 = alloca [3 x <8 x float>], align 8
+// CHECK-256-NEXT: %local_arr_f64 = alloca [3 x <4 x double>], align 8
+
+// CHECK-512:      %local_arr_i8 = alloca [3 x <64 x i8>], align 8
+// CHECK-512-NEXT: %local_arr_i16 = alloca [3 x <32 x i16>], align 8
+// CHECK-512-NEXT: %local_arr_i32 = alloca [3 x <16 x i32>], align 8
+// CHECK-512-NEXT: %local_arr_i64 = alloca [3 x <8 x i64>], align 8
+// CHECK-512-NEXT: %local_arr_u8 = alloca [3 x <64 x i8>], align 8
+// CHECK-512-NEXT: %local_arr_u16 = alloca [3 x <32 x i16>], align 8
+// CHECK-512-NEXT: %local_arr_u32 = alloca [3 x <16 x i32>], align 8
+// CHECK-512-NEXT: %local_arr_u64 = alloca [3 x <8 x i64>], align 8
+// CHECK-512-NEXT: %local_arr_f32 = alloca [3 x <16 x float>], align 8
+// CHECK-512-NEXT: %local_arr_f64 = alloca [3 x <8 x double>], align 8
+
+// CHECK-1024:       %local_arr_i8 = alloca [3 x <128 x i8>], align 8
+// CHECK-1024-NEXT:  %local_arr_i16 = alloca [3 x <64 x i16>], align 8
+// CHECK-1024-NEXT:  %local_arr_i32 = alloca [3 x <32 x i32>], align 8
+// CHECK-1024-NEXT:  %local_arr_i64 = alloca [3 x <16 x i64>], align 8
+// CHECK-1024-NEXT:  %local_arr_u8 = alloca [3 x <128 x i8>], align 8
+// CHECK-1024-NEXT:  %local_arr_u16 = alloca [3 x <64 x i16>], align 8
+// CHECK-1024-NEXT:  %local_arr_u32 = alloca [3 x <32 x i32>], align 8
+// CHECK-1024-NEXT:  %local_arr_u64 = alloca [3 x <16 x i64>], align 8
+// CHECK-1024-NEXT:  %local_arr_f32 = alloca [3 x <32 x float>], align 8
+// CHECK-1024-NEXT:  %local_arr_f64 = alloca [3 x <16 x double>], align 8
+
+//===----------------------------------------------------------------------===//
+// ILP32 ABI
+//===----------------------------------------------------------------------===//
+// CHECK-ILP32: @global_i32 ={{.*}} global <16 x i32> zeroinitializer, align 8
+// CHECK-ILP32: @global_i64 ={{.*}} global <8 x i64> zeroinitializer, align 8
+// CHECK-ILP32: @global_u32 ={{.*}} global <16 x i32> zeroinitializer, align 8
+// CHECK-ILP32: @global_u64 ={{.*}} global <8 x i64> zeroinitializer, align 8
diff --git a/clang/test/CodeGen/builtins-nondeterministic-value.c b/clang/test/CodeGen/builtins-nondeterministic-value.c
index 5b22902de2e7d..aa040edf73d98 100644
--- a/clang/test/CodeGen/builtins-nondeterministic-value.c
+++ b/clang/test/CodeGen/builtins-nondeterministic-value.c
@@ -52,7 +52,7 @@ void clang_nondet_fv( ) {
 void clang_nondet_bv( ) {
 // CHECK: [[A:%.*]] = alloca i8, align
 // CHECK: [[V:%.*]] = freeze <4 x i1> poison
-// CHECK: [[SV:%.*]] = shufflevector <4 x i1> [[V]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+// CHECK: [[SV:%.*]] = shufflevector <4 x i1> [[V]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 // CHECK: [[BC:%.*]] = bitcast <8 x i1> [[SV]] to i8
 // CHECK: store i8 [[BC]], ptr [[A]], align
 // CHECK: ret void
diff --git a/clang/test/CodeGen/builtinshufflevector2.c b/clang/test/CodeGen/builtinshufflevector2.c
index 10e8fe19f9b77..da957503f86b6 100644
--- a/clang/test/CodeGen/builtinshufflevector2.c
+++ b/clang/test/CodeGen/builtinshufflevector2.c
@@ -34,9 +34,9 @@ void clang_shufflevector_v_v_c( float4* A, float4 x, float4 y) {
   *A = __builtin_shufflevector( x, y, 0, 4, 1, 5 );
 }
 
-// CHECK-LABEL: define {{.*}}void @clang_shufflevector_v_v_undef(
-void clang_shufflevector_v_v_undef( float4* A, float4 x, float4 y) {
-// CHECK: [[V:%.*]] = shufflevector <4 x float> {{%.*}}, <4 x float> {{%.*}}, <4 x i32> <i32 0, i32 4, i32 undef, i32 5>
+// CHECK-LABEL: define {{.*}}void @clang_shufflevector_v_v_poison(
+void clang_shufflevector_v_v_poison( float4* A, float4 x, float4 y) {
+// CHECK: [[V:%.*]] = shufflevector <4 x float> {{%.*}}, <4 x float> {{%.*}}, <4 x i32> <i32 0, i32 4, i32 poison, i32 5>
 // CHECK: store <4 x float> [[V]], ptr {{%.*}}
   *A = __builtin_shufflevector( x, y, 0, 4, -1, 5 );
 }
diff --git a/clang/test/CodeGen/debug-prefix-map.c b/clang/test/CodeGen/debug-prefix-map.c
index f604f1643ba02..e2421806b055b 100644
--- a/clang/test/CodeGen/debug-prefix-map.c
+++ b/clang/test/CodeGen/debug-prefix-map.c
@@ -9,6 +9,10 @@
 // RUN: %clang -g -fdebug-prefix-map=%p=./UNLIKELY_PATH/empty -S -c %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-REL
 // RUN: %clang -g -ffile-prefix-map=%p=./UNLIKELY_PATH/empty -S -c %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-REL
 
+// RUN: rm -rf %t && mkdir -p %t/a/b && cp %s %t/a/b/c.c
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=standalone -I%S -fdebug-prefix-map=%t/a/b=y -fdebug-prefix-map=%t/a=x %t/a/b/c.c -o - | FileCheck %s --check-prefix=CHECK-X
+// RUN: %clang_cc1 -emit-llvm -debug-info-kind=standalone -I%S -fdebug-prefix-map=%t/a=x -fdebug-prefix-map=%t/a/b=y %t/a/b/c.c -o - | FileCheck %s --check-prefix=CHECK-Y
+
 #include "Inputs/stdio.h"
 
 int main(int argc, char **argv) {
@@ -47,3 +51,6 @@ void test_rewrite_includes(void) {
 // CHECK-REL: !DIFile(filename: "./UNLIKELY_PATH/empty{{/|\\\\}}{{.*}}",
 // CHECK-REL: !DIFile(filename: "./UNLIKELY_PATH/empty{{/|\\\\}}{{.*}}Inputs/stdio.h",
 // CHECK-REL-SAME:    directory: ""
+
+// CHECK-X: !DIFile(filename: "x{{/|\\\\}}b{{/|\\\\}}c.c", directory: "")
+// CHECK-Y: !DIFile(filename: "y{{/|\\\\}}c.c", directory: "")
diff --git a/clang/test/CodeGen/debug-prefix-map.cpp b/clang/test/CodeGen/debug-prefix-map.cpp
new file mode 100644
index 0000000000000..5e90aedd8ed73
--- /dev/null
+++ b/clang/test/CodeGen/debug-prefix-map.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -debug-info-kind=standalone -fdebug-prefix-map=%p=./UNLIKELY_PATH/empty -S %s -emit-llvm -o - | FileCheck %s
+
+struct alignas(64) an {
+  struct {
+    unsigned char x{0};
+  } arr[64];
+};
+
+struct an *pan = new an;
+
+// CHECK: !DISubprogram(name: "(unnamed struct at ./UNLIKELY_PATH/empty{{/|\\\\}}{{.*}}",
diff --git a/clang/test/CodeGen/denormalfpmode-f32.c b/clang/test/CodeGen/denormalfpmode-f32.c
index f89202bcc9ed8..2e2306f5a3478 100644
--- a/clang/test/CodeGen/denormalfpmode-f32.c
+++ b/clang/test/CodeGen/denormalfpmode-f32.c
@@ -2,21 +2,30 @@
 // RUN: %clang_cc1 -S -fdenormal-fp-math=ieee %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-NONE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=preserve-sign %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PS,CHECK-F32-NONE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PZ,CHECK-F32-NONE
+// RUN: %clang_cc1 -S -fdenormal-fp-math=dynamic %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-DYNAMIC,CHECK-F32-NONE
 
 // RUN: %clang_cc1 -S -fdenormal-fp-math-f32=ieee %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-NONE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=ieee -fdenormal-fp-math-f32=ieee %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-NONE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=preserve-sign -fdenormal-fp-math-f32=ieee %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PS,CHECK-F32-IEEE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=positive-zero -fdenormal-fp-math-f32=ieee %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PZ,CHECK-F32-IEEE
+// RUN: %clang_cc1 -S -fdenormal-fp-math=positive-zero -fdenormal-fp-math-f32=dynamic %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PZ,CHECK-F32-DYNAMIC
+
 
 // RUN: %clang_cc1 -S -fdenormal-fp-math-f32=preserve-sign %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-PS
 // RUN: %clang_cc1 -S -fdenormal-fp-math=ieee -fdenormal-fp-math-f32=preserve-sign %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-PS
 // RUN: %clang_cc1 -S -fdenormal-fp-math=preserve-sign -fdenormal-fp-math-f32=preserve-sign %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PS,CHECK-F32-NONE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=positive-zero -fdenormal-fp-math-f32=preserve-sign %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PZ,CHECK-F32-PS
+// RUN: %clang_cc1 -S -fdenormal-fp-math=ieee -fdenormal-fp-math-f32=dynamic %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-DYNAMIC
+
 
 // RUN: %clang_cc1 -S -fdenormal-fp-math-f32=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-PZ
+// RUN: %clang_cc1 -S -fdenormal-fp-math-f32=dynamic %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-DYNAMIC
 // RUN: %clang_cc1 -S -fdenormal-fp-math=ieee -fdenormal-fp-math-f32=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-NONE,CHECK-F32-PZ
+// RUN: %clang_cc1 -S -fdenormal-fp-math=dynamic -fdenormal-fp-math-f32=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-DYNAMIC,CHECK-F32-PZ
 // RUN: %clang_cc1 -S -fdenormal-fp-math=preserve-sign -fdenormal-fp-math-f32=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PS,CHECK-F32-PZ
 // RUN: %clang_cc1 -S -fdenormal-fp-math=positive-zero -fdenormal-fp-math-f32=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-PZ,CHECK-F32-NONE
+// RUN: %clang_cc1 -S -fdenormal-fp-math=dynamic -fdenormal-fp-math-f32=dynamic %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK-ATTR,CHECK-DYNAMIC,CHECK-F32-NONE
+
 
 // CHECK-LABEL: main
 
@@ -25,11 +34,16 @@
 // CHECK-IEEE: "denormal-fp-math"="ieee,ieee"
 // CHECK-PS: "denormal-fp-math"="preserve-sign,preserve-sign"
 // CHECK-PZ: "denormal-fp-math"="positive-zero,positive-zero"
+// CHECK-DYNAMIC: "denormal-fp-math"="dynamic,dynamic"
+
 // CHECK-F32-NONE-NOT:"denormal-fp-math-f32"
 // CHECK-F32-IEEE: "denormal-fp-math-f32"="ieee,ieee"
 // CHECK-F32-PS: "denormal-fp-math-f32"="preserve-sign,preserve-sign"
 // CHECK-F32-PZ: "denormal-fp-math-f32"="positive-zero,positive-zero"
 
+
+// CHECK-F32-DYNAMIC: "denormal-fp-math-f32"="dynamic,dynamic"
+
 int main(void) {
   return 0;
 }
diff --git a/clang/test/CodeGen/denormalfpmode.c b/clang/test/CodeGen/denormalfpmode.c
index 6213454a675dc..36f25038ce2bc 100644
--- a/clang/test/CodeGen/denormalfpmode.c
+++ b/clang/test/CodeGen/denormalfpmode.c
@@ -1,6 +1,7 @@
 // RUN: %clang_cc1 -S -fdenormal-fp-math=ieee %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-IEEE
 // RUN: %clang_cc1 -S -fdenormal-fp-math=preserve-sign %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-PS
 // RUN: %clang_cc1 -S -fdenormal-fp-math=positive-zero %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-PZ
+// RUN: %clang_cc1 -S -fdenormal-fp-math=dynamic %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK-DYNAMIC
 
 // CHECK-LABEL: main
 
@@ -8,6 +9,7 @@
 // CHECK-IEEE-NOT:"denormal-fp-math"
 // CHECK-PS: attributes #0 = {{.*}}"denormal-fp-math"="preserve-sign,preserve-sign"{{.*}}
 // CHECK-PZ: attributes #0 = {{.*}}"denormal-fp-math"="positive-zero,positive-zero"{{.*}}
+// CHECK-DYNAMIC: attributes #0 = {{.*}}"denormal-fp-math"="dynamic,dynamic"{{.*}}
 
 int main(void) {
   return 0;
diff --git a/clang/test/CodeGen/ext-int.c b/clang/test/CodeGen/ext-int.c
index 8112d9b044a51..4cb399d108f29 100644
--- a/clang/test/CodeGen/ext-int.c
+++ b/clang/test/CodeGen/ext-int.c
@@ -3,6 +3,10 @@
 // RUN: %clang_cc1 -triple i386-gnu-linux -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,LIN32
 // RUN: %clang_cc1 -triple i386-windows-pc -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WIN32
 
+//GH62207
+unsigned _BitInt(1) GlobSize1 = 0;
+// CHECK: @GlobSize1 = {{.*}}global i1 false
+
 void GenericTest(_BitInt(3) a, unsigned _BitInt(3) b, _BitInt(4) c) {
   // CHECK: define {{.*}}void @GenericTest
   int which = _Generic(a, _BitInt(3): 1, unsigned _BitInt(3) : 2, _BitInt(4) : 3);
diff --git a/clang/test/CodeGen/ext-vector.c b/clang/test/CodeGen/ext-vector.c
index 2fce096e2531a..a3edabf0fc0d4 100644
--- a/clang/test/CodeGen/ext-vector.c
+++ b/clang/test/CodeGen/ext-vector.c
@@ -235,7 +235,7 @@ int test11(void) {
 
 // CHECK: @test12
 // CHECK: shufflevector {{.*}} <i32 2, i32 1, i32 0>
-// CHECK: shufflevector {{.*}} <i32 0, i32 1, i32 2, i32 undef>
+// CHECK: shufflevector {{.*}} <i32 0, i32 1, i32 2, i32 poison>
 // CHECK: shufflevector {{.*}} <i32 4, i32 5, i32 6, i32 3>
 int4 test12(int4 V) {
   V.xyz = V.zyx;
@@ -328,7 +328,7 @@ void test_rgba(void) {
   // CHECK: extractelement {{.*}} 0
   f = vec4_2.rg.r;
   // CHECK: shufflevector {{.*}} <i32 2, i32 1, i32 0>
-  // CHECK: shufflevector {{.*}} <i32 0, i32 1, i32 2, i32 undef>
+  // CHECK: shufflevector {{.*}} <i32 0, i32 1, i32 2, i32 poison>
   // CHECK: shufflevector {{.*}} <i32 4, i32 5, i32 6, i32 3>
   vec4.rgb = vec4.bgr;
 
diff --git a/clang/test/CodeGen/inline-asm-aarch64-flag-output.c b/clang/test/CodeGen/inline-asm-aarch64-flag-output.c
new file mode 100644
index 0000000000000..b1ef6e3883754
--- /dev/null
+++ b/clang/test/CodeGen/inline-asm-aarch64-flag-output.c
@@ -0,0 +1,130 @@
+// RUN: %clang_cc1 -O2 -emit-llvm %s -o - -triple aarch64 | FileCheck %s
+
+int test_cceq(int a, int* b) {
+// CHECK-LABEL: @test_cceq
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cceq},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cceq"(*b));
+  return a;
+}
+
+int test_ccne(int a, int* b) {
+// CHECK-LABEL: @test_ccne
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccne},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccne"(*b));
+  return a;
+}
+
+int test_cccs(int a, int* b) {
+// CHECK-LABEL: @test_cccs
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cccs},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cccs"(*b));
+  return a;
+}
+
+int test_cchs(int a, int* b) {
+// CHECK-LABEL: @test_cchs
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cchs},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cchs"(*b));
+  return a;
+}
+
+int test_cccc(int a, int* b) {
+// CHECK-LABEL: @test_cccc
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cccc},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cccc"(*b));
+  return a;
+}
+
+int test_cclo(int a, int* b) {
+// CHECK-LABEL: @test_cclo
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cclo},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cclo"(*b));
+  return a;
+}
+
+int test_ccmi(int a, int* b) {
+// CHECK-LABEL: @test_ccmi
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccmi},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccmi"(*b));
+  return a;
+}
+
+int test_ccpl(int a, int* b) {
+// CHECK-LABEL: @test_ccpl
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccpl},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccpl"(*b));
+  return a;
+}
+
+int test_ccvs(int a, int* b) {
+// CHECK-LABEL: @test_ccvs
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccvs},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccvs"(*b));
+  return a;
+}
+
+int test_ccvc(int a, int* b) {
+// CHECK-LABEL: @test_ccvc
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccvc},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccvc"(*b));
+  return a;
+}
+
+int test_cchi(int a, int* b) {
+// CHECK-LABEL: @test_cchi
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cchi},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cchi"(*b));
+  return a;
+}
+
+int test_ccls(int a, int* b) {
+// CHECK-LABEL: @test_ccls
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccls},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccls"(*b));
+  return a;
+}
+
+
+int test_ccge(int a, int* b) {
+// CHECK-LABEL: @test_ccge
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccge},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccge"(*b));
+  return a;
+}
+
+int test_cclt(int a, int* b) {
+// CHECK-LABEL: @test_cclt
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@cclt},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@cclt"(*b));
+  return a;
+}
+
+int test_ccgt(int a, int* b) {
+// CHECK-LABEL: @test_ccgt
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccgt},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccgt"(*b));
+  return a;
+}
+
+int test_ccle(int a, int* b) {
+// CHECK-LABEL: @test_ccle
+// CHECK: = tail call { i32, i32 } asm "ands ${0:w}, ${0:w}, #3", "=r,={@ccle},0"(i32 %a)
+  asm("ands %w[a], %w[a], #3"
+      : [a] "+r"(a), "=@ccle"(*b));
+  return a;
+}
diff --git a/clang/test/CodeGen/mozilla-ms-inline-asm.c b/clang/test/CodeGen/mozilla-ms-inline-asm.c
index f01e9200852f3..2e379c1bb8a23 100644
--- a/clang/test/CodeGen/mozilla-ms-inline-asm.c
+++ b/clang/test/CodeGen/mozilla-ms-inline-asm.c
@@ -27,7 +27,7 @@ void invoke(void* that, unsigned methodIndex,
 // CHECK-SAME: sub esp,eax
 // CHECK-SAME: mov ecx,esp
 // CHECK-SAME: push $0
-// CHECK-SAME: call dword ptr ${2:P}
+// CHECK-SAME: call ${2:P}
 // CHECK-SAME: {{.*}}__MSASMLABEL_.${:uid}__noparams:
 // CHECK-SAME: mov ecx,$3
 // CHECK-SAME: push ecx
diff --git a/clang/test/CodeGen/ms-inline-asm-64.c b/clang/test/CodeGen/ms-inline-asm-64.c
index 7c9a9f2e2bb2e..d688c10ca7080 100644
--- a/clang/test/CodeGen/ms-inline-asm-64.c
+++ b/clang/test/CodeGen/ms-inline-asm-64.c
@@ -68,7 +68,7 @@ void t5(void) {
   }
   // CHECK: t5
   // CHECK: call void asm sideeffect inteldialect
-  // CHECK-SAME: call qword ptr ${0:P}
-  // CHECK-SAME: jmp qword ptr ${1:P}
+  // CHECK-SAME: call ${0:P}
+  // CHECK-SAME: jmp ${1:P}
   // CHECK-SAME: "*m,*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(void (...)) @bar, ptr elementtype(void (...)) @bar)
 }
diff --git a/clang/test/CodeGen/ms-inline-asm.c b/clang/test/CodeGen/ms-inline-asm.c
index 32e9fb0d47139..9498be1034768 100644
--- a/clang/test/CodeGen/ms-inline-asm.c
+++ b/clang/test/CodeGen/ms-inline-asm.c
@@ -312,7 +312,7 @@ void t24_helper(void) {}
 void t24(void) {
   __asm call t24_helper
 // CHECK: t24
-// CHECK: call void asm sideeffect inteldialect "call dword ptr ${0:P}", "*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(void ()) @t24_helper)
+// CHECK: call void asm sideeffect inteldialect "call ${0:P}", "*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(void ()) @t24_helper)
 }
 
 void t25(void) {
@@ -684,7 +684,7 @@ void dot_operator(void){
 void call_clobber(void) {
   __asm call t41
   // CHECK-LABEL: define{{.*}} void @call_clobber
-  // CHECK: call void asm sideeffect inteldialect "call dword ptr ${0:P}", "*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(void (i16)) @t41)
+  // CHECK: call void asm sideeffect inteldialect "call ${0:P}", "*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(void (i16)) @t41)
 }
 
 void xgetbv(void) {
diff --git a/clang/test/CodeGen/ms-inline-asm.cpp b/clang/test/CodeGen/ms-inline-asm.cpp
index da55176071247..a1fb19f3f640c 100644
--- a/clang/test/CodeGen/ms-inline-asm.cpp
+++ b/clang/test/CodeGen/ms-inline-asm.cpp
@@ -109,7 +109,7 @@ void test5() {
   __asm mov x, eax
   // CHECK: call void asm sideeffect inteldialect
   // CHECK-SAME: push $0
-  // CHECK-SAME: call dword ptr ${2:P}
+  // CHECK-SAME: call ${2:P}
   // CHECK-SAME: mov $1, eax
   // CHECK-SAME: "=*m,=*m,*m,~{esp},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i32) %y, ptr elementtype(i32) %x, ptr elementtype(i32 (float)) @_ZN2T5IiE6createIfEEiT_)
 }
diff --git a/clang/test/CodeGen/ms_this.cpp b/clang/test/CodeGen/ms_this.cpp
index 655fcdf9ab549..24d134b3404be 100644
--- a/clang/test/CodeGen/ms_this.cpp
+++ b/clang/test/CodeGen/ms_this.cpp
@@ -20,7 +20,7 @@ void t2::runc() {
       mov rax,[this]
       // CHECK: [[THIS_ADDR_T2:%.+]] = alloca ptr
       // CHECK: [[THIS1_T2:%.+]] = load ptr, ptr [[THIS_ADDR_T2]],
-      // CHECK: call void asm sideeffect inteldialect "mov rax,qword ptr $1{{.*}}ptr [[THIS1_T2]]
+      // CHECK: call void asm sideeffect inteldialect "mov rax,$1\0A\09mov rbx,[rax]\0A\09mov $0, rbx", "=*m,m,~{rax},~{rbx},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(double) %num, ptr [[THIS1_T2]])
       mov rbx,[rax]
       mov num, rbx
 	   };
@@ -33,7 +33,7 @@ void t1::runc() {
        mov rax,[this]
        // CHECK: [[THIS_ADDR_T1:%.+]] = alloca ptr
        // CHECK: [[THIS1_T1:%.+]] = load ptr, ptr [[THIS_ADDR_T1]],
-       // CHECK: call void asm sideeffect inteldialect "mov rax,qword ptr $1{{.*}}ptr [[THIS1_T1]]
+       // CHECK: call void asm sideeffect inteldialect "mov rax,$1{{.*}}ptr [[THIS1_T1]]
         mov rbx,[rax]
         mov num, rbx
 	   };
@@ -46,7 +46,7 @@ struct s {
     __asm mov rax, [this]
     // CHECK: [[THIS_ADDR_S:%.+]] = alloca ptr
     // CHECK: [[THIS1_S:%.+]] = load ptr, ptr [[THIS_ADDR_S]],
-    // CHECK: call void asm sideeffect inteldialect "mov rax, qword ptr $0{{.*}}ptr [[THIS1_S]]
+    // CHECK: call void asm sideeffect inteldialect "mov rax, $0{{.*}}ptr [[THIS1_S]]
   }
 } f3;
 
diff --git a/clang/test/CodeGen/nullptr.c b/clang/test/CodeGen/nullptr.c
new file mode 100644
index 0000000000000..7ea951213df0e
--- /dev/null
+++ b/clang/test/CodeGen/nullptr.c
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -S %s -std=c2x -emit-llvm -o - | FileCheck %s
+
+// Test that null <-> nullptr_t conversions work as expected.
+typedef typeof(nullptr) nullptr_t;
+
+nullptr_t nullptr_t_val;
+
+void bool_func(bool);
+void nullptr_func(nullptr_t);
+
+void test() {
+  // Test initialization
+  bool bool_from_nullptr_t = nullptr_t_val;
+  nullptr_t nullptr_t_from_nullptr = nullptr;
+  void *vp_from_nullptr_t = nullptr_t_val;
+  nullptr_t nullptr_t_from_vp = (void *)0;
+  nullptr_t nullptr_t_from_int = 0;
+
+  // Test assignment
+  bool_from_nullptr_t = nullptr_t_val;
+  nullptr_t_from_nullptr = nullptr;
+  vp_from_nullptr_t = nullptr_t_val;
+  nullptr_t_from_vp = (void *)0;
+  nullptr_t_from_int = 0;
+
+  // Test calls
+  bool_func(nullptr_t_from_nullptr);
+  nullptr_func(nullptr_t_from_nullptr);
+  nullptr_func(0);
+  nullptr_func((void *)0);
+  nullptr_func(nullptr);
+  nullptr_func(false);
+
+  // Allocation of locals
+  // CHECK: %[[bool_from_nullptr_t:.*]] = alloca i8
+  // CHECK: %[[nullptr_t_from_nullptr:.*]] = alloca ptr
+  // CHECK: %[[vp_from_nullptr_t:.*]] = alloca ptr
+  // CHECK: %[[nullptr_t_from_vp:.*]] = alloca ptr
+  // CHECK: %[[nullptr_t_from_int:.*]] = alloca ptr
+
+  // Initialization of locals
+  // CHECK: store i8 0, ptr %[[bool_from_nullptr_t]]
+  // CHECK: store ptr null, ptr %[[nullptr_t_from_nullptr]]
+  // CHECK: store ptr null, ptr %[[vp_from_nullptr_t]]
+  // CHECK: store ptr null, ptr %[[nullptr_t_from_vp]]
+  // CHECK: store ptr null, ptr %[[nullptr_t_from_int]]
+
+  // Assignment expressions
+  // CHECK: store i8 0, ptr %[[bool_from_nullptr_t]]
+  // CHECK: store ptr null, ptr %[[nullptr_t_from_nullptr]]
+  // CHECK: store ptr null, ptr %[[vp_from_nullptr_t]]
+  // CHECK: store ptr null, ptr %[[nullptr_t_from_vp]]
+  // CHECK: store ptr null, ptr %[[nullptr_t_from_int]]
+
+  // Calls
+  // CHECK: call void @bool_func(i1 noundef {{(zeroext )?}}false)
+  // CHECK: call void @nullptr_func(ptr null)
+  // CHECK: call void @nullptr_func(ptr null)
+  // CHECK: call void @nullptr_func(ptr null)
+  // CHECK: call void @nullptr_func(ptr null)
+  // CHECK: call void @nullptr_func(ptr null)
+}
+
diff --git a/clang/test/CodeGen/paren-list-agg-init.cpp b/clang/test/CodeGen/paren-list-agg-init.cpp
index a860196d8f500..0e68beb5c3706 100644
--- a/clang/test/CodeGen/paren-list-agg-init.cpp
+++ b/clang/test/CodeGen/paren-list-agg-init.cpp
@@ -90,6 +90,23 @@ namespace gh61145 {
   };
 }
 
+namespace gh62266 {
+  // CHECK-DAG: [[STRUCT_H:%.*H.*]] = type { i32, i32 }
+  template <int J>
+  struct H {
+    int i;
+    int j = J;
+  };
+}
+
+namespace gh61567 {
+  // CHECK-DAG: [[STRUCT_I:%.*I.*]] = type { i32, ptr }
+  struct I {
+    int a;
+    int&& r = 2;
+  };
+}
+
 // CHECK-DAG: [[A1:@.*a1.*]] = internal constant [[STRUCT_A]] { i8 3, double 2.000000e+00 }, align 8
 constexpr A a1(3.1, 2.0);
 // CHECK-DAG: [[A2:@.*a2.*]] = internal constant [[STRUCT_A]] { i8 99, double 0.000000e+00 }, align 8
@@ -421,3 +438,78 @@ namespace gh61145 {
     make2<0>();
   }
 }
+
+namespace gh62266 {
+  // CHECK: define {{.*}} void {{.*foo20.*}}
+  // CHECK-NEXT: entry:
+  // CHECK-NEXT: [[H:%.*h.*]] = alloca [[STRUCT_H]], align 4
+  // CHECK-NEXT: [[I:%.*i.*]] = getelementptr inbounds [[STRUCT_H]], ptr [[H]], i32 0, i32 0
+  // CHECK-NEXT: store i32 1, ptr [[I]], align 4
+  // CHECK-NEXT: [[J:%.*j.*]] = getelementptr inbounds [[STRUCT_H]], ptr [[H]], i32 0, i32 1
+  // CHECK-NEXT: store i32 2, ptr [[J]], align 4
+  // CHECK-NEXT: ret void
+  void foo20() {
+    H<2> h(1);
+  }
+}
+
+namespace gh61567 {
+  int foo20();
+
+  // CHECK: define {{.*}} void @{{.*foo21.*}} {
+  // CHECK-NEXT: entry
+  // CHECK-NEXT: [[AGG_TMP_ENSURED:%.*]] = alloca [[STRUCT_I]], align 8
+  // CHECK-NEXT: [[REF_TMP:%.*]] = alloca i32, align 4
+  // CHECK-NEXT: [[A:%.*a.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 0
+  // CHECK-NEXT: store i32 0, ptr [[A]], align 8
+  // CHECK-NEXT: [[R:%.*r.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 1
+  // CHECK-NEXT: store i32 1, ptr [[REF_TMP]], align 4
+  // CHECK-NEXT: store ptr [[REF_TMP]], ptr [[R]], align 8
+  // CHECK-NEXT: ret void
+  void foo21() {
+    I(0, 1);
+  }
+
+  // CHECK: define {{.*}} void @{{.*foo22.*}} {
+  // CHECK-NEXT: entry
+  // CHECK-NEXT: [[AGG_TMP_ENSURED:%.*]] = alloca [[STRUCT_I]], align 8
+  // CHECK-NEXT: [[REF_TMP:%.*]] = alloca i32, align 4
+  // CHECK-NEXT: [[A:%.*a.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 0
+  // CHECK-NEXT: store i32 0, ptr [[A]], align 8
+  // CHECK-NEXT: [[R:%.*r.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 1
+  // CHECK-NEXT: [[CALL:%.*call*]] = call noundef i32 @{{.*foo20.*}}
+  // CHECK-NEXT: store i32 [[CALL]], ptr [[REF_TMP]], align 4
+  // CHECK-NEXT: store ptr [[REF_TMP]], ptr [[R]], align 8
+  // CHECK-NEXT: ret void
+  void foo22() {
+    I(0, foo20());
+  }
+
+  // CHECK: define {{.*}} void @{{.*foo23.*}}(i32 noundef [[I:%.*i.*]])
+  // CHECK-NEXT: entry
+  // CHECK-NEXT: [[I_ADDR:%.*i.*]] = alloca i32, align 4
+  // CHECK-NEXT: [[AGG_TMP_ENSURED:%.*]] = alloca [[STRUCT_I]], align 8
+  // CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
+  // CHECK-NEXT: [[A:%.*a.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 0
+  // CHECK-NEXT: store i32 0, ptr [[A]], align 8
+  // CHECK-NEXT: [[R:%.*r.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 1
+  // CHECK-NEXT: store ptr [[I_ADDR]], ptr [[R]], align 8
+  // CHECK-NEXT: ret void
+  void foo23(int i) {
+    I(0, static_cast<int&&>(i));
+  }
+
+  // CHECK: define {{.*}} void @{{.*foo24.*}} {
+  // CHECK-NEXT: entry
+  // CHECK-NEXT: [[AGG_TMP_ENSURED:%.*]] = alloca [[STRUCT_I]], align 8
+  // CHECK-NEXT: [[REF_TMP:%.*]] = alloca i32, align 4
+  // CHECK-NEXT: [[A:%.*a.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 0
+  // CHECK-NEXT: store i32 0, ptr [[A]], align 8
+  // CHECK-NEXT: [[R:%.*r.*]] = getelementptr inbounds [[STRUCT_I]], ptr [[AGG_TMP_ENSURED]], i32 0, i32 1
+  // CHECK-NEXT: store i32 2, ptr [[REF_TMP]], align 4
+  // CHECK-NEXT: store ptr [[REF_TMP]], ptr [[R]], align 8
+  // CHECK-NEXT: ret void
+  void foo24() {
+    I(0);
+  }
+}
diff --git a/clang/test/CodeGen/riscv-rvv-vls-arith-ops.c b/clang/test/CodeGen/riscv-rvv-vls-arith-ops.c
new file mode 100644
index 0000000000000..a090e8b9bb958
--- /dev/null
+++ b/clang/test/CodeGen/riscv-rvv-vls-arith-ops.c
@@ -0,0 +1,1819 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +zve64d \
+// RUN: -target-feature +f -target-feature +d -disable-O0-optnone \
+// RUN: -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: opt -S -passes=sroa | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+// ADDITION
+
+// CHECK-LABEL: @add_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t add_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t add_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t add_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t add_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t add_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t add_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t add_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t add_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t add_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t add_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_inplace_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t add_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t add_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t add_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t add_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t add_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t add_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t add_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t add_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t add_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_inplace_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t add_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a += b;
+}
+
+// CHECK-LABEL: @add_scalar_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t add_scalar_i8(fixed_int8m1_t a, int8_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t add_scalar_i16(fixed_int16m1_t a, int16_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t add_scalar_i32(fixed_int32m1_t a, int32_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t add_scalar_i64(fixed_int64m1_t a, int64_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t add_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t add_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t add_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t add_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = fadd <8 x float> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t add_scalar_f32(fixed_float32m1_t a, float b) {
+  return a + b;
+}
+
+// CHECK-LABEL: @add_scalar_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x double> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[ADD]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t add_scalar_f64(fixed_float64m1_t a, double b) {
+  return a + b;
+}
+
+// SUBTRACTION
+
+// CHECK-LABEL: @sub_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t sub_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t sub_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t sub_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t sub_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t sub_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t sub_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t sub_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t sub_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t sub_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = fsub <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t sub_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t sub_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t sub_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t sub_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t sub_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t sub_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t sub_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t sub_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t sub_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t sub_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_inplace_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SUB:%.*]] = fsub <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t sub_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t sub_scalar_i8(fixed_int8m1_t a, int8_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t sub_scalar_i16(fixed_int16m1_t a, int16_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t sub_scalar_i32(fixed_int32m1_t a, int32_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t sub_scalar_i64(fixed_int64m1_t a, int64_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t sub_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t sub_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t sub_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t sub_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = fsub <8 x float> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t sub_scalar_f32(fixed_float32m1_t a, float b) {
+  return a - b;
+}
+
+// CHECK-LABEL: @sub_scalar_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SUB:%.*]] = fsub <4 x double> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[SUB]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t sub_scalar_f64(fixed_float64m1_t a, double b) {
+  return a - b;
+}
+
+// MULTIPLICATION
+
+// CHECK-LABEL: @mul_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t mul_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t mul_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t mul_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t mul_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t mul_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t mul_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t mul_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t mul_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t mul_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t mul_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t mul_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t mul_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t mul_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t mul_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t mul_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t mul_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t mul_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t mul_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t mul_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_inplace_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t mul_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t mul_scalar_i8(fixed_int8m1_t a, int8_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t mul_scalar_i16(fixed_int16m1_t a, int16_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t mul_scalar_i32(fixed_int32m1_t a, int32_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t mul_scalar_i64(fixed_int64m1_t a, int64_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t mul_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t mul_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t mul_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t mul_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = fmul <8 x float> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t mul_scalar_f32(fixed_float32m1_t a, float b) {
+  return a * b;
+}
+
+// CHECK-LABEL: @mul_scalar_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[MUL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t mul_scalar_f64(fixed_float64m1_t a, double b) {
+  return a * b;
+}
+
+// DIVISION
+
+// CHECK-LABEL: @div_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t div_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t div_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t div_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t div_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t div_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t div_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t div_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t div_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t div_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t div_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t div_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t div_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t div_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t div_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t div_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t div_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t div_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t div_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t div_inplace_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_inplace_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t div_inplace_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t div_scalar_i8(fixed_int8m1_t a, int8_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t div_scalar_i16(fixed_int16m1_t a, int16_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t div_scalar_i32(fixed_int32m1_t a, int32_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = sdiv <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t div_scalar_i64(fixed_int64m1_t a, int64_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t div_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t div_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t div_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = udiv <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t div_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x float> [[SPLAT_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv <8 x float> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v8f32(<vscale x 2 x float> undef, <8 x float> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x float> [[CASTSCALABLESVE]]
+//
+fixed_float32m1_t div_scalar_f32(fixed_float32m1_t a, float b) {
+  return a / b;
+}
+
+// CHECK-LABEL: @div_scalar_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[DIV:%.*]] = fdiv <4 x double> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[DIV]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+//
+fixed_float64m1_t div_scalar_f64(fixed_float64m1_t a, double b) {
+  return a / b;
+}
+
+// REMAINDER
+
+// CHECK-LABEL: @rem_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t rem_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t rem_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t rem_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t rem_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t rem_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t rem_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t rem_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t rem_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t rem_inplace_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t rem_inplace_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t rem_inplace_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = srem <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t rem_inplace_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t rem_inplace_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t rem_inplace_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t rem_inplace_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_inplace_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[REM:%.*]] = urem <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t rem_inplace_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = srem <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t rem_scalar_i8(fixed_int8m1_t a, int8_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = srem <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t rem_scalar_i16(fixed_int16m1_t a, int16_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = srem <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t rem_scalar_i32(fixed_int32m1_t a, int32_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = srem <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t rem_scalar_i64(fixed_int64m1_t a, int64_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = urem <32 x i8> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t rem_scalar_u8(fixed_uint8m1_t a, uint8_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = urem <16 x i16> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t rem_scalar_u16(fixed_uint16m1_t a, uint16_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = urem <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t rem_scalar_u32(fixed_uint32m1_t a, uint32_t b) {
+  return a % b;
+}
+
+// CHECK-LABEL: @rem_scalar_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[REM:%.*]] = urem <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[REM]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t rem_scalar_u64(fixed_uint64m1_t a, uint64_t b) {
+  return a % b;
+}
diff --git a/clang/test/CodeGen/riscv-rvv-vls-bitwise-ops.c b/clang/test/CodeGen/riscv-rvv-vls-bitwise-ops.c
new file mode 100644
index 0000000000000..9821e26ce0bf8
--- /dev/null
+++ b/clang/test/CodeGen/riscv-rvv-vls-bitwise-ops.c
@@ -0,0 +1,417 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +zve64d \
+// RUN: -target-feature +f -target-feature +d -disable-O0-optnone \
+// RUN: -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: opt -S -passes=sroa | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+// AND
+
+// CHECK-LABEL: @and_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t and_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t and_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t and_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t and_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t and_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t and_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t and_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a & b;
+}
+
+// CHECK-LABEL: @and_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[AND:%.*]] = and <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[AND]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t and_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a & b;
+}
+
+// OR
+
+// CHECK-LABEL: @or_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t or_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t or_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t or_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t or_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t or_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t or_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t or_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a | b;
+}
+
+// CHECK-LABEL: @or_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[OR:%.*]] = or <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[OR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t or_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a | b;
+}
+
+// XOR
+
+// CHECK-LABEL: @xor_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t xor_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t xor_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t xor_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t xor_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t xor_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t xor_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t xor_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a ^ b;
+}
+
+// CHECK-LABEL: @xor_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[XOR:%.*]] = xor <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[XOR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t xor_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a ^ b;
+}
+
+// NEG
+
+// CHECK-LABEL: @not_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <32 x i8> [[A]], <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t not_i8(fixed_int8m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <16 x i16> [[A]], <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t not_i16(fixed_int16m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i32> [[A]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t not_i32(fixed_int32m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <4 x i64> [[A]], <i64 -1, i64 -1, i64 -1, i64 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t not_i64(fixed_int64m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <32 x i8> [[A]], <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t not_u8(fixed_uint8m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <16 x i16> [[A]], <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t not_u16(fixed_uint16m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <8 x i32> [[A]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t not_u32(fixed_uint32m1_t a) {
+  return ~a;
+}
+
+// CHECK-LABEL: @not_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[NOT:%.*]] = xor <4 x i64> [[A]], <i64 -1, i64 -1, i64 -1, i64 -1>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[NOT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t not_u64(fixed_uint64m1_t a) {
+  return ~a;
+}
diff --git a/clang/test/CodeGen/riscv-rvv-vls-compare-ops.c b/clang/test/CodeGen/riscv-rvv-vls-compare-ops.c
new file mode 100644
index 0000000000000..39bc764a3c570
--- /dev/null
+++ b/clang/test/CodeGen/riscv-rvv-vls-compare-ops.c
@@ -0,0 +1,825 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +zve64d \
+// RUN: -target-feature +f -target-feature +d -disable-O0-optnone \
+// RUN: -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: opt -S -passes=sroa | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+// EQ
+
+// CHECK-LABEL: @eq_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t eq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t eq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t eq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t eq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t eq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t eq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t eq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t eq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t eq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a == b;
+}
+
+// CHECK-LABEL: @eq_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t eq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a == b;
+}
+
+// NEQ
+
+// CHECK-LABEL: @neq_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t neq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t neq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t neq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t neq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t neq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t neq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t neq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t neq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp une <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t neq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a != b;
+}
+
+// CHECK-LABEL: @neq_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp une <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t neq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a != b;
+}
+
+// LT
+
+// CHECK-LABEL: @lt_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t lt_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t lt_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t lt_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t lt_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ult <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t lt_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ult <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t lt_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ult <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t lt_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ult <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t lt_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t lt_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a < b;
+}
+
+// CHECK-LABEL: @lt_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t lt_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a < b;
+}
+
+// LEQ
+
+// CHECK-LABEL: @leq_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sle <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t leq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sle <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t leq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sle <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t leq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sle <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t leq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ule <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t leq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ule <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t leq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ule <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t leq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ule <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t leq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t leq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a <= b;
+}
+
+// CHECK-LABEL: @leq_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t leq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a <= b;
+}
+
+// GT
+
+// CHECK-LABEL: @gt_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t gt_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t gt_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t gt_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t gt_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t gt_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t gt_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t gt_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t gt_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t gt_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a > b;
+}
+
+// CHECK-LABEL: @gt_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t gt_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a > b;
+}
+
+// GEQ
+
+// CHECK-LABEL: @geq_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sge <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t geq_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sge <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t geq_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sge <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t geq_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp sge <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t geq_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp uge <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <32 x i1> [[CMP]] to <32 x i8>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t geq_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp uge <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <16 x i1> [[CMP]] to <16 x i16>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t geq_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp uge <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t geq_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = icmp uge <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t geq_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <8 x float> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[CMP]] to <8 x i32>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t geq_f32(fixed_float32m1_t a, fixed_float32m1_t b) {
+  return a >= b;
+}
+
+// CHECK-LABEL: @geq_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <4 x double> [[A]], [[B]]
+// CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i64>
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SEXT]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t geq_f64(fixed_float64m1_t a, fixed_float64m1_t b) {
+  return a >= b;
+}
diff --git a/clang/test/CodeGen/riscv-rvv-vls-shift-ops.c b/clang/test/CodeGen/riscv-rvv-vls-shift-ops.c
new file mode 100644
index 0000000000000..f5ef09d7262b8
--- /dev/null
+++ b/clang/test/CodeGen/riscv-rvv-vls-shift-ops.c
@@ -0,0 +1,657 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +zve64d \
+// RUN: -target-feature +f -target-feature +d -disable-O0-optnone \
+// RUN: -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: opt -S -passes=sroa | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+// CHECK-LABEL: @lshift_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t lshift_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_i8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t rshift_i8(fixed_int8m1_t a, fixed_int8m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t lshift_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i8> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t rshift_u8(fixed_uint8m1_t a, fixed_uint8m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t lshift_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_i16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t rshift_i16(fixed_int16m1_t a, fixed_int16m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t lshift_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i16> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t rshift_u16(fixed_uint16m1_t a, fixed_uint16m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t lshift_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_i32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t rshift_i32(fixed_int32m1_t a, fixed_int32m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t lshift_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i32> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t rshift_u32(fixed_uint32m1_t a, fixed_uint32m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t lshift_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_i64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t rshift_i64(fixed_int64m1_t a, fixed_int64m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t lshift_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @rshift_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[B:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[B_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i64> [[A]], [[B]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t rshift_u64(fixed_uint64m1_t a, fixed_uint64m1_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @lshift_i8_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
+// CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t lshift_i8_rsplat(fixed_int8m1_t a, int8_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_i8_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t lshift_i8_lsplat(fixed_int8m1_t a, int8_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_i8_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i8> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t rshift_i8_rsplat(fixed_int8m1_t a, int8_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_i8_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i8> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_int8m1_t rshift_i8_lsplat(fixed_int8m1_t a, int8_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_u8_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
+// CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t lshift_u8_rsplat(fixed_uint8m1_t a, uint8_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_u8_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i8> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t lshift_u8_lsplat(fixed_uint8m1_t a, uint8_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_u8_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i8>
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i8> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t rshift_u8_rsplat(fixed_uint8m1_t a, uint8_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_u8_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i8> poison, i8 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i8> [[SPLAT_SPLATINSERT]], <32 x i8> poison, <32 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i8> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i8> @llvm.vector.insert.nxv8i8.v32i8(<vscale x 8 x i8> undef, <32 x i8> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 8 x i8> [[CASTSCALABLESVE]]
+//
+fixed_uint8m1_t rshift_u8_lsplat(fixed_uint8m1_t a, uint8_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_i16_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
+// CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t lshift_i16_rsplat(fixed_int16m1_t a, int16_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_i16_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t lshift_i16_lsplat(fixed_int16m1_t a, int16_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_i16_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i16> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t rshift_i16_rsplat(fixed_int16m1_t a, int16_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_i16_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i16> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_int16m1_t rshift_i16_lsplat(fixed_int16m1_t a, int16_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_u16_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
+// CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t lshift_u16_rsplat(fixed_uint16m1_t a, uint16_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_u16_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i16> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t lshift_u16_lsplat(fixed_uint16m1_t a, uint16_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_u16_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[B:%.*]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <16 x i32> [[SPLAT_SPLAT]] to <16 x i16>
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i16> [[A]], [[SH_PROM]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t rshift_u16_rsplat(fixed_uint16m1_t a, uint16_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_u16_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i16> poison, i16 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i16> [[SPLAT_SPLATINSERT]], <16 x i16> poison, <16 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i16> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i16> @llvm.vector.insert.nxv4i16.v16i16(<vscale x 4 x i16> undef, <16 x i16> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 4 x i16> [[CASTSCALABLESVE]]
+//
+fixed_uint16m1_t rshift_u16_lsplat(fixed_uint16m1_t a, uint16_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_i32_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t lshift_i32_rsplat(fixed_int32m1_t a, int32_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_i32_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t lshift_i32_lsplat(fixed_int32m1_t a, int32_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_i32_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t rshift_i32_rsplat(fixed_int32m1_t a, int32_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_i32_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i32> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_int32m1_t rshift_i32_lsplat(fixed_int32m1_t a, int32_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_u32_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t lshift_u32_rsplat(fixed_uint32m1_t a, uint32_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_u32_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t lshift_u32_lsplat(fixed_uint32m1_t a, uint32_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_u32_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i32> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t rshift_u32_rsplat(fixed_uint32m1_t a, uint32_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_u32_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i32> [[SPLAT_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i32> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v8i32(<vscale x 2 x i32> undef, <8 x i32> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 2 x i32> [[CASTSCALABLESVE]]
+//
+fixed_uint32m1_t rshift_u32_lsplat(fixed_uint32m1_t a, uint32_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_i64_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t lshift_i64_rsplat(fixed_int64m1_t a, int64_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_i64_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t lshift_i64_lsplat(fixed_int64m1_t a, int64_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_i64_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t rshift_i64_rsplat(fixed_int64m1_t a, int64_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_i64_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i64> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_int64m1_t rshift_i64_lsplat(fixed_int64m1_t a, int64_t b) {
+  return b >> a;
+}
+
+// CHECK-LABEL: @lshift_u64_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t lshift_u64_rsplat(fixed_uint64m1_t a, uint64_t b) {
+  return a << b;
+}
+
+// CHECK-LABEL: @lshift_u64_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i64> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHL]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t lshift_u64_lsplat(fixed_uint64m1_t a, uint64_t b) {
+  return b << a;
+}
+
+// CHECK-LABEL: @rshift_u64_rsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i64> [[A]], [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t rshift_u64_rsplat(fixed_uint64m1_t a, uint64_t b) {
+  return a >> b;
+}
+
+// CHECK-LABEL: @rshift_u64_lsplat(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[B:%.*]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i64> [[SPLAT_SPLAT]], [[A]]
+// CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[SHR]], i64 0)
+// CHECK-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+//
+fixed_uint64m1_t rshift_u64_lsplat(fixed_uint64m1_t a, uint64_t b) {
+  return b >> a;
+}
diff --git a/clang/test/CodeGen/riscv-rvv-vls-subscript-ops.c b/clang/test/CodeGen/riscv-rvv-vls-subscript-ops.c
new file mode 100644
index 0000000000000..aed783ba71488
--- /dev/null
+++ b/clang/test/CodeGen/riscv-rvv-vls-subscript-ops.c
@@ -0,0 +1,134 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +zve64d \
+// RUN: -target-feature +f -target-feature +d -disable-O0-optnone \
+// RUN: -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: opt -S -passes=sroa | FileCheck %s
+
+// REQUIRES: riscv-registered-target
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+// CHECK-LABEL: @subscript_int8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <32 x i8> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i8 [[VECEXT]]
+//
+int8_t subscript_int8(fixed_int8m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_uint8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <32 x i8> @llvm.vector.extract.v32i8.nxv8i8(<vscale x 8 x i8> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <32 x i8> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i8 [[VECEXT]]
+//
+uint8_t subscript_uint8(fixed_uint8m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_int16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <16 x i16> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i16 [[VECEXT]]
+//
+int16_t subscript_int16(fixed_int16m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_uint16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <16 x i16> @llvm.vector.extract.v16i16.nxv4i16(<vscale x 4 x i16> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <16 x i16> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i16 [[VECEXT]]
+//
+uint16_t subscript_uint16(fixed_uint16m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_int32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <8 x i32> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i32 [[VECEXT]]
+//
+int32_t subscript_int32(fixed_int32m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_uint32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x i32> @llvm.vector.extract.v8i32.nxv2i32(<vscale x 2 x i32> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <8 x i32> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i32 [[VECEXT]]
+//
+uint32_t subscript_uint32(fixed_uint32m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_int64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x i64> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i64 [[VECEXT]]
+//
+int64_t subscript_int64(fixed_int64m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_uint64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x i64> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret i64 [[VECEXT]]
+//
+uint64_t subscript_uint64(fixed_uint64m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_float32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <8 x float> @llvm.vector.extract.v8f32.nxv2f32(<vscale x 2 x float> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <8 x float> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret float [[VECEXT]]
+//
+float subscript_float32(fixed_float32m1_t a, size_t b) {
+  return a[b];
+}
+
+// CHECK-LABEL: @subscript_float64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[A_COERCE:%.*]], i64 0)
+// CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <4 x double> [[A]], i64 [[B:%.*]]
+// CHECK-NEXT:    ret double [[VECEXT]]
+//
+double subscript_float64(fixed_float64m1_t a, size_t b) {
+  return a[b];
+}
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index a7650c88df201..3649ddd651ffc 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -176,12 +176,12 @@
 
 // RUN: %clang_cc1 -triple amdgcn-unknown -target-cpu hawaii -o - -emit-llvm %s \
 // RUN: | FileCheck %s -check-prefix=R600SI
-// R600SI: target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+// R600SI: target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 
 // Test default -target-cpu
 // RUN: %clang_cc1 -triple amdgcn-unknown -o - -emit-llvm %s \
 // RUN: | FileCheck %s -check-prefix=R600SIDefault
-// R600SIDefault: target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+// R600SIDefault: target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 
 // RUN: %clang_cc1 -triple arm64-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=AARCH64
diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index c85ce0f6a27da..9967586fea9bc 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -1,7 +1,12 @@
 ; FIXME: This test should use CHECK-NEXT to keep up-to-date.
 ; REQUIRES: x86-registered-target
 
-; Validate ThinLTO post link pipeline at O2 and O3
+;; Validate that we set up the ThinLTO post link pipeline at O2 and O3
+;; for a ThinLTO distributed backend invoked via clang.
+;; Since LLVM tests already more thoroughly test this pipeline, and to
+;; avoid making this clang test too sensitive to LLVM pipeline changes,
+;; here we simply confirm that an LTO backend-specific pass is getting
+;; invoked (WPD).
 
 ; RUN: opt -thinlto-bc -o %t.o %s
 
@@ -17,94 +22,9 @@
 ; RUN: %clang -target x86_64-grtev4-linux-gnu \
 ; RUN:   -O3 -Xclang -fdebug-pass-manager \
 ; RUN:   -c -fthinlto-index=%t.o.thinlto.bc \
-; RUN:   -o %t.native.o -x ir %t.o 2>&1 | FileCheck -check-prefixes=CHECK-O,CHECK-O3 %s --dump-input=fail
+; RUN:   -o %t.native.o -x ir %t.o 2>&1 | FileCheck -check-prefixes=CHECK-O %s --dump-input=fail
 
 ; CHECK-O: Running pass: WholeProgramDevirtPass
-; CHECK-O: Running pass: LowerTypeTestsPass
-; CHECK-O: Running pass: PGOIndirectCallPromotion
-; CHECK-O: Running pass: InferFunctionAttrsPass
-; CHECK-O: Running pass: LowerExpectIntrinsicPass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: SROAPass on main
-; CHECK-O: Running pass: EarlyCSEPass on main
-; CHECK-O3: Running pass: CallSiteSplittingPass on main
-; CHECK-O: Running pass: LowerTypeTestsPass
-; CHECK-O: Running pass: IPSCCPPass
-; CHECK-O: Running pass: CalledValuePropagationPass
-; CHECK-O: Running pass: GlobalOptPass
-; CHECK-O: Running pass: PromotePass
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: InlinerPass on (main)
-; CHECK-O: Running pass: PostOrderFunctionAttrsPass on (main)
-; CHECK-O3: Running pass: ArgumentPromotionPass on (main)
-; CHECK-O: Running pass: SROAPass on main
-; CHECK-O: Running pass: EarlyCSEPass on main
-; CHECK-O: Running pass: SpeculativeExecutionPass on main
-; CHECK-O: Running pass: JumpThreadingPass on main
-; CHECK-O: Running pass: CorrelatedValuePropagationPass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O3: Running pass: AggressiveInstCombinePass on main
-; CHECK-O: Running pass: LibCallsShrinkWrapPass on main
-; CHECK-O: Running pass: TailCallElimPass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: ReassociatePass on main
-; CHECK-O: Running pass: LoopSimplifyPass on main
-; CHECK-O: Running pass: LCSSAPass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: LoopSimplifyPass on main
-; CHECK-O: Running pass: LCSSAPass on main
-; CHECK-O: Running pass: SROAPass on main
-; CHECK-O: Running pass: MergedLoadStoreMotionPass on main
-; CHECK-O: Running pass: GVNPass on main
-; CHECK-O: Running pass: SCCPPass on main
-; CHECK-O: Running pass: BDCEPass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: JumpThreadingPass on main
-; CHECK-O: Running pass: CorrelatedValuePropagationPass on main
-; CHECK-O: Running pass: ADCEPass on main
-; CHECK-O: Running pass: MemCpyOptPass on main
-; CHECK-O: Running pass: DSEPass on main
-; CHECK-O: Running pass: LoopSimplifyPass on main
-; CHECK-O: Running pass: LCSSAPass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: DeadArgumentEliminationPass
-; CHECK-O: Running pass: GlobalOptPass
-; CHECK-O: Running pass: GlobalDCEPass
-; CHECK-O: Running pass: EliminateAvailableExternallyPass
-; CHECK-O: Running pass: ReversePostOrderFunctionAttrsPass
-; CHECK-O: Running pass: RecomputeGlobalsAAPass
-; CHECK-O: Running pass: Float2IntPass on main
-; CHECK-O: Running pass: LowerConstantIntrinsicsPass on main
-; CHECK-O: Running pass: LoopSimplifyPass on main
-; CHECK-O: Running pass: LCSSAPass on main
-; CHECK-O: Running pass: LoopRotatePass on b
-; CHECK-O: Running pass: LoopDistributePass on main
-; CHECK-O: Running pass: InjectTLIMappings on main
-; CHECK-O: Running pass: LoopVectorizePass on main
-; CHECK-O: Running pass: LoopLoadEliminationPass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: SLPVectorizerPass on main
-; CHECK-O: Running pass: VectorCombinePass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: LoopUnrollPass on main
-; CHECK-O: Running pass: WarnMissedTransformationsPass on main
-; CHECK-O: Running pass: InstCombinePass on main
-; CHECK-O: Running pass: LoopSimplifyPass on main
-; CHECK-O: Running pass: LCSSAPass on main
-; CHECK-O: Running pass: LICMPass on b
-; CHECK-O: Running pass: AlignmentFromAssumptionsPass on main
-; CHECK-O: Running pass: LoopSinkPass on main
-; CHECK-O: Running pass: InstSimplifyPass on main
-; CHECK-O: Running pass: DivRemPairsPass on main
-; CHECK-O: Running pass: SimplifyCFGPass on main
-; CHECK-O: Running pass: GlobalDCEPass
-; CHECK-O: Running pass: ConstantMergePass
-; CHECK-O: Running pass: CGProfilePass
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
diff --git a/clang/test/CodeGenCUDA/Inputs/ocml-sample.cl b/clang/test/CodeGenCUDA/Inputs/ocml-sample.cl
new file mode 100644
index 0000000000000..46bd33a1c5947
--- /dev/null
+++ b/clang/test/CodeGenCUDA/Inputs/ocml-sample.cl
@@ -0,0 +1,18 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+half do_f16_stuff(half a, half b, half c) {
+  return __builtin_fmaf16(a, b, c) + 4.0h;
+}
+
+float do_f32_stuff(float a, float b, float c) {
+  return __builtin_fmaf(a, b, c) + 4.0f;
+}
+
+double do_f64_stuff(double a, double b, double c) {
+  return __builtin_fma(a, b, c) + 4.0;
+}
+
+__attribute__((weak))
+float weak_do_f32_stuff(float a, float b, float c) {
+  return c * (a / b);
+}
diff --git a/clang/test/CodeGenCUDA/link-builtin-bitcode-denormal-fp-mode.cu b/clang/test/CodeGenCUDA/link-builtin-bitcode-denormal-fp-mode.cu
new file mode 100644
index 0000000000000..910d2470315f1
--- /dev/null
+++ b/clang/test/CodeGenCUDA/link-builtin-bitcode-denormal-fp-mode.cu
@@ -0,0 +1,157 @@
+// Verify the behavior of the denormal-fp-mode attributes in the way that
+// rocm-device-libs should be built with. The bitcode should be compiled with
+// denormal-fp-math-f32=dynamic, and should be replaced with the denormal mode
+// of the final TU.
+
+// Build the fake device library in the way rocm-device-libs should be built.
+//
+// RUN: %clang_cc1 -x cl -triple amdgcn-amd-amdhsa -fdenormal-fp-math-f32=dynamic \
+// RUN:   -mcode-object-version=none -emit-llvm-bc \
+// RUN:   %S/Inputs/ocml-sample.cl -o %t.dynamic.f32.bc
+//
+// RUN: %clang_cc1 -x cl -triple amdgcn-amd-amdhsa -fdenormal-fp-math=dynamic \
+// RUN:   -mcode-object-version=none -emit-llvm-bc \
+// RUN:   %S/Inputs/ocml-sample.cl -o %t.dynamic.full.bc
+
+
+
+// Check the default behavior with no denormal-fp-math arguments.
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 -fcuda-is-device \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.f32.bc \
+// RUN:   -emit-llvm %s -o - | FileCheck -implicit-check-not=denormal-fp-math %s --check-prefixes=CHECK,INTERNALIZE
+
+
+// Check an explicit full ieee request
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 -fcuda-is-device \
+// RUN:    -fdenormal-fp-math=ieee \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.f32.bc \
+// RUN:   -emit-llvm %s -o - | FileCheck -implicit-check-not=denormal-fp-math %s --check-prefixes=CHECK,INTERNALIZE
+
+
+// Check explicit f32-only flushing request
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 \
+// RUN:   -fcuda-is-device -fdenormal-fp-math-f32=preserve-sign \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.f32.bc -emit-llvm %s -o - \
+// RUN: | FileCheck -implicit-check-not=denormal-fp-math --enable-var-scope %s --check-prefixes=CHECK,INTERNALIZE,IEEEF64-PSZF32
+
+
+// Check explicit flush all request. Only the f32 component of the library is
+// dynamic, so the linked functions should use IEEE as the base mode and the new
+// functions preserve-sign.
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 \
+// RUN:   -fcuda-is-device -fdenormal-fp-math=preserve-sign \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.f32.bc -emit-llvm %s -o - \
+// RUN: | FileCheck -implicit-check-not=denormal-fp-math --enable-var-scope %s --check-prefixes=CHECK,INTERNALIZE,PSZ
+
+
+// Check explicit f32-only, ieee-other flushing request
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 \
+// RUN:   -fcuda-is-device -fdenormal-fp-math=ieee -fdenormal-fp-math-f32=preserve-sign \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.f32.bc -emit-llvm %s -o - \
+// RUN: | FileCheck -implicit-check-not=denormal-fp-math --enable-var-scope %s --check-prefixes=CHECK,INTERNALIZE,IEEEF64-PSZF32
+
+
+// Check inverse of normal usage. Requesting IEEE f32, with flushed f16/f64
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 \
+// RUN:   -fcuda-is-device -fdenormal-fp-math=preserve-sign -fdenormal-fp-math-f32=ieee \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.f32.bc -emit-llvm %s -o - \
+// RUN: | FileCheck -implicit-check-not=denormal-fp-math --enable-var-scope %s --check-prefixes=CHECK,INTERNALIZE,IEEEF32-PSZF64-DYNF32
+
+
+// Check backwards from the normal usage where both library components can be
+// overridden.
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 \
+// RUN:   -fcuda-is-device -fdenormal-fp-math=preserve-sign -fdenormal-fp-math-f32=ieee \
+// RUN:   -mlink-builtin-bitcode %t.dynamic.full.bc -emit-llvm %s -o - \
+// RUN: | FileCheck -implicit-check-not=denormal-fp-math --enable-var-scope %s --check-prefixes=CHECK,INTERNALIZE,IEEEF32-PSZF64-DYNFULL
+
+
+
+// Check the case where no internalization is performed
+// RUN: %clang_cc1 -x hip -triple amdgcn-amd-amdhsa -target-cpu gfx803 \
+// RUN:   -fcuda-is-device -fdenormal-fp-math=preserve-sign -fdenormal-fp-math-f32=ieee \
+// RUN:   -mlink-bitcode-file %t.dynamic.full.bc -emit-llvm %s -o - \
+// RUN: | FileCheck -implicit-check-not=denormal-fp-math --enable-var-scope %s --check-prefixes=CHECK,NOINTERNALIZE,NOINTERNALIZE-IEEEF32-PSZF64-DYNFULL
+
+
+
+#define __device__ __attribute__((device))
+#define __global__ __attribute__((global))
+
+typedef _Float16 half;
+
+extern "C" {
+__device__ half do_f16_stuff(half a, half  b, half c);
+__device__ float do_f32_stuff(float a, float b, float c);
+
+// Currently all library functions are internalized. Check a weak function in
+// case we ever choose to not internalize these. In that case, the safest thing
+// to do would likely be to preserve the dynamic denormal-fp-math.
+__attribute__((weak)) __device__ float weak_do_f32_stuff(float a, float b, float c);
+__device__ double do_f64_stuff(double a, double b, double c);
+
+
+  // CHECK: kernel_f16({{.*}}) #[[$KERNELATTR:[0-9]+]]
+__global__ void kernel_f16(float* out, float* a, float* b, float* c) {
+  int id = 0;
+  out[id] = do_f16_stuff(a[id], b[id], c[id]);
+}
+
+// CHECK: kernel_f32({{.*}}) #[[$KERNELATTR]]
+__global__ void kernel_f32(float* out, float* a, float* b, float* c) {
+  int id = 0;
+  out[id] = do_f32_stuff(a[id], b[id], c[id]);
+  out[id] += weak_do_f32_stuff(a[id], b[id], c[id]);
+}
+
+// CHECK: kernel_f64({{.*}}) #[[$KERNELATTR]]
+__global__ void kernel_f64(double* out, double* a, double* b, double* c) {
+  int id = 0;
+  out[id] = do_f64_stuff(a[id], b[id], c[id]);
+}
+}
+
+// INTERNALIZE: define internal half @do_f16_stuff({{.*}}) #[[$FUNCATTR:[0-9]+]]
+// INTERNALIZE: define internal float @do_f32_stuff({{.*}}) #[[$FUNCATTR]]
+// INTERNALIZE: define internal double @do_f64_stuff({{.*}}) #[[$FUNCATTR]]
+// INTERNALIZE: define internal float @weak_do_f32_stuff({{.*}}) #[[$WEAK_FUNCATTR:[0-9]+]]
+
+
+// NOINTERNALIZE: define dso_local half @do_f16_stuff({{.*}}) #[[$FUNCATTR:[0-9]+]]
+// NOINTERNALIZE: define dso_local float @do_f32_stuff({{.*}}) #[[$FUNCATTR]]
+// NOINTERNALIZE: define dso_local double @do_f64_stuff({{.*}}) #[[$FUNCATTR]]
+// NOINTERNALIZE: define weak float @weak_do_f32_stuff({{.*}}) #[[$WEAK_FUNCATTR:[0-9]+]]
+
+
+
+// We should not be littering call sites with the attribute
+// Everything should use the default ieee with no explicit attribute
+
+// FIXME: Should check-not "denormal-fp-math" within the denormal-fp-math-f32
+// lines.
+
+// Default mode relies on the implicit check-not for the denormal-fp-math.
+
+// PSZ: #[[$KERNELATTR]] = { {{.*}} "denormal-fp-math"="preserve-sign,preserve-sign" {{.*}} "target-cpu"="gfx803" {{.*}} }
+// PSZ: #[[$FUNCATTR]] = { {{.*}} "denormal-fp-math-f32"="preserve-sign,preserve-sign" {{.*}} "target-cpu"="gfx803" {{.*}} }
+// PSZ: #[[$WEAK_FUNCATTR]] = { {{.*}} "denormal-fp-math-f32"="preserve-sign,preserve-sign" {{.*}} "target-cpu"="gfx803" {{.*}} }
+
+// FIXME: Should check-not "denormal-fp-math" within the line
+// IEEEF64-PSZF32: #[[$KERNELATTR]] = { {{.*}} "denormal-fp-math-f32"="preserve-sign,preserve-sign" {{.*}} "target-cpu"="gfx803" {{.*}} }
+// IEEEF64-PSZF32: #[[$FUNCATTR]] = { {{.*}} "denormal-fp-math-f32"="preserve-sign,preserve-sign" {{.*}} "target-cpu"="gfx803" {{.*}} }
+// IEEEF64-PSZF32: #[[$WEAK_FUNCATTR]] = { {{.*}} "denormal-fp-math-f32"="preserve-sign,preserve-sign" {{.*}} "target-cpu"="gfx803" {{.*}} }
+
+// IEEEF32-PSZF64-DYNF32: #[[$KERNELATTR]] = { {{.*}} "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" {{.*}} "target-cpu"="gfx803" {{.*}}  }
+// implicit check-not
+// implicit check-not
+
+
+// IEEEF32-PSZF64-DYNFULL: #[[$KERNELATTR]] = { {{.*}} "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" {{.*}} "target-cpu"="gfx803" {{.*}} }
+// IEEEF32-PSZF64-DYNFULL: #[[$FUNCATTR]] = { {{.*}} "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" {{.*}} "target-cpu"="gfx803" {{.*}}  }
+// IEEEF32-PSZF64-DYNFULL: #[[$WEAK_FUNCATTR]] = { {{.*}} "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee"  {{.*}} "target-cpu"="gfx803" {{.*}} }
+
+
+// -mlink-bitcode-file doesn't internalize or propagate attributes.
+// NOINTERNALIZE-IEEEF32-PSZF64-DYNFULL: #[[$KERNELATTR]] = { {{.*}} "denormal-fp-math"="preserve-sign,preserve-sign" "denormal-fp-math-f32"="ieee,ieee" {{.*}} "target-cpu"="gfx803" {{.*}} }
+// NOINTERNALIZE-IEEEF32-PSZF64-DYNFULL: #[[$FUNCATTR]] = { {{.*}} "denormal-fp-math"="dynamic,dynamic" {{.*}} }
+// NOINTERNALIZE-IEEEF32-PSZF64-DYNFULL: #[[$WEAK_FUNCATTR]] = { {{.*}} "denormal-fp-math"="dynamic,dynamic" {{.*}} }
diff --git a/clang/test/CodeGenCXX/cxx2b-consteval-if.cpp b/clang/test/CodeGenCXX/cxx2b-consteval-if.cpp
index a69a2271f661b..343b6a0bbd8a6 100644
--- a/clang/test/CodeGenCXX/cxx2b-consteval-if.cpp
+++ b/clang/test/CodeGenCXX/cxx2b-consteval-if.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 %s -emit-llvm -o - | FileCheck %s
 
 void should_be_used_1();
 void should_be_used_2();
diff --git a/clang/test/CodeGenCXX/cxx2b-static-call-operator.cpp b/clang/test/CodeGenCXX/cxx2b-static-call-operator.cpp
index 1d0b05ba10aa3..fd53649c9b061 100644
--- a/clang/test/CodeGenCXX/cxx2b-static-call-operator.cpp
+++ b/clang/test/CodeGenCXX/cxx2b-static-call-operator.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b %s -emit-llvm -triple x86_64-linux -o - | FileCheck %s
-// RUN: %clang_cc1 -std=c++2b %s -emit-llvm -triple x86_64-windows-msvc -o - | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 %s -emit-llvm -triple x86_64-linux -o - | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 %s -emit-llvm -triple x86_64-windows-msvc -o - | FileCheck %s
 
 struct Functor {
   static int operator()(int x, int y) {
diff --git a/clang/test/CodeGenCXX/cxx2b-static-subscript-operator.cpp b/clang/test/CodeGenCXX/cxx2b-static-subscript-operator.cpp
index 3d8957698f5a7..5dbd2c50cc56b 100644
--- a/clang/test/CodeGenCXX/cxx2b-static-subscript-operator.cpp
+++ b/clang/test/CodeGenCXX/cxx2b-static-subscript-operator.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b %s -emit-llvm -triple x86_64-linux -o - | FileCheck %s
-// RUN: %clang_cc1 -std=c++2b %s -emit-llvm -triple x86_64-windows-msvc -o - | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 %s -emit-llvm -triple x86_64-linux -o - | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 %s -emit-llvm -triple x86_64-windows-msvc -o - | FileCheck %s
 
 struct Functor {
   static int operator[](int x, int y) {
diff --git a/clang/test/CodeGenCXX/lto-visibility-inference.cpp b/clang/test/CodeGenCXX/lto-visibility-inference.cpp
index 1246743115604..e7ce3436a88b1 100644
--- a/clang/test/CodeGenCXX/lto-visibility-inference.cpp
+++ b/clang/test/CodeGenCXX/lto-visibility-inference.cpp
@@ -74,10 +74,10 @@ void f(C1 *c1, C2 *c2, C3 *c3, C4 *c4, C5 *c5, C6 *c6, std::C7 *c7,
   // MS: type.test{{.*}}!"?AUC2@@"
   c2->f();
   // ITANIUM: type.test{{.*}}!"_ZTS2C3"
-  // MS: type.test{{.*}}!"?AUC3@@"
+  // MS-NOT: type.test{{.*}}!"?AUC3@@"
   c3->f();
   // ITANIUM: type.test{{.*}}!"_ZTS2C4"
-  // MS: type.test{{.*}}!"?AUC4@@"
+  // MS-NOT: type.test{{.*}}!"?AUC4@@"
   c4->f();
   // ITANIUM-NOT: type.test{{.*}}!"_ZTS2C5"
   // MS-NOT: type.test{{.*}}!"?AUC5@@"
diff --git a/clang/test/CodeGenCXX/mangle-class-nttp.cpp b/clang/test/CodeGenCXX/mangle-class-nttp.cpp
index 70e5be8456cc0..12c81f2ba0514 100644
--- a/clang/test/CodeGenCXX/mangle-class-nttp.cpp
+++ b/clang/test/CodeGenCXX/mangle-class-nttp.cpp
@@ -15,7 +15,7 @@ template<B> void f() {}
 
 int n = 0;
 // CHECK: define weak_odr void @_Z1fIXtl1BadL_Z1nEEEEvv(
-// MSABI: define {{.*}} @"??$f@$2UB@@PEBH1?n@@3HAH0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UB@@PEBHE?n@@3HAH0A@@@@YAXXZ"
 template void f<B{&n}>();
 // CHECK: define weak_odr void @_Z1fIXtl1BLPKi0ELi1EEEEvv(
 // MSABI: define {{.*}} @"??$f@$2UB@@PEBH0A@H00@@@YAXXZ"
@@ -36,15 +36,19 @@ template void f<B{fold(reinterpret_cast<int*>(0))}>();
 
 // Pointers to subobjects.
 struct Nested { union { int k; int arr[2]; }; } nested[2];
-struct Derived : A, Nested { int z; } extern derived;
+struct Derived : A, Nested { int z; A a_field; } extern derived;
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z7derivedE16EEEEvv
 // MSABI: define {{.*}} void @"??$f@$2UB@@PEBH56E?derived@@3UDerived@@Az@@@H0A@@@@YAXXZ"
 template void f<B{&derived.z}>();
-// FIXME: We don't know the MS ABI mangling for array subscripting and
-// past-the-end pointers yet.
-#ifndef _WIN32
+// CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z7derivedE20EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UB@@PEBH566E?derived@@3UDerived@@Aa_field@@a@@@H0A@@@@YAXXZ"
+template void f<B{&derived.a_field.a}>();
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z6nestedE_EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UB@@PEBH56CE?nested@@3PAUNested@@A0A@@k@@@H0A@@@@YAXXZ"
 template void f<B{&nested[0].k}>();
+// Mangling of pointers to nested array elements and past-the-end pointers
+// is still incorrect in MSVC.
+#ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z6nestedE16_0pEEEEvv
 template void f<B{&nested[1].arr[2]}>();
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z7derivedE8pEEEEvv
@@ -59,14 +63,16 @@ template<BR> void f() {}
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z7derivedE16EEEEvv
 // MSABI: define {{.*}} void @"??$f@$2UBR@@AEBH6E?derived@@3UDerived@@Az@@@@@YAXXZ"
 template void f<BR{derived.z}>();
-// FIXME: We don't know the MS ABI mangling for array subscripting yet.
-#ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z6nestedE_EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UBR@@AEBH6CE?nested@@3PAUNested@@A0A@@k@@@@@YAXXZ"
 template void f<BR{nested[0].k}>();
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z6nestedE12_0EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UBR@@AEBHC6CE?nested@@3PAUNested@@A00@arr@@00@@@@YAXXZ"
 template void f<BR{nested[1].arr[1]}>();
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z7derivedE4EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UBR@@AEBH66E?derived@@3UDerived@@AA@@b@@@@@YAXXZ"
 template void f<BR{derived.b}>();
+#ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl2BRdecvPKiplcvPcadL_Z7derivedELl16EEEEvv
 template void f<BR{fold(*(&derived.b + 3))}>();
 #endif
@@ -77,42 +83,93 @@ template<C> void f() {}
 // CHECK: define weak_odr void @_Z1fIXtl1CadsoKiL_Z7derivedE16EEEEvv
 // MSABI: define {{.*}} void @"??$f@$2UC@@PEBH56E?derived@@3UDerived@@Az@@@@@@YAXXZ"
 template void f<C{&derived.z}>();
-#ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl1CadsoKiL_Z7derivedE4EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UC@@PEBH566E?derived@@3UDerived@@AA@@b@@@@@@YAXXZ"
 template void f<C{&derived.b}>();
-#endif
 
 // Pointers to members.
 struct D { const int Derived::*p; int k; };
 template<D> void f() {}
 // CHECK: define weak_odr void @_Z1fIXtl1DLM7DerivedKi0ELi1EEEEvv
-// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H0?0H00@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@HNH00@@@YAXXZ"
 template void f<D{nullptr, 1}>();
 // CHECK: define weak_odr void @_Z1fIXtl1DEEEvv
-// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H0?0H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@HNH0A@@@@YAXXZ"
 template void f<D{nullptr}>();
 // CHECK: define weak_odr void @_Z1fIXtl1DadL_ZN7Derived1zEEEEEvv
-// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H0BA@H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H82@z@@H0A@@@@YAXXZ"
 template void f<D{&Derived::z}>();
-#ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl1DmcM7DerivedKiadL_ZN1A1aEEEEEEvv
-// MSABI-FIXME: define {{.*}} @"??$f@$2UD@@PERDerived@@H0A@H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H8A@@a@@H0A@@@@YAXXZ"
 template void f<D{&A::a}>();
 // CHECK: define weak_odr void @_Z1fIXtl1DmcM7DerivedKiadL_ZN1A1bEEEEEEvv
-// MSABI-FIXME: define {{.*}} @"??$f@$2UD@@PERDerived@@H03H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H8A@@b@@H0A@@@@YAXXZ"
 template void f<D{&A::b}>();
 // FIXME: Is the Ut_1 mangling here correct?
 // CHECK: define weak_odr void @_Z1fIXtl1DmcM7DerivedKiadL_ZN6NestedUt_1kEE8ELi2EEEEvv
-// FIXME: This mangles the same as &A::a (bug in the MS ABI).
-// MSABI-FIXME: define {{.*}} @"??$f@$2UD@@PERDerived@@H0A@H01@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H8<unnamed-tag>@Nested@@k@@H01@@@YAXXZ"
 template void f<D{&Nested::k, 2}>();
 struct MoreDerived : A, Derived { int z; };
 // CHECK: define weak_odr void @_Z1fIXtl1DmcM7DerivedKiadL_ZN11MoreDerived1zEEn8EEEEvv
-// MSABI-FIXME: define {{.*}} @"??$f@$2UD@@PERDerived@@H0BI@H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UD@@PERDerived@@H8MoreDerived@@z@@H0A@@@@YAXXZ"
 template void f<D{(int Derived::*)&MoreDerived::z}>();
-#endif
 
-// FIXME: Pointers to member functions.
+struct DerivedVirtually : virtual A, Nested { int z; };
+struct D2 { const int DerivedVirtually::*p; int k; };
+template<D2> void f() {}
+// CHECK: define weak_odr void @_Z1fIXtl2D2LM16DerivedVirtuallyKi0ELi1EEEEvv
+// MSABI: define {{.*}} @"??$f@$2UD2@@PERDerivedVirtually@@HFA@?0H00@@@YAXXZ"
+template void f<D2{nullptr, 1}>();
+// CHECK: define weak_odr void @_Z1fIXtl2D2EEEvv
+// MSABI: define {{.*}} @"??$f@$2UD2@@PERDerivedVirtually@@HFA@?0H0A@@@@YAXXZ"
+template void f<D2{nullptr}>();
+// CHECK: define weak_odr void @_Z1fIXtl2D2adL_ZN16DerivedVirtually1zEEEEEvv
+// MSABI: define {{.*}} @"??$f@$2UD2@@PERDerivedVirtually@@HFBA@A@H0A@@@@YAXXZ"
+template void f<D2{&DerivedVirtually::z}>();
+
+// Forward-decl without MS inheritance keyword means unspecified inheritance
+// which is different from e. g. single inheritance.
+struct UnspecInherStruct;
+struct D3 { const int UnspecInherStruct::*p; };
+template<D3> void f() {}
+struct UnspecInherStruct { int i; };
+// CHECK: define weak_odr void @_Z1fIXtl2D3adL_ZN17UnspecInherStruct1iEEEEEvv
+// MSABI: define {{.*}} @"??$f@$2UD3@@PERUnspecInherStruct@@HGA@A@A@@@@YAXXZ"
+template void f<D3{&UnspecInherStruct::i}>();
+
+// Pointers to member functions.
+// Test struct templates instead of function templates so as to cover
+// the separate code which handles nullptr in their pointer-to-member arguments.
+struct Derived2 : A, Nested { void f(); virtual void g(); };
+struct D4 { void (Derived2::*p)(); };
+template <D4> struct S1 { static void fn() {} };
+// CHECK: define weak_odr void @_ZN2S1IXtl2D4adL_ZN8Derived21fEvEEEE2fnEv
+// MSABI: define {{.*}} @"?fn@?$S1@$2UD4@@P8Derived2@@EAAXXZE?f@2@QEAAXXZ@@@SAXXZ"
+template void S1<D4{&Derived2::f}>::fn();
+// CHECK: define weak_odr void @_ZN2S1IXtl2D4adL_ZN8Derived21gEvEEEE2fnEv
+// MSABI: define {{.*}} @"?fn@?$S1@$2UD4@@P8Derived2@@EAAXXZE??_92@$BA@AA@@@SAXXZ"
+template void S1<D4{&Derived2::g}>::fn();
+// CHECK: define weak_odr void @_ZN2S1IXtl2D4EEE2fnEv
+// MSABI: define {{.*}} @"?fn@?$S1@$2UD4@@P8Derived2@@EAAXXZHA@@@@SAXXZ"
+template void S1<D4{nullptr}>::fn();
+
+struct NoInheritance { void f(); };
+struct D5 { void (NoInheritance::*p)(); };
+template <D5> struct S2 { static void fn() {} };
+// CHECK: define weak_odr void @_ZN2S2IXtl2D5adL_ZN13NoInheritance1fEvEEEE2fnEv
+// MSABI: define {{.*}} @"?fn@?$S2@$2UD5@@P8NoInheritance@@EAAXXZE?f@2@QEAAXXZ@@@SAXXZ"
+template void S2<D5{&NoInheritance::f}>::fn();
+// CHECK: define weak_odr void @_ZN2S2IXtl2D5EEE2fnEv
+// MSABI: define {{.*}} @"?fn@?$S2@$2UD5@@P8NoInheritance@@EAAXXZN@@@SAXXZ"
+template void S2<D5{nullptr}>::fn();
+
+struct NoInheritanceButUnspecified;
+struct D6 { void (NoInheritanceButUnspecified::*p)(); };
+template <D6> struct S3 { static void fn() {} };
+// CHECK: define weak_odr void @_ZN2S3IXtl2D6EEE2fnEv
+// MSABI: define {{.*}} @"?fn@?$S3@$2UD6@@P8NoInheritanceButUnspecified@@EAAXXZJA@A@?0@@@SAXXZ"
+template void S3<D6{nullptr}>::fn();
+
 
 union E {
   int n;
@@ -204,8 +261,6 @@ template void f<G{1, 2}>();
 template void f<G{-8, -32}>();
 
 // Empty and nearly-empty unions.
-// Some of the MSVC manglings here are our invention, because MSVC rejects, but
-// seem likely to be right.
 union H1 {};
 union H2 { int : 1, : 2, : 3; };
 union H3 { int : 1, a, : 2, b, : 3; };
diff --git a/clang/test/CodeGenCXX/mangle-ms-templates.cpp b/clang/test/CodeGenCXX/mangle-ms-templates.cpp
index 7402d367ae3e9..cefe1e0cb5a55 100644
--- a/clang/test/CodeGenCXX/mangle-ms-templates.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-templates.cpp
@@ -272,7 +272,7 @@ struct type1 {
 };
 extern const record inst;
 void recref(type1<inst>) {}
-// CHECK: "?recref@@YAXU?$type1@$E?inst@@3Urecord@@B@@@Z"
+// CHECK: "?recref@@YAXU?$type1@$1?inst@@3Urecord@@B@@@Z"
 
 struct _GUID {};
 struct __declspec(uuid("{12345678-1234-1234-1234-1234567890aB}")) uuid;
@@ -286,7 +286,7 @@ struct UUIDType2 {};
 void fun(UUIDType1<uuid> a) {}
 // CHECK: "?fun@@YAXU?$UUIDType1@Uuuid@@$1?_GUID_12345678_1234_1234_1234_1234567890ab@@3U__s_GUID@@B@@@Z"
 void fun(UUIDType2<uuid> b) {}
-// CHECK: "?fun@@YAXU?$UUIDType2@Uuuid@@$E?_GUID_12345678_1234_1234_1234_1234567890ab@@3U__s_GUID@@B@@@Z"
+// CHECK: "?fun@@YAXU?$UUIDType2@Uuuid@@$1?_GUID_12345678_1234_1234_1234_1234567890ab@@3U__s_GUID@@B@@@Z"
 
 template <typename T> struct TypeWithFriendDefinition {
   friend void FunctionDefinedWithInjectedName(TypeWithFriendDefinition<T>) {}
diff --git a/clang/test/CodeGenCXX/ms-inline-asm-fields.cpp b/clang/test/CodeGenCXX/ms-inline-asm-fields.cpp
index 9f79fd6e2a17b..403e9c8427760 100644
--- a/clang/test/CodeGenCXX/ms-inline-asm-fields.cpp
+++ b/clang/test/CodeGenCXX/ms-inline-asm-fields.cpp
@@ -53,4 +53,4 @@ template void msvc_dcas_x86<false>::store();
 // CHECK: %[[P:.*]] = alloca %"struct.make_storage_type<false>::type", align 4
 // CHECK: %[[B:.*]] = getelementptr inbounds %"struct.make_storage_type<false>::type", ptr %[[P]], i32 0, i32 0
 // CHECK: %[[X:.*]] = getelementptr inbounds %"struct.make_storage_type<false>::type::B", ptr %[[B]], i32 0, i32 1
-// CHECK: call void asm sideeffect inteldialect "mov edx, dword ptr $0", "*m,~{edx},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i32) %[[X]])
+// CHECK: call void asm sideeffect inteldialect "mov edx, $0", "*m,~{edx},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i32) %[[X]])
diff --git a/clang/test/CodeGenCXX/new_hot_cold.cpp b/clang/test/CodeGenCXX/new_hot_cold.cpp
new file mode 100644
index 0000000000000..014e815201485
--- /dev/null
+++ b/clang/test/CodeGenCXX/new_hot_cold.cpp
@@ -0,0 +1,130 @@
+// Test to check that the appropriate attributes are added to the __hot_cold_t
+// versions of operator new.
+
+// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -faligned-allocation -emit-llvm -o - | FileCheck %s
+
+typedef __typeof__(sizeof(0)) size_t;
+
+namespace std {
+struct nothrow_t {};
+enum class align_val_t : size_t;
+} // namespace std
+std::nothrow_t nothrow;
+
+typedef unsigned char uint8_t;
+
+// See the following link for how this type is declared in tcmalloc:
+// https://github.com/google/tcmalloc/blob/220043886d4e2efff7a5702d5172cb8065253664/tcmalloc/malloc_extension.h#L53.
+enum class __hot_cold_t : uint8_t;
+
+// Test handling of declaration that uses a type alias, ensuring that it still
+// recognizes the expected __hot_cold_t type name.
+namespace malloc_namespace {
+using hot_cold_t = __hot_cold_t;
+}
+
+void *operator new(size_t size,
+                   malloc_namespace::hot_cold_t hot_cold) noexcept(false);
+void *operator new[](size_t size,
+                     malloc_namespace::hot_cold_t hot_cold) noexcept(false);
+void *operator new(size_t size, const std::nothrow_t &,
+                   malloc_namespace::hot_cold_t hot_cold) noexcept;
+void *operator new[](size_t size, const std::nothrow_t &,
+                     malloc_namespace::hot_cold_t hot_cold) noexcept;
+void *operator new(size_t size, std::align_val_t alignment,
+                   malloc_namespace::hot_cold_t hot_cold) noexcept(false);
+void *operator new[](size_t size, std::align_val_t alignment,
+                     malloc_namespace::hot_cold_t hot_cold) noexcept(false);
+void *operator new(size_t size, std::align_val_t alignment,
+                   const std::nothrow_t &,
+                   malloc_namespace::hot_cold_t hot_cold) noexcept;
+void *operator new[](size_t size, std::align_val_t alignment,
+                     const std::nothrow_t &,
+                     malloc_namespace::hot_cold_t hot_cold) noexcept;
+
+// All explicit operator new calls should not get any builtin attribute, whereas
+// all implicit new expressions should get builtin attributes. All of the
+// declarations should get nobuiltin attributes.
+
+void hot_cold_new() {
+  // CHECK: call noalias noundef nonnull ptr @_Znwm12__hot_cold_t(i64 noundef 1, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_CALL:#[^ ]*]]
+  operator new(1, (__hot_cold_t)0);
+  // CHECK: call noalias noundef nonnull ptr @_Znwm12__hot_cold_t(i64 noundef 4, i8 noundef zeroext 0) [[ATTR_BUILTIN_CALL:#[^ ]*]]
+  new ((__hot_cold_t)0) int;
+}
+
+// CHECK: declare noundef nonnull ptr @_Znwm12__hot_cold_t(i64 noundef, i8 noundef zeroext) [[ATTR_NOBUILTIN:#[^ ]*]]
+
+void hot_cold_new_array() {
+  // CHECK: call noalias noundef nonnull ptr @_Znam12__hot_cold_t(i64 noundef 1, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_CALL:#[^ ]*]]
+  operator new[](1, (__hot_cold_t)0);
+  // CHECK: call noalias noundef nonnull ptr @_Znam12__hot_cold_t(i64 noundef 4, i8 noundef zeroext 0) [[ATTR_BUILTIN_CALL:#[^ ]*]]
+  new ((__hot_cold_t)0) int[1];
+}
+
+// CHECK: declare noundef nonnull ptr @_Znam12__hot_cold_t(i64 noundef, i8 noundef zeroext) [[ATTR_NOBUILTIN:#[^ ]*]]
+
+void hot_cold_new_nothrow() {
+  // CHECK: call noalias noundef ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 noundef 1, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  operator new(1, nothrow, (__hot_cold_t)0);
+  // CHECK: call noalias noundef ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 noundef 4, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  new (nothrow, (__hot_cold_t)0) int;
+}
+
+// CHECK: declare noundef ptr @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 noundef, ptr noundef nonnull align 1 dereferenceable(1), i8 noundef zeroext) [[ATTR_NOBUILTIN_NOTHROW:#[^ ]*]]
+
+void hot_cold_new_nothrow_array() {
+  // CHECK: call noalias noundef ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 noundef 1, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  operator new[](1, nothrow, (__hot_cold_t)0);
+  // CHECK: call noalias noundef ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 noundef 4, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  new (nothrow, (__hot_cold_t)0) int[1];
+}
+
+// CHECK: declare noundef ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 noundef, ptr noundef nonnull align 1 dereferenceable(1), i8 noundef zeroext) [[ATTR_NOBUILTIN_NOTHROW:#[^ ]*]]
+
+class alignas(32) alignedstruct {
+  int x;
+};
+
+void hot_cold_new_align() {
+  // CHECK: call noalias noundef nonnull align 32 ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64 noundef 1, i64 noundef 32, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_CALL:#[^ ]*]]
+  operator new(1, (std::align_val_t)32, (__hot_cold_t)0);
+  // CHECK: call noalias noundef nonnull align 32 ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64 noundef 32, i64 noundef 32, i8 noundef zeroext 0) [[ATTR_BUILTIN_CALL:#[^ ]*]]
+  new ((__hot_cold_t)0) alignedstruct;
+}
+
+// CHECK: declare noundef nonnull ptr @_ZnwmSt11align_val_t12__hot_cold_t(i64 noundef, i64 noundef, i8 noundef zeroext) [[ATTR_NOBUILTIN:#[^ ]*]]
+
+void hot_cold_new_align_array() {
+  // CHECK: call noalias noundef nonnull align 32 ptr @_ZnamSt11align_val_t12__hot_cold_t(i64 noundef 1, i64 noundef 32, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_CALL:#[^ ]*]]
+  operator new[](1, (std::align_val_t)32, (__hot_cold_t)0);
+  // CHECK: call noalias noundef nonnull align 32 ptr @_ZnamSt11align_val_t12__hot_cold_t(i64 noundef 32, i64 noundef 32, i8 noundef zeroext 0) [[ATTR_BUILTIN_CALL:#[^ ]*]]
+  new ((__hot_cold_t)0) alignedstruct[1];
+}
+
+// CHECK: declare noundef nonnull ptr @_ZnamSt11align_val_t12__hot_cold_t(i64 noundef, i64 noundef, i8 noundef zeroext) [[ATTR_NOBUILTIN:#[^ ]*]]
+
+void hot_cold_new_align_nothrow() {
+  // CHECK: call noalias noundef align 32 ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef 1, i64 noundef 32, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  operator new(1, (std::align_val_t)32, nothrow, (__hot_cold_t)0);
+  // CHECK: call noalias noundef align 32 ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef 32, i64 noundef 32, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  new (nothrow, (__hot_cold_t)0) alignedstruct;
+}
+
+// CHECK: declare noundef ptr @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1), i8 noundef zeroext) [[ATTR_NOBUILTIN_NOTHROW:#[^ ]*]]
+
+void hot_cold_new_align_nothrow_array() {
+  // CHECK: call noalias noundef align 32 ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef 1, i64 noundef 32, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_NO_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  operator new[](1, (std::align_val_t)32, nothrow, (__hot_cold_t)0);
+  // CHECK: call noalias noundef align 32 ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef 32, i64 noundef 32, {{.*}} @nothrow, i8 noundef zeroext 0) [[ATTR_BUILTIN_NOTHROW_CALL:#[^ ]*]]
+  new (nothrow, (__hot_cold_t)0) alignedstruct[1];
+}
+
+// CHECK: declare noundef ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1), i8 noundef zeroext) [[ATTR_NOBUILTIN_NOTHROW:#[^ ]*]]
+
+// CHECK-DAG: attributes [[ATTR_NOBUILTIN]] = { nobuiltin allocsize(0) {{.*}} }
+// CHECK-DAG: attributes [[ATTR_NOBUILTIN_NOTHROW]] = { nobuiltin nounwind allocsize(0) {{.*}} }
+// CHECK-DAG: attributes [[ATTR_NO_BUILTIN_CALL]] = { allocsize(0) }
+// CHECK-DAG: attributes [[ATTR_BUILTIN_CALL]] = { builtin allocsize(0) }
+// CHECK-DAG: attributes [[ATTR_NO_BUILTIN_NOTHROW_CALL]] = { nounwind allocsize(0) }
+// CHECK-DAG: attributes [[ATTR_BUILTIN_NOTHROW_CALL]] = { builtin nounwind allocsize(0) }
diff --git a/clang/test/CodeGenCXX/riscv-mangle-rvv-fixed-vectors.cpp b/clang/test/CodeGenCXX/riscv-mangle-rvv-fixed-vectors.cpp
new file mode 100644
index 0000000000000..ff2f928c065f1
--- /dev/null
+++ b/clang/test/CodeGenCXX/riscv-mangle-rvv-fixed-vectors.cpp
@@ -0,0 +1,116 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=2 -mvscale-max=2 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-128
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-256
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-512
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=16 -mvscale-max=16 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-1024
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+template <typename T> struct S {};
+
+// CHECK-64: _Z2f11SI9__RVV_VLSIu14__rvv_int8m1_tLj64EEE
+// CHECK-128: _Z2f11SI9__RVV_VLSIu14__rvv_int8m1_tLj128EEE
+// CHECK-256: _Z2f11SI9__RVV_VLSIu14__rvv_int8m1_tLj256EEE
+// CHECK-512: _Z2f11SI9__RVV_VLSIu14__rvv_int8m1_tLj512EEE
+// CHECK-1024: _Z2f11SI9__RVV_VLSIu14__rvv_int8m1_tLj1024EEE
+void f1(S<fixed_int8m1_t>) {}
+
+// CHECK-64: _Z2f21SI9__RVV_VLSIu15__rvv_int16m1_tLj64EEE
+// CHECK-128: _Z2f21SI9__RVV_VLSIu15__rvv_int16m1_tLj128EEE
+// CHECK-256: _Z2f21SI9__RVV_VLSIu15__rvv_int16m1_tLj256EEE
+// CHECK-512: _Z2f21SI9__RVV_VLSIu15__rvv_int16m1_tLj512EEE
+// CHECK-1024: _Z2f21SI9__RVV_VLSIu15__rvv_int16m1_tLj1024EEE
+void f2(S<fixed_int16m1_t>) {}
+
+// CHECK-64: _Z2f31SI9__RVV_VLSIu15__rvv_int32m1_tLj64EEE
+// CHECK-128: _Z2f31SI9__RVV_VLSIu15__rvv_int32m1_tLj128EEE
+// CHECK-256: _Z2f31SI9__RVV_VLSIu15__rvv_int32m1_tLj256EEE
+// CHECK-512: _Z2f31SI9__RVV_VLSIu15__rvv_int32m1_tLj512EEE
+// CHECK-1024: _Z2f31SI9__RVV_VLSIu15__rvv_int32m1_tLj1024EEE
+void f3(S<fixed_int32m1_t>) {}
+
+// CHECK-64: _Z2f41SI9__RVV_VLSIu15__rvv_int64m1_tLj64EEE
+// CHECK-128: _Z2f41SI9__RVV_VLSIu15__rvv_int64m1_tLj128EEE
+// CHECK-256: _Z2f41SI9__RVV_VLSIu15__rvv_int64m1_tLj256EEE
+// CHECK-512: _Z2f41SI9__RVV_VLSIu15__rvv_int64m1_tLj512EEE
+// CHECK-1024: _Z2f41SI9__RVV_VLSIu15__rvv_int64m1_tLj1024EEE
+void f4(S<fixed_int64m1_t>) {}
+
+// CHECK-64: _Z2f51SI9__RVV_VLSIu15__rvv_uint8m1_tLj64EEE
+// CHECK-128: _Z2f51SI9__RVV_VLSIu15__rvv_uint8m1_tLj128EEE
+// CHECK-256: _Z2f51SI9__RVV_VLSIu15__rvv_uint8m1_tLj256EEE
+// CHECK-512: _Z2f51SI9__RVV_VLSIu15__rvv_uint8m1_tLj512EEE
+// CHECK-1024: _Z2f51SI9__RVV_VLSIu15__rvv_uint8m1_tLj1024EEE
+void f5(S<fixed_uint8m1_t>) {}
+
+// CHECK-64: _Z2f61SI9__RVV_VLSIu16__rvv_uint16m1_tLj64EEE
+// CHECK-128: _Z2f61SI9__RVV_VLSIu16__rvv_uint16m1_tLj128EEE
+// CHECK-256: _Z2f61SI9__RVV_VLSIu16__rvv_uint16m1_tLj256EEE
+// CHECK-512: _Z2f61SI9__RVV_VLSIu16__rvv_uint16m1_tLj512EEE
+// CHECK-1024: _Z2f61SI9__RVV_VLSIu16__rvv_uint16m1_tLj1024EEE
+void f6(S<fixed_uint16m1_t>) {}
+
+// CHECK-64: _Z2f71SI9__RVV_VLSIu16__rvv_uint32m1_tLj64EEE
+// CHECK-128: _Z2f71SI9__RVV_VLSIu16__rvv_uint32m1_tLj128EEE
+// CHECK-256: _Z2f71SI9__RVV_VLSIu16__rvv_uint32m1_tLj256EEE
+// CHECK-512: _Z2f71SI9__RVV_VLSIu16__rvv_uint32m1_tLj512EEE
+// CHECK-1024: _Z2f71SI9__RVV_VLSIu16__rvv_uint32m1_tLj1024EEE
+void f7(S<fixed_uint32m1_t>) {}
+
+// CHECK-64: _Z2f81SI9__RVV_VLSIu16__rvv_uint64m1_tLj64EEE
+// CHECK-128: _Z2f81SI9__RVV_VLSIu16__rvv_uint64m1_tLj128EEE
+// CHECK-256: _Z2f81SI9__RVV_VLSIu16__rvv_uint64m1_tLj256EEE
+// CHECK-512: _Z2f81SI9__RVV_VLSIu16__rvv_uint64m1_tLj512EEE
+// CHECK-1024: _Z2f81SI9__RVV_VLSIu16__rvv_uint64m1_tLj1024EEE
+void f8(S<fixed_uint64m1_t>) {}
+
+// CHECK-64: _Z2f91SI9__RVV_VLSIu17__rvv_float32m1_tLj64EEE
+// CHECK-128: _Z2f91SI9__RVV_VLSIu17__rvv_float32m1_tLj128EEE
+// CHECK-256: _Z2f91SI9__RVV_VLSIu17__rvv_float32m1_tLj256EEE
+// CHECK-512: _Z2f91SI9__RVV_VLSIu17__rvv_float32m1_tLj512EEE
+// CHECK-1024: _Z2f91SI9__RVV_VLSIu17__rvv_float32m1_tLj1024EEE
+void f9(S<fixed_float32m1_t>) {}
+
+// CHECK-64: _Z3f101SI9__RVV_VLSIu17__rvv_float64m1_tLj64EEE
+// CHECK-128: _Z3f101SI9__RVV_VLSIu17__rvv_float64m1_tLj128EEE
+// CHECK-256: _Z3f101SI9__RVV_VLSIu17__rvv_float64m1_tLj256EEE
+// CHECK-512: _Z3f101SI9__RVV_VLSIu17__rvv_float64m1_tLj512EEE
+// CHECK-1024: _Z3f101SI9__RVV_VLSIu17__rvv_float64m1_tLj1024EEE
+void f10(S<fixed_float64m1_t>) {}
diff --git a/clang/test/CodeGenCXX/riscv-rvv-fixedtypeinfo.cpp b/clang/test/CodeGenCXX/riscv-rvv-fixedtypeinfo.cpp
new file mode 100644
index 0000000000000..16b73d820a674
--- /dev/null
+++ b/clang/test/CodeGenCXX/riscv-rvv-fixedtypeinfo.cpp
@@ -0,0 +1,121 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-64
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=2 -mvscale-max=2 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-128
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-256
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-512
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu %s -emit-llvm -o - \
+// RUN:  -target-feature +f -target-feature +d \
+// RUN:  -target-feature +zve64d -mvscale-min=16 -mvscale-max=16 \
+// RUN:  | FileCheck %s --check-prefix=CHECK-1024
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+namespace std {
+class type_info;
+};
+
+auto &fs8 = typeid(fixed_int8m1_t);
+auto &fs16 = typeid(fixed_int16m1_t);
+auto &fs32 = typeid(fixed_int32m1_t);
+auto &fs64 = typeid(fixed_int64m1_t);
+
+auto &fu8 = typeid(fixed_uint8m1_t);
+auto &fu16 = typeid(fixed_uint16m1_t);
+auto &fu32 = typeid(fixed_uint32m1_t);
+auto &fu64 = typeid(fixed_uint64m1_t);
+
+auto &ff32 = typeid(fixed_float32m1_t);
+auto &ff64 = typeid(fixed_float64m1_t);
+
+// CHECK-64: @_ZTI9__RVV_VLSIu14__rvv_int8m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu14__rvv_int8m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu14__rvv_int8m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu14__rvv_int8m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu14__rvv_int8m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu14__rvv_int8m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu14__rvv_int8m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu14__rvv_int8m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu14__rvv_int8m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu14__rvv_int8m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu15__rvv_int16m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int16m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu15__rvv_int16m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int16m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu15__rvv_int16m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int16m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu15__rvv_int16m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int16m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu15__rvv_int16m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int16m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu15__rvv_int32m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int32m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu15__rvv_int32m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int32m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu15__rvv_int32m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int32m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu15__rvv_int32m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int32m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu15__rvv_int32m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int32m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu15__rvv_int64m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int64m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu15__rvv_int64m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int64m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu15__rvv_int64m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int64m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu15__rvv_int64m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int64m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu15__rvv_int64m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_int64m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu15__rvv_uint8m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_uint8m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu15__rvv_uint8m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_uint8m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu15__rvv_uint8m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_uint8m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu15__rvv_uint8m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_uint8m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu15__rvv_uint8m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu15__rvv_uint8m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu16__rvv_uint16m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint16m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu16__rvv_uint16m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint16m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu16__rvv_uint16m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint16m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu16__rvv_uint16m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint16m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu16__rvv_uint16m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint16m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu16__rvv_uint32m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint32m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu16__rvv_uint32m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint32m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu16__rvv_uint32m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint32m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu16__rvv_uint32m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint32m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu16__rvv_uint32m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint32m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu16__rvv_uint64m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint64m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu16__rvv_uint64m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint64m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu16__rvv_uint64m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint64m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu16__rvv_uint64m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint64m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu16__rvv_uint64m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu16__rvv_uint64m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu17__rvv_float32m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float32m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu17__rvv_float32m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float32m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu17__rvv_float32m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float32m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu17__rvv_float32m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float32m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu17__rvv_float32m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float32m1_tLj1024EE
+
+// CHECK-64: @_ZTI9__RVV_VLSIu17__rvv_float64m1_tLj64EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float64m1_tLj64EE
+// CHECK-128: @_ZTI9__RVV_VLSIu17__rvv_float64m1_tLj128EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float64m1_tLj128EE
+// CHECK-256: @_ZTI9__RVV_VLSIu17__rvv_float64m1_tLj256EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float64m1_tLj256EE
+// CHECK-512: @_ZTI9__RVV_VLSIu17__rvv_float64m1_tLj512EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float64m1_tLj512EE
+// CHECK-1024: @_ZTI9__RVV_VLSIu17__rvv_float64m1_tLj1024EE = {{.*}} @_ZTVN10__cxxabiv123__fundamental_type_infoE, {{.*}} @_ZTS9__RVV_VLSIu17__rvv_float64m1_tLj1024EE
diff --git a/clang/test/CodeGenOpenCL/amdgpu-env-amdgcn.cl b/clang/test/CodeGenOpenCL/amdgpu-env-amdgcn.cl
index bd1529187ca11..a085c3dd70ebc 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-env-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-env-amdgcn.cl
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -O0 -triple amdgcn -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 %s -O0 -triple amdgcn---opencl -emit-llvm -o - | FileCheck %s
 
-// CHECK: target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+// CHECK: target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 void foo(void) {}
diff --git a/clang/test/CodeGenOpenCL/as_type.cl b/clang/test/CodeGenOpenCL/as_type.cl
index afc76b7b57a9d..d8e75db936f73 100644
--- a/clang/test/CodeGenOpenCL/as_type.cl
+++ b/clang/test/CodeGenOpenCL/as_type.cl
@@ -13,7 +13,7 @@ char3 f1(char4 x) {
 }
 
 //CHECK: define{{.*}} spir_func <4 x i8> @f2(<3 x i8> noundef %[[x:.*]])
-//CHECK: %[[astype:.*]] = shufflevector <3 x i8> %[[x]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+//CHECK: %[[astype:.*]] = shufflevector <3 x i8> %[[x]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 //CHECK: ret <4 x i8> %[[astype]]
 char4 f2(char3 x) {
   return __builtin_astype(x, char4);
@@ -36,7 +36,7 @@ char4 f4(int x) {
 }
 
 //CHECK: define{{.*}} spir_func i32 @f5(<3 x i8> noundef %[[x:.*]])
-//CHECK: %[[shuffle:.*]] = shufflevector <3 x i8> %[[x]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+//CHECK: %[[shuffle:.*]] = shufflevector <3 x i8> %[[x]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 //CHECK: %[[astype:.*]] = bitcast <4 x i8> %[[shuffle]] to i32
 //CHECK: ret i32 %[[astype]]
 int f5(char3 x) {
@@ -98,7 +98,7 @@ char3 ptr_to_char3(int *x) {
 }
 
 //CHECK: define{{.*}} spir_func ptr @char3_to_ptr(<3 x i8> noundef %[[x:.*]])
-//CHECK: %[[astype:.*]] = shufflevector <3 x i8> %[[x]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+//CHECK: %[[astype:.*]] = shufflevector <3 x i8> %[[x]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 //CHECK: %[[cast1:.*]] = bitcast <4 x i8> %[[astype]] to i32
 //CHECK: %[[cast2:.*]] = inttoptr i32 %[[cast1]] to ptr
 //CHECK: ret ptr %[[cast2]]
diff --git a/clang/test/CodeGenOpenCL/partial_initializer.cl b/clang/test/CodeGenOpenCL/partial_initializer.cl
index 5e246ca71655a..cfdacd150e046 100644
--- a/clang/test/CodeGenOpenCL/partial_initializer.cl
+++ b/clang/test/CodeGenOpenCL/partial_initializer.cl
@@ -48,8 +48,8 @@ void f(void) {
 
   // CHECK: store <2 x i32> <i32 1, i32 2>, ptr %[[compoundliteral1]], align 8
   // CHECK: %[[v6:.*]] = load <2 x i32>, ptr %[[compoundliteral1]], align 8
-  // CHECK: %[[vext:.*]] = shufflevector <2 x i32> %[[v6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-  // CHECK: %[[vecinit:.*]] = shufflevector <4 x i32> %[[vext]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  // CHECK: %[[vext:.*]] = shufflevector <2 x i32> %[[v6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  // CHECK: %[[vecinit:.*]] = shufflevector <4 x i32> %[[vext]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   // CHECK: %[[vecinit2:.*]] = insertelement <4 x i32> %[[vecinit]], i32 3, i32 2
   // CHECK: %[[vecinit3:.*]] = insertelement <4 x i32> %[[vecinit2]], i32 4, i32 3
   // CHECK: store <4 x i32> %[[vecinit3]], ptr %[[compoundliteral]], align 16
diff --git a/clang/test/CodeGenOpenCL/preserve_vec3.cl b/clang/test/CodeGenOpenCL/preserve_vec3.cl
index d91df758546ed..19f0cdff60a9d 100644
--- a/clang/test/CodeGenOpenCL/preserve_vec3.cl
+++ b/clang/test/CodeGenOpenCL/preserve_vec3.cl
@@ -25,7 +25,7 @@ void kernel float4_to_float3(global float3 *a, global float4 *b) {
 void kernel float3_to_float4(global float3 *a, global float4 *b) {
   // CHECK-LABEL: spir_kernel void @float3_to_float4
   // CHECK: %[[LOAD_A:.*]] = load <3 x float>, ptr addrspace(1) %a, align 16
-  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x float> %[[LOAD_A]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x float> %[[LOAD_A]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   // CHECK: store <4 x float> %[[ASTYPE]], ptr addrspace(1) %b, align 16
   *b = __builtin_astype(*a, float4);
 }
@@ -33,7 +33,7 @@ void kernel float3_to_float4(global float3 *a, global float4 *b) {
 void kernel float3_to_double2(global float3 *a, global double2 *b) {
   // CHECK-LABEL: spir_kernel void @float3_to_double2
   // CHECK: %[[LOAD_A:.*]] = load <3 x float>, ptr addrspace(1) %a, align 16
-  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x float> %[[LOAD_A]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x float> %[[LOAD_A]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   // CHECK: store <4 x float> %[[ASTYPE]], ptr addrspace(1) %b, align 16
   *b = __builtin_astype(*a, double2);
 }
@@ -48,14 +48,14 @@ void kernel char8_to_short3(global short3 *a, global char8 *b) {
 
 void from_char3(char3 a, global int *out) {
   // CHECK-LABEL: void @from_char3
-  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x i8> %a, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x i8> %a, <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   // CHECK: store <4 x i8> %[[ASTYPE]], ptr addrspace(1) %out
   *out = __builtin_astype(a, int);
 }
 
 void from_short3(short3 a, global long *out) {
   // CHECK-LABEL: void @from_short3
-  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x i16> %a, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  // CHECK: %[[ASTYPE:.*]] = shufflevector <3 x i16> %a, <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   // CHECK: store <4 x i16> %[[ASTYPE]], ptr addrspace(1) %out
   *out = __builtin_astype(a, long);
 }
diff --git a/clang/test/CodeGenOpenCL/vector_literals.cl b/clang/test/CodeGenOpenCL/vector_literals.cl
index d8d36c7ea493f..6fbe38310db31 100644
--- a/clang/test/CodeGenOpenCL/vector_literals.cl
+++ b/clang/test/CodeGenOpenCL/vector_literals.cl
@@ -19,32 +19,32 @@ void vector_literals_valid() {
   int4 a_1_1_1_1 = (int4)(1, 2, c1.s2, c2.s3);
 
   //CHECK: store <2 x i32> <i32 1, i32 2>, ptr
-  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   //CHECK: insertelement <4 x i32> %{{.+}}, i32 3, i32 2
   //CHECK: insertelement <4 x i32> %{{.+}}, i32 4, i32 3
   int4 a_2_1_1 = (int4)((int2)(1, 2), 3, 4);
 
   //CHECK: store <2 x i32> <i32 2, i32 3>, ptr
-  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-  //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 undef>
+  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
   //CHECK: insertelement <4 x i32> %{{.+}}, i32 4, i32 3
   int4 a_1_2_1 = (int4)(1, (int2)(2, 3), 4);
 
   //CHECK: store <2 x i32> <i32 3, i32 4>, ptr
-  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   //CHECK: shufflevector <4 x i32> <i32 1, i32 2, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   int4 a_1_1_2 = (int4)(1, 2, (int2)(3, 4));
 
   //CHECK: store <2 x i32> <i32 1, i32 2>, ptr
-  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
   //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> <i32 3, i32 3, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   int4 a_2_2 = (int4)((int2)(1, 2), (int2)(3));
 
   //CHECK: store <4 x i32> <i32 2, i32 3, i32 4, i32 undef>, ptr
   //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
-  //CHECK: shufflevector <3 x i32> %{{.+}}, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  //CHECK: shufflevector <3 x i32> %{{.+}}, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
   //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
   int4 a_1_3 = (int4)(1, (int3)(2, 3, 4));
 
@@ -53,10 +53,10 @@ void vector_literals_valid() {
 
   //CHECK: load <4 x i32>, ptr %a
   //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-  //CHECK: shufflevector <8 x i32> <i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  //CHECK: shufflevector <8 x i32> <i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
   //CHECK: load <4 x i32>, ptr %a
-  //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
   //CHECK: shufflevector <8 x i32> %{{.+}}, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   int8 b = (int8)(1, 2, a.xy, a);
 
diff --git a/clang/test/CodeGenOpenCL/vector_shufflevector.cl b/clang/test/CodeGenOpenCL/vector_shufflevector.cl
index 0953c66f58e2c..a7018ca908f40 100644
--- a/clang/test/CodeGenOpenCL/vector_shufflevector.cl
+++ b/clang/test/CodeGenOpenCL/vector_shufflevector.cl
@@ -8,6 +8,6 @@
 typedef unsigned int uint2 __attribute((ext_vector_type(2)));
 
 void vector_shufflevector_valid(void) {
-    //CHECK: {{%.*}} = shufflevector <2 x i32> {{%.*}}, <2 x i32> undef, <2 x i32> <i32 0, i32 undef>
+    //CHECK: {{%.*}} = shufflevector <2 x i32> {{%.*}}, <2 x i32> undef, <2 x i32> <i32 0, i32 poison>
     (uint2)(((uint2)(0)).s0, 0);
 }
diff --git a/clang/test/CoverageMapping/if.cpp b/clang/test/CoverageMapping/if.cpp
index f477db980e7f4..4326add3b3443 100644
--- a/clang/test/CoverageMapping/if.cpp
+++ b/clang/test/CoverageMapping/if.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fms-extensions -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++2b -triple %itanium_abi_triple -main-file-name if.cpp %s | FileCheck %s
+// RUN: %clang_cc1 -fms-extensions -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++23 -triple %itanium_abi_triple -main-file-name if.cpp %s | FileCheck %s
 
 int nop() { return 0; }
 struct S {
diff --git a/clang/test/CoverageMapping/strong_order.cpp b/clang/test/CoverageMapping/strong_order.cpp
index ebda311d2489b..eeed85f826069 100644
--- a/clang/test/CoverageMapping/strong_order.cpp
+++ b/clang/test/CoverageMapping/strong_order.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++2b -triple %itanium_abi_triple -main-file-name if.cpp %s
+// RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -std=c++23 -triple %itanium_abi_triple -main-file-name if.cpp %s
 
 // No crash for following example.
 // See https://github.com/llvm/llvm-project/issues/45481
diff --git a/clang/test/Driver/Inputs/rewrite-2.map b/clang/test/Driver/Inputs/rewrite-2.map
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/clang/test/Driver/Inputs/rewrite.map b/clang/test/Driver/Inputs/rewrite.map
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/clang/test/Driver/Wframe-larger-than.c b/clang/test/Driver/Wframe-larger-than.c
index cc5b5d0a2ca1a..768ff4407624b 100644
--- a/clang/test/Driver/Wframe-larger-than.c
+++ b/clang/test/Driver/Wframe-larger-than.c
@@ -8,6 +8,7 @@
 // RUN:   -v -E %s 2>&1 | FileCheck %s --check-prefix=NOARG
 // RUN: not %clang -Wframe-larger-than \
 // RUN:   -v -E %s 2>&1 | FileCheck %s --check-prefix=NOARG
+// RUN: not %clang -Wframe-larger-than=0x2a -E %s 2>&1 | FileCheck %s --check-prefix=INVALID
 
 // ENABLE: cc1 {{.*}} -fwarn-stack-size=42 {{.*}} -Wframe-larger-than=42
 // ENABLE: frame-larger-than:
@@ -21,7 +22,8 @@
 // REENABLE: frame-larger-than:
 // REENABLE-SAME: warning
 
-// NOARG: error: argument to '-Wframe-larger-than=' is missing
+// NOARG: error: invalid argument '' to -Wframe-larger-than=
+// INVALID: error: invalid argument '0x2a' to -Wframe-larger-than=
 
 // We need to create some state transitions before the pragma will dump anything.
 #pragma clang diagnostic push
diff --git a/clang/test/Driver/aix-vec_extabi.c b/clang/test/Driver/aix-vec_extabi.c
index 93de4a25e3e76..83c37979f28d7 100644
--- a/clang/test/Driver/aix-vec_extabi.c
+++ b/clang/test/Driver/aix-vec_extabi.c
@@ -13,4 +13,4 @@
 // EXTABI-SAME:  "-mabi=vec-extabi"
 
 // DFLTABI:      "-cc1"
-// DFLTABI-SAME: "-mabi=vec-default"
+// DFLTABI-NOT:  "-mabi=vec-extabi"
diff --git a/clang/test/Driver/amdgpu-toolchain.c b/clang/test/Driver/amdgpu-toolchain.c
index 3477d50527c66..b8b6667333d81 100644
--- a/clang/test/Driver/amdgpu-toolchain.c
+++ b/clang/test/Driver/amdgpu-toolchain.c
@@ -11,6 +11,6 @@
 // DWARF_VER: "-dwarf-version=5"
 
 // RUN: %clang -### --target=amdgcn-amd-amdhsa -mcpu=gfx906 -nogpulib \
-// RUN:   -flto %s 2>&1 | FileCheck -check-prefix=LTO %s
-// LTO: clang{{.*}} "-flto=full"
+// RUN:   -flto -fconvergent-functions %s 2>&1 | FileCheck -check-prefix=LTO %s
+// LTO: clang{{.*}} "-flto=full"{{.*}}"-fconvergent-functions"
 // LTO: ld.lld{{.*}}-plugin-opt=mcpu=gfx906
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index 7f2334493c529..c00713a5d30fb 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -15,7 +15,7 @@
 // CHECK-V6M-C-SAME: "-T" "semihosted.lds" "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-V6M-C-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
 // CHECK-V6M-C-SAME: "-L[[RESOURCE_DIR:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}baremetal"
-// CHECK-V6M-C-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "-o" "{{.*}}.tmp.out"
+// CHECK-V6M-C-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "--target2=rel" "-o" "{{.*}}.tmp.out"
 
 // RUN: %clang %s -### --target=armv6m-none-eabi -nostdlibinc -nobuiltininc 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-LIBINC %s
@@ -42,7 +42,7 @@
 // CHECK-V6M-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
 // CHECK-V6M-DEFAULTCXX-SAME: "-L[[RESOURCE_DIR]]{{[/\\]+}}lib{{[/\\]+}}baremetal"
 // CHECK-V6M-DEFAULTCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-V6M-DEFAULTCXX-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "-o" "a.out"
+// CHECK-V6M-DEFAULTCXX-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "--target2=rel" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi -stdlib=libc++ 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-LIBCXX %s
@@ -53,7 +53,7 @@
 // CHECK-V6M-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
 // CHECK-V6M-LIBCXX-SAME: "-L[[RESOURCE_DIR]]{{[/\\]+}}lib{{[/\\]+}}baremetal"
 // CHECK-V6M-LIBCXX-SAME: "-lc++" "-lc++abi" "-lunwind"
-// CHECK-V6M-LIBCXX-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "-o" "a.out"
+// CHECK-V6M-LIBCXX-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "--target2=rel" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm \
@@ -66,7 +66,7 @@
 // CHECK-V6M-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
 // CHECK-V6M-LIBSTDCXX-SAME: "-L[[RESOURCE_DIR]]{{[/\\]+}}lib{{[/\\]+}}baremetal"
 // CHECK-V6M-LIBSTDCXX-SAME: "-lstdc++" "-lsupc++" "-lunwind"
-// CHECK-V6M-LIBSTDCXX-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "-o" "a.out"
+// CHECK-V6M-LIBSTDCXX-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m" "--target2=rel" "-o" "a.out"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm \
diff --git a/clang/test/Driver/canonical-system-headers.c b/clang/test/Driver/canonical-system-headers.c
new file mode 100644
index 0000000000000..a7ab57521fc22
--- /dev/null
+++ b/clang/test/Driver/canonical-system-headers.c
@@ -0,0 +1,6 @@
+// RUN: %clang -MD -no-canonical-prefixes -### %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO
+// RUN: %clang -MD -canonical-prefixes -### %s 2>&1 | FileCheck %s --check-prefix=CHECK-YES
+// RUN: %clang -MD -### %s 2>&1 | FileCheck %s --check-prefix=CHECK-YES
+
+// CHECK-YES: "-canonical-system-headers"
+// CHECK-NO-NOT: "-canonical-system-headers"
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 1d588bdfc1b00..b32b44ab31648 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -604,7 +604,7 @@
 // STDCXX20: -std=c++20
 
 // RUN: %clang_cl -fmsc-version=1900 -TP -std:c++latest -### -- %s 2>&1 | FileCheck -check-prefix=STDCXXLATEST %s
-// STDCXXLATEST: -std=c++2b
+// STDCXXLATEST: -std=c++23
 
 // RUN: env CL="/Gy" %clang_cl -### -- %s 2>&1 | FileCheck -check-prefix=ENV-CL %s
 // ENV-CL: "-ffunction-sections"
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index 446bb07ee6cb7..67ec82a09f852 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -140,6 +140,10 @@
 // CHECK-PROFILE-USE-DIR: "-fprofile-instrument-use-path={{.*}}.d/some/dir{{/|\\\\}}default.profdata"
 // CHECK-PROFILE-USE-FILE: "-fprofile-instrument-use-path=/tmp/somefile.prof"
 
+// RUN: %clang -### -S -fprofile-instr-use=%t.profdata -fdiagnostics-misexpect-tolerance=10 -Wmisexpect %s 2>&1 | FileCheck %s --check-prefix=CHECK-MISEXPECT-TOLLERANCE
+// CHECK-MISEXPECT-TOLLERANCE: "-fdiagnostics-misexpect-tolerance=10"
+// CHECK-MISEXPECT-TOLLERANCE-NOT: argument unused
+
 // RUN: %clang -### -S -fvectorize %s 2>&1 | FileCheck -check-prefix=CHECK-VECTORIZE %s
 // RUN: %clang -### -S -fno-vectorize -fvectorize %s 2>&1 | FileCheck -check-prefix=CHECK-VECTORIZE %s
 // RUN: %clang -### -S -fno-vectorize %s 2>&1 | FileCheck -check-prefix=CHECK-NO-VECTORIZE %s
diff --git a/clang/test/Driver/cuda-cross-compiling.c b/clang/test/Driver/cuda-cross-compiling.c
index 9750f09508010..7a62470b4ff1b 100644
--- a/clang/test/Driver/cuda-cross-compiling.c
+++ b/clang/test/Driver/cuda-cross-compiling.c
@@ -68,3 +68,12 @@
 //      DEFAULT: -cc1" "-triple" "nvptx64-nvidia-cuda" "-S" {{.*}} "-target-cpu" "sm_35" "-target-feature" "+ptx{{[0-9]+}}" {{.*}} "-o" "[[PTX:.+]].s"
 // DEFAULT-NEXT: ptxas{{.*}}"-m64" "-O0" "--gpu-name" "sm_35" "--output-file" "[[CUBIN:.+]].cubin" "[[PTX]].s" "-c"
 // DEFAULT-NEXT: nvlink{{.*}}"-o" "a.out" "-arch" "sm_35" {{.*}} "[[CUBIN]].cubin"
+
+//
+// Test to ensure that we enable handling global constructors in a freestanding
+// Nvidia compilation.
+//
+// RUN: %clang -target nvptx64-nvidia-cuda -march=sm_70 %s -### 2>&1 \
+// RUN:   | FileCheck -check-prefix=LOWERING %s
+
+// LOWERING: -cc1" "-triple" "nvptx64-nvidia-cuda" {{.*}} "-mllvm" "--nvptx-lower-global-ctor-dtor"
diff --git a/clang/test/Driver/darwin-ld-lto.c b/clang/test/Driver/darwin-ld-lto.c
index efa28b6746fac..2e049769b0cd4 100644
--- a/clang/test/Driver/darwin-ld-lto.c
+++ b/clang/test/Driver/darwin-ld-lto.c
@@ -40,3 +40,11 @@
 // GISEL: {{ld(.exe)?"}}
 // GISEL: "-mllvm" "-global-isel"
 // GISEL: "-mllvm" "-global-isel-abort=0"
+
+
+// Check that we disable atexit()-based global destructor lowering when
+// compiling/linking for kernel/kext/freestanding.
+// RUN: %clang -target arm64-apple-darwin %s -flto -fapple-kext -### 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEXT %s
+// KEXT: {{ld(.exe)?"}}
+// KEXT: "-mllvm" "-disable-atexit-based-global-dtor-lowering"
diff --git a/clang/test/Driver/denormal-fp-math.c b/clang/test/Driver/denormal-fp-math.c
index ea4dc8699ecc8..243de3b203fc4 100644
--- a/clang/test/Driver/denormal-fp-math.c
+++ b/clang/test/Driver/denormal-fp-math.c
@@ -1,6 +1,9 @@
 // RUN: %clang -### -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=ieee -v 2>&1 | FileCheck -check-prefix=CHECK-IEEE %s
 // RUN: %clang -### -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=preserve-sign -v 2>&1 | FileCheck -check-prefix=CHECK-PS %s
 // RUN: %clang -### -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=positive-zero -v 2>&1 | FileCheck -check-prefix=CHECK-PZ %s
+
+// RUN: %clang -### -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=dynamic -v 2>&1 | FileCheck -check-prefix=CHECK-DYNAMIC %s
+
 // RUN: %clang -### -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=ieee -fno-fast-math -v 2>&1 | FileCheck -check-prefix=CHECK-NO-UNSAFE %s
 // RUN: %clang -### -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=ieee -fno-unsafe-math-optimizations -v 2>&1 | FileCheck -check-prefix=CHECK-NO-UNSAFE %s
 // RUN: not %clang -target arm-unknown-linux-gnu -c %s -fdenormal-fp-math=foo -v 2>&1 | FileCheck -check-prefix=CHECK-INVALID0 %s
@@ -12,6 +15,9 @@
 // CHECK-IEEE-NOT: -fdenormal-fp-math=
 // CHECK-PS: "-fdenormal-fp-math=preserve-sign,preserve-sign"
 // CHECK-PZ: "-fdenormal-fp-math=positive-zero,positive-zero"
+// CHECK-DYNAMIC: "-fdenormal-fp-math=dynamic,dynamic"
+
+
 // CHECK-NO-UNSAFE-NOT: "-fdenormal-fp-math=ieee"
 // CHECK-INVALID0: error: invalid value 'foo' in '-fdenormal-fp-math=foo'
 // CHECK-INVALID1: error: invalid value 'ieee,foo' in '-fdenormal-fp-math=ieee,foo'
diff --git a/clang/test/Driver/flang/flang.f90 b/clang/test/Driver/flang/flang.f90
index 9b16f233b231a..5d8edf6308b00 100644
--- a/clang/test/Driver/flang/flang.f90
+++ b/clang/test/Driver/flang/flang.f90
@@ -13,7 +13,7 @@
 ! * (no type specified, resulting in an object file)
 
 ! All invocations should begin with flang -fc1, consume up to here.
-! ALL-LABEL: "{{[^"]*}}flang-new" "-fc1"
+! ALL-LABEL: "{{[^"]*}}flang-new{{[^"/]*}}" "-fc1"
 
 ! Check that f90 files are not treated as "previously preprocessed"
 ! ... in --driver-mode=flang.
diff --git a/clang/test/Driver/flang/flang_ucase.F90 b/clang/test/Driver/flang/flang_ucase.F90
index 113ef75f45b87..50305ee337e10 100644
--- a/clang/test/Driver/flang/flang_ucase.F90
+++ b/clang/test/Driver/flang/flang_ucase.F90
@@ -13,7 +13,7 @@
 ! * (no type specified, resulting in an object file)
 
 ! All invocations should begin with flang -fc1, consume up to here.
-! ALL-LABEL: "{{[^"]*}}flang-new" "-fc1"
+! ALL-LABEL: "{{[^"]*}}flang-new{{[^"/]*}}" "-fc1"
 
 ! Check that f90 files are not treated as "previously preprocessed"
 ! ... in --driver-mode=flang.
diff --git a/clang/test/Driver/flang/multiple-inputs.f90 b/clang/test/Driver/flang/multiple-inputs.f90
index f6ee60e48fef3..ada999e927a6a 100644
--- a/clang/test/Driver/flang/multiple-inputs.f90
+++ b/clang/test/Driver/flang/multiple-inputs.f90
@@ -1,7 +1,7 @@
 ! Check that flang driver can handle multiple inputs at once.
 
 ! RUN: %clang --driver-mode=flang -### -fsyntax-only %S/Inputs/one.f90 %S/Inputs/two.f90 2>&1 | FileCheck --check-prefixes=CHECK-SYNTAX-ONLY %s
-! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new" "-fc1"
+! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new{{[^"/]*}}" "-fc1"
 ! CHECK-SYNTAX-ONLY: "{{[^"]*}}/Inputs/one.f90"
-! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new" "-fc1"
+! CHECK-SYNTAX-ONLY-LABEL: "{{[^"]*}}flang-new{{[^"/]*}}" "-fc1"
 ! CHECK-SYNTAX-ONLY: "{{[^"]*}}/Inputs/two.f90"
diff --git a/clang/test/Driver/m68k-features.cpp b/clang/test/Driver/m68k-features.cpp
index 0ee9edcfd1647..a5222a72a57ff 100644
--- a/clang/test/Driver/m68k-features.cpp
+++ b/clang/test/Driver/m68k-features.cpp
@@ -59,3 +59,23 @@
 // RUN: FileCheck --check-prefix=CHECK-FIXED-D7 < %t %s
 // CHECK-FIXED-D7: "-target-feature" "+reserve-d7"
 
+// ==== Floating point ====
+// RUN: %clang -target m68k -m68000 -mhard-float -### %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-MX881 < %t %s
+// RUN: %clang -target m68k -m68000 -m68881 -### %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-MX881 < %t %s
+
+// RUN: %clang -target m68k -m68010 -mhard-float -### %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-MX881 < %t %s
+// RUN: %clang -target m68k -m68010 -m68881 -### %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-MX881 < %t %s
+
+// RUN: %clang -target m68k -m68020 -### %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-MX881 < %t %s
+
+// RUN: %clang -target m68k -m68030 -### %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-MX882 < %t %s
+
+// CHECK-MX881: "-target-feature" "+isa-68881"
+// CHECK-MX882: "-target-feature" "+isa-68882"
+
diff --git a/clang/test/Driver/m68k-macros.cpp b/clang/test/Driver/m68k-macros.cpp
index 8f85815e7552c..19954f5a0c81b 100644
--- a/clang/test/Driver/m68k-macros.cpp
+++ b/clang/test/Driver/m68k-macros.cpp
@@ -1,10 +1,19 @@
 // Check macro definitions
-// RUN: %clang -target m68k-unknown-linux -m68000 -dM -E %s | FileCheck --check-prefix=CHECK-MX %s
+
+// Since '__HAVE_68881__' sorted before most of the 'mc680x0' macros, we need to put it here.
+// CHECK-MX881: #define __HAVE_68881__ 1
+// CHECK-NOMX881-NOT: #define __HAVE_68881__ 1
+
+// RUN: %clang -target m68k-unknown-linux -m68000 -dM -E %s | FileCheck --check-prefixes=CHECK-MX,CHECK-NOMX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68000 -mhard-float -dM -E %s | FileCheck --check-prefix=CHECK-MX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68000 -m68881 -dM -E %s | FileCheck --check-prefix=CHECK-MX881 %s
 // CHECK-MX: #define __mc68000 1
 // CHECK-MX: #define __mc68000__ 1
 // CHECK-MX: #define mc68000 1
 
-// RUN: %clang -target m68k-unknown-linux -m68010 -dM -E %s | FileCheck --check-prefix=CHECK-MX10 %s
+// RUN: %clang -target m68k-unknown-linux -m68010 -dM -E %s | FileCheck --check-prefixes=CHECK-MX10,CHECK-NOMX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68010 -mhard-float -dM -E %s | FileCheck --check-prefix=CHECK-MX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68010 -m68881 -dM -E %s | FileCheck --check-prefix=CHECK-MX881 %s
 // CHECK-MX10: #define __mc68000 1
 // CHECK-MX10: #define __mc68000__ 1
 // CHECK-MX10: #define __mc68010 1
@@ -12,7 +21,8 @@
 // CHECK-MX10: #define mc68000 1
 // CHECK-MX10: #define mc68010 1
 
-// RUN: %clang -target m68k-unknown-linux -m68020 -dM -E %s | FileCheck --check-prefix=CHECK-MX20 %s
+// RUN: %clang -target m68k-unknown-linux -m68020 -dM -E %s | FileCheck --check-prefixes=CHECK-MX20,CHECK-MX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68020 -msoft-float -dM -E %s | FileCheck --check-prefix=CHECK-NOMX881 %s
 // CHECK-MX20: #define __mc68000 1
 // CHECK-MX20: #define __mc68000__ 1
 // CHECK-MX20: #define __mc68020 1
@@ -20,7 +30,8 @@
 // CHECK-MX20: #define mc68000 1
 // CHECK-MX20: #define mc68020 1
 
-// RUN: %clang -target m68k-unknown-linux -m68030 -dM -E %s | FileCheck --check-prefix=CHECK-MX30 %s
+// RUN: %clang -target m68k-unknown-linux -m68030 -dM -E %s | FileCheck --check-prefixes=CHECK-MX30,CHECK-MX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68030 -msoft-float -dM -E %s | FileCheck --check-prefix=CHECK-NOMX881 %s
 // CHECK-MX30: #define __mc68000 1
 // CHECK-MX30: #define __mc68000__ 1
 // CHECK-MX30: #define __mc68030 1
@@ -28,7 +39,8 @@
 // CHECK-MX30: #define mc68000 1
 // CHECK-MX30: #define mc68030 1
 
-// RUN: %clang -target m68k-unknown-linux -m68040 -dM -E %s | FileCheck --check-prefix=CHECK-MX40 %s
+// RUN: %clang -target m68k-unknown-linux -m68040 -dM -E %s | FileCheck --check-prefixes=CHECK-MX40,CHECK-MX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68040 -msoft-float -dM -E %s | FileCheck --check-prefix=CHECK-NOMX881 %s
 // CHECK-MX40: #define __mc68000 1
 // CHECK-MX40: #define __mc68000__ 1
 // CHECK-MX40: #define __mc68040 1
@@ -36,7 +48,8 @@
 // CHECK-MX40: #define mc68000 1
 // CHECK-MX40: #define mc68040 1
 
-// RUN: %clang -target m68k-unknown-linux -m68060 -dM -E %s | FileCheck --check-prefix=CHECK-MX60 %s
+// RUN: %clang -target m68k-unknown-linux -m68060 -dM -E %s | FileCheck --check-prefixes=CHECK-MX60,CHECK-MX881 %s
+// RUN: %clang -target m68k-unknown-linux -m68060 -msoft-float -dM -E %s | FileCheck --check-prefix=CHECK-NOMX881 %s
 // CHECK-MX60: #define __mc68000 1
 // CHECK-MX60: #define __mc68000__ 1
 // CHECK-MX60: #define __mc68060 1
diff --git a/clang/test/Driver/modules-fprebuilt-mdoule-path.cpp b/clang/test/Driver/modules-fprebuilt-mdoule-path.cpp
index a2c830bfda00b..900645546394b 100644
--- a/clang/test/Driver/modules-fprebuilt-mdoule-path.cpp
+++ b/clang/test/Driver/modules-fprebuilt-mdoule-path.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang -std=c++2a -fprebuilt-module-path=. -### -c %s 2>&1 | FileCheck %s
 // RUN: %clang -std=c++20 -fprebuilt-module-path=. -### -c %s 2>&1 | FileCheck %s
-// RUN: %clang -std=c++2b -fprebuilt-module-path=. -### -c %s 2>&1 | FileCheck %s
+// RUN: %clang -std=c++23 -fprebuilt-module-path=. -### -c %s 2>&1 | FileCheck %s
 // RUN: %clang -std=c++latest -fprebuilt-module-path=. -### -c %s 2>&1 | FileCheck %s
 //
 // CHECK-NOT: warning: argument unused during compilation
diff --git a/clang/test/Driver/rewrite-map-files.c b/clang/test/Driver/rewrite-map-files.c
deleted file mode 100644
index f079347c59438..0000000000000
--- a/clang/test/Driver/rewrite-map-files.c
+++ /dev/null
@@ -1,2 +0,0 @@
-// RUN: %clang -### -frewrite-map-file=%t.map -c %s -o /dev/null 2>&1 | FileCheck %s
-// CHECK: error: no such file or directory:
diff --git a/clang/test/Driver/rewrite-map-in-diagnostics.c b/clang/test/Driver/rewrite-map-in-diagnostics.c
deleted file mode 100644
index 5ca8b897e199a..0000000000000
--- a/clang/test/Driver/rewrite-map-in-diagnostics.c
+++ /dev/null
@@ -1,12 +0,0 @@
-// RUN: export LSAN_OPTIONS=detect_leaks=0
-// RUN: rm -rf "%t"
-// RUN: mkdir -p "%t"
-// RUN: env TMPDIR="%t" TEMP="%t" TMP="%t" RC_DEBUG_OPTION=1 \
-// RUN:     not %clang -fsyntax-only -frewrite-map-file=%p/Inputs/rewrite.map %s 2>&1 \
-// RUN:   | FileCheck %s
-
-#pragma clang __debug parser_crash
-
-// CHECK: note: diagnostic msg: {{.*}}rewrite.map
-
-// REQUIRES: crash-recovery
diff --git a/clang/test/Driver/riscv-arch.c b/clang/test/Driver/riscv-arch.c
index 4d11db2a5e09a..5b9a45deb06a2 100644
--- a/clang/test/Driver/riscv-arch.c
+++ b/clang/test/Driver/riscv-arch.c
@@ -355,12 +355,20 @@
 // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-TARGET %s
 // RUN: %clang --target=riscv64-unknown-elf -march=rv32i -### %s \
 // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-TARGET %s
+// RUN: %clang --target=riscv32-unknown-elf -mcpu=generic-rv32 -### %s \
+// RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-TARGET %s
+// RUN: %clang --target=riscv64-unknown-elf -mcpu=generic-rv32 -### %s \
+// RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-TARGET %s
 // RV32-TARGET: "-triple" "riscv32-unknown-unknown-elf"
 
 // RUN: %clang --target=riscv32-unknown-elf -march=rv64i -### %s \
 // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV64-TARGET %s
 // RUN: %clang --target=riscv64-unknown-elf -march=rv64i -### %s \
 // RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV64-TARGET %s
+// RUN: %clang --target=riscv32-unknown-elf -mcpu=generic-rv64 -### %s \
+// RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV64-TARGET %s
+// RUN: %clang --target=riscv64-unknown-elf -mcpu=generic-rv64 -### %s \
+// RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV64-TARGET %s
 // RV64-TARGET: "-triple" "riscv64-unknown-unknown-elf"
 
 // RUN: %clang --target=riscv32-unknown-elf -march=rv32ifzfh01p0 -### %s \
diff --git a/clang/test/Driver/riscv-cpus.c b/clang/test/Driver/riscv-cpus.c
index b107d89773e1c..76325311668d8 100644
--- a/clang/test/Driver/riscv-cpus.c
+++ b/clang/test/Driver/riscv-cpus.c
@@ -174,6 +174,3 @@
 
 // RUN: %clang --target=riscv32 -### -c %s 2>&1 -mcpu=generic-rv32 -march=rv64i | FileCheck -check-prefix=MISMATCH-ARCH %s
 // MISMATCH-ARCH: cpu 'generic-rv32' does not support rv64
-
-// RUN: %clang --target=riscv32 -### -c %s 2>&1 -mcpu=generic-rv64 | FileCheck -check-prefix=MISMATCH-MCPU %s
-// MISMATCH-MCPU: error: cpu 'generic-rv64' does not support rv32
diff --git a/clang/test/Driver/riscv-rvv-vector-bits.c b/clang/test/Driver/riscv-rvv-vector-bits.c
index 2cc427502b149..273d692708ff9 100644
--- a/clang/test/Driver/riscv-rvv-vector-bits.c
+++ b/clang/test/Driver/riscv-rvv-vector-bits.c
@@ -43,3 +43,24 @@
 // RUN:  -mrvv-vector-bits=64 2>&1 | FileCheck --check-prefix=CHECK-BAD-VALUE-ERROR %s
 
 // CHECK-BAD-VALUE-ERROR: error: unsupported argument '{{.*}}' to option '-mrvv-vector-bits='
+
+// Error if using attribute without -msve-vector-bits=<bits> or if using -msve-vector-bits=<bits>+ syntax
+// -----------------------------------------------------------------------------
+// RUN: not %clang -c %s -o /dev/null -target riscv64-linux-gnu \
+// RUN:  -march=rv64gc_zve64x 2>&1 | FileCheck --check-prefix=CHECK-NO-FLAG-ERROR %s
+// RUN: not %clang -c %s -o /dev/null -target riscv64-linux-gnu \
+// RUN:  -march=rv64gc_zve64x -mrvv-vector-bits=scalable 2>&1 | FileCheck --check-prefix=CHECK-NO-FLAG-ERROR %s
+
+typedef __rvv_int32m1_t vint32m1_t;
+typedef vint32m1_t noflag __attribute__((riscv_rvv_vector_bits(256)));
+
+// CHECK-NO-FLAG-ERROR: error: 'riscv_rvv_vector_bits' is only supported when '-mrvv-vector-bits=<bits>' is specified with a value of "zvl" or a power 2 in the range [64,65536]
+
+// Error if attribute vector size != -mrvv-vector-bits
+// -----------------------------------------------------------------------------
+// RUN: not %clang -c %s -o /dev/null -target riscv64-linux-gnu \
+// RUN:  -march=rv64gc_zve64x -mrvv-vector-bits=128 2>&1 | FileCheck --check-prefix=CHECK-BAD-VECTOR-SIZE-ERROR %s
+
+typedef vint32_t bad_vector_size __attribute__((riscv_rvv_vector_bits(256)));
+
+// CHECK-BAD-VECTOR-SIZE-ERROR: error: invalid RVV vector size '256', must match value set by '-mrvv-vector-bits' ('128')
diff --git a/clang/test/Driver/std.cpp b/clang/test/Driver/std.cpp
index c04221b363d7f..fff604fa98ece 100644
--- a/clang/test/Driver/std.cpp
+++ b/clang/test/Driver/std.cpp
@@ -11,8 +11,8 @@
 // RUN: not %clang -std=gnu++1z %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX1Z %s
 // RUN: not %clang -std=c++2a %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CXX2A %s
 // RUN: not %clang -std=gnu++2a %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX2A %s
-// RUN: not %clang -std=c++2b %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CXX2B %s
-// RUN: not %clang -std=gnu++2b %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX2B %s
+// RUN: not %clang -std=c++23 %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CXX2B %s
+// RUN: not %clang -std=gnu++23 %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX2B %s
 
 void f(int n) {
   typeof(n)();
diff --git a/clang/test/Driver/symbol-rewriter.c b/clang/test/Driver/symbol-rewriter.c
deleted file mode 100644
index 2bee07b4b0281..0000000000000
--- a/clang/test/Driver/symbol-rewriter.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: %clang -frewrite-map-file=%S/Inputs/rewrite.map -### %s 2>&1 | FileCheck %s -check-prefix CHECK-SINGLE
-
-// CHECK-SINGLE: "-frewrite-map-file={{.*[\\/]}}rewrite.map"
-
-// RUN: %clang -frewrite-map-file=%S/Inputs/rewrite-1.map -frewrite-map-file=%S/Inputs/rewrite-2.map -### %s 2>&1 | FileCheck %s -check-prefix CHECK-MULTIPLE
-
-// CHECK-MULTIPLE: "-frewrite-map-file={{.*[\\/]}}rewrite-1.map" "-frewrite-map-file={{.*[\\/]}}rewrite-2.map"
diff --git a/clang/test/Driver/test-uwtable.cpp b/clang/test/Driver/test-uwtable.cpp
new file mode 100644
index 0000000000000..8498c68982a0c
--- /dev/null
+++ b/clang/test/Driver/test-uwtable.cpp
@@ -0,0 +1,26 @@
+// This test is added to provide test coverage for the uwtable attribute. It
+// aims to make sure that for an X86_64 output compiled with -fexceptions or
+//  -fno-exceptions, a uwtable attribute is emitted. However, for an AArch64
+//  output with -fexceptions a uwtable attribute is emitted, but with 
+// -fno-exceptions a uwtable attribute is not emitted.
+
+// RUN: %clang -g -fexceptions --target=x86_64-apple-darwin %s -S -emit-llvm -o - | FileCheck %s --check-prefix=X86_64_UWTABLE_EXCEPTIONS
+// X86_64_UWTABLE_EXCEPTIONS: attributes #0 = { {{.*}} uwtable
+// X86_64_UWTABLE_EXCEPTIONS: !{{[0-9]+}} = !{i32 7, !"uwtable", i32 2}
+
+// RUN: %clang -g -fno-exceptions --target=x86_64-apple-darwin %s -S -emit-llvm -o - | FileCheck %s --check-prefix=X86_64_UWTABLE
+// X86_64_UWTABLE: attributes #0 = { {{.*}} uwtable
+// X86_64_UWTABLE: !{{[0-9]+}} = !{i32 7, !"uwtable", i32 2}
+
+// RUN: %clang -g -fexceptions --target=arm64-apple-darwin %s -S -emit-llvm -o - | FileCheck %s --check-prefix=ARM64_UWTABLE_EXCEPTIONS
+// ARM64_UWTABLE_EXCEPTIONS: attributes #0 = { {{.*}} uwtable
+// ARM64_UWTABLE_EXCEPTIONS: !{{[0-9]+}} = !{i32 7, !"uwtable", i32 1}
+
+// RUN: %clang -g -fno-exceptions --target=arm64-apple-darwin %s -S -emit-llvm -o - | FileCheck %s --check-prefix=ARM64_UWTABLE
+// ARM64_UWTABLE-NOT: attributes #0 = { {{.*}} uwtable
+// ARM64_UWTABLE-NOT: !{{[0-9]+}} = !{i32 7, !"uwtable", i32 2}
+// ARM64_UWTABLE: attributes #0 =
+
+int main() {
+    return 1;
+}
diff --git a/clang/test/Driver/unknown-std.cpp b/clang/test/Driver/unknown-std.cpp
index d1722b78499ab..1490e9e3a1164 100644
--- a/clang/test/Driver/unknown-std.cpp
+++ b/clang/test/Driver/unknown-std.cpp
@@ -17,8 +17,8 @@
 // CHECK-NEXT: note: use 'gnu++17' for 'ISO C++ 2017 with amendments and GNU extensions' standard
 // CHECK-NEXT: note: use 'c++20' for 'ISO C++ 2020 DIS' standard
 // CHECK-NEXT: note: use 'gnu++20' for 'ISO C++ 2020 DIS with GNU extensions' standard
-// CHECK-NEXT: note: use 'c++2b' for 'Working draft for ISO C++ 2023 DIS' standard
-// CHECK-NEXT: note: use 'gnu++2b' for 'Working draft for ISO C++ 2023 DIS with GNU extensions' standard
+// CHECK-NEXT: note: use 'c++23' for 'ISO C++ 2023 DIS' standard
+// CHECK-NEXT: note: use 'gnu++23' for 'ISO C++ 2023 DIS with GNU extensions' standard
 // CUDA-NEXT: note: use 'cuda' for 'NVIDIA CUDA(tm)' standard
 
 // Make sure that no other output is present.
diff --git a/clang/test/Driver/unsupported-option.c b/clang/test/Driver/unsupported-option.c
index 3b7a2b5af2743..3f4227b52b3ba 100644
--- a/clang/test/Driver/unsupported-option.c
+++ b/clang/test/Driver/unsupported-option.c
@@ -13,3 +13,7 @@
 // RUN: not %clang --target=powerpc64-ibm-aix %s -mlong-double-128 2>&1 | \
 // RUN: FileCheck %s --check-prefix=AIX64-LONGDOUBLE128-ERR
 // AIX64-LONGDOUBLE128-ERR: error: unsupported option '-mlong-double-128' for target 'powerpc64-ibm-aix'
+
+// RUN: not %clang -fprofile-sample-use=code.prof --target=powerpc-ibm-aix %s 2>&1 | \
+// RUN: FileCheck %s --check-prefix=AIX-PROFILE-SAMPLE
+// AIX-PROFILE-SAMPLE: error: unsupported option '-fprofile-sample-use=' for target
diff --git a/clang/test/Driver/unsupported-target-arch.c b/clang/test/Driver/unsupported-target-arch.c
index 7c55850f49943..b43a77e0486d4 100644
--- a/clang/test/Driver/unsupported-target-arch.c
+++ b/clang/test/Driver/unsupported-target-arch.c
@@ -2,24 +2,32 @@
 //
 // RUN: not %clang --target=noarch-unknown-linux -o %t.o %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-LINUX %s
-// CHECK-NOARCH-LINUX: error: unknown target triple 'noarch-unknown-linux', please use -triple or -arch
+// CHECK-NOARCH-LINUX: error: unknown target triple 'noarch-unknown-linux'{{$}}
 //
 // RUN: not %clang --target=noarch-unknown-darwin -o %t.o %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-DARWIN %s
-// CHECK-NOARCH-DARWIN: error: unknown target triple 'unknown-unknown-macosx{{.+}}', please use -triple or -arch
+// CHECK-NOARCH-DARWIN: error: unknown target triple 'unknown-unknown-macosx{{.+}}'{{$}}
 //
 // RUN: not %clang --target=noarch-unknown-windows -o %t.o %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-WINDOWS %s
-// CHECK-NOARCH-WINDOWS: error: unknown target triple 'noarch-unknown-windows-{{.+}}', please use -triple or -arch
+// CHECK-NOARCH-WINDOWS: error: unknown target triple 'noarch-unknown-windows-{{.+}}'{{$}}
 //
 // RUN: not %clang --target=noarch-unknown-freebsd -o %t.o %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-FREEBSD %s
-// CHECK-NOARCH-FREEBSD: error: unknown target triple 'noarch-unknown-freebsd', please use -triple or -arch
+// CHECK-NOARCH-FREEBSD: error: unknown target triple 'noarch-unknown-freebsd'{{$}}
 //
 // RUN: not %clang --target=noarch-unknown-netbsd -o %t.o %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-NETBSD %s
-// CHECK-NOARCH-NETBSD: error: unknown target triple 'noarch-unknown-netbsd', please use -triple or -arch
+// CHECK-NOARCH-NETBSD: error: unknown target triple 'noarch-unknown-netbsd'{{$}}
 //
 // RUN: not %clang --target=noarch-unknown-nacl -o %t.o %s 2> %t.err
 // RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-NACL %s
 // CHECK-NOARCH-NACL:  error: the target architecture 'noarch' is not supported by the target 'Native Client'
+
+// RUN: not %clang --target=noarch-unknown-windows-gnu -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-MINGW %s
+// CHECK-NOARCH-MINGW: error: unknown target triple 'noarch-unknown-windows-gnu'
+
+// RUN: not %clang --target=noarch-unknown-windows-itanium -o %t.o %s 2> %t.err
+// RUN: FileCheck --input-file=%t.err --check-prefix=CHECK-NOARCH-CROSSWINDOWS %s
+// CHECK-NOARCH-CROSSWINDOWS: error: unknown target triple 'noarch-unknown-windows-itanium'
diff --git a/clang/test/ExtractAPI/availability.c b/clang/test/ExtractAPI/availability.c
index 7d071909a092e..0c8cd3629f3fd 100644
--- a/clang/test/ExtractAPI/availability.c
+++ b/clang/test/ExtractAPI/availability.c
@@ -76,7 +76,7 @@ void e(void) __attribute__((availability(tvos, unavailable)));
         },
         {
           "kind": "text",
-          "spelling": "()"
+          "spelling": "();"
         }
       ],
       "functionSignature": {
@@ -150,7 +150,7 @@ void e(void) __attribute__((availability(tvos, unavailable)));
         },
         {
           "kind": "text",
-          "spelling": "()"
+          "spelling": "();"
         }
       ],
       "functionSignature": {
@@ -234,7 +234,7 @@ void e(void) __attribute__((availability(tvos, unavailable)));
         },
         {
           "kind": "text",
-          "spelling": "()"
+          "spelling": "();"
         }
       ],
       "functionSignature": {
@@ -334,7 +334,7 @@ void e(void) __attribute__((availability(tvos, unavailable)));
         },
         {
           "kind": "text",
-          "spelling": "()"
+          "spelling": "();"
         }
       ],
       "functionSignature": {
@@ -416,7 +416,7 @@ void e(void) __attribute__((availability(tvos, unavailable)));
         },
         {
           "kind": "text",
-          "spelling": "()"
+          "spelling": "();"
         }
       ],
       "functionSignature": {
diff --git a/clang/test/ExtractAPI/global_record.c b/clang/test/ExtractAPI/global_record.c
index 8516ac50be858..7ca89875d66ae 100644
--- a/clang/test/ExtractAPI/global_record.c
+++ b/clang/test/ExtractAPI/global_record.c
@@ -189,7 +189,7 @@ char unavailable __attribute__((unavailable));
         },
         {
           "kind": "text",
-          "spelling": ")"
+          "spelling": ");"
         }
       ],
       "docComment": {
diff --git a/clang/test/ExtractAPI/global_record_multifile.c b/clang/test/ExtractAPI/global_record_multifile.c
index 0eacb8a94b77f..b1e9f9ed21d42 100644
--- a/clang/test/ExtractAPI/global_record_multifile.c
+++ b/clang/test/ExtractAPI/global_record_multifile.c
@@ -191,7 +191,7 @@ char unavailable __attribute__((unavailable));
         },
         {
           "kind": "text",
-          "spelling": ")"
+          "spelling": ");"
         }
       ],
       "docComment": {
diff --git a/clang/test/ExtractAPI/macro_undefined.c b/clang/test/ExtractAPI/macro_undefined.c
index c7d3dd6cfea8c..f150c10fb2ed6 100644
--- a/clang/test/ExtractAPI/macro_undefined.c
+++ b/clang/test/ExtractAPI/macro_undefined.c
@@ -67,7 +67,7 @@ FUNC_GEN(bar, const int *, unsigned);
         },
         {
           "kind": "text",
-          "spelling": "()"
+          "spelling": "();"
         }
       ],
       "functionSignature": {
@@ -173,7 +173,7 @@ FUNC_GEN(bar, const int *, unsigned);
         },
         {
           "kind": "text",
-          "spelling": ")"
+          "spelling": ");"
         }
       ],
       "functionSignature": {
diff --git a/clang/test/FixIt/fixit-c++11.cpp b/clang/test/FixIt/fixit-c++11.cpp
index 9a2020c41870a..a5a47b7c937ba 100644
--- a/clang/test/FixIt/fixit-c++11.cpp
+++ b/clang/test/FixIt/fixit-c++11.cpp
@@ -59,8 +59,8 @@ void S2::f(int i) {
   (void)[] mutable {};
   (void)[]->int{};
 #if __cplusplus <= 202002L
-  // expected-warning@-3{{is a C++2b extension}}
-  // expected-warning@-3{{is a C++2b extension}}
+  // expected-warning@-3{{is a C++23 extension}}
+  // expected-warning@-3{{is a C++23 extension}}
 #endif
 
   delete []() { return new int; }(); // expected-error{{'[]' after delete interpreted as 'delete[]'}}
diff --git a/clang/test/Headers/__clang_hip_math.hip b/clang/test/Headers/__clang_hip_math.hip
index ec774c43f8124..e546ce5ac24ab 100644
--- a/clang/test/Headers/__clang_hip_math.hip
+++ b/clang/test/Headers/__clang_hip_math.hip
@@ -767,6 +767,7 @@ extern "C" __device__ float test_exp2f(float x) {
   return exp2f(x);
 }
 
+//
 // DEFAULT-LABEL: @test_exp2(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call contract double @__ocml_exp2_f64(double noundef [[X:%.*]]) #[[ATTR14]]
@@ -937,13 +938,13 @@ extern "C" __device__ double test_floor(double x) {
 
 // DEFAULT-LABEL: @test_fmaf(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call contract float @__ocml_fma_f32(float noundef [[X:%.*]], float noundef [[Y:%.*]], float noundef [[Z:%.*]]) #[[ATTR13]]
-// DEFAULT-NEXT:    ret float [[CALL_I]]
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+// DEFAULT-NEXT:    ret float [[TMP0]]
 //
 // FINITEONLY-LABEL: @test_fmaf(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call nnan ninf contract nofpclass(nan inf) float @__ocml_fma_f32(float noundef nofpclass(nan inf) [[X:%.*]], float noundef nofpclass(nan inf) [[Y:%.*]], float noundef nofpclass(nan inf) [[Z:%.*]]) #[[ATTR13]]
-// FINITEONLY-NEXT:    ret float [[CALL_I]]
+// FINITEONLY-NEXT:    [[TMP0:%.*]] = tail call nnan ninf contract float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+// FINITEONLY-NEXT:    ret float [[TMP0]]
 //
 extern "C" __device__ float test_fmaf(float x, float y, float z) {
   return fmaf(x, y, z);
@@ -951,13 +952,13 @@ extern "C" __device__ float test_fmaf(float x, float y, float z) {
 
 // DEFAULT-LABEL: @test_fma(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call contract double @__ocml_fma_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], double noundef [[Z:%.*]]) #[[ATTR13]]
-// DEFAULT-NEXT:    ret double [[CALL_I]]
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract double @llvm.fma.f64(double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]])
+// DEFAULT-NEXT:    ret double [[TMP0]]
 //
 // FINITEONLY-LABEL: @test_fma(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call nnan ninf contract nofpclass(nan inf) double @__ocml_fma_f64(double noundef nofpclass(nan inf) [[X:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) #[[ATTR13]]
-// FINITEONLY-NEXT:    ret double [[CALL_I]]
+// FINITEONLY-NEXT:    [[TMP0:%.*]] = tail call nnan ninf contract double @llvm.fma.f64(double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]])
+// FINITEONLY-NEXT:    ret double [[TMP0]]
 //
 extern "C" __device__ double test_fma(double x, double y, double z) {
   return fma(x, y, z);
@@ -965,13 +966,13 @@ extern "C" __device__ double test_fma(double x, double y, double z) {
 
 // DEFAULT-LABEL: @test_fma_rn(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call contract double @__ocml_fma_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], double noundef [[Z:%.*]]) #[[ATTR13]]
-// DEFAULT-NEXT:    ret double [[CALL_I]]
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract double @llvm.fma.f64(double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]])
+// DEFAULT-NEXT:    ret double [[TMP0]]
 //
 // FINITEONLY-LABEL: @test_fma_rn(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call nnan ninf contract nofpclass(nan inf) double @__ocml_fma_f64(double noundef nofpclass(nan inf) [[X:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) #[[ATTR13]]
-// FINITEONLY-NEXT:    ret double [[CALL_I]]
+// FINITEONLY-NEXT:    [[TMP0:%.*]] = tail call nnan ninf contract double @llvm.fma.f64(double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]])
+// FINITEONLY-NEXT:    ret double [[TMP0]]
 //
 extern "C" __device__ double test_fma_rn(double x, double y, double z) {
   return __fma_rn(x, y, z);
@@ -3414,13 +3415,13 @@ extern "C" __device__ float test___fdividef(float x, float y) {
 
 // DEFAULT-LABEL: @test__fmaf_rn(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call contract float @__ocml_fma_f32(float noundef [[X:%.*]], float noundef [[Y:%.*]], float noundef [[Z:%.*]]) #[[ATTR13]]
-// DEFAULT-NEXT:    ret float [[CALL_I]]
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+// DEFAULT-NEXT:    ret float [[TMP0]]
 //
 // FINITEONLY-LABEL: @test__fmaf_rn(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call nnan ninf contract nofpclass(nan inf) float @__ocml_fma_f32(float noundef nofpclass(nan inf) [[X:%.*]], float noundef nofpclass(nan inf) [[Y:%.*]], float noundef nofpclass(nan inf) [[Z:%.*]]) #[[ATTR13]]
-// FINITEONLY-NEXT:    ret float [[CALL_I]]
+// FINITEONLY-NEXT:    [[TMP0:%.*]] = tail call nnan ninf contract float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+// FINITEONLY-NEXT:    ret float [[TMP0]]
 //
 extern "C" __device__ float test__fmaf_rn(float x, float y, float z) {
   return __fmaf_rn(x, y, z);
@@ -3692,13 +3693,13 @@ extern "C" __device__ double test___dsqrt_rn(double x) {
 
 // DEFAULT-LABEL: @test__fma_rn(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call contract double @__ocml_fma_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], double noundef [[Z:%.*]]) #[[ATTR13]]
-// DEFAULT-NEXT:    ret double [[CALL_I]]
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract double @llvm.fma.f64(double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]])
+// DEFAULT-NEXT:    ret double [[TMP0]]
 //
 // FINITEONLY-LABEL: @test__fma_rn(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call nnan ninf contract nofpclass(nan inf) double @__ocml_fma_f64(double noundef nofpclass(nan inf) [[X:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], double noundef nofpclass(nan inf) [[Z:%.*]]) #[[ATTR13]]
-// FINITEONLY-NEXT:    ret double [[CALL_I]]
+// FINITEONLY-NEXT:    [[TMP0:%.*]] = tail call nnan ninf contract double @llvm.fma.f64(double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]])
+// FINITEONLY-NEXT:    ret double [[TMP0]]
 //
 extern "C" __device__ double test__fma_rn(double x, double y, double z) {
   return __fma_rn(x, y, z);
diff --git a/clang/test/Interpreter/Inputs/dynamic-library-test.cpp b/clang/test/Interpreter/Inputs/dynamic-library-test.cpp
deleted file mode 100644
index 1f143ba040cb6..0000000000000
--- a/clang/test/Interpreter/Inputs/dynamic-library-test.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-int ultimate_answer = 0;
-
-int calculate_answer() {
-  ultimate_answer = 42;
-  return 5;
-}
diff --git a/clang/test/Interpreter/Inputs/libdynamic-library-test.so b/clang/test/Interpreter/Inputs/libdynamic-library-test.so
new file mode 100755
index 0000000000000..bb7c7b5a54317
Binary files /dev/null and b/clang/test/Interpreter/Inputs/libdynamic-library-test.so differ
diff --git a/clang/test/Interpreter/disambiguate-decl-stmt.cpp b/clang/test/Interpreter/disambiguate-decl-stmt.cpp
index 0022ad7f546e6..6f97310475980 100644
--- a/clang/test/Interpreter/disambiguate-decl-stmt.cpp
+++ b/clang/test/Interpreter/disambiguate-decl-stmt.cpp
@@ -26,6 +26,13 @@ I x = 10;
 x.I::~I();
 x = 20;
 
+struct Dtor1 {~Dtor1();};
+Dtor1::~Dtor1() { printf("Dtor1\n"); }
+Dtor1 d1;
+
+struct ANestedDtor { struct A1 { struct A2 { ~A2(); }; }; };
+ANestedDtor::A1::A2::~A2() { printf("Dtor A::A1::A2::~A2\n"); }
+
 // Ctors
 
 // Deduction guide
diff --git a/clang/test/Interpreter/dynamic-library.cpp b/clang/test/Interpreter/dynamic-library.cpp
index 145c58bc2ae26..6c4621f729c1c 100644
--- a/clang/test/Interpreter/dynamic-library.cpp
+++ b/clang/test/Interpreter/dynamic-library.cpp
@@ -1,13 +1,25 @@
-// REQUIRES: host-supports-jit, system-linux
-// UNSUPPORTED: target={{.*-(ps4|ps5)}}
+// REQUIRES: host-supports-jit, x86_64-linux
 
-// RUN: %clang -xc++ -o %T/libdynamic-library-test.so -fPIC -shared -DLIBRARY %S/Inputs/dynamic-library-test.cpp
-// RUN: cat %s | env LD_LIBRARY_PATH=%T:$LD_LIBRARY_PATH clang-repl | FileCheck %s
+// To generate libdynamic-library-test.so :
+// clang -xc++ -o libdynamic-library-test.so -fPIC -shared
+//
+// extern "C" {
+//
+// int ultimate_answer = 0;
+// 
+// int calculate_answer() {
+//   ultimate_answer = 42;
+//   return 5;
+// }
+//
+// }
 
-#include <cstdio>
+// RUN: cat %s | env LD_LIBRARY_PATH=%S/Inputs:$LD_LIBRARY_PATH clang-repl | FileCheck %s
 
-extern int ultimate_answer;
-int calculate_answer();
+extern "C" int printf(const char* format, ...);
+
+extern "C" int ultimate_answer;
+extern "C" int calculate_answer();
 
 %lib libdynamic-library-test.so
 
diff --git a/clang/test/Lexer/char-escapes-delimited.c b/clang/test/Lexer/char-escapes-delimited.c
index 3b1deffe936db..5327ef700b0e2 100644
--- a/clang/test/Lexer/char-escapes-delimited.c
+++ b/clang/test/Lexer/char-escapes-delimited.c
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -x c++ -std=gnu++11 -fsyntax-only -pedantic -verify=ext,expected %s
 // RUN: %clang_cc1 -x c -std=gnu11 -fsyntax-only -pedantic -verify=ext,expected %s
-// RUN: %clang_cc1 -x c++ -std=c++2b -fsyntax-only -pedantic -verify=cxx2b,expected -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -x c++ -std=c++23 -fsyntax-only -pedantic -verify=cxx23,expected -Wpre-c++23-compat %s
 // RUN: %clang_cc1 -x c++ -std=gnu++11 -fwchar-type=short -fno-signed-wchar -fsyntax-only -pedantic -verify=ext,expected %s
 // RUN: %clang_cc1 -x c -std=gnu11 -fwchar-type=short -fno-signed-wchar -fsyntax-only -pedantic -verify=ext,expected %s
 // RUN: %clang_cc1 -x c++ -std=c++17 -ftrigraphs -fsyntax-only -pedantic -verify=ext,expected -DTRIGRAPHS=1 %s
@@ -21,47 +21,47 @@ const char *errors =
 
 void ucn(void) {
   char a = '\u{1234}'; // expected-error {{character too large for enclosing character literal type}} \
-                       // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+                       // ext-warning {{extension}} cxx23-warning {{C++23}}
 
-  unsigned b = U'\u{1234}';   // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned b = U'\u{1234}';   // ext-warning {{extension}} cxx23-warning {{C++23}}
 
 #ifdef __cplusplus
-  unsigned b2 = U'\u{1}';     // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned b2 = U'\u{1}';     // ext-warning {{extension}} cxx23-warning {{C++23}}
 #else
   unsigned b2 = U'\u{1}';     //expected-error {{universal character name refers to a control character}}
 #endif
 
-  unsigned c = U'\u{000000000001234}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned c = U'\u{000000000001234}'; // ext-warning {{extension}} cxx23-warning {{C++23}}
   unsigned d = U'\u{111111111}';       //expected-error {{hex escape sequence out of range}}
 }
 
 void hex(void) {
-  char a = '\x{1}';             // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  char a = '\x{1}';             // ext-warning {{extension}} cxx23-warning {{C++23}}
   char b = '\x{abcdegggggabc}'; // expected-error 5{{invalid digit 'g' in escape sequence}}
   char c = '\x{ff1}';           // expected-error {{hex escape sequence out of range}}
 
 #if __WCHAR_MAX__ > 0xFFFF
-  unsigned d = L'\x{FFFFFFFF}';  // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned d = L'\x{FFFFFFFF}';  // ext-warning {{extension}} cxx23-warning {{C++23}}
   unsigned e = L'\x{100000000}'; // expected-error {{hex escape sequence out of range}}
 #else
-  unsigned f = L'\x{FFFF}';   // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned f = L'\x{FFFF}';   // ext-warning {{extension}} cxx23-warning {{C++23}}
   unsigned g = L'\x{10000}';  // expected-error {{hex escape sequence out of range}}
 #endif
-  unsigned h = U'\x{FFFFFFFF}';  // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned h = U'\x{FFFFFFFF}';  // ext-warning {{extension}} cxx23-warning {{C++23}}
   unsigned i = U'\x{100000000}'; // expected-error {{hex escape sequence out of range}}
 }
 
 void octal(void) {
-  char a = '\o{1}';              // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  char a = '\o{1}';              // ext-warning {{extension}} cxx23-warning {{C++23}}
   char b = '\o{12345678881238}'; // expected-error 4{{invalid digit '8' in escape sequence}}
   char c = '\o{777}';            // //expected-error {{octal escape sequence out of range}}
 #if __WCHAR_MAX__ > 0xFFFF
-  unsigned d = L'\o{37777777777}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned d = L'\o{37777777777}'; // ext-warning {{extension}} cxx23-warning {{C++23}}
   unsigned e = L'\o{40000000000}'; // expected-error {{octal escape sequence out of range}}
   unsigned f = L'\o{100000000000}'; // expected-error {{octal escape sequence out of range}}
   unsigned g = L'\o{200000000000}'; // expected-error {{octal escape sequence out of range}}
 #else
-  unsigned d = L'\o{177777}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned d = L'\o{177777}'; // ext-warning {{extension}} cxx23-warning {{C++23}}
   unsigned e = L'\o{200000}'; // expected-error {{octal escape sequence out of range}}
 #endif
 }
@@ -78,9 +78,9 @@ void concat(void) {
 
 void named(void) {
   char a = '\N{LOTUS}'; // expected-error{{character too large for enclosing character literal type}} \
-                        // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+                        // ext-warning {{extension}} cxx23-warning {{C++23}}
 
-  char b  = '\N{DOLLAR SIGN}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  char b  = '\N{DOLLAR SIGN}'; // ext-warning {{extension}} cxx23-warning {{C++23}}
   char b_ = '\N{ DOL-LAR _SIGN }'; // expected-error {{' DOL-LAR _SIGN ' is not a valid Unicode character name}} \
                                // expected-note {{characters names in Unicode escape sequences are sensitive to case and whitespaces}}
 
@@ -89,13 +89,13 @@ void named(void) {
   char d = '\N{}';          // expected-error {{delimited escape sequence cannot be empty}}
   char e = '\N{';           // expected-error {{incomplete universal character name}}
 
-  unsigned f = L'\N{GREEK CAPITAL LETTER DELTA}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned f = L'\N{GREEK CAPITAL LETTER DELTA}'; // ext-warning {{extension}} cxx23-warning {{C++23}}
 
   unsigned g = u'\N{LOTUS}'; // expected-error {{character too large for enclosing character literal type}} \
-                             // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+                             // ext-warning {{extension}} cxx23-warning {{C++23}}
 
-  unsigned h = U'\N{LOTUS}';                      // ext-warning {{extension}} cxx2b-warning {{C++2b}}
-  unsigned i = u'\N{GREEK CAPITAL LETTER DELTA}'; // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+  unsigned h = U'\N{LOTUS}';                      // ext-warning {{extension}} cxx23-warning {{C++23}}
+  unsigned i = u'\N{GREEK CAPITAL LETTER DELTA}'; // ext-warning {{extension}} cxx23-warning {{C++23}}
   char j = '\NN';                                 // expected-error {{expected '{' after '\N' escape sequence}} expected-warning {{multi-character character constant}}
   unsigned k = u'\N{LOTUS';                       // expected-error {{incomplete universal character name}}
 
@@ -115,11 +115,11 @@ void separators(void) {
                  // expected-warning@-2 3{{expression result unused}}
 }
 
-#if L'\N{GREEK CAPITAL LETTER GAMMA}' != L'Γ' // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+#if L'\N{GREEK CAPITAL LETTER GAMMA}' != L'Γ' // ext-warning {{extension}} cxx23-warning {{C++23}}
 #error "oh no!"
 #endif
 
 #ifdef TRIGRAPHS
 static_assert('\N??<DOLLAR SIGN??>' == '$'); // expected-warning 2{{trigraph converted}} \
-                                             // ext-warning {{extension}} cxx2b-warning {{C++2b}}
+                                             // ext-warning {{extension}} cxx23-warning {{C++23}}
 #endif
diff --git a/clang/test/Lexer/char-literal.cpp b/clang/test/Lexer/char-literal.cpp
index a71500f8f0107..f2d72280e66cd 100644
--- a/clang/test/Lexer/char-literal.cpp
+++ b/clang/test/Lexer/char-literal.cpp
@@ -1,5 +1,8 @@
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -Wfour-char-constants -fsyntax-only -verify %s
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c11 -x c -Wfour-char-constants -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -Wfour-char-constants -fsyntax-only -verify=cxx,expected %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++17 -Wfour-char-constants -fsyntax-only -verify=cxx,expected %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++20 -Wfour-char-constants -fsyntax-only -verify=cxx,expected %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c11 -x c -Wfour-char-constants -fsyntax-only -verify=c,expected %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c2x -x c -Wfour-char-constants -fsyntax-only -verify=c,expected %s
 
 #ifndef __cplusplus
 typedef __WCHAR_TYPE__ wchar_t;
@@ -38,3 +41,81 @@ char16_t q[2] = u"\U00010000";
 #ifdef __cplusplus
 // expected-error@-2 {{too long}}
 #endif
+
+// UTF-8 character literal code point ranges.
+#if __cplusplus >= 201703L || __STDC_VERSION__ >= 201710L
+_Static_assert(u8'\U00000000' == 0x00, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(u8'\U0000007F' == 0x7F, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(u8'\U00000080', ""); // c-error {{universal character name refers to a control character}}
+                                    // cxx-error@-1 {{character too large for enclosing character literal type}}
+_Static_assert((unsigned char)u8'\xFF' == (unsigned char)0xFF, "");
+#endif
+
+// UTF-8 string literal code point ranges.
+_Static_assert(u8"\U00000000"[0] == 0x00, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(u8"\U0000007F"[0] == 0x7F, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert((unsigned char)u8"\U00000080"[0] == (unsigned char)0xC2, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert((unsigned char)u8"\U00000080"[1] == (unsigned char)0x80, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert((unsigned char)u8"\U000007FF"[0] == (unsigned char)0xDF, "");
+_Static_assert((unsigned char)u8"\U000007FF"[1] == (unsigned char)0xBF, "");
+_Static_assert((unsigned char)u8"\U00000800"[0] == (unsigned char)0xE0, "");
+_Static_assert((unsigned char)u8"\U00000800"[1] == (unsigned char)0xA0, "");
+_Static_assert((unsigned char)u8"\U00000800"[2] == (unsigned char)0x80, "");
+_Static_assert(u8"\U0000D800"[0], ""); // expected-error {{invalid universal character}}
+_Static_assert(u8"\U0000DFFF"[0], ""); // expected-error {{invalid universal character}}
+_Static_assert((unsigned char)u8"\U0000FFFF"[0] == (unsigned char)0xEF, "");
+_Static_assert((unsigned char)u8"\U0000FFFF"[1] == (unsigned char)0xBF, "");
+_Static_assert((unsigned char)u8"\U0000FFFF"[2] == (unsigned char)0xBF, "");
+_Static_assert((unsigned char)u8"\U00010000"[0] == (unsigned char)0xF0, "");
+_Static_assert((unsigned char)u8"\U00010000"[1] == (unsigned char)0x90, "");
+_Static_assert((unsigned char)u8"\U00010000"[2] == (unsigned char)0x80, "");
+_Static_assert((unsigned char)u8"\U00010000"[3] == (unsigned char)0x80, "");
+_Static_assert((unsigned char)u8"\U0010FFFF"[0] == (unsigned char)0xF4, "");
+_Static_assert((unsigned char)u8"\U0010FFFF"[1] == (unsigned char)0x8F, "");
+_Static_assert((unsigned char)u8"\U0010FFFF"[2] == (unsigned char)0xBF, "");
+_Static_assert((unsigned char)u8"\U0010FFFF"[3] == (unsigned char)0xBF, "");
+_Static_assert(u8"\U00110000"[0], ""); // expected-error {{invalid universal character}}
+
+#if !defined(__STDC_UTF_16__)
+#error __STDC_UTF_16__ is not defined.
+#endif
+#if __STDC_UTF_16__ != 1
+#error __STDC_UTF_16__ has the wrong value.
+#endif
+
+// UTF-16 character literal code point ranges.
+_Static_assert(u'\U00000000' == 0x0000, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(u'\U0000D800', ""); // expected-error {{invalid universal character}}
+_Static_assert(u'\U0000DFFF', ""); // expected-error {{invalid universal character}}
+_Static_assert(u'\U0000FFFF' == 0xFFFF, "");
+_Static_assert(u'\U00010000', ""); // expected-error {{character too large for enclosing character literal type}}
+
+// UTF-16 string literal code point ranges.
+_Static_assert(u"\U00000000"[0] == 0x0000, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(u"\U0000D800"[0], ""); // expected-error {{invalid universal character}}
+_Static_assert(u"\U0000DFFF"[0], ""); // expected-error {{invalid universal character}}
+_Static_assert(u"\U0000FFFF"[0] == 0xFFFF, "");
+_Static_assert(u"\U00010000"[0] == 0xD800, "");
+_Static_assert(u"\U00010000"[1] == 0xDC00, "");
+_Static_assert(u"\U0010FFFF"[0] == 0xDBFF, "");
+_Static_assert(u"\U0010FFFF"[1] == 0xDFFF, "");
+_Static_assert(u"\U00110000"[0], ""); // expected-error {{invalid universal character}}
+
+#if !defined(__STDC_UTF_32__)
+#error __STDC_UTF_32__ is not defined.
+#endif
+#if __STDC_UTF_32__ != 1
+#error __STDC_UTF_32__ has the wrong value.
+#endif
+
+// UTF-32 character literal code point ranges.
+_Static_assert(U'\U00000000' == 0x00000000, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(U'\U0010FFFF' == 0x0010FFFF, "");
+_Static_assert(U'\U00110000', ""); // expected-error {{invalid universal character}}
+
+// UTF-32 string literal code point ranges.
+_Static_assert(U"\U00000000"[0] == 0x00000000, ""); // c-error {{universal character name refers to a control character}}
+_Static_assert(U"\U0000D800"[0], ""); // expected-error {{invalid universal character}}
+_Static_assert(U"\U0000DFFF"[0], ""); // expected-error {{invalid universal character}}
+_Static_assert(U"\U0010FFFF"[0] == 0x0010FFFF, "");
+_Static_assert(U"\U00110000"[0], ""); // expected-error {{invalid universal character}}
diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp
index ca7f987b78ab5..5d157ea2fac1c 100644
--- a/clang/test/Lexer/cxx-features.cpp
+++ b/clang/test/Lexer/cxx-features.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 -fcxx-exceptions -fsized-deallocation -verify %s
 // RUN: %clang_cc1 -std=c++17 -fcxx-exceptions -fsized-deallocation -verify %s
 // RUN: %clang_cc1 -std=c++20 -fcxx-exceptions -fsized-deallocation -verify %s
-// RUN: %clang_cc1 -std=c++2b -fcxx-exceptions -fsized-deallocation -verify %s
+// RUN: %clang_cc1 -std=c++23 -fcxx-exceptions -fsized-deallocation -verify %s
 //
 // RUN: %clang_cc1 -std=c++17 -fcxx-exceptions -fsized-deallocation -frelaxed-template-template-args -DRELAXED_TEMPLATE_TEMPLATE_ARGS=1 -verify %s
 // RUN: %clang_cc1 -std=c++17 -fcxx-exceptions -fsized-deallocation -DCONCEPTS_TS=1 -verify %s
@@ -28,7 +28,7 @@
 #define check(macro, cxx98, cxx11, cxx14, cxx17, cxx20, cxx23) (cxx23 == 0 ? defined(__cpp_##macro) : __cpp_##macro != cxx23)
 #endif
 
-// --- C++2b features ---
+// --- C++23 features ---
 
 #if check(implicit_move, 0, 0, 0, 0, 0, 202011)
 #error "wrong value for __cpp_implicit_move"
diff --git a/clang/test/Lexer/size_t-literal.cpp b/clang/test/Lexer/size_t-literal.cpp
index 3e3e8094b070d..d056feae39729 100644
--- a/clang/test/Lexer/size_t-literal.cpp
+++ b/clang/test/Lexer/size_t-literal.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s
 
 #if 1z != 1
 #error "z suffix must be recognized by preprocessor"
diff --git a/clang/test/Misc/serialized-diags-unknown-target.c b/clang/test/Misc/serialized-diags-unknown-target.c
index 040dfa4b2849e..c6d18811333df 100644
--- a/clang/test/Misc/serialized-diags-unknown-target.c
+++ b/clang/test/Misc/serialized-diags-unknown-target.c
@@ -1,4 +1,4 @@
 // RUN: rm -rf %t && mkdir %t
 // RUN: not %clang_cc1 %s -triple blah-unknown-unknown -serialize-diagnostic-file %t/diag -o /dev/null 2>&1 | FileCheck %s
 
-// CHECK: error: unknown target triple 'blah-unknown-unknown', please use -triple or -arch
+// CHECK: error: unknown target triple 'blah-unknown-unknown'{{$}}
diff --git a/clang/test/Modules/gmodules-deduction-guide.cpp b/clang/test/Modules/gmodules-deduction-guide.cpp
index 6b5a4086b3635..eab7db653ee55 100644
--- a/clang/test/Modules/gmodules-deduction-guide.cpp
+++ b/clang/test/Modules/gmodules-deduction-guide.cpp
@@ -1,6 +1,6 @@
 // REQUIRES: asserts
 
-// RUN: %clang_cc1 -std=c++2b -x c++-header -emit-pch -fmodule-format=obj -I %S/Inputs \
+// RUN: %clang_cc1 -std=c++23 -x c++-header -emit-pch -fmodule-format=obj -I %S/Inputs \
 // RUN:   -o %t.pch %S/Inputs/gmodules-deduction-guide.h \
 // RUN:   -mllvm -debug-only=pchcontainer &>%t-pch.ll
 // RUN: cat %t-pch.ll | FileCheck %s
diff --git a/clang/test/Modules/merge-constrained-friends.cpp b/clang/test/Modules/merge-constrained-friends.cpp
index 13147d566e614..8f0e9ed83cf29 100644
--- a/clang/test/Modules/merge-constrained-friends.cpp
+++ b/clang/test/Modules/merge-constrained-friends.cpp
@@ -2,8 +2,8 @@
 // RUN: mkdir -p %t
 // RUN: split-file %s %t
 //
-// RUN: %clang_cc1 -std=c++2b %t/A.cppm -emit-module-interface -o %t/A.pcm
-// RUN: %clang_cc1 -std=c++2b %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify
+// RUN: %clang_cc1 -std=c++23 %t/A.cppm -emit-module-interface -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++23 %t/Use.cpp -fprebuilt-module-path=%t -fsyntax-only -verify
 
 //--- A.cppm
 module;
diff --git a/clang/test/Modules/pr62359.cppm b/clang/test/Modules/pr62359.cppm
new file mode 100644
index 0000000000000..4632457e57f18
--- /dev/null
+++ b/clang/test/Modules/pr62359.cppm
@@ -0,0 +1,43 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/Hello.cppm -o %t/Hello.pcm
+// RUN: not %clang_cc1 -std=c++20 -fopenmp %t/use.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only \
+// RUN:     2>&1 | FileCheck %t/use.cpp
+// RUN: not %clang_cc1 -std=c++20 -fopenmp %t/use2.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only \
+// RUN:     2>&1 | FileCheck %t/use2.cpp
+//
+// RUN: %clang_cc1 -std=c++20 -fopenmp -emit-module-interface %t/Hello.cppm -o %t/Hello.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp %t/use.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only -verify
+// RUN: %clang_cc1 -std=c++20 -fopenmp %t/use2.cpp -fmodule-file=hello=%t/Hello.pcm -fsyntax-only -verify
+
+//--- Hello.cppm
+export module hello;
+export void hello() {
+  
+}
+
+//--- use.cpp
+// expected-no-diagnostics
+import hello;
+int use() {
+  for(int i=0;i<10;i++)
+    hello();
+  return 0;
+}
+
+// CHECK: OpenMP{{.*}}differs in PCH file vs. current file
+
+//--- use2.cpp
+// expected-no-diagnostics
+import hello;
+int use2() {
+#pragma omp parallel for
+  for(int i=0;i<10;i++)
+    hello();
+  return 0;
+}
+
+// CHECK: OpenMP{{.*}}differs in PCH file vs. current file
+// CHECK: use of undeclared identifier 'pragma'
diff --git a/clang/test/OpenMP/target_update_messages.cpp b/clang/test/OpenMP/target_update_messages.cpp
index f936a075e1b48..a8505626196f8 100644
--- a/clang/test/OpenMP/target_update_messages.cpp
+++ b/clang/test/OpenMP/target_update_messages.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -verify=expected,ge50,lt51 -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 // RUN: %clang_cc1 -verify=expected,ge50,ge51 -fopenmp-simd -fopenmp-version=51 -ferror-limit 100 -o - -std=c++11 %s -Wuninitialized
 
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,cxx2b -fopenmp -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++2b %s -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,cxx23 -fopenmp -fopenmp-simd -fopenmp-version=51 -x c++ -std=c++23 %s -Wuninitialized
 
 void xxx(int argc) {
   int x; // expected-note {{initialize the variable 'x' to silence this warning}}
@@ -228,7 +228,7 @@ struct bar {
 
 #if defined(__cplusplus) && __cplusplus >= 202101L
 
-namespace cxx2b {
+namespace cxx23 {
 
 struct S {
   int operator[](auto...);
@@ -240,22 +240,22 @@ void f() {
 
 #pragma omp target update to(test[1])
 
-#pragma omp target update to(test[1, 2]) // cxx2b-error {{type 'int[10]' does not provide a subscript operator}} \
-                                         // cxx2b-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(test[1, 2]) // cxx23-error {{type 'int[10]' does not provide a subscript operator}} \
+                                         // cxx23-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 
 #pragma omp target update to(test [1:1:1])
 
-#pragma omp target update to(test [1, 2:1:1]) // cxx2b-error {{expected ']'}} // expected-note {{'['}} \
-                                            // cxx2b-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(test [1, 2:1:1]) // cxx23-error {{expected ']'}} // expected-note {{'['}} \
+                                            // cxx23-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 
-#pragma omp target update to(test [1, 2:]) // cxx2b-error {{expected ']'}} // expected-note {{'['}} \
-                                            // cxx2b-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(test [1, 2:]) // cxx23-error {{expected ']'}} // expected-note {{'['}} \
+                                            // cxx23-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 
-#pragma omp target update to(test[1, 2 ::]) // cxx2b-error {{expected ']'}} // expected-note {{'['}} \
-                                            // cxx2b-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(test[1, 2 ::]) // cxx23-error {{expected ']'}} // expected-note {{'['}} \
+                                            // cxx23-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 
-#pragma omp target update to(test[]) // cxx2b-error {{type 'int[10]' does not provide a subscript operator}} \
-                                            // cxx2b-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(test[]) // cxx23-error {{type 'int[10]' does not provide a subscript operator}} \
+                                            // cxx23-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
   S s;
   (void)s[0];
   (void)s[];
diff --git a/clang/test/Parser/cxx-concepts-requires-clause.cpp b/clang/test/Parser/cxx-concepts-requires-clause.cpp
index e4bdd28f61ec1..1ec1eefa12865 100644
--- a/clang/test/Parser/cxx-concepts-requires-clause.cpp
+++ b/clang/test/Parser/cxx-concepts-requires-clause.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2a -x c++ %s -verify
-// RUN: %clang_cc1 -std=c++2b -x c++ %s -verify
+// RUN: %clang_cc1 -std=c++20 -x c++ %s -verify
+// RUN: %clang_cc1 -std=c++23 -x c++ %s -verify
 
 // Test parsing of the optional requires-clause in a template-declaration.
 
@@ -166,5 +166,5 @@ auto lambda3 = []<auto> requires(sizeof(char) == 1){};
 
 auto lambda4 = [] requires(sizeof(char) == 1){}; // expected-error {{expected body of lambda expression}}
 #if __cplusplus <= 202002L
-// expected-warning@-2{{lambda without a parameter clause is a C++2b extension}}
+// expected-warning@-2{{lambda without a parameter clause is a C++23 extension}}
 #endif
diff --git a/clang/test/Parser/cxx-member-initializers.cpp b/clang/test/Parser/cxx-member-initializers.cpp
index c29260b731223..57f707cee52be 100644
--- a/clang/test/Parser/cxx-member-initializers.cpp
+++ b/clang/test/Parser/cxx-member-initializers.cpp
@@ -108,4 +108,9 @@ class G {
   // expected-error@-2 {{type name requires a specifier or qualifier}}
   // expected-error@-3 {{expected '>'}}
   // expected-note@-4 {{to match this '<'}}
+
+  void n(int x = 1 ? 2) {}
+  // expected-error@-1 {{expected ':'}}
+  // expected-note@-2 {{to match this '?'}}
+  // expected-error@-3 {{expected expression}}
 };
diff --git a/clang/test/Parser/cxx0x-lambda-expressions.cpp b/clang/test/Parser/cxx0x-lambda-expressions.cpp
index 00a9b70ccd016..72b315a497c06 100644
--- a/clang/test/Parser/cxx0x-lambda-expressions.cpp
+++ b/clang/test/Parser/cxx0x-lambda-expressions.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++11 -Wno-c99-designator %s
-// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++2a -Wno-c99-designator %s
-// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++2b -Wno-c99-designator %s
+// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++20 -Wno-c99-designator %s
+// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++23 -Wno-c99-designator %s
 
 enum E { e };
 
@@ -31,8 +31,8 @@ class C {
     [] -> int { return 0; };
     [] mutable -> int { return 0; };
 #if __cplusplus <= 202002L
-    // expected-warning@-3 {{lambda without a parameter clause is a C++2b extension}}
-    // expected-warning@-3 {{is a C++2b extension}}
+    // expected-warning@-3 {{lambda without a parameter clause is a C++23 extension}}
+    // expected-warning@-3 {{is a C++23 extension}}
 #endif
     [](int) -> {}; // PR13652 expected-error {{expected a type}}
     return 1;
@@ -108,7 +108,7 @@ class C {
   void attributes() {
     [] __attribute__((noreturn)){};
 #if __cplusplus <= 202002L
-    // expected-warning@-2 {{is a C++2b extension}}
+    // expected-warning@-2 {{is a C++23 extension}}
 #endif
     []() [[]]
       mutable {}; // expected-error {{expected body of lambda expression}}
@@ -128,17 +128,17 @@ class C {
     // rather than the type.
     [][[]](){};
 #if __cplusplus <= 202002L
-    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++2b extension}}
+    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}}
 #endif
 #if __cplusplus > 201703L
     []<typename>[[]](){};
 #if __cplusplus <= 202002L
-    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++2b extension}}
+    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}}
 #endif
 #endif
     [][[]]{};
 #if __cplusplus <= 202002L
-    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++2b extension}}
+    // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}}
 #endif
   }
 
@@ -146,8 +146,8 @@ class C {
     [] mutable {};
     [] noexcept {};
 #if __cplusplus <= 202002L
-    // expected-warning@-3 {{is a C++2b extension}}
-    // expected-warning@-3 {{is a C++2b extension}}
+    // expected-warning@-3 {{is a C++23 extension}}
+    // expected-warning@-3 {{is a C++23 extension}}
 #endif
   }
 };
diff --git a/clang/test/Parser/cxx1z-constexpr-lambdas.cpp b/clang/test/Parser/cxx1z-constexpr-lambdas.cpp
index 0c2f81d318a14..87584ee5ca91b 100644
--- a/clang/test/Parser/cxx1z-constexpr-lambdas.cpp
+++ b/clang/test/Parser/cxx1z-constexpr-lambdas.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b %s -verify
+// RUN: %clang_cc1 -std=c++23 %s -verify
 // RUN: %clang_cc1 -std=c++20 %s -verify
 // RUN: %clang_cc1 -std=c++17 %s -verify
 // RUN: %clang_cc1 -std=c++14 %s -verify
@@ -9,7 +9,7 @@ auto XL0 = [] constexpr { return true; };
 // expected-warning@-2 {{is a C++17 extension}}
 #endif
 #if __cplusplus <= 202002L
-// expected-warning@-5 {{lambda without a parameter clause is a C++2b extension}}
+// expected-warning@-5 {{lambda without a parameter clause is a C++23 extension}}
 #endif
 auto XL1 = []() mutable //
     mutable             // expected-error{{cannot appear multiple times}}
diff --git a/clang/test/Parser/cxx2a-template-lambdas.cpp b/clang/test/Parser/cxx2a-template-lambdas.cpp
index f85280f7ece50..5cf1a862d878c 100644
--- a/clang/test/Parser/cxx2a-template-lambdas.cpp
+++ b/clang/test/Parser/cxx2a-template-lambdas.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b %s -verify
-// RUN: %clang_cc1 -std=c++2a %s -verify
+// RUN: %clang_cc1 -std=c++23 %s -verify
+// RUN: %clang_cc1 -std=c++20 %s -verify
 
 auto L0 = []<> { }; //expected-error {{cannot be empty}}
 
@@ -14,14 +14,14 @@ auto L6 = []<auto>{};
 auto L7 = []<auto>() noexcept {};
 auto L8 = []<auto> noexcept {};
 #if __cplusplus <= 202002L
-// expected-warning@-2 {{lambda without a parameter clause is a C++2b extension}}
+// expected-warning@-2 {{lambda without a parameter clause is a C++23 extension}}
 #endif
 auto L9 = []<auto> requires true {};
 auto L10 = []<auto> requires true(){};
 auto L11 = []<auto> requires true() noexcept {};
 auto L12 = []<auto> requires true noexcept {};
 #if __cplusplus <= 202002L
-// expected-warning@-2 {{is a C++2b extension}}
+// expected-warning@-2 {{is a C++23 extension}}
 #endif
 auto L13 = []<auto>() noexcept requires true {};
 auto L14 = []<auto> requires true() noexcept requires true {};
@@ -29,6 +29,6 @@ auto L14 = []<auto> requires true() noexcept requires true {};
 auto XL0 = []<auto> noexcept requires true {};               // expected-error {{expected body of lambda expression}}
 auto XL1 = []<auto> requires true noexcept requires true {}; // expected-error {{expected body}}
 #if __cplusplus <= 202002L
-// expected-warning@-3 {{is a C++2b extension}}
-// expected-warning@-3 {{is a C++2b extension}}
+// expected-warning@-3 {{is a C++23 extension}}
+// expected-warning@-3 {{is a C++23 extension}}
 #endif
diff --git a/clang/test/Parser/cxx2b-auto-x.cpp b/clang/test/Parser/cxx2b-auto-x.cpp
index 28687dced5445..5ca11d5f39950 100644
--- a/clang/test/Parser/cxx2b-auto-x.cpp
+++ b/clang/test/Parser/cxx2b-auto-x.cpp
@@ -1,14 +1,14 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx2b -std=c++2b -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx23 -std=c++23 -Wpre-c++23-compat %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx20 -std=c++20 %s
 
 void looks_like_decltype_auto() {
   decltype(auto(42)) b = 42; // cxx20-error {{'auto' not allowed here}} \
-                                cxx2b-warning {{'auto' as a functional-style cast is incompatible with C++ standards before C++2b}}
+                                cxx23-warning {{'auto' as a functional-style cast is incompatible with C++ standards before C++23}}
   decltype(long *) a = 42;   // expected-error {{expected '(' for function-style cast or type construction}} \
                                 expected-error {{expected expression}}
   decltype(auto *) a = 42;   // expected-error {{expected '(' for function-style cast or type construction}} \
                                 expected-error {{expected expression}}
-  decltype(auto()) c = 42;   // cxx2b-error {{initializer for functional-style cast to 'auto' is empty}} \
+  decltype(auto()) c = 42;   // cxx23-error {{initializer for functional-style cast to 'auto' is empty}} \
                                 cxx20-error {{'auto' not allowed here}}
 }
 
diff --git a/clang/test/Parser/cxx2b-init-statement.cpp b/clang/test/Parser/cxx2b-init-statement.cpp
index 8f99b46edeb65..4986b402bf51d 100644
--- a/clang/test/Parser/cxx2b-init-statement.cpp
+++ b/clang/test/Parser/cxx2b-init-statement.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=expected -std=c++2b -Wall %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected -std=c++23 -Wall %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,expected-cxx20 -std=c++20 -Wall %s
 
 namespace ns {
@@ -7,13 +7,13 @@ namespace ns {
 }
 void f() {
 
-    for (using foo = int;true;); //expected-cxx20-warning {{alias declaration in this context is a C++2b extension}}
+    for (using foo = int;true;); //expected-cxx20-warning {{alias declaration in this context is a C++23 extension}}
 
-    switch(using foo = int; 0) { //expected-cxx20-warning {{alias declaration in this context is a C++2b extension}}
+    switch(using foo = int; 0) { //expected-cxx20-warning {{alias declaration in this context is a C++23 extension}}
         case 0: break;
     }
 
-    if(using foo = int; false) {} //expected-cxx20-warning {{alias declaration in this context is a C++2b extension}}
+    if(using foo = int; false) {} //expected-cxx20-warning {{alias declaration in this context is a C++23 extension}}
 
 
     if (using enum ns::e; false){}  // expected-error {{expected '='}}
diff --git a/clang/test/Parser/cxx2b-label.cpp b/clang/test/Parser/cxx2b-label.cpp
index 2560c8d71b982..a490370409cc0 100644
--- a/clang/test/Parser/cxx2b-label.cpp
+++ b/clang/test/Parser/cxx2b-label.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx2b -std=c++2b -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx23 -std=c++23 -Wpre-c++23-compat %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx20 -std=c++20 %s
 
 void test_label_in_func() {
@@ -7,8 +7,8 @@ void test_label_in_func() {
 label2:
     x = 1;
 label3: label4: label5:
-} // cxx20-warning {{label at end of compound statement is a C++2b extension}} \
-     cxx2b-warning {{label at end of compound statement is incompatible with C++ standards before C++2b}}
+} // cxx20-warning {{label at end of compound statement is a C++23 extension}} \
+     cxx23-warning {{label at end of compound statement is incompatible with C++ standards before C++23}}
 
 int test_label_in_switch(int v) {
     switch (v) {
@@ -17,15 +17,15 @@ int test_label_in_switch(int v) {
     case 2:
         return 2;
     case 3: case 4: case 5:
-    } // cxx20-warning {{label at end of compound statement is a C++2b extension}} \
-         cxx2b-warning {{label at end of compound statement is incompatible with C++ standards before C++2b}}
+    } // cxx20-warning {{label at end of compound statement is a C++23 extension}} \
+         cxx23-warning {{label at end of compound statement is incompatible with C++ standards before C++23}}
 
     switch (v) {
     case 6:
         return 6;
     default:
-    } // cxx20-warning {{label at end of compound statement is a C++2b extension}} \
-         cxx2b-warning {{label at end of compound statement is incompatible with C++ standards before C++2b}}
+    } // cxx20-warning {{label at end of compound statement is a C++23 extension}} \
+         cxx23-warning {{label at end of compound statement is incompatible with C++ standards before C++23}}
 
     return 0;
 }
diff --git a/clang/test/Parser/cxx2b-lambdas-ext-warns.cpp b/clang/test/Parser/cxx2b-lambdas-ext-warns.cpp
index 3a80f59aba565..7ffb7aae9d391 100644
--- a/clang/test/Parser/cxx2b-lambdas-ext-warns.cpp
+++ b/clang/test/Parser/cxx2b-lambdas-ext-warns.cpp
@@ -1,16 +1,16 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20
-// RUN: %clang_cc1 -std=c++2b %s -verify=cxx2b
-// RUN: %clang_cc1 -std=c++2b -Wpre-c++2b-compat %s -verify=precxx2b
-// RUN: %clang_cc1 -std=c++2b -pedantic %s -verify=cxx2b
+// RUN: %clang_cc1 -std=c++23 %s -verify=cxx23
+// RUN: %clang_cc1 -std=c++23 -Wpre-c++23-compat %s -verify=precxx23
+// RUN: %clang_cc1 -std=c++23 -pedantic %s -verify=cxx23
 
-//cxx2b-no-diagnostics
+//cxx23-no-diagnostics
 
 auto L1 = [] constexpr {};
-// cxx20-warning@-1 {{lambda without a parameter clause is a C++2b extension}}
+// cxx20-warning@-1 {{lambda without a parameter clause is a C++23 extension}}
 auto L2 = []() static {};
-// cxx20-warning@-1 {{static lambdas are a C++2b extension}}
-// precxx2b-warning@-2 {{static lambdas are incompatible with C++ standards before C++2b}}
+// cxx20-warning@-1 {{static lambdas are a C++23 extension}}
+// precxx23-warning@-2 {{static lambdas are incompatible with C++ standards before C++23}}
 auto L3 = [] static {};
-// cxx20-warning@-1 {{lambda without a parameter clause is a C++2b extension}}
-// cxx20-warning@-2 {{static lambdas are a C++2b extension}}
-// precxx2b-warning@-3 {{static lambdas are incompatible with C++ standards before C++2b}}
+// cxx20-warning@-1 {{lambda without a parameter clause is a C++23 extension}}
+// cxx20-warning@-2 {{static lambdas are a C++23 extension}}
+// precxx23-warning@-3 {{static lambdas are incompatible with C++ standards before C++23}}
diff --git a/clang/test/Parser/cxx2b-lambdas.cpp b/clang/test/Parser/cxx2b-lambdas.cpp
index 2de4b6593cd3c..bb9ed226afffa 100644
--- a/clang/test/Parser/cxx2b-lambdas.cpp
+++ b/clang/test/Parser/cxx2b-lambdas.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b %s -verify
+// RUN: %clang_cc1 -std=c++23 %s -verify
 
 auto LL0 = [] {};
 auto LL1 = []() {};
diff --git a/clang/test/Parser/cxx2b-subscript.cpp b/clang/test/Parser/cxx2b-subscript.cpp
index c56df48a379d9..478bf36272e37 100644
--- a/clang/test/Parser/cxx2b-subscript.cpp
+++ b/clang/test/Parser/cxx2b-subscript.cpp
@@ -1,34 +1,34 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx2b -std=c++2b %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx23 -std=c++23 %s
 // RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx20 -std=c++20 %s
 
-//cxx2b-no-diagnostics
+//cxx23-no-diagnostics
 
 struct S {
   constexpr int operator[](int i) {
     return i;
   }
-  constexpr int operator[](int a, int b) { // cxx20-error {{overloaded 'operator[]' cannot have more than one parameter before C++2b}}
+  constexpr int operator[](int a, int b) { // cxx20-error {{overloaded 'operator[]' cannot have more than one parameter before C++23}}
     return a + b;
   }
-  constexpr int operator[]() { // cxx20-error {{overloaded 'operator[]' cannot have no parameter before C++2b}}
+  constexpr int operator[]() { // cxx20-error {{overloaded 'operator[]' cannot have no parameter before C++23}}
     return 42;
   }
 };
 
 struct Defaults {
-  constexpr int operator[](int i = 0) { // cxx20-error {{overloaded 'operator[]' cannot have a defaulted parameter before C++2b}}
+  constexpr int operator[](int i = 0) { // cxx20-error {{overloaded 'operator[]' cannot have a defaulted parameter before C++23}}
     return 0;
   }
-  constexpr int operator[](int a, int b, int c = 0) { // cxx20-error {{overloaded 'operator[]' cannot have a defaulted parameter before C++2b}}\
-                                                         // cxx20-error {{cannot have more than one parameter before C++2b}}
+  constexpr int operator[](int a, int b, int c = 0) { // cxx20-error {{overloaded 'operator[]' cannot have a defaulted parameter before C++23}}\
+                                                         // cxx20-error {{cannot have more than one parameter before C++23}}
     return 0;
   }
 };
 
 template <typename... T>
 struct T1 {
-  constexpr auto operator[](T &&...arg); // cxx20-error {{overloaded 'operator[]' cannot have no parameter before C++2b}} \
-                                           // cxx20-error {{overloaded 'operator[]' cannot have more than one parameter before C++2b}}
+  constexpr auto operator[](T &&...arg); // cxx20-error {{overloaded 'operator[]' cannot have no parameter before C++23}} \
+                                           // cxx20-error {{overloaded 'operator[]' cannot have more than one parameter before C++23}}
 };
 
 T1<> t10;         // cxx20-note {{requested here}}
@@ -43,16 +43,16 @@ void f() {
   S s;
   (void)s[0];
   (void)s[1, 2]; // cxx20-warning {{left operand of comma operator has no effect}}\
-                   // cxx20-warning {{top-level comma expression in array subscript is deprecated in C++20 and unsupported in C++2b}}
+                   // cxx20-warning {{top-level comma expression in array subscript is deprecated in C++20 and unsupported in C++23}}
   (void)S{}[];   // cxx20-error {{expected expression}}
 
   (void)Defaults{}[1];
   (void)Defaults{}[];     // cxx20-error {{expected expression}}
   (void)Defaults{}[1, 2]; // cxx20-warning {{left operand of comma operator has no effect}}\
-                            // cxx20-warning {{top-level comma expression in array subscript is deprecated in C++20 and unsupported in C++2b}}
+                            // cxx20-warning {{top-level comma expression in array subscript is deprecated in C++20 and unsupported in C++23}}
 
   Variadic{}[]; // cxx20-error {{expected expression}}
   Variadic{}[1];
   Variadic{}[1, 2]; // cxx20-warning {{left operand of comma operator has no effect}}\
-                       // cxx20-warning {{top-level comma expression in array subscript is deprecated in C++20 and unsupported in C++2b}}
+                       // cxx20-warning {{top-level comma expression in array subscript is deprecated in C++20 and unsupported in C++23}}
 }
diff --git a/clang/test/Parser/switch-recovery.cpp b/clang/test/Parser/switch-recovery.cpp
index f422a8757d963..861e520ec4465 100644
--- a/clang/test/Parser/switch-recovery.cpp
+++ b/clang/test/Parser/switch-recovery.cpp
@@ -161,14 +161,14 @@ void missing_statement_case(int x) {
   switch (x) {
     case 1:
     case 0:
-  } // expected-warning {{label at end of compound statement is a C++2b extension}}
+  } // expected-warning {{label at end of compound statement is a C++23 extension}}
 }
 
 void missing_statement_default(int x) {
   switch (x) {
     case 0:
     default:
-  } // expected-warning {{label at end of compound statement is a C++2b extension}}
+  } // expected-warning {{label at end of compound statement is a C++23 extension}}
 }
 
 void pr19022_1() {
@@ -179,7 +179,7 @@ void pr19022_1() {
 void pr19022_1a(int x) {
   switch(x) {
   case 1  // expected-error{{expected ':' after 'case'}}
-  } // expected-warning {{label at end of compound statement is a C++2b extension}}
+  } // expected-warning {{label at end of compound statement is a C++23 extension}}
 }
 
 void pr19022_1b(int x) {
@@ -211,7 +211,7 @@ void pr19022_5(int x) {
   switch(x) {
   case 1: case // expected-error{{expected ':' after 'case'}}
   }  // expected-error{{expected expression}} \
-     // expected-warning {{label at end of compound statement is a C++2b extension}}
+     // expected-warning {{label at end of compound statement is a C++23 extension}}
 }
 
 namespace pr19022 {
diff --git a/clang/test/Driver/Inputs/rewrite-1.map b/clang/test/Preprocessor/Inputs/canonical-system-headers/a.h
similarity index 100%
rename from clang/test/Driver/Inputs/rewrite-1.map
rename to clang/test/Preprocessor/Inputs/canonical-system-headers/a.h
diff --git a/clang/test/Preprocessor/aarch64_asm_flag_output.c b/clang/test/Preprocessor/aarch64_asm_flag_output.c
new file mode 100644
index 0000000000000..28b5282132099
--- /dev/null
+++ b/clang/test/Preprocessor/aarch64_asm_flag_output.c
@@ -0,0 +1,3 @@
+// RUN: %clang -target aarch64-unknown-unknown -x c -E -dM -o - %s | FileCheck -match-full-lines %s
+
+// CHECK: #define __GCC_ASM_FLAG_OUTPUTS__ 1
diff --git a/clang/test/Preprocessor/canonical-system-headers.c b/clang/test/Preprocessor/canonical-system-headers.c
new file mode 100644
index 0000000000000..0afa73c3e8225
--- /dev/null
+++ b/clang/test/Preprocessor/canonical-system-headers.c
@@ -0,0 +1,16 @@
+// don't create symlinks on windows
+// UNSUPPORTED: system-windows
+// REQUIRES: shell
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/foo/
+// RUN: ln -f -s %S/Inputs/canonical-system-headers %t/foo/include
+// RUN: %clang_cc1 -isystem %t/foo/include -sys-header-deps -MT foo.o -dependency-file %t2 %s -fsyntax-only
+// RUN: FileCheck %s --check-prefix=NOCANON --implicit-check-not=a.h < %t2
+// RUN: %clang_cc1 -isystem %t/foo/include -sys-header-deps -MT foo.o -dependency-file %t2 %s -fsyntax-only -canonical-system-headers
+// RUN: FileCheck %s --check-prefix=CANON --implicit-check-not=a.h < %t2
+
+// NOCANON: foo/include/a.h
+// CANON: Inputs/canonical-system-headers/a.h
+
+#include <a.h>
diff --git a/clang/test/Preprocessor/ext-pp-directive.c b/clang/test/Preprocessor/ext-pp-directive.c
index 97454b4d9002a..a3d74143dfbc6 100644
--- a/clang/test/Preprocessor/ext-pp-directive.c
+++ b/clang/test/Preprocessor/ext-pp-directive.c
@@ -6,11 +6,11 @@
 // RUN: not %clang_cc1 -std=c2x -fsyntax-only -verify %s
 
 // For C++
-// RUN: %clang_cc1 -x c++ -fsyntax-only -verify=pre-cpp2b-pedantic -pedantic %s
-// RUN: %clang_cc1 -x c++ -std=c++2b -fsyntax-only -verify=pre-cpp2b-compat -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify=pre-cpp23-pedantic -pedantic %s
+// RUN: %clang_cc1 -x c++ -std=c++23 -fsyntax-only -verify=pre-cpp23-compat -Wpre-c++23-compat %s
 // RUN: not %clang_cc1 -x c++ -fsyntax-only -verify %s
-// RUN: not %clang_cc1 -x c++ -std=c++2b -fsyntax-only -verify -pedantic %s
-// RUN: not %clang_cc1 -x c++ -std=c++2b -fsyntax-only -verify %s
+// RUN: not %clang_cc1 -x c++ -std=c++23 -fsyntax-only -verify -pedantic %s
+// RUN: not %clang_cc1 -x c++ -std=c++23 -fsyntax-only -verify %s
 
 int x;
 
@@ -22,8 +22,8 @@ int x;
 // pre-c2x-compat-warning@#1 {{use of a '#elifdef' directive is incompatible with C standards before C2x}}
 
 // For C++
-// pre-cpp2b-pedantic-warning@#1 {{use of a '#elifdef' directive is a C++2b extension}}
-// pre-cpp2b-compat-warning@#1 {{use of a '#elifdef' directive is incompatible with C++ standards before C++2b}}
+// pre-cpp23-pedantic-warning@#1 {{use of a '#elifdef' directive is a C++23 extension}}
+// pre-cpp23-compat-warning@#1 {{use of a '#elifdef' directive is incompatible with C++ standards before C++23}}
 
 #if 1
 #elifndef B // #2
@@ -33,8 +33,8 @@ int x;
 // pre-c2x-compat-warning@#2 {{use of a '#elifndef' directive is incompatible with C standards before C2x}}
 
 // For C++
-// pre-cpp2b-pedantic-warning@#2 {{use of a '#elifndef' directive is a C++2b extension}}
-// pre-cpp2b-compat-warning@#2 {{use of a '#elifndef' directive is incompatible with C++ standards before C++2b}}
+// pre-cpp23-pedantic-warning@#2 {{use of a '#elifndef' directive is a C++23 extension}}
+// pre-cpp23-compat-warning@#2 {{use of a '#elifndef' directive is incompatible with C++ standards before C++23}}
 
 #if 0
 #elifdef C
@@ -44,8 +44,8 @@ int x;
 // pre-c2x-compat-warning@-4 {{use of a '#elifdef' directive is incompatible with C standards before C2x}}
 
 // For C++
-// pre-cpp2b-pedantic-warning@-7 {{use of a '#elifdef' directive is a C++2b extension}}
-// pre-cpp2b-compat-warning@-8 {{use of a '#elifdef' directive is incompatible with C++ standards before C++2b}}
+// pre-cpp23-pedantic-warning@-7 {{use of a '#elifdef' directive is a C++23 extension}}
+// pre-cpp23-compat-warning@-8 {{use of a '#elifdef' directive is incompatible with C++ standards before C++23}}
 
 #if 0
 #elifndef D
@@ -55,8 +55,8 @@ int x;
 // pre-c2x-compat-warning@-4 {{use of a '#elifndef' directive is incompatible with C standards before C2x}}
 
 // For C++
-// pre-cpp2b-pedantic-warning@-7 {{use of a '#elifndef' directive is a C++2b extension}}
-// pre-cpp2b-compat-warning@-8 {{use of a '#elifndef' directive is incompatible with C++ standards before C++2b}}
+// pre-cpp23-pedantic-warning@-7 {{use of a '#elifndef' directive is a C++23 extension}}
+// pre-cpp23-compat-warning@-8 {{use of a '#elifndef' directive is incompatible with C++ standards before C++23}}
 
 #warning foo
 // For C
@@ -66,7 +66,7 @@ int x;
 // pre-c2x-compat-warning@-5 {{foo}}
 
 // For C++
-// pre-cpp2b-pedantic-warning@-8 {{#warning is a C++2b extension}}
-// pre-cpp2b-pedantic-warning@-9 {{foo}}
-// pre-cpp2b-compat-warning@-10 {{#warning is incompatible with C++ standards before C++2b}}
-// pre-cpp2b-compat-warning@-11 {{foo}}
+// pre-cpp23-pedantic-warning@-8 {{#warning is a C++23 extension}}
+// pre-cpp23-pedantic-warning@-9 {{foo}}
+// pre-cpp23-compat-warning@-10 {{#warning is incompatible with C++ standards before C++23}}
+// pre-cpp23-compat-warning@-11 {{foo}}
diff --git a/clang/test/Preprocessor/init-aarch64.c b/clang/test/Preprocessor/init-aarch64.c
index 869d7ca8a0636..4ef62f0e2aa8d 100644
--- a/clang/test/Preprocessor/init-aarch64.c
+++ b/clang/test/Preprocessor/init-aarch64.c
@@ -106,6 +106,7 @@
 // AARCH64-NEXT: #define __FLT_RADIX__ 2
 // AARCH64-NEXT: #define __FP_FAST_FMA 1
 // AARCH64-NEXT: #define __FP_FAST_FMAF 1
+// AARCH64-NEXT: #define __GCC_ASM_FLAG_OUTPUTS__ 1
 // AARCH64-NEXT: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 1
 // AARCH64-NEXT: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
 // AARCH64-NEXT: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c
index 46f79414fb7c3..f4d4a5793b080 100644
--- a/clang/test/Preprocessor/init.c
+++ b/clang/test/Preprocessor/init.c
@@ -9,7 +9,7 @@
 // BLOCKS:#define __block __attribute__((__blocks__(byref)))
 //
 //
-// RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=c++2b -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix CXX2B %s
+// RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=c++23 -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix CXX2B %s
 //
 // CXX2B:#define __GNUG__ 4
 // CXX2B:#define __GXX_EXPERIMENTAL_CXX0X__ 1
@@ -133,7 +133,7 @@
 // RUN: %clang_cc1 -ffreestanding -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix FREESTANDING %s
 // FREESTANDING:#define __STDC_HOSTED__ 0
 //
-// RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=gnu++2b -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GXX2B %s
+// RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=gnu++23 -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GXX2B %s
 //
 // GXX2B:#define __GNUG__ 4
 // GXX2B:#define __GXX_WEAK__ 1
diff --git a/clang/test/Preprocessor/macro_vaopt_check.cpp b/clang/test/Preprocessor/macro_vaopt_check.cpp
index d0469d7e884cf..28f92fc17c6c5 100644
--- a/clang/test/Preprocessor/macro_vaopt_check.cpp
+++ b/clang/test/Preprocessor/macro_vaopt_check.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 %s -Eonly -verify -Wno-all -Wno-c++2b-extensions -pedantic -std=c++20
-// RUN: %clang_cc1 %s -Eonly -verify -Wno-all -Wno-c++2b-extensions -pedantic -std=c++11
+// RUN: %clang_cc1 %s -Eonly -verify -Wno-all -Wno-c++23-extensions -pedantic -std=c++20
+// RUN: %clang_cc1 %s -Eonly -verify -Wno-all -Wno-c++23-extensions -pedantic -std=c++11
 // RUN: %clang_cc1 -x c %s -Eonly -verify -Wno-all -Wno-c2x-extensions -pedantic -std=c99
 
 //expected-error@+1{{missing '('}}
diff --git a/clang/test/Preprocessor/multiple-inclusion-opt.cpp b/clang/test/Preprocessor/multiple-inclusion-opt.cpp
new file mode 100644
index 0000000000000..0da20331fada6
--- /dev/null
+++ b/clang/test/Preprocessor/multiple-inclusion-opt.cpp
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -E -P -H %s 2>&1 | grep "multiple-inclusion-opt.h" | count 1
+
+#include "multiple-inclusion-opt.h"
+#include "multiple-inclusion-opt.h"
+#include "multiple-inclusion-opt.h"
+#include "multiple-inclusion-opt.h"
+#include "multiple-inclusion-opt.h"
diff --git a/clang/test/Preprocessor/multiple-inclusion-opt.h b/clang/test/Preprocessor/multiple-inclusion-opt.h
new file mode 100644
index 0000000000000..d346c9d5d22e4
--- /dev/null
+++ b/clang/test/Preprocessor/multiple-inclusion-opt.h
@@ -0,0 +1,18 @@
+# // null directive and comments before include guard
+
+#ifndef MULTIPLE_INCLUSION_OPT
+
+int foo();
+
+// The position of the define should not matter
+#define MULTIPLE_INCLUSION_OPT
+
+int bar();
+
+#endif
+
+#
+#
+/* Two null directives
+   and a multiline comment
+   after the #endif */
diff --git a/clang/test/Preprocessor/predefined-win-macros.c b/clang/test/Preprocessor/predefined-win-macros.c
index f402ce91838d6..1ac249550a489 100644
--- a/clang/test/Preprocessor/predefined-win-macros.c
+++ b/clang/test/Preprocessor/predefined-win-macros.c
@@ -52,7 +52,7 @@
 // CHECK-MS-CPP20: #define _MSVC_LANG 202002L
 
 // RUN: %clang_cc1 %s -x c++ -E -dM -triple i686-pc-win32 -fms-extensions -fms-compatibility \
-// RUN:     -fms-compatibility-version=19.00 -std=c++2b -o - | FileCheck -match-full-lines %s --check-prefix=CHECK-MS-CPP2B
+// RUN:     -fms-compatibility-version=19.00 -std=c++23 -o - | FileCheck -match-full-lines %s --check-prefix=CHECK-MS-CPP2B
 // CHECK-MS-CPP2B: #define _MSC_VER 1900
 // CHECK-MS-CPP2B: #define _MSVC_LANG 202004L
 
@@ -129,4 +129,5 @@
 // CHECK-ARM64-MINGW: #define WINNT 1
 // CHECK-ARM64-MINGW: #define _WIN32 1
 // CHECK-ARM64-MINGW: #define _WIN64 1
+// CHECK-ARM64-MINGW: #define __GCC_ASM_FLAG_OUTPUTS__ 1
 // CHECK-ARM64-MINGW: #define __aarch64__ 1
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index c4135e4d3ddfa..2dadbe2c420db 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -48,6 +48,7 @@
 // CHECK-NOT: __riscv_zcb {{.*$}}
 // CHECK-NOT: __riscv_zcd {{.*$}}
 // CHECK-NOT: __riscv_zcf {{.*$}}
+// CHECK-NOT: __riscv_zcmt {{.*$}}
 // CHECK-NOT: __riscv_h {{.*$}}
 // CHECK-NOT: __riscv_zvbb {{.*$}}
 // CHECK-NOT: __riscv_zvbc {{.*$}}
@@ -63,6 +64,8 @@
 // CHECK-NOT: __riscv_zvksh {{.*$}}
 // CHECK-NOT: __riscv_zvkt {{.*$}}
 // CHECK-NOT: __riscv_zicond {{.*$}}
+// CHECK-NOT: __riscv_smaia {{.*$}}
+// CHECK-NOT: __riscv_ssaia {{.*$}}
 
 // RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32i -x c -E -dM %s \
 // RUN: -o - | FileCheck %s
@@ -509,6 +512,13 @@
 // RUN: -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-ZCF-EXT %s
 // CHECK-ZCF-EXT: __riscv_zcf 1000000{{$}}
 
+// RUN: %clang -target riscv32 -march=rv32izcmt1p0 -menable-experimental-extensions \
+// RUN: -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-ZCMT-EXT %s
+// RUN: %clang -target riscv64 -march=rv64izcmt1p0 -menable-experimental-extensions \
+// RUN: -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-ZCMT-EXT %s
+// CHECK-ZCMT-EXT: __riscv_zca 1000000{{$}}
+// CHECK-ZCMT-EXT: __riscv_zcmt 1000000{{$}}
+
 // RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32izicsr2p0 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-ZICSR-EXT %s
 // RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64izicsr2p0 -x c -E -dM %s \
@@ -640,3 +650,19 @@
 // RUN: -march=rv64i_zicond1p0 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-ZICOND-EXT %s
 // CHECK-ZICOND-EXT: __riscv_zicond  1000000{{$}}
+
+// RUN: %clang -target riscv32 -menable-experimental-extensions \
+// RUN: -march=rv32ismaia1p0 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-SMAIA-EXT %s
+// RUN: %clang -target riscv64 -menable-experimental-extensions \
+// RUN: -march=rv64ismaia1p0 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-SMAIA-EXT %s
+// CHECK-SMAIA-EXT: __riscv_smaia  1000000{{$}}
+
+// RUN: %clang -target riscv32 -menable-experimental-extensions \
+// RUN: -march=rv32issaia1p0 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-SSAIA-EXT %s
+// RUN: %clang -target riscv64 -menable-experimental-extensions \
+// RUN: -march=rv64issaia1p0 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-SSAIA-EXT %s
+// CHECK-SSAIA-EXT: __riscv_ssaia  1000000{{$}}
diff --git a/clang/test/Preprocessor/suggest-typoed-directive.c b/clang/test/Preprocessor/suggest-typoed-directive.c
index cc15efee09e1c..c7130ad8031ca 100644
--- a/clang/test/Preprocessor/suggest-typoed-directive.c
+++ b/clang/test/Preprocessor/suggest-typoed-directive.c
@@ -1,19 +1,19 @@
-// RUN: %clang_cc1 -fsyntax-only -verify=pre-c2x-cpp2b %s
-// RUN: %clang_cc1 -std=c2x -fsyntax-only -verify=c2x-cpp2b %s
-// RUN: %clang_cc1 -x c++ -std=c++2b -fsyntax-only -verify=c2x-cpp2b %s
-// RUN: %clang_cc1 -x c++ -std=c++2b -fsyntax-only %s -fdiagnostics-parseable-fixits 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -fsyntax-only -verify=pre-c2x-cpp23 %s
+// RUN: %clang_cc1 -std=c2x -fsyntax-only -verify=c2x-cpp23 %s
+// RUN: %clang_cc1 -x c++ -std=c++23 -fsyntax-only -verify=c2x-cpp23 %s
+// RUN: %clang_cc1 -x c++ -std=c++23 -fsyntax-only %s -fdiagnostics-parseable-fixits 2>&1 | FileCheck %s
 
-// id:        pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#if'?}}
-// ifd:       pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#if'?}}
-// ifde:      pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#ifdef'?}}
-// elf:       pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#elif'?}}
-// elsif:     pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#elif'?}}
-// elseif:    pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#elif'?}}
+// id:        pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#if'?}}
+// ifd:       pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#if'?}}
+// ifde:      pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#ifdef'?}}
+// elf:       pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#elif'?}}
+// elsif:     pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#elif'?}}
+// elseif:    pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#elif'?}}
 // elfidef:   not suggested to '#elifdef'
 // elfindef:  not suggested to '#elifdef'
 // elfinndef: not suggested to '#elifndef'
-// els:       pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#else'?}}
-// endi:      pre-c2x-cpp2b-warning@+12 {{invalid preprocessing directive, did you mean '#endif'?}}
+// els:       pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#else'?}}
+// endi:      pre-c2x-cpp23-warning@+12 {{invalid preprocessing directive, did you mean '#endif'?}}
 #ifdef UNDEFINED
 #id
 #ifd
@@ -27,17 +27,17 @@
 #els
 #endi
 #endif
-// id:        c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#if'?}}
-// ifd:       c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#if'?}}
-// ifde:      c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#ifdef'?}}
-// elf:       c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#elif'?}}
-// elsif:     c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#elif'?}}
-// elseif:    c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#elif'?}}
-// elfidef:   c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#elifdef'?}}
-// elfindef:  c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#elifdef'?}}
-// elfinndef: c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#elifndef'?}}
-// els:       c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#else'?}}
-// endi:      c2x-cpp2b-warning@-12 {{invalid preprocessing directive, did you mean '#endif'?}}
+// id:        c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#if'?}}
+// ifd:       c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#if'?}}
+// ifde:      c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#ifdef'?}}
+// elf:       c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#elif'?}}
+// elsif:     c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#elif'?}}
+// elseif:    c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#elif'?}}
+// elfidef:   c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#elifdef'?}}
+// elfindef:  c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#elifdef'?}}
+// elfinndef: c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#elifndef'?}}
+// els:       c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#else'?}}
+// endi:      c2x-cpp23-warning@-12 {{invalid preprocessing directive, did you mean '#endif'?}}
 
 // CHECK: fix-it:{{.*}}:{[[@LINE-24]]:2-[[@LINE-24]]:4}:"if"
 // CHECK: fix-it:{{.*}}:{[[@LINE-24]]:2-[[@LINE-24]]:5}:"if"
diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c
index c47ed21ff3158..e1bcdfdcaa364 100644
--- a/clang/test/Preprocessor/ucn-pp-identifier.c
+++ b/clang/test/Preprocessor/ucn-pp-identifier.c
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 %s -fsyntax-only -std=c99 -pedantic -verify=expected,ext -Wundef -DTRIGRAPHS=1
 // RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify=expected,ext -Wundef -fno-trigraphs
-// RUN: %clang_cc1 %s -fsyntax-only -x c++ -std=c++2b -pedantic -ftrigraphs -DTRIGRAPHS=1 -verify=expected,cxx2b -Wundef -Wpre-c++2b-compat
+// RUN: %clang_cc1 %s -fsyntax-only -x c++ -std=c++23 -pedantic -ftrigraphs -DTRIGRAPHS=1 -verify=expected,cxx23 -Wundef -Wpre-c++23-compat
 // RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify=expected,ext -Wundef -ftrigraphs -DTRIGRAPHS=1
 // RUN: not %clang_cc1 %s -fsyntax-only -std=c99 -pedantic -Wundef 2>&1 | FileCheck -strict-whitespace %s
 
@@ -18,7 +18,7 @@
 #error "This should never happen"
 #endif
 
-#if a\u{FD}() // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
+#if a\u{FD}() // ext-warning {{extension}} cxx23-warning {{before C++23}}
 #error "This should never happen"
 #endif
 
@@ -34,12 +34,12 @@
 #define \U10000000      // expected-error {{macro name must be an identifier}}
 #define \u0061          // expected-error {{character 'a' cannot be specified by a universal character name}} expected-error {{macro name must be an identifier}}
 #define \u{fffe}        // expected-error {{macro name must be an identifier}} \
-                        // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
+                        // ext-warning {{extension}} cxx23-warning {{before C++23}}
 #define \N{ALERT}       // expected-error {{universal character name refers to a control character}} \
                    // expected-error {{macro name must be an identifier}} \
-                   // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
+                   // ext-warning {{extension}} cxx23-warning {{before C++23}}
 #define \N{WASTEBASKET} // expected-error {{macro name must be an identifier}} \
-                        // ext-warning {{extension}} cxx2b-warning {{before C++2b}}
+                        // ext-warning {{extension}} cxx23-warning {{before C++23}}
 #define a\u0024
 
 #if \u0110 // expected-warning {{is not defined, evaluates to 0}}
@@ -141,10 +141,10 @@ int CONCAT(\N{GREEK
 
 int \N{\
 LATIN CAPITAL LETTER A WITH GRAVE};
-//ext-warning@-2 {{extension}} cxx2b-warning@-2 {{before C++2b}}
+//ext-warning@-2 {{extension}} cxx23-warning@-2 {{before C++23}}
 
 #ifdef TRIGRAPHS
-int \N??<GREEK CAPITAL LETTER ALPHA??> = 0; // cxx2b-warning {{before C++2b}} \
+int \N??<GREEK CAPITAL LETTER ALPHA??> = 0; // cxx23-warning {{before C++23}} \
                                             //ext-warning {{extension}}\
                                             // expected-warning 2{{trigraph converted}}
 
diff --git a/clang/test/Profile/coverage-prefix-map.c b/clang/test/Profile/coverage-prefix-map.c
index 6f2c0fba0b09f..de9e3773f66d2 100644
--- a/clang/test/Profile/coverage-prefix-map.c
+++ b/clang/test/Profile/coverage-prefix-map.c
@@ -19,3 +19,13 @@
 
 // RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -mllvm -enable-name-compression=false -main-file-name coverage-prefix-map.c %t/root/nested/coverage-prefix-map.c -fcoverage-compilation-dir=/custom -fcoverage-prefix-map=/custom=/nonsense -o - | FileCheck --check-prefix=COVERAGE-COMPILATION-DIR %s
 // COVERAGE-COMPILATION-DIR: @__llvm_coverage_mapping = {{.*"\\02.*}}nonsense
+
+// Test that last -fcoverage-prefix-map option (-fcoverage-prefix-map==newpath) is applied.
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -mllvm -enable-name-compression=false -main-file-name coverage-prefix-map.c %t/root/nested/coverage-prefix-map.c -fcoverage-prefix-map=%/t/root=. -fcoverage-prefix-map==newpath -o - | FileCheck --check-prefix=COVERAGE-PREFIX-MAP-ORDER %s
+// COVERAGE-PREFIX-MAP-ORDER: @__llvm_coverage_mapping = {{.*"\\02.*newpath.*root.*nested.*coverage-prefix-map\.c}}
+
+// Test that last -fcoverage-prefix-map option (-fcoverage-prefix-map=%/t/root=.) is applied.
+// RUN: %clang_cc1 -fprofile-instrument=clang -fcoverage-mapping -emit-llvm -mllvm -enable-name-compression=false -main-file-name coverage-prefix-map.c %t/root/nested/coverage-prefix-map.c -fcoverage-prefix-map==newpath -fcoverage-prefix-map=%/t/root=. -o - | FileCheck --check-prefix=COVERAGE-PREFIX-MAP-REORDER %s
+// COVERAGE-PREFIX-MAP-REORDER: @__llvm_coverage_mapping =
+// COVERAGE-PREFIX-MAP-REORDER-NOT: newpath
+// COVERAGE-PREFIX-MAP-REORDER-SAME: nested{{.*coverage-prefix-map\.c}}
diff --git a/clang/test/Sema/attr-riscv-rvv-vector-bits.c b/clang/test/Sema/attr-riscv-rvv-vector-bits.c
index 2da69191878e4..bc1a70830eafc 100644
--- a/clang/test/Sema/attr-riscv-rvv-vector-bits.c
+++ b/clang/test/Sema/attr-riscv-rvv-vector-bits.c
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -ffreestanding -fsyntax-only -verify %s
-
-// TODO: Support for a arm_sve_vector_bits like attribute will come in the future.
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -ffreestanding -fsyntax-only -verify -mvscale-min=1 -mvscale-max=1 %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -ffreestanding -fsyntax-only -verify -mvscale-min=2 -mvscale-max=2 %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -ffreestanding -fsyntax-only -verify -mvscale-min=4 -mvscale-max=4 %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -ffreestanding -fsyntax-only -verify -mvscale-min=8 -mvscale-max=8 %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -ffreestanding -fsyntax-only -verify -mvscale-min=16 -mvscale-max=16 %s
 
 #include <stdint.h>
 
-#define N 64
-
 typedef __rvv_int8m1_t vint8m1_t;
 typedef __rvv_uint8m1_t vuint8m1_t;
 typedef __rvv_int16m1_t vint16m1_t;
@@ -17,64 +17,299 @@ typedef __rvv_uint64m1_t vuint64m1_t;
 typedef __rvv_float32m1_t vfloat32m1_t;
 typedef __rvv_float64m1_t vfloat64m1_t;
 
+// Define valid fixed-width RVV types
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint16m1_t fixed_int16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_int32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint64m1_t fixed_int64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vuint8m1_t fixed_uint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint16m1_t fixed_uint16m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint32m1_t fixed_uint32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vuint64m1_t fixed_uint64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+typedef vfloat32m1_t fixed_float32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t fixed_float64m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
 // GNU vector types
-typedef int8_t gnu_int8_t __attribute__((vector_size(N / 8)));
-typedef int16_t gnu_int16_t __attribute__((vector_size(N / 8)));
-typedef int32_t gnu_int32_t __attribute__((vector_size(N / 8)));
-typedef int64_t gnu_int64_t __attribute__((vector_size(N / 8)));
+typedef int8_t gnu_int8m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef int16_t gnu_int16m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef int32_t gnu_int32m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef int64_t gnu_int64m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+
+typedef uint8_t gnu_uint8m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef uint16_t gnu_uint16m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef uint32_t gnu_uint32m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef uint64_t gnu_uint64m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+
+typedef float gnu_float32m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+typedef double gnu_float64m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+
+// Attribute must have a single argument
+typedef vint8m1_t no_argument __attribute__((riscv_rvv_vector_bits));         // expected-error {{'riscv_rvv_vector_bits' attribute takes one argument}}
+typedef vint8m1_t two_arguments __attribute__((riscv_rvv_vector_bits(2, 4))); // expected-error {{'riscv_rvv_vector_bits' attribute takes one argument}}
+
+// The number of RVV vector bits must be an integer constant expression
+typedef vint8m1_t non_int_size1 __attribute__((riscv_rvv_vector_bits(2.0)));   // expected-error {{'riscv_rvv_vector_bits' attribute requires an integer constant}}
+typedef vint8m1_t non_int_size2 __attribute__((riscv_rvv_vector_bits("256"))); // expected-error {{'riscv_rvv_vector_bits' attribute requires an integer constant}}
+
+// Attribute must be attached to a single RVV vector or predicate type.
+typedef void *badtype1 __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));         // expected-error {{'riscv_rvv_vector_bits' attribute applied to non-RVV type 'void *'}}
+typedef int badtype2 __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));           // expected-error {{'riscv_rvv_vector_bits' attribute applied to non-RVV type 'int'}}
+typedef float badtype3 __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));         // expected-error {{'riscv_rvv_vector_bits' attribute applied to non-RVV type 'float'}}
+
+// Attribute only applies to typedefs.
+vint8m1_t non_typedef_type __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));  // expected-error {{'riscv_rvv_vector_bits' attribute only applies to typedefs}}
+
+// Test that we can define non-local fixed-length RVV types (unsupported for
+// sizeless types).
+fixed_int8m1_t global_int8;
+
+extern fixed_int8m1_t extern_int8;
+
+static fixed_int8m1_t static_int8;
+
+fixed_int8m1_t *global_int8_ptr;
+extern fixed_int8m1_t *extern_int8_ptr;
+static fixed_int8m1_t *static_int8_ptr;
+__thread fixed_int8m1_t thread_int8;
 
-typedef uint8_t gnu_uint8_t __attribute__((vector_size(N / 8)));
-typedef uint16_t gnu_uint16_t __attribute__((vector_size(N / 8)));
-typedef uint32_t gnu_uint32_t __attribute__((vector_size(N / 8)));
-typedef uint64_t gnu_uint64_t __attribute__((vector_size(N / 8)));
+typedef fixed_int8m1_t int8_typedef;
+typedef fixed_int8m1_t *int8_ptr_typedef;
 
-typedef float gnu_float32_t __attribute__((vector_size(N / 8)));
-typedef double gnu_float64_t __attribute__((vector_size(N / 8)));
+// Test sized expressions
+int sizeof_int8 = sizeof(global_int8);
+int sizeof_int8_var = sizeof(*global_int8_ptr);
+int sizeof_int8_var_ptr = sizeof(global_int8_ptr);
 
+extern fixed_int8m1_t *extern_int8_ptr;
+
+int alignof_int8 = __alignof__(extern_int8);
+int alignof_int8_var = __alignof__(*extern_int8_ptr);
+int alignof_int8_var_ptr = __alignof__(extern_int8_ptr);
 
 void f(int c) {
+  fixed_int8m1_t fs8;
   vint8m1_t ss8;
-  gnu_int8_t gs8;
+  gnu_int8m1_t gs8;
 
   // Check conditional expressions where the result is ambiguous are
   // ill-formed.
   void *sel __attribute__((unused));
+  sel = c ? ss8 : fs8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
+  sel = c ? fs8 : ss8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
 
   sel = c ? gs8 : ss8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
   sel = c ? ss8 : gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
+  sel = c ? gs8 : fs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+  sel = c ? fs8 : gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+
   // Check binary expressions where the result is ambiguous are ill-formed.
+  ss8 = ss8 + fs8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
   ss8 = ss8 + gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
+  fs8 = fs8 + ss8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
+  fs8 = fs8 + gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+
   gs8 = gs8 + ss8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+  gs8 = gs8 + fs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
+  ss8 += fs8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
   ss8 += gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
+  fs8 += ss8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
+  fs8 += gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+
   gs8 += ss8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+  gs8 += fs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
+  ss8 = ss8 == fs8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
   ss8 = ss8 == gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
+  fs8 = fs8 == ss8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
+  fs8 = fs8 == gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+
   gs8 = gs8 == ss8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+  gs8 = gs8 == fs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+
+  ss8 = ss8 & fs8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
+  ss8 = ss8 & gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
-  ss8 = ss8 & gs8; // expected-error {{invalid operands to binary expression ('vint8m1_t' (aka '__rvv_int8m1_t') and 'gnu_int8_t' (vector of 8 'int8_t' values))}}
+  fs8 = fs8 & ss8; // expected-error {{cannot combine fixed-length and sizeless RVV vectors in expression, result is ambiguous}}
+  fs8 = fs8 & gs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 
-  gs8 = gs8 & ss8; // expected-error {{invalid operands to binary expression ('gnu_int8_t' (vector of 8 'int8_t' values) and 'vint8m1_t' (aka '__rvv_int8m1_t'))}}
+  gs8 = gs8 & ss8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
+  gs8 = gs8 & fs8; // expected-error {{cannot combine GNU and RVV vectors in expression, result is ambiguous}}
 }
 
+// --------------------------------------------------------------------------//
+// Sizeof
+
+#define VECTOR_SIZE ((__riscv_v_fixed_vlen / 8))
+
+_Static_assert(sizeof(fixed_int8m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_int16m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_int32m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_int64m1_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_uint8m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_uint16m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_uint32m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_int64m1_t) == VECTOR_SIZE, "");
+
+_Static_assert(sizeof(fixed_float32m1_t) == VECTOR_SIZE, "");
+_Static_assert(sizeof(fixed_float64m1_t) == VECTOR_SIZE, "");
+
+// --------------------------------------------------------------------------//
+// Alignof
+
+#define VECTOR_ALIGN 8
+
+_Static_assert(__alignof__(fixed_int8m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_int16m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_int32m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_int64m1_t) == VECTOR_ALIGN, "");
+
+_Static_assert(__alignof__(fixed_uint8m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_uint16m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_uint32m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_uint64m1_t) == VECTOR_ALIGN, "");
+
+_Static_assert(__alignof__(fixed_float32m1_t) == VECTOR_ALIGN, "");
+_Static_assert(__alignof__(fixed_float64m1_t) == VECTOR_ALIGN, "");
+
+// --------------------------------------------------------------------------//
+// Structs
+
+struct struct_int64 { fixed_int64m1_t x, y[5]; };
+struct struct_float64 { fixed_float64m1_t x, y[5]; };
+
+// --------------------------------------------------------------------------//
+// Unions
+union union_int64 { fixed_int64m1_t x, y[5]; };
+union union_float64 { fixed_float64m1_t x, y[5]; };
+
 // --------------------------------------------------------------------------//
 // Implicit casts
 
-gnu_int8_t to_gnu_int8_t_from_vint8m1_t_(vint8m1_t x) { return x; } // expected-error {{returning 'vint8m1_t' (aka '__rvv_int8m1_t') from a function with incompatible result type 'gnu_int8_t' (vector of 8 'int8_t' values)}}
-vint8m1_t from_gnu_int8_t_to_vint8m1_t(gnu_int8_t x) { return x; } // expected-error {{returning 'gnu_int8_t' (vector of 8 'int8_t' values) from a function with incompatible result type 'vint8m1_t' (aka '__rvv_int8m1_t')}}
+#define TEST_CAST_COMMON(TYPE)                                              \
+  v##TYPE##_t to_v##TYPE##_t_from_fixed(fixed_##TYPE##_t x) { return x; } \
+  fixed_##TYPE##_t from_##TYPE##_t_to_fixed(v##TYPE##_t x) { return x; }
+
+#define TEST_CAST_GNU(PREFIX, TYPE)                                                          \
+  gnu_##TYPE##_t to_gnu_##TYPE##_t_from_##PREFIX##TYPE##_t(PREFIX##TYPE##_t x) { return x; } \
+  PREFIX##TYPE##_t from_gnu_##TYPE##_t_to_##PREFIX##TYPE##_t(gnu_##TYPE##_t x) { return x; }
+
+#define TEST_CAST_VECTOR(TYPE) \
+  TEST_CAST_COMMON(TYPE)       \
+  TEST_CAST_GNU(v, TYPE)      \
+  TEST_CAST_GNU(fixed_, TYPE)
+
+TEST_CAST_VECTOR(int8m1)
+TEST_CAST_VECTOR(int16m1)
+TEST_CAST_VECTOR(int32m1)
+TEST_CAST_VECTOR(int64m1)
+TEST_CAST_VECTOR(uint8m1)
+TEST_CAST_VECTOR(uint16m1)
+TEST_CAST_VECTOR(uint32m1)
+TEST_CAST_VECTOR(uint64m1)
+TEST_CAST_VECTOR(float32m1)
+TEST_CAST_VECTOR(float64m1)
 
 // --------------------------------------------------------------------------//
-// Test passing GNU vector scalable function
+// Test the scalable and fixed-length types can be used interchangeably
 
 vint32m1_t __attribute__((overloadable)) vfunc(vint32m1_t op1, vint32m1_t op2);
 vfloat64m1_t __attribute__((overloadable)) vfunc(vfloat64m1_t op1, vfloat64m1_t op2);
 
-gnu_int32_t call_int32_ff(gnu_int32_t op1, gnu_int32_t op2) {
-  return vfunc(op1, op2); // expected-error {{no matching function for call to 'vfunc'}}
-                          // expected-note@-5 {{candidate function not viable: no known conversion from 'gnu_int32_t' (vector of 2 'int32_t' values) to 'vint32m1_t' (aka '__rvv_int32m1_t') for 1st argument}}
-                          // expected-note@-5 {{candidate function not viable: no known conversion from 'gnu_int32_t' (vector of 2 'int32_t' values) to 'vfloat64m1_t' (aka '__rvv_float64m1_t') for 1st argument}}
-}
+#define TEST_CALL(TYPE)                                              \
+  fixed_##TYPE##_t                                                   \
+      call_##TYPE##_ff(fixed_##TYPE##_t op1, fixed_##TYPE##_t op2) { \
+    return vfunc(op1, op2);                                         \
+  }                                                                  \
+  fixed_##TYPE##_t                                                   \
+      call_##TYPE##_fs(fixed_##TYPE##_t op1, v##TYPE##_t op2) {     \
+    return vfunc(op1, op2);                                         \
+  }                                                                  \
+  fixed_##TYPE##_t                                                   \
+      call_##TYPE##_sf(v##TYPE##_t op1, fixed_##TYPE##_t op2) {     \
+    return vfunc(op1, op2);                                         \
+  }
+
+TEST_CALL(int32m1)
+TEST_CALL(float64m1)
+
+// --------------------------------------------------------------------------//
+// Vector initialization
+
+#if __RISCV_RVV_VLEN_BITS == 256
+
+typedef vint32m1_t int32x8 __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vfloat64m1_t float64x4 __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+
+int32x8 foo = {1, 2, 3, 4, 5, 6, 7, 8};
+int32x8 foo2 = {1, 2, 3, 4, 5, 6, 7, 8, 9}; // expected-warning{{excess elements in vector initializer}}
+
+float64x4 bar = {1.0, 2.0, 3.0, 4.0};
+float64x4 bar2 = {1.0, 2.0, 3.0, 4.0, 5.0}; // expected-warning{{excess elements in vector initializer}}
+
+#endif
+
+// --------------------------------------------------------------------------//
+// Vector ops
+
+#define TEST_BINARY(TYPE, NAME, OP)                  \
+  TYPE NAME##_##TYPE(TYPE op1, TYPE op2) {           \
+    return op1 OP op2;                               \
+  }                                                  \
+  TYPE compound##NAME##_##TYPE(TYPE op1, TYPE op2) { \
+    op1 OP##= op2;                                   \
+    return op1;                                      \
+  }
+
+#define TEST_COMPARISON(TYPE, NAME, OP)    \
+  TYPE NAME##_##TYPE(TYPE op1, TYPE op2) { \
+    return op1 OP op2;                     \
+  }
+
+#define TEST_UNARY(TYPE, NAME, OP) \
+  TYPE NAME##_##TYPE(TYPE op1) {   \
+    return OP op1;                 \
+  }
+
+#define TEST_OPS(TYPE)           \
+  TEST_BINARY(TYPE, add, +)      \
+  TEST_BINARY(TYPE, sub, -)      \
+  TEST_BINARY(TYPE, mul, *)      \
+  TEST_BINARY(TYPE, div, /)      \
+  TEST_COMPARISON(TYPE, eq, ==)  \
+  TEST_COMPARISON(TYPE, ne, !=)  \
+  TEST_COMPARISON(TYPE, lt, <)   \
+  TEST_COMPARISON(TYPE, gt, >)   \
+  TEST_COMPARISON(TYPE, lte, <=) \
+  TEST_COMPARISON(TYPE, gte, >=) \
+  TEST_UNARY(TYPE, nop, +)       \
+  TEST_UNARY(TYPE, neg, -)
+
+#define TEST_INT_OPS(TYPE)   \
+  TEST_OPS(TYPE)             \
+  TEST_BINARY(TYPE, mod, %)  \
+  TEST_BINARY(TYPE, and, &)  \
+  TEST_BINARY(TYPE, or, |)   \
+  TEST_BINARY(TYPE, xor, ^)  \
+  TEST_BINARY(TYPE, shl, <<) \
+  TEST_BINARY(TYPE, shr, <<) \
+  TEST_UNARY(TYPE, not, ~)
+
+TEST_INT_OPS(fixed_int8m1_t)
+TEST_INT_OPS(fixed_int16m1_t)
+TEST_INT_OPS(fixed_int32m1_t)
+TEST_INT_OPS(fixed_int64m1_t)
+TEST_INT_OPS(fixed_uint8m1_t)
+TEST_INT_OPS(fixed_uint16m1_t)
+TEST_INT_OPS(fixed_uint32m1_t)
+TEST_INT_OPS(fixed_uint64m1_t)
+
+TEST_OPS(fixed_float32m1_t)
+TEST_OPS(fixed_float64m1_t)
diff --git a/clang/test/Sema/gnu-asm-pmf.cpp b/clang/test/Sema/gnu-asm-pmf.cpp
index 30677c85696df..3758dbc992da9 100644
--- a/clang/test/Sema/gnu-asm-pmf.cpp
+++ b/clang/test/Sema/gnu-asm-pmf.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-windows-msvc -std=c++2b -fsyntax-only %s -verify
-// RUN: %clang_cc1 -triple x86_64-unknown-windows-itanium -std=c++2b -fsyntax-only %s -verify
+// RUN: %clang_cc1 -triple x86_64-unknown-windows-msvc -std=c++23 -fsyntax-only %s -verify
+// RUN: %clang_cc1 -triple x86_64-unknown-windows-itanium -std=c++23 -fsyntax-only %s -verify
 
 struct S {
   void operator()();
diff --git a/clang/test/Sema/nullptr.c b/clang/test/Sema/nullptr.c
index 46a9111ee278e..d11765a9c881a 100644
--- a/clang/test/Sema/nullptr.c
+++ b/clang/test/Sema/nullptr.c
@@ -16,8 +16,8 @@ nullptr_t f(nullptr_t null)
   p = null;
   int *pi = nullptr;
   pi = null;
-  null = 0; // expected-error {{assigning to 'nullptr_t' from incompatible type 'int'}}
-  bool b = nullptr; // expected-error {{initializing 'bool' with an expression of incompatible type 'nullptr_t'}}
+  null = 0;
+  bool b = nullptr;
 
   // Can't convert nullptr to integral implicitly.
   uintptr_t i = nullptr; // expected-error-re {{initializing 'uintptr_t' (aka '{{.*}}') with an expression of incompatible type 'nullptr_t'}}
@@ -77,6 +77,9 @@ void h() {
 
 static_assert(sizeof(nullptr_t) == sizeof(void*), "");
 
+static_assert(!nullptr, "");
+static_assert(!(bool){nullptr}, "");
+
 static_assert(!(nullptr < nullptr), ""); // expected-error {{invalid operands to binary expression}}
 static_assert(!(nullptr > nullptr), ""); // expected-error {{invalid operands to binary expression}}
 static_assert(  nullptr <= nullptr, ""); // expected-error {{invalid operands to binary expression}}
diff --git a/clang/test/Sema/riscv-bad-intrinsic-pragma.c b/clang/test/Sema/riscv-bad-intrinsic-pragma.c
index 78fa1edf4c596..fc8e18ff130e0 100644
--- a/clang/test/Sema/riscv-bad-intrinsic-pragma.c
+++ b/clang/test/Sema/riscv-bad-intrinsic-pragma.c
@@ -2,7 +2,7 @@
 // RUN:            2>&1 | FileCheck %s
 
 #pragma clang riscv intrinsic vvvv
-// CHECK:      warning: unexpected argument 'vvvv' to '#pragma riscv'; expected 'vector' [-Wignored-pragmas]
+// CHECK:      warning: unexpected argument 'vvvv' to '#pragma riscv'; expected 'vector' or 'sifive_vector' [-Wignored-pragmas]
 
 #pragma clang riscv what + 3241
 // CHECK:      warning: unexpected argument 'what' to '#pragma riscv'; expected 'intrinsic' [-Wignored-pragmas]
diff --git a/clang/test/Sema/riscv-interrupt-attr.c b/clang/test/Sema/riscv-interrupt-attr.c
index 2f5f2a44c778d..e66a5799bb394 100644
--- a/clang/test/Sema/riscv-interrupt-attr.c
+++ b/clang/test/Sema/riscv-interrupt-attr.c
@@ -4,26 +4,21 @@
 // RUN: %clang_cc1 %s -triple riscv64-unknown-elf -verify -fsyntax-only
 
 #if defined(CHECK_IR)
-// CHECK-LABEL:  @foo_user() #0
-// CHECK: ret void
-__attribute__((interrupt("user"))) void foo_user(void) {}
-// CHECK-LABEL:  @foo_supervisor() #1
+// CHECK-LABEL:  @foo_supervisor() #0
 // CHECK: ret void
 __attribute__((interrupt("supervisor"))) void foo_supervisor(void) {}
-// CHECK-LABEL:  @foo_machine() #2
+// CHECK-LABEL:  @foo_machine() #1
 // CHECK: ret void
 __attribute__((interrupt("machine"))) void foo_machine(void) {}
-// CHECK-LABEL:  @foo_default() #2
+// CHECK-LABEL:  @foo_default() #1
 // CHECK: ret void
 __attribute__((interrupt())) void foo_default(void) {}
-// CHECK-LABEL:  @foo_default2() #2
+// CHECK-LABEL:  @foo_default2() #1
 // CHECK: ret void
 __attribute__((interrupt())) void foo_default2(void) {}
 // CHECK: attributes #0
-// CHECK: "interrupt"="user"
-// CHECK: attributes #1
 // CHECK: "interrupt"="supervisor"
-// CHECK: attributes #2
+// CHECK: attributes #1
 // CHECK: "interrupt"="machine"
 #else
 struct a { int b; };
@@ -31,8 +26,10 @@ struct a { int b; };
 struct a test __attribute__((interrupt)); // expected-warning {{'interrupt' attribute only applies to functions}}
 
 __attribute__((interrupt("USER"))) void foo1(void) {} // expected-warning {{'interrupt' attribute argument not supported: USER}}
+__attribute__((interrupt("user"))) void foo1b(void) {} // expected-warning {{'interrupt' attribute argument not supported: user}}
+__attribute__((interrupt("MACHINE"))) void foo1c(void) {} // expected-warning {{'interrupt' attribute argument not supported: MACHINE}}
 
-__attribute__((interrupt("user", 1))) void foo2(void) {} // expected-error {{'interrupt' attribute takes no more than 1 argument}}
+__attribute__((interrupt("machine", 1))) void foo2(void) {} // expected-error {{'interrupt' attribute takes no more than 1 argument}}
 
 __attribute__((interrupt)) int foo3(void) {return 0;} // expected-warning {{RISC-V 'interrupt' attribute only applies to functions that have a 'void' return type}}
 
@@ -41,7 +38,7 @@ __attribute__((interrupt())) void foo4(void) {}
 
 __attribute__((interrupt())) void foo5(int a) {} // expected-warning {{RISC-V 'interrupt' attribute only applies to functions that have no parameters}}
 
-__attribute__((interrupt("user"), interrupt("supervisor"))) void foo6(void) {} // expected-warning {{repeated RISC-V 'interrupt' attribute}} \
+__attribute__((interrupt("machine"), interrupt("supervisor"))) void foo6(void) {} // expected-warning {{repeated RISC-V 'interrupt' attribute}} \
   // expected-note {{repeated RISC-V 'interrupt' attribute is here}}
 
 __attribute__((interrupt, interrupt)) void foo7(void) {} // expected-warning {{repeated RISC-V 'interrupt' attribute}} \
@@ -49,11 +46,9 @@ __attribute__((interrupt, interrupt)) void foo7(void) {} // expected-warning {{r
 
 __attribute__((interrupt(""))) void foo8(void) {} // expected-warning {{'interrupt' attribute argument not supported}}
 
-__attribute__((interrupt("user"))) void foo9(void);
 __attribute__((interrupt("supervisor"))) void foo9(void);
 __attribute__((interrupt("machine"))) void foo9(void);
 
-__attribute__((interrupt("user"))) void foo10(void) {}
 __attribute__((interrupt("supervisor"))) void foo11(void) {}
 __attribute__((interrupt("machine"))) void foo12(void) {}
 __attribute__((interrupt())) void foo13(void) {}
diff --git a/clang/test/Sema/riscv-rvv-explicit-casts-fixed-size.c b/clang/test/Sema/riscv-rvv-explicit-casts-fixed-size.c
new file mode 100644
index 0000000000000..17a73a77f68e8
--- /dev/null
+++ b/clang/test/Sema/riscv-rvv-explicit-casts-fixed-size.c
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=2 -mvscale-max=2 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=16 -mvscale-max=16 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+
+// expected-no-diagnostics
+
+// REQUIRES: riscv-registered-target
+
+#define FIXED_ATTR __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)))
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vfloat32m1_t fixed_float32m1_t FIXED_ATTR;
+typedef vfloat64m1_t fixed_float64m1_t FIXED_ATTR;
+typedef vint32m1_t fixed_int32m1_t FIXED_ATTR;
+typedef vint64m1_t fixed_int64m1_t FIXED_ATTR;
+
+// RVV VLS types can be cast to RVV VLA types, regardless of lane size.
+// NOTE: the list below is NOT exhaustive for all RVV types.
+
+#define CAST(from, to) \
+    void from##_to_##to(from a, to b) { \
+        b = (to) a; \
+    }
+
+#define TESTCASE(ty1, ty2) \
+    CAST(ty1, ty2) \
+    CAST(ty2, ty1)
+
+TESTCASE(fixed_float32m1_t, vfloat32m1_t)
+TESTCASE(fixed_float32m1_t, vfloat64m1_t)
+TESTCASE(fixed_float32m1_t, vint32m1_t)
+TESTCASE(fixed_float32m1_t, vint64m1_t)
+
+TESTCASE(fixed_float64m1_t, vfloat32m1_t)
+TESTCASE(fixed_float64m1_t, vfloat64m1_t)
+TESTCASE(fixed_float64m1_t, vint32m1_t)
+TESTCASE(fixed_float64m1_t, vint64m1_t)
+
+TESTCASE(fixed_int32m1_t, vfloat32m1_t)
+TESTCASE(fixed_int32m1_t, vfloat64m1_t)
+TESTCASE(fixed_int32m1_t, vint32m1_t)
+TESTCASE(fixed_int32m1_t, vint64m1_t)
+
+TESTCASE(fixed_int64m1_t, vfloat32m1_t)
+TESTCASE(fixed_int64m1_t, vfloat64m1_t)
+TESTCASE(fixed_int64m1_t, vint32m1_t)
+TESTCASE(fixed_int64m1_t, vint64m1_t)
diff --git a/clang/test/Sema/riscv-rvv-lax-vector-conversions.c b/clang/test/Sema/riscv-rvv-lax-vector-conversions.c
new file mode 100644
index 0000000000000..ff3e028aa314d
--- /dev/null
+++ b/clang/test/Sema/riscv-rvv-lax-vector-conversions.c
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify=lax-vector-none %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=integer -ffreestanding -fsyntax-only -verify=lax-vector-integer %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=all -ffreestanding -fsyntax-only -verify=lax-vector-all %s
+
+// lax-vector-all-no-diagnostics
+
+// REQUIRES: riscv-registered-target
+
+#define RVV_FIXED_ATTR __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)))
+#define GNU_FIXED_ATTR __attribute__((vector_size(__riscv_v_fixed_vlen / 8)))
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vfloat32m1_t rvv_fixed_float32m1_t RVV_FIXED_ATTR;
+typedef vint32m1_t rvv_fixed_int32m1_t RVV_FIXED_ATTR;
+typedef float gnu_fixed_float32m1_t GNU_FIXED_ATTR;
+typedef int gnu_fixed_int32m1_t GNU_FIXED_ATTR;
+
+void rvv_allowed_with_integer_lax_conversions() {
+  rvv_fixed_int32m1_t fi32;
+  vint64m1_t si64;
+
+  // The implicit cast here should fail if -flax-vector-conversions=none, but pass if
+  // -flax-vector-conversions={integer,all}.
+  fi32 = si64;
+  // lax-vector-none-error@-1 {{assigning to 'rvv_fixed_int32m1_t' (vector of 16 'int' values) from incompatible type}}
+  si64 = fi32;
+  // lax-vector-none-error@-1 {{assigning to 'vint64m1_t' (aka '__rvv_int64m1_t') from incompatible type}}
+}
+
+void rvv_allowed_with_all_lax_conversions() {
+  rvv_fixed_float32m1_t ff32;
+  vfloat64m1_t sf64;
+
+  // The implicit cast here should fail if -flax-vector-conversions={none,integer}, but pass if
+  // -flax-vector-conversions=all.
+  ff32 = sf64;
+  // lax-vector-none-error@-1 {{assigning to 'rvv_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'rvv_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  sf64 = ff32;
+  // lax-vector-none-error@-1 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+}
+
+void gnu_allowed_with_integer_lax_conversions() {
+  gnu_fixed_int32m1_t fi32;
+  vint64m1_t si64;
+
+  // The implicit cast here should fail if -flax-vector-conversions=none, but pass if
+  // -flax-vector-conversions={integer,all}.
+  fi32 = si64;
+  // lax-vector-none-error@-1 {{assigning to 'gnu_fixed_int32m1_t' (vector of 16 'int' values) from incompatible type}}
+  si64 = fi32;
+  // lax-vector-none-error@-1 {{assigning to 'vint64m1_t' (aka '__rvv_int64m1_t') from incompatible type}}
+}
+
+void gnu_allowed_with_all_lax_conversions() {
+  gnu_fixed_float32m1_t ff32;
+  vfloat64m1_t sf64;
+
+  // The implicit cast here should fail if -flax-vector-conversions={none,integer}, but pass if
+  // -flax-vector-conversions=all.
+  ff32 = sf64;
+  // lax-vector-none-error@-1 {{assigning to 'gnu_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'gnu_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  sf64 = ff32;
+  // lax-vector-none-error@-1 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+}
diff --git a/clang/test/Sema/riscv-vector-types-support.c b/clang/test/Sema/riscv-vector-types-support.c
new file mode 100644
index 0000000000000..e92bb5f59a3a0
--- /dev/null
+++ b/clang/test/Sema/riscv-vector-types-support.c
@@ -0,0 +1,3 @@
+// RUN: %clang_cc1 %s -triple riscv64 -fsyntax-only -verify
+
+typedef __attribute__((riscv_rvv_vector_bits(256))) void norvvflag; // expected-error{{'riscv_rvv_vector_bits' attribute is not supported on targets missing 'zve32x'; specify an appropriate -march= or -mcpu=}}
diff --git a/clang/test/Sema/stmtexpr-init.c b/clang/test/Sema/stmtexpr-init.c
index 73c87e0d4b726..4ad31adecf20a 100644
--- a/clang/test/Sema/stmtexpr-init.c
+++ b/clang/test/Sema/stmtexpr-init.c
@@ -6,7 +6,7 @@ void T1(void) {
   int *x[1] = {({ static int _x = 10; &_x; })}; // expected-no-error
 
   /* Before commit
-     683e83c5 [Clang][C++2b] P2242R3: Non-literal variables [...] in constexpr
+     683e83c5 [Clang][C++23] P2242R3: Non-literal variables [...] in constexpr
      (i.e in clang-14 and earlier)
      this was silently accepted, but generated incorrect code.
   */
diff --git a/clang/test/SemaCXX/P1155.cpp b/clang/test/SemaCXX/P1155.cpp
index 6dcbaa750557d..87f71ca3bd76c 100644
--- a/clang/test/SemaCXX/P1155.cpp
+++ b/clang/test/SemaCXX/P1155.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -fcxx-exceptions -verify %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify %s
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -fcxx-exceptions -verify %s
 // expected-no-diagnostics
diff --git a/clang/test/SemaCXX/attr-declspec-ignored.cpp b/clang/test/SemaCXX/attr-declspec-ignored.cpp
index f46e3910c56d7..dfea8cc4d47c8 100644
--- a/clang/test/SemaCXX/attr-declspec-ignored.cpp
+++ b/clang/test/SemaCXX/attr-declspec-ignored.cpp
@@ -9,6 +9,10 @@ namespace test1 {
   // expected-warning{{attribute 'aligned' is ignored, place it after "union" to apply attribute to type declaration}}
   __attribute__((visibility("hidden")))  __attribute__((aligned)) enum D {D}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum" to apply attribute to type declaration}} \
   // expected-warning{{attribute 'aligned' is ignored, place it after "enum" to apply attribute to type declaration}}
+  __attribute__((visibility("hidden")))  __attribute__((aligned)) enum class EC {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum class" to apply attribute to type declaration}} \
+  // expected-warning{{attribute 'aligned' is ignored, place it after "enum class" to apply attribute to type declaration}}
+  __attribute__((visibility("hidden")))  __attribute__((aligned)) enum struct ES {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum struct" to apply attribute to type declaration}} \
+  // expected-warning{{attribute 'aligned' is ignored, place it after "enum struct" to apply attribute to type declaration}}
 
   // Test that we get the same warnings for type declarations nested in a record.
   struct X {
@@ -20,7 +24,11 @@ namespace test1 {
     // expected-warning{{attribute 'aligned' is ignored, place it after "union" to apply attribute to type declaration}}
     __attribute__((visibility("hidden")))  __attribute__((aligned)) enum D {D}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum" to apply attribute to type declaration}} \
     // expected-warning{{attribute 'aligned' is ignored, place it after "enum" to apply attribute to type declaration}}
-
+    __attribute__((visibility("hidden")))  __attribute__((aligned)) enum class EC {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum class" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'aligned' is ignored, place it after "enum class" to apply attribute to type declaration}}
+    __attribute__((visibility("hidden")))  __attribute__((aligned)) enum struct ES {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum struct" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'aligned' is ignored, place it after "enum struct" to apply attribute to type declaration}}
+  
     // Also test [[]] attribute syntax. (On a non-nested declaration, these
     // generate a hard "misplaced attributes" error, which we test for
     // elsewhere.)
@@ -32,6 +40,10 @@ namespace test1 {
     // expected-warning{{attribute 'aligned' is ignored, place it after "union" to apply attribute to type declaration}}
     [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum H {H}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum" to apply attribute to type declaration}} \
     // expected-warning{{attribute 'aligned' is ignored, place it after "enum" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum class I {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum class" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'aligned' is ignored, place it after "enum class" to apply attribute to type declaration}}
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum struct J {}; // expected-warning{{attribute 'visibility' is ignored, place it after "enum struct" to apply attribute to type declaration}} \
+    // expected-warning{{attribute 'aligned' is ignored, place it after "enum struct" to apply attribute to type declaration}}
   };
 }
 
@@ -40,16 +52,22 @@ namespace test2 {
   __attribute__((visibility("hidden")))  __attribute__((aligned)) struct B {} b;
   __attribute__((visibility("hidden")))  __attribute__((aligned)) union C {} c;
   __attribute__((visibility("hidden")))  __attribute__((aligned)) enum D {D} d;
+  __attribute__((visibility("hidden")))  __attribute__((aligned)) enum class EC {} ec;
+  __attribute__((visibility("hidden")))  __attribute__((aligned)) enum struct ES {} es;
 
   struct X {
     __attribute__((visibility("hidden")))  __attribute__((aligned)) class A {} a;
     __attribute__((visibility("hidden")))  __attribute__((aligned)) struct B {} b;
     __attribute__((visibility("hidden")))  __attribute__((aligned)) union C {} c;
     __attribute__((visibility("hidden")))  __attribute__((aligned)) enum D {D} d;
+    __attribute__((visibility("hidden")))  __attribute__((aligned)) enum class EC {} ec;
+    __attribute__((visibility("hidden")))  __attribute__((aligned)) enum struct ES {} es;
 
     [[gnu::visibility("hidden")]]  [[gnu::aligned]] class E {} e;
     [[gnu::visibility("hidden")]]  [[gnu::aligned]] struct F {} f;
     [[gnu::visibility("hidden")]]  [[gnu::aligned]] union G {} g;
     [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum H {H} h;
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum class I {} i;
+    [[gnu::visibility("hidden")]]  [[gnu::aligned]] enum struct J {} j;
   };
 }
diff --git a/clang/test/SemaCXX/attr-riscv-rvv-vector-bits.cpp b/clang/test/SemaCXX/attr-riscv-rvv-vector-bits.cpp
new file mode 100644
index 0000000000000..6d18883b46fe2
--- /dev/null
+++ b/clang/test/SemaCXX/attr-riscv-rvv-vector-bits.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +zve64x -ffreestanding -fsyntax-only -verify -std=c++11 -mvscale-min=4 -mvscale-max=4 -Wconversion %s
+// expected-no-diagnostics
+
+#include <stdint.h>
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef vint8m1_t fixed_int8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef int8_t gnu_int8m1_t __attribute__((vector_size(__riscv_v_fixed_vlen / 8)));
+
+template<typename T> struct S { T var; };
+
+S<fixed_int8m1_t> s;
+
+// Test implicit casts between VLA and VLS vectors
+vint8m1_t to_vint8m1_t(fixed_int8m1_t x) { return x; }
+fixed_int8m1_t from_vint8m1_t(vint8m1_t x) { return x; }
+
+// Test implicit casts between GNU and VLA vectors
+vint8m1_t to_vint8m1_t__from_gnu_int8m1_t(gnu_int8m1_t x) { return x; }
+gnu_int8m1_t from_vint8m1_t__to_gnu_int8m1_t(vint8m1_t x) { return x; }
+
+// Test implicit casts between GNU and VLS vectors
+fixed_int8m1_t to_fixed_int8m1_t__from_gnu_int8m1_t(gnu_int8m1_t x) { return x; }
+gnu_int8m1_t from_fixed_int8m1_t__to_gnu_int8m1_t(fixed_int8m1_t x) { return x; }
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index 5e7a80c81eaee..fa9ad786b289e 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1,12 +1,12 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx20_2b,cxx2b    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx11_20,cxx20_2b -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx20_23,cxx23    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx11_20,cxx20_23 -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify=expected,cxx11_20,cxx11    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
 
 namespace StaticAssertFoldTest {
 
 int x;
 static_assert(++x, "test"); // expected-error {{not an integral constant expression}}
-// cxx20_2b-note@-1 {{cannot modify an object that is visible outside that expression}}
+// cxx20_23-note@-1 {{cannot modify an object that is visible outside that expression}}
 static_assert(false, "test"); // expected-error {{test}}
 
 }
@@ -360,7 +360,7 @@ extern char externalvar[];
 constexpr bool constaddress = (void *)externalvar == (void *)0x4000UL; // expected-error {{must be initialized by a constant expression}} expected-note {{reinterpret_cast}}
 constexpr bool litaddress = "foo" == "foo"; // expected-error {{must be initialized by a constant expression}}
 // expected-note@-1 {{comparison of addresses of literals has unspecified value}}
-// cxx20_2b-warning@-2 {{comparison between two arrays is deprecated}}
+// cxx20_23-warning@-2 {{comparison between two arrays is deprecated}}
 static_assert(0 != "foo", "");
 
 }
@@ -1777,12 +1777,12 @@ namespace std { struct type_info; }
 namespace TypeId {
   struct A { virtual ~A(); };
   A f();
-  A &g(); // cxx20_2b-note {{declared here}}
+  A &g(); // cxx20_23-note {{declared here}}
   constexpr auto &x = typeid(f());
   constexpr auto &y = typeid(g()); // expected-error{{constant expression}}
   // cxx11-note@-1 {{typeid applied to expression of polymorphic type 'A' is not allowed in a constant expression}}
   // expected-warning@-2 {{expression with side effects will be evaluated despite being used as an operand to 'typeid'}}
-  // cxx20_2b-note@-3 {{non-constexpr function 'g' cannot be used in a constant expression}}
+  // cxx20_23-note@-3 {{non-constexpr function 'g' cannot be used in a constant expression}}
 }
 
 namespace PR14203 {
@@ -1921,14 +1921,14 @@ namespace VirtualFromBase {
   static_assert(p->f() == sizeof(X<S1>), "");
   // cxx11-error@-1    {{not an integral constant expression}}
   // cxx11-note@-2     {{call to virtual function}}
-  // cxx20_2b-error@-3 {{static assertion failed}}
-  // cxx20_2b-note@-4 {{8 == 16}}
+  // cxx20_23-error@-3 {{static assertion failed}}
+  // cxx20_23-note@-4 {{8 == 16}}
 
   // Non-virtual f(), OK.
   constexpr X<X<S2>> xxs2;
   constexpr X<S2> *q = const_cast<X<X<S2>>*>(&xxs2);
-  static_assert(q->f() == sizeof(S2), ""); // cxx20_2b-error {{static assertion failed}} \
-                                           // cxx20_2b-note {{16 == 8}}
+  static_assert(q->f() == sizeof(S2), ""); // cxx20_23-error {{static assertion failed}} \
+                                           // cxx20_23-note {{16 == 8}}
 }
 
 namespace ConstexprConstructorRecovery {
@@ -1954,7 +1954,7 @@ namespace Lifetime {
   }
 
   constexpr int &get(int &&n) { return n; }
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
   constexpr int &&get_rv(int &&n) { return static_cast<int&&>(n); }
   struct S {
     int &&r;
@@ -2281,13 +2281,13 @@ namespace InheritedCtor {
   constexpr D d(0); // cxx11-error {{constant expression}} cxx11-note {{derived class}}
 
   struct E : virtual A { using A::A; }; // expected-note {{here}}
-  // cxx20_2b-note@-1 {{struct with virtual base class is not a literal type}}
+  // cxx20_23-note@-1 {{struct with virtual base class is not a literal type}}
   // We wrap a function around this to avoid implicit zero-initialization
   // happening first; the zero-initialization step would produce the same
   // error and defeat the point of this test.
   void f() {
     constexpr E e(0); // cxx11-error {{constant expression}} cxx11-note {{derived class}}
-    // cxx20_2b-error@-1 {{constexpr variable cannot have non-literal type}}
+    // cxx20_23-error@-1 {{constexpr variable cannot have non-literal type}}
   }
   // FIXME: This produces a note with no source location.
   //constexpr E e(0);
diff --git a/clang/test/SemaCXX/constant-expression-cxx14.cpp b/clang/test/SemaCXX/constant-expression-cxx14.cpp
index 9f8d8f7954b12..52a7cceea2a3f 100644
--- a/clang/test/SemaCXX/constant-expression-cxx14.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx14.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx20_2b,cxx2b          %s -fcxx-exceptions -triple=x86_64-linux-gnu
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_2b,cxx20 %s -fcxx-exceptions -triple=x86_64-linux-gnu
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx20_23,cxx23          %s -fcxx-exceptions -triple=x86_64-linux-gnu
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_23,cxx20 %s -fcxx-exceptions -triple=x86_64-linux-gnu
 // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,cxx14_20,cxx14          %s -fcxx-exceptions -triple=x86_64-linux-gnu
 
 struct S {
@@ -46,12 +46,12 @@ constexpr int g(int k) {
 static_assert(g(2) == 42, "");
 constexpr int h(int n) {  // expected-error {{constexpr function never produces a constant expression}}
   static const int m = n; // expected-note {{control flows through the definition of a static variable}} \
-                          // cxx14_20-warning {{definition of a static variable in a constexpr function is a C++2b extension}}
+                          // cxx14_20-warning {{definition of a static variable in a constexpr function is a C++23 extension}}
   return m;
 }
 constexpr int i(int n) {        // expected-error {{constexpr function never produces a constant expression}}
   thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \
-                                // cxx14_20-warning {{definition of a thread_local variable in a constexpr function is a C++2b extension}}
+                                // cxx14_20-warning {{definition of a thread_local variable in a constexpr function is a C++23 extension}}
   return m;
 }
 
@@ -229,7 +229,7 @@ namespace subobject {
     // expected-note@-2 {{assignment to member 'c' of union with active member 'n'}}
   }
   constexpr bool check(D &d) { return d.c.a.y == 3; }
-  // cxx20_2b-note@-1 {{read of member 'y' of union with active member 'x'}}
+  // cxx20_23-note@-1 {{read of member 'y' of union with active member 'x'}}
 
   constexpr bool g() { D d; f(d); return d.c.a.y == 3; }
   static_assert(g(), "");
@@ -242,7 +242,7 @@ namespace subobject {
   static_assert(i(), ""); // expected-error {{constant expression}} expected-note {{in call}}
 
   constexpr bool j() { D d; d.c.a.x = 3; return check(d); } // cxx14-note {{assignment to member 'x' of union with active member 'y'}}
-  // cxx20_2b-note@-1 {{in call to 'check(d)'}}
+  // cxx20_23-note@-1 {{in call to 'check(d)'}}
   static_assert(j(), ""); // expected-error {{constant expression}} expected-note {{in call}}
 }
 
@@ -271,12 +271,12 @@ namespace null {
 
 namespace incdec {
   template<typename T> constexpr T &ref(T &&r) { return r; }
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
   template<typename T> constexpr T postinc(T &&r) { return (r++, r); }
   template<typename T> constexpr T postdec(T &&r) { return (r--, r); }
 
   template int &ref<int>(int &&);
-  // cxx2b-note@-1  {{in instantiation of function template specialization}}
+  // cxx23-note@-1  {{in instantiation of function template specialization}}
 
   static_assert(postinc(0) == 1, "");
   static_assert(postdec(0) == -1, "");
@@ -345,7 +345,7 @@ namespace incdec {
   }
   constexpr int wrong_member = f({0}); // expected-error {{constant}} expected-note {{in call to 'f({.a = 0})'}}
   constexpr int vol = --ref<volatile int>(0); // expected-error {{constant}} expected-note {{decrement of volatile-qualified}}
-  // cxx20_2b-warning@-1 {{decrement of object of volatile-qualified type 'volatile int' is deprecated}}
+  // cxx20_23-warning@-1 {{decrement of object of volatile-qualified type 'volatile int' is deprecated}}
 
   constexpr int incr(int k) {
     int x = k;
@@ -877,7 +877,7 @@ namespace VirtualFromBase {
 
 namespace Lifetime {
   constexpr int &get(int &&r) { return r; }
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
   constexpr int f() {
     int &r = get(123);
     return r;
diff --git a/clang/test/SemaCXX/constant-expression-cxx2b.cpp b/clang/test/SemaCXX/constant-expression-cxx2b.cpp
index e52dc1400f3d5..2ee1d48d1cd69 100644
--- a/clang/test/SemaCXX/constant-expression-cxx2b.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx2b.cpp
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx2a %s -fcxx-exceptions -triple=x86_64-linux-gnu -Wno-c++2b-extensions
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx2b %s -fcxx-exceptions -triple=x86_64-linux-gnu -Wpre-c++2b-compat
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx2a %s -fcxx-exceptions -triple=x86_64-linux-gnu -Wno-c++23-extensions
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx23 %s -fcxx-exceptions -triple=x86_64-linux-gnu -Wpre-c++23-compat
 
 struct NonLiteral { // cxx2a-note {{'NonLiteral' is not literal}} \
-                    // cxx2b-note 2{{'NonLiteral' is not literal}}
+                    // cxx23-note 2{{'NonLiteral' is not literal}}
   NonLiteral() {}
 };
 
@@ -12,42 +12,42 @@ struct Constexpr{};
 
 constexpr int f(int n) {  // expected-error {{constexpr function never produces a constant expression}}
   static const int m = n; // expected-note {{control flows through the definition of a static variable}} \
-                          // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                          // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 constexpr int g(int n) {        // expected-error {{constexpr function never produces a constant expression}}
   thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \
-                                // cxx2b-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                                // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int c_thread_local(int n) { // expected-error {{constexpr function never produces a constant expression}}
   static _Thread_local int m = 0;     // expected-note {{control flows through the definition of a thread_local variable}} \
-                                      // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                                      // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int gnu_thread_local(int n) { // expected-error {{constexpr function never produces a constant expression}}
   static __thread int m = 0;            // expected-note {{control flows through the definition of a thread_local variable}} \
-                                        // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                                        // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int h(int n) {  // expected-error {{constexpr function never produces a constant expression}}
   static const int m = n; // expected-note {{control flows through the definition of a static variable}} \
-                          // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                          // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return &m - &m;
 }
 constexpr int i(int n) {        // expected-error {{constexpr function never produces a constant expression}}
   thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \
-                                 // cxx2b-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                                 // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
   return &m - &m;
 }
 
 constexpr int j(int n) {
   if (!n)
     return 0;
-  static const int m = n; // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  static const int m = n; // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 constexpr int j0 = j(0);
@@ -55,7 +55,7 @@ constexpr int j0 = j(0);
 constexpr int k(int n) {
   if (!n)
     return 0;
-  thread_local const int m = n; // cxx2b-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  thread_local const int m = n; // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
 
   return m;
 }
@@ -65,7 +65,7 @@ constexpr int j_evaluated(int n) {
   if (!n)
     return 0;
   static const int m = n; // expected-note {{control flows through the definition of a static variable}} \
-                          // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                          // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
@@ -76,7 +76,7 @@ constexpr int k_evaluated(int n) {
   if (!n)
     return 0;
   thread_local const int m = n; // expected-note {{control flows through the definition of a thread_local variable}} \
-                                // cxx2b-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
+                                // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
 
   return m;
 }
@@ -85,21 +85,21 @@ constexpr int ke = k_evaluated(1); // expected-error {{constexpr variable 'ke' m
                                    // expected-note {{in call}}
 
 constexpr int static_constexpr() {
-  static constexpr int m = 42;     // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
-  static constexpr Constexpr foo; // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  static constexpr int m = 42;     // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
+  static constexpr Constexpr foo; // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int thread_local_constexpr() {
-  thread_local constexpr int m = 42; // cxx2b-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
-  thread_local constexpr Constexpr foo; // cxx2b-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++2b}}
+  thread_local constexpr int m = 42; // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
+  thread_local constexpr Constexpr foo; // cxx23-warning {{definition of a thread_local variable in a constexpr function is incompatible with C++ standards before C++23}}
   return m;
 }
 
 constexpr int non_literal(bool b) {
   if (!b)
     return 0;
-  NonLiteral n; // cxx2b-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++2b}}
+  NonLiteral n; // cxx23-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}}
 }
 
 constexpr int non_literal_1 = non_literal(false);
@@ -111,7 +111,7 @@ constexpr int f(int x) {
     return 0;
   } else {
     goto test; // expected-note {{subexpression not valid in a constant expression}} \
-               // cxx2b-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++2b}}
+               // cxx23-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}}
   }
 test:
   return 0;
@@ -124,7 +124,7 @@ constexpr int c = f(1);
 
 constexpr int label() {
 
-test: // cxx2b-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++2b}}
+test: // cxx23-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}}
   return 0;
 }
 
@@ -138,14 +138,14 @@ constexpr int d = label();
 // This is to be contrasted with the test for implicitly constexpr lambdas below.
 int test_in_lambdas() {
   auto a = []() constexpr {
-    static const int m = 32; // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+    static const int m = 32; // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
     return m;
   };
 
   auto b = [](int n) constexpr {
     if (!n)
       return 0;
-    static const int m = n; // cxx2b-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++2b}}
+    static const int m = n; // cxx23-warning {{definition of a static variable in a constexpr function is incompatible with C++ standards before C++23}}
     return m;
   }
   (1);
@@ -155,7 +155,7 @@ int test_in_lambdas() {
       return 0;
     else
       goto test; // expected-note {{subexpression not valid in a constant expression}} \
-                 // cxx2b-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++2b}}
+                 // cxx23-warning {{use of this statement in a constexpr function is incompatible with C++ standards before C++23}}
   test:
     return 1;
   };
@@ -165,15 +165,15 @@ int test_in_lambdas() {
 
   auto non_literal = [](bool b) constexpr {
     if (!b)
-      NonLiteral n; // cxx2b-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}} \
-                    // cxx2a-error {{variable of non-literal type 'NonLiteral' cannot be defined in a constexpr function before C++2b}} \
-                    // cxx2b-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++2b}}
+      NonLiteral n; // cxx23-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}} \
+                    // cxx2a-error {{variable of non-literal type 'NonLiteral' cannot be defined in a constexpr function before C++23}} \
+                    // cxx23-warning {{definition of a variable of non-literal type in a constexpr function is incompatible with C++ standards before C++23}}
     return 0;
   };
 
 #if __cplusplus > 202002L
-  constexpr auto non_literal_ko = non_literal(false); // cxx2b-error {{constexpr variable 'non_literal_ko' must be initialized by a constant expression}} \
-                                                      // cxx2b-note {{in call}}
+  constexpr auto non_literal_ko = non_literal(false); // cxx23-error {{constexpr variable 'non_literal_ko' must be initialized by a constant expression}} \
+                                                      // cxx23-note {{in call}}
 
   constexpr auto non_literal_ok = non_literal(true);
 #endif
@@ -187,7 +187,7 @@ int test_lambdas_implicitly_constexpr() {
   auto b = [](int n) { // cxx2a-note 2{{declared here}}
     if (!n)
       return 0;
-    static const int m = n; // cxx2b-note {{control flows through the definition of a static variable}}
+    static const int m = n; // cxx23-note {{control flows through the definition of a static variable}}
     return m;
   };
 
@@ -197,13 +197,13 @@ int test_lambdas_implicitly_constexpr() {
 
   constexpr auto b3 = b(1); // expected-error{{constexpr variable 'b3' must be initialized by a constant expression}} \
                             // cxx2a-note {{non-constexpr function}} \
-                            // cxx2b-note {{in call}}
+                            // cxx23-note {{in call}}
 
   auto c = [](int n) { // cxx2a-note 2{{declared here}}
     if (!n)
       return 0;
     else
-      goto test; // cxx2b-note {{subexpression not valid in a constant expression}}
+      goto test; // cxx23-note {{subexpression not valid in a constant expression}}
   test:
     return 1;
   };
@@ -213,17 +213,17 @@ int test_lambdas_implicitly_constexpr() {
 
   constexpr auto c_error = c(1); // expected-error {{constexpr variable 'c_error' must be initialized by a constant expression}} \
                                  // cxx2a-note {{non-constexpr function}} \
-                                 // cxx2b-note {{in call to}}
+                                 // cxx23-note {{in call to}}
 
   auto non_literal = [](bool b) { // cxx2a-note 2{{declared here}}
     if (b)
-      NonLiteral n; // cxx2b-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}}
+      NonLiteral n; // cxx23-note {{non-literal type 'NonLiteral' cannot be used in a constant expression}}
     return 0;
   };
 
   constexpr auto non_literal_ko = non_literal(true); // expected-error {{constexpr variable 'non_literal_ko' must be initialized by a constant expression}} \
                                                      // cxx2a-note {{non-constexpr function}} \
-                                                     // cxx2b-note {{in call}}
+                                                     // cxx23-note {{in call}}
 
   constexpr auto non_literal_ok = non_literal(false); // cxx2a-error {{must be initialized by a constant expression}} \
                                                       // cxx2a-note {{non-constexpr function}}
@@ -238,8 +238,8 @@ constexpr auto dependent_var_def_lambda() {
   };
 }
 
-constexpr auto non_literal_valid_in_cxx2b = dependent_var_def_lambda<NonLiteral>()(true); // \
-    // cxx2a-error {{constexpr variable 'non_literal_valid_in_cxx2b' must be initialized by a constant expression}} \
+constexpr auto non_literal_valid_in_cxx23 = dependent_var_def_lambda<NonLiteral>()(true); // \
+    // cxx2a-error {{constexpr variable 'non_literal_valid_in_cxx23' must be initialized by a constant expression}} \
     // cxx2a-note {{non-constexpr function}}
 
 
@@ -249,8 +249,8 @@ constexpr double evaluate_static_constexpr() {
       return 42;
     }
   };
-  thread_local constexpr Constexpr t; // cxx2b-warning {{before C++2b}}
-  static constexpr Constexpr s; // cxx2b-warning {{before C++2b}}
+  thread_local constexpr Constexpr t; // cxx23-warning {{before C++23}}
+  static constexpr Constexpr s; // cxx23-warning {{before C++23}}
   return t.f() + s.f();
 }
 static_assert(evaluate_static_constexpr() == 84);
diff --git a/clang/test/SemaCXX/conversion-function.cpp b/clang/test/SemaCXX/conversion-function.cpp
index e5f7577d1489b..b6e6142d17906 100644
--- a/clang/test/SemaCXX/conversion-function.cpp
+++ b/clang/test/SemaCXX/conversion-function.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected                -triple %itanium_abi_triple -Wbind-to-temporary-copy %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected                -triple %itanium_abi_triple -Wbind-to-temporary-copy %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected                -triple %itanium_abi_triple -Wbind-to-temporary-copy %s
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify=expected,cxx98_11,cxx11 -triple %itanium_abi_triple -Wbind-to-temporary-copy %s
 // RUN: %clang_cc1 -std=c++98 -fsyntax-only -verify=expected,cxx98_11,cxx98 -triple %itanium_abi_triple -Wbind-to-temporary-copy %s
diff --git a/clang/test/SemaCXX/coroutines.cpp b/clang/test/SemaCXX/coroutines.cpp
index 782f4b2f63333..2292932583fff 100644
--- a/clang/test/SemaCXX/coroutines.cpp
+++ b/clang/test/SemaCXX/coroutines.cpp
@@ -1,8 +1,8 @@
 // This file contains references to sections of the Coroutines TS, which can be
 // found at http://wg21.link/coroutines.
 
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx20_2b,cxx2b    %s -fcxx-exceptions -fexceptions -Wunused-result
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_2b %s -fcxx-exceptions -fexceptions -Wunused-result
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx20_23,cxx23    %s -fcxx-exceptions -fexceptions -Wunused-result
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_23 %s -fcxx-exceptions -fexceptions -Wunused-result
 
 void no_coroutine_traits_bad_arg_await() {
   co_await a; // expected-error {{include <coroutine>}}
@@ -933,7 +933,7 @@ struct std::coroutine_traits<int, mismatch_gro_type_tag2> {
 };
 
 extern "C" int f(mismatch_gro_type_tag2) {
-  // cxx2b-error@-1 {{cannot initialize return object of type 'int' with an rvalue of type 'void *'}}
+  // cxx23-error@-1 {{cannot initialize return object of type 'int' with an rvalue of type 'void *'}}
   // cxx14_20-error@-2 {{cannot initialize return object of type 'int' with an lvalue of type 'void *'}}
   co_return; //expected-note {{function is a coroutine due to use of 'co_return' here}}
 }
@@ -1031,7 +1031,7 @@ struct await_suspend_type_test {
   // expected-error@+2 {{return type of 'await_suspend' is required to be 'void' or 'bool' (have 'bool &')}}
   // expected-error@+1 {{return type of 'await_suspend' is required to be 'void' or 'bool' (have 'bool &&')}}
   SuspendTy await_suspend(std::coroutine_handle<>);
-  // cxx20_2b-warning@-1 {{volatile-qualified return type 'const volatile bool' is deprecated}}
+  // cxx20_23-warning@-1 {{volatile-qualified return type 'const volatile bool' is deprecated}}
   void await_resume();
 };
 void test_bad_suspend() {
@@ -1053,7 +1053,7 @@ void test_bad_suspend() {
     await_suspend_type_test<bool &&> a;
     await_suspend_type_test<bool &> b;
     await_suspend_type_test<const void> c;
-    await_suspend_type_test<const volatile bool> d; // cxx20_2b-note {{in instantiation of template class}}
+    await_suspend_type_test<const volatile bool> d; // cxx20_23-note {{in instantiation of template class}}
     co_await a; // expected-note {{call to 'await_suspend' implicitly required by coroutine function here}}
     co_await b; // expected-note {{call to 'await_suspend' implicitly required by coroutine function here}}
     co_await c; // OK
@@ -1141,7 +1141,7 @@ struct TestType {
   }
 
   CoroMemberTag test_qual(int *, const float &&, volatile void *volatile) const {
-    // cxx20_2b-warning@-1 {{volatile-qualified parameter type}}
+    // cxx20_23-warning@-1 {{volatile-qualified parameter type}}
     auto TC = co_yield 0;
     static_assert(TC.MatchesArgs<const TestType &, int *, const float &&, volatile void *volatile>, "");
   }
@@ -1240,7 +1240,7 @@ struct DepTestType {
   }
 
   CoroMemberTag test_qual(int *, const float &&, volatile void *volatile) const {
-    // cxx20_2b-warning@-1 {{volatile-qualified parameter type}}
+    // cxx20_23-warning@-1 {{volatile-qualified parameter type}}
     auto TC = co_yield 0;
     static_assert(TC.template MatchesArgs<const DepTestType &, int *, const float &&, volatile void *volatile>, "");
   }
diff --git a/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp b/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp
index abeb27fd03e35..8bce70608be58 100644
--- a/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval-default-params.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++20 %s
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2b %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++23 %s
 
 consteval int undefined();  // expected-note 4 {{declared here}}
 
diff --git a/clang/test/SemaCXX/cxx2b-ast-print.cpp b/clang/test/SemaCXX/cxx2b-ast-print.cpp
index 58eb785a00771..a245647d2eb57 100644
--- a/clang/test/SemaCXX/cxx2b-ast-print.cpp
+++ b/clang/test/SemaCXX/cxx2b-ast-print.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -ast-print %s | FileCheck %s
 
 template <template <class...> class C>
 void test_auto_expr(long long y, auto &&z) {
diff --git a/clang/test/SemaCXX/cxx2b-consteval-if.cpp b/clang/test/SemaCXX/cxx2b-consteval-if.cpp
index 68921b3852f22..72ba58b676247 100644
--- a/clang/test/SemaCXX/cxx2b-consteval-if.cpp
+++ b/clang/test/SemaCXX/cxx2b-consteval-if.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 namespace PR52206 {
 constexpr auto f() {
diff --git a/clang/test/SemaCXX/cxx2b-designated-initializers.cpp b/clang/test/SemaCXX/cxx2b-designated-initializers.cpp
index 3cddb5d6410ec..3774565513673 100644
--- a/clang/test/SemaCXX/cxx2b-designated-initializers.cpp
+++ b/clang/test/SemaCXX/cxx2b-designated-initializers.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s
 
 namespace PR61118 {
 
diff --git a/clang/test/SemaCXX/cxx2b-init-statement.cpp b/clang/test/SemaCXX/cxx2b-init-statement.cpp
index de7dfcb80bc37..53c439fd7f571 100644
--- a/clang/test/SemaCXX/cxx2b-init-statement.cpp
+++ b/clang/test/SemaCXX/cxx2b-init-statement.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify -std=c++2b -Wall -Wshadow %s
+// RUN: %clang_cc1 -verify -std=c++23 -Wall -Wshadow %s
 
 void f() {
 
diff --git a/clang/test/SemaCXX/cxx2b-overloaded-operator-pedantic.cpp b/clang/test/SemaCXX/cxx2b-overloaded-operator-pedantic.cpp
index 53782c61c1c53..4a39f2a699681 100644
--- a/clang/test/SemaCXX/cxx2b-overloaded-operator-pedantic.cpp
+++ b/clang/test/SemaCXX/cxx2b-overloaded-operator-pedantic.cpp
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -verify -std=c++2b -pedantic %s
-// RUN: %clang_cc1 -verify=compat -std=c++2b -Wpre-c++2b-compat %s
+// RUN: %clang_cc1 -verify -std=c++23 -pedantic %s
+// RUN: %clang_cc1 -verify=compat -std=c++23 -Wpre-c++23-compat %s
 
 // expected-no-diagnostics
 
 struct GH61582 {
   // We accidentally would issue this diagnostic in pedantic mode; show that we
   // only issue it when enabling the compat warnings now.
-  void operator[](int, int); // compat-warning {{overloaded 'operator[]' with more than one parameter is a C++2b extension}}
+  void operator[](int, int); // compat-warning {{overloaded 'operator[]' with more than one parameter is a C++23 extension}}
 };
 
diff --git a/clang/test/SemaCXX/cxx2b-overloaded-operator.cpp b/clang/test/SemaCXX/cxx2b-overloaded-operator.cpp
index f9a83c813dcf0..c734b82987083 100644
--- a/clang/test/SemaCXX/cxx2b-overloaded-operator.cpp
+++ b/clang/test/SemaCXX/cxx2b-overloaded-operator.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify -std=c++2b %s
+// RUN: %clang_cc1 -verify -std=c++23 %s
 
 namespace N {
 
@@ -73,3 +73,47 @@ struct T2 {
 static_assert(T2{}[] == 1);
 static_assert(T2{}[1] == 2);
 static_assert(T2{}[1, 1] == 3);
+
+namespace test_packs {
+
+struct foo_t {
+template<typename... Ts>
+constexpr int operator[](Ts... idx) {
+    return (0 + ... + idx);
+}
+};
+
+template<int... Is>
+constexpr int cxx_subscript() {
+  foo_t foo;
+  return foo[Is...];
+}
+
+template<int... Is>
+int cxx_subscript_unexpanded() {
+  foo_t foo;
+  return foo[Is]; // expected-error {{expression contains unexpanded parameter pack 'Is'}}
+}
+
+template<int... Is>
+constexpr int c_array() {
+  int arr[] = {1, 2, 3};
+  return arr[Is...]; // expected-error 2{{type 'int[3]' does not provide a subscript operator}}
+}
+
+template<int... Is>
+int c_array_unexpanded() {
+  int arr[] = {1, 2, 3};
+  return arr[Is]; // expected-error {{expression contains unexpanded parameter pack 'Is'}}
+}
+
+void test() {
+  static_assert(cxx_subscript<1, 2, 3>() == 6);
+  static_assert(c_array<1>() == 2);
+
+  c_array<>(); // expected-note {{in instantiation}}
+  c_array<1>();
+  c_array<1, 2>(); // expected-note {{in instantiation}}
+}
+
+}
diff --git a/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp b/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp
index 17247c4473178..d40491834d398 100644
--- a/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp
+++ b/clang/test/SemaCXX/cxx2b-p2266-disable-with-msvc-compat.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only                    -verify=cxx2b,new %s
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -fms-compatibility -verify=cxx2b,old %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only                    -verify=cxx23,new %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fms-compatibility -verify=cxx23,old %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only                    -verify=cxx20,old %s
 
 // FIXME: This is a test for a temporary workaround where we disable simpler implicit moves
@@ -18,21 +18,21 @@ struct nocopy {
 };
 
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy mt3(nocopy x) { return x; }
 
 namespace {
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy mt3(nocopy x) { return x; }
 } // namespace
 
 namespace foo {
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 namespace std {
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy mt3(nocopy x) { return x; }
 } // namespace std
 } // namespace foo
@@ -40,18 +40,18 @@ nocopy mt3(nocopy x) { return x; }
 namespace std {
 
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy mt3(nocopy x) { return x; }
 
 namespace {
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy mt3(nocopy x) { return x; }
 } // namespace
 
 namespace foo {
 int &&mt1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &mt2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &mt2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy mt3(nocopy x) { return x; }
 } // namespace foo
 
@@ -65,22 +65,22 @@ nocopy mt3(nocopy x) { return x; }
 #elif !defined(SYSTEM)
 
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 
 namespace {
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 } // namespace
 
 namespace foo {
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 namespace std {
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 } // namespace std
 } // namespace foo
@@ -88,18 +88,18 @@ nocopy ut3(nocopy x) { return x; }
 namespace std {
 
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 
 namespace {
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 } // namespace
 
 namespace foo {
 int &&ut1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &ut2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &ut2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy ut3(nocopy x) { return x; }
 } // namespace foo
 
@@ -110,22 +110,22 @@ nocopy ut3(nocopy x) { return x; }
 #pragma GCC system_header
 
 int &&st1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &st2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &st2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy st3(nocopy x) { return x; }
 
 namespace {
 int &&st1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &st2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &st2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy st3(nocopy x) { return x; }
 } // namespace
 
 namespace foo {
 int &&st1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &st2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &st2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy st3(nocopy x) { return x; }
 namespace std {
 int &&st1(int &&x) { return x; } // cxx20-error {{cannot bind to lvalue}}
-int &st2(int &&x) { return x; }  // cxx2b-error {{cannot bind to a temporary}}
+int &st2(int &&x) { return x; }  // cxx23-error {{cannot bind to a temporary}}
 nocopy st3(nocopy x) { return x; }
 } // namespace std
 } // namespace foo
diff --git a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp
index 41acb44315b63..aa67f5368e77a 100644
--- a/clang/test/SemaCXX/deduced-return-type-cxx14.cpp
+++ b/clang/test/SemaCXX/deduced-return-type-cxx14.cpp
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx20_2b,cxx2b    %s
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx20_2b,cxx2b    %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx20_23,cxx23    %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx20_23,cxx23    %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING
 
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_2b %s
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_2b %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_23 %s
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx14_20,cxx20_23 %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING
 
 // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,cxx14_20,cxx14    %s
 // RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify=expected,cxx14_20,cxx14    %s -fdelayed-template-parsing -DDELAYED_TEMPLATE_PARSING
@@ -129,14 +129,14 @@ namespace Templates {
     return T() + 1;
   }
   template<typename T> auto &f2(T &&v) { return v; }
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
-  // cxx2b-error@-2 {{non-const lvalue reference to type 'double' cannot bind to a temporary of type 'double'}}
-  // cxx2b-note@-3  {{candidate template ignored: substitution failure [with T = double]}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-2 {{non-const lvalue reference to type 'double' cannot bind to a temporary of type 'double'}}
+  // cxx23-note@-3  {{candidate template ignored: substitution failure [with T = double]}}
   int a = f1<int>();
-  const int &b = f2(0); // cxx2b-note {{in instantiation of function template specialization 'Templates::f2<int>' requested here}}
+  const int &b = f2(0); // cxx23-note {{in instantiation of function template specialization 'Templates::f2<int>' requested here}}
   double d;
   float &c = f2(0.0); // expected-error {{non-const lvalue reference to type 'float' cannot bind to a value of unrelated type 'double'}}
-  // cxx2b-note@-1 {{in instantiation of function template specialization 'Templates::f2<double>' requested here}}
+  // cxx23-note@-1 {{in instantiation of function template specialization 'Templates::f2<double>' requested here}}
 
   template<typename T> auto fwd_decl(); // expected-note {{declared here}}
   int e = fwd_decl<int>(); // expected-error {{cannot be used before it is defined}}
@@ -150,7 +150,7 @@ namespace Templates {
   auto (*q)() = f1<int>; // ok
 
   typedef decltype(f2(1.2)) dbl; // cxx14_20-note {{previous}}
-  // cxx2b-error@-1 {{no matching function for call to 'f2'}}
+  // cxx23-error@-1 {{no matching function for call to 'f2'}}
   typedef float dbl; // cxx14_20-error {{typedef redefinition with different types ('float' vs 'decltype(f2(1.2))' (aka 'double &'))}}
 
   extern template auto fwd_decl<double>();
@@ -299,7 +299,7 @@ namespace Constexpr {
     constexpr int q = Y<int>().f(); // expected-error {{must be initialized by a constant expression}} expected-note {{in call to '&Y<int>()->f()'}}
   }
   struct NonLiteral { ~NonLiteral(); } nl; // cxx14-note {{user-provided destructor}}
-  // cxx20_2b-note@-1 {{'NonLiteral' is not literal because its destructor is not constexpr}}
+  // cxx20_23-note@-1 {{'NonLiteral' is not literal because its destructor is not constexpr}}
   constexpr auto f2(int n) { return nl; } // expected-error {{return type 'struct NonLiteral' is not a literal type}}
 }
 
diff --git a/clang/test/SemaCXX/lambda-capture-type-deduction.cpp b/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
index 8c9f6f69e91db..e524d3bc20abc 100644
--- a/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
+++ b/clang/test/SemaCXX/lambda-capture-type-deduction.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -verify -fsyntax-only %s
+// RUN: %clang_cc1 -std=c++23 -verify -fsyntax-only %s
 
 template <typename T, typename U>
 constexpr bool is_same = false;
diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp
index 67853c991ce53..aeecf2109beff 100644
--- a/clang/test/SemaCXX/lambda-expressions.cpp
+++ b/clang/test/SemaCXX/lambda-expressions.cpp
@@ -709,7 +709,7 @@ static_assert([]() constexpr {
 // be properly handled after D124351
 constexpr int i = 2;
 void foo() {
-  (void)[=][[gnu::aligned(i)]] () {}; // expected-warning{{C++2b extension}}
+  (void)[=][[gnu::aligned(i)]] () {}; // expected-warning{{C++23 extension}}
   // CHECK: AlignedAttr
   // CHECK-NEXT: ConstantExpr
   // CHECK-NEXT: value: Int 2
diff --git a/clang/test/SemaCXX/lambda-unevaluated.cpp b/clang/test/SemaCXX/lambda-unevaluated.cpp
index da00cb48a87fd..135c6f6bfc05e 100644
--- a/clang/test/SemaCXX/lambda-unevaluated.cpp
+++ b/clang/test/SemaCXX/lambda-unevaluated.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -std=c++20 %s -Wno-c++2b-extensions -verify
-// RUN: %clang_cc1 -std=c++2b %s -verify
+// RUN: %clang_cc1 -std=c++20 %s -Wno-c++23-extensions -verify
+// RUN: %clang_cc1 -std=c++23 %s -verify
 
 
 template <auto> struct Nothing {};
diff --git a/clang/test/SemaCXX/libcxx_is_trivially_equality_comparable_hack.cpp b/clang/test/SemaCXX/libcxx_is_trivially_equality_comparable_hack.cpp
index b236ce09b2970..37adc5fc0b04b 100644
--- a/clang/test/SemaCXX/libcxx_is_trivially_equality_comparable_hack.cpp
+++ b/clang/test/SemaCXX/libcxx_is_trivially_equality_comparable_hack.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++20 -fms-extensions -Wno-microsoft %s
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++2b -fms-extensions -Wno-microsoft %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++23 -fms-extensions -Wno-microsoft %s
 
 template <class T, class U>
 struct Same {
diff --git a/clang/test/SemaCXX/overloaded-operator-decl.cpp b/clang/test/SemaCXX/overloaded-operator-decl.cpp
index a21e13f3fceb8..777be6c5a0487 100644
--- a/clang/test/SemaCXX/overloaded-operator-decl.cpp
+++ b/clang/test/SemaCXX/overloaded-operator-decl.cpp
@@ -51,7 +51,7 @@ struct PR10839 {
 
 namespace PR14120 {
   struct A {
-    static void operator()(int& i) { ++i; } // expected-warning{{is a C++2b extension}}
+    static void operator()(int& i) { ++i; } // expected-warning{{is a C++23 extension}}
   };
   void f() {
     int i = 0;
diff --git a/clang/test/SemaCXX/paren-list-agg-init.cpp b/clang/test/SemaCXX/paren-list-agg-init.cpp
index a5f39ff6c477d..7bdf49b8fd0c1 100644
--- a/clang/test/SemaCXX/paren-list-agg-init.cpp
+++ b/clang/test/SemaCXX/paren-list-agg-init.cpp
@@ -9,7 +9,7 @@ struct A { // expected-note 4{{candidate constructor}}
 struct B {
   A a;
   int b[20];
-  int &&c; // expected-note {{reference member declared here}}
+  int &&c;
 };
 
 struct C { // expected-note 5{{candidate constructor}}
@@ -21,9 +21,9 @@ struct D : public C, public A {
   int a;
 };
 
-struct E { // expected-note 3{{candidate constructor}}
-  struct F {
-    F(int, int);
+struct E {
+  struct F { // expected-note 2{{candidate constructor}}
+    F(int, int); // expected-note {{candidate constructor}}
   };
   int a;
   F f;
@@ -56,6 +56,22 @@ struct J {
   int b[]; // expected-note {{initialized flexible array member 'b' is here}}
 };
 
+enum K { K0, K1, K2 };
+
+struct L {
+  K k : 1;
+};
+
+struct M {
+  struct N {
+    private:
+    N(int);
+    // expected-note@-1 {{declared private here}}
+  };
+  int i;
+  N n;
+};
+
 union U {
   int a;
   char* b;
@@ -74,7 +90,7 @@ T Construct(Args... args) {
   // beforecxx20-warning@-1 {{aggregate initialization of type 'A' from a parenthesized list of values is a C++20 extension}}
 }
 
-void foo() {
+void foo(int n) {
   A a1(1954, 9, 21);
   // expected-error@-1 {{excess elements in struct initializer}}
   A a2(2.1);
@@ -96,9 +112,8 @@ void foo() {
   B b1(2022, {7, 8});
   // expected-error@-1 {{no viable conversion from 'int' to 'A'}}
   B b2(A(1), {}, 1);
-  // expected-error@-1 {{reference member 'c' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  // beforecxx20-warning@-2 {{aggregate initialization of type 'A' from a parenthesized list of values is a C++20 extension}}
-  // beforecxx20-warning@-3 {{aggregate initialization of type 'B' from a parenthesized list of values is a C++20 extension}}
+  // beforecxx20-warning@-1 {{aggregate initialization of type 'A' from a parenthesized list of values is a C++20 extension}}
+  // beforecxx20-warning@-2 {{aggregate initialization of type 'B' from a parenthesized list of values is a C++20 extension}}
 
   C c(A(1), 1, 2, 3, 4);
   // expected-error@-1 {{array initializer must be an initializer list}}
@@ -130,7 +145,7 @@ void foo() {
   // expected-error@-1 {{excess elements in union initializer}}
 
   E e1(1);
-  // expected-error@-1 {{no matching constructor for initialization of 'E'}}
+  // expected-error@-1 {{no matching constructor for initialization of 'F'}}
 
   constexpr F f1(1);
   // expected-error@-1 {{constexpr variable 'f1' must be initialized by a constant expression}}
@@ -148,18 +163,29 @@ void foo() {
   A a7 = Construct<A>('i', 2.2);
   // beforecxx20-note@-1 {{in instantiation of function template specialization 'Construct<A, char, double>' requested here}}
 
+  L l(K::K2);
+  // expected-warning@-1 {{implicit truncation}}
+  // beforecxx20-warning@-2 {{aggregate initialization of type 'L' from a parenthesized list of values is a C++20 extension}}
+
   int arr4[](1, 2);
   // beforecxx20-warning@-1 {{aggregate initialization of type 'int[2]' from a parenthesized list of values is a C++20 extension}}
 
   int arr5[2](1, 2);
   // beforecxx20-warning@-1 {{aggregate initialization of type 'int[2]' from a parenthesized list of values is a C++20 extension}}
 
+  int arr6[n](1, 2, 3);
+  // expected-error@-1 {{variable-sized object may not be initialized}}
+
   I i(1, 2);
   // expected-error@-1 {{no matching constructor for initialization of 'I'}}
 
   J j(1, {2, 3});
   // expected-error@-1 {{initialization of flexible array member is not allowed}}
 
+  M m(1, 1);
+  // expected-error@-1 {{field of type 'N' has private constructor}}
+  // beforecxx20-warning@-2 {{aggregate initialization of type 'M' from a parenthesized list of values is a C++20 extension}}
+
   static_assert(__is_trivially_constructible(A, char, double));
   static_assert(__is_trivially_constructible(A, char, int));
   static_assert(__is_trivially_constructible(A, char));
@@ -200,3 +226,43 @@ void bar() {
   // expected-error@-1 {{call to implicitly-deleted copy constructor of 'V'}}
 }
 }
+
+namespace gh62296 {
+struct L {
+protected:
+  L(int);
+  // expected-note@-1 2{{declared protected here}}
+};
+
+struct M : L {};
+
+struct N {
+  L l;
+};
+
+M m(42);
+// expected-error@-1 {{base class 'L' has protected constructor}}
+// beforecxx20-warning@-2 {{aggregate initialization of type 'M' from a parenthesized list of values is a C++20 extension}}
+
+N n(43);
+// expected-error@-1 {{field of type 'L' has protected constructor}}
+// beforecxx20-warning@-2 {{aggregate initialization of type 'N' from a parenthesized list of values is a C++20 extension}}
+}
+
+namespace gh61567 {
+struct O {
+  int i;
+  int &&j;
+  // expected-note@-1 {{uninitialized reference member is here}}
+  int &&k = 1;
+};
+
+O o1(0, 0, 0); // no-error
+// beforecxx20-warning@-1 {{aggregate initialization of type 'O' from a parenthesized list of values is a C++20 extension}}
+
+O o2(0, 0); // no-error
+// beforecxx20-warning@-1 {{aggregate initialization of type 'O' from a parenthesized list of values is a C++20 extension}}
+
+O o3(0);
+// expected-error@-1 {{reference member of type 'int &&' uninitialized}}
+}
diff --git a/clang/test/SemaCXX/return-stack-addr.cpp b/clang/test/SemaCXX/return-stack-addr.cpp
index 2f3b5b5d6c660..a66197b5acf62 100644
--- a/clang/test/SemaCXX/return-stack-addr.cpp
+++ b/clang/test/SemaCXX/return-stack-addr.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -verify=expected,cxx2b          %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx23          %s
 // RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx11_20       %s
 // RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify=expected,cxx11_20,cxx11 %s
 
@@ -30,7 +30,7 @@ int* ret_local_array_element_const_index() {
 int& ret_local_ref() {
   int x = 1;
   return x; // cxx11_20-warning {{reference to stack memory}}
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
 }
 
 int* ret_local_addrOf() {
@@ -156,9 +156,9 @@ void ret_from_lambda() {
   (void) [=]() mutable -> int& { return a; };
   (void) [=]() mutable -> int& { return b; };
   (void) [&]() -> int& { int a; return a; }; // cxx11_20-warning {{reference to stack}}
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
   (void) [=]() -> int& { int a; return a; }; // cxx11_20-warning {{reference to stack}}
-  // cxx2b-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
+  // cxx23-error@-1 {{non-const lvalue reference to type 'int' cannot bind to a temporary of type 'int'}}
   (void) [&]() -> int& { int &a = b; return a; };
   (void) [=]() mutable -> int& { int &a = b; return a; };
 
diff --git a/clang/test/SemaCXX/riscv-rvv-explicit-casts-fixed-size.cpp b/clang/test/SemaCXX/riscv-rvv-explicit-casts-fixed-size.cpp
new file mode 100644
index 0000000000000..74acfdf39b779
--- /dev/null
+++ b/clang/test/SemaCXX/riscv-rvv-explicit-casts-fixed-size.cpp
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=1 -mvscale-max=1 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=2 -mvscale-max=2 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=4 -mvscale-max=4 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=16 -mvscale-max=16 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify %s
+
+// REQUIRES: riscv-registered-target
+
+// expected-no-diagnostics
+
+#define FIXED_ATTR __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)))
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vfloat32m1_t fixed_float32m1_t FIXED_ATTR;
+typedef vfloat64m1_t fixed_float64m1_t FIXED_ATTR;
+typedef vint32m1_t fixed_int32m1_t FIXED_ATTR;
+typedef vint64m1_t fixed_int64m1_t FIXED_ATTR;
+
+// RVV VLS types can be cast to RVV VLA types, regardless of lane size.
+// NOTE: the list below is NOT exhaustive for all RVV types.
+
+#define CAST(from, to) \
+    void from##_to_##to(from a, to b) { \
+        b = (to) a; \
+    }
+
+#define TESTCASE(ty1, ty2) \
+    CAST(ty1, ty2) \
+    CAST(ty2, ty1)
+
+TESTCASE(fixed_float32m1_t, vfloat32m1_t)
+TESTCASE(fixed_float32m1_t, vfloat64m1_t)
+TESTCASE(fixed_float32m1_t, vint32m1_t)
+TESTCASE(fixed_float32m1_t, vint64m1_t)
+
+TESTCASE(fixed_float64m1_t, vfloat32m1_t)
+TESTCASE(fixed_float64m1_t, vfloat64m1_t)
+TESTCASE(fixed_float64m1_t, vint32m1_t)
+TESTCASE(fixed_float64m1_t, vint64m1_t)
+
+TESTCASE(fixed_int32m1_t, vfloat32m1_t)
+TESTCASE(fixed_int32m1_t, vfloat64m1_t)
+TESTCASE(fixed_int32m1_t, vint32m1_t)
+TESTCASE(fixed_int32m1_t, vint64m1_t)
+
+TESTCASE(fixed_int64m1_t, vfloat32m1_t)
+TESTCASE(fixed_int64m1_t, vfloat64m1_t)
+TESTCASE(fixed_int64m1_t, vint32m1_t)
+TESTCASE(fixed_int64m1_t, vint64m1_t)
diff --git a/clang/test/SemaCXX/riscv-rvv-lax-vector-conversions.cpp b/clang/test/SemaCXX/riscv-rvv-lax-vector-conversions.cpp
new file mode 100644
index 0000000000000..ff3e028aa314d
--- /dev/null
+++ b/clang/test/SemaCXX/riscv-rvv-lax-vector-conversions.cpp
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=none -ffreestanding -fsyntax-only -verify=lax-vector-none %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=integer -ffreestanding -fsyntax-only -verify=lax-vector-integer %s
+// RUN: %clang_cc1 -triple riscv64-none-linux-gnu -target-feature +f -target-feature +d -target-feature +zve64d -mvscale-min=8 -mvscale-max=8 -flax-vector-conversions=all -ffreestanding -fsyntax-only -verify=lax-vector-all %s
+
+// lax-vector-all-no-diagnostics
+
+// REQUIRES: riscv-registered-target
+
+#define RVV_FIXED_ATTR __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)))
+#define GNU_FIXED_ATTR __attribute__((vector_size(__riscv_v_fixed_vlen / 8)))
+
+typedef __rvv_int8m1_t vint8m1_t;
+typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int16m1_t vint16m1_t;
+typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int32m1_t vint32m1_t;
+typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int64m1_t vint64m1_t;
+typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float64m1_t vfloat64m1_t;
+
+typedef vfloat32m1_t rvv_fixed_float32m1_t RVV_FIXED_ATTR;
+typedef vint32m1_t rvv_fixed_int32m1_t RVV_FIXED_ATTR;
+typedef float gnu_fixed_float32m1_t GNU_FIXED_ATTR;
+typedef int gnu_fixed_int32m1_t GNU_FIXED_ATTR;
+
+void rvv_allowed_with_integer_lax_conversions() {
+  rvv_fixed_int32m1_t fi32;
+  vint64m1_t si64;
+
+  // The implicit cast here should fail if -flax-vector-conversions=none, but pass if
+  // -flax-vector-conversions={integer,all}.
+  fi32 = si64;
+  // lax-vector-none-error@-1 {{assigning to 'rvv_fixed_int32m1_t' (vector of 16 'int' values) from incompatible type}}
+  si64 = fi32;
+  // lax-vector-none-error@-1 {{assigning to 'vint64m1_t' (aka '__rvv_int64m1_t') from incompatible type}}
+}
+
+void rvv_allowed_with_all_lax_conversions() {
+  rvv_fixed_float32m1_t ff32;
+  vfloat64m1_t sf64;
+
+  // The implicit cast here should fail if -flax-vector-conversions={none,integer}, but pass if
+  // -flax-vector-conversions=all.
+  ff32 = sf64;
+  // lax-vector-none-error@-1 {{assigning to 'rvv_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'rvv_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  sf64 = ff32;
+  // lax-vector-none-error@-1 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+}
+
+void gnu_allowed_with_integer_lax_conversions() {
+  gnu_fixed_int32m1_t fi32;
+  vint64m1_t si64;
+
+  // The implicit cast here should fail if -flax-vector-conversions=none, but pass if
+  // -flax-vector-conversions={integer,all}.
+  fi32 = si64;
+  // lax-vector-none-error@-1 {{assigning to 'gnu_fixed_int32m1_t' (vector of 16 'int' values) from incompatible type}}
+  si64 = fi32;
+  // lax-vector-none-error@-1 {{assigning to 'vint64m1_t' (aka '__rvv_int64m1_t') from incompatible type}}
+}
+
+void gnu_allowed_with_all_lax_conversions() {
+  gnu_fixed_float32m1_t ff32;
+  vfloat64m1_t sf64;
+
+  // The implicit cast here should fail if -flax-vector-conversions={none,integer}, but pass if
+  // -flax-vector-conversions=all.
+  ff32 = sf64;
+  // lax-vector-none-error@-1 {{assigning to 'gnu_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'gnu_fixed_float32m1_t' (vector of 16 'float' values) from incompatible type}}
+  sf64 = ff32;
+  // lax-vector-none-error@-1 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+  // lax-vector-integer-error@-2 {{assigning to 'vfloat64m1_t' (aka '__rvv_float64m1_t') from incompatible type}}
+}
diff --git a/clang/test/SemaCXX/size_t-literal.cpp b/clang/test/SemaCXX/size_t-literal.cpp
index 02e644697002e..73be677d47bd2 100644
--- a/clang/test/SemaCXX/size_t-literal.cpp
+++ b/clang/test/SemaCXX/size_t-literal.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -std=c++2b -triple x86_64-linux -Wpre-c++2b-compat -fsyntax-only -verify=cxx2b %s
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-linux -Wpre-c++23-compat -fsyntax-only -verify=cxx23 %s
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-linux -fsyntax-only -verify=cxx20 %s
-// RUN: %clang_cc1 -std=c++2b -triple i686-linux -fsyntax-only -verify=cxx2b-32 %s
+// RUN: %clang_cc1 -std=c++23 -triple i686-linux -fsyntax-only -verify=cxx23-32 %s
 // RUN: %clang_cc1 -x c -std=c11 -fsyntax-only -verify=c11 %s
 
 #ifdef __cplusplus
@@ -17,83 +17,83 @@ struct is_same<T, T> { static constexpr bool value = true; };
 
 void SSizeT(void) {
   auto a1 = 1z;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a1), ssize_t>::value);
 
   auto a2 = 1Z;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a2), ssize_t>::value);
 }
 
 void SizeT(void) {
   auto a1 = 1uz;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a1), size_t>::value);
 
   auto a2 = 1uZ;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a2), size_t>::value);
 
   auto a3 = 1Uz;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a3), size_t>::value);
 
   auto a4 = 1UZ;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a4), size_t>::value);
 
   auto a5 = 1zu;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a5), size_t>::value);
 
   auto a6 = 1Zu;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a6), size_t>::value);
 
   auto a7 = 1zU;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a7), size_t>::value);
 
   auto a8 = 1ZU;
-  // cxx2b-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++2b}}
-  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++2b extension}}
+  // cxx23-warning@-1 {{'size_t' suffix for literals is incompatible with C++ standards before C++23}}
+  // cxx20-warning@-2 {{'size_t' suffix for literals is a C++23 extension}}
   static_assert(is_same<decltype(a8), size_t>::value);
 }
 
 void oor(void) {
 #if __i386__
-  (void)3'000'000'000z; // cxx2b-32-error {{signed 'size_t' literal is out of range of possible signed 'size_t' values}}
+  (void)3'000'000'000z; // cxx23-32-error {{signed 'size_t' literal is out of range of possible signed 'size_t' values}}
   (void)3'000'000'000uz;
-  (void)5'000'000'000uz; // cxx2b-32-error {{'size_t' literal is out of range of possible 'size_t' values}}
+  (void)5'000'000'000uz; // cxx23-32-error {{'size_t' literal is out of range of possible 'size_t' values}}
 
   (void)0x80000000z;
   (void)0x80000000uz;
-  (void)0x180000000uz; //cxx2b-32-error {{'size_t' literal is out of range of possible 'size_t' values}}
+  (void)0x180000000uz; //cxx23-32-error {{'size_t' literal is out of range of possible 'size_t' values}}
 #endif
 }
 
 #else
 
 void f(void) {
-  (void)1z;  // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1Z;  // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1uz; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1uZ; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1Uz; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1UZ; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1zu; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1Zu; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1zU; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
-  (void)1ZU; // c11-error {{'size_t' suffix for literals is a C++2b feature}}
+  (void)1z;  // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1Z;  // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1uz; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1uZ; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1Uz; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1UZ; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1zu; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1Zu; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1zU; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
+  (void)1ZU; // c11-error {{'size_t' suffix for literals is a C++23 feature}}
 }
 
 #endif
diff --git a/clang/test/SemaCXX/undefined-partial-specialization.cpp b/clang/test/SemaCXX/undefined-partial-specialization.cpp
new file mode 100644
index 0000000000000..b07a513270fd6
--- /dev/null
+++ b/clang/test/SemaCXX/undefined-partial-specialization.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -std=c++17 -verify %s
+// RUN: %clang_cc1 -std=c++20 -verify %s
+
+namespace GH61356 {
+
+template <typename T, bool b>
+class boo {void foo();};
+
+template <typename T>
+class boo<T, true>;
+
+template<typename T>
+void boo<T, true>::foo(){} // expected-error{{out-of-line definition of 'foo' from class 'boo<type-parameter-0-0, true>' without definition}}
+
+}
diff --git a/clang/test/SemaCXX/warn-shadow-in-lambdas.cpp b/clang/test/SemaCXX/warn-shadow-in-lambdas.cpp
index 9dddbf8b5f3b7..e5d4c69dcee06 100644
--- a/clang/test/SemaCXX/warn-shadow-in-lambdas.cpp
+++ b/clang/test/SemaCXX/warn-shadow-in-lambdas.cpp
@@ -2,6 +2,7 @@
 // RUN: %clang_cc1 -std=c++14 -verify -fsyntax-only -Wshadow -Wshadow-uncaptured-local %s
 // RUN: %clang_cc1 -std=c++14 -verify -fsyntax-only -Wshadow-all %s
 // RUN: %clang_cc1 -std=c++17 -verify -fsyntax-only -Wshadow-all %s
+// RUN: %clang_cc1 -std=c++20 -verify -fsyntax-only -Wshadow-all %s
 
 void foo(int param) { // expected-note 1+ {{previous declaration is here}}
   int var = 0; // expected-note 1+ {{previous declaration is here}}
@@ -146,3 +147,22 @@ void avoidWarningWhenRedefining() {
     int b = 0; // expected-error {{redefinition of 'b'}}
   };
 }
+
+namespace GH61105 {
+void f() {
+  int y = 0;
+  int x = 0;
+#if __cplusplus >= 202002L
+  auto l1 = [y]<typename y>(y) { return 0; }; // expected-error {{declaration of 'y' shadows template parameter}} \
+                                              // expected-note {{template parameter is declared here}}
+  auto l2 = [=]<typename y>() { int a = y; return 0; }; // expected-error {{'y' does not refer to a value}} \
+                                                        // expected-note {{declared here}}
+  auto l3 = [&, y]<typename y, typename>(y) { int a = x; return 0; }; // expected-error {{declaration of 'y' shadows template parameter}} \
+                                                                      // expected-note {{template parameter is declared here}}
+  auto l4 = [x, y]<typename y, int x>() { return 0; }; // expected-error {{declaration of 'y' shadows template parameter}} \
+                                                       // expected-error {{declaration of 'x' shadows template parameter}} \
+                                                       // expected-note 2{{template parameter is declared here}}
+  auto l5 = []<typename y>(y) { return 0; }; // No diagnostic
+#endif
+}
+}
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-local-var-span.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-local-var-span.cpp
index ca3f7a4b172a8..cb6519a153fee 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-local-var-span.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-local-var-span.cpp
@@ -2,6 +2,8 @@
 typedef int * Int_ptr_t;
 typedef int Int_t;
 
+#define DEFINE_PTR(X) int* ptr = (X);
+
 void local_array_subscript_simple() {
   int tmp;
   int *p = new int[10];
@@ -222,3 +224,28 @@ void unsupported_subscript_negative(int i, unsigned j, unsigned long k) {
   tmp = r[j] + r[k]; // both `j` and `k` are unsigned so they must be non-negative
   tmp = r[(unsigned int)-1]; // a cast-to-unsigned-expression is also non-negative
 }
+
+void all_vars_in_macro() {
+  int* local;
+  DEFINE_PTR(local)
+  ptr[1] = 0;
+}
+
+void few_vars_in_macro() {
+  int* local;
+  DEFINE_PTR(local)
+  ptr[1] = 0;
+  int tmp;
+  ptr[2] = 30;
+  auto p = new int[10];
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:11}:"std::span<int> p"
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"{"
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-3]]:23-[[@LINE-3]]:23}:", 10}"
+  tmp = p[5];
+  int val = *p;
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-1]]:13-[[@LINE-1]]:14}:""
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-2]]:15-[[@LINE-2]]:15}:"[0]"
+  val = *p + 30;
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-1]]:9-[[@LINE-1]]:10}:""
+  // CHECK-DAG: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:"[0]"
+}
diff --git a/clang/test/SemaObjC/nullable-result.m b/clang/test/SemaObjC/nullable-result.m
index 6125141e8c261..17cc1dd63810a 100644
--- a/clang/test/SemaObjC/nullable-result.m
+++ b/clang/test/SemaObjC/nullable-result.m
@@ -25,9 +25,9 @@ void test_conversion(void) {
 }
 
 void test_dup(void) {
-  id _Nullable_result _Nullable_result a; // expected-warning {{duplicate nullability specifier _Nullable_result}}
-  id _Nullable _Nullable_result b; // expected-error{{nullability specifier _Nullable_result conflicts with existing specifier '_Nullable'}}
-  id _Nullable_result _Nonnull c; // expected-error{{nullability specifier '_Nonnull' conflicts with existing specifier _Nullable_result}}
+  id _Nullable_result _Nullable_result a; // expected-warning {{duplicate nullability specifier '_Nullable_result'}}
+  id _Nullable _Nullable_result b; // expected-error{{nullability specifier '_Nullable_result' conflicts with existing specifier '_Nullable'}}
+  id _Nullable_result _Nonnull c; // expected-error{{nullability specifier '_Nonnull' conflicts with existing specifier '_Nullable_result'}}
 }
 
 @interface NoContextSensitive
diff --git a/clang/test/SemaObjCXX/block-capture.mm b/clang/test/SemaObjCXX/block-capture.mm
index c89c088a871ba..8ba02f919e015 100644
--- a/clang/test/SemaObjCXX/block-capture.mm
+++ b/clang/test/SemaObjCXX/block-capture.mm
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -fobjc-arc -fblocks                       -verify=cxx98_2b,cxx11_2b,cxx2b %s
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fobjc-arc -fblocks                       -verify=cxx98_2b,cxx11_2b       %s
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fobjc-arc -fblocks                       -verify=cxx98_2b,cxx11_2b       %s
-// RUN: %clang_cc1 -std=c++98 -fsyntax-only -fobjc-arc -fblocks -Wno-c++11-extensions -verify=cxx98_2b,cxx98          %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fobjc-arc -fblocks                       -verify=cxx98_23,cxx11_23,cxx23 %s
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fobjc-arc -fblocks                       -verify=cxx98_23,cxx11_23       %s
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fobjc-arc -fblocks                       -verify=cxx98_23,cxx11_23       %s
+// RUN: %clang_cc1 -std=c++98 -fsyntax-only -fobjc-arc -fblocks -Wno-c++11-extensions -verify=cxx98_23,cxx98          %s
 
 #define TEST(T) void test_##T() { \
   __block T x;                    \
@@ -9,10 +9,10 @@
 }
 
 struct CopyOnly {
-  CopyOnly();           // cxx2b-note {{not viable}}
-  CopyOnly(CopyOnly &); // cxx2b-note {{not viable}}
+  CopyOnly();           // cxx23-note {{not viable}}
+  CopyOnly(CopyOnly &); // cxx23-note {{not viable}}
 };
-TEST(CopyOnly); // cxx2b-error {{no matching constructor}}
+TEST(CopyOnly); // cxx23-error {{no matching constructor}}
 
 // Both ConstCopyOnly and NonConstCopyOnly are
 // "pure" C++98 tests (pretend 'delete' means 'private').
@@ -28,16 +28,16 @@
 struct NonConstCopyOnly {
   NonConstCopyOnly();
   NonConstCopyOnly(NonConstCopyOnly &);
-  NonConstCopyOnly(const NonConstCopyOnly &) = delete; // cxx11_2b-note {{marked deleted here}}
+  NonConstCopyOnly(const NonConstCopyOnly &) = delete; // cxx11_23-note {{marked deleted here}}
 };
-TEST(NonConstCopyOnly); // cxx11_2b-error {{call to deleted constructor}}
+TEST(NonConstCopyOnly); // cxx11_23-error {{call to deleted constructor}}
 
 struct CopyNoMove {
   CopyNoMove();
   CopyNoMove(CopyNoMove &);
-  CopyNoMove(CopyNoMove &&) = delete; // cxx98_2b-note {{marked deleted here}}
+  CopyNoMove(CopyNoMove &&) = delete; // cxx98_23-note {{marked deleted here}}
 };
-TEST(CopyNoMove); // cxx98_2b-error {{call to deleted constructor}}
+TEST(CopyNoMove); // cxx98_23-error {{call to deleted constructor}}
 
 struct MoveOnly {
   MoveOnly();
@@ -49,9 +49,9 @@
 struct NoCopyNoMove {
   NoCopyNoMove();
   NoCopyNoMove(NoCopyNoMove &) = delete;
-  NoCopyNoMove(NoCopyNoMove &&) = delete; // cxx98_2b-note {{marked deleted here}}
+  NoCopyNoMove(NoCopyNoMove &&) = delete; // cxx98_23-note {{marked deleted here}}
 };
-TEST(NoCopyNoMove); // cxx98_2b-error {{call to deleted constructor}}
+TEST(NoCopyNoMove); // cxx98_23-error {{call to deleted constructor}}
 
 struct ConvertingRVRef {
   ConvertingRVRef();
@@ -69,11 +69,11 @@
   ConvertingCLVRef(ConvertingCLVRef &);
 
   struct X {};
-  ConvertingCLVRef(X &&); // cxx98_2b-note {{passing argument to parameter here}}
+  ConvertingCLVRef(X &&); // cxx98_23-note {{passing argument to parameter here}}
   operator X() const &;
-  operator X() && = delete; // cxx98_2b-note {{marked deleted here}}
+  operator X() && = delete; // cxx98_23-note {{marked deleted here}}
 };
-TEST(ConvertingCLVRef); // cxx98_2b-error {{invokes a deleted function}}
+TEST(ConvertingCLVRef); // cxx98_23-error {{invokes a deleted function}}
 
 struct SubSubMove {};
 struct SubMove : SubSubMove {
diff --git a/clang/test/SemaOpenCLCXX/address-space-lambda.clcpp b/clang/test/SemaOpenCLCXX/address-space-lambda.clcpp
index 66dc4da13d2c3..54c7c88087be8 100644
--- a/clang/test/SemaOpenCLCXX/address-space-lambda.clcpp
+++ b/clang/test/SemaOpenCLCXX/address-space-lambda.clcpp
@@ -63,7 +63,7 @@ __kernel void test_qual() {
 
   [&] __private {} (); // expected-error{{no matching function for call to object of type '(lambda at}} expected-note{{candidate function not viable: 'this' object is in default address space, but method expects object in address space '__private'}}
 #if __cplusplus <= 202002L
-// expected-warning@-2{{lambda without a parameter clause is a C++2b extension}}
+// expected-warning@-2{{lambda without a parameter clause is a C++23 extension}}
 #endif
 
   [&] () mutable __private {} ();
diff --git a/clang/test/SemaTemplate/concepts-inherited-ctor.cpp b/clang/test/SemaTemplate/concepts-inherited-ctor.cpp
new file mode 100644
index 0000000000000..6ac5a1618784e
--- /dev/null
+++ b/clang/test/SemaTemplate/concepts-inherited-ctor.cpp
@@ -0,0 +1,91 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s
+
+namespace GH62361 {
+  template <typename T, typename U = void*> struct B { // expected-note 14{{candidate}}
+    B() // expected-note 7{{not viable}}
+      requires __is_same(T, int); // expected-note 7{{because '__is_same(char, int)' evaluated to false}}
+  };
+
+  template <typename U> struct B<void, U> : B<int, U> {
+    using B<int, U>::B;
+  };
+
+  template<typename T>
+  void g(B<T>); // expected-note {{cannot convert}}
+
+  void f1() {
+    B<void> b1;
+    B<void> b2{};
+    B<void> b3 = {};
+    new B<void>{};
+    new B<void>();
+    g<void>({});
+    B<void>{};
+    B<void>();
+  }
+
+  void f2() {
+    B<int> b1;
+    B<int> b2{};
+    B<int> b3 = {};
+    new B<int>{};
+    new B<int>();
+    g<int>({});
+    B<int>{};
+    B<int>();
+  }
+
+  void f3() {
+    B<char> b1; // expected-error {{no matching constructor}}
+    B<char> b2{}; // expected-error {{no matching constructor}}
+    B<char> b3 = {}; // expected-error {{no matching constructor}}
+    new B<char>{}; // expected-error {{no matching constructor}}
+    new B<char>(); // expected-error {{no matching constructor}}
+    g<char>({}); // expected-error {{no matching function}}
+    B<char>{}; // expected-error {{no matching constructor}}
+    B<char>(); // expected-error {{no matching constructor}}
+  }
+}
+
+namespace no_early_substitution {
+  template <typename T> concept X = true;
+
+  struct A {};
+
+  template <typename T> struct B {
+    B() requires X<T*>;
+    B();
+  };
+
+  template <typename U = int, typename V = A>
+  struct C : public B<V&> {
+    using B<V&>::B;
+  };
+
+  void foo() {
+    // OK, we only substitute T ~> V& into X<T*> in a SFINAE context,
+    // during satisfaction checks.
+    C();
+  }
+}
+
+namespace GH62362 {
+  template<typename T>
+    concept C = true;
+  template <typename T> struct Test {
+    Test()
+      requires(C<T>);
+  };
+  struct Bar : public Test<int> {
+    using Test<int>::Test;
+  };
+  template <>
+    struct Test<void> : public Test<int> {
+      using Test<int>::Test;
+    };
+
+  void foo() {
+    Bar();
+    Test<void>();
+  }
+}
diff --git a/clang/test/SemaTemplate/deduction.cpp b/clang/test/SemaTemplate/deduction.cpp
index 7a45d1a425728..54bb9d6cfb158 100644
--- a/clang/test/SemaTemplate/deduction.cpp
+++ b/clang/test/SemaTemplate/deduction.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++11
-// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++1z
-// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++2b
+// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++17
+// RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++23
 
 // Template argument deduction with template template parameters.
 template<typename T, template<T> class A> 
diff --git a/clang/test/SemaTemplate/instantiate-subscript.cpp b/clang/test/SemaTemplate/instantiate-subscript.cpp
index 3015f720cf1f1..cb20f2dd721f7 100644
--- a/clang/test/SemaTemplate/instantiate-subscript.cpp
+++ b/clang/test/SemaTemplate/instantiate-subscript.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2b %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++23 %s
 
 struct Sub0 {
   int &operator[](int);
diff --git a/clang/test/SemaTemplate/instantiation-dependence.cpp b/clang/test/SemaTemplate/instantiation-dependence.cpp
index 82665b26b1229..1a97dbf769ec6 100644
--- a/clang/test/SemaTemplate/instantiation-dependence.cpp
+++ b/clang/test/SemaTemplate/instantiation-dependence.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 // Ensure we substitute into instantiation-dependent but non-dependent
 // constructs. The poster-child for this is...
diff --git a/clang/test/SemaTemplate/make_integer_seq.cpp b/clang/test/SemaTemplate/make_integer_seq.cpp
index 12505b0a7481b..12520167b93e5 100644
--- a/clang/test/SemaTemplate/make_integer_seq.cpp
+++ b/clang/test/SemaTemplate/make_integer_seq.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -triple x86_64-linux-gnu -ast-dump -verify -xc++ < %s | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -triple x86_64-linux-gnu -ast-dump -verify -xc++ < %s | FileCheck %s
 
 template <class A1, A1... A2> struct A {};
 
diff --git a/clang/test/SemaTemplate/pr52909.cpp b/clang/test/SemaTemplate/pr52909.cpp
index ccde5903342a6..998a88ae7be27 100644
--- a/clang/test/SemaTemplate/pr52909.cpp
+++ b/clang/test/SemaTemplate/pr52909.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -std=c++20 -verify %s
-// RUN: %clang_cc1 -std=c++2b -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 namespace PR52905 {
 template <class> concept C = true;
diff --git a/clang/test/SemaTemplate/template-friend-definition-in-template.cpp b/clang/test/SemaTemplate/template-friend-definition-in-template.cpp
new file mode 100644
index 0000000000000..cfcf00b58b026
--- /dev/null
+++ b/clang/test/SemaTemplate/template-friend-definition-in-template.cpp
@@ -0,0 +1,26 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+template <class F1> int foo1(F1 X1);
+
+template <int A1> struct A {
+  template <class F2> friend int foo1(F2 X2) {
+    return A1;
+  }
+};
+
+template struct A<1>;
+int main() { 
+  foo1(1.0);
+}
+
+template <class F1> int foo2(F1 X1);
+
+template <int A1> struct B {
+  template <class F2> friend int foo2(F2 X2) {
+    return A1;
+  }
+};
+
+template struct B<1>;
+template int foo2<float>(float X1);
diff --git a/clang/test/SemaTemplate/type_pack_element.cpp b/clang/test/SemaTemplate/type_pack_element.cpp
index c9d6b98a1cec1..260b90bf506c5 100644
--- a/clang/test/SemaTemplate/type_pack_element.cpp
+++ b/clang/test/SemaTemplate/type_pack_element.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2b -fsyntax-only -triple x86_64-linux-gnu -ast-dump -verify -xc++ < %s | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -triple x86_64-linux-gnu -ast-dump -verify -xc++ < %s | FileCheck %s
 
 using test1 = __type_pack_element<0, int>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <<stdin>:3:1, col:41> col:7 test1 '__type_pack_element<0, int>':'int'
diff --git a/clang/test/clang-rename/NonExistFile.cpp b/clang/test/clang-rename/NonExistFile.cpp
new file mode 100644
index 0000000000000..f45839be80473
--- /dev/null
+++ b/clang/test/clang-rename/NonExistFile.cpp
@@ -0,0 +1,2 @@
+// RUN: not clang-rename -offset=0 -new-name=bar non-existing-file 2>&1 | FileCheck %s
+// CHECK: clang-rename: non-existing-file does not exist.
diff --git a/clang/tools/clang-rename/ClangRename.cpp b/clang/tools/clang-rename/ClangRename.cpp
index e7ceac7dbf303..24c9d8521dc27 100644
--- a/clang/tools/clang-rename/ClangRename.cpp
+++ b/clang/tools/clang-rename/ClangRename.cpp
@@ -229,6 +229,10 @@ int main(int argc, const char **argv) {
     Tool.applyAllReplacements(Rewrite);
     for (const auto &File : Files) {
       auto Entry = FileMgr.getFile(File);
+      if (!Entry) {
+        errs() << "clang-rename: " << File << " does not exist.\n";
+        return 1;
+      }
       const auto ID = Sources.getOrCreateFileID(*Entry, SrcMgr::C_User);
       Rewrite.getEditBuffer(ID).write(outs());
     }
diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index ee54a0f7b0eab..ecfb1be8c55af 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -97,7 +97,7 @@ struct AssemblerInvocation {
   std::string DwarfDebugFlags;
   std::string DwarfDebugProducer;
   std::string DebugCompilationDir;
-  std::map<const std::string, const std::string> DebugPrefixMap;
+  llvm::SmallVector<std::pair<std::string, std::string>, 0> DebugPrefixMap;
   llvm::DebugCompressionType CompressDebugSections =
       llvm::DebugCompressionType::None;
   std::string MainFileName;
@@ -275,8 +275,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
 
   for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ)) {
     auto Split = StringRef(Arg).split('=');
-    Opts.DebugPrefixMap.insert(
-        {std::string(Split.first), std::string(Split.second)});
+    Opts.DebugPrefixMap.emplace_back(Split.first, Split.second);
   }
 
   // Frontend Options
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index bb536112e2094..448681bfe533a 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -3964,7 +3964,7 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
       TUKind, CacheCodeCompletionResults, IncludeBriefCommentsInCodeCompletion,
       /*AllowPCHWithCompilerErrors=*/true, SkipFunctionBodies, SingleFileParse,
       /*UserFilesAreVolatile=*/true, ForSerialization, RetainExcludedCB,
-      CXXIdx->getPCHContainerOperations()->getRawReader().getFormat(),
+      CXXIdx->getPCHContainerOperations()->getRawReader().getFormats().front(),
       &ErrUnit));
 
   // Early failures in LoadFromCommandLine may return with ErrUnit unset.
diff --git a/clang/tools/libclang/Indexing.cpp b/clang/tools/libclang/Indexing.cpp
index 24ca98d4de416..b6dfb1f47b598 100644
--- a/clang/tools/libclang/Indexing.cpp
+++ b/clang/tools/libclang/Indexing.cpp
@@ -552,7 +552,7 @@ static CXErrorCode clang_indexSourceFile_Impl(
 
   // Make sure to use the raw module format.
   CInvok->getHeaderSearchOpts().ModuleFormat = std::string(
-      CXXIdx->getPCHContainerOperations()->getRawReader().getFormat());
+      CXXIdx->getPCHContainerOperations()->getRawReader().getFormats().front());
 
   auto Unit = ASTUnit::create(CInvok, Diags, CaptureDiagnostics,
                               /*UserFilesAreVolatile=*/true);
diff --git a/clang/unittests/Analysis/FlowSensitive/LoggerTest.cpp b/clang/unittests/Analysis/FlowSensitive/LoggerTest.cpp
index 0b8579a45b2df..7201771e0b118 100644
--- a/clang/unittests/Analysis/FlowSensitive/LoggerTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/LoggerTest.cpp
@@ -37,14 +37,16 @@ class TestAnalysis : public DataflowAnalysis<TestAnalysis, TestLattice> {
 
   static TestLattice initialElement() { return TestLattice{}; }
   void transfer(const CFGElement &, TestLattice &L, Environment &E) {
-    E.logger().log([](llvm::raw_ostream &OS) { OS << "transfer()"; });
+    E.getDataflowAnalysisContext().getOptions().Log->log(
+        [](llvm::raw_ostream &OS) { OS << "transfer()"; });
     ++L.Elements;
   }
   void transferBranch(bool Branch, const Stmt *S, TestLattice &L,
                       Environment &E) {
-    E.logger().log([&](llvm::raw_ostream &OS) {
-      OS << "transferBranch(" << Branch << ")";
-    });
+    E.getDataflowAnalysisContext().getOptions().Log->log(
+        [&](llvm::raw_ostream &OS) {
+          OS << "transferBranch(" << Branch << ")";
+        });
     ++L.Branches;
   }
 };
@@ -176,6 +178,7 @@ TEST(LoggerTest, HTML) {
       << "has analysis point state";
   EXPECT_THAT(Logs[0], HasSubstr("transferBranch(0)")) << "has analysis logs";
   EXPECT_THAT(Logs[0], HasSubstr("LocToVal")) << "has built-in lattice dump";
+  EXPECT_THAT(Logs[0], HasSubstr("\"type\": \"int\"")) << "has value dump";
 }
 
 } // namespace
diff --git a/clang/unittests/Analysis/FlowSensitive/MultiVarConstantPropagationTest.cpp b/clang/unittests/Analysis/FlowSensitive/MultiVarConstantPropagationTest.cpp
index bc51b5c59d19f..ed95887a45f1a 100644
--- a/clang/unittests/Analysis/FlowSensitive/MultiVarConstantPropagationTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/MultiVarConstantPropagationTest.cpp
@@ -57,7 +57,8 @@ struct ValueLattice {
   //  * `Defined` -> top.
   ValueState State;
 
-  // When `None`, the lattice is either at top or bottom, based on `State`.
+  // When `std::nullopt`, the lattice is either at top or bottom, based on
+  // `State`.
   std::optional<int64_t> Value;
 
   constexpr ValueLattice()
diff --git a/clang/unittests/Analysis/FlowSensitive/SingleVarConstantPropagationTest.cpp b/clang/unittests/Analysis/FlowSensitive/SingleVarConstantPropagationTest.cpp
index 4b6f575641c5c..b76ce4fa42642 100644
--- a/clang/unittests/Analysis/FlowSensitive/SingleVarConstantPropagationTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/SingleVarConstantPropagationTest.cpp
@@ -59,7 +59,7 @@ struct ConstantPropagationLattice {
       return Lhs.Var == Rhs.Var && Lhs.Value == Rhs.Value;
     }
   };
-  // `None` is "bottom".
+  // `std::nullopt` is "bottom".
   std::optional<VarValue> Data;
 
   static constexpr ConstantPropagationLattice bottom() {
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 1589067a81c7b..0283fe74e7790 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -53,9 +53,10 @@ void runDataflow(llvm::StringRef Code, Matcher Match,
       [UseBuiltinModel = Options.BuiltinOpts.has_value()](ASTContext &C,
                                                           Environment &Env) {
         return NoopAnalysis(
-            C, DataflowAnalysisOptions{UseBuiltinModel
-                                           ? Env.getAnalysisOptions()
-                                           : std::optional<BuiltinOptions>()});
+            C,
+            DataflowAnalysisOptions{
+                UseBuiltinModel ? Env.getDataflowAnalysisContext().getOptions()
+                                : std::optional<BuiltinOptions>()});
       });
   AI.ASTBuildArgs = ASTBuildArgs;
   if (Options.BuiltinOpts)
@@ -403,14 +404,9 @@ TEST(TransferTest, ReferenceVarDecl) {
 
         const StorageLocation *FooLoc =
             Env.getStorageLocation(*FooDecl, SkipPast::None);
-        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
-
-        const ReferenceValue *FooVal =
-            cast<ReferenceValue>(Env.getValue(*FooLoc));
-        const StorageLocation &FooReferentLoc = FooVal->getReferentLoc();
-        EXPECT_TRUE(isa<AggregateStorageLocation>(&FooReferentLoc));
+        ASSERT_TRUE(isa_and_nonnull<AggregateStorageLocation>(FooLoc));
 
-        const Value *FooReferentVal = Env.getValue(FooReferentLoc);
+        const Value *FooReferentVal = Env.getValue(*FooLoc);
         EXPECT_TRUE(isa_and_nonnull<StructValue>(FooReferentVal));
       });
 }
@@ -494,11 +490,9 @@ TEST(TransferTest, SelfReferentialReferenceVarDecl) {
     ASSERT_THAT(BazRefDecl, NotNull());
     ASSERT_THAT(BazPtrDecl, NotNull());
 
-    const auto *FooLoc = cast<ScalarStorageLocation>(
+    const auto *FooLoc = cast<AggregateStorageLocation>(
         Env.getStorageLocation(*FooDecl, SkipPast::None));
-    const auto *FooVal = cast<ReferenceValue>(Env.getValue(*FooLoc));
-    const auto *FooReferentVal =
-        cast<StructValue>(Env.getValue(FooVal->getReferentLoc()));
+    const auto *FooReferentVal = cast<StructValue>(Env.getValue(*FooLoc));
 
     const auto *BarVal =
         cast<ReferenceValue>(FooReferentVal->getChild(*BarDecl));
@@ -507,13 +501,17 @@ TEST(TransferTest, SelfReferentialReferenceVarDecl) {
 
     const auto *FooRefVal =
         cast<ReferenceValue>(BarReferentVal->getChild(*FooRefDecl));
-    const StorageLocation &FooReferentLoc = FooRefVal->getReferentLoc();
-    EXPECT_THAT(Env.getValue(FooReferentLoc), IsNull());
+    const auto &FooReferentLoc =
+        cast<AggregateStorageLocation>(FooRefVal->getReferentLoc());
+    EXPECT_THAT(Env.getValue(FooReferentLoc), NotNull());
+    EXPECT_THAT(Env.getValue(FooReferentLoc.getChild(*BarDecl)), IsNull());
 
     const auto *FooPtrVal =
         cast<PointerValue>(BarReferentVal->getChild(*FooPtrDecl));
-    const StorageLocation &FooPtrPointeeLoc = FooPtrVal->getPointeeLoc();
-    EXPECT_THAT(Env.getValue(FooPtrPointeeLoc), IsNull());
+    const auto &FooPtrPointeeLoc =
+        cast<AggregateStorageLocation>(FooPtrVal->getPointeeLoc());
+    EXPECT_THAT(Env.getValue(FooPtrPointeeLoc), NotNull());
+    EXPECT_THAT(Env.getValue(FooPtrPointeeLoc.getChild(*BarDecl)), IsNull());
 
     const auto *BazRefVal =
         cast<ReferenceValue>(BarReferentVal->getChild(*BazRefDecl));
@@ -1058,16 +1056,9 @@ TEST(TransferTest, ReferenceParamDecl) {
 
         const StorageLocation *FooLoc =
             Env.getStorageLocation(*FooDecl, SkipPast::None);
-        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
-
-        const ReferenceValue *FooVal =
-            dyn_cast<ReferenceValue>(Env.getValue(*FooLoc));
-        ASSERT_THAT(FooVal, NotNull());
-
-        const StorageLocation &FooReferentLoc = FooVal->getReferentLoc();
-        EXPECT_TRUE(isa<AggregateStorageLocation>(&FooReferentLoc));
+        ASSERT_TRUE(isa_and_nonnull<AggregateStorageLocation>(FooLoc));
 
-        const Value *FooReferentVal = Env.getValue(FooReferentLoc);
+        const Value *FooReferentVal = Env.getValue(*FooLoc);
         EXPECT_TRUE(isa_and_nonnull<StructValue>(FooReferentVal));
       });
 }
@@ -1905,15 +1896,13 @@ TEST(TransferTest, DefaultInitializerReference) {
         const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
         ASSERT_THAT(FooDecl, NotNull());
 
-        const auto *FooVal =
-            cast<ReferenceValue>(Env.getValue(*FooDecl, SkipPast::None));
+        const auto *FooLoc = Env.getStorageLocation(*FooDecl);
 
         const ValueDecl *QuxDecl = findValueDecl(ASTCtx, "Qux");
         ASSERT_THAT(QuxDecl, NotNull());
 
-        const auto *QuxVal =
-            cast<ReferenceValue>(Env.getValue(*QuxDecl, SkipPast::None));
-        EXPECT_EQ(&QuxVal->getReferentLoc(), &FooVal->getReferentLoc());
+        const auto *QuxLoc = Env.getStorageLocation(*QuxDecl);
+        EXPECT_EQ(QuxLoc, FooLoc);
       });
 }
 
@@ -2593,9 +2582,8 @@ TEST(TransferTest, DerefDependentPtr) {
 
         const auto *FooVal =
             cast<PointerValue>(Env.getValue(*FooDecl, SkipPast::None));
-        const auto *BarVal =
-            cast<ReferenceValue>(Env.getValue(*BarDecl, SkipPast::None));
-        EXPECT_EQ(&BarVal->getReferentLoc(), &FooVal->getPointeeLoc());
+        const auto *BarLoc = Env.getStorageLocation(*BarDecl);
+        EXPECT_EQ(BarLoc, &FooVal->getPointeeLoc());
       });
 }
 
@@ -2643,17 +2631,20 @@ TEST(TransferTest, VarDeclInDoWhile) {
     void target(int *Foo) {
       do {
         int Bar = *Foo;
-      } while (true);
+        // [[in_loop]]
+      } while (false);
       (void)0;
-      /*[[p]]*/
+      // [[after_loop]]
     }
   )";
   runDataflow(
       Code,
       [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
          ASTContext &ASTCtx) {
-        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
-        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+        const Environment &EnvInLoop =
+            getEnvironmentAtAnnotation(Results, "in_loop");
+        const Environment &EnvAfterLoop =
+            getEnvironmentAtAnnotation(Results, "after_loop");
 
         const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
         ASSERT_THAT(FooDecl, NotNull());
@@ -2662,15 +2653,35 @@ TEST(TransferTest, VarDeclInDoWhile) {
         ASSERT_THAT(BarDecl, NotNull());
 
         const auto *FooVal =
-            cast<PointerValue>(Env.getValue(*FooDecl, SkipPast::None));
+            cast<PointerValue>(EnvAfterLoop.getValue(*FooDecl));
         const auto *FooPointeeVal =
-            cast<IntegerValue>(Env.getValue(FooVal->getPointeeLoc()));
-
-        const auto *BarVal = dyn_cast_or_null<IntegerValue>(
-            Env.getValue(*BarDecl, SkipPast::None));
-        ASSERT_THAT(BarVal, NotNull());
+            cast<IntegerValue>(EnvAfterLoop.getValue(FooVal->getPointeeLoc()));
 
+        const auto *BarVal = cast<IntegerValue>(EnvInLoop.getValue(*BarDecl));
         EXPECT_EQ(BarVal, FooPointeeVal);
+
+        // FIXME: This assertion documents current behavior, but we would prefer
+        // declarations to be removed from the environment when their lifetime
+        // ends. Once this is the case, change this assertion accordingly.
+        ASSERT_THAT(EnvAfterLoop.getValue(*BarDecl), BarVal);
+      });
+}
+
+TEST(TransferTest, UnreachableAfterWhileTrue) {
+  std::string Code = R"(
+    void target() {
+      while (true) {}
+      (void)0;
+      /*[[p]]*/
+    }
+  )";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        // The node after the while-true is pruned because it is trivially
+        // known to be unreachable.
+        ASSERT_TRUE(Results.empty());
       });
 }
 
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index b2d79f4ca11e6..0e47abed12472 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -247,6 +247,8 @@ TEST(ConfigParseTest, ParsesConfiguration) {
               SpacesBeforeTrailingComments, 1234u);
   CHECK_PARSE("IndentWidth: 32", IndentWidth, 32u);
   CHECK_PARSE("ContinuationIndentWidth: 11", ContinuationIndentWidth, 11u);
+  CHECK_PARSE("BracedInitializerIndentWidth: 34", BracedInitializerIndentWidth,
+              34);
   CHECK_PARSE("CommentPragmas: '// abc$'", CommentPragmas, "// abc$");
 
   Style.QualifierAlignment = FormatStyle::QAS_Right;
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index c5c1f2150d3b1..099a5cb913643 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -2998,6 +2998,17 @@ TEST_F(FormatTest, FormatsLabels) {
                "test_label:;\n"
                "  int i = 0;\n"
                "}");
+  verifyFormat("{\n"
+               "  some_code();\n"
+               "test_label: { some_other_code(); }\n"
+               "}");
+  verifyFormat("{\n"
+               "  some_code();\n"
+               "test_label: {\n"
+               "  some_other_code();\n"
+               "  some_other_code();\n"
+               "}\n"
+               "}");
   FormatStyle Style = getLLVMStyle();
   Style.IndentGotoLabels = false;
   verifyFormat("void f() {\n"
@@ -3022,6 +3033,23 @@ TEST_F(FormatTest, FormatsLabels) {
                "test_label:;\n"
                "  int i = 0;\n"
                "}");
+  verifyFormat("{\n"
+               "  some_code();\n"
+               "test_label: { some_other_code(); }\n"
+               "}",
+               Style);
+  // The opening brace may either be on the same unwrapped line as the colon or
+  // on a separate one. The formatter should recognize both.
+  Style = getLLVMStyle();
+  Style.BreakBeforeBraces = FormatStyle::BraceBreakingStyle::BS_Allman;
+  verifyFormat("{\n"
+               "  some_code();\n"
+               "test_label:\n"
+               "{\n"
+               "  some_other_code();\n"
+               "}\n"
+               "}",
+               Style);
 }
 
 TEST_F(FormatTest, MultiLineControlStatements) {
@@ -4855,6 +4883,191 @@ TEST_F(FormatTest, DesignatedInitializers) {
                "    [5] = eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee};");
 }
 
+TEST_F(FormatTest, BracedInitializerIndentWidth) {
+  auto Style = getLLVMStyleWithColumns(60);
+  Style.BinPackArguments = true;
+  Style.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak;
+  Style.BracedInitializerIndentWidth = 6;
+
+  // Non-initializing braces are unaffected by BracedInitializerIndentWidth.
+  verifyFormat("enum class {\n"
+               "  One,\n"
+               "  Two,\n"
+               "};\n",
+               Style);
+  verifyFormat("class Foo {\n"
+               "  Foo() {}\n"
+               "  void bar();\n"
+               "};\n",
+               Style);
+  verifyFormat("void foo() {\n"
+               "  auto bar = baz;\n"
+               "  return baz;\n"
+               "};\n",
+               Style);
+  verifyFormat("auto foo = [&] {\n"
+               "  auto bar = baz;\n"
+               "  return baz;\n"
+               "};\n",
+               Style);
+  verifyFormat("{\n"
+               "  auto bar = baz;\n"
+               "  return baz;\n"
+               "};\n",
+               Style);
+  // Non-brace initialization is unaffected by BracedInitializerIndentWidth.
+  verifyFormat("SomeClass clazz(\n"
+               "    \"xxxxxxxxxxxxxxxxxx\", \"yyyyyyyyyyyyyyyyyy\",\n"
+               "    \"zzzzzzzzzzzzzzzzzz\");\n",
+               Style);
+
+  // The following types of initialization are all affected by
+  // BracedInitializerIndentWidth. Aggregate initialization.
+  verifyFormat("int LooooooooooooooooooooooooongVariable[2] = {\n"
+               "      10000000, 20000000};",
+               Style);
+  verifyFormat("SomeStruct s{\n"
+               "      \"xxxxxxxxxxxxxxxx\", \"yyyyyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzzzzz\"};\n",
+               Style);
+  // Designated initializers.
+  verifyFormat("int LooooooooooooooooooooooooongVariable[1] = {\n"
+               "      [0] = 10000000, [1] = 20000000};",
+               Style);
+  verifyFormat("SomeStruct s{\n"
+               "      .foo = \"xxxxxxxxxxxxx\",\n"
+               "      .bar = \"yyyyyyyyyyyyy\",\n"
+               "      .baz = \"zzzzzzzzzzzzz\"};\n",
+               Style);
+  // List initialization.
+  verifyFormat("SomeStruct s{\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  verifyFormat("SomeStruct{\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  verifyFormat("new SomeStruct{\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  // Member initializer.
+  verifyFormat("class SomeClass {\n"
+               "  SomeStruct s{\n"
+               "        \"xxxxxxxxxxxxx\",\n"
+               "        \"yyyyyyyyyyyyy\",\n"
+               "        \"zzzzzzzzzzzzz\",\n"
+               "  };\n"
+               "};\n",
+               Style);
+  // Constructor member initializer.
+  verifyFormat("SomeClass::SomeClass : strct{\n"
+               "                             \"xxxxxxxxxxxxx\",\n"
+               "                             \"yyyyyyyyyyyyy\",\n"
+               "                             \"zzzzzzzzzzzzz\",\n"
+               "                       } {}\n",
+               Style);
+  // Copy initialization.
+  verifyFormat("SomeStruct s = SomeStruct{\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  // Copy list initialization.
+  verifyFormat("SomeStruct s = {\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  // Assignment operand initialization.
+  verifyFormat("s = {\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  // Returned object initialization.
+  verifyFormat("return {\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  // Initializer list.
+  verifyFormat("auto initializerList = {\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "};\n",
+               Style);
+  // Function parameter initialization.
+  verifyFormat("func({\n"
+               "      \"xxxxxxxxxxxxx\",\n"
+               "      \"yyyyyyyyyyyyy\",\n"
+               "      \"zzzzzzzzzzzzz\",\n"
+               "});\n",
+               Style);
+  // Nested init lists.
+  verifyFormat("SomeStruct s = {\n"
+               "      {{init1, init2, init3, init4, init5},\n"
+               "       {init1, init2, init3, init4, init5}}};\n",
+               Style);
+  verifyFormat("SomeStruct s = {\n"
+               "      {{\n"
+               "             .init1 = 1,\n"
+               "             .init2 = 2,\n"
+               "             .init3 = 3,\n"
+               "             .init4 = 4,\n"
+               "             .init5 = 5,\n"
+               "       },\n"
+               "       {init1, init2, init3, init4, init5}}};\n",
+               Style);
+  verifyFormat("SomeArrayT a[3] = {\n"
+               "      {\n"
+               "            foo,\n"
+               "            bar,\n"
+               "      },\n"
+               "      {\n"
+               "            foo,\n"
+               "            bar,\n"
+               "      },\n"
+               "      SomeArrayT{},\n"
+               "}\n",
+               Style);
+  verifyFormat("SomeArrayT a[3] = {\n"
+               "      {foo},\n"
+               "      {\n"
+               "            {\n"
+               "                  init1,\n"
+               "                  init2,\n"
+               "                  init3,\n"
+               "            },\n"
+               "            {\n"
+               "                  init1,\n"
+               "                  init2,\n"
+               "                  init3,\n"
+               "            },\n"
+               "      },\n"
+               "      {baz},\n"
+               "}\n",
+               Style);
+
+  // Aligning after open braces unaffected by BracedInitializerIndentWidth.
+  Style.AlignAfterOpenBracket = FormatStyle::BAS_Align;
+  verifyFormat("SomeStruct s{\"xxxxxxxxxxxxx\", \"yyyyyyyyyyyyy\",\n"
+               "             \"zzzzzzzzzzzzz\"};\n",
+               Style);
+}
+
 TEST_F(FormatTest, NestedStaticInitializers) {
   verifyFormat("static A x = {{{}}};\n");
   verifyFormat("static A x = {{{init1, init2, init3, init4},\n"
@@ -16777,6 +16990,19 @@ TEST_F(FormatTest, ConfigurableSpaceBeforeColon) {
                "}\n"
                "}",
                CaseStyle);
+  // Goto labels should not be affected.
+  verifyFormat("switch (x) {\n"
+               "goto_label:\n"
+               "default :\n"
+               "}",
+               CaseStyle);
+  verifyFormat("switch (x) {\n"
+               "goto_label: { break; }\n"
+               "default : {\n"
+               "  break;\n"
+               "}\n"
+               "}",
+               CaseStyle);
 
   FormatStyle NoSpaceStyle = getLLVMStyle();
   EXPECT_EQ(NoSpaceStyle.SpaceBeforeCaseColon, false);
diff --git a/clang/unittests/Format/FormatTestVerilog.cpp b/clang/unittests/Format/FormatTestVerilog.cpp
index ff6a1cdc57fe7..a7ba3c57c55bb 100644
--- a/clang/unittests/Format/FormatTestVerilog.cpp
+++ b/clang/unittests/Format/FormatTestVerilog.cpp
@@ -359,6 +359,12 @@ TEST_F(FormatTestVerilog, Headers) {
                "     input var shortreal in2,\n"
                "     output tagged_st out);\n"
                "endmodule");
+  // There should be a space following the type but not the variable name.
+  verifyFormat("module test\n"
+               "    (input wire [7 : 0] a,\n"
+               "     input wire b[7 : 0],\n"
+               "     input wire [7 : 0] c[7 : 0]);\n"
+               "endmodule");
   // Ports should be grouped by types.
   verifyFormat("module test\n"
                "    (input [7 : 0] a,\n"
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 76800b3fc8e37..9760c80dcc619 100755
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1343,6 +1343,29 @@ TEST_F(QualifierFixerTest, TemplatesLeft) {
                "TemplateType<Container const> t;", Style);
 }
 
+TEST_F(QualifierFixerTest, Ranges) {
+  FormatStyle Style = getLLVMStyle();
+  Style.QualifierAlignment = FormatStyle::QAS_Custom;
+  Style.QualifierOrder = {"const", "volatile", "type"};
+
+  // Only the first line should be formatted; the second should remain as is.
+  verifyFormat("template <typename T> const Foo f();\n"
+               "template <typename T> Foo const f();",
+               "template <typename T> Foo const f();\n"
+               "template <typename T> Foo const f();",
+               Style, {tooling::Range(0, 36)});
+
+  // Only the middle line should be formatted; the first and last should remain
+  // as is.
+  verifyFormat("template <typename T> Foo const f();\n"
+               "template <typename T> const Foo f();\n"
+               "template <typename T> Foo const f();",
+               "template <typename T> Foo const f();\n"
+               "template <typename T> Foo const f();\n"
+               "template <typename T> Foo const f();",
+               Style, {tooling::Range(37, 36)});
+}
+
 } // namespace
 } // namespace test
 } // namespace format
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index cd39661f7f8de..908a81d09f0e9 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -1615,13 +1615,20 @@ TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
   Tokens = Annotate("extern function [1 : 0] x;");
   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
   EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
+  Tokens = Annotate("module test\n"
+                    "    (input wire [7 : 0] a[7 : 0]);\n"
+                    "endmodule");
+  ASSERT_EQ(Tokens.size(), 20u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);
+  EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);
+  EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);
   // Test case labels and ternary operators.
   Tokens = Annotate("case (x)\n"
                     "  x:\n"
                     "    x;\n"
                     "endcase\n");
   ASSERT_EQ(Tokens.size(), 10u) << Tokens;
-  EXPECT_TOKEN(Tokens[5], tok::colon, TT_GotoLabelColon);
+  EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);
   Tokens = Annotate("case (x)\n"
                     "  x ? x : x:\n"
                     "    x;\n"
@@ -1629,7 +1636,7 @@ TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
   ASSERT_EQ(Tokens.size(), 14u) << Tokens;
   EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
   EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
-  EXPECT_TOKEN(Tokens[9], tok::colon, TT_GotoLabelColon);
+  EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);
   // Non-blocking assignments.
   Tokens = Annotate("a <= b;");
   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
@@ -1752,6 +1759,21 @@ TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {
   EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
 }
 
+TEST_F(TokenAnnotatorTest, UnderstandsLabels) {
+  auto Tokens = annotate("{ x: break; }");
+  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
+  Tokens = annotate("{ case x: break; }");
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
+  Tokens = annotate("{ x: { break; } }");
+  ASSERT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
+  Tokens = annotate("{ case x: { break; } }");
+  ASSERT_EQ(Tokens.size(), 10u) << Tokens;
+  EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
diff --git a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
index 5222df9fb9eb2..2f8804784a2e4 100644
--- a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
+++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
@@ -90,7 +90,8 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) {
                                            "#pragma pop_macro(A)\n"
                                            "#pragma include_alias(<A>, <B>)\n"
                                            "export module m;\n"
-                                           "import m;\n",
+                                           "import m;\n"
+                                           "#pragma clang system_header\n",
                                            Out, Tokens, Directives));
   EXPECT_EQ(pp_define, Directives[0].Kind);
   EXPECT_EQ(pp_undef, Directives[1].Kind);
@@ -113,7 +114,8 @@ TEST(MinimizeSourceToDependencyDirectivesTest, AllTokens) {
   EXPECT_EQ(pp_pragma_include_alias, Directives[18].Kind);
   EXPECT_EQ(cxx_export_module_decl, Directives[19].Kind);
   EXPECT_EQ(cxx_import_decl, Directives[20].Kind);
-  EXPECT_EQ(pp_eof, Directives[21].Kind);
+  EXPECT_EQ(pp_pragma_system_header, Directives[21].Kind);
+  EXPECT_EQ(pp_eof, Directives[22].Kind);
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest, EmptyHash) {
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 8f46b08b1366f..6ef5790731a6b 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -389,7 +389,7 @@ class Intrinsic {
       Mods = getNextModifiers(Proto, Pos);
     }
 
-    for (auto Type : Types) {
+    for (const auto &Type : Types) {
       // If this builtin takes an immediate argument, we need to #define it rather
       // than use a standard declaration, so that SemaChecking can range check
       // the immediate passed by the user.
@@ -2353,7 +2353,6 @@ void NeonEmitter::run(raw_ostream &OS) {
   OS << "#include <stdint.h>\n\n";
 
   OS << "#include <arm_bf16.h>\n";
-  OS << "typedef __bf16 bfloat16_t;\n";
 
   // Emit NEON-specific scalar typedefs.
   OS << "typedef float float32_t;\n";
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index bc50cbad4b541..d5d3f5fe558a8 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -1103,7 +1103,6 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
   OS << "typedef __SVBFloat16_t svbfloat16_t;\n";
 
   OS << "#include <arm_bf16.h>\n";
-  OS << "typedef __bf16 bfloat16_t;\n";
 
   OS << "typedef __SVFloat32_t svfloat32_t;\n";
   OS << "typedef __SVFloat64_t svfloat64_t;\n";
diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp
index a67e1d1af5d75..2fc6639d832f0 100644
--- a/clang/utils/TableGen/TableGen.cpp
+++ b/clang/utils/TableGen/TableGen.cpp
@@ -91,6 +91,9 @@ enum ActionType {
   GenRISCVVectorBuiltins,
   GenRISCVVectorBuiltinCG,
   GenRISCVVectorBuiltinSema,
+  GenRISCVSiFiveVectorBuiltins,
+  GenRISCVSiFiveVectorBuiltinCG,
+  GenRISCVSiFiveVectorBuiltinSema,
   GenAttrDocs,
   GenDiagDocs,
   GenOptDocs,
@@ -251,6 +254,12 @@ cl::opt<ActionType> Action(
                    "Generate riscv_vector_builtin_cg.inc for clang"),
         clEnumValN(GenRISCVVectorBuiltinSema, "gen-riscv-vector-builtin-sema",
                    "Generate riscv_vector_builtin_sema.inc for clang"),
+        clEnumValN(GenRISCVSiFiveVectorBuiltins, "gen-riscv-sifive-vector-builtins",
+                   "Generate riscv_sifive_vector_builtins.inc for clang"),
+        clEnumValN(GenRISCVSiFiveVectorBuiltinCG, "gen-riscv-sifive-vector-builtin-codegen",
+                   "Generate riscv_sifive_vector_builtin_cg.inc for clang"),
+        clEnumValN(GenRISCVSiFiveVectorBuiltinSema, "gen-riscv-sifive-vector-builtin-sema",
+                   "Generate riscv_sifive_vector_builtin_sema.inc for clang"),
         clEnumValN(GenAttrDocs, "gen-attr-docs",
                    "Generate attribute documentation"),
         clEnumValN(GenDiagDocs, "gen-diag-docs",
@@ -472,6 +481,15 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenRISCVVectorBuiltinSema:
     EmitRVVBuiltinSema(Records, OS);
     break;
+  case GenRISCVSiFiveVectorBuiltins:
+    EmitRVVBuiltins(Records, OS);
+    break;
+  case GenRISCVSiFiveVectorBuiltinCG:
+    EmitRVVBuiltinCG(Records, OS);
+    break;
+  case GenRISCVSiFiveVectorBuiltinSema:
+    EmitRVVBuiltinSema(Records, OS);
+    break;
   case GenAttrDocs:
     EmitClangAttrDocs(Records, OS);
     break;
diff --git a/clang/www/OpenProjects.html b/clang/www/OpenProjects.html
index fb3bd1b5edec7..d3aedd281121c 100755
--- a/clang/www/OpenProjects.html
+++ b/clang/www/OpenProjects.html
@@ -22,29 +22,91 @@ <h1>Open Clang Projects</h1>
 or to verify that one of these isn't already completed.</p>
 
 <ul>
-<li><b>Implement an tool to generate code documentation</b>: Clang's
-library-based design allows it to be used by a variety of tools that reason
-about source code. One great application of Clang would be to build an
-auto-documentation system like doxygen that generates code documentation from
-source code. The advantage of using Clang for such a tool is that the tool would
-use the same preprocessor/parser/ASTs as the compiler itself, giving it a very
-rich understanding of the code. Clang is already able to read and understand
-doxygen markup, but cannot yet generate documentation from it.</li>
-
-<li><b>Use clang libraries to implement better versions of existing tools</b>:
-Clang is built as a set of libraries, which means that it is possible to
-implement capabilities similar to other source language tools, improving them
-in various ways.  Three examples are <a
-href="https://github.com/distcc">distcc</a>, the <a
-href="http://delta.tigris.org/">delta testcase reduction tool</a>, and the
-"indent" source reformatting tool.
-distcc can be improved to scale better and be more efficient.  Delta could be
-faster and more efficient at reducing C-family programs if built on the clang
-preprocessor. The clang-based indent replacement,
-<a href="https://clang.llvm.org/docs/ClangFormat.html">clang-format</a>,
-could be taught to handle simple structural rules like those in <a
-href="https://llvm.org/docs/CodingStandards.html#use-early-exits-and-continue-to-simplify-code">the LLVM coding
-standards</a>.</li>
+<li><b>Refresh and improve Clang's documentation</b>: Clang is inconsistent
+with documenting implementation-defined behaviors. We have significant
+documentation in the <a href="https://clang.llvm.org/docs/LanguageExtensions.html">
+Language Extensions</a> page, but the information is incomplete and the page is
+difficult to navigate. We would appreciate help with:
+<ul>
+  <li>improving the way this information is presented to users,</li>
+  <li><a href="https://llvm.org/docs/TableGen/">table generating</a>
+      documentation where possible, such as for implementation limits or other
+      target-specific information,</li>
+  <li>adding documentation for currently
+     <a href="https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/AttrDocs.td">
+     undocumented attributes</a>,</li>
+  <li>documenting <a href="https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/DiagnosticDocs.td">
+      diagnostic group flags</a> (adding code examples of what is diagnosed, or
+      other relevant information), or</li>
+  <li>documenting <a href="https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Driver/Options.td">
+      command line options</a>, or</li>
+  <li>help with completing other missing documentation.</li>
+</ul>
+These projects are independent of each other.</li>
+
+<li><b>Complete the investigation into Clang's C conformance</b>: Clang's
+<a href="c_status.html">C status page</a> contain a number of entries marked as
+<code>Unknown</code>. Completing the investigation involves adding
+<a href="https://github.com/llvm/llvm-project/tree/main/clang/test/C">test
+coverage</a> for the various standards papers and updating the documentation
+accordingly.
+</li>
+
+<li><b>Improve Clang's C and C++ standard conformance test coverage</b>:
+Clang's test suite is structured such that most tests are written to provide
+coverage for what part of the compiler the feature's implementation exists in;
+we have parsing tests in <code>clang/test/Parser</code>, and semantic analysis
+tests in <code>clang/test/Sema*</code>, etc. We also have tests written to
+provide coverage for the standard requirements (<code>clang/test/CXX</code> and
+<code>clang/test/C</code>). The standards coverage is not structured in a way
+that makes it easy to maintain as the standards change over time. No commercial
+conformance test suite has a license model suitable for open source projects,
+so we would appreciate help in improving the existing coverage we have both in
+terms of layout of the tests as well as in coverage of the various standard
+modes.</li>
+
+<li><b>Complete the investigation into Clang's C and C++ Defect Report
+conformance</b>: Separate from (but related to) general conformance testing is
+determining which <a href="c_dr_status.html">C defect reports</a> and
+<a href="cxx_dr_status.html">C++ defect reports</a> Clang implements. These
+lists currently have a number of entries marked as <code>Unknown</code>.
+Completing the investigation involves adding test coverage for
+<a href="https://github.com/llvm/llvm-project/tree/main/clang/test/C/drs">C</a>
+and
+<a href="https://github.com/llvm/llvm-project/tree/main/clang/test/CXX/drs">C++</a>
+defect reports and updating the documentation accordingly.</li>
+
+<li><b>Bug triage</b>: Clang's <a href="https://github.com/llvm/llvm-project/issues">
+issue tracker</a>currently has over 20,000 open issues, many of which are not
+appropriately tagged, are no longer reproducible, could use a reduced test case,
+or otherwise needs some manual interaction. We can always use help with
+<a href="https://llvm.org/docs/BugLifeCycle.html#triaging-bugs">bug triage and
+issue tracker maintenance</a>.
+</li>
+
+<li><b>Improve build times with Clang</b>: the time it takes Clang to process a
+translation unit is very important to our users; the lower the build time, the
+better the overall user experience. It would be good to improve Clang's
+performance as well as to find ways to proactively alert us when we've
+introduced a change that has significant negative impact on build times.</li>
+
+<li><b>Complete support for the experimental constant expression interpreter
+</b>: Clang's production constant expression interpreter computes a constant
+expression result by walking over AST nodes, performing calculations as it
+goes. This does not have good performance properties, and so we've begun work
+on an <a href="https://clang.llvm.org/docs/ConstantInterpreter.html">
+experimental constant expression interpreter</a> that works by converting the
+AST into bytecode that is interpreted. This effort has a long tail of work left
+to complete because it requires implementing byte code for every kind of
+expression and type that can be used in a constant expression for C++ and C.
+</li>
+
+<li><b>Improve clang-doc</b>: Clang's library-based design allows it to be used
+by a variety of tools that reason about source code.
+<a href="https://clang.llvm.org/extra/clang-doc.html">clang-doc</a> is one
+great application of this functionality, which generates code documentation
+from source code. The tool is in early stages of development and could use more
+dedicated effort to complete the implementation.</li>
 
 <li><b>Self-testing using clang</b>: There are several neat ways to
 improve the quality of clang by self-testing. Some examples:
@@ -61,7 +123,7 @@ <h1>Open Clang Projects</h1>
 </ul>
 </li>
 
-<li><b>Continue work on C++20, C++2b, and C2x support</b>:
+<li><b>Continue work on C++20, C++23, and C2x support</b>:
   There are still several C++20 features to complete, and work has begun on
   supporting the latest language standards. Please see the
   <a href="cxx_status.html">C++ status report page</a> to find out what is
@@ -80,7 +142,7 @@ <h1>Open Clang Projects</h1>
 
 <p>A tool that would investigate hosts and targets, and store the configuration
 in files that can later be used by Clang itself to avoid command-line options,
-especially the ones regarding which target options to use, would greatle alleviate
+especially the ones regarding which target options to use, would greatly alleviate
 this problem. A simple tool, with little or no dependency on LLVM itself, that
 will investigate a target architecture by probing hardware, software, libraries
 and compiling and executing code to identify all properties that would be relevant
@@ -92,7 +154,7 @@ <h1>Open Clang Projects</h1>
 <p>The second stage would be to produce a configuration file (that can be used
 independently of the Host) so that Clang can read it and not need a gazillion
 of command-line options. Such file should be simple JSON / INI or anything that
-a text editor could change.</p>
+a text editor could change.</p></li>
 </ul>
 
 <p>If you hit a bug with Clang, it is very useful for us if you reduce the code
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index 9ddd01b9fd445..9f476c8b013e2 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -929,7 +929,7 @@ <h2 id="c2x">C2x implementation status</h2>
     <tr>
       <td>char16_t & char32_t string literals shall be UTF-16 & UTF-32</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n2728.htm">N2728</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
     </tr>
     <tr>
       <td>IEC 60559 binding</td>
@@ -1215,13 +1215,7 @@ <h2 id="c2x">C2x implementation status</h2>
     <tr>
       <td>Introduce the nullptr constant</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3042.htm">N3042</a></td>
-      <td class="partial" align="center">
-        <details><summary>Partial</summary>
-          Parts of the implementation may be incorrect until WG14 has completed NB comment
-          resolution for incompatibilities with C++ that were discovered. The major use cases
-          and usage patterns should work well, though.
-        </details>
-      </td>
+      <td class="unreleased" align="center">Clang 17</td>
     </tr>
     <tr>
       <td>Memory layout of unions</td>
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 29532e4e42299..6206ea6664873 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -10733,7 +10733,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/1821.html">1821</a></td>
     <td>CD6</td>
     <td>Qualified redeclarations in a class <I>member-specification</I></td>
-    <td class="none" align="center">Unknown</td>
+    <td class="full" align="center">Yes</td>
   </tr>
   <tr id="1822">
     <td><a href="https://cplusplus.github.io/CWG/issues/1822.html">1822</a></td>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index fe2a16d9cee1f..e4ecb5f832e93 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -61,8 +61,8 @@ <h1>C++ Support in Clang</h1>
  <td class="partial" align="center">Partial</td>
 </tr>
 <tr>
- <td><a href="#cxx23">C++2b (tentatively C++23)</a></td>
- <td><tt>-std=c++2b</tt></td>
+ <td><a href="#cxx23">C++23</a></td>
+ <td><tt>-std=c++23</tt></td>
  <td class="partial" align="center">Partial</td>
 </tr>
 </table>
@@ -1290,12 +1290,12 @@ <h2 id="cxx20">C++20 implementation status</h2>
 </p>
 </details>
 
-<h2 id="cxx23">C++2b implementation status</h2>
+<h2 id="cxx23">C++23 implementation status</h2>
 
-<p>Clang has support for some of the features of the C++ standard following
-C++20, informally referred to as C++2b.</p>
+<!-- TODO add link to the ISO Standard after it's published. -->
+<p>Clang has support for some of the features of the ISO C++ 2023 standard.</p>
 
-<p>You can use Clang in C++2b mode with the <code>-std=c++2b</code> option.</p>
+<p>You can use Clang in C++23 mode with the <code>-std=c++23</code> option.</p>
 
 <details open>
 <summary>List of features and minimum Clang version with support</summary>
@@ -1303,7 +1303,7 @@ <h2 id="cxx23">C++2b implementation status</h2>
 <table width="689" border="1" cellspacing="0">
  <tr>
     <th>Language Feature</th>
-    <th>C++2b Proposal</th>
+    <th>C++23 Proposal</th>
     <th>Available in Clang?</th>
  </tr>
     <!-- Fall 2020 papers -->
@@ -1480,7 +1480,14 @@ <h2 id="cxx23">C++2b implementation status</h2>
     <tr>
       <td>Relaxing some constexpr restrictions</td>
       <td><a href="https://wg21.link/P2448R2">P2448R2</a></td>
-      <td class="none" align="center">No</td>
+      <td class="partial" align="center">
+        <details><summary>Clang 17 (Partial)</summary>
+	  We do not support outside of defaulted special memeber functions the change that constexpr functions no
+          longer have to be constexpr compatible but rather support a less restricted requirements for constexpr
+          functions. Which include allowing non-literal types as return values and paremeters, allow calling of
+          non-constexpr functions and constructors.
+        </details></td>
+      </td>
     </tr>
     <tr>
       <td>Using unknown pointers and references in constant expressions</td>
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index 045221312a820..752ce2ff1c8f5 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -508,7 +508,7 @@ function(add_compiler_rt_test test_suite test_name arch)
   set(output_dir "${output_dir}/${CMAKE_CFG_INTDIR}")
   file(MAKE_DIRECTORY "${output_dir}")
   set(output_bin "${output_dir}/${test_name}")
-  if(MSVC)
+  if(WIN32)
     set(output_bin "${output_bin}.exe")
   endif()
 
diff --git a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
index f15917f7872b5..25e7823716fc2 100644
--- a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
@@ -359,7 +359,7 @@ macro(construct_compiler_rt_default_triple)
       message(FATAL_ERROR "COMPILER_RT_DEFAULT_TARGET_TRIPLE isn't supported when building for default target only")
     endif()
     if ("${CMAKE_C_COMPILER_TARGET}" STREQUAL "")
-      message(FATAL_ERROR "CMAKE_C_COMPILER_TARGET must also be set when COMPILER_RT_DEFAUT_TARGET_ONLY is ON")
+      message(FATAL_ERROR "CMAKE_C_COMPILER_TARGET must also be set when COMPILER_RT_DEFAULT_TARGET_ONLY is ON")
     endif()
     message(STATUS "cmake c compiler target: ${CMAKE_C_COMPILER_TARGET}")
     set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${CMAKE_C_COMPILER_TARGET})
diff --git a/compiler-rt/include/sanitizer/common_interface_defs.h b/compiler-rt/include/sanitizer/common_interface_defs.h
index 2f415bd9e8540..983df7cea16ed 100644
--- a/compiler-rt/include/sanitizer/common_interface_defs.h
+++ b/compiler-rt/include/sanitizer/common_interface_defs.h
@@ -129,26 +129,23 @@ int __sanitizer_acquire_crash_state();
 /// state <c>mid == end</c>, so that should be the final state when the
 /// container is destroyed or when the container reallocates the storage.
 ///
-/// For ASan, <c><i>beg</i></c> should be 8-aligned and <c><i>end</i></c>
-/// should be either 8-aligned or it should point to the end of a separate
-/// heap-, stack-, or global-allocated buffer. So the following example will
-/// not work:
+/// For ASan, <c><i>beg</i></c> no longer needs to be 8-aligned,
+/// first and last granule may be shared with other objects
+/// and therefore the function can be used for any allocator.
 ///
-/// \code
-///   int64_t x[2]; // 16 bytes, 8-aligned
-///   char *beg = (char *)&x[0];
-///   char *end = beg + 12; // Not 8-aligned, not the end of the buffer
-/// \endcode
+/// The following example shows how to use the function:
 ///
-/// The following, however, will work:
 /// \code
-///   int32_t x[3]; // 12 bytes, but 8-aligned under ASan.
+///   int32_t x[3]; // 12 bytes
 ///   char *beg = (char*)&x[0];
-///   char *end = beg + 12; // Not 8-aligned, but is the end of the buffer
+///   char *end = beg + 12;
+///   __sanitizer_annotate_contiguous_container(beg, end, beg, end);
 /// \endcode
 ///
 /// \note  Use this function with caution and do not use for anything other
 /// than vector-like classes.
+/// \note  Unaligned <c><i>beg</i></c> or <c><i>end</i></c> may miss bugs in
+/// these granules.
 ///
 /// \param beg Beginning of memory region.
 /// \param end End of memory region.
diff --git a/compiler-rt/include/sanitizer/hwasan_interface.h b/compiler-rt/include/sanitizer/hwasan_interface.h
index 14035c05c6353..ee742c7f30317 100644
--- a/compiler-rt/include/sanitizer/hwasan_interface.h
+++ b/compiler-rt/include/sanitizer/hwasan_interface.h
@@ -1,4 +1,4 @@
-//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===//
+//===-- sanitizer/hwasan_interface.h ----------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index 013bdb64038d8..9c5a7e4c32884 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -656,6 +656,7 @@ void InitializeAsanInterceptors() {
   static bool was_called_once;
   CHECK(!was_called_once);
   was_called_once = true;
+  InitializePlatformInterceptors();
   InitializeCommonInterceptors();
   InitializeSignalInterceptors();
 
@@ -706,11 +707,11 @@ void InitializeAsanInterceptors() {
 #endif
   // Indirectly intercept std::rethrow_exception.
 #if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION
-  INTERCEPT_FUNCTION(_Unwind_RaiseException);
+  ASAN_INTERCEPT_FUNC(_Unwind_RaiseException);
 #endif
   // Indirectly intercept std::rethrow_exception.
 #if ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION
-  INTERCEPT_FUNCTION(_Unwind_SjLj_RaiseException);
+  ASAN_INTERCEPT_FUNC(_Unwind_SjLj_RaiseException);
 #endif
 
   // Intercept threading-related functions
@@ -741,8 +742,6 @@ void InitializeAsanInterceptors() {
   ASAN_INTERCEPT_FUNC(vfork);
 #endif
 
-  InitializePlatformInterceptors();
-
   VReport(1, "AddressSanitizer: libc interceptors initialized\n");
 }
 
diff --git a/compiler-rt/lib/asan/asan_interceptors.h b/compiler-rt/lib/asan/asan_interceptors.h
index c4bf087ea17f0..e7bec02f048d9 100644
--- a/compiler-rt/lib/asan/asan_interceptors.h
+++ b/compiler-rt/lib/asan/asan_interceptors.h
@@ -78,8 +78,8 @@ void InitializePlatformInterceptors();
 # define ASAN_INTERCEPT___LONGJMP_CHK 0
 #endif
 
-#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \
-    !SANITIZER_NETBSD
+#if ASAN_HAS_EXCEPTIONS && !SANITIZER_SOLARIS && !SANITIZER_NETBSD && \
+    (!SANITIZER_WINDOWS || (defined(__MINGW32__) && defined(__i386__)))
 # define ASAN_INTERCEPT___CXA_THROW 1
 # define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1
 # if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__))
diff --git a/compiler-rt/lib/asan/asan_malloc_win.cpp b/compiler-rt/lib/asan/asan_malloc_win.cpp
index 4b76d4ebd3eb4..ff78d7646a90d 100644
--- a/compiler-rt/lib/asan/asan_malloc_win.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_win.cpp
@@ -508,10 +508,10 @@ void ReplaceSystemMalloc() {
   TryToOverrideFunction("_expand_base", (uptr)_expand);
 
   if (flags()->windows_hook_rtl_allocators) {
-    INTERCEPT_FUNCTION(HeapSize);
-    INTERCEPT_FUNCTION(HeapFree);
-    INTERCEPT_FUNCTION(HeapReAlloc);
-    INTERCEPT_FUNCTION(HeapAlloc);
+    ASAN_INTERCEPT_FUNC(HeapSize);
+    ASAN_INTERCEPT_FUNC(HeapFree);
+    ASAN_INTERCEPT_FUNC(HeapReAlloc);
+    ASAN_INTERCEPT_FUNC(HeapAlloc);
 
     // Undocumented functions must be intercepted by name, not by symbol.
     __interception::OverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap),
diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp
index 7dbd7ab98a17b..25f2e6cd551fe 100644
--- a/compiler-rt/lib/asan/asan_win.cpp
+++ b/compiler-rt/lib/asan/asan_win.cpp
@@ -159,6 +159,8 @@ INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security,
 namespace __asan {
 
 void InitializePlatformInterceptors() {
+  __interception::SetErrorReportCallback(Report);
+
   // The interceptors were not designed to be removable, so we have to keep this
   // module alive for the life of the process.
   HMODULE pinned;
@@ -194,9 +196,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
 }
 
 void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+  // Only asan on 64-bit Windows supports committing shadow memory on demand.
+#if SANITIZER_WINDOWS64
   // Since asan's mapping is compacting, the shadow chunk may be
   // not page-aligned, so we only flush the page-aligned portion.
   ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+#endif
 }
 
 // ---------------------- TSD ---------------- {{{
diff --git a/compiler-rt/lib/dfsan/dfsan_custom.cpp b/compiler-rt/lib/dfsan/dfsan_custom.cpp
index 6f41e225d9e8c..8bb5d39ee8f24 100644
--- a/compiler-rt/lib/dfsan/dfsan_custom.cpp
+++ b/compiler-rt/lib/dfsan/dfsan_custom.cpp
@@ -204,6 +204,57 @@ SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strpbrk(
   return const_cast<char *>(ret);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strsep(char **s, const char *delim,
+                                                  dfsan_label s_label,
+                                                  dfsan_label delim_label,
+                                                  dfsan_label *ret_label) {
+  dfsan_label base_label = dfsan_read_label(s, sizeof(*s));
+  char *base = *s;
+  char *res = strsep(s, delim);
+  if (res != *s) {
+    char *token_start = res;
+    int token_length = strlen(res);
+    // the delimiter byte has been set to NULL
+    dfsan_set_label(0, token_start + token_length, 1);
+  }
+
+  if (flags().strict_data_dependencies) {
+    *ret_label = res ? base_label : 0;
+  } else {
+    size_t s_bytes_read = (res ? strlen(res) : strlen(base)) + 1;
+    *ret_label = dfsan_union(
+        dfsan_union(base_label, dfsan_read_label(base, sizeof(s_bytes_read))),
+        dfsan_union(dfsan_read_label(delim, strlen(delim) + 1),
+                    dfsan_union(s_label, delim_label)));
+  }
+
+  return res;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strsep(
+    char **s, const char *delim, dfsan_label s_label, dfsan_label delim_label,
+    dfsan_label *ret_label, dfsan_origin s_origin, dfsan_origin delim_origin,
+    dfsan_origin *ret_origin) {
+  dfsan_origin base_origin = dfsan_read_origin_of_first_taint(s, sizeof(*s));
+  char *res = __dfsw_strsep(s, delim, s_label, delim_label, ret_label);
+  if (flags().strict_data_dependencies) {
+    if (res)
+      *ret_origin = base_origin;
+  } else {
+    if (*ret_label) {
+      if (base_origin) {
+        *ret_origin = base_origin;
+      } else {
+        dfsan_origin o =
+            dfsan_read_origin_of_first_taint(delim, strlen(delim) + 1);
+        *ret_origin = o ? o : (s_label ? s_origin : delim_origin);
+      }
+    }
+  }
+
+  return res;
+}
+
 static int dfsan_memcmp_bcmp(const void *s1, const void *s2, size_t n,
                              size_t *bytes_read) {
   const char *cs1 = (const char *) s1, *cs2 = (const char *) s2;
diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index ff8a37fbf426f..88ec5cf504bae 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -283,6 +283,7 @@ fun:strncmp=custom
 fun:strpbrk=custom
 fun:strrchr=custom
 fun:strstr=custom
+fun:strsep=custom
 
 # Functions which take action based on global state, such as running a callback
 # set by a separate function.
diff --git a/compiler-rt/lib/fuzzer/FuzzerCorpus.h b/compiler-rt/lib/fuzzer/FuzzerCorpus.h
index e01891e18fe31..912082be8fbae 100644
--- a/compiler-rt/lib/fuzzer/FuzzerCorpus.h
+++ b/compiler-rt/lib/fuzzer/FuzzerCorpus.h
@@ -77,7 +77,7 @@ struct InputInfo {
     SumIncidence = 0.0;
 
     // Apply add-one smoothing to locally discovered features.
-    for (auto F : FeatureFreqs) {
+    for (const auto &F : FeatureFreqs) {
       double LocalIncidence = F.second + 1;
       Energy -= LocalIncidence * log(LocalIncidence);
       SumIncidence += LocalIncidence;
diff --git a/compiler-rt/lib/gwp_asan/tests/backtrace.cpp b/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
index 7cbbcf5fb48ce..6a84a2a4ae5a5 100644
--- a/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
+++ b/compiler-rt/lib/gwp_asan/tests/backtrace.cpp
@@ -21,7 +21,7 @@ TEST_P(BacktraceGuardedPoolAllocatorDeathTest, DoubleFree) {
   DeathRegex.append("was deallocated.*DeallocateMemory[^2].*");
   DeathRegex.append("was allocated.*AllocateMemory");
   if (!Recoverable) {
-    ASSERT_DEATH(DeallocateMemory2(GPA, Ptr), DeathRegex);
+    EXPECT_DEATH(DeallocateMemory2(GPA, Ptr), DeathRegex);
     return;
   }
 
@@ -51,7 +51,7 @@ TEST_P(BacktraceGuardedPoolAllocatorDeathTest, UseAfterFree) {
   DeathRegex.append("was allocated.*AllocateMemory");
 
   if (!Recoverable) {
-    ASSERT_DEATH(TouchMemory(Ptr), DeathRegex);
+    EXPECT_DEATH(TouchMemory(Ptr), DeathRegex);
     return;
   }
 
diff --git a/compiler-rt/lib/gwp_asan/tests/harness.cpp b/compiler-rt/lib/gwp_asan/tests/harness.cpp
index 6d416300aeafe..4633d3ec6e9ed 100644
--- a/compiler-rt/lib/gwp_asan/tests/harness.cpp
+++ b/compiler-rt/lib/gwp_asan/tests/harness.cpp
@@ -47,8 +47,15 @@ void CheckOnlyOneGwpAsanCrash(const std::string &OutputBuffer) {
       << OutputBuffer;
 }
 
+// Fuchsia does not support recoverable GWP-ASan.
+#if defined(__Fuchsia__)
+INSTANTIATE_TEST_SUITE_P(RecoverableAndNonRecoverableTests,
+                         BacktraceGuardedPoolAllocatorDeathTest,
+                         /* Recoverable */ testing::Values(false));
+#else
 INSTANTIATE_TEST_SUITE_P(RecoverableTests, BacktraceGuardedPoolAllocator,
                          /* Recoverable */ testing::Values(true));
 INSTANTIATE_TEST_SUITE_P(RecoverableAndNonRecoverableTests,
                          BacktraceGuardedPoolAllocatorDeathTest,
                          /* Recoverable */ testing::Bool());
+#endif
diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp
index 097136fe5480c..238fcc4d61853 100644
--- a/compiler-rt/lib/hwasan/hwasan.cpp
+++ b/compiler-rt/lib/hwasan/hwasan.cpp
@@ -445,16 +445,32 @@ void __hwasan_print_shadow(const void *p, uptr sz) {
 sptr __hwasan_test_shadow(const void *p, uptr sz) {
   if (sz == 0)
     return -1;
-  tag_t ptr_tag = GetTagFromPointer((uptr)p);
-  uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
+  uptr ptr = reinterpret_cast<uptr>(p);
+  tag_t ptr_tag = GetTagFromPointer(ptr);
+  uptr ptr_raw = UntagAddr(ptr);
   uptr shadow_first = MemToShadow(ptr_raw);
-  uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
-  for (uptr s = shadow_first; s <= shadow_last; ++s)
-    if (*(tag_t *)s != ptr_tag) {
-      sptr offset = ShadowToMem(s) - ptr_raw;
+  uptr shadow_last = MemToShadow(ptr_raw + sz);
+  for (uptr s = shadow_first; s < shadow_last; ++s) {
+    if (UNLIKELY(*(tag_t *)s != ptr_tag)) {
+      uptr short_size =
+          ShortTagSize(*(tag_t *)s, AddTagToPointer(ShadowToMem(s), ptr_tag));
+      sptr offset = ShadowToMem(s) - ptr_raw + short_size;
       return offset < 0 ? 0 : offset;
     }
-  return -1;
+  }
+
+  uptr end = ptr + sz;
+  uptr tail_sz = end & (kShadowAlignment - 1);
+  if (!tail_sz)
+    return -1;
+
+  uptr short_size =
+      ShortTagSize(*(tag_t *)shadow_last, end & ~(kShadowAlignment - 1));
+  if (LIKELY(tail_sz <= short_size))
+    return -1;
+
+  sptr offset = sz - tail_sz + short_size;
+  return offset < 0 ? 0 : offset;
 }
 
 u16 __sanitizer_unaligned_load16(const uu16 *p) {
@@ -563,7 +579,7 @@ uptr __hwasan_tag_pointer(uptr p, u8 tag) {
 void __hwasan_handle_longjmp(const void *sp_dst) {
   uptr dst = (uptr)sp_dst;
   // HWASan does not support tagged SP.
-  CHECK(GetTagFromPointer(dst) == 0);
+  CHECK_EQ(GetTagFromPointer(dst), 0);
 
   uptr sp = (uptr)__builtin_frame_address(0);
   static const uptr kMaxExpectedCleanupSize = 64 << 20;  // 64M
diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h
index c3d71a28142f6..37ef482228511 100644
--- a/compiler-rt/lib/hwasan/hwasan.h
+++ b/compiler-rt/lib/hwasan/hwasan.h
@@ -16,6 +16,7 @@
 
 #include "hwasan_flags.h"
 #include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
@@ -78,12 +79,23 @@ const unsigned kRecordFPShift = 48;
 const unsigned kRecordFPLShift = 4;
 const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift);
 
+static inline bool InTaggableRegion(uptr addr) {
+#if defined(HWASAN_ALIASING_MODE)
+  // Aliases are mapped next to shadow so that the upper bits match the shadow
+  // base.
+  return (addr >> kTaggableRegionCheckShift) ==
+         (__hwasan::GetShadowOffset() >> kTaggableRegionCheckShift);
+#endif
+  return true;
+}
+
 static inline tag_t GetTagFromPointer(uptr p) {
-  return (p >> kAddressTagShift) & kTagMask;
+  return InTaggableRegion(p) ? ((p >> kAddressTagShift) & kTagMask) : 0;
 }
 
 static inline uptr UntagAddr(uptr tagged_addr) {
-  return tagged_addr & ~kAddressTagMask;
+  return InTaggableRegion(tagged_addr) ? (tagged_addr & ~kAddressTagMask)
+                                       : tagged_addr;
 }
 
 static inline void *UntagPtr(const void *tagged_ptr) {
@@ -92,7 +104,9 @@ static inline void *UntagPtr(const void *tagged_ptr) {
 }
 
 static inline uptr AddTagToPointer(uptr p, tag_t tag) {
-  return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift);
+  return InTaggableRegion(p)
+             ? ((p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift))
+             : p;
 }
 
 namespace __hwasan {
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
index d3cb5c8457259..3b59741df6e36 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
@@ -213,7 +213,10 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
     ReportOutOfMemory(size, stack);
   }
   if (zeroise) {
-    internal_memset(allocated, 0, size);
+    // The secondary allocator mmaps memory, which should be zero-inited so we
+    // don't need to explicitly clear it.
+    if (allocator.FromPrimary(allocated))
+      internal_memset(allocated, 0, size);
   } else if (flags()->max_malloc_fill_size > 0) {
     uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size);
     internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
@@ -287,9 +290,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   CHECK(tagged_ptr);
   RunFreeHooks(tagged_ptr);
 
-  bool in_taggable_region =
-      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr));
-  void *untagged_ptr = in_taggable_region ? UntagPtr(tagged_ptr) : tagged_ptr;
+  void *untagged_ptr = UntagPtr(tagged_ptr);
 
   if (CheckInvalidFree(stack, untagged_ptr, tagged_ptr))
     return;
@@ -308,6 +309,9 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   u32 alloc_context_id = meta->GetAllocStackId();
   u32 alloc_thread_id = meta->GetAllocThreadId();
 
+  bool in_taggable_region =
+      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr));
+
   // Check tail magic.
   uptr tagged_size = TaggedSize(orig_size);
   if (flags()->free_checks_tail_magic && orig_size &&
@@ -370,10 +374,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
 
 static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
                               uptr new_size, uptr alignment) {
-  void *untagged_ptr_old =
-      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr_old))
-          ? UntagPtr(tagged_ptr_old)
-          : tagged_ptr_old;
+  void *untagged_ptr_old = UntagPtr(tagged_ptr_old);
   if (CheckInvalidFree(stack, untagged_ptr_old, tagged_ptr_old))
     return nullptr;
   void *tagged_ptr_new =
@@ -381,9 +382,9 @@ static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
   if (tagged_ptr_old && tagged_ptr_new) {
     Metadata *meta =
         reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
-    internal_memcpy(
-        UntagPtr(tagged_ptr_new), untagged_ptr_old,
-        Min(new_size, static_cast<uptr>(meta->GetRequestedSize())));
+    void *untagged_ptr_new = UntagPtr(tagged_ptr_new);
+    internal_memcpy(untagged_ptr_new, untagged_ptr_old,
+                    Min(new_size, static_cast<uptr>(meta->GetRequestedSize())));
     HwasanDeallocate(stack, tagged_ptr_old);
   }
   return tagged_ptr_new;
@@ -426,12 +427,13 @@ static const void *AllocationBegin(const void *p) {
   return (const void *)AddTagToPointer((uptr)beg, tag);
 }
 
-static uptr AllocationSize(const void *tagged_ptr) {
-  const void *untagged_ptr = UntagPtr(tagged_ptr);
+static uptr AllocationSize(const void *p) {
+  const void *untagged_ptr = UntagPtr(p);
   if (!untagged_ptr) return 0;
   const void *beg = allocator.GetBlockBegin(untagged_ptr);
-  Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr);
-  if (beg != untagged_ptr) return 0;
+  if (!beg)
+    return 0;
+  Metadata *b = (Metadata *)allocator.GetMetaData(beg);
   return b->GetRequestedSize();
 }
 
@@ -540,7 +542,7 @@ void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
 }
 
 uptr PointsIntoChunk(void *p) {
-  p = __hwasan::InTaggableRegion(reinterpret_cast<uptr>(p)) ? UntagPtr(p) : p;
+  p = UntagPtr(p);
   uptr addr = reinterpret_cast<uptr>(p);
   uptr chunk =
       reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBeginFastLocked(p));
@@ -558,8 +560,7 @@ uptr PointsIntoChunk(void *p) {
 }
 
 uptr GetUserBegin(uptr chunk) {
-  if (__hwasan::InTaggableRegion(chunk))
-    CHECK_EQ(UntagAddr(chunk), chunk);
+  CHECK_EQ(UntagAddr(chunk), chunk);
   void *block = __hwasan::allocator.GetBlockBeginFastLocked(
       reinterpret_cast<void *>(chunk));
   if (!block)
@@ -573,15 +574,14 @@ uptr GetUserBegin(uptr chunk) {
 }
 
 uptr GetUserAddr(uptr chunk) {
-  tag_t mem_tag = *(tag_t *)__hwasan::MemToShadow(chunk);
-  if (!__hwasan::InTaggableRegion(chunk))
+  if (!InTaggableRegion(chunk))
     return chunk;
+  tag_t mem_tag = *(tag_t *)__hwasan::MemToShadow(chunk);
   return AddTagToPointer(chunk, mem_tag);
 }
 
 LsanMetadata::LsanMetadata(uptr chunk) {
-  if (__hwasan::InTaggableRegion(chunk))
-    CHECK_EQ(UntagAddr(chunk), chunk);
+  CHECK_EQ(UntagAddr(chunk), chunk);
   metadata_ =
       chunk ? __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk))
             : nullptr;
@@ -619,7 +619,7 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) {
 }
 
 IgnoreObjectResult IgnoreObject(const void *p) {
-  p = __hwasan::InTaggableRegion(reinterpret_cast<uptr>(p)) ? UntagPtr(p) : p;
+  p = UntagPtr(p);
   uptr addr = reinterpret_cast<uptr>(p);
   uptr chunk = reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBegin(p));
   if (!chunk)
@@ -674,3 +674,5 @@ const void *__sanitizer_get_allocated_begin(const void *p) {
 }
 
 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
+
+void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h
index b7a06da0b6534..ecf3f6816fc7e 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.h
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.h
@@ -127,16 +127,6 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
 
 void GetAllocatorStats(AllocatorStatCounters s);
 
-inline bool InTaggableRegion(uptr addr) {
-#if defined(HWASAN_ALIASING_MODE)
-  // Aliases are mapped next to shadow so that the upper bits match the shadow
-  // base.
-  return (addr >> kTaggableRegionCheckShift) ==
-         (GetShadowOffset() >> kTaggableRegionCheckShift);
-#endif
-  return true;
-}
-
 } // namespace __hwasan
 
 #endif // HWASAN_ALLOCATOR_H
diff --git a/compiler-rt/lib/hwasan/hwasan_checks.h b/compiler-rt/lib/hwasan/hwasan_checks.h
index 514d351cf7d72..0911af30dcb8f 100644
--- a/compiler-rt/lib/hwasan/hwasan_checks.h
+++ b/compiler-rt/lib/hwasan/hwasan_checks.h
@@ -125,8 +125,22 @@ __attribute__((always_inline)) static void SigTrap(uptr p, uptr size) {
   // __builtin_unreachable();
 }
 
-__attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches(
-    tag_t mem_tag, uptr ptr, uptr sz) {
+__attribute__((always_inline, nodebug)) static inline uptr ShortTagSize(
+    tag_t mem_tag, uptr ptr) {
+  DCHECK(IsAligned(ptr, kShadowAlignment));
+  tag_t ptr_tag = GetTagFromPointer(ptr);
+  if (ptr_tag == mem_tag)
+    return kShadowAlignment;
+  if (!mem_tag || mem_tag >= kShadowAlignment)
+    return 0;
+  if (*(u8 *)(ptr | (kShadowAlignment - 1)) != ptr_tag)
+    return 0;
+  return mem_tag;
+}
+
+__attribute__((always_inline, nodebug)) static inline bool
+PossiblyShortTagMatches(tag_t mem_tag, uptr ptr, uptr sz) {
+  DCHECK(IsAligned(ptr, kShadowAlignment));
   tag_t ptr_tag = GetTagFromPointer(ptr);
   if (ptr_tag == mem_tag)
     return true;
@@ -134,9 +148,6 @@ __attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches(
     return false;
   if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag)
     return false;
-#if !defined(__aarch64__) && !(SANITIZER_RISCV64)
-  ptr = UntagAddr(ptr);
-#endif
   return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag;
 }
 
@@ -169,7 +180,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
         __builtin_unreachable();
     }
   uptr end = p + sz;
-  uptr tail_sz = end & 0xf;
+  uptr tail_sz = end & (kShadowAlignment - 1);
   if (UNLIKELY(tail_sz != 0 &&
                !PossiblyShortTagMatches(
                    *shadow_last, end & ~(kShadowAlignment - 1), tail_sz))) {
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index 16ac85eb85894..67edba40b5b90 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -14,9 +14,11 @@
 // sanitizer_common/sanitizer_common_interceptors.h
 //===----------------------------------------------------------------------===//
 
-#include "interception/interception.h"
 #include "hwasan.h"
+#include "hwasan_checks.h"
 #include "hwasan_thread.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 
 #if !SANITIZER_FUCHSIA
@@ -28,31 +30,47 @@ using namespace __hwasan;
 struct ThreadStartArg {
   thread_callback_t callback;
   void *param;
+  __sanitizer_sigset_t starting_sigset_;
 };
 
 static void *HwasanThreadStartFunc(void *arg) {
   __hwasan_thread_enter();
   ThreadStartArg A = *reinterpret_cast<ThreadStartArg*>(arg);
+  SetSigProcMask(&A.starting_sigset_, nullptr);
   UnmapOrDie(arg, GetPageSizeCached());
   return A.callback(A.param);
 }
 
+#    define COMMON_SYSCALL_PRE_READ_RANGE(p, s) __hwasan_loadN((uptr)p, (uptr)s)
+#    define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+      __hwasan_storeN((uptr)p, (uptr)s)
+#    define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+      do {                                       \
+        (void)(p);                               \
+        (void)(s);                               \
+      } while (false)
+#    define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+      do {                                        \
+        (void)(p);                                \
+        (void)(s);                                \
+      } while (false)
+#    include "sanitizer_common/sanitizer_common_syscalls.inc"
+#    include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
+
 INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
             void * param) {
   EnsureMainThreadIDIsCorrect();
   ScopedTaggingDisabler tagging_disabler;
   ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie(
       GetPageSizeCached(), "pthread_create"));
-  *A = {callback, param};
-  int res;
-  {
-    // ASAN uses the same approach to disable leaks from pthread_create.
+  A->callback = callback;
+  A->param = param;
+  ScopedBlockSignals block(&A->starting_sigset_);
+  // ASAN uses the same approach to disable leaks from pthread_create.
 #    if CAN_SANITIZE_LEAKS
-    __lsan::ScopedInterceptorDisabler lsan_disabler;
+  __lsan::ScopedInterceptorDisabler lsan_disabler;
 #    endif
-    res = REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
-  }
-  return res;
+  return REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
 }
 
 INTERCEPTOR(int, pthread_join, void *t, void **arg) {
diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp
index d3e4b5390e827..abf92d290c846 100644
--- a/compiler-rt/lib/hwasan/hwasan_linux.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp
@@ -283,7 +283,7 @@ void InitThreads() {
 bool MemIsApp(uptr p) {
 // Memory outside the alias range has non-zero tags.
 #  if !defined(HWASAN_ALIASING_MODE)
-  CHECK(GetTagFromPointer(p) == 0);
+  CHECK_EQ(GetTagFromPointer(p), 0);
 #  endif
 
   return (p >= kHighMemStart && p <= kHighMemEnd) ||
diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp
index 8f9dc6cf13e04..935bae3e7bf79 100644
--- a/compiler-rt/lib/hwasan/hwasan_report.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_report.cpp
@@ -707,7 +707,8 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size,
 
   sptr offset =
       __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size);
-  CHECK(offset >= 0 && offset < static_cast<sptr>(access_size));
+  CHECK_GE(offset, 0);
+  CHECK_LT(offset, static_cast<sptr>(access_size));
   tag_t ptr_tag = GetTagFromPointer(tagged_addr);
   tag_t *tag_ptr =
       reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset));
diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.cpp b/compiler-rt/lib/hwasan/hwasan_thread_list.cpp
index fa46e658b69d5..d528f520cc9eb 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread_list.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread_list.cpp
@@ -7,7 +7,7 @@ static HwasanThreadList *hwasan_thread_list;
 HwasanThreadList &hwasanThreadList() { return *hwasan_thread_list; }
 
 void InitThreadList(uptr storage, uptr size) {
-  CHECK(hwasan_thread_list == nullptr);
+  CHECK_EQ(hwasan_thread_list, nullptr);
   hwasan_thread_list =
       new (thread_list_placeholder) HwasanThreadList(storage, size);
 }
diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.h b/compiler-rt/lib/hwasan/hwasan_thread_list.h
index 97485b195b64e..99d2d460261fc 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread_list.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread_list.h
@@ -199,7 +199,7 @@ class SANITIZER_MUTEX HwasanThreadList {
     CHECK(IsAligned(free_space_, align));
     Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
     free_space_ += thread_alloc_size_;
-    CHECK(free_space_ <= free_space_end_ && "out of thread memory");
+    CHECK_LE(free_space_, free_space_end_);
     return t;
   }
 
diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp
index faaa8ee15381d..00c317510e420 100644
--- a/compiler-rt/lib/interception/interception_win.cpp
+++ b/compiler-rt/lib/interception/interception_win.cpp
@@ -141,8 +141,29 @@ static const int kBranchLength =
     FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
 static const int kDirectBranchLength = kBranchLength + kAddressLength;
 
+#  if defined(_MSC_VER)
+#    define INTERCEPTION_FORMAT(f, a)
+#  else
+#    define INTERCEPTION_FORMAT(f, a) __attribute__((format(printf, f, a)))
+#  endif
+
+static void (*ErrorReportCallback)(const char *format, ...)
+    INTERCEPTION_FORMAT(1, 2);
+
+void SetErrorReportCallback(void (*callback)(const char *format, ...)) {
+  ErrorReportCallback = callback;
+}
+
+#  define ReportError(...)                \
+    do {                                  \
+      if (ErrorReportCallback)            \
+        ErrorReportCallback(__VA_ARGS__); \
+    } while (0)
+
 static void InterceptionFailed() {
-  // Do we have a good way to abort with an error message here?
+  ReportError("interception_win: failed due to an unrecoverable error.\n");
+  // This acts like an abort when no debugger is attached. According to an old
+  // comment, calling abort() leads to an infinite recursion in CheckFailed.
   __debugbreak();
 }
 
@@ -249,8 +270,13 @@ static void WritePadding(uptr from, uptr size) {
 }
 
 static void WriteJumpInstruction(uptr from, uptr target) {
-  if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
+  if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) {
+    ReportError(
+        "interception_win: cannot write jmp further than 2GB away, from %p to "
+        "%p.\n",
+        (void *)from, (void *)target);
     InterceptionFailed();
+  }
   ptrdiff_t offset = target - from - kJumpInstructionLength;
   *(u8*)from = 0xE9;
   *(u32*)(from + 1) = offset;
@@ -274,6 +300,10 @@ static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
   int offset = indirect_target - from - kIndirectJumpInstructionLength;
   if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
                             indirect_target)) {
+    ReportError(
+        "interception_win: cannot write indirect jmp with target further than "
+        "2GB away, from %p to %p.\n",
+        (void *)from, (void *)indirect_target);
     InterceptionFailed();
   }
   *(u16*)from = 0x25FF;
@@ -492,6 +522,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xFF8B:  // 8B FF : mov edi, edi
     case 0xEC8B:  // 8B EC : mov ebp, esp
     case 0xc889:  // 89 C8 : mov eax, ecx
+    case 0xE589:  // 89 E5 : mov ebp, esp
     case 0xC18B:  // 8B C1 : mov eax, ecx
     case 0xC033:  // 33 C0 : xor eax, eax
     case 0xC933:  // 33 C9 : xor ecx, ecx
@@ -641,6 +672,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x24448B:  // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
     case 0x244C8B:  // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
     case 0x24548B:  // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
+    case 0x245C8B:  // 8B 5C 24 XX : mov ebx, dword ptr [esp + XX]
+    case 0x246C8B:  // 8B 6C 24 XX : mov ebp, dword ptr [esp + XX]
     case 0x24748B:  // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
     case 0x247C8B:  // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
       return 4;
@@ -652,12 +685,20 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
   }
 #endif
 
-  // Unknown instruction!
-  // FIXME: Unknown instruction failures might happen when we add a new
-  // interceptor or a new compiler version. In either case, they should result
-  // in visible and readable error messages. However, merely calling abort()
-  // leads to an infinite recursion in CheckFailed.
-  InterceptionFailed();
+  // Unknown instruction! This might happen when we add a new interceptor, use
+  // a new compiler version, or if Windows changed how some functions are
+  // compiled. In either case, we print the address and 8 bytes of instructions
+  // to notify the user about the error and to help identify the unknown
+  // instruction. Don't treat this as a fatal error, though we can break the
+  // debugger if one has been attached.
+  u8 *bytes = (u8 *)address;
+  ReportError(
+      "interception_win: unhandled instruction at %p: %02x %02x %02x %02x %02x "
+      "%02x %02x %02x\n",
+      (void *)address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4],
+      bytes[5], bytes[6], bytes[7]);
+  if (::IsDebuggerPresent())
+    __debugbreak();
   return 0;
 }
 
@@ -678,6 +719,8 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
   while (cursor != size) {
     size_t rel_offset = 0;
     size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
+    if (!instruction_size)
+      return false;
     _memcpy((void*)(to + cursor), (void*)(from + cursor),
             (size_t)instruction_size);
     if (rel_offset) {
@@ -895,6 +938,10 @@ static void **InterestingDLLsAvailable() {
       "msvcr120.dll",      // VS2013
       "vcruntime140.dll",  // VS2015
       "ucrtbase.dll",      // Universal CRT
+#if (defined(__MINGW32__) && defined(__i386__))
+      "libc++.dll",        // libc++
+      "libunwind.dll",     // libunwind
+#endif
       // NTDLL should go last as it exports some functions that we should
       // override in the CRT [presumably only used internally].
       "ntdll.dll", NULL};
diff --git a/compiler-rt/lib/interception/interception_win.h b/compiler-rt/lib/interception/interception_win.h
index 4590013019e37..f6eca82191cba 100644
--- a/compiler-rt/lib/interception/interception_win.h
+++ b/compiler-rt/lib/interception/interception_win.h
@@ -41,6 +41,11 @@ bool OverrideImportedFunction(const char *module_to_patch,
                               const char *function_name, uptr new_function,
                               uptr *orig_old_func);
 
+// Sets a callback to be used for reporting errors by interception_win. The
+// callback will be called with printf-like arguments. Intended to be used with
+// __sanitizer::Report. Pass nullptr to disable error reporting (default).
+void SetErrorReportCallback(void (*callback)(const char *format, ...));
+
 #if !SANITIZER_WINDOWS64
 // Exposed for unittests
 bool OverrideFunctionWithDetour(
diff --git a/compiler-rt/lib/interception/tests/CMakeLists.txt b/compiler-rt/lib/interception/tests/CMakeLists.txt
index 37bf99edaf068..688edc3750f5f 100644
--- a/compiler-rt/lib/interception/tests/CMakeLists.txt
+++ b/compiler-rt/lib/interception/tests/CMakeLists.txt
@@ -39,6 +39,11 @@ if(MSVC)
     -Wl,-nodefaultlib:libcmt,-defaultlib:msvcrt,-defaultlib:oldnames
     )
 endif()
+if(MINGW)
+  list(APPEND INTERCEPTION_TEST_LINK_FLAGS_COMMON
+    -Wl,--large-address-aware
+    )
+endif()
 list(APPEND INTERCEPTION_TEST_LINK_FLAGS_COMMON -g)
 
 if(NOT MSVC)
diff --git a/compiler-rt/lib/interception/tests/interception_win_test.cpp b/compiler-rt/lib/interception/tests/interception_win_test.cpp
index 01b8d3132c37c..9159ce405f2dc 100644
--- a/compiler-rt/lib/interception/tests/interception_win_test.cpp
+++ b/compiler-rt/lib/interception/tests/interception_win_test.cpp
@@ -18,6 +18,8 @@
 #if !SANITIZER_DEBUG
 #if SANITIZER_WINDOWS
 
+#include <stdarg.h>
+
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
@@ -283,6 +285,37 @@ const u8 kPatchableCode11[] = {
     0x83, 0x64, 0x24, 0x28, 0x00,   // and     dword ptr [rsp+28h],0
 };
 
+const u8 kPatchableCode12[] = {
+    0x55,                           // push    ebp
+    0x53,                           // push    ebx
+    0x57,                           // push    edi
+    0x56,                           // push    esi
+    0x8b, 0x6c, 0x24, 0x18,         // mov     ebp,dword ptr[esp+18h]
+};
+
+const u8 kPatchableCode13[] = {
+    0x55,                           // push    ebp
+    0x53,                           // push    ebx
+    0x57,                           // push    edi
+    0x56,                           // push    esi
+    0x8b, 0x5c, 0x24, 0x14,         // mov     ebx,dword ptr[esp+14h]
+};
+
+const u8 kPatchableCode14[] = {
+    0x55,                           // push    ebp
+    0x89, 0xe5,                     // mov     ebp,esp
+    0x53,                           // push    ebx
+    0x57,                           // push    edi
+    0x56,                           // push    esi
+};
+
+const u8 kUnsupportedCode1[] = {
+    0x0f, 0x0b,                     // ud2
+    0x0f, 0x0b,                     // ud2
+    0x0f, 0x0b,                     // ud2
+    0x0f, 0x0b,                     // ud2
+};
+
 // A buffer holding the dynamically generated code under test.
 u8* ActiveCode;
 const size_t ActiveCodeLength = 4096;
@@ -679,8 +712,11 @@ TEST(Interception, PatchableFunctionWithTrampoline) {
   EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode9, override, prefix));
 #else
   EXPECT_TRUE(TestFunctionPatching(kPatchableCode3, override, prefix));
+  EXPECT_TRUE(TestFunctionPatching(kPatchableCode12, override, prefix));
+  EXPECT_TRUE(TestFunctionPatching(kPatchableCode13, override, prefix));
 #endif
   EXPECT_FALSE(TestFunctionPatching(kPatchableCode4, override, prefix));
+  EXPECT_TRUE(TestFunctionPatching(kPatchableCode14, override, prefix));
 
   EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode1, override, prefix));
   EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode2, override, prefix));
@@ -690,6 +726,43 @@ TEST(Interception, PatchableFunctionWithTrampoline) {
   EXPECT_FALSE(TestFunctionPatching(kUnpatchableCode6, override, prefix));
 }
 
+TEST(Interception, UnsupportedInstructionWithTrampoline) {
+  TestOverrideFunction override = OverrideFunctionWithTrampoline;
+  FunctionPrefixKind prefix = FunctionPrefixPadding;
+
+  static bool reportCalled;
+  reportCalled = false;
+
+  struct Local {
+    static void Report(const char *format, ...) {
+      if (reportCalled)
+        FAIL() << "Report called more times than expected";
+      reportCalled = true;
+      ASSERT_STREQ(
+          "interception_win: unhandled instruction at %p: %02x %02x %02x %02x "
+          "%02x %02x %02x %02x\n",
+          format);
+      va_list args;
+      va_start(args, format);
+      u8 *ptr = va_arg(args, u8 *);
+      for (int i = 0; i < 8; i++) EXPECT_EQ(kUnsupportedCode1[i], ptr[i]);
+      int bytes[8];
+      for (int i = 0; i < 8; i++) {
+        bytes[i] = va_arg(args, int);
+        EXPECT_EQ(kUnsupportedCode1[i], bytes[i]);
+      }
+      va_end(args);
+    }
+  };
+
+  SetErrorReportCallback(Local::Report);
+  EXPECT_FALSE(TestFunctionPatching(kUnsupportedCode1, override, prefix));
+  SetErrorReportCallback(nullptr);
+
+  if (!reportCalled)
+    ADD_FAILURE() << "Report not called";
+}
+
 TEST(Interception, PatchableFunctionPadding) {
   TestOverrideFunction override = OverrideFunction;
   FunctionPrefixKind prefix = FunctionPrefixPadding;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 3bd73cf87483e..490a8b12d8b17 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -7748,8 +7748,7 @@ static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
   for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, internal_strlen(*pp) + 1);
 
-  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
-                                  pp_size * sizeof(char **));
+  COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
 }
 
 INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
@@ -7855,8 +7854,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
     for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
 
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
-                                   nn_size * sizeof(char **));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
   }
   return n;
 }
@@ -7877,8 +7875,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyname, const char *name) {
     for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
 
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
-                                   nn_size * sizeof(char **));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
   }
   return n;
 }
@@ -7897,8 +7894,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) {
     for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
 
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
-                                   nn_size * sizeof(char **));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
   }
   return n;
 }
@@ -10090,41 +10086,6 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
 #define INIT_GETRANDOM
 #endif
 
-#if SANITIZER_INTERCEPT_CRYPT
-INTERCEPTOR(char *, crypt, char *key, char *salt) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, crypt, key, salt);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
-  char *res = REAL(crypt)(key, salt);
-  if (res != nullptr)
-    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
-  return res;
-}
-#define INIT_CRYPT COMMON_INTERCEPT_FUNCTION(crypt);
-#else
-#define INIT_CRYPT
-#endif
-
-#if SANITIZER_INTERCEPT_CRYPT_R
-INTERCEPTOR(char *, crypt_r, char *key, char *salt, void *data) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, crypt_r, key, salt, data);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
-  char *res = REAL(crypt_r)(key, salt, data);
-  if (res != nullptr) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data,
-                                   __sanitizer::struct_crypt_data_sz);
-    COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
-  }
-  return res;
-}
-#define INIT_CRYPT_R COMMON_INTERCEPT_FUNCTION(crypt_r);
-#else
-#define INIT_CRYPT_R
-#endif
-
 #if SANITIZER_INTERCEPT_GETENTROPY
 INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
   void *ctx;
@@ -10702,8 +10663,6 @@ static void InitializeCommonInterceptors() {
   INIT_GETUSERSHELL;
   INIT_SL_INIT;
   INIT_GETRANDOM;
-  INIT_CRYPT;
-  INIT_CRYPT_R;
   INIT_GETENTROPY;
   INIT_QSORT;
   INIT_QSORT_R;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
index 93b988ba16393..3900bcf22b7a1 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
@@ -1374,9 +1374,8 @@ PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) {
 }
 
 POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) {
-  if (res >= 0) {
-    if (ctx)
-      POST_WRITE(ctx, sizeof(*ctx));
+  if (res >= 0 && ctx) {
+    POST_WRITE(ctx, sizeof(*ctx));
     // (*ctx) is actually a pointer to a kernel mapped page, and there are
     // people out there who are crazy enough to peek into that page's 32-byte
     // header.
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index eb39fabfd5983..c82ab5c210562 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -569,8 +569,6 @@
 #define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD
 #define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_ANDROID)
 #define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD)
-#define SANITIZER_INTERCEPT_CRYPT (SI_POSIX && !SI_ANDROID)
-#define SANITIZER_INTERCEPT_CRYPT_R (SI_LINUX && !SI_ANDROID)
 
 #define SANITIZER_INTERCEPT_GETRANDOM \
   ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
index a04eed7aa5a6e..6d61d276d77e3 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
@@ -177,10 +177,6 @@ typedef struct user_fpregs elf_fpregset_t;
 #  include "sanitizer_platform_interceptors.h"
 #  include "sanitizer_platform_limits_posix.h"
 
-#if SANITIZER_INTERCEPT_CRYPT_R
-#include <crypt.h>
-#endif
-
 namespace __sanitizer {
   unsigned struct_utsname_sz = sizeof(struct utsname);
   unsigned struct_stat_sz = sizeof(struct stat);
@@ -300,10 +296,6 @@ namespace __sanitizer {
   unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
 #endif // SANITIZER_GLIBC
 
-#if SANITIZER_INTERCEPT_CRYPT_R
-  unsigned struct_crypt_data_sz = sizeof(struct crypt_data);
-#endif
-
 #if SANITIZER_LINUX && !SANITIZER_ANDROID
   unsigned struct_timex_sz = sizeof(struct timex);
   unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index cfca7bdedbe55..58244c9944a03 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -309,7 +309,6 @@ extern unsigned struct_msqid_ds_sz;
 extern unsigned struct_mq_attr_sz;
 extern unsigned struct_timex_sz;
 extern unsigned struct_statvfs_sz;
-extern unsigned struct_crypt_data_sz;
 #endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
 
 struct __sanitizer_iovec {
@@ -578,8 +577,13 @@ struct __sanitizer_sigset_t {
 #endif
 
 struct __sanitizer_siginfo_pad {
+#if SANITIZER_X32
+  // x32 siginfo_t is aligned to 8 bytes.
+  u64 pad[128 / sizeof(u64)];
+#else
   // Require uptr, because siginfo_t is always pointer-size aligned on Linux.
   uptr pad[128 / sizeof(uptr)];
+#endif
 };
 
 #if SANITIZER_LINUX
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 4ea1ba367764f..b954b7c0bf2ae 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -201,16 +201,18 @@ template <typename Config> class SizeClassAllocator64 {
         // cause a recursive allocation). However, The number of free blocks may
         // be less than two. Therefore, populate the free list before inserting
         // the blocks.
-        if (Size >= 2U) {
+        const bool NeedToRefill = Size == 1U && Region->FreeList.empty();
+        // If BatchClass has been exhausted, the program should have been
+        // aborted.
+        DCHECK(!Region->Exhausted);
+
+        if (UNLIKELY(
+                NeedToRefill &&
+                !populateFreeList(C, SizeClassMap::BatchClassId, Region))) {
+          PrintStats = true;
+        } else {
           pushBlocksImpl(C, SizeClassMap::BatchClassId, Region, Array, Size);
           Region->Stats.PushedBlocks += Size;
-        } else {
-          const bool RegionIsExhausted = Region->Exhausted;
-          if (UNLIKELY(
-                  RegionIsExhausted ||
-                  !populateFreeList(C, SizeClassMap::BatchClassId, Region))) {
-            PrintStats = !RegionIsExhausted && Region->Exhausted;
-          }
         }
       }
 
@@ -227,6 +229,7 @@ template <typename Config> class SizeClassAllocator64 {
         // when it happens.
         reportOutOfBatchClass();
       }
+
       return;
     }
 
diff --git a/compiler-rt/test/dfsan/custom.cpp b/compiler-rt/test/dfsan/custom.cpp
index 6808cb19dd05a..6dbf0d71c9663 100644
--- a/compiler-rt/test/dfsan/custom.cpp
+++ b/compiler-rt/test/dfsan/custom.cpp
@@ -1630,6 +1630,51 @@ void test_strpbrk() {
 #endif
 }
 
+void test_strsep() {
+  char *s = strdup("Hello world/");
+  char *delim = strdup(" /");
+
+  char *p_s = s;
+  char *base = s;
+  char *p_delim = delim;
+
+  // taint delim bytes
+  dfsan_set_label(i_label, p_delim, strlen(p_delim));
+  // taint delim pointer
+  dfsan_set_label(j_label, &p_delim, sizeof(p_delim));
+  // taint the string data bytes
+  dfsan_set_label(k_label, s, 5);
+  // taint the string pointer
+  dfsan_set_label(m_label, &p_s, sizeof(p_s));
+
+  char *rv = strsep(&p_s, p_delim);
+  assert(rv == &base[0]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_LABEL(rv, m_label);
+  ASSERT_READ_LABEL(rv, strlen(rv), k_label);
+#else
+  ASSERT_LABEL(rv, dfsan_union(dfsan_union(i_label, j_label),
+                               dfsan_union(k_label, m_label)));
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, p_s);
+#endif
+
+  // taint the remaining string's pointer
+  char **pp_s = &p_s;
+  char **pp_s_base = pp_s;
+  dfsan_set_label(n_label, pp_s, sizeof(pp_s));
+
+  rv = strsep(pp_s, p_delim);
+
+  assert(rv == &base[6]);
+#ifdef STRICT_DATA_DEPENDENCIES
+  ASSERT_LABEL(rv, n_label);
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, *pp_s);
+#else
+  ASSERT_LABEL(rv, dfsan_union(dfsan_union(i_label, j_label), n_label));
+  ASSERT_INIT_ORIGIN_EQ_ORIGIN(&rv, *pp_s);
+#endif
+}
+
 void test_memchr() {
   char str1[] = "str1";
   dfsan_set_label(i_label, &str1[3], 1);
@@ -2044,6 +2089,7 @@ int main(void) {
   test_strncmp();
   test_strncpy();
   test_strpbrk();
+  test_strsep();
   test_strrchr();
   test_strstr();
   test_strtod();
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp b/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp
new file mode 100644
index 0000000000000..d7bc34ef324f5
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/Linux/syscalls.cpp
@@ -0,0 +1,33 @@
+// RUN: %clangxx_hwasan -O0 %s -o %t && not %env_hwasan_opts=symbolize=0 %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -O3 %s -o %t && not %env_hwasan_opts=symbolize=0 %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: android
+
+#include <assert.h>
+#include <errno.h>
+#include <glob.h>
+#include <malloc.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <sanitizer/hwasan_interface.h>
+#include <sanitizer/linux_syscall_hooks.h>
+
+/* Test the presence of __sanitizer_syscall_ in the tool runtime, and general
+   sanity of their behaviour. */
+
+int main(int argc, char *argv[]) {
+  // lit.cfg.py currently sets 'disable_allocator_tagging=1'
+  __hwasan_enable_allocator_tagging();
+
+  char *buf = (char *)malloc(1000);
+  assert(buf != NULL);
+
+  __sanitizer_syscall_pre_recvmsg(0, buf - 1, 0);
+  // CHECK: HWAddressSanitizer: tag-mismatch on address [[PTR:0x[a-f0-9]+]]
+  // CHECK: Cause: heap-buffer-overflow
+  // CHECK: [[PTR]] is located 1 bytes before a 1000-byte region
+
+  free(buf);
+  return 0;
+}
diff --git a/compiler-rt/test/hwasan/TestCases/short-granule-and-match-all-tag.cpp b/compiler-rt/test/hwasan/TestCases/short-granule-and-match-all-tag.cpp
new file mode 100644
index 0000000000000..5d23ecc3ddd22
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/short-granule-and-match-all-tag.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_hwasan -mllvm -hwasan-match-all-tag=0 %s -o %t && %run %t
+
+#include <sanitizer/hwasan_interface.h>
+#include <stdlib.h>
+
+int main() {
+  __hwasan_enable_allocator_tagging();
+  char *x = (char *)malloc(40);
+  char volatile z = *x;
+  free(x);
+  return 0;
+}
diff --git a/compiler-rt/test/hwasan/TestCases/test_shadow.c b/compiler-rt/test/hwasan/TestCases/test_shadow.c
new file mode 100644
index 0000000000000..693ba255a030e
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/test_shadow.c
@@ -0,0 +1,41 @@
+// RUN: %clang_hwasan %s -o %t && %run %t
+
+#include <assert.h>
+#include <sanitizer/hwasan_interface.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+  __hwasan_enable_allocator_tagging();
+  for (int sz = 0; sz < 64; ++sz) {
+    fprintf(stderr, "sz: %d\n", sz);
+    char *x = (char *)malloc(sz);
+    do {
+      // Empty range is always OK.
+      for (int b = -16; b < sz + 32; ++b)
+        assert(__hwasan_test_shadow(x + b, 0) == -1);
+
+      int real_sz = sz ? sz : 1;
+      // Unlucky case when we cant distinguish between tag and short granule size.
+      if (__hwasan_tag_pointer(x, real_sz % 16) == x)
+        break;
+
+      // Underflow - the first byte is bad.
+      for (int b = -16; b < 0; ++b)
+        assert(__hwasan_test_shadow(x + b, real_sz) == 0);
+
+      // Inbound ranges.
+      for (int b = 0; b < real_sz; ++b)
+        for (int e = b; e <= real_sz; ++e)
+          assert(__hwasan_test_shadow(x + b, e - b) == -1);
+
+      // Overflow - the first byte after the buffer is bad.
+      for (int b = 0; b <= real_sz; ++b)
+        for (int e = real_sz + 1; e <= real_sz + 64; ++e)
+          assert(__hwasan_test_shadow(x + b, e - b) == (real_sz - b));
+
+    } while (0);
+    free(x);
+  }
+  return 0;
+}
diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/objc-Foo-foo-class-method.S b/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/objc-Foo-foo-class-method.S
new file mode 100644
index 0000000000000..12a1f359eb5ae
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/objc-Foo-foo-class-method.S
@@ -0,0 +1,88 @@
+	.section	__TEXT,__text,regular,pure_instructions
+	.build_version macos, 14, 0	sdk_version 14, 0
+	.p2align	2
+"+[Foo foo]":
+	mov	w0, wzr
+	ret
+
+	.section	__TEXT,__objc_classname,cstring_literals
+l_OBJC_CLASS_NAME_:
+	.asciz	"Foo"
+
+	.section	__TEXT,__objc_methname,cstring_literals
+l_OBJC_METH_VAR_NAME_:
+	.asciz	"foo"
+
+	.section	__TEXT,__objc_methtype,cstring_literals
+l_OBJC_METH_VAR_TYPE_:
+	.asciz	"i16@0:8"
+
+	.section	__DATA,__objc_const
+	.p2align	3
+__OBJC_$_CLASS_METHODS_Foo:
+	.long	24
+	.long	1
+	.quad	l_OBJC_METH_VAR_NAME_
+	.quad	l_OBJC_METH_VAR_TYPE_
+	.quad	"+[Foo foo]"
+
+	.p2align	3
+__OBJC_METACLASS_RO_$_Foo:
+	.long	1
+	.long	40
+	.long	40
+	.space	4
+	.quad	0
+	.quad	l_OBJC_CLASS_NAME_
+	.quad	__OBJC_$_CLASS_METHODS_Foo
+	.quad	0
+	.quad	0
+	.quad	0
+	.quad	0
+
+	.section	__DATA,__objc_data
+	.globl	_OBJC_METACLASS_$_Foo
+	.p2align	3
+_OBJC_METACLASS_$_Foo:
+	.quad	_OBJC_METACLASS_$_NSObject
+	.quad	_OBJC_METACLASS_$_NSObject
+	.quad	__objc_empty_cache
+	.quad	0
+	.quad	__OBJC_METACLASS_RO_$_Foo
+
+	.section	__DATA,__objc_const
+	.p2align	3
+__OBJC_CLASS_RO_$_Foo:
+	.long	0
+	.long	8
+	.long	8
+	.space	4
+	.quad	0
+	.quad	l_OBJC_CLASS_NAME_
+	.quad	0
+	.quad	0
+	.quad	0
+	.quad	0
+	.quad	0
+
+	.section	__DATA,__objc_data
+	.globl	_OBJC_CLASS_$_Foo
+	.p2align	3
+_OBJC_CLASS_$_Foo:
+	.quad	_OBJC_METACLASS_$_Foo
+	.quad	_OBJC_CLASS_$_NSObject
+	.quad	__objc_empty_cache
+	.quad	0
+	.quad	__OBJC_CLASS_RO_$_Foo
+
+	.section	__DATA,__objc_classlist,regular,no_dead_strip
+	.p2align	3
+l_OBJC_LABEL_CLASS_$:
+	.quad	_OBJC_CLASS_$_Foo
+
+	.section	__DATA,__objc_imageinfo,regular,no_dead_strip
+L_OBJC_IMAGE_INFO:
+	.long	0
+	.long	64
+
+.subsections_via_symbols
diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-cross-file-method-call.S b/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-cross-file-method-call.S
new file mode 100644
index 0000000000000..0d0d2debd4679
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/objc-cross-file-method-call.S
@@ -0,0 +1,37 @@
+// Test that calls to Objective-C methods in other files work.
+//
+// RUN: %clang -c -o %t.o %s
+// RUN: %clang -c -o %t.aux.o %S/Inputs/objc-Foo-foo-class-method.S
+// RUN: %llvm_jitlink -preload libobjc.A.dylib %t.o %t.aux.o
+
+        .section	__TEXT,__text,regular,pure_instructions
+	.build_version macos, 14, 0	sdk_version 14, 0
+	.globl	_main
+	.p2align	2
+_main:
+	adrp	x8, _OBJC_CLASSLIST_REFERENCES_$_@PAGE
+	ldr	x0, [x8, _OBJC_CLASSLIST_REFERENCES_$_@PAGEOFF]
+	adrp	x8, _OBJC_SELECTOR_REFERENCES_@PAGE
+	ldr	x1, [x8, _OBJC_SELECTOR_REFERENCES_@PAGEOFF]
+	b	_objc_msgSend
+
+	.section	__DATA,__objc_classrefs,regular,no_dead_strip
+	.p2align	3, 0x0
+_OBJC_CLASSLIST_REFERENCES_$_:
+	.quad	_OBJC_CLASS_$_Foo
+
+	.section	__TEXT,__objc_methname,cstring_literals
+l_OBJC_METH_VAR_NAME_:
+	.asciz	"foo"
+
+	.section	__DATA,__objc_selrefs,literal_pointers,no_dead_strip
+	.p2align	3, 0x0
+_OBJC_SELECTOR_REFERENCES_:
+	.quad	l_OBJC_METH_VAR_NAME_
+
+	.section	__DATA,__objc_imageinfo,regular,no_dead_strip
+L_OBJC_IMAGE_INFO:
+	.long	0
+	.long	64
+
+.subsections_via_symbols
diff --git a/compiler-rt/test/sanitizer_common/CMakeLists.txt b/compiler-rt/test/sanitizer_common/CMakeLists.txt
index f28ab0f627350..f2df8cec3549b 100644
--- a/compiler-rt/test/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/test/sanitizer_common/CMakeLists.txt
@@ -7,7 +7,7 @@ set(SANITIZER_COMMON_TESTSUITES)
 # FIXME(dliew): We should switch to COMPILER_RT_SANITIZERS_TO_BUILD instead of
 # the hard coded `SUPPORTED_TOOLS_INIT` list once we know that the other
 # sanitizers work.
-set(SUPPORTED_TOOLS_INIT asan lsan msan tsan ubsan)
+set(SUPPORTED_TOOLS_INIT asan lsan hwasan msan tsan ubsan)
 set(SUPPORTED_TOOLS)
   foreach(SANITIZER_TOOL ${SUPPORTED_TOOLS_INIT})
     string(TOUPPER ${SANITIZER_TOOL} SANITIZER_TOOL_UPPER)
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp
index 03ddf06ae2d41..1c740153a81d7 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/allow_user_segv.cpp
@@ -3,6 +3,9 @@
 // Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46860
 // XFAIL: !compiler-rt-optimized && tsan
 
+// FIXME: Implement.
+// XFAIL: hwasan
+
 // RUN: %clangxx -O0 %s -o %t
 
 // RUN: %env_tool_opts=handle_segv=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/crypt_r.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/crypt_r.cpp
deleted file mode 100644
index 69bfb46aa5f17..0000000000000
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/crypt_r.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-// RUN: %clangxx -O0 -g %s -lcrypt -o %t && %run %t
-
-// crypt.h is missing from Android.
-// UNSUPPORTED: android
-
-#include <assert.h>
-#include <unistd.h>
-#include <cstring>
-#include <crypt.h>
-
-int main(int argc, char **argv) {
-  {
-    crypt_data cd;
-    cd.initialized = 0;
-    char *p = crypt_r("abcdef", "xz", &cd);
-    volatile size_t z = strlen(p);
-  }
-  {
-    crypt_data cd;
-    cd.initialized = 0;
-    char *p = crypt_r("abcdef", "$1$", &cd);
-    volatile size_t z = strlen(p);
-  }
-  {
-    crypt_data cd;
-    cd.initialized = 0;
-    char *p = crypt_r("abcdef", "$5$", &cd);
-    volatile size_t z = strlen(p);
-  }
-  {
-    crypt_data cd;
-    cd.initialized = 0;
-    char *p = crypt_r("abcdef", "$6$", &cd);
-    volatile size_t z = strlen(p);
-  }
-}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/deepbind.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/deepbind.cpp
index 81150fae977e0..4482713dd424d 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/deepbind.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/deepbind.cpp
@@ -1,6 +1,9 @@
 // RUN: %clangxx %s -o %t && %run not %t 1 2>&1 | FileCheck %s
 // UNSUPPORTED: lsan,ubsan,android
 
+// FIXME: Implement.
+// XFAIL: hwasan
+
 #include <dlfcn.h>
 #include <stdio.h>
 #include <string>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/malloc_usable_size.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/malloc_usable_size.c
index c51ab73774eea..07abb1fa29363 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/malloc_usable_size.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/malloc_usable_size.c
@@ -1,19 +1,27 @@
 // RUN: %clang -O2 %s -o %t && %run %t
 
-// Ubsan does not provide allocator.
-// UNSUPPORTED: ubsan
+// Must not be implemented, no other reason to install interceptors.
+// XFAIL: ubsan
 
 #include <assert.h>
 #include <malloc.h>
 #include <sanitizer/allocator_interface.h>
 #include <stdlib.h>
 
+void *p;
+
 int main() {
   assert(__sanitizer_get_allocated_size(NULL) == 0);
   assert(malloc_usable_size(NULL) == 0);
 
-  int size = 1234567;
-  void *p = malloc(size);
+  int size = 1;
+  p = malloc(size);
+  assert(__sanitizer_get_allocated_size(p) == size);
+  assert(malloc_usable_size(p) == size);
+  free(p);
+
+  size = 1234567;
+  p = malloc(size);
   assert(__sanitizer_get_allocated_size(p) == size);
   assert(malloc_usable_size(p) == size);
   free(p);
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/mlock_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/mlock_test.cpp
index a952922aaabc8..09b05d1a44434 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/mlock_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/mlock_test.cpp
@@ -1,13 +1,24 @@
+// Test that sanitizers suppress mlock.
 // RUN: %clang  %s -o %t && %run %t
-// XFAIL: ubsan,lsan
+
+// No shadow, so no need to disable mlock.
+// XFAIL: ubsan, lsan
+
+// FIXME: Implement.
+// XFAIL: hwasan
 
 #include <assert.h>
 #include <sys/mman.h>
 
 int main() {
+#if defined(__has_feature)
+#  if __has_feature(hwaddress_sanitizer)
+  // Don't mlock, it may claim all memory.
+  abort();
+#  endif
+#endif
   assert(0 == mlockall(MCL_CURRENT));
   assert(0 == mlock((void *)0x12345, 0x5678));
   assert(0 == munlockall());
   assert(0 == munlock((void *)0x987, 0x654));
 }
-
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
new file mode 100644
index 0000000000000..b14ac7bcf1924
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
@@ -0,0 +1,30 @@
+// RUN: %clangxx %s -pie -fPIE -o %t && %run %t
+// REQUIRES: x86_64-target-arch
+
+#include <assert.h>
+#include <stdio.h>
+#include <sys/mman.h>
+
+int main() {
+    for (int j = 0; j < 1024; j++) {
+        // Try 1TB offsets. This attempts to find memory addresses where the
+        // shadow mappings - which assume a 47-bit address space - are invalid.
+        unsigned long long target = (1ULL << 56) - (2 * 4096) - (j * (1ULL << 40));
+
+        // Since we don't use MAP_FIXED, mmap might return an address that is
+        // lower in the address space (due to sanitizer and/or kernel limits).
+        // That is fine - if the app is also restricted from making high
+        // allocations, then they are safe.
+        char* ptr = (char*) mmap ((void*) target, 4096, PROT_READ | PROT_WRITE,
+                                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        printf ("Allocated at %p\n", ptr);
+
+        assert (ptr != MAP_FAILED);
+        for (int i = 0; i < 100; i++) {
+            ptr [i] = 0;
+        }
+        munmap (ptr, 4096);
+    }
+
+    return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/netent.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/netent.cpp
index 296af338e5c84..659a14c570fd0 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/netent.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/netent.cpp
@@ -23,6 +23,11 @@ void test1() {
   assert(ntp && ntp->n_name);
   assert(ntp->n_addrtype == 2);
   assert(ntp->n_net == 127);
+  char **aliases = ntp->n_aliases;
+  while (aliases) {
+    printf("%s\n", *aliases);
+    aliases++;
+  }
   endnetent();
 }
 
@@ -31,6 +36,11 @@ void test2() {
   assert(ntp && ntp->n_name);
   assert(ntp->n_addrtype == 2);
   assert(ntp->n_net == 127);
+  char **aliases = ntp->n_aliases;
+  while (aliases) {
+    printf("%s\n", *aliases);
+    aliases++;
+  }
   endnetent();
 }
 
@@ -41,6 +51,11 @@ void test3() {
   assert(ntp && ntp->n_name);
   assert(ntp->n_addrtype == 2);
   assert(ntp->n_net == 127);
+  char **aliases = ntp->n_aliases;
+  while (aliases) {
+    printf("%s\n", *aliases);
+    aliases++;
+  }
   endnetent();
 }
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/pie_no_aslr.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/pie_no_aslr.cpp
new file mode 100644
index 0000000000000..246794bca7399
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/pie_no_aslr.cpp
@@ -0,0 +1,4 @@
+// RUN: %clangxx %s -pie -fPIE -o %t && %run setarch x86_64 -R %t
+// REQUIRES: x86_64-target-arch
+
+int main() { return 0; }
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp
index ff5226f771a52..890b8bb7db059 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/protoent.cpp
@@ -15,6 +15,11 @@ void test1() {
   struct protoent *ptp = getprotoent();
   assert(ptp && ptp->p_name);
   assert(ptp->p_proto == 0);
+  char **aliases = ptp->p_aliases;
+  while (aliases) {
+    printf("%s\n", *aliases);
+    aliases++;
+  }
   endprotoent();
 }
 
@@ -22,6 +27,11 @@ void test2() {
   struct protoent *ptp = getprotobyname("tcp");
   assert(ptp && ptp->p_name);
   assert(ptp->p_proto == 6);
+  char **aliases = ptp->p_aliases;
+  while (aliases) {
+    printf("%s\n", *aliases);
+    aliases++;
+  }
   endprotoent();
 }
 
@@ -29,6 +39,11 @@ void test3() {
   struct protoent *ptp = getprotobynumber(1);
   assert(ptp && ptp->p_name);
   assert(ptp->p_proto == 1);
+  char **aliases = ptp->p_aliases;
+  while (aliases) {
+    printf("%s\n", *aliases);
+    aliases++;
+  }
   endprotoent();
 }
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp
new file mode 100644
index 0000000000000..83af7d3e611e0
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp
@@ -0,0 +1,90 @@
+// RUN: %clangxx %s -o %t
+// RUN: env %tool_options=allocator_release_to_os_interval_ms=-1 %run %t
+
+// Not needed, no allocator.
+// XFAIL: ubsan
+
+// FIXME: Implement
+// XFAIL: lsan, msan, tsan, hwasan-aliasing
+
+#include <algorithm>
+#include <assert.h>
+#include <fcntl.h>
+#include <random>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sanitizer/allocator_interface.h>
+
+const size_t kPageSize = 4096;
+
+void sync_rss() {
+  char *page =
+      (char *)mmap((void *)&sync_rss, kPageSize, PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+  // Linux kernel updates RSS counters after a set number of page faults.
+  for (int i = 0; i < 10000; ++i) {
+    page[0] = 42;
+    madvise(page, kPageSize, MADV_DONTNEED);
+  }
+  munmap(page, kPageSize);
+}
+
+size_t current_rss() {
+  sync_rss();
+  int statm_fd = open("/proc/self/statm", O_RDONLY);
+  assert(statm_fd >= 0);
+
+  char buf[100];
+  assert(read(statm_fd, &buf, sizeof(buf)) > 0);
+  size_t size, rss;
+  assert(sscanf(buf, "%zu %zu", &size, &rss) == 2);
+
+  close(statm_fd);
+  return rss;
+}
+
+void MallocReleaseStress() {
+  const size_t kNumChunks = 10000;
+  const size_t kAllocSize = 100;
+  const size_t kNumIter = 100;
+  uintptr_t *chunks[kNumChunks] = {0};
+  std::mt19937 r;
+
+  for (size_t iter = 0; iter < kNumIter; iter++) {
+    std::shuffle(chunks, chunks + kNumChunks, r);
+    size_t to_replace = rand() % kNumChunks;
+    for (size_t i = 0; i < kNumChunks; i++) {
+      if (chunks[i])
+        assert(chunks[i][0] == (uintptr_t)chunks[i]);
+      if (i < to_replace) {
+        delete[] chunks[i];
+        chunks[i] = new uintptr_t[kAllocSize];
+        chunks[i][0] = (uintptr_t)chunks[i];
+      }
+    }
+  }
+  fprintf(stderr, "before delete: %zu\n", current_rss());
+  for (auto p : chunks)
+    delete[] p;
+}
+
+int main(int argc, char **argv) {
+  // 32bit asan allocator is unsupported.
+  if (sizeof(void *) < 8 && __has_feature(address_sanitizer))
+    return 0;
+  auto a = current_rss();
+  MallocReleaseStress();
+  auto b = current_rss();
+  __sanitizer_purge_allocator();
+  auto c = current_rss();
+  fprintf(stderr, "a:%zu b:%zu c:%zu\n", a, b, c);
+  assert(a * 2 < b);
+  assert(c * 2 < b);
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_name.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_name.c
index e29cb1e730362..58a2d457f359e 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_name.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_name.c
@@ -6,6 +6,9 @@
 // RUN: %env_tool_opts=handle_sigbus=2 not %run %t 4 2>&1 | FileCheck %s -DSIGNAME=BUS
 // RUN: %env_tool_opts=handle_sigtrap=2 not %run %t 5 2>&1 | FileCheck %s -DSIGNAME=TRAP
 
+// FIXME: Hwasan misclassify TRAP as tag missmatch.
+// XFAIL: hwasan && !hwasan-aliasing
+
 #include <signal.h>
 #include <stdlib.h>
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/crypt.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/crypt.cpp
deleted file mode 100644
index 3a8faaa1ae768..0000000000000
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/crypt.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-// RUN: %clangxx -O0 -g %s -o %t -lcrypt && %run %t
-
-// crypt() is missing from Android and -lcrypt from darwin.
-// UNSUPPORTED: android, darwin
-
-#include <assert.h>
-#include <unistd.h>
-#include <cstring>
-#if __has_include(<crypt.h>)
-#include <crypt.h>
-#endif
-
-int
-main (int argc, char** argv)
-{
-  {
-    char *p = crypt("abcdef", "xz");
-    volatile size_t z = strlen(p);
-  }
-  {
-    char *p = crypt("abcdef", "$1$");
-    volatile size_t z = strlen(p);
-  }
-  {
-    char *p = crypt("abcdef", "$5$");
-    volatile size_t z = strlen(p);
-  }
-  {
-    char *p = crypt("abcdef", "$6$");
-    volatile size_t z = strlen(p);
-  }
-}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/dump_registers.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/dump_registers.cpp
index 07e87bedc131c..f09b2bf4447cc 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/dump_registers.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/dump_registers.cpp
@@ -4,11 +4,7 @@
 // RUN: %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-DUMP
 //
 // FIXME: Implement.
-// UNSUPPORTED: asan
-// UNSUPPORTED: lsan
-// UNSUPPORTED: msan
-// UNSUPPORTED: tsan
-// UNSUPPORTED: ubsan
+// XFAIL: *
 
 #include <signal.h>
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/getpass.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/getpass.cpp
index f46447fa388b8..2711cfb112959 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/getpass.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/getpass.cpp
@@ -1,7 +1,11 @@
-// RUN: %clangxx -O0 -g %s -lutil -o %t && %run %t | FileCheck %s
+// RUN: %clangxx -O0 -g %s -lutil -o %t
+
+// Ignore leaks as this is not the point of test, but HWASAN repors one here.
+// RUN: %env_tool_opts=detect_leaks=0 %run %t | FileCheck %s
 
 // REQUIRES: stable-runtime
 // XFAIL: android && asan
+
 // No libutil.
 // UNSUPPORTED: target={{.*solaris.*}}
 
@@ -9,41 +13,40 @@
 #include <stdio.h>
 #include <string.h>
 #if __linux__
-#include <pty.h>
+#  include <pty.h>
 #elif defined(__FreeBSD__)
-#include <libutil.h>
-#include <pwd.h>
-#include <sys/ioctl.h>
-#include <sys/termios.h>
-#include <sys/types.h>
+#  include <libutil.h>
+#  include <pwd.h>
+#  include <sys/ioctl.h>
+#  include <sys/termios.h>
+#  include <sys/types.h>
 #elif defined(__sun__) && defined(__svr4__)
-#include <termios.h>
+#  include <termios.h>
 #else
-#include <util.h>
+#  include <util.h>
 #endif
 #include <unistd.h>
 
-int
-main (int argc, char** argv)
-{
-    int m;
-    int pid = forkpty(&m, NULL, NULL, NULL);
+int main(int argc, char **argv) {
+  int m;
+  int pid = forkpty(&m, NULL, NULL, NULL);
 
-    if(pid == -1) {
-      fprintf(stderr, "forkpty failed\n");
-      return 1;
-    } else if (pid > 0) {
-      char buf[1024];
-      int res = read(m, buf, sizeof(buf));
+  if (pid == -1) {
+    fprintf(stderr, "forkpty failed\n");
+    return 1;
+  } else if (pid > 0) {
+    char buf[1024];
+    int res = read(m, buf, sizeof(buf));
+    write(1, buf, res);
+    write(m, "password\n", 9);
+    while ((res = read(m, buf, sizeof(buf))) > 0)
       write(1, buf, res);
-      write(m, "password\n", 9);
-      while ((res = read(m, buf, sizeof(buf))) > 0) write(1, buf, res);
-    } else {
-      char *s = getpass("prompt");
-      assert(strcmp(s, "password") == 0);
-      write(1, "done\n", 5);
-    }
-    return 0;
+  } else {
+    char *s = getpass("prompt");
+    assert(strcmp(s, "password") == 0);
+    write(1, "done\n", 5);
+  }
+  return 0;
 }
 
 // CHECK: prompt
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/mmap_write_exec.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/mmap_write_exec.cpp
index eec2161573a77..f4165b73db5c1 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/mmap_write_exec.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/mmap_write_exec.cpp
@@ -7,6 +7,9 @@
 // TODO: Fix option on Android, it hangs there for unknown reasons.
 // XFAIL: android
 
+// FIXME: Implement.
+// XFAIL: hwasan
+
 #if defined(__APPLE__)
 #include <Availability.h>
 #endif
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/print-module-map.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/print-module-map.cpp
index c0e80d9debab7..00df7242b9ed2 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/print-module-map.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/print-module-map.cpp
@@ -18,8 +18,8 @@
 // FIXME: Add linux support.
 // XFAIL: msan && target={{.*linux.*}}
 
-// FIXME: Add lsan support.
-// XFAIL: lsan
+// FIXME: Implement.
+// XFAIL: lsan, hwasan
 
 int global;
 
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/weak_hook_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/weak_hook_test.cpp
index ba98f47b35e08..e95de739fe784 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/weak_hook_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/weak_hook_test.cpp
@@ -1,10 +1,11 @@
 // Test the weak hooks.
-// RUN: %clangxx %s -o %t
-// RUN: %run %t
+// RUN: %clangxx %s -o %t && %run %t
 
-// Hooks are not implemented for lsan.
-// XFAIL: lsan
-// XFAIL: ubsan
+// Must not be implemented, no other reason to install interceptors.
+// XFAIL: lsan, ubsan
+
+// FIXME: Implement.
+// XFAIL: hwasan
 
 #include <assert.h>
 #include <string.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/get_allocated_begin.cpp b/compiler-rt/test/sanitizer_common/TestCases/get_allocated_begin.cpp
index 1683063baea26..5d757eeacd5fc 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/get_allocated_begin.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/get_allocated_begin.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx -O0 -g %s -o %t && %run %t
 
-// UBSan does not have its own allocator
-// UNSUPPORTED: ubsan
+// Must not be implemented, no other reason to install interceptors.
+// XFAIL: ubsan
 
 #include <assert.h>
 #include <sanitizer/allocator_interface.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/malloc_hook.cpp b/compiler-rt/test/sanitizer_common/TestCases/malloc_hook.cpp
index 160e4e1bd1754..123910818abda 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/malloc_hook.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/malloc_hook.cpp
@@ -2,8 +2,13 @@
 
 // Malloc/free hooks are not supported on Windows.
 // XFAIL: target={{.*windows-msvc.*}}
+
+// Must not be implemented, no other reason to install interceptors.
 // XFAIL: ubsan
 
+// FIXME: Implement.
+// XFAIL: hwasan
+
 #include <stdlib.h>
 #include <unistd.h>
 #include <sanitizer/allocator_interface.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/symbolize_pc.cpp b/compiler-rt/test/sanitizer_common/TestCases/symbolize_pc.cpp
index 8c794b5f166e3..bb47edd6ba8d9 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/symbolize_pc.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/symbolize_pc.cpp
@@ -2,6 +2,10 @@
 // RUN: %env_tool_opts=strip_path_prefix=/TestCases/ %run %t 2>&1 | FileCheck %s
 //
 // Tests __sanitizer_symbolize_pc.
+
+// FIXME: Investigate why it does not print GLOBAL_VAR_ABC.
+// XFAIL: hwasan && target=aarch64{{.*}}
+
 #include <stdio.h>
 #include <sanitizer/common_interface_defs.h>
 
diff --git a/compiler-rt/test/sanitizer_common/lit.common.cfg.py b/compiler-rt/test/sanitizer_common/lit.common.cfg.py
index 13a21c6c4adcd..a35884cab91da 100644
--- a/compiler-rt/test/sanitizer_common/lit.common.cfg.py
+++ b/compiler-rt/test/sanitizer_common/lit.common.cfg.py
@@ -10,6 +10,14 @@
 if config.tool_name == "asan":
   tool_cflags = ["-fsanitize=address"]
   tool_options = "ASAN_OPTIONS"
+elif config.tool_name == "hwasan":
+  tool_cflags = ["-fsanitize=hwaddress", "-fuse-ld=lld"]
+  if config.target_arch == "x86_64":
+    tool_cflags += ["-fsanitize-hwaddress-experimental-aliasing"]
+    config.available_features.add("hwasan-aliasing")
+  tool_options = "HWASAN_OPTIONS"
+  if not config.has_lld:
+    config.unsupported = True
 elif config.tool_name == "tsan":
   tool_cflags = ["-fsanitize=thread"]
   tool_options = "TSAN_OPTIONS"
diff --git a/compiler-rt/test/tsan/Linux/pie_no_aslr.cpp b/compiler-rt/test/tsan/Linux/pie_no_aslr.cpp
deleted file mode 100644
index b99342da06bb7..0000000000000
--- a/compiler-rt/test/tsan/Linux/pie_no_aslr.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: %clang_tsan %s -pie -fPIE -o %t && %run setarch x86_64 -R %t
-// REQUIRES: x86_64-target-arch
-
-int main() {
-  return 0;
-}
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 9be6cd7eb1f59..ab254871fbb48 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -13,6 +13,14 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
+if (LLVM_ENABLE_EH)
+  # To match with the flang guidelines we currently disable
+  # building with exception support in core LLVM.
+  message(FATAL_ERROR "Flang does not currently support building with \
+          LLVM exceptions enabled.  Please disable LLVM_ENABLE_EH when building \
+          flang.")
+endif()
+
 set(FLANG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 if (CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE)
diff --git a/flang/docs/OpenMP-semantics.md b/flang/docs/OpenMP-semantics.md
index 46dc189456f99..579c40692ba4b 100644
--- a/flang/docs/OpenMP-semantics.md
+++ b/flang/docs/OpenMP-semantics.md
@@ -318,6 +318,7 @@ w/
 For the following OpenMP regions:
 
 * `target` regions
+* `target data` regions
 * `teams` regions
 * `parallel` regions
 * `simd` regions
@@ -407,6 +408,16 @@ More details are listed in the following table:
    <td>OmpLastPrivate
    </td>
   </tr>
+  <tr>
+   <td>use_device_ptr
+   </td>
+   <td>Yes
+   </td>
+   <td>New Symbol
+   </td>
+   <td>OmpUseDevicePtr
+   </td>
+  </tr>
 </table>
 
 To determine the right data-sharing attribute,
@@ -519,6 +530,12 @@ will be determined and represented as `Flag` in the `Symbol` object
 of the variable.
 No `Symbol` or `Scope` will be created.
 
+However, there are some exceptions for this, Pointers that appear in a
+use_device_ptr clause are privatized and the device pointers to the
+corresponding list items in the device data environment are assigned into the
+private versions so it is best to follow the representation for privatised
+variables i.e represent them with a new Symbol and `OmpUseDevicePtr` flag.
+
 The basic steps to determine the data-mapping attribute are:
 
 1. If _map_ clause is present,
diff --git a/flang/docs/doxygen.cfg.in b/flang/docs/doxygen.cfg.in
index 6ee771c23d2b6..2782ded84aa00 100644
--- a/flang/docs/doxygen.cfg.in
+++ b/flang/docs/doxygen.cfg.in
@@ -2105,7 +2105,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2150,7 +2150,7 @@ TEMPLATE_RELATIONS     = NO
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2159,7 +2159,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/flang/examples/CMakeLists.txt b/flang/examples/CMakeLists.txt
index 23fea3920efb6..8cc66ddbbbb0e 100644
--- a/flang/examples/CMakeLists.txt
+++ b/flang/examples/CMakeLists.txt
@@ -1,3 +1,4 @@
 add_subdirectory(ExternalHelloWorld)
 add_subdirectory(PrintFlangFunctionNames)
 add_subdirectory(FlangOmpReport)
+add_subdirectory(FeatureList)
diff --git a/flang/examples/FeatureList/CMakeLists.txt b/flang/examples/FeatureList/CMakeLists.txt
new file mode 100644
index 0000000000000..e17a7bebbff05
--- /dev/null
+++ b/flang/examples/FeatureList/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_llvm_example_library(flangFeatureList
+    MODULE
+    FeatureList.cpp
+
+    DEPENDS
+    acc_gen
+    flangFrontend
+    omp_gen
+)
diff --git a/flang/examples/FeatureList/FeatureList.cpp b/flang/examples/FeatureList/FeatureList.cpp
new file mode 100644
index 0000000000000..5efda806856f6
--- /dev/null
+++ b/flang/examples/FeatureList/FeatureList.cpp
@@ -0,0 +1,775 @@
+//===-- FeatureList.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A plugin that counts the amount of times a particular parse tree node
+// occurs.  This plugin should cover each feature covered in dump-parse-tree.h
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Frontend/FrontendActions.h"
+#include "flang/Frontend/FrontendPluginRegistry.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/parsing.h"
+
+#include <algorithm>
+#include <cstring>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+using namespace Fortran::frontend;
+using namespace Fortran::parser;
+using namespace Fortran;
+
+#define READ_FEATURE_CUST(classname, n) \
+  bool Pre(const classname &) { \
+    record(#n); \
+    return true; \
+  } \
+  void Post(const classname &) {}
+
+#define READ_FEATURE(classname) READ_FEATURE_CUST(classname, classname)
+
+struct NodeVisitor {
+private:
+  std::unordered_map<const char *, unsigned int> frequencies;
+
+  void record(const char *name) {
+    auto [it, ins] = frequencies.insert({name, 1});
+    if (!ins) {
+      it->second++;
+    }
+  }
+
+public:
+  const std::unordered_map<const char *, unsigned int> &getFrequencies() const {
+    return frequencies;
+  }
+
+  READ_FEATURE_CUST(format::ControlEditDesc, ControlEditDesc)
+  READ_FEATURE_CUST(format::DerivedTypeDataEditDesc, DerivedTypeDataEditDesc)
+  READ_FEATURE_CUST(format::FormatItem, FormatItem)
+  READ_FEATURE_CUST(format::FormatSpecification, FormatSpecification)
+  READ_FEATURE_CUST(
+      format::IntrinsicTypeDataEditDesc, IntrinsicTypeDataEditDesc)
+  READ_FEATURE(Abstract)
+  READ_FEATURE(AccAtomicCapture)
+  READ_FEATURE(AccAtomicCapture::Stmt1)
+  READ_FEATURE(AccAtomicCapture::Stmt2)
+  READ_FEATURE(AccAtomicRead)
+  READ_FEATURE(AccAtomicUpdate)
+  READ_FEATURE(AccAtomicWrite)
+  READ_FEATURE(AccBeginBlockDirective)
+  READ_FEATURE(AccBeginCombinedDirective)
+  READ_FEATURE(AccBeginLoopDirective)
+  READ_FEATURE(AccBlockDirective)
+  READ_FEATURE(AccClause)
+  READ_FEATURE(AccBindClause)
+  READ_FEATURE(AccDefaultClause)
+  READ_FEATURE(AccClauseList)
+  READ_FEATURE(AccCombinedDirective)
+  READ_FEATURE(AccDataModifier)
+  READ_FEATURE(AccDataModifier::Modifier)
+  READ_FEATURE(AccDeclarativeDirective)
+  READ_FEATURE(AccEndAtomic)
+  READ_FEATURE(AccEndBlockDirective)
+  READ_FEATURE(AccEndCombinedDirective)
+  READ_FEATURE(AccGangArgument)
+  READ_FEATURE(AccObject)
+  READ_FEATURE(AccObjectList)
+  READ_FEATURE(AccObjectListWithModifier)
+  READ_FEATURE(AccObjectListWithReduction)
+  READ_FEATURE(AccReductionOperator)
+  READ_FEATURE(AccReductionOperator::Operator)
+  READ_FEATURE(AccSizeExpr)
+  READ_FEATURE(AccSizeExprList)
+  READ_FEATURE(AccSelfClause)
+  READ_FEATURE(AccStandaloneDirective)
+  READ_FEATURE(AccDeviceTypeExpr)
+  READ_FEATURE(AccDeviceTypeExprList)
+  READ_FEATURE(AccTileExpr)
+  READ_FEATURE(AccTileExprList)
+  READ_FEATURE(AccLoopDirective)
+  READ_FEATURE(AccWaitArgument)
+  READ_FEATURE(AcImpliedDo)
+  READ_FEATURE(AcImpliedDoControl)
+  READ_FEATURE(AcValue)
+  READ_FEATURE(AccessStmt)
+  READ_FEATURE(AccessId)
+  READ_FEATURE(AccessSpec)
+  READ_FEATURE(AccessSpec::Kind)
+  READ_FEATURE(AcSpec)
+  READ_FEATURE(ActionStmt)
+  READ_FEATURE(ActualArg)
+  READ_FEATURE(ActualArg::PercentRef)
+  READ_FEATURE(ActualArg::PercentVal)
+  READ_FEATURE(ActualArgSpec)
+  READ_FEATURE(AcValue::Triplet)
+  READ_FEATURE(AllocOpt)
+  READ_FEATURE(AllocOpt::Mold)
+  READ_FEATURE(AllocOpt::Source)
+  READ_FEATURE(Allocatable)
+  READ_FEATURE(AllocatableStmt)
+  READ_FEATURE(AllocateCoarraySpec)
+  READ_FEATURE(AllocateObject)
+  READ_FEATURE(AllocateShapeSpec)
+  READ_FEATURE(AllocateStmt)
+  READ_FEATURE(Allocation)
+  READ_FEATURE(AltReturnSpec)
+  READ_FEATURE(ArithmeticIfStmt)
+  READ_FEATURE(ArrayConstructor)
+  READ_FEATURE(ArrayElement)
+  READ_FEATURE(ArraySpec)
+  READ_FEATURE(AssignStmt)
+  READ_FEATURE(AssignedGotoStmt)
+  READ_FEATURE(AssignmentStmt)
+  READ_FEATURE(AssociateConstruct)
+  READ_FEATURE(AssociateStmt)
+  READ_FEATURE(Association)
+  READ_FEATURE(AssumedImpliedSpec)
+  READ_FEATURE(AssumedRankSpec)
+  READ_FEATURE(AssumedShapeSpec)
+  READ_FEATURE(AssumedSizeSpec)
+  READ_FEATURE(Asynchronous)
+  READ_FEATURE(AsynchronousStmt)
+  READ_FEATURE(AttrSpec)
+  READ_FEATURE(BOZLiteralConstant)
+  READ_FEATURE(BackspaceStmt)
+  READ_FEATURE(BasedPointer)
+  READ_FEATURE(BasedPointerStmt)
+  READ_FEATURE(BindAttr)
+  READ_FEATURE(BindAttr::Deferred)
+  READ_FEATURE(BindAttr::Non_Overridable)
+  READ_FEATURE(BindEntity)
+  READ_FEATURE(BindEntity::Kind)
+  READ_FEATURE(BindStmt)
+  READ_FEATURE(Block)
+  READ_FEATURE(BlockConstruct)
+  READ_FEATURE(BlockData)
+  READ_FEATURE(BlockDataStmt)
+  READ_FEATURE(BlockSpecificationPart)
+  READ_FEATURE(BlockStmt)
+  READ_FEATURE(BoundsRemapping)
+  READ_FEATURE(BoundsSpec)
+  READ_FEATURE(Call)
+  READ_FEATURE(CallStmt)
+  READ_FEATURE(CaseConstruct)
+  READ_FEATURE(CaseConstruct::Case)
+  READ_FEATURE(CaseSelector)
+  READ_FEATURE(CaseStmt)
+  READ_FEATURE(CaseValueRange)
+  READ_FEATURE(CaseValueRange::Range)
+  READ_FEATURE(ChangeTeamConstruct)
+  READ_FEATURE(ChangeTeamStmt)
+  READ_FEATURE(CharLength)
+  READ_FEATURE(CharLiteralConstant)
+  READ_FEATURE(CharLiteralConstantSubstring)
+  READ_FEATURE(CharSelector)
+  READ_FEATURE(CharSelector::LengthAndKind)
+  READ_FEATURE(CloseStmt)
+  READ_FEATURE(CloseStmt::CloseSpec)
+  READ_FEATURE(CoarrayAssociation)
+  READ_FEATURE(CoarraySpec)
+  READ_FEATURE(CodimensionDecl)
+  READ_FEATURE(CodimensionStmt)
+  READ_FEATURE(CoindexedNamedObject)
+  READ_FEATURE(CommonBlockObject)
+  READ_FEATURE(CommonStmt)
+  READ_FEATURE(CommonStmt::Block)
+  READ_FEATURE(CompilerDirective)
+  READ_FEATURE(CompilerDirective::IgnoreTKR)
+  READ_FEATURE(CompilerDirective::LoopCount)
+  READ_FEATURE(CompilerDirective::NameValue)
+  READ_FEATURE(ComplexLiteralConstant)
+  READ_FEATURE(ComplexPart)
+  READ_FEATURE(ComponentArraySpec)
+  READ_FEATURE(ComponentAttrSpec)
+  READ_FEATURE(ComponentDataSource)
+  READ_FEATURE(ComponentDecl)
+  READ_FEATURE(FillDecl)
+  READ_FEATURE(ComponentOrFill)
+  READ_FEATURE(ComponentDefStmt)
+  READ_FEATURE(ComponentSpec)
+  READ_FEATURE(ComputedGotoStmt)
+  READ_FEATURE(ConcurrentControl)
+  READ_FEATURE(ConcurrentHeader)
+  READ_FEATURE(ConnectSpec)
+  READ_FEATURE(ConnectSpec::CharExpr)
+  READ_FEATURE(ConnectSpec::CharExpr::Kind)
+  READ_FEATURE(ConnectSpec::Newunit)
+  READ_FEATURE(ConnectSpec::Recl)
+  READ_FEATURE(ContainsStmt)
+  READ_FEATURE(Contiguous)
+  READ_FEATURE(ContiguousStmt)
+  READ_FEATURE(ContinueStmt)
+  READ_FEATURE(CriticalConstruct)
+  READ_FEATURE(CriticalStmt)
+  READ_FEATURE(CycleStmt)
+  READ_FEATURE(DataComponentDefStmt)
+  READ_FEATURE(DataIDoObject)
+  READ_FEATURE(DataImpliedDo)
+  READ_FEATURE(DataRef)
+  READ_FEATURE(DataStmt)
+  READ_FEATURE(DataStmtConstant)
+  READ_FEATURE(DataStmtObject)
+  READ_FEATURE(DataStmtRepeat)
+  READ_FEATURE(DataStmtSet)
+  READ_FEATURE(DataStmtValue)
+  READ_FEATURE(DeallocateStmt)
+  READ_FEATURE(DeclarationConstruct)
+  READ_FEATURE(DeclarationTypeSpec)
+  READ_FEATURE(DeclarationTypeSpec::Class)
+  READ_FEATURE(DeclarationTypeSpec::ClassStar)
+  READ_FEATURE(DeclarationTypeSpec::Record)
+  READ_FEATURE(DeclarationTypeSpec::Type)
+  READ_FEATURE(DeclarationTypeSpec::TypeStar)
+  READ_FEATURE(Default)
+  READ_FEATURE(DeferredCoshapeSpecList)
+  READ_FEATURE(DeferredShapeSpecList)
+  READ_FEATURE(DefinedOpName)
+  READ_FEATURE(DefinedOperator)
+  READ_FEATURE(DefinedOperator::IntrinsicOperator)
+  READ_FEATURE(DerivedTypeDef)
+  READ_FEATURE(DerivedTypeSpec)
+  READ_FEATURE(DerivedTypeStmt)
+  READ_FEATURE(Designator)
+  READ_FEATURE(DimensionStmt)
+  READ_FEATURE(DimensionStmt::Declaration)
+  READ_FEATURE(DoConstruct)
+  READ_FEATURE(DummyArg)
+  READ_FEATURE(ElseIfStmt)
+  READ_FEATURE(ElseStmt)
+  READ_FEATURE(ElsewhereStmt)
+  READ_FEATURE(EndAssociateStmt)
+  READ_FEATURE(EndBlockDataStmt)
+  READ_FEATURE(EndBlockStmt)
+  READ_FEATURE(EndChangeTeamStmt)
+  READ_FEATURE(EndCriticalStmt)
+  READ_FEATURE(EndDoStmt)
+  READ_FEATURE(EndEnumStmt)
+  READ_FEATURE(EndForallStmt)
+  READ_FEATURE(EndFunctionStmt)
+  READ_FEATURE(EndIfStmt)
+  READ_FEATURE(EndInterfaceStmt)
+  READ_FEATURE(EndLabel)
+  READ_FEATURE(EndModuleStmt)
+  READ_FEATURE(EndMpSubprogramStmt)
+  READ_FEATURE(EndProgramStmt)
+  READ_FEATURE(EndSelectStmt)
+  READ_FEATURE(EndSubmoduleStmt)
+  READ_FEATURE(EndSubroutineStmt)
+  READ_FEATURE(EndTypeStmt)
+  READ_FEATURE(EndWhereStmt)
+  READ_FEATURE(EndfileStmt)
+  READ_FEATURE(EntityDecl)
+  READ_FEATURE(EntryStmt)
+  READ_FEATURE(EnumDef)
+  READ_FEATURE(EnumDefStmt)
+  READ_FEATURE(Enumerator)
+  READ_FEATURE(EnumeratorDefStmt)
+  READ_FEATURE(EorLabel)
+  READ_FEATURE(EquivalenceObject)
+  READ_FEATURE(EquivalenceStmt)
+  READ_FEATURE(ErrLabel)
+  READ_FEATURE(ErrorRecovery)
+  READ_FEATURE(EventPostStmt)
+  READ_FEATURE(EventWaitStmt)
+  READ_FEATURE(EventWaitStmt::EventWaitSpec)
+  READ_FEATURE(ExecutableConstruct)
+  READ_FEATURE(ExecutionPart)
+  READ_FEATURE(ExecutionPartConstruct)
+  READ_FEATURE(ExitStmt)
+  READ_FEATURE(ExplicitCoshapeSpec)
+  READ_FEATURE(ExplicitShapeSpec)
+  READ_FEATURE(Expr)
+  READ_FEATURE(Expr::Parentheses)
+  READ_FEATURE(Expr::UnaryPlus)
+  READ_FEATURE(Expr::Negate)
+  READ_FEATURE(Expr::NOT)
+  READ_FEATURE(Expr::PercentLoc)
+  READ_FEATURE(Expr::DefinedUnary)
+  READ_FEATURE(Expr::Power)
+  READ_FEATURE(Expr::Multiply)
+  READ_FEATURE(Expr::Divide)
+  READ_FEATURE(Expr::Add)
+  READ_FEATURE(Expr::Subtract)
+  READ_FEATURE(Expr::Concat)
+  READ_FEATURE(Expr::LT)
+  READ_FEATURE(Expr::LE)
+  READ_FEATURE(Expr::EQ)
+  READ_FEATURE(Expr::NE)
+  READ_FEATURE(Expr::GE)
+  READ_FEATURE(Expr::GT)
+  READ_FEATURE(Expr::AND)
+  READ_FEATURE(Expr::OR)
+  READ_FEATURE(Expr::EQV)
+  READ_FEATURE(Expr::NEQV)
+  READ_FEATURE(Expr::DefinedBinary)
+  READ_FEATURE(Expr::ComplexConstructor)
+  READ_FEATURE(External)
+  READ_FEATURE(ExternalStmt)
+  READ_FEATURE(FailImageStmt)
+  READ_FEATURE(FileUnitNumber)
+  READ_FEATURE(FinalProcedureStmt)
+  READ_FEATURE(FlushStmt)
+  READ_FEATURE(ForallAssignmentStmt)
+  READ_FEATURE(ForallBodyConstruct)
+  READ_FEATURE(ForallConstruct)
+  READ_FEATURE(ForallConstructStmt)
+  READ_FEATURE(ForallStmt)
+  READ_FEATURE(FormTeamStmt)
+  READ_FEATURE(FormTeamStmt::FormTeamSpec)
+  READ_FEATURE(Format)
+  READ_FEATURE(FormatStmt)
+  READ_FEATURE(FunctionReference)
+  READ_FEATURE(FunctionStmt)
+  READ_FEATURE(FunctionSubprogram)
+  READ_FEATURE(GenericSpec)
+  READ_FEATURE(GenericSpec::Assignment)
+  READ_FEATURE(GenericSpec::ReadFormatted)
+  READ_FEATURE(GenericSpec::ReadUnformatted)
+  READ_FEATURE(GenericSpec::WriteFormatted)
+  READ_FEATURE(GenericSpec::WriteUnformatted)
+  READ_FEATURE(GenericStmt)
+  READ_FEATURE(GotoStmt)
+  READ_FEATURE(HollerithLiteralConstant)
+  READ_FEATURE(IdExpr)
+  READ_FEATURE(IdVariable)
+  READ_FEATURE(IfConstruct)
+  READ_FEATURE(IfConstruct::ElseBlock)
+  READ_FEATURE(IfConstruct::ElseIfBlock)
+  READ_FEATURE(IfStmt)
+  READ_FEATURE(IfThenStmt)
+  READ_FEATURE(TeamValue)
+  READ_FEATURE(ImageSelector)
+  READ_FEATURE(ImageSelectorSpec)
+  READ_FEATURE(ImageSelectorSpec::Stat)
+  READ_FEATURE(ImageSelectorSpec::Team_Number)
+  READ_FEATURE(ImplicitPart)
+  READ_FEATURE(ImplicitPartStmt)
+  READ_FEATURE(ImplicitSpec)
+  READ_FEATURE(ImplicitStmt)
+  READ_FEATURE(ImplicitStmt::ImplicitNoneNameSpec)
+  READ_FEATURE(ImpliedShapeSpec)
+  READ_FEATURE(ImportStmt)
+  READ_FEATURE(Initialization)
+  READ_FEATURE(InputImpliedDo)
+  READ_FEATURE(InputItem)
+  READ_FEATURE(InquireSpec)
+  READ_FEATURE(InquireSpec::CharVar)
+  READ_FEATURE(InquireSpec::CharVar::Kind)
+  READ_FEATURE(InquireSpec::IntVar)
+  READ_FEATURE(InquireSpec::IntVar::Kind)
+  READ_FEATURE(InquireSpec::LogVar)
+  READ_FEATURE(InquireSpec::LogVar::Kind)
+  READ_FEATURE(InquireStmt)
+  READ_FEATURE(InquireStmt::Iolength)
+  READ_FEATURE(IntegerTypeSpec)
+  READ_FEATURE(IntentSpec)
+  READ_FEATURE(IntentSpec::Intent)
+  READ_FEATURE(IntentStmt)
+  READ_FEATURE(InterfaceBlock)
+  READ_FEATURE(InterfaceBody)
+  READ_FEATURE(InterfaceBody::Function)
+  READ_FEATURE(InterfaceBody::Subroutine)
+  READ_FEATURE(InterfaceSpecification)
+  READ_FEATURE(InterfaceStmt)
+  READ_FEATURE(InternalSubprogram)
+  READ_FEATURE(InternalSubprogramPart)
+  READ_FEATURE(Intrinsic)
+  READ_FEATURE(IntrinsicStmt)
+  READ_FEATURE(IntrinsicTypeSpec)
+  READ_FEATURE(IntrinsicTypeSpec::Character)
+  READ_FEATURE(IntrinsicTypeSpec::Complex)
+  READ_FEATURE(IntrinsicTypeSpec::DoubleComplex)
+  READ_FEATURE(IntrinsicTypeSpec::DoublePrecision)
+  READ_FEATURE(IntrinsicTypeSpec::Logical)
+  READ_FEATURE(IntrinsicTypeSpec::Real)
+  READ_FEATURE(IoControlSpec)
+  READ_FEATURE(IoControlSpec::Asynchronous)
+  READ_FEATURE(IoControlSpec::CharExpr)
+  READ_FEATURE(IoControlSpec::CharExpr::Kind)
+  READ_FEATURE(IoControlSpec::Pos)
+  READ_FEATURE(IoControlSpec::Rec)
+  READ_FEATURE(IoControlSpec::Size)
+  READ_FEATURE(IoUnit)
+  READ_FEATURE(Keyword)
+  READ_FEATURE(KindParam)
+  READ_FEATURE(KindSelector)
+  READ_FEATURE(KindSelector::StarSize)
+  READ_FEATURE(LabelDoStmt)
+  READ_FEATURE(LanguageBindingSpec)
+  READ_FEATURE(LengthSelector)
+  READ_FEATURE(LetterSpec)
+  READ_FEATURE(LiteralConstant)
+  READ_FEATURE(IntLiteralConstant)
+  READ_FEATURE(LocalitySpec)
+  READ_FEATURE(LocalitySpec::DefaultNone)
+  READ_FEATURE(LocalitySpec::Local)
+  READ_FEATURE(LocalitySpec::LocalInit)
+  READ_FEATURE(LocalitySpec::Shared)
+  READ_FEATURE(LockStmt)
+  READ_FEATURE(LockStmt::LockStat)
+  READ_FEATURE(LogicalLiteralConstant)
+  READ_FEATURE(LoopControl)
+  READ_FEATURE(LoopControl::Concurrent)
+  READ_FEATURE(MainProgram)
+  READ_FEATURE(Map)
+  READ_FEATURE(Map::EndMapStmt)
+  READ_FEATURE(Map::MapStmt)
+  READ_FEATURE(MaskedElsewhereStmt)
+  READ_FEATURE(Module)
+  READ_FEATURE(ModuleStmt)
+  READ_FEATURE(ModuleSubprogram)
+  READ_FEATURE(ModuleSubprogramPart)
+  READ_FEATURE(MpSubprogramStmt)
+  READ_FEATURE(MsgVariable)
+  READ_FEATURE(Name)
+  READ_FEATURE(NamedConstant)
+  READ_FEATURE(NamedConstantDef)
+  READ_FEATURE(NamelistStmt)
+  READ_FEATURE(NamelistStmt::Group)
+  READ_FEATURE(NonLabelDoStmt)
+  READ_FEATURE(NoPass)
+  READ_FEATURE(NullifyStmt)
+  READ_FEATURE(NullInit)
+  READ_FEATURE(ObjectDecl)
+  READ_FEATURE(OldParameterStmt)
+  READ_FEATURE(OmpAlignedClause)
+  READ_FEATURE(OmpAtomic)
+  READ_FEATURE(OmpAtomicCapture)
+  READ_FEATURE(OmpAtomicCapture::Stmt1)
+  READ_FEATURE(OmpAtomicCapture::Stmt2)
+  READ_FEATURE(OmpAtomicRead)
+  READ_FEATURE(OmpAtomicUpdate)
+  READ_FEATURE(OmpAtomicWrite)
+  READ_FEATURE(OmpBeginBlockDirective)
+  READ_FEATURE(OmpBeginLoopDirective)
+  READ_FEATURE(OmpBeginSectionsDirective)
+  READ_FEATURE(OmpBlockDirective)
+  READ_FEATURE(OmpCancelType)
+  READ_FEATURE(OmpCancelType::Type)
+  READ_FEATURE(OmpClause)
+  READ_FEATURE(OmpClauseList)
+  READ_FEATURE(OmpCriticalDirective)
+  READ_FEATURE(OmpDeclareTargetSpecifier)
+  READ_FEATURE(OmpDeclareTargetWithClause)
+  READ_FEATURE(OmpDeclareTargetWithList)
+  READ_FEATURE(OmpDefaultClause)
+  READ_FEATURE(OmpDefaultClause::Type)
+  READ_FEATURE(OmpDefaultmapClause)
+  READ_FEATURE(OmpDefaultmapClause::ImplicitBehavior)
+  READ_FEATURE(OmpDefaultmapClause::VariableCategory)
+  READ_FEATURE(OmpDependClause)
+  READ_FEATURE(OmpDependClause::InOut)
+  READ_FEATURE(OmpDependClause::Sink)
+  READ_FEATURE(OmpDependClause::Source)
+  READ_FEATURE(OmpDependenceType)
+  READ_FEATURE(OmpDependenceType::Type)
+  READ_FEATURE(OmpDependSinkVec)
+  READ_FEATURE(OmpDependSinkVecLength)
+  READ_FEATURE(OmpEndAtomic)
+  READ_FEATURE(OmpEndBlockDirective)
+  READ_FEATURE(OmpEndCriticalDirective)
+  READ_FEATURE(OmpEndLoopDirective)
+  READ_FEATURE(OmpEndSectionsDirective)
+  READ_FEATURE(OmpIfClause)
+  READ_FEATURE(OmpIfClause::DirectiveNameModifier)
+  READ_FEATURE(OmpLinearClause)
+  READ_FEATURE(OmpLinearClause::WithModifier)
+  READ_FEATURE(OmpLinearClause::WithoutModifier)
+  READ_FEATURE(OmpLinearModifier)
+  READ_FEATURE(OmpLinearModifier::Type)
+  READ_FEATURE(OmpLoopDirective)
+  READ_FEATURE(OmpMapClause)
+  READ_FEATURE(OmpMapType)
+  READ_FEATURE(OmpMapType::Always)
+  READ_FEATURE(OmpMapType::Type)
+  READ_FEATURE(OmpObject)
+  READ_FEATURE(OmpObjectList)
+  READ_FEATURE(OmpOrderClause)
+  READ_FEATURE(OmpOrderClause::Type)
+  READ_FEATURE(OmpOrderModifier)
+  READ_FEATURE(OmpOrderModifier::Kind)
+  READ_FEATURE(OmpProcBindClause)
+  READ_FEATURE(OmpProcBindClause::Type)
+  READ_FEATURE(OmpReductionClause)
+  READ_FEATURE(OmpInReductionClause)
+  READ_FEATURE(OmpReductionCombiner)
+  READ_FEATURE(OmpReductionCombiner::FunctionCombiner)
+  READ_FEATURE(OmpReductionInitializerClause)
+  READ_FEATURE(OmpReductionOperator)
+  READ_FEATURE(OmpAllocateClause)
+  READ_FEATURE(OmpAllocateClause::Allocator)
+  READ_FEATURE(OmpScheduleClause)
+  READ_FEATURE(OmpScheduleClause::ScheduleType)
+  READ_FEATURE(OmpDeviceClause)
+  READ_FEATURE(OmpDeviceClause::DeviceModifier)
+  READ_FEATURE(OmpDeviceTypeClause)
+  READ_FEATURE(OmpDeviceTypeClause::Type)
+  READ_FEATURE(OmpScheduleModifier)
+  READ_FEATURE(OmpScheduleModifier::Modifier1)
+  READ_FEATURE(OmpScheduleModifier::Modifier2)
+  READ_FEATURE(OmpScheduleModifierType)
+  READ_FEATURE(OmpScheduleModifierType::ModType)
+  READ_FEATURE(OmpSectionBlocks)
+  READ_FEATURE(OmpSectionsDirective)
+  READ_FEATURE(OmpSimpleStandaloneDirective)
+  READ_FEATURE(Only)
+  READ_FEATURE(OpenACCAtomicConstruct)
+  READ_FEATURE(OpenACCBlockConstruct)
+  READ_FEATURE(OpenACCCacheConstruct)
+  READ_FEATURE(OpenACCCombinedConstruct)
+  READ_FEATURE(OpenACCConstruct)
+  READ_FEATURE(OpenACCDeclarativeConstruct)
+  READ_FEATURE(OpenACCLoopConstruct)
+  READ_FEATURE(OpenACCRoutineConstruct)
+  READ_FEATURE(OpenACCStandaloneDeclarativeConstruct)
+  READ_FEATURE(OpenACCStandaloneConstruct)
+  READ_FEATURE(OpenACCWaitConstruct)
+  READ_FEATURE(OpenMPAtomicConstruct)
+  READ_FEATURE(OpenMPBlockConstruct)
+  READ_FEATURE(OpenMPCancelConstruct)
+  READ_FEATURE(OpenMPCancelConstruct::If)
+  READ_FEATURE(OpenMPCancellationPointConstruct)
+  READ_FEATURE(OpenMPConstruct)
+  READ_FEATURE(OpenMPCriticalConstruct)
+  READ_FEATURE(OpenMPDeclarativeAllocate)
+  READ_FEATURE(OpenMPDeclarativeConstruct)
+  READ_FEATURE(OpenMPDeclareReductionConstruct)
+  READ_FEATURE(OpenMPDeclareSimdConstruct)
+  READ_FEATURE(OpenMPDeclareTargetConstruct)
+  READ_FEATURE(OmpMemoryOrderClause)
+  READ_FEATURE(OmpAtomicClause)
+  READ_FEATURE(OmpAtomicClauseList)
+  READ_FEATURE(OmpAtomicDefaultMemOrderClause)
+  READ_FEATURE(OmpAtomicDefaultMemOrderClause::Type)
+  READ_FEATURE(OpenMPFlushConstruct)
+  READ_FEATURE(OpenMPLoopConstruct)
+  READ_FEATURE(OpenMPExecutableAllocate)
+  READ_FEATURE(OpenMPRequiresConstruct)
+  READ_FEATURE(OpenMPSimpleStandaloneConstruct)
+  READ_FEATURE(OpenMPStandaloneConstruct)
+  READ_FEATURE(OpenMPSectionConstruct)
+  READ_FEATURE(OpenMPSectionsConstruct)
+  READ_FEATURE(OpenMPThreadprivate)
+  READ_FEATURE(OpenStmt)
+  READ_FEATURE(Optional)
+  READ_FEATURE(OptionalStmt)
+  READ_FEATURE(OtherSpecificationStmt)
+  READ_FEATURE(OutputImpliedDo)
+  READ_FEATURE(OutputItem)
+  READ_FEATURE(Parameter)
+  READ_FEATURE(ParameterStmt)
+  READ_FEATURE(ParentIdentifier)
+  READ_FEATURE(Pass)
+  READ_FEATURE(PauseStmt)
+  READ_FEATURE(Pointer)
+  READ_FEATURE(PointerAssignmentStmt)
+  READ_FEATURE(PointerAssignmentStmt::Bounds)
+  READ_FEATURE(PointerDecl)
+  READ_FEATURE(PointerObject)
+  READ_FEATURE(PointerStmt)
+  READ_FEATURE(PositionOrFlushSpec)
+  READ_FEATURE(PrefixSpec)
+  READ_FEATURE(PrefixSpec::Elemental)
+  READ_FEATURE(PrefixSpec::Impure)
+  READ_FEATURE(PrefixSpec::Module)
+  READ_FEATURE(PrefixSpec::Non_Recursive)
+  READ_FEATURE(PrefixSpec::Pure)
+  READ_FEATURE(PrefixSpec::Recursive)
+  READ_FEATURE(PrintStmt)
+  READ_FEATURE(PrivateStmt)
+  READ_FEATURE(PrivateOrSequence)
+  READ_FEATURE(ProcAttrSpec)
+  READ_FEATURE(ProcComponentAttrSpec)
+  READ_FEATURE(ProcComponentDefStmt)
+  READ_FEATURE(ProcComponentRef)
+  READ_FEATURE(ProcDecl)
+  READ_FEATURE(ProcInterface)
+  READ_FEATURE(ProcPointerInit)
+  READ_FEATURE(ProcedureDeclarationStmt)
+  READ_FEATURE(ProcedureDesignator)
+  READ_FEATURE(ProcedureStmt)
+  READ_FEATURE(ProcedureStmt::Kind)
+  READ_FEATURE(Program)
+  READ_FEATURE(ProgramStmt)
+  READ_FEATURE(ProgramUnit)
+  READ_FEATURE(Protected)
+  READ_FEATURE(ProtectedStmt)
+  READ_FEATURE(ReadStmt)
+  READ_FEATURE(RealLiteralConstant)
+  READ_FEATURE(RealLiteralConstant::Real)
+  READ_FEATURE(Rename)
+  READ_FEATURE(Rename::Names)
+  READ_FEATURE(Rename::Operators)
+  READ_FEATURE(ReturnStmt)
+  READ_FEATURE(RewindStmt)
+  READ_FEATURE(Save)
+  READ_FEATURE(SaveStmt)
+  READ_FEATURE(SavedEntity)
+  READ_FEATURE(SavedEntity::Kind)
+  READ_FEATURE(SectionSubscript)
+  READ_FEATURE(SelectCaseStmt)
+  READ_FEATURE(SelectRankCaseStmt)
+  READ_FEATURE(SelectRankCaseStmt::Rank)
+  READ_FEATURE(SelectRankConstruct)
+  READ_FEATURE(SelectRankConstruct::RankCase)
+  READ_FEATURE(SelectRankStmt)
+  READ_FEATURE(SelectTypeConstruct)
+  READ_FEATURE(SelectTypeConstruct::TypeCase)
+  READ_FEATURE(SelectTypeStmt)
+  READ_FEATURE(Selector)
+  READ_FEATURE(SeparateModuleSubprogram)
+  READ_FEATURE(SequenceStmt)
+  READ_FEATURE(Sign)
+  READ_FEATURE(SignedComplexLiteralConstant)
+  READ_FEATURE(SignedIntLiteralConstant)
+  READ_FEATURE(SignedRealLiteralConstant)
+  READ_FEATURE(SpecificationConstruct)
+  READ_FEATURE(SpecificationExpr)
+  READ_FEATURE(SpecificationPart)
+  READ_FEATURE(Star)
+  READ_FEATURE(StatOrErrmsg)
+  READ_FEATURE(StatVariable)
+  READ_FEATURE(StatusExpr)
+  READ_FEATURE(StmtFunctionStmt)
+  READ_FEATURE(StopCode)
+  READ_FEATURE(StopStmt)
+  READ_FEATURE(StopStmt::Kind)
+  READ_FEATURE(StructureComponent)
+  READ_FEATURE(StructureConstructor)
+  READ_FEATURE(StructureDef)
+  READ_FEATURE(StructureDef::EndStructureStmt)
+  READ_FEATURE(StructureField)
+  READ_FEATURE(StructureStmt)
+  READ_FEATURE(Submodule)
+  READ_FEATURE(SubmoduleStmt)
+  READ_FEATURE(SubroutineStmt)
+  READ_FEATURE(SubroutineSubprogram)
+  READ_FEATURE(SubscriptTriplet)
+  READ_FEATURE(Substring)
+  READ_FEATURE(SubstringInquiry)
+  READ_FEATURE(SubstringRange)
+  READ_FEATURE(Suffix)
+  READ_FEATURE(SyncAllStmt)
+  READ_FEATURE(SyncImagesStmt)
+  READ_FEATURE(SyncImagesStmt::ImageSet)
+  READ_FEATURE(SyncMemoryStmt)
+  READ_FEATURE(SyncTeamStmt)
+  READ_FEATURE(Target)
+  READ_FEATURE(TargetStmt)
+  READ_FEATURE(TypeAttrSpec)
+  READ_FEATURE(TypeAttrSpec::BindC)
+  READ_FEATURE(TypeAttrSpec::Extends)
+  READ_FEATURE(TypeBoundGenericStmt)
+  READ_FEATURE(TypeBoundProcBinding)
+  READ_FEATURE(TypeBoundProcDecl)
+  READ_FEATURE(TypeBoundProcedurePart)
+  READ_FEATURE(TypeBoundProcedureStmt)
+  READ_FEATURE(TypeBoundProcedureStmt::WithInterface)
+  READ_FEATURE(TypeBoundProcedureStmt::WithoutInterface)
+  READ_FEATURE(TypeDeclarationStmt)
+  READ_FEATURE(TypeGuardStmt)
+  READ_FEATURE(TypeGuardStmt::Guard)
+  READ_FEATURE(TypeParamDecl)
+  READ_FEATURE(TypeParamDefStmt)
+  READ_FEATURE(common::TypeParamAttr)
+  READ_FEATURE(TypeParamSpec)
+  READ_FEATURE(TypeParamValue)
+  READ_FEATURE(TypeParamValue::Deferred)
+  READ_FEATURE(TypeSpec)
+  READ_FEATURE(Union)
+  READ_FEATURE(Union::EndUnionStmt)
+  READ_FEATURE(Union::UnionStmt)
+  READ_FEATURE(UnlockStmt)
+  READ_FEATURE(UseStmt)
+  READ_FEATURE(UseStmt::ModuleNature)
+  READ_FEATURE(Value)
+  READ_FEATURE(ValueStmt)
+  READ_FEATURE(Variable)
+  READ_FEATURE(Verbatim)
+  READ_FEATURE(Volatile)
+  READ_FEATURE(VolatileStmt)
+  READ_FEATURE(WaitSpec)
+  READ_FEATURE(WaitStmt)
+  READ_FEATURE(WhereBodyConstruct)
+  READ_FEATURE(WhereConstruct)
+  READ_FEATURE(WhereConstruct::Elsewhere)
+  READ_FEATURE(WhereConstruct::MaskedElsewhere)
+  READ_FEATURE(WhereConstructStmt)
+  READ_FEATURE(WhereStmt)
+  READ_FEATURE(WriteStmt)
+
+  READ_FEATURE(llvm::omp::Directive)
+  READ_FEATURE(llvm::omp::Clause)
+  READ_FEATURE(llvm::acc::Directive)
+  READ_FEATURE(llvm::acc::DefaultValue)
+
+  template <typename A> bool Pre(const A &) { return true; }
+  template <typename A> void Post(const A &) {}
+
+  template <typename T> bool Pre(const Statement<T> &) { return true; }
+  template <typename T> void Post(const Statement<T> &) {}
+
+  template <typename T> bool Pre(const UnlabeledStatement<T> &) { return true; }
+  template <typename T> void Post(const UnlabeledStatement<T> &) {}
+
+  template <typename T> bool Pre(const common::Indirection<T> &) {
+    return true;
+  }
+  template <typename T> void Post(const common::Indirection<T> &) {}
+
+  template <typename A> bool Pre(const Scalar<A> &) { return true; }
+  template <typename A> void Post(const Scalar<A> &) {}
+
+  template <typename A> bool Pre(const Constant<A> &) { return true; }
+  template <typename A> void Post(const Constant<A> &) {}
+
+  template <typename A> bool Pre(const Integer<A> &) { return true; }
+  template <typename A> void Post(const Integer<A> &) {}
+
+  template <typename A> bool Pre(const Logical<A> &) { return true; }
+  template <typename A> void Post(const Logical<A> &) {}
+
+  template <typename A> bool Pre(const DefaultChar<A> &) { return true; }
+  template <typename A> void Post(const DefaultChar<A> &) {}
+
+  template <typename... A> bool Pre(const std::tuple<A...> &) { return true; }
+  template <typename... A> void Post(const std::tuple<A...> &) {}
+
+  template <typename... A> bool Pre(const std::variant<A...> &) { return true; }
+  template <typename... A> void Post(const std::variant<A...> &) {}
+};
+
+bool sortNodes(std::pair<const char *, int> a, std::pair<const char *, int> b) {
+  return (a.second == b.second) ? (std::strcmp(a.first, b.first) < 0)
+                                : a.second > b.second;
+}
+
+class FeatureListAction : public PluginParseTreeAction {
+  void executeAction() override {
+    NodeVisitor visitor;
+    Fortran::parser::Walk(getParsing().parseTree(), visitor);
+
+    const auto &frequencyMap = visitor.getFrequencies();
+    std::vector<std::pair<const char *, int>> frequencies(
+        frequencyMap.begin(), frequencyMap.end());
+
+    std::sort(frequencies.begin(), frequencies.end(), sortNodes);
+    for (auto const &[feature, frequency] : frequencies) {
+      llvm::outs() << feature << ": " << frequency << "\n";
+    }
+  }
+
+  bool beginSourceFileAction() override { return runPrescan() && runParse(); }
+};
+
+static FrontendPluginRegistry::Add<FeatureListAction> X(
+    "feature-list", "List program features");
diff --git a/flang/include/flang/Common/uint128.h b/flang/include/flang/Common/uint128.h
index b12e1729341fe..bfd2eef01f6f0 100644
--- a/flang/include/flang/Common/uint128.h
+++ b/flang/include/flang/Common/uint128.h
@@ -261,7 +261,7 @@ template <bool IS_SIGNED = false> class Int128 {
 using UnsignedInt128 = Int128<false>;
 using SignedInt128 = Int128<true>;
 
-#if !AVOID_NATIVE_UINT128_t && (defined __GNUC__ || defined __clang__) && \
+#if !AVOID_NATIVE_UINT128_T && (defined __GNUC__ || defined __clang__) && \
     defined __SIZEOF_INT128__
 using uint128_t = __uint128_t;
 using int128_t = __int128_t;
diff --git a/flang/include/flang/Frontend/LangOptions.h b/flang/include/flang/Frontend/LangOptions.h
index c8edbfdbc8d1c..7adf2eec9ca3d 100644
--- a/flang/include/flang/Frontend/LangOptions.h
+++ b/flang/include/flang/Frontend/LangOptions.h
@@ -15,6 +15,8 @@
 #ifndef LLVM_FLANG_FRONTEND_LANGOPTIONS_H
 #define LLVM_FLANG_FRONTEND_LANGOPTIONS_H
 
+#include <string>
+
 namespace Fortran::frontend {
 
 /// Bitfields of LangOptions, split out from LangOptions to ensure
@@ -52,6 +54,10 @@ class LangOptions : public LangOptionsBase {
   void set##Name(Type Value) { Name = static_cast<unsigned>(Value); }
 #include "flang/Frontend/LangOptions.def"
 
+  /// Name of the IR file that contains the result of the OpenMP target
+  /// host code generation.
+  std::string OMPHostIRFile;
+
   LangOptions();
 };
 
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index d82999fdb7dd9..0b1e36590f10b 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -27,6 +27,7 @@ namespace hlfir {
 
 class AssociateOp;
 class ElementalOp;
+class ElementalAddrOp;
 class YieldElementOp;
 
 /// Is this an SSA value type for the value of a Fortran procedure
@@ -181,6 +182,11 @@ class Entity : public mlir::Value {
     return base.getDefiningOp<fir::FortranVariableOpInterface>();
   }
 
+  bool isOptional() const {
+    auto varIface = getIfVariableInterface();
+    return varIface ? varIface.isOptional() : false;
+  }
+
   // Get the entity as an mlir SSA value containing all the shape, type
   // parameters and dynamic shape information.
   mlir::Value getBase() const { return *this; }
@@ -315,6 +321,11 @@ llvm::SmallVector<mlir::Value> getIndexExtents(mlir::Location loc,
                                                fir::FirOpBuilder &builder,
                                                mlir::Value shape);
 
+/// Return explicit extents. If the base is a fir.box, this won't read it to
+/// return the extents and will instead return an empty vector.
+llvm::SmallVector<mlir::Value>
+getExplicitExtentsFromShape(mlir::Value shape, fir::FirOpBuilder &builder);
+
 /// Read length parameters into result if this entity has any.
 void genLengthParameters(mlir::Location loc, fir::FirOpBuilder &builder,
                          Entity entity,
@@ -380,6 +391,11 @@ std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>
 convertToBox(mlir::Location loc, fir::FirOpBuilder &builder,
              const hlfir::Entity &entity, mlir::Type targetType);
 
+/// Clone an hlfir.elemental_addr into an hlfir.elemental value.
+hlfir::ElementalOp cloneToElementalOp(mlir::Location loc,
+                                      fir::FirOpBuilder &builder,
+                                      hlfir::ElementalAddrOp elementalAddrOp);
+
 } // namespace hlfir
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_HLFIRTOOLS_H
diff --git a/flang/include/flang/Optimizer/CodeGen/CGPasses.td b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
index c215cb647849d..0014298a27a22 100644
--- a/flang/include/flang/Optimizer/CodeGen/CGPasses.td
+++ b/flang/include/flang/Optimizer/CodeGen/CGPasses.td
@@ -53,7 +53,7 @@ def TargetRewritePass : Pass<"target-rewrite", "mlir::ModuleOp"> {
       representations that may differ based on the target machine.
   }];
   let constructor = "::fir::createFirTargetRewritePass()";
-  let dependentDialects = [ "fir::FIROpsDialect" ];
+  let dependentDialects = [ "fir::FIROpsDialect", "mlir::func::FuncDialect" ];
   let options = [
     Option<"forcedTargetTriple", "target", "std::string", /*default=*/"",
            "Override module's target triple.">,
diff --git a/flang/include/flang/Optimizer/HLFIR/CMakeLists.txt b/flang/include/flang/Optimizer/HLFIR/CMakeLists.txt
index bf90996edf3c3..823918bf8e9f9 100644
--- a/flang/include/flang/Optimizer/HLFIR/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/HLFIR/CMakeLists.txt
@@ -7,6 +7,8 @@ mlir_tablegen(HLFIRAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=hlfir)
 mlir_tablegen(HLFIRAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=hlfir)
 
 set(LLVM_TARGET_DEFINITIONS HLFIROps.td)
+mlir_tablegen(HLFIROpInterfaces.h.inc -gen-op-interface-decls)
+mlir_tablegen(HLFIROpInterfaces.cpp.inc -gen-op-interface-defs)
 mlir_tablegen(HLFIROps.h.inc -gen-op-decls)
 mlir_tablegen(HLFIROps.cpp.inc -gen-op-defs)
 
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.h b/flang/include/flang/Optimizer/HLFIR/HLFIROps.h
index 88f50fd110db5..26bfc9a805dcc 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.h
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.h
@@ -11,6 +11,7 @@
 
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/FortranVariableInterface.h"
 #include "flang/Optimizer/HLFIR/HLFIRDialect.h"
@@ -19,6 +20,7 @@
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
+#include "flang/Optimizer/HLFIR/HLFIROpInterfaces.h.inc"
 #define GET_OP_CLASSES
 #include "flang/Optimizer/HLFIR/HLFIROps.h.inc"
 
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 79bba3dc39e2b..87136197851a0 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -317,6 +317,31 @@ def hlfir_ConcatOp : hlfir_Op<"concat", []> {
   let hasVerifier = 1;
 }
 
+def hlfir_ProductOp : hlfir_Op<"product", [AttrSizedOperandSegments,
+    DeclareOpInterfaceMethods<ArithFastMathInterface>]> {
+  let summary = "PRODUCT transformational intrinsic";
+  let description = [{
+    Multiplies the elements of an array, optionally along a particular dimension,
+    optionally if a mask is true.
+  }];
+
+  let arguments = (ins
+    AnyFortranNumericalArrayObject:$array,
+    Optional<AnyIntegerType>:$dim,
+    Optional<AnyFortranLogicalOrI1ArrayObject>:$mask,
+    DefaultValuedAttr<Arith_FastMathAttr,
+                      "::mlir::arith::FastMathFlags::none">:$fastmath
+  );
+
+  let results = (outs hlfir_ExprType);
+
+  let assemblyFormat = [{
+    $array (`dim` $dim^)? (`mask` $mask^)? attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
 def hlfir_SetLengthOp : hlfir_Op<"set_length", []> {
   let summary = "change the length of a character entity";
   let description = [{
@@ -814,4 +839,468 @@ def hlfir_GetExtentOp : hlfir_Op<"get_extent", [Pure]> {
   let builders = [OpBuilder<(ins "mlir::Value":$shape, "unsigned":$dim)>];
 }
 
+def hlfir_OrderedAssignmentTreeOpInterface : OpInterface<"OrderedAssignmentTreeOpInterface"> {
+  let description = [{
+    Interface for the operations representing Forall and Where constructs and
+    statements as an mlir::Region tree.
+
+    These operations all have in common that they have "leaf" regions that contains
+    some code that should be evaluated for "all active combinations of Forall
+    index-name values" before the next OrderedAssignmentTreeOpInterface is
+    evaluated.
+
+    These operations are ordered in a tree fashion: Some operations, like
+    hlfir.forall or hlfir.where, contain a list of OrderedAssignmentTreeOpInterface
+    that should be evaluated after the "Leaf" regions, and before the next
+    OrderedAssignmentTreeOpInterface.
+
+    Nested OrderedAssignmentTreeOpInterface operations are affected by the
+    OrderedAssignmentTreeOpInterface operations that contain them (e.g:
+    hlfir.region_assign may be masked by the value of the mask region of
+    an hlfir.where that contains it).
+
+    OrderedAssignmentTreeOpInterface operations that contain nested operation
+    must return a "sub-tree" region that contains the list of nested
+    OrderedAssignmentTreeOpInterface operations.
+
+    There is no constraints over what IR a leaf region may contain. There is also
+    no restriction regarding how many leaf regions an
+    OrderedAssignmentTreeOpInterface operation may contain.
+
+    A "sub-tree" region, if any, must contain only OrderedAssignmentTreeOpInterface
+    operations and, maybe, a fir.end terminator.
+  }];
+
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/"Get the OrderedAssignmentTreeOpInterface leaf regions that contain evaluation code",
+      /*retTy=*/"void",
+      /*methodName=*/"getLeafRegions",
+      /*args=*/(ins "llvm::SmallVectorImpl<mlir::Region*>&":$regions),
+      /*methodBody=*/[{}]
+    >,
+    InterfaceMethod<
+      /*desc=*/"Get the region, if any, containing the list of sub-tree OrderedAssignmentTreeOpInterface nodes",
+      /*retTy=*/"mlir::Region*",
+      /*methodName=*/"getSubTreeRegion",
+      /*args=*/(ins),
+      /*methodBody=*/[{}]
+    >,
+  ];
+
+  let extraClassDeclaration = [{
+    /// Interface verifier imlementation.
+    mlir::LogicalResult verifyImpl();
+  }];
+
+  let verify = [{
+    return ::mlir::cast<::hlfir::OrderedAssignmentTreeOpInterface>($_op).verifyImpl();
+  }];
+
+  let cppNamespace = "hlfir";
+}
+
+
+def hlfir_RegionAssignOp : hlfir_Op<"region_assign", [hlfir_OrderedAssignmentTreeOpInterface]> {
+  let summary = "represent a Fortran assignment using regions for the LHS and RHS evaluation";
+  let description = [{
+    This operation can represent Forall and Where assignment when inside an
+    hlfir.forall or hlfir.where "ordered assignment tree". It can
+    also represent user defined assignments and assignment to vector
+    subscripted entities without requiring the materialization of the
+    right-hand side temporary copy that may be needed to implement Fortran
+    assignment semantic.
+
+    The right-hand side and left-hand side evaluations are held in their
+    own regions terminated with hlfir.yield operations (or hlfir.elemental_addr
+    for a left-hand side with vector subscript).
+
+    An optional region may be added to implement user defined assignment.
+    This region provides two block arguments with the same type as the
+    yielded rhs and lhs entities (in that order), or the element type if this
+    is an elemental user defined assignment.
+
+    If this optional region is not provided, intrinsic assignment is performed.
+
+    Example: "X = Y",  where "=" is a user defined elemental assignment "foo"
+    taking Y by value.
+    ```
+    hlfir.region_assign {
+      hlfir.yield %y : !fir.box<!fir.array<?x!f32>>
+    } to {
+      hlfir.yield %x : !fir.box<!fir.array<?x!fir.type<t>>>
+    } user_defined_assignment (%rhs_elt: !fir.ref<f32>) to (%lhs_elt: !fir.ref<!fir.type<t>>) {
+      %0 = fir.load %rhs_elt : !fir.ref<f32>
+      fir.call @foo(%lhs_elt, %0) : (!fir.ref<!fir.type<t>>, f32) -> ()
+    }
+    ```
+
+    TODO: add optional "realloc" semantics like for hlfir.assign.
+  }];
+
+  let regions = (region  SizedRegion<1>:$rhs_region,
+                         SizedRegion<1>:$lhs_region,
+                         MaxSizedRegion<1>:$user_defined_assignment);
+
+  let extraClassDeclaration = [{
+    mlir::Value getUserAssignmentRhs() {
+      return getUserDefinedAssignment().getArguments()[0];
+    }
+    mlir::Value getUserAssignmentLhs() {
+      return getUserDefinedAssignment().getArguments()[1];
+    }
+    void getLeafRegions(llvm::SmallVectorImpl<mlir::Region*>& regions) {
+      regions.push_back(&getRhsRegion());
+      regions.push_back(&getLhsRegion());
+      if (!getUserDefinedAssignment().empty())
+        regions.push_back(&getUserDefinedAssignment());
+    }
+    mlir::Region* getSubTreeRegion() { return nullptr; }
+
+  }];
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+}
+
+def hlfir_YieldOp : hlfir_Op<"yield", [Terminator, ParentOneOf<["RegionAssignOp",
+    "ElementalAddrOp", "ForallOp", "ForallMaskOp", "WhereOp", "ElseWhereOp"]>,
+    SingleBlockImplicitTerminator<"fir::FirEndOp">]> {
+
+  let summary = "Yield a value or variable inside a forall, where or region assignment";
+
+  let description = [{
+    Terminator operation that yields an HLFIR value or variable that was computed in
+    a region and hold the yielded entity cleanup, if any, into its own region.
+    This allows representing any Fortran expression evaluation in its own region so
+    that the evaluation can easily be scheduled/moved around in a pass.
+
+    Example: "foo(x)" where foo returns an allocatable array.
+    ```
+    {
+      // In some region.
+      %0 = fir.call @foo(x) (!fir.ref<f32>) -> !fir.box<fir.heap<!fir.array<?xf32>>>
+      hlfir.yield %0 : !fir.box<!fir.heap<!fir.array<?xf32>>> cleanup {
+        %1 = fir.box_addr %0 : !fir.box<!fir.heap<!fir.array<?xf32>>> -> !fir.heap<!fir.array<?xf32>>
+        %fir.freemem %1 : !fir.heap<!fir.array<?xf32>>
+      }
+    }
+    ```
+  }];
+
+  let arguments = (ins AnyFortranEntity:$entity);
+  let regions = (region  MaxSizedRegion<1>:$cleanup);
+
+  let assemblyFormat = "$entity attr-dict `:` type($entity) custom<YieldOpCleanup>($cleanup)";
+}
+
+def hlfir_ElementalAddrOp : hlfir_Op<"elemental_addr", [Terminator, HasParent<"RegionAssignOp">, RecursiveMemoryEffects, RecursivelySpeculatable]> {
+  let summary = "Yield the address of a vector subscripted variable inside an hlfir.region_assign";
+  let description = [{
+    Special terminator node for the left-hand side region of an hlfir.region_assign
+    to a vector subscripted entity.
+
+    It represents how the address of an element of such entity is computed given
+    one based indices.
+
+    It is very similar to hlfir.elemental, except that it does not produce an SSA
+    value because there is no hlfir type to describe a vector subscripted entity
+    (the codegen of such type would be problematic). Hence, it is tightly linked
+    to an hlfir.region_assign by its terminator property.
+
+    An optional cleanup region may be provided if any of the subscript expressions
+    of the designator require a cleanup.
+    This allows documenting cleanups that cannot be generated after the vector
+    subscripted designator usage (that has not been materizaled yet). The cleanups
+    will be evaluated after the assignment once the related
+    hlfir.region_assign is lowered.
+
+    Example: "X(VECTOR) = Y"
+
+    ```
+    hlfir.region_assign {
+      hlfir.yield %y : !fir.ref<!fir.array<20xf32>>
+    } to {
+      hlfir.elemental_addr %vector_shape  : !fir.shape<1> {
+        ^bb0(%i: index):
+        %0 = hlfir.designate %vector (%i)  : (!fir.ref<!fir.array<20xi32>>, index) -> !fir.ref<i32>
+        %1 = fir.load %0 : !fir.ref<i32>
+        %x_element_addr = hlfir.designate %x (%1)  : (!fir.ref<!fir.array<100xf32>>, i32) -> !fir.ref<f32>
+        hlfir.yield %x_element_addr : !fir.ref<f32>
+      }
+    }
+    ```
+  }];
+
+  let arguments = (ins
+    fir_ShapeType:$shape,
+    Variadic<AnyIntegerType>:$typeparams);
+
+  let regions = (region  SizedRegion<1>:$body,
+                         MaxSizedRegion<1>:$cleanup);
+
+  let builders = [
+    OpBuilder<(ins "mlir::Value":$shape)>
+  ];
+
+  let assemblyFormat = [{
+    $shape (`typeparams` $typeparams^)?
+    attr-dict `:` type(operands) $body
+    custom<YieldOpCleanup>($cleanup)}];
+
+  let extraClassDeclaration = [{
+    mlir::Region::BlockArgListType getIndices() {
+      return getBody().getArguments();
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+/// Define ODS constraints to verify that a region ends with a yield of a
+/// certain type.
+def YieldIntegerOrEmpty : CPred<"yieldsIntegerOrEmpty($_self)">;
+def YieldIntegerRegion : RegionConstraint<
+  And<[SizedRegion<1>.predicate, YieldIntegerOrEmpty]>,
+  "single block region that yields an integer scalar value">;
+def MaybeYieldIntegerRegion : RegionConstraint<
+  And<[MaxSizedRegion<1>.predicate, YieldIntegerOrEmpty]>,
+  "optional single block region that yields an integer scalar value">;
+
+def hlfir_ForallOp : hlfir_Op<"forall", [hlfir_OrderedAssignmentTreeOpInterface]> {
+  let summary = "represent a Fortran forall";
+  let description = [{
+    This operation allows representing Fortran forall. It computes
+    a set of "index-name" values based on lower bound, upper bound,
+    and step values whose evaluations are represented in their own
+    regions.
+
+    Operations nested in its body region are evaluated in order.
+    As opposed to a regular loop, each nested operation is
+    fully evaluated for all the values in the "active set of
+    index-name" before the next nested operation. In practice, the
+    nested operation evaluation may be fused if it is proven that
+    they do not have data dependency.
+
+    The "index-name" value is represented as the argument of the
+    body region.
+
+    The lower, upper, and step region (if provided), must be terminated
+    by hlfir.yield that yields scalar integers.
+
+    The body region must only contain other OrderedAssignmentTreeOpInterface
+    operations (like hlfir.region_assign, or other hlfir.forall).
+
+    A Fortran forall with several indices is represented as a nest
+    of hlfir.forall.
+
+    Example: FORALL(I=1:10) X(I) = FOO(I)
+    ```
+      hlfir.forall lb {
+        hlfir.yield %c1 : index
+      } ub {
+        hlfir.yield %c10 : index
+      } (%i : index) {
+        hlfir.region_assign {
+          %res = fir.call @foo(%i) : (index) -> f32
+          hlfir.yield %res : f32
+        } to {
+          %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+          hlfir.yield %xi : !fir.ref<f32>
+        }
+      }
+    ```
+
+  }];
+
+  let regions = (region  YieldIntegerRegion:$lb_region,
+                         YieldIntegerRegion:$ub_region,
+                         MaybeYieldIntegerRegion:$step_region,
+                         SizedRegion<1>:$body);
+
+  let extraClassDeclaration = [{
+    mlir::Value getForallIndexValue() {
+      return getBody().getArguments()[0];
+    }
+    void getLeafRegions(llvm::SmallVectorImpl<mlir::Region*>& regions) {
+      regions.push_back(&getLbRegion());
+      regions.push_back(&getUbRegion());
+      if (!getStepRegion().empty())
+        regions.push_back(&getStepRegion());
+    }
+    mlir::Region* getSubTreeRegion() { return &getBody(); }
+  }];
+
+  let assemblyFormat = [{
+    attr-dict `lb` $lb_region
+    `ub` $ub_region
+    (`step` $step_region^)?
+    custom<ForallOpBody>($body)
+  }];
+}
+
+/// Shared definition for hlfir.forall_mask and hlfir.where
+/// that have the same structure and assembly format, but not the same
+/// constraints.
+class hlfir_AssignmentMaskOp<string mnemonic> : hlfir_Op<mnemonic,
+    [hlfir_OrderedAssignmentTreeOpInterface]> {
+  let regions = (region  SizedRegion<1>:$mask_region,
+                         SizedRegion<1>:$body);
+
+  let extraClassDeclaration = [{
+    void getLeafRegions(llvm::SmallVectorImpl<mlir::Region*>& regions) {
+      regions.push_back(&getMaskRegion());
+    }
+    mlir::Region* getSubTreeRegion() { return &getBody(); }
+  }];
+
+  let assemblyFormat = [{
+    $mask_region
+    attr-dict `do`
+    custom<AssignmentMaskOpBody>($body)
+  }];
+}
+
+def hlfir_ForallMaskOp : hlfir_AssignmentMaskOp<"forall_mask"> {
+  let summary = "Represent a Fortran forall mask";
+  let description = [{
+    Fortran Forall can have a scalar mask expression that depends on the
+    Forall index-name value.
+    hlfir.forall_mask allows representing this mask. The expression
+    evaluation is held in the mask region that must yield an i1 scalar
+    value.
+    An hlfir.forall_mask must be directly nested in the body region of
+    an hlfir.forall. It is a separate operation so that it can use the
+    index SSA value defined by the hlfir.forall body region.
+
+    Example: "FORALL(I=1:10, SOME_CONDITION(I)) X(I) = FOO(I)"
+    ```
+    hlfir.forall lb {
+      hlfir.yield %c1 : index
+    } ub {
+      hlfir.yield %c10 : index
+    } (%i : index) {
+      hlfir.forall_mask {
+        %mask = fir.call @some_condition(%i) : (index) -> i1
+        hlfir.yield %mask : i1
+      } do {
+        hlfir.region_assign {
+          %res = fir.call @foo(%i) : (index) -> f32
+          hlfir.yield %res : f32
+        } to {
+          %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+          hlfir.yield %xi : !fir.ref<f32>
+        }
+      }
+    }
+    ```
+  }];
+  let hasVerifier = 1;
+}
+
+def hlfir_WhereOp : hlfir_AssignmentMaskOp<"where"> {
+  let summary = "Represent a Fortran where construct or statement";
+  let description = [{
+    Represent Fortran "where" construct or statement. The mask
+    expression evaluation is held in the mask region that must yield
+    logical array that has the same shape as all the nested
+    hlfir.region_assign left-hand sides, and all the nested hlfir.where
+    or hlfir.elsewhere masks.
+
+    The values of the where and elsewhere masks form a control mask that
+    controls all the nested hlfir.region_assign: only the array element for
+    which the related control mask value is true are assigned. Any right-hand
+    side elemental expression is only evaluated for elements where the control
+    mask is true. See Fortran standard 2018 section 10.2.3 for more detailed
+    about the control mask semantic.
+
+    An hlfir.where must not contain any hlfir.forall but it may be contained
+    in such operation. This matches Fortran rules.
+  }];
+  let hasVerifier = 1;
+}
+
+def hlfir_ElseWhereOp : hlfir_Op<"elsewhere", [Terminator,
+    ParentOneOf<["WhereOp", "ElseWhereOp"]>, hlfir_OrderedAssignmentTreeOpInterface]> {
+  let summary = "Represent a Fortran elsewhere statement";
+
+  let description = [{
+    Represent Fortran "elsewhere" construct or statement.
+
+    It has an optional mask region to hold the evaluation of Fortran
+    optional elsewhere mask expressions. If this region is provided,
+    it must satisfy the same constraints as hlfir.where mask region.
+
+    An hlfir.elsewhere must be the last operation of an hlfir.where or,
+    hlfir.elsewhere body, which is enforced by its terminator property.
+
+    Like in Fortran, an hlfir.elsewhere negate the current control mask,
+    and if provided, adds the mask the resulting control mask (with a logical
+    AND).
+  }];
+
+  let regions = (region  MaxSizedRegion<1>:$mask_region,
+                         SizedRegion<1>:$body);
+
+  let extraClassDeclaration = [{
+    void getLeafRegions(llvm::SmallVectorImpl<mlir::Region*>& regions) {
+      if (!getMaskRegion().empty())
+        regions.push_back(&getMaskRegion());
+    }
+    mlir::Region* getSubTreeRegion() { return &getBody(); }
+  }];
+
+  let assemblyFormat = [{
+    (`mask` $mask_region^)?
+    attr-dict `do`
+    custom<AssignmentMaskOpBody>($body)
+  }];
+  let hasVerifier = 1;
+}
+
+def hlfir_ForallIndexOp : hlfir_Op<"forall_index", [fir_FortranVariableOpInterface,
+    hlfir_OrderedAssignmentTreeOpInterface, Pure]> {
+  let summary = "represent a Fortran forall index declaration";
+  let description = [{
+    This operation allows placing an hlfir.forall index in memory with
+    the related Fortran index-value name and type.
+
+    So far, lowering needs to manipulate symbols as memory entities.
+    This operation allows fulfilling this requirements without allowing
+    bare alloca/declare/store inside the body of hlfir.forall, which would
+    make their analysis more complex.
+
+    Given Forall index-value cannot be modified it also allows defining
+    a canonicalization of all its loads into a fir.convert of the
+    hlfir.forall index, which helps simplifying the data dependency analysis
+    of hlfir.forall.
+  }];
+
+  let arguments = (ins AnyIntegerType:$index,
+                       Builtin_StringAttr:$name);
+
+  let results = (outs AnyFortranVariable);
+
+  let assemblyFormat = [{
+    $name $index attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    /// Implement FortranVariableInterface interface.
+    std::optional<fir::FortranVariableFlagsEnum> getFortranAttrs() const {
+      return std::nullopt;
+    }
+    mlir::Value getShape() const {return mlir::Value{};}
+    mlir::OperandRange getExplicitTypeParams() const {
+      // Return an empty range.
+      return {(*this)->getOperands().begin(), (*this)->getOperands().begin()};
+    }
+    /// Implement OrderedAssignmentTreeOpInterface interface.
+    void getLeafRegions(llvm::SmallVectorImpl<mlir::Region*>& regions) {}
+    mlir::Region* getSubTreeRegion() { return nullptr; }
+  }];
+
+  let hasCanonicalizeMethod = 1;
+}
+
 #endif // FORTRAN_DIALECT_HLFIR_OPS
diff --git a/flang/include/flang/Optimizer/HLFIR/Passes.h b/flang/include/flang/Optimizer/HLFIR/Passes.h
index 2d9ac0c8e64b6..a5aa35b92ae59 100644
--- a/flang/include/flang/Optimizer/HLFIR/Passes.h
+++ b/flang/include/flang/Optimizer/HLFIR/Passes.h
@@ -13,6 +13,7 @@
 #ifndef FORTRAN_OPTIMIZER_HLFIR_PASSES_H
 #define FORTRAN_OPTIMIZER_HLFIR_PASSES_H
 
+#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassRegistry.h"
 #include <memory>
@@ -24,6 +25,8 @@ namespace hlfir {
 std::unique_ptr<mlir::Pass> createConvertHLFIRtoFIRPass();
 std::unique_ptr<mlir::Pass> createBufferizeHLFIRPass();
 std::unique_ptr<mlir::Pass> createLowerHLFIRIntrinsicsPass();
+std::unique_ptr<mlir::Pass> createSimplifyHLFIRIntrinsicsPass();
+std::unique_ptr<mlir::Pass> createLowerHLFIROrderedAssignmentsPass();
 
 #define GEN_PASS_REGISTRATION
 #include "flang/Optimizer/HLFIR/Passes.h.inc"
diff --git a/flang/include/flang/Optimizer/HLFIR/Passes.td b/flang/include/flang/Optimizer/HLFIR/Passes.td
index faf5cd99d8ac3..e2cf8447dff47 100644
--- a/flang/include/flang/Optimizer/HLFIR/Passes.td
+++ b/flang/include/flang/Optimizer/HLFIR/Passes.td
@@ -25,4 +25,14 @@ def LowerHLFIRIntrinsics : Pass<"lower-hlfir-intrinsics", "::mlir::ModuleOp"> {
   let constructor = "hlfir::createLowerHLFIRIntrinsicsPass()";
 }
 
+def LowerHLFIROrderedAssignments : Pass<"lower-hlfir-ordered-assignments", "::mlir::ModuleOp"> {
+  let summary = "Lower HLFIR ordered assignments like forall and where operations";
+  let constructor = "hlfir::createLowerHLFIROrderedAssignmentsPass()";
+}
+
+def SimplifyHLFIRIntrinsics : Pass<"simplify-hlfir-intrinsics", "::mlir::func::FuncOp"> {
+  let summary = "Simplify HLFIR intrinsic operations that don't need to result in runtime calls";
+  let constructor = "hlfir::createSimplifyHLFIRIntrinsicsPass()";
+}
+
 #endif //FORTRAN_DIALECT_HLFIR_PASSES
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index ddeb73f09af15..41a03a2b6a949 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -560,6 +560,7 @@ class Symbol {
       OmpShared, OmpPrivate, OmpLinear, OmpFirstPrivate, OmpLastPrivate,
       // OpenMP data-mapping attribute
       OmpMapTo, OmpMapFrom, OmpMapAlloc, OmpMapRelease, OmpMapDelete,
+      OmpUseDevicePtr,
       // OpenMP data-copying attribute
       OmpCopyIn, OmpCopyPrivate,
       // OpenMP miscellaneous flags
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index 037741e18d407..a799427f023e4 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -238,8 +238,11 @@ inline void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
 ///   passes pipeline
 inline void createHLFIRToFIRPassPipeline(
     mlir::PassManager &pm, llvm::OptimizationLevel optLevel = defaultOptLevel) {
-  if (optLevel.isOptimizingForSpeed())
+  if (optLevel.isOptimizingForSpeed()) {
     addCanonicalizerPassWithoutRegionSimplification(pm);
+    pm.addPass(hlfir::createSimplifyHLFIRIntrinsicsPass());
+  }
+  pm.addPass(hlfir::createLowerHLFIROrderedAssignmentsPass());
   pm.addPass(hlfir::createLowerHLFIRIntrinsicsPass());
   pm.addPass(hlfir::createBufferizeHLFIRPass());
   pm.addPass(hlfir::createConvertHLFIRtoFIRPass());
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 19d31dffaac20..a28b45ca27612 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -23,13 +23,14 @@ struct OffloadModuleOpts {
   OffloadModuleOpts() {}
   OffloadModuleOpts(uint32_t OpenMPTargetDebug, bool OpenMPTeamSubscription,
       bool OpenMPThreadSubscription, bool OpenMPNoThreadState,
-      bool OpenMPNoNestedParallelism, bool OpenMPIsDevice)
+      bool OpenMPNoNestedParallelism, bool OpenMPIsDevice,
+      std::string OMPHostIRFile = {})
       : OpenMPTargetDebug(OpenMPTargetDebug),
         OpenMPTeamSubscription(OpenMPTeamSubscription),
         OpenMPThreadSubscription(OpenMPThreadSubscription),
         OpenMPNoThreadState(OpenMPNoThreadState),
         OpenMPNoNestedParallelism(OpenMPNoNestedParallelism),
-        OpenMPIsDevice(OpenMPIsDevice) {}
+        OpenMPIsDevice(OpenMPIsDevice), OMPHostIRFile(OMPHostIRFile) {}
 
   OffloadModuleOpts(Fortran::frontend::LangOptions &Opts)
       : OpenMPTargetDebug(Opts.OpenMPTargetDebug),
@@ -37,7 +38,8 @@ struct OffloadModuleOpts {
         OpenMPThreadSubscription(Opts.OpenMPThreadSubscription),
         OpenMPNoThreadState(Opts.OpenMPNoThreadState),
         OpenMPNoNestedParallelism(Opts.OpenMPNoNestedParallelism),
-        OpenMPIsDevice(Opts.OpenMPIsDevice) {}
+        OpenMPIsDevice(Opts.OpenMPIsDevice), OMPHostIRFile(Opts.OMPHostIRFile) {
+  }
 
   uint32_t OpenMPTargetDebug = 0;
   bool OpenMPTeamSubscription = false;
@@ -45,6 +47,7 @@ struct OffloadModuleOpts {
   bool OpenMPNoThreadState = false;
   bool OpenMPNoNestedParallelism = false;
   bool OpenMPIsDevice = false;
+  std::string OMPHostIRFile = {};
 };
 
 //  Shares assinging of the OpenMP OffloadModuleInterface and its assorted
@@ -59,6 +62,9 @@ void setOffloadModuleInterfaceAttributes(
       offloadMod.setFlags(Opts.OpenMPTargetDebug, Opts.OpenMPTeamSubscription,
           Opts.OpenMPThreadSubscription, Opts.OpenMPNoThreadState,
           Opts.OpenMPNoNestedParallelism);
+
+      if (!Opts.OMPHostIRFile.empty())
+        offloadMod.setHostIRFilePath(Opts.OMPHostIRFile);
     }
   }
 }
diff --git a/flang/lib/Decimal/CMakeLists.txt b/flang/lib/Decimal/CMakeLists.txt
index 5c5a498f1a3ca..a81d329b4a5de 100644
--- a/flang/lib/Decimal/CMakeLists.txt
+++ b/flang/lib/Decimal/CMakeLists.txt
@@ -11,6 +11,7 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
 
   set(LLVM_COMMON_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../cmake")
   set(LLVM_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../llvm/cmake")
+  set(CLANG_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../clang/cmake")
 
   # Add path for custom modules
   list(INSERT CMAKE_MODULE_PATH 0
@@ -20,8 +21,10 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     "${LLVM_COMMON_CMAKE_UTILS}/Modules"
     "${LLVM_CMAKE_UTILS}"
     "${LLVM_CMAKE_UTILS}/modules"
+    "${CLANG_CMAKE_UTILS}/modules"
     )
 
+  include(AddClang)
   include(AddLLVM)
   include(AddFlang)
   include(HandleLLVMOptions)
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index c6d5152c38d17..6672777f34377 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -726,6 +726,16 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
     if (args.hasArg(clang::driver::options::OPT_fopenmp_is_device)) {
       res.getLangOpts().OpenMPIsDevice = 1;
 
+      // Get OpenMP host file path if any and report if a non existent file is
+      // found
+      if (auto *arg = args.getLastArg(
+              clang::driver::options::OPT_fopenmp_host_ir_file_path)) {
+        res.getLangOpts().OMPHostIRFile = arg->getValue();
+        if (!llvm::sys::fs::exists(res.getLangOpts().OMPHostIRFile))
+          diags.Report(clang::diag::err_drv_omp_host_ir_file_not_found)
+              << res.getLangOpts().OMPHostIRFile;
+      }
+
       if (args.hasFlag(
               clang::driver::options::OPT_fopenmp_assume_teams_oversubscription,
               clang::driver::options::
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 15e0d0528e2c6..fdc6649972471 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -2983,6 +2983,42 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     }
   }
 
+  /// Given converted LHS and RHS of the assignment, generate
+  /// explicit type conversion for implicit Logical<->Integer
+  /// conversion. Return Value representing the converted RHS,
+  /// if the implicit Logical<->Integer is detected, otherwise,
+  /// return nullptr. The caller is responsible for inserting
+  /// DestroyOp in case the returned value has hlfir::ExprType.
+  mlir::Value
+  genImplicitLogicalConvert(const Fortran::evaluate::Assignment &assign,
+                            hlfir::Entity lhs, hlfir::Entity rhs) {
+    mlir::Type fromTy = rhs.getFortranElementType();
+    mlir::Type toTy = lhs.getFortranElementType();
+    if (fromTy == toTy)
+      return nullptr;
+
+    if (!fromTy.isa<mlir::IntegerType, fir::LogicalType>())
+      return nullptr;
+    if (!toTy.isa<mlir::IntegerType, fir::LogicalType>())
+      return nullptr;
+
+    mlir::Location loc = toLocation();
+    auto &builder = getFirOpBuilder();
+    if (assign.rhs.Rank() == 0)
+      return builder.createConvert(loc, toTy, rhs);
+
+    mlir::Value shape = hlfir::genShape(loc, builder, rhs);
+    auto genKernel =
+        [&rhs, &toTy](mlir::Location loc, fir::FirOpBuilder &builder,
+                      mlir::ValueRange oneBasedIndices) -> hlfir::Entity {
+      auto elementPtr = hlfir::getElementAt(loc, builder, rhs, oneBasedIndices);
+      auto val = hlfir::loadTrivialScalar(loc, builder, elementPtr);
+      return hlfir::EntityWithAttributes{builder.createConvert(loc, toTy, val)};
+    };
+    return hlfir::genElementalOp(loc, builder, toTy, shape, /*typeParams=*/{},
+                                 genKernel);
+  }
+
   /// Shared for both assignments and pointer assignments.
   void genAssignment(const Fortran::evaluate::Assignment &assign) {
     mlir::Location loc = toLocation();
@@ -2994,6 +3030,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
           Fortran::common::visitors{
               // [1] Plain old assignment.
               [&](const Fortran::evaluate::Assignment::Intrinsic &) {
+                if (Fortran::evaluate::HasVectorSubscript(assign.lhs))
+                  TODO(loc, "assignment to vector subscripted entity");
                 Fortran::lower::StatementContext stmtCtx;
                 hlfir::Entity rhs = Fortran::lower::convertExprToHLFIR(
                     loc, *this, assign.rhs, localSymbols, stmtCtx);
@@ -3018,9 +3056,24 @@ class FirConverter : public Fortran::lower::AbstractConverter {
                   // Dereference pointer LHS: the target is being assigned to.
                   lhs = hlfir::derefPointersAndAllocatables(loc, builder, lhs);
                 }
+
+                // Logical<->Integer assignments are allowed as an extension,
+                // but there is no explicit Convert expression for the RHS.
+                // Recognize the type mismatch here and insert explicit
+                // scalar convert or ElementalOp for array assignment.
+                mlir::Value logicalConvert =
+                    genImplicitLogicalConvert(assign, lhs, rhs);
+                if (logicalConvert)
+                  rhs = hlfir::EntityWithAttributes{logicalConvert};
+
                 builder.create<hlfir::AssignOp>(
                     loc, rhs, lhs, isWholeAllocatableAssignment,
                     keepLhsLengthInAllocatableAssignment);
+
+                // Mark the end of life range of the ElementalOp's result.
+                if (logicalConvert &&
+                    logicalConvert.getType().isa<hlfir::ExprType>())
+                  builder.create<hlfir::DestroyOp>(loc, rhs);
               },
               // [2] User defined assignment. If the context is a scalar
               // expression then call the procedure.
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 39a8be38cb297..35e34511d5425 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -71,11 +71,9 @@ static fir::ExtendedValue toExtendedValue(mlir::Location loc, mlir::Value base,
 /// Lower a type(C_PTR/C_FUNPTR) argument with VALUE attribute into a
 /// reference. A C pointer can correspond to a Fortran dummy argument of type
 /// C_PTR with the VALUE attribute. (see 18.3.6 note 3).
-static mlir::Value
-genRecordCPtrValueArg(Fortran::lower::AbstractConverter &converter,
-                      mlir::Value rec, mlir::Type ty) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  mlir::Location loc = converter.getCurrentLocation();
+static mlir::Value genRecordCPtrValueArg(fir::FirOpBuilder &builder,
+                                         mlir::Location loc, mlir::Value rec,
+                                         mlir::Type ty) {
   mlir::Value cAddr = fir::factory::genCPtrOrCFunptrAddr(builder, loc, rec, ty);
   mlir::Value cVal = builder.create<fir::LoadOp>(loc, cAddr);
   return builder.createConvert(loc, cAddr.getType(), cVal);
@@ -354,7 +352,7 @@ fir::ExtendedValue Fortran::lower::genCallOpAndResult(
       mlir::Type fromTy = fir::unwrapRefType(fst.getType());
       if (fir::isa_builtin_cptr_type(fromTy) &&
           Fortran::lower::isCPtrArgByValueType(snd)) {
-        cast = genRecordCPtrValueArg(converter, fst, fromTy);
+        cast = genRecordCPtrValueArg(builder, loc, fst, fromTy);
       } else if (fir::isa_derived(snd)) {
         // FIXME: This seems like a serious bug elsewhere in lowering. Paper
         // over the problem for now.
@@ -856,7 +854,10 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     const PreparedActualArgument &preparedActual, mlir::Type dummyType,
     const Fortran::lower::CallerInterface::PassedEntity &arg,
     const Fortran::lower::SomeExpr &expr,
-    Fortran::evaluate::FoldingContext &foldingContext) {
+    Fortran::lower::AbstractConverter &converter) {
+
+  Fortran::evaluate::FoldingContext &foldingContext =
+      converter.getFoldingContext();
 
   // Step 1: get the actual argument, which includes addressing the
   // element if this is an array in an elemental call.
@@ -933,8 +934,9 @@ static PreparedDummyArgument preparePresentUserCallActualArgument(
     if (mustSetDynamicTypeToDummyType)
       TODO(loc, "passing polymorphic array expression to non polymorphic "
                 "contiguous dummy");
+    mlir::Type storageType = converter.genType(expr);
     hlfir::AssociateOp associate = hlfir::genAssociateExpr(
-        loc, builder, entity, dummyType, "adapt.valuebyref");
+        loc, builder, entity, storageType, "adapt.valuebyref");
     entity = hlfir::Entity{associate.getBase()};
     preparedDummy.setExprAssociateCleanUp(associate.getFirBase(),
                                           associate.getMustFreeStrorageFlag());
@@ -985,10 +987,10 @@ static PreparedDummyArgument prepareUserCallActualArgument(
     const PreparedActualArgument &preparedActual, mlir::Type dummyType,
     const Fortran::lower::CallerInterface::PassedEntity &arg,
     const Fortran::lower::SomeExpr &expr,
-    Fortran::evaluate::FoldingContext &foldingContext) {
+    Fortran::lower::AbstractConverter &converter) {
   if (!preparedActual.handleDynamicOptional())
     return preparePresentUserCallActualArgument(
-        loc, builder, preparedActual, dummyType, arg, expr, foldingContext);
+        loc, builder, preparedActual, dummyType, arg, expr, converter);
 
   // Conditional dummy argument preparation. The actual may be absent
   // at runtime, causing any addressing, copy, and packaging to have
@@ -1009,8 +1011,8 @@ static PreparedDummyArgument prepareUserCallActualArgument(
   mlir::Block *preparationBlock = &badIfOp.getThenRegion().front();
   builder.setInsertionPointToStart(preparationBlock);
   PreparedDummyArgument unconditionalDummy =
-      preparePresentUserCallActualArgument(
-          loc, builder, preparedActual, dummyType, arg, expr, foldingContext);
+      preparePresentUserCallActualArgument(loc, builder, preparedActual,
+                                           dummyType, arg, expr, converter);
   builder.restoreInsertionPoint(insertPt);
 
   // TODO: when forwarding an optional to an optional of the same kind
@@ -1077,9 +1079,24 @@ genUserCall(PreparedActualArguments &loweredActuals,
       // True pass-by-value semantics.
       assert(!preparedActual->handleDynamicOptional() && "cannot be optional");
       hlfir::Entity actual = preparedActual->getActual(loc, builder);
-      auto value = hlfir::loadTrivialScalar(loc, builder, actual);
-      if (!value.isValue())
-        TODO(loc, "Passing CPTR an CFUNCTPTR VALUE in HLFIR");
+      hlfir::Entity value = hlfir::loadTrivialScalar(loc, builder, actual);
+
+      mlir::Type eleTy = value.getFortranElementType();
+      if (fir::isa_builtin_cptr_type(eleTy)) {
+        // Pass-by-value argument of type(C_PTR/C_FUNPTR).
+        // Load the __address component and pass it by value.
+        if (value.isValue()) {
+          auto associate = hlfir::genAssociateExpr(loc, builder, value, eleTy,
+                                                   "adapt.cptrbyval");
+          value = hlfir::Entity{genRecordCPtrValueArg(
+              builder, loc, associate.getFirBase(), eleTy)};
+          builder.create<hlfir::EndAssociateOp>(loc, associate);
+        } else {
+          value =
+              hlfir::Entity{genRecordCPtrValueArg(builder, loc, value, eleTy)};
+        }
+      }
+
       caller.placeInput(arg, builder.createConvert(loc, argTy, value));
     } break;
     case PassBy::BaseAddressValueAttribute:
@@ -1087,9 +1104,9 @@ genUserCall(PreparedActualArguments &loweredActuals,
     case PassBy::Box:
     case PassBy::BaseAddress:
     case PassBy::BoxChar: {
-      PreparedDummyArgument preparedDummy = prepareUserCallActualArgument(
-          loc, builder, *preparedActual, argTy, arg, *expr,
-          callContext.converter.getFoldingContext());
+      PreparedDummyArgument preparedDummy =
+          prepareUserCallActualArgument(loc, builder, *preparedActual, argTy,
+                                        arg, *expr, callContext.converter);
       if (preparedDummy.maybeCleanUp.has_value())
         callCleanUps.emplace_back(std::move(*preparedDummy.maybeCleanUp));
       caller.placeInput(arg, preparedDummy.dummy);
@@ -1247,8 +1264,10 @@ genIntrinsicRefCore(PreparedActualArguments &loweredActuals,
   hlfir::EntityWithAttributes resultEntity = extendedValueToHlfirEntity(
       loc, builder, resultExv, ".tmp.intrinsic_result");
   // Move result into memory into an hlfir.expr since they are immutable from
-  // that point, and the result storage is some temp.
-  if (resultEntity.isVariable()) {
+  // that point, and the result storage is some temp. "Null" is special: it
+  // returns a null pointer variable that should not be transformed into a value
+  // (what matters is the memory address).
+  if (resultEntity.isVariable() && intrinsicName != "null") {
     hlfir::AsExprOp asExpr;
     // Character/Derived MERGE lowering returns one of its argument address
     // (this is the only intrinsic implemented in that way so far). The
@@ -1319,20 +1338,49 @@ genHLFIRIntrinsicRefCore(PreparedActualArguments &loweredActuals,
     return hlfir::ExprType::get(builder.getContext(), resultShape, elementType,
                                 /*polymorphic=*/false);
   };
-  const std::string intrinsicName = callContext.getProcedureName();
-  if (intrinsicName == "sum") {
+
+  auto buildSumOperation = [](fir::FirOpBuilder &builder, mlir::Location loc,
+                              mlir::Type resultTy, mlir::Value array,
+                              mlir::Value dim, mlir::Value mask) {
+    return builder.create<hlfir::SumOp>(loc, resultTy, array, dim, mask);
+  };
+
+  auto buildProductOperation = [](fir::FirOpBuilder &builder,
+                                  mlir::Location loc, mlir::Type resultTy,
+                                  mlir::Value array, mlir::Value dim,
+                                  mlir::Value mask) {
+    return builder.create<hlfir::ProductOp>(loc, resultTy, array, dim, mask);
+  };
+
+  auto buildReductionIntrinsic =
+      [&](PreparedActualArguments &loweredActuals, mlir::Location loc,
+          fir::FirOpBuilder &builder, CallContext &callContext,
+          std::function<mlir::Operation *(fir::FirOpBuilder &, mlir::Location,
+                                          mlir::Type, mlir::Value, mlir::Value,
+                                          mlir::Value)>
+              buildFunc) -> std::optional<hlfir::EntityWithAttributes> {
+    // shared logic for building the product and sum operations
     llvm::SmallVector<mlir::Value> operands = getOperandVector(loweredActuals);
     assert(operands.size() == 3);
+    // dim, mask can be NULL if these arguments were not given
     mlir::Value array = operands[0];
     mlir::Value dim = operands[1];
     if (dim)
       dim = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{dim});
     mlir::Value mask = operands[2];
     mlir::Type resultTy = computeResultType(array, *callContext.resultType);
-    // dim, mask can be NULL if these arguments were not given
-    hlfir::SumOp sumOp =
-        builder.create<hlfir::SumOp>(loc, resultTy, array, dim, mask);
-    return {hlfir::EntityWithAttributes{sumOp.getResult()}};
+    auto *intrinsicOp = buildFunc(builder, loc, resultTy, array, dim, mask);
+    return {hlfir::EntityWithAttributes{intrinsicOp->getResult(0)}};
+  };
+
+  const std::string intrinsicName = callContext.getProcedureName();
+  if (intrinsicName == "sum") {
+    return buildReductionIntrinsic(loweredActuals, loc, builder, callContext,
+                                   buildSumOperation);
+  }
+  if (intrinsicName == "product") {
+    return buildReductionIntrinsic(loweredActuals, loc, builder, callContext,
+                                   buildProductOperation);
   }
   if (intrinsicName == "matmul") {
     llvm::SmallVector<mlir::Value> operands = getOperandVector(loweredActuals);
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 2dd61bb15833d..87c0099814fa8 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -35,6 +35,18 @@ namespace {
 
 /// Lower Designators to HLFIR.
 class HlfirDesignatorBuilder {
+private:
+  /// Internal entry point on the rightest part of a evaluate::Designator.
+  template <typename T>
+  hlfir::EntityWithAttributes
+  genLeafPartRef(const T &designatorNode,
+                 bool vectorSubscriptDesignatorToValue) {
+    hlfir::EntityWithAttributes result = gen(designatorNode);
+    if (vectorSubscriptDesignatorToValue)
+      return turnVectorSubscriptedDesignatorIntoValue(result);
+    return result;
+  }
+
 public:
   HlfirDesignatorBuilder(mlir::Location loc,
                          Fortran::lower::AbstractConverter &converter,
@@ -42,40 +54,62 @@ class HlfirDesignatorBuilder {
                          Fortran::lower::StatementContext &stmtCtx)
       : converter{converter}, symMap{symMap}, stmtCtx{stmtCtx}, loc{loc} {}
 
+  /// Public entry points to lower a Designator<T> (given its .u member, to
+  /// avoid the template arguments which does not matter here).
+  /// This lowers a designator to an hlfir variable SSA value (that can be
+  /// assigned to), except for vector subscripted designators that are
+  /// lowered by default to hlfir.expr value since they cannot be
+  /// represented as HLFIR variable SSA values.
+
   // Character designators variant contains substrings
   using CharacterDesignators =
       decltype(Fortran::evaluate::Designator<Fortran::evaluate::Type<
                    Fortran::evaluate::TypeCategory::Character, 1>>::u);
   hlfir::EntityWithAttributes
-  gen(const CharacterDesignators &designatorVariant) {
+  gen(const CharacterDesignators &designatorVariant,
+      bool vectorSubscriptDesignatorToValue = true) {
     return std::visit(
-        [&](const auto &x) -> hlfir::EntityWithAttributes { return gen(x); },
+        [&](const auto &x) -> hlfir::EntityWithAttributes {
+          return genLeafPartRef(x, vectorSubscriptDesignatorToValue);
+        },
         designatorVariant);
   }
   // Character designators variant contains complex parts
   using RealDesignators =
       decltype(Fortran::evaluate::Designator<Fortran::evaluate::Type<
                    Fortran::evaluate::TypeCategory::Real, 4>>::u);
-  hlfir::EntityWithAttributes gen(const RealDesignators &designatorVariant) {
+  hlfir::EntityWithAttributes
+  gen(const RealDesignators &designatorVariant,
+      bool vectorSubscriptDesignatorToValue = true) {
     return std::visit(
-        [&](const auto &x) -> hlfir::EntityWithAttributes { return gen(x); },
+        [&](const auto &x) -> hlfir::EntityWithAttributes {
+          return genLeafPartRef(x, vectorSubscriptDesignatorToValue);
+        },
         designatorVariant);
   }
   // All other designators are similar
   using OtherDesignators =
       decltype(Fortran::evaluate::Designator<Fortran::evaluate::Type<
                    Fortran::evaluate::TypeCategory::Integer, 4>>::u);
-  hlfir::EntityWithAttributes gen(const OtherDesignators &designatorVariant) {
+  hlfir::EntityWithAttributes
+  gen(const OtherDesignators &designatorVariant,
+      bool vectorSubscriptDesignatorToValue = true) {
     return std::visit(
-        [&](const auto &x) -> hlfir::EntityWithAttributes { return gen(x); },
+        [&](const auto &x) -> hlfir::EntityWithAttributes {
+          return genLeafPartRef(x, vectorSubscriptDesignatorToValue);
+        },
         designatorVariant);
   }
 
   hlfir::EntityWithAttributes
-  gen(const Fortran::evaluate::NamedEntity &namedEntity) {
+  genNamedEntity(const Fortran::evaluate::NamedEntity &namedEntity,
+                 bool vectorSubscriptDesignatorToValue = true) {
     if (namedEntity.IsSymbol())
-      return gen(Fortran::evaluate::SymbolRef{namedEntity.GetLastSymbol()});
-    return gen(namedEntity.GetComponent());
+      return genLeafPartRef(
+          Fortran::evaluate::SymbolRef{namedEntity.GetLastSymbol()},
+          vectorSubscriptDesignatorToValue);
+    return genLeafPartRef(namedEntity.GetComponent(),
+                          vectorSubscriptDesignatorToValue);
   }
 
 private:
@@ -151,11 +185,29 @@ class HlfirDesignatorBuilder {
   fir::FortranVariableOpInterface
   genDesignate(mlir::Type designatorType, PartInfo &partInfo,
                fir::FortranVariableFlagsAttr attributes) {
-    auto designate = getBuilder().create<hlfir::DesignateOp>(
+    fir::FirOpBuilder &builder = getBuilder();
+    // Once a part with vector subscripts has been lowered, the following
+    // hlfir.designator (for the parts on the right of the designator) must
+    // be lowered inside the hlfir.elemental_addr because they depend on the
+    // hlfir.elemental_addr indices.
+    // All the subsequent Fortran indices however, should be lowered before
+    // the hlfir.elemental_addr because they should only be evaluated once,
+    // hence, the insertion point is restored outside of the
+    // hlfir.elemental_addr after generating the hlfir.designate. Example: in
+    // "X(VECTOR)%COMP(FOO(), BAR())", the calls to bar() and foo() must be
+    // generated outside of the hlfir.elemental, but the related hlfir.designate
+    // that depends on the scalar hlfir.designate of X(VECTOR) that was
+    // generated inside the hlfir.elemental_addr should be generated in the
+    // hlfir.elemental_addr.
+    if (auto elementalAddrOp = getVectorSubscriptElementAddrOp())
+      builder.setInsertionPointToEnd(&elementalAddrOp->getBody().front());
+    auto designate = builder.create<hlfir::DesignateOp>(
         getLoc(), designatorType, partInfo.base.value().getBase(),
         partInfo.componentName, partInfo.componentShape, partInfo.subscripts,
         partInfo.substring, partInfo.complexPart, partInfo.resultShape,
         partInfo.typeParams, attributes);
+    if (auto elementalAddrOp = getVectorSubscriptElementAddrOp())
+      builder.setInsertionPoint(*elementalAddrOp);
     return mlir::cast<fir::FortranVariableOpInterface>(
         designate.getOperation());
   }
@@ -414,8 +466,20 @@ class HlfirDesignatorBuilder {
     };
     auto frontEndResultShape =
         Fortran::evaluate::GetShape(converter.getFoldingContext(), arrayRef);
+    auto tryGettingExtentFromFrontEnd =
+        [&](unsigned dim) -> std::pair<mlir::Value, fir::SequenceType::Extent> {
+      // Use constant extent if possible. The main advantage to do this now
+      // is to get the best FIR array types as possible while lowering.
+      if (frontEndResultShape)
+        if (auto maybeI64 =
+                Fortran::evaluate::ToInt64(frontEndResultShape->at(dim)))
+          return {builder.createIntegerConstant(loc, idxTy, *maybeI64),
+                  *maybeI64};
+      return {mlir::Value{}, fir::SequenceType::getUnknownExtent()};
+    };
     llvm::SmallVector<mlir::Value> resultExtents;
     fir::SequenceType::Shape resultTypeShape;
+    bool sawVectorSubscripts = false;
     for (auto subscript : llvm::enumerate(arrayRef.subscript())) {
       if (const auto *triplet =
               std::get_if<Fortran::evaluate::Triplet>(&subscript.value().u)) {
@@ -432,35 +496,43 @@ class HlfirDesignatorBuilder {
         ub = builder.createConvert(loc, idxTy, ub);
         mlir::Value stride = genSubscript(triplet->stride());
         stride = builder.createConvert(loc, idxTy, stride);
-        mlir::Value extent;
-        // Use constant extent if possible. The main advantage to do this now
-        // is to get the best FIR array types as possible while lowering.
-        if (frontEndResultShape)
-          if (auto maybeI64 = Fortran::evaluate::ToInt64(
-                  frontEndResultShape->at(resultExtents.size()))) {
-            resultTypeShape.push_back(*maybeI64);
-            extent = builder.createIntegerConstant(loc, idxTy, *maybeI64);
-          }
-        if (!extent) {
-          extent = builder.genExtentFromTriplet(loc, lb, ub, stride, idxTy);
-          resultTypeShape.push_back(fir::SequenceType::getUnknownExtent());
-        }
+        auto [extentValue, shapeExtent] =
+            tryGettingExtentFromFrontEnd(resultExtents.size());
+        resultTypeShape.push_back(shapeExtent);
+        if (!extentValue)
+          extentValue =
+              builder.genExtentFromTriplet(loc, lb, ub, stride, idxTy);
+        resultExtents.push_back(extentValue);
         partInfo.subscripts.emplace_back(
             hlfir::DesignateOp::Triplet{lb, ub, stride});
-        resultExtents.push_back(extent);
       } else {
         const auto &expr =
             std::get<Fortran::evaluate::IndirectSubscriptIntegerExpr>(
                 subscript.value().u)
                 .value();
-        if (expr.Rank() > 0)
-          TODO(getLoc(), "vector subscripts in HLFIR");
-        partInfo.subscripts.push_back(genSubscript(expr));
+        hlfir::Entity subscript = genSubscript(expr);
+        partInfo.subscripts.push_back(subscript);
+        if (expr.Rank() > 0) {
+          sawVectorSubscripts = true;
+          auto [extentValue, shapeExtent] =
+              tryGettingExtentFromFrontEnd(resultExtents.size());
+          resultTypeShape.push_back(shapeExtent);
+          if (!extentValue)
+            extentValue = hlfir::genExtent(loc, builder, subscript, /*dim=*/0);
+          resultExtents.push_back(extentValue);
+        }
       }
     }
-
     assert(resultExtents.size() == resultTypeShape.size() &&
            "inconsistent hlfir.designate shape");
+
+    // For vector subscripts, create an hlfir.elemental_addr and continue
+    // lowering the designator inside it as if it was addressing an element of
+    // the vector subscripts.
+    if (sawVectorSubscripts)
+      return createVectorSubscriptElementAddrOp(partInfo, baseType,
+                                                resultExtents);
+
     mlir::Type resultType = baseType.cast<fir::SequenceType>().getEleTy();
     if (!resultTypeShape.empty()) {
       // Ranked array section. The result shape comes from the array section
@@ -625,11 +697,132 @@ class HlfirDesignatorBuilder {
     return {baseType, fieldType};
   }
 
-  /// Lower a subscript expression. If it is a scalar subscript that is
-  /// a variable, it is loaded into an integer value.
+  // Compute: "lb + (i-1)*step".
+  mlir::Value computeTripletPosition(mlir::Location loc,
+                                     fir::FirOpBuilder &builder,
+                                     hlfir::DesignateOp::Triplet &triplet,
+                                     mlir::Value oneBasedIndex) {
+    mlir::Type idxTy = builder.getIndexType();
+    mlir::Value lb = builder.createConvert(loc, idxTy, std::get<0>(triplet));
+    mlir::Value step = builder.createConvert(loc, idxTy, std::get<2>(triplet));
+    mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
+    oneBasedIndex = builder.createConvert(loc, idxTy, oneBasedIndex);
+    mlir::Value zeroBased =
+        builder.create<mlir::arith::SubIOp>(loc, oneBasedIndex, one);
+    mlir::Value offset =
+        builder.create<mlir::arith::MulIOp>(loc, zeroBased, step);
+    return builder.create<mlir::arith::AddIOp>(loc, lb, offset);
+  }
+
+  /// Create an hlfir.element_addr operation to deal with vector subscripted
+  /// entities. This transforms the current vector subscripted array-ref into a
+  /// a scalar array-ref that is addressing the vector subscripted part given
+  /// the one based indices of the hlfir.element_addr.
+  /// The rest of the designator lowering will continue lowering any further
+  /// parts inside the hlfir.elemental as a scalar reference.
+  /// At the end of the designator lowering, the hlfir.elemental_addr will
+  /// be turned into an hlfir.elemental value, unless the caller of this
+  /// utility requested to get the hlfir.elemental_addr instead of lowering
+  /// the designator to an mlir::Value.
+  mlir::Type createVectorSubscriptElementAddrOp(
+      PartInfo &partInfo, mlir::Type baseType,
+      llvm::ArrayRef<mlir::Value> resultExtents) {
+    fir::FirOpBuilder &builder = getBuilder();
+    mlir::Value shape = builder.genShape(loc, resultExtents);
+    // The type parameters to be added on the hlfir.elemental_addr are the ones
+    // of the whole designator (not the ones of the vector subscripted part).
+    // These are not yet known and will be added when finalizing the designator
+    // lowering.
+    auto elementalAddrOp = builder.create<hlfir::ElementalAddrOp>(loc, shape);
+    setVectorSubscriptElementAddrOp(elementalAddrOp);
+    builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
+    mlir::Region::BlockArgListType indices = elementalAddrOp.getIndices();
+    auto indicesIterator = indices.begin();
+    auto getNextOneBasedIndex = [&]() -> mlir::Value {
+      assert(indicesIterator != indices.end() && "ill formed ElementalAddrOp");
+      return *(indicesIterator++);
+    };
+    // Transform the designator into a scalar designator computing the vector
+    // subscripted entity element address given one based indices (for the shape
+    // of the vector subscripted designator).
+    for (hlfir::DesignateOp::Subscript &subscript : partInfo.subscripts) {
+      if (auto *triplet =
+              std::get_if<hlfir::DesignateOp::Triplet>(&subscript)) {
+        // subscript = (lb + (i-1)*step)
+        mlir::Value scalarSubscript = computeTripletPosition(
+            loc, builder, *triplet, getNextOneBasedIndex());
+        subscript = scalarSubscript;
+      } else {
+        hlfir::Entity valueSubscript{std::get<mlir::Value>(subscript)};
+        if (valueSubscript.isScalar())
+          continue;
+        // subscript = vector(i + (vector_lb-1))
+        hlfir::Entity scalarSubscript = hlfir::getElementAt(
+            loc, builder, valueSubscript, {getNextOneBasedIndex()});
+        scalarSubscript =
+            hlfir::loadTrivialScalar(loc, builder, scalarSubscript);
+        subscript = scalarSubscript;
+      }
+    }
+    builder.setInsertionPoint(elementalAddrOp);
+    return baseType.cast<fir::SequenceType>().getEleTy();
+  }
+
+  /// Yield the designator for the final part-ref inside the
+  /// hlfir.elemental_addr.
+  void finalizeElementAddrOp(hlfir::ElementalAddrOp elementalAddrOp,
+                             hlfir::EntityWithAttributes elementAddr) {
+    fir::FirOpBuilder &builder = getBuilder();
+    builder.setInsertionPointToEnd(&elementalAddrOp.getBody().front());
+    builder.create<hlfir::YieldOp>(loc, elementAddr);
+    builder.setInsertionPointAfter(elementalAddrOp);
+  }
+
+  /// If the lowered designator has vector subscripts turn it into an
+  /// ElementalOp, otherwise, return the lowered designator. This should
+  /// only be called if the user did not request to get the
+  /// hlfir.elemental_addr. In Fortran, vector subscripted designators are only
+  /// writable on the left-hand side of an assignment and in input IO
+  /// statements. Otherwise, they are not variables (cannot be modified, their
+  /// value is taken at the place they appear).
+  hlfir::EntityWithAttributes turnVectorSubscriptedDesignatorIntoValue(
+      hlfir::EntityWithAttributes loweredDesignator) {
+    std::optional<hlfir::ElementalAddrOp> elementalAddrOp =
+        getVectorSubscriptElementAddrOp();
+    if (!elementalAddrOp)
+      return loweredDesignator;
+    finalizeElementAddrOp(*elementalAddrOp, loweredDesignator);
+    // This vector subscript designator is only being read, transform the
+    // hlfir.elemental_addr into an hlfir.elemental.  The content of the
+    // hlfir.elemental_addr is cloned, and the resulting address is loaded to
+    // get the new element value.
+    fir::FirOpBuilder &builder = getBuilder();
+    mlir::Location loc = getLoc();
+    mlir::Value elemental =
+        hlfir::cloneToElementalOp(loc, builder, *elementalAddrOp);
+    (*elementalAddrOp)->erase();
+    setVectorSubscriptElementAddrOp(std::nullopt);
+    fir::FirOpBuilder *bldr = &builder;
+    getStmtCtx().attachCleanup(
+        [=]() { bldr->create<hlfir::DestroyOp>(loc, elemental); });
+    return hlfir::EntityWithAttributes{elemental};
+  }
+
+  /// Lower a subscript expression. If it is a scalar subscript that is a
+  /// variable, it is loaded into an integer value. If it is an array (for
+  /// vector subscripts) it is dereferenced if this is an allocatable or
+  /// pointer.
   template <typename T>
-  hlfir::EntityWithAttributes
-  genSubscript(const Fortran::evaluate::Expr<T> &expr);
+  hlfir::Entity genSubscript(const Fortran::evaluate::Expr<T> &expr);
+
+  const std::optional<hlfir::ElementalAddrOp> &
+  getVectorSubscriptElementAddrOp() const {
+    return vectorSubscriptElementAddrOp;
+  }
+  void setVectorSubscriptElementAddrOp(
+      std::optional<hlfir::ElementalAddrOp> elementalAddrOp) {
+    vectorSubscriptElementAddrOp = elementalAddrOp;
+  }
 
   mlir::Location getLoc() const { return loc; }
   Fortran::lower::AbstractConverter &getConverter() { return converter; }
@@ -640,6 +833,9 @@ class HlfirDesignatorBuilder {
   Fortran::lower::AbstractConverter &converter;
   Fortran::lower::SymMap &symMap;
   Fortran::lower::StatementContext &stmtCtx;
+  // If there is a vector subscript, an elementalAddrOp is created
+  // to compute the address of the designator elements.
+  std::optional<hlfir::ElementalAddrOp> vectorSubscriptElementAddrOp{};
   mlir::Location loc;
 };
 
@@ -1253,7 +1449,7 @@ class HlfirBuilder {
     hlfir::EntityWithAttributes entity =
         HlfirDesignatorBuilder(getLoc(), getConverter(), getSymMap(),
                                getStmtCtx())
-            .gen(desc.base());
+            .genNamedEntity(desc.base());
     using ResTy = Fortran::evaluate::DescriptorInquiry::Result;
     mlir::Type resultType =
         getConverter().genType(ResTy::category, ResTy::kind);
@@ -1304,25 +1500,20 @@ class HlfirBuilder {
 };
 
 template <typename T>
-hlfir::EntityWithAttributes
+hlfir::Entity
 HlfirDesignatorBuilder::genSubscript(const Fortran::evaluate::Expr<T> &expr) {
   auto loweredExpr =
       HlfirBuilder(getLoc(), getConverter(), getSymMap(), getStmtCtx())
           .gen(expr);
-  if (!loweredExpr.isArray()) {
-    fir::FirOpBuilder &builder = getBuilder();
-    if (loweredExpr.isVariable())
+  fir::FirOpBuilder &builder = getBuilder();
+  // Skip constant conversions that litters designators and makes generated
+  // IR harder to read: directly use index constants for constant subscripts.
+  mlir::Type idxTy = builder.getIndexType();
+  if (!loweredExpr.isArray() && loweredExpr.getType() != idxTy)
+    if (auto cstIndex = fir::getIntIfConstant(loweredExpr))
       return hlfir::EntityWithAttributes{
-          hlfir::loadTrivialScalar(loc, builder, loweredExpr).getBase()};
-    // Skip constant conversions that litters designators and makes generated
-    // IR harder to read: directly use index constants for constant subscripts.
-    mlir::Type idxTy = builder.getIndexType();
-    if (loweredExpr.getType() != idxTy)
-      if (auto cstIndex = fir::getIntIfConstant(loweredExpr))
-        return hlfir::EntityWithAttributes{
-            builder.createIntegerConstant(getLoc(), idxTy, *cstIndex)};
-  }
-  return loweredExpr;
+          builder.createIntegerConstant(getLoc(), idxTy, *cstIndex)};
+  return hlfir::loadTrivialScalar(loc, builder, loweredExpr);
 }
 
 } // namespace
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 4d0375bfad4eb..65ae694d23f66 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -721,9 +721,12 @@ static void deallocateIntentOut(Fortran::lower::AbstractConverter &converter,
     if (auto mutBox = extVal.getBoxOf<fir::MutableBoxValue>()) {
       // The dummy argument is not passed in the ENTRY so it should not be
       // deallocated.
-      if (mlir::Operation *op = mutBox->getAddr().getDefiningOp())
-        if (mlir::isa<fir::AllocaOp>(op))
+      if (mlir::Operation *op = mutBox->getAddr().getDefiningOp()) {
+        if (auto declOp = mlir::dyn_cast<hlfir::DeclareOp>(op))
+          op = declOp.getMemref().getDefiningOp();
+        if (op && mlir::isa<fir::AllocaOp>(op))
           return;
+      }
       mlir::Location loc = converter.getCurrentLocation();
       fir::FirOpBuilder &builder = converter.getFirOpBuilder();
       auto genDeallocateWithTypeDesc = [&]() {
@@ -1837,14 +1840,15 @@ void Fortran::lower::mapSymbolAttributes(
     }
   }
 
+  if (addr && addr.getDefiningOp<fir::UnboxCharOp>()) {
+    // Ensure proper type is given to array/scalar that transited via
+    // fir.boxchar arg.
+    mlir::Type castTy = builder.getRefType(converter.genType(var));
+    addr = builder.createConvert(loc, castTy, addr);
+  }
+
   // Compute array extents and lower bounds.
   if (ba.isArray()) {
-    if (addr && addr.getDefiningOp<fir::UnboxCharOp>()) {
-      // Ensure proper type is given to array that transited via fir.boxchar
-      // arg.
-      mlir::Type castTy = builder.getRefType(converter.genType(var));
-      addr = builder.createConvert(loc, castTy, addr);
-    }
     if (ba.isStaticArray()) {
       if (ba.lboundIsAllOnes()) {
         for (std::int64_t extent :
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index e8a492c6a2076..a778e078f23b3 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -101,15 +101,80 @@ genObjectList(const Fortran::parser::AccObjectList &objectList,
   }
 }
 
+/// Generate the acc.bounds operation from the descriptor information.
+static llvm::SmallVector<mlir::Value>
+genBoundsOpsFromBox(fir::FirOpBuilder &builder, mlir::Location loc,
+                    Fortran::lower::AbstractConverter &converter,
+                    fir::ExtendedValue dataExv, mlir::Value box) {
+  llvm::SmallVector<mlir::Value> bounds;
+  mlir::Type idxTy = builder.getIndexType();
+  mlir::Type boundTy = builder.getType<mlir::acc::DataBoundsType>();
+  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
+  assert(box.getType().isa<fir::BaseBoxType>() && "expect firbox or fir.class");
+  for (unsigned dim = 0; dim < dataExv.rank(); ++dim) {
+    mlir::Value d = builder.createIntegerConstant(loc, idxTy, dim);
+    mlir::Value baseLb =
+        fir::factory::readLowerBound(builder, loc, dataExv, dim, one);
+    auto dimInfo =
+        builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy, box, d);
+    mlir::Value lb = builder.createIntegerConstant(loc, idxTy, 0);
+    mlir::Value ub =
+        builder.create<mlir::arith::SubIOp>(loc, dimInfo.getExtent(), one);
+    mlir::Value bound = builder.create<mlir::acc::DataBoundsOp>(
+        loc, boundTy, lb, ub, mlir::Value(), dimInfo.getByteStride(), true,
+        baseLb);
+    bounds.push_back(bound);
+  }
+  return bounds;
+}
+
+/// Generate acc.bounds operation for base array without any subscripts
+/// provided.
+static llvm::SmallVector<mlir::Value>
+genBaseBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
+                 Fortran::lower::AbstractConverter &converter,
+                 fir::ExtendedValue dataExv, mlir::Value baseAddr) {
+  mlir::Type idxTy = builder.getIndexType();
+  mlir::Type boundTy = builder.getType<mlir::acc::DataBoundsType>();
+  llvm::SmallVector<mlir::Value> bounds;
+
+  if (dataExv.rank() == 0)
+    return bounds;
+
+  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
+  for (std::size_t dim = 0; dim < dataExv.rank(); ++dim) {
+    mlir::Value baseLb =
+        fir::factory::readLowerBound(builder, loc, dataExv, dim, one);
+    mlir::Value ext = fir::factory::readExtent(builder, loc, dataExv, dim);
+    mlir::Value lb =
+        baseLb == one ? builder.createIntegerConstant(loc, idxTy, 0) : baseLb;
+
+    // ub = baseLb + extent - 1
+    mlir::Value lbExt = builder.create<mlir::arith::AddIOp>(loc, ext, baseLb);
+    mlir::Value ub = builder.create<mlir::arith::SubIOp>(loc, lbExt, one);
+    mlir::Value bound = builder.create<mlir::acc::DataBoundsOp>(
+        loc, boundTy, lb, ub, mlir::Value(), one, false, baseLb);
+    bounds.push_back(bound);
+  }
+  return bounds;
+}
+
+/// Generate acc.bounds operations for an array section when subscripts are
+/// provided.
 static llvm::SmallVector<mlir::Value>
 genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
              Fortran::lower::AbstractConverter &converter,
+             Fortran::lower::StatementContext &stmtCtx,
              const std::list<Fortran::parser::SectionSubscript> &subscripts,
-             std::stringstream &asFortran, const Fortran::parser::Name &name) {
+             std::stringstream &asFortran, fir::ExtendedValue &dataExv,
+             mlir::Value baseAddr) {
   int dimension = 0;
   mlir::Type idxTy = builder.getIndexType();
   mlir::Type boundTy = builder.getType<mlir::acc::DataBoundsType>();
   llvm::SmallVector<mlir::Value> bounds;
+
+  mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
+  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
   for (const auto &subscript : subscripts) {
     if (const auto *triplet{
             std::get_if<Fortran::parser::SubscriptTriplet>(&subscript.u)}) {
@@ -117,12 +182,19 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
         asFortran << ',';
       mlir::Value lbound, ubound, extent;
       std::optional<std::int64_t> lval, uval;
-      mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-      fir::ExtendedValue dataExv =
-          converter.getSymbolExtendedValue(*name.symbol);
       mlir::Value baseLb =
           fir::factory::readLowerBound(builder, loc, dataExv, dimension, one);
       bool defaultLb = baseLb == one;
+      mlir::Value stride = one;
+      bool strideInBytes = false;
+
+      if (fir::unwrapRefType(baseAddr.getType()).isa<fir::BaseBoxType>()) {
+        mlir::Value d = builder.createIntegerConstant(loc, idxTy, dimension);
+        auto dimInfo = builder.create<fir::BoxDimsOp>(loc, idxTy, idxTy, idxTy,
+                                                      baseAddr, d);
+        stride = dimInfo.getByteStride();
+        strideInBytes = true;
+      }
 
       const auto &lower{std::get<0>(triplet->t)};
       if (lower) {
@@ -136,8 +208,16 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
           }
           asFortran << *lval;
         } else {
-          TODO(loc, "non constant lower bound in array section");
+          const Fortran::lower::SomeExpr *lexpr =
+              Fortran::semantics::GetExpr(*lower);
+          mlir::Value lb =
+              fir::getBase(converter.genExprValue(loc, *lexpr, stmtCtx));
+          lb = builder.createConvert(loc, baseLb.getType(), lb);
+          lbound = builder.create<mlir::arith::SubIOp>(loc, lb, baseLb);
+          asFortran << lexpr->AsFortran();
         }
+      } else {
+        lbound = defaultLb ? zero : baseLb;
       }
       asFortran << ':';
       const auto &upper{std::get<1>(triplet->t)};
@@ -152,7 +232,13 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
           }
           asFortran << *uval;
         } else {
-          TODO(loc, "non constant upper bound in array section");
+          const Fortran::lower::SomeExpr *uexpr =
+              Fortran::semantics::GetExpr(*upper);
+          mlir::Value ub =
+              fir::getBase(converter.genExprValue(loc, *uexpr, stmtCtx));
+          ub = builder.createConvert(loc, baseLb.getType(), ub);
+          ubound = builder.create<mlir::arith::SubIOp>(loc, ub, baseLb);
+          asFortran << uexpr->AsFortran();
         }
       }
       if (lower && upper) {
@@ -168,14 +254,16 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
           }
         }
       }
+      // ub = baseLb + extent - 1
       if (!ubound) {
         mlir::Value ext =
             fir::factory::readExtent(builder, loc, dataExv, dimension);
-        extent = builder.create<mlir::arith::SubIOp>(loc, ext, baseLb);
+        mlir::Value lbExt =
+            builder.create<mlir::arith::AddIOp>(loc, ext, baseLb);
+        ubound = builder.create<mlir::arith::SubIOp>(loc, lbExt, one);
       }
-      mlir::Value empty;
       mlir::Value bound = builder.create<mlir::acc::DataBoundsOp>(
-          loc, boundTy, lbound, ubound, extent, empty, false, baseLb);
+          loc, boundTy, lbound, ubound, extent, stride, strideInBytes, baseLb);
       bounds.push_back(bound);
       ++dimension;
     }
@@ -183,6 +271,175 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc,
   return bounds;
 }
 
+static mlir::Value
+getDataOperandBaseAddr(Fortran::lower::AbstractConverter &converter,
+                       fir::FirOpBuilder &builder,
+                       Fortran::lower::SymbolRef sym, mlir::Location loc) {
+  mlir::Value symAddr = converter.getSymbolAddress(sym);
+  // TODO: Might need revisiting to handle for non-shared clauses
+  if (!symAddr) {
+    if (const auto *details =
+            sym->detailsIf<Fortran::semantics::HostAssocDetails>())
+      symAddr = converter.getSymbolAddress(details->symbol());
+  }
+
+  if (!symAddr)
+    llvm::report_fatal_error("could not retrieve symbol address");
+
+  if (auto boxTy =
+          fir::unwrapRefType(symAddr.getType()).dyn_cast<fir::BaseBoxType>()) {
+    if (boxTy.getEleTy().isa<fir::RecordType>())
+      TODO(loc, "derived type");
+
+    // Load the box when baseAddr is a `fir.ref<fir.box<T>>` or a
+    // `fir.ref<fir.class<T>>` type.
+    if (symAddr.getType().isa<fir::ReferenceType>())
+      return builder.create<fir::LoadOp>(loc, symAddr);
+  }
+  return symAddr;
+}
+
+static mlir::Value gatherDataOperandAddrAndBounds(
+    Fortran::lower::AbstractConverter &converter, fir::FirOpBuilder &builder,
+    Fortran::semantics::SemanticsContext &semanticsContext,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::AccObject &accObject, mlir::Location operandLocation,
+    std::stringstream &asFortran, llvm::SmallVector<mlir::Value> &bounds) {
+  mlir::Value baseAddr;
+  std::visit(
+      Fortran::common::visitors{
+          [&](const Fortran::parser::Designator &designator) {
+            if (auto expr{Fortran::semantics::AnalyzeExpr(semanticsContext,
+                                                          designator)}) {
+              if ((*expr).Rank() > 0 &&
+                  Fortran::parser::Unwrap<Fortran::parser::ArrayElement>(
+                      designator)) {
+                const auto *arrayElement =
+                    Fortran::parser::Unwrap<Fortran::parser::ArrayElement>(
+                        designator);
+                const auto *dataRef =
+                    std::get_if<Fortran::parser::DataRef>(&designator.u);
+                fir::ExtendedValue dataExv;
+                if (Fortran::parser::Unwrap<
+                        Fortran::parser::StructureComponent>(
+                        arrayElement->base)) {
+                  auto exprBase = Fortran::semantics::AnalyzeExpr(
+                      semanticsContext, arrayElement->base);
+                  dataExv = converter.genExprAddr(operandLocation, *exprBase,
+                                                  stmtCtx);
+                  baseAddr = fir::getBase(dataExv);
+                  asFortran << (*exprBase).AsFortran();
+                } else {
+                  const Fortran::parser::Name &name =
+                      Fortran::parser::GetLastName(*dataRef);
+                  baseAddr = getDataOperandBaseAddr(
+                      converter, builder, *name.symbol, operandLocation);
+                  dataExv = converter.getSymbolExtendedValue(*name.symbol);
+                  asFortran << name.ToString();
+                }
+
+                if (!arrayElement->subscripts.empty()) {
+                  asFortran << '(';
+                  bounds = genBoundsOps(builder, operandLocation, converter,
+                                        stmtCtx, arrayElement->subscripts,
+                                        asFortran, dataExv, baseAddr);
+                }
+                asFortran << ')';
+              } else if (Fortran::parser::Unwrap<
+                             Fortran::parser::StructureComponent>(designator)) {
+                fir::ExtendedValue compExv =
+                    converter.genExprAddr(operandLocation, *expr, stmtCtx);
+                baseAddr = fir::getBase(compExv);
+                if (fir::unwrapRefType(baseAddr.getType())
+                        .isa<fir::SequenceType>())
+                  bounds = genBaseBoundsOps(builder, operandLocation, converter,
+                                            compExv, baseAddr);
+                asFortran << (*expr).AsFortran();
+
+                // If the component is an allocatable or pointer the result of
+                // genExprAddr will be the result of a fir.box_addr operation.
+                // Retrieve the box so we handle it like other descriptor.
+                if (auto boxAddrOp = mlir::dyn_cast_or_null<fir::BoxAddrOp>(
+                        baseAddr.getDefiningOp())) {
+                  baseAddr = boxAddrOp.getVal();
+                  bounds = genBoundsOpsFromBox(builder, operandLocation,
+                                               converter, compExv, baseAddr);
+                }
+              } else {
+                // Scalar or full array.
+                if (const auto *dataRef{
+                        std::get_if<Fortran::parser::DataRef>(&designator.u)}) {
+                  const Fortran::parser::Name &name =
+                      Fortran::parser::GetLastName(*dataRef);
+                  fir::ExtendedValue dataExv =
+                      converter.getSymbolExtendedValue(*name.symbol);
+                  baseAddr = getDataOperandBaseAddr(
+                      converter, builder, *name.symbol, operandLocation);
+                  if (fir::unwrapRefType(baseAddr.getType())
+                          .isa<fir::BaseBoxType>())
+                    bounds = genBoundsOpsFromBox(builder, operandLocation,
+                                                 converter, dataExv, baseAddr);
+                  if (fir::unwrapRefType(baseAddr.getType())
+                          .isa<fir::SequenceType>())
+                    bounds = genBaseBoundsOps(builder, operandLocation,
+                                              converter, dataExv, baseAddr);
+                  asFortran << name.ToString();
+                } else { // Unsupported
+                  llvm::report_fatal_error(
+                      "Unsupported type of OpenACC operand");
+                }
+              }
+            }
+          },
+          [&](const Fortran::parser::Name &name) {
+            baseAddr = getDataOperandBaseAddr(converter, builder, *name.symbol,
+                                              operandLocation);
+            asFortran << name.ToString();
+          }},
+      accObject.u);
+  return baseAddr;
+}
+
+static mlir::Location
+genOperandLocation(Fortran::lower::AbstractConverter &converter,
+                   const Fortran::parser::AccObject &accObject) {
+  mlir::Location loc = converter.genUnknownLocation();
+  std::visit(Fortran::common::visitors{
+                 [&](const Fortran::parser::Designator &designator) {
+                   loc = converter.genLocation(designator.source);
+                 },
+                 [&](const Fortran::parser::Name &name) {
+                   loc = converter.genLocation(name.source);
+                 }},
+             accObject.u);
+  return loc;
+}
+
+template <typename Op>
+static Op createDataEntryOp(fir::FirOpBuilder &builder, mlir::Location loc,
+                            mlir::Value baseAddr, std::stringstream &name,
+                            mlir::SmallVector<mlir::Value> bounds,
+                            bool structured, mlir::acc::DataClause dataClause) {
+  mlir::Value varPtrPtr;
+  if (auto boxTy = baseAddr.getType().dyn_cast<fir::BaseBoxType>())
+    baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
+
+  Op op = builder.create<Op>(loc, baseAddr.getType(), baseAddr);
+  op.setNameAttr(builder.getStringAttr(name.str()));
+  op.setStructured(structured);
+  op.setDataClause(dataClause);
+
+  unsigned insPos = 1;
+  if (varPtrPtr)
+    op->insertOperands(insPos++, varPtrPtr);
+  if (bounds.size() > 0)
+    op->insertOperands(insPos, bounds);
+  op->setAttr(Op::getOperandSegmentSizeAttr(),
+              builder.getDenseI32ArrayAttr(
+                  {1, varPtrPtr ? 1 : 0, static_cast<int32_t>(bounds.size())}));
+  return op;
+}
+
 template <typename Op>
 static void
 genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
@@ -191,95 +448,34 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList,
                          Fortran::lower::StatementContext &stmtCtx,
                          llvm::SmallVectorImpl<mlir::Value> &dataOperands,
                          mlir::acc::DataClause dataClause, bool structured) {
-
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-
-  auto createOpAndAddOperand = [&](Fortran::lower::SymbolRef sym,
-                                   llvm::StringRef name,
-                                   mlir::Location loc) -> Op {
-    mlir::Value symAddr = converter.getSymbolAddress(sym);
-    // TODO: Might need revisiting to handle for non-shared clauses
-    if (!symAddr) {
-      if (const auto *details =
-              sym->detailsIf<Fortran::semantics::HostAssocDetails>())
-        symAddr = converter.getSymbolAddress(details->symbol());
-    }
-
-    if (!symAddr)
-      llvm::report_fatal_error("could not retrieve symbol address");
-
-    if (symAddr.getType().isa<fir::BaseBoxType>())
-      TODO(loc, "data operand operation creation for box types");
-    Op op = builder.create<Op>(loc, symAddr.getType(), symAddr);
-    op.setNameAttr(builder.getStringAttr(name));
-    op.setStructured(structured);
-    op.setDataClause(dataClause);
-    op->setAttr(Op::getOperandSegmentSizeAttr(),
-                builder.getDenseI32ArrayAttr({1, 0, 0}));
+  for (const auto &accObject : objectList.v) {
+    llvm::SmallVector<mlir::Value> bounds;
+    std::stringstream asFortran;
+    mlir::Location operandLocation = genOperandLocation(converter, accObject);
+    mlir::Value baseAddr = gatherDataOperandAddrAndBounds(
+        converter, builder, semanticsContext, stmtCtx, accObject,
+        operandLocation, asFortran, bounds);
+    Op op = createDataEntryOp<Op>(builder, operandLocation, baseAddr, asFortran,
+                                  bounds, structured, dataClause);
     dataOperands.push_back(op.getAccPtr());
-    return op;
-  };
+  }
+}
 
-  for (const auto &accObject : objectList.v) {
-    std::visit(
-        Fortran::common::visitors{
-            [&](const Fortran::parser::Designator &designator) {
-              mlir::Location operandLocation =
-                  converter.genLocation(designator.source);
-              if (auto expr{Fortran::semantics::AnalyzeExpr(semanticsContext,
-                                                            designator)}) {
-                if ((*expr).Rank() > 0 &&
-                    Fortran::parser::Unwrap<Fortran::parser::ArrayElement>(
-                        designator)) {
-                  const auto *arrayElement =
-                      Fortran::parser::Unwrap<Fortran::parser::ArrayElement>(
-                          designator);
-                  llvm::SmallVector<mlir::Value> bounds;
-                  const auto *dataRef =
-                      std::get_if<Fortran::parser::DataRef>(&designator.u);
-                  const Fortran::parser::Name &name =
-                      Fortran::parser::GetLastName(*dataRef);
-                  std::stringstream asFortran;
-                  asFortran << name.ToString();
-                  if (!arrayElement->subscripts.empty()) {
-                    asFortran << '(';
-                    bounds =
-                        genBoundsOps(builder, operandLocation, converter,
-                                     arrayElement->subscripts, asFortran, name);
-                  }
-                  asFortran << ')';
-                  Op op = createOpAndAddOperand(*name.symbol, asFortran.str(),
-                                                operandLocation);
-                  op->insertOperands(1, bounds);
-                  op->setAttr(Op::getOperandSegmentSizeAttr(),
-                              builder.getDenseI32ArrayAttr(
-                                  {1, 0, static_cast<int32_t>(bounds.size())}));
-                } else if (Fortran::parser::Unwrap<
-                               Fortran::parser::StructureComponent>(
-                               designator)) {
-                  TODO(operandLocation, "OpenACC derived-type data operand");
-                } else {
-                  // Scalar or full array.
-                  if (const auto *dataRef{std::get_if<Fortran::parser::DataRef>(
-                          &designator.u)}) {
-                    const Fortran::parser::Name &name =
-                        Fortran::parser::GetLastName(*dataRef);
-                    createOpAndAddOperand(*name.symbol, name.ToString(),
-                                          operandLocation);
-                  } else { // Unsupported
-                    llvm::report_fatal_error(
-                        "Unsupported type of OpenACC operand");
-                  }
-                }
-              }
-            },
-            [&](const Fortran::parser::Name &name) {
-              mlir::Location operandLocation =
-                  converter.genLocation(name.source);
-              createOpAndAddOperand(*name.symbol, name.ToString(),
-                                    operandLocation);
-            }},
-        accObject.u);
+template <typename EntryOp, typename ExitOp>
+static void genDataExitOperations(fir::FirOpBuilder &builder,
+                                  llvm::SmallVector<mlir::Value> operands,
+                                  bool structured, bool implicit) {
+  for (mlir::Value operand : operands) {
+    auto entryOp = mlir::dyn_cast_or_null<EntryOp>(operand.getDefiningOp());
+    assert(entryOp && "data entry op expected");
+    mlir::Value varPtr;
+    if constexpr (std::is_same_v<ExitOp, mlir::acc::CopyoutOp>)
+      varPtr = entryOp.getVarPtr();
+    builder.create<ExitOp>(entryOp.getLoc(), entryOp.getAccPtr(), varPtr,
+                           entryOp.getBounds(), entryOp.getDataClause(),
+                           structured, implicit,
+                           builder.getStringAttr(*entryOp.getName()));
   }
 }
 
@@ -514,12 +710,14 @@ createLoopOp(Fortran::lower::AbstractConverter &converter,
                        &clause.u)) {
       genObjectList(privateClause->v, converter, semanticsContext, stmtCtx,
                     privateOperands);
+    } else if (std::get_if<Fortran::parser::AccClause::Reduction>(&clause.u)) {
+      // Reduction clause is left out for the moment as the clause will probably
+      // end up having its own operation.
+      TODO(clauseLocation, "OpenACC compute construct reduction lowering");
     }
-    // Reduction clause is left out for the moment as the clause will probably
-    // end up having its own operation.
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value> operands;
   llvm::SmallVector<int32_t> operandSegments;
   addOperand(operands, operandSegments, gangNum);
@@ -585,6 +783,28 @@ static void genACC(Fortran::lower::AbstractConverter &converter,
   }
 }
 
+template <typename Op, typename Clause>
+static void genDataOperandOperationsWithModifier(
+    const Clause *x, Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semanticsContext,
+    Fortran::lower::StatementContext &stmtCtx,
+    Fortran::parser::AccDataModifier::Modifier mod,
+    llvm::SmallVectorImpl<mlir::Value> &dataClauseOperands,
+    const mlir::acc::DataClause clause,
+    const mlir::acc::DataClause clauseWithModifier) {
+  const Fortran::parser::AccObjectListWithModifier &listWithModifier = x->v;
+  const auto &accObjectList =
+      std::get<Fortran::parser::AccObjectList>(listWithModifier.t);
+  const auto &modifier =
+      std::get<std::optional<Fortran::parser::AccDataModifier>>(
+          listWithModifier.t);
+  mlir::acc::DataClause dataClause =
+      (modifier && (*modifier).v == mod) ? clauseWithModifier : clause;
+  genDataOperandOperations<Op>(accObjectList, converter, semanticsContext,
+                               stmtCtx, dataClauseOperands, dataClause,
+                               /*structured=*/true);
+}
+
 template <typename Op>
 static Op
 createComputeOp(Fortran::lower::AbstractConverter &converter,
@@ -601,11 +821,13 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
   mlir::Value ifCond;
   mlir::Value selfCond;
   mlir::Value waitDevnum;
-  llvm::SmallVector<mlir::Value, 2> waitOperands, reductionOperands,
-      copyOperands, copyinOperands, copyinReadonlyOperands, copyoutOperands,
-      copyoutZeroOperands, createOperands, createZeroOperands, noCreateOperands,
-      presentOperands, devicePtrOperands, attachOperands, firstprivateOperands,
-      privateOperands, dataClauseOperands;
+  llvm::SmallVector<mlir::Value> waitOperands, attachEntryOperands,
+      copyEntryOperands, copyoutEntryOperands, createEntryOperands,
+      dataClauseOperands;
+
+  // TODO: need to more work/design.
+  llvm::SmallVector<mlir::Value> reductionOperands, privateOperands,
+      firstprivateOperands;
 
   // Async, wait and self clause have optional values but can be present with
   // no value as well. When there is no value, the op has an attribute to
@@ -614,7 +836,7 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
   bool addWaitAttr = false;
   bool addSelfAttr = false;
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
   // Lower clauses values mapped to operands.
   // Keep track of each group of operands separatly as clauses can appear
@@ -657,8 +879,8 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
           if (*optCondition) {
             mlir::Value cond = fir::getBase(converter.genExprValue(
                 *Fortran::semantics::GetExpr(*optCondition), stmtCtx));
-            selfCond = firOpBuilder.createConvert(
-                clauseLocation, firOpBuilder.getI1Type(), cond);
+            selfCond = builder.createConvert(clauseLocation,
+                                             builder.getI1Type(), cond);
           }
         } else if (const auto *accClauseList =
                        std::get_if<Fortran::parser::AccObjectList>(
@@ -670,8 +892,8 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
                     std::get_if<Fortran::parser::Designator>(&accObject.u)) {
               if (const auto *name = getDesignatorNameIfDataRef(*designator)) {
                 auto cond = converter.getSymbolAddress(*name->symbol);
-                selfCond = firOpBuilder.createConvert(
-                    clauseLocation, firOpBuilder.getI1Type(), cond);
+                selfCond = builder.createConvert(clauseLocation,
+                                                 builder.getI1Type(), cond);
               }
             }
           }
@@ -681,46 +903,62 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
       }
     } else if (const auto *copyClause =
                    std::get_if<Fortran::parser::AccClause::Copy>(&clause.u)) {
-      genObjectList(copyClause->v, converter, semanticsContext, stmtCtx,
-                    copyOperands);
+      genDataOperandOperations<mlir::acc::CopyinOp>(
+          copyClause->v, converter, semanticsContext, stmtCtx,
+          copyEntryOperands, mlir::acc::DataClause::acc_copy,
+          /*structured=*/true);
     } else if (const auto *copyinClause =
                    std::get_if<Fortran::parser::AccClause::Copyin>(&clause.u)) {
-      genObjectListWithModifier<Fortran::parser::AccClause::Copyin>(
+      genDataOperandOperationsWithModifier<mlir::acc::CopyinOp,
+                                           Fortran::parser::AccClause::Copyin>(
           copyinClause, converter, semanticsContext, stmtCtx,
           Fortran::parser::AccDataModifier::Modifier::ReadOnly,
-          copyinReadonlyOperands, copyinOperands);
+          dataClauseOperands, mlir::acc::DataClause::acc_copyin,
+          mlir::acc::DataClause::acc_copyin_readonly);
     } else if (const auto *copyoutClause =
                    std::get_if<Fortran::parser::AccClause::Copyout>(
                        &clause.u)) {
-      genObjectListWithModifier<Fortran::parser::AccClause::Copyout>(
+      genDataOperandOperationsWithModifier<mlir::acc::CreateOp,
+                                           Fortran::parser::AccClause::Copyout>(
           copyoutClause, converter, semanticsContext, stmtCtx,
-          Fortran::parser::AccDataModifier::Modifier::Zero, copyoutZeroOperands,
-          copyoutOperands);
+          Fortran::parser::AccDataModifier::Modifier::ReadOnly,
+          copyoutEntryOperands, mlir::acc::DataClause::acc_copyout,
+          mlir::acc::DataClause::acc_copyout_zero);
     } else if (const auto *createClause =
                    std::get_if<Fortran::parser::AccClause::Create>(&clause.u)) {
-      genObjectListWithModifier<Fortran::parser::AccClause::Create>(
+      genDataOperandOperationsWithModifier<mlir::acc::CreateOp,
+                                           Fortran::parser::AccClause::Create>(
           createClause, converter, semanticsContext, stmtCtx,
-          Fortran::parser::AccDataModifier::Modifier::Zero, createZeroOperands,
-          createOperands);
+          Fortran::parser::AccDataModifier::Modifier::Zero, createEntryOperands,
+          mlir::acc::DataClause::acc_create,
+          mlir::acc::DataClause::acc_create_zero);
     } else if (const auto *noCreateClause =
                    std::get_if<Fortran::parser::AccClause::NoCreate>(
                        &clause.u)) {
-      genObjectList(noCreateClause->v, converter, semanticsContext, stmtCtx,
-                    noCreateOperands);
+      genDataOperandOperations<mlir::acc::NoCreateOp>(
+          noCreateClause->v, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, mlir::acc::DataClause::acc_no_create,
+          /*structured=*/true);
     } else if (const auto *presentClause =
                    std::get_if<Fortran::parser::AccClause::Present>(
                        &clause.u)) {
-      genObjectList(presentClause->v, converter, semanticsContext, stmtCtx,
-                    presentOperands);
+      genDataOperandOperations<mlir::acc::PresentOp>(
+          presentClause->v, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, mlir::acc::DataClause::acc_present,
+          /*structured=*/true);
     } else if (const auto *devicePtrClause =
                    std::get_if<Fortran::parser::AccClause::Deviceptr>(
                        &clause.u)) {
-      genObjectList(devicePtrClause->v, converter, semanticsContext, stmtCtx,
-                    devicePtrOperands);
+      genDataOperandOperations<mlir::acc::DevicePtrOp>(
+          devicePtrClause->v, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, mlir::acc::DataClause::acc_deviceptr,
+          /*structured=*/true);
     } else if (const auto *attachClause =
                    std::get_if<Fortran::parser::AccClause::Attach>(&clause.u)) {
-      genObjectList(attachClause->v, converter, semanticsContext, stmtCtx,
-                    attachOperands);
+      genDataOperandOperations<mlir::acc::AttachOp>(
+          attachClause->v, converter, semanticsContext, stmtCtx,
+          attachEntryOperands, mlir::acc::DataClause::acc_attach,
+          /*structured=*/true);
     } else if (const auto *privateClause =
                    std::get_if<Fortran::parser::AccClause::Private>(
                        &clause.u)) {
@@ -731,10 +969,17 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
                        &clause.u)) {
       genObjectList(firstprivateClause->v, converter, semanticsContext, stmtCtx,
                     firstprivateOperands);
+    } else if (std::get_if<Fortran::parser::AccClause::Reduction>(&clause.u)) {
+      TODO(clauseLocation, "compute construct reduction clause lowering");
     }
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  dataClauseOperands.append(attachEntryOperands);
+  dataClauseOperands.append(copyEntryOperands);
+  dataClauseOperands.append(copyoutEntryOperands);
+  dataClauseOperands.append(createEntryOperands);
+
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value, 8> operands;
   llvm::SmallVector<int32_t, 8> operandSegments;
   addOperand(operands, operandSegments, async);
@@ -748,17 +993,7 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
   addOperand(operands, operandSegments, selfCond);
   if constexpr (!std::is_same_v<Op, mlir::acc::KernelsOp>)
     addOperands(operands, operandSegments, reductionOperands);
-  addOperands(operands, operandSegments, copyOperands);
-  addOperands(operands, operandSegments, copyinOperands);
-  addOperands(operands, operandSegments, copyinReadonlyOperands);
-  addOperands(operands, operandSegments, copyoutOperands);
-  addOperands(operands, operandSegments, copyoutZeroOperands);
-  addOperands(operands, operandSegments, createOperands);
-  addOperands(operands, operandSegments, createZeroOperands);
-  addOperands(operands, operandSegments, noCreateOperands);
-  addOperands(operands, operandSegments, presentOperands);
-  addOperands(operands, operandSegments, devicePtrOperands);
-  addOperands(operands, operandSegments, attachOperands);
+  operandSegments.append({0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0});
   if constexpr (!std::is_same_v<Op, mlir::acc::KernelsOp>) {
     addOperands(operands, operandSegments, privateOperands);
     addOperands(operands, operandSegments, firstprivateOperands);
@@ -768,18 +1003,32 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
   Op computeOp;
   if constexpr (std::is_same_v<Op, mlir::acc::KernelsOp>)
     computeOp = createRegionOp<Op, mlir::acc::TerminatorOp>(
-        firOpBuilder, currentLocation, operands, operandSegments);
+        builder, currentLocation, operands, operandSegments);
   else
     computeOp = createRegionOp<Op, mlir::acc::YieldOp>(
-        firOpBuilder, currentLocation, operands, operandSegments);
+        builder, currentLocation, operands, operandSegments);
 
   if (addAsyncAttr)
-    computeOp.setAsyncAttrAttr(firOpBuilder.getUnitAttr());
+    computeOp.setAsyncAttrAttr(builder.getUnitAttr());
   if (addWaitAttr)
-    computeOp.setWaitAttrAttr(firOpBuilder.getUnitAttr());
+    computeOp.setWaitAttrAttr(builder.getUnitAttr());
   if (addSelfAttr)
-    computeOp.setSelfAttrAttr(firOpBuilder.getUnitAttr());
-
+    computeOp.setSelfAttrAttr(builder.getUnitAttr());
+
+  auto insPt = builder.saveInsertionPoint();
+  builder.setInsertionPointAfter(computeOp);
+
+  // Create the exit operations after the region.
+  genDataExitOperations<mlir::acc::CopyinOp, mlir::acc::CopyoutOp>(
+      builder, copyEntryOperands, /*structured=*/true, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::CreateOp, mlir::acc::CopyoutOp>(
+      builder, copyoutEntryOperands, /*structured=*/true, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::AttachOp, mlir::acc::DetachOp>(
+      builder, attachEntryOperands, /*structured=*/true, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::CreateOp, mlir::acc::DeleteOp>(
+      builder, createEntryOperands, /*structured=*/true, /*implicit=*/false);
+
+  builder.restoreInsertionPoint(insPt);
   return computeOp;
 }
 
@@ -789,12 +1038,10 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter,
                          Fortran::lower::StatementContext &stmtCtx,
                          const Fortran::parser::AccClauseList &accClauseList) {
   mlir::Value ifCond;
-  llvm::SmallVector<mlir::Value> copyOperands, copyinOperands,
-      copyinReadonlyOperands, copyoutOperands, copyoutZeroOperands,
-      createOperands, createZeroOperands, noCreateOperands, presentOperands,
-      deviceptrOperands, attachOperands, dataClauseOperands;
+  llvm::SmallVector<mlir::Value> attachEntryOperands, createEntryOperands,
+      copyEntryOperands, copyoutEntryOperands, dataClauseOperands;
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
   // Lower clauses values mapped to operands.
   // Keep track of each group of operands separatly as clauses can appear
@@ -806,68 +1053,94 @@ static void genACCDataOp(Fortran::lower::AbstractConverter &converter,
       genIfClause(converter, clauseLocation, ifClause, ifCond, stmtCtx);
     } else if (const auto *copyClause =
                    std::get_if<Fortran::parser::AccClause::Copy>(&clause.u)) {
-      genObjectList(copyClause->v, converter, semanticsContext, stmtCtx,
-                    copyOperands);
+      genDataOperandOperations<mlir::acc::CopyinOp>(
+          copyClause->v, converter, semanticsContext, stmtCtx,
+          copyEntryOperands, mlir::acc::DataClause::acc_copy,
+          /*structured=*/true);
     } else if (const auto *copyinClause =
                    std::get_if<Fortran::parser::AccClause::Copyin>(&clause.u)) {
-      genObjectListWithModifier<Fortran::parser::AccClause::Copyin>(
+      genDataOperandOperationsWithModifier<mlir::acc::CopyinOp,
+                                           Fortran::parser::AccClause::Copyin>(
           copyinClause, converter, semanticsContext, stmtCtx,
           Fortran::parser::AccDataModifier::Modifier::ReadOnly,
-          copyinReadonlyOperands, copyinOperands);
+          dataClauseOperands, mlir::acc::DataClause::acc_copyin,
+          mlir::acc::DataClause::acc_copyin_readonly);
     } else if (const auto *copyoutClause =
                    std::get_if<Fortran::parser::AccClause::Copyout>(
                        &clause.u)) {
-      genObjectListWithModifier<Fortran::parser::AccClause::Copyout>(
+      genDataOperandOperationsWithModifier<mlir::acc::CreateOp,
+                                           Fortran::parser::AccClause::Copyout>(
           copyoutClause, converter, semanticsContext, stmtCtx,
-          Fortran::parser::AccDataModifier::Modifier::Zero, copyoutZeroOperands,
-          copyoutOperands);
+          Fortran::parser::AccDataModifier::Modifier::Zero,
+          copyoutEntryOperands, mlir::acc::DataClause::acc_copyout,
+          mlir::acc::DataClause::acc_copyout_zero);
     } else if (const auto *createClause =
                    std::get_if<Fortran::parser::AccClause::Create>(&clause.u)) {
-      genObjectListWithModifier<Fortran::parser::AccClause::Create>(
+      genDataOperandOperationsWithModifier<mlir::acc::CreateOp,
+                                           Fortran::parser::AccClause::Create>(
           createClause, converter, semanticsContext, stmtCtx,
-          Fortran::parser::AccDataModifier::Modifier::Zero, createZeroOperands,
-          createOperands);
+          Fortran::parser::AccDataModifier::Modifier::Zero, createEntryOperands,
+          mlir::acc::DataClause::acc_create,
+          mlir::acc::DataClause::acc_create_zero);
     } else if (const auto *noCreateClause =
                    std::get_if<Fortran::parser::AccClause::NoCreate>(
                        &clause.u)) {
-      genObjectList(noCreateClause->v, converter, semanticsContext, stmtCtx,
-                    noCreateOperands);
+      genDataOperandOperations<mlir::acc::NoCreateOp>(
+          noCreateClause->v, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, mlir::acc::DataClause::acc_no_create,
+          /*structured=*/true);
     } else if (const auto *presentClause =
                    std::get_if<Fortran::parser::AccClause::Present>(
                        &clause.u)) {
-      genObjectList(presentClause->v, converter, semanticsContext, stmtCtx,
-                    presentOperands);
+      genDataOperandOperations<mlir::acc::PresentOp>(
+          presentClause->v, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, mlir::acc::DataClause::acc_present,
+          /*structured=*/true);
     } else if (const auto *deviceptrClause =
                    std::get_if<Fortran::parser::AccClause::Deviceptr>(
                        &clause.u)) {
-      genObjectList(deviceptrClause->v, converter, semanticsContext, stmtCtx,
-                    deviceptrOperands);
+      genDataOperandOperations<mlir::acc::DevicePtrOp>(
+          deviceptrClause->v, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, mlir::acc::DataClause::acc_deviceptr,
+          /*structured=*/true);
     } else if (const auto *attachClause =
                    std::get_if<Fortran::parser::AccClause::Attach>(&clause.u)) {
-      genObjectList(attachClause->v, converter, semanticsContext, stmtCtx,
-                    attachOperands);
+      genDataOperandOperations<mlir::acc::AttachOp>(
+          attachClause->v, converter, semanticsContext, stmtCtx,
+          attachEntryOperands, mlir::acc::DataClause::acc_attach,
+          /*structured=*/true);
     }
   }
 
+  dataClauseOperands.append(attachEntryOperands);
+  dataClauseOperands.append(copyEntryOperands);
+  dataClauseOperands.append(copyoutEntryOperands);
+  dataClauseOperands.append(createEntryOperands);
+
   // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value> operands;
   llvm::SmallVector<int32_t> operandSegments;
   addOperand(operands, operandSegments, ifCond);
-  addOperands(operands, operandSegments, copyOperands);
-  addOperands(operands, operandSegments, copyinOperands);
-  addOperands(operands, operandSegments, copyinReadonlyOperands);
-  addOperands(operands, operandSegments, copyoutOperands);
-  addOperands(operands, operandSegments, copyoutZeroOperands);
-  addOperands(operands, operandSegments, createOperands);
-  addOperands(operands, operandSegments, createZeroOperands);
-  addOperands(operands, operandSegments, noCreateOperands);
-  addOperands(operands, operandSegments, presentOperands);
-  addOperands(operands, operandSegments, deviceptrOperands);
-  addOperands(operands, operandSegments, attachOperands);
+  operandSegments.append({0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0});
   addOperands(operands, operandSegments, dataClauseOperands);
 
-  createRegionOp<mlir::acc::DataOp, mlir::acc::TerminatorOp>(
-      firOpBuilder, currentLocation, operands, operandSegments);
+  auto dataOp = createRegionOp<mlir::acc::DataOp, mlir::acc::TerminatorOp>(
+      builder, currentLocation, operands, operandSegments);
+
+  auto insPt = builder.saveInsertionPoint();
+  builder.setInsertionPointAfter(dataOp);
+
+  // Create the exit operations after the region.
+  genDataExitOperations<mlir::acc::CopyinOp, mlir::acc::CopyoutOp>(
+      builder, copyEntryOperands, /*structured=*/true, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::CreateOp, mlir::acc::CopyoutOp>(
+      builder, copyoutEntryOperands, /*structured=*/true, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::AttachOp, mlir::acc::DetachOp>(
+      builder, attachEntryOperands, /*structured=*/true, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::CreateOp, mlir::acc::DeleteOp>(
+      builder, createEntryOperands, /*structured=*/true, /*implicit=*/false);
+
+  builder.restoreInsertionPoint(insPt);
 }
 
 static void
@@ -945,8 +1218,7 @@ genACCEnterDataOp(Fortran::lower::AbstractConverter &converter,
                   Fortran::lower::StatementContext &stmtCtx,
                   const Fortran::parser::AccClauseList &accClauseList) {
   mlir::Value ifCond, async, waitDevnum;
-  llvm::SmallVector<mlir::Value> copyinOperands, createOperands,
-      createZeroOperands, attachOperands, waitOperands, dataClauseOperands;
+  llvm::SmallVector<mlir::Value> waitOperands, dataClauseOperands;
 
   // Async, wait and self clause have optional values but can be present with
   // no value as well. When there is no value, the op has an attribute to
@@ -957,7 +1229,7 @@ genACCEnterDataOp(Fortran::lower::AbstractConverter &converter,
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   // Lower clauses values mapped to operands.
-  // Keep track of each group of operands separatly as clauses can appear
+  // Keep track of each group of operands separately as clauses can appear
   // more than once.
   for (const Fortran::parser::AccClause &clause : accClauseList.v) {
     mlir::Location clauseLocation = converter.genLocation(clause.source);
@@ -989,16 +1261,13 @@ genACCEnterDataOp(Fortran::lower::AbstractConverter &converter,
       const auto &modifier =
           std::get<std::optional<Fortran::parser::AccDataModifier>>(
               listWithModifier.t);
+      mlir::acc::DataClause clause = mlir::acc::DataClause::acc_create;
       if (modifier &&
-          (*modifier).v == Fortran::parser::AccDataModifier::Modifier::Zero) {
-        genDataOperandOperations<mlir::acc::CreateOp>(
-            accObjectList, converter, semanticsContext, stmtCtx,
-            dataClauseOperands, mlir::acc::DataClause::acc_create_zero, false);
-      } else {
-        genDataOperandOperations<mlir::acc::CreateOp>(
-            accObjectList, converter, semanticsContext, stmtCtx,
-            dataClauseOperands, mlir::acc::DataClause::acc_create, false);
-      }
+          (*modifier).v == Fortran::parser::AccDataModifier::Modifier::Zero)
+        clause = mlir::acc::DataClause::acc_create_zero;
+      genDataOperandOperations<mlir::acc::CreateOp>(
+          accObjectList, converter, semanticsContext, stmtCtx,
+          dataClauseOperands, clause, false);
     } else if (const auto *attachClause =
                    std::get_if<Fortran::parser::AccClause::Attach>(&clause.u)) {
       genDataOperandOperations<mlir::acc::AttachOp>(
@@ -1010,17 +1279,14 @@ genACCEnterDataOp(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value, 16> operands;
   llvm::SmallVector<int32_t, 8> operandSegments;
   addOperand(operands, operandSegments, ifCond);
   addOperand(operands, operandSegments, async);
   addOperand(operands, operandSegments, waitDevnum);
   addOperands(operands, operandSegments, waitOperands);
-  addOperands(operands, operandSegments, copyinOperands);
-  addOperands(operands, operandSegments, createOperands);
-  addOperands(operands, operandSegments, createZeroOperands);
-  addOperands(operands, operandSegments, attachOperands);
+  operandSegments.append({0, 0, 0, 0});
   addOperands(operands, operandSegments, dataClauseOperands);
 
   mlir::acc::EnterDataOp enterDataOp = createSimpleOp<mlir::acc::EnterDataOp>(
@@ -1039,8 +1305,8 @@ genACCExitDataOp(Fortran::lower::AbstractConverter &converter,
                  Fortran::lower::StatementContext &stmtCtx,
                  const Fortran::parser::AccClauseList &accClauseList) {
   mlir::Value ifCond, async, waitDevnum;
-  llvm::SmallVector<mlir::Value> copyoutOperands, deleteOperands,
-      detachOperands, waitOperands, dataClauseOperands;
+  llvm::SmallVector<mlir::Value> waitOperands, dataClauseOperands,
+      copyoutOperands, deleteOperands, detachOperands;
 
   // Async and wait clause have optional values but can be present with
   // no value as well. When there is no value, the op has an attribute to
@@ -1049,7 +1315,7 @@ genACCExitDataOp(Fortran::lower::AbstractConverter &converter,
   bool addWaitAttr = false;
   bool addFinalizeAttr = false;
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
 
   // Lower clauses values mapped to operands.
   // Keep track of each group of operands separatly as clauses can appear
@@ -1073,42 +1339,54 @@ genACCExitDataOp(Fortran::lower::AbstractConverter &converter,
           copyoutClause->v;
       const auto &accObjectList =
           std::get<Fortran::parser::AccObjectList>(listWithModifier.t);
-      genObjectList(accObjectList, converter, semanticsContext, stmtCtx,
-                    copyoutOperands);
+      genDataOperandOperations<mlir::acc::GetDevicePtrOp>(
+          accObjectList, converter, semanticsContext, stmtCtx, copyoutOperands,
+          mlir::acc::DataClause::acc_copyout, false);
     } else if (const auto *deleteClause =
                    std::get_if<Fortran::parser::AccClause::Delete>(&clause.u)) {
-      genObjectList(deleteClause->v, converter, semanticsContext, stmtCtx,
-                    deleteOperands);
+      genDataOperandOperations<mlir::acc::GetDevicePtrOp>(
+          deleteClause->v, converter, semanticsContext, stmtCtx, deleteOperands,
+          mlir::acc::DataClause::acc_delete, false);
     } else if (const auto *detachClause =
                    std::get_if<Fortran::parser::AccClause::Detach>(&clause.u)) {
-      genObjectList(detachClause->v, converter, semanticsContext, stmtCtx,
-                    detachOperands);
+      genDataOperandOperations<mlir::acc::GetDevicePtrOp>(
+          detachClause->v, converter, semanticsContext, stmtCtx, detachOperands,
+          mlir::acc::DataClause::acc_detach, false);
     } else if (std::get_if<Fortran::parser::AccClause::Finalize>(&clause.u)) {
       addFinalizeAttr = true;
     }
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  dataClauseOperands.append(copyoutOperands);
+  dataClauseOperands.append(deleteOperands);
+  dataClauseOperands.append(detachOperands);
+
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value, 14> operands;
   llvm::SmallVector<int32_t, 7> operandSegments;
   addOperand(operands, operandSegments, ifCond);
   addOperand(operands, operandSegments, async);
   addOperand(operands, operandSegments, waitDevnum);
   addOperands(operands, operandSegments, waitOperands);
-  addOperands(operands, operandSegments, copyoutOperands);
-  addOperands(operands, operandSegments, deleteOperands);
-  addOperands(operands, operandSegments, detachOperands);
+  operandSegments.append({0, 0, 0});
   addOperands(operands, operandSegments, dataClauseOperands);
 
   mlir::acc::ExitDataOp exitDataOp = createSimpleOp<mlir::acc::ExitDataOp>(
-      firOpBuilder, currentLocation, operands, operandSegments);
+      builder, currentLocation, operands, operandSegments);
 
   if (addAsyncAttr)
-    exitDataOp.setAsyncAttr(firOpBuilder.getUnitAttr());
+    exitDataOp.setAsyncAttr(builder.getUnitAttr());
   if (addWaitAttr)
-    exitDataOp.setWaitAttr(firOpBuilder.getUnitAttr());
+    exitDataOp.setWaitAttr(builder.getUnitAttr());
   if (addFinalizeAttr)
-    exitDataOp.setFinalizeAttr(firOpBuilder.getUnitAttr());
+    exitDataOp.setFinalizeAttr(builder.getUnitAttr());
+
+  genDataExitOperations<mlir::acc::GetDevicePtrOp, mlir::acc::CopyoutOp>(
+      builder, copyoutOperands, /*structured=*/false, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::GetDevicePtrOp, mlir::acc::DeleteOp>(
+      builder, deleteOperands, /*structured=*/false, /*implicit=*/false);
+  genDataExitOperations<mlir::acc::GetDevicePtrOp, mlir::acc::DetachOp>(
+      builder, detachOperands, /*structured=*/false, /*implicit=*/false);
 }
 
 template <typename Op>
@@ -1143,7 +1421,7 @@ genACCInitShutdownOp(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value, 6> operands;
   llvm::SmallVector<int32_t, 3> operandSegments;
   addOperands(operands, operandSegments, deviceTypeOperands);
@@ -1203,7 +1481,7 @@ genACCUpdateOp(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value> operands;
   llvm::SmallVector<int32_t> operandSegments;
   addOperand(operands, operandSegments, ifCond);
@@ -1312,7 +1590,7 @@ static void genACC(Fortran::lower::AbstractConverter &converter,
     }
   }
 
-  // Prepare the operand segement size attribute and the operands value range.
+  // Prepare the operand segment size attribute and the operands value range.
   llvm::SmallVector<mlir::Value> operands;
   llvm::SmallVector<int32_t> operandSegments;
   addOperands(operands, operandSegments, waitOperands);
diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp
index 8c201e0d30a2b..8ab9dc40a50b0 100644
--- a/flang/lib/Lower/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP.cpp
@@ -23,7 +23,6 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/tools.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 
 using namespace mlir;
@@ -106,10 +105,12 @@ bool DataSharingProcessor::process() {
   insPt = firOpBuilder.saveInsertionPoint();
   collectSymbolsForPrivatization();
   insertLastPrivateCompare(op);
-  if (mlir::isa<mlir::omp::SectionOp>(op))
-    firOpBuilder.setInsertionPointToStart(&op->getRegion(0).back());
-  else
+  if (mlir::isa<mlir::omp::SectionOp>(op)) {
+    if (!eval.lowerAsUnstructured())
+      firOpBuilder.setInsertionPointToStart(&op->getRegion(0).back());
+  } else {
     firOpBuilder.setInsertionPointToStart(firOpBuilder.getAllocaBlock());
+  }
   privatize();
   collectDefaultSymbols();
   defaultPrivatize();
@@ -222,52 +223,65 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
           //  omp.section {
           //      fir.allocate for `private`/`firstprivate`
           //      <More operations here>
-          //      scf.if %true {
+          //      fir.if %true {
           //          ^%lpv_update_blk
           //      }
           //  }
           // }
           //
           // To keep code consistency while handling privatization
-          // through this control flow, add a `scf.if` operation
+          // through this control flow, add a `fir.if` operation
           // that always evaluates to true, in order to create
           // a dedicated sub-region in `omp.section` where
           // lastprivate FIR can reside. Later canonicalizations
           // will optimize away this operation.
-
-          mlir::scf::IfOp ifOp = firOpBuilder.create<mlir::scf::IfOp>(
-              op->getLoc(),
-              firOpBuilder.createIntegerConstant(
-                  op->getLoc(), firOpBuilder.getIntegerType(1), 0x1),
-              /*else*/ false);
-          firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-
-          const Fortran::parser::OpenMPConstruct *parentOmpConstruct =
-              eval.parentConstruct->getIf<Fortran::parser::OpenMPConstruct>();
-          assert(parentOmpConstruct &&
-                 "Expected a valid enclosing OpenMP construct");
-          const Fortran::parser::OpenMPSectionsConstruct *sectionsConstruct =
-              std::get_if<Fortran::parser::OpenMPSectionsConstruct>(
-                  &parentOmpConstruct->u);
-          assert(sectionsConstruct &&
-                 "Expected an enclosing omp.sections construct");
-          const Fortran::parser::OmpClauseList &sectionsEndClauseList =
-              std::get<Fortran::parser::OmpClauseList>(
-                  std::get<Fortran::parser::OmpEndSectionsDirective>(
-                      sectionsConstruct->t)
-                      .t);
-          for (const Fortran::parser::OmpClause &otherClause :
-               sectionsEndClauseList.v)
-            if (std::get_if<Fortran::parser::OmpClause::Nowait>(&otherClause.u))
-              // Emit implicit barrier to synchronize threads and avoid data
-              // races on post-update of lastprivate variables when `nowait`
-              // clause is present.
-              firOpBuilder.create<mlir::omp::BarrierOp>(
-                  converter.getCurrentLocation());
-          firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-          lastPrivIP = firOpBuilder.saveInsertionPoint();
-          firOpBuilder.setInsertionPoint(ifOp);
-          insPt = firOpBuilder.saveInsertionPoint();
+          if (!eval.lowerAsUnstructured()) {
+            auto ifOp = firOpBuilder.create<fir::IfOp>(
+                op->getLoc(),
+                firOpBuilder.createIntegerConstant(
+                    op->getLoc(), firOpBuilder.getIntegerType(1), 0x1),
+                /*else*/ false);
+            firOpBuilder.setInsertionPointToStart(
+                &ifOp.getThenRegion().front());
+
+            const Fortran::parser::OpenMPConstruct *parentOmpConstruct =
+                eval.parentConstruct->getIf<Fortran::parser::OpenMPConstruct>();
+            assert(parentOmpConstruct &&
+                   "Expected a valid enclosing OpenMP construct");
+            const Fortran::parser::OpenMPSectionsConstruct *sectionsConstruct =
+                std::get_if<Fortran::parser::OpenMPSectionsConstruct>(
+                    &parentOmpConstruct->u);
+            assert(sectionsConstruct &&
+                   "Expected an enclosing omp.sections construct");
+            const Fortran::parser::OmpClauseList &sectionsEndClauseList =
+                std::get<Fortran::parser::OmpClauseList>(
+                    std::get<Fortran::parser::OmpEndSectionsDirective>(
+                        sectionsConstruct->t)
+                        .t);
+            for (const Fortran::parser::OmpClause &otherClause :
+                 sectionsEndClauseList.v)
+              if (std::get_if<Fortran::parser::OmpClause::Nowait>(
+                      &otherClause.u))
+                // Emit implicit barrier to synchronize threads and avoid data
+                // races on post-update of lastprivate variables when `nowait`
+                // clause is present.
+                firOpBuilder.create<mlir::omp::BarrierOp>(
+                    converter.getCurrentLocation());
+            firOpBuilder.setInsertionPointToStart(
+                &ifOp.getThenRegion().front());
+            lastPrivIP = firOpBuilder.saveInsertionPoint();
+            firOpBuilder.setInsertionPoint(ifOp);
+            insPt = firOpBuilder.saveInsertionPoint();
+          } else {
+            // Lastprivate operation is inserted at the end
+            // of the lexically last section in the sections
+            // construct
+            mlir::OpBuilder::InsertPoint unstructuredSectionsIP =
+                firOpBuilder.saveInsertionPoint();
+            firOpBuilder.setInsertionPointToStart(&op->getRegion(0).back());
+            lastPrivIP = firOpBuilder.saveInsertionPoint();
+            firOpBuilder.restoreInsertionPoint(unstructuredSectionsIP);
+          }
         }
       } else if (mlir::isa<omp::WsLoopOp>(op)) {
         mlir::Operation *lastOper = op->getRegion(0).back().getTerminator();
@@ -280,7 +294,7 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
         // omp.wsloop {            ...
         //    ...                  store
         //    store       ===>     %cmp = llvm.icmp "eq" %iv %ub
-        //    omp.yield            scf.if %cmp {
+        //    omp.yield            fir.if %cmp {
         // }                         ^%lpv_update_blk:
         //                         }
         //                         omp.yield
@@ -295,8 +309,8 @@ void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
               op->getRegion(0).front().getArguments()[0],
               mlir::dyn_cast<mlir::omp::WsLoopOp>(op).getUpperBound()[0]);
         }
-        mlir::scf::IfOp ifOp = firOpBuilder.create<mlir::scf::IfOp>(
-            op->getLoc(), cmpOp, /*else*/ false);
+        auto ifOp =
+            firOpBuilder.create<fir::IfOp>(op->getLoc(), cmpOp, /*else*/ false);
         firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
         lastPrivIP = firOpBuilder.saveInsertionPoint();
       } else {
@@ -1181,6 +1195,18 @@ static Value getReductionInitValue(mlir::Location loc, mlir::Type type,
     unsigned bits = type.getIntOrFloatBitWidth();
     int64_t maxInt = llvm::APInt::getSignedMaxValue(bits).getSExtValue();
     return builder.createIntegerConstant(loc, type, maxInt);
+  } else if (reductionOpName.contains("ior")) {
+    unsigned bits = type.getIntOrFloatBitWidth();
+    int64_t zeroInt = llvm::APInt::getZero(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, type, zeroInt);
+  } else if (reductionOpName.contains("ieor")) {
+    unsigned bits = type.getIntOrFloatBitWidth();
+    int64_t zeroInt = llvm::APInt::getZero(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, type, zeroInt);
+  } else if (reductionOpName.contains("iand")) {
+    unsigned bits = type.getIntOrFloatBitWidth();
+    int64_t allOnInt = llvm::APInt::getAllOnes(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, type, allOnInt);
   } else {
     if (type.isa<FloatType>())
       return builder.create<mlir::arith::ConstantOp>(
@@ -1268,6 +1294,15 @@ createReductionDecl(fir::FirOpBuilder &builder, llvm::StringRef reductionOpName,
       reductionOp =
           getReductionOperation<mlir::arith::MinFOp, mlir::arith::MinSIOp>(
               builder, type, loc, op1, op2);
+    } else if (name->source == "ior") {
+      assert((type.isIntOrIndex()) && "only integer is expected");
+      reductionOp = builder.create<mlir::arith::OrIOp>(loc, op1, op2);
+    } else if (name->source == "ieor") {
+      assert((type.isIntOrIndex()) && "only integer is expected");
+      reductionOp = builder.create<mlir::arith::XOrIOp>(loc, op1, op2);
+    } else if (name->source == "iand") {
+      assert((type.isIntOrIndex()) && "only integer is expected");
+      reductionOp = builder.create<mlir::arith::AndIOp>(loc, op1, op2);
     } else {
       TODO(loc, "Reduction of some intrinsic operators is not supported");
     }
@@ -1594,7 +1629,9 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                          &redOperator.u)) {
         if (const auto *name{Fortran::parser::Unwrap<Fortran::parser::Name>(
                 reductionIntrinsic)}) {
-          if ((name->source != "max") && (name->source != "min")) {
+          if ((name->source != "max") && (name->source != "min") &&
+              (name->source != "ior") && (name->source != "ieor") &&
+              (name->source != "iand")) {
             TODO(currentLocation,
                  "Reduction of intrinsic procedures is not supported");
           }
@@ -1890,6 +1927,36 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   }
 }
 
+static bool checkForSingleVariableOnRHS(
+    const Fortran::parser::AssignmentStmt &assignmentStmt) {
+  // Check if the assignment statement has a single variable on the RHS
+  const Fortran::parser::Expr &expr{
+      std::get<Fortran::parser::Expr>(assignmentStmt.t)};
+  const Fortran::common::Indirection<Fortran::parser::Designator> *designator =
+      std::get_if<Fortran::common::Indirection<Fortran::parser::Designator>>(
+          &expr.u);
+  const Fortran::parser::Name *name =
+      designator ? getDesignatorNameIfDataRef(designator->value()) : nullptr;
+  return name != nullptr;
+}
+
+static bool
+checkForSymbolMatch(const Fortran::parser::AssignmentStmt &assignmentStmt) {
+  // Check if the symbol on the LHS of the assignment statement is present in
+  // the RHS expression
+  const auto &var{std::get<Fortran::parser::Variable>(assignmentStmt.t)};
+  const auto &expr{std::get<Fortran::parser::Expr>(assignmentStmt.t)};
+  const auto *e{Fortran::semantics::GetExpr(expr)};
+  const auto *v{Fortran::semantics::GetExpr(var)};
+  const Fortran::semantics::Symbol &varSymbol =
+      Fortran::evaluate::GetSymbolVector(*v).front();
+  for (const Fortran::semantics::Symbol &symbol :
+       Fortran::evaluate::GetSymbolVector(*e))
+    if (varSymbol == symbol)
+      return true;
+  return false;
+}
+
 static void genOmpAtomicHintAndMemoryOrderClauses(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::OmpAtomicClauseList &clauseList,
@@ -1928,20 +1995,68 @@ static void genOmpAtomicHintAndMemoryOrderClauses(
   }
 }
 
+static void genOmpAtomicCaptureStatement(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::pft::Evaluation &eval, mlir::Value from_address,
+    mlir::Value to_address,
+    const Fortran::parser::OmpAtomicClauseList *leftHandClauseList,
+    const Fortran::parser::OmpAtomicClauseList *rightHandClauseList,
+    mlir::Type elementType) {
+  // Generate `omp.atomic.read` operation for atomic assigment statements
+  auto &firOpBuilder = converter.getFirOpBuilder();
+  auto currentLocation = converter.getCurrentLocation();
+
+  // If no hint clause is specified, the effect is as if
+  // hint(omp_sync_hint_none) had been specified.
+  mlir::IntegerAttr hint = nullptr;
+
+  mlir::omp::ClauseMemoryOrderKindAttr memory_order = nullptr;
+  if (leftHandClauseList)
+    genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, hint,
+                                          memory_order);
+  if (rightHandClauseList)
+    genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, hint,
+                                          memory_order);
+  firOpBuilder.create<mlir::omp::AtomicReadOp>(
+      currentLocation, from_address, to_address,
+      mlir::TypeAttr::get(elementType), hint, memory_order);
+}
+
+static void genOmpAtomicWriteStatement(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::pft::Evaluation &eval, mlir::Value lhs_addr,
+    mlir::Value rhs_expr,
+    const Fortran::parser::OmpAtomicClauseList *leftHandClauseList,
+    const Fortran::parser::OmpAtomicClauseList *rightHandClauseList,
+    mlir::Value *evaluatedExprValue = nullptr) {
+  // Generate `omp.atomic.write` operation for atomic assignment statements
+  auto &firOpBuilder = converter.getFirOpBuilder();
+  auto currentLocation = converter.getCurrentLocation();
+  // If no hint clause is specified, the effect is as if
+  // hint(omp_sync_hint_none) had been specified.
+  mlir::IntegerAttr hint = nullptr;
+  mlir::omp::ClauseMemoryOrderKindAttr memory_order = nullptr;
+  if (leftHandClauseList)
+    genOmpAtomicHintAndMemoryOrderClauses(converter, *leftHandClauseList, hint,
+                                          memory_order);
+  if (rightHandClauseList)
+    genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, hint,
+                                          memory_order);
+  firOpBuilder.create<mlir::omp::AtomicWriteOp>(currentLocation, lhs_addr,
+                                                rhs_expr, hint, memory_order);
+}
+
 static void genOmpAtomicUpdateStatement(
     Fortran::lower::AbstractConverter &converter,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::Variable &assignmentStmtVariable,
+    Fortran::lower::pft::Evaluation &eval, mlir::Value lhs_addr,
+    mlir::Type varType, const Fortran::parser::Variable &assignmentStmtVariable,
     const Fortran::parser::Expr &assignmentStmtExpr,
     const Fortran::parser::OmpAtomicClauseList *leftHandClauseList,
     const Fortran::parser::OmpAtomicClauseList *rightHandClauseList) {
   // Generate `omp.atomic.update` operation for atomic assignment statements
   auto &firOpBuilder = converter.getFirOpBuilder();
   auto currentLocation = converter.getCurrentLocation();
-  Fortran::lower::StatementContext stmtCtx;
 
-  mlir::Value address = fir::getBase(converter.genExprAddr(
-      *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx));
   // If no hint clause is specified, the effect is as if
   // hint(omp_sync_hint_none) had been specified.
   mlir::IntegerAttr hint = nullptr;
@@ -1953,16 +2068,11 @@ static void genOmpAtomicUpdateStatement(
     genOmpAtomicHintAndMemoryOrderClauses(converter, *rightHandClauseList, hint,
                                           memoryOrder);
   auto atomicUpdateOp = firOpBuilder.create<mlir::omp::AtomicUpdateOp>(
-      currentLocation, address, hint, memoryOrder);
+      currentLocation, lhs_addr, hint, memoryOrder);
 
   //// Generate body of Atomic Update operation
   // If an argument for the region is provided then create the block with that
   // argument. Also update the symbol's address with the argument mlir value.
-  mlir::Type varType =
-      fir::getBase(
-          converter.genExprValue(
-              *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx))
-          .getType();
   SmallVector<Type> varTys = {varType};
   SmallVector<Location> locs = {currentLocation};
   firOpBuilder.createBlock(&atomicUpdateOp.getRegion(), {}, varTys, locs);
@@ -1974,6 +2084,9 @@ static void genOmpAtomicUpdateStatement(
   assert(varDesignator && "Variable designator for atomic update assignment "
                           "statement does not exist");
   const auto *name = getDesignatorNameIfDataRef(varDesignator->value());
+  if (!name)
+    TODO(converter.getCurrentLocation(),
+         "Array references as atomic update variable");
   assert(name && name->symbol &&
          "No symbol attached to atomic update variable");
   converter.bindSymbol(*name->symbol, val);
@@ -1982,10 +2095,11 @@ static void genOmpAtomicUpdateStatement(
   mlir::Block &block = atomicUpdateOp.getRegion().back();
   firOpBuilder.setInsertionPointToEnd(&block);
 
-  mlir::Value result = fir::getBase(converter.genExprValue(
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::Value rhs_expr = fir::getBase(converter.genExprValue(
       *Fortran::semantics::GetExpr(assignmentStmtExpr), stmtCtx));
   mlir::Value convertResult =
-      firOpBuilder.createConvert(currentLocation, varType, result);
+      firOpBuilder.createConvert(currentLocation, varType, rhs_expr);
   // Insert the terminator: YieldOp.
   firOpBuilder.create<mlir::omp::YieldOp>(currentLocation, convertResult);
   // Reset the insert point to before the terminator.
@@ -1996,8 +2110,6 @@ static void
 genOmpAtomicWrite(Fortran::lower::AbstractConverter &converter,
                   Fortran::lower::pft::Evaluation &eval,
                   const Fortran::parser::OmpAtomicWrite &atomicWrite) {
-  auto &firOpBuilder = converter.getFirOpBuilder();
-  auto currentLocation = converter.getCurrentLocation();
   // Get the value and address of atomic write operands.
   const Fortran::parser::OmpAtomicClauseList &rightHandClauseList =
       std::get<2>(atomicWrite.t);
@@ -2008,27 +2120,18 @@ genOmpAtomicWrite(Fortran::lower::AbstractConverter &converter,
   const auto &assignmentStmtVariable = std::get<Fortran::parser::Variable>(
       std::get<3>(atomicWrite.t).statement.t);
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value value = fir::getBase(converter.genExprValue(
+  // Get the value and address of atomic write operands.
+  mlir::Value rhs_expr = fir::getBase(converter.genExprValue(
       *Fortran::semantics::GetExpr(assignmentStmtExpr), stmtCtx));
-  mlir::Value address = fir::getBase(converter.genExprAddr(
+  mlir::Value lhs_addr = fir::getBase(converter.genExprAddr(
       *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx));
-  // If no hint clause is specified, the effect is as if
-  // hint(omp_sync_hint_none) had been specified.
-  mlir::IntegerAttr hint = nullptr;
-  mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr;
-  genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint,
-                                        memoryOrder);
-  genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint,
-                                        memoryOrder);
-  firOpBuilder.create<mlir::omp::AtomicWriteOp>(currentLocation, address, value,
-                                                hint, memoryOrder);
+  genOmpAtomicWriteStatement(converter, eval, lhs_addr, rhs_expr,
+                             &leftHandClauseList, &rightHandClauseList);
 }
 
 static void genOmpAtomicRead(Fortran::lower::AbstractConverter &converter,
                              Fortran::lower::pft::Evaluation &eval,
                              const Fortran::parser::OmpAtomicRead &atomicRead) {
-  auto &firOpBuilder = converter.getFirOpBuilder();
-  auto currentLocation = converter.getCurrentLocation();
   // Get the address of atomic read operands.
   const Fortran::parser::OmpAtomicClauseList &rightHandClauseList =
       std::get<2>(atomicRead.t);
@@ -2047,17 +2150,9 @@ static void genOmpAtomicRead(Fortran::lower::AbstractConverter &converter,
       fir::getBase(converter.genExprAddr(fromExpr, stmtCtx));
   mlir::Value toAddress = fir::getBase(converter.genExprAddr(
       *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx));
-  // If no hint clause is specified, the effect is as if
-  // hint(omp_sync_hint_none) had been specified.
-  mlir::IntegerAttr hint = nullptr;
-  mlir::omp::ClauseMemoryOrderKindAttr memoryOrder = nullptr;
-  genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint,
-                                        memoryOrder);
-  genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint,
-                                        memoryOrder);
-  firOpBuilder.create<mlir::omp::AtomicReadOp>(
-      currentLocation, fromAddress, toAddress, mlir::TypeAttr::get(elementType),
-      hint, memoryOrder);
+  genOmpAtomicCaptureStatement(converter, eval, fromAddress, toAddress,
+                               &leftHandClauseList, &rightHandClauseList,
+                               elementType);
 }
 
 static void
@@ -2073,9 +2168,17 @@ genOmpAtomicUpdate(Fortran::lower::AbstractConverter &converter,
   const auto &assignmentStmtVariable = std::get<Fortran::parser::Variable>(
       std::get<3>(atomicUpdate.t).statement.t);
 
-  genOmpAtomicUpdateStatement(converter, eval, assignmentStmtVariable,
-                              assignmentStmtExpr, &leftHandClauseList,
-                              &rightHandClauseList);
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::Value lhs_addr = fir::getBase(converter.genExprAddr(
+      *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx));
+  mlir::Type varType =
+      fir::getBase(
+          converter.genExprValue(
+              *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx))
+          .getType();
+  genOmpAtomicUpdateStatement(converter, eval, lhs_addr, varType,
+                              assignmentStmtVariable, assignmentStmtExpr,
+                              &leftHandClauseList, &rightHandClauseList);
 }
 
 static void genOmpAtomic(Fortran::lower::AbstractConverter &converter,
@@ -2091,10 +2194,137 @@ static void genOmpAtomic(Fortran::lower::AbstractConverter &converter,
       std::get<Fortran::parser::Statement<Fortran::parser::AssignmentStmt>>(
           atomicConstruct.t)
           .statement.t);
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::Value lhs_addr = fir::getBase(converter.genExprAddr(
+      *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx));
+  mlir::Type varType =
+      fir::getBase(
+          converter.genExprValue(
+              *Fortran::semantics::GetExpr(assignmentStmtVariable), stmtCtx))
+          .getType();
   // If atomic-clause is not present on the construct, the behaviour is as if
   // the update clause is specified
-  genOmpAtomicUpdateStatement(converter, eval, assignmentStmtVariable,
-                              assignmentStmtExpr, &atomicClauseList, nullptr);
+  genOmpAtomicUpdateStatement(converter, eval, lhs_addr, varType,
+                              assignmentStmtVariable, assignmentStmtExpr,
+                              &atomicClauseList, nullptr);
+}
+
+static void
+genOmpAtomicCapture(Fortran::lower::AbstractConverter &converter,
+                    Fortran::lower::pft::Evaluation &eval,
+                    const Fortran::parser::OmpAtomicCapture &atomicCapture) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  mlir::Location currentLocation = converter.getCurrentLocation();
+
+  mlir::IntegerAttr hint = nullptr;
+  mlir::omp::ClauseMemoryOrderKindAttr memory_order = nullptr;
+  const Fortran::parser::OmpAtomicClauseList &rightHandClauseList =
+      std::get<2>(atomicCapture.t);
+  const Fortran::parser::OmpAtomicClauseList &leftHandClauseList =
+      std::get<0>(atomicCapture.t);
+  genOmpAtomicHintAndMemoryOrderClauses(converter, leftHandClauseList, hint,
+                                        memory_order);
+  genOmpAtomicHintAndMemoryOrderClauses(converter, rightHandClauseList, hint,
+                                        memory_order);
+
+  const Fortran::parser::AssignmentStmt &stmt1 =
+      std::get<3>(atomicCapture.t).v.statement;
+  const auto &stmt1Var{std::get<Fortran::parser::Variable>(stmt1.t)};
+  const auto &stmt1Expr{std::get<Fortran::parser::Expr>(stmt1.t)};
+  const Fortran::parser::AssignmentStmt &stmt2 =
+      std::get<4>(atomicCapture.t).v.statement;
+  const auto &stmt2Var{std::get<Fortran::parser::Variable>(stmt2.t)};
+  const auto &stmt2Expr{std::get<Fortran::parser::Expr>(stmt2.t)};
+
+  // Pre-evaluate expressions to be used in the various operations inside
+  // `omp.atomic.capture` since it is not desirable to have anything other than
+  // a `omp.atomic.read`, `omp.atomic.write`, or `omp.atomic.update` operation
+  // inside `omp.atomic.capture`
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::Value stmt1LHSArg, stmt1RHSArg, stmt2LHSArg, stmt2RHSArg;
+  mlir::Type elementType;
+  // LHS evaluations are common to all combinations of `omp.atomic.capture`
+  stmt1LHSArg = fir::getBase(
+      converter.genExprAddr(*Fortran::semantics::GetExpr(stmt1Var), stmtCtx));
+  stmt2LHSArg = fir::getBase(
+      converter.genExprAddr(*Fortran::semantics::GetExpr(stmt2Var), stmtCtx));
+
+  // Operation specific RHS evaluations
+  if (checkForSingleVariableOnRHS(stmt1)) {
+    // Atomic capture construct is of the form [capture-stmt, update-stmt] or
+    // of the form [capture-stmt, write-stmt]
+    stmt1RHSArg = fir::getBase(converter.genExprAddr(
+        *Fortran::semantics::GetExpr(stmt1Expr), stmtCtx));
+    stmt2RHSArg = fir::getBase(converter.genExprValue(
+        *Fortran::semantics::GetExpr(stmt2Expr), stmtCtx));
+
+  } else {
+    // Atomic capture construct is of the form [update-stmt, capture-stmt]
+    stmt1RHSArg = fir::getBase(converter.genExprValue(
+        *Fortran::semantics::GetExpr(stmt1Expr), stmtCtx));
+    stmt2RHSArg = fir::getBase(converter.genExprAddr(
+        *Fortran::semantics::GetExpr(stmt2Expr), stmtCtx));
+  }
+  // Type information used in generation of `omp.atomic.update` operation
+  mlir::Type stmt1VarType =
+      fir::getBase(converter.genExprValue(
+                       *Fortran::semantics::GetExpr(stmt1Var), stmtCtx))
+          .getType();
+  mlir::Type stmt2VarType =
+      fir::getBase(converter.genExprValue(
+                       *Fortran::semantics::GetExpr(stmt2Var), stmtCtx))
+          .getType();
+
+  auto atomicCaptureOp = firOpBuilder.create<mlir::omp::AtomicCaptureOp>(
+      currentLocation, hint, memory_order);
+  firOpBuilder.createBlock(&atomicCaptureOp.getRegion());
+  mlir::Block &block = atomicCaptureOp.getRegion().back();
+  firOpBuilder.setInsertionPointToStart(&block);
+  if (checkForSingleVariableOnRHS(stmt1)) {
+    if (checkForSymbolMatch(stmt2)) {
+      // Atomic capture construct is of the form [capture-stmt, update-stmt]
+      const Fortran::semantics::SomeExpr &fromExpr =
+          *Fortran::semantics::GetExpr(stmt1Expr);
+      elementType = converter.genType(fromExpr);
+      genOmpAtomicCaptureStatement(converter, eval, stmt1RHSArg, stmt1LHSArg,
+                                   /*leftHandClauseList=*/nullptr,
+                                   /*rightHandClauseList=*/nullptr,
+                                   elementType);
+      genOmpAtomicUpdateStatement(converter, eval, stmt1RHSArg, stmt2VarType,
+                                  stmt2Var, stmt2Expr,
+                                  /*leftHandClauseList=*/nullptr,
+                                  /*rightHandClauseList=*/nullptr);
+    } else {
+      // Atomic capture construct is of the form [capture-stmt, write-stmt]
+      const Fortran::semantics::SomeExpr &fromExpr =
+          *Fortran::semantics::GetExpr(stmt1Expr);
+      elementType = converter.genType(fromExpr);
+      genOmpAtomicCaptureStatement(converter, eval, stmt1RHSArg, stmt1LHSArg,
+                                   /*leftHandClauseList=*/nullptr,
+                                   /*rightHandClauseList=*/nullptr,
+                                   elementType);
+      genOmpAtomicWriteStatement(converter, eval, stmt1RHSArg, stmt2RHSArg,
+                                 /*leftHandClauseList=*/nullptr,
+                                 /*rightHandClauseList=*/nullptr);
+    }
+  } else {
+    // Atomic capture construct is of the form [update-stmt, capture-stmt]
+    firOpBuilder.setInsertionPointToEnd(&block);
+    const Fortran::semantics::SomeExpr &fromExpr =
+        *Fortran::semantics::GetExpr(stmt2Expr);
+    elementType = converter.genType(fromExpr);
+    genOmpAtomicCaptureStatement(converter, eval, stmt1LHSArg, stmt2LHSArg,
+                                 /*leftHandClauseList=*/nullptr,
+                                 /*rightHandClauseList=*/nullptr, elementType);
+    firOpBuilder.setInsertionPointToStart(&block);
+    genOmpAtomicUpdateStatement(converter, eval, stmt1LHSArg, stmt1VarType,
+                                stmt1Var, stmt1Expr,
+                                /*leftHandClauseList=*/nullptr,
+                                /*rightHandClauseList=*/nullptr);
+  }
+  firOpBuilder.setInsertionPointToEnd(&block);
+  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+  firOpBuilder.setInsertionPointToStart(&block);
 }
 
 static void
@@ -2114,8 +2344,8 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                  [&](const Fortran::parser::OmpAtomicUpdate &atomicUpdate) {
                    genOmpAtomicUpdate(converter, eval, atomicUpdate);
                  },
-                 [&](const auto &) {
-                   TODO(converter.getCurrentLocation(), "Atomic capture");
+                 [&](const Fortran::parser::OmpAtomicCapture &atomicCapture) {
+                   genOmpAtomicCapture(converter, eval, atomicCapture);
                  },
              },
              atomicConstruct.u);
@@ -2364,7 +2594,10 @@ void Fortran::lower::genOpenMPReduction(
                          &redOperator.u)) {
         if (const auto *name{Fortran::parser::Unwrap<Fortran::parser::Name>(
                 reductionIntrinsic)}) {
-          if ((name->source != "max") && (name->source != "min")) {
+          std::string redName = name->ToString();
+          if ((name->source != "max") && (name->source != "min") &&
+              (name->source != "ior") && (name->source != "ieor") &&
+              (name->source != "iand")) {
             continue;
           }
           for (const auto &ompObject : objectList.v) {
@@ -2383,12 +2616,21 @@ void Fortran::lower::genOpenMPReduction(
                         findReductionChain(loadVal, &reductionVal);
                     if (reductionOp == nullptr)
                       continue;
-                    assert(mlir::isa<mlir::arith::SelectOp>(reductionOp) &&
-                           "Selection Op not found in reduction intrinsic");
-                    mlir::Operation *compareOp =
-                        getCompareFromReductionOp(reductionOp, loadVal);
-                    updateReduction(compareOp, firOpBuilder, loadVal,
-                                    reductionVal);
+
+                    if (redName == "max" || redName == "min") {
+                      assert(mlir::isa<mlir::arith::SelectOp>(reductionOp) &&
+                             "Selection Op not found in reduction intrinsic");
+                      mlir::Operation *compareOp =
+                          getCompareFromReductionOp(reductionOp, loadVal);
+                      updateReduction(compareOp, firOpBuilder, loadVal,
+                                      reductionVal);
+                    }
+                    if (redName == "ior" || redName == "ieor" ||
+                        redName == "iand") {
+
+                      updateReduction(reductionOp, firOpBuilder, loadVal,
+                                      reductionVal);
+                    }
                   }
                 }
               }
diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp
index 2c63852ce95ee..ec4c7218b58f6 100644
--- a/flang/lib/Lower/PFTBuilder.cpp
+++ b/flang/lib/Lower/PFTBuilder.cpp
@@ -1451,6 +1451,14 @@ struct SymbolDependenceAnalysis {
     // in some cases. Non-dummy procedures don't.
     if (semantics::IsProcedure(sym) && !isProcedurePointerOrDummy)
       return 0;
+    // Derived type component symbols may be collected by "CollectSymbols"
+    // below when processing something like "real :: x(derived%component)". The
+    // symbol "component" has "ObjectEntityDetails", but it should not be
+    // instantiated: it is is part of "derived" that should be the only one to
+    // be instantiated.
+    if (sym.owner().IsDerivedType())
+      return 0;
+
     semantics::Symbol ultimate = sym.GetUltimate();
     if (const auto *details =
             ultimate.detailsIf<semantics::NamelistDetails>()) {
diff --git a/flang/lib/Optimizer/Builder/Character.cpp b/flang/lib/Optimizer/Builder/Character.cpp
index c9b0be948831b..eb3825c1d4d60 100644
--- a/flang/lib/Optimizer/Builder/Character.cpp
+++ b/flang/lib/Optimizer/Builder/Character.cpp
@@ -400,10 +400,19 @@ void fir::factory::CharacterExprHelper::createLengthOneAssign(
   auto addr = lhs.getBuffer();
   auto toTy = fir::unwrapRefType(addr.getType());
   mlir::Value val = rhs.getBuffer();
-  if (fir::isa_ref_type(val.getType()))
-    val = builder.create<fir::LoadOp>(loc, val);
-  val = builder.createConvert(loc, toTy, val);
-  builder.create<fir::StoreOp>(loc, val, addr);
+  if (fir::isa_ref_type(val.getType())) {
+    auto fromCharLen1RefTy = builder.getRefType(getSingletonCharType(
+        builder.getContext(),
+        getCharacterKind(fir::unwrapRefType(val.getType()))));
+    val = builder.create<fir::LoadOp>(
+        loc, builder.createConvert(loc, fromCharLen1RefTy, val));
+  }
+  auto toCharLen1Ty =
+      getSingletonCharType(builder.getContext(), getCharacterKind(toTy));
+  val = builder.createConvert(loc, toCharLen1Ty, val);
+  builder.create<fir::StoreOp>(
+      loc, val,
+      builder.createConvert(loc, builder.getRefType(toCharLen1Ty), addr));
 }
 
 /// Returns the minimum of integer mlir::Value \p a and \b.
@@ -418,11 +427,29 @@ void fir::factory::CharacterExprHelper::createAssign(
     const fir::CharBoxValue &lhs, const fir::CharBoxValue &rhs) {
   auto rhsCstLen = getCompileTimeLength(rhs);
   auto lhsCstLen = getCompileTimeLength(lhs);
-  bool compileTimeSameLength =
-      (lhsCstLen && rhsCstLen && *lhsCstLen == *rhsCstLen) ||
-      (rhs.getLen() == lhs.getLen());
-
-  if (compileTimeSameLength && lhsCstLen && *lhsCstLen == 1) {
+  bool compileTimeSameLength = false;
+  bool isLengthOneAssign = false;
+
+  if (lhsCstLen && rhsCstLen && *lhsCstLen == *rhsCstLen) {
+    compileTimeSameLength = true;
+    if (*lhsCstLen == 1)
+      isLengthOneAssign = true;
+  } else if (rhs.getLen() == lhs.getLen()) {
+    compileTimeSameLength = true;
+
+    // If the length values are the same for LHS and RHS,
+    // then we can rely on the constant length deduced from
+    // any of the two types.
+    if (lhsCstLen && *lhsCstLen == 1)
+      isLengthOneAssign = true;
+    if (rhsCstLen && *rhsCstLen == 1)
+      isLengthOneAssign = true;
+
+    // We could have recognized constant operations here (e.g.
+    // two different arith.constant ops may produce the same value),
+    // but for now leave it to CSE to get rid of the duplicates.
+  }
+  if (isLengthOneAssign) {
     createLengthOneAssign(lhs, rhs);
     return;
   }
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 020d6c7b27eb1..8ffff7ad7809a 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -70,6 +70,8 @@ mlir::Type fir::FirOpBuilder::getRealType(int kind) {
   switch (kindMap.getRealTypeID(kind)) {
   case llvm::Type::TypeID::HalfTyID:
     return mlir::FloatType::getF16(getContext());
+  case llvm::Type::TypeID::BFloatTyID:
+    return mlir::FloatType::getBF16(getContext());
   case llvm::Type::TypeID::FloatTyID:
     return mlir::FloatType::getF32(getContext());
   case llvm::Type::TypeID::DoubleTyID:
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 9b9174eea80ae..1c2e3cfffd111 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -23,8 +23,9 @@
 
 // Return explicit extents. If the base is a fir.box, this won't read it to
 // return the extents and will instead return an empty vector.
-static llvm::SmallVector<mlir::Value>
-getExplicitExtentsFromShape(mlir::Value shape, fir::FirOpBuilder &builder) {
+llvm::SmallVector<mlir::Value>
+hlfir::getExplicitExtentsFromShape(mlir::Value shape,
+                                   fir::FirOpBuilder &builder) {
   llvm::SmallVector<mlir::Value> result;
   auto *shapeOp = shape.getDefiningOp();
   if (auto s = mlir::dyn_cast_or_null<fir::ShapeOp>(shapeOp)) {
@@ -36,11 +37,22 @@ getExplicitExtentsFromShape(mlir::Value shape, fir::FirOpBuilder &builder) {
   } else if (mlir::dyn_cast_or_null<fir::ShiftOp>(shapeOp)) {
     return {};
   } else if (auto s = mlir::dyn_cast_or_null<hlfir::ShapeOfOp>(shapeOp)) {
+    hlfir::ExprType expr = s.getExpr().getType().cast<hlfir::ExprType>();
+    llvm::ArrayRef<int64_t> exprShape = expr.getShape();
+    mlir::Type indexTy = builder.getIndexType();
     fir::ShapeType shapeTy = shape.getType().cast<fir::ShapeType>();
     result.reserve(shapeTy.getRank());
     for (unsigned i = 0; i < shapeTy.getRank(); ++i) {
-      auto op = builder.create<hlfir::GetExtentOp>(shape.getLoc(), shape, i);
-      result.emplace_back(op.getResult());
+      int64_t extent = exprShape[i];
+      mlir::Value extentVal;
+      if (extent == expr.getUnknownExtent()) {
+        auto op = builder.create<hlfir::GetExtentOp>(shape.getLoc(), shape, i);
+        extentVal = op.getResult();
+      } else {
+        extentVal =
+            builder.createIntegerConstant(shape.getLoc(), indexTy, extent);
+      }
+      result.emplace_back(extentVal);
     }
   } else {
     TODO(shape.getLoc(), "read fir.shape to get extents");
@@ -51,7 +63,7 @@ static llvm::SmallVector<mlir::Value>
 getExplicitExtents(fir::FortranVariableOpInterface var,
                    fir::FirOpBuilder &builder) {
   if (mlir::Value shape = var.getShape())
-    return getExplicitExtentsFromShape(var.getShape(), builder);
+    return hlfir::getExplicitExtentsFromShape(var.getShape(), builder);
   return {};
 }
 
@@ -258,6 +270,8 @@ mlir::Value hlfir::genVariableRawAddress(mlir::Location loc,
   if (var.isMutableBox())
     baseAddr = builder.create<fir::LoadOp>(loc, baseAddr);
   // Get raw address.
+  if (var.getType().isa<fir::BoxCharType>())
+    baseAddr = genUnboxChar(loc, builder, var.getBase()).getAddr();
   if (baseAddr.getType().isa<fir::BaseBoxType>())
     baseAddr = builder.create<fir::BoxAddrOp>(loc, baseAddr);
   return baseAddr;
@@ -298,9 +312,12 @@ hlfir::Entity hlfir::genVariableBox(mlir::Location loc,
       var.getFortranElementType().dyn_cast<fir::CharacterType>();
   if (!maybeCharType || maybeCharType.hasDynamicLen())
     hlfir::genLengthParameters(loc, builder, var, typeParams);
+  mlir::Value addr = var.getBase();
+  if (var.getType().isa<fir::BoxCharType>())
+    addr = genVariableRawAddress(loc, builder, var);
   mlir::Type boxType = fir::BoxType::get(var.getElementOrSequenceType());
   auto embox =
-      builder.create<fir::EmboxOp>(loc, boxType, var, shape,
+      builder.create<fir::EmboxOp>(loc, boxType, addr, shape,
                                    /*slice=*/mlir::Value{}, typeParams);
   return hlfir::Entity{embox.getResult()};
 }
@@ -393,7 +410,7 @@ hlfir::genBounds(mlir::Location loc, fir::FirOpBuilder &builder,
   assert((shape.getType().isa<fir::ShapeShiftType>() ||
           shape.getType().isa<fir::ShapeType>()) &&
          "shape must contain extents");
-  auto extents = getExplicitExtentsFromShape(shape, builder);
+  auto extents = hlfir::getExplicitExtentsFromShape(shape, builder);
   auto lowers = getExplicitLboundsFromShape(shape);
   assert(lowers.empty() || lowers.size() == extents.size());
   mlir::Type idxTy = builder.getIndexType();
@@ -516,7 +533,7 @@ llvm::SmallVector<mlir::Value>
 hlfir::getIndexExtents(mlir::Location loc, fir::FirOpBuilder &builder,
                        mlir::Value shape) {
   llvm::SmallVector<mlir::Value> extents =
-      getExplicitExtentsFromShape(shape, builder);
+      hlfir::getExplicitExtentsFromShape(shape, builder);
   mlir::Type indexType = builder.getIndexType();
   for (auto &extent : extents)
     extent = builder.createConvert(loc, indexType, extent);
@@ -527,7 +544,7 @@ mlir::Value hlfir::genExtent(mlir::Location loc, fir::FirOpBuilder &builder,
                              hlfir::Entity entity, unsigned dim) {
   entity = followShapeInducingSource(entity);
   if (auto shape = tryRetrievingShapeOrShift(entity)) {
-    auto extents = getExplicitExtentsFromShape(shape, builder);
+    auto extents = hlfir::getExplicitExtentsFromShape(shape, builder);
     if (!extents.empty()) {
       assert(extents.size() > dim && "bad inquiry");
       return extents[dim];
@@ -779,7 +796,7 @@ translateVariableToExtendedValue(mlir::Location loc, fir::FirOpBuilder &builder,
 
   if (firBase.getType().isa<fir::BaseBoxType>()) {
     if (!variable.isSimplyContiguous() || variable.isPolymorphic() ||
-        variable.isDerivedWithLengthParameters()) {
+        variable.isDerivedWithLengthParameters() || variable.isOptional()) {
       llvm::SmallVector<mlir::Value> nonDefaultLbounds =
           getNonDefaultLowerBounds(loc, builder, variable);
       return fir::BoxValue(firBase, nonDefaultLbounds,
@@ -917,3 +934,52 @@ hlfir::convertToAddress(mlir::Location loc, fir::FirOpBuilder &builder,
     exv = placeTrivialInMemory(loc, builder, base, targetType);
   return {exv, cleanup};
 }
+
+/// Clone:
+/// ```
+/// hlfir.elemental_addr %shape : !fir.shape<1> {
+///   ^bb0(%i : index)
+///    .....
+///    %hlfir.yield %scalarAddress : fir.ref<T>
+/// }
+/// ```
+//
+/// into
+///
+/// ```
+/// %expr = hlfir.elemental %shape : (!fir.shape<1>) -> hlfir.expr<?xT> {
+///   ^bb0(%i : index)
+///    .....
+///    %value = fir.load %scalarAddress : fir.ref<T>
+///    %hlfir.yield_element %value : T
+///  }
+/// ```
+hlfir::ElementalOp
+hlfir::cloneToElementalOp(mlir::Location loc, fir::FirOpBuilder &builder,
+                          hlfir::ElementalAddrOp elementalAddrOp) {
+  hlfir::Entity scalarAddress =
+      hlfir::Entity{mlir::cast<hlfir::YieldOp>(
+                        elementalAddrOp.getBody().back().getTerminator())
+                        .getEntity()};
+  llvm::SmallVector<mlir::Value, 1> typeParams;
+  hlfir::genLengthParameters(loc, builder, scalarAddress, typeParams);
+
+  builder.setInsertionPointAfter(elementalAddrOp);
+  auto genKernel = [&](mlir::Location l, fir::FirOpBuilder &b,
+                       mlir::ValueRange oneBasedIndices) -> hlfir::Entity {
+    mlir::IRMapping mapper;
+    mapper.map(elementalAddrOp.getIndices(), oneBasedIndices);
+    mlir::Operation *newOp = nullptr;
+    for (auto &op : elementalAddrOp.getBody().back().getOperations())
+      newOp = b.clone(op, mapper);
+    auto newYielOp = mlir::dyn_cast_or_null<hlfir::YieldOp>(newOp);
+    assert(newYielOp && "hlfir.elemental_addr is ill formed");
+    hlfir::Entity newAddr{newYielOp.getEntity()};
+    newYielOp->erase();
+    return hlfir::loadTrivialScalar(l, b, newAddr);
+  };
+  mlir::Type elementType = scalarAddress.getFortranElementType();
+  return hlfir::genElementalOp(loc, builder, elementType,
+                               elementalAddrOp.getShape(), typeParams,
+                               genKernel);
+}
diff --git a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
index 813e32eeced97..b38c64cc790b2 100644
--- a/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
+++ b/flang/lib/Optimizer/CodeGen/TargetRewrite.cpp
@@ -91,6 +91,13 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
         mod.getContext(), fir::getTargetTriple(mod), fir::getKindMapping(mod));
     setMembers(specifics.get(), &rewriter);
 
+    // We may need to call stacksave/stackrestore later, so
+    // create the FuncOps beforehand.
+    fir::FirOpBuilder builder(rewriter, fir::getKindMapping(mod));
+    builder.setInsertionPointToStart(mod.getBody());
+    stackSaveFn = fir::factory::getLlvmStackSave(builder);
+    stackRestoreFn = fir::factory::getLlvmStackRestore(builder);
+
     // Perform type conversion on signatures and call sites.
     if (mlir::failed(convertTypes(mod))) {
       mlir::emitError(mlir::UnknownLoc::get(&context),
@@ -121,7 +128,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
   template <typename A, typename B, typename C>
   std::optional<std::function<mlir::Value(mlir::Operation *)>>
   rewriteCallComplexResultType(mlir::Location loc, A ty, B &newResTys,
-                               B &newInTys, C &newOpers) {
+                               B &newInTys, C &newOpers,
+                               mlir::Value &savedStackPtr) {
     if (noComplexConversion) {
       newResTys.push_back(ty);
       return std::nullopt;
@@ -134,6 +142,11 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     auto attr = std::get<fir::CodeGenSpecifics::Attributes>(m[0]);
     if (attr.isSRet()) {
       assert(fir::isa_ref_type(resTy) && "must be a memory reference type");
+      // Save the stack pointer, if it has not been saved for this call yet.
+      // We will need to restore it after the call, because the alloca
+      // needs to be deallocated.
+      if (!savedStackPtr)
+        savedStackPtr = genStackSave(loc);
       mlir::Value stack =
           rewriter->create<fir::AllocaOp>(loc, fir::dyn_cast_ptrEleTy(resTy));
       newInTys.push_back(resTy);
@@ -145,7 +158,10 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       };
     }
     newResTys.push_back(resTy);
-    return [=](mlir::Operation *call) -> mlir::Value {
+    return [=, &savedStackPtr](mlir::Operation *call) -> mlir::Value {
+      // We are going to generate an alloca, so save the stack pointer.
+      if (!savedStackPtr)
+        savedStackPtr = genStackSave(loc);
       auto mem = rewriter->create<fir::AllocaOp>(loc, resTy);
       rewriter->create<fir::StoreOp>(loc, call->getResult(0), mem);
       auto memTy = fir::ReferenceType::get(ty);
@@ -156,7 +172,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
 
   template <typename A, typename B, typename C>
   void rewriteCallComplexInputType(A ty, mlir::Value oper, B &newInTys,
-                                   C &newOpers) {
+                                   C &newOpers, mlir::Value &savedStackPtr) {
     if (noComplexConversion) {
       newInTys.push_back(ty);
       newOpers.push_back(oper);
@@ -173,6 +189,9 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       auto resTy = std::get<mlir::Type>(m[0]);
       auto attr = std::get<fir::CodeGenSpecifics::Attributes>(m[0]);
       auto oldRefTy = fir::ReferenceType::get(ty);
+      // We are going to generate an alloca, so save the stack pointer.
+      if (!savedStackPtr)
+        savedStackPtr = genStackSave(loc);
       if (attr.isByVal()) {
         auto mem = rewriter->create<fir::AllocaOp>(loc, ty);
         rewriter->create<fir::StoreOp>(loc, oper, mem);
@@ -210,6 +229,7 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     llvm::SmallVector<mlir::Type> newResTys;
     llvm::SmallVector<mlir::Type> newInTys;
     llvm::SmallVector<mlir::Value> newOpers;
+    mlir::Value savedStackPtr = nullptr;
 
     // If the call is indirect, the first argument must still be the function
     // to call.
@@ -231,11 +251,11 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       llvm::TypeSwitch<mlir::Type>(ty)
           .template Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
             wrap = rewriteCallComplexResultType(loc, cmplx, newResTys, newInTys,
-                                                newOpers);
+                                                newOpers, savedStackPtr);
           })
           .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
             wrap = rewriteCallComplexResultType(loc, cmplx, newResTys, newInTys,
-                                                newOpers);
+                                                newOpers, savedStackPtr);
           })
           .Default([&](mlir::Type ty) { newResTys.push_back(ty); });
     } else if (fnTy.getResults().size() > 1) {
@@ -290,10 +310,12 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
             }
           })
           .template Case<fir::ComplexType>([&](fir::ComplexType cmplx) {
-            rewriteCallComplexInputType(cmplx, oper, newInTys, newOpers);
+            rewriteCallComplexInputType(cmplx, oper, newInTys, newOpers,
+                                        savedStackPtr);
           })
           .template Case<mlir::ComplexType>([&](mlir::ComplexType cmplx) {
-            rewriteCallComplexInputType(cmplx, oper, newInTys, newOpers);
+            rewriteCallComplexInputType(cmplx, oper, newInTys, newOpers,
+                                        savedStackPtr);
           })
           .template Case<mlir::TupleType>([&](mlir::TupleType tuple) {
             if (fir::isCharacterProcedureTuple(tuple)) {
@@ -343,6 +365,8 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
     }
     newInTys.insert(newInTys.end(), trailingInTys.begin(), trailingInTys.end());
     newOpers.insert(newOpers.end(), trailingOpers.begin(), trailingOpers.end());
+
+    llvm::SmallVector<mlir::Value, 1> newCallResults;
     if constexpr (std::is_same_v<std::decay_t<A>, fir::CallOp>) {
       fir::CallOp newCall;
       if (callOp.getCallee()) {
@@ -357,18 +381,40 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
       }
       LLVM_DEBUG(llvm::dbgs() << "replacing call with " << newCall << '\n');
       if (wrap)
-        replaceOp(callOp, (*wrap)(newCall.getOperation()));
+        newCallResults.push_back((*wrap)(newCall.getOperation()));
       else
-        replaceOp(callOp, newCall.getResults());
+        newCallResults.append(newCall.result_begin(), newCall.result_end());
     } else {
       fir::DispatchOp dispatchOp = rewriter->create<A>(
           loc, newResTys, rewriter->getStringAttr(callOp.getMethod()),
           callOp.getOperands()[0], newOpers,
           rewriter->getI32IntegerAttr(*callOp.getPassArgPos() + passArgShift));
       if (wrap)
-        replaceOp(callOp, (*wrap)(dispatchOp.getOperation()));
+        newCallResults.push_back((*wrap)(dispatchOp.getOperation()));
       else
-        replaceOp(callOp, dispatchOp.getResults());
+        newCallResults.append(dispatchOp.result_begin(),
+                              dispatchOp.result_end());
+    }
+
+    if (newCallResults.size() <= 1) {
+      if (savedStackPtr) {
+        if (newCallResults.size() == 1) {
+          // We assume that all the allocas are inserted before
+          // the operation that defines the new call result.
+          rewriter->setInsertionPointAfterValue(newCallResults[0]);
+        } else {
+          // If the call does not have results, then insert
+          // stack restore after the original call operation.
+          rewriter->setInsertionPointAfter(callOp);
+        }
+        genStackRestore(loc, savedStackPtr);
+      }
+      replaceOp(callOp, newCallResults);
+    } else {
+      // The TODO is duplicated here to make sure this part
+      // handles the stackrestore insertion properly, if
+      // we add support for multiple call results.
+      TODO(loc, "multiple results not supported yet");
     }
   }
 
@@ -974,8 +1020,22 @@ class TargetRewrite : public fir::impl::TargetRewritePassBase<TargetRewrite> {
 
   inline void clearMembers() { setMembers(nullptr, nullptr); }
 
+  // Inserts a call to llvm.stacksave at the current insertion
+  // point and the given location. Returns the call's result Value.
+  inline mlir::Value genStackSave(mlir::Location loc) {
+    return rewriter->create<fir::CallOp>(loc, stackSaveFn).getResult(0);
+  }
+
+  // Inserts a call to llvm.stackrestore at the current insertion
+  // point and the given location and argument.
+  inline void genStackRestore(mlir::Location loc, mlir::Value sp) {
+    rewriter->create<fir::CallOp>(loc, stackRestoreFn, mlir::ValueRange{sp});
+  }
+
   fir::CodeGenSpecifics *specifics = nullptr;
   mlir::OpBuilder *rewriter = nullptr;
+  mlir::func::FuncOp stackSaveFn = nullptr;
+  mlir::func::FuncOp stackRestoreFn = nullptr;
 }; // namespace
 } // namespace
 
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index a479d64f39db8..7599b8d2756e0 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -489,35 +489,18 @@ void hlfir::ConcatOp::build(mlir::OpBuilder &builder,
 }
 
 //===----------------------------------------------------------------------===//
-// SetLengthOp
-//===----------------------------------------------------------------------===//
-
-void hlfir::SetLengthOp::build(mlir::OpBuilder &builder,
-                               mlir::OperationState &result, mlir::Value string,
-                               mlir::Value len) {
-  fir::CharacterType::LenType resultTypeLen = fir::CharacterType::unknownLen();
-  if (auto cstLen = fir::getIntIfConstant(len))
-    resultTypeLen = *cstLen;
-  unsigned kind = getCharacterKind(string.getType());
-  auto resultType = hlfir::ExprType::get(
-      builder.getContext(), hlfir::ExprType::Shape{},
-      fir::CharacterType::get(builder.getContext(), kind, resultTypeLen),
-      false);
-  build(builder, result, resultType, string, len);
-}
-
-//===----------------------------------------------------------------------===//
-// SumOp
+// ReductionOp
 //===----------------------------------------------------------------------===//
 
-mlir::LogicalResult hlfir::SumOp::verify() {
-  mlir::Operation *op = getOperation();
+template <typename ReductionOp>
+static mlir::LogicalResult verifyReductionOp(ReductionOp reductionOp) {
+  mlir::Operation *op = reductionOp->getOperation();
 
   auto results = op->getResultTypes();
   assert(results.size() == 1);
 
-  mlir::Value array = getArray();
-  mlir::Value mask = getMask();
+  mlir::Value array = reductionOp->getArray();
+  mlir::Value mask = reductionOp->getMask();
 
   fir::SequenceType arrayTy =
       hlfir::getFortranElementOrSequenceType(array.getType())
@@ -537,7 +520,7 @@ mlir::LogicalResult hlfir::SumOp::verify() {
 
     if (!maskShape.empty()) {
       if (maskShape.size() != arrayShape.size())
-        return emitWarning("MASK must be conformable to ARRAY");
+        return reductionOp->emitWarning("MASK must be conformable to ARRAY");
       static_assert(fir::SequenceType::getUnknownExtent() ==
                     hlfir::ExprType::getUnknownExtent());
       constexpr int64_t unknownExtent = fir::SequenceType::getUnknownExtent();
@@ -546,7 +529,7 @@ mlir::LogicalResult hlfir::SumOp::verify() {
         int64_t maskExtent = maskShape[i];
         if ((arrayExtent != maskExtent) && (arrayExtent != unknownExtent) &&
             (maskExtent != unknownExtent))
-          return emitWarning("MASK must be conformable to ARRAY");
+          return reductionOp->emitWarning("MASK must be conformable to ARRAY");
       }
     }
   }
@@ -554,24 +537,59 @@ mlir::LogicalResult hlfir::SumOp::verify() {
   if (resultTy.isArray()) {
     // Result is of the same type as ARRAY
     if (resultTy.getEleTy() != numTy)
-      return emitOpError(
+      return reductionOp->emitOpError(
           "result must have the same element type as ARRAY argument");
 
     llvm::ArrayRef<int64_t> resultShape = resultTy.getShape();
 
     // Result has rank n-1
     if (resultShape.size() != (arrayShape.size() - 1))
-      return emitOpError("result rank must be one less than ARRAY");
+      return reductionOp->emitOpError(
+          "result rank must be one less than ARRAY");
   } else {
     // Result is of the same type as ARRAY
     if (resultTy.getElementType() != numTy)
-      return emitOpError(
+      return reductionOp->emitOpError(
           "result must have the same element type as ARRAY argument");
   }
 
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// ProductOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult hlfir::ProductOp::verify() {
+  return verifyReductionOp<hlfir::ProductOp *>(this);
+}
+
+//===----------------------------------------------------------------------===//
+// SetLengthOp
+//===----------------------------------------------------------------------===//
+
+void hlfir::SetLengthOp::build(mlir::OpBuilder &builder,
+                               mlir::OperationState &result, mlir::Value string,
+                               mlir::Value len) {
+  fir::CharacterType::LenType resultTypeLen = fir::CharacterType::unknownLen();
+  if (auto cstLen = fir::getIntIfConstant(len))
+    resultTypeLen = *cstLen;
+  unsigned kind = getCharacterKind(string.getType());
+  auto resultType = hlfir::ExprType::get(
+      builder.getContext(), hlfir::ExprType::Shape{},
+      fir::CharacterType::get(builder.getContext(), kind, resultTypeLen),
+      false);
+  build(builder, result, resultType, string, len);
+}
+
+//===----------------------------------------------------------------------===//
+// SumOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult hlfir::SumOp::verify() {
+  return verifyReductionOp<hlfir::SumOp *>(this);
+}
+
 //===----------------------------------------------------------------------===//
 // MatmulOp
 //===----------------------------------------------------------------------===//
@@ -960,5 +978,294 @@ mlir::LogicalResult hlfir::GetExtentOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// RegionAssignOp
+//===----------------------------------------------------------------------===//
+
+/// Add a fir.end terminator to a parsed region if it does not already has a
+/// terminator.
+static void ensureTerminator(mlir::Region &region, mlir::Builder &builder,
+                             mlir::Location loc) {
+  // Borrow YielOp::ensureTerminator MLIR generated implementation to add a
+  // fir.end if there is no terminator. This has nothing to do with YielOp,
+  // other than the fact that yieldOp has the
+  // SingleBlocklicitTerminator<"fir::FirEndOp"> interface that
+  // cannot be added on other HLFIR operations with several regions which are
+  // not all terminated the same way.
+  hlfir::YieldOp::ensureTerminator(region, builder, loc);
+}
+
+mlir::ParseResult hlfir::RegionAssignOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  mlir::Region &rhsRegion = *result.addRegion();
+  if (parser.parseRegion(rhsRegion))
+    return mlir::failure();
+  mlir::Region &lhsRegion = *result.addRegion();
+  if (parser.parseKeyword("to") || parser.parseRegion(lhsRegion))
+    return mlir::failure();
+  mlir::Region &userDefinedAssignmentRegion = *result.addRegion();
+  if (succeeded(parser.parseOptionalKeyword("user_defined_assign"))) {
+    mlir::OpAsmParser::Argument rhsArg, lhsArg;
+    if (parser.parseLParen() || parser.parseArgument(rhsArg) ||
+        parser.parseColon() || parser.parseType(rhsArg.type) ||
+        parser.parseRParen() || parser.parseKeyword("to") ||
+        parser.parseLParen() || parser.parseArgument(lhsArg) ||
+        parser.parseColon() || parser.parseType(lhsArg.type) ||
+        parser.parseRParen())
+      return mlir::failure();
+    if (parser.parseRegion(userDefinedAssignmentRegion, {rhsArg, lhsArg}))
+      return mlir::failure();
+    ensureTerminator(userDefinedAssignmentRegion, parser.getBuilder(),
+                     result.location);
+  }
+  return mlir::success();
+}
+
+void hlfir::RegionAssignOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printRegion(getRhsRegion(), /*printEntryBlockArgs=*/false,
+                /*printBlockTerminators=*/true);
+  p << " to ";
+  p.printRegion(getLhsRegion(), /*printEntryBlockArgs=*/false,
+                /*printBlockTerminators=*/true);
+  if (!getUserDefinedAssignment().empty()) {
+    p << " user_defined_assign ";
+    mlir::Value userAssignmentRhs = getUserAssignmentRhs();
+    mlir::Value userAssignmentLhs = getUserAssignmentLhs();
+    p << " (" << userAssignmentRhs << ": " << userAssignmentRhs.getType()
+      << ") to (";
+    p << userAssignmentLhs << ": " << userAssignmentLhs.getType() << ") ";
+    p.printRegion(getUserDefinedAssignment(), /*printEntryBlockArgs=*/false,
+                  /*printBlockTerminators=*/false);
+  }
+}
+
+static mlir::Operation *getTerminator(mlir::Region &region) {
+  if (region.empty() || region.back().empty())
+    return nullptr;
+  return &region.back().back();
+}
+
+mlir::LogicalResult hlfir::RegionAssignOp::verify() {
+  if (!mlir::isa_and_nonnull<hlfir::YieldOp>(getTerminator(getRhsRegion())))
+    return emitOpError(
+        "right-hand side region must be terminated by an hlfir.yield");
+  if (!mlir::isa_and_nonnull<hlfir::YieldOp, hlfir::ElementalAddrOp>(
+          getTerminator(getLhsRegion())))
+    return emitOpError("left-hand side region must be terminated by an "
+                       "hlfir.yield or hlfir.elemental_addr");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// YieldOp
+//===----------------------------------------------------------------------===//
+
+static mlir::ParseResult parseYieldOpCleanup(mlir::OpAsmParser &parser,
+                                             mlir::Region &cleanup) {
+  if (succeeded(parser.parseOptionalKeyword("cleanup"))) {
+    if (parser.parseRegion(cleanup, /*arguments=*/{},
+                           /*argTypes=*/{}))
+      return mlir::failure();
+    hlfir::YieldOp::ensureTerminator(cleanup, parser.getBuilder(),
+                                     parser.getBuilder().getUnknownLoc());
+  }
+  return mlir::success();
+}
+
+template <typename YieldOp>
+static void printYieldOpCleanup(mlir::OpAsmPrinter &p, YieldOp yieldOp,
+                                mlir::Region &cleanup) {
+  if (!cleanup.empty()) {
+    p << "cleanup ";
+    p.printRegion(cleanup, /*printEntryBlockArgs=*/false,
+                  /*printBlockTerminators=*/false);
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// ElementalAddrOp
+//===----------------------------------------------------------------------===//
+
+void hlfir::ElementalAddrOp::build(mlir::OpBuilder &builder,
+                                   mlir::OperationState &odsState,
+                                   mlir::Value shape) {
+  odsState.addOperands(shape);
+  mlir::Region *bodyRegion = odsState.addRegion();
+  bodyRegion->push_back(new mlir::Block{});
+  if (auto shapeType = shape.getType().dyn_cast<fir::ShapeType>()) {
+    unsigned dim = shapeType.getRank();
+    mlir::Type indexType = builder.getIndexType();
+    for (unsigned d = 0; d < dim; ++d)
+      bodyRegion->front().addArgument(indexType, odsState.location);
+  }
+  // Push cleanUp region.
+  odsState.addRegion();
+}
+
+mlir::LogicalResult hlfir::ElementalAddrOp::verify() {
+  hlfir::YieldOp yieldOp =
+      mlir::dyn_cast_or_null<hlfir::YieldOp>(getTerminator(getBody()));
+  if (!yieldOp)
+    return emitOpError("body region must be terminated by an hlfir.yield");
+  mlir::Type elementAddrType = yieldOp.getEntity().getType();
+  if (!hlfir::isFortranVariableType(elementAddrType) ||
+      hlfir::getFortranElementOrSequenceType(elementAddrType)
+          .isa<fir::SequenceType>())
+    return emitOpError("body must compute the address of a scalar entity");
+  unsigned shapeRank = getShape().getType().cast<fir::ShapeType>().getRank();
+  if (shapeRank != getIndices().size())
+    return emitOpError("body number of indices must match shape rank");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// OrderedAssignmentTreeOpInterface
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult hlfir::OrderedAssignmentTreeOpInterface::verifyImpl() {
+  if (mlir::Region *body = getSubTreeRegion())
+    if (!body->empty())
+      for (mlir::Operation &op : body->front())
+        if (!mlir::isa<hlfir::OrderedAssignmentTreeOpInterface, fir::FirEndOp>(
+                op))
+          return emitOpError(
+              "body region must only contain OrderedAssignmentTreeOpInterface "
+              "operations or fir.end");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// ForallOp
+//===----------------------------------------------------------------------===//
+
+static mlir::ParseResult parseForallOpBody(mlir::OpAsmParser &parser,
+                                           mlir::Region &body) {
+  mlir::OpAsmParser::Argument bodyArg;
+  if (parser.parseLParen() || parser.parseArgument(bodyArg) ||
+      parser.parseColon() || parser.parseType(bodyArg.type) ||
+      parser.parseRParen())
+    return mlir::failure();
+  if (parser.parseRegion(body, {bodyArg}))
+    return mlir::failure();
+  ensureTerminator(body, parser.getBuilder(),
+                   parser.getBuilder().getUnknownLoc());
+  return mlir::success();
+}
+
+static void printForallOpBody(mlir::OpAsmPrinter &p, hlfir::ForallOp forall,
+                              mlir::Region &body) {
+  mlir::Value forallIndex = forall.getForallIndexValue();
+  p << " (" << forallIndex << ": " << forallIndex.getType() << ") ";
+  p.printRegion(body, /*printEntryBlockArgs=*/false,
+                /*printBlockTerminators=*/false);
+}
+
+/// Predicate implementation of YieldIntegerOrEmpty.
+static bool yieldsIntegerOrEmpty(mlir::Region &region) {
+  if (region.empty())
+    return true;
+  auto yield = mlir::dyn_cast_or_null<hlfir::YieldOp>(getTerminator(region));
+  return yield && fir::isa_integer(yield.getEntity().getType());
+}
+
+//===----------------------------------------------------------------------===//
+// ForallMaskOp
+//===----------------------------------------------------------------------===//
+
+static mlir::ParseResult parseAssignmentMaskOpBody(mlir::OpAsmParser &parser,
+                                                   mlir::Region &body) {
+  if (parser.parseRegion(body))
+    return mlir::failure();
+  ensureTerminator(body, parser.getBuilder(),
+                   parser.getBuilder().getUnknownLoc());
+  return mlir::success();
+}
+
+template <typename ConcreteOp>
+static void printAssignmentMaskOpBody(mlir::OpAsmPrinter &p, ConcreteOp,
+                                      mlir::Region &body) {
+  // ElseWhereOp is a WhereOp/ElseWhereOp terminator that should be printed.
+  bool printBlockTerminators =
+      !body.empty() &&
+      mlir::isa_and_nonnull<hlfir::ElseWhereOp>(body.back().getTerminator());
+  p.printRegion(body, /*printEntryBlockArgs=*/false, printBlockTerminators);
+}
+
+static bool yieldsLogical(mlir::Region &region, bool mustBeScalarI1) {
+  if (region.empty())
+    return false;
+  auto yield = mlir::dyn_cast_or_null<hlfir::YieldOp>(getTerminator(region));
+  if (!yield)
+    return false;
+  mlir::Type yieldType = yield.getEntity().getType();
+  if (mustBeScalarI1)
+    return hlfir::isI1Type(yieldType);
+  return hlfir::isMaskArgument(yieldType) &&
+         hlfir::getFortranElementOrSequenceType(yieldType)
+             .isa<fir::SequenceType>();
+}
+
+mlir::LogicalResult hlfir::ForallMaskOp::verify() {
+  if (!yieldsLogical(getMaskRegion(), /*mustBeScalarI1=*/true))
+    return emitOpError("mask region must yield a scalar i1");
+  mlir::Operation *op = getOperation();
+  hlfir::ForallOp forallOp =
+      mlir::dyn_cast_or_null<hlfir::ForallOp>(op->getParentOp());
+  if (!forallOp || op->getParentRegion() != &forallOp.getBody())
+    return emitOpError("must be inside the body region of an hlfir.forall");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// WhereOp and ElseWhereOp
+//===----------------------------------------------------------------------===//
+
+template <typename ConcreteOp>
+static mlir::LogicalResult verifyWhereAndElseWhereBody(ConcreteOp &concreteOp) {
+  for (mlir::Operation &op : concreteOp.getBody().front())
+    if (mlir::isa<hlfir::ForallOp>(op))
+      return concreteOp.emitOpError(
+          "body region must not contain hlfir.forall");
+  return mlir::success();
+}
+
+mlir::LogicalResult hlfir::WhereOp::verify() {
+  if (!yieldsLogical(getMaskRegion(), /*mustBeScalarI1=*/false))
+    return emitOpError("mask region must yield a logical array");
+  return verifyWhereAndElseWhereBody(*this);
+}
+
+mlir::LogicalResult hlfir::ElseWhereOp::verify() {
+  if (!getMaskRegion().empty())
+    if (!yieldsLogical(getMaskRegion(), /*mustBeScalarI1=*/false))
+      return emitOpError(
+          "mask region must yield a logical array when provided");
+  return verifyWhereAndElseWhereBody(*this);
+}
+
+//===----------------------------------------------------------------------===//
+// ForallIndexOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult
+hlfir::ForallIndexOp::canonicalize(hlfir::ForallIndexOp indexOp,
+                                   mlir::PatternRewriter &rewriter) {
+  auto insertPt = rewriter.saveInsertionPoint();
+  for (mlir::Operation *user : indexOp->getResult(0).getUsers())
+    if (auto loadOp = mlir::dyn_cast_or_null<fir::LoadOp>(user)) {
+      rewriter.setInsertionPoint(loadOp);
+      rewriter.replaceOpWithNewOp<fir::ConvertOp>(
+          user, loadOp.getResult().getType(), indexOp.getIndex());
+    }
+  rewriter.restoreInsertionPoint(insertPt);
+  if (indexOp.use_empty()) {
+    rewriter.eraseOp(indexOp);
+    return mlir::success();
+  }
+  return mlir::failure();
+}
+
+#include "flang/Optimizer/HLFIR/HLFIROpInterfaces.cpp.inc"
 #define GET_OP_CLASSES
 #include "flang/Optimizer/HLFIR/HLFIROps.cpp.inc"
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
index 0323dce441640..df2e2bd68f4ed 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/HLFIR/Transforms/CMakeLists.txt
@@ -4,6 +4,8 @@ add_flang_library(HLFIRTransforms
   BufferizeHLFIR.cpp
   ConvertToFIR.cpp
   LowerHLFIRIntrinsics.cpp
+  LowerHLFIROrderedAssignments.cpp
+  SimplifyHLFIRIntrinsics.cpp
 
   DEPENDS
   FIRDialect
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index de4ec72776436..0225ac3631b9e 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -91,12 +91,21 @@ class AssignOpConversion : public mlir::OpRewritePattern<hlfir::AssignOp> {
       if (rhsIsValue) {
         // createBox can only be called for fir::ExtendedValue that are
         // already in memory. Place the integer/real/complex/logical scalar
-        // in memory (convert to the LHS type so that i1 are allocated in
-        // a proper Fortran logical storage).
-        mlir::Type lhsValueType = lhs.getFortranElementType();
-        mlir::Value rhsVal =
-            builder.createConvert(loc, lhsValueType, fir::getBase(rhsExv));
-        mlir::Value temp = builder.create<fir::AllocaOp>(loc, lhsValueType);
+        // in memory.
+        // The RHS might be i1, which is not supported for emboxing.
+        // If LHS is not polymorphic, we may cast the RHS to the LHS type
+        // before emboxing. If LHS is polymorphic we have to figure out
+        // the data type for RHS emboxing anyway.
+        // It is probably a good idea to make sure that the data type
+        // of the RHS is always a valid Fortran storage data type.
+        // For the time being, just handle i1 explicitly here.
+        mlir::Type rhsType = rhs.getFortranElementType();
+        mlir::Value rhsVal = fir::getBase(rhsExv);
+        if (rhsType == builder.getI1Type()) {
+          rhsType = fir::LogicalType::get(builder.getContext(), 4);
+          rhsVal = builder.createConvert(loc, rhsType, rhsVal);
+        }
+        mlir::Value temp = builder.create<fir::AllocaOp>(loc, rhsType);
         builder.create<fir::StoreOp>(loc, rhsVal, temp);
         rhsExv = temp;
       }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index 95192a3e79d49..fbf7670413df2 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -163,39 +163,56 @@ class HlfirIntrinsicConversion : public mlir::OpRewritePattern<OP> {
   }
 };
 
-struct SumOpConversion : public HlfirIntrinsicConversion<hlfir::SumOp> {
-  using HlfirIntrinsicConversion<hlfir::SumOp>::HlfirIntrinsicConversion;
+template <class OP>
+class HlfirReductionIntrinsicConversion : public HlfirIntrinsicConversion<OP> {
+  using HlfirIntrinsicConversion<OP>::HlfirIntrinsicConversion;
+  using IntrinsicArgument =
+      typename HlfirIntrinsicConversion<OP>::IntrinsicArgument;
 
+public:
   mlir::LogicalResult
-  matchAndRewrite(hlfir::SumOp sum,
+  matchAndRewrite(OP operation,
                   mlir::PatternRewriter &rewriter) const override {
+    std::string opName;
+    if constexpr (std::is_same_v<OP, hlfir::SumOp>) {
+      opName = "sum";
+    } else if constexpr (std::is_same_v<OP, hlfir::ProductOp>) {
+      opName = "product";
+    } else {
+      return mlir::failure();
+    }
     fir::KindMapping kindMapping{rewriter.getContext()};
     fir::FirOpBuilder builder{rewriter, kindMapping};
-    const mlir::Location &loc = sum->getLoc();
+    const mlir::Location &loc = operation->getLoc();
 
     mlir::Type i32 = builder.getI32Type();
     mlir::Type logicalType = fir::LogicalType::get(
         builder.getContext(), builder.getKindMap().defaultLogicalKind());
-
     llvm::SmallVector<IntrinsicArgument, 3> inArgs;
-    inArgs.push_back({sum.getArray(), sum.getArray().getType()});
-    inArgs.push_back({sum.getDim(), i32});
-    inArgs.push_back({sum.getMask(), logicalType});
+    inArgs.push_back({operation.getArray(), operation.getArray().getType()});
+    inArgs.push_back({operation.getDim(), i32});
+    inArgs.push_back({operation.getMask(), logicalType});
 
-    auto *argLowering = fir::getIntrinsicArgumentLowering("sum");
+    auto *argLowering = fir::getIntrinsicArgumentLowering(opName);
     llvm::SmallVector<fir::ExtendedValue, 3> args =
-        lowerArguments(sum, inArgs, rewriter, argLowering);
+        this->lowerArguments(operation, inArgs, rewriter, argLowering);
 
-    mlir::Type scalarResultType = hlfir::getFortranElementType(sum.getType());
+    mlir::Type scalarResultType =
+        hlfir::getFortranElementType(operation.getType());
 
     auto [resultExv, mustBeFreed] =
-        fir::genIntrinsicCall(builder, loc, "sum", scalarResultType, args);
+        fir::genIntrinsicCall(builder, loc, opName, scalarResultType, args);
 
-    processReturnValue(sum, resultExv, mustBeFreed, builder, rewriter);
+    this->processReturnValue(operation, resultExv, mustBeFreed, builder,
+                             rewriter);
     return mlir::success();
   }
 };
 
+using SumOpConversion = HlfirReductionIntrinsicConversion<hlfir::SumOp>;
+
+using ProductOpConversion = HlfirReductionIntrinsicConversion<hlfir::ProductOp>;
+
 struct MatmulOpConversion : public HlfirIntrinsicConversion<hlfir::MatmulOp> {
   using HlfirIntrinsicConversion<hlfir::MatmulOp>::HlfirIntrinsicConversion;
 
@@ -304,14 +321,16 @@ class LowerHLFIRIntrinsics
     mlir::ModuleOp module = this->getOperation();
     mlir::MLIRContext *context = &getContext();
     mlir::RewritePatternSet patterns(context);
-    patterns.insert<MatmulOpConversion, MatmulTransposeOpConversion,
-                    SumOpConversion, TransposeOpConversion>(context);
+    patterns
+        .insert<MatmulOpConversion, MatmulTransposeOpConversion,
+                SumOpConversion, ProductOpConversion, TransposeOpConversion>(
+            context);
     mlir::ConversionTarget target(*context);
     target.addLegalDialect<mlir::BuiltinDialect, mlir::arith::ArithDialect,
                            mlir::func::FuncDialect, fir::FIROpsDialect,
                            hlfir::hlfirDialect>();
     target.addIllegalOp<hlfir::MatmulOp, hlfir::MatmulTransposeOp, hlfir::SumOp,
-                        hlfir::TransposeOp>();
+                        hlfir::ProductOp, hlfir::TransposeOp>();
     target.markUnknownOpDynamicallyLegal(
         [](mlir::Operation *) { return true; });
     if (mlir::failed(
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
new file mode 100644
index 0000000000000..85e32ffa0ec2d
--- /dev/null
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -0,0 +1,106 @@
+//===- LowerHLFIROrderedAssignments.cpp - Lower HLFIR ordered assignments -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file defines a pass to lower HLFIR ordered assignments.
+// Ordered assignments are all the operations with the
+// OrderedAssignmentTreeOpInterface that implements user defined assignments,
+// assignment to vector subscripted entities, and assignments inside forall and
+// where.
+// The pass lowers these operations to regular hlfir.assign, loops and, if
+// needed, introduces temporary storage to fulfill Fortran semantics.
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Optimizer/HLFIR/Passes.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace hlfir {
+#define GEN_PASS_DEF_LOWERHLFIRORDEREDASSIGNMENTS
+#include "flang/Optimizer/HLFIR/Passes.h.inc"
+} // namespace hlfir
+
+using namespace mlir;
+
+namespace {
+
+class ForallOpConversion : public mlir::OpRewritePattern<hlfir::ForallOp> {
+public:
+  explicit ForallOpConversion(mlir::MLIRContext *ctx) : OpRewritePattern{ctx} {}
+
+  mlir::LogicalResult
+  matchAndRewrite(hlfir::ForallOp forallOp,
+                  mlir::PatternRewriter &rewriter) const override {
+    TODO(forallOp.getLoc(), "FORALL construct or statement in HLFIR");
+    return mlir::failure();
+  }
+};
+
+class WhereOpConversion : public mlir::OpRewritePattern<hlfir::WhereOp> {
+public:
+  explicit WhereOpConversion(mlir::MLIRContext *ctx) : OpRewritePattern{ctx} {}
+
+  mlir::LogicalResult
+  matchAndRewrite(hlfir::WhereOp whereOp,
+                  mlir::PatternRewriter &rewriter) const override {
+    TODO(whereOp.getLoc(), "WHERE construct or statement in HLFIR");
+    return mlir::failure();
+  }
+};
+
+class RegionAssignConversion
+    : public mlir::OpRewritePattern<hlfir::RegionAssignOp> {
+public:
+  explicit RegionAssignConversion(mlir::MLIRContext *ctx)
+      : OpRewritePattern{ctx} {}
+
+  mlir::LogicalResult
+  matchAndRewrite(hlfir::RegionAssignOp regionAssignOp,
+                  mlir::PatternRewriter &rewriter) const override {
+    if (!regionAssignOp.getUserDefinedAssignment().empty())
+      TODO(regionAssignOp.getLoc(), "user defined assignment in HLFIR");
+    else
+      TODO(regionAssignOp.getLoc(),
+           "assignments to vector subscripted entity in HLFIR");
+    return mlir::failure();
+  }
+};
+
+class LowerHLFIROrderedAssignments
+    : public hlfir::impl::LowerHLFIROrderedAssignmentsBase<
+          LowerHLFIROrderedAssignments> {
+public:
+  void runOnOperation() override {
+    // Running on a ModuleOp because this pass may generate FuncOp declaration
+    // for runtime calls. This could be a FuncOp pass otherwise.
+    auto module = this->getOperation();
+    auto *context = &getContext();
+    mlir::RewritePatternSet patterns(context);
+    // Patterns are only defined for the OrderedAssignmentTreeOpInterface
+    // operations that can be the root of ordered assignments. The other
+    // operations will be taken care of while rewriting these trees (they
+    // cannot exist outside of these operations given their verifiers/traits).
+    patterns
+        .insert<ForallOpConversion, WhereOpConversion, RegionAssignConversion>(
+            context);
+    mlir::ConversionTarget target(*context);
+    target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) {
+      return !mlir::isa<hlfir::OrderedAssignmentTreeOpInterface>(op);
+    });
+    if (mlir::failed(mlir::applyPartialConversion(module, target,
+                                                  std::move(patterns)))) {
+      mlir::emitError(mlir::UnknownLoc::get(context),
+                      "failure in HLFIR ordered assignments lowering pass");
+      signalPassFailure();
+    }
+  }
+};
+} // namespace
+
+std::unique_ptr<mlir::Pass> hlfir::createLowerHLFIROrderedAssignmentsPass() {
+  return std::make_unique<LowerHLFIROrderedAssignments>();
+}
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
new file mode 100644
index 0000000000000..21df162265cda
--- /dev/null
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -0,0 +1,114 @@
+//===- SimplifyHLFIRIntrinsics.cpp - Simplify HLFIR Intrinsics ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Normally transformational intrinsics are lowered to calls to runtime
+// functions. However, some cases of the intrinsics are faster when inlined
+// into the calling function.
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/Support/KindMapping.h"
+#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+#include "flang/Optimizer/HLFIR/Passes.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/Location.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace hlfir {
+#define GEN_PASS_DEF_SIMPLIFYHLFIRINTRINSICS
+#include "flang/Optimizer/HLFIR/Passes.h.inc"
+} // namespace hlfir
+
+namespace {
+
+class TransposeAsElementalConversion
+    : public mlir::OpRewritePattern<hlfir::TransposeOp> {
+public:
+  using mlir::OpRewritePattern<hlfir::TransposeOp>::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(hlfir::TransposeOp transpose,
+                  mlir::PatternRewriter &rewriter) const override {
+    mlir::Location loc = transpose.getLoc();
+    fir::KindMapping kindMapping{rewriter.getContext()};
+    fir::FirOpBuilder builder{rewriter, kindMapping};
+    hlfir::ExprType expr = transpose.getType();
+    mlir::Type elementType = expr.getElementType();
+    hlfir::Entity array = hlfir::Entity{transpose.getArray()};
+    mlir::Value resultShape = genResultShape(loc, builder, array);
+    llvm::SmallVector<mlir::Value, 1> typeParams;
+    hlfir::genLengthParameters(loc, builder, array, typeParams);
+
+    auto genKernel = [&array](mlir::Location loc, fir::FirOpBuilder &builder,
+                              mlir::ValueRange inputIndices) -> hlfir::Entity {
+      assert(inputIndices.size() == 2 && "checked in TransposeOp::validate");
+      mlir::ValueRange transposedIndices{{inputIndices[1], inputIndices[0]}};
+      hlfir::Entity element =
+          hlfir::getElementAt(loc, builder, array, transposedIndices);
+      hlfir::Entity val = hlfir::loadTrivialScalar(loc, builder, element);
+      return val;
+    };
+    hlfir::ElementalOp elementalOp = hlfir::genElementalOp(
+        loc, builder, elementType, resultShape, typeParams, genKernel);
+
+    rewriter.replaceOp(transpose, elementalOp.getResult());
+    return mlir::success();
+  }
+
+private:
+  static mlir::Value genResultShape(mlir::Location loc,
+                                    fir::FirOpBuilder &builder,
+                                    hlfir::Entity array) {
+    mlir::Value inShape = hlfir::genShape(loc, builder, array);
+    llvm::SmallVector<mlir::Value> inExtents =
+        hlfir::getExplicitExtentsFromShape(inShape, builder);
+    if (inShape.getUses().empty())
+      inShape.getDefiningOp()->erase();
+
+    // transpose indices
+    assert(inExtents.size() == 2 && "checked in TransposeOp::validate");
+    return builder.create<fir::ShapeOp>(
+        loc, mlir::ValueRange{inExtents[1], inExtents[0]});
+  }
+};
+
+class SimplifyHLFIRIntrinsics
+    : public hlfir::impl::SimplifyHLFIRIntrinsicsBase<SimplifyHLFIRIntrinsics> {
+public:
+  void runOnOperation() override {
+    mlir::func::FuncOp func = this->getOperation();
+    mlir::MLIRContext *context = &getContext();
+    mlir::RewritePatternSet patterns(context);
+    patterns.insert<TransposeAsElementalConversion>(context);
+    mlir::ConversionTarget target(*context);
+    // don't transform transpose of polymorphic arrays (not currently supported
+    // by hlfir.elemental)
+    target.addDynamicallyLegalOp<hlfir::TransposeOp>(
+        [](hlfir::TransposeOp transpose) {
+          return transpose.getType().cast<hlfir::ExprType>().isPolymorphic();
+        });
+    target.markUnknownOpDynamicallyLegal(
+        [](mlir::Operation *) { return true; });
+    if (mlir::failed(
+            mlir::applyFullConversion(func, target, std::move(patterns)))) {
+      mlir::emitError(func->getLoc(),
+                      "failure in HLFIR intrinsic simplification");
+      signalPassFailure();
+    }
+  }
+};
+} // namespace
+
+std::unique_ptr<mlir::Pass> hlfir::createSimplifyHLFIRIntrinsicsPass() {
+  return std::make_unique<SimplifyHLFIRIntrinsics>();
+}
diff --git a/flang/lib/Optimizer/Transforms/AbstractResult.cpp b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
index 2c0576eaa5cc4..7fcdb4d5650be 100644
--- a/flang/lib/Optimizer/Transforms/AbstractResult.cpp
+++ b/flang/lib/Optimizer/Transforms/AbstractResult.cpp
@@ -217,6 +217,10 @@ class ReturnOpConversion : public mlir::OpRewritePattern<mlir::func::ReturnOp> {
     if (auto *op = returnedValue.getDefiningOp())
       if (auto load = mlir::dyn_cast<fir::LoadOp>(op)) {
         auto resultStorage = load.getMemref();
+        // The result alloca may be behind a fir.declare, if any.
+        if (auto declare = mlir::dyn_cast_or_null<fir::DeclareOp>(
+                resultStorage.getDefiningOp()))
+          resultStorage = declare.getMemref();
         // TODO: This should be generalized for derived types, and it is
         // architecture and OS dependent.
         if (fir::isa_builtin_cptr_type(returnedValue.getType())) {
@@ -232,7 +236,7 @@ class ReturnOpConversion : public mlir::OpRewritePattern<mlir::func::ReturnOp> {
               ret, mlir::ValueRange{retValue});
           return mlir::success();
         }
-        load.getMemref().replaceAllUsesWith(newArg);
+        resultStorage.replaceAllUsesWith(newArg);
         replacedStorage = true;
         if (auto *alloc = resultStorage.getDefiningOp())
           if (alloc->use_empty())
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index ab9f1d768b039..f5d0602c270cc 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -340,7 +340,8 @@ mlir::LogicalResult SelectTypeConv::matchAndRewrite(
     std::optional<mlir::ValueRange> destOps =
         selectType.getSuccessorOperands(operands, idx);
     if (typeGuards[idx].dyn_cast<mlir::UnitAttr>())
-      rewriter.replaceOpWithNewOp<mlir::cf::BranchOp>(selectType, dest);
+      rewriter.replaceOpWithNewOp<mlir::cf::BranchOp>(
+          selectType, dest, destOps.value_or(mlir::ValueRange{}));
     else if (mlir::failed(genTypeLadderStep(loc, selector, typeGuards[idx],
                                             dest, destOps, mod, rewriter,
                                             kindMap)))
diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
index 244a462cb7225..4896ca0f668a8 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
@@ -42,7 +42,6 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include <llvm/CodeGen/SelectionDAGNodes.h>
 #include <llvm/Support/ErrorHandling.h>
 #include <mlir/Dialect/Arith/IR/Arith.h>
 #include <mlir/IR/BuiltinTypes.h>
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index e7d628d4c428f..cec08e18e44c5 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -309,7 +309,7 @@ TYPE_PARSER(
     "TO" >> construct<OmpClause>(construct<OmpClause::To>(
                 parenthesized(Parser<OmpObjectList>{}))) ||
     "USE_DEVICE_PTR" >> construct<OmpClause>(construct<OmpClause::UseDevicePtr>(
-                            parenthesized(nonemptyList(name)))) ||
+                            parenthesized(Parser<OmpObjectList>{}))) ||
     "UNIFIED_ADDRESS" >>
         construct<OmpClause>(construct<OmpClause::UnifiedAddress>()) ||
     "UNIFIED_SHARED_MEMORY" >>
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 9794424e266d6..c7bc0197630c9 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -2766,11 +2766,12 @@ const parser::OmpObjectList *OmpStructureChecker::GetOmpObjectList(
     const parser::OmpClause &clause) {
 
   // Clauses with OmpObjectList as its data member
-  using MemberObjectListClauses = std::tuple<parser::OmpClause::Copyprivate,
-      parser::OmpClause::Copyin, parser::OmpClause::Firstprivate,
-      parser::OmpClause::From, parser::OmpClause::Lastprivate,
-      parser::OmpClause::Link, parser::OmpClause::Private,
-      parser::OmpClause::Shared, parser::OmpClause::To>;
+  using MemberObjectListClauses =
+      std::tuple<parser::OmpClause::Copyprivate, parser::OmpClause::Copyin,
+          parser::OmpClause::Firstprivate, parser::OmpClause::From,
+          parser::OmpClause::Lastprivate, parser::OmpClause::Link,
+          parser::OmpClause::Private, parser::OmpClause::Shared,
+          parser::OmpClause::To, parser::OmpClause::UseDevicePtr>;
 
   // Clauses with OmpObjectList in the tuple
   using TupleObjectListClauses = std::tuple<parser::OmpClause::Allocate,
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 1bb1e67f89304..9eb4d98364916 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -436,6 +436,11 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     return false;
   }
 
+  bool Pre(const parser::OmpClause::UseDevicePtr &x) {
+    ResolveOmpObjectList(x.v, Symbol::Flag::OmpUseDevicePtr);
+    return false;
+  }
+
   void Post(const parser::Name &);
 
   // Keep track of labels in the statements that causes jumps to target labels
@@ -506,7 +511,7 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
       Symbol::Flag::OmpPrivate, Symbol::Flag::OmpLinear,
       Symbol::Flag::OmpFirstPrivate, Symbol::Flag::OmpLastPrivate,
       Symbol::Flag::OmpReduction, Symbol::Flag::OmpCriticalLock,
-      Symbol::Flag::OmpCopyIn};
+      Symbol::Flag::OmpCopyIn, Symbol::Flag::OmpUseDevicePtr};
 
   static constexpr Symbol::Flags ompFlagsRequireMark{
       Symbol::Flag::OmpThreadprivate};
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 820b3d8fda0b6..57870a7ccce02 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1414,7 +1414,6 @@ bool OmpVisitor::NeedsScope(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &beginDir{std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
   switch (beginDir.v) {
-  case llvm::omp::Directive::OMPD_target_data:
   case llvm::omp::Directive::OMPD_master:
   case llvm::omp::Directive::OMPD_ordered:
   case llvm::omp::Directive::OMPD_taskgroup:
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index 20e78ff6654af..2a7e74c175533 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -26,6 +26,7 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
 
   set(LLVM_COMMON_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../cmake")
   set(LLVM_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../llvm/cmake")
+  set(CLANG_CMAKE_UTILS "${FLANG_SOURCE_DIR}/../clang/cmake")
 
   # Add path for custom modules
   list(INSERT CMAKE_MODULE_PATH 0
@@ -35,8 +36,10 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     "${LLVM_COMMON_CMAKE_UTILS}/Modules"
     "${LLVM_CMAKE_UTILS}"
     "${LLVM_CMAKE_UTILS}/modules"
+    "${CLANG_CMAKE_UTILS}/modules"
     )
 
+  include(AddClang)
   include(AddLLVM)
   include(AddFlang)
   include(HandleLLVMOptions)
diff --git a/flang/runtime/assign.cpp b/flang/runtime/assign.cpp
index 095c4d3b9eefd..88f5500588bfa 100644
--- a/flang/runtime/assign.cpp
+++ b/flang/runtime/assign.cpp
@@ -223,12 +223,13 @@ static void BlankPadCharacterAssignment(Descriptor &to, const Descriptor &from,
     SubscriptValue toAt[], SubscriptValue fromAt[], std::size_t elements,
     std::size_t toElementBytes, std::size_t fromElementBytes) {
   std::size_t padding{(toElementBytes - fromElementBytes) / sizeof(CHAR)};
+  std::size_t copiedCharacters{fromElementBytes / sizeof(CHAR)};
   for (; elements-- > 0;
        to.IncrementSubscripts(toAt), from.IncrementSubscripts(fromAt)) {
     CHAR *p{to.Element<CHAR>(toAt)};
     std::memmove(
         p, from.Element<std::add_const_t<CHAR>>(fromAt), fromElementBytes);
-    p += fromElementBytes;
+    p += copiedCharacters;
     for (auto n{padding}; n-- > 0;) {
       *p++ = CHAR{' '};
     }
@@ -545,6 +546,17 @@ void RTNAME(Assign)(Descriptor &to, const Descriptor &from,
 void RTNAME(AssignTemporary)(Descriptor &to, const Descriptor &from,
     const char *sourceFile, int sourceLine) {
   Terminator terminator{sourceFile, sourceLine};
+  // Initialize the "to" if it is of derived type that needs initialization.
+  if (const DescriptorAddendum * addendum{to.Addendum()}) {
+    if (const auto *derived{addendum->derivedType()}) {
+      if (!derived->noInitializationNeeded()) {
+        if (ReturnError(terminator, Initialize(to, *derived, terminator)) !=
+            StatOk) {
+          return;
+        }
+      }
+    }
+  }
   Assign(to, from, terminator, PolymorphicLHS);
 }
 
diff --git a/flang/runtime/time-intrinsic.cpp b/flang/runtime/time-intrinsic.cpp
index 5f7b50964fe2a..68d63253139f1 100644
--- a/flang/runtime/time-intrinsic.cpp
+++ b/flang/runtime/time-intrinsic.cpp
@@ -56,7 +56,12 @@ template <typename Unused = void> double GetCpuTime(fallback_implementation) {
   return -1.0;
 }
 
-#if defined CLOCK_PROCESS_CPUTIME_ID
+#if defined __MINGW32__
+// clock_gettime is implemented in the pthread library for MinGW.
+// Using it here would mean that all programs that link libFortranRuntime are
+// required to also link to pthread. Instead, don't use the function.
+#undef CLOCKID
+#elif defined CLOCK_PROCESS_CPUTIME_ID
 #define CLOCKID CLOCK_PROCESS_CPUTIME_ID
 #elif defined CLOCK_THREAD_CPUTIME_ID
 #define CLOCKID CLOCK_THREAD_CPUTIME_ID
diff --git a/flang/test/CMakeLists.txt b/flang/test/CMakeLists.txt
index 4de1036dfb52b..7d96a72e5f36d 100644
--- a/flang/test/CMakeLists.txt
+++ b/flang/test/CMakeLists.txt
@@ -79,6 +79,7 @@ if (LLVM_BUILD_EXAMPLES)
   list(APPEND FLANG_TEST_DEPENDS
     flangPrintFunctionNames
     flangOmpReport
+    flangFeatureList
     )
 endif ()
 
diff --git a/flang/test/Driver/code-gen-aarch64.f90 b/flang/test/Driver/code-gen-aarch64.f90
index b6be486395316..e258b59c05f03 100644
--- a/flang/test/Driver/code-gen-aarch64.f90
+++ b/flang/test/Driver/code-gen-aarch64.f90
@@ -1,22 +1,9 @@
 ! Test -emit-obj (X86)
 
-! REQUIRES: aarch64-registered-target, x86-registered-target
+! REQUIRES: aarch64-registered-target
 
-! RUN: rm -f %t.o
-! RUN: %flang_fc1 -emit-obj -triple aarch64-unknown-linux-gnu %s -o %t.o
-! RUN: llvm-objdump --triple aarch64-unknown-linux-gnu --disassemble-all %t.o | FileCheck %s --check-prefix=CORRECT_TRIPLE
-! RUN: rm -f %t.o
-! RUN: %flang -c --target=aarch64-unknown-linux-gnu %s -o %t.o
-! RUN: llvm-objdump --triple aarch64-unknown-linux-gnu --disassemble-all %t.o | FileCheck %s --check-prefix=CORRECT_TRIPLE
-
-! RUN: %flang -c --target=aarch64-unknown-linux-gnu %s -o %t.o
-! RUN: llvm-objdump --triple x86_64-unknown-linux-gnu --disassemble-all %t.o | FileCheck %s --check-prefix=INCORRECT_TRIPLE
-
-! CORRECT_TRIPLE-LABEL: <_QQmain>:
-! CORRECT_TRIPLE-NEXT:  	ret
-
-! When incorrect triple is used to disassemble, there won't be a ret instruction at all.
-! INCORRECT_TRIPLE-LABEL: <_QQmain>:
-! INCORRECT_TRIPLE-NOT:  	ret
+! RUN: %flang_fc1 -triple aarch64-unknown-linux-gnu -emit-obj %s -o - | \
+! RUN: llvm-readobj -h - | FileCheck %s
 
+! CHECK: Arch: aarch64
 end program
diff --git a/flang/test/Driver/code-gen-rv64.f90 b/flang/test/Driver/code-gen-rv64.f90
index c29ec9eff0aa2..f7bdb842559a4 100644
--- a/flang/test/Driver/code-gen-rv64.f90
+++ b/flang/test/Driver/code-gen-rv64.f90
@@ -2,16 +2,15 @@
 
 ! REQUIRES: riscv-registered-target
 
-! RUN: rm -f %t.o
 ! RUN: %flang_fc1 -triple riscv64-unknown-linux-gnu \
-! RUN:   -target-feature +d -target-feature +c -emit-obj %s -o %t.o
-! RUN: llvm-readobj -h %t.o | FileCheck %s
+! RUN:   -target-feature +d -target-feature +c -emit-obj %s -o - | \
+! RUN: llvm-readobj -h - | FileCheck %s
 
-! RUN: rm -f %t.o
-! RUN: %flang --target=riscv64-unknown-linux-gnu -c %s -o %t.o
-! RUN: llvm-readobj -h %t.o | FileCheck %s
+! RUN: %flang --target=riscv64-unknown-linux-gnu -c %s -o - | \
+! RUN: llvm-readobj -h - | FileCheck %s
 
 ! If Flang failed to emit target-feature info, then Flags will be 0x0.
+! CHECK: Arch: riscv64
 ! CHECK: Flags [ (0x5)
 ! CHECK-NEXT: EF_RISCV_FLOAT_ABI_DOUBLE (0x4)
 ! CHECK-NEXT: EF_RISCV_RVC (0x1)
diff --git a/flang/test/Driver/code-gen-x86.f90 b/flang/test/Driver/code-gen-x86.f90
index b2bf95613efae..a6e5096f0de4b 100644
--- a/flang/test/Driver/code-gen-x86.f90
+++ b/flang/test/Driver/code-gen-x86.f90
@@ -1,22 +1,9 @@
 ! Test -emit-obj (X86)
 
-! REQUIRES: x86-registered-target, aarch64-registered-target
+! REQUIRES: x86-registered-target
 
-! RUN: rm -f %t.o
-! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-obj %s -o %t.o
-! RUN: llvm-objdump --triple x86_64-unknown-linux-gnu --disassemble-all %t.o | FileCheck %s --check-prefix=CORRECT_TRIPLE
-! RUN: rm -f %t.o
-! RUN: %flang --target=x86_64-unknown-linux-gnu -c %s -o %t.o
-! RUN: llvm-objdump --triple x86_64-unknown-linux-gnu --disassemble-all %t.o | FileCheck %s --check-prefix=CORRECT_TRIPLE
-
-! RUN: %flang -c --target=x86_64-unknown-linux-gnu %s -o %t.o
-! RUN: llvm-objdump --triple aarch64-unknown-linux-gnu --disassemble-all %t.o | FileCheck %s --check-prefix=INCORRECT_TRIPLE
-
-! CORRECT_TRIPLE-LABEL: <_QQmain>:
-! CORRECT_TRIPLE-NEXT:  	retq
-
-! When incorrect triple is used to disassemble, there won't be a ret(q) instruction at all.
-! INCORRECT_TRIPLE-LABEL: <_QQmain>:
-! INCORRECT_TRIPLE-NOT:  	ret
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-obj %s -o - | \
+! RUN: llvm-readobj -h - | FileCheck %s
 
+! CHECK: Arch: x86_64
 end program
diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90
index f57fd32b0838e..655178677f2ff 100644
--- a/flang/test/Driver/driver-help.f90
+++ b/flang/test/Driver/driver-help.f90
@@ -153,6 +153,8 @@
 ! HELP-FC1-NEXT: -fno-version-loops-for-stride
 ! HELP-FC1-NEXT:                        Do not create unit-strided loops (default)
 ! HELP-FC1-NEXT: -fopenacc              Enable OpenACC
+! HELP-FC1-NEXT: -fopenmp-host-ir-file-path <value> 
+! HELP-FC1-NEXT:                        Path to the IR file produced by the frontend for the host.
 ! HELP-FC1-NEXT: -fopenmp-is-device     Generate code only for an OpenMP target device.
 ! HELP-FC1-NEXT: -fopenmp-target-debug  Enable debugging in the OpenMP offloading device RTL
 ! HELP-FC1-NEXT: -fopenmp               Parse OpenMP pragmas and generate parallel code.
diff --git a/flang/test/Driver/mlir-debug-pass-pipeline.f90 b/flang/test/Driver/mlir-debug-pass-pipeline.f90
index d397b29bb56b2..320f06a854b34 100644
--- a/flang/test/Driver/mlir-debug-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-debug-pass-pipeline.f90
@@ -25,6 +25,7 @@
 ! ALL: Pass statistics report
 
 ! ALL: Fortran::lower::VerifierPass
+! ALL-NEXT: LowerHLFIROrderedAssignments
 ! ALL-NEXT: LowerHLFIRIntrinsics
 ! ALL-NEXT: BufferizeHLFIR
 ! ALL-NEXT: ConvertHLFIRtoFIR
diff --git a/flang/test/Driver/mlir-pass-pipeline.f90 b/flang/test/Driver/mlir-pass-pipeline.f90
index 3e8df1615adaa..44d253a467bb4 100644
--- a/flang/test/Driver/mlir-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-pass-pipeline.f90
@@ -13,6 +13,9 @@
 
 ! ALL: Fortran::lower::VerifierPass
 ! O2-NEXT: Canonicalizer
+! O2-NEXT: 'func.func' Pipeline
+! O2-NEXT:   SimplifyHLFIRIntrinsics
+! ALL-NEXT: LowerHLFIROrderedAssignments
 ! ALL-NEXT: LowerHLFIRIntrinsics
 ! ALL-NEXT: BufferizeHLFIR
 ! ALL-NEXT: ConvertHLFIRtoFIR
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index ec107261d4c2b..b48e3100316a8 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -47,15 +47,15 @@
 ! RUN: %flang -### -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa %s 2>&1 | FileCheck --check-prefixes=CHECK-OPENMP-IS-DEVICE %s
 ! CHECK-OPENMP-IS-DEVICE: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" {{.*}}.f90"
 
-! Testing fembed-offload-object and fopenmp-is-device
+! Testing appropriate flags are gnerated and appropriately assigned by the driver when offloading
 ! RUN: %flang -S -### %s -o %t 2>&1 \
 ! RUN: -fopenmp --offload-arch=gfx90a \
 ! RUN: --target=aarch64-unknown-linux-gnu \
-! RUN:   | FileCheck %s --check-prefixes=CHECK-OPENMP-EMBED
-! CHECK-OPENMP-EMBED: "{{[^"]*}}flang-new" "-fc1" "-triple" "aarch64-unknown-linux-gnu" {{.*}} "-fopenmp" {{.*}}.f90"
-! CHECK-OPENMP-EMBED-NEXT: "{{[^"]*}}flang-new" "-fc1" "-triple" "amdgcn-amd-amdhsa" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" {{.*}}.f90"
-! CHECK-OPENMP-EMBED: "{{[^"]*}}clang-offload-packager{{.*}} "--image=file={{.*}}.bc,triple=amdgcn-amd-amdhsa,arch=gfx90a,kind=openmp"
-! CHECK-OPENMP-EMBED-NEXT: "{{[^"]*}}flang-new" "-fc1" "-triple" "aarch64-unknown-linux-gnu" {{.*}} "-fopenmp" {{.*}} "-fembed-offload-object={{.*}}.out" {{.*}}.bc"
+! RUN:   | FileCheck %s --check-prefix=OPENMP-OFFLOAD-ARGS
+! OPENMP-OFFLOAD-ARGS: "{{[^"]*}}flang-new" "-fc1" "-triple" "aarch64-unknown-linux-gnu" {{.*}} "-fopenmp" {{.*}}.f90"
+! OPENMP-OFFLOAD-ARGS-NEXT: "{{[^"]*}}flang-new" "-fc1" "-triple" "amdgcn-amd-amdhsa" {{.*}} "-fopenmp" {{.*}} "-fopenmp-host-ir-file-path" "{{.*}}.bc" "-fopenmp-is-device" {{.*}}.f90"
+! OPENMP-OFFLOAD-ARGS: "{{[^"]*}}clang-offload-packager{{.*}}" {{.*}} "--image=file={{.*}}.bc,triple=amdgcn-amd-amdhsa,arch=gfx90a,kind=openmp"
+! OPENMP-OFFLOAD-ARGS-NEXT: "{{[^"]*}}flang-new" "-fc1" "-triple" "aarch64-unknown-linux-gnu" {{.*}} "-fopenmp" {{.*}} "-fembed-offload-object={{.*}}.out" {{.*}}.bc"
 
 ! Test -fopenmp with offload for RTL Flag Options
 ! RUN: %flang -### %s -o %t 2>&1 \
@@ -103,3 +103,9 @@
 ! CHECK-RTL-ALL: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-target-debug" "-fopenmp-assume-teams-oversubscription"
 ! CHECK-RTL-ALL: "-fopenmp-assume-threads-oversubscription" "-fopenmp-assume-no-thread-state" "-fopenmp-assume-no-nested-parallelism"
 ! CHECK-RTL-ALL: {{.*}}.f90"
+
+! Test diagnostic error when host IR file is non-existent 
+! RUN: not %flang_fc1 %s -o %t 2>&1 -fopenmp -fopenmp-is-device \
+! RUN: -fopenmp-host-ir-file-path non-existant-file.bc \
+! RUN: | FileCheck %s --check-prefix=HOST-IR-MISSING
+! HOST-IR-MISSING: error: provided host compiler IR file 'non-existant-file.bc' is required to generate code for OpenMP target regions but cannot be found
diff --git a/flang/test/Examples/feature-list-class.f90 b/flang/test/Examples/feature-list-class.f90
new file mode 100644
index 0000000000000..61b18c7fdd974
--- /dev/null
+++ b/flang/test/Examples/feature-list-class.f90
@@ -0,0 +1,88 @@
+! UNSUPPORTED: system-windows
+! REQUIRES: plugins, shell, examples
+
+! RUN: %flang_fc1 -load %llvmshlibdir/flangFeatureList%pluginext \
+! RUN:            -plugin feature-list %s 2>&1 | FileCheck %s
+
+module list_features_test
+    implicit none
+
+    type :: test_class_1
+        integer :: a
+        real :: b
+    contains
+        procedure :: sum => sum_test_class_1
+        procedure :: set => set_values_test_class_1
+    end type
+contains
+    real function sum_test_class_1(self)
+        class(test_class_1), intent(in) :: self
+        sum_test_class_1 = self%a + self%b
+    end function
+
+    subroutine set_values_test_class_1(self, a, b)
+        class(test_class_1), intent(out) :: self
+        integer, intent(in) :: a, b
+        self%a = a
+        self%b = b
+    end subroutine
+end module list_features_test
+
+! CHECK: Name: 32
+! CHECK-NEXT: DataRef: 11
+! CHECK-NEXT: Designator: 7
+! CHECK-NEXT: DeclarationTypeSpec: 6
+! CHECK-NEXT: Expr: 5
+! CHECK-NEXT: DeclarationConstruct: 4
+! CHECK-NEXT: EntityDecl: 4
+! CHECK-NEXT: IntrinsicTypeSpec: 4
+! CHECK-NEXT: SpecificationConstruct: 4
+! CHECK-NEXT: StructureComponent: 4
+! CHECK-NEXT: ActionStmt: 3
+! CHECK-NEXT: AssignmentStmt: 3
+! CHECK-NEXT: AttrSpec: 3
+! CHECK-NEXT: DummyArg: 3
+! CHECK-NEXT: ExecutableConstruct: 3
+! CHECK-NEXT: ExecutionPartConstruct: 3
+! CHECK-NEXT: ImplicitPart: 3
+! CHECK-NEXT: IntentSpec: 3
+! CHECK-NEXT: IntentSpec::Intent: 3
+! CHECK-NEXT: SpecificationPart: 3
+! CHECK-NEXT: TypeDeclarationStmt: 3
+! CHECK-NEXT: Variable: 3
+! CHECK-NEXT: Block: 2
+! CHECK-NEXT: ComponentDecl: 2
+! CHECK-NEXT: ComponentDefStmt: 2
+! CHECK-NEXT: ComponentOrFill: 2
+! CHECK-NEXT: ContainsStmt: 2
+! CHECK-NEXT: DataComponentDefStmt: 2
+! CHECK-NEXT: DeclarationTypeSpec::Class: 2
+! CHECK-NEXT: DerivedTypeSpec: 2
+! CHECK-NEXT: ExecutionPart: 2
+! CHECK-NEXT: IntegerTypeSpec: 2
+! CHECK-NEXT: IntrinsicTypeSpec::Real: 2
+! CHECK-NEXT: ModuleSubprogram: 2
+! CHECK-NEXT: TypeBoundProcBinding: 2
+! CHECK-NEXT: TypeBoundProcDecl: 2
+! CHECK-NEXT: TypeBoundProcedureStmt: 2
+! CHECK-NEXT: TypeBoundProcedureStmt::WithoutInterface: 2
+! CHECK-NEXT: DerivedTypeDef: 1
+! CHECK-NEXT: DerivedTypeStmt: 1
+! CHECK-NEXT: EndFunctionStmt: 1
+! CHECK-NEXT: EndModuleStmt: 1
+! CHECK-NEXT: EndSubroutineStmt: 1
+! CHECK-NEXT: EndTypeStmt: 1
+! CHECK-NEXT: Expr::Add: 1
+! CHECK-NEXT: FunctionStmt: 1
+! CHECK-NEXT: FunctionSubprogram: 1
+! CHECK-NEXT: ImplicitPartStmt: 1
+! CHECK-NEXT: ImplicitStmt: 1
+! CHECK-NEXT: Module: 1
+! CHECK-NEXT: ModuleStmt: 1
+! CHECK-NEXT: ModuleSubprogramPart: 1
+! CHECK-NEXT: PrefixSpec: 1
+! CHECK-NEXT: Program: 1
+! CHECK-NEXT: ProgramUnit: 1
+! CHECK-NEXT: SubroutineStmt: 1
+! CHECK-NEXT: SubroutineSubprogram: 1
+! CHECK-NEXT: TypeBoundProcedurePart: 1
diff --git a/flang/test/Examples/feature-list-functions.f90 b/flang/test/Examples/feature-list-functions.f90
new file mode 100644
index 0000000000000..386e30d5d18ad
--- /dev/null
+++ b/flang/test/Examples/feature-list-functions.f90
@@ -0,0 +1,76 @@
+! UNSUPPORTED: system-windows
+! REQUIRES: plugins, shell, examples
+
+! RUN: %flang_fc1 -load %llvmshlibdir/flangFeatureList%pluginext \
+! RUN:            -plugin feature-list %s 2>&1 | FileCheck %s
+
+program list_features_test
+    implicit none
+    call test_sub(test_func(2, 3), 4)
+contains
+    subroutine test_sub(a, b)
+        integer, intent(in) :: a, b
+        print "(I0)", a + b
+    end subroutine
+
+    integer function test_func(a, b)
+        integer, intent(in) :: a, b
+        test_func = a * b
+    end function
+end program list_features_test
+
+! CHECK: Name: 19
+! CHECK-NEXT: Expr: 11
+! CHECK-NEXT: DataRef: 5
+! CHECK-NEXT: Designator: 5
+! CHECK-NEXT: ActualArg: 4
+! CHECK-NEXT: ActualArgSpec: 4
+! CHECK-NEXT: EntityDecl: 4
+! CHECK-NEXT: LiteralConstant: 4
+! CHECK-NEXT: ActionStmt: 3
+! CHECK-NEXT: Block: 3
+! CHECK-NEXT: DeclarationTypeSpec: 3
+! CHECK-NEXT: ExecutableConstruct: 3
+! CHECK-NEXT: ExecutionPart: 3
+! CHECK-NEXT: ExecutionPartConstruct: 3
+! CHECK-NEXT: ImplicitPart: 3
+! CHECK-NEXT: IntLiteralConstant: 3
+! CHECK-NEXT: IntegerTypeSpec: 3
+! CHECK-NEXT: IntrinsicTypeSpec: 3
+! CHECK-NEXT: SpecificationPart: 3
+! CHECK-NEXT: AttrSpec: 2
+! CHECK-NEXT: Call: 2
+! CHECK-NEXT: DeclarationConstruct: 2
+! CHECK-NEXT: DummyArg: 2
+! CHECK-NEXT: IntentSpec: 2
+! CHECK-NEXT: IntentSpec::Intent: 2
+! CHECK-NEXT: InternalSubprogram: 2
+! CHECK-NEXT: ProcedureDesignator: 2
+! CHECK-NEXT: SpecificationConstruct: 2
+! CHECK-NEXT: TypeDeclarationStmt: 2
+! CHECK-NEXT: AssignmentStmt: 1
+! CHECK-NEXT: CallStmt: 1
+! CHECK-NEXT: CharLiteralConstant: 1
+! CHECK-NEXT: ContainsStmt: 1
+! CHECK-NEXT: EndFunctionStmt: 1
+! CHECK-NEXT: EndProgramStmt: 1
+! CHECK-NEXT: EndSubroutineStmt: 1
+! CHECK-NEXT: Expr::Add: 1
+! CHECK-NEXT: Expr::Multiply: 1
+! CHECK-NEXT: Format: 1
+! CHECK-NEXT: FunctionReference: 1
+! CHECK-NEXT: FunctionStmt: 1
+! CHECK-NEXT: FunctionSubprogram: 1
+! CHECK-NEXT: ImplicitPartStmt: 1
+! CHECK-NEXT: ImplicitStmt: 1
+! CHECK-NEXT: InternalSubprogramPart: 1
+! CHECK-NEXT: MainProgram: 1
+! CHECK-NEXT: OutputItem: 1
+! CHECK-NEXT: PrefixSpec: 1
+! CHECK-NEXT: PrintStmt: 1
+! CHECK-NEXT: Program: 1
+! CHECK-NEXT: ProgramStmt: 1
+! CHECK-NEXT: ProgramUnit: 1
+! CHECK-NEXT: SubroutineStmt: 1
+! CHECK-NEXT: SubroutineSubprogram: 1
+! CHECK-NEXT: Variable: 1
diff --git a/flang/test/Fir/abstract-results.fir b/flang/test/Fir/abstract-results.fir
index 374c0d18753bb..42ff2a5c8eb2a 100644
--- a/flang/test/Fir/abstract-results.fir
+++ b/flang/test/Fir/abstract-results.fir
@@ -106,6 +106,33 @@ func.func @retcptr() -> !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__addres
   // FUNC-BOX: return %[[VAL]] : i64
 }
 
+// FUNC-REF-LABEL:  func private @arrayfunc_callee_declare(
+// FUNC-REF-SAME: %[[buffer:.*]]: !fir.ref<!fir.array<?xf32>>, %[[n:.*]]: index) {
+// FUNC-BOX-LABEL:  func private @arrayfunc_callee_declare(
+// FUNC-BOX-SAME: %[[box:.*]]: !fir.box<!fir.array<?xf32>>, %[[n:.*]]: index) {
+func.func private @arrayfunc_callee_declare(%n : index) -> !fir.array<?xf32> {
+  %buffer_alloc = fir.alloca !fir.array<?xf32>, %n
+  %shape = fir.shape %n : (index) -> !fir.shape<1>
+  %buffer = fir.declare %buffer_alloc(%shape) {uniq_name = "x"}: (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<?xf32>>
+  // Do something with result (res(4) = 42.)
+  %c4 = arith.constant 4 : i64
+  %coor = fir.coordinate_of %buffer, %c4 : (!fir.ref<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
+  %cst = arith.constant 4.200000e+01 : f32
+  fir.store %cst to %coor : !fir.ref<f32>
+  %res = fir.load %buffer : !fir.ref<!fir.array<?xf32>>
+  return %res : !fir.array<?xf32>
+
+  // FUNC-REF-DAG: %[[buffer_declare:.*]] = fir.declare %[[buffer]](%{{.*}}) {uniq_name = "x"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<?xf32>>
+  // FUNC-REF-DAG: %[[coor:.*]] = fir.coordinate_of %[[buffer_declare]], %{{.*}} : (!fir.ref<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
+  // FUNC-REF-DAG: fir.store %{{.*}} to %[[coor]] : !fir.ref<f32>
+  // FUNC-REF: return
+
+  // FUNC-BOX: %[[buffer:.*]] = fir.box_addr %[[box]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+  // FUNC-BOX-DAG: %[[buffer_declare:.*]] = fir.declare %[[buffer]](%{{.*}}) {uniq_name = "x"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<?xf32>>
+  // FUNC-BOX-DAG: %[[coor:.*]] = fir.coordinate_of %[[buffer_declare]], %{{.*}} : (!fir.ref<!fir.array<?xf32>>, i64) -> !fir.ref<f32>
+  // FUNC-BOX-DAG: fir.store %{{.*}} to %[[coor]] : !fir.ref<f32>
+  // FUNC-BOX: return
+}
 
 // ------------------------ Test caller rewrite --------------------------------
 
diff --git a/flang/test/Fir/basic-program.fir b/flang/test/Fir/basic-program.fir
index e096fb62591e5..e6d849c2f4659 100644
--- a/flang/test/Fir/basic-program.fir
+++ b/flang/test/Fir/basic-program.fir
@@ -17,6 +17,9 @@ func.func @_QQmain() {
 // PASSES: Pass statistics report
 
 // PASSES:        Canonicalizer
+// PASSES-NEXT: 'func.func' Pipeline
+// PASSES-NEXT:   SimplifyHLFIRIntrinsics
+// PASSES-NEXT:   LowerHLFIROrderedAssignments
 // PASSES-NEXT:   LowerHLFIRIntrinsics
 // PASSES-NEXT:   BufferizeHLFIR
 // PASSES-NEXT:   ConvertHLFIRtoFIR
diff --git a/flang/test/Fir/target-rewrite-complex16.fir b/flang/test/Fir/target-rewrite-complex16.fir
index 2627af41c9b8e..0903075ad223f 100644
--- a/flang/test/Fir/target-rewrite-complex16.fir
+++ b/flang/test/Fir/target-rewrite-complex16.fir
@@ -63,57 +63,65 @@ func.func @addrof() {
 // CHECK:         func.func private @paramcomplex16(!fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>})
 
 // CHECK-LABEL:   func.func @callcomplex16() {
-// CHECK:           %[[VAL_0:.*]] = fir.alloca tuple<!fir.real<16>, !fir.real<16>>
-// CHECK:           fir.call @returncomplex16(%[[VAL_0]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
-// CHECK:           %[[VAL_1:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_2]] to %[[VAL_3]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           fir.call @paramcomplex16(%[[VAL_4]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           %[[VAL_0:.*]] = fir.call @llvm.stacksave() : () -> !fir.ref<i8>
+// CHECK:           %[[VAL_1:.*]] = fir.alloca tuple<!fir.real<16>, !fir.real<16>>
+// CHECK:           fir.call @returncomplex16(%[[VAL_1]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.ref<!fir.complex<16>>
+// CHECK:           fir.call @llvm.stackrestore(%[[VAL_0]]) : (!fir.ref<i8>) -> ()
+// CHECK:           %[[VAL_4:.*]] = fir.call @llvm.stacksave() : () -> !fir.ref<i8>
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.complex<16>
+// CHECK:           fir.store %[[VAL_3]] to %[[VAL_5]] : !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
+// CHECK:           fir.call @paramcomplex16(%[[VAL_6]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           fir.call @llvm.stackrestore(%[[VAL_4]]) : (!fir.ref<i8>) -> ()
 // CHECK:           return
 // CHECK:         }
 // CHECK:         func.func private @calleemultipleparamscomplex16(!fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>})
 
 // CHECK-LABEL:   func.func @multipleparamscomplex16(
-// CHECK-SAME:       %[[VAL_0:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, %[[VAL_1:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, %[[VAL_2:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}) {
+// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, %[[VAL_1:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}, %[[VAL_2:.*]]: !fir.ref<tuple<!fir.real<16>, !fir.real<16>>> {llvm.align = 16 : i32, llvm.byval = tuple<!fir.real<16>, !fir.real<16>>}) {
 // CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
 // CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<!fir.complex<16>>
 // CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
 // CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.complex<16>>
 // CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> !fir.ref<!fir.complex<16>>
 // CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_9:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_8]] to %[[VAL_9]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_6]] to %[[VAL_11]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           %[[VAL_13:.*]] = fir.alloca !fir.complex<16>
-// CHECK:           fir.store %[[VAL_4]] to %[[VAL_13]] : !fir.ref<!fir.complex<16>>
-// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
-// CHECK:           fir.call @calleemultipleparamscomplex16(%[[VAL_10]], %[[VAL_12]], %[[VAL_14]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           %[[VAL_9:.*]] = fir.call @llvm.stacksave() : () -> !fir.ref<i8>
+// CHECK:           %[[VAL_10:.*]] = fir.alloca !fir.complex<16>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_10]] : !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
+// CHECK:           %[[VAL_12:.*]] = fir.alloca !fir.complex<16>
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_12]] : !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
+// CHECK:           %[[VAL_14:.*]] = fir.alloca !fir.complex<16>
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_14]] : !fir.ref<!fir.complex<16>>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<!fir.complex<16>>) -> !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>
+// CHECK:           fir.call @calleemultipleparamscomplex16(%[[VAL_11]], %[[VAL_13]], %[[VAL_15]]) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>, !fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
+// CHECK:           fir.call @llvm.stackrestore(%[[VAL_9]]) : (!fir.ref<i8>) -> ()
 // CHECK:           return
 // CHECK:         }
 
 // CHECK-LABEL:   func.func private @mlircomplexf128(
-// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.sret = tuple<f128, f128>}, %[[VAL_1:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>},  %[[VAL_2:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}) {
+// CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.sret = tuple<f128, f128>}, %[[VAL_1:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}, %[[VAL_2:.*]]: !fir.ref<tuple<f128, f128>> {llvm.align = 16 : i32, llvm.byval = tuple<f128, f128>}) {
 // CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_1]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
 // CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] : !fir.ref<complex<f128>>
-// CHECK:           %[[VAL_7:.*]] = fir.alloca tuple<f128, f128>
-// CHECK:           %[[VAL_8:.*]] = fir.alloca complex<f128>
-// CHECK:           fir.store %[[VAL_6]] to %[[VAL_8]] : !fir.ref<complex<f128>>
-// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
-// CHECK:           %[[VAL_10:.*]] = fir.alloca complex<f128>
-// CHECK:           fir.store %[[VAL_4]] to %[[VAL_10]] : !fir.ref<complex<f128>>
-// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
-// CHECK:           fir.call @mlircomplexf128(%[[VAL_7]], %[[VAL_9]], %[[VAL_11]]) : (!fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>) -> ()
-// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
-// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_12]] : !fir.ref<complex<f128>>
-// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
-// CHECK:           fir.store %[[VAL_13]] to %[[VAL_14]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_7:.*]] = fir.call @llvm.stacksave() : () -> !fir.ref<i8>
+// CHECK:           %[[VAL_8:.*]] = fir.alloca tuple<f128, f128>
+// CHECK:           %[[VAL_9:.*]] = fir.alloca complex<f128>
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_9]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
+// CHECK:           %[[VAL_11:.*]] = fir.alloca complex<f128>
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_11]] : !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (!fir.ref<complex<f128>>) -> !fir.ref<tuple<f128, f128>>
+// CHECK:           fir.call @mlircomplexf128(%[[VAL_8]], %[[VAL_10]], %[[VAL_12]]) : (!fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>, !fir.ref<tuple<f128, f128>>) -> ()
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_8]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_13]] : !fir.ref<complex<f128>>
+// CHECK:           fir.call @llvm.stackrestore(%[[VAL_7]]) : (!fir.ref<i8>) -> ()
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<tuple<f128, f128>>) -> !fir.ref<complex<f128>>
+// CHECK:           fir.store %[[VAL_14]] to %[[VAL_15]] : !fir.ref<complex<f128>>
 // CHECK:           return
 // CHECK:         }
 
@@ -122,3 +130,5 @@ func.func @addrof() {
 // CHECK:           %[[VAL_1:.*]] = fir.address_of(@paramcomplex16) : (!fir.ref<tuple<!fir.real<16>, !fir.real<16>>>) -> ()
 // CHECK:           return
 // CHECK:         }
+// CHECK:         func.func private @llvm.stacksave() -> !fir.ref<i8>
+// CHECK:         func.func private @llvm.stackrestore(!fir.ref<i8>)
diff --git a/flang/test/Fir/target.fir b/flang/test/Fir/target.fir
index f6cf0587fcae9..653b100bccbd4 100644
--- a/flang/test/Fir/target.fir
+++ b/flang/test/Fir/target.fir
@@ -87,7 +87,7 @@ func.func @call8() {
   // PPC: = call { double, double } @gen8()
   %1 = fir.call @gen8() : () -> !fir.complex<8>
   // I32: call void @sink8(ptr %
-  // X64: call void @sink8(double %4, double %5)
+  // X64: call void @sink8(double %{{[0-9]*}}, double %{{[0-9]*}})
   // AARCH64: call void @sink8([2 x double] %
   // PPC: call void @sink8(double %{{.*}}, double %{{.*}})
   fir.call @sink8(%1) : (!fir.complex<8>) -> ()
diff --git a/flang/test/HLFIR/assign-codegen.fir b/flang/test/HLFIR/assign-codegen.fir
index 6da5acc7dec2d..401d55bb53940 100644
--- a/flang/test/HLFIR/assign-codegen.fir
+++ b/flang/test/HLFIR/assign-codegen.fir
@@ -206,3 +206,42 @@ func.func @test_alloc_assign_polymorphic(%lhs: !fir.ref<!fir.class<!fir.heap<!fi
 // CHECK:  %[[VAL_7:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<t>>>>>) -> !fir.ref<!fir.box<none>>
 // CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_1]] : (!fir.class<!fir.array<?x!fir.type<t>>>) -> !fir.box<none>
 // CHECK:  %[[VAL_10:.*]] = fir.call @_FortranAAssignPolymorphic(%[[VAL_7]], %[[VAL_8]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+
+func.func @assing_scalar_int_to_polymorphic(%arg0: !fir.ref<!fir.class<!fir.heap<none>>>) {
+  %c123_i32 = arith.constant 123 : i32
+  %0:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "x"} : (!fir.ref<!fir.class<!fir.heap<none>>>) -> (!fir.ref<!fir.class<!fir.heap<none>>>, !fir.ref<!fir.class<!fir.heap<none>>>)
+  hlfir.assign %c123_i32 to %0#0 realloc : i32, !fir.ref<!fir.class<!fir.heap<none>>>
+  return
+}
+
+// CHECK-LABEL:   func.func @assing_scalar_int_to_polymorphic(
+// CHECK-SAME:        %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<none>>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 123 : i32
+// CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "x"} : (!fir.ref<!fir.class<!fir.heap<none>>>) -> !fir.ref<!fir.class<!fir.heap<none>>>
+// CHECK:           %[[VAL_3:.*]] = fir.alloca i32
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_3]] : !fir.ref<i32>
+// CHECK:           %[[VAL_4:.*]] = fir.embox %[[VAL_3]] : (!fir.ref<i32>) -> !fir.box<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.class<!fir.heap<none>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_4]] : (!fir.box<i32>) -> !fir.box<none>
+// CHECK:           %[[VAL_11:.*]] = fir.call @_FortranAAssignPolymorphic(%[[VAL_8]], %[[VAL_9]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+
+func.func @assign_i1_to_polymorphic(%arg0: !fir.ref<!fir.class<!fir.heap<none>>>) {
+  %false = arith.constant false
+  %0:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "x"} : (!fir.ref<!fir.class<!fir.heap<none>>>) -> (!fir.ref<!fir.class<!fir.heap<none>>>, !fir.ref<!fir.class<!fir.heap<none>>>)
+  %1 = hlfir.no_reassoc %false : i1
+  hlfir.assign %1 to %0#0 realloc : i1, !fir.ref<!fir.class<!fir.heap<none>>>
+  return
+}
+
+// CHECK-LABEL:   func.func @assign_i1_to_polymorphic(
+// CHECK-SAME:                                        %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<none>>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "x"} : (!fir.ref<!fir.class<!fir.heap<none>>>) -> !fir.ref<!fir.class<!fir.heap<none>>>
+// CHECK:           %[[VAL_3:.*]] = fir.no_reassoc %[[VAL_1]] : i1
+// CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i1) -> !fir.logical<4>
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.logical<4>
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref<!fir.logical<4>>
+// CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]] : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.class<!fir.heap<none>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_6]] : (!fir.box<!fir.logical<4>>) -> !fir.box<none>
+// CHECK:           %[[VAL_13:.*]] = fir.call @_FortranAAssignPolymorphic(%[[VAL_10]], %[[VAL_11]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none
diff --git a/flang/test/HLFIR/boxchar_emboxing.f90 b/flang/test/HLFIR/boxchar_emboxing.f90
new file mode 100644
index 0000000000000..aec6a95fda965
--- /dev/null
+++ b/flang/test/HLFIR/boxchar_emboxing.f90
@@ -0,0 +1,87 @@
+! RUN: bbc -polymorphic-type -emit-fir -hlfir %s -o - | FileCheck %s
+
+! CHECK-LABEL:   func.func @_QPtest1(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.class<none> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ex"} : (!fir.class<none>) -> (!fir.class<none>, !fir.class<none>)
+! CHECK:           fir.select_type %[[VAL_1]]#1 : !fir.class<none> [#fir.type_is<!fir.char<1,?>>, ^bb1, unit, ^bb2]
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_2:.*]] = fir.box_addr %[[VAL_1]]#1 : (!fir.class<none>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_3:.*]] = fir.box_elesize %[[VAL_1]]#1 : (!fir.class<none>) -> index
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_2]] typeparams %[[VAL_3]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:           %[[VAL_5:.*]]:2 = fir.unboxchar %[[VAL_4]]#0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]]#0 typeparams %[[VAL_3]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK:           %[[VAL_7:.*]] = fir.rebox %[[VAL_6]] : (!fir.box<!fir.char<1,?>>) -> !fir.class<none>
+! CHECK:           fir.call @_QPprint(%[[VAL_7]]) fastmath<contract> : (!fir.class<none>) -> ()
+! CHECK:           cf.br ^bb3
+! CHECK:         ^bb2:
+! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1]]#1 {uniq_name = "_QFtest1Ex"} : (!fir.class<none>) -> (!fir.class<none>, !fir.class<none>)
+! CHECK:           %[[VAL_9:.*]] = fir.address_of(@_QQcl.4641494C) : !fir.ref<!fir.char<1,4>>
+! CHECK:           %[[VAL_10:.*]] = arith.constant 4 : index
+! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]] typeparams %[[VAL_10]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQcl.4641494C"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]]#1 : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+! CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+! CHECK:           %[[VAL_14:.*]] = arith.constant false
+! CHECK:           %[[VAL_15:.*]] = arith.constant false
+! CHECK:           %[[VAL_16:.*]] = fir.call @_FortranAStopStatementText(%[[VAL_12]], %[[VAL_13]], %[[VAL_14]], %[[VAL_15]]) fastmath<contract> : (!fir.ref<i8>, i64, i1, i1) -> none
+! CHECK:           fir.unreachable
+! CHECK:         ^bb3:
+! CHECK:           return
+! CHECK:         }
+subroutine test1(x)
+  interface
+     subroutine print(x)
+       class(*) x
+     end subroutine print
+  end interface
+  class(*) x
+  select type(x)
+  type is (Character(*))
+     call print(x)
+  class Default
+     stop 'FAIL'
+  end select
+end subroutine test1
+
+! CHECK-LABEL:   func.func @_QPtest2(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.class<!fir.array<10xnone>> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ex"} : (!fir.class<!fir.array<10xnone>>) -> (!fir.class<!fir.array<10xnone>>, !fir.class<!fir.array<10xnone>>)
+! CHECK:           fir.select_type %[[VAL_1]]#1 : !fir.class<!fir.array<10xnone>> [#fir.type_is<!fir.char<1,?>>, ^bb1, unit, ^bb2]
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.class<!fir.array<10xnone>>) -> !fir.box<!fir.array<10x!fir.char<1,?>>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest2Ex"} : (!fir.box<!fir.array<10x!fir.char<1,?>>>) -> (!fir.box<!fir.array<10x!fir.char<1,?>>>, !fir.box<!fir.array<10x!fir.char<1,?>>>)
+! CHECK:           %[[VAL_4:.*]] = fir.box_elesize %[[VAL_3]]#1 : (!fir.box<!fir.array<10x!fir.char<1,?>>>) -> index
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_6:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_5]])  typeparams %[[VAL_4]] : (!fir.box<!fir.array<10x!fir.char<1,?>>>, index, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[VAL_6]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_8:.*]] = fir.embox %[[VAL_7]]#0 typeparams %[[VAL_4]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK:           %[[VAL_9:.*]] = fir.rebox %[[VAL_8]] : (!fir.box<!fir.char<1,?>>) -> !fir.class<none>
+! CHECK:           fir.call @_QPprint(%[[VAL_9]]) fastmath<contract> : (!fir.class<none>) -> ()
+! CHECK:           cf.br ^bb3
+! CHECK:         ^bb2:
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]]#1 {uniq_name = "_QFtest2Ex"} : (!fir.class<!fir.array<10xnone>>) -> (!fir.class<!fir.array<10xnone>>, !fir.class<!fir.array<10xnone>>)
+! CHECK:           %[[VAL_11:.*]] = fir.address_of(@_QQcl.4641494C) : !fir.ref<!fir.char<1,4>>
+! CHECK:           %[[VAL_12:.*]] = arith.constant 4 : index
+! CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_11]] typeparams %[[VAL_12]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQcl.4641494C"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]]#1 : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+! CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_12]] : (index) -> i64
+! CHECK:           %[[VAL_16:.*]] = arith.constant false
+! CHECK:           %[[VAL_17:.*]] = arith.constant false
+! CHECK:           %[[VAL_18:.*]] = fir.call @_FortranAStopStatementText(%[[VAL_14]], %[[VAL_15]], %[[VAL_16]], %[[VAL_17]]) fastmath<contract> : (!fir.ref<i8>, i64, i1, i1) -> none
+! CHECK:           fir.unreachable
+! CHECK:         ^bb3:
+! CHECK:           return
+! CHECK:         }
+subroutine test2(x)
+  interface
+     subroutine print(x)
+       class(*) x
+     end subroutine print
+  end interface
+  class(*) x(10)
+  select type(x)
+  type is (Character(*))
+     call print(x(1))
+  class Default
+     stop 'FAIL'
+  end select
+end subroutine test2
diff --git a/flang/test/HLFIR/c_ptr_byvalue.f90 b/flang/test/HLFIR/c_ptr_byvalue.f90
new file mode 100644
index 0000000000000..7c549335efcc1
--- /dev/null
+++ b/flang/test/HLFIR/c_ptr_byvalue.f90
@@ -0,0 +1,41 @@
+! RUN: bbc -emit-fir -hlfir %s -o - | FileCheck %s
+
+! CHECK-LABEL:   func.func @_QPtest1() {
+! CHECK:           %[[VAL_110:.*]]:3 = hlfir.associate %{{.*}} {uniq_name = "adapt.cptrbyval"} : (!hlfir.expr<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1)
+! CHECK:           %[[VAL_111:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_112:.*]] = fir.coordinate_of %[[VAL_110]]#1, %[[VAL_111]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+! CHECK:           %[[VAL_113:.*]] = fir.load %[[VAL_112]] : !fir.ref<i64>
+! CHECK:           %[[VAL_114:.*]] = fir.convert %[[VAL_113]] : (i64) -> !fir.ref<i64>
+! CHECK:           hlfir.end_associate %[[VAL_110]]#1, %[[VAL_110]]#2 : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, i1
+! CHECK:           fir.call @get_expected_f(%[[VAL_114]]) fastmath<contract> : (!fir.ref<i64>) -> ()
+subroutine test1
+  use iso_c_binding
+  interface
+     subroutine get_expected_f(src) bind(c)
+       use iso_c_binding
+       type(c_ptr), value :: src
+     end subroutine get_expected_f
+  end interface
+  real, target,  dimension(1) :: r_src
+  call get_expected_f(c_loc(r_src))
+end
+
+! CHECK-LABEL:   func.func @_QPtest2(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>> {fir.bindc_name = "cptr"}) {
+! CHECK:           %[[VAL_97:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ecptr"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>)
+! CHECK:           %[[VAL_98:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_99:.*]] = fir.coordinate_of %[[VAL_97]]#0, %[[VAL_98]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+! CHECK:           %[[VAL_100:.*]] = fir.load %[[VAL_99]] : !fir.ref<i64>
+! CHECK:           %[[VAL_101:.*]] = fir.convert %[[VAL_100]] : (i64) -> !fir.ref<i64>
+! CHECK:           fir.call @get_expected_f(%[[VAL_101]]) fastmath<contract> : (!fir.ref<i64>) -> ()
+subroutine test2(cptr)
+  use iso_c_binding
+  interface
+     subroutine get_expected_f(src) bind(c)
+       use iso_c_binding
+       type(c_ptr), value :: src
+     end subroutine get_expected_f
+  end interface
+  type(c_ptr) :: cptr
+  call get_expected_f(cptr)
+end
diff --git a/flang/test/HLFIR/call_with_poly_dummy.f90 b/flang/test/HLFIR/call_with_poly_dummy.f90
new file mode 100644
index 0000000000000..19776ba68ff06
--- /dev/null
+++ b/flang/test/HLFIR/call_with_poly_dummy.f90
@@ -0,0 +1,44 @@
+! RUN: bbc -polymorphic-type -emit-fir -hlfir %s -o - | FileCheck %s
+
+! Test passing arguments to subprograms with polymorphic dummy arguments.
+
+! CHECK-LABEL:   func.func @_QPtest1() {
+! CHECK:           %[[VAL_0:.*]] = arith.constant 17 : i32
+! CHECK:           %[[VAL_1:.*]]:3 = hlfir.associate %[[VAL_0]] {uniq_name = "adapt.valuebyref"} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK:           %[[VAL_2:.*]] = fir.embox %[[VAL_1]]#0 : (!fir.ref<i32>) -> !fir.box<i32>
+! CHECK:           %[[VAL_3:.*]] = fir.rebox %[[VAL_2]] : (!fir.box<i32>) -> !fir.class<none>
+! CHECK:           fir.call @_QPcallee(%[[VAL_3]]) fastmath<contract> : (!fir.class<none>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_1]]#1, %[[VAL_1]]#2 : !fir.ref<i32>, i1
+! CHECK:           return
+! CHECK:         }
+subroutine test1
+  interface
+     subroutine callee(x)
+       class(*) x
+     end subroutine callee
+  end interface
+  call callee(17)
+end subroutine test1
+
+! CHECK-LABEL:   func.func @_QPtest2(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<f32> {fir.bindc_name = "x"}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ex"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<f32>
+! CHECK:           %[[VAL_3:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:           %[[VAL_4:.*]] = arith.cmpf oeq, %[[VAL_2]], %[[VAL_3]] : f32
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
+! CHECK:           %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_5]] {uniq_name = "adapt.valuebyref"} : (!fir.logical<4>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, i1)
+! CHECK:           %[[VAL_7:.*]] = fir.embox %[[VAL_6]]#0 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+! CHECK:           %[[VAL_8:.*]] = fir.rebox %[[VAL_7]] : (!fir.box<!fir.logical<4>>) -> !fir.class<none>
+! CHECK:           fir.call @_QPcallee(%[[VAL_8]]) fastmath<contract> : (!fir.class<none>) -> ()
+! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<!fir.logical<4>>, i1
+! CHECK:           return
+! CHECK:         }
+subroutine test2(x)
+  interface
+     subroutine callee(x)
+       class(*) x
+     end subroutine callee
+  end interface
+  call callee(x.eq.0)
+end subroutine test2
diff --git a/flang/test/HLFIR/char_assign.fir b/flang/test/HLFIR/char_assign.fir
new file mode 100644
index 0000000000000..06748cdcebe6c
--- /dev/null
+++ b/flang/test/HLFIR/char_assign.fir
@@ -0,0 +1,44 @@
+// RUN: fir-opt --split-input-file %s -convert-hlfir-to-fir | FileCheck %s
+
+// Verify that the special case of length-one character assignment
+// is using valid fir.converts.
+
+// -----
+
+// CHECK-LABEL: func.func @_QPtest1
+// CHECK: [[C_DECL:%[0-9]*]] = fir.declare{{.*}}{uniq_name = "_QFtest1Ec"} : (!fir.ref<!fir.char<1,?>>, index) -> !fir.ref<!fir.char<1,?>>
+// CHECK: [[LOCAL_DECL:%[0-9]*]] = fir.declare{{.*}}{uniq_name = "_QFtest1Elocal"} : (!fir.ref<!fir.char<1>>, index) -> !fir.ref<!fir.char<1>>
+// CHECK: [[CVT:%[0-9]*]] = fir.convert [[C_DECL]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+// CHECK: [[LD:%[0-9]*]] = fir.load [[CVT]] : !fir.ref<!fir.char<1>>
+// CHECK: fir.store [[LD]] to [[LOCAL_DECL]] : !fir.ref<!fir.char<1>>
+module {
+  func.func @_QPtest1(%arg0: !fir.boxchar<1> {fir.bindc_name = "c"}) {
+    %c1 = arith.constant 1 : index
+    %0:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+    %1:2 = hlfir.declare %0#0 typeparams %c1 {uniq_name = "_QFtest1Ec"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+    %2 = fir.alloca !fir.char<1> {bindc_name = "local", uniq_name = "_QFtest1Elocal"}
+    %3:2 = hlfir.declare %2 typeparams %c1 {uniq_name = "_QFtest1Elocal"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+    hlfir.assign %1#0 to %3#0 : !fir.boxchar<1>, !fir.ref<!fir.char<1>>
+    return
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func.func @_QPtest2
+// CHECK: [[C_DECL:%[0-9]*]] = fir.declare{{.*}}{uniq_name = "_QFtest2Ec"} : (!fir.ref<!fir.char<1,?>>, index) -> !fir.ref<!fir.char<1,?>>
+// CHECK: [[LOCAL_DECL:%[0-9]*]] = fir.declare{{.*}}{uniq_name = "_QFtest2Elocal"} : (!fir.ref<!fir.char<1>>, index) -> !fir.ref<!fir.char<1>>
+// CHECK: [[LD:%[0-9]*]] = fir.load [[LOCAL_DECL]] : !fir.ref<!fir.char<1>>
+// CHECK: [[CVT:%[0-9]*]] = fir.convert [[C_DECL]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+// CHECK: fir.store [[LD]] to [[CVT]] : !fir.ref<!fir.char<1>>
+module {
+  func.func @_QPtest2(%arg0: !fir.boxchar<1> {fir.bindc_name = "c"}) {
+    %c1 = arith.constant 1 : index
+    %0:2 = fir.unboxchar %arg0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+    %1:2 = hlfir.declare %0#0 typeparams %c1 {uniq_name = "_QFtest2Ec"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+    %2 = fir.alloca !fir.char<1> {bindc_name = "local", uniq_name = "_QFtest2Elocal"}
+    %3:2 = hlfir.declare %2 typeparams %c1 {uniq_name = "_QFtest2Elocal"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+    hlfir.assign %3#0 to %1#0 : !fir.ref<!fir.char<1>>, !fir.boxchar<1>
+    return
+  }
+}
diff --git a/flang/test/HLFIR/dummy_deallocation.f90 b/flang/test/HLFIR/dummy_deallocation.f90
new file mode 100644
index 0000000000000..9d3c51c843bcc
--- /dev/null
+++ b/flang/test/HLFIR/dummy_deallocation.f90
@@ -0,0 +1,16 @@
+! RUN: bbc -emit-fir -hlfir %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+
+! Test that the intent(out) allocatable dummy argument
+! is not deallocated in entry SUB_B.
+
+! CHECK-LABEL: func.func @_QPsub_a
+! CHECK: fir.freemem
+
+! CHECK-LABEL: func.func @_QPsub_b
+! CHECK-NOT: fir.freemem
+SUBROUTINE SUB_A(A)
+  INTEGER, INTENT(out), ALLOCATABLE, DIMENSION (:) :: A
+  RETURN
+  ENTRY SUB_B
+END SUBROUTINE SUB_A
diff --git a/flang/test/HLFIR/element-addr.fir b/flang/test/HLFIR/element-addr.fir
new file mode 100644
index 0000000000000..c6f2b06b9f688
--- /dev/null
+++ b/flang/test/HLFIR/element-addr.fir
@@ -0,0 +1,83 @@
+// Test hlfir.element_addr operation parse, verify (no errors), and unparse.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @test_element_addr(%x: !fir.ref<!fir.array<100xf32>>, %vector: !fir.ref<!fir.array<20xi32>>, %y: !fir.ref<!fir.array<20xf32>>) {
+  %c20 = arith.constant 20 : index
+  %vector_shape = fir.shape %c20 : (index) -> !fir.shape<1>
+  hlfir.region_assign {
+    hlfir.yield %y : !fir.ref<!fir.array<20xf32>>
+  } to {
+    hlfir.elemental_addr %vector_shape  : !fir.shape<1> {
+      ^bb0(%i: index):
+      %0 = hlfir.designate %vector (%i)  : (!fir.ref<!fir.array<20xi32>>, index) -> !fir.ref<i32>
+      %1 = fir.load %0 : !fir.ref<i32>
+      %x_element_addr = hlfir.designate %x (%1)  : (!fir.ref<!fir.array<100xf32>>, i32) -> !fir.ref<f32>
+      hlfir.yield %x_element_addr : !fir.ref<f32>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_element_addr(
+// CHECK-SAME:                                 %[[VAL_0:.*]]: !fir.ref<!fir.array<100xf32>>,
+// CHECK-SAME:                                 %[[VAL_1:.*]]: !fir.ref<!fir.array<20xi32>>,
+// CHECK-SAME:                                 %[[VAL_2:.*]]: !fir.ref<!fir.array<20xf32>>) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant 20 : index
+// CHECK:           %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+// CHECK:           hlfir.region_assign {
+// CHECK:             hlfir.yield %[[VAL_2]] : !fir.ref<!fir.array<20xf32>>
+// CHECK:           } to {
+// CHECK:             hlfir.elemental_addr %[[VAL_4]] : !fir.shape<1> {
+// CHECK:             ^bb0(%[[VAL_5:.*]]: index):
+// CHECK:               %[[VAL_6:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_5]])  : (!fir.ref<!fir.array<20xi32>>, index) -> !fir.ref<i32>
+// CHECK:               %[[VAL_7:.*]] = fir.load %[[VAL_6]] : !fir.ref<i32>
+// CHECK:               %[[VAL_8:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_7]])  : (!fir.ref<!fir.array<100xf32>>, i32) -> !fir.ref<f32>
+// CHECK:               hlfir.yield %[[VAL_8]] : !fir.ref<f32>
+// CHECK:             }
+// CHECK:           }
+
+
+
+func.func @test_element_addr_cleanup(%x: !fir.box<!fir.array<?x!fir.char<1,?>>>, %y: !fir.box<!fir.array<?x!fir.char<1,?>>>) {
+  hlfir.region_assign {
+    hlfir.yield %y : !fir.box<!fir.array<?x!fir.char<1,?>>>
+  } to {
+    %c0 = arith.constant 0 : index
+    %len = fir.box_elesize %x : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
+    %vector = fir.call @returns_alloc() : () -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+    %vector_dim:3 = fir.box_dims %x, %c0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index) -> (index, index, index)
+    %vector_shape = fir.shape %vector_dim#1 : (index) -> !fir.shape<1>
+    hlfir.elemental_addr %vector_shape  typeparams %len : !fir.shape<1>, index {
+      ^bb0(%i: index):
+      %0 = hlfir.designate %vector (%i)  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+      %1 = fir.load %0 : !fir.ref<i32>
+      %x_element_addr = hlfir.designate %x (%1) typeparams %len  : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32, index) -> !fir.boxchar<1>
+      hlfir.yield %x_element_addr : !fir.boxchar<1>
+    } cleanup {
+      %addr = fir.box_addr %vector : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+      fir.freemem %addr : !fir.heap<!fir.array<?xi32>>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_element_addr_cleanup(
+// CHECK-SAME:                                         %[[VAL_0:[^:]*]]: !fir.box<!fir.array<?x!fir.char<1,?>>>,
+// CHECK-SAME:                                         %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>>) {
+// CHECK:           hlfir.region_assign {
+// CHECK:             hlfir.yield %[[VAL_1]] : !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           } to {
+// CHECK:             %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:             %[[VAL_3:.*]] = fir.box_elesize %[[VAL_0]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
+// CHECK:             %[[VAL_4:.*]] = fir.call @returns_alloc() : () -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:             %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index) -> (index, index, index)
+// CHECK:             %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
+// CHECK:             hlfir.elemental_addr %[[VAL_6]] typeparams %[[VAL_3]] : !fir.shape<1>, index {
+// CHECK:             ^bb0(%[[VAL_7:.*]]: index):
+// CHECK:               %[[VAL_8:.*]] = hlfir.designate %[[VAL_4]] (%[[VAL_7]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+// CHECK:               %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref<i32>
+// CHECK:               %[[VAL_10:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_9]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32, index) -> !fir.boxchar<1>
+// CHECK:               hlfir.yield %[[VAL_10]] : !fir.boxchar<1>
+// CHECK:             } cleanup {
+// CHECK:               %[[VAL_11:.*]] = fir.box_addr %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+// CHECK:               fir.freemem %[[VAL_11]] : !fir.heap<!fir.array<?xi32>>
+// CHECK:             }
+// CHECK:           }
diff --git a/flang/test/HLFIR/elsewhere.fir b/flang/test/HLFIR/elsewhere.fir
new file mode 100644
index 0000000000000..b0033a6dc5b4f
--- /dev/null
+++ b/flang/test/HLFIR/elsewhere.fir
@@ -0,0 +1,82 @@
+// Test hlfir.elsewhere operation parse, verify (no errors), and unparse.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @test_elsewhere(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>, %x: !fir.ref<!fir.array<10xf32>>, %y: !fir.box<!fir.array<?xf32>>) {
+  hlfir.where {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+    hlfir.elsewhere do {
+      hlfir.region_assign {
+        hlfir.yield %y : !fir.box<!fir.array<?xf32>>
+      } to {
+        hlfir.yield %x : !fir.ref<!fir.array<10xf32>>
+      }
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_elsewhere(
+// CHECK-SAME:                              %[[VAL_0:.*]]: !fir.ref<!fir.array<10x!fir.logical<4>>>,
+// CHECK-SAME:                              %[[VAL_1:.*]]: !fir.ref<!fir.array<10xf32>>,
+// CHECK-SAME:                              %[[VAL_2:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           hlfir.where {
+// CHECK:             hlfir.yield %[[VAL_0]] : !fir.ref<!fir.array<10x!fir.logical<4>>>
+// CHECK:           } do {
+// CHECK:             hlfir.elsewhere do {
+// CHECK:               hlfir.region_assign {
+// CHECK:                 hlfir.yield %[[VAL_2]] : !fir.box<!fir.array<?xf32>>
+// CHECK:               } to {
+// CHECK:                 hlfir.yield %[[VAL_1]] : !fir.ref<!fir.array<10xf32>>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+
+func.func @test_masked_elsewhere(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>, %x: !fir.ref<!fir.array<10xf32>>, %y: !fir.box<!fir.array<?xf32>>) {
+  hlfir.where {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+    hlfir.elsewhere mask {
+      %other_mask = fir.call @get_mask() : () -> !fir.ptr<!fir.array<10x!fir.logical<4>>>
+      hlfir.yield %other_mask : !fir.ptr<!fir.array<10x!fir.logical<4>>>
+    } do {
+      hlfir.region_assign {
+        hlfir.yield %y : !fir.box<!fir.array<?xf32>>
+      } to {
+        hlfir.yield %x : !fir.ref<!fir.array<10xf32>>
+      }
+      hlfir.elsewhere do {
+        hlfir.region_assign {
+          hlfir.yield %x : !fir.ref<!fir.array<10xf32>>
+        } to {
+          hlfir.yield %y : !fir.box<!fir.array<?xf32>>
+        }
+      }
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_masked_elsewhere(
+// CHECK-SAME:                                     %[[VAL_0:.*]]: !fir.ref<!fir.array<10x!fir.logical<4>>>,
+// CHECK-SAME:                                     %[[VAL_1:.*]]: !fir.ref<!fir.array<10xf32>>,
+// CHECK-SAME:                                     %[[VAL_2:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           hlfir.where {
+// CHECK:             hlfir.yield %[[VAL_0]] : !fir.ref<!fir.array<10x!fir.logical<4>>>
+// CHECK:           } do {
+// CHECK:             hlfir.elsewhere mask {
+// CHECK:               %[[VAL_3:.*]] = fir.call @get_mask() : () -> !fir.ptr<!fir.array<10x!fir.logical<4>>>
+// CHECK:               hlfir.yield %[[VAL_3]] : !fir.ptr<!fir.array<10x!fir.logical<4>>>
+// CHECK:             } do {
+// CHECK:               hlfir.region_assign {
+// CHECK:                 hlfir.yield %[[VAL_2]] : !fir.box<!fir.array<?xf32>>
+// CHECK:               } to {
+// CHECK:                 hlfir.yield %[[VAL_1]] : !fir.ref<!fir.array<10xf32>>
+// CHECK:               }
+// CHECK:               hlfir.elsewhere do {
+// CHECK:                 hlfir.region_assign {
+// CHECK:                   hlfir.yield %[[VAL_1]] : !fir.ref<!fir.array<10xf32>>
+// CHECK:                 } to {
+// CHECK:                   hlfir.yield %[[VAL_2]] : !fir.box<!fir.array<?xf32>>
+// CHECK:                 }
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
diff --git a/flang/test/HLFIR/extents-of-shape-of.f90 b/flang/test/HLFIR/extents-of-shape-of.f90
index 85479ebb407f6..8fc7133985ffc 100644
--- a/flang/test/HLFIR/extents-of-shape-of.f90
+++ b/flang/test/HLFIR/extents-of-shape-of.f90
@@ -1,7 +1,7 @@
 ! RUN: bbc -emit-fir -hlfir %s -o - | FileCheck --check-prefix CHECK-ALL --check-prefix CHECK-HLFIR %s
 ! RUN: bbc -emit-fir -hlfir %s -o - | fir-opt --lower-hlfir-intrinsics | fir-opt --bufferize-hlfir | fir-opt --convert-hlfir-to-fir | FileCheck --check-prefix CHECK-ALL --check-prefix CHECK-FIR %s
 subroutine foo(a, b)
-  real :: a(:, :), b(:, :)
+  real :: a(2, 2), b(:, :)
   interface
     elemental subroutine elem_sub(x)
       real, intent(in) :: x
@@ -10,19 +10,19 @@ elemental subroutine elem_sub(x)
   call elem_sub(matmul(a, b))
 end subroutine
 ! CHECK-ALL-LABEL:   func.func @_QPfoo
-! CHECK-ALL:             %[[A_ARG:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "a"}
+! CHECK-ALL:             %[[A_ARG:.*]]: !fir.ref<!fir.array<2x2xf32>> {fir.bindc_name = "a"}
 ! CHECK-ALL:             %[[B_ARG:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "b"}
 
 ! CHECK-HLFIR-DAG:     %[[A_VAR:.*]]:2 = hlfir.declare %[[A_ARG]]
 ! CHECK-HLFIR-DAG:     %[[B_VAR:.*]]:2 = hlfir.declare %[[B_ARG]]
 ! CHECK-HLFIR-NEXT:    %[[MUL:.*]] = hlfir.matmul %[[A_VAR]]#0 %[[B_VAR]]#0
-! CHECK-HLFIR-NEXT:    %[[SHAPE:.*]] = hlfir.shape_of %[[MUL]] : (!hlfir.expr<?x?xf32>) -> !fir.shape<2>
-! CHECK-HLFIR-NEXT:    %[[EXT0:.*]] = hlfir.get_extent %[[SHAPE]] {dim = 0 : index} : (!fir.shape<2>) -> index
+! CHECK-HLFIR-NEXT:    %[[SHAPE:.*]] = hlfir.shape_of %[[MUL]] : (!hlfir.expr<2x?xf32>) -> !fir.shape<2>
+! CHECK-HLFIR-NEXT:    %[[EXT0:.*]] = arith.constant 2 : index
 ! CHECK-HLFIR-NEXT:    %[[EXT1:.*]] = hlfir.get_extent %[[SHAPE]] {dim = 1 : index} : (!fir.shape<2>) -> index
 ! CHECK-HLFIR-NEXT:    %[[C1:.*]] = arith.constant 1 : index
 ! CHECK-HLFIR-NEXT:    fir.do_loop %[[ARG2:.*]] = %[[C1]] to %[[EXT1]] step %[[C1]] {
 ! CHECK-HLFIR-NEXT:      fir.do_loop %[[ARG3:.*]] = %[[C1]] to %[[EXT0]] step %[[C1]] {
-! CHECK-HLFIR-NEXT:        %[[ELE:.*]] = hlfir.apply %[[MUL]], %[[ARG3]], %[[ARG2]] : (!hlfir.expr<?x?xf32>, index, index) -> f32
+! CHECK-HLFIR-NEXT:        %[[ELE:.*]] = hlfir.apply %[[MUL]], %[[ARG3]], %[[ARG2]] : (!hlfir.expr<2x?xf32>, index, index) -> f32
 ! CHECK-HLFIR-NEXT:        %[[ASSOC:.*]]:3 = hlfir.associate %[[ELE]] {uniq_name = "adapt.valuebyref"} : (f32) -> (!fir.ref<f32>, !fir.ref<f32>, i1)
 ! CHECK-HLFIR-NEXT:        fir.call
 ! CHECK-HLFIR-NEXT:        hlfir.end_associate
@@ -39,9 +39,10 @@ elemental subroutine elem_sub(x)
 ! CHECK-FIR-NEXT:      %[[DIMS1:.*]]:3 = fir.box_dims %[[MUL]], %[[C1]]
 ! ...
 ! CHECK-FIR:           %[[SHAPE:.*]] = fir.shape %[[DIMS0]]#1, %[[DIMS1]]#1
+! CHECK-FIR-NEXT:      %[[C2:.*]] = arith.constant 2 : index
 ! CHECK-FIR-NEXT:      %[[C1_1:.*]] = arith.constant 1 : index
 ! CHECK-FIR-NEXT:      fir.do_loop %[[ARG2:.*]] = %[[C1_1]] to %[[DIMS1]]#1 step %[[C1_1]] {
-! CHECK-FIR-NEXT:        fir.do_loop %[[ARG3:.*]] = %[[C1_1]] to %[[DIMS0]]#1 step %[[C1_1]] {
+! CHECK-FIR-NEXT:        fir.do_loop %[[ARG3:.*]] = %[[C1_1]] to %[[C2]] step %[[C1_1]] {
 ! ...
 
 ! CHECK-ALL:           return
diff --git a/flang/test/HLFIR/forall-index.fir b/flang/test/HLFIR/forall-index.fir
new file mode 100644
index 0000000000000..02c5f412ebd3a
--- /dev/null
+++ b/flang/test/HLFIR/forall-index.fir
@@ -0,0 +1,44 @@
+// Test hlfir.forall_index operation parse, verify (no errors), unparse,
+// and canonicalization.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+// RUN: fir-opt -canonicalize %s | FileCheck %s --check-prefix=CANONICALIZATION
+
+func.func @forall_index(%x: !fir.ref<!fir.array<10xf32>>, %y: !fir.ref<!fir.array<10xf32>>) {
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  hlfir.forall lb {
+    hlfir.yield %c1 : index
+  } ub {
+    hlfir.yield %c10 : index
+  } (%arg0: i64) {
+    %i = hlfir.forall_index "i" %arg0 : (i64) -> !fir.ref<i64>
+    hlfir.region_assign {
+      %ival = fir.load %i : !fir.ref<i64>
+      %yi = hlfir.designate %y(%ival) : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+      hlfir.yield %yi : !fir.ref<f32>
+    } to {
+      %ival = fir.load %i : !fir.ref<i64>
+      %xi = hlfir.designate %x(%ival) : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+      hlfir.yield %xi : !fir.ref<f32>
+    }
+    hlfir.region_assign {
+      %res = fir.call @taking_address(%i) : (!fir.ref<i64>) -> f32
+      hlfir.yield %res : f32
+    } to {
+      %ival = fir.load %i : !fir.ref<i64>
+      %xi = hlfir.designate %x(%ival) : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+      hlfir.yield %xi : !fir.ref<f32>
+    }
+  }
+  return
+}
+// CHECK:           hlfir.forall lb {
+// CHECK:           } ub {
+// CHECK:           }  (%[[VAL_4:.*]]: i64) {
+// CHECK:             hlfir.forall_index "i" %[[VAL_4]] : (i64) -> !fir.ref<i64>
+
+// CANONICALIZATION:  %[[VAL_5:.*]] = hlfir.forall_index "i" %[[VAL_4:.*]] : (i64) -> !fir.ref<i64>
+// CANONICALIZATION:  hlfir.designate %{{.*}} (%[[VAL_4]])  : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+// CANONICALIZATION:  hlfir.designate %{{.*}} (%[[VAL_4]])  : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+// CANONICALIZATION:  fir.call @taking_address(%[[VAL_5]]) : (!fir.ref<i64>) -> f32
+// CANONICALIZATION:  hlfir.designate %{{.*}} (%[[VAL_4]])  : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
diff --git a/flang/test/HLFIR/forall.fir b/flang/test/HLFIR/forall.fir
new file mode 100644
index 0000000000000..2ae91deafcacd
--- /dev/null
+++ b/flang/test/HLFIR/forall.fir
@@ -0,0 +1,110 @@
+// Test hlfir.forall operation parse, verify (no errors), and unparse.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @forall_test(%x: !fir.box<!fir.array<?xf32>>, %x2: !fir.box<!fir.array<?x?xf32>>) {
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  hlfir.forall lb {
+    hlfir.yield %c1 : index
+  } ub {
+    hlfir.yield %c10 : index
+  } (%i : index) {
+    hlfir.region_assign {
+      %res = fir.call @foo(%i) : (index) -> f32
+      hlfir.yield %res : f32
+    } to {
+      %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+      hlfir.yield %xi : !fir.ref<f32>
+    }
+    hlfir.forall lb {
+      hlfir.yield %c1 : index
+    } ub {
+      hlfir.yield %c10 : index
+    } (%j : index) {
+      hlfir.region_assign {
+        %jf = fir.convert %j : (index) -> f32
+        hlfir.yield %jf : f32
+      } to {
+        %xij = hlfir.designate %x2(%i, %j) : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+        hlfir.yield %xij : !fir.ref<f32>
+      }
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @forall_test(
+// CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>,
+// CHECK-SAME:                           %[[VAL_1:.*]]: !fir.box<!fir.array<?x?xf32>>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           hlfir.forall lb {
+// CHECK:             hlfir.yield %[[VAL_2]] : index
+// CHECK:           } ub {
+// CHECK:             hlfir.yield %[[VAL_3]] : index
+// CHECK:           }  (%[[VAL_4:.*]]: index) {
+// CHECK:             hlfir.region_assign {
+// CHECK:               %[[VAL_5:.*]] = fir.call @foo(%[[VAL_4]]) : (index) -> f32
+// CHECK:               hlfir.yield %[[VAL_5]] : f32
+// CHECK:             } to {
+// CHECK:               %[[VAL_6:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_4]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               hlfir.yield %[[VAL_6]] : !fir.ref<f32>
+// CHECK:             }
+// CHECK:             hlfir.forall lb {
+// CHECK:               hlfir.yield %[[VAL_2]] : index
+// CHECK:             } ub {
+// CHECK:               hlfir.yield %[[VAL_3]] : index
+// CHECK:             }  (%[[VAL_7:.*]]: index) {
+// CHECK:               hlfir.region_assign {
+// CHECK:                 %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (index) -> f32
+// CHECK:                 hlfir.yield %[[VAL_8]] : f32
+// CHECK:               } to {
+// CHECK:                 %[[VAL_9:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_4]], %[[VAL_7]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.yield %[[VAL_9]] : !fir.ref<f32>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+
+func.func @forall_test_step(%x : !fir.box<!fir.array<10xf32>>, %y: !fir.box<!fir.array<?xf32>>) {
+  hlfir.forall lb {
+    %c1 = arith.constant 1 : index
+    hlfir.yield %c1 : index
+  } ub {
+    %c10 = arith.constant 10 : index
+    hlfir.yield %c10 : index
+  } step {
+    %c2 = arith.constant 2 : index
+    hlfir.yield %c2 : index
+  } (%i : index) {
+    hlfir.region_assign {
+      %yi = hlfir.designate %y(%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+      %val = fir.load %yi : !fir.ref<f32>
+      hlfir.yield %val : f32
+    } to {
+      %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+      hlfir.yield %xi : !fir.ref<f32>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @forall_test_step(
+// CHECK-SAME:                                %[[VAL_0:.*]]: !fir.box<!fir.array<10xf32>>,
+// CHECK-SAME:                                %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           hlfir.forall lb {
+// CHECK:             %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:             hlfir.yield %[[VAL_2]] : index
+// CHECK:           } ub {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:             hlfir.yield %[[VAL_3]] : index
+// CHECK:           } step {
+// CHECK:             %[[VAL_4:.*]] = arith.constant 2 : index
+// CHECK:             hlfir.yield %[[VAL_4]] : index
+// CHECK:           }  (%[[VAL_5:.*]]: index) {
+// CHECK:             hlfir.region_assign {
+// CHECK:               %[[VAL_6:.*]] = hlfir.designate %[[VAL_1]] (%[[VAL_5]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_7:.*]] = fir.load %[[VAL_6]] : !fir.ref<f32>
+// CHECK:               hlfir.yield %[[VAL_7]] : f32
+// CHECK:             } to {
+// CHECK:               %[[VAL_8:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_5]])  : (!fir.box<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+// CHECK:               hlfir.yield %[[VAL_8]] : !fir.ref<f32>
+// CHECK:             }
+// CHECK:           }
diff --git a/flang/test/HLFIR/forall_mask.fir b/flang/test/HLFIR/forall_mask.fir
new file mode 100644
index 0000000000000..b0ca270084e38
--- /dev/null
+++ b/flang/test/HLFIR/forall_mask.fir
@@ -0,0 +1,49 @@
+// Test hlfir.forall_mask operation parse, verify (no errors), and unparse.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @forall_mask_test(%x: !fir.box<!fir.array<?xf32>>) {
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  hlfir.forall lb {
+    hlfir.yield %c1 : index
+  } ub {
+    hlfir.yield %c10 : index
+  } (%i : index) {
+    hlfir.forall_mask {
+      %mask = fir.call @some_condition(%i) : (index) -> i1
+      hlfir.yield %mask : i1
+    } do {
+      hlfir.region_assign {
+        %res = fir.call @foo(%i) : (index) -> f32
+        hlfir.yield %res : f32
+      } to {
+        %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+        hlfir.yield %xi : !fir.ref<f32>
+      }
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @forall_mask_test(
+// CHECK-SAME:                                %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 10 : index
+// CHECK:           hlfir.forall lb {
+// CHECK:             hlfir.yield %[[VAL_1]] : index
+// CHECK:           } ub {
+// CHECK:             hlfir.yield %[[VAL_2]] : index
+// CHECK:           }  (%[[VAL_3:.*]]: index) {
+// CHECK:             hlfir.forall_mask {
+// CHECK:               %[[VAL_4:.*]] = fir.call @some_condition(%[[VAL_3]]) : (index) -> i1
+// CHECK:               hlfir.yield %[[VAL_4]] : i1
+// CHECK:             } do {
+// CHECK:               hlfir.region_assign {
+// CHECK:                 %[[VAL_5:.*]] = fir.call @foo(%[[VAL_3]]) : (index) -> f32
+// CHECK:                 hlfir.yield %[[VAL_5]] : f32
+// CHECK:               } to {
+// CHECK:                 %[[VAL_6:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_3]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.yield %[[VAL_6]] : !fir.ref<f32>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index 9b80cbdc25c4f..6b3ed77eb1aa3 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -296,6 +296,30 @@ func.func @bad_concat_4(%arg0: !fir.ref<!fir.char<1,30>>) {
   return
 }
 
+// -----
+func.func @bad_product1(%arg0: !hlfir.expr<?xi32>, %arg1: i32, %arg2: !fir.box<!fir.logical<4>>) {
+  // expected-error@+1 {{'hlfir.product' op result must have the same element type as ARRAY argument}}
+  %0 = hlfir.product %arg0 dim %arg1 mask %arg2 : (!hlfir.expr<?xi32>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<f32>
+}
+
+// -----
+func.func @bad_product2(%arg0: !hlfir.expr<?xi32>, %arg1: i32, %arg2: !fir.box<!fir.array<?x?x?x?x?x!fir.logical<4>>>) {
+  // expected-warning@+1 {{MASK must be conformable to ARRAY}}
+  %0 = hlfir.product %arg0 dim %arg1 mask %arg2 : (!hlfir.expr<?xi32>, i32, !fir.box<!fir.array<?x?x?x?x?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+}
+
+// -----
+func.func @bad_product3(%arg0: !hlfir.expr<?x5x?xi32>, %arg1: i32, %arg2: !fir.box<!fir.array<2x6x?x!fir.logical<4>>>) {
+  // expected-warning@+1 {{MASK must be conformable to ARRAY}}
+  %0 = hlfir.product %arg0 dim %arg1 mask %arg2 : (!hlfir.expr<?x5x?xi32>, i32, !fir.box<!fir.array<2x6x?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+}
+
+// -----
+func.func @bad_product4(%arg0: !hlfir.expr<?xi32>, %arg1: i32, %arg2: !fir.box<!fir.logical<4>>) {
+  // expected-error@+1 {{'hlfir.product' op result rank must be one less than ARRAY}}
+  %0 = hlfir.product %arg0 dim %arg1 mask %arg2 : (!hlfir.expr<?xi32>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<?x?xi32>
+}
+
 // -----
 func.func @bad_sum1(%arg0: !hlfir.expr<?xi32>, %arg1: i32, %arg2: !fir.box<!fir.logical<4>>) {
   // expected-error@+1 {{'hlfir.sum' op result must have the same element type as ARRAY argument}}
@@ -518,3 +542,234 @@ func.func @bad_getextent(%arg0: !fir.shape<1>) {
   // expected-error@+1 {{'hlfir.get_extent' op dimension index out of bounds}}
   %0 = hlfir.get_extent %arg0 {dim = 1 : index} : (!fir.shape<1>) -> index
 }
+
+// -----
+func.func @bad_region_assign_1(%x: !fir.box<!fir.array<?xf32>>) {
+// expected-error@+1 {{'hlfir.region_assign' op right-hand side region must be terminated by an hlfir.yield}}
+  hlfir.region_assign {
+    %c100 = arith.constant 100 : index
+  } to {
+    hlfir.yield %x : !fir.box<!fir.array<?xf32>>
+  }
+  return
+}
+
+// -----
+func.func @bad_region_assign_2(%x: !fir.box<!fir.array<?xf32>>) {
+// expected-error@+1 {{'hlfir.region_assign' op left-hand side region must be terminated by an hlfir.yield or hlfir.elemental_addr}}
+  hlfir.region_assign {
+    hlfir.yield %x : !fir.box<!fir.array<?xf32>>
+  } to {
+    %c100 = arith.constant 100 : index
+  } user_defined_assign  (%rhs: !fir.ref<i64>) to (%lhs: !fir.ref<f32>) {
+  }
+  return
+}
+
+// -----
+func.func @bad_element_addr_1(%x: !fir.ref<!fir.array<20xf32>>) {
+  %c20 = arith.constant 20 : index
+  %vector_shape = fir.shape %c20 : (index) -> !fir.shape<1>
+  hlfir.region_assign {
+    hlfir.yield %x : !fir.ref<!fir.array<20xf32>>
+  } to {
+    // expected-error@+1 {{'hlfir.elemental_addr' op body must compute the address of a scalar entity}}
+    hlfir.elemental_addr %vector_shape  : !fir.shape<1> {
+      ^bb0(%i: index):
+      %c42 = arith.constant 42.0 : f32
+      hlfir.yield %c42 : f32
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_element_addr_2(%x: !fir.ref<!fir.array<20xf32>>) {
+  %c20 = arith.constant 20 : index
+  %vector_shape = fir.shape %c20 : (index) -> !fir.shape<1>
+  hlfir.region_assign {
+    hlfir.yield %x : !fir.ref<!fir.array<20xf32>>
+  } to {
+    // expected-error@+1 {{'hlfir.elemental_addr' op body must compute the address of a scalar entity}}
+    hlfir.elemental_addr %vector_shape  : !fir.shape<1> {
+      ^bb0(%i: index):
+      hlfir.yield %x : !fir.ref<!fir.array<20xf32>>
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_element_addr_3(%x: !fir.ref<!fir.array<20xf32>>) {
+  %c20 = arith.constant 20 : index
+  %vector_shape = fir.shape %c20 : (index) -> !fir.shape<1>
+  hlfir.region_assign {
+    hlfir.yield %x : !fir.ref<!fir.array<20xf32>>
+  } to {
+    // expected-error@+1 {{'hlfir.elemental_addr' op body region must be terminated by an hlfir.yield}}
+    hlfir.elemental_addr %vector_shape  : !fir.shape<1> {
+      ^bb0(%i: index):
+      %c42 = arith.constant 42.0 : f32
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_element_addr_4(%x: !fir.ref<!fir.array<20xf32>>, %y: !fir.ref<!fir.array<20x20xf32>>) {
+  %c20 = arith.constant 20 : index
+  %vector_shape = fir.shape %c20 : (index) -> !fir.shape<1>
+  hlfir.region_assign {
+    hlfir.yield %x : !fir.ref<!fir.array<20xf32>>
+  } to {
+    // expected-error@+1 {{'hlfir.elemental_addr' op body number of indices must match shape rank}}
+    hlfir.elemental_addr %vector_shape  : !fir.shape<1> {
+      ^bb0(%i: index, %j: index):
+       %elt = hlfir.designate %y(%i, %j) : (!fir.ref<!fir.array<20x20xf32>>, index, index) -> !fir.ref<f32>
+       hlfir.yield %elt : !fir.ref<f32>
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_forall(%x : !fir.box<!fir.array<10xf32>>, %y: f32, %bad : !fir.ref<!fir.array<10xindex>>) {
+  // expected-error@+1 {{'hlfir.forall' op region #0 ('lb_region') failed to verify constraint: single block region that yields an integer scalar value}}
+  hlfir.forall lb {
+    hlfir.yield %bad : !fir.ref<!fir.array<10xindex>>
+  } ub {
+    %c10 = arith.constant 10 : index
+    hlfir.yield %c10 : index
+  } (%i : index) {
+    hlfir.region_assign {
+      hlfir.yield %y : f32
+    } to {
+      %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+      hlfir.yield %xi : !fir.ref<f32>
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_forall_2(%x : !fir.box<!fir.array<10xf32>>, %y: f32) {
+  // expected-error@+1 {{'hlfir.forall' op body region must only contain OrderedAssignmentTreeOpInterface operations or fir.end}}
+  hlfir.forall lb {
+    %c1 = arith.constant 1 : index
+    hlfir.yield %c1 : index
+  } ub {
+    %c10 = arith.constant 10 : index
+    hlfir.yield %c10 : index
+  } (%i : index) {
+    %xi = hlfir.designate %x(%i) : (!fir.box<!fir.array<10xf32>>, index) -> !fir.ref<f32>
+    hlfir.assign %y to %xi : f32, !fir.ref<f32>
+  }
+  return
+}
+
+// -----
+func.func @bad_forall_mask(%i: index) {
+  // expected-error@+1 {{'hlfir.forall_mask' op must be inside the body region of an hlfir.forall}}
+  hlfir.forall_mask {
+    %mask = fir.call @some_condition(%i) : (index) -> i1
+    hlfir.yield %mask : i1
+  } do {
+  }
+  return
+}
+
+// -----
+func.func @bad_forall_mask_2(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>) {
+  %c1 = arith.constant 1 : index
+  hlfir.forall lb {
+    hlfir.yield %c1 : index
+  } ub {
+    hlfir.yield %c1 : index
+  } (%i: index) {
+    // expected-error@+1 {{'hlfir.forall_mask' op mask region must yield a scalar i1}}
+    hlfir.forall_mask {
+      hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+    } do {
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_where_1(%bad_mask: !fir.ref<!fir.array<10xf32>>) {
+  // expected-error@+1 {{'hlfir.where' op mask region must yield a logical array}}
+  hlfir.where {
+    hlfir.yield %bad_mask : !fir.ref<!fir.array<10xf32>>
+  } do {
+  }
+  return
+}
+
+// -----
+func.func @bad_where_2(%bad_mask: i1) {
+  // expected-error@+1 {{'hlfir.where' op mask region must yield a logical array}}
+  hlfir.where {
+    hlfir.yield %bad_mask : i1
+  } do {
+  }
+  return
+}
+
+// -----
+func.func @bad_where_3(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>, %n: index) {
+  // expected-error@+1 {{'hlfir.where' op body region must not contain hlfir.forall}}
+  hlfir.where {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+    hlfir.forall lb {
+      hlfir.yield %n : index
+    } ub {
+      hlfir.yield %n : index
+    } (%i: index) {
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_elsewhere_1(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>, %bad_mask: i1) {
+  hlfir.where {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+    // expected-error@+1 {{'hlfir.elsewhere' op mask region must yield a logical array when provided}}
+    hlfir.elsewhere mask {
+      hlfir.yield %bad_mask : i1
+    } do {
+    }
+  }
+  return
+}
+
+// -----
+func.func @bad_elsewhere_2(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>) {
+  // expected-error@+1 {{'hlfir.elsewhere' op expects parent op to be one of 'hlfir.where, hlfir.elsewhere'}}
+  hlfir.elsewhere mask {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+  }
+  return
+}
+
+// -----
+func.func @bad_elsewhere_3(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>, %x: !fir.ref<!fir.array<10xf32>>, %y: !fir.box<!fir.array<?xf32>>) {
+  hlfir.where {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+    // expected-error@+1 {{'hlfir.elsewhere' op must be the last operation in the parent block}}
+    hlfir.elsewhere mask {
+      hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+    } do {
+    }
+    hlfir.region_assign {
+      hlfir.yield %y : !fir.box<!fir.array<?xf32>>
+    } to {
+      hlfir.yield %x : !fir.ref<!fir.array<10xf32>>
+    }
+  }
+  return
+}
diff --git a/flang/test/HLFIR/optional_dummy.f90 b/flang/test/HLFIR/optional_dummy.f90
new file mode 100644
index 0000000000000..9dd2076f82a1a
--- /dev/null
+++ b/flang/test/HLFIR/optional_dummy.f90
@@ -0,0 +1,28 @@
+! RUN: bbc -emit-fir -hlfir %s -o - | FileCheck %s
+
+! Check that the lowering does not generate fir.box_addr for
+! the optional box. It will cause segfault during execution.
+
+! CHECK-LABEL:   func.func @_QPtest(
+! CHECK-SAME:        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "ext_buf", fir.contiguous, fir.optional}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<contiguous, optional>, uniq_name = "_QFtestEext_buf"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#1 : (!fir.box<!fir.array<?xi32>>) -> i1
+! CHECK:           cf.cond_br %[[VAL_2]], ^bb1, ^bb2
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+! CHECK:           %[[VAL_4:.*]] = arith.constant false
+! CHECK:           %[[VAL_5:.*]] = arith.constant false
+! CHECK:           %[[VAL_6:.*]] = fir.call @_FortranAStopStatement(%[[VAL_3]], %[[VAL_4]], %[[VAL_5]]) fastmath<contract> : (i32, i1, i1) -> none
+! CHECK:           fir.unreachable
+! CHECK:         ^bb2:
+! CHECK:           cf.br ^bb3
+! CHECK:         ^bb3:
+! CHECK:           return
+! CHECK:         }
+subroutine test(ext_buf)
+  integer, contiguous, optional :: ext_buf(:)
+  if (present(ext_buf)) then
+     stop
+  endif
+  return
+end subroutine test
diff --git a/flang/test/HLFIR/ordered-assignments-codegen-todo.fir b/flang/test/HLFIR/ordered-assignments-codegen-todo.fir
new file mode 100644
index 0000000000000..ccbd97ce5caba
--- /dev/null
+++ b/flang/test/HLFIR/ordered-assignments-codegen-todo.fir
@@ -0,0 +1,24 @@
+// Just test that Ordered assignment pass TODOs are properly reported.
+// RUN: %not_todo_cmd fir-opt --lower-hlfir-ordered-assignments %s 2>&1 | FileCheck %s
+
+
+// CHECK: not yet implemented: FORALL construct or statement in HLFIR
+
+func.func @forall_todo(%arg0: !fir.ref<!fir.array<10xf32>>, %arg1: !fir.ref<!fir.array<10xf32>>) {
+  %c1 = arith.constant 1 : index
+  %c10 = arith.constant 10 : index
+  hlfir.forall lb {
+    hlfir.yield %c1 : index
+  } ub {
+    hlfir.yield %c10 : index
+  }  (%arg2: i64) {
+    hlfir.region_assign {
+      %1 = hlfir.designate %arg1 (%arg2)  : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+      hlfir.yield %1 : !fir.ref<f32>
+    } to {
+      %1 = hlfir.designate %arg0 (%arg2)  : (!fir.ref<!fir.array<10xf32>>, i64) -> !fir.ref<f32>
+      hlfir.yield %1 : !fir.ref<f32>
+    }
+  }
+  return
+}
diff --git a/flang/test/HLFIR/product-lowering.fir b/flang/test/HLFIR/product-lowering.fir
new file mode 100644
index 0000000000000..e22869ffc4c7d
--- /dev/null
+++ b/flang/test/HLFIR/product-lowering.fir
@@ -0,0 +1,171 @@
+// Test hlfir.product operation lowering to fir runtime call
+// RUN: fir-opt %s -lower-hlfir-intrinsics | FileCheck %s
+
+// one argument product
+func.func @_QPproduct1(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFsum1Ea"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFsum1Es"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = hlfir.product %0#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>) -> !hlfir.expr<i32>
+  hlfir.assign %2 to %1#0 : !hlfir.expr<i32>, !fir.ref<i32>
+  hlfir.destroy %2 : !hlfir.expr<i32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPproduct1(
+// CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[ARG1:.*]]: !fir.ref<i32>
+// CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[ARG1]]
+// CHECK-DAG:     %[[MASK:.*]] = fir.absent !fir.box<i1>
+// CHECK-DAG:     %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1 :  (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK]] : (!fir.box<i1>) -> !fir.box<none>
+// CHECK:         %[[RET:.*]] = fir.call @_FortranAProductInteger4(%[[ARRAY_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[INT:.*]], %[[MASK_ARG]]) : (!fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> i32
+// CHECK-NEXT:    hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// product with DIM argument by-ref
+func.func @_QPproduct2(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}, %arg2: !fir.ref<index> {fir.bindc_name = "d"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFproduct2Ea"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFproduct2Ed"} : (!fir.ref<index>) -> (!fir.ref<index>, !fir.ref<index>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFproduct2Es"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %3 = fir.load %1#0 : !fir.ref<index>
+  %4 = hlfir.product %0#0 dim %3 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x?xi32>>, index) -> !hlfir.expr<?xi32>
+  hlfir.assign %4 to %2#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %4 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPproduct2(
+// CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x?xi32>>
+// CHECK:           %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[ARG2:.*]]: !fir.ref<index>
+// CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:     %[[DIM_VAR:.*]]:2 = hlfir.declare %[[ARG2]]
+// CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[ARG1]]
+
+// CHECK-DAG:     %[[RET_BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK-DAG:     %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+// CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:     %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
+// CHECK-DAG:     %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]])
+// CHECK-DAG:     fir.store %[[RET_EMBOX]] to %[[RET_BOX]]
+
+// CHECK-DAG:     %[[MASK:.*]] = fir.absent !fir.box<i1>
+// CHECK-DAG:     %[[DIM_IDX:.*]] = fir.load %[[DIM_VAR]]#0 : !fir.ref<index>
+// CHECK-DAG:     %[[DIM:.*]] = fir.convert %[[DIM_IDX]] : (index) -> i32
+
+// CHECK-DAG:     %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]]
+// CHECK-DAG:     %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK]]
+
+// CHECK:         %[[NONE:.*]] = fir.call @_FortranAProductDim(%[[RET_ARG]], %[[ARRAY_ARG]], %[[DIM]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[MASK_ARG]])  : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32, !fir.box<none>) -> none
+// CHECK:         %[[RET:.*]] = fir.load %[[RET_BOX]]
+// CHECK:         %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[RET]]
+// CHECK-NEXT:    %[[ADDR:.*]] = fir.box_addr %[[RET]]
+// CHECK-NEXT:    %[[SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS]]#0, %[[BOX_DIMS]]#1
+// CHECK-NEXT:    %[[TMP:.*]]:2 = hlfir.declare %[[ADDR]](%[[SHIFT]]) {uniq_name = ".tmp.intrinsic_result"}
+// CHECK:         %[[TRUE:.*]] = arith.constant true
+// CHECK:         %[[EXPR:.*]] = hlfir.as_expr %[[TMP]]#0 move %[[TRUE]] : (!fir.box<!fir.array<?xi32>>, i1) -> !hlfir.expr<?xi32>
+// CHECK:         hlfir.assign %[[EXPR]] to %[[RES]]#0
+// CHECK:         hlfir.destroy %[[EXPR]]
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// product with scalar mask
+func.func @_QPproduct3(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}, %arg2: !fir.ref<!fir.logical<4>> {fir.bindc_name = "m"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFproduct3Ea"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFproduct3Em"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFproduct3Es"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = hlfir.product %0#0 mask %1#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.logical<4>>) -> !hlfir.expr<i32>
+  hlfir.assign %3 to %2#0 : !hlfir.expr<i32>, !fir.ref<i32>
+  hlfir.destroy %3 : !hlfir.expr<i32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPproduct3(
+// CHECK:            %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>>
+// CHECK:            %[[ARG1:.*]]: !fir.ref<i32>
+// CHECK:            %[[ARG2:.*]]: !fir.ref<!fir.logical<4>>
+// CHECK-DAG:      %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:      %[[RES:.*]]:2 = hlfir.declare %[[ARG1]]
+// CHECK-DAG:      %[[MASK:.*]]:2 = hlfir.declare %[[ARG2]]
+// CHECK-DAG:      %[[MASK_BOX:.*]] = fir.embox %[[MASK]]#1 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-DAG:      %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK-DAG:      %[[MASK_ARG:.*]] = fir.convert %[[MASK_BOX]]  : (!fir.box<!fir.logical<4>>) -> !fir.box<none>
+// CHECK:          %[[RET:.*]] = fir.call @_FortranAProductInteger4(%[[ARRAY_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[INT:.*]], %[[MASK_ARG]]) : (!fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> i32
+// CHECK-NEXT:     hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:     return
+// CHECK-NEXT:   }
+
+// product with array mask
+func.func @_QPproduct4(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}, %arg2: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "m"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFproduct4Ea"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFproduct4Em"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFproduct4Es"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %3 = hlfir.product %0#0 mask %1#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+  hlfir.assign %3 to %2#0 : !hlfir.expr<i32>, !fir.ref<i32>
+  hlfir.destroy %3 : !hlfir.expr<i32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPproduct4(
+// CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>
+// CHECK:           %[[ARG1:.*]]: !fir.ref<i32>
+// CHECK:           %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>>
+// CHECK-DAG:     %[[ARRAY]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:     %[[RES]]:2 = hlfir.declare %[[ARG1]]
+// CHECK-DAG:     %[[MASK]]:2 = hlfir.declare %[[ARG2]]
+// CHECK-DAG:     %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY]]#1  : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK]]#1 : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> !fir.box<none>
+// CHECK:         %[[RET:.*]] = fir.call @_FortranAProductInteger4(%[[ARRAY_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[INT:.*]], %[[MASK_ARG]]) : (!fir.box<none>, !fir.ref<i8>, i32, i32, !fir.box<none>) -> i32
+// CHECK-NEXT:    hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+
+// product with all 3 arguments
+func.func @_QPproduct5(%arg0: !fir.ref<!fir.array<2xi32>> {fir.bindc_name = "s"}) {
+  %0 = fir.address_of(@_QFproduct5Ea) : !fir.ref<!fir.array<2x2xi32>>
+  %c2 = arith.constant 2 : index
+  %c2_0 = arith.constant 2 : index
+  %1 = fir.shape %c2, %c2_0 : (index, index) -> !fir.shape<2>
+  %2:2 = hlfir.declare %0(%1) {uniq_name = "_QFproduct5Ea"} : (!fir.ref<!fir.array<2x2xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<2x2xi32>>, !fir.ref<!fir.array<2x2xi32>>)
+  %c2_1 = arith.constant 2 : index
+  %3 = fir.shape %c2_1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg0(%3) {uniq_name = "_QFproduct5Es"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+  %c1_i32 = arith.constant 1 : i32
+  %true = arith.constant true
+  %5 = hlfir.product %2#0 dim %c1_i32 mask %true {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<2x2xi32>>, i32, i1) -> !hlfir.expr<2xi32>
+  hlfir.assign %5 to %4#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
+  hlfir.destroy %5 : !hlfir.expr<2xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPproduct5(
+// CHECK:           %[[ARG0:.*]]: !fir.ref<!fir.array<2xi32>>
+// CHECK-DAG:     %[[RET_BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK-DAG:     %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+// CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:     %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
+// CHECK-DAG:     %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]])
+// CHECK-DAG:     fir.store %[[RET_EMBOX]] to %[[RET_BOX]]
+
+// CHECK-DAG:     %[[RES_VAR:.*]] = hlfir.declare %[[ARG0]](%[[RES_SHAPE:.*]])
+
+// CHECK-DAG:     %[[MASK_ALLOC:.*]] = fir.alloca !fir.logical<4>
+// CHECK-DAG:     %[[TRUE:.*]] = arith.constant true
+// CHECK-DAG:     %[[MASK_VAL:.*]]  = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-DAG:     fir.store %[[MASK_VAL]] to %[[MASK_ALLOC]] : !fir.ref<!fir.logical<4>>
+// CHECK-DAG:     %[[MASK_BOX:.*]] = fir.embox %[[MASK_ALLOC]]
+
+// CHECK-DAG:     %[[ARRAY_ADDR:.*]] = fir.address_of
+// CHECK-DAG:     %[[ARRAY_VAR:.*]]:2 = hlfir.declare %[[ARRAY_ADDR]](%[[ARRAY_SHAPE:.*]])
+// CHECK-DAG:     %[[ARRAY_BOX:.*]] = fir.embox %[[ARRAY_VAR]]#1(%[[ARRAY_SHAPE:.*]])
+
+// CHECK-DAG:     %[[DIM:.*]] = arith.constant 1 : i32
+
+// CHECK-DAG:     %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]]
+// CHECK-DAG:     %[[ARRAY_ARG:.*]] = fir.convert %[[ARRAY_BOX]] : (!fir.box<!fir.array<2x2xi32>>) -> !fir.box<none>
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK_BOX]] : (!fir.box<!fir.logical<4>>) -> !fir.box<none>
+// CHECK:         %[[NONE:.*]] = fir.call @_FortranAProductDim(%[[RET_ARG]], %[[ARRAY_ARG]], %[[DIM]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[MASK_ARG]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, !fir.ref<i8>, i32, !fir.box<none>) -> none
diff --git a/flang/test/HLFIR/product.fir b/flang/test/HLFIR/product.fir
new file mode 100644
index 0000000000000..b0c7dfe1797ea
--- /dev/null
+++ b/flang/test/HLFIR/product.fir
@@ -0,0 +1,240 @@
+// Test hlfir.product operation parse, verify (no errors), and unparse
+
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+// array is an expression of known shape
+func.func @product0(%arg0: !hlfir.expr<42xi32>) {
+  %mask = fir.alloca !fir.logical<4>
+  %c_1 = arith.constant 1 : index
+  %true = arith.constant true
+  %true_logical = fir.convert %true : (i1) -> !fir.logical<4>
+  fir.store %true_logical to %mask : !fir.ref<!fir.logical<4>>
+  %mask_box = fir.embox %mask : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %0 = hlfir.product %arg0 dim %c_1 mask %mask_box : (!hlfir.expr<42xi32>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product0(%[[ARRAY:.*]]: !hlfir.expr<42xi32>) {
+// CHECK-NEXT:   %[[MASK:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+// CHECK-NEXT:   %[[LOGICAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:   fir.store %[[LOGICAL]] to %[[MASK]] : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:   %[[BOX:.*]] = fir.embox %0 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[C1]] mask %[[BOX]] : (!hlfir.expr<42xi32>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// array is an expression of assumed shape
+func.func @product1(%arg0: !hlfir.expr<?xi32>) {
+  %mask = fir.alloca !fir.logical<4>
+  %c_1 = arith.constant 1 : index
+  %true = arith.constant true
+  %true_logical = fir.convert %true : (i1) -> !fir.logical<4>
+  fir.store %true_logical to %mask : !fir.ref<!fir.logical<4>>
+  %mask_box = fir.embox %mask : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %0 = hlfir.product %arg0 dim %c_1 mask %mask_box : (!hlfir.expr<?xi32>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product1(%[[ARRAY:.*]]: !hlfir.expr<?xi32>) {
+// CHECK-NEXT:   %[[MASK:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+// CHECK-NEXT:   %[[LOGICAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:   fir.store %[[LOGICAL:.*]] to %[[MASK:.*]] : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:   %[[BOX:.*]] = fir.embox %[[MASK:.*]] : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-NEXT:   hlfir.product %[[ARRAY:.*]] dim %[[C1]] mask %[[BOX]] : (!hlfir.expr<?xi32>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// boxed array
+func.func @product2(%arg0: !fir.box<!fir.array<42xi32>>) {
+  %mask = fir.alloca !fir.logical<4>
+  %c_1 = arith.constant 1 : index
+  %true = arith.constant true
+  %true_logical = fir.convert %true : (i1) -> !fir.logical<4>
+  fir.store %true_logical to %mask : !fir.ref<!fir.logical<4>>
+  %mask_box = fir.embox %mask : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %0 = hlfir.product %arg0 dim %c_1 mask %mask_box : (!fir.box<!fir.array<42xi32>>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product2(%[[ARRAY:.*]]: !fir.box<!fir.array<42xi32>>) {
+// CHECK-NEXT:   %[[MASK:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+// CHECK-NEXT:   %[[LOGICAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:   fir.store %[[LOGICAL:.*]] to %[[MASK:.*]] : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:   %[[BOX:.*]] = fir.embox %[[MASK:.*]] : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-NEXT:   hlfir.product %[[ARRAY:.*]] dim %[[C1]] mask %[[BOX]] : (!fir.box<!fir.array<42xi32>>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// assumed shape boxed array
+func.func @product3(%arg0: !fir.box<!fir.array<?xi32>>) {
+  %mask = fir.alloca !fir.logical<4>
+  %c_1 = arith.constant 1 : index
+  %true = arith.constant true
+  %true_logical = fir.convert %true : (i1) -> !fir.logical<4>
+  fir.store %true_logical to %mask : !fir.ref<!fir.logical<4>>
+  %mask_box = fir.embox %mask : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %0 = hlfir.product %arg0 dim %c_1 mask %mask_box : (!fir.box<!fir.array<?xi32>>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product3(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>) {
+// CHECK-NEXT:   %[[MASK:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+// CHECK-NEXT:   %[[LOGICAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:   fir.store %[[LOGICAL:.*]] to %[[MASK:.*]] : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:   %[[BOX:.*]] = fir.embox %[[MASK:.*]] : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-NEXT:   hlfir.product %[[ARRAY:.*]] dim %[[C1]] mask %[[BOX]] : (!fir.box<!fir.array<?xi32>>, index, !fir.box<!fir.logical<4>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// known shape expr mask
+func.func @product4(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !hlfir.expr<42x!fir.logical<4>>) {
+  %c_1 = arith.constant 1 : index
+  %0 = hlfir.product %arg0 dim %c_1 mask %arg1 : (!fir.box<!fir.array<?xi32>>, index, !hlfir.expr<42x!fir.logical<4>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product4(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: !hlfir.expr<42x!fir.logical<4>>) {
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[C1]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, index, !hlfir.expr<42x!fir.logical<4>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// assumed shape expr mask
+func.func @product5(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !hlfir.expr<?x!fir.logical<4>>) {
+  %c_1 = arith.constant 1 : index
+  %0 = hlfir.product %arg0 dim %c_1 mask %arg1 : (!fir.box<!fir.array<?xi32>>, index, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product5(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: !hlfir.expr<?x!fir.logical<4>>) {
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[C1]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, index, !hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// known shape array mask
+func.func @product6(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.box<!fir.array<42x!fir.logical<4>>>) {
+  %c_1 = arith.constant 1 : index
+  %0 = hlfir.product %arg0 dim %c_1 mask %arg1 : (!fir.box<!fir.array<?xi32>>, index, !fir.box<!fir.array<42x!fir.logical<4>>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product6(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: !fir.box<!fir.array<42x!fir.logical<4>>>) {
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[C1]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, index, !fir.box<!fir.array<42x!fir.logical<4>>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// assumed shape array mask
+func.func @product7(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.box<!fir.array<?x!fir.logical<4>>>) {
+  %c_1 = arith.constant 1 : index
+  %0 = hlfir.product %arg0 dim %c_1 mask %arg1 : (!fir.box<!fir.array<?xi32>>, index, !fir.box<!fir.array<?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product7(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>>) {
+// CHECK-NEXT:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[C1]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, index, !fir.box<!fir.array<?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// known shape expr return
+func.func @product8(%arg0: !fir.box<!fir.array<2x2xi32>>, %arg1: i32) {
+  %mask = fir.alloca !fir.logical<4>
+  %true = arith.constant true
+  %true_logical = fir.convert %true : (i1) -> !fir.logical<4>
+  fir.store %true_logical to %mask : !fir.ref<!fir.logical<4>>
+  %mask_box = fir.embox %mask : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %0 = hlfir.product %arg0 dim %arg1 mask %mask_box : (!fir.box<!fir.array<2x2xi32>>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<2xi32>
+  return
+}
+// CHECK:      func.func @product8(%[[ARRAY:.*]]: !fir.box<!fir.array<2x2xi32>>, %[[DIM:.*]]: i32) {
+// CHECK-NEXT:   %[[MASK:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+// CHECK-NEXT:   %[[LOGICAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:   fir.store %[[LOGICAL]] to %[[MASK]] : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:   %[[BOX:.*]] = fir.embox %0 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[DIM]] mask %[[BOX]] : (!fir.box<!fir.array<2x2xi32>>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<2xi32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+
+// assumed shape expr return
+func.func @product9(%arg0: !fir.box<!fir.array<?x?xi32>>, %arg1: i32) {
+  %mask = fir.alloca !fir.logical<4>
+  %true = arith.constant true
+  %true_logical = fir.convert %true : (i1) -> !fir.logical<4>
+  fir.store %true_logical to %mask : !fir.ref<!fir.logical<4>>
+  %mask_box = fir.embox %mask : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %0 = hlfir.product %arg0 dim %arg1 mask %mask_box : (!fir.box<!fir.array<?x?xi32>>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<?xi32>
+  return
+}
+// CHECK:      func.func @product9(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?xi32>>, %[[DIM:.*]]: i32) {
+// CHECK-NEXT:   %[[MASK:.*]] = fir.alloca !fir.logical<4>
+// CHECK-NEXT:   %[[TRUE:.*]] = arith.constant true
+// CHECK-NEXT:   %[[LOGICAL:.*]] = fir.convert %[[TRUE]] : (i1) -> !fir.logical<4>
+// CHECK-NEXT:   fir.store %[[LOGICAL]] to %[[MASK]] : !fir.ref<!fir.logical<4>>
+// CHECK-NEXT:   %[[BOX:.*]] = fir.embox %0 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK-NEXT:   hlfir.product %[[ARRAY]] dim %[[DIM]] mask %[[BOX]] : (!fir.box<!fir.array<?x?xi32>>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<?xi32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// hlfir.product with only an array argument
+func.func @product10(%arg0: !fir.box<!fir.array<?x?xi32>>) {
+  %product = hlfir.product %arg0 : (!fir.box<!fir.array<?x?xi32>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product10(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?xi32>>
+// CHECK-NEXT:   %[[PRODUCT:.*]] = hlfir.product %[[ARRAY]] : (!fir.box<!fir.array<?x?xi32>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// hlfir.product with array and dim argument
+func.func @product11(%arg0: !fir.box<!fir.array<?x?xi32>>, %arg1: i32) {
+  %product = hlfir.product %arg0 dim %arg1 : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?xi32>
+  return
+}
+// CHECK:      func.func @product11(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?xi32>>, %[[DIM:.*]]: i32
+// CHECK-NEXT:   %[[PRODUCT:.*]] = hlfir.product %[[ARRAY]] dim %[[DIM]] : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?xi32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// hlfir.product with array and mask argument
+func.func @product12(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.logical<4>) {
+  %product = hlfir.product %arg0 mask %arg1 : (!fir.box<!fir.array<?xi32>>, !fir.logical<4>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product12(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: !fir.logical<4>
+// CHECK-NEXT:   %[[PRODUCT:.*]] = hlfir.product %[[ARRAY]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, !fir.logical<4>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// hlfir.product with dim argument with an unusual type
+func.func @product13(%arg0: !fir.box<!fir.array<?x?xi32>>, %arg1: index) {
+  %product = hlfir.product %arg0 dim %arg1 : (!fir.box<!fir.array<?x?xi32>>, index) -> !hlfir.expr<?xi32>
+  return
+}
+// CHECK:      func.func @product13(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?xi32>>, %[[DIM:.*]]: index
+// CHECK-NEXT:   %[[PRODUCT:.*]] = hlfir.product %[[ARRAY]] dim %[[DIM]] : (!fir.box<!fir.array<?x?xi32>>, index) -> !hlfir.expr<?xi32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// hlfir.product with mask argument of unusual type
+func.func @product14(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: i1) {
+  %product = hlfir.product %arg0 mask %arg1 : (!fir.box<!fir.array<?xi32>>, i1) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product14(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: i1
+// CHECK-NEXT:   %[[PRODUCT:.*]] = hlfir.product %[[ARRAY]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, i1) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// hlfir.product with mask argument of ref<array<>> type
+func.func @product15(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<!fir.array<?x!fir.logical<4>>>) {
+  %product = hlfir.product %arg0 mask %arg1 : (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+  return
+}
+// CHECK:      func.func @product15(%[[ARRAY:.*]]: !fir.box<!fir.array<?xi32>>, %[[MASK:.*]]: !fir.ref<!fir.array<?x!fir.logical<4>>>
+// CHECK-NEXT:   %[[PRODUCT:.*]] = hlfir.product %[[ARRAY]] mask %[[MASK]] : (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
diff --git a/flang/test/HLFIR/region-assign.fir b/flang/test/HLFIR/region-assign.fir
new file mode 100644
index 0000000000000..39724d818cb02
--- /dev/null
+++ b/flang/test/HLFIR/region-assign.fir
@@ -0,0 +1,87 @@
+// Test hlfir.region_assign and hlfir.yield operation parse, verify (no errors),
+// and unparse.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @region_assign_test(%y : !fir.box<!fir.array<?xi64>>, %x: !fir.box<!fir.array<?xf32>>) {
+  %c100 = arith.constant 100 : index
+  %shape = fir.shape %c100 : (index) -> !fir.shape<1>
+  hlfir.region_assign {
+    %expr = hlfir.elemental %shape : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+    ^bb0(%i : index):
+      %yelt = hlfir.designate %y(%i) : (!fir.box<!fir.array<?xi64>>, index) -> !fir.ref<i64>
+      %elt = fir.call @some_elemental(%yelt) : (!fir.ref<i64>) -> f32
+      hlfir.yield_element %elt : f32
+    }
+    hlfir.yield %expr : !hlfir.expr<?xf32>
+  } to {
+    hlfir.yield %x : !fir.box<!fir.array<?xf32>>
+  }
+  return
+}
+// CHECK-LABEL:   func.func @region_assign_test(
+// CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.box<!fir.array<?xi64>>,
+// CHECK-SAME:                                  %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 100 : index
+// CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
+// CHECK:           hlfir.region_assign {
+// CHECK:             %[[VAL_4:.*]] = hlfir.elemental %[[VAL_3]] : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:             ^bb0(%[[VAL_5:.*]]: index):
+// CHECK:               %[[VAL_6:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_5]])  : (!fir.box<!fir.array<?xi64>>, index) -> !fir.ref<i64>
+// CHECK:               %[[VAL_7:.*]] = fir.call @some_elemental(%[[VAL_6]]) : (!fir.ref<i64>) -> f32
+// CHECK:               hlfir.yield_element %[[VAL_7]] : f32
+// CHECK:             }
+// CHECK:             hlfir.yield %[[VAL_8:.*]] : !hlfir.expr<?xf32>
+// CHECK:           } to {
+// CHECK:             hlfir.yield %[[VAL_1]] : !fir.box<!fir.array<?xf32>>
+// CHECK:           }
+
+func.func @region_user_assign_test(%y : !fir.box<!fir.array<?xi64>>, %x: !fir.box<!fir.array<?xf32>>) {
+  hlfir.region_assign {
+    hlfir.yield %y : !fir.box<!fir.array<?xi64>>
+  } to {
+    hlfir.yield %x : !fir.box<!fir.array<?xf32>>
+  } user_defined_assign (%rhs : !fir.ref<i64>) to (%lhs : !fir.ref<f32>) {
+    %0 = fir.load %rhs : !fir.ref<i64>
+    fir.call @user_assign(%lhs, %0) : (!fir.ref<f32>, i64) -> ()
+  }
+  return
+}
+// CHECK-LABEL:   func.func @region_user_assign_test(
+// CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.box<!fir.array<?xi64>>,
+// CHECK-SAME:                                       %[[VAL_1:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           hlfir.region_assign {
+// CHECK:             hlfir.yield %[[VAL_0]] : !fir.box<!fir.array<?xi64>>
+// CHECK:           } to {
+// CHECK:             hlfir.yield %[[VAL_1]] : !fir.box<!fir.array<?xf32>>
+// CHECK:           } user_defined_assign  (%[[VAL_2:.*]]: !fir.ref<i64>) to (%[[VAL_3:.*]]: !fir.ref<f32>) {
+// CHECK:             %[[VAL_4:.*]] = fir.load %[[VAL_2]] : !fir.ref<i64>
+// CHECK:             fir.call @user_assign(%[[VAL_3]], %[[VAL_4]]) : (!fir.ref<f32>, i64) -> ()
+// CHECK:           }
+
+func.func @yield_cleanup(%x: !fir.box<!fir.array<?xf32>>) {
+  hlfir.region_assign {
+    %0 = fir.allocmem !fir.array<100xf32>
+    fir.call @fillin_some_values(%0) : (!fir.heap<!fir.array<100xf32>>) -> ()
+    hlfir.yield %0 : !fir.heap<!fir.array<100xf32>> cleanup {
+      fir.freemem %0 : !fir.heap<!fir.array<100xf32>>
+    }
+  } to {
+    hlfir.yield %x : !fir.box<!fir.array<?xf32>>
+  }
+  return
+}
+// CHECK-LABEL:   func.func @yield_cleanup(
+// CHECK-SAME:                             %[[VAL_0:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           hlfir.region_assign {
+// CHECK:             %[[VAL_1:.*]] = fir.allocmem !fir.array<100xf32>
+// CHECK:             fir.call @fillin_some_values(%[[VAL_1]]) : (!fir.heap<!fir.array<100xf32>>) -> ()
+// CHECK:             hlfir.yield %[[VAL_1]] : !fir.heap<!fir.array<100xf32>> cleanup {
+// CHECK:               fir.freemem %[[VAL_1]] : !fir.heap<!fir.array<100xf32>>
+// CHECK:             }
+// CHECK:           } to {
+// CHECK:             hlfir.yield %[[VAL_0]] : !fir.box<!fir.array<?xf32>>
+// CHECK:           }
+
+func.func private @user_assign(!fir.ref<f32>, i64) -> ()
+func.func private @some_elemental(!fir.ref<i64>) -> f32
+func.func private @fillin_some_values(!fir.heap<!fir.array<100xf32>>) -> ()
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics.fir
new file mode 100644
index 0000000000000..e75b6fba885ba
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics.fir
@@ -0,0 +1,95 @@
+// RUN: fir-opt --simplify-hlfir-intrinsics %s | FileCheck %s
+
+// box with known extents
+func.func @transpose0(%arg0: !fir.box<!fir.array<1x2xi32>>) {
+  %res = hlfir.transpose %arg0 : (!fir.box<!fir.array<1x2xi32>>) -> !hlfir.expr<2x1xi32>
+  return
+}
+// CHECK-LABEL:   func.func @transpose0(
+// CHECK-SAME:                          %[[ARG0:.*]]: !fir.box<!fir.array<1x2xi32>>) {
+// CHECK:           %[[C1:.*]] = arith.constant 1 : index
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[SHAPE:.*]] = fir.shape %[[C2]], %[[C1]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[EXPR:.*]] = hlfir.elemental %[[SHAPE]] : (!fir.shape<2>) -> !hlfir.expr<2x1xi32> {
+// CHECK:           ^bb0(%[[I:.*]]: index, %[[J:.*]]: index):
+// CHECK:             %[[C0:.*]] = arith.constant 0 : index
+// CHECK:             %[[DIMS0:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.array<1x2xi32>>, index) -> (index, index, index)
+// CHECK:             %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK:             %[[DIMS1:.*]]:3 = fir.box_dims %[[ARG0]], %[[C1_1]] : (!fir.box<!fir.array<1x2xi32>>, index) -> (index, index, index)
+// CHECK:             %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK:             %[[LOWER_BOUND0:.*]] = arith.subi %[[DIMS0]]#0, %[[C1_2]] : index
+// CHECK:             %[[J_OFFSET:.*]] = arith.addi %[[J]], %[[LOWER_BOUND0]] : index
+// CHECK:             %[[LOWER_BOUND1:.*]] = arith.subi %[[DIMS1]]#0, %[[C1_2]] : index
+// CHECK:             %[[I_OFFSET:.*]] = arith.addi %[[I]], %[[LOWER_BOUND1]] : index
+// CHECK:             %[[ELEMENT_REF:.*]] = hlfir.designate %[[ARG0]] (%[[J_OFFSET]], %[[I_OFFSET]])  : (!fir.box<!fir.array<1x2xi32>>, index, index) -> !fir.ref<i32>
+// CHECK:             %[[ELEMENT:.*]] = fir.load %[[ELEMENT_REF]] : !fir.ref<i32>
+// CHECK:             hlfir.yield_element %[[ELEMENT]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// expr with known extents
+func.func @transpose1(%arg0: !hlfir.expr<1x2xi32>) {
+  %res = hlfir.transpose %arg0 : (!hlfir.expr<1x2xi32>) -> !hlfir.expr<2x1xi32>
+  return
+}
+// CHECK-LABEL:   func.func @transpose1(
+// CHECK-SAME:                          %[[ARG0:.*]]: !hlfir.expr<1x2xi32>) {
+// CHECK:           %[[C1:.*]] = arith.constant 1 : index
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[SHAPE:.*]] = fir.shape %[[C2]], %[[C1]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[EXPR:.*]] = hlfir.elemental %[[SHAPE]] : (!fir.shape<2>) -> !hlfir.expr<2x1xi32> {
+// CHECK:           ^bb0(%[[I:.*]]: index, %[[J:.*]]: index):
+// CHECK:             %[[ELEMENT:.*]] = hlfir.apply %[[ARG0]], %[[J]], %[[I]] : (!hlfir.expr<1x2xi32>, index, index) -> i32
+// CHECK:             hlfir.yield_element %[[ELEMENT]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// box with unknown extent
+func.func @transpose2(%arg0: !fir.box<!fir.array<?x2xi32>>) {
+  %res = hlfir.transpose %arg0 : (!fir.box<!fir.array<?x2xi32>>) -> !hlfir.expr<2x?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @transpose2(
+// CHECK-SAME:                          %[[ARG0:.*]]: !fir.box<!fir.array<?x2xi32>>) {
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[DIMS0:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0]] : (!fir.box<!fir.array<?x2xi32>>, index) -> (index, index, index)
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[SHAPE:.*]] = fir.shape %[[C2]], %[[DIMS0]]#1 : (index, index) -> !fir.shape<2>
+// CHECK:           %[[EXPR:.*]] = hlfir.elemental %[[SHAPE]] : (!fir.shape<2>) -> !hlfir.expr<2x?xi32> {
+// CHECK:           ^bb0(%[[I:.*]]: index, %[[J:.*]]: index):
+// CHECK:             %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK:             %[[DIMS0:.*]]:3 = fir.box_dims %[[ARG0]], %[[C0_1]] : (!fir.box<!fir.array<?x2xi32>>, index) -> (index, index, index)
+// CHECK:             %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK:             %[[DIMS1_1:.*]]:3 = fir.box_dims %[[ARG0]], %[[C1_1]] : (!fir.box<!fir.array<?x2xi32>>, index) -> (index, index, index)
+// CHECK:             %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK:             %[[LOWER_BOUND0:.*]] = arith.subi %[[DIMS0]]#0, %[[C1_2]] : index
+// CHECK:             %[[J_OFFSET:.*]] = arith.addi %[[J]], %[[LOWER_BOUND0]] : index
+// CHECK:             %[[LOWER_BOUND1:.*]] = arith.subi %[[DIMS1_1]]#0, %[[C1_2]] : index
+// CHECK:             %[[I_OFFSET:.*]] = arith.addi %[[I]], %[[LOWER_BOUND1]] : index
+// CHECK:             %[[ELE_REF:.*]] = hlfir.designate %[[ARG0]] (%[[J_OFFSET]], %[[I_OFFSET]])  : (!fir.box<!fir.array<?x2xi32>>, index, index) -> !fir.ref<i32>
+// CHECK:             %[[ELEMENT:.*]] = fir.load %[[ELE_REF]] : !fir.ref<i32>
+// CHECK:             hlfir.yield_element %[[ELEMENT]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+
+// expr with unknown extent
+func.func @transpose3(%arg0: !hlfir.expr<?x2xi32>) {
+  %res = hlfir.transpose %arg0 : (!hlfir.expr<?x2xi32>) -> !hlfir.expr<2x?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @transpose3(
+// CHECK-SAME:                          %[[ARG0:.*]]: !hlfir.expr<?x2xi32>) {
+// CHECK:           %[[IN_SHAPE:.*]] = hlfir.shape_of %[[ARG0]] : (!hlfir.expr<?x2xi32>) -> !fir.shape<2>
+// CHECK:           %[[EXTENT0:.*]] = hlfir.get_extent %[[IN_SHAPE]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[OUT_SHAPE:.*]] = fir.shape %[[C2]], %[[EXTENT0]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[EXPR:.*]] = hlfir.elemental %[[OUT_SHAPE]] : (!fir.shape<2>) -> !hlfir.expr<2x?xi32> {
+// CHECK:           ^bb0(%[[I:.*]]: index, %[[J:.*]]: index):
+// CHECK:             %[[ELEMENT:.*]] = hlfir.apply %[[ARG0]], %[[J]], %[[I]] : (!hlfir.expr<?x2xi32>, index, index) -> i32
+// CHECK:             hlfir.yield_element %[[ELEMENT]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/HLFIR/where.fir b/flang/test/HLFIR/where.fir
new file mode 100644
index 0000000000000..2aa242c9491df
--- /dev/null
+++ b/flang/test/HLFIR/where.fir
@@ -0,0 +1,28 @@
+// Test hlfir.where operation parse, verify (no errors), and unparse.
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+func.func @test_where(%mask: !fir.ref<!fir.array<10x!fir.logical<4>>>, %x: !fir.ref<!fir.array<10xf32>>, %y: !fir.box<!fir.array<?xf32>>) {
+  hlfir.where {
+    hlfir.yield %mask : !fir.ref<!fir.array<10x!fir.logical<4>>>
+  } do {
+    hlfir.region_assign {
+      hlfir.yield %y : !fir.box<!fir.array<?xf32>>
+    } to {
+      hlfir.yield %x : !fir.ref<!fir.array<10xf32>>
+    }
+  }
+  return
+}
+// CHECK-LABEL:   func.func @test_where(
+// CHECK-SAME:                          %[[VAL_0:.*]]: !fir.ref<!fir.array<10x!fir.logical<4>>>,
+// CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<!fir.array<10xf32>>,
+// CHECK-SAME:                          %[[VAL_2:.*]]: !fir.box<!fir.array<?xf32>>) {
+// CHECK:           hlfir.where {
+// CHECK:             hlfir.yield %[[VAL_0]] : !fir.ref<!fir.array<10x!fir.logical<4>>>
+// CHECK:           } do {
+// CHECK:             hlfir.region_assign {
+// CHECK:               hlfir.yield %[[VAL_2]] : !fir.box<!fir.array<?xf32>>
+// CHECK:             } to {
+// CHECK:               hlfir.yield %[[VAL_1]] : !fir.ref<!fir.array<10xf32>>
+// CHECK:             }
+// CHECK:           }
diff --git a/flang/test/Lower/HLFIR/assignment-intrinsics.f90 b/flang/test/Lower/HLFIR/assignment-intrinsics.f90
index 78b7083857215..169e5fa4b7027 100644
--- a/flang/test/Lower/HLFIR/assignment-intrinsics.f90
+++ b/flang/test/Lower/HLFIR/assignment-intrinsics.f90
@@ -74,7 +74,8 @@ subroutine scalar_logical_2(x)
 ! CHECK-LABEL: func.func @_QPscalar_logical_2(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}  {uniq_name = "_QFscalar_logical_2Ex"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant true
-! CHECK:  hlfir.assign %[[VAL_2]] to %[[VAL_1]]#0 : i1, !fir.ref<!fir.logical<4>>
+! CHECK:  %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<4>
+! CHECK:  hlfir.assign %[[VAL_3]] to %[[VAL_1]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 
 subroutine scalar_real_2(x)
   real :: x
diff --git a/flang/test/Lower/HLFIR/calls-character-singleton-result.f90 b/flang/test/Lower/HLFIR/calls-character-singleton-result.f90
index 960484c6415cf..638a8078a949e 100644
--- a/flang/test/Lower/HLFIR/calls-character-singleton-result.f90
+++ b/flang/test/Lower/HLFIR/calls-character-singleton-result.f90
@@ -11,7 +11,8 @@ subroutine scalar_char(c, i)
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_char(
 ! CHECK:  %[[VAL_2:.*]] = fir.alloca !fir.char<1>
-! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %{{.*}}#0 typeparams %{{.*}}  {{.*}}Ec
+! CHECK:  %[[VAL_4:.*]] = fir.convert %{{.*}}#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] typeparams %{{.*}}  {{.*}}Ec
 ! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %{{.*}}  {{.*}}Ei
 ! CHECK:  %[[VAL_7:.*]] = fir.load %[[VAL_6]]#1 : !fir.ref<i64>
 ! CHECK:  %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i64) -> i8
@@ -20,7 +21,7 @@ subroutine scalar_char(c, i)
 ! CHECK:  fir.store %[[VAL_10]] to %[[VAL_2]] : !fir.ref<!fir.char<1>>
 ! CHECK:  %[[VAL_11:.*]] = arith.constant false
 ! CHECK:  %[[VAL_12:.*]] = hlfir.as_expr %[[VAL_2]] move %[[VAL_11]] : (!fir.ref<!fir.char<1>>, i1) -> !hlfir.expr<!fir.char<1>>
-! CHECK:  hlfir.assign %[[VAL_12]] to %[[VAL_5]]#0 : !hlfir.expr<!fir.char<1>>, !fir.boxchar<1>
+! CHECK:  hlfir.assign %[[VAL_12]] to %[[VAL_5]]#0 : !hlfir.expr<!fir.char<1>>, !fir.ref<!fir.char<1>>
 
 subroutine scalar_bindc(c)
   character(1) :: c
@@ -32,12 +33,13 @@ character(1) function bar() bind(c)
 end subroutine
 ! CHECK-LABEL: func.func @_QPscalar_bindc(
 ! CHECK:  %[[VAL_1:.*]] = fir.alloca !fir.char<1>
-! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %{{.*}}#0 typeparams %{{.*}}  {{.*}}Ec
+! CHECK:  %[[VAL_3:.*]] = fir.convert %{{.*}}#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] typeparams %{{.*}}  {{.*}}Ec
 ! CHECK:  %[[VAL_5:.*]] = fir.call @bar() fastmath<contract> : () -> !fir.char<1>
 ! CHECK:  fir.store %[[VAL_5]] to %[[VAL_1]] : !fir.ref<!fir.char<1>>
 ! CHECK:  %[[VAL_6:.*]] = arith.constant false
 ! CHECK:  %[[VAL_7:.*]] = hlfir.as_expr %[[VAL_1]] move %[[VAL_6]] : (!fir.ref<!fir.char<1>>, i1) -> !hlfir.expr<!fir.char<1>>
-! CHECK:  hlfir.assign %[[VAL_7]] to %[[VAL_4]]#0 : !hlfir.expr<!fir.char<1>>, !fir.boxchar<1>
+! CHECK:  hlfir.assign %[[VAL_7]] to %[[VAL_4]]#0 : !hlfir.expr<!fir.char<1>>, !fir.ref<!fir.char<1>>
 
 subroutine array_char(c, i)
   character(1) :: c(100)
diff --git a/flang/test/Lower/HLFIR/convert-variable.f90 b/flang/test/Lower/HLFIR/convert-variable.f90
index ab9425afe3f1e..1905241fc0274 100644
--- a/flang/test/Lower/HLFIR/convert-variable.f90
+++ b/flang/test/Lower/HLFIR/convert-variable.f90
@@ -23,7 +23,8 @@ subroutine scalar_character_cst_len(c)
 ! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1>
 ! CHECK:  %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 10 : index
-! CHECK:  %[[VAL_3:.*]] = hlfir.declare %[[VAL_1]]#0 typeparams %[[VAL_2]] {uniq_name = "_QFscalar_character_cst_lenEc"} : (!fir.ref<!fir.char<1,?>>, index) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+! CHECK:  %[[VAL_3:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
+! CHECK:  %[[VAL_4:.*]] = hlfir.declare %[[VAL_3]] typeparams %[[VAL_2]] {uniq_name = "_QFscalar_character_cst_lenEc"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
 
 subroutine array_numeric(x)
   integer :: x(10, 20)
@@ -95,3 +96,16 @@ subroutine scalar_numeric_parameter()
 ! CHECK-LABEL: func.func @_QPscalar_numeric_parameter() {
 ! CHECK:  %[[VAL_0:.*]] = fir.address_of(@_QFscalar_numeric_parameterECp) : !fir.ref<i32>
 ! CHECK:  %[[VAL_1:.*]] = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QFscalar_numeric_parameterECp"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+subroutine test_component_in_spec_expr(x, derived)
+  type t
+    integer :: component
+  end type
+  type(t) :: derived
+  ! Test that we do not try to instantiate "component" just because
+  ! its symbol appears in a specification expression.
+  real :: x(derived%component)
+end subroutine
+! CHECK-LABEL: func.func @_QPtest_component_in_spec_expr(
+! CHECK-NOT: alloca
+! CHECK: return
diff --git a/flang/test/Lower/HLFIR/expr-as-inquired.f90 b/flang/test/Lower/HLFIR/expr-as-inquired.f90
index 9dac1006649e5..8543921ca242e 100644
--- a/flang/test/Lower/HLFIR/expr-as-inquired.f90
+++ b/flang/test/Lower/HLFIR/expr-as-inquired.f90
@@ -16,7 +16,8 @@ subroutine test_isAllocated(x, l)
 ! CHECK:  %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (!fir.heap<!fir.array<?xf32>>) -> i64
 ! CHECK:  %[[VAL_7:.*]] = arith.constant 0 : i64
 ! CHECK:  %[[VAL_8:.*]] = arith.cmpi ne, %[[VAL_6]], %[[VAL_7]] : i64
-! CHECK:  hlfir.assign %[[VAL_8]] to %[[VAL_2]]#0 : i1, !fir.ref<!fir.logical<4>>
+! CHECK:  %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i1) -> !fir.logical<4>
+! CHECK:  hlfir.assign %[[VAL_9]] to %[[VAL_2]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
 ! CHECK:  return
 ! CHECK:  }
 
diff --git a/flang/test/Lower/HLFIR/expr-box.f90 b/flang/test/Lower/HLFIR/expr-box.f90
index e42c01a8f91d9..e7ab006751a06 100644
--- a/flang/test/Lower/HLFIR/expr-box.f90
+++ b/flang/test/Lower/HLFIR/expr-box.f90
@@ -1,5 +1,5 @@
 ! Test lowering of of expressions as fir.box
-! RUN: bbc -hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -hlfir -o - %s 2>&1 --use-hlfir-intrinsic-ops=false | FileCheck %s
 
 ! CHECK-LABEL: func.func @_QPfoo(
 ! CHECK-SAME:  %[[VAL_0:.*]]: !fir.ref<!fir.array<10xi32>>
diff --git a/flang/test/Lower/HLFIR/implicit-call-mismatch.f90 b/flang/test/Lower/HLFIR/implicit-call-mismatch.f90
index 1c722fc549cf1..4c10b32fba189 100644
--- a/flang/test/Lower/HLFIR/implicit-call-mismatch.f90
+++ b/flang/test/Lower/HLFIR/implicit-call-mismatch.f90
@@ -37,10 +37,11 @@ subroutine pass_kind2_char_to_char(c)
   call takes_char(c)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpass_kind2_char_to_char(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}Ec
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}Ec
+! CHECK:  %[[VAL_3:.*]] = fir.emboxchar %[[VAL_2]]#1, %{{.*}} : (!fir.ref<!fir.char<2,4>>, index) -> !fir.boxchar<2>
 ! CHECK:  %[[VAL_4:.*]] = fir.address_of(@_QPtakes_char) : (!fir.boxchar<1>) -> ()
 ! CHECK:  %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : ((!fir.boxchar<1>) -> ()) -> ((!fir.boxchar<2>) -> ())
-! CHECK:  fir.call %[[VAL_5]](%[[VAL_3]]#0) {{.*}}: (!fir.boxchar<2>) -> ()
+! CHECK:  fir.call %[[VAL_5]](%[[VAL_3]]) {{.*}}: (!fir.boxchar<2>) -> ()
 
 subroutine takes_real(r)
   real(8) :: r
@@ -60,10 +61,11 @@ subroutine pass_char_to_real(c)
   call takes_real(c)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpass_char_to_real(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}Ec
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}Ec
+! CHECK:  %[[VAL_3:.*]] = fir.emboxchar %[[VAL_2]]#1, %{{.*}} : (!fir.ref<!fir.char<1,8>>, index) -> !fir.boxchar<1>
 ! CHECK:  %[[VAL_4:.*]] = fir.address_of(@_QPtakes_real) : (!fir.ref<f64>) -> ()
 ! CHECK:  %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : ((!fir.ref<f64>) -> ()) -> ((!fir.boxchar<1>) -> ())
-! CHECK:  fir.call %[[VAL_5]](%[[VAL_3]]#0) {{.*}}: (!fir.boxchar<1>) -> ()
+! CHECK:  fir.call %[[VAL_5]](%[[VAL_3]]) {{.*}}: (!fir.boxchar<1>) -> ()
 
 subroutine pass_proc_to_real()
   real(8), external :: proc
@@ -105,10 +107,11 @@ subroutine pass_char_to_char_proc(c)
   call takes_char_proc(c)
 end subroutine
 ! CHECK-LABEL: func.func @_QPpass_char_to_char_proc(
-! CHECK:  %[[VAL_3:.*]]:2 = hlfir.declare {{.*}}Ec
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare {{.*}}Ec
+! CHECK:  %[[VAL_3:.*]] = fir.emboxchar %[[VAL_2]]#1, %{{.*}} : (!fir.ref<!fir.char<1,8>>, index) -> !fir.boxchar<1>
 ! CHECK:  %[[VAL_4:.*]] = fir.address_of(@_QPtakes_char_proc) : (tuple<!fir.boxproc<() -> ()>, i64>) -> ()
 ! CHECK:  %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : ((tuple<!fir.boxproc<() -> ()>, i64>) -> ()) -> ((!fir.boxchar<1>) -> ())
-! CHECK:  fir.call %[[VAL_5]](%[[VAL_3]]#0) {{.*}}: (!fir.boxchar<1>) -> ()
+! CHECK:  fir.call %[[VAL_5]](%[[VAL_3]]) {{.*}}: (!fir.boxchar<1>) -> ()
 
 subroutine takes_proc(proc)
   real(8), external :: proc
diff --git a/flang/test/Lower/HLFIR/implicit-type-conversion.f90 b/flang/test/Lower/HLFIR/implicit-type-conversion.f90
new file mode 100644
index 0000000000000..66bf7f81f78de
--- /dev/null
+++ b/flang/test/Lower/HLFIR/implicit-type-conversion.f90
@@ -0,0 +1,137 @@
+! RUN: bbc -emit-fir -hlfir %s -o - | FileCheck %s
+
+! CHECK-LABEL:   func.func @_QPtest1(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest1Ey"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.logical<4>) -> i32
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
+! CHECK:           return
+! CHECK:         }
+subroutine test1(x, y)
+  integer :: x
+  logical :: y
+  x = y
+end subroutine test1
+
+! CHECK-LABEL:   func.func @_QPtest2(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest2Ey"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i32) -> !fir.logical<4>
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+! CHECK:           return
+! CHECK:         }
+subroutine test2(x, y)
+  integer :: x
+  logical :: y
+  y = x
+end subroutine test2
+
+! CHECK-LABEL:   func.func @_QPtest3(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest3Ex"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest3Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.cmpi eq, %[[VAL_4]], %[[VAL_5]] : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.no_reassoc %[[VAL_6]] : i1
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> !fir.logical<4>
+! CHECK:           hlfir.assign %[[VAL_8]] to %[[VAL_2]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+! CHECK:           return
+! CHECK:         }
+subroutine test3(x, y)
+  logical :: x
+  integer :: y
+  x = (y.eq.1)
+end subroutine test3
+
+! CHECK-LABEL:   func.func @_QPtest4(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest4Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest4Ey"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.cmpi eq, %[[VAL_4]], %[[VAL_5]] : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.no_reassoc %[[VAL_6]] : i1
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i1) -> i32
+! CHECK:           hlfir.assign %[[VAL_8]] to %[[VAL_2]]#0 : i32, !fir.ref<i32>
+! CHECK:           return
+! CHECK:         }
+subroutine test4(x, y)
+  integer :: x
+  integer :: y
+  x = (y.eq.1)
+end subroutine test4
+
+! CHECK-LABEL:   func.func @_QPtest5(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest5Ex"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest5Ey"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           %[[VAL_4:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (!fir.logical<4>) -> i32
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_2]]#0 : i32, !fir.box<!fir.array<?xi32>>
+! CHECK:           return
+! CHECK:         }
+subroutine test5(x, y)
+  integer :: x(:)
+  logical :: y
+  x = y
+end subroutine test5
+
+! CHECK-LABEL:   func.func @_QPtest6(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest6Ex"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest6Ey"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]]#0, %[[VAL_4]] : (!fir.box<!fir.array<?x!fir.logical<4>>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+! CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+! CHECK:             %[[VAL_9:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_8]])  : (!fir.box<!fir.array<?x!fir.logical<4>>>, index) -> !fir.ref<!fir.logical<4>>
+! CHECK:             %[[VAL_10:.*]] = fir.load %[[VAL_9]] : !fir.ref<!fir.logical<4>>
+! CHECK:             %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (!fir.logical<4>) -> i32
+! CHECK:             hlfir.yield_element %[[VAL_11]] : i32
+! CHECK:           }
+! CHECK:           hlfir.assign %[[VAL_12:.*]] to %[[VAL_2]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_12]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+subroutine test6(x, y)
+  integer :: x(:)
+  logical :: y(:)
+  x = y
+end subroutine test6
+
+! CHECK-LABEL:   func.func @_QPtest7(
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "x"},
+! CHECK-SAME:                        %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "y"}) {
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest7Ex"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFtest7Ey"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]]#0, %[[VAL_4]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_5]]#1 : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_7:.*]] = hlfir.elemental %[[VAL_6]] : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+! CHECK:           ^bb0(%[[VAL_8:.*]]: index):
+! CHECK:             %[[VAL_9:.*]] = hlfir.designate %[[VAL_3]]#0 (%[[VAL_8]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK:             %[[VAL_10:.*]] = fir.load %[[VAL_9]] : !fir.ref<i32>
+! CHECK:             %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i32) -> !fir.logical<4>
+! CHECK:             hlfir.yield_element %[[VAL_11]] : !fir.logical<4>
+! CHECK:           }
+! CHECK:           hlfir.assign %[[VAL_12:.*]] to %[[VAL_2]]#0 : !hlfir.expr<?x!fir.logical<4>>, !fir.box<!fir.array<?x!fir.logical<4>>>
+! CHECK:           hlfir.destroy %[[VAL_12]] : !hlfir.expr<?x!fir.logical<4>>
+! CHECK:           return
+! CHECK:         }
+subroutine test7(x, y)
+  logical :: x(:)
+  integer :: y(:)
+  x = y
+end subroutine test7
diff --git a/flang/test/Lower/HLFIR/null.f90 b/flang/test/Lower/HLFIR/null.f90
new file mode 100644
index 0000000000000..6ae44082f3164
--- /dev/null
+++ b/flang/test/Lower/HLFIR/null.f90
@@ -0,0 +1,20 @@
+! Test lowering of NULL(MOLD) to HLFIR.
+! RUN: bbc -emit-fir -hlfir -o - %s | FileCheck %s
+subroutine test(mold)
+  integer, pointer :: mold(:)
+  interface
+    subroutine takes_ptr(p)
+      integer, pointer :: p(:)
+    end subroutine
+  end interface
+  call takes_ptr(null(mold))
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest(
+! CHECK:  %[[VAL_1:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:  %[[VAL_3:.*]] = fir.zero_bits !fir.ptr<!fir.array<?xi32>>
+! CHECK:  %[[VAL_4:.*]] = arith.constant 0 : index
+! CHECK:  %[[VAL_5:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_6:.*]] = fir.embox %[[VAL_3]](%[[VAL_5]]) : (!fir.ptr<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xi32>>>
+! CHECK:  fir.store %[[VAL_6]] to %[[VAL_1]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = ".tmp.intrinsic_result"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! CHECK:  fir.call @_QPtakes_ptr(%[[VAL_7]]#0) fastmath<contract> : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> ()
diff --git a/flang/test/Lower/HLFIR/product.f90 b/flang/test/Lower/HLFIR/product.f90
new file mode 100644
index 0000000000000..148d53bf2ccdb
--- /dev/null
+++ b/flang/test/Lower/HLFIR/product.f90
@@ -0,0 +1,113 @@
+! Test lowering of PRODUCT intrinsic to HLFIR
+! RUN: bbc -emit-fir -hlfir -o - %s 2>&1 | FileCheck %s
+
+! simple 1 argument PRODUCT
+subroutine product1(a, s)
+  integer :: a(:), s
+  s = PRODUCT(a)
+end subroutine
+! CHECK-LABEL: func.func @_QPproduct1(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32>
+! CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.product %[[ARRAY]]#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>) -> !hlfir.expr<i32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<i32>, !fir.ref<i32>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]]
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! product with by-ref DIM argument
+subroutine product2(a, s, d)
+  integer :: a(:,:), s(:), d
+  s = PRODUCT(a, d)
+end subroutine
+! CHECK-LABEL: func.func @_QPproduct2(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}, %[[ARG2:.*]]: !fir.ref<i32>
+! CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-DAG:     %[[DIM_REF:.*]]:2 = hlfir.declare %[[ARG2]]
+! CHECK-NEXT:    %[[DIM:.*]] = fir.load %[[DIM_REF]]#0 : !fir.ref<i32>
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.product %[[ARRAY]]#0 dim %[[DIM]] {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?xi32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]]
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! product with scalar mask argument
+subroutine product3(a, s, m)
+  integer :: a(:), s
+  logical :: m
+  s = PRODUCT(a, m)
+end subroutine
+! CHECK-LABEL: func.func @_QPproduct3(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}, %[[ARG2:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "m"})
+! CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG2]]
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.product %[[ARRAY]]#0 mask %[[MASK]]#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.logical<4>>) -> !hlfir.expr<i32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<i32>, !fir.ref<i32>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]]
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! product with array mask argument
+subroutine product4(a, s, m)
+  integer :: a(:), s
+  logical :: m(:)
+  s = PRODUCT(a, m)
+end subroutine
+
+! CHECK-LABEL: func.func @_QPproduct4(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}, %arg2: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "m"})
+! CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG2]]
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.product %[[ARRAY]]#0 mask %[[MASK]]#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?x!fir.logical<4>>>) -> !hlfir.expr<i32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<i32>, !fir.ref<i32>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]]
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! product with all 3 arguments, dim is by-val, array isn't boxed
+subroutine product5(s)
+  integer :: s(2)
+  integer :: a(2,2) = reshape((/1, 2, 3, 4/), [2,2])
+  s = PRODUCT(a, 1, .true.)
+end subroutine
+
+! CHECK-LABEL: func.func @_QPproduct5(
+! CHECK:           %[[ARG0:.*]]: !fir.ref<!fir.array<2xi32>>
+! CHECK-DAG:     %[[ADDR:.*]] = fir.address_of({{.*}}) : !fir.ref<!fir.array<2x2xi32>>
+! CHECK-DAG:     %[[ARRAY_SHAPE:.*]] = fir.shape {{.*}} -> !fir.shape<2>
+! CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ADDR]](%[[ARRAY_SHAPE]])
+! CHECK-DAG:     %[[OUT_SHAPE:.*]] = fir.shape {{.*}} -> !fir.shape<1>
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG0]](%[[OUT_SHAPE]])
+! CHECK-DAG:     %[[C1:.*]] = arith.constant 1 : i32
+! CHECK-DAG:     %[[TRUE:.*]] = arith.constant true
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.product %[[ARRAY]]#0 dim %[[C1]] mask %[[TRUE]]  {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<2x2xi32>>, i32, i1) -> !hlfir.expr<2xi32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! product with dimesnsion from pointer
+subroutine product6(a, s, d)
+  integer, pointer :: d
+  real :: a(:,:), s(:)
+  s = PRODUCT(a, (d))
+end subroutine
+
+! CHECK-LABEL: func.func @_QPproduct6(
+! CHECK:           %[[ARG0:.*]]:  !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "s"}, %[[ARG2:.*]]: !fir.ref<!fir.box<!fir.ptr<i32>>> {fir.bindc_name = "d"})
+! CHECK-DAG:     %[[ARRAY:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-DAG:     %[[DIM:.*]]:2 = hlfir.declare %[[ARG2]]
+! CHECK-NEXT:    %[[DIM_BOX:.*]] = fir.load %[[DIM]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+! CHECK-NEXT:    %[[DIM_ADDR:.*]] = fir.box_addr %[[DIM_BOX]]  : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+! CHECK-NEXT:    %[[DIM0:.*]] = fir.load %[[DIM_ADDR]] : !fir.ptr<i32>
+! CHECK-NEXT:    %[[DIM1:.*]] = hlfir.no_reassoc %[[DIM0]] : i32
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.product %[[ARRAY]]#0 dim %[[DIM1]]  {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x?xf32>>, i32) -> !hlfir.expr<?xf32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<?xf32>, !fir.box<!fir.array<?xf32>>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]]
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
diff --git a/flang/test/Lower/HLFIR/substrings.f90 b/flang/test/Lower/HLFIR/substrings.f90
index d97f75eb51d65..3b69b151c2e3e 100644
--- a/flang/test/Lower/HLFIR/substrings.f90
+++ b/flang/test/Lower/HLFIR/substrings.f90
@@ -24,7 +24,7 @@ subroutine cst_len(array, scalar)
 ! CHECK:  %[[VAL_34:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_15]]:%[[VAL_5]]:%[[VAL_15]]) substr %[[VAL_16]], %[[VAL_3]]  shape %[[VAL_6]] typeparams %[[VAL_33]] : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, index, index, index, index, index, !fir.shape<1>, index) -> !fir.box<!fir.array<100x!fir.char<1,9>>>
 
   print *, scalar(2:5)
-! CHECK:  %[[VAL_40:.*]] = hlfir.designate %[[VAL_9]]#0  substr %[[VAL_16]], %[[VAL_17]]  typeparams %[[VAL_18]] : (!fir.boxchar<1>, index, index, index) -> !fir.ref<!fir.char<1,4>>
+! CHECK:  %[[VAL_40:.*]] = hlfir.designate %[[VAL_9]]#0  substr %[[VAL_16]], %[[VAL_17]]  typeparams %[[VAL_18]] : (!fir.ref<!fir.char<1,10>>, index, index, index) -> !fir.ref<!fir.char<1,4>>
 end subroutine
 
 ! CHECK-LABEL:   func.func @_QPdyn_len(
diff --git a/flang/test/Lower/HLFIR/vector-subscript-as-value.f90 b/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
new file mode 100644
index 0000000000000..067bb65eeffe8
--- /dev/null
+++ b/flang/test/Lower/HLFIR/vector-subscript-as-value.f90
@@ -0,0 +1,184 @@
+! Test lowering of vector subscript designators outside of the
+! assignment left-and side and input IO context.
+! RUN: bbc -emit-fir -hlfir -o - -I nw %s 2>&1 | FileCheck %s
+
+subroutine foo(x, y)
+  integer :: x(100)
+  integer(8) :: y(20)
+  call bar(x(y))
+end subroutine
+! CHECK-LABEL:   func.func @_QPfoo(
+! CHECK:  %[[VAL_2:.*]] = arith.constant 100 : index
+! CHECK:  %[[VAL_3:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]](%[[VAL_3:[a-z0-9]*]])  {{.*}}Ex
+! CHECK:  %[[VAL_5:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_6:.*]] = fir.shape %[[VAL_5]] : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_6:[a-z0-9]*]])  {{.*}}Ey
+! CHECK:  %[[VAL_8:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_10:.*]] = hlfir.elemental %[[VAL_9]] : (!fir.shape<1>) -> !hlfir.expr<20xi32> {
+! CHECK:  ^bb0(%[[VAL_11:.*]]: index):
+! CHECK:    %[[VAL_12:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_11]])  : (!fir.ref<!fir.array<20xi64>>, index) -> !fir.ref<i64>
+! CHECK:    %[[VAL_13:.*]] = fir.load %[[VAL_12]] : !fir.ref<i64>
+! CHECK:    %[[VAL_14:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<100xi32>>, i64) -> !fir.ref<i32>
+! CHECK:    %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK:    hlfir.yield_element %[[VAL_15]] : i32
+! CHECK:  }
+! CHECK:  %[[VAL_16:.*]]:3 = hlfir.associate %[[VAL_17:.*]](%[[VAL_9]]) {uniq_name = "adapt.valuebyref"} : (!hlfir.expr<20xi32>, !fir.shape<1>) -> (!fir.ref<!fir.array<20xi32>>, !fir.ref<!fir.array<20xi32>>, i1)
+! CHECK:  fir.call @_QPbar(%[[VAL_16]]#1) fastmath<contract> : (!fir.ref<!fir.array<20xi32>>) -> ()
+! CHECK:  hlfir.end_associate %[[VAL_16]]#1, %[[VAL_16]]#2 : !fir.ref<!fir.array<20xi32>>, i1
+! CHECK:  hlfir.destroy %[[VAL_17]] : !hlfir.expr<20xi32>
+
+subroutine foo2(x, y)
+  integer :: x(10, 30, 100)
+  integer(8) :: y(20)
+  call bar2(x(1:8:2, 5, y))
+end subroutine
+! CHECK-LABEL:   func.func @_QPfoo2(
+! CHECK:  %[[VAL_2:.*]] = arith.constant 10 : index
+! CHECK:  %[[VAL_3:.*]] = arith.constant 30 : index
+! CHECK:  %[[VAL_4:.*]] = arith.constant 100 : index
+! CHECK:  %[[VAL_5:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : (index, index, index) -> !fir.shape<3>
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]](%[[VAL_5:[a-z0-9]*]])  {{.*}}Ex
+! CHECK:  %[[VAL_7:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_8:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_8:[a-z0-9]*]])  {{.*}}Ey
+! CHECK:  %[[VAL_10:.*]] = arith.constant 1 : index
+! CHECK:  %[[VAL_11:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_12:.*]] = arith.constant 4 : index
+! CHECK:  %[[VAL_13:.*]] = arith.constant 5 : index
+! CHECK:  %[[VAL_14:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_15:.*]] = fir.shape %[[VAL_12]], %[[VAL_14]] : (index, index) -> !fir.shape<2>
+! CHECK:  %[[VAL_16:.*]] = hlfir.elemental %[[VAL_15]] : (!fir.shape<2>) -> !hlfir.expr<4x20xi32> {
+! CHECK:  ^bb0(%[[VAL_17:.*]]: index, %[[VAL_18:.*]]: index):
+! CHECK:    %[[VAL_19:.*]] = arith.constant 1 : index
+! CHECK:    %[[VAL_20:.*]] = arith.subi %[[VAL_17]], %[[VAL_19]] : index
+! CHECK:    %[[VAL_21:.*]] = arith.muli %[[VAL_20]], %[[VAL_11]] : index
+! CHECK:    %[[VAL_22:.*]] = arith.addi %[[VAL_10]], %[[VAL_21]] : index
+! CHECK:    %[[VAL_23:.*]] = hlfir.designate %[[VAL_9]]#0 (%[[VAL_18]])  : (!fir.ref<!fir.array<20xi64>>, index) -> !fir.ref<i64>
+! CHECK:    %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<i64>
+! CHECK:    %[[VAL_25:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_22]], %[[VAL_13]], %[[VAL_24]])  : (!fir.ref<!fir.array<10x30x100xi32>>, index, index, i64) -> !fir.ref<i32>
+! CHECK:    %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<i32>
+! CHECK:    hlfir.yield_element %[[VAL_26]] : i32
+! CHECK:  }
+
+subroutine foo3(x, y)
+  integer, pointer :: x(:, :, :)
+  integer(8) :: y(20)
+  call bar2(x(1:8:2, 5, y))
+end subroutine
+! CHECK-LABEL:   func.func @_QPfoo3(
+! CHECK:  %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFfoo3Ex"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>)
+! CHECK:  %[[VAL_3:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_4:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]](%[[VAL_4:[a-z0-9]*]])  {{.*}}Ey
+! CHECK:  %[[VAL_6:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>>
+! CHECK:  %[[VAL_7:.*]] = arith.constant 1 : index
+! CHECK:  %[[VAL_8:.*]] = arith.constant 2 : index
+! CHECK:  %[[VAL_9:.*]] = arith.constant 4 : index
+! CHECK:  %[[VAL_10:.*]] = arith.constant 5 : index
+! CHECK:  %[[VAL_11:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_12:.*]] = fir.shape %[[VAL_9]], %[[VAL_11]] : (index, index) -> !fir.shape<2>
+! CHECK:  %[[VAL_13:.*]] = hlfir.elemental %[[VAL_12]] : (!fir.shape<2>) -> !hlfir.expr<4x20xi32> {
+! CHECK:  ^bb0(%[[VAL_14:.*]]: index, %[[VAL_15:.*]]: index):
+! CHECK:    %[[VAL_16:.*]] = arith.constant 1 : index
+! CHECK:    %[[VAL_17:.*]] = arith.subi %[[VAL_14]], %[[VAL_16]] : index
+! CHECK:    %[[VAL_18:.*]] = arith.muli %[[VAL_17]], %[[VAL_8]] : index
+! CHECK:    %[[VAL_19:.*]] = arith.addi %[[VAL_7]], %[[VAL_18]] : index
+! CHECK:    %[[VAL_20:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_15]])  : (!fir.ref<!fir.array<20xi64>>, index) -> !fir.ref<i64>
+! CHECK:    %[[VAL_21:.*]] = fir.load %[[VAL_20]] : !fir.ref<i64>
+! CHECK:    %[[VAL_22:.*]] = hlfir.designate %[[VAL_6]] (%[[VAL_19]], %[[VAL_10]], %[[VAL_21]])  : (!fir.box<!fir.ptr<!fir.array<?x?x?xi32>>>, index, index, i64) -> !fir.ref<i32>
+! CHECK:    %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<i32>
+! CHECK:    hlfir.yield_element %[[VAL_23]] : i32
+! CHECK:  }
+
+subroutine foo4(at1, vector, i, j, k, l, step)
+  type t0
+    complex :: x(10, 20)
+  end type
+  type t1
+    type(t0) :: at0(30, 40, 50)
+  end type
+  type(t1) :: at1(:)
+  integer(8) :: vector(:), step, i, j, k, l
+  call bar3(at1(i)%at0(1:8:step, j, vector)%x(k, l)%im)
+end subroutine
+! CHECK-LABEL:   func.func @_QPfoo4(
+! CHECK:  %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]]  {{.*}}Eat1
+! CHECK:  %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_2:[a-z0-9]*]]  {{.*}}Ei
+! CHECK:  %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_3:[a-z0-9]*]]  {{.*}}Ej
+! CHECK:  %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_4:[a-z0-9]*]]  {{.*}}Ek
+! CHECK:  %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_5:[a-z0-9]*]]  {{.*}}El
+! CHECK:  %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_6:[a-z0-9]*]]  {{.*}}Estep
+! CHECK:  %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]]  {{.*}}Evector
+! CHECK:  %[[VAL_14:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_15:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_14]])  : (!fir.box<!fir.array<?x!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>}>>>, i64) -> !fir.ref<!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>}>>
+! CHECK:  %[[VAL_16:.*]] = arith.constant 30 : index
+! CHECK:  %[[VAL_17:.*]] = arith.constant 40 : index
+! CHECK:  %[[VAL_18:.*]] = arith.constant 50 : index
+! CHECK:  %[[VAL_19:.*]] = fir.shape %[[VAL_16]], %[[VAL_17]], %[[VAL_18]] : (index, index, index) -> !fir.shape<3>
+! CHECK:  %[[VAL_20:.*]] = arith.constant 1 : index
+! CHECK:  %[[VAL_21:.*]] = arith.constant 8 : index
+! CHECK:  %[[VAL_22:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_23:.*]] = fir.convert %[[VAL_22]] : (i64) -> index
+! CHECK:  %[[VAL_24:.*]] = arith.constant 0 : index
+! CHECK:  %[[VAL_25:.*]] = arith.subi %[[VAL_21]], %[[VAL_20]] : index
+! CHECK:  %[[VAL_26:.*]] = arith.addi %[[VAL_25]], %[[VAL_23]] : index
+! CHECK:  %[[VAL_27:.*]] = arith.divsi %[[VAL_26]], %[[VAL_23]] : index
+! CHECK:  %[[VAL_28:.*]] = arith.cmpi sgt, %[[VAL_27]], %[[VAL_24]] : index
+! CHECK:  %[[VAL_29:.*]] = arith.select %[[VAL_28]], %[[VAL_27]], %[[VAL_24]] : index
+! CHECK:  %[[VAL_30:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_31:.*]] = arith.constant 0 : index
+! CHECK:  %[[VAL_32:.*]]:3 = fir.box_dims %[[VAL_13]]#0, %[[VAL_31]] : (!fir.box<!fir.array<?xi64>>, index) -> (index, index, index)
+! CHECK:  %[[VAL_33:.*]] = fir.shape %[[VAL_29]], %[[VAL_32]]#1 : (index, index) -> !fir.shape<2>
+! CHECK:  %[[VAL_34:.*]] = arith.constant 10 : index
+! CHECK:  %[[VAL_35:.*]] = arith.constant 20 : index
+! CHECK:  %[[VAL_36:.*]] = fir.shape %[[VAL_34]], %[[VAL_35]] : (index, index) -> !fir.shape<2>
+! CHECK:  %[[VAL_37:.*]] = fir.load %[[VAL_10]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_38:.*]] = fir.load %[[VAL_11]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_39:.*]] = hlfir.elemental %[[VAL_33]] : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+! CHECK:  ^bb0(%[[VAL_40:.*]]: index, %[[VAL_41:.*]]: index):
+! CHECK:    %[[VAL_42:.*]] = arith.constant 1 : index
+! CHECK:    %[[VAL_43:.*]] = arith.subi %[[VAL_40]], %[[VAL_42]] : index
+! CHECK:    %[[VAL_44:.*]] = arith.muli %[[VAL_43]], %[[VAL_23]] : index
+! CHECK:    %[[VAL_45:.*]] = arith.addi %[[VAL_20]], %[[VAL_44]] : index
+! CHECK:    %[[VAL_46:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_41]])  : (!fir.box<!fir.array<?xi64>>, index) -> !fir.ref<i64>
+! CHECK:    %[[VAL_47:.*]] = fir.load %[[VAL_46]] : !fir.ref<i64>
+! CHECK:    %[[VAL_48:.*]] = hlfir.designate %[[VAL_15]]{"at0"} <%[[VAL_19]]> (%[[VAL_45]], %[[VAL_30]], %[[VAL_47]])  : (!fir.ref<!fir.type<_QFfoo4Tt1{at0:!fir.array<30x40x50x!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>}>>, !fir.shape<3>, index, i64, i64) -> !fir.ref<!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>
+! CHECK:    %[[VAL_49:.*]] = hlfir.designate %[[VAL_48]]{"x"} <%[[VAL_36]]> (%[[VAL_37]], %[[VAL_38]]) imag : (!fir.ref<!fir.type<_QFfoo4Tt0{x:!fir.array<10x20x!fir.complex<4>>}>>, !fir.shape<2>, i64, i64) -> !fir.ref<f32>
+! CHECK:    %[[VAL_50:.*]] = fir.load %[[VAL_49]] : !fir.ref<f32>
+! CHECK:    hlfir.yield_element %[[VAL_50]] : f32
+! CHECK:  }
+
+subroutine substring(c, vector, i, j)
+  character(*) :: c(:)
+  integer(8) :: vector(:), step, i, j
+  call bar4(c(vector)(i:j))
+end subroutine
+! CHECK-LABEL:   func.func @_QPsubstring(
+! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]]  {{.*}}Ec
+! CHECK:  %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2:[a-z0-9]*]]  {{.*}}Ei
+! CHECK:  %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3:[a-z0-9]*]]  {{.*}}Ej
+! CHECK:  %[[VAL_7:.*]] = fir.alloca i64 {bindc_name = "step", uniq_name = "_QFsubstringEstep"}
+! CHECK:  %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7:[a-z0-9]*]]  {{.*}}Estep
+! CHECK:  %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]]  {{.*}}Evector
+! CHECK:  %[[VAL_10:.*]] = arith.constant 0 : index
+! CHECK:  %[[VAL_11:.*]]:3 = fir.box_dims %[[VAL_9]]#0, %[[VAL_10]] : (!fir.box<!fir.array<?xi64>>, index) -> (index, index, index)
+! CHECK:  %[[VAL_12:.*]] = fir.shape %[[VAL_11]]#1 : (index) -> !fir.shape<1>
+! CHECK:  %[[VAL_13:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_14:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i64>
+! CHECK:  %[[VAL_15:.*]] = fir.convert %[[VAL_13]] : (i64) -> index
+! CHECK:  %[[VAL_16:.*]] = fir.convert %[[VAL_14]] : (i64) -> index
+! CHECK:  %[[VAL_17:.*]] = arith.constant 1 : index
+! CHECK:  %[[VAL_18:.*]] = arith.subi %[[VAL_16]], %[[VAL_15]] : index
+! CHECK:  %[[VAL_19:.*]] = arith.addi %[[VAL_18]], %[[VAL_17]] : index
+! CHECK:  %[[VAL_20:.*]] = arith.constant 0 : index
+! CHECK:  %[[VAL_21:.*]] = arith.cmpi sgt, %[[VAL_19]], %[[VAL_20]] : index
+! CHECK:  %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_19]], %[[VAL_20]] : index
+! CHECK:  %[[VAL_23:.*]] = hlfir.elemental %[[VAL_12]] typeparams %[[VAL_22]] : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,?>> {
+! CHECK:  ^bb0(%[[VAL_24:.*]]: index):
+! CHECK:    %[[VAL_25:.*]] = hlfir.designate %[[VAL_9]]#0 (%[[VAL_24]])  : (!fir.box<!fir.array<?xi64>>, index) -> !fir.ref<i64>
+! CHECK:    %[[VAL_26:.*]] = fir.load %[[VAL_25]] : !fir.ref<i64>
+! CHECK:    %[[VAL_27:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_26]]) substr %[[VAL_15]], %[[VAL_16]]  typeparams %[[VAL_22]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i64, index, index, index) -> !fir.boxchar<1>
+! CHECK:    hlfir.yield_element %[[VAL_27]] : !fir.boxchar<1>
+! CHECK:  }
diff --git a/flang/test/Lower/Intrinsics/achar.f90 b/flang/test/Lower/Intrinsics/achar.f90
index 31afaa90af671..db9fc9368cfad 100644
--- a/flang/test/Lower/Intrinsics/achar.f90
+++ b/flang/test/Lower/Intrinsics/achar.f90
@@ -1,21 +1,22 @@
 ! RUN: bbc -emit-fir %s -o - | fir-opt --canonicalize | FileCheck %s
 ! RUN: %flang_fc1 -emit-fir %s -o - | fir-opt --canonicalize | FileCheck %s
 
-! CHECK-LABEL: test1
-! CHECK-SAME: (%[[XREF:.*]]: !fir.ref<i32> {{.*}}, %[[CBOX:.*]]: !fir.boxchar<1> {{.*}})
-! CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
-! CHECK-DAG: %[[FALSE:.*]] = arith.constant false
-! CHECK: %[[TEMP:.*]] = fir.alloca !fir.char<1> {adapt.valuebyref}
-! CHECK: %[[C:.*]]:2 = fir.unboxchar %[[CBOX]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK: %[[X:.*]] = fir.load %[[XREF]] : !fir.ref<i32>
-! CHECK: %[[X_I8:.*]] = fir.convert %[[X]] : (i32) -> i8
-! CHECK: %[[UNDEF:.*]] = fir.undefined !fir.char<1>
-! CHECK: %[[XCHAR:.*]] = fir.insert_value %[[UNDEF]], %[[X_I8]], [0 : index] : (!fir.char<1>, i8) -> !fir.char<1>
-! CHECK: fir.store %[[XCHAR]] to %[[TEMP]] : !fir.ref<!fir.char<1>>
-! CHECK: %[[C1_I64:.*]] = fir.convert %[[C1]] : (index) -> i64
-! CHECK: %[[C_CVT:.*]] = fir.convert %[[C]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-! CHECK: %[[TEMP_WITH_XCHAR:.*]] = fir.convert %[[TEMP]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<i8>
-! CHECK: fir.call @llvm.memmove.p0.p0.i64(%[[C_CVT]], %[[TEMP_WITH_XCHAR]], %[[C1_I64]], %[[FALSE]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
+
+! CHECK-LABEL:   func.func @_QPtest1(
+! CHECK-SAME:        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "x"},
+! CHECK-SAME:        %[[VAL_1:.*]]: !fir.boxchar<1> {fir.bindc_name = "c"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.char<1> {adapt.valuebyref}
+! CHECK:           %[[VAL_3:.*]]:2 = fir.unboxchar %[[VAL_1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_0]] : !fir.ref<i32>
+! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i32) -> i8
+! CHECK:           %[[VAL_7:.*]] = fir.undefined !fir.char<1>
+! CHECK:           %[[VAL_8:.*]] = fir.insert_value %[[VAL_7]], %[[VAL_6]], [0 : index] : (!fir.char<1>, i8) -> !fir.char<1>
+! CHECK:           fir.store %[[VAL_8]] to %[[VAL_2]] : !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_2]] : !fir.ref<!fir.char<1>>
+! CHECK:           fir.store %[[VAL_9]] to %[[VAL_4]] : !fir.ref<!fir.char<1>>
+! CHECK:           return
+! CHECK:         }
 subroutine test1(x, c)
   integer :: x
   character :: c
diff --git a/flang/test/Lower/Intrinsics/get_command_argument.f90 b/flang/test/Lower/Intrinsics/get_command_argument.f90
index 60f0c2a4fae1e..4af448387044d 100644
--- a/flang/test/Lower/Intrinsics/get_command_argument.f90
+++ b/flang/test/Lower/Intrinsics/get_command_argument.f90
@@ -17,15 +17,15 @@ subroutine number_and_value_only(num, value)
 character(len=32) :: value
 call get_command_argument(num, value)
 ! CHECK: %[[valueUnboxed:.*]]:2 = fir.unboxchar %[[value]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[valueLength:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[valueCast:.*]] = fir.convert %[[valueUnboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[numUnbox:.*]] = fir.load %[[num]] : !fir.ref<i[[DEFAULT_INTEGER_SIZE]]>
-! CHECK-NEXT: %[[valueBoxed:.*]] = fir.embox %[[valueUnboxed]]#0 typeparams %[[valueLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[valueBoxed:.*]] = fir.embox %[[valueCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %[[length:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[errmsg:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[sourceFileString:.*]] = fir.address_of(@_QQcl{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 8]] : i32
 ! CHECK-64-NEXT: %[[numCast:.*]] = fir.convert %[[numUnbox]] : (i64) -> i32
-! CHECK-NEXT: %[[valueCast:.*]] = fir.convert %[[valueBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[valueCast:.*]] = fir.convert %[[valueBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-32-NEXT: %{{[0-9]+}} = fir.call @_FortranAGetCommandArgument(%[[numUnbox]], %[[valueCast]], %[[length]], %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (i32, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-64-NEXT: %{{[0-9]+}} = fir.call @_FortranAGetCommandArgument(%[[numCast]], %[[valueCast]], %[[length]], %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (i32, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -38,19 +38,19 @@ subroutine all_arguments(num, value, length, status, errmsg)
     character(len=32) :: value, errmsg
     call get_command_argument(num, value, length, status, errmsg)
 ! CHECK: %[[errmsgUnboxed:.*]]:2 = fir.unboxchar %[[errmsg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[errmsgLen:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[errmsgCast:.*]] = fir.convert %[[errmsgUnboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[valueUnboxed:.*]]:2 = fir.unboxchar %[[value]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[valueLen:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[valueCast:.*]] = fir.convert %[[valueUnboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[numUnboxed:.*]] = fir.load %[[num]] : !fir.ref<i[[DEFAULT_INTEGER_SIZE]]>
-! CHECK-NEXT: %[[valueBoxed:.*]] = fir.embox %[[valueUnboxed]]#0 typeparams %[[valueLen]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[valueBoxed:.*]] = fir.embox %[[valueCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %[[lengthBoxed:.*]] = fir.embox %[[length]] : (!fir.ref<i[[DEFAULT_INTEGER_SIZE]]>) -> !fir.box<i[[DEFAULT_INTEGER_SIZE]]>
-! CHECK-NEXT: %[[errmsgBoxed:.*]] = fir.embox %[[errmsgUnboxed]]#0 typeparams %[[errmsgLen]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[errmsgBoxed:.*]] = fir.embox %[[errmsgCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %[[sourceFileString:.*]] = fir.address_of(@_QQcl{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 10]] : i32
 ! CHECK-64-NEXT: %[[numCast:.*]] = fir.convert %[[numUnboxed]] : (i64) -> i32
-! CHECK-NEXT: %[[valueBuffer:.*]] = fir.convert %[[valueBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[valueBuffer:.*]] = fir.convert %[[valueBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[lengthBuffer:.*]] = fir.convert %[[lengthBoxed]] : (!fir.box<i[[DEFAULT_INTEGER_SIZE]]>) -> !fir.box<none>
-! CHECK-NEXT: %[[errmsgBuffer:.*]] = fir.convert %[[errmsgBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[errmsgBuffer:.*]] = fir.convert %[[errmsgBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-32-NEXT: %[[statusResult:.*]] = fir.call @_FortranAGetCommandArgument(%[[numUnboxed]], %[[valueBuffer]], %[[lengthBuffer]], %[[errmsgBuffer]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (i32, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-64-NEXT: %[[statusResult32:.*]] = fir.call @_FortranAGetCommandArgument(%[[numCast]], %[[valueBuffer]], %[[lengthBuffer]], %[[errmsgBuffer]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (i32, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -102,15 +102,15 @@ subroutine number_and_errmsg_only(num, errmsg)
     character(len=32) :: errmsg
     call get_command_argument(num, ERRMSG=errmsg)
 ! CHECK: %[[errmsgUnboxed:.*]]:2 = fir.unboxchar %[[errmsg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[errmsgLength:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[errmsgCast:.*]] = fir.convert %[[errmsgUnboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[numUnboxed:.*]] = fir.load %[[num]] : !fir.ref<i[[DEFAULT_INTEGER_SIZE]]>
-! CHECK-NEXT: %[[errmsgBoxed:.*]] = fir.embox %[[errmsgUnboxed]]#0 typeparams %[[errmsgLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[errmsgBoxed:.*]] = fir.embox %[[errmsgCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %[[value:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[length:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[sourceFileString:.*]] = fir.address_of(@_QQcl{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 8]] : i32
 ! CHECK-64: %[[numCast:.*]] = fir.convert %[[numUnboxed]] : (i64) -> i32
-! CHECK-NEXT: %[[errmsg:.*]] = fir.convert %[[errmsgBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[errmsg:.*]] = fir.convert %[[errmsgBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-32-NEXT: %{{[0-9]+}} = fir.call @_FortranAGetCommandArgument(%[[numUnboxed]], %[[value]], %[[length]], %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (i32, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-64-NEXT: %{{[0-9]+}} = fir.call @_FortranAGetCommandArgument(%[[numCast]], %[[value]], %[[length]], %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (i32, !fir.box<none>, !fir.box<none>, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/flang/test/Lower/Intrinsics/get_environment_variable.f90 b/flang/test/Lower/Intrinsics/get_environment_variable.f90
index b4cecbe742ff2..ad9b45da18e79 100644
--- a/flang/test/Lower/Intrinsics/get_environment_variable.f90
+++ b/flang/test/Lower/Intrinsics/get_environment_variable.f90
@@ -17,18 +17,18 @@ subroutine name_and_value_only(name, value)
     character(len=32) :: name, value
     call get_environment_variable(name, value)
 ! CHECK: %[[nameUnbox:.*]]:2 = fir.unboxchar %[[nameArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[nameLength:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[nameCast:.*]] = fir.convert %[[nameUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[valueUnbox:.*]]:2 = fir.unboxchar %[[valueArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[valueLength:.*]] = arith.constant 32 : index
-! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameUnbox]]#0 typeparams %[[nameLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
-! CHECK-NEXT: %[[valueBox:.*]] = fir.embox %[[valueUnbox]]#0 typeparams %[[valueLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[valueCast:.*]] = fir.convert %[[valueUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
+! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
+! CHECK-NEXT: %[[valueBox:.*]] = fir.embox %[[valueCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %true = arith.constant true
 ! CHECK-NEXT: %[[length:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[errmsg:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[sourceFileString:.*]] = fir.address_of(@_QQcl{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 11]] : i32
-! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
-! CHECK-NEXT: %[[value:.*]] = fir.convert %[[valueBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
+! CHECK-NEXT: %[[value:.*]] = fir.convert %[[valueBox]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-NEXT: %{{[0-9]+}} = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %true, %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-NEXT: return
@@ -42,15 +42,15 @@ subroutine name_and_length_only(name, length)
     integer :: length
     call get_environment_variable(name, LENGTH=length)
 ! CHECK: %[[nameUnbox:.*]]:2 = fir.unboxchar %[[nameArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[nameLength:.*]] = arith.constant 32 : index
-! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %0#0 typeparams %[[nameLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[nameCast:.*]] = fir.convert %[[nameUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
+! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %[[lengthBox:.*]] = fir.embox %arg1 : (!fir.ref<i[[DEFAULT_INTEGER_SIZE]]>) -> !fir.box<i[[DEFAULT_INTEGER_SIZE]]>
 ! CHECK-NEXT: %true = arith.constant true
 ! CHECK-NEXT: %[[value:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[errmsg:.*]] = fir.absent !fir.box<none>
 ! CHECK: %[[sourceFileString:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 9]] : i32
-! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[length:.*]] = fir.convert %[[lengthBox]] : (!fir.box<i[[DEFAULT_INTEGER_SIZE]]>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-NEXT: %{{.*}} = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %true, %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -64,15 +64,15 @@ subroutine name_and_status_only(name, status)
     integer :: status
     call get_environment_variable(name, STATUS=status)
 ! CHECK: %[[nameUnbox:.*]]:2 = fir.unboxchar %[[nameArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[nameLength:.*]] = arith.constant 32 : index
-! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameUnbox]]#0 typeparams %[[nameLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[nameCast:.*]] = fir.convert %[[nameUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
+! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %true = arith.constant true
 ! CHECK-NEXT: %[[value:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[length:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[errmsg:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[sourceFileString:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 9]] : i32
-! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-32-NEXT: %[[status:.*]] = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %true, %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-64-NEXT: %[[status32:.*]] = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %true, %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
@@ -98,18 +98,18 @@ subroutine name_and_errmsg_only(name, errmsg)
     character(len=32) :: name, errmsg
     call get_environment_variable(name, ERRMSG=errmsg)
 ! CHECK: %[[errmsgUnbox:.*]]:2 = fir.unboxchar %[[errmsgArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[errmsgLength:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[errmsgCast:.*]] = fir.convert %[[errmsgUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[nameUnbox:.*]]:2 = fir.unboxchar %[[nameArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[nameLength:.*]] = arith.constant 32 : index
-! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameUnbox]]#0 typeparams %[[nameLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
-! CHECK-NEXT: %[[errmsgBox:.*]] = fir.embox %[[errmsgUnbox]]#0 typeparams %c32 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[nameCast:.*]] = fir.convert %[[nameUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
+! CHECK-NEXT: %[[nameBox:.*]] = fir.embox %[[nameCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
+! CHECK-NEXT: %[[errmsgBox:.*]] = fir.embox %[[errmsgCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %true = arith.constant true
 ! CHECK-NEXT: %[[value:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[length:.*]] = fir.absent !fir.box<none>
 ! CHECK-NEXT: %[[sourceFileString:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,[[sourceFileLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 11]] : i32
-! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
-! CHECK-NEXT: %[[errmsg:.*]] = fir.convert %[[errmsgBox]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBox]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
+! CHECK-NEXT: %[[errmsg:.*]] = fir.convert %[[errmsgBox]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[sourceFileLength]]>>) -> !fir.ref<i8>
 ! CHECK-NEXT: %{{[0-9]+}} = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %true, %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-NEXT: return
@@ -128,15 +128,15 @@ subroutine all_arguments(name, value, length, status, trim_name, errmsg)
     logical :: trim_name
     call get_environment_variable(name, value, length, status, trim_name, errmsg)
 ! CHECK: %[[errmsgUnbox:.*]]:2 = fir.unboxchar %[[errmsgArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[errmsgLength:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[errmsgCast:.*]] = fir.convert %[[errmsgUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[nameUnbox:.*]]:2 = fir.unboxchar %[[nameArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[nameLength:.*]] = arith.constant 32 : index
+! CHECK-NEXT: %[[nameCast:.*]] = fir.convert %[[nameUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
 ! CHECK-NEXT: %[[valueUnbox:.*]]:2 = fir.unboxchar %[[valueArg]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-NEXT: %[[valueLength:.*]] = arith.constant 32 : index
-! CHECK-NEXT: %[[nameBoxed:.*]] = fir.embox %[[nameUnbox]]#0 typeparams %[[nameLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
-! CHECK-NEXT: %[[valueBoxed:.*]] = fir.embox %[[valueUnbox]]#0 typeparams %[[valueLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[valueCast:.*]] = fir.convert %[[valueUnbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,32>>
+! CHECK-NEXT: %[[nameBoxed:.*]] = fir.embox %[[nameCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
+! CHECK-NEXT: %[[valueBoxed:.*]] = fir.embox %[[valueCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK-NEXT: %[[lengthBoxed:.*]] = fir.embox %[[lengthArg]] : (!fir.ref<i[[DEFAULT_INTEGER_SIZE]]>) -> !fir.box<i[[DEFAULT_INTEGER_SIZE]]>
-! CHECK-NEXT: %[[errmsgBoxed:.*]] = fir.embox %[[errmsgUnbox]]#0 typeparams %[[errmsgLength]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.box<!fir.char<1,?>>
+! CHECK-NEXT: %[[errmsgBoxed:.*]] = fir.embox %[[errmsgCast]] : (!fir.ref<!fir.char<1,32>>) -> !fir.box<!fir.char<1,32>>
 ! CHECK:      %[[trimName:.*]] = fir.if %{{.*}} -> (i1) {
 ! CHECK-NEXT:   %[[trimNameLoaded:.*]] = fir.load %[[trimNameArg]] : !fir.ref<!fir.logical<4>>
 ! CHECK-NEXT:   %[[trimCast:.*]] = fir.convert %[[trimNameLoaded]] : (!fir.logical<4>) -> i1
@@ -147,10 +147,10 @@ subroutine all_arguments(name, value, length, status, trim_name, errmsg)
 ! CHECK-NEXT: }
 ! CHECK: %[[sourceFileString:.*]] = fir.address_of(@_QQcl.[[fileString:.*]]) : !fir.ref<!fir.char<1,[[fileStringLength:.*]]>>
 ! CHECK-NEXT: %[[sourceLine:.*]] = arith.constant [[# @LINE - 20]] : i32
-! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
-! CHECK-NEXT: %[[value:.*]] = fir.convert %[[valueBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[name:.*]] = fir.convert %[[nameBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
+! CHECK-NEXT: %[[value:.*]] = fir.convert %[[valueBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[length:.*]] = fir.convert %[[lengthBoxed]] : (!fir.box<i[[DEFAULT_INTEGER_SIZE]]>) -> !fir.box<none>
-! CHECK-NEXT: %[[errmsg:.*]] = fir.convert %[[errmsgBoxed]] : (!fir.box<!fir.char<1,?>>) -> !fir.box<none>
+! CHECK-NEXT: %[[errmsg:.*]] = fir.convert %[[errmsgBoxed]] : (!fir.box<!fir.char<1,32>>) -> !fir.box<none>
 ! CHECK-NEXT: %[[sourceFile:.*]] = fir.convert %[[sourceFileString]] : (!fir.ref<!fir.char<1,[[fileStringLength]]>>) -> !fir.ref<i8>
 ! CHECK-32-NEXT: %[[status:.*]] = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %[[trimName]], %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
 ! CHECK-64-NEXT: %[[status32:.*]] = fir.call @_FortranAGetEnvVariable(%[[name]], %[[value]], %[[length]], %[[trimName]], %[[errmsg]], %[[sourceFile]], %[[sourceLine]]) {{.*}}: (!fir.box<none>, !fir.box<none>, !fir.box<none>, i1, !fir.box<none>, !fir.ref<i8>, i32) -> i32
diff --git a/flang/test/Lower/Intrinsics/ichar.f90 b/flang/test/Lower/Intrinsics/ichar.f90
index e99fae0f5b193..fa0cdc56ef588 100644
--- a/flang/test/Lower/Intrinsics/ichar.f90
+++ b/flang/test/Lower/Intrinsics/ichar.f90
@@ -6,9 +6,9 @@ subroutine ichar_test(c)
   character(1) :: c
   character :: str(10)
   ! CHECK-DAG: %[[unbox:.*]]:2 = fir.unboxchar
+  ! CHECK-DAG: %[[BOX:.*]] = fir.convert %[[unbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
   ! CHECK-DAG: %[[J:.*]] = fir.alloca i32 {{{.*}}uniq_name = "{{.*}}Ej"}
   ! CHECK-DAG: %[[STR:.*]] = fir.alloca !fir.array{{.*}} {{{.*}}uniq_name = "{{.*}}Estr"}
-  ! CHECK: %[[BOX:.*]] = fir.convert %[[unbox]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
   ! CHECK: %[[PTR:.*]] = fir.load %[[BOX]] : !fir.ref<!fir.char<1>>
   ! CHECK: %[[CHAR:.*]] = fir.extract_value %[[PTR]], [0 : index] :
   ! CHECK: %[[ARG:.*]] = arith.extui %[[CHAR]] : i8 to i32
diff --git a/flang/test/Lower/Intrinsics/merge.f90 b/flang/test/Lower/Intrinsics/merge.f90
index cfd224ef0a2f7..0ceb670f20fbd 100644
--- a/flang/test/Lower/Intrinsics/merge.f90
+++ b/flang/test/Lower/Intrinsics/merge.f90
@@ -8,11 +8,13 @@ function merge_test(o1, o2, mask)
 logical :: mask
 merge_test = merge(o1, o2, mask)
 ! CHECK:  %[[a0:.*]]:2 = fir.unboxchar %[[arg2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK-DAG:  %[[a1:.*]]:2 = fir.unboxchar %[[arg3]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index) 
+! CHECK: %[[a0_cast:.*]] = fir.convert %[[a0]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+! CHECK:  %[[a1:.*]]:2 = fir.unboxchar %[[arg3]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index) 
+! CHECK: %[[a1_cast:.*]] = fir.convert %[[a1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
 ! CHECK: %[[a2:.*]] = fir.load %[[arg4]] : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[a3:.*]] = fir.convert %[[a2]] : (!fir.logical<4>) -> i1
-! CHECK: %[[a4:.*]] = arith.select %[[a3]], %[[a0]]#0, %[[a1]]#0 : !fir.ref<!fir.char<1,?>>
-! CHECK-DAG:  %{{.*}} = fir.convert %[[a4]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! CHECK: %[[a4:.*]] = arith.select %[[a3]], %[[a0_cast]], %[[a1_cast]] : !fir.ref<!fir.char<1>>
+! CHECK:  %{{.*}} = fir.convert %[[a4]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<i8>
 end
 
 ! CHECK-LABEL: func @_QPmerge_test2(
diff --git a/flang/test/Lower/OpenACC/acc-data-operands.f90 b/flang/test/Lower/OpenACC/acc-data-operands.f90
index a756d61661124..2f6838e94aa81 100644
--- a/flang/test/Lower/OpenACC/acc-data-operands.f90
+++ b/flang/test/Lower/OpenACC/acc-data-operands.f90
@@ -1,6 +1,6 @@
 ! This test checks lowering of complex OpenACC data operands.
 
-! RUN: bbc --use-desc-for-alloc=false -fopenacc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s
 
 module acc_data_operand
 
@@ -16,39 +16,25 @@ subroutine acc_operand_array_section()
 
   !$acc data copyin(a(1:50)) copyout(a(51:100))
   !$acc end data
-
-  !CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section
-
-  !CHECK: %[[ARR:.*]] = fir.alloca !fir.array<100xf32>
-
-  !CHECK: %[[C1:.*]] = arith.constant 1 : i64
-  !CHECK: %[[LB1:.*]] = fir.convert %[[C1]] : (i64) -> index
-  !CHECK: %[[C1_I64:.*]] = arith.constant 1 : i64
-  !CHECK: %[[STEP1:.*]] = fir.convert %[[C1_I64]] : (i64) -> index
-  !CHECK: %[[C50_I64:.*]] = arith.constant 50 : i64
-  !CHECK: %[[UB1:.*]] = fir.convert %[[C50_I64]] : (i64) -> index
-  !CHECK: %[[SHAPE1:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-  !CHECK: %[[SLICE1:.*]] = fir.slice %[[LB1]], %[[UB1]], %[[STEP1]] : (index, index, index) -> !fir.slice<1>
-  !CHECK: %[[ARR_SECTION1:.*]] = fir.embox %[[ARR]](%[[SHAPE1]]) [%[[SLICE1]]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
-  !CHECK: %[[MEM1:.*]] = fir.alloca !fir.box<!fir.array<50xf32>>
-  !CHECK: fir.store %[[ARR_SECTION1]] to %[[MEM1]] : !fir.ref<!fir.box<!fir.array<50xf32>>>
-
-  !CHECK: %[[C51_I64:.*]] = arith.constant 51 : i64
-  !CHECK: %[[LB2:.*]] = fir.convert %[[C51_I64]] : (i64) -> index
-  !CHECK: %[[C1_I64:.*]] = arith.constant 1 : i64
-  !CHECK: %[[STEP2:.*]] = fir.convert %[[C1_I64]] : (i64) -> index
-  !CHECK: %[[C100_I64:.*]] = arith.constant 100 : i64
-  !CHECK: %[[UB2:.*]] = fir.convert %[[C100_I64]] : (i64) -> index
-  !CHECK: %[[SHAPE2:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-  !CHECK: %[[SLICE2:.*]] = fir.slice %[[LB2]], %[[UB2]], %[[STEP2]] : (index, index, index) -> !fir.slice<1>
-  !CHECK: %[[ARR_SECTION2:.*]] = fir.embox %[[ARR]](%[[SHAPE2]]) [%[[SLICE2]]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
-  !CHECK: %[[MEM2:.*]] = fir.alloca !fir.box<!fir.array<50xf32>>
-  !CHECK: fir.store %[[ARR_SECTION2]] to %[[MEM2]] : !fir.ref<!fir.box<!fir.array<50xf32>>>
-
-  !CHECK: acc.data copyin(%[[MEM1]] : !fir.ref<!fir.box<!fir.array<50xf32>>>) copyout(%[[MEM2]] : !fir.ref<!fir.box<!fir.array<50xf32>>>)
-
 end subroutine
 
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section
+! CHECK: %[[ARR:.*]] = fir.alloca !fir.array<100xf32>
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.constant 49 : index
+! CHECK: %[[BOUND_1_50:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[ARR]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND_1_50]]) -> !fir.ref<!fir.array<100xf32>> {name = "a(1:50)"}
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 50 : index
+! CHECK: %[[UB:.*]] = arith.constant 99 : index
+! CHECK: %[[BOUND_51_100:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+! CHECK: %[[COPYOUT_CREATE:.*]] = acc.create varPtr(%[[ARR]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND_51_100]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = 4 : i64, name = "a(51:100)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]], %[[COPYOUT_CREATE]] : !fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPYOUT_CREATE]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND_51_100]]) to varPtr(%[[ARR]] : !fir.ref<!fir.array<100xf32>>) {name = "a(51:100)"}
+    
 ! Testing array sections of a derived-type component
 subroutine acc_operand_array_section_component()
 
@@ -56,40 +42,46 @@ subroutine acc_operand_array_section_component()
 
   !$acc data copy(w%data(1:20))
   !$acc end data
-
-  !CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_component
-  !CHECK: %[[W:.*]] = fir.alloca !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
-  !CHECK: %[[FIELD_INDEX:.*]] = fir.field_index data, !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
-  !CHECK: %[[DATA_COORD:.*]] = fir.coordinate_of %[[W]], %[[FIELD_INDEX]] : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.field) -> !fir.ref<!fir.array<100xf32>>
-  !CHECK: %[[C1_I64_1:.*]] = arith.constant 1 : i64
-  !CHECK: %[[LB:.*]] = fir.convert %[[C1_I64_1]] : (i64) -> index
-  !CHECK: %[[C1_I64_2:.*]] = arith.constant 1 : i64
-  !CHECK: %[[STEP:.*]] = fir.convert %[[C1_I64_2]] : (i64) -> index
-  !CHECK: %[[C20_I64:.*]] = arith.constant 20 : i64
-  !CHECK: %[[UB:.*]] = fir.convert %[[C20_I64]] : (i64) -> index
-  !CHECK: %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-  !CHECK: %[[SLICE:.*]] = fir.slice %[[LB]], %[[UB]], %[[STEP]] : (index, index, index) -> !fir.slice<1>
-  !CHECK: %[[ARR_SECTION:.*]] = fir.embox %[[DATA_COORD]](%[[SHAPE]]) [%[[SLICE]]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>, !fir.slice<1>) -> !fir.box<!fir.array<20xf32>>
-  !CHECK: %[[MEM:.*]] = fir.alloca !fir.box<!fir.array<20xf32>> 
-  !CHECK: fir.store %[[ARR_SECTION]] to %[[MEM]] : !fir.ref<!fir.box<!fir.array<20xf32>>> 
-  !CHECK: acc.data copy(%[[MEM]] : !fir.ref<!fir.box<!fir.array<20xf32>>>)
-
 end subroutine
 
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_component() {
+! CHECK: %[[W:.*]] = fir.alloca !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}> {bindc_name = "w", uniq_name = "_QMacc_data_operandFacc_operand_array_section_componentEw"}
+! CHECK: %[[FIELD_DATA:.*]] = fir.field_index data, !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
+! CHECK: %[[COORD_DATA:.*]] = fir.coordinate_of %[[W]], %[[FIELD_DATA]] : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.field) -> !fir.ref<!fir.array<100xf32>>
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.constant 19 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = 3 : i64, name = "w%data(1:20)"}
+! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = 3 : i64, name = "w%data(1:20)"}
+
 ! Testing derived-type component without section
 subroutine acc_operand_derived_type_component()
   type(wrapper) :: w
 
   !$acc data copy(w%data)
   !$acc end data
+end subroutine
 
-  !CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_derived_type_component
-  !CHECK: %[[W:.*]] = fir.alloca !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
-  !CHECK: %[[FIELD_INDEX:.*]] = fir.field_index data, !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
-  !CHECK: %[[DATA_COORD:.*]] = fir.coordinate_of %[[W]], %[[FIELD_INDEX]] : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.field) -> !fir.ref<!fir.array<100xf32>>
-  !CHECK: acc.data copy(%[[DATA_COORD]] : !fir.ref<!fir.array<100xf32>>) {
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_derived_type_component() {
+! CHECK: %[[W:.*]] = fir.alloca !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}> {bindc_name = "w", uniq_name = "_QMacc_data_operandFacc_operand_derived_type_componentEw"}
+! CHECK: %[[FIELD_DATA:.*]] = fir.field_index data, !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
+! CHECK: %[[COORD_DATA:.*]] = fir.coordinate_of %[[W]], %[[FIELD_DATA]] : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10
+! CHECK: %[[EXT:.*]] = arith.constant 100 : index
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[LBEXT:.*]] = arith.addi %[[EXT]], %[[ONE]] : index
+! CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = 3 : i64, name = "w%data"}
+! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = 3 : i64, name = "w%data"}
 
-end subroutine
 
 ! Testing array of derived-type component without section
 subroutine acc_operand_array_derived_type_component()
@@ -97,19 +89,28 @@ subroutine acc_operand_array_derived_type_component()
 
   !$acc data copy(w(1)%data)
   !$acc end data
-
-  !CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_derived_type_component
-  !CHECK: %[[W:.*]] = fir.alloca !fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>
-  !CHECK: %[[C1_I64_1:.*]] = arith.constant 1 : i64
-  !CHECK: %[[C1_I64_2:.*]] = arith.constant 1 : i64
-  !CHECK: %[[IDX:.*]] = arith.subi %[[C1_I64_1]], %[[C1_I64_2]] : i64
-  !CHECK: %[[COORD1:.*]] = fir.coordinate_of %[[W]], %[[IDX]] : (!fir.ref<!fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>>, i64) -> !fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>
-  !CHECK: %[[COORD2:.*]] = fir.field_index data, !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
-  !CHECK: %[[COORD_OF:.*]] = fir.coordinate_of %[[COORD1]], %[[COORD2]] : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.field) -> !fir.ref<!fir.array<100xf32>>
-  !CHECK: acc.data copy(%[[COORD_OF]] : !fir.ref<!fir.array<100xf32>>)
-
 end subroutine
 
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_derived_type_component() {
+! CHECK: %[[W:.*]] = fir.alloca !fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>> {bindc_name = "w", uniq_name = "_QMacc_data_operandFacc_operand_array_derived_type_componentEw"}
+! CHECK: %[[C1:.*]] = arith.constant 1 : i64
+! CHECK: %[[SUB:.*]] = arith.constant 1 : i64
+! CHECK: %[[IDX:.*]] = arith.subi %[[C1]], %[[SUB]] : i64
+! CHECK: %[[W_1:.*]] = fir.coordinate_of %[[W]], %[[IDX]] : (!fir.ref<!fir.array<10x!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>>, i64) -> !fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>
+! CHECK: %[[FIELD_DATA:.*]] = fir.field_index data, !fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>
+! CHECK: %[[COORD_W1_DATA:.*]] = fir.coordinate_of %[[W_1]], %[[FIELD_DATA]] : (!fir.ref<!fir.type<_QMacc_data_operandTwrapper{data:!fir.array<100xf32>}>>, !fir.field) -> !fir.ref<!fir.array<100xf32>>
+! CHECK: %[[EXT:.*]] = arith.constant 100 : index
+! CHECK: %[[ONE:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[LBEXT:.*]] = arith.addi %[[EXT]], %[[ONE]] : index
+! CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index) 
+! CHECK: %[[COPY_COPYIN:.*]] = acc.copyin varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {dataClause = 3 : i64, name = "w(1_8)%data"}
+! CHECK: acc.data dataOperands(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPY_COPYIN]] : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) to varPtr(%[[COORD_W1_DATA]] : !fir.ref<!fir.array<100xf32>>) {dataClause = 3 : i64, name = "w(1_8)%data"}
+
 ! Testing array sections on allocatable array
 subroutine acc_operand_array_section_allocatable()
   real, allocatable :: a(:)
@@ -119,39 +120,42 @@ subroutine acc_operand_array_section_allocatable()
   !$acc data copyin(a(1:50)) copyout(a(51:100))
   !$acc end data
 
-  !CHECK: %[[ARR_HEAP:.*]] = fir.alloca !fir.heap<!fir.array<?xf32>> {uniq_name = "_QMacc_data_operandFacc_operand_array_section_allocatableEa.addr"}
-
-  !CHECK: %[[LOAD_ARR0:.*]] = fir.load %[[ARR_HEAP]] : !fir.ref<!fir.heap<!fir.array<?xf32>>>
-  !CHECK: %[[C1_I64:.*]] = arith.constant 1 : i64
-  !CHECK: %[[LB0:.*]] = fir.convert %[[C1_I64]] : (i64) -> index
-  !CHECK: %[[C1_STEP:.*]] = arith.constant 1 : i64
-  !CHECK: %[[STEP0:.*]] = fir.convert %[[C1_STEP]] : (i64) -> index
-  !CHECK: %[[C50_I64:.*]] = arith.constant 50 : i64
-  !CHECK: %[[UB0:.*]] = fir.convert %[[C50_I64]] : (i64) -> index
-  !CHECK: %[[SHAPE_SHIFT0:.*]] = fir.shape_shift %{{.*}}, %{{.*}} : (index, index) -> !fir.shapeshift<1>
-  !CHECK: %[[SLICE0:.*]] = fir.slice %[[LB0]], %[[UB0]], %[[STEP0]] : (index, index, index) -> !fir.slice<1>
-  !CHECK: %[[ARR_SECTION0:.*]] = fir.embox %[[LOAD_ARR0]](%[[SHAPE_SHIFT0]]) [%[[SLICE0]]] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
-  !CHECK: %[[MEM0:.*]] = fir.alloca !fir.box<!fir.array<50xf32>>
-  !CHECK: fir.store %[[ARR_SECTION0]] to %[[MEM0]] : !fir.ref<!fir.box<!fir.array<50xf32>>>
-
-  !CHECK: %[[LOAD_ARR1:.*]] = fir.load %[[ARR_HEAP]] : !fir.ref<!fir.heap<!fir.array<?xf32>>>
-  !CHECK: %[[C51_I64:.*]] = arith.constant 51 : i64
-  !CHECK: %[[LB1:.*]] = fir.convert %[[C51_I64]] : (i64) -> index
-  !CHECK: %[[C1_STEP:.*]] = arith.constant 1 : i64
-  !CHECK: %[[STEP1:.*]] = fir.convert %[[C1_STEP]] : (i64) -> index
-  !CHECK: %[[C100_I64:.*]] = arith.constant 100 : i64
-  !CHECK: %[[UB1:.*]] = fir.convert %[[C100_I64]] : (i64) -> index
-  !CHECK: %[[SHAPE_SHIFT1:.*]] = fir.shape_shift %{{.*}}, %{{.*}} : (index, index) -> !fir.shapeshift<1>
-  !CHECK: %[[SLICE1:.*]] = fir.slice %[[LB1]], %[[UB1]], %[[STEP1]] : (index, index, index) -> !fir.slice<1>
-  !CHECK: %[[ARR_SECTION1:.*]] = fir.embox %[[LOAD_ARR1]](%[[SHAPE_SHIFT1]]) [%[[SLICE1]]] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
-  !CHECK: %[[MEM1:.*]] = fir.alloca !fir.box<!fir.array<50xf32>>
-  !CHECK: fir.store %[[ARR_SECTION1]] to %[[MEM1]] : !fir.ref<!fir.box<!fir.array<50xf32>>>
-
-  !CHECK: acc.data copyin(%[[MEM0]] : !fir.ref<!fir.box<!fir.array<50xf32>>>) copyout(%[[MEM1]] : !fir.ref<!fir.box<!fir.array<50xf32>>>)
-
   deallocate(a)
 end subroutine
 
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_allocatable() {
+! CHECK: %[[A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QMacc_data_operandFacc_operand_array_section_allocatableEa"}
+! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[LOAD_BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.subi %[[C1]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[C50:.*]] = arith.constant 50 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C50]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(1:50)"}
+! CHECK: %[[LOAD_BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[LOAD_BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C51:.*]] = arith.constant 51 : index
+! CHECK: %[[LB:.*]] = arith.subi %[[C51]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[C100:.*]] = arith.constant 100 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C100]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+! CHECK: %[[COPYOUT_CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {dataClause = 4 : i64, name = "a(51:100)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]], %[[COPYOUT_CREATE]] : !fir.heap<!fir.array<?xf32>>, !fir.heap<!fir.array<?xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
+! CHECK: acc.copyout accPtr(%[[COPYOUT_CREATE]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) to varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) {name = "a(51:100)"}
+
 
 ! Testing array sections on pointer array
 subroutine acc_operand_array_section_pointer()
@@ -162,30 +166,26 @@ subroutine acc_operand_array_section_pointer()
 
   !$acc data copyin(p(1:50))
   !$acc end data
+end subroutine
 
-  !CHECK: %[[C100:.*]] = arith.constant 100 : index
-  !CHECK: %[[ARR:.*]] = fir.alloca !fir.array<100xf32> {bindc_name = "a", fir.target, uniq_name = "_QMacc_data_operandFacc_operand_array_section_pointerEa"}
-  !CHECK: %[[PTR:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>> {bindc_name = "p", uniq_name = "_QMacc_data_operandFacc_operand_array_section_pointerEp"}
-  !CHECK: %[[SHAPE0:.*]] = fir.shape %[[C100]] : (index) -> !fir.shape<1>
-  !CHECK: %[[EMBOX0:.*]] = fir.embox %[[ARR]](%[[SHAPE0]]) : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<?xf32>>>
-  !CHECK: fir.store %[[EMBOX0]] to %[[PTR]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
-  !CHECK: %[[PTR_LOAD:.*]] = fir.load %[[PTR]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
-  !CHECK: %[[C0:.*]] = arith.constant 0 : index
-  !CHECK: %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[PTR_LOAD]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
-  !CHECK: %[[C1_I64:.*]] = arith.constant 1 : i64
-  !CHECK: %[[LB0:.*]] = fir.convert %[[C1_I64]] : (i64) -> index
-  !CHECK: %[[C1_STEP:.*]] = arith.constant 1 : i64
-  !CHECK: %[[STEP0:.*]] = fir.convert %[[C1_STEP]] : (i64) -> index
-  !CHECK: %[[C50_I64:.*]] = arith.constant 50 : i64
-  !CHECK: %[[UB0:.*]] = fir.convert %[[C50_I64]] : (i64) -> index
-  !CHECK: %[[SHIFT0:.*]] = fir.shift %[[BOX_DIMS]]#0 : (index) -> !fir.shift<1>
-  !CHECK: %[[SLICE0:.*]] = fir.slice %[[LB0]], %[[UB0]], %[[STEP0]] : (index, index, index) -> !fir.slice<1>
-  !CHECK: %[[REBOX0:.*]] = fir.rebox %7(%[[SHIFT0]]) [%[[SLICE0]]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, !fir.shift<1>, !fir.slice<1>) -> !fir.box<!fir.array<50xf32>>
-  !CHECK: %[[MEM0:.*]] = fir.alloca !fir.box<!fir.array<50xf32>>
-  !CHECK: fir.store %[[REBOX0]] to %[[MEM0]] : !fir.ref<!fir.box<!fir.array<50xf32>>>
-  
-  !CHECK: acc.data copyin(%[[MEM0]] : !fir.ref<!fir.box<!fir.array<50xf32>>>) {
+! CHECK-LABEL: func.func @_QMacc_data_operandPacc_operand_array_section_pointer() {
+! CHECK: %[[P:.*]] = fir.alloca !fir.box<!fir.ptr<!fir.array<?xf32>>> {bindc_name = "p", uniq_name = "_QMacc_data_operandFacc_operand_array_section_pointerEp"}
+! CHECK: %[[LOAD_BOX_P_0:.*]] = fir.load %[[P]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: %[[LOAD_BOX_P_1:.*]] = fir.load %[[P]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_0:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_1]], %[[C0:.*]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[LOAD_BOX_P_0]], %[[C0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>, index) -> (index, index, index)
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[LB:.*]] = arith.subi %[[C1]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[C50:.*]] = arith.constant 50 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C50]], %[[DIMS0_0]]#0 : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0_0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD_BOX_P_0]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
+! CHECK: %[[COPYIN:.*]] = acc.copyin varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "p(1:50)"}
+! CHECK: acc.data dataOperands(%[[COPYIN]] : !fir.ptr<!fir.array<?xf32>>) {
+! CHECK:   acc.terminator
+! CHECK: }
 
-end subroutine
 
 end module
diff --git a/flang/test/Lower/OpenACC/acc-data.f90 b/flang/test/Lower/OpenACC/acc-data.f90
index 1716ac99629e4..c8f44affea37a 100644
--- a/flang/test/Lower/OpenACC/acc-data.f90
+++ b/flang/test/Lower/OpenACC/acc-data.f90
@@ -7,91 +7,136 @@ subroutine acc_data
   real, pointer :: d, e
   logical :: ifCondition = .TRUE.
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[D:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
-!CHECK: [[E:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+! CHECK: %[[E:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
 
   !$acc data if(.TRUE.) copy(a)
   !$acc end data
 
-!CHECK:      [[IF1:%.*]] = arith.constant true
-!CHECK:      acc.data if([[IF1]]) copy([[A]] : !fir.ref<!fir.array<10x10xf32>>)  {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[IF1:.*]] = arith.constant true
+! CHECK:      %[[COPYIN:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.data if(%[[IF1]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>)  {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
 
   !$acc data copy(a) if(ifCondition)
   !$acc end data
 
-!CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
-!CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-!CHECK:      acc.data if([[IF2]]) copy([[A]] : !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"}
+! CHECK:      %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+! CHECK:      %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK:      acc.data if(%[[IF2]]) dataOperands(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[COPYIN]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
 
   !$acc data copy(a, b, c)
   !$acc end data
 
-!CHECK:      acc.data copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"}
+! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK: acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+! CHECK: acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc data copy(a) copy(b) copy(c)
   !$acc end data
 
-!CHECK:      acc.data copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"}
+! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK: acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+! CHECK: acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc data copyin(a) copyin(readonly: b, c)
   !$acc end data
 
-!CHECK:      acc.data copyin([[A]] : !fir.ref<!fir.array<10x10xf32>>) copyin_readonly([[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "b"}
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "c"}
+! CHECK:      acc.data dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc data copyout(a) copyout(zero: b) copyout(c)
   !$acc end data
 
-!CHECK:      acc.data copyout([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 5 : i64, name = "b"}
+! CHECK:      %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "c"}
+! CHECK:      acc.data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 5 : i64, name = "b"}
+! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc data create(a, b) create(zero: c)
   !$acc end data
 
-!CHECK:      acc.data create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+! CHECK:      acc.data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 8 : i64, name = "c"}
+
 
   !$acc data no_create(a, b) create(zero: c)
   !$acc end data
 
-!CHECK:      acc.data create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) no_create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+! CHECK:      acc.data dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 8 : i64, name = "c"}
 
   !$acc data present(a, b, c)
   !$acc end data
 
-!CHECK:      acc.data present([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK:      %[[PRESENT_C:.*]] = acc.present varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.data dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc data deviceptr(b, c)
   !$acc end data
 
-!CHECK:      acc.data deviceptr([[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"} 
+! CHECK:      %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"} 
+! CHECK:      acc.data dataOperands(%[[DEVICEPTR_B]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc data attach(d, e)
   !$acc end data
 
-!CHECK:      acc.data attach([[D]], [[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%{{.*}} : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"}
+! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%{{.*}} : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"}
+! CHECK:      acc.data dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = 10 : i64, name = "d"}
+! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = 10 : i64, name = "e"}
 
 end subroutine acc_data
 
diff --git a/flang/test/Lower/OpenACC/acc-enter-data.f90 b/flang/test/Lower/OpenACC/acc-enter-data.f90
index a618d5eff1a3b..1aa7f579945a2 100644
--- a/flang/test/Lower/OpenACC/acc-enter-data.f90
+++ b/flang/test/Lower/OpenACC/acc-enter-data.f90
@@ -8,6 +8,7 @@ subroutine acc_enter_data
   real, pointer :: d
   logical :: ifCondition = .TRUE.
 
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
 !CHECK: %[[EXTENT_C10:.*]] = arith.constant 10 : index
 !CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
 !CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
@@ -15,133 +16,251 @@ subroutine acc_enter_data
 !CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
 
   !$acc enter data create(a)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) if(.true.)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: [[IF1:%.*]] = arith.constant true
 !CHECK: acc.enter_data if([[IF1]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) if(ifCondition)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXTENT_C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 !CHECK: [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
 !CHECK: acc.enter_data if([[IF2]]) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) create(b) create(c)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
-!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
-!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "c", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data create(a) create(b) create(zero: c)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
-!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
-!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
 
   !$acc enter data copyin(a) create(b) attach(d)
-!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
-!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
-!CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>)   -> !fir.ref<!fir.box<!fir.ptr<f32>>> {name = "d", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]], %[[CREATE_B]], %[[ATTACH_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.box<!fir.ptr<f32>>>){{$}}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]])  -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "b", structured = false}
+!CHECK: %[[BOX_D:.*]] = fir.load %[[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+!CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+!CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]], %[[CREATE_B]], %[[ATTACH_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<f32>){{$}}
 
   !$acc enter data create(a) async
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
 
   !$acc enter data create(a) wait
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
 
   !$acc enter data create(a) async wait
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
 
   !$acc enter data create(a) async(1)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[ASYNC1:.*]] = arith.constant 1 : i32
 !CHECK: acc.enter_data async(%[[ASYNC1]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) async(async)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[ASYNC2:.*]] = fir.load %{{.*}} : !fir.ref<i32>
 !CHECK: acc.enter_data async(%[[ASYNC2]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) wait(1)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[WAIT1:.*]] = arith.constant 1 : i32
 !CHECK: acc.enter_data wait(%[[WAIT1]] : i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) wait(queues: 1, 2)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[WAIT2:.*]] = arith.constant 1 : i32
 !CHECK: %[[WAIT3:.*]] = arith.constant 2 : i32
 !CHECK: acc.enter_data wait(%[[WAIT2]], %[[WAIT3]] : i32, i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data create(a) wait(devnum: 1: queues: 1, 2)
-!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a", structured = false}
 !CHECK: %[[WAIT4:.*]] = arith.constant 1 : i32
 !CHECK: %[[WAIT5:.*]] = arith.constant 2 : i32
 !CHECK: %[[WAIT6:.*]] = arith.constant 1 : i32
 !CHECK: acc.enter_data wait_devnum(%[[WAIT6]] : i32) wait(%[[WAIT4]], %[[WAIT5]] : i32, i32) dataOperands(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data copyin(a(1:10,1:5))
-!CHECK: %[[LB1:.*]] = arith.constant 0 : index
-!CHECK: %[[UB1:.*]] = arith.constant 9 : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) startIdx(%c1{{.*}} : index)
-!CHECK: %[[LB2:.*]] = arith.constant 0 : index
-!CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) startIdx(%c1{{.*}} : index) 
-!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:10,1:5)", structured = false}
+!CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND0]], %[[BOUND1]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:10,1:5)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data copyin(a(1:,1:5))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index 
 !CHECK: %[[LB1:.*]] = arith.constant 0 : index
-!CHECK: %[[EXTENT:.*]] = arith.subi %[[EXTENT_C10:.*]], %c1{{.*}} : index 
-!CHECK: %[[BOUND1:.*]] = acc.bounds   lowerbound(%[[LB1]] : index) extent(%[[EXTENT]] : index)  startIdx(%c1{{.*}} : index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %c10{{.*}}, %[[ONE]] : index
+!CHECK: %[[UB1:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index 
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[LB2:.*]] = arith.constant 0 : index
 !CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index)  startIdx(%c1{{.*}} : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(1:,1:5)", structured = false}
 !CHECK: acc.enter_data   dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data copyin(a(:10,1:5))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[UB1:.*]] = arith.constant 9 : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds   upperbound(%[[UB1]] : index)
-!CHECK: %[[LB2:.*]] = arith.constant 0 : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB1]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
 !CHECK: %[[UB2:.*]] = arith.constant 4 : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB2]] : index) upperbound(%[[UB2]] : index)  startIdx(%c1{{.*}} : index)
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB2]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:10,1:5)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
 
   !$acc enter data copyin(a(:,:))
-!CHECK: %[[C1:.*]] = arith.constant 1 : index
-!CHECK: %[[EXT:.*]] = arith.subi %c10{{.*}}, %[[C1]] : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds extent(%[[EXT]] : index) startIdx(%[[C1]] : index)
-!CHECK: %[[C1:.*]] = arith.constant 1 : index
-!CHECK: %[[EXT:.*]] = arith.subi %c10{{.*}}, %[[C1]] : index
-!CHECK: %[[BOUND2:.*]] = acc.bounds extent(%[[EXT]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %c10{{.*}}, %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %c10{{.*}}, %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index 
+!CHECK: %[[BOUND2:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
 !CHECK: %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%[[BOUND1]], %[[BOUND2]]) -> !fir.ref<!fir.array<10x10xf32>> {name = "a(:,:)", structured = false}
-!CHECK: acc.enter_data dataOperands(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>)
-
 end subroutine acc_enter_data
 
-
-subroutine acc_enter_data_dummy(a)
+subroutine acc_enter_data_dummy(a, b, n, m)
+  integer :: n, m
   real :: a(1:10)
+  real :: b(n:m)
 
 !CHECK-LABEL: func.func @_QPacc_enter_data_dummy
-!CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "a"}
+!CHECK-SAME:    %[[A:.*]]: !fir.ref<!fir.array<10xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "b"}, %[[N:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[M:.*]]: !fir.ref<i32> {fir.bindc_name = "m"}
+
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[N]] : !fir.ref<i32>
+!CHECK: %[[N_I64:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64
+!CHECK: %[[N_IDX:.*]] = fir.convert %[[N_I64]] : (i64) -> index
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[M]] : !fir.ref<i32>
+!CHECK: %[[M_I64:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64
+!CHECK: %[[M_IDX:.*]] = fir.convert %[[M_I64]] : (i64) -> index
+!CHECK: %[[M_N:.*]] = arith.subi %[[M_IDX]], %[[N_IDX]] : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[M_N_1:.*]] = arith.addi %[[M_N]], %[[C1]] : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[CMP:.*]] = arith.cmpi sgt, %[[M_N_1]], %[[C0]] : index
+!CHECK: %[[EXT_B:.*]] = arith.select %[[CMP]], %[[M_N_1]], %[[C0]] : index
+
+  !$acc enter data create(a)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(b)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%{{.*}} : index) upperbound(%{{.*}} : index) stride(%c1{{.*}} : index) startIdx(%{{.*}} : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
 
   !$acc enter data create(a(5:10))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[LB1:.*]] = arith.constant 4 : index
 !CHECK: %[[UB1:.*]] = arith.constant 9 : index
-!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index)  startIdx(%c1{{.*}} : index)
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB1]] : index) upperbound(%[[UB1]] : index) stride(%[[ONE]] : index) startIdx(%c1{{.*}} : index)
 !CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<10xf32>> {name = "a(5:10)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<10xf32>>)
 
+  !$acc enter data create(b(n:m))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[N]] : !fir.ref<i32> 
+!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[N_IDX]] : index
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[M]] : !fir.ref<i32>
+!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[N_IDX]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[N_IDX]] : index)
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(n:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b(n:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index 
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[N]] : !fir.ref<i32> 
+!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[N_IDX]] : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXT_B]], %[[N_IDX]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT:.*]], %[[ONE]] : index 
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[N_IDX]] : index) 
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(n:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b(:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index 
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXT_B]], %[[N_IDX]] : index 
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[N_IDX]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[N_IDX]] : index)
+!CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND1]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE1]] : !fir.ref<!fir.array<?xf32>>)
+
 end subroutine
 
 ! Test lowering of array section for non default lower bound.
@@ -150,36 +269,428 @@ subroutine acc_enter_data_non_default_lb()
 
 !CHECK-LABEL: func.func @_QPacc_enter_data_non_default_lb() {
 !CHECK: %[[BASELB:.*]] = arith.constant 0 : index
+!CHECK: %[[EXTENT_C10:.*]] = arith.constant 10 : index
 !CHECK: %[[A:.*]] = fir.alloca !fir.array<10xi32> {bindc_name = "a", uniq_name = "_QFacc_enter_data_non_default_lbEa"}
 
   !$acc enter data create(a(5:9))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[SECTIONLB:.*]] = arith.constant 5 : index
 !CHECK: %[[LB:.*]] = arith.subi %[[SECTIONLB]], %[[BASELB]] : index
 !CHECK: %[[SECTIONUB:.*]] = arith.constant 9 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[SECTIONUB]], %[[BASELB]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[BASELB]] : index)
 !CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(5:9)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
 
   !$acc enter data create(a(:))
-!CHECK: %[[EXT:.*]] = arith.subi %c10{{.*}}, %[[BASELB]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds extent(%[[EXT]] : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXTENT_C10]], %[[BASELB]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[BASELB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[BASELB]] : index)
 !CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(:)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
 
   !$acc enter data create(a(:6))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[SECTIONUB:.*]] = arith.constant 6 : index
 !CHECK: %[[UB:.*]] = arith.subi %[[SECTIONUB]], %[[BASELB]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds upperbound(%[[UB]] : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[BASELB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[BASELB]] : index)
 !CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(:6)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
 
   !$acc enter data create(a(4:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
 !CHECK: %[[SECTIONLB:.*]] = arith.constant 4 : index
 !CHECK: %[[LB:.*]] = arith.subi %[[SECTIONLB]], %[[BASELB]] : index
-!CHECK: %[[EXT:.*]] = arith.subi %c10{{.*}}, %[[BASELB]] : index
-!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) extent(%[[EXT]] : index) startIdx(%[[BASELB]] : index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[EXTENT_C10]], %[[BASELB]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[BASELB]] : index)
 !CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "a(4:)", structured = false}
 !CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
 
 end subroutine
+
+! Test lowering of assumed size arrays.
+subroutine acc_enter_data_assumed(a, b, n, m)
+  integer :: n, m
+  real :: a(:)
+  real :: b(10:)
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_assumed(
+!CHECK-SAME: %[[A:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}, %[[B:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "b"}, %[[N:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}, %[[M:.*]]: !fir.ref<i32> {fir.bindc_name = "m"}) {
+
+!CHECK: %[[LB_C10:.*]] = arith.constant 10 : i64
+!CHECK: %[[LB_C10_IDX:.*]] = fir.convert %[[LB_C10]] : (i64) -> index
+
+  !$acc enter data create(a)
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 0 : index 
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS]]#1, %[[C1]] : index 
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS]]#2 : index) startIdx(%[[C1]] : index) {strideInBytes = true} 
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[DIMS1]]#1, %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(2:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[DIMS1]]#1, %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(2:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:4))
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.constant 3 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:4)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(6:10))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LB:.*]] = arith.constant 5 : index
+!CHECK: %[[UB:.*]] = arith.constant 9 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(6:10)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(n:))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[N]] : !fir.ref<i32>
+!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[ONE]] : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[DIMS]]#1, %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(n:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:m))
+!CHECK: %[[BASELB:.*]] = arith.constant 0 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[M]] : !fir.ref<i32>
+!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[BASELB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(n:m))
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[A]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LOAD_N:.*]] = fir.load %[[N]] : !fir.ref<i32>
+!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index
+!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[ONE]] : index
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[M]] : !fir.ref<i32>
+!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[A]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "a(n:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b(:m))
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[B]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[LOAD_M:.*]] = fir.load %[[M]] : !fir.ref<i32>
+!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index
+!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[LB_C10_IDX]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB_C10_IDX]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[B]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b(:m)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+  !$acc enter data create(b)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[B]], %[[C0]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[C0]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[B]] : (!fir.box<!fir.array<?xf32>>) -> !fir.ref<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<?xf32>> {name = "b", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<?xf32>>)
+
+end subroutine
+
+subroutine acc_enter_data_allocatable()
+  real, allocatable :: a(:)
+  integer, allocatable :: i
+  
+!CHECK-LABEL: func.func @_QPacc_enter_data_allocatable() {
+!CHECK: %[[A:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFacc_enter_data_allocatableEa"}
+!CHECK: %[[I:.*]] = fir.alloca !fir.box<!fir.heap<i32>> {bindc_name = "i", uniq_name = "_QFacc_enter_data_allocatableEi"}
+
+  !$acc enter data create(a)
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0_0:.*]] = arith.constant 0 : index
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0_1:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0_1]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS1]]#1, %c1{{.*}} : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:))
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[BOX_A_2:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_2]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[DIMS2]]#1, %[[DIMS0]]#0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[DIMS0]]#0 : index) upperbound(%[[UB:.*]] : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(2:5))
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C2:.*]] = arith.constant 2 : index
+!CHECK: %[[LB:.*]] = arith.subi %[[C2]], %[[DIMS0]]#0 : index
+!CHECK: %[[C5:.*]] = arith.constant 5 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C5]], %[[DIMS0]]#0 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(2:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(3:))
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C3:.*]] = arith.constant 3 : index
+!CHECK: %[[LB:.*]] = arith.subi %[[C3]], %[[DIMS0]]#0 : index
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS2:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[DIMS2:.*]]#1, %[[DIMS0]]#0 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(3:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(a(:7))
+!CHECK: %[[BOX_A_0:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[BOX_A_1:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[BOX_A_1]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[BOX_A_0]], %[[C0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+!CHECK: %[[C7:.*]] = arith.constant 7 : index
+!CHECK: %[[UB:.*]] = arith.subi %[[C7]], %[[DIMS0]]#0 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[DIMS0]]#0 : index) upperbound(%[[UB]] : index) stride(%[[DIMS1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_A_0]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xf32>> {name = "a(:7)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xf32>>)
+
+  !$acc enter data create(i)
+!CHECK: %[[BOX_I:.*]] = fir.load %[[I]] : !fir.ref<!fir.box<!fir.heap<i32>>>
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX_I]] : (!fir.box<!fir.heap<i32>>) -> !fir.heap<i32>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<i32>) -> !fir.heap<i32> {name = "i", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<i32>)
+
+end subroutine
+
+subroutine acc_enter_data_derived_type()
+  type :: dt
+    real :: data
+    real :: array(1:10)
+  end type
+
+  type :: t
+    type(dt) :: d
+  end type
+
+  type :: z
+    integer, allocatable :: data(:)
+  end type
+
+  type :: tt
+    type(dt) :: d(10)
+  end type
+
+  type(dt) :: a
+  type(t) :: b
+  type(dt) :: aa(10)
+  type(z) :: c
+  type(tt) :: d
+
+!CHECK-LABEL: func.func @_QPacc_enter_data_derived_type() {
+!CHECK: %[[A:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}> {bindc_name = "a", uniq_name = "_QFacc_enter_data_derived_typeEa"}
+!CHECK: %[[AA:.*]] = fir.alloca !fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>> {bindc_name = "aa", uniq_name = "_QFacc_enter_data_derived_typeEaa"}
+!CHECK: %[[B:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}> {bindc_name = "b", uniq_name = "_QFacc_enter_data_derived_typeEb"}
+!CHECK: %[[C:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTz{data:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "c", uniq_name = "_QFacc_enter_data_derived_typeEc"}
+!CHECK: %[[D:.*]] = fir.alloca !fir.type<_QFacc_enter_data_derived_typeTtt{d:!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>}> {bindc_name = "d", uniq_name = "_QFacc_enter_data_derived_typeEd"}
+
+  !$acc enter data create(a%data)
+!CHECK: %[[DATA_FIELD:.*]] = fir.field_index data, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}> 
+!CHECK: %[[DATA_COORD:.*]] = fir.coordinate_of %[[A]], %[[DATA_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<f32> 
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DATA_COORD]] : !fir.ref<f32>) -> !fir.ref<f32> {name = "a%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<f32>)
+
+  !$acc enter data create(b%d%data)
+!CHECK: %[[D_FIELD:.*]] = fir.field_index d, !fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>
+!CHECK: %[[DATA_FIELD:.*]] = fir.field_index data, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[DATA_COORD:.*]] = fir.coordinate_of %[[B]], %[[D_FIELD]], %[[DATA_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>>, !fir.field, !fir.field) -> !fir.ref<f32>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[DATA_COORD]] : !fir.ref<f32>) -> !fir.ref<f32> {name = "b%d%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<f32>)
+
+  !$acc enter data create(a%array)
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[A]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[C1]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(:))
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[A]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[C1]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(1:5))
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[A]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C0:.*]] = arith.constant 0 : index
+!CHECK: %[[C4:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[C0]] : index) upperbound(%[[C4]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(1:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(:5))
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[A]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[LB:.*]] = arith.constant 0 : index 
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[C4:.*]] = arith.constant 4 : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[C4]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(:5)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+  !$acc enter data create(a%array(2:))
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[A]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 1 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[ONE]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[ONE]] : index) startIdx(%[[ONE]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "a%array(2:)", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+!$acc enter data create(b%d%array)
+!CHECK: %[[D_FIELD:.*]] = fir.field_index d, !fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[B]], %[[D_FIELD]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTt{d:!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>}>>, !fir.field, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[C1]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "b%d%array", structured = false}
+
+  !$acc enter data create(c%data)
+!CHECK: %[[DATA_FIELD:.*]] = fir.field_index data, !fir.type<_QFacc_enter_data_derived_typeTz{data:!fir.box<!fir.heap<!fir.array<?xi32>>>}>
+!CHECK: %[[DATA_COORD:.*]] = fir.coordinate_of %[[C]], %[[DATA_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTz{data:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.field) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+!CHECK: %[[DATA_BOX:.*]] = fir.load %[[DATA_COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+!CHECK: %[[DIM0:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DATA_BOX]], %[[DIM0]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+!CHECK: %[[ONE:.*]] = arith.constant 1 : index
+!CHECK: %[[DIM0_1:.*]] = arith.constant 0 : index
+!CHECK: %[[DIMS0_1:.*]]:3 = fir.box_dims %[[DATA_BOX]], %[[DIM0_1]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+!CHECK: %[[UB:.*]] = arith.subi %[[DIMS0_1]]#1, %[[ONE]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) stride(%[[DIMS0_1]]#2 : index) startIdx(%[[DIMS0]]#0 : index) {strideInBytes = true}
+!CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DATA_BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xi32>> {name = "c%data", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
+
+  !$acc enter data create (d%d(1)%array)
+!CHECK: %[[D_FIELD:.*]] = fir.field_index d, !fir.type<_QFacc_enter_data_derived_typeTtt{d:!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>}>
+!CHECK: %[[D_COORD:.*]] = fir.coordinate_of %[[D]], %[[D_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTtt{d:!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>}>>, !fir.field) -> !fir.ref<!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>>
+!CHECK: %[[IDX:.*]] = arith.constant 1 : i64
+!CHECK: %[[ONE:.*]] = arith.constant 1 : i64
+!CHECK: %[[NORMALIZED_IDX:.*]] = arith.subi %[[IDX]], %[[ONE]] : i64
+!CHECK: %[[D1_COORD:.*]] = fir.coordinate_of %[[D_COORD]], %[[NORMALIZED_IDX]] : (!fir.ref<!fir.array<10x!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>>, i64) -> !fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>
+!CHECK: %[[ARRAY_FIELD:.*]] = fir.field_index array, !fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>
+!CHECK: %[[ARRAY_COORD:.*]] = fir.coordinate_of %[[D1_COORD]], %[[ARRAY_FIELD]] : (!fir.ref<!fir.type<_QFacc_enter_data_derived_typeTdt{data:f32,array:!fir.array<10xf32>}>>, !fir.field) -> !fir.ref<!fir.array<10xf32>>
+!CHECK: %[[C10:.*]] = arith.constant 10 : index
+!CHECK: %[[C1:.*]] = arith.constant 1 : index
+!CHECK: %[[LB:.*]] = arith.constant 0 : index
+!CHECK: %[[LBEXT:.*]] = arith.addi %[[C10]], %[[C1]] : index
+!CHECK: %[[UB:.*]] = arith.subi %[[LBEXT]], %[[C1]] : index
+!CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+!CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[ARRAY_COORD]] : !fir.ref<!fir.array<10xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xf32>> {name = "d%d(1_8)%array", structured = false}
+!CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xf32>>)
+
+end subroutine
diff --git a/flang/test/Lower/OpenACC/acc-exit-data.f90 b/flang/test/Lower/OpenACC/acc-exit-data.f90
index 5bae065bf1ea6..88ed4f580a9fe 100644
--- a/flang/test/Lower/OpenACC/acc-exit-data.f90
+++ b/flang/test/Lower/OpenACC/acc-exit-data.f90
@@ -8,59 +8,96 @@ subroutine acc_exit_data
   real, pointer :: d
   logical :: ifCondition = .TRUE.
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[D:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+!CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+!CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+!CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+!CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
 
   !$acc exit data delete(a)
-!CHECK: acc.exit_data delete([[A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) if(.true.)
-!CHECK: [[IF1:%.*]] = arith.constant true
-!CHECK: acc.exit_data if([[IF1]]) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[IF1:.*]] = arith.constant true
+!CHECK: acc.exit_data if(%[[IF1]]) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) 
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) if(ifCondition)
-!CHECK: [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
-!CHECK: [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-!CHECK: acc.exit_data if([[IF2]]) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[IFCOND:.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+!CHECK: %[[IF2:.*]] = fir.convert %[[IFCOND]] : (!fir.logical<4>) -> i1
+!CHECK: acc.exit_data if(%[[IF2]]) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) delete(b) delete(c)
-!CHECK: acc.exit_data delete([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "b", structured = false}
+!CHECK: %[[DEVPTR_C:.*]] = acc.getdeviceptr varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "c", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>){{$}}
+!CHECK: acc.delete accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "b", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "c", structured = false}
 
   !$acc exit data copyout(a) delete(b) detach(d)
-!CHECK: acc.exit_data copyout([[A]] : !fir.ref<!fir.array<10x10xf32>>) delete([[B]] : !fir.ref<!fir.array<10x10xf32>>) detach([[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>){{$}}
+!CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "a", structured = false}
+!CHECK: %[[DEVPTR_B:.*]] = acc.getdeviceptr varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>)   bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "b", structured = false}
+!CHECK: %[[BOX_D:.*]] = fir.load %[[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>>
+!CHECK: %[[D_ADDR:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
+!CHECK: %[[DEVPTR_D:.*]] = acc.getdeviceptr varPtr(%[[D_ADDR]] : !fir.ptr<f32>) -> !fir.ptr<f32> {dataClause = 11 : i64, name = "d", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR_A]], %[[DEVPTR_B]], %[[DEVPTR_D]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ptr<f32>)
+!CHECK: acc.copyout accPtr(%[[DEVPTR_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a", structured = false}
+!CHECK: acc.delete accPtr(%[[DEVPTR_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "b", structured = false}
+!CHECK: acc.detach accPtr(%[[DEVPTR_D]] : !fir.ptr<f32>) {name = "d", structured = false}
 
   !$acc exit data delete(a) async
-!CHECK: acc.exit_data delete([[A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait
-!CHECK: acc.exit_data delete([[A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {wait}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) async wait
-!CHECK: acc.exit_data delete([[A]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: acc.exit_data dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) attributes {async, wait}
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) async(1)
-!CHECK: [[ASYNC1:%.*]] = arith.constant 1 : i32
-!CHECK: acc.exit_data async([[ASYNC1]] : i32) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[ASYNC1:.*]] = arith.constant 1 : i32
+!CHECK: acc.exit_data async(%[[ASYNC1]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
+
 
   !$acc exit data delete(a) async(async)
-!CHECK: [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK: acc.exit_data async([[ASYNC2]] : i32) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[ASYNC2:.*]] = fir.load %{{.*}} : !fir.ref<i32>
+!CHECK: acc.exit_data async(%[[ASYNC2]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait(1)
-!CHECK: [[WAIT1:%.*]] = arith.constant 1 : i32
-!CHECK: acc.exit_data wait([[WAIT1]] : i32) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[WAIT1:.*]] = arith.constant 1 : i32
+!CHECK: acc.exit_data wait(%[[WAIT1]] : i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait(queues: 1, 2)
-!CHECK: [[WAIT2:%.*]] = arith.constant 1 : i32
-!CHECK: [[WAIT3:%.*]] = arith.constant 2 : i32
-!CHECK: acc.exit_data wait([[WAIT2]], [[WAIT3]] : i32, i32) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[WAIT2:.*]] = arith.constant 1 : i32
+!CHECK: %[[WAIT3:.*]] = arith.constant 2 : i32
+!CHECK: acc.exit_data wait(%[[WAIT2]], %[[WAIT3]] : i32, i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
   !$acc exit data delete(a) wait(devnum: 1: queues: 1, 2)
-!CHECK: [[WAIT4:%.*]] = arith.constant 1 : i32
-!CHECK: [[WAIT5:%.*]] = arith.constant 2 : i32
-!CHECK: [[WAIT6:%.*]] = arith.constant 1 : i32
-!CHECK: acc.exit_data wait_devnum([[WAIT6]] : i32) wait([[WAIT4]], [[WAIT5]] : i32, i32) delete([[A]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: %[[DEVPTR:.*]] = acc.getdeviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 9 : i64, name = "a", structured = false}
+!CHECK: %[[WAIT4:.*]] = arith.constant 1 : i32
+!CHECK: %[[WAIT5:.*]] = arith.constant 2 : i32
+!CHECK: %[[WAIT6:.*]] = arith.constant 1 : i32
+!CHECK: acc.exit_data wait_devnum(%[[WAIT6]] : i32) wait(%[[WAIT4]], %[[WAIT5]] : i32, i32) dataOperands(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>)
+!CHECK: acc.delete accPtr(%[[DEVPTR]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {name = "a", structured = false}
 
 end subroutine acc_exit_data
diff --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index b160fc528070f..6aad08b13499c 100644
--- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -22,25 +22,25 @@ subroutine acc_kernels_loop
   integer :: vectorNum = 128
   integer, parameter :: tileSize = 2
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[F:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "f", uniq_name = "{{.*}}Ef"}
-!CHECK: [[G:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "g", uniq_name = "{{.*}}Eg"}
-!CHECK: [[IFCONDITION:%.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK: %[[F:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "f", uniq_name = "{{.*}}Ef"}
+! CHECK: %[[G:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "g", uniq_name = "{{.*}}Eg"}
+! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 
   !$acc kernels loop
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop async
   DO i = 1, n
@@ -48,529 +48,559 @@ subroutine acc_kernels_loop
   END DO
   !$acc end kernels loop
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: } attributes {asyncAttr}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: } attributes {asyncAttr}
 
   !$acc kernels loop async(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.kernels async([[ASYNC1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.kernels async([[ASYNC1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop async(async)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels async([[ASYNC2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels async([[ASYNC2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop wait
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: } attributes {waitAttr}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: } attributes {waitAttr}
 
   !$acc kernels loop wait(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.kernels wait([[WAIT1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.kernels wait([[WAIT1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop wait(1, 2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
-!CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-!CHECK:      acc.kernels wait([[WAIT2]], [[WAIT3]] : i32, i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
+! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
+! CHECK:      acc.kernels wait([[WAIT2]], [[WAIT3]] : i32, i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop wait(wait1, wait2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels wait([[WAIT4]], [[WAIT5]] : i32, i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels wait([[WAIT4]], [[WAIT5]] : i32, i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop num_gangs(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.kernels num_gangs([[NUMGANGS1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.kernels num_gangs([[NUMGANGS1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop num_gangs(numGangs)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels num_gangs([[NUMGANGS2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels num_gangs([[NUMGANGS2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop num_workers(10)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
-!CHECK:      acc.kernels num_workers([[NUMWORKERS1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
+! CHECK:      acc.kernels num_workers([[NUMWORKERS1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop num_workers(numWorkers)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels num_workers([[NUMWORKERS2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels num_workers([[NUMWORKERS2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop vector_length(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
-!CHECK:      acc.kernels vector_length([[VECTORLENGTH1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
+! CHECK:      acc.kernels vector_length([[VECTORLENGTH1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop vector_length(vectorLength)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels vector_length([[VECTORLENGTH2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels vector_length([[VECTORLENGTH2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop if(.TRUE.)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[IF1:%.*]] = arith.constant true
-!CHECK:      acc.kernels if([[IF1]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IF1:%.*]] = arith.constant true
+! CHECK:      acc.kernels if([[IF1]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop if(ifCondition)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
-!CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-!CHECK:      acc.kernels if([[IF2]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK:      acc.kernels if([[IF2]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop self(.TRUE.)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[SELF1:%.*]] = arith.constant true
-!CHECK:      acc.kernels self([[SELF1]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[SELF1:%.*]] = arith.constant true
+! CHECK:      acc.kernels self([[SELF1]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop self
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: } attributes {selfAttr}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: } attributes {selfAttr}
 
   !$acc kernels loop self(ifCondition)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[SELF2:%.*]] = fir.convert [[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
-!CHECK:      acc.kernels self([[SELF2]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[SELF2:.*]] = fir.convert %[[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
+! CHECK:      acc.kernels self(%[[SELF2]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop copy(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels copy([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "b"}
 
   !$acc kernels loop copy(a) copy(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels copy([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "b"}
 
   !$acc kernels loop copyin(a) copyin(readonly: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels copyin([[A]] : !fir.ref<!fir.array<10xf32>>) copyin_readonly([[B]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 2 : i64, name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop copyout(a) copyout(zero: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels copyout([[A]] : !fir.ref<!fir.array<10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 4 : i64, name = "a"} 
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 4 : i64, name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {name = "a"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {name = "b"}
 
   !$acc kernels loop create(b) create(zero: a)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels create([[B]] : !fir.ref<!fir.array<10xf32>>) create_zero([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 8 : i64, name = "a"}
+! CHECK:      acc.kernels dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = 7 : i64, name = "b"}
+! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = 8 : i64, name = "a"}
 
   !$acc kernels loop no_create(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels no_create([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop present(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels present([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop deviceptr(a) deviceptr(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels deviceptr([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.kernels dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop attach(f, g)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels attach([[F]], [[G]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[BOX_F:.*]] = fir.load %[[F]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_F:.*]] = fir.box_addr %[[BOX_F]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[BOX_ADDR_F]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "f"}
+! CHECK:      %[[BOX_G:.*]] = fir.load %[[G]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
+! CHECK:      acc.kernels dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop seq
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {seq}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {seq}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop auto
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {auto}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {auto}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop independent
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {independent}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {independent}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop gang
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop gang {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop gang {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop gang(num: 8)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-!CHECK-NEXT:   acc.loop gang(num=[[GANGNUM1]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
+! CHECK-NEXT:   acc.loop gang(num=[[GANGNUM1]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop gang(num: gangNum)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK-NEXT:   acc.loop gang(num=[[GANGNUM2]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK-NEXT:   acc.loop gang(num=[[GANGNUM2]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
  !$acc kernels loop gang(num: gangNum, static: gangStatic)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop gang(num=%{{.*}}: i32, static=%{{.*}}: i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop vector
   DO i = 1, n
     a(i) = b(i)
   END DO
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop vector {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop vector {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop vector(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
-!CHECK:        acc.loop vector([[CONSTANT128]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
+! CHECK:        acc.loop vector([[CONSTANT128]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop vector(vectorLength)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop worker
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop worker {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop worker {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop worker(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
-!CHECK:        acc.loop worker([[WORKER128]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
+! CHECK:        acc.loop worker([[WORKER128]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop collapse(2)
   DO i = 1, n
@@ -579,14 +609,14 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:            fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {collapse = 2 : i64}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:            fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {collapse = 2 : i64}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop
   DO i = 1, n
@@ -596,45 +626,45 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:            acc.loop {
-!CHECK:              fir.do_loop
-!CHECK:              acc.yield
-!CHECK-NEXT:     }{{$}}
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:            acc.loop {
+! CHECK:              fir.do_loop
+! CHECK:              acc.yield
+! CHECK-NEXT:     }{{$}}
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
  !$acc kernels loop tile(2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
-!CHECK:        acc.loop tile([[TILESIZE]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
+! CHECK:        acc.loop tile([[TILESIZE]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
  !$acc kernels loop tile(*)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
-!CHECK:        acc.loop tile([[TILESIZEM1]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
+! CHECK:        acc.loop tile([[TILESIZEM1]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop tile(2, 2)
   DO i = 1, n
@@ -643,28 +673,28 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
-!CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
-!CHECK:        acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
+! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
+! CHECK:        acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop tile(tileSize)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop tile(%{{.*}}: i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop tile(%{{.*}} : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels loop tile(tileSize, tileSize)
   DO i = 1, n
@@ -673,12 +703,12 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-!CHECK:      acc.kernels {
-!CHECK:        acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels {
+! CHECK:        acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
 end subroutine
diff --git a/flang/test/Lower/OpenACC/acc-kernels.f90 b/flang/test/Lower/OpenACC/acc-kernels.f90
index 65730fba16817..70a4cca6e094f 100644
--- a/flang/test/Lower/OpenACC/acc-kernels.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels.f90
@@ -15,225 +15,266 @@ subroutine acc_kernels
   real, dimension(10, 10) :: a, b, c
   real, pointer :: d, e
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[D:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
-!CHECK: [[E:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
-!CHECK: [[IFCONDITION:%.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+! CHECK: %[[E:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
+! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 
   !$acc kernels
   !$acc end kernels
 
-!CHECK:      acc.kernels  {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.kernels  {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels async
   !$acc end kernels
 
-!CHECK:      acc.kernels  {
-!CHECK:        acc.terminator
-!CHECK-NEXT: } attributes {asyncAttr}
+! CHECK:      acc.kernels  {
+! CHECK:        acc.terminator
+! CHECK-NEXT: } attributes {asyncAttr}
 
   !$acc kernels async(1)
   !$acc end kernels
 
-!CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.kernels  async([[ASYNC1]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.kernels  async([[ASYNC1]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels async(async)
   !$acc end kernels
 
-!CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels  async([[ASYNC2]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels  async([[ASYNC2]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels wait
   !$acc end kernels
 
-!CHECK:      acc.kernels  {
-!CHECK:        acc.terminator
-!CHECK-NEXT: } attributes {waitAttr}
+! CHECK:      acc.kernels  {
+! CHECK:        acc.terminator
+! CHECK-NEXT: } attributes {waitAttr}
 
   !$acc kernels wait(1)
   !$acc end kernels
 
-!CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.kernels  wait([[WAIT1]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.kernels  wait([[WAIT1]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels wait(1, 2)
   !$acc end kernels
 
-!CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
-!CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-!CHECK:      acc.kernels  wait([[WAIT2]], [[WAIT3]] : i32, i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
+! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
+! CHECK:      acc.kernels  wait([[WAIT2]], [[WAIT3]] : i32, i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels wait(wait1, wait2)
   !$acc end kernels
 
-!CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels  wait([[WAIT4]], [[WAIT5]] : i32, i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels  wait([[WAIT4]], [[WAIT5]] : i32, i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels num_gangs(1)
   !$acc end kernels
 
-!CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.kernels  num_gangs([[NUMGANGS1]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.kernels  num_gangs([[NUMGANGS1]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels num_gangs(numGangs)
   !$acc end kernels
 
-!CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels  num_gangs([[NUMGANGS2]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels  num_gangs([[NUMGANGS2]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels num_workers(10)
   !$acc end kernels
 
-!CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
-!CHECK:      acc.kernels  num_workers([[NUMWORKERS1]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
+! CHECK:      acc.kernels  num_workers([[NUMWORKERS1]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels num_workers(numWorkers)
   !$acc end kernels
 
-!CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels  num_workers([[NUMWORKERS2]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels  num_workers([[NUMWORKERS2]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels vector_length(128)
   !$acc end kernels
 
-!CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
-!CHECK:      acc.kernels  vector_length([[VECTORLENGTH1]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
+! CHECK:      acc.kernels  vector_length([[VECTORLENGTH1]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels vector_length(vectorLength)
   !$acc end kernels
 
-!CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.kernels  vector_length([[VECTORLENGTH2]] : i32) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.kernels  vector_length([[VECTORLENGTH2]] : i32) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels if(.TRUE.)
   !$acc end kernels
 
-!CHECK:      [[IF1:%.*]] = arith.constant true
-!CHECK:      acc.kernels  if([[IF1]]) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IF1:%.*]] = arith.constant true
+! CHECK:      acc.kernels  if([[IF1]]) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels if(ifCondition)
   !$acc end kernels
 
-!CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
-!CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-!CHECK:      acc.kernels  if([[IF2]]) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK:      acc.kernels  if([[IF2]]) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels self(.TRUE.)
   !$acc end kernels
 
-!CHECK:      [[SELF1:%.*]] = arith.constant true
-!CHECK:      acc.kernels  self([[SELF1]]) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[SELF1:%.*]] = arith.constant true
+! CHECK:      acc.kernels  self([[SELF1]]) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels self
   !$acc end kernels
 
-!CHECK:      acc.kernels  {
-!CHECK:        acc.terminator
-!CHECK-NEXT: } attributes {selfAttr}
+! CHECK:      acc.kernels  {
+! CHECK:        acc.terminator
+! CHECK-NEXT: } attributes {selfAttr}
 
   !$acc kernels self(ifCondition)
   !$acc end kernels
 
-!CHECK:      [[SELF2:%.*]] = fir.convert [[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
-!CHECK:      acc.kernels  self([[SELF2]]) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[SELF2:.*]] = fir.convert %[[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
+! CHECK:      acc.kernels self(%[[SELF2]]) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels copy(a, b, c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"} 
+! CHECK:      acc.kernels  dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc kernels copy(a) copy(b) copy(c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"} 
+! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc kernels copyin(a) copyin(readonly: b, c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  copyin([[A]] : !fir.ref<!fir.array<10x10xf32>>) copyin_readonly([[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "b"} 
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "c"} 
+! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels copyout(a) copyout(zero: b) copyout(c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  copyout([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "c"}
+! CHECK:      acc.kernels dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc kernels create(a, b) create(zero: c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+! CHECK:      acc.kernels dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 8 : i64, name = "c"}
 
   !$acc kernels no_create(a, b) create(zero: c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) no_create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+! CHECK:      acc.kernels dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels present(a, b, c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  present([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.kernels dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels deviceptr(a) deviceptr(c)
   !$acc end kernels
 
-!CHECK:      acc.kernels  deviceptr([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.kernels dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
   !$acc kernels attach(d, e)
   !$acc end kernels
 
-!CHECK:      acc.kernels  attach([[D]], [[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
-!CHECK:        acc.terminator
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[BOX_D:.*]] = fir.load %[[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"} 
+! CHECK:      %[[BOX_E:.*]] = fir.load %[[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_E:.*]] = fir.box_addr %[[BOX_E]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_E:.*]] = acc.attach varPtr(%[[BOX_ADDR_E]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"} 
+! CHECK:      acc.kernels dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) { 
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
 
 end subroutine acc_kernels
diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index ab662ab35a53e..fa7323ddf6b08 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -22,25 +22,25 @@ subroutine acc_parallel_loop
   integer :: vectorNum = 128
   integer, parameter :: tileSize = 2
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[F:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "f", uniq_name = "{{.*}}Ef"}
-!CHECK: [[G:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "g", uniq_name = "{{.*}}Eg"}
-!CHECK: [[IFCONDITION:%.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK: %[[F:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "f", uniq_name = "{{.*}}Ef"}
+! CHECK: %[[G:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "g", uniq_name = "{{.*}}Eg"}
+! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 
   !$acc parallel loop
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop async
   DO i = 1, n
@@ -48,542 +48,572 @@ subroutine acc_parallel_loop
   END DO
   !$acc end parallel loop
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: } attributes {asyncAttr}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {asyncAttr}
 
   !$acc parallel loop async(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.parallel async([[ASYNC1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.parallel async([[ASYNC1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop async(async)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.parallel async([[ASYNC2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.parallel async([[ASYNC2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop wait
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: } attributes {waitAttr}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {waitAttr}
 
   !$acc parallel loop wait(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.parallel wait([[WAIT1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.parallel wait([[WAIT1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop wait(1, 2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
-!CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-!CHECK:      acc.parallel wait([[WAIT2]], [[WAIT3]] : i32, i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
+! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
+! CHECK:      acc.parallel wait([[WAIT2]], [[WAIT3]] : i32, i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop wait(wait1, wait2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.parallel wait([[WAIT4]], [[WAIT5]] : i32, i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.parallel wait([[WAIT4]], [[WAIT5]] : i32, i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop num_gangs(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.parallel num_gangs([[NUMGANGS1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.parallel num_gangs([[NUMGANGS1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop num_gangs(numGangs)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.parallel num_gangs([[NUMGANGS2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.parallel num_gangs([[NUMGANGS2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop num_workers(10)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
-!CHECK:      acc.parallel num_workers([[NUMWORKERS1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
+! CHECK:      acc.parallel num_workers([[NUMWORKERS1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop num_workers(numWorkers)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.parallel num_workers([[NUMWORKERS2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.parallel num_workers([[NUMWORKERS2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop vector_length(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
-!CHECK:      acc.parallel vector_length([[VECTORLENGTH1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
+! CHECK:      acc.parallel vector_length([[VECTORLENGTH1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop vector_length(vectorLength)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.parallel vector_length([[VECTORLENGTH2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.parallel vector_length([[VECTORLENGTH2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop if(.TRUE.)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[IF1:%.*]] = arith.constant true
-!CHECK:      acc.parallel if([[IF1]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IF1:%.*]] = arith.constant true
+! CHECK:      acc.parallel if([[IF1]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop if(ifCondition)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
-!CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-!CHECK:      acc.parallel if([[IF2]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK:      acc.parallel if([[IF2]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop self(.TRUE.)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[SELF1:%.*]] = arith.constant true
-!CHECK:      acc.parallel self([[SELF1]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[SELF1:%.*]] = arith.constant true
+! CHECK:      acc.parallel self([[SELF1]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop self
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: } attributes {selfAttr}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {selfAttr}
 
   !$acc parallel loop self(ifCondition)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[SELF2:%.*]] = fir.convert [[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
-!CHECK:      acc.parallel self([[SELF2]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[SELF2:.*]] = fir.convert %[[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
+! CHECK:      acc.parallel self(%[[SELF2]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop copy(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel copy([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "b"}
 
   !$acc parallel loop copy(a) copy(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel copy([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "b"}
 
   !$acc parallel loop copyin(a) copyin(readonly: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel copyin([[A]] : !fir.ref<!fir.array<10xf32>>) copyin_readonly([[B]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 2 : i64, name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop copyout(a) copyout(zero: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel copyout([[A]] : !fir.ref<!fir.array<10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 4 : i64, name = "a"} 
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 4 : i64, name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {name = "a"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {name = "b"}
 
   !$acc parallel loop create(b) create(zero: a)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel create([[B]] : !fir.ref<!fir.array<10xf32>>) create_zero([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 8 : i64, name = "a"}
+! CHECK:      acc.parallel dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = 7 : i64, name = "b"}
+! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = 8 : i64, name = "a"}
 
   !$acc parallel loop no_create(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel no_create([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop present(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel present([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop deviceptr(a) deviceptr(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel deviceptr([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.parallel dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop attach(f, g)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel attach([[F]], [[G]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[BOX_F:.*]] = fir.load %[[F]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_F:.*]] = fir.box_addr %[[BOX_F]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[BOX_ADDR_F]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "f"}
+! CHECK:      %[[BOX_G:.*]] = fir.load %[[G]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
+! CHECK:      acc.parallel dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop private(a) firstprivate(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel firstprivate([[B]] : !fir.ref<!fir.array<10xf32>>) private([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop private([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel firstprivate(%[[B]] : !fir.ref<!fir.array<10xf32>>) private(%[[A]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop private(%[[A]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop seq
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {seq}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {seq}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop auto
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {auto}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {auto}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop independent
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {independent}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {independent}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop gang
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop gang {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop gang {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop gang(num: 8)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-!CHECK-NEXT:   acc.loop gang(num=[[GANGNUM1]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
+! CHECK-NEXT:   acc.loop gang(num=[[GANGNUM1]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop gang(num: gangNum)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK-NEXT:   acc.loop gang(num=[[GANGNUM2]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK-NEXT:   acc.loop gang(num=[[GANGNUM2]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
  !$acc parallel loop gang(num: gangNum, static: gangStatic)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop vector
   DO i = 1, n
     a(i) = b(i)
   END DO
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop vector {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop vector {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop vector(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
-!CHECK:        acc.loop vector([[CONSTANT128]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
+! CHECK:        acc.loop vector([[CONSTANT128]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop vector(vectorLength)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop worker
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop worker {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop worker {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop worker(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
-!CHECK:        acc.loop worker([[WORKER128]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
+! CHECK:        acc.loop worker([[WORKER128]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop collapse(2)
   DO i = 1, n
@@ -592,14 +622,14 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:            fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {collapse = 2 : i64}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:            fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {collapse = 2 : i64}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop
   DO i = 1, n
@@ -609,45 +639,45 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:            acc.loop {
-!CHECK:              fir.do_loop
-!CHECK:              acc.yield
-!CHECK-NEXT:     }{{$}}
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:            acc.loop {
+! CHECK:              fir.do_loop
+! CHECK:              acc.yield
+! CHECK-NEXT:     }{{$}}
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
  !$acc parallel loop tile(2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
-!CHECK:        acc.loop tile([[TILESIZE]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
+! CHECK:        acc.loop tile([[TILESIZE]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
  !$acc parallel loop tile(*)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
-!CHECK:        acc.loop tile([[TILESIZEM1]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
+! CHECK:        acc.loop tile([[TILESIZEM1]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop tile(2, 2)
   DO i = 1, n
@@ -656,28 +686,28 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
-!CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
-!CHECK:        acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
+! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
+! CHECK:        acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop tile(tileSize)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop tile(%{{.*}} : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop tile(%{{.*}} : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc parallel loop tile(tileSize, tileSize)
   DO i = 1, n
@@ -686,12 +716,12 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-!CHECK:      acc.parallel {
-!CHECK:        acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.parallel {
+! CHECK:        acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
 end subroutine acc_parallel_loop
diff --git a/flang/test/Lower/OpenACC/acc-parallel.f90 b/flang/test/Lower/OpenACC/acc-parallel.f90
index ab793a8050496..8229890f9a875 100644
--- a/flang/test/Lower/OpenACC/acc-parallel.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel.f90
@@ -15,12 +15,12 @@ subroutine acc_parallel
   real, dimension(10, 10) :: a, b, c
   real, pointer :: d, e
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[D:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
-!CHECK: [[E:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
-!CHECK: [[IFCONDITION:%.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
+!CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+!CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+!CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+!CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+!CHECK: %[[E:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
+!CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 
   !$acc parallel
   !$acc end parallel
@@ -168,78 +168,123 @@ subroutine acc_parallel
   !$acc parallel self(ifCondition)
   !$acc end parallel
 
-!CHECK:      [[SELF2:%.*]] = fir.convert [[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
-!CHECK:      acc.parallel self([[SELF2]]) {
+!CHECK:      %[[SELF2:.*]] = fir.convert %[[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
+!CHECK:      acc.parallel self(%[[SELF2]]) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
 
   !$acc parallel copy(a, b, c)
   !$acc end parallel
 
-!CHECK:      acc.parallel copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"} 
+!CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"} 
+!CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"} 
+!CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
+!CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+!CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+!CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc parallel copy(a) copy(b) copy(c)
   !$acc end parallel
 
-!CHECK:      acc.parallel copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"} 
+!CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"} 
+!CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"} 
+!CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
+!CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+!CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+!CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc parallel copyin(a) copyin(readonly: b, c)
   !$acc end parallel
 
-!CHECK:      acc.parallel copyin([[A]] : !fir.ref<!fir.array<10x10xf32>>) copyin_readonly([[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+!CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "b"} 
+!CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "c"} 
+!CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
 
   !$acc parallel copyout(a) copyout(zero: b) copyout(c)
   !$acc end parallel
 
-!CHECK:      acc.parallel copyout([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "a"}
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "b"}
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "c"}
+!CHECK:      acc.parallel dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
+!CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+!CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
+!CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc parallel create(a, b) create(zero: c)
   !$acc end parallel
 
-!CHECK:      acc.parallel create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+!CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+!CHECK:      acc.parallel dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
+!CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "a"}
+!CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "b"}
+!CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 8 : i64, name = "c"}
 
   !$acc parallel no_create(a, b) create(zero: c)
   !$acc end parallel
 
-!CHECK:      acc.parallel create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) no_create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+!CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+!CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+!CHECK:      acc.parallel dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
+!CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 8 : i64, name = "c"}
+
 
   !$acc parallel present(a, b, c)
   !$acc end parallel
 
-!CHECK:      acc.parallel present([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+!CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+!CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+!CHECK:      acc.parallel dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
 
   !$acc parallel deviceptr(a) deviceptr(c)
   !$acc end parallel
 
-!CHECK:      acc.parallel deviceptr([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK: %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+!CHECK: %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+!CHECK:      acc.parallel dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
 
   !$acc parallel attach(d, e)
   !$acc end parallel
 
-!CHECK:      acc.parallel attach([[D]], [[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
+!CHECK: %[[BOX_D:.*]] = fir.load %[[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+!CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+!CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"} 
+!CHECK: %[[BOX_E:.*]] = fir.load %[[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+!CHECK: %[[BOX_ADDR_E:.*]] = fir.box_addr %[[BOX_E]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+!CHECK: %[[ATTACH_E:.*]] = acc.attach varPtr(%[[BOX_ADDR_E]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"} 
+!CHECK: acc.parallel dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) { 
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
+!CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = 10 : i64, name = "d"}
+!CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = 10 : i64, name = "e"}
 
   !$acc parallel private(a) firstprivate(b) private(c)
   !$acc end parallel
 
-!CHECK:      acc.parallel firstprivate([[B]] : !fir.ref<!fir.array<10x10xf32>>) private([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+!CHECK:      acc.parallel firstprivate(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) private(%[[A]], %[[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 !CHECK:        acc.yield
 !CHECK-NEXT: }{{$}}
 
diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index c2ee38a7c4f00..673a90f567087 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -22,25 +22,25 @@ subroutine acc_serial_loop
   integer :: vectorNum = 128
   integer, parameter :: tileSize = 2
 
-!CHECK: [[A:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-!CHECK: [[B:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-!CHECK: [[C:%.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-!CHECK: [[F:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "f", uniq_name = "{{.*}}Ef"}
-!CHECK: [[G:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "g", uniq_name = "{{.*}}Eg"}
-!CHECK: [[IFCONDITION:%.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK: %[[F:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "f", uniq_name = "{{.*}}Ef"}
+! CHECK: %[[G:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "g", uniq_name = "{{.*}}Eg"}
+! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 
   !$acc serial loop
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop async
   DO i = 1, n
@@ -48,458 +48,488 @@ subroutine acc_serial_loop
   END DO
   !$acc end serial loop
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: } attributes {asyncAttr}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {asyncAttr}
 
   !$acc serial loop async(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.serial async([[ASYNC1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.serial async([[ASYNC1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop async(async)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.serial async([[ASYNC2]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.serial async([[ASYNC2]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop wait
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: } attributes {waitAttr}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {waitAttr}
 
   !$acc serial loop wait(1)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-!CHECK:      acc.serial wait([[WAIT1]] : i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
+! CHECK:      acc.serial wait([[WAIT1]] : i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop wait(1, 2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
-!CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-!CHECK:      acc.serial wait([[WAIT2]], [[WAIT3]] : i32, i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
+! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
+! CHECK:      acc.serial wait([[WAIT2]], [[WAIT3]] : i32, i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop wait(wait1, wait2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:      acc.serial wait([[WAIT4]], [[WAIT5]] : i32, i32) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:      acc.serial wait([[WAIT4]], [[WAIT5]] : i32, i32) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop if(.TRUE.)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[IF1:%.*]] = arith.constant true
-!CHECK:      acc.serial if([[IF1]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IF1:%.*]] = arith.constant true
+! CHECK:      acc.serial if([[IF1]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop if(ifCondition)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
-!CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-!CHECK:      acc.serial if([[IF2]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
+! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
+! CHECK:      acc.serial if([[IF2]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop self(.TRUE.)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[SELF1:%.*]] = arith.constant true
-!CHECK:      acc.serial self([[SELF1]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      [[SELF1:%.*]] = arith.constant true
+! CHECK:      acc.serial self([[SELF1]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop self
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: } attributes {selfAttr}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: } attributes {selfAttr}
 
   !$acc serial loop self(ifCondition)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      [[SELF2:%.*]] = fir.convert [[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
-!CHECK:      acc.serial self([[SELF2]]) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[SELF2:.*]] = fir.convert %[[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
+! CHECK:      acc.serial self(%[[SELF2]]) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop copy(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial copy([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "b"}
 
   !$acc serial loop copy(a) copy(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial copy([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {dataClause = 3 : i64, name = "b"}
 
   !$acc serial loop copyin(a) copyin(readonly: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial copyin([[A]] : !fir.ref<!fir.array<10xf32>>) copyin_readonly([[B]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 2 : i64, name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop copyout(a) copyout(zero: b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial copyout([[A]] : !fir.ref<!fir.array<10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 4 : i64, name = "a"} 
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 4 : i64, name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) {name = "a"}
+! CHECK:      acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) {name = "b"}
 
   !$acc serial loop create(b) create(zero: a)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial create([[B]] : !fir.ref<!fir.array<10xf32>>) create_zero([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = 8 : i64, name = "a"}
+! CHECK:      acc.serial dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+! CHECK:      acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = 7 : i64, name = "b"}
+! CHECK:      acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) {dataClause = 8 : i64, name = "a"}
 
   !$acc serial loop no_create(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial no_create([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop present(a, b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial present([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop deviceptr(a) deviceptr(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial deviceptr([[A]], [[B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"} 
+! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"} 
+! CHECK:      acc.serial dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop attach(f, g)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial attach([[F]], [[G]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      %[[BOX_F:.*]] = fir.load %[[F]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_F:.*]] = fir.box_addr %[[BOX_F]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_F:.*]] = acc.attach varPtr(%[[BOX_ADDR_F]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "f"}
+! CHECK:      %[[BOX_G:.*]] = fir.load %[[G]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
+! CHECK:      acc.serial dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop private(a) firstprivate(b)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial firstprivate([[B]] : !fir.ref<!fir.array<10xf32>>) private([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:        acc.loop private([[A]] : !fir.ref<!fir.array<10xf32>>) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial firstprivate(%[[B]] : !fir.ref<!fir.array<10xf32>>) private(%[[A]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:        acc.loop private(%[[A]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop seq
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {seq}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {seq}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop auto
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {auto}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {auto}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop independent
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {independent}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {independent}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop gang
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop gang {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop gang {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop gang(num: 8)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-!CHECK-NEXT:   acc.loop gang(num=[[GANGNUM1]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
+! CHECK-NEXT:   acc.loop gang(num=[[GANGNUM1]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop gang(num: gangNum)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK-NEXT:   acc.loop gang(num=[[GANGNUM2]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK-NEXT:   acc.loop gang(num=[[GANGNUM2]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
  !$acc serial loop gang(num: gangNum, static: gangStatic)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop gang(num=%{{.*}} : i32, static=%{{.*}} : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop vector
   DO i = 1, n
     a(i) = b(i)
   END DO
-!CHECK:      acc.serial {
-!CHECK:        acc.loop vector {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop vector {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop vector(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
-!CHECK:        acc.loop vector([[CONSTANT128]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
+! CHECK:        acc.loop vector([[CONSTANT128]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop vector(vectorLength)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-!CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
+! CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop worker
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop worker {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop worker {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop worker(128)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
-!CHECK:        acc.loop worker([[WORKER128]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
+! CHECK:        acc.loop worker([[WORKER128]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop collapse(2)
   DO i = 1, n
@@ -508,14 +538,14 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:            fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   } attributes {collapse = 2 : i64}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:            fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   } attributes {collapse = 2 : i64}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop
   DO i = 1, n
@@ -525,45 +555,45 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop {
-!CHECK:          fir.do_loop
-!CHECK:            acc.loop {
-!CHECK:              fir.do_loop
-!CHECK:              acc.yield
-!CHECK-NEXT:     }{{$}}
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop {
+! CHECK:          fir.do_loop
+! CHECK:            acc.loop {
+! CHECK:              fir.do_loop
+! CHECK:              acc.yield
+! CHECK-NEXT:     }{{$}}
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
  !$acc serial loop tile(2)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
-!CHECK:        acc.loop tile([[TILESIZE]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
+! CHECK:        acc.loop tile([[TILESIZE]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
  !$acc serial loop tile(*)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
-!CHECK:        acc.loop tile([[TILESIZEM1]] : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
+! CHECK:        acc.loop tile([[TILESIZEM1]] : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop tile(2, 2)
   DO i = 1, n
@@ -572,28 +602,28 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
-!CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
-!CHECK:        acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
+! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
+! CHECK:        acc.loop tile([[TILESIZE1]], [[TILESIZE2]] : i32, i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop tile(tileSize)
   DO i = 1, n
     a(i) = b(i)
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop tile(%{{.*}} : i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop tile(%{{.*}} : i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
   !$acc serial loop tile(tileSize, tileSize)
   DO i = 1, n
@@ -602,12 +632,12 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-!CHECK:      acc.serial {
-!CHECK:        acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) {
-!CHECK:          fir.do_loop
-!CHECK:          acc.yield
-!CHECK-NEXT:   }{{$}}
-!CHECK:        acc.yield
-!CHECK-NEXT: }{{$}}
+! CHECK:      acc.serial {
+! CHECK:        acc.loop tile(%{{.*}}, %{{.*}} : i32, i32) {
+! CHECK:          fir.do_loop
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
 
 end subroutine acc_serial_loop
diff --git a/flang/test/Lower/OpenACC/acc-serial.f90 b/flang/test/Lower/OpenACC/acc-serial.f90
index 5bc19984fef50..96ecf80f28cd5 100644
--- a/flang/test/Lower/OpenACC/acc-serial.f90
+++ b/flang/test/Lower/OpenACC/acc-serial.f90
@@ -15,12 +15,12 @@ subroutine acc_serial
   real, dimension(10, 10) :: a, b, c
   real, pointer :: d, e
 
-! CHECK: [[A:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
-! CHECK: [[B:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
-! CHECK: [[C:%.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
-! CHECK: [[D:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
-! CHECK: [[E:%.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
-! CHECK: [[IFCONDITION:%.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
+! CHECK: %[[A:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ea"}
+! CHECK: %[[B:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Eb"}
+! CHECK: %[[C:.*]] = fir.alloca !fir.array<10x10xf32> {{{.*}}uniq_name = "{{.*}}Ec"}
+! CHECK: %[[D:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "d", uniq_name = "{{.*}}Ed"}
+! CHECK: %[[E:.*]] = fir.alloca !fir.box<!fir.ptr<f32>> {bindc_name = "e", uniq_name = "{{.*}}Ee"}
+! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 
   !$acc serial
   !$acc end serial
@@ -120,78 +120,121 @@ subroutine acc_serial
   !$acc serial self(ifCondition)
   !$acc end serial
 
-! CHECK:      [[SELF2:%.*]] = fir.convert [[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
-! CHECK:      acc.serial self([[SELF2]]) {
+! CHECK:      %[[SELF2:.*]] = fir.convert %[[IFCONDITION]] : (!fir.ref<!fir.logical<4>>) -> i1
+! CHECK:      acc.serial self(%[[SELF2]]) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
   !$acc serial copy(a, b, c)
   !$acc end serial
 
-! CHECK:      acc.serial copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"} 
+! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc serial copy(a) copy(b) copy(c)
   !$acc end serial
 
-! CHECK:      acc.serial copy([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "a"} 
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "b"} 
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 3 : i64, name = "c"} 
+! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "a"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "b"}
+! CHECK:      acc.copyout accPtr(%[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {dataClause = 3 : i64, name = "c"}
 
   !$acc serial copyin(a) copyin(readonly: b, c)
   !$acc end serial
 
-! CHECK:      acc.serial copyin([[A]] : !fir.ref<!fir.array<10x10xf32>>) copyin_readonly([[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "b"} 
+! CHECK:      %[[COPYIN_C:.*]] = acc.copyin varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 2 : i64, name = "c"} 
+! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]], %[[COPYIN_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
   !$acc serial copyout(a) copyout(zero: b) copyout(c)
   !$acc end serial
 
-! CHECK:      acc.serial copyout([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) copyout_zero([[B]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 4 : i64, name = "c"}
+! CHECK:      acc.serial dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.copyout accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) {name = "a"}
+! CHECK: acc.copyout accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) {name = "b"}
+! CHECK: acc.copyout accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) {name = "c"}
 
   !$acc serial create(a, b) create(zero: c)
   !$acc end serial
 
-! CHECK:      acc.serial create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK: %[[CREATE_A:.*]] = acc.create varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[CREATE_B:.*]] = acc.create varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+! CHECK:      acc.serial dataOperands(%[[CREATE_A]], %[[CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.delete accPtr(%[[CREATE_A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "a"}
+! CHECK: acc.delete accPtr(%[[CREATE_B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 7 : i64, name = "b"}
+! CHECK: acc.delete accPtr(%[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) {dataClause = 8 : i64, name = "c"}
 
   !$acc serial no_create(a, b) create(zero: c)
   !$acc end serial
 
-! CHECK:      acc.serial create_zero([[C]] : !fir.ref<!fir.array<10x10xf32>>) no_create([[A]], [[B]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK: %[[NO_CREATE_A:.*]] = acc.nocreate varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[NO_CREATE_B:.*]] = acc.nocreate varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[CREATE_C:.*]] = acc.create varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {dataClause = 8 : i64, name = "c"}
+! CHECK:      acc.serial dataOperands(%[[NO_CREATE_A]], %[[NO_CREATE_B]], %[[CREATE_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
   !$acc serial present(a, b, c)
   !$acc end serial
 
-! CHECK:      acc.serial present([[A]], [[B]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK: %[[PRESENT_A:.*]] = acc.present varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[PRESENT_B:.*]] = acc.present varPtr(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "b"}
+! CHECK: %[[PRESENT_C:.*]] = acc.present varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.serial dataOperands(%[[PRESENT_A]], %[[PRESENT_B]], %[[PRESENT_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
   !$acc serial deviceptr(a) deviceptr(c)
   !$acc end serial
 
-! CHECK:      acc.serial deviceptr([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK: %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[A]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "a"}
+! CHECK: %[[DEVICEPTR_C:.*]] = acc.deviceptr varPtr(%[[C]] : !fir.ref<!fir.array<10x10xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<10x10xf32>> {name = "c"}
+! CHECK:      acc.serial dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
-
+  
   !$acc serial attach(d, e)
   !$acc end serial
 
-! CHECK:      acc.serial attach([[D]], [[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>>, !fir.ref<!fir.box<!fir.ptr<f32>>>) {
+! CHECK: %[[BOX_D:.*]] = fir.load %[[D]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK: %[[BOX_ADDR_D:.*]] = fir.box_addr %[[BOX_D]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK: %[[ATTACH_D:.*]] = acc.attach varPtr(%[[BOX_ADDR_D]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "d"} 
+! CHECK: %[[BOX_E:.*]] = fir.load %[[E]] : !fir.ref<!fir.box<!fir.ptr<f32>>> 
+! CHECK: %[[BOX_ADDR_E:.*]] = fir.box_addr %[[BOX_E]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32> 
+! CHECK: %[[ATTACH_E:.*]] = acc.attach varPtr(%[[BOX_ADDR_E]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "e"} 
+! CHECK:      acc.serial dataOperands(%[[ATTACH_D]], %[[ATTACH_E]] : !fir.ptr<f32>, !fir.ptr<f32>) { 
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
+! CHECK: acc.detach accPtr(%[[ATTACH_D]] : !fir.ptr<f32>) {dataClause = 10 : i64, name = "d"}
+! CHECK: acc.detach accPtr(%[[ATTACH_E]] : !fir.ptr<f32>) {dataClause = 10 : i64, name = "e"}
 
   !$acc serial private(a) firstprivate(b) private(c)
   !$acc end serial
 
-! CHECK:      acc.serial firstprivate([[B]] : !fir.ref<!fir.array<10x10xf32>>) private([[A]], [[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
+! CHECK:      acc.serial firstprivate(%[[B]] : !fir.ref<!fir.array<10x10xf32>>) private(%[[A]], %[[C]] : !fir.ref<!fir.array<10x10xf32>>, !fir.ref<!fir.array<10x10xf32>>) {
 ! CHECK:        acc.yield
 ! CHECK-NEXT: }{{$}}
 
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-iand.f90 b/flang/test/Lower/OpenMP/Todo/reduction-iand.f90
deleted file mode 100644
index f1b9b025d2a16..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/reduction-iand.f90
+++ /dev/null
@@ -1,16 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Reduction of intrinsic procedures is not supported
-subroutine reduction_iand(y)
-  integer :: x, y(:)
-  x = 0
-  !$omp parallel
-  !$omp do reduction(iand:x)
-  do i=1, 100
-    x = iand(x, y(i))
-  end do
-  !$omp end do
-  !$omp end parallel
-  print *, x
-end subroutine
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-ieor.f90 b/flang/test/Lower/OpenMP/Todo/reduction-ieor.f90
deleted file mode 100644
index 2303a8edfaa94..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/reduction-ieor.f90
+++ /dev/null
@@ -1,16 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Reduction of intrinsic procedures is not supported
-subroutine reduction_ieor(y)
-  integer :: x, y(:)
-  x = 0
-  !$omp parallel
-  !$omp do reduction(ieor:x)
-  do i=1, 100
-    x = ieor(x, y(i))
-  end do
-  !$omp end do
-  !$omp end parallel
-  print *, x
-end subroutine
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-ior.f90 b/flang/test/Lower/OpenMP/Todo/reduction-ior.f90
deleted file mode 100644
index 3460df2b9d6d2..0000000000000
--- a/flang/test/Lower/OpenMP/Todo/reduction-ior.f90
+++ /dev/null
@@ -1,16 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Reduction of intrinsic procedures is not supported
-subroutine reduction_ior(y)
-  integer :: x, y(:)
-  x = 0
-  !$omp parallel
-  !$omp do reduction(ior:x)
-  do i=1, 100
-    x = ior(x, y(i))
-  end do
-  !$omp end do
-  !$omp end parallel
-  print *, x
-end subroutine
diff --git a/flang/test/Lower/OpenMP/atomic-capture.f90 b/flang/test/Lower/OpenMP/atomic-capture.f90
new file mode 100644
index 0000000000000..f48a5efaf4354
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-capture.f90
@@ -0,0 +1,117 @@
+! RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | FileCheck %s
+
+! This test checks the lowering of atomic capture
+
+program OmpAtomicCapture
+    use omp_lib                                                                                                       
+    integer :: x, y
+
+!CHECK: %[[X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
+!CHECK: %[[Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
+!CHECK: omp.atomic.capture   memory_order(release) {
+!CHECK: omp.atomic.read %[[X]] = %[[Y]] : !fir.ref<i32>
+!CHECK: omp.atomic.update %[[Y]] : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:.*]]: i32):
+!CHECK: %[[temp:.*]] = fir.load %[[X]] : !fir.ref<i32>
+!CHECK: %[[result:.*]] = arith.addi %[[temp]], %[[ARG]] : i32
+!CHECK: omp.yield(%[[result]] : i32)
+!CHECK: }
+!CHECK: }
+
+    !$omp atomic capture release
+        x = y
+        y = x + y
+    !$omp end atomic
+
+
+!CHECK: omp.atomic.capture   hint(uncontended) {
+!CHECK: omp.atomic.update %[[Y]] : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:.*]]: i32):
+!CHECK: %[[temp:.*]] = fir.load %[[X]] : !fir.ref<i32>
+!CHECK: %[[result:.*]] = arith.muli %[[temp]], %[[ARG]] : i32
+!CHECK: omp.yield(%[[result]] : i32)
+!CHECK: }
+!CHECK: omp.atomic.read %[[X]] = %[[Y]] : !fir.ref<i32>
+!CHECK: }
+
+    !$omp atomic hint(omp_sync_hint_uncontended) capture
+        y = x * y 
+        x = y
+    !$omp end atomic
+
+!CHECK: %[[constant_20:.*]] = arith.constant 20 : i32
+!CHECK: %[[constant_8:.*]] = arith.constant 8 : i32
+!CHECK: %[[temp:.*]] = fir.load %[[X]] : !fir.ref<i32>
+!CHECK: %[[result:.*]] = arith.subi %[[constant_8]], %[[temp]] : i32
+!CHECK: %[[result_noreassoc:.*]] = fir.no_reassoc %[[result]] : i32
+!CHECK: %[[result:.*]] = arith.addi %[[constant_20]], %[[result_noreassoc]] : i32
+!CHECK: omp.atomic.capture   memory_order(acquire) hint(nonspeculative) {
+!CHECK: omp.atomic.read %[[X]] = %[[Y]] : !fir.ref<i32>
+!CHECK: omp.atomic.write %[[Y]] = %[[result]] : !fir.ref<i32>, i32
+!CHECK: }
+
+    !$omp atomic hint(omp_lock_hint_nonspeculative) capture acquire
+        x = y
+        y = 2 * 10 + (8 - x) 
+    !$omp end atomic
+
+
+!CHECK: %[[constant_20:.*]] = arith.constant 20 : i32
+!CHECK: %[[constant_8:.*]] = arith.constant 8 : i32
+!CHECK: %[[temp:.*]] = fir.load %[[X]] : !fir.ref<i32>
+!CHECK: %[[result:.*]] = arith.subi %[[constant_8]], %[[temp]] : i32
+!CHECK: %[[result_noreassoc:.*]] = fir.no_reassoc %[[result]] : i32
+!CHECK: %[[result:.*]] = arith.addi %[[constant_20]], %[[result_noreassoc]] : i32
+!CHECK: omp.atomic.capture {
+!CHECK: omp.atomic.read %[[X]] = %[[Y]] : !fir.ref<i32>
+!CHECK: omp.atomic.write %[[Y]] = %[[result]] : !fir.ref<i32>, i32
+!CHECK: }
+
+    !$omp atomic capture
+        x = y
+        y = 2 * 10 + (8 - x) 
+    !$omp end atomic 
+end program
+
+
+
+subroutine pointers_in_atomic_capture()
+!CHECK: %[[A:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "a", uniq_name = "_QFpointers_in_atomic_captureEa"}
+!CHECK: {{.*}} = fir.zero_bits !fir.ptr<i32>
+!CHECK: {{.*}} = fir.embox {{.*}} : (!fir.ptr<i32>) -> !fir.box<!fir.ptr<i32>>
+!CHECK: fir.store {{.*}} to %[[A]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: %[[B:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "b", uniq_name = "_QFpointers_in_atomic_captureEb"}
+!CHECK: {{.*}} = fir.zero_bits !fir.ptr<i32>
+!CHECK: {{.*}} = fir.embox {{.*}} : (!fir.ptr<i32>) -> !fir.box<!fir.ptr<i32>>
+!CHECK: fir.store {{.*}} to %[[B]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: %[[C:.*]] = fir.alloca i32 {bindc_name = "c", fir.target, uniq_name = "_QFpointers_in_atomic_captureEc"}
+!CHECK: %[[D:.*]] = fir.alloca i32 {bindc_name = "d", fir.target, uniq_name = "_QFpointers_in_atomic_captureEd"}
+!CHECK: {{.*}} = fir.embox {{.*}} : (!fir.ref<i32>) -> !fir.box<!fir.ptr<i32>>
+!CHECK: fir.store {{.*}} to %[[A]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: {{.*}} = fir.embox {{.*}} : (!fir.ref<i32>) -> !fir.box<!fir.ptr<i32>>
+!CHECK: fir.store {{.*}} to %[[B]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: %[[loaded_A:.*]] = fir.load %[[A]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: %[[loaded_A_addr:.*]] = fir.box_addr %[[loaded_A]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK: %[[loaded_B:.*]] = fir.load %[[B]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: %[[loaded_B_addr:.*]] = fir.box_addr %[[loaded_B]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK: omp.atomic.capture   {
+!CHECK: omp.atomic.update %[[loaded_A_addr]] : !fir.ptr<i32> {
+!CHECK: ^bb0(%[[ARG:.*]]: i32):
+!CHECK: %[[PRIVATE_LOADED_B:.*]] = fir.load %[[B]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK: %[[PRIVATE_LOADED_B_addr:.*]] = fir.box_addr %[[PRIVATE_LOADED_B]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK: %[[loaded_value:.*]] = fir.load %[[PRIVATE_LOADED_B_addr]] : !fir.ptr<i32>
+!CHECK: %[[result:.*]] = arith.addi %[[ARG]], %[[loaded_value]] : i32
+!CHECK: omp.yield(%[[result]] : i32)
+!CHECK: }
+!CHECK: omp.atomic.read %[[loaded_B_addr]] = %[[loaded_A_addr]] : !fir.ptr<i32>, i32
+!CHECK: }
+    integer, pointer :: a, b
+    integer, target :: c, d
+    a=>c
+    b=>d
+
+    !$omp atomic capture
+        a = a + b
+        b = a
+    !$omp end atomic
+end subroutine
diff --git a/flang/test/Lower/OpenMP/omp-host-ir-flag.f90 b/flang/test/Lower/OpenMP/omp-host-ir-flag.f90
new file mode 100644
index 0000000000000..f25a024afb1d5
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-host-ir-flag.f90
@@ -0,0 +1,6 @@
+!RUN: %flang_fc1 -emit-llvm-bc -fopenmp -o %t.bc %s 2>&1
+!RUN: %flang_fc1 -emit-mlir -fopenmp -fopenmp-is-device -fopenmp-host-ir-file-path %t.bc -o - %s 2>&1 | FileCheck %s
+
+!CHECK: module attributes {{{.*}}, omp.host_ir_filepath = "{{.*}}.bc", omp.is_device = #omp.isdevice<is_device = true>{{.*}}}
+subroutine omp_subroutine()
+end subroutine omp_subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
index cd0d8786d05f8..9cadd61d6c35a 100644
--- a/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-lastprivate-clause-scalar.f90
@@ -6,6 +6,7 @@
 !CHECK: func @_QPlastprivate_character(%[[ARG1:.*]]: !fir.boxchar<1>{{.*}}) {
 !CHECK-DAG: %[[ARG1_UNBOX:.*]]:2 = fir.unboxchar
 !CHECK-DAG: %[[FIVE:.*]] = arith.constant 5 : index
+!CHECK-DAG: %[[ARG1_REF:.*]] = fir.convert %[[ARG1_UNBOX]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,5>>
 
 !CHECK: omp.parallel {
 !CHECK-DAG: %[[ARG1_PVT:.*]] = fir.alloca !fir.char<1,5> {bindc_name = "arg1", 
@@ -24,10 +25,10 @@
 
 ! Testing last iteration check
 !CHECK-NEXT: %[[IV_CMP:.*]] = arith.cmpi eq, %[[INDX_WS]]
-!CHECK: scf.if %[[IV_CMP]] {
+!CHECK: fir.if %[[IV_CMP]] {
 
 ! Testing lastprivate val update
-!CHECK-DAG: %[[CVT:.*]] = fir.convert %[[ARG1_UNBOX]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+!CHECK-DAG: %[[CVT:.*]] = fir.convert %[[ARG1_REF]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
 !CHECK-DAG: %[[CVT1:.*]] = fir.convert %[[ARG1_PVT]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
 !CHECK-DAG: fir.call @llvm.memmove.p0.p0.i64(%[[CVT]], %[[CVT1]]{{.*}})
 !CHECK-DAG: } 
@@ -52,7 +53,7 @@ subroutine lastprivate_character(arg1)
 
 ! Testing last iteration check
 !CHECK-DAG: %[[IV_CMP:.*]] = arith.cmpi eq, %[[INDX_WS]]
-!CHECK-DAG: scf.if %[[IV_CMP]] {
+!CHECK-DAG: fir.if %[[IV_CMP]] {
 
 ! Testing lastprivate val update
 !CHECK-NEXT: %[[CLONE_LD:.*]] = fir.load %[[CLONE]] : !fir.ref<i32>
@@ -81,7 +82,7 @@ subroutine lastprivate_int(arg1)
 
 ! Testing last iteration check
 !CHECK: %[[IV_CMP1:.*]] = arith.cmpi eq, %[[INDX_WS]]
-!CHECK-NEXT: scf.if %[[IV_CMP1]] {
+!CHECK-NEXT: fir.if %[[IV_CMP1]] {
 ! Testing lastprivate val update
 !CHECK-DAG: %[[CLONE_LD1:.*]] = fir.load %[[CLONE1]] : !fir.ref<i32>
 !CHECK-DAG: fir.store %[[CLONE_LD1]] to %[[ARG1]] : !fir.ref<i32>
@@ -112,7 +113,7 @@ subroutine mult_lastprivate_int(arg1, arg2)
 
 !Testing last iteration check
 !CHECK: %[[IV_CMP1:.*]] = arith.cmpi eq, %[[INDX_WS]]
-!CHECK-NEXT: scf.if %[[IV_CMP1]] {
+!CHECK-NEXT: fir.if %[[IV_CMP1]] {
 !Testing lastprivate val update
 !CHECK-DAG: %[[CLONE_LD2:.*]] = fir.load %[[CLONE2]] : !fir.ref<i32>
 !CHECK-DAG: fir.store %[[CLONE_LD2]] to %[[ARG2]] : !fir.ref<i32>
@@ -148,7 +149,7 @@ subroutine mult_lastprivate_int2(arg1, arg2)
 
 ! Testing last iteration check
 !CHECK: %[[IV_CMP1:.*]] = arith.cmpi eq, %[[INDX_WS]]
-!CHECK-NEXT: scf.if %[[IV_CMP1]] {
+!CHECK-NEXT: fir.if %[[IV_CMP1]] {
 ! Testing lastprivate val update
 !CHECK-NEXT: %[[CLONE_LD:.*]] = fir.load %[[CLONE2]] : !fir.ref<i32>
 !CHECK-NEXT: fir.store %[[CLONE_LD]] to %[[ARG2]] : !fir.ref<i32>
@@ -179,7 +180,7 @@ subroutine firstpriv_lastpriv_int(arg1, arg2)
 !CHECK: omp.wsloop for (%[[INDX_WS:.*]]) : {{.*}} {
 ! Testing last iteration check
 !CHECK: %[[IV_CMP1:.*]] = arith.cmpi eq, %[[INDX_WS]]
-!CHECK-NEXT: scf.if %[[IV_CMP1]] {
+!CHECK-NEXT: fir.if %[[IV_CMP1]] {
 ! Testing lastprivate val update
 !CHECK-NEXT: %[[CLONE_LD:.*]] = fir.load %[[CLONE1]] : !fir.ref<i32>
 !CHECK-NEXT: fir.store %[[CLONE_LD]] to %[[ARG1]] : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index 3bec578c6bd57..02ab64d9fa172 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -131,7 +131,7 @@ subroutine lastprivate()
 !CHECK: %[[const:.*]] = arith.constant 1 : i32
 !CHECK: %[[result:.*]] = arith.addi %[[temp]], %[[const]] : i32
 !CHECK: fir.store %[[result]] to %[[PRIVATE_X]] : !fir.ref<i32>
-!CHECK: scf.if %[[true]] {
+!CHECK: fir.if %[[true]] {
 !CHECK: %[[temp:.*]] = fir.load %[[PRIVATE_X]] : !fir.ref<i32>
 !CHECK: fir.store %[[temp]] to %[[X]] : !fir.ref<i32>
 !CHECK: }
@@ -168,7 +168,7 @@ subroutine lastprivate()
 !CHECK: %[[const:.*]] = arith.constant 1 : i32
 !CHECK: %[[result:.*]] = arith.addi %[[temp]], %[[const]] : i32
 !CHECK: fir.store %[[result]] to %[[PRIVATE_X]] : !fir.ref<i32>
-!CHECK: scf.if %true {
+!CHECK: fir.if %true {
 !CHECK: %[[temp:.*]] = fir.load %[[PRIVATE_X]] : !fir.ref<i32>
 !CHECK: fir.store %[[temp]] to %[[X]] : !fir.ref<i32>
 !CHECK: }
@@ -205,7 +205,7 @@ subroutine lastprivate()
 !CHECK: %[[const:.*]] = arith.constant 1 : i32
 !CHECK: %[[result:.*]] = arith.addi %[[temp]], %[[const]] : i32
 !CHECK: fir.store %[[result]] to %[[PRIVATE_X]] : !fir.ref<i32>
-!CHECK: scf.if %true {
+!CHECK: fir.if %true {
 !CHECK: %[[temp:.*]] = fir.load %[[PRIVATE_X]] : !fir.ref<i32>
 !CHECK: fir.store %[[temp]] to %[[X]] : !fir.ref<i32>
 !CHECK: omp.barrier
@@ -217,4 +217,26 @@ subroutine lastprivate()
 !CHECK: omp.terminator
 !CHECK: }
     !$omp end sections nowait
+
+!CHECK: omp.sections {
+!CHECK: omp.section {
+!CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFlastprivateEx"}
+!CHECK: cf.br ^bb1
+!CHECK: ^bb1:  // pred: ^bb0
+!CHECK: %[[INNER_PRIVATE_X:.*]] = fir.load %[[PRIVATE_X]] : !fir.ref<i32>
+!CHECK: %[[const:.*]] = arith.constant 1 : i32
+!CHECK: %[[result:.*]] = arith.addi %[[INNER_PRIVATE_X]], %[[const]] : i32
+!CHECK: fir.store %[[result]] to %[[PRIVATE_X]] : !fir.ref<i32>
+!CHECK: %[[loaded_value:.*]] = fir.load %[[PRIVATE_X]] : !fir.ref<i32>
+!CHECK: fir.store %[[loaded_value]] to %[[X]] : !fir.ref<i32>
+!CHECK: omp.terminator
+!CHECK: }
+!CHECK: omp.terminator
+!CHECK: }
+
+    !$omp sections lastprivate(x)
+        !$omp section
+                goto 30
+        30  x = x + 1
+    !$omp end sections
 end subroutine
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
new file mode 100644
index 0000000000000..c18971e14ff38
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90
@@ -0,0 +1,34 @@
+! RUN: bbc -emit-fir -fopenmp %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | FileCheck %s
+
+!CHECK: omp.reduction.declare @[[IAND_DECLARE_I:.*]] : i32 init {
+!CHECK: %[[ZERO_VAL_I:.*]] = arith.constant -1 : i32
+!CHECK: omp.yield(%[[ZERO_VAL_I]] : i32)
+!CHECK: combiner
+!CHECK: ^bb0(%[[ARG0_I:.*]]: i32, %[[ARG1_I:.*]]: i32):
+!CHECK: %[[IAND_VAL_I:.*]] = arith.andi %[[ARG0_I]], %[[ARG1_I]] : i32
+!CHECK: omp.yield(%[[IAND_VAL_I]] : i32)
+
+!CHECK-LABEL: @_QPreduction_iand
+!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box<!fir.array<?xi32>> 
+!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iandEx"}
+!CHECK: omp.parallel
+!CHECK: omp.wsloop reduction(@[[IAND_DECLARE_I]] -> %[[X_REF]] : !fir.ref<i32>) for 
+!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]]
+!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref<i32>
+!CHECK: omp.reduction %[[Y_I]], %[[X_REF]] : i32, !fir.ref<i32>
+!CHECK: omp.yield
+!CHECK: omp.terminator
+
+subroutine reduction_iand(y)
+  integer :: x, y(:)
+  x = 0
+  !$omp parallel
+  !$omp do reduction(iand:x)
+  do i=1, 100
+    x = iand(x, y(i))
+  end do
+  !$omp end do
+  !$omp end parallel
+  print *, x
+end subroutine
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
new file mode 100644
index 0000000000000..93d1539bc63a9
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor.f90
@@ -0,0 +1,34 @@
+! RUN: bbc -emit-fir -fopenmp %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | FileCheck %s
+
+!CHECK: omp.reduction.declare @[[IEOR_DECLARE_I:.*]] : i32 init {
+!CHECK: %[[ZERO_VAL_I:.*]] = arith.constant 0 : i32
+!CHECK: omp.yield(%[[ZERO_VAL_I]] : i32)
+!CHECK: combiner
+!CHECK: ^bb0(%[[ARG0_I:.*]]: i32, %[[ARG1_I:.*]]: i32):
+!CHECK: %[[IEOR_VAL_I:.*]] = arith.xori %[[ARG0_I]], %[[ARG1_I]] : i32
+!CHECK: omp.yield(%[[IEOR_VAL_I]] : i32)
+
+!CHECK-LABEL: @_QPreduction_ieor
+!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box<!fir.array<?xi32>> 
+!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_ieorEx"}
+!CHECK: omp.parallel
+!CHECK: omp.wsloop reduction(@[[IEOR_DECLARE_I]] -> %[[X_REF]] : !fir.ref<i32>) for 
+!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]]
+!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref<i32>
+!CHECK: omp.reduction %[[Y_I]], %[[X_REF]] : i32, !fir.ref<i32>
+!CHECK: omp.yield
+!CHECK: omp.terminator
+
+subroutine reduction_ieor(y)
+  integer :: x, y(:)
+  x = 0
+  !$omp parallel
+  !$omp do reduction(ieor:x)
+  do i=1, 100
+    x = ieor(x, y(i))
+  end do
+  !$omp end do
+  !$omp end parallel
+  print *, x
+end subroutine
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
new file mode 100644
index 0000000000000..87e77c265fe1a
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90
@@ -0,0 +1,34 @@
+! RUN: bbc -emit-fir -fopenmp %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | FileCheck %s
+
+!CHECK: omp.reduction.declare @[[IOR_DECLARE_I:.*]] : i32 init {
+!CHECK: %[[ZERO_VAL_I:.*]] = arith.constant 0 : i32
+!CHECK: omp.yield(%[[ZERO_VAL_I]] : i32)
+!CHECK: combiner
+!CHECK: ^bb0(%[[ARG0_I:.*]]: i32, %[[ARG1_I:.*]]: i32):
+!CHECK: %[[IOR_VAL_I:.*]] = arith.ori %[[ARG0_I]], %[[ARG1_I]] : i32
+!CHECK: omp.yield(%[[IOR_VAL_I]] : i32)
+
+!CHECK-LABEL: @_QPreduction_ior
+!CHECK-SAME: %[[Y_BOX:.*]]: !fir.box<!fir.array<?xi32>> 
+!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iorEx"}
+!CHECK: omp.parallel
+!CHECK: omp.wsloop reduction(@[[IOR_DECLARE_I]] -> %[[X_REF]] : !fir.ref<i32>) for 
+!CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]]
+!CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref<i32>
+!CHECK: omp.reduction %[[Y_I]], %[[X_REF]] : i32, !fir.ref<i32>
+!CHECK: omp.yield
+!CHECK: omp.terminator
+
+subroutine reduction_ior(y)
+  integer :: x, y(:)
+  x = 0
+  !$omp parallel
+  !$omp do reduction(ior:x)
+  do i=1, 100
+    x = ior(x, y(i))
+  end do
+  !$omp end do
+  !$omp end parallel
+  print *, x
+end subroutine
diff --git a/flang/test/Lower/array-elemental-calls-char.f90 b/flang/test/Lower/array-elemental-calls-char.f90
index 4d11cabf42a8e..15a00d6fe2724 100644
--- a/flang/test/Lower/array-elemental-calls-char.f90
+++ b/flang/test/Lower/array-elemental-calls-char.f90
@@ -105,31 +105,34 @@ subroutine foo2(i, j, c)
   i = elem2(c, j)
 end subroutine
 
-! CHECK-LABEL: func @_QMchar_elemPfoo2b(
-! CHECK-SAME: %[[VAL_67:[^:]+]]: !fir.ref<!fir.array<10xi32>>{{.*}}, %[[VAL_64:[^:]+]]: !fir.ref<!fir.array<10xi32>>{{.*}}, %[[VAL_56:.*]]: !fir.boxchar<1>{{.*}}) {
+! CHECK-LABEL:   func.func @_QMchar_elemPfoo2b(
+! CHECK-SAME:        %[[VAL_0:.*]]: !fir.ref<!fir.array<10xi32>>{{.*}}, %[[VAL_1:.*]]: !fir.ref<!fir.array<10xi32>>{{.*}}, %[[VAL_2:.*]]: !fir.boxchar<1>{{.*}}) {
 subroutine foo2b(i, j, c)
   integer :: i(10), j(10)
   character(10) :: c
-! CHECK-DAG:   %[[VAL_52:.*]] = arith.constant 10 : index
-! CHECK-DAG:   %[[VAL_53:.*]] = arith.constant 0 : index
-! CHECK-DAG:   %[[VAL_54:.*]] = arith.constant 1 : index
-! CHECK:   %[[VAL_55:.*]]:2 = fir.unboxchar %[[VAL_56]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:   %[[VAL_57:.*]] = fir.shape %[[VAL_52]] : (index) -> !fir.shape<1>
-! CHECK:   br ^bb1(%[[VAL_53]], %[[VAL_52]] : index, index)
-! CHECK: ^bb1(%[[VAL_58:.*]]: index, %[[VAL_59:.*]]: index):
-! CHECK:   %[[VAL_60:.*]] = arith.cmpi sgt, %[[VAL_59]], %[[VAL_53]] : index
-! CHECK:   cond_br %[[VAL_60]], ^bb2, ^bb3
-! CHECK: ^bb2:
-! CHECK:   %[[VAL_61:.*]] = fir.emboxchar %[[VAL_55]]#0, %[[VAL_52]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-! CHECK:   %[[VAL_62:.*]] = arith.addi %[[VAL_58]], %[[VAL_54]] : index
-! CHECK:   %[[VAL_63:.*]] = fir.array_coor %[[VAL_64]](%[[VAL_57]]) %[[VAL_62]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
-! CHECK:   %[[VAL_65:.*]] = fir.call @_QPelem2(%[[VAL_61]], %[[VAL_63]]) {{.*}}: (!fir.boxchar<1>, !fir.ref<i32>) -> i32
-! CHECK:   %[[VAL_66:.*]] = fir.array_coor %[[VAL_67]](%[[VAL_57]]) %[[VAL_62]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
-! CHECK:   fir.store %[[VAL_65]] to %[[VAL_66]] : !fir.ref<i32>
-! CHECK:   %[[VAL_68:.*]] = arith.subi %[[VAL_59]], %[[VAL_54]] : index
-! CHECK:   br ^bb1(%[[VAL_62]], %[[VAL_68]] : index, index)
-! CHECK: ^bb3:
-! CHECK:   return
+! CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 10 : index
+! CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : index
+! CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[VAL_2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
+! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_3]] : (index) -> !fir.shape<1>
+! CHECK:           cf.br ^bb1(%[[VAL_4]], %[[VAL_3]] : index, index)
+! CHECK:         ^bb1(%[[VAL_9:.*]]: index, %[[VAL_10:.*]]: index):
+! CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_10]], %[[VAL_4]] : index
+! CHECK:           cf.cond_br %[[VAL_11]], ^bb2, ^bb3
+! CHECK:         ^bb2:
+! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_13:.*]] = fir.emboxchar %[[VAL_12]], %[[VAL_3]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_9]], %[[VAL_5]] : index
+! CHECK:           %[[VAL_15:.*]] = fir.array_coor %[[VAL_1]](%[[VAL_8]]) %[[VAL_14]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
+! CHECK:           %[[VAL_16:.*]] = fir.call @_QPelem2(%[[VAL_13]], %[[VAL_15]]) fastmath<contract> : (!fir.boxchar<1>, !fir.ref<i32>) -> i32
+! CHECK:           %[[VAL_17:.*]] = fir.array_coor %[[VAL_0]](%[[VAL_8]]) %[[VAL_14]] : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
+! CHECK:           fir.store %[[VAL_16]] to %[[VAL_17]] : !fir.ref<i32>
+! CHECK:           %[[VAL_18:.*]] = arith.subi %[[VAL_10]], %[[VAL_5]] : index
+! CHECK:           cf.br ^bb1(%[[VAL_14]], %[[VAL_18]] : index, index)
+! CHECK:         ^bb3:
+! CHECK:           return
+! CHECK:         }
   i = elem2(c, j)
 end subroutine
 
diff --git a/flang/test/Lower/call-by-value.f90 b/flang/test/Lower/call-by-value.f90
index 8fa8def0f3d99..de11a27d86c34 100644
--- a/flang/test/Lower/call-by-value.f90
+++ b/flang/test/Lower/call-by-value.f90
@@ -62,14 +62,16 @@ subroutine test_complex_value(x) bind(c)
   call internal_call3(x)
 end
 
-! CHECK-LABEL: func.func @test_char_value(
-! CHECK-SAME:                             %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test_char_value"} {
-! CHECK:         %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:         %[[VAL_2:.*]] = arith.constant 1 : index
-! CHECK:         %[[VAL_3:.*]] = fir.emboxchar %[[VAL_1]]#0, %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-! CHECK:         fir.call @_QPinternal_call4(%[[VAL_3]]) {{.*}}: (!fir.boxchar<1>) -> ()
-! CHECK:         return
-! CHECK:       }
+! CHECK-LABEL:   func.func @test_char_value(
+! CHECK-SAME:                               %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test_char_value"} {
+! CHECK:           %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_5:.*]] = fir.emboxchar %[[VAL_4]], %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           fir.call @_QPinternal_call4(%[[VAL_5]]) fastmath<contract> : (!fir.boxchar<1>) -> ()
+! CHECK:           return
+! CHECK:         }
 
 subroutine test_char_value(x) bind(c)
   character(1), value :: x
diff --git a/flang/test/Lower/call-parenthesized-arg.f90 b/flang/test/Lower/call-parenthesized-arg.f90
index 84c5f9404fcae..de5b4d7658dfc 100644
--- a/flang/test/Lower/call-parenthesized-arg.f90
+++ b/flang/test/Lower/call-parenthesized-arg.f90
@@ -22,25 +22,27 @@ subroutine foo_num_scalar(x)
 ! CHECK-SAME:         %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}) {
 subroutine foo_char_scalar(x)
   character(5) :: x
-! CHECK:         %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:         %[[VAL_2:.*]] = arith.constant 5 : index
-! CHECK:         %[[VAL_3:.*]] = fir.emboxchar %[[VAL_1]]#0, %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-! CHECK:         fir.call @_QPbar_char_scalar(%[[VAL_3]]) {{.*}}: (!fir.boxchar<1>) -> ()
+! CHECK:           %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_2:.*]] = arith.constant 5 : index
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,5>>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_5:.*]] = fir.emboxchar %[[VAL_4]], %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           fir.call @_QPbar_char_scalar(%[[VAL_5]]) fastmath<contract> : (!fir.boxchar<1>) -> ()
   call bar_char_scalar(x)
-! CHECK:         %[[VAL_4:.*]] = fir.no_reassoc %[[VAL_1]]#0 : !fir.ref<!fir.char<1,?>>
-! CHECK:         %[[VAL_5:.*]] = fir.alloca !fir.char<1,5> {bindc_name = ".chrtmp"}
-! CHECK:         %[[VAL_6:.*]] = arith.constant 1 : i64
-! CHECK:         %[[VAL_7:.*]] = fir.convert %[[VAL_2]] : (index) -> i64
-! CHECK:         %[[VAL_8:.*]] = arith.muli %[[VAL_6]], %[[VAL_7]] : i64
-! CHECK:         %[[VAL_9:.*]] = arith.constant false
-! CHECK:         %[[VAL_10:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
-! CHECK:         %[[VAL_11:.*]] = fir.convert %[[VAL_4]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-! CHECK:         fir.call @llvm.memmove.p0.p0.i64(%[[VAL_10]], %[[VAL_11]], %[[VAL_8]], %[[VAL_9]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
-! CHECK:         %[[VAL_12:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<!fir.char<1,?>>
-! CHECK:         %[[VAL_13:.*]] = fir.emboxchar %[[VAL_12]], %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
-! CHECK:         fir.call @_QPbar_char_scalar(%[[VAL_13]]) {{.*}}: (!fir.boxchar<1>) -> ()
-! CHECK:         return
-! CHECK:       }
+! CHECK:           %[[VAL_6:.*]] = fir.no_reassoc %[[VAL_3]] : !fir.ref<!fir.char<1,5>>
+! CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.char<1,5> {bindc_name = ".chrtmp"}
+! CHECK:           %[[VAL_8:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_2]] : (index) -> i64
+! CHECK:           %[[VAL_10:.*]] = arith.muli %[[VAL_8]], %[[VAL_9]] : i64
+! CHECK:           %[[VAL_11:.*]] = arith.constant false
+! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
+! CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<i8>
+! CHECK:           fir.call @llvm.memmove.p0.p0.i64(%[[VAL_12]], %[[VAL_13]], %[[VAL_10]], %[[VAL_11]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
+! CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.char<1,5>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_15:.*]] = fir.emboxchar %[[VAL_14]], %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           fir.call @_QPbar_char_scalar(%[[VAL_15]]) fastmath<contract> : (!fir.boxchar<1>) -> ()
+! CHECK:           return
+! CHECK:         }
   call bar_char_scalar((x))
 end subroutine
 
diff --git a/flang/test/Lower/character-substrings.f90 b/flang/test/Lower/character-substrings.f90
index 9015af1258b8e..7b0a303e39595 100644
--- a/flang/test/Lower/character-substrings.f90
+++ b/flang/test/Lower/character-substrings.f90
@@ -69,61 +69,63 @@ end subroutine array_substring_embox
 ! CHECK-LABEL: func @_QPsubstring_assignment(
 ! CHECK-SAME:        %[[VAL_0:.*]]: !fir.boxchar<1>{{.*}}, %[[VAL_1:.*]]: !fir.boxchar<1>{{.*}}) {
 subroutine substring_assignment(a,b)
-  ! CHECK:         %[[VAL_2:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-  ! CHECK:         %[[VAL_3:.*]]:2 = fir.unboxchar %[[VAL_1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-  ! CHECK:         %[[VAL_4:.*]] = arith.constant 3 : i64
-  ! CHECK:         %[[VAL_5:.*]] = arith.constant 4 : i64
-  ! CHECK:         %[[VAL_6:.*]] = fir.convert %[[VAL_4]] : (i64) -> index
-  ! CHECK:         %[[VAL_7:.*]] = fir.convert %[[VAL_5]] : (i64) -> index
-  ! CHECK:         %[[VAL_8:.*]] = arith.constant 1 : index
-  ! CHECK:         %[[VAL_9:.*]] = arith.subi %[[VAL_6]], %[[VAL_8]] : index
-  ! CHECK:         %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
-  ! CHECK:         %[[VAL_11:.*]] = fir.coordinate_of %[[VAL_10]], %[[VAL_9]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-  ! CHECK:         %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
-  ! CHECK:         %[[VAL_13:.*]] = arith.subi %[[VAL_7]], %[[VAL_6]] : index
-  ! CHECK:         %[[VAL_14:.*]] = arith.addi %[[VAL_13]], %[[VAL_8]] : index
-  ! CHECK:         %[[VAL_15:.*]] = arith.constant 0 : index
-  ! CHECK:         %[[VAL_16:.*]] = arith.cmpi slt, %[[VAL_14]], %[[VAL_15]] : index
-  ! CHECK:         %[[VAL_17:.*]] = arith.select %[[VAL_16]], %[[VAL_15]], %[[VAL_14]] : index
-  ! CHECK:         %[[VAL_18:.*]] = arith.constant 1 : i64
-  ! CHECK:         %[[VAL_19:.*]] = arith.constant 2 : i64
-  ! CHECK:         %[[VAL_20:.*]] = fir.convert %[[VAL_18]] : (i64) -> index
-  ! CHECK:         %[[VAL_21:.*]] = fir.convert %[[VAL_19]] : (i64) -> index
-  ! CHECK:         %[[VAL_22:.*]] = arith.constant 1 : index
-  ! CHECK:         %[[VAL_23:.*]] = arith.subi %[[VAL_20]], %[[VAL_22]] : index
-  ! CHECK:         %[[VAL_24:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
-  ! CHECK:         %[[VAL_25:.*]] = fir.coordinate_of %[[VAL_24]], %[[VAL_23]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-  ! CHECK:         %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
-  ! CHECK:         %[[VAL_27:.*]] = arith.subi %[[VAL_21]], %[[VAL_20]] : index
-  ! CHECK:         %[[VAL_28:.*]] = arith.addi %[[VAL_27]], %[[VAL_22]] : index
-  ! CHECK:         %[[VAL_29:.*]] = arith.constant 0 : index
-  ! CHECK:         %[[VAL_30:.*]] = arith.cmpi slt, %[[VAL_28]], %[[VAL_29]] : index
-  ! CHECK:         %[[VAL_31:.*]] = arith.select %[[VAL_30]], %[[VAL_29]], %[[VAL_28]] : index
-  ! CHECK:         %[[VAL_32:.*]] = arith.cmpi slt, %[[VAL_31]], %[[VAL_17]] : index
-  ! CHECK:         %[[VAL_33:.*]] = arith.select %[[VAL_32]], %[[VAL_31]], %[[VAL_17]] : index
-  ! CHECK:         %[[VAL_34:.*]] = arith.constant 1 : i64
-  ! CHECK:         %[[VAL_35:.*]] = fir.convert %[[VAL_33]] : (index) -> i64
-  ! CHECK:         %[[VAL_36:.*]] = arith.muli %[[VAL_34]], %[[VAL_35]] : i64
-  ! CHECK:         %[[VAL_37:.*]] = arith.constant false
-  ! CHECK:         %[[VAL_38:.*]] = fir.convert %[[VAL_26]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-  ! CHECK:         %[[VAL_39:.*]] = fir.convert %[[VAL_12]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-  ! CHECK:         fir.call @llvm.memmove.p0.p0.i64(%[[VAL_38]], %[[VAL_39]], %[[VAL_36]], %[[VAL_37]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
-  ! CHECK:         %[[VAL_40:.*]] = arith.constant 1 : index
-  ! CHECK:         %[[VAL_41:.*]] = arith.subi %[[VAL_31]], %[[VAL_40]] : index
-  ! CHECK:         %[[VAL_42:.*]] = arith.constant 32 : i8
-  ! CHECK:         %[[VAL_43:.*]] = fir.undefined !fir.char<1>
-  ! CHECK:         %[[VAL_44:.*]] = fir.insert_value %[[VAL_43]], %[[VAL_42]], [0 : index] : (!fir.char<1>, i8) -> !fir.char<1>
-  ! CHECK:         %[[VAL_45:.*]] = arith.constant 1 : index
-  ! CHECK:         fir.do_loop %[[VAL_46:.*]] = %[[VAL_33]] to %[[VAL_41]] step %[[VAL_45]] {
-  ! CHECK:           %[[VAL_47:.*]] = fir.convert %[[VAL_26]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
-  ! CHECK:           %[[VAL_48:.*]] = fir.coordinate_of %[[VAL_47]], %[[VAL_46]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-  ! CHECK:           fir.store %[[VAL_44]] to %[[VAL_48]] : !fir.ref<!fir.char<1>>
-  ! CHECK:         }
+! CHECK:           %[[VAL_2:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
+! CHECK:           %[[VAL_4:.*]]:2 = fir.unboxchar %[[VAL_1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
+! CHECK:           %[[VAL_6:.*]] = arith.constant 3 : i64
+! CHECK:           %[[VAL_7:.*]] = arith.constant 4 : i64
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_6]] : (i64) -> index
+! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_7]] : (i64) -> index
+! CHECK:           %[[VAL_10:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_11:.*]] = arith.subi %[[VAL_8]], %[[VAL_10]] : index
+! CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<!fir.array<4x!fir.char<1>>>
+! CHECK:           %[[VAL_13:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_11]] : (!fir.ref<!fir.array<4x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_15:.*]] = arith.subi %[[VAL_9]], %[[VAL_8]] : index
+! CHECK:           %[[VAL_16:.*]] = arith.addi %[[VAL_15]], %[[VAL_10]] : index
+! CHECK:           %[[VAL_17:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_18:.*]] = arith.cmpi slt, %[[VAL_16]], %[[VAL_17]] : index
+! CHECK:           %[[VAL_19:.*]] = arith.select %[[VAL_18]], %[[VAL_17]], %[[VAL_16]] : index
+! CHECK:           %[[VAL_20:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_21:.*]] = arith.constant 2 : i64
+! CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_20]] : (i64) -> index
+! CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_21]] : (i64) -> index
+! CHECK:           %[[VAL_24:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_25:.*]] = arith.subi %[[VAL_22]], %[[VAL_24]] : index
+! CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<!fir.array<4x!fir.char<1>>>
+! CHECK:           %[[VAL_27:.*]] = fir.coordinate_of %[[VAL_26]], %[[VAL_25]] : (!fir.ref<!fir.array<4x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_27]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:           %[[VAL_29:.*]] = arith.subi %[[VAL_23]], %[[VAL_22]] : index
+! CHECK:           %[[VAL_30:.*]] = arith.addi %[[VAL_29]], %[[VAL_24]] : index
+! CHECK:           %[[VAL_31:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_32:.*]] = arith.cmpi slt, %[[VAL_30]], %[[VAL_31]] : index
+! CHECK:           %[[VAL_33:.*]] = arith.select %[[VAL_32]], %[[VAL_31]], %[[VAL_30]] : index
+! CHECK:           %[[VAL_34:.*]] = arith.cmpi slt, %[[VAL_33]], %[[VAL_19]] : index
+! CHECK:           %[[VAL_35:.*]] = arith.select %[[VAL_34]], %[[VAL_33]], %[[VAL_19]] : index
+! CHECK:           %[[VAL_36:.*]] = arith.constant 1 : i64
+! CHECK:           %[[VAL_37:.*]] = fir.convert %[[VAL_35]] : (index) -> i64
+! CHECK:           %[[VAL_38:.*]] = arith.muli %[[VAL_36]], %[[VAL_37]] : i64
+! CHECK:           %[[VAL_39:.*]] = arith.constant false
+! CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_28]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! CHECK:           %[[VAL_41:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! CHECK:           fir.call @llvm.memmove.p0.p0.i64(%[[VAL_40]], %[[VAL_41]], %[[VAL_38]], %[[VAL_39]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
+! CHECK:           %[[VAL_42:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_43:.*]] = arith.subi %[[VAL_33]], %[[VAL_42]] : index
+! CHECK:           %[[VAL_44:.*]] = arith.constant 32 : i8
+! CHECK:           %[[VAL_45:.*]] = fir.undefined !fir.char<1>
+! CHECK:           %[[VAL_46:.*]] = fir.insert_value %[[VAL_45]], %[[VAL_44]], [0 : index] : (!fir.char<1>, i8) -> !fir.char<1>
+! CHECK:           %[[VAL_47:.*]] = arith.constant 1 : index
+! CHECK:           fir.do_loop %[[VAL_48:.*]] = %[[VAL_35]] to %[[VAL_43]] step %[[VAL_47]] {
+! CHECK:             %[[VAL_49:.*]] = fir.convert %[[VAL_28]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
+! CHECK:             %[[VAL_50:.*]] = fir.coordinate_of %[[VAL_49]], %[[VAL_48]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! CHECK:             fir.store %[[VAL_46]] to %[[VAL_50]] : !fir.ref<!fir.char<1>>
+! CHECK:           }
+! CHECK:           return
+! CHECK:         }
   
   character(4) :: a, b
   a(1:2) = b(3:4)
-  ! CHECK:         return
-  ! CHECK:       }
 end subroutine substring_assignment
 
 ! CHECK-LABEL: func @_QParray_substring_assignment(
diff --git a/flang/test/Lower/complex-real.f90 b/flang/test/Lower/complex-real.f90
new file mode 100644
index 0000000000000..72bd3cf300970
--- /dev/null
+++ b/flang/test/Lower/complex-real.f90
@@ -0,0 +1,15 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK-LABEL: @_QPsb_complex_bfloat
+! CHECK: %[[C_REF:.*]] = fir.alloca !fir.complex<3> {bindc_name = "c", uniq_name = "_QFsb_complex_bfloatEc"}
+! CHECK: %[[R_REF:.*]] = fir.alloca bf16 {bindc_name = "r", uniq_name = "_QFsb_complex_bfloatEr"}
+! CHECK: %[[R_VAL:.*]] = fir.load %[[R_REF]] : !fir.ref<bf16>
+! CHECK: %[[C0:.*]] = arith.constant 0 : i32
+! CHECK: %[[CREAL_REF:.*]] = fir.coordinate_of %[[C_REF]], %[[C0]] : (!fir.ref<!fir.complex<3>>, i32) -> !fir.ref<bf16>
+! CHECK: fir.store %[[R_VAL]] to %[[CREAL_REF]] : !fir.ref<bf16>
+subroutine sb_complex_bfloat
+  complex(kind=3) :: c
+  real(kind=3) :: r
+  c%re = r
+end subroutine
diff --git a/flang/test/Lower/dummy-argument-optional.f90 b/flang/test/Lower/dummy-argument-optional.f90
index 749df6277addf..351a1977abd67 100644
--- a/flang/test/Lower/dummy-argument-optional.f90
+++ b/flang/test/Lower/dummy-argument-optional.f90
@@ -52,8 +52,9 @@ subroutine call_intrinsic_f77_array()
 ! CHECK-SAME: %[[arg0:.*]]: !fir.boxchar<1> {fir.bindc_name = "x", fir.optional}) {
 subroutine character_scalar(x)
   ! CHECK: %[[unboxed:.*]]:2 = fir.unboxchar %[[arg0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  ! CHECK: %[[ref:.*]] = fir.convert %[[unboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
   character(10), optional :: x
-  ! CHECK: fir.is_present %[[unboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> i1
+  ! CHECK: fir.is_present %[[ref]] : (!fir.ref<!fir.char<1,10>>) -> i1
   print *, present(x)
 end subroutine
 ! CHECK-LABEL: func @_QMoptPcall_character_scalar()
diff --git a/flang/test/Lower/forall/scalar-substring.f90 b/flang/test/Lower/forall/scalar-substring.f90
index 340c283eceb8e..55734491bd761 100644
--- a/flang/test/Lower/forall/scalar-substring.f90
+++ b/flang/test/Lower/forall/scalar-substring.f90
@@ -7,75 +7,75 @@ subroutine s(ch)
   end forall
 end subroutine s
 
-! CHECK-LABEL: func @_QPs(
-! CHECK-SAME:    %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "ch"}) {
-! CHECK:         %[[VAL_1:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}
-! CHECK:         %[[VAL_2:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:         %[[VAL_3:.*]] = arith.constant 1 : i32
-! CHECK:         %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i32) -> index
-! CHECK:         %[[VAL_5:.*]] = arith.constant 4 : i32
-! CHECK:         %[[VAL_6:.*]] = fir.convert %[[VAL_5]] : (i32) -> index
-! CHECK:         %[[VAL_7:.*]] = arith.constant 1 : index
-! CHECK:         fir.do_loop %[[VAL_8:.*]] = %[[VAL_4]] to %[[VAL_6]] step %[[VAL_7]] unordered {
-! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (index) -> i32
-! CHECK:           fir.store %[[VAL_9]] to %[[VAL_1]] : !fir.ref<i32>
-! CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
-! CHECK:           %[[VAL_11:.*]] = arith.constant 1 : i32
-! CHECK:           %[[VAL_12:.*]] = arith.addi %[[VAL_10]], %[[VAL_11]] : i32
-! CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> i64
-! CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
-! CHECK:           %[[VAL_15:.*]] = arith.constant 1 : i32
-! CHECK:           %[[VAL_16:.*]] = arith.addi %[[VAL_14]], %[[VAL_15]] : i32
-! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_16]] : (i32) -> i64
-! CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_13]] : (i64) -> index
-! CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_17]] : (i64) -> index
-! CHECK:           %[[VAL_20:.*]] = arith.constant 1 : index
-! CHECK:           %[[VAL_21:.*]] = arith.subi %[[VAL_18]], %[[VAL_20]] : index
-! CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
-! CHECK:           %[[VAL_23:.*]] = fir.coordinate_of %[[VAL_22]], %[[VAL_21]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_23]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
-! CHECK:           %[[VAL_25:.*]] = arith.subi %[[VAL_19]], %[[VAL_18]] : index
-! CHECK:           %[[VAL_26:.*]] = arith.addi %[[VAL_25]], %[[VAL_20]] : index
-! CHECK:           %[[VAL_27:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_28:.*]] = arith.cmpi slt, %[[VAL_26]], %[[VAL_27]] : index
-! CHECK:           %[[VAL_29:.*]] = arith.select %[[VAL_28]], %[[VAL_27]], %[[VAL_26]] : index
-! CHECK:           %[[VAL_30:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
-! CHECK:           %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (i32) -> i64
-! CHECK:           %[[VAL_32:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
-! CHECK:           %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (i32) -> i64
-! CHECK:           %[[VAL_34:.*]] = fir.convert %[[VAL_31]] : (i64) -> index
-! CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_33]] : (i64) -> index
-! CHECK:           %[[VAL_36:.*]] = arith.constant 1 : index
-! CHECK:           %[[VAL_37:.*]] = arith.subi %[[VAL_34]], %[[VAL_36]] : index
-! CHECK:           %[[VAL_38:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
-! CHECK:           %[[VAL_39:.*]] = fir.coordinate_of %[[VAL_38]], %[[VAL_37]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
-! CHECK:           %[[VAL_41:.*]] = arith.subi %[[VAL_35]], %[[VAL_34]] : index
-! CHECK:           %[[VAL_42:.*]] = arith.addi %[[VAL_41]], %[[VAL_36]] : index
-! CHECK:           %[[VAL_43:.*]] = arith.constant 0 : index
-! CHECK:           %[[VAL_44:.*]] = arith.cmpi slt, %[[VAL_42]], %[[VAL_43]] : index
-! CHECK:           %[[VAL_45:.*]] = arith.select %[[VAL_44]], %[[VAL_43]], %[[VAL_42]] : index
-! CHECK:           %[[VAL_46:.*]] = arith.cmpi slt, %[[VAL_45]], %[[VAL_29]] : index
-! CHECK:           %[[VAL_47:.*]] = arith.select %[[VAL_46]], %[[VAL_45]], %[[VAL_29]] : index
-! CHECK:           %[[VAL_48:.*]] = arith.constant 1 : i64
-! CHECK:           %[[VAL_49:.*]] = fir.convert %[[VAL_47]] : (index) -> i64
-! CHECK:           %[[VAL_50:.*]] = arith.muli %[[VAL_48]], %[[VAL_49]] : i64
-! CHECK:           %[[VAL_51:.*]] = arith.constant false
-! CHECK:           %[[VAL_52:.*]] = fir.convert %[[VAL_40]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-! CHECK:           %[[VAL_53:.*]] = fir.convert %[[VAL_24]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
-! CHECK:           fir.call @llvm.memmove.p0.p0.i64(%[[VAL_52]], %[[VAL_53]], %[[VAL_50]], %[[VAL_51]]) {{.*}}: (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
-! CHECK:           %[[VAL_54:.*]] = arith.constant 1 : index
-! CHECK:           %[[VAL_55:.*]] = arith.subi %[[VAL_45]], %[[VAL_54]] : index
-! CHECK:           %[[VAL_56:.*]] = arith.constant 32 : i8
-! CHECK:           %[[VAL_57:.*]] = fir.undefined !fir.char<1>
-! CHECK:           %[[VAL_58:.*]] = fir.insert_value %[[VAL_57]], %[[VAL_56]], [0 : index] : (!fir.char<1>, i8) -> !fir.char<1>
-! CHECK:           %[[VAL_59:.*]] = arith.constant 1 : index
-! CHECK:           fir.do_loop %[[VAL_60:.*]] = %[[VAL_47]] to %[[VAL_55]] step %[[VAL_59]] {
-! CHECK:             %[[VAL_61:.*]] = fir.convert %[[VAL_40]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
-! CHECK:             %[[VAL_62:.*]] = fir.coordinate_of %[[VAL_61]], %[[VAL_60]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
-! CHECK:             fir.store %[[VAL_58]] to %[[VAL_62]] : !fir.ref<!fir.char<1>>
+! CHECK-LABEL:   func.func @_QPs(
+! CHECK-SAME:                    %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "ch"}) {
+! CHECK:           %[[VAL_1:.*]] = fir.alloca i32 {adapt.valuebyref, bindc_name = "i"}
+! CHECK:           %[[VAL_2:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
+! CHECK:           %[[VAL_4:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i32) -> index
+! CHECK:           %[[VAL_6:.*]] = arith.constant 4 : i32
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i32) -> index
+! CHECK:           %[[VAL_8:.*]] = arith.constant 1 : index
+! CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_5]] to %[[VAL_7]] step %[[VAL_8]] unordered {
+! CHECK:             %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (index) -> i32
+! CHECK:             fir.store %[[VAL_10]] to %[[VAL_1]] : !fir.ref<i32>
+! CHECK:             %[[VAL_11:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
+! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
+! CHECK:             %[[VAL_13:.*]] = arith.addi %[[VAL_11]], %[[VAL_12]] : i32
+! CHECK:             %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> i64
+! CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
+! CHECK:             %[[VAL_16:.*]] = arith.constant 1 : i32
+! CHECK:             %[[VAL_17:.*]] = arith.addi %[[VAL_15]], %[[VAL_16]] : i32
+! CHECK:             %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> i64
+! CHECK:             %[[VAL_19:.*]] = fir.convert %[[VAL_14]] : (i64) -> index
+! CHECK:             %[[VAL_20:.*]] = fir.convert %[[VAL_18]] : (i64) -> index
+! CHECK:             %[[VAL_21:.*]] = arith.constant 1 : index
+! CHECK:             %[[VAL_22:.*]] = arith.subi %[[VAL_19]], %[[VAL_21]] : index
+! CHECK:             %[[VAL_23:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<!fir.array<10x!fir.char<1>>>
+! CHECK:             %[[VAL_24:.*]] = fir.coordinate_of %[[VAL_23]], %[[VAL_22]] : (!fir.ref<!fir.array<10x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! CHECK:             %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:             %[[VAL_26:.*]] = arith.subi %[[VAL_20]], %[[VAL_19]] : index
+! CHECK:             %[[VAL_27:.*]] = arith.addi %[[VAL_26]], %[[VAL_21]] : index
+! CHECK:             %[[VAL_28:.*]] = arith.constant 0 : index
+! CHECK:             %[[VAL_29:.*]] = arith.cmpi slt, %[[VAL_27]], %[[VAL_28]] : index
+! CHECK:             %[[VAL_30:.*]] = arith.select %[[VAL_29]], %[[VAL_28]], %[[VAL_27]] : index
+! CHECK:             %[[VAL_31:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
+! CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (i32) -> i64
+! CHECK:             %[[VAL_33:.*]] = fir.load %[[VAL_1]] : !fir.ref<i32>
+! CHECK:             %[[VAL_34:.*]] = fir.convert %[[VAL_33]] : (i32) -> i64
+! CHECK:             %[[VAL_35:.*]] = fir.convert %[[VAL_32]] : (i64) -> index
+! CHECK:             %[[VAL_36:.*]] = fir.convert %[[VAL_34]] : (i64) -> index
+! CHECK:             %[[VAL_37:.*]] = arith.constant 1 : index
+! CHECK:             %[[VAL_38:.*]] = arith.subi %[[VAL_35]], %[[VAL_37]] : index
+! CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<!fir.array<10x!fir.char<1>>>
+! CHECK:             %[[VAL_40:.*]] = fir.coordinate_of %[[VAL_39]], %[[VAL_38]] : (!fir.ref<!fir.array<10x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! CHECK:             %[[VAL_41:.*]] = fir.convert %[[VAL_40]] : (!fir.ref<!fir.char<1>>) -> !fir.ref<!fir.char<1,?>>
+! CHECK:             %[[VAL_42:.*]] = arith.subi %[[VAL_36]], %[[VAL_35]] : index
+! CHECK:             %[[VAL_43:.*]] = arith.addi %[[VAL_42]], %[[VAL_37]] : index
+! CHECK:             %[[VAL_44:.*]] = arith.constant 0 : index
+! CHECK:             %[[VAL_45:.*]] = arith.cmpi slt, %[[VAL_43]], %[[VAL_44]] : index
+! CHECK:             %[[VAL_46:.*]] = arith.select %[[VAL_45]], %[[VAL_44]], %[[VAL_43]] : index
+! CHECK:             %[[VAL_47:.*]] = arith.cmpi slt, %[[VAL_46]], %[[VAL_30]] : index
+! CHECK:             %[[VAL_48:.*]] = arith.select %[[VAL_47]], %[[VAL_46]], %[[VAL_30]] : index
+! CHECK:             %[[VAL_49:.*]] = arith.constant 1 : i64
+! CHECK:             %[[VAL_50:.*]] = fir.convert %[[VAL_48]] : (index) -> i64
+! CHECK:             %[[VAL_51:.*]] = arith.muli %[[VAL_49]], %[[VAL_50]] : i64
+! CHECK:             %[[VAL_52:.*]] = arith.constant false
+! CHECK:             %[[VAL_53:.*]] = fir.convert %[[VAL_41]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! CHECK:             %[[VAL_54:.*]] = fir.convert %[[VAL_25]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<i8>
+! CHECK:             fir.call @llvm.memmove.p0.p0.i64(%[[VAL_53]], %[[VAL_54]], %[[VAL_51]], %[[VAL_52]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<i8>, i64, i1) -> ()
+! CHECK:             %[[VAL_55:.*]] = arith.constant 1 : index
+! CHECK:             %[[VAL_56:.*]] = arith.subi %[[VAL_46]], %[[VAL_55]] : index
+! CHECK:             %[[VAL_57:.*]] = arith.constant 32 : i8
+! CHECK:             %[[VAL_58:.*]] = fir.undefined !fir.char<1>
+! CHECK:             %[[VAL_59:.*]] = fir.insert_value %[[VAL_58]], %[[VAL_57]], [0 : index] : (!fir.char<1>, i8) -> !fir.char<1>
+! CHECK:             %[[VAL_60:.*]] = arith.constant 1 : index
+! CHECK:             fir.do_loop %[[VAL_61:.*]] = %[[VAL_48]] to %[[VAL_56]] step %[[VAL_60]] {
+! CHECK:               %[[VAL_62:.*]] = fir.convert %[[VAL_41]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1>>>
+! CHECK:               %[[VAL_63:.*]] = fir.coordinate_of %[[VAL_62]], %[[VAL_61]] : (!fir.ref<!fir.array<?x!fir.char<1>>>, index) -> !fir.ref<!fir.char<1>>
+! CHECK:               fir.store %[[VAL_59]] to %[[VAL_63]] : !fir.ref<!fir.char<1>>
+! CHECK:             }
 ! CHECK:           }
+! CHECK:           return
 ! CHECK:         }
-! CHECK:         return
-! CHECK:       }
-
diff --git a/flang/test/Lower/implicit-call-mismatch.f90 b/flang/test/Lower/implicit-call-mismatch.f90
index f87dbbd5b46df..37d717f8f890b 100644
--- a/flang/test/Lower/implicit-call-mismatch.f90
+++ b/flang/test/Lower/implicit-call-mismatch.f90
@@ -57,7 +57,8 @@ subroutine pass_char_to_proc(a)
 ! CHECK-LABEL: func.func @_QPpass_char_to_proc(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.boxchar<1>
 ! CHECK: %[[charAndLen:.*]]:2 = fir.unboxchar %[[arg0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK: %[[procAddr:.*]] = fir.convert %[[charAndLen]]#0 : (!fir.ref<!fir.char<1,?>>) -> (() -> ())
+! CHECK: %[[charRef:.*]] = fir.convert %[[charAndLen]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,8>>
+! CHECK: %[[procAddr:.*]] = fir.convert %[[charRef]] : (!fir.ref<!fir.char<1,8>>) -> (() -> ())
 ! CHECK: %[[boxProc:.*]] = fir.emboxproc %[[procAddr]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: fir.call @_QPtakes_proc(%[[boxProc]]) {{.*}}: (!fir.boxproc<() -> ()>) -> ()
 
@@ -234,7 +235,8 @@ subroutine pass_char_to_char_proc(a)
 ! CHECK-SAME: %[[arg0:.*]]: !fir.boxchar<1>
 ! CHECK: %[[charRefAndLen:.*]]:2 = fir.unboxchar %[[arg0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
 ! CHECK: %[[charLen:.*]] = arith.constant 8 : index
-! CHECK: %[[procAddr:.*]] = fir.convert %[[charRefAndLen]]#0 : (!fir.ref<!fir.char<1,?>>) -> (() -> ())
+! CHECK: %[[charRef:.*]] = fir.convert %[[charRefAndLen]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,8>>
+! CHECK: %[[procAddr:.*]] = fir.convert %[[charRef]] : (!fir.ref<!fir.char<1,8>>) -> (() -> ())
 ! CHECK: %[[boxProc:.*]] = fir.emboxproc %[[procAddr]] : (() -> ()) -> !fir.boxproc<() -> ()>
 ! CHECK: %[[charLen2:.*]] = fir.convert %[[charLen]] : (index) -> i64
 ! CHECK: %[[tuple:.*]] = fir.undefined tuple<!fir.boxproc<() -> ()>, i64>
diff --git a/flang/test/Lower/select-type-2.fir b/flang/test/Lower/select-type-2.fir
new file mode 100644
index 0000000000000..011a32b5def06
--- /dev/null
+++ b/flang/test/Lower/select-type-2.fir
@@ -0,0 +1,105 @@
+// RUN: fir-opt --fir-polymorphic-op %s | FileCheck %s
+
+// Test that cf.br for the default select case passes the blocks argument
+// as expected.
+
+// Original code:
+//subroutine test()
+//  interface
+//     function fstar()
+//       class(*),pointer :: fstar
+//     end function fstar
+//     subroutine foo(x)
+//       class(*) :: x
+//     end subroutine foo
+//  end interface
+//
+//  select type(y=>fstar())
+//  type is (integer)
+//    y = 777
+//  class default
+//     call foo(y)
+//     stop 'fail type'
+//  end select
+//
+//  select type(y=>fstar())
+//  type is (integer)
+//    y = 23
+//  class default
+//     call foo(y)
+//     stop 'fail type'
+//  end select
+//end subroutine test
+
+// CHECK-LABEL:   func.func @_QPtest() {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 23 : i32
+// CHECK:           %[[VAL_1:.*]] = arith.constant false
+// CHECK:           %[[VAL_2:.*]] = arith.constant 9 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 777 : i32
+// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.class<!fir.ptr<none>> {bindc_name = ".result"}
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.class<!fir.ptr<none>> {bindc_name = ".result"}
+// CHECK:           %[[VAL_6:.*]] = fir.call @_QPfstar() fastmath<contract> : () -> !fir.class<!fir.ptr<none>>
+// CHECK:           fir.save_result %[[VAL_6]] to %[[VAL_5]] : !fir.class<!fir.ptr<none>>, !fir.ref<!fir.class<!fir.ptr<none>>>
+// CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_5]] : !fir.ref<!fir.class<!fir.ptr<none>>>
+// CHECK:           %[[VAL_8:.*]] = arith.constant 9 : i8
+// CHECK:           %[[VAL_9:.*]] = fir.box_typecode %[[VAL_7]] : (!fir.class<!fir.ptr<none>>) -> i8
+// CHECK:           %[[VAL_10:.*]] = arith.cmpi eq, %[[VAL_9]], %[[VAL_8]] : i8
+// CHECK:           cf.cond_br %[[VAL_10]], ^bb2, ^bb1
+// CHECK:         ^bb1:
+// CHECK:           cf.br ^bb3(%[[VAL_7]] : !fir.class<!fir.ptr<none>>)
+// CHECK:         ^bb2:
+// CHECK:           %[[VAL_11:.*]] = fir.box_addr %[[VAL_7]] : (!fir.class<!fir.ptr<none>>) -> !fir.ptr<i32>
+// CHECK:           fir.store %[[VAL_3]] to %[[VAL_11]] : !fir.ptr<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.call @_QPfstar() fastmath<contract> : () -> !fir.class<!fir.ptr<none>>
+// CHECK:           fir.save_result %[[VAL_12]] to %[[VAL_4]] : !fir.class<!fir.ptr<none>>, !fir.ref<!fir.class<!fir.ptr<none>>>
+// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.class<!fir.ptr<none>>>
+// CHECK:           %[[VAL_14:.*]] = arith.constant 9 : i8
+// CHECK:           %[[VAL_15:.*]] = fir.box_typecode %[[VAL_13]] : (!fir.class<!fir.ptr<none>>) -> i8
+// CHECK:           %[[VAL_16:.*]] = arith.cmpi eq, %[[VAL_15]], %[[VAL_14]] : i8
+// CHECK:           cf.cond_br %[[VAL_16]], ^bb5, ^bb4
+// CHECK:         ^bb3(%[[VAL_17:.*]]: !fir.class<!fir.ptr<none>>):
+// CHECK:           %[[VAL_18:.*]] = fir.rebox %[[VAL_17]] : (!fir.class<!fir.ptr<none>>) -> !fir.class<none>
+// CHECK:           fir.call @_QPfoo(%[[VAL_18]]) fastmath<contract> : (!fir.class<none>) -> ()
+// CHECK:           %[[VAL_19:.*]] = fir.address_of(@_QQcl.6661696C2074797065) : !fir.ref<!fir.char<1,9>>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (!fir.ref<!fir.char<1,9>>) -> !fir.ref<i8>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_2]] : (index) -> i64
+// CHECK:           %[[VAL_22:.*]] = fir.call @_FortranAStopStatementText(%[[VAL_20]], %[[VAL_21]], %[[VAL_1]], %[[VAL_1]]) fastmath<contract> : (!fir.ref<i8>, i64, i1, i1) -> none
+// CHECK:           fir.unreachable
+// CHECK:         ^bb4:
+// CHECK:           cf.br ^bb3(%[[VAL_13]] : !fir.class<!fir.ptr<none>>)
+// CHECK:         ^bb5:
+// CHECK:           %[[VAL_23:.*]] = fir.box_addr %[[VAL_13]] : (!fir.class<!fir.ptr<none>>) -> !fir.ptr<i32>
+// CHECK:           fir.store %[[VAL_0]] to %[[VAL_23]] : !fir.ptr<i32>
+// CHECK:           return
+// CHECK:         }
+func.func @_QPtest() {
+  %c23_i32 = arith.constant 23 : i32
+  %false = arith.constant false
+  %c9 = arith.constant 9 : index
+  %c777_i32 = arith.constant 777 : i32
+  %0 = fir.alloca !fir.class<!fir.ptr<none>> {bindc_name = ".result"}
+  %1 = fir.alloca !fir.class<!fir.ptr<none>> {bindc_name = ".result"}
+  %2 = fir.call @_QPfstar() fastmath<contract> : () -> !fir.class<!fir.ptr<none>>
+  fir.save_result %2 to %1 : !fir.class<!fir.ptr<none>>, !fir.ref<!fir.class<!fir.ptr<none>>>
+  %3 = fir.load %1 : !fir.ref<!fir.class<!fir.ptr<none>>>
+  fir.select_type %3 : !fir.class<!fir.ptr<none>> [#fir.type_is<i32>, ^bb1, unit, ^bb2(%3 : !fir.class<!fir.ptr<none>>)]
+^bb1:  // pred: ^bb0
+  %4 = fir.box_addr %3 : (!fir.class<!fir.ptr<none>>) -> !fir.ptr<i32>
+  fir.store %c777_i32 to %4 : !fir.ptr<i32>
+  %5 = fir.call @_QPfstar() fastmath<contract> : () -> !fir.class<!fir.ptr<none>>
+  fir.save_result %5 to %0 : !fir.class<!fir.ptr<none>>, !fir.ref<!fir.class<!fir.ptr<none>>>
+  %6 = fir.load %0 : !fir.ref<!fir.class<!fir.ptr<none>>>
+  fir.select_type %6 : !fir.class<!fir.ptr<none>> [#fir.type_is<i32>, ^bb3, unit, ^bb2(%6 : !fir.class<!fir.ptr<none>>)]
+^bb2(%7: !fir.class<!fir.ptr<none>>):  // 2 preds: ^bb0, ^bb1
+  %8 = fir.rebox %7 : (!fir.class<!fir.ptr<none>>) -> !fir.class<none>
+  fir.call @_QPfoo(%8) fastmath<contract> : (!fir.class<none>) -> ()
+  %9 = fir.address_of(@_QQcl.6661696C2074797065) : !fir.ref<!fir.char<1,9>>
+  %10 = fir.convert %9 : (!fir.ref<!fir.char<1,9>>) -> !fir.ref<i8>
+  %11 = fir.convert %c9 : (index) -> i64
+  %12 = fir.call @_FortranAStopStatementText(%10, %11, %false, %false) fastmath<contract> : (!fir.ref<i8>, i64, i1, i1) -> none
+  fir.unreachable
+^bb3:  // pred: ^bb1
+  %13 = fir.box_addr %6 : (!fir.class<!fir.ptr<none>>) -> !fir.ptr<i32>
+  fir.store %c23_i32 to %13 : !fir.ptr<i32>
+  return
+}
diff --git a/flang/test/Lower/statement-function.f90 b/flang/test/Lower/statement-function.f90
index d7dc7625334ff..a706a4be69342 100644
--- a/flang/test/Lower/statement-function.f90
+++ b/flang/test/Lower/statement-function.f90
@@ -92,9 +92,11 @@ integer function test_stmt_character(c, j)
   integer :: i, j, func, argj
   character(10) :: c, argc
   ! CHECK-DAG: %[[unboxed:.*]]:2 = fir.unboxchar %arg0 :
+  ! CHECK-DAG: %[[ref:.*]] = fir.convert %[[unboxed]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
   ! CHECK-DAG: %[[c10:.*]] = arith.constant 10 :
+  ! CHECK-DAG: %[[ref_cast:.*]] = fir.convert %[[ref]] : (!fir.ref<!fir.char<1,10>>) -> !fir.ref<!fir.char<1,?>>
   ! CHECK: %[[c10_cast:.*]] = fir.convert %[[c10]] : (i32) -> index
-  ! CHECK: %[[c:.*]] = fir.emboxchar %[[unboxed]]#0, %[[c10_cast]]
+  ! CHECK: %[[c:.*]] = fir.emboxchar %[[ref_cast]], %[[c10_cast]]
 
   func(argc, argj) = len_trim(argc, 4) + argj
   ! CHECK: addi %{{.*}}, %{{.*}} : i
diff --git a/flang/test/Semantics/OpenMP/use_device_ptr.f90 b/flang/test/Semantics/OpenMP/use_device_ptr.f90
new file mode 100644
index 0000000000000..738904bbab64c
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/use_device_ptr.f90
@@ -0,0 +1,21 @@
+! RUN: %flang -fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
+! OpenMP Version 5.0
+! 2.10.1 use_device_ptr clause
+! List items that appear in a use_device_ptr clause are converted into device
+! pointers to the corresponding list item in the device data environment.
+
+subroutine omp_target_data
+   use iso_c_binding
+   integer :: a(1024)
+   !CHECK: b size=8 offset=4096: ObjectEntity type: TYPE(c_ptr)
+   type(C_PTR) :: b
+   integer, pointer :: arrayB
+   a = 1
+   !$omp target data map(tofrom: a, arrayB) use_device_ptr(b)
+   !CHECK: b (OmpUseDevicePtr)
+      allocate(arrayB)
+      call c_f_pointer(b, arrayB)
+      a = arrayB
+   !$omp end target data
+end subroutine omp_target_data
+
diff --git a/flang/test/Transforms/loop-versioning.fir b/flang/test/Transforms/loop-versioning.fir
index f26dbd9bb80e9..bad02b7a226c5 100644
--- a/flang/test/Transforms/loop-versioning.fir
+++ b/flang/test/Transforms/loop-versioning.fir
@@ -112,4 +112,276 @@ func.func @sum1dfixed(%arg0: !fir.ref<!fir.array<?xf64>> {fir.bindc_name = "a"},
 // CHECK: %[[COORD:.*]] = fir.coordinate_of %[[ARG0]], {{.*}}
 // CHECK: %{{.*}} = fir.load %[[COORD]]
 
+// -----
+
+// RUN: fir-opt --loop-versioning %s | FileCheck %s
+
+// Check that "no result" from a versioned loop works correctly
+// This code was the basis for this, but `read` is replaced with a function called Func
+// subroutine test3(x, y)
+//  integer :: y(:)
+//  integer :: x(:)
+//  read(*,*) x(y)
+// end subroutine
+
+  func.func @test3(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "x"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "y"}) {
+    %c0 = arith.constant 0 : index
+    %3:3 = fir.box_dims %arg1, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+    %c1 = arith.constant 1 : index
+    %4 = fir.slice %c1, %3#1, %c1 : (index, index, index) -> !fir.slice<1>
+    %c1_0 = arith.constant 1 : index
+    %c0_1 = arith.constant 0 : index
+    %5 = arith.subi %3#1, %c1_0 : index
+    fir.do_loop %arg2 = %c0_1 to %5 step %c1_0 {
+      %7 = fir.coordinate_of %arg1, %arg2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+      %8 = fir.load %7 : !fir.ref<i32>
+      %9 = fir.convert %8 : (i32) -> index
+      %10 = fir.array_coor %arg0 [%4] %9 : (!fir.box<!fir.array<?xi32>>, !fir.slice<1>, index) -> !fir.ref<i32>
+      %12 = fir.call @Func(%10) fastmath<contract> : (!fir.ref<i32>) -> i1
+    }
+    return
+  }
+  func.func private @Func(!fir.ref<i8>, !fir.ref<i32>) -> i1
+
+// CHECK-LABEL: func.func @test3(
+// CHECK-SAME:                    %[[X:.*]]: !fir.box<!fir.array<?xi32>> {{.*}},
+// CHECK-SAME:                    %[[Y:.*]]: !fir.box<!fir.array<?xi32>> {{.*}}) {
+// Look for arith.subi to locate the correct part of code.
+// CHECK:    {{.*}} arith.subi {{.*}}
+// CHECK:    %[[ZERO:.*]] = arith.constant 0 : index
+// CHECK:     %[[DIMS:.*]]:3 = fir.box_dims %[[Y]], %[[ZERO]]
+// CHECK:    %[[FOUR:.*]] = arith.constant 4 : index
+// CHECK:    %[[COMP:.*]] = arith.cmpi eq, %[[DIMS]]#2, %[[FOUR]] : index
+// CHECK:    fir.if %[[COMP]] {
+// CHECK:    %[[CONV:.*]]  = fir.convert %[[Y]] : {{.*}}
+// CHECK:    %[[BOX_ADDR:.*]] = fir.box_addr %[[CONV]] : {{.*}}
+// CHECK:    fir.do_loop %[[INDEX:.*]] = {{.*}}
+// CHECK:    %[[IND_PLUS_1:.*]] = arith.addi %{{.*}}, %[[INDEX]]
+// CHECK:    %[[YADDR:.*]] = fir.coordinate_of %[[BOX_ADDR]], %[[IND_PLUS_1]]
+// CHECK:    %[[YINT:.*]] = fir.load %[[YADDR]] : {{.*}}
+// CHECK:    %[[YINDEX:.*]] = fir.convert %[[YINT]]
+// CHECK:    %[[XADDR:.*]] = fir.array_coor %[[X]] [%{{.*}}] %[[YINDEX]]
+// CHECK:    fir.call @Func(%[[XADDR]])
+// CHECK-NEXT: }
+// CHECK-NEXT: } else {
+// CHECK:    fir.do_loop %[[INDEX2:.*]] = {{.*}}
+// CHECK:    %[[YADDR2:.*]] = fir.coordinate_of %[[Y]], %[[INDEX2]]
+// CHECK:    %[[YINT2:.*]] = fir.load %[[YADDR2]] : {{.*}}
+// CHECK:    %[[YINDEX2:.*]] = fir.convert %[[YINT2]]
+// CHECK:    %[[XADDR2:.*]] = fir.array_coor %[[X]] [%{{.*}}] %[[YINDEX2]]
+// CHECK:    fir.call @Func(%[[XADDR2]])
+// CHECK-NEXT: }
+
+
+// ----
+
+// Test array initialization.
+//
+// This code has been modified to simplify it - removing the realloc generated to grow
+// the constructed 
+//subroutine test4(a, b, n1, m1)
+//  real :: a(:)  
+//  real :: b(:,:)
+//          
+//  a = [ ((b(i,j), j=1,n1,m1), i=1,n1,m1) ]
+//end subroutine test4
+
+  func.func @test4(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "b"}, %arg2: !fir.ref<i32> {fir.bindc_name = "n1"}, %arg3: !fir.ref<i32> {fir.bindc_name = "m1"}) {
+    %0 = fir.alloca index {bindc_name = ".buff.pos"}
+    %1 = fir.alloca index {bindc_name = ".buff.size"}
+    %c0 = arith.constant 0 : index
+    %2:3 = fir.box_dims %arg0, %c0 : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+    %3 = fir.array_load %arg0 : (!fir.box<!fir.array<?xf32>>) -> !fir.array<?xf32>
+    %c0_0 = arith.constant 0 : index
+    fir.store %c0_0 to %0 : !fir.ref<index>
+    %c32 = arith.constant 32 : index
+    %4 = fir.allocmem f32, %c32
+    fir.store %c32 to %1 : !fir.ref<index>
+    %c1_i64 = arith.constant 1 : i64
+    %5 = fir.convert %c1_i64 : (i64) -> index
+    %6 = fir.load %arg2 : !fir.ref<i32>
+    %7 = fir.convert %6 : (i32) -> i64
+    %8 = fir.convert %7 : (i64) -> index
+    %9 = fir.load %arg3 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> i64
+    %11 = fir.convert %10 : (i64) -> index
+    %12 = fir.do_loop %arg4 = %5 to %8 step %11 iter_args(%arg5 = %4) -> (!fir.heap<f32>) {
+      %c1_i64_2 = arith.constant 1 : i64
+      %19 = fir.convert %c1_i64_2 : (i64) -> index
+      %20 = fir.load %arg2 : !fir.ref<i32>
+      %21 = fir.convert %20 : (i32) -> i64
+      %22 = fir.convert %21 : (i64) -> index
+      %23 = fir.load %arg3 : !fir.ref<i32>
+      %24 = fir.convert %23 : (i32) -> i64
+      %25 = fir.convert %24 : (i64) -> index
+      %26 = fir.do_loop %arg6 = %19 to %22 step %25 iter_args(%arg7 = %arg5) -> (!fir.heap<f32>) {
+        %27 = fir.convert %arg4 : (index) -> i32
+        %28 = fir.convert %27 : (i32) -> i64
+        %c1_i64_3 = arith.constant 1 : i64
+        %29 = arith.subi %28, %c1_i64_3 : i64
+        %30 = fir.convert %arg6 : (index) -> i32
+        %31 = fir.convert %30 : (i32) -> i64
+        %c1_i64_4 = arith.constant 1 : i64
+        %32 = arith.subi %31, %c1_i64_4 : i64
+        %33 = fir.coordinate_of %arg1, %29, %32 : (!fir.box<!fir.array<?x?xf32>>, i64, i64) -> !fir.ref<f32>
+        %34 = fir.load %33 : !fir.ref<f32>
+        %c1_5 = arith.constant 1 : index
+        %35 = fir.zero_bits !fir.ref<!fir.array<?xf32>>
+        %36 = fir.coordinate_of %35, %c1_5 : (!fir.ref<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+        %37 = fir.convert %36 : (!fir.ref<f32>) -> index
+        %38 = fir.load %0 : !fir.ref<index>
+        %39 = fir.load %1 : !fir.ref<index>
+        %c1_6 = arith.constant 1 : index
+        %40 = arith.addi %38, %c1_6 : index
+
+        fir.store %40 to %0 : !fir.ref<index>
+        fir.result %arg7 : !fir.heap<f32>
+      }
+      fir.result %26 : !fir.heap<f32>
+    }
+    %13 = fir.convert %12 : (!fir.heap<f32>) -> !fir.heap<!fir.array<?xf32>>
+    %14 = fir.load %0 : !fir.ref<index>
+    %15 = fir.shape %14 : (index) -> !fir.shape<1>
+    %16 = fir.array_load %13(%15) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.array<?xf32>
+    %c1 = arith.constant 1 : index
+    %c0_1 = arith.constant 0 : index
+    %17 = arith.subi %2#1, %c1 : index
+    %18 = fir.do_loop %arg4 = %c0_1 to %17 step %c1 unordered iter_args(%arg5 = %3) -> (!fir.array<?xf32>) {
+      %19 = fir.array_fetch %16, %arg4 : (!fir.array<?xf32>, index) -> f32
+      %20 = fir.array_update %arg5, %19, %arg4 : (!fir.array<?xf32>, f32, index) -> !fir.array<?xf32>
+      fir.result %20 : !fir.array<?xf32>
+    }
+    fir.array_merge_store %3, %18 to %arg0 : !fir.array<?xf32>, !fir.array<?xf32>, !fir.box<!fir.array<?xf32>>
+    fir.freemem %13 : !fir.heap<!fir.array<?xf32>>
+    return
+  }
+
+// CHECK: func.func @test4(
+// CHECK-SAME:            %[[A:.*]]: !fir.box<!fir.array<?xf32>>
+// CHECK-SAME:            %[[B:.*]]: !fir.box<!fir.array<?x?xf32>>
+// CHECK-SAME:            %[[N1:.*]]: !fir.ref<i32> {{.*}},
+// CHECK-SAME:            %[[M1:.*]]: !fir.ref<i32> {{.*}}) {
+// CHECK: fir.do_loop
+// CHECL:   %[[FOUR:.*]] = arith.constant 4 : index
+// CHECK:   %[[COMP:.*]] = arith.cmpi {{.*}}, %[[FOUR]]
+// CHECK:   fir.if %[[COMP]] -> {{.*}} {
+// CHECK:     %[[CONV:.*]] = fir.convert %[[B]] :
+// CHECK:     %[[BOX_ADDR:.*]] = fir.box_addr %[[CONV]]
+// CHECK:     %[[RES:.*]] = fir.do_loop {{.*}} {
+// CHECK:     %[[ADDR:.*]] = fir.coordinate_of %[[BOX_ADDR]], %{{.*}}
+// CHECK:     %45 = fir.load %[[ADDR]] : !fir.ref<f32>
+// CHECK:   }
+// CHECK:   fir.result %[[RES]] : {{.*}}
+// CHECK: } else {
+// CHECK:   %[[RES2:.*]] = fir.do_loop
+// CHECK:     %{{.*}} = fir.coordinate_of %[[B]], %{{.*}}
+// CHECK:   }
+// CHECK:   fir.result %[[RES2]]
+// CHECK: }
+
+// -----
+
+
+// Check that 2D arrays are identified and converted.
+// Source code:
+//   subroutine sum2d(a, nx, ny)
+//    real*8 :: a(:,:)
+//    integer :: nx, ny
+//    real*8 :: sum
+//    integer :: i, j
+//    sum = 0
+//    do i=1,nx
+//       do j=1,ny
+//          sum = sum + a(j,i)
+//       end do
+//    end do
+//  end subroutine sum2d
+
+  func.func @sum2d(%arg0: !fir.box<!fir.array<?x?xf64>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "nx"}, %arg2: !fir.ref<i32> {fir.bindc_name = "ny"}) {
+    %0 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QMmoduleFsum2dEi"}
+    %1 = fir.alloca i32 {bindc_name = "j", uniq_name = "_QMmoduleFsum2dEj"}
+    %2 = fir.alloca f64 {bindc_name = "sum", uniq_name = "_QMmoduleFsum2dEsum"}
+    %cst = arith.constant 0.000000e+00 : f64
+    fir.store %cst to %2 : !fir.ref<f64>
+    %c1_i32 = arith.constant 1 : i32
+    %3 = fir.convert %c1_i32 : (i32) -> index
+    %4 = fir.load %arg1 : !fir.ref<i32>
+    %5 = fir.convert %4 : (i32) -> index
+    %c1 = arith.constant 1 : index
+    %6 = fir.convert %3 : (index) -> i32
+    %7:2 = fir.do_loop %arg3 = %3 to %5 step %c1 iter_args(%arg4 = %6) -> (index, i32) {
+      fir.store %arg4 to %0 : !fir.ref<i32>
+      %c1_i32_0 = arith.constant 1 : i32
+      %8 = fir.convert %c1_i32_0 : (i32) -> index
+      %9 = fir.load %arg2 : !fir.ref<i32>
+      %10 = fir.convert %9 : (i32) -> index
+      %c1_1 = arith.constant 1 : index
+      %11 = fir.convert %8 : (index) -> i32
+      %12:2 = fir.do_loop %arg5 = %8 to %10 step %c1_1 iter_args(%arg6 = %11) -> (index, i32) {
+        fir.store %arg6 to %1 : !fir.ref<i32>
+        %17 = fir.load %2 : !fir.ref<f64>
+        %18 = fir.load %1 : !fir.ref<i32>
+        %19 = fir.convert %18 : (i32) -> i64
+        %c1_i64 = arith.constant 1 : i64
+        %20 = arith.subi %19, %c1_i64 : i64
+        %21 = fir.load %0 : !fir.ref<i32>
+        %22 = fir.convert %21 : (i32) -> i64
+        %c1_i64_2 = arith.constant 1 : i64
+        %23 = arith.subi %22, %c1_i64_2 : i64
+        %24 = fir.coordinate_of %arg0, %20, %23 : (!fir.box<!fir.array<?x?xf64>>, i64, i64) -> !fir.ref<f64>
+        %25 = fir.load %24 : !fir.ref<f64>
+        %26 = arith.addf %17, %25 fastmath<contract> : f64
+        fir.store %26 to %2 : !fir.ref<f64>
+        %27 = arith.addi %arg5, %c1_1 : index
+        %28 = fir.convert %c1_1 : (index) -> i32
+        %29 = fir.load %1 : !fir.ref<i32>
+        %30 = arith.addi %29, %28 : i32
+        fir.result %27, %30 : index, i32
+      }
+      fir.store %12#1 to %1 : !fir.ref<i32>
+      %13 = arith.addi %arg3, %c1 : index
+      %14 = fir.convert %c1 : (index) -> i32
+      %15 = fir.load %0 : !fir.ref<i32>
+      %16 = arith.addi %15, %14 : i32
+      fir.result %13, %16 : index, i32
+    }
+    fir.store %7#1 to %0 : !fir.ref<i32>
+    return
+  }
+
+// Note this only checks the expected transformation, not the entire generated code:
+// CHECK-LABEL: func.func @sum2d(
+// CHECK-SAME:                  %[[ARG0:.*]]: !fir.box<!fir.array<?x?xf64>> {{.*}})
+// Only inner loop should be verisoned.
+// CHECK: fir.do_loop
+// CHECK: %[[ZERO:.*]] = arith.constant 0 : index
+// CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[ARG0]], %[[ZERO]] : {{.*}}
+// CHECK: %[[SIZE:.*]] = arith.constant 8 : index
+// CHECK: %[[CMP:.*]] = arith.cmpi eq, %[[DIMS]]#2, %[[SIZE]]
+// CHECK: %[[IF_RES:.*]]:2 = fir.if %[[CMP]] -> {{.*}}
+// CHECK: %[[NEWARR:.*]] = fir.convert %[[ARG0]]
+// CHECK: %[[BOXADDR:.*]] = fir.box_addr %[[NEWARR]] : {{.*}} -> !fir.ref<!fir.array<?xf64>>
+// CHECK: %[[LOOP_RES:.*]]:2 = fir.do_loop {{.*}}
+// Check the 2D -> 1D coordinate conversion, should have a multiply and a final add.
+// Some other operations are checked to synch the different parts.
+// CHECK: arith.muli %[[DIMS]]#1, {{.*}}
+// CHECK: %[[OUTER_IDX:.*]] = arith.addi {{.*}}
+// CHECK: %[[INNER_IDX:.*]] = fir.convert {{.*}}
+// CHECK: %[[C2D:.*]] = arith.addi %[[OUTER_IDX]], %[[INNER_IDX]]
+// CHECK: %[[COORD:.*]] = fir.coordinate_of %[[BOXADDR]], %[[C2D]] : (!fir.ref<!fir.array<?xf64>>, index) -> !fir.ref<f64>
+// CHECK: %{{.*}} = fir.load %[[COORD]] : !fir.ref<f64>
+// CHECK: fir.result %{{.*}}, %{{.*}}
+// CHECK: }
+// CHECK  fir.result %[[LOOP_RES]]#0, %[[LOOP_RES]]#1
+// CHECK: } else {
+// CHECK: %[[LOOP_RES2:.*]]:2 = fir.do_loop {{.*}}
+// CHECK: %[[COORD2:.*]] = fir.coordinate_of %[[ARG0]], %{{.*}} : (!fir.box<!fir.array<?x?xf64>>, i64, i64) -> !fir.ref<f64>
+// CHECK: %{{.*}}= fir.load %[[COORD2]] : !fir.ref<f64>
+// CHECK: fir.result %{{.*}}, %{{.*}}
+// CHECK: }
+// CHECK  fir.result %[[LOOP_RES2]]#0, %[[LOOP_RES2]]#1
+// CHECK: }
+// CHECK: fir.store %[[IF_RES]]#1 to %{{.*}}
+// CHECK: return
+
 } // End module
diff --git a/flang/tools/fir-opt/fir-opt.cpp b/flang/tools/fir-opt/fir-opt.cpp
index b605c9dc52af4..d4457c57a0411 100644
--- a/flang/tools/fir-opt/fir-opt.cpp
+++ b/flang/tools/fir-opt/fir-opt.cpp
@@ -35,5 +35,5 @@ int main(int argc, char **argv) {
   DialectRegistry registry;
   fir::support::registerDialects(registry);
   return failed(MlirOptMain(argc, argv, "FIR modular optimizer driver\n",
-      registry, /*preloadDialectsInContext=*/false));
+      registry));
 }
diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index 958de3b30ad20..e0e5b69b47cf8 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -75,11 +75,15 @@ option(LLVM_LIBC_ENABLE_LINTING "Enables linting of libc source files" OFF)
 option(LIBC_GPU_BUILD "Build libc for the GPU. All CPU build options will be ignored." OFF)
 set(LIBC_TARGET_TRIPLE "" CACHE STRING "The target triple for the libc build.")
 
+set(LIBC_ENABLE_UNITTESTS ON)
+set(LIBC_ENABLE_HERMETIC_TESTS ${LLVM_LIBC_FULL_BUILD})
+
 # Defines LIBC_TARGET_ARCHITECTURE and associated macros.
 include(LLVMLibCArchitectures)
 
 if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
   include(prepare_libc_gpu_build)
+  set(LIBC_ENABLE_UNITTESTS OFF)
 endif()
 
 include(LLVMLibCCheckMPFR)
diff --git a/libc/cmake/modules/LLVMLibCArchitectures.cmake b/libc/cmake/modules/LLVMLibCArchitectures.cmake
index 63ec2ef7a286a..f3852e1b9e056 100644
--- a/libc/cmake/modules/LLVMLibCArchitectures.cmake
+++ b/libc/cmake/modules/LLVMLibCArchitectures.cmake
@@ -76,7 +76,7 @@ function(get_arch_and_system_from_triple triple arch_var sys_var)
   set(${sys_var} ${target_sys} PARENT_SCOPE)
 endfunction(get_arch_and_system_from_triple)
 
-execute_process(COMMAND ${CMAKE_CXX_COMPILER} -v
+execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version -v
                 RESULT_VARIABLE libc_compiler_info_result
                 OUTPUT_VARIABLE libc_compiler_info
                 ERROR_VARIABLE libc_compiler_info)
diff --git a/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake b/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake
index f70cffbf81347..0b522318aaa12 100644
--- a/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake
+++ b/libc/cmake/modules/LLVMLibCCheckCpuFeatures.cmake
@@ -5,10 +5,6 @@
 # Initialize ALL_CPU_FEATURES as empty list.
 set(ALL_CPU_FEATURES "")
 
-if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
-  return()
-endif()
-
 if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
   set(ALL_CPU_FEATURES SSE2 SSE4_2 AVX AVX2 AVX512F AVX512BW FMA)
   set(LIBC_COMPILE_OPTIONS_NATIVE -march=native)
@@ -26,6 +22,10 @@ list(SORT ALL_CPU_FEATURES)
 #   <list of cpu features>
 # )
 function(cpu_supports output_var features)
+  if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
+    unset(${output_var} PARENT_SCOPE)
+    return()
+  endif()
   _intersection(var "${LIBC_CPU_FEATURES}" "${features}")
   if("${var}" STREQUAL "${features}")
     set(${output_var} TRUE PARENT_SCOPE)
diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake
index 3a1475d3807ca..688997228fd35 100644
--- a/libc/cmake/modules/LLVMLibCObjectRules.cmake
+++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake
@@ -136,6 +136,9 @@ function(_build_gpu_objects fq_target_name internal_target_name)
 
   set(include_dirs ${LIBC_BUILD_DIR}/include ${LIBC_SOURCE_DIR} ${LIBC_BUILD_DIR})
   set(common_compile_options ${ADD_GPU_OBJ_COMPILE_OPTIONS})
+  if(NOT ADD_GPU_OBJ_CXX_STANDARD)
+    set(ADD_GPU_OBJ_CXX_STANDARD ${CMAKE_CXX_STANDARD})
+  endif()
 
   foreach(add_gpu_obj_src ${ADD_GPU_OBJ_SRCS})
     # The packaged version will be built for every target GPU architecture. We do
@@ -170,6 +173,11 @@ function(_build_gpu_objects fq_target_name internal_target_name)
       target_compile_options(${gpu_target_name} PRIVATE ${compile_options})
       target_include_directories(${gpu_target_name} PRIVATE ${include_dirs})
       target_compile_definitions(${gpu_target_name} PRIVATE LIBC_COPT_PUBLIC_PACKAGING)
+      set_target_properties(
+        ${gpu_target_name}
+        PROPERTIES
+          CXX_STANDARD ${ADD_GPU_OBJ_CXX_STANDARD}
+      )
       if(ADD_GPU_OBJ_DEPENDS)
         add_dependencies(${gpu_target_name} ${ADD_GPU_OBJ_DEPENDS})
       endif()
@@ -324,6 +332,7 @@ function(create_object_library fq_target_name)
       SRCS ${ADD_OBJECT_SRCS}
       HDRS ${ADD_OBJECT_HDRS}
       DEPENDS ${fq_deps_list}
+      CXX_STANDARD ${ADD_OBJECT_CXX_STANDARD}
       COMPILE_OPTIONS ${compile_options}
     )
   else()
@@ -598,6 +607,7 @@ function(create_entrypoint_object fq_target_name)
       SRCS ${ADD_ENTRYPOINT_OBJ_SRCS}
       HDRS ${ADD_ENTRYPOINT_OBJ_HDRS}
       COMPILE_OPTIONS ${common_compile_options}
+      CXX_STANDARD ${ADD_ENTRYPOINT_OBJ_CXX_STANDARD}
       DEPENDS ${full_deps_list}
       FLAGS "${ADD_ENTRYPOINT_OBJ_FLAGS}"
     )
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index 0757bd27cd392..4c1b143b66186 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -77,7 +77,7 @@ function(create_libc_unittest fq_target_name)
 
   cmake_parse_arguments(
     "LIBC_UNITTEST"
-    "NO_RUN_POSTBUILD;NO_LIBC_UNITTEST_TEST_MAIN" # Optional arguments
+    "NO_RUN_POSTBUILD" # Optional arguments
     "SUITE;CXX_STANDARD" # Single value arguments
     "SRCS;HDRS;DEPENDS;COMPILE_OPTIONS;LINK_LIBRARIES;FLAGS" # Multi-value arguments
     ${ARGN}
@@ -179,11 +179,7 @@ function(create_libc_unittest fq_target_name)
   )
 
   # LibcUnitTest should not depend on anything in LINK_LIBRARIES.
-  if(NO_LIBC_UNITTEST_TEST_MAIN)
-    list(APPEND link_libraries LibcUnitTest)
-  else()
-    list(APPEND link_libraries LibcUnitTest LibcUnitTestMain)
-  endif()
+  list(APPEND link_libraries LibcUnitTestMain LibcUnitTest)
 
   target_link_libraries(${fq_build_target_name} PRIVATE ${link_libraries})
 
@@ -201,6 +197,7 @@ function(create_libc_unittest fq_target_name)
       ${fq_target_name}
     )
   endif()
+  add_dependencies(libc-unit-tests ${fq_target_name})
 endfunction(create_libc_unittest)
 
 # Internal function, used by `add_libc_unittest`.
@@ -298,11 +295,6 @@ function(add_libc_unittest target_name)
   )
 endfunction(add_libc_unittest)
 
-function(add_libc_testsuite suite_name)
-  add_custom_target(${suite_name})
-  add_dependencies(libc-unit-tests ${suite_name})
-endfunction(add_libc_testsuite)
-
 function(add_libc_exhaustive_testsuite suite_name)
   add_custom_target(${suite_name})
   add_dependencies(exhaustive-check-libc ${suite_name})
@@ -390,6 +382,8 @@ function(add_libc_fuzzer target_name)
 
 endfunction(add_libc_fuzzer)
 
+# DEPRECATED: Use add_hermetic_test instead.
+#
 # Rule to add an integration test. An integration test is like a unit test
 # but does not use the system libc. Not even the startup objects from the
 # system libc are linked in to the final executable. The final exe is fully
@@ -503,12 +497,12 @@ function(add_integration_test test_name)
   # The GPU build requires overriding the default CMake triple and architecture.
   if(LIBC_GPU_TARGET_ARCHITECTURE_IS_AMDGPU)
     target_compile_options(${fq_build_target_name} PRIVATE
-                           -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -flto
-                           --target=${LIBC_GPU_TARGET_TRIPLE})
+                           -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE}
+                           -flto --target=${LIBC_GPU_TARGET_TRIPLE})
   elseif(LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
     get_nvptx_compile_options(nvptx_options ${LIBC_GPU_TARGET_ARCHITECTURE})
     target_compile_options(${fq_build_target_name} PRIVATE
-                           ${nvptx_options}
+                           ${nvptx_options} -fno-use-cxa-atexit
                            --target=${LIBC_GPU_TARGET_TRIPLE})
   endif()
 
@@ -548,3 +542,173 @@ function(add_integration_test test_name)
   )
   add_dependencies(${INTEGRATION_TEST_SUITE} ${fq_target_name})
 endfunction(add_integration_test)
+
+set(LIBC_HERMETIC_TEST_COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_DEFAULT}
+    -fpie -ffreestanding -fno-exceptions -fno-rtti)
+# The GPU build requires overriding the default CMake triple and architecture.
+if(LIBC_GPU_TARGET_ARCHITECTURE_IS_AMDGPU)
+  list(APPEND LIBC_HERMETIC_TEST_COMPILE_OPTIONS
+       -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -flto --target=${LIBC_GPU_TARGET_TRIPLE})
+elseif(LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
+  get_nvptx_compile_options(nvptx_options ${LIBC_GPU_TARGET_ARCHITECTURE})
+  list(APPEND LIBC_HERMETIC_TEST_COMPILE_OPTIONS
+       ${nvptx_options} -fno-use-cxa-atexit --target=${LIBC_GPU_TARGET_TRIPLE})
+endif()
+
+# Rule to add a hermetic test. A hermetic test is one whose executable is fully
+# statically linked and consists of pieces drawn only from LLVM's libc. Nothing,
+# including the startup objects, come from the system libc.
+#
+# Usage:
+#   add_libc_hermetic_test(
+#     <target name>
+#     SUITE <the suite to which the test should belong>
+#     SRCS <src1.cpp> [src2.cpp ...]
+#     HDRS [hdr1.cpp ...]
+#     DEPENDS <list of entrypoint or other object targets>
+#     ARGS <list of command line arguments to be passed to the test>
+#     ENV <list of environment variables to set before running the test>
+#     COMPILE_OPTIONS <list of special compile options for the test>
+#     LINK_LIBRARIES <list of linking libraries for this target>
+#     LOADER_ARGS <list of special args to loaders (like the GPU loader)>
+#   )
+function(add_libc_hermetic_test test_name)
+  if(NOT TARGET libc.startup.${LIBC_TARGET_OS}.crt1)
+    message(VERBOSE "Skipping ${fq_target_name} as it is not available on ${LIBC_TARGET_OS}.")
+    return()
+  endif()
+  cmake_parse_arguments(
+    "HERMETIC_TEST"
+    "" # No optional arguments
+    "SUITE" # Single value arguments
+    "SRCS;HDRS;DEPENDS;ARGS;ENV;COMPILE_OPTIONS;LINK_LIBRARIES;LOADER_ARGS" # Multi-value arguments
+    ${ARGN}
+  )
+
+  if(NOT HERMETIC_TEST_SUITE)
+    message(FATAL_ERROR "SUITE not specified for ${fq_target_name}")
+  endif()
+  if(NOT HERMETIC_TEST_SRCS)
+    message(FATAL_ERROR "The SRCS list for add_integration_test is missing.")
+  endif()
+
+  get_fq_target_name(${test_name} fq_target_name)
+  get_fq_target_name(${test_name}.libc fq_libc_target_name)
+
+  get_fq_deps_list(fq_deps_list ${HERMETIC_TEST_DEPENDS})
+  list(APPEND fq_deps_list
+      # Hermetic tests use the platform's startup object. So, their deps also
+      # have to be collected.
+      libc.startup.${LIBC_TARGET_OS}.crt1
+      # We always add the memory functions objects. This is because the
+      # compiler's codegen can emit calls to the C memory functions.
+      libc.src.string.bcmp
+      libc.src.string.bzero
+      libc.src.string.memcmp
+      libc.src.string.memcpy
+      libc.src.string.memmove
+      libc.src.string.memset
+  )
+  list(REMOVE_DUPLICATES fq_deps_list)
+
+  # TODO: Instead of gathering internal object files from entrypoints,
+  # collect the object files with public names of entrypoints.
+  get_object_files_for_test(
+      link_object_files skipped_entrypoints_list ${fq_deps_list})
+  if(skipped_entrypoints_list)
+    set(msg "Skipping unittest ${fq_target_name} as it has missing deps: "
+            "${skipped_entrypoints_list}.")
+    message(STATUS ${msg})
+    return()
+  endif()
+  list(REMOVE_DUPLICATES link_object_files)
+
+  # Make a library of all deps
+  add_library(
+    ${fq_target_name}.__libc__
+    STATIC
+    EXCLUDE_FROM_ALL
+    ${link_object_files}
+  )
+  set_target_properties(${fq_target_name}.__libc__
+      PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  set_target_properties(${fq_target_name}.__libc__
+      PROPERTIES ARCHIVE_OUTPUT_NAME ${fq_target_name}.libc)
+
+  set(fq_build_target_name ${fq_target_name}.__build__)
+  add_executable(
+    ${fq_build_target_name}
+    EXCLUDE_FROM_ALL
+    # The NVIDIA 'nvlink' linker does not currently support static libraries.
+    $<$<BOOL:${LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX}>:${link_object_files}>
+    ${HERMETIC_TEST_SRCS}
+    ${HERMETIC_TEST_HDRS}
+  )
+  set_target_properties(${fq_build_target_name}
+    PROPERTIES
+      RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      #OUTPUT_NAME ${fq_target_name}
+  )
+  target_include_directories(
+    ${fq_build_target_name}
+    PRIVATE
+      ${LIBC_SOURCE_DIR}
+      ${LIBC_BUILD_DIR}
+      ${LIBC_BUILD_DIR}/include
+  )
+  target_compile_options(${fq_build_target_name}
+      PRIVATE ${LIBC_HERMETIC_TEST_COMPILE_OPTIONS} ${HERMETIC_TEST_COMPILE_OPTIONS})
+
+  if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
+    target_link_options(${fq_build_target_name} PRIVATE -nostdlib -static)
+  else()
+    target_link_options(${fq_build_target_name} PRIVATE -nolibc -nostartfiles -nostdlib++ -static)
+  endif()
+  target_link_libraries(
+    ${fq_build_target_name}
+    PRIVATE
+      ${HERMETIC_TEST_LINK_LIBRARIES}
+      libc.startup.${LIBC_TARGET_OS}.crt1
+      LibcHermeticTestMain LibcHermeticTest
+      # The NVIDIA 'nvlink' linker does not currently support static libraries.
+      $<$<NOT:$<BOOL:${LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX}>>:${fq_target_name}.__libc__>)
+  add_dependencies(${fq_build_target_name}
+                   LibcHermeticTest
+                   ${fq_deps_list})
+
+  # Tests on the GPU require an external loader utility to launch the kernel.
+  if(TARGET libc.utils.gpu.loader)
+    add_dependencies(${fq_build_target_name} libc.utils.gpu.loader)
+    get_target_property(gpu_loader_exe libc.utils.gpu.loader "EXECUTABLE")
+  endif()
+
+  set(test_cmd ${HERMETIC_TEST_ENV}
+      $<$<BOOL:${LIBC_TARGET_ARCHITECTURE_IS_GPU}>:${gpu_loader_exe}> ${HERMETIC_TEST_LOADER_ARGS}
+      $<TARGET_FILE:${fq_build_target_name}> ${HERMETIC_TEST_ARGS})
+  add_custom_target(
+    ${fq_target_name}
+    COMMAND ${test_cmd}
+    COMMAND_EXPAND_LISTS
+    COMMENT "Running hermetic test ${fq_target_name}"
+  )
+
+  add_dependencies(${HERMETIC_TEST_SUITE} ${fq_target_name})
+  add_dependencies(libc-hermetic-tests ${fq_target_name})
+endfunction(add_libc_hermetic_test)
+
+# A convenience function to add both a unit test as well as a hermetic test.
+function(add_libc_test test_name)
+  cmake_parse_arguments(
+    "LIBC_TEST"
+    "UNIT_TEST_ONLY;HERMETIC_TEST_ONLY" # Optional arguments
+    "" # Single value arguments
+    "" # Multi-value arguments
+    ${ARGN}
+  )
+  if(LIBC_ENABLE_UNITTESTS AND NOT LIBC_TEST_HERMETIC_TEST_ONLY)
+    add_libc_unittest(${test_name}.__unit__ ${LIBC_TEST_UNPARSED_ARGUMENTS})
+  endif()
+  if(LIBC_ENABLE_HERMETIC_TESTS AND NOT LIBC_TEST_UNIT_TEST_ONLY)
+    add_libc_hermetic_test(${test_name}.__hermetic__ ${LIBC_TEST_UNPARSED_ARGUMENTS})
+  endif()
+endfunction(add_libc_test)
diff --git a/libc/cmake/modules/prepare_libc_gpu_build.cmake b/libc/cmake/modules/prepare_libc_gpu_build.cmake
index 5d7d8ac2b0fca..79afa383b00e3 100644
--- a/libc/cmake/modules/prepare_libc_gpu_build.cmake
+++ b/libc/cmake/modules/prepare_libc_gpu_build.cmake
@@ -44,7 +44,7 @@ foreach(arch_tool ${LIBC_NVPTX_ARCH} ${LIBC_AMDGPU_ARCH})
   if(arch_tool)
     execute_process(COMMAND ${arch_tool}
                     OUTPUT_VARIABLE arch_tool_output
-                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+                    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
     string(REPLACE "\n" ";" arch_list "${arch_tool_output}")
     list(APPEND detected_gpu_architectures "${arch_list}")
   endif()
diff --git a/libc/config/gpu/api.td b/libc/config/gpu/api.td
index 9036bbc89238e..c5f555c90178c 100644
--- a/libc/config/gpu/api.td
+++ b/libc/config/gpu/api.td
@@ -5,3 +5,14 @@ include "spec/stdc.td"
 def StringAPI : PublicAPI<"string.h"> {
   let Types = ["size_t"];
 }
+
+def StdlibAPI : PublicAPI<"stdlib.h"> {
+  let Types = [
+    "size_t",
+    "__atexithandler_t",
+  ];
+}
+
+def FenvAPI: PublicAPI<"fenv.h"> {
+  let Types = ["fenv_t"];
+}
diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index 4c78b2fc79648..170f21c73f808 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -53,6 +53,25 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strstr
     libc.src.string.strtok
     libc.src.string.strtok_r
+
+    # stdlib.h entrypoints
+    libc.src.stdlib.abs
+    libc.src.stdlib.labs
+    libc.src.stdlib.llabs
+
+    # stdlib.h entrypoints
+    libc.src.stdlib._Exit
+    libc.src.stdlib.atexit
+    libc.src.stdlib.exit
+
+    # Only implemented in the test suite
+    libc.src.stdlib.malloc
+    libc.src.stdlib.aligned_alloc
+    libc.src.stdlib.realloc
+    libc.src.stdlib.free
+
+    # errno.h entrypoints
+    libc.src.errno.errno
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/config/gpu/headers.txt b/libc/config/gpu/headers.txt
index 2280244883de4..608ff42cbdbc5 100644
--- a/libc/config/gpu/headers.txt
+++ b/libc/config/gpu/headers.txt
@@ -1,4 +1,7 @@
 set(TARGET_PUBLIC_HEADERS
     libc.include.ctype
     libc.include.string
+    libc.include.fenv
+    libc.include.errno
+    libc.include.stdlib
 )
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 8fac4e041f1bc..4aa06c7845e1f 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -203,6 +203,10 @@ def SysSelectAPI : PublicAPI<"sys/select.h"> {
                "struct timeval"];
 }
 
+def SysSocketAPI : PublicAPI<"sys/socket.h"> {
+  let Types = ["struct sockaddr", "sa_family_t"];
+}
+
 def SysResourceAPI : PublicAPI<"sys/resource.h"> {
   let Types = ["rlim_t", "struct rlimit"];
 }
diff --git a/libc/config/linux/riscv64/entrypoints.txt b/libc/config/linux/riscv64/entrypoints.txt
index dc700941c667c..c36a623701e5c 100644
--- a/libc/config/linux/riscv64/entrypoints.txt
+++ b/libc/config/linux/riscv64/entrypoints.txt
@@ -341,6 +341,12 @@ if(LLVM_LIBC_FULL_BUILD)
     # assert.h entrypoints
     libc.src.assert.__assert_fail
 
+    # dirent.h entrypoints
+    libc.src.dirent.closedir
+    libc.src.dirent.dirfd
+    libc.src.dirent.opendir
+    libc.src.dirent.readdir
+
     # network.h entrypoints
     libc.src.network.htonl
     libc.src.network.htons
diff --git a/libc/config/linux/riscv64/headers.txt b/libc/config/linux/riscv64/headers.txt
index 2d8ed5001c143..aaa75a9dd08cb 100644
--- a/libc/config/linux/riscv64/headers.txt
+++ b/libc/config/linux/riscv64/headers.txt
@@ -1,6 +1,7 @@
 set(TARGET_PUBLIC_HEADERS
     libc.include.assert
     libc.include.ctype
+    libc.include.dirent
     libc.include.errno
     libc.include.fcntl
     libc.include.fenv
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 5ac24c6daade0..532a7dc72d84f 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -145,6 +145,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     # sys/sendfile entrypoints
     libc.src.sys.sendfile.sendfile
 
+    # sys/socket.h entrypoints
+    libc.src.sys.socket.socket
+
     # sys/stat.h entrypoints
     libc.src.sys.stat.chmod
     libc.src.sys.stat.fchmod
diff --git a/libc/docs/contributing.rst b/libc/docs/contributing.rst
index 65ba9a4079704..7f02630145eb9 100644
--- a/libc/docs/contributing.rst
+++ b/libc/docs/contributing.rst
@@ -48,9 +48,9 @@ a list of open projects that one can start with:
    CI builders.
 
 #. **double and higher precision math functions** - These are under active
-   developement but you can take a shot at those not yet implemented. See
+   development but you can take a shot at those not yet implemented. See
    :ref:`math` for more information.
 
 #. **Contribute a new OS/Architecture port** - You can contribute a new
    operating system or target architecture port. See :ref:`porting` for more
-   informaton.
+   information.
diff --git a/libc/docs/dev/api_test.rst b/libc/docs/dev/api_test.rst
index e39d506c3a92f..85239bbe78941 100644
--- a/libc/docs/dev/api_test.rst
+++ b/libc/docs/dev/api_test.rst
@@ -7,7 +7,7 @@ The implementation of libc-project is unique because our public C header files
 are generated using information from ground truth captured in TableGen files.
 Unit tests only exercise the internal C++ implementations and don't ensure the
 headers were generated by the build system and that the generated header files
-contain the extpected declarations and definitions. A simple solution is to have
+contain the expected declarations and definitions. A simple solution is to have
 contributors write an integration test for each individual function as a C
 program; however, this would place a large burden on contributors and duplicates
 some effort from the unit tests.
diff --git a/libc/docs/dev/code_style.rst b/libc/docs/dev/code_style.rst
index e29ebf99d03e2..edbd0ef75438c 100644
--- a/libc/docs/dev/code_style.rst
+++ b/libc/docs/dev/code_style.rst
@@ -15,11 +15,11 @@ differences are as follows:
 #. **Non-const variables** - This includes function arguments, struct and
    class data members, non-const globals and local variables. They all use the
    ``snake_case`` style.
-#. **const and constexpr variables** - They use the capitlized
+#. **const and constexpr variables** - They use the capitalized
    ``SNAKE_CASE`` irrespective of whether they are local or global.
 #. **Function and methods** - They use the ``snake_case`` style like the
    non-const variables.
-#. **Internal type names** - These are types which are interal to the libc
+#. **Internal type names** - These are types which are internal to the libc
    implementation. They use the ``CaptilizedCamelCase`` style.
 #. **Public names** - These are the names as prescribed by the standards and
    will follow the style as prescribed by the standards.
@@ -39,7 +39,7 @@ We define two kinds of macros: **code defined** and **build defined** macros.
 
    * **Properties** - Build related properties like used compiler, target
      architecture or enabled CPU features defined by introspecting compiler
-     defined preprocessor defininitions. e.g., ``LIBC_TARGET_ARCH_IS_ARM``,
+     defined preprocessor definitions. e.g., ``LIBC_TARGET_ARCH_IS_ARM``,
      ``LIBC_TARGET_CPU_HAS_AVX2``, ``LIBC_COMPILER_IS_CLANG``, ...
    * **Attributes** - Compiler agnostic attributes or functions to handle
      specific operations. e.g., ``LIBC_INLINE``, ``LIBC_NO_LOOP_UNROLL``,
@@ -98,7 +98,7 @@ followed:
 
 #. The header file ``src/errno/libc_errno.h`` is shipped as part of the target
    corresponding to the ``errno`` entrypoint ``libc.src.errno.errno``. We do
-   not in general allow dependecies between entrypoints. However, the ``errno``
+   not in general allow dependencies between entrypoints. However, the ``errno``
    entrypoint is the only exceptional entrypoint on which other entrypoints
    should explicitly depend on if they set ``errno`` to indicate error
    conditions.
@@ -155,5 +155,5 @@ The only exception to using the above pattern is if allocating using the
 failures will still need to be handled gracefully. Further, keep in mind that
 these functions do not call the constructors and destructors of the
 allocated/deallocated objects. So, use these functions carefully and only
-when it is absolutely clear that constructor and desctructor invocation is
+when it is absolutely clear that constructor and destructor invocation is
 not required.
diff --git a/libc/docs/dev/mechanics_of_public_api.rst b/libc/docs/dev/mechanics_of_public_api.rst
index 992c8ac304a6f..257ab3d71bc17 100644
--- a/libc/docs/dev/mechanics_of_public_api.rst
+++ b/libc/docs/dev/mechanics_of_public_api.rst
@@ -17,7 +17,7 @@ The API config file lists two kinds of items:
 
 1. The list of standards from which the public entities available on the platform
    are derived from.
-2. For each header file exposed on the platfrom, the list of public members
+2. For each header file exposed on the platform, the list of public members
    provided in that header file.
 
 Note that, the header generator only learns the names of the public entities
diff --git a/libc/docs/gpu/index.rst b/libc/docs/gpu/index.rst
new file mode 100644
index 0000000000000..0ea54a7235459
--- /dev/null
+++ b/libc/docs/gpu/index.rst
@@ -0,0 +1,18 @@
+.. _libc_gpu:
+
+=============
+libc for GPUs
+=============
+
+.. note:: This feature is very experimental and may change in the future.
+
+The *GPU* support for LLVM's libc project aims to make a subset of the standard
+C library available on GPU based accelerators. Navigate using the links below to
+learn more about this project.
+
+.. toctree::
+
+   using
+   support
+   testing
+   rpc
diff --git a/libc/docs/gpu/rpc.rst b/libc/docs/gpu/rpc.rst
new file mode 100644
index 0000000000000..bdc2c4ac312cf
--- /dev/null
+++ b/libc/docs/gpu/rpc.rst
@@ -0,0 +1,17 @@
+.. _libc_gpu_rpc:
+
+======================
+Remote Procedure Calls
+======================
+
+.. contents:: Table of Contents
+  :depth: 4
+  :local:
+
+Remote Procedure Call Implementation
+====================================
+
+Certain features from the standard C library, such as allocation or printing,
+require support from the operating system. We instead implement a remote
+procedure call (RPC) interface to allow submitting work from the GPU to a host
+server that forwards it to the host system.
diff --git a/libc/docs/gpu/support.rst b/libc/docs/gpu/support.rst
new file mode 100644
index 0000000000000..cc3c326f9e565
--- /dev/null
+++ b/libc/docs/gpu/support.rst
@@ -0,0 +1,101 @@
+.. _libc_gpu_support:
+
+===================
+Supported Functions
+===================
+
+.. include:: ../check.rst
+
+.. contents:: Table of Contents
+  :depth: 4
+  :local:
+
+The following functions and headers are supported at least partially on the
+device. Some functions are implemented fully on the GPU, while others require a
+`remote procedure call <libc_gpu_rpc>`.
+
+ctype.h
+-------
+
+=============  =========  ============
+Function Name  Available  RPC Required
+=============  =========  ============
+isalnum        |check|
+isalpha        |check|
+isascii        |check|
+isblank        |check|
+iscntrl        |check|
+isdigit        |check|
+isgraph        |check|
+islower        |check|
+isprint        |check|
+ispunct        |check|
+isspace        |check|
+isupper        |check|
+isxdigit       |check|
+toascii        |check|
+tolower        |check|
+toupper        |check|
+=============  =========  ============
+
+string.h
+--------
+
+=============  =========  ============
+Function Name  Available  RPC Required
+=============  =========  ============
+bcmp           |check|
+bzero          |check|
+memccpy        |check|
+memchr         |check|
+memcmp         |check|
+memcpy         |check|
+memmove        |check|
+mempcpy        |check|
+memrchr        |check|
+memset         |check|
+stpcpy         |check|
+stpncpy        |check|
+strcat         |check|
+strchr         |check|
+strcmp         |check|
+strcpy         |check|
+strcspn        |check|
+strlcat        |check|
+strlcpy        |check|
+strlen         |check|
+strncat        |check|
+strncmp        |check|
+strncpy        |check|
+strnlen        |check|
+strpbrk        |check|
+strrchr        |check|
+strspn         |check|
+strstr         |check|
+strtok         |check|
+strtok_r       |check|
+strdup
+strndup
+=============  =========  ============
+
+stdlib.h
+--------
+
+=============  =========  ============
+Function Name  Available  RPC Required
+=============  =========  ============
+abs            |check|
+atoi           
+atof           
+atol           
+atoll          
+labs           |check|
+llabs          |check|
+strtod         
+strtof         
+strtol         
+strtold        
+strtoll        
+strtoul        
+strtoull       
+=============  =========  ============
diff --git a/libc/docs/gpu/testing.rst b/libc/docs/gpu/testing.rst
new file mode 100644
index 0000000000000..376a51ec8067b
--- /dev/null
+++ b/libc/docs/gpu/testing.rst
@@ -0,0 +1,37 @@
+.. _libc_gpu_testing:
+
+
+============================
+Testing the GPU libc library
+============================
+
+.. note::
+   Running GPU tests with high parallelism is likely to cause spurious failures,
+   out of resource errors, or indefinate hangs. limiting the number of threads 
+   used while testing is highly recommended.
+
+.. contents:: Table of Contents
+  :depth: 4
+  :local:
+
+Testing Infrastructure
+======================
+
+The testing support in LLVM's libc implementation for GPUs is designed to mimic
+the standard unit tests as much as possible. We use the `remote procedure call
+<libc_gpu_rpc>` support to provide the necessary utilities like printing from
+the GPU. Execution is performed by emitting a ``_start`` kernel from the GPU
+that is then called by an external loader utility. This is an example of how
+this can be done manually:
+
+.. code-block:: sh
+
+   $> clang++ crt1.o test.cpp --target=amdgcn-amd-amdhsa -mcpu=gfx90a -flto
+   $> ./amdhsa_loader --threads 1 --blocks 1 a.out
+   Test Passed!
+
+Unlike the exported ``libcgpu.a``, the testing architecture can only support a
+single architecture at a time. This is either detected automatically, or set
+manually by the user using ``LIBC_GPU_TEST_ARCHITECTURE``. The latter is useful
+in cases where the user does not build LLVM's libc on machine with the GPU to
+use for testing.
diff --git a/libc/docs/gpu/using.rst b/libc/docs/gpu/using.rst
new file mode 100644
index 0000000000000..6808f05ad13b6
--- /dev/null
+++ b/libc/docs/gpu/using.rst
@@ -0,0 +1,87 @@
+.. _libc_gpu_usage:
+
+
+===================
+Using libc for GPUs
+===================
+
+.. contents:: Table of Contents
+  :depth: 4
+  :local:
+
+Building the GPU library
+========================
+
+LLVM's libc GPU support *must* be built with an up-to-date ``clang`` compiler
+due to heavy reliance on ``clang``'s GPU support. This can be done automatically
+using the ``LLVM_ENABLE_RUNTIMES=libc`` option. To enable libc for the GPU,
+enable the ``LIBC_GPU_BUILD`` option. By default, ``libcgpu.a`` will be built
+using every supported GPU architecture. To restrict the number of architectures
+build, either set ``LLVM_LIBC_GPU_ARCHITECTURES`` to the list of desired
+architectures manually or use ``native`` to detect the GPUs on your system. A
+typical ``cmake`` configuration will look like this:
+
+.. code-block:: sh
+
+  $> cd llvm-project  # The llvm-project checkout
+  $> mkdir build
+  $> cd build
+  $> cmake ../llvm -G Ninja                                \
+     -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt"        \
+     -DLLVM_ENABLE_RUNTIMES="libc;openmp"                  \
+     -DCMAKE_BUILD_TYPE=<Debug|Release>   \ # Select build type
+     -DLIBC_GPU_BUILD=ON                  \ # Build in GPU mode
+     -DLLVM_LIBC_GPU_ARCHITECTURES=all    \ # Build all supported architectures
+     -DCMAKE_INSTALL_PREFIX=<PATH>        \ # Where 'libcgpu.a' will live
+  $> ninja install
+
+Since we want to include ``clang``, ``lld`` and ``compiler-rt`` in our
+toolchain, we list them in ``LLVM_ENABLE_PROJECTS``. To ensure ``libc`` is built
+using a compatible compiler and to support ``openmp`` offloading, we list them
+in ``LLVM_ENABLE_RUNTIMES`` to build them after the enabled projects using the
+newly built compiler. ``CMAKE_INSTALL_PREFIX`` specifies the installation
+directory in which to install the ``libcgpu.a`` library and headers along with
+LLVM. The generated headers will be placed in ``include/gpu-none-llvm``.
+
+Usage
+=====
+
+Once the ``libcgpu.a`` static archive has been built it can be linked directly
+with offloading applications as a standard library. This process is described in
+the `clang documentation <https://clang.llvm.org/docs/OffloadingDesign.html>`_.
+This linking mode is used by the OpenMP toolchain, but is currently opt-in for
+the CUDA and HIP toolchains through the ``--offload-new-driver``` and
+``-fgpu-rdc`` flags. A typical usage will look this this:
+
+.. code-block:: sh
+
+  $> clang foo.c -fopenmp --offload-arch=gfx90a -lcgpu
+
+The ``libcgpu.a`` static archive is a fat-binary containing LLVM-IR for each
+supported target device. The supported architectures can be seen using LLVM's
+``llvm-objdump`` with the ``--offloading`` flag:
+
+.. code-block:: sh
+
+  $> llvm-objdump --offloading libcgpu.a
+  libcgpu.a(strcmp.cpp.o):    file format elf64-x86-64
+
+  OFFLOADING IMAGE [0]:
+  kind            llvm ir
+  arch            gfx90a
+  triple          amdgcn-amd-amdhsa
+  producer        none
+
+Because the device code is stored inside a fat binary, it can be difficult to
+inspect the resulting code. This can be done using the following utilities:
+
+.. code-block:: sh
+
+   $> llvm-ar x libcgpu.a strcmp.cpp.o
+   $> clang-offload-packager strcmp.cpp.o --image=arch=gfx90a,file=gfx90a.bc
+   $> opt -S out.bc
+   ...
+
+Please note that this fat binary format is provided for compatibility with
+existing offloading toolchains. The implementation in ``libc`` does not depend
+on any existing offloading languages and is completely freestanding.
diff --git a/libc/docs/gpu_mode.rst b/libc/docs/gpu_mode.rst
deleted file mode 100644
index b71b6eec5daee..0000000000000
--- a/libc/docs/gpu_mode.rst
+++ /dev/null
@@ -1,169 +0,0 @@
-.. _GPU_mode:
-
-==============
-GPU Mode
-==============
-
-.. include:: check.rst
-
-.. contents:: Table of Contents
-  :depth: 4
-  :local:
-
-.. note:: This feature is very experimental and may change in the future.
-
-The *GPU* mode of LLVM's libc is an experimental mode used to support calling
-libc routines during GPU execution. The goal of this project is to provide
-access to the standard C library on systems running accelerators. To begin using
-this library, build and install the ``libcgpu.a`` static archive following the
-instructions in :ref:`building_gpu_mode` and link with your offloading
-application.
-
-.. _building_gpu_mode:
-
-Building the GPU library
-========================
-
-LLVM's libc GPU support *must* be built using the same compiler as the final
-application to ensure relative LLVM bitcode compatibility. This can be done
-automatically using the ``LLVM_ENABLE_RUNTIMES=libc`` option. Furthermore,
-building for the GPU is only supported in :ref:`fullbuild_mode`. To enable the
-GPU build, set the target OS to ``gpu`` via ``LLVM_LIBC_TARGET_OS=gpu``. By
-default, ``libcgpu.a`` will be built using every supported GPU architecture. To
-restrict the number of architectures build, set ``LLVM_LIBC_GPU_ARCHITECTURES``
-to the list of desired architectures or use ``all``. A typical ``cmake``
-configuration will look like this:
-
-.. code-block:: sh
-
-  $> cd llvm-project  # The llvm-project checkout
-  $> mkdir build
-  $> cd build
-  $> cmake ../llvm -G Ninja                                \
-     -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt"        \
-     -DLLVM_ENABLE_RUNTIMES="libc;openmp"                  \
-     -DCMAKE_BUILD_TYPE=<Debug|Release>  \ # Select build type
-     -DLLVM_LIBC_FULL_BUILD=ON           \ # We need the full libc
-     -DLIBC_GPU_BUILD=ON                 \ # Build in GPU mode
-     -DLLVM_LIBC_GPU_ARCHITECTURES=all   \ # Build all supported architectures
-     -DCMAKE_INSTALL_PREFIX=<PATH>       \ # Where 'libcgpu.a' will live
-  $> ninja install
-
-Since we want to include ``clang``, ``lld`` and ``compiler-rt`` in our
-toolchain, we list them in ``LLVM_ENABLE_PROJECTS``. To ensure ``libc`` is built
-using a compatible compiler and to support ``openmp`` offloading, we list them
-in ``LLVM_ENABLE_RUNTIMES`` to build them after the enabled projects using the
-newly built compiler. ``CMAKE_INSTALL_PREFIX`` specifies the installation
-directory in which to install the ``libcgpu.a`` library along with LLVM.
-
-Usage
-=====
-
-Once the ``libcgpu.a`` static archive has been built in
-:ref:`building_gpu_mode`, it can be linked directly with offloading applications
-as a standard library. This process is described in the `clang documentation
-<https://clang.llvm.org/docs/OffloadingDesign.html>_`. This linking mode is used
-by the OpenMP toolchain, but is currently opt-in for the CUDA and HIP toolchains
-using the ``--offload-new-driver``` and ``-fgpu-rdc`` flags. A typical usage
-will look this this:
-
-.. code-block:: sh
-
-  $> clang foo.c -fopenmp --offload-arch=gfx90a -lcgpu
-
-The ``libcgpu.a`` static archive is a fat-binary containing LLVM-IR for each
-supported target device. The supported architectures can be seen using LLVM's
-objdump with the ``--offloading`` flag:
-
-.. code-block:: sh
-
-  $> llvm-objdump --offloading libcgpu.a
-  libcgpu.a(strcmp.cpp.o):    file format elf64-x86-64
-
-  OFFLOADING IMAGE [0]:
-  kind            llvm ir
-  arch            gfx90a
-  triple          amdgcn-amd-amdhsa
-  producer        <none>
-
-Because the device code is stored inside a fat binary, it can be difficult to
-inspect the resulting code. This can be done using the following utilities:
-
-.. code-block:: sh
-
-   $> llvm-ar x libcgpu.a strcmp.cpp.o
-   $> clang-offload-packager strcmp.cpp.o --image=arch=gfx90a,file=gfx90a.bc
-   $> opt -S out.bc
-   ...
-
-Supported Functions
-===================
-
-The following functions and headers are supported at least partially on the
-device. Currently, only basic device functions that do not require an operating
-system are supported on the device. Supporting functions like `malloc` using an
-RPC mechanism is a work-in-progress.
-
-ctype.h
--------
-
-=============  =========
-Function Name  Available
-=============  =========
-isalnum        |check|
-isalpha        |check|
-isascii        |check|
-isblank        |check|
-iscntrl        |check|
-isdigit        |check|
-isgraph        |check|
-islower        |check|
-isprint        |check|
-ispunct        |check|
-isspace        |check|
-isupper        |check|
-isxdigit       |check|
-toascii        |check|
-tolower        |check|
-toupper        |check|
-=============  =========
-
-string.h
---------
-
-=============   =========
-Function Name   Available
-=============   =========
-bcmp            |check|
-bzero           |check|
-memccpy         |check|
-memchr          |check|
-memcmp          |check|
-memcpy          |check|
-memmove         |check|
-mempcpy         |check|
-memrchr         |check|
-memset          |check|
-stpcpy          |check|
-stpncpy         |check|
-strcat          |check|
-strchr          |check|
-strcmp          |check|
-strcpy          |check|
-strcspn         |check|
-strlcat         |check|
-strlcpy         |check|
-strlen          |check|
-strncat         |check|
-strncmp         |check|
-strncpy         |check|
-strnlen         |check|
-strpbrk         |check|
-strrchr         |check|
-strspn          |check|
-strstr          |check|
-strtok          |check|
-strtok_r        |check|
-strdup
-strndup
-=============   =========
diff --git a/libc/docs/index.rst b/libc/docs/index.rst
index 90422617403e6..5e9a602b5a96a 100644
--- a/libc/docs/index.rst
+++ b/libc/docs/index.rst
@@ -52,7 +52,7 @@ stages there is no ABI stability in any form.
    usage_modes
    overlay_mode
    fullbuild_mode
-   gpu_mode
+   gpu/index.rst
 
 .. toctree::
    :hidden:
diff --git a/libc/docs/math/log.rst b/libc/docs/math/log.rst
index c8095556cc2ac..83d03a83d131e 100644
--- a/libc/docs/math/log.rst
+++ b/libc/docs/math/log.rst
@@ -70,7 +70,7 @@ d. add step a and step c results.
 How to derive `r`
 -----------------
 
-For an efficient implementation, we would like to use the first `M` signficicant
+For an efficient implementation, we would like to use the first `M` significant
 bits of `m_x` to look up for `r`.  In particular, we would like to find a value
 of `r` that works for all `m_x` satisfying:
 
diff --git a/libc/docs/overlay_mode.rst b/libc/docs/overlay_mode.rst
index 723ec5bb2522d..f9b566658bb49 100644
--- a/libc/docs/overlay_mode.rst
+++ b/libc/docs/overlay_mode.rst
@@ -109,7 +109,7 @@ Linking the static archive to other LLVM binaries
 
 Since the libc and other LLVM binaries are developed in the same source tree,
 linking ``libllvmlibc.a`` to those LLVM binaries does not require any special
-install step or explicity passing any special linker flags/options. One can
+install step or explicitly passing any special linker flags/options. One can
 simply add ``llvmlibc`` as a link library to that binary's target. For example,
 if you want to link ``libllvmlibc.a`` to ``llvm-objcopy``, all you have to do
 is to add a CMake command as follows:
diff --git a/libc/docs/porting.rst b/libc/docs/porting.rst
index 42f7fa33bc334..3f41eb2c96991 100644
--- a/libc/docs/porting.rst
+++ b/libc/docs/porting.rst
@@ -107,13 +107,13 @@ updated.
 The headers.txt file
 ====================
 
-Another important piece of config informtion is listed in a file named
+Another important piece of config information is listed in a file named
 ``headers.txt``. It lists the targets for the set of public headers that are
 provided by the libc. This is relevant only if the libc is to be used in the
 :ref:`fullbuild_mode` on the target operating system and architecture. As with
 the ``entrypoints.txt`` file, one ``headers.txt`` file should be listed for
 each individual target architecture if you are doing an architecture specific
-bring up. The Linux config has ``headers.txt`` file listed seperately for the
+bring up. The Linux config has ``headers.txt`` file listed separately for the
 `aarch64 <https://github.com/llvm/llvm-project/tree/main/libc/config/linux/aarch64>`_
 config and the
 `x86_64 <https://github.com/llvm/llvm-project/tree/main/libc/config/linux/x86_64>`_
diff --git a/libc/fuzzing/stdlib/CMakeLists.txt b/libc/fuzzing/stdlib/CMakeLists.txt
index 3127848ee193f..785efd82a1ed3 100644
--- a/libc/fuzzing/stdlib/CMakeLists.txt
+++ b/libc/fuzzing/stdlib/CMakeLists.txt
@@ -14,6 +14,8 @@ add_libc_fuzzer(
     StringParserOutputDiff.h
   DEPENDS
     libc.src.stdlib.atof
+  COMPILE_OPTIONS
+    -DLLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
 )
 
 add_libc_fuzzer(
diff --git a/libc/fuzzing/stdlib/atof_differential_fuzz.cpp b/libc/fuzzing/stdlib/atof_differential_fuzz.cpp
index b368129960d3b..2f330e1af2e32 100644
--- a/libc/fuzzing/stdlib/atof_differential_fuzz.cpp
+++ b/libc/fuzzing/stdlib/atof_differential_fuzz.cpp
@@ -16,6 +16,40 @@
 
 #include "fuzzing/stdlib/StringParserOutputDiff.h"
 
+// TODO: Remove this once glibc fixes hex subnormal rounding. See
+// https://sourceware.org/bugzilla/show_bug.cgi?id=30220
+#ifdef LLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
+#include <ctype.h>
+constexpr double MIN_NORMAL = 0x1p-1022;
+
+bool has_hex_prefix(const uint8_t *str) {
+  size_t index = 0;
+
+  // Skip over leading whitespace
+  while (isspace(str[index])) {
+    ++index;
+  }
+  // Skip over sign
+  if (str[index] == '-' || str[index] == '+') {
+    ++index;
+  }
+  return str[index] == '0' && (tolower(str[index + 1])) == 'x';
+}
+
+bool should_be_skipped(const uint8_t *str) {
+  double init_result = ::atof(reinterpret_cast<const char *>(str));
+  if (init_result < 0) {
+    init_result = -init_result;
+  }
+  if (init_result < MIN_NORMAL && init_result != 0) {
+    return has_hex_prefix(str);
+  }
+  return false;
+}
+#else
+bool should_be_skipped(const uint8_t *) { return false; }
+#endif // LLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   uint8_t *container = new uint8_t[size + 1];
   if (!container)
@@ -26,7 +60,10 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     container[i] = data[i];
   container[size] = '\0'; // Add null terminator to container.
 
-  StringParserOutputDiff<double>(&__llvm_libc::atof, &::atof, container, size);
+  if (!should_be_skipped(container)) {
+    StringParserOutputDiff<double>(&__llvm_libc::atof, &::atof, container,
+                                   size);
+  }
   delete[] container;
   return 0;
 }
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index f3b23a5e3e26a..9482e16073aab 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -388,6 +388,8 @@ add_gen_header(
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_socket_macros
+    .llvm-libc-types.struct_sockaddr
+    .llvm-libc-types.sa_family_t
 )
 
 add_gen_header(
diff --git a/libc/include/errno.h.def b/libc/include/errno.h.def
index 580a53f6a7716..d8f79dd47a0d1 100644
--- a/libc/include/errno.h.def
+++ b/libc/include/errno.h.def
@@ -43,7 +43,9 @@
 #include <llvm-libc-macros/generic-error-number-macros.h>
 #endif
 
+#if !defined(__AMDGPU__) && !defined(__NVPTX__)
 extern _Thread_local int __llvmlibc_errno;
 #define errno __llvmlibc_errno
+#endif
 
 #endif // LLVM_LIBC_ERRNO_H
diff --git a/libc/include/llvm-libc-macros/linux/sys-socket-macros.h b/libc/include/llvm-libc-macros/linux/sys-socket-macros.h
index 74a886408bd4e..7de410225b712 100644
--- a/libc/include/llvm-libc-macros/linux/sys-socket-macros.h
+++ b/libc/include/llvm-libc-macros/linux/sys-socket-macros.h
@@ -18,4 +18,11 @@
 #define AF_INET 2   // Internet IPv4 Protocol
 #define AF_INET6 10 // IP version 6
 
+#define SOCK_STREAM 1
+#define SOCK_DGRAM 2
+#define SOCK_RAW 3
+#define SOCK_RDM 4
+#define SOCK_SEQPACKET 5
+#define SOCK_PACKET 10
+
 #endif // __LLVM_LIBC_MACROS_LINUX_SYS_SOCKET_MACROS_H
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 37230208e458a..0ff124ac9e31a 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -87,3 +87,5 @@ add_header(struct_termios HDR struct_termios.h DEPENDS .cc_t .speed_t .tcflag_t)
 add_header(__getoptargv_t HDR __getoptargv_t.h)
 add_header(wchar_t HDR wchar_t.h)
 add_header(wint_t HDR wint_t.h)
+add_header(sa_family_t HDR sa_family_t.h)
+add_header(struct_sockaddr HDR struct_sockaddr.h)
diff --git a/libc/include/llvm-libc-types/fenv_t.h b/libc/include/llvm-libc-types/fenv_t.h
index 6d2bea74560db..86fcf2e49a7ff 100644
--- a/libc/include/llvm-libc-types/fenv_t.h
+++ b/libc/include/llvm-libc-types/fenv_t.h
@@ -25,6 +25,10 @@ typedef struct {
 } fenv_t;
 #elif defined(__riscv)
 typedef unsigned int fenv_t;
+#elif defined(__AMDGPU__) || defined(__NVPTX__)
+typedef struct {
+  unsigned int __fpc;
+} fenv_t;
 #else
 #error "fenv_t not defined for your platform"
 #endif
diff --git a/libc/include/llvm-libc-types/sa_family_t.h b/libc/include/llvm-libc-types/sa_family_t.h
new file mode 100644
index 0000000000000..52b69957b0d3f
--- /dev/null
+++ b/libc/include/llvm-libc-types/sa_family_t.h
@@ -0,0 +1,19 @@
+//===-- Definition of sa_family_t type ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_SA_FAMILY_T_H__
+#define __LLVM_LIBC_TYPES_SA_FAMILY_T_H__
+
+// The posix standard only says of sa_family_t that it must be unsigned. The
+// linux man page for "address_families" lists approximately 32 different
+// address families, meaning that a short 16 bit number will have plenty of
+// space for all of them.
+
+typedef unsigned short sa_family_t;
+
+#endif // __LLVM_LIBC_TYPES_SA_FAMILY_T_H__
diff --git a/libc/include/llvm-libc-types/struct_dirent.h b/libc/include/llvm-libc-types/struct_dirent.h
index 44bda4caae452..fde3f8ce47980 100644
--- a/libc/include/llvm-libc-types/struct_dirent.h
+++ b/libc/include/llvm-libc-types/struct_dirent.h
@@ -18,6 +18,7 @@ struct dirent {
   off_t d_off;
   unsigned short d_reclen;
 #endif
+  unsigned char d_type;
   // The user code should use strlen to determine actual the size of d_name.
   // Likewise, it is incorrect and prohibited by the POSIX standard to detemine
   // the size of struct dirent type using sizeof. The size should be got using
diff --git a/libc/include/llvm-libc-types/struct_sockaddr.h b/libc/include/llvm-libc-types/struct_sockaddr.h
new file mode 100644
index 0000000000000..1ef907904ca3e
--- /dev/null
+++ b/libc/include/llvm-libc-types/struct_sockaddr.h
@@ -0,0 +1,21 @@
+//===-- Definition of struct stat -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_STRUCT_STAT_H__
+#define __LLVM_LIBC_TYPES_STRUCT_STAT_H__
+
+#include <llvm-libc-types/sa_family_t.h>
+
+struct sockaddr {
+  sa_family_t sa_family;
+  // sa_data is a variable length array. It is provided with a length of one
+  // here as a placeholder.
+  char sa_data[1];
+};
+
+#endif // __LLVM_LIBC_TYPES_STRUCT_STAT_H__
diff --git a/libc/spec/linux.td b/libc/spec/linux.td
index d5b876cadebe5..ba5f99c12ecd1 100644
--- a/libc/spec/linux.td
+++ b/libc/spec/linux.td
@@ -110,19 +110,6 @@ def Linux : StandardSpec<"Linux"> {
       ]
   >;
 
-  HeaderSpec SysSocket = HeaderSpec<
-      "sys/socket.h",
-      [
-        Macro<"AF_UNSPEC">,
-        Macro<"AF_LOCAL">,
-        Macro<"AF_INET">,
-        Macro<"AF_INET6">,
-      ],
-      [], // Types
-      [], // Enumerations
-      []  // Functions
-  >;
-
   HeaderSpec SysTime = HeaderSpec<
       "sys/time.h",
       [
@@ -184,7 +171,6 @@ def Linux : StandardSpec<"Linux"> {
     SysMMan,
     SysPrctl,
     SysRandom,
-    SysSocket,
     SysTime,
     Signal,
   ];
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 2246e49eeaaa6..b1aa2fb472124 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -80,6 +80,10 @@ def RestrictedFdSetPtr : RestrictedPtrType<FdSet>;
 
 def GetoptArgvT : NamedType<"__getoptargv_t">;
 
+def StructSockAddr : NamedType<"struct sockaddr">;
+def StructSockAddrPtr : PtrType<StructSockAddr>;
+def SAFamilyType : NamedType<"sa_family_t">;
+
 def POSIX : StandardSpec<"POSIX"> {
   PtrType CharPtr = PtrType<CharType>;
   RestrictedPtrType RestrictedCharPtr = RestrictedPtrType<CharType>;
@@ -1347,6 +1351,34 @@ def POSIX : StandardSpec<"POSIX"> {
     ]
   >;
 
+  HeaderSpec SysSocket = HeaderSpec<
+      "sys/socket.h",
+      [
+        Macro<"AF_UNSPEC">,
+        Macro<"AF_UNIX">,
+        Macro<"AF_LOCAL">,
+        Macro<"AF_INET">,
+        Macro<"AF_INET6">,
+        Macro<"SOCK_STREAM">,
+        Macro<"SOCK_DGRAM">,
+        Macro<"SOCK_RAW">,
+        Macro<"SOCK_RDM">,
+        Macro<"SOCK_SEQPACKET">,
+        Macro<"SOCK_PACKET">,
+      ], // Macros
+      [
+        StructSockAddr, SAFamilyType,
+      ], // Types
+      [], // Enumerations
+      [
+        FunctionSpec<
+          "socket",
+          RetValSpec<IntType>,
+          [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntType>]
+        >,
+      ]  // Functions
+  >;
+
   HeaderSpec SysTypes = HeaderSpec<
     "sys/types.h",
     [], // Macros
@@ -1373,6 +1405,7 @@ def POSIX : StandardSpec<"POSIX"> {
     SysMMan,
     SysResource,
     SysSelect,
+    SysSocket,
     SysStat,
     SysTypes,
     SysUtsName,
diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 7f60e5df42185..469d81dccaaf9 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -45,6 +45,15 @@ add_header_library(
     libc.src.__support.CPP.expected
 )
 
+add_header_library(
+  c_string
+  HDRS
+    c_string.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.CPP.string
+)
+
 add_header_library(
   ctype_utils
   HDRS
diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt
index 63f1fc6673fde..147e84c0e39aa 100644
--- a/libc/src/__support/CPP/CMakeLists.txt
+++ b/libc/src/__support/CPP/CMakeLists.txt
@@ -60,6 +60,7 @@ add_header_library(
   HDRS
     string.h
   DEPENDS
+    libc.include.stdlib
     .string_view
     libc.src.__support.common
     libc.src.__support.integer_to_string
diff --git a/libc/src/__support/CPP/atomic.h b/libc/src/__support/CPP/atomic.h
index 495d492a0b7c6..6922a367289a9 100644
--- a/libc/src/__support/CPP/atomic.h
+++ b/libc/src/__support/CPP/atomic.h
@@ -90,6 +90,10 @@ template <typename T> struct Atomic {
     return __atomic_fetch_or(&val, mask, int(mem_ord));
   }
 
+  T fetch_and(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) {
+    return __atomic_fetch_and(&val, mask, int(mem_ord));
+  }
+
   T fetch_sub(T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) {
     return __atomic_fetch_sub(&val, decrement, int(mem_ord));
   }
diff --git a/libc/src/__support/CPP/functional.h b/libc/src/__support/CPP/functional.h
index 5919306ad707e..e346011debab1 100644
--- a/libc/src/__support/CPP/functional.h
+++ b/libc/src/__support/CPP/functional.h
@@ -9,19 +9,54 @@
 #ifndef LLVM_LIBC_SRC_SUPPORT_CPP_FUNCTIONAL_H
 #define LLVM_LIBC_SRC_SUPPORT_CPP_FUNCTIONAL_H
 
+#include "src/__support/CPP/type_traits.h"
+#include "src/__support/CPP/utility.h"
+#include "src/__support/macros/attributes.h"
+
+#include <stdint.h>
+
 namespace __llvm_libc {
 namespace cpp {
 
-template <typename F> class function;
+/// A function type adapted from LLVM's function_ref.
+/// This class does not own the callable, so it is not in general safe to
+/// store a function.
+template <typename Fn> class function;
+
+template <typename Ret, typename... Params> class function<Ret(Params...)> {
+  Ret (*callback)(intptr_t callable, Params... params) = nullptr;
+  intptr_t callable;
 
-template <typename R, typename... Args> class function<R(Args...)> {
-  R (*func)(Args...) = nullptr;
+  template <typename Callable>
+  LIBC_INLINE static Ret callback_fn(intptr_t callable, Params... params) {
+    return (*reinterpret_cast<Callable *>(callable))(
+        forward<Params>(params)...);
+  }
 
 public:
-  constexpr function() = default;
-  template <typename F> constexpr function(F &&f) : func(f) {}
+  LIBC_INLINE function() = default;
+  LIBC_INLINE function(decltype(nullptr)) {}
+  LIBC_INLINE ~function() = default;
+
+  template <typename Callable>
+  LIBC_INLINE function(
+      Callable &&callable,
+      // This is not the copy-constructor.
+      enable_if_t<!is_same<remove_cvref_t<Callable>, function>::value> * =
+          nullptr,
+      // Functor must be callable and return a suitable type.
+      enable_if_t<is_void_v<Ret> ||
+                  is_convertible_v<
+                      decltype(declval<Callable>()(declval<Params>()...)), Ret>>
+          * = nullptr)
+      : callback(callback_fn<remove_reference_t<Callable>>),
+        callable(reinterpret_cast<intptr_t>(&callable)) {}
+
+  LIBC_INLINE Ret operator()(Params... params) const {
+    return callback(callable, forward<Params>(params)...);
+  }
 
-  constexpr R operator()(Args... params) { return func(params...); }
+  LIBC_INLINE explicit operator bool() const { return callback; }
 };
 
 } // namespace cpp
diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h
index e5614021ca17a..f888bdcb8e112 100644
--- a/libc/src/__support/CPP/type_traits.h
+++ b/libc/src/__support/CPP/type_traits.h
@@ -12,7 +12,9 @@
 namespace __llvm_libc {
 namespace cpp {
 
-template <typename T> struct type_identity { using type = T; };
+template <typename T> struct type_identity {
+  using type = T;
+};
 
 template <bool B, typename T> struct enable_if;
 template <typename T> struct enable_if<true, T> : type_identity<T> {};
@@ -49,6 +51,19 @@ template <typename T> struct remove_cv<volatile T> : type_identity<T> {};
 template <typename T> struct remove_cv<const volatile T> : type_identity<T> {};
 template <typename T> using remove_cv_t = typename remove_cv<T>::type;
 
+template <typename T> struct remove_reference : type_identity<T> {};
+template <typename T> struct remove_reference<T &> : type_identity<T> {};
+template <typename T> struct remove_reference<T &&> : type_identity<T> {};
+template <typename T>
+using remove_reference_t = typename remove_reference<T>::type;
+
+template <typename T> struct add_rvalue_reference : type_identity<T &&> {};
+
+template <typename T> struct remove_cvref {
+  using type = remove_cv_t<remove_reference_t<T>>;
+};
+template <typename T> using remove_cvref_t = typename remove_cvref<T>::type;
+
 namespace details {
 template <typename T, typename... Args> constexpr bool is_unqualified_any_of() {
   return (... || is_same_v<remove_cv_t<T>, Args>);
@@ -148,6 +163,26 @@ struct conditional<false, T, F> : type_identity<F> {};
 template <bool B, typename T, typename F>
 using conditional_t = typename conditional<B, T, F>::type;
 
+template <typename T>
+struct is_void : is_same<void, typename remove_cv<T>::type> {};
+template <typename T> inline constexpr bool is_void_v = is_void<T>::value;
+template <class T> T declval();
+
+// Compile time checks on implicit conversions.
+namespace details {
+template <typename...> using void_t = void;
+template <typename T> void convertible_to_helper(T);
+} // namespace details
+
+template <typename F, typename T, typename = void>
+inline constexpr bool is_convertible_v = false;
+
+template <typename F, typename T>
+inline constexpr bool
+    is_convertible_v<F, T,
+                     details::void_t<decltype(details::convertible_to_helper<T>(
+                         declval<F>()))>> = true;
+
 } // namespace cpp
 } // namespace __llvm_libc
 
diff --git a/libc/src/__support/CPP/utility.h b/libc/src/__support/CPP/utility.h
index 52044eb0b9b72..40455d5e61f87 100644
--- a/libc/src/__support/CPP/utility.h
+++ b/libc/src/__support/CPP/utility.h
@@ -10,6 +10,7 @@
 #define LLVM_LIBC_SRC_SUPPORT_CPP_UTILITY_H
 
 #include "src/__support/CPP/type_traits.h"
+#include "src/__support/macros/attributes.h"
 
 namespace __llvm_libc::cpp {
 
@@ -35,6 +36,21 @@ template <typename T, int N>
 using make_integer_sequence =
     typename internal::make_integer_sequence<T, N - 1>::type;
 
+template <typename T>
+LIBC_INLINE constexpr T &&forward(typename remove_reference<T>::type &value) {
+  return static_cast<T &&>(value);
+}
+
+template <typename T>
+LIBC_INLINE constexpr T &&forward(typename remove_reference<T>::type &&value) {
+  return static_cast<T &&>(value);
+}
+
+template <typename T>
+LIBC_INLINE constexpr typename remove_reference<T>::type &&move(T &&value) {
+  return static_cast<typename remove_reference<T>::type &&>(value);
+}
+
 } // namespace __llvm_libc::cpp
 
 #endif // LLVM_LIBC_SRC_SUPPORT_CPP_UTILITY_H
diff --git a/libc/src/__support/FPUtil/PlatformDefs.h b/libc/src/__support/FPUtil/PlatformDefs.h
index 488bfc3bbd96a..175a62a1c930e 100644
--- a/libc/src/__support/FPUtil/PlatformDefs.h
+++ b/libc/src/__support/FPUtil/PlatformDefs.h
@@ -17,8 +17,9 @@
 
 // https://developer.arm.com/documentation/dui0491/i/C-and-C---Implementation-Details/Basic-data-types
 // https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms
-#if defined(_WIN32) || defined(__arm__) ||                                     \
-    (defined(__APPLE__) && defined(__aarch64__))
+// https://docs.amd.com/bundle/HIP-Programming-Guide-v5.1/page/Programming_with_HIP.html
+#if defined(_WIN32) || defined(__arm__) || defined(__NVPTX__) ||               \
+    defined(__AMDGPU__) || (defined(__APPLE__) && defined(__aarch64__))
 #define LONG_DOUBLE_IS_DOUBLE
 #endif
 
diff --git a/libc/src/__support/File/CMakeLists.txt b/libc/src/__support/File/CMakeLists.txt
index 2d94c2f30a49d..79de9250c642b 100644
--- a/libc/src/__support/File/CMakeLists.txt
+++ b/libc/src/__support/File/CMakeLists.txt
@@ -11,7 +11,6 @@ add_object_library(
   HDRS
     file.h
   DEPENDS
-    
     libc.src.__support.CPP.new
     libc.src.__support.CPP.span
     libc.src.__support.threads.mutex
@@ -37,7 +36,6 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}_file.cpp)
       ${LIBC_TARGET_OS}_file.cpp
     DEPENDS
       .file
-      
       libc.include.fcntl
       libc.include.stdio
       libc.include.sys_syscall
diff --git a/libc/src/__support/File/dir.cpp b/libc/src/__support/File/dir.cpp
index 70f550ffab72d..e632c9293d617 100644
--- a/libc/src/__support/File/dir.cpp
+++ b/libc/src/__support/File/dir.cpp
@@ -12,8 +12,6 @@
 #include "src/__support/error_or.h"
 #include "src/errno/libc_errno.h" // For error macros
 
-#include <stdlib.h>
-
 namespace __llvm_libc {
 
 ErrorOr<Dir *> Dir::open(const char *path) {
diff --git a/libc/src/__support/File/linux_dir.cpp b/libc/src/__support/File/linux_dir.cpp
index 86aaaae907d22..aae565ffb337a 100644
--- a/libc/src/__support/File/linux_dir.cpp
+++ b/libc/src/__support/File/linux_dir.cpp
@@ -34,8 +34,13 @@ ErrorOr<int> platform_opendir(const char *name) {
 }
 
 ErrorOr<size_t> platform_fetch_dirents(int fd, cpp::span<uint8_t> buffer) {
-  long size =
-      __llvm_libc::syscall_impl(SYS_getdents, fd, buffer.data(), buffer.size());
+#ifdef SYS_getdents64
+  long size = __llvm_libc::syscall_impl(SYS_getdents64, fd, buffer.data(),
+                                        buffer.size());
+#else
+#error "getdents64 syscalls not available to perform a fetch dirents operation."
+#endif
+
   if (size < 0) {
     return __llvm_libc::Error(static_cast<int>(-size));
   }
diff --git a/libc/src/__support/GPU/amdgpu/utils.h b/libc/src/__support/GPU/amdgpu/utils.h
index be90cb3edbc7a..87cd6451445a3 100644
--- a/libc/src/__support/GPU/amdgpu/utils.h
+++ b/libc/src/__support/GPU/amdgpu/utils.h
@@ -14,11 +14,136 @@
 #include <stdint.h>
 
 namespace __llvm_libc {
+namespace gpu {
 
+/// The number of threads that execute in lock-step in a lane.
+constexpr const uint64_t LANE_SIZE = __AMDGCN_WAVEFRONT_SIZE;
+
+/// Returns the number of workgroups in the 'x' dimension of the grid.
+LIBC_INLINE uint32_t get_num_blocks_x() {
+  return __builtin_amdgcn_grid_size_x() / __builtin_amdgcn_workgroup_size_x();
+}
+
+/// Returns the number of workgroups in the 'y' dimension of the grid.
+LIBC_INLINE uint32_t get_num_blocks_y() {
+  return __builtin_amdgcn_grid_size_y() / __builtin_amdgcn_workgroup_size_y();
+}
+
+/// Returns the number of workgroups in the 'z' dimension of the grid.
+LIBC_INLINE uint32_t get_num_blocks_z() {
+  return __builtin_amdgcn_grid_size_z() / __builtin_amdgcn_workgroup_size_z();
+}
+
+/// Returns the total number of workgruops in the grid.
+LIBC_INLINE uint64_t get_num_blocks() {
+  return get_num_blocks_x() * get_num_blocks_y() * get_num_blocks_z();
+}
+
+/// Returns the 'x' dimension of the current AMD workgroup's id.
 LIBC_INLINE uint32_t get_block_id_x() {
   return __builtin_amdgcn_workgroup_id_x();
 }
 
+/// Returns the 'y' dimension of the current AMD workgroup's id.
+LIBC_INLINE uint32_t get_block_id_y() {
+  return __builtin_amdgcn_workgroup_id_y();
+}
+
+/// Returns the 'z' dimension of the current AMD workgroup's id.
+LIBC_INLINE uint32_t get_block_id_z() {
+  return __builtin_amdgcn_workgroup_id_z();
+}
+
+/// Returns the absolute id of the AMD workgroup.
+LIBC_INLINE uint64_t get_block_id() {
+  return get_block_id_x() + get_num_blocks_x() * get_block_id_y() +
+         get_num_blocks_x() * get_num_blocks_y() * get_block_id_z();
+}
+
+/// Returns the number of workitems in the 'x' dimension.
+LIBC_INLINE uint32_t get_num_threads_x() {
+  return __builtin_amdgcn_workgroup_size_x();
+}
+
+/// Returns the number of workitems in the 'y' dimension.
+LIBC_INLINE uint32_t get_num_threads_y() {
+  return __builtin_amdgcn_workgroup_size_y();
+}
+
+/// Returns the number of workitems in the 'z' dimension.
+LIBC_INLINE uint32_t get_num_threads_z() {
+  return __builtin_amdgcn_workgroup_size_z();
+}
+
+/// Returns the total number of workitems in the workgroup.
+LIBC_INLINE uint64_t get_num_threads() {
+  return get_num_threads_x() * get_num_threads_y() * get_num_threads_z();
+}
+
+/// Returns the 'x' dimension id of the workitem in the current AMD workgroup.
+LIBC_INLINE uint32_t get_thread_id_x() {
+  return __builtin_amdgcn_workitem_id_x();
+}
+
+/// Returns the 'y' dimension id of the workitem in the current AMD workgroup.
+LIBC_INLINE uint32_t get_thread_id_y() {
+  return __builtin_amdgcn_workitem_id_y();
+}
+
+/// Returns the 'z' dimension id of the workitem in the current AMD workgroup.
+LIBC_INLINE uint32_t get_thread_id_z() {
+  return __builtin_amdgcn_workitem_id_z();
+}
+
+/// Returns the absolute id of the thread in the current AMD workgroup.
+LIBC_INLINE uint64_t get_thread_id() {
+  return get_thread_id_x() + get_num_threads_x() * get_thread_id_y() +
+         get_num_threads_x() * get_num_threads_y() * get_thread_id_z();
+}
+
+/// Returns the size of an AMD wavefront. Either 32 or 64 depending on hardware.
+LIBC_INLINE uint32_t get_lane_size() { return LANE_SIZE; }
+
+/// Returns the id of the thread inside of an AMD wavefront executing together.
+[[clang::convergent]] LIBC_INLINE uint32_t get_lane_id() {
+  if constexpr (LANE_SIZE == 64)
+    return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
+  else
+    return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
+}
+
+/// Returns the bit-mask of active threads in the current wavefront.
+[[clang::convergent]] LIBC_INLINE uint64_t get_lane_mask() {
+  return __builtin_amdgcn_read_exec();
+}
+
+/// Copies the value from the first active thread in the wavefront to the rest.
+[[clang::convergent]] LIBC_INLINE uint32_t broadcast_value(uint32_t x) {
+  return __builtin_amdgcn_readfirstlane(x);
+}
+
+[[clang::convergent]] LIBC_INLINE uint64_t ballot(uint64_t lane_mask, bool x) {
+  // the lane_mask & gives the nvptx semantics when lane_mask is a subset of
+  // the active threads
+  if constexpr (LANE_SIZE == 64) {
+    return lane_mask & __builtin_amdgcn_ballot_w64(x);
+  } else {
+    return lane_mask & __builtin_amdgcn_ballot_w32(x);
+  }
+}
+
+/// Waits for all the threads in the block to converge and issues a fence.
+[[clang::convergent]] LIBC_INLINE void sync_threads() {
+  __builtin_amdgcn_s_barrier();
+  __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup");
+}
+
+/// Wait for all threads in the wavefront to converge, this is a noop on AMDGPU.
+[[clang::convergent]] LIBC_INLINE void sync_lane(uint64_t) {
+  __builtin_amdgcn_wave_barrier();
+}
+
+} // namespace gpu
 } // namespace __llvm_libc
 
 #endif
diff --git a/libc/src/__support/GPU/generic/utils.h b/libc/src/__support/GPU/generic/utils.h
index d54551f953712..546e83e033afa 100644
--- a/libc/src/__support/GPU/generic/utils.h
+++ b/libc/src/__support/GPU/generic/utils.h
@@ -14,9 +14,60 @@
 #include <stdint.h>
 
 namespace __llvm_libc {
+namespace gpu {
+
+constexpr const uint64_t LANE_SIZE = 1;
+
+LIBC_INLINE uint32_t get_num_blocks_x() { return 1; }
+
+LIBC_INLINE uint32_t get_num_blocks_y() { return 1; }
+
+LIBC_INLINE uint32_t get_num_blocks_z() { return 1; }
+
+LIBC_INLINE uint64_t get_num_blocks() { return 1; }
 
 LIBC_INLINE uint32_t get_block_id_x() { return 0; }
 
+LIBC_INLINE uint32_t get_block_id_y() { return 0; }
+
+LIBC_INLINE uint32_t get_block_id_z() { return 0; }
+
+LIBC_INLINE uint64_t get_block_id() { return 0; }
+
+LIBC_INLINE uint32_t get_num_threads_x() { return 1; }
+
+LIBC_INLINE uint32_t get_num_threads_y() { return 1; }
+
+LIBC_INLINE uint32_t get_num_threads_z() { return 1; }
+
+LIBC_INLINE uint64_t get_num_threads() { return 1; }
+
+LIBC_INLINE uint32_t get_thread_id_x() { return 0; }
+
+LIBC_INLINE uint32_t get_thread_id_y() { return 0; }
+
+LIBC_INLINE uint32_t get_thread_id_z() { return 0; }
+
+LIBC_INLINE uint64_t get_thread_id() { return 0; }
+
+LIBC_INLINE uint32_t get_lane_size() { return LANE_SIZE; }
+
+LIBC_INLINE uint32_t get_lane_id() { return 0; }
+
+LIBC_INLINE uint64_t get_lane_mask() { return 1; }
+
+LIBC_INLINE uint32_t broadcast_value(uint32_t x) { return x; }
+
+LIBC_INLINE uint64_t ballot(uint64_t lane_mask, bool x) {
+  (void)lane_mask;
+  return x;
+}
+
+LIBC_INLINE void sync_threads() {}
+
+LIBC_INLINE void sync_lane(uint64_t) {}
+
+} // namespace gpu
 } // namespace __llvm_libc
 
 #endif
diff --git a/libc/src/__support/GPU/nvptx/utils.h b/libc/src/__support/GPU/nvptx/utils.h
index fa361cdbdf5c5..763675735a2b2 100644
--- a/libc/src/__support/GPU/nvptx/utils.h
+++ b/libc/src/__support/GPU/nvptx/utils.h
@@ -2,7 +2,7 @@
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-License-id: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -14,9 +14,126 @@
 #include <stdint.h>
 
 namespace __llvm_libc {
+namespace gpu {
 
+/// The number of threads that execute in lock-step in a warp.
+constexpr const uint64_t LANE_SIZE = 32;
+
+/// Returns the number of CUDA blocks in the 'x' dimension.
+LIBC_INLINE uint32_t get_num_blocks_x() {
+  return __nvvm_read_ptx_sreg_nctaid_x();
+}
+
+/// Returns the number of CUDA blocks in the 'y' dimension.
+LIBC_INLINE uint32_t get_num_blocks_y() {
+  return __nvvm_read_ptx_sreg_nctaid_y();
+}
+
+/// Returns the number of CUDA blocks in the 'z' dimension.
+LIBC_INLINE uint32_t get_num_blocks_z() {
+  return __nvvm_read_ptx_sreg_nctaid_z();
+}
+
+/// Returns the total number of CUDA blocks.
+LIBC_INLINE uint64_t get_num_blocks() {
+  return get_num_blocks_x() * get_num_blocks_y() * get_num_blocks_z();
+}
+
+/// Returns the 'x' dimension of the current CUDA block's id.
 LIBC_INLINE uint32_t get_block_id_x() { return __nvvm_read_ptx_sreg_ctaid_x(); }
 
+/// Returns the 'y' dimension of the current CUDA block's id.
+LIBC_INLINE uint32_t get_block_id_y() { return __nvvm_read_ptx_sreg_ctaid_y(); }
+
+/// Returns the 'z' dimension of the current CUDA block's id.
+LIBC_INLINE uint32_t get_block_id_z() { return __nvvm_read_ptx_sreg_ctaid_z(); }
+
+/// Returns the absolute id of the CUDA block.
+LIBC_INLINE uint64_t get_block_id() {
+  return get_block_id_x() + get_num_blocks_x() * get_block_id_y() +
+         get_num_blocks_x() * get_num_blocks_y() * get_block_id_z();
+}
+
+/// Returns the number of CUDA threads in the 'x' dimension.
+LIBC_INLINE uint32_t get_num_threads_x() {
+  return __nvvm_read_ptx_sreg_ntid_x();
+}
+
+/// Returns the number of CUDA threads in the 'y' dimension.
+LIBC_INLINE uint32_t get_num_threads_y() {
+  return __nvvm_read_ptx_sreg_ntid_y();
+}
+
+/// Returns the number of CUDA threads in the 'z' dimension.
+LIBC_INLINE uint32_t get_num_threads_z() {
+  return __nvvm_read_ptx_sreg_ntid_z();
+}
+
+/// Returns the total number of threads in the block.
+LIBC_INLINE uint64_t get_num_threads() {
+  return get_num_threads_x() * get_num_threads_y() * get_num_threads_z();
+}
+
+/// Returns the 'x' dimension id of the thread in the current CUDA block.
+LIBC_INLINE uint32_t get_thread_id_x() { return __nvvm_read_ptx_sreg_tid_x(); }
+
+/// Returns the 'y' dimension id of the thread in the current CUDA block.
+LIBC_INLINE uint32_t get_thread_id_y() { return __nvvm_read_ptx_sreg_tid_y(); }
+
+/// Returns the 'z' dimension id of the thread in the current CUDA block.
+LIBC_INLINE uint32_t get_thread_id_z() { return __nvvm_read_ptx_sreg_tid_z(); }
+
+/// Returns the absolute id of the thread in the current CUDA block.
+LIBC_INLINE uint64_t get_thread_id() {
+  return get_thread_id_x() + get_num_threads_x() * get_thread_id_y() +
+         get_num_threads_x() * get_num_threads_y() * get_thread_id_z();
+}
+
+/// Returns the size of a CUDA warp.
+LIBC_INLINE uint32_t get_lane_size() { return LANE_SIZE; }
+
+/// Returns the id of the thread inside of a CUDA warp executing together.
+[[clang::convergent]] LIBC_INLINE uint32_t get_lane_id() {
+  return get_thread_id() & (get_lane_size() - 1);
+}
+
+/// Returns the bit-mask of active threads in the current warp.
+[[clang::convergent]] LIBC_INLINE uint64_t get_lane_mask() {
+  uint32_t mask;
+  asm volatile("activemask.b32 %0;" : "=r"(mask));
+  return mask;
+}
+
+/// Copies the value from the first active thread in the warp to the rest.
+[[clang::convergent]] LIBC_INLINE uint32_t broadcast_value(uint32_t x) {
+  // NOTE: This is not sufficient in all cases on Volta hardware or later. The
+  // lane mask returned here is not always the true lane mask used by the
+  // intrinsics in cases of incedental or enforced divergence by the user.
+  uint64_t lane_mask = get_lane_mask();
+  uint64_t id = __builtin_ffsl(lane_mask) - 1;
+#if __CUDA_ARCH__ >= 600
+  return __nvvm_shfl_sync_idx_i32(lane_mask, x, id, get_lane_size() - 1);
+#else
+  return __nvvm_shfl_idx_i32(x, id, get_lane_size() - 1);
+#endif
+}
+
+[[clang::convergent]] LIBC_INLINE uint64_t ballot(uint64_t lane_mask, bool x) {
+#if __CUDA_ARCH__ >= 600
+  return __nvvm_vote_ballot_sync(lane_mask, x);
+#else
+  return lane_mask & __nvvm_vote_ballot(x);
+#endif
+}
+/// Waits for all the threads in the block to converge and issues a fence.
+[[clang::convergent]] LIBC_INLINE void sync_threads() { __syncthreads(); }
+
+/// Waits for all threads in the warp to reconverge for independent scheduling.
+[[clang::convergent]] LIBC_INLINE void sync_lane(uint64_t mask) {
+  __nvvm_bar_warp_sync(mask);
+}
+
+} // namespace gpu
 } // namespace __llvm_libc
 
 #endif
diff --git a/libc/src/__support/OSUtil/gpu/io.cpp b/libc/src/__support/OSUtil/gpu/io.cpp
index 13ab96dacc284..62eedd5113f69 100644
--- a/libc/src/__support/OSUtil/gpu/io.cpp
+++ b/libc/src/__support/OSUtil/gpu/io.cpp
@@ -15,8 +15,8 @@
 namespace __llvm_libc {
 
 void write_to_stderr(cpp::string_view msg) {
-  rpc::Port port = rpc::client.open(rpc::PRINT_TO_STDERR);
-  port.send_n(msg.data(), msg.size() + 1);
+  rpc::Client::Port port = rpc::client.open(rpc::PRINT_TO_STDERR);
+  port.send_n(msg.data(), msg.size());
   port.close();
 }
 
diff --git a/libc/src/__support/OSUtil/gpu/quick_exit.cpp b/libc/src/__support/OSUtil/gpu/quick_exit.cpp
index b56af4a30d9d4..70f8bfd466472 100644
--- a/libc/src/__support/OSUtil/gpu/quick_exit.cpp
+++ b/libc/src/__support/OSUtil/gpu/quick_exit.cpp
@@ -17,7 +17,7 @@
 namespace __llvm_libc {
 
 void quick_exit(int status) {
-  rpc::Port port = rpc::client.open(rpc::EXIT);
+  rpc::Client::Port port = rpc::client.open(rpc::EXIT);
   port.send([&](rpc::Buffer *buffer) {
     reinterpret_cast<uint32_t *>(buffer->data)[0] = status;
   });
diff --git a/libc/src/__support/RPC/CMakeLists.txt b/libc/src/__support/RPC/CMakeLists.txt
index 6bcec0f6526d6..96bfd8e729549 100644
--- a/libc/src/__support/RPC/CMakeLists.txt
+++ b/libc/src/__support/RPC/CMakeLists.txt
@@ -10,6 +10,8 @@ add_header_library(
   DEPENDS
     libc.src.__support.common
     libc.src.__support.CPP.atomic
+    libc.src.__support.CPP.optional
+    libc.src.__support.CPP.functional
     libc.src.__support.GPU.utils
 )
 
@@ -20,5 +22,6 @@ add_object_library(
   HDRS
     rpc_client.h
   DEPENDS
+    libc.src.__support.GPU.utils
     .rpc
 )
diff --git a/libc/src/__support/RPC/rpc.h b/libc/src/__support/RPC/rpc.h
index be5856bfa7254..170f74830b141 100644
--- a/libc/src/__support/RPC/rpc.h
+++ b/libc/src/__support/RPC/rpc.h
@@ -20,6 +20,7 @@
 
 #include "rpc_util.h"
 #include "src/__support/CPP/atomic.h"
+#include "src/__support/CPP/functional.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/GPU/utils.h"
 #include "src/string/memory_utils/memcpy_implementations.h"
@@ -38,16 +39,47 @@ enum Opcode : uint16_t {
 };
 
 /// A fixed size channel used to communicate between the RPC client and server.
-struct alignas(64) Buffer {
-  uint8_t data[62];
-  uint16_t opcode;
+struct Buffer {
+  uint64_t data[8];
 };
 static_assert(sizeof(Buffer) == 64, "Buffer size mismatch");
 
+/// The information associated with a packet. This indicates which operations to
+/// perform and which threads are active in the slots.
+struct Header {
+  uint64_t mask;
+  uint16_t opcode;
+};
+
+/// The data payload for the associated packet. We provide enough space for each
+/// thread in the cooperating lane to have a buffer.
+struct Payload {
+#if defined(LIBC_TARGET_ARCH_IS_GPU)
+  Buffer slot[gpu::LANE_SIZE];
+#else
+  // Flexible array size allocated at runtime to the appropriate size.
+  Buffer slot[];
+#endif
+};
+
+/// A packet used to share data between the client and server across an entire
+/// lane. We use a lane as the minimum granularity for execution.
+struct alignas(64) Packet {
+  Header header;
+  Payload payload;
+};
+
 /// A common process used to synchronize communication between a client and a
 /// server. The process contains an inbox and an outbox used for signaling
 /// ownership of the shared buffer between both sides.
 ///
+/// No process writes to its inbox. Each toggles the bit in the outbox to pass
+/// ownership to the other process.
+/// When inbox == outbox, the current state machine owns the buffer.
+/// Initially the client is able to open any port as it will load 0 from both.
+/// The server inbox read is inverted, so it loads inbox==1, outbox==0 until
+/// the client has written to its outbox.
+///
 /// This process is designed to support mostly arbitrary combinations of 'send'
 /// and 'recv' operations on the shared buffer as long as these operations are
 /// mirrored by the other process. These operations exchange ownership of the
@@ -58,58 +90,137 @@ static_assert(sizeof(Buffer) == 64, "Buffer size mismatch");
 ///   - For every 'send' / 'recv' call on one side of the process there is a
 ///     mirrored 'recv' / 'send' call.
 ///
-/// The communication protocol is organized as a pair of two-state state
-/// machines. One state machine tracks outgoing sends and the other tracks
-/// incoming receives. For example, a 'send' operation uses its input 'Ack' bit
-/// and its output 'Data' bit. If these bits are equal the sender owns the
-/// buffer, otherwise the receiver owns the buffer and we wait. Similarly, a
-/// 'recv' operation uses its output 'Ack' bit and input 'Data' bit. If these
-/// bits are not equal the receiver owns the buffer, otherwise the sender owns
-/// the buffer.
-struct Process {
+template <bool InvertInbox> struct Process {
   LIBC_INLINE Process() = default;
   LIBC_INLINE Process(const Process &) = default;
   LIBC_INLINE Process &operator=(const Process &) = default;
   LIBC_INLINE ~Process() = default;
 
-  static constexpr uint32_t Data = 0b01;
-  static constexpr uint32_t Ack = 0b10;
-
+  uint32_t lane_size;
   cpp::Atomic<uint32_t> *lock;
   cpp::Atomic<uint32_t> *inbox;
   cpp::Atomic<uint32_t> *outbox;
-  Buffer *buffer;
+  Packet *buffer;
 
   /// Initialize the communication channels.
-  LIBC_INLINE void reset(void *lock, void *inbox, void *outbox, void *buffer) {
+  LIBC_INLINE void reset(uint32_t lane_size, void *lock, void *inbox,
+                         void *outbox, void *buffer) {
     *this = {
+        lane_size,
         reinterpret_cast<cpp::Atomic<uint32_t> *>(lock),
         reinterpret_cast<cpp::Atomic<uint32_t> *>(inbox),
         reinterpret_cast<cpp::Atomic<uint32_t> *>(outbox),
-        reinterpret_cast<Buffer *>(buffer),
+        reinterpret_cast<Packet *>(buffer),
     };
   }
 
-  /// Determines if this process owns the buffer for a send. We can send data if
-  /// the output data bit matches the input acknowledge bit.
-  LIBC_INLINE static bool can_send_data(uint32_t in, uint32_t out) {
-    return bool(in & Process::Ack) == bool(out & Process::Data);
+  /// Inverting the bits loaded from the inbox in exactly one of the pair of
+  /// processes means that each can use the same state transitions.
+  /// Whichever process has InvertInbox==false is the initial owner.
+  /// Inbox equal Outbox => current process owns the buffer
+  /// Inbox difer Outbox => current process does not own the buffer
+  /// At startup, memory is zero initialised and raw loads of either mailbox
+  /// would return zero. Thus both would succeed in opening a port and data
+  /// races result. If either inbox or outbox is inverted for one process, that
+  /// process interprets memory as Inbox!=Outbox and thus waits for the other.
+  /// It is simpler to invert reads from the inbox than writes to the outbox.
+  LIBC_INLINE uint32_t load_inbox(uint64_t index) {
+    uint32_t i = inbox[index].load(cpp::MemoryOrder::RELAXED);
+    return InvertInbox ? !i : i;
+  }
+
+  /// Determines if this process needs to wait for ownership of the buffer.
+  LIBC_INLINE static bool buffer_unavailable(uint32_t in, uint32_t out) {
+    return in != out;
+  }
+
+  /// Attempt to claim the lock at index. Return true on lock taken.
+  /// lane_mask is a bitmap of the threads in the warp that would hold the
+  /// single lock on success, e.g. the result of gpu::get_lane_mask()
+  /// The lock is held when the zeroth bit of the uint32_t at lock[index]
+  /// is set, and available when that bit is clear. Bits [1, 32) are zero.
+  /// Or with one is a no-op when the lock is already held.
+  [[clang::convergent]] LIBC_INLINE bool try_lock(uint64_t lane_mask,
+                                                  uint64_t index) {
+    // On amdgpu, test and set to lock[index] and a sync_lane would suffice
+    // On volta, need to handle differences between the threads running and
+    // the threads that were detected in the previous call to get_lane_mask()
+    //
+    // All threads in lane_mask try to claim the lock. At most one can succeed.
+    // There may be threads active which are not in lane mask which must not
+    // succeed in taking the lock, as otherwise it will leak. This is handled
+    // by making threads which are not in lane_mask or with 0, a no-op.
+    uint32_t id = gpu::get_lane_id();
+    bool id_in_lane_mask = lane_mask & (1ul << id);
+
+    // All threads in the warp call fetch_or. Possibly at the same time.
+    bool before =
+        lock[index].fetch_or(id_in_lane_mask, cpp::MemoryOrder::RELAXED);
+    uint64_t packed = gpu::ballot(lane_mask, before);
+
+    // If every bit set in lane_mask is also set in packed, every single thread
+    // in the warp failed to get the lock. Ballot returns unset for threads not
+    // in the lane mask.
+    //
+    // Cases, per thread:
+    // mask==0 -> unspecified before, discarded by ballot -> 0
+    // mask==1 and before==0 (success), set zero by ballot -> 0
+    // mask==1 and before==1 (failure), set one by ballot -> 1
+    //
+    // mask != packed implies at least one of the threads got the lock
+    // atomic semantics of fetch_or mean at most one of the threads for the lock
+    return lane_mask != packed;
   }
 
-  /// Determines if this process owns the buffer for a receive. We can send data
-  /// if the output acknowledge bit does not match the input data bit.
-  LIBC_INLINE static bool can_recv_data(uint32_t in, uint32_t out) {
-    return bool(in & Process::Data) != bool(out & Process::Ack);
+  /// Unlock the lock at index. We need a lane sync to keep this function
+  /// convergent, otherwise the compiler will sink the store and deadlock.
+  [[clang::convergent]] LIBC_INLINE void unlock(uint64_t lane_mask,
+                                                uint64_t index) {
+    // Wait for other threads in the warp to finish using the lock
+    gpu::sync_lane(lane_mask);
+
+    // Use exactly one thread to clear the bit at position 0 in lock[index]
+    // Must restrict to a single thread to avoid one thread dropping the lock,
+    // then an unrelated warp claiming the lock, then a second thread in this
+    // warp dropping the lock again.
+    uint32_t and_mask = ~(rpc::is_first_lane(lane_mask) ? 1 : 0);
+    lock[index].fetch_and(and_mask, cpp::MemoryOrder::RELAXED);
+    gpu::sync_lane(lane_mask);
+  }
+
+  /// Invokes a function accross every active buffer across the total lane size.
+  LIBC_INLINE void invoke_rpc(cpp::function<void(Buffer *)> fn,
+                              uint32_t index) {
+    if constexpr (is_process_gpu()) {
+      fn(&buffer[index].payload.slot[gpu::get_lane_id()]);
+    } else {
+      for (uint32_t i = 0; i < lane_size; i += gpu::get_lane_size())
+        if (buffer[index].header.mask & 1ul << i)
+          fn(&buffer[index].payload.slot[i]);
+    }
+  }
+
+  /// Alternate version that also provides the index of the current lane.
+  LIBC_INLINE void invoke_rpc(cpp::function<void(Buffer *, uint32_t)> fn,
+                              uint32_t index) {
+    if constexpr (is_process_gpu()) {
+      fn(&buffer[index].payload.slot[gpu::get_lane_id()], gpu::get_lane_id());
+    } else {
+      for (uint32_t i = 0; i < lane_size; i += gpu::get_lane_size())
+        if (buffer[index].header.mask & 1ul << i)
+          fn(&buffer[index].payload.slot[i], i);
+    }
   }
 };
 
 /// The port provides the interface to communicate between the multiple
 /// processes. A port is conceptually an index into the memory provided by the
 /// underlying process that is guarded by a lock bit.
-struct Port {
+template <bool T> struct Port {
   // TODO: This should be move-only.
-  LIBC_INLINE Port(Process &process, uint64_t index, uint32_t out)
-      : process(process), index(index), out(out) {}
+  LIBC_INLINE Port(Process<T> &process, uint64_t lane_mask, uint64_t index,
+                   uint32_t out)
+      : process(process), lane_mask(lane_mask), index(index), out(out) {}
   LIBC_INLINE Port(const Port &) = default;
   LIBC_INLINE Port &operator=(const Port &) = delete;
   LIBC_INLINE ~Port() = default;
@@ -123,78 +234,80 @@ struct Port {
   template <typename A> LIBC_INLINE void recv_n(A alloc);
 
   LIBC_INLINE uint16_t get_opcode() const {
-    return process.buffer[index].opcode;
+    return process.buffer[index].header.opcode;
   }
 
-  LIBC_INLINE void close() {
-    process.lock[index].store(0, cpp::MemoryOrder::RELAXED);
-  }
+  LIBC_INLINE void close() { process.unlock(lane_mask, index); }
 
 private:
-  Process &process;
+  Process<T> &process;
+  uint64_t lane_mask;
   uint64_t index;
   uint32_t out;
 };
 
 /// The RPC client used to make requests to the server.
-struct Client : public Process {
+struct Client : public Process<false> {
   LIBC_INLINE Client() = default;
   LIBC_INLINE Client(const Client &) = default;
   LIBC_INLINE Client &operator=(const Client &) = default;
   LIBC_INLINE ~Client() = default;
 
+  using Port = rpc::Port<false>;
   LIBC_INLINE cpp::optional<Port> try_open(uint16_t opcode);
   LIBC_INLINE Port open(uint16_t opcode);
 };
 
 /// The RPC server used to respond to the client.
-struct Server : public Process {
+struct Server : public Process<true> {
   LIBC_INLINE Server() = default;
   LIBC_INLINE Server(const Server &) = default;
   LIBC_INLINE Server &operator=(const Server &) = default;
   LIBC_INLINE ~Server() = default;
 
+  using Port = rpc::Port<true>;
   LIBC_INLINE cpp::optional<Port> try_open();
   LIBC_INLINE Port open();
 };
 
 /// Applies \p fill to the shared buffer and initiates a send operation.
-template <typename F> LIBC_INLINE void Port::send(F fill) {
-  uint32_t in = process.inbox[index].load(cpp::MemoryOrder::RELAXED);
+template <bool T> template <typename F> LIBC_INLINE void Port<T>::send(F fill) {
+  uint32_t in = process.load_inbox(index);
 
   // We need to wait until we own the buffer before sending.
-  while (!Process::can_send_data(in, out)) {
+  while (Process<T>::buffer_unavailable(in, out)) {
     sleep_briefly();
-    in = process.inbox[index].load(cpp::MemoryOrder::RELAXED);
+    in = process.load_inbox(index);
   }
 
   // Apply the \p fill function to initialize the buffer and release the memory.
-  fill(&process.buffer[index]);
-  out = out ^ Process::Data;
+  process.invoke_rpc(fill, index);
+  out = !out;
   atomic_thread_fence(cpp::MemoryOrder::RELEASE);
   process.outbox[index].store(out, cpp::MemoryOrder::RELAXED);
 }
 
 /// Applies \p use to the shared buffer and acknowledges the send.
-template <typename U> LIBC_INLINE void Port::recv(U use) {
-  uint32_t in = process.inbox[index].load(cpp::MemoryOrder::RELAXED);
+template <bool T> template <typename U> LIBC_INLINE void Port<T>::recv(U use) {
+  uint32_t in = process.load_inbox(index);
 
   // We need to wait until we own the buffer before receiving.
-  while (!Process::can_recv_data(in, out)) {
+  while (Process<T>::buffer_unavailable(in, out)) {
     sleep_briefly();
-    in = process.inbox[index].load(cpp::MemoryOrder::RELAXED);
+    in = process.load_inbox(index);
   }
   atomic_thread_fence(cpp::MemoryOrder::ACQUIRE);
 
   // Apply the \p use function to read the memory out of the buffer.
-  use(&process.buffer[index]);
-  out = out ^ Process::Ack;
+  process.invoke_rpc(use, index);
+  out = !out;
   process.outbox[index].store(out, cpp::MemoryOrder::RELAXED);
 }
 
 /// Combines a send and receive into a single function.
+template <bool T>
 template <typename F, typename U>
-LIBC_INLINE void Port::send_and_recv(F fill, U use) {
+LIBC_INLINE void Port<T>::send_and_recv(F fill, U use) {
   send(fill);
   recv(use);
 }
@@ -202,17 +315,23 @@ LIBC_INLINE void Port::send_and_recv(F fill, U use) {
 /// Combines a receive and send operation into a single function. The \p work
 /// function modifies the buffer in-place and the send is only used to initiate
 /// the copy back.
-template <typename W> LIBC_INLINE void Port::recv_and_send(W work) {
+template <bool T>
+template <typename W>
+LIBC_INLINE void Port<T>::recv_and_send(W work) {
   recv(work);
   send([](Buffer *) { /* no-op */ });
 }
 
 /// Sends an arbitrarily sized data buffer \p src across the shared channel in
 /// multiples of the packet length.
-LIBC_INLINE void Port::send_n(const void *src, uint64_t size) {
+template <bool T>
+LIBC_INLINE void Port<T>::send_n(const void *src, uint64_t size) {
   // TODO: We could send the first bytes in this call and potentially save an
   // extra send operation.
-  send([=](Buffer *buffer) { buffer->data[0] = size; });
+  // TODO: We may need a way for the CPU to send different strings per thread.
+  send([=](Buffer *buffer) {
+    reinterpret_cast<uint64_t *>(buffer->data)[0] = size;
+  });
   const uint8_t *ptr = reinterpret_cast<const uint8_t *>(src);
   for (uint64_t idx = 0; idx < size; idx += sizeof(Buffer::data)) {
     send([=](Buffer *buffer) {
@@ -221,21 +340,51 @@ LIBC_INLINE void Port::send_n(const void *src, uint64_t size) {
       inline_memcpy(buffer->data, ptr + idx, len);
     });
   }
+  gpu::sync_lane(process.buffer[index].header.mask);
 }
 
 /// Receives an arbitrarily sized data buffer across the shared channel in
 /// multiples of the packet length. The \p alloc function is called with the
 /// size of the data so that we can initialize the size of the \p dst buffer.
-template <typename A> LIBC_INLINE void Port::recv_n(A alloc) {
-  uint64_t size = 0;
-  recv([&](Buffer *buffer) { size = buffer->data[0]; });
-  uint8_t *dst = reinterpret_cast<uint8_t *>(alloc(size));
-  for (uint64_t idx = 0; idx < size; idx += sizeof(Buffer::data)) {
-    recv([=](Buffer *buffer) {
-      uint64_t len =
-          size - idx > sizeof(Buffer::data) ? sizeof(Buffer::data) : size - idx;
-      inline_memcpy(dst + idx, buffer->data, len);
+template <bool T>
+template <typename A>
+LIBC_INLINE void Port<T>::recv_n(A alloc) {
+  // The GPU handles thread private variables and masking implicitly through its
+  // execution model. If this is the CPU we need to manually handle the
+  // possibility that the sent data is of different length.
+  if constexpr (is_process_gpu()) {
+    uint64_t size = 0;
+    recv([&](Buffer *buffer) {
+      size = reinterpret_cast<uint64_t *>(buffer->data)[0];
+    });
+    uint8_t *dst = reinterpret_cast<uint8_t *>(alloc(size), gpu::get_lane_id());
+    for (uint64_t idx = 0; idx < size; idx += sizeof(Buffer::data)) {
+      recv([=](Buffer *buffer) {
+        uint64_t len = size - idx > sizeof(Buffer::data) ? sizeof(Buffer::data)
+                                                         : size - idx;
+        inline_memcpy(dst + idx, buffer->data, len);
+      });
+    }
+    return;
+  } else {
+    uint64_t size[MAX_LANE_SIZE];
+    uint8_t *dst[MAX_LANE_SIZE];
+    uint64_t max = 0;
+    recv([&](Buffer *buffer, uint32_t id) {
+      size[id] = reinterpret_cast<uint64_t *>(buffer->data)[0];
+      dst[id] = reinterpret_cast<uint8_t *>(alloc(size[id], id));
+      max = size[id] > max ? size[id] : max;
     });
+    for (uint64_t idx = 0; idx < max; idx += sizeof(Buffer::data)) {
+      recv([=](Buffer *buffer, uint32_t id) {
+        uint64_t len = size[id] - idx > sizeof(Buffer::data)
+                           ? sizeof(Buffer::data)
+                           : size[id] - idx;
+        if (idx < size[id])
+          inline_memcpy(dst[id] + idx, buffer->data, len);
+      });
+    }
+    return;
   }
 }
 
@@ -243,28 +392,41 @@ template <typename A> LIBC_INLINE void Port::recv_n(A alloc) {
 /// port if we find an index that is in a valid sending state. That is, there
 /// are send operations pending that haven't been serviced on this port. Each
 /// port instance uses an associated \p opcode to tell the server what to do.
-LIBC_INLINE cpp::optional<Port> Client::try_open(uint16_t opcode) {
+/// Opening a port is only valid if the `opcode` is the sam accross every
+/// participating thread.
+[[clang::convergent]] LIBC_INLINE cpp::optional<Client::Port>
+Client::try_open(uint16_t opcode) {
+  constexpr uint64_t index = 0;
+  const uint64_t lane_mask = gpu::get_lane_mask();
+
   // Attempt to acquire the lock on this index.
-  if (lock->fetch_or(1, cpp::MemoryOrder::RELAXED))
+  if (!try_lock(lane_mask, index))
     return cpp::nullopt;
 
-  uint32_t in = inbox->load(cpp::MemoryOrder::RELAXED);
-  uint32_t out = outbox->load(cpp::MemoryOrder::RELAXED);
+  // The mailbox state must be read with the lock held.
+  atomic_thread_fence(cpp::MemoryOrder::ACQUIRE);
+
+  uint32_t in = load_inbox(index);
+  uint32_t out = outbox[index].load(cpp::MemoryOrder::RELAXED);
 
   // Once we acquire the index we need to check if we are in a valid sending
   // state.
-  if (!can_send_data(in, out)) {
-    lock->store(0, cpp::MemoryOrder::RELAXED);
+  if (buffer_unavailable(in, out)) {
+    unlock(lane_mask, index);
     return cpp::nullopt;
   }
 
-  buffer->opcode = opcode;
-  return Port(*this, 0, out);
+  if (is_first_lane(lane_mask)) {
+    buffer[index].header.opcode = opcode;
+    buffer[index].header.mask = lane_mask;
+  }
+  gpu::sync_lane(lane_mask);
+  return Port(*this, lane_mask, index, out);
 }
 
-LIBC_INLINE Port Client::open(uint16_t opcode) {
+LIBC_INLINE Client::Port Client::open(uint16_t opcode) {
   for (;;) {
-    if (cpp::optional<Port> p = try_open(opcode))
+    if (cpp::optional<Client::Port> p = try_open(opcode))
       return p.value();
     sleep_briefly();
   }
@@ -272,33 +434,40 @@ LIBC_INLINE Port Client::open(uint16_t opcode) {
 
 /// Attempts to open a port to use as the server. The server can only open a
 /// port if it has a pending receive operation
-LIBC_INLINE cpp::optional<Port> Server::try_open() {
-  uint32_t in = inbox->load(cpp::MemoryOrder::RELAXED);
-  uint32_t out = outbox->load(cpp::MemoryOrder::RELAXED);
+[[clang::convergent]] LIBC_INLINE cpp::optional<Server::Port>
+Server::try_open() {
+  constexpr uint64_t index = 0;
+  const uint64_t lane_mask = gpu::get_lane_mask();
+
+  uint32_t in = load_inbox(index);
+  uint32_t out = outbox[index].load(cpp::MemoryOrder::RELAXED);
 
   // The server is passive, if there is no work pending don't bother
   // opening a port.
-  if (!can_recv_data(in, out))
+  if (buffer_unavailable(in, out))
     return cpp::nullopt;
 
   // Attempt to acquire the lock on this index.
-  if (lock->fetch_or(1, cpp::MemoryOrder::RELAXED))
+  if (!try_lock(lane_mask, index))
     return cpp::nullopt;
 
-  in = inbox->load(cpp::MemoryOrder::RELAXED);
-  out = outbox->load(cpp::MemoryOrder::RELAXED);
+  // The mailbox state must be read with the lock held.
+  atomic_thread_fence(cpp::MemoryOrder::ACQUIRE);
+
+  in = load_inbox(index);
+  out = outbox[index].load(cpp::MemoryOrder::RELAXED);
 
-  if (!can_recv_data(in, out)) {
-    lock->store(0, cpp::MemoryOrder::RELAXED);
+  if (buffer_unavailable(in, out)) {
+    unlock(lane_mask, index);
     return cpp::nullopt;
   }
 
-  return Port(*this, 0, out);
+  return Port(*this, lane_mask, index, out);
 }
 
-LIBC_INLINE Port Server::open() {
+LIBC_INLINE Server::Port Server::open() {
   for (;;) {
-    if (cpp::optional<Port> p = try_open())
+    if (cpp::optional<Server::Port> p = try_open())
       return p.value();
     sleep_briefly();
   }
diff --git a/libc/src/__support/RPC/rpc_util.h b/libc/src/__support/RPC/rpc_util.h
index 53a993292fb35..224723ef20c92 100644
--- a/libc/src/__support/RPC/rpc_util.h
+++ b/libc/src/__support/RPC/rpc_util.h
@@ -9,12 +9,16 @@
 #ifndef LLVM_LIBC_SRC_SUPPORT_RPC_RPC_UTILS_H
 #define LLVM_LIBC_SRC_SUPPORT_RPC_RPC_UTILS_H
 
+#include "src/__support/GPU/utils.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/properties/architectures.h"
 
 namespace __llvm_libc {
 namespace rpc {
 
+/// Maximum amount of data a single lane can use.
+constexpr uint64_t MAX_LANE_SIZE = 64;
+
 /// Suspend the thread briefly to assist the thread scheduler during busy loops.
 LIBC_INLINE void sleep_briefly() {
 #if defined(LIBC_TARGET_ARCH_IS_NVPTX) && __CUDA_ARCH__ >= 700
@@ -26,6 +30,25 @@ LIBC_INLINE void sleep_briefly() {
 #endif
 }
 
+/// Get the first active thread inside the lane.
+LIBC_INLINE uint64_t get_first_lane_id(uint64_t lane_mask) {
+  return __builtin_ffsl(lane_mask) - 1;
+}
+
+/// Conditional that is only true for a single thread in a lane.
+LIBC_INLINE bool is_first_lane(uint64_t lane_mask) {
+  return gpu::get_lane_id() == get_first_lane_id(lane_mask);
+}
+
+/// Conditional to indicate if this process is running on the GPU.
+LIBC_INLINE constexpr bool is_process_gpu() {
+#if defined(LIBC_TARGET_ARCH_IS_GPU)
+  return true;
+#else
+  return false;
+#endif
+}
+
 } // namespace rpc
 } // namespace __llvm_libc
 
diff --git a/libc/src/__support/c_string.h b/libc/src/__support/c_string.h
new file mode 100644
index 0000000000000..eaac4e7ad61d4
--- /dev/null
+++ b/libc/src/__support/c_string.h
@@ -0,0 +1,36 @@
+//===-- Implementation of a struct to hold a string in menory -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SUPPORT_C_STRING_H
+#define LLVM_LIBC_SRC_SUPPORT_C_STRING_H
+
+#include "src/__support/CPP/string.h"
+#include "src/__support/macros/attributes.h" // for LIBC_INLINE
+
+namespace __llvm_libc {
+
+// The CString class is a companion to the cpp::string class. Its use case is as
+// a return value for a function that in C would return a char* and a flag for
+// if that char* needs to be freed.
+class CString {
+  cpp::string str;
+
+public:
+  // These constructors can be implemented iff required.
+  CString() = delete;
+  CString(const CString &) = delete;
+  CString(CString &&) = delete;
+
+  LIBC_INLINE CString(cpp::string in_str) : str(in_str) {}
+
+  LIBC_INLINE operator const char *() const { return str.c_str(); }
+};
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SUPPORT_C_STRING_H
diff --git a/libc/src/__support/threads/gpu/CMakeLists.txt b/libc/src/__support/threads/gpu/CMakeLists.txt
new file mode 100644
index 0000000000000..ea89feb0c5c68
--- /dev/null
+++ b/libc/src/__support/threads/gpu/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_header_library(
+  mutex
+  HDRS
+    mutex.h
+)
diff --git a/libc/src/__support/threads/gpu/mutex.h b/libc/src/__support/threads/gpu/mutex.h
new file mode 100644
index 0000000000000..c4a75907eed7c
--- /dev/null
+++ b/libc/src/__support/threads/gpu/mutex.h
@@ -0,0 +1,31 @@
+//===--- Implementation of a GPU mutex class --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SUPPORT_THREAD_GPU_MUTEX_H
+#define LLVM_LIBC_SRC_SUPPORT_THREAD_GPU_MUTEX_H
+
+#include "src/__support/macros/attributes.h"
+#include "src/__support/threads/mutex_common.h"
+
+namespace __llvm_libc {
+
+/// Implementation of a simple passthrough mutex which guards nothing. A
+/// complete Mutex locks in general cannot be implemented on the GPU. We simply
+/// define the Mutex interface and require that only a single thread executes
+/// code requiring a mutex lock.
+struct Mutex {
+  LIBC_INLINE constexpr Mutex(bool, bool, bool) {}
+
+  LIBC_INLINE MutexError lock() { return MutexError::NONE; }
+  LIBC_INLINE MutexError unlock() { return MutexError::NONE; }
+  LIBC_INLINE MutexError reset() { return MutexError::NONE; }
+};
+
+} // namespace __llvm_libc
+
+#endif
diff --git a/libc/src/__support/threads/mutex.h b/libc/src/__support/threads/mutex.h
index 1015333b677a9..3c2291d8f41fd 100644
--- a/libc/src/__support/threads/mutex.h
+++ b/libc/src/__support/threads/mutex.h
@@ -9,6 +9,8 @@
 #ifndef LLVM_LIBC_SRC_SUPPORT_THREAD_MUTEX_H
 #define LLVM_LIBC_SRC_SUPPORT_THREAD_MUTEX_H
 
+#include "src/__support/macros/properties/architectures.h"
+
 // Platform independent code will include this header file which pulls
 // the platfrom specific specializations using platform macros.
 //
@@ -35,8 +37,10 @@
 // few global locks. So, to avoid static initialization order fiasco, we
 // want the constructors of the Mutex classes to be constexprs.
 
-#ifdef __unix__
+#if defined(__unix__)
 #include "linux/mutex.h"
+#elif defined(LIBC_TARGET_ARCH_IS_GPU)
+#include "gpu/mutex.h"
 #endif // __unix__
 
 namespace __llvm_libc {
diff --git a/libc/src/errno/libc_errno.cpp b/libc/src/errno/libc_errno.cpp
index b8c276cb853fb..a52e3afbbf1a9 100644
--- a/libc/src/errno/libc_errno.cpp
+++ b/libc/src/errno/libc_errno.cpp
@@ -6,8 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/macros/properties/architectures.h"
+
 namespace __llvm_libc {
 
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+struct ErrnoConsumer {
+  void operator=(int) {}
+};
+#endif
+
 extern "C" {
 #ifdef LIBC_COPT_PUBLIC_PACKAGING
 // TODO: Declare __llvmlibc_errno only under LIBC_COPT_PUBLIC_PACKAGING and
@@ -18,9 +26,17 @@ extern "C" {
 // macro defined in LLVM libc's public errno.h header file.
 // TODO: Use a macro to distinguish full build and overlay build which can be
 //       used to exclude __llvmlibc_errno under overlay build.
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+ErrnoConsumer __llvmlibc_errno;
+#else
 thread_local int __llvmlibc_errno;
+#endif // LIBC_TARGET_ARCH_IS_GPU
+#else
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+ErrnoConsumer __llvmlibc_internal_errno;
 #else
 thread_local int __llvmlibc_internal_errno;
+#endif // LIBC_TARGET_ARCH_IS_GPU
 #endif
 } // extern "C"
 
diff --git a/libc/src/errno/libc_errno.h b/libc/src/errno/libc_errno.h
index 28f8d0d13e736..585da1577940c 100644
--- a/libc/src/errno/libc_errno.h
+++ b/libc/src/errno/libc_errno.h
@@ -9,21 +9,42 @@
 #ifndef LLVM_LIBC_SRC_ERRNO_LLVMLIBC_ERRNO_H
 #define LLVM_LIBC_SRC_ERRNO_LLVMLIBC_ERRNO_H
 
+#include "src/__support/macros/properties/architectures.h"
+
 #include <errno.h>
 
+// If we are targeting the GPU we currently don't support 'errno'. We simply
+// consume it.
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+namespace __llvm_libc {
+struct ErrnoConsumer {
+  void operator=(int) {}
+};
+} // namespace __llvm_libc
+#endif
+
 // All of the libc runtime and test code should use the "libc_errno" macro. They
 // should not refer to the "errno" macro directly.
 #ifdef LIBC_COPT_PUBLIC_PACKAGING
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+extern "C" __llvm_libc::ErrnoConsumer __llvmlibc_errno;
+#define libc_errno __llvmlibc_errno
+#else
 // This macro will resolve to errno from the errno.h file included above. Under
 // full build, this will be LLVM libc's errno. In overlay build, it will be
 // system libc's errno.
 #define libc_errno errno
+#endif
 #else
 namespace __llvm_libc {
 
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+extern "C" ErrnoConsumer __llvmlibc_internal_errno;
+#else // LIBC_TARGET_ARCH_IS_GPU
 extern "C" {
 extern thread_local int __llvmlibc_internal_errno;
 } // extern "C"
+#endif
 
 // TODO: After all of libc/src and libc/test are switched over to use
 // libc_errno, this header file will be "shipped" via an add_entrypoint_object
diff --git a/libc/src/sched/linux/sched_get_priority_max.cpp b/libc/src/sched/linux/sched_get_priority_max.cpp
index 87d52f43c915c..859c9755c783d 100644
--- a/libc/src/sched/linux/sched_get_priority_max.cpp
+++ b/libc/src/sched/linux/sched_get_priority_max.cpp
@@ -17,7 +17,7 @@
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(int, sched_get_priority_max, (int policy)) {
-  long ret = __llvm_libc::syscall_impl(SYS_sched_get_priority_min, policy);
+  long ret = __llvm_libc::syscall_impl(SYS_sched_get_priority_max, policy);
   if (ret < 0) {
     libc_errno = -ret;
     return -1;
diff --git a/libc/src/stdio/fprintf.cpp b/libc/src/stdio/fprintf.cpp
index da8fabf5ab542..900e18f72167f 100644
--- a/libc/src/stdio/fprintf.cpp
+++ b/libc/src/stdio/fprintf.cpp
@@ -8,7 +8,6 @@
 
 #include "src/stdio/fprintf.h"
 
-#include "src/__support/File/file.h"
 #include "src/__support/arg_list.h"
 #include "src/stdio/printf_core/vfprintf_internal.h"
 
@@ -18,6 +17,7 @@
 namespace __llvm_libc {
 
 #ifndef LIBC_COPT_PRINTF_USE_SYSTEM_FILE
+#include "src/__support/File/file.h"
 using FileT = __llvm_libc::File;
 #else  // defined(LIBC_COPT_PRINTF_USE_SYSTEM_FILE)
 using FileT = ::FILE;
diff --git a/libc/src/stdio/printf_core/converter.cpp b/libc/src/stdio/printf_core/converter.cpp
index 950c246c247ed..ee9baf28ea782 100644
--- a/libc/src/stdio/printf_core/converter.cpp
+++ b/libc/src/stdio/printf_core/converter.cpp
@@ -28,6 +28,23 @@ int convert(Writer *writer, const FormatSection &to_conv) {
   if (!to_conv.has_conv)
     return writer->write(to_conv.raw_string);
 
+#ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT
+  // TODO(michaelrj): Undo this once decimal long double support is done.
+  if (to_conv.length_modifier == LengthModifier::L) {
+    switch (to_conv.conv_name) {
+    case 'f':
+    case 'F':
+    case 'e':
+    case 'E':
+    case 'g':
+    case 'G':
+      return convert_float_hex_exp(writer, to_conv);
+    default:
+      break;
+    }
+  }
+#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT
+
   switch (to_conv.conv_name) {
   case '%':
     return writer->write("%");
diff --git a/libc/src/stdio/printf_core/float_hex_converter.h b/libc/src/stdio/printf_core/float_hex_converter.h
index faf97b5b5dd9e..a26a3e796e4df 100644
--- a/libc/src/stdio/printf_core/float_hex_converter.h
+++ b/libc/src/stdio/printf_core/float_hex_converter.h
@@ -29,9 +29,9 @@ using MantissaInt = fputil::FPBits<long double>::UIntType;
 LIBC_INLINE int convert_float_hex_exp(Writer *writer,
                                       const FormatSection &to_conv) {
   // All of the letters will be defined relative to variable a, which will be
-  // the appropriate case based on the name of the conversion.
-  // Since the name of the conversion is also 'a', we can just use it directly.
-  const char a = to_conv.conv_name;
+  // the appropriate case based on the name of the conversion. This converts any
+  // conversion name into the letter 'a' with the appropriate case.
+  const char a = (to_conv.conv_name & 32) | 'A';
 
   bool is_negative;
   int exponent;
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index a9de969a55114..8067eebb532a4 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -313,9 +313,13 @@ endif()
 
 add_entrypoint_object(
   _Exit
-  ALIAS
+  SRCS
+    _Exit.cpp
+  HDRS
+    _Exit.h
   DEPENDS
-    .${LIBC_TARGET_OS}._Exit
+    libc.include.stdlib
+    libc.src.__support.OSUtil.osutil
 )
 
 add_entrypoint_object(
@@ -341,6 +345,7 @@ add_entrypoint_object(
   DEPENDS
     ._Exit
     .atexit
+    libc.src.__support.OSUtil.osutil
 )
 
 add_entrypoint_object(
diff --git a/libc/src/stdlib/linux/_Exit.cpp b/libc/src/stdlib/_Exit.cpp
similarity index 59%
rename from libc/src/stdlib/linux/_Exit.cpp
rename to libc/src/stdlib/_Exit.cpp
index 0300a08847cea..5aec134b1c71c 100644
--- a/libc/src/stdlib/linux/_Exit.cpp
+++ b/libc/src/stdlib/_Exit.cpp
@@ -1,4 +1,4 @@
-//===------------------- Linux Implementation of _Exit --------------------===//
+//===------------------- Implementation of _Exit --------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "include/sys/syscall.h"          // For syscall numbers.
-#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/OSUtil/quick_exit.h"
 #include "src/__support/common.h"
 
 #include "src/stdlib/_Exit.h"
@@ -15,10 +14,8 @@
 namespace __llvm_libc {
 
 LLVM_LIBC_FUNCTION(void, _Exit, (int status)) {
-  for (;;) {
-    __llvm_libc::syscall_impl(SYS_exit_group, status);
-    __llvm_libc::syscall_impl(SYS_exit, status);
-  }
+  quick_exit(status);
+  __builtin_unreachable();
 }
 
 } // namespace __llvm_libc
diff --git a/libc/src/stdlib/exit.cpp b/libc/src/stdlib/exit.cpp
index 92faea4da6ad0..71f5b1ecbf2c9 100644
--- a/libc/src/stdlib/exit.cpp
+++ b/libc/src/stdlib/exit.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdlib/exit.h"
+#include "src/__support/OSUtil/quick_exit.h"
 #include "src/__support/common.h"
-#include "src/stdlib/_Exit.h"
 
 namespace __llvm_libc {
 
@@ -18,7 +18,7 @@ void call_exit_callbacks();
 
 LLVM_LIBC_FUNCTION(void, exit, (int status)) {
   internal::call_exit_callbacks();
-  _Exit(status);
+  quick_exit(status);
 }
 
 } // namespace __llvm_libc
diff --git a/libc/src/stdlib/linux/CMakeLists.txt b/libc/src/stdlib/linux/CMakeLists.txt
index 360d3634f361c..1d3c00a5e0ddb 100644
--- a/libc/src/stdlib/linux/CMakeLists.txt
+++ b/libc/src/stdlib/linux/CMakeLists.txt
@@ -1,15 +1,3 @@
-add_entrypoint_object(
-  _Exit
-  SRCS
-    _Exit.cpp
-  HDRS
-    ../_Exit.h
-  DEPENDS
-    libc.include.sys_syscall
-    libc.include.stdlib
-    libc.src.__support.OSUtil.osutil
-)
-
 add_entrypoint_object(
   abort
   SRCS
diff --git a/libc/src/sys/CMakeLists.txt b/libc/src/sys/CMakeLists.txt
index 94f62123d2c40..bf869ddc6a23c 100644
--- a/libc/src/sys/CMakeLists.txt
+++ b/libc/src/sys/CMakeLists.txt
@@ -2,6 +2,7 @@ add_subdirectory(mman)
 add_subdirectory(random)
 add_subdirectory(resource)
 add_subdirectory(select)
+add_subdirectory(socket)
 add_subdirectory(sendfile)
 add_subdirectory(stat)
 add_subdirectory(utsname)
diff --git a/libc/src/sys/socket/CMakeLists.txt b/libc/src/sys/socket/CMakeLists.txt
new file mode 100644
index 0000000000000..7079d6e4466c6
--- /dev/null
+++ b/libc/src/sys/socket/CMakeLists.txt
@@ -0,0 +1,11 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+endif()
+
+add_entrypoint_object(
+  socket
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.socket
+)
+
diff --git a/libc/src/sys/socket/linux/CMakeLists.txt b/libc/src/sys/socket/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..41bcc9c9055f4
--- /dev/null
+++ b/libc/src/sys/socket/linux/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_entrypoint_object(
+  socket
+  SRCS
+    socket.cpp
+  HDRS
+    ../socket.h
+  DEPENDS
+    libc.include.sys_syscall
+    libc.include.sys_socket
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
diff --git a/libc/src/sys/socket/linux/socket.cpp b/libc/src/sys/socket/linux/socket.cpp
new file mode 100644
index 0000000000000..9ce20fd1685b4
--- /dev/null
+++ b/libc/src/sys/socket/linux/socket.cpp
@@ -0,0 +1,38 @@
+//===-- Linux implementation of socket ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/socket/socket.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+
+#include "src/errno/libc_errno.h"
+
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(int, socket, (int domain, int type, int protocol)) {
+#ifdef SYS_socket
+  long ret = __llvm_libc::syscall_impl(SYS_socket, domain, type, protocol);
+#elif defined(SYS_socketcall)
+  unsigned long sockcall_args[3] = {domain, type, protocol};
+  long ret =
+      __llvm_libc::syscall_impl(SYS_socketcall, SYS_SOCKET, sockcall_args);
+#else
+#error "socket and socketcall syscalls unavailable for this platform."
+#endif
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+  return ret;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/sys/socket/socket.h b/libc/src/sys/socket/socket.h
new file mode 100644
index 0000000000000..6a5a4c9e623f4
--- /dev/null
+++ b/libc/src/sys/socket/socket.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for socket ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_SOCKET_SENDFILE_H
+#define LLVM_LIBC_SRC_SYS_SOCKET_SENDFILE_H
+
+namespace __llvm_libc {
+
+int socket(int domain, int type, int protocol);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_SYS_SOCKET_SENDFILE_H
diff --git a/libc/src/time/asctime.h b/libc/src/time/asctime.h
index 9d75b40148afd..f053ad23eb4e3 100644
--- a/libc/src/time/asctime.h
+++ b/libc/src/time/asctime.h
@@ -18,5 +18,3 @@ char *asctime(const struct tm *timeptr);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_ASCTIME_H
-
-#include "include/time.h"
diff --git a/libc/src/time/asctime_r.h b/libc/src/time/asctime_r.h
index 8521e782a509d..21bba7ebaa3f5 100644
--- a/libc/src/time/asctime_r.h
+++ b/libc/src/time/asctime_r.h
@@ -18,5 +18,3 @@ char *asctime_r(const struct tm *timeptr, char *buffer);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_ASCTIME_R_H
-
-#include "include/time.h"
diff --git a/libc/src/time/clock_gettime.cpp b/libc/src/time/clock_gettime.cpp
index e8f9994ae696e..0bd14a1149363 100644
--- a/libc/src/time/clock_gettime.cpp
+++ b/libc/src/time/clock_gettime.cpp
@@ -17,6 +17,7 @@
 
 namespace __llvm_libc {
 
+// TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, clock_gettime,
                    (clockid_t clockid, struct timespec *tp)) {
   long ret_val =
diff --git a/libc/src/time/difftime.h b/libc/src/time/difftime.h
index feadef135933a..249dfed4fe819 100644
--- a/libc/src/time/difftime.h
+++ b/libc/src/time/difftime.h
@@ -18,5 +18,3 @@ double difftime(time_t end, time_t beginning);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_DIFFTIME_H
-
-#include "include/time.h"
diff --git a/libc/src/time/gettimeofday.cpp b/libc/src/time/gettimeofday.cpp
index 7c16c01d1dd91..914b5826e9cfb 100644
--- a/libc/src/time/gettimeofday.cpp
+++ b/libc/src/time/gettimeofday.cpp
@@ -16,6 +16,7 @@
 
 namespace __llvm_libc {
 
+// TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, gettimeofday,
                    (struct timeval * tv, [[maybe_unused]] void *unused)) {
   if (tv == nullptr)
diff --git a/libc/src/time/gettimeofday.h b/libc/src/time/gettimeofday.h
index 42182ee28e2a9..ee08efba8381b 100644
--- a/libc/src/time/gettimeofday.h
+++ b/libc/src/time/gettimeofday.h
@@ -18,5 +18,3 @@ int gettimeofday(struct timeval *tv, void *tz);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_GETTIMEOFDAY_H
-
-#include "include/time.h"
diff --git a/libc/src/time/gmtime.h b/libc/src/time/gmtime.h
index 8891a8c917ac5..950ba4f5673ab 100644
--- a/libc/src/time/gmtime.h
+++ b/libc/src/time/gmtime.h
@@ -18,5 +18,3 @@ struct tm *gmtime(const time_t *timer);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_GMTIME_H
-
-#include "include/time.h"
diff --git a/libc/src/time/gmtime_r.h b/libc/src/time/gmtime_r.h
index 8e9fc94b5ceed..619af603c98db 100644
--- a/libc/src/time/gmtime_r.h
+++ b/libc/src/time/gmtime_r.h
@@ -18,5 +18,3 @@ struct tm *gmtime_r(const time_t *timer, struct tm *result);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_GMTIME_R_H
-
-#include "include/time.h"
diff --git a/libc/src/time/linux/clock.cpp b/libc/src/time/linux/clock.cpp
index ff91e95488088..f72e4fcf37296 100644
--- a/libc/src/time/linux/clock.cpp
+++ b/libc/src/time/linux/clock.cpp
@@ -27,8 +27,7 @@ LLVM_LIBC_FUNCTION(clock_t, clock, ()) {
     return clock_t(-1);
   }
 
-  // The above syscall gets the CPU time in seconds plus nanoseconds. We should
-  // make sure that corresponding clocks can actually be represented by clock-t.
+  // The above syscall gets the CPU time in seconds plus nanoseconds.
   // The standard requires that we return clock_t(-1) if we cannot represent
   // clocks as a clock_t value.
   constexpr clock_t CLOCK_SECS_MAX =
diff --git a/libc/src/time/mktime.h b/libc/src/time/mktime.h
index 05dced8b6077c..efe9cf44565a4 100644
--- a/libc/src/time/mktime.h
+++ b/libc/src/time/mktime.h
@@ -18,5 +18,3 @@ time_t mktime(struct tm *t1);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_MKTIME_H
-
-#include "include/time.h"
diff --git a/libc/src/time/nanosleep.cpp b/libc/src/time/nanosleep.cpp
index de2503953f98b..f0458de2d1a85 100644
--- a/libc/src/time/nanosleep.cpp
+++ b/libc/src/time/nanosleep.cpp
@@ -15,6 +15,7 @@
 
 namespace __llvm_libc {
 
+// TODO(michaelrj): Move this into time/linux with the other syscalls.
 LLVM_LIBC_FUNCTION(int, nanosleep,
                    (const struct timespec *req, struct timespec *rem)) {
   int ret = __llvm_libc::syscall_impl(SYS_nanosleep, req, rem);
diff --git a/libc/src/time/nanosleep.h b/libc/src/time/nanosleep.h
index 0abe73948be4a..d8ecfb758f93c 100644
--- a/libc/src/time/nanosleep.h
+++ b/libc/src/time/nanosleep.h
@@ -18,5 +18,3 @@ int nanosleep(const struct timespec *req, struct timespec *rem);
 } // namespace __llvm_libc
 
 #endif // LLVM_LIBC_SRC_TIME_NANOSLEEP_H
-
-#include "include/time.h"
diff --git a/libc/src/time/time_utils.cpp b/libc/src/time/time_utils.cpp
index ca5f16cb7c9a8..b25b737d1f8d0 100644
--- a/libc/src/time/time_utils.cpp
+++ b/libc/src/time/time_utils.cpp
@@ -85,19 +85,19 @@ int64_t update_from_seconds(int64_t total_seconds, struct tm *tm) {
     numOfFourHundredYearCycles--;
   }
 
-  // The reminder number of years after computing number of
+  // The remaining number of years after computing the number of
   // "four hundred year cycles" will be 4 hundred year cycles or less in 400
   // years.
   int64_t numOfHundredYearCycles = computeRemainingYears(
       TimeConstants::DAYS_PER100_YEARS, 4, &remainingDays);
 
-  // The reminder number of years after computing number of
+  // The remaining number of years after computing the number of
   // "hundred year cycles" will be 25 four year cycles or less in 100 years.
   int64_t numOfFourYearCycles =
       computeRemainingYears(TimeConstants::DAYS_PER4_YEARS, 25, &remainingDays);
 
-  // The reminder number of years after computing number of "four year cycles"
-  // will be 4 one year cycles or less in 4 years.
+  // The remaining number of years after computing the number of
+  // "four year cycles" will be 4 one year cycles or less in 4 years.
   int64_t remainingYears = computeRemainingYears(
       TimeConstants::DAYS_PER_NON_LEAP_YEAR, 4, &remainingDays);
 
@@ -109,6 +109,8 @@ int64_t update_from_seconds(int64_t total_seconds, struct tm *tm) {
   int leapDay =
       !remainingYears && (numOfFourYearCycles || !numOfHundredYearCycles);
 
+  // We add 31 and 28 for the number of days in January and February, since our
+  // starting point was March 1st.
   int64_t yday = remainingDays + 31 + 28 + leapDay;
   if (yday >= TimeConstants::DAYS_PER_NON_LEAP_YEAR + leapDay)
     yday -= TimeConstants::DAYS_PER_NON_LEAP_YEAR + leapDay;
diff --git a/libc/src/time/time_utils.h b/libc/src/time/time_utils.h
index 362b4a9085a9f..8075bfe32027a 100644
--- a/libc/src/time/time_utils.h
+++ b/libc/src/time/time_utils.h
@@ -37,18 +37,21 @@ enum Month : int {
 
 struct TimeConstants {
   static constexpr int SECONDS_PER_MIN = 60;
-  static constexpr int SECONDS_PER_HOUR = 3600;
-  static constexpr int SECONDS_PER_DAY = 86400;
   static constexpr int MINUTES_PER_HOUR = 60;
+  static constexpr int HOURS_PER_DAY = 24;
   static constexpr int DAYS_PER_WEEK = 7;
   static constexpr int MONTHS_PER_YEAR = 12;
   static constexpr int DAYS_PER_NON_LEAP_YEAR = 365;
   static constexpr int DAYS_PER_LEAP_YEAR = 366;
+
+  static constexpr int SECONDS_PER_HOUR = SECONDS_PER_MIN * MINUTES_PER_HOUR;
+  static constexpr int SECONDS_PER_DAY = SECONDS_PER_HOUR * HOURS_PER_DAY;
+  static constexpr int NUMBER_OF_SECONDS_IN_LEAP_YEAR =
+      DAYS_PER_LEAP_YEAR * SECONDS_PER_DAY;
+
   static constexpr int TIME_YEAR_BASE = 1900;
   static constexpr int EPOCH_YEAR = 1970;
   static constexpr int EPOCH_WEEK_DAY = 4;
-  static constexpr int NUMBER_OF_SECONDS_IN_LEAP_YEAR =
-      (DAYS_PER_NON_LEAP_YEAR + 1) * SECONDS_PER_DAY;
 
   // For asctime the behavior is undefined if struct tm's tm_wday or tm_mon are
   // not within the normal ranges as defined in <time.h>, or if struct tm's
@@ -64,10 +67,10 @@ struct TimeConstants {
   static constexpr int WEEK_DAY_OF2000_MARCH_FIRST = 3;
 
   static constexpr int DAYS_PER400_YEARS =
-      (DAYS_PER_NON_LEAP_YEAR * 400 + (400 / 4) - 3);
+      (DAYS_PER_NON_LEAP_YEAR * 400) + (400 / 4) - 3;
   static constexpr int DAYS_PER100_YEARS =
-      (DAYS_PER_NON_LEAP_YEAR * 100 + (100 / 4) - 1);
-  static constexpr int DAYS_PER4_YEARS = (DAYS_PER_NON_LEAP_YEAR * 4 + 1);
+      (DAYS_PER_NON_LEAP_YEAR * 100) + (100 / 4) - 1;
+  static constexpr int DAYS_PER4_YEARS = (DAYS_PER_NON_LEAP_YEAR * 4) + 1;
 
   // The latest time that can be represented in this form is 03:14:07 UTC on
   // Tuesday, 19 January 2038 (corresponding to 2,147,483,647 seconds since the
@@ -82,6 +85,10 @@ struct TimeConstants {
 // "total_seconds" is the number of seconds since January 1st, 1970.
 extern int64_t update_from_seconds(int64_t total_seconds, struct tm *tm);
 
+// TODO(michaelrj): move these functions to use ErrorOr instead of setting
+// errno. They always accompany a specific return value so we only need the one
+// variable.
+
 // POSIX.1-2017 requires this.
 LIBC_INLINE time_t out_of_range() {
   libc_errno = EOVERFLOW;
@@ -114,6 +121,10 @@ LIBC_INLINE char *asctime(const struct tm *timeptr, char *buffer,
   static const char *months_name[TimeConstants::MONTHS_PER_YEAR] = {
       "Jan", "Feb", "Mar", "Apr", "May", "Jun",
       "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
+
+  // TODO(michaelr): look into removing this call to __builtin_snprintf that may
+  // be emitted as a call to snprintf. Alternatively, look into using our
+  // internal printf machinery.
   int written_size = __builtin_snprintf(
       buffer, bufferLength, "%.3s %.3s%3d %.2d:%.2d:%.2d %d\n",
       week_days_name[timeptr->tm_wday], months_name[timeptr->tm_mon],
diff --git a/libc/startup/gpu/amdgpu/CMakeLists.txt b/libc/startup/gpu/amdgpu/CMakeLists.txt
index 891d20993b080..0f1d4edf285df 100644
--- a/libc/startup/gpu/amdgpu/CMakeLists.txt
+++ b/libc/startup/gpu/amdgpu/CMakeLists.txt
@@ -4,6 +4,9 @@ add_startup_object(
     start.cpp
   DEPENDS
     libc.src.__support.RPC.rpc_client
+    libc.src.__support.GPU.utils
+    libc.src.stdlib.exit
+    libc.src.stdlib.atexit
   COMPILE_OPTIONS
     -ffreestanding # To avoid compiler warnings about calling the main function.
     -fno-builtin
diff --git a/libc/startup/gpu/amdgpu/start.cpp b/libc/startup/gpu/amdgpu/start.cpp
index 66f06b086a233..d1dfc7b8c11da 100644
--- a/libc/startup/gpu/amdgpu/start.cpp
+++ b/libc/startup/gpu/amdgpu/start.cpp
@@ -6,16 +6,65 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/GPU/utils.h"
 #include "src/__support/RPC/rpc_client.h"
-
-static __llvm_libc::cpp::Atomic<uint32_t> lock;
+#include "src/stdlib/atexit.h"
+#include "src/stdlib/exit.h"
 
 extern "C" int main(int argc, char **argv, char **envp);
 
+namespace __llvm_libc {
+
+static cpp::Atomic<uint32_t> lock = 0;
+
+extern "C" uintptr_t __init_array_start[];
+extern "C" uintptr_t __init_array_end[];
+extern "C" uintptr_t __fini_array_start[];
+extern "C" uintptr_t __fini_array_end[];
+
+using InitCallback = void(int, char **, char **);
+using FiniCallback = void(void);
+
+static void call_init_array_callbacks(int argc, char **argv, char **env) {
+  size_t init_array_size = __init_array_end - __init_array_start;
+  for (size_t i = 0; i < init_array_size; ++i)
+    reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
+}
+
+static void call_fini_array_callbacks() {
+  size_t fini_array_size = __fini_array_end - __fini_array_start;
+  for (size_t i = 0; i < fini_array_size; ++i)
+    reinterpret_cast<FiniCallback *>(__fini_array_start[i])();
+}
+
+} // namespace __llvm_libc
+
 extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
-_start(int argc, char **argv, char **envp, int *ret, void *in, void *out,
-       void *buffer) {
-  __llvm_libc::rpc::client.reset(&lock, in, out, buffer);
+_begin(int argc, char **argv, char **env, void *in, void *out, void *buffer) {
+  // We need to set up the RPC client first in case any of the constructors
+  // require it.
+  __llvm_libc::rpc::client.reset(__llvm_libc::gpu::get_lane_size(),
+                                 &__llvm_libc::lock, in, out, buffer);
+
+  // We want the fini array callbacks to be run after other atexit
+  // callbacks are run. So, we register them before running the init
+  // array callbacks as they can potentially register their own atexit
+  // callbacks.
+  __llvm_libc::atexit(&__llvm_libc::call_fini_array_callbacks);
+  __llvm_libc::call_init_array_callbacks(argc, argv, env);
+}
 
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+_start(int argc, char **argv, char **envp, int *ret) {
+  // Invoke the 'main' function with every active thread that the user launched
+  // the _start kernel with.
   __atomic_fetch_or(ret, main(argc, argv, envp), __ATOMIC_RELAXED);
 }
+
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+_end(int retval) {
+  // Only a single thread should call `exit` here, the rest should gracefully
+  // return from the kernel. This is so only one thread calls the destructors
+  // registred with 'atexit' above.
+  __llvm_libc::exit(retval);
+}
diff --git a/libc/startup/gpu/nvptx/CMakeLists.txt b/libc/startup/gpu/nvptx/CMakeLists.txt
index 49661691ecb57..49fa489c6129b 100644
--- a/libc/startup/gpu/nvptx/CMakeLists.txt
+++ b/libc/startup/gpu/nvptx/CMakeLists.txt
@@ -5,6 +5,9 @@ add_startup_object(
     start.cpp
   DEPENDS
     libc.src.__support.RPC.rpc_client
+    libc.src.__support.GPU.utils
+    libc.src.stdlib.exit
+    libc.src.stdlib.atexit
   COMPILE_OPTIONS
     -ffreestanding # To avoid compiler warnings about calling the main function.
     -fno-builtin
diff --git a/libc/startup/gpu/nvptx/start.cpp b/libc/startup/gpu/nvptx/start.cpp
index 9939c6e21330b..83453ae1e47a6 100644
--- a/libc/startup/gpu/nvptx/start.cpp
+++ b/libc/startup/gpu/nvptx/start.cpp
@@ -6,16 +6,68 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/GPU/utils.h"
 #include "src/__support/RPC/rpc_client.h"
-
-static __llvm_libc::cpp::Atomic<uint32_t> lock;
+#include "src/stdlib/atexit.h"
+#include "src/stdlib/exit.h"
 
 extern "C" int main(int argc, char **argv, char **envp);
 
-extern "C" [[gnu::visibility("protected")]] __attribute__((nvptx_kernel)) void
-_start(int argc, char **argv, char **envp, int *ret, void *in, void *out,
-       void *buffer) {
-  __llvm_libc::rpc::client.reset(&lock, in, out, buffer);
+namespace __llvm_libc {
+
+static cpp::Atomic<uint32_t> lock = 0;
+
+extern "C" {
+// Nvidia's 'nvlink' linker does not provide these symbols. We instead need
+// to manually create them and update the globals in the loader implememtation.
+uintptr_t *__init_array_start [[gnu::visibility("protected")]];
+uintptr_t *__init_array_end [[gnu::visibility("protected")]];
+uintptr_t *__fini_array_start [[gnu::visibility("protected")]];
+uintptr_t *__fini_array_end [[gnu::visibility("protected")]];
+}
+
+using InitCallback = void(int, char **, char **);
+using FiniCallback = void(void);
+
+static void call_init_array_callbacks(int argc, char **argv, char **env) {
+  size_t init_array_size = __init_array_end - __init_array_start;
+  for (size_t i = 0; i < init_array_size; ++i)
+    reinterpret_cast<InitCallback *>(__init_array_start[i])(argc, argv, env);
+}
 
+static void call_fini_array_callbacks() {
+  size_t fini_array_size = __fini_array_end - __fini_array_start;
+  for (size_t i = 0; i < fini_array_size; ++i)
+    reinterpret_cast<FiniCallback *>(__fini_array_start[i])();
+}
+
+} // namespace __llvm_libc
+
+extern "C" [[gnu::visibility("protected"), clang::nvptx_kernel]] void
+_begin(int argc, char **argv, char **env, void *in, void *out, void *buffer) {
+  // We need to set up the RPC client first in case any of the constructors
+  // require it.
+  __llvm_libc::rpc::client.reset(__llvm_libc::gpu::get_lane_size(),
+                                 &__llvm_libc::lock, in, out, buffer);
+
+  // We want the fini array callbacks to be run after other atexit
+  // callbacks are run. So, we register them before running the init
+  // array callbacks as they can potentially register their own atexit
+  // callbacks.
+  __llvm_libc::atexit(&__llvm_libc::call_fini_array_callbacks);
+  __llvm_libc::call_init_array_callbacks(argc, argv, env);
+}
+
+extern "C" [[gnu::visibility("protected"), clang::nvptx_kernel]] void
+_start(int argc, char **argv, char **envp, int *ret) {
+  // Invoke the 'main' function with every active thread that the user launched
+  // the _start kernel with.
   __atomic_fetch_or(ret, main(argc, argv, envp), __ATOMIC_RELAXED);
 }
+
+extern "C" [[gnu::visibility("protected"), clang::nvptx_kernel]] void
+_end(int retval) {
+  // To finis the execution we invoke all the callbacks registered via 'atexit'
+  // and then exit with the appropriate return value.
+  __llvm_libc::exit(retval);
+}
diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt
index 710f1138a6f2b..027028de71640 100644
--- a/libc/test/CMakeLists.txt
+++ b/libc/test/CMakeLists.txt
@@ -1,13 +1,12 @@
 add_custom_target(check-libc)
 add_custom_target(libc-unit-tests)
-add_dependencies(check-libc libc-unit-tests)
+add_custom_target(libc-hermetic-tests)
+add_dependencies(check-libc libc-unit-tests libc-hermetic-tests)
 
 add_custom_target(exhaustive-check-libc)
 add_custom_target(libc-long-running-tests)
 
-if(NOT LIBC_TARGET_ARCHITECTURE_IS_GPU)
-  add_subdirectory(UnitTest)
-endif()
+add_subdirectory(UnitTest)
 
 add_header_library(
   errno_setter_matcher
@@ -23,10 +22,8 @@ if(LIBC_TARGET_ARCHITECTURE_IS_GPU AND
   return()
 endif()
 
-if(NOT LIBC_TARGET_ARCHITECTURE_IS_GPU)
-  add_subdirectory(src)
-  add_subdirectory(utils)
-endif()
+add_subdirectory(src)
+add_subdirectory(utils)
 
 if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_BAREMETAL)
   add_subdirectory(IntegrationTest)
diff --git a/libc/test/IntegrationTest/test.cpp b/libc/test/IntegrationTest/test.cpp
index 4d2a7f08cc06e..e86e0a8d22c82 100644
--- a/libc/test/IntegrationTest/test.cpp
+++ b/libc/test/IntegrationTest/test.cpp
@@ -22,6 +22,7 @@ int memcmp(const void *lhs, const void *rhs, size_t count);
 void *memcpy(void *__restrict, const void *__restrict, size_t);
 void *memmove(void *dst, const void *src, size_t count);
 void *memset(void *ptr, int value, size_t count);
+int atexit(void (*func)(void));
 
 } // namespace __llvm_libc
 
@@ -44,6 +45,9 @@ void *memset(void *ptr, int value, size_t count) {
   return __llvm_libc::memset(ptr, value, count);
 }
 
+// This is needed if the test was compiled with '-fno-use-cxa-atexit'.
+int atexit(void (*func)(void)) { return __llvm_libc::atexit(func); }
+
 } // extern "C"
 
 // Integration tests cannot use the SCUDO standalone allocator as SCUDO pulls
diff --git a/libc/test/UnitTest/BazelFilePath.cpp b/libc/test/UnitTest/BazelFilePath.cpp
new file mode 100644
index 0000000000000..7bee46c4029e5
--- /dev/null
+++ b/libc/test/UnitTest/BazelFilePath.cpp
@@ -0,0 +1,25 @@
+//===-- Implementation of the file path generator for bazel ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LibcTest.h"
+
+#include <stdlib.h>
+
+#include "src/__support/CPP/string.h"
+#include "src/__support/c_string.h"
+
+namespace __llvm_libc::testing {
+
+CString libc_make_test_file_path_func(const char *file_name) {
+  // This is the path to the folder bazel wants the test outputs written to.
+  const char *UNDECLARED_OUTPUTS_PATH = getenv("TEST_UNDECLARED_OUTPUTS_DIR");
+
+  return cpp::string(UNDECLARED_OUTPUTS_PATH) + file_name;
+}
+
+} // namespace __llvm_libc::testing
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index f3d5a2b191943..7a54d502b50ce 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -1,38 +1,75 @@
-set(libc_unit_test_srcs
-  ExecuteFunction.h
+set(libc_test_srcs_common
   Test.h
   LibcTest.cpp
+  CmakeFilePath.cpp
   LibcTest.h
   TestLogger.cpp
   TestLogger.h
 )
 
+set(libc_death_test_srcs ExecuteFunction.h)
 if(${LIBC_TARGET_OS} STREQUAL "linux")
-  list(APPEND libc_unit_test_srcs ExecuteFunctionUnix.cpp)
+  list(APPEND libc_death_test_srcs 
+       LibcDeathTestExecutors.cpp ExecuteFunctionUnix.cpp)
+endif()
+
+# The Nvidia 'nvlink' linker does not support static libraries.
+if(LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
+  set(library_type OBJECT)
+else()
+  set(library_type STATIC)
 endif()
 
 add_library(
   LibcUnitTest
-  ${libc_unit_test_srcs}
+  ${library_type}
+  EXCLUDE_FROM_ALL
+  ${libc_test_srcs_common}
+  ${libc_death_test_srcs}
 )
 
-target_include_directories(LibcUnitTest PUBLIC ${LIBC_SOURCE_DIR})
-add_dependencies(
-  LibcUnitTest
-  libc.src.__support.CPP.string
-  libc.src.__support.CPP.string_view
-  libc.src.__support.CPP.type_traits
-  libc.src.__support.OSUtil.osutil
-  libc.src.__support.uint128)
+add_library(
+  LibcHermeticTest
+  ${library_type}
+  EXCLUDE_FROM_ALL
+  ${libc_test_srcs_common}
+  HermeticTestUtils.cpp
+)
 
 add_library(
   LibcUnitTestMain
+  ${library_type}
+  EXCLUDE_FROM_ALL
   LibcTestMain.cpp
 )
-
-target_include_directories(LibcUnitTestMain PUBLIC ${LIBC_SOURCE_DIR})
 add_dependencies(LibcUnitTestMain LibcUnitTest)
-target_link_libraries(LibcUnitTestMain PUBLIC LibcUnitTest)
+
+add_library(
+  LibcHermeticTestMain
+  ${library_type}
+  EXCLUDE_FROM_ALL
+  LibcTestMain.cpp
+)
+add_dependencies(LibcHermeticTestMain LibcHermeticTest)
+
+foreach(lib LibcUnitTest LibcUnitTestMain LibcHermeticTest LibcHermeticTestMain)
+  target_include_directories(${lib} PUBLIC ${LIBC_SOURCE_DIR})
+  target_compile_options(${lib} PRIVATE -fno-exceptions -fno-rtti)
+  add_dependencies(${lib}
+    libc.src.__support.c_string
+    libc.src.__support.CPP.string
+    libc.src.__support.CPP.string_view
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.uint128
+  )
+endforeach()
+
+foreach(lib LibcHermeticTest LibcHermeticTestMain)
+  target_include_directories(${lib} PRIVATE ${LIBC_BUILD_DIR}/include)
+  target_compile_options(${lib}
+      PRIVATE ${LIBC_HERMETIC_TEST_COMPILE_OPTIONS} -ffreestanding -nostdlib -nostdlib++)
+endforeach()
 
 add_header_library(
   string_utils
@@ -45,14 +82,10 @@ add_header_library(
 
 add_library(
   LibcFPTestHelpers
-  FPExceptMatcher.cpp
-  FPExceptMatcher.h
   FPMatcher.h
   RoundingModeUtils.cpp
   RoundingModeUtils.h
 )
-target_include_directories(LibcFPTestHelpers PUBLIC ${LIBC_SOURCE_DIR})
-target_link_libraries(LibcFPTestHelpers LibcUnitTest)
 add_dependencies(
   LibcFPTestHelpers
   LibcUnitTest
@@ -61,14 +94,24 @@ add_dependencies(
   libc.src.__support.FPUtil.fenv_impl
   libc.test.UnitTest.string_utils
 )
+add_library(
+  LibcFPExceptionHelpers
+  FPExceptMatcher.cpp
+  FPExceptMatcher.h
+)
+add_dependencies(
+  LibcFPExceptionHelpers
+  LibcUnitTest
+  libc.src.__support.FPUtil.fp_bits
+  libc.src.__support.FPUtil.fenv_impl
+)
 
 add_library(
   LibcMemoryHelpers
+  ${library_type}
   MemoryMatcher.h
   MemoryMatcher.cpp
 )
-target_include_directories(LibcMemoryHelpers PUBLIC ${LIBC_SOURCE_DIR})
-target_link_libraries(LibcMemoryHelpers LibcUnitTest)
 add_dependencies(
   LibcMemoryHelpers
   LibcUnitTest
@@ -80,8 +123,6 @@ add_library(
   PrintfMatcher.h
   PrintfMatcher.cpp
 )
-target_include_directories(LibcPrintfHelpers PUBLIC ${LIBC_SOURCE_DIR})
-target_link_libraries(LibcPrintfHelpers LibcUnitTest)
 add_dependencies(
   LibcPrintfHelpers
   LibcUnitTest
@@ -95,8 +136,6 @@ add_library(
   ScanfMatcher.h
   ScanfMatcher.cpp
 )
-target_include_directories(LibcScanfHelpers PUBLIC ${LIBC_SOURCE_DIR})
-target_link_libraries(LibcScanfHelpers LibcUnitTest)
 add_dependencies(
   LibcScanfHelpers
   LibcUnitTest
@@ -104,3 +143,19 @@ add_dependencies(
   libc.src.stdio.scanf_core.core_structs
   libc.test.UnitTest.string_utils
 )
+
+foreach(lib LibcFPTestHelpers LibcFPExceptionHelpers LibcMemoryHelpers 
+            LibcPrintfHelpers LibcScanfHelpers)
+  target_include_directories(${lib} PUBLIC ${LIBC_SOURCE_DIR})
+  target_compile_options(${lib} PRIVATE -fno-exceptions -fno-rtti)
+  target_link_libraries(${lib} LibcUnitTest)
+endforeach()
+
+# The GPU needs these flags applied to override the system triple.
+if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
+  foreach(lib LibcMemoryHelpers)
+    target_include_directories(${lib} PRIVATE ${LIBC_BUILD_DIR}/include)
+    target_compile_options(${lib}
+        PRIVATE ${LIBC_HERMETIC_TEST_COMPILE_OPTIONS} -ffreestanding -nostdlib -nostdlib++)
+  endforeach()
+endif()
diff --git a/libc/test/UnitTest/CmakeFilePath.cpp b/libc/test/UnitTest/CmakeFilePath.cpp
new file mode 100644
index 0000000000000..1e89167a00275
--- /dev/null
+++ b/libc/test/UnitTest/CmakeFilePath.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of the file path generator for cmake ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LibcTest.h"
+
+#include "src/__support/CPP/string.h"
+#include "src/__support/c_string.h"
+
+namespace __llvm_libc::testing {
+
+CString libc_make_test_file_path_func(const char *file_name) {
+  return cpp::string(file_name);
+}
+
+} // namespace __llvm_libc::testing
diff --git a/libc/test/UnitTest/HermeticTestUtils.cpp b/libc/test/UnitTest/HermeticTestUtils.cpp
new file mode 100644
index 0000000000000..c8279e54e15dd
--- /dev/null
+++ b/libc/test/UnitTest/HermeticTestUtils.cpp
@@ -0,0 +1,99 @@
+//===-- Implementation of libc death test executors -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace __llvm_libc {
+
+int bcmp(const void *lhs, const void *rhs, size_t count);
+void bzero(void *ptr, size_t count);
+int memcmp(const void *lhs, const void *rhs, size_t count);
+void *memcpy(void *__restrict, const void *__restrict, size_t);
+void *memmove(void *dst, const void *src, size_t count);
+void *memset(void *ptr, int value, size_t count);
+int atexit(void (*func)(void));
+
+} // namespace __llvm_libc
+
+namespace {
+
+// Integration tests cannot use the SCUDO standalone allocator as SCUDO pulls
+// various other parts of the libc. Since SCUDO development does not use
+// LLVM libc build rules, it is very hard to keep track or pull all that SCUDO
+// requires. Hence, as a work around for this problem, we use a simple allocator
+// which just hands out continuous blocks from a statically allocated chunk of
+// memory.
+static uint8_t memory[16384];
+static uint8_t *ptr = memory;
+
+} // anonymous namespace
+
+extern "C" {
+
+// Hermetic tests rely on the following memory functions. This is because the
+// compiler code generation can emit calls to them. We want to map the external
+// entrypoint to the internal implementation of the function used for testing.
+// This is done manually as not all targets support aliases.
+
+int bcmp(const void *lhs, const void *rhs, size_t count) {
+  return __llvm_libc::bcmp(lhs, rhs, count);
+}
+void bzero(void *ptr, size_t count) { __llvm_libc::bzero(ptr, count); }
+int memcmp(const void *lhs, const void *rhs, size_t count) {
+  return __llvm_libc::memcmp(lhs, rhs, count);
+}
+void *memcpy(void *__restrict dst, const void *__restrict src, size_t count) {
+  return __llvm_libc::memcpy(dst, src, count);
+}
+void *memmove(void *dst, const void *src, size_t count) {
+  return __llvm_libc::memmove(dst, src, count);
+}
+void *memset(void *ptr, int value, size_t count) {
+  return __llvm_libc::memset(ptr, value, count);
+}
+
+// This is needed if the test was compiled with '-fno-use-cxa-atexit'.
+int atexit(void (*func)(void)) { return __llvm_libc::atexit(func); }
+
+constexpr uint64_t ALIGNMENT = alignof(uintptr_t);
+
+void *malloc(size_t s) {
+  // Keep the bump pointer aligned on an eight byte boundary.
+  s = ((s + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT;
+  void *mem = ptr;
+  ptr += s;
+  return mem;
+}
+
+void free(void *) {}
+
+void *realloc(void *ptr, size_t s) {
+  free(ptr);
+  return malloc(s);
+}
+
+// The unit test framework uses pure virtual functions. Since hermetic tests
+// cannot depend C++ runtime libraries, implement dummy functions to support
+// the virtual function runtime.
+void __cxa_pure_virtual() {
+  // A pure virtual being called is an error so we just trap.
+  __builtin_trap();
+}
+
+// Hermetic tests are linked with -nostdlib. BFD linker expects
+// __dso_handle when -nostdlib is used.
+void *__dso_handle = nullptr;
+
+} // extern "C"
+
+void operator delete(void *) {
+  // The libc runtime should not use the global delete operator. Hence,
+  // we just trap here to catch any such accidental usages.
+  __builtin_trap();
+}
diff --git a/libc/test/UnitTest/LibcDeathTestExecutors.cpp b/libc/test/UnitTest/LibcDeathTestExecutors.cpp
new file mode 100644
index 0000000000000..c756a8dbe0302
--- /dev/null
+++ b/libc/test/UnitTest/LibcDeathTestExecutors.cpp
@@ -0,0 +1,102 @@
+//===-- Implementation of libc death test executors -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LibcTest.h"
+
+#include "test/UnitTest/ExecuteFunction.h"
+#include "test/UnitTest/TestLogger.h"
+
+#include <cassert>
+
+namespace __llvm_libc {
+namespace testing {
+
+bool Test::testProcessKilled(testutils::FunctionCaller *Func, int Signal,
+                             const char *LHSStr, const char *RHSStr,
+                             const char *File, unsigned long Line) {
+  testutils::ProcessStatus Result = testutils::invoke_in_subprocess(Func, 500);
+
+  if (const char *error = Result.get_error()) {
+    Ctx->markFail();
+    tlog << File << ":" << Line << ": FAILURE\n" << error << '\n';
+    return false;
+  }
+
+  if (Result.timed_out()) {
+    Ctx->markFail();
+    tlog << File << ":" << Line << ": FAILURE\n"
+         << "Process timed out after " << 500 << " milliseconds.\n";
+    return false;
+  }
+
+  if (Result.exited_normally()) {
+    Ctx->markFail();
+    tlog << File << ":" << Line << ": FAILURE\n"
+         << "Expected " << LHSStr
+         << " to be killed by a signal\nBut it exited normally!\n";
+    return false;
+  }
+
+  int KilledBy = Result.get_fatal_signal();
+  assert(KilledBy != 0 && "Not killed by any signal");
+  if (Signal == -1 || KilledBy == Signal)
+    return true;
+
+  using testutils::signal_as_string;
+  Ctx->markFail();
+  tlog << File << ":" << Line << ": FAILURE\n"
+       << "              Expected: " << LHSStr << '\n'
+       << "To be killed by signal: " << Signal << '\n'
+       << "              Which is: " << signal_as_string(Signal) << '\n'
+       << "  But it was killed by: " << KilledBy << '\n'
+       << "              Which is: " << signal_as_string(KilledBy) << '\n';
+  return false;
+}
+
+bool Test::testProcessExits(testutils::FunctionCaller *Func, int ExitCode,
+                            const char *LHSStr, const char *RHSStr,
+                            const char *File, unsigned long Line) {
+  testutils::ProcessStatus Result = testutils::invoke_in_subprocess(Func, 500);
+
+  if (const char *error = Result.get_error()) {
+    Ctx->markFail();
+    tlog << File << ":" << Line << ": FAILURE\n" << error << '\n';
+    return false;
+  }
+
+  if (Result.timed_out()) {
+    Ctx->markFail();
+    tlog << File << ":" << Line << ": FAILURE\n"
+         << "Process timed out after " << 500 << " milliseconds.\n";
+    return false;
+  }
+
+  if (!Result.exited_normally()) {
+    Ctx->markFail();
+    tlog << File << ":" << Line << ": FAILURE\n"
+         << "Expected " << LHSStr << '\n'
+         << "to exit with exit code " << ExitCode << '\n'
+         << "But it exited abnormally!\n";
+    return false;
+  }
+
+  int ActualExit = Result.get_exit_code();
+  if (ActualExit == ExitCode)
+    return true;
+
+  Ctx->markFail();
+  tlog << File << ":" << Line << ": FAILURE\n"
+       << "Expected exit code of: " << LHSStr << '\n'
+       << "             Which is: " << ActualExit << '\n'
+       << "       To be equal to: " << RHSStr << '\n'
+       << "             Which is: " << ExitCode << '\n';
+  return false;
+}
+
+} // namespace testing
+} // namespace __llvm_libc
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index b3a7fc18dfda2..dabfe2b6bd8a0 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -11,28 +11,11 @@
 #include "src/__support/CPP/string.h"
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/UInt128.h"
-#include "test/UnitTest/ExecuteFunction.h"
 #include "test/UnitTest/TestLogger.h"
-#include <cassert>
 
 namespace __llvm_libc {
 namespace testing {
 
-// This need not be a class as all it has is a single read-write state variable.
-// But, we make it class as then its implementation can be hidden from the
-// header file.
-class RunContext {
-public:
-  enum RunResult { Result_Pass = 1, Result_Fail = 2 };
-
-  RunResult status() const { return Status; }
-
-  void markFail() { Status = Result_Fail; }
-
-private:
-  RunResult Status = Result_Pass;
-};
-
 namespace internal {
 
 // When the value is UInt128 or __uint128_t, show its hexadecimal digits.
@@ -149,6 +132,12 @@ bool test(RunContext *Ctx, TestCondition Cond, ValType LHS, ValType RHS,
 Test *Test::Start = nullptr;
 Test *Test::End = nullptr;
 
+int argc = 0;
+char **argv = nullptr;
+char **envp = nullptr;
+
+using internal::RunContext;
+
 void Test::addTest(Test *T) {
   if (End == nullptr) {
     Start = T;
@@ -326,90 +315,5 @@ bool Test::testMatch(bool MatchResult, MatcherBase &Matcher, const char *LHSStr,
   return false;
 }
 
-#ifdef ENABLE_SUBPROCESS_TESTS
-
-bool Test::testProcessKilled(testutils::FunctionCaller *Func, int Signal,
-                             const char *LHSStr, const char *RHSStr,
-                             const char *File, unsigned long Line) {
-  testutils::ProcessStatus Result = testutils::invoke_in_subprocess(Func, 500);
-
-  if (const char *error = Result.get_error()) {
-    Ctx->markFail();
-    tlog << File << ":" << Line << ": FAILURE\n" << error << '\n';
-    return false;
-  }
-
-  if (Result.timed_out()) {
-    Ctx->markFail();
-    tlog << File << ":" << Line << ": FAILURE\n"
-         << "Process timed out after " << 500 << " milliseconds.\n";
-    return false;
-  }
-
-  if (Result.exited_normally()) {
-    Ctx->markFail();
-    tlog << File << ":" << Line << ": FAILURE\n"
-         << "Expected " << LHSStr
-         << " to be killed by a signal\nBut it exited normally!\n";
-    return false;
-  }
-
-  int KilledBy = Result.get_fatal_signal();
-  assert(KilledBy != 0 && "Not killed by any signal");
-  if (Signal == -1 || KilledBy == Signal)
-    return true;
-
-  using testutils::signal_as_string;
-  Ctx->markFail();
-  tlog << File << ":" << Line << ": FAILURE\n"
-       << "              Expected: " << LHSStr << '\n'
-       << "To be killed by signal: " << Signal << '\n'
-       << "              Which is: " << signal_as_string(Signal) << '\n'
-       << "  But it was killed by: " << KilledBy << '\n'
-       << "              Which is: " << signal_as_string(KilledBy) << '\n';
-  return false;
-}
-
-bool Test::testProcessExits(testutils::FunctionCaller *Func, int ExitCode,
-                            const char *LHSStr, const char *RHSStr,
-                            const char *File, unsigned long Line) {
-  testutils::ProcessStatus Result = testutils::invoke_in_subprocess(Func, 500);
-
-  if (const char *error = Result.get_error()) {
-    Ctx->markFail();
-    tlog << File << ":" << Line << ": FAILURE\n" << error << '\n';
-    return false;
-  }
-
-  if (Result.timed_out()) {
-    Ctx->markFail();
-    tlog << File << ":" << Line << ": FAILURE\n"
-         << "Process timed out after " << 500 << " milliseconds.\n";
-    return false;
-  }
-
-  if (!Result.exited_normally()) {
-    Ctx->markFail();
-    tlog << File << ":" << Line << ": FAILURE\n"
-         << "Expected " << LHSStr << '\n'
-         << "to exit with exit code " << ExitCode << '\n'
-         << "But it exited abnormally!\n";
-    return false;
-  }
-
-  int ActualExit = Result.get_exit_code();
-  if (ActualExit == ExitCode)
-    return true;
-
-  Ctx->markFail();
-  tlog << File << ":" << Line << ": FAILURE\n"
-       << "Expected exit code of: " << LHSStr << '\n'
-       << "             Which is: " << ActualExit << '\n'
-       << "       To be equal to: " << RHSStr << '\n'
-       << "             Which is: " << ExitCode << '\n';
-  return false;
-}
-
-#endif // ENABLE_SUBPROCESS_TESTS
 } // namespace testing
 } // namespace __llvm_libc
diff --git a/libc/test/UnitTest/LibcTest.h b/libc/test/UnitTest/LibcTest.h
index a1343e2520f78..c802b230700b3 100644
--- a/libc/test/UnitTest/LibcTest.h
+++ b/libc/test/UnitTest/LibcTest.h
@@ -9,22 +9,32 @@
 #ifndef LLVM_LIBC_UTILS_UNITTEST_LIBCTEST_H
 #define LLVM_LIBC_UTILS_UNITTEST_LIBCTEST_H
 
-// This file can only include headers from src/__support/CPP/ or
-// utils/testutils. No other headers should be included.
+// This is defined as a simple macro in test.h so that it exists for platforms
+// that don't use our test infrastructure. It's defined as a proper function
+// below.
+#ifdef libc_make_test_file_path
+#undef libc_make_test_file_path
+#endif // libc_make_test_file_path
+
+// This is defined as a macro here to avoid namespace issues.
+#define libc_make_test_file_path(file_name)                                    \
+  (__llvm_libc::testing::libc_make_test_file_path_func(file_name))
+
+// This file can only include headers from src/__support/ or test/UnitTest. No
+// other headers should be included.
 
 #include "PlatformDefs.h"
 
 #include "src/__support/CPP/string.h"
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/CPP/type_traits.h"
+#include "src/__support/c_string.h"
 #include "test/UnitTest/ExecuteFunction.h"
 #include "test/UnitTest/TestLogger.h"
 
 namespace __llvm_libc {
 namespace testing {
 
-class RunContext;
-
 // Only the following conditions are supported. Notice that we do not have
 // a TRUE or FALSE condition. That is because, C library funtions do not
 // return boolean values, but use integral return values to indicate true or
@@ -42,6 +52,18 @@ enum TestCondition {
 
 namespace internal {
 
+class RunContext {
+public:
+  enum RunResult { Result_Pass = 1, Result_Fail = 2 };
+
+  RunResult status() const { return Status; }
+
+  void markFail() { Status = Result_Fail; }
+
+private:
+  RunResult Status = Result_Pass;
+};
+
 template <typename ValType>
 bool test(RunContext *Ctx, TestCondition Cond, ValType LHS, ValType RHS,
           const char *LHSStr, const char *RHSStr, const char *File,
@@ -65,9 +87,9 @@ template <typename T> struct Matcher : public MatcherBase {
 class Test {
 private:
   Test *Next = nullptr;
-  RunContext *Ctx = nullptr;
+  internal::RunContext *Ctx = nullptr;
 
-  void setContext(RunContext *C) { Ctx = C; }
+  void setContext(internal::RunContext *C) { Ctx = C; }
 
 public:
   virtual ~Test() {}
@@ -161,6 +183,10 @@ class Test {
   static Test *End;
 };
 
+extern int argc;
+extern char **argv;
+extern char **envp;
+
 namespace internal {
 
 constexpr bool same_prefix(char const *lhs, char const *rhs, int const len) {
@@ -245,6 +271,8 @@ template <typename... Types> struct TypeList {
 // Make TypeList visible in __llvm_libc::testing.
 template <typename... Types> using TypeList = internal::TypeList<Types...>;
 
+CString libc_make_test_file_path_func(const char *file_name);
+
 } // namespace testing
 } // namespace __llvm_libc
 
diff --git a/libc/test/UnitTest/LibcTestMain.cpp b/libc/test/UnitTest/LibcTestMain.cpp
index 46e19dd027365..756d1196bc365 100644
--- a/libc/test/UnitTest/LibcTestMain.cpp
+++ b/libc/test/UnitTest/LibcTestMain.cpp
@@ -12,7 +12,11 @@ static const char *getTestFilter(int argc, char *argv[]) {
   return argc > 1 ? argv[1] : nullptr;
 }
 
-int main(int argc, char *argv[]) {
+extern "C" int main(int argc, char **argv, char **envp) {
+  __llvm_libc::testing::argc = argc;
+  __llvm_libc::testing::argv = argv;
+  __llvm_libc::testing::envp = envp;
+
   const char *TestFilter = getTestFilter(argc, argv);
   return __llvm_libc::testing::Test::runTests(TestFilter);
 }
diff --git a/libc/test/UnitTest/Test.h b/libc/test/UnitTest/Test.h
index 19c4149b3ac00..7fd6f025b152f 100644
--- a/libc/test/UnitTest/Test.h
+++ b/libc/test/UnitTest/Test.h
@@ -9,6 +9,13 @@
 #ifndef LLVM_LIBC_UTILS_UNITTEST_TEST_H
 #define LLVM_LIBC_UTILS_UNITTEST_TEST_H
 
+// This macro takes a file name and returns a value implicitly castable to
+// a const char*. That const char* is the path to a file with the provided name
+// in a directory where the test is allowed to write. By default it writes
+// directly to the filename provided, but implementations are allowed to
+// redefine it as necessary.
+#define libc_make_test_file_path(file_name) (file_name)
+
 #ifdef LIBC_COPT_TEST_USE_FUCHSIA
 #include "FuchsiaTest.h"
 #elif defined(LIBC_COPT_TEST_USE_PIGWEED)
diff --git a/libc/test/integration/startup/gpu/CMakeLists.txt b/libc/test/integration/startup/gpu/CMakeLists.txt
index bc44fe14bb313..d2028cc941f05 100644
--- a/libc/test/integration/startup/gpu/CMakeLists.txt
+++ b/libc/test/integration/startup/gpu/CMakeLists.txt
@@ -22,6 +22,17 @@ add_integration_test(
     libc.src.__support.RPC.rpc_client
     libc.src.__support.GPU.utils
   LOADER_ARGS
-    --blocks 16
-    --threads 1
+    --blocks-x 2
+    --blocks-y 2
+    --blocks-z 2
+    --threads-x 4
+    --threads-y 4
+    --threads-z 4
+)
+
+add_integration_test(
+  init_fini_array_test
+  SUITE libc-startup-tests
+  SRCS
+    init_fini_array_test.cpp
 )
diff --git a/libc/test/integration/startup/gpu/init_fini_array_test.cpp b/libc/test/integration/startup/gpu/init_fini_array_test.cpp
new file mode 100644
index 0000000000000..1e61711f0fc4d
--- /dev/null
+++ b/libc/test/integration/startup/gpu/init_fini_array_test.cpp
@@ -0,0 +1,60 @@
+//===-- Loader test to test init and fini array iteration -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "test/IntegrationTest/test.h"
+
+#include <stddef.h>
+
+int global_destroyed = false;
+
+class A {
+private:
+  int val[1024];
+
+public:
+  A(int i, int a) {
+    for (int k = 0; k < 1024; ++k)
+      val[k] = 0;
+    val[i] = a;
+  }
+
+  ~A() { global_destroyed = true; }
+
+  int get(int i) const { return val[i]; }
+};
+
+int GLOBAL_INDEX = 512;
+int INITVAL_INITIALIZER = 0x600D;
+int BEFORE_INITIALIZER = 0xFEED;
+
+A global(GLOBAL_INDEX, INITVAL_INITIALIZER);
+
+int initval = 0;
+int before = 0;
+
+__attribute__((constructor(101))) void run_before() {
+  before = BEFORE_INITIALIZER;
+}
+
+__attribute__((constructor(65535))) void run_after() {
+  ASSERT_EQ(before, BEFORE_INITIALIZER);
+}
+
+__attribute__((constructor)) void set_initval() {
+  initval = INITVAL_INITIALIZER;
+}
+__attribute__((destructor(1))) void reset_initval() {
+  ASSERT_TRUE(global_destroyed);
+  initval = 0;
+}
+
+TEST_MAIN(int argc, char **argv, char **env) {
+  ASSERT_EQ(global.get(GLOBAL_INDEX), INITVAL_INITIALIZER);
+  ASSERT_EQ(initval, INITVAL_INITIALIZER);
+  return 0;
+}
diff --git a/libc/test/integration/startup/gpu/rpc_test.cpp b/libc/test/integration/startup/gpu/rpc_test.cpp
index 0d3f13718d594..9dc2214fde411 100644
--- a/libc/test/integration/startup/gpu/rpc_test.cpp
+++ b/libc/test/integration/startup/gpu/rpc_test.cpp
@@ -13,10 +13,11 @@
 using namespace __llvm_libc;
 
 static void test_add_simple() {
-  uint32_t num_additions = 1000 + 10 * get_block_id_x();
+  uint32_t num_additions =
+      10 + 10 * gpu::get_thread_id() + 10 * gpu::get_block_id();
   uint64_t cnt = 0;
   for (uint32_t i = 0; i < num_additions; ++i) {
-    rpc::Port port = rpc::client.open(rpc::TEST_INCREMENT);
+    rpc::Client::Port port = rpc::client.open(rpc::TEST_INCREMENT);
     port.send_and_recv(
         [=](rpc::Buffer *buffer) {
           reinterpret_cast<uint64_t *>(buffer->data)[0] = cnt;
@@ -29,8 +30,20 @@ static void test_add_simple() {
   ASSERT_TRUE(cnt == num_additions && "Incorrect sum");
 }
 
+// Test to ensure that the RPC mechanism doesn't hang on divergence.
+static void test_noop(uint8_t data) {
+  rpc::Client::Port port = rpc::client.open(rpc::NOOP);
+  port.send([=](rpc::Buffer *buffer) { buffer->data[0] = data; });
+  port.close();
+}
+
 TEST_MAIN(int argc, char **argv, char **envp) {
   test_add_simple();
 
+  if (gpu::get_thread_id() % 2)
+    test_noop(1);
+  else
+    test_noop(2);
+
   return 0;
 }
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index f3f8ca88190e0..9e6c02c1f9f4b 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -1,7 +1,7 @@
 function(add_fp_unittest name)
   cmake_parse_arguments(
     "MATH_UNITTEST"
-    "NEED_MPFR" # Optional arguments
+    "NEED_MPFR;HERMETIC_TEST_ONLY" # Optional arguments
     "" # Single value arguments
     "LINK_LIBRARIES" # Multi-value arguments
     ${ARGN}
@@ -9,18 +9,26 @@ function(add_fp_unittest name)
 
   if(MATH_UNITTEST_NEED_MPFR)
     if(NOT LIBC_TESTS_CAN_USE_MPFR)
-      message("WARNING: Math test ${name} will be skipped as MPFR library is not available.")
+      message(VERBOSE "Math test ${name} will be skipped as MPFR library is not available.")
       return()
     endif()
   endif()
 
+  if(MATH_UNITTEST_HERMETIC_TEST_ONLY)
+    set(test_type HERMETIC_TEST_ONLY)
+  endif()
   if(MATH_UNITTEST_NEED_MPFR)
+    if(MATH_UNITTEST_HERMETIC_TEST_ONLY)
+      message(FATAL "Hermetic math test cannot require MPFR.")
+    endif()
+    set(test_type UNIT_TEST_ONLY)
     list(APPEND MATH_UNITTEST_LINK_LIBRARIES libcMPFRWrapper libc_math_test_utils -lmpfr -lgmp)
   endif()
   list(APPEND MATH_UNITTEST_LINK_LIBRARIES LibcFPTestHelpers)
 
   add_libc_unittest(
     ${name}
+    ${test_type}
     LINK_LIBRARIES "${MATH_UNITTEST_LINK_LIBRARIES}"
     "${MATH_UNITTEST_UNPARSED_ARGUMENTS}"
   )
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index a8c5517238882..7390ed2471f72 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -1,40 +1,43 @@
-add_libc_testsuite(libc_support_unittests)
+add_custom_target(libc-support-tests)
 
-add_libc_unittest(
+add_libc_test(
   blockstore_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     blockstore_test.cpp
   DEPENDS
     libc.src.__support.blockstore
 )
 
-add_libc_unittest(
+add_libc_test(
   endian_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     endian_test.cpp
   DEPENDS
     libc.src.__support.common
 )
 
-add_libc_unittest(
-  high_precision_decimal_test
-  SUITE
-    libc_support_unittests
-  SRCS
-  high_precision_decimal_test.cpp
-  DEPENDS
-    libc.src.__support.high_precision_decimal
-    libc.src.__support.uint128
-)
+# This test fails with a segmentation fault on NVPTX.
+if(NOT LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
+  add_libc_test(
+    high_precision_decimal_test
+    SUITE
+      libc-support-tests
+    SRCS
+    high_precision_decimal_test.cpp
+    DEPENDS
+      libc.src.__support.high_precision_decimal
+      libc.src.__support.uint128
+  )
+endif()
 
-add_libc_unittest(
+add_libc_test(
   str_to_float_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     str_to_float_test.cpp
   DEPENDS
@@ -43,10 +46,10 @@ add_libc_unittest(
     libc.src.errno.errno
 )
 
-add_libc_unittest(
+add_libc_test(
   integer_to_string_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     integer_to_string_test.cpp
   DEPENDS
@@ -54,20 +57,23 @@ add_libc_unittest(
     libc.src.__support.CPP.string_view
 )
 
-add_libc_unittest(
-  arg_list_test
-  SUITE
-    libc_support_unittests
-  SRCS
-    arg_list_test.cpp
-  DEPENDS
-    libc.src.__support.arg_list
-)
+# The GPU does not support varargs currently.
+if(NOT LIBC_TARGET_ARCHITECTURE_IS_GPU)
+  add_libc_test(
+    arg_list_test
+    SUITE
+      libc-support-tests
+    SRCS
+      arg_list_test.cpp
+    DEPENDS
+      libc.src.__support.arg_list
+  )
+endif()
 
-add_libc_unittest(
+add_libc_test(
   uint_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     uint_test.cpp
   DEPENDS
@@ -75,20 +81,22 @@ add_libc_unittest(
     libc.src.__support.CPP.optional
 )
 
-add_libc_unittest(
+add_libc_test(
   fixedvector_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     fixedvector_test.cpp
   DEPENDS
     libc.src.__support.fixedvector
 )
 
-add_libc_unittest(
+add_libc_test(
   char_vector_test
+  # This test relies on 'realloc' which is not implemented for hermetic tests.
+  UNIT_TEST_ONLY
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     char_vector_test.cpp
   DEPENDS
diff --git a/libc/test/src/__support/CPP/CMakeLists.txt b/libc/test/src/__support/CPP/CMakeLists.txt
index 9a9519559db33..28db8bdd5c87b 100644
--- a/libc/test/src/__support/CPP/CMakeLists.txt
+++ b/libc/test/src/__support/CPP/CMakeLists.txt
@@ -1,39 +1,39 @@
-add_libc_testsuite(libc_cpp_utils_unittests)
+add_custom_target(libc-cpp-utils-tests)
 
-add_libc_unittest(
+add_libc_test(
   bitset_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     bitset_test.cpp
   DEPENDS
     libc.src.__support.CPP.bitset
 )
 
-add_libc_unittest(
+add_libc_test(
   cstddef_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     cstddef_test.cpp
   DEPENDS
     libc.src.__support.CPP.cstddef
 )
 
-add_libc_unittest(
+add_libc_test(
   stringview_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     stringview_test.cpp
   DEPENDS
     libc.src.__support.CPP.string_view
 )
 
-add_libc_unittest(
+add_libc_test(
   limits_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     limits_test.cpp
   DEPENDS
@@ -41,61 +41,70 @@ add_libc_unittest(
     libc.src.__support.uint
 )
 
-add_libc_unittest(
+add_libc_test(
   int_seq_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     integer_sequence_test.cpp
   DEPENDS
     libc.src.__support.CPP.utility
 )
 
-add_libc_unittest(
-  atomic_test
-  SUITE
-    libc_cpp_utils_unittests
-  SRCS
-    atomic_test.cpp
-  DEPENDS
-    libc.src.__support.CPP.atomic
-)
 
-add_libc_unittest(
-  stringstream_test
-  SUITE
-    libc_cpp_utils_unittests
-  SRCS
-    stringstream_test.cpp
-  DEPENDS
-    libc.src.__support.CPP.span
-    libc.src.__support.CPP.stringstream
-)
+# This test fails with invalid address space operations on sm_60
+if(NOT LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
+  add_libc_test(
+    atomic_test
+    SUITE
+      libc-cpp-utils-tests
+    SRCS
+      atomic_test.cpp
+    DEPENDS
+      libc.src.__support.CPP.atomic
+  )
+endif()
+
+# This test fails with a segmentation fault on NVPTX.
+if(NOT LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX)
+  add_libc_test(
+    stringstream_test
+    SUITE
+      libc-cpp-utils-tests
+    SRCS
+      stringstream_test.cpp
+    DEPENDS
+      libc.src.__support.CPP.span
+      libc.src.__support.CPP.stringstream
+  )
+endif()
 
-add_libc_unittest(
+add_libc_test(
   optional_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     optional_test.cpp
   DEPENDS
     libc.src.__support.CPP.optional
 )
 
-add_libc_unittest(
+add_libc_test(
   span_test
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     span_test.cpp
   DEPENDS
     libc.src.__support.CPP.span
 )
 
-add_libc_unittest(
+add_libc_test(
   string_test
+  # This test relies on 'realloc' which is not implemented for hermetic tests.
+  UNIT_TEST_ONLY
   SUITE
-    libc_cpp_utils_unittests
+    libc-cpp-utils-tests
   SRCS
     string_test.cpp
   DEPENDS
diff --git a/libc/test/src/__support/CPP/atomic_test.cpp b/libc/test/src/__support/CPP/atomic_test.cpp
index 3bd1d752ae9eb..16408f7ad1467 100644
--- a/libc/test/src/__support/CPP/atomic_test.cpp
+++ b/libc/test/src/__support/CPP/atomic_test.cpp
@@ -14,21 +14,21 @@
 
 TEST(LlvmLibcAtomicTest, LoadStore) {
   __llvm_libc::cpp::Atomic<int> aint(123);
-  ASSERT_EQ(aint.load(), 123);
+  ASSERT_EQ(aint.load(__llvm_libc::cpp::MemoryOrder::RELAXED), 123);
 
-  aint.store(100);
-  ASSERT_EQ(aint.load(), 100);
+  aint.store(100, __llvm_libc::cpp::MemoryOrder::RELAXED);
+  ASSERT_EQ(aint.load(__llvm_libc::cpp::MemoryOrder::RELAXED), 100);
 
   aint = 1234; // Equivalent of store
-  ASSERT_EQ(aint.load(), 1234);
+  ASSERT_EQ(aint.load(__llvm_libc::cpp::MemoryOrder::RELAXED), 1234);
 }
 
 TEST(LlvmLibcAtomicTest, CompareExchangeStrong) {
   int desired = 123;
   __llvm_libc::cpp::Atomic<int> aint(desired);
   ASSERT_TRUE(aint.compare_exchange_strong(desired, 100));
-  ASSERT_EQ(aint.load(), 100);
+  ASSERT_EQ(aint.load(__llvm_libc::cpp::MemoryOrder::RELAXED), 100);
 
   ASSERT_FALSE(aint.compare_exchange_strong(desired, 100));
-  ASSERT_EQ(aint.load(), 100);
+  ASSERT_EQ(aint.load(__llvm_libc::cpp::MemoryOrder::RELAXED), 100);
 }
diff --git a/libc/test/src/__support/FPUtil/CMakeLists.txt b/libc/test/src/__support/FPUtil/CMakeLists.txt
index 47d88b169c588..2e4f7f897773f 100644
--- a/libc/test/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/test/src/__support/FPUtil/CMakeLists.txt
@@ -1,10 +1,10 @@
-add_libc_testsuite(libc_fputil_unittests)
+add_custom_target(libc-fputil-tests)
 
 add_fp_unittest(
   dyadic_float_test
   NEED_MPFR
   SUITE
-    libc_fputil_unittests
+    libc-fputil-tests
   SRCS
     dyadic_float_test.cpp
   DEPENDS
diff --git a/libc/test/src/__support/File/CMakeLists.txt b/libc/test/src/__support/File/CMakeLists.txt
index 4ee7a3255aee5..d4fed45fdaa04 100644
--- a/libc/test/src/__support/File/CMakeLists.txt
+++ b/libc/test/src/__support/File/CMakeLists.txt
@@ -1,13 +1,14 @@
-if(NOT (TARGET libc.src.__support.threads.mutex))
+if(NOT (TARGET libc.src.__support.threads.mutex) OR LIBC_TARGET_ARCHITECTURE_IS_GPU)
   # Not all platforms have a mutex implementation. If mutex is unvailable,
-  # we just skip everything about files.
+  # we just skip everything about files. The GPU does not currently support
+  # files as well.
   return()
 endif()
 
-add_libc_unittest(
+add_libc_test(
   file_test
   SUITE
-    libc_support_unittests
+    libc-support-tests
   SRCS
     file_test.cpp
   LINK_LIBRARIES
@@ -21,10 +22,10 @@ add_libc_unittest(
 )
 
 if (TARGET libc.src.__support.File.platform_file)
-  add_libc_unittest(
+  add_libc_test(
     platform_file_test
     SUITE
-      libc_support_unittests
+      libc-support-tests
     SRCS
       platform_file_test.cpp
     DEPENDS
diff --git a/libc/test/src/__support/OSUtil/CMakeLists.txt b/libc/test/src/__support/OSUtil/CMakeLists.txt
index c70fedf3ff046..c8818d4715d0f 100644
--- a/libc/test/src/__support/OSUtil/CMakeLists.txt
+++ b/libc/test/src/__support/OSUtil/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_osutil_tests)
+add_custom_target(libc-osutil-tests)
 
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   add_subdirectory(${LIBC_TARGET_OS})
diff --git a/libc/test/src/__support/OSUtil/linux/x86_64/CMakeLists.txt b/libc/test/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
index e0f2549bd8de2..577fd3169cc1f 100644
--- a/libc/test/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
+++ b/libc/test/src/__support/OSUtil/linux/x86_64/CMakeLists.txt
@@ -1,6 +1,6 @@
-add_libc_unittest(
-  syscall_unittest
-  SUITE libc_osutil_tests
+add_libc_test(
+  syscall_test
+  SUITE libc-osutil-tests
   SRCS syscall_test.cpp
   COMPILE_OPTIONS
     -Wno-unused-variable # Only signature tests, declared variables are unused.
diff --git a/libc/test/src/__support/blockstore_test.cpp b/libc/test/src/__support/blockstore_test.cpp
index d372a62a9c8e9..10277af49116d 100644
--- a/libc/test/src/__support/blockstore_test.cpp
+++ b/libc/test/src/__support/blockstore_test.cpp
@@ -90,10 +90,11 @@ TEST_F(LlvmLibcBlockStoreTest, PopulateAndIterateReverse10) {
   populate_and_iterate<4, 10, true>();
 }
 
-TEST_F(LlvmLibcBlockStoreTest, Back) {
-  back_test<false>();
-  back_test<true>();
-}
+TEST_F(LlvmLibcBlockStoreTest, Back) { back_test<false>(); }
+
+// FIXME: Combing this test with the above test makes the AMDGPU backend
+// generate code which hangs. This should be fixed in the clang compiler.
+TEST_F(LlvmLibcBlockStoreTest, BackReverse) { back_test<true>(); }
 
 TEST_F(LlvmLibcBlockStoreTest, Empty) {
   empty_test<false>();
diff --git a/libc/test/src/__support/char_vector_test.cpp b/libc/test/src/__support/char_vector_test.cpp
index 70f73729cc517..cbca178283941 100644
--- a/libc/test/src/__support/char_vector_test.cpp
+++ b/libc/test/src/__support/char_vector_test.cpp
@@ -71,8 +71,8 @@ TEST(LlvmLibcCharVectorTest, AppendLong) {
       "ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy";
   for (size_t i = 0; test_str[i] != '\0'; ++i) {
     ASSERT_EQ(v.length(), i);
-    v.append(test_str[i]);
+    ASSERT_TRUE(v.append(test_str[i]));
   }
-  ASSERT_STREQ(v.c_str(), test_str);
   ASSERT_EQ(v.length(), size_t(1000));
+  ASSERT_STREQ(v.c_str(), test_str);
 }
diff --git a/libc/test/src/assert/CMakeLists.txt b/libc/test/src/assert/CMakeLists.txt
index f672230910272..e66aca8307eeb 100644
--- a/libc/test/src/assert/CMakeLists.txt
+++ b/libc/test/src/assert/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_assert_unittests)
+add_custom_target(libc_assert_unittests)
 
 add_libc_unittest(
   assert_test
diff --git a/libc/test/src/ctype/CMakeLists.txt b/libc/test/src/ctype/CMakeLists.txt
index eadec223c8ea3..0621f83c2bae7 100644
--- a/libc/test/src/ctype/CMakeLists.txt
+++ b/libc/test/src/ctype/CMakeLists.txt
@@ -1,159 +1,159 @@
-add_libc_testsuite(libc_ctype_unittests)
+add_custom_target(libc-ctype-tests)
 
-add_libc_unittest(
+add_libc_test(
   isalnum_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isalnum_test.cpp
   DEPENDS
     libc.src.ctype.isalnum
 )
 
-add_libc_unittest(
+add_libc_test(
   isalpha_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isalpha_test.cpp
   DEPENDS
     libc.src.ctype.isalpha
 )
 
-add_libc_unittest(
+add_libc_test(
   isascii_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isascii_test.cpp
   DEPENDS
     libc.src.ctype.isascii
 )
 
-add_libc_unittest(
+add_libc_test(
   isblank_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isblank_test.cpp
   DEPENDS
     libc.src.ctype.isblank
 )
 
-add_libc_unittest(
+add_libc_test(
   iscntrl_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     iscntrl_test.cpp
   DEPENDS
     libc.src.ctype.iscntrl
 )
 
-add_libc_unittest(
+add_libc_test(
   isdigit_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isdigit_test.cpp
   DEPENDS
     libc.src.ctype.isdigit
 )
 
-add_libc_unittest(
+add_libc_test(
   isgraph_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isgraph_test.cpp
   DEPENDS
     libc.src.ctype.isgraph
 )
 
-add_libc_unittest(
+add_libc_test(
   islower_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     islower_test.cpp
   DEPENDS
     libc.src.ctype.islower
 )
 
-add_libc_unittest(
+add_libc_test(
   isprint_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isprint_test.cpp
   DEPENDS
     libc.src.ctype.isprint
 )
 
-add_libc_unittest(
+add_libc_test(
   ispunct_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     ispunct_test.cpp
   DEPENDS
     libc.src.ctype.ispunct
 )
 
-add_libc_unittest(
+add_libc_test(
   isspace_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isspace_test.cpp
   DEPENDS
     libc.src.ctype.isspace
 )
 
-add_libc_unittest(
+add_libc_test(
   isupper_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isupper_test.cpp
   DEPENDS
     libc.src.ctype.isupper
 )
 
-add_libc_unittest(
+add_libc_test(
   isxdigit_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     isxdigit_test.cpp
   DEPENDS
     libc.src.ctype.isxdigit
 )
 
-add_libc_unittest(
+add_libc_test(
   toascii_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     toascii_test.cpp
   DEPENDS
     libc.src.ctype.toascii
 )
 
-add_libc_unittest(
+add_libc_test(
   tolower_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     tolower_test.cpp
   DEPENDS
     libc.src.ctype.tolower
 )
 
-add_libc_unittest(
+add_libc_test(
   toupper_test
   SUITE
-    libc_ctype_unittests
+    libc-ctype-tests
   SRCS
     toupper_test.cpp
   DEPENDS
diff --git a/libc/test/src/dirent/CMakeLists.txt b/libc/test/src/dirent/CMakeLists.txt
index 014fe37cc09f6..b8ae813141c0d 100644
--- a/libc/test/src/dirent/CMakeLists.txt
+++ b/libc/test/src/dirent/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_subdirectory(testdata)
-add_libc_testsuite(libc_dirent_unittests)
+add_custom_target(libc_dirent_unittests)
 
 add_libc_unittest(
   dirent_test
diff --git a/libc/test/src/errno/CMakeLists.txt b/libc/test/src/errno/CMakeLists.txt
index 70520735b9d6f..633d46a1f5f88 100644
--- a/libc/test/src/errno/CMakeLists.txt
+++ b/libc/test/src/errno/CMakeLists.txt
@@ -1,8 +1,8 @@
-if(NOT LLVM_LIBC_FULL_BUILD)
+if(NOT LLVM_LIBC_FULL_BUILD OR LIBC_TARGET_ARCHITECTURE_IS_GPU)
   return()
 endif()
 
-add_libc_testsuite(libc_errno_unittests)
+add_custom_target(libc_errno_unittests)
 
 add_libc_unittest(
   errno_test
diff --git a/libc/test/src/fcntl/CMakeLists.txt b/libc/test/src/fcntl/CMakeLists.txt
index d90b2fab10347..f3d265ec238bd 100644
--- a/libc/test/src/fcntl/CMakeLists.txt
+++ b/libc/test/src/fcntl/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_fcntl_unittests)
+add_custom_target(libc_fcntl_unittests)
 
 add_subdirectory(testdata)
 
diff --git a/libc/test/src/fenv/CMakeLists.txt b/libc/test/src/fenv/CMakeLists.txt
index cf27fa5b5188c..11179cc88c380 100644
--- a/libc/test/src/fenv/CMakeLists.txt
+++ b/libc/test/src/fenv/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_fenv_unittests)
+add_custom_target(libc_fenv_unittests)
 
 add_libc_unittest(
   rounding_mode_test
@@ -104,6 +104,8 @@ if (NOT (LLVM_USE_SANITIZER OR (${LIBC_TARGET_OS} STREQUAL "windows")
       libc.src.fenv.feraiseexcept
       libc.src.fenv.fetestexcept
       libc.src.__support.FPUtil.fenv_impl
+    LINK_LIBRARIES
+      LibcFPExceptionHelpers
   )
 
   add_fp_unittest(
@@ -116,5 +118,7 @@ if (NOT (LLVM_USE_SANITIZER OR (${LIBC_TARGET_OS} STREQUAL "windows")
       libc.include.fenv
       libc.src.fenv.feholdexcept
       libc.src.__support.FPUtil.fenv_impl
+    LINK_LIBRARIES
+      LibcFPExceptionHelpers
   )
 endif()
diff --git a/libc/test/src/inttypes/CMakeLists.txt b/libc/test/src/inttypes/CMakeLists.txt
index 7fee729bc641c..cbcd8463747e8 100644
--- a/libc/test/src/inttypes/CMakeLists.txt
+++ b/libc/test/src/inttypes/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_inttypes_unittests)
+add_custom_target(libc_inttypes_unittests)
 
 add_libc_unittest(
   strtoimax_test
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 0cececdc0cdb4..c71090706cc40 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_math_unittests)
+add_custom_target(libc_math_unittests)
 
 add_library(
   libc_math_test_utils
@@ -1571,5 +1571,8 @@ add_fp_unittest(
 )
 
 add_subdirectory(generic)
-add_subdirectory(exhaustive)
-add_subdirectory(differential_testing)
+
+if(NOT LLVM_LIBC_FULL_BUILD)
+  add_subdirectory(exhaustive)
+  add_subdirectory(differential_testing)
+endif()
diff --git a/libc/test/src/math/exhaustive/CMakeLists.txt b/libc/test/src/math/exhaustive/CMakeLists.txt
index 107e7915650e3..4c3f524f0b904 100644
--- a/libc/test/src/math/exhaustive/CMakeLists.txt
+++ b/libc/test/src/math/exhaustive/CMakeLists.txt
@@ -10,6 +10,7 @@ add_object_library(
 
 add_fp_unittest(
   sqrtf_test
+  NO_RUN_POSTBUILD
   NEED_MPFR
   SUITE
     libc_math_exhaustive_tests
diff --git a/libc/test/src/network/CMakeLists.txt b/libc/test/src/network/CMakeLists.txt
index 7bff6da77ed09..222205dfe247a 100644
--- a/libc/test/src/network/CMakeLists.txt
+++ b/libc/test/src/network/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_network_unittests)
+add_custom_target(libc_network_unittests)
 
 add_libc_unittest(
   htonl
diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt
index db0516cecc4fd..fb0d22ab9d2a5 100644
--- a/libc/test/src/pthread/CMakeLists.txt
+++ b/libc/test/src/pthread/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_pthread_unittests)
+add_custom_target(libc_pthread_unittests)
 
 add_libc_unittest(
   pthread_attr_test
diff --git a/libc/test/src/sched/CMakeLists.txt b/libc/test/src/sched/CMakeLists.txt
index 15a6257702346..ef6cdf34843b3 100644
--- a/libc/test/src/sched/CMakeLists.txt
+++ b/libc/test/src/sched/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sched_unittests)
+add_custom_target(libc_sched_unittests)
 
 add_libc_unittest(
   affinity_test
diff --git a/libc/test/src/sched/get_priority_test.cpp b/libc/test/src/sched/get_priority_test.cpp
index fb532e25688e9..15ee958ec16f4 100644
--- a/libc/test/src/sched/get_priority_test.cpp
+++ b/libc/test/src/sched/get_priority_test.cpp
@@ -89,7 +89,7 @@ TEST(LlvmLibcSchedGetPriorityTest, SmokeTest) {
 
     ASSERT_LE(max_priority, 99);
     ASSERT_GE(min_priority, 0);
-    ASSERT_GE(max_priority, min_priority);
+    ASSERT_GT(max_priority, min_priority);
   }
 
   {
@@ -103,6 +103,6 @@ TEST(LlvmLibcSchedGetPriorityTest, SmokeTest) {
 
     ASSERT_LE(max_priority, 99);
     ASSERT_GE(min_priority, 0);
-    ASSERT_GE(max_priority, min_priority);
+    ASSERT_GT(max_priority, min_priority);
   }
 }
diff --git a/libc/test/src/setjmp/CMakeLists.txt b/libc/test/src/setjmp/CMakeLists.txt
index cfa35089d3b67..f637842c03425 100644
--- a/libc/test/src/setjmp/CMakeLists.txt
+++ b/libc/test/src/setjmp/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_setjmp_unittests)
+add_custom_target(libc_setjmp_unittests)
 
 # Skip setjmp tests under sanitizers
 if(LLVM_USE_SANITIZER)
diff --git a/libc/test/src/signal/CMakeLists.txt b/libc/test/src/signal/CMakeLists.txt
index fe49328668032..3016f2a0a6d40 100644
--- a/libc/test/src/signal/CMakeLists.txt
+++ b/libc/test/src/signal/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_signal_unittests)
+add_custom_target(libc_signal_unittests)
 
 add_libc_unittest(
   raise_test
diff --git a/libc/test/src/spawn/CMakeLists.txt b/libc/test/src/spawn/CMakeLists.txt
index c7e7bc29ad02b..39b63df190849 100644
--- a/libc/test/src/spawn/CMakeLists.txt
+++ b/libc/test/src/spawn/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_spawn_unittests)
+add_custom_target(libc_spawn_unittests)
 
 add_libc_unittest(
   posix_spawn_file_actions_test
diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt
index 4261238e7361c..5b66cadcdc127 100644
--- a/libc/test/src/stdio/CMakeLists.txt
+++ b/libc/test/src/stdio/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_stdio_unittests)
+add_custom_target(libc_stdio_unittests)
 
 add_libc_unittest(
   fileop_test
@@ -318,6 +318,10 @@ add_libc_unittest(
     libc.src.stdio.setvbuf
 )
 
+if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
+  return()
+endif()
+
 add_subdirectory(printf_core)
 add_subdirectory(scanf_core)
 add_subdirectory(testdata)
diff --git a/libc/test/src/stdio/fprintf_test.cpp b/libc/test/src/stdio/fprintf_test.cpp
index 20b3c0faed6f7..a521bb47eb136 100644
--- a/libc/test/src/stdio/fprintf_test.cpp
+++ b/libc/test/src/stdio/fprintf_test.cpp
@@ -34,8 +34,10 @@ using ::fread;
 } // namespace printf_test
 
 TEST(LlvmLibcFPrintfTest, WriteToFile) {
-  constexpr char FILENAME[] = "testdata/fprintf_output.test";
-  ::FILE *file = printf_test::fopen(FILENAME, "w");
+  const char *FILENAME = "fprintf_output.test";
+  auto FILE_PATH = libc_make_test_file_path(FILENAME);
+
+  ::FILE *file = printf_test::fopen(FILE_PATH, "w");
   ASSERT_FALSE(file == nullptr);
 
   int written;
@@ -55,7 +57,7 @@ TEST(LlvmLibcFPrintfTest, WriteToFile) {
 
   ASSERT_EQ(0, printf_test::fclose(file));
 
-  file = printf_test::fopen(FILENAME, "r");
+  file = printf_test::fopen(FILE_PATH, "r");
   ASSERT_FALSE(file == nullptr);
 
   char data[50];
diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt
index 55bc28606e6eb..01d3257b148db 100644
--- a/libc/test/src/stdlib/CMakeLists.txt
+++ b/libc/test/src/stdlib/CMakeLists.txt
@@ -1,9 +1,9 @@
-add_libc_testsuite(libc_stdlib_unittests)
+add_custom_target(libc-stdlib-tests)
 
-add_libc_unittest(
+add_libc_test(
   atof_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     atof_test.cpp
   DEPENDS
@@ -20,10 +20,10 @@ add_header_library(
     libc.src.__support.CPP.type_traits
 )
 
-add_libc_unittest(
+add_libc_test(
   atoi_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     atoi_test.cpp
   DEPENDS
@@ -31,10 +31,10 @@ add_libc_unittest(
     libc.src.stdlib.atoi
 )
 
-add_libc_unittest(
+add_libc_test(
   atol_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     atol_test.cpp
   DEPENDS
@@ -42,10 +42,10 @@ add_libc_unittest(
     libc.src.stdlib.atol
 )
 
-add_libc_unittest(
+add_libc_test(
   atoll_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     atoll_test.cpp
   DEPENDS
@@ -56,23 +56,25 @@ add_libc_unittest(
 add_fp_unittest(
   strtod_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtod_test.cpp
   DEPENDS
     libc.src.errno.errno
     libc.src.stdlib.strtod
+    libc.src.__support.FPUtil.fenv_impl
 )
 
 add_fp_unittest(
   strtof_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtof_test.cpp
   DEPENDS
     libc.src.errno.errno
     libc.src.stdlib.strtof
+    libc.src.__support.FPUtil.fenv_impl
 )
 
 add_header_library(
@@ -85,10 +87,10 @@ add_header_library(
     libc.src.errno.errno
 )
 
-add_libc_unittest(
+add_libc_test(
   strtol_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtol_test.cpp
   DEPENDS
@@ -96,10 +98,10 @@ add_libc_unittest(
     .strtol_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   strtold_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtold_test.cpp
   DEPENDS
@@ -108,10 +110,10 @@ add_libc_unittest(
     libc.src.stdlib.strtold
 )
 
-add_libc_unittest(
+add_libc_test(
   strtoll_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtoll_test.cpp
   DEPENDS
@@ -119,10 +121,10 @@ add_libc_unittest(
     .strtol_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   strtoul_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtoul_test.cpp
   DEPENDS
@@ -130,10 +132,10 @@ add_libc_unittest(
     .strtol_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   strtoull_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     strtoull_test.cpp
   DEPENDS
@@ -141,40 +143,40 @@ add_libc_unittest(
     .strtol_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   abs_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     abs_test.cpp
   DEPENDS
     libc.src.stdlib.abs
 )
 
-add_libc_unittest(
+add_libc_test(
   labs_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     labs_test.cpp
   DEPENDS
     libc.src.stdlib.labs
 )
 
-add_libc_unittest(
+add_libc_test(
   llabs_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     llabs_test.cpp
   DEPENDS
     libc.src.stdlib.llabs
 )
 
-add_libc_unittest(
+add_libc_test(
   div_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     div_test.cpp
   HDRS
@@ -184,10 +186,10 @@ add_libc_unittest(
     libc.src.stdlib.div
 )
 
-add_libc_unittest(
+add_libc_test(
   ldiv_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     ldiv_test.cpp
   HDRS
@@ -197,10 +199,10 @@ add_libc_unittest(
     libc.src.stdlib.ldiv
 )
 
-add_libc_unittest(
+add_libc_test(
   lldiv_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     lldiv_test.cpp
   HDRS
@@ -210,10 +212,10 @@ add_libc_unittest(
     libc.src.stdlib.lldiv
 )
 
-add_libc_unittest(
+add_libc_test(
   bsearch_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     bsearch_test.cpp
   DEPENDS
@@ -221,10 +223,10 @@ add_libc_unittest(
     libc.src.stdlib.bsearch
 )
 
-add_libc_unittest(
+add_libc_test(
   qsort_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     qsort_test.cpp
   DEPENDS
@@ -232,10 +234,10 @@ add_libc_unittest(
     libc.src.stdlib.qsort
 )
 
-add_libc_unittest(
+add_libc_test(
   rand_test
   SUITE
-    libc_stdlib_unittests
+    libc-stdlib-tests
   SRCS
     rand_test.cpp
   DEPENDS
@@ -246,10 +248,12 @@ add_libc_unittest(
 
 if(LLVM_LIBC_FULL_BUILD)
 
-  add_libc_unittest(
+  add_libc_test(
     _Exit_test
+    # The EXPECT_EXITS test is only availible for unit tests.
+    UNIT_TEST_ONLY
     SUITE
-      libc_stdlib_unittests
+      libc-stdlib-tests
     SRCS
       _Exit_test.cpp
     DEPENDS
@@ -258,10 +262,12 @@ if(LLVM_LIBC_FULL_BUILD)
       libc.src.stdlib.exit
   )
 
-  add_libc_unittest(
+  add_libc_test(
     atexit_test
+    # The EXPECT_EXITS test is only availible for unit tests.
+    UNIT_TEST_ONLY
     SUITE
-      libc_stdlib_unittests
+      libc-stdlib-tests
     SRCS
       atexit_test.cpp
     DEPENDS
@@ -273,10 +279,12 @@ if(LLVM_LIBC_FULL_BUILD)
       libc.src.__support.CPP.utility
   )
 
-  add_libc_unittest(
+  add_libc_test(
     abort_test
+    # The EXPECT_DEATH test is only availible for unit tests.
+    UNIT_TEST_ONLY
     SUITE
-      libc_stdlib_unittests
+      libc-stdlib-tests
     SRCS
       abort_test.cpp
     DEPENDS
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index af5ddfeb3cdce..b90da43f9a7b9 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -1,11 +1,11 @@
-add_libc_testsuite(libc_string_unittests)
+add_custom_target(libc-string-tests)
 
 add_subdirectory(memory_utils)
 
-add_libc_unittest(
+add_libc_test(
   bcopy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     bcopy_test.cpp
   DEPENDS
@@ -20,10 +20,10 @@ add_header_library(
     StrchrTest.h
 )
 
-add_libc_unittest(
+add_libc_test(
   index_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     index_test.cpp
   DEPENDS
@@ -31,60 +31,60 @@ add_libc_unittest(
     .strchr_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   memccpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     memccpy_test.cpp
   DEPENDS
     libc.src.string.memccpy
 )
 
-add_libc_unittest(
+add_libc_test(
   mempcpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     mempcpy_test.cpp
   DEPENDS
     libc.src.string.mempcpy
 )
 
-add_libc_unittest(
+add_libc_test(
   memmem_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     memmem_test.cpp
   DEPENDS
     libc.src.string.memmem
 )
 
-add_libc_unittest(
+add_libc_test(
   memchr_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     memchr_test.cpp
   DEPENDS
     libc.src.string.memchr
 )
 
-add_libc_unittest(
+add_libc_test(
   memrchr_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     memrchr_test.cpp
   DEPENDS
     libc.src.string.memrchr
 )
 
-add_libc_unittest(
+add_libc_test(
   rindex_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     rindex_test.cpp
   DEPENDS
@@ -92,40 +92,40 @@ add_libc_unittest(
     .strchr_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   stpcpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     stpcpy_test.cpp
   DEPENDS
     libc.src.string.stpcpy
 )
 
-add_libc_unittest(
+add_libc_test(
   stpncpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     stpncpy_test.cpp
   DEPENDS
     libc.src.string.stpncpy
 )
 
-add_libc_unittest(
+add_libc_test(
   strcat_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcat_test.cpp
   DEPENDS
     libc.src.string.strcat
 )
 
-add_libc_unittest(
+add_libc_test(
   strchr_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strchr_test.cpp
   DEPENDS
@@ -133,80 +133,80 @@ add_libc_unittest(
     .strchr_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   strchrnul_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strchrnul_test.cpp
   DEPENDS
     libc.src.string.strchrnul
 )
 
-add_libc_unittest(
+add_libc_test(
   strcmp_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcmp_test.cpp
   DEPENDS
     libc.src.string.strcmp
 )
 
-add_libc_unittest(
+add_libc_test(
   strcasecmp_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcasecmp_test.cpp
   DEPENDS
     libc.src.string.strcasecmp
 )
 
-add_libc_unittest(
+add_libc_test(
   strcasestr_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcasestr_test.cpp
   DEPENDS
     libc.src.string.strcasestr
 )
 
-add_libc_unittest(
+add_libc_test(
   strcoll_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcoll_test.cpp
   DEPENDS
     libc.src.string.strcoll
 )
 
-add_libc_unittest(
+add_libc_test(
   strcpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcpy_test.cpp
   DEPENDS
     libc.src.string.strcpy
 )
 
-add_libc_unittest(
+add_libc_test(
   strcspn_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strcspn_test.cpp
   DEPENDS
     libc.src.string.strcspn
 )
 
-add_libc_unittest(
+add_libc_test(
   strdup_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strdup_test.cpp
   DEPENDS
@@ -215,10 +215,10 @@ add_libc_unittest(
     libc.src.errno.errno
 )
 
-add_libc_unittest(
+add_libc_test(
   strerror_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strerror_test.cpp
   DEPENDS
@@ -226,90 +226,90 @@ add_libc_unittest(
 )
 
 
-add_libc_unittest(
+add_libc_test(
   strerror_r_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strerror_r_test.cpp
   DEPENDS
     libc.src.string.strerror_r
 )
 
-add_libc_unittest(
+add_libc_test(
   strlcat_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strlcat_test.cpp
   DEPENDS
     libc.src.string.strlcat
 )
 
-add_libc_unittest(
+add_libc_test(
   strlcpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strlcpy_test.cpp
   DEPENDS
     libc.src.string.strlcpy
 )
 
-add_libc_unittest(
+add_libc_test(
   strlen_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strlen_test.cpp
   DEPENDS
     libc.src.string.strlen
 )
 
-add_libc_unittest(
+add_libc_test(
   strncat_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strncat_test.cpp
   DEPENDS
     libc.src.string.strncat
 )
 
-add_libc_unittest(
+add_libc_test(
   strncmp_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strncmp_test.cpp
   DEPENDS
     libc.src.string.strncmp
 )
 
-add_libc_unittest(
+add_libc_test(
   strncasecmp_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strncasecmp_test.cpp
   DEPENDS
     libc.src.string.strncasecmp
 )
 
-add_libc_unittest(
+add_libc_test(
   strncpy_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strncpy_test.cpp
   DEPENDS
     libc.src.string.strncpy
 )
 
-add_libc_unittest(
+add_libc_test(
   strndup_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strndup_test.cpp
   DEPENDS
@@ -317,30 +317,30 @@ add_libc_unittest(
     libc.src.string.strndup
 )
 
-add_libc_unittest(
+add_libc_test(
   strnlen_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strnlen_test.cpp
   DEPENDS
     libc.src.string.strnlen
 )
 
-add_libc_unittest(
+add_libc_test(
   strpbrk_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strpbrk_test.cpp
   DEPENDS
     libc.src.string.strpbrk
 )
 
-add_libc_unittest(
+add_libc_test(
   strrchr_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strrchr_test.cpp
   DEPENDS
@@ -348,70 +348,70 @@ add_libc_unittest(
     .strchr_test_support
 )
 
-add_libc_unittest(
+add_libc_test(
   strsep_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strsep_test.cpp
   DEPENDS
     libc.src.string.strsep
 )
 
-add_libc_unittest(
+add_libc_test(
   strsignal_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strsignal_test.cpp
   DEPENDS
     libc.src.string.strsignal
 )
 
-add_libc_unittest(
+add_libc_test(
   strspn_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strspn_test.cpp
   DEPENDS
     libc.src.string.strspn
 )
 
-add_libc_unittest(
+add_libc_test(
   strstr_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strstr_test.cpp
   DEPENDS
     libc.src.string.strstr
 )
 
-add_libc_unittest(
+add_libc_test(
   strtok_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strtok_test.cpp
   DEPENDS
     libc.src.string.strtok
 )
 
-add_libc_unittest(
+add_libc_test(
   strtok_r_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strtok_r_test.cpp
   DEPENDS
     libc.src.string.strtok_r
 )
 
-add_libc_unittest(
+add_libc_test(
   strxfrm_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     strxfrm_test.cpp
   DEPENDS
@@ -425,10 +425,10 @@ function(add_libc_multi_impl_test name)
     get_target_property(required_cpu_features ${fq_config_name} REQUIRE_CPU_FEATURES)
     cpu_supports(can_run "${required_cpu_features}")
     if(can_run)
-      add_libc_unittest(
+      add_libc_test(
         ${fq_config_name}_test
         SUITE
-          libc_string_unittests
+          libc-string-tests
         COMPILE_OPTIONS
           ${LIBC_COMPILE_OPTIONS_NATIVE}
         LINK_LIBRARIES
diff --git a/libc/test/src/string/StrchrTest.h b/libc/test/src/string/StrchrTest.h
index 1c5bee52863cc..71903c987cacb 100644
--- a/libc/test/src/string/StrchrTest.h
+++ b/libc/test/src/string/StrchrTest.h
@@ -120,7 +120,7 @@ template <auto Func> struct StrrchrTest : public __llvm_libc::testing::Test {
   }
 
   void findsLastBehindFirstNullTerminator() {
-    const char src[6] = {'a', 'a', '\0', 'b', '\0', 'c'};
+    static const char src[6] = {'a', 'a', '\0', 'b', '\0', 'c'};
     // 'b' is behind a null terminator, so should not be found.
     ASSERT_STREQ(Func(src, 'b'), nullptr);
     // Same goes for 'c'.
diff --git a/libc/test/src/string/bzero_test.cpp b/libc/test/src/string/bzero_test.cpp
index a476c43bdf6e6..e809506386132 100644
--- a/libc/test/src/string/bzero_test.cpp
+++ b/libc/test/src/string/bzero_test.cpp
@@ -15,7 +15,6 @@ namespace __llvm_libc {
 // Adapt CheckMemset signature to op implementation signatures.
 template <auto FnImpl>
 void BzeroAdaptor(cpp::span<char> p1, uint8_t value, size_t size) {
-  assert(value == 0);
   FnImpl(p1.begin(), size);
 }
 
diff --git a/libc/test/src/string/memory_utils/CMakeLists.txt b/libc/test/src/string/memory_utils/CMakeLists.txt
index e222a616321bb..06b98381390bd 100644
--- a/libc/test/src/string/memory_utils/CMakeLists.txt
+++ b/libc/test/src/string/memory_utils/CMakeLists.txt
@@ -1,7 +1,8 @@
-add_libc_unittest(
+# This test currently uses too much memory to be made hermetic.
+add_libc_test(
   utils_test
   SUITE
-    libc_string_unittests
+    libc-string-tests
   SRCS
     op_tests.cpp
     utils_test.cpp
@@ -14,4 +15,5 @@ add_libc_unittest(
     libc.src.__support.CPP.span
     libc.src.__support.macros.sanitizer
     libc.src.string.memory_utils.memory_utils
+  UNIT_TEST_ONLY
 )
diff --git a/libc/test/src/string/memory_utils/memory_check_utils.h b/libc/test/src/string/memory_utils/memory_check_utils.h
index 46c362d6b3701..c2b910edd952a 100644
--- a/libc/test/src/string/memory_utils/memory_check_utils.h
+++ b/libc/test/src/string/memory_utils/memory_check_utils.h
@@ -12,7 +12,6 @@
 #include "src/__support/CPP/span.h"
 #include "src/__support/macros/sanitizer.h"
 #include "src/string/memory_utils/utils.h"
-#include <assert.h> // assert
 #include <stddef.h> // size_t
 #include <stdint.h> // uintxx_t
 #include <stdlib.h> // malloc/free
@@ -24,7 +23,6 @@ namespace __llvm_libc {
 // This is a utility class to be used by Buffer below, do not use directly.
 struct PoisonedBuffer {
   PoisonedBuffer(size_t size) : ptr((char *)malloc(size)) {
-    assert(ptr);
     ASAN_POISON_MEMORY_REGION(ptr, size);
   }
   ~PoisonedBuffer() { free(ptr); }
@@ -45,11 +43,8 @@ struct Buffer : private PoisonedBuffer {
       : PoisonedBuffer(size + kLeeway), size(size) {
     offset_ptr = ptr;
     offset_ptr += distance_to_next_aligned<kAlign>(ptr);
-    assert((uintptr_t)(offset_ptr) % kAlign == 0);
     if (aligned == Aligned::NO)
       ++offset_ptr;
-    assert(offset_ptr > ptr);
-    assert((offset_ptr + size) < (ptr + size + kLeeway));
     ASAN_UNPOISON_MEMORY_REGION(offset_ptr, size);
   }
   cpp::span<char> span() { return cpp::span<char>(offset_ptr, size); }
@@ -77,7 +72,6 @@ static inline void Randomize(cpp::span<char> buffer) {
 // Copy one span to another.
 static inline void ReferenceCopy(cpp::span<char> dst,
                                  const cpp::span<char> src) {
-  assert(dst.size() == src.size());
   for (size_t i = 0; i < dst.size(); ++i)
     dst[i] = src[i];
 }
@@ -85,8 +79,6 @@ static inline void ReferenceCopy(cpp::span<char> dst,
 // Checks that FnImpl implements the memcpy semantic.
 template <auto FnImpl>
 bool CheckMemcpy(cpp::span<char> dst, cpp::span<char> src, size_t size) {
-  assert(dst.size() == src.size());
-  assert(dst.size() == size);
   Randomize(dst);
   FnImpl(dst, src, size);
   for (size_t i = 0; i < size; ++i)
@@ -109,7 +101,6 @@ bool CheckMemset(cpp::span<char> dst, uint8_t value, size_t size) {
 // Checks that FnImpl implements the bcmp semantic.
 template <auto FnImpl>
 bool CheckBcmp(cpp::span<char> span1, cpp::span<char> span2, size_t size) {
-  assert(span1.size() == span2.size());
   ReferenceCopy(span2, span1);
   // Compare equal
   if (int cmp = FnImpl(span1, span2, size); cmp != 0)
@@ -129,7 +120,6 @@ bool CheckBcmp(cpp::span<char> span1, cpp::span<char> span2, size_t size) {
 // Checks that FnImpl implements the memcmp semantic.
 template <auto FnImpl>
 bool CheckMemcmp(cpp::span<char> span1, cpp::span<char> span2, size_t size) {
-  assert(span1.size() == span2.size());
   ReferenceCopy(span2, span1);
   // Compare equal
   if (int cmp = FnImpl(span1, span2, size); cmp != 0)
diff --git a/libc/test/src/string/memory_utils/op_tests.cpp b/libc/test/src/string/memory_utils/op_tests.cpp
index f845f49a01d90..e38edc54e8a9f 100644
--- a/libc/test/src/string/memory_utils/op_tests.cpp
+++ b/libc/test/src/string/memory_utils/op_tests.cpp
@@ -13,8 +13,6 @@
 #include "src/string/memory_utils/op_x86.h"
 #include "test/UnitTest/Test.h"
 
-#include <assert.h>
-
 #if defined(LIBC_TARGET_ARCH_IS_X86_64) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
 #define LLVM_LIBC_HAS_UINT64
 #endif
@@ -75,7 +73,6 @@ void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
 }
 template <size_t Size, auto FnImpl>
 void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) {
-  assert(size == Size);
   FnImpl(as_byte(dst), as_byte(src));
 }
 
@@ -157,7 +154,6 @@ void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
 }
 template <size_t Size, auto FnImpl>
 void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) {
-  assert(size == Size);
   FnImpl(as_byte(dst), value);
 }
 
@@ -235,7 +231,6 @@ int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
 }
 template <size_t Size, auto FnImpl>
 int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) {
-  assert(size == Size);
   return (int)FnImpl(as_byte(p1), as_byte(p2));
 }
 
diff --git a/libc/test/src/sys/CMakeLists.txt b/libc/test/src/sys/CMakeLists.txt
index 94f62123d2c40..5ef97fe817700 100644
--- a/libc/test/src/sys/CMakeLists.txt
+++ b/libc/test/src/sys/CMakeLists.txt
@@ -3,6 +3,7 @@ add_subdirectory(random)
 add_subdirectory(resource)
 add_subdirectory(select)
 add_subdirectory(sendfile)
+add_subdirectory(socket)
 add_subdirectory(stat)
 add_subdirectory(utsname)
 add_subdirectory(wait)
diff --git a/libc/test/src/sys/mman/linux/CMakeLists.txt b/libc/test/src/sys/mman/linux/CMakeLists.txt
index 8b35842cd810a..e493c8cb6f22c 100644
--- a/libc/test/src/sys/mman/linux/CMakeLists.txt
+++ b/libc/test/src/sys/mman/linux/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_mman_unittests)
+add_custom_target(libc_sys_mman_unittests)
 
 add_libc_unittest(
   mmap_test
diff --git a/libc/test/src/sys/random/linux/CMakeLists.txt b/libc/test/src/sys/random/linux/CMakeLists.txt
index 6277749087697..32c1b6128166e 100644
--- a/libc/test/src/sys/random/linux/CMakeLists.txt
+++ b/libc/test/src/sys/random/linux/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_random_unittests)
+add_custom_target(libc_sys_random_unittests)
 
 add_libc_unittest(
   getrandom_test
diff --git a/libc/test/src/sys/resource/CMakeLists.txt b/libc/test/src/sys/resource/CMakeLists.txt
index 574a4dc8d64c0..90dc0e846df6d 100644
--- a/libc/test/src/sys/resource/CMakeLists.txt
+++ b/libc/test/src/sys/resource/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_resource_unittests)
+add_custom_target(libc_sys_resource_unittests)
 
 add_subdirectory(testdata)
 
diff --git a/libc/test/src/sys/select/CMakeLists.txt b/libc/test/src/sys/select/CMakeLists.txt
index 9d45a447797b7..3ad73d390a4ed 100644
--- a/libc/test/src/sys/select/CMakeLists.txt
+++ b/libc/test/src/sys/select/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_select_unittests)
+add_custom_target(libc_sys_select_unittests)
 
 add_libc_unittest(
   select_ui_test
diff --git a/libc/test/src/sys/sendfile/CMakeLists.txt b/libc/test/src/sys/sendfile/CMakeLists.txt
index d2aa830fb0529..57dac4accd828 100644
--- a/libc/test/src/sys/sendfile/CMakeLists.txt
+++ b/libc/test/src/sys/sendfile/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_sendfile_unittests)
+add_custom_target(libc_sys_sendfile_unittests)
 
 add_subdirectory(testdata)
 
diff --git a/libc/test/src/sys/socket/CMakeLists.txt b/libc/test/src/sys/socket/CMakeLists.txt
new file mode 100644
index 0000000000000..b4bbe81c92ff2
--- /dev/null
+++ b/libc/test/src/sys/socket/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/test/src/sys/socket/linux/CMakeLists.txt b/libc/test/src/sys/socket/linux/CMakeLists.txt
new file mode 100644
index 0000000000000..4380597e55157
--- /dev/null
+++ b/libc/test/src/sys/socket/linux/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_custom_target(libc_sys_socket_unittests)
+
+add_libc_unittest(
+  socket_test
+  SUITE
+    libc_sys_socket_unittests
+  SRCS
+    socket_test.cpp
+  DEPENDS
+    libc.include.sys_socket
+    libc.src.errno.errno
+    libc.src.sys.socket.socket
+    libc.src.unistd.close
+)
diff --git a/libc/test/src/sys/socket/linux/socket_test.cpp b/libc/test/src/sys/socket/linux/socket_test.cpp
new file mode 100644
index 0000000000000..6826594e52d7b
--- /dev/null
+++ b/libc/test/src/sys/socket/linux/socket_test.cpp
@@ -0,0 +1,24 @@
+//===-- Unittests for socket ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/socket/socket.h"
+
+#include "src/unistd/close.h"
+
+#include "src/errno/libc_errno.h"
+#include "test/UnitTest/Test.h"
+
+#include <sys/socket.h> // For AF_LOCAL and SOCK_DGRAM
+
+TEST(LlvmLibcSocketTest, LocalSocket) {
+  int sock = __llvm_libc::socket(AF_LOCAL, SOCK_DGRAM, 0);
+  ASSERT_GE(sock, 0);
+  ASSERT_EQ(libc_errno, 0);
+
+  __llvm_libc::close(sock);
+}
diff --git a/libc/test/src/sys/stat/CMakeLists.txt b/libc/test/src/sys/stat/CMakeLists.txt
index f21b7a0323a0f..877a129b627dd 100644
--- a/libc/test/src/sys/stat/CMakeLists.txt
+++ b/libc/test/src/sys/stat/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_stat_unittests)
+add_custom_target(libc_sys_stat_unittests)
 
 add_subdirectory(testdata)
 
diff --git a/libc/test/src/sys/utsname/CMakeLists.txt b/libc/test/src/sys/utsname/CMakeLists.txt
index a459820c86ea1..d1c0ba2aadd73 100644
--- a/libc/test/src/sys/utsname/CMakeLists.txt
+++ b/libc/test/src/sys/utsname/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_utsname_unittests)
+add_custom_target(libc_sys_utsname_unittests)
 
 add_libc_unittest(
   uname_test
diff --git a/libc/test/src/sys/wait/CMakeLists.txt b/libc/test/src/sys/wait/CMakeLists.txt
index 59176b8d6d7be..4ba1a939d6a8e 100644
--- a/libc/test/src/sys/wait/CMakeLists.txt
+++ b/libc/test/src/sys/wait/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_sys_wait_unittests)
+add_custom_target(libc_sys_wait_unittests)
 
 add_libc_unittest(
   waitpid_test
diff --git a/libc/test/src/termios/CMakeLists.txt b/libc/test/src/termios/CMakeLists.txt
index 991d8d613539d..9f770da101394 100644
--- a/libc/test/src/termios/CMakeLists.txt
+++ b/libc/test/src/termios/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_termios_unittests)
+add_custom_target(libc_termios_unittests)
 
 add_libc_unittest(
   termios_test
diff --git a/libc/test/src/time/CMakeLists.txt b/libc/test/src/time/CMakeLists.txt
index 378d754ca3e0e..6f15bdbc07d4a 100644
--- a/libc/test/src/time/CMakeLists.txt
+++ b/libc/test/src/time/CMakeLists.txt
@@ -1,7 +1,7 @@
-add_libc_testsuite(libc_time_unittests)
+add_custom_target(libc_time_unittests)
 
 add_libc_unittest(
-  asctime
+  asctime_test
   SUITE
     libc_time_unittests
   SRCS
@@ -16,7 +16,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  asctime_r
+  asctime_r_test
   SUITE
     libc_time_unittests
   SRCS
@@ -31,7 +31,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  clock_gettime
+  clock_gettime_test
   SUITE
     libc_time_unittests
   SRCS
@@ -41,7 +41,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  difftime
+  difftime_test
   SUITE
     libc_time_unittests
   SRCS
@@ -51,7 +51,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  gettimeofday
+  gettimeofday_test
   SUITE
     libc_time_unittests
   SRCS
@@ -69,7 +69,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  gmtime
+  gmtime_test
   SUITE
     libc_time_unittests
   SRCS
@@ -81,7 +81,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  gmtime_r
+  gmtime_r_test
   SUITE
     libc_time_unittests
   SRCS
@@ -93,7 +93,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  mktime
+  mktime_test
   SUITE
     libc_time_unittests
   SRCS
@@ -108,7 +108,7 @@ add_libc_unittest(
 )
 
 add_libc_unittest(
-  nanosleep
+  nanosleep_test
   SUITE
     libc_time_unittests
   SRCS
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index 1ec29d36c65f9..ceee888669b7e 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_unistd_unittests)
+add_custom_target(libc_unistd_unittests)
 
 add_subdirectory(testdata)
 
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index 8b03002e90224..cabf42ba9b0b2 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_libc_testsuite(libc_wchar_unittests)
+add_custom_target(libc_wchar_unittests)
 
 add_libc_unittest(
   btowc_test
diff --git a/libc/test/utils/FPUtil/x86_long_double_test.cpp b/libc/test/utils/FPUtil/x86_long_double_test.cpp
index 16248ae267619..c2435b43ab72b 100644
--- a/libc/test/utils/FPUtil/x86_long_double_test.cpp
+++ b/libc/test/utils/FPUtil/x86_long_double_test.cpp
@@ -13,7 +13,7 @@
 
 using FPBits = __llvm_libc::fputil::FPBits<long double>;
 
-TEST(X86LongDoubleTest, is_nan) {
+TEST(LlvmLibcX86LongDoubleTest, is_nan) {
   // In the nan checks below, we use the macro isnan from math.h to ensure that
   // a number is actually a NaN. The isnan macro resolves to the compiler
   // builtin function. Hence, matching LLVM-libc's notion of NaN with the
@@ -21,66 +21,66 @@ TEST(X86LongDoubleTest, is_nan) {
   // behavior.
 
   FPBits bits(0.0l);
-  bits.exponent = FPBits::MAX_EXPONENT;
+  bits.set_unbiased_exponent(FPBits::MAX_EXPONENT);
   for (unsigned int i = 0; i < 1000000; ++i) {
     // If exponent has the max value and the implicit bit is 0,
     // then the number is a NaN for all values of mantissa.
-    bits.mantissa = i;
+    bits.set_mantissa(i);
     long double nan = bits;
-    ASSERT_NE(isnan(nan), 0);
+    ASSERT_NE(static_cast<int>(isnan(nan)), 0);
     ASSERT_TRUE(bits.is_nan());
   }
 
-  bits.implicitBit = 1;
+  bits.set_implicit_bit(1);
   for (unsigned int i = 1; i < 1000000; ++i) {
     // If exponent has the max value and the implicit bit is 1,
     // then the number is a NaN for all non-zero values of mantissa.
     // Note the initial value of |i| of 1 to avoid a zero mantissa.
-    bits.mantissa = i;
+    bits.set_mantissa(i);
     long double nan = bits;
-    ASSERT_NE(isnan(nan), 0);
+    ASSERT_NE(static_cast<int>(isnan(nan)), 0);
     ASSERT_TRUE(bits.is_nan());
   }
 
-  bits.exponent = 1;
-  bits.implicitBit = 0;
+  bits.set_unbiased_exponent(1);
+  bits.set_implicit_bit(0);
   for (unsigned int i = 0; i < 1000000; ++i) {
     // If exponent is non-zero and also not max, and the implicit bit is 0,
     // then the number is a NaN for all values of mantissa.
-    bits.mantissa = i;
+    bits.set_mantissa(i);
     long double nan = bits;
-    ASSERT_NE(isnan(nan), 0);
+    ASSERT_NE(static_cast<int>(isnan(nan)), 0);
     ASSERT_TRUE(bits.is_nan());
   }
 
-  bits.exponent = 1;
-  bits.implicitBit = 1;
+  bits.set_unbiased_exponent(1);
+  bits.set_implicit_bit(1);
   for (unsigned int i = 0; i < 1000000; ++i) {
     // If exponent is non-zero and also not max, and the implicit bit is 1,
     // then the number is normal value for all values of mantissa.
-    bits.mantissa = i;
+    bits.set_mantissa(i);
     long double valid = bits;
-    ASSERT_EQ(isnan(valid), 0);
+    ASSERT_EQ(static_cast<int>(isnan(valid)), 0);
     ASSERT_FALSE(bits.is_nan());
   }
 
-  bits.exponent = 0;
-  bits.implicitBit = 1;
+  bits.set_unbiased_exponent(0);
+  bits.set_implicit_bit(1);
   for (unsigned int i = 0; i < 1000000; ++i) {
     // If exponent is zero, then the number is a valid but denormal value.
-    bits.mantissa = i;
+    bits.set_mantissa(i);
     long double valid = bits;
-    ASSERT_EQ(isnan(valid), 0);
+    ASSERT_EQ(static_cast<int>(isnan(valid)), 0);
     ASSERT_FALSE(bits.is_nan());
   }
 
-  bits.exponent = 0;
-  bits.implicitBit = 0;
+  bits.set_unbiased_exponent(0);
+  bits.set_implicit_bit(0);
   for (unsigned int i = 0; i < 1000000; ++i) {
     // If exponent is zero, then the number is a valid but denormal value.
-    bits.mantissa = i;
+    bits.set_mantissa(i);
     long double valid = bits;
-    ASSERT_EQ(isnan(valid), 0);
+    ASSERT_EQ(static_cast<int>(isnan(valid)), 0);
     ASSERT_FALSE(bits.is_nan());
   }
 }
diff --git a/libc/test/utils/UnitTest/CMakeLists.txt b/libc/test/utils/UnitTest/CMakeLists.txt
index d6031895e4153..6f61e0ffefb00 100644
--- a/libc/test/utils/UnitTest/CMakeLists.txt
+++ b/libc/test/utils/UnitTest/CMakeLists.txt
@@ -1,4 +1,8 @@
-add_libc_testsuite(libc_unittest_tests)
+if(LIBC_TARGET_ARCHITECTURE_IS_GPU)
+  return()
+endif()
+
+add_custom_target(libc_unittest_tests)
 
 add_libc_unittest(
   testfilter_test
diff --git a/libc/utils/gpu/loader/CMakeLists.txt b/libc/utils/gpu/loader/CMakeLists.txt
index 3f63ef0bc90e4..689cf086b4763 100644
--- a/libc/utils/gpu/loader/CMakeLists.txt
+++ b/libc/utils/gpu/loader/CMakeLists.txt
@@ -12,7 +12,9 @@ else()
 endif()
 
 find_package(CUDAToolkit QUIET)
-if(CUDAToolkit_FOUND)
+# The CUDA loader requires LLVM to traverse the ELF image for symbols.
+find_package(LLVM QUIET)
+if(CUDAToolkit_FOUND AND LLVM_FOUND)
   add_subdirectory(nvptx)
 else()
   message(STATUS "Skipping CUDA loader for gpu target, no CUDA was detected")
diff --git a/libc/utils/gpu/loader/Loader.h b/libc/utils/gpu/loader/Loader.h
index 9c6413ee45d8d..2f55b3ac8fc47 100644
--- a/libc/utils/gpu/loader/Loader.h
+++ b/libc/utils/gpu/loader/Loader.h
@@ -23,12 +23,40 @@ struct LaunchParameters {
   uint32_t num_blocks_z;
 };
 
+/// The arguments to the '_begin' kernel.
+struct begin_args_t {
+  int argc;
+  void *argv;
+  void *envp;
+  void *inbox;
+  void *outbox;
+  void *buffer;
+};
+
+/// The arguments to the '_start' kernel.
+struct start_args_t {
+  int argc;
+  void *argv;
+  void *envp;
+  void *ret;
+};
+
+/// The arguments to the '_end' kernel.
+struct end_args_t {
+  int argc;
+};
+
 /// Generic interface to load the \p image and launch execution of the _start
 /// kernel on the target device. Copies \p argc and \p argv to the device.
 /// Returns the final value of the `main` function on the device.
 int load(int argc, char **argv, char **evnp, void *image, size_t size,
          const LaunchParameters &params);
 
+/// Return \p V aligned "upwards" according to \p Align.
+template <typename V, typename A> inline V align_up(V val, A align) {
+  return ((val + V(align) - 1) / V(align)) * V(align);
+}
+
 /// Copy the system's argument vector to GPU memory allocated using \p alloc.
 template <typename Allocator>
 void *copy_argument_vector(int argc, char **argv, Allocator alloc) {
diff --git a/libc/utils/gpu/loader/Server.h b/libc/utils/gpu/loader/Server.h
index af432fcfe1fb7..6ffb32955c895 100644
--- a/libc/utils/gpu/loader/Server.h
+++ b/libc/utils/gpu/loader/Server.h
@@ -30,10 +30,19 @@ void handle_server() {
 
   switch (port->get_opcode()) {
   case __llvm_libc::rpc::Opcode::PRINT_TO_STDERR: {
-    void *str = nullptr;
-    port->recv_n([&](uint64_t size) { return str = malloc(size); });
-    fputs(reinterpret_cast<char *>(str), stderr);
-    free(str);
+    uint64_t str_size[__llvm_libc::rpc::MAX_LANE_SIZE] = {0};
+    char *strs[__llvm_libc::rpc::MAX_LANE_SIZE] = {nullptr};
+    port->recv_n([&](uint64_t size, uint32_t id) {
+      str_size[id] = size;
+      strs[id] = new char[size];
+      return strs[id];
+    });
+    for (uint64_t i = 0; i < __llvm_libc::rpc::MAX_LANE_SIZE; ++i) {
+      if (strs[i]) {
+        fwrite(strs[i], str_size[i], 1, stderr);
+        delete[] strs[i];
+      }
+    }
     break;
   }
   case __llvm_libc::rpc::Opcode::EXIT: {
@@ -49,8 +58,7 @@ void handle_server() {
     break;
   }
   default:
-    port->recv([](__llvm_libc::rpc::Buffer *) { /* no-op */ });
-    return;
+    port->recv([](__llvm_libc::rpc::Buffer *buffer) {});
   }
   port->close();
 }
diff --git a/libc/utils/gpu/loader/amdgpu/Loader.cpp b/libc/utils/gpu/loader/amdgpu/Loader.cpp
index af5f00878e656..ee12d6d63ffb8 100644
--- a/libc/utils/gpu/loader/amdgpu/Loader.cpp
+++ b/libc/utils/gpu/loader/amdgpu/Loader.cpp
@@ -24,20 +24,6 @@
 #include <cstring>
 #include <utility>
 
-/// The name of the kernel we will launch. All AMDHSA kernels end with '.kd'.
-constexpr const char *KERNEL_START = "_start.kd";
-
-/// The arguments to the '_start' kernel.
-struct kernel_args_t {
-  int argc;
-  void *argv;
-  void *envp;
-  void *ret;
-  void *inbox;
-  void *outbox;
-  void *buffer;
-};
-
 /// Print the error code and exit if \p code indicates an error.
 static void handle_error(hsa_status_t code) {
   if (code == HSA_STATUS_SUCCESS || code == HSA_STATUS_INFO_BREAK)
@@ -145,6 +131,105 @@ hsa_status_t get_agent_memory_pool(hsa_agent_t agent,
   return iterate_agent_memory_pools(agent, cb);
 }
 
+template <typename args_t>
+hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable,
+                           hsa_amd_memory_pool_t kernargs_pool,
+                           hsa_queue_t *queue, const LaunchParameters &params,
+                           const char *kernel_name, args_t kernel_args) {
+  // Look up the '_start' kernel in the loaded executable.
+  hsa_executable_symbol_t symbol;
+  if (hsa_status_t err = hsa_executable_get_symbol_by_name(
+          executable, kernel_name, &dev_agent, &symbol))
+    return err;
+
+  // Retrieve different properties of the kernel symbol used for launch.
+  uint64_t kernel;
+  uint32_t args_size;
+  uint32_t group_size;
+  uint32_t private_size;
+
+  std::pair<hsa_executable_symbol_info_t, void *> symbol_infos[] = {
+      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &kernel},
+      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_KERNARG_SEGMENT_SIZE, &args_size},
+      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_GROUP_SEGMENT_SIZE, &group_size},
+      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_PRIVATE_SEGMENT_SIZE, &private_size}};
+
+  for (auto &[info, value] : symbol_infos)
+    if (hsa_status_t err = hsa_executable_symbol_get_info(symbol, info, value))
+      return err;
+
+  // Allocate space for the kernel arguments on the host and allow the GPU agent
+  // to access it.
+  void *args;
+  if (hsa_status_t err = hsa_amd_memory_pool_allocate(kernargs_pool, args_size,
+                                                      /*flags=*/0, &args))
+    handle_error(err);
+  hsa_amd_agents_allow_access(1, &dev_agent, nullptr, args);
+
+  // Initialie all the arguments (explicit and implicit) to zero, then set the
+  // explicit arguments to the values created above.
+  std::memset(args, 0, args_size);
+  std::memcpy(args, &kernel_args, sizeof(args_t));
+
+  // Obtain a packet from the queue.
+  uint64_t packet_id = hsa_queue_add_write_index_relaxed(queue, 1);
+  while (packet_id - hsa_queue_load_read_index_scacquire(queue) >= queue->size)
+    ;
+
+  const uint32_t mask = queue->size - 1;
+  hsa_kernel_dispatch_packet_t *packet =
+      static_cast<hsa_kernel_dispatch_packet_t *>(queue->base_address) +
+      (packet_id & mask);
+
+  // Set up the packet for exeuction on the device. We currently only launch
+  // with one thread on the device, forcing the rest of the wavefront to be
+  // masked off.
+  std::memset(packet, 0, sizeof(hsa_kernel_dispatch_packet_t));
+  packet->setup = (1 + (params.num_blocks_y * params.num_threads_y != 1) +
+                   (params.num_blocks_z * params.num_threads_z != 1))
+                  << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS;
+  packet->workgroup_size_x = params.num_threads_x;
+  packet->workgroup_size_y = params.num_threads_y;
+  packet->workgroup_size_z = params.num_threads_z;
+  packet->grid_size_x = params.num_blocks_x * params.num_threads_x;
+  packet->grid_size_y = params.num_blocks_y * params.num_threads_y;
+  packet->grid_size_z = params.num_blocks_z * params.num_threads_z;
+  packet->private_segment_size = private_size;
+  packet->group_segment_size = group_size;
+  packet->kernel_object = kernel;
+  packet->kernarg_address = args;
+
+  // Create a signal to indicate when this packet has been completed.
+  if (hsa_status_t err =
+          hsa_signal_create(1, 0, nullptr, &packet->completion_signal))
+    handle_error(err);
+
+  // Initialize the packet header and set the doorbell signal to begin execution
+  // by the HSA runtime.
+  uint16_t setup = packet->setup;
+  uint16_t header =
+      (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) |
+      (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) |
+      (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE);
+  __atomic_store_n(&packet->header, header | (setup << 16), __ATOMIC_RELEASE);
+  hsa_signal_store_relaxed(queue->doorbell_signal, packet_id);
+
+  // Wait until the kernel has completed execution on the device. Periodically
+  // check the RPC client for work to be performed on the server.
+  while (hsa_signal_wait_scacquire(
+             packet->completion_signal, HSA_SIGNAL_CONDITION_EQ, 0,
+             /*timeout_hint=*/1024, HSA_WAIT_STATE_ACTIVE) != 0)
+    handle_server();
+
+  // Destroy the resources acquired to launch the kernel and return.
+  if (hsa_status_t err = hsa_amd_memory_pool_free(args))
+    handle_error(err);
+  if (hsa_status_t err = hsa_signal_destroy(packet->completion_signal))
+    handle_error(err);
+
+  return HSA_STATUS_SUCCESS;
+}
+
 int load(int argc, char **argv, char **envp, void *image, size_t size,
          const LaunchParameters &params) {
   // Initialize the HSA runtime used to communicate with the device.
@@ -169,18 +254,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   if (hsa_status_t err = get_agent<HSA_DEVICE_TYPE_CPU>(&host_agent))
     handle_error(err);
 
-  // Obtain a queue with the minimum (power of two) size, used to send commands
-  // to the HSA runtime and launch execution on the device.
-  uint64_t queue_size;
-  if (hsa_status_t err = hsa_agent_get_info(
-          dev_agent, HSA_AGENT_INFO_QUEUE_MIN_SIZE, &queue_size))
-    handle_error(err);
-  hsa_queue_t *queue = nullptr;
-  if (hsa_status_t err =
-          hsa_queue_create(dev_agent, queue_size, HSA_QUEUE_TYPE_SINGLE,
-                           nullptr, nullptr, UINT32_MAX, UINT32_MAX, &queue))
-    handle_error(err);
-
   // Load the code object's ISA information and executable data segments.
   hsa_code_object_t object;
   if (hsa_status_t err = hsa_code_object_deserialize(image, size, "", &object))
@@ -228,36 +301,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
               dev_agent, &coarsegrained_pool))
     handle_error(err);
 
-  // Look up the '_start' kernel in the loaded executable.
-  hsa_executable_symbol_t symbol;
-  if (hsa_status_t err = hsa_executable_get_symbol_by_name(
-          executable, KERNEL_START, &dev_agent, &symbol))
-    handle_error(err);
-
-  // Retrieve different properties of the kernel symbol used for launch.
-  uint64_t kernel;
-  uint32_t args_size;
-  uint32_t group_size;
-  uint32_t private_size;
-
-  std::pair<hsa_executable_symbol_info_t, void *> symbol_infos[] = {
-      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &kernel},
-      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_KERNARG_SEGMENT_SIZE, &args_size},
-      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_GROUP_SEGMENT_SIZE, &group_size},
-      {HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_PRIVATE_SEGMENT_SIZE, &private_size}};
-
-  for (auto &[info, value] : symbol_infos)
-    if (hsa_status_t err = hsa_executable_symbol_get_info(symbol, info, value))
-      handle_error(err);
-
-  // Allocate space for the kernel arguments on the host and allow the GPU agent
-  // to access it.
-  void *args;
-  if (hsa_status_t err = hsa_amd_memory_pool_allocate(kernargs_pool, args_size,
-                                                      /*flags=*/0, &args))
-    handle_error(err);
-  hsa_amd_agents_allow_access(1, &dev_agent, nullptr, args);
-
   // Allocate fine-grained memory on the host to hold the pointer array for the
   // copied argv and allow the GPU agent to access it.
   auto allocator = [&](uint64_t size) -> void * {
@@ -287,6 +330,10 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   hsa_amd_memory_fill(dev_ret, 0, sizeof(int));
 
   // Allocate finegrained memory for the RPC server and client to share.
+  uint32_t wavefront_size = 0;
+  if (hsa_status_t err = hsa_agent_get_info(
+          dev_agent, HSA_AGENT_INFO_WAVEFRONT_SIZE, &wavefront_size))
+    handle_error(err);
   void *server_inbox;
   void *server_outbox;
   void *buffer;
@@ -299,76 +346,43 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
           /*flags=*/0, &server_outbox))
     handle_error(err);
   if (hsa_status_t err = hsa_amd_memory_pool_allocate(
-          finegrained_pool, sizeof(__llvm_libc::rpc::Buffer),
+          finegrained_pool,
+          align_up(sizeof(__llvm_libc::rpc::Header) +
+                       (wavefront_size * sizeof(__llvm_libc::rpc::Buffer)),
+                   alignof(__llvm_libc::rpc::Packet)),
           /*flags=*/0, &buffer))
     handle_error(err);
   hsa_amd_agents_allow_access(1, &dev_agent, nullptr, server_inbox);
   hsa_amd_agents_allow_access(1, &dev_agent, nullptr, server_outbox);
   hsa_amd_agents_allow_access(1, &dev_agent, nullptr, buffer);
 
-  // Initialie all the arguments (explicit and implicit) to zero, then set the
-  // explicit arguments to the values created above.
-  std::memset(args, 0, args_size);
-  kernel_args_t *kernel_args = reinterpret_cast<kernel_args_t *>(args);
-  kernel_args->argc = argc;
-  kernel_args->argv = dev_argv;
-  kernel_args->envp = dev_envp;
-  kernel_args->ret = dev_ret;
-  kernel_args->inbox = server_outbox;
-  kernel_args->outbox = server_inbox;
-  kernel_args->buffer = buffer;
-
-  // Obtain a packet from the queue.
-  uint64_t packet_id = hsa_queue_add_write_index_relaxed(queue, 1);
-  while (packet_id - hsa_queue_load_read_index_scacquire(queue) >= queue_size)
-    ;
-
-  const uint32_t mask = queue_size - 1;
-  hsa_kernel_dispatch_packet_t *packet =
-      (hsa_kernel_dispatch_packet_t *)queue->base_address + (packet_id & mask);
-
-  // Set up the packet for exeuction on the device. We currently only launch
-  // with one thread on the device, forcing the rest of the wavefront to be
-  // masked off.
-  std::memset(packet, 0, sizeof(hsa_kernel_dispatch_packet_t));
-  packet->setup = (1 + (params.num_blocks_y * params.num_threads_y != 1) +
-                   (params.num_blocks_z * params.num_threads_z != 1))
-                  << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS;
-  packet->workgroup_size_x = params.num_threads_x;
-  packet->workgroup_size_y = params.num_threads_y;
-  packet->workgroup_size_z = params.num_threads_z;
-  packet->grid_size_x = params.num_blocks_x * params.num_threads_x;
-  packet->grid_size_y = params.num_blocks_y * params.num_threads_y;
-  packet->grid_size_z = params.num_blocks_z * params.num_threads_z;
-  packet->private_segment_size = private_size;
-  packet->group_segment_size = group_size;
-  packet->kernel_object = kernel;
-  packet->kernarg_address = args;
+  // Initialize the RPC server's buffer for host-device communication.
+  server.reset(wavefront_size, &lock, server_inbox, server_outbox, buffer);
 
-  // Create a signal to indicate when this packet has been completed.
+  // Obtain a queue with the minimum (power of two) size, used to send commands
+  // to the HSA runtime and launch execution on the device.
+  uint64_t queue_size;
+  if (hsa_status_t err = hsa_agent_get_info(
+          dev_agent, HSA_AGENT_INFO_QUEUE_MIN_SIZE, &queue_size))
+    handle_error(err);
+  hsa_queue_t *queue = nullptr;
   if (hsa_status_t err =
-          hsa_signal_create(1, 0, nullptr, &packet->completion_signal))
+          hsa_queue_create(dev_agent, queue_size, HSA_QUEUE_TYPE_MULTI, nullptr,
+                           nullptr, UINT32_MAX, UINT32_MAX, &queue))
     handle_error(err);
 
-  // Initialize the RPC server's buffer for host-device communication.
-  server.reset(&lock, server_inbox, server_outbox, buffer);
-
-  // Initialize the packet header and set the doorbell signal to begin execution
-  // by the HSA runtime.
-  uint16_t header =
-      (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) |
-      (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) |
-      (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE);
-  __atomic_store_n(&packet->header, header | (packet->setup << 16),
-                   __ATOMIC_RELEASE);
-  hsa_signal_store_relaxed(queue->doorbell_signal, packet_id);
+  LaunchParameters single_threaded_params = {1, 1, 1, 1, 1, 1};
+  begin_args_t init_args = {argc,          dev_argv,     dev_envp,
+                            server_outbox, server_inbox, buffer};
+  if (hsa_status_t err =
+          launch_kernel(dev_agent, executable, kernargs_pool, queue,
+                        single_threaded_params, "_begin.kd", init_args))
+    handle_error(err);
 
-  // Wait until the kernel has completed execution on the device. Periodically
-  // check the RPC client for work to be performed on the server.
-  while (hsa_signal_wait_scacquire(
-             packet->completion_signal, HSA_SIGNAL_CONDITION_EQ, 0,
-             /*timeout_hint=*/1024, HSA_WAIT_STATE_ACTIVE) != 0)
-    handle_server();
+  start_args_t args = {argc, dev_argv, dev_envp, dev_ret};
+  if (hsa_status_t err = launch_kernel(dev_agent, executable, kernargs_pool,
+                                       queue, params, "_start.kd", args))
+    handle_error(err);
 
   // Create a memory signal and copy the return value back from the device into
   // a new buffer.
@@ -395,9 +409,13 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   // Save the return value and perform basic clean-up.
   int ret = *static_cast<int *>(host_ret);
 
-  // Free the memory allocated for the device.
-  if (hsa_status_t err = hsa_amd_memory_pool_free(args))
+  end_args_t fini_args = {ret};
+  if (hsa_status_t err =
+          launch_kernel(dev_agent, executable, kernargs_pool, queue,
+                        single_threaded_params, "_end.kd", fini_args))
     handle_error(err);
+
+  // Free the memory allocated for the device.
   if (hsa_status_t err = hsa_amd_memory_pool_free(dev_argv))
     handle_error(err);
   if (hsa_status_t err = hsa_amd_memory_pool_free(dev_ret))
@@ -413,10 +431,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
 
   if (hsa_status_t err = hsa_signal_destroy(memory_signal))
     handle_error(err);
-
-  if (hsa_status_t err = hsa_signal_destroy(packet->completion_signal))
-    handle_error(err);
-
   if (hsa_status_t err = hsa_queue_destroy(queue))
     handle_error(err);
 
diff --git a/libc/utils/gpu/loader/nvptx/CMakeLists.txt b/libc/utils/gpu/loader/nvptx/CMakeLists.txt
index f88914383a98e..9e85357920678 100644
--- a/libc/utils/gpu/loader/nvptx/CMakeLists.txt
+++ b/libc/utils/gpu/loader/nvptx/CMakeLists.txt
@@ -1,8 +1,14 @@
 add_executable(nvptx_loader Loader.cpp)
 add_dependencies(nvptx_loader libc.src.__support.RPC.rpc)
 
+if(NOT LLVM_ENABLE_RTTI)
+  target_compile_options(nvptx_loader PRIVATE -fno-rtti)
+endif()
+target_include_directories(nvptx_loader PRIVATE ${LLVM_INCLUDE_DIRS})
 target_link_libraries(nvptx_loader
   PRIVATE
   gpu_loader
   CUDA::cuda_driver
+  LLVMObject
+  LLVMSupport
 )
diff --git a/libc/utils/gpu/loader/nvptx/Loader.cpp b/libc/utils/gpu/loader/nvptx/Loader.cpp
index eb8db7f48572e..ca18da939f4cb 100644
--- a/libc/utils/gpu/loader/nvptx/Loader.cpp
+++ b/libc/utils/gpu/loader/nvptx/Loader.cpp
@@ -17,21 +17,18 @@
 #include "Server.h"
 
 #include "cuda.h"
+
+#include "llvm/Object/ELF.h"
+#include "llvm/Object/ELFObjectFile.h"
+
 #include <cstddef>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <vector>
 
-/// The arguments to the '_start' kernel.
-struct kernel_args_t {
-  int argc;
-  void *argv;
-  void *envp;
-  void *ret;
-  void *inbox;
-  void *outbox;
-  void *buffer;
-};
+using namespace llvm;
+using namespace object;
 
 static void handle_error(CUresult err) {
   if (err == CUDA_SUCCESS)
@@ -51,11 +48,152 @@ static void handle_error(const char *msg) {
   exit(EXIT_FAILURE);
 }
 
+// Gets the names of all the globals that contain functions to initialize or
+// deinitialize. We need to do this manually because the NVPTX toolchain does
+// not contain the necessary binary manipulation tools.
+template <typename Alloc>
+Expected<void *> get_ctor_dtor_array(const void *image, const size_t size,
+                                     Alloc allocator, CUmodule binary) {
+  auto mem_buffer = MemoryBuffer::getMemBuffer(
+      StringRef(reinterpret_cast<const char *>(image), size), "image",
+      /*RequiresNullTerminator=*/false);
+  Expected<ELF64LEObjectFile> elf_or_err =
+      ELF64LEObjectFile::create(*mem_buffer);
+  if (!elf_or_err)
+    handle_error(toString(elf_or_err.takeError()).c_str());
+
+  std::vector<std::pair<const char *, uint16_t>> ctors;
+  std::vector<std::pair<const char *, uint16_t>> dtors;
+  // CUDA has no way to iterate over all the symbols so we need to inspect the
+  // ELF directly using the LLVM libraries.
+  for (const auto &symbol : elf_or_err->symbols()) {
+    auto name_or_err = symbol.getName();
+    if (!name_or_err)
+      handle_error(toString(name_or_err.takeError()).c_str());
+
+    // Search for all symbols that contain a constructor or destructor.
+    if (!name_or_err->starts_with("__init_array_object_") &&
+        !name_or_err->starts_with("__fini_array_object_"))
+      continue;
+
+    uint16_t priority;
+    if (name_or_err->rsplit('_').second.getAsInteger(10, priority))
+      handle_error("Invalid priority for constructor or destructor");
+
+    if (name_or_err->starts_with("__init"))
+      ctors.emplace_back(std::make_pair(name_or_err->data(), priority));
+    else
+      dtors.emplace_back(std::make_pair(name_or_err->data(), priority));
+  }
+  // Lower priority constructors are run before higher ones. The reverse is true
+  // for destructors.
+  llvm::sort(ctors, [](auto x, auto y) { return x.second < y.second; });
+  llvm::sort(dtors, [](auto x, auto y) { return x.second < y.second; });
+  llvm::reverse(dtors);
+
+  // Allocate host pinned memory to make these arrays visible to the GPU.
+  CUdeviceptr *dev_memory = reinterpret_cast<CUdeviceptr *>(allocator(
+      ctors.size() * sizeof(CUdeviceptr) + dtors.size() * sizeof(CUdeviceptr)));
+  uint64_t global_size = 0;
+
+  // Get the address of the global and then store the address of the constructor
+  // function to call in the constructor array.
+  CUdeviceptr *dev_ctors_start = dev_memory;
+  CUdeviceptr *dev_ctors_end = dev_ctors_start + ctors.size();
+  for (uint64_t i = 0; i < ctors.size(); ++i) {
+    CUdeviceptr dev_ptr;
+    if (CUresult err =
+            cuModuleGetGlobal(&dev_ptr, &global_size, binary, ctors[i].first))
+      handle_error(err);
+    if (CUresult err =
+            cuMemcpyDtoH(&dev_ctors_start[i], dev_ptr, sizeof(uintptr_t)))
+      handle_error(err);
+  }
+
+  // Get the address of the global and then store the address of the destructor
+  // function to call in the destructor array.
+  CUdeviceptr *dev_dtors_start = dev_ctors_end;
+  CUdeviceptr *dev_dtors_end = dev_dtors_start + dtors.size();
+  for (uint64_t i = 0; i < dtors.size(); ++i) {
+    CUdeviceptr dev_ptr;
+    if (CUresult err =
+            cuModuleGetGlobal(&dev_ptr, &global_size, binary, dtors[i].first))
+      handle_error(err);
+    if (CUresult err =
+            cuMemcpyDtoH(&dev_dtors_start[i], dev_ptr, sizeof(uintptr_t)))
+      handle_error(err);
+  }
+
+  // Obtain the address of the pointers the startup implementation uses to
+  // iterate the constructors and destructors.
+  CUdeviceptr init_start;
+  if (CUresult err = cuModuleGetGlobal(&init_start, &global_size, binary,
+                                       "__init_array_start"))
+    handle_error(err);
+  CUdeviceptr init_end;
+  if (CUresult err = cuModuleGetGlobal(&init_end, &global_size, binary,
+                                       "__init_array_end"))
+    handle_error(err);
+  CUdeviceptr fini_start;
+  if (CUresult err = cuModuleGetGlobal(&fini_start, &global_size, binary,
+                                       "__fini_array_start"))
+    handle_error(err);
+  CUdeviceptr fini_end;
+  if (CUresult err = cuModuleGetGlobal(&fini_end, &global_size, binary,
+                                       "__fini_array_end"))
+    handle_error(err);
+
+  // Copy the pointers to the newly written array to the symbols so the startup
+  // implementation can iterate them.
+  if (CUresult err =
+          cuMemcpyHtoD(init_start, &dev_ctors_start, sizeof(uintptr_t)))
+    handle_error(err);
+  if (CUresult err = cuMemcpyHtoD(init_end, &dev_ctors_end, sizeof(uintptr_t)))
+    handle_error(err);
+  if (CUresult err =
+          cuMemcpyHtoD(fini_start, &dev_dtors_start, sizeof(uintptr_t)))
+    handle_error(err);
+  if (CUresult err = cuMemcpyHtoD(fini_end, &dev_dtors_end, sizeof(uintptr_t)))
+    handle_error(err);
+
+  return dev_memory;
+}
+
+template <typename args_t>
+CUresult launch_kernel(CUmodule binary, CUstream stream,
+                       const LaunchParameters &params, const char *kernel_name,
+                       args_t kernel_args) {
+  // look up the '_start' kernel in the loaded module.
+  CUfunction function;
+  if (CUresult err = cuModuleGetFunction(&function, binary, kernel_name))
+    handle_error(err);
+
+  // Set up the arguments to the '_start' kernel on the GPU.
+  uint64_t args_size = sizeof(args_t);
+  void *args_config[] = {CU_LAUNCH_PARAM_BUFFER_POINTER, &kernel_args,
+                         CU_LAUNCH_PARAM_BUFFER_SIZE, &args_size,
+                         CU_LAUNCH_PARAM_END};
+
+  // Call the kernel with the given arguments.
+  if (CUresult err = cuLaunchKernel(
+          function, params.num_blocks_x, params.num_blocks_y,
+          params.num_blocks_z, params.num_threads_x, params.num_threads_y,
+          params.num_threads_z, 0, stream, nullptr, args_config))
+    handle_error(err);
+
+  // Wait until the kernel has completed execution on the device. Periodically
+  // check the RPC client for work to be performed on the server.
+  while (cuStreamQuery(stream) == CUDA_ERROR_NOT_READY)
+    handle_server();
+
+  return CUDA_SUCCESS;
+}
+
 int load(int argc, char **argv, char **envp, void *image, size_t size,
          const LaunchParameters &params) {
+
   if (CUresult err = cuInit(0))
     handle_error(err);
-
   // Obtain the first device found on the system.
   CUdevice device;
   if (CUresult err = cuDeviceGet(&device, 0))
@@ -78,11 +216,6 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   if (CUresult err = cuModuleLoadDataEx(&binary, image, 0, nullptr, nullptr))
     handle_error(err);
 
-  // look up the '_start' kernel in the loaded module.
-  CUfunction function;
-  if (CUresult err = cuModuleGetFunction(&function, binary, "_start"))
-    handle_error(err);
-
   // Allocate pinned memory on the host to hold the pointer array for the
   // copied argv and allow the GPU device to access it.
   auto allocator = [&](uint64_t size) -> void * {
@@ -91,6 +224,11 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
       handle_error(err);
     return dev_ptr;
   };
+
+  auto memory_or_err = get_ctor_dtor_array(image, size, allocator, binary);
+  if (!memory_or_err)
+    handle_error(toString(memory_or_err.takeError()).c_str());
+
   void *dev_argv = copy_argument_vector(argc, argv, allocator);
   if (!dev_argv)
     handle_error("Failed to allocate device argv");
@@ -108,41 +246,31 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   if (CUresult err = cuMemsetD32(dev_ret, 0, 1))
     handle_error(err);
 
+  uint32_t warp_size = 32;
   void *server_inbox = allocator(sizeof(__llvm_libc::cpp::Atomic<int>));
   void *server_outbox = allocator(sizeof(__llvm_libc::cpp::Atomic<int>));
-  void *buffer = allocator(sizeof(__llvm_libc::rpc::Buffer));
+  void *buffer =
+      allocator(align_up(sizeof(__llvm_libc::rpc::Header) +
+                             (warp_size * sizeof(__llvm_libc::rpc::Buffer)),
+                         alignof(__llvm_libc::rpc::Packet)));
   if (!server_inbox || !server_outbox || !buffer)
     handle_error("Failed to allocate memory the RPC client / server.");
 
-  // Set up the arguments to the '_start' kernel on the GPU.
-  uint64_t args_size = sizeof(kernel_args_t);
-  kernel_args_t args;
-  std::memset(&args, 0, args_size);
-  args.argc = argc;
-  args.argv = dev_argv;
-  args.envp = dev_envp;
-  args.ret = reinterpret_cast<void *>(dev_ret);
-  args.inbox = server_outbox;
-  args.outbox = server_inbox;
-  args.buffer = buffer;
-  void *args_config[] = {CU_LAUNCH_PARAM_BUFFER_POINTER, &args,
-                         CU_LAUNCH_PARAM_BUFFER_SIZE, &args_size,
-                         CU_LAUNCH_PARAM_END};
-
   // Initialize the RPC server's buffer for host-device communication.
-  server.reset(&lock, server_inbox, server_outbox, buffer);
+  server.reset(warp_size, &lock, server_inbox, server_outbox, buffer);
 
-  // Call the kernel with the given arguments.
-  if (CUresult err = cuLaunchKernel(
-          function, params.num_blocks_x, params.num_blocks_y,
-          params.num_blocks_z, params.num_threads_x, params.num_threads_y,
-          params.num_threads_z, 0, stream, nullptr, args_config))
+  LaunchParameters single_threaded_params = {1, 1, 1, 1, 1, 1};
+  // Call the kernel to
+  begin_args_t init_args = {argc,          dev_argv,     dev_envp,
+                            server_outbox, server_inbox, buffer};
+  if (CUresult err = launch_kernel(binary, stream, single_threaded_params,
+                                   "_begin", init_args))
     handle_error(err);
 
-  // Wait until the kernel has completed execution on the device. Periodically
-  // check the RPC client for work to be performed on the server.
-  while (cuStreamQuery(stream) == CUDA_ERROR_NOT_READY)
-    handle_server();
+  start_args_t args = {argc, dev_argv, dev_envp,
+                       reinterpret_cast<void *>(dev_ret)};
+  if (CUresult err = launch_kernel(binary, stream, params, "_start", args))
+    handle_error(err);
 
   // Copy the return value back from the kernel and wait.
   int host_ret = 0;
@@ -152,7 +280,14 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   if (CUresult err = cuStreamSynchronize(stream))
     handle_error(err);
 
+  end_args_t fini_args = {host_ret};
+  if (CUresult err = launch_kernel(binary, stream, single_threaded_params,
+                                   "_end", fini_args))
+    handle_error(err);
+
   // Free the memory allocated for the device.
+  if (CUresult err = cuMemFreeHost(*memory_or_err))
+    handle_error(err);
   if (CUresult err = cuMemFree(dev_ret))
     handle_error(err);
   if (CUresult err = cuMemFreeHost(dev_argv))
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 2b6f9987c6750..cf572af74bd13 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -68,6 +68,7 @@ endif()
 option(LIBCXX_ENABLE_FILESYSTEM "Build filesystem as part of the main libc++ library"
     ${ENABLE_FILESYSTEM_DEFAULT})
 option(LIBCXX_INCLUDE_TESTS "Build the libc++ tests." ${LLVM_INCLUDE_TESTS})
+option(LIBCXX_ENABLE_PARALLEL_ALGORITHMS "Enable the parallel algorithms library. This requires the PSTL to be available." OFF)
 option(LIBCXX_ENABLE_DEBUG_MODE
   "Whether to build libc++ with the debug mode enabled.
    By default, this is turned off. Turning it on results in a different ABI (additional
diff --git a/libcxx/cmake/caches/With-pstl.cmake b/libcxx/cmake/caches/With-pstl.cmake
new file mode 100644
index 0000000000000..82dd90f968cb6
--- /dev/null
+++ b/libcxx/cmake/caches/With-pstl.cmake
@@ -0,0 +1,4 @@
+# TODO: Remove this cache file once the PSTL is under `-fexperimental-library`
+set(LIBCXX_TEST_PARAMS "std=c++17" CACHE STRING "")
+set(LIBCXXABI_TEST_PARAMS "${LIBCXX_TEST_PARAMS}" CACHE STRING "")
+set(LIBCXX_ENABLE_PARALLEL_ALGORITHMS ON CACHE BOOL "")
diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
index 8c0b3a1ddfd2d..90b5c8b09b815 100644
--- a/libcxx/docs/Contributing.rst
+++ b/libcxx/docs/Contributing.rst
@@ -162,8 +162,7 @@ Below is a short description of the most interesting CI builds [#]_:
 * ``ARM`` tests libc++ on various Linux ARM platforms.
 * ``AIX`` tests libc++ on AIX.
 
-.. [#] Not all all steps are listed: steps are added and removed when the need
-   arises.
+.. [#] Not all steps are listed: steps are added and removed when the need arises.
 .. [#] Clang modules are not the same as C++20's modules.
 
 Infrastructure
@@ -194,7 +193,7 @@ your system.
 run-buildbot
 ~~~~~~~~~~~~
 
-Contains the buld script executed on Buildkite. This script can be executed
+Contains the build script executed on Buildkite. This script can be executed
 locally or inside ``run-buildbot-container``. The script must be called with
 the target to test. For example, ``run-buildbot generic-cxx20`` will build
 libc++ and test it using C++20.
diff --git a/libcxx/docs/DesignDocs/ABIVersioning.rst b/libcxx/docs/DesignDocs/ABIVersioning.rst
index 3b82f3cc60a44..ad7218687d55f 100644
--- a/libcxx/docs/DesignDocs/ABIVersioning.rst
+++ b/libcxx/docs/DesignDocs/ABIVersioning.rst
@@ -22,3 +22,11 @@ Internally, each ABI-changing feature is placed under its own C++ macro,
 ``_LIBCPP_ABI_VERSION``, which is controlled by the ``LIBCXX_ABI_VERSION`` set
 at build time. Libc++ does not intend users to interact with these C++ macros
 directly.
+
+-----------------
+MSVC environments
+-----------------
+
+The exception to this is MSVC environments. Libc++ does not currently have users
+that require a stable ABI in MSVC environments, so MSVC-only changes may be
+applied unconditionally.
diff --git a/libcxx/docs/DesignDocs/PSTLIntegration.rst b/libcxx/docs/DesignDocs/PSTLIntegration.rst
new file mode 100644
index 0000000000000..e2b9e89f050eb
--- /dev/null
+++ b/libcxx/docs/DesignDocs/PSTLIntegration.rst
@@ -0,0 +1,22 @@
+================
+PSTL integration
+================
+
+The PSTL (or Parallel STL) project is quite complex in its current form and does not provide everything that libc++
+requires, for example ``_LIBCPP_HIDE_FROM_ABI`` or similar annotations and including granularized headers. Furthermore,
+the PSTL provides various layers of indirection that make sense in a generic implementation of the parallel algorithms,
+but are unnecessarily complex in the context of a single standard library implementation. Because of these drawbacks, we
+decided to adopt a modified PSTL in libc++. Specifically, the goals of the modified PSTL are
+
+- No ``<__pstl_algorithm>`` and similar glue headers -- instead, the implementation files are included directly in
+  ``<algorithm>`` and friends.
+- No ``<pstl/internal/algorithm_impl.h>`` and ``<pstl/internal/algorithm_fwd.h>`` headers and friends -- these contain
+  the implementation and forward declarations for internal functions respectively. The implementation lives inside
+  ``<__algorithm/pstl_any_of.h>`` and friends, and the forward declarations are not needed inside libc++.
+- No ``<pstl/internal/glue_algorithm_defs.h>`` and ``<pstl/internal/glue_algorithm_impl.h>`` headers and friends --
+  these contain the public API. It lives inside ``<__algorithm/pstl_any_of.h>`` and friends instead.
+- The headers implementing backends are kept with as few changes as possible to make it easier to keep the backends in
+  sync with the backends from the original PSTL.
+- The configuration headers ``__pstl_config_site.in`` and ``pstl_config.h`` are removed, and any required configuration
+  is done inside ``__config_site.in`` and ``__config`` respectively.
+- libc++-style tests for the public PSTL API
diff --git a/libcxx/docs/Status/Cxx2bIssues.csv b/libcxx/docs/Status/Cxx2bIssues.csv
index 8fa96fe2cb658..5ed62484f0407 100644
--- a/libcxx/docs/Status/Cxx2bIssues.csv
+++ b/libcxx/docs/Status/Cxx2bIssues.csv
@@ -119,7 +119,7 @@
 `3560 <https://wg21.link/LWG3560>`__,"``ranges::equal`` and ``ranges::is_permutation`` should short-circuit for ``sized_ranges``","October 2021","","","|ranges|"
 `3561 <https://wg21.link/LWG3561>`__,"Issue with internal counter in ``discard_block_engine``","October 2021","",""
 `3563 <https://wg21.link/LWG3563>`__,"``keys_view`` example is broken","October 2021","","","|ranges|"
-`3566 <https://wg21.link/LWG3566>`__,"Constraint recursion for ``operator<=>(optional<T>, U)``","October 2021","","","|spaceship|"
+`3566 <https://wg21.link/LWG3566>`__,"Constraint recursion for ``operator<=>(optional<T>, U)``","October 2021","Complete","17.0","|spaceship|"
 `3567 <https://wg21.link/LWG3567>`__,"Formatting move-only iterators take two","October 2021","|Complete|","16.0","|format| |ranges|"
 `3568 <https://wg21.link/LWG3568>`__,"``basic_istream_view`` needs to initialize ``value_``","October 2021","|Complete|","16.0","|ranges|"
 `3570 <https://wg21.link/LWG3570>`__,"``basic_osyncstream::emit`` should be an unformatted output function","October 2021","",""
@@ -206,7 +206,7 @@
 "`3738 <https://wg21.link/LWG3738>`__","Missing preconditions for ``take_view`` constructor", "November 2022","|Complete|","16.0","|ranges|"
 "`3743 <https://wg21.link/LWG3743>`__","``ranges::to``'s reserve may be ill-formed", "November 2022","","","|ranges|"
 "`3745 <https://wg21.link/LWG3745>`__","``std::atomic_wait`` and its friends lack ``noexcept``", "November 2022","|Complete|","16.0",""
-"`3746 <https://wg21.link/LWG3746>`__","``optional``'s spaceship with ``U`` with a type derived from optional causes infinite constraint meta-recursion", "November 2022","","","|spaceship|"
+"`3746 <https://wg21.link/LWG3746>`__","``optional``'s spaceship with ``U`` with a type derived from optional causes infinite constraint meta-recursion", "November 2022","Complete","17.0","|spaceship|"
 "`3747 <https://wg21.link/LWG3747>`__","``ranges::uninitialized_copy_n``, ``ranges::uninitialized_move_n``, and ``ranges::destroy_n`` should use ``std::move``", "November 2022","","","|ranges|"
 "`3750 <https://wg21.link/LWG3750>`__","Too many papers bump ``__cpp_lib_format``", "November 2022","|Partial| [#note-LWG3750]_","","|format|"
 "`3751 <https://wg21.link/LWG3751>`__","Missing feature macro for ``flat_set``", "November 2022","","","|flat_containers|"
diff --git a/libcxx/docs/Status/FormatPaper.csv b/libcxx/docs/Status/FormatPaper.csv
index 2cd430ff5b2ba..32e4505fb2ea0 100644
--- a/libcxx/docs/Status/FormatPaper.csv
+++ b/libcxx/docs/Status/FormatPaper.csv
@@ -5,8 +5,8 @@ Section,Description,Dependencies,Assignee,Status,First released version
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::utc_time<Duration>``",A ``<chrono>`` implementation,Not assigned,,,
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::tai_time<Duration>``",A ``<chrono>`` implementation,Not assigned,,,
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::gps_time<Duration>``",A ``<chrono>`` implementation,Not assigned,,,
-`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::file_time<Duration>``",,Mark de Wever,|In Progress|,,
-`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local_time<Duration>``",,Mark de Wever,|In Progress|,,
+`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::file_time<Duration>``",,Mark de Wever,|Complete|, Clang 17
+`[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local_time<Duration>``",,Mark de Wever,|Complete|, Clang 17
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::local-time-format-t<Duration>``",A ``<chrono>`` implementation,Not assigned,,,
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::day``",,Mark de Wever,|Complete|, Clang 16
 `[time.syn] <https://wg21.link/time.syn>`_,"Formatter ``chrono::month``",,Mark de Wever,|Complete|, Clang 16
diff --git a/libcxx/docs/Status/SpaceshipProjects.csv b/libcxx/docs/Status/SpaceshipProjects.csv
index e8624cb6a6724..3eecc354b8113 100644
--- a/libcxx/docs/Status/SpaceshipProjects.csv
+++ b/libcxx/docs/Status/SpaceshipProjects.csv
@@ -22,7 +22,7 @@ Section,Description,Dependencies,Assignee,Complete
 "| `[optional.relops] <https://wg21.link/optional.relops>`_
 | `[optional.nullops] <https://wg21.link/optional.nullops>`_
 | `[optional.comp.with.t] <https://wg21.link/optional.comp.with.t>`_","| `optional <https://reviews.llvm.org/D146392>`_
-| `nullopt <https://reviews.llvm.org/D146392>`_",None,Hristo Hristov,|In Progress|
+| `nullopt <https://reviews.llvm.org/D146392>`_",None,Hristo Hristov,|Complete|
 "| `[variant.relops] <https://wg21.link/variant.relops>`_
 | `[variant.monostate.relops] <https://wg21.link/variant.monostate.relops>`_","| `monostate <https://reviews.llvm.org/D131372>`_
 | `variant <https://reviews.llvm.org/D131372>`_",None,Kent Ross,|Complete|
diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst
index ce944b5a6e9d5..a7ab788dbc597 100644
--- a/libcxx/docs/UsingLibcxx.rst
+++ b/libcxx/docs/UsingLibcxx.rst
@@ -523,3 +523,32 @@ Standard. Libc++ retroactively updates the Unicode Standard in older C++
 versions. This allows the library to have better estimates for newly introduced
 Unicode code points, without requiring the user to use the latest C++ version
 in their code base.
+=======
+Turning off ASan annotation in containers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``__asan_annotate_container_with_allocator`` is a customization point to allow users to disable
+`Address Sanitizer annotations for containers <https://github.com/google/sanitizers/wiki/AddressSanitizerContainerOverflow>`_ for specific allocators. This may be necessary for allocators that access allocated memory.
+This customization point exists only when ``_LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS`` Feature Test Macro is defined.
+
+For allocators not running destructors, it is also possible to `bulk-unpoison memory <https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning>`_ instead of disabling annotations altogether.
+
+The struct may be specialized for user-defined allocators. It is a `Cpp17UnaryTypeTrait <http://eel.is/c++draft/type.traits#meta.rqmts>`_ with a base characteristic of ``true_type`` if the container is allowed to use annotations and ``false_type`` otherwise.
+
+The annotations for a ``user_allocator`` can be disabled like this:
+
+.. code-block:: cpp
+
+  #ifdef _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS
+  template <class T>
+  struct std::__asan_annotate_container_with_allocator<user_allocator<T>> : std::false_type {};
+  #endif
+
+Why may I want to turn it off?
+------------------------------
+
+There are a few reasons why you may want to turn off annotations for an allocator.
+Unpoisoning may not be an option, if (for example) you are not maintaining the allocator.
+
+* You are using allocator, which does not call destructor during deallocation.
+* You are aware that memory allocated with an allocator may be accessed, even when unused by container.
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index be89c177db3f0..4af4d9f1704f5 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -120,7 +120,7 @@ Target platform Target architecture       Notes
 macOS 10.9+     i386, x86_64, arm64       Building the shared library itself requires targetting macOS 10.13+
 FreeBSD 12+     i386, x86_64, arm
 Linux           i386, x86_64, arm, arm64  Only glibc-2.24 and later and no other libc is officially supported
-Windows         i386, x86_64              Both MSVC and MinGW style environments
+Windows         i386, x86_64              Both MSVC and MinGW style environments, ABI in MSVC environments is :doc:`unstable <DesignDocs/ABIVersioning>`
 AIX 7.2TL5+     powerpc, powerpc64
 =============== ========================= ============================
 
@@ -179,6 +179,7 @@ Design Documents
    DesignDocs/FileTimeType
    DesignDocs/HeaderRemovalPolicy
    DesignDocs/NoexceptPolicy
+   DesignDocs/PSTLIntegration
    DesignDocs/ThreadingSupportAPI
    DesignDocs/UniquePtrTrivialAbi
    DesignDocs/UnspecifiedBehaviorRandomization
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index a28f8c1e8a035..8ca0ce73a65fc 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -69,6 +69,10 @@ set(files
   __algorithm/partition_point.h
   __algorithm/pop_heap.h
   __algorithm/prev_permutation.h
+  __algorithm/pstl_any_all_none_of.h
+  __algorithm/pstl_fill.h
+  __algorithm/pstl_find.h
+  __algorithm/pstl_for_each.h
   __algorithm/push_heap.h
   __algorithm/ranges_adjacent_find.h
   __algorithm/ranges_all_of.h
@@ -385,11 +389,17 @@ set(files
   __functional/unwrap_ref.h
   __functional/weak_result_type.h
   __fwd/array.h
+  __fwd/fstream.h
   __fwd/get.h
   __fwd/hash.h
+  __fwd/ios.h
+  __fwd/istream.h
   __fwd/memory_resource.h
+  __fwd/ostream.h
   __fwd/pair.h
   __fwd/span.h
+  __fwd/sstream.h
+  __fwd/streambuf.h
   __fwd/string.h
   __fwd/string_view.h
   __fwd/subrange.h
@@ -494,6 +504,41 @@ set(files
   __numeric/transform_exclusive_scan.h
   __numeric/transform_inclusive_scan.h
   __numeric/transform_reduce.h
+  __pstl/internal/algorithm_fwd.h
+  __pstl/internal/algorithm_impl.h
+  __pstl/internal/execution_defs.h
+  __pstl/internal/execution_impl.h
+  __pstl/internal/glue_algorithm_defs.h
+  __pstl/internal/glue_algorithm_impl.h
+  __pstl/internal/glue_memory_defs.h
+  __pstl/internal/glue_memory_impl.h
+  __pstl/internal/glue_numeric_defs.h
+  __pstl/internal/glue_numeric_impl.h
+  __pstl/internal/memory_impl.h
+  __pstl/internal/numeric_fwd.h
+  __pstl/internal/numeric_impl.h
+  __pstl/internal/omp/parallel_for.h
+  __pstl/internal/omp/parallel_for_each.h
+  __pstl/internal/omp/parallel_invoke.h
+  __pstl/internal/omp/parallel_merge.h
+  __pstl/internal/omp/parallel_scan.h
+  __pstl/internal/omp/parallel_stable_partial_sort.h
+  __pstl/internal/omp/parallel_stable_sort.h
+  __pstl/internal/omp/parallel_transform_reduce.h
+  __pstl/internal/omp/parallel_transform_scan.h
+  __pstl/internal/omp/util.h
+  __pstl/internal/parallel_backend.h
+  __pstl/internal/parallel_backend_omp.h
+  __pstl/internal/parallel_backend_serial.h
+  __pstl/internal/parallel_backend_tbb.h
+  __pstl/internal/parallel_backend_utils.h
+  __pstl/internal/parallel_impl.h
+  __pstl/internal/pstl_config.h
+  __pstl/internal/unseq_backend_simd.h
+  __pstl/internal/utils.h
+  __pstl_algorithm
+  __pstl_memory
+  __pstl_numeric
   __random/bernoulli_distribution.h
   __random/binomial_distribution.h
   __random/cauchy_distribution.h
@@ -570,6 +615,7 @@ set(files
   __ranges/views.h
   __ranges/zip_view.h
   __split_buffer
+  __std_mbstate_t.h
   __string/char_traits.h
   __string/constexpr_c_functions.h
   __string/extern_template_lists.h
@@ -657,6 +703,7 @@ set(files
   __type_traits/is_empty.h
   __type_traits/is_enum.h
   __type_traits/is_equality_comparable.h
+  __type_traits/is_execution_policy.h
   __type_traits/is_final.h
   __type_traits/is_floating_point.h
   __type_traits/is_function.h
@@ -757,6 +804,7 @@ set(files
   __utility/priority_tag.h
   __utility/rel_ops.h
   __utility/swap.h
+  __utility/terminate_on_exception.h
   __utility/to_underlying.h
   __utility/unreachable.h
   __variant/monostate.h
diff --git a/libcxx/include/__algorithm/all_of.h b/libcxx/include/__algorithm/all_of.h
index 284c34ffcda9b..237f8495c645f 100644
--- a/libcxx/include/__algorithm/all_of.h
+++ b/libcxx/include/__algorithm/all_of.h
@@ -19,7 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _InputIterator, class _Predicate>
-_LIBCPP_NODISCARD_EXT inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
 all_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
   for (; __first != __last; ++__first)
     if (!__pred(*__first))
diff --git a/libcxx/include/__algorithm/pstl_any_all_none_of.h b/libcxx/include/__algorithm/pstl_any_all_none_of.h
new file mode 100644
index 0000000000000..6c01634eafdf3
--- /dev/null
+++ b/libcxx/include/__algorithm/pstl_any_all_none_of.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PSTL_ANY_ALL_NONE_OF_H
+#define _LIBCPP___ALGORITHM_PSTL_ANY_ALL_NONE_OF_H
+
+#include <__algorithm/any_of.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__pstl/internal/parallel_impl.h>
+#include <__pstl/internal/unseq_backend_simd.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/terminate_on_exception.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool
+any_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
+                __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    return std::__terminate_on_exception([&] {
+      return __pstl::__internal::__parallel_or(
+          __pstl::__internal::__par_backend_tag{},
+          __policy,
+          __first,
+          __last,
+          [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            return std::any_of(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred);
+          });
+    });
+  } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
+                       __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    return __pstl::__unseq_backend::__simd_or(__first, __last - __first, __pred);
+  } else {
+    return std::any_of(__first, __last, __pred);
+  }
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Pred,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool
+all_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) {
+  return !std::any_of(__policy, __first, __last, [&](__iter_reference<_ForwardIterator> __value) {
+    return !__pred(__value);
+  });
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Pred,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool
+none_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) {
+  return !std::any_of(__policy, __first, __last, __pred);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___ALGORITHM_PSTL_ANY_ALL_NONE_OF_H
diff --git a/libcxx/include/__algorithm/pstl_fill.h b/libcxx/include/__algorithm/pstl_fill.h
new file mode 100644
index 0000000000000..0ee14d6c8ee47
--- /dev/null
+++ b/libcxx/include/__algorithm/pstl_fill.h
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PSTL_FILL_H
+#define _LIBCPP___ALGORITHM_PSTL_FILL_H
+
+#include <__algorithm/fill.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__pstl/internal/parallel_impl.h>
+#include <__pstl/internal/unseq_backend_simd.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/terminate_on_exception.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+fill(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
+                __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    std::__terminate_on_exception([&] {
+      __pstl::__par_backend::__parallel_for(
+          __pstl::__internal::__par_backend_tag{},
+          __policy,
+          __first,
+          __last,
+          [&__policy, &__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            std::fill(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __value);
+          });
+    });
+  } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
+                       __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    __pstl::__unseq_backend::__simd_fill_n(__first, __last - __first, __value);
+  } else {
+    std::fill(__first, __last, __value);
+  }
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _SizeT,
+          class _Tp,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+fill_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _SizeT __n, const _Tp& __value) {
+  if constexpr (__is_cpp17_random_access_iterator<_ForwardIterator>::value)
+    std::fill(__policy, __first, __first + __n, __value);
+  else
+    std::fill_n(__first, __n, __value);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___ALGORITHM_PSTL_FILL_H
diff --git a/libcxx/include/__algorithm/pstl_find.h b/libcxx/include/__algorithm/pstl_find.h
new file mode 100644
index 0000000000000..8edb14bc59f74
--- /dev/null
+++ b/libcxx/include/__algorithm/pstl_find.h
@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PSTL_FIND_H
+#define _LIBCPP___ALGORITHM_PSTL_FIND_H
+
+#include <__algorithm/comp.h>
+#include <__algorithm/find.h>
+#include <__config>
+#include <__pstl/internal/parallel_impl.h>
+#include <__pstl/internal/unseq_backend_simd.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/terminate_on_exception.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+find(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
+                __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    return std::__terminate_on_exception([&] {
+      return __pstl::__internal::__parallel_find(
+          __pstl::__internal::__par_backend_tag{},
+          __policy,
+          __first,
+          __last,
+          [&__policy, &__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            return std::find(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __value);
+          },
+          less<>{},
+          true);
+    });
+  } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
+                       __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    using __diff_t = __iter_diff_t<_ForwardIterator>;
+    return __pstl::__unseq_backend::__simd_first(
+        __first, __diff_t(0), __last - __first, [&__value](_ForwardIterator __iter, __diff_t __i) {
+          return __iter[__i] == __value;
+        });
+  } else {
+    return std::find(__first, __last, __value);
+  }
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
+                __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    return std::__terminate_on_exception([&] {
+      return __pstl::__internal::__parallel_find(
+          __pstl::__internal::__par_backend_tag{},
+          __policy,
+          __first,
+          __last,
+          [&__policy, &__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            return std::find_if(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __pred);
+          },
+          less<>{},
+          true);
+    });
+  } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
+                       __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    using __diff_t = __iter_diff_t<_ForwardIterator>;
+    return __pstl::__unseq_backend::__simd_first(
+        __first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) {
+          return __pred(__iter[__i]);
+        });
+  } else {
+    return std::find_if(__first, __last, __pred);
+  }
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Predicate,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+find_if_not(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  return std::find_if(__policy, __first, __last, [&](__iter_reference<_ForwardIterator> __value) {
+    return !__pred(__value);
+  });
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_PSTL_FIND_H
diff --git a/libcxx/include/__algorithm/pstl_for_each.h b/libcxx/include/__algorithm/pstl_for_each.h
new file mode 100644
index 0000000000000..48bbeff2fc5d7
--- /dev/null
+++ b/libcxx/include/__algorithm/pstl_for_each.h
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PSTL_FOR_EACH_H
+#define _LIBCPP___ALGORITHM_PSTL_FOR_EACH_H
+
+#include <__algorithm/for_each.h>
+#include <__algorithm/for_each_n.h>
+#include <__config>
+#include <__iterator/iterator_traits.h>
+#include <__pstl/internal/parallel_backend.h>
+#include <__pstl/internal/unseq_backend_simd.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__utility/terminate_on_exception.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if defined(_LIBCPP_HAS_PARALLEL_ALGORITHMS) && _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Function,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) {
+  if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
+                __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    std::__terminate_on_exception([&] {
+      __pstl::__par_backend::__parallel_for(
+          {},
+          __policy,
+          __first,
+          __last,
+          [&__policy, __func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
+            std::for_each(std::__remove_parallel_policy(__policy), __brick_first, __brick_last, __func);
+          });
+    });
+  } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
+                       __is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    __pstl::__unseq_backend::__simd_walk_1(__first, __last - __first, __func);
+  } else {
+    std::for_each(__first, __last, __func);
+  }
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Size,
+          class _Function,
+          enable_if_t<is_execution_policy_v<__remove_cvref_t<_ExecutionPolicy>>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI void
+for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) {
+  if constexpr (__is_cpp17_random_access_iterator<_ForwardIterator>::value) {
+    std::for_each(__policy, __first, __first + __size, __func);
+  } else {
+    std::for_each_n(__first, __size, __func);
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // defined(_LIBCPP_HAS_PARALLEL_ALGORITHMS) && _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___ALGORITHM_PSTL_FOR_EACH_H
diff --git a/libcxx/include/__chrono/convert_to_tm.h b/libcxx/include/__chrono/convert_to_tm.h
index 95cde204f6946..f4dcddece74cd 100644
--- a/libcxx/include/__chrono/convert_to_tm.h
+++ b/libcxx/include/__chrono/convert_to_tm.h
@@ -10,9 +10,11 @@
 #ifndef _LIBCPP___CHRONO_CONVERT_TO_TM_H
 #define _LIBCPP___CHRONO_CONVERT_TO_TM_H
 
+#include <__chrono/calendar.h>
 #include <__chrono/concepts.h>
 #include <__chrono/day.h>
 #include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
 #include <__chrono/hh_mm_ss.h>
 #include <__chrono/month.h>
 #include <__chrono/month_weekday.h>
@@ -73,6 +75,24 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _Date& __date, chrono::weekday _
   return __result;
 }
 
+template <class _Tm, class _Duration>
+_LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const chrono::sys_time<_Duration> __tp) {
+  chrono::sys_days __days = chrono::time_point_cast<chrono::days>(__tp);
+  chrono::year_month_day __ymd{__days};
+
+  _Tm __result = std::__convert_to_tm<_Tm>(chrono::year_month_day{__ymd}, chrono::weekday{__days});
+
+  uint64_t __sec =
+      chrono::duration_cast<chrono::seconds>(__tp - chrono::time_point_cast<chrono::seconds>(__days)).count();
+  __sec %= 24 * 3600;
+  __result.tm_hour = __sec / 3600;
+  __sec %= 3600;
+  __result.tm_min = __sec / 60;
+  __result.tm_sec = __sec % 60;
+
+  return __result;
+}
+
 // Convert a chrono (calendar) time point, or dururation to the given _Tm type,
 // which must have the same properties as std::tm.
 template <class _Tm, class _ChronoT>
@@ -83,22 +103,13 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _ChronoT& __value) {
 #  endif
 
   if constexpr (__is_time_point<_ChronoT>) {
-    if constexpr (same_as<typename _ChronoT::clock, chrono::system_clock>) {
-      chrono::sys_days __days = chrono::time_point_cast<chrono::days>(__value);
-      chrono::year_month_day __ymd{__days};
-
-      __result = std::__convert_to_tm<_Tm>(chrono::year_month_day{__ymd}, chrono::weekday{__days});
-
-      // TODO FMT D138826 has improvements for this part.
-      // TODO FMT Since this is identical for duration and system time it would be good to avoid code duplication.
-      uint64_t __sec =
-          chrono::duration_cast<chrono::seconds>(__value - chrono::time_point_cast<chrono::seconds>(__days)).count();
-      __sec %= 24 * 3600;
-      __result.tm_hour = __sec / 3600;
-      __sec %= 3600;
-      __result.tm_min = __sec / 60;
-      __result.tm_sec = __sec % 60;
-    } else
+    if constexpr (same_as<typename _ChronoT::clock, chrono::system_clock>)
+      return std::__convert_to_tm<_Tm>(__value);
+    else if constexpr (same_as<typename _ChronoT::clock, chrono::file_clock>)
+      return std::__convert_to_tm<_Tm>(_ChronoT::clock::to_sys(__value));
+    else if constexpr (same_as<typename _ChronoT::clock, chrono::local_t>)
+      return std::__convert_to_tm<_Tm>(chrono::sys_time<typename _ChronoT::duration>{__value.time_since_epoch()});
+    else
       static_assert(sizeof(_ChronoT) == 0, "TODO: Add the missing clock specialization");
   } else if constexpr (chrono::__is_duration<_ChronoT>::value) {
     // [time.format]/6
diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h
index e73b043761eb1..a91a592a88e21 100644
--- a/libcxx/include/__chrono/formatter.h
+++ b/libcxx/include/__chrono/formatter.h
@@ -15,6 +15,7 @@
 #include <__chrono/convert_to_tm.h>
 #include <__chrono/day.h>
 #include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
 #include <__chrono/hh_mm_ss.h>
 #include <__chrono/month.h>
 #include <__chrono/month_weekday.h>
@@ -591,6 +592,29 @@ struct _LIBCPP_TEMPLATE_VIS formatter<chrono::sys_time<_Duration>, _CharT> : pub
   }
 };
 
+template <class _Duration, __fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::file_time<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto parse(basic_format_parse_context<_CharT>& __parse_ctx)
+      -> decltype(__parse_ctx.begin()) {
+    return _Base::__parse(__parse_ctx, __format_spec::__fields_chrono, __format_spec::__flags::__clock);
+  }
+};
+
+template <class _Duration, __fmt_char_type _CharT>
+struct _LIBCPP_TEMPLATE_VIS formatter<chrono::local_time<_Duration>, _CharT> : public __formatter_chrono<_CharT> {
+public:
+  using _Base = __formatter_chrono<_CharT>;
+
+  _LIBCPP_HIDE_FROM_ABI constexpr auto parse(basic_format_parse_context<_CharT>& __parse_ctx)
+      -> decltype(__parse_ctx.begin()) {
+    // The flags are not __clock since there is no associated time-zone.
+    return _Base::__parse(__parse_ctx, __format_spec::__fields_chrono, __format_spec::__flags::__date_time);
+  }
+};
+
 template <class _Rep, class _Period, __fmt_char_type _CharT>
 struct formatter<chrono::duration<_Rep, _Period>, _CharT> : public __formatter_chrono<_CharT> {
 public:
diff --git a/libcxx/include/__chrono/ostream.h b/libcxx/include/__chrono/ostream.h
index 26df8822dc84a..d044cc56903ab 100644
--- a/libcxx/include/__chrono/ostream.h
+++ b/libcxx/include/__chrono/ostream.h
@@ -10,8 +10,10 @@
 #ifndef _LIBCPP___CHRONO_OSTREAM_H
 #define _LIBCPP___CHRONO_OSTREAM_H
 
+#include <__chrono/calendar.h>
 #include <__chrono/day.h>
 #include <__chrono/duration.h>
+#include <__chrono/file_clock.h>
 #include <__chrono/hh_mm_ss.h>
 #include <__chrono/month.h>
 #include <__chrono/month_weekday.h>
@@ -45,6 +47,18 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const sys_time<_Duration> __tp)
   return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T}"), __tp);
 }
 
+template <class _CharT, class _Traits, class _Duration>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const file_time<_Duration> __tp) {
+  return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%F %T}"), __tp);
+}
+
+template <class _CharT, class _Traits, class _Duration>
+_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os, const local_time<_Duration> __tp) {
+  return __os << sys_time<_Duration>{__tp.time_since_epoch()};
+}
+
 // Depending on the type the return is a const _CharT* or a basic_string<_CharT>
 template <class _CharT, class _Period>
 _LIBCPP_HIDE_FROM_ABI auto __units_suffix() {
diff --git a/libcxx/include/__format/buffer.h b/libcxx/include/__format/buffer.h
index 16db88aec7fbe..e74ca8f579d42 100644
--- a/libcxx/include/__format/buffer.h
+++ b/libcxx/include/__format/buffer.h
@@ -28,12 +28,17 @@
 #include <__iterator/iterator_traits.h>
 #include <__iterator/wrap_iter.h>
 #include <__memory/addressof.h>
+#include <__memory/allocate_at_least.h>
+#include <__memory/allocator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/ranges_construct_at.h>
+#include <__memory/uninitialized_algorithms.h>
 #include <__type_traits/add_pointer.h>
 #include <__type_traits/conditional.h>
+#include <__utility/exception_guard.h>
 #include <__utility/move.h>
 #include <cstddef>
 #include <string_view>
-#include <vector>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -512,8 +517,13 @@ struct _LIBCPP_TEMPLATE_VIS __format_to_n_buffer final
 // context and the format arguments need to be retargeted to the new context.
 // This retargeting is done by a basic_format_context specialized for the
 // __iterator of this container.
+//
+// This class uses its own buffer management, since using vector
+// would lead to a circular include with formatter for vector<bool>.
 template <__fmt_char_type _CharT>
 class _LIBCPP_TEMPLATE_VIS __retarget_buffer {
+  using _Alloc = allocator<_CharT>;
+
 public:
   using value_type = _CharT;
 
@@ -537,31 +547,89 @@ class _LIBCPP_TEMPLATE_VIS __retarget_buffer {
     __retarget_buffer* __buffer_;
   };
 
-  _LIBCPP_HIDE_FROM_ABI explicit __retarget_buffer(size_t __size_hint) { __buffer_.reserve(__size_hint); }
+  __retarget_buffer(const __retarget_buffer&)            = delete;
+  __retarget_buffer& operator=(const __retarget_buffer&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI explicit __retarget_buffer(size_t __size_hint) {
+    auto __result = std::__allocate_at_least(__alloc_, __size_hint ? __size_hint : 256 / sizeof(_CharT));
+    __ptr_        = __result.ptr;
+    __capacity_   = __result.count;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI ~__retarget_buffer() {
+    ranges::destroy_n(__ptr_, __size_);
+    allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __capacity_);
+  }
 
   _LIBCPP_HIDE_FROM_ABI __iterator __make_output_iterator() { return __iterator{*this}; }
 
-  _LIBCPP_HIDE_FROM_ABI void push_back(_CharT __c) { __buffer_.push_back(__c); }
+  _LIBCPP_HIDE_FROM_ABI void push_back(_CharT __c) {
+    std::construct_at(__ptr_ + __size_, __c);
+    ++__size_;
+
+    if (__size_ == __capacity_)
+      __grow_buffer();
+  }
 
   template <__fmt_char_type _InCharT>
   _LIBCPP_HIDE_FROM_ABI void __copy(basic_string_view<_InCharT> __str) {
-    __buffer_.insert(__buffer_.end(), __str.begin(), __str.end());
+    size_t __n = __str.size();
+    if (__size_ + __n >= __capacity_)
+      // Push_back requires the buffer to have room for at least one character.
+      __grow_buffer(__size_ + __n + 1);
+
+    std::uninitialized_copy_n(__str.data(), __n, __ptr_ + __size_);
+    __size_ += __n;
   }
 
   template <__fmt_char_type _InCharT, class _UnaryOperation>
   _LIBCPP_HIDE_FROM_ABI void __transform(const _InCharT* __first, const _InCharT* __last, _UnaryOperation __operation) {
     _LIBCPP_ASSERT(__first <= __last, "not a valid range");
-    std::transform(__first, __last, std::back_inserter(__buffer_), std::move(__operation));
+
+    size_t __n = static_cast<size_t>(__last - __first);
+    if (__size_ + __n >= __capacity_)
+      // Push_back requires the buffer to have room for at least one character.
+      __grow_buffer(__size_ + __n + 1);
+
+    std::uninitialized_default_construct_n(__ptr_ + __size_, __n);
+    std::transform(__first, __last, __ptr_ + __size_, std::move(__operation));
+    __size_ += __n;
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __fill(size_t __n, _CharT __value) { __buffer_.insert(__buffer_.end(), __n, __value); }
+  _LIBCPP_HIDE_FROM_ABI void __fill(size_t __n, _CharT __value) {
+    if (__size_ + __n >= __capacity_)
+      // Push_back requires the buffer to have room for at least one character.
+      __grow_buffer(__size_ + __n + 1);
+
+    std::uninitialized_fill_n(__ptr_ + __size_, __n, __value);
+    __size_ += __n;
+  }
 
-  _LIBCPP_HIDE_FROM_ABI basic_string_view<_CharT> __view() { return {__buffer_.data(), __buffer_.size()}; }
+  _LIBCPP_HIDE_FROM_ABI basic_string_view<_CharT> __view() { return {__ptr_, __size_}; }
 
 private:
-  // Use vector instead of string to avoid adding zeros after every append
-  // operation. The buffer is exposed as a string_view and not as a c-string.
-  vector<_CharT> __buffer_;
+  _LIBCPP_HIDE_FROM_ABI void __grow_buffer() { __grow_buffer(__capacity_ * 1.6); }
+
+  _LIBCPP_HIDE_FROM_ABI void __grow_buffer(size_t __capacity) {
+    _LIBCPP_ASSERT(__capacity > __capacity_, "the buffer must grow");
+    auto __result = std::__allocate_at_least(__alloc_, __capacity);
+    auto __guard  = std::__make_exception_guard([&] {
+      allocator_traits<_Alloc>::deallocate(__alloc_, __result.ptr, __result.count);
+    });
+    // This shouldn't throw, but just to be safe. Not that at -O1 this
+    // guard is optimized away so there is no runtime overhead.
+    std::uninitialized_move_n(__ptr_, __size_, __result.ptr);
+    __guard.__complete();
+    ranges::destroy_n(__ptr_, __size_);
+    allocator_traits<_Alloc>::deallocate(__alloc_, __ptr_, __capacity_);
+
+    __ptr_      = __result.ptr;
+    __capacity_ = __result.count;
+  }
+  _LIBCPP_NO_UNIQUE_ADDRESS _Alloc __alloc_;
+  _CharT* __ptr_;
+  size_t __capacity_;
+  size_t __size_{0};
 };
 
 } // namespace __format
diff --git a/libcxx/include/__functional/bind.h b/libcxx/include/__functional/bind.h
index 8e5d598ad257b..1d5e531a2c696 100644
--- a/libcxx/include/__functional/bind.h
+++ b/libcxx/include/__functional/bind.h
@@ -51,7 +51,14 @@ namespace placeholders
 
 template <int _Np> struct __ph {};
 
-#if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY)
+// C++17 recommends that we implement placeholders as `inline constexpr`, but allows
+// implementing them as `extern <implementation-defined>`. Libc++ implements them as
+// `extern const` in all standard modes to avoid an ABI break in C++03: making them
+// `inline constexpr` requires removing their definition in the shared library to
+// avoid ODR violations, which is an ABI break.
+//
+// In practice, since placeholders are empty, `extern const` is almost impossible
+// to distinguish from `inline constexpr` from a usage stand point.
 _LIBCPP_FUNC_VIS extern const __ph<1>   _1;
 _LIBCPP_FUNC_VIS extern const __ph<2>   _2;
 _LIBCPP_FUNC_VIS extern const __ph<3>   _3;
@@ -62,29 +69,6 @@ _LIBCPP_FUNC_VIS extern const __ph<7>   _7;
 _LIBCPP_FUNC_VIS extern const __ph<8>   _8;
 _LIBCPP_FUNC_VIS extern const __ph<9>   _9;
 _LIBCPP_FUNC_VIS extern const __ph<10> _10;
-#elif _LIBCPP_STD_VER >= 17
-inline constexpr __ph<1>   _1{};
-inline constexpr __ph<2>   _2{};
-inline constexpr __ph<3>   _3{};
-inline constexpr __ph<4>   _4{};
-inline constexpr __ph<5>   _5{};
-inline constexpr __ph<6>   _6{};
-inline constexpr __ph<7>   _7{};
-inline constexpr __ph<8>   _8{};
-inline constexpr __ph<9>   _9{};
-inline constexpr __ph<10> _10{};
-#else
-constexpr __ph<1>   _1{};
-constexpr __ph<2>   _2{};
-constexpr __ph<3>   _3{};
-constexpr __ph<4>   _4{};
-constexpr __ph<5>   _5{};
-constexpr __ph<6>   _6{};
-constexpr __ph<7>   _7{};
-constexpr __ph<8>   _8{};
-constexpr __ph<9>   _9{};
-constexpr __ph<10> _10{};
-#endif // defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY)
 
 } // namespace placeholders
 
@@ -96,7 +80,7 @@ struct is_placeholder<placeholders::__ph<_Np> >
 #ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Uj>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
 _Tp&
 __mu(reference_wrapper<_Tp> __t, _Uj&)
 {
@@ -104,7 +88,7 @@ __mu(reference_wrapper<_Tp> __t, _Uj&)
 }
 
 template <class _Ti, class ..._Uj, size_t ..._Indx>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
 typename __invoke_of<_Ti&, _Uj...>::type
 __mu_expand(_Ti& __ti, tuple<_Uj...>& __uj, __tuple_indices<_Indx...>)
 {
@@ -112,7 +96,7 @@ __mu_expand(_Ti& __ti, tuple<_Uj...>& __uj, __tuple_indices<_Indx...>)
 }
 
 template <class _Ti, class ..._Uj>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
 typename __enable_if_t
 <
     is_bind_expression<_Ti>::value,
@@ -134,7 +118,7 @@ struct __mu_return2<true, _Ti, _Uj>
 };
 
 template <class _Ti, class _Uj>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
 typename enable_if
 <
     0 < is_placeholder<_Ti>::value,
@@ -147,7 +131,7 @@ __mu(_Ti&, _Uj& __uj)
 }
 
 template <class _Ti, class _Uj>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
 typename enable_if
 <
     !is_bind_expression<_Ti>::value &&
@@ -265,7 +249,7 @@ struct __bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj, true>
 };
 
 template <class _Fp, class _BoundArgs, size_t ..._Indx, class _Args>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
 typename __bind_return<_Fp, _BoundArgs, _Args>::type
 __apply_functor(_Fp& __f, _BoundArgs& __bound_args, __tuple_indices<_Indx...>,
                 _Args&& __args)
diff --git a/libcxx/include/__functional/invoke.h b/libcxx/include/__functional/invoke.h
index c4906840f03dd..7f511c86e7c9d 100644
--- a/libcxx/include/__functional/invoke.h
+++ b/libcxx/include/__functional/invoke.h
@@ -474,7 +474,7 @@ template <class _Ret, bool = is_void<_Ret>::value>
 struct __invoke_void_return_wrapper
 {
     template <class ..._Args>
-    _LIBCPP_HIDE_FROM_ABI static _Ret __call(_Args&&... __args) {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
         return std::__invoke(std::forward<_Args>(__args)...);
     }
 };
@@ -483,7 +483,7 @@ template <class _Ret>
 struct __invoke_void_return_wrapper<_Ret, true>
 {
     template <class ..._Args>
-    _LIBCPP_HIDE_FROM_ABI static void __call(_Args&&... __args) {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
         std::__invoke(std::forward<_Args>(__args)...);
     }
 };
diff --git a/libcxx/include/__fwd/fstream.h b/libcxx/include/__fwd/fstream.h
new file mode 100644
index 0000000000000..0eed60a42c2b4
--- /dev/null
+++ b/libcxx/include/__fwd/fstream.h
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_FSTREAM_H
+#define _LIBCPP___FWD_FSTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_filebuf;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ifstream;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ofstream;
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_fstream;
+
+typedef basic_filebuf<char> filebuf;
+typedef basic_ifstream<char> ifstream;
+typedef basic_ofstream<char> ofstream;
+typedef basic_fstream<char> fstream;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_filebuf<wchar_t> wfilebuf;
+typedef basic_ifstream<wchar_t> wifstream;
+typedef basic_ofstream<wchar_t> wofstream;
+typedef basic_fstream<wchar_t> wfstream;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(filebuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wfilebuf)) basic_filebuf;
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ifstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wifstream)) basic_ifstream;
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ofstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wofstream)) basic_ofstream;
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(fstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wfstream)) basic_fstream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_FSTREAM_H
diff --git a/libcxx/include/__fwd/ios.h b/libcxx/include/__fwd/ios.h
new file mode 100644
index 0000000000000..e2512471e9729
--- /dev/null
+++ b/libcxx/include/__fwd/ios.h
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_IOS_H
+#define _LIBCPP___FWD_IOS_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ios;
+
+typedef basic_ios<char> ios;
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_ios<wchar_t> wios;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ios) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wios)) basic_ios;
+
+#if defined(_NEWLIB_VERSION)
+// On newlib, off_t is 'long int'
+typedef long int streamoff; // for char_traits in <string>
+#else
+typedef long long streamoff; // for char_traits in <string>
+#endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_IOS_H
diff --git a/libcxx/include/__fwd/istream.h b/libcxx/include/__fwd/istream.h
new file mode 100644
index 0000000000000..5172da095ece1
--- /dev/null
+++ b/libcxx/include/__fwd/istream.h
@@ -0,0 +1,43 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_ISTREAM_H
+#define _LIBCPP___FWD_ISTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_istream;
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_iostream;
+
+typedef basic_istream<char> istream;
+typedef basic_iostream<char> iostream;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_istream<wchar_t> wistream;
+typedef basic_iostream<wchar_t> wiostream;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(istream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wistream)) basic_istream;
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(iostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wiostream)) basic_iostream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_ISTREAM_H
diff --git a/libcxx/include/__fwd/ostream.h b/libcxx/include/__fwd/ostream.h
new file mode 100644
index 0000000000000..5a6a01f3b8341
--- /dev/null
+++ b/libcxx/include/__fwd/ostream.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_OSTREAM_H
+#define _LIBCPP___FWD_OSTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ostream;
+
+typedef basic_ostream<char> ostream;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_ostream<wchar_t> wostream;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(ostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wostream)) basic_ostream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_OSTREAM_H
diff --git a/libcxx/include/__fwd/sstream.h b/libcxx/include/__fwd/sstream.h
new file mode 100644
index 0000000000000..83e82bf8a272a
--- /dev/null
+++ b/libcxx/include/__fwd/sstream.h
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_SSTREAM_H
+#define _LIBCPP___FWD_SSTREAM_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_stringbuf;
+
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_istringstream;
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_ostringstream;
+template <class _CharT, class _Traits = char_traits<_CharT>, class _Allocator = allocator<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_stringstream;
+
+typedef basic_stringbuf<char> stringbuf;
+typedef basic_istringstream<char> istringstream;
+typedef basic_ostringstream<char> ostringstream;
+typedef basic_stringstream<char> stringstream;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_stringbuf<wchar_t> wstringbuf;
+typedef basic_istringstream<wchar_t> wistringstream;
+typedef basic_ostringstream<wchar_t> wostringstream;
+typedef basic_stringstream<wchar_t> wstringstream;
+#endif
+
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(stringbuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstringbuf)) basic_stringbuf;
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(istringstream)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wistringstream)) basic_istringstream;
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(ostringstream)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wostringstream)) basic_ostringstream;
+template <class _CharT, class _Traits, class _Allocator>
+class _LIBCPP_PREFERRED_NAME(stringstream)
+    _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstringstream)) basic_stringstream;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_SSTREAM_H
diff --git a/libcxx/include/__fwd/streambuf.h b/libcxx/include/__fwd/streambuf.h
new file mode 100644
index 0000000000000..f1ee9115ed247
--- /dev/null
+++ b/libcxx/include/__fwd/streambuf.h
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___FWD_STREAMBUF_H
+#define _LIBCPP___FWD_STREAMBUF_H
+
+#include <__config>
+#include <__fwd/string.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _Traits = char_traits<_CharT> >
+class _LIBCPP_TEMPLATE_VIS basic_streambuf;
+
+typedef basic_streambuf<char> streambuf;
+
+#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
+typedef basic_streambuf<wchar_t> wstreambuf;
+#endif
+
+template <class _CharT, class _Traits>
+class _LIBCPP_PREFERRED_NAME(streambuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstreambuf)) basic_streambuf;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___FWD_STREAMBUF_H
diff --git a/libcxx/include/__iterator/iterator_traits.h b/libcxx/include/__iterator/iterator_traits.h
index 0318e4b62591e..f066b962e7a54 100644
--- a/libcxx/include/__iterator/iterator_traits.h
+++ b/libcxx/include/__iterator/iterator_traits.h
@@ -534,6 +534,9 @@ using __iter_diff_t = typename iterator_traits<_Iter>::difference_type;
 template<class _InputIterator>
 using __iter_value_type = typename iterator_traits<_InputIterator>::value_type;
 
+template <class _Iter>
+using __iter_reference = typename iterator_traits<_Iter>::reference;
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___ITERATOR_ITERATOR_TRAITS_H
diff --git a/libcxx/include/__locale b/libcxx/include/__locale
index 994613083cdff..a812427acba4e 100644
--- a/libcxx/include/__locale
+++ b/libcxx/include/__locale
@@ -25,6 +25,8 @@
 
 #ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
 #  include <cwchar>
+#else
+#  include <__std_mbstate_t.h>
 #endif
 
 #if defined(_LIBCPP_MSVCRT_LIKE)
diff --git a/libcxx/include/__mbstate_t.h b/libcxx/include/__mbstate_t.h
index 487a6d092c715..d793787fa0cd6 100644
--- a/libcxx/include/__mbstate_t.h
+++ b/libcxx/include/__mbstate_t.h
@@ -16,29 +16,27 @@
 #  pragma GCC system_header
 #endif
 
-// TODO(ldionne):
-// The goal of this header is to provide mbstate_t without having to pull in
-// <wchar.h> or <uchar.h>. This is necessary because we need that type even
-// when we don't have (or try to provide) support for wchar_t, because several
-// types like std::fpos are defined in terms of mbstate_t.
+// The goal of this header is to provide mbstate_t without requiring all of
+// <uchar.h> or <wchar.h>. It's also used by the libc++ versions of <uchar.h>
+// and <wchar.h> to get mbstate_t when the C library doesn't provide <uchar.h>
+// or <wchar.h>, hence the #include_next of those headers instead of #include.
+// (e.g. if <wchar.h> isn't present in the C library, the libc++ <wchar.h>
+// will include this header. This header needs to not turn around and cyclically
+// include <wchar.h>, but fall through to <uchar.h>.)
 //
-// This is a gruesome hack, but I don't know how to make it cleaner for
-// the time being.
+// This does not define std::mbstate_t -- this only brings in the declaration
+// in the global namespace.
 
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-#   include <wchar.h> // for mbstate_t
-#elif __has_include(<bits/types/mbstate_t.h>)
+#if __has_include(<bits/types/mbstate_t.h>)
 #   include <bits/types/mbstate_t.h> // works on most Unixes
 #elif __has_include(<sys/_types/_mbstate_t.h>)
 #   include <sys/_types/_mbstate_t.h> // works on Darwin
+#elif !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS) && __has_include_next(<wchar.h>)
+#   include_next <wchar.h> // fall back to the C standard provider of mbstate_t
+#elif __has_include_next(<uchar.h>)
+#   include_next <uchar.h> // <uchar.h> is also required to make mbstate_t visible
 #else
-#   error "The library was configured without support for wide-characters, but we don't know how to get the definition of mbstate_t without <wchar.h> on your platform."
+#   error "We don't know how to get the definition of mbstate_t without <wchar.h> on your platform."
 #endif
 
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
-
-_LIBCPP_END_NAMESPACE_STD
-
 #endif // _LIBCPP___MBSTATE_T_H
diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h
index 6e7800cbd2b9f..98f68899e4efb 100644
--- a/libcxx/include/__memory/allocator.h
+++ b/libcxx/include/__memory/allocator.h
@@ -21,7 +21,6 @@
 #include <__utility/forward.h>
 #include <cstddef>
 #include <new>
-#include <stdexcept>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__memory/allocator_traits.h b/libcxx/include/__memory/allocator_traits.h
index 57f88a38151fd..4ed44165f916b 100644
--- a/libcxx/include/__memory/allocator_traits.h
+++ b/libcxx/include/__memory/allocator_traits.h
@@ -401,6 +401,25 @@ struct __is_cpp17_copy_insertable<_Alloc, __enable_if_t<
     : __is_cpp17_move_insertable<_Alloc>
 { };
 
+// ASan choices
+#ifndef _LIBCPP_HAS_NO_ASAN
+#   define _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS 1
+#endif
+
+#ifdef _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS
+template <class _Alloc>
+struct __asan_annotate_container_with_allocator
+#   if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1600
+      : true_type {};
+#   else
+      // TODO LLVM18: Remove the special-casing
+      : false_type {};
+#   endif
+
+template <class _Tp>
+struct __asan_annotate_container_with_allocator<allocator<_Tp> > : true_type {};
+#endif
+
 #undef _LIBCPP_ALLOCATOR_TRAITS_HAS_XXX
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__pstl/internal/algorithm_fwd.h b/libcxx/include/__pstl/internal/algorithm_fwd.h
new file mode 100644
index 0000000000000..4f6d73103ec81
--- /dev/null
+++ b/libcxx/include/__pstl/internal/algorithm_fwd.h
@@ -0,0 +1,1769 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_ALGORITHM_FWD_H
+#define _PSTL_ALGORITHM_FWD_H
+
+#include <iterator>
+#include <type_traits>
+#include <utility>
+
+#include "pstl_config.h"
+
+namespace __pstl {
+namespace __internal {
+
+//------------------------------------------------------------------------
+// walk1 (pseudo)
+//
+// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _Function>
+void __brick_walk1(_ForwardIterator,
+                   _ForwardIterator,
+                   _Function,
+                   /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _Function>
+void __brick_walk1(_RandomAccessIterator,
+                   _RandomAccessIterator,
+                   _Function,
+                   /*vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Function>
+void __pattern_walk1(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Function) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Function>
+void __pattern_walk1(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Function);
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Brick>
+void __pattern_walk_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Brick) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Brick>
+void __pattern_walk_brick(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Brick);
+
+//------------------------------------------------------------------------
+// walk1_n
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _Size, class _Function>
+_ForwardIterator __brick_walk1_n(
+    _ForwardIterator,
+    _Size,
+    _Function,
+    /*_IsVectorTag=*/std::false_type);
+
+template <class _RandomAccessIterator, class _DifferenceType, class _Function>
+_RandomAccessIterator __brick_walk1_n(
+    _RandomAccessIterator,
+    _DifferenceType,
+    _Function,
+    /*vectorTag=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
+_ForwardIterator __pattern_walk1_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _Size, _Function) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function>
+_RandomAccessIterator
+__pattern_walk1_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, _Function);
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
+_ForwardIterator __pattern_walk_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _Size, _Brick) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
+_RandomAccessIterator
+__pattern_walk_brick_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, _Brick);
+
+//------------------------------------------------------------------------
+// walk2 (pseudo)
+//
+// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
+_ForwardIterator2 __brick_walk2(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _Function,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Function>
+_RandomAccessIterator2 __brick_walk2(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _Function,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
+_ForwardIterator2 __brick_walk2_n(
+    _ForwardIterator1,
+    _Size,
+    _ForwardIterator2,
+    _Function,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Function>
+_RandomAccessIterator2 __brick_walk2_n(
+    _RandomAccessIterator1,
+    _Size,
+    _RandomAccessIterator2,
+    _Function,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function>
+_ForwardIterator2
+__pattern_walk2(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Function) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Function>
+_RandomAccessIterator2 __pattern_walk2(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _Function);
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _Size,
+          class _ForwardIterator2,
+          class _Function>
+_ForwardIterator2
+__pattern_walk2_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _Size, _ForwardIterator2, _Function) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _Size,
+          class _RandomAccessIterator2,
+          class _Function>
+_RandomAccessIterator2 __pattern_walk2_n(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _Size, _RandomAccessIterator2, _Function);
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
+_ForwardIterator2 __pattern_walk2_brick(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Brick) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Brick>
+_RandomAccessIterator2 __pattern_walk2_brick(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _Brick);
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _Size,
+          class _ForwardIterator2,
+          class _Brick>
+_ForwardIterator2
+__pattern_walk2_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _Size, _ForwardIterator2, _Brick) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _Size,
+          class _RandomAccessIterator2,
+          class _Brick>
+_RandomAccessIterator2 __pattern_walk2_brick_n(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _Size, _RandomAccessIterator2, _Brick);
+
+//------------------------------------------------------------------------
+// walk3 (pseudo)
+//
+// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
+_ForwardIterator3 __brick_walk3(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator3,
+    _Function,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
+_RandomAccessIterator3 __brick_walk3(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator3,
+    _Function,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator3,
+          class _Function>
+_ForwardIterator3 __pattern_walk3(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator3,
+    _Function) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _Function>
+_RandomAccessIterator3 __pattern_walk3(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator3,
+    _Function);
+
+//------------------------------------------------------------------------
+// equal
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __brick_equal(_ForwardIterator1,
+                   _ForwardIterator1,
+                   _ForwardIterator2,
+                   _BinaryPredicate,
+                   /* is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+bool __brick_equal(_RandomAccessIterator1,
+                   _RandomAccessIterator1,
+                   _RandomAccessIterator2,
+                   _BinaryPredicate,
+                   /* is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __pattern_equal(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _BinaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+bool __pattern_equal(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _BinaryPredicate);
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __brick_equal(_ForwardIterator1,
+                   _ForwardIterator1,
+                   _ForwardIterator2,
+                   _ForwardIterator2,
+                   _BinaryPredicate,
+                   /* is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+bool __brick_equal(_RandomAccessIterator1,
+                   _RandomAccessIterator1,
+                   _RandomAccessIterator2,
+                   _RandomAccessIterator2,
+                   _BinaryPredicate,
+                   /* is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __pattern_equal(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+bool __pattern_equal(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate);
+
+//------------------------------------------------------------------------
+// find_end
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __brick_find_end(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_find_end(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __pattern_find_end(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator1 __pattern_find_end(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// find_first_of
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __brick_find_first_of(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_find_first_of(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __pattern_find_first_of(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator1 __pattern_find_first_of(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// search
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __brick_search(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_search(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __pattern_search(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _BinaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator1 __pattern_search(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _BinaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// search_n
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
+_ForwardIterator __brick_search_n(
+    _ForwardIterator,
+    _ForwardIterator,
+    _Size,
+    const _Tp&,
+    _BinaryPredicate,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate>
+_RandomAccessIterator __brick_search_n(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Size,
+    const _Tp&,
+    _BinaryPredicate,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
+_ForwardIterator __pattern_search_n(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Size, const _Tp&, _BinaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _Size,
+          class _Tp,
+          class _BinaryPredicate>
+_RandomAccessIterator __pattern_search_n(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Size,
+    const _Tp&,
+    _BinaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// copy_n
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _Size, class _OutputIterator>
+_OutputIterator __brick_copy_n(_ForwardIterator,
+                               _Size,
+                               _OutputIterator,
+                               /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _Size, class _OutputIterator>
+_OutputIterator __brick_copy_n(_RandomAccessIterator,
+                               _Size,
+                               _OutputIterator,
+                               /*vector=*/std::true_type) noexcept;
+
+//------------------------------------------------------------------------
+// copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator __brick_copy(_ForwardIterator,
+                             _ForwardIterator,
+                             _OutputIterator,
+                             /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __brick_copy(_RandomAccessIterator,
+                             _RandomAccessIterator,
+                             _OutputIterator,
+                             /*vector=*/std::true_type) noexcept;
+
+//------------------------------------------------------------------------
+// move
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator __brick_move(_ForwardIterator,
+                             _ForwardIterator,
+                             _OutputIterator,
+                             /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __brick_move(_RandomAccessIterator,
+                             _RandomAccessIterator,
+                             _OutputIterator,
+                             /*vector=*/std::true_type) noexcept;
+
+//------------------------------------------------------------------------
+// swap_ranges
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator __brick_swap_ranges(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __brick_swap_ranges(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    /*vector=*/std::true_type) noexcept;
+
+//------------------------------------------------------------------------
+// copy_if
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
+_OutputIterator __brick_copy_if(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    _UnaryPredicate,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator, class _UnaryPredicate>
+_OutputIterator __brick_copy_if(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _UnaryPredicate,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
+std::pair<_DifferenceType, _DifferenceType> __brick_calc_mask_1(
+    _ForwardIterator,
+    _ForwardIterator,
+    bool* __restrict,
+    _UnaryPredicate,
+    /*vector=*/std::false_type) noexcept;
+template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
+std::pair<_DifferenceType, _DifferenceType> __brick_calc_mask_1(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    bool* __restrict,
+    _UnaryPredicate,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _ForwardIterator, class _OutputIterator>
+void __brick_copy_by_mask(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    bool*,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator>
+void __brick_copy_by_mask(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    bool* __restrict,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
+void __brick_partition_by_mask(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator1,
+    _OutputIterator2,
+    bool*,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2>
+void __brick_partition_by_mask(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator1,
+    _OutputIterator2,
+    bool*,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
+_OutputIterator __pattern_copy_if(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator, _UnaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryPredicate>
+_OutputIterator __pattern_copy_if(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _UnaryPredicate);
+
+//------------------------------------------------------------------------
+// count
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _Predicate>
+typename std::iterator_traits<_RandomAccessIterator>::difference_type __brick_count(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Predicate,
+    /* is_vector = */ std::true_type) noexcept;
+
+template <class _ForwardIterator, class _Predicate>
+typename std::iterator_traits<_ForwardIterator>::difference_type __brick_count(
+    _ForwardIterator,
+    _ForwardIterator,
+    _Predicate,
+    /* is_vector = */ std::false_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
+typename std::iterator_traits<_ForwardIterator>::difference_type
+__pattern_count(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Predicate) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
+typename std::iterator_traits<_RandomAccessIterator>::difference_type __pattern_count(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Predicate);
+
+//------------------------------------------------------------------------
+// unique
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator __brick_unique(
+    _ForwardIterator,
+    _ForwardIterator,
+    _BinaryPredicate,
+    /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __brick_unique(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _BinaryPredicate,
+    /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator
+__pattern_unique(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _BinaryPredicate) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __pattern_unique(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _BinaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// unique_copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
+OutputIterator __brick_unique_copy(
+    _ForwardIterator,
+    _ForwardIterator,
+    OutputIterator,
+    _BinaryPredicate,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator, class _BinaryPredicate>
+_OutputIterator __brick_unique_copy(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _BinaryPredicate,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate>
+_OutputIterator __pattern_unique_copy(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator, _BinaryPredicate) noexcept;
+
+template <class _ExecutionPolicy, class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
+_DifferenceType __brick_calc_mask_2(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    bool* __restrict,
+    _BinaryPredicate,
+    /*vector=*/std::false_type) noexcept;
+
+template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
+_DifferenceType __brick_calc_mask_2(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    bool* __restrict,
+    _BinaryPredicate,
+    /*vector=*/std::true_type) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _BinaryPredicate>
+_OutputIterator __pattern_unique_copy(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _BinaryPredicate);
+
+//------------------------------------------------------------------------
+// reverse
+//------------------------------------------------------------------------
+
+template <class _BidirectionalIterator>
+void __brick_reverse(_BidirectionalIterator,
+                     _BidirectionalIterator,
+                     /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator>
+void __brick_reverse(_RandomAccessIterator,
+                     _RandomAccessIterator,
+                     /*__is_vector=*/std::true_type) noexcept;
+
+template <class _BidirectionalIterator>
+void __brick_reverse(_BidirectionalIterator,
+                     _BidirectionalIterator,
+                     _BidirectionalIterator,
+                     /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator>
+void __brick_reverse(_RandomAccessIterator,
+                     _RandomAccessIterator,
+                     _RandomAccessIterator,
+                     /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator>
+void __pattern_reverse(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
+void __pattern_reverse(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator);
+
+//------------------------------------------------------------------------
+// reverse_copy
+//------------------------------------------------------------------------
+
+template <class _BidirectionalIterator, class _OutputIterator>
+_OutputIterator __brick_reverse_copy(
+    _BidirectionalIterator,
+    _BidirectionalIterator,
+    _OutputIterator,
+    /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __brick_reverse_copy(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator>
+_OutputIterator __pattern_reverse_copy(
+    _Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator, _OutputIterator) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __pattern_reverse_copy(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _OutputIterator);
+
+//------------------------------------------------------------------------
+// rotate
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator>
+_ForwardIterator __brick_rotate(
+    _ForwardIterator,
+    _ForwardIterator,
+    _ForwardIterator,
+    /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator>
+_RandomAccessIterator __brick_rotate(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator>
+_ForwardIterator
+__pattern_rotate(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _ForwardIterator) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
+_RandomAccessIterator __pattern_rotate(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator);
+
+//------------------------------------------------------------------------
+// rotate_copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator __brick_rotate_copy(
+    _ForwardIterator,
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __brick_rotate_copy(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator>
+_OutputIterator __pattern_rotate_copy(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _ForwardIterator, _OutputIterator) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator>
+_OutputIterator __pattern_rotate_copy(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator);
+
+//------------------------------------------------------------------------
+// is_partitioned
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _UnaryPredicate>
+bool __brick_is_partitioned(_ForwardIterator,
+                            _ForwardIterator,
+                            _UnaryPredicate,
+                            /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+bool __brick_is_partitioned(_RandomAccessIterator,
+                            _RandomAccessIterator,
+                            _UnaryPredicate,
+                            /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+bool __pattern_is_partitioned(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+bool __pattern_is_partitioned(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate);
+
+//------------------------------------------------------------------------
+// partition
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator __brick_partition(
+    _ForwardIterator,
+    _ForwardIterator,
+    _UnaryPredicate,
+    /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __brick_partition(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _UnaryPredicate,
+    /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator
+__pattern_partition(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __pattern_partition(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate);
+
+//------------------------------------------------------------------------
+// stable_partition
+//------------------------------------------------------------------------
+
+template <class _BidirectionalIterator, class _UnaryPredicate>
+_BidirectionalIterator __brick_stable_partition(
+    _BidirectionalIterator,
+    _BidirectionalIterator,
+    _UnaryPredicate,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __brick_stable_partition(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _UnaryPredicate,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
+_BidirectionalIterator __pattern_stable_partition(
+    _Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator, _UnaryPredicate) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __pattern_stable_partition(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _UnaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// partition_copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
+std::pair<_OutputIterator1, _OutputIterator2> __brick_partition_copy(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator1,
+    _OutputIterator2,
+    _UnaryPredicate,
+    /*is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
+std::pair<_OutputIterator1, _OutputIterator2> __brick_partition_copy(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator1,
+    _OutputIterator2,
+    _UnaryPredicate,
+    /*is_vector=*/std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _OutputIterator1,
+          class _OutputIterator2,
+          class _UnaryPredicate>
+std::pair<_OutputIterator1, _OutputIterator2> __pattern_partition_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator1,
+    _OutputIterator2,
+    _UnaryPredicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator1,
+          class _OutputIterator2,
+          class _UnaryPredicate>
+std::pair<_OutputIterator1, _OutputIterator2> __pattern_partition_copy(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator1,
+    _OutputIterator2,
+    _UnaryPredicate);
+
+//------------------------------------------------------------------------
+// sort
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsMoveConstructible>
+void __pattern_sort(
+    _Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare, _IsMoveConstructible) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_sort(__parallel_tag<_IsVector>,
+                    _ExecutionPolicy&&,
+                    _RandomAccessIterator,
+                    _RandomAccessIterator,
+                    _Compare,
+                    /*is_move_constructible=*/std::true_type);
+
+//------------------------------------------------------------------------
+// stable_sort
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_stable_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_stable_sort(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare);
+
+//------------------------------------------------------------------------
+// partial_sort
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_partial_sort(
+    _Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_partial_sort(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// partial_sort_copy
+//------------------------------------------------------------------------
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+_RandomAccessIterator2 __pattern_partial_sort_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+_RandomAccessIterator2 __pattern_partial_sort_copy(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Compare);
+
+//------------------------------------------------------------------------
+// adjacent_find
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __brick_adjacent_find(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _BinaryPredicate,
+    /* IsVector = */ std::true_type,
+    bool) noexcept;
+
+template <class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator __brick_adjacent_find(
+    _ForwardIterator,
+    _ForwardIterator,
+    _BinaryPredicate,
+    /* IsVector = */ std::false_type,
+    bool) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator
+__pattern_adjacent_find(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _BinaryPredicate, bool) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __pattern_adjacent_find(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _BinaryPredicate,
+    bool);
+
+//------------------------------------------------------------------------
+// nth_element
+//------------------------------------------------------------------------
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_nth_element(
+    _Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_nth_element(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare) noexcept;
+
+//------------------------------------------------------------------------
+// fill, fill_n
+//------------------------------------------------------------------------
+template <class _RandomAccessIterator, class _Tp>
+void __brick_fill(_RandomAccessIterator,
+                  _RandomAccessIterator,
+                  const _Tp&,
+                  /* __is_vector = */ std::true_type) noexcept;
+
+template <class _ForwardIterator, class _Tp>
+void __brick_fill(_ForwardIterator,
+                  _ForwardIterator,
+                  const _Tp&,
+                  /* __is_vector = */ std::false_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+void __pattern_fill(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, const _Tp&) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp>
+_RandomAccessIterator
+__pattern_fill(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, const _Tp&);
+
+template <class _RandomAccessIterator, class _Size, class _Tp>
+_RandomAccessIterator
+__brick_fill_n(_RandomAccessIterator,
+               _Size,
+               const _Tp&,
+               /* __is_vector = */ std::true_type) noexcept;
+
+template <class _OutputIterator, class _Size, class _Tp>
+_OutputIterator
+__brick_fill_n(_OutputIterator,
+               _Size,
+               const _Tp&,
+               /* __is_vector = */ std::false_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp>
+_OutputIterator __pattern_fill_n(_Tag, _ExecutionPolicy&&, _OutputIterator, _Size, const _Tp&) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp>
+_RandomAccessIterator
+__pattern_fill_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, const _Tp&);
+
+//------------------------------------------------------------------------
+// generate, generate_n
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _Generator>
+void __brick_generate(_RandomAccessIterator,
+                      _RandomAccessIterator,
+                      _Generator,
+                      /* is_vector = */ std::true_type) noexcept;
+
+template <class _ForwardIterator, class _Generator>
+void __brick_generate(_ForwardIterator,
+                      _ForwardIterator,
+                      _Generator,
+                      /* is_vector = */ std::false_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Generator>
+void __pattern_generate(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Generator) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Generator>
+_RandomAccessIterator __pattern_generate(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Generator);
+
+template <class _RandomAccessIterator, class Size, class _Generator>
+_RandomAccessIterator __brick_generate_n(
+    _RandomAccessIterator,
+    Size,
+    _Generator,
+    /* is_vector = */ std::true_type) noexcept;
+
+template <class OutputIterator, class Size, class _Generator>
+OutputIterator __brick_generate_n(
+    OutputIterator,
+    Size,
+    _Generator,
+    /* is_vector = */ std::false_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class OutputIterator, class Size, class _Generator>
+OutputIterator __pattern_generate_n(_Tag, _ExecutionPolicy&&, OutputIterator, Size, _Generator) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class Size, class _Generator>
+_RandomAccessIterator
+__pattern_generate_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, Size, _Generator);
+
+//------------------------------------------------------------------------
+// remove
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator __brick_remove_if(
+    _ForwardIterator,
+    _ForwardIterator,
+    _UnaryPredicate,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __brick_remove_if(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _UnaryPredicate,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator
+__pattern_remove_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __pattern_remove_if(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _UnaryPredicate) noexcept;
+
+//------------------------------------------------------------------------
+// merge
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_merge(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_merge(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_merge(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_merge(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// inplace_merge
+//------------------------------------------------------------------------
+
+template <class _BidirectionalIterator, class _Compare>
+void __brick_inplace_merge(
+    _BidirectionalIterator,
+    _BidirectionalIterator,
+    _BidirectionalIterator,
+    _Compare,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _Compare>
+void __brick_inplace_merge(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
+void __pattern_inplace_merge(
+    _Tag,
+    _ExecutionPolicy&&,
+    _BidirectionalIterator,
+    _BidirectionalIterator,
+    _BidirectionalIterator,
+    _Compare) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_inplace_merge(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// includes
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+bool __pattern_includes(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+bool __pattern_includes(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Compare);
+
+//------------------------------------------------------------------------
+// set_union
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_union(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_union(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_union(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_union(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// set_intersection
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_intersection(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_intersection(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_intersection(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_intersection(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// set_difference
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_difference(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_difference(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_difference(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_difference(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// set_symmetric_difference
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_symmetric_difference(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_symmetric_difference(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_symmetric_difference(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _OutputIterator,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_symmetric_difference(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _OutputIterator,
+    _Compare);
+
+//------------------------------------------------------------------------
+// is_heap_until
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __brick_is_heap_until(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __brick_is_heap_until(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator
+__pattern_is_heap_until(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __pattern_is_heap_until(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
+
+//------------------------------------------------------------------------
+// min_element
+//------------------------------------------------------------------------
+
+template <typename _ForwardIterator, typename _Compare>
+_ForwardIterator __brick_min_element(
+    _ForwardIterator,
+    _ForwardIterator,
+    _Compare,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <typename _RandomAccessIterator, typename _Compare>
+_RandomAccessIterator __brick_min_element(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
+_ForwardIterator __pattern_min_element(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Compare) noexcept;
+
+template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
+_RandomAccessIterator __pattern_min_element(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare);
+
+//------------------------------------------------------------------------
+// minmax_element
+//------------------------------------------------------------------------
+
+template <typename _ForwardIterator, typename _Compare>
+std::pair<_ForwardIterator, _ForwardIterator> __brick_minmax_element(
+    _ForwardIterator,
+    _ForwardIterator,
+    _Compare,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <typename _RandomAccessIterator, typename _Compare>
+std::pair<_RandomAccessIterator, _RandomAccessIterator> __brick_minmax_element(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Compare,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
+std::pair<_ForwardIterator, _ForwardIterator>
+__pattern_minmax_element(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Compare) noexcept;
+
+template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
+std::pair<_RandomAccessIterator, _RandomAccessIterator> __pattern_minmax_element(
+    __parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare);
+
+//------------------------------------------------------------------------
+// mismatch
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+std::pair<_ForwardIterator1, _ForwardIterator2> __brick_mismatch(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _Predicate,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate>
+std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __brick_mismatch(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Predicate,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+std::pair<_ForwardIterator1, _ForwardIterator2> __pattern_mismatch(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _Predicate) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Predicate>
+std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __pattern_mismatch(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Predicate) noexcept;
+
+//------------------------------------------------------------------------
+// lexicographical_compare
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+bool __brick_lexicographical_compare(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _Compare,
+    /* __is_vector = */ std::false_type) noexcept;
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Compare>
+bool __brick_lexicographical_compare(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Compare,
+    /* __is_vector = */ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+bool __pattern_lexicographical_compare(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _ForwardIterator2,
+    _Compare) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+bool __pattern_lexicographical_compare(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _RandomAccessIterator2,
+    _Compare) noexcept;
+
+} // namespace __internal
+} // namespace __pstl
+
+#endif /* _PSTL_ALGORITHM_FWD_H */
diff --git a/libcxx/include/__pstl/internal/algorithm_impl.h b/libcxx/include/__pstl/internal/algorithm_impl.h
new file mode 100644
index 0000000000000..1f51651065f2a
--- /dev/null
+++ b/libcxx/include/__pstl/internal/algorithm_impl.h
@@ -0,0 +1,4257 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_ALGORITHM_IMPL_H
+#define _PSTL_ALGORITHM_IMPL_H
+
+#include <__assert>
+#include <algorithm>
+#include <functional>
+#include <iterator>
+#include <type_traits>
+#include <utility>
+
+#include "execution_impl.h"
+#include "memory_impl.h"
+#include "parallel_backend.h"
+#include "parallel_backend_utils.h"
+#include "parallel_impl.h"
+#include "pstl_config.h"
+#include "unseq_backend_simd.h"
+
+namespace __pstl {
+namespace __internal {
+
+// [alg.foreach]
+// for_each_n with no policy
+
+template <class _ForwardIterator, class _Size, class _Function>
+_ForwardIterator __for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f) {
+  for (; __n > 0; ++__first, --__n)
+    __f(__first);
+  return __first;
+}
+
+//------------------------------------------------------------------------
+// walk1 (pseudo)
+//
+// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _Function>
+void __brick_walk1(
+    _ForwardIterator __first, _ForwardIterator __last, _Function __f, /*vector=*/std::false_type) noexcept {
+  std::for_each(__first, __last, __f);
+}
+
+template <class _RandomAccessIterator, class _Function>
+void __brick_walk1(_RandomAccessIterator __first,
+                   _RandomAccessIterator __last,
+                   _Function __f,
+                   /*vector=*/std::true_type) noexcept {
+  __unseq_backend::__simd_walk_1(__first, __last - __first, __f);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Function>
+void __pattern_walk1(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __f) noexcept {
+  __internal::__brick_walk1(__first, __last, __f, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Function>
+void __pattern_walk1(__parallel_tag<_IsVector> __tag,
+                     _ExecutionPolicy&& __exec,
+                     _RandomAccessIterator __first,
+                     _RandomAccessIterator __last,
+                     _Function __f) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        [__f](_RandomAccessIterator __i, _RandomAccessIterator __j) {
+          __internal::__brick_walk1(__i, __j, __f, _IsVector{});
+        });
+  });
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Brick>
+void __pattern_walk_brick(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Brick __brick) noexcept {
+  __brick(__first, __last);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Brick>
+void __pattern_walk_brick(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Brick __brick) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j); });
+  });
+}
+
+//------------------------------------------------------------------------
+// walk1_n
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _Size, class _Function>
+_ForwardIterator __brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type) {
+  return __internal::__for_each_n_it_serial(__first, __n, [&__f](_ForwardIterator __it) {
+    __f(*__it);
+  }); // calling serial version
+}
+
+template <class _RandomAccessIterator, class _DifferenceType, class _Function>
+_RandomAccessIterator
+__brick_walk1_n(_RandomAccessIterator __first,
+                _DifferenceType __n,
+                _Function __f,
+                /*vectorTag=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_walk_1(__first, __n, __f);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
+_ForwardIterator
+__pattern_walk1_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f) noexcept {
+  return __internal::__brick_walk1_n(__first, __n, __f, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function>
+_RandomAccessIterator __pattern_walk1_n(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _Size __n,
+    _Function __f) {
+  __internal::__pattern_walk1(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f);
+
+  return __first + __n;
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
+_ForwardIterator
+__pattern_walk_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Brick __brick) noexcept {
+  return __brick(__first, __n);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
+_RandomAccessIterator __pattern_walk_brick_n(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _Size __n,
+    _Brick __brick) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __first + __n,
+        [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); });
+    return __first + __n;
+  });
+}
+
+//------------------------------------------------------------------------
+// walk2 (pseudo)
+//
+// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
+//------------------------------------------------------------------------
+template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
+_ForwardIterator2
+__brick_walk2(_ForwardIterator1 __first1,
+              _ForwardIterator1 __last1,
+              _ForwardIterator2 __first2,
+              _Function __f,
+              /*vector=*/std::false_type) noexcept {
+  for (; __first1 != __last1; ++__first1, ++__first2)
+    __f(*__first1, *__first2);
+  return __first2;
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Function>
+_RandomAccessIterator2
+__brick_walk2(_RandomAccessIterator1 __first1,
+              _RandomAccessIterator1 __last1,
+              _RandomAccessIterator2 __first2,
+              _Function __f,
+              /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_walk_2(__first1, __last1 - __first1, __first2, __f);
+}
+
+template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
+_ForwardIterator2 __brick_walk2_n(
+    _ForwardIterator1 __first1,
+    _Size __n,
+    _ForwardIterator2 __first2,
+    _Function __f,
+    /*vector=*/std::false_type) noexcept {
+  for (; __n > 0; --__n, ++__first1, ++__first2)
+    __f(*__first1, *__first2);
+  return __first2;
+}
+
+template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Function>
+_RandomAccessIterator2 __brick_walk2_n(
+    _RandomAccessIterator1 __first1,
+    _Size __n,
+    _RandomAccessIterator2 __first2,
+    _Function __f,
+    /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_walk_2(__first1, __n, __first2, __f);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function>
+_ForwardIterator2 __pattern_walk2(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Function __f) noexcept {
+  return __internal::__brick_walk2(__first1, __last1, __first2, __f, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Function>
+_RandomAccessIterator2 __pattern_walk2(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _Function __f) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        [__f, __first1, __first2](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          __internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, _IsVector{});
+        });
+    return __first2 + (__last1 - __first1);
+  });
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _Size,
+          class _ForwardIterator2,
+          class _Function>
+_ForwardIterator2 __pattern_walk2_n(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _Size __n,
+    _ForwardIterator2 __first2,
+    _Function __f) noexcept {
+  return __internal::__brick_walk2_n(__first1, __n, __first2, __f, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _Size,
+          class _RandomAccessIterator2,
+          class _Function>
+_RandomAccessIterator2 __pattern_walk2_n(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _Size __n,
+    _RandomAccessIterator2 __first2,
+    _Function __f) {
+  return __internal::__pattern_walk2(
+      __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n, __first2, __f);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
+_ForwardIterator2 __pattern_walk2_brick(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Brick __brick) noexcept {
+  return __brick(__first1, __last1, __first2);
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Brick>
+_RandomAccessIterator2 __pattern_walk2_brick(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _Brick __brick) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          __brick(__i, __j, __first2 + (__i - __first1));
+        });
+    return __first2 + (__last1 - __first1);
+  });
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _Size,
+          class _ForwardIterator2,
+          class _Brick>
+_ForwardIterator2 __pattern_walk2_brick_n(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _Size __n,
+    _ForwardIterator2 __first2,
+    _Brick __brick) noexcept {
+  return __brick(__first1, __n, __first2);
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _Size,
+          class _RandomAccessIterator2,
+          class _Brick>
+_RandomAccessIterator2 __pattern_walk2_brick_n(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _Size __n,
+    _RandomAccessIterator2 __first2,
+    _Brick __brick) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __first1 + __n,
+        [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          __brick(__i, __j - __i, __first2 + (__i - __first1));
+        });
+    return __first2 + __n;
+  });
+}
+
+//------------------------------------------------------------------------
+// walk3 (pseudo)
+//
+// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
+//------------------------------------------------------------------------
+template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
+_ForwardIterator3 __brick_walk3(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator3 __first3,
+    _Function __f,
+    /*vector=*/std::false_type) noexcept {
+  for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
+    __f(*__first1, *__first2, *__first3);
+  return __first3;
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
+_RandomAccessIterator3 __brick_walk3(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator3 __first3,
+    _Function __f,
+    /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_walk_3(__first1, __last1 - __first1, __first2, __first3, __f);
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator3,
+          class _Function>
+_ForwardIterator3 __pattern_walk3(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator3 __first3,
+    _Function __f) noexcept {
+  return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _Function>
+_RandomAccessIterator3 __pattern_walk3(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator3 __first3,
+    _Function __f) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        [__f, __first1, __first2, __first3](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          __internal::__brick_walk3(
+              __i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f, _IsVector{});
+        });
+    return __first3 + (__last1 - __first1);
+  });
+}
+
+//------------------------------------------------------------------------
+// equal
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __brick_equal(_ForwardIterator1 __first1,
+                   _ForwardIterator1 __last1,
+                   _ForwardIterator2 __first2,
+                   _ForwardIterator2 __last2,
+                   _BinaryPredicate __p,
+                   /* IsVector = */ std::false_type) noexcept {
+  return std::equal(__first1, __last1, __first2, __last2, __p);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+bool __brick_equal(_RandomAccessIterator1 __first1,
+                   _RandomAccessIterator1 __last1,
+                   _RandomAccessIterator2 __first2,
+                   _RandomAccessIterator2 __last2,
+                   _BinaryPredicate __p,
+                   /* is_vector = */ std::true_type) noexcept {
+  if (__last1 - __first1 != __last2 - __first2)
+    return false;
+
+  return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __pattern_equal(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate __p) noexcept {
+  return __internal::__brick_equal(__first1, __last1, __first2, __last2, __p, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+bool __pattern_equal(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _BinaryPredicate __p) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__last1 - __first1 != __last2 - __first2)
+    return false;
+
+  return __internal::__except_handler([&]() {
+    return !__internal::__parallel_or(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        [__first1, __first2, __p](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          return !__internal::__brick_equal(
+              __i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1), __p, _IsVector{});
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// equal version for sequences with equal length
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __brick_equal(_ForwardIterator1 __first1,
+                   _ForwardIterator1 __last1,
+                   _ForwardIterator2 __first2,
+                   _BinaryPredicate __p,
+                   /* IsVector = */ std::false_type) noexcept {
+  return std::equal(__first1, __last1, __first2, __p);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+bool __brick_equal(_RandomAccessIterator1 __first1,
+                   _RandomAccessIterator1 __last1,
+                   _RandomAccessIterator2 __first2,
+                   _BinaryPredicate __p,
+                   /* is_vector = */ std::true_type) noexcept {
+  return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+bool __pattern_equal(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _BinaryPredicate __p) noexcept {
+  return __internal::__brick_equal(__first1, __last1, __first2, __p, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+bool __pattern_equal(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _BinaryPredicate __p) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    return !__internal::__parallel_or(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        [__first1, __first2, __p](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, _IsVector{});
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// find_end
+//------------------------------------------------------------------------
+
+// find the first occurrence of the subsequence [s_first, s_last)
+//   or the  last occurrence of the subsequence in the range [first, last)
+// b_first determines what occurrence we want to find (first or last)
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate, class _IsVector>
+_RandomAccessIterator1 __find_subrange(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator1 __global_last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred,
+    bool __b_first,
+    _IsVector __is_vector) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _ValueType;
+  auto __n2 = __s_last - __s_first;
+  if (__n2 < 1) {
+    return __b_first ? __first : __last;
+  }
+
+  auto __n1 = __global_last - __first;
+  if (__n1 < __n2) {
+    return __last;
+  }
+
+  auto __cur = __last;
+  while (__first != __last && (__global_last - __first >= __n2)) {
+    // find position of *s_first in [first, last) (it can be start of subsequence)
+    __first = __internal::__brick_find_if(
+        __first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector);
+
+    // if position that was found previously is the start of subsequence
+    // then we can exit the loop (b_first == true) or keep the position
+    // (b_first == false)
+    if (__first != __last && (__global_last - __first >= __n2) &&
+        __internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector)) {
+      if (__b_first) {
+        return __first;
+      } else {
+        __cur = __first;
+      }
+    } else if (__first == __last) {
+      break;
+    } else {
+    }
+
+    // in case of b_first == false we try to find new start position
+    // for the next subsequence
+    ++__first;
+  }
+  return __cur;
+}
+
+template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate, class _IsVector>
+_RandomAccessIterator __find_subrange(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _RandomAccessIterator __global_last,
+    _Size __count,
+    const _Tp& __value,
+    _BinaryPredicate __pred,
+    _IsVector __is_vector) noexcept {
+  if (static_cast<_Size>(__global_last - __first) < __count || __count < 1) {
+    return __last; // According to the standard last shall be returned when count < 1
+  }
+
+  auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred);
+  while (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count)) {
+    __first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector);
+
+    // check that all of elements in [first+1, first+count) equal to value
+    if (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count) &&
+        !__internal::__brick_any_of(__first + 1, __first + __count, std::not_fn(__unary_pred), __is_vector)) {
+      return __first;
+    } else if (__first == __last) {
+      break;
+    } else {
+      ++__first;
+    }
+  }
+  return __last;
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __brick_find_end(
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::find_end(__first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_find_end(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred,
+    /*__is_vector=*/std::true_type) noexcept {
+  return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, false, std::true_type());
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __pattern_find_end(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred) noexcept {
+  return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator1 __pattern_find_end(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__last - __first == __s_last - __s_first) {
+    const bool __res =
+        __internal::__pattern_equal(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __pred);
+    return __res ? __first : __last;
+  } else {
+    return __internal::__except_handler([&]() {
+      return __internal::__parallel_find(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __first,
+          __last,
+          [__last, __s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+            return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false, _IsVector{});
+          },
+          std::greater<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
+          /*is_first=*/false);
+    });
+  }
+}
+
+//------------------------------------------------------------------------
+// find_first_of
+//------------------------------------------------------------------------
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __brick_find_first_of(
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::find_first_of(__first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_find_first_of(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred,
+    /*__is_vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_find_first_of(__first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __pattern_find_first_of(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred) noexcept {
+  return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator1 __pattern_find_first_of(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    return __internal::__parallel_find(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        [__s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, _IsVector{});
+        },
+        std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
+        /*is_first=*/true);
+  });
+}
+
+//------------------------------------------------------------------------
+// search
+//------------------------------------------------------------------------
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_search(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred,
+    /*vector=*/std::false_type) noexcept {
+  return std::search(__first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator1 __brick_search(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred,
+    /*vector=*/std::true_type) noexcept {
+  return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type());
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+_ForwardIterator1 __pattern_search(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred) noexcept {
+  return __internal::__brick_search(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator1 __pattern_search(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __s_first,
+    _RandomAccessIterator2 __s_last,
+    _BinaryPredicate __pred) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__last - __first == __s_last - __s_first) {
+    const bool __res =
+        __internal::__pattern_equal(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __pred);
+    return __res ? __first : __last;
+  } else {
+    return __internal::__except_handler([&]() {
+      return __internal::__parallel_find(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __first,
+          __last,
+          [__last, __s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+            return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true, _IsVector{});
+          },
+          std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
+          /*is_first=*/true);
+    });
+  }
+}
+
+//------------------------------------------------------------------------
+// search_n
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
+_ForwardIterator __brick_search_n(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Size __count,
+    const _Tp& __value,
+    _BinaryPredicate __pred,
+    /*vector=*/std::false_type) noexcept {
+  return std::search_n(__first, __last, __count, __value, __pred);
+}
+
+template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate>
+_RandomAccessIterator __brick_search_n(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Size __count,
+    const _Tp& __value,
+    _BinaryPredicate __pred,
+    /*vector=*/std::true_type) noexcept {
+  return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type());
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
+_ForwardIterator __pattern_search_n(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Size __count,
+    const _Tp& __value,
+    _BinaryPredicate __pred) noexcept {
+  return __internal::__brick_search_n(__first, __last, __count, __value, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _Size,
+          class _Tp,
+          class _BinaryPredicate>
+_RandomAccessIterator __pattern_search_n(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Size __count,
+    const _Tp& __value,
+    _BinaryPredicate __pred) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (static_cast<_Size>(__last - __first) == __count) {
+    const bool __result = !__internal::__pattern_any_of(
+        __tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [&__value, &__pred](const _Tp& __val) {
+          return !__pred(__val, __value);
+        });
+    return __result ? __first : __last;
+  } else {
+    return __internal::__except_handler([&__exec, __first, __last, __count, &__value, __pred]() {
+      return __internal::__parallel_find(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __first,
+          __last,
+          [__last, __count, &__value, __pred](_RandomAccessIterator __i, _RandomAccessIterator __j) {
+            return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, _IsVector{});
+          },
+          std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(),
+          /*is_first=*/true);
+    });
+  }
+}
+
+//------------------------------------------------------------------------
+// copy_n
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _Size, class _OutputIterator>
+_OutputIterator
+__brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::false_type) noexcept {
+  return std::copy_n(__first, __n, __result);
+}
+
+template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2>
+_RandomAccessIterator2
+__brick_copy_n(_RandomAccessIterator1 __first,
+               _Size __n,
+               _RandomAccessIterator2 __result,
+               /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_assign(
+      __first, __n, __result, [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) {
+        *__result = *__first;
+      });
+}
+
+//------------------------------------------------------------------------
+// copy
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator
+__brick_copy(_ForwardIterator __first,
+             _ForwardIterator __last,
+             _OutputIterator __result,
+             /*vector=*/std::false_type) noexcept {
+  return std::copy(__first, __last, __result);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2
+__brick_copy(_RandomAccessIterator1 __first,
+             _RandomAccessIterator1 __last,
+             _RandomAccessIterator2 __result,
+             /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_assign(
+      __first, __last - __first, __result, [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) {
+        *__result = *__first;
+      });
+}
+
+//------------------------------------------------------------------------
+// move
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator
+__brick_move(_ForwardIterator __first,
+             _ForwardIterator __last,
+             _OutputIterator __result,
+             /*vector=*/std::false_type) noexcept {
+  return std::move(__first, __last, __result);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2
+__brick_move(_RandomAccessIterator1 __first,
+             _RandomAccessIterator1 __last,
+             _RandomAccessIterator2 __result,
+             /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_assign(
+      __first, __last - __first, __result, [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) {
+        *__result = std::move(*__first);
+      });
+}
+
+struct __brick_move_destroy {
+  template <typename _RandomAccessIterator1, typename _RandomAccessIterator2>
+  _RandomAccessIterator2
+  operator()(_RandomAccessIterator1 __first,
+             _RandomAccessIterator1 __last,
+             _RandomAccessIterator2 __result,
+             /*vec*/ std::true_type) const {
+    using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
+
+    return __unseq_backend::__simd_assign(
+        __first, __last - __first, __result, [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) {
+          *__result = std::move(*__first);
+          (*__first).~_IteratorValueType();
+        });
+  }
+
+  template <typename _RandomAccessIterator1, typename _RandomAccessIterator2>
+  _RandomAccessIterator2
+  operator()(_RandomAccessIterator1 __first,
+             _RandomAccessIterator1 __last,
+             _RandomAccessIterator2 __result,
+             /*vec*/ std::false_type) const {
+    using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
+
+    for (; __first != __last; ++__first, ++__result) {
+      *__result = std::move(*__first);
+      (*__first).~_IteratorValueType();
+    }
+    return __result;
+  }
+};
+
+//------------------------------------------------------------------------
+// swap_ranges
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator __brick_swap_ranges(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    /*vector=*/std::false_type) noexcept {
+  return std::swap_ranges(__first, __last, __result);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2 __brick_swap_ranges(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    /*vector=*/std::true_type) noexcept {
+  using std::iter_swap;
+  return __unseq_backend::__simd_assign(
+      __first, __last - __first, __result, iter_swap<_RandomAccessIterator1, _RandomAccessIterator2>);
+}
+
+//------------------------------------------------------------------------
+// copy_if
+//------------------------------------------------------------------------
+template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
+_OutputIterator __brick_copy_if(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    _UnaryPredicate __pred,
+    /*vector=*/std::false_type) noexcept {
+  return std::copy_if(__first, __last, __result, __pred);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _UnaryPredicate>
+_RandomAccessIterator2 __brick_copy_if(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    _UnaryPredicate __pred,
+    /*vector=*/std::true_type) noexcept {
+#if defined(_PSTL_MONOTONIC_PRESENT)
+  return __unseq_backend::__simd_copy_if(__first, __last - __first, __result, __pred);
+#else
+  return std::copy_if(__first, __last, __result, __pred);
+#endif
+}
+
+// TODO: Try to use transform_reduce for combining __brick_copy_if_phase1 on IsVector.
+template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
+std::pair<_DifferenceType, _DifferenceType> __brick_calc_mask_1(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    bool* __restrict __mask,
+    _UnaryPredicate __pred,
+    /*vector=*/std::false_type) noexcept {
+  auto __count_true = _DifferenceType(0);
+  auto __size       = __last - __first;
+
+  static_assert(__are_random_access_iterators<_ForwardIterator>::value,
+                "Pattern-brick error. Should be a random access iterator.");
+
+  for (; __first != __last; ++__first, ++__mask) {
+    *__mask = __pred(*__first);
+    if (*__mask) {
+      ++__count_true;
+    }
+  }
+  return std::make_pair(__count_true, __size - __count_true);
+}
+
+template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
+std::pair<_DifferenceType, _DifferenceType> __brick_calc_mask_1(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    bool* __mask,
+    _UnaryPredicate __pred,
+    /*vector=*/std::true_type) noexcept {
+  auto __result = __unseq_backend::__simd_calc_mask_1(__first, __last - __first, __mask, __pred);
+  return std::make_pair(__result, (__last - __first) - __result);
+}
+
+template <class _ForwardIterator, class _OutputIterator, class _Assigner>
+void __brick_copy_by_mask(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    bool* __mask,
+    _Assigner __assigner,
+    /*vector=*/std::false_type) noexcept {
+  for (; __first != __last; ++__first, ++__mask) {
+    if (*__mask) {
+      __assigner(__first, __result);
+      ++__result;
+    }
+  }
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Assigner>
+void __brick_copy_by_mask(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    bool* __restrict __mask,
+    _Assigner __assigner,
+    /*vector=*/std::true_type) noexcept {
+#if defined(_PSTL_MONOTONIC_PRESENT)
+  __unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner);
+#else
+  __internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type());
+#endif
+}
+
+template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
+void __brick_partition_by_mask(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator1 __out_true,
+    _OutputIterator2 __out_false,
+    bool* __mask,
+    /*vector=*/std::false_type) noexcept {
+  for (; __first != __last; ++__first, ++__mask) {
+    if (*__mask) {
+      *__out_true = *__first;
+      ++__out_true;
+    } else {
+      *__out_false = *__first;
+      ++__out_false;
+    }
+  }
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3>
+void __brick_partition_by_mask(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __out_true,
+    _RandomAccessIterator3 __out_false,
+    bool* __mask,
+    /*vector=*/std::true_type) noexcept {
+#if defined(_PSTL_MONOTONIC_PRESENT)
+  __unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask);
+#else
+  __internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type());
+#endif
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
+_OutputIterator __pattern_copy_if(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    _UnaryPredicate __pred) noexcept {
+  return __internal::__brick_copy_if(__first, __last, __result, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _UnaryPredicate>
+_RandomAccessIterator2 __pattern_copy_if(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    _UnaryPredicate __pred) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
+  const _DifferenceType __n = __last - __first;
+  if (_DifferenceType(1) < __n) {
+    __par_backend::__buffer<bool> __mask_buf(__n);
+    return __internal::__except_handler([&__exec, __n, __first, __result, __pred, &__mask_buf]() {
+      bool* __mask = __mask_buf.get();
+      _DifferenceType __m{};
+      __par_backend::__parallel_strict_scan(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __n,
+          _DifferenceType(0),
+          [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
+            return __internal::__brick_calc_mask_1<_DifferenceType>(
+                       __first + __i, __first + (__i + __len), __mask + __i, __pred, _IsVector{})
+                .first;
+          },
+          std::plus<_DifferenceType>(),                                                // Combine
+          [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
+            __internal::__brick_copy_by_mask(
+                __first + __i,
+                __first + (__i + __len),
+                __result + __initial,
+                __mask + __i,
+                [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = *__x; },
+                _IsVector{});
+          },
+          [&__m](_DifferenceType __total) { __m = __total; });
+      return __result + __m;
+    });
+  }
+  // trivial sequence - use serial algorithm
+  return __internal::__brick_copy_if(__first, __last, __result, __pred, _IsVector{});
+}
+
+//------------------------------------------------------------------------
+// count
+//------------------------------------------------------------------------
+template <class _RandomAccessIterator, class _Predicate>
+typename std::iterator_traits<_RandomAccessIterator>::difference_type
+__brick_count(_RandomAccessIterator __first,
+              _RandomAccessIterator __last,
+              _Predicate __pred,
+              /* is_vector = */ std::true_type) noexcept {
+  return __unseq_backend::__simd_count(__first, __last - __first, __pred);
+}
+
+template <class _ForwardIterator, class _Predicate>
+typename std::iterator_traits<_ForwardIterator>::difference_type
+__brick_count(_ForwardIterator __first,
+              _ForwardIterator __last,
+              _Predicate __pred,
+              /* is_vector = */ std::false_type) noexcept {
+  return std::count_if(__first, __last, __pred);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
+typename std::iterator_traits<_ForwardIterator>::difference_type __pattern_count(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) noexcept {
+  return __internal::__brick_count(__first, __last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
+typename std::iterator_traits<_RandomAccessIterator>::difference_type __pattern_count(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Predicate __pred) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
+  return __internal::__except_handler([&]() {
+    return __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        _SizeType(0),
+        [__pred](_RandomAccessIterator __begin, _RandomAccessIterator __end, _SizeType __value) -> _SizeType {
+          return __value + __internal::__brick_count(__begin, __end, __pred, _IsVector{});
+        },
+        std::plus<_SizeType>());
+  });
+}
+
+//------------------------------------------------------------------------
+// unique
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator
+__brick_unique(_RandomAccessIterator __first,
+               _RandomAccessIterator __last,
+               _BinaryPredicate __pred,
+               /*is_vector=*/std::false_type) noexcept {
+  return std::unique(__first, __last, __pred);
+}
+
+template <class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator
+__brick_unique(_RandomAccessIterator __first,
+               _RandomAccessIterator __last,
+               _BinaryPredicate __pred,
+               /*is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::unique(__first, __last, __pred);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator __pattern_unique(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred) noexcept {
+  return __internal::__brick_unique(__first, __last, __pred, typename _Tag::__is_vector{});
+}
+
+// That function is shared between two algorithms - remove_if (__pattern_remove_if) and unique (pattern unique). But a
+// mask calculation is different. So, a caller passes _CalcMask brick into remove_elements.
+template <class _IsVector, class _ExecutionPolicy, class _ForwardIterator, class _CalcMask>
+_ForwardIterator __remove_elements(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _CalcMask __calc_mask) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType;
+  typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
+  _DifferenceType __n = __last - __first;
+  __par_backend::__buffer<bool> __mask_buf(__n);
+  // 1. find a first iterator that should be removed
+  return __internal::__except_handler([&]() {
+    bool* __mask          = __mask_buf.get();
+    _DifferenceType __min = __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        _DifferenceType(0),
+        __n,
+        __n,
+        [__first, __mask, &__calc_mask](
+            _DifferenceType __i, _DifferenceType __j, _DifferenceType __local_min) -> _DifferenceType {
+          // Create mask
+          __calc_mask(__mask + __i, __mask + __j, __first + __i);
+
+          // if minimum was found in a previous range we shouldn't do anymore
+          if (__local_min < __i) {
+            return __local_min;
+          }
+          // find first iterator that should be removed
+          bool* __result = __internal::__brick_find_if(
+              __mask + __i, __mask + __j, [](bool __val) { return !__val; }, _IsVector{});
+          if (__result - __mask == __j) {
+            return __local_min;
+          }
+          return std::min(__local_min, _DifferenceType(__result - __mask));
+        },
+        [](_DifferenceType __local_min1, _DifferenceType __local_min2) -> _DifferenceType {
+          return std::min(__local_min1, __local_min2);
+        });
+
+    // No elements to remove - exit
+    if (__min == __n) {
+      return __last;
+    }
+    __n -= __min;
+    __first += __min;
+
+    __par_backend::__buffer<_Tp> __buf(__n);
+    _Tp* __result = __buf.get();
+    __mask += __min;
+    _DifferenceType __m{};
+    // 2. Elements that doesn't satisfy pred are moved to result
+    __par_backend::__parallel_strict_scan(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __n,
+        _DifferenceType(0),
+        [__mask](_DifferenceType __i, _DifferenceType __len) {
+          return __internal::__brick_count(
+              __mask + __i, __mask + __i + __len, [](bool __val) { return __val; }, _IsVector{});
+        },
+        std::plus<_DifferenceType>(),
+        [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) {
+          __internal::__brick_copy_by_mask(
+              __first + __i,
+              __first + __i + __len,
+              __result + __initial,
+              __mask + __i,
+              [](_ForwardIterator __x, _Tp* __z) {
+                __internal::__invoke_if_else(
+                    std::is_trivial<_Tp>(),
+                    [&]() { *__z = std::move(*__x); },
+                    [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
+              },
+              _IsVector{});
+        },
+        [&__m](_DifferenceType __total) { __m = __total; });
+
+    // 3. Elements from result are moved to [first, last)
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __result,
+        __result + __m,
+        [__result, __first](_Tp* __i, _Tp* __j) {
+          __invoke_if_else(
+              std::is_trivial<_Tp>(),
+              [&]() { __brick_move(__i, __j, __first + (__i - __result), _IsVector{}); },
+              [&]() { __brick_move_destroy()(__i, __j, __first + (__i - __result), _IsVector{}); });
+        });
+    return __first + __m;
+  });
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __pattern_unique(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _BinaryPredicate __pred) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
+
+  if (__first == __last) {
+    return __last;
+  }
+  if (__first + 1 == __last || __first + 2 == __last) {
+    // Trivial sequence - use serial algorithm
+    return __internal::__brick_unique(__first, __last, __pred, _IsVector{});
+  }
+  return __internal::__remove_elements(
+      __tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      ++__first,
+      __last,
+      [&__pred](bool* __b, bool* __e, _RandomAccessIterator __it) {
+        __internal::__brick_walk3(
+            __b,
+            __e,
+            __it - 1,
+            __it,
+            [&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); },
+            _IsVector{});
+      });
+}
+
+//------------------------------------------------------------------------
+// unique_copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
+OutputIterator __brick_unique_copy(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    OutputIterator __result,
+    _BinaryPredicate __pred,
+    /*vector=*/std::false_type) noexcept {
+  return std::unique_copy(__first, __last, __result, __pred);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
+_RandomAccessIterator2 __brick_unique_copy(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    _BinaryPredicate __pred,
+    /*vector=*/std::true_type) noexcept {
+#if defined(_PSTL_MONOTONIC_PRESENT)
+  return __unseq_backend::__simd_unique_copy(__first, __last - __first, __result, __pred);
+#else
+  return std::unique_copy(__first, __last, __result, __pred);
+#endif
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate>
+_OutputIterator __pattern_unique_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    _BinaryPredicate __pred) noexcept {
+  return __internal::__brick_unique_copy(__first, __last, __result, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
+_DifferenceType __brick_calc_mask_2(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    bool* __restrict __mask,
+    _BinaryPredicate __pred,
+    /*vector=*/std::false_type) noexcept {
+  _DifferenceType __count = 0;
+  for (; __first != __last; ++__first, ++__mask) {
+    *__mask = !__pred(*__first, *(__first - 1));
+    __count += *__mask;
+  }
+  return __count;
+}
+
+template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
+_DifferenceType __brick_calc_mask_2(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    bool* __restrict __mask,
+    _BinaryPredicate __pred,
+    /*vector=*/std::true_type) noexcept {
+  return __unseq_backend::__simd_calc_mask_2(__first, __last - __first, __mask, __pred);
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryPredicate>
+_RandomAccessIterator2 __pattern_unique_copy(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    _BinaryPredicate __pred) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
+  const _DifferenceType __n = __last - __first;
+  if (_DifferenceType(2) < __n) {
+    __par_backend::__buffer<bool> __mask_buf(__n);
+    if (_DifferenceType(2) < __n) {
+      return __internal::__except_handler([&__exec, __n, __first, __result, __pred, &__mask_buf]() {
+        bool* __mask = __mask_buf.get();
+        _DifferenceType __m{};
+        __par_backend::__parallel_strict_scan(
+            __backend_tag{},
+            std::forward<_ExecutionPolicy>(__exec),
+            __n,
+            _DifferenceType(0),
+            [=](_DifferenceType __i, _DifferenceType __len) -> _DifferenceType { // Reduce
+              _DifferenceType __extra = 0;
+              if (__i == 0) {
+                // Special boundary case
+                __mask[__i] = true;
+                if (--__len == 0)
+                  return 1;
+                ++__i;
+                ++__extra;
+              }
+              return __internal::__brick_calc_mask_2<_DifferenceType>(
+                         __first + __i, __first + (__i + __len), __mask + __i, __pred, _IsVector{}) +
+                     __extra;
+            },
+            std::plus<_DifferenceType>(),                                                // Combine
+            [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
+              // Phase 2 is same as for __pattern_copy_if
+              __internal::__brick_copy_by_mask(
+                  __first + __i,
+                  __first + (__i + __len),
+                  __result + __initial,
+                  __mask + __i,
+                  [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = *__x; },
+                  _IsVector{});
+            },
+            [&__m](_DifferenceType __total) { __m = __total; });
+        return __result + __m;
+      });
+    }
+  }
+  // trivial sequence - use serial algorithm
+  return __internal::__brick_unique_copy(__first, __last, __result, __pred, _IsVector{});
+}
+
+//------------------------------------------------------------------------
+// reverse
+//------------------------------------------------------------------------
+template <class _BidirectionalIterator>
+void __brick_reverse(
+    _BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::false_type) noexcept {
+  std::reverse(__first, __last);
+}
+
+template <class _RandomAccessIterator>
+void __brick_reverse(
+    _RandomAccessIterator __first, _RandomAccessIterator __last, /*__is_vector=*/std::true_type) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
+
+  const auto __n = (__last - __first) / 2;
+  __unseq_backend::__simd_walk_2(
+      __first, __n, std::reverse_iterator<_RandomAccessIterator>(__last), [](_ReferenceType __x, _ReferenceType __y) {
+        using std::swap;
+        swap(__x, __y);
+      });
+}
+
+// this brick is called in parallel version, so we can use iterator arithmetic
+template <class _BidirectionalIterator>
+void __brick_reverse(_BidirectionalIterator __first,
+                     _BidirectionalIterator __last,
+                     _BidirectionalIterator __d_last,
+                     /*is_vector=*/std::false_type) noexcept {
+  for (--__d_last; __first != __last; ++__first, --__d_last) {
+    using std::iter_swap;
+    iter_swap(__first, __d_last);
+  }
+}
+
+// this brick is called in parallel version, so we can use iterator arithmetic
+template <class _RandomAccessIterator>
+void __brick_reverse(_RandomAccessIterator __first,
+                     _RandomAccessIterator __last,
+                     _RandomAccessIterator __d_last,
+                     /*is_vector=*/std::true_type) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
+
+  __unseq_backend::__simd_walk_2(
+      __first,
+      __last - __first,
+      std::reverse_iterator<_RandomAccessIterator>(__d_last),
+      [](_ReferenceType __x, _ReferenceType __y) {
+        using std::swap;
+        swap(__x, __y);
+      });
+}
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator>
+void __pattern_reverse(
+    _Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last) noexcept {
+  __internal::__brick_reverse(__first, __last, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
+void __pattern_reverse(__parallel_tag<_IsVector> __tag,
+                       _ExecutionPolicy&& __exec,
+                       _RandomAccessIterator __first,
+                       _RandomAccessIterator __last) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __par_backend::__parallel_for(
+      __backend_tag{},
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __first + (__last - __first) / 2,
+      [__first, __last](_RandomAccessIterator __inner_first, _RandomAccessIterator __inner_last) {
+        __internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), _IsVector{});
+      });
+}
+
+//------------------------------------------------------------------------
+// reverse_copy
+//------------------------------------------------------------------------
+
+template <class _BidirectionalIterator, class _OutputIterator>
+_OutputIterator __brick_reverse_copy(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    _OutputIterator __d_first,
+    /*is_vector=*/std::false_type) noexcept {
+  return std::reverse_copy(__first, __last, __d_first);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2 __brick_reverse_copy(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __d_first,
+    /*is_vector=*/std::true_type) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
+  typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
+
+  return __unseq_backend::__simd_walk_2(
+      std::reverse_iterator<_RandomAccessIterator1>(__last),
+      __last - __first,
+      __d_first,
+      [](_ReferenceType1 __x, _ReferenceType2 __y) { __y = __x; });
+}
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator>
+_OutputIterator __pattern_reverse_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    _OutputIterator __d_first) noexcept {
+  return __internal::__brick_reverse_copy(__first, __last, __d_first, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2 __pattern_reverse_copy(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __d_first) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  auto __len = __last - __first;
+  __par_backend::__parallel_for(
+      __backend_tag{},
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      [__first, __len, __d_first](_RandomAccessIterator1 __inner_first, _RandomAccessIterator1 __inner_last) {
+        __internal::__brick_reverse_copy(
+            __inner_first, __inner_last, __d_first + (__len - (__inner_last - __first)), _IsVector{});
+      });
+  return __d_first + __len;
+}
+
+//------------------------------------------------------------------------
+// rotate
+//------------------------------------------------------------------------
+template <class _ForwardIterator>
+_ForwardIterator
+__brick_rotate(_ForwardIterator __first,
+               _ForwardIterator __middle,
+               _ForwardIterator __last,
+               /*is_vector=*/std::false_type) noexcept {
+#if defined(_PSTL_CPP11_STD_ROTATE_BROKEN)
+  std::rotate(__first, __middle, __last);
+  return std::next(__first, std::distance(__middle, __last));
+#else
+  return std::rotate(__first, __middle, __last);
+#endif
+}
+
+template <class _RandomAccessIterator>
+_RandomAccessIterator
+__brick_rotate(_RandomAccessIterator __first,
+               _RandomAccessIterator __middle,
+               _RandomAccessIterator __last,
+               /*is_vector=*/std::true_type) noexcept {
+  auto __n                          = __last - __first;
+  auto __m                          = __middle - __first;
+  const _RandomAccessIterator __ret = __first + (__last - __middle);
+
+  bool __is_left = (__m <= __n / 2);
+  if (!__is_left)
+    __m = __n - __m;
+
+  while (__n > 1 && __m > 0) {
+    using std::iter_swap;
+    const auto __m_2 = __m * 2;
+    if (__is_left) {
+      for (; __last - __first >= __m_2; __first += __m) {
+        __unseq_backend::__simd_assign(
+            __first, __m, __first + __m, iter_swap<_RandomAccessIterator, _RandomAccessIterator>);
+      }
+    } else {
+      for (; __last - __first >= __m_2; __last -= __m) {
+        __unseq_backend::__simd_assign(
+            __last - __m, __m, __last - __m_2, iter_swap<_RandomAccessIterator, _RandomAccessIterator>);
+      }
+    }
+    __is_left = !__is_left;
+    __m       = __n % __m;
+    __n       = __last - __first;
+  }
+
+  return __ret;
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator>
+_ForwardIterator __pattern_rotate(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) noexcept {
+  return __internal::__brick_rotate(__first, __middle, __last, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
+_RandomAccessIterator __pattern_rotate(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __middle,
+    _RandomAccessIterator __last) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
+  auto __n = __last - __first;
+  auto __m = __middle - __first;
+  if (__m <= __n / 2) {
+    __par_backend::__buffer<_Tp> __buf(__n - __m);
+    return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, &__buf]() {
+      _Tp* __result = __buf.get();
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __middle,
+          __last,
+          [__middle, __result](_RandomAccessIterator __b, _RandomAccessIterator __e) {
+            __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), _IsVector{});
+          });
+
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __first,
+          __middle,
+          [__last, __middle](_RandomAccessIterator __b, _RandomAccessIterator __e) {
+            __internal::__brick_move(__b, __e, __b + (__last - __middle), _IsVector{});
+          });
+
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __result,
+          __result + (__n - __m),
+          [__first, __result](_Tp* __b, _Tp* __e) {
+            __brick_move_destroy()(__b, __e, __first + (__b - __result), _IsVector{});
+          });
+
+      return __first + (__last - __middle);
+    });
+  } else {
+    __par_backend::__buffer<_Tp> __buf(__m);
+    return __internal::__except_handler([&__exec, __n, __m, __first, __middle, __last, &__buf]() {
+      _Tp* __result = __buf.get();
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __first,
+          __middle,
+          [__first, __result](_RandomAccessIterator __b, _RandomAccessIterator __e) {
+            __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __first), _IsVector{});
+          });
+
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __middle,
+          __last,
+          [__first, __middle](_RandomAccessIterator __b, _RandomAccessIterator __e) {
+            __internal::__brick_move(__b, __e, __first + (__b - __middle), _IsVector{});
+          });
+
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __result,
+          __result + __m,
+          [__n, __m, __first, __result](_Tp* __b, _Tp* __e) {
+            __brick_move_destroy()(__b, __e, __first + ((__n - __m) + (__b - __result)), _IsVector{});
+          });
+
+      return __first + (__last - __middle);
+    });
+  }
+}
+
+//------------------------------------------------------------------------
+// rotate_copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator>
+_OutputIterator __brick_rotate_copy(
+    _ForwardIterator __first,
+    _ForwardIterator __middle,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::rotate_copy(__first, __middle, __last, __result);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2 __brick_rotate_copy(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __middle,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result,
+    /*__is_vector=*/std::true_type) noexcept {
+  _RandomAccessIterator2 __res = __internal::__brick_copy(__middle, __last, __result, std::true_type());
+  return __internal::__brick_copy(__first, __middle, __res, std::true_type());
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator>
+_OutputIterator __pattern_rotate_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __middle,
+    _ForwardIterator __last,
+    _OutputIterator __result) noexcept {
+  return __internal::__brick_rotate_copy(__first, __middle, __last, __result, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2>
+_RandomAccessIterator2 __pattern_rotate_copy(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __middle,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __result) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __par_backend::__parallel_for(
+      __backend_tag{},
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      [__first, __last, __middle, __result](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e) {
+        if (__b > __middle) {
+          __internal::__brick_copy(__b, __e, __result + (__b - __middle), _IsVector{});
+        } else {
+          _RandomAccessIterator2 __new_result = __result + ((__last - __middle) + (__b - __first));
+          if (__e < __middle) {
+            __internal::__brick_copy(__b, __e, __new_result, _IsVector{});
+          } else {
+            __internal::__brick_copy(__b, __middle, __new_result, _IsVector{});
+            __internal::__brick_copy(__middle, __e, __result, _IsVector{});
+          }
+        }
+      });
+  return __result + (__last - __first);
+}
+
+//------------------------------------------------------------------------
+// is_partitioned
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _UnaryPredicate>
+bool __brick_is_partitioned(_ForwardIterator __first,
+                            _ForwardIterator __last,
+                            _UnaryPredicate __pred,
+                            /*is_vector=*/std::false_type) noexcept {
+  return std::is_partitioned(__first, __last, __pred);
+}
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+bool __brick_is_partitioned(_RandomAccessIterator __first,
+                            _RandomAccessIterator __last,
+                            _UnaryPredicate __pred,
+                            /*is_vector=*/std::true_type) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
+  if (__first == __last) {
+    return true;
+  } else {
+    _RandomAccessIterator __result = __unseq_backend::__simd_first(
+        __first, _SizeType(0), __last - __first, [&__pred](_RandomAccessIterator __it, _SizeType __i) {
+          return !__pred(__it[__i]);
+        });
+    if (__result == __last) {
+      return true;
+    } else {
+      ++__result;
+      return !__unseq_backend::__simd_or(__result, __last - __result, __pred);
+    }
+  }
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+bool __pattern_is_partitioned(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred) noexcept {
+  return __internal::__brick_is_partitioned(__first, __last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+bool __pattern_is_partitioned(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred) {
+  if (__first == __last) {
+    return true;
+  } else {
+    using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+    return __internal::__except_handler([&]() {
+      // State of current range:
+      // broken     - current range is not partitioned by pred
+      // all_true   - all elements in current range satisfy pred
+      // all_false  - all elements in current range don't satisfy pred
+      // true_false - elements satisfy pred are placed before elements that don't satisfy pred
+      enum _ReduceType { __not_init = -1, __broken, __all_true, __all_false, __true_false };
+      _ReduceType __init = __not_init;
+
+      // Array with states that we'll have when state from the left branch is merged with state from the right branch.
+      // State is calculated by formula: new_state = table[left_state * 4 + right_state]
+      _ReduceType __table[] = {
+          __broken,
+          __broken,
+          __broken,
+          __broken,
+          __broken,
+          __all_true,
+          __true_false,
+          __true_false,
+          __broken,
+          __broken,
+          __all_false,
+          __broken,
+          __broken,
+          __broken,
+          __true_false,
+          __broken};
+
+      __init = __par_backend::__parallel_reduce(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __first,
+          __last,
+          __init,
+          [&__pred,
+           &__table](_RandomAccessIterator __i, _RandomAccessIterator __j, _ReduceType __value) -> _ReduceType {
+            if (__value == __broken) {
+              return __broken;
+            }
+            _ReduceType __res = __not_init;
+            // if first element satisfy pred
+            if (__pred(*__i)) {
+              // find first element that don't satisfy pred
+              _RandomAccessIterator __x = __internal::__brick_find_if(__i + 1, __j, std::not_fn(__pred), _IsVector{});
+              if (__x != __j) {
+                // find first element after "x" that satisfy pred
+                _RandomAccessIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, _IsVector{});
+                // if it was found then range isn't partitioned by pred
+                if (__y != __j) {
+                  return __broken;
+                } else {
+                  __res = __true_false;
+                }
+              } else {
+                __res = __all_true;
+              }
+            } else { // if first element doesn't satisfy pred
+              // then we should find the first element that satisfy pred.
+              // If we found it then range isn't partitioned by pred
+              if (__internal::__brick_find_if(__i + 1, __j, __pred, _IsVector{}) != __j) {
+                return __broken;
+              } else {
+                __res = __all_false;
+              }
+            }
+            // if we have value from left range then we should calculate the result
+            return (__value == -1) ? __res : __table[__value * 4 + __res];
+          },
+
+          [&__table](_ReduceType __val1, _ReduceType __val2) -> _ReduceType {
+            if (__val1 == __broken || __val2 == __broken) {
+              return __broken;
+            }
+            // calculate the result for new big range
+            return __table[__val1 * 4 + __val2];
+          });
+      return __init != __broken;
+    });
+  }
+}
+
+//------------------------------------------------------------------------
+// partition
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator __brick_partition(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _UnaryPredicate __pred,
+    /*is_vector=*/std::false_type) noexcept {
+  return std::partition(__first, __last, __pred);
+}
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __brick_partition(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred,
+    /*is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::partition(__first, __last, __pred);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator __pattern_partition(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred) noexcept {
+  return __internal::__brick_partition(__first, __last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __pattern_partition(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  // partitioned range: elements before pivot satisfy pred (true part),
+  //                    elements after pivot don't satisfy pred (false part)
+  struct _PartitionRange {
+    _RandomAccessIterator __begin;
+    _RandomAccessIterator __pivot;
+    _RandomAccessIterator __end;
+  };
+
+  return __internal::__except_handler([&]() {
+    _PartitionRange __init{__last, __last, __last};
+
+    // lambda for merging two partitioned ranges to one partitioned range
+    auto __reductor = [&__exec](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
+      auto __size1     = __val1.__end - __val1.__pivot;
+      auto __size2     = __val2.__pivot - __val2.__begin;
+      auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
+
+      // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
+      if (__val1.__end == __val1.__pivot) {
+        return {__new_begin, __val2.__pivot, __val2.__end};
+      }
+      // if true part of right range greater than false part of left range
+      // then we should swap the false part of left range and last part of true part of right range
+      else if (__size2 > __size1) {
+        __par_backend::__parallel_for(
+            __backend_tag{},
+            std::forward<_ExecutionPolicy>(__exec),
+            __val1.__pivot,
+            __val1.__pivot + __size1,
+            [__val1, __val2, __size1](_RandomAccessIterator __i, _RandomAccessIterator __j) {
+              __internal::__brick_swap_ranges(
+                  __i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot), _IsVector{});
+            });
+        return {__new_begin, __val2.__pivot - __size1, __val2.__end};
+      }
+      // else we should swap the first part of false part of left range and true part of right range
+      else {
+        __par_backend::__parallel_for(
+            __backend_tag{},
+            std::forward<_ExecutionPolicy>(__exec),
+            __val1.__pivot,
+            __val1.__pivot + __size2,
+            [__val1, __val2](_RandomAccessIterator __i, _RandomAccessIterator __j) {
+              __internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), _IsVector{});
+            });
+        return {__new_begin, __val1.__pivot + __size2, __val2.__end};
+      }
+    };
+
+    _PartitionRange __result = __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        __init,
+        [__pred,
+         __reductor](_RandomAccessIterator __i, _RandomAccessIterator __j, _PartitionRange __value) -> _PartitionRange {
+          // 1. serial partition
+          _RandomAccessIterator __pivot = __internal::__brick_partition(__i, __j, __pred, _IsVector{});
+
+          // 2. merging of two ranges (left and right respectively)
+          return __reductor(__value, {__i, __pivot, __j});
+        },
+        __reductor);
+    return __result.__pivot;
+  });
+}
+
+//------------------------------------------------------------------------
+// stable_partition
+//------------------------------------------------------------------------
+
+template <class _BidirectionalIterator, class _UnaryPredicate>
+_BidirectionalIterator __brick_stable_partition(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    _UnaryPredicate __pred,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::stable_partition(__first, __last, __pred);
+}
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __brick_stable_partition(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred,
+    /*__is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::stable_partition(__first, __last, __pred);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
+_BidirectionalIterator __pattern_stable_partition(
+    _Tag,
+    _ExecutionPolicy&&,
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __last,
+    _UnaryPredicate __pred) noexcept {
+  return __internal::__brick_stable_partition(__first, __last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __pattern_stable_partition(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  // partitioned range: elements before pivot satisfy pred (true part),
+  //                    elements after pivot don't satisfy pred (false part)
+  struct _PartitionRange {
+    _RandomAccessIterator __begin;
+    _RandomAccessIterator __pivot;
+    _RandomAccessIterator __end;
+  };
+
+  return __internal::__except_handler([&]() {
+    _PartitionRange __init{__last, __last, __last};
+
+    // lambda for merging two partitioned ranges to one partitioned range
+    auto __reductor = [](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange {
+      auto __size1     = __val1.__end - __val1.__pivot;
+      auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
+
+      // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
+      if (__val1.__end == __val1.__pivot) {
+        return {__new_begin, __val2.__pivot, __val2.__end};
+      }
+      // if true part of right range greater than false part of left range
+      // then we should swap the false part of left range and last part of true part of right range
+      else {
+        __internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, _IsVector{});
+        return {__new_begin, __val2.__pivot - __size1, __val2.__end};
+      }
+    };
+
+    _PartitionRange __result = __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        __init,
+        [&__pred,
+         __reductor](_RandomAccessIterator __i, _RandomAccessIterator __j, _PartitionRange __value) -> _PartitionRange {
+          // 1. serial stable_partition
+          _RandomAccessIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, _IsVector{});
+
+          // 2. merging of two ranges (left and right respectively)
+          return __reductor(__value, {__i, __pivot, __j});
+        },
+        __reductor);
+    return __result.__pivot;
+  });
+}
+
+//------------------------------------------------------------------------
+// partition_copy
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
+std::pair<_OutputIterator1, _OutputIterator2> __brick_partition_copy(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator1 __out_true,
+    _OutputIterator2 __out_false,
+    _UnaryPredicate __pred,
+    /*is_vector=*/std::false_type) noexcept {
+  return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
+}
+
+template <class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _UnaryPredicate>
+std::pair<_RandomAccessIterator2, _RandomAccessIterator3> __brick_partition_copy(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __out_true,
+    _RandomAccessIterator3 __out_false,
+    _UnaryPredicate __pred,
+    /*is_vector=*/std::true_type) noexcept {
+#if defined(_PSTL_MONOTONIC_PRESENT)
+  return __unseq_backend::__simd_partition_copy(__first, __last - __first, __out_true, __out_false, __pred);
+#else
+  return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
+#endif
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _OutputIterator1,
+          class _OutputIterator2,
+          class _UnaryPredicate>
+std::pair<_OutputIterator1, _OutputIterator2> __pattern_partition_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator1 __out_true,
+    _OutputIterator2 __out_false,
+    _UnaryPredicate __pred) noexcept {
+  return __internal::__brick_partition_copy(
+      __first, __last, __out_true, __out_false, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _UnaryPredicate>
+std::pair<_RandomAccessIterator2, _RandomAccessIterator3> __pattern_partition_copy(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __out_true,
+    _RandomAccessIterator3 __out_false,
+    _UnaryPredicate __pred) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
+  typedef std::pair<_DifferenceType, _DifferenceType> _ReturnType;
+  const _DifferenceType __n = __last - __first;
+  if (_DifferenceType(1) < __n) {
+    __par_backend::__buffer<bool> __mask_buf(__n);
+    return __internal::__except_handler([&__exec, __n, __first, __out_true, __out_false, __pred, &__mask_buf]() {
+      bool* __mask = __mask_buf.get();
+      _ReturnType __m{};
+      __par_backend::__parallel_strict_scan(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __n,
+          std::make_pair(_DifferenceType(0), _DifferenceType(0)),
+          [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
+            return __internal::__brick_calc_mask_1<_DifferenceType>(
+                __first + __i, __first + (__i + __len), __mask + __i, __pred, _IsVector{});
+          },
+          [](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType {
+            return std::make_pair(__x.first + __y.first, __x.second + __y.second);
+          },                                                                       // Combine
+          [=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) { // Scan
+            __internal::__brick_partition_by_mask(
+                __first + __i,
+                __first + (__i + __len),
+                __out_true + __initial.first,
+                __out_false + __initial.second,
+                __mask + __i,
+                _IsVector{});
+          },
+          [&__m](_ReturnType __total) { __m = __total; });
+      return std::make_pair(__out_true + __m.first, __out_false + __m.second);
+    });
+  }
+  // trivial sequence - use serial algorithm
+  return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, _IsVector{});
+}
+
+//------------------------------------------------------------------------
+// sort
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsMoveConstructible>
+void __pattern_sort(_Tag,
+                    _ExecutionPolicy&&,
+                    _RandomAccessIterator __first,
+                    _RandomAccessIterator __last,
+                    _Compare __comp,
+                    _IsMoveConstructible) noexcept {
+  std::sort(__first, __last, __comp);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_sort(__parallel_tag<_IsVector> __tag,
+                    _ExecutionPolicy&& __exec,
+                    _RandomAccessIterator __first,
+                    _RandomAccessIterator __last,
+                    _Compare __comp,
+                    /*is_move_constructible=*/std::true_type) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __internal::__except_handler([&]() {
+    __par_backend::__parallel_stable_sort(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        __comp,
+        [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+          std::sort(__first, __last, __comp);
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// stable_sort
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_stable_sort(
+    _Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) noexcept {
+  std::stable_sort(__first, __last, __comp);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_stable_sort(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __internal::__except_handler([&]() {
+    __par_backend::__parallel_stable_sort(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        __comp,
+        [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+          std::stable_sort(__first, __last, __comp);
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// partial_sort
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_partial_sort(
+    _Tag,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __middle,
+    _RandomAccessIterator __last,
+    _Compare __comp) noexcept {
+  std::partial_sort(__first, __middle, __last, __comp);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_partial_sort(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __middle,
+    _RandomAccessIterator __last,
+    _Compare __comp) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  const auto __n = __middle - __first;
+  if (__n == 0)
+    return;
+
+  __internal::__except_handler([&]() {
+    __par_backend::__parallel_stable_sort(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        __comp,
+        [__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp) {
+          if (__n < __end - __begin)
+            std::partial_sort(__begin, __begin + __n, __end, __comp);
+          else
+            std::sort(__begin, __end, __comp);
+        },
+        __n);
+  });
+}
+
+//------------------------------------------------------------------------
+// partial_sort_copy
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __pattern_partial_sort_copy(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _RandomAccessIterator __d_first,
+    _RandomAccessIterator __d_last,
+    _Compare __comp) noexcept {
+  return std::partial_sort_copy(__first, __last, __d_first, __d_last, __comp);
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+_RandomAccessIterator2 __pattern_partial_sort_copy(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __d_first,
+    _RandomAccessIterator2 __d_last,
+    _Compare __comp) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__last == __first || __d_last == __d_first) {
+    return __d_first;
+  }
+  auto __n1 = __last - __first;
+  auto __n2 = __d_last - __d_first;
+  return __internal::__except_handler([&]() {
+    if (__n2 >= __n1) {
+      __par_backend::__parallel_stable_sort(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __d_first,
+          __d_first + __n1,
+          __comp,
+          [__first, __d_first](_RandomAccessIterator2 __i, _RandomAccessIterator2 __j, _Compare __comp) {
+            _RandomAccessIterator1 __i1 = __first + (__i - __d_first);
+            _RandomAccessIterator1 __j1 = __first + (__j - __d_first);
+
+        // 1. Copy elements from input to output
+#if !defined(_PSTL_ICC_18_OMP_SIMD_BROKEN)
+            __internal::__brick_copy(__i1, __j1, __i, _IsVector{});
+#else
+            std::copy(__i1, __j1, __i);
+#endif
+            // 2. Sort elements in output sequence
+            std::sort(__i, __j, __comp);
+          },
+          __n1);
+      return __d_first + __n1;
+    } else {
+      typedef typename std::iterator_traits<_RandomAccessIterator1>::value_type _T1;
+      typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _T2;
+      __par_backend::__buffer<_T1> __buf(__n1);
+      _T1* __r = __buf.get();
+
+      __par_backend::__parallel_stable_sort(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __r,
+          __r + __n1,
+          __comp,
+          [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp) {
+            _RandomAccessIterator1 __it = __first + (__i - __r);
+
+            // 1. Copy elements from input to raw memory
+            for (_T1* __k = __i; __k != __j; ++__k, ++__it) {
+              ::new (__k) _T2(*__it);
+            }
+
+            // 2. Sort elements in temporary __buffer
+            if (__n2 < __j - __i)
+              std::partial_sort(__i, __i + __n2, __j, __comp);
+            else
+              std::sort(__i, __j, __comp);
+          },
+          __n2);
+
+      // 3. Move elements from temporary __buffer to output
+      __par_backend::__parallel_for(
+          __backend_tag{},
+          std::forward<_ExecutionPolicy>(__exec),
+          __r,
+          __r + __n2,
+          [__r, __d_first](_T1* __i, _T1* __j) {
+            __brick_move_destroy()(__i, __j, __d_first + (__i - __r), _IsVector{});
+          });
+      __par_backend::__parallel_for(
+          __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r + __n2, __r + __n1, [](_T1* __i, _T1* __j) {
+            __brick_destroy(__i, __j, _IsVector{});
+          });
+
+      return __d_first + __n2;
+    }
+  });
+}
+
+//------------------------------------------------------------------------
+// adjacent_find
+//------------------------------------------------------------------------
+template <class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __brick_adjacent_find(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _BinaryPredicate __pred,
+    /* IsVector = */ std::true_type,
+    bool __or_semantic) noexcept {
+  return __unseq_backend::__simd_adjacent_find(__first, __last, __pred, __or_semantic);
+}
+
+template <class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator __brick_adjacent_find(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _BinaryPredicate __pred,
+    /* IsVector = */ std::false_type,
+    bool) noexcept {
+  return std::adjacent_find(__first, __last, __pred);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
+_ForwardIterator __pattern_adjacent_find(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _BinaryPredicate __pred,
+    bool __or_semantic) noexcept {
+  return __internal::__brick_adjacent_find(__first, __last, __pred, typename _Tag::__is_vector{}, __or_semantic);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
+_RandomAccessIterator __pattern_adjacent_find(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _BinaryPredicate __pred,
+    bool __or_semantic) {
+  if (__last - __first < 2)
+    return __last;
+
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    return __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        __last,
+        [__last, __pred, __or_semantic](
+            _RandomAccessIterator __begin, _RandomAccessIterator __end, _RandomAccessIterator __value)
+            -> _RandomAccessIterator {
+          // TODO: investigate performance benefits from the use of shared variable for the result,
+          // checking (compare_and_swap idiom) its __value at __first.
+          if (__or_semantic && __value < __last) { // found
+            __par_backend::__cancel_execution();
+            return __value;
+          }
+
+          if (__value > __begin) {
+            // modify __end to check the predicate on the boundary __values;
+            // TODO: to use a custom range with boundaries overlapping
+            // TODO: investigate what if we remove "if" below and run algorithm on range [__first, __last-1)
+            // then check the pair [__last-1, __last)
+            if (__end != __last)
+              ++__end;
+
+            // correct the global result iterator if the "brick" returns a local "__last"
+            const _RandomAccessIterator __res =
+                __internal::__brick_adjacent_find(__begin, __end, __pred, _IsVector{}, __or_semantic);
+            if (__res < __end)
+              __value = __res;
+          }
+          return __value;
+        },
+        [](_RandomAccessIterator __x, _RandomAccessIterator __y) -> _RandomAccessIterator {
+          return __x < __y ? __x : __y;
+        } // reduce a __value
+    );
+  });
+}
+
+//------------------------------------------------------------------------
+// nth_element
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_nth_element(
+    _Tag,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __nth,
+    _RandomAccessIterator __last,
+    _Compare __comp) noexcept {
+  std::nth_element(__first, __nth, __last, __comp);
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_nth_element(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __nth,
+    _RandomAccessIterator __last,
+    _Compare __comp) noexcept {
+  if (__first == __last || __nth == __last) {
+    return;
+  }
+
+  using std::iter_swap;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
+  _RandomAccessIterator __x;
+  do {
+    __x = __internal::__pattern_partition(
+        __tag, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, [&__comp, __first](const _Tp& __x) {
+          return __comp(__x, *__first);
+        });
+    --__x;
+    if (__x != __first) {
+      iter_swap(__first, __x);
+    }
+    // if x > nth then our new range for partition is [first, x)
+    if (__x - __nth > 0) {
+      __last = __x;
+    }
+    // if x < nth then our new range for partition is [x, last)
+    else if (__x - __nth < 0) {
+      // if *x == *nth then we can start new partition with x+1
+      if (!__comp(*__nth, *__x) && !__comp(*__x, *__nth)) {
+        ++__x;
+      } else {
+        iter_swap(__nth, __x);
+      }
+      __first = __x;
+    }
+  } while (__x != __nth);
+}
+
+//------------------------------------------------------------------------
+// generate, generate_n
+//------------------------------------------------------------------------
+template <class _RandomAccessIterator, class _Generator>
+void __brick_generate(_RandomAccessIterator __first,
+                      _RandomAccessIterator __last,
+                      _Generator __g,
+                      /* is_vector = */ std::true_type) noexcept {
+  __unseq_backend::__simd_generate_n(__first, __last - __first, __g);
+}
+
+template <class _ForwardIterator, class _Generator>
+void __brick_generate(_ForwardIterator __first,
+                      _ForwardIterator __last,
+                      _Generator __g,
+                      /* is_vector = */ std::false_type) noexcept {
+  std::generate(__first, __last, __g);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Generator>
+void __pattern_generate(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __g) noexcept {
+  __internal::__brick_generate(__first, __last, __g, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Generator>
+_RandomAccessIterator __pattern_generate(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Generator __g) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_for(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        [__g](_RandomAccessIterator __begin, _RandomAccessIterator __end) {
+          __internal::__brick_generate(__begin, __end, __g, _IsVector{});
+        });
+    return __last;
+  });
+}
+
+template <class _RandomAccessIterator, class Size, class _Generator>
+_RandomAccessIterator __brick_generate_n(
+    _RandomAccessIterator __first,
+    Size __count,
+    _Generator __g,
+    /* is_vector = */ std::true_type) noexcept {
+  return __unseq_backend::__simd_generate_n(__first, __count, __g);
+}
+
+template <class OutputIterator, class Size, class _Generator>
+OutputIterator
+__brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::false_type) noexcept {
+  return std::generate_n(__first, __count, __g);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator>
+_OutputIterator
+__pattern_generate_n(_Tag, _ExecutionPolicy&&, _OutputIterator __first, _Size __count, _Generator __g) noexcept {
+  return __internal::__brick_generate_n(__first, __count, __g, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Generator>
+_RandomAccessIterator __pattern_generate_n(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _Size __count,
+    _Generator __g) {
+  static_assert(__are_random_access_iterators<_RandomAccessIterator>::value,
+                "Pattern-brick error. Should be a random access iterator.");
+  return __internal::__pattern_generate(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __count, __g);
+}
+
+//------------------------------------------------------------------------
+// remove
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator __brick_remove_if(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _UnaryPredicate __pred,
+    /* __is_vector = */ std::false_type) noexcept {
+  return std::remove_if(__first, __last, __pred);
+}
+
+template <class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __brick_remove_if(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred,
+    /* __is_vector = */ std::true_type) noexcept {
+#if defined(_PSTL_MONOTONIC_PRESENT)
+  return __unseq_backend::__simd_remove_if(__first, __last - __first, __pred);
+#else
+  return std::remove_if(__first, __last, __pred);
+#endif
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+_ForwardIterator __pattern_remove_if(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred) noexcept {
+  return __internal::__brick_remove_if(__first, __last, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
+_RandomAccessIterator __pattern_remove_if(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _UnaryPredicate __pred) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
+
+  if (__first == __last || __first + 1 == __last) {
+    // Trivial sequence - use serial algorithm
+    return __internal::__brick_remove_if(__first, __last, __pred, _IsVector{});
+  }
+
+  return __internal::__remove_elements(
+      __tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      [&__pred](bool* __b, bool* __e, _RandomAccessIterator __it) {
+        __internal::__brick_walk2(
+            __b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); }, _IsVector{});
+      });
+}
+
+//------------------------------------------------------------------------
+// merge
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_merge(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __d_first,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
+_RandomAccessIterator3 __brick_merge(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __d_first,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+  // TODO: vectorize
+  return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_merge(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __d_first,
+    _Compare __comp) noexcept {
+  return __internal::__brick_merge(
+      __first1, __last1, __first2, __last2, __d_first, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _Compare>
+_RandomAccessIterator3 __pattern_merge(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __d_first,
+    _Compare __comp) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  __par_backend::__parallel_merge(
+      __backend_tag{},
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __last2,
+      __d_first,
+      __comp,
+      [](_RandomAccessIterator1 __f1,
+         _RandomAccessIterator1 __l1,
+         _RandomAccessIterator2 __f2,
+         _RandomAccessIterator2 __l2,
+         _RandomAccessIterator3 __f3,
+         _Compare __comp) { return __internal::__brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, _IsVector{}); });
+  return __d_first + (__last1 - __first1) + (__last2 - __first2);
+}
+
+//------------------------------------------------------------------------
+// inplace_merge
+//------------------------------------------------------------------------
+template <class _BidirectionalIterator, class _Compare>
+void __brick_inplace_merge(
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __middle,
+    _BidirectionalIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  std::inplace_merge(__first, __middle, __last, __comp);
+}
+
+template <class _RandomAccessIterator, class _Compare>
+void __brick_inplace_merge(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __middle,
+    _RandomAccessIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+  // TODO: vectorize
+  std::inplace_merge(__first, __middle, __last, __comp);
+}
+
+template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
+void __pattern_inplace_merge(
+    _Tag,
+    _ExecutionPolicy&&,
+    _BidirectionalIterator __first,
+    _BidirectionalIterator __middle,
+    _BidirectionalIterator __last,
+    _Compare __comp) noexcept {
+  __internal::__brick_inplace_merge(__first, __middle, __last, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+void __pattern_inplace_merge(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __middle,
+    _RandomAccessIterator __last,
+    _Compare __comp) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__first == __last || __first == __middle || __middle == __last) {
+    return;
+  }
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
+  auto __n = __last - __first;
+  __par_backend::__buffer<_Tp> __buf(__n);
+  _Tp* __r = __buf.get();
+  __internal::__except_handler([&]() {
+    auto __move_values = [](_RandomAccessIterator __x, _Tp* __z) {
+      __internal::__invoke_if_else(
+          std::is_trivial<_Tp>(),
+          [&]() { *__z = std::move(*__x); },
+          [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
+    };
+
+    auto __move_sequences = [](_RandomAccessIterator __first1, _RandomAccessIterator __last1, _Tp* __first2) {
+      return __internal::__brick_uninitialized_move(__first1, __last1, __first2, _IsVector());
+    };
+
+    __par_backend::__parallel_merge(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __middle,
+        __middle,
+        __last,
+        __r,
+        __comp,
+        [__n, __move_values, __move_sequences](
+            _RandomAccessIterator __f1,
+            _RandomAccessIterator __l1,
+            _RandomAccessIterator __f2,
+            _RandomAccessIterator __l2,
+            _Tp* __f3,
+            _Compare __comp) {
+          (__utils::__serial_move_merge(__n))(
+              __f1, __l1, __f2, __l2, __f3, __comp, __move_values, __move_values, __move_sequences, __move_sequences);
+          return __f3 + (__l1 - __f1) + (__l2 - __f2);
+        });
+    __par_backend::__parallel_for(
+        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n, [__r, __first](_Tp* __i, _Tp* __j) {
+          __brick_move_destroy()(__i, __j, __first + (__i - __r), _IsVector{});
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// includes
+//------------------------------------------------------------------------
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+bool __pattern_includes(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Compare __comp) noexcept {
+  return std::includes(__first1, __last1, __first2, __last2, __comp);
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+bool __pattern_includes(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _Compare __comp) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__first2 >= __last2)
+    return true;
+
+  if (__first1 >= __last1 || __comp(*__first2, *__first1) || __comp(*(__last1 - 1), *(__last2 - 1)))
+    return false;
+
+  __first1 = std::lower_bound(__first1, __last1, *__first2, __comp);
+  if (__first1 == __last1)
+    return false;
+
+  if (__last2 - __first2 == 1)
+    return !__comp(*__first1, *__first2) && !__comp(*__first2, *__first1);
+
+  return __internal::__except_handler([&]() {
+    return !__internal::__parallel_or(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first2,
+        __last2,
+        [__first1, __last1, __first2, __last2, &__comp](_RandomAccessIterator2 __i, _RandomAccessIterator2 __j) {
+          _LIBCPP_ASSERT(__j > __i, "");
+          //_LIBCPP_ASSERT(__j - __i > 1, "");
+
+          // 1. moving boundaries to "consume" subsequence of equal elements
+          auto __is_equal = [&__comp](_RandomAccessIterator2 __a, _RandomAccessIterator2 __b) -> bool {
+            return !__comp(*__a, *__b) && !__comp(*__b, *__a);
+          };
+
+          // 1.1 left bound, case "aaa[aaaxyz...]" - searching "x"
+          if (__i > __first2 && __is_equal(__i, __i - 1)) {
+            // whole subrange continues to content equal elements - return "no op"
+            if (__is_equal(__i, __j - 1))
+              return false;
+
+            __i = std::upper_bound(__i, __last2, *__i, __comp);
+          }
+
+          // 1.2 right bound, case "[...aaa]aaaxyz" - searching "x"
+          if (__j < __last2 && __is_equal(__j - 1, __j))
+            __j = std::upper_bound(__j, __last2, *__j, __comp);
+
+          // 2. testing is __a subsequence of the second range included into the first range
+          auto __b = std::lower_bound(__first1, __last1, *__i, __comp);
+
+          _LIBCPP_ASSERT(!__comp(*(__last1 - 1), *__b), "");
+          _LIBCPP_ASSERT(!__comp(*(__j - 1), *__i), "");
+          return !std::includes(__b, __last1, __i, __j, __comp);
+        });
+  });
+}
+
+constexpr auto __set_algo_cut_off = 1000;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare,
+          class _SizeFunction,
+          class _SetOP>
+_OutputIterator __parallel_set_op(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    _SizeFunction __size_func,
+    _SetOP __set_op) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
+  typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
+
+  struct _SetRange {
+    _DifferenceType __pos, __len, __buf_pos;
+    bool empty() const { return __len == 0; }
+  };
+
+  const _DifferenceType __n1 = __last1 - __first1;
+  const _DifferenceType __n2 = __last2 - __first2;
+
+  __par_backend::__buffer<_Tp> __buf(__size_func(__n1, __n2));
+
+  return __internal::__except_handler(
+      [&__exec, __n1, __first1, __last1, __first2, __last2, __result, __comp, __size_func, __set_op, &__buf]() {
+        auto __buffer = __buf.get();
+        _DifferenceType __m{};
+        auto __scan = [=](_DifferenceType, _DifferenceType, const _SetRange& __s) { // Scan
+          if (!__s.empty())
+            __brick_move_destroy()(
+                __buffer + __s.__buf_pos, __buffer + (__s.__buf_pos + __s.__len), __result + __s.__pos, _IsVector{});
+        };
+        __par_backend::__parallel_strict_scan(
+            __backend_tag{},
+            std::forward<_ExecutionPolicy>(__exec),
+            __n1,
+            _SetRange{0, 0, 0},                               //-1, 0},
+            [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
+              //[__b; __e) - a subrange of the first sequence, to reduce
+              _ForwardIterator1 __b = __first1 + __i, __e = __first1 + (__i + __len);
+
+              // try searching for the first element which not equal to *__b
+              if (__b != __first1)
+                __b = std::upper_bound(__b, __last1, *__b, __comp);
+
+              // try searching for the first element which not equal to *__e
+              if (__e != __last1)
+                __e = std::upper_bound(__e, __last1, *__e, __comp);
+
+              // check is [__b; __e) empty
+              if (__e - __b < 1) {
+                _ForwardIterator2 __bb = __last2;
+                if (__b != __last1)
+                  __bb = std::lower_bound(__first2, __last2, *__b, __comp);
+
+                const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
+                return _SetRange{0, 0, __buf_pos};
+              }
+
+              // try searching for "corresponding" subrange [__bb; __ee) in the second sequence
+              _ForwardIterator2 __bb = __first2;
+              if (__b != __first1)
+                __bb = std::lower_bound(__first2, __last2, *__b, __comp);
+
+              _ForwardIterator2 __ee = __last2;
+              if (__e != __last1)
+                __ee = std::lower_bound(__bb, __last2, *__e, __comp);
+
+              const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
+              auto __buffer_b                 = __buffer + __buf_pos;
+              auto __res                      = __set_op(__b, __e, __bb, __ee, __buffer_b, __comp);
+
+              return _SetRange{0, __res - __buffer_b, __buf_pos};
+            },
+            [](const _SetRange& __a, const _SetRange& __b) { // Combine
+              if (__b.__buf_pos > __a.__buf_pos || ((__b.__buf_pos == __a.__buf_pos) && !__b.empty()))
+                return _SetRange{__a.__pos + __a.__len + __b.__pos, __b.__len, __b.__buf_pos};
+              return _SetRange{__b.__pos + __b.__len + __a.__pos, __a.__len, __a.__buf_pos};
+            },
+            __scan,                                     // Scan
+            [&__m, &__scan](const _SetRange& __total) { // Apex
+              // final scan
+              __scan(0, 0, __total);
+              __m = __total.__pos + __total.__len;
+            });
+        return __result + __m;
+      });
+}
+
+// a shared parallel pattern for '__pattern_set_union' and '__pattern_set_symmetric_difference'
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare,
+          class _SetUnionOp>
+_OutputIterator __parallel_set_union_op(
+    _Tag __tag,
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    _SetUnionOp __set_union_op) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
+
+  const auto __n1 = __last1 - __first1;
+  const auto __n2 = __last2 - __first2;
+
+  auto copy_range1 = [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res) {
+    return __internal::__brick_copy(__begin, __end, __res, typename _Tag::__is_vector{});
+  };
+  auto copy_range2 = [](_ForwardIterator2 __begin, _ForwardIterator2 __end, _OutputIterator __res) {
+    return __internal::__brick_copy(__begin, __end, __res, typename _Tag::__is_vector{});
+  };
+
+  // {1} {}: parallel copying just first sequence
+  if (__n2 == 0)
+    return __internal::__pattern_walk2_brick(
+        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, copy_range1);
+
+  // {} {2}: parallel copying justmake  second sequence
+  if (__n1 == 0)
+    return __internal::__pattern_walk2_brick(
+        __tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result, copy_range2);
+
+  // testing  whether the sequences are intersected
+  _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
+
+  if (__left_bound_seq_1 == __last1) {
+    //{1} < {2}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
+    __par_backend::__parallel_invoke(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        [=] {
+          __internal::__pattern_walk2_brick(
+              __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result, copy_range1);
+        },
+        [=] {
+          __internal::__pattern_walk2_brick(
+              __tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result + __n1, copy_range2);
+        });
+    return __result + __n1 + __n2;
+  }
+
+  // testing  whether the sequences are intersected
+  _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
+
+  if (__left_bound_seq_2 == __last2) {
+    //{2} < {1}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
+    __par_backend::__parallel_invoke(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        [=] {
+          __internal::__pattern_walk2_brick(
+              __tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2, __result, copy_range2);
+        },
+        [=] {
+          __internal::__pattern_walk2_brick(
+              __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result + __n2, copy_range1);
+        });
+    return __result + __n1 + __n2;
+  }
+
+  const auto __m1 = __left_bound_seq_1 - __first1;
+  if (__m1 > __set_algo_cut_off) {
+    auto __res_or = __result;
+    __result += __m1; // we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
+    __par_backend::__parallel_invoke(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        // do parallel copying of [first1; left_bound_seq_1)
+        [=] {
+          __internal::__pattern_walk2_brick(
+              __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __left_bound_seq_1, __res_or, copy_range1);
+        },
+        [=, &__result] {
+          __result = __internal::__parallel_set_op(
+              __tag,
+              std::forward<_ExecutionPolicy>(__exec),
+              __left_bound_seq_1,
+              __last1,
+              __first2,
+              __last2,
+              __result,
+              __comp,
+              [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
+              __set_union_op);
+        });
+    return __result;
+  }
+
+  const auto __m2 = __left_bound_seq_2 - __first2;
+  _LIBCPP_ASSERT(__m1 == 0 || __m2 == 0, "");
+  if (__m2 > __set_algo_cut_off) {
+    auto __res_or = __result;
+    __result += __m2; // we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
+    __par_backend::__parallel_invoke(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        // do parallel copying of [first2; left_bound_seq_2)
+        [=] {
+          __internal::__pattern_walk2_brick(
+              __tag, std::forward<_ExecutionPolicy>(__exec), __first2, __left_bound_seq_2, __res_or, copy_range2);
+        },
+        [=, &__result] {
+          __result = __internal::__parallel_set_op(
+              __tag,
+              std::forward<_ExecutionPolicy>(__exec),
+              __first1,
+              __last1,
+              __left_bound_seq_2,
+              __last2,
+              __result,
+              __comp,
+              [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
+              __set_union_op);
+        });
+    return __result;
+  }
+
+  return __internal::__parallel_set_op(
+      __tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __last2,
+      __result,
+      __comp,
+      [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
+      __set_union_op);
+}
+
+//------------------------------------------------------------------------
+// set_union
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_union(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <typename _IsVector>
+struct __BrickCopyConstruct {
+  template <typename _ForwardIterator, typename _OutputIterator>
+  _OutputIterator operator()(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result) {
+    return __brick_uninitialized_copy(__first, __last, __result, _IsVector());
+  }
+};
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_union(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    /*__is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_union(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) noexcept {
+  return __internal::__brick_set_union(
+      __first1, __last1, __first2, __last2, __result, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_union(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) {
+  const auto __n1 = __last1 - __first1;
+  const auto __n2 = __last2 - __first2;
+
+  // use serial algorithm
+  if (__n1 + __n2 <= __set_algo_cut_off)
+    return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
+
+  typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
+  return __parallel_set_union_op(
+      __tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __last2,
+      __result,
+      __comp,
+      [](_RandomAccessIterator1 __first1,
+         _RandomAccessIterator1 __last1,
+         _RandomAccessIterator2 __first2,
+         _RandomAccessIterator2 __last2,
+         _Tp* __result,
+         _Compare __comp) {
+        return __pstl::__utils::__set_union_construct(
+            __first1, __last1, __first2, __last2, __result, __comp, __BrickCopyConstruct<_IsVector>());
+      });
+}
+
+//------------------------------------------------------------------------
+// set_intersection
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_intersection(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
+_RandomAccessIterator3 __brick_set_intersection(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __result,
+    _Compare __comp,
+    /*__is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_intersection(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) noexcept {
+  return __internal::__brick_set_intersection(
+      __first1, __last1, __first2, __last2, __result, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _Compare>
+_RandomAccessIterator3 __pattern_set_intersection(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __result,
+    _Compare __comp) {
+  typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
+
+  const auto __n1 = __last1 - __first1;
+  const auto __n2 = __last2 - __first2;
+
+  // intersection is empty
+  if (__n1 == 0 || __n2 == 0)
+    return __result;
+
+  // testing  whether the sequences are intersected
+  _RandomAccessIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
+  //{1} < {2}: seq 2 is wholly greater than seq 1, so, the intersection is empty
+  if (__left_bound_seq_1 == __last1)
+    return __result;
+
+  // testing  whether the sequences are intersected
+  _RandomAccessIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
+  //{2} < {1}: seq 1 is wholly greater than seq 2, so, the intersection is empty
+  if (__left_bound_seq_2 == __last2)
+    return __result;
+
+  const auto __m1 = __last1 - __left_bound_seq_1 + __n2;
+  if (__m1 > __set_algo_cut_off) {
+    // we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
+    return __internal::__parallel_set_op(
+        __tag,
+        std::forward<_ExecutionPolicy>(__exec),
+        __left_bound_seq_1,
+        __last1,
+        __first2,
+        __last2,
+        __result,
+        __comp,
+        [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
+        [](_RandomAccessIterator1 __first1,
+           _RandomAccessIterator1 __last1,
+           _RandomAccessIterator2 __first2,
+           _RandomAccessIterator2 __last2,
+           _Tp* __result,
+           _Compare __comp) {
+          return __pstl::__utils::__set_intersection_construct(__first1, __last1, __first2, __last2, __result, __comp);
+        });
+  }
+
+  const auto __m2 = __last2 - __left_bound_seq_2 + __n1;
+  if (__m2 > __set_algo_cut_off) {
+    // we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
+    __result = __internal::__parallel_set_op(
+        __tag,
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        __left_bound_seq_2,
+        __last2,
+        __result,
+        __comp,
+        [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
+        [](_RandomAccessIterator1 __first1,
+           _RandomAccessIterator1 __last1,
+           _RandomAccessIterator2 __first2,
+           _RandomAccessIterator2 __last2,
+           _Tp* __result,
+           _Compare __comp) {
+          return __pstl::__utils::__set_intersection_construct(__first2, __last2, __first1, __last1, __result, __comp);
+        });
+    return __result;
+  }
+
+  // [left_bound_seq_1; last1) and [left_bound_seq_2; last2) - use serial algorithm
+  return std::set_intersection(__left_bound_seq_1, __last1, __left_bound_seq_2, __last2, __result, __comp);
+}
+
+//------------------------------------------------------------------------
+// set_difference
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_difference(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
+_RandomAccessIterator3 __brick_set_difference(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __result,
+    _Compare __comp,
+    /*__is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_difference(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) noexcept {
+  return __internal::__brick_set_difference(
+      __first1, __last1, __first2, __last2, __result, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _Compare>
+_RandomAccessIterator3 __pattern_set_difference(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __result,
+    _Compare __comp) {
+  typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
+
+  const auto __n1 = __last1 - __first1;
+  const auto __n2 = __last2 - __first2;
+
+  // {} \ {2}: the difference is empty
+  if (__n1 == 0)
+    return __result;
+
+  // {1} \ {}: parallel copying just first sequence
+  if (__n2 == 0)
+    return __internal::__pattern_walk2_brick(
+        __tag,
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        __result,
+        [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res) {
+          return __internal::__brick_copy(__begin, __end, __res, _IsVector{});
+        });
+
+  // testing  whether the sequences are intersected
+  _RandomAccessIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
+  //{1} < {2}: seq 2 is wholly greater than seq 1, so, parallel copying just first sequence
+  if (__left_bound_seq_1 == __last1)
+    return __internal::__pattern_walk2_brick(
+        __tag,
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        __result,
+        [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res) {
+          return __internal::__brick_copy(__begin, __end, __res, _IsVector{});
+        });
+
+  // testing  whether the sequences are intersected
+  _RandomAccessIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
+  //{2} < {1}: seq 1 is wholly greater than seq 2, so, parallel copying just first sequence
+  if (__left_bound_seq_2 == __last2)
+    return __internal::__pattern_walk2_brick(
+        __tag,
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        __result,
+        [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res) {
+          return __internal::__brick_copy(__begin, __end, __res, _IsVector{});
+        });
+
+  if (__n1 + __n2 > __set_algo_cut_off)
+    return __parallel_set_op(
+        __tag,
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        __first2,
+        __last2,
+        __result,
+        __comp,
+        [](_DifferenceType __n, _DifferenceType) { return __n; },
+        [](_RandomAccessIterator1 __first1,
+           _RandomAccessIterator1 __last1,
+           _RandomAccessIterator2 __first2,
+           _RandomAccessIterator2 __last2,
+           _Tp* __result,
+           _Compare __comp) {
+          return __pstl::__utils::__set_difference_construct(
+              __first1, __last1, __first2, __last2, __result, __comp, __BrickCopyConstruct<_IsVector>());
+        });
+
+  // use serial algorithm
+  return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+//------------------------------------------------------------------------
+// set_symmetric_difference
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
+_OutputIterator __brick_set_symmetric_difference(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp,
+    /*__is_vector=*/std::false_type) noexcept {
+  return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
+_RandomAccessIterator3 __brick_set_symmetric_difference(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __result,
+    _Compare __comp,
+    /*__is_vector=*/std::true_type) noexcept {
+  // TODO: vectorize
+  return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _OutputIterator,
+          class _Compare>
+_OutputIterator __pattern_set_symmetric_difference(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _OutputIterator __result,
+    _Compare __comp) noexcept {
+  return __internal::__brick_set_symmetric_difference(
+      __first1, __last1, __first2, __last2, __result, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _RandomAccessIterator3,
+          class _Compare>
+_RandomAccessIterator3 __pattern_set_symmetric_difference(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _RandomAccessIterator3 __result,
+    _Compare __comp) {
+  const auto __n1 = __last1 - __first1;
+  const auto __n2 = __last2 - __first2;
+
+  // use serial algorithm
+  if (__n1 + __n2 <= __set_algo_cut_off)
+    return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
+
+  typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
+  return __internal::__parallel_set_union_op(
+      __tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __last2,
+      __result,
+      __comp,
+      [](_RandomAccessIterator1 __first1,
+         _RandomAccessIterator1 __last1,
+         _RandomAccessIterator2 __first2,
+         _RandomAccessIterator2 __last2,
+         _Tp* __result,
+         _Compare __comp) {
+        return __pstl::__utils::__set_symmetric_difference_construct(
+            __first1, __last1, __first2, __last2, __result, __comp, __BrickCopyConstruct<_IsVector>());
+      });
+}
+
+//------------------------------------------------------------------------
+// is_heap_until
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __brick_is_heap_until(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  return std::is_heap_until(__first, __last, __comp);
+}
+
+template <class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __brick_is_heap_until(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+  if (__last - __first < 2)
+    return __last;
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
+  return __unseq_backend::__simd_first(
+      __first, _SizeType(0), __last - __first, [&__comp](_RandomAccessIterator __it, _SizeType __i) {
+        return __comp(__it[(__i - 1) / 2], __it[__i]);
+      });
+}
+
+template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __pattern_is_heap_until(
+    _Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) noexcept {
+  return __internal::__brick_is_heap_until(__first, __last, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
+_RandomAccessIterator __is_heap_until_local(
+    _RandomAccessIterator __first,
+    _DifferenceType __begin,
+    _DifferenceType __end,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  _DifferenceType __i = __begin;
+  for (; __i < __end; ++__i) {
+    if (__comp(__first[(__i - 1) / 2], __first[__i])) {
+      break;
+    }
+  }
+  return __first + __i;
+}
+
+template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
+_RandomAccessIterator __is_heap_until_local(
+    _RandomAccessIterator __first,
+    _DifferenceType __begin,
+    _DifferenceType __end,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+  return __unseq_backend::__simd_first(
+      __first, __begin, __end, [&__comp](_RandomAccessIterator __it, _DifferenceType __i) {
+        return __comp(__it[(__i - 1) / 2], __it[__i]);
+      });
+}
+
+template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+_RandomAccessIterator __pattern_is_heap_until(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__last - __first < 2)
+    return __last;
+
+  return __internal::__except_handler([&]() {
+    return __parallel_find(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        [__first, __comp](_RandomAccessIterator __i, _RandomAccessIterator __j) {
+          return __internal::__is_heap_until_local(__first, __i - __first, __j - __first, __comp, _IsVector{});
+        },
+        std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(),
+        /*is_first=*/true);
+  });
+}
+
+//------------------------------------------------------------------------
+// min_element
+//------------------------------------------------------------------------
+
+template <typename _ForwardIterator, typename _Compare>
+_ForwardIterator __brick_min_element(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  return std::min_element(__first, __last, __comp);
+}
+
+template <typename _RandomAccessIterator, typename _Compare>
+_RandomAccessIterator __brick_min_element(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+#if defined(_PSTL_UDR_PRESENT)
+  return __unseq_backend::__simd_min_element(__first, __last - __first, __comp);
+#else
+  return std::min_element(__first, __last, __comp);
+#endif
+}
+
+template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
+_ForwardIterator __pattern_min_element(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) noexcept {
+  return __internal::__brick_min_element(__first, __last, __comp, typename _Tag::__is_vector{});
+}
+
+template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
+_RandomAccessIterator __pattern_min_element(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp) {
+  if (__first == __last)
+    return __last;
+
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    return __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first + 1,
+        __last,
+        __first,
+        [=](_RandomAccessIterator __begin, _RandomAccessIterator __end, _RandomAccessIterator __init)
+            -> _RandomAccessIterator {
+          const _RandomAccessIterator subresult = __internal::__brick_min_element(__begin, __end, __comp, _IsVector{});
+          return __internal::__cmp_iterators_by_values(__init, subresult, __comp);
+        },
+        [=](_RandomAccessIterator __it1, _RandomAccessIterator __it2) -> _RandomAccessIterator {
+          return __internal::__cmp_iterators_by_values(__it1, __it2, __comp);
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// minmax_element
+//------------------------------------------------------------------------
+
+template <typename _ForwardIterator, typename _Compare>
+std::pair<_ForwardIterator, _ForwardIterator> __brick_minmax_element(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  return std::minmax_element(__first, __last, __comp);
+}
+
+template <typename _RandomAccessIterator, typename _Compare>
+std::pair<_RandomAccessIterator, _RandomAccessIterator> __brick_minmax_element(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+#if defined(_PSTL_UDR_PRESENT)
+  return __unseq_backend::__simd_minmax_element(__first, __last - __first, __comp);
+#else
+  return std::minmax_element(__first, __last, __comp);
+#endif
+}
+
+template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
+std::pair<_ForwardIterator, _ForwardIterator> __pattern_minmax_element(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) noexcept {
+  return __internal::__brick_minmax_element(__first, __last, __comp, typename _Tag::__is_vector{});
+}
+
+template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
+std::pair<_RandomAccessIterator, _RandomAccessIterator> __pattern_minmax_element(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Compare __comp) {
+  if (__first == __last)
+    return std::make_pair(__first, __first);
+
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    typedef std::pair<_RandomAccessIterator, _RandomAccessIterator> _Result;
+
+    return __par_backend::__parallel_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first + 1,
+        __last,
+        std::make_pair(__first, __first),
+        [=](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Result __init) -> _Result {
+          const _Result __subresult = __internal::__brick_minmax_element(__begin, __end, __comp, _IsVector{});
+          return std::make_pair(
+              __internal::__cmp_iterators_by_values(__subresult.first, __init.first, __comp),
+              __internal::__cmp_iterators_by_values(__init.second, __subresult.second, std::not_fn(__comp)));
+        },
+        [=](_Result __p1, _Result __p2) -> _Result {
+          return std::make_pair(__internal::__cmp_iterators_by_values(__p1.first, __p2.first, __comp),
+                                __internal::__cmp_iterators_by_values(__p2.second, __p1.second, std::not_fn(__comp)));
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// mismatch
+//------------------------------------------------------------------------
+template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+std::pair<_ForwardIterator1, _ForwardIterator2> __mismatch_serial(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _BinaryPredicate __pred) {
+  return std::mismatch(__first1, __last1, __first2, __last2, __pred);
+}
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+std::pair<_ForwardIterator1, _ForwardIterator2> __brick_mismatch(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Predicate __pred,
+    /* __is_vector = */ std::false_type) noexcept {
+  return __mismatch_serial(__first1, __last1, __first2, __last2, __pred);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate>
+std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __brick_mismatch(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _Predicate __pred,
+    /* __is_vector = */ std::true_type) noexcept {
+  auto __n = std::min(__last1 - __first1, __last2 - __first2);
+  return __unseq_backend::__simd_first(__first1, __n, __first2, std::not_fn(__pred));
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+std::pair<_ForwardIterator1, _ForwardIterator2> __pattern_mismatch(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Predicate __pred) noexcept {
+  return __internal::__brick_mismatch(__first1, __last1, __first2, __last2, __pred, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Predicate>
+std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __pattern_mismatch(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _Predicate __pred) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    auto __n      = std::min(__last1 - __first1, __last2 - __first2);
+    auto __result = __internal::__parallel_find(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __first1 + __n,
+        [__first1, __first2, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          return __internal::__brick_mismatch(
+                     __i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1), __pred, _IsVector{})
+              .first;
+        },
+        std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
+        /*is_first=*/true);
+    return std::make_pair(__result, __first2 + (__result - __first1));
+  });
+}
+
+//------------------------------------------------------------------------
+// lexicographical_compare
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+bool __brick_lexicographical_compare(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Compare __comp,
+    /* __is_vector = */ std::false_type) noexcept {
+  return std::lexicographical_compare(__first1, __last1, __first2, __last2, __comp);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Compare>
+bool __brick_lexicographical_compare(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _Compare __comp,
+    /* __is_vector = */ std::true_type) noexcept {
+  if (__first2 == __last2) {        // if second sequence is empty
+    return false;
+  } else if (__first1 == __last1) { // if first sequence is empty
+    return true;
+  } else {
+    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference ref_type1;
+    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference ref_type2;
+    --__last1;
+    --__last2;
+    auto __n = std::min(__last1 - __first1, __last2 - __first2);
+    std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __result = __unseq_backend::__simd_first(
+        __first1, __n, __first2, [__comp](const ref_type1 __x, const ref_type2 __y) mutable {
+          return __comp(__x, __y) || __comp(__y, __x);
+        });
+
+    if (__result.first == __last1 && __result.second != __last2) { // if first sequence shorter than second
+      return !__comp(*__result.second, *__result.first);
+    } else { // if second sequence shorter than first or both have the same number of elements
+      return __comp(*__result.first, *__result.second);
+    }
+  }
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+bool __pattern_lexicographical_compare(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Compare __comp) noexcept {
+  return __internal::__brick_lexicographical_compare(
+      __first1, __last1, __first2, __last2, __comp, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Compare>
+bool __pattern_lexicographical_compare(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _RandomAccessIterator2 __last2,
+    _Compare __comp) noexcept {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  if (__first2 == __last2) {        // if second sequence is empty
+    return false;
+  } else if (__first1 == __last1) { // if first sequence is empty
+    return true;
+  } else {
+    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _RefType1;
+    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _RefType2;
+    --__last1;
+    --__last2;
+    auto __n      = std::min(__last1 - __first1, __last2 - __first2);
+    auto __result = __internal::__parallel_find(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __first1 + __n,
+        [__first1, __first2, &__comp](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
+          return __internal::__brick_mismatch(
+                     __i,
+                     __j,
+                     __first2 + (__i - __first1),
+                     __first2 + (__j - __first1),
+                     [&__comp](const _RefType1 __x, const _RefType2 __y) {
+                       return !__comp(__x, __y) && !__comp(__y, __x);
+                     },
+                     _IsVector{})
+              .first;
+        },
+        std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
+        /*is_first=*/true);
+
+    if (__result == __last1 && __first2 + (__result - __first1) != __last2) { // if first sequence shorter than second
+      return !__comp(*(__first2 + (__result - __first1)), *__result);
+    } else { // if second sequence shorter than first or both have the same number of elements
+      return __comp(*__result, *(__first2 + (__result - __first1)));
+    }
+  }
+}
+
+} // namespace __internal
+} // namespace __pstl
+
+#endif /* _PSTL_ALGORITHM_IMPL_H */
diff --git a/libcxx/include/pstl/internal/execution_defs.h b/libcxx/include/__pstl/internal/execution_defs.h
similarity index 69%
rename from libcxx/include/pstl/internal/execution_defs.h
rename to libcxx/include/__pstl/internal/execution_defs.h
index d7c4126972338..af6164bc722cb 100644
--- a/libcxx/include/pstl/internal/execution_defs.h
+++ b/libcxx/include/__pstl/internal/execution_defs.h
@@ -10,37 +10,26 @@
 #ifndef _PSTL_EXECUTION_POLICY_DEFS_H
 #define _PSTL_EXECUTION_POLICY_DEFS_H
 
-#include <type_traits>
+#include <__type_traits/decay.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
 
 #include "pstl_config.h"
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace execution
-{
-inline namespace v1
-{
+namespace __pstl {
+namespace execution {
+inline namespace v1 {
 
 // 2.4, Sequential execution policy
-class sequenced_policy
-{
-};
+class sequenced_policy {};
 
 // 2.5, Parallel execution policy
-class parallel_policy
-{
-};
+class parallel_policy {};
 
 // 2.6, Parallel+Vector execution policy
-class parallel_unsequenced_policy
-{
-};
+class parallel_unsequenced_policy {};
 
-class unsequenced_policy
-{
-};
+class unsequenced_policy {};
 
 // 2.8, Execution policy objects
 constexpr sequenced_policy seq{};
@@ -49,42 +38,28 @@ constexpr parallel_unsequenced_policy par_unseq{};
 constexpr unsequenced_policy unseq{};
 
 // 2.3, Execution policy type trait
-template <class T>
-struct is_execution_policy : std::false_type
-{
-};
+template <class>
+struct is_execution_policy : std::false_type {};
 
 template <>
-struct is_execution_policy<__pstl::execution::sequenced_policy> : std::true_type
-{
-};
+struct is_execution_policy<__pstl::execution::sequenced_policy> : std::true_type {};
 template <>
-struct is_execution_policy<__pstl::execution::parallel_policy> : std::true_type
-{
-};
+struct is_execution_policy<__pstl::execution::parallel_policy> : std::true_type {};
 template <>
-struct is_execution_policy<__pstl::execution::parallel_unsequenced_policy> : std::true_type
-{
-};
+struct is_execution_policy<__pstl::execution::parallel_unsequenced_policy> : std::true_type {};
 template <>
-struct is_execution_policy<__pstl::execution::unsequenced_policy> : std::true_type
-{
-};
-
-#if defined(_PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT)
-template <class T>
-constexpr bool is_execution_policy_v = __pstl::execution::is_execution_policy<T>::value;
-#endif
+struct is_execution_policy<__pstl::execution::unsequenced_policy> : std::true_type {};
 
+template <class _Tp>
+constexpr bool is_execution_policy_v = __pstl::execution::is_execution_policy<_Tp>::value;
 } // namespace v1
 } // namespace execution
 
-namespace __internal
-{
-template <class ExecPolicy, class T>
+namespace __internal {
+template <class _ExecPolicy, class _Tp>
 using __enable_if_execution_policy =
-    typename std::enable_if<__pstl::execution::is_execution_policy<typename std::decay<ExecPolicy>::type>::value,
-                            T>::type;
+    typename std::enable_if<__pstl::execution::is_execution_policy<typename std::decay<_ExecPolicy>::type>::value,
+                            _Tp>::type;
 
 template <class _IsVector>
 struct __serial_tag;
@@ -95,6 +70,4 @@ struct __parallel_tag;
 
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_EXECUTION_POLICY_DEFS_H */
diff --git a/libcxx/include/pstl/internal/execution_impl.h b/libcxx/include/__pstl/internal/execution_impl.h
similarity index 59%
rename from libcxx/include/pstl/internal/execution_impl.h
rename to libcxx/include/__pstl/internal/execution_impl.h
index 5dc622b441221..cc60bce8216b7 100644
--- a/libcxx/include/pstl/internal/execution_impl.h
+++ b/libcxx/include/__pstl/internal/execution_impl.h
@@ -10,18 +10,13 @@
 #ifndef _PSTL_EXECUTION_IMPL_H
 #define _PSTL_EXECUTION_IMPL_H
 
-#include <iterator>
-#include <type_traits>
+#include <__iterator/iterator_traits.h>
 
-#include "pstl_config.h"
 #include "execution_defs.h"
+#include "pstl_config.h"
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
+namespace __pstl {
+namespace __internal {
 
 template <typename _IteratorTag, typename... _IteratorTypes>
 using __are_iterators_of = std::conjunction<
@@ -30,15 +25,9 @@ using __are_iterators_of = std::conjunction<
 template <typename... _IteratorTypes>
 using __are_random_access_iterators = __are_iterators_of<std::random_access_iterator_tag, _IteratorTypes...>;
 
-struct __serial_backend_tag
-{
-};
-struct __tbb_backend_tag
-{
-};
-struct __openmp_backend_tag
-{
-};
+struct __serial_backend_tag {};
+struct __tbb_backend_tag {};
+struct __openmp_backend_tag {};
 
 #if defined(_PSTL_PAR_BACKEND_TBB)
 using __par_backend_tag = __tbb_backend_tag;
@@ -47,59 +36,53 @@ using __par_backend_tag = __openmp_backend_tag;
 #elif defined(_PSTL_PAR_BACKEND_SERIAL)
 using __par_backend_tag = __serial_backend_tag;
 #else
-#    error "A parallel backend must be specified";
+#  error "A parallel backend must be specified";
 #endif
 
 template <class _IsVector>
-struct __serial_tag
-{
-    using __is_vector = _IsVector;
+struct __serial_tag {
+  using __is_vector = _IsVector;
 };
 
 template <class _IsVector>
-struct __parallel_tag
-{
-    using __is_vector = _IsVector;
-    // backend tag can be change depending on
-    // TBB availability in the environment
-    using __backend_tag = __par_backend_tag;
+struct __parallel_tag {
+  using __is_vector = _IsVector;
+  // backend tag can be change depending on
+  // TBB availability in the environment
+  using __backend_tag = __par_backend_tag;
 };
 
 template <class _IsVector, class... _IteratorTypes>
-using __tag_type = typename std::conditional<__internal::__are_random_access_iterators<_IteratorTypes...>::value,
-                                             __parallel_tag<_IsVector>, __serial_tag<_IsVector>>::type;
+using __tag_type =
+    typename std::conditional<__internal::__are_random_access_iterators<_IteratorTypes...>::value,
+                              __parallel_tag<_IsVector>,
+                              __serial_tag<_IsVector>>::type;
 
 template <class... _IteratorTypes>
-__serial_tag</*_IsVector = */ std::false_type>
-__select_backend(__pstl::execution::sequenced_policy, _IteratorTypes&&...)
-{
-    return {};
+_LIBCPP_HIDE_FROM_ABI __serial_tag</*_IsVector = */ std::false_type>
+__select_backend(__pstl::execution::sequenced_policy, _IteratorTypes&&...) {
+  return {};
 }
 
 template <class... _IteratorTypes>
-__serial_tag<__internal::__are_random_access_iterators<_IteratorTypes...>>
-__select_backend(__pstl::execution::unsequenced_policy, _IteratorTypes&&...)
-{
-    return {};
+_LIBCPP_HIDE_FROM_ABI __serial_tag<__internal::__are_random_access_iterators<_IteratorTypes...>>
+__select_backend(__pstl::execution::unsequenced_policy, _IteratorTypes&&...) {
+  return {};
 }
 
 template <class... _IteratorTypes>
-__tag_type</*_IsVector = */ std::false_type, _IteratorTypes...>
-__select_backend(__pstl::execution::parallel_policy, _IteratorTypes&&...)
-{
-    return {};
+_LIBCPP_HIDE_FROM_ABI __tag_type</*_IsVector = */ std::false_type, _IteratorTypes...>
+__select_backend(__pstl::execution::parallel_policy, _IteratorTypes&&...) {
+  return {};
 }
 
 template <class... _IteratorTypes>
-__tag_type<__internal::__are_random_access_iterators<_IteratorTypes...>, _IteratorTypes...>
-__select_backend(__pstl::execution::parallel_unsequenced_policy, _IteratorTypes&&...)
-{
-    return {};
+_LIBCPP_HIDE_FROM_ABI __tag_type<__internal::__are_random_access_iterators<_IteratorTypes...>, _IteratorTypes...>
+__select_backend(__pstl::execution::parallel_unsequenced_policy, _IteratorTypes&&...) {
+  return {};
 }
 
 } // namespace __internal
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_EXECUTION_IMPL_H */
diff --git a/libcxx/include/pstl/internal/glue_algorithm_defs.h b/libcxx/include/__pstl/internal/glue_algorithm_defs.h
similarity index 63%
rename from libcxx/include/pstl/internal/glue_algorithm_defs.h
rename to libcxx/include/__pstl/internal/glue_algorithm_defs.h
index 28a7f92163b1c..f751802b47a13 100644
--- a/libcxx/include/pstl/internal/glue_algorithm_defs.h
+++ b/libcxx/include/__pstl/internal/glue_algorithm_defs.h
@@ -16,76 +16,45 @@
 #include "execution_defs.h"
 #include "pstl_config.h"
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [alg.any_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.all_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-all_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.none_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-none_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.foreach]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-for_each(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-for_each_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, _Function __f);
-
-// [alg.find]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if_not(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
+namespace std {
 
 // [alg.find.end]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last, _BinaryPredicate __pred);
+find_end(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first,
+         _ForwardIterator1 __last,
+         _ForwardIterator2 __s_first,
+         _ForwardIterator2 __s_last,
+         _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
+find_end(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first,
+         _ForwardIterator1 __last,
+         _ForwardIterator2 __s_first,
          _ForwardIterator2 __s_last);
 
 // [alg.find_first_of]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1> find_first_of(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last);
+find_first_of(_ExecutionPolicy&& __exec,
+              _ForwardIterator1 __first,
+              _ForwardIterator1 __last,
+              _ForwardIterator2 __s_first,
+              _ForwardIterator2 __s_last);
 
 // [alg.adjacent_find]
 
@@ -113,23 +82,33 @@ count_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator _
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last, _BinaryPredicate __pred);
+search(_ExecutionPolicy&& __exec,
+       _ForwardIterator1 __first,
+       _ForwardIterator1 __last,
+       _ForwardIterator2 __s_first,
+       _ForwardIterator2 __s_last,
+       _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
+search(_ExecutionPolicy&& __exec,
+       _ForwardIterator1 __first,
+       _ForwardIterator1 __last,
+       _ForwardIterator2 __s_first,
        _ForwardIterator2 __s_last);
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value, _BinaryPredicate __pred);
+search_n(_ExecutionPolicy&& __exec,
+         _ForwardIterator __first,
+         _ForwardIterator __last,
+         _Size __count,
+         const _Tp& __value,
+         _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> search_n(
+    _ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value);
 
 // [alg.copy]
 
@@ -143,60 +122,76 @@ copy_n(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _Size __n, _Forward
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 result,
+copy_if(_ExecutionPolicy&& __exec,
+        _ForwardIterator1 __first,
+        _ForwardIterator1 __last,
+        _ForwardIterator2 result,
         _Predicate __pred);
 
 // [alg.swap]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-swap_ranges(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-            _ForwardIterator2 __first2);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> swap_ranges(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2);
 
 // [alg.transform]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
+transform(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first,
+          _ForwardIterator1 __last,
+          _ForwardIterator2 __result,
           _UnaryOperation __op);
 
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
           class _BinaryOperation>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator __result, _BinaryOperation __op);
+transform(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first1,
+          _ForwardIterator1 __last1,
+          _ForwardIterator2 __first2,
+          _ForwardIterator __result,
+          _BinaryOperation __op);
 
 // [alg.replace]
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _Tp>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
+replace_if(_ExecutionPolicy&& __exec,
+           _ForwardIterator __first,
+           _ForwardIterator __last,
+           _UnaryPredicate __pred,
            const _Tp& __new_value);
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value,
+replace(_ExecutionPolicy&& __exec,
+        _ForwardIterator __first,
+        _ForwardIterator __last,
+        const _Tp& __old_value,
         const _Tp& __new_value);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                _ForwardIterator2 __result, _UnaryPredicate __pred, const _Tp& __new_value);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> replace_copy_if(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _UnaryPredicate __pred,
+    const _Tp& __new_value);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-             const _Tp& __old_value, const _Tp& __new_value);
-
-// [alg.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, const _Tp& __value);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> replace_copy(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    const _Tp& __old_value,
+    const _Tp& __new_value);
 
 // [alg.generate]
 template <class _ExecutionPolicy, class _ForwardIterator, class _Generator>
@@ -210,13 +205,19 @@ generate_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size count, _Ge
 // [alg.remove]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Predicate __pred);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> remove_copy_if(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Predicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
+remove_copy(_ExecutionPolicy&& __exec,
+            _ForwardIterator1 __first,
+            _ForwardIterator1 __last,
+            _ForwardIterator2 __result,
             const _Tp& __value);
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
@@ -239,7 +240,10 @@ unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __l
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
+unique_copy(_ExecutionPolicy&& __exec,
+            _ForwardIterator1 __first,
+            _ForwardIterator1 __last,
+            _ForwardIterator2 __result,
             _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
@@ -254,7 +258,9 @@ reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _Bidirectiona
 
 template <class _ExecutionPolicy, class _BidirectionalIterator, class _ForwardIterator>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
+reverse_copy(_ExecutionPolicy&& __exec,
+             _BidirectionalIterator __first,
+             _BidirectionalIterator __last,
              _ForwardIterator __d_first);
 
 // [alg.rotate]
@@ -265,7 +271,10 @@ rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __m
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __middle, _ForwardIterator1 __last,
+rotate_copy(_ExecutionPolicy&& __exec,
+            _ForwardIterator1 __first,
+            _ForwardIterator1 __middle,
+            _ForwardIterator1 __last,
             _ForwardIterator2 __result);
 
 // [alg.partitions]
@@ -279,15 +288,21 @@ __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardItera
 partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator>
-stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                 _UnaryPredicate __pred);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator> stable_partition(
+    _ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred);
 
-template <class _ExecutionPolicy, class _ForwardIterator, class _ForwardIterator1, class _ForwardIterator2,
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
           class _UnaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-partition_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-               _ForwardIterator1 __out_true, _ForwardIterator2 __out_false, _UnaryPredicate __pred);
+partition_copy(_ExecutionPolicy&& __exec,
+               _ForwardIterator __first,
+               _ForwardIterator __last,
+               _ForwardIterator1 __out_true,
+               _ForwardIterator2 __out_false,
+               _UnaryPredicate __pred);
 
 // [alg.sort]
 
@@ -313,17 +328,27 @@ stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAcc
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _BinaryPredicate __pred);
+mismatch(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _ForwardIterator2 __last2,
+         _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
+mismatch(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
          _BinaryPredicate __pred);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
+mismatch(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
          _ForwardIterator2 __last2);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
@@ -334,7 +359,10 @@ mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
+equal(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
       _BinaryPredicate __p);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
@@ -343,12 +371,19 @@ equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 _
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _BinaryPredicate __p);
+equal(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _BinaryPredicate __p);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
+equal(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
       _ForwardIterator2 __last2);
 
 // [alg.move]
@@ -360,25 +395,37 @@ move(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __l
 
 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last, _Compare __comp);
+partial_sort(_ExecutionPolicy&& __exec,
+             _RandomAccessIterator __first,
+             _RandomAccessIterator __middle,
+             _RandomAccessIterator __last,
+             _Compare __comp);
 
 template <class _ExecutionPolicy, class _RandomAccessIterator>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
+partial_sort(_ExecutionPolicy&& __exec,
+             _RandomAccessIterator __first,
+             _RandomAccessIterator __middle,
              _RandomAccessIterator __last);
 
 // [partial.sort.copy]
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator> partial_sort_copy(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _RandomAccessIterator __d_first,
+    _RandomAccessIterator __d_last,
+    _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator> partial_sort_copy(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _RandomAccessIterator __d_first,
+    _RandomAccessIterator __d_last);
 
 // [is.sorted]
 template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
@@ -401,100 +448,176 @@ is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator
 
 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last, _Compare __comp);
+nth_element(_ExecutionPolicy&& __exec,
+            _RandomAccessIterator __first,
+            _RandomAccessIterator __nth,
+            _RandomAccessIterator __last,
+            _Compare __comp);
 
 template <class _ExecutionPolicy, class _RandomAccessIterator>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
+nth_element(_ExecutionPolicy&& __exec,
+            _RandomAccessIterator __first,
+            _RandomAccessIterator __nth,
             _RandomAccessIterator __last);
 
 // [alg.merge]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
           class _Compare>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first, _Compare __comp);
+merge(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardIterator __d_first,
+      _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first);
+merge(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardIterator __d_first);
 
 template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last, _Compare __comp);
+inplace_merge(_ExecutionPolicy&& __exec,
+              _BidirectionalIterator __first,
+              _BidirectionalIterator __middle,
+              _BidirectionalIterator __last,
+              _Compare __comp);
 
 template <class _ExecutionPolicy, class _BidirectionalIterator>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
+inplace_merge(_ExecutionPolicy&& __exec,
+              _BidirectionalIterator __first,
+              _BidirectionalIterator __middle,
               _BidirectionalIterator __last);
 
 // [includes]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _Compare __comp);
+includes(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _ForwardIterator2 __last2,
+         _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
+includes(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
          _ForwardIterator2 __last2);
 
 // [set.union]
 
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
           class _Compare>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp);
+set_union(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first1,
+          _ForwardIterator1 __last1,
+          _ForwardIterator2 __first2,
+          _ForwardIterator2 __last2,
+          _ForwardIterator __result,
+          _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
 __pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result);
+set_union(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first1,
+          _ForwardIterator1 __last1,
+          _ForwardIterator2 __first2,
+          _ForwardIterator2 __last2,
+          _ForwardIterator __result);
 
 // [set.intersection]
 
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
           class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_intersection(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result,
+    _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_intersection(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result);
 
 // [set.difference]
 
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
           class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result,
+    _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result);
 
 // [set.symmetric.difference]
 
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
           class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator result,
-                         _Compare __comp);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_symmetric_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator result,
+    _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_symmetric_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result);
 
 // [is.heap]
 template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
@@ -542,17 +665,22 @@ minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIter
 // [alg.lex.comparison]
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool> lexicographical_compare(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Compare __comp);
 
 template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2);
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool> lexicographical_compare(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2);
 
 } // namespace std
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_GLUE_ALGORITHM_DEFS_H */
diff --git a/libcxx/include/__pstl/internal/glue_algorithm_impl.h b/libcxx/include/__pstl/internal/glue_algorithm_impl.h
new file mode 100644
index 0000000000000..3d207185401bd
--- /dev/null
+++ b/libcxx/include/__pstl/internal/glue_algorithm_impl.h
@@ -0,0 +1,1170 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_GLUE_ALGORITHM_IMPL_H
+#define _PSTL_GLUE_ALGORITHM_IMPL_H
+
+#include <functional>
+
+#include "pstl_config.h"
+
+#include "algorithm_fwd.h"
+#include "execution_defs.h"
+#include "numeric_fwd.h" /* count and count_if use __pattern_transform_reduce */
+#include "utils.h"
+
+#include "execution_impl.h"
+
+namespace std {
+
+// [alg.find.end]
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
+find_end(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first,
+         _ForwardIterator1 __last,
+         _ForwardIterator2 __s_first,
+         _ForwardIterator2 __s_last,
+         _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
+
+  return __pstl::__internal::__pattern_find_end(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
+find_end(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first,
+         _ForwardIterator1 __last,
+         _ForwardIterator2 __s_first,
+         _ForwardIterator2 __s_last) {
+  return std::find_end(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, std::equal_to<>());
+}
+
+// [alg.find_first_of]
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1> find_first_of(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __s_first,
+    _ForwardIterator2 __s_last,
+    _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
+
+  return __pstl::__internal::__pattern_find_first_of(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
+find_first_of(_ExecutionPolicy&& __exec,
+              _ForwardIterator1 __first,
+              _ForwardIterator1 __last,
+              _ForwardIterator2 __s_first,
+              _ForwardIterator2 __s_last) {
+  return std::find_first_of(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, std::equal_to<>());
+}
+
+// [alg.adjacent_find]
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  return __pstl::__internal::__pattern_adjacent_find(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      std::equal_to<_ValueType>(),
+      /*first_semantic*/ false);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_adjacent_find(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred, /*first_semantic*/ false);
+}
+
+// [alg.count]
+
+// Implementation note: count and count_if call the pattern directly instead of calling std::transform_reduce
+// so that we do not have to include <numeric>.
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
+                                                 typename iterator_traits<_ForwardIterator>::difference_type>
+count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  return __pstl::__internal::__pattern_count(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [&__value](const _ValueType& __x) {
+        return __value == __x;
+      });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
+                                                 typename iterator_traits<_ForwardIterator>::difference_type>
+count_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_count(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
+}
+
+// [alg.search]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
+search(_ExecutionPolicy&& __exec,
+       _ForwardIterator1 __first,
+       _ForwardIterator1 __last,
+       _ForwardIterator2 __s_first,
+       _ForwardIterator2 __s_last,
+       _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
+
+  return __pstl::__internal::__pattern_search(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
+search(_ExecutionPolicy&& __exec,
+       _ForwardIterator1 __first,
+       _ForwardIterator1 __last,
+       _ForwardIterator2 __s_first,
+       _ForwardIterator2 __s_last) {
+  return std::search(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, std::equal_to<>());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+search_n(_ExecutionPolicy&& __exec,
+         _ForwardIterator __first,
+         _ForwardIterator __last,
+         _Size __count,
+         const _Tp& __value,
+         _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_search_n(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __count, __value, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> search_n(
+    _ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value) {
+  return std::search_n(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __count,
+      __value,
+      std::equal_to<typename iterator_traits<_ForwardIterator>::value_type>());
+}
+
+// [alg.copy]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__pattern_walk2_brick(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _ForwardIterator2 __res) {
+        return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{});
+      });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+copy_n(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _Size __n, _ForwardIterator2 __result) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__pattern_walk2_brick_n(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __n,
+      __result,
+      [](_ForwardIterator1 __begin, _Size __sz, _ForwardIterator2 __res) {
+        return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{});
+      });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+copy_if(_ExecutionPolicy&& __exec,
+        _ForwardIterator1 __first,
+        _ForwardIterator1 __last,
+        _ForwardIterator2 __result,
+        _Predicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_copy_if(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __pred);
+}
+
+// [alg.swap]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> swap_ranges(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
+  typedef typename iterator_traits<_ForwardIterator1>::reference _ReferenceType1;
+  typedef typename iterator_traits<_ForwardIterator2>::reference _ReferenceType2;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  return __pstl::__internal::__pattern_walk2(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      [](_ReferenceType1 __x, _ReferenceType2 __y) {
+        using std::swap;
+        swap(__x, __y);
+      });
+}
+
+// [alg.transform]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+transform(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first,
+          _ForwardIterator1 __last,
+          _ForwardIterator2 __result,
+          _UnaryOperation __op) {
+  typedef typename iterator_traits<_ForwardIterator1>::reference _InputType;
+  typedef typename iterator_traits<_ForwardIterator2>::reference _OutputType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_walk2(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      [__op](_InputType __x, _OutputType __y) mutable { __y = __op(__x); });
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
+          class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+transform(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first1,
+          _ForwardIterator1 __last1,
+          _ForwardIterator2 __first2,
+          _ForwardIterator __result,
+          _BinaryOperation __op) {
+  typedef typename iterator_traits<_ForwardIterator1>::reference _Input1Type;
+  typedef typename iterator_traits<_ForwardIterator2>::reference _Input2Type;
+  typedef typename iterator_traits<_ForwardIterator>::reference _OutputType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
+
+  return __pstl::__internal::__pattern_walk3(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __result,
+      [__op](_Input1Type x, _Input2Type y, _OutputType z) mutable { z = __op(x, y); });
+}
+
+// [alg.replace]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+replace_if(_ExecutionPolicy&& __exec,
+           _ForwardIterator __first,
+           _ForwardIterator __last,
+           _UnaryPredicate __pred,
+           const _Tp& __new_value) {
+  typedef typename iterator_traits<_ForwardIterator>::reference _ElementType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__pattern_walk1(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      [&__pred, &__new_value](_ElementType __elem) {
+        if (__pred(__elem)) {
+          __elem = __new_value;
+        }
+      });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+replace(_ExecutionPolicy&& __exec,
+        _ForwardIterator __first,
+        _ForwardIterator __last,
+        const _Tp& __old_value,
+        const _Tp& __new_value) {
+  std::replace_if(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __pstl::__internal::__equal_value<_Tp>(__old_value),
+      __new_value);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryPredicate, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> replace_copy_if(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _UnaryPredicate __pred,
+    const _Tp& __new_value) {
+  typedef typename iterator_traits<_ForwardIterator1>::reference _InputType;
+  typedef typename iterator_traits<_ForwardIterator2>::reference _OutputType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_walk2(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      [__pred, &__new_value](_InputType __x, _OutputType __y) mutable { __y = __pred(__x) ? __new_value : __x; });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> replace_copy(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    const _Tp& __old_value,
+    const _Tp& __new_value) {
+  return std::replace_copy_if(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __pstl::__internal::__equal_value<_Tp>(__old_value),
+      __new_value);
+}
+
+// [alg.generate]
+template <class _ExecutionPolicy, class _ForwardIterator, class _Generator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__pattern_generate(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __g);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Generator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+generate_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, _Generator __g) {
+  if (__count <= 0)
+    return __first;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_generate_n(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __count, __g);
+}
+
+// [alg.remove]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> remove_copy_if(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Predicate __pred) {
+  return std::copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, std::not_fn(__pred));
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+remove_copy(_ExecutionPolicy&& __exec,
+            _ForwardIterator1 __first,
+            _ForwardIterator1 __last,
+            _ForwardIterator2 __result,
+            const _Tp& __value) {
+  return std::copy_if(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __pstl::__internal::__not_equal_value<_Tp>(__value));
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_remove_if(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+remove(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  return std::remove_if(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __pstl::__internal::__equal_value<_Tp>(__value));
+}
+
+// [alg.unique]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_unique(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  return std::unique(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::equal_to<>());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+unique_copy(_ExecutionPolicy&& __exec,
+            _ForwardIterator1 __first,
+            _ForwardIterator1 __last,
+            _ForwardIterator2 __result,
+            _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_unique_copy(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> unique_copy(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result) {
+  return std::unique_copy(__exec, __first, __last, __result, std::equal_to<>());
+}
+
+// [alg.reverse]
+
+template <class _ExecutionPolicy, class _BidirectionalIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__pattern_reverse(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last);
+}
+
+template <class _ExecutionPolicy, class _BidirectionalIterator, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+reverse_copy(_ExecutionPolicy&& __exec,
+             _BidirectionalIterator __first,
+             _BidirectionalIterator __last,
+             _ForwardIterator __d_first) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
+
+  return __pstl::__internal::__pattern_reverse_copy(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first);
+}
+
+// [alg.rotate]
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_rotate(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __middle, __last);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+rotate_copy(_ExecutionPolicy&& __exec,
+            _ForwardIterator1 __first,
+            _ForwardIterator1 __middle,
+            _ForwardIterator1 __last,
+            _ForwardIterator2 __result) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_rotate_copy(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __middle, __last, __result);
+}
+
+// [alg.partitions]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_is_partitioned(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_partition(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
+}
+
+template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator> stable_partition(
+    _ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_stable_partition(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _UnaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
+partition_copy(_ExecutionPolicy&& __exec,
+               _ForwardIterator __first,
+               _ForwardIterator __last,
+               _ForwardIterator1 __out_true,
+               _ForwardIterator2 __out_false,
+               _UnaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __out_true, __out_false);
+
+  return __pstl::__internal::__pattern_partition_copy(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __out_true, __out_false, __pred);
+}
+
+// [alg.sort]
+
+template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  return __pstl::__internal::__pattern_sort(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __comp,
+      typename std::is_move_constructible<_InputType>::type());
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last) {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  std::sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
+}
+
+// [stable.sort]
+
+template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_stable_sort(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last) {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  std::stable_sort(__exec, __first, __last, std::less<_InputType>());
+}
+
+// [mismatch]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
+mismatch(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _ForwardIterator2 __last2,
+         _BinaryPredicate __pred) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  return __pstl::__internal::__pattern_mismatch(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
+mismatch(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _BinaryPredicate __pred) {
+  return std::mismatch(
+      __exec, __first1, __last1, __first2, std::next(__first2, std::distance(__first1, __last1)), __pred);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
+mismatch(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _ForwardIterator2 __last2) {
+  return std::mismatch(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::equal_to<>());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
+mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
+  // TODO: to get rid of "distance"
+  return std::mismatch(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      std::next(__first2, std::distance(__first1, __last1)));
+}
+
+// [alg.equal]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+equal(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _BinaryPredicate __p) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  return __pstl::__internal::__pattern_equal(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __p);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
+  return std::equal(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, std::equal_to<>());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+equal(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _BinaryPredicate __p) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  return __pstl::__internal::__pattern_equal(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __p);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+equal(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2) {
+  return equal(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::equal_to<>());
+}
+
+// [alg.move]
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
+move(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__pattern_walk2_brick(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __d_first,
+      [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _ForwardIterator2 __res) {
+        return __pstl::__internal::__brick_move(__begin, __end, __res, __is_vector{});
+      });
+}
+
+// [partial.sort]
+
+template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+partial_sort(_ExecutionPolicy&& __exec,
+             _RandomAccessIterator __first,
+             _RandomAccessIterator __middle,
+             _RandomAccessIterator __last,
+             _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__pattern_partial_sort(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __middle, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+partial_sort(_ExecutionPolicy&& __exec,
+             _RandomAccessIterator __first,
+             _RandomAccessIterator __middle,
+             _RandomAccessIterator __last) {
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  std::partial_sort(__exec, __first, __middle, __last, std::less<_InputType>());
+}
+
+// [partial.sort.copy]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator> partial_sort_copy(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _RandomAccessIterator __d_first,
+    _RandomAccessIterator __d_last,
+    _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
+
+  return __pstl::__internal::__pattern_partial_sort_copy(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, __d_last, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator> partial_sort_copy(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _RandomAccessIterator __d_first,
+    _RandomAccessIterator __d_last) {
+  return std::partial_sort_copy(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, __d_last, std::less<>());
+}
+
+// [is.sorted]
+template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  auto __dispatch_tag          = __pstl::__internal::__select_backend(__exec, __first);
+  const _ForwardIterator __res = __pstl::__internal::__pattern_adjacent_find(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __pstl::__internal::__reorder_pred<_Compare>(__comp),
+      /*first_semantic*/ false);
+  return __res == __last ? __last : std::next(__res);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
+  return is_sorted_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_adjacent_find(
+             __dispatch_tag,
+             std::forward<_ExecutionPolicy>(__exec),
+             __first,
+             __last,
+             __pstl::__internal::__reorder_pred<_Compare>(__comp),
+             /*or_semantic*/ true) == __last;
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
+  return std::is_sorted(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
+}
+
+// [alg.merge]
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
+          class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+merge(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardIterator __d_first,
+      _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __d_first);
+
+  return __pstl::__internal::__pattern_merge(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+merge(_ExecutionPolicy&& __exec,
+      _ForwardIterator1 __first1,
+      _ForwardIterator1 __last1,
+      _ForwardIterator2 __first2,
+      _ForwardIterator2 __last2,
+      _ForwardIterator __d_first) {
+  return std::merge(
+      std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first, std::less<>());
+}
+
+template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+inplace_merge(_ExecutionPolicy&& __exec,
+              _BidirectionalIterator __first,
+              _BidirectionalIterator __middle,
+              _BidirectionalIterator __last,
+              _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__pattern_inplace_merge(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __middle, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _BidirectionalIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+inplace_merge(_ExecutionPolicy&& __exec,
+              _BidirectionalIterator __first,
+              _BidirectionalIterator __middle,
+              _BidirectionalIterator __last) {
+  typedef typename std::iterator_traits<_BidirectionalIterator>::value_type _InputType;
+  std::inplace_merge(__exec, __first, __middle, __last, std::less<_InputType>());
+}
+
+// [includes]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+includes(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _ForwardIterator2 __last2,
+         _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  return __pstl::__internal::__pattern_includes(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+includes(_ExecutionPolicy&& __exec,
+         _ForwardIterator1 __first1,
+         _ForwardIterator1 __last1,
+         _ForwardIterator2 __first2,
+         _ForwardIterator2 __last2) {
+  return std::includes(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::less<>());
+}
+
+// [set.union]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
+          class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+set_union(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first1,
+          _ForwardIterator1 __last1,
+          _ForwardIterator2 __first2,
+          _ForwardIterator2 __last2,
+          _ForwardIterator __result,
+          _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
+
+  return __pstl::__internal::__pattern_set_union(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+set_union(_ExecutionPolicy&& __exec,
+          _ForwardIterator1 __first1,
+          _ForwardIterator1 __last1,
+          _ForwardIterator2 __first2,
+          _ForwardIterator2 __last2,
+          _ForwardIterator __result) {
+  return std::set_union(
+      std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, std::less<>());
+}
+
+// [set.intersection]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
+          class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_intersection(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result,
+    _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
+
+  return __pstl::__internal::__pattern_set_intersection(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_intersection(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result) {
+  return std::set_intersection(
+      std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, std::less<>());
+}
+
+// [set.difference]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
+          class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result,
+    _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
+
+  return __pstl::__internal::__pattern_set_difference(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result) {
+  return std::set_difference(
+      std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, std::less<>());
+}
+
+// [set.symmetric.difference]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _ForwardIterator,
+          class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_symmetric_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result,
+    _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
+
+  return __pstl::__internal::__pattern_set_symmetric_difference(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> set_symmetric_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _ForwardIterator __result) {
+  return std::set_symmetric_difference(
+      std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, std::less<>());
+}
+
+// [is.heap]
+template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
+is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__pattern_is_heap_until(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
+is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last) {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  return std::is_heap_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) {
+  return std::is_heap_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp) == __last;
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
+is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last) {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  return std::is_heap(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
+}
+
+// [alg.min.max]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_min_element(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
+  return std::min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  return min_element(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __pstl::__internal::__reorder_pred<_Compare>(__comp));
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
+  return std::min_element(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __pstl::__internal::__reorder_pred<std::less<_InputType>>(std::less<_InputType>()));
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
+minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_minmax_element(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
+minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  return std::minmax_element(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_ValueType>());
+}
+
+// [alg.nth.element]
+
+template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+nth_element(_ExecutionPolicy&& __exec,
+            _RandomAccessIterator __first,
+            _RandomAccessIterator __nth,
+            _RandomAccessIterator __last,
+            _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__pattern_nth_element(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __nth, __last, __comp);
+}
+
+template <class _ExecutionPolicy, class _RandomAccessIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+nth_element(_ExecutionPolicy&& __exec,
+            _RandomAccessIterator __first,
+            _RandomAccessIterator __nth,
+            _RandomAccessIterator __last) {
+  typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
+  std::nth_element(std::forward<_ExecutionPolicy>(__exec), __first, __nth, __last, std::less<_InputType>());
+}
+
+// [alg.lex.comparison]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool> lexicographical_compare(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2,
+    _Compare __comp) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  return __pstl::__internal::__pattern_lexicographical_compare(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __comp);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool> lexicographical_compare(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _ForwardIterator2 __last2) {
+  return std::lexicographical_compare(
+      std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::less<>());
+}
+
+} // namespace std
+
+#endif /* _PSTL_GLUE_ALGORITHM_IMPL_H */
diff --git a/libcxx/include/pstl/internal/glue_memory_defs.h b/libcxx/include/__pstl/internal/glue_memory_defs.h
similarity index 98%
rename from libcxx/include/pstl/internal/glue_memory_defs.h
rename to libcxx/include/__pstl/internal/glue_memory_defs.h
index ae523339d52d6..7357546028375 100644
--- a/libcxx/include/pstl/internal/glue_memory_defs.h
+++ b/libcxx/include/__pstl/internal/glue_memory_defs.h
@@ -13,10 +13,7 @@
 #include "execution_defs.h"
 #include "pstl_config.h"
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
+namespace std {
 
 // [uninitialized.copy]
 
@@ -80,6 +77,4 @@ uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __fi
 
 } //  namespace std
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_GLUE_MEMORY_DEFS_H */
diff --git a/libcxx/include/__pstl/internal/glue_memory_impl.h b/libcxx/include/__pstl/internal/glue_memory_impl.h
new file mode 100644
index 0000000000000..e65d6947dabed
--- /dev/null
+++ b/libcxx/include/__pstl/internal/glue_memory_impl.h
@@ -0,0 +1,379 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_GLUE_MEMORY_IMPL_H
+#define _PSTL_GLUE_MEMORY_IMPL_H
+
+#include "pstl_config.h"
+
+#include "algorithm_fwd.h"
+#include "execution_defs.h"
+#include "utils.h"
+
+#include "execution_impl.h"
+
+namespace std {
+
+// [uninitialized.copy]
+
+template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> uninitialized_copy(
+    _ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result) {
+  typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
+  typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__invoke_if_else(
+      std::integral_constant < bool,
+      std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
+      [&]() {
+        return __pstl::__internal::__pattern_walk2_brick(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __last,
+            __result,
+            [](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res) {
+              return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{});
+            });
+      },
+      [&]() {
+        return __pstl::__internal::__pattern_walk2(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __last,
+            __result,
+            [](_ReferenceType1 __val1, _ReferenceType2 __val2) { ::new (std::addressof(__val2)) _ValueType2(__val1); });
+      });
+}
+
+template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result) {
+  typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
+  typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__invoke_if_else(
+      std::integral_constant < bool,
+      std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
+      [&]() {
+        return __pstl::__internal::__pattern_walk2_brick_n(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __n,
+            __result,
+            [](_InputIterator __begin, _Size __sz, _ForwardIterator __res) {
+              return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{});
+            });
+      },
+      [&]() {
+        return __pstl::__internal::__pattern_walk2_n(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __n,
+            __result,
+            [](_ReferenceType1 __val1, _ReferenceType2 __val2) { ::new (std::addressof(__val2)) _ValueType2(__val1); });
+      });
+}
+
+// [uninitialized.move]
+
+template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator> uninitialized_move(
+    _ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result) {
+  typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
+  typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__invoke_if_else(
+      std::integral_constant < bool,
+      std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
+      [&]() {
+        return __pstl::__internal::__pattern_walk2_brick(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __last,
+            __result,
+            [](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res) {
+              return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{});
+            });
+      },
+      [&]() {
+        return __pstl::__internal::__pattern_walk2(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __last,
+            __result,
+            [](_ReferenceType1 __val1, _ReferenceType2 __val2) {
+              ::new (std::addressof(__val2)) _ValueType2(std::move(__val1));
+            });
+      });
+}
+
+template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result) {
+  typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
+  typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__invoke_if_else(
+      std::integral_constant < bool,
+      std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
+      [&]() {
+        return __pstl::__internal::__pattern_walk2_brick_n(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __n,
+            __result,
+            [](_InputIterator __begin, _Size __sz, _ForwardIterator __res) {
+              return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{});
+            });
+      },
+      [&]() {
+        return __pstl::__internal::__pattern_walk2_n(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __n,
+            __result,
+            [](_ReferenceType1 __val1, _ReferenceType2 __val2) {
+              ::new (std::addressof(__val2)) _ValueType2(std::move(__val1));
+            });
+      });
+}
+
+// [uninitialized.fill]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  __pstl::__internal::__invoke_if_else(
+      std::is_arithmetic<_ValueType>(),
+      [&]() {
+        __pstl::__internal::__pattern_walk_brick(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __last,
+            [&__value](_ForwardIterator __begin, _ForwardIterator __end) {
+              __pstl::__internal::__brick_fill(__begin, __end, _ValueType(__value), __is_vector{});
+            });
+      },
+      [&]() {
+        __pstl::__internal::__pattern_walk1(
+            __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [&__value](_ReferenceType __val) {
+              ::new (std::addressof(__val)) _ValueType(__value);
+            });
+      });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__invoke_if_else(
+      std::is_arithmetic<_ValueType>(),
+      [&]() {
+        return __pstl::__internal::__pattern_walk_brick_n(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __n,
+            [&__value](_ForwardIterator __begin, _Size __count) {
+              return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(__value), __is_vector{});
+            });
+      },
+      [&]() {
+        return __pstl::__internal::__pattern_walk1_n(
+            __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [&__value](_ReferenceType __val) {
+              ::new (std::addressof(__val)) _ValueType(__value);
+            });
+      });
+}
+
+// [specialized.destroy]
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__invoke_if_not(std::is_trivially_destructible<_ValueType>(), [&]() {
+    __pstl::__internal::__pattern_walk1(
+        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) {
+          __val.~_ValueType();
+        });
+  });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__invoke_if_else(
+      std::is_trivially_destructible<_ValueType>(),
+      [&]() { return std::next(__first, __n); },
+      [&]() {
+        return __pstl::__internal::__pattern_walk1_n(
+            __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) {
+              __val.~_ValueType();
+            });
+      });
+}
+
+// [uninitialized.construct.default]
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  __pstl::__internal::__invoke_if_not(std::is_trivial<_ValueType>(), [&]() {
+    __pstl::__internal::__pattern_walk1(
+        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) {
+          ::new (std::addressof(__val)) _ValueType;
+        });
+  });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  return __pstl::__internal::__invoke_if_else(
+      std::is_trivial<_ValueType>(),
+      [&]() { return std::next(__first, __n); },
+      [&]() {
+        return __pstl::__internal::__pattern_walk1_n(
+            __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) {
+              ::new (std::addressof(__val)) _ValueType;
+            });
+      });
+}
+
+// [uninitialized.construct.value]
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
+uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  __pstl::__internal::__invoke_if_else(
+      std::is_trivial<_ValueType>(),
+      [&]() {
+        __pstl::__internal::__pattern_walk_brick(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __last,
+            [](_ForwardIterator __begin, _ForwardIterator __end) {
+              __pstl::__internal::__brick_fill(__begin, __end, _ValueType(), __is_vector{});
+            });
+      },
+      [&]() {
+        __pstl::__internal::__pattern_walk1(
+            __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, [](_ReferenceType __val) {
+              ::new (std::addressof(__val)) _ValueType();
+            });
+      });
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
+uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+
+  using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
+
+  return __pstl::__internal::__invoke_if_else(
+      std::is_trivial<_ValueType>(),
+      [&]() {
+        return __pstl::__internal::__pattern_walk_brick_n(
+            __dispatch_tag,
+            std::forward<_ExecutionPolicy>(__exec),
+            __first,
+            __n,
+            [](_ForwardIterator __begin, _Size __count) {
+              return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(), __is_vector{});
+            });
+      },
+      [&]() {
+        return __pstl::__internal::__pattern_walk1_n(
+            __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, [](_ReferenceType __val) {
+              ::new (std::addressof(__val)) _ValueType();
+            });
+      });
+}
+
+} // namespace std
+
+#endif /* _PSTL_GLUE_MEMORY_IMPL_H */
diff --git a/libcxx/include/__pstl/internal/glue_numeric_defs.h b/libcxx/include/__pstl/internal/glue_numeric_defs.h
new file mode 100644
index 0000000000000..40e0202624735
--- /dev/null
+++ b/libcxx/include/__pstl/internal/glue_numeric_defs.h
@@ -0,0 +1,175 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_GLUE_NUMERIC_DEFS_H
+#define _PSTL_GLUE_NUMERIC_DEFS_H
+
+#include <iterator>
+
+#include "execution_defs.h"
+#include "pstl_config.h"
+
+namespace std {
+// [reduce]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
+reduce(_ExecutionPolicy&& __exec,
+       _ForwardIterator __first,
+       _ForwardIterator __last,
+       _Tp __init,
+       _BinaryOperation __binary_op);
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
+reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init);
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
+                                                 typename iterator_traits<_ForwardIterator>::value_type>
+reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init);
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2);
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op);
+
+// [exclusive.scan]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Tp __init);
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Tp __init,
+    _BinaryOperation __binary_op);
+
+// [inclusive.scan]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result);
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op);
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op,
+    _Tp __init);
+
+// [transform.exclusive.scan]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_exclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op);
+
+// [transform.inclusive.scan]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _BinaryOperation,
+          class _UnaryOperation,
+          class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op,
+    _Tp __init);
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _UnaryOperation,
+          class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op);
+
+// [adjacent.difference]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __d_first,
+    _BinaryOperation op);
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first);
+
+} // namespace std
+
+#endif /* _PSTL_GLUE_NUMERIC_DEFS_H */
diff --git a/libcxx/include/__pstl/internal/glue_numeric_impl.h b/libcxx/include/__pstl/internal/glue_numeric_impl.h
new file mode 100644
index 0000000000000..4923191319431
--- /dev/null
+++ b/libcxx/include/__pstl/internal/glue_numeric_impl.h
@@ -0,0 +1,320 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_GLUE_NUMERIC_IMPL_H
+#define _PSTL_GLUE_NUMERIC_IMPL_H
+
+#include <functional>
+
+#include "pstl_config.h"
+
+#include "execution_impl.h"
+#include "numeric_fwd.h"
+#include "utils.h"
+
+namespace std {
+
+// [reduce]
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
+reduce(_ExecutionPolicy&& __exec,
+       _ForwardIterator __first,
+       _ForwardIterator __last,
+       _Tp __init,
+       _BinaryOperation __binary_op) {
+  return transform_reduce(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, __binary_op, __pstl::__internal::__no_op());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
+reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init) {
+  return transform_reduce(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, std::plus<_Tp>(), __pstl::__internal::__no_op());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
+                                                 typename iterator_traits<_ForwardIterator>::value_type>
+reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last) {
+  typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
+  return transform_reduce(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      _ValueType{},
+      std::plus<_ValueType>(),
+      __pstl::__internal::__no_op());
+}
+
+// [transform.reduce]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+
+  typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
+  return __pstl::__internal::__pattern_transform_reduce(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __init,
+      std::plus<_InputType>(),
+      std::multiplies<_InputType>());
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
+  return __pstl::__internal::__pattern_transform_reduce(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first1,
+      __last1,
+      __first2,
+      __init,
+      __binary_op1,
+      __binary_op2);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp> transform_reduce(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
+  return __pstl::__internal::__pattern_transform_reduce(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, __binary_op, __unary_op);
+}
+
+// [exclusive.scan]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Tp __init) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using namespace __pstl;
+  return __internal::__pattern_transform_scan(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __pstl::__internal::__no_op(),
+      __init,
+      std::plus<_Tp>(),
+      /*inclusive=*/std::false_type());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> exclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Tp __init,
+    _BinaryOperation __binary_op) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  using namespace __pstl;
+  return __internal::__pattern_transform_scan(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __pstl::__internal::__no_op(),
+      __init,
+      __binary_op,
+      /*inclusive=*/std::false_type());
+}
+
+// [inclusive.scan]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result) {
+  typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
+  return transform_inclusive_scan(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      std::plus<_InputType>(),
+      __pstl::__internal::__no_op());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op) {
+  return transform_inclusive_scan(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __binary_op, __pstl::__internal::__no_op());
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op,
+    _Tp __init) {
+  return transform_inclusive_scan(
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __binary_op,
+      __pstl::__internal::__no_op(),
+      __init);
+}
+
+// [transform.exclusive.scan]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_exclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_transform_scan(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __unary_op,
+      __init,
+      __binary_op,
+      /*inclusive=*/std::false_type());
+}
+
+// [transform.inclusive.scan]
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _BinaryOperation,
+          class _UnaryOperation,
+          class _Tp>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op,
+    _Tp __init) {
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
+
+  return __pstl::__internal::__pattern_transform_scan(
+      __dispatch_tag,
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last,
+      __result,
+      __unary_op,
+      __init,
+      __binary_op,
+      /*inclusive=*/std::true_type());
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _UnaryOperation,
+          class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> transform_inclusive_scan(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __result,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op) {
+  if (__first != __last) {
+    auto __tmp = __unary_op(*__first);
+    *__result  = __tmp;
+    return transform_inclusive_scan(
+        std::forward<_ExecutionPolicy>(__exec), ++__first, __last, ++__result, __binary_op, __unary_op, __tmp);
+  } else {
+    return __result;
+  }
+}
+
+// [adjacent.difference]
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
+    _ExecutionPolicy&& __exec,
+    _ForwardIterator1 __first,
+    _ForwardIterator1 __last,
+    _ForwardIterator2 __d_first,
+    _BinaryOperation __op) {
+  if (__first == __last)
+    return __d_first;
+
+  auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
+
+  return __pstl::__internal::__pattern_adjacent_difference(
+      __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, __op);
+}
+
+template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
+__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2> adjacent_difference(
+    _ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first) {
+  typedef typename iterator_traits<_ForwardIterator1>::value_type _ValueType;
+  return adjacent_difference(
+      std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, std::minus<_ValueType>());
+}
+
+} // namespace std
+
+#endif /* _PSTL_GLUE_NUMERIC_IMPL_H_ */
diff --git a/libcxx/include/__pstl/internal/memory_impl.h b/libcxx/include/__pstl/internal/memory_impl.h
new file mode 100644
index 0000000000000..1967d5d7b735d
--- /dev/null
+++ b/libcxx/include/__pstl/internal/memory_impl.h
@@ -0,0 +1,106 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_MEMORY_IMPL_H
+#define _PSTL_MEMORY_IMPL_H
+
+#include <iterator>
+
+#include "pstl_config.h"
+#include "unseq_backend_simd.h"
+
+namespace __pstl {
+namespace __internal {
+
+//------------------------------------------------------------------------
+// uninitialized_move
+//------------------------------------------------------------------------
+
+template <typename _ForwardIterator, typename _OutputIterator>
+_OutputIterator __brick_uninitialized_move(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    /*vector=*/std::false_type) noexcept {
+  using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
+  for (; __first != __last; ++__first, ++__result) {
+    ::new (std::addressof(*__result)) _ValueType(std::move(*__first));
+  }
+  return __result;
+}
+
+template <typename _RandomAccessIterator, typename _OutputIterator>
+_OutputIterator __brick_uninitialized_move(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    /*vector=*/std::true_type) noexcept {
+  using __ValueType     = typename std::iterator_traits<_OutputIterator>::value_type;
+  using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
+  using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
+
+  return __unseq_backend::__simd_walk_2(
+      __first, __last - __first, __result, [](_ReferenceType1 __x, _ReferenceType2 __y) {
+        ::new (std::addressof(__y)) __ValueType(std::move(__x));
+      });
+}
+
+template <typename _Iterator>
+void __brick_destroy(_Iterator __first, _Iterator __last, /*vector*/ std::false_type) noexcept {
+  using _ValueType = typename std::iterator_traits<_Iterator>::value_type;
+
+  for (; __first != __last; ++__first)
+    __first->~_ValueType();
+}
+
+template <typename _RandomAccessIterator>
+void __brick_destroy(_RandomAccessIterator __first, _RandomAccessIterator __last, /*vector*/ std::true_type) noexcept {
+  using _ValueType     = typename std::iterator_traits<_RandomAccessIterator>::value_type;
+  using _ReferenceType = typename std::iterator_traits<_RandomAccessIterator>::reference;
+
+  __unseq_backend::__simd_walk_1(__first, __last - __first, [](_ReferenceType __x) { __x.~_ValueType(); });
+}
+
+//------------------------------------------------------------------------
+// uninitialized copy
+//------------------------------------------------------------------------
+
+template <typename _ForwardIterator, typename _OutputIterator>
+_OutputIterator __brick_uninitialized_copy(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    /*vector=*/std::false_type) noexcept {
+  using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
+  for (; __first != __last; ++__first, ++__result) {
+    ::new (std::addressof(*__result)) _ValueType(*__first);
+  }
+  return __result;
+}
+
+template <typename _RandomAccessIterator, typename _OutputIterator>
+_OutputIterator __brick_uninitialized_copy(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    /*vector=*/std::true_type) noexcept {
+  using __ValueType     = typename std::iterator_traits<_OutputIterator>::value_type;
+  using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
+  using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
+
+  return __unseq_backend::__simd_walk_2(
+      __first, __last - __first, __result, [](_ReferenceType1 __x, _ReferenceType2 __y) {
+        ::new (std::addressof(__y)) __ValueType(__x);
+      });
+}
+
+} // namespace __internal
+} // namespace __pstl
+
+#endif /* _PSTL_MEMORY_IMPL_H */
diff --git a/libcxx/include/__pstl/internal/numeric_fwd.h b/libcxx/include/__pstl/internal/numeric_fwd.h
new file mode 100644
index 0000000000000..ccf7b634b2f0f
--- /dev/null
+++ b/libcxx/include/__pstl/internal/numeric_fwd.h
@@ -0,0 +1,252 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_NUMERIC_FWD_H
+#define _PSTL_NUMERIC_FWD_H
+
+#include <type_traits>
+#include <utility>
+
+#include "pstl_config.h"
+
+namespace __pstl {
+namespace __internal {
+
+//------------------------------------------------------------------------
+// transform_reduce (version with two binary functions, according to draft N4659)
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+_Tp __brick_transform_reduce(
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _Tp,
+    _BinaryOperation1,
+    _BinaryOperation2,
+    /*__is_vector=*/std::true_type) noexcept;
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
+_Tp __brick_transform_reduce(
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _Tp,
+    _BinaryOperation1,
+    _BinaryOperation2,
+    /*__is_vector=*/std::false_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+_Tp __pattern_transform_reduce(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1,
+    _ForwardIterator1,
+    _ForwardIterator2,
+    _Tp,
+    _BinaryOperation1,
+    _BinaryOperation2) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+_Tp __pattern_transform_reduce(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator1,
+    _RandomAccessIterator1,
+    _RandomAccessIterator2,
+    _Tp,
+    _BinaryOperation1,
+    _BinaryOperation2);
+
+//------------------------------------------------------------------------
+// transform_reduce (version with unary and binary functions)
+//------------------------------------------------------------------------
+
+template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
+_Tp __brick_transform_reduce(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Tp,
+    _BinaryOperation,
+    _UnaryOperation,
+    /*is_vector=*/std::true_type) noexcept;
+
+template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+_Tp __brick_transform_reduce(
+    _ForwardIterator,
+    _ForwardIterator,
+    _Tp,
+    _BinaryOperation,
+    _UnaryOperation,
+    /*is_vector=*/std::false_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation>
+_Tp __pattern_transform_reduce(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Tp, _BinaryOperation, _UnaryOperation) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation>
+_Tp __pattern_transform_reduce(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _Tp,
+    _BinaryOperation,
+    _UnaryOperation);
+
+//------------------------------------------------------------------------
+// transform_exclusive_scan
+//
+// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
+std::pair<_OutputIterator, _Tp> __brick_transform_scan(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    _UnaryOperation,
+    _Tp,
+    _BinaryOperation,
+    /*Inclusive*/ std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
+std::pair<_OutputIterator, _Tp> __brick_transform_scan(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _UnaryOperation,
+    _Tp,
+    _BinaryOperation,
+    /*Inclusive*/ std::true_type) noexcept;
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+_OutputIterator __pattern_transform_scan(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    _UnaryOperation,
+    _Tp,
+    _BinaryOperation,
+    _Inclusive) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _UnaryOperation,
+    _Tp,
+    _BinaryOperation,
+    _Inclusive);
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _UnaryOperation,
+    _Tp,
+    _BinaryOperation,
+    _Inclusive);
+
+//------------------------------------------------------------------------
+// adjacent_difference
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
+_OutputIterator __brick_adjacent_difference(
+    _ForwardIterator,
+    _ForwardIterator,
+    _OutputIterator,
+    _BinaryOperation,
+    /*is_vector*/ std::false_type) noexcept;
+
+template <class _RandomAccessIterator, class _OutputIterator, class _BinaryOperation>
+_OutputIterator __brick_adjacent_difference(
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _BinaryOperation,
+    /*is_vector*/ std::true_type) noexcept;
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
+_OutputIterator __pattern_adjacent_difference(
+    _Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator, _BinaryOperation) noexcept;
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _BinaryOperation>
+_OutputIterator __pattern_adjacent_difference(
+    __parallel_tag<_IsVector>,
+    _ExecutionPolicy&&,
+    _RandomAccessIterator,
+    _RandomAccessIterator,
+    _OutputIterator,
+    _BinaryOperation);
+
+} // namespace __internal
+} // namespace __pstl
+
+#endif /* _PSTL_NUMERIC_FWD_H */
diff --git a/libcxx/include/__pstl/internal/numeric_impl.h b/libcxx/include/__pstl/internal/numeric_impl.h
new file mode 100644
index 0000000000000..7c852d7b0825d
--- /dev/null
+++ b/libcxx/include/__pstl/internal/numeric_impl.h
@@ -0,0 +1,536 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_NUMERIC_IMPL_H
+#define _PSTL_NUMERIC_IMPL_H
+
+#include <__assert>
+#include <iterator>
+#include <type_traits>
+#include <numeric>
+
+#include "parallel_backend.h"
+#include "pstl_config.h"
+#include "execution_impl.h"
+#include "unseq_backend_simd.h"
+#include "algorithm_fwd.h"
+
+namespace __pstl {
+namespace __internal {
+
+//------------------------------------------------------------------------
+// transform_reduce (version with two binary functions, according to draft N4659)
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
+_Tp __brick_transform_reduce(
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2,
+    /*is_vector=*/std::false_type) noexcept {
+  return std::inner_product(__first1, __last1, __first2, __init, __binary_op1, __binary_op2);
+}
+
+template <class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+_Tp __brick_transform_reduce(
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2,
+    /*is_vector=*/std::true_type) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
+  return __unseq_backend::__simd_transform_reduce(
+      __last1 - __first1, __init, __binary_op1, [=, &__binary_op2](_DifferenceType __i) {
+        return __binary_op2(__first1[__i], __first2[__i]);
+      });
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator1,
+          class _ForwardIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+_Tp __pattern_transform_reduce(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator1 __first1,
+    _ForwardIterator1 __last1,
+    _ForwardIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2) noexcept {
+  return __brick_transform_reduce(
+      __first1, __last1, __first2, __init, __binary_op1, __binary_op2, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _Tp,
+          class _BinaryOperation1,
+          class _BinaryOperation2>
+_Tp __pattern_transform_reduce(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first1,
+    _RandomAccessIterator1 __last1,
+    _RandomAccessIterator2 __first2,
+    _Tp __init,
+    _BinaryOperation1 __binary_op1,
+    _BinaryOperation2 __binary_op2) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    return __par_backend::__parallel_transform_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first1,
+        __last1,
+        [__first1, __first2, __binary_op2](_RandomAccessIterator1 __i) mutable {
+          return __binary_op2(*__i, *(__first2 + (__i - __first1)));
+        },
+        __init,
+        __binary_op1, // Combine
+        [__first1, __first2, __binary_op1, __binary_op2](
+            _RandomAccessIterator1 __i, _RandomAccessIterator1 __j, _Tp __init) -> _Tp {
+          return __internal::__brick_transform_reduce(
+              __i, __j, __first2 + (__i - __first1), __init, __binary_op1, __binary_op2, _IsVector{});
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// transform_reduce (version with unary and binary functions)
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
+_Tp __brick_transform_reduce(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op,
+    /*is_vector=*/std::false_type) noexcept {
+  return std::transform_reduce(__first, __last, __init, __binary_op, __unary_op);
+}
+
+template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
+_Tp __brick_transform_reduce(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op,
+    /*is_vector=*/std::true_type) noexcept {
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
+  return __unseq_backend::__simd_transform_reduce(
+      __last - __first, __init, __binary_op, [=, &__unary_op](_DifferenceType __i) {
+        return __unary_op(__first[__i]);
+      });
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation>
+_Tp __pattern_transform_reduce(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op) noexcept {
+  return __internal::__brick_transform_reduce(
+      __first, __last, __init, __binary_op, __unary_op, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _Tp,
+          class _BinaryOperation,
+          class _UnaryOperation>
+_Tp __pattern_transform_reduce(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _UnaryOperation __unary_op) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  return __internal::__except_handler([&]() {
+    return __par_backend::__parallel_transform_reduce(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __first,
+        __last,
+        [__unary_op](_RandomAccessIterator __i) mutable { return __unary_op(*__i); },
+        __init,
+        __binary_op,
+        [__unary_op, __binary_op](_RandomAccessIterator __i, _RandomAccessIterator __j, _Tp __init) {
+          return __internal::__brick_transform_reduce(__i, __j, __init, __binary_op, __unary_op, _IsVector{});
+        });
+  });
+}
+
+//------------------------------------------------------------------------
+// transform_exclusive_scan
+//
+// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
+//------------------------------------------------------------------------
+
+// Exclusive form
+template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
+std::pair<_OutputIterator, _Tp> __brick_transform_scan(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    /*Inclusive*/ std::false_type,
+    /*is_vector=*/std::false_type) noexcept {
+  for (; __first != __last; ++__first, ++__result) {
+    *__result = __init;
+    __init = __binary_op(__init, __unary_op(*__first));
+  }
+  return std::make_pair(__result, __init);
+}
+
+// Inclusive form
+template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
+std::pair<_OutputIterator, _Tp> __brick_transform_scan(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    /*Inclusive*/ std::true_type,
+    /*is_vector=*/std::false_type) noexcept {
+  for (; __first != __last; ++__first, ++__result) {
+    __init    = __binary_op(__init, __unary_op(*__first));
+    *__result = __init;
+  }
+  return std::make_pair(__result, __init);
+}
+
+// type is arithmetic and binary operation is a user defined operation.
+template <typename _Tp, typename _BinaryOperation>
+using is_arithmetic_udop =
+    std::integral_constant<bool,
+                           std::is_arithmetic<_Tp>::value && !std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
+
+// [restriction] - T shall be DefaultConstructible.
+// [violation] - default ctor of T shall set the identity value for binary_op.
+template <class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+typename std::enable_if<!is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
+__brick_transform_scan(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _Inclusive,
+    /*is_vector=*/std::true_type) noexcept {
+#if defined(_PSTL_UDS_PRESENT)
+  return __unseq_backend::__simd_scan(
+      __first, __last - __first, __result, __unary_op, __init, __binary_op, _Inclusive());
+#else
+  // We need to call serial brick here to call function for inclusive and exclusive scan that depends on _Inclusive()
+  // value
+  return __internal::__brick_transform_scan(
+      __first,
+      __last,
+      __result,
+      __unary_op,
+      __init,
+      __binary_op,
+      _Inclusive(),
+      /*is_vector=*/std::false_type());
+#endif
+}
+
+template <class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+typename std::enable_if<is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
+__brick_transform_scan(
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _Inclusive,
+    /*is_vector=*/std::true_type) noexcept {
+  return __internal::__brick_transform_scan(
+      __first,
+      __last,
+      __result,
+      __unary_op,
+      __init,
+      __binary_op,
+      _Inclusive(),
+      /*is_vector=*/std::false_type());
+}
+
+template <class _Tag,
+          class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+_OutputIterator __pattern_transform_scan(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _Inclusive) noexcept {
+  return __internal::__brick_transform_scan(
+             __first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(), typename _Tag::__is_vector{})
+      .first;
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _Inclusive) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
+
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_transform_scan(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __last - __first,
+        [__first, __unary_op](_DifferenceType __i) mutable { return __unary_op(__first[__i]); },
+        __init,
+        __binary_op,
+        [__first, __unary_op, __binary_op](_DifferenceType __i, _DifferenceType __j, _Tp __init) {
+          // Execute serial __brick_transform_reduce, due to the explicit SIMD vectorization (reduction) requires a
+          // commutative operation for the guarantee of correct scan.
+          return __internal::__brick_transform_reduce(
+              __first + __i,
+              __first + __j,
+              __init,
+              __binary_op,
+              __unary_op,
+              /*__is_vector*/ std::false_type());
+        },
+        [__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __j, _Tp __init) {
+          return __internal::__brick_transform_scan(
+                     __first + __i,
+                     __first + __j,
+                     __result + __i,
+                     __unary_op,
+                     __init,
+                     __binary_op,
+                     _Inclusive(),
+                     _IsVector{})
+              .second;
+        });
+    return __result + (__last - __first);
+  });
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator,
+          class _OutputIterator,
+          class _UnaryOperation,
+          class _Tp,
+          class _BinaryOperation,
+          class _Inclusive>
+typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type __pattern_transform_scan(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator __first,
+    _RandomAccessIterator __last,
+    _OutputIterator __result,
+    _UnaryOperation __unary_op,
+    _Tp __init,
+    _BinaryOperation __binary_op,
+    _Inclusive) {
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
+  _DifferenceType __n = __last - __first;
+
+  if (__n <= 0) {
+    return __result;
+  }
+  return __internal::__except_handler([&]() {
+    __par_backend::__parallel_strict_scan(
+        __backend_tag{},
+        std::forward<_ExecutionPolicy>(__exec),
+        __n,
+        __init,
+        [__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __len) {
+          return __internal::__brick_transform_scan(
+                     __first + __i,
+                     __first + (__i + __len),
+                     __result + __i,
+                     __unary_op,
+                     _Tp{},
+                     __binary_op,
+                     _Inclusive(),
+                     _IsVector{})
+              .second;
+        },
+        __binary_op,
+        [__result, &__binary_op](_DifferenceType __i, _DifferenceType __len, _Tp __initial) {
+          return *(std::transform(__result + __i,
+                                  __result + __i + __len,
+                                  __result + __i,
+                                  [&__initial, &__binary_op](const _Tp& __x) {
+                                    return __binary_op(__initial, __x);
+                                  }) -
+                   1);
+        },
+        [](_Tp) {});
+    return __result + (__last - __first);
+  });
+}
+
+//------------------------------------------------------------------------
+// adjacent_difference
+//------------------------------------------------------------------------
+
+template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
+_OutputIterator __brick_adjacent_difference(
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __d_first,
+    _BinaryOperation __op,
+    /*is_vector*/ std::false_type) noexcept {
+  return std::adjacent_difference(__first, __last, __d_first, __op);
+}
+
+template <class _RandomAccessIterator1, class _RandomAccessIterator2, class BinaryOperation>
+_RandomAccessIterator2 __brick_adjacent_difference(
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __d_first,
+    BinaryOperation __op,
+    /*is_vector=*/std::true_type) noexcept {
+  _LIBCPP_ASSERT(__first != __last, "Range cannot be empty");
+
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
+  typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
+
+  auto __n   = __last - __first;
+  *__d_first = *__first;
+  return __unseq_backend::__simd_walk_3(
+      __first + 1,
+      __n - 1,
+      __first,
+      __d_first + 1,
+      [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__x, __y); });
+}
+
+template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
+_OutputIterator __pattern_adjacent_difference(
+    _Tag,
+    _ExecutionPolicy&&,
+    _ForwardIterator __first,
+    _ForwardIterator __last,
+    _OutputIterator __d_first,
+    _BinaryOperation __op) noexcept {
+  return __internal::__brick_adjacent_difference(__first, __last, __d_first, __op, typename _Tag::__is_vector{});
+}
+
+template <class _IsVector,
+          class _ExecutionPolicy,
+          class _RandomAccessIterator1,
+          class _RandomAccessIterator2,
+          class _BinaryOperation>
+_RandomAccessIterator2 __pattern_adjacent_difference(
+    __parallel_tag<_IsVector> __tag,
+    _ExecutionPolicy&& __exec,
+    _RandomAccessIterator1 __first,
+    _RandomAccessIterator1 __last,
+    _RandomAccessIterator2 __d_first,
+    _BinaryOperation __op) {
+  _LIBCPP_ASSERT(__first != __last, "range cannot be empty");
+  typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
+  typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
+
+  using __backend_tag = typename decltype(__tag)::__backend_tag;
+
+  *__d_first = *__first;
+  __par_backend::__parallel_for(
+      __backend_tag{},
+      std::forward<_ExecutionPolicy>(__exec),
+      __first,
+      __last - 1,
+      [&__op, __d_first, __first](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e) {
+        _RandomAccessIterator2 __d_b = __d_first + (__b - __first);
+        __internal::__brick_walk3(
+            __b,
+            __e,
+            __b + 1,
+            __d_b + 1,
+            [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__y, __x); },
+            _IsVector{});
+      });
+  return __d_first + (__last - __first);
+}
+
+} // namespace __internal
+} // namespace __pstl
+
+#endif /* _PSTL_NUMERIC_IMPL_H */
diff --git a/libcxx/include/pstl/internal/omp/parallel_for.h b/libcxx/include/__pstl/internal/omp/parallel_for.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_for.h
rename to libcxx/include/__pstl/internal/omp/parallel_for.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_for_each.h b/libcxx/include/__pstl/internal/omp/parallel_for_each.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_for_each.h
rename to libcxx/include/__pstl/internal/omp/parallel_for_each.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_invoke.h b/libcxx/include/__pstl/internal/omp/parallel_invoke.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_invoke.h
rename to libcxx/include/__pstl/internal/omp/parallel_invoke.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_merge.h b/libcxx/include/__pstl/internal/omp/parallel_merge.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_merge.h
rename to libcxx/include/__pstl/internal/omp/parallel_merge.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_reduce.h b/libcxx/include/__pstl/internal/omp/parallel_reduce.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_reduce.h
rename to libcxx/include/__pstl/internal/omp/parallel_reduce.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_scan.h b/libcxx/include/__pstl/internal/omp/parallel_scan.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_scan.h
rename to libcxx/include/__pstl/internal/omp/parallel_scan.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_stable_partial_sort.h b/libcxx/include/__pstl/internal/omp/parallel_stable_partial_sort.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_stable_partial_sort.h
rename to libcxx/include/__pstl/internal/omp/parallel_stable_partial_sort.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_stable_sort.h b/libcxx/include/__pstl/internal/omp/parallel_stable_sort.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_stable_sort.h
rename to libcxx/include/__pstl/internal/omp/parallel_stable_sort.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_transform_reduce.h b/libcxx/include/__pstl/internal/omp/parallel_transform_reduce.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_transform_reduce.h
rename to libcxx/include/__pstl/internal/omp/parallel_transform_reduce.h
diff --git a/libcxx/include/pstl/internal/omp/parallel_transform_scan.h b/libcxx/include/__pstl/internal/omp/parallel_transform_scan.h
similarity index 100%
rename from libcxx/include/pstl/internal/omp/parallel_transform_scan.h
rename to libcxx/include/__pstl/internal/omp/parallel_transform_scan.h
diff --git a/libcxx/include/pstl/internal/omp/util.h b/libcxx/include/__pstl/internal/omp/util.h
similarity index 99%
rename from libcxx/include/pstl/internal/omp/util.h
rename to libcxx/include/__pstl/internal/omp/util.h
index c88d9808379cf..3c1a92e03cb95 100644
--- a/libcxx/include/pstl/internal/omp/util.h
+++ b/libcxx/include/__pstl/internal/omp/util.h
@@ -31,8 +31,6 @@
 #    define _PSTL_PRAGMA(x) _Pragma(#    x)
 #endif
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
 namespace __pstl
 {
 namespace __omp_backend
diff --git a/libcxx/include/pstl/internal/parallel_backend.h b/libcxx/include/__pstl/internal/parallel_backend.h
similarity index 93%
rename from libcxx/include/pstl/internal/parallel_backend.h
rename to libcxx/include/__pstl/internal/parallel_backend.h
index 4da871bfe74f2..2890581f5aeb3 100644
--- a/libcxx/include/pstl/internal/parallel_backend.h
+++ b/libcxx/include/__pstl/internal/parallel_backend.h
@@ -17,7 +17,7 @@
 namespace __pstl
 {
 namespace __par_backend = __serial_backend;
-}
+} // namespace __pstl
 #elif defined(_PSTL_PAR_BACKEND_TBB)
 #    include "parallel_backend_tbb.h"
 namespace __pstl
@@ -31,7 +31,7 @@ namespace __pstl
 namespace __par_backend = __omp_backend;
 }
 #else
-_PSTL_PRAGMA_MESSAGE("Parallel backend was not specified");
+#  error "No backend set"
 #endif
 
 #endif /* _PSTL_PARALLEL_BACKEND_H */
diff --git a/libcxx/include/pstl/internal/parallel_backend_omp.h b/libcxx/include/__pstl/internal/parallel_backend_omp.h
similarity index 100%
rename from libcxx/include/pstl/internal/parallel_backend_omp.h
rename to libcxx/include/__pstl/internal/parallel_backend_omp.h
diff --git a/libcxx/include/pstl/internal/parallel_backend_serial.h b/libcxx/include/__pstl/internal/parallel_backend_serial.h
similarity index 87%
rename from libcxx/include/pstl/internal/parallel_backend_serial.h
rename to libcxx/include/__pstl/internal/parallel_backend_serial.h
index ad2c5fc60ec3f..c73a45c5c4833 100644
--- a/libcxx/include/pstl/internal/parallel_backend_serial.h
+++ b/libcxx/include/__pstl/internal/parallel_backend_serial.h
@@ -10,16 +10,12 @@
 #ifndef _PSTL_PARALLEL_BACKEND_SERIAL_H
 #define _PSTL_PARALLEL_BACKEND_SERIAL_H
 
-#include <algorithm>
-#include <cstddef>
-#include <memory>
-#include <numeric>
-#include <utility>
+#include <__memory/allocator.h>
+#include <__pstl/internal/execution_impl.h>
+#include <__utility/forward.h>
 
 #include "pstl_config.h"
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
 namespace __pstl
 {
 namespace __serial_backend
@@ -36,31 +32,32 @@ class __buffer
     operator=(const __buffer&) = delete;
 
   public:
+    _LIBCPP_HIDE_FROM_ABI
     __buffer(std::size_t __n) : __allocator_(), __ptr_(__allocator_.allocate(__n)), __buf_size_(__n) {}
 
-    operator bool() const { return __ptr_ != nullptr; }
-    _Tp*
+    _LIBCPP_HIDE_FROM_ABI operator bool() const { return __ptr_ != nullptr; }
+    _LIBCPP_HIDE_FROM_ABI _Tp*
     get() const
     {
         return __ptr_;
     }
-    ~__buffer() { __allocator_.deallocate(__ptr_, __buf_size_); }
+    _LIBCPP_HIDE_FROM_ABI ~__buffer() { __allocator_.deallocate(__ptr_, __buf_size_); }
 };
 
-inline void
+_LIBCPP_HIDE_FROM_ABI inline void
 __cancel_execution()
 {
 }
 
 template <class _ExecutionPolicy, class _Index, class _Fp>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __parallel_for(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last, _Fp __f)
 {
     __f(__first, __last);
 }
 
 template <class _ExecutionPolicy, class _Value, class _Index, typename _RealBody, typename _Reduction>
-_Value
+_LIBCPP_HIDE_FROM_ABI _Value
 __parallel_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
                   const _Value& __identity, const _RealBody& __real_body, const _Reduction&)
 {
@@ -75,7 +72,7 @@ __parallel_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&,
 }
 
 template <class _ExecutionPolicy, class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
-_Tp
+_LIBCPP_HIDE_FROM_ABI _Tp
 __parallel_transform_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
                             _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce)
 {
@@ -83,7 +80,7 @@ __parallel_transform_reduce(__pstl::__internal::__serial_backend_tag, _Execution
 }
 
 template <class _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp, typename _Ap>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __parallel_strict_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __n, _Tp __initial,
                        _Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
 {
@@ -96,7 +93,7 @@ __parallel_strict_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolic
 }
 
 template <class _ExecutionPolicy, class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce, class _Scan>
-_Tp
+_LIBCPP_HIDE_FROM_ABI _Tp
 __parallel_transform_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __n, _UnaryOp,
                           _Tp __init, _BinaryOp, _Reduce, _Scan __scan)
 {
@@ -104,7 +101,7 @@ __parallel_transform_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPo
 }
 
 template <class _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __parallel_stable_sort(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
                        _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort, std::size_t = 0)
 {
@@ -113,7 +110,7 @@ __parallel_stable_sort(__pstl::__internal::__serial_backend_tag, _ExecutionPolic
 
 template <class _ExecutionPolicy, typename _RandomAccessIterator1, typename _RandomAccessIterator2,
           typename _RandomAccessIterator3, typename _Compare, typename _LeafMerge>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __parallel_merge(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator1 __first1,
                  _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
                  _RandomAccessIterator3 __outit, _Compare __comp, _LeafMerge __leaf_merge)
@@ -122,7 +119,7 @@ __parallel_merge(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _
 }
 
 template <class _ExecutionPolicy, typename _F1, typename _F2>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __parallel_invoke(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _F1&& __f1, _F2&& __f2)
 {
     std::forward<_F1>(__f1)();
@@ -132,6 +129,4 @@ __parallel_invoke(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&,
 } // namespace __serial_backend
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_PARALLEL_BACKEND_SERIAL_H */
diff --git a/libcxx/include/pstl/internal/parallel_backend_tbb.h b/libcxx/include/__pstl/internal/parallel_backend_tbb.h
similarity index 96%
rename from libcxx/include/pstl/internal/parallel_backend_tbb.h
rename to libcxx/include/__pstl/internal/parallel_backend_tbb.h
index e336f69799eb0..3bee7e2764e1b 100644
--- a/libcxx/include/pstl/internal/parallel_backend_tbb.h
+++ b/libcxx/include/__pstl/internal/parallel_backend_tbb.h
@@ -10,6 +10,7 @@
 #ifndef _PSTL_PARALLEL_BACKEND_TBB_H
 #define _PSTL_PARALLEL_BACKEND_TBB_H
 
+#include <__assert>
 #include <algorithm>
 #include <type_traits>
 
@@ -30,8 +31,6 @@
 #    error Intel(R) Threading Building Blocks 2018 is required; older versions are not supported.
 #endif
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
 namespace __pstl
 {
 namespace __tbb_backend
@@ -528,7 +527,7 @@ class __task : public tbb::detail::d1::task
     __task*
     allocate_func_task(_Fn&& __f)
     {
-        _PSTL_ASSERT(_M_execute_data != nullptr);
+        _LIBCPP_ASSERT(_M_execute_data != nullptr, "");
         tbb::detail::d1::small_object_allocator __alloc{};
         auto __t =
             __alloc.new_object<__func_task<typename std::decay<_Fn>::type>>(*_M_execute_data, std::forward<_Fn>(__f));
@@ -573,7 +572,7 @@ class __task : public tbb::detail::d1::task
     make_additional_child_of(__task* __parent, _Fn&& __f)
     {
         auto __t = make_child_of(__parent, std::forward<_Fn>(__f));
-        _PSTL_ASSERT(__parent->_M_refcount.load(std::memory_order_relaxed) > 0);
+        _LIBCPP_ASSERT(__parent->_M_refcount.load(std::memory_order_relaxed) > 0, "");
         ++__parent->_M_refcount;
         return __t;
     }
@@ -594,7 +593,7 @@ class __task : public tbb::detail::d1::task
     inline void
     spawn(__task* __t)
     {
-        _PSTL_ASSERT(_M_execute_data != nullptr);
+        _LIBCPP_ASSERT(_M_execute_data != nullptr, "");
         tbb::detail::d1::spawn(*__t, *_M_execute_data->context);
     }
 
@@ -647,11 +646,11 @@ class __func_task : public __task
 
         this->~__func_task();
 
-        _PSTL_ASSERT(__parent != nullptr);
-        _PSTL_ASSERT(__parent->_M_refcount.load(std::memory_order_relaxed) > 0);
+        _LIBCPP_ASSERT(__parent != nullptr, "");
+        _LIBCPP_ASSERT(__parent->_M_refcount.load(std::memory_order_relaxed) > 0, "");
         if (--__parent->_M_refcount == 0)
         {
-            _PSTL_ASSERT(__next == nullptr);
+            _LIBCPP_ASSERT(__next == nullptr, "");
             __alloc.deallocate(this, *__ed);
             return __parent;
         }
@@ -863,20 +862,20 @@ class __merge_func
     {
         const auto __nx = (_M_xe - _M_xs);
         const auto __ny = (_M_ye - _M_ys);
-        _PSTL_ASSERT(__nx > 0 && __ny > 0);
+        _LIBCPP_ASSERT(__nx > 0 && __ny > 0, "");
 
-        _PSTL_ASSERT(_x_orig == _y_orig);
-        _PSTL_ASSERT(!is_partial());
+        _LIBCPP_ASSERT(_x_orig == _y_orig, "");
+        _LIBCPP_ASSERT(!is_partial(), "");
 
         if (_x_orig)
         {
-            _PSTL_ASSERT(std::is_sorted(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_comp));
-            _PSTL_ASSERT(std::is_sorted(_M_x_beg + _M_ys, _M_x_beg + _M_ye, _M_comp));
+            _LIBCPP_ASSERT(std::is_sorted(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_comp), "");
+            _LIBCPP_ASSERT(std::is_sorted(_M_x_beg + _M_ys, _M_x_beg + _M_ye, _M_comp), "");
             return !_M_comp(*(_M_x_beg + _M_ys), *(_M_x_beg + _M_xe - 1));
         }
 
-        _PSTL_ASSERT(std::is_sorted(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_comp));
-        _PSTL_ASSERT(std::is_sorted(_M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_comp));
+        _LIBCPP_ASSERT(std::is_sorted(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_comp), "");
+        _LIBCPP_ASSERT(std::is_sorted(_M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_comp), "");
         return !_M_comp(*(_M_z_beg + _M_zs + __nx), *(_M_z_beg + _M_zs + __nx - 1));
     }
     void
@@ -884,7 +883,7 @@ class __merge_func
     {
         const auto __nx = (_M_xe - _M_xs);
         const auto __ny = (_M_ye - _M_ys);
-        _PSTL_ASSERT(__nx > 0 && __ny > 0);
+        _LIBCPP_ASSERT(__nx > 0 && __ny > 0, "");
 
         if (_x_orig)
             __move_range_construct()(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_z_beg + _M_zs);
@@ -915,7 +914,7 @@ class __merge_func
     __task*
     merge_ranges(__task* __self)
     {
-        _PSTL_ASSERT(_x_orig == _y_orig); //two merged subrange must be lie into the same buffer
+        _LIBCPP_ASSERT(_x_orig == _y_orig, ""); //two merged subrange must be lie into the same buffer
 
         const auto __nx = (_M_xe - _M_xs);
         const auto __ny = (_M_ye - _M_ys);
@@ -931,15 +930,15 @@ class __merge_func
             _M_leaf_merge(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_x_beg + _M_ys, _M_x_beg + _M_ye, _M_z_beg + _M_zs,
                           _M_comp, __move_value_construct(), __move_value_construct(), __move_range_construct(),
                           __move_range_construct());
-            _PSTL_ASSERT(parent_merge(__self)); //not root merging task
+            _LIBCPP_ASSERT(parent_merge(__self), ""); //not root merging task
         }
         //merge to "origin"
         else
         {
-            _PSTL_ASSERT(_x_orig == _y_orig);
+            _LIBCPP_ASSERT(_x_orig == _y_orig, "");
 
-            _PSTL_ASSERT(is_partial() || std::is_sorted(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_comp));
-            _PSTL_ASSERT(is_partial() || std::is_sorted(_M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_comp));
+            _LIBCPP_ASSERT(is_partial() || std::is_sorted(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_comp), "");
+            _LIBCPP_ASSERT(is_partial() || std::is_sorted(_M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_comp), "");
 
             const auto __nx = (_M_xe - _M_xs);
             const auto __ny = (_M_ye - _M_ys);
@@ -956,8 +955,8 @@ class __merge_func
     __task*
     process_ranges(__task* __self)
     {
-        _PSTL_ASSERT(_x_orig == _y_orig);
-        _PSTL_ASSERT(!_split);
+        _LIBCPP_ASSERT(_x_orig == _y_orig, "");
+        _LIBCPP_ASSERT(!_split, "");
 
         auto p = parent_merge(__self);
 
@@ -1003,7 +1002,7 @@ class __merge_func
     __task*
     split_merging(__task* __self)
     {
-        _PSTL_ASSERT(_x_orig == _y_orig);
+        _LIBCPP_ASSERT(_x_orig == _y_orig, "");
         const auto __nx = (_M_xe - _M_xs);
         const auto __ny = (_M_ye - _M_ys);
 
@@ -1075,8 +1074,8 @@ operator()(__task* __self)
     {
         const _SizeType __nx = (_M_xe - _M_xs);
         const _SizeType __ny = (_M_ye - _M_ys);
-        _PSTL_ASSERT(__nx > 0);
-        _PSTL_ASSERT(__nx > 0);
+        _LIBCPP_ASSERT(__nx > 0, "");
+        _LIBCPP_ASSERT(__nx > 0, "");
 
         if (__nx < __ny)
             move_x_range();
@@ -1132,7 +1131,7 @@ __stable_sort_func<_RandomAccessIterator1, _RandomAccessIterator2, _Compare, _Le
     if (__n <= __sort_cut_off)
     {
         _M_leaf_sort(_M_xs, _M_xe, _M_comp);
-        _PSTL_ASSERT(!_M_root);
+        _LIBCPP_ASSERT(!_M_root, "");
         return nullptr;
     }
 
@@ -1291,6 +1290,4 @@ __parallel_invoke(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _F1
 } // namespace __tbb_backend
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_PARALLEL_BACKEND_TBB_H */
diff --git a/libcxx/include/pstl/internal/parallel_backend_utils.h b/libcxx/include/__pstl/internal/parallel_backend_utils.h
similarity index 98%
rename from libcxx/include/pstl/internal/parallel_backend_utils.h
rename to libcxx/include/__pstl/internal/parallel_backend_utils.h
index e176d7e935184..b27d1ef2b1ce4 100644
--- a/libcxx/include/pstl/internal/parallel_backend_utils.h
+++ b/libcxx/include/__pstl/internal/parallel_backend_utils.h
@@ -10,14 +10,14 @@
 #ifndef _PSTL_PARALLEL_BACKEND_UTILS_H
 #define _PSTL_PARALLEL_BACKEND_UTILS_H
 
-#include <iterator>
-#include <utility>
+#include <__assert>
+#include <__iterator/iterator_traits.h>
+#include <__memory/addressof.h>
+
 #include "utils.h"
 
 #include "pstl_config.h"
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
 namespace __pstl
 {
 
@@ -57,7 +57,7 @@ struct __serial_move_merge
         constexpr bool __same_move_seq = std::is_same<_MoveSequenceX, _MoveSequenceY>::value;
 
         auto __n = _M_nmerge;
-        _PSTL_ASSERT(__n > 0);
+        _LIBCPP_ASSERT(__n > 0, "");
 
         auto __nx = __xe - __xs;
         //auto __ny = __ye - __ys;
@@ -258,6 +258,4 @@ __set_symmetric_difference_construct(_ForwardIterator1 __first1, _ForwardIterato
 } // namespace __utils
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_PARALLEL_BACKEND_UTILS_H */
diff --git a/libcxx/include/pstl/internal/parallel_impl.h b/libcxx/include/__pstl/internal/parallel_impl.h
similarity index 91%
rename from libcxx/include/pstl/internal/parallel_impl.h
rename to libcxx/include/__pstl/internal/parallel_impl.h
index 76b3f43e0d523..961afd75f5129 100644
--- a/libcxx/include/pstl/internal/parallel_impl.h
+++ b/libcxx/include/__pstl/internal/parallel_impl.h
@@ -12,11 +12,8 @@
 
 #include "pstl_config.h"
 
-#include <atomic>
-// This header defines the minimum set of parallel routines required to support Parallel STL,
-// implemented on top of Intel(R) Threading Building Blocks (Intel(R) TBB) library
-
-_PSTL_HIDE_FROM_ABI_PUSH
+#include <__atomic/atomic.h>
+#include <__pstl/internal/parallel_backend.h>
 
 namespace __pstl
 {
@@ -29,7 +26,7 @@ namespace __internal
 /** Return extremum value returned by brick f[i,j) for subranges [i,j) of [first,last)
 Each f[i,j) must return a value in [i,j). */
 template <class _BackendTag, class _ExecutionPolicy, class _Index, class _Brick, class _Compare>
-_Index
+_LIBCPP_HIDE_FROM_ABI _Index
 __parallel_find(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _Index __last, _Brick __f,
                 _Compare __comp, bool __b_first)
 {
@@ -66,9 +63,8 @@ __parallel_find(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _I
 //------------------------------------------------------------------------
 //! Return true if brick f[i,j) returns true for some subrange [i,j) of [first,last)
 template <class _BackendTag, class _ExecutionPolicy, class _Index, class _Brick>
-bool
-__parallel_or(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _Index __last, _Brick __f)
-{
+_LIBCPP_HIDE_FROM_ABI
+bool __parallel_or(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _Index __last, _Brick __f) {
     std::atomic<bool> __found(false);
     __par_backend::__parallel_for(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
                                   [__f, &__found](_Index __i, _Index __j)
@@ -85,6 +81,4 @@ __parallel_or(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _Ind
 } // namespace __internal
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_PARALLEL_IMPL_H */
diff --git a/libcxx/include/__pstl/internal/pstl_config.h b/libcxx/include/__pstl/internal/pstl_config.h
new file mode 100644
index 0000000000000..763d1399764a6
--- /dev/null
+++ b/libcxx/include/__pstl/internal/pstl_config.h
@@ -0,0 +1,55 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_CONFIG_H
+#define _PSTL_CONFIG_H
+
+#include <__config>
+
+// TODO: Make this a proper configuration option
+#define _PSTL_PAR_BACKEND_SERIAL
+
+#define _PSTL_PRAGMA(x) _Pragma(#    x)
+
+// Enable SIMD for compilers that support OpenMP 4.0
+#if (defined(_OPENMP) && _OPENMP >= 201307)
+
+#  define _PSTL_UDR_PRESENT
+#  define _PSTL_PRAGMA_SIMD _PSTL_PRAGMA(omp simd)
+#  define _PSTL_PRAGMA_DECLARE_SIMD _PSTL_PRAGMA(omp declare simd)
+#  define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _PSTL_PRAGMA(omp simd reduction(PRM))
+#  define _PSTL_PRAGMA_SIMD_SCAN(PRM) _PSTL_PRAGMA(omp simd reduction(inscan, PRM))
+#  define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan inclusive(PRM))
+#  define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan exclusive(PRM))
+
+// Declaration of reduction functor, where
+// NAME - the name of the functor
+// OP - type of the callable object with the reduction operation
+// omp_in - refers to the local partial result
+// omp_out - refers to the final value of the combiner operator
+// omp_priv - refers to the private copy of the initial value
+// omp_orig - refers to the original variable to be reduced
+#  define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)                                                                     \
+    _PSTL_PRAGMA(omp declare reduction(NAME:OP : omp_out(omp_in)) initializer(omp_priv = omp_orig))
+
+#else // (defined(_OPENMP) && _OPENMP >= 201307)
+
+#  define _PSTL_PRAGMA_SIMD
+#  define _PSTL_PRAGMA_DECLARE_SIMD
+#  define _PSTL_PRAGMA_SIMD_REDUCTION(PRM)
+#  define _PSTL_PRAGMA_SIMD_SCAN(PRM)
+#  define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM)
+#  define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM)
+#  define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)
+
+#endif // (defined(_OPENMP) && _OPENMP >= 201307)
+
+#define _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
+
+#endif /* _PSTL_CONFIG_H */
diff --git a/libcxx/include/pstl/internal/unseq_backend_simd.h b/libcxx/include/__pstl/internal/unseq_backend_simd.h
similarity index 77%
rename from libcxx/include/pstl/internal/unseq_backend_simd.h
rename to libcxx/include/__pstl/internal/unseq_backend_simd.h
index af2a143bc58ab..5a2f18c7c191e 100644
--- a/libcxx/include/pstl/internal/unseq_backend_simd.h
+++ b/libcxx/include/__pstl/internal/unseq_backend_simd.h
@@ -10,7 +10,8 @@
 #ifndef _PSTL_UNSEQ_BACKEND_SIMD_H
 #define _PSTL_UNSEQ_BACKEND_SIMD_H
 
-#include <type_traits>
+#include <__functional/operations.h>
+#include <__utility/pair.h>
 
 #include "pstl_config.h"
 #include "utils.h"
@@ -18,8 +19,6 @@
 // This header defines the minimum set of vector routines required
 // to support parallel STL.
 
-_PSTL_HIDE_FROM_ABI_PUSH
-
 namespace __pstl
 {
 namespace __unseq_backend
@@ -29,7 +28,7 @@ namespace __unseq_backend
 const std::size_t __lane_size = 64;
 
 template <class _Iterator, class _DifferenceType, class _Function>
-_Iterator
+_LIBCPP_HIDE_FROM_ABI _Iterator
 __simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept
 {
     _PSTL_PRAGMA_SIMD
@@ -40,7 +39,7 @@ __simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept
 }
 
 template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Function>
-_Iterator2
+_LIBCPP_HIDE_FROM_ABI _Iterator2
 __simd_walk_2(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept
 {
     _PSTL_PRAGMA_SIMD
@@ -50,7 +49,7 @@ __simd_walk_2(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Fu
 }
 
 template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Iterator3, class _Function>
-_Iterator3
+_LIBCPP_HIDE_FROM_ABI _Iterator3
 __simd_walk_3(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Iterator3 __first3,
               _Function __f) noexcept
 {
@@ -62,18 +61,9 @@ __simd_walk_3(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _It
 
 // TODO: check whether __simd_first() can be used here
 template <class _Index, class _DifferenceType, class _Pred>
-bool
+_LIBCPP_HIDE_FROM_ABI bool
 __simd_or(_Index __first, _DifferenceType __n, _Pred __pred) noexcept
 {
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    _DifferenceType __i;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED
-    _PSTL_PRAGMA_SIMD_EARLYEXIT
-    for (__i = 0; __i < __n; ++__i)
-        if (__pred(__first[__i]))
-            break;
-    return __i < __n;
-#else
     _DifferenceType __block_size = 4 < __n ? 4 : __n;
     const _Index __last = __first + __n;
     while (__last != __first)
@@ -98,32 +88,18 @@ __simd_or(_Index __first, _DifferenceType __n, _Pred __pred) noexcept
         }
     }
     return false;
-#endif
 }
 
 template <class _Index, class _DifferenceType, class _Compare>
-_Index
+_LIBCPP_HIDE_FROM_ABI _Index
 __simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp) noexcept
 {
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    _DifferenceType __i = __begin;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
-        _PSTL_PRAGMA_SIMD_EARLYEXIT for (; __i < __end; ++__i)
-    {
-        if (__comp(__first, __i))
-        {
-            break;
-        }
-    }
-    return __first + __i;
-#else
     // Experiments show good block sizes like this
     const _DifferenceType __block_size = 8;
     alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
     while (__end - __begin >= __block_size)
     {
         _DifferenceType __found = 0;
-        _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
             _PSTL_PRAGMA_SIMD_REDUCTION(|
                                         : __found) for (_DifferenceType __i = __begin; __i < __begin + __block_size;
                                                         ++__i)
@@ -158,22 +134,12 @@ __simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Co
         ++__begin;
     }
     return __first + __end;
-#endif //_PSTL_EARLYEXIT_PRESENT
 }
 
 template <class _Index1, class _DifferenceType, class _Index2, class _Pred>
-std::pair<_Index1, _Index2>
+_LIBCPP_HIDE_FROM_ABI std::pair<_Index1, _Index2>
 __simd_first(_Index1 __first1, _DifferenceType __n, _Index2 __first2, _Pred __pred) noexcept
 {
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    _DifferenceType __i = 0;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED
-    _PSTL_PRAGMA_SIMD_EARLYEXIT
-    for (; __i < __n; ++__i)
-        if (__pred(__first1[__i], __first2[__i]))
-            break;
-    return std::make_pair(__first1 + __i, __first2 + __i);
-#else
     const _Index1 __last1 = __first1 + __n;
     const _Index2 __last2 = __first2 + __n;
     // Experiments show good block sizes like this
@@ -183,7 +149,6 @@ __simd_first(_Index1 __first1, _DifferenceType __n, _Index2 __first2, _Pred __pr
     {
         _DifferenceType __found = 0;
         _DifferenceType __i;
-        _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
             _PSTL_PRAGMA_SIMD_REDUCTION(|
                                         : __found) for (__i = 0; __i < __block_size; ++__i)
         {
@@ -212,11 +177,10 @@ __simd_first(_Index1 __first1, _DifferenceType __n, _Index2 __first2, _Pred __pr
             return std::make_pair(__first1, __first2);
 
     return std::make_pair(__last1, __last2);
-#endif //_PSTL_EARLYEXIT_PRESENT
 }
 
 template <class _Index, class _DifferenceType, class _Pred>
-_DifferenceType
+_LIBCPP_HIDE_FROM_ABI _DifferenceType
 __simd_count(_Index __index, _DifferenceType __n, _Pred __pred) noexcept
 {
     _DifferenceType __count = 0;
@@ -229,7 +193,7 @@ __simd_count(_Index __index, _DifferenceType __n, _Pred __pred) noexcept
 }
 
 template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator
+_LIBCPP_HIDE_FROM_ABI _OutputIterator
 __simd_unique_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator __result,
                    _BinaryPredicate __pred) noexcept
 {
@@ -242,7 +206,6 @@ __simd_unique_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator
     _PSTL_PRAGMA_SIMD
     for (_DifferenceType __i = 1; __i < __n; ++__i)
     {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
         if (!__pred(__first[__i], __first[__i - 1]))
         {
             __result[__cnt] = __first[__i];
@@ -253,7 +216,7 @@ __simd_unique_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator
 }
 
 template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _Assigner>
-_OutputIterator
+_LIBCPP_HIDE_FROM_ABI _OutputIterator
 __simd_assign(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, _Assigner __assigner) noexcept
 {
     _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
@@ -264,7 +227,7 @@ __simd_assign(_InputIterator __first, _DifferenceType __n, _OutputIterator __res
 }
 
 template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
+_LIBCPP_HIDE_FROM_ABI _OutputIterator
 __simd_copy_if(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, _UnaryPredicate __pred) noexcept
 {
     _DifferenceType __cnt = 0;
@@ -272,7 +235,6 @@ __simd_copy_if(_InputIterator __first, _DifferenceType __n, _OutputIterator __re
     _PSTL_PRAGMA_SIMD
     for (_DifferenceType __i = 0; __i < __n; ++__i)
     {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
         if (__pred(__first[__i]))
         {
             __result[__cnt] = __first[__i];
@@ -283,7 +245,7 @@ __simd_copy_if(_InputIterator __first, _DifferenceType __n, _OutputIterator __re
 }
 
 template <class _InputIterator, class _DifferenceType, class _BinaryPredicate>
-_DifferenceType
+_LIBCPP_HIDE_FROM_ABI _DifferenceType
 __simd_calc_mask_2(_InputIterator __first, _DifferenceType __n, bool* __mask, _BinaryPredicate __pred) noexcept
 {
     _DifferenceType __count = 0;
@@ -298,7 +260,7 @@ __simd_calc_mask_2(_InputIterator __first, _DifferenceType __n, bool* __mask, _B
 }
 
 template <class _InputIterator, class _DifferenceType, class _UnaryPredicate>
-_DifferenceType
+_LIBCPP_HIDE_FROM_ABI _DifferenceType
 __simd_calc_mask_1(_InputIterator __first, _DifferenceType __n, bool* __mask, _UnaryPredicate __pred) noexcept
 {
     _DifferenceType __count = 0;
@@ -313,7 +275,7 @@ __simd_calc_mask_1(_InputIterator __first, _DifferenceType __n, bool* __mask, _U
 }
 
 template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _Assigner>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __simd_copy_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, bool* __mask,
                     _Assigner __assigner) noexcept
 {
@@ -323,7 +285,6 @@ __simd_copy_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator
     {
         if (__mask[__i])
         {
-            _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
             {
                 __assigner(__first + __i, __result + __cnt);
                 ++__cnt;
@@ -333,7 +294,7 @@ __simd_copy_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator
 }
 
 template <class _InputIterator, class _DifferenceType, class _OutputIterator1, class _OutputIterator2>
-void
+_LIBCPP_HIDE_FROM_ABI void
 __simd_partition_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator1 __out_true,
                          _OutputIterator2 __out_false, bool* __mask) noexcept
 {
@@ -341,7 +302,6 @@ __simd_partition_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIte
     _PSTL_PRAGMA_SIMD
     for (_DifferenceType __i = 0; __i < __n; ++__i)
     {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(__cnt_true : 1, __cnt_false : 1)
         if (__mask[__i])
         {
             __out_true[__cnt_true] = __first[__i];
@@ -356,7 +316,7 @@ __simd_partition_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIte
 }
 
 template <class _Index, class _DifferenceType, class _Tp>
-_Index
+_LIBCPP_HIDE_FROM_ABI _Index
 __simd_fill_n(_Index __first, _DifferenceType __n, const _Tp& __value) noexcept
 {
     _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
@@ -367,7 +327,7 @@ __simd_fill_n(_Index __first, _DifferenceType __n, const _Tp& __value) noexcept
 }
 
 template <class _Index, class _DifferenceType, class _Generator>
-_Index
+_LIBCPP_HIDE_FROM_ABI _Index
 __simd_generate_n(_Index __first, _DifferenceType __size, _Generator __g) noexcept
 {
     _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
@@ -378,7 +338,7 @@ __simd_generate_n(_Index __first, _DifferenceType __size, _Generator __g) noexce
 }
 
 template <class _Index, class _BinaryPredicate>
-_Index
+_LIBCPP_HIDE_FROM_ABI _Index
 __simd_adjacent_find(_Index __first, _Index __last, _BinaryPredicate __pred, bool __or_semantic) noexcept
 {
     if (__last - __first < 2)
@@ -387,17 +347,6 @@ __simd_adjacent_find(_Index __first, _Index __last, _BinaryPredicate __pred, boo
     typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
     _DifferenceType __i = 0;
 
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    //Some compiler versions fail to compile the following loop when iterators are used. Indices are used instead
-    const _DifferenceType __n = __last - __first - 1;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED
-    _PSTL_PRAGMA_SIMD_EARLYEXIT
-    for (; __i < __n; ++__i)
-        if (__pred(__first[__i], __first[__i + 1]))
-            break;
-
-    return __i < __n ? __first + __i : __last;
-#else
     // Experiments show good block sizes like this
     //TODO: to consider tuning block_size for various data types
     const _DifferenceType __block_size = 8;
@@ -405,7 +354,6 @@ __simd_adjacent_find(_Index __first, _Index __last, _BinaryPredicate __pred, boo
     while (__last - __first >= __block_size)
     {
         _DifferenceType __found = 0;
-        _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
             _PSTL_PRAGMA_SIMD_REDUCTION(|
                                         : __found) for (__i = 0; __i < __block_size - 1; ++__i)
         {
@@ -438,7 +386,6 @@ __simd_adjacent_find(_Index __first, _Index __last, _BinaryPredicate __pred, boo
             return __first;
 
     return __last;
-#endif
 }
 
 // It was created to reduce the code inside std::enable_if
@@ -447,7 +394,7 @@ using is_arithmetic_plus = std::integral_constant<bool, std::is_arithmetic<_Tp>:
                                                             std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
 
 template <typename _DifferenceType, typename _Tp, typename _BinaryOperation, typename _UnaryOperation>
-typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
+_LIBCPP_HIDE_FROM_ABI typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
 __simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _UnaryOperation __f) noexcept
 {
     _PSTL_PRAGMA_SIMD_REDUCTION(+ : __init)
@@ -457,14 +404,14 @@ __simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _Unar
 }
 
 template <typename _Size, typename _Tp, typename _BinaryOperation, typename _UnaryOperation>
-typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
+_LIBCPP_HIDE_FROM_ABI typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
 __simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _UnaryOperation __f) noexcept
 {
     const _Size __block_size = __lane_size / sizeof(_Tp);
     if (__n > 2 * __block_size && __block_size > 1)
     {
-        alignas(__lane_size) char __lane_[__lane_size];
-        _Tp* __lane = reinterpret_cast<_Tp*>(__lane_);
+        alignas(__lane_size) char __lane_buffer[__lane_size];
+        _Tp* __lane = reinterpret_cast<_Tp*>(__lane_buffer);
 
         // initializer
         _PSTL_PRAGMA_SIMD
@@ -474,8 +421,8 @@ __simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _Un
         }
         // main loop
         _Size __i = 2 * __block_size;
-        const _Size last_iteration = __block_size * (__n / __block_size);
-        for (; __i < last_iteration; __i += __block_size)
+        const _Size __last_iteration = __block_size * (__n / __block_size);
+        for (; __i < __last_iteration; __i += __block_size)
         {
             _PSTL_PRAGMA_SIMD
             for (_Size __j = 0; __j < __block_size; ++__j)
@@ -485,9 +432,9 @@ __simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _Un
         }
         // remainder
         _PSTL_PRAGMA_SIMD
-        for (_Size __j = 0; __j < __n - last_iteration; ++__j)
+        for (_Size __j = 0; __j < __n - __last_iteration; ++__j)
         {
-            __lane[__j] = __binary_op(__lane[__j], __f(last_iteration + __j));
+            __lane[__j] = __binary_op(__lane[__j], __f(__last_iteration + __j));
         }
         // combiner
         for (_Size __j = 0; __j < __block_size; ++__j)
@@ -514,6 +461,7 @@ __simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _Un
 // Exclusive scan for "+" and arithmetic types
 template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
           class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI
 typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
 __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
             _BinaryOperation, /*Inclusive*/ std::false_type)
@@ -532,46 +480,48 @@ __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryO
 template <typename _Tp, typename _BinaryOp>
 struct _Combiner
 {
-    _Tp __value;
-    _BinaryOp* __bin_op; // Here is a pointer to function because of default ctor
+    _Tp __value_;
+    _BinaryOp* __bin_op_; // Here is a pointer to function because of default ctor
 
-    _Combiner() : __value{}, __bin_op(nullptr) {}
-    _Combiner(const _Tp& value, const _BinaryOp* bin_op) : __value(value), __bin_op(const_cast<_BinaryOp*>(bin_op)) {}
-    _Combiner(const _Combiner& __obj) : __value{}, __bin_op(__obj.__bin_op) {}
+    _LIBCPP_HIDE_FROM_ABI _Combiner() : __value_{}, __bin_op_(nullptr) {}
+    _LIBCPP_HIDE_FROM_ABI
+    _Combiner(const _Tp& __value, const _BinaryOp* __bin_op)
+        : __value_(__value), __bin_op_(const_cast<_BinaryOp*>(__bin_op)) {}
+    _LIBCPP_HIDE_FROM_ABI _Combiner(const _Combiner& __obj) : __value_{}, __bin_op_(__obj.__bin_op) {}
 
-    void
+    _LIBCPP_HIDE_FROM_ABI void
     operator()(const _Combiner& __obj)
     {
-        __value = (*__bin_op)(__value, __obj.__value);
+        __value_ = (*__bin_op_)(__value_, __obj.__value_);
     }
 };
 
 // Exclusive scan for other binary operations and types
 template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
           class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI
 typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
 __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
             _BinaryOperation __binary_op, /*Inclusive*/ std::false_type)
 {
     typedef _Combiner<_Tp, _BinaryOperation> _CombinerType;
-    _CombinerType __init_{__init, &__binary_op};
+    _CombinerType __combined_init{__init, &__binary_op};
 
     _PSTL_PRAGMA_DECLARE_REDUCTION(__bin_op, _CombinerType)
-
     _PSTL_PRAGMA_SIMD_SCAN(__bin_op : __init_)
     for (_Size __i = 0; __i < __n; ++__i)
     {
-        __result[__i] = __init_.__value;
+        __result[__i] = __combined_init.__value_;
         _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(__init_)
-        _PSTL_PRAGMA_FORCEINLINE
-        __init_.__value = __binary_op(__init_.__value, __unary_op(__first[__i]));
+        __combined_init.__value_ = __binary_op(__combined_init.__value_, __unary_op(__first[__i]));
     }
-    return std::make_pair(__result + __n, __init_.__value);
+    return std::make_pair(__result + __n, __combined_init.__value_);
 }
 
 // Inclusive scan for "+" and arithmetic types
 template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
           class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI
 typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
 __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
             _BinaryOperation, /*Inclusive*/ std::true_type)
@@ -589,30 +539,29 @@ __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryO
 // Inclusive scan for other binary operations and types
 template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
           class _BinaryOperation>
+_LIBCPP_HIDE_FROM_ABI
 typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
 __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
             _BinaryOperation __binary_op, std::true_type)
 {
     typedef _Combiner<_Tp, _BinaryOperation> _CombinerType;
-    _CombinerType __init_{__init, &__binary_op};
+    _CombinerType __combined_init{__init, &__binary_op};
 
     _PSTL_PRAGMA_DECLARE_REDUCTION(__bin_op, _CombinerType)
-
     _PSTL_PRAGMA_SIMD_SCAN(__bin_op : __init_)
     for (_Size __i = 0; __i < __n; ++__i)
     {
-        _PSTL_PRAGMA_FORCEINLINE
-        __init_.__value = __binary_op(__init_.__value, __unary_op(__first[__i]));
+        __combined_init.__value_ = __binary_op(__combined_init.__value_, __unary_op(__first[__i]));
         _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(__init_)
-        __result[__i] = __init_.__value;
+        __result[__i] = __combined_init.__value_;
     }
-    return std::make_pair(__result + __n, __init_.__value);
+    return std::make_pair(__result + __n, __combined_init.__value_);
 }
 
 // [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
 // complexity [violation] - We will have at most (__n-1 + number_of_lanes) comparisons instead of at most __n-1.
 template <typename _ForwardIterator, typename _Size, typename _Compare>
-_ForwardIterator
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
 __simd_min_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcept
 {
     if (__n == 0)
@@ -623,29 +572,29 @@ __simd_min_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcep
     typedef typename std::iterator_traits<_ForwardIterator>::value_type _ValueType;
     struct _ComplexType
     {
-        _ValueType __min_val;
-        _Size __min_ind;
-        _Compare* __min_comp;
+        _ValueType __min_val_;
+        _Size __min_ind_;
+        _Compare* __min_comp_;
 
-        _ComplexType() : __min_val{}, __min_ind{}, __min_comp(nullptr) {}
-        _ComplexType(const _ValueType& val, const _Compare* comp)
-            : __min_val(val), __min_ind(0), __min_comp(const_cast<_Compare*>(comp))
+        _LIBCPP_HIDE_FROM_ABI _ComplexType() : __min_val_{}, __min_ind_{}, __min_comp_(nullptr) {}
+        _LIBCPP_HIDE_FROM_ABI _ComplexType(const _ValueType& __val, const _Compare* __comp)
+            : __min_val_(__val), __min_ind_(0), __min_comp_(const_cast<_Compare*>(__comp))
         {
         }
-        _ComplexType(const _ComplexType& __obj)
-            : __min_val(__obj.__min_val), __min_ind(__obj.__min_ind), __min_comp(__obj.__min_comp)
+        _LIBCPP_HIDE_FROM_ABI _ComplexType(const _ComplexType& __obj)
+            : __min_val_(__obj.__min_val_), __min_ind_(__obj.__min_ind_), __min_comp_(__obj.__min_comp_)
         {
         }
 
         _PSTL_PRAGMA_DECLARE_SIMD
-        void
+        _LIBCPP_HIDE_FROM_ABI void
         operator()(const _ComplexType& __obj)
         {
-            if (!(*__min_comp)(__min_val, __obj.__min_val) &&
-                ((*__min_comp)(__obj.__min_val, __min_val) || __obj.__min_ind - __min_ind < 0))
+            if (!(*__min_comp_)(__min_val_, __obj.__min_val_) &&
+                ((*__min_comp_)(__obj.__min_val_, __min_val_) || __obj.__min_ind_ - __min_ind_ < 0))
             {
-                __min_val = __obj.__min_val;
-                __min_ind = __obj.__min_ind;
+                __min_val_ = __obj.__min_val_;
+                __min_ind_ = __obj.__min_ind_;
             }
         }
     };
@@ -657,21 +606,21 @@ __simd_min_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcep
     _PSTL_PRAGMA_SIMD_REDUCTION(__min_func : __init)
     for (_Size __i = 1; __i < __n; ++__i)
     {
-        const _ValueType __min_val = __init.__min_val;
+        const _ValueType __min_val = __init.__min_val_;
         const _ValueType __current = __first[__i];
         if (__comp(__current, __min_val))
         {
-            __init.__min_val = __current;
-            __init.__min_ind = __i;
+            __init.__min_val_ = __current;
+            __init.__min_ind_ = __i;
         }
     }
-    return __first + __init.__min_ind;
+    return __first + __init.__min_ind_;
 }
 
 // [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
 // complexity [violation] - We will have at most (2*(__n-1) + 4*number_of_lanes) comparisons instead of at most [1.5*(__n-1)].
 template <typename _ForwardIterator, typename _Size, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
+_LIBCPP_HIDE_FROM_ABI std::pair<_ForwardIterator, _ForwardIterator>
 __simd_minmax_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcept
 {
     if (__n == 0)
@@ -682,49 +631,51 @@ __simd_minmax_element(_ForwardIterator __first, _Size __n, _Compare __comp) noex
 
     struct _ComplexType
     {
-        _ValueType __min_val;
-        _ValueType __max_val;
-        _Size __min_ind;
-        _Size __max_ind;
+        _ValueType __min_val_;
+        _ValueType __max_val_;
+        _Size __min_ind_;
+        _Size __max_ind_;
         _Compare* __minmax_comp;
 
-        _ComplexType() : __min_val{}, __max_val{}, __min_ind{}, __max_ind{}, __minmax_comp(nullptr) {}
-        _ComplexType(const _ValueType& min_val, const _ValueType& max_val, const _Compare* comp)
-            : __min_val(min_val), __max_val(max_val), __min_ind(0), __max_ind(0),
-              __minmax_comp(const_cast<_Compare*>(comp))
+        _LIBCPP_HIDE_FROM_ABI _ComplexType()
+            : __min_val_{}, __max_val_{}, __min_ind_{}, __max_ind_{}, __minmax_comp(nullptr) {}
+        _LIBCPP_HIDE_FROM_ABI _ComplexType(
+                const _ValueType& __min_val, const _ValueType& __max_val, const _Compare* __comp)
+            : __min_val_(__min_val), __max_val_(__max_val), __min_ind_(0), __max_ind_(0),
+              __minmax_comp(const_cast<_Compare*>(__comp))
         {
         }
-        _ComplexType(const _ComplexType& __obj)
-            : __min_val(__obj.__min_val), __max_val(__obj.__max_val), __min_ind(__obj.__min_ind),
-              __max_ind(__obj.__max_ind), __minmax_comp(__obj.__minmax_comp)
+        _LIBCPP_HIDE_FROM_ABI _ComplexType(const _ComplexType& __obj)
+            : __min_val_(__obj.__min_val_), __max_val_(__obj.__max_val_), __min_ind_(__obj.__min_ind_),
+              __max_ind_(__obj.__max_ind_), __minmax_comp(__obj.__minmax_comp)
         {
         }
 
-        void
+        _LIBCPP_HIDE_FROM_ABI void
         operator()(const _ComplexType& __obj)
         {
             // min
-            if ((*__minmax_comp)(__obj.__min_val, __min_val))
+            if ((*__minmax_comp)(__obj.__min_val_, __min_val_))
             {
-                __min_val = __obj.__min_val;
-                __min_ind = __obj.__min_ind;
+                __min_val_ = __obj.__min_val_;
+                __min_ind_ = __obj.__min_ind_;
             }
-            else if (!(*__minmax_comp)(__min_val, __obj.__min_val))
+            else if (!(*__minmax_comp)(__min_val_, __obj.__min_val_))
             {
-                __min_val = __obj.__min_val;
-                __min_ind = (__min_ind - __obj.__min_ind < 0) ? __min_ind : __obj.__min_ind;
+                __min_val_ = __obj.__min_val_;
+                __min_ind_ = (__min_ind_ - __obj.__min_ind_ < 0) ? __min_ind_ : __obj.__min_ind_;
             }
 
             // max
-            if ((*__minmax_comp)(__max_val, __obj.__max_val))
+            if ((*__minmax_comp)(__max_val_, __obj.__max_val_))
             {
-                __max_val = __obj.__max_val;
-                __max_ind = __obj.__max_ind;
+                __max_val_ = __obj.__max_val_;
+                __max_ind_ = __obj.__max_ind_;
             }
-            else if (!(*__minmax_comp)(__obj.__max_val, __max_val))
+            else if (!(*__minmax_comp)(__obj.__max_val_, __max_val_))
             {
-                __max_val = __obj.__max_val;
-                __max_ind = (__max_ind - __obj.__max_ind < 0) ? __obj.__max_ind : __max_ind;
+                __max_val_ = __obj.__max_val_;
+                __max_ind_ = (__max_ind_ - __obj.__max_ind_ < 0) ? __obj.__max_ind_ : __max_ind_;
             }
         }
     };
@@ -736,26 +687,26 @@ __simd_minmax_element(_ForwardIterator __first, _Size __n, _Compare __comp) noex
     _PSTL_PRAGMA_SIMD_REDUCTION(__min_func : __init)
     for (_Size __i = 1; __i < __n; ++__i)
     {
-        auto __min_val = __init.__min_val;
-        auto __max_val = __init.__max_val;
+        auto __min_val = __init.__min_val_;
+        auto __max_val = __init.__max_val_;
         auto __current = __first + __i;
         if (__comp(*__current, __min_val))
         {
-            __init.__min_val = *__current;
-            __init.__min_ind = __i;
+            __init.__min_val_ = *__current;
+            __init.__min_ind_ = __i;
         }
         else if (!__comp(*__current, __max_val))
         {
-            __init.__max_val = *__current;
-            __init.__max_ind = __i;
+            __init.__max_val_ = *__current;
+            __init.__max_ind_ = __i;
         }
     }
-    return std::make_pair(__first + __init.__min_ind, __first + __init.__max_ind);
+    return std::make_pair(__first + __init.__min_ind_, __first + __init.__max_ind_);
 }
 
 template <class _InputIterator, class _DifferenceType, class _OutputIterator1, class _OutputIterator2,
           class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
+_LIBCPP_HIDE_FROM_ABI std::pair<_OutputIterator1, _OutputIterator2>
 __simd_partition_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator1 __out_true,
                       _OutputIterator2 __out_false, _UnaryPredicate __pred) noexcept
 {
@@ -764,7 +715,6 @@ __simd_partition_copy(_InputIterator __first, _DifferenceType __n, _OutputIterat
     _PSTL_PRAGMA_SIMD
     for (_DifferenceType __i = 0; __i < __n; ++__i)
     {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(__cnt_true : 1, __cnt_false : 1)
         if (__pred(__first[__i]))
         {
             __out_true[__cnt_true] = __first[__i];
@@ -780,7 +730,7 @@ __simd_partition_copy(_InputIterator __first, _DifferenceType __n, _OutputIterat
 }
 
 template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator1
 __simd_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
                      _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
 {
@@ -826,7 +776,7 @@ __simd_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _Forwa
 }
 
 template <class _RandomAccessIterator, class _DifferenceType, class _UnaryPredicate>
-_RandomAccessIterator
+_LIBCPP_HIDE_FROM_ABI _RandomAccessIterator
 __simd_remove_if(_RandomAccessIterator __first, _DifferenceType __n, _UnaryPredicate __pred) noexcept
 {
     // find first element we need to remove
@@ -845,7 +795,6 @@ __simd_remove_if(_RandomAccessIterator __first, _DifferenceType __n, _UnaryPredi
     _PSTL_PRAGMA_SIMD
     for (_DifferenceType __i = 1; __i < __n; ++__i)
     {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
         if (!__pred(__current[__i]))
         {
             __current[__cnt] = std::move(__current[__i]);
@@ -857,6 +806,4 @@ __simd_remove_if(_RandomAccessIterator __first, _DifferenceType __n, _UnaryPredi
 } // namespace __unseq_backend
 } // namespace __pstl
 
-_PSTL_HIDE_FROM_ABI_POP
-
 #endif /* _PSTL_UNSEQ_BACKEND_SIMD_H */
diff --git a/libcxx/include/__pstl/internal/utils.h b/libcxx/include/__pstl/internal/utils.h
new file mode 100644
index 0000000000000..73da2cb0ca9cb
--- /dev/null
+++ b/libcxx/include/__pstl/internal/utils.h
@@ -0,0 +1,140 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _PSTL_UTILS_H
+#define _PSTL_UTILS_H
+
+#include <__exception/terminate.h>
+#include <__pstl/internal/pstl_config.h>
+#include <__utility/forward.h>
+#include <new>
+
+namespace __pstl {
+namespace __internal {
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI auto __except_handler(_Fp __f) -> decltype(__f()) {
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    return __f();
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (const std::bad_alloc&) {
+    throw;            // re-throw bad_alloc according to the standard [algorithms.parallel.exceptions]
+  } catch (...) {
+    std::terminate(); // Good bye according to the standard [algorithms.parallel.exceptions]
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+}
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI void __invoke_if(std::true_type, _Fp __f) {
+  __f();
+}
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI void __invoke_if(std::false_type, _Fp) {}
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI void __invoke_if_not(std::false_type, _Fp __f) {
+  __f();
+}
+
+template <typename _Fp>
+_LIBCPP_HIDE_FROM_ABI void __invoke_if_not(std::true_type, _Fp) {}
+
+template <typename _F1, typename _F2>
+_LIBCPP_HIDE_FROM_ABI auto __invoke_if_else(std::true_type, _F1 __f1, _F2) -> decltype(__f1()) {
+  return __f1();
+}
+
+template <typename _F1, typename _F2>
+_LIBCPP_HIDE_FROM_ABI auto __invoke_if_else(std::false_type, _F1, _F2 __f2) -> decltype(__f2()) {
+  return __f2();
+}
+
+//! Unary operator that returns reference to its argument.
+struct __no_op {
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI _Tp&& operator()(_Tp&& __a) const {
+    return std::forward<_Tp>(__a);
+  }
+};
+
+template <typename _Pred>
+class __reorder_pred {
+  _Pred __pred_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __reorder_pred(_Pred __pred) : __pred_(__pred) {}
+
+  template <typename _FTp, typename _STp>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(_FTp&& __a, _STp&& __b) {
+    return __pred_(std::forward<_STp>(__b), std::forward<_FTp>(__a));
+  }
+};
+
+//! Like a polymorphic lambda for pred(...,value)
+template <typename _Tp, typename _Predicate>
+class __equal_value_by_pred {
+  const _Tp& __value_;
+  _Predicate __pred_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI __equal_value_by_pred(const _Tp& __value, _Predicate __pred)
+      : __value_(__value), __pred_(__pred) {}
+
+  template <typename _Arg>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(_Arg&& __arg) {
+    return __pred_(std::forward<_Arg>(__arg), __value_);
+  }
+};
+
+//! Like a polymorphic lambda for ==value
+template <typename _Tp>
+class __equal_value {
+  const _Tp& __value_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __equal_value(const _Tp& __value) : __value_(__value) {}
+
+  template <typename _Arg>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(_Arg&& __arg) const {
+    return std::forward<_Arg>(__arg) == __value_;
+  }
+};
+
+//! Logical negation of ==value
+template <typename _Tp>
+class __not_equal_value {
+  const _Tp& __value_;
+
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit __not_equal_value(const _Tp& __value) : __value_(__value) {}
+
+  template <typename _Arg>
+  _LIBCPP_HIDE_FROM_ABI bool operator()(_Arg&& __arg) const {
+    return !(std::forward<_Arg>(__arg) == __value_);
+  }
+};
+
+template <typename _ForwardIterator, typename _Compare>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator
+__cmp_iterators_by_values(_ForwardIterator __a, _ForwardIterator __b, _Compare __comp) {
+  if (__a < __b) { // we should return closer iterator
+    return __comp(*__b, *__a) ? __b : __a;
+  } else {
+    return __comp(*__a, *__b) ? __a : __b;
+  }
+}
+
+} // namespace __internal
+} // namespace __pstl
+
+#endif /* _PSTL_UTILS_H */
diff --git a/libcxx/include/__ranges/as_rvalue_view.h b/libcxx/include/__ranges/as_rvalue_view.h
index 422d8a8e08347..c4b1a51b5e2ee 100644
--- a/libcxx/include/__ranges/as_rvalue_view.h
+++ b/libcxx/include/__ranges/as_rvalue_view.h
@@ -125,7 +125,7 @@ struct __fn : __range_adaptor_closure<__fn> {
 } // namespace __as_rvalue
 
 inline namespace __cpo {
-constexpr auto as_rvalue = __as_rvalue::__fn{};
+inline constexpr auto as_rvalue = __as_rvalue::__fn{};
 } // namespace __cpo
 } // namespace views
 } // namespace ranges
diff --git a/libcxx/include/__std_mbstate_t.h b/libcxx/include/__std_mbstate_t.h
new file mode 100644
index 0000000000000..e79cc789fddf9
--- /dev/null
+++ b/libcxx/include/__std_mbstate_t.h
@@ -0,0 +1,29 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___STD_MBSTATE_T_H
+#define _LIBCPP___STD_MBSTATE_T_H
+
+#include <__config>
+#include <__mbstate_t.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// The goal of this header is to provide std::mbstate_t without requiring all
+// of <cuchar> or <cwchar>.
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+using ::mbstate_t _LIBCPP_USING_IF_EXISTS;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___STD_MBSTATE_T_H
diff --git a/libcxx/include/__tuple/sfinae_helpers.h b/libcxx/include/__tuple/sfinae_helpers.h
index f00fad5010089..384e3ca5aaa66 100644
--- a/libcxx/include/__tuple/sfinae_helpers.h
+++ b/libcxx/include/__tuple/sfinae_helpers.h
@@ -108,19 +108,6 @@ struct _LIBCPP_TEMPLATE_VIS tuple_element<_Ip, tuple<_Tp...> >
     typedef _LIBCPP_NODEBUG typename tuple_element<_Ip, __tuple_types<_Tp...> >::type type;
 };
 
-template <bool _IsTuple, class _SizeTrait, size_t _Expected>
-struct __tuple_like_with_size_imp : false_type {};
-
-template <class _SizeTrait, size_t _Expected>
-struct __tuple_like_with_size_imp<true, _SizeTrait, _Expected>
-    : integral_constant<bool, _SizeTrait::value == _Expected> {};
-
-template <class _Tuple, size_t _ExpectedSize, class _RawTuple = __libcpp_remove_reference_t<_Tuple> >
-using __tuple_like_with_size _LIBCPP_NODEBUG = __tuple_like_with_size_imp<
-                                   __tuple_like_ext<_RawTuple>::value,
-                                   tuple_size<_RawTuple>, _ExpectedSize
-                              >;
-
 struct _LIBCPP_TYPE_VIS __check_tuple_constructor_fail {
 
     static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_explicit_default() { return false; }
diff --git a/libcxx/include/__type_traits/is_convertible.h b/libcxx/include/__type_traits/is_convertible.h
index 873a64b5a486c..749d6fd47af9c 100644
--- a/libcxx/include/__type_traits/is_convertible.h
+++ b/libcxx/include/__type_traits/is_convertible.h
@@ -24,11 +24,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
+#if __has_builtin(__is_convertible) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS is_convertible : public integral_constant<bool, __is_convertible(_T1, _T2)> {};
+
+#elif __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
 
 template <class _T1, class _T2> struct _LIBCPP_TEMPLATE_VIS is_convertible
     : public integral_constant<bool, __is_convertible_to(_T1, _T2)> {};
 
+// TODO: Remove this fallback when GCC < 13 support is no longer required.
+// GCC 13 has the __is_convertible built-in.
 #else  // __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
 
 namespace __is_convertible_imp
diff --git a/libcxx/include/__type_traits/is_execution_policy.h b/libcxx/include/__type_traits/is_execution_policy.h
new file mode 100644
index 0000000000000..b97d94edd269e
--- /dev/null
+++ b/libcxx/include/__type_traits/is_execution_policy.h
@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+#define _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
+
+#include <__config>
+#include <__type_traits/remove_cvref.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class>
+inline constexpr bool is_execution_policy_v = false;
+
+template <class>
+inline constexpr bool __is_unsequenced_execution_policy_impl = false;
+
+template <class _Tp>
+inline constexpr bool __is_unsequenced_execution_policy_v =
+    __is_unsequenced_execution_policy_impl<__remove_cvref_t<_Tp>>;
+
+template <class>
+inline constexpr bool __is_parallel_execution_policy_impl = false;
+
+template <class _Tp>
+inline constexpr bool __is_parallel_execution_policy_v = __is_parallel_execution_policy_impl<__remove_cvref_t<_Tp>>;
+
+// Removes the "parallel" part of an execution policy.
+// For example, turns par_unseq into unseq, and par into seq.
+template <class _ExecutionPolicy>
+const auto& __remove_parallel_policy(_ExecutionPolicy&&);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___TYPE_TRAITS_IS_EXECUTION_POLICY_H
diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h
index 3f803687e5954..0d009061575b3 100644
--- a/libcxx/include/__utility/pair.h
+++ b/libcxx/include/__utility/pair.h
@@ -11,10 +11,14 @@
 
 #include <__compare/common_comparison_category.h>
 #include <__compare/synth_three_way.h>
+#include <__concepts/different_from.h>
 #include <__config>
 #include <__functional/unwrap_ref.h>
+#include <__fwd/array.h>
 #include <__fwd/get.h>
+#include <__fwd/subrange.h>
 #include <__fwd/tuple.h>
+#include <__tuple/pair_like.h>
 #include <__tuple/sfinae_helpers.h>
 #include <__tuple/tuple_element.h>
 #include <__tuple/tuple_indices.h>
@@ -23,6 +27,7 @@
 #include <__type_traits/common_type.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/decay.h>
+#include <__type_traits/integral_constant.h>
 #include <__type_traits/is_assignable.h>
 #include <__type_traits/is_constructible.h>
 #include <__type_traits/is_convertible.h>
@@ -39,6 +44,8 @@
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_swappable.h>
 #include <__type_traits/nat.h>
+#include <__type_traits/remove_cvref.h>
+#include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/piecewise_construct.h>
@@ -58,6 +65,14 @@ struct __non_trivially_copyable_base {
   __non_trivially_copyable_base(__non_trivially_copyable_base const&) _NOEXCEPT {}
 };
 
+#if _LIBCPP_STD_VER >= 23
+template <class _Tp>
+struct __is_specialization_of_subrange : false_type {};
+
+template <class _Iter, class _Sent, ranges::subrange_kind _Kind>
+struct __is_specialization_of_subrange<ranges::subrange<_Iter, _Sent, _Kind>> : true_type {};
+#endif
+
 template <class _T1, class _T2>
 struct _LIBCPP_TEMPLATE_VIS pair
 #if defined(_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR)
@@ -132,32 +147,6 @@ struct _LIBCPP_TEMPLATE_VIS pair
     using _CheckArgsDep _LIBCPP_NODEBUG = typename conditional<
       _MaybeEnable, _CheckArgs, __check_tuple_constructor_fail>::type;
 
-    struct _CheckTupleLikeConstructor {
-        template <class _Tuple>
-        static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit() {
-            return __tuple_convertible<_Tuple, pair>::value;
-        }
-
-        template <class _Tuple>
-        static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_explicit() {
-            return __tuple_constructible<_Tuple, pair>::value
-               && !__tuple_convertible<_Tuple, pair>::value;
-        }
-
-        template <class _Tuple>
-        static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_assign() {
-            return __tuple_assignable<_Tuple, pair>::value;
-        }
-    };
-
-    template <class _Tuple>
-    using _CheckTLC _LIBCPP_NODEBUG = __conditional_t<
-        __tuple_like_with_size<_Tuple, 2>::value
-            && !is_same<__decay_t<_Tuple>, pair>::value,
-        _CheckTupleLikeConstructor,
-        __check_tuple_constructor_fail
-    >;
-
     template<bool _Dummy = true, typename enable_if<
             _CheckArgsDep<_Dummy>::__enable_explicit_default()
     >::type* = nullptr>
@@ -279,21 +268,24 @@ struct _LIBCPP_TEMPLATE_VIS pair
         : first(std::move(__p.first)), second(std::move(__p.second)) {}
 #endif
 
-    template<class _Tuple, typename enable_if<
-            _CheckTLC<_Tuple>::template __enable_explicit<_Tuple>()
-    >::type* = nullptr>
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit pair(_Tuple&& __p)
-        : first(std::get<0>(std::forward<_Tuple>(__p))),
-          second(std::get<1>(std::forward<_Tuple>(__p))) {}
+#  if _LIBCPP_STD_VER >= 23
+    // This is a workaround for http://llvm.org/PR60710. We should be able to remove it once Clang is fixed.
+    template <class _PairLike, bool _Enable = tuple_size<remove_cvref_t<_PairLike>>::value == 2>
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __pair_like_explicit_wknd() {
+        if constexpr (tuple_size<remove_cvref_t<_PairLike>>::value == 2) {
+            return !is_convertible_v<decltype(std::get<0>(std::declval<_PairLike&&>())), first_type> ||
+                   !is_convertible_v<decltype(std::get<1>(std::declval<_PairLike&&>())), second_type>;
+        }
+        return false;
+    }
 
-    template<class _Tuple, typename enable_if<
-            _CheckTLC<_Tuple>::template __enable_implicit<_Tuple>()
-    >::type* = nullptr>
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
-    pair(_Tuple&& __p)
-        : first(std::get<0>(std::forward<_Tuple>(__p))),
-          second(std::get<1>(std::forward<_Tuple>(__p))) {}
+    template <__pair_like _PairLike>
+      requires(is_constructible_v<first_type, decltype(std::get<0>(std::declval<_PairLike&&>()))> &&
+               is_constructible_v<second_type, decltype(std::get<1>(std::declval<_PairLike&&>()))>)
+    _LIBCPP_HIDE_FROM_ABI constexpr explicit(__pair_like_explicit_wknd<_PairLike>())
+        pair(_PairLike&& __p)
+        : first(std::get<0>(std::forward<_PairLike>(__p))), second(std::get<1>(std::forward<_PairLike>(__p))) {}
+#  endif
 
     template <class... _Args1, class... _Args2>
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
@@ -331,7 +323,29 @@ struct _LIBCPP_TEMPLATE_VIS pair
         return *this;
     }
 
-#if _LIBCPP_STD_VER >= 23
+    template <class _U1, class _U2, __enable_if_t<
+        is_assignable<first_type&, _U1 const&>::value &&
+        is_assignable<second_type&, _U2 const&>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+    pair& operator=(pair<_U1, _U2> const& __p) {
+        first = __p.first;
+        second = __p.second;
+        return *this;
+    }
+
+    template <class _U1, class _U2, __enable_if_t<
+        is_assignable<first_type&, _U1>::value &&
+        is_assignable<second_type&, _U2>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+    pair& operator=(pair<_U1, _U2>&& __p) {
+        first = std::forward<_U1>(__p.first);
+        second = std::forward<_U2>(__p.second);
+        return *this;
+    }
+
+#  if _LIBCPP_STD_VER >= 23
     _LIBCPP_HIDE_FROM_ABI constexpr
     const pair& operator=(pair const& __p) const
       noexcept(is_nothrow_copy_assignable_v<const first_type> &&
@@ -373,17 +387,165 @@ struct _LIBCPP_TEMPLATE_VIS pair
         second = std::forward<_U2>(__p.second);
         return *this;
     }
-#endif // _LIBCPP_STD_VER >= 23
 
-    template <class _Tuple, typename enable_if<
-            _CheckTLC<_Tuple>::template __enable_assign<_Tuple>()
-     >::type* = nullptr>
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-    pair& operator=(_Tuple&& __p) {
-        first = std::get<0>(std::forward<_Tuple>(__p));
-        second = std::get<1>(std::forward<_Tuple>(__p));
+    template <__pair_like _PairLike>
+      requires(__different_from<_PairLike, pair> &&
+               !__is_specialization_of_subrange<remove_cvref_t<_PairLike>>::value &&
+               is_assignable_v<first_type&, decltype(std::get<0>(std::declval<_PairLike>()))> &&
+               is_assignable_v<second_type&, decltype(std::get<1>(std::declval<_PairLike>()))>)
+    _LIBCPP_HIDE_FROM_ABI constexpr pair& operator=(_PairLike&& __p) {
+        first  = std::get<0>(std::forward<_PairLike>(__p));
+        second = std::get<1>(std::forward<_PairLike>(__p));
+        return *this;
+    }
+
+    template <__pair_like _PairLike>
+      requires(__different_from<_PairLike, pair> &&
+               !__is_specialization_of_subrange<remove_cvref_t<_PairLike>>::value &&
+               is_assignable_v<first_type const&, decltype(std::get<0>(std::declval<_PairLike>()))> &&
+               is_assignable_v<second_type const&, decltype(std::get<1>(std::declval<_PairLike>()))>)
+    _LIBCPP_HIDE_FROM_ABI constexpr pair const& operator=(_PairLike&& __p) const {
+        first  = std::get<0>(std::forward<_PairLike>(__p));
+        second = std::get<1>(std::forward<_PairLike>(__p));
+        return *this;
+    }
+#  endif // _LIBCPP_STD_VER >= 23
+
+    // Prior to C++23, we provide an approximation of constructors and assignment operators from
+    // pair-like types. This was historically provided as an extension.
+#if _LIBCPP_STD_VER < 23
+    // from std::tuple
+    template<class _U1, class _U2, __enable_if_t<
+        is_convertible<_U1 const&, _T1>::value &&
+        is_convertible<_U2 const&, _T2>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair(tuple<_U1, _U2> const& __p)
+        : first(std::get<0>(__p)),
+          second(std::get<1>(__p)) {}
+
+    template<class _U1, class _U2, __enable_if_t<
+        is_constructible<_T1, _U1 const&>::value &&
+        is_constructible<_T2, _U2 const&>::value &&
+        !(is_convertible<_U1 const&, _T1>::value &&
+          is_convertible<_U2 const&, _T2>::value)
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    explicit
+    pair(tuple<_U1, _U2> const& __p)
+        : first(std::get<0>(__p)),
+          second(std::get<1>(__p)) {}
+
+    template<class _U1, class _U2, __enable_if_t<
+        is_convertible<_U1, _T1>::value &&
+        is_convertible<_U2, _T2>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair(tuple<_U1, _U2>&& __p)
+        : first(std::get<0>(std::move(__p))),
+          second(std::get<1>(std::move(__p))) {}
+
+    template<class _U1, class _U2, __enable_if_t<
+        is_constructible<_T1, _U1>::value &&
+        is_constructible<_T2, _U2>::value &&
+        !(is_convertible<_U1, _T1>::value &&
+          is_convertible<_U2, _T2>::value)
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    explicit
+    pair(tuple<_U1, _U2>&& __p)
+        : first(std::get<0>(std::move(__p))),
+          second(std::get<1>(std::move(__p))) {}
+
+
+    template<class _U1, class _U2, __enable_if_t<
+        is_assignable<_T1&, _U1 const&>::value &&
+        is_assignable<_T2&, _U2 const&>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair& operator=(tuple<_U1, _U2> const& __p) {
+        first = std::get<0>(__p);
+        second = std::get<1>(__p);
+        return *this;
+    }
+
+    template<class _U1, class _U2, __enable_if_t<
+        is_assignable<_T1&, _U1&&>::value &&
+        is_assignable<_T2&, _U2&&>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair& operator=(tuple<_U1, _U2>&& __p) {
+        first = std::get<0>(std::move(__p));
+        second = std::get<1>(std::move(__p));
+        return *this;
+    }
+
+    // from std::array
+    template<class _Up, __enable_if_t<
+        is_convertible<_Up const&, _T1>::value &&
+        is_convertible<_Up const&, _T2>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair(array<_Up, 2> const& __p)
+        : first(__p[0]),
+          second(__p[1]) {}
+
+    template<class _Up, __enable_if_t<
+        is_constructible<_T1, _Up const&>::value &&
+        is_constructible<_T2, _Up const&>::value &&
+        !(is_convertible<_Up const&, _T1>::value &&
+          is_convertible<_Up const&, _T2>::value)
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    explicit
+    pair(array<_Up, 2> const& __p)
+        : first(__p[0]),
+          second(__p[1]) {}
+
+    template<class _Up, __enable_if_t<
+        is_convertible<_Up, _T1>::value &&
+        is_convertible<_Up, _T2>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair(array<_Up, 2>&& __p)
+        : first(std::move(__p)[0]),
+          second(std::move(__p)[1]) {}
+
+    template<class _Up, __enable_if_t<
+        is_constructible<_T1, _Up>::value &&
+        is_constructible<_T2, _Up>::value &&
+        !(is_convertible<_Up, _T1>::value &&
+          is_convertible<_Up, _T2>::value)
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    explicit
+    pair(array<_Up, 2>&& __p)
+        : first(std::move(__p)[0]),
+          second(std::move(__p)[1]) {}
+
+
+    template<class _Up, __enable_if_t<
+        is_assignable<_T1&, _Up const&>::value &&
+        is_assignable<_T2&, _Up const&>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair& operator=(array<_Up, 2> const& __p) {
+        first = std::get<0>(__p);
+        second = std::get<1>(__p);
+        return *this;
+    }
+
+    template<class _Up, __enable_if_t<
+        is_assignable<_T1&, _Up>::value &&
+        is_assignable<_T2&, _Up>::value
+    >* = nullptr>
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+    pair& operator=(array<_Up, 2>&& __p) {
+        first = std::get<0>(std::move(__p));
+        second = std::get<1>(std::move(__p));
         return *this;
     }
+#endif // _LIBCPP_STD_VER < 23
 #endif // _LIBCPP_CXX03_LANG
 
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
diff --git a/libcxx/include/__utility/terminate_on_exception.h b/libcxx/include/__utility/terminate_on_exception.h
new file mode 100644
index 0000000000000..c046dee1e9569
--- /dev/null
+++ b/libcxx/include/__utility/terminate_on_exception.h
@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_TERMINATE_ON_EXCEPTION_H
+#define _LIBCPP___UTILITY_TERMINATE_ON_EXCEPTION_H
+
+#include <__config>
+#include <__exception/terminate.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+
+template <class _Func>
+_LIBCPP_HIDE_FROM_ABI auto __terminate_on_exception(_Func __func) {
+  try {
+    return __func();
+  } catch (...) {
+    std::terminate();
+  }
+}
+
+#  else  // _LIBCPP_HAS_NO_EXCEPTIONS
+
+template <class _Func>
+_LIBCPP_HIDE_FROM_ABI auto __terminate_on_exception(_Func __func) {
+  return __func();
+}
+
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___UTILITY_TERMINATE_ON_EXCEPTION_H
diff --git a/libcxx/include/__verbose_abort b/libcxx/include/__verbose_abort
index 048d9ff9a0033..a38284b711f36 100644
--- a/libcxx/include/__verbose_abort
+++ b/libcxx/include/__verbose_abort
@@ -47,7 +47,7 @@ void __libcpp_verbose_abort(const char *__format, ...);
 
 #  if defined(_LIBCPP_AVAILABILITY_HAS_NO_VERBOSE_ABORT)
 // The decltype is there to suppress -Wunused warnings in this configuration.
-_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR void __use(const char*, ...) { }
+void __use(const char*, ...);
 #    define _LIBCPP_VERBOSE_ABORT(...) (decltype(::std::__use(__VA_ARGS__))(), __builtin_abort())
 #  else
 #    define _LIBCPP_VERBOSE_ABORT(...) ::std::__libcpp_verbose_abort(__VA_ARGS__)
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index f5dcffa0a3f84..41ce675309434 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -1908,6 +1908,13 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/unwrap_iter.h>
 #include <__algorithm/upper_bound.h>
 
+#ifdef _LIBCPP_HAS_PARALLEL_ALGORITHMS
+#  include <__algorithm/pstl_any_all_none_of.h>
+#  include <__algorithm/pstl_fill.h>
+#  include <__algorithm/pstl_find.h>
+#  include <__algorithm/pstl_for_each.h>
+#endif
+
 // standard-mandated includes
 
 // [algorithm.syn]
diff --git a/libcxx/include/any b/libcxx/include/any
index d2f85269599f8..e056444f83c98 100644
--- a/libcxx/include/any
+++ b/libcxx/include/any
@@ -721,6 +721,7 @@ _LIBCPP_END_NAMESPACE_STD
 #  include <iosfwd>
 #  include <iterator>
 #  include <memory>
+#  include <stdexcept>
 #  include <type_traits>
 #  include <variant>
 #endif
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index b5d3b0318742a..483e42d8b2cc3 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -315,6 +315,10 @@ public:
 template<class Duration>
   using file_time = time_point<file_clock, Duration>;   // C++20
 
+template<class charT, class traits, class Duration>     // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const file_time<Duration>& tp);
+
 class steady_clock
 {
 public:
@@ -336,6 +340,10 @@ template<class Duration>
 using local_seconds = local_time<seconds>;
 using local_days    = local_time<days>;
 
+template<class charT, class traits, class Duration>     // C++20
+  basic_ostream<charT, traits>&
+    operator<<(basic_ostream<charT, traits>& os, const local_time<Duration>& tp);
+
 // 25.8.2, class last_spec    // C++20
 struct last_spec;
 
@@ -692,6 +700,10 @@ bool operator>=(const time_zone& x, const time_zone& y) noexcept;
 namespace std {
   template<class Duration, class charT>
     struct formatter<chrono::sys_time<Duration>, charT>;                          // C++20
+  template<class Duration, class charT>
+    struct formatter<chrono::filetime<Duration>, charT>;                          // C++20
+  template<class Duration, class charT>
+    struct formatter<chrono::local_time<Duration>, charT>;                        // C++20
   template<class Rep, class Period, class charT>
     struct formatter<chrono::duration<Rep, Period>, charT>;                       // C++20
   template<class charT> struct formatter<chrono::day, charT>;                     // C++20
diff --git a/libcxx/include/execution b/libcxx/include/execution
index b68cdfac2c6d3..d73445dca112b 100644
--- a/libcxx/include/execution
+++ b/libcxx/include/execution
@@ -10,12 +10,142 @@
 #ifndef _LIBCPP_EXECUTION
 #define _LIBCPP_EXECUTION
 
+/*
+namespace std::execution {
+  struct sequenced_policy;
+  struct parallel_policy;
+  struct parallel_unsequenced_policy;
+  struct unsequenced_policy; // since C++20
+
+  inline constexpr sequenced_policy seq = implementation-defined;
+  inline constexpr parallel_policy par = implementation-defined;
+  inline constexpr parallel_unsequenced_policy par_unseq = implementation-defined;
+  inline constexpr unsequenced_policy unseq = implementation-defined; // since C++20
+}
+
+namespace std {
+  template <class T>
+  struct is_execution_policy;
+
+  template <class T>
+  inline constexpr bool is_execution_policy_v;
+}
+*/
+
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__config>
+#include <__type_traits/is_execution_policy.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_cvref.h>
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+#if defined(_LIBCPP_HAS_PARALLEL_ALGORITHMS) && _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+namespace execution {
+struct __disable_user_instantiations_tag {
+  explicit __disable_user_instantiations_tag() = default;
+};
+
+struct sequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit sequenced_policy(__disable_user_instantiations_tag) {}
+  sequenced_policy(const sequenced_policy&)            = delete;
+  sequenced_policy& operator=(const sequenced_policy&) = delete;
+};
+
+inline constexpr sequenced_policy seq{__disable_user_instantiations_tag{}};
+
+struct parallel_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit parallel_policy(__disable_user_instantiations_tag) {}
+  parallel_policy(const parallel_policy&)            = delete;
+  parallel_policy& operator=(const parallel_policy&) = delete;
+};
+
+inline constexpr parallel_policy par{__disable_user_instantiations_tag{}};
+
+struct parallel_unsequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit parallel_unsequenced_policy(__disable_user_instantiations_tag) {}
+  parallel_unsequenced_policy(const parallel_unsequenced_policy&)            = delete;
+  parallel_unsequenced_policy& operator=(const parallel_unsequenced_policy&) = delete;
+};
+
+inline constexpr parallel_unsequenced_policy par_unseq{__disable_user_instantiations_tag{}};
+
+struct __unsequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit __unsequenced_policy(__disable_user_instantiations_tag) {}
+  __unsequenced_policy(const __unsequenced_policy&)            = delete;
+  __unsequenced_policy& operator=(const __unsequenced_policy&) = delete;
+};
+
+constexpr __unsequenced_policy __unseq{__disable_user_instantiations_tag{}};
+
+#  if _LIBCPP_STD_VER >= 20
+
+struct unsequenced_policy {
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit unsequenced_policy(__disable_user_instantiations_tag) {}
+  unsequenced_policy(const unsequenced_policy&)            = delete;
+  unsequenced_policy& operator=(const unsequenced_policy&) = delete;
+};
+
+inline constexpr unsequenced_policy unseq{__disable_user_instantiations_tag{}};
+
+#  endif // _LIBCPP_STD_VER >= 20
+
+} // namespace execution
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::sequenced_policy> = true;
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::parallel_policy> = true;
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::parallel_unsequenced_policy> = true;
+
+template <>
+inline constexpr bool is_execution_policy_v<execution::__unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_parallel_execution_policy_impl<execution::parallel_policy> = true;
+
+template <>
+inline constexpr bool __is_parallel_execution_policy_impl<execution::parallel_unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_unsequenced_execution_policy_impl<execution::__unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_unsequenced_execution_policy_impl<execution::parallel_unsequenced_policy> = true;
+
+#  if _LIBCPP_STD_VER >= 20
+template <>
+inline constexpr bool is_execution_policy_v<execution::unsequenced_policy> = true;
+
+template <>
+inline constexpr bool __is_unsequenced_execution_policy_impl<execution::unsequenced_policy> = true;
+
+#  endif
+
+template <class _Tp>
+struct is_execution_policy : bool_constant<is_execution_policy_v<_Tp>> {};
+
+template <class _ExecutionPolicy>
+const auto& __remove_parallel_policy(_ExecutionPolicy&&) {
+  using _ExecPol = __remove_cvref_t<_ExecutionPolicy>;
+  if constexpr (is_same_v<_ExecPol, execution::parallel_policy>) {
+    return execution::seq;
+  } else if constexpr (is_same_v<_ExecPol, execution::parallel_unsequenced_policy>) {
+    return execution::__unseq;
+  }
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // defined(_LIBCPP_HAS_PARALLEL_ALGORITHMS) && _LIBCPP_STD_VER >= 17
+
 #endif // _LIBCPP_EXECUTION
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index a312d18c49383..1aa5414152a85 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -183,6 +183,7 @@ typedef basic_fstream<wchar_t> wfstream;
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__availability>
 #include <__config>
+#include <__fwd/fstream.h>
 #include <__locale>
 #include <__utility/move.h>
 #include <__utility/swap.h>
diff --git a/libcxx/include/ios b/libcxx/include/ios
index c97daac1ee548..f811343f0324d 100644
--- a/libcxx/include/ios
+++ b/libcxx/include/ios
@@ -217,6 +217,7 @@ storage-class-specifier const error_category& iostream_category() noexcept;
 #endif
 
 #include <__assert> // all public C++ headers provide the assertion handler
+#include <__fwd/ios.h>
 #include <__ios/fpos.h>
 #include <__locale>
 #include <__system_error/error_category.h>
diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd
index 7948606e6453e..e3cd9faa70b7b 100644
--- a/libcxx/include/iosfwd
+++ b/libcxx/include/iosfwd
@@ -96,8 +96,14 @@ using u32streampos = fpos<char_traits<char32_t>::state_type>;
 
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__config>
+#include <__fwd/fstream.h>
+#include <__fwd/ios.h>
+#include <__fwd/istream.h>
+#include <__fwd/ostream.h>
+#include <__fwd/sstream.h>
+#include <__fwd/streambuf.h>
 #include <__fwd/string.h>
-#include <__mbstate_t.h>
+#include <__std_mbstate_t.h>
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -108,112 +114,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 class _LIBCPP_TYPE_VIS ios_base;
 
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_ios;
-
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_streambuf;
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_istream;
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_ostream;
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_iostream;
-
-template <class _CharT, class _Traits = char_traits<_CharT>,
-          class _Allocator = allocator<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_stringbuf;
-template <class _CharT, class _Traits = char_traits<_CharT>,
-          class _Allocator = allocator<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_istringstream;
-template <class _CharT, class _Traits = char_traits<_CharT>,
-          class _Allocator = allocator<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_ostringstream;
-template <class _CharT, class _Traits = char_traits<_CharT>,
-          class _Allocator = allocator<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_stringstream;
-
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_filebuf;
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_ifstream;
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_ofstream;
-template <class _CharT, class _Traits = char_traits<_CharT> >
-    class _LIBCPP_TEMPLATE_VIS basic_fstream;
-
 template <class _CharT, class _Traits = char_traits<_CharT> >
     class _LIBCPP_TEMPLATE_VIS istreambuf_iterator;
 template <class _CharT, class _Traits = char_traits<_CharT> >
     class _LIBCPP_TEMPLATE_VIS ostreambuf_iterator;
 
-typedef basic_ios<char>              ios;
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-typedef basic_ios<wchar_t>           wios;
-#endif
-
-typedef basic_streambuf<char>        streambuf;
-typedef basic_istream<char>          istream;
-typedef basic_ostream<char>          ostream;
-typedef basic_iostream<char>         iostream;
-
-typedef basic_stringbuf<char>        stringbuf;
-typedef basic_istringstream<char>    istringstream;
-typedef basic_ostringstream<char>    ostringstream;
-typedef basic_stringstream<char>     stringstream;
-
-typedef basic_filebuf<char>          filebuf;
-typedef basic_ifstream<char>         ifstream;
-typedef basic_ofstream<char>         ofstream;
-typedef basic_fstream<char>          fstream;
-
-#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
-typedef basic_streambuf<wchar_t>     wstreambuf;
-typedef basic_istream<wchar_t>       wistream;
-typedef basic_ostream<wchar_t>       wostream;
-typedef basic_iostream<wchar_t>      wiostream;
-
-typedef basic_stringbuf<wchar_t>     wstringbuf;
-typedef basic_istringstream<wchar_t> wistringstream;
-typedef basic_ostringstream<wchar_t> wostringstream;
-typedef basic_stringstream<wchar_t>  wstringstream;
-
-typedef basic_filebuf<wchar_t>       wfilebuf;
-typedef basic_ifstream<wchar_t>      wifstream;
-typedef basic_ofstream<wchar_t>      wofstream;
-typedef basic_fstream<wchar_t>       wfstream;
-#endif
-
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(ios) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wios)) basic_ios;
-
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(streambuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstreambuf)) basic_streambuf;
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(istream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wistream)) basic_istream;
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(ostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wostream)) basic_ostream;
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(iostream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wiostream)) basic_iostream;
-
-template <class _CharT, class _Traits, class _Allocator>
-    class _LIBCPP_PREFERRED_NAME(stringbuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstringbuf)) basic_stringbuf;
-template <class _CharT, class _Traits, class _Allocator>
-    class _LIBCPP_PREFERRED_NAME(istringstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wistringstream)) basic_istringstream;
-template <class _CharT, class _Traits, class _Allocator>
-    class _LIBCPP_PREFERRED_NAME(ostringstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wostringstream)) basic_ostringstream;
-template <class _CharT, class _Traits, class _Allocator>
-    class _LIBCPP_PREFERRED_NAME(stringstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wstringstream)) basic_stringstream;
-
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(filebuf) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wfilebuf)) basic_filebuf;
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(ifstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wifstream)) basic_ifstream;
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(ofstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wofstream)) basic_ofstream;
-template <class _CharT, class _Traits>
-    class _LIBCPP_PREFERRED_NAME(fstream) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wfstream)) basic_fstream;
-
 template <class _State>             class _LIBCPP_TEMPLATE_VIS fpos;
 typedef fpos<mbstate_t>    streampos;
 typedef fpos<mbstate_t>    wstreampos;
@@ -223,13 +128,6 @@ typedef fpos<mbstate_t>    u8streampos;
 typedef fpos<mbstate_t>    u16streampos;
 typedef fpos<mbstate_t>    u32streampos;
 
-#if defined(_NEWLIB_VERSION)
-// On newlib, off_t is 'long int'
-typedef long int streamoff;         // for char_traits in <string>
-#else
-typedef long long streamoff;        // for char_traits in <string>
-#endif
-
 // Include other forward declarations here
 template <class _Tp, class _Alloc = allocator<_Tp> >
 class _LIBCPP_TEMPLATE_VIS vector;
diff --git a/libcxx/include/istream b/libcxx/include/istream
index 7350e11fe129d..8b440c036ddb2 100644
--- a/libcxx/include/istream
+++ b/libcxx/include/istream
@@ -160,6 +160,7 @@ template <class Stream, class T>
 
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__config>
+#include <__fwd/istream.h>
 #include <__iterator/istreambuf_iterator.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/enable_if.h>
diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
index d6e26d8844451..271812e9eb9d4 100644
--- a/libcxx/include/libcxx.imp
+++ b/libcxx/include/libcxx.imp
@@ -37,6 +37,7 @@
   { include: [ "@<__memory_resource/.*>", "private", "<memory_resource>", "public" ] },
   { include: [ "@<__mutex/.*>", "private", "<mutex>", "public" ] },
   { include: [ "@<__numeric/.*>", "private", "<numeric>", "public" ] },
+  { include: [ "@<__pstl/.*>", "private", "<pstl>", "public" ] },
   { include: [ "@<__random/.*>", "private", "<random>", "public" ] },
   { include: [ "@<__ranges/.*>", "private", "<ranges>", "public" ] },
   { include: [ "@<__string/.*>", "private", "<string>", "public" ] },
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 8f8333692558e..45a22d9d4822a 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -877,7 +877,11 @@ module std [system] {
       module file_type                    { private header "__filesystem/file_type.h" }
       module filesystem_error             { private header "__filesystem/filesystem_error.h" }
       module operations                   { private header "__filesystem/operations.h" }
-      module path                         { private header "__filesystem/path.h" }
+      module path                         {
+        private header "__filesystem/path.h"
+        export functional.__functional.hash
+        export functional.__functional.unary_function
+      }
       module path_iterator                { private header "__filesystem/path_iterator.h" }
       module perm_options                 { private header "__filesystem/perm_options.h" }
       module perms                        { private header "__filesystem/perms.h" }
@@ -1012,6 +1016,14 @@ module std [system] {
   module iosfwd {
     header "iosfwd"
     export *
+    module __iosfwd {
+      module fstream_fwd   { private header "__fwd/fstream.h" }
+      module ios_fwd       { private header "__fwd/ios.h" }
+      module istream_fwd   { private header "__fwd/istream.h" }
+      module ostream_fwd   { private header "__fwd/ostream.h" }
+      module sstream_fwd   { private header "__fwd/sstream.h" }
+      module streambuf_fwd { private header "__fwd/streambuf.h" }
+    }
   }
   module iostream {
     @requires_LIBCXX_ENABLE_LOCALIZATION@
@@ -1547,6 +1559,7 @@ module std [system] {
       private header "__type_traits/is_equality_comparable.h"
       export integral_constant
     }
+    module is_execution_policy                 { private header "__type_traits/is_execution_policy.h" }
     module is_final                            { private header "__type_traits/is_final.h" }
     module is_floating_point                   { private header "__type_traits/is_floating_point.h" }
     module is_function                         { private header "__type_traits/is_function.h" }
@@ -1664,29 +1677,30 @@ module std [system] {
     export *
 
     module __utility {
-      module as_const            { private header "__utility/as_const.h" }
-      module auto_cast           {
+      module as_const               { private header "__utility/as_const.h" }
+      module auto_cast              {
         private header "__utility/auto_cast.h"
         export type_traits.decay
       }
-      module cmp                 { private header "__utility/cmp.h" }
-      module convert_to_integral { private header "__utility/convert_to_integral.h" }
-      module declval             { private header "__utility/declval.h" }
-      module exception_guard     { private header "__utility/exception_guard.h" }
-      module exchange            { private header "__utility/exchange.h" }
-      module forward             { private header "__utility/forward.h" }
-      module forward_like        { private header "__utility/forward_like.h" }
-      module in_place            { private header "__utility/in_place.h" }
-      module integer_sequence    { private header "__utility/integer_sequence.h" }
-      module move                { private header "__utility/move.h" }
-      module pair                { private header "__utility/pair.h" }
-      module pair_fwd            { private header "__fwd/pair.h" }
-      module piecewise_construct { private header "__utility/piecewise_construct.h" }
-      module priority_tag        { private header "__utility/priority_tag.h" }
-      module rel_ops             { private header "__utility/rel_ops.h" }
-      module swap                { private header "__utility/swap.h" }
-      module to_underlying       { private header "__utility/to_underlying.h" }
-      module unreachable         { private header "__utility/unreachable.h" }
+      module cmp                    { private header "__utility/cmp.h" }
+      module convert_to_integral    { private header "__utility/convert_to_integral.h" }
+      module declval                { private header "__utility/declval.h" }
+      module exception_guard        { private header "__utility/exception_guard.h" }
+      module exchange               { private header "__utility/exchange.h" }
+      module forward                { private header "__utility/forward.h" }
+      module forward_like           { private header "__utility/forward_like.h" }
+      module in_place               { private header "__utility/in_place.h" }
+      module integer_sequence       { private header "__utility/integer_sequence.h" }
+      module move                   { private header "__utility/move.h" }
+      module pair                   { private header "__utility/pair.h" }
+      module pair_fwd               { private header "__fwd/pair.h" }
+      module piecewise_construct    { private header "__utility/piecewise_construct.h" }
+      module priority_tag           { private header "__utility/priority_tag.h" }
+      module rel_ops                { private header "__utility/rel_ops.h" }
+      module swap                   { private header "__utility/swap.h" }
+      module terminate_on_exception { private header "__utility/terminate_on_exception.h" }
+      module to_underlying          { private header "__utility/to_underlying.h" }
+      module unreachable            { private header "__utility/unreachable.h" }
     }
   }
   module valarray {
@@ -1725,6 +1739,7 @@ module std [system] {
   module __mbstate_t         { private header "__mbstate_t.h"       export * }
   module __node_handle       { private header "__node_handle"       export * }
   module __split_buffer      { private header "__split_buffer"      export * }
+  module __std_mbstate_t     { private header "__std_mbstate_t.h"   export * }
   module __threading_support {         header "__threading_support" export * }
   module __tree              {         header "__tree"              export * }
   module __undef_macros      {         header "__undef_macros"      export * }
diff --git a/libcxx/include/optional b/libcxx/include/optional
index 77f52b757dc70..8895ed152dc82 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -16,107 +16,126 @@
 // C++1z
 
 namespace std {
-  // 23.6.3, optional for object types
-  template <class T> class optional;
+  // [optional.optional], class template optional
+  template <class T>
+    class optional;
 
-  // 23.6.4, no-value state indicator
+  template<class T>
+    concept is-derived-from-optional = requires(const T& t) {       // exposition only
+      []<class U>(const optional<U>&){ }(t);
+    };
+
+  // [optional.nullopt], no-value state indicator
   struct nullopt_t{see below };
   inline constexpr nullopt_t nullopt(unspecified );
 
-  // 23.6.5, class bad_optional_access
+  // [optional.bad.access], class bad_optional_access
   class bad_optional_access;
 
-  // 23.6.6, relational operators
+  // [optional.relops], relational operators
   template <class T, class U>
-  constexpr bool operator==(const optional<T>&, const optional<U>&);
+    constexpr bool operator==(const optional<T>&, const optional<U>&);
   template <class T, class U>
-  constexpr bool operator!=(const optional<T>&, const optional<U>&);
+    constexpr bool operator!=(const optional<T>&, const optional<U>&);
   template <class T, class U>
-  constexpr bool operator<(const optional<T>&, const optional<U>&);
+    constexpr bool operator<(const optional<T>&, const optional<U>&);
   template <class T, class U>
-  constexpr bool operator>(const optional<T>&, const optional<U>&);
+    constexpr bool operator>(const optional<T>&, const optional<U>&);
   template <class T, class U>
-  constexpr bool operator<=(const optional<T>&, const optional<U>&);
+    constexpr bool operator<=(const optional<T>&, const optional<U>&);
   template <class T, class U>
-  constexpr bool operator>=(const optional<T>&, const optional<U>&);
-
-  // 23.6.7 comparison with nullopt
-  template <class T> constexpr bool operator==(const optional<T>&, nullopt_t) noexcept;
-  template <class T> constexpr bool operator==(nullopt_t, const optional<T>&) noexcept;
-  template <class T> constexpr bool operator!=(const optional<T>&, nullopt_t) noexcept;
-  template <class T> constexpr bool operator!=(nullopt_t, const optional<T>&) noexcept;
-  template <class T> constexpr bool operator<(const optional<T>&, nullopt_t) noexcept;
-  template <class T> constexpr bool operator<(nullopt_t, const optional<T>&) noexcept;
-  template <class T> constexpr bool operator<=(const optional<T>&, nullopt_t) noexcept;
-  template <class T> constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept;
-  template <class T> constexpr bool operator>(const optional<T>&, nullopt_t) noexcept;
-  template <class T> constexpr bool operator>(nullopt_t, const optional<T>&) noexcept;
-  template <class T> constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept;
-  template <class T> constexpr bool operator>=(nullopt_t, const optional<T>&) noexcept;
-
-  // 23.6.8, comparison with T
-  template <class T, class U> constexpr bool operator==(const optional<T>&, const U&);
-  template <class T, class U> constexpr bool operator==(const T&, const optional<U>&);
-  template <class T, class U> constexpr bool operator!=(const optional<T>&, const U&);
-  template <class T, class U> constexpr bool operator!=(const T&, const optional<U>&);
-  template <class T, class U> constexpr bool operator<(const optional<T>&, const U&);
-  template <class T, class U> constexpr bool operator<(const T&, const optional<U>&);
-  template <class T, class U> constexpr bool operator<=(const optional<T>&, const U&);
-  template <class T, class U> constexpr bool operator<=(const T&, const optional<U>&);
-  template <class T, class U> constexpr bool operator>(const optional<T>&, const U&);
-  template <class T, class U> constexpr bool operator>(const T&, const optional<U>&);
-  template <class T, class U> constexpr bool operator>=(const optional<T>&, const U&);
-  template <class T, class U> constexpr bool operator>=(const T&, const optional<U>&);
-
-  // 23.6.9, specialized algorithms
-  template <class T> void swap(optional<T>&, optional<T>&) noexcept(see below ); // constexpr in C++20
-  template <class T> constexpr optional<see below > make_optional(T&&);
-  template <class T, class... Args>
+    constexpr bool operator>=(const optional<T>&, const optional<U>&);
+  template<class T, three_way_comparable_with<T> U>
+    constexpr compare_three_way_result_t<T, U>
+      operator<=>(const optional<T>&, const optional<U>&); // since C++20
+
+  // [optional.nullops], comparison with nullopt
+  template<class T> constexpr bool operator==(const optional<T>&, nullopt_t) noexcept;
+  template<class T> constexpr bool operator==(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T> constexpr bool operator!=(const optional<T>&, nullopt_t) noexcept; // until C++17
+  template<class T> constexpr bool operator!=(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T> constexpr bool operator<(const optional<T>&, nullopt_t) noexcept;  // until C++17
+  template<class T> constexpr bool operator<(nullopt_t, const optional<T>&) noexcept;  // until C++17
+  template<class T> constexpr bool operator<=(const optional<T>&, nullopt_t) noexcept; // until C++17
+  template<class T> constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T> constexpr bool operator>(const optional<T>&, nullopt_t) noexcept;  // until C++17
+  template<class T> constexpr bool operator>(nullopt_t, const optional<T>&) noexcept;  // until C++17
+  template<class T> constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept; // until C++17
+  template<class T> constexpr bool operator>=(nullopt_t, const optional<T>&) noexcept; // until C++17
+  template<class T>
+    constexpr strong_ordering operator<=>(const optional<T>&, nullopt_t) noexcept;     // since C++20
+
+  // [optional.comp.with.t], comparison with T
+  template<class T, class U> constexpr bool operator==(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator==(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator!=(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator!=(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator<(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator<(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator<=(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator<=(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator>(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator>(const T&, const optional<U>&);
+  template<class T, class U> constexpr bool operator>=(const optional<T>&, const U&);
+  template<class T, class U> constexpr bool operator>=(const T&, const optional<U>&);
+  template<class T, class U>
+      requires (!is-derived-from-optional<U>) && three_way_comparable_with<T, U>
+    constexpr compare_three_way_result_t<T, U>
+      operator<=>(const optional<T>&, const U&);                                       // since C++20
+
+  // [optional.specalg], specialized algorithms
+  template<class T>
+    void swap(optional<T>&, optional<T>&) noexcept(see below ); // constexpr in C++20
+
+  template<class T>
+    constexpr optional<see below > make_optional(T&&);
+  template<class T, class... Args>
     constexpr optional<T> make_optional(Args&&... args);
-  template <class T, class U, class... Args>
+  template<class T, class U, class... Args>
     constexpr optional<T> make_optional(initializer_list<U> il, Args&&... args);
 
-  // 23.6.10, hash support
-  template <class T> struct hash;
-  template <class T> struct hash<optional<T>>;
+  // [optional.hash], hash support
+  template<class T> struct hash;
+  template<class T> struct hash<optional<T>>;
 
-  template <class T> class optional {
+  template<class T>
+  class optional {
   public:
     using value_type = T;
 
-    // 23.6.3.1, constructors
+    // [optional.ctor], constructors
     constexpr optional() noexcept;
     constexpr optional(nullopt_t) noexcept;
     constexpr optional(const optional &);
     constexpr optional(optional &&) noexcept(see below);
-    template <class... Args> constexpr explicit optional(in_place_t, Args &&...);
-    template <class U, class... Args>
+    template<class... Args>
+      constexpr explicit optional(in_place_t, Args &&...);
+    template<class U, class... Args>
       constexpr explicit optional(in_place_t, initializer_list<U>, Args &&...);
-    template <class U = T>
+    template<class U = T>
       constexpr explicit(see-below) optional(U &&);
-    template <class U>
-      explicit(see-below) optional(const optional<U> &);         // constexpr in C++20
-    template <class U>
-      explicit(see-below) optional(optional<U> &&);              // constexpr in C++20
+    template<class U>
+      explicit(see-below) optional(const optional<U> &);                          // constexpr in C++20
+    template<class U>
+      explicit(see-below) optional(optional<U> &&);                               // constexpr in C++20
 
-    // 23.6.3.2, destructor
+    // [optional.dtor], destructor
     ~optional(); // constexpr in C++20
 
-    // 23.6.3.3, assignment
-    optional &operator=(nullopt_t) noexcept;                     // constexpr in C++20
+    // [optional.assign], assignment
+    optional &operator=(nullopt_t) noexcept;                                      // constexpr in C++20
     constexpr optional &operator=(const optional &);
     constexpr optional &operator=(optional &&) noexcept(see below);
-    template <class U = T> optional &operator=(U &&);            // constexpr in C++20
-    template <class U> optional &operator=(const optional<U> &); // constexpr in C++20
-    template <class U> optional &operator=(optional<U> &&);      // constexpr in C++20
-    template <class... Args> T& emplace(Args &&...);             // constexpr in C++20
-    template <class U, class... Args>
-      T& emplace(initializer_list<U>, Args &&...);               // constexpr in C++20
-
-    // 23.6.3.4, swap
+    template<class U = T> optional &operator=(U &&);                              // constexpr in C++20
+    template<class U> optional &operator=(const optional<U> &);                   // constexpr in C++20
+    template<class U> optional &operator=(optional<U> &&);                        // constexpr in C++20
+    template<class... Args> T& emplace(Args &&...);                               // constexpr in C++20
+    template<class U, class... Args> T& emplace(initializer_list<U>, Args &&...); // constexpr in C++20
+
+    // [optional.swap], swap
     void swap(optional &) noexcept(see below ); // constexpr in C++20
 
-    // 23.6.3.5, observers
+    // [optional.observe], observers
     constexpr T const *operator->() const;
     constexpr T *operator->();
     constexpr T const &operator*() const &;
@@ -129,8 +148,8 @@ namespace std {
     constexpr T &value() &;
     constexpr T &&value() &&;
     constexpr const T &&value() const &&;
-    template <class U> constexpr T value_or(U &&) const &;
-    template <class U> constexpr T value_or(U &&) &&;
+    template<class U> constexpr T value_or(U &&) const &;
+    template<class U> constexpr T value_or(U &&) &&;
 
     // [optional.monadic], monadic operations
     template<class F> constexpr auto and_then(F&& f) &;         // since C++23
@@ -144,15 +163,15 @@ namespace std {
     template<class F> constexpr optional or_else(F&& f) &&;     // since C++23
     template<class F> constexpr optional or_else(F&& f) const&; // since C++23
 
-    // 23.6.3.6, modifiers
-    void reset() noexcept; // constexpr in C++20
+    // [optional.mod], modifiers
+    void reset() noexcept;                                      // constexpr in C++20
 
   private:
-    T *val; // exposition only
+    T *val;         // exposition only
   };
 
-template<class T>
-  optional(T) -> optional<T>;
+  template<class T>
+    optional(T) -> optional<T>;
 
 } // namespace std
 
@@ -160,6 +179,8 @@ template<class T>
 
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__availability>
+#include <__compare/compare_three_way_result.h>
+#include <__compare/three_way_comparable.h>
 #include <__concepts/invocable.h>
 #include <__config>
 #include <__functional/hash.h>
@@ -633,12 +654,20 @@ using __optional_sfinae_assign_base_t = __sfinae_assign_base<
 
 template<class _Tp>
 class optional;
+
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+concept __is_derived_from_optional = requires(const _Tp& __t) { []<class __Up>(const optional<__Up>&) {}(__t); };
+
+#  endif // _LIBCPP_STD_VER >= 20
+
 template <class _Tp>
 struct __is_std_optional : false_type {};
 template <class _Tp> struct __is_std_optional<optional<_Tp>> : true_type {};
 
 template <class _Tp>
-class optional
+class _LIBCPP_DECLSPEC_EMPTY_BASES optional
     : private __optional_move_assign_base<_Tp>
     , private __optional_sfinae_ctor_base_t<_Tp>
     , private __optional_sfinae_assign_base_t<_Tp>
@@ -1288,6 +1317,18 @@ operator>=(const optional<_Tp>& __x, const optional<_Up>& __y)
     return *__x >= *__y;
 }
 
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, three_way_comparable_with<_Tp> _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Tp, _Up>
+operator<=>(const optional<_Tp>& __x, const optional<_Up>& __y) {
+    if (__x && __y)
+        return *__x <=> *__y;
+    return __x.has_value() <=> __y.has_value();
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
 // Comparisons with nullopt
 template <class _Tp>
 _LIBCPP_INLINE_VISIBILITY constexpr
@@ -1297,6 +1338,8 @@ operator==(const optional<_Tp>& __x, nullopt_t) noexcept
     return !static_cast<bool>(__x);
 }
 
+#if _LIBCPP_STD_VER <= 17
+
 template <class _Tp>
 _LIBCPP_INLINE_VISIBILITY constexpr
 bool
@@ -1385,6 +1428,15 @@ operator>=(nullopt_t, const optional<_Tp>& __x) noexcept
     return !static_cast<bool>(__x);
 }
 
+#else // _LIBCPP_STD_VER <= 17
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const optional<_Tp>& __x, nullopt_t) noexcept {
+    return __x.has_value() <=> false;
+}
+
+#endif // _LIBCPP_STD_VER <= 17
+
 // Comparisons with T
 template <class _Tp, class _Up>
 _LIBCPP_INLINE_VISIBILITY constexpr
@@ -1530,6 +1582,17 @@ operator>=(const _Tp& __v, const optional<_Up>& __x)
     return static_cast<bool>(__x) ? __v >= *__x : true;
 }
 
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp, class _Up>
+  requires(!__is_derived_from_optional<_Up>) && three_way_comparable_with<_Tp, _Up>
+_LIBCPP_HIDE_FROM_ABI constexpr compare_three_way_result_t<_Tp, _Up>
+operator<=>(const optional<_Tp>& __x, const _Up& __v) {
+    return __x.has_value() ? *__x <=> __v : strong_ordering::less;
+}
+
+#endif // _LIBCPP_STD_VER >= 20
+
 
 template <class _Tp>
 inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index 610a283942044..168a75376bd94 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -166,6 +166,7 @@ basic_ostream<wchar_t, traits>& operator<<(basic_ostream<wchar_t, traits>&, cons
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__config>
 #include <__exception/operations.h>
+#include <__fwd/ostream.h>
 #include <__memory/shared_ptr.h>
 #include <__memory/unique_ptr.h>
 #include <__system_error/error_code.h>
diff --git a/libcxx/include/pstl/internal/algorithm_fwd.h b/libcxx/include/pstl/internal/algorithm_fwd.h
deleted file mode 100644
index 3dcf9e9f5e9eb..0000000000000
--- a/libcxx/include/pstl/internal/algorithm_fwd.h
+++ /dev/null
@@ -1,1202 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_ALGORITHM_FWD_H
-#define _PSTL_ALGORITHM_FWD_H
-
-#include <iterator>
-#include <type_traits>
-#include <utility>
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// any_of
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Pred>
-bool
-__brick_any_of(const _ForwardIterator, const _ForwardIterator, _Pred,
-               /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Pred>
-bool
-__brick_any_of(const _RandomAccessIterator, const _RandomAccessIterator, _Pred,
-               /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Pred>
-bool
-__pattern_any_of(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Pred) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Pred>
-bool
-__pattern_any_of(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Pred);
-
-//------------------------------------------------------------------------
-// walk1 (pseudo)
-//
-// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Function>
-void __brick_walk1(_ForwardIterator, _ForwardIterator, _Function,
-                   /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Function>
-void __brick_walk1(_RandomAccessIterator, _RandomAccessIterator, _Function,
-                   /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Function>
-void
-__pattern_walk1(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Function>
-void
-__pattern_walk1(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Brick>
-void
-__pattern_walk_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Brick>
-void
-__pattern_walk_brick(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                     _Brick);
-
-//------------------------------------------------------------------------
-// walk1_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator __brick_walk1_n(_ForwardIterator, _Size, _Function,
-                                 /*_IsVectorTag=*/std::false_type);
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Function>
-_RandomAccessIterator __brick_walk1_n(_RandomAccessIterator, _DifferenceType, _Function,
-                                      /*vectorTag=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__pattern_walk1_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _Size, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function>
-_RandomAccessIterator
-__pattern_walk1_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
-_ForwardIterator
-__pattern_walk_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _Size, _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
-_RandomAccessIterator
-__pattern_walk_brick_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, _Brick);
-
-//------------------------------------------------------------------------
-// walk2 (pseudo)
-//
-// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2 __brick_walk2(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Function,
-                                /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2 __brick_walk2(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _Function,
-                                     /*vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
-_ForwardIterator2 __brick_walk2_n(_ForwardIterator1, _Size, _ForwardIterator2, _Function,
-                                  /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2 __brick_walk2_n(_RandomAccessIterator1, _Size, _RandomAccessIterator2, _Function,
-                                       /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__pattern_walk2(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Function>
-_RandomAccessIterator2
-__pattern_walk2(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Function>
-_ForwardIterator2
-__pattern_walk2_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _Size, _ForwardIterator2, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__pattern_walk2_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _Size, _RandomAccessIterator2,
-                  _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                      _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                      _RandomAccessIterator2, _Brick);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _Size, _ForwardIterator2, _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _Size,
-                        _RandomAccessIterator2, _Brick);
-
-//------------------------------------------------------------------------
-// walk3 (pseudo)
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
-_ForwardIterator3 __brick_walk3(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator3, _Function,
-                                /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3 __brick_walk3(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                     _RandomAccessIterator3, _Function,
-                                     /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
-          class _Function>
-_ForwardIterator3
-__pattern_walk3(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator3,
-                _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3
-__pattern_walk3(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _RandomAccessIterator3, _Function);
-
-//------------------------------------------------------------------------
-// equal
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool __brick_equal(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _BinaryPredicate,
-                   /* is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool __brick_equal(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _BinaryPredicate,
-                   /* is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _BinaryPredicate);
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool __brick_equal(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2, _BinaryPredicate,
-                   /* is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool __brick_equal(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                   _BinaryPredicate, /* is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate);
-
-//------------------------------------------------------------------------
-// find_if
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Predicate>
-_ForwardIterator __brick_find_if(_ForwardIterator, _ForwardIterator, _Predicate,
-                                 /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator __brick_find_if(_RandomAccessIterator, _RandomAccessIterator, _Predicate,
-                                      /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-_ForwardIterator
-__pattern_find_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Predicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator
-__pattern_find_if(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                  _Predicate);
-
-//------------------------------------------------------------------------
-// find_end
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1 __brick_find_end(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                   _BinaryPredicate,
-                                   /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1 __brick_find_end(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                        _RandomAccessIterator2, _BinaryPredicate,
-                                        /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_end(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                   _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_end(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                   _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// find_first_of
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1 __brick_find_first_of(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                        _BinaryPredicate,
-                                        /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1 __brick_find_first_of(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                             _RandomAccessIterator2, _BinaryPredicate,
-                                             /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_first_of(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                        _ForwardIterator2, _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_first_of(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                        _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// search
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1 __brick_search(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                 _BinaryPredicate,
-                                 /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1 __brick_search(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                      _RandomAccessIterator2, _BinaryPredicate,
-                                      /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_search(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                 _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_search(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                 _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// search_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__brick_search_n(_ForwardIterator, _ForwardIterator, _Size, const _Tp&, _BinaryPredicate,
-                 /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_search_n(_RandomAccessIterator, _RandomAccessIterator, _Size, const _Tp&, _BinaryPredicate,
-                 /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__pattern_search_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Size, const _Tp&,
-                   _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp,
-          class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_search_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Size,
-                   const _Tp&, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// copy_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _OutputIterator>
-_OutputIterator __brick_copy_n(_ForwardIterator, _Size, _OutputIterator,
-                               /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Size, class _OutputIterator>
-_OutputIterator __brick_copy_n(_RandomAccessIterator, _Size, _OutputIterator,
-                               /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_copy(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                             /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_copy(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                             /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// move
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_move(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                             /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_move(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                             /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// swap_ranges
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_swap_ranges(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                                    /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_swap_ranges(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                    /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// copy_if
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator __brick_copy_if(_ForwardIterator, _ForwardIterator, _OutputIterator, _UnaryPredicate,
-                                /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator __brick_copy_if(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator, _UnaryPredicate,
-                                /*vector=*/std::true_type) noexcept;
-
-template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_ForwardIterator, _ForwardIterator, bool* __restrict, _UnaryPredicate,
-                    /*vector=*/std::false_type) noexcept;
-template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_RandomAccessIterator, _RandomAccessIterator, bool* __restrict, _UnaryPredicate,
-                    /*vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator, class _OutputIterator>
-void
-__brick_copy_by_mask(_ForwardIterator, _ForwardIterator, _OutputIterator, bool*,
-                     /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-void
-__brick_copy_by_mask(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator, bool* __restrict,
-                     /*vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
-void
-__brick_partition_by_mask(_ForwardIterator, _ForwardIterator, _OutputIterator1, _OutputIterator2, bool*,
-                          /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2>
-void
-__brick_partition_by_mask(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator1, _OutputIterator2, bool*,
-                          /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__pattern_copy_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                  _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryPredicate>
-_OutputIterator
-__pattern_copy_if(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                  _OutputIterator, _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// count
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-    __brick_count(_RandomAccessIterator, _RandomAccessIterator, _Predicate,
-                  /* is_vector = */ std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-    __brick_count(_ForwardIterator, _ForwardIterator, _Predicate,
-                  /* is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-__pattern_count(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Predicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-__pattern_count(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                _Predicate);
-
-//------------------------------------------------------------------------
-// unique
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator __brick_unique(_ForwardIterator, _ForwardIterator, _BinaryPredicate,
-                                /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator __brick_unique(_RandomAccessIterator, _RandomAccessIterator, _BinaryPredicate,
-                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_unique(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_unique(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                 _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// unique_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
-OutputIterator __brick_unique_copy(_ForwardIterator, _ForwardIterator, OutputIterator, _BinaryPredicate,
-                                   /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator __brick_unique_copy(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator, _BinaryPredicate,
-                                    /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator
-__pattern_unique_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                      _BinaryPredicate) noexcept;
-
-template <class _ExecutionPolicy, class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator, _RandomAccessIterator, bool* __restrict, _BinaryPredicate,
-                    /*vector=*/std::false_type) noexcept;
-
-template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator, _RandomAccessIterator, bool* __restrict, _BinaryPredicate,
-                    /*vector=*/std::true_type) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _BinaryPredicate>
-_OutputIterator
-__pattern_unique_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _OutputIterator, _BinaryPredicate);
-
-//------------------------------------------------------------------------
-// reverse
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator>
-void __brick_reverse(_BidirectionalIterator, _BidirectionalIterator,
-                     /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator>
-void __brick_reverse(_RandomAccessIterator, _RandomAccessIterator,
-                     /*__is_vector=*/std::true_type) noexcept;
-
-template <class _BidirectionalIterator>
-void __brick_reverse(_BidirectionalIterator, _BidirectionalIterator, _BidirectionalIterator,
-                     /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator>
-void __brick_reverse(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator>
-void
-__pattern_reverse(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-void
-__pattern_reverse(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator);
-
-//------------------------------------------------------------------------
-// reverse_copy
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator __brick_reverse_copy(_BidirectionalIterator, _BidirectionalIterator, _OutputIterator,
-                                     /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_reverse_copy(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator
-__pattern_reverse_copy(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator,
-                       _OutputIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator
-__pattern_reverse_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                       _OutputIterator);
-
-//------------------------------------------------------------------------
-// rotate
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator>
-_ForwardIterator __brick_rotate(_ForwardIterator, _ForwardIterator, _ForwardIterator,
-                                /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator>
-_RandomAccessIterator __brick_rotate(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator>
-_ForwardIterator
-__pattern_rotate(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _ForwardIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-_RandomAccessIterator
-__pattern_rotate(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                 _RandomAccessIterator);
-
-//------------------------------------------------------------------------
-// rotate_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_rotate_copy(_ForwardIterator, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                                    /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_rotate_copy(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                                    _OutputIterator,
-                                    /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__pattern_rotate_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _ForwardIterator,
-                      _OutputIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator
-__pattern_rotate_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _RandomAccessIterator, _OutputIterator);
-
-//------------------------------------------------------------------------
-// is_partitioned
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-bool __brick_is_partitioned(_ForwardIterator, _ForwardIterator, _UnaryPredicate,
-                            /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-bool __brick_is_partitioned(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                            /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// partition
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator __brick_partition(_ForwardIterator, _ForwardIterator, _UnaryPredicate,
-                                   /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator __brick_partition(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                                        /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_partition(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_partition(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                    _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// stable_partition
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator __brick_stable_partition(_BidirectionalIterator, _BidirectionalIterator, _UnaryPredicate,
-                                                /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator __brick_stable_partition(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                                               /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator
-__pattern_stable_partition(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator,
-                           _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_stable_partition(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                           _UnaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// partition_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-    __brick_partition_copy(_ForwardIterator, _ForwardIterator, _OutputIterator1, _OutputIterator2, _UnaryPredicate,
-                           /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2> __brick_partition_copy(_RandomAccessIterator, _RandomAccessIterator,
-                                                                     _OutputIterator1, _OutputIterator2,
-                                                                     _UnaryPredicate,
-                                                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
-          class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__pattern_partition_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator1,
-                         _OutputIterator2, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator1,
-          class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__pattern_partition_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _OutputIterator1, _OutputIterator2, _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsMoveConstructible>
-void
-__pattern_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare,
-               _IsMoveConstructible) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_sort(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare,
-               /*is_move_constructible=*/std::true_type);
-
-//------------------------------------------------------------------------
-// stable_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _Compare);
-
-//------------------------------------------------------------------------
-// partial_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                       _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                       _RandomAccessIterator, _Compare);
-
-//------------------------------------------------------------------------
-// partial_sort_copy
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-_RandomAccessIterator2
-__pattern_partial_sort_copy(_Tag, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                            _RandomAccessIterator2, _RandomAccessIterator2, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-_RandomAccessIterator2
-__pattern_partial_sort_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                            _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2, _Compare);
-
-//------------------------------------------------------------------------
-// adjacent_find
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_adjacent_find(_RandomAccessIterator, _RandomAccessIterator, _BinaryPredicate,
-                      /* IsVector = */ std::true_type, bool) noexcept;
-
-template <class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__brick_adjacent_find(_ForwardIterator, _ForwardIterator, _BinaryPredicate,
-                      /* IsVector = */ std::false_type, bool) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_adjacent_find(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _BinaryPredicate, bool) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_adjacent_find(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                        _BinaryPredicate, bool);
-
-//------------------------------------------------------------------------
-// nth_element
-//------------------------------------------------------------------------
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                      _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _RandomAccessIterator, _Compare) noexcept;
-
-//------------------------------------------------------------------------
-// fill, fill_n
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Tp>
-void
-__brick_fill(_RandomAccessIterator, _RandomAccessIterator, const _Tp&,
-             /* __is_vector = */ std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Tp>
-void
-__brick_fill(_ForwardIterator, _ForwardIterator, const _Tp&,
-             /* __is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-void
-__pattern_fill(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, const _Tp&) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp>
-_RandomAccessIterator
-__pattern_fill(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, const _Tp&);
-
-template <class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__brick_fill_n(_RandomAccessIterator, _Size, const _Tp&,
-               /* __is_vector = */ std::true_type) noexcept;
-
-template <class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__brick_fill_n(_OutputIterator, _Size, const _Tp&,
-               /* __is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__pattern_fill_n(_Tag, _ExecutionPolicy&&, _OutputIterator, _Size, const _Tp&) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__pattern_fill_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, const _Tp&);
-
-//------------------------------------------------------------------------
-// generate, generate_n
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Generator>
-void __brick_generate(_RandomAccessIterator, _RandomAccessIterator, _Generator,
-                      /* is_vector = */ std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Generator>
-void __brick_generate(_ForwardIterator, _ForwardIterator, _Generator,
-                      /* is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-void
-__pattern_generate(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Generator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Generator>
-_RandomAccessIterator
-__pattern_generate(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                   _Generator);
-
-template <class _RandomAccessIterator, class Size, class _Generator>
-_RandomAccessIterator __brick_generate_n(_RandomAccessIterator, Size, _Generator,
-                                         /* is_vector = */ std::true_type) noexcept;
-
-template <class OutputIterator, class Size, class _Generator>
-OutputIterator __brick_generate_n(OutputIterator, Size, _Generator,
-                                  /* is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class OutputIterator, class Size, class _Generator>
-OutputIterator
-__pattern_generate_n(_Tag, _ExecutionPolicy&&, OutputIterator, Size, _Generator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class Size, class _Generator>
-_RandomAccessIterator
-__pattern_generate_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, Size, _Generator);
-
-//------------------------------------------------------------------------
-// remove
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator __brick_remove_if(_ForwardIterator, _ForwardIterator, _UnaryPredicate,
-                                   /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator __brick_remove_if(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                                        /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_remove_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_remove_if(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                    _UnaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// merge
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_merge(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                              _OutputIterator, _Compare,
-                              /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_merge(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                              _RandomAccessIterator2, _OutputIterator, _Compare,
-                              /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_merge(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_merge(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// inplace_merge
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _Compare>
-void __brick_inplace_merge(_BidirectionalIterator, _BidirectionalIterator, _BidirectionalIterator, _Compare,
-                           /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Compare>
-void __brick_inplace_merge(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator, _Compare,
-                           /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-void
-__pattern_inplace_merge(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator,
-                        _BidirectionalIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_inplace_merge(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                        _RandomAccessIterator, _Compare);
-
-//------------------------------------------------------------------------
-// includes
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_includes(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                   _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_includes(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                   _RandomAccessIterator2, _RandomAccessIterator2, _Compare);
-
-//------------------------------------------------------------------------
-// set_union
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_union(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                  _OutputIterator, _Compare,
-                                  /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_union(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                  _RandomAccessIterator2, _OutputIterator, _Compare,
-                                  /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_union(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                    _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_union(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                    _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// set_intersection
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_intersection(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                         _OutputIterator, _Compare,
-                                         /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_intersection(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                         _RandomAccessIterator2, _OutputIterator, _Compare,
-                                         /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_intersection(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                           _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_intersection(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                           _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator,
-                           _Compare);
-
-//------------------------------------------------------------------------
-// set_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_difference(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                       _OutputIterator, _Compare,
-                                       /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_difference(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                       _RandomAccessIterator2, _OutputIterator, _Compare,
-                                       /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                         _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_difference(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                         _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// set_symmetric_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_symmetric_difference(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                                                 _ForwardIterator2, _OutputIterator, _Compare,
-                                                 /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_symmetric_difference(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                                 _RandomAccessIterator2, _OutputIterator, _Compare,
-                                                 /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_symmetric_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                                   _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_symmetric_difference(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                                   _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                                   _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// is_heap_until
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator __brick_is_heap_until(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                                            /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator __brick_is_heap_until(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                                            /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                        _Compare) noexcept;
-
-//------------------------------------------------------------------------
-// min_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-_ForwardIterator __brick_min_element(_ForwardIterator, _ForwardIterator, _Compare,
-                                     /* __is_vector = */ std::false_type) noexcept;
-
-template <typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator __brick_min_element(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                                          /* __is_vector = */ std::true_type) noexcept;
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__pattern_min_element(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Compare) noexcept;
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator
-__pattern_min_element(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _Compare);
-
-//------------------------------------------------------------------------
-// minmax_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator> __brick_minmax_element(_ForwardIterator, _ForwardIterator, _Compare,
-                                                                     /* __is_vector = */ std::false_type) noexcept;
-
-template <typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-    __brick_minmax_element(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                           /* __is_vector = */ std::true_type) noexcept;
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__pattern_minmax_element(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Compare) noexcept;
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-__pattern_minmax_element(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _Compare);
-
-//------------------------------------------------------------------------
-// mismatch
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2> __brick_mismatch(_ForwardIterator1, _ForwardIterator1,
-                                                                 _ForwardIterator2, _ForwardIterator2, _Predicate,
-                                                                 /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-    __brick_mismatch(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                     _Predicate,
-                     /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__pattern_mismatch(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                   _Predicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-__pattern_mismatch(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                   _RandomAccessIterator2, _RandomAccessIterator2, _Predicate) noexcept;
-
-//------------------------------------------------------------------------
-// lexicographical_compare
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool __brick_lexicographical_compare(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                     _Compare,
-                                     /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Compare>
-bool __brick_lexicographical_compare(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                     _RandomAccessIterator2, _Compare,
-                                     /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_lexicographical_compare(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                                  _ForwardIterator2, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_lexicographical_compare(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                                  _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                                  _Compare) noexcept;
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_ALGORITHM_FWD_H */
diff --git a/libcxx/include/pstl/internal/algorithm_impl.h b/libcxx/include/pstl/internal/algorithm_impl.h
deleted file mode 100644
index 2b505d94611f6..0000000000000
--- a/libcxx/include/pstl/internal/algorithm_impl.h
+++ /dev/null
@@ -1,3819 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_ALGORITHM_IMPL_H
-#define _PSTL_ALGORITHM_IMPL_H
-
-#include <iterator>
-#include <type_traits>
-#include <utility>
-#include <functional>
-#include <algorithm>
-
-#include "execution_impl.h"
-#include "memory_impl.h"
-#include "parallel_backend.h"
-#include "parallel_backend_utils.h"
-#include "parallel_impl.h"
-#include "pstl_config.h"
-#include "unseq_backend_simd.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// any_of
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Pred>
-bool
-__brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
-               /*__is_vector=*/std::false_type) noexcept
-{
-    return std::any_of(__first, __last, __pred);
-};
-
-template <class _RandomAccessIterator, class _Pred>
-bool
-__brick_any_of(const _RandomAccessIterator __first, const _RandomAccessIterator __last, _Pred __pred,
-               /*__is_vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_or(__first, __last - __first, __pred);
-};
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Pred>
-bool
-__pattern_any_of(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) noexcept
-{
-    return __internal::__brick_any_of(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Pred>
-bool
-__pattern_any_of(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _RandomAccessIterator __last, _Pred __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __internal::__parallel_or(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                             [__pred](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                                             { return __internal::__brick_any_of(__i, __j, __pred, _IsVector{}); });
-        });
-}
-
-// [alg.foreach]
-// for_each_n with no policy
-
-template <class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f)
-{
-    for (; __n > 0; ++__first, --__n)
-        __f(__first);
-    return __first;
-}
-
-//------------------------------------------------------------------------
-// walk1 (pseudo)
-//
-// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Function>
-void
-__brick_walk1(_ForwardIterator __first, _ForwardIterator __last, _Function __f, /*vector=*/std::false_type) noexcept
-{
-    std::for_each(__first, __last, __f);
-}
-
-template <class _RandomAccessIterator, class _Function>
-void
-__brick_walk1(_RandomAccessIterator __first, _RandomAccessIterator __last, _Function __f,
-              /*vector=*/std::true_type) noexcept
-{
-    __unseq_backend::__simd_walk_1(__first, __last - __first, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Function>
-void
-__pattern_walk1(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __f) noexcept
-{
-    __internal::__brick_walk1(__first, __last, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Function>
-void
-__pattern_walk1(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                _RandomAccessIterator __last, _Function __f)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [__f](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                                          { __internal::__brick_walk1(__i, __j, __f, _IsVector{}); });
-        });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Brick>
-void
-__pattern_walk_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                     _Brick __brick) noexcept
-{
-    __brick(__first, __last);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Brick>
-void
-__pattern_walk_brick(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                     _RandomAccessIterator __last, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                                          { __brick(__i, __j); });
-        });
-}
-
-//------------------------------------------------------------------------
-// walk1_n
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type)
-{
-    return __internal::__for_each_n_it_serial(__first, __n,
-                                              [&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Function>
-_RandomAccessIterator
-__brick_walk1_n(_RandomAccessIterator __first, _DifferenceType __n, _Function __f,
-                /*vectorTag=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_1(__first, __n, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__pattern_walk1_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f) noexcept
-{
-    return __internal::__brick_walk1_n(__first, __n, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function>
-_RandomAccessIterator
-__pattern_walk1_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n,
-                  _Function __f)
-{
-    __internal::__pattern_walk1(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f);
-
-    return __first + __n;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
-_ForwardIterator
-__pattern_walk_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Brick __brick) noexcept
-{
-    return __brick(__first, __n);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
-_RandomAccessIterator
-__pattern_walk_brick_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                       _Size __n, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __first + __n,
-                [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); });
-            return __first + __n;
-        });
-}
-
-//------------------------------------------------------------------------
-// walk2 (pseudo)
-//
-// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
-              /*vector=*/std::false_type) noexcept
-{
-    for (; __first1 != __last1; ++__first1, ++__first2)
-        __f(*__first1, *__first2);
-    return __first2;
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__brick_walk2(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _Function __f,
-              /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_2(__first1, __last1 - __first1, __first2, __f);
-}
-
-template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
-                /*vector=*/std::false_type) noexcept
-{
-    for (; __n > 0; --__n, ++__first1, ++__first2)
-        __f(*__first1, *__first2);
-    return __first2;
-}
-
-template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__brick_walk2_n(_RandomAccessIterator1 __first1, _Size __n, _RandomAccessIterator2 __first2, _Function __f,
-                /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_2(__first1, __n, __first2, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__pattern_walk2(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _Function __f) noexcept
-{
-    return __internal::__brick_walk2(__first1, __last1, __first2, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Function>
-_RandomAccessIterator2
-__pattern_walk2(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Function __f)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__f, __first1, __first2](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { __internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, _IsVector{}); });
-            return __first2 + (__last1 - __first1);
-        });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Function>
-_ForwardIterator2
-__pattern_walk2_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
-                  _Function __f) noexcept
-{
-    return __internal::__brick_walk2_n(__first1, __n, __first2, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__pattern_walk2_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                  _Size __n, _RandomAccessIterator2 __first2, _Function __f)
-{
-    return __internal::__pattern_walk2(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-                                       __first2, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                      _ForwardIterator2 __first2, _Brick __brick) noexcept
-{
-    return __brick(__first1, __last1, __first2);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                      _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { __brick(__i, __j, __first2 + (__i - __first1)); });
-            return __first2 + (__last1 - __first1);
-        });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
-                        _Brick __brick) noexcept
-{
-    return __brick(__first1, __n, __first2);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                        _Size __n, _RandomAccessIterator2 __first2, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-                [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { __brick(__i, __j - __i, __first2 + (__i - __first1)); });
-            return __first2 + __n;
-        });
-}
-
-//------------------------------------------------------------------------
-// walk3 (pseudo)
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
-_ForwardIterator3
-__brick_walk3(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-              _ForwardIterator3 __first3, _Function __f, /*vector=*/std::false_type) noexcept
-{
-    for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
-        __f(*__first1, *__first2, *__first3);
-    return __first3;
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3
-__brick_walk3(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _RandomAccessIterator3 __first3, _Function __f, /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_3(__first1, __last1 - __first1, __first2, __first3, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
-          class _Function>
-_ForwardIterator3
-__pattern_walk3(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _ForwardIterator3 __first3, _Function __f) noexcept
-{
-    return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3
-__pattern_walk3(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator3 __first3,
-                _Function __f)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__f, __first1, __first2, __first3](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
-                    __internal::__brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f,
-                                              _IsVector{});
-                });
-            return __first3 + (__last1 - __first1);
-        });
-}
-
-//------------------------------------------------------------------------
-// equal
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-              _ForwardIterator2 __last2, _BinaryPredicate __p, /* IsVector = */ std::false_type) noexcept
-{
-    return std::equal(__first1, __last1, __first2, __last2, __p);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _RandomAccessIterator2 __last2, _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
-{
-    if (__last1 - __first1 != __last2 - __first2)
-        return false;
-
-    return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _ForwardIterator2 __last2, _BinaryPredicate __p) noexcept
-{
-    return __internal::__brick_equal(__first1, __last1, __first2, __last2, __p, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                _BinaryPredicate __p)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last1 - __first1 != __last2 - __first2)
-        return false;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return !__internal::__parallel_or(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __p](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                {
-                    return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1),
-                                                      __first2 + (__j - __first1), __p, _IsVector{});
-                });
-        });
-}
-
-//------------------------------------------------------------------------
-// equal version for sequences with equal length
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __p,
-              /* IsVector = */ std::false_type) noexcept
-{
-    return std::equal(__first1, __last1, __first2, __p);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _BinaryPredicate __p) noexcept
-{
-    return __internal::__brick_equal(__first1, __last1, __first2, __p, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _BinaryPredicate __p)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return !__internal::__parallel_or(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __p](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, _IsVector{}); });
-        });
-}
-
-//------------------------------------------------------------------------
-// find_if
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Predicate>
-_ForwardIterator
-__brick_find_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
-                /*is_vector=*/std::false_type) noexcept
-{
-    return std::find_if(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator
-__brick_find_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
-                /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    return __unseq_backend::__simd_first(
-        __first, _SizeType(0), __last - __first,
-        [&__pred](_RandomAccessIterator __it, _SizeType __i) { return __pred(__it[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-_ForwardIterator
-__pattern_find_if(_Tag __tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                  _Predicate __pred) noexcept
-{
-    return __internal::__brick_find_if(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator
-__pattern_find_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                  _RandomAccessIterator __last, _Predicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __internal::__parallel_find(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__pred](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                { return __internal::__brick_find_if(__i, __j, __pred, _IsVector{}); },
-                std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(),
-                /*is_first=*/true);
-        });
-}
-
-//------------------------------------------------------------------------
-// find_end
-//------------------------------------------------------------------------
-
-// find the first occurrence of the subsequence [s_first, s_last)
-//   or the  last occurrence of the subsequence in the range [first, last)
-// b_first determines what occurrence we want to find (first or last)
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate, class _IsVector>
-_RandomAccessIterator1
-__find_subrange(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator1 __global_last,
-                _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last, _BinaryPredicate __pred,
-                bool __b_first, _IsVector __is_vector) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _ValueType;
-    auto __n2 = __s_last - __s_first;
-    if (__n2 < 1)
-    {
-        return __b_first ? __first : __last;
-    }
-
-    auto __n1 = __global_last - __first;
-    if (__n1 < __n2)
-    {
-        return __last;
-    }
-
-    auto __cur = __last;
-    while (__first != __last && (__global_last - __first >= __n2))
-    {
-        // find position of *s_first in [first, last) (it can be start of subsequence)
-        __first = __internal::__brick_find_if(
-            __first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector);
-
-        // if position that was found previously is the start of subsequence
-        // then we can exit the loop (b_first == true) or keep the position
-        // (b_first == false)
-        if (__first != __last && (__global_last - __first >= __n2) &&
-            __internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector))
-        {
-            if (__b_first)
-            {
-                return __first;
-            }
-            else
-            {
-                __cur = __first;
-            }
-        }
-        else if (__first == __last)
-        {
-            break;
-        }
-        else
-        {
-        }
-
-        // in case of b_first == false we try to find new start position
-        // for the next subsequence
-        ++__first;
-    }
-    return __cur;
-}
-
-template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate, class _IsVector>
-_RandomAccessIterator
-__find_subrange(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __global_last,
-                _Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector) noexcept
-{
-    if (static_cast<_Size>(__global_last - __first) < __count || __count < 1)
-    {
-        return __last; // According to the standard last shall be returned when count < 1
-    }
-
-    auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred);
-    while (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count))
-    {
-        __first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector);
-
-        // check that all of elements in [first+1, first+count) equal to value
-        if (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count) &&
-            !__internal::__brick_any_of(__first + 1, __first + __count, std::not_fn(__unary_pred), __is_vector))
-        {
-            return __first;
-        }
-        else if (__first == __last)
-        {
-            break;
-        }
-        else
-        {
-            ++__first;
-        }
-    }
-    return __last;
-}
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-                 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
-{
-    return std::find_end(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_find_end(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-                 _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
-{
-    return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, false, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_end(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                   _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_end(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                   _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last,
-                   _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last - __first == __s_last - __s_first)
-    {
-        const bool __res = __internal::__pattern_equal(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                       __s_first, __pred);
-        return __res ? __first : __last;
-    }
-    else
-    {
-        return __internal::__except_handler(
-            [&]()
-            {
-                return __internal::__parallel_find(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    [__last, __s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
-                        return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false,
-                                                           _IsVector{});
-                    },
-                    std::greater<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
-                    /*is_first=*/false);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// find_first_of
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-                      _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
-{
-    return std::find_first_of(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_find_first_of(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-                      _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_find_first_of(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_first_of(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                        _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred,
-                                             typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_first_of(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                        _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-                        _RandomAccessIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __internal::__parallel_find(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, _IsVector{}); },
-                std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
-        });
-}
-
-//------------------------------------------------------------------------
-// search
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_search(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-               _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
-{
-    return std::search(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_search(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-               _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-    return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_search(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_search(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_search(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                 _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last,
-                 _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last - __first == __s_last - __s_first)
-    {
-        const bool __res = __internal::__pattern_equal(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                       __s_first, __pred);
-        return __res ? __first : __last;
-    }
-    else
-    {
-        return __internal::__except_handler(
-            [&]()
-            {
-                return __internal::__parallel_find(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    [__last, __s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
-                        return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true,
-                                                           _IsVector{});
-                    },
-                    std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
-                    /*is_first=*/true);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// search_n
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
-                 _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
-{
-    return std::search_n(__first, __last, __count, __value, __pred);
-}
-
-template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_search_n(_RandomAccessIterator __first, _RandomAccessIterator __last, _Size __count, const _Tp& __value,
-                 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-    return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__pattern_search_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-                   const _Tp& __value, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_search_n(__first, __last, __count, __value, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp,
-          class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_search_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                   _RandomAccessIterator __last, _Size __count, const _Tp& __value, _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (static_cast<_Size>(__last - __first) == __count)
-    {
-        const bool __result =
-            !__internal::__pattern_any_of(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [&__value, &__pred](const _Tp& __val) { return !__pred(__val, __value); });
-        return __result ? __first : __last;
-    }
-    else
-    {
-        return __internal::__except_handler(
-            [&__exec, __first, __last, __count, &__value, __pred]()
-            {
-                return __internal::__parallel_find(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    [__last, __count, &__value, __pred](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                    { return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, _IsVector{}); },
-                    std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(),
-                    /*is_first=*/true);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// copy_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _OutputIterator>
-_OutputIterator
-__brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::false_type) noexcept
-{
-    return std::copy_n(__first, __n, __result);
-}
-
-template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_copy_n(_RandomAccessIterator1 __first, _Size __n, _RandomAccessIterator2 __result,
-               /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_assign(
-        __first, __n, __result,
-        [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) { *__result = *__first; });
-}
-
-//------------------------------------------------------------------------
-// copy
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-             /*vector=*/std::false_type) noexcept
-{
-    return std::copy(__first, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-             /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_assign(
-        __first, __last - __first, __result,
-        [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) { *__result = *__first; });
-}
-
-//------------------------------------------------------------------------
-// move
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-             /*vector=*/std::false_type) noexcept
-{
-    return std::move(__first, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_move(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-             /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_assign(
-        __first, __last - __first, __result,
-        [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) { *__result = std::move(*__first); });
-}
-
-struct __brick_move_destroy
-{
-    template <typename _RandomAccessIterator1, typename _RandomAccessIterator2>
-    _RandomAccessIterator2
-    operator()(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-               /*vec*/ std::true_type) const
-    {
-        using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
-
-        return __unseq_backend::__simd_assign(__first, __last - __first, __result,
-                                              [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) {
-                                                  *__result = std::move(*__first);
-                                                  (*__first).~_IteratorValueType();
-                                              });
-    }
-
-    template <typename _RandomAccessIterator1, typename _RandomAccessIterator2>
-    _RandomAccessIterator2
-    operator()(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-               /*vec*/ std::false_type) const
-    {
-        using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
-
-        for (; __first != __last; ++__first, ++__result)
-        {
-            *__result = std::move(*__first);
-            (*__first).~_IteratorValueType();
-        }
-        return __result;
-    }
-};
-
-//------------------------------------------------------------------------
-// swap_ranges
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                    /*vector=*/std::false_type) noexcept
-{
-    return std::swap_ranges(__first, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_swap_ranges(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                    /*vector=*/std::true_type) noexcept
-{
-    using std::iter_swap;
-    return __unseq_backend::__simd_assign(__first, __last - __first, __result,
-                                          iter_swap<_RandomAccessIterator1, _RandomAccessIterator2>);
-}
-
-//------------------------------------------------------------------------
-// copy_if
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
-                /*vector=*/std::false_type) noexcept
-{
-    return std::copy_if(__first, __last, __result, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _UnaryPredicate>
-_RandomAccessIterator2
-__brick_copy_if(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                _UnaryPredicate __pred,
-                /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_copy_if(__first, __last - __first, __result, __pred);
-#else
-    return std::copy_if(__first, __last, __result, __pred);
-#endif
-}
-
-// TODO: Try to use transform_reduce for combining __brick_copy_if_phase1 on IsVector.
-template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_ForwardIterator __first, _ForwardIterator __last, bool* __restrict __mask, _UnaryPredicate __pred,
-                    /*vector=*/std::false_type) noexcept
-{
-    auto __count_true = _DifferenceType(0);
-    auto __size = __last - __first;
-
-    static_assert(__are_random_access_iterators<_ForwardIterator>::value,
-                  "Pattern-brick error. Should be a random access iterator.");
-
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        *__mask = __pred(*__first);
-        if (*__mask)
-        {
-            ++__count_true;
-        }
-    }
-    return std::make_pair(__count_true, __size - __count_true);
-}
-
-template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __mask, _UnaryPredicate __pred,
-                    /*vector=*/std::true_type) noexcept
-{
-    auto __result = __unseq_backend::__simd_calc_mask_1(__first, __last - __first, __mask, __pred);
-    return std::make_pair(__result, (__last - __first) - __result);
-}
-
-template <class _ForwardIterator, class _OutputIterator, class _Assigner>
-void
-__brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, bool* __mask,
-                     _Assigner __assigner, /*vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        if (*__mask)
-        {
-            __assigner(__first, __result);
-            ++__result;
-        }
-    }
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Assigner>
-void
-__brick_copy_by_mask(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                     bool* __restrict __mask, _Assigner __assigner, /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    __unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner);
-#else
-    __internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type());
-#endif
-}
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
-void
-__brick_partition_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
-                          _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        if (*__mask)
-        {
-            *__out_true = *__first;
-            ++__out_true;
-        }
-        else
-        {
-            *__out_false = *__first;
-            ++__out_false;
-        }
-    }
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3>
-void
-__brick_partition_by_mask(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last,
-                          _RandomAccessIterator2 __out_true, _RandomAccessIterator3 __out_false, bool* __mask,
-                          /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    __unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask);
-#else
-    __internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type());
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__pattern_copy_if(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                  _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_copy_if(__first, __last, __result, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _UnaryPredicate>
-_RandomAccessIterator2
-__pattern_copy_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                  _RandomAccessIterator1 __last, _RandomAccessIterator2 __result, _UnaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    const _DifferenceType __n = __last - __first;
-    if (_DifferenceType(1) < __n)
-    {
-        __par_backend::__buffer<bool> __mask_buf(__n);
-        return __internal::__except_handler(
-            [&__exec, __n, __first, __result, __pred, &__mask_buf]()
-            {
-                bool* __mask = __mask_buf.get();
-                _DifferenceType __m{};
-                __par_backend::__parallel_strict_scan(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
-                    [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
-                        return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
-                                                                                __mask + __i, __pred, _IsVector{})
-                            .first;
-                    },
-                    std::plus<_DifferenceType>(),                                                // Combine
-                    [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
-                        __internal::__brick_copy_by_mask(
-                            __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
-                            [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = *__x; }, _IsVector{});
-                    },
-                    [&__m](_DifferenceType __total) { __m = __total; });
-                return __result + __m;
-            });
-    }
-    // trivial sequence - use serial algorithm
-    return __internal::__brick_copy_if(__first, __last, __result, __pred, _IsVector{});
-}
-
-//------------------------------------------------------------------------
-// count
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-__brick_count(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
-              /* is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_count(__first, __last - __first, __pred);
-}
-
-template <class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-__brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
-              /* is_vector = */ std::false_type) noexcept
-{
-    return std::count_if(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-__pattern_count(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) noexcept
-{
-    return __internal::__brick_count(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-__pattern_count(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                _RandomAccessIterator __last, _Predicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, _SizeType(0),
-                [__pred](_RandomAccessIterator __begin, _RandomAccessIterator __end, _SizeType __value) -> _SizeType
-                { return __value + __internal::__brick_count(__begin, __end, __pred, _IsVector{}); },
-                std::plus<_SizeType>());
-        });
-}
-
-//------------------------------------------------------------------------
-// unique
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_unique(_RandomAccessIterator __first, _RandomAccessIterator __last, _BinaryPredicate __pred,
-               /*is_vector=*/std::false_type) noexcept
-{
-    return std::unique(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_unique(_RandomAccessIterator __first, _RandomAccessIterator __last, _BinaryPredicate __pred,
-               /*is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::unique(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_unique(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                 _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_unique(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-// That function is shared between two algorithms - remove_if (__pattern_remove_if) and unique (pattern unique). But a mask calculation is different.
-// So, a caller passes _CalcMask brick into remove_elements.
-template <class _IsVector, class _ExecutionPolicy, class _ForwardIterator, class _CalcMask>
-_ForwardIterator
-__remove_elements(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _ForwardIterator __first,
-                  _ForwardIterator __last, _CalcMask __calc_mask)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType;
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
-    _DifferenceType __n = __last - __first;
-    __par_backend::__buffer<bool> __mask_buf(__n);
-    // 1. find a first iterator that should be removed
-    return __internal::__except_handler([&]() {
-        bool* __mask = __mask_buf.get();
-        _DifferenceType __min = __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), _DifferenceType(0), __n, __n,
-            [__first, __mask, &__calc_mask](_DifferenceType __i, _DifferenceType __j,
-                                            _DifferenceType __local_min) -> _DifferenceType
-            {
-                // Create mask
-                __calc_mask(__mask + __i, __mask + __j, __first + __i);
-
-                // if minimum was found in a previous range we shouldn't do anymore
-                if (__local_min < __i)
-                {
-                    return __local_min;
-                }
-                // find first iterator that should be removed
-                bool* __result = __internal::__brick_find_if(
-                    __mask + __i, __mask + __j, [](bool __val) { return !__val; }, _IsVector{});
-                if (__result - __mask == __j)
-                {
-                    return __local_min;
-                }
-                return std::min(__local_min, _DifferenceType(__result - __mask));
-            },
-            [](_DifferenceType __local_min1, _DifferenceType __local_min2) -> _DifferenceType
-            { return std::min(__local_min1, __local_min2); });
-
-        // No elements to remove - exit
-        if (__min == __n)
-        {
-            return __last;
-        }
-        __n -= __min;
-        __first += __min;
-
-        __par_backend::__buffer<_Tp> __buf(__n);
-        _Tp* __result = __buf.get();
-        __mask += __min;
-        _DifferenceType __m{};
-        // 2. Elements that doesn't satisfy pred are moved to result
-        __par_backend::__parallel_strict_scan(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
-            [__mask](_DifferenceType __i, _DifferenceType __len)
-            {
-                return __internal::__brick_count(
-                    __mask + __i, __mask + __i + __len, [](bool __val) { return __val; }, _IsVector{});
-            },
-            std::plus<_DifferenceType>(),
-            [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial)
-            {
-                __internal::__brick_copy_by_mask(
-                    __first + __i, __first + __i + __len, __result + __initial, __mask + __i,
-                    [](_ForwardIterator __x, _Tp* __z)
-                    {
-                        __internal::__invoke_if_else(
-                            std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
-                            [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
-                    },
-                    _IsVector{});
-            },
-            [&__m](_DifferenceType __total) { __m = __total; });
-
-        // 3. Elements from result are moved to [first, last)
-        __par_backend::__parallel_for(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
-            [__result, __first](_Tp* __i, _Tp* __j)
-            {
-                __invoke_if_else(
-                    std::is_trivial<_Tp>(), [&]() { __brick_move(__i, __j, __first + (__i - __result), _IsVector{}); },
-                    [&]() { __brick_move_destroy()(__i, __j, __first + (__i - __result), _IsVector{}); });
-            });
-        return __first + __m;
-    });
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_unique(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _RandomAccessIterator __last, _BinaryPredicate __pred) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    if (__first == __last)
-    {
-        return __last;
-    }
-    if (__first + 1 == __last || __first + 2 == __last)
-    {
-        // Trivial sequence - use serial algorithm
-        return __internal::__brick_unique(__first, __last, __pred, _IsVector{});
-    }
-    return __internal::__remove_elements(
-        __tag, std::forward<_ExecutionPolicy>(__exec), ++__first, __last,
-        [&__pred](bool* __b, bool* __e, _RandomAccessIterator __it)
-        {
-            __internal::__brick_walk3(
-                __b, __e, __it - 1, __it,
-                [&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); }, _IsVector{});
-        });
-}
-
-//------------------------------------------------------------------------
-// unique_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
-OutputIterator
-__brick_unique_copy(_ForwardIterator __first, _ForwardIterator __last, OutputIterator __result, _BinaryPredicate __pred,
-                    /*vector=*/std::false_type) noexcept
-{
-    return std::unique_copy(__first, __last, __result, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator2
-__brick_unique_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                    _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_unique_copy(__first, __last - __first, __result, __pred);
-#else
-    return std::unique_copy(__first, __last, __result, __pred);
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator
-__pattern_unique_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                      _OutputIterator __result, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_unique_copy(__first, __last, __result, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
-                    _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
-{
-    _DifferenceType __count = 0;
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        *__mask = !__pred(*__first, *(__first - 1));
-        __count += *__mask;
-    }
-    return __count;
-}
-
-template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
-                    _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_calc_mask_2(__first, __last - __first, __mask, __pred);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator2
-__pattern_unique_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                      _RandomAccessIterator1 __last, _RandomAccessIterator2 __result, _BinaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    const _DifferenceType __n = __last - __first;
-    if (_DifferenceType(2) < __n)
-    {
-        __par_backend::__buffer<bool> __mask_buf(__n);
-        if (_DifferenceType(2) < __n)
-        {
-            return __internal::__except_handler(
-                [&__exec, __n, __first, __result, __pred, &__mask_buf]()
-                {
-                    bool* __mask = __mask_buf.get();
-                    _DifferenceType __m{};
-                    __par_backend::__parallel_strict_scan(
-                        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
-                        [=](_DifferenceType __i, _DifferenceType __len) -> _DifferenceType { // Reduce
-                            _DifferenceType __extra = 0;
-                            if (__i == 0)
-                            {
-                                // Special boundary case
-                                __mask[__i] = true;
-                                if (--__len == 0)
-                                    return 1;
-                                ++__i;
-                                ++__extra;
-                            }
-                            return __internal::__brick_calc_mask_2<_DifferenceType>(
-                                       __first + __i, __first + (__i + __len), __mask + __i, __pred, _IsVector{}) +
-                                   __extra;
-                        },
-                        std::plus<_DifferenceType>(),                                                // Combine
-                        [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
-                            // Phase 2 is same as for __pattern_copy_if
-                            __internal::__brick_copy_by_mask(
-                                __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
-                                [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = *__x; },
-                                _IsVector{});
-                        },
-                        [&__m](_DifferenceType __total) { __m = __total; });
-                    return __result + __m;
-                });
-        }
-    }
-    // trivial sequence - use serial algorithm
-    return __internal::__brick_unique_copy(__first, __last, __result, __pred, _IsVector{});
-}
-
-//------------------------------------------------------------------------
-// reverse
-//------------------------------------------------------------------------
-template <class _BidirectionalIterator>
-void
-__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::false_type) noexcept
-{
-    std::reverse(__first, __last);
-}
-
-template <class _RandomAccessIterator>
-void
-__brick_reverse(_RandomAccessIterator __first, _RandomAccessIterator __last, /*__is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    const auto __n = (__last - __first) / 2;
-    __unseq_backend::__simd_walk_2(__first, __n, std::reverse_iterator<_RandomAccessIterator>(__last),
-                                   [](_ReferenceType __x, _ReferenceType __y) {
-                                       using std::swap;
-                                       swap(__x, __y);
-                                   });
-}
-
-// this brick is called in parallel version, so we can use iterator arithmetic
-template <class _BidirectionalIterator>
-void
-__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
-                /*is_vector=*/std::false_type) noexcept
-{
-    for (--__d_last; __first != __last; ++__first, --__d_last)
-    {
-        using std::iter_swap;
-        iter_swap(__first, __d_last);
-    }
-}
-
-// this brick is called in parallel version, so we can use iterator arithmetic
-template <class _RandomAccessIterator>
-void
-__brick_reverse(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __d_last,
-                /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    __unseq_backend::__simd_walk_2(__first, __last - __first, std::reverse_iterator<_RandomAccessIterator>(__d_last),
-                                   [](_ReferenceType __x, _ReferenceType __y) {
-                                       using std::swap;
-                                       swap(__x, __y);
-                                   });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator>
-void
-__pattern_reverse(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last) noexcept
-{
-    __internal::__brick_reverse(__first, __last, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-void
-__pattern_reverse(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                  _RandomAccessIterator __last)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __par_backend::__parallel_for(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __first + (__last - __first) / 2,
-        [__first, __last](_RandomAccessIterator __inner_first, _RandomAccessIterator __inner_last)
-        { __internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), _IsVector{}); });
-}
-
-//------------------------------------------------------------------------
-// reverse_copy
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator
-__brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
-                     /*is_vector=*/std::false_type) noexcept
-{
-    return std::reverse_copy(__first, __last, __d_first);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_reverse_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __d_first,
-                     /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
-
-    return __unseq_backend::__simd_walk_2(std::reverse_iterator<_RandomAccessIterator1>(__last), __last - __first,
-                                          __d_first, [](_ReferenceType1 __x, _ReferenceType2 __y) { __y = __x; });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator
-__pattern_reverse_copy(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                       _OutputIterator __d_first) noexcept
-{
-    return __internal::__brick_reverse_copy(__first, __last, __d_first, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__pattern_reverse_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                       _RandomAccessIterator1 __last, _RandomAccessIterator2 __d_first)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    auto __len = __last - __first;
-    __par_backend::__parallel_for(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-        [__first, __len, __d_first](_RandomAccessIterator1 __inner_first, _RandomAccessIterator1 __inner_last)
-        {
-            __internal::__brick_reverse_copy(__inner_first, __inner_last,
-                                             __d_first + (__len - (__inner_last - __first)), _IsVector{});
-        });
-    return __d_first + __len;
-}
-
-//------------------------------------------------------------------------
-// rotate
-//------------------------------------------------------------------------
-template <class _ForwardIterator>
-_ForwardIterator
-__brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
-               /*is_vector=*/std::false_type) noexcept
-{
-#if defined(_PSTL_CPP11_STD_ROTATE_BROKEN)
-    std::rotate(__first, __middle, __last);
-    return std::next(__first, std::distance(__middle, __last));
-#else
-    return std::rotate(__first, __middle, __last);
-#endif
-}
-
-template <class _RandomAccessIterator>
-_RandomAccessIterator
-__brick_rotate(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last,
-               /*is_vector=*/std::true_type) noexcept
-{
-    auto __n = __last - __first;
-    auto __m = __middle - __first;
-    const _RandomAccessIterator __ret = __first + (__last - __middle);
-
-    bool __is_left = (__m <= __n / 2);
-    if (!__is_left)
-        __m = __n - __m;
-
-    while (__n > 1 && __m > 0)
-    {
-        using std::iter_swap;
-        const auto __m_2 = __m * 2;
-        if (__is_left)
-        {
-            for (; __last - __first >= __m_2; __first += __m)
-            {
-                __unseq_backend::__simd_assign(__first, __m, __first + __m,
-                                               iter_swap<_RandomAccessIterator, _RandomAccessIterator>);
-            }
-        }
-        else
-        {
-            for (; __last - __first >= __m_2; __last -= __m)
-            {
-                __unseq_backend::__simd_assign(__last - __m, __m, __last - __m_2,
-                                               iter_swap<_RandomAccessIterator, _RandomAccessIterator>);
-            }
-        }
-        __is_left = !__is_left;
-        __m = __n % __m;
-        __n = __last - __first;
-    }
-
-    return __ret;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator>
-_ForwardIterator
-__pattern_rotate(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle,
-                 _ForwardIterator __last) noexcept
-{
-    return __internal::__brick_rotate(__first, __middle, __last, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-_RandomAccessIterator
-__pattern_rotate(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _RandomAccessIterator __middle, _RandomAccessIterator __last)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
-    auto __n = __last - __first;
-    auto __m = __middle - __first;
-    if (__m <= __n / 2)
-    {
-        __par_backend::__buffer<_Tp> __buf(__n - __m);
-        return __internal::__except_handler(
-            [&__exec, __n, __m, __first, __middle, __last, &__buf]()
-            {
-                _Tp* __result = __buf.get();
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __middle, __last,
-                    [__middle, __result](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __middle,
-                    [__last, __middle](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_move(__b, __e, __b + (__last - __middle), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __result, __result + (__n - __m),
-                    [__first, __result](_Tp* __b, _Tp* __e)
-                    { __brick_move_destroy()(__b, __e, __first + (__b - __result), _IsVector{}); });
-
-                return __first + (__last - __middle);
-            });
-    }
-    else
-    {
-        __par_backend::__buffer<_Tp> __buf(__m);
-        return __internal::__except_handler(
-            [&__exec, __n, __m, __first, __middle, __last, &__buf]()
-            {
-                _Tp* __result = __buf.get();
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __middle,
-                    [__first, __result](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __first), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __middle, __last,
-                    [__first, __middle](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_move(__b, __e, __first + (__b - __middle), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
-                    [__n, __m, __first, __result](_Tp* __b, _Tp* __e)
-                    { __brick_move_destroy()(__b, __e, __first + ((__n - __m) + (__b - __result)), _IsVector{}); });
-
-                return __first + (__last - __middle);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// rotate_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
-                    _OutputIterator __result, /*__is_vector=*/std::false_type) noexcept
-{
-    return std::rotate_copy(__first, __middle, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_rotate_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __middle, _RandomAccessIterator1 __last,
-                    _RandomAccessIterator2 __result, /*__is_vector=*/std::true_type) noexcept
-{
-    _RandomAccessIterator2 __res = __internal::__brick_copy(__middle, __last, __result, std::true_type());
-    return __internal::__brick_copy(__first, __middle, __res, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__pattern_rotate_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle,
-                      _ForwardIterator __last, _OutputIterator __result) noexcept
-{
-    return __internal::__brick_rotate_copy(__first, __middle, __last, __result, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__pattern_rotate_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                      _RandomAccessIterator1 __middle, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __par_backend::__parallel_for(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-        [__first, __last, __middle, __result](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e)
-        {
-            if (__b > __middle)
-            {
-                __internal::__brick_copy(__b, __e, __result + (__b - __middle), _IsVector{});
-            }
-            else
-            {
-                _RandomAccessIterator2 __new_result = __result + ((__last - __middle) + (__b - __first));
-                if (__e < __middle)
-                {
-                    __internal::__brick_copy(__b, __e, __new_result, _IsVector{});
-                }
-                else
-                {
-                    __internal::__brick_copy(__b, __middle, __new_result, _IsVector{});
-                    __internal::__brick_copy(__middle, __e, __result, _IsVector{});
-                }
-            }
-        });
-    return __result + (__last - __first);
-}
-
-//------------------------------------------------------------------------
-// is_partitioned
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-bool
-__brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-                       /*is_vector=*/std::false_type) noexcept
-{
-    return std::is_partitioned(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-bool
-__brick_is_partitioned(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                       /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    if (__first == __last)
-    {
-        return true;
-    }
-    else
-    {
-        _RandomAccessIterator __result = __unseq_backend::__simd_first(
-            __first, _SizeType(0), __last - __first,
-            [&__pred](_RandomAccessIterator __it, _SizeType __i) { return !__pred(__it[__i]); });
-        if (__result == __last)
-        {
-            return true;
-        }
-        else
-        {
-            ++__result;
-            return !__unseq_backend::__simd_or(__result, __last - __result, __pred);
-        }
-    }
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_is_partitioned(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _UnaryPredicate __pred)
-{
-    if (__first == __last)
-    {
-        return true;
-    }
-    else
-    {
-        using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-        return __internal::__except_handler([&]() {
-            // State of current range:
-            // broken     - current range is not partitioned by pred
-            // all_true   - all elements in current range satisfy pred
-            // all_false  - all elements in current range don't satisfy pred
-            // true_false - elements satisfy pred are placed before elements that don't satisfy pred
-            enum _ReduceType
-            {
-                __not_init = -1,
-                __broken,
-                __all_true,
-                __all_false,
-                __true_false
-            };
-            _ReduceType __init = __not_init;
-
-            // Array with states that we'll have when state from the left branch is merged with state from the right branch.
-            // State is calculated by formula: new_state = table[left_state * 4 + right_state]
-            _ReduceType __table[] = {__broken,     __broken,     __broken,     __broken, __broken,    __all_true,
-                                     __true_false, __true_false, __broken,     __broken, __all_false, __broken,
-                                     __broken,     __broken,     __true_false, __broken};
-
-            __init = __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
-                [&__pred, &__table](_RandomAccessIterator __i, _RandomAccessIterator __j,
-                                    _ReduceType __value) -> _ReduceType
-                {
-                    if (__value == __broken)
-                    {
-                        return __broken;
-                    }
-                    _ReduceType __res = __not_init;
-                    // if first element satisfy pred
-                    if (__pred(*__i))
-                    {
-                        // find first element that don't satisfy pred
-                        _RandomAccessIterator __x =
-                            __internal::__brick_find_if(__i + 1, __j, std::not_fn(__pred), _IsVector{});
-                        if (__x != __j)
-                        {
-                            // find first element after "x" that satisfy pred
-                            _RandomAccessIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, _IsVector{});
-                            // if it was found then range isn't partitioned by pred
-                            if (__y != __j)
-                            {
-                                return __broken;
-                            }
-                            else
-                            {
-                                __res = __true_false;
-                            }
-                        }
-                        else
-                        {
-                            __res = __all_true;
-                        }
-                    }
-                    else
-                    { // if first element doesn't satisfy pred
-                        // then we should find the first element that satisfy pred.
-                        // If we found it then range isn't partitioned by pred
-                        if (__internal::__brick_find_if(__i + 1, __j, __pred, _IsVector{}) != __j)
-                        {
-                            return __broken;
-                        }
-                        else
-                        {
-                            __res = __all_false;
-                        }
-                    }
-                    // if we have value from left range then we should calculate the result
-                    return (__value == -1) ? __res : __table[__value * 4 + __res];
-                },
-
-                [&__table](_ReduceType __val1, _ReduceType __val2) -> _ReduceType
-                {
-                    if (__val1 == __broken || __val2 == __broken)
-                    {
-                        return __broken;
-                    }
-                    // calculate the result for new big range
-                    return __table[__val1 * 4 + __val2];
-                });
-            return __init != __broken;
-        });
-    }
-}
-
-//------------------------------------------------------------------------
-// partition
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-                  /*is_vector=*/std::false_type) noexcept
-{
-    return std::partition(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__brick_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                  /*is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::partition(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_partition(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                    _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_partition(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_partition(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                    _RandomAccessIterator __last, _UnaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    // partitioned range: elements before pivot satisfy pred (true part),
-    //                    elements after pivot don't satisfy pred (false part)
-    struct _PartitionRange
-    {
-        _RandomAccessIterator __begin;
-        _RandomAccessIterator __pivot;
-        _RandomAccessIterator __end;
-    };
-
-    return __internal::__except_handler([&]() {
-        _PartitionRange __init{__last, __last, __last};
-
-        // lambda for merging two partitioned ranges to one partitioned range
-        auto __reductor = [&__exec](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange
-        {
-            auto __size1 = __val1.__end - __val1.__pivot;
-            auto __size2 = __val2.__pivot - __val2.__begin;
-            auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
-
-            // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
-            if (__val1.__end == __val1.__pivot)
-            {
-                return {__new_begin, __val2.__pivot, __val2.__end};
-            }
-            // if true part of right range greater than false part of left range
-            // then we should swap the false part of left range and last part of true part of right range
-            else if (__size2 > __size1)
-            {
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size1,
-                    [__val1, __val2, __size1](_RandomAccessIterator __i, _RandomAccessIterator __j) {
-                        __internal::__brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot),
-                                                        _IsVector{});
-                    });
-                return {__new_begin, __val2.__pivot - __size1, __val2.__end};
-            }
-            // else we should swap the first part of false part of left range and true part of right range
-            else
-            {
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size2,
-                    [__val1, __val2](_RandomAccessIterator __i, _RandomAccessIterator __j) {
-                        __internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), _IsVector{});
-                    });
-                return {__new_begin, __val1.__pivot + __size2, __val2.__end};
-            }
-        };
-
-        _PartitionRange __result = __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
-            [__pred, __reductor](_RandomAccessIterator __i, _RandomAccessIterator __j,
-                                 _PartitionRange __value) -> _PartitionRange
-            {
-                //1. serial partition
-                _RandomAccessIterator __pivot = __internal::__brick_partition(__i, __j, __pred, _IsVector{});
-
-                // 2. merging of two ranges (left and right respectively)
-                return __reductor(__value, {__i, __pivot, __j});
-            },
-            __reductor);
-        return __result.__pivot;
-    });
-}
-
-//------------------------------------------------------------------------
-// stable_partition
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator
-__brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
-                         /*__is_vector=*/std::false_type) noexcept
-{
-    return std::stable_partition(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__brick_stable_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                         /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::stable_partition(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator
-__pattern_stable_partition(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                           _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_stable_partition(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_stable_partition(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                           _RandomAccessIterator __last, _UnaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    // partitioned range: elements before pivot satisfy pred (true part),
-    //                    elements after pivot don't satisfy pred (false part)
-    struct _PartitionRange
-    {
-        _RandomAccessIterator __begin;
-        _RandomAccessIterator __pivot;
-        _RandomAccessIterator __end;
-    };
-
-    return __internal::__except_handler([&]() {
-        _PartitionRange __init{__last, __last, __last};
-
-        // lambda for merging two partitioned ranges to one partitioned range
-        auto __reductor = [](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange
-        {
-            auto __size1 = __val1.__end - __val1.__pivot;
-            auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
-
-            // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
-            if (__val1.__end == __val1.__pivot)
-            {
-                return {__new_begin, __val2.__pivot, __val2.__end};
-            }
-            // if true part of right range greater than false part of left range
-            // then we should swap the false part of left range and last part of true part of right range
-            else
-            {
-                __internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, _IsVector{});
-                return {__new_begin, __val2.__pivot - __size1, __val2.__end};
-            }
-        };
-
-        _PartitionRange __result = __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
-            [&__pred, __reductor](_RandomAccessIterator __i, _RandomAccessIterator __j,
-                                  _PartitionRange __value) -> _PartitionRange
-            {
-                //1. serial stable_partition
-                _RandomAccessIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, _IsVector{});
-
-                // 2. merging of two ranges (left and right respectively)
-                return __reductor(__value, {__i, __pivot, __j});
-            },
-            __reductor);
-        return __result.__pivot;
-    });
-}
-
-//------------------------------------------------------------------------
-// partition_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
-                       _OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::false_type) noexcept
-{
-    return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3,
-          class _UnaryPredicate>
-std::pair<_RandomAccessIterator2, _RandomAccessIterator3>
-__brick_partition_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __out_true,
-                       _RandomAccessIterator3 __out_false, _UnaryPredicate __pred,
-                       /*is_vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_partition_copy(__first, __last - __first, __out_true, __out_false, __pred);
-#else
-    return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
-          class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__pattern_partition_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred,
-                                              typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _UnaryPredicate>
-std::pair<_RandomAccessIterator2, _RandomAccessIterator3>
-__pattern_partition_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                         _RandomAccessIterator1 __last, _RandomAccessIterator2 __out_true,
-                         _RandomAccessIterator3 __out_false, _UnaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    typedef std::pair<_DifferenceType, _DifferenceType> _ReturnType;
-    const _DifferenceType __n = __last - __first;
-    if (_DifferenceType(1) < __n)
-    {
-        __par_backend::__buffer<bool> __mask_buf(__n);
-        return __internal::__except_handler(
-            [&__exec, __n, __first, __out_true, __out_false, __pred, &__mask_buf]()
-            {
-                bool* __mask = __mask_buf.get();
-                _ReturnType __m{};
-                __par_backend::__parallel_strict_scan(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n,
-                    std::make_pair(_DifferenceType(0), _DifferenceType(0)),
-                    [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
-                        return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
-                                                                                __mask + __i, __pred, _IsVector{});
-                    },
-                    [](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType
-                    { return std::make_pair(__x.first + __y.first, __x.second + __y.second); }, // Combine
-                    [=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) {    // Scan
-                        __internal::__brick_partition_by_mask(
-                            __first + __i, __first + (__i + __len), __out_true + __initial.first,
-                            __out_false + __initial.second, __mask + __i, _IsVector{});
-                    },
-                    [&__m](_ReturnType __total) { __m = __total; });
-                return std::make_pair(__out_true + __m.first, __out_false + __m.second);
-            });
-    }
-    // trivial sequence - use serial algorithm
-    return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, _IsVector{});
-}
-
-//------------------------------------------------------------------------
-// sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsMoveConstructible>
-void
-__pattern_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-               _IsMoveConstructible) noexcept
-{
-    std::sort(__first, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_sort(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-               _RandomAccessIterator __last, _Compare __comp, /*is_move_constructible=*/std::true_type)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
-                [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-                { std::sort(__first, __last, __comp); });
-        });
-}
-
-//------------------------------------------------------------------------
-// stable_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last,
-                      _Compare __comp) noexcept
-{
-    std::stable_sort(__first, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                      _RandomAccessIterator __last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
-                [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-                { std::stable_sort(__first, __last, __comp); });
-        });
-}
-
-//------------------------------------------------------------------------
-// partial_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-                       _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    std::partial_sort(__first, __middle, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                       _RandomAccessIterator __middle, _RandomAccessIterator __last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    const auto __n = __middle - __first;
-    if (__n == 0)
-        return;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
-                [__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp)
-                {
-                    if (__n < __end - __begin)
-                        std::partial_sort(__begin, __begin + __n, __end, __comp);
-                    else
-                        std::sort(__begin, __end, __comp);
-                },
-                __n);
-        });
-}
-
-//------------------------------------------------------------------------
-// partial_sort_copy
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_partial_sort_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                            _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp) noexcept
-{
-    return std::partial_sort_copy(__first, __last, __d_first, __d_last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-_RandomAccessIterator2
-__pattern_partial_sort_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                            _RandomAccessIterator1 __last, _RandomAccessIterator2 __d_first,
-                            _RandomAccessIterator2 __d_last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last == __first || __d_last == __d_first)
-    {
-        return __d_first;
-    }
-    auto __n1 = __last - __first;
-    auto __n2 = __d_last - __d_first;
-    return __internal::__except_handler([&]() {
-        if (__n2 >= __n1)
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __d_first, __d_first + __n1, __comp,
-                [__first, __d_first](_RandomAccessIterator2 __i, _RandomAccessIterator2 __j, _Compare __comp)
-                {
-                    _RandomAccessIterator1 __i1 = __first + (__i - __d_first);
-                    _RandomAccessIterator1 __j1 = __first + (__j - __d_first);
-
-                // 1. Copy elements from input to output
-#if !defined(_PSTL_ICC_18_OMP_SIMD_BROKEN)
-                    __internal::__brick_copy(__i1, __j1, __i, _IsVector{});
-#else
-                    std::copy(__i1, __j1, __i);
-#endif
-                    // 2. Sort elements in output sequence
-                    std::sort(__i, __j, __comp);
-                },
-                __n1);
-            return __d_first + __n1;
-        }
-        else
-        {
-            typedef typename std::iterator_traits<_RandomAccessIterator1>::value_type _T1;
-            typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _T2;
-            __par_backend::__buffer<_T1> __buf(__n1);
-            _T1* __r = __buf.get();
-
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp,
-                [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp)
-                {
-                    _RandomAccessIterator1 __it = __first + (__i - __r);
-
-                    // 1. Copy elements from input to raw memory
-                    for (_T1* __k = __i; __k != __j; ++__k, ++__it)
-                    {
-                        ::new (__k) _T2(*__it);
-                    }
-
-                    // 2. Sort elements in temporary __buffer
-                    if (__n2 < __j - __i)
-                        std::partial_sort(__i, __i + __n2, __j, __comp);
-                    else
-                        std::sort(__i, __j, __comp);
-                },
-                __n2);
-
-            // 3. Move elements from temporary __buffer to output
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n2,
-                                          [__r, __d_first](_T1* __i, _T1* __j)
-                                          { __brick_move_destroy()(__i, __j, __d_first + (__i - __r), _IsVector{}); });
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r + __n2,
-                                          __r + __n1,
-                                          [](_T1* __i, _T1* __j) { __brick_destroy(__i, __j, _IsVector{}); });
-
-            return __d_first + __n2;
-        }
-    });
-}
-
-//------------------------------------------------------------------------
-// adjacent_find
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_adjacent_find(_RandomAccessIterator __first, _RandomAccessIterator __last, _BinaryPredicate __pred,
-                      /* IsVector = */ std::true_type, bool __or_semantic) noexcept
-{
-    return __unseq_backend::__simd_adjacent_find(__first, __last, __pred, __or_semantic);
-}
-
-template <class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
-                      /* IsVector = */ std::false_type, bool) noexcept
-{
-    return std::adjacent_find(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_adjacent_find(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                        _BinaryPredicate __pred, bool __or_semantic) noexcept
-{
-    return __internal::__brick_adjacent_find(__first, __last, __pred, typename _Tag::__is_vector{}, __or_semantic);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_adjacent_find(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                        _RandomAccessIterator __last, _BinaryPredicate __pred, bool __or_semantic)
-{
-    if (__last - __first < 2)
-        return __last;
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __last,
-                [__last, __pred, __or_semantic](_RandomAccessIterator __begin, _RandomAccessIterator __end,
-                                                _RandomAccessIterator __value) -> _RandomAccessIterator
-                {
-                    // TODO: investigate performance benefits from the use of shared variable for the result,
-                    // checking (compare_and_swap idiom) its __value at __first.
-                    if (__or_semantic && __value < __last)
-                    { //found
-                        __par_backend::__cancel_execution();
-                        return __value;
-                    }
-
-                    if (__value > __begin)
-                    {
-                        // modify __end to check the predicate on the boundary __values;
-                        // TODO: to use a custom range with boundaries overlapping
-                        // TODO: investigate what if we remove "if" below and run algorithm on range [__first, __last-1)
-                        // then check the pair [__last-1, __last)
-                        if (__end != __last)
-                            ++__end;
-
-                        //correct the global result iterator if the "brick" returns a local "__last"
-                        const _RandomAccessIterator __res =
-                            __internal::__brick_adjacent_find(__begin, __end, __pred, _IsVector{}, __or_semantic);
-                        if (__res < __end)
-                            __value = __res;
-                    }
-                    return __value;
-                },
-                [](_RandomAccessIterator __x, _RandomAccessIterator __y) -> _RandomAccessIterator
-                { return __x < __y ? __x : __y; } //reduce a __value
-            );
-        });
-}
-
-//------------------------------------------------------------------------
-// nth_element
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-                      _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    std::nth_element(__first, __nth, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                      _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    if (__first == __last || __nth == __last)
-    {
-        return;
-    }
-
-    using std::iter_swap;
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
-    _RandomAccessIterator __x;
-    do
-    {
-        __x = __internal::__pattern_partition(__tag, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last,
-                                              [&__comp, __first](const _Tp& __x) { return __comp(__x, *__first); });
-        --__x;
-        if (__x != __first)
-        {
-            iter_swap(__first, __x);
-        }
-        // if x > nth then our new range for partition is [first, x)
-        if (__x - __nth > 0)
-        {
-            __last = __x;
-        }
-        // if x < nth then our new range for partition is [x, last)
-        else if (__x - __nth < 0)
-        {
-            // if *x == *nth then we can start new partition with x+1
-            if (!__comp(*__nth, *__x) && !__comp(*__x, *__nth))
-            {
-                ++__x;
-            }
-            else
-            {
-                iter_swap(__nth, __x);
-            }
-            __first = __x;
-        }
-    } while (__x != __nth);
-}
-
-//------------------------------------------------------------------------
-// fill, fill_n
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Tp>
-void
-__brick_fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value,
-             /* __is_vector = */ std::true_type) noexcept
-{
-    __unseq_backend::__simd_fill_n(__first, __last - __first, __value);
-}
-
-template <class _ForwardIterator, class _Tp>
-void
-__brick_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
-             /* __is_vector = */ std::false_type) noexcept
-{
-    std::fill(__first, __last, __value);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-void
-__pattern_fill(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) noexcept
-{
-    __internal::__brick_fill(__first, __last, __value, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp>
-_RandomAccessIterator
-__pattern_fill(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-               _RandomAccessIterator __last, const _Tp& __value)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&__exec, __first, __last, &__value]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [&__value](_RandomAccessIterator __begin, _RandomAccessIterator __end)
-                                          { __internal::__brick_fill(__begin, __end, __value, _IsVector{}); });
-            return __last;
-        });
-}
-
-template <class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__brick_fill_n(_RandomAccessIterator __first, _Size __count, const _Tp& __value,
-               /* __is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_fill_n(__first, __count, __value);
-}
-
-template <class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__brick_fill_n(_OutputIterator __first, _Size __count, const _Tp& __value, /* __is_vector = */ std::false_type) noexcept
-{
-    return std::fill_n(__first, __count, __value);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__pattern_fill_n(_Tag, _ExecutionPolicy&&, _OutputIterator __first, _Size __count, const _Tp& __value) noexcept
-{
-    return __internal::__brick_fill_n(__first, __count, __value, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__pattern_fill_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _Size __count, const _Tp& __value)
-{
-    return __internal::__pattern_fill(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __count,
-                                      __value);
-}
-
-//------------------------------------------------------------------------
-// generate, generate_n
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Generator>
-void
-__brick_generate(_RandomAccessIterator __first, _RandomAccessIterator __last, _Generator __g,
-                 /* is_vector = */ std::true_type) noexcept
-{
-    __unseq_backend::__simd_generate_n(__first, __last - __first, __g);
-}
-
-template <class _ForwardIterator, class _Generator>
-void
-__brick_generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __g,
-                 /* is_vector = */ std::false_type) noexcept
-{
-    std::generate(__first, __last, __g);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-void
-__pattern_generate(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __g) noexcept
-{
-    __internal::__brick_generate(__first, __last, __g, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Generator>
-_RandomAccessIterator
-__pattern_generate(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                   _RandomAccessIterator __last, _Generator __g)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [__g](_RandomAccessIterator __begin, _RandomAccessIterator __end)
-                                          { __internal::__brick_generate(__begin, __end, __g, _IsVector{}); });
-            return __last;
-        });
-}
-
-template <class _RandomAccessIterator, class Size, class _Generator>
-_RandomAccessIterator
-__brick_generate_n(_RandomAccessIterator __first, Size __count, _Generator __g,
-                   /* is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_generate_n(__first, __count, __g);
-}
-
-template <class OutputIterator, class Size, class _Generator>
-OutputIterator
-__brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::false_type) noexcept
-{
-    return std::generate_n(__first, __count, __g);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator>
-_OutputIterator
-__pattern_generate_n(_Tag, _ExecutionPolicy&&, _OutputIterator __first, _Size __count, _Generator __g) noexcept
-{
-    return __internal::__brick_generate_n(__first, __count, __g, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Generator>
-_RandomAccessIterator
-__pattern_generate_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                     _Size __count, _Generator __g)
-{
-    static_assert(__are_random_access_iterators<_RandomAccessIterator>::value,
-                  "Pattern-brick error. Should be a random access iterator.");
-    return __internal::__pattern_generate(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __count,
-                                          __g);
-}
-
-//------------------------------------------------------------------------
-// remove
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__brick_remove_if(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-                  /* __is_vector = */ std::false_type) noexcept
-{
-    return std::remove_if(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__brick_remove_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                  /* __is_vector = */ std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_remove_if(__first, __last - __first, __pred);
-#else
-    return std::remove_if(__first, __last, __pred);
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_remove_if(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                    _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_remove_if(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_remove_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                    _RandomAccessIterator __last, _UnaryPredicate __pred) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    if (__first == __last || __first + 1 == __last)
-    {
-        // Trivial sequence - use serial algorithm
-        return __internal::__brick_remove_if(__first, __last, __pred, _IsVector{});
-    }
-
-    return __internal::__remove_elements(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-        [&__pred](bool* __b, bool* __e, _RandomAccessIterator __it)
-        {
-            __internal::__brick_walk2(
-                __b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); }, _IsVector{});
-        });
-}
-
-//------------------------------------------------------------------------
-// merge
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_merge(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-              _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp,
-              /* __is_vector = */ std::false_type) noexcept
-{
-    return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_merge(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _RandomAccessIterator2 __last2, _RandomAccessIterator3 __d_first, _Compare __comp,
-              /* __is_vector = */ std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_merge(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __d_first,
-                _Compare __comp) noexcept
-{
-    return __internal::__brick_merge(__first1, __last1, __first2, __last2, __d_first, __comp,
-                                     typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_merge(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                _RandomAccessIterator3 __d_first, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __par_backend::__parallel_merge(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first,
-        __comp,
-        [](_RandomAccessIterator1 __f1, _RandomAccessIterator1 __l1, _RandomAccessIterator2 __f2,
-           _RandomAccessIterator2 __l2, _RandomAccessIterator3 __f3, _Compare __comp)
-        { return __internal::__brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, _IsVector{}); });
-    return __d_first + (__last1 - __first1) + (__last2 - __first2);
-}
-
-//------------------------------------------------------------------------
-// inplace_merge
-//------------------------------------------------------------------------
-template <class _BidirectionalIterator, class _Compare>
-void
-__brick_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last,
-                      _Compare __comp, /* __is_vector = */ std::false_type) noexcept
-{
-    std::inplace_merge(__first, __middle, __last, __comp);
-}
-
-template <class _RandomAccessIterator, class _Compare>
-void
-__brick_inplace_merge(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last,
-                      _Compare __comp, /* __is_vector = */ std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial")
-    std::inplace_merge(__first, __middle, __last, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-void
-__pattern_inplace_merge(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-                        _BidirectionalIterator __last, _Compare __comp) noexcept
-{
-    __internal::__brick_inplace_merge(__first, __middle, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_inplace_merge(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                        _RandomAccessIterator __middle, _RandomAccessIterator __last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__first == __last || __first == __middle || __middle == __last)
-    {
-        return;
-    }
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
-    auto __n = __last - __first;
-    __par_backend::__buffer<_Tp> __buf(__n);
-    _Tp* __r = __buf.get();
-    __internal::__except_handler(
-        [&]()
-        {
-            auto __move_values = [](_RandomAccessIterator __x, _Tp* __z)
-            {
-                __internal::__invoke_if_else(
-                    std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
-                    [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
-            };
-
-            auto __move_sequences = [](_RandomAccessIterator __first1, _RandomAccessIterator __last1, _Tp* __first2)
-            { return __internal::__brick_uninitialized_move(__first1, __last1, __first2, _IsVector()); };
-
-            __par_backend::__parallel_merge(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __middle, __middle, __last, __r,
-                __comp,
-                [__n, __move_values, __move_sequences](_RandomAccessIterator __f1, _RandomAccessIterator __l1,
-                                                       _RandomAccessIterator __f2, _RandomAccessIterator __l2,
-                                                       _Tp* __f3, _Compare __comp)
-                {
-                    (__utils::__serial_move_merge(__n))(__f1, __l1, __f2, __l2, __f3, __comp, __move_values,
-                                                        __move_values, __move_sequences, __move_sequences);
-                    return __f3 + (__l1 - __f1) + (__l2 - __f2);
-                });
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n,
-                                          [__r, __first](_Tp* __i, _Tp* __j)
-                                          { __brick_move_destroy()(__i, __j, __first + (__i - __r), _IsVector{}); });
-        });
-}
-
-//------------------------------------------------------------------------
-// includes
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_includes(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                   _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp) noexcept
-{
-    return std::includes(__first1, __last1, __first2, __last2, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_includes(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                   _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                   _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__first2 >= __last2)
-        return true;
-
-    if (__first1 >= __last1 || __comp(*__first2, *__first1) || __comp(*(__last1 - 1), *(__last2 - 1)))
-        return false;
-
-    __first1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-    if (__first1 == __last1)
-        return false;
-
-    if (__last2 - __first2 == 1)
-        return !__comp(*__first1, *__first2) && !__comp(*__first2, *__first1);
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return !__internal::__parallel_or(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                [__first1, __last1, __first2, __last2, &__comp](_RandomAccessIterator2 __i, _RandomAccessIterator2 __j)
-                {
-                    _PSTL_ASSERT(__j > __i);
-                    //_PSTL_ASSERT(__j - __i > 1);
-
-                    //1. moving boundaries to "consume" subsequence of equal elements
-                    auto __is_equal = [&__comp](_RandomAccessIterator2 __a, _RandomAccessIterator2 __b) -> bool
-                    { return !__comp(*__a, *__b) && !__comp(*__b, *__a); };
-
-                    //1.1 left bound, case "aaa[aaaxyz...]" - searching "x"
-                    if (__i > __first2 && __is_equal(__i, __i - 1))
-                    {
-                        //whole subrange continues to content equal elements - return "no op"
-                        if (__is_equal(__i, __j - 1))
-                            return false;
-
-                        __i = std::upper_bound(__i, __last2, *__i, __comp);
-                    }
-
-                    //1.2 right bound, case "[...aaa]aaaxyz" - searching "x"
-                    if (__j < __last2 && __is_equal(__j - 1, __j))
-                        __j = std::upper_bound(__j, __last2, *__j, __comp);
-
-                    //2. testing is __a subsequence of the second range included into the first range
-                    auto __b = std::lower_bound(__first1, __last1, *__i, __comp);
-
-                    _PSTL_ASSERT(!__comp(*(__last1 - 1), *__b));
-                    _PSTL_ASSERT(!__comp(*(__j - 1), *__i));
-                    return !std::includes(__b, __last1, __i, __j, __comp);
-                });
-        });
-}
-
-constexpr auto __set_algo_cut_off = 1000;
-
-template <class _IsVector, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2,
-          class _OutputIterator, class _Compare, class _SizeFunction, class _SetOP>
-_OutputIterator
-__parallel_set_op(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _ForwardIterator1 __first1,
-                  _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
-                  _OutputIterator __result, _Compare __comp, _SizeFunction __size_func, _SetOP __set_op)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
-    typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
-
-    struct _SetRange
-    {
-        _DifferenceType __pos, __len, __buf_pos;
-        bool
-        empty() const
-        {
-            return __len == 0;
-        }
-    };
-
-    const _DifferenceType __n1 = __last1 - __first1;
-    const _DifferenceType __n2 = __last2 - __first2;
-
-    __par_backend::__buffer<_Tp> __buf(__size_func(__n1, __n2));
-
-    return __internal::__except_handler(
-        [&__exec, __n1, __first1, __last1, __first2, __last2, __result, __comp, __size_func, __set_op, &__buf]()
-        {
-            auto __buffer = __buf.get();
-            _DifferenceType __m{};
-            auto __scan = [=](_DifferenceType, _DifferenceType, const _SetRange& __s) { // Scan
-                if (!__s.empty())
-                    __brick_move_destroy()(__buffer + __s.__buf_pos, __buffer + (__s.__buf_pos + __s.__len),
-                                           __result + __s.__pos, _IsVector{});
-            };
-            __par_backend::__parallel_strict_scan(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n1, _SetRange{0, 0, 0}, //-1, 0},
-                [=](_DifferenceType __i, _DifferenceType __len) {                                  // Reduce
-                    //[__b; __e) - a subrange of the first sequence, to reduce
-                    _ForwardIterator1 __b = __first1 + __i, __e = __first1 + (__i + __len);
-
-                    //try searching for the first element which not equal to *__b
-                    if (__b != __first1)
-                        __b = std::upper_bound(__b, __last1, *__b, __comp);
-
-                    //try searching for the first element which not equal to *__e
-                    if (__e != __last1)
-                        __e = std::upper_bound(__e, __last1, *__e, __comp);
-
-                    //check is [__b; __e) empty
-                    if (__e - __b < 1)
-                    {
-                        _ForwardIterator2 __bb = __last2;
-                        if (__b != __last1)
-                            __bb = std::lower_bound(__first2, __last2, *__b, __comp);
-
-                        const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
-                        return _SetRange{0, 0, __buf_pos};
-                    }
-
-                    //try searching for "corresponding" subrange [__bb; __ee) in the second sequence
-                    _ForwardIterator2 __bb = __first2;
-                    if (__b != __first1)
-                        __bb = std::lower_bound(__first2, __last2, *__b, __comp);
-
-                    _ForwardIterator2 __ee = __last2;
-                    if (__e != __last1)
-                        __ee = std::lower_bound(__bb, __last2, *__e, __comp);
-
-                    const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
-                    auto __buffer_b = __buffer + __buf_pos;
-                    auto __res = __set_op(__b, __e, __bb, __ee, __buffer_b, __comp);
-
-                    return _SetRange{0, __res - __buffer_b, __buf_pos};
-                },
-                [](const _SetRange& __a, const _SetRange& __b) { // Combine
-                    if (__b.__buf_pos > __a.__buf_pos || ((__b.__buf_pos == __a.__buf_pos) && !__b.empty()))
-                        return _SetRange{__a.__pos + __a.__len + __b.__pos, __b.__len, __b.__buf_pos};
-                    return _SetRange{__b.__pos + __b.__len + __a.__pos, __a.__len, __a.__buf_pos};
-                },
-                __scan,                                     // Scan
-                [&__m, &__scan](const _SetRange& __total) { // Apex
-                    //final scan
-                    __scan(0, 0, __total);
-                    __m = __total.__pos + __total.__len;
-                });
-            return __result + __m;
-        });
-}
-
-//a shared parallel pattern for '__pattern_set_union' and '__pattern_set_symmetric_difference'
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare, class _SetUnionOp>
-_OutputIterator
-__parallel_set_union_op(_Tag __tag, _ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                        _Compare __comp, _SetUnionOp __set_union_op)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    auto copy_range1 = [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res)
-    { return __internal::__brick_copy(__begin, __end, __res, typename _Tag::__is_vector{}); };
-    auto copy_range2 = [](_ForwardIterator2 __begin, _ForwardIterator2 __end, _OutputIterator __res)
-    { return __internal::__brick_copy(__begin, __end, __res, typename _Tag::__is_vector{}); };
-
-    // {1} {}: parallel copying just first sequence
-    if (__n2 == 0)
-        return __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                                                 __result, copy_range1);
-
-    // {} {2}: parallel copying justmake  second sequence
-    if (__n1 == 0)
-        return __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                                                 __result, copy_range2);
-
-    // testing  whether the sequences are intersected
-    _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-
-    if (__left_bound_seq_1 == __last1)
-    {
-        //{1} < {2}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                                                  __result, copy_range1);
-            },
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                                                  __result + __n1, copy_range2);
-            });
-        return __result + __n1 + __n2;
-    }
-
-    // testing  whether the sequences are intersected
-    _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
-
-    if (__left_bound_seq_2 == __last2)
-    {
-        //{2} < {1}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                                                  __result, copy_range2);
-            },
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                                                  __result + __n2, copy_range1);
-            });
-        return __result + __n1 + __n2;
-    }
-
-    const auto __m1 = __left_bound_seq_1 - __first1;
-    if (__m1 > __set_algo_cut_off)
-    {
-        auto __res_or = __result;
-        __result += __m1; //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            //do parallel copying of [first1; left_bound_seq_1)
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                  __left_bound_seq_1, __res_or, copy_range1);
-            },
-            [=, &__result]
-            {
-                __result = __internal::__parallel_set_op(
-                    __tag, std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2,
-                    __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
-                    __set_union_op);
-            });
-        return __result;
-    }
-
-    const auto __m2 = __left_bound_seq_2 - __first2;
-    _PSTL_ASSERT(__m1 == 0 || __m2 == 0);
-    if (__m2 > __set_algo_cut_off)
-    {
-        auto __res_or = __result;
-        __result += __m2; //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            //do parallel copying of [first2; left_bound_seq_2)
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2,
-                                                  __left_bound_seq_2, __res_or, copy_range2);
-            },
-            [=, &__result]
-            {
-                __result = __internal::__parallel_set_op(
-                    __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2,
-                    __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
-                    __set_union_op);
-            });
-        return __result;
-    }
-
-    return __internal::__parallel_set_op(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-        [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op);
-}
-
-//------------------------------------------------------------------------
-// set_union
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_union(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                  _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                  /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <typename _IsVector>
-struct __BrickCopyConstruct
-{
-    template <typename _ForwardIterator, typename _OutputIterator>
-    _OutputIterator
-    operator()(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result)
-    {
-        return __brick_uninitialized_copy(__first, __last, __result, _IsVector());
-    }
-};
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_union(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                  _RandomAccessIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                  /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_union(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                    _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                    _Compare __comp) noexcept
-{
-    return __internal::__brick_set_union(__first1, __last1, __first2, __last2, __result, __comp,
-                                         typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_union(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                    _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                    _OutputIterator __result, _Compare __comp)
-{
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // use serial algorithm
-    if (__n1 + __n2 <= __set_algo_cut_off)
-        return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
-
-    typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
-    return __parallel_set_union_op(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-        [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-           _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp)
-        {
-            return __pstl::__utils::__set_union_construct(__first1, __last1, __first2, __last2, __result, __comp,
-                                                          __BrickCopyConstruct<_IsVector>());
-        });
-}
-
-//------------------------------------------------------------------------
-// set_intersection
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_intersection(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                         _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                         /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_set_intersection(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                         _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                         _RandomAccessIterator3 __result, _Compare __comp,
-                         /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_intersection(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                           _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                           _Compare __comp) noexcept
-{
-    return __internal::__brick_set_intersection(__first1, __last1, __first2, __last2, __result, __comp,
-                                                typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_set_intersection(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                           _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                           _RandomAccessIterator2 __last2, _RandomAccessIterator3 __result, _Compare __comp)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // intersection is empty
-    if (__n1 == 0 || __n2 == 0)
-        return __result;
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-    //{1} < {2}: seq 2 is wholly greater than seq 1, so, the intersection is empty
-    if (__left_bound_seq_1 == __last1)
-        return __result;
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
-    //{2} < {1}: seq 1 is wholly greater than seq 2, so, the intersection is empty
-    if (__left_bound_seq_2 == __last2)
-        return __result;
-
-    const auto __m1 = __last1 - __left_bound_seq_1 + __n2;
-    if (__m1 > __set_algo_cut_off)
-    {
-        //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
-        return __internal::__parallel_set_op(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result,
-            __comp, [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
-            [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-               _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp) {
-                return __pstl::__utils::__set_intersection_construct(__first1, __last1, __first2, __last2, __result,
-                                                                     __comp);
-            });
-    }
-
-    const auto __m2 = __last2 - __left_bound_seq_2 + __n1;
-    if (__m2 > __set_algo_cut_off)
-    {
-        //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
-        __result = __internal::__parallel_set_op(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result,
-            __comp, [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
-            [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-               _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp) {
-                return __pstl::__utils::__set_intersection_construct(__first2, __last2, __first1, __last1, __result,
-                                                                     __comp);
-            });
-        return __result;
-    }
-
-    // [left_bound_seq_1; last1) and [left_bound_seq_2; last2) - use serial algorithm
-    return std::set_intersection(__left_bound_seq_1, __last1, __left_bound_seq_2, __last2, __result, __comp);
-}
-
-//------------------------------------------------------------------------
-// set_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                       _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                       /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_set_difference(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                       _RandomAccessIterator2 __last2, _RandomAccessIterator3 __result, _Compare __comp,
-                       /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                         _Compare __comp) noexcept
-{
-    return __internal::__brick_set_difference(__first1, __last1, __first2, __last2, __result, __comp,
-                                              typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_set_difference(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                         _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                         _RandomAccessIterator2 __last2, _RandomAccessIterator3 __result, _Compare __comp)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // {} \ {2}: the difference is empty
-    if (__n1 == 0)
-        return __result;
-
-    // {1} \ {}: parallel copying just first sequence
-    if (__n2 == 0)
-        return __internal::__pattern_walk2_brick(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
-            [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res)
-            { return __internal::__brick_copy(__begin, __end, __res, _IsVector{}); });
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-    //{1} < {2}: seq 2 is wholly greater than seq 1, so, parallel copying just first sequence
-    if (__left_bound_seq_1 == __last1)
-        return __internal::__pattern_walk2_brick(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
-            [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res)
-            { return __internal::__brick_copy(__begin, __end, __res, _IsVector{}); });
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
-    //{2} < {1}: seq 1 is wholly greater than seq 2, so, parallel copying just first sequence
-    if (__left_bound_seq_2 == __last2)
-        return __internal::__pattern_walk2_brick(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
-            [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res)
-            { return __internal::__brick_copy(__begin, __end, __res, _IsVector{}); });
-
-    if (__n1 + __n2 > __set_algo_cut_off)
-        return __parallel_set_op(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-            [](_DifferenceType __n, _DifferenceType) { return __n; },
-            [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-               _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp)
-            {
-                return __pstl::__utils::__set_difference_construct(__first1, __last1, __first2, __last2, __result,
-                                                                   __comp, __BrickCopyConstruct<_IsVector>());
-            });
-
-    // use serial algorithm
-    return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-//------------------------------------------------------------------------
-// set_symmetric_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_symmetric_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                                 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                                 /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_set_symmetric_difference(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                                 _RandomAccessIterator3 __result, _Compare __comp,
-                                 /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_symmetric_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                                   _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                                   _Compare __comp) noexcept
-{
-    return __internal::__brick_set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp,
-                                                        typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_set_symmetric_difference(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec,
-                                   _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                   _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                                   _RandomAccessIterator3 __result, _Compare __comp)
-{
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // use serial algorithm
-    if (__n1 + __n2 <= __set_algo_cut_off)
-        return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
-
-    typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
-    return __internal::__parallel_set_union_op(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-        [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-           _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp)
-        {
-            return __pstl::__utils::__set_symmetric_difference_construct(__first1, __last1, __first2, __last2, __result,
-                                                                         __comp, __BrickCopyConstruct<_IsVector>());
-        });
-}
-
-//------------------------------------------------------------------------
-// is_heap_until
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                      /* __is_vector = */ std::false_type) noexcept
-{
-    return std::is_heap_until(__first, __last, __comp);
-}
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                      /* __is_vector = */ std::true_type) noexcept
-{
-    if (__last - __first < 2)
-        return __last;
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    return __unseq_backend::__simd_first(
-        __first, _SizeType(0), __last - __first,
-        [&__comp](_RandomAccessIterator __it, _SizeType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last,
-                        _Compare __comp) noexcept
-{
-    return __internal::__brick_is_heap_until(__first, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
-_RandomAccessIterator
-__is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
-                      /* __is_vector = */ std::false_type) noexcept
-{
-    _DifferenceType __i = __begin;
-    for (; __i < __end; ++__i)
-    {
-        if (__comp(__first[(__i - 1) / 2], __first[__i]))
-        {
-            break;
-        }
-    }
-    return __first + __i;
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
-_RandomAccessIterator
-__is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
-                      /* __is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_first(
-        __first, __begin, __end,
-        [&__comp](_RandomAccessIterator __it, _DifferenceType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                        _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last - __first < 2)
-        return __last;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __parallel_find(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__first, __comp](_RandomAccessIterator __i, _RandomAccessIterator __j) {
-                    return __internal::__is_heap_until_local(__first, __i - __first, __j - __first, __comp,
-                                                             _IsVector{});
-                },
-                std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
-        });
-}
-
-//------------------------------------------------------------------------
-// min_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__brick_min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
-                    /* __is_vector = */ std::false_type) noexcept
-{
-    return std::min_element(__first, __last, __comp);
-}
-
-template <typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator
-__brick_min_element(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                    /* __is_vector = */ std::true_type) noexcept
-{
-#if defined(_PSTL_UDR_PRESENT)
-    return __unseq_backend::__simd_min_element(__first, __last - __first, __comp);
-#else
-    return std::min_element(__first, __last, __comp);
-#endif
-}
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__pattern_min_element(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                      _Compare __comp) noexcept
-{
-    return __internal::__brick_min_element(__first, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator
-__pattern_min_element(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                      _RandomAccessIterator __last, _Compare __comp)
-{
-    if (__first == __last)
-        return __last;
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, __first,
-                [=](_RandomAccessIterator __begin, _RandomAccessIterator __end,
-                    _RandomAccessIterator __init) -> _RandomAccessIterator
-                {
-                    const _RandomAccessIterator subresult =
-                        __internal::__brick_min_element(__begin, __end, __comp, _IsVector{});
-                    return __internal::__cmp_iterators_by_values(__init, subresult, __comp);
-                },
-                [=](_RandomAccessIterator __it1, _RandomAccessIterator __it2) -> _RandomAccessIterator
-                { return __internal::__cmp_iterators_by_values(__it1, __it2, __comp); });
-        });
-}
-
-//------------------------------------------------------------------------
-// minmax_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__brick_minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
-                       /* __is_vector = */ std::false_type) noexcept
-{
-    return std::minmax_element(__first, __last, __comp);
-}
-
-template <typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-__brick_minmax_element(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                       /* __is_vector = */ std::true_type) noexcept
-{
-#if defined(_PSTL_UDR_PRESENT)
-    return __unseq_backend::__simd_minmax_element(__first, __last - __first, __comp);
-#else
-    return std::minmax_element(__first, __last, __comp);
-#endif
-}
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__pattern_minmax_element(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _Compare __comp) noexcept
-{
-    return __internal::__brick_minmax_element(__first, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-__pattern_minmax_element(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _Compare __comp)
-{
-    if (__first == __last)
-        return std::make_pair(__first, __first);
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler([&]() {
-        typedef std::pair<_RandomAccessIterator, _RandomAccessIterator> _Result;
-
-        return __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last,
-            std::make_pair(__first, __first),
-            [=](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Result __init) -> _Result
-            {
-                const _Result __subresult = __internal::__brick_minmax_element(__begin, __end, __comp, _IsVector{});
-                return std::make_pair(
-                    __internal::__cmp_iterators_by_values(__subresult.first, __init.first, __comp),
-                    __internal::__cmp_iterators_by_values(__init.second, __subresult.second, std::not_fn(__comp)));
-            },
-            [=](_Result __p1, _Result __p2) -> _Result
-            {
-                return std::make_pair(
-                    __internal::__cmp_iterators_by_values(__p1.first, __p2.first, __comp),
-                    __internal::__cmp_iterators_by_values(__p2.second, __p1.second, std::not_fn(__comp)));
-            });
-    });
-}
-
-//------------------------------------------------------------------------
-// mismatch
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__mismatch_serial(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                  _ForwardIterator2 __last2, _BinaryPredicate __pred)
-{
-#if defined(_PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT)
-    return std::mismatch(__first1, __last1, __first2, __last2, __pred);
-#else
-    for (; __first1 != __last1 && __first2 != __last2 && __pred(*__first1, *__first2); ++__first1, ++__first2)
-    {
-    }
-    return std::make_pair(__first1, __first2);
-#endif
-}
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__brick_mismatch(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                 _ForwardIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::false_type) noexcept
-{
-    return __mismatch_serial(__first1, __last1, __first2, __last2, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-__brick_mismatch(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                 _RandomAccessIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::true_type) noexcept
-{
-    auto __n = std::min(__last1 - __first1, __last2 - __first2);
-    return __unseq_backend::__simd_first(__first1, __n, __first2, std::not_fn(__pred));
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__pattern_mismatch(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                   _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Predicate __pred) noexcept
-{
-    return __internal::__brick_mismatch(__first1, __last1, __first2, __last2, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-__pattern_mismatch(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                   _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                   _Predicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler([&]() {
-        auto __n = std::min(__last1 - __first1, __last2 - __first2);
-        auto __result = __internal::__parallel_find(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-            [__first1, __first2, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-            {
-                return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
-                                                    __pred, _IsVector{})
-                    .first;
-            },
-            std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
-        return std::make_pair(__result, __first2 + (__result - __first1));
-    });
-}
-
-//------------------------------------------------------------------------
-// lexicographical_compare
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__brick_lexicographical_compare(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                                _ForwardIterator2 __last2, _Compare __comp,
-                                /* __is_vector = */ std::false_type) noexcept
-{
-    return std::lexicographical_compare(__first1, __last1, __first2, __last2, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Compare>
-bool
-__brick_lexicographical_compare(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _Compare __comp,
-                                /* __is_vector = */ std::true_type) noexcept
-{
-    if (__first2 == __last2)
-    { // if second sequence is empty
-        return false;
-    }
-    else if (__first1 == __last1)
-    { // if first sequence is empty
-        return true;
-    }
-    else
-    {
-        typedef typename std::iterator_traits<_RandomAccessIterator1>::reference ref_type1;
-        typedef typename std::iterator_traits<_RandomAccessIterator2>::reference ref_type2;
-        --__last1;
-        --__last2;
-        auto __n = std::min(__last1 - __first1, __last2 - __first2);
-        std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __result = __unseq_backend::__simd_first(
-            __first1, __n, __first2, [__comp](const ref_type1 __x, const ref_type2 __y) mutable {
-                return __comp(__x, __y) || __comp(__y, __x);
-            });
-
-        if (__result.first == __last1 && __result.second != __last2)
-        { // if first sequence shorter than second
-            return !__comp(*__result.second, *__result.first);
-        }
-        else
-        { // if second sequence shorter than first or both have the same number of elements
-            return __comp(*__result.first, *__result.second);
-        }
-    }
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_lexicographical_compare(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                                  _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp) noexcept
-{
-    return __internal::__brick_lexicographical_compare(__first1, __last1, __first2, __last2, __comp,
-                                                       typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_lexicographical_compare(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec,
-                                  _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                  _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                                  _Compare __comp) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__first2 == __last2)
-    { // if second sequence is empty
-        return false;
-    }
-    else if (__first1 == __last1)
-    { // if first sequence is empty
-        return true;
-    }
-    else
-    {
-        typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _RefType1;
-        typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _RefType2;
-        --__last1;
-        --__last2;
-        auto __n = std::min(__last1 - __first1, __last2 - __first2);
-        auto __result = __internal::__parallel_find(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-            [__first1, __first2, &__comp](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-            {
-                return __internal::__brick_mismatch(
-                           __i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
-                           [&__comp](const _RefType1 __x, const _RefType2 __y)
-                           { return !__comp(__x, __y) && !__comp(__y, __x); },
-                           _IsVector{})
-                    .first;
-            },
-            std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
-
-        if (__result == __last1 && __first2 + (__result - __first1) != __last2)
-        { // if first sequence shorter than second
-            return !__comp(*(__first2 + (__result - __first1)), *__result);
-        }
-        else
-        { // if second sequence shorter than first or both have the same number of elements
-            return __comp(*__result, *(__first2 + (__result - __first1)));
-        }
-    }
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_ALGORITHM_IMPL_H */
diff --git a/libcxx/include/pstl/internal/glue_algorithm_impl.h b/libcxx/include/pstl/internal/glue_algorithm_impl.h
deleted file mode 100644
index fb0c19d776658..0000000000000
--- a/libcxx/include/pstl/internal/glue_algorithm_impl.h
+++ /dev/null
@@ -1,1108 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_ALGORITHM_IMPL_H
-#define _PSTL_GLUE_ALGORITHM_IMPL_H
-
-#include <functional>
-
-#include "pstl_config.h"
-
-#include "execution_defs.h"
-#include "utils.h"
-#include "algorithm_fwd.h"
-#include "numeric_fwd.h" /* count and count_if use __pattern_transform_reduce */
-
-#include "execution_impl.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [alg.any_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_any_of(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __pred);
-}
-
-// [alg.all_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Pred>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-all_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred)
-{
-    return !std::any_of(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::not_fn(__pred));
-}
-
-// [alg.none_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-none_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    return !std::any_of(std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
-}
-
-// [alg.foreach]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-for_each(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __f);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-for_each_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, _Function __f)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_walk1_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                                                 __f);
-}
-
-// [alg.find]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_find_if(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                 __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if_not(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    return std::find_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::not_fn(__pred));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    return std::find_if(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                        __pstl::__internal::__equal_value<_Tp>(__value));
-}
-
-// [alg.find.end]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
-
-    return __pstl::__internal::__pattern_find_end(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                  __last, __s_first, __s_last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last)
-{
-    return std::find_end(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last,
-                         std::equal_to<>());
-}
-
-// [alg.find_first_of]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
-
-    return __pstl::__internal::__pattern_find_first_of(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __s_first, __s_last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last)
-{
-    return std::find_first_of(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last,
-                              std::equal_to<>());
-}
-
-// [alg.adjacent_find]
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, std::equal_to<_ValueType>(), /*first_semantic*/ false);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __pred, /*first_semantic*/ false);
-}
-
-// [alg.count]
-
-// Implementation note: count and count_if call the pattern directly instead of calling std::transform_reduce
-// so that we do not have to include <numeric>.
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::difference_type>
-count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return __pstl::__internal::__pattern_count(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                               [&__value](const _ValueType& __x) { return __value == __x; });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::difference_type>
-count_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_count(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                               __pred);
-}
-
-// [alg.search]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
-
-    return __pstl::__internal::__pattern_search(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __s_first, __s_last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last)
-{
-    return std::search(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_search_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                  __last, __count, __value, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value)
-{
-    return std::search_n(std::forward<_ExecutionPolicy>(__exec), __first, __last, __count, __value,
-                         std::equal_to<typename iterator_traits<_ForwardIterator>::value_type>());
-}
-
-// [alg.copy]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__pattern_walk2_brick(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-        [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _ForwardIterator2 __res)
-        { return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{}); });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_n(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _Size __n, _ForwardIterator2 __result)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__pattern_walk2_brick_n(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-        [](_ForwardIterator1 __begin, _Size __sz, _ForwardIterator2 __res)
-        { return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{}); });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-        _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_copy_if(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                 __last, __result, __pred);
-}
-
-// [alg.swap]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-swap_ranges(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-            _ForwardIterator2 __first2)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_walk2(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2,
-                                               [](_ReferenceType1 __x, _ReferenceType2 __y)
-                                               {
-                                                   using std::swap;
-                                                   swap(__x, __y);
-                                               });
-}
-
-// [alg.transform]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-          _UnaryOperation __op)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _InputType;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _OutputType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_walk2(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                               __result,
-                                               [__op](_InputType __x, _OutputType __y) mutable { __y = __op(__x); });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator __result, _BinaryOperation __op)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _Input1Type;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _Input2Type;
-    typedef typename iterator_traits<_ForwardIterator>::reference _OutputType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_walk3(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __result,
-        [__op](_Input1Type x, _Input2Type y, _OutputType z) mutable { z = __op(x, y); });
-}
-
-// [alg.replace]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-           const _Tp& __new_value)
-{
-    typedef typename iterator_traits<_ForwardIterator>::reference _ElementType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                        [&__pred, &__new_value](_ElementType __elem)
-                                        {
-                                            if (__pred(__elem))
-                                            {
-                                                __elem = __new_value;
-                                            }
-                                        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value,
-        const _Tp& __new_value)
-{
-    std::replace_if(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    __pstl::__internal::__equal_value<_Tp>(__old_value), __new_value);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                _ForwardIterator2 __result, _UnaryPredicate __pred, const _Tp& __new_value)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _InputType;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _OutputType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_walk2(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-        [__pred, &__new_value](_InputType __x, _OutputType __y) mutable { __y = __pred(__x) ? __new_value : __x; });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-             const _Tp& __old_value, const _Tp& __new_value)
-{
-    return std::replace_copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                                __pstl::__internal::__equal_value<_Tp>(__old_value), __new_value);
-}
-
-// [alg.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_fill(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                       __value);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, const _Tp& __value)
-{
-    if (__count <= 0)
-        return __first;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_fill_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                __count, __value);
-}
-
-// [alg.generate]
-template <class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_generate(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                           __g);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Generator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-generate_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, _Generator __g)
-{
-    if (__count <= 0)
-        return __first;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_generate_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                    __count, __g);
-}
-
-// [alg.remove]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Predicate __pred)
-{
-    return std::copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, std::not_fn(__pred));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-            const _Tp& __value)
-{
-    return std::copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                        __pstl::__internal::__not_equal_value<_Tp>(__value));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_remove_if(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                   __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-remove(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    return std::remove_if(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                          __pstl::__internal::__equal_value<_Tp>(__value));
-}
-
-// [alg.unique]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_unique(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    return std::unique(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-            _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_unique_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __last, __result, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result)
-{
-    return std::unique_copy(__exec, __first, __last, __result, std::equal_to<>());
-}
-
-// [alg.reverse]
-
-template <class _ExecutionPolicy, class _BidirectionalIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_reverse(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last);
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-             _ForwardIterator __d_first)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    return __pstl::__internal::__pattern_reverse_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                      __last, __d_first);
-}
-
-// [alg.rotate]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_rotate(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                __middle, __last);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __middle, _ForwardIterator1 __last,
-            _ForwardIterator2 __result)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_rotate_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __middle, __last, __result);
-}
-
-// [alg.partitions]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_is_partitioned(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_partition(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                   __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator>
-stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                 _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_stable_partition(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first, __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _ForwardIterator1, class _ForwardIterator2,
-          class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-partition_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-               _ForwardIterator1 __out_true, _ForwardIterator2 __out_false, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __out_true, __out_false);
-
-    return __pstl::__internal::__pattern_partition_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __out_true, __out_false, __pred);
-}
-
-// [alg.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    return __pstl::__internal::__pattern_sort(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                              __comp, typename std::is_move_constructible<_InputType>::type());
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-// [stable.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_stable_sort(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::stable_sort(__exec, __first, __last, std::less<_InputType>());
-}
-
-// [mismatch]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_mismatch(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                  __last1, __first2, __last2, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _BinaryPredicate __pred)
-{
-    return std::mismatch(__exec, __first1, __last1, __first2, std::next(__first2, std::distance(__first1, __last1)),
-                         __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2)
-{
-    return std::mismatch(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2,
-                         std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2)
-{
-    //TODO: to get rid of "distance"
-    return std::mismatch(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2,
-                         std::next(__first2, std::distance(__first1, __last1)));
-}
-
-// [alg.equal]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _BinaryPredicate __p)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_equal(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2, __p);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2)
-{
-    return std::equal(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _BinaryPredicate __p)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_equal(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2, __last2, __p);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2)
-{
-    return equal(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::equal_to<>());
-}
-
-// [alg.move]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-move(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__pattern_walk2_brick(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first,
-        [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _ForwardIterator2 __res)
-        { return __pstl::__internal::__brick_move(__begin, __end, __res, __is_vector{}); });
-}
-
-// [partial.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_partial_sort(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                               __middle, __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last)
-{
-    typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::partial_sort(__exec, __first, __middle, __last, std::less<_InputType>());
-}
-
-// [partial.sort.copy]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    return __pstl::__internal::__pattern_partial_sort_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                           __first, __last, __d_first, __d_last, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last)
-{
-    return std::partial_sort_copy(std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, __d_last,
-                                  std::less<>());
-}
-
-// [is.sorted]
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    const _ForwardIterator __res =
-        __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                    __last, __pstl::__internal::__reorder_pred<_Compare>(__comp),
-                                                    /*first_semantic*/ false);
-    return __res == __last ? __last : std::next(__res);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return is_sorted_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __pstl::__internal::__reorder_pred<_Compare>(__comp),
-                                                       /*or_semantic*/ true) == __last;
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return std::is_sorted(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-// [alg.merge]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __d_first);
-
-    return __pstl::__internal::__pattern_merge(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2, __last2, __d_first, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first)
-{
-    return std::merge(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first,
-                      std::less<>());
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_inplace_merge(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                __middle, __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last)
-{
-    typedef typename std::iterator_traits<_BidirectionalIterator>::value_type _InputType;
-    std::inplace_merge(__exec, __first, __middle, __last, std::less<_InputType>());
-}
-
-// [includes]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_includes(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                  __last1, __first2, __last2, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2)
-{
-    return std::includes(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::less<>());
-}
-
-// [set.union]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_union(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                   __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_union(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                          std::less<>());
-}
-
-// [set.intersection]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_intersection(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_intersection(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                                 std::less<>());
-}
-
-// [set.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_difference(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                        __first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_difference(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                               std::less<>());
-}
-
-// [set.symmetric.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result,
-                         _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_symmetric_difference(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_symmetric_difference(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2,
-                                         __result, std::less<>());
-}
-
-// [is.heap]
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_is_heap_until(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    return std::is_heap_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    return std::is_heap_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp) == __last;
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    return std::is_heap(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-// [alg.min.max]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_min_element(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return std::min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    return min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                       __pstl::__internal::__reorder_pred<_Compare>(__comp));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return std::min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                            __pstl::__internal::__reorder_pred<std::less<_InputType>>(std::less<_InputType>()));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
-minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_minmax_element(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
-minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return std::minmax_element(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_ValueType>());
-}
-
-// [alg.nth.element]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_nth_element(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __nth,
-                                              __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last)
-{
-    typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::nth_element(std::forward<_ExecutionPolicy>(__exec), __first, __nth, __last, std::less<_InputType>());
-}
-
-// [alg.lex.comparison]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_lexicographical_compare(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                                 __first1, __last1, __first2, __last2, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2)
-{
-    return std::lexicographical_compare(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2,
-                                        std::less<>());
-}
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_ALGORITHM_IMPL_H */
diff --git a/libcxx/include/pstl/internal/glue_execution_defs.h b/libcxx/include/pstl/internal/glue_execution_defs.h
deleted file mode 100644
index df9a477f721ea..0000000000000
--- a/libcxx/include/pstl/internal/glue_execution_defs.h
+++ /dev/null
@@ -1,55 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_EXECUTION_DEFS_H
-#define _PSTL_GLUE_EXECUTION_DEFS_H
-
-#include <type_traits>
-
-#include "execution_defs.h"
-#include "pstl_config.h"
-
-namespace std
-{
-// Type trait
-using __pstl::execution::is_execution_policy;
-#if defined(_PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT)
-#    if defined(__INTEL_COMPILER)
-template <class T>
-constexpr bool is_execution_policy_v = is_execution_policy<T>::value;
-#    else
-using __pstl::execution::is_execution_policy_v;
-#    endif
-#endif
-
-namespace execution
-{
-// Standard C++ policy classes
-using __pstl::execution::parallel_policy;
-using __pstl::execution::parallel_unsequenced_policy;
-using __pstl::execution::sequenced_policy;
-
-// Standard predefined policy instances
-using __pstl::execution::par;
-using __pstl::execution::par_unseq;
-using __pstl::execution::seq;
-
-// Implementation-defined names
-// Unsequenced policy is not yet standard, but for consistency
-// we include it into namespace std::execution as well
-using __pstl::execution::unseq;
-using __pstl::execution::unsequenced_policy;
-} // namespace execution
-} // namespace std
-
-#include "algorithm_impl.h"
-#include "numeric_impl.h"
-#include "parallel_backend.h"
-
-#endif /* _PSTL_GLUE_EXECUTION_DEFS_H */
diff --git a/libcxx/include/pstl/internal/glue_memory_impl.h b/libcxx/include/pstl/internal/glue_memory_impl.h
deleted file mode 100644
index 39c595d697a28..0000000000000
--- a/libcxx/include/pstl/internal/glue_memory_impl.h
+++ /dev/null
@@ -1,352 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_MEMORY_IMPL_H
-#define _PSTL_GLUE_MEMORY_IMPL_H
-
-#include "pstl_config.h"
-
-#include "execution_defs.h"
-#include "utils.h"
-#include "algorithm_fwd.h"
-
-#include "execution_impl.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [uninitialized.copy]
-
-template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_copy(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                [](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __result,
-                                                       [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                                                       { ::new (std::addressof(__val2)) _ValueType2(__val1); });
-        });
-}
-
-template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-                [](_InputIterator __begin, _Size __sz, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                         __first, __n, __result,
-                                                         [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                                                         { ::new (std::addressof(__val2)) _ValueType2(__val1); });
-        });
-}
-
-// [uninitialized.move]
-
-template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_move(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                [](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                { ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); });
-        });
-}
-
-template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-                [](_InputIterator __begin, _Size __sz, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-                [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                { ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); });
-        });
-}
-
-// [uninitialized.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    __pstl::__internal::__invoke_if_else(
-        std::is_arithmetic<_ValueType>(),
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [&__value](_ForwardIterator __begin, _ForwardIterator __end)
-                { __pstl::__internal::__brick_fill(__begin, __end, _ValueType(__value), __is_vector{}); });
-        },
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [&__value](_ReferenceType __val)
-                                                { ::new (std::addressof(__val)) _ValueType(__value); });
-        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_arithmetic<_ValueType>(),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [&__value](_ForwardIterator __begin, _Size __count)
-                { return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(__value), __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [&__value](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(__value); });
-        });
-}
-
-// [specialized.destroy]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__invoke_if_not(std::is_trivially_destructible<_ValueType>(),
-                                        [&]()
-                                        {
-                                            __pstl::__internal::__pattern_walk1(
-                                                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [](_ReferenceType __val) { __val.~_ValueType(); });
-                                        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_trivially_destructible<_ValueType>(), [&]() { return std::next(__first, __n); },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                         __first, __n,
-                                                         [](_ReferenceType __val) { __val.~_ValueType(); });
-        });
-}
-
-// [uninitialized.construct.default]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__invoke_if_not(std::is_trivial<_ValueType>(),
-                                        [&]()
-                                        {
-                                            __pstl::__internal::__pattern_walk1(
-                                                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; });
-                                        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_trivial<_ValueType>(), [&]() { return std::next(__first, __n); },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; });
-        });
-}
-
-// [uninitialized.construct.value]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    __pstl::__internal::__invoke_if_else(
-        std::is_trivial<_ValueType>(),
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [](_ForwardIterator __begin, _ForwardIterator __end)
-                { __pstl::__internal::__brick_fill(__begin, __end, _ValueType(), __is_vector{}); });
-        },
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [](_ReferenceType __val)
-                                                { ::new (std::addressof(__val)) _ValueType(); });
-        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_trivial<_ValueType>(),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [](_ForwardIterator __begin, _Size __count)
-                { return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(), __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(); });
-        });
-}
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_MEMORY_IMPL_H */
diff --git a/libcxx/include/pstl/internal/glue_numeric_defs.h b/libcxx/include/pstl/internal/glue_numeric_defs.h
deleted file mode 100644
index 86cd38b34b137..0000000000000
--- a/libcxx/include/pstl/internal/glue_numeric_defs.h
+++ /dev/null
@@ -1,124 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_NUMERIC_DEFS_H
-#define _PSTL_GLUE_NUMERIC_DEFS_H
-
-#include <iterator>
-
-#include "execution_defs.h"
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-// [reduce]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-       _BinaryOperation __binary_op);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::value_type>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1,
-                 _BinaryOperation2 __binary_op2);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-                 _BinaryOperation __binary_op, _UnaryOperation __unary_op);
-
-// [exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op);
-
-// [inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op, _Tp __init);
-
-// [transform.exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op,
-                         _UnaryOperation __unary_op);
-
-// [transform.inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation,
-          class _UnaryOperation, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op,
-                         _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op);
-
-// [adjacent.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first, _BinaryOperation op);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first);
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_NUMERIC_DEFS_H */
diff --git a/libcxx/include/pstl/internal/glue_numeric_impl.h b/libcxx/include/pstl/internal/glue_numeric_impl.h
deleted file mode 100644
index ad268b51a16c9..0000000000000
--- a/libcxx/include/pstl/internal/glue_numeric_impl.h
+++ /dev/null
@@ -1,232 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_NUMERIC_IMPL_H
-#define _PSTL_GLUE_NUMERIC_IMPL_H
-
-#include <functional>
-
-#include "pstl_config.h"
-
-#include "utils.h"
-#include "numeric_fwd.h"
-#include "execution_impl.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [reduce]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-       _BinaryOperation __binary_op)
-{
-    return transform_reduce(std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, __binary_op,
-                            __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init)
-{
-    return transform_reduce(std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, std::plus<_Tp>(),
-                            __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::value_type>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return transform_reduce(std::forward<_ExecutionPolicy>(__exec), __first, __last, _ValueType{},
-                            std::plus<_ValueType>(), __pstl::__internal::__no_op());
-}
-
-// [transform.reduce]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
-    return __pstl::__internal::__pattern_transform_reduce(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first1, __last1, __first2, __init, std::plus<_InputType>(),
-                                                          std::multiplies<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-    return __pstl::__internal::__pattern_transform_reduce(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first1, __last1, __first2, __init, __binary_op1,
-                                                          __binary_op2);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-                 _BinaryOperation __binary_op, _UnaryOperation __unary_op)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_transform_reduce(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first, __last, __init, __binary_op, __unary_op);
-}
-
-// [exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using namespace __pstl;
-    return __internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __result, __pstl::__internal::__no_op(), __init, std::plus<_Tp>(),
-                                                /*inclusive=*/std::false_type());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using namespace __pstl;
-    return __internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __result, __pstl::__internal::__no_op(), __init, __binary_op,
-                                                /*inclusive=*/std::false_type());
-}
-
-// [inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
-    return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                                    std::plus<_InputType>(), __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op)
-{
-    return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __binary_op,
-                                    __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op, _Tp __init)
-{
-    return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __binary_op,
-                                    __pstl::__internal::__no_op(), __init);
-}
-
-// [transform.exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op,
-                         _UnaryOperation __unary_op)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __result, __unary_op, __init, __binary_op,
-                                                        /*inclusive=*/std::false_type());
-}
-
-// [transform.inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation,
-          class _UnaryOperation, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op,
-                         _Tp __init)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __result, __unary_op, __init, __binary_op,
-                                                        /*inclusive=*/std::true_type());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op)
-{
-    if (__first != __last)
-    {
-        auto __tmp = __unary_op(*__first);
-        *__result = __tmp;
-        return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), ++__first, __last, ++__result,
-                                        __binary_op, __unary_op, __tmp);
-    }
-    else
-    {
-        return __result;
-    }
-}
-
-// [adjacent.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first, _BinaryOperation __op)
-{
-
-    if (__first == __last)
-        return __d_first;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    return __pstl::__internal::__pattern_adjacent_difference(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                             __first, __last, __d_first, __op);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::value_type _ValueType;
-    return adjacent_difference(std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first,
-                               std::minus<_ValueType>());
-}
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_NUMERIC_IMPL_H_ */
diff --git a/libcxx/include/pstl/internal/memory_impl.h b/libcxx/include/pstl/internal/memory_impl.h
deleted file mode 100644
index 942a30ec1a65f..0000000000000
--- a/libcxx/include/pstl/internal/memory_impl.h
+++ /dev/null
@@ -1,112 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_MEMORY_IMPL_H
-#define _PSTL_MEMORY_IMPL_H
-
-#include <iterator>
-
-#include "pstl_config.h"
-#include "unseq_backend_simd.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// uninitialized_move
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                           /*vector=*/std::false_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    for (; __first != __last; ++__first, ++__result)
-    {
-        ::new (std::addressof(*__result)) _ValueType(std::move(*__first));
-    }
-    return __result;
-}
-
-template <typename _RandomAccessIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_move(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                           /*vector=*/std::true_type) noexcept
-{
-    using __ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
-    using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
-
-    return __unseq_backend::__simd_walk_2(
-        __first, __last - __first, __result,
-        [](_ReferenceType1 __x, _ReferenceType2 __y) { ::new (std::addressof(__y)) __ValueType(std::move(__x)); });
-}
-
-template <typename _Iterator>
-void
-__brick_destroy(_Iterator __first, _Iterator __last, /*vector*/ std::false_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_Iterator>::value_type;
-
-    for (; __first != __last; ++__first)
-        __first->~_ValueType();
-}
-
-template <typename _RandomAccessIterator>
-void
-__brick_destroy(_RandomAccessIterator __first, _RandomAccessIterator __last, /*vector*/ std::true_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_RandomAccessIterator>::value_type;
-    using _ReferenceType = typename std::iterator_traits<_RandomAccessIterator>::reference;
-
-    __unseq_backend::__simd_walk_1(__first, __last - __first, [](_ReferenceType __x) { __x.~_ValueType(); });
-}
-
-//------------------------------------------------------------------------
-// uninitialized copy
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                           /*vector=*/std::false_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    for (; __first != __last; ++__first, ++__result)
-    {
-        ::new (std::addressof(*__result)) _ValueType(*__first);
-    }
-    return __result;
-}
-
-template <typename _RandomAccessIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                           /*vector=*/std::true_type) noexcept
-{
-    using __ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
-    using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
-
-    return __unseq_backend::__simd_walk_2(
-        __first, __last - __first, __result,
-        [](_ReferenceType1 __x, _ReferenceType2 __y) { ::new (std::addressof(__y)) __ValueType(__x); });
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_MEMORY_IMPL_H */
diff --git a/libcxx/include/pstl/internal/numeric_fwd.h b/libcxx/include/pstl/internal/numeric_fwd.h
deleted file mode 100644
index 7f7845bd3e260..0000000000000
--- a/libcxx/include/pstl/internal/numeric_fwd.h
+++ /dev/null
@@ -1,139 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_NUMERIC_FWD_H
-#define _PSTL_NUMERIC_FWD_H
-
-#include <type_traits>
-#include <utility>
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// transform_reduce (version with two binary functions, according to draft N4659)
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-_Tp __brick_transform_reduce(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _Tp,
-                             _BinaryOperation1, _BinaryOperation2,
-                             /*__is_vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp __brick_transform_reduce(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Tp, _BinaryOperation1,
-                             _BinaryOperation2,
-                             /*__is_vector=*/std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp,
-          class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Tp,
-                           _BinaryOperation1, _BinaryOperation2) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                           _RandomAccessIterator1, _RandomAccessIterator2, _Tp, _BinaryOperation1, _BinaryOperation2);
-
-//------------------------------------------------------------------------
-// transform_reduce (version with unary and binary functions)
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
-_Tp __brick_transform_reduce(_RandomAccessIterator, _RandomAccessIterator, _Tp, _BinaryOperation, _UnaryOperation,
-                             /*is_vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-_Tp __brick_transform_reduce(_ForwardIterator, _ForwardIterator, _Tp, _BinaryOperation, _UnaryOperation,
-                             /*is_vector=*/std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Tp, _BinaryOperation,
-                           _UnaryOperation) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                           _Tp, _BinaryOperation, _UnaryOperation);
-
-//------------------------------------------------------------------------
-// transform_exclusive_scan
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp> __brick_transform_scan(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                                                       _UnaryOperation, _Tp, _BinaryOperation,
-                                                       /*Inclusive*/ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp> __brick_transform_scan(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                                       _UnaryOperation, _Tp, _BinaryOperation,
-                                                       /*Inclusive*/ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryOperation,
-          class _Tp, class _BinaryOperation, class _Inclusive>
-_OutputIterator
-__pattern_transform_scan(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator, _UnaryOperation,
-                         _Tp, _BinaryOperation, _Inclusive) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&&, _RandomAccessIterator,
-                         _RandomAccessIterator, _OutputIterator, _UnaryOperation, _Tp, _BinaryOperation, _Inclusive);
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _OutputIterator, _UnaryOperation, _Tp, _BinaryOperation, _Inclusive);
-
-//------------------------------------------------------------------------
-// adjacent_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator __brick_adjacent_difference(_ForwardIterator, _ForwardIterator, _OutputIterator, _BinaryOperation,
-                                            /*is_vector*/ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator __brick_adjacent_difference(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                            _BinaryOperation,
-                                            /*is_vector*/ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator
-__pattern_adjacent_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                              _BinaryOperation) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _BinaryOperation>
-_OutputIterator
-__pattern_adjacent_difference(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator,
-                              _RandomAccessIterator, _OutputIterator, _BinaryOperation);
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_NUMERIC_FWD_H */
diff --git a/libcxx/include/pstl/internal/numeric_impl.h b/libcxx/include/pstl/internal/numeric_impl.h
deleted file mode 100644
index a0387aef50b00..0000000000000
--- a/libcxx/include/pstl/internal/numeric_impl.h
+++ /dev/null
@@ -1,383 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_NUMERIC_IMPL_H
-#define _PSTL_NUMERIC_IMPL_H
-
-#include <iterator>
-#include <type_traits>
-#include <numeric>
-
-#include "parallel_backend.h"
-#include "pstl_config.h"
-#include "execution_impl.h"
-#include "unseq_backend_simd.h"
-#include "algorithm_fwd.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// transform_reduce (version with two binary functions, according to draft N4659)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__brick_transform_reduce(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Tp __init,
-                         _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2,
-                         /*is_vector=*/std::false_type) noexcept
-{
-    return std::inner_product(__first1, __last1, __first2, __init, __binary_op1, __binary_op2);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-_Tp
-__brick_transform_reduce(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                         _RandomAccessIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1,
-                         _BinaryOperation2 __binary_op2,
-                         /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    return __unseq_backend::__simd_transform_reduce(
-        __last1 - __first1, __init, __binary_op1,
-        [=, &__binary_op2](_DifferenceType __i) { return __binary_op2(__first1[__i], __first2[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp,
-          class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                           _ForwardIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1,
-                           _BinaryOperation2 __binary_op2) noexcept
-{
-    return __brick_transform_reduce(__first1, __last1, __first2, __init, __binary_op1, __binary_op2,
-                                    typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                           _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Tp __init,
-                           _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_transform_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __binary_op2](_RandomAccessIterator1 __i) mutable
-                { return __binary_op2(*__i, *(__first2 + (__i - __first1))); },
-                __init,
-                __binary_op1, // Combine
-                [__first1, __first2, __binary_op1, __binary_op2](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j,
-                                                                 _Tp __init) -> _Tp
-                {
-                    return __internal::__brick_transform_reduce(__i, __j, __first2 + (__i - __first1), __init,
-                                                                __binary_op1, __binary_op2, _IsVector{});
-                });
-        });
-}
-
-//------------------------------------------------------------------------
-// transform_reduce (version with unary and binary functions)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-_Tp
-__brick_transform_reduce(_ForwardIterator __first, _ForwardIterator __last, _Tp __init, _BinaryOperation __binary_op,
-                         _UnaryOperation __unary_op, /*is_vector=*/std::false_type) noexcept
-{
-    return std::transform_reduce(__first, __last, __init, __binary_op, __unary_op);
-}
-
-template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
-_Tp
-__brick_transform_reduce(_RandomAccessIterator __first, _RandomAccessIterator __last, _Tp __init,
-                         _BinaryOperation __binary_op, _UnaryOperation __unary_op,
-                         /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-    return __unseq_backend::__simd_transform_reduce(
-        __last - __first, __init, __binary_op,
-        [=, &__unary_op](_DifferenceType __i) { return __unary_op(__first[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-                           _BinaryOperation __binary_op, _UnaryOperation __unary_op) noexcept
-{
-    return __internal::__brick_transform_reduce(__first, __last, __init, __binary_op, __unary_op,
-                                                typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                           _RandomAccessIterator __last, _Tp __init, _BinaryOperation __binary_op,
-                           _UnaryOperation __unary_op)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_transform_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__unary_op](_RandomAccessIterator __i) mutable { return __unary_op(*__i); }, __init, __binary_op,
-                [__unary_op, __binary_op](_RandomAccessIterator __i, _RandomAccessIterator __j, _Tp __init) {
-                    return __internal::__brick_transform_reduce(__i, __j, __init, __binary_op, __unary_op, _IsVector{});
-                });
-        });
-}
-
-//------------------------------------------------------------------------
-// transform_exclusive_scan
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-
-// Exclusive form
-template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp>
-__brick_transform_scan(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
-                       /*Inclusive*/ std::false_type, /*is_vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__result)
-    {
-        *__result = __init;
-        _PSTL_PRAGMA_FORCEINLINE
-        __init = __binary_op(__init, __unary_op(*__first));
-    }
-    return std::make_pair(__result, __init);
-}
-
-// Inclusive form
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp>
-__brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
-                       /*Inclusive*/ std::true_type, /*is_vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__result)
-    {
-        _PSTL_PRAGMA_FORCEINLINE
-        __init = __binary_op(__init, __unary_op(*__first));
-        *__result = __init;
-    }
-    return std::make_pair(__result, __init);
-}
-
-// type is arithmetic and binary operation is a user defined operation.
-template <typename _Tp, typename _BinaryOperation>
-using is_arithmetic_udop = std::integral_constant<bool, std::is_arithmetic<_Tp>::value &&
-                                                            !std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
-
-// [restriction] - T shall be DefaultConstructible.
-// [violation] - default ctor of T shall set the identity value for binary_op.
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation,
-          class _Inclusive>
-typename std::enable_if<!is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op, _Inclusive,
-                       /*is_vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_UDS_PRESENT)
-    return __unseq_backend::__simd_scan(__first, __last - __first, __result, __unary_op, __init, __binary_op,
-                                        _Inclusive());
-#else
-    // We need to call serial brick here to call function for inclusive and exclusive scan that depends on _Inclusive() value
-    return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(),
-                                              /*is_vector=*/std::false_type());
-#endif
-}
-
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation,
-          class _Inclusive>
-typename std::enable_if<is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op, _Inclusive,
-                       /*is_vector=*/std::true_type) noexcept
-{
-    return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(),
-                                              /*is_vector=*/std::false_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryOperation,
-          class _Tp, class _BinaryOperation, class _Inclusive>
-_OutputIterator
-__pattern_transform_scan(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
-                         _Inclusive) noexcept
-{
-    return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(),
-                                              typename _Tag::__is_vector{})
-        .first;
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-                         _BinaryOperation __binary_op, _Inclusive)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_transform_scan(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __last - __first,
-                [__first, __unary_op](_DifferenceType __i) mutable { return __unary_op(__first[__i]); }, __init,
-                __binary_op,
-                [__first, __unary_op, __binary_op](_DifferenceType __i, _DifferenceType __j, _Tp __init)
-                {
-                    // Execute serial __brick_transform_reduce, due to the explicit SIMD vectorization (reduction) requires a commutative operation for the guarantee of correct scan.
-                    return __internal::__brick_transform_reduce(__first + __i, __first + __j, __init, __binary_op,
-                                                                __unary_op,
-                                                                /*__is_vector*/ std::false_type());
-                },
-                [__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __j, _Tp __init)
-                {
-                    return __internal::__brick_transform_scan(__first + __i, __first + __j, __result + __i, __unary_op,
-                                                              __init, __binary_op, _Inclusive(), _IsVector{})
-                        .second;
-                });
-            return __result + (__last - __first);
-        });
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-                         _BinaryOperation __binary_op, _Inclusive)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-    _DifferenceType __n = __last - __first;
-
-    if (__n <= 0)
-    {
-        return __result;
-    }
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_strict_scan(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, __init,
-                [__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __len)
-                {
-                    return __internal::__brick_transform_scan(__first + __i, __first + (__i + __len), __result + __i,
-                                                              __unary_op, _Tp{}, __binary_op, _Inclusive(), _IsVector{})
-                        .second;
-                },
-                __binary_op,
-                [__result, &__binary_op](_DifferenceType __i, _DifferenceType __len, _Tp __initial)
-                {
-                    return *(std::transform(__result + __i, __result + __i + __len, __result + __i,
-                                            [&__initial, &__binary_op](const _Tp& __x)
-                                            {
-                                                _PSTL_PRAGMA_FORCEINLINE
-                                                return __binary_op(__initial, __x);
-                                            }) -
-                             1);
-                },
-                [](_Tp) {});
-            return __result + (__last - __first);
-        });
-}
-
-//------------------------------------------------------------------------
-// adjacent_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator
-__brick_adjacent_difference(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __d_first,
-                            _BinaryOperation __op, /*is_vector*/ std::false_type) noexcept
-{
-    return std::adjacent_difference(__first, __last, __d_first, __op);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class BinaryOperation>
-_RandomAccessIterator2
-__brick_adjacent_difference(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last,
-                            _RandomAccessIterator2 __d_first, BinaryOperation __op,
-                            /*is_vector=*/std::true_type) noexcept
-{
-    _PSTL_ASSERT(__first != __last);
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
-
-    auto __n = __last - __first;
-    *__d_first = *__first;
-    return __unseq_backend::__simd_walk_3(
-        __first + 1, __n - 1, __first, __d_first + 1,
-        [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__x, __y); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator
-__pattern_adjacent_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                              _OutputIterator __d_first, _BinaryOperation __op) noexcept
-{
-    return __internal::__brick_adjacent_difference(__first, __last, __d_first, __op, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryOperation>
-_RandomAccessIterator2
-__pattern_adjacent_difference(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec,
-                              _RandomAccessIterator1 __first, _RandomAccessIterator1 __last,
-                              _RandomAccessIterator2 __d_first, _BinaryOperation __op)
-{
-    _PSTL_ASSERT(__first != __last);
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    *__d_first = *__first;
-    __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last - 1,
-                                  [&__op, __d_first, __first](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e)
-                                  {
-                                      _RandomAccessIterator2 __d_b = __d_first + (__b - __first);
-                                      __internal::__brick_walk3(
-                                          __b, __e, __b + 1, __d_b + 1,
-                                          [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z)
-                                          { __z = __op(__y, __x); },
-                                          _IsVector{});
-                                  });
-    return __d_first + (__last - __first);
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_NUMERIC_IMPL_H */
diff --git a/libcxx/include/pstl/internal/pstl_config.h b/libcxx/include/pstl/internal/pstl_config.h
deleted file mode 100644
index de89e7bd977e7..0000000000000
--- a/libcxx/include/pstl/internal/pstl_config.h
+++ /dev/null
@@ -1,204 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_CONFIG_H
-#define _PSTL_CONFIG_H
-
-#include <__pstl_config_site>
-
-// The version is XYYZ, where X is major, YY is minor, and Z is patch (i.e. X.YY.Z)
-#define _PSTL_VERSION 16000
-#define _PSTL_VERSION_MAJOR (_PSTL_VERSION / 1000)
-#define _PSTL_VERSION_MINOR ((_PSTL_VERSION % 1000) / 10)
-#define _PSTL_VERSION_PATCH (_PSTL_VERSION % 10)
-
-#if !defined(_PSTL_PAR_BACKEND_SERIAL) && !defined(_PSTL_PAR_BACKEND_TBB) && !defined(_PSTL_PAR_BACKEND_OPENMP)
-#    error "A parallel backend must be specified"
-#endif
-
-// Check the user-defined macro for warnings
-#if defined(PSTL_USAGE_WARNINGS)
-#    define _PSTL_USAGE_WARNINGS
-#endif
-
-#if defined(_LIBCPP_VERSION)
-#    include <__assert>
-#    define _PSTL_ASSERT(pred) _LIBCPP_ASSERT(pred, "")
-#elif defined(__GLIBCXX__)
-#    define _PSTL_ASSERT(pred) __glibcxx_assert(pred)
-#else
-#    include <cassert>
-#    define _PSTL_ASSERT(pred) (assert((pred)))
-#endif
-
-// Portability "#pragma" definition
-#ifdef _MSC_VER
-#    define _PSTL_PRAGMA(x) __pragma(x)
-#else
-#    define _PSTL_PRAGMA(x) _Pragma(#    x)
-#endif
-
-#define _PSTL_STRING_AUX(x) #x
-#define _PSTL_STRING(x) _PSTL_STRING_AUX(x)
-#define _PSTL_STRING_CONCAT(x, y) x #y
-
-#ifdef _PSTL_HIDE_FROM_ABI_PER_TU
-#    define _PSTL_HIDE_FROM_ABI_PUSH                                                                                   \
-        _Pragma("clang attribute push(__attribute__((internal_linkage)), apply_to=any(function,record))")
-#    define _PSTL_HIDE_FROM_ABI_POP _Pragma("clang attribute pop")
-#else
-#    define _PSTL_HIDE_FROM_ABI_PUSH /* nothing */
-#    define _PSTL_HIDE_FROM_ABI_POP  /* nothing */
-#endif
-
-// note that when ICC or Clang is in use, _PSTL_GCC_VERSION might not fully match
-// the actual GCC version on the system.
-#define _PSTL_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
-
-#if defined(__clang__)
-// according to clang documentation, version can be vendor specific
-#    define _PSTL_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
-#endif
-
-// Enable SIMD for compilers that support OpenMP 4.0
-#if (defined(_OPENMP) && _OPENMP >= 201307) || \
-    (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1600) || \
-    (!defined(__INTEL_COMPILER) && _PSTL_GCC_VERSION >= 40900) || \
-    defined(__clang__)
-#    define _PSTL_PRAGMA_SIMD _PSTL_PRAGMA(omp simd)
-#    define _PSTL_PRAGMA_DECLARE_SIMD _PSTL_PRAGMA(omp declare simd)
-#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _PSTL_PRAGMA(omp simd reduction(PRM))
-#elif !defined(_MSC_VER) //#pragma simd
-#    define _PSTL_PRAGMA_SIMD _PSTL_PRAGMA(simd)
-#    define _PSTL_PRAGMA_DECLARE_SIMD
-#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _PSTL_PRAGMA(simd reduction(PRM))
-#else //no simd
-#    define _PSTL_PRAGMA_SIMD
-#    define _PSTL_PRAGMA_DECLARE_SIMD
-#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM)
-#endif //Enable SIMD
-
-#if defined(__INTEL_COMPILER)
-#    define _PSTL_PRAGMA_FORCEINLINE _PSTL_PRAGMA(forceinline)
-#else
-#    define _PSTL_PRAGMA_FORCEINLINE
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900
-#    define _PSTL_PRAGMA_SIMD_SCAN(PRM) _PSTL_PRAGMA(omp simd reduction(inscan, PRM))
-#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan inclusive(PRM))
-#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan exclusive(PRM))
-#else
-#    define _PSTL_PRAGMA_SIMD_SCAN(PRM)
-#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM)
-#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM)
-#endif
-
-// Should be defined to 1 for environments with a vendor implementation of C++17 execution policies
-#define _PSTL_CPP17_EXECUTION_POLICIES_PRESENT (_MSC_VER >= 1912 && _MSVC_LANG >= 201703L) ||                          \
-    (_GLIBCXX_RELEASE >= 9 && __GLIBCXX__ >= 20190503 && __cplusplus >= 201703L)
-
-#if (defined(_MSC_VER) && _MSC_VER >= 1900) || \
-    __cplusplus >= 201300L || \
-    __cpp_lib_robust_nonmodifying_seq_ops == 201304
-#   define _PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT
-#endif
-#if (defined(_MSC_VER) && _MSC_VER >= 1900) || \
-    __cplusplus >= 201402L || \
-    __cpp_lib_make_reverse_iterator == 201402
-#   define _PSTL_CPP14_MAKE_REVERSE_ITERATOR_PRESENT
-#endif
-#if (defined(_MSC_VER) && _MSC_VER >= 1900) || __cplusplus >= 201402L
-#   define _PSTL_CPP14_INTEGER_SEQUENCE_PRESENT
-#endif
-#if (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1700) || \
-    (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 190023918) || \
-    __cplusplus >= 201402L
-#   define _PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1800
-#   define _PSTL_EARLYEXIT_PRESENT
-#   define _PSTL_MONOTONIC_PRESENT
-#endif
-
-#if (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900) || \
-    (!defined(__INTEL_COMPILER) && _PSTL_GCC_VERSION >= 40900) || \
-    (defined(_OPENMP) && _OPENMP >= 201307)
-#    define _PSTL_UDR_PRESENT
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900 && __INTEL_COMPILER_BUILD_DATE >= 20180626
-#   define _PSTL_UDS_PRESENT
-#endif
-
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-#    define _PSTL_PRAGMA_SIMD_EARLYEXIT _PSTL_PRAGMA(omp simd early_exit)
-#else
-#    define _PSTL_PRAGMA_SIMD_EARLYEXIT
-#endif
-
-#if defined(_PSTL_MONOTONIC_PRESENT)
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(PRM) _PSTL_PRAGMA(omp ordered simd monotonic(PRM))
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(PRM1, PRM2) _PSTL_PRAGMA(omp ordered simd monotonic(PRM1, PRM2))
-#else
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(PRM)
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(PRM1, PRM2)
-#endif
-
-// Declaration of reduction functor, where
-// NAME - the name of the functor
-// OP - type of the callable object with the reduction operation
-// omp_in - refers to the local partial result
-// omp_out - refers to the final value of the combiner operator
-// omp_priv - refers to the private copy of the initial value
-// omp_orig - refers to the original variable to be reduced
-#define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)                                                                       \
-    _PSTL_PRAGMA(omp declare reduction(NAME:OP : omp_out(omp_in)) initializer(omp_priv = omp_orig))
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1600
-#    define _PSTL_PRAGMA_VECTOR_UNALIGNED _PSTL_PRAGMA(vector unaligned)
-#else
-#    define _PSTL_PRAGMA_VECTOR_UNALIGNED
-#endif
-
-// Check the user-defined macro to use non-temporal stores
-#if defined(PSTL_USE_NONTEMPORAL_STORES) && (__INTEL_COMPILER >= 1600)
-#    define _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED _PSTL_PRAGMA(vector nontemporal)
-#else
-#    define _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
-#endif
-
-#if defined(_MSC_VER) || defined(__INTEL_COMPILER) // the preprocessors don't type a message location
-#    define _PSTL_PRAGMA_LOCATION __FILE__ ":" _PSTL_STRING(__LINE__) ": [Parallel STL message]: "
-#else
-#    define _PSTL_PRAGMA_LOCATION " [Parallel STL message]: "
-#endif
-
-#define _PSTL_PRAGMA_MESSAGE_IMPL(x) _PSTL_PRAGMA(message(_PSTL_STRING_CONCAT(_PSTL_PRAGMA_LOCATION, x)))
-
-#if defined(_PSTL_USAGE_WARNINGS)
-#    define _PSTL_PRAGMA_MESSAGE(x) _PSTL_PRAGMA_MESSAGE_IMPL(x)
-#    define _PSTL_PRAGMA_MESSAGE_POLICIES(x) _PSTL_PRAGMA_MESSAGE_IMPL(x)
-#else
-#    define _PSTL_PRAGMA_MESSAGE(x)
-#    define _PSTL_PRAGMA_MESSAGE_POLICIES(x)
-#endif
-
-// broken macros
-#if (defined(__GLIBCXX__) && __GLIBCXX__ < 20150716) || \
-    (defined(_MSC_VER) && _MSC_VER < 1800)
-#   define _PSTL_CPP11_STD_ROTATE_BROKEN
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER == 1800
-#   define _PSTL_ICC_18_OMP_SIMD_BROKEN
-#endif
-
-#endif /* _PSTL_CONFIG_H */
diff --git a/libcxx/include/pstl/internal/utils.h b/libcxx/include/pstl/internal/utils.h
deleted file mode 100644
index ec9d46713a682..0000000000000
--- a/libcxx/include/pstl/internal/utils.h
+++ /dev/null
@@ -1,177 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_UTILS_H
-#define _PSTL_UTILS_H
-
-#include <new>
-#include <iterator>
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-template <typename _Fp>
-auto
-__except_handler(_Fp __f) -> decltype(__f())
-{
-    try
-    {
-        return __f();
-    }
-    catch (const std::bad_alloc&)
-    {
-        throw; // re-throw bad_alloc according to the standard [algorithms.parallel.exceptions]
-    }
-    catch (...)
-    {
-        std::terminate(); // Good bye according to the standard [algorithms.parallel.exceptions]
-    }
-}
-
-template <typename _Fp>
-void
-__invoke_if(std::true_type, _Fp __f)
-{
-    __f();
-}
-
-template <typename _Fp>
-void __invoke_if(std::false_type, _Fp)
-{
-}
-
-template <typename _Fp>
-void
-__invoke_if_not(std::false_type, _Fp __f)
-{
-    __f();
-}
-
-template <typename _Fp>
-void __invoke_if_not(std::true_type, _Fp)
-{
-}
-
-template <typename _F1, typename _F2>
-auto
-__invoke_if_else(std::true_type, _F1 __f1, _F2) -> decltype(__f1())
-{
-    return __f1();
-}
-
-template <typename _F1, typename _F2>
-auto
-__invoke_if_else(std::false_type, _F1, _F2 __f2) -> decltype(__f2())
-{
-    return __f2();
-}
-
-//! Unary operator that returns reference to its argument.
-struct __no_op
-{
-    template <typename _Tp>
-    _Tp&&
-    operator()(_Tp&& __a) const
-    {
-        return std::forward<_Tp>(__a);
-    }
-};
-
-template <typename _Pred>
-class __reorder_pred
-{
-    _Pred _M_pred;
-
-  public:
-    explicit __reorder_pred(_Pred __pred) : _M_pred(__pred) {}
-
-    template <typename _FTp, typename _STp>
-    bool
-    operator()(_FTp&& __a, _STp&& __b)
-    {
-        return _M_pred(std::forward<_STp>(__b), std::forward<_FTp>(__a));
-    }
-};
-
-//! Like a polymorphic lambda for pred(...,value)
-template <typename _Tp, typename _Predicate>
-class __equal_value_by_pred
-{
-    const _Tp& _M_value;
-    _Predicate _M_pred;
-
-  public:
-    __equal_value_by_pred(const _Tp& __value, _Predicate __pred) : _M_value(__value), _M_pred(__pred) {}
-
-    template <typename _Arg>
-    bool
-    operator()(_Arg&& __arg)
-    {
-        return _M_pred(std::forward<_Arg>(__arg), _M_value);
-    }
-};
-
-//! Like a polymorphic lambda for ==value
-template <typename _Tp>
-class __equal_value
-{
-    const _Tp& _M_value;
-
-  public:
-    explicit __equal_value(const _Tp& __value) : _M_value(__value) {}
-
-    template <typename _Arg>
-    bool
-    operator()(_Arg&& __arg) const
-    {
-        return std::forward<_Arg>(__arg) == _M_value;
-    }
-};
-
-//! Logical negation of ==value
-template <typename _Tp>
-class __not_equal_value
-{
-    const _Tp& _M_value;
-
-  public:
-    explicit __not_equal_value(const _Tp& __value) : _M_value(__value) {}
-
-    template <typename _Arg>
-    bool
-    operator()(_Arg&& __arg) const
-    {
-        return !(std::forward<_Arg>(__arg) == _M_value);
-    }
-};
-
-template <typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__cmp_iterators_by_values(_ForwardIterator __a, _ForwardIterator __b, _Compare __comp)
-{
-    if (__a < __b)
-    { // we should return closer iterator
-        return __comp(*__b, *__a) ? __b : __a;
-    }
-    else
-    {
-        return __comp(*__a, *__b) ? __a : __b;
-    }
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_UTILS_H */
diff --git a/libcxx/include/sstream b/libcxx/include/sstream
index 389bbf6aae1e3..6af2cb8fa87ff 100644
--- a/libcxx/include/sstream
+++ b/libcxx/include/sstream
@@ -187,6 +187,7 @@ typedef basic_stringstream<wchar_t> wstringstream;
 
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__config>
+#include <__fwd/sstream.h>
 #include <__utility/swap.h>
 #include <istream>
 #include <ostream>
diff --git a/libcxx/include/streambuf b/libcxx/include/streambuf
index d473df103475c..f491a14659096 100644
--- a/libcxx/include/streambuf
+++ b/libcxx/include/streambuf
@@ -109,6 +109,7 @@ protected:
 
 #include <__assert> // all public C++ headers provide the assertion handler
 #include <__config>
+#include <__fwd/streambuf.h>
 #include <cstdint>
 #include <ios>
 #include <iosfwd>
diff --git a/libcxx/include/uchar.h b/libcxx/include/uchar.h
index 3a51bb7a9f61f..546113f7eab46 100644
--- a/libcxx/include/uchar.h
+++ b/libcxx/include/uchar.h
@@ -42,10 +42,12 @@ size_t c32rtomb(char* s, char32_t c32, mbstate_t* ps);
 
 // Some platforms don't implement <uchar.h> and we don't want to give a hard
 // error on those platforms. When the platform doesn't provide <uchar.h>, at
-// least include <stddef.h> so we get the declaration for size_t.
+// least include <stddef.h> so we get the declaration for size_t, and try to
+// get the declaration of mbstate_t too.
 #if __has_include_next(<uchar.h>)
 # include_next <uchar.h>
 #else
+# include <__mbstate_t.h>
 # include <stddef.h>
 #endif
 
diff --git a/libcxx/include/utility b/libcxx/include/utility
index 1dc16b1158390..9ed7c6dac8ad2 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -84,14 +84,15 @@ struct pair
     explicit(see-below) constexpr pair();
     explicit(see-below) pair(const T1& x, const T2& y);                          // constexpr in C++14
     template <class U = T1, class V = T2> explicit(see-below) pair(U&&, V&&);    // constexpr in C++14
-    template <class U, class V> constexpr explicit(see below) pair(pair<U, V>&); // since C++23
+    template <class U, class V> constexpr explicit(see-below) pair(pair<U, V>&); // since C++23
     template <class U, class V> explicit(see-below) pair(const pair<U, V>& p);   // constexpr in C++14
     template <class U, class V> explicit(see-below) pair(pair<U, V>&& p);        // constexpr in C++14
     template <class U, class V>
-    constexpr explicit(see below) pair(const pair<U, V>&&);                      // since C++23
+    constexpr explicit(see-below) pair(const pair<U, V>&&);                      // since C++23
+    template <pair-like P> constexpr explicit(see-below) pair(P&&);              // since C++23
     template <class... Args1, class... Args2>
-        pair(piecewise_construct_t, tuple<Args1...> first_args,
-             tuple<Args2...> second_args);                                       // constexpr in C++20
+        pair(piecewise_construct_t, tuple<Args1...> first_args,                  // constexpr in C++20
+                                    tuple<Args2...> second_args);
 
     constexpr const pair& operator=(const pair& p) const;                        // since C++23
     template <class U, class V> pair& operator=(const pair<U, V>& p);            // constexpr in C++20
@@ -103,6 +104,8 @@ struct pair
     template <class U, class V> pair& operator=(pair<U, V>&& p);                 // constexpr in C++20
     template <class U, class V>
     constexpr const pair& operator=(pair<U, V>&& p) const;                       // since C++23
+    template <pair-like P> constexpr pair& operator=(P&&);                       // since C++23
+    template <pair-like P> constexpr const pair& operator=(P&&) const;           // since C++23
 
     void swap(pair& p) noexcept(is_nothrow_swappable_v<T1> &&
                                 is_nothrow_swappable_v<T2>);                     // constexpr in C++20
diff --git a/libcxx/include/valarray b/libcxx/include/valarray
index 76ae4a3f8c668..001e111a07d5b 100644
--- a/libcxx/include/valarray
+++ b/libcxx/include/valarray
@@ -4937,6 +4937,7 @@ _LIBCPP_POP_MACROS
 #  include <cstdlib>
 #  include <cstring>
 #  include <functional>
+#  include <stdexcept>
 #  include <type_traits>
 #endif
 
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 7dcbe1f8d0d49..2ee7f4891f1ab 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -1294,7 +1294,7 @@ using __best_match_t =
 } // namespace __variant_detail
 
 template <class... _Types>
-class _LIBCPP_TEMPLATE_VIS variant
+class _LIBCPP_TEMPLATE_VIS _LIBCPP_DECLSPEC_EMPTY_BASES variant
     : private __sfinae_ctor_base<
           __all<is_copy_constructible_v<_Types>...>::value,
           __all<is_move_constructible_v<_Types>...>::value>,
diff --git a/libcxx/include/vector b/libcxx/include/vector
index 40538f211d48f..d90eb1666c360 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -301,6 +301,7 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 #include <__iterator/wrap_iter.h>
 #include <__memory/addressof.h>
 #include <__memory/allocate_at_least.h>
+#include <__memory/allocator_traits.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/swap_allocator.h>
 #include <__memory/temp_value.h>
@@ -441,11 +442,11 @@ private:
       _LIBCPP_CONSTEXPR __destroy_vector(vector& __vec) : __vec_(__vec) {}
 
       _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() {
-          __vec_.__annotate_delete();
           std::__debug_db_erase_c(std::addressof(__vec_));
 
           if (__vec_.__begin_ != nullptr) {
             __vec_.__clear();
+            __vec_.__annotate_delete();
             __alloc_traits::deallocate(__vec_.__alloc(), __vec_.__begin_, __vec_.capacity());
           }
       }
@@ -745,8 +746,7 @@ private:
                                          const void *__old_mid,
                                          const void *__new_mid) const
     {
-
-      if (!__libcpp_is_constant_evaluated() && __beg && is_same<allocator_type, __default_allocator_type>::value)
+      if (!__libcpp_is_constant_evaluated() && __beg != nullptr && __asan_annotate_container_with_allocator<_Allocator>::value)
         __sanitizer_annotate_contiguous_container(__beg, __end, __old_mid, __new_mid);
     }
 #else
@@ -868,6 +868,7 @@ private:
     if (__alloc() != __c.__alloc())
     {
       __clear();
+      __annotate_delete();
       __alloc_traits::deallocate(__alloc(), this->__begin_, capacity());
       this->__begin_ = this->__end_ = __end_cap() = nullptr;
     }
@@ -956,6 +957,7 @@ vector<_Tp, _Allocator>::__vdeallocate() _NOEXCEPT
     if (this->__begin_ != nullptr)
     {
         clear();
+        __annotate_delete();
         __alloc_traits::deallocate(this->__alloc(), this->__begin_, capacity());
         this->__begin_ = this->__end_ = this->__end_cap() = nullptr;
     }
diff --git a/libcxx/include/wchar.h b/libcxx/include/wchar.h
index c684508dc2ccd..db624cea2bee3 100644
--- a/libcxx/include/wchar.h
+++ b/libcxx/include/wchar.h
@@ -122,6 +122,8 @@ size_t wcsrtombs(char* restrict dst, const wchar_t** restrict src, size_t len,
 
 #  if __has_include_next(<wchar.h>)
 #    include_next <wchar.h>
+#  else
+#    include <__mbstate_t.h> // make sure we have mbstate_t regardless of the existence of <wchar.h>
 #  endif
 
 // Determine whether we have const-correct overloads for wcschr and friends.
diff --git a/libcxx/test/configs/ibm-libc++-shared.cfg.in b/libcxx/test/configs/ibm-libc++-shared.cfg.in
index a93b3085c21ca..7881556603cfa 100644
--- a/libcxx/test/configs/ibm-libc++-shared.cfg.in
+++ b/libcxx/test/configs/ibm-libc++-shared.cfg.in
@@ -4,9 +4,9 @@
 
 lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
-config.substitutions.append(('%{flags}', ''))
+config.substitutions.append(('%{flags}', '-pthread'))
 config.substitutions.append(('%{compile_flags}',
-    '-nostdinc++ -D__LIBC_NO_CPP_MATH_OVERLOADS__ -I %{include} -I %{libcxx}/test/support -pthread'
+    '-nostdinc++ -D__LIBC_NO_CPP_MATH_OVERLOADS__ -I %{include} -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
     '-nostdlib++ -L %{lib} -lc++ -lc++abi -latomic -Wl,-bbigtoc'
diff --git a/libcxx/test/configs/llvm-libc++-mingw.cfg.in b/libcxx/test/configs/llvm-libc++-mingw.cfg.in
index 449b030dd3fc6..eb77f1142db61 100644
--- a/libcxx/test/configs/llvm-libc++-mingw.cfg.in
+++ b/libcxx/test/configs/llvm-libc++-mingw.cfg.in
@@ -11,7 +11,7 @@ config.substitutions.append(('%{link_flags}',
     '-nostdlib++ -L %{lib} -lc++'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/libcxx/test/configs/llvm-libc++-shared-clangcl.cfg.in b/libcxx/test/configs/llvm-libc++-shared-clangcl.cfg.in
index 3814bd4cfe22d..fe406044223f8 100644
--- a/libcxx/test/configs/llvm-libc++-shared-clangcl.cfg.in
+++ b/libcxx/test/configs/llvm-libc++-shared-clangcl.cfg.in
@@ -11,7 +11,7 @@ config.substitutions.append(('%{link_flags}',
     '-nostdlib -L %{lib} -lc++ -lmsvcrt -lmsvcprt -loldnames'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/libcxx/test/configs/llvm-libc++-shared-gcc.cfg.in b/libcxx/test/configs/llvm-libc++-shared-gcc.cfg.in
index eeeafcf0c99b2..df4aaa311dc49 100644
--- a/libcxx/test/configs/llvm-libc++-shared-gcc.cfg.in
+++ b/libcxx/test/configs/llvm-libc++-shared-gcc.cfg.in
@@ -4,12 +4,12 @@
 
 lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
-config.substitutions.append(('%{flags}', ''))
+config.substitutions.append(('%{flags}', '-pthread'))
 config.substitutions.append(('%{compile_flags}',
     '-nostdinc++ -I %{include} -I %{target-include} -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
-    '-L %{lib} -Wl,-rpath,%{lib} -nodefaultlibs -lc++ -lm -lgcc_s -lgcc -lpthread -lc -lgcc_s -lgcc -latomic'
+    '-L %{lib} -Wl,-rpath,%{lib} -nodefaultlibs -lc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc -latomic'
 ))
 config.substitutions.append(('%{exec}',
     '%{executor} --execdir %T -- '
diff --git a/libcxx/test/configs/llvm-libc++-shared-no-vcruntime-clangcl.cfg.in b/libcxx/test/configs/llvm-libc++-shared-no-vcruntime-clangcl.cfg.in
index bdaa834477507..a3ef060f28ba5 100644
--- a/libcxx/test/configs/llvm-libc++-shared-no-vcruntime-clangcl.cfg.in
+++ b/libcxx/test/configs/llvm-libc++-shared-no-vcruntime-clangcl.cfg.in
@@ -12,7 +12,7 @@ config.substitutions.append(('%{link_flags}',
     '-nostdlib -L %{lib} -lc++ -lmsvcrt -lmsvcprt -loldnames'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/libcxx/test/configs/llvm-libc++-shared.cfg.in b/libcxx/test/configs/llvm-libc++-shared.cfg.in
index eb86b88f956ad..143b3b3feae11 100644
--- a/libcxx/test/configs/llvm-libc++-shared.cfg.in
+++ b/libcxx/test/configs/llvm-libc++-shared.cfg.in
@@ -4,13 +4,13 @@
 lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
 config.substitutions.append(('%{flags}',
-    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
+    '-pthread' + (' -isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else '')
 ))
 config.substitutions.append(('%{compile_flags}',
     '-nostdinc++ -I %{include} -I %{target-include} -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++ -pthread'
+    '-nostdlib++ -L %{lib} -Wl,-rpath,%{lib} -lc++'
 ))
 config.substitutions.append(('%{exec}',
     '%{executor} --execdir %T -- '
diff --git a/libcxx/test/configs/llvm-libc++-static.cfg.in b/libcxx/test/configs/llvm-libc++-static.cfg.in
index 8b66a129098be..e866d4f52a062 100644
--- a/libcxx/test/configs/llvm-libc++-static.cfg.in
+++ b/libcxx/test/configs/llvm-libc++-static.cfg.in
@@ -4,13 +4,13 @@
 lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
 
 config.substitutions.append(('%{flags}',
-    '-isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else ''
+    '-pthread' + (' -isysroot {}'.format('@CMAKE_OSX_SYSROOT@') if '@CMAKE_OSX_SYSROOT@' else '')
 ))
 config.substitutions.append(('%{compile_flags}',
     '-nostdinc++ -I %{include} -I %{target-include} -I %{libcxx}/test/support'
 ))
 config.substitutions.append(('%{link_flags}',
-    '-nostdlib++ -L %{lib} -lc++ -lc++abi -pthread'
+    '-nostdlib++ -L %{lib} -lc++ -lc++abi'
 ))
 config.substitutions.append(('%{exec}',
     '%{executor} --execdir %T -- '
diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
index 40275f58cb32c..5b58de0c86ac0 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
@@ -29,40 +29,52 @@ void do_exit() {
 
 int main(int, char**)
 {
-#if TEST_STD_VER >= 11
-    {
-        typedef int T;
-        typedef std::vector<T, min_allocator<T>> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        c.reserve(2*c.size());
-        volatile T foo = c[c.size()];    // bad, but not caught by ASAN
-        ((void)foo);
-    }
+#if TEST_STD_VER >= 11 && TEST_CLANG_VER >= 1600
+  // TODO LLVM18: Remove the special-casing
+  {
+    typedef int T;
+    typedef cpp17_input_iterator<T*> MyInputIter;
+    std::vector<T, min_allocator<T>> v;
+    v.reserve(1);
+    int i[] = {42};
+    v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 1));
+    assert(v[0] == 42);
+    assert(is_contiguous_container_asan_correct(v));
+  }
+  {
+    typedef char T;
+    typedef cpp17_input_iterator<T*> MyInputIter;
+    std::vector<T, unaligned_allocator<T>> v;
+    v.reserve(1);
+    char i[] = {'a', 'b'};
+    v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 2));
+    assert(v[0] == 'a');
+    assert(v[1] == 'b');
+    assert(is_contiguous_container_asan_correct(v));
+  }
 #endif
+  {
+    typedef cpp17_input_iterator<int*> MyInputIter;
+    // Sould not trigger ASan.
+    std::vector<int> v;
+    v.reserve(1);
+    int i[] = {42};
+    v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 1));
+    assert(v[0] == 42);
+    assert(is_contiguous_container_asan_correct(v));
+  }
 
-    {
-        typedef cpp17_input_iterator<int*> MyInputIter;
-        // Sould not trigger ASan.
-        std::vector<int> v;
-        v.reserve(1);
-        int i[] = {42};
-        v.insert(v.begin(), MyInputIter(i), MyInputIter(i + 1));
-        assert(v[0] == 42);
-        assert(is_contiguous_container_asan_correct(v));
-    }
-
-    __sanitizer_set_death_callback(do_exit);
-    {
-        typedef int T;
-        typedef std::vector<T> C;
-        const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-        C c(std::begin(t), std::end(t));
-        c.reserve(2*c.size());
-        assert(is_contiguous_container_asan_correct(c));
-        assert(!__sanitizer_verify_contiguous_container( c.data(), c.data() + 1, c.data() + c.capacity()));
-        volatile T foo = c[c.size()]; // should trigger ASAN. Use volatile to prevent being optimized away.
-        assert(false);          // if we got here, ASAN didn't trigger
-        ((void)foo);
-    }
+  __sanitizer_set_death_callback(do_exit);
+  {
+    typedef int T;
+    typedef std::vector<T> C;
+    const T t[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    C c(std::begin(t), std::end(t));
+    c.reserve(2 * c.size());
+    assert(is_contiguous_container_asan_correct(c));
+    assert(!__sanitizer_verify_contiguous_container(c.data(), c.data() + 1, c.data() + c.capacity()));
+    volatile T foo = c[c.size()]; // should trigger ASAN. Use volatile to prevent being optimized away.
+    assert(false);                // if we got here, ASAN didn't trigger
+    ((void)foo);
+  }
 }
diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp
new file mode 100644
index 0000000000000..e2760271d60a5
--- /dev/null
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan_turning_off.pass.cpp
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03
+
+// <vector>
+
+// Test based on: https://bugs.chromium.org/p/chromium/issues/detail?id=1419798#c5
+// Some allocators during deallocation may not call destructors and just reuse memory.
+// In those situations, one may want to deactivate annotations for a specific allocator.
+// It's possible with __asan_annotate_container_with_allocator template class.
+// This test confirms that those allocators work after turning off annotations.
+
+#include <assert.h>
+#include <stdlib.h>
+#include <vector>
+#include <new>
+
+struct reuse_allocator {
+  static size_t const N = 100;
+  reuse_allocator() {
+    for (size_t i = 0; i < N; ++i)
+      __buffers[i] = malloc(8*1024);
+  }
+  ~reuse_allocator() {
+    for (size_t i = 0; i < N; ++i)
+      free(__buffers[i]);
+  }
+  void* alloc() {
+    assert(__next_id < N);
+    return __buffers[__next_id++];
+  }
+  void reset() { __next_id = 0; }
+  void* __buffers[N];
+  size_t __next_id = 0;
+} reuse_buffers;
+
+template <typename T>
+struct user_allocator {
+  using value_type = T;
+  user_allocator() = default;
+  template <class U>
+  user_allocator(user_allocator<U>) {}
+  friend bool operator==(user_allocator, user_allocator) {return true;}
+  friend bool operator!=(user_allocator x, user_allocator y) {return !(x == y);}
+
+  T* allocate(size_t) { return (T*)reuse_buffers.alloc(); }
+  void deallocate(T*, size_t) noexcept {}
+};
+
+#ifdef _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS
+template <class T>
+struct std::__asan_annotate_container_with_allocator<user_allocator<T>> : false_type {};
+#endif
+
+int main(int, char**) {
+  using V = std::vector<int, user_allocator<int>>;
+
+  {
+    V* v = new (reuse_buffers.alloc()) V();
+    for (int i = 0; i < 100; i++)
+      v->push_back(i);
+  }
+  reuse_buffers.reset();
+  {
+    V v;
+    for (int i = 0; i < 1000; i++)
+      v.push_back(i);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/diagnostics/pstl.nodiscard_extensions.compile.pass.cpp b/libcxx/test/libcxx/diagnostics/pstl.nodiscard_extensions.compile.pass.cpp
new file mode 100644
index 0000000000000..c45c356a625a4
--- /dev/null
+++ b/libcxx/test/libcxx/diagnostics/pstl.nodiscard_extensions.compile.pass.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Check that PSTL algorithms aren't marked [[nodiscard]] when
+// _LIBCPP_DISBALE_NODISCARD_EXT is defined
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_NODISCARD_EXT
+
+// REQUIRES: with-pstl
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+#include <algorithm>
+#include <execution>
+
+void test() {
+  int a[] = {1};
+  auto pred = [](auto) { return false; };
+  std::all_of(std::execution::par, std::begin(a), std::end(a), pred);
+  std::any_of(std::execution::par, std::begin(a), std::end(a), pred);
+  std::none_of(std::execution::par, std::begin(a), std::end(a), pred);
+}
diff --git a/libcxx/test/libcxx/diagnostics/pstl.nodiscard_extensions.verify.cpp b/libcxx/test/libcxx/diagnostics/pstl.nodiscard_extensions.verify.cpp
new file mode 100644
index 0000000000000..c1817adb2951d
--- /dev/null
+++ b/libcxx/test/libcxx/diagnostics/pstl.nodiscard_extensions.verify.cpp
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Check that PSTL algorithms are marked [[nodiscard]] as a conforming extension
+
+// REQUIRES: with-pstl
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+#include <algorithm>
+#include <execution>
+
+void test() {
+  int a[] = {1};
+  auto pred = [](auto) { return false; };
+  std::all_of(std::execution::par, std::begin(a), std::end(a), pred); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::any_of(std::execution::par, std::begin(a), std::end(a), pred); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  std::none_of(std::execution::par, std::begin(a), std::end(a), pred); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+}
diff --git a/libcxx/test/libcxx/private_headers.verify.cpp b/libcxx/test/libcxx/private_headers.verify.cpp
index e511411985876..0720e1f72a77f 100644
--- a/libcxx/test/libcxx/private_headers.verify.cpp
+++ b/libcxx/test/libcxx/private_headers.verify.cpp
@@ -427,11 +427,17 @@ END-SCRIPT
 #include <__functional/unwrap_ref.h> // expected-error@*:* {{use of private header from outside its module: '__functional/unwrap_ref.h'}}
 #include <__functional/weak_result_type.h> // expected-error@*:* {{use of private header from outside its module: '__functional/weak_result_type.h'}}
 #include <__fwd/array.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/array.h'}}
+#include <__fwd/fstream.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/fstream.h'}}
 #include <__fwd/get.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/get.h'}}
 #include <__fwd/hash.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/hash.h'}}
+#include <__fwd/ios.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/ios.h'}}
+#include <__fwd/istream.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/istream.h'}}
 #include <__fwd/memory_resource.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/memory_resource.h'}}
+#include <__fwd/ostream.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/ostream.h'}}
 #include <__fwd/pair.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/pair.h'}}
 #include <__fwd/span.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/span.h'}}
+#include <__fwd/sstream.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/sstream.h'}}
+#include <__fwd/streambuf.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/streambuf.h'}}
 #include <__fwd/string.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/string.h'}}
 #include <__fwd/string_view.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/string_view.h'}}
 #include <__fwd/subrange.h> // expected-error@*:* {{use of private header from outside its module: '__fwd/subrange.h'}}
@@ -608,6 +614,7 @@ END-SCRIPT
 #include <__ranges/views.h> // expected-error@*:* {{use of private header from outside its module: '__ranges/views.h'}}
 #include <__ranges/zip_view.h> // expected-error@*:* {{use of private header from outside its module: '__ranges/zip_view.h'}}
 #include <__split_buffer> // expected-error@*:* {{use of private header from outside its module: '__split_buffer'}}
+#include <__std_mbstate_t.h> // expected-error@*:* {{use of private header from outside its module: '__std_mbstate_t.h'}}
 #include <__string/char_traits.h> // expected-error@*:* {{use of private header from outside its module: '__string/char_traits.h'}}
 #include <__string/constexpr_c_functions.h> // expected-error@*:* {{use of private header from outside its module: '__string/constexpr_c_functions.h'}}
 #include <__string/extern_template_lists.h> // expected-error@*:* {{use of private header from outside its module: '__string/extern_template_lists.h'}}
@@ -677,6 +684,7 @@ END-SCRIPT
 #include <__type_traits/is_empty.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_empty.h'}}
 #include <__type_traits/is_enum.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_enum.h'}}
 #include <__type_traits/is_equality_comparable.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_equality_comparable.h'}}
+#include <__type_traits/is_execution_policy.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_execution_policy.h'}}
 #include <__type_traits/is_final.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_final.h'}}
 #include <__type_traits/is_floating_point.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_floating_point.h'}}
 #include <__type_traits/is_function.h> // expected-error@*:* {{use of private header from outside its module: '__type_traits/is_function.h'}}
@@ -776,6 +784,7 @@ END-SCRIPT
 #include <__utility/priority_tag.h> // expected-error@*:* {{use of private header from outside its module: '__utility/priority_tag.h'}}
 #include <__utility/rel_ops.h> // expected-error@*:* {{use of private header from outside its module: '__utility/rel_ops.h'}}
 #include <__utility/swap.h> // expected-error@*:* {{use of private header from outside its module: '__utility/swap.h'}}
+#include <__utility/terminate_on_exception.h> // expected-error@*:* {{use of private header from outside its module: '__utility/terminate_on_exception.h'}}
 #include <__utility/to_underlying.h> // expected-error@*:* {{use of private header from outside its module: '__utility/to_underlying.h'}}
 #include <__utility/unreachable.h> // expected-error@*:* {{use of private header from outside its module: '__utility/unreachable.h'}}
 #include <__variant/monostate.h> // expected-error@*:* {{use of private header from outside its module: '__variant/monostate.h'}}
diff --git a/libcxx/test/libcxx/transitive_includes.sh.cpp b/libcxx/test/libcxx/transitive_includes.sh.cpp
index 28a1ed586b931..8523db6cfa284 100644
--- a/libcxx/test/libcxx/transitive_includes.sh.cpp
+++ b/libcxx/test/libcxx/transitive_includes.sh.cpp
@@ -32,6 +32,9 @@
 // this test instead.
 // UNSUPPORTED: transitive-includes-disabled
 
+// FIXME: This should pass with the PSTL enabled
+// XFAIL: with-pstl
+
 // Prevent <ext/hash_map> from generating deprecated warnings for this test.
 #if defined(__DEPRECATED)
 #    undef __DEPRECATED
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index ef7efa519d4b6..e5aee81cc70d8 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -213,6 +213,7 @@ exception cstddef
 exception cstdlib
 exception type_traits
 exception version
+execution cstddef
 execution version
 expected cstddef
 expected initializer_list
@@ -332,7 +333,6 @@ format stdexcept
 format string
 format string_view
 format tuple
-format vector
 format version
 forward_list algorithm
 forward_list atomic
@@ -868,7 +868,6 @@ thread string_view
 thread system_error
 thread tuple
 thread type_traits
-thread vector
 thread version
 tuple compare
 tuple cstddef
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index 3eecead0ae1b1..f696ff6c2b152 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -213,6 +213,7 @@ exception cstddef
 exception cstdlib
 exception type_traits
 exception version
+execution cstddef
 execution version
 expected cstddef
 expected initializer_list
@@ -332,7 +333,6 @@ format stdexcept
 format string
 format string_view
 format tuple
-format vector
 format version
 forward_list algorithm
 forward_list atomic
@@ -869,7 +869,6 @@ thread string_view
 thread system_error
 thread tuple
 thread type_traits
-thread vector
 thread version
 tuple compare
 tuple cstddef
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index ec68a819d0aed..5e7d9ca2700bf 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -213,6 +213,7 @@ exception cstddef
 exception cstdlib
 exception type_traits
 exception version
+execution cstddef
 execution version
 expected cstddef
 expected initializer_list
@@ -334,7 +335,6 @@ format stdexcept
 format string
 format string_view
 format tuple
-format vector
 format version
 forward_list algorithm
 forward_list atomic
@@ -871,7 +871,6 @@ thread string_view
 thread system_error
 thread tuple
 thread type_traits
-thread vector
 thread version
 tuple compare
 tuple cstddef
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index ec68a819d0aed..5e7d9ca2700bf 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -213,6 +213,7 @@ exception cstddef
 exception cstdlib
 exception type_traits
 exception version
+execution cstddef
 execution version
 expected cstddef
 expected initializer_list
@@ -334,7 +335,6 @@ format stdexcept
 format string
 format string_view
 format tuple
-format vector
 format version
 forward_list algorithm
 forward_list atomic
@@ -871,7 +871,6 @@ thread string_view
 thread system_error
 thread tuple
 thread type_traits
-thread vector
 thread version
 tuple compare
 tuple cstddef
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index c3a309a709520..a96081233f326 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -127,7 +127,6 @@ chrono string
 chrono string_view
 chrono tuple
 chrono type_traits
-chrono vector
 chrono version
 cinttypes cstdint
 cmath type_traits
@@ -221,6 +220,7 @@ exception cstddef
 exception cstdlib
 exception type_traits
 exception version
+execution cstddef
 execution version
 expected cstddef
 expected initializer_list
@@ -342,7 +342,6 @@ format stdexcept
 format string
 format string_view
 format tuple
-format vector
 format version
 forward_list algorithm
 forward_list atomic
@@ -877,7 +876,6 @@ thread string_view
 thread system_error
 thread tuple
 thread type_traits
-thread vector
 thread version
 tuple compare
 tuple cstddef
diff --git a/libcxx/test/libcxx/transitive_includes/cxx2b.csv b/libcxx/test/libcxx/transitive_includes/cxx2b.csv
index 7b82ae1016a5e..06bdb93e57ab5 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx2b.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx2b.csv
@@ -13,7 +13,6 @@ any cstring
 any initializer_list
 any limits
 any new
-any stdexcept
 any typeinfo
 any version
 array compare
@@ -71,6 +70,7 @@ chrono ctime
 chrono initializer_list
 chrono limits
 chrono locale
+chrono new
 chrono optional
 chrono ostream
 chrono ratio
@@ -79,7 +79,6 @@ chrono stdexcept
 chrono string
 chrono string_view
 chrono tuple
-chrono vector
 chrono version
 cinttypes cstdint
 cmath version
@@ -140,6 +139,7 @@ deque version
 exception cstddef
 exception cstdlib
 exception version
+execution cstddef
 execution version
 expected cstddef
 expected initializer_list
@@ -224,6 +224,7 @@ format cstdlib
 format initializer_list
 format limits
 format locale
+format new
 format optional
 format queue
 format stack
@@ -231,7 +232,6 @@ format stdexcept
 format string
 format string_view
 format tuple
-format vector
 format version
 forward_list compare
 forward_list cstddef
@@ -581,12 +581,12 @@ thread ctime
 thread iosfwd
 thread limits
 thread locale
+thread new
 thread ratio
 thread stdexcept
 thread string
 thread string_view
 thread tuple
-thread vector
 thread version
 tuple compare
 tuple cstddef
@@ -634,7 +634,6 @@ valarray cstdint
 valarray initializer_list
 valarray limits
 valarray new
-valarray stdexcept
 valarray version
 variant compare
 variant cstddef
diff --git a/libcxx/test/libcxx/utilities/expol/policies.compile.pass.cpp b/libcxx/test/libcxx/utilities/expol/policies.compile.pass.cpp
new file mode 100644
index 0000000000000..fb97cffe22444
--- /dev/null
+++ b/libcxx/test/libcxx/utilities/expol/policies.compile.pass.cpp
@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we don't allow copying the execution policies in any way to avoid users relying on it.
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+#include <execution>
+#include <type_traits>
+
+#include "test_macros.h"
+
+static_assert(!std::is_default_constructible_v<std::execution::sequenced_policy>);
+static_assert(!std::is_copy_constructible_v<std::execution::sequenced_policy>);
+static_assert(!std::is_move_constructible_v<std::execution::sequenced_policy>);
+static_assert(!std::is_copy_assignable_v<std::execution::sequenced_policy>);
+static_assert(!std::is_move_assignable_v<std::execution::sequenced_policy>);
+
+static_assert(!std::is_default_constructible_v<std::execution::parallel_policy>);
+static_assert(!std::is_copy_constructible_v<std::execution::parallel_policy>);
+static_assert(!std::is_move_constructible_v<std::execution::parallel_policy>);
+static_assert(!std::is_copy_assignable_v<std::execution::parallel_policy>);
+static_assert(!std::is_move_assignable_v<std::execution::parallel_policy>);
+
+static_assert(!std::is_default_constructible_v<std::execution::parallel_unsequenced_policy>);
+static_assert(!std::is_copy_constructible_v<std::execution::parallel_unsequenced_policy>);
+static_assert(!std::is_move_constructible_v<std::execution::parallel_unsequenced_policy>);
+static_assert(!std::is_copy_assignable_v<std::execution::parallel_unsequenced_policy>);
+static_assert(!std::is_move_assignable_v<std::execution::parallel_unsequenced_policy>);
+
+#if TEST_STD_VER >= 20
+static_assert(!std::is_default_constructible_v<std::execution::unsequenced_policy>);
+static_assert(!std::is_copy_constructible_v<std::execution::unsequenced_policy>);
+static_assert(!std::is_move_constructible_v<std::execution::unsequenced_policy>);
+static_assert(!std::is_copy_assignable_v<std::execution::unsequenced_policy>);
+static_assert(!std::is_move_assignable_v<std::execution::unsequenced_policy>);
+#endif
diff --git a/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp
new file mode 100644
index 0000000000000..b928ed7432791
--- /dev/null
+++ b/libcxx/test/libcxx/utilities/optional/optional.object/optional_size.pass.cpp
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// <optional>
+
+// template <class T> class optional;
+
+#include <optional>
+
+template <class T>
+struct type_with_bool {
+  T value;
+  bool has_value;
+};
+
+int main(int, char**) {
+  // Test that std::optional achieves the expected size. See https://llvm.org/PR61095.
+  static_assert(sizeof(std::optional<char>) == sizeof(type_with_bool<char>));
+  static_assert(sizeof(std::optional<int>) == sizeof(type_with_bool<int>));
+  static_assert(sizeof(std::optional<long>) == sizeof(type_with_bool<long>));
+  static_assert(sizeof(std::optional<std::size_t>) == sizeof(type_with_bool<std::size_t>));
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp b/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp
index a2dc58bce1b64..9011e61e78808 100644
--- a/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp
+++ b/libcxx/test/libcxx/utilities/variant/variant.variant/variant_size.pass.cpp
@@ -60,6 +60,16 @@ void test_index_internals() {
   static_assert(std::__variant_npos<IndexT> == IndexLim::max(), "");
 }
 
+template <class LargestType>
+struct type_with_index {
+  LargestType largest;
+#ifdef _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
+  unsigned char index;
+#else
+  unsigned int index;
+#endif
+};
+
 int main(int, char**) {
   test_index_type<unsigned char>();
   // This won't compile due to template depth issues.
@@ -67,5 +77,12 @@ int main(int, char**) {
   test_index_internals<unsigned char>();
   test_index_internals<unsigned short>();
 
+  // Test that std::variant achieves the expected size. See https://llvm.org/PR61095.
+  static_assert(sizeof(std::variant<char, char, char>) == sizeof(type_with_index<char>));
+  static_assert(sizeof(std::variant<int, int, int>) == sizeof(type_with_index<int>));
+  static_assert(sizeof(std::variant<long, long, long>) == sizeof(type_with_index<long>));
+  static_assert(sizeof(std::variant<char, int, long>) == sizeof(type_with_index<long>));
+  static_assert(sizeof(std::variant<std::size_t, std::size_t, std::size_t>) == sizeof(type_with_index<std::size_t>));
+
   return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/pstl.fill.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/pstl.fill.pass.cpp
new file mode 100644
index 0000000000000..69c703fae3ba3
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/pstl.fill.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <algorithm>
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// template<class ExecutionPolicy, class ForwardIterator, class T>
+//   void fill(ExecutionPolicy&& exec,
+//             ForwardIterator first, ForwardIterator last, const T& value);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(fill);
+
+static_assert(sfinae_test_fill<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_fill<std::execution::parallel_policy, int*, int*, int>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    { // simple test
+      int a[4];
+      std::fill(policy, Iter(std::begin(a)), Iter(std::end(a)), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+    { // check that an empty range works
+      int a[1] = {2};
+      std::fill(policy, Iter(std::begin(a)), Iter(std::begin(a)), 33);
+      assert(a[0] == 2);
+    }
+    { // check that a one-element range works
+      int a[1];
+      std::fill(policy, Iter(std::begin(a)), Iter(std::end(a)), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+    { // check that a two-element range works
+      int a[2];
+      std::fill(policy, Iter(std::begin(a)), Iter(std::end(a)), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+    { // check that a large range works
+      std::vector<int> a(234, 2);
+      std::fill(policy, Iter(std::data(a)), Iter(std::data(a) + std::size(a)), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+  }
+};
+
+struct ThrowOnCopy {
+  ThrowOnCopy& operator=(const ThrowOnCopy&) { throw int{}; }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  ThrowOnCopy a[2];
+  try {
+    (void)std::fill(std::execution::par, std::begin(a), std::end(a), ThrowOnCopy{});
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/pstl.fill_n.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/pstl.fill_n.pass.cpp
new file mode 100644
index 0000000000000..9767f3987d581
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.fill/pstl.fill_n.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <algorithm>
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// template<class ExecutionPolicy, class ForwardIterator, class Size, class T>
+//   ForwardIterator fill_n(ExecutionPolicy&& exec,
+//                          ForwardIterator first, Size n, const T& value);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(fill_n);
+
+static_assert(sfinae_test_fill_n<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_fill_n<std::execution::parallel_policy, int*, int*, int>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    { // simple test
+      int a[4];
+      std::fill_n(policy, Iter(std::begin(a)), std::size(a), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+    { // check that an empty range works
+      int a[1] = {2};
+      std::fill_n(policy, Iter(std::begin(a)), 0, 33);
+      assert(a[0] == 2);
+    }
+    { // check that a one-element range works
+      int a[1];
+      std::fill_n(policy, Iter(std::begin(a)), std::size(a), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+    { // check that a two-element range works
+      int a[2];
+      std::fill_n(policy, Iter(std::begin(a)), std::size(a), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+    { // check that a large range works
+      std::vector<int> a(234, 2);
+      std::fill_n(policy, Iter(std::data(a)), std::size(a), 33);
+      assert(std::all_of(std::begin(a), std::end(a), [](int i) { return i == 33; }));
+    }
+  }
+};
+
+struct ThrowOnCopy {
+  ThrowOnCopy& operator=(const ThrowOnCopy&) { throw int{}; }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  ThrowOnCopy a[2];
+  try {
+    (void)std::fill_n(std::execution::par, std::begin(a), std::size(a), ThrowOnCopy{});
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.all_of/pstl.all_of.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.all_of/pstl.all_of.pass.cpp
new file mode 100644
index 0000000000000..5280970e6f549
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.all_of/pstl.all_of.pass.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Predicate>
+//   bool all_of(ExecutionPolicy&& exec, ForwardIterator first, ForwardIterator last,
+//               Predicate pred);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(all_of);
+
+static_assert(sfinae_test_all_of<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_all_of<std::execution::parallel_policy, int*, int*, bool (*)(int)>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int a[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    // simple test
+    assert(std::all_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i < 9; }));
+    assert(!std::all_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i < 8; }));
+
+    // check that an empty range works
+    assert(std::all_of(policy, Iter(std::begin(a)), Iter(std::begin(a)), [](int) { return true; }));
+
+    // check that a single-element range works
+    assert(std::all_of(policy, Iter(a), Iter(a + 1), [](int i) { return i < 2; }));
+
+    // check that a two-element range works
+    assert(std::all_of(policy, Iter(a), Iter(a + 2), [](int i) { return i < 3; }));
+
+    // check that false is returned if no element satisfies the condition
+    assert(!std::all_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 9; }));
+
+    // check that false is returned if only one elements satisfies the condition
+    assert(!std::all_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 1; }));
+
+    // check that a one-element range works
+    assert(std::all_of(policy, Iter(std::begin(a)), Iter(std::begin(a) + 1), [](int i) { return i == 1; }));
+
+    // check that a two-element range works
+    assert(std::all_of(policy, Iter(std::begin(a)), Iter(std::begin(a) + 2), [](int i) { return i < 3; }));
+
+    // check that a large number of elements works
+    std::vector<int> vec(100);
+    std::fill(vec.begin(), vec.end(), 3);
+    assert(std::all_of(Iter(vec.data()), Iter(vec.data() + vec.size()), [](int i) { return i == 3; }));
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    (void)std::all_of(std::execution::par, std::begin(a), std::end(a), [](int i) -> bool { throw i; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.any_of/pstl.any_of.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.any_of/pstl.any_of.pass.cpp
new file mode 100644
index 0000000000000..c47aa1820f263
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.any_of/pstl.any_of.pass.cpp
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Predicate>
+//   bool any_of(ExecutionPolicy&& exec, ForwardIterator first, ForwardIterator last,
+//               Predicate pred);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(any_of);
+
+static_assert(sfinae_test_any_of<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_any_of<std::execution::parallel_policy, int*, int*, bool (*)(int)>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int a[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    // simple test
+    assert(std::any_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i < 9; }));
+    assert(!std::any_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i > 8; }));
+
+    // check that an empty range works
+    assert(!std::any_of(policy, Iter(std::begin(a)), Iter(std::begin(a)), [](int) { return false; }));
+
+    // check that false is returned if no element satisfies the condition
+    assert(!std::any_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 9; }));
+
+    // check that true is returned if only one elements satisfies the condition
+    assert(std::any_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 1; }));
+
+    // check that a one-element range works
+    assert(std::any_of(policy, Iter(std::begin(a)), Iter(std::begin(a) + 1), [](int i) { return i == 1; }));
+
+    // check that a two-element range works
+    assert(std::any_of(policy, Iter(std::begin(a)), Iter(std::begin(a) + 2), [](int i) { return i == 2; }));
+
+    // check that a large number of elements works
+    std::vector<int> vec(100, 2);
+    vec[96] = 3;
+    assert(std::any_of(Iter(vec.data()), Iter(vec.data() + vec.size()), [](int i) { return i == 3; }));
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    (void)std::any_of(std::execution::par, std::begin(a), std::end(a), [](int i) -> bool { throw i; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find.pass.cpp
new file mode 100644
index 0000000000000..004f6d6d8f900
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find.pass.cpp
@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class T>
+//   ForwardIterator find(ExecutionPolicy&& exec, ForwardIterator first, ForwardIterator last,
+//                        const T& value);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(find);
+
+static_assert(sfinae_test_find<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_find<std::execution::parallel_policy, int*, int*, int>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int a[] = {1, 2, 3, 4, 5, 6, 7, 8};
+
+    // simple test
+    assert(base(std::find(policy, Iter(std::begin(a)), Iter(std::end(a)), 3)) == a + 2);
+
+    // check that last is returned if no element matches
+    assert(base(std::find(policy, Iter(std::begin(a)), Iter(std::end(a)), 0)) == std::end(a));
+
+    // check that the first element is returned
+    assert(base(std::find(policy, Iter(std::begin(a)), Iter(std::end(a)), 1)) == std::begin(a));
+
+    // check that an empty range works
+    assert(base(std::find(policy, Iter(std::begin(a)), Iter(std::begin(a)), 1)) == std::begin(a));
+
+    // check that a one-element range works
+    assert(base(std::find(policy, Iter(std::begin(a)), Iter(std::begin(a) + 1), 1)) == std::begin(a));
+
+    // check that a two-element range works
+    assert(base(std::find(policy, Iter(std::begin(a)), Iter(std::begin(a) + 2), 2)) == std::begin(a) + 1);
+
+    // check that a large number of elements works
+    std::vector<int> vec(200, 4);
+    vec[176] = 5;
+    assert(base(std::find(policy, Iter(std::data(vec)), Iter(std::data(vec) + std::size(vec)), 5)) ==
+           std::data(vec) + 176);
+  }
+};
+
+struct ThrowOnCompare {};
+
+bool operator==(ThrowOnCompare, ThrowOnCompare) { throw int{}; }
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  ThrowOnCompare a[2];
+  try {
+    (void)std::find(std::execution::par, std::begin(a), std::end(a), ThrowOnCompare{});
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find_if.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find_if.pass.cpp
new file mode 100644
index 0000000000000..bff446bc079ee
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find_if.pass.cpp
@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Predicate>
+//   ForwardIterator find_if(ExecutionPolicy&& exec, ForwardIterator first, ForwardIterator last,
+//                           Predicate pred);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(find_if);
+
+static_assert(sfinae_test_find_if<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_find_if<std::execution::parallel_policy, int*, int*, int>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int a[] = {1, 2, 3, 4, 5, 6, 7, 8};
+
+    // simple test
+    assert(base(std::find_if(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 3; })) == a + 2);
+
+    // check that last is returned if no element matches
+    assert(base(std::find_if(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 0; })) ==
+           std::end(a));
+
+    // check that the first element is returned
+    assert(base(std::find_if(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 1; })) ==
+           std::begin(a));
+
+    // check that an empty range works
+    assert(base(std::find_if(policy, Iter(std::begin(a)), Iter(std::begin(a)), [](int i) { return i == 1; })) ==
+           std::begin(a));
+
+    // check that a one-element range works
+    assert(base(std::find_if(policy, Iter(std::begin(a)), Iter(std::begin(a) + 1), [](int i) { return i == 1; })) ==
+           std::begin(a));
+
+    // check that a two-element range works
+    assert(base(std::find_if(policy, Iter(std::begin(a)), Iter(std::begin(a) + 2), [](int i) { return i == 2; })) ==
+           std::begin(a) + 1);
+
+    // check that a large number of elements works
+    std::vector<int> vec(200, 4);
+    vec[176] = 5;
+    assert(base(std::find_if(policy, Iter(std::data(vec)), Iter(std::data(vec) + std::size(vec)), [](int i) {
+             return i == 5;
+           })) == std::data(vec) + 176);
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    (void)std::find_if(std::execution::par, std::begin(a), std::end(a), [](int) -> bool { throw int{}; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find_if_not.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find_if_not.pass.cpp
new file mode 100644
index 0000000000000..2cf123e4f9516
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.find/pstl.find_if_not.pass.cpp
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Predicate>
+//   ForwardIterator find_if_not(ExecutionPolicy&& exec, ForwardIterator first, ForwardIterator last,
+//                               Predicate pred);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(find_if_not);
+
+static_assert(sfinae_test_find_if_not<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_find_if_not<std::execution::parallel_policy, int*, int*, int>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int a[] = {1, 2, 3, 4, 5, 6, 7, 8};
+
+    // simple test
+    assert(base(std::find_if_not(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i != 3; })) ==
+           a + 2);
+
+    // check that last is returned if no element matches
+    assert(base(std::find_if_not(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i != 0; })) ==
+           std::end(a));
+
+    // check that the first element is returned
+    assert(base(std::find_if_not(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i != 1; })) ==
+           std::begin(a));
+
+    // check that an empty range works
+    assert(base(std::find_if_not(policy, Iter(std::begin(a)), Iter(std::begin(a)), [](int i) { return i != 1; })) ==
+           std::begin(a));
+
+    // check that a one-element range works
+    assert(base(std::find_if_not(policy, Iter(std::begin(a)), Iter(std::begin(a) + 1), [](int i) { return i != 1; })) ==
+           std::begin(a));
+
+    // check that a two-element range works
+    assert(base(std::find_if_not(policy, Iter(std::begin(a)), Iter(std::begin(a) + 2), [](int i) { return i != 2; })) ==
+           std::begin(a) + 1);
+
+    // check that a large number of elements works
+    std::vector<int> vec(200, 4);
+    vec[176] = 5;
+    assert(base(std::find_if_not(policy, Iter(std::data(vec)), Iter(std::data(vec) + std::size(vec)), [](int i) {
+             return i != 5;
+           })) == std::data(vec) + 176);
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    (void)std::find_if_not(std::execution::par, std::begin(a), std::end(a), [](int) -> bool { throw int{}; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/pstl.for_each.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/pstl.for_each.pass.cpp
new file mode 100644
index 0000000000000..49d3bd16dc021
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/pstl.for_each.pass.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Function>
+//   void for_each(ExecutionPolicy&& exec,
+//                 ForwardIterator first, ForwardIterator last,
+//                 Function f);
+
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(for_each);
+
+static_assert(sfinae_test_for_each<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_for_each<std::execution::parallel_policy, int*, int*, bool (*)(int)>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int sizes[] = {0, 1, 2, 100};
+    for (auto size : sizes) {
+      std::vector<int> a(size);
+      std::vector<Bool> called(size);
+      std::for_each(policy, Iter(std::data(a)), Iter(std::data(a) + std::size(a)), [&](int& v) {
+        assert(!called[&v - a.data()]);
+        called[&v - a.data()] = true;
+      });
+      assert(std::all_of(std::begin(called), std::end(called), [](bool b) { return b; }));
+    }
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    std::for_each(std::execution::par, std::begin(a), std::end(a), [](int) { throw int{}; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/pstl.for_each_n.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/pstl.for_each_n.pass.cpp
new file mode 100644
index 0000000000000..994563861b0f2
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/pstl.for_each_n.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Size, class Function>
+//   ForwardIterator for_each_n(ExecutionPolicy&& exec, ForwardIterator first, Size n,
+//                              Function f);
+
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(for_each);
+
+static_assert(sfinae_test_for_each<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_for_each<std::execution::parallel_policy, int*, int*, bool (*)(int)>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int sizes[] = {0, 1, 2, 100};
+    for (auto size : sizes) {
+      std::vector<int> a(size);
+      std::vector<Bool> called(size);
+      std::for_each_n(policy, Iter(std::data(a)), std::size(a), [&](int& v) {
+        assert(!called[&v - a.data()]);
+        called[&v - a.data()] = true;
+      });
+      assert(std::all_of(std::begin(called), std::end(called), [](bool b) { return b; }));
+    }
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    std::for_each_n(std::execution::par, std::data(a), std::size(a), [](int) { throw int{}; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.none_of/pstl.none_of.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.none_of/pstl.none_of.pass.cpp
new file mode 100644
index 0000000000000..524bb8e652b06
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.none_of/pstl.none_of.pass.cpp
@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: with-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator, class Predicate>
+//   bool none_of(ExecutionPolicy&& exec, ForwardIterator first, ForwardIterator last,
+//                Predicate pred);
+
+#include <algorithm>
+#include <cassert>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(none_of);
+
+static_assert(sfinae_test_none_of<int, int*, int*, bool (*)(int)>);
+static_assert(!sfinae_test_none_of<std::execution::parallel_policy, int*, int*, bool (*)(int)>);
+
+template <class Iter>
+struct Test {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    int a[] = {1, 2, 3, 4, 5, 6, 7, 8};
+    // simple test
+    assert(std::none_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i > 9; }));
+    assert(!std::none_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i >= 8; }));
+
+    // check that an empty range works
+    assert(std::none_of(policy, Iter(std::begin(a)), Iter(std::begin(a)), [](int) { return false; }));
+
+    // check that true is returned if no element satisfies the condition
+    assert(std::none_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 9; }));
+
+    // check that false is returned if only one elements satisfies the condition
+    assert(!std::none_of(policy, Iter(std::begin(a)), Iter(std::end(a)), [](int i) { return i == 1; }));
+
+    // check that a one-element range works
+    assert(std::none_of(policy, Iter(std::begin(a)), Iter(std::begin(a) + 1), [](int i) { return i != 1; }));
+
+    // check that a two-element range works
+    assert(std::none_of(policy, Iter(std::begin(a)), Iter(std::begin(a) + 2), [](int i) { return i > 2; }));
+
+    // check that a large number of elements works
+    std::vector<int> vec(100);
+    std::fill(vec.begin(), vec.end(), 3);
+    assert(std::none_of(Iter(vec.data()), Iter(vec.data() + vec.size()), [](int i) { return i != 3; }));
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, TestIteratorWithPolicies<Test>{});
+
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  std::set_terminate(terminate_successful);
+  int a[] = {1, 2};
+  try {
+    (void)std::none_of(std::execution::par, std::begin(a), std::end(a), [](int i) -> bool { throw i; });
+  } catch (int) {
+    assert(false);
+  }
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
index 504616c106652..e1d78de670475 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
@@ -25,6 +25,7 @@
 // strong_ordering operator<=>(path const&, path const&) noexcept;
 //
 // size_t hash_value(path const&) noexcept;
+// template<> struct hash<filesystem::path>;
 
 #include "filesystem_include.h"
 #include <type_traits>
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.hash_enabled.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.hash_enabled.pass.cpp
new file mode 100644
index 0000000000000..bc1db879f03e3
--- /dev/null
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.hash_enabled.pass.cpp
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03
+
+// <filesystem>
+
+// Test that <filesystem> provides all of the arithmetic, enum, and pointer
+// hash specializations.
+
+#include "filesystem_include.h"
+#include "poisoned_hash_helper.h"
+
+int main(int, char**) {
+  test_library_hash_specializations_available();
+  test_hash_enabled_for_type<fs::path>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.clock/time.clock.file/ostream.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.file/ostream.pass.cpp
new file mode 100644
index 0000000000000..759387be93f1a
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.file/ostream.pass.cpp
@@ -0,0 +1,200 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-localization
+// UNSUPPORTED: libcpp-has-no-incomplete-format
+
+// TODO FMT Investigate Windows issues.
+// UNSUPPORTED msvc, target={{.+}}-windows-gnu
+// TODO FMT It seems GCC uses too much memory in the CI and fails.
+// UNSUPPORTED: gcc-12
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+
+// class system_clock;
+
+// template<class charT, class traits, class Duration>
+//   basic_ostream<charT, traits>&
+//     operator<<(basic_ostream<charT, traits>& os, const sys_time<Duration>& tp);
+
+#include <chrono>
+#include <cassert>
+#include <ratio>
+#include <sstream>
+
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+#define SV(S) MAKE_STRING_VIEW(CharT, S)
+
+// Modeled after the system clock's usings, this make adapating the
+// tests easier.
+template <class Duration>
+using file_time    = std::chrono::time_point<std::chrono::file_clock, Duration>;
+using file_seconds = file_time<std::chrono::seconds>;
+using file_days    = file_time<std::chrono::days>;
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_c_locale(file_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_fr_FR_locale(file_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_fr_FR_UTF_8);
+  sstr.imbue(locale);
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_ja_JP_locale(file_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_ja_JP_UTF_8);
+  sstr.imbue(locale);
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT>
+static void test_c() {
+  using namespace std::literals::chrono_literals;
+
+  assert(stream_c_locale<CharT>(file_time<std::chrono::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("2000-01-01 01:02:03.123456789"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::microseconds>{946'688'523'123'456us}) ==
+         SV("2000-01-01 01:02:03.123456"));
+
+  assert(stream_c_locale<CharT>(file_time<std::chrono::milliseconds>{946'684'800'123ms}) ==
+         SV("2000-01-01 00:00:00.123"));
+  assert(stream_c_locale<CharT>(file_seconds{1'234'567'890s}) == SV("2009-02-13 23:31:30"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::minutes>{20'576'131min}) == SV("2009-02-13 23:31:00"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::hours>{342'935h}) == SV("2009-02-13 23:00:00"));
+  assert(stream_c_locale<CharT>(file_days{std::chrono::days{14'288}}) == SV("2009-02-13 00:00:00"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::weeks>{std::chrono::weeks{2041}}) == SV("2009-02-12 00:00:00"));
+
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<signed char, std::ratio<2, 1>>>{
+             std::chrono::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1970-01-01 00:02:00"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<short, std::ratio<1, 2>>>{
+             std::chrono::duration<short, std::ratio<1, 2>>{3600}}) == SV("1970-01-01 00:30:00.0"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<int, std::ratio<1, 4>>>{
+             std::chrono::duration<int, std::ratio<1, 4>>{3600}}) == SV("1970-01-01 00:15:00.00"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<long, std::ratio<1, 10>>>{
+             std::chrono::duration<long, std::ratio<1, 10>>{36611}}) == SV("1970-01-01 01:01:01.1"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<long long, std::ratio<1, 100>>>{
+             std::chrono::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("2009-02-13 23:31:30.10"));
+
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<float, std::ratio<1, 1>>>{
+             std::chrono::duration<float, std::ratio<1, 1>>{123.456}}) == SV("1970-01-01 00:02:03"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<double, std::ratio<1, 10>>>{
+             std::chrono::duration<double, std::ratio<1, 10>>{123.456}}) == SV("1970-01-01 00:00:12.3"));
+  assert(stream_c_locale<CharT>(file_time<std::chrono::duration<long double, std::ratio<1, 100>>>{
+             std::chrono::duration<long double, std::ratio<1, 100>>{123.456}}) == SV("1970-01-01 00:00:01.23"));
+}
+
+template <class CharT>
+static void test_fr_FR() {
+  using namespace std::literals::chrono_literals;
+
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("2000-01-01 01:02:03,123456789"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::microseconds>{946'688'523'123'456us}) ==
+         SV("2000-01-01 01:02:03,123456"));
+
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::milliseconds>{946'684'800'123ms}) ==
+         SV("2000-01-01 00:00:00,123"));
+  assert(stream_fr_FR_locale<CharT>(file_seconds{1'234'567'890s}) == SV("2009-02-13 23:31:30"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::minutes>{20'576'131min}) == SV("2009-02-13 23:31:00"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::hours>{342'935h}) == SV("2009-02-13 23:00:00"));
+  assert(stream_fr_FR_locale<CharT>(file_days{std::chrono::days{14'288}}) == SV("2009-02-13 00:00:00"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::weeks>{std::chrono::weeks{2041}}) ==
+         SV("2009-02-12 00:00:00"));
+
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<signed char, std::ratio<2, 1>>>{
+             std::chrono::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1970-01-01 00:02:00"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<short, std::ratio<1, 2>>>{
+             std::chrono::duration<short, std::ratio<1, 2>>{3600}}) == SV("1970-01-01 00:30:00,0"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<int, std::ratio<1, 4>>>{
+             std::chrono::duration<int, std::ratio<1, 4>>{3600}}) == SV("1970-01-01 00:15:00,00"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<long, std::ratio<1, 10>>>{
+             std::chrono::duration<long, std::ratio<1, 10>>{36611}}) == SV("1970-01-01 01:01:01,1"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<long long, std::ratio<1, 100>>>{
+             std::chrono::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("2009-02-13 23:31:30,10"));
+
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<float, std::ratio<1, 1>>>{
+             std::chrono::duration<float, std::ratio<1, 1>>{123.456}}) == SV("1970-01-01 00:02:03"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<double, std::ratio<1, 10>>>{
+             std::chrono::duration<double, std::ratio<1, 10>>{123.456}}) == SV("1970-01-01 00:00:12,3"));
+  assert(stream_fr_FR_locale<CharT>(file_time<std::chrono::duration<long double, std::ratio<1, 100>>>{
+             std::chrono::duration<long double, std::ratio<1, 100>>{123.456}}) == SV("1970-01-01 00:00:01,23"));
+}
+
+template <class CharT>
+static void test_ja_JP() {
+  using namespace std::literals::chrono_literals;
+
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("2000-01-01 01:02:03.123456789"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::microseconds>{946'688'523'123'456us}) ==
+         SV("2000-01-01 01:02:03.123456"));
+
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::milliseconds>{946'684'800'123ms}) ==
+         SV("2000-01-01 00:00:00.123"));
+  assert(stream_ja_JP_locale<CharT>(file_seconds{1'234'567'890s}) == SV("2009-02-13 23:31:30"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::minutes>{20'576'131min}) == SV("2009-02-13 23:31:00"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::hours>{342'935h}) == SV("2009-02-13 23:00:00"));
+  assert(stream_ja_JP_locale<CharT>(file_days{std::chrono::days{14'288}}) == SV("2009-02-13 00:00:00"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::weeks>{std::chrono::weeks{2041}}) ==
+         SV("2009-02-12 00:00:00"));
+
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<signed char, std::ratio<2, 1>>>{
+             std::chrono::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1970-01-01 00:02:00"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<short, std::ratio<1, 2>>>{
+             std::chrono::duration<short, std::ratio<1, 2>>{3600}}) == SV("1970-01-01 00:30:00.0"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<int, std::ratio<1, 4>>>{
+             std::chrono::duration<int, std::ratio<1, 4>>{3600}}) == SV("1970-01-01 00:15:00.00"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<long, std::ratio<1, 10>>>{
+             std::chrono::duration<long, std::ratio<1, 10>>{36611}}) == SV("1970-01-01 01:01:01.1"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<long long, std::ratio<1, 100>>>{
+             std::chrono::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("2009-02-13 23:31:30.10"));
+
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<float, std::ratio<1, 1>>>{
+             std::chrono::duration<float, std::ratio<1, 1>>{123.456}}) == SV("1970-01-01 00:02:03"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<double, std::ratio<1, 10>>>{
+             std::chrono::duration<double, std::ratio<1, 10>>{123.456}}) == SV("1970-01-01 00:00:12.3"));
+  assert(stream_ja_JP_locale<CharT>(file_time<std::chrono::duration<long double, std::ratio<1, 100>>>{
+             std::chrono::duration<long double, std::ratio<1, 100>>{123.456}}) == SV("1970-01-01 00:00:01.23"));
+}
+
+template <class CharT>
+static void test() {
+  test_c<CharT>();
+  test_fr_FR<CharT>();
+  test_ja_JP<CharT>();
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.clock/time.clock.local/ostream.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.local/ostream.pass.cpp
new file mode 100644
index 0000000000000..37d0a245ca9f6
--- /dev/null
+++ b/libcxx/test/std/time/time.clock/time.clock.local/ostream.pass.cpp
@@ -0,0 +1,199 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-localization
+// UNSUPPORTED: libcpp-has-no-incomplete-format
+
+// TODO FMT Investigate Windows issues.
+// UNSUPPORTED msvc, target={{.+}}-windows-gnu
+// TODO FMT It seems GCC uses too much memory in the CI and fails.
+// UNSUPPORTED: gcc-12
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+
+// class local_t;
+
+// template<class charT, class traits, class Duration>
+//   basic_ostream<charT, traits>&
+//     operator<<(basic_ostream<charT, traits>& os, const local_time<Duration>& tp);
+
+#include <chrono>
+#include <cassert>
+#include <ratio>
+#include <sstream>
+
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+#define SV(S) MAKE_STRING_VIEW(CharT, S)
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_c_locale(std::chrono::local_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_fr_FR_locale(std::chrono::local_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_fr_FR_UTF_8);
+  sstr.imbue(locale);
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT, class Duration>
+static std::basic_string<CharT> stream_ja_JP_locale(std::chrono::local_time<Duration> time_point) {
+  std::basic_stringstream<CharT> sstr;
+  const std::locale locale(LOCALE_ja_JP_UTF_8);
+  sstr.imbue(locale);
+  sstr << std::fixed << time_point;
+  return sstr.str();
+}
+
+template <class CharT>
+static void test_c() {
+  using namespace std::literals::chrono_literals;
+
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("2000-01-01 01:02:03.123456789"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::microseconds>{946'688'523'123'456us}) ==
+         SV("2000-01-01 01:02:03.123456"));
+
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::milliseconds>{946'684'800'123ms}) ==
+         SV("2000-01-01 00:00:00.123"));
+  assert(stream_c_locale<CharT>(std::chrono::local_seconds{1'234'567'890s}) == SV("2009-02-13 23:31:30"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::minutes>{20'576'131min}) ==
+         SV("2009-02-13 23:31:00"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::hours>{342'935h}) == SV("2009-02-13 23:00:00"));
+  assert(stream_c_locale<CharT>(std::chrono::local_days{std::chrono::days{14'288}}) == SV("2009-02-13 00:00:00"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::weeks>{std::chrono::weeks{2041}}) ==
+         SV("2009-02-12 00:00:00"));
+
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<signed char, std::ratio<2, 1>>>{
+             std::chrono::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1970-01-01 00:02:00"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<short, std::ratio<1, 2>>>{
+             std::chrono::duration<short, std::ratio<1, 2>>{3600}}) == SV("1970-01-01 00:30:00.0"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<int, std::ratio<1, 4>>>{
+             std::chrono::duration<int, std::ratio<1, 4>>{3600}}) == SV("1970-01-01 00:15:00.00"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<long, std::ratio<1, 10>>>{
+             std::chrono::duration<long, std::ratio<1, 10>>{36611}}) == SV("1970-01-01 01:01:01.1"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<long long, std::ratio<1, 100>>>{
+             std::chrono::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("2009-02-13 23:31:30.10"));
+
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<float, std::ratio<1, 1>>>{
+             std::chrono::duration<float, std::ratio<1, 1>>{123.456}}) == SV("1970-01-01 00:02:03"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<double, std::ratio<1, 10>>>{
+             std::chrono::duration<double, std::ratio<1, 10>>{123.456}}) == SV("1970-01-01 00:00:12.3"));
+  assert(stream_c_locale<CharT>(std::chrono::local_time<std::chrono::duration<long double, std::ratio<1, 100>>>{
+             std::chrono::duration<long double, std::ratio<1, 100>>{123.456}}) == SV("1970-01-01 00:00:01.23"));
+}
+
+template <class CharT>
+static void test_fr_FR() {
+  using namespace std::literals::chrono_literals;
+
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("2000-01-01 01:02:03,123456789"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::microseconds>{946'688'523'123'456us}) ==
+         SV("2000-01-01 01:02:03,123456"));
+
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::milliseconds>{946'684'800'123ms}) ==
+         SV("2000-01-01 00:00:00,123"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_seconds{1'234'567'890s}) == SV("2009-02-13 23:31:30"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::minutes>{20'576'131min}) ==
+         SV("2009-02-13 23:31:00"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::hours>{342'935h}) ==
+         SV("2009-02-13 23:00:00"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_days{std::chrono::days{14'288}}) == SV("2009-02-13 00:00:00"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::weeks>{std::chrono::weeks{2041}}) ==
+         SV("2009-02-12 00:00:00"));
+
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<signed char, std::ratio<2, 1>>>{
+             std::chrono::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1970-01-01 00:02:00"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<short, std::ratio<1, 2>>>{
+             std::chrono::duration<short, std::ratio<1, 2>>{3600}}) == SV("1970-01-01 00:30:00,0"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<int, std::ratio<1, 4>>>{
+             std::chrono::duration<int, std::ratio<1, 4>>{3600}}) == SV("1970-01-01 00:15:00,00"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<long, std::ratio<1, 10>>>{
+             std::chrono::duration<long, std::ratio<1, 10>>{36611}}) == SV("1970-01-01 01:01:01,1"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<long long, std::ratio<1, 100>>>{
+             std::chrono::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("2009-02-13 23:31:30,10"));
+
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<float, std::ratio<1, 1>>>{
+             std::chrono::duration<float, std::ratio<1, 1>>{123.456}}) == SV("1970-01-01 00:02:03"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<double, std::ratio<1, 10>>>{
+             std::chrono::duration<double, std::ratio<1, 10>>{123.456}}) == SV("1970-01-01 00:00:12,3"));
+  assert(stream_fr_FR_locale<CharT>(std::chrono::local_time<std::chrono::duration<long double, std::ratio<1, 100>>>{
+             std::chrono::duration<long double, std::ratio<1, 100>>{123.456}}) == SV("1970-01-01 00:00:01,23"));
+}
+
+template <class CharT>
+static void test_ja_JP() {
+  using namespace std::literals::chrono_literals;
+
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::nanoseconds>{946'688'523'123'456'789ns}) ==
+         SV("2000-01-01 01:02:03.123456789"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::microseconds>{946'688'523'123'456us}) ==
+         SV("2000-01-01 01:02:03.123456"));
+
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::milliseconds>{946'684'800'123ms}) ==
+         SV("2000-01-01 00:00:00.123"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_seconds{1'234'567'890s}) == SV("2009-02-13 23:31:30"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::minutes>{20'576'131min}) ==
+         SV("2009-02-13 23:31:00"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::hours>{342'935h}) ==
+         SV("2009-02-13 23:00:00"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_days{std::chrono::days{14'288}}) == SV("2009-02-13 00:00:00"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::weeks>{std::chrono::weeks{2041}}) ==
+         SV("2009-02-12 00:00:00"));
+
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<signed char, std::ratio<2, 1>>>{
+             std::chrono::duration<signed char, std::ratio<2, 1>>{60}}) == SV("1970-01-01 00:02:00"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<short, std::ratio<1, 2>>>{
+             std::chrono::duration<short, std::ratio<1, 2>>{3600}}) == SV("1970-01-01 00:30:00.0"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<int, std::ratio<1, 4>>>{
+             std::chrono::duration<int, std::ratio<1, 4>>{3600}}) == SV("1970-01-01 00:15:00.00"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<long, std::ratio<1, 10>>>{
+             std::chrono::duration<long, std::ratio<1, 10>>{36611}}) == SV("1970-01-01 01:01:01.1"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<long long, std::ratio<1, 100>>>{
+             std::chrono::duration<long long, std::ratio<1, 100>>{12'345'678'9010}}) == SV("2009-02-13 23:31:30.10"));
+
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<float, std::ratio<1, 1>>>{
+             std::chrono::duration<float, std::ratio<1, 1>>{123.456}}) == SV("1970-01-01 00:02:03"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<double, std::ratio<1, 10>>>{
+             std::chrono::duration<double, std::ratio<1, 10>>{123.456}}) == SV("1970-01-01 00:00:12.3"));
+  assert(stream_ja_JP_locale<CharT>(std::chrono::local_time<std::chrono::duration<long double, std::ratio<1, 100>>>{
+             std::chrono::duration<long double, std::ratio<1, 100>>{123.456}}) == SV("1970-01-01 00:00:01.23"));
+}
+
+template <class CharT>
+static void test() {
+  test_c<CharT>();
+  test_fr_FR<CharT>();
+  test_ja_JP<CharT>();
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp
new file mode 100644
index 0000000000000..8cc8ecff830b8
--- /dev/null
+++ b/libcxx/test/std/time/time.syn/formatter.file_time.pass.cpp
@@ -0,0 +1,940 @@
+//===----------------------------------------------------------------------===//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-localization
+// UNSUPPORTED: libcpp-has-no-incomplete-format
+
+// XFAIL: LIBCXX-FREEBSD-FIXME
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// TODO FMT Investigate Windows issues.
+// UNSUPPORTED: msvc, target={{.+}}-windows-gnu
+// TODO FMT It seems GCC uses too much memory in the CI and fails.
+// UNSUPPORTED: gcc-12
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+//
+// template<class charT> struct formatter<chrono::file_time, charT>;
+
+#include <chrono>
+#include <format>
+
+#include <cassert>
+#include <concepts>
+#include <locale>
+#include <iostream>
+#include <type_traits>
+
+#include "formatter_tests.h"
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+// Modeled after the system clock's usings, this make adapting the
+// tests easier.
+template <class Duration>
+using file_time    = std::chrono::time_point<std::chrono::file_clock, Duration>;
+using file_seconds = file_time<std::chrono::seconds>;
+using file_days    = file_time<std::chrono::days>;
+
+template <class CharT>
+static void test_no_chrono_specs() {
+  using namespace std::literals::chrono_literals;
+
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output
+  check(SV("1970-01-01 00:00:00"), SV("{}"), file_seconds(0s));
+  check(SV("2000-01-01 00:00:00"), SV("{}"), file_seconds(946'684'800s));
+  check(SV("2000-01-01 01:02:03"), SV("{}"), file_seconds(946'688'523s));
+
+  check(SV("2000-01-01 01:02:03.123"), SV("{}"), file_time<std::chrono::milliseconds>(946'688'523'123ms));
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_year() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        fmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%C='19'\t%EC='昭和'\t%y='70'\t%Oy='七十'\t%Ey='45'\t%Y='1970'\t%EY='昭和45年'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%C='20'\t%EC='平成'\t%y='09'\t%Oy='九'\t%Ey='21'\t%Y='2009'\t%EY='平成21年'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                 // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_month() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%b='Jan'\t%h='Jan'\t%B='January'\t%m='01'\t%Om='01'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%b='May'\t%h='May'\t%B='May'\t%m='05'\t%Om='05'\n"),
+        fmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%b='jan'\t%h='jan'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+#else
+  check(SV("%b='janv.'\t%h='janv.'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+#endif
+
+  check(SV("%b='mai'\t%h='mai'\t%B='mai'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#ifdef _WIN32
+  check(loc,
+        SV("%b='1'\t%h='1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b='5'\t%h='5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(_AIX)                    // _WIN32
+  check(loc,
+        SV("%b='1月'\t%h='1月'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b='5月'\t%h='5月'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(__APPLE__)               // _WIN32
+  check(loc,
+        SV("%b=' 1'\t%h=' 1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5'\t%h=' 5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#else                                  // _WIN32
+  check(loc,
+        SV("%b=' 1月'\t%h=' 1月'\t%B='1月'\t%m='01'\t%Om='一'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5月'\t%h=' 5月'\t%B='5月'\t%m='05'\t%Om='五'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#endif                                 // _WIN32
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"), fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        fmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"), lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                  // _WIN32
+  check(loc,
+        SV("%d='01'\t%Od='一'\t%e=' 1'\t%Oe='一'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%d='13'\t%Od='十三'\t%e='13'\t%Oe='十三'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+#endif                                 // _WIN32
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_weekday() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%a='Thu'\t%A='Thursday'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='Sun'\t%A='Sunday'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        fmt,
+        file_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%a='Jeu'\t%A='Jeudi'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='Dim'\t%A='Dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        file_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#else
+  check(SV("%a='jeu.'\t%A='jeudi'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='dim.'\t%A='dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        file_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#endif
+
+  // Use supplied locale (ja_JP).
+  // This locale has a different alternate, but not on all platforms
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%a='木'\t%A='木曜日'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        file_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#else                                  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%a='木'\t%A='木曜日'\t%u='4'\t%Ou='四'\t%w='4'\t%Ow='四'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='七'\t%w='0'\t%Ow='〇'\n"),
+        lfmt,
+        file_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#endif                                 // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day_of_year() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%j='%j'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%j='%j'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%j='001'\n"), fmt, file_seconds(0s));             // 00:00:00 UTC Thursday, 1 January 1970
+  check(SV("%j='138'\n"), fmt, file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%j='001'\n"), lfmt, file_seconds(0s));             // 00:00:00 UTC Thursday, 1 January 1970
+  check(SV("%j='138'\n"), lfmt, file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+  check(loc, SV("%j='001'\n"), lfmt, file_seconds(0s));             // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc, SV("%j='138'\n"), lfmt, file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_week() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"), fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        fmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"), lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#else                                  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%U='00'\t%OU='〇'\t%W='00'\t%OW='〇'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%U='20'\t%OU='二十'\t%W='20'\t%OW='二十'\n"),
+        lfmt,
+        file_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#endif                                 // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_iso_8601_week() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"), fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        fmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"), lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%g='70'\t%G='1970'\t%V='01'\t%OV='一'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='七'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                 // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01/01/70'\t%Ex='01/01/70'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='02/13/09'\t%Ex='02/13/09'\n"),
+        fmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01.01.1970'\t%Ex='01.01.1970'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13.02.2009'\t%Ex='13.02.2009'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01/01/1970'\t%Ex='01/01/1970'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13/02/2009'\t%Ex='13/02/2009'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%D='01/01/70'\t%F='1970-01-01'\t%x='1970/01/01'\t%Ex='1970/01/01'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009/02/13'\t%Ex='2009/02/13'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                  // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%D='01/01/70'\t%F='1970-01-01'\t%x='1970年01月01日'\t%Ex='昭和45年01月01日'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009年02月13日'\t%Ex='平成21年02月13日'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                 // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt = SV(
+      "{:"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV(
+      "{:L"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+           "%p='AM'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='12:00:00 AM'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+           "%p='PM'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='11:31:30 PM'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        fmt,
+        std::chrono::sys_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+  // Use the global locale (fr_FR)
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#if defined(_AIX)
+           "%p='AM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#ifdef _WIN32
+           "%r='00:00:00'\t"
+#elif defined(_AIX)
+           "%r='12:00:00 AM'\t"
+#elif defined(__APPLE__)
+           "%r=''\t"
+#else
+           "%r='12:00:00 '\t"
+#endif
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30,123'\t"
+           "%OS='30,123'\t"
+#if defined(_AIX)
+           "%p='PM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='23:31'\t"
+           "%T='23:31:30,123'\t"
+#ifdef _WIN32
+           "%r='23:31:30'\t"
+#elif defined(_AIX)
+           "%r='11:31:30 PM'\t"
+#elif defined(__APPLE__)
+           "%r=''\t"
+#else
+           "%r='11:31:30 '\t"
+#endif
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        lfmt,
+        std::chrono::sys_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#  if defined(__APPLE__)
+           "%p='AM'\t"
+#  else
+           "%p='午前'\t"
+#  endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#  if defined(__APPLE__)
+           "%r='12:00:00 AM'\t"
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+#  else
+           "%r='午前12:00:00'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+#  endif
+           "\n"),
+        lfmt,
+        std::chrono::hh_mm_ss(0s));
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+#  if defined(__APPLE__)
+           "%p='PM'\t"
+#  else
+           "%p='午後'\t"
+#  endif
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+#  if defined(__APPLE__)
+           "%r='11:31:30 PM'\t"
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+#  else
+           "%r='午後11:31:30'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+#  endif
+           "\n"),
+        lfmt,
+        std::chrono::hh_mm_ss(23h + 31min + 30s + 123ms));
+#else  // defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='〇'\t"
+           "%I='12'\t"
+           "%OI='十二'\t"
+           "%M='00'\t"
+           "%OM='〇'\t"
+           "%S='00'\t"
+           "%OS='〇'\t"
+           "%p='午前'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='午前12時00分00秒'\t"
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+           "\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='二十三'\t"
+           "%I='11'\t"
+           "%OI='十一'\t"
+           "%M='31'\t"
+           "%OM='三十一'\t"
+           "%S='30.123'\t"
+           "%OS='三十.123'\t"
+           "%p='午後'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='午後11時31分30秒'\t"
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+           "\n"),
+        lfmt,
+        std::chrono::sys_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif // defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date_time() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%c='%c'%t%%Ec='%Ec'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%c='%c'%t%%Ec='%Ec'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%c='Thu Jan  1 00:00:00 1970'\t%Ec='Thu Jan  1 00:00:00 1970'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%c='Fri Feb 13 23:31:30 2009'\t%Ec='Fri Feb 13 23:31:30 2009'\n"),
+        fmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='jeu. 01 janv. 1970 00:00:00 GMT'\t%Ec='jeu. 01 janv. 1970 00:00:00 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c=' 1 janvier 1970 à 00:00:00 UTC'\t%Ec=' 1 janvier 1970 à 00:00:00 UTC'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Jeu  1 jan 00:00:00 1970'\t%Ec='Jeu  1 jan 00:00:00 1970'\n"),
+#else
+      SV("%c='jeu. 01 janv. 1970 00:00:00'\t%Ec='jeu. 01 janv. 1970 00:00:00'\n"),
+#endif
+      lfmt,
+      file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c='13 février 2009 à 23:31:30 UTC'\t%Ec='13 février 2009 à 23:31:30 UTC'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Ven 13 fév 23:31:30 2009'\t%Ec='Ven 13 fév 23:31:30 2009'\n"),
+#else
+      SV("%c='ven. 13 févr. 2009 23:31:30'\t%Ec='ven. 13 févr. 2009 23:31:30'\n"),
+#endif
+      lfmt,
+      file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__)
+  check(loc,
+        SV("%c='木  1/ 1 00:00:00 1970'\t%Ec='木  1/ 1 00:00:00 1970'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='金  2/13 23:31:30 2009'\t%Ec='金  2/13 23:31:30 2009'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#elif defined(_AIX)                    // __APPLE__
+  check(loc,
+        SV("%c='1970年01月 1日 00:00:00 UTC'\t%Ec='1970年01月 1日 00:00:00 UTC'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='2009年02月13日 23:31:30 UTC'\t%Ec='2009年02月13日 23:31:30 UTC'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                  // __APPLE__
+  check(loc,
+        SV("%c='1970年01月01日 00時00分00秒'\t%Ec='昭和45年01月01日 00時00分00秒'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%c='2009年02月13日 23時31分30秒'\t%Ec='平成21年02月13日 23時31分30秒'\n"),
+        lfmt,
+        file_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                 // __APPLE__
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time_zone() {
+// The Apple CI gives %z='-0700'	%Ez='-0700'	%Oz='-0700'	%Z='UTC'
+// -0700 looks like the local time where the CI happens to recide, therefore
+// omit this test on Apple.
+#if !defined(__APPLE__)
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+#  if defined(_AIX)
+  // Non localized output using C-locale
+  check(SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  // Use the global locale (fr_FR)
+  check(SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+  check(loc,
+        SV("%z='UTC'\t%Ez='UTC'\t%Oz='UTC'\t%Z='UTC'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+#  else                    // defined(_AIX)
+  // Non localized output using C-locale
+  check(SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"),
+        fmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  // Use the global locale (fr_FR)
+  check(SV("%z='+0000'\t%Ez='+0000'\t%Oz='+0000'\t%Z='UTC'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+  check(loc,
+        SV("%z='+0000'\t%Ez='+0000'\t%Oz='+〇'\t%Z='UTC'\n"),
+        lfmt,
+        file_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+#  endif                   // defined(_AIX)
+  std::locale::global(std::locale::classic());
+#endif                     // !defined(__APPLE__)
+}
+
+template <class CharT>
+static void test_valid_values() {
+  test_valid_values_year<CharT>();
+  test_valid_values_month<CharT>();
+  test_valid_values_day<CharT>();
+  test_valid_values_weekday<CharT>();
+  test_valid_values_day_of_year<CharT>();
+  test_valid_values_week<CharT>();
+  test_valid_values_iso_8601_week<CharT>();
+  test_valid_values_date<CharT>();
+  test_valid_values_time<CharT>();
+  test_valid_values_date_time<CharT>();
+  test_valid_values_time_zone<CharT>();
+}
+
+template <class CharT>
+static void test() {
+  using namespace std::literals::chrono_literals;
+
+  test_no_chrono_specs<CharT>();
+  test_valid_values<CharT>();
+
+  check_invalid_types<CharT>(
+      {SV("a"),  SV("A"),  SV("b"),  SV("B"),  SV("c"),  SV("C"),  SV("d"),  SV("D"),  SV("e"),  SV("F"),  SV("g"),
+       SV("G"),  SV("h"),  SV("H"),  SV("I"),  SV("j"),  SV("m"),  SV("M"),  SV("p"),  SV("r"),  SV("R"),  SV("S"),
+       SV("T"),  SV("u"),  SV("U"),  SV("V"),  SV("w"),  SV("W"),  SV("x"),  SV("X"),  SV("y"),  SV("Y"),  SV("z"),
+       SV("Z"),  SV("Ec"), SV("EC"), SV("Ex"), SV("EX"), SV("Ey"), SV("EY"), SV("Ez"), SV("Od"), SV("Oe"), SV("OH"),
+       SV("OI"), SV("Om"), SV("OM"), SV("OS"), SV("Ou"), SV("OU"), SV("OV"), SV("Ow"), SV("OW"), SV("Oy"), SV("Oz")},
+      file_seconds(0s));
+
+  check_exception("Expected '%' or '}' in the chrono format-string", SV("{:A"), file_seconds(0s));
+  check_exception("The chrono-specs contains a '{'", SV("{:%%{"), file_seconds(0s));
+  check_exception("End of input while parsing the modifier chrono conversion-spec", SV("{:%"), file_seconds(0s));
+  check_exception("End of input while parsing the modifier E", SV("{:%E"), file_seconds(0s));
+  check_exception("End of input while parsing the modifier O", SV("{:%O"), file_seconds(0s));
+
+  // Precision not allowed
+  check_exception("Expected '%' or '}' in the chrono format-string", SV("{:.3}"), file_seconds(0s));
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp b/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp
new file mode 100644
index 0000000000000..09e0c26fab683
--- /dev/null
+++ b/libcxx/test/std/time/time.syn/formatter.local_time.pass.cpp
@@ -0,0 +1,892 @@
+//===----------------------------------------------------------------------===//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: no-localization
+// UNSUPPORTED: libcpp-has-no-incomplete-format
+
+// XFAIL: LIBCXX-FREEBSD-FIXME
+
+// TODO FMT This test should not require std::to_chars(floating-point)
+// XFAIL: availability-fp_to_chars-missing
+
+// TODO FMT Investigate Windows issues.
+// UNSUPPORTED: msvc, target={{.+}}-windows-gnu
+// TODO FMT It seems GCC uses too much memory in the CI and fails.
+// UNSUPPORTED: gcc-12
+
+// REQUIRES: locale.fr_FR.UTF-8
+// REQUIRES: locale.ja_JP.UTF-8
+
+// <chrono>
+//
+// template<class charT> struct formatter<chrono::day, charT>;
+
+#include <chrono>
+#include <format>
+
+#include <cassert>
+#include <concepts>
+#include <locale>
+#include <iostream>
+#include <type_traits>
+
+#include "formatter_tests.h"
+#include "make_string.h"
+#include "platform_support.h" // locale name macros
+#include "test_macros.h"
+
+template <class CharT>
+static void test_no_chrono_specs() {
+  using namespace std::literals::chrono_literals;
+
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output
+  check(SV("1970-01-01 00:00:00"), SV("{}"), std::chrono::local_seconds(0s));
+  check(SV("2000-01-01 00:00:00"), SV("{}"), std::chrono::local_seconds(946'684'800s));
+  check(SV("2000-01-01 01:02:03"), SV("{}"), std::chrono::local_seconds(946'688'523s));
+
+  check(SV("2000-01-01 01:02:03.123"), SV("{}"), std::chrono::local_time<std::chrono::milliseconds>(946'688'523'123ms));
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_year() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%C='%C'%t%%EC='%EC'%t%%y='%y'%t%%Oy='%Oy'%t%%Ey='%Ey'%t%%Y='%Y'%t%%EY='%EY'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        fmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%C='19'\t%EC='19'\t%y='70'\t%Oy='70'\t%Ey='70'\t%Y='1970'\t%EY='1970'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%C='20'\t%EC='20'\t%y='09'\t%Oy='09'\t%Ey='09'\t%Y='2009'\t%EY='2009'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                                // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%C='19'\t%EC='昭和'\t%y='70'\t%Oy='七十'\t%Ey='45'\t%Y='1970'\t%EY='昭和45年'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%C='20'\t%EC='平成'\t%y='09'\t%Oy='九'\t%Ey='21'\t%Y='2009'\t%EY='平成21年'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                               // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_month() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%b='%b'%t%%h='%h'%t%%B='%B'%t%%m='%m'%t%%Om='%Om'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%b='Jan'\t%h='Jan'\t%B='January'\t%m='01'\t%Om='01'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%b='May'\t%h='May'\t%B='May'\t%m='05'\t%Om='05'\n"),
+        fmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%b='jan'\t%h='jan'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+#else
+  check(SV("%b='janv.'\t%h='janv.'\t%B='janvier'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+#endif
+
+  check(SV("%b='mai'\t%h='mai'\t%B='mai'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#ifdef _WIN32
+  check(loc,
+        SV("%b='1'\t%h='1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b='5'\t%h='5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(_AIX)                                  // _WIN32
+  check(loc,
+        SV("%b='1月'\t%h='1月'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b='5月'\t%h='5月'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#elif defined(__APPLE__)                             // _WIN32
+  check(loc,
+        SV("%b=' 1'\t%h=' 1'\t%B='1月'\t%m='01'\t%Om='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5'\t%h=' 5'\t%B='5月'\t%m='05'\t%Om='05'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#else                                                // _WIN32
+  check(loc,
+        SV("%b=' 1月'\t%h=' 1月'\t%B='1月'\t%m='01'\t%Om='一'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%b=' 5月'\t%h=' 5月'\t%B='5月'\t%m='05'\t%Om='五'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#endif                                               // _WIN32
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%d='%d'%t%%Od='%Od'%t%%e='%e'%t%%Oe='%Oe'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        fmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%d='01'\t%Od='01'\t%e=' 1'\t%Oe=' 1'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%d='13'\t%Od='13'\t%e='13'\t%Oe='13'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                                // _WIN32
+  check(loc,
+        SV("%d='01'\t%Od='一'\t%e=' 1'\t%Oe='一'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%d='13'\t%Od='十三'\t%e='13'\t%Oe='十三'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+#endif                                               // _WIN32
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_weekday() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt =
+      SV("{:%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+  constexpr std::basic_string_view<CharT> lfmt =
+      SV("{:L%%a='%a'%t%%A='%A'%t%%u='%u'%t%%Ou='%Ou'%t%%w='%w'%t%%Ow='%Ow'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%a='Thu'\t%A='Thursday'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='Sun'\t%A='Sunday'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        fmt,
+        std::chrono::local_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%a='Jeu'\t%A='Jeudi'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='Dim'\t%A='Dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        std::chrono::local_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#else
+  check(SV("%a='jeu.'\t%A='jeudi'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%a='dim.'\t%A='dimanche'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        std::chrono::local_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#endif
+
+  // Use supplied locale (ja_JP).
+  // This locale has a different alternate, but not on all platforms
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%a='木'\t%A='木曜日'\t%u='4'\t%Ou='4'\t%w='4'\t%Ow='4'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='7'\t%w='0'\t%Ow='0'\n"),
+        lfmt,
+        std::chrono::local_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#else                                                // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%a='木'\t%A='木曜日'\t%u='4'\t%Ou='四'\t%w='4'\t%Ow='四'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%a='日'\t%A='日曜日'\t%u='7'\t%Ou='七'\t%w='0'\t%Ow='〇'\n"),
+        lfmt,
+        std::chrono::local_seconds(4'294'967'295s)); // 06:28:15 UTC on Sunday, 7 February 2106
+#endif                                               // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_day_of_year() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%j='%j'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%j='%j'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%j='001'\n"), fmt, std::chrono::local_seconds(0s));             // 00:00:00 UTC Thursday, 1 January 1970
+  check(SV("%j='138'\n"), fmt, std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%j='001'\n"), lfmt, std::chrono::local_seconds(0s));             // 00:00:00 UTC Thursday, 1 January 1970
+  check(SV("%j='138'\n"), lfmt, std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+  check(loc, SV("%j='001'\n"), lfmt, std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%j='138'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_week() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%U='%U'%t%%OU='%OU'%t%%W='%W'%t%%OW='%OW'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        fmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use the global locale (fr_FR)
+  check(SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%U='00'\t%OU='00'\t%W='00'\t%OW='00'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%U='20'\t%OU='20'\t%W='20'\t%OW='20'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#else                                                // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%U='00'\t%OU='〇'\t%W='00'\t%OW='〇'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%U='20'\t%OU='二十'\t%W='20'\t%OW='二十'\n"),
+        lfmt,
+        std::chrono::local_seconds(2'000'000'000s)); // 03:33:20 UTC on Wednesday, 18 May 2033
+#endif                                               // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_iso_8601_week() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%g='%g'%t%%G='%G'%t%%V='%V'%t%%OV='%OV'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        fmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%g='70'\t%G='1970'\t%V='01'\t%OV='01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='07'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                                // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%g='70'\t%G='1970'\t%V='01'\t%OV='一'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%g='09'\t%G='2009'\t%V='07'\t%OV='七'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                               // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%D='%D'%t%%F='%F'%t%%x='%x'%t%%Ex='%Ex'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01/01/70'\t%Ex='01/01/70'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='02/13/09'\t%Ex='02/13/09'\n"),
+        fmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+#if defined(__APPLE__)
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01.01.1970'\t%Ex='01.01.1970'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13.02.2009'\t%Ex='13.02.2009'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else
+  check(SV("%D='01/01/70'\t%F='1970-01-01'\t%x='01/01/1970'\t%Ex='01/01/1970'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%D='02/13/09'\t%F='2009-02-13'\t%x='13/02/2009'\t%Ex='13/02/2009'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.
+#if defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%D='01/01/70'\t%F='1970-01-01'\t%x='1970/01/01'\t%Ex='1970/01/01'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009/02/13'\t%Ex='2009/02/13'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                                // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%D='01/01/70'\t%F='1970-01-01'\t%x='1970年01月01日'\t%Ex='昭和45年01月01日'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%D='02/13/09'\t%F='2009-02-13'\t%x='2009年02月13日'\t%Ex='平成21年02月13日'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                               // defined(_WIN32) || defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_time() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt = SV(
+      "{:"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV(
+      "{:L"
+      "%%H='%H'%t"
+      "%%OH='%OH'%t"
+      "%%I='%I'%t"
+      "%%OI='%OI'%t"
+      "%%M='%M'%t"
+      "%%OM='%OM'%t"
+      "%%S='%S'%t"
+      "%%OS='%OS'%t"
+      "%%p='%p'%t"
+      "%%R='%R'%t"
+      "%%T='%T'%t"
+      "%%r='%r'%t"
+      "%%X='%X'%t"
+      "%%EX='%EX'%t"
+      "%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+           "%p='AM'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='12:00:00 AM'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+           "%p='PM'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='11:31:30 PM'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        fmt,
+        std::chrono::local_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+  // Use the global locale (fr_FR)
+  check(SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#if defined(_AIX)
+           "%p='AM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#ifdef _WIN32
+           "%r='00:00:00'\t"
+#elif defined(_AIX)
+           "%r='12:00:00 AM'\t"
+#elif defined(__APPLE__)
+           "%r=''\t"
+#else
+           "%r='12:00:00 '\t"
+#endif
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+           "\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30,123'\t"
+           "%OS='30,123'\t"
+#if defined(_AIX)
+           "%p='PM'\t"
+#else
+           "%p=''\t"
+#endif
+           "%R='23:31'\t"
+           "%T='23:31:30,123'\t"
+#ifdef _WIN32
+           "%r='23:31:30'\t"
+#elif defined(_AIX)
+           "%r='11:31:30 PM'\t"
+#elif defined(__APPLE__)
+           "%r=''\t"
+#else
+           "%r='11:31:30 '\t"
+#endif
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+           "\n"),
+        lfmt,
+        std::chrono::local_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='00'\t"
+           "%I='12'\t"
+           "%OI='12'\t"
+           "%M='00'\t"
+           "%OM='00'\t"
+           "%S='00'\t"
+           "%OS='00'\t"
+#  if defined(__APPLE__)
+           "%p='AM'\t"
+#  else
+           "%p='午前'\t"
+#  endif
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+#  if defined(__APPLE__)
+           "%r='12:00:00 AM'\t"
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+#  else
+           "%r='午前12:00:00'\t"
+           "%X='00:00:00'\t"
+           "%EX='00:00:00'\t"
+#  endif
+           "\n"),
+        lfmt,
+        std::chrono::hh_mm_ss(0s));
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='23'\t"
+           "%I='11'\t"
+           "%OI='11'\t"
+           "%M='31'\t"
+           "%OM='31'\t"
+           "%S='30.123'\t"
+           "%OS='30.123'\t"
+#  if defined(__APPLE__)
+           "%p='PM'\t"
+#  else
+           "%p='午後'\t"
+#  endif
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+#  if defined(__APPLE__)
+           "%r='11:31:30 PM'\t"
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+#  else
+           "%r='午後11:31:30'\t"
+           "%X='23:31:30'\t"
+           "%EX='23:31:30'\t"
+#  endif
+           "\n"),
+        lfmt,
+        std::chrono::hh_mm_ss(23h + 31min + 30s + 123ms));
+#else  // defined(__APPLE__) || defined(_AIX)
+  check(loc,
+        SV("%H='00'\t"
+           "%OH='〇'\t"
+           "%I='12'\t"
+           "%OI='十二'\t"
+           "%M='00'\t"
+           "%OM='〇'\t"
+           "%S='00'\t"
+           "%OS='〇'\t"
+           "%p='午前'\t"
+           "%R='00:00'\t"
+           "%T='00:00:00'\t"
+           "%r='午前12時00分00秒'\t"
+           "%X='00時00分00秒'\t"
+           "%EX='00時00分00秒'\t"
+           "\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%H='23'\t"
+           "%OH='二十三'\t"
+           "%I='11'\t"
+           "%OI='十一'\t"
+           "%M='31'\t"
+           "%OM='三十一'\t"
+           "%S='30.123'\t"
+           "%OS='三十.123'\t"
+           "%p='午後'\t"
+           "%R='23:31'\t"
+           "%T='23:31:30.123'\t"
+           "%r='午後11時31分30秒'\t"
+           "%X='23時31分30秒'\t"
+           "%EX='23時31分30秒'\t"
+           "\n"),
+        lfmt,
+        std::chrono::local_time<std::chrono::milliseconds>(
+            1'234'567'890'123ms)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif // defined(__APPLE__) || defined(_AIX)
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values_date_time() {
+  using namespace std::literals::chrono_literals;
+
+  constexpr std::basic_string_view<CharT> fmt  = SV("{:%%c='%c'%t%%Ec='%Ec'%n}");
+  constexpr std::basic_string_view<CharT> lfmt = SV("{:L%%c='%c'%t%%Ec='%Ec'%n}");
+
+  const std::locale loc(LOCALE_ja_JP_UTF_8);
+  std::locale::global(std::locale(LOCALE_fr_FR_UTF_8));
+
+  // Non localized output using C-locale
+  check(SV("%c='Thu Jan  1 00:00:00 1970'\t%Ec='Thu Jan  1 00:00:00 1970'\n"),
+        fmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(SV("%c='Fri Feb 13 23:31:30 2009'\t%Ec='Fri Feb 13 23:31:30 2009'\n"),
+        fmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use the global locale (fr_FR)
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='jeu. 01 janv. 1970 00:00:00 GMT'\t%Ec='jeu. 01 janv. 1970 00:00:00 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c=' 1 janvier 1970 à 00:00:00 UTC'\t%Ec=' 1 janvier 1970 à 00:00:00 UTC'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Jeu  1 jan 00:00:00 1970'\t%Ec='Jeu  1 jan 00:00:00 1970'\n"),
+#else
+      SV("%c='jeu. 01 janv. 1970 00:00:00'\t%Ec='jeu. 01 janv. 1970 00:00:00'\n"),
+#endif
+      lfmt,
+      std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(
+// https://sourceware.org/bugzilla/show_bug.cgi?id=24054
+#if defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 29
+      SV("%c='ven. 13 févr. 2009 23:31:30 GMT'\t%Ec='ven. 13 févr. 2009 23:31:30 GMT'\n"),
+#elif defined(_AIX)
+      SV("%c='13 février 2009 à 23:31:30 UTC'\t%Ec='13 février 2009 à 23:31:30 UTC'\n"),
+#elif defined(__APPLE__)
+      SV("%c='Ven 13 fév 23:31:30 2009'\t%Ec='Ven 13 fév 23:31:30 2009'\n"),
+#else
+      SV("%c='ven. 13 févr. 2009 23:31:30'\t%Ec='ven. 13 févr. 2009 23:31:30'\n"),
+#endif
+      lfmt,
+      std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+
+  // Use supplied locale (ja_JP). This locale has a different alternate.a
+#if defined(__APPLE__)
+  check(loc,
+        SV("%c='木  1/ 1 00:00:00 1970'\t%Ec='木  1/ 1 00:00:00 1970'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='金  2/13 23:31:30 2009'\t%Ec='金  2/13 23:31:30 2009'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#elif defined(_AIX)                                  // __APPLE__
+  check(loc,
+        SV("%c='1970年01月 1日 00:00:00 UTC'\t%Ec='1970年01月 1日 00:00:00 UTC'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+  check(loc,
+        SV("%c='2009年02月13日 23:31:30 UTC'\t%Ec='2009年02月13日 23:31:30 UTC'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#else                                                // __APPLE__
+  check(loc,
+        SV("%c='1970年01月01日 00時00分00秒'\t%Ec='昭和45年01月01日 00時00分00秒'\n"),
+        lfmt,
+        std::chrono::local_seconds(0s)); // 00:00:00 UTC Thursday, 1 January 1970
+
+  check(loc,
+        SV("%c='2009年02月13日 23時31分30秒'\t%Ec='平成21年02月13日 23時31分30秒'\n"),
+        lfmt,
+        std::chrono::local_seconds(1'234'567'890s)); // 23:31:30 UTC on Friday, 13 February 2009
+#endif                                               // __APPLE__
+
+  std::locale::global(std::locale::classic());
+}
+
+template <class CharT>
+static void test_valid_values() {
+  test_valid_values_year<CharT>();
+  test_valid_values_month<CharT>();
+  test_valid_values_day<CharT>();
+  test_valid_values_weekday<CharT>();
+  test_valid_values_day_of_year<CharT>();
+  test_valid_values_week<CharT>();
+  test_valid_values_iso_8601_week<CharT>();
+  test_valid_values_date<CharT>();
+  test_valid_values_time<CharT>();
+  test_valid_values_date_time<CharT>();
+}
+
+template <class CharT>
+static void test() {
+  using namespace std::literals::chrono_literals;
+
+  test_no_chrono_specs<CharT>();
+  test_valid_values<CharT>();
+  check_invalid_types<CharT>(
+      {
+          SV("a"),  SV("A"),  SV("b"),  SV("B"),  SV("c"),  SV("C"),  SV("d"),  SV("D"),  SV("e"),  SV("F"),  SV("g"),
+          SV("G"),  SV("h"),  SV("H"),  SV("I"),  SV("j"),  SV("m"),  SV("M"),  SV("p"),  SV("r"),  SV("R"),  SV("S"),
+          SV("T"),  SV("u"),  SV("U"),  SV("V"),  SV("w"),  SV("W"),  SV("x"),  SV("X"),  SV("y"),  SV("Y"),  SV("Ec"),
+          SV("EC"), SV("Ex"), SV("EX"), SV("Ey"), SV("EY"), SV("Od"), SV("Oe"), SV("OH"), SV("OI"), SV("Om"), SV("OM"),
+          SV("OS"), SV("Ou"), SV("OU"), SV("OV"), SV("Ow"), SV("OW"), SV("Oy"),
+      },
+      std::chrono::local_seconds(0s));
+
+  check_exception("Expected '%' or '}' in the chrono format-string", SV("{:A"), std::chrono::local_seconds(0s));
+  check_exception("The chrono-specs contains a '{'", SV("{:%%{"), std::chrono::local_seconds(0s));
+  check_exception(
+      "End of input while parsing the modifier chrono conversion-spec", SV("{:%"), std::chrono::local_seconds(0s));
+  check_exception("End of input while parsing the modifier E", SV("{:%E"), std::chrono::local_seconds(0s));
+  check_exception("End of input while parsing the modifier O", SV("{:%O"), std::chrono::local_seconds(0s));
+
+  // Precision not allowed
+  check_exception("Expected '%' or '}' in the chrono format-string", SV("{:.3}"), std::chrono::local_seconds(0s));
+}
+
+int main(int, char**) {
+  test<char>();
+
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/expol/is_execution_policy.compile.pass.cpp b/libcxx/test/std/utilities/expol/is_execution_policy.compile.pass.cpp
new file mode 100644
index 0000000000000..57845c5146614
--- /dev/null
+++ b/libcxx/test/std/utilities/expol/is_execution_policy.compile.pass.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// template<class T> struct is_execution_policy;
+// template<class T> constexpr bool is_execution_policy_v = is_execution_policy<T>::value;
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+#include <execution>
+
+#include "test_macros.h"
+
+static_assert(std::is_execution_policy<std::execution::sequenced_policy>::value);
+static_assert(std::is_execution_policy_v<std::execution::sequenced_policy>);
+static_assert(std::is_execution_policy<std::execution::parallel_policy>::value);
+static_assert(std::is_execution_policy_v<std::execution::parallel_policy>);
+static_assert(std::is_execution_policy<std::execution::parallel_unsequenced_policy>::value);
+static_assert(std::is_execution_policy_v<std::execution::parallel_unsequenced_policy>);
+
+#if TEST_STD_VER >= 20
+static_assert(std::is_execution_policy<std::execution::unsequenced_policy>::value);
+static_assert(std::is_execution_policy_v<std::execution::unsequenced_policy>);
+#endif
diff --git a/libcxx/test/std/utilities/expol/policies.compile.pass.cpp b/libcxx/test/std/utilities/expol/policies.compile.pass.cpp
new file mode 100644
index 0000000000000..4de439f1acacf
--- /dev/null
+++ b/libcxx/test/std/utilities/expol/policies.compile.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// class sequenced_policy;
+// class parallel_policy;
+// class parallel_unsequenced_policy;
+// class unsequenced_policy; // since C++20
+//
+// inline constexpr sequenced_policy seq = implementation-defined;
+// inline constexpr parallel_policy par = implementation-defined;
+// inline constexpr parallel_unsequenced_policy par_unseq = implementation-defined;
+// inline constexpr unsequenced_policy unseq = implementation-defined; // since C++20
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// REQUIRES: with-pstl
+
+#include <execution>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <class T>
+using remove_cvref_t = std::remove_cv_t<std::remove_reference_t<T>>;
+
+template <class T>
+TEST_NOINLINE void use(T&) {}
+
+static_assert(std::is_same_v<remove_cvref_t<decltype(std::execution::seq)>, std::execution::sequenced_policy>);
+static_assert(std::is_same_v<remove_cvref_t<decltype(std::execution::par)>, std::execution::parallel_policy>);
+static_assert(
+    std::is_same_v<remove_cvref_t<decltype(std::execution::par_unseq)>, std::execution::parallel_unsequenced_policy>);
+
+#if TEST_STD_VER >= 20
+static_assert(std::is_same_v<remove_cvref_t<decltype(std::execution::unseq)>, std::execution::unsequenced_policy>);
+#endif
+
+int main(int, char**) {
+  use(std::execution::seq);
+  use(std::execution::par);
+  use(std::execution::par_unseq);
+#if TEST_STD_VER >= 20
+  use(std::execution::unseq);
+#endif
+}
diff --git a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp
index 6957c7b14e356..1154d0d2d5eaa 100644
--- a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp
@@ -140,8 +140,8 @@ void test_P1361() {
   //assert_is_formattable<std::chrono::utc_time<std::chrono::microseconds>, CharT>();
   //assert_is_formattable<std::chrono::tai_time<std::chrono::microseconds>, CharT>();
   //assert_is_formattable<std::chrono::gps_time<std::chrono::microseconds>, CharT>();
-  assert_is_not_formattable<std::chrono::file_time<std::chrono::microseconds>, CharT>();
-  assert_is_not_formattable<std::chrono::local_time<std::chrono::microseconds>, CharT>();
+  assert_is_formattable<std::chrono::file_time<std::chrono::microseconds>, CharT>();
+  assert_is_formattable<std::chrono::local_time<std::chrono::microseconds>, CharT>();
 
   assert_is_formattable<std::chrono::day, CharT>();
   assert_is_formattable<std::chrono::month, CharT>();
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/PR23141_invoke_not_constexpr.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/PR23141.pass.cpp
similarity index 65%
rename from libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/PR23141_invoke_not_constexpr.pass.cpp
rename to libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/PR23141.pass.cpp
index eeb87e0db0a8f..f1c5b07aa51b1 100644
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/PR23141_invoke_not_constexpr.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/PR23141.pass.cpp
@@ -11,9 +11,9 @@
 // <functional>
 
 // template<CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 // template<Returnable R, CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 
 // https://llvm.org/PR23141
 #include <functional>
@@ -21,18 +21,23 @@
 
 #include "test_macros.h"
 
-struct Fun
-{
+struct Fun {
   template<typename T, typename U>
-  void operator()(T &&, U &&) const
-  {
+  TEST_CONSTEXPR_CXX20 void operator()(T &&, U &&) const {
     static_assert(std::is_same<U, int &>::value, "");
   }
 };
 
-int main(int, char**)
-{
-    std::bind(Fun{}, std::placeholders::_1, 42)("hello");
+TEST_CONSTEXPR_CXX20 bool test() {
+  std::bind(Fun{}, std::placeholders::_1, 42)("hello");
+  return true;
+}
+
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_int_0.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_int_0.pass.cpp
index 4d3f679893d13..37c914467df1f 100644
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_int_0.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_int_0.pass.cpp
@@ -11,9 +11,9 @@
 // <functional>
 
 // template<CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 // template<Returnable R, CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 
 #include <functional>
 #include <cassert>
@@ -21,29 +21,25 @@
 #include "test_macros.h"
 
 template <class R, class F>
-void
-test(F f, R expected)
-{
+TEST_CONSTEXPR_CXX20
+void test(F f, R expected) {
     assert(f() == expected);
 }
 
 template <class R, class F>
-void
-test_const(const F& f, R expected)
-{
+TEST_CONSTEXPR_CXX20
+void test_const(const F& f, R expected) {
     assert(f() == expected);
 }
 
-int f() {return 1;}
+TEST_CONSTEXPR_CXX20 int f() {return 1;}
 
-struct A_int_0
-{
-    int operator()() {return 4;}
-    int operator()() const {return 5;}
+struct A_int_0 {
+    TEST_CONSTEXPR_CXX20 int operator()() {return 4;}
+    TEST_CONSTEXPR_CXX20 int operator()() const {return 5;}
 };
 
-int main(int, char**)
-{
+TEST_CONSTEXPR_CXX20 bool test_all() {
     test(std::bind(f), 1);
     test(std::bind(&f), 1);
     test(std::bind(A_int_0()), 4);
@@ -53,6 +49,14 @@ int main(int, char**)
     test(std::bind<int>(&f), 1);
     test(std::bind<int>(A_int_0()), 4);
     test_const(std::bind<int>(A_int_0()), 5);
+    return true;
+}
+
+int main(int, char**) {
+    test_all();
+#if TEST_STD_VER >= 20
+    static_assert(test_all());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_lvalue.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_lvalue.pass.cpp
index 49547a22b39a9..82431bb673779 100644
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_lvalue.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_lvalue.pass.cpp
@@ -11,9 +11,9 @@
 // <functional>
 
 // template<CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 // template<Returnable R, CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 
 #include <stdio.h>
 
@@ -140,27 +140,22 @@ test_void_1()
 
 // 1 arg, return int
 
-int f_int_1(int i)
-{
+TEST_CONSTEXPR_CXX20 int f_int_1(int i) {
     return i + 1;
 }
 
-struct A_int_1
-{
-    A_int_1() : data_(5) {}
-    int operator()(int i)
-    {
+struct A_int_1 {
+    TEST_CONSTEXPR_CXX20 A_int_1() : data_(5) {}
+    TEST_CONSTEXPR_CXX20 int operator()(int i) {
         return i - 1;
     }
 
-    int mem1() {return 3;}
-    int mem2() const {return 4;}
+    TEST_CONSTEXPR_CXX20 int mem1() { return 3; }
+    TEST_CONSTEXPR_CXX20 int mem2() const { return 4; }
     int data_;
 };
 
-void
-test_int_1()
-{
+TEST_CONSTEXPR_CXX20 bool test_int_1() {
     using namespace std::placeholders;
     // function
     {
@@ -212,6 +207,8 @@ test_int_1()
     std::bind(&A_int_1::data_, _1)(ap) = 7;
     assert(std::bind(&A_int_1::data_, _1)(ap) == 7);
     }
+
+    return true;
 }
 
 // 2 arg, return void
@@ -263,31 +260,45 @@ test_void_2()
     }
 }
 
-struct TFENode
-{
-    bool foo(unsigned long long) const
-    {
+struct ConstQualifiedMemberFunction {
+    TEST_CONSTEXPR_CXX20 bool foo(unsigned long long) const {
         return true;
     }
 };
 
-void
-test3()
-{
+TEST_CONSTEXPR_CXX20 bool test_const_qualified_member() {
     using namespace std;
     using namespace std::placeholders;
-    const auto f = bind(&TFENode::foo, _1, 0UL);
-    const TFENode n = TFENode{};
+    const auto f = bind(&ConstQualifiedMemberFunction::foo, _1, 0UL);
+    const ConstQualifiedMemberFunction n = ConstQualifiedMemberFunction{};
     bool b = f(n);
     assert(b);
+    return true;
 }
 
-int main(int, char**)
-{
+TEST_CONSTEXPR_CXX20 bool test_many_args() {
+    using namespace std::placeholders;
+    auto f = [](int& a, char&, float&, long&) -> int& { return a; };
+    auto bound = std::bind(f, _4, _3, _2, _1);
+    int a = 3; char b = '2'; float c = 1.0f; long d = 0l;
+    int& result = bound(d, c, b, a);
+    assert(&result == &a);
+    return true;
+}
+
+int main(int, char**) {
     test_void_1();
     test_int_1();
     test_void_2();
-    test3();
+    test_const_qualified_member();
+    test_many_args();
+
+    // The other tests are not constexpr-friendly since they need to use a global variable
+#if TEST_STD_VER >= 20
+    static_assert(test_int_1());
+    static_assert(test_const_qualified_member());
+    static_assert(test_many_args());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_rvalue.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_rvalue.pass.cpp
index cd15727da90df..91ccbcf9eec9e 100644
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_rvalue.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/invoke_rvalue.pass.cpp
@@ -11,9 +11,9 @@
 // <functional>
 
 // template<CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 // template<Returnable R, CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 
 #include <stdio.h>
 
@@ -130,27 +130,22 @@ test_void_1()
 
 // 1 arg, return int
 
-int f_int_1(int i)
-{
+TEST_CONSTEXPR_CXX20 int f_int_1(int i) {
     return i + 1;
 }
 
-struct A_int_1
-{
-    A_int_1() : data_(5) {}
-    int operator()(int i)
-    {
+struct A_int_1 {
+    TEST_CONSTEXPR_CXX20 A_int_1() : data_(5) {}
+    TEST_CONSTEXPR_CXX20 int operator()(int i) {
         return i - 1;
     }
 
-    int mem1() {return 3;}
-    int mem2() const {return 4;}
+    TEST_CONSTEXPR_CXX20 int mem1() { return 3; }
+    TEST_CONSTEXPR_CXX20 int mem2() const { return 4; }
     int data_;
 };
 
-void
-test_int_1()
-{
+TEST_CONSTEXPR_CXX20 bool test_int_1() {
     using namespace std::placeholders;
     // function
     {
@@ -196,6 +191,7 @@ test_int_1()
     std::bind(&A_int_1::data_, _1)(&a) = 7;
     assert(std::bind(&A_int_1::data_, _1)(&a) == 7);
     }
+    return true;
 }
 
 // 2 arg, return void
@@ -244,28 +240,39 @@ test_void_2()
     }
 }
 
-int f_nested(int i)
-{
+TEST_CONSTEXPR_CXX20 int f_nested(int i) {
     return i+1;
 }
 
-int g_nested(int i)
-{
+TEST_CONSTEXPR_CXX20 int g_nested(int i) {
     return i*10;
 }
 
-void test_nested()
-{
+TEST_CONSTEXPR_CXX20 bool test_nested() {
     using namespace std::placeholders;
     assert(std::bind(f_nested, std::bind(g_nested, _1))(3) == 31);
+    return true;
 }
 
-int main(int, char**)
-{
+TEST_CONSTEXPR_CXX20 bool test_many_args() {
+    using namespace std::placeholders;
+    auto f = [](int a, char, float, long) { return a; };
+    auto bound = std::bind(f, _4, _3, _2, _1);
+    assert(bound(0l, 1.0f, '2', 3) == 3);
+    return true;
+}
+
+int main(int, char**) {
     test_void_1();
     test_int_1();
     test_void_2();
     test_nested();
 
-  return 0;
+    // The other tests are not constexpr-friendly since they need to use a global variable
+#if TEST_STD_VER >= 20
+    static_assert(test_int_1());
+    static_assert(test_nested());
+#endif
+
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/nested.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/nested.pass.cpp
index 76e9d84b63fdd..0fe72220efa49 100644
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/nested.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.bind/nested.pass.cpp
@@ -11,45 +11,45 @@
 // <functional>
 
 // template<CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 // template<Returnable R, CopyConstructible Fn, CopyConstructible... Types>
-//   unspecified bind(Fn, Types...);
+//   unspecified bind(Fn, Types...);    // constexpr since C++20
 
 // https://llvm.org/PR16343
 
-#include <cmath>
 #include <functional>
 #include <cassert>
 
 #include "test_macros.h"
 
-struct power
-{
+struct multiply {
   template <typename T>
-  T
-  operator()(T a, T b)
-  {
-    return static_cast<T>(std::pow(a, b));
+  TEST_CONSTEXPR_CXX20 T operator()(T a, T b) {
+    return a * b;
   }
 };
 
-struct plus_one
-{
+struct plus_one {
   template <typename T>
-  T
-  operator()(T a)
-  {
+  TEST_CONSTEXPR_CXX20 T operator()(T a) {
     return a + 1;
   }
 };
 
-int main(int, char**)
-{
-    using std::placeholders::_1;
+TEST_CONSTEXPR_CXX20 bool test() {
+  using std::placeholders::_1;
+  auto g = std::bind(multiply(), 2, _1);
+  assert(g(5) == 10);
+  assert(std::bind(plus_one(), g)(5) == 11);
 
-    auto g = std::bind(power(), 2, _1);
-    assert(g(5) == 32);
-    assert(std::bind(plus_one(), g)(5) == 33);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 20
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.place/placeholders.pass.cpp b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.place/placeholders.pass.cpp
index b71aae818a062..ac408699434a4 100644
--- a/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.place/placeholders.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/bind/func.bind/func.bind.place/placeholders.pass.cpp
@@ -8,9 +8,25 @@
 
 // <functional>
 
-// placeholders
-// The placeholders are constexpr in C++17 and beyond.
-// libc++ provides constexpr placeholders in C++11 and beyond.
+// namespace placeholders {
+//   // M is the implementation-defined number of placeholders
+//   extern unspecified _1;
+//   extern unspecified _2;
+//   .
+//   .
+//   .
+//   extern unspecified _Mp;
+// }
+
+// The Standard recommends implementing them as `inline constexpr` in C++17.
+//
+// Libc++ implements the placeholders as `extern const` in all standard modes
+// to avoid an ABI break in C++03: making them `inline constexpr` requires removing
+// their definition in the shared library to avoid ODR violations, which is an
+// ABI break.
+//
+// Concretely, `extern const` is almost indistinguishable from constexpr for the
+// placeholders since they are empty types.
 
 #include <functional>
 #include <type_traits>
@@ -18,81 +34,50 @@
 #include "test_macros.h"
 
 template <class T>
-void
-test(const T& t)
-{
-    // Test default constructible.
-    T t2;
-    ((void)t2);
-    // Test copy constructible.
-    T t3 = t;
-    ((void)t3);
+TEST_CONSTEXPR_CXX17 void test(const T& t) {
+  // Test default constructible.
+  {
+    T x; (void)x;
+  }
+
+  // Test copy constructible.
+  {
+    T x = t; (void)x;
     static_assert(std::is_nothrow_copy_constructible<T>::value, "");
     static_assert(std::is_nothrow_move_constructible<T>::value, "");
-}
+  }
 
-#if TEST_STD_VER >= 11
-constexpr decltype(std::placeholders::_1)  default1{};
-constexpr decltype(std::placeholders::_2)  default2{};
-constexpr decltype(std::placeholders::_3)  default3{};
-constexpr decltype(std::placeholders::_4)  default4{};
-constexpr decltype(std::placeholders::_5)  default5{};
-constexpr decltype(std::placeholders::_6)  default6{};
-constexpr decltype(std::placeholders::_7)  default7{};
-constexpr decltype(std::placeholders::_8)  default8{};
-constexpr decltype(std::placeholders::_9)  default9{};
-constexpr decltype(std::placeholders::_10) default10{};
-
-constexpr decltype(std::placeholders::_1)  cp1 = std::placeholders::_1;
-constexpr decltype(std::placeholders::_2)  cp2 = std::placeholders::_2;
-constexpr decltype(std::placeholders::_3)  cp3 = std::placeholders::_3;
-constexpr decltype(std::placeholders::_4)  cp4 = std::placeholders::_4;
-constexpr decltype(std::placeholders::_5)  cp5 = std::placeholders::_5;
-constexpr decltype(std::placeholders::_6)  cp6 = std::placeholders::_6;
-constexpr decltype(std::placeholders::_7)  cp7 = std::placeholders::_7;
-constexpr decltype(std::placeholders::_8)  cp8 = std::placeholders::_8;
-constexpr decltype(std::placeholders::_9)  cp9 = std::placeholders::_9;
-constexpr decltype(std::placeholders::_10) cp10 = std::placeholders::_10;
-#endif // TEST_STD_VER >= 11
-
-void use_placeholders_to_prevent_unused_warning() {
-#if TEST_STD_VER >= 11
-  ((void)cp1);
-  ((void)cp2);
-  ((void)cp3);
-  ((void)cp4);
-  ((void)cp5);
-  ((void)cp6);
-  ((void)cp7);
-  ((void)cp8);
-  ((void)cp9);
-  ((void)cp10);
-  ((void)default1);
-  ((void)default2);
-  ((void)default3);
-  ((void)default4);
-  ((void)default5);
-  ((void)default6);
-  ((void)default7);
-  ((void)default8);
-  ((void)default9);
-  ((void)default10);
+  // It is implementation-defined whether placeholder types are CopyAssignable.
+  // CopyAssignable placeholders' copy assignment operators shall not throw exceptions.
+#ifdef _LIBCPP_VERSION
+  {
+    T x;
+    x = t;
+    static_assert(std::is_nothrow_copy_assignable<T>::value, "");
+    static_assert(std::is_nothrow_move_assignable<T>::value, "");
+  }
 #endif
 }
 
-int main(int, char**)
-{
-    use_placeholders_to_prevent_unused_warning();
-    test(std::placeholders::_1);
-    test(std::placeholders::_2);
-    test(std::placeholders::_3);
-    test(std::placeholders::_4);
-    test(std::placeholders::_5);
-    test(std::placeholders::_6);
-    test(std::placeholders::_7);
-    test(std::placeholders::_8);
-    test(std::placeholders::_9);
-    test(std::placeholders::_10);
+TEST_CONSTEXPR_CXX17 bool test_all() {
+  test(std::placeholders::_1);
+  test(std::placeholders::_2);
+  test(std::placeholders::_3);
+  test(std::placeholders::_4);
+  test(std::placeholders::_5);
+  test(std::placeholders::_6);
+  test(std::placeholders::_7);
+  test(std::placeholders::_8);
+  test(std::placeholders::_9);
+  test(std::placeholders::_10);
+  return true;
+}
+
+int main(int, char**) {
+  test_all();
+#if TEST_STD_VER >= 17
+  static_assert(test_all(), "");
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/optional/optional.comp_with_t/compare.three_way.pass.cpp b/libcxx/test/std/utilities/optional/optional.comp_with_t/compare.three_way.pass.cpp
new file mode 100644
index 0000000000000..18abbdc1229d0
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/optional.comp_with_t/compare.three_way.pass.cpp
@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <optional>
+
+// [optional.comp.with.t], comparison with T
+
+// template<class T, class U>
+//     requires (!is-derived-from-optional<U>) && three_way_comparable_with<T, U>
+//   constexpr compare_three_way_result_t<T, U>
+//     operator<=>(const optional<T>&, const U&);
+
+#include <cassert>
+#include <compare>
+#include <optional>
+
+#include "test_comparisons.h"
+
+struct SomeInt {
+  int value_;
+
+  constexpr explicit SomeInt(int value = 0) : value_(value) {}
+
+  auto operator<=>(const SomeInt&) const = default;
+};
+
+template <class T, class U>
+concept HasSpaceship = requires(T t, U u) { t <=> u; };
+
+// SFINAE tests.
+
+static_assert(std::three_way_comparable_with<std::optional<int>, std::optional<int>>);
+static_assert(HasSpaceship<std::optional<int>, std::optional<int>>);
+
+static_assert(std::three_way_comparable_with<std::optional<SomeInt>, std::optional<SomeInt>>);
+static_assert(HasSpaceship<std::optional<SomeInt>, std::optional<SomeInt>>);
+
+static_assert(!HasSpaceship<std::optional<int>, std::optional<SomeInt>>);
+
+// Runtime and static tests.
+
+constexpr void test_custom_integral() {
+  {
+    SomeInt t{3};
+    std::optional<SomeInt> op{3};
+    assert((t <=> op) == std::strong_ordering::equal);
+    assert(testOrder(t, op, std::strong_ordering::equal));
+  }
+  {
+    SomeInt t{2};
+    std::optional<SomeInt> op{3};
+    assert((t <=> op) == std::strong_ordering::less);
+    assert(testOrder(t, op, std::strong_ordering::less));
+  }
+  {
+    SomeInt t{3};
+    std::optional<SomeInt> op{2};
+    assert((t <=> op) == std::strong_ordering::greater);
+    assert(testOrder(t, op, std::strong_ordering::greater));
+  }
+}
+
+constexpr void test_int() {
+  {
+    int t{3};
+    std::optional<int> op{3};
+    assert((t <=> op) == std::strong_ordering::equal);
+    assert(testOrder(t, op, std::strong_ordering::equal));
+  }
+  {
+    int t{2};
+    std::optional<int> op{3};
+    assert((t <=> op) == std::strong_ordering::less);
+    assert(testOrder(t, op, std::strong_ordering::less));
+  }
+  {
+    int t{3};
+    std::optional<int> op{2};
+    assert((t <=> op) == std::strong_ordering::greater);
+    assert(testOrder(t, op, std::strong_ordering::greater));
+  }
+}
+
+constexpr bool test() {
+  test_custom_integral();
+  test_int();
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/optional/optional.nullops/compare.three_way.pass.cpp b/libcxx/test/std/utilities/optional/optional.nullops/compare.three_way.pass.cpp
new file mode 100644
index 0000000000000..9acc9b7909d0d
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/optional.nullops/compare.three_way.pass.cpp
@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <optional>
+
+// [optional.nullops], comparison with nullopt
+
+// template<class T>
+//   constexpr strong_ordering operator<=>(const optional<T>&, nullopt_t) noexcept;
+
+#include <cassert>
+#include <compare>
+#include <optional>
+
+#include "test_comparisons.h"
+
+constexpr bool test() {
+  {
+    std::optional<int> op;
+    assert((std::nullopt <=> op) == std::strong_ordering::equal);
+    assert(testOrder(std::nullopt, op, std::strong_ordering::equal));
+    assert((op <=> std::nullopt) == std::strong_ordering::equal);
+    assert(testOrder(op, std::nullopt, std::strong_ordering::equal));
+  }
+  {
+    std::optional<int> op{1};
+    assert((std::nullopt <=> op) == std::strong_ordering::less);
+    assert(testOrder(std::nullopt, op, std::strong_ordering::less));
+  }
+  {
+    std::optional<int> op{1};
+    assert((op <=> std::nullopt) == std::strong_ordering::greater);
+    assert(testOrder(op, std::nullopt, std::strong_ordering::greater));
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/optional/optional.relops/compare.three_way.pass.cpp b/libcxx/test/std/utilities/optional/optional.relops/compare.three_way.pass.cpp
new file mode 100644
index 0000000000000..b00fe46f37ff5
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/optional.relops/compare.three_way.pass.cpp
@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <optional>
+
+// [optional.relops], relational operators
+
+// template<class T, three_way_comparable_with<T> U>
+//   constexpr compare_three_way_result_t<T, U>
+//     operator<=>(const optional<T>&, const optional<U>&);
+
+#include <cassert>
+#include <compare>
+#include <optional>
+
+#include "test_comparisons.h"
+
+constexpr bool test() {
+  {
+    std::optional<int> op1;
+    std::optional<int> op2;
+
+    assert((op1 <=> op2) == std::strong_ordering::equal);
+    assert(testOrder(op1, op2, std::strong_ordering::equal));
+  }
+  {
+    std::optional<int> op1{3};
+    std::optional<int> op2{3};
+    assert((op1 <=> op1) == std::strong_ordering::equal);
+    assert(testOrder(op1, op1, std::strong_ordering::equal));
+    assert((op1 <=> op2) == std::strong_ordering::equal);
+    assert(testOrder(op1, op2, std::strong_ordering::equal));
+    assert((op2 <=> op1) == std::strong_ordering::equal);
+    assert(testOrder(op2, op1, std::strong_ordering::equal));
+  }
+  {
+    std::optional<int> op;
+    std::optional<int> op1{2};
+    std::optional<int> op2{3};
+    assert((op <=> op2) == std::strong_ordering::less);
+    assert(testOrder(op, op2, std::strong_ordering::less));
+    assert((op1 <=> op2) == std::strong_ordering::less);
+    assert(testOrder(op1, op2, std::strong_ordering::less));
+  }
+  {
+    std::optional<int> op;
+    std::optional<int> op1{3};
+    std::optional<int> op2{2};
+    assert((op1 <=> op) == std::strong_ordering::greater);
+    assert(testOrder(op1, op, std::strong_ordering::greater));
+    assert((op1 <=> op2) == std::strong_ordering::greater);
+    assert(testOrder(op1, op2, std::strong_ordering::greater));
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/assign.pair_like_rv.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/assign.pair_like_rv.pass.cpp
new file mode 100644
index 0000000000000..92c6033512603
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/assign.pair_like_rv.pass.cpp
@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <utility>
+
+// template <class T1, class T2> struct pair
+
+//  template <pair-like P> constexpr pair& operator=(P&&);  // since C++23
+
+#include <array>
+#include <cassert>
+#include <concepts>
+#include <ranges>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+constexpr bool test() {
+  // Make sure assignment works from array and tuple
+  {
+    // Check from std::array
+    {
+      std::array<int, 2> a = {1, 2};
+      std::pair<int, int> p;
+      std::same_as<std::pair<int, int>&> decltype(auto) result = (p = a);
+      assert(&result == &p);
+      assert(p.first == 1);
+      assert(p.second == 2);
+      static_assert(!std::is_assignable_v<std::pair<int, int>&, std::array<int, 1>>); // too small
+      static_assert( std::is_assignable_v<std::pair<int, int>&, std::array<int, 2>>); // works (test the test)
+      static_assert(!std::is_assignable_v<std::pair<int, int>&, std::array<int, 3>>); // too large
+    }
+
+    // Check from std::tuple
+    {
+      std::tuple<int, int> a = {1, 2};
+      std::pair<int, int> p;
+      std::same_as<std::pair<int, int>&> decltype(auto) result = (p = a);
+      assert(&result == &p);
+      assert(p.first == 1);
+      assert(p.second == 2);
+      static_assert(!std::is_assignable_v<std::pair<int, int>&, std::tuple<int>>); // too small
+      static_assert( std::is_assignable_v<std::pair<int, int>&, std::tuple<int, int>>); // works (test the test)
+      static_assert(!std::is_assignable_v<std::pair<int, int>&, std::tuple<int, int, int>>); // too large
+    }
+
+    // Make sure it works for ranges::subrange. This actually deserves an explanation: even though
+    // the assignment operator explicitly excludes ranges::subrange specializations, such assignments
+    // end up working because of ranges::subrange's implicit conversion to pair-like types.
+    // This test ensures that the interoperability works as intended.
+    {
+      struct Assignable {
+        int* ptr = nullptr;
+        Assignable() = default;
+        constexpr Assignable(int* p) : ptr(p) { } // enable `subrange::operator pair-like`
+        constexpr Assignable& operator=(Assignable const&) = default;
+      };
+      int data[] = {1, 2, 3, 4, 5};
+      std::ranges::subrange<int*> a(data);
+      std::pair<Assignable, Assignable> p;
+      std::same_as<std::pair<Assignable, Assignable>&> decltype(auto) result = (p = a);
+      assert(&result == &p);
+      assert(p.first.ptr == data);
+      assert(p.second.ptr == data + 5);
+    }
+  }
+
+  // Make sure we allow element conversion from a pair-like
+  {
+    std::tuple<int, char const*> a = {34, "hello world"};
+    std::pair<long, std::string> p;
+    std::same_as<std::pair<long, std::string>&> decltype(auto) result = (p = a);
+    assert(&result == &p);
+    assert(p.first == 34);
+    assert(p.second == std::string("hello world"));
+    static_assert(!std::is_assignable_v<std::pair<long, std::string>&, std::tuple<char*, std::string>>); // first not convertible
+    static_assert(!std::is_assignable_v<std::pair<long, std::string>&, std::tuple<long, void*>>); // second not convertible
+    static_assert( std::is_assignable_v<std::pair<long, std::string>&, std::tuple<long, std::string>>); // works (test the test)
+  }
+
+  // Make sure we forward the pair-like elements
+  {
+    struct NoCopy {
+      NoCopy() = default;
+      NoCopy(NoCopy const&) = delete;
+      NoCopy(NoCopy&&) = default;
+      NoCopy& operator=(NoCopy const&) = delete;
+      NoCopy& operator=(NoCopy&&) = default;
+    };
+    std::tuple<NoCopy, NoCopy> a;
+    std::pair<NoCopy, NoCopy> p;
+    p = std::move(a);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/assign.pair_like_rv_const.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/assign.pair_like_rv_const.pass.cpp
new file mode 100644
index 0000000000000..1ebcb7ac9e2fe
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/assign.pair_like_rv_const.pass.cpp
@@ -0,0 +1,124 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// GCC 12 chokes on using a mutable variable inside a constexpr context
+// XFAIL: gcc-12
+
+// <utility>
+
+// template <class T1, class T2> struct pair
+
+//  template <pair-like P> constexpr const pair& operator=(P&&) const;  // since C++23
+
+#include <array>
+#include <cassert>
+#include <concepts>
+#include <ranges>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+constexpr bool test() {
+  // Make sure assignment works from array and tuple
+  {
+    // Check from std::array
+    {
+      int x = 91, y = 92;
+      std::array<int, 2> a = {1, 2};
+      std::pair<int&, int&> const p = {x, y};
+      std::same_as<std::pair<int&, int&> const&> decltype(auto) result = (p = a);
+      assert(&result == &p);
+      assert(x == 1);
+      assert(y == 2);
+      static_assert(!std::is_assignable_v<std::pair<int&, int&> const&, std::array<int, 1>>); // too small
+      static_assert( std::is_assignable_v<std::pair<int&, int&> const&, std::array<int, 2>>); // works (test the test)
+      static_assert(!std::is_assignable_v<std::pair<int&, int&> const&, std::array<int, 3>>); // too large
+    }
+
+    // Check from std::tuple
+    {
+      int x = 91, y = 92;
+      std::tuple<int, int> a = {1, 2};
+      std::pair<int&, int&> const p = {x, y};
+      std::same_as<std::pair<int&, int&> const&> decltype(auto) result = (p = a);
+      assert(&result == &p);
+      assert(x == 1);
+      assert(y == 2);
+      static_assert(!std::is_assignable_v<std::pair<int&, int&> const&, std::tuple<int>>); // too small
+      static_assert( std::is_assignable_v<std::pair<int&, int&> const&, std::tuple<int, int>>); // works (test the test)
+      static_assert(!std::is_assignable_v<std::pair<int&, int&> const&, std::tuple<int, int, int>>); // too large
+    }
+
+    // Make sure it works for ranges::subrange. This actually deserves an explanation: even though
+    // the assignment operator explicitly excludes ranges::subrange specializations, such assignments
+    // end up working because of ranges::subrange's implicit conversion to pair-like types.
+    // This test ensures that the interoperability works as intended.
+    {
+      struct ConstAssignable {
+        mutable int* ptr = nullptr;
+        ConstAssignable() = default;
+        constexpr ConstAssignable(int* p) : ptr(p) { } // enable `subrange::operator pair-like`
+        constexpr ConstAssignable const& operator=(ConstAssignable const& other) const { ptr = other.ptr; return *this; }
+
+        constexpr ConstAssignable(ConstAssignable const&) = default; // defeat -Wdeprecated-copy
+        constexpr ConstAssignable& operator=(ConstAssignable const&) = default; // defeat -Wdeprecated-copy
+      };
+      int data[] = {1, 2, 3, 4, 5};
+      std::ranges::subrange<int*> a(data);
+      std::pair<ConstAssignable, ConstAssignable> const p;
+      std::same_as<std::pair<ConstAssignable, ConstAssignable> const&> decltype(auto) result = (p = a);
+      assert(&result == &p);
+      assert(p.first.ptr == data);
+      assert(p.second.ptr == data + 5);
+    }
+  }
+
+  // Make sure we allow element conversion from a pair-like
+  {
+    struct ConstAssignable {
+      mutable int val = 0;
+      ConstAssignable() = default;
+      constexpr ConstAssignable const& operator=(int v) const { val = v; return *this; }
+    };
+    std::tuple<int, int> a = {1, 2};
+    std::pair<ConstAssignable, ConstAssignable> const p;
+    std::same_as<std::pair<ConstAssignable, ConstAssignable> const&> decltype(auto) result = (p = a);
+    assert(&result == &p);
+    assert(p.first.val == 1);
+    assert(p.second.val == 2);
+    static_assert(!std::is_assignable_v<std::pair<ConstAssignable, ConstAssignable> const&, std::tuple<void*, int>>); // first not convertible
+    static_assert(!std::is_assignable_v<std::pair<ConstAssignable, ConstAssignable> const&, std::tuple<int, void*>>); // second not convertible
+    static_assert( std::is_assignable_v<std::pair<ConstAssignable, ConstAssignable> const&, std::tuple<int, int>>); // works (test the test)
+  }
+
+  // Make sure we forward the pair-like elements
+  {
+    struct NoCopy {
+      NoCopy() = default;
+      NoCopy(NoCopy const&) = delete;
+      NoCopy(NoCopy&&) = default;
+      NoCopy& operator=(NoCopy const&) = delete;
+      constexpr NoCopy const& operator=(NoCopy&&) const { return *this; }
+    };
+    std::tuple<NoCopy, NoCopy> a;
+    std::pair<NoCopy, NoCopy> const p;
+    p = std::move(a);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.pair_like.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.pair_like.pass.cpp
new file mode 100644
index 0000000000000..3362a872a5857
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.pair_like.pass.cpp
@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <utility>
+
+// template <class T1, class T2> struct pair
+
+// template <pair-like P>
+// constexpr explicit(see-below) pair(P&&); // since C++23
+
+#include <array>
+#include <cassert>
+#include <ranges>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+constexpr bool test() {
+  // Make sure construction works from array, tuple, and ranges::subrange
+  {
+    // Check from std::array
+    {
+      std::array<int, 2> a = {1, 2};
+      std::pair<int, int> p(a);
+      assert(p.first == 1);
+      assert(p.second == 2);
+      static_assert(!std::is_constructible_v<std::pair<int, int>, std::array<int, 1>>); // too small
+      static_assert( std::is_constructible_v<std::pair<int, int>, std::array<int, 2>>); // works (test the test)
+      static_assert(!std::is_constructible_v<std::pair<int, int>, std::array<int, 3>>); // too large
+    }
+
+    // Check from std::tuple
+    {
+      std::tuple<int, int> a = {1, 2};
+      std::pair<int, int> p(a);
+      assert(p.first == 1);
+      assert(p.second == 2);
+      static_assert(!std::is_constructible_v<std::pair<int, int>, std::tuple<int>>); // too small
+      static_assert( std::is_constructible_v<std::pair<int, int>, std::tuple<int, int>>); // works (test the test)
+      static_assert(!std::is_constructible_v<std::pair<int, int>, std::tuple<int, int, int>>); // too large
+    }
+
+    // Check from ranges::subrange
+    {
+      int data[] = {1, 2, 3, 4, 5};
+      std::ranges::subrange a(data);
+      {
+        std::pair<int*, int*> p(a);
+        assert(p.first == data + 0);
+        assert(p.second == data + 5);
+      }
+      {
+        std::pair<int*, int*> p{a};
+        assert(p.first == data + 0);
+        assert(p.second == data + 5);
+      }
+      {
+        std::pair<int*, int*> p = a;
+        assert(p.first == data + 0);
+        assert(p.second == data + 5);
+      }
+    }
+  }
+
+  // Make sure we allow element conversion from a pair-like
+  {
+    std::tuple<int, char const*> a = {34, "hello world"};
+    std::pair<long, std::string> p(a);
+    assert(p.first == 34);
+    assert(p.second == std::string("hello world"));
+    static_assert(!std::is_constructible_v<std::pair<long, std::string>, std::tuple<char*, std::string>>); // first not convertible
+    static_assert(!std::is_constructible_v<std::pair<long, std::string>, std::tuple<long, void*>>); // second not convertible
+    static_assert( std::is_constructible_v<std::pair<long, std::string>, std::tuple<long, std::string>>); // works (test the test)
+  }
+
+  // Make sure we forward the pair-like elements
+  {
+    struct NoCopy {
+      NoCopy() = default;
+      NoCopy(NoCopy const&) = delete;
+      NoCopy(NoCopy&&) = default;
+    };
+    std::tuple<NoCopy, NoCopy> a;
+    std::pair<NoCopy, NoCopy> p(std::move(a));
+    (void)p;
+  }
+
+  // Make sure the constructor is implicit iff both elements can be converted
+  {
+    struct To { };
+    struct FromImplicit {
+      constexpr operator To() const { return To{}; }
+    };
+    struct FromExplicit {
+      constexpr explicit operator To() const { return To{}; }
+    };
+    // If both are convertible, the constructor is not explicit
+    {
+      std::tuple<FromImplicit, float> a = {FromImplicit{}, 2.3f};
+      std::pair<To, double> p = a;
+      (void)p;
+      static_assert(std::is_convertible_v<std::tuple<FromImplicit, float>, std::pair<To, double>>);
+    }
+    // Otherwise, the constructor is explicit
+    {
+      static_assert( std::is_constructible_v<std::pair<To, int>, std::tuple<FromExplicit, int>>);
+      static_assert(!std::is_convertible_v<std::tuple<FromExplicit, int>, std::pair<To, int>>);
+
+      static_assert( std::is_constructible_v<std::pair<int, To>, std::tuple<int, FromExplicit>>);
+      static_assert(!std::is_convertible_v<std::tuple<int, FromExplicit>, std::pair<int, To>>);
+
+      static_assert( std::is_constructible_v<std::pair<To, To>, std::tuple<FromExplicit, FromExplicit>>);
+      static_assert(!std::is_convertible_v<std::tuple<FromExplicit, FromExplicit>, std::pair<To, To>>);
+    }
+  }
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/support/test_execution_policies.h b/libcxx/test/support/test_execution_policies.h
new file mode 100644
index 0000000000000..d0cf0ddb6f0d5
--- /dev/null
+++ b/libcxx/test/support/test_execution_policies.h
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TEST_SUPPORT_TEST_EXECUTION_POLICIES
+#define TEST_SUPPORT_TEST_EXECUTION_POLICIES
+
+#include <cstdlib>
+#include <exception>
+#include <execution>
+#include <type_traits>
+#include <utility>
+
+#include "test_macros.h"
+
+#define EXECUTION_POLICY_SFINAE_TEST(function)                                                                         \
+  template <class, class...>                                                                                           \
+  struct sfinae_test_##function##_impl : std::true_type {};                                                            \
+                                                                                                                       \
+  template <class... Args>                                                                                             \
+  struct sfinae_test_##function##_impl<std::void_t<decltype(std::function(std::declval<Args>()...))>, Args...>         \
+      : std::false_type {};                                                                                            \
+                                                                                                                       \
+  template <class... Args>                                                                                             \
+  constexpr bool sfinae_test_##function = sfinae_test_##function##_impl<void, Args...>::value;
+
+template <class Functor>
+bool test_execution_policies(Functor func) {
+  func(std::execution::seq);
+#if TEST_STD_VER >= 20
+  func(std::execution::unseq);
+#endif
+  func(std::execution::par);
+  func(std::execution::par_unseq);
+
+  return true;
+}
+
+template <template <class Iter> class TestClass>
+struct TestIteratorWithPolicies {
+  template <class Iter>
+  void operator()() {
+    test_execution_policies(TestClass<Iter>{});
+  }
+};
+
+struct Bool {
+  bool b_;
+  Bool() = default;
+  Bool(bool b) : b_(b) {}
+
+  operator bool&() {
+    return b_;
+  }
+};
+
+[[noreturn]] inline void terminate_successful() { std::exit(0); }
+
+#endif // TEST_SUPPORT_TEST_EXECUTION_POLICIES
diff --git a/libcxx/test/tools/clang_tidy_checks/uglify_attributes.cpp b/libcxx/test/tools/clang_tidy_checks/uglify_attributes.cpp
index 038788d94c915..c4a51fad72720 100644
--- a/libcxx/test/tools/clang_tidy_checks/uglify_attributes.cpp
+++ b/libcxx/test/tools/clang_tidy_checks/uglify_attributes.cpp
@@ -25,6 +25,24 @@ bool isUgly(std::string_view str) {
   return str.find("__") != std::string_view::npos;
 }
 
+// Starting with Clang 17 ToT C++23 support is provided by CPlusPlus23 instead
+// of C++23 support is provided by CPlusPlus2b. To allow a smooth transition for
+// libc++ use "reflection" to select the proper member. Since the change
+// happens in the development cycle it's not possible to use #ifdefs.
+template <class T>
+bool CPlusPlus23(const T& lang_opts)
+  requires requires { T::CPlusPlus2b; }
+{
+  return lang_opts.CPlusPlus2b;
+}
+
+template <class T>
+bool CPlusPlus23(const T& lang_opts)
+  requires requires { T::CPlusPlus23; }
+{
+  return lang_opts.CPlusPlus23;
+}
+
 std::vector<const char*> get_standard_attributes(const clang::LangOptions& lang_opts) {
   std::vector<const char*> attributes = {"noreturn", "carries_dependency"};
 
@@ -43,7 +61,7 @@ std::vector<const char*> get_standard_attributes(const clang::LangOptions& lang_
     attributes.emplace_back("no_unique_address");
   }
 
-  if (lang_opts.CPlusPlus2b) {
+  if (CPlusPlus23(lang_opts)) {
     attributes.emplace_back("assume");
   }
 
diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index 85d0ff17b2a84..e8c5d5dad3312 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -84,11 +84,15 @@ RUN apt-get update && apt-get install -y clang-tools-$(($LLVM_HEAD_VERSION - 3))
 RUN apt-get update && apt-get install -y clang-format-$(($LLVM_HEAD_VERSION - 2)) clang-format-$(($LLVM_HEAD_VERSION - 1))
 
 # Install clang-tidy
-RUN apt-get update && apt-get install -y clang-tidy-$(($LLVM_HEAD_VERSION - 2)) clang-tidy-$((LLVM_HEAD_VERSION - 1))
+# TODO LLVM 17 revert D148831 to only install $(($LLVM_HEAD_VERSION - 1)) and $LLVM_HEAD_VERSION
+# The usage of the ToT version is needed due to module issues with Clang 16
+RUN apt-get update && apt-get install -y clang-tidy-$(($LLVM_HEAD_VERSION - 2)) clang-tidy-$(($LLVM_HEAD_VERSION - 1)) clang-tidy-$LLVM_HEAD_VERSION
 
 # Install llvm-dev and libclang-dev to compile custom clang-tidy checks
-RUN apt-get update && apt-get install -y llvm-$(($LLVM_HEAD_VERSION - 1))-dev     llvm-$(($LLVM_HEAD_VERSION - 2))-dev \
-                                     libclang-$(($LLVM_HEAD_VERSION - 1))-dev libclang-$(($LLVM_HEAD_VERSION - 2))-dev
+# TODO LLVM 17 revert D148831 to only install $(($LLVM_HEAD_VERSION - 1)) and $LLVM_HEAD_VERSION
+# The usage of the ToT version is needed due to module issues with Clang 16
+RUN apt-get update && apt-get install -y llvm-$(($LLVM_HEAD_VERSION - 2))-dev llvm-$(($LLVM_HEAD_VERSION - 1))-dev     llvm-$LLVM_HEAD_VERSION-dev \
+                                     libclang-$(($LLVM_HEAD_VERSION - 2))-dev libclang-$(($LLVM_HEAD_VERSION - 1))-dev libclang-$LLVM_HEAD_VERSION-dev
 
 # Install the most recent GCC, like clang install the previous version as a transition.
 ENV GCC_LATEST_VERSION=12
diff --git a/libcxx/utils/ci/buildkite-pipeline-clang.yml b/libcxx/utils/ci/buildkite-pipeline-clang.yml
index 7ed58745bf10b..3e40c2bc0fef3 100644
--- a/libcxx/utils/ci/buildkite-pipeline-clang.yml
+++ b/libcxx/utils/ci/buildkite-pipeline-clang.yml
@@ -19,9 +19,12 @@ steps:
   - label: "Building clang"
     commands:
       - "mkdir install"
-      - "cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=install -DLLVM_ENABLE_PROJECTS=\"clang;compiler-rt\""
+      # We use Release here to avoid including debug information. Otherwise, the clang binary is very large, which
+      # is problematic because we need to upload the artifacts for other jobs to use. This may seem like nothing,
+      # but with the number of jobs we run daily, this can result in thousands of GB of network I/O.
+      - "cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install -DLLVM_ENABLE_PROJECTS=\"clang;compiler-rt\""
       - "ninja -C build install-clang install-clang-resource-headers"
-      - "buildkite-agent artifact upload 'install/**/*'"
+      - "buildkite-agent artifact upload --debug 'install/**/*'"
     env:
         CC: "clang-${LLVM_HEAD_VERSION}"
         CXX: "clang++-${LLVM_HEAD_VERSION}"
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index 474138d21f4f0..17a1dc4633be8 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -332,6 +332,7 @@ steps:
           - exit_status: -1  # Agent was lost
             limit: 2
       timeout_in_minutes: 120
+      soft_fail: true # TODO: Investigate why this job is failing only in the CI environment
 
     - label: "UBSAN"
       command: "libcxx/utils/ci/run-buildbot generic-ubsan"
@@ -368,6 +369,7 @@ steps:
           - exit_status: -1  # Agent was lost
             limit: 2
       timeout_in_minutes: 120
+      soft_fail: true # TODO: Investigate why this job is failing only in the CI environment
 
   # Tests with the various supported ways to build libc++.
   - label: "Bootstrapping build"
@@ -463,6 +465,22 @@ steps:
           limit: 2
     timeout_in_minutes: 120
 
+  - label: "With PSTL support"
+    command: "libcxx/utils/ci/run-buildbot with-pstl"
+    artifact_paths:
+      - "**/test-results.xml"
+    env:
+      CC: "clang-${LLVM_HEAD_VERSION}"
+      CXX: "clang++-${LLVM_HEAD_VERSION}"
+    agents:
+      queue: "libcxx-builders"
+      os: "linux"
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+    timeout_in_minutes: 120
+
   - label: "With LLVM's libunwind"
     command: "libcxx/utils/ci/run-buildbot generic-with_llvm_unwinder"
     artifact_paths:
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index cd70d73a52f5d..34e3792efd5d4 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -448,6 +448,11 @@ generic-abi-unstable)
     generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/Generic-abi-unstable.cmake"
     check-runtimes
 ;;
+with-pstl)
+    clean
+    generate-cmake -C "${MONOREPO_ROOT}/libcxx/cmake/caches/With-pstl.cmake"
+    check-runtimes
+;;
 apple-system)
     clean
 
diff --git a/libcxx/utils/data/ignore_format.txt b/libcxx/utils/data/ignore_format.txt
index 044a395b352ce..cfc617c1fb3fd 100644
--- a/libcxx/utils/data/ignore_format.txt
+++ b/libcxx/utils/data/ignore_format.txt
@@ -506,39 +506,26 @@ libcxx/include/__numeric/transform_inclusive_scan.h
 libcxx/include/__numeric/transform_reduce.h
 libcxx/include/optional
 libcxx/include/ostream
-libcxx/include/pstl/internal/algorithm_fwd.h
-libcxx/include/pstl/internal/algorithm_impl.h
-libcxx/include/pstl/internal/execution_defs.h
-libcxx/include/pstl/internal/execution_impl.h
-libcxx/include/pstl/internal/glue_algorithm_defs.h
-libcxx/include/pstl/internal/glue_algorithm_impl.h
-libcxx/include/pstl/internal/glue_execution_defs.h
-libcxx/include/pstl/internal/glue_memory_defs.h
-libcxx/include/pstl/internal/glue_memory_impl.h
-libcxx/include/pstl/internal/glue_numeric_defs.h
-libcxx/include/pstl/internal/glue_numeric_impl.h
-libcxx/include/pstl/internal/memory_impl.h
-libcxx/include/pstl/internal/numeric_fwd.h
-libcxx/include/pstl/internal/numeric_impl.h
-libcxx/include/pstl/internal/omp/parallel_for_each.h
-libcxx/include/pstl/internal/omp/parallel_for.h
-libcxx/include/pstl/internal/omp/parallel_invoke.h
-libcxx/include/pstl/internal/omp/parallel_merge.h
-libcxx/include/pstl/internal/omp/parallel_reduce.h
-libcxx/include/pstl/internal/omp/parallel_scan.h
-libcxx/include/pstl/internal/omp/parallel_stable_partial_sort.h
-libcxx/include/pstl/internal/omp/parallel_stable_sort.h
-libcxx/include/pstl/internal/omp/parallel_transform_reduce.h
-libcxx/include/pstl/internal/omp/parallel_transform_scan.h
-libcxx/include/pstl/internal/omp/util.h
-libcxx/include/pstl/internal/parallel_backend.h
-libcxx/include/pstl/internal/parallel_backend_serial.h
-libcxx/include/pstl/internal/parallel_backend_tbb.h
-libcxx/include/pstl/internal/parallel_backend_utils.h
-libcxx/include/pstl/internal/parallel_impl.h
-libcxx/include/pstl/internal/pstl_config.h
-libcxx/include/pstl/internal/unseq_backend_simd.h
-libcxx/include/pstl/internal/utils.h
+libcxx/include/__pstl/internal/numeric_impl.h
+libcxx/include/__pstl/internal/omp/parallel_for_each.h
+libcxx/include/__pstl/internal/omp/parallel_for.h
+libcxx/include/__pstl/internal/omp/parallel_invoke.h
+libcxx/include/__pstl/internal/omp/parallel_merge.h
+libcxx/include/__pstl/internal/omp/parallel_reduce.h
+libcxx/include/__pstl/internal/omp/parallel_scan.h
+libcxx/include/__pstl/internal/omp/parallel_stable_partial_sort.h
+libcxx/include/__pstl/internal/omp/parallel_stable_sort.h
+libcxx/include/__pstl/internal/omp/parallel_transform_reduce.h
+libcxx/include/__pstl/internal/omp/parallel_transform_scan.h
+libcxx/include/__pstl/internal/omp/util.h
+libcxx/include/__pstl/internal/parallel_backend.h
+libcxx/include/__pstl/internal/parallel_backend_serial.h
+libcxx/include/__pstl/internal/parallel_backend_tbb.h
+libcxx/include/__pstl/internal/parallel_backend_utils.h
+libcxx/include/__pstl/internal/parallel_impl.h
+libcxx/include/__pstl/internal/pstl_config.h
+libcxx/include/__pstl/internal/unseq_backend_simd.h
+libcxx/include/__pstl/internal/utils.h
 libcxx/include/queue
 libcxx/include/__random/bernoulli_distribution.h
 libcxx/include/__random/binomial_distribution.h
diff --git a/libcxx/utils/generate_header_tests.py b/libcxx/utils/generate_header_tests.py
index d2f18883f8abe..eacf1f8d84192 100755
--- a/libcxx/utils/generate_header_tests.py
+++ b/libcxx/utils/generate_header_tests.py
@@ -128,7 +128,7 @@ def main():
     experimental_headers = sorted(str(p.relative_to(include)) for p in include.glob('experimental/[a-z]*') if is_header(p))
     extended_headers     = sorted(str(p.relative_to(include)) for p in include.glob('ext/[a-z]*') if is_header(p))
     public_headers       = toplevel_headers + experimental_headers + extended_headers
-    private_headers      = sorted(str(p.relative_to(include)) for p in include.rglob('*') if is_header(p) and str(p.relative_to(include)).startswith('__'))
+    private_headers      = sorted(str(p.relative_to(include)) for p in include.rglob('*') if is_header(p) and str(p.relative_to(include)).startswith('__') and not p.name.startswith('pstl'))
     variables = {
         'toplevel_headers': toplevel_headers,
         'experimental_headers': experimental_headers,
diff --git a/libcxx/utils/generate_iwyu_mapping.py b/libcxx/utils/generate_iwyu_mapping.py
index cb27d46773955..f8377a976f08e 100644
--- a/libcxx/utils/generate_iwyu_mapping.py
+++ b/libcxx/utils/generate_iwyu_mapping.py
@@ -56,11 +56,12 @@ def generate_map(include):
         elif i == '__pstl_memory': continue
         elif i == '__pstl_numeric': continue
         elif i == '__split_buffer': public = ['deque', 'vector']
+        elif i == '__std_mbstate_t.h': continue
         elif i == '__threading_support': public = ['atomic', 'mutex', 'semaphore', 'thread']
         elif i == '__tree': public = ['map', 'set']
         elif i == '__undef_macros': continue
         elif i == '__verbose_abort': continue
-        else: panic()
+        else: panic(i)
 
         for p in public:
             result.append(f'{generate(f"<{i}>", p)},')
diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py
index 99ccb5af9f5d1..9f59d3b447e68 100644
--- a/libcxx/utils/libcxx/test/dsl.py
+++ b/libcxx/utils/libcxx/test/dsl.py
@@ -105,7 +105,7 @@ def _executeScriptInternal(test, commands):
     out = out[:-len(conflatedErrorOutput)]
     err += conflatedErrorOutput
 
-  return (out, err, exitCode, timeoutInfo)
+  return (out, err, exitCode, timeoutInfo, parsedCommands)
 
 def _makeConfigTest(config):
   # Make sure the support directories exist, which is needed to create
@@ -146,7 +146,7 @@ def sourceBuilds(config, source, additionalFlags=[]):
   with _makeConfigTest(config) as test:
     with open(test.getSourcePath(), 'w') as sourceFile:
       sourceFile.write(source)
-    _, _, exitCode, _ = _executeScriptInternal(test, ['%{{build}} {}'.format(' '.join(additionalFlags))])
+    _, _, exitCode, _, _ = _executeScriptInternal(test, ['%{{build}} {}'.format(' '.join(additionalFlags))])
     return exitCode == 0
 
 @_memoizeExpensiveOperation(lambda c, p, args=None: (c.substitutions, c.environment, p, args))
@@ -164,13 +164,13 @@ def programOutput(config, program, args=None):
   with _makeConfigTest(config) as test:
     with open(test.getSourcePath(), 'w') as source:
       source.write(program)
-    _, err, exitCode, _ = _executeScriptInternal(test, ['%{build}'])
+    _, err, exitCode, _, buildcmd = _executeScriptInternal(test, ['%{build}'])
     if exitCode != 0:
-      raise ConfigurationCompilationError("Failed to build program, stderr is:\n{}".format(err))
+      raise ConfigurationCompilationError("Failed to build program, cmd:\n{}\nstderr is:\n{}".format(buildcmd, err))
 
-    out, err, exitCode, _ = _executeScriptInternal(test, ["%{{run}} {}".format(' '.join(args))])
+    out, err, exitCode, _, runcmd = _executeScriptInternal(test, ["%{{run}} {}".format(' '.join(args))])
     if exitCode != 0:
-      raise ConfigurationRuntimeError("Failed to run program, stderr is:\n{}".format(err))
+      raise ConfigurationRuntimeError("Failed to run program, cmd:\n{}\nstderr is:\n{}".format(runcmd, err))
 
     actualOut = re.search("# command output:\n(.+)\n$", out, flags=re.DOTALL)
     actualOut = actualOut.group(1) if actualOut else ""
@@ -201,7 +201,7 @@ def hasCompileFlag(config, flag):
   checking whether that succeeds.
   """
   with _makeConfigTest(config) as test:
-    out, err, exitCode, timeoutInfo = _executeScriptInternal(test, [
+    out, err, exitCode, timeoutInfo, _ = _executeScriptInternal(test, [
       "%{{cxx}} -xc++ {} -Werror -fsyntax-only %{{flags}} %{{compile_flags}} {}".format(os.devnull, flag)
     ])
     return exitCode == 0
@@ -215,7 +215,7 @@ def runScriptExitCode(config, script):
   could appear on the right-hand-side of a `RUN:` keyword.
   """
   with _makeConfigTest(config) as test:
-    _, _, exitCode, _ = _executeScriptInternal(test, script)
+    _, _, exitCode, _, _ = _executeScriptInternal(test, script)
     return exitCode
 
 @_memoizeExpensiveOperation(lambda c, s: (c.substitutions, c.environment, s))
@@ -228,7 +228,7 @@ def commandOutput(config, command):
   could appear on the right-hand-side of a `RUN:` keyword.
   """
   with _makeConfigTest(config) as test:
-    out, _, exitCode, _ = _executeScriptInternal(test, command)
+    out, _, exitCode, _, _ = _executeScriptInternal(test, command)
     if exitCode != 0:
      raise ConfigurationRuntimeError()
     return out
@@ -281,7 +281,7 @@ def compilerMacros(config, flags=''):
       #  include <__config_site>
       #endif
       """)
-    unparsedOutput, err, exitCode, _ = _executeScriptInternal(test, [
+    unparsedOutput, err, exitCode, _, _ = _executeScriptInternal(test, [
       "%{{cxx}} %s -dM -E %{{flags}} %{{compile_flags}} {}".format(flags)
     ])
     if exitCode != 0:
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index dd5163f844453..49f5e70334327 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -209,6 +209,7 @@ def _getSuitableClangTidy(cfg):
   '_LIBCPP_HAS_NO_WIDE_CHARACTERS': 'no-wide-characters',
   '_LIBCPP_HAS_NO_UNICODE': 'libcpp-has-no-unicode',
   '_LIBCPP_ENABLE_DEBUG_MODE': 'libcpp-has-debug-mode',
+  '_LIBCPP_HAS_PARALLEL_ALGORITHMS': 'with-pstl',
 }
 for macro, feature in macros.items():
   DEFAULT_FEATURES.append(
diff --git a/libcxx/utils/run.py b/libcxx/utils/run.py
index bcb7b8412700c..347a013243c84 100755
--- a/libcxx/utils/run.py
+++ b/libcxx/utils/run.py
@@ -23,7 +23,8 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--execdir', type=str, required=True)
     parser.add_argument('--codesign_identity', type=str, required=False, default=None)
-    parser.add_argument('--env', type=str, nargs='*', required=False, default=dict())
+    parser.add_argument('--env', type=str, nargs='*', required=False, default=[])
+    parser.add_argument('--prepend_env', type=str, nargs='*', required=False, default=[])
     parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
     args = parser.parse_args()
     commandLine = args.command
@@ -43,6 +44,14 @@ def main():
 
     # Extract environment variables into a dictionary
     env = {k : v  for (k, v) in map(lambda s: s.split('=', 1), args.env)}
+
+    # Set environment variables where we prepend the given value to the
+    # existing environment variable.
+    for (k, v) in map(lambda s: s.split('=', 1), args.prepend_env):
+        if k in os.environ:
+            v = v + os.pathsep + os.environ[k]
+        env[k] = v
+
     if platform.system() == 'Windows':
         # Pass some extra variables through on Windows:
         # COMSPEC is needed for running subprocesses via std::system().
diff --git a/libcxx/utils/ssh.py b/libcxx/utils/ssh.py
index 6c1d706984dd8..a5f17d6ecd598 100755
--- a/libcxx/utils/ssh.py
+++ b/libcxx/utils/ssh.py
@@ -47,7 +47,8 @@ def main():
     parser.add_argument('--extra-ssh-args', type=str, required=False)
     parser.add_argument('--extra-scp-args', type=str, required=False)
     parser.add_argument('--codesign_identity', type=str, required=False, default=None)
-    parser.add_argument('--env', type=str, nargs='*', required=False, default=dict())
+    parser.add_argument('--env', type=str, nargs='*', required=False, default=[])
+    parser.add_argument('--prepend_env', type=str, nargs='*', required=False, default=[])
     parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
     args = parser.parse_args()
     commandLine = args.command
@@ -108,8 +109,15 @@ def main():
         # temporary directory on the remote host.
         commandLine = (pathOnRemote(x) if isTestExe(x) else x for x in commandLine)
         remoteCommands.append('cd {}'.format(tmp))
+
+        if args.prepend_env:
+            # We can't sensibly know the original value of the env vars
+            # in order to prepend to them, so just overwrite these variables.
+            args.env.extend(args.prepend_env)
+
         if args.env:
-            remoteCommands.append('export {}'.format(cmd_quote(' '.join(args.env))))
+            env = list(map(cmd_quote, args.env))
+            remoteCommands.append('export {}'.format(' '.join(args.env)))
         remoteCommands.append(subprocess.list2cmdline(commandLine))
 
         # Finally, SSH to the remote host and execute all the commands.
diff --git a/libcxxabi/test/configs/llvm-libc++abi-mingw.cfg.in b/libcxxabi/test/configs/llvm-libc++abi-mingw.cfg.in
index 6f88304fb0c52..1e9f202e8dd8a 100644
--- a/libcxxabi/test/configs/llvm-libc++abi-mingw.cfg.in
+++ b/libcxxabi/test/configs/llvm-libc++abi-mingw.cfg.in
@@ -11,7 +11,7 @@ config.substitutions.append(('%{link_flags}',
     '-nostdlib++ -L %{lib} -lc++'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/libcxxabi/test/configs/llvm-libc++abi-shared-clangcl.cfg.in b/libcxxabi/test/configs/llvm-libc++abi-shared-clangcl.cfg.in
index a375722ba7c2f..cfdfc0f86090c 100644
--- a/libcxxabi/test/configs/llvm-libc++abi-shared-clangcl.cfg.in
+++ b/libcxxabi/test/configs/llvm-libc++abi-shared-clangcl.cfg.in
@@ -11,7 +11,7 @@ config.substitutions.append(('%{link_flags}',
     '-nostdlib -L %{lib} -lc++ -lc++abi -lmsvcrt -lmsvcprt -loldnames'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/libcxxabi/test/configs/llvm-libc++abi-static-clangcl.cfg.in b/libcxxabi/test/configs/llvm-libc++abi-static-clangcl.cfg.in
index 445ee02b1a988..ba67c8b2b7653 100644
--- a/libcxxabi/test/configs/llvm-libc++abi-static-clangcl.cfg.in
+++ b/libcxxabi/test/configs/llvm-libc++abi-static-clangcl.cfg.in
@@ -11,7 +11,7 @@ config.substitutions.append(('%{link_flags}',
     '-nostdlib -L %{lib} -llibc++ -llibc++abi -lmsvcrt -lmsvcprt -loldnames'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/libunwind/test/configs/llvm-libunwind-mingw.cfg.in b/libunwind/test/configs/llvm-libunwind-mingw.cfg.in
index 99d0aac9015ca..33d061a3efc6f 100644
--- a/libunwind/test/configs/llvm-libunwind-mingw.cfg.in
+++ b/libunwind/test/configs/llvm-libunwind-mingw.cfg.in
@@ -11,7 +11,7 @@ config.substitutions.append(('%{link_flags}',
     '-L %{lib} -lunwind'
 ))
 config.substitutions.append(('%{exec}',
-    '%{executor} --execdir %T --env PATH=%{lib} -- '
+    '%{executor} --execdir %T --prepend_env PATH=%{lib} -- '
 ))
 
 import os, site
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 8d272c2cc6bf7..6326e466930a4 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -445,9 +445,12 @@ void LinkerDriver::parseDirectives(InputFile *file) {
     case OPT_editandcontinue:
     case OPT_guardsym:
     case OPT_throwingnew:
+    case OPT_inferasanlibs:
+    case OPT_inferasanlibs_no:
       break;
     default:
-      error(arg->getSpelling() + " is not allowed in .drectve");
+      error(arg->getSpelling() + " is not allowed in .drectve (" +
+            toString(file) + ")");
     }
   }
 }
@@ -1932,6 +1935,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       args.hasFlag(OPT_stdcall_fixup, OPT_stdcall_fixup_no, config->mingw);
   config->warnStdcallFixup = !args.hasArg(OPT_stdcall_fixup);
 
+  if (args.hasFlag(OPT_inferasanlibs, OPT_inferasanlibs_no, false))
+    warn("ignoring '/inferasanlibs', this flag is not supported");
+
   // Don't warn about long section names, such as .debug_info, for mingw or
   // when -debug:dwarf is requested.
   if (config->mingw || config->debugDwarf)
diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp
index fb3ba2c6e9d78..cca14be0fc2b4 100644
--- a/lld/COFF/LTO.cpp
+++ b/lld/COFF/LTO.cpp
@@ -55,9 +55,8 @@ static std::unique_ptr<raw_fd_ostream> openFile(StringRef file) {
 }
 
 std::string BitcodeCompiler::getThinLTOOutputFile(StringRef path) {
-  return lto::getThinLTOOutputFile(
-      std::string(path), std::string(ctx.config.thinLTOPrefixReplaceOld),
-      std::string(ctx.config.thinLTOPrefixReplaceNew));
+  return lto::getThinLTOOutputFile(path, ctx.config.thinLTOPrefixReplaceOld,
+                                   ctx.config.thinLTOPrefixReplaceNew);
 }
 
 lto::Config BitcodeCompiler::createConfig() {
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index df527bb48ae72..2a89509be5fe0 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -194,6 +194,9 @@ defm highentropyva : B<"highentropyva",
 defm incremental : B<"incremental",
                      "Keep original import library if contents are unchanged",
                      "Overwrite import library even if contents are unchanged">;
+defm inferasanlibs : B<"inferasanlibs",
+                       "Unused, generates a warning",
+                       "No effect (default)">;
 defm integritycheck : B<"integritycheck",
                         "Set FORCE_INTEGRITY bit in PE header",
                         "No effect (default)">;
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index 94af920d5e7a1..8ca74ebc661ea 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -950,7 +950,7 @@ void DebugSHandler::finish() {
   // must also be rewritten to use the PDB string table.
   for (const UnrelocatedFpoData &subsec : frameDataSubsecs) {
     // Relocate the first four bytes of the subection and reinterpret them as a
-    // 32 bit integer.
+    // 32 bit little-endian integer.
     SectionChunk *debugChunk = subsec.debugChunk;
     ArrayRef<uint8_t> subsecData = subsec.subsecData;
     uint32_t relocIndex = subsec.relocIndex;
@@ -959,8 +959,9 @@ void DebugSHandler::finish() {
     debugChunk->writeAndRelocateSubsection(debugChunk->getContents(),
                                            unrelocatedRvaStart, relocIndex,
                                            &relocatedRvaStart[0]);
-    uint32_t rvaStart;
-    memcpy(&rvaStart, &relocatedRvaStart[0], sizeof(uint32_t));
+    // Use of memcpy here avoids violating type-based aliasing rules.
+    support::ulittle32_t rvaStart;
+    memcpy(&rvaStart, &relocatedRvaStart[0], sizeof(support::ulittle32_t));
 
     // Copy each frame data record, add in rvaStart, translate string table
     // indices, and add the record to the PDB.
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 603703e65290b..645c63f690409 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -1266,7 +1266,9 @@ void Writer::createSymbolAndStringTable() {
   // solution where discardable sections have long names preserved and
   // non-discardable sections have their names truncated, to ensure that any
   // section which is mapped at runtime also has its name mapped at runtime.
+  bool HasDwarfSection = false;
   for (OutputSection *sec : ctx.outputSections) {
+    HasDwarfSection |= sec->name.startswith(".debug_");
     if (sec->name.size() <= COFF::NameSize)
       continue;
     if ((sec->header.Characteristics & IMAGE_SCN_MEM_DISCARDABLE) == 0)
@@ -1279,7 +1281,7 @@ void Writer::createSymbolAndStringTable() {
     sec->setStringTableOff(addEntryToStringTable(sec->name));
   }
 
-  if (ctx.config.debugDwarf || ctx.config.debugSymtab) {
+  if (ctx.config.debugDwarf || ctx.config.debugSymtab || HasDwarfSection) {
     for (ObjFile *file : ctx.objFileInstances) {
       for (Symbol *b : file->getSymbols()) {
         auto *d = dyn_cast_or_null<Defined>(b);
diff --git a/lld/ELF/Arch/AVR.cpp b/lld/ELF/Arch/AVR.cpp
index 8a5087df49344..4905d61796fa9 100644
--- a/lld/ELF/Arch/AVR.cpp
+++ b/lld/ELF/Arch/AVR.cpp
@@ -28,6 +28,7 @@
 #include "InputFiles.h"
 #include "Symbols.h"
 #include "Target.h"
+#include "Thunks.h"
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Endian.h"
@@ -42,9 +43,13 @@ using namespace lld::elf;
 namespace {
 class AVR final : public TargetInfo {
 public:
+  AVR() { needsThunks = true; }
   uint32_t calcEFlags() const override;
   RelExpr getRelExpr(RelType type, const Symbol &s,
                      const uint8_t *loc) const override;
+  bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
+                  uint64_t branchAddr, const Symbol &s,
+                  int64_t a) const override;
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
 };
@@ -71,8 +76,10 @@ RelExpr AVR::getRelExpr(RelType type, const Symbol &s,
   case R_AVR_HH8_LDI:
   case R_AVR_MS8_LDI_NEG:
   case R_AVR_MS8_LDI:
+  case R_AVR_LO8_LDI_GS:
   case R_AVR_LO8_LDI_PM:
   case R_AVR_LO8_LDI_PM_NEG:
+  case R_AVR_HI8_LDI_GS:
   case R_AVR_HI8_LDI_PM:
   case R_AVR_HI8_LDI_PM_NEG:
   case R_AVR_HH8_LDI_PM:
@@ -96,6 +103,19 @@ static void writeLDI(uint8_t *loc, uint64_t val) {
   write16le(loc, (read16le(loc) & 0xf0f0) | (val & 0xf0) << 4 | (val & 0x0f));
 }
 
+bool AVR::needsThunk(RelExpr expr, RelType type, const InputFile *file,
+                     uint64_t branchAddr, const Symbol &s, int64_t a) const {
+  switch (type) {
+  case R_AVR_LO8_LDI_GS:
+  case R_AVR_HI8_LDI_GS:
+    // A thunk is needed if the symbol's virtual address is out of range
+    // [0, 0x1ffff].
+    return s.getVA() >= 0x20000;
+  default:
+    return false;
+  }
+}
+
 void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
   case R_AVR_8:
@@ -160,10 +180,16 @@ void AVR::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
     writeLDI(loc, (val >> 24) & 0xff);
     break;
 
+  case R_AVR_LO8_LDI_GS:
+    checkUInt(loc, val, 17, rel);
+    [[fallthrough]];
   case R_AVR_LO8_LDI_PM:
     checkAlignment(loc, val, 2, rel);
     writeLDI(loc, (val >> 1) & 0xff);
     break;
+  case R_AVR_HI8_LDI_GS:
+    checkUInt(loc, val, 17, rel);
+    [[fallthrough]];
   case R_AVR_HI8_LDI_PM:
     checkAlignment(loc, val, 2, rel);
     writeLDI(loc, (val >> 9) & 0xff);
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index 26977015dab3a..aa67617a5be25 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -662,23 +662,8 @@ static bool relax(InputSection &sec) {
   const uint64_t secAddr = sec.getVA();
   auto &aux = *sec.relaxAux;
   bool changed = false;
-
-  // Get st_value delta for symbols relative to this section from the previous
-  // iteration.
-  DenseMap<const Defined *, uint64_t> valueDelta;
   ArrayRef<SymbolAnchor> sa = ArrayRef(aux.anchors);
   uint32_t delta = 0;
-  for (auto [i, r] : llvm::enumerate(sec.relocs())) {
-    for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(1))
-      if (!sa[0].end)
-        valueDelta[sa[0].d] = delta;
-    delta = aux.relocDeltas[i];
-  }
-  for (const SymbolAnchor &sa : sa)
-    if (!sa.end)
-      valueDelta[sa.d] = delta;
-  sa = ArrayRef(aux.anchors);
-  delta = 0;
 
   std::fill_n(aux.relocTypes.get(), sec.relocs().size(), R_RISCV_NONE);
   aux.writes.clear();
@@ -725,7 +710,7 @@ static bool relax(InputSection &sec) {
       if (sa[0].end)
         sa[0].d->size = sa[0].offset - delta - sa[0].d->value;
       else
-        sa[0].d->value -= delta - valueDelta.find(sa[0].d)->second;
+        sa[0].d->value = sa[0].offset - delta;
     }
     delta += remove;
     if (delta != cur) {
@@ -738,7 +723,7 @@ static bool relax(InputSection &sec) {
     if (a.end)
       a.d->size = a.offset - delta - a.d->value;
     else
-      a.d->value -= delta - valueDelta.find(a.d)->second;
+      a.d->value = a.offset - delta;
   }
   // Inform assignAddresses that the size has changed.
   if (!isUInt<16>(delta))
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index c446508dfa2f0..9c9a49d9f20ed 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -407,6 +407,12 @@ struct Config {
   bool androidMemtagStack;
 
   unsigned threadCount;
+
+  // If an input file equals a key, remap it to the value.
+  llvm::DenseMap<llvm::StringRef, llvm::StringRef> remapInputs;
+  // If an input file matches a wildcard pattern, remap it to the value.
+  llvm::SmallVector<std::pair<llvm::GlobPattern, llvm::StringRef>, 0>
+      remapInputsWildcards;
 };
 struct ConfigWrapper {
   Config c;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 79f16a281df9b..f235ffcbb9559 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1077,6 +1077,25 @@ static bool isValidReportString(StringRef arg) {
   return arg == "none" || arg == "warning" || arg == "error";
 }
 
+// Process a remap pattern 'from-glob=to-file'.
+static bool remapInputs(StringRef line, const Twine &location) {
+  SmallVector<StringRef, 0> fields;
+  line.split(fields, '=');
+  if (fields.size() != 2 || fields[1].empty()) {
+    error(location + ": parse error, not 'from-glob=to-file'");
+    return true;
+  }
+  if (!hasWildcard(fields[0]))
+    config->remapInputs[fields[0]] = fields[1];
+  else if (Expected<GlobPattern> pat = GlobPattern::create(fields[0]))
+    config->remapInputsWildcards.emplace_back(std::move(*pat), fields[1]);
+  else {
+    error(location + ": " + toString(pat.takeError()));
+    return true;
+  }
+  return false;
+}
+
 // Initializes Config members by the command line options.
 static void readConfigs(opt::InputArgList &args) {
   errorHandler().verbose = args.hasArg(OPT_verbose);
@@ -1335,6 +1354,21 @@ static void readConfigs(opt::InputArgList &args) {
       config->optEL = true;
   }
 
+  for (opt::Arg *arg : args.filtered(OPT_remap_inputs)) {
+    StringRef value(arg->getValue());
+    remapInputs(value, arg->getSpelling());
+  }
+  for (opt::Arg *arg : args.filtered(OPT_remap_inputs_file)) {
+    StringRef filename(arg->getValue());
+    std::optional<MemoryBufferRef> buffer = readFile(filename);
+    if (!buffer)
+      continue;
+    // Parse 'from-glob=to-file' lines, ignoring #-led comments.
+    for (auto [lineno, line] : llvm::enumerate(args::getLines(*buffer)))
+      if (remapInputs(line, filename + ":" + Twine(lineno + 1)))
+        break;
+  }
+
   for (opt::Arg *arg : args.filtered(OPT_shuffle_sections)) {
     constexpr StringRef errPrefix = "--shuffle-sections=: ";
     std::pair<StringRef, StringRef> kv = StringRef(arg->getValue()).split('=');
diff --git a/lld/ELF/DriverUtils.cpp b/lld/ELF/DriverUtils.cpp
index 254f4b7224628..50e9f509bcb47 100644
--- a/lld/ELF/DriverUtils.cpp
+++ b/lld/ELF/DriverUtils.cpp
@@ -192,6 +192,7 @@ std::string elf::createResponseFile(const opt::InputArgList &args) {
     case OPT_export_dynamic_symbol_list:
     case OPT_just_symbols:
     case OPT_library_path:
+    case OPT_remap_inputs_file:
     case OPT_retain_symbols_file:
     case OPT_rpath:
     case OPT_script:
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index b7686e8e3ba99..b8291fb4307bc 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -195,6 +195,29 @@ std::optional<MemoryBufferRef> elf::readFile(StringRef path) {
   if (!config->chroot.empty() && path.startswith("/"))
     path = saver().save(config->chroot + path);
 
+  bool remapped = false;
+  auto it = config->remapInputs.find(path);
+  if (it != config->remapInputs.end()) {
+    path = it->second;
+    remapped = true;
+  } else {
+    for (const auto &[pat, toFile] : config->remapInputsWildcards) {
+      if (pat.match(path)) {
+        path = toFile;
+        remapped = true;
+        break;
+      }
+    }
+  }
+  if (remapped) {
+    // Use /dev/null to indicate an input file that should be ignored. Change
+    // the path to NUL on Windows.
+#ifdef _WIN32
+    if (path == "/dev/null")
+      path = "NUL";
+#endif
+  }
+
   log(path);
   config->dependencyFiles.insert(llvm::CachedHashString(path));
 
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 8b5c23ee6a476..f16abc188b76f 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -67,9 +67,8 @@ static std::unique_ptr<raw_fd_ostream> openLTOOutputFile(StringRef file) {
 }
 
 static std::string getThinLTOOutputFile(StringRef modulePath) {
-  return lto::getThinLTOOutputFile(
-      std::string(modulePath), std::string(config->thinLTOPrefixReplaceOld),
-      std::string(config->thinLTOPrefixReplaceNew));
+  return lto::getThinLTOOutputFile(modulePath, config->thinLTOPrefixReplaceOld,
+                                   config->thinLTOPrefixReplaceNew);
 }
 
 static lto::Config createConfig() {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 5b3d9f6c40dee..8f2a440fd67a2 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -359,6 +359,14 @@ defm relax_gp: BB<"relax-gp",
   "Enable global pointer relaxation",
   "Disable global pointer relaxation (default)">;
 
+defm remap_inputs: EEq<"remap-inputs",
+  "Remap input files matching <from-glob> to <to-file>">,
+  MetaVarName<"<from-glob>=<to-file>">;
+
+def remap_inputs_file: JJ<"remap-inputs-file=">,
+  HelpText<"Each line contains 'from-glob=to-file'. An input file matching <from-glob> is remapped to <to-file>">,
+  MetaVarName<"<file>">;
+
 defm reproduce:
   EEq<"reproduce",
      "Write tar file containing inputs and command to reproduce link">;
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 2251ecc83cde6..97f676f000fdc 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -217,7 +217,7 @@ void OutputSection::finalizeInputSections() {
       });
       if (i == mergeSections.end()) {
         MergeSyntheticSection *syn =
-            createMergeSynthetic(name, ms->type, ms->flags, ms->addralign);
+            createMergeSynthetic(s->name, ms->type, ms->flags, ms->addralign);
         mergeSections.push_back(syn);
         i = std::prev(mergeSections.end());
         syn->entsize = ms->entsize;
@@ -534,7 +534,7 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
     taskSize += sections[i]->getSize();
     bool done = ++i == numSections;
     if (done || taskSize >= taskSizeLimit) {
-      tg.execute([=] { fn(begin, i); });
+      tg.spawn([=] { fn(begin, i); });
       if (done)
         break;
       begin = i;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 6f2280b678b45..2f7fcd664c6a5 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1534,16 +1534,10 @@ template <class ELFT> void elf::scanRelocations() {
           scanner.template scanSection<ELFT>(*s);
       }
     };
-    if (serial)
-      fn();
-    else
-      tg.execute(fn);
+    tg.spawn(fn, serial);
   }
 
-  // Both the main thread and thread pool index 0 use getThreadIndex()==0. Be
-  // careful that they don't concurrently run scanSections. When serial is
-  // true, fn() has finished at this point, so running execute is safe.
-  tg.execute([] {
+  tg.spawn([] {
     RelocationScanner scanner;
     for (Partition &part : partitions) {
       for (EhInputSection *sec : part.ehFrame->sections)
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index e95cca993bec6..2dc263d12f089 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -1564,9 +1564,11 @@ int64_t DynamicReloc::computeAddend() const {
     assert(sym != nullptr);
     return addend;
   case AddendOnlyWithTargetVA:
-  case AgainstSymbolWithTargetVA:
-    return InputSection::getRelocTargetVA(inputSec->file, type, addend,
-                                          getOffset(), *sym, expr);
+  case AgainstSymbolWithTargetVA: {
+    uint64_t ca = InputSection::getRelocTargetVA(inputSec->file, type, addend,
+                                                 getOffset(), *sym, expr);
+    return config->is64 ? ca : SignExtend64<32>(ca);
+  }
   case MipsMultiGotPage:
     assert(sym == nullptr);
     return getMipsPageAddr(outputSec->addr) + addend;
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index edcc6a92109be..26f3041021f30 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -292,6 +292,16 @@ class ThumbV4ABSLongThunk final : public ThumbThunk {
   void addSymbols(ThunkSection &isec) override;
 };
 
+// The AVR devices need thunks for R_AVR_LO8_LDI_GS/R_AVR_HI8_LDI_GS
+// when their destination is out of range [0, 0x1ffff].
+class AVRThunk : public Thunk {
+public:
+  AVRThunk(Symbol &dest, int64_t addend) : Thunk(dest, addend) {}
+  uint32_t size() override { return 4; }
+  void writeTo(uint8_t *buf) override;
+  void addSymbols(ThunkSection &isec) override;
+};
+
 // MIPS LA25 thunk
 class MipsThunk final : public Thunk {
 public:
@@ -892,6 +902,17 @@ void ThumbV4PILongThunk::addSymbols(ThunkSection &isec) {
   addSymbol("$d", STT_NOTYPE, 16, isec);
 }
 
+// Use the long jump which covers a range up to 8MiB.
+void AVRThunk::writeTo(uint8_t *buf) {
+  write32(buf, 0x940c); // jmp func
+  target->relocateNoSym(buf, R_AVR_CALL, destination.getVA());
+}
+
+void AVRThunk::addSymbols(ThunkSection &isec) {
+  addSymbol(saver().save("__AVRThunk_" + destination.getName()), STT_FUNC, 0,
+            isec);
+}
+
 // Write MIPS LA25 thunk code to call PIC function from the non-PIC one.
 void MipsThunk::writeTo(uint8_t *buf) {
   uint64_t s = destination.getVA();
@@ -1314,6 +1335,16 @@ static Thunk *addThunkArm(RelType reloc, Symbol &s, int64_t a) {
   fatal("unrecognized relocation type");
 }
 
+static Thunk *addThunkAVR(RelType type, Symbol &s, int64_t a) {
+  switch (type) {
+  case R_AVR_LO8_LDI_GS:
+  case R_AVR_HI8_LDI_GS:
+    return make<AVRThunk>(s, a);
+  default:
+    fatal("unrecognized relocation type " + toString(type));
+  }
+}
+
 static Thunk *addThunkMips(RelType type, Symbol &s) {
   if ((s.stOther & STO_MIPS_MICROMIPS) && isMipsR6())
     return make<MicroMipsR6Thunk>(s);
@@ -1365,6 +1396,8 @@ Thunk *elf::addThunk(const InputSection &isec, Relocation &rel) {
     return addThunkAArch64(rel.type, s, a);
   case EM_ARM:
     return addThunkArm(rel.type, s, a);
+  case EM_AVR:
+    return addThunkAVR(rel.type, s, a);
   case EM_MIPS:
     return addThunkMips(rel.type, s);
   case EM_PPC:
@@ -1372,6 +1405,6 @@ Thunk *elf::addThunk(const InputSection &isec, Relocation &rel) {
   case EM_PPC64:
     return addThunkPPC64(rel.type, s, a);
   default:
-    llvm_unreachable("add Thunk only supported for ARM, Mips and PowerPC");
+    llvm_unreachable("add Thunk only supported for ARM, AVR, Mips and PowerPC");
   }
 }
diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp
index 481ac9444ef07..a2d3934c5bced 100644
--- a/lld/MachO/LTO.cpp
+++ b/lld/MachO/LTO.cpp
@@ -46,9 +46,8 @@ static std::unique_ptr<raw_fd_ostream> openFile(StringRef file) {
 }
 
 static std::string getThinLTOOutputFile(StringRef modulePath) {
-  return lto::getThinLTOOutputFile(
-      std::string(modulePath), std::string(config->thinLTOPrefixReplaceOld),
-      std::string(config->thinLTOPrefixReplaceNew));
+  return lto::getThinLTOOutputFile(modulePath, config->thinLTOPrefixReplaceOld,
+                                   config->thinLTOPrefixReplaceNew);
 }
 
 static lto::Config createConfig() {
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 50c556bb8b311..393b3f7d848de 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -26,6 +26,9 @@ Non-comprehensive list of changes in this release
 ELF Improvements
 ----------------
 
+* ``--remap-inputs=`` and ``--remap-inputs-file=`` are added to remap input files.
+  (`D148859 <https://reviews.llvm.org/D148859>`_)
+
 Breaking changes
 ----------------
 
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index d97cd26a6378b..e3db3a20678d9 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -497,6 +497,21 @@ Restore the states saved by
 Enable global pointer relaxation for RISC-V.
 .It Fl -relocatable , Fl r
 Create relocatable object file.
+.It Fl -remap-inputs Ns = Ns Ar from-glob=to-file
+Input files matching
+.Cm from-glob
+are mapped to
+.Cm to-file.
+Use
+.Cm /dev/null
+to ignore an input file.
+.It Fl -remap-inputs-file Ns = Ns Ar file
+Remap input files based on patterns in
+.Ar file .
+Each line in the remap file is of the format
+.Cm from-glob=to-file
+or a comment starting with
+.Cm # .
 .It Fl -reproduce Ns = Ns Ar path
 Write a tar file to
 .Ar path,
diff --git a/lld/test/COFF/directives-unsupported.s b/lld/test/COFF/directives-unsupported.s
new file mode 100644
index 0000000000000..1b8231178df83
--- /dev/null
+++ b/lld/test/COFF/directives-unsupported.s
@@ -0,0 +1,15 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -triple=x86_64-windows %s -filetype=obj -o %t.obj
+
+# RUN: not lld-link -dll -out:%t.dll -entry:entry %t.obj -subsystem:console 2>&1 | FileCheck %s
+
+# CHECK: warning: ignoring unknown argument: -unknowndirectivename
+# CHECK: error: -unknowndirectivename is not allowed in .drectve ({{.*}}.obj)
+
+  .global entry
+  .text
+entry:
+  ret
+  .section .drectve
+  .ascii " -unknowndirectivename "
diff --git a/lld/test/COFF/export.test b/lld/test/COFF/export.test
index e53e3dea9aa29..755a552a57ff8 100644
--- a/lld/test/COFF/export.test
+++ b/lld/test/COFF/export.test
@@ -4,8 +4,9 @@
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK1 %s
 
 CHECK1:      Export Table:
-CHECK1:      DLL name: export.test.tmp.dll
-CHECK1:      Ordinal      RVA  Name
+CHECK1-NEXT: DLL name: export.test.tmp.dll
+CHECK1-NEXT: Ordinal base: 1
+CHECK1-NEXT: Ordinal      RVA  Name
 CHECK1-NEXT:       1   0x1008  exportfn1
 CHECK1-NEXT:       2   0x1010  exportfn2
 
@@ -13,12 +14,9 @@ CHECK1-NEXT:       2   0x1010  exportfn2
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK2 %s
 
 CHECK2:      Export Table:
-CHECK2:      DLL name: export.test.tmp.dll
-CHECK2:      Ordinal      RVA  Name
-CHECK2-NEXT:       1        0
-CHECK2-NEXT:       2        0
-CHECK2-NEXT:       3        0
-CHECK2-NEXT:       4        0
+CHECK2-NEXT: DLL name: export.test.tmp.dll
+CHECK2-NEXT: Ordinal base: 1
+CHECK2-NEXT: Ordinal      RVA  Name
 CHECK2-NEXT:       5   0x1008  exportfn1
 CHECK2-NEXT:       6   0x1010  exportfn2
 CHECK2-NEXT:       7   0x1010  exportfn3
@@ -27,12 +25,9 @@ CHECK2-NEXT:       7   0x1010  exportfn3
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK3 %s
 
 CHECK3:      Export Table:
-CHECK3:      DLL name: export.test.tmp.dll
-CHECK3:      Ordinal      RVA  Name
-CHECK3-NEXT:       1        0
-CHECK3-NEXT:       2        0
-CHECK3-NEXT:       3        0
-CHECK3-NEXT:       4        0
+CHECK3-NEXT: DLL name: export.test.tmp.dll
+CHECK3-NEXT: Ordinal base: 1
+CHECK3-NEXT: Ordinal      RVA  Name
 CHECK3-NEXT:       5   0x1008
 CHECK3-NEXT:       6   0x1010  exportfn2
 
@@ -41,8 +36,9 @@ CHECK3-NEXT:       6   0x1010  exportfn2
 # RUN: llvm-nm %t.lib | FileCheck -check-prefix=CHECK4-NM %s
 
 CHECK4:      Export Table:
-CHECK4:      DLL name: export.test.tmp.dll
-CHECK4:      Ordinal      RVA  Name
+CHECK4-NEXT: DLL name: export.test.tmp.dll
+CHECK4-NEXT: Ordinal base: 1
+CHECK4-NEXT: Ordinal      RVA  Name
 CHECK4-NEXT:       1   0x1010  exportfn3
 CHECK4-NEXT:       2   0x1008  f1
 CHECK4-NEXT:       3   0x1010  f2
@@ -55,9 +51,9 @@ CHECK4-NM: 00000000 T f2
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK5 %s
 
 CHECK5:      Export Table:
-CHECK5:      DLL name: export.test.tmp.dll
-CHECK5:      Ordinal      RVA  Name
-CHECK5-NEXT:       1        0
+CHECK5-NEXT: DLL name: export.test.tmp.dll
+CHECK5-NEXT: Ordinal base: 1
+CHECK5-NEXT: Ordinal      RVA  Name
 CHECK5-NEXT:       2   0x1010  fn2
 CHECK5-NEXT:       3   0x1008  exportfn1
 CHECK5-NEXT:       4   0x1010  exportfn3
diff --git a/lld/test/COFF/export32.test b/lld/test/COFF/export32.test
index 4e15177c16128..8e27b0834024a 100644
--- a/lld/test/COFF/export32.test
+++ b/lld/test/COFF/export32.test
@@ -8,8 +8,9 @@
 # RUN: llvm-readobj --file-headers --sections %t.dll | FileCheck -check-prefix=HEADER-MERGE %s
 
 # CHECK1:      Export Table:
-# CHECK1:      DLL name: export32.test.tmp.dll
-# CHECK1:      Ordinal      RVA  Name
+# CHECK1-NEXT: DLL name: export32.test.tmp.dll
+# CHECK1-NEXT: Ordinal base: 1
+# CHECK1-NEXT: Ordinal      RVA  Name
 # CHECK1-NEXT:       1   0x1008  exportfn1
 # CHECK1-NEXT:       2   0x1010  exportfn2
 
@@ -24,12 +25,9 @@
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK2 %s
 
 # CHECK2:      Export Table:
-# CHECK2:      DLL name: export32.test.tmp.dll
-# CHECK2:      Ordinal      RVA  Name
-# CHECK2-NEXT:       1        0
-# CHECK2-NEXT:       2        0
-# CHECK2-NEXT:       3        0
-# CHECK2-NEXT:       4        0
+# CHECK2-NEXT: DLL name: export32.test.tmp.dll
+# CHECK2-NEXT: Ordinal base: 1
+# CHECK2-NEXT: Ordinal      RVA  Name
 # CHECK2-NEXT:       5   0x1008  exportfn1
 # CHECK2-NEXT:       6   0x1010  exportfn2
 # CHECK2-NEXT:       7   0x1010  exportfn3
@@ -39,12 +37,9 @@
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK3 %s
 
 # CHECK3:      Export Table:
-# CHECK3:      DLL name: export32.test.tmp.dll
-# CHECK3:      Ordinal      RVA  Name
-# CHECK3-NEXT:       1        0
-# CHECK3-NEXT:       2        0
-# CHECK3-NEXT:       3        0
-# CHECK3-NEXT:       4        0
+# CHECK3-NEXT: DLL name: export32.test.tmp.dll
+# CHECK3-NEXT: Ordinal base: 1
+# CHECK3-NEXT: Ordinal      RVA  Name
 # CHECK3-NEXT:       5   0x1008
 # CHECK3-NEXT:       6   0x1010  exportfn2
 
@@ -52,8 +47,9 @@
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK4 %s
 
 # CHECK4:      Export Table:
-# CHECK4:      DLL name: export32.test.tmp.dll
-# CHECK4:      Ordinal      RVA  Name
+# CHECK4-NEXT: DLL name: export32.test.tmp.dll
+# CHECK4-NEXT: Ordinal base: 1
+# CHECK4-NEXT: Ordinal      RVA  Name
 # CHECK4-NEXT:       1   0x1010  exportfn3
 # CHECK4-NEXT:       2   0x1008  f1
 # CHECK4-NEXT:       3   0x1010  f2
@@ -69,9 +65,9 @@
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK5 %s
 
 # CHECK5:      Export Table:
-# CHECK5:      DLL name: export32.test.tmp.dll
-# CHECK5:      Ordinal      RVA  Name
-# CHECK5-NEXT:       1        0
+# CHECK5-NEXT: DLL name: export32.test.tmp.dll
+# CHECK5-NEXT: Ordinal base: 1
+# CHECK5-NEXT: Ordinal      RVA  Name
 # CHECK5-NEXT:       2   0x1010  fn2
 # CHECK5-NEXT:       3   0x1008  exportfn1
 # CHECK5-NEXT:       4   0x1010  exportfn3
@@ -87,9 +83,10 @@
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=CHECK7 %s
 
 # CHECK7:      Export Table:
-# CHECK7:      DLL name: export32.test.tmp.dll
-# CHECK7:      Ordinal      RVA  Name
-# CHECK7-NEXT:       1        0
+# CHECK7-NEXT: DLL name: export32.test.tmp.dll
+# CHECK7-NEXT: Ordinal base: 1
+# CHECK7-NEXT: Ordinal      RVA  Name
+# CHECK7-NEXT:       1   0x1010  exportfn3
 # CHECK7-NEXT:       2   0x1010  foo
 
 --- !COFF
diff --git a/lld/test/COFF/inferasanlibs-drectve.s b/lld/test/COFF/inferasanlibs-drectve.s
new file mode 100644
index 0000000000000..8406658cea389
--- /dev/null
+++ b/lld/test/COFF/inferasanlibs-drectve.s
@@ -0,0 +1,16 @@
+# REQUIRES: x86
+
+# RUN: llvm-mc -triple=x86_64-windows %s -filetype=obj -o %t.obj
+
+# RUN: lld-link -dll -out:%t.dll -entry:entry %t.obj -subsystem:console 2>&1 | FileCheck --allow-empty --ignore-case %s
+
+# CHECK-NOT: ignoring unknown argument
+# CHECK-NOT: inferasanlibs
+# CHECK-NOT: is not allowed in .drectve
+
+  .global entry
+  .text
+entry:
+  ret
+  .section .drectve
+  .ascii " /INFERASANLIBS "
diff --git a/lld/test/COFF/inferasanlibs.test b/lld/test/COFF/inferasanlibs.test
new file mode 100644
index 0000000000000..e8f36602b5f6b
--- /dev/null
+++ b/lld/test/COFF/inferasanlibs.test
@@ -0,0 +1,8 @@
+# RUN: yaml2obj %p/Inputs/ret42.yaml -o %t.obj
+
+# RUN: lld-link /out:%t.exe /entry:main %t.obj /inferasanlibs 2>&1 | FileCheck --check-prefix=POS %s
+# RUN: lld-link /out:%t.exe /entry:main %t.obj /inferasanlibs:no 2>&1 | FileCheck --allow-empty --check-prefix=NEG %s
+
+POS: ignoring '/inferasanlibs', this flag is not supported
+
+NEG-NOT: ignoring '/inferasanlibs', this flag is not supported
diff --git a/lld/test/COFF/symtab.test b/lld/test/COFF/symtab.test
index 41419a942d87b..f0a160cec790f 100644
--- a/lld/test/COFF/symtab.test
+++ b/lld/test/COFF/symtab.test
@@ -8,6 +8,8 @@
 
 # RUN: lld-link /debug /out:%t.exe /entry:main %t.obj %p/Inputs/std64.lib
 # RUN: llvm-readobj --symbols %t.exe | FileCheck -check-prefix=NO %s
+# RUN: lld-link /debug /merge:.blank=.debug_info /out:%t.exe /entry:main %t.obj %p/Inputs/std64.lib
+# RUN: llvm-readobj --symbols %t.exe | FileCheck %s
 
 # CHECK:      Symbols [
 # CHECK-NEXT:   Symbol {
@@ -84,7 +86,8 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
-# NO: Symbols [
+# NO:       Symbols [
+# NO-NEXT:  ]
 
 --- !COFF
 header:
@@ -141,6 +144,10 @@ sections:
     Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ]
     Alignment:       4
     SectionData:     48656C6C6F0048656C6C6F20576F726C6400
+  - Name:            .blank
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    Alignment:       1
+    SectionData:     0000
 symbols:
   - Name:            "@comp.id"
     Value:           10394907
diff --git a/lld/test/ELF/arm-pie-relative.s b/lld/test/ELF/arm-pie-relative.s
index 20421f2d0c8e8..0628d945063d9 100644
--- a/lld/test/ELF/arm-pie-relative.s
+++ b/lld/test/ELF/arm-pie-relative.s
@@ -3,6 +3,10 @@
 // RUN: ld.lld %t.o --pie -o %t
 // RUN: llvm-readobj -r %t | FileCheck %s
 // RUN: llvm-readelf -x .got %t | FileCheck %s --check-prefix=GOT
+// RUN: ld.lld %t.o --pie --image-base=0x80000000 --check-dynamic-relocations -o %t1 2>&1 | \
+// RUN:   FileCheck %s -allow-empty -check-prefix=NOERR
+// RUN: llvm-readobj -r %t1 | FileCheck %s --check-prefix=CHECK1
+// RUN: llvm-readelf -x .got %t1 | FileCheck %s --check-prefix=GOT1
 
 // Test that a R_ARM_GOT_BREL relocation with PIE results in a R_ARM_RELATIVE
 // dynamic relocation
@@ -23,3 +27,12 @@ sym:
 
 // GOT:      section '.got':
 // GOT-NEXT: 0x000201e4 e8010300
+
+// NOERR-NOT: internal linker error
+
+// CHECK1:      Relocations [
+// CHECK1-NEXT:   Section (5) .rel.dyn {
+// CHECK1-NEXT:     0x800201E4 R_ARM_RELATIVE
+
+// GOT1:      section '.got':
+// GOT1-NEXT: 0x800201e4 e8010380
diff --git a/lld/test/ELF/avr-reloc.s b/lld/test/ELF/avr-reloc.s
index 36904f80dabb2..172c0e03ba74b 100644
--- a/lld/test/ELF/avr-reloc.s
+++ b/lld/test/ELF/avr-reloc.s
@@ -1,12 +1,12 @@
 ; REQUIRES: avr
 ; RUN: llvm-mc -filetype=obj -triple=avr -mcpu=atmega328p %s -o %t0.o
-; RUN: ld.lld %t0.o --defsym=a=0x12345678 --defsym=b=30 -o %t0
+; RUN: ld.lld %t0.o --defsym=a=0x12345678 --defsym=b=30 --defsym=c=0x15554 -o %t0
 ; RUN: llvm-objdump -d --print-imm-hex --mcpu=atmega328p %t0 | \
 ; RUN:     FileCheck --check-prefixes=CHECK,AVR %s
 ; RUN: llvm-objdump -s --mcpu=atmega328p %t0 | \
 ; RUN:     FileCheck --check-prefixes=HEX,AVRHEX %s
 ; RUN: llvm-mc -filetype=obj -triple=avr -mcpu=attiny10 %s --defsym=TINY=1 -o %t1.o
-; RUN: ld.lld %t1.o --defsym=a=0x12345678 --defsym=b=30 -o %t1
+; RUN: ld.lld %t1.o --defsym=a=0x12345678 --defsym=b=30 --defsym=c=0x15554 -o %t1
 ; RUN: llvm-objdump -d --print-imm-hex --mcpu=attiny10 %t1 | FileCheck %s
 ; RUN: llvm-objdump -s --mcpu=attiny10 %t1 | \
 ; RUN:     FileCheck --check-prefixes=HEX,TINYHEX %s
@@ -20,7 +20,10 @@
 ; CHECK-NEXT:  ldi     r20, 0x3c
 ; CHECK-NEXT:  ldi     r20, 0x2b
 ; CHECK-NEXT:  ldi     r20, 0x1a
+; CHECK-NEXT:  ldi     r20, 0xaa
+; CHECK-NEXT:  ldi     r20, 0xaa
 ; CHECK-NEXT:  ldi     r20, 0xff
+
 ldi r20, lo8(a)     ; R_AVR_LO8_LDI
 ldi r20, hi8(a)     ; R_AVR_HI8_LDI
 ldi r20, hh8(a)     ; R_AVR_HH8_LDI
@@ -30,6 +33,9 @@ ldi r20, pm_lo8(a)  ; R_AVR_LO8_LDI_PM
 ldi r20, pm_hi8(a)  ; R_AVR_HI8_LDI_PM
 ldi r20, pm_hh8(a)  ; R_AVR_HH8_LDI_PM
 
+ldi r20, lo8_gs(c)  ; R_AVR_LO8_LDI_GS
+ldi r20, hi8_gs(c)  ; R_AVR_HI8_LDI_GS
+
 ldi r20, b+225
 
 .section .LDI_NEG,"ax",@progbits
diff --git a/lld/test/ELF/avr-thunk-ldi-gs.s b/lld/test/ELF/avr-thunk-ldi-gs.s
new file mode 100644
index 0000000000000..58de33780ba1e
--- /dev/null
+++ b/lld/test/ELF/avr-thunk-ldi-gs.s
@@ -0,0 +1,33 @@
+; REQUIRES: avr
+; RUN: llvm-mc -filetype=obj -triple=avr -mcpu=atmega2560 %s -o %t.o
+; RUN: ld.lld %t.o --defsym=a=0x1fffe --defsym=b=0x20000 -o %t
+; RUN: llvm-objdump -d --print-imm-hex --no-show-raw-insn --mcpu=atmega2560 %t \
+; RUN:     | FileCheck %s
+
+.section .LDI,"ax",@progbits
+
+;; CHECK-LABEL: <__AVRThunk_b>:
+;; CHECK-NEXT:  110b4: jmp  0x20000
+;; CHECK-LABEL: <__init>:
+;; CHECK-NEXT:  110b8:  ldi r30, 0xff
+;; CHECK-NEXT:  110ba:  ldi r31, 0xff
+;; CHECK-NEXT:  110bc:  eicall
+;; The destination of the following two LDI instructions is
+;; __AVRThunk_b == 0x110b4, so they actually are
+;;                      ldi r30, ((0x110b4) >> 1) & 0xff
+;;                      ldi r31, ((0x110b4) >> 9)
+;; CHECK-NEXT:  110be:  ldi r30, 0x5a
+;; CHECK-NEXT:  110c0:  ldi r31, 0x88
+;; CHECK-NEXT:  110c2:  eicall
+;; CHECK-NOT:   __AVRThunk_a
+
+.globl __init
+__init:
+;; No thunk is needed, since the destination is in range [0, 0x1ffff].
+ldi r30, lo8_gs(a)  ; R_AVR_LO8_LDI_GS
+ldi r31, hi8_gs(a)  ; R_AVR_HI8_LDI_GS
+eicall
+;; A thunk is needed, since the destination is out of range.
+ldi r30, lo8_gs(b)  ; R_AVR_LO8_LDI_GS
+ldi r31, hi8_gs(b)  ; R_AVR_HI8_LDI_GS
+eicall
diff --git a/lld/test/ELF/basic-avr.s b/lld/test/ELF/basic-avr.s
index 2096446b2fff3..71b8412bb2ac7 100644
--- a/lld/test/ELF/basic-avr.s
+++ b/lld/test/ELF/basic-avr.s
@@ -1,14 +1,81 @@
 # REQUIRES: avr
 # RUN: llvm-mc -filetype=obj -triple=avr-unknown-linux -mcpu=atmega328p %s -o %t.o
 # RUN: ld.lld %t.o -o %t.exe -Ttext=0
-# RUN: llvm-objdump -d %t.exe | FileCheck %s
+# RUN: llvm-objdump -d %t.exe --mcpu=atmega328 | FileCheck %s --check-prefix=ASM
+# RUN: llvm-readelf --file-headers --sections -l --symbols %t.exe \
+# RUN:     | FileCheck %s --check-prefix=ELF
 
 main:
-  call foo
+  call  foo
 foo:
-  jmp foo
+  jmp   foo
+  rcall foo
+  rjmp  foo
 
-# CHECK:      <main>:
-# CHECK-NEXT:   0: 0e 94 02 00 <unknown>
-# CHECK:      <foo>:
-# CHECK-NEXT:   4: 0c 94 02 00 <unknown>
+# ASM:      <main>:
+# ASM-NEXT:   0: 0e 94 02 00  call  0x4
+# ASM:      <foo>:
+# ASM-NEXT:   4: 0c 94 02 00  jmp   0x4
+# ASM-NEXT:   8: fd df        rcall .-6
+# ASM-NEXT:   a: fc cf        rjmp  .-8
+
+# ELF:      ELF Header:
+# ELF-NEXT:   Magic:   7f 45 4c 46 01 01 01 00 00 00 00 00 00 00 00 00
+# ELF-NEXT:   Class:                             ELF32
+# ELF-NEXT:   Data:                              2's complement, little endian
+# ELF-NEXT:   Version:                           1 (current)
+# ELF-NEXT:   OS/ABI:                            UNIX - System V
+# ELF-NEXT:   ABI Version:                       0
+# ELF-NEXT:   Type:                              EXEC (Executable file)
+# ELF-NEXT:   Machine:                           Atmel AVR 8-bit microcontroller
+# ELF-NEXT:   Version:                           0x1
+# ELF-NEXT:   Entry point address:               0x0
+# ELF-NEXT:   Start of program headers:          52 (bytes into file)
+# ELF-NEXT:   Start of section headers:          4216 (bytes into file)
+# ELF-NEXT:   Flags:                             0x85, EF_AVR_ARCH_AVR5, relaxable
+# ELF-NEXT:   Size of this header:               52 (bytes)
+# ELF-NEXT:   Size of program headers:           32 (bytes)
+# ELF-NEXT:   Number of program headers:         4
+# ELF-NEXT:   Size of section headers:           40 (bytes)
+# ELF-NEXT:   Number of section headers:         6
+# ELF-NEXT:   Section header string table index: 4
+# ELF-NEXT: There are 6 section headers, starting at offset 0x1078:
+
+# ELF:      Section Headers:
+# ELF-NEXT:   [Nr] Name              Type            Address  Off    Size   ES Flg Lk Inf Al
+# ELF-NEXT:   [ 0]                   NULL            00000000 000000 000000 00      0   0  0
+# ELF-NEXT:   [ 1] .text             PROGBITS        00000000 001000 00000c 00  AX  0   0  4
+# ELF-NEXT:   [ 2] .comment          PROGBITS        00000000 00100c 000008 01  MS  0   0  1
+# ELF-NEXT:   [ 3] .symtab           SYMTAB          00000000 001014 000030 10      5   3  4
+# ELF-NEXT:   [ 4] .shstrtab         STRTAB          00000000 001044 00002a 00      0   0  1
+# ELF-NEXT:   [ 5] .strtab           STRTAB          00000000 00106e 00000a 00      0   0  1
+# ELF-NEXT: Key to Flags:
+# ELF-NEXT:   W (write), A (alloc), X (execute), M (merge), S (strings), I (info),
+# ELF-NEXT:   L (link order), O (extra OS processing required), G (group), T (TLS),
+# ELF-NEXT:   C (compressed), x (unknown), o (OS specific), E (exclude),
+# ELF-NEXT:   R (retain), p (processor specific)
+
+# ELF:      Elf file type is EXEC (Executable file)
+# ELF-NEXT: Entry point 0x0
+# ELF-NEXT: There are 4 program headers, starting at offset 52
+
+# ELF:      Program Headers:
+# ELF-NEXT:   Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
+# ELF-NEXT:   PHDR           0x000034 0x00010034 0x00010034 0x00080 0x00080 R   0x4
+# ELF-NEXT:   LOAD           0x000000 0x00010000 0x00010000 0x000b4 0x000b4 R   0x1000
+# ELF-NEXT:   LOAD           0x001000 0x00000000 0x00000000 0x0000c 0x0000c R E 0x1000
+# ELF-NEXT:   GNU_STACK      0x000000 0x00000000 0x00000000 0x00000 0x00000 RW  0x0
+
+# ELF:      Section to Segment mapping:
+# ELF-NEXT:   Segment Sections...
+# ELF-NEXT:    00
+# ELF-NEXT:    01
+# ELF-NEXT:    02     .text
+# ELF-NEXT:    03
+# ELF-NEXT:    None   .comment .symtab .shstrtab .strtab
+
+# ELF:      Symbol table '.symtab' contains 3 entries:
+# ELF-NEXT:    Num:    Value  Size Type    Bind   Vis       Ndx Name
+# ELF-NEXT:      0: 00000000     0 NOTYPE  LOCAL  DEFAULT   UND
+# ELF-NEXT:      1: 00000000     0 NOTYPE  LOCAL  DEFAULT     1 main
+# ELF-NEXT:      2: 00000004     0 NOTYPE  LOCAL  DEFAULT     1 foo
diff --git a/lld/test/ELF/merge-entsize.s b/lld/test/ELF/merge-entsize.s
index 080d71b329713..f45b334f63656 100644
--- a/lld/test/ELF/merge-entsize.s
+++ b/lld/test/ELF/merge-entsize.s
@@ -1,8 +1,12 @@
 // REQUIRES: x86
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-// RUN: ld.lld %t.o -o %t
+// RUN: ld.lld %t.o -o %t -M | FileCheck %s --check-prefix=MAP
 // RUN: llvm-readobj -S %t | FileCheck %s
 
+// MAP:      .rodata{{$}}
+// MAP-NEXT:         <internal>:(.rodata.1)
+// MAP-NEXT:         <internal>:(.rodata.2)
+
         .section  .rodata.1,"aM",@progbits,1
         .p2align 2
         .byte 0x42
diff --git a/lld/test/ELF/merge-string-align.s b/lld/test/ELF/merge-string-align.s
index 0098d95b23d33..661019dd9b5b7 100644
--- a/lld/test/ELF/merge-string-align.s
+++ b/lld/test/ELF/merge-string-align.s
@@ -1,8 +1,13 @@
 // REQUIRES: x86
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-// RUN: ld.lld %t.o -o %t.so -shared
+// RUN: ld.lld %t.o -o %t.so -shared -M | FileCheck %s --check-prefix=MAP
 // RUN: llvm-readobj -S --section-data %t.so | FileCheck %s
 
+// MAP:      .rodata{{$}}
+// MAP-NEXT:         <internal>:(.rodata.foo)
+// MAP-NEXT: .rodata2
+// MAP-NEXT:         <internal>:(.rodata2)
+
         .section        .rodata.foo,"aMS",@progbits,1
         .align  16
         .asciz "foo"
diff --git a/lld/test/ELF/remap-inputs.test b/lld/test/ELF/remap-inputs.test
new file mode 100644
index 0000000000000..8a2d2e9891064
--- /dev/null
+++ b/lld/test/ELF/remap-inputs.test
@@ -0,0 +1,85 @@
+# REQUIRES: x86
+## --remap-inputs and --remap-inputs-file= remap input files.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: llvm-as b.ll -o b.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 c.s -o c.o && llvm-ar rc c.a c.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 d.s -o d.o && ld.lld -shared -soname=d d.o -o d.so
+# RUN: ld.lld --remap-inputs-file=1.map --remap-inputs-file=2.map --remap-inputs='d*.so=d.so' --reproduce=repro.tar aa.o bb.bc cc.a dd.so empty -o out
+# RUN: tar tf repro.tar | FileCheck %s --check-prefix=REPRO
+
+# REPRO:      1.map
+# REPRO-NEXT: 2.map
+# REPRO-NEXT: a.o
+# REPRO-NEXT: b.o
+# REPRO-NEXT: c.a
+# REPRO-NEXT: d.so
+
+## --remap-inputs can also be specified multiple times.
+# RUN: ld.lld --remap-inputs 'aa.o=a.o' --remap-inputs='d[d].so=d.so' aa.o b.o c.a d.so -o /dev/null
+
+## A multiple-to-one pattern may easily cause issues. Users should be careful.
+# RUN: not ld.lld --remap-inputs-file=3.map aa.o bb.bc -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=DUPLICATE --implicit-check-not=error:
+# DUPLICATE: error: duplicate symbol: _start
+
+# RUN: not ld.lld --remap-inputs-file=err1.map aa.o bb.bc -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR1 --implicit-check-not=error:
+# ERR1:      error: err1.map:2: parse error, not 'from-glob=to-file'
+# ERR1-NEXT: error: cannot open bb.bc: {{.*}}
+
+# RUN: not ld.lld --remap-inputs-file=err2.map aa.o -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR2 --implicit-check-not=error:
+# ERR2:      error: err2.map:1: invalid glob pattern: aa.[o
+# ERR2-NEXT: error: cannot open aa.o: {{.*}}
+
+# RUN: not ld.lld --remap-inputs=aa.o aa.o -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR3 --implicit-check-not=error:
+# RUN: not ld.lld --remap-inputs=aa.o= aa.o -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR3 --implicit-check-not=error:
+# ERR3:      error: --remap-inputs: parse error, not 'from-glob=to-file'
+# ERR3-NEXT: error: cannot open aa.o: {{.*}}
+
+#--- a.s
+.globl _start
+_start:
+  call b
+  call c
+  call d
+
+#--- b.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @b() {
+  ret void
+}
+
+#--- c.s
+.globl c
+c:
+
+#--- d.s
+.globl d
+d:
+
+#--- 1.map
+aa.o=a.o
+b?.[b]c=b.o
+
+#--- 2.map
+cc.a=c.a
+## Use /dev/null to indicate an input file which should be ignored.
+empty=/dev/null
+
+#--- 3.map
+*=a.o
+
+#--- err1.map
+aa.o=a.o
+bb.bc
+cc.a
+
+#--- err2.map
+aa.[o=a.o
diff --git a/lld/test/MachO/lit.local.cfg b/lld/test/MachO/lit.local.cfg
index 929160d0c001f..08bec0a1aff19 100644
--- a/lld/test/MachO/lit.local.cfg
+++ b/lld/test/MachO/lit.local.cfg
@@ -2,6 +2,12 @@
 
 import os
 
+# FIXME: The MachO back-end currently does not respect endianness when
+# accessing binary data structures, and therefore only works correctly
+# on little-endian host systems.  Skip all tests on big-endian hosts.
+if sys.byteorder == 'big':
+    config.unsupported = True
+
 # We specify the most commonly-used archs and platform versions in our tests
 # here. Tests which need different settings can just append to this, as only
 # the last value will be used.
diff --git a/lldb/CODE_OWNERS.txt b/lldb/CODE_OWNERS.txt
index ae679e90d9243..c62c5ade7634d 100644
--- a/lldb/CODE_OWNERS.txt
+++ b/lldb/CODE_OWNERS.txt
@@ -18,8 +18,7 @@ N: Jim Ingham
 E: jingham@apple.com
 D: Overall LLDB architecture, Thread plans, Expression parser, ValueObject, Breakpoints, ABI
 D: Watchpoints, Trampolines, Target, Command Interpreter, C++ / Objective-C Language runtime
-D: Expression evaluator, IR interpreter, Clang integration
-D: Data Formatters
+D: Expression evaluator, IR interpreter, Clang integration, Data Formatters
 
 N: Ed Maste
 E: emaste@freebsd.org
@@ -47,3 +46,9 @@ N: Walter Erquinigo
 E: a20012251@gmail.com
 E: waltermelon@fb.com
 D: Trace, TraceCursor, TraceExport, intel-pt, lldb-vscode, Data Formatters
+
+N: Med Ismail Bennani
+E: ismail@bennani.ma
+E: m.i.b@apple.com
+D: Breakpoints, Target, General Process Execution, Command Interpreter
+D: Script Interpreter, API, Data Formatters, CrashLog, Scripted Process
diff --git a/lldb/bindings/interface/SBTargetExtensions.i b/lldb/bindings/interface/SBTargetExtensions.i
index a062a109fa0b5..02c40b09c857f 100644
--- a/lldb/bindings/interface/SBTargetExtensions.i
+++ b/lldb/bindings/interface/SBTargetExtensions.i
@@ -88,7 +88,7 @@ STRING_EXTENSION_LEVEL_OUTSIDE(SBTarget, lldb::eDescriptionLevelBrief)
 
         def get_modules_access_object(self):
             '''An accessor function that returns a modules_access() object which allows lazy module access from a lldb.SBTarget object.'''
-            return self.modules_access (self)
+            return self.modules_access(self)
 
         def get_modules_array(self):
             '''An accessor function that returns a list() that contains all modules in a lldb.SBTarget object.'''
@@ -107,18 +107,80 @@ STRING_EXTENSION_LEVEL_OUTSIDE(SBTarget, lldb::eDescriptionLevelBrief)
             object.'''
             return lldb_iter(self, 'GetNumBreakpoints', 'GetBreakpointAtIndex')
 
+        class bkpts_access(object):
+            '''A helper object that will lazily hand out bkpts for a target when supplied an index.'''
+            def __init__(self, sbtarget):
+                self.sbtarget = sbtarget
+
+            def __len__(self):
+                if self.sbtarget:
+                    return int(self.sbtarget.GetNumBreakpoints())
+                return 0
+
+            def __getitem__(self, key):
+                if isinstance(key, int):
+                    count = len(self)
+                    if -count <= key < count:
+                        key %= count
+                        return self.sbtarget.GetBreakpointAtIndex(key)
+                return None
+
+        def get_bkpts_access_object(self):
+            '''An accessor function that returns a bkpts_access() object which allows lazy bkpt access from a lldb.SBtarget object.'''
+            return self.bkpts_access(self)
+
+        def get_target_bkpts(self):
+            '''An accessor function that returns a list() that contains all bkpts in a lldb.SBtarget object.'''
+            bkpts = []
+            for idx in range(self.GetNumBreakpoints()):
+                bkpts.append(self.GetBreakpointAtIndex(idx))
+            return bkpts
+
         def watchpoint_iter(self):
             '''Returns an iterator over all watchpoints in a lldb.SBTarget
             object.'''
             return lldb_iter(self, 'GetNumWatchpoints', 'GetWatchpointAtIndex')
 
+        class watchpoints_access(object):
+            '''A helper object that will lazily hand out watchpoints for a target when supplied an index.'''
+            def __init__(self, sbtarget):
+                self.sbtarget = sbtarget
+
+            def __len__(self):
+                if self.sbtarget:
+                    return int(self.sbtarget.GetNumWatchpoints())
+                return 0
+
+            def __getitem__(self, key):
+                if isinstance(key, int):
+                    count = len(self)
+                    if -count <= key < count:
+                        key %= count
+                        return self.sbtarget.GetWatchpointAtIndex(key)
+                return None
+
+        def get_watchpoints_access_object(self):
+            '''An accessor function that returns a watchpoints_access() object which allows lazy watchpoint access from a lldb.SBtarget object.'''
+            return self.watchpoints_access(self)
+
+        def get_target_watchpoints(self):
+            '''An accessor function that returns a list() that contains all watchpoints in a lldb.SBtarget object.'''
+            watchpoints = []
+            for idx in range(self.GetNumWatchpoints()):
+                bkpts.append(self.GetWatchpointAtIndex(idx))
+            return watchpoints
+
         modules = property(get_modules_array, None, doc='''A read only property that returns a list() of lldb.SBModule objects contained in this target. This list is a list all modules that the target currently is tracking (the main executable and all dependent shared libraries).''')
         module = property(get_modules_access_object, None, doc=r'''A read only property that returns an object that implements python operator overloading with the square brackets().\n    target.module[<int>] allows array access to any modules.\n    target.module[<str>] allows access to modules by basename, full path, or uuid string value.\n    target.module[uuid.UUID()] allows module access by UUID.\n    target.module[re] allows module access using a regular expression that matches the module full path.''')
         process = property(GetProcess, None, doc='''A read only property that returns an lldb object that represents the process (lldb.SBProcess) that this target owns.''')
         executable = property(GetExecutable, None, doc='''A read only property that returns an lldb object that represents the main executable module (lldb.SBModule) for this target.''')
         debugger = property(GetDebugger, None, doc='''A read only property that returns an lldb object that represents the debugger (lldb.SBDebugger) that owns this target.''')
         num_breakpoints = property(GetNumBreakpoints, None, doc='''A read only property that returns the number of breakpoints that this target has as an integer.''')
+        breakpoints = property(get_target_bkpts, None, doc='''A read only property that returns a list() of lldb.SBBreakpoint objects for all breakpoints in this target.''')
+        breakpoint = property(get_bkpts_access_object, None, doc='''A read only property that returns an object that can be used to access breakpoints as an array ("bkpt_12 = lldb.target.bkpt[12]").''')
         num_watchpoints = property(GetNumWatchpoints, None, doc='''A read only property that returns the number of watchpoints that this target has as an integer.''')
+        watchpoints = property(get_target_watchpoints, None, doc='''A read only property that returns a list() of lldb.SBwatchpoint objects for all watchpoints in this target.''')
+        watchpoint = property(get_watchpoints_access_object, None, doc='''A read only property that returns an object that can be used to access watchpoints as an array ("watchpoint_12 = lldb.target.watchpoint[12]").''')
         broadcaster = property(GetBroadcaster, None, doc='''A read only property that an lldb object that represents the broadcaster (lldb.SBBroadcaster) for this target.''')
         byte_order = property(GetByteOrder, None, doc='''A read only property that returns an lldb enumeration value (lldb.eByteOrderLittle, lldb.eByteOrderBig, lldb.eByteOrderInvalid) that represents the byte order for this target.''')
         addr_size = property(GetAddressByteSize, None, doc='''A read only property that returns the size in bytes of an address for this target.''')
diff --git a/lldb/bindings/macros.swig b/lldb/bindings/macros.swig
index eee504cb2e6f4..cb013daa158d9 100644
--- a/lldb/bindings/macros.swig
+++ b/lldb/bindings/macros.swig
@@ -1,18 +1,3 @@
-%define STRING_EXTENSION_LEVEL(Class, Level)
-%extend {
-  std::string lldb:: ## Class ## ::__repr__(){
-    lldb::SBStream stream;
-    $self->GetDescription (stream, Level);
-    const char *desc = stream.GetData();
-    size_t desc_len = stream.GetSize();
-    if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r')) {
-      --desc_len;
-    }
-    return std::string(desc, desc_len);
-  }
-}
-%enddef
-
 %define STRING_EXTENSION_LEVEL_OUTSIDE(Class, Level)
 %extend lldb:: ## Class ## {
   std::string __repr__(){
@@ -28,25 +13,6 @@
 }
 %enddef
 
-%define STRING_EXTENSION(Class)
-%extend {
-  std::string lldb:: ## Class ## ::__repr__(){
-    lldb::SBStream stream;
-    $self->GetDescription (stream);
-    const char *desc = stream.GetData();
-    size_t desc_len = stream.GetSize();
-    if (desc_len > 0 && (desc[desc_len-1] == '\n' || desc[desc_len-1] == '\r')) {
-      --desc_len;
-    }
-    return std::string(desc, desc_len);
-  }
-}
-%enddef
-
-// NOTE:
-// This is for when you are peforming the extension from outside of the class
-// instead of inside of it. If this change works out, it will replace
-// STRING_EXTENSION entirely.
 %define STRING_EXTENSION_OUTSIDE(Class)
 %extend lldb:: ## Class ## {
   std::string __repr__(){
diff --git a/lldb/bindings/python/python-wrapper.swig b/lldb/bindings/python/python-wrapper.swig
index d82c2dace18f3..bb452658a8380 100644
--- a/lldb/bindings/python/python-wrapper.swig
+++ b/lldb/bindings/python/python-wrapper.swig
@@ -367,6 +367,38 @@ bool lldb_private::LLDBSWIGPythonCallThreadPlan(
   return false;
 }
 
+bool lldb_private::LLDBSWIGPythonCallThreadPlan(
+    void *implementor, const char *method_name, lldb_private::Stream *stream,
+    bool &got_error) {
+  got_error = false;
+
+  PyErr_Cleaner py_err_cleaner(false);
+  PythonObject self(PyRefType::Borrowed, static_cast<PyObject *>(implementor));
+  auto pfunc = self.ResolveName<PythonCallable>(method_name);
+
+  if (!pfunc.IsAllocated()) 
+    return false;
+
+  auto *sb_stream = new lldb::SBStream();
+  PythonObject sb_stream_arg =
+      ToSWIGWrapper(std::unique_ptr<lldb::SBStream>(sb_stream));
+
+  PythonObject result;
+  result = pfunc(sb_stream_arg);
+
+  if (PyErr_Occurred()) {
+    printf("Error occured for call to %s.\n",
+           method_name);
+    PyErr_Print();
+    got_error = true;
+    return false;
+  }
+  if (stream)
+    stream->PutCString(sb_stream->GetData());
+  return true;
+
+}
+
 PythonObject lldb_private::LLDBSwigPythonCreateScriptedBreakpointResolver(
     const char *python_class_name, const char *session_dictionary_name,
     const StructuredDataImpl &args_impl,
@@ -716,6 +748,18 @@ void *lldb_private::LLDBSWIGPython_CastPyObjectToSBData(PyObject * data) {
   return sb_ptr;
 }
 
+void *lldb_private::LLDBSWIGPython_CastPyObjectToSBBreakpoint(PyObject * data) {
+  lldb::SBBreakpoint *sb_ptr = nullptr;
+
+  int valid_cast =
+      SWIG_ConvertPtr(data, (void **)&sb_ptr, SWIGTYPE_p_lldb__SBBreakpoint, 0);
+
+  if (valid_cast == -1)
+    return NULL;
+
+  return sb_ptr;
+}
+
 void *lldb_private::LLDBSWIGPython_CastPyObjectToSBAttachInfo(PyObject * data) {
   lldb::SBAttachInfo *sb_ptr = nullptr;
 
diff --git a/lldb/docs/doxygen.cfg.in b/lldb/docs/doxygen.cfg.in
index 5712779e6b2c5..be8a21a535bdd 100644
--- a/lldb/docs/doxygen.cfg.in
+++ b/lldb/docs/doxygen.cfg.in
@@ -1507,7 +1507,7 @@ CLASS_GRAPH            = YES
 # indirect implementation dependencies (inheritance, containment, and
 # class references variables) of the class with other documented classes.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
 # will generate a graph for groups, showing the direct groups dependencies
@@ -1530,14 +1530,14 @@ TEMPLATE_RELATIONS     = YES
 # file showing the direct and indirect include dependencies of the file with
 # other documented files.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
 # HAVE_DOT tags are set to YES then doxygen will generate a graph for each
 # documented header file showing the documented files that directly or
 # indirectly include this file.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH and HAVE_DOT options are set to YES then
 # doxygen will generate a call dependency graph for every global function
diff --git a/lldb/docs/lldb-gdb-remote.txt b/lldb/docs/lldb-gdb-remote.txt
index 570e70f9e54a9..b426978a94907 100644
--- a/lldb/docs/lldb-gdb-remote.txt
+++ b/lldb/docs/lldb-gdb-remote.txt
@@ -1636,6 +1636,24 @@ for this region.
 //                                  Example:
 //                                  thread-pcs:dec14,2cf872b0,2cf8681c,2d02d68c,2cf716a8;
 //
+//  "addressing_bits" unsigned optional  Specifies how many bits in addresses 
+//                                       are significant for addressing, base 
+//                                       10.  If bits 38..0 in a 64-bit 
+//                                       pointer are significant for 
+//                                       addressing, then the value is 39.  
+//                                       This is needed on e.g. AArch64
+//                                       v8.3 ABIs that use pointer 
+//                                       authentication in the high bits.
+//                                       This value is normally sent in the
+//                                       qHostInfo packet response, and if the
+//                                       value cannot change during the process
+//                                       lifetime, it does not need to be 
+//                                       duplicated here in the stop packet.
+//                                       For a firmware environment with early
+//                                       start code that may be changing the
+//                                       page table setup, a dynamically set
+//                                       value may be needed.
+//
 // BEST PRACTICES:
 //  Since register values can be supplied with this packet, it is often useful
 //  to return the PC, SP, FP, LR (if any), and FLAGS registers so that separate
diff --git a/lldb/examples/python/scripted_process/scripted_process.py b/lldb/examples/python/scripted_process/scripted_process.py
index e4d25214da766..a0880fbb6268f 100644
--- a/lldb/examples/python/scripted_process/scripted_process.py
+++ b/lldb/examples/python/scripted_process/scripted_process.py
@@ -161,30 +161,25 @@ def attach(self, attach_info):
         """
         return lldb.SBError()
 
-    def resume(self):
+    def resume(self, should_stop=True):
         """ Simulate the scripted process resume.
 
-        Returns:
-            lldb.SBError: An `lldb.SBError` with error code 0.
-        """
-        return lldb.SBError()
-
-    @abstractmethod
-    def should_stop(self):
-        """ Check if the scripted process plugin should produce the stop event.
-
-        Returns:
-            bool: True if scripted process should broadcast a stop event.
-                  False otherwise.
-        """
-        pass
-
-    def stop(self):
-        """ Trigger the scripted process stop.
+        Args:
+            should_stop (bool): If True, resume will also force the process
+            state to stopped after running it.
 
         Returns:
             lldb.SBError: An `lldb.SBError` with error code 0.
         """
+        process = self.target.GetProcess()
+        if not process:
+            error = lldb.SBError()
+            error.SetErrorString("Invalid process.")
+            return error
+
+        process.ForceScriptedState(lldb.eStateRunning);
+        if (should_stop):
+            process.ForceScriptedState(lldb.eStateStopped);
         return lldb.SBError()
 
     @abstractmethod
@@ -214,6 +209,23 @@ def get_process_metadata(self):
         """
         return self.metadata
 
+    def create_breakpoint(self, addr, error):
+        """ Create a breakpoint in the scripted process from an address.
+            This is mainly used with interactive scripted process debugging.
+
+        Args:
+            addr (int): Address at which the breakpoint should be set.
+            error (lldb.SBError): Error object.
+
+        Returns:
+            SBBreakpoint: A valid breakpoint object that was created a the specified
+                          address. None if the breakpoint creation failed.
+        """
+        error.SetErrorString("%s doesn't support creating breakpoints."
+                             % self.__class__.__name__)
+        return False
+
+
 class ScriptedThread(metaclass=ABCMeta):
 
     """
diff --git a/lldb/examples/python/scripted_step.py b/lldb/examples/python/scripted_step.py
index 011e24d70cb26..6e53072e9735c 100644
--- a/lldb/examples/python/scripted_step.py
+++ b/lldb/examples/python/scripted_step.py
@@ -77,6 +77,10 @@
 #    is stale.  In this case, if the step_out plan that the FinishPrintAndContinue
 #    plan is driving is stale, so is ours, and it is time to do our printing.
 #
+# 4) If you implement the "stop_description(SBStream stream)" method in your
+#    python class, then that will show up as the "plan completed" reason when
+#    your thread plan is complete.
+#
 # Both examples show stepping through an address range for 20 bytes from the
 # current PC.  The first one does it by single stepping and checking a condition.
 # It doesn't, however handle the case where you step into another frame while
@@ -122,6 +126,8 @@ def should_stop(self, event):
     def should_step(self):
         return True
 
+    def stop_description(self, stream):
+        stream.Print("Simple step completed")
 
 class StepWithPlan:
 
@@ -146,6 +152,9 @@ def should_stop(self, event):
     def should_step(self):
         return False
 
+    def stop_description(self, stream):
+        self.step_thread_plan.GetDescription(stream, lldb.eDescriptionLevelBrief)
+
 # Here's another example which does "step over" through the current function,
 # and when it stops at each line, it checks some condition (in this example the
 # value of a variable) and stops if that condition is true.
@@ -205,6 +214,9 @@ def should_stop(self, event):
     def should_step(self):
         return True
 
+    def stop_description(self, stream):
+        stream.Print(f"Stepped until a == 20")
+
 # Here's an example that steps out of the current frame, gathers some information
 # and then continues.  The information in this case is rax.  Currently the thread
 # plans are not a safe place to call lldb command-line commands, so the information
@@ -242,3 +254,6 @@ def do_print(self):
             print("RAX on exit: ", rax_value.GetValue())
         else:
             print("Couldn't get rax value:", rax_value.GetError().GetCString())
+
+    def stop_description(self, stream):
+        self.step_out_thread_plan.GetDescription(stream, lldb.eDescriptionLevelBrief)
diff --git a/lldb/examples/python/symbolication.py b/lldb/examples/python/symbolication.py
index 9784c28a44766..3a42f340ea578 100755
--- a/lldb/examples/python/symbolication.py
+++ b/lldb/examples/python/symbolication.py
@@ -401,6 +401,11 @@ def add_module(self, target):
                     with open(tf.name, 'w') as f:
                         f.write(json.dumps(data, indent=4))
                     self.module = target.AddModule(tf.name, None, uuid_str)
+                    if self.module:
+                        # If we were able to add the module with inlined
+                        # symbols, we should mark it as available so load_module
+                        # does not exit early.
+                        self.unavailable = False
             if not self.module and not self.unavailable:
                 return 'error: unable to get module for (%s) "%s"' % (
                     self.arch, self.get_resolved_path())
diff --git a/lldb/include/lldb/API/SBAttachInfo.h b/lldb/include/lldb/API/SBAttachInfo.h
index a4dbce29855f2..ea1145e625856 100644
--- a/lldb/include/lldb/API/SBAttachInfo.h
+++ b/lldb/include/lldb/API/SBAttachInfo.h
@@ -168,6 +168,25 @@ class LLDB_API SBAttachInfo {
   /// allows a different listener to be used to listen for process events.
   void SetListener(SBListener &listener);
 
+  /// Get the shadow listener that receive public process events,
+  /// additionally to the default process event listener.
+  ///
+  /// If no listener has been set via a call to
+  /// SBLaunchInfo::SetShadowListener(), then an invalid SBListener will
+  /// be returned (SBListener::IsValid() will return false). If a listener
+  /// has been set, then the valid listener object will be returned.
+  SBListener GetShadowListener();
+
+  /// Set the shadow listener that will receive public process events,
+  /// additionally to the default process event listener.
+  ///
+  /// By default a process have no shadow event listener.
+  /// Calling this function allows public process events to be broadcasted to an
+  /// additional listener on top of the default process event listener.
+  /// If the `listener` argument is invalid (SBListener::IsValid() will
+  /// return false), this will clear the shadow listener.
+  void SetShadowListener(SBListener &listener);
+
   const char *GetScriptedProcessClassName() const;
 
   void SetScriptedProcessClassName(const char *class_name);
diff --git a/lldb/include/lldb/API/SBBreakpoint.h b/lldb/include/lldb/API/SBBreakpoint.h
index bca6566ce325c..247743929bef1 100644
--- a/lldb/include/lldb/API/SBBreakpoint.h
+++ b/lldb/include/lldb/API/SBBreakpoint.h
@@ -13,6 +13,10 @@
 
 class SBBreakpointListImpl;
 
+namespace lldb_private {
+class ScriptInterpreter;
+}
+
 namespace lldb {
 
 class LLDB_API SBBreakpoint {
@@ -155,6 +159,8 @@ class LLDB_API SBBreakpoint {
   friend class SBBreakpointName;
   friend class SBTarget;
 
+  friend class lldb_private::ScriptInterpreter;
+
   lldb::BreakpointSP GetSP() const;
 
   lldb::BreakpointWP m_opaque_wp;
diff --git a/lldb/include/lldb/API/SBDefines.h b/lldb/include/lldb/API/SBDefines.h
index 4f04cd1f5057d..15f808b29fa8d 100644
--- a/lldb/include/lldb/API/SBDefines.h
+++ b/lldb/include/lldb/API/SBDefines.h
@@ -30,7 +30,7 @@
 // Don't add the deprecated attribute when generating the bindings or when
 // building for anything older than C++14 which is the first version that
 // supports the attribute.
-#if defined(SWIG) or _cplusplus < 201402L
+#if defined(SWIG) || _cplusplus < 201402L
 #undef LLDB_DEPRECATED
 #define LLDB_DEPRECATED(MSG, FIX)
 #endif
diff --git a/lldb/include/lldb/API/SBError.h b/lldb/include/lldb/API/SBError.h
index 16300bd93f69d..f6d5b748c56db 100644
--- a/lldb/include/lldb/API/SBError.h
+++ b/lldb/include/lldb/API/SBError.h
@@ -23,6 +23,8 @@ class LLDB_API SBError {
 
   SBError(const lldb::SBError &rhs);
 
+  SBError(const char *message);
+
 #ifndef SWIG
   SBError(const lldb_private::Status &error);
 #endif
diff --git a/lldb/include/lldb/API/SBFile.h b/lldb/include/lldb/API/SBFile.h
index d8b348b25c815..ebdc5607b7942 100644
--- a/lldb/include/lldb/API/SBFile.h
+++ b/lldb/include/lldb/API/SBFile.h
@@ -11,6 +11,8 @@
 
 #include "lldb/API/SBDefines.h"
 
+#include <cstdio>
+
 namespace lldb {
 
 class LLDB_API SBFile {
diff --git a/lldb/include/lldb/API/SBLaunchInfo.h b/lldb/include/lldb/API/SBLaunchInfo.h
index 8c625e1b9068e..06e72efc30f9f 100644
--- a/lldb/include/lldb/API/SBLaunchInfo.h
+++ b/lldb/include/lldb/API/SBLaunchInfo.h
@@ -92,6 +92,25 @@ class LLDB_API SBLaunchInfo {
   /// allows a different listener to be used to listen for process events.
   void SetListener(SBListener &listener);
 
+  /// Get the shadow listener that receive public process events,
+  /// additionally to the default process event listener.
+  ///
+  /// If no listener has been set via a call to
+  /// SBLaunchInfo::SetShadowListener(), then an invalid SBListener will
+  /// be returned (SBListener::IsValid() will return false). If a listener
+  /// has been set, then the valid listener object will be returned.
+  SBListener GetShadowListener();
+
+  /// Set the shadow listener that will receive public process events,
+  /// additionally to the default process event listener.
+  ///
+  /// By default a process have no shadow event listener.
+  /// Calling this function allows public process events to be broadcasted to an
+  /// additional listener on top of the default process event listener.
+  /// If the `listener` argument is invalid (SBListener::IsValid() will
+  /// return false), this will clear the shadow listener.
+  void SetShadowListener(SBListener &listener);
+
   uint32_t GetNumArguments();
 
   const char *GetArgumentAtIndex(uint32_t idx);
diff --git a/lldb/include/lldb/API/SBProcess.h b/lldb/include/lldb/API/SBProcess.h
index b7eb2036dbe47..d1e15cc5c1f71 100644
--- a/lldb/include/lldb/API/SBProcess.h
+++ b/lldb/include/lldb/API/SBProcess.h
@@ -186,6 +186,14 @@ class LLDB_API SBProcess {
   ///   The stop event corresponding to stop ID.
   lldb::SBEvent GetStopEventForStopID(uint32_t stop_id);
 
+  /// If the process is a scripted process, changes its state to the new state.
+  /// No-op otherwise.
+  ///
+  /// \param [in] new_state
+  ///   The new state that the scripted process should be set to.
+  ///
+  void ForceScriptedState(StateType new_state);
+
   size_t ReadMemory(addr_t addr, void *buf, size_t size, lldb::SBError &error);
 
   size_t WriteMemory(addr_t addr, const void *buf, size_t size,
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index 1d9d691bc2b06..f05d8d4e6b231 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -269,7 +269,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   const FormatEntity::Entry *GetFrameFormatUnique() const;
 
-  uint32_t GetStopDisassemblyMaxSize() const;
+  uint64_t GetStopDisassemblyMaxSize() const;
 
   const FormatEntity::Entry *GetThreadFormat() const;
 
@@ -283,7 +283,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   bool SetREPLLanguage(lldb::LanguageType repl_lang);
 
-  uint32_t GetTerminalWidth() const;
+  uint64_t GetTerminalWidth() const;
 
   bool SetTerminalWidth(uint32_t term_width);
 
@@ -293,9 +293,12 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
   void SetPrompt(const char *) = delete;
 
   bool GetUseExternalEditor() const;
-
   bool SetUseExternalEditor(bool use_external_editor_p);
 
+  llvm::StringRef GetExternalEditor() const;
+
+  bool SetExternalEditor(llvm::StringRef editor);
+
   bool GetUseColor() const;
 
   bool SetUseColor(bool use_color);
@@ -326,11 +329,11 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   llvm::StringRef GetStopShowColumnAnsiSuffix() const;
 
-  uint32_t GetStopSourceLineCount(bool before) const;
+  uint64_t GetStopSourceLineCount(bool before) const;
 
   StopDisassemblyType GetStopDisassemblyDisplay() const;
 
-  uint32_t GetDisassemblyLineCount() const;
+  uint64_t GetDisassemblyLineCount() const;
 
   llvm::StringRef GetStopShowLineMarkerAnsiPrefix() const;
 
@@ -346,7 +349,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   bool SetPrintDecls(bool b);
 
-  uint32_t GetTabSize() const;
+  uint64_t GetTabSize() const;
 
   bool SetTabSize(uint32_t tab_size);
 
@@ -497,6 +500,19 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
   SetDestroyCallback(lldb_private::DebuggerDestroyCallback destroy_callback,
                      void *baton);
 
+  /// Manually start the global event handler thread. It is useful to plugins
+  /// that directly use the \a lldb_private namespace and want to use the
+  /// debugger's default event handler thread instead of defining their own.
+  bool StartEventHandlerThread();
+
+  /// Manually stop the debugger's default event handler.
+  void StopEventHandlerThread();
+
+  /// Force flushing the process's pending stdout and stderr to the debugger's
+  /// asynchronous stdout and stderr streams.
+  void FlushProcessOutput(Process &process, bool flush_stdout,
+                          bool flush_stderr);
+
 protected:
   friend class CommandInterpreter;
   friend class REPL;
@@ -545,10 +561,6 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   void PrintProgress(const ProgressEventData &data);
 
-  bool StartEventHandlerThread();
-
-  void StopEventHandlerThread();
-
   void PushIOHandler(const lldb::IOHandlerSP &reader_sp,
                      bool cancel_top_handler = true);
 
@@ -584,8 +596,6 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   // Ensures two threads don't attempt to flush process output in parallel.
   std::mutex m_output_flush_mutex;
-  void FlushProcessOutput(Process &process, bool flush_stdout,
-                          bool flush_stderr);
 
   SourceManager::SourceFileCache &GetSourceFileCache() {
     return m_source_file_cache;
diff --git a/lldb/include/lldb/Core/SearchFilter.h b/lldb/include/lldb/Core/SearchFilter.h
index 491e3ddc5985f..7645499e948d0 100644
--- a/lldb/include/lldb/Core/SearchFilter.h
+++ b/lldb/include/lldb/Core/SearchFilter.h
@@ -9,7 +9,7 @@
 #ifndef LLDB_CORE_SEARCHFILTER_H
 #define LLDB_CORE_SEARCHFILTER_H
 
-#include "lldb/Core/FileSpecList.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/StructuredData.h"
 
 #include "lldb/Utility/FileSpec.h"
diff --git a/lldb/include/lldb/Core/UserSettingsController.h b/lldb/include/lldb/Core/UserSettingsController.h
index 8afef06359738..f320057ef120d 100644
--- a/lldb/include/lldb/Core/UserSettingsController.h
+++ b/lldb/include/lldb/Core/UserSettingsController.h
@@ -9,6 +9,7 @@
 #ifndef LLDB_CORE_USERSETTINGSCONTROLLER_H
 #define LLDB_CORE_USERSETTINGSCONTROLLER_H
 
+#include "lldb/Interpreter/OptionValueProperties.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private-enumerations.h"
@@ -47,7 +48,6 @@ class Properties {
 
   virtual lldb::OptionValueSP GetPropertyValue(const ExecutionContext *exe_ctx,
                                                llvm::StringRef property_path,
-                                               bool will_modify,
                                                Status &error) const;
 
   virtual Status SetPropertyValue(const ExecutionContext *exe_ctx,
@@ -83,6 +83,27 @@ class Properties {
 
   static bool IsSettingExperimental(llvm::StringRef setting);
 
+  template <typename T>
+  T GetPropertyAtIndexAs(uint32_t idx, T default_value,
+                         const ExecutionContext *exe_ctx = nullptr) const {
+    return m_collection_sp->GetPropertyAtIndexAs<T>(idx, exe_ctx)
+        .value_or(default_value);
+  }
+
+  template <typename T, typename U = typename std::remove_pointer<T>::type,
+            std::enable_if_t<std::is_pointer_v<T>, bool> = true>
+  const U *
+  GetPropertyAtIndexAs(uint32_t idx,
+                       const ExecutionContext *exe_ctx = nullptr) const {
+    return m_collection_sp->GetPropertyAtIndexAs<T>(idx, exe_ctx);
+  }
+
+  template <typename T>
+  bool SetPropertyAtIndex(uint32_t idx, T t,
+                          const ExecutionContext *exe_ctx = nullptr) const {
+    return m_collection_sp->SetPropertyAtIndex<T>(idx, t, exe_ctx);
+  }
+
 protected:
   lldb::OptionValuePropertiesSP m_collection_sp;
 };
diff --git a/lldb/include/lldb/DataFormatters/FormattersHelpers.h b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
index e9af6656e3d7d..a2e8521d96651 100644
--- a/lldb/include/lldb/DataFormatters/FormattersHelpers.h
+++ b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
@@ -58,8 +58,6 @@ size_t ExtractIndexFromString(const char *item_name);
 
 Address GetArrayAddressOrPointerValue(ValueObject &valobj);
 
-lldb::ValueObjectSP GetValueOfLibCXXCompressedPair(ValueObject &pair);
-
 time_t GetOSXEpoch();
 
 struct InferiorSizedWord {
diff --git a/lldb/include/lldb/Expression/REPL.h b/lldb/include/lldb/Expression/REPL.h
index a0df53eb9b4c1..edfe7c0b341da 100644
--- a/lldb/include/lldb/Expression/REPL.h
+++ b/lldb/include/lldb/Expression/REPL.h
@@ -107,6 +107,24 @@ class REPL : public IOHandlerDelegate {
                          CompletionRequest &request) override;
 
 protected:
+  /// Method that can be optionally overriden by subclasses to get notified
+  /// whenever an expression has been evaluated. The params of this method
+  /// include the inputs and outputs of the expression evaluation.
+  ///
+  /// Note: meta commands that start with : are not covered by this method.
+  ///
+  /// \return
+  ///   An \a Error object that, if it is a failure, aborts the regular
+  ///   REPL expression result handling.
+  virtual llvm::Error
+  OnExpressionEvaluated(const ExecutionContext &exe_ctx, llvm::StringRef code,
+                        const EvaluateExpressionOptions &expr_options,
+                        lldb::ExpressionResults execution_results,
+                        const lldb::ValueObjectSP &result_valobj_sp,
+                        const Status &error) {
+    return llvm::Error::success();
+  }
+
   static int CalculateActualIndentation(const StringList &lines);
 
   // Subclasses should override these functions to implement a functional REPL.
diff --git a/lldb/include/lldb/Host/Host.h b/lldb/include/lldb/Host/Host.h
index 4fc2bd128b025..30549cd789149 100644
--- a/lldb/include/lldb/Host/Host.h
+++ b/lldb/include/lldb/Host/Host.h
@@ -236,8 +236,9 @@ class Host {
                                 bool run_in_shell = true,
                                 bool hide_stderr = false);
 
-  static bool OpenFileInExternalEditor(const FileSpec &file_spec,
-                                       uint32_t line_no);
+  static llvm::Error OpenFileInExternalEditor(llvm::StringRef editor,
+                                              const FileSpec &file_spec,
+                                              uint32_t line_no);
 
   /// Check if we're running in an interactive graphical session.
   ///
diff --git a/lldb/include/lldb/Host/HostInfoBase.h b/lldb/include/lldb/Host/HostInfoBase.h
index 42f71d91f9bf9..6c86c71e552dc 100644
--- a/lldb/include/lldb/Host/HostInfoBase.h
+++ b/lldb/include/lldb/Host/HostInfoBase.h
@@ -121,6 +121,14 @@ class HostInfoBase {
     return {};
   }
 
+  /// Returns the distribution id of the host
+  ///
+  /// This will be something like "ubuntu", "fedora", etc. on Linux.
+  ///
+  /// \return Returns either std::nullopt or a reference to a const std::string
+  /// containing the distribution id
+  static llvm::StringRef GetDistributionId() { return llvm::StringRef(); }
+
 protected:
   static bool ComputeSharedLibraryDirectory(FileSpec &file_spec);
   static bool ComputeSupportExeDirectory(FileSpec &file_spec);
diff --git a/lldb/include/lldb/Host/ProcessLaunchInfo.h b/lldb/include/lldb/Host/ProcessLaunchInfo.h
index b9beea4d11b8c..891f46f650c8c 100644
--- a/lldb/include/lldb/Host/ProcessLaunchInfo.h
+++ b/lldb/include/lldb/Host/ProcessLaunchInfo.h
@@ -120,19 +120,6 @@ class ProcessLaunchInfo : public ProcessInfo {
 
   PseudoTerminal &GetPTY() { return *m_pty; }
 
-  // Get and set the actual listener that will be used for the process events
-  lldb::ListenerSP GetListener() const { return m_listener_sp; }
-
-  void SetListener(const lldb::ListenerSP &listener_sp) {
-    m_listener_sp = listener_sp;
-  }
-
-  lldb::ListenerSP GetHijackListener() const { return m_hijack_listener_sp; }
-
-  void SetHijackListener(const lldb::ListenerSP &listener_sp) {
-    m_hijack_listener_sp = listener_sp;
-  }
-
   void SetLaunchEventData(const char *data) { m_event_data.assign(data); }
 
   const char *GetLaunchEventData() const { return m_event_data.c_str(); }
@@ -154,8 +141,6 @@ class ProcessLaunchInfo : public ProcessInfo {
   Host::MonitorChildProcessCallback m_monitor_callback;
   std::string m_event_data; // A string passed to the plugin launch, having no
                             // meaning to the upper levels of lldb.
-  lldb::ListenerSP m_listener_sp;
-  lldb::ListenerSP m_hijack_listener_sp;
 };
 }
 
diff --git a/lldb/include/lldb/Interpreter/CommandCompletions.h b/lldb/include/lldb/Interpreter/CommandCompletions.h
index c13bc4997ff3e..8578782e9fdd7 100644
--- a/lldb/include/lldb/Interpreter/CommandCompletions.h
+++ b/lldb/include/lldb/Interpreter/CommandCompletions.h
@@ -11,10 +11,10 @@
 
 #include <set>
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/SearchFilter.h"
 #include "lldb/Interpreter/Options.h"
 #include "lldb/Utility/CompletionRequest.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/lldb-private.h"
 
diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h
index e428614fbedb1..f6a9a4d90bc1d 100644
--- a/lldb/include/lldb/Interpreter/CommandInterpreter.h
+++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h
@@ -324,8 +324,9 @@ class CommandInterpreter : public Broadcaster,
                          lldb::CommandObjectSP &command_obj_sp,
                          llvm::StringRef args_string = llvm::StringRef());
 
-  // Remove a command if it is removable (python or regex command)
-  bool RemoveCommand(llvm::StringRef cmd);
+  /// Remove a command if it is removable (python or regex command). If \b force
+  /// is provided, the command is removed regardless of its removable status.
+  bool RemoveCommand(llvm::StringRef cmd, bool force = false);
 
   bool RemoveAlias(llvm::StringRef alias_name);
 
diff --git a/lldb/include/lldb/Interpreter/OptionGroupWatchpoint.h b/lldb/include/lldb/Interpreter/OptionGroupWatchpoint.h
index 33818043cf63f..201ab1d134f25 100644
--- a/lldb/include/lldb/Interpreter/OptionGroupWatchpoint.h
+++ b/lldb/include/lldb/Interpreter/OptionGroupWatchpoint.h
@@ -43,6 +43,7 @@ class OptionGroupWatchpoint : public OptionGroup {
   WatchType watch_type;
   uint32_t watch_size;
   bool watch_type_specified;
+  lldb::LanguageType language_type;
 
 private:
   OptionGroupWatchpoint(const OptionGroupWatchpoint &) = delete;
diff --git a/lldb/include/lldb/Interpreter/OptionValue.h b/lldb/include/lldb/Interpreter/OptionValue.h
index 3c842d1f5ce79..8735106c3d142 100644
--- a/lldb/include/lldb/Interpreter/OptionValue.h
+++ b/lldb/include/lldb/Interpreter/OptionValue.h
@@ -13,6 +13,8 @@
 #include "lldb/Utility/Cloneable.h"
 #include "lldb/Utility/CompletionRequest.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-private-enumerations.h"
@@ -108,7 +110,6 @@ class OptionValue {
   // Subclasses can override these functions
   virtual lldb::OptionValueSP GetSubValue(const ExecutionContext *exe_ctx,
                                           llvm::StringRef name,
-                                          bool will_modify,
                                           Status &error) const {
     error.SetErrorStringWithFormat("'%s' is not a value subvalue", name.str().c_str());
     return lldb::OptionValueSP();
@@ -182,10 +183,6 @@ class OptionValue {
   CreateValueFromCStringForTypeMask(const char *value_cstr, uint32_t type_mask,
                                     Status &error);
 
-  // Get this value as a uint64_t value if it is encoded as a boolean, uint64_t
-  // or int64_t. Other types will cause "fail_value" to be returned
-  uint64_t GetUInt64Value(uint64_t fail_value, bool *success_ptr);
-
   OptionValueArch *GetAsArch();
 
   const OptionValueArch *GetAsArch() const;
@@ -262,31 +259,31 @@ class OptionValue {
 
   const OptionValueFormatEntity *GetAsFormatEntity() const;
 
-  bool GetBooleanValue(bool fail_value = false) const;
+  std::optional<bool> GetBooleanValue() const;
 
   bool SetBooleanValue(bool new_value);
 
-  char GetCharValue(char fail_value) const;
+  std::optional<char> GetCharValue() const;
 
   char SetCharValue(char new_value);
 
-  int64_t GetEnumerationValue(int64_t fail_value = -1) const;
+  std::optional<int64_t> GetEnumerationValue() const;
 
   bool SetEnumerationValue(int64_t value);
 
-  FileSpec GetFileSpecValue() const;
+  std::optional<FileSpec> GetFileSpecValue() const;
+
+  bool SetFileSpecValue(FileSpec file_spec);
 
-  bool SetFileSpecValue(const FileSpec &file_spec);
+  bool AppendFileSpecValue(FileSpec file_spec);
 
-  FileSpecList GetFileSpecListValue() const;
+  std::optional<FileSpecList> GetFileSpecListValue() const;
 
-  lldb::Format
-  GetFormatValue(lldb::Format fail_value = lldb::eFormatDefault) const;
+  std::optional<lldb::Format> GetFormatValue() const;
 
   bool SetFormatValue(lldb::Format new_value);
 
-  lldb::LanguageType GetLanguageValue(
-      lldb::LanguageType fail_value = lldb::eLanguageTypeUnknown) const;
+  std::optional<lldb::LanguageType> GetLanguageValue() const;
 
   bool SetLanguageValue(lldb::LanguageType new_language);
 
@@ -294,16 +291,15 @@ class OptionValue {
 
   const RegularExpression *GetRegexValue() const;
 
-  int64_t GetSInt64Value(int64_t fail_value = 0) const;
+  std::optional<int64_t> GetSInt64Value() const;
 
   bool SetSInt64Value(int64_t new_value);
 
-  llvm::StringRef GetStringValue(llvm::StringRef fail_value) const;
-  llvm::StringRef GetStringValue() const { return GetStringValue(llvm::StringRef()); }
+  std::optional<llvm::StringRef> GetStringValue() const;
 
   bool SetStringValue(llvm::StringRef new_value);
 
-  uint64_t GetUInt64Value(uint64_t fail_value = 0) const;
+  std::optional<uint64_t> GetUInt64Value() const;
 
   bool SetUInt64Value(uint64_t new_value);
 
@@ -330,6 +326,57 @@ class OptionValue {
       m_callback();
   }
 
+  template <typename T, std::enable_if_t<!std::is_pointer_v<T>, bool> = true>
+  std::optional<T> GetValueAs() const {
+    if constexpr (std::is_same_v<T, uint64_t>)
+      return GetUInt64Value();
+    if constexpr (std::is_same_v<T, int64_t>)
+      return GetSInt64Value();
+    if constexpr (std::is_same_v<T, bool>)
+      return GetBooleanValue();
+    if constexpr (std::is_same_v<T, char>)
+      return GetCharValue();
+    if constexpr (std::is_same_v<T, lldb::Format>)
+      return GetFormatValue();
+    if constexpr (std::is_same_v<T, FileSpec>)
+      return GetFileSpecValue();
+    if constexpr (std::is_same_v<T, FileSpecList>)
+      return GetFileSpecListValue();
+    if constexpr (std::is_same_v<T, lldb::LanguageType>)
+      return GetLanguageValue();
+    if constexpr (std::is_same_v<T, llvm::StringRef>)
+      return GetStringValue();
+    if constexpr (std::is_enum_v<T>)
+      if (std::optional<int64_t> value = GetEnumerationValue())
+        return static_cast<T>(*value);
+    return {};
+  }
+
+  template <typename T,
+            typename U = typename std::remove_const<
+                typename std::remove_pointer<T>::type>::type,
+            std::enable_if_t<std::is_pointer_v<T>, bool> = true>
+  T GetValueAs() const {
+    if constexpr (std::is_same_v<U, FormatEntity::Entry>)
+      return GetFormatEntity();
+    if constexpr (std::is_same_v<U, RegularExpression>)
+      return GetRegexValue();
+    return {};
+  }
+
+  bool SetValueAs(bool v) { return SetBooleanValue(v); }
+
+  bool SetValueAs(llvm::StringRef v) { return SetStringValue(v); }
+
+  bool SetValueAs(lldb::LanguageType v) { return SetLanguageValue(v); }
+
+  bool SetValueAs(FileSpec v) { return SetFileSpecValue(v); }
+
+  template <typename T, std::enable_if_t<std::is_enum_v<T>, bool> = true>
+  bool SetValueAs(T t) {
+    return SetEnumerationValue(t);
+  }
+
 protected:
   using TopmostBase = OptionValue;
 
diff --git a/lldb/include/lldb/Interpreter/OptionValueArray.h b/lldb/include/lldb/Interpreter/OptionValueArray.h
index 147e15ef4335d..0e1bae103d41f 100644
--- a/lldb/include/lldb/Interpreter/OptionValueArray.h
+++ b/lldb/include/lldb/Interpreter/OptionValueArray.h
@@ -46,7 +46,7 @@ class OptionValueArray : public Cloneable<OptionValueArray, OptionValue> {
   bool IsAggregateValue() const override { return true; }
 
   lldb::OptionValueSP GetSubValue(const ExecutionContext *exe_ctx,
-                                  llvm::StringRef name, bool will_modify,
+                                  llvm::StringRef name,
                                   Status &error) const override;
 
   // Subclass specific functions
diff --git a/lldb/include/lldb/Interpreter/OptionValueDictionary.h b/lldb/include/lldb/Interpreter/OptionValueDictionary.h
index 1fdccdeccef50..18ef448237157 100644
--- a/lldb/include/lldb/Interpreter/OptionValueDictionary.h
+++ b/lldb/include/lldb/Interpreter/OptionValueDictionary.h
@@ -9,11 +9,11 @@
 #ifndef LLDB_INTERPRETER_OPTIONVALUEDICTIONARY_H
 #define LLDB_INTERPRETER_OPTIONVALUEDICTIONARY_H
 
-#include <map>
-
 #include "lldb/Interpreter/OptionValue.h"
 #include "lldb/lldb-private-types.h"
 
+#include "llvm/ADT/StringMap.h"
+
 namespace lldb_private {
 
 class OptionValueDictionary
@@ -58,30 +58,28 @@ class OptionValueDictionary
 
   size_t GetNumValues() const { return m_values.size(); }
 
-  lldb::OptionValueSP GetValueForKey(ConstString key) const;
+  lldb::OptionValueSP GetValueForKey(llvm::StringRef key) const;
 
   lldb::OptionValueSP GetSubValue(const ExecutionContext *exe_ctx,
-                                  llvm::StringRef name, bool will_modify,
+                                  llvm::StringRef name,
                                   Status &error) const override;
 
   Status SetSubValue(const ExecutionContext *exe_ctx, VarSetOperationType op,
                      llvm::StringRef name, llvm::StringRef value) override;
 
-  bool SetValueForKey(ConstString key,
-                      const lldb::OptionValueSP &value_sp,
+  bool SetValueForKey(llvm::StringRef key, const lldb::OptionValueSP &value_sp,
                       bool can_replace = true);
 
-  bool DeleteValueForKey(ConstString key);
+  bool DeleteValueForKey(llvm::StringRef key);
 
   size_t GetArgs(Args &args) const;
 
   Status SetArgs(const Args &args, VarSetOperationType op);
 
 protected:
-  typedef std::map<ConstString, lldb::OptionValueSP> collection;
   uint32_t m_type_mask;
   OptionEnumValues m_enum_values;
-  collection m_values;
+  llvm::StringMap<lldb::OptionValueSP> m_values;
   bool m_raw_value_dump;
 };
 
diff --git a/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h b/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
index 29641c3a20879..bda6b5071d599 100644
--- a/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
+++ b/lldb/include/lldb/Interpreter/OptionValueFileSpecList.h
@@ -11,8 +11,8 @@
 
 #include <mutex>
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Interpreter/OptionValue.h"
+#include "lldb/Utility/FileSpecList.h"
 
 namespace lldb_private {
 
diff --git a/lldb/include/lldb/Interpreter/OptionValueProperties.h b/lldb/include/lldb/Interpreter/OptionValueProperties.h
index 814e0a895cc61..004ed55e3b119 100644
--- a/lldb/include/lldb/Interpreter/OptionValueProperties.h
+++ b/lldb/include/lldb/Interpreter/OptionValueProperties.h
@@ -68,8 +68,6 @@ class OptionValueProperties
 
   // Subclass specific functions
 
-  virtual size_t GetNumProperties() const;
-
   // Get the index of a property given its exact name in this property
   // collection, "name" can't be a path to a property path that refers to a
   // property within a property
@@ -78,124 +76,69 @@ class OptionValueProperties
   // Get a property by exact name exists in this property collection, name can
   // not be a path to a property path that refers to a property within a
   // property
-  virtual const Property *GetProperty(const ExecutionContext *exe_ctx,
-                                      bool will_modify,
-                                      ConstString name) const;
-
-  virtual const Property *GetPropertyAtIndex(const ExecutionContext *exe_ctx,
-                                             bool will_modify,
-                                             uint32_t idx) const;
+  virtual const Property *
+  GetProperty(ConstString name,
+              const ExecutionContext *exe_ctx = nullptr) const;
+
+  virtual const Property *
+  GetPropertyAtIndex(uint32_t idx,
+                     const ExecutionContext *exe_ctx = nullptr) const {
+    return ProtectedGetPropertyAtIndex(idx);
+  }
 
   // Property can be be a property path like
   // "target.process.extra-startup-command"
-  virtual const Property *GetPropertyAtPath(const ExecutionContext *exe_ctx,
-                                            bool will_modify,
-    llvm::StringRef property_path) const;
+  virtual const Property *
+  GetPropertyAtPath(const ExecutionContext *exe_ctx,
+
+                    llvm::StringRef property_path) const;
 
   virtual lldb::OptionValueSP
-  GetPropertyValueAtIndex(const ExecutionContext *exe_ctx, bool will_modify,
-                          uint32_t idx) const;
+  GetPropertyValueAtIndex(uint32_t idx, const ExecutionContext *exe_ctx) const;
 
   virtual lldb::OptionValueSP GetValueForKey(const ExecutionContext *exe_ctx,
-                                             ConstString key,
-                                             bool value_will_be_modified) const;
+                                             ConstString key) const;
 
   lldb::OptionValueSP GetSubValue(const ExecutionContext *exe_ctx,
                                   llvm::StringRef name,
-                                  bool value_will_be_modified,
                                   Status &error) const override;
 
   Status SetSubValue(const ExecutionContext *exe_ctx, VarSetOperationType op,
                      llvm::StringRef path, llvm::StringRef value) override;
 
-  OptionValueArch *
-  GetPropertyAtIndexAsOptionValueArch(const ExecutionContext *exe_ctx,
-                                      uint32_t idx) const;
-
-  OptionValueLanguage *
-  GetPropertyAtIndexAsOptionValueLanguage(const ExecutionContext *exe_ctx,
-                                          uint32_t idx) const;
-
-  bool SetPropertyAtIndexAsLanguage(const ExecutionContext *exe_ctx,
-                                    uint32_t idx, lldb::LanguageType lang);
-
-  bool GetPropertyAtIndexAsArgs(const ExecutionContext *exe_ctx, uint32_t idx,
-                                Args &args) const;
-
-  bool SetPropertyAtIndexFromArgs(const ExecutionContext *exe_ctx, uint32_t idx,
-                                  const Args &args);
-
-  bool GetPropertyAtIndexAsBoolean(const ExecutionContext *exe_ctx,
-                                   uint32_t idx, bool fail_value) const;
-
-  bool SetPropertyAtIndexAsBoolean(const ExecutionContext *exe_ctx,
-                                   uint32_t idx, bool new_value);
-
-  OptionValueDictionary *
-  GetPropertyAtIndexAsOptionValueDictionary(const ExecutionContext *exe_ctx,
-                                            uint32_t idx) const;
-
-  int64_t GetPropertyAtIndexAsEnumeration(const ExecutionContext *exe_ctx,
-                                          uint32_t idx,
-                                          int64_t fail_value) const;
+  OptionValueArch *GetPropertyAtIndexAsOptionValueArch(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
-  bool SetPropertyAtIndexAsEnumeration(const ExecutionContext *exe_ctx,
-                                       uint32_t idx, int64_t new_value);
+  OptionValueLanguage *GetPropertyAtIndexAsOptionValueLanguage(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
-  const FormatEntity::Entry *
-  GetPropertyAtIndexAsFormatEntity(const ExecutionContext *exe_ctx,
-                                   uint32_t idx);
+  bool
+  GetPropertyAtIndexAsArgs(uint32_t idx, Args &args,
+                           const ExecutionContext *exe_ctx = nullptr) const;
 
-  const RegularExpression *
-  GetPropertyAtIndexAsOptionValueRegex(const ExecutionContext *exe_ctx,
-                                       uint32_t idx) const;
+  bool SetPropertyAtIndexFromArgs(uint32_t idx, const Args &args,
+                                  const ExecutionContext *exe_ctx = nullptr);
 
-  OptionValueSInt64 *
-  GetPropertyAtIndexAsOptionValueSInt64(const ExecutionContext *exe_ctx,
-                                        uint32_t idx) const;
+  OptionValueDictionary *GetPropertyAtIndexAsOptionValueDictionary(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
-  OptionValueUInt64 *
-  GetPropertyAtIndexAsOptionValueUInt64(const ExecutionContext *exe_ctx,
-                                        uint32_t idx) const;
+  OptionValueSInt64 *GetPropertyAtIndexAsOptionValueSInt64(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
-  int64_t GetPropertyAtIndexAsSInt64(const ExecutionContext *exe_ctx,
-                                     uint32_t idx, int64_t fail_value) const;
+  OptionValueUInt64 *GetPropertyAtIndexAsOptionValueUInt64(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
-  bool SetPropertyAtIndexAsSInt64(const ExecutionContext *exe_ctx, uint32_t idx,
-                                  int64_t new_value);
+  OptionValueString *GetPropertyAtIndexAsOptionValueString(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
-  uint64_t GetPropertyAtIndexAsUInt64(const ExecutionContext *exe_ctx,
-                                      uint32_t idx, uint64_t fail_value) const;
-
-  bool SetPropertyAtIndexAsUInt64(const ExecutionContext *exe_ctx, uint32_t idx,
-                                  uint64_t new_value);
-
-  llvm::StringRef GetPropertyAtIndexAsString(const ExecutionContext *exe_ctx,
-                                         uint32_t idx,
-                                         llvm::StringRef fail_value) const;
-
-  bool SetPropertyAtIndexAsString(const ExecutionContext *exe_ctx, uint32_t idx,
-                                  llvm::StringRef new_value);
-
-  OptionValueString *
-  GetPropertyAtIndexAsOptionValueString(const ExecutionContext *exe_ctx,
-                                        bool will_modify, uint32_t idx) const;
-
-  OptionValueFileSpec *
-  GetPropertyAtIndexAsOptionValueFileSpec(const ExecutionContext *exe_ctx,
-                                          bool will_modify, uint32_t idx) const;
-
-  FileSpec GetPropertyAtIndexAsFileSpec(const ExecutionContext *exe_ctx,
-                                        uint32_t idx) const;
-
-  bool SetPropertyAtIndexAsFileSpec(const ExecutionContext *exe_ctx,
-                                    uint32_t idx, const FileSpec &file_spec);
+  OptionValueFileSpec *GetPropertyAtIndexAsOptionValueFileSpec(
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
   OptionValuePathMappings *GetPropertyAtIndexAsOptionValuePathMappings(
-      const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const;
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
   OptionValueFileSpecList *GetPropertyAtIndexAsOptionValueFileSpecList(
-      const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const;
+      uint32_t idx, const ExecutionContext *exe_ctx = nullptr) const;
 
   void AppendProperty(ConstString name, llvm::StringRef desc, bool is_global,
                       const lldb::OptionValueSP &value_sp);
@@ -206,12 +149,39 @@ class OptionValueProperties
   void SetValueChangedCallback(uint32_t property_idx,
                                std::function<void()> callback);
 
+  template <typename T>
+  auto GetPropertyAtIndexAs(uint32_t idx,
+                            const ExecutionContext *exe_ctx = nullptr) const {
+    if (const Property *property = GetPropertyAtIndex(idx, exe_ctx)) {
+      if (OptionValue *value = property->GetValue().get())
+        return value->GetValueAs<T>();
+    }
+    if constexpr (std::is_pointer_v<T>)
+      return T{nullptr};
+    else
+      return std::optional<T>{std::nullopt};
+  }
+
+  template <typename T>
+  bool SetPropertyAtIndex(uint32_t idx, T t,
+                          const ExecutionContext *exe_ctx = nullptr) const {
+    if (const Property *property = GetPropertyAtIndex(idx, exe_ctx)) {
+      if (OptionValue *value = property->GetValue().get()) {
+        value->SetValueAs(t);
+        return true;
+      }
+    }
+    return false;
+  }
+
 protected:
   Property *ProtectedGetPropertyAtIndex(uint32_t idx) {
+    assert(idx < m_properties.size() && "invalid property index");
     return ((idx < m_properties.size()) ? &m_properties[idx] : nullptr);
   }
 
   const Property *ProtectedGetPropertyAtIndex(uint32_t idx) const {
+    assert(idx < m_properties.size() && "invalid property index");
     return ((idx < m_properties.size()) ? &m_properties[idx] : nullptr);
   }
 
diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
index 557406b43a531..0095905fd320c 100644
--- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h
+++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
@@ -10,6 +10,7 @@
 #define LLDB_INTERPRETER_SCRIPTINTERPRETER_H
 
 #include "lldb/API/SBAttachInfo.h"
+#include "lldb/API/SBBreakpoint.h"
 #include "lldb/API/SBData.h"
 #include "lldb/API/SBError.h"
 #include "lldb/API/SBLaunchInfo.h"
@@ -313,6 +314,14 @@ class ScriptInterpreter : public PluginInterface {
     return lldb::eStateStepping;
   }
 
+  virtual bool
+  ScriptedThreadPlanGetStopDescription(StructuredData::ObjectSP implementor_sp,
+                                       lldb_private::Stream *stream,
+                                       bool &script_error) {
+    script_error = true;
+    return false;
+  }
+
   virtual StructuredData::GenericSP
   CreateScriptedBreakpointResolver(const char *class_name,
                                    const StructuredDataImpl &args_data,
@@ -587,6 +596,9 @@ class ScriptInterpreter : public PluginInterface {
 
   Status GetStatusFromSBError(const lldb::SBError &error) const;
 
+  lldb::BreakpointSP
+  GetOpaqueTypeFromSBBreakpoint(const lldb::SBBreakpoint &breakpoint) const;
+
   lldb::ProcessAttachInfoSP
   GetOpaqueTypeFromSBAttachInfo(const lldb::SBAttachInfo &attach_info) const;
 
diff --git a/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h b/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h
index 895fc5e2fd3d3..210b1081cddcd 100644
--- a/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h
+++ b/lldb/include/lldb/Interpreter/ScriptedProcessInterface.h
@@ -38,10 +38,6 @@ class ScriptedProcessInterface : virtual public ScriptedInterface {
 
   virtual Status Resume() { return Status("ScriptedProcess did not resume"); }
 
-  virtual bool ShouldStop() { return true; }
-
-  virtual Status Stop() { return Status("ScriptedProcess did not stop"); }
-
   virtual std::optional<MemoryRegionInfo>
   GetMemoryRegionContainingAddress(lldb::addr_t address, Status &error) {
     error.SetErrorString("ScriptedProcess have no memory region.");
@@ -50,6 +46,11 @@ class ScriptedProcessInterface : virtual public ScriptedInterface {
 
   virtual StructuredData::DictionarySP GetThreadsInfo() { return {}; }
 
+  virtual bool CreateBreakpoint(lldb::addr_t addr, Status &error) {
+    error.SetErrorString("ScriptedProcess don't support creating breakpoints.");
+    return {};
+  }
+
   virtual lldb::DataExtractorSP
   ReadMemoryAtAddress(lldb::addr_t address, size_t size, Status &error) {
     return {};
diff --git a/lldb/include/lldb/Symbol/CompileUnit.h b/lldb/include/lldb/Symbol/CompileUnit.h
index 44e1d673f1fd1..229ee2d27a703 100644
--- a/lldb/include/lldb/Symbol/CompileUnit.h
+++ b/lldb/include/lldb/Symbol/CompileUnit.h
@@ -9,13 +9,13 @@
 #ifndef LLDB_SYMBOL_COMPILEUNIT_H
 #define LLDB_SYMBOL_COMPILEUNIT_H
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/ModuleChild.h"
 #include "lldb/Core/SourceLocationSpec.h"
 #include "lldb/Symbol/DebugMacros.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/LineTable.h"
 #include "lldb/Symbol/SourceModule.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-enumerations.h"
diff --git a/lldb/include/lldb/Symbol/LocateSymbolFile.h b/lldb/include/lldb/Symbol/LocateSymbolFile.h
index 6f0506e671ce4..0bf6555909676 100644
--- a/lldb/include/lldb/Symbol/LocateSymbolFile.h
+++ b/lldb/include/lldb/Symbol/LocateSymbolFile.h
@@ -11,8 +11,8 @@
 
 #include <cstdint>
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/lldb-forward.h"
 
diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index 8862630495e60..9b006dfe05e7e 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -9,7 +9,6 @@
 #ifndef LLDB_SYMBOL_OBJECTFILE_H
 #define LLDB_SYMBOL_OBJECTFILE_H
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/ModuleChild.h"
 #include "lldb/Core/PluginInterface.h"
 #include "lldb/Symbol/Symtab.h"
@@ -17,6 +16,7 @@
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/UUID.h"
 #include "lldb/lldb-private.h"
 #include "llvm/Support/Threading.h"
diff --git a/lldb/include/lldb/Target/ABI.h b/lldb/include/lldb/Target/ABI.h
index f0753172d3e71..f600e29c7c4b0 100644
--- a/lldb/include/lldb/Target/ABI.h
+++ b/lldb/include/lldb/Target/ABI.h
@@ -147,6 +147,8 @@ class ABI : public PluginInterface {
 
   virtual bool GetPointerReturnRegister(const char *&name) { return false; }
 
+  virtual uint64_t GetStackFrameSize() { return 512 * 1024; }
+
   static lldb::ABISP FindPlugin(lldb::ProcessSP process_sp, const ArchSpec &arch);
 
 protected:
diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index 808226067d5a6..207b4939d0232 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -122,8 +122,6 @@ class ProcessAttachInfo : public ProcessInstanceInfo {
     ProcessInfo::operator=(launch_info);
     SetProcessPluginName(launch_info.GetProcessPluginName());
     SetResumeCount(launch_info.GetResumeCount());
-    SetListener(launch_info.GetListener());
-    SetHijackListener(launch_info.GetHijackListener());
     m_detach_on_error = launch_info.GetDetachOnError();
   }
 
@@ -174,28 +172,13 @@ class ProcessAttachInfo : public ProcessInstanceInfo {
     return false;
   }
 
-  lldb::ListenerSP GetHijackListener() const { return m_hijack_listener_sp; }
-
-  void SetHijackListener(const lldb::ListenerSP &listener_sp) {
-    m_hijack_listener_sp = listener_sp;
-  }
-
   bool GetDetachOnError() const { return m_detach_on_error; }
 
   void SetDetachOnError(bool enable) { m_detach_on_error = enable; }
 
-  // Get and set the actual listener that will be used for the process events
-  lldb::ListenerSP GetListener() const { return m_listener_sp; }
-
-  void SetListener(const lldb::ListenerSP &listener_sp) {
-    m_listener_sp = listener_sp;
-  }
-
   lldb::ListenerSP GetListenerForProcess(Debugger &debugger);
 
 protected:
-  lldb::ListenerSP m_listener_sp;
-  lldb::ListenerSP m_hijack_listener_sp;
   std::string m_plugin_name;
   uint32_t m_resume_count = 0; // How many times do we resume after launching
   bool m_wait_for_launch = false;
@@ -397,6 +380,10 @@ class Process : public std::enable_shared_from_this<Process>,
     return GetStaticBroadcasterClass();
   }
 
+  void SetShadowListener(lldb::ListenerSP listener_sp) override {
+    Broadcaster::SetShadowListener(listener_sp);
+  }
+
 /// A notification structure that can be used by clients to listen
 /// for changes in a process's lifetime.
 ///
@@ -2565,6 +2552,8 @@ void PruneThreadPlans();
 
   virtual void *GetImplementation() { return nullptr; }
 
+  virtual void ForceScriptedState(lldb::StateType state) {}
+
 protected:
   friend class Trace;
 
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index b557dd26a3acc..d15f0705630ce 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -204,6 +204,12 @@ class TargetProperties : public Properties {
 
   uint64_t GetExprErrorLimit() const;
 
+  uint64_t GetExprAllocAddress() const;
+
+  uint64_t GetExprAllocSize() const;
+
+  uint64_t GetExprAllocAlign() const;
+
   bool GetUseHexImmediates() const;
 
   bool GetUseFastStepping() const;
diff --git a/lldb/include/lldb/Target/ThreadPlanPython.h b/lldb/include/lldb/Target/ThreadPlanPython.h
index f148f88d4c46d..64854d66b8f25 100644
--- a/lldb/include/lldb/Target/ThreadPlanPython.h
+++ b/lldb/include/lldb/Target/ThreadPlanPython.h
@@ -51,6 +51,9 @@ class ThreadPlanPython : public ThreadPlan {
   void DidPush() override;
 
   bool IsPlanStale() override;
+  
+  bool DoWillResume(lldb::StateType resume_state, bool current_plan) override;
+
 
 protected:
   bool DoPlanExplainsStop(Event *event_ptr) override;
@@ -64,6 +67,7 @@ class ThreadPlanPython : public ThreadPlan {
   StructuredDataImpl m_args_data;
   std::string m_error_str;
   StructuredData::ObjectSP m_implementation_sp;
+  StreamString m_stop_description; // Cache the stop description here
   bool m_did_push;
   bool m_stop_others;
 
diff --git a/lldb/include/lldb/Utility/ArchSpec.h b/lldb/include/lldb/Utility/ArchSpec.h
index 2de517d765b2a..a226a3a5a9b71 100644
--- a/lldb/include/lldb/Utility/ArchSpec.h
+++ b/lldb/include/lldb/Utility/ArchSpec.h
@@ -10,7 +10,6 @@
 #define LLDB_UTILITY_ARCHSPEC_H
 
 #include "lldb/Utility/CompletionRequest.h"
-#include "lldb/Utility/ConstString.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private-enumerations.h"
@@ -342,20 +341,6 @@ class ArchSpec {
   /// \return An LLVM arch type.
   llvm::Triple::ArchType GetMachine() const;
 
-  /// Returns the distribution id of the architecture.
-  ///
-  /// This will be something like "ubuntu", "fedora", etc. on Linux.
-  ///
-  /// \return A ConstString ref containing the distribution id,
-  ///         potentially empty.
-  ConstString GetDistributionId() const;
-
-  /// Set the distribution id of the architecture.
-  ///
-  /// This will be something like "ubuntu", "fedora", etc. on Linux. This
-  /// should be the same value returned by HostInfo::GetDistributionId ().
-  void SetDistributionId(const char *distribution_id);
-
   /// Tests if this ArchSpec is valid.
   ///
   /// \return True if the current architecture is valid, false
@@ -555,8 +540,6 @@ class ArchSpec {
   // these are application specific extensions like micromips, mips16 etc.
   uint32_t m_flags = 0;
 
-  ConstString m_distribution_id;
-
   // Called when m_def or m_entry are changed.  Fills in all remaining members
   // with default values.
   void CoreUpdated(bool update_triple);
diff --git a/lldb/include/lldb/Utility/Broadcaster.h b/lldb/include/lldb/Utility/Broadcaster.h
index 9c025a7c7dd88..85e2468a111f5 100644
--- a/lldb/include/lldb/Utility/Broadcaster.h
+++ b/lldb/include/lldb/Utility/Broadcaster.h
@@ -406,6 +406,10 @@ class Broadcaster {
 
   lldb::BroadcasterManagerSP GetManager();
 
+  virtual void SetShadowListener(lldb::ListenerSP listener_sp) {
+    m_broadcaster_sp->m_shadow_listener = listener_sp;
+  }
+
 protected:
   /// BroadcasterImpl contains the actual Broadcaster implementation.  The
   /// Broadcaster makes a BroadcasterImpl which lives as long as it does.  The
@@ -513,6 +517,10 @@ class Broadcaster {
     /// for now this is just for private hijacking.
     std::vector<uint32_t> m_hijacking_masks;
 
+    /// A optional listener that all private events get also broadcasted to,
+    /// on top the hijacked / default listeners.
+    lldb::ListenerSP m_shadow_listener = nullptr;
+
   private:
     BroadcasterImpl(const BroadcasterImpl &) = delete;
     const BroadcasterImpl &operator=(const BroadcasterImpl &) = delete;
diff --git a/lldb/include/lldb/Utility/FileSpec.h b/lldb/include/lldb/Utility/FileSpec.h
index 08fd1017eccd2..919b5e8564583 100644
--- a/lldb/include/lldb/Utility/FileSpec.h
+++ b/lldb/include/lldb/Utility/FileSpec.h
@@ -329,8 +329,8 @@ class FileSpec {
   /// filename has no extension, ConstString(nullptr) is returned. The dot
   /// ('.') character is the first character in the returned string.
   ///
-  /// \return Returns the extension of the file as a ConstString object.
-  ConstString GetFileNameExtension() const;
+  /// \return Returns the extension of the file as a StringRef.
+  llvm::StringRef GetFileNameExtension() const;
 
   /// Return the filename without the extension part
   ///
@@ -408,8 +408,6 @@ class FileSpec {
   ///     A boolean value indicating whether the path was updated.
   bool RemoveLastPathComponent();
 
-  ConstString GetLastPathComponent() const;
-
 protected:
   // Convenience method for setting the file without changing the style.
   void SetFile(llvm::StringRef path);
diff --git a/lldb/include/lldb/Core/FileSpecList.h b/lldb/include/lldb/Utility/FileSpecList.h
similarity index 97%
rename from lldb/include/lldb/Core/FileSpecList.h
rename to lldb/include/lldb/Utility/FileSpecList.h
index a5720d85e7cde..6b88cd2f30913 100644
--- a/lldb/include/lldb/Core/FileSpecList.h
+++ b/lldb/include/lldb/Utility/FileSpecList.h
@@ -8,18 +8,16 @@
 
 #ifndef LLDB_CORE_FILESPECLIST_H
 #define LLDB_CORE_FILESPECLIST_H
-#if defined(__cplusplus)
 
 #include "lldb/Utility/FileSpec.h"
 
-#include <vector>
-
 #include <cstddef>
+#include <vector>
 
 namespace lldb_private {
 class Stream;
 
-/// \class FileSpecList FileSpecList.h "lldb/Core/FileSpecList.h"
+/// \class FileSpecList FileSpecList.h "lldb/Utility/FileSpecList.h"
 /// A file collection class.
 ///
 /// A class that contains a mutable list of FileSpec objects.
@@ -84,7 +82,7 @@ class FileSpecList {
   ///
   /// \param[in] args
   ///     Arguments to create the FileSpec
-  template <class... Args> void EmplaceBack(Args &&... args) {
+  template <class... Args> void EmplaceBack(Args &&...args) {
     m_files.emplace_back(std::forward<Args>(args)...);
   }
 
@@ -227,5 +225,4 @@ class FileSpecList {
 
 } // namespace lldb_private
 
-#endif // #if defined(__cplusplus)
 #endif // LLDB_CORE_FILESPECLIST_H
diff --git a/lldb/include/lldb/Utility/Listener.h b/lldb/include/lldb/Utility/Listener.h
index d70e778c94803..8d50fe77d6a29 100644
--- a/lldb/include/lldb/Utility/Listener.h
+++ b/lldb/include/lldb/Utility/Listener.h
@@ -95,6 +95,8 @@ class Listener : public std::enable_shared_from_this<Listener> {
 
   size_t HandleBroadcastEvent(lldb::EventSP &event_sp);
 
+  void SetShadow(bool is_shadow) { m_is_shadow = is_shadow; }
+
 private:
   // Classes that inherit from Listener can see and modify these
   struct BroadcasterInfo {
@@ -134,6 +136,7 @@ class Listener : public std::enable_shared_from_this<Listener> {
   std::mutex m_events_mutex; // Protects m_broadcasters and m_events
   std::condition_variable m_events_condition;
   broadcaster_manager_collection m_broadcaster_managers;
+  bool m_is_shadow = false;
 
   void BroadcasterWillDestruct(Broadcaster *);
 
diff --git a/lldb/include/lldb/Utility/ProcessInfo.h b/lldb/include/lldb/Utility/ProcessInfo.h
index 8bddb04b1d474..f56b30ab82ec6 100644
--- a/lldb/include/lldb/Utility/ProcessInfo.h
+++ b/lldb/include/lldb/Utility/ProcessInfo.h
@@ -97,6 +97,25 @@ class ProcessInfo {
     m_scripted_metadata_sp = metadata_sp;
   }
 
+  // Get and set the actual listener that will be used for the process events
+  lldb::ListenerSP GetListener() const { return m_listener_sp; }
+
+  void SetListener(const lldb::ListenerSP &listener_sp) {
+    m_listener_sp = listener_sp;
+  }
+
+  lldb::ListenerSP GetHijackListener() const { return m_hijack_listener_sp; }
+
+  void SetHijackListener(const lldb::ListenerSP &listener_sp) {
+    m_hijack_listener_sp = listener_sp;
+  }
+
+  lldb::ListenerSP GetShadowListener() const { return m_shadow_listener_sp; }
+
+  void SetShadowListener(const lldb::ListenerSP &listener_sp) {
+    m_shadow_listener_sp = listener_sp;
+  }
+
 protected:
   FileSpec m_executable;
   std::string m_arg0; // argv[0] if supported. If empty, then use m_executable.
@@ -109,6 +128,9 @@ class ProcessInfo {
   ArchSpec m_arch;
   lldb::pid_t m_pid = LLDB_INVALID_PROCESS_ID;
   lldb::ScriptedMetadataSP m_scripted_metadata_sp = nullptr;
+  lldb::ListenerSP m_listener_sp = nullptr;
+  lldb::ListenerSP m_hijack_listener_sp = nullptr;
+  lldb::ListenerSP m_shadow_listener_sp = nullptr;
 };
 
 // ProcessInstanceInfo
diff --git a/lldb/include/lldb/module.modulemap b/lldb/include/lldb/module.modulemap
deleted file mode 100644
index 3e5a903dc1c6c..0000000000000
--- a/lldb/include/lldb/module.modulemap
+++ /dev/null
@@ -1,142 +0,0 @@
-
-module lldb_API {
-  requires cplusplus
-
-  textual header "Utility/Instrumentation.h"
-
-  umbrella "API"
-  module * { export * }
-}
-
-module lldb_Host {
-  requires cplusplus
-
-  // Because we have OS-specific headers in Host, we just list
-  // all OS-independent headers here that will include the correct
-  // OS-specific header for us.
-  module ConnectionFileDescriptor { header "Host/ConnectionFileDescriptor.h" export * }
-  module Debug { header "Host/Debug.h" export * }
-  module Editline { header "Host/Editline.h" export * }
-  module FileCache { header "Host/FileCache.h" export * }
-  module File { header "Host/File.h" export * }
-  module FileAction { header "Host/FileAction.h" export * }
-  module FileSystem { header "Host/FileSystem.h" export * }
-  module HostGetOpt { header "Host/HostGetOpt.h" export * }
-  module Host { header "Host/Host.h" export * }
-  module HostInfoBase { header "Host/HostInfoBase.h" export * }
-  module HostInfo { header "Host/HostInfo.h" export * }
-  module HostNativeProcessBase { header "Host/HostNativeProcessBase.h" export * }
-  module HostNativeProcess { header "Host/HostNativeProcess.h" export * }
-  module HostNativeThreadBase { header "Host/HostNativeThreadBase.h" export * }
-  module HostNativeThreadForward { header "Host/HostNativeThreadForward.h" export * }
-  module HostNativeThread { header "Host/HostNativeThread.h" export * }
-  module HostProcess { header "Host/HostProcess.h" export * }
-  module HostThread { header "Host/HostThread.h" export * }
-  module LockFileBase { header "Host/LockFileBase.h" export * }
-  module LockFile { header "Host/LockFile.h" export * }
-  module MainLoopBase { header "Host/MainLoopBase.h" export * }
-  module MainLoop { header "Host/MainLoop.h" export * }
-  module MonitoringProcessLauncher { header "Host/MonitoringProcessLauncher.h" export * }
-  module OptionParser { header "Host/OptionParser.h" export * }
-  module PipeBase { header "Host/PipeBase.h" export * }
-  module Pipe { header "Host/Pipe.h" export * }
-  module PosixApi { header "Host/PosixApi.h" export * }
-  module ProcessLauncher { header "Host/ProcessLauncher.h" export * }
-  module ProcessLaunchInfo { header "Host/ProcessLaunchInfo.h" export * }
-  module ProcessRunLock { header "Host/ProcessRunLock.h" export * }
-  module PseudoTerminal { header "Host/PseudoTerminal.h" export * }
-  module SafeMachO { header "Host/SafeMachO.h" export * }
-  module SocketAddress { header "Host/SocketAddress.h" export * }
-  module Socket { header "Host/Socket.h" export * }
-  module Terminal { header "Host/Terminal.h" export * }
-  module ThreadLauncher { header "Host/ThreadLauncher.h" export * }
-  module Time { header "Host/Time.h" export * }
-  module XML { header "Host/XML.h" export * }
-
-  module common {
-    umbrella "Host/common"
-    module * { export * }
-  }
-
-  export *
-}
-
-module lldb_Initialization {
-  requires cplusplus
-
-  umbrella "Initialization"
-  module * { export * }
-}
-
-
-module lldb_Wrapper {
-
-  module lldb_Breakpoint {
-    requires cplusplus
-
-    umbrella "Breakpoint"
-    module * { export * }
-  }
-
-  module lldb_Core {
-    requires cplusplus
-
-    umbrella "Core"
-    module * { export * }
-  }
-
-  module lldb_DataFormatters {
-    requires cplusplus
-
-    umbrella "DataFormatters"
-    module * { export * }
-  }
-
-  module lldb_Expression {
-    requires cplusplus
-
-    umbrella "Expression"
-    module * { export * }
-  }
-
-  module lldb_Interpreter {
-    requires cplusplus
-
-    umbrella "Interpreter"
-    module * { export * }
-  }
-
-  module lldb_Symbol {
-    requires cplusplus
-
-    umbrella "Symbol"
-    module * { export * }
-  }
-  module lldb_Target {
-    requires cplusplus
-
-    umbrella "Target"
-    textual header "Target/AppleArm64ExceptionClass.def"
-    module * { export * }
-  }
-}
-
-
-module lldb_Utility {
-  requires cplusplus
-
-  umbrella "Utility"
-  module * { export * }
-
-  module lldb_defines { header "lldb-defines.h" export * }
-  module lldb_enumerations { header "lldb-enumerations.h" export * }
-  module lldb_forward { header "lldb-forward.h" export * }
-  module lldb_private_enumerations { header "lldb-private-enumerations.h" export * }
-  module lldb_private_forward { header "lldb-private-forward.h" export * }
-  module lldb_private { header "lldb-private.h" export * }
-  module lldb_private_interfaces { header "lldb-private-interfaces.h" export * }
-  module lldb_private_types { header "lldb-private-types.h" export * }
-  module lldb_public { header "lldb-public.h" export * }
-  module lldb_types { header "lldb-types.h" export * }
-  module lldb_versioning { header "lldb-versioning.h" export * }
-}
diff --git a/lldb/include/module.modulemap b/lldb/include/module.modulemap
new file mode 100644
index 0000000000000..9500427eed581
--- /dev/null
+++ b/lldb/include/module.modulemap
@@ -0,0 +1,142 @@
+
+module lldb_API {
+  requires cplusplus
+
+  textual header "lldb/Utility/Instrumentation.h"
+
+  umbrella "lldb/API"
+  module * { export * }
+}
+
+module lldb_Host {
+  requires cplusplus
+
+  // Because we have OS-specific headers in Host, we just list
+  // all OS-independent headers here that will include the correct
+  // OS-specific header for us.
+  module ConnectionFileDescriptor { header "lldb/Host/ConnectionFileDescriptor.h" export * }
+  module Debug { header "lldb/Host/Debug.h" export * }
+  module Editline { header "lldb/Host/Editline.h" export * }
+  module FileCache { header "lldb/Host/FileCache.h" export * }
+  module File { header "lldb/Host/File.h" export * }
+  module FileAction { header "lldb/Host/FileAction.h" export * }
+  module FileSystem { header "lldb/Host/FileSystem.h" export * }
+  module HostGetOpt { header "lldb/Host/HostGetOpt.h" export * }
+  module Host { header "lldb/Host/Host.h" export * }
+  module HostInfoBase { header "lldb/Host/HostInfoBase.h" export * }
+  module HostInfo { header "lldb/Host/HostInfo.h" export * }
+  module HostNativeProcessBase { header "lldb/Host/HostNativeProcessBase.h" export * }
+  module HostNativeProcess { header "lldb/Host/HostNativeProcess.h" export * }
+  module HostNativeThreadBase { header "lldb/Host/HostNativeThreadBase.h" export * }
+  module HostNativeThreadForward { header "lldb/Host/HostNativeThreadForward.h" export * }
+  module HostNativeThread { header "lldb/Host/HostNativeThread.h" export * }
+  module HostProcess { header "lldb/Host/HostProcess.h" export * }
+  module HostThread { header "lldb/Host/HostThread.h" export * }
+  module LockFileBase { header "lldb/Host/LockFileBase.h" export * }
+  module LockFile { header "lldb/Host/LockFile.h" export * }
+  module MainLoopBase { header "lldb/Host/MainLoopBase.h" export * }
+  module MainLoop { header "lldb/Host/MainLoop.h" export * }
+  module MonitoringProcessLauncher { header "lldb/Host/MonitoringProcessLauncher.h" export * }
+  module OptionParser { header "lldb/Host/OptionParser.h" export * }
+  module PipeBase { header "lldb/Host/PipeBase.h" export * }
+  module Pipe { header "lldb/Host/Pipe.h" export * }
+  module PosixApi { header "lldb/Host/PosixApi.h" export * }
+  module ProcessLauncher { header "lldb/Host/ProcessLauncher.h" export * }
+  module ProcessLaunchInfo { header "lldb/Host/ProcessLaunchInfo.h" export * }
+  module ProcessRunLock { header "lldb/Host/ProcessRunLock.h" export * }
+  module PseudoTerminal { header "lldb/Host/PseudoTerminal.h" export * }
+  module SafeMachO { header "lldb/Host/SafeMachO.h" export * }
+  module SocketAddress { header "lldb/Host/SocketAddress.h" export * }
+  module Socket { header "lldb/Host/Socket.h" export * }
+  module Terminal { header "lldb/Host/Terminal.h" export * }
+  module ThreadLauncher { header "lldb/Host/ThreadLauncher.h" export * }
+  module Time { header "lldb/Host/Time.h" export * }
+  module XML { header "lldb/Host/XML.h" export * }
+
+  module common {
+    umbrella "lldb/Host/common"
+    module * { export * }
+  }
+
+  export *
+}
+
+module lldb_Initialization {
+  requires cplusplus
+
+  umbrella "lldb/Initialization"
+  module * { export * }
+}
+
+
+module lldb_Wrapper {
+
+  module lldb_Breakpoint {
+    requires cplusplus
+
+    umbrella "lldb/Breakpoint"
+    module * { export * }
+  }
+
+  module lldb_Core {
+    requires cplusplus
+
+    umbrella "lldb/Core"
+    module * { export * }
+  }
+
+  module lldb_DataFormatters {
+    requires cplusplus
+
+    umbrella "lldb/DataFormatters"
+    module * { export * }
+  }
+
+  module lldb_Expression {
+    requires cplusplus
+
+    umbrella "lldb/Expression"
+    module * { export * }
+  }
+
+  module lldb_Interpreter {
+    requires cplusplus
+
+    umbrella "lldb/Interpreter"
+    module * { export * }
+  }
+
+  module lldb_Symbol {
+    requires cplusplus
+
+    umbrella "lldb/Symbol"
+    module * { export * }
+  }
+  module lldb_Target {
+    requires cplusplus
+
+    umbrella "lldb/Target"
+    textual header "lldb/Target/AppleArm64ExceptionClass.def"
+    module * { export * }
+  }
+}
+
+
+module lldb_Utility {
+  requires cplusplus
+
+  umbrella "lldb/Utility"
+  module * { export * }
+
+  module lldb_defines { header "lldb/lldb-defines.h" export * }
+  module lldb_enumerations { header "lldb/lldb-enumerations.h" export * }
+  module lldb_forward { header "lldb/lldb-forward.h" export * }
+  module lldb_private_enumerations { header "lldb/lldb-private-enumerations.h" export * }
+  module lldb_private_forward { header "lldb/lldb-private-forward.h" export * }
+  module lldb_private { header "lldb/lldb-private.h" export * }
+  module lldb_private_interfaces { header "lldb/lldb-private-interfaces.h" export * }
+  module lldb_private_types { header "lldb/lldb-private-types.h" export * }
+  module lldb_public { header "lldb/lldb-public.h" export * }
+  module lldb_types { header "lldb/lldb-types.h" export * }
+  module lldb_versioning { header "lldb/lldb-versioning.h" export * }
+}
diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index d174c5af069b8..309b51baae9be 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -1202,18 +1202,25 @@ def start_listening_from(broadcaster, event_mask):
     broadcaster.AddListener(listener, event_mask)
     return listener
 
-def fetch_next_event(test, listener, broadcaster, timeout=10):
+def fetch_next_event(test, listener, broadcaster, match_class=False, timeout=10):
     """Fetch one event from the listener and return it if it matches the provided broadcaster.
+    If `match_class` is set to True, this will match an event with an entire broadcaster class.
     Fails otherwise."""
 
     event = lldb.SBEvent()
 
     if listener.WaitForEvent(timeout, event):
-        if event.BroadcasterMatchesRef(broadcaster):
-            return event
+        if match_class:
+            if event.GetBroadcasterClass() == broadcaster:
+                return event
+        else:
+            if event.BroadcasterMatchesRef(broadcaster):
+                return event
 
+        stream = lldb.SBStream()
+        event.GetDescription(stream)
         test.fail("received event '%s' from unexpected broadcaster '%s'." %
-                  (event.GetDescription(), event.GetBroadcaster().GetName()))
+                  (stream.GetData(), event.GetBroadcaster().GetName()))
 
     test.fail("couldn't fetch an event before reaching the timeout.")
 
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
index c2de4ad5c7d9a..16d26a8fa2216 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
@@ -588,6 +588,15 @@ def request_continue(self, threadId=None):
         # Caller must still call wait_for_stopped.
         return response
 
+    def request_restart(self):
+        command_dict = {
+            'command': 'restart',
+            'type': 'request',
+        }
+        response = self.send_recv(command_dict)
+        # Caller must still call wait_for_stopped.
+        return response
+
     def request_disconnect(self, terminateDebuggee=None):
         args_dict = {}
         if terminateDebuggee is not None:
diff --git a/lldb/source/API/SBAttachInfo.cpp b/lldb/source/API/SBAttachInfo.cpp
index 0d2dacbf9c78c..1cd1146f5d317 100644
--- a/lldb/source/API/SBAttachInfo.cpp
+++ b/lldb/source/API/SBAttachInfo.cpp
@@ -254,6 +254,27 @@ void SBAttachInfo::SetListener(SBListener &listener) {
   m_opaque_sp->SetListener(listener.GetSP());
 }
 
+SBListener SBAttachInfo::GetShadowListener() {
+  LLDB_INSTRUMENT_VA(this);
+
+  lldb::ListenerSP shadow_sp = m_opaque_sp->GetShadowListener();
+  if (!shadow_sp)
+    return SBListener();
+  return SBListener(shadow_sp);
+}
+
+void SBAttachInfo::SetShadowListener(SBListener &listener) {
+  LLDB_INSTRUMENT_VA(this, listener);
+
+  ListenerSP listener_sp = listener.GetSP();
+  if (listener_sp && listener.IsValid())
+    listener_sp->SetShadow(true);
+  else
+    listener_sp = nullptr;
+
+  m_opaque_sp->SetShadowListener(listener_sp);
+}
+
 const char *SBAttachInfo::GetScriptedProcessClassName() const {
   LLDB_INSTRUMENT_VA(this);
 
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index 4f9c2fa1b223a..7574b87eb85ac 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -1363,7 +1363,7 @@ SBDebugger::GetInternalVariableValue(const char *var_name,
     ExecutionContext exe_ctx(
         debugger_sp->GetCommandInterpreter().GetExecutionContext());
     lldb::OptionValueSP value_sp(
-        debugger_sp->GetPropertyValue(&exe_ctx, var_name, false, error));
+        debugger_sp->GetPropertyValue(&exe_ctx, var_name, error));
     if (value_sp) {
       StreamString value_strm;
       value_sp->DumpValue(&exe_ctx, value_strm, OptionValue::eDumpOptionValue);
diff --git a/lldb/source/API/SBError.cpp b/lldb/source/API/SBError.cpp
index 1a034154955d8..2eb9e927514ac 100644
--- a/lldb/source/API/SBError.cpp
+++ b/lldb/source/API/SBError.cpp
@@ -25,6 +25,12 @@ SBError::SBError(const SBError &rhs) {
   m_opaque_up = clone(rhs.m_opaque_up);
 }
 
+SBError::SBError(const char *message) {
+  LLDB_INSTRUMENT_VA(this, message);
+
+  SetErrorString(message);
+}
+
 SBError::SBError(const lldb_private::Status &status)
     : m_opaque_up(new Status(status)) {
   LLDB_INSTRUMENT_VA(this, status);
diff --git a/lldb/source/API/SBFileSpec.cpp b/lldb/source/API/SBFileSpec.cpp
index a7df9afc4b8eb..8668b64b4ce76 100644
--- a/lldb/source/API/SBFileSpec.cpp
+++ b/lldb/source/API/SBFileSpec.cpp
@@ -114,9 +114,7 @@ const char *SBFileSpec::GetFilename() const {
 const char *SBFileSpec::GetDirectory() const {
   LLDB_INSTRUMENT_VA(this);
 
-  FileSpec directory{*m_opaque_up};
-  directory.ClearFilename();
-  return directory.GetPathAsConstString().GetCString();
+  return m_opaque_up->GetDirectory().GetCString();
 }
 
 void SBFileSpec::SetFilename(const char *filename) {
diff --git a/lldb/source/API/SBFileSpecList.cpp b/lldb/source/API/SBFileSpecList.cpp
index cf81c42340879..74a368a3cabeb 100644
--- a/lldb/source/API/SBFileSpecList.cpp
+++ b/lldb/source/API/SBFileSpecList.cpp
@@ -10,9 +10,9 @@
 #include "Utils.h"
 #include "lldb/API/SBFileSpec.h"
 #include "lldb/API/SBStream.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Host/PosixApi.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Instrumentation.h"
 #include "lldb/Utility/Stream.h"
 
diff --git a/lldb/source/API/SBLaunchInfo.cpp b/lldb/source/API/SBLaunchInfo.cpp
index 21ae84b4e3cc4..b3ed1b3ac2e60 100644
--- a/lldb/source/API/SBLaunchInfo.cpp
+++ b/lldb/source/API/SBLaunchInfo.cpp
@@ -17,6 +17,7 @@
 #include "lldb/API/SBStructuredData.h"
 #include "lldb/Core/StructuredDataImpl.h"
 #include "lldb/Host/ProcessLaunchInfo.h"
+#include "lldb/Utility/Listener.h"
 #include "lldb/Utility/ScriptedMetadata.h"
 
 using namespace lldb;
@@ -387,3 +388,24 @@ void SBLaunchInfo::SetScriptedProcessDictionary(lldb::SBStructuredData dict) {
   metadata_sp = std::make_shared<ScriptedMetadata>(class_name, dict_sp);
   m_opaque_sp->SetScriptedMetadata(metadata_sp);
 }
+
+SBListener SBLaunchInfo::GetShadowListener() {
+  LLDB_INSTRUMENT_VA(this);
+
+  lldb::ListenerSP shadow_sp = m_opaque_sp->GetShadowListener();
+  if (!shadow_sp)
+    return SBListener();
+  return SBListener(shadow_sp);
+}
+
+void SBLaunchInfo::SetShadowListener(SBListener &listener) {
+  LLDB_INSTRUMENT_VA(this, listener);
+
+  ListenerSP listener_sp = listener.GetSP();
+  if (listener_sp && listener.IsValid())
+    listener_sp->SetShadow(true);
+  else
+    listener_sp = nullptr;
+
+  m_opaque_sp->SetShadowListener(listener_sp);
+}
diff --git a/lldb/source/API/SBProcess.cpp b/lldb/source/API/SBProcess.cpp
index ca473175f18f0..4af1a983fe20f 100644
--- a/lldb/source/API/SBProcess.cpp
+++ b/lldb/source/API/SBProcess.cpp
@@ -466,6 +466,16 @@ SBEvent SBProcess::GetStopEventForStopID(uint32_t stop_id) {
   return sb_event;
 }
 
+void SBProcess::ForceScriptedState(StateType new_state) {
+  LLDB_INSTRUMENT_VA(this, new_state);
+
+  if (ProcessSP process_sp = GetSP()) {
+    std::lock_guard<std::recursive_mutex> guard(
+        process_sp->GetTarget().GetAPIMutex());
+    process_sp->ForceScriptedState(new_state);
+  }
+}
+
 StateType SBProcess::GetState() {
   LLDB_INSTRUMENT_VA(this);
 
@@ -769,8 +779,8 @@ SBProcess::GetStructuredDataFromEvent(const lldb::SBEvent &event) {
 bool SBProcess::EventIsProcessEvent(const SBEvent &event) {
   LLDB_INSTRUMENT_VA(event);
 
-  return (event.GetBroadcasterClass() == SBProcess::GetBroadcasterClass()) &&
-         !EventIsStructuredDataEvent(event);
+  return Process::ProcessEventData::GetEventDataFromEvent(event.get()) !=
+         nullptr;
 }
 
 bool SBProcess::EventIsStructuredDataEvent(const lldb::SBEvent &event) {
diff --git a/lldb/source/Commands/CommandCompletions.cpp b/lldb/source/Commands/CommandCompletions.cpp
index ae1ee1fdd30b8..53a3f966a6c79 100644
--- a/lldb/source/Commands/CommandCompletions.cpp
+++ b/lldb/source/Commands/CommandCompletions.cpp
@@ -10,7 +10,6 @@
 #include "llvm/ADT/StringSet.h"
 
 #include "lldb/Breakpoint/Watchpoint.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/DataFormatters/DataVisualization.h"
@@ -27,6 +26,7 @@
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/TildeExpressionResolver.h"
 
diff --git a/lldb/source/Commands/CommandObjectBreakpoint.cpp b/lldb/source/Commands/CommandObjectBreakpoint.cpp
index 8c31630231b55..3debbb3b576e6 100644
--- a/lldb/source/Commands/CommandObjectBreakpoint.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpoint.cpp
@@ -1739,7 +1739,8 @@ class CommandObjectBreakpointNameConfigure : public CommandObjectParsed {
     // check the error:
     BreakpointSP bp_sp;
     if (m_bp_id.m_breakpoint.OptionWasSet()) {
-      lldb::break_id_t bp_id = m_bp_id.m_breakpoint.GetUInt64Value();
+      lldb::break_id_t bp_id =
+          m_bp_id.m_breakpoint.GetUInt64Value().value_or(0);
       bp_sp = target.GetBreakpointByID(bp_id);
       if (!bp_sp) {
         result.AppendErrorWithFormatv("Could not find specified breakpoint {0}",
@@ -1755,7 +1756,8 @@ class CommandObjectBreakpointNameConfigure : public CommandObjectParsed {
       if (!bp_name)
         continue;
       if (m_bp_id.m_help_string.OptionWasSet())
-        bp_name->SetHelp(m_bp_id.m_help_string.GetStringValue().str().c_str());
+        bp_name->SetHelp(
+            m_bp_id.m_help_string.GetStringValue().value_or("").str().c_str());
 
       if (bp_sp)
         target.ConfigureBreakpointName(*bp_name, bp_sp->GetOptions(),
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index 70dd6ea947bec..a8896f8c1fea6 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -1047,7 +1047,8 @@ class CommandObjectMemoryFind : public CommandObjectParsed {
     DataBufferHeap buffer;
 
     if (m_memory_options.m_string.OptionWasSet()) {
-      llvm::StringRef str = m_memory_options.m_string.GetStringValue();
+      llvm::StringRef str =
+          m_memory_options.m_string.GetStringValue().value_or("");
       if (str.empty()) {
         result.AppendError("search string must have non-zero length.");
         return false;
@@ -1058,7 +1059,8 @@ class CommandObjectMemoryFind : public CommandObjectParsed {
       ValueObjectSP result_sp;
       if ((eExpressionCompleted ==
            process->GetTarget().EvaluateExpression(
-               m_memory_options.m_expr.GetStringValue(), frame, result_sp)) &&
+               m_memory_options.m_expr.GetStringValue().value_or(""), frame,
+               result_sp)) &&
           result_sp) {
         uint64_t value = result_sp->GetValueAsUnsigned(0);
         std::optional<uint64_t> size =
diff --git a/lldb/source/Commands/CommandObjectRegister.cpp b/lldb/source/Commands/CommandObjectRegister.cpp
index a6ea64229eccb..80813cda04b8b 100644
--- a/lldb/source/Commands/CommandObjectRegister.cpp
+++ b/lldb/source/Commands/CommandObjectRegister.cpp
@@ -171,8 +171,8 @@ class CommandObjectRegisterRead : public CommandObjectParsed {
       const size_t set_array_size = m_command_options.set_indexes.GetSize();
       if (set_array_size > 0) {
         for (size_t i = 0; i < set_array_size; ++i) {
-          set_idx = m_command_options.set_indexes[i]->GetUInt64Value(UINT32_MAX,
-                                                                     nullptr);
+          set_idx = m_command_options.set_indexes[i]->GetUInt64Value().value_or(
+              UINT32_MAX);
           if (set_idx < reg_ctx->GetRegisterSetCount()) {
             if (!DumpRegisterSet(m_exe_ctx, strm, reg_ctx, set_idx)) {
               if (errno)
diff --git a/lldb/source/Commands/CommandObjectSettings.cpp b/lldb/source/Commands/CommandObjectSettings.cpp
index d658e137a4fdb..a2ad87a4395cf 100644
--- a/lldb/source/Commands/CommandObjectSettings.cpp
+++ b/lldb/source/Commands/CommandObjectSettings.cpp
@@ -161,8 +161,8 @@ insert-before or insert-after.");
     const char *setting_var_name =
         request.GetParsedLine().GetArgumentAtIndex(setting_var_idx);
     Status error;
-    lldb::OptionValueSP value_sp(GetDebugger().GetPropertyValue(
-        &m_exe_ctx, setting_var_name, false, error));
+    lldb::OptionValueSP value_sp(
+        GetDebugger().GetPropertyValue(&m_exe_ctx, setting_var_name, error));
     if (!value_sp)
       return;
     value_sp->AutoComplete(m_interpreter, request);
@@ -520,7 +520,6 @@ class CommandObjectSettingsList : public CommandObjectParsed {
   bool DoExecute(Args &args, CommandReturnObject &result) override {
     result.SetStatus(eReturnStatusSuccessFinishResult);
 
-    const bool will_modify = false;
     const size_t argc = args.GetArgumentCount();
     if (argc > 0) {
       const bool dump_qualified_name = true;
@@ -530,7 +529,7 @@ class CommandObjectSettingsList : public CommandObjectParsed {
 
         const Property *property =
             GetDebugger().GetValueProperties()->GetPropertyAtPath(
-                &m_exe_ctx, will_modify, property_path);
+                &m_exe_ctx, property_path);
 
         if (property) {
           property->DumpDescription(m_interpreter, result.GetOutputStream(), 0,
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index aecdf595ee5b5..0a1cacfaf7509 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -2161,9 +2161,9 @@ class CommandObjectTargetModulesDumpClangPCMInfo : public CommandObjectParsed {
     }
 
     const char *pcm_path = command.GetArgumentAtIndex(0);
-    FileSpec pcm_file{pcm_path};
+    const FileSpec pcm_file{pcm_path};
 
-    if (pcm_file.GetFileNameExtension().GetStringRef() != ".pcm") {
+    if (pcm_file.GetFileNameExtension() != ".pcm") {
       result.AppendError("file must have a .pcm extension");
       return false;
     }
diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp
index ae52808793660..b6c379bdb4342 100644
--- a/lldb/source/Commands/CommandObjectType.cpp
+++ b/lldb/source/Commands/CommandObjectType.cpp
@@ -39,9 +39,6 @@
 #include <functional>
 #include <memory>
 
-#define CHECK_FORMATTER_KIND_MASK(VAL)                                         \
-  ((m_formatter_kind_mask & (VAL)) == (VAL))
-
 using namespace lldb;
 using namespace lldb_private;
 
@@ -100,6 +97,22 @@ static bool WarnOnPotentialUnquotedUnsignedType(Args &command,
   return false;
 }
 
+const char *FormatCategoryToString(FormatCategoryItem item, bool long_name) {
+  switch (item) {
+  case eFormatCategoryItemSummary:
+    return "summary";
+  case eFormatCategoryItemFilter:
+    return "filter";
+  case eFormatCategoryItemSynth:
+    if (long_name)
+      return "synthetic child provider";
+    return "synthetic";
+  case eFormatCategoryItemFormat:
+    return "format";
+  }
+  llvm_unreachable("Fully covered switch above!");
+}
+
 #define LLDB_OPTIONS_type_summary_add
 #include "CommandOptions.inc"
 
@@ -754,16 +767,25 @@ class CommandObjectTypeFormatterDelete : public CommandObjectParsed {
   };
 
   CommandOptions m_options;
-  uint32_t m_formatter_kind_mask;
+  FormatCategoryItem m_formatter_kind;
 
   Options *GetOptions() override { return &m_options; }
 
+  static constexpr const char *g_short_help_template =
+      "Delete an existing %s for a type.";
+
+  static constexpr const char *g_long_help_template =
+      "Delete an existing %s for a type.  Unless you specify a "
+      "specific category or all categories, only the "
+      "'default' category is searched.  The names must be exactly as "
+      "shown in the 'type %s list' output";
+
 public:
   CommandObjectTypeFormatterDelete(CommandInterpreter &interpreter,
-                                   uint32_t formatter_kind_mask,
-                                   const char *name, const char *help)
-      : CommandObjectParsed(interpreter, name, help, nullptr),
-        m_formatter_kind_mask(formatter_kind_mask) {
+                                   FormatCategoryItem formatter_kind)
+      : CommandObjectParsed(interpreter,
+                            FormatCategoryToString(formatter_kind, false)),
+        m_formatter_kind(formatter_kind) {
     CommandArgumentEntry type_arg;
     CommandArgumentData type_style_arg;
 
@@ -773,6 +795,19 @@ class CommandObjectTypeFormatterDelete : public CommandObjectParsed {
     type_arg.push_back(type_style_arg);
 
     m_arguments.push_back(type_arg);
+
+    const char *kind = FormatCategoryToString(formatter_kind, true);
+    const char *short_kind = FormatCategoryToString(formatter_kind, false);
+
+    StreamString s;
+    s.Printf(g_short_help_template, kind);
+    SetHelp(s.GetData());
+    s.Clear();
+    s.Printf(g_long_help_template, kind, short_kind);
+    SetHelpLong(s.GetData());
+    s.Clear();
+    s.Printf("type %s delete", short_kind);
+    SetCommandName(s.GetData());
   }
 
   ~CommandObjectTypeFormatterDelete() override = default;
@@ -785,7 +820,7 @@ class CommandObjectTypeFormatterDelete : public CommandObjectParsed {
 
     DataVisualization::Categories::ForEach(
         [this, &request](const lldb::TypeCategoryImplSP &category_sp) {
-          category_sp->AutoComplete(request, m_formatter_kind_mask);
+          category_sp->AutoComplete(request, m_formatter_kind);
           return true;
         });
   }
@@ -812,7 +847,7 @@ class CommandObjectTypeFormatterDelete : public CommandObjectParsed {
     if (m_options.m_delete_all) {
       DataVisualization::Categories::ForEach(
           [this, typeCS](const lldb::TypeCategoryImplSP &category_sp) -> bool {
-            category_sp->Delete(typeCS, m_formatter_kind_mask);
+            category_sp->Delete(typeCS, m_formatter_kind);
             return true;
           });
       result.SetStatus(eReturnStatusSuccessFinishNoResult);
@@ -827,14 +862,14 @@ class CommandObjectTypeFormatterDelete : public CommandObjectParsed {
       DataVisualization::Categories::GetCategory(m_options.m_language,
                                                  category);
       if (category)
-        delete_category = category->Delete(typeCS, m_formatter_kind_mask);
+        delete_category = category->Delete(typeCS, m_formatter_kind);
       extra_deletion = FormatterSpecificDeletion(typeCS);
     } else {
       lldb::TypeCategoryImplSP category;
       DataVisualization::Categories::GetCategory(
           ConstString(m_options.m_category.c_str()), category);
       if (category)
-        delete_category = category->Delete(typeCS, m_formatter_kind_mask);
+        delete_category = category->Delete(typeCS, m_formatter_kind);
       extra_deletion = FormatterSpecificDeletion(typeCS);
     }
 
@@ -888,16 +923,16 @@ class CommandObjectTypeFormatterClear : public CommandObjectParsed {
   };
 
   CommandOptions m_options;
-  uint32_t m_formatter_kind_mask;
+  FormatCategoryItem m_formatter_kind;
 
   Options *GetOptions() override { return &m_options; }
 
 public:
   CommandObjectTypeFormatterClear(CommandInterpreter &interpreter,
-                                  uint32_t formatter_kind_mask,
+                                  FormatCategoryItem formatter_kind,
                                   const char *name, const char *help)
       : CommandObjectParsed(interpreter, name, help, nullptr),
-        m_formatter_kind_mask(formatter_kind_mask) {
+        m_formatter_kind(formatter_kind) {
     CommandArgumentData category_arg{eArgTypeName, eArgRepeatOptional};
     m_arguments.push_back({category_arg});
   }
@@ -911,7 +946,7 @@ class CommandObjectTypeFormatterClear : public CommandObjectParsed {
     if (m_options.m_delete_all) {
       DataVisualization::Categories::ForEach(
           [this](const TypeCategoryImplSP &category_sp) -> bool {
-            category_sp->Clear(m_formatter_kind_mask);
+            category_sp->Clear(m_formatter_kind);
             return true;
           });
     } else {
@@ -924,7 +959,7 @@ class CommandObjectTypeFormatterClear : public CommandObjectParsed {
         DataVisualization::Categories::GetCategory(ConstString(nullptr),
                                                    category);
       }
-      category->Clear(m_formatter_kind_mask);
+      category->Clear(m_formatter_kind);
     }
 
     FormatterSpecificDeletion();
@@ -940,8 +975,7 @@ class CommandObjectTypeFormatDelete : public CommandObjectTypeFormatterDelete {
 public:
   CommandObjectTypeFormatDelete(CommandInterpreter &interpreter)
       : CommandObjectTypeFormatterDelete(
-            interpreter, eFormatCategoryItemFormat, "type format delete",
-            "Delete an existing formatting style for a type.") {}
+            interpreter, eFormatCategoryItemFormat) {}
 
   ~CommandObjectTypeFormatDelete() override = default;
 };
@@ -1609,8 +1643,7 @@ class CommandObjectTypeSummaryDelete : public CommandObjectTypeFormatterDelete {
 public:
   CommandObjectTypeSummaryDelete(CommandInterpreter &interpreter)
       : CommandObjectTypeFormatterDelete(
-            interpreter, eFormatCategoryItemSummary, "type summary delete",
-            "Delete an existing summary for a type.") {}
+            interpreter, eFormatCategoryItemSummary) {}
 
   ~CommandObjectTypeSummaryDelete() override = default;
 
@@ -2159,8 +2192,7 @@ class CommandObjectTypeFilterDelete : public CommandObjectTypeFormatterDelete {
 public:
   CommandObjectTypeFilterDelete(CommandInterpreter &interpreter)
       : CommandObjectTypeFormatterDelete(
-            interpreter, eFormatCategoryItemFilter, "type filter delete",
-            "Delete an existing filter for a type.") {}
+            interpreter, eFormatCategoryItemFilter) {}
 
   ~CommandObjectTypeFilterDelete() override = default;
 };
@@ -2173,8 +2205,7 @@ class CommandObjectTypeSynthDelete : public CommandObjectTypeFormatterDelete {
 public:
   CommandObjectTypeSynthDelete(CommandInterpreter &interpreter)
       : CommandObjectTypeFormatterDelete(
-            interpreter, eFormatCategoryItemSynth, "type synthetic delete",
-            "Delete an existing synthetic provider for a type.") {}
+            interpreter, eFormatCategoryItemSynth) {}
 
   ~CommandObjectTypeSynthDelete() override = default;
 };
diff --git a/lldb/source/Commands/CommandObjectWatchpoint.cpp b/lldb/source/Commands/CommandObjectWatchpoint.cpp
index 8b8c196e4505b..14180f9b56d36 100644
--- a/lldb/source/Commands/CommandObjectWatchpoint.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpoint.cpp
@@ -835,8 +835,7 @@ corresponding to the byte size of the data type.");
     m_arguments.push_back(arg);
 
     // Absorb the '-w' and '-s' options into our option group.
-    m_option_group.Append(&m_option_watchpoint, LLDB_OPT_SET_ALL,
-                          LLDB_OPT_SET_1);
+    m_option_group.Append(&m_option_watchpoint, LLDB_OPT_SET_1, LLDB_OPT_SET_1);
     m_option_group.Finalize();
   }
 
@@ -990,6 +989,7 @@ class CommandObjectWatchpointSetExpression : public CommandObjectRaw {
       : CommandObjectRaw(
             interpreter, "watchpoint set expression",
             "Set a watchpoint on an address by supplying an expression. "
+            "Use the '-l' option to specify the language of the expression. "
             "Use the '-w' option to specify the type of watchpoint and "
             "the '-s' option to specify the byte size to watch for. "
             "If no '-w' option is specified, it defaults to write. "
@@ -1083,6 +1083,8 @@ class CommandObjectWatchpointSetExpression : public CommandObjectRaw {
     options.SetKeepInMemory(false);
     options.SetTryAllThreads(true);
     options.SetTimeout(std::nullopt);
+    if (m_option_watchpoint.language_type != eLanguageTypeUnknown)
+      options.SetLanguage(m_option_watchpoint.language_type);
 
     ExpressionResults expr_result =
         target->EvaluateExpression(expr, frame, valobj_sp, options);
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index f0220beae032a..74ad4556c9ed7 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -36,7 +36,6 @@ add_lldb_library(lldbCore
   DynamicLoader.cpp
   EmulateInstruction.cpp
   FileLineResolver.cpp
-  FileSpecList.cpp
   FormatEntity.cpp
   Highlighter.cpp
   IOHandler.cpp
diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td
index 62729923f366b..4287cb1fff029 100644
--- a/lldb/source/Core/CoreProperties.td
+++ b/lldb/source/Core/CoreProperties.td
@@ -74,7 +74,7 @@ let Definition = "debugger" in {
     Global,
     DefaultEnumValue<"eLanguageTypeUnknown">,
     Desc<"The language to use for the REPL.">;
-  def StopDisassemblyCount: Property<"stop-disassembly-count", "SInt64">,
+  def StopDisassemblyCount: Property<"stop-disassembly-count", "UInt64">,
     Global,
     DefaultUnsignedValue<4>,
     Desc<"The number of disassembly lines to show when displaying a stopped context.">;
@@ -87,11 +87,11 @@ let Definition = "debugger" in {
     Global,
     DefaultUnsignedValue<32000>,
     Desc<"The size limit to use when disassembling large functions (default: 32KB).">;
-  def StopLineCountAfter: Property<"stop-line-count-after", "SInt64">,
+  def StopLineCountAfter: Property<"stop-line-count-after", "UInt64">,
     Global,
     DefaultUnsignedValue<3>,
     Desc<"The number of sources lines to display that come after the current source line when displaying a stopped context.">;
-  def StopLineCountBefore: Property<"stop-line-count-before", "SInt64">,
+  def StopLineCountBefore: Property<"stop-line-count-before", "UInt64">,
     Global,
     DefaultUnsignedValue<3>,
     Desc<"The number of sources lines to display that come before the current source line when displaying a stopped context.">;
@@ -135,6 +135,10 @@ let Definition = "debugger" in {
     Global,
     DefaultFalse,
     Desc<"Whether to use an external editor or not.">;
+  def ExternalEditor: Property<"external-editor", "String">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"External editor to use when use-external-editor is enabled.">;
   def UseColor: Property<"use-color", "Boolean">,
     Global,
     DefaultTrue,
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 401561e8e06d6..5eb2d1ff5c61d 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -262,47 +262,47 @@ Status Debugger::SetPropertyValue(const ExecutionContext *exe_ctx,
 }
 
 bool Debugger::GetAutoConfirm() const {
-  const uint32_t idx = ePropertyAutoConfirm;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  constexpr uint32_t idx = ePropertyAutoConfirm;
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 const FormatEntity::Entry *Debugger::GetDisassemblyFormat() const {
-  const uint32_t idx = ePropertyDisassemblyFormat;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  constexpr uint32_t idx = ePropertyDisassemblyFormat;
+  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
 }
 
 const FormatEntity::Entry *Debugger::GetFrameFormat() const {
-  const uint32_t idx = ePropertyFrameFormat;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  constexpr uint32_t idx = ePropertyFrameFormat;
+  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
 }
 
 const FormatEntity::Entry *Debugger::GetFrameFormatUnique() const {
-  const uint32_t idx = ePropertyFrameFormatUnique;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  constexpr uint32_t idx = ePropertyFrameFormatUnique;
+  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
 }
 
-uint32_t Debugger::GetStopDisassemblyMaxSize() const {
-  const uint32_t idx = ePropertyStopDisassemblyMaxSize;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+uint64_t Debugger::GetStopDisassemblyMaxSize() const {
+  constexpr uint32_t idx = ePropertyStopDisassemblyMaxSize;
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_debugger_properties[idx].default_uint_value);
 }
 
 bool Debugger::GetNotifyVoid() const {
-  const uint32_t idx = ePropertyNotiftVoid;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  constexpr uint32_t idx = ePropertyNotiftVoid;
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 llvm::StringRef Debugger::GetPrompt() const {
-  const uint32_t idx = ePropertyPrompt;
-  return m_collection_sp->GetPropertyAtIndexAsString(
-      nullptr, idx, g_debugger_properties[idx].default_cstr_value);
+  constexpr uint32_t idx = ePropertyPrompt;
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 void Debugger::SetPrompt(llvm::StringRef p) {
-  const uint32_t idx = ePropertyPrompt;
-  m_collection_sp->SetPropertyAtIndexAsString(nullptr, idx, p);
+  constexpr uint32_t idx = ePropertyPrompt;
+  SetPropertyAtIndex(idx, p);
   llvm::StringRef new_prompt = GetPrompt();
   std::string str =
       lldb_private::ansi::FormatAnsiTerminalCodes(new_prompt, GetUseColor());
@@ -312,31 +312,31 @@ void Debugger::SetPrompt(llvm::StringRef p) {
 }
 
 const FormatEntity::Entry *Debugger::GetThreadFormat() const {
-  const uint32_t idx = ePropertyThreadFormat;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  constexpr uint32_t idx = ePropertyThreadFormat;
+  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
 }
 
 const FormatEntity::Entry *Debugger::GetThreadStopFormat() const {
-  const uint32_t idx = ePropertyThreadStopFormat;
-  return m_collection_sp->GetPropertyAtIndexAsFormatEntity(nullptr, idx);
+  constexpr uint32_t idx = ePropertyThreadStopFormat;
+  return GetPropertyAtIndexAs<const FormatEntity::Entry *>(idx);
 }
 
 lldb::ScriptLanguage Debugger::GetScriptLanguage() const {
   const uint32_t idx = ePropertyScriptLanguage;
-  return (lldb::ScriptLanguage)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<lldb::ScriptLanguage>(
+      idx, static_cast<lldb::ScriptLanguage>(
+               g_debugger_properties[idx].default_uint_value));
 }
 
 bool Debugger::SetScriptLanguage(lldb::ScriptLanguage script_lang) {
   const uint32_t idx = ePropertyScriptLanguage;
-  return m_collection_sp->SetPropertyAtIndexAsEnumeration(nullptr, idx,
-                                                          script_lang);
+  return SetPropertyAtIndex(idx, script_lang);
 }
 
 lldb::LanguageType Debugger::GetREPLLanguage() const {
   const uint32_t idx = ePropertyREPLLanguage;
   OptionValueLanguage *value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueLanguage(nullptr, idx);
+      m_collection_sp->GetPropertyAtIndexAsOptionValueLanguage(idx);
   if (value)
     return value->GetCurrentValue();
   return LanguageType();
@@ -344,13 +344,13 @@ lldb::LanguageType Debugger::GetREPLLanguage() const {
 
 bool Debugger::SetREPLLanguage(lldb::LanguageType repl_lang) {
   const uint32_t idx = ePropertyREPLLanguage;
-  return m_collection_sp->SetPropertyAtIndexAsLanguage(nullptr, idx, repl_lang);
+  return SetPropertyAtIndex(idx, repl_lang);
 }
 
-uint32_t Debugger::GetTerminalWidth() const {
+uint64_t Debugger::GetTerminalWidth() const {
   const uint32_t idx = ePropertyTerminalWidth;
-  return m_collection_sp->GetPropertyAtIndexAsSInt64(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<int64_t>(
+      idx, g_debugger_properties[idx].default_uint_value);
 }
 
 bool Debugger::SetTerminalWidth(uint32_t term_width) {
@@ -358,80 +358,94 @@ bool Debugger::SetTerminalWidth(uint32_t term_width) {
     handler_sp->TerminalSizeChanged();
 
   const uint32_t idx = ePropertyTerminalWidth;
-  return m_collection_sp->SetPropertyAtIndexAsSInt64(nullptr, idx, term_width);
+  return SetPropertyAtIndex(idx, term_width);
 }
 
 bool Debugger::GetUseExternalEditor() const {
   const uint32_t idx = ePropertyUseExternalEditor;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::SetUseExternalEditor(bool b) {
   const uint32_t idx = ePropertyUseExternalEditor;
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  return SetPropertyAtIndex(idx, b);
+}
+
+llvm::StringRef Debugger::GetExternalEditor() const {
+  const uint32_t idx = ePropertyExternalEditor;
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
+}
+
+bool Debugger::SetExternalEditor(llvm::StringRef editor) {
+  const uint32_t idx = ePropertyExternalEditor;
+  return SetPropertyAtIndex(idx, editor);
 }
 
 bool Debugger::GetUseColor() const {
   const uint32_t idx = ePropertyUseColor;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::SetUseColor(bool b) {
   const uint32_t idx = ePropertyUseColor;
-  bool ret = m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  bool ret = SetPropertyAtIndex(idx, b);
   SetPrompt(GetPrompt());
   return ret;
 }
 
 bool Debugger::GetShowProgress() const {
   const uint32_t idx = ePropertyShowProgress;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::SetShowProgress(bool show_progress) {
   const uint32_t idx = ePropertyShowProgress;
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx,
-                                                      show_progress);
+  return SetPropertyAtIndex(idx, show_progress);
 }
 
 llvm::StringRef Debugger::GetShowProgressAnsiPrefix() const {
   const uint32_t idx = ePropertyShowProgressAnsiPrefix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 llvm::StringRef Debugger::GetShowProgressAnsiSuffix() const {
   const uint32_t idx = ePropertyShowProgressAnsiSuffix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 bool Debugger::GetUseAutosuggestion() const {
   const uint32_t idx = ePropertyShowAutosuggestion;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 llvm::StringRef Debugger::GetAutosuggestionAnsiPrefix() const {
   const uint32_t idx = ePropertyShowAutosuggestionAnsiPrefix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 llvm::StringRef Debugger::GetAutosuggestionAnsiSuffix() const {
   const uint32_t idx = ePropertyShowAutosuggestionAnsiSuffix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 bool Debugger::GetUseSourceCache() const {
   const uint32_t idx = ePropertyUseSourceCache;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::SetUseSourceCache(bool b) {
   const uint32_t idx = ePropertyUseSourceCache;
-  bool ret = m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  bool ret = SetPropertyAtIndex(idx, b);
   if (!ret) {
     m_source_file_cache.Clear();
   }
@@ -439,102 +453,111 @@ bool Debugger::SetUseSourceCache(bool b) {
 }
 bool Debugger::GetHighlightSource() const {
   const uint32_t idx = ePropertyHighlightSource;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 StopShowColumn Debugger::GetStopShowColumn() const {
   const uint32_t idx = ePropertyStopShowColumn;
-  return (lldb::StopShowColumn)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<lldb::StopShowColumn>(
+      idx, static_cast<lldb::StopShowColumn>(
+               g_debugger_properties[idx].default_uint_value));
 }
 
 llvm::StringRef Debugger::GetStopShowColumnAnsiPrefix() const {
   const uint32_t idx = ePropertyStopShowColumnAnsiPrefix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 llvm::StringRef Debugger::GetStopShowColumnAnsiSuffix() const {
   const uint32_t idx = ePropertyStopShowColumnAnsiSuffix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 llvm::StringRef Debugger::GetStopShowLineMarkerAnsiPrefix() const {
   const uint32_t idx = ePropertyStopShowLineMarkerAnsiPrefix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
 llvm::StringRef Debugger::GetStopShowLineMarkerAnsiSuffix() const {
   const uint32_t idx = ePropertyStopShowLineMarkerAnsiSuffix;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx, "");
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
 }
 
-uint32_t Debugger::GetStopSourceLineCount(bool before) const {
+uint64_t Debugger::GetStopSourceLineCount(bool before) const {
   const uint32_t idx =
       before ? ePropertyStopLineCountBefore : ePropertyStopLineCountAfter;
-  return m_collection_sp->GetPropertyAtIndexAsSInt64(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_debugger_properties[idx].default_uint_value);
 }
 
 Debugger::StopDisassemblyType Debugger::GetStopDisassemblyDisplay() const {
   const uint32_t idx = ePropertyStopDisassemblyDisplay;
-  return (Debugger::StopDisassemblyType)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<Debugger::StopDisassemblyType>(
+      idx, static_cast<Debugger::StopDisassemblyType>(
+               g_debugger_properties[idx].default_uint_value));
 }
 
-uint32_t Debugger::GetDisassemblyLineCount() const {
+uint64_t Debugger::GetDisassemblyLineCount() const {
   const uint32_t idx = ePropertyStopDisassemblyCount;
-  return m_collection_sp->GetPropertyAtIndexAsSInt64(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_debugger_properties[idx].default_uint_value);
 }
 
 bool Debugger::GetAutoOneLineSummaries() const {
   const uint32_t idx = ePropertyAutoOneLineSummaries;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, true);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::GetEscapeNonPrintables() const {
   const uint32_t idx = ePropertyEscapeNonPrintables;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, true);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::GetAutoIndent() const {
   const uint32_t idx = ePropertyAutoIndent;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, true);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::SetAutoIndent(bool b) {
   const uint32_t idx = ePropertyAutoIndent;
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  return SetPropertyAtIndex(idx, b);
 }
 
 bool Debugger::GetPrintDecls() const {
   const uint32_t idx = ePropertyPrintDecls;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, true);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_debugger_properties[idx].default_uint_value != 0);
 }
 
 bool Debugger::SetPrintDecls(bool b) {
   const uint32_t idx = ePropertyPrintDecls;
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  return SetPropertyAtIndex(idx, b);
 }
 
-uint32_t Debugger::GetTabSize() const {
+uint64_t Debugger::GetTabSize() const {
   const uint32_t idx = ePropertyTabSize;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_debugger_properties[idx].default_uint_value);
 }
 
 bool Debugger::SetTabSize(uint32_t tab_size) {
   const uint32_t idx = ePropertyTabSize;
-  return m_collection_sp->SetPropertyAtIndexAsUInt64(nullptr, idx, tab_size);
+  return SetPropertyAtIndex(idx, tab_size);
 }
 
 lldb::DWIMPrintVerbosity Debugger::GetDWIMPrintVerbosity() const {
   const uint32_t idx = ePropertyDWIMPrintVerbosity;
-  return (lldb::DWIMPrintVerbosity)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_debugger_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<lldb::DWIMPrintVerbosity>(
+      idx, static_cast<lldb::DWIMPrintVerbosity>(
+               g_debugger_properties[idx].default_uint_value));
 }
 
 #pragma mark Debugger
@@ -607,8 +630,8 @@ LoadPluginCallback(void *baton, llvm::sys::fs::file_type ft,
                    llvm::StringRef path) {
   Status error;
 
-  static ConstString g_dylibext(".dylib");
-  static ConstString g_solibext(".so");
+  static constexpr llvm::StringLiteral g_dylibext(".dylib");
+  static constexpr llvm::StringLiteral g_solibext(".so");
 
   if (!baton)
     return FileSystem::eEnumerateDirectoryResultQuit;
@@ -789,6 +812,24 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
                     GetStaticBroadcasterClass().AsCString()),
       m_forward_listener_sp(), m_clear_once() {
   m_instance_name.SetString(llvm::formatv("debugger_{0}", GetID()).str());
+  // Initialize the debugger properties as early as possible as other parts of
+  // LLDB will start querying them during construction.
+  m_collection_sp->Initialize(g_debugger_properties);
+  m_collection_sp->AppendProperty(
+      ConstString("target"), "Settings specify to debugging targets.", true,
+      Target::GetGlobalProperties().GetValueProperties());
+  m_collection_sp->AppendProperty(
+      ConstString("platform"), "Platform settings.", true,
+      Platform::GetGlobalPlatformProperties().GetValueProperties());
+  m_collection_sp->AppendProperty(
+      ConstString("symbols"), "Symbol lookup and cache settings.", true,
+      ModuleList::GetGlobalModuleListProperties().GetValueProperties());
+  if (m_command_interpreter_up) {
+    m_collection_sp->AppendProperty(
+        ConstString("interpreter"),
+        "Settings specify to the debugger's command interpreter.", true,
+        m_command_interpreter_up->GetValueProperties());
+  }
   if (log_callback)
     m_callback_handler_sp =
         std::make_shared<CallbackLogHandler>(log_callback, baton);
@@ -810,25 +851,9 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
   }
   assert(m_dummy_target_sp.get() && "Couldn't construct dummy target?");
 
-  m_collection_sp->Initialize(g_debugger_properties);
-  m_collection_sp->AppendProperty(
-      ConstString("target"), "Settings specify to debugging targets.", true,
-      Target::GetGlobalProperties().GetValueProperties());
-  m_collection_sp->AppendProperty(
-      ConstString("platform"), "Platform settings.", true,
-      Platform::GetGlobalPlatformProperties().GetValueProperties());
-  m_collection_sp->AppendProperty(
-      ConstString("symbols"), "Symbol lookup and cache settings.", true,
-      ModuleList::GetGlobalModuleListProperties().GetValueProperties());
-  if (m_command_interpreter_up) {
-    m_collection_sp->AppendProperty(
-        ConstString("interpreter"),
-        "Settings specify to the debugger's command interpreter.", true,
-        m_command_interpreter_up->GetValueProperties());
-  }
   OptionValueSInt64 *term_width =
       m_collection_sp->GetPropertyAtIndexAsOptionValueSInt64(
-          nullptr, ePropertyTerminalWidth);
+          ePropertyTerminalWidth);
   term_width->SetMinimumValue(10);
   term_width->SetMaximumValue(1024);
 
@@ -1242,7 +1267,7 @@ void Debugger::CancelInterruptRequest() {
 
 bool Debugger::InterruptRequested() {
   // This is the one we should call internally.  This will return true either
-  // if there's a debugger interrupt and we aren't on the IOHandler thread, 
+  // if there's a debugger interrupt and we aren't on the IOHandler thread,
   // or if we are on the IOHandler thread and there's a CommandInterpreter
   // interrupt.
   if (!IsIOHandlerThreadCurrentThread()) {
@@ -2006,7 +2031,7 @@ void Debugger::HandleDiagnosticEvent(const lldb::EventSP &event_sp) {
 }
 
 bool Debugger::HasIOHandlerThread() const {
-  return m_io_handler_thread.IsJoinable(); 
+  return m_io_handler_thread.IsJoinable();
 }
 
 HostThread Debugger::SetIOHandlerThread(HostThread &new_thread) {
diff --git a/lldb/source/Core/Disassembler.cpp b/lldb/source/Core/Disassembler.cpp
index 570332f582df2..797d30a520f33 100644
--- a/lldb/source/Core/Disassembler.cpp
+++ b/lldb/source/Core/Disassembler.cpp
@@ -253,7 +253,7 @@ bool Disassembler::ElideMixedSourceAndDisassemblyLine(
     if (target_sp) {
       Status error;
       OptionValueSP value_sp = target_sp->GetDebugger().GetPropertyValue(
-          &exe_ctx, "target.process.thread.step-avoid-regexp", false, error);
+          &exe_ctx, "target.process.thread.step-avoid-regexp", error);
       if (value_sp && value_sp->GetType() == OptionValue::eTypeRegex) {
         OptionValueRegex *re = value_sp->GetAsRegex();
         if (re) {
@@ -755,7 +755,7 @@ OptionValueSP Instruction::ReadDictionary(FILE *in_file, Stream *out_stream) {
   char buffer[1024];
 
   auto option_value_sp = std::make_shared<OptionValueDictionary>();
-  static ConstString encoding_key("data_encoding");
+  static constexpr llvm::StringLiteral encoding_key("data_encoding");
   OptionValue::Type data_type = OptionValue::eTypeInvalid;
 
   while (!done) {
@@ -802,7 +802,6 @@ OptionValueSP Instruction::ReadDictionary(FILE *in_file, Stream *out_stream) {
         return option_value_sp;
       }
 
-      ConstString const_key(key.c_str());
       // Check value to see if it's the start of an array or dictionary.
 
       lldb::OptionValueSP value_sp;
@@ -838,15 +837,14 @@ OptionValueSP Instruction::ReadDictionary(FILE *in_file, Stream *out_stream) {
         value_sp = std::make_shared<OptionValueString>(value.c_str(), "");
       }
 
-      if (const_key == encoding_key) {
+      if (key == encoding_key) {
         // A 'data_encoding=..." is NOT a normal key-value pair; it is meta-data
-        // indicating the
-        // data type of an upcoming array (usually the next bit of data to be
-        // read in).
-        if (strcmp(value.c_str(), "uint32_t") == 0)
+        // indicating the data type of an upcoming array (usually the next bit
+        // of data to be read in).
+        if (llvm::StringRef(value) == "uint32_t")
           data_type = OptionValue::eTypeUInt64;
       } else
-        option_value_sp->GetAsDictionary()->SetValueForKey(const_key, value_sp,
+        option_value_sp->GetAsDictionary()->SetValueForKey(key, value_sp,
                                                            false);
     }
   }
@@ -910,7 +908,7 @@ bool Instruction::TestEmulation(Stream *out_stream, const char *file_name) {
     return false;
   }
 
-  SetDescription(value_sp->GetStringValue());
+  SetDescription(value_sp->GetStringValue().value_or(""));
 
   value_sp = data_dictionary->GetValueForKey(triple_key);
   if (!value_sp) {
@@ -920,7 +918,7 @@ bool Instruction::TestEmulation(Stream *out_stream, const char *file_name) {
   }
 
   ArchSpec arch;
-  arch.SetTriple(llvm::Triple(value_sp->GetStringValue()));
+  arch.SetTriple(llvm::Triple(value_sp->GetStringValue().value_or("")));
 
   bool success = false;
   std::unique_ptr<EmulateInstruction> insn_emulator_up(
diff --git a/lldb/source/Core/FileLineResolver.cpp b/lldb/source/Core/FileLineResolver.cpp
index 612614a41bb0e..7c3cf70572ee3 100644
--- a/lldb/source/Core/FileLineResolver.cpp
+++ b/lldb/source/Core/FileLineResolver.cpp
@@ -8,10 +8,10 @@
 
 #include "lldb/Core/FileLineResolver.h"
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/LineTable.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Stream.h"
 
 #include <string>
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 17d8043852ab7..d527bdc76309d 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -12,7 +12,6 @@
 #include "lldb/Core/AddressResolverFileLine.h"
 #include "lldb/Core/DataFileCache.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Mangled.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/SearchFilter.h"
@@ -39,6 +38,7 @@
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/DataBufferHeap.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index 5aa58d9bba6bd..212b1186b4ef9 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Core/ModuleList.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Host/FileSystem.h"
@@ -22,6 +21,7 @@
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/UUID.h"
@@ -97,79 +97,71 @@ ModuleListProperties::ModuleListProperties() {
 
 bool ModuleListProperties::GetEnableExternalLookup() const {
   const uint32_t idx = ePropertyEnableExternalLookup;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_modulelist_properties[idx].default_uint_value != 0);
 }
 
 bool ModuleListProperties::SetEnableExternalLookup(bool new_value) {
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(
-      nullptr, ePropertyEnableExternalLookup, new_value);
+  return SetPropertyAtIndex(ePropertyEnableExternalLookup, new_value);
 }
 
 bool ModuleListProperties::GetEnableBackgroundLookup() const {
   const uint32_t idx = ePropertyEnableBackgroundLookup;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_modulelist_properties[idx].default_uint_value != 0);
 }
 
 FileSpec ModuleListProperties::GetClangModulesCachePath() const {
-  return m_collection_sp
-      ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
-                                                ePropertyClangModulesCachePath)
-      ->GetCurrentValue();
+  const uint32_t idx = ePropertyClangModulesCachePath;
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 bool ModuleListProperties::SetClangModulesCachePath(const FileSpec &path) {
-  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
-      nullptr, ePropertyClangModulesCachePath, path);
+  const uint32_t idx = ePropertyClangModulesCachePath;
+  return SetPropertyAtIndex(idx, path);
 }
 
 FileSpec ModuleListProperties::GetLLDBIndexCachePath() const {
-  return m_collection_sp
-      ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
-                                                ePropertyLLDBIndexCachePath)
-      ->GetCurrentValue();
+  const uint32_t idx = ePropertyLLDBIndexCachePath;
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 bool ModuleListProperties::SetLLDBIndexCachePath(const FileSpec &path) {
-  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
-      nullptr, ePropertyLLDBIndexCachePath, path);
+  const uint32_t idx = ePropertyLLDBIndexCachePath;
+  return SetPropertyAtIndex(idx, path);
 }
 
 bool ModuleListProperties::GetEnableLLDBIndexCache() const {
   const uint32_t idx = ePropertyEnableLLDBIndexCache;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_modulelist_properties[idx].default_uint_value != 0);
 }
 
 bool ModuleListProperties::SetEnableLLDBIndexCache(bool new_value) {
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(
-      nullptr, ePropertyEnableLLDBIndexCache, new_value);
+  return SetPropertyAtIndex(ePropertyEnableLLDBIndexCache, new_value);
 }
 
 uint64_t ModuleListProperties::GetLLDBIndexCacheMaxByteSize() {
   const uint32_t idx = ePropertyLLDBIndexCacheMaxByteSize;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_modulelist_properties[idx].default_uint_value);
 }
 
 uint64_t ModuleListProperties::GetLLDBIndexCacheMaxPercent() {
   const uint32_t idx = ePropertyLLDBIndexCacheMaxPercent;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_modulelist_properties[idx].default_uint_value);
 }
 
 uint64_t ModuleListProperties::GetLLDBIndexCacheExpirationDays() {
   const uint32_t idx = ePropertyLLDBIndexCacheExpirationDays;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_modulelist_properties[idx].default_uint_value);
 }
 
 void ModuleListProperties::UpdateSymlinkMappings() {
-  FileSpecList list = m_collection_sp
-                          ->GetPropertyAtIndexAsOptionValueFileSpecList(
-                              nullptr, false, ePropertySymLinkPaths)
-                          ->GetCurrentValue();
+  FileSpecList list =
+      GetPropertyAtIndexAs<FileSpecList>(ePropertySymLinkPaths, {});
   llvm::sys::ScopedWriter lock(m_symlink_paths_mutex);
   const bool notify = false;
   m_symlink_paths.Clear(notify);
@@ -188,8 +180,8 @@ PathMappingList ModuleListProperties::GetSymlinkMappings() const {
 
 bool ModuleListProperties::GetLoadSymbolOnDemand() {
   const uint32_t idx = ePropertyLoadSymbolOnDemand;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_modulelist_properties[idx].default_uint_value != 0);
 }
 
 ModuleList::ModuleList() : m_modules(), m_modules_mutex() {}
diff --git a/lldb/source/Core/UserSettingsController.cpp b/lldb/source/Core/UserSettingsController.cpp
index 9ea4f78ddd062..bc03b7891b052 100644
--- a/lldb/source/Core/UserSettingsController.cpp
+++ b/lldb/source/Core/UserSettingsController.cpp
@@ -32,11 +32,10 @@ using namespace lldb_private;
 
 lldb::OptionValueSP
 Properties::GetPropertyValue(const ExecutionContext *exe_ctx,
-                             llvm::StringRef path, bool will_modify,
-                             Status &error) const {
+                             llvm::StringRef path, Status &error) const {
   OptionValuePropertiesSP properties_sp(GetValueProperties());
   if (properties_sp)
-    return properties_sp->GetSubValue(exe_ctx, path, will_modify, error);
+    return properties_sp->GetSubValue(exe_ctx, path, error);
   return lldb::OptionValueSP();
 }
 
diff --git a/lldb/source/DataFormatters/FormattersHelpers.cpp b/lldb/source/DataFormatters/FormattersHelpers.cpp
index 87da31769ff12..085ed3d0a2f29 100644
--- a/lldb/source/DataFormatters/FormattersHelpers.cpp
+++ b/lldb/source/DataFormatters/FormattersHelpers.cpp
@@ -126,14 +126,3 @@ lldb_private::formatters::GetArrayAddressOrPointerValue(ValueObject &valobj) {
 
   return data_addr;
 }
-
-lldb::ValueObjectSP
-lldb_private::formatters::GetValueOfLibCXXCompressedPair(ValueObject &pair) {
-  ValueObjectSP value =
-      pair.GetChildMemberWithName(ConstString("__value_"), true);
-  if (!value) {
-    // pre-r300140 member name
-    value = pair.GetChildMemberWithName(ConstString("__first_"), true);
-  }
-  return value;
-}
diff --git a/lldb/source/Expression/IRMemoryMap.cpp b/lldb/source/Expression/IRMemoryMap.cpp
index 951444db86a87..de631370bb048 100644
--- a/lldb/source/Expression/IRMemoryMap.cpp
+++ b/lldb/source/Expression/IRMemoryMap.cpp
@@ -92,26 +92,26 @@ lldb::addr_t IRMemoryMap::FindSpace(size_t size) {
     ret = llvm::alignTo(addr + alloc_size, 4096);
   }
 
+  uint64_t end_of_memory;
+  switch (GetAddressByteSize()) {
+  case 2:
+    end_of_memory = 0xffffull;
+    break;
+  case 4:
+    end_of_memory = 0xffffffffull;
+    break;
+  case 8:
+    end_of_memory = 0xffffffffffffffffull;
+    break;
+  default:
+    lldbassert(false && "Invalid address size.");
+    return LLDB_INVALID_ADDRESS;
+  }
+
   // Now, if it's possible to use the GetMemoryRegionInfo API to detect mapped
   // regions, walk forward through memory until a region is found that has
   // adequate space for our allocation.
   if (process_is_alive) {
-    uint64_t end_of_memory;
-    switch (process_sp->GetAddressByteSize()) {
-    case 2:
-      end_of_memory = 0xffffull;
-      break;
-    case 4:
-      end_of_memory = 0xffffffffull;
-      break;
-    case 8:
-      end_of_memory = 0xffffffffffffffffull;
-      break;
-    default:
-      lldbassert(false && "Invalid address size.");
-      return LLDB_INVALID_ADDRESS;
-    }
-
     MemoryRegionInfo region_info;
     Status err = process_sp->GetMemoryRegionInfo(ret, region_info);
     if (err.Success()) {
@@ -147,29 +147,40 @@ lldb::addr_t IRMemoryMap::FindSpace(size_t size) {
   // to the end of the allocations we've already reported, or use a 'sensible'
   // default if this is our first allocation.
   if (m_allocations.empty()) {
-    uint32_t address_byte_size = GetAddressByteSize();
-    if (address_byte_size != UINT32_MAX) {
-      switch (address_byte_size) {
-      case 2:
-        ret = 0x8000ull;
-        break;
-      case 4:
-        ret = 0xee000000ull;
-        break;
-      case 8:
-        ret = 0xdead0fff00000000ull;
-        break;
-      default:
-        lldbassert(false && "Invalid address size.");
+    uint64_t alloc_address = target_sp->GetExprAllocAddress();
+    if (alloc_address > 0) {
+      if (alloc_address >= end_of_memory) {
+        lldbassert(0 && "The allocation address for expression evaluation must "
+                        "be within process address space");
         return LLDB_INVALID_ADDRESS;
       }
+      ret = alloc_address;
+    } else {
+      uint32_t address_byte_size = GetAddressByteSize();
+      if (address_byte_size != UINT32_MAX) {
+        switch (address_byte_size) {
+        case 2:
+          ret = 0x8000ull;
+          break;
+        case 4:
+          ret = 0xee000000ull;
+          break;
+        case 8:
+          ret = 0xdead0fff00000000ull;
+          break;
+        default:
+          lldbassert(false && "Invalid address size.");
+          return LLDB_INVALID_ADDRESS;
+        }
+      }
     }
   } else {
     auto back = m_allocations.rbegin();
     lldb::addr_t addr = back->first;
     size_t alloc_size = back->second.m_size;
-    auto arch = target_sp->GetArchitecture().GetTriple().getArch();
-    auto align = arch == llvm::Triple::msp430 ? 512 : 4096;
+    uint64_t align = target_sp->GetExprAllocAlign();
+    if (align == 0)
+      align = 4096;
     ret = llvm::alignTo(addr + alloc_size, align);
   }
 
diff --git a/lldb/source/Expression/LLVMUserExpression.cpp b/lldb/source/Expression/LLVMUserExpression.cpp
index af63a1629a647..9d01bfcb72200 100644
--- a/lldb/source/Expression/LLVMUserExpression.cpp
+++ b/lldb/source/Expression/LLVMUserExpression.cpp
@@ -23,6 +23,7 @@
 #include "lldb/Symbol/SymbolVendor.h"
 #include "lldb/Symbol/Type.h"
 #include "lldb/Symbol/VariableList.h"
+#include "lldb/Target/ABI.h"
 #include "lldb/Target/ExecutionContext.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/StackFrame.h"
@@ -34,6 +35,7 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
 
+using namespace lldb;
 using namespace lldb_private;
 
 char LLVMUserExpression::ID;
@@ -333,9 +335,14 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
     if (m_can_interpret && m_stack_frame_bottom == LLDB_INVALID_ADDRESS) {
       Status alloc_error;
 
-      auto arch = target->GetArchitecture().GetTriple().getArch();
-      const size_t stack_frame_size =
-          arch == llvm::Triple::msp430 ? 512 : 512 * 1024;
+      size_t stack_frame_size = target->GetExprAllocSize();
+      if (stack_frame_size == 0) {
+        ABISP abi_sp;
+        if (process && (abi_sp = process->GetABI()))
+          stack_frame_size = abi_sp->GetStackFrameSize();
+        else
+          stack_frame_size = 512 * 1024;
+      }
 
       const bool zero_memory = false;
 
diff --git a/lldb/source/Expression/ObjectFileJIT.cpp b/lldb/source/Expression/ObjectFileJIT.cpp
index 106652d4499d2..10fac9a9a4452 100644
--- a/lldb/source/Expression/ObjectFileJIT.cpp
+++ b/lldb/source/Expression/ObjectFileJIT.cpp
@@ -8,7 +8,6 @@
 
 #include "llvm/ADT/StringRef.h"
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
@@ -21,6 +20,7 @@
 #include "lldb/Utility/DataBuffer.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Timer.h"
 #include "lldb/Utility/UUID.h"
diff --git a/lldb/source/Expression/REPL.cpp b/lldb/source/Expression/REPL.cpp
index 5b1937245d967..31c9978b24226 100644
--- a/lldb/source/Expression/REPL.cpp
+++ b/lldb/source/Expression/REPL.cpp
@@ -342,9 +342,11 @@ void REPL::IOHandlerInputComplete(IOHandler &io_handler, std::string &code) {
                                    expr_prefix, result_valobj_sp, error,
                                    nullptr); // fixed expression
 
-      // CommandInterpreter &ci = debugger.GetCommandInterpreter();
-
-      if (process_sp && process_sp->IsAlive()) {
+      if (llvm::Error err = OnExpressionEvaluated(exe_ctx, code, expr_options,
+                                                  execution_results,
+                                                  result_valobj_sp, error)) {
+        *error_sp << llvm::toString(std::move(err)) << "\n";
+      } else if (process_sp && process_sp->IsAlive()) {
         bool add_to_code = true;
         bool handled = false;
         if (result_valobj_sp) {
diff --git a/lldb/source/Host/common/Host.cpp b/lldb/source/Host/common/Host.cpp
index 33b550008b746..49eac0b0fa7b0 100644
--- a/lldb/source/Host/common/Host.cpp
+++ b/lldb/source/Host/common/Host.cpp
@@ -546,9 +546,11 @@ void Host::Kill(lldb::pid_t pid, int signo) { ::kill(pid, signo); }
 #endif
 
 #if !defined(__APPLE__)
-bool Host::OpenFileInExternalEditor(const FileSpec &file_spec,
-                                    uint32_t line_no) {
-  return false;
+llvm::Error Host::OpenFileInExternalEditor(llvm::StringRef editor,
+                                           const FileSpec &file_spec,
+                                           uint32_t line_no) {
+  return llvm::errorCodeToError(
+      std::error_code(ENOTSUP, std::system_category()));
 }
 
 bool Host::IsInteractiveGraphicSession() { return false; }
diff --git a/lldb/source/Host/common/ProcessLaunchInfo.cpp b/lldb/source/Host/common/ProcessLaunchInfo.cpp
index 8e09601b5715b..e737815750b2c 100644
--- a/lldb/source/Host/common/ProcessLaunchInfo.cpp
+++ b/lldb/source/Host/common/ProcessLaunchInfo.cpp
@@ -31,8 +31,8 @@ using namespace lldb_private;
 
 ProcessLaunchInfo::ProcessLaunchInfo()
     : ProcessInfo(), m_working_dir(), m_plugin_name(), m_flags(0),
-      m_file_actions(), m_pty(new PseudoTerminal), m_monitor_callback(nullptr),
-      m_listener_sp(), m_hijack_listener_sp() {}
+      m_file_actions(), m_pty(new PseudoTerminal), m_monitor_callback(nullptr) {
+}
 
 ProcessLaunchInfo::ProcessLaunchInfo(const FileSpec &stdin_file_spec,
                                      const FileSpec &stdout_file_spec,
diff --git a/lldb/source/Host/linux/HostInfoLinux.cpp b/lldb/source/Host/linux/HostInfoLinux.cpp
index 018be54ecf7aa..c66f787db0cf9 100644
--- a/lldb/source/Host/linux/HostInfoLinux.cpp
+++ b/lldb/source/Host/linux/HostInfoLinux.cpp
@@ -200,17 +200,13 @@ void HostInfoLinux::ComputeHostArchitectureSupport(ArchSpec &arch_32,
                                                    ArchSpec &arch_64) {
   HostInfoPosix::ComputeHostArchitectureSupport(arch_32, arch_64);
 
-  const char *distribution_id = GetDistributionId().data();
-
   // On Linux, "unknown" in the vendor slot isn't what we want for the default
   // triple.  It's probably an artifact of config.guess.
   if (arch_32.IsValid()) {
-    arch_32.SetDistributionId(distribution_id);
     if (arch_32.GetTriple().getVendor() == llvm::Triple::UnknownVendor)
       arch_32.GetTriple().setVendorName(llvm::StringRef());
   }
   if (arch_64.IsValid()) {
-    arch_64.SetDistributionId(distribution_id);
     if (arch_64.GetTriple().getVendor() == llvm::Triple::UnknownVendor)
       arch_64.GetTriple().setVendorName(llvm::StringRef());
   }
diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index 303a138559e93..5a151b9ecc8ff 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -325,12 +325,37 @@ repeat with the_window in (get windows)\n\
 
 #endif // TARGET_OS_OSX
 
-bool Host::OpenFileInExternalEditor(const FileSpec &file_spec,
-                                    uint32_t line_no) {
+llvm::Error Host::OpenFileInExternalEditor(llvm::StringRef editor,
+                                           const FileSpec &file_spec,
+                                           uint32_t line_no) {
 #if !TARGET_OS_OSX
-  return false;
+  return llvm::errorCodeToError(
+      std::error_code(ENOTSUP, std::system_category()));
 #else // !TARGET_OS_OSX
-  // We attach this to an 'odoc' event to specify a particular selection
+  Log *log = GetLog(LLDBLog::Host);
+
+  const std::string file_path = file_spec.GetPath();
+
+  LLDB_LOG(log, "Sending {0}:{1} to external editor",
+           file_path.empty() ? "<invalid>" : file_path, line_no);
+
+  if (file_path.empty())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "no file specified");
+
+  CFCString file_cfstr(file_path.c_str(), kCFStringEncodingUTF8);
+  CFCReleaser<CFURLRef> file_URL = ::CFURLCreateWithFileSystemPath(
+      /*allocator=*/NULL,
+      /*filePath*/ file_cfstr.get(),
+      /*pathStyle=*/kCFURLPOSIXPathStyle,
+      /*isDirectory=*/false);
+
+  if (!file_URL.get())
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        llvm::formatv("could not create CFURL from path \"{0}\"", file_path));
+
+  // Create a new Apple Event descriptor.
   typedef struct {
     int16_t reserved0; // must be zero
     int16_t fLineNumber;
@@ -340,18 +365,7 @@ repeat with the_window in (get windows)\n\
     uint32_t reserved2; // must be zero
   } BabelAESelInfo;
 
-  Log *log = GetLog(LLDBLog::Host);
-  char file_path[PATH_MAX];
-  file_spec.GetPath(file_path, PATH_MAX);
-  CFCString file_cfstr(file_path, kCFStringEncodingUTF8);
-  CFCReleaser<CFURLRef> file_URL(::CFURLCreateWithFileSystemPath(
-      NULL, file_cfstr.get(), kCFURLPOSIXPathStyle, false));
-
-  LLDB_LOGF(log,
-            "Sending source file: \"%s\" and line: %d to external editor.\n",
-            file_path, line_no);
-
-  long error;
+  // We attach this to an 'odoc' event to specify a particular selection.
   BabelAESelInfo file_and_line_info = {
       0,                      // reserved0
       (int16_t)(line_no - 1), // fLineNumber (zero based line number)
@@ -362,64 +376,69 @@ repeat with the_window in (get windows)\n\
   };
 
   AEKeyDesc file_and_line_desc;
-
-  error = ::AECreateDesc(typeUTF8Text, &file_and_line_info,
-                         sizeof(file_and_line_info),
-                         &(file_and_line_desc.descContent));
-
-  if (error != noErr) {
-    LLDB_LOGF(log, "Error creating AEDesc: %ld.\n", error);
-    return false;
-  }
-
   file_and_line_desc.descKey = keyAEPosition;
+  long error = ::AECreateDesc(/*typeCode=*/typeUTF8Text,
+                              /*dataPtr=*/&file_and_line_info,
+                              /*dataSize=*/sizeof(file_and_line_info),
+                              /*result=*/&(file_and_line_desc.descContent));
+
+  if (error != noErr)
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        llvm::formatv("creating Apple Event descriptor failed: error {0}",
+                      error));
+
+  // Deallocate the descriptor on exit.
+  auto on_exit = llvm::make_scope_exit(
+      [&]() { AEDisposeDesc(&(file_and_line_desc.descContent)); });
+
+  if (editor.empty()) {
+    if (const char *lldb_external_editor = ::getenv("LLDB_EXTERNAL_EDITOR"))
+      editor = lldb_external_editor;
+  }
 
-  static std::string g_app_name;
-  static FSRef g_app_fsref;
+  std::optional<FSRef> app_fsref;
+  if (!editor.empty()) {
+    LLDB_LOG(log, "Looking for external editor: {0}", editor);
+
+    app_fsref.emplace();
+    CFCString editor_name(editor.data(), kCFStringEncodingUTF8);
+    long app_error = ::LSFindApplicationForInfo(
+        /*inCreator=*/kLSUnknownCreator, /*inBundleID=*/NULL,
+        /*inName=*/editor_name.get(), /*outAppRef=*/&(*app_fsref),
+        /*outAppURL=*/NULL);
+    if (app_error != noErr)
+      return llvm::createStringError(
+          llvm::inconvertibleErrorCode(),
+          llvm::formatv("could not find external editor \"{0}\": "
+                        "LSFindApplicationForInfo returned error {1}",
+                        editor, app_error));
+  }
 
+  // Build app launch parameters.
   LSApplicationParameters app_params;
   ::memset(&app_params, 0, sizeof(app_params));
   app_params.flags =
       kLSLaunchDefaults | kLSLaunchDontAddToRecents | kLSLaunchDontSwitch;
-
-  char *external_editor = ::getenv("LLDB_EXTERNAL_EDITOR");
-
-  if (external_editor) {
-    LLDB_LOGF(log, "Looking for external editor \"%s\".\n", external_editor);
-
-    if (g_app_name.empty() ||
-        strcmp(g_app_name.c_str(), external_editor) != 0) {
-      CFCString editor_name(external_editor, kCFStringEncodingUTF8);
-      error = ::LSFindApplicationForInfo(kLSUnknownCreator, NULL,
-                                         editor_name.get(), &g_app_fsref, NULL);
-
-      // If we found the app, then store away the name so we don't have to
-      // re-look it up.
-      if (error != noErr) {
-        LLDB_LOGF(log,
-                  "Could not find External Editor application, error: %ld.\n",
-                  error);
-        return false;
-      }
-    }
-    app_params.application = &g_app_fsref;
-  }
+  if (app_fsref)
+    app_params.application = &(*app_fsref);
 
   ProcessSerialNumber psn;
-  CFCReleaser<CFArrayRef> file_array(
-      CFArrayCreate(NULL, (const void **)file_URL.ptr_address(false), 1, NULL));
-  error = ::LSOpenURLsWithRole(file_array.get(), kLSRolesAll,
-                               &file_and_line_desc, &app_params, &psn, 1);
-
-  AEDisposeDesc(&(file_and_line_desc.descContent));
-
-  if (error != noErr) {
-    LLDB_LOGF(log, "LSOpenURLsWithRole failed, error: %ld.\n", error);
-
-    return false;
-  }
-
-  return true;
+  std::array<CFURLRef, 1> file_array = {file_URL.get()};
+  CFCReleaser<CFArrayRef> cf_array(
+      CFArrayCreate(/*allocator=*/NULL, /*values=*/(const void **)&file_array,
+                    /*numValues*/ 1, /*callBacks=*/NULL));
+  error = ::LSOpenURLsWithRole(
+      /*inURLs=*/cf_array.get(), /*inRole=*/kLSRolesEditor,
+      /*inAEParam=*/&file_and_line_desc,
+      /*inAppParams=*/&app_params, /*outPSNs=*/&psn, /*inMaxPSNCount=*/1);
+
+  if (error != noErr)
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        llvm::formatv("LSOpenURLsWithRole failed: error {0}", error));
+
+  return llvm::Error::success();
 #endif // TARGET_OS_OSX
 }
 
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 09fc8e216bc00..735f18dd5004b 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -146,73 +146,73 @@ CommandInterpreter::CommandInterpreter(Debugger &debugger,
 
 bool CommandInterpreter::GetExpandRegexAliases() const {
   const uint32_t idx = ePropertyExpandRegexAliases;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 bool CommandInterpreter::GetPromptOnQuit() const {
   const uint32_t idx = ePropertyPromptOnQuit;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 void CommandInterpreter::SetPromptOnQuit(bool enable) {
   const uint32_t idx = ePropertyPromptOnQuit;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
+  SetPropertyAtIndex(idx, enable);
 }
 
 bool CommandInterpreter::GetSaveSessionOnQuit() const {
   const uint32_t idx = ePropertySaveSessionOnQuit;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 void CommandInterpreter::SetSaveSessionOnQuit(bool enable) {
   const uint32_t idx = ePropertySaveSessionOnQuit;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
+  SetPropertyAtIndex(idx, enable);
 }
 
 bool CommandInterpreter::GetOpenTranscriptInEditor() const {
   const uint32_t idx = ePropertyOpenTranscriptInEditor;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 void CommandInterpreter::SetOpenTranscriptInEditor(bool enable) {
   const uint32_t idx = ePropertyOpenTranscriptInEditor;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
+  SetPropertyAtIndex(idx, enable);
 }
 
 FileSpec CommandInterpreter::GetSaveSessionDirectory() const {
   const uint32_t idx = ePropertySaveSessionDirectory;
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 void CommandInterpreter::SetSaveSessionDirectory(llvm::StringRef path) {
   const uint32_t idx = ePropertySaveSessionDirectory;
-  m_collection_sp->SetPropertyAtIndexAsString(nullptr, idx, path);
+  SetPropertyAtIndex(idx, path);
 }
 
 bool CommandInterpreter::GetEchoCommands() const {
   const uint32_t idx = ePropertyEchoCommands;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 void CommandInterpreter::SetEchoCommands(bool enable) {
   const uint32_t idx = ePropertyEchoCommands;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
+  SetPropertyAtIndex(idx, enable);
 }
 
 bool CommandInterpreter::GetEchoCommentCommands() const {
   const uint32_t idx = ePropertyEchoCommentCommands;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 void CommandInterpreter::SetEchoCommentCommands(bool enable) {
   const uint32_t idx = ePropertyEchoCommentCommands;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, enable);
+  SetPropertyAtIndex(idx, enable);
 }
 
 void CommandInterpreter::AllowExitCodeOnQuit(bool allow) {
@@ -246,26 +246,26 @@ void CommandInterpreter::ResolveCommand(const char *command_line,
 
 bool CommandInterpreter::GetStopCmdSourceOnError() const {
   const uint32_t idx = ePropertyStopCmdSourceOnError;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 bool CommandInterpreter::GetSpaceReplPrompts() const {
   const uint32_t idx = ePropertySpaceReplPrompts;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 bool CommandInterpreter::GetRepeatPreviousCommand() const {
   const uint32_t idx = ePropertyRepeatPreviousCommand;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 bool CommandInterpreter::GetRequireCommandOverwrite() const {
   const uint32_t idx = ePropertyRequireCommandOverwrite;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_interpreter_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_interpreter_properties[idx].default_uint_value != 0);
 }
 
 void CommandInterpreter::Initialize() {
@@ -1374,11 +1374,12 @@ bool CommandInterpreter::RemoveAlias(llvm::StringRef alias_name) {
   return false;
 }
 
-bool CommandInterpreter::RemoveCommand(llvm::StringRef cmd) {
+bool CommandInterpreter::RemoveCommand(llvm::StringRef cmd, bool force) {
   auto pos = m_command_dict.find(std::string(cmd));
   if (pos != m_command_dict.end()) {
-    if (pos->second->IsRemovable()) {
-      // Only regular expression objects or python commands are removable
+    if (force || pos->second->IsRemovable()) {
+      // Only regular expression objects or python commands are removable under
+      // normal circumstances.
       m_command_dict.erase(pos);
       return true;
     }
@@ -1689,20 +1690,19 @@ CommandObject *CommandInterpreter::BuildAliasResult(
           return nullptr;
         }
         llvm::StringRef arg_text = entry.ref();
-        if (strpos - start_fudge + arg_text.size() + len_fudge 
-            > raw_input_string.size()) {
+        if (strpos - start_fudge + arg_text.size() + len_fudge >
+            raw_input_string.size()) {
           result.AppendError("Unmatched quote at command end.");
-          return nullptr;  
+          return nullptr;
         }
         raw_input_string = raw_input_string.erase(
-            strpos - start_fudge, 
+            strpos - start_fudge,
             strlen(cmd_args.GetArgumentAtIndex(index)) + len_fudge);
       }
       if (quote_char == '\0')
         result_str.Printf("%s", cmd_args.GetArgumentAtIndex(index));
       else
-        result_str.Printf("%c%s%c", quote_char, 
-                          entry.c_str(), quote_char);
+        result_str.Printf("%c%s%c", quote_char, entry.c_str(), quote_char);
     }
   }
 
@@ -1759,7 +1759,7 @@ Status CommandInterpreter::PreprocessCommand(std::string &command) {
     command.insert(start_backtick, std::string(expr_str));
     pos = start_backtick + expr_str.size();
   }
-  return error;                        
+  return error;
 }
 
 Status
@@ -1794,7 +1794,7 @@ CommandInterpreter::PreprocessToken(std::string &expr_str) {
           expr_result_valobj_sp->GetQualifiedRepresentationIfAvailable(
               expr_result_valobj_sp->GetDynamicValueType(), true);
     if (expr_result_valobj_sp->ResolveValue(scalar)) {
-      
+
       StreamString value_strm;
       const bool show_type = false;
       scalar.GetValue(&value_strm, show_type);
@@ -1982,11 +1982,11 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
 
   CommandObject *cmd_obj = ResolveCommandImpl(command_string, result);
 
-  // We have to preprocess the whole command string for Raw commands, since we 
+  // We have to preprocess the whole command string for Raw commands, since we
   // don't know the structure of the command.  For parsed commands, we only
   // treat backticks as quote characters specially.
   // FIXME: We probably want to have raw commands do their own preprocessing.
-  // For instance, I don't think people expect substitution in expr expressions. 
+  // For instance, I don't think people expect substitution in expr expressions.
   if (cmd_obj && cmd_obj->WantsRawCommandString()) {
     Status error(PreprocessCommand(command_string));
 
@@ -3270,8 +3270,11 @@ bool CommandInterpreter::SaveTranscript(
   if (GetOpenTranscriptInEditor() && Host::IsInteractiveGraphicSession()) {
     const FileSpec file_spec;
     error = file->GetFileSpec(const_cast<FileSpec &>(file_spec));
-    if (error.Success())
-      Host::OpenFileInExternalEditor(file_spec, 1);
+    if (error.Success()) {
+      if (llvm::Error e = Host::OpenFileInExternalEditor(
+              m_debugger.GetExternalEditor(), file_spec, 1))
+        result.AppendError(llvm::toString(std::move(e)));
+    }
   }
 
   return true;
@@ -3391,8 +3394,8 @@ CommandInterpreterRunResult CommandInterpreter::RunCommandInterpreter(
     // If the current thread is not managed by a host thread, we won't detect
     // that this IS the CommandInterpreter IOHandler thread, so make it so:
     HostThread new_io_handler_thread(Host::GetCurrentThread());
-    HostThread old_io_handler_thread 
-        = m_debugger.SetIOHandlerThread(new_io_handler_thread);
+    HostThread old_io_handler_thread =
+        m_debugger.SetIOHandlerThread(new_io_handler_thread);
     m_debugger.RunIOHandlers();
     m_debugger.SetIOHandlerThread(old_io_handler_thread);
 
diff --git a/lldb/source/Interpreter/CommandObject.cpp b/lldb/source/Interpreter/CommandObject.cpp
index c13ce4ece4297..313d24f0657be 100644
--- a/lldb/source/Interpreter/CommandObject.cpp
+++ b/lldb/source/Interpreter/CommandObject.cpp
@@ -23,11 +23,11 @@
 
 // These are for the Sourcename completers.
 // FIXME: Make a separate file for the completers.
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/DataFormatters/FormatManager.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 
 #include "lldb/Target/Language.h"
 
diff --git a/lldb/source/Interpreter/OptionGroupWatchpoint.cpp b/lldb/source/Interpreter/OptionGroupWatchpoint.cpp
index 5559e82d47224..c2f78d8c2dd14 100644
--- a/lldb/source/Interpreter/OptionGroupWatchpoint.cpp
+++ b/lldb/source/Interpreter/OptionGroupWatchpoint.cpp
@@ -10,6 +10,7 @@
 
 #include "lldb/Host/OptionParser.h"
 #include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Target/Language.h"
 #include "lldb/lldb-enumerations.h"
 
 using namespace lldb;
@@ -62,7 +63,17 @@ static constexpr OptionDefinition g_option_table[] = {
      "Specify the type of watching to perform."},
     {LLDB_OPT_SET_1, false, "size", 's', OptionParser::eRequiredArgument,
      nullptr, OptionEnumValues(g_watch_size), 0, eArgTypeByteSize,
-     "Number of bytes to use to watch a region."}};
+     "Number of bytes to use to watch a region."},
+    {LLDB_OPT_SET_2,
+     false,
+     "language",
+     'l',
+     OptionParser::eRequiredArgument,
+     nullptr,
+     {},
+     0,
+     eArgTypeLanguage,
+     "Language of expression to run"}};
 
 bool OptionGroupWatchpoint::IsWatchSizeSupported(uint32_t watch_size) {
   for (const auto& size : g_watch_size) {
@@ -81,6 +92,18 @@ OptionGroupWatchpoint::SetOptionValue(uint32_t option_idx,
   Status error;
   const int short_option = g_option_table[option_idx].short_option;
   switch (short_option) {
+  case 'l': {
+    language_type = Language::GetLanguageTypeFromString(option_arg);
+    if (language_type == eLanguageTypeUnknown) {
+      StreamString sstr;
+      sstr.Printf("Unknown language type: '%s' for expression. List of "
+                  "supported languages:\n",
+                  option_arg.str().c_str());
+      Language::PrintSupportedLanguagesForExpressions(sstr, " ", "\n");
+      error.SetErrorString(sstr.GetString());
+    }
+    break;
+  }
   case 'w': {
     WatchType tmp_watch_type;
     tmp_watch_type = (WatchType)OptionArgParser::ToOptionEnum(
@@ -108,6 +131,7 @@ void OptionGroupWatchpoint::OptionParsingStarting(
   watch_type_specified = false;
   watch_type = eWatchInvalid;
   watch_size = 0;
+  language_type = eLanguageTypeUnknown;
 }
 
 llvm::ArrayRef<OptionDefinition> OptionGroupWatchpoint::GetDefinitions() {
diff --git a/lldb/source/Interpreter/OptionValue.cpp b/lldb/source/Interpreter/OptionValue.cpp
index 79bdf74b6fd9b..b9da1f122ebee 100644
--- a/lldb/source/Interpreter/OptionValue.cpp
+++ b/lldb/source/Interpreter/OptionValue.cpp
@@ -15,26 +15,6 @@
 using namespace lldb;
 using namespace lldb_private;
 
-// Get this value as a uint64_t value if it is encoded as a boolean, uint64_t
-// or int64_t. Other types will cause "fail_value" to be returned
-uint64_t OptionValue::GetUInt64Value(uint64_t fail_value, bool *success_ptr) {
-  if (success_ptr)
-    *success_ptr = true;
-  switch (GetType()) {
-  case OptionValue::eTypeBoolean:
-    return static_cast<OptionValueBoolean *>(this)->GetCurrentValue();
-  case OptionValue::eTypeSInt64:
-    return static_cast<OptionValueSInt64 *>(this)->GetCurrentValue();
-  case OptionValue::eTypeUInt64:
-    return static_cast<OptionValueUInt64 *>(this)->GetCurrentValue();
-  default:
-    break;
-  }
-  if (success_ptr)
-    *success_ptr = false;
-  return fail_value;
-}
-
 Status OptionValue::SetSubValue(const ExecutionContext *exe_ctx,
                                 VarSetOperationType op, llvm::StringRef name,
                                 llvm::StringRef value) {
@@ -271,11 +251,10 @@ const OptionValueUUID *OptionValue::GetAsUUID() const {
   return nullptr;
 }
 
-bool OptionValue::GetBooleanValue(bool fail_value) const {
-  const OptionValueBoolean *option_value = GetAsBoolean();
-  if (option_value)
+std::optional<bool> OptionValue::GetBooleanValue() const {
+  if (const OptionValueBoolean *option_value = GetAsBoolean())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetBooleanValue(bool new_value) {
@@ -287,11 +266,10 @@ bool OptionValue::SetBooleanValue(bool new_value) {
   return false;
 }
 
-char OptionValue::GetCharValue(char fail_value) const {
-  const OptionValueChar *option_value = GetAsChar();
-  if (option_value)
+std::optional<char> OptionValue::GetCharValue() const {
+  if (const OptionValueChar *option_value = GetAsChar())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 char OptionValue::SetCharValue(char new_value) {
@@ -303,11 +281,10 @@ char OptionValue::SetCharValue(char new_value) {
   return false;
 }
 
-int64_t OptionValue::GetEnumerationValue(int64_t fail_value) const {
-  const OptionValueEnumeration *option_value = GetAsEnumeration();
-  if (option_value)
+std::optional<int64_t> OptionValue::GetEnumerationValue() const {
+  if (const OptionValueEnumeration *option_value = GetAsEnumeration())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetEnumerationValue(int64_t value) {
@@ -319,14 +296,14 @@ bool OptionValue::SetEnumerationValue(int64_t value) {
   return false;
 }
 
-FileSpec OptionValue::GetFileSpecValue() const {
+std::optional<FileSpec> OptionValue::GetFileSpecValue() const {
   const OptionValueFileSpec *option_value = GetAsFileSpec();
   if (option_value)
     return option_value->GetCurrentValue();
-  return FileSpec();
+  return {};
 }
 
-bool OptionValue::SetFileSpecValue(const FileSpec &file_spec) {
+bool OptionValue::SetFileSpecValue(FileSpec file_spec) {
   OptionValueFileSpec *option_value = GetAsFileSpec();
   if (option_value) {
     option_value->SetCurrentValue(file_spec, false);
@@ -335,18 +312,24 @@ bool OptionValue::SetFileSpecValue(const FileSpec &file_spec) {
   return false;
 }
 
-FileSpecList OptionValue::GetFileSpecListValue() const {
-  const OptionValueFileSpecList *option_value = GetAsFileSpecList();
-  if (option_value)
+bool OptionValue::AppendFileSpecValue(FileSpec file_spec) {
+  if (OptionValueFileSpecList *option_value = GetAsFileSpecList()) {
+    option_value->AppendCurrentValue(file_spec);
+    return true;
+  }
+  return false;
+}
+
+std::optional<FileSpecList> OptionValue::GetFileSpecListValue() const {
+  if (const OptionValueFileSpecList *option_value = GetAsFileSpecList())
     return option_value->GetCurrentValue();
-  return FileSpecList();
+  return {};
 }
 
-lldb::Format OptionValue::GetFormatValue(lldb::Format fail_value) const {
-  const OptionValueFormat *option_value = GetAsFormat();
-  if (option_value)
+std::optional<lldb::Format> OptionValue::GetFormatValue() const {
+  if (const OptionValueFormat *option_value = GetAsFormat())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetFormatValue(lldb::Format new_value) {
@@ -358,12 +341,10 @@ bool OptionValue::SetFormatValue(lldb::Format new_value) {
   return false;
 }
 
-lldb::LanguageType
-OptionValue::GetLanguageValue(lldb::LanguageType fail_value) const {
-  const OptionValueLanguage *option_value = GetAsLanguage();
-  if (option_value)
+std::optional<lldb::LanguageType> OptionValue::GetLanguageValue() const {
+  if (const OptionValueLanguage *option_value = GetAsLanguage())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetLanguageValue(lldb::LanguageType new_language) {
@@ -389,11 +370,10 @@ const RegularExpression *OptionValue::GetRegexValue() const {
   return nullptr;
 }
 
-int64_t OptionValue::GetSInt64Value(int64_t fail_value) const {
-  const OptionValueSInt64 *option_value = GetAsSInt64();
-  if (option_value)
+std::optional<int64_t> OptionValue::GetSInt64Value() const {
+  if (const OptionValueSInt64 *option_value = GetAsSInt64())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetSInt64Value(int64_t new_value) {
@@ -405,11 +385,10 @@ bool OptionValue::SetSInt64Value(int64_t new_value) {
   return false;
 }
 
-llvm::StringRef OptionValue::GetStringValue(llvm::StringRef fail_value) const {
-  const OptionValueString *option_value = GetAsString();
-  if (option_value)
+std::optional<llvm::StringRef> OptionValue::GetStringValue() const {
+  if (const OptionValueString *option_value = GetAsString())
     return option_value->GetCurrentValueAsRef();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetStringValue(llvm::StringRef new_value) {
@@ -421,11 +400,10 @@ bool OptionValue::SetStringValue(llvm::StringRef new_value) {
   return false;
 }
 
-uint64_t OptionValue::GetUInt64Value(uint64_t fail_value) const {
-  const OptionValueUInt64 *option_value = GetAsUInt64();
-  if (option_value)
+std::optional<uint64_t> OptionValue::GetUInt64Value() const {
+  if (const OptionValueUInt64 *option_value = GetAsUInt64())
     return option_value->GetCurrentValue();
-  return fail_value;
+  return {};
 }
 
 bool OptionValue::SetUInt64Value(uint64_t new_value) {
diff --git a/lldb/source/Interpreter/OptionValueArgs.cpp b/lldb/source/Interpreter/OptionValueArgs.cpp
index 0e20154907b02..e9f73e631dfd1 100644
--- a/lldb/source/Interpreter/OptionValueArgs.cpp
+++ b/lldb/source/Interpreter/OptionValueArgs.cpp
@@ -15,10 +15,7 @@ using namespace lldb_private;
 
 size_t OptionValueArgs::GetArgs(Args &args) const {
   args.Clear();
-  for (const auto &value : m_values) {
-    llvm::StringRef string_value = value->GetStringValue();
-    args.AppendArgument(string_value);
-  }
-
+  for (const auto &value : m_values)
+    args.AppendArgument(value->GetStringValue().value_or(""));
   return args.GetArgumentCount();
 }
diff --git a/lldb/source/Interpreter/OptionValueArray.cpp b/lldb/source/Interpreter/OptionValueArray.cpp
index 40357d5f4b062..796b6529b6c88 100644
--- a/lldb/source/Interpreter/OptionValueArray.cpp
+++ b/lldb/source/Interpreter/OptionValueArray.cpp
@@ -94,8 +94,7 @@ Status OptionValueArray::SetValueFromString(llvm::StringRef value,
 
 lldb::OptionValueSP
 OptionValueArray::GetSubValue(const ExecutionContext *exe_ctx,
-                              llvm::StringRef name, bool will_modify,
-                              Status &error) const {
+                              llvm::StringRef name, Status &error) const {
   if (name.empty() || name.front() != '[') {
     error.SetErrorStringWithFormat(
       "invalid value path '%s', %s values only support '[<index>]' subvalues "
@@ -129,8 +128,7 @@ OptionValueArray::GetSubValue(const ExecutionContext *exe_ctx,
   if (new_idx < array_count) {
     if (m_values[new_idx]) {
       if (!sub_value.empty())
-        return m_values[new_idx]->GetSubValue(exe_ctx, sub_value,
-                                              will_modify, error);
+        return m_values[new_idx]->GetSubValue(exe_ctx, sub_value, error);
       else
         return m_values[new_idx];
     }
@@ -155,9 +153,9 @@ size_t OptionValueArray::GetArgs(Args &args) const {
   args.Clear();
   const uint32_t size = m_values.size();
   for (uint32_t i = 0; i < size; ++i) {
-    llvm::StringRef string_value = m_values[i]->GetStringValue();
-    if (!string_value.empty())
-      args.AppendArgument(string_value);
+    std::optional<llvm::StringRef> string_value = m_values[i]->GetStringValue();
+    if (string_value)
+      args.AppendArgument(*string_value);
   }
 
   return args.GetArgumentCount();
diff --git a/lldb/source/Interpreter/OptionValueDictionary.cpp b/lldb/source/Interpreter/OptionValueDictionary.cpp
index d18510166eeb7..c7d501fd45216 100644
--- a/lldb/source/Interpreter/OptionValueDictionary.cpp
+++ b/lldb/source/Interpreter/OptionValueDictionary.cpp
@@ -33,20 +33,24 @@ void OptionValueDictionary::DumpValue(const ExecutionContext *exe_ctx,
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" =");
 
-    collection::iterator pos, end = m_values.end();
-
     if (!one_line)
       strm.IndentMore();
 
-    for (pos = m_values.begin(); pos != end; ++pos) {
-      OptionValue *option_value = pos->second.get();
+    // m_values is not guaranteed to be sorted alphabetically, so for
+    // consistentcy we will sort them here before dumping
+    std::map<llvm::StringRef, OptionValue *> sorted_values;
+    for (const auto &value : m_values) {
+      sorted_values[value.first()] = value.second.get();
+    }
+    for (const auto &value : sorted_values) {
+      OptionValue *option_value = value.second;
 
       if (one_line)
         strm << ' ';
       else
         strm.EOL();
 
-      strm.Indent(pos->first.GetStringRef());
+      strm.Indent(value.first);
 
       const uint32_t extra_dump_options = m_raw_value_dump ? eDumpOptionRaw : 0;
       switch (dict_type) {
@@ -87,18 +91,17 @@ llvm::json::Value
 OptionValueDictionary::ToJSON(const ExecutionContext *exe_ctx) {
   llvm::json::Object dict;
   for (const auto &value : m_values) {
-    dict.try_emplace(value.first.GetCString(), value.second->ToJSON(exe_ctx));
+    dict.try_emplace(value.first(), value.second->ToJSON(exe_ctx));
   }
   return dict;
 }
 
 size_t OptionValueDictionary::GetArgs(Args &args) const {
   args.Clear();
-  collection::const_iterator pos, end = m_values.end();
-  for (pos = m_values.begin(); pos != end; ++pos) {
+  for (const auto &value : m_values) {
     StreamString strm;
-    strm.Printf("%s=", pos->first.GetCString());
-    pos->second->DumpValue(nullptr, strm, eDumpOptionValue | eDumpOptionRaw);
+    strm.Printf("%s=", value.first().data());
+    value.second->DumpValue(nullptr, strm, eDumpOptionValue | eDumpOptionRaw);
     args.AppendArgument(strm.GetString());
   }
   return args.GetArgumentCount();
@@ -178,7 +181,7 @@ Status OptionValueDictionary::SetArgs(const Args &args,
         if (error.Fail())
           return error;
         m_value_was_set = true;
-        SetValueForKey(ConstString(key), enum_value, true);
+        SetValueForKey(key, enum_value, true);
       } else {
         lldb::OptionValueSP value_sp(CreateValueFromCStringForTypeMask(
             value.str().c_str(), m_type_mask, error));
@@ -186,7 +189,7 @@ Status OptionValueDictionary::SetArgs(const Args &args,
           if (error.Fail())
             return error;
           m_value_was_set = true;
-          SetValueForKey(ConstString(key), value_sp, true);
+          SetValueForKey(key, value_sp, true);
         } else {
           error.SetErrorString("dictionaries that can contain multiple types "
                                "must subclass OptionValueArray");
@@ -198,11 +201,11 @@ Status OptionValueDictionary::SetArgs(const Args &args,
   case eVarSetOperationRemove:
     if (argc > 0) {
       for (size_t i = 0; i < argc; ++i) {
-        ConstString key(args.GetArgumentAtIndex(i));
+        llvm::StringRef key(args.GetArgumentAtIndex(i));
         if (!DeleteValueForKey(key)) {
           error.SetErrorStringWithFormat(
               "no value found named '%s', aborting remove operation",
-              key.GetCString());
+              key.data());
           break;
         }
       }
@@ -231,8 +234,7 @@ Status OptionValueDictionary::SetValueFromString(llvm::StringRef value,
 
 lldb::OptionValueSP
 OptionValueDictionary::GetSubValue(const ExecutionContext *exe_ctx,
-                                   llvm::StringRef name, bool will_modify,
-                                   Status &error) const {
+                                   llvm::StringRef name, Status &error) const {
   lldb::OptionValueSP value_sp;
   if (name.empty())
     return nullptr;
@@ -267,7 +269,7 @@ OptionValueDictionary::GetSubValue(const ExecutionContext *exe_ctx,
     return nullptr;
   }
 
-  value_sp = GetValueForKey(ConstString(key));
+  value_sp = GetValueForKey(key);
   if (!value_sp) {
     error.SetErrorStringWithFormat(
       "dictionary does not contain a value for the key name '%s'",
@@ -277,7 +279,7 @@ OptionValueDictionary::GetSubValue(const ExecutionContext *exe_ctx,
 
   if (sub_name.empty())
     return value_sp;
-  return value_sp->GetSubValue(exe_ctx, sub_name, will_modify, error);
+  return value_sp->GetSubValue(exe_ctx, sub_name, error);
 }
 
 Status OptionValueDictionary::SetSubValue(const ExecutionContext *exe_ctx,
@@ -285,8 +287,7 @@ Status OptionValueDictionary::SetSubValue(const ExecutionContext *exe_ctx,
                                           llvm::StringRef name,
                                           llvm::StringRef value) {
   Status error;
-  const bool will_modify = true;
-  lldb::OptionValueSP value_sp(GetSubValue(exe_ctx, name, will_modify, error));
+  lldb::OptionValueSP value_sp(GetSubValue(exe_ctx, name, error));
   if (value_sp)
     error = value_sp->SetValueFromString(value, op);
   else {
@@ -297,22 +298,22 @@ Status OptionValueDictionary::SetSubValue(const ExecutionContext *exe_ctx,
 }
 
 lldb::OptionValueSP
-OptionValueDictionary::GetValueForKey(ConstString key) const {
+OptionValueDictionary::GetValueForKey(llvm::StringRef key) const {
   lldb::OptionValueSP value_sp;
-  collection::const_iterator pos = m_values.find(key);
+  auto pos = m_values.find(key);
   if (pos != m_values.end())
     value_sp = pos->second;
   return value_sp;
 }
 
-bool OptionValueDictionary::SetValueForKey(ConstString key,
+bool OptionValueDictionary::SetValueForKey(llvm::StringRef key,
                                            const lldb::OptionValueSP &value_sp,
                                            bool can_replace) {
   // Make sure the value_sp object is allowed to contain values of the type
   // passed in...
   if (value_sp && (m_type_mask & value_sp->GetTypeAsMask())) {
     if (!can_replace) {
-      collection::const_iterator pos = m_values.find(key);
+      auto pos = m_values.find(key);
       if (pos != m_values.end())
         return false;
     }
@@ -322,8 +323,8 @@ bool OptionValueDictionary::SetValueForKey(ConstString key,
   return false;
 }
 
-bool OptionValueDictionary::DeleteValueForKey(ConstString key) {
-  collection::iterator pos = m_values.find(key);
+bool OptionValueDictionary::DeleteValueForKey(llvm::StringRef key) {
+  auto pos = m_values.find(key);
   if (pos != m_values.end()) {
     m_values.erase(pos);
     return true;
diff --git a/lldb/source/Interpreter/OptionValueProperties.cpp b/lldb/source/Interpreter/OptionValueProperties.cpp
index 46e677348352e..dafe398f9c8eb 100644
--- a/lldb/source/Interpreter/OptionValueProperties.cpp
+++ b/lldb/source/Interpreter/OptionValueProperties.cpp
@@ -22,15 +22,12 @@ using namespace lldb_private;
 
 OptionValueProperties::OptionValueProperties(ConstString name) : m_name(name) {}
 
-size_t OptionValueProperties::GetNumProperties() const {
-  return m_properties.size();
-}
-
 void OptionValueProperties::Initialize(const PropertyDefinitions &defs) {
   for (const auto &definition : defs) {
     Property property(definition);
     assert(property.IsValid());
-    m_name_to_index.Append(ConstString(property.GetName()), m_properties.size());
+    m_name_to_index.Append(ConstString(property.GetName()),
+                           m_properties.size());
     property.GetValue()->SetParent(shared_from_this());
     m_properties.push_back(property);
   }
@@ -75,19 +72,17 @@ void OptionValueProperties::AppendProperty(ConstString name,
 //
 lldb::OptionValueSP
 OptionValueProperties::GetValueForKey(const ExecutionContext *exe_ctx,
-                                      ConstString key,
-                                      bool will_modify) const {
+                                      ConstString key) const {
   lldb::OptionValueSP value_sp;
   size_t idx = m_name_to_index.Find(key, SIZE_MAX);
   if (idx < m_properties.size())
-    value_sp = GetPropertyAtIndex(exe_ctx, will_modify, idx)->GetValue();
+    value_sp = GetPropertyAtIndex(idx, exe_ctx)->GetValue();
   return value_sp;
 }
 
 lldb::OptionValueSP
 OptionValueProperties::GetSubValue(const ExecutionContext *exe_ctx,
-                                   llvm::StringRef name, bool will_modify,
-                                   Status &error) const {
+                                   llvm::StringRef name, Status &error) const {
   lldb::OptionValueSP value_sp;
   if (name.empty())
     return OptionValueSP();
@@ -101,7 +96,7 @@ OptionValueProperties::GetSubValue(const ExecutionContext *exe_ctx,
   } else
     key.SetString(name);
 
-  value_sp = GetValueForKey(exe_ctx, key, will_modify);
+  value_sp = GetValueForKey(exe_ctx, key);
   if (sub_name.empty() || !value_sp)
     return value_sp;
 
@@ -109,14 +104,14 @@ OptionValueProperties::GetSubValue(const ExecutionContext *exe_ctx,
   case '.': {
     lldb::OptionValueSP return_val_sp;
     return_val_sp =
-        value_sp->GetSubValue(exe_ctx, sub_name.drop_front(), will_modify, error);
+        value_sp->GetSubValue(exe_ctx, sub_name.drop_front(), error);
     if (!return_val_sp) {
       if (Properties::IsSettingExperimental(sub_name.drop_front())) {
         size_t experimental_len =
             strlen(Properties::GetExperimentalSettingsName());
         if (sub_name[experimental_len + 1] == '.')
           return_val_sp = value_sp->GetSubValue(
-              exe_ctx, sub_name.drop_front(experimental_len + 2), will_modify, error);
+              exe_ctx, sub_name.drop_front(experimental_len + 2), error);
         // It isn't an error if an experimental setting is not present.
         if (!return_val_sp)
           error.Clear();
@@ -127,7 +122,7 @@ OptionValueProperties::GetSubValue(const ExecutionContext *exe_ctx,
   case '[':
     // Array or dictionary access for subvalues like: "[12]"       -- access
     // 12th array element "['hello']"  -- dictionary access of key named hello
-    return value_sp->GetSubValue(exe_ctx, sub_name, will_modify, error);
+    return value_sp->GetSubValue(exe_ctx, sub_name, error);
 
   default:
     value_sp.reset();
@@ -141,7 +136,6 @@ Status OptionValueProperties::SetSubValue(const ExecutionContext *exe_ctx,
                                           llvm::StringRef name,
                                           llvm::StringRef value) {
   Status error;
-  const bool will_modify = true;
   llvm::SmallVector<llvm::StringRef, 8> components;
   name.split(components, '.');
   bool name_contains_experimental = false;
@@ -149,41 +143,33 @@ Status OptionValueProperties::SetSubValue(const ExecutionContext *exe_ctx,
     if (Properties::IsSettingExperimental(part))
       name_contains_experimental = true;
 
-  lldb::OptionValueSP value_sp(GetSubValue(exe_ctx, name, will_modify, error));
+  lldb::OptionValueSP value_sp(GetSubValue(exe_ctx, name, error));
   if (value_sp)
     error = value_sp->SetValueFromString(value, op);
   else {
     // Don't set an error if the path contained .experimental. - those are
     // allowed to be missing and should silently fail.
     if (!name_contains_experimental && error.AsCString() == nullptr) {
-      error.SetErrorStringWithFormat("invalid value path '%s'", name.str().c_str());
+      error.SetErrorStringWithFormat("invalid value path '%s'",
+                                     name.str().c_str());
     }
   }
   return error;
 }
 
-uint32_t
-OptionValueProperties::GetPropertyIndex(ConstString name) const {
+uint32_t OptionValueProperties::GetPropertyIndex(ConstString name) const {
   return m_name_to_index.Find(name, SIZE_MAX);
 }
 
 const Property *
-OptionValueProperties::GetProperty(const ExecutionContext *exe_ctx,
-                                   bool will_modify,
-                                   ConstString name) const {
-  return GetPropertyAtIndex(
-      exe_ctx, will_modify,
-      m_name_to_index.Find(name, SIZE_MAX));
-}
-
-const Property *OptionValueProperties::GetPropertyAtIndex(
-    const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const {
-  return ProtectedGetPropertyAtIndex(idx);
+OptionValueProperties::GetProperty(ConstString name,
+                                   const ExecutionContext *exe_ctx) const {
+  return GetPropertyAtIndex(m_name_to_index.Find(name, SIZE_MAX), exe_ctx);
 }
 
 lldb::OptionValueSP OptionValueProperties::GetPropertyValueAtIndex(
-    const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const {
-  const Property *setting = GetPropertyAtIndex(exe_ctx, will_modify, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *setting = GetPropertyAtIndex(idx, exe_ctx);
   if (setting)
     return setting->GetValue();
   return OptionValueSP();
@@ -191,8 +177,8 @@ lldb::OptionValueSP OptionValueProperties::GetPropertyValueAtIndex(
 
 OptionValuePathMappings *
 OptionValueProperties::GetPropertyAtIndexAsOptionValuePathMappings(
-    const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const {
-  OptionValueSP value_sp(GetPropertyValueAtIndex(exe_ctx, will_modify, idx));
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  OptionValueSP value_sp(GetPropertyValueAtIndex(idx, exe_ctx));
   if (value_sp)
     return value_sp->GetAsPathMappings();
   return nullptr;
@@ -200,16 +186,16 @@ OptionValueProperties::GetPropertyAtIndexAsOptionValuePathMappings(
 
 OptionValueFileSpecList *
 OptionValueProperties::GetPropertyAtIndexAsOptionValueFileSpecList(
-    const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const {
-  OptionValueSP value_sp(GetPropertyValueAtIndex(exe_ctx, will_modify, idx));
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  OptionValueSP value_sp(GetPropertyValueAtIndex(idx, exe_ctx));
   if (value_sp)
     return value_sp->GetAsFileSpecList();
   return nullptr;
 }
 
 OptionValueArch *OptionValueProperties::GetPropertyAtIndexAsOptionValueArch(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (property)
     return property->GetValue()->GetAsArch();
   return nullptr;
@@ -217,27 +203,16 @@ OptionValueArch *OptionValueProperties::GetPropertyAtIndexAsOptionValueArch(
 
 OptionValueLanguage *
 OptionValueProperties::GetPropertyAtIndexAsOptionValueLanguage(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (property)
     return property->GetValue()->GetAsLanguage();
   return nullptr;
 }
 
-bool OptionValueProperties::SetPropertyAtIndexAsLanguage(
-    const ExecutionContext *exe_ctx, uint32_t idx, const LanguageType lang) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->SetLanguageValue(lang);
-  }
-  return false;
-}
-
 bool OptionValueProperties::GetPropertyAtIndexAsArgs(
-    const ExecutionContext *exe_ctx, uint32_t idx, Args &args) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, Args &args, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (!property)
     return false;
 
@@ -267,8 +242,8 @@ bool OptionValueProperties::GetPropertyAtIndexAsArgs(
 }
 
 bool OptionValueProperties::SetPropertyAtIndexFromArgs(
-    const ExecutionContext *exe_ctx, uint32_t idx, const Args &args) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
+    uint32_t idx, const Args &args, const ExecutionContext *exe_ctx) {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (!property)
     return false;
 
@@ -291,77 +266,19 @@ bool OptionValueProperties::SetPropertyAtIndexFromArgs(
   return false;
 }
 
-bool OptionValueProperties::GetPropertyAtIndexAsBoolean(
-    const ExecutionContext *exe_ctx, uint32_t idx, bool fail_value) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetBooleanValue(fail_value);
-  }
-  return fail_value;
-}
-
-bool OptionValueProperties::SetPropertyAtIndexAsBoolean(
-    const ExecutionContext *exe_ctx, uint32_t idx, bool new_value) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value) {
-      value->SetBooleanValue(new_value);
-      return true;
-    }
-  }
-  return false;
-}
-
 OptionValueDictionary *
 OptionValueProperties::GetPropertyAtIndexAsOptionValueDictionary(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (property)
     return property->GetValue()->GetAsDictionary();
   return nullptr;
 }
 
-int64_t OptionValueProperties::GetPropertyAtIndexAsEnumeration(
-    const ExecutionContext *exe_ctx, uint32_t idx, int64_t fail_value) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetEnumerationValue(fail_value);
-  }
-  return fail_value;
-}
-
-bool OptionValueProperties::SetPropertyAtIndexAsEnumeration(
-    const ExecutionContext *exe_ctx, uint32_t idx, int64_t new_value) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->SetEnumerationValue(new_value);
-  }
-  return false;
-}
-
-const FormatEntity::Entry *
-OptionValueProperties::GetPropertyAtIndexAsFormatEntity(
-    const ExecutionContext *exe_ctx, uint32_t idx) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetFormatEntity();
-  }
-  return nullptr;
-}
-
 OptionValueFileSpec *
 OptionValueProperties::GetPropertyAtIndexAsOptionValueFileSpec(
-    const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (property) {
     OptionValue *value = property->GetValue().get();
     if (value)
@@ -370,44 +287,9 @@ OptionValueProperties::GetPropertyAtIndexAsOptionValueFileSpec(
   return nullptr;
 }
 
-FileSpec OptionValueProperties::GetPropertyAtIndexAsFileSpec(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetFileSpecValue();
-  }
-  return FileSpec();
-}
-
-bool OptionValueProperties::SetPropertyAtIndexAsFileSpec(
-    const ExecutionContext *exe_ctx, uint32_t idx,
-    const FileSpec &new_file_spec) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->SetFileSpecValue(new_file_spec);
-  }
-  return false;
-}
-
-const RegularExpression *
-OptionValueProperties::GetPropertyAtIndexAsOptionValueRegex(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetRegexValue();
-  }
-  return nullptr;
-}
-
 OptionValueSInt64 *OptionValueProperties::GetPropertyAtIndexAsOptionValueSInt64(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (property) {
     OptionValue *value = property->GetValue().get();
     if (value)
@@ -417,8 +299,8 @@ OptionValueSInt64 *OptionValueProperties::GetPropertyAtIndexAsOptionValueSInt64(
 }
 
 OptionValueUInt64 *OptionValueProperties::GetPropertyAtIndexAsOptionValueUInt64(
-    const ExecutionContext *exe_ctx, uint32_t idx) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  const Property *property = GetPropertyAtIndex(idx, exe_ctx);
   if (property) {
     OptionValue *value = property->GetValue().get();
     if (value)
@@ -427,81 +309,14 @@ OptionValueUInt64 *OptionValueProperties::GetPropertyAtIndexAsOptionValueUInt64(
   return nullptr;
 }
 
-int64_t OptionValueProperties::GetPropertyAtIndexAsSInt64(
-    const ExecutionContext *exe_ctx, uint32_t idx, int64_t fail_value) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetSInt64Value(fail_value);
-  }
-  return fail_value;
-}
-
-bool OptionValueProperties::SetPropertyAtIndexAsSInt64(
-    const ExecutionContext *exe_ctx, uint32_t idx, int64_t new_value) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->SetSInt64Value(new_value);
-  }
-  return false;
-}
-
-llvm::StringRef OptionValueProperties::GetPropertyAtIndexAsString(
-    const ExecutionContext *exe_ctx, uint32_t idx,
-    llvm::StringRef fail_value) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetStringValue(fail_value);
-  }
-  return fail_value;
-}
-
-bool OptionValueProperties::SetPropertyAtIndexAsString(
-    const ExecutionContext *exe_ctx, uint32_t idx, llvm::StringRef new_value) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->SetStringValue(new_value);
-  }
-  return false;
-}
-
 OptionValueString *OptionValueProperties::GetPropertyAtIndexAsOptionValueString(
-    const ExecutionContext *exe_ctx, bool will_modify, uint32_t idx) const {
-  OptionValueSP value_sp(GetPropertyValueAtIndex(exe_ctx, will_modify, idx));
+    uint32_t idx, const ExecutionContext *exe_ctx) const {
+  OptionValueSP value_sp(GetPropertyValueAtIndex(idx, exe_ctx));
   if (value_sp)
     return value_sp->GetAsString();
   return nullptr;
 }
 
-uint64_t OptionValueProperties::GetPropertyAtIndexAsUInt64(
-    const ExecutionContext *exe_ctx, uint32_t idx, uint64_t fail_value) const {
-  const Property *property = GetPropertyAtIndex(exe_ctx, false, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->GetUInt64Value(fail_value);
-  }
-  return fail_value;
-}
-
-bool OptionValueProperties::SetPropertyAtIndexAsUInt64(
-    const ExecutionContext *exe_ctx, uint32_t idx, uint64_t new_value) {
-  const Property *property = GetPropertyAtIndex(exe_ctx, true, idx);
-  if (property) {
-    OptionValue *value = property->GetValue().get();
-    if (value)
-      return value->SetUInt64Value(new_value);
-  }
-  return false;
-}
-
 void OptionValueProperties::Clear() {
   const size_t num_properties = m_properties.size();
   for (size_t i = 0; i < num_properties; ++i)
@@ -537,7 +352,7 @@ void OptionValueProperties::DumpValue(const ExecutionContext *exe_ctx,
                                       Stream &strm, uint32_t dump_mask) {
   const size_t num_properties = m_properties.size();
   for (size_t i = 0; i < num_properties; ++i) {
-    const Property *property = GetPropertyAtIndex(exe_ctx, false, i);
+    const Property *property = GetPropertyAtIndex(i, exe_ctx);
     if (property) {
       OptionValue *option_value = property->GetValue().get();
       assert(option_value);
@@ -554,7 +369,7 @@ OptionValueProperties::ToJSON(const ExecutionContext *exe_ctx) {
   llvm::json::Object json_properties;
   const size_t num_properties = m_properties.size();
   for (size_t i = 0; i < num_properties; ++i) {
-    const Property *property = GetPropertyAtIndex(exe_ctx, false, i);
+    const Property *property = GetPropertyAtIndex(i, exe_ctx);
     if (property) {
       OptionValue *option_value = property->GetValue().get();
       assert(option_value);
@@ -568,11 +383,10 @@ OptionValueProperties::ToJSON(const ExecutionContext *exe_ctx) {
 Status OptionValueProperties::DumpPropertyValue(const ExecutionContext *exe_ctx,
                                                 Stream &strm,
                                                 llvm::StringRef property_path,
-                                                uint32_t dump_mask, bool is_json) {
+                                                uint32_t dump_mask,
+                                                bool is_json) {
   Status error;
-  const bool will_modify = false;
-  lldb::OptionValueSP value_sp(
-      GetSubValue(exe_ctx, property_path, will_modify, error));
+  lldb::OptionValueSP value_sp(GetSubValue(exe_ctx, property_path, error));
   if (value_sp) {
     if (!value_sp->ValueIsTransparent()) {
       if (dump_mask & eDumpOptionName)
@@ -581,7 +395,9 @@ Status OptionValueProperties::DumpPropertyValue(const ExecutionContext *exe_ctx,
         strm.PutChar(' ');
     }
     if (is_json) {
-      strm.Printf("%s", llvm::formatv("{0:2}", value_sp->ToJSON(exe_ctx)).str().c_str());
+      strm.Printf(
+          "%s",
+          llvm::formatv("{0:2}", value_sp->ToJSON(exe_ctx)).str().c_str());
     } else
       value_sp->DumpValue(exe_ctx, strm, dump_mask);
   }
@@ -616,8 +432,9 @@ OptionValueProperties::DeepCopy(const OptionValueSP &new_parent) const {
   return copy_sp;
 }
 
-const Property *OptionValueProperties::GetPropertyAtPath(
-    const ExecutionContext *exe_ctx, bool will_modify, llvm::StringRef name) const {
+const Property *
+OptionValueProperties::GetPropertyAtPath(const ExecutionContext *exe_ctx,
+                                         llvm::StringRef name) const {
   const Property *property = nullptr;
   if (name.empty())
     return nullptr;
@@ -631,7 +448,7 @@ const Property *OptionValueProperties::GetPropertyAtPath(
   } else
     key.SetString(name);
 
-  property = GetProperty(exe_ctx, will_modify, key);
+  property = GetProperty(key, exe_ctx);
   if (sub_name.empty() || !property)
     return property;
 
@@ -639,8 +456,7 @@ const Property *OptionValueProperties::GetPropertyAtPath(
     OptionValueProperties *sub_properties =
         property->GetValue()->GetAsProperties();
     if (sub_properties)
-      return sub_properties->GetPropertyAtPath(exe_ctx, will_modify,
-                                                sub_name.drop_front());
+      return sub_properties->GetPropertyAtPath(exe_ctx, sub_name.drop_front());
   }
   return nullptr;
 }
@@ -694,7 +510,7 @@ void OptionValueProperties::Apropos(
 lldb::OptionValuePropertiesSP
 OptionValueProperties::GetSubProperty(const ExecutionContext *exe_ctx,
                                       ConstString name) {
-  lldb::OptionValueSP option_value_sp(GetValueForKey(exe_ctx, name, false));
+  lldb::OptionValueSP option_value_sp(GetValueForKey(exe_ctx, name));
   if (option_value_sp) {
     OptionValueProperties *ov_properties = option_value_sp->GetAsProperties();
     if (ov_properties)
diff --git a/lldb/source/Interpreter/Property.cpp b/lldb/source/Interpreter/Property.cpp
index 681596224d31a..56e45363be89a 100644
--- a/lldb/source/Interpreter/Property.cpp
+++ b/lldb/source/Interpreter/Property.cpp
@@ -226,6 +226,7 @@ Property::Property(const PropertyDefinition &definition)
     }
     break;
   }
+  assert(m_value_sp && "invalid property definition");
 }
 
 Property::Property(llvm::StringRef name, llvm::StringRef desc, bool is_global,
diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp
index bb3b769a41b5d..fb3fa74d0b978 100644
--- a/lldb/source/Interpreter/ScriptInterpreter.cpp
+++ b/lldb/source/Interpreter/ScriptInterpreter.cpp
@@ -80,6 +80,11 @@ ScriptInterpreter::GetDataExtractorFromSBData(const lldb::SBData &data) const {
   return data.m_opaque_sp;
 }
 
+lldb::BreakpointSP ScriptInterpreter::GetOpaqueTypeFromSBBreakpoint(
+    const lldb::SBBreakpoint &breakpoint) const {
+  return breakpoint.m_opaque_wp.lock();
+}
+
 lldb::ProcessAttachInfoSP ScriptInterpreter::GetOpaqueTypeFromSBAttachInfo(
     const lldb::SBAttachInfo &attach_info) const {
   return attach_info.m_opaque_sp;
diff --git a/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.h b/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.h
index 91f48a8416112..8ef39e7aa1bb7 100644
--- a/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.h
+++ b/lldb/source/Plugins/ABI/MSP430/ABISysV_msp430.h
@@ -55,6 +55,8 @@ class ABISysV_msp430 : public lldb_private::RegInfoBasedABI {
   const lldb_private::RegisterInfo *
   GetRegisterInfoArray(uint32_t &count) override;
 
+  uint64_t GetStackFrameSize() override { return 512; }
+
   //------------------------------------------------------------------
   // Static Functions
   //------------------------------------------------------------------
diff --git a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
index e84a7d6a047e8..846fdbbaa4aba 100644
--- a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
+++ b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
@@ -113,16 +113,17 @@ class DynamicLoaderDarwinKernelProperties : public Properties {
 
   bool GetLoadKexts() const {
     const uint32_t idx = ePropertyLoadKexts;
-    return m_collection_sp->GetPropertyAtIndexAsBoolean(
-        nullptr, idx,
+    return GetPropertyAtIndexAs<bool>(
+        idx,
         g_dynamicloaderdarwinkernel_properties[idx].default_uint_value != 0);
   }
 
   KASLRScanType GetScanType() const {
     const uint32_t idx = ePropertyScanType;
-    return (KASLRScanType)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-        nullptr, idx,
-        g_dynamicloaderdarwinkernel_properties[idx].default_uint_value);
+    return GetPropertyAtIndexAs<KASLRScanType>(
+        idx,
+        static_cast<KASLRScanType>(
+            g_dynamicloaderdarwinkernel_properties[idx].default_uint_value));
   }
 };
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h
index 3a87570bf8041..7be0ce6c7ae57 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h
@@ -9,7 +9,7 @@
 #ifndef LLDB_SOURCE_PLUGINS_EXPRESSIONPARSER_CLANG_CPPMODULECONFIGURATION_H
 #define LLDB_SOURCE_PLUGINS_EXPRESSIONPARSER_CLANG_CPPMODULECONFIGURATION_H
 
-#include <lldb/Core/FileSpecList.h>
+#include <lldb/Utility/FileSpecList.h>
 #include <llvm/Support/Regex.h>
 
 namespace lldb_private {
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
index 115527a1ec02d..a8cc86c3715ae 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
@@ -18,7 +18,6 @@
 #include "lldb/Interpreter/OptionValueDictionary.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Utility/ArchSpec.h"
-#include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Stream.h"
 
 #include "Plugins/Process/Utility/ARMDefines.h"
@@ -14353,9 +14352,9 @@ bool EmulateInstructionARM::TestEmulation(Stream *out_stream, ArchSpec &arch,
     return false;
   }
 
-  static ConstString opcode_key("opcode");
-  static ConstString before_key("before_state");
-  static ConstString after_key("after_state");
+  static constexpr llvm::StringLiteral opcode_key("opcode");
+  static constexpr llvm::StringLiteral before_key("before_state");
+  static constexpr llvm::StringLiteral after_key("after_state");
 
   OptionValueSP value_sp = test_data->GetValueForKey(opcode_key);
 
@@ -14365,7 +14364,7 @@ bool EmulateInstructionARM::TestEmulation(Stream *out_stream, ArchSpec &arch,
     out_stream->Printf("TestEmulation: Error reading opcode from test file.\n");
     return false;
   }
-  test_opcode = value_sp->GetUInt64Value();
+  test_opcode = value_sp->GetUInt64Value().value_or(0);
 
   if (arch.GetTriple().getArch() == llvm::Triple::thumb ||
       arch.IsAlwaysThumbInstructions()) {
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
index 7b8849ede9c7d..1260b45b9d97b 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
@@ -11,7 +11,6 @@
 
 #include "Plugins/Process/Utility/ARMDefines.h"
 #include "lldb/Core/EmulateInstruction.h"
-#include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Status.h"
 #include <optional>
 
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulationStateARM.cpp b/lldb/source/Plugins/Instruction/ARM/EmulationStateARM.cpp
index ca118375b8abc..fd7875599d887 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulationStateARM.cpp
+++ b/lldb/source/Plugins/Instruction/ARM/EmulationStateARM.cpp
@@ -264,11 +264,10 @@ bool EmulationStateARM::LoadRegistersStateFromDictionary(
   for (int i = 0; i < num; ++i) {
     sstr.Clear();
     sstr.Printf("%c%d", kind, i);
-    OptionValueSP value_sp =
-        reg_dict->GetValueForKey(ConstString(sstr.GetString()));
+    OptionValueSP value_sp = reg_dict->GetValueForKey(sstr.GetString());
     if (value_sp.get() == nullptr)
       return false;
-    uint64_t reg_value = value_sp->GetUInt64Value();
+    uint64_t reg_value = value_sp->GetUInt64Value().value_or(0);
     StorePseudoRegisterValue(first_reg + i, reg_value);
   }
 
@@ -277,8 +276,8 @@ bool EmulationStateARM::LoadRegistersStateFromDictionary(
 
 bool EmulationStateARM::LoadStateFromDictionary(
     OptionValueDictionary *test_data) {
-  static ConstString memory_key("memory");
-  static ConstString registers_key("registers");
+  static constexpr llvm::StringLiteral memory_key("memory");
+  static constexpr llvm::StringLiteral registers_key("registers");
 
   if (!test_data)
     return false;
@@ -288,8 +287,8 @@ bool EmulationStateARM::LoadStateFromDictionary(
   // Load memory, if present.
 
   if (value_sp.get() != nullptr) {
-    static ConstString address_key("address");
-    static ConstString data_key("data");
+    static constexpr llvm::StringLiteral address_key("address");
+    static constexpr llvm::StringLiteral data_key("data");
     uint64_t start_address = 0;
 
     OptionValueDictionary *mem_dict = value_sp->GetAsDictionary();
@@ -297,7 +296,7 @@ bool EmulationStateARM::LoadStateFromDictionary(
     if (value_sp.get() == nullptr)
       return false;
     else
-      start_address = value_sp->GetUInt64Value();
+      start_address = value_sp->GetUInt64Value().value_or(0);
 
     value_sp = mem_dict->GetValueForKey(data_key);
     OptionValueArray *mem_array = value_sp->GetAsArray();
@@ -311,7 +310,7 @@ bool EmulationStateARM::LoadStateFromDictionary(
       value_sp = mem_array->GetValueAtIndex(i);
       if (value_sp.get() == nullptr)
         return false;
-      uint64_t value = value_sp->GetUInt64Value();
+      uint64_t value = value_sp->GetUInt64Value().value_or(0);
       StoreToPseudoAddress(address, value);
       address = address + 4;
     }
@@ -327,11 +326,11 @@ bool EmulationStateARM::LoadStateFromDictionary(
   if (!LoadRegistersStateFromDictionary(reg_dict, 'r', dwarf_r0, 16))
     return false;
 
-  static ConstString cpsr_name("cpsr");
+  static constexpr llvm::StringLiteral cpsr_name("cpsr");
   value_sp = reg_dict->GetValueForKey(cpsr_name);
   if (value_sp.get() == nullptr)
     return false;
-  StorePseudoRegisterValue(dwarf_cpsr, value_sp->GetUInt64Value());
+  StorePseudoRegisterValue(dwarf_cpsr, value_sp->GetUInt64Value().value_or(0));
 
   // Load s/d Registers
   // To prevent you giving both types in a state and overwriting
diff --git a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
index c0321b6a2b59f..a0b02d6b848f7 100644
--- a/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
+++ b/lldb/source/Plugins/Instruction/ARM64/EmulateInstructionARM64.cpp
@@ -12,7 +12,6 @@
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Symbol/UnwindPlan.h"
 #include "lldb/Utility/ArchSpec.h"
-#include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Stream.h"
 
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index cebe8d8441607..946e4b356a363 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -94,7 +94,7 @@ const char *thread_sanitizer_retrieve_report_data_command = R"(
 const int REPORT_TRACE_SIZE = 128;
 const int REPORT_ARRAY_SIZE = 4;
 
-struct data {
+struct {
     void *report;
     const char *description;
     int report_count;
diff --git a/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp b/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp
index a9868336f02e3..b78f45cf1efdf 100644
--- a/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp
+++ b/lldb/source/Plugins/JITLoader/GDB/JITLoaderGDB.cpp
@@ -99,9 +99,10 @@ class PluginProperties : public Properties {
   }
 
   EnableJITLoaderGDB GetEnable() const {
-    return (EnableJITLoaderGDB)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-        nullptr, ePropertyEnable,
-        g_jitloadergdb_properties[ePropertyEnable].default_uint_value);
+    return GetPropertyAtIndexAs<EnableJITLoaderGDB>(
+        ePropertyEnable,
+        static_cast<EnableJITLoaderGDB>(
+            g_jitloadergdb_properties[ePropertyEnable].default_uint_value));
   }
 };
 } // namespace
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index faa33ecefc2d0..236a8129eeb34 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -46,6 +46,38 @@ lldb::ValueObjectSP lldb_private::formatters::GetChildMemberWithName(
   return {};
 }
 
+lldb::ValueObjectSP
+lldb_private::formatters::GetFirstValueOfLibCXXCompressedPair(
+    ValueObject &pair) {
+  ValueObjectSP value;
+  ValueObjectSP first_child = pair.GetChildAtIndex(0, true);
+  if (first_child)
+    value = first_child->GetChildMemberWithName(ConstString("__value_"), true);
+  if (!value) {
+    // pre-r300140 member name
+    value = pair.GetChildMemberWithName(ConstString("__first_"), true);
+  }
+  return value;
+}
+
+lldb::ValueObjectSP
+lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
+    ValueObject &pair) {
+  ValueObjectSP value;
+  if (pair.GetNumChildren() > 1) {
+    ValueObjectSP second_child = pair.GetChildAtIndex(1, true);
+    if (second_child) {
+      value =
+          second_child->GetChildMemberWithName(ConstString("__value_"), true);
+    }
+  }
+  if (!value) {
+    // pre-r300140 member name
+    value = pair.GetChildMemberWithName(ConstString("__second_"), true);
+  }
+  return value;
+}
+
 bool lldb_private::formatters::LibcxxOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   ValueObjectSP valobj_sp(valobj.GetNonSyntheticValue());
@@ -170,7 +202,7 @@ bool lldb_private::formatters::LibcxxUniquePointerSummaryProvider(
   if (!ptr_sp)
     return false;
 
-  ptr_sp = GetValueOfLibCXXCompressedPair(*ptr_sp);
+  ptr_sp = GetFirstValueOfLibCXXCompressedPair(*ptr_sp);
   if (!ptr_sp)
     return false;
 
@@ -658,7 +690,9 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
 
 size_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
     CalculateNumChildren() {
-  return (m_value_ptr_sp ? 1 : 0);
+  if (m_value_ptr_sp)
+    return m_deleter_sp ? 2 : 1;
+  return 0;
 }
 
 lldb::ValueObjectSP
@@ -670,7 +704,10 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::GetChildAtIndex(
   if (idx == 0)
     return m_value_ptr_sp;
 
-  if (idx == 1) {
+  if (idx == 1)
+    return m_deleter_sp;
+
+  if (idx == 2) {
     Status status;
     auto value_sp = m_value_ptr_sp->Dereference(status);
     if (status.Success()) {
@@ -691,7 +728,15 @@ bool lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::Update() {
   if (!ptr_sp)
     return false;
 
-  m_value_ptr_sp = GetValueOfLibCXXCompressedPair(*ptr_sp);
+  // Retrieve the actual pointer and the deleter, and clone them to give them
+  // user-friendly names.
+  ValueObjectSP value_pointer_sp = GetFirstValueOfLibCXXCompressedPair(*ptr_sp);
+  if (value_pointer_sp)
+    m_value_ptr_sp = value_pointer_sp->Clone(ConstString("pointer"));
+
+  ValueObjectSP deleter_sp = GetSecondValueOfLibCXXCompressedPair(*ptr_sp);
+  if (deleter_sp)
+    m_deleter_sp = deleter_sp->Clone(ConstString("deleter"));
 
   return false;
 }
@@ -703,10 +748,12 @@ bool lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
 
 size_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
-  if (name == "__value_")
+  if (name == "pointer")
     return 0;
-  if (name == "$$dereference$$")
+  if (name == "deleter")
     return 1;
+  if (name == "$$dereference$$")
+    return 2;
   return UINT32_MAX;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index 9ab22153a92a8..2dae71b24419d 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -23,6 +23,10 @@ lldb::ValueObjectSP
 GetChildMemberWithName(ValueObject &obj,
                        llvm::ArrayRef<ConstString> alternative_names);
 
+lldb::ValueObjectSP GetFirstValueOfLibCXXCompressedPair(ValueObject &pair);
+lldb::ValueObjectSP GetSecondValueOfLibCXXCompressedPair(ValueObject &pair);
+
+
 bool LibcxxStringSummaryProviderASCII(
     ValueObject &valobj, Stream &stream,
     const TypeSummaryOptions &summary_options); // libc++ std::string
@@ -200,6 +204,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 
 private:
   lldb::ValueObjectSP m_value_ptr_sp;
+  lldb::ValueObjectSP m_deleter_sp;
 };
 
 SyntheticChildrenFrontEnd *
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index a1953a1c7a227..3ae26a3012ac8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -300,7 +300,7 @@ bool ForwardListFrontEnd::Update() {
       m_backend.GetChildMemberWithName(ConstString("__before_begin_"), true));
   if (!impl_sp)
     return false;
-  impl_sp = GetValueOfLibCXXCompressedPair(*impl_sp);
+  impl_sp = GetFirstValueOfLibCXXCompressedPair(*impl_sp);
   if (!impl_sp)
     return false;
   m_head = impl_sp->GetChildMemberWithName(ConstString("__next_"), true).get();
@@ -321,7 +321,7 @@ size_t ListFrontEnd::CalculateNumChildren() {
   ValueObjectSP size_alloc(
       m_backend.GetChildMemberWithName(ConstString("__size_alloc_"), true));
   if (size_alloc) {
-    ValueObjectSP value = GetValueOfLibCXXCompressedPair(*size_alloc);
+    ValueObjectSP value = GetFirstValueOfLibCXXCompressedPair(*size_alloc);
     if (value) {
       m_count = value->GetValueAsUnsigned(UINT32_MAX);
     }
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
index bc8738094316c..95a3694c64b00 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
@@ -96,18 +96,6 @@ class AppleObjCRuntimeV2 : public AppleObjCRuntime {
   void GetValuesForGlobalCFBooleans(lldb::addr_t &cf_true,
                                     lldb::addr_t &cf_false) override;
 
-  // none of these are valid ISAs - we use them to infer the type
-  // of tagged pointers - if we have something meaningful to say
-  // we report an actual type - otherwise, we just say tagged
-  // there is no connection between the values here and the tagged pointers map
-  static const ObjCLanguageRuntime::ObjCISA g_objc_Tagged_ISA = 1;
-  static const ObjCLanguageRuntime::ObjCISA g_objc_Tagged_ISA_NSAtom = 2;
-  static const ObjCLanguageRuntime::ObjCISA g_objc_Tagged_ISA_NSNumber = 3;
-  static const ObjCLanguageRuntime::ObjCISA g_objc_Tagged_ISA_NSDateTS = 4;
-  static const ObjCLanguageRuntime::ObjCISA g_objc_Tagged_ISA_NSManagedObject =
-      5;
-  static const ObjCLanguageRuntime::ObjCISA g_objc_Tagged_ISA_NSDate = 6;
-
 protected:
   lldb::BreakpointResolverSP
   CreateExceptionResolver(const lldb::BreakpointSP &bkpt, bool catch_bp,
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
index 0c39a4f68e878..82171dee221ec 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.cpp
@@ -13,7 +13,6 @@
 #include <optional>
 #include <unordered_map>
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
@@ -27,6 +26,7 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/DataBufferHeap.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RangeMap.h"
@@ -623,7 +623,7 @@ size_t ObjectFileELF::GetModuleSpecifications(
             if (!gnu_debuglink_crc) {
               LLDB_SCOPED_TIMERF(
                   "Calculating module crc32 %s with size %" PRIu64 " KiB",
-                  file.GetLastPathComponent().AsCString(),
+                  file.GetFilename().AsCString(),
                   (length - file_offset) / 1024);
 
               // For core files - which usually don't happen to have a
@@ -2029,7 +2029,7 @@ unsigned ObjectFileELF::ParseSymbols(Symtab *symtab, user_id_t start_id,
   // contain the note section specifying the environment to Android but the
   // custom extension and file name makes it highly unlikely that this will
   // collide with anything else.
-  ConstString file_extension = m_file.GetFileNameExtension();
+  llvm::StringRef file_extension = m_file.GetFileNameExtension();
   bool skip_oatdata_oatexec =
       file_extension == ".oat" || file_extension == ".odex";
 
@@ -2637,6 +2637,44 @@ static void ApplyELF64ABS32Relocation(Symtab *symtab, ELFRelocation &rel,
   }
 }
 
+static void ApplyELF32ABS32RelRelocation(Symtab *symtab, ELFRelocation &rel,
+                                         DataExtractor &debug_data,
+                                         Section *rel_section) {
+  Log *log = GetLog(LLDBLog::Modules);
+  Symbol *symbol = symtab->FindSymbolByID(ELFRelocation::RelocSymbol32(rel));
+  if (symbol) {
+    addr_t value = symbol->GetAddressRef().GetFileAddress();
+    if (value == LLDB_INVALID_ADDRESS) {
+      const char *name = symbol->GetName().GetCString();
+      LLDB_LOGF(log, "Debug info symbol invalid: %s", name);
+      return;
+    }
+    assert(llvm::isUInt<32>(value) && "Valid addresses are 32-bit");
+    DataBufferSP &data_buffer_sp = debug_data.GetSharedDataBuffer();
+    // ObjectFileELF creates a WritableDataBuffer in CreateInstance.
+    WritableDataBuffer *data_buffer =
+        llvm::cast<WritableDataBuffer>(data_buffer_sp.get());
+    uint8_t *dst = data_buffer->GetBytes() + rel_section->GetFileOffset() +
+                   ELFRelocation::RelocOffset32(rel);
+    // Implicit addend is stored inline as a signed value.
+    int32_t addend;
+    memcpy(&addend, dst, sizeof(int32_t));
+    // The sum must be positive. This extra check prevents UB from overflow in
+    // the actual range check below.
+    if (addend < 0 && static_cast<uint32_t>(-addend) > value) {
+      LLDB_LOGF(log, "Debug info relocation overflow: 0x%" PRIx64,
+                static_cast<int64_t>(value) + addend);
+      return;
+    }
+    if (!llvm::isUInt<32>(value + addend)) {
+      LLDB_LOGF(log, "Debug info relocation out of range: 0x%" PRIx64, value);
+      return;
+    }
+    uint32_t addr = value + addend;
+    memcpy(dst, &addr, sizeof(uint32_t));
+  }
+}
+
 unsigned ObjectFileELF::ApplyRelocations(
     Symtab *symtab, const ELFHeader *hdr, const ELFSectionHeader *rel_hdr,
     const ELFSectionHeader *symtab_hdr, const ELFSectionHeader *debug_hdr,
@@ -2667,6 +2705,21 @@ unsigned ObjectFileELF::ApplyRelocations(
 
     if (hdr->Is32Bit()) {
       switch (hdr->e_machine) {
+      case llvm::ELF::EM_ARM:
+        switch (reloc_type(rel)) {
+        case R_ARM_ABS32:
+          ApplyELF32ABS32RelRelocation(symtab, rel, debug_data, rel_section);
+          break;
+        case R_ARM_REL32:
+          GetModule()->ReportError("unsupported AArch32 relocation:"
+                                   " .rel{0}[{1}], type {2}",
+                                   rel_section->GetName().AsCString(), i,
+                                   reloc_type(rel));
+          break;
+        default:
+          assert(false && "unexpected relocation type");
+        }
+        break;
       case llvm::ELF::EM_386:
         switch (reloc_type(rel)) {
         case R_386_32:
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 621df79e52e56..8ff613d0cf9bc 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -14,7 +14,6 @@
 #include "Plugins/Process/Utility/RegisterContextDarwin_i386.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_x86_64.h"
 #include "lldb/Core/Debugger.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
@@ -36,6 +35,7 @@
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/DataBuffer.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RangeMap.h"
@@ -531,21 +531,18 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
       lldb::offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRAltRegSet:
-      case GPRRegSet:
-        // On ARM, the CPSR register is also included in the count but it is
-        // not included in gpr.r so loop until (count-1).
-
-        // Prevent static analysis warnings by explicitly contstraining 'count'
-        // to acceptable range. Handle possible underflow of count-1
-        if (count > 0 && count <= sizeof(gpr.r) / sizeof(gpr.r[0])) {
+      case GPRRegSet: {
+        // r0-r15, plus CPSR
+        uint32_t gpr_buf_count = (sizeof(gpr.r) / sizeof(gpr.r[0])) + 1;
+        if (count == gpr_buf_count) {
           for (uint32_t i = 0; i < (count - 1); ++i) {
             gpr.r[i] = data.GetU32(&offset);
           }
-        }
-        // Save cpsr explicitly.
-        gpr.cpsr = data.GetU32(&offset);
+          gpr.cpsr = data.GetU32(&offset);
 
-        SetError(GPRRegSet, Read, 0);
+          SetError(GPRRegSet, Read, 0);
+        }
+      }
         offset = next_thread_state;
         break;
 
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index d4f42a8d61e3b..bea77c3d4eb95 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -10,10 +10,10 @@
 #define LLDB_SOURCE_PLUGINS_OBJECTFILE_MACH_O_OBJECTFILEMACHO_H
 
 #include "lldb/Core/Address.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Host/SafeMachO.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/RangeMap.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/UUID.h"
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 515c43ec6a720..cded6b5fe698f 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -10,7 +10,6 @@
 #include "PECallFrameInfo.h"
 #include "WindowsMiniDump.h"
 
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
@@ -25,6 +24,7 @@
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
@@ -90,14 +90,13 @@ class PluginProperties : public Properties {
   }
 
   llvm::Triple::EnvironmentType ABI() const {
-    return (llvm::Triple::EnvironmentType)
-        m_collection_sp->GetPropertyAtIndexAsEnumeration(
-            nullptr, ePropertyABI, llvm::Triple::UnknownEnvironment);
+    return GetPropertyAtIndexAs<llvm::Triple::EnvironmentType>(
+        ePropertyABI, llvm::Triple::UnknownEnvironment);
   }
 
   OptionValueDictionary *ModuleABIMap() const {
     return m_collection_sp->GetPropertyAtIndexAsOptionValueDictionary(
-        nullptr, ePropertyModuleABIMap);
+        ePropertyModuleABIMap);
   }
 };
 
@@ -166,7 +165,7 @@ static UUID GetCoffUUID(llvm::object::COFFObjectFile &coff_obj) {
     auto raw_data = coff_obj.getData();
     LLDB_SCOPED_TIMERF(
         "Calculating module crc32 %s with size %" PRIu64 " KiB",
-        FileSpec(coff_obj.getFileName()).GetLastPathComponent().AsCString(),
+        FileSpec(coff_obj.getFileName()).GetFilename().AsCString(),
         static_cast<lldb::offset_t>(raw_data.size()) / 1024);
     gnu_debuglink_crc = llvm::crc32(0, llvm::arrayRefFromStringRef(raw_data));
   }
@@ -295,25 +294,23 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
   const auto *map = GetGlobalPluginProperties().ModuleABIMap();
   if (map->GetNumValues() > 0) {
     // Step 1: Try with the exact file name.
-    auto name = file.GetLastPathComponent();
+    auto name = file.GetFilename();
     module_env_option = map->GetValueForKey(name);
     if (!module_env_option) {
       // Step 2: Try with the file name in lowercase.
       auto name_lower = name.GetStringRef().lower();
-      module_env_option =
-          map->GetValueForKey(ConstString(llvm::StringRef(name_lower)));
+      module_env_option = map->GetValueForKey(llvm::StringRef(name_lower));
     }
     if (!module_env_option) {
       // Step 3: Try with the file name with ".debug" suffix stripped.
       auto name_stripped = name.GetStringRef();
       if (name_stripped.consume_back_insensitive(".debug")) {
-        module_env_option = map->GetValueForKey(ConstString(name_stripped));
+        module_env_option = map->GetValueForKey(name_stripped);
         if (!module_env_option) {
           // Step 4: Try with the file name in lowercase with ".debug" suffix
           // stripped.
           auto name_lower = name_stripped.lower();
-          module_env_option =
-              map->GetValueForKey(ConstString(llvm::StringRef(name_lower)));
+          module_env_option = map->GetValueForKey(llvm::StringRef(name_lower));
         }
       }
     }
@@ -321,7 +318,8 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
   llvm::Triple::EnvironmentType env;
   if (module_env_option)
     env =
-        (llvm::Triple::EnvironmentType)module_env_option->GetEnumerationValue();
+        (llvm::Triple::EnvironmentType)module_env_option->GetEnumerationValue()
+            .value_or(0);
   else
     env = GetGlobalPluginProperties().ABI();
 
diff --git a/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp b/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp
index 1afce4f41a823..64c1d1e48a9ef 100644
--- a/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp
+++ b/lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp
@@ -281,7 +281,7 @@ uint32_t PlatformAndroid::GetSdkVersion() {
 Status PlatformAndroid::DownloadSymbolFile(const lldb::ModuleSP &module_sp,
                                            const FileSpec &dst_file_spec) {
   // For oat file we can try to fetch additional debug info from the device
-  ConstString extension = module_sp->GetFileSpec().GetFileNameExtension();
+  llvm::StringRef extension = module_sp->GetFileSpec().GetFileNameExtension();
   if (extension != ".oat" && extension != ".odex")
     return Status(
         "Symbol file downloading only supported for oat and odex files");
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index e9b717e91e047..76c6b535679a6 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -136,8 +136,7 @@ class PlatformDarwinProperties : public Properties {
   const char *GetIgnoredExceptions() const {
     const uint32_t idx = ePropertyIgnoredExceptions;
     const OptionValueString *option_value =
-        m_collection_sp->GetPropertyAtIndexAsOptionValueString(nullptr, false,
-                                                               idx);
+        m_collection_sp->GetPropertyAtIndexAsOptionValueString(idx);
     assert(option_value);
     return option_value->GetCurrentValue();
   }
@@ -145,8 +144,7 @@ class PlatformDarwinProperties : public Properties {
   OptionValueString *GetIgnoredExceptionValue() {
     const uint32_t idx = ePropertyIgnoredExceptions;
     OptionValueString *option_value =
-        m_collection_sp->GetPropertyAtIndexAsOptionValueString(nullptr, false,
-                                                               idx);
+        m_collection_sp->GetPropertyAtIndexAsOptionValueString(idx);
     assert(option_value);
     return option_value;
   }
@@ -1240,10 +1238,9 @@ lldb_private::Status PlatformDarwin::FindBundleBinaryInExecSearchPaths(
 
     FileSpec platform_pull_upart(platform_file);
     std::vector<std::string> path_parts;
-    path_parts.push_back(
-        platform_pull_upart.GetLastPathComponent().AsCString());
+    path_parts.push_back(platform_pull_upart.GetFilename().AsCString());
     while (platform_pull_upart.RemoveLastPathComponent()) {
-      ConstString part = platform_pull_upart.GetLastPathComponent();
+      ConstString part = platform_pull_upart.GetFilename();
       path_parts.push_back(part.AsCString());
     }
     const size_t path_parts_size = path_parts.size();
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
index 4fd7521e031b7..a51e0b9cf2ac3 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.h
@@ -10,12 +10,12 @@
 #define LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMDARWIN_H
 
 #include "Plugins/Platform/POSIX/PlatformPOSIX.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/ProcessLaunchInfo.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/XcodeSDK.h"
@@ -51,7 +51,7 @@ class PlatformDarwin : public PlatformPOSIX {
   static lldb::PlatformSP CreateInstance(bool force, const ArchSpec *arch);
 
   static void DebuggerInitialize(lldb_private::Debugger &debugger);
-  
+
   static void Initialize();
 
   static void Terminate();
@@ -108,7 +108,7 @@ class PlatformDarwin : public PlatformPOSIX {
   FileSpec LocateExecutable(const char *basename) override;
 
   Status LaunchProcess(ProcessLaunchInfo &launch_info) override;
-  
+
   Args GetExtraStartupCommands() override;
 
   static std::tuple<llvm::VersionTuple, llvm::StringRef>
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
index 5d81b7caef712..c29007fb9a340 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.cpp
@@ -201,11 +201,7 @@ class PlatformDarwinKernelProperties : public Properties {
 
   FileSpecList GetKextDirectories() const {
     const uint32_t idx = ePropertyKextDirectories;
-    const OptionValueFileSpecList *option_value =
-        m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(
-            NULL, false, idx);
-    assert(option_value);
-    return option_value->GetCurrentValue();
+    return GetPropertyAtIndexAs<FileSpecList>(idx, {});
   }
 };
 
@@ -418,11 +414,11 @@ void PlatformDarwinKernel::AddSDKSubdirsToSearchPaths(const std::string &dir) {
 FileSystem::EnumerateDirectoryResult
 PlatformDarwinKernel::FindKDKandSDKDirectoriesInDirectory(
     void *baton, llvm::sys::fs::file_type ft, llvm::StringRef path) {
-  static ConstString g_sdk_suffix = ConstString(".sdk");
-  static ConstString g_kdk_suffix = ConstString(".kdk");
+  static constexpr llvm::StringLiteral g_sdk_suffix = ".sdk";
+  static constexpr llvm::StringLiteral g_kdk_suffix = ".kdk";
 
   PlatformDarwinKernel *thisp = (PlatformDarwinKernel *)baton;
-  FileSpec file_spec(path);
+  const FileSpec file_spec(path);
   if (ft == llvm::sys::fs::file_type::directory_file &&
       (file_spec.GetFileNameExtension() == g_sdk_suffix ||
        file_spec.GetFileNameExtension() == g_kdk_suffix)) {
@@ -480,12 +476,11 @@ FileSystem::EnumerateDirectoryResult
 PlatformDarwinKernel::GetKernelsAndKextsInDirectoryHelper(
     void *baton, llvm::sys::fs::file_type ft, llvm::StringRef path,
     bool recurse) {
-  static ConstString g_kext_suffix = ConstString(".kext");
-  static ConstString g_dsym_suffix = ConstString(".dSYM");
-  static ConstString g_bundle_suffix = ConstString("Bundle");
+  static constexpr llvm::StringLiteral g_kext_suffix = ".kext";
+  static constexpr llvm::StringLiteral g_dsym_suffix = ".dSYM";
 
-  FileSpec file_spec(path);
-  ConstString file_spec_extension = file_spec.GetFileNameExtension();
+  const FileSpec file_spec(path);
+  llvm::StringRef file_spec_extension = file_spec.GetFileNameExtension();
 
   Log *log = GetLog(LLDBLog::Platform);
 
@@ -567,8 +562,7 @@ PlatformDarwinKernel::GetKernelsAndKextsInDirectoryHelper(
 
   // Don't recurse into dSYM/kext/bundle directories
   if (recurse && file_spec_extension != g_dsym_suffix &&
-      file_spec_extension != g_kext_suffix &&
-      file_spec_extension != g_bundle_suffix) {
+      file_spec_extension != g_kext_suffix) {
     LLDB_LOGV(log, "PlatformDarwinKernel descending into directory '{0}'",
               file_spec);
     return FileSystem::eEnumerateDirectoryResultEnter;
@@ -666,7 +660,7 @@ bool PlatformDarwinKernel::KernelHasdSYMSibling(const FileSpec &kernel_binary) {
 //    /dir/dir/mach.development.t7004
 bool PlatformDarwinKernel::KerneldSYMHasNoSiblingBinary(
     const FileSpec &kernel_dsym) {
-  static ConstString g_dsym_suffix = ConstString(".dSYM");
+  static constexpr llvm::StringLiteral g_dsym_suffix = ".dSYM";
   std::string possible_path = kernel_dsym.GetPath();
   if (kernel_dsym.GetFileNameExtension() != g_dsym_suffix)
     return false;
@@ -694,9 +688,9 @@ bool PlatformDarwinKernel::KerneldSYMHasNoSiblingBinary(
 // it should iterate over every binary in the DWARF subdir
 // and return them all.
 std::vector<FileSpec>
-PlatformDarwinKernel::GetDWARFBinaryInDSYMBundle(FileSpec dsym_bundle) {
+PlatformDarwinKernel::GetDWARFBinaryInDSYMBundle(const FileSpec &dsym_bundle) {
   std::vector<FileSpec> results;
-  static ConstString g_dsym_suffix = ConstString(".dSYM");
+  static constexpr llvm::StringLiteral g_dsym_suffix = ".dSYM";
   if (dsym_bundle.GetFileNameExtension() != g_dsym_suffix) {
     return results;
   }
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.h b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.h
index 1b0708cd9f8dc..e28f77cd44d55 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwinKernel.h
@@ -137,7 +137,8 @@ class PlatformDarwinKernel : public PlatformDarwin {
   // with the binary inside it ('.../foo.dSYM/Contents/Resources/DWARF/foo').
   // A dSYM bundle may have multiple DWARF binaries in them, so a vector
   // of matches is returned.
-  static std::vector<FileSpec> GetDWARFBinaryInDSYMBundle(FileSpec dsym_bundle);
+  static std::vector<FileSpec>
+  GetDWARFBinaryInDSYMBundle(const FileSpec &dsym_bundle);
 
   Status GetSharedModuleKext(const ModuleSpec &module_spec, Process *process,
                              lldb::ModuleSP &module_sp,
diff --git a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
index 222dbfa719647..1e91f2ccd1982 100644
--- a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
+++ b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
@@ -401,6 +401,7 @@ lldb::ProcessSP PlatformPOSIX::Attach(ProcessAttachInfo &attach_info,
           attach_info.SetHijackListener(listener_sp);
         }
         process_sp->HijackProcessEvents(listener_sp);
+        process_sp->SetShadowListener(attach_info.GetShadowListener());
         error = process_sp->Attach(attach_info);
       }
     }
@@ -458,6 +459,7 @@ lldb::ProcessSP PlatformPOSIX::DebugProcess(ProcessLaunchInfo &launch_info,
   LLDB_LOG(log, "successfully created process");
 
   process_sp->HijackProcessEvents(launch_info.GetHijackListener());
+  process_sp->SetShadowListener(launch_info.GetShadowListener());
 
   // Log file actions.
   if (log) {
diff --git a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
index e0bc8bf97dcf6..6e94ab51354c7 100644
--- a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
+++ b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
@@ -41,33 +41,28 @@ class PluginProperties : public Properties {
   }
 
   llvm::StringRef GetArchitecture() {
-    return m_collection_sp->GetPropertyAtIndexAsString(
-        nullptr, ePropertyArchitecture, "");
+    return GetPropertyAtIndexAs<llvm::StringRef>(ePropertyArchitecture, "");
   }
 
   FileSpec GetEmulatorPath() {
-    return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr,
-                                                         ePropertyEmulatorPath);
+    return GetPropertyAtIndexAs<FileSpec>(ePropertyEmulatorPath, {});
   }
 
   Args GetEmulatorArgs() {
     Args result;
-    m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyEmulatorArgs,
-                                              result);
+    m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyEmulatorArgs, result);
     return result;
   }
 
   Environment GetEmulatorEnvVars() {
     Args args;
-    m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyEmulatorEnvVars,
-                                              args);
+    m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyEmulatorEnvVars, args);
     return Environment(args);
   }
 
   Environment GetTargetEnvVars() {
     Args args;
-    m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyTargetEnvVars,
-                                              args);
+    m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyTargetEnvVars, args);
     return Environment(args);
   }
 };
diff --git a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
index 446776519dd24..a6106dccdb7f8 100644
--- a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
+++ b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
@@ -423,6 +423,7 @@ PlatformRemoteGDBServer::DebugProcess(ProcessLaunchInfo &launch_info,
 
         if (process_sp) {
           process_sp->HijackProcessEvents(launch_info.GetHijackListener());
+          process_sp->SetShadowListener(launch_info.GetShadowListener());
 
           error = process_sp->ConnectRemote(connect_url.c_str());
           // Retry the connect remote one time...
@@ -515,6 +516,7 @@ lldb::ProcessSP PlatformRemoteGDBServer::Attach(
               ListenerSP listener_sp = attach_info.GetHijackListener();
               if (listener_sp)
                 process_sp->HijackProcessEvents(listener_sp);
+              process_sp->SetShadowListener(attach_info.GetShadowListener());
               error = process_sp->Attach(attach_info);
             }
 
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
index c57b9499d5247..0ec152f0643a4 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
@@ -61,7 +61,7 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
   case llvm::Triple::aarch64: {
     // Configure register sets supported by this AArch64 target.
     // Read SVE header to check for SVE support.
-    struct user_sve_header sve_header;
+    struct sve::user_sve_header sve_header;
     struct iovec ioVec;
     ioVec.iov_base = &sve_header;
     ioVec.iov_len = sizeof(sve_header);
@@ -380,7 +380,7 @@ Status NativeRegisterContextLinux_arm64::WriteRegister(
       if (GetRegisterInfo().IsSVERegVG(reg)) {
         uint64_t vg_value = reg_value.GetAsUInt64();
 
-        if (sve_vl_valid(vg_value * 8)) {
+        if (sve::vl_valid(vg_value * 8)) {
           if (m_sve_header_is_valid && vg_value == GetSVERegVG())
             return error;
 
@@ -566,7 +566,7 @@ Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(
   if (contains_sve_reg_data) {
     // We have SVE register data first write SVE header.
     ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());
-    if (!sve_vl_valid(m_sve_header.vl)) {
+    if (!sve::vl_valid(m_sve_header.vl)) {
       m_sve_header_is_valid = false;
       error.SetErrorStringWithFormat("NativeRegisterContextLinux_arm64::%s "
                                      "Invalid SVE header in data_sp",
@@ -934,7 +934,7 @@ void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
       // On every stop we configure SVE vector length by calling
       // ConfigureVectorLength regardless of current SVEState of this thread.
       uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;
-      if (sve_vl_valid(m_sve_header.vl))
+      if (sve::vl_valid(m_sve_header.vl))
         vq = sve::vq_from_vl(m_sve_header.vl);
 
       GetRegisterInfo().ConfigureVectorLength(vq);
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
index 9b29d1b8f1585..7f984233f0c26 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp
@@ -78,8 +78,8 @@ class PluginProperties : public Properties {
 
   uint64_t GetPacketTimeout() {
     const uint32_t idx = ePropertyKDPPacketTimeout;
-    return m_collection_sp->GetPropertyAtIndexAsUInt64(
-        NULL, idx, g_processkdp_properties[idx].default_uint_value);
+    return GetPropertyAtIndexAs<uint64_t>(
+        idx, g_processkdp_properties[idx].default_uint_value);
   }
 };
 
@@ -887,7 +887,7 @@ class CommandObjectProcessKDPPacketSend : public CommandObjectParsed {
           "the --command option must be set to a valid command byte");
     } else {
       const uint64_t command_byte =
-          m_command_byte.GetOptionValue().GetUInt64Value(0);
+          m_command_byte.GetOptionValue().GetUInt64Value().value_or(0);
       if (command_byte > 0 && command_byte <= UINT8_MAX) {
         ProcessKDP *process =
             (ProcessKDP *)m_interpreter.GetExecutionContext().GetProcessPtr();
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index 18552ea66b029..0501c5bd0ce04 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -69,10 +69,10 @@ GDBRemoteCommunicationClient::GDBRemoteCommunicationClient()
       m_supports_vFileSize(true), m_supports_vFileMode(true),
       m_supports_vFileExists(true), m_supports_vRun(true),
 
-      m_host_arch(), m_process_arch(), m_os_build(), m_os_kernel(),
-      m_hostname(), m_gdb_server_name(), m_default_packet_timeout(0),
-      m_qSupported_response(), m_supported_async_json_packets_sp(),
-      m_qXfer_memory_map() {}
+      m_host_arch(), m_host_distribution_id(), m_process_arch(), m_os_build(),
+      m_os_kernel(), m_hostname(), m_gdb_server_name(),
+      m_default_packet_timeout(0), m_qSupported_response(),
+      m_supported_async_json_packets_sp(), m_qXfer_memory_map() {}
 
 // Destructor
 GDBRemoteCommunicationClient::~GDBRemoteCommunicationClient() {
@@ -307,6 +307,7 @@ void GDBRemoteCommunicationClient::ResetDiscoverableSettings(bool did_exec) {
     m_qSymbol_requests_done = false;
     m_supports_qModuleInfo = true;
     m_host_arch.Clear();
+    m_host_distribution_id.clear();
     m_os_version = llvm::VersionTuple();
     m_os_build.clear();
     m_os_kernel.clear();
@@ -1206,7 +1207,6 @@ bool GDBRemoteCommunicationClient::GetHostInfo(bool force) {
         std::string environment;
         std::string vendor_name;
         std::string triple;
-        std::string distribution_id;
         uint32_t pointer_byte_size = 0;
         ByteOrder byte_order = eByteOrderInvalid;
         uint32_t num_keys_decoded = 0;
@@ -1228,7 +1228,7 @@ bool GDBRemoteCommunicationClient::GetHostInfo(bool force) {
             ++num_keys_decoded;
           } else if (name.equals("distribution_id")) {
             StringExtractor extractor(value);
-            extractor.GetHexByteString(distribution_id);
+            extractor.GetHexByteString(m_host_distribution_id);
             ++num_keys_decoded;
           } else if (name.equals("os_build")) {
             StringExtractor extractor(value);
@@ -1376,8 +1376,6 @@ bool GDBRemoteCommunicationClient::GetHostInfo(bool force) {
                     m_host_arch.GetTriple().getTriple().c_str(),
                     triple.c_str());
         }
-        if (!distribution_id.empty())
-          m_host_arch.SetDistributionId(distribution_id.c_str());
       }
     }
   }
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
index b32e09cf8e76f..02185cf52e33e 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
@@ -583,6 +583,7 @@ class GDBRemoteCommunicationClient : public GDBRemoteClientBase {
   uint32_t m_addressing_bits = 0;
 
   ArchSpec m_host_arch;
+  std::string m_host_distribution_id;
   ArchSpec m_process_arch;
   UUID m_process_standalone_uuid;
   lldb::addr_t m_process_standalone_value = LLDB_INVALID_ADDRESS;
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index 7b10fd4217e05..9e90e98b1f2ab 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -187,8 +187,8 @@ GDBRemoteCommunicationServerCommon::Handle_qHostInfo(
   response.PutStringAsRawHex8(host_triple.getTriple());
   response.Printf(";ptrsize:%u;", host_arch.GetAddressByteSize());
 
-  const char *distribution_id = host_arch.GetDistributionId().AsCString();
-  if (distribution_id) {
+  llvm::StringRef distribution_id = HostInfo::GetDistributionId();
+  if (!distribution_id.empty()) {
     response.PutCString("distribution_id:");
     response.PutStringAsRawHex8(distribution_id);
     response.PutCString(";");
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 087344198e226..272cd4be3c671 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -142,30 +142,29 @@ class PluginProperties : public Properties {
 
   uint64_t GetPacketTimeout() {
     const uint32_t idx = ePropertyPacketTimeout;
-    return m_collection_sp->GetPropertyAtIndexAsUInt64(
-        nullptr, idx, g_processgdbremote_properties[idx].default_uint_value);
+    return GetPropertyAtIndexAs<uint64_t>(
+        idx, g_processgdbremote_properties[idx].default_uint_value);
   }
 
   bool SetPacketTimeout(uint64_t timeout) {
     const uint32_t idx = ePropertyPacketTimeout;
-    return m_collection_sp->SetPropertyAtIndexAsUInt64(nullptr, idx, timeout);
+    return SetPropertyAtIndex(idx, timeout);
   }
 
   FileSpec GetTargetDefinitionFile() const {
     const uint32_t idx = ePropertyTargetDefinitionFile;
-    return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+    return GetPropertyAtIndexAs<FileSpec>(idx, {});
   }
 
   bool GetUseSVR4() const {
     const uint32_t idx = ePropertyUseSVR4;
-    return m_collection_sp->GetPropertyAtIndexAsBoolean(
-        nullptr, idx,
-        g_processgdbremote_properties[idx].default_uint_value != 0);
+    return GetPropertyAtIndexAs<bool>(
+        idx, g_processgdbremote_properties[idx].default_uint_value != 0);
   }
 
   bool GetUseGPacketForReading() const {
     const uint32_t idx = ePropertyUseGPacketForReading;
-    return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, true);
+    return GetPropertyAtIndexAs<bool>(idx, true);
   }
 };
 
@@ -2258,6 +2257,13 @@ StateType ProcessGDBRemote::SetThreadStopInfo(StringExtractor &stop_packet) {
         StreamString ostr;
         ostr.Printf("%" PRIu64 " %" PRIu64, pid_tid->first, pid_tid->second);
         description = std::string(ostr.GetString());
+      } else if (key.compare("addressing_bits") == 0) {
+        uint64_t addressing_bits;
+        if (!value.getAsInteger(0, addressing_bits)) {
+          addr_t address_mask = ~((1ULL << addressing_bits) - 1);
+          SetCodeAddressMask(address_mask);
+          SetDataAddressMask(address_mask);
+        }
       } else if (key.size() == 2 && ::isxdigit(key[0]) && ::isxdigit(key[1])) {
         uint32_t reg = UINT32_MAX;
         if (!key.getAsInteger(16, reg))
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
index 23ff8811e3fd7..46f99f6b1e5b5 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
+++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp
@@ -150,22 +150,15 @@ Status ScriptedProcess::DoLoadCore() {
 
 Status ScriptedProcess::DoLaunch(Module *exe_module,
                                  ProcessLaunchInfo &launch_info) {
-  /* FIXME: This doesn't reflect how lldb actually launches a process.
-           In reality, it attaches to debugserver, then resume the process. */
+  LLDB_LOGF(GetLog(LLDBLog::Process), "ScriptedProcess::%s launching process", __FUNCTION__);
+  
+  /* MARK: This doesn't reflect how lldb actually launches a process.
+           In reality, it attaches to debugserver, then resume the process.
+           That's not true in all cases.  If debugserver is remote, lldb
+           asks debugserver to launch the process for it. */
   Status error = GetInterface().Launch();
-  SetPrivateState(eStateRunning);
-
-  if (error.Fail())
-    return error;
-
-  // TODO: Fetch next state from stopped event queue then send stop event
-  //  const StateType state = SetThreadStopInfo(response);
-  //  if (state != eStateInvalid) {
-  //    SetPrivateState(state);
-
   SetPrivateState(eStateStopped);
-
-  return {};
+  return error;
 }
 
 void ScriptedProcess::DidLaunch() { m_pid = GetInterface().GetProcessID(); }
@@ -177,25 +170,9 @@ void ScriptedProcess::DidResume() {
 }
 
 Status ScriptedProcess::DoResume() {
-  Log *log = GetLog(LLDBLog::Process);
-  // FIXME: Fetch data from thread.
-  const StateType thread_resume_state = eStateRunning;
-  LLDB_LOGF(log, "ScriptedProcess::%s thread_resume_state = %s", __FUNCTION__,
-            StateAsCString(thread_resume_state));
-
-  bool resume = (thread_resume_state == eStateRunning);
-  assert(thread_resume_state == eStateRunning && "invalid thread resume state");
-
-  Status error;
-  if (resume) {
-    LLDB_LOGF(log, "ScriptedProcess::%s sending resume", __FUNCTION__);
+  LLDB_LOGF(GetLog(LLDBLog::Process), "ScriptedProcess::%s resuming process", __FUNCTION__);
 
-    SetPrivateState(eStateRunning);
-    SetPrivateState(eStateStopped);
-    error = GetInterface().Resume();
-  }
-
-  return error;
+  return GetInterface().Resume();
 }
 
 Status ScriptedProcess::DoAttach(const ProcessAttachInfo &attach_info) {
@@ -226,19 +203,6 @@ void ScriptedProcess::DidAttach(ArchSpec &process_arch) {
   process_arch = GetArchitecture();
 }
 
-Status ScriptedProcess::DoStop() {
-  Log *log = GetLog(LLDBLog::Process);
-
-  if (GetInterface().ShouldStop()) {
-    SetPrivateState(eStateStopped);
-    LLDB_LOGF(log, "ScriptedProcess::%s Immediate stop", __FUNCTION__);
-    return {};
-  }
-
-  LLDB_LOGF(log, "ScriptedProcess::%s Delayed stop", __FUNCTION__);
-  return GetInterface().Stop();
-}
-
 Status ScriptedProcess::DoDestroy() { return Status(); }
 
 bool ScriptedProcess::IsAlive() { return GetInterface().IsAlive(); }
@@ -285,6 +249,23 @@ size_t ScriptedProcess::DoWriteMemory(lldb::addr_t vm_addr, const void *buf,
   return bytes_written;
 }
 
+Status ScriptedProcess::EnableBreakpointSite(BreakpointSite *bp_site) {
+  assert(bp_site != nullptr);
+
+  if (bp_site->IsEnabled()) {
+    return {};
+  }
+
+  if (bp_site->HardwareRequired()) {
+    return Status("Scripted Processes don't support hardware breakpoints");
+  }
+
+  Status error;
+  GetInterface().CreateBreakpoint(bp_site->GetLoadAddress(), error);
+
+  return error;
+}
+
 ArchSpec ScriptedProcess::GetArchitecture() {
   return GetTarget().GetArchitecture();
 }
diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.h b/lldb/source/Plugins/Process/scripted/ScriptedProcess.h
index 60f42cbaf7f2c..24b5a317b54fa 100644
--- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.h
+++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.h
@@ -74,6 +74,8 @@ class ScriptedProcess : public Process {
   size_t DoWriteMemory(lldb::addr_t vm_addr, const void *buf, size_t size,
                        Status &error) override;
 
+  Status EnableBreakpointSite(BreakpointSite *bp_site) override;
+
   ArchSpec GetArchitecture();
 
   Status
@@ -90,12 +92,14 @@ class ScriptedProcess : public Process {
 
   void *GetImplementation() override;
 
+  void ForceScriptedState(lldb::StateType state) override {
+    SetPrivateState(state);
+  }
+
 protected:
   ScriptedProcess(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp,
                   const ScriptedMetadata &scripted_metadata, Status &error);
 
-  Status DoStop();
-
   void Clear();
 
   bool DoUpdateThreadList(ThreadList &old_thread_list,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
index b82a2647e9a08..9c2227cc3884e 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
@@ -131,14 +131,13 @@ llvm::Error Lua::CheckSyntax(llvm::StringRef buffer) {
 }
 
 llvm::Error Lua::LoadModule(llvm::StringRef filename) {
-  FileSpec file(filename);
+  const FileSpec file(filename);
   if (!FileSystem::Instance().Exists(file)) {
     return llvm::make_error<llvm::StringError>("invalid path",
                                                llvm::inconvertibleErrorCode());
   }
 
-  ConstString module_extension = file.GetFileNameExtension();
-  if (module_extension != ".lua") {
+  if (file.GetFileNameExtension() != ".lua") {
     return llvm::make_error<llvm::StringError>("invalid extension",
                                                llvm::inconvertibleErrorCode());
   }
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
index 365d499bead86..33092209b1778 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
@@ -201,6 +201,7 @@ template <> struct PythonFormat<short> : PassthroughFormat<short, 'h'> {};
 template <>
 struct PythonFormat<unsigned short> : PassthroughFormat<unsigned short, 'H'> {};
 template <> struct PythonFormat<int> : PassthroughFormat<int, 'i'> {};
+template <> struct PythonFormat<bool> : PassthroughFormat<bool, 'p'> {};
 template <>
 struct PythonFormat<unsigned int> : PassthroughFormat<unsigned int, 'I'> {};
 template <> struct PythonFormat<long> : PassthroughFormat<long, 'l'> {};
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
index 2812cc78189df..78b1375aedad9 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/SWIGPythonBridge.h
@@ -92,6 +92,7 @@ python::ScopedPythonObject<lldb::SBEvent> ToSWIGWrapper(Event *event);
 } // namespace python
 
 void *LLDBSWIGPython_CastPyObjectToSBData(PyObject *data);
+void *LLDBSWIGPython_CastPyObjectToSBBreakpoint(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBAttachInfo(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBLaunchInfo(PyObject *data);
 void *LLDBSWIGPython_CastPyObjectToSBError(PyObject *data);
@@ -147,6 +148,11 @@ python::PythonObject LLDBSwigPythonCreateScriptedThreadPlan(
 bool LLDBSWIGPythonCallThreadPlan(void *implementor, const char *method_name,
                                   lldb_private::Event *event_sp,
                                   bool &got_error);
+                                  
+bool LLDBSWIGPythonCallThreadPlan(void *implementor, 
+                                  const char *method_name,
+                                  lldb_private::Stream *stream,
+                                  bool &got_error);
 
 python::PythonObject LLDBSwigPythonCreateScriptedBreakpointResolver(
     const char *python_class_name, const char *session_dictionary_name,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 6472a9301ed6e..a98ac58e488bc 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -306,7 +306,7 @@ void ScriptInterpreterPython::SharedLibraryDirectoryHelper(
   // On windows, we need to manually back out of the python tree, and go into
   // the bin directory. This is pretty much the inverse of what ComputePythonDir
   // does.
-  if (this_file.GetFileNameExtension() == ConstString(".pyd")) {
+  if (this_file.GetFileNameExtension() == ".pyd") {
     this_file.RemoveLastPathComponent(); // _lldb.pyd or _lldb_d.pyd
     this_file.RemoveLastPathComponent(); // lldb
     llvm::StringRef libdir = LLDB_PYTHON_RELATIVE_LIBDIR;
@@ -1743,7 +1743,7 @@ bool ScriptInterpreterPythonImpl::ScriptedThreadPlanIsStale(
     Locker py_lock(this,
                    Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
     is_stale = LLDBSWIGPythonCallThreadPlan(generic->GetValue(), "is_stale",
-                                            nullptr, script_error);
+                                            (Event *) nullptr, script_error);
     if (script_error)
       return true;
   }
@@ -1760,7 +1760,7 @@ lldb::StateType ScriptInterpreterPythonImpl::ScriptedThreadPlanGetRunState(
     Locker py_lock(this,
                    Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
     should_step = LLDBSWIGPythonCallThreadPlan(
-        generic->GetValue(), "should_step", nullptr, script_error);
+        generic->GetValue(), "should_step", (Event *) nullptr, script_error);
     if (script_error)
       should_step = true;
   }
@@ -1769,6 +1769,24 @@ lldb::StateType ScriptInterpreterPythonImpl::ScriptedThreadPlanGetRunState(
   return lldb::eStateRunning;
 }
 
+bool
+ScriptInterpreterPythonImpl::ScriptedThreadPlanGetStopDescription(
+    StructuredData::ObjectSP implementor_sp, lldb_private::Stream *stream, 
+    bool &script_error) {
+  StructuredData::Generic *generic = nullptr;
+  if (implementor_sp)
+    generic = implementor_sp->GetAsGeneric();
+  if (!generic) {
+    script_error = true;
+    return false;
+  }
+  Locker py_lock(this,
+                   Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
+  return LLDBSWIGPythonCallThreadPlan(generic->GetValue(), "stop_description", 
+      stream, script_error);
+}
+
+
 StructuredData::GenericSP
 ScriptInterpreterPythonImpl::CreateScriptedBreakpointResolver(
     const char *class_name, const StructuredDataImpl &args_data,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
index 21fdf12505f86..b810bdbd9a0c7 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
@@ -97,6 +97,11 @@ class ScriptInterpreterPythonImpl : public ScriptInterpreterPython {
   ScriptedThreadPlanGetRunState(StructuredData::ObjectSP implementor_sp,
                                 bool &script_error) override;
 
+  bool
+  ScriptedThreadPlanGetStopDescription(StructuredData::ObjectSP implementor_sp,
+                                lldb_private::Stream *s,
+                                bool &script_error) override;
+
   StructuredData::GenericSP
   CreateScriptedBreakpointResolver(const char *class_name,
                                    const StructuredDataImpl &args_data,
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
index 17fc2e5e1852a..2b8defb5ae3e9 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.cpp
@@ -81,21 +81,8 @@ Status ScriptedProcessPythonInterface::Launch() {
 }
 
 Status ScriptedProcessPythonInterface::Resume() {
-  return GetStatusFromMethod("resume");
-}
-
-bool ScriptedProcessPythonInterface::ShouldStop() {
-  Status error;
-  StructuredData::ObjectSP obj = Dispatch("is_alive", error);
-
-  if (!CheckStructuredDataObject(LLVM_PRETTY_FUNCTION, obj, error))
-    return {};
-
-  return obj->GetBooleanValue();
-}
-
-Status ScriptedProcessPythonInterface::Stop() {
-  return GetStatusFromMethod("stop");
+  // When calling ScriptedProcess.Resume from lldb we should always stop.
+  return GetStatusFromMethod("resume", /*should_stop=*/true);
 }
 
 std::optional<MemoryRegionInfo>
@@ -123,6 +110,22 @@ StructuredData::DictionarySP ScriptedProcessPythonInterface::GetThreadsInfo() {
   return dict;
 }
 
+bool ScriptedProcessPythonInterface::CreateBreakpoint(lldb::addr_t addr,
+                                                      Status &error) {
+  Status py_error;
+  StructuredData::ObjectSP obj =
+      Dispatch("create_breakpoint", py_error, addr, error);
+
+  // If there was an error on the python call, surface it to the user.
+  if (py_error.Fail())
+    error = py_error;
+
+  if (!CheckStructuredDataObject(LLVM_PRETTY_FUNCTION, obj, error))
+    return {};
+
+  return obj->GetBooleanValue();
+}
+
 lldb::DataExtractorSP ScriptedProcessPythonInterface::ReadMemoryAtAddress(
     lldb::addr_t address, size_t size, Status &error) {
   Status py_error;
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h
index a96a55df8f4bc..ff03eab07648a 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedProcessPythonInterface.h
@@ -37,16 +37,14 @@ class ScriptedProcessPythonInterface : public ScriptedProcessInterface,
 
   Status Resume() override;
 
-  bool ShouldStop() override;
-
-  Status Stop() override;
-
   std::optional<MemoryRegionInfo>
   GetMemoryRegionContainingAddress(lldb::addr_t address,
                                    Status &error) override;
 
   StructuredData::DictionarySP GetThreadsInfo() override;
 
+  bool CreateBreakpoint(lldb::addr_t addr, Status &error) override;
+
   lldb::DataExtractorSP ReadMemoryAtAddress(lldb::addr_t address, size_t size,
                                             Status &error) override;
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.cpp
index 9da46237280ff..61a326ac2b18c 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.cpp
@@ -70,6 +70,22 @@ ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::DataExtractorSP>(
   return m_interpreter.GetDataExtractorFromSBData(*sb_data);
 }
 
+template <>
+lldb::BreakpointSP
+ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::BreakpointSP>(
+    python::PythonObject &p, Status &error) {
+  lldb::SBBreakpoint *sb_breakpoint = reinterpret_cast<lldb::SBBreakpoint *>(
+      LLDBSWIGPython_CastPyObjectToSBBreakpoint(p.get()));
+
+  if (!sb_breakpoint) {
+    error.SetErrorString(
+        "Couldn't cast lldb::SBBreakpoint to lldb::BreakpointSP.");
+    return nullptr;
+  }
+
+  return m_interpreter.GetOpaqueTypeFromSBBreakpoint(*sb_breakpoint);
+}
+
 template <>
 lldb::ProcessAttachInfoSP ScriptedPythonInterface::ExtractValueFromPythonObject<
     lldb::ProcessAttachInfoSP>(python::PythonObject &p, Status &error) {
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.h
index a015bd106b5c6..04e265159e439 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptedPythonInterface.h
@@ -113,6 +113,11 @@ class ScriptedPythonInterface : virtual public ScriptedInterface {
     return {object};
   }
 
+  python::PythonObject Transform(bool arg) {
+    // Boolean arguments need to be turned into python objects.
+    return python::PythonBoolean(arg);
+  }
+
   python::PythonObject Transform(Status arg) {
     return python::ToSWIGWrapper(arg);
   }
@@ -141,6 +146,19 @@ class ScriptedPythonInterface : virtual public ScriptedInterface {
     original_arg = ExtractValueFromPythonObject<T>(transformed_arg, error);
   }
 
+
+  void ReverseTransform(bool &original_arg,
+                        python::PythonObject transformed_arg, Status &error) {
+    python::PythonBoolean boolean_arg = python::PythonBoolean(
+        python::PyRefType::Borrowed, transformed_arg.get());
+    if (boolean_arg.IsValid())
+      original_arg = boolean_arg.GetValue();
+    else
+      error.SetErrorString(
+          llvm::formatv("{}: Invalid boolean argument.", LLVM_PRETTY_FUNCTION)
+              .str());
+  }
+
   template <std::size_t... I, typename... Args>
   auto TransformTuple(const std::tuple<Args...> &args,
                       std::index_sequence<I...>) {
@@ -210,6 +228,11 @@ template <>
 Status ScriptedPythonInterface::ExtractValueFromPythonObject<Status>(
     python::PythonObject &p, Status &error);
 
+template <>
+lldb::BreakpointSP
+ScriptedPythonInterface::ExtractValueFromPythonObject<lldb::BreakpointSP>(
+    python::PythonObject &p, Status &error);
+
 template <>
 lldb::ProcessAttachInfoSP ScriptedPythonInterface::ExtractValueFromPythonObject<
     lldb::ProcessAttachInfoSP>(python::PythonObject &p, Status &error);
diff --git a/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp b/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
index fadf0cc412996..deebf0700f947 100644
--- a/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
+++ b/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
@@ -131,14 +131,14 @@ class StructuredDataDarwinLogProperties : public Properties {
 
   bool GetEnableOnStartup() const {
     const uint32_t idx = ePropertyEnableOnStartup;
-    return m_collection_sp->GetPropertyAtIndexAsBoolean(
-        nullptr, idx, g_darwinlog_properties[idx].default_uint_value != 0);
+    return GetPropertyAtIndexAs<bool>(
+        idx, g_darwinlog_properties[idx].default_uint_value != 0);
   }
 
   llvm::StringRef GetAutoEnableOptions() const {
     const uint32_t idx = ePropertyAutoEnableOptions;
-    return m_collection_sp->GetPropertyAtIndexAsString(
-        nullptr, idx, g_darwinlog_properties[idx].default_cstr_value);
+    return GetPropertyAtIndexAs<llvm::StringRef>(
+        idx, g_darwinlog_properties[idx].default_cstr_value);
   }
 
   const char *GetLoggingModuleName() const { return "libsystem_trace.dylib"; }
@@ -975,9 +975,10 @@ EnableOptionsSP ParseAutoEnableOptions(Status &error, Debugger &debugger) {
 
   // Parse the arguments.
   auto options_property_sp =
-      debugger.GetPropertyValue(nullptr, "plugin.structured-data.darwin-log."
-                                         "auto-enable-options",
-                                false, error);
+      debugger.GetPropertyValue(nullptr,
+                                "plugin.structured-data.darwin-log."
+                                "auto-enable-options",
+                                error);
   if (!error.Success())
     return EnableOptionsSP();
   if (!options_property_sp) {
@@ -1267,7 +1268,7 @@ void StructuredDataDarwinLog::ModulesDidLoad(Process &process,
 
     auto &file_spec = module_sp->GetFileSpec();
     found_logging_support_module =
-        (file_spec.GetLastPathComponent() == logging_module_name);
+        (file_spec.GetFilename() == logging_module_name);
     if (found_logging_support_module)
       break;
   }
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
index 3942f4f7ae247..4a01a64202eed 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
@@ -10,11 +10,11 @@
 #define LLDB_SOURCE_PLUGINS_SYMBOLFILE_BREAKPAD_SYMBOLFILEBREAKPAD_H
 
 #include "Plugins/ObjectFile/Breakpad/BreakpadRecords.h"
-#include "lldb/Core/FileSpecList.h"
 #include "lldb/Symbol/LineTable.h"
 #include "lldb/Symbol/PostfixExpression.h"
 #include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Symbol/UnwindPlan.h"
+#include "lldb/Utility/FileSpecList.h"
 #include <optional>
 
 namespace lldb_private {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 879949b9e7dcf..674977cd7b59a 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -125,8 +125,7 @@ class PluginProperties : public Properties {
   }
 
   bool IgnoreFileIndexes() const {
-    return m_collection_sp->GetPropertyAtIndexAsBoolean(
-        nullptr, ePropertyIgnoreIndexes, false);
+    return GetPropertyAtIndexAs<bool>(ePropertyIgnoreIndexes, false);
   }
 };
 
diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index a132e0ce202f2..f81f2353d867f 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -99,29 +99,27 @@ PlatformProperties::PlatformProperties() {
 
 bool PlatformProperties::GetUseModuleCache() const {
   const auto idx = ePropertyUseModuleCache;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_platform_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_platform_properties[idx].default_uint_value != 0);
 }
 
 bool PlatformProperties::SetUseModuleCache(bool use_module_cache) {
-  return m_collection_sp->SetPropertyAtIndexAsBoolean(
-      nullptr, ePropertyUseModuleCache, use_module_cache);
+  return SetPropertyAtIndex(ePropertyUseModuleCache, use_module_cache);
 }
 
 FileSpec PlatformProperties::GetModuleCacheDirectory() const {
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(
-      nullptr, ePropertyModuleCacheDirectory);
+  return GetPropertyAtIndexAs<FileSpec>(ePropertyModuleCacheDirectory, {});
 }
 
 bool PlatformProperties::SetModuleCacheDirectory(const FileSpec &dir_spec) {
-  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
-      nullptr, ePropertyModuleCacheDirectory, dir_spec);
+  return m_collection_sp->SetPropertyAtIndex(ePropertyModuleCacheDirectory,
+                                             dir_spec);
 }
 
 void PlatformProperties::SetDefaultModuleCacheDirectory(
     const FileSpec &dir_spec) {
   auto f_spec_opt = m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpec(
-        nullptr, false, ePropertyModuleCacheDirectory);
+      ePropertyModuleCacheDirectory);
   assert(f_spec_opt);
   f_spec_opt->SetDefaultValue(dir_spec);
 }
@@ -435,7 +433,7 @@ RecurseCopy_Callback(void *baton, llvm::sys::fs::file_type ft,
     // make the new directory and get in there
     FileSpec dst_dir = rc_baton->dst;
     if (!dst_dir.GetFilename())
-      dst_dir.SetFilename(src.GetLastPathComponent());
+      dst_dir.SetFilename(src.GetFilename());
     Status error = rc_baton->platform_ptr->MakeDirectory(
         dst_dir, lldb::eFilePermissionsDirectoryDefault);
     if (error.Fail()) {
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 5de32e2770238..0f7c15a75bbf4 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -92,9 +92,9 @@ class ProcessOptionValueProperties
 public:
   ProcessOptionValueProperties(ConstString name) : Cloneable(name) {}
 
-  const Property *GetPropertyAtIndex(const ExecutionContext *exe_ctx,
-                                     bool will_modify,
-                                     uint32_t idx) const override {
+  const Property *
+  GetPropertyAtIndex(uint32_t idx,
+                     const ExecutionContext *exe_ctx) const override {
     // When getting the value for a key from the process options, we will
     // always try and grab the setting from the current process if there is
     // one. Else we just use the one from this instance.
@@ -190,169 +190,171 @@ ProcessProperties::~ProcessProperties() = default;
 
 bool ProcessProperties::GetDisableMemoryCache() const {
   const uint32_t idx = ePropertyDisableMemCache;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 uint64_t ProcessProperties::GetMemoryCacheLineSize() const {
   const uint32_t idx = ePropertyMemCacheLineSize;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_process_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_process_properties[idx].default_uint_value);
 }
 
 Args ProcessProperties::GetExtraStartupCommands() const {
   Args args;
   const uint32_t idx = ePropertyExtraStartCommand;
-  m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, idx, args);
+  m_collection_sp->GetPropertyAtIndexAsArgs(idx, args);
   return args;
 }
 
 void ProcessProperties::SetExtraStartupCommands(const Args &args) {
   const uint32_t idx = ePropertyExtraStartCommand;
-  m_collection_sp->SetPropertyAtIndexFromArgs(nullptr, idx, args);
+  m_collection_sp->SetPropertyAtIndexFromArgs(idx, args);
 }
 
 FileSpec ProcessProperties::GetPythonOSPluginPath() const {
   const uint32_t idx = ePropertyPythonOSPluginPath;
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 uint32_t ProcessProperties::GetVirtualAddressableBits() const {
   const uint32_t idx = ePropertyVirtualAddressableBits;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_process_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_process_properties[idx].default_uint_value);
 }
 
 void ProcessProperties::SetVirtualAddressableBits(uint32_t bits) {
   const uint32_t idx = ePropertyVirtualAddressableBits;
-  m_collection_sp->SetPropertyAtIndexAsUInt64(nullptr, idx, bits);
+  SetPropertyAtIndex(idx, bits);
 }
 void ProcessProperties::SetPythonOSPluginPath(const FileSpec &file) {
   const uint32_t idx = ePropertyPythonOSPluginPath;
-  m_collection_sp->SetPropertyAtIndexAsFileSpec(nullptr, idx, file);
+  SetPropertyAtIndex(idx, file);
 }
 
 bool ProcessProperties::GetIgnoreBreakpointsInExpressions() const {
   const uint32_t idx = ePropertyIgnoreBreakpointsInExpressions;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 void ProcessProperties::SetIgnoreBreakpointsInExpressions(bool ignore) {
   const uint32_t idx = ePropertyIgnoreBreakpointsInExpressions;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, ignore);
+  SetPropertyAtIndex(idx, ignore);
 }
 
 bool ProcessProperties::GetUnwindOnErrorInExpressions() const {
   const uint32_t idx = ePropertyUnwindOnErrorInExpressions;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 void ProcessProperties::SetUnwindOnErrorInExpressions(bool ignore) {
   const uint32_t idx = ePropertyUnwindOnErrorInExpressions;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, ignore);
+  SetPropertyAtIndex(idx, ignore);
 }
 
 bool ProcessProperties::GetStopOnSharedLibraryEvents() const {
   const uint32_t idx = ePropertyStopOnSharedLibraryEvents;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 void ProcessProperties::SetStopOnSharedLibraryEvents(bool stop) {
   const uint32_t idx = ePropertyStopOnSharedLibraryEvents;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, stop);
+  SetPropertyAtIndex(idx, stop);
 }
 
 bool ProcessProperties::GetDisableLangRuntimeUnwindPlans() const {
   const uint32_t idx = ePropertyDisableLangRuntimeUnwindPlans;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 void ProcessProperties::SetDisableLangRuntimeUnwindPlans(bool disable) {
   const uint32_t idx = ePropertyDisableLangRuntimeUnwindPlans;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, disable);
+  SetPropertyAtIndex(idx, disable);
   m_process->Flush();
 }
 
 bool ProcessProperties::GetDetachKeepsStopped() const {
   const uint32_t idx = ePropertyDetachKeepsStopped;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 void ProcessProperties::SetDetachKeepsStopped(bool stop) {
   const uint32_t idx = ePropertyDetachKeepsStopped;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, stop);
+  SetPropertyAtIndex(idx, stop);
 }
 
 bool ProcessProperties::GetWarningsOptimization() const {
   const uint32_t idx = ePropertyWarningOptimization;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 bool ProcessProperties::GetWarningsUnsupportedLanguage() const {
   const uint32_t idx = ePropertyWarningUnsupportedLanguage;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 bool ProcessProperties::GetStopOnExec() const {
   const uint32_t idx = ePropertyStopOnExec;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 std::chrono::seconds ProcessProperties::GetUtilityExpressionTimeout() const {
   const uint32_t idx = ePropertyUtilityExpressionTimeout;
-  uint64_t value = m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_process_properties[idx].default_uint_value);
+  uint64_t value = GetPropertyAtIndexAs<uint64_t>(
+      idx, g_process_properties[idx].default_uint_value);
   return std::chrono::seconds(value);
 }
 
 std::chrono::seconds ProcessProperties::GetInterruptTimeout() const {
   const uint32_t idx = ePropertyInterruptTimeout;
-  uint64_t value = m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_process_properties[idx].default_uint_value);
+  uint64_t value = GetPropertyAtIndexAs<uint64_t>(
+      idx, g_process_properties[idx].default_uint_value);
   return std::chrono::seconds(value);
 }
 
 bool ProcessProperties::GetSteppingRunsAllThreads() const {
   const uint32_t idx = ePropertySteppingRunsAllThreads;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_process_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_process_properties[idx].default_uint_value != 0);
 }
 
 bool ProcessProperties::GetOSPluginReportsAllThreads() const {
   const bool fail_value = true;
   const Property *exp_property =
-      m_collection_sp->GetPropertyAtIndex(nullptr, true, ePropertyExperimental);
+      m_collection_sp->GetPropertyAtIndex(ePropertyExperimental);
   OptionValueProperties *exp_values =
       exp_property->GetValue()->GetAsProperties();
   if (!exp_values)
     return fail_value;
 
-  return exp_values->GetPropertyAtIndexAsBoolean(
-      nullptr, ePropertyOSPluginReportsAllThreads, fail_value);
+  return exp_values
+      ->GetPropertyAtIndexAs<bool>(ePropertyOSPluginReportsAllThreads)
+      .value_or(fail_value);
 }
 
 void ProcessProperties::SetOSPluginReportsAllThreads(bool does_report) {
   const Property *exp_property =
-      m_collection_sp->GetPropertyAtIndex(nullptr, true, ePropertyExperimental);
+      m_collection_sp->GetPropertyAtIndex(ePropertyExperimental);
   OptionValueProperties *exp_values =
       exp_property->GetValue()->GetAsProperties();
   if (exp_values)
-    exp_values->SetPropertyAtIndexAsBoolean(
-        nullptr, ePropertyOSPluginReportsAllThreads, does_report);
+    exp_values->SetPropertyAtIndex(ePropertyOSPluginReportsAllThreads,
+                                   does_report);
 }
 
 FollowForkMode ProcessProperties::GetFollowForkMode() const {
   const uint32_t idx = ePropertyFollowForkMode;
-  return (FollowForkMode)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-      nullptr, idx, g_process_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<FollowForkMode>(
+      idx, static_cast<FollowForkMode>(
+               g_process_properties[idx].default_uint_value));
 }
 
 ProcessSP Process::FindPlugin(lldb::TargetSP target_sp,
@@ -479,8 +481,7 @@ Process::Process(lldb::TargetSP target_sp, ListenerSP listener_sp,
 
   // Allow the platform to override the default cache line size
   OptionValueSP value_sp =
-      m_collection_sp
-          ->GetPropertyAtIndex(nullptr, true, ePropertyMemCacheLineSize)
+      m_collection_sp->GetPropertyAtIndex(ePropertyMemCacheLineSize)
           ->GetValue();
   uint32_t platform_cache_line_size =
       target_sp->GetPlatform()->GetDefaultMemoryCacheLineSize();
@@ -1055,14 +1056,16 @@ bool Process::SetExitStatus(int status, const char *cstr) {
   std::lock_guard<std::mutex> guard(m_exit_status_mutex);
 
   Log *log(GetLog(LLDBLog::State | LLDBLog::Process));
-  LLDB_LOGF(
-      log, "Process::SetExitStatus (status=%i (0x%8.8x), description=%s%s%s)",
-      status, status, cstr ? "\"" : "", cstr ? cstr : "NULL", cstr ? "\"" : "");
+  LLDB_LOG(log, "(plugin = %s status=%i (0x%8.8x), description=%s%s%s)",
+           GetPluginName().data(), status, status, cstr ? "\"" : "",
+           cstr ? cstr : "NULL", cstr ? "\"" : "");
 
   // We were already in the exited state
   if (m_private_state.GetValue() == eStateExited) {
-    LLDB_LOGF(log, "Process::SetExitStatus () ignoring exit status because "
-                   "state was already set to eStateExited");
+    LLDB_LOG(log,
+             "(plugin = %s) ignoring exit status because state was already set "
+             "to eStateExited",
+             GetPluginName().data());
     return false;
   }
 
@@ -1314,8 +1317,8 @@ void Process::SetPublicState(StateType new_state, bool restarted) {
   }
 
   Log *log(GetLog(LLDBLog::State | LLDBLog::Process));
-  LLDB_LOGF(log, "Process::SetPublicState (state = %s, restarted = %i)",
-            StateAsCString(new_state), restarted);
+  LLDB_LOG(log, "(plugin = %s, state = %s, restarted = %i)",
+           GetPluginName().data(), StateAsCString(new_state), restarted);
   const StateType old_state = m_public_state.GetValue();
   m_public_state.SetValue(new_state);
 
@@ -1324,16 +1327,16 @@ void Process::SetPublicState(StateType new_state, bool restarted) {
   // program to run.
   if (!StateChangedIsExternallyHijacked()) {
     if (new_state == eStateDetached) {
-      LLDB_LOGF(log,
-                "Process::SetPublicState (%s) -- unlocking run lock for detach",
-                StateAsCString(new_state));
+      LLDB_LOG(log,
+               "(plugin = %s, state = %s) -- unlocking run lock for detach",
+               GetPluginName().data(), StateAsCString(new_state));
       m_public_run_lock.SetStopped();
     } else {
       const bool old_state_is_stopped = StateIsStoppedState(old_state, false);
       if ((old_state_is_stopped != new_state_is_stopped)) {
         if (new_state_is_stopped && !restarted) {
-          LLDB_LOGF(log, "Process::SetPublicState (%s) -- unlocking run lock",
-                    StateAsCString(new_state));
+          LLDB_LOG(log, "(plugin = %s, state = %s) -- unlocking run lock",
+                   GetPluginName().data(), StateAsCString(new_state));
           m_public_run_lock.SetStopped();
         }
       }
@@ -1343,10 +1346,11 @@ void Process::SetPublicState(StateType new_state, bool restarted) {
 
 Status Process::Resume() {
   Log *log(GetLog(LLDBLog::State | LLDBLog::Process));
-  LLDB_LOGF(log, "Process::Resume -- locking run lock");
+  LLDB_LOG(log, "(plugin = %s) -- locking run lock", GetPluginName().data());
   if (!m_public_run_lock.TrySetRunning()) {
     Status error("Resume request failed - process still running.");
-    LLDB_LOGF(log, "Process::Resume: -- TrySetRunning failed, not resuming.");
+    LLDB_LOG(log, "(plugin = %s) -- TrySetRunning failed, not resuming.",
+             GetPluginName().data());
     return error;
   }
   Status error = PrivateResume();
@@ -1419,7 +1423,8 @@ void Process::SetPrivateState(StateType new_state) {
   Log *log(GetLog(LLDBLog::State | LLDBLog::Process | LLDBLog::Unwind));
   bool state_changed = false;
 
-  LLDB_LOGF(log, "Process::SetPrivateState (%s)", StateAsCString(new_state));
+  LLDB_LOG(log, "(plugin = %s, state = %s)", GetPluginName().data(),
+           StateAsCString(new_state));
 
   std::lock_guard<std::recursive_mutex> thread_guard(m_thread_list.GetMutex());
   std::lock_guard<std::recursive_mutex> guard(m_private_state.GetMutex());
@@ -1460,15 +1465,15 @@ void Process::SetPrivateState(StateType new_state) {
       if (!m_mod_id.IsLastResumeForUserExpression())
         m_mod_id.SetStopEventForLastNaturalStopID(event_sp);
       m_memory_cache.Clear();
-      LLDB_LOGF(log, "Process::SetPrivateState (%s) stop_id = %u",
-                StateAsCString(new_state), m_mod_id.GetStopID());
+      LLDB_LOG(log, "(plugin = %s, state = %s, stop_id = %u",
+               GetPluginName().data(), StateAsCString(new_state),
+               m_mod_id.GetStopID());
     }
 
     m_private_state_broadcaster.BroadcastEvent(event_sp);
   } else {
-    LLDB_LOGF(log,
-              "Process::SetPrivateState (%s) state didn't change. Ignoring...",
-              StateAsCString(new_state));
+    LLDB_LOG(log, "(plugin = %s, state = %s) state didn't change. Ignoring...",
+             GetPluginName().data(), StateAsCString(new_state));
   }
 }
 
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index c5946d15a6bc3..0910273421109 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -293,6 +293,7 @@ void Target::Destroy() {
   m_stop_hooks.clear();
   m_stop_hook_next_id = 0;
   m_suppress_stop_hooks = false;
+  m_repl_map.clear();
   Args signal_args;
   ClearDummySignals(signal_args);
 }
@@ -3193,6 +3194,7 @@ Status Target::Launch(ProcessLaunchInfo &launch_info, Stream *stream) {
     // Since we didn't have a platform launch the process, launch it here.
     if (m_process_sp) {
       m_process_sp->HijackProcessEvents(launch_info.GetHijackListener());
+      m_process_sp->SetShadowListener(launch_info.GetShadowListener());
       error = m_process_sp->Launch(launch_info);
     }
   }
@@ -4005,9 +4007,9 @@ class TargetOptionValueProperties
 public:
   TargetOptionValueProperties(ConstString name) : Cloneable(name) {}
 
-  const Property *GetPropertyAtIndex(const ExecutionContext *exe_ctx,
-                                     bool will_modify,
-                                     uint32_t idx) const override {
+  const Property *
+  GetPropertyAtIndex(uint32_t idx,
+                     const ExecutionContext *exe_ctx = nullptr) const override {
     // When getting the value for a key from the target options, we will always
     // try and grab the setting from the current target if there is one. Else
     // we just use the one from this instance.
@@ -4128,13 +4130,14 @@ void TargetProperties::UpdateLaunchInfoFromProperties() {
 
 bool TargetProperties::GetInjectLocalVariables(
     ExecutionContext *exe_ctx) const {
-  const Property *exp_property = m_collection_sp->GetPropertyAtIndex(
-      exe_ctx, false, ePropertyExperimental);
+  const Property *exp_property =
+      m_collection_sp->GetPropertyAtIndex(ePropertyExperimental, exe_ctx);
   OptionValueProperties *exp_values =
       exp_property->GetValue()->GetAsProperties();
   if (exp_values)
-    return exp_values->GetPropertyAtIndexAsBoolean(
-        exe_ctx, ePropertyInjectLocalVars, true);
+    return exp_values
+        ->GetPropertyAtIndexAs<bool>(ePropertyInjectLocalVars, exe_ctx)
+        .value_or(true);
   else
     return true;
 }
@@ -4142,17 +4145,16 @@ bool TargetProperties::GetInjectLocalVariables(
 void TargetProperties::SetInjectLocalVariables(ExecutionContext *exe_ctx,
                                                bool b) {
   const Property *exp_property =
-      m_collection_sp->GetPropertyAtIndex(exe_ctx, true, ePropertyExperimental);
+      m_collection_sp->GetPropertyAtIndex(ePropertyExperimental, exe_ctx);
   OptionValueProperties *exp_values =
       exp_property->GetValue()->GetAsProperties();
   if (exp_values)
-    exp_values->SetPropertyAtIndexAsBoolean(exe_ctx, ePropertyInjectLocalVars,
-                                            true);
+    exp_values->SetPropertyAtIndex(ePropertyInjectLocalVars, true, exe_ctx);
 }
 
 ArchSpec TargetProperties::GetDefaultArchitecture() const {
   OptionValueArch *value = m_collection_sp->GetPropertyAtIndexAsOptionValueArch(
-      nullptr, ePropertyDefaultArch);
+      ePropertyDefaultArch);
   if (value)
     return value->GetCurrentValue();
   return ArchSpec();
@@ -4160,82 +4162,82 @@ ArchSpec TargetProperties::GetDefaultArchitecture() const {
 
 void TargetProperties::SetDefaultArchitecture(const ArchSpec &arch) {
   OptionValueArch *value = m_collection_sp->GetPropertyAtIndexAsOptionValueArch(
-      nullptr, ePropertyDefaultArch);
+      ePropertyDefaultArch);
   if (value)
     return value->SetCurrentValue(arch, true);
 }
 
 bool TargetProperties::GetMoveToNearestCode() const {
   const uint32_t idx = ePropertyMoveToNearestCode;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 lldb::DynamicValueType TargetProperties::GetPreferDynamicValue() const {
   const uint32_t idx = ePropertyPreferDynamic;
-  return (lldb::DynamicValueType)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<lldb::DynamicValueType>(
+      idx, static_cast<lldb::DynamicValueType>(
+               g_target_properties[idx].default_uint_value));
 }
 
 bool TargetProperties::SetPreferDynamicValue(lldb::DynamicValueType d) {
   const uint32_t idx = ePropertyPreferDynamic;
-  return m_collection_sp->SetPropertyAtIndexAsEnumeration(nullptr, idx, d);
+  return SetPropertyAtIndex(idx, d);
 }
 
 bool TargetProperties::GetPreloadSymbols() const {
   const uint32_t idx = ePropertyPreloadSymbols;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetPreloadSymbols(bool b) {
   const uint32_t idx = ePropertyPreloadSymbols;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 bool TargetProperties::GetDisableASLR() const {
   const uint32_t idx = ePropertyDisableASLR;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetDisableASLR(bool b) {
   const uint32_t idx = ePropertyDisableASLR;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 bool TargetProperties::GetInheritTCC() const {
   const uint32_t idx = ePropertyInheritTCC;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetInheritTCC(bool b) {
   const uint32_t idx = ePropertyInheritTCC;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 bool TargetProperties::GetDetachOnError() const {
   const uint32_t idx = ePropertyDetachOnError;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetDetachOnError(bool b) {
   const uint32_t idx = ePropertyDetachOnError;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 bool TargetProperties::GetDisableSTDIO() const {
   const uint32_t idx = ePropertyDisableSTDIO;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetDisableSTDIO(bool b) {
   const uint32_t idx = ePropertyDisableSTDIO;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 const char *TargetProperties::GetDisassemblyFlavor() const {
@@ -4243,38 +4245,41 @@ const char *TargetProperties::GetDisassemblyFlavor() const {
   const char *return_value;
 
   x86DisassemblyFlavor flavor_value =
-      (x86DisassemblyFlavor)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_target_properties[idx].default_uint_value);
+      GetPropertyAtIndexAs<x86DisassemblyFlavor>(
+          idx, static_cast<x86DisassemblyFlavor>(
+                   g_target_properties[idx].default_uint_value));
+
   return_value = g_x86_dis_flavor_value_types[flavor_value].string_value;
   return return_value;
 }
 
 InlineStrategy TargetProperties::GetInlineStrategy() const {
   const uint32_t idx = ePropertyInlineStrategy;
-  return (InlineStrategy)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<InlineStrategy>(
+      idx,
+      static_cast<InlineStrategy>(g_target_properties[idx].default_uint_value));
 }
 
 llvm::StringRef TargetProperties::GetArg0() const {
   const uint32_t idx = ePropertyArg0;
-  return m_collection_sp->GetPropertyAtIndexAsString(nullptr, idx,
-                                                     llvm::StringRef());
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_target_properties[idx].default_cstr_value);
 }
 
 void TargetProperties::SetArg0(llvm::StringRef arg) {
   const uint32_t idx = ePropertyArg0;
-  m_collection_sp->SetPropertyAtIndexAsString(nullptr, idx, arg);
+  SetPropertyAtIndex(idx, arg);
   m_launch_info.SetArg0(arg);
 }
 
 bool TargetProperties::GetRunArguments(Args &args) const {
   const uint32_t idx = ePropertyRunArgs;
-  return m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, idx, args);
+  return m_collection_sp->GetPropertyAtIndexAsArgs(idx, args);
 }
 
 void TargetProperties::SetRunArguments(const Args &args) {
   const uint32_t idx = ePropertyRunArgs;
-  m_collection_sp->SetPropertyAtIndexFromArgs(nullptr, idx, args);
+  m_collection_sp->SetPropertyAtIndexFromArgs(idx, args);
   m_launch_info.GetArguments() = args;
 }
 
@@ -4282,8 +4287,8 @@ Environment TargetProperties::ComputeEnvironment() const {
   Environment env;
 
   if (m_target &&
-      m_collection_sp->GetPropertyAtIndexAsBoolean(
-          nullptr, ePropertyInheritEnv,
+      GetPropertyAtIndexAs<bool>(
+          ePropertyInheritEnv,
           g_target_properties[ePropertyInheritEnv].default_uint_value != 0)) {
     if (auto platform_sp = m_target->GetPlatform()) {
       Environment platform_env = platform_sp->GetEnvironment();
@@ -4293,14 +4298,13 @@ Environment TargetProperties::ComputeEnvironment() const {
   }
 
   Args property_unset_env;
-  m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyUnsetEnvVars,
+  m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyUnsetEnvVars,
                                             property_unset_env);
   for (const auto &var : property_unset_env)
     env.erase(var.ref());
 
   Args property_env;
-  m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyEnvVars,
-                                            property_env);
+  m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyEnvVars, property_env);
   for (const auto &KV : Environment(property_env))
     env[KV.first()] = KV.second;
 
@@ -4317,8 +4321,8 @@ Environment TargetProperties::GetInheritedEnvironment() const {
   if (m_target == nullptr)
     return environment;
 
-  if (!m_collection_sp->GetPropertyAtIndexAsBoolean(
-          nullptr, ePropertyInheritEnv,
+  if (!GetPropertyAtIndexAs<bool>(
+          ePropertyInheritEnv,
           g_target_properties[ePropertyInheritEnv].default_uint_value != 0))
     return environment;
 
@@ -4331,7 +4335,7 @@ Environment TargetProperties::GetInheritedEnvironment() const {
     environment[KV.first()] = KV.second;
 
   Args property_unset_environment;
-  m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyUnsetEnvVars,
+  m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyUnsetEnvVars,
                                             property_unset_environment);
   for (const auto &var : property_unset_environment)
     environment.erase(var.ref());
@@ -4341,7 +4345,7 @@ Environment TargetProperties::GetInheritedEnvironment() const {
 
 Environment TargetProperties::GetTargetEnvironment() const {
   Args property_environment;
-  m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, ePropertyEnvVars,
+  m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyEnvVars,
                                             property_environment);
   Environment environment;
   for (const auto &KV : Environment(property_environment))
@@ -4353,106 +4357,93 @@ Environment TargetProperties::GetTargetEnvironment() const {
 void TargetProperties::SetEnvironment(Environment env) {
   // TODO: Get rid of the Args intermediate step
   const uint32_t idx = ePropertyEnvVars;
-  m_collection_sp->SetPropertyAtIndexFromArgs(nullptr, idx, Args(env));
+  m_collection_sp->SetPropertyAtIndexFromArgs(idx, Args(env));
 }
 
 bool TargetProperties::GetSkipPrologue() const {
   const uint32_t idx = ePropertySkipPrologue;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 PathMappingList &TargetProperties::GetSourcePathMap() const {
   const uint32_t idx = ePropertySourceMap;
   OptionValuePathMappings *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValuePathMappings(nullptr,
-                                                                   false, idx);
+      m_collection_sp->GetPropertyAtIndexAsOptionValuePathMappings(idx);
   assert(option_value);
   return option_value->GetCurrentValue();
 }
 
 bool TargetProperties::GetAutoSourceMapRelative() const {
   const uint32_t idx = ePropertyAutoSourceMapRelative;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::AppendExecutableSearchPaths(const FileSpec &dir) {
   const uint32_t idx = ePropertyExecutableSearchPaths;
   OptionValueFileSpecList *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(nullptr,
-                                                                   false, idx);
+      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(idx);
   assert(option_value);
   option_value->AppendCurrentValue(dir);
 }
 
 FileSpecList TargetProperties::GetExecutableSearchPaths() {
   const uint32_t idx = ePropertyExecutableSearchPaths;
-  const OptionValueFileSpecList *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(nullptr,
-                                                                   false, idx);
-  assert(option_value);
-  return option_value->GetCurrentValue();
+  return GetPropertyAtIndexAs<FileSpecList>(idx, {});
 }
 
 FileSpecList TargetProperties::GetDebugFileSearchPaths() {
   const uint32_t idx = ePropertyDebugFileSearchPaths;
-  const OptionValueFileSpecList *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(nullptr,
-                                                                   false, idx);
-  assert(option_value);
-  return option_value->GetCurrentValue();
+  return GetPropertyAtIndexAs<FileSpecList>(idx, {});
 }
 
 FileSpecList TargetProperties::GetClangModuleSearchPaths() {
   const uint32_t idx = ePropertyClangModuleSearchPaths;
-  const OptionValueFileSpecList *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(nullptr,
-                                                                   false, idx);
-  assert(option_value);
-  return option_value->GetCurrentValue();
+  return GetPropertyAtIndexAs<FileSpecList>(idx, {});
 }
 
 bool TargetProperties::GetEnableAutoImportClangModules() const {
   const uint32_t idx = ePropertyAutoImportClangModules;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 ImportStdModule TargetProperties::GetImportStdModule() const {
   const uint32_t idx = ePropertyImportStdModule;
-  return (ImportStdModule)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<ImportStdModule>(
+      idx, static_cast<ImportStdModule>(
+               g_target_properties[idx].default_uint_value));
 }
 
 DynamicClassInfoHelper TargetProperties::GetDynamicClassInfoHelper() const {
   const uint32_t idx = ePropertyDynamicClassInfoHelper;
-  return (DynamicClassInfoHelper)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<DynamicClassInfoHelper>(
+      idx, static_cast<DynamicClassInfoHelper>(
+               g_target_properties[idx].default_uint_value));
 }
 
 bool TargetProperties::GetEnableAutoApplyFixIts() const {
   const uint32_t idx = ePropertyAutoApplyFixIts;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 uint64_t TargetProperties::GetNumberOfRetriesWithFixits() const {
   const uint32_t idx = ePropertyRetriesWithFixIts;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
 }
 
 bool TargetProperties::GetEnableNotifyAboutFixIts() const {
   const uint32_t idx = ePropertyNotifyAboutFixIts;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 FileSpec TargetProperties::GetSaveJITObjectsDir() const {
   const uint32_t idx = ePropertySaveObjectsDir;
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 void TargetProperties::CheckJITObjectsDir() {
@@ -4468,7 +4459,7 @@ void TargetProperties::CheckJITObjectsDir() {
   if (exists && is_directory && writable)
     return;
 
-  m_collection_sp->GetPropertyAtIndex(nullptr, true, ePropertySaveObjectsDir)
+  m_collection_sp->GetPropertyAtIndex(ePropertySaveObjectsDir)
       ->GetValue()
       ->Clear();
 
@@ -4490,77 +4481,77 @@ void TargetProperties::CheckJITObjectsDir() {
 
 bool TargetProperties::GetEnableSyntheticValue() const {
   const uint32_t idx = ePropertyEnableSynthetic;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 uint32_t TargetProperties::GetMaxZeroPaddingInFloatFormat() const {
   const uint32_t idx = ePropertyMaxZeroPaddingInFloatFormat;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
 }
 
 uint32_t TargetProperties::GetMaximumNumberOfChildrenToDisplay() const {
   const uint32_t idx = ePropertyMaxChildrenCount;
-  return m_collection_sp->GetPropertyAtIndexAsSInt64(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<int64_t>(
+      idx, g_target_properties[idx].default_uint_value);
 }
 
 std::pair<uint32_t, bool>
 TargetProperties::GetMaximumDepthOfChildrenToDisplay() const {
   const uint32_t idx = ePropertyMaxChildrenDepth;
   auto *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueUInt64(nullptr, idx);
+      m_collection_sp->GetPropertyAtIndexAsOptionValueUInt64(idx);
   bool is_default = !option_value->OptionWasSet();
   return {option_value->GetCurrentValue(), is_default};
 }
 
 uint32_t TargetProperties::GetMaximumSizeOfStringSummary() const {
   const uint32_t idx = ePropertyMaxSummaryLength;
-  return m_collection_sp->GetPropertyAtIndexAsSInt64(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
 }
 
 uint32_t TargetProperties::GetMaximumMemReadSize() const {
   const uint32_t idx = ePropertyMaxMemReadSize;
-  return m_collection_sp->GetPropertyAtIndexAsSInt64(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
 }
 
 FileSpec TargetProperties::GetStandardInputPath() const {
   const uint32_t idx = ePropertyInputPath;
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 void TargetProperties::SetStandardInputPath(llvm::StringRef path) {
   const uint32_t idx = ePropertyInputPath;
-  m_collection_sp->SetPropertyAtIndexAsString(nullptr, idx, path);
+  SetPropertyAtIndex(idx, path);
 }
 
 FileSpec TargetProperties::GetStandardOutputPath() const {
   const uint32_t idx = ePropertyOutputPath;
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 void TargetProperties::SetStandardOutputPath(llvm::StringRef path) {
   const uint32_t idx = ePropertyOutputPath;
-  m_collection_sp->SetPropertyAtIndexAsString(nullptr, idx, path);
+  SetPropertyAtIndex(idx, path);
 }
 
 FileSpec TargetProperties::GetStandardErrorPath() const {
   const uint32_t idx = ePropertyErrorPath;
-  return m_collection_sp->GetPropertyAtIndexAsFileSpec(nullptr, idx);
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
 }
 
 void TargetProperties::SetStandardErrorPath(llvm::StringRef path) {
   const uint32_t idx = ePropertyErrorPath;
-  m_collection_sp->SetPropertyAtIndexAsString(nullptr, idx, path);
+  SetPropertyAtIndex(idx, path);
 }
 
 LanguageType TargetProperties::GetLanguage() const {
   OptionValueLanguage *value =
       m_collection_sp->GetPropertyAtIndexAsOptionValueLanguage(
-          nullptr, ePropertyLanguage);
+          ePropertyLanguage);
   if (value)
     return value->GetCurrentValue();
   return LanguageType();
@@ -4569,8 +4560,7 @@ LanguageType TargetProperties::GetLanguage() const {
 llvm::StringRef TargetProperties::GetExpressionPrefixContents() {
   const uint32_t idx = ePropertyExprPrefix;
   OptionValueFileSpec *file =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
-                                                               idx);
+      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpec(idx);
   if (file) {
     DataBufferSP data_sp(file->GetFileContents());
     if (data_sp)
@@ -4583,89 +4573,110 @@ llvm::StringRef TargetProperties::GetExpressionPrefixContents() {
 
 uint64_t TargetProperties::GetExprErrorLimit() const {
   const uint32_t idx = ePropertyExprErrorLimit;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
+}
+
+uint64_t TargetProperties::GetExprAllocAddress() const {
+  const uint32_t idx = ePropertyExprAllocAddress;
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
+}
+
+uint64_t TargetProperties::GetExprAllocSize() const {
+  const uint32_t idx = ePropertyExprAllocSize;
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
+}
+
+uint64_t TargetProperties::GetExprAllocAlign() const {
+  const uint32_t idx = ePropertyExprAllocAlign;
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_target_properties[idx].default_uint_value);
 }
 
 bool TargetProperties::GetBreakpointsConsultPlatformAvoidList() {
   const uint32_t idx = ePropertyBreakpointUseAvoidList;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 bool TargetProperties::GetUseHexImmediates() const {
   const uint32_t idx = ePropertyUseHexImmediates;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 bool TargetProperties::GetUseFastStepping() const {
   const uint32_t idx = ePropertyUseFastStepping;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 bool TargetProperties::GetDisplayExpressionsInCrashlogs() const {
   const uint32_t idx = ePropertyDisplayExpressionsInCrashlogs;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 LoadScriptFromSymFile TargetProperties::GetLoadScriptFromSymbolFile() const {
   const uint32_t idx = ePropertyLoadScriptFromSymbolFile;
-  return (LoadScriptFromSymFile)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<LoadScriptFromSymFile>(
+      idx, static_cast<LoadScriptFromSymFile>(
+               g_target_properties[idx].default_uint_value));
 }
 
 LoadCWDlldbinitFile TargetProperties::GetLoadCWDlldbinitFile() const {
   const uint32_t idx = ePropertyLoadCWDlldbinitFile;
-  return (LoadCWDlldbinitFile)m_collection_sp->GetPropertyAtIndexAsEnumeration(
-      nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<LoadCWDlldbinitFile>(
+      idx, static_cast<LoadCWDlldbinitFile>(
+               g_target_properties[idx].default_uint_value));
 }
 
 Disassembler::HexImmediateStyle TargetProperties::GetHexImmediateStyle() const {
   const uint32_t idx = ePropertyHexImmediateStyle;
-  return (Disassembler::HexImmediateStyle)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<Disassembler::HexImmediateStyle>(
+      idx, static_cast<Disassembler::HexImmediateStyle>(
+               g_target_properties[idx].default_uint_value));
 }
 
 MemoryModuleLoadLevel TargetProperties::GetMemoryModuleLoadLevel() const {
   const uint32_t idx = ePropertyMemoryModuleLoadLevel;
-  return (MemoryModuleLoadLevel)
-      m_collection_sp->GetPropertyAtIndexAsEnumeration(
-          nullptr, idx, g_target_properties[idx].default_uint_value);
+  return GetPropertyAtIndexAs<MemoryModuleLoadLevel>(
+      idx, static_cast<MemoryModuleLoadLevel>(
+               g_target_properties[idx].default_uint_value));
 }
 
 bool TargetProperties::GetUserSpecifiedTrapHandlerNames(Args &args) const {
   const uint32_t idx = ePropertyTrapHandlerNames;
-  return m_collection_sp->GetPropertyAtIndexAsArgs(nullptr, idx, args);
+  return m_collection_sp->GetPropertyAtIndexAsArgs(idx, args);
 }
 
 void TargetProperties::SetUserSpecifiedTrapHandlerNames(const Args &args) {
   const uint32_t idx = ePropertyTrapHandlerNames;
-  m_collection_sp->SetPropertyAtIndexFromArgs(nullptr, idx, args);
+  m_collection_sp->SetPropertyAtIndexFromArgs(idx, args);
 }
 
 bool TargetProperties::GetDisplayRuntimeSupportValues() const {
   const uint32_t idx = ePropertyDisplayRuntimeSupportValues;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, false);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetDisplayRuntimeSupportValues(bool b) {
   const uint32_t idx = ePropertyDisplayRuntimeSupportValues;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 bool TargetProperties::GetDisplayRecognizedArguments() const {
   const uint32_t idx = ePropertyDisplayRecognizedArguments;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(nullptr, idx, false);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetDisplayRecognizedArguments(bool b) {
   const uint32_t idx = ePropertyDisplayRecognizedArguments;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  SetPropertyAtIndex(idx, b);
 }
 
 const ProcessLaunchInfo &TargetProperties::GetProcessLaunchInfo() const {
@@ -4702,19 +4713,19 @@ void TargetProperties::SetProcessLaunchInfo(
 
 bool TargetProperties::GetRequireHardwareBreakpoints() const {
   const uint32_t idx = ePropertyRequireHardwareBreakpoints;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetRequireHardwareBreakpoints(bool b) {
   const uint32_t idx = ePropertyRequireHardwareBreakpoints;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, b);
+  m_collection_sp->SetPropertyAtIndex(idx, b);
 }
 
 bool TargetProperties::GetAutoInstallMainExecutable() const {
   const uint32_t idx = ePropertyAutoInstallMainExecutable;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::Arg0ValueChangedCallback() {
@@ -4776,13 +4787,13 @@ void TargetProperties::DisableSTDIOValueChangedCallback() {
 
 bool TargetProperties::GetDebugUtilityExpression() const {
   const uint32_t idx = ePropertyDebugUtilityExpression;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_target_properties[idx].default_uint_value != 0);
 }
 
 void TargetProperties::SetDebugUtilityExpression(bool debug) {
   const uint32_t idx = ePropertyDebugUtilityExpression;
-  m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, debug);
+  SetPropertyAtIndex(idx, debug);
 }
 
 // Target::TargetEventData
diff --git a/lldb/source/Target/TargetProperties.td b/lldb/source/Target/TargetProperties.td
index 51bfd28914af9..e8350cb9e2799 100644
--- a/lldb/source/Target/TargetProperties.td
+++ b/lldb/source/Target/TargetProperties.td
@@ -24,6 +24,15 @@ let Definition = "target" in {
     DefaultUnsignedValue<5>,
     Desc<"The maximum amount of errors to emit while parsing an expression. "
          "A value of 0 means to always continue parsing if possible.">;
+  def ExprAllocAddress: Property<"expr-alloc-address", "UInt64">,
+    DefaultUnsignedValue<0>,
+    Desc<"Start address within the process address space of memory allocation for expression evaluation.">;
+  def ExprAllocSize: Property<"expr-alloc-size", "UInt64">,
+    DefaultUnsignedValue<0>,
+    Desc<"Amount of memory in bytes to allocate for expression evaluation.">;
+  def ExprAllocAlign: Property<"expr-alloc-align", "UInt64">,
+    DefaultUnsignedValue<0>,
+    Desc<"Alignment for each memory allocation for expression evaluation.">;
   def PreferDynamic: Property<"prefer-dynamic-value", "Enum">,
     DefaultEnumValue<"eDynamicDontRunTarget">,
     EnumValues<"OptionEnumValues(g_dynamic_value_types)">,
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 98d05fa292a40..b5caf402b66fa 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -78,9 +78,9 @@ class ThreadOptionValueProperties
 public:
   ThreadOptionValueProperties(ConstString name) : Cloneable(name) {}
 
-  const Property *GetPropertyAtIndex(const ExecutionContext *exe_ctx,
-                                     bool will_modify,
-                                     uint32_t idx) const override {
+  const Property *
+  GetPropertyAtIndex(uint32_t idx,
+                     const ExecutionContext *exe_ctx) const override {
     // When getting the value for a key from the thread options, we will always
     // try and grab the setting from the current thread if there is one. Else
     // we just use the one from this instance.
@@ -112,40 +112,36 @@ ThreadProperties::~ThreadProperties() = default;
 
 const RegularExpression *ThreadProperties::GetSymbolsToAvoidRegexp() {
   const uint32_t idx = ePropertyStepAvoidRegex;
-  return m_collection_sp->GetPropertyAtIndexAsOptionValueRegex(nullptr, idx);
+  return GetPropertyAtIndexAs<const RegularExpression *>(idx);
 }
 
 FileSpecList ThreadProperties::GetLibrariesToAvoid() const {
   const uint32_t idx = ePropertyStepAvoidLibraries;
-  const OptionValueFileSpecList *option_value =
-      m_collection_sp->GetPropertyAtIndexAsOptionValueFileSpecList(nullptr,
-                                                                   false, idx);
-  assert(option_value);
-  return option_value->GetCurrentValue();
+  return GetPropertyAtIndexAs<FileSpecList>(idx, {});
 }
 
 bool ThreadProperties::GetTraceEnabledState() const {
   const uint32_t idx = ePropertyEnableThreadTrace;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_thread_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_thread_properties[idx].default_uint_value != 0);
 }
 
 bool ThreadProperties::GetStepInAvoidsNoDebug() const {
   const uint32_t idx = ePropertyStepInAvoidsNoDebug;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_thread_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_thread_properties[idx].default_uint_value != 0);
 }
 
 bool ThreadProperties::GetStepOutAvoidsNoDebug() const {
   const uint32_t idx = ePropertyStepOutAvoidsNoDebug;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_thread_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<bool>(
+      idx, g_thread_properties[idx].default_uint_value != 0);
 }
 
 uint64_t ThreadProperties::GetMaxBacktraceDepth() const {
   const uint32_t idx = ePropertyMaxBacktraceDepth;
-  return m_collection_sp->GetPropertyAtIndexAsUInt64(
-      nullptr, idx, g_thread_properties[idx].default_uint_value != 0);
+  return GetPropertyAtIndexAs<uint64_t>(
+      idx, g_thread_properties[idx].default_uint_value);
 }
 
 // Thread Event Data
@@ -303,10 +299,17 @@ bool Thread::SetSelectedFrameByIndexNoisily(uint32_t frame_idx,
       bool already_shown = false;
       SymbolContext frame_sc(
           frame_sp->GetSymbolContext(eSymbolContextLineEntry));
-      if (GetProcess()->GetTarget().GetDebugger().GetUseExternalEditor() &&
-          frame_sc.line_entry.file && frame_sc.line_entry.line != 0) {
-        already_shown = Host::OpenFileInExternalEditor(
-            frame_sc.line_entry.file, frame_sc.line_entry.line);
+      const Debugger &debugger = GetProcess()->GetTarget().GetDebugger();
+      if (debugger.GetUseExternalEditor() && frame_sc.line_entry.file &&
+          frame_sc.line_entry.line != 0) {
+        if (llvm::Error e = Host::OpenFileInExternalEditor(
+                debugger.GetExternalEditor(), frame_sc.line_entry.file,
+                frame_sc.line_entry.line)) {
+          LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(e),
+                         "OpenFileInExternalEditor failed: {0}");
+        } else {
+          already_shown = true;
+        }
       }
 
       bool show_frame_info = true;
@@ -707,7 +710,7 @@ bool Thread::ShouldResume(StateType resume_state) {
   return need_to_resume;
 }
 
-void Thread::DidResume() { 
+void Thread::DidResume() {
   SetResumeSignal(LLDB_INVALID_SIGNAL_NUMBER);
   // This will get recomputed each time when we stop.
   SetShouldRunBeforePublicStop(false);
@@ -751,7 +754,7 @@ bool Thread::ShouldStop(Event *event_ptr) {
                                    : LLDB_INVALID_ADDRESS);
     return false;
   }
-  
+
   // Clear the "must run me before stop" if it was set:
   SetShouldRunBeforePublicStop(false);
 
@@ -981,7 +984,7 @@ Vote Thread::ShouldReportStop(Event *event_ptr) {
   }
 
   if (GetPlans().AnyCompletedPlans()) {
-    // Pass skip_private = false to GetCompletedPlan, since we want to ask 
+    // Pass skip_private = false to GetCompletedPlan, since we want to ask
     // the last plan, regardless of whether it is private or not.
     LLDB_LOGF(log,
               "Thread::ShouldReportStop() tid = 0x%4.4" PRIx64
@@ -1019,7 +1022,7 @@ Vote Thread::ShouldReportRun(Event *event_ptr) {
 
   Log *log = GetLog(LLDBLog::Step);
   if (GetPlans().AnyCompletedPlans()) {
-    // Pass skip_private = false to GetCompletedPlan, since we want to ask 
+    // Pass skip_private = false to GetCompletedPlan, since we want to ask
     // the last plan, regardless of whether it is private or not.
     LLDB_LOGF(log,
               "Current Plan for thread %d(%p) (0x%4.4" PRIx64
@@ -1072,7 +1075,7 @@ void Thread::PushPlan(ThreadPlanSP thread_plan_sp) {
               static_cast<void *>(this), s.GetData(),
               thread_plan_sp->GetThread().GetID());
   }
-  
+
   GetPlans().PushPlan(std::move(thread_plan_sp));
 }
 
@@ -1090,7 +1093,7 @@ void Thread::DiscardPlan() {
   ThreadPlanSP discarded_plan_sp = GetPlans().DiscardPlan();
 
   LLDB_LOGF(log, "Discarding plan: \"%s\", tid = 0x%4.4" PRIx64 ".",
-            discarded_plan_sp->GetName(), 
+            discarded_plan_sp->GetName(),
             discarded_plan_sp->GetThread().GetID());
 }
 
@@ -1219,7 +1222,7 @@ Status Thread::UnwindInnermostExpression() {
   if (!innermost_expr_plan) {
     error.SetErrorString("No expressions currently active on this thread");
     return error;
-  }  
+  }
   DiscardThreadPlansUpToPlan(innermost_expr_plan);
   return error;
 }
@@ -1363,8 +1366,8 @@ ThreadPlanSP Thread::QueueThreadPlanForStepUntil(
 }
 
 lldb::ThreadPlanSP Thread::QueueThreadPlanForStepScripted(
-    bool abort_other_plans, const char *class_name, 
-    StructuredData::ObjectSP extra_args_sp,  bool stop_other_threads,
+    bool abort_other_plans, const char *class_name,
+    StructuredData::ObjectSP extra_args_sp, bool stop_other_threads,
     Status &status) {
 
   ThreadPlanSP thread_plan_sp(new ThreadPlanPython(
@@ -1725,8 +1728,12 @@ size_t Thread::GetStatus(Stream &strm, uint32_t start_frame,
         SymbolContext frame_sc(
             frame_sp->GetSymbolContext(eSymbolContextLineEntry));
         if (frame_sc.line_entry.line != 0 && frame_sc.line_entry.file) {
-          Host::OpenFileInExternalEditor(frame_sc.line_entry.file,
-                                         frame_sc.line_entry.line);
+          if (llvm::Error e = Host::OpenFileInExternalEditor(
+                  target->GetDebugger().GetExternalEditor(),
+                  frame_sc.line_entry.file, frame_sc.line_entry.line)) {
+            LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(e),
+                           "OpenFileInExternalEditor failed: {0}");
+          }
         }
       }
     }
diff --git a/lldb/source/Target/ThreadPlanPython.cpp b/lldb/source/Target/ThreadPlanPython.cpp
index bc2a7a02e99c6..d6de6b3c3cf04 100644
--- a/lldb/source/Target/ThreadPlanPython.cpp
+++ b/lldb/source/Target/ThreadPlanPython.cpp
@@ -136,8 +136,11 @@ bool ThreadPlanPython::MischiefManaged() {
     // I don't really need mischief_managed, since it's simpler to just call
     // SetPlanComplete in should_stop.
     mischief_managed = IsPlanComplete();
-    if (mischief_managed)
+    if (mischief_managed) {
+      // We need to cache the stop reason here we'll need it in GetDescription.
+      GetDescription(&m_stop_description, eDescriptionLevelBrief);
       m_implementation_sp.reset();
+    }
   }
   return mischief_managed;
 }
@@ -158,15 +161,40 @@ lldb::StateType ThreadPlanPython::GetPlanRunState() {
   return run_state;
 }
 
-// The ones below are not currently exported to Python.
 void ThreadPlanPython::GetDescription(Stream *s, lldb::DescriptionLevel level) {
-  s->Printf("Python thread plan implemented by class %s.",
+  Log *log = GetLog(LLDBLog::Thread);
+  LLDB_LOGF(log, "%s called on Python Thread Plan: %s )", LLVM_PRETTY_FUNCTION,
+            m_class_name.c_str());
+  if (m_implementation_sp) {
+    ScriptInterpreter *script_interp = GetScriptInterpreter();
+    if (script_interp) {
+      bool script_error;
+      bool added_desc = script_interp->ScriptedThreadPlanGetStopDescription(
+          m_implementation_sp, s, script_error);
+      if (script_error || !added_desc)
+        s->Printf("Python thread plan implemented by class %s.",
             m_class_name.c_str());
+    }
+    return;
+  }
+  // It's an error not to have a description, so if we get here, we should
+  // add something.
+  if (m_stop_description.Empty())
+    s->Printf("Python thread plan implemented by class %s.",
+              m_class_name.c_str());
+  s->PutCString(m_stop_description.GetData());
 }
 
+// The ones below are not currently exported to Python.
 bool ThreadPlanPython::WillStop() {
   Log *log = GetLog(LLDBLog::Thread);
   LLDB_LOGF(log, "%s called on Python Thread Plan: %s )", LLVM_PRETTY_FUNCTION,
             m_class_name.c_str());
   return true;
 }
+
+bool ThreadPlanPython::DoWillResume(lldb::StateType resume_state, 
+                                  bool current_plan) {
+  m_stop_description.Clear();
+  return true;                                  
+}
diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index 475d0c8a63a7b..fb0e985a0d565 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -543,7 +543,6 @@ void ArchSpec::Clear() {
   m_triple = llvm::Triple();
   m_core = kCore_invalid;
   m_byte_order = eByteOrderInvalid;
-  m_distribution_id.Clear();
   m_flags = 0;
 }
 
@@ -689,14 +688,6 @@ llvm::Triple::ArchType ArchSpec::GetMachine() const {
   return llvm::Triple::UnknownArch;
 }
 
-ConstString ArchSpec::GetDistributionId() const {
-  return m_distribution_id;
-}
-
-void ArchSpec::SetDistributionId(const char *distribution_id) {
-  m_distribution_id.SetCString(distribution_id);
-}
-
 uint32_t ArchSpec::GetAddressByteSize() const {
   const CoreDefinition *core_def = FindCoreDefinition(m_core);
   if (core_def) {
@@ -979,8 +970,6 @@ static bool IsCompatibleEnvironment(llvm::Triple::EnvironmentType lhs,
 }
 
 bool ArchSpec::IsMatch(const ArchSpec &rhs, MatchType match) const {
-  // explicitly ignoring m_distribution_id in this method.
-
   if (GetByteOrder() != rhs.GetByteOrder() ||
       !cores_match(GetCore(), rhs.GetCore(), true, match == ExactMatch))
     return false;
diff --git a/lldb/source/Utility/Broadcaster.cpp b/lldb/source/Utility/Broadcaster.cpp
index 31530f8a6443d..4e6710e1108b3 100644
--- a/lldb/source/Utility/Broadcaster.cpp
+++ b/lldb/source/Utility/Broadcaster.cpp
@@ -228,6 +228,8 @@ void Broadcaster::BroadcasterImpl::PrivateBroadcastEvent(EventSP &event_sp,
                       &m_broadcaster, event_type))
       return;
     hijacking_listener_sp->AddEvent(event_sp);
+    if (m_shadow_listener)
+      m_shadow_listener->AddEvent(event_sp);
   } else {
     for (auto &pair : GetListeners()) {
       if (!(pair.second & event_type))
@@ -237,6 +239,8 @@ void Broadcaster::BroadcasterImpl::PrivateBroadcastEvent(EventSP &event_sp,
         continue;
 
       pair.first->AddEvent(event_sp);
+      if (m_shadow_listener)
+        m_shadow_listener->AddEvent(event_sp);
     }
   }
 }
diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt
index c48ccdd7c1ed1..d8c18b189f6c7 100644
--- a/lldb/source/Utility/CMakeLists.txt
+++ b/lldb/source/Utility/CMakeLists.txt
@@ -39,6 +39,7 @@ add_lldb_library(lldbUtility NO_INTERNAL_DEPENDENCIES
   Environment.cpp
   Event.cpp
   FileSpec.cpp
+  FileSpecList.cpp
   GDBRemote.cpp
   IOObject.cpp
   LLDBAssert.cpp
diff --git a/lldb/source/Utility/FileSpec.cpp b/lldb/source/Utility/FileSpec.cpp
index 939f967d7a91b..6688e45650505 100644
--- a/lldb/source/Utility/FileSpec.cpp
+++ b/lldb/source/Utility/FileSpec.cpp
@@ -399,9 +399,8 @@ void FileSpec::GetPath(llvm::SmallVectorImpl<char> &path,
     Denormalize(path, m_style);
 }
 
-ConstString FileSpec::GetFileNameExtension() const {
-  return ConstString(
-      llvm::sys::path::extension(m_filename.GetStringRef(), m_style));
+llvm::StringRef FileSpec::GetFileNameExtension() const {
+  return llvm::sys::path::extension(m_filename.GetStringRef(), m_style);
 }
 
 ConstString FileSpec::GetFileNameStrippingExtension() const {
@@ -430,12 +429,6 @@ FileSpec FileSpec::CopyByRemovingLastPathComponent() const {
   return *this;
 }
 
-ConstString FileSpec::GetLastPathComponent() const {
-  llvm::SmallString<64> current_path;
-  GetPath(current_path, false);
-  return ConstString(llvm::sys::path::filename(current_path, m_style));
-}
-
 void FileSpec::PrependPathComponent(llvm::StringRef component) {
   llvm::SmallString<64> new_path(component);
   llvm::SmallString<64> current_path;
@@ -478,8 +471,8 @@ bool FileSpec::RemoveLastPathComponent() {
 ///     \b true if the filespec represents an implementation source
 ///     file, \b false otherwise.
 bool FileSpec::IsSourceImplementationFile() const {
-  ConstString extension(GetFileNameExtension());
-  if (!extension)
+  llvm::StringRef extension = GetFileNameExtension();
+  if (extension.empty())
     return false;
 
   static RegularExpression g_source_file_regex(llvm::StringRef(
@@ -487,7 +480,7 @@ bool FileSpec::IsSourceImplementationFile() const {
       "cC][pP]|[sS]|[aA][sS][mM]|[fF]|[fF]77|[fF]90|[fF]95|[fF]03|[fF][oO]["
       "rR]|[fF][tT][nN]|[fF][pP][pP]|[aA][dD][aA]|[aA][dD][bB]|[aA][dD][sS])"
       "$"));
-  return g_source_file_regex.Execute(extension.GetStringRef());
+  return g_source_file_regex.Execute(extension);
 }
 
 bool FileSpec::IsRelative() const {
diff --git a/lldb/source/Core/FileSpecList.cpp b/lldb/source/Utility/FileSpecList.cpp
similarity index 99%
rename from lldb/source/Core/FileSpecList.cpp
rename to lldb/source/Utility/FileSpecList.cpp
index 7fa511176e468..e5e0ac3e5981b 100644
--- a/lldb/source/Core/FileSpecList.cpp
+++ b/lldb/source/Utility/FileSpecList.cpp
@@ -6,14 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Core/FileSpecList.h"
-
+#include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Stream.h"
 
-#include <utility>
-
 #include <cstdint>
+#include <utility>
 
 using namespace lldb_private;
 
diff --git a/lldb/source/Utility/Listener.cpp b/lldb/source/Utility/Listener.cpp
index 8bb55f93bbf8c..2286827c51385 100644
--- a/lldb/source/Utility/Listener.cpp
+++ b/lldb/source/Utility/Listener.cpp
@@ -35,7 +35,7 @@ class BroadcasterManagerWPMatcher {
 
 Listener::Listener(const char *name)
     : m_name(name), m_broadcasters(), m_broadcasters_mutex(), m_events(),
-      m_events_mutex() {
+      m_events_mutex(), m_is_shadow() {
   Log *log = GetLog(LLDBLog::Object);
   if (log != nullptr)
     LLDB_LOGF(log, "%p Listener::Listener('%s')", static_cast<void *>(this),
@@ -302,7 +302,8 @@ bool Listener::FindNextEventInternal(
       // to return it so it should be okay to get the next event off the queue
       // here - and it might be useful to do that in the "DoOnRemoval".
       lock.unlock();
-      event_sp->DoOnRemoval();
+      if (!m_is_shadow)
+        event_sp->DoOnRemoval();
     }
     return true;
   }
diff --git a/lldb/source/Utility/ProcessInfo.cpp b/lldb/source/Utility/ProcessInfo.cpp
index 595c0b090401a..6b2a7114dfb4c 100644
--- a/lldb/source/Utility/ProcessInfo.cpp
+++ b/lldb/source/Utility/ProcessInfo.cpp
@@ -22,12 +22,14 @@ using namespace lldb;
 using namespace lldb_private;
 
 ProcessInfo::ProcessInfo()
-    : m_executable(), m_arguments(), m_environment(), m_arch() {}
+    : m_executable(), m_arguments(), m_environment(), m_arch(), m_listener_sp(),
+      m_hijack_listener_sp(), m_shadow_listener_sp() {}
 
 ProcessInfo::ProcessInfo(const char *name, const ArchSpec &arch,
                          lldb::pid_t pid)
     : m_executable(name), m_arguments(), m_environment(), m_arch(arch),
-      m_pid(pid) {}
+      m_pid(pid), m_listener_sp(), m_hijack_listener_sp(),
+      m_shadow_listener_sp() {}
 
 void ProcessInfo::Clear() {
   m_executable.Clear();
diff --git a/lldb/source/Utility/XcodeSDK.cpp b/lldb/source/Utility/XcodeSDK.cpp
index 9e382f1b51087..84f3ccbd01e2d 100644
--- a/lldb/source/Utility/XcodeSDK.cpp
+++ b/lldb/source/Utility/XcodeSDK.cpp
@@ -242,7 +242,7 @@ bool XcodeSDK::SupportsSwift() const {
 
 bool XcodeSDK::SDKSupportsModules(XcodeSDK::Type desired_type,
                                   const FileSpec &sdk_path) {
-  ConstString last_path_component = sdk_path.GetLastPathComponent();
+  ConstString last_path_component = sdk_path.GetFilename();
 
   if (!last_path_component)
     return false;
diff --git a/lldb/test/API/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py b/lldb/test/API/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py
index e14f160f1e576..2ee90181d9327 100644
--- a/lldb/test/API/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py
+++ b/lldb/test/API/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py
@@ -2,4 +2,4 @@
 from lldbsuite.test import decorators
 
 lldbinline.MakeInlineTest(__file__, globals(),
-  [decorators.expectedFailureAll(bugnumber="llvm.org/pr50814", compiler="gcc")])
+  [decorators.expectedFailureAll(bugnumber="llvm.org/pr50814", compiler="gcc"), decorators.expectedFailureAll(oslist=["windows"])])
diff --git a/lldb/test/API/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py
index 1cf227e4c3a89..60a23e95acea2 100644
--- a/lldb/test/API/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py
@@ -31,7 +31,7 @@ def test(self):
             "s",
             result_type=ptr_type,
             result_summary="3",
-            result_children=[ValueCheck(name="__value_")])
+            result_children=[ValueCheck(name="pointer")])
         self.expect_expr("*s", result_type="int", result_value="3")
         self.expect_expr("*s = 5", result_type="int", result_value="5")
         self.expect_expr("*s", result_type="int", result_value="5")
diff --git a/lldb/test/API/commands/expression/memory-allocation/Makefile b/lldb/test/API/commands/expression/memory-allocation/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/test/API/commands/expression/memory-allocation/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/expression/memory-allocation/TestMemoryAllocSettings.py b/lldb/test/API/commands/expression/memory-allocation/TestMemoryAllocSettings.py
new file mode 100644
index 0000000000000..3dc8dc16495c9
--- /dev/null
+++ b/lldb/test/API/commands/expression/memory-allocation/TestMemoryAllocSettings.py
@@ -0,0 +1,35 @@
+"""
+Test changing setting for expression memory allocation.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestMemoryAllocSettings(TestBase):
+
+    def test(self):
+        """Test changing settings for expression memory allocation."""
+        self.build()
+        target = self.createTestTarget()
+
+        self.log_file = self.getBuildArtifact("log-expr.txt")
+
+        self.runCmd("settings set target.expr-alloc-address 0xdead0000")
+        self.runCmd("settings set target.expr-alloc-size 10000")
+        self.runCmd("settings set target.expr-alloc-align 0x1000")
+
+        self.runCmd("log enable lldb expr -f " + self.log_file)
+        self.runCmd("expression -- int foo; &foo")
+
+        self.assertTrue(os.path.isfile(self.log_file))
+        with open(self.log_file, 'r') as f:
+            log = f.read()
+
+        alloc0 = re.search('^.*IRMemoryMap::Malloc.+?0xdead0000.*$', log, re.MULTILINE)
+        # Malloc adds additional bytes to allocation size, hence 10007
+        alloc1 = re.search('^.*IRMemoryMap::Malloc\s*?\(10007.+?0xdead1000.*$', log, re.MULTILINE)
+        self.assertTrue(alloc0, "Couldn't find an allocation at a given address.")
+        self.assertTrue(alloc1, "Couldn't find an allocation of a given size at a given address.")
+
diff --git a/lldb/test/API/commands/expression/memory-allocation/main.cpp b/lldb/test/API/commands/expression/memory-allocation/main.cpp
new file mode 100644
index 0000000000000..4cce7f667ff72
--- /dev/null
+++ b/lldb/test/API/commands/expression/memory-allocation/main.cpp
@@ -0,0 +1,3 @@
+int main() {
+  return 0;
+}
diff --git a/lldb/test/API/commands/expression/save_jit_objects/TestSaveJITObjects.py b/lldb/test/API/commands/expression/save_jit_objects/TestSaveJITObjects.py
index 157bed4880a3d..e3e20ad94707d 100644
--- a/lldb/test/API/commands/expression/save_jit_objects/TestSaveJITObjects.py
+++ b/lldb/test/API/commands/expression/save_jit_objects/TestSaveJITObjects.py
@@ -22,7 +22,6 @@ def cleanJITFiles(self):
             os.remove(j)
         return
 
-    @expectedFailureAll(oslist=["windows"])
     def test_save_jit_objects(self):
         self.build()
         os.chdir(self.getBuildDir())
diff --git a/lldb/test/API/commands/settings/TestSettings.py b/lldb/test/API/commands/settings/TestSettings.py
index e50a4f385344d..82d3d886276ba 100644
--- a/lldb/test/API/commands/settings/TestSettings.py
+++ b/lldb/test/API/commands/settings/TestSettings.py
@@ -60,7 +60,7 @@ def cleanup(setting):
         self.assertEqual(self.dbg.GetSelectedTarget().GetNumBreakpoints(), 1)
 
     def test_append_target_env_vars(self):
-        """Test that 'append target.run-args' works."""
+        """Test that 'append target.env-vars' works."""
         # Append the env-vars.
         self.runCmd('settings append target.env-vars MY_ENV_VAR=YES')
         # And add hooks to restore the settings during tearDown().
@@ -268,7 +268,7 @@ def do_test_run_args_and_env_vars(self, use_launchsimple):
                 found_env_var = True
                 break
         self.assertTrue(found_env_var,
-                        "MY_ENV_VAR was not set in LunchInfo object")
+                        "MY_ENV_VAR was not set in LaunchInfo object")
 
         self.assertEqual(launch_info.GetNumArguments(), 3)
         self.assertEqual(launch_info.GetArgumentAtIndex(0), "A")
diff --git a/lldb/test/API/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py b/lldb/test/API/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py
index 8a143bdf3817a..edac8b1981fdf 100644
--- a/lldb/test/API/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py
+++ b/lldb/test/API/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py
@@ -13,20 +13,17 @@ class TestScriptedResolver(TestBase):
 
     NO_DEBUG_INFO_TESTCASE = True
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24528")
     def test_scripted_resolver(self):
         """Use a scripted resolver to set a by symbol name breakpoint"""
         self.build()
         self.do_test()
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24528")
     def test_search_depths(self):
         """ Make sure we are called at the right depths depending on what we return
             from __get_depth__"""
         self.build()
         self.do_test_depths()
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24528")
     def test_command_line(self):
         """ Test setting a resolver breakpoint from the command line """
         self.build()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py b/lldb/test/API/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py
index 08375c699a5b0..9025de207e6dc 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py
@@ -17,6 +17,7 @@ def setUp(self):
         self.line = line_number('main.cpp', '// Set break point at this line.')
 
     @expectedFlakeyNetBSD
+    @expectedFailureAll(oslist=["windows"])
     def test_with_run_command(self):
         """Test that that file and class static variables display correctly."""
         self.build()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
index cc147de4ff481..a2d16375f1b86 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
@@ -40,7 +40,7 @@ def test_unique_ptr_variables(self):
             "up_empty",
             type=self.make_expected_type("int"),
             summary="nullptr",
-            children=[ValueCheck(name="__value_")],
+            children=[ValueCheck(name="pointer")],
         )
         self.assertEqual(
             valobj.child[0].GetValueAsUnsigned(lldb.LLDB_INVALID_ADDRESS), 0
@@ -54,7 +54,7 @@ def test_unique_ptr_variables(self):
             "up_int",
             type=self.make_expected_type("int"),
             summary="10",
-            children=[ValueCheck(name="__value_")],
+            children=[ValueCheck(name="pointer")],
         )
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
@@ -62,7 +62,7 @@ def test_unique_ptr_variables(self):
             "up_int_ref",
             type=self.make_expected_type("int", qualifiers="&"),
             summary="10",
-            children=[ValueCheck(name="__value_")],
+            children=[ValueCheck(name="pointer")],
         )
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
@@ -70,7 +70,7 @@ def test_unique_ptr_variables(self):
             "up_int_ref_ref",
             type=self.make_expected_type("int", qualifiers="&&"),
             summary="10",
-            children=[ValueCheck(name="__value_")],
+            children=[ValueCheck(name="pointer")],
         )
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
@@ -78,7 +78,7 @@ def test_unique_ptr_variables(self):
             "up_str",
             type=self.make_expected_basic_string_ptr(),
             summary='"hello"',
-            children=[ValueCheck(name="__value_", summary='"hello"')],
+            children=[ValueCheck(name="pointer", summary='"hello"')],
         )
 
         valobj = self.expect_var_path(
@@ -95,7 +95,20 @@ def test_unique_ptr_variables(self):
                 ValueCheck(name="name", summary='"steph"'),
             ],
         )
-        self.assertEqual(str(valobj), '(User) *__value_ = (id = 30, name = "steph")')
+        self.assertEqual(str(valobj), '(User) *pointer = (id = 30, name = "steph")')
+
+        valobj = self.expect_var_path(
+            "up_non_empty_deleter",
+            type="std::unique_ptr<int, NonEmptyIntDeleter>",
+            summary="1234",
+            children=[
+                ValueCheck(name="pointer"),
+                ValueCheck(name="deleter", children=[
+                    ValueCheck(name="dummy_", value="9999")
+                ]),
+            ],
+        )
+        self.assertNotEqual(valobj.child[0].unsigned, 0)
 
         self.expect_var_path("up_user->id", type="int", value="30")
         self.expect_var_path("up_user->name", type="std::string", summary='"steph"')
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
index 5201b2a6e9cb3..ab65d95b248ef 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
@@ -6,6 +6,15 @@ struct User {
   std::string name = "steph";
 };
 
+// libc++ stores unique_ptr data in a compressed pair, which has a specialized
+// representation when the type of the second element is an empty class. So
+// we need a deleter class with a dummy data member to trigger the other path.
+struct NonEmptyIntDeleter {
+  void operator()(int* ptr) { delete ptr; }
+
+  int dummy_ = 9999;
+};
+
 int main() {
   std::unique_ptr<int> up_empty;
   std::unique_ptr<int> up_int = std::make_unique<int>(10);
@@ -13,6 +22,8 @@ int main() {
   std::unique_ptr<int> &up_int_ref = up_int;
   std::unique_ptr<int> &&up_int_ref_ref = std::make_unique<int>(10);
   std::unique_ptr<User> up_user = std::make_unique<User>();
+  auto up_non_empty_deleter =
+      std::unique_ptr<int, NonEmptyIntDeleter>(new int(1234));
 
   return 0; // break here
 }
diff --git a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
index 7c5ae442cc5b9..62ff7a5047d2d 100644
--- a/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
+++ b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
@@ -27,7 +27,6 @@ def test_step_over_with_python_api(self):
         self.inline_stepping_step_over()
 
     @add_test_categories(['pyapi'])
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_step_in_template_with_python_api(self):
         """Test stepping in to templated functions."""
         self.build()
diff --git a/lldb/test/API/functionalities/interactive_scripted_process/Makefile b/lldb/test/API/functionalities/interactive_scripted_process/Makefile
new file mode 100644
index 0000000000000..669f80a17ae62
--- /dev/null
+++ b/lldb/test/API/functionalities/interactive_scripted_process/Makefile
@@ -0,0 +1,7 @@
+CXX_SOURCES := main.cpp
+CXXFLAGS=--std=c++17 -g
+ARCH=$(shell uname -m)
+ENABLE_THREADS := YES
+
+include Makefile.rules
+
diff --git a/lldb/test/API/functionalities/interactive_scripted_process/TestInteractiveScriptedProcess.py b/lldb/test/API/functionalities/interactive_scripted_process/TestInteractiveScriptedProcess.py
new file mode 100644
index 0000000000000..807fcd76fd547
--- /dev/null
+++ b/lldb/test/API/functionalities/interactive_scripted_process/TestInteractiveScriptedProcess.py
@@ -0,0 +1,288 @@
+"""
+Test the functionality of interactive scripted processes
+"""
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+import json, os
+
+
+class TestInteractiveScriptedProcess(TestBase):
+
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Build and load test program
+        self.build()
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+        self.main_source_file = lldb.SBFileSpec("main.cpp")
+        self.script_module = "interactive_scripted_process"
+        self.script_file = self.script_module + ".py"
+
+    @skipUnlessDarwin
+    def test_passthrough_launch(self):
+        """Test a simple pass-through process launch"""
+        self.passthrough_launch()
+
+        lldbutil.run_break_set_by_source_regexp(self, "also break here")
+        self.assertEqual(self.mux_target.GetNumBreakpoints(), 2)
+        error = self.mux_process.Continue()
+        self.assertSuccess(error, "Resuming multiplexer scripted process")
+        self.assertTrue(self.mux_process.IsValid(), "Got a valid process")
+
+        event = lldbutil.fetch_next_event(
+            self, self.mux_process_listener, self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateRunning)
+        event = lldbutil.fetch_next_event(
+            self, self.mux_process_listener, self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateStopped)
+
+    @skipUnlessDarwin
+    def test_multiplexed_launch(self):
+        """Test a multiple interactive scripted process debugging"""
+        self.passthrough_launch()
+        self.assertEqual(self.dbg.GetNumTargets(), 2)
+
+        driving_target = self.mux_process.GetScriptedImplementation().driving_target
+        self.assertTrue(driving_target.IsValid(), "Driving target is invalid")
+
+        # Create a target for the multiplexed even scripted process
+        even_target = self.duplicate_target(driving_target)
+        self.assertTrue(
+            even_target.IsValid(),
+            "Couldn't duplicate driving target to launch multiplexed even scripted process",
+        )
+
+        class_name = f"{self.script_module}.MultiplexedScriptedProcess"
+        dictionary = {"driving_target_idx": self.dbg.GetIndexOfTarget(self.mux_target)}
+
+        dictionary["parity"] = 0
+        muxed_launch_info = self.get_launch_info(class_name, dictionary)
+
+        # Launch Even Child Scripted Process
+        error = lldb.SBError()
+        even_process = even_target.Launch(muxed_launch_info, error)
+        self.assertTrue(
+            even_process, "Couldn't launch multiplexed even scripted process"
+        )
+        self.multiplex(even_process)
+
+        # Check that the even process started running
+        event = lldbutil.fetch_next_event(
+            self, self.dbg.GetListener(), even_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateRunning)
+        # Check that the even process stopped
+        event = lldbutil.fetch_next_event(
+            self, self.dbg.GetListener(), even_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateStopped)
+
+        self.assertTrue(even_process.IsValid(), "Got a valid process")
+        self.assertState(
+            even_process.GetState(), lldb.eStateStopped, "Process is stopped"
+        )
+
+        # Create a target for the multiplexed odd scripted process
+        odd_target = self.duplicate_target(driving_target)
+        self.assertTrue(
+            odd_target.IsValid(),
+            "Couldn't duplicate driving target to launch multiplexed odd scripted process",
+        )
+
+        dictionary["parity"] = 1
+        muxed_launch_info = self.get_launch_info(class_name, dictionary)
+
+        # Launch Odd Child Scripted Process
+        error = lldb.SBError()
+        odd_process = odd_target.Launch(muxed_launch_info, error)
+        self.assertTrue(odd_process, "Couldn't launch multiplexed odd scripted process")
+        self.multiplex(odd_process)
+
+        # Check that the odd process started running
+        event = lldbutil.fetch_next_event(
+            self, self.dbg.GetListener(), odd_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateRunning)
+        # Check that the odd process stopped
+        event = lldbutil.fetch_next_event(
+            self, self.dbg.GetListener(), odd_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateStopped)
+
+        self.assertTrue(odd_process.IsValid(), "Got a valid process")
+        self.assertState(
+            odd_process.GetState(), lldb.eStateStopped, "Process is stopped"
+        )
+
+        # Set a breakpoint on the odd child process
+        bkpt = odd_target.BreakpointCreateBySourceRegex(
+            "also break here", self.main_source_file
+        )
+        self.assertEqual(odd_target.GetNumBreakpoints(), 1)
+        self.assertTrue(bkpt, "Second breakpoint set on child scripted process")
+        self.assertEqual(bkpt.GetNumLocations(), 1, "Second breakpoint has 1 location")
+
+        # Verify that the breakpoint was also set on the multiplexer & real target
+        self.assertEqual(self.mux_target.GetNumBreakpoints(), 2)
+        bkpt = self.mux_target.GetBreakpointAtIndex(1)
+        self.assertEqual(
+            bkpt.GetNumLocations(), 1, "Second breakpoint set on mux scripted process"
+        )
+        self.assertTrue(bkpt.MatchesName("multiplexed_scripted_process_421"))
+
+        self.assertGreater(driving_target.GetNumBreakpoints(), 1)
+
+        # Resume execution on child process
+        error = odd_process.Continue()
+        self.assertSuccess(error, "Resuming odd child scripted process")
+        self.assertTrue(odd_process.IsValid(), "Got a valid process")
+
+        # Since all the execution is asynchronous, the order in which events
+        # arrive is non-deterministic, so we need a data structure to make sure
+        # we received both the running and stopped event for each target.
+
+        # Initialize the execution event "bingo book", that maps a process index
+        # to a dictionary that contains flags that are not set for the process
+        # events that we care about (running & stopped)
+
+        execution_events = {
+            1: {lldb.eStateRunning: False, lldb.eStateStopped: False},
+            2: {lldb.eStateRunning: False, lldb.eStateStopped: False},
+            3: {lldb.eStateRunning: False, lldb.eStateStopped: False},
+        }
+
+        def fetch_process_event(self, execution_events):
+            event = lldbutil.fetch_next_event(
+                self,
+                self.dbg.GetListener(),
+                lldb.SBProcess.GetBroadcasterClass(),
+                match_class=True,
+            )
+            state = lldb.SBProcess.GetStateFromEvent(event)
+            self.assertIn(state, [lldb.eStateRunning, lldb.eStateStopped])
+            event_process = lldb.SBProcess.GetProcessFromEvent(event)
+            self.assertTrue(event_process.IsValid())
+            event_target = event_process.GetTarget()
+            event_target_idx = self.dbg.GetIndexOfTarget(event_target)
+            self.assertFalse(
+                execution_events[event_target_idx][state],
+                "Event already received for this process",
+            )
+            execution_events[event_target_idx][state] = True
+
+        event = lldbutil.fetch_next_event(
+            self, self.mux_process_listener, self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateRunning)
+
+        event = lldbutil.fetch_next_event(
+            self, self.mux_process_listener, self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateStopped)
+
+        for _ in range((self.dbg.GetNumTargets() - 1) * 2):
+            fetch_process_event(self, execution_events)
+
+        for target_index, event_states in execution_events.items():
+            for state, is_set in event_states.items():
+                self.assertTrue(is_set, f"Target {target_index} has state {state} set")
+
+    def duplicate_target(self, driving_target):
+        exe = driving_target.executable.fullpath
+        triple = driving_target.triple
+        return self.dbg.CreateTargetWithFileAndTargetTriple(exe, triple)
+
+    def get_launch_info(self, class_name, script_dict):
+        structured_data = lldb.SBStructuredData()
+        structured_data.SetFromJSON(json.dumps(script_dict))
+
+        launch_info = lldb.SBLaunchInfo(None)
+        launch_info.SetProcessPluginName("ScriptedProcess")
+        launch_info.SetScriptedProcessClassName(class_name)
+        launch_info.SetScriptedProcessDictionary(structured_data)
+        return launch_info
+
+    def multiplex(self, muxed_process):
+        muxed_process.GetScriptedImplementation().multiplexer = (
+            self.mux_process.GetScriptedImplementation()
+        )
+        self.mux_process.GetScriptedImplementation().multiplexed_processes[
+            muxed_process.GetProcessID()
+        ] = muxed_process
+
+    def passthrough_launch(self):
+        """Test that a simple passthrough wrapper functions correctly"""
+        # First build the real target:
+        self.assertEqual(self.dbg.GetNumTargets(), 1)
+        real_target_id = 0
+        real_target = self.dbg.GetTargetAtIndex(real_target_id)
+        lldbutil.run_break_set_by_source_regexp(self, "Break here")
+        self.assertEqual(real_target.GetNumBreakpoints(), 1)
+
+        # Now source in the scripted module:
+        script_path = os.path.join(self.getSourceDir(), self.script_file)
+        self.runCmd(f"command script import '{script_path}'")
+
+        self.mux_target = self.duplicate_target(real_target)
+        self.assertTrue(self.mux_target.IsValid(), "duplicate target succeeded")
+
+        mux_class = f"{self.script_module}.MultiplexerScriptedProcess"
+        script_dict = {"driving_target_idx": real_target_id}
+        mux_launch_info = self.get_launch_info(mux_class, script_dict)
+        self.mux_process_listener = lldb.SBListener(
+            "lldb.test.interactive-scripted-process.listener"
+        )
+        mux_launch_info.SetShadowListener(self.mux_process_listener)
+
+        self.dbg.SetAsync(True)
+        error = lldb.SBError()
+        self.mux_process = self.mux_target.Launch(mux_launch_info, error)
+        self.assertSuccess(error, "Launched multiplexer scripted process")
+        self.assertTrue(self.mux_process.IsValid(), "Got a valid process")
+
+        # Check that the mux process started running
+        event = lldbutil.fetch_next_event(
+            self, self.mux_process_listener, self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateRunning)
+        # Check that the real process started running
+        event = lldbutil.fetch_next_event(
+            self, self.dbg.GetListener(), self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateRunning)
+
+        # Check that the real process stopped
+        event = lldbutil.fetch_next_event(
+            self, self.dbg.GetListener(), self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateStopped)
+        # Check that the mux process stopped
+        event = lldbutil.fetch_next_event(
+            self, self.mux_process_listener, self.mux_process.GetBroadcaster()
+        )
+        self.assertState(lldb.SBProcess.GetStateFromEvent(event), lldb.eStateStopped)
+
+        real_process = real_target.GetProcess()
+        self.assertTrue(real_process.IsValid(), "Got a valid process")
+        self.assertState(
+            real_process.GetState(), lldb.eStateStopped, "Process is stopped"
+        )
+
+        # This is a passthrough, so the two processes should have the same state:
+        # Check that we got the right threads:
+        self.assertEqual(
+            len(real_process.threads),
+            len(self.mux_process.threads),
+            "Same number of threads",
+        )
+        for id in range(len(real_process.threads)):
+            real_pc = real_process.threads[id].frame[0].pc
+            mux_pc = self.mux_process.threads[id].frame[0].pc
+            self.assertEqual(real_pc, mux_pc, f"PC's equal for {id}")
diff --git a/lldb/test/API/functionalities/interactive_scripted_process/interactive_scripted_process.py b/lldb/test/API/functionalities/interactive_scripted_process/interactive_scripted_process.py
new file mode 100644
index 0000000000000..c979099066798
--- /dev/null
+++ b/lldb/test/API/functionalities/interactive_scripted_process/interactive_scripted_process.py
@@ -0,0 +1,574 @@
+# Usage:
+# ./bin/lldb $LLVM/lldb/test/API/functionalities/interactive_scripted_process/main \
+#   -o "br set -p 'Break here'" \
+#   -o "command script import $LLVM/lldb/test/API/functionalities/interactive_scripted_process/interactive_scripted_process.py" \
+#   -o "create_mux" \
+#   -o "create_sub" \
+#   -o "br set -p 'also break here'" -o 'continue'
+
+import os, json, struct, signal, tempfile
+
+from threading import Thread
+from typing import Any, Dict
+
+import lldb
+from lldb.plugins.scripted_process import ScriptedProcess
+from lldb.plugins.scripted_process import ScriptedThread
+
+
+class PassthruScriptedProcess(ScriptedProcess):
+    driving_target = None
+    driving_process = None
+
+    def __init__(
+        self,
+        exe_ctx: lldb.SBExecutionContext,
+        args: lldb.SBStructuredData,
+        launched_driving_process: bool = True,
+    ):
+        super().__init__(exe_ctx, args)
+
+        self.driving_target = None
+        self.driving_process = None
+
+        self.driving_target_idx = args.GetValueForKey("driving_target_idx")
+        if self.driving_target_idx and self.driving_target_idx.IsValid():
+            if self.driving_target_idx.GetType() == lldb.eStructuredDataTypeInteger:
+                idx = self.driving_target_idx.GetIntegerValue(42)
+            if self.driving_target_idx.GetType() == lldb.eStructuredDataTypeString:
+                idx = int(self.driving_target_idx.GetStringValue(100))
+            self.driving_target = self.target.GetDebugger().GetTargetAtIndex(idx)
+
+            if launched_driving_process:
+                self.driving_process = self.driving_target.GetProcess()
+                for driving_thread in self.driving_process:
+                    structured_data = lldb.SBStructuredData()
+                    structured_data.SetFromJSON(
+                        json.dumps(
+                            {
+                                "driving_target_idx": idx,
+                                "thread_idx": driving_thread.GetIndexID(),
+                            }
+                        )
+                    )
+
+                    self.threads[driving_thread.GetThreadID()] = PassthruScriptedThread(
+                        self, structured_data
+                    )
+
+                for module in self.driving_target.modules:
+                    path = module.file.fullpath
+                    load_addr = module.GetObjectFileHeaderAddress().GetLoadAddress(
+                        self.driving_target
+                    )
+                    self.loaded_images.append({"path": path, "load_addr": load_addr})
+
+    def get_memory_region_containing_address(
+        self, addr: int
+    ) -> lldb.SBMemoryRegionInfo:
+        mem_region = lldb.SBMemoryRegionInfo()
+        error = self.driving_process.GetMemoryRegionInfo(addr, mem_region)
+        if error.Fail():
+            return None
+        return mem_region
+
+    def read_memory_at_address(
+        self, addr: int, size: int, error: lldb.SBError
+    ) -> lldb.SBData:
+        data = lldb.SBData()
+        bytes_read = self.driving_process.ReadMemory(addr, size, error)
+
+        if error.Fail():
+            return data
+
+        data.SetDataWithOwnership(
+            error,
+            bytes_read,
+            self.driving_target.GetByteOrder(),
+            self.driving_target.GetAddressByteSize(),
+        )
+
+        return data
+
+    def write_memory_at_address(
+        self, addr: int, data: lldb.SBData, error: lldb.SBError
+    ) -> int:
+        return self.driving_process.WriteMemory(
+            addr, bytearray(data.uint8.all()), error
+        )
+
+    def get_process_id(self) -> int:
+        return 42
+
+    def is_alive(self) -> bool:
+        return True
+
+    def get_scripted_thread_plugin(self) -> str:
+        return f"{PassthruScriptedThread.__module__}.{PassthruScriptedThread.__name__}"
+
+
+class MultiplexedScriptedProcess(PassthruScriptedProcess):
+    def __init__(self, exe_ctx: lldb.SBExecutionContext, args: lldb.SBStructuredData):
+        super().__init__(exe_ctx, args)
+        self.multiplexer = None
+        if isinstance(self.driving_process, lldb.SBProcess) and self.driving_process:
+            parity = args.GetValueForKey("parity")
+            # TODO: Change to Walrus operator (:=) with oneline if assignment
+            # Requires python 3.8
+            val = extract_value_from_structured_data(parity, 0)
+            if val is not None:
+                self.parity = val
+
+            # Turn PassThruScriptedThread into MultiplexedScriptedThread
+            for thread in self.threads.values():
+                thread.__class__ = MultiplexedScriptedThread
+
+    def get_process_id(self) -> int:
+        return self.parity + 420
+
+    def launch(self, should_stop: bool = True) -> lldb.SBError:
+        self.first_launch = True
+        return lldb.SBError()
+
+    def resume(self, should_stop: bool) -> lldb.SBError:
+        if self.first_launch:
+            self.first_launch = False
+            return super().resume()
+        else:
+            if not self.multiplexer:
+                error = lldb.SBError("Multiplexer is not set.")
+                return error
+            return self.multiplexer.resume(should_stop)
+
+    def get_threads_info(self) -> Dict[int, Any]:
+        if not self.multiplexer:
+            return super().get_threads_info()
+        filtered_threads = self.multiplexer.get_threads_info(pid=self.get_process_id())
+        # Update the filtered thread class from PassthruScriptedThread to MultiplexedScriptedThread
+        return dict(
+            map(
+                lambda pair: (pair[0], MultiplexedScriptedThread(pair[1])),
+                filtered_threads.items(),
+            )
+        )
+
+    def create_breakpoint(self, addr, error, pid=None):
+        if not self.multiplexer:
+            error.SetErrorString("Multiplexer is not set.")
+        return self.multiplexer.create_breakpoint(addr, error, self.get_process_id())
+
+    def get_scripted_thread_plugin(self) -> str:
+        return f"{MultiplexedScriptedThread.__module__}.{MultiplexedScriptedThread.__name__}"
+
+
+class PassthruScriptedThread(ScriptedThread):
+    def __init__(self, process, args):
+        super().__init__(process, args)
+        driving_target_idx = args.GetValueForKey("driving_target_idx")
+        thread_idx = args.GetValueForKey("thread_idx")
+
+        # TODO: Change to Walrus operator (:=) with oneline if assignment
+        # Requires python 3.8
+        val = extract_value_from_structured_data(thread_idx, 0)
+        if val is not None:
+            self.idx = val
+
+        self.driving_target = None
+        self.driving_process = None
+        self.driving_thread = None
+
+        # TODO: Change to Walrus operator (:=) with oneline if assignment
+        # Requires python 3.8
+        val = extract_value_from_structured_data(driving_target_idx, 42)
+        if val is not None:
+            self.driving_target = self.target.GetDebugger().GetTargetAtIndex(val)
+            self.driving_process = self.driving_target.GetProcess()
+            self.driving_thread = self.driving_process.GetThreadByIndexID(self.idx)
+
+        if self.driving_thread:
+            self.id = self.driving_thread.GetThreadID()
+
+    def get_thread_id(self) -> int:
+        return self.id
+
+    def get_name(self) -> str:
+        return f"{PassthruScriptedThread.__name__}.thread-{self.idx}"
+
+    def get_stop_reason(self) -> Dict[str, Any]:
+        stop_reason = {"type": lldb.eStopReasonInvalid, "data": {}}
+
+        if (
+            self.driving_thread
+            and self.driving_thread.IsValid()
+            and self.get_thread_id() == self.driving_thread.GetThreadID()
+        ):
+            stop_reason["type"] = lldb.eStopReasonNone
+
+            if self.driving_thread.GetStopReason() != lldb.eStopReasonNone:
+                if "arm64" in self.scripted_process.arch:
+                    stop_reason["type"] = lldb.eStopReasonException
+                    stop_reason["data"][
+                        "desc"
+                    ] = self.driving_thread.GetStopDescription(100)
+                elif self.scripted_process.arch == "x86_64":
+                    stop_reason["type"] = lldb.eStopReasonSignal
+                    stop_reason["data"]["signal"] = signal.SIGTRAP
+                else:
+                    stop_reason["type"] = self.driving_thread.GetStopReason()
+
+        return stop_reason
+
+    def get_register_context(self) -> str:
+        if not self.driving_thread or self.driving_thread.GetNumFrames() == 0:
+            return None
+        frame = self.driving_thread.GetFrameAtIndex(0)
+
+        GPRs = None
+        registerSet = frame.registers  # Returns an SBValueList.
+        for regs in registerSet:
+            if "general purpose" in regs.name.lower():
+                GPRs = regs
+                break
+
+        if not GPRs:
+            return None
+
+        for reg in GPRs:
+            self.register_ctx[reg.name] = int(reg.value, base=16)
+
+        return struct.pack(f"{len(self.register_ctx)}Q", *self.register_ctx.values())
+
+
+class MultiplexedScriptedThread(PassthruScriptedThread):
+    def get_name(self) -> str:
+        parity = "Odd" if self.scripted_process.parity % 2 else "Even"
+        return f"{parity}{MultiplexedScriptedThread.__name__}.thread-{self.idx}"
+
+
+class MultiplexerScriptedProcess(PassthruScriptedProcess):
+    listener = None
+    multiplexed_processes = None
+
+    def wait_for_driving_process_to_stop(self):
+        def handle_process_state_event():
+            # Update multiplexer process
+            log("Updating interactive scripted process threads")
+            dbg = self.driving_target.GetDebugger()
+            log("Clearing interactive scripted process threads")
+            self.threads.clear()
+            for driving_thread in self.driving_process:
+                log(f"{len(self.threads)} New thread {hex(driving_thread.id)}")
+                structured_data = lldb.SBStructuredData()
+                structured_data.SetFromJSON(
+                    json.dumps(
+                        {
+                            "driving_target_idx": dbg.GetIndexOfTarget(
+                                self.driving_target
+                            ),
+                            "thread_idx": driving_thread.GetIndexID(),
+                        }
+                    )
+                )
+
+                self.threads[driving_thread.GetThreadID()] = PassthruScriptedThread(
+                    self, structured_data
+                )
+
+            mux_process = self.target.GetProcess()
+            mux_process.ForceScriptedState(lldb.eStateRunning)
+            mux_process.ForceScriptedState(lldb.eStateStopped)
+
+            for child_process in self.multiplexed_processes.values():
+                child_process.ForceScriptedState(lldb.eStateRunning)
+                child_process.ForceScriptedState(lldb.eStateStopped)
+
+        event = lldb.SBEvent()
+        while True:
+            if self.listener.WaitForEvent(1, event):
+                event_mask = event.GetType()
+                if event_mask & lldb.SBProcess.eBroadcastBitStateChanged:
+                    state = lldb.SBProcess.GetStateFromEvent(event)
+                    log(f"Received public process state event: {state}")
+                    if state == lldb.eStateStopped:
+                        # If it's a stop event, iterate over the driving process
+                        # thread, looking for a breakpoint stop reason, if internal
+                        # continue.
+                        handle_process_state_event()
+            else:
+                continue
+
+    def __init__(self, exe_ctx: lldb.SBExecutionContext, args: lldb.SBStructuredData):
+        super().__init__(exe_ctx, args, launched_driving_process=False)
+        if isinstance(self.driving_target, lldb.SBTarget) and self.driving_target:
+            self.listener = lldb.SBListener(
+                "lldb.listener.multiplexer-scripted-process"
+            )
+            self.multiplexed_processes = {}
+
+            # Copy breakpoints from real target to passthrough
+            with tempfile.NamedTemporaryFile() as tf:
+                bkpt_file = lldb.SBFileSpec(tf.name)
+                error = self.driving_target.BreakpointsWriteToFile(bkpt_file)
+                if error.Fail():
+                    log(
+                        "Failed to save breakpoints from driving target (%s)"
+                        % error.GetCString()
+                    )
+                bkpts_list = lldb.SBBreakpointList(self.target)
+                error = self.target.BreakpointsCreateFromFile(bkpt_file, bkpts_list)
+                if error.Fail():
+                    log(
+                        "Failed create breakpoints from driving target \
+                        (bkpt file: %s)"
+                        % tf.name
+                    )
+
+            # Copy breakpoint from passthrough to real target
+            if error.Success():
+                self.driving_target.DeleteAllBreakpoints()
+                for bkpt in self.target.breakpoints:
+                    if bkpt.IsValid():
+                        for bl in bkpt:
+                            real_bpkt = self.driving_target.BreakpointCreateBySBAddress(
+                                bl.GetAddress()
+                            )
+                            if not real_bpkt.IsValid():
+                                log(
+                                    "Failed to set breakpoint at address %s in \
+                                    driving target"
+                                    % hex(bl.GetLoadAddress())
+                                )
+
+            self.listener_thread = Thread(
+                target=self.wait_for_driving_process_to_stop, daemon=True
+            )
+            self.listener_thread.start()
+
+    def launch(self, should_stop: bool = True) -> lldb.SBError:
+        if not self.driving_target:
+            return lldb.SBError(
+                f"{self.__class__.__name__}.resume: Invalid driving target."
+            )
+
+        if self.driving_process:
+            return lldb.SBError(
+                f"{self.__class__.__name__}.resume: Invalid driving process."
+            )
+
+        error = lldb.SBError()
+        launch_info = lldb.SBLaunchInfo(None)
+        launch_info.SetListener(self.listener)
+        driving_process = self.driving_target.Launch(launch_info, error)
+
+        if not driving_process or error.Fail():
+            return error
+
+        self.driving_process = driving_process
+
+        for module in self.driving_target.modules:
+            path = module.file.fullpath
+            load_addr = module.GetObjectFileHeaderAddress().GetLoadAddress(
+                self.driving_target
+            )
+            self.loaded_images.append({"path": path, "load_addr": load_addr})
+
+        self.first_resume = True
+        return error
+
+    def resume(self, should_stop: bool = True) -> lldb.SBError:
+        if self.first_resume:
+            # When we resume the multiplexer process for the first time,
+            # we shouldn't do anything because lldb's execution machinery
+            # will resume the driving process by itself.
+
+            # Also, no need to update the multiplexer scripted process state
+            # here because since it's listening for the real process stop events.
+            # Once it receives the stop event from the driving process,
+            # `wait_for_driving_process_to_stop` will update the multiplexer
+            # state for us.
+
+            self.first_resume = False
+            return lldb.SBError()
+
+        if not self.driving_process:
+            return lldb.SBError(
+                f"{self.__class__.__name__}.resume: Invalid driving process."
+            )
+
+        return self.driving_process.Continue()
+
+    def get_threads_info(self, pid: int = None) -> Dict[int, Any]:
+        if not pid:
+            return super().get_threads_info()
+        parity = pid % 2
+        return dict(filter(lambda pair: pair[0] % 2 == parity, self.threads.items()))
+
+    def create_breakpoint(self, addr, error, pid=None):
+        if not self.driving_target:
+            error.SetErrorString("%s has no driving target." % self.__class__.__name__)
+            return False
+
+        def create_breakpoint_with_name(target, load_addr, name, error):
+            addr = lldb.SBAddress(load_addr, target)
+            if not addr.IsValid():
+                error.SetErrorString("Invalid breakpoint address %s" % hex(load_addr))
+                return False
+            bkpt = target.BreakpointCreateBySBAddress(addr)
+            if not bkpt.IsValid():
+                error.SetErrorString(
+                    "Failed to create breakpoint at address %s"
+                    % hex(addr.GetLoadAddress())
+                )
+                return False
+            error = bkpt.AddNameWithErrorHandling(name)
+            return error.Success()
+
+        name = (
+            "multiplexer_scripted_process"
+            if not pid
+            else f"multiplexed_scripted_process_{pid}"
+        )
+
+        if pid is not None:
+            # This means that this method has been called from one of the
+            # multiplexed scripted process. That also means that the multiplexer
+            # target doesn't have this breakpoint created.
+            mux_error = lldb.SBError()
+            bkpt = create_breakpoint_with_name(self.target, addr, name, mux_error)
+            if mux_error.Fail():
+                error.SetError(
+                    "Failed to create breakpoint in multiplexer \
+                               target: %s"
+                    % mux_error.GetCString()
+                )
+                return False
+        return create_breakpoint_with_name(self.driving_target, addr, name, error)
+
+
+def multiplex(mux_process, muxed_process):
+    muxed_process.GetScriptedImplementation().multiplexer = (
+        mux_process.GetScriptedImplementation()
+    )
+    mux_process.GetScriptedImplementation().multiplexed_processes[
+        muxed_process.GetProcessID()
+    ] = muxed_process
+
+
+def launch_scripted_process(target, class_name, dictionary):
+    structured_data = lldb.SBStructuredData()
+    structured_data.SetFromJSON(json.dumps(dictionary))
+
+    launch_info = lldb.SBLaunchInfo(None)
+    launch_info.SetProcessPluginName("ScriptedProcess")
+    launch_info.SetScriptedProcessClassName(class_name)
+    launch_info.SetScriptedProcessDictionary(structured_data)
+
+    error = lldb.SBError()
+    return target.Launch(launch_info, error)
+
+
+def duplicate_target(driving_target):
+    error = lldb.SBError()
+    exe = driving_target.executable.fullpath
+    triple = driving_target.triple
+    debugger = driving_target.GetDebugger()
+    return debugger.CreateTargetWithFileAndTargetTriple(exe, triple)
+
+
+def extract_value_from_structured_data(data, default_val):
+    if data and data.IsValid():
+        if data.GetType() == lldb.eStructuredDataTypeInteger:
+            return data.GetIntegerValue(default_val)
+        if data.GetType() == lldb.eStructuredDataTypeString:
+            return int(data.GetStringValue(100))
+    return default_val
+
+
+def create_mux_process(debugger, command, exe_ctx, result, dict):
+    if not debugger.GetNumTargets() > 0:
+        return result.SetError(
+            "Interactive scripted processes requires one non scripted process."
+        )
+
+    debugger.SetAsync(True)
+
+    driving_target = debugger.GetSelectedTarget()
+    if not driving_target:
+        return result.SetError("Driving target is invalid")
+
+    # Create a seconde target for the multiplexer scripted process
+    mux_target = duplicate_target(driving_target)
+    if not mux_target:
+        return result.SetError(
+            "Couldn't duplicate driving target to launch multiplexer scripted process"
+        )
+
+    class_name = f"{__name__}.{MultiplexerScriptedProcess.__name__}"
+    dictionary = {"driving_target_idx": debugger.GetIndexOfTarget(driving_target)}
+    mux_process = launch_scripted_process(mux_target, class_name, dictionary)
+    if not mux_process:
+        return result.SetError("Couldn't launch multiplexer scripted process")
+
+
+def create_child_processes(debugger, command, exe_ctx, result, dict):
+    if not debugger.GetNumTargets() >= 2:
+        return result.SetError("Scripted Multiplexer process not setup")
+
+    debugger.SetAsync(True)
+
+    # Create a seconde target for the multiplexer scripted process
+    mux_target = debugger.GetSelectedTarget()
+    if not mux_target:
+        return result.SetError("Couldn't get multiplexer scripted process target")
+    mux_process = mux_target.GetProcess()
+    if not mux_process:
+        return result.SetError("Couldn't get multiplexer scripted process")
+
+    driving_target = mux_process.GetScriptedImplementation().driving_target
+    if not driving_target:
+        return result.SetError("Driving target is invalid")
+
+    # Create a target for the multiplexed even scripted process
+    even_target = duplicate_target(driving_target)
+    if not even_target:
+        return result.SetError(
+            "Couldn't duplicate driving target to launch multiplexed even scripted process"
+        )
+
+    class_name = f"{__name__}.{MultiplexedScriptedProcess.__name__}"
+    dictionary = {"driving_target_idx": debugger.GetIndexOfTarget(mux_target)}
+    dictionary["parity"] = 0
+    even_process = launch_scripted_process(even_target, class_name, dictionary)
+    if not even_process:
+        return result.SetError("Couldn't launch multiplexed even scripted process")
+    multiplex(mux_process, even_process)
+
+    # Create a target for the multiplexed odd scripted process
+    odd_target = duplicate_target(driving_target)
+    if not odd_target:
+        return result.SetError(
+            "Couldn't duplicate driving target to launch multiplexed odd scripted process"
+        )
+
+    dictionary["parity"] = 1
+    odd_process = launch_scripted_process(odd_target, class_name, dictionary)
+    if not odd_process:
+        return result.SetError("Couldn't launch multiplexed odd scripted process")
+    multiplex(mux_process, odd_process)
+
+
+def log(message):
+    # FIXME: For now, we discard the log message until we can pass it to an lldb
+    # logging channel.
+    should_log = False
+    if should_log:
+        print(message)
+
+
+def __lldb_init_module(dbg, dict):
+    dbg.HandleCommand(
+        "command script add -o -f interactive_scripted_process.create_mux_process create_mux"
+    )
+    dbg.HandleCommand(
+        "command script add -o -f interactive_scripted_process.create_child_processes create_sub"
+    )
diff --git a/lldb/test/API/functionalities/interactive_scripted_process/main.cpp b/lldb/test/API/functionalities/interactive_scripted_process/main.cpp
new file mode 100644
index 0000000000000..523474b70ef0a
--- /dev/null
+++ b/lldb/test/API/functionalities/interactive_scripted_process/main.cpp
@@ -0,0 +1,35 @@
+#include <iostream>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+void spawn_thread(int index) {
+  std::string name = "I'm thread " + std::to_string(index) + " !";
+  bool done = false;
+  std::string state = "Started execution!";
+  while (true) {
+    if (done) // also break here
+      break;
+  }
+
+  state = "Stopped execution!";
+}
+
+int main() {
+  constexpr size_t num_threads = 10;
+  std::vector<std::thread> threads;
+
+  for (size_t i = 0; i < num_threads; i++) {
+    threads.push_back(std::thread(spawn_thread, i));
+  }
+
+  std::cout << "Spawned " << threads.size() << " threads!" << std::endl; // Break here
+
+  for (auto &t : threads) {
+    if (t.joinable())
+      t.join();
+  }
+
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py b/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
index 17ac2e067e299..fbaf0dd7ad108 100644
--- a/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
+++ b/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
@@ -14,7 +14,6 @@
 class StepAvoidsNoDebugTestCase(TestBase):
 
     @add_test_categories(['pyapi'])
-    @expectedFailureAll(archs=["aarch64"], oslist=["windows"], bugnumber="llvm.org/pr56292")
     def test_step_out_with_python(self):
         """Test stepping out using avoid-no-debug with dsyms."""
         self.build()
@@ -32,7 +31,6 @@ def test_step_out_with_python(self):
         archs=["i386"],
         oslist=no_match(["freebsd"]),
         bugnumber="llvm.org/pr28549")
-    @expectedFailureAll(archs=["aarch64"], oslist=["windows"], bugnumber="llvm.org/pr56292")
     def test_step_over_with_python(self):
         """Test stepping over using avoid-no-debug with dwarf."""
         self.build()
@@ -50,7 +48,6 @@ def test_step_over_with_python(self):
         archs=["i386"],
         oslist=no_match(["freebsd"]),
         bugnumber="llvm.org/pr28549")
-    @expectedFailureAll(archs=["aarch64"], oslist=["windows"], bugnumber="llvm.org/pr56292")
     def test_step_in_with_python(self):
         """Test stepping in using avoid-no-debug with dwarf."""
         self.build()
diff --git a/lldb/test/API/functionalities/step_scripted/Steps.py b/lldb/test/API/functionalities/step_scripted/Steps.py
index d41d01d43095e..6b49de772f8a0 100644
--- a/lldb/test/API/functionalities/step_scripted/Steps.py
+++ b/lldb/test/API/functionalities/step_scripted/Steps.py
@@ -19,6 +19,11 @@ def should_stop(self, event):
     def should_step(self):
         return False
 
+    def stop_description(self, stream):
+        if self.child_thread_plan.IsPlanComplete():
+            return self.child_thread_plan.GetDescription(stream)
+        return True
+
     def queue_child_thread_plan(self):
         return None
 
@@ -39,19 +44,18 @@ def queue_child_thread_plan(self):
 # This plan does a step-over until a variable changes value.
 class StepUntil(StepWithChild):
     def __init__(self, thread_plan, args_data, dict):
+        self.thread_plan = thread_plan
         self.frame = thread_plan.GetThread().frames[0]
         self.target = thread_plan.GetThread().GetProcess().GetTarget()
-        func_entry = args_data.GetValueForKey("variable_name")
+        var_entry = args_data.GetValueForKey("variable_name")
         
-        if not func_entry.IsValid():
+        if not var_entry.IsValid():
             print("Did not get a valid entry for variable_name")
-        func_name = func_entry.GetStringValue(100)
+        self.var_name = var_entry.GetStringValue(100)
 
-        self.value = self.frame.FindVariable(func_name)
+        self.value = self.frame.FindVariable(self.var_name)
         if self.value.GetError().Fail():
             print("Failed to get foo value: %s"%(self.value.GetError().GetCString()))
-        else:
-            print("'foo' value: %d"%(self.value.GetValueAsUnsigned()))
 
         StepWithChild.__init__(self, thread_plan)
 
@@ -70,17 +74,23 @@ def should_stop(self, event):
 
         # If we've stepped out of this frame, stop.
         if not self.frame.IsValid():
+            self.thread_plan.SetPlanComplete(True)
             return True
 
         if not self.value.IsValid():
+            self.thread_plan.SetPlanComplete(True)
             return True
 
         if not self.value.GetValueDidChange():
             self.child_thread_plan = self.queue_child_thread_plan()
             return False
         else:
+            self.thread_plan.SetPlanComplete(True)
             return True
 
+    def stop_description(self, stream):
+        stream.Print(f"Stepped until {self.var_name} changed.")
+        
 # This plan does nothing, but sets stop_mode to the
 # value of GetStopOthers for this plan.
 class StepReportsStopOthers():
@@ -92,7 +102,6 @@ def __init__(self, thread_plan, args_data, dict):
         
     def should_stop(self, event):
         self.thread_plan.SetPlanComplete(True)
-        print("Called in should_stop")
         StepReportsStopOthers.stop_mode_dict[self.key] = self.thread_plan.GetStopOthers()
         return True
 
diff --git a/lldb/test/API/functionalities/step_scripted/TestStepScripted.py b/lldb/test/API/functionalities/step_scripted/TestStepScripted.py
index b02c5ebfd90ca..50482782a3665 100644
--- a/lldb/test/API/functionalities/step_scripted/TestStepScripted.py
+++ b/lldb/test/API/functionalities/step_scripted/TestStepScripted.py
@@ -41,7 +41,8 @@ def step_out_with_scripted_plan(self, name):
 
         frame = thread.GetFrameAtIndex(0)
         self.assertEqual("main", frame.GetFunctionName())
-
+        stop_desc = thread.GetStopDescription(1000)
+        self.assertIn("Stepping out from", stop_desc, "Got right description")
 
     def test_misspelled_plan_name(self):
         """Test that we get a useful error if we misspell the plan class name"""
@@ -106,6 +107,10 @@ def do_test_checking_variable(self, use_cli):
         # And foo should have changed:
         self.assertTrue(foo_val.GetValueDidChange(), "Foo changed")
 
+        # And we should have a reasonable stop description:
+        desc = thread.GetStopDescription(1000)
+        self.assertIn("Stepped until foo changed", desc, "Got right stop description")
+
     def test_stop_others_from_command(self):
         """Test that the stop-others flag is set correctly by the command line.
            Also test that the run-all-threads property overrides this."""
@@ -119,10 +124,12 @@ def run_step(self, stop_others_value, run_mode, token):
         cmd = "thread step-scripted -C Steps.StepReportsStopOthers -k token -v %s"%(token)
         if run_mode != None:
             cmd = cmd + " --run-mode %s"%(run_mode)
-        print(cmd)
+        if self.TraceOn():
+            print(cmd)
         interp.HandleCommand(cmd, result)
         self.assertTrue(result.Succeeded(), "Step scripted failed: %s."%(result.GetError()))
-        print(Steps.StepReportsStopOthers.stop_mode_dict)
+        if self.TraceOn():
+            print(Steps.StepReportsStopOthers.stop_mode_dict)
         value = Steps.StepReportsStopOthers.stop_mode_dict[token]
         self.assertEqual(value, stop_others_value, "Stop others has the correct value.")
 
diff --git a/lldb/test/API/functionalities/watchpoint/large-watchpoint/Makefile b/lldb/test/API/functionalities/watchpoint/large-watchpoint/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/functionalities/watchpoint/large-watchpoint/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py b/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py
new file mode 100644
index 0000000000000..e8d88024547bf
--- /dev/null
+++ b/lldb/test/API/functionalities/watchpoint/large-watchpoint/TestLargeWatchpoint.py
@@ -0,0 +1,59 @@
+"""
+Watch larger-than-8-bytes regions of memory, confirm that
+writes to those regions are detected.
+"""
+
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class UnalignedWatchpointTestCase(TestBase):
+
+    def continue_and_report_stop_reason(self, process, iter_str):
+        process.Continue()
+        self.assertIn(process.GetState(), [lldb.eStateStopped, lldb.eStateExited],
+                         iter_str)
+        thread = process.GetSelectedThread()
+        return thread.GetStopReason()
+
+
+    NO_DEBUG_INFO_TESTCASE = True
+    # debugserver on AArch64 has this feature.
+    @skipIf(archs=no_match(['arm64', 'arm64e', 'aarch64']))
+    @skipUnlessDarwin
+
+    # debugserver only gained the ability to watch larger regions
+    # with this patch.
+    @skipIfOutOfTreeDebugserver
+
+    def test_large_watchpoint(self):
+        """Test watchpoint that covers a large region of memory."""
+        self.build()
+        self.main_source_file = lldb.SBFileSpec("main.c")
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+             "break here", self.main_source_file)
+
+        frame = thread.GetFrameAtIndex(0)
+
+        array_addr = frame.GetValueForVariablePath("array").GetValueAsUnsigned()
+
+        # watch 256 uint32_t elements in the middle of the array, 
+        # don't assume that the heap allocated array is aligned 
+        # to a 1024 byte boundary to begin with, force alignment.
+        wa_256_addr = ((array_addr + 1024) & ~(1024-1))
+        err = lldb.SBError()
+        wp = target.WatchAddress(wa_256_addr, 1024, False, True, err)
+        self.assertTrue(wp.IsValid())
+        self.assertSuccess(err)
+
+        c_count = 0
+        reason = self.continue_and_report_stop_reason(process, "continue #%d" % c_count)
+        while reason == lldb.eStopReasonWatchpoint:
+            c_count = c_count + 1
+            reason = self.continue_and_report_stop_reason(process, "continue #%d" % c_count)
+            self.assertLessEqual(c_count, 16)
+
+        self.assertEqual(c_count, 16)
diff --git a/lldb/test/API/functionalities/watchpoint/large-watchpoint/main.c b/lldb/test/API/functionalities/watchpoint/large-watchpoint/main.c
new file mode 100644
index 0000000000000..0b0d3f2c80477
--- /dev/null
+++ b/lldb/test/API/functionalities/watchpoint/large-watchpoint/main.c
@@ -0,0 +1,16 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+int main() {
+  const int count = 65535;
+  int *array = (int*) malloc(sizeof (int) * count);
+  memset (array, 0, count * sizeof (int));
+
+  puts ("break here");
+
+  for (int i = 0; i < count - 16; i += 16) 
+    array[i] += 10;
+
+  puts ("done, exiting.");
+}
diff --git a/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/Makefile b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py
new file mode 100644
index 0000000000000..62ed1280d51d1
--- /dev/null
+++ b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/TestUnalignedSpanningDwords.py
@@ -0,0 +1,61 @@
+"""
+Watch 4 bytes which spawn two doubleword aligned regions.
+On a target that supports 8 byte watchpoints, this will
+need to be implemented with a hardware watchpoint on both
+doublewords.
+"""
+
+
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class UnalignedWatchpointTestCase(TestBase):
+
+    def hit_watchpoint_and_continue(self, process, iter_str):
+        process.Continue()
+        self.assertEqual(process.GetState(), lldb.eStateStopped,
+                         iter_str)
+        thread = process.GetSelectedThread()
+        self.assertEqual(thread.GetStopReason(), lldb.eStopReasonWatchpoint, iter_str)
+        self.assertEqual(thread.GetStopReasonDataCount(), 1, iter_str)
+        wp_num = thread.GetStopReasonDataAtIndex(0)
+        self.assertEqual(wp_num, 1, iter_str)
+
+    NO_DEBUG_INFO_TESTCASE = True
+    # debugserver on AArch64 has this feature.
+    @skipIf(archs=no_match(['x86_64', 'arm64', 'arm64e', 'aarch64']))
+    @skipUnlessDarwin
+    # debugserver only started returning an exception address within
+    # a range lldb expects in https://reviews.llvm.org/D147820 2023-04-12.
+    # older debugservers will return the base address of the doubleword
+    # which lldb doesn't understand, and will stop executing without a
+    # proper stop reason.
+    @skipIfOutOfTreeDebugserver
+
+    def test_unaligned_watchpoint(self):
+        """Test a watchpoint that is handled by two hardware watchpoint registers."""
+        self.build()
+        self.main_source_file = lldb.SBFileSpec("main.c")
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+             "break here", self.main_source_file)
+
+        frame = thread.GetFrameAtIndex(0)
+
+        a_bytebuf_6 = frame.GetValueForVariablePath("a.bytebuf[6]")
+        a_bytebuf_6_addr = a_bytebuf_6.GetAddress().GetLoadAddress(target)
+        err = lldb.SBError()
+        wp = target.WatchAddress(a_bytebuf_6_addr, 4, False, True, err)
+        self.assertTrue(err.Success())
+        self.assertTrue(wp.IsEnabled())
+        self.assertEqual(wp.GetWatchSize(), 4)
+        self.assertGreater(wp.GetWatchAddress() % 8, 4, "watched region spans two doublewords")
+
+        # We will hit our watchpoint 6 times during the execution
+        # of the inferior.  If the remote stub does not actually split
+        # the watched region into two doubleword watchpoints, we will 
+        # exit before we get to 6 watchpoint hits.
+        for i in range(1, 7):
+            self.hit_watchpoint_and_continue(process, "wp hit number %s" % i)
diff --git a/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/main.c b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/main.c
new file mode 100644
index 0000000000000..09c108343e743
--- /dev/null
+++ b/lldb/test/API/functionalities/watchpoint/unaligned-spanning-two-dwords/main.c
@@ -0,0 +1,17 @@
+#include <stdint.h>
+#include <stdio.h>
+int main() {
+  union {
+    uint8_t bytebuf[16];
+    uint16_t shortbuf[8];
+    uint64_t dwordbuf[2];
+  } a;
+  a.dwordbuf[0] = a.dwordbuf[1] = 0;
+  a.bytebuf[0] = 0; // break here
+  for (int i = 0; i < 8; i++) {
+    a.shortbuf[i] += i;
+  }
+  for (int i = 0; i < 8; i++) {
+    a.shortbuf[i] += i;
+  }
+}
diff --git a/lldb/test/API/lang/c/step-target/TestStepTarget.py b/lldb/test/API/lang/c/step-target/TestStepTarget.py
index 3521870ddbdca..98f2ac0989d1a 100644
--- a/lldb/test/API/lang/c/step-target/TestStepTarget.py
+++ b/lldb/test/API/lang/c/step-target/TestStepTarget.py
@@ -47,7 +47,6 @@ def get_to_start(self):
         thread = threads[0]
         return thread
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_with_end_line(self):
         """Test stepping over vrs. hitting breakpoints & subsequent stepping in various forms."""
 
@@ -59,7 +58,6 @@ def test_with_end_line(self):
 
         self.assertEqual(frame.name, "lotsOfArgs", "Stepped to lotsOfArgs.")
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_with_end_line_bad_name(self):
         """Test stepping over vrs. hitting breakpoints & subsequent stepping in various forms."""
 
@@ -81,7 +79,6 @@ def test_with_end_line_deeper(self):
         frame = thread.frames[0]
         self.assertEqual(frame.name, "modifyInt", "Stepped to modifyInt.")
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_with_command_and_block(self):
         """Test stepping over vrs. hitting breakpoints & subsequent stepping in various forms."""
 
@@ -97,7 +94,6 @@ def test_with_command_and_block(self):
         frame = thread.frames[0]
         self.assertEqual(frame.name, "lotsOfArgs", "Stepped to lotsOfArgs.")
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_with_command_and_block_and_bad_name(self):
         """Test stepping over vrs. hitting breakpoints & subsequent stepping in various forms."""
 
diff --git a/lldb/test/API/lang/cpp/constructors/TestCppConstructors.py b/lldb/test/API/lang/cpp/constructors/TestCppConstructors.py
index d726108ab7411..9815705b330b4 100644
--- a/lldb/test/API/lang/cpp/constructors/TestCppConstructors.py
+++ b/lldb/test/API/lang/cpp/constructors/TestCppConstructors.py
@@ -6,6 +6,7 @@
 class TestCase(TestBase):
 
     @expectedFailureAll(bugnumber="llvm.org/pr50814", compiler="gcc")
+    @expectedFailureAll(oslist=["windows"])
     def test_constructors(self):
         self.build()
         lldbutil.run_to_source_breakpoint(self,"// break here", lldb.SBFileSpec("main.cpp"))
diff --git a/lldb/test/API/lang/cpp/global_variables/TestCPPGlobalVariables.py b/lldb/test/API/lang/cpp/global_variables/TestCPPGlobalVariables.py
index 28e53a0da2842..6df030086d38b 100644
--- a/lldb/test/API/lang/cpp/global_variables/TestCPPGlobalVariables.py
+++ b/lldb/test/API/lang/cpp/global_variables/TestCPPGlobalVariables.py
@@ -44,7 +44,6 @@ def test(self):
         self.assertEqual(var.GetName(), "abc::(anonymous namespace)::g_anon_namespace_const_int")
         self.assertEqual(var.GetValue(), "100")
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24764")
     def test_access_by_mangled_name(self):
         self.build()
 
diff --git a/lldb/test/API/lang/cpp/namespace/TestNamespace.py b/lldb/test/API/lang/cpp/namespace/TestNamespace.py
index ad534cd58ac08..9914ee347bbc6 100644
--- a/lldb/test/API/lang/cpp/namespace/TestNamespace.py
+++ b/lldb/test/API/lang/cpp/namespace/TestNamespace.py
@@ -43,6 +43,7 @@ def test_breakpoints_func_auto(self):
                 "make sure breakpoint locations are correct for 'func' with eFunctionNameTypeAuto")
 
     @expectedFailureAll(bugnumber="llvm.org/pr28548", compiler="gcc")
+    @expectedFailureAll(oslist=["windows"])
     def test_breakpoints_func_full(self):
         """Test that we can set breakpoints correctly by fullname to find all functions whose fully qualified name is "func"
            (no namespaces)."""
diff --git a/lldb/test/API/lang/cpp/standards/cpp20/Makefile b/lldb/test/API/lang/cpp/standards/cpp20/Makefile
index 4f79c0a900c3a..ec2f8d171f49a 100644
--- a/lldb/test/API/lang/cpp/standards/cpp20/Makefile
+++ b/lldb/test/API/lang/cpp/standards/cpp20/Makefile
@@ -1,3 +1,4 @@
+USE_LIBCPP := 1
 CXX_SOURCES := main.cpp
 CXXFLAGS_EXTRAS := -std=c++20
 
diff --git a/lldb/test/API/lang/cpp/standards/cpp20/TestCPP20Standard.py b/lldb/test/API/lang/cpp/standards/cpp20/TestCPP20Standard.py
index dbca9092e3abf..4c305f32b08bb 100644
--- a/lldb/test/API/lang/cpp/standards/cpp20/TestCPP20Standard.py
+++ b/lldb/test/API/lang/cpp/standards/cpp20/TestCPP20Standard.py
@@ -4,6 +4,8 @@
 from lldbsuite.test import lldbutil
 
 class TestCPP20Standard(TestBase):
+    @add_test_categories(["libc++"])
+    @skipIf(compiler="clang", compiler_version=['<', '11.0'])
     def test_cpp20(self):
         """
         Tests that we can evaluate an expression in C++20 mode
diff --git a/lldb/test/API/lang/cpp/static_members/TestCPPStaticMembers.py b/lldb/test/API/lang/cpp/static_members/TestCPPStaticMembers.py
index ce5f76e5fe744..98f44ba4b992f 100644
--- a/lldb/test/API/lang/cpp/static_members/TestCPPStaticMembers.py
+++ b/lldb/test/API/lang/cpp/static_members/TestCPPStaticMembers.py
@@ -39,8 +39,6 @@ def test_access_without_scope(self):
         self.expect("expression s_c", error=True,
                     startstr="error: use of undeclared identifier 's_d'")
 
-    # We fail to lookup static members on Windows.
-    @expectedFailureAll(oslist=["windows"])
     def test_no_crash_in_IR_arithmetic(self):
         """
         Test that LLDB doesn't crash on evaluating specific expression involving
diff --git a/lldb/test/API/lang/cpp/this_class_type_mixing/TestThisClassTypeMixing.py b/lldb/test/API/lang/cpp/this_class_type_mixing/TestThisClassTypeMixing.py
index 29c3c5d9fdd17..15ce278a5eea6 100644
--- a/lldb/test/API/lang/cpp/this_class_type_mixing/TestThisClassTypeMixing.py
+++ b/lldb/test/API/lang/cpp/this_class_type_mixing/TestThisClassTypeMixing.py
@@ -16,6 +16,7 @@ class by injecting a "$__lldb_expr" member function into the class. This
 class TestCase(TestBase):
 
     @no_debug_info_test
+    @expectedFailureAll(oslist=["windows"])
     def test(self):
         self.build()
         lldbutil.run_to_source_breakpoint(self, "// break in member function",
diff --git a/lldb/test/API/macosx/arm-corefile-regctx/Makefile b/lldb/test/API/macosx/arm-corefile-regctx/Makefile
new file mode 100644
index 0000000000000..e1d0354441cd4
--- /dev/null
+++ b/lldb/test/API/macosx/arm-corefile-regctx/Makefile
@@ -0,0 +1,6 @@
+MAKE_DSYM := NO
+
+CXX_SOURCES := create-arm-corefiles.cpp
+
+include Makefile.rules
+
diff --git a/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py b/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py
new file mode 100644
index 0000000000000..9e84291816084
--- /dev/null
+++ b/lldb/test/API/macosx/arm-corefile-regctx/TestArmMachoCorefileRegctx.py
@@ -0,0 +1,69 @@
+"""Test that Mach-O armv7/arm64 corefile register contexts are read by lldb."""
+
+
+
+import os
+import re
+import subprocess
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestArmMachoCorefileRegctx(TestBase):
+
+    NO_DEBUG_INFO_TESTCASE = True
+    @skipUnlessDarwin
+
+    def setUp(self):
+        TestBase.setUp(self)
+        self.build()
+        self.create_corefile = self.getBuildArtifact("a.out")
+        self.corefile = self.getBuildArtifact("core")
+
+    def test_armv7_corefile(self):
+        ### Create corefile
+        retcode = call(self.create_corefile + " armv7 " + self.corefile, shell=True)
+
+        target = self.dbg.CreateTarget('')
+        err = lldb.SBError()
+        process = target.LoadCore(self.corefile)
+        self.assertEqual(process.IsValid(), True)
+        thread = process.GetSelectedThread()
+        frame = thread.GetSelectedFrame()
+
+        lr = frame.FindRegister("lr")
+        self.assertTrue(lr.IsValid())
+        self.assertEqual(lr.GetValueAsUnsigned(), 0x000f0000)
+
+        pc = frame.FindRegister("pc")
+        self.assertTrue(pc.IsValid())
+        self.assertEqual(pc.GetValueAsUnsigned(), 0x00100000)
+
+        exception = frame.FindRegister("exception")
+        self.assertTrue(exception.IsValid())
+        self.assertEqual(exception.GetValueAsUnsigned(), 0x00003f5c)
+
+    def test_arm64_corefile(self):
+        ### Create corefile
+        retcode = call(self.create_corefile + " arm64 " + self.corefile, shell=True)
+
+        target = self.dbg.CreateTarget('')
+        err = lldb.SBError()
+        process = target.LoadCore(self.corefile)
+        self.assertEqual(process.IsValid(), True)
+        thread = process.GetSelectedThread()
+        frame = thread.GetSelectedFrame()
+
+        lr = frame.FindRegister("lr")
+        self.assertTrue(lr.IsValid())
+        self.assertEqual(lr.GetValueAsUnsigned(), 0x000000018cd97f28)
+
+        pc = frame.FindRegister("pc")
+        self.assertTrue(pc.IsValid())
+        self.assertEqual(pc.GetValueAsUnsigned(), 0x0000000100003f5c)
+
+        exception = frame.FindRegister("far")
+        self.assertTrue(exception.IsValid())
+        self.assertEqual(exception.GetValueAsUnsigned(), 0x0000000100003f5c)
diff --git a/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp b/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp
new file mode 100644
index 0000000000000..5517a2397ae52
--- /dev/null
+++ b/lldb/test/API/macosx/arm-corefile-regctx/create-arm-corefiles.cpp
@@ -0,0 +1,214 @@
+#include <mach-o/loader.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+
+
+// Normally these are picked up by including <mach/thread_status.h>
+// but that does a compile time check for the build host arch and
+// only defines the ARM register context constants when building on
+// an arm system.  We're creating fake corefiles, and might be
+// creating them on an intel system.
+#define ARM_THREAD_STATE 1
+#define ARM_THREAD_STATE_COUNT 17
+#define ARM_EXCEPTION_STATE 3
+#define ARM_EXCEPTION_STATE_COUNT 3
+#define ARM_THREAD_STATE64 6
+#define ARM_THREAD_STATE64_COUNT 68
+#define ARM_EXCEPTION_STATE64 7
+#define ARM_EXCEPTION_STATE64_COUNT 4
+
+
+union uint32_buf {
+  uint8_t bytebuf[4];
+  uint32_t val;
+};
+
+union uint64_buf {
+  uint8_t bytebuf[8];
+  uint64_t val;
+};
+
+void add_uint64(std::vector<uint8_t> &buf, uint64_t val) {
+  uint64_buf conv;
+  conv.val = val;
+  for (int i = 0; i < 8; i++)
+    buf.push_back(conv.bytebuf[i]);
+}
+
+void add_uint32(std::vector<uint8_t> &buf, uint32_t val) {
+  uint32_buf conv;
+  conv.val = val;
+  for (int i = 0; i < 4; i++)
+    buf.push_back(conv.bytebuf[i]);
+}
+
+std::vector<uint8_t> armv7_lc_thread_load_command() {
+  std::vector<uint8_t> data;
+  add_uint32(data, LC_THREAD);              // thread_command.cmd
+  add_uint32(data, 104);                    // thread_command.cmdsize
+  add_uint32(data, ARM_THREAD_STATE);       // thread_command.flavor
+  add_uint32(data, ARM_THREAD_STATE_COUNT); // thread_command.count
+  add_uint32(data, 0x00010000);             // r0
+  add_uint32(data, 0x00020000);             // r1
+  add_uint32(data, 0x00030000);             // r2
+  add_uint32(data, 0x00040000);             // r3
+  add_uint32(data, 0x00050000);             // r4
+  add_uint32(data, 0x00060000);             // r5
+  add_uint32(data, 0x00070000);             // r6
+  add_uint32(data, 0x00080000);             // r7
+  add_uint32(data, 0x00090000);             // r8
+  add_uint32(data, 0x000a0000);             // r9
+  add_uint32(data, 0x000b0000);             // r10
+  add_uint32(data, 0x000c0000);             // r11
+  add_uint32(data, 0x000d0000);             // r12
+  add_uint32(data, 0x000e0000);             // sp
+  add_uint32(data, 0x000f0000);             // lr
+  add_uint32(data, 0x00100000);             // pc
+  add_uint32(data, 0x00110000);             // cpsr
+
+  add_uint32(data, ARM_EXCEPTION_STATE);       // thread_command.flavor
+  add_uint32(data, ARM_EXCEPTION_STATE_COUNT); // thread_command.count
+  add_uint32(data, 0x00003f5c);                // far
+  add_uint32(data, 0xf2000000);                // esr
+  add_uint32(data, 0x00000000);                // exception
+
+  return data;
+}
+
+std::vector<uint8_t> arm64_lc_thread_load_command() {
+  std::vector<uint8_t> data;
+  add_uint32(data, LC_THREAD);                // thread_command.cmd
+  add_uint32(data, 312);                      // thread_command.cmdsize
+  add_uint32(data, ARM_THREAD_STATE64);       // thread_command.flavor
+  add_uint32(data, ARM_THREAD_STATE64_COUNT); // thread_command.count
+  add_uint64(data, 0x0000000000000001);       // x0
+  add_uint64(data, 0x000000016fdff3c0);       // x1
+  add_uint64(data, 0x000000016fdff3d0);       // x2
+  add_uint64(data, 0x000000016fdff510);       // x3
+  add_uint64(data, 0x0000000000000000);       // x4
+  add_uint64(data, 0x0000000000000000);       // x5
+  add_uint64(data, 0x0000000000000000);       // x6
+  add_uint64(data, 0x0000000000000000);       // x7
+  add_uint64(data, 0x000000010000d910);       // x8
+  add_uint64(data, 0x0000000000000001);       // x9
+  add_uint64(data, 0xe1e88de000000000);       // x10
+  add_uint64(data, 0x0000000000000003);       // x11
+  add_uint64(data, 0x0000000000000148);       // x12
+  add_uint64(data, 0x0000000000004000);       // x13
+  add_uint64(data, 0x0000000000000008);       // x14
+  add_uint64(data, 0x0000000000000000);       // x15
+  add_uint64(data, 0x0000000000000000);       // x16
+  add_uint64(data, 0x0000000100003f5c);       // x17
+  add_uint64(data, 0x0000000000000000);       // x18
+  add_uint64(data, 0x0000000100003f5c);       // x19
+  add_uint64(data, 0x000000010000c000);       // x20
+  add_uint64(data, 0x000000010000d910);       // x21
+  add_uint64(data, 0x000000016fdff250);       // x22
+  add_uint64(data, 0x000000018ce12366);       // x23
+  add_uint64(data, 0x000000016fdff1d0);       // x24
+  add_uint64(data, 0x0000000000000001);       // x25
+  add_uint64(data, 0x0000000000000000);       // x26
+  add_uint64(data, 0x0000000000000000);       // x27
+  add_uint64(data, 0x0000000000000000);       // x28
+  add_uint64(data, 0x000000016fdff3a0);       // fp
+  add_uint64(data, 0x000000018cd97f28);       // lr
+  add_uint64(data, 0x000000016fdff140);       // sp
+  add_uint64(data, 0x0000000100003f5c);       // pc
+  add_uint32(data, 0x80001000);               // cpsr
+
+  add_uint32(data, 0x00000000); // padding
+
+  add_uint32(data, ARM_EXCEPTION_STATE64);       // thread_command.flavor
+  add_uint32(data, ARM_EXCEPTION_STATE64_COUNT); // thread_command.count
+  add_uint64(data, 0x0000000100003f5c);          // far
+  add_uint32(data, 0xf2000000);                  // esr
+  add_uint32(data, 0x00000000);                  // exception
+
+  return data;
+}
+
+enum arch { unspecified, armv7, arm64 };
+
+int main(int argc, char **argv) {
+  if (argc != 3) {
+    fprintf(stderr,
+            "usage: create-arm-corefiles [armv7|arm64] <output-core-name>\n");
+    exit(1);
+  }
+
+  arch arch = unspecified;
+
+  if (strcmp(argv[1], "armv7") == 0)
+    arch = armv7;
+  else if (strcmp(argv[1], "arm64") == 0)
+    arch = arm64;
+  else {
+    fprintf(stderr, "unrecognized architecture %s\n", argv[1]);
+    exit(1);
+  }
+
+  // An array of load commands (in the form of byte arrays)
+  std::vector<std::vector<uint8_t>> load_commands;
+
+  // An array of corefile contents (page data, lc_note data, etc)
+  std::vector<uint8_t> payload;
+
+  // First add all the load commands / payload so we can figure out how large
+  // the load commands will actually be.
+  if (arch == armv7)
+    load_commands.push_back(armv7_lc_thread_load_command());
+  else if (arch == arm64)
+    load_commands.push_back(arm64_lc_thread_load_command());
+
+  int size_of_load_commands = 0;
+  for (const auto &lc : load_commands)
+    size_of_load_commands += lc.size();
+
+  int header_and_load_cmd_room =
+      sizeof(struct mach_header_64) + size_of_load_commands;
+
+  // Erase the load commands / payload now that we know how much space is
+  // needed, redo it.
+  load_commands.clear();
+  payload.clear();
+
+  if (arch == armv7)
+    load_commands.push_back(armv7_lc_thread_load_command());
+  else if (arch == arm64)
+    load_commands.push_back(arm64_lc_thread_load_command());
+
+  struct mach_header_64 mh;
+  mh.magic = MH_MAGIC_64;
+  if (arch == armv7) {
+    mh.cputype = CPU_TYPE_ARM;
+    mh.cpusubtype = CPU_SUBTYPE_ARM_V7M;
+  } else if (arch == arm64) {
+    mh.cputype = CPU_TYPE_ARM64;
+    mh.cpusubtype = CPU_SUBTYPE_ARM64_ALL;
+  }
+  mh.filetype = MH_CORE;
+  mh.ncmds = load_commands.size();
+  mh.sizeofcmds = size_of_load_commands;
+  mh.flags = 0;
+  mh.reserved = 0;
+
+  FILE *f = fopen(argv[2], "w");
+
+  if (f == nullptr) {
+    fprintf(stderr, "Unable to open file %s for writing\n", argv[2]);
+    exit(1);
+  }
+
+  fwrite(&mh, sizeof(struct mach_header_64), 1, f);
+
+  for (const auto &lc : load_commands)
+    fwrite(lc.data(), lc.size(), 1, f);
+
+  fseek(f, header_and_load_cmd_room, SEEK_SET);
+
+  fwrite(payload.data(), payload.size(), 1, f);
+
+  fclose(f);
+}
diff --git a/lldb/test/API/python_api/function_symbol/TestDisasmAPI.py b/lldb/test/API/python_api/function_symbol/TestDisasmAPI.py
index dbdbe751ec745..1f1e4b1867c1e 100644
--- a/lldb/test/API/python_api/function_symbol/TestDisasmAPI.py
+++ b/lldb/test/API/python_api/function_symbol/TestDisasmAPI.py
@@ -19,7 +19,6 @@ def setUp(self):
         self.line2 = line_number(
             'main.c', '// Find the line number for breakpoint 2 here.')
 
-    @expectedFailureAll(oslist=["windows"], bugnumber='llvm.org/pr21765')
     def test(self):
         """Exercise getting SBAddress objects, disassembly, and SBAddress APIs."""
         self.build()
diff --git a/lldb/test/API/python_api/function_symbol/TestSymbolAPI.py b/lldb/test/API/python_api/function_symbol/TestSymbolAPI.py
index 14cc3ebe870f4..2517f9b8e2960 100644
--- a/lldb/test/API/python_api/function_symbol/TestSymbolAPI.py
+++ b/lldb/test/API/python_api/function_symbol/TestSymbolAPI.py
@@ -19,7 +19,6 @@ def setUp(self):
         self.line2 = line_number(
             'main.c', '// Find the line number for breakpoint 2 here.')
 
-    @expectedFailureAll(oslist=["windows"], bugnumber='llvm.org/pr21765')
     def test(self):
         """Exercise some SBSymbol and SBAddress APIs."""
         self.build()
diff --git a/lldb/test/API/python_api/symbol-context/TestSymbolContext.py b/lldb/test/API/python_api/symbol-context/TestSymbolContext.py
index 8094845936f89..3521bc36b142a 100644
--- a/lldb/test/API/python_api/symbol-context/TestSymbolContext.py
+++ b/lldb/test/API/python_api/symbol-context/TestSymbolContext.py
@@ -17,7 +17,6 @@ def setUp(self):
         self.line = line_number(
             'main.c', '// Find the line number of function "c" here.')
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24778")
     def test(self):
         """Exercise SBSymbolContext API extensively."""
         self.build()
diff --git a/lldb/test/API/python_api/target/TestTargetAPI.py b/lldb/test/API/python_api/target/TestTargetAPI.py
index 15ac04cfa74c4..501a7cdbb1813 100644
--- a/lldb/test/API/python_api/target/TestTargetAPI.py
+++ b/lldb/test/API/python_api/target/TestTargetAPI.py
@@ -42,7 +42,6 @@ def test_find_compile_units(self):
         self.setTearDownCleanup(dictionary=d)
         self.find_compile_units(self.getBuildArtifact('b.out'))
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24778")
     def test_find_functions(self):
         """Exercise SBTarget.FindFunctions() API."""
         d = {'EXE': 'b.out'}
@@ -55,7 +54,6 @@ def test_get_description(self):
         self.build()
         self.get_description()
 
-    @expectedFailureAll(oslist=["windows"], bugnumber='llvm.org/pr21765')
     def test_resolve_symbol_context_with_address(self):
         """Exercise SBTarget.ResolveSymbolContextForAddress() API."""
         self.build()
diff --git a/lldb/test/API/python_api/value/TestValueAPI.py b/lldb/test/API/python_api/value/TestValueAPI.py
index 85d8b494e3d3d..00d171f2bebb1 100644
--- a/lldb/test/API/python_api/value/TestValueAPI.py
+++ b/lldb/test/API/python_api/value/TestValueAPI.py
@@ -18,7 +18,6 @@ def setUp(self):
         # Find the line number to of function 'c'.
         self.line = line_number('main.c', '// Break at this line')
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24772")
     def test(self):
         """Exercise some SBValue APIs."""
         d = {'EXE': self.exe_name}
diff --git a/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
index 8177e391468fe..d67f9f7883e98 100644
--- a/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
+++ b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
@@ -129,10 +129,15 @@ def test_watch_address_with_invalid_watch_size(self):
             "pointee",
             value.GetValueAsUnsigned(0),
             value.GetType().GetPointeeType())
-        # Watch for write to *g_char_ptr.
-        error = lldb.SBError()
-        watchpoint = target.WatchAddress(
-            value.GetValueAsUnsigned(), 365, False, True, error)
-        self.assertFalse(watchpoint)
-        self.expect(error.GetCString(), exe=False,
-                    substrs=['watch size of %d is not supported' % 365])
+
+        # debugserver on Darwin AArch64 systems can watch large regions
+        # of memory via https://reviews.llvm.org/D149792 , don't run this
+        # test there.
+        if self.getArchitecture() not in ['arm64', 'arm64e', 'arm64_32']:
+          # Watch for write to *g_char_ptr.
+          error = lldb.SBError()
+          watchpoint = target.WatchAddress(
+              value.GetValueAsUnsigned(), 365, False, True, error)
+          self.assertFalse(watchpoint)
+          self.expect(error.GetCString(), exe=False,
+                      substrs=['watch size of %d is not supported' % 365])
diff --git a/lldb/test/API/tools/lldb-vscode/restart/Makefile b/lldb/test/API/tools/lldb-vscode/restart/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/restart/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart.py b/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart.py
new file mode 100644
index 0000000000000..6715b167bcdb0
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart.py
@@ -0,0 +1,82 @@
+"""
+Test lldb-vscode RestartRequest.
+"""
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import line_number
+import lldbvscode_testcase
+
+
+class TestVSCode_restart(lldbvscode_testcase.VSCodeTestCaseBase):
+
+    @skipIfWindows
+    @skipIfRemote
+    def test_basic_functionality(self):
+        '''
+            Tests the basic restarting functionality: set two breakpoints in
+            sequence, restart at the second, check that we hit the first one.
+        '''
+        line_A = line_number('main.c', '// breakpoint A')
+        line_B = line_number('main.c', '// breakpoint B')
+
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        [bp_A, bp_B] = self.set_source_breakpoints('main.c', [line_A, line_B])
+
+        # Verify we hit A, then B.
+        self.vscode.request_configurationDone()
+        self.verify_breakpoint_hit([bp_A])
+        self.vscode.request_continue()
+        self.verify_breakpoint_hit([bp_B])
+
+        # Make sure i has been modified from its initial value of 0.
+        self.assertEquals(int(self.vscode.get_local_variable_value('i')),
+                          1234, 'i != 1234 after hitting breakpoint B')
+
+        # Restart then check we stop back at A and program state has been reset.
+        self.vscode.request_restart()
+        self.verify_breakpoint_hit([bp_A])
+        self.assertEquals(int(self.vscode.get_local_variable_value('i')),
+                          0, 'i != 0 after hitting breakpoint A on restart')
+
+
+    @skipIfWindows
+    @skipIfRemote
+    def test_stopOnEntry(self):
+        '''
+            Check that the stopOnEntry setting is still honored after a restart.
+        '''
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program, stopOnEntry=True)
+        [bp_main] = self.set_function_breakpoints(['main'])
+        self.vscode.request_configurationDone()
+
+        # Once the "configuration done" event is sent, we should get a stopped
+        # event immediately because of stopOnEntry.
+        stopped_events = self.vscode.wait_for_stopped()
+        for stopped_event in stopped_events:
+            if 'body' in stopped_event:
+                body = stopped_event['body']
+                if 'reason' in body:
+                    reason = body['reason']
+                    self.assertNotEqual(
+                        reason, 'breakpoint',
+                        'verify stop isn\'t "main" breakpoint')
+
+        # Then, if we continue, we should hit the breakpoint at main.
+        self.vscode.request_continue()
+        self.verify_breakpoint_hit([bp_main])
+
+        # Restart and check that we still get a stopped event before reaching
+        # main.
+        self.vscode.request_restart()
+        stopped_events = self.vscode.wait_for_stopped()
+        for stopped_event in stopped_events:
+            if 'body' in stopped_event:
+                body = stopped_event['body']
+                if 'reason' in body:
+                    reason = body['reason']
+                    self.assertNotEqual(
+                        reason, 'breakpoint',
+                        'verify stop after restart isn\'t "main" breakpoint')
+
diff --git a/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart_runInTerminal.py b/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart_runInTerminal.py
new file mode 100644
index 0000000000000..f250e014cfe3d
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart_runInTerminal.py
@@ -0,0 +1,104 @@
+"""
+Test lldb-vscode RestartRequest.
+"""
+
+import os
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import line_number
+import lldbvscode_testcase
+
+
+class TestVSCode_restart_runInTerminal(lldbvscode_testcase.VSCodeTestCaseBase):
+
+    def isTestSupported(self):
+        try:
+            # We skip this test for debug builds because it takes too long
+            # parsing lldb's own debug info. Release builds are fine.
+            # Checking the size of the lldb-vscode binary seems to be a decent
+            # proxy for a quick detection. It should be far less than 1 MB in
+            # Release builds.
+            return os.path.getsize(os.environ["LLDBVSCODE_EXEC"]) < 1000000
+        except:
+            return False
+
+
+    @skipIfWindows
+    @skipIfRemote
+    def test_basic_functionality(self):
+        '''
+            Test basic restarting functionality when the process is running in
+            a terminal.
+        '''
+        if not self.isTestSupported():
+            return
+        line_A = line_number('main.c', '// breakpoint A')
+        line_B = line_number('main.c', '// breakpoint B')
+
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program, runInTerminal=True)
+        [bp_A, bp_B] = self.set_source_breakpoints('main.c', [line_A, line_B])
+
+        # Verify we hit A, then B.
+        self.vscode.request_configurationDone()
+        self.verify_breakpoint_hit([bp_A])
+        self.vscode.request_continue()
+        self.verify_breakpoint_hit([bp_B])
+
+        # Make sure i has been modified from its initial value of 0.
+        self.assertEquals(int(self.vscode.get_local_variable_value('i')),
+                          1234, 'i != 1234 after hitting breakpoint B')
+
+        # Restart.
+        self.vscode.request_restart()
+
+        # Finally, check we stop back at A and program state has been reset.
+        self.verify_breakpoint_hit([bp_A])
+        self.assertEquals(int(self.vscode.get_local_variable_value('i')),
+                          0, 'i != 0 after hitting breakpoint A on restart')
+
+    @skipIfWindows
+    @skipIfRemote
+    def test_stopOnEntry(self):
+        '''
+            Check that stopOnEntry works correctly when using runInTerminal.
+        '''
+        if not self.isTestSupported():
+            return
+        line_A = line_number('main.c', '// breakpoint A')
+        line_B = line_number('main.c', '// breakpoint B')
+
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program, runInTerminal=True, stopOnEntry=True)
+        [bp_main] = self.set_function_breakpoints(['main'])
+        self.vscode.request_configurationDone()
+
+        # When using stopOnEntry, configurationDone doesn't result in a running
+        # process, we should immediately get a stopped event instead.
+        stopped_events = self.vscode.wait_for_stopped()
+        # We should be stopped at the entry point.
+        for stopped_event in stopped_events:
+            if 'body' in stopped_event:
+                body = stopped_event['body']
+                if 'reason' in body:
+                    reason = body['reason']
+                    self.assertNotEqual(
+                        reason, 'breakpoint',
+                        'verify stop isn\'t a breakpoint')
+
+        # Then, if we continue, we should hit the breakpoint at main.
+        self.vscode.request_continue()
+        self.verify_breakpoint_hit([bp_main])
+
+        # Restart and check that we still get a stopped event before reaching
+        # main.
+        self.vscode.request_restart()
+        stopped_events = self.vscode.wait_for_stopped()
+        for stopped_event in stopped_events:
+            if 'body' in stopped_event:
+                body = stopped_event['body']
+                if 'reason' in body:
+                    reason = body['reason']
+                    self.assertNotEqual(
+                        reason, 'breakpoint',
+                        'verify stop after restart isn\'t "main" breakpoint')
+
diff --git a/lldb/test/API/tools/lldb-vscode/restart/main.c b/lldb/test/API/tools/lldb-vscode/restart/main.c
new file mode 100644
index 0000000000000..710af8e975020
--- /dev/null
+++ b/lldb/test/API/tools/lldb-vscode/restart/main.c
@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+int main(int argc, char const *argv[], char const *envp[]) {
+  int i = 0;
+  printf("Do something\n"); // breakpoint A
+  printf("Do something else\n");
+  i = 1234;
+  return 0;  // breakpoint B
+}
diff --git a/lldb/test/Shell/ObjectFile/ELF/aarch32-relocations.yaml b/lldb/test/Shell/ObjectFile/ELF/aarch32-relocations.yaml
new file mode 100644
index 0000000000000..1a12ba74e417e
--- /dev/null
+++ b/lldb/test/Shell/ObjectFile/ELF/aarch32-relocations.yaml
@@ -0,0 +1,62 @@
+# RUN: yaml2obj %s -o %t
+# RUN: lldb-test object-file -contents %t | FileCheck %s
+
+## Test that R_ARM_ABS32 relocations are resolved in .debug_info sections on aarch32.
+## REL-type relocations store implicit addend as signed values inline.
+## We relocate the symbol foo with 4 different addends and bar once in the .debug_info section.
+## Results that exceed the 32-bit range or overflow are logged and ignored.
+
+# CHECK:      Name: .debug_info
+# CHECK:      Data:  (
+#
+#              Addends: Zero     Positive Negative Overflow Out-of-range
+#                       00000000 04030201 D6FFFFFF D5FFFFFF FFFFFF7F
+# CHECK-NEXT:     0000: 2A000000 2E030201 00000000 D5FFFFFF FFFFFF7F
+# CHECK-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_ARM
+  Flags:           [ EF_ARM_EABI_VER5 ]
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+  - Name:            .debug_info
+    Type:            SHT_PROGBITS
+    Content:         0000000004030201D6FFFFFFD5FFFFFFFFFFFF7F
+  - Name:            .rel.debug_info
+    Type:            SHT_REL
+    Info:            .debug_info
+    Relocations:
+      - Offset:          0x0
+        Symbol:          foo
+        Type:            R_ARM_ABS32
+      - Offset:          0x4
+        Symbol:          foo
+        Type:            R_ARM_ABS32
+      - Offset:          0x8
+        Symbol:          foo
+        Type:            R_ARM_ABS32
+      - Offset:          0xC
+        Symbol:          foo
+        Type:            R_ARM_ABS32
+      - Offset:          0x10
+        Symbol:          bar
+        Type:            R_ARM_ABS32
+Symbols:
+  - Name:            .debug_info
+    Type:            STT_SECTION
+    Section:         .debug_info
+  - Name:            foo
+    Type:            STT_FUNC
+    Section:         .debug_info
+    Value:           0x0000002A
+  - Name:            bar
+    Type:            STT_FUNC
+    Section:         .debug_info
+    Value:           0xFF000000
+...
diff --git a/lldb/test/Shell/Watchpoint/ExpressionLanguage.test b/lldb/test/Shell/Watchpoint/ExpressionLanguage.test
new file mode 100644
index 0000000000000..3966923778620
--- /dev/null
+++ b/lldb/test/Shell/Watchpoint/ExpressionLanguage.test
@@ -0,0 +1,20 @@
+# UNSUPPORTED: system-windows
+# RUN: %clangxx_host %p/Inputs/languages.cpp -g -o %t.out
+# RUN: %lldb -b -o 'settings set interpreter.stop-command-source-on-error false' -s %s %t.out 2>&1 | FileCheck %s
+
+settings  show interpreter.stop-command-source-on-error
+# CHECK: interpreter.stop-command-source-on-error (boolean) = false
+
+b main
+run
+# CHECK: stopped
+# CHECK-NEXT: stop reason = breakpoint
+
+watchpoint set expression -- &g_foo
+# CHECK: Watchpoint created:
+
+watchpoint set expression -l c++ -- &g_bar
+# CHECK: Watchpoint created:
+
+watchpoint set expression -l fake -- &g_baz
+# CHECK: Unknown language type: 'fake' for expression. List of supported languages:
diff --git a/lldb/test/Shell/Watchpoint/Inputs/languages.cpp b/lldb/test/Shell/Watchpoint/Inputs/languages.cpp
new file mode 100644
index 0000000000000..98b653f3d404a
--- /dev/null
+++ b/lldb/test/Shell/Watchpoint/Inputs/languages.cpp
@@ -0,0 +1,12 @@
+#include <cstdint>
+#include <iostream>
+
+uint64_t g_foo = 5;
+uint64_t g_bar = 6;
+uint64_t g_baz = 7;
+
+int main() {
+  int val = 8;
+  printf("Hello world! %d\n", val);
+  return 0;
+}
diff --git a/lldb/tools/debugserver/debugserver.xcodeproj/project.pbxproj b/lldb/tools/debugserver/debugserver.xcodeproj/project.pbxproj
index 4c18f73339368..79175c6d9dd73 100644
--- a/lldb/tools/debugserver/debugserver.xcodeproj/project.pbxproj
+++ b/lldb/tools/debugserver/debugserver.xcodeproj/project.pbxproj
@@ -39,8 +39,6 @@
 		26CE05BD115C36430022F371 /* PThreadMutex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2672DBEE0EEF446700E92059 /* PThreadMutex.cpp */; };
 		26CE05BE115C36440022F371 /* SysSignal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 26C638010C71334A0024798E /* SysSignal.cpp */; };
 		26CE05BF115C364D0022F371 /* DNBArchImplX86_64.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 26CF99A21142EB7400011AAB /* DNBArchImplX86_64.cpp */; };
-		26CE05C0115C364F0022F371 /* DNBArchImplI386.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 26C637EA0C71334A0024798E /* DNBArchImplI386.cpp */; };
-		26CE05C1115C36510022F371 /* DNBArchImpl.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2675D4220CCEB705000F49AF /* DNBArchImpl.cpp */; };
 		26CE05C3115C36580022F371 /* CFString.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2695DD9B0D3EC160007E4CA2 /* CFString.cpp */; };
 		26CE05C5115C36590022F371 /* CFBundle.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2695DD910D3EBFF6007E4CA2 /* CFBundle.cpp */; };
 		26CE05CF115C36F70022F371 /* CoreFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 26ACA3340D3E956300A2120B /* CoreFoundation.framework */; settings = {ATTRIBUTES = (Required, ); }; };
@@ -70,8 +68,6 @@
 		456F675C1AD46CE9002850C2 /* PThreadMutex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2672DBEE0EEF446700E92059 /* PThreadMutex.cpp */; };
 		456F675D1AD46CE9002850C2 /* SysSignal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 26C638010C71334A0024798E /* SysSignal.cpp */; };
 		456F675E1AD46CE9002850C2 /* DNBArchImplX86_64.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 26CF99A21142EB7400011AAB /* DNBArchImplX86_64.cpp */; };
-		456F675F1AD46CE9002850C2 /* DNBArchImplI386.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 26C637EA0C71334A0024798E /* DNBArchImplI386.cpp */; };
-		456F67601AD46CE9002850C2 /* DNBArchImpl.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2675D4220CCEB705000F49AF /* DNBArchImpl.cpp */; };
 		456F67621AD46CE9002850C2 /* CFString.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2695DD9B0D3EC160007E4CA2 /* CFString.cpp */; };
 		456F67641AD46CE9002850C2 /* CFBundle.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2695DD910D3EBFF6007E4CA2 /* CFBundle.cpp */; };
 		456F67651AD46CE9002850C2 /* PseudoTerminal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = AF67ABFF0D34604D0022D128 /* PseudoTerminal.cpp */; };
@@ -107,8 +103,6 @@
 		266B5ECF1460A68200E43F0A /* DNBArchImplARM64.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = DNBArchImplARM64.cpp; sourceTree = "<group>"; };
 		266B5ED01460A68200E43F0A /* DNBArchImplARM64.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DNBArchImplARM64.h; sourceTree = "<group>"; };
 		2672DBEE0EEF446700E92059 /* PThreadMutex.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PThreadMutex.cpp; sourceTree = "<group>"; };
-		2675D4220CCEB705000F49AF /* DNBArchImpl.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; name = DNBArchImpl.cpp; path = arm/DNBArchImpl.cpp; sourceTree = "<group>"; };
-		2675D4230CCEB705000F49AF /* DNBArchImpl.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; name = DNBArchImpl.h; path = arm/DNBArchImpl.h; sourceTree = "<group>"; };
 		2695DD910D3EBFF6007E4CA2 /* CFBundle.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CFBundle.cpp; sourceTree = "<group>"; };
 		2695DD920D3EBFF6007E4CA2 /* CFBundle.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFBundle.h; sourceTree = "<group>"; };
 		2695DD9A0D3EC160007E4CA2 /* CFString.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFString.h; sourceTree = "<group>"; };
@@ -142,8 +136,6 @@
 		26C637E30C71334A0024798E /* DNBRegisterInfo.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = DNBRegisterInfo.h; sourceTree = "<group>"; };
 		26C637E70C71334A0024798E /* CFUtils.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = CFUtils.h; sourceTree = "<group>"; };
 		26C637E80C71334A0024798E /* dbgnub-mig.defs */ = {isa = PBXFileReference; explicitFileType = sourcecode.mig; fileEncoding = 30; path = "dbgnub-mig.defs"; sourceTree = "<group>"; };
-		26C637EA0C71334A0024798E /* DNBArchImplI386.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; path = DNBArchImplI386.cpp; sourceTree = "<group>"; };
-		26C637EB0C71334A0024798E /* DNBArchImplI386.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = DNBArchImplI386.h; sourceTree = "<group>"; };
 		26C637EE0C71334A0024798E /* MachException.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; path = MachException.cpp; sourceTree = "<group>"; };
 		26C637EF0C71334A0024798E /* MachException.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = MachException.h; sourceTree = "<group>"; };
 		26C637F00C71334A0024798E /* MachProcess.mm */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.objcpp; path = MachProcess.mm; sourceTree = "<group>"; };
@@ -170,9 +162,12 @@
 		26E6B9DA0D1329010037ECDD /* RNBDefs.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RNBDefs.h; sourceTree = "<group>"; };
 		456F67721AD46CE9002850C2 /* debugserver-nonui */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = "debugserver-nonui"; sourceTree = BUILT_PRODUCTS_DIR; };
 		49D404611E39260F00570CDC /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
-		49F530111331519C008956F6 /* MachRegisterStatesI386.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MachRegisterStatesI386.h; sourceTree = "<group>"; };
 		49F5301213316D7F008956F6 /* MachRegisterStatesX86_64.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MachRegisterStatesX86_64.h; sourceTree = "<group>"; };
 		9457ECF61419864100DFE7D8 /* stack_logging.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = stack_logging.h; sourceTree = "<group>"; };
+		9684D93A29FCA1E10046D45E /* debugserver-macosx-private-entitlements.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; name = "debugserver-macosx-private-entitlements.plist"; path = "resources/debugserver-macosx-private-entitlements.plist"; sourceTree = SOURCE_ROOT; };
+		9684D93B29FCA1E10046D45E /* lldb-debugserver-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; name = "lldb-debugserver-Info.plist"; path = "resources/lldb-debugserver-Info.plist"; sourceTree = SOURCE_ROOT; };
+		9684D93C29FCA1E10046D45E /* debugserver-macosx-entitlements.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; name = "debugserver-macosx-entitlements.plist"; path = "resources/debugserver-macosx-entitlements.plist"; sourceTree = SOURCE_ROOT; };
+		9684D93D29FCA1E10046D45E /* debugserver-entitlements.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; name = "debugserver-entitlements.plist"; path = "resources/debugserver-entitlements.plist"; sourceTree = SOURCE_ROOT; };
 		AF0934BA18E12B92005A11FD /* Genealogy.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Genealogy.h; sourceTree = "<group>"; };
 		AF0934BB18E12B92005A11FD /* GenealogySPI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GenealogySPI.h; sourceTree = "<group>"; };
 		AF48558B1D75126800D19C07 /* StdStringExtractor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = StdStringExtractor.cpp; sourceTree = "<group>"; };
@@ -239,18 +234,13 @@
 			path = arm64;
 			sourceTree = "<group>";
 		};
-		2675D41C0CCEB6CF000F49AF /* arm */ = {
-			isa = PBXGroup;
-			children = (
-				2675D4220CCEB705000F49AF /* DNBArchImpl.cpp */,
-				2675D4230CCEB705000F49AF /* DNBArchImpl.h */,
-			);
-			name = arm;
-			sourceTree = "<group>";
-		};
 		26A028FE114AB6A60029C479 /* Resources */ = {
 			isa = PBXGroup;
 			children = (
+				9684D93D29FCA1E10046D45E /* debugserver-entitlements.plist */,
+				9684D93C29FCA1E10046D45E /* debugserver-macosx-entitlements.plist */,
+				9684D93A29FCA1E10046D45E /* debugserver-macosx-private-entitlements.plist */,
+				9684D93B29FCA1E10046D45E /* lldb-debugserver-Info.plist */,
 				26A4BAED0D498B7D00A9BEAB /* com.apple.debugserver.plist */,
 				EF88788B0D9C7558001831DA /* com.apple.debugserver.applist.plist */,
 				269E8DF8164B2ED200AD65F6 /* com.apple.debugserver.posix.plist */,
@@ -336,9 +326,7 @@
 				2695DD9B0D3EC160007E4CA2 /* CFString.cpp */,
 				26C637E70C71334A0024798E /* CFUtils.h */,
 				2307CCCC1D4A5DAE0016ABC0 /* CMakeLists.txt */,
-				2675D41C0CCEB6CF000F49AF /* arm */,
 				266B5ECE1460A68200E43F0A /* arm64 */,
-				26C637E90C71334A0024798E /* i386 */,
 				26CF99A11142EB7400011AAB /* x86_64 */,
 				26C637E80C71334A0024798E /* dbgnub-mig.defs */,
 				AFEC3363194A8B0B00FF05C6 /* Genealogy.cpp */,
@@ -365,16 +353,6 @@
 			path = MacOSX;
 			sourceTree = "<group>";
 		};
-		26C637E90C71334A0024798E /* i386 */ = {
-			isa = PBXGroup;
-			children = (
-				26C637EA0C71334A0024798E /* DNBArchImplI386.cpp */,
-				26C637EB0C71334A0024798E /* DNBArchImplI386.h */,
-				49F530111331519C008956F6 /* MachRegisterStatesI386.h */,
-			);
-			path = i386;
-			sourceTree = "<group>";
-		};
 		26CF99A11142EB7400011AAB /* x86_64 */ = {
 			isa = PBXGroup;
 			children = (
@@ -422,7 +400,6 @@
 			buildPhases = (
 				26CE0591115C31C20022F371 /* Sources */,
 				26CE0592115C31C20022F371 /* Frameworks */,
-				4C3326CB18B2A2F600EB5DD7 /* ShellScript */,
 			);
 			buildRules = (
 			);
@@ -478,22 +455,6 @@
 		};
 /* End PBXProject section */
 
-/* Begin PBXShellScriptBuildPhase section */
-		4C3326CB18B2A2F600EB5DD7 /* ShellScript */ = {
-			isa = PBXShellScriptBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			inputPaths = (
-			);
-			outputPaths = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-			shellPath = "/bin/sh -x";
-			shellScript = "if [ \"${CONFIGURATION}\" != BuildAndIntegration ]\nthen\n    if [ -n \"${DEBUGSERVER_USE_FROM_SYSTEM}\" ]\n    then\n\t\tditto \"${DEVELOPER_DIR}/../SharedFrameworks/LLDB.framework/Resources/debugserver\" \"${TARGET_BUILD_DIR}/${TARGET_NAME}\"\n    elif [ \"${DEBUGSERVER_DISABLE_CODESIGN}\" == \"\" ]\n    then\n        codesign -f -s lldb_codesign --entitlements ${SRCROOT}/resources/debugserver-macosx-entitlements.plist \"${TARGET_BUILD_DIR}/${TARGET_NAME}\"\n    fi\nfi\n";
-		};
-/* End PBXShellScriptBuildPhase section */
-
 /* Begin PBXSourcesBuildPhase section */
 		26CE0591115C31C20022F371 /* Sources */ = {
 			isa = PBXSourcesBuildPhase;
@@ -529,8 +490,6 @@
 				26CE05BD115C36430022F371 /* PThreadMutex.cpp in Sources */,
 				26CE05BE115C36440022F371 /* SysSignal.cpp in Sources */,
 				26CE05BF115C364D0022F371 /* DNBArchImplX86_64.cpp in Sources */,
-				26CE05C0115C364F0022F371 /* DNBArchImplI386.cpp in Sources */,
-				26CE05C1115C36510022F371 /* DNBArchImpl.cpp in Sources */,
 				26CE05C5115C36590022F371 /* CFBundle.cpp in Sources */,
 				26CE05C3115C36580022F371 /* CFString.cpp in Sources */,
 				26CE05F1115C387C0022F371 /* PseudoTerminal.cpp in Sources */,
@@ -570,8 +529,6 @@
 				456F675C1AD46CE9002850C2 /* PThreadMutex.cpp in Sources */,
 				456F675D1AD46CE9002850C2 /* SysSignal.cpp in Sources */,
 				456F675E1AD46CE9002850C2 /* DNBArchImplX86_64.cpp in Sources */,
-				456F675F1AD46CE9002850C2 /* DNBArchImplI386.cpp in Sources */,
-				456F67601AD46CE9002850C2 /* DNBArchImpl.cpp in Sources */,
 				23043C9E1D35DBFA00FC25CA /* StringConvert.cpp in Sources */,
 				AF588449206077BD00A0CB5A /* SocketAddress.cpp in Sources */,
 				456F67621AD46CE9002850C2 /* CFString.cpp in Sources */,
@@ -600,7 +557,7 @@
 				CLANG_WARN_INT_CONVERSION = YES;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "";
+				CODE_SIGN_IDENTITY = "-";
 				COPY_PHASE_STRIP = NO;
 				CURRENT_PROJECT_VERSION = 360.99.0;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
@@ -666,10 +623,10 @@
 			buildSettings = {
 				CLANG_CXX_LANGUAGE_STANDARD = "c++17";
 				CLANG_CXX_LIBRARY = "libc++";
-				"CODE_SIGN_ENTITLEMENTS[sdk=*]" = "source/debugserver-entitlements.plist";
-				"CODE_SIGN_ENTITLEMENTS[sdk=macosx*]" = "source/debugserver-macosx-entitlements.plist";
-				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "";
-				"CODE_SIGN_IDENTITY[sdk=macosx*]" = "";
+				"CODE_SIGN_ENTITLEMENTS[sdk=*]" = "resources/debugserver-entitlements.plist";
+				"CODE_SIGN_ENTITLEMENTS[sdk=macosx*]" = "resources/debugserver-macosx-entitlements.plist";
+				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "-";
+				"CODE_SIGN_IDENTITY[sdk=macosx*]" = "-";
 				COPY_PHASE_STRIP = YES;
 				CURRENT_PROJECT_VERSION = 360.99.0;
 				FRAMEWORK_SEARCH_PATHS = $SDKROOT/System/Library/PrivateFrameworks;
@@ -766,10 +723,10 @@
 			buildSettings = {
 				CLANG_CXX_LANGUAGE_STANDARD = "c++17";
 				CLANG_CXX_LIBRARY = "libc++";
-				"CODE_SIGN_ENTITLEMENTS[sdk=*]" = "source/debugserver-entitlements.plist";
-				"CODE_SIGN_ENTITLEMENTS[sdk=macosx*]" = "source/debugserver-macosx-entitlements.plist";
-				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "";
-				"CODE_SIGN_IDENTITY[sdk=macosx*]" = "";
+				"CODE_SIGN_ENTITLEMENTS[sdk=*]" = "resources/debugserver-entitlements.plist";
+				"CODE_SIGN_ENTITLEMENTS[sdk=macosx*]" = "resources/debugserver-macosx-entitlements.plist";
+				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "-";
+				"CODE_SIGN_IDENTITY[sdk=macosx*]" = "-";
 				COPY_PHASE_STRIP = YES;
 				CURRENT_PROJECT_VERSION = 360.99.0;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
@@ -875,8 +832,8 @@
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
 				"CODE_SIGN_ENTITLEMENTS[sdk=*]" = "source/debugserver-entitlements.plist";
 				"CODE_SIGN_ENTITLEMENTS[sdk=macosx*]" = "source/debugserver-macosx-entitlements.plist";
-				CODE_SIGN_IDENTITY = "";
-				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "";
+				CODE_SIGN_IDENTITY = "-";
+				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "-";
 				"CODE_SIGN_IDENTITY[sdk=macosx*]" = "";
 				COPY_PHASE_STRIP = NO;
 				CURRENT_PROJECT_VERSION = 360.99.0;
@@ -955,7 +912,7 @@
 				CLANG_CXX_LIBRARY = "libc++";
 				"CODE_SIGN_ENTITLEMENTS[sdk=*]" = "source/debugserver-entitlements.plist";
 				"CODE_SIGN_ENTITLEMENTS[sdk=macosx*]" = "source/debugserver-macosx-entitlements.plist";
-				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "";
+				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "-";
 				"CODE_SIGN_IDENTITY[sdk=macosx*]" = "";
 				COPY_PHASE_STRIP = YES;
 				CURRENT_PROJECT_VERSION = 360.99.0;
diff --git a/lldb/tools/debugserver/source/CMakeLists.txt b/lldb/tools/debugserver/source/CMakeLists.txt
index 2aaefb56cb4b6..43accc363ef3c 100644
--- a/lldb/tools/debugserver/source/CMakeLists.txt
+++ b/lldb/tools/debugserver/source/CMakeLists.txt
@@ -3,9 +3,7 @@ include(CheckLibraryExists)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/..)
 include_directories(${LLDB_SOURCE_DIR}/source)
 include_directories(MacOSX)
-include_directories(MacOSX/i386)
 include_directories(MacOSX/x86_64)
-include_directories(MacOSX/arm)
 include_directories(MacOSX/arm64)
 
 function(check_certificate identity result_valid)
@@ -228,9 +226,7 @@ set(lldbDebugserverCommonSources
   MacOSX/MachVMMemory.cpp
   MacOSX/MachVMRegion.cpp
   MacOSX/OsLogger.cpp
-  MacOSX/arm/DNBArchImpl.cpp
   MacOSX/arm64/DNBArchImplARM64.cpp
-  MacOSX/i386/DNBArchImplI386.cpp
   MacOSX/x86_64/DNBArchImplX86_64.cpp
   ${generated_mach_interfaces}
   ${DEBUGSERVER_VERS_GENERATED_FILE})
diff --git a/lldb/tools/debugserver/source/DNB.cpp b/lldb/tools/debugserver/source/DNB.cpp
index 07423ea56e6e2..bab5b4a813058 100644
--- a/lldb/tools/debugserver/source/DNB.cpp
+++ b/lldb/tools/debugserver/source/DNB.cpp
@@ -1768,10 +1768,8 @@ std::string DNBGetMacCatalystVersionString() {
 void DNBInitialize() {
   DNBLogThreadedIf(LOG_PROCESS, "DNBInitialize ()");
 #if defined(__i386__) || defined(__x86_64__)
-  DNBArchImplI386::Initialize();
   DNBArchImplX86_64::Initialize();
 #elif defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
-  DNBArchMachARM::Initialize();
   DNBArchMachARM64::Initialize();
 #endif
 }
diff --git a/lldb/tools/debugserver/source/DNBArch.h b/lldb/tools/debugserver/source/DNBArch.h
index b673e403feb43..85b2d1caf9af1 100644
--- a/lldb/tools/debugserver/source/DNBArch.h
+++ b/lldb/tools/debugserver/source/DNBArch.h
@@ -122,9 +122,7 @@ class DNBArchProtocol {
                       // SaveRegisterState/RestoreRegisterState
 };
 
-#include "MacOSX/arm/DNBArchImpl.h"
 #include "MacOSX/arm64/DNBArchImplARM64.h"
-#include "MacOSX/i386/DNBArchImplI386.h"
 #include "MacOSX/x86_64/DNBArchImplX86_64.h"
 
 #endif
diff --git a/lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.cpp b/lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.cpp
deleted file mode 100644
index 92717f1c57544..0000000000000
--- a/lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.cpp
+++ /dev/null
@@ -1,2195 +0,0 @@
-//===-- DNBArchImpl.cpp -----------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  Created by Greg Clayton on 6/25/07.
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
-
-#include "MacOSX/arm/DNBArchImpl.h"
-#include "ARM_DWARF_Registers.h"
-#include "ARM_ehframe_Registers.h"
-#include "DNB.h"
-#include "DNBBreakpoint.h"
-#include "DNBLog.h"
-#include "DNBRegisterInfo.h"
-#include "MacOSX/MachProcess.h"
-#include "MacOSX/MachThread.h"
-
-#include <cinttypes>
-#include <sys/sysctl.h>
-
-// BCR address match type
-#define BCR_M_IMVA_MATCH ((uint32_t)(0u << 21))
-#define BCR_M_CONTEXT_ID_MATCH ((uint32_t)(1u << 21))
-#define BCR_M_IMVA_MISMATCH ((uint32_t)(2u << 21))
-#define BCR_M_RESERVED ((uint32_t)(3u << 21))
-
-// Link a BVR/BCR or WVR/WCR pair to another
-#define E_ENABLE_LINKING ((uint32_t)(1u << 20))
-
-// Byte Address Select
-#define BAS_IMVA_PLUS_0 ((uint32_t)(1u << 5))
-#define BAS_IMVA_PLUS_1 ((uint32_t)(1u << 6))
-#define BAS_IMVA_PLUS_2 ((uint32_t)(1u << 7))
-#define BAS_IMVA_PLUS_3 ((uint32_t)(1u << 8))
-#define BAS_IMVA_0_1 ((uint32_t)(3u << 5))
-#define BAS_IMVA_2_3 ((uint32_t)(3u << 7))
-#define BAS_IMVA_ALL ((uint32_t)(0xfu << 5))
-
-// Break only in privileged or user mode
-#define S_RSVD ((uint32_t)(0u << 1))
-#define S_PRIV ((uint32_t)(1u << 1))
-#define S_USER ((uint32_t)(2u << 1))
-#define S_PRIV_USER ((S_PRIV) | (S_USER))
-
-#define BCR_ENABLE ((uint32_t)(1u))
-#define WCR_ENABLE ((uint32_t)(1u))
-
-// Watchpoint load/store
-#define WCR_LOAD ((uint32_t)(1u << 3))
-#define WCR_STORE ((uint32_t)(1u << 4))
-
-// Definitions for the Debug Status and Control Register fields:
-// [5:2] => Method of debug entry
-//#define WATCHPOINT_OCCURRED     ((uint32_t)(2u))
-// I'm seeing this, instead.
-#define WATCHPOINT_OCCURRED ((uint32_t)(10u))
-
-// 0xE120BE70
-static const uint8_t g_arm_breakpoint_opcode[] = {0x70, 0xBE, 0x20, 0xE1};
-static const uint8_t g_thumb_breakpoint_opcode[] = {0x70, 0xBE};
-
-// A watchpoint may need to be implemented using two watchpoint registers.
-// e.g. watching an 8-byte region when the device can only watch 4-bytes.
-//
-// This stores the lo->hi mappings.  It's safe to initialize to all 0's
-// since hi > lo and therefore LoHi[i] cannot be 0.
-static uint32_t LoHi[16] = {0};
-
-// ARM constants used during decoding
-#define REG_RD 0
-#define LDM_REGLIST 1
-#define PC_REG 15
-#define PC_REGLIST_BIT 0x8000
-
-// ARM conditions
-#define COND_EQ 0x0
-#define COND_NE 0x1
-#define COND_CS 0x2
-#define COND_HS 0x2
-#define COND_CC 0x3
-#define COND_LO 0x3
-#define COND_MI 0x4
-#define COND_PL 0x5
-#define COND_VS 0x6
-#define COND_VC 0x7
-#define COND_HI 0x8
-#define COND_LS 0x9
-#define COND_GE 0xA
-#define COND_LT 0xB
-#define COND_GT 0xC
-#define COND_LE 0xD
-#define COND_AL 0xE
-#define COND_UNCOND 0xF
-
-#define MASK_CPSR_T (1u << 5)
-#define MASK_CPSR_J (1u << 24)
-
-#define MNEMONIC_STRING_SIZE 32
-#define OPERAND_STRING_SIZE 128
-
-#if !defined(__arm64__) && !defined(__aarch64__)
-// Returns true if the first 16 bit opcode of a thumb instruction indicates
-// the instruction will be a 32 bit thumb opcode
-static bool IsThumb32Opcode(uint16_t opcode) {
-  if (((opcode & 0xE000) == 0xE000) && (opcode & 0x1800))
-    return true;
-  return false;
-}
-#endif
-
-void DNBArchMachARM::Initialize() {
-  DNBArchPluginInfo arch_plugin_info = {
-      CPU_TYPE_ARM, DNBArchMachARM::Create, DNBArchMachARM::GetRegisterSetInfo,
-      DNBArchMachARM::SoftwareBreakpointOpcode};
-
-  // Register this arch plug-in with the main protocol class
-  DNBArchProtocol::RegisterArchPlugin(arch_plugin_info);
-}
-
-DNBArchProtocol *DNBArchMachARM::Create(MachThread *thread) {
-  DNBArchMachARM *obj = new DNBArchMachARM(thread);
-  return obj;
-}
-
-const uint8_t *DNBArchMachARM::SoftwareBreakpointOpcode(nub_size_t byte_size) {
-  switch (byte_size) {
-  case 2:
-    return g_thumb_breakpoint_opcode;
-  case 4:
-    return g_arm_breakpoint_opcode;
-  }
-  return NULL;
-}
-
-uint32_t DNBArchMachARM::GetCPUType() { return CPU_TYPE_ARM; }
-
-uint64_t DNBArchMachARM::GetPC(uint64_t failValue) {
-  // Get program counter
-  if (GetGPRState(false) == KERN_SUCCESS)
-    return m_state.context.gpr.__pc;
-  return failValue;
-}
-
-kern_return_t DNBArchMachARM::SetPC(uint64_t value) {
-  // Get program counter
-  kern_return_t err = GetGPRState(false);
-  if (err == KERN_SUCCESS) {
-    m_state.context.gpr.__pc = (uint32_t)value;
-    err = SetGPRState();
-  }
-  return err == KERN_SUCCESS;
-}
-
-uint64_t DNBArchMachARM::GetSP(uint64_t failValue) {
-  // Get stack pointer
-  if (GetGPRState(false) == KERN_SUCCESS)
-    return m_state.context.gpr.__sp;
-  return failValue;
-}
-
-kern_return_t DNBArchMachARM::GetGPRState(bool force) {
-  int set = e_regSetGPR;
-  // Check if we have valid cached registers
-  if (!force && m_state.GetError(set, Read) == KERN_SUCCESS)
-    return KERN_SUCCESS;
-
-  // Read the registers from our thread
-  mach_msg_type_number_t count = ARM_THREAD_STATE_COUNT;
-  kern_return_t kret =
-      ::thread_get_state(m_thread->MachPortNumber(), ARM_THREAD_STATE,
-                         (thread_state_t)&m_state.context.gpr, &count);
-  uint32_t *r = &m_state.context.gpr.__r[0];
-  DNBLogThreadedIf(
-      LOG_THREAD, "thread_get_state(0x%4.4x, %u, &gpr, %u) => 0x%8.8x (count = "
-                  "%u) regs r0=%8.8x r1=%8.8x r2=%8.8x r3=%8.8x r4=%8.8x "
-                  "r5=%8.8x r6=%8.8x r7=%8.8x r8=%8.8x r9=%8.8x r10=%8.8x "
-                  "r11=%8.8x s12=%8.8x sp=%8.8x lr=%8.8x pc=%8.8x cpsr=%8.8x",
-      m_thread->MachPortNumber(), ARM_THREAD_STATE, ARM_THREAD_STATE_COUNT,
-      kret, count, r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9],
-      r[10], r[11], r[12], r[13], r[14], r[15], r[16]);
-  m_state.SetError(set, Read, kret);
-  return kret;
-}
-
-kern_return_t DNBArchMachARM::GetVFPState(bool force) {
-  int set = e_regSetVFP;
-  // Check if we have valid cached registers
-  if (!force && m_state.GetError(set, Read) == KERN_SUCCESS)
-    return KERN_SUCCESS;
-
-  kern_return_t kret;
-
-#if defined(__arm64__) || defined(__aarch64__)
-  // Read the registers from our thread
-  mach_msg_type_number_t count = ARM_NEON_STATE_COUNT;
-  kret = ::thread_get_state(m_thread->MachPortNumber(), ARM_NEON_STATE,
-                            (thread_state_t)&m_state.context.vfp, &count);
-  if (DNBLogEnabledForAny(LOG_THREAD)) {
-    DNBLogThreaded(
-        "thread_get_state(0x%4.4x, %u, &vfp, %u) => 0x%8.8x (count = %u) regs"
-        "\n   q0  = 0x%16.16llx%16.16llx"
-        "\n   q1  = 0x%16.16llx%16.16llx"
-        "\n   q2  = 0x%16.16llx%16.16llx"
-        "\n   q3  = 0x%16.16llx%16.16llx"
-        "\n   q4  = 0x%16.16llx%16.16llx"
-        "\n   q5  = 0x%16.16llx%16.16llx"
-        "\n   q6  = 0x%16.16llx%16.16llx"
-        "\n   q7  = 0x%16.16llx%16.16llx"
-        "\n   q8  = 0x%16.16llx%16.16llx"
-        "\n   q9  = 0x%16.16llx%16.16llx"
-        "\n   q10 = 0x%16.16llx%16.16llx"
-        "\n   q11 = 0x%16.16llx%16.16llx"
-        "\n   q12 = 0x%16.16llx%16.16llx"
-        "\n   q13 = 0x%16.16llx%16.16llx"
-        "\n   q14 = 0x%16.16llx%16.16llx"
-        "\n   q15 = 0x%16.16llx%16.16llx"
-        "\n  fpsr = 0x%8.8x"
-        "\n  fpcr = 0x%8.8x\n\n",
-        m_thread->MachPortNumber(), ARM_NEON_STATE, ARM_NEON_STATE_COUNT, kret,
-        count, ((uint64_t *)&m_state.context.vfp.__v[0])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[0])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[1])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[1])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[2])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[2])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[3])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[3])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[4])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[4])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[5])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[5])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[6])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[6])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[7])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[7])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[8])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[8])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[9])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[9])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[10])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[10])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[11])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[11])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[12])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[12])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[13])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[13])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[14])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[14])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[15])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[15])[1],
-        m_state.context.vfp.__fpsr, m_state.context.vfp.__fpcr);
-  }
-#else
-  // Read the registers from our thread
-  mach_msg_type_number_t count = ARM_VFP_STATE_COUNT;
-  kret = ::thread_get_state(m_thread->MachPortNumber(), ARM_VFP_STATE,
-                            (thread_state_t)&m_state.context.vfp, &count);
-
-  if (DNBLogEnabledForAny(LOG_THREAD)) {
-    uint32_t *r = &m_state.context.vfp.__r[0];
-    DNBLogThreaded(
-        "thread_get_state(0x%4.4x, %u, &gpr, %u) => 0x%8.8x (count => %u)",
-        m_thread->MachPortNumber(), ARM_THREAD_STATE, ARM_THREAD_STATE_COUNT,
-        kret, count);
-    DNBLogThreaded("   s0=%8.8x  s1=%8.8x  s2=%8.8x  s3=%8.8x  s4=%8.8x  "
-                   "s5=%8.8x  s6=%8.8x  s7=%8.8x",
-                   r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7]);
-    DNBLogThreaded("   s8=%8.8x  s9=%8.8x s10=%8.8x s11=%8.8x s12=%8.8x "
-                   "s13=%8.8x s14=%8.8x s15=%8.8x",
-                   r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15]);
-    DNBLogThreaded("  s16=%8.8x s17=%8.8x s18=%8.8x s19=%8.8x s20=%8.8x "
-                   "s21=%8.8x s22=%8.8x s23=%8.8x",
-                   r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23]);
-    DNBLogThreaded("  s24=%8.8x s25=%8.8x s26=%8.8x s27=%8.8x s28=%8.8x "
-                   "s29=%8.8x s30=%8.8x s31=%8.8x",
-                   r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31]);
-    DNBLogThreaded("  s32=%8.8x s33=%8.8x s34=%8.8x s35=%8.8x s36=%8.8x "
-                   "s37=%8.8x s38=%8.8x s39=%8.8x",
-                   r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39]);
-    DNBLogThreaded("  s40=%8.8x s41=%8.8x s42=%8.8x s43=%8.8x s44=%8.8x "
-                   "s45=%8.8x s46=%8.8x s47=%8.8x",
-                   r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47]);
-    DNBLogThreaded("  s48=%8.8x s49=%8.8x s50=%8.8x s51=%8.8x s52=%8.8x "
-                   "s53=%8.8x s54=%8.8x s55=%8.8x",
-                   r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55]);
-    DNBLogThreaded("  s56=%8.8x s57=%8.8x s58=%8.8x s59=%8.8x s60=%8.8x "
-                   "s61=%8.8x s62=%8.8x s63=%8.8x fpscr=%8.8x",
-                   r[56], r[57], r[58], r[59], r[60], r[61], r[62], r[63],
-                   r[64]);
-  }
-
-#endif
-  m_state.SetError(set, Read, kret);
-  return kret;
-}
-
-kern_return_t DNBArchMachARM::GetEXCState(bool force) {
-  int set = e_regSetEXC;
-  // Check if we have valid cached registers
-  if (!force && m_state.GetError(set, Read) == KERN_SUCCESS)
-    return KERN_SUCCESS;
-
-  // Read the registers from our thread
-  mach_msg_type_number_t count = ARM_EXCEPTION_STATE_COUNT;
-  kern_return_t kret =
-      ::thread_get_state(m_thread->MachPortNumber(), ARM_EXCEPTION_STATE,
-                         (thread_state_t)&m_state.context.exc, &count);
-  m_state.SetError(set, Read, kret);
-  return kret;
-}
-
-#if 0
-static void DumpDBGState(const DNBArchMachARM::DBG &dbg) {
-  uint32_t i = 0;
-  for (i = 0; i < 16; i++) {
-    DNBLogThreadedIf(LOG_STEP, "BVR%-2u/BCR%-2u = { 0x%8.8x, 0x%8.8x } "
-                               "WVR%-2u/WCR%-2u = { 0x%8.8x, 0x%8.8x }",
-                     i, i, dbg.__bvr[i], dbg.__bcr[i], i, i, dbg.__wvr[i],
-                     dbg.__wcr[i]);
-  }
-}
-#endif
-
-kern_return_t DNBArchMachARM::GetDBGState(bool force) {
-  int set = e_regSetDBG;
-
-  // Check if we have valid cached registers
-  if (!force && m_state.GetError(set, Read) == KERN_SUCCESS)
-    return KERN_SUCCESS;
-
-// Read the registers from our thread
-#if defined(ARM_DEBUG_STATE32) && (defined(__arm64__) || defined(__aarch64__))
-  mach_msg_type_number_t count = ARM_DEBUG_STATE32_COUNT;
-  kern_return_t kret =
-      ::thread_get_state(m_thread->MachPortNumber(), ARM_DEBUG_STATE32,
-                         (thread_state_t)&m_state.dbg, &count);
-#else
-  mach_msg_type_number_t count = ARM_DEBUG_STATE_COUNT;
-  kern_return_t kret =
-      ::thread_get_state(m_thread->MachPortNumber(), ARM_DEBUG_STATE,
-                         (thread_state_t)&m_state.dbg, &count);
-#endif
-  m_state.SetError(set, Read, kret);
-
-  return kret;
-}
-
-kern_return_t DNBArchMachARM::SetGPRState() {
-  int set = e_regSetGPR;
-  kern_return_t kret = ::thread_set_state(
-      m_thread->MachPortNumber(), ARM_THREAD_STATE,
-      (thread_state_t)&m_state.context.gpr, ARM_THREAD_STATE_COUNT);
-  m_state.SetError(set, Write,
-                   kret); // Set the current write error for this register set
-  m_state.InvalidateRegisterSetState(set); // Invalidate the current register
-                                           // state in case registers are read
-                                           // back differently
-  return kret;                             // Return the error code
-}
-
-kern_return_t DNBArchMachARM::SetVFPState() {
-  int set = e_regSetVFP;
-  kern_return_t kret;
-  mach_msg_type_number_t count;
-
-#if defined(__arm64__) || defined(__aarch64__)
-  count = ARM_NEON_STATE_COUNT;
-  kret = ::thread_set_state(m_thread->MachPortNumber(), ARM_NEON_STATE,
-                            (thread_state_t)&m_state.context.vfp, count);
-#else
-  count = ARM_VFP_STATE_COUNT;
-  kret = ::thread_set_state(m_thread->MachPortNumber(), ARM_VFP_STATE,
-                            (thread_state_t)&m_state.context.vfp, count);
-#endif
-
-#if defined(__arm64__) || defined(__aarch64__)
-  if (DNBLogEnabledForAny(LOG_THREAD)) {
-    DNBLogThreaded(
-        "thread_set_state(0x%4.4x, %u, &vfp, %u) => 0x%8.8x (count = %u) regs"
-        "\n   q0  = 0x%16.16llx%16.16llx"
-        "\n   q1  = 0x%16.16llx%16.16llx"
-        "\n   q2  = 0x%16.16llx%16.16llx"
-        "\n   q3  = 0x%16.16llx%16.16llx"
-        "\n   q4  = 0x%16.16llx%16.16llx"
-        "\n   q5  = 0x%16.16llx%16.16llx"
-        "\n   q6  = 0x%16.16llx%16.16llx"
-        "\n   q7  = 0x%16.16llx%16.16llx"
-        "\n   q8  = 0x%16.16llx%16.16llx"
-        "\n   q9  = 0x%16.16llx%16.16llx"
-        "\n   q10 = 0x%16.16llx%16.16llx"
-        "\n   q11 = 0x%16.16llx%16.16llx"
-        "\n   q12 = 0x%16.16llx%16.16llx"
-        "\n   q13 = 0x%16.16llx%16.16llx"
-        "\n   q14 = 0x%16.16llx%16.16llx"
-        "\n   q15 = 0x%16.16llx%16.16llx"
-        "\n  fpsr = 0x%8.8x"
-        "\n  fpcr = 0x%8.8x\n\n",
-        m_thread->MachPortNumber(), ARM_NEON_STATE, ARM_NEON_STATE_COUNT, kret,
-        count, ((uint64_t *)&m_state.context.vfp.__v[0])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[0])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[1])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[1])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[2])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[2])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[3])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[3])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[4])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[4])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[5])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[5])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[6])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[6])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[7])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[7])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[8])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[8])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[9])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[9])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[10])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[10])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[11])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[11])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[12])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[12])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[13])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[13])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[14])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[14])[1],
-        ((uint64_t *)&m_state.context.vfp.__v[15])[0],
-        ((uint64_t *)&m_state.context.vfp.__v[15])[1],
-        m_state.context.vfp.__fpsr, m_state.context.vfp.__fpcr);
-  }
-#else
-  if (DNBLogEnabledForAny(LOG_THREAD)) {
-    uint32_t *r = &m_state.context.vfp.__r[0];
-    DNBLogThreaded(
-        "thread_get_state(0x%4.4x, %u, &gpr, %u) => 0x%8.8x (count => %u)",
-        m_thread->MachPortNumber(), ARM_THREAD_STATE, ARM_THREAD_STATE_COUNT,
-        kret, count);
-    DNBLogThreaded("   s0=%8.8x  s1=%8.8x  s2=%8.8x  s3=%8.8x  s4=%8.8x  "
-                   "s5=%8.8x  s6=%8.8x  s7=%8.8x",
-                   r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7]);
-    DNBLogThreaded("   s8=%8.8x  s9=%8.8x s10=%8.8x s11=%8.8x s12=%8.8x "
-                   "s13=%8.8x s14=%8.8x s15=%8.8x",
-                   r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15]);
-    DNBLogThreaded("  s16=%8.8x s17=%8.8x s18=%8.8x s19=%8.8x s20=%8.8x "
-                   "s21=%8.8x s22=%8.8x s23=%8.8x",
-                   r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23]);
-    DNBLogThreaded("  s24=%8.8x s25=%8.8x s26=%8.8x s27=%8.8x s28=%8.8x "
-                   "s29=%8.8x s30=%8.8x s31=%8.8x",
-                   r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31]);
-    DNBLogThreaded("  s32=%8.8x s33=%8.8x s34=%8.8x s35=%8.8x s36=%8.8x "
-                   "s37=%8.8x s38=%8.8x s39=%8.8x",
-                   r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39]);
-    DNBLogThreaded("  s40=%8.8x s41=%8.8x s42=%8.8x s43=%8.8x s44=%8.8x "
-                   "s45=%8.8x s46=%8.8x s47=%8.8x",
-                   r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47]);
-    DNBLogThreaded("  s48=%8.8x s49=%8.8x s50=%8.8x s51=%8.8x s52=%8.8x "
-                   "s53=%8.8x s54=%8.8x s55=%8.8x",
-                   r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55]);
-    DNBLogThreaded("  s56=%8.8x s57=%8.8x s58=%8.8x s59=%8.8x s60=%8.8x "
-                   "s61=%8.8x s62=%8.8x s63=%8.8x fpscr=%8.8x",
-                   r[56], r[57], r[58], r[59], r[60], r[61], r[62], r[63],
-                   r[64]);
-  }
-#endif
-
-  m_state.SetError(set, Write,
-                   kret); // Set the current write error for this register set
-  m_state.InvalidateRegisterSetState(set); // Invalidate the current register
-                                           // state in case registers are read
-                                           // back differently
-  return kret;                             // Return the error code
-}
-
-kern_return_t DNBArchMachARM::SetEXCState() {
-  int set = e_regSetEXC;
-  kern_return_t kret = ::thread_set_state(
-      m_thread->MachPortNumber(), ARM_EXCEPTION_STATE,
-      (thread_state_t)&m_state.context.exc, ARM_EXCEPTION_STATE_COUNT);
-  m_state.SetError(set, Write,
-                   kret); // Set the current write error for this register set
-  m_state.InvalidateRegisterSetState(set); // Invalidate the current register
-                                           // state in case registers are read
-                                           // back differently
-  return kret;                             // Return the error code
-}
-
-kern_return_t DNBArchMachARM::SetDBGState(bool also_set_on_task) {
-  int set = e_regSetDBG;
-#if defined(ARM_DEBUG_STATE32) && (defined(__arm64__) || defined(__aarch64__))
-  kern_return_t kret =
-      ::thread_set_state(m_thread->MachPortNumber(), ARM_DEBUG_STATE32,
-                         (thread_state_t)&m_state.dbg, ARM_DEBUG_STATE32_COUNT);
-  if (also_set_on_task) {
-    kern_return_t task_kret = ::task_set_state(
-        m_thread->Process()->Task().TaskPort(), ARM_DEBUG_STATE32,
-        (thread_state_t)&m_state.dbg, ARM_DEBUG_STATE32_COUNT);
-    if (task_kret != KERN_SUCCESS)
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::SetDBGState failed to "
-                                        "set debug control register state: "
-                                        "0x%8.8x.",
-                       kret);
-  }
-#else
-  kern_return_t kret =
-      ::thread_set_state(m_thread->MachPortNumber(), ARM_DEBUG_STATE,
-                         (thread_state_t)&m_state.dbg, ARM_DEBUG_STATE_COUNT);
-  if (also_set_on_task) {
-    kern_return_t task_kret = ::task_set_state(
-        m_thread->Process()->Task().TaskPort(), ARM_DEBUG_STATE,
-        (thread_state_t)&m_state.dbg, ARM_DEBUG_STATE_COUNT);
-    if (task_kret != KERN_SUCCESS)
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::SetDBGState failed to "
-                                        "set debug control register state: "
-                                        "0x%8.8x.",
-                       kret);
-  }
-#endif
-
-  m_state.SetError(set, Write,
-                   kret); // Set the current write error for this register set
-  m_state.InvalidateRegisterSetState(set); // Invalidate the current register
-                                           // state in case registers are read
-                                           // back differently
-  return kret;                             // Return the error code
-}
-
-void DNBArchMachARM::ThreadWillResume() {
-  // Do we need to step this thread? If so, let the mach thread tell us so.
-  if (m_thread->IsStepping()) {
-    // This is the primary thread, let the arch do anything it needs
-    if (NumSupportedHardwareBreakpoints() > 0) {
-      if (EnableHardwareSingleStep(true) != KERN_SUCCESS) {
-        DNBLogThreaded("DNBArchMachARM::ThreadWillResume() failed to enable "
-                       "hardware single step");
-      }
-    }
-  }
-
-  // Disable the triggered watchpoint temporarily before we resume.
-  // Plus, we try to enable hardware single step to execute past the instruction
-  // which triggered our watchpoint.
-  if (m_watchpoint_did_occur) {
-    if (m_watchpoint_hw_index >= 0) {
-      kern_return_t kret = GetDBGState(false);
-      if (kret == KERN_SUCCESS &&
-          !IsWatchpointEnabled(m_state.dbg, m_watchpoint_hw_index)) {
-        // The watchpoint might have been disabled by the user.  We don't need
-        // to do anything at all
-        // to enable hardware single stepping.
-        m_watchpoint_did_occur = false;
-        m_watchpoint_hw_index = -1;
-        return;
-      }
-
-      DisableHardwareWatchpoint(m_watchpoint_hw_index, false);
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::ThreadWillResume() "
-                                        "DisableHardwareWatchpoint(%d) called",
-                       m_watchpoint_hw_index);
-
-      // Enable hardware single step to move past the watchpoint-triggering
-      // instruction.
-      m_watchpoint_resume_single_step_enabled =
-          (EnableHardwareSingleStep(true) == KERN_SUCCESS);
-
-      // If we are not able to enable single step to move past the
-      // watchpoint-triggering instruction,
-      // at least we should reset the two watchpoint member variables so that
-      // the next time around
-      // this callback function is invoked, the enclosing logical branch is
-      // skipped.
-      if (!m_watchpoint_resume_single_step_enabled) {
-        // Reset the two watchpoint member variables.
-        m_watchpoint_did_occur = false;
-        m_watchpoint_hw_index = -1;
-        DNBLogThreadedIf(
-            LOG_WATCHPOINTS,
-            "DNBArchMachARM::ThreadWillResume() failed to enable single step");
-      } else
-        DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::ThreadWillResume() "
-                                          "succeeded to enable single step");
-    }
-  }
-}
-
-bool DNBArchMachARM::ThreadDidStop() {
-  bool success = true;
-
-  m_state.InvalidateRegisterSetState(e_regSetALL);
-
-  if (m_watchpoint_resume_single_step_enabled) {
-    // Great!  We now disable the hardware single step as well as re-enable the
-    // hardware watchpoint.
-    // See also ThreadWillResume().
-    if (EnableHardwareSingleStep(false) == KERN_SUCCESS) {
-      if (m_watchpoint_did_occur && m_watchpoint_hw_index >= 0) {
-        ReenableHardwareWatchpoint(m_watchpoint_hw_index);
-        m_watchpoint_resume_single_step_enabled = false;
-        m_watchpoint_did_occur = false;
-        m_watchpoint_hw_index = -1;
-      } else {
-        DNBLogError("internal error detected: m_watchpoint_resume_step_enabled "
-                    "is true but (m_watchpoint_did_occur && "
-                    "m_watchpoint_hw_index >= 0) does not hold!");
-      }
-    } else {
-      DNBLogError("internal error detected: m_watchpoint_resume_step_enabled "
-                  "is true but unable to disable single step!");
-    }
-  }
-
-  // Are we stepping a single instruction?
-  if (GetGPRState(true) == KERN_SUCCESS) {
-    // We are single stepping, was this the primary thread?
-    if (m_thread->IsStepping()) {
-      success = EnableHardwareSingleStep(false) == KERN_SUCCESS;
-    } else {
-      // The MachThread will automatically restore the suspend count
-      // in ThreadDidStop(), so we don't need to do anything here if
-      // we weren't the primary thread the last time
-    }
-  }
-  return success;
-}
-
-bool DNBArchMachARM::NotifyException(MachException::Data &exc) {
-  switch (exc.exc_type) {
-  default:
-    break;
-  case EXC_BREAKPOINT:
-    if (exc.exc_data.size() == 2 && exc.exc_data[0] == EXC_ARM_DA_DEBUG) {
-      // The data break address is passed as exc_data[1].
-      nub_addr_t addr = exc.exc_data[1];
-      // Find the hardware index with the side effect of possibly massaging the
-      // addr to return the starting address as seen from the debugger side.
-      uint32_t hw_index = GetHardwareWatchpointHit(addr);
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::NotifyException "
-                                        "watchpoint %d was hit on address "
-                                        "0x%llx",
-                       hw_index, (uint64_t)addr);
-      const uint32_t num_watchpoints = NumSupportedHardwareWatchpoints();
-      for (uint32_t i = 0; i < num_watchpoints; i++) {
-        if (LoHi[i] != 0 && LoHi[i] == hw_index && LoHi[i] != i &&
-            GetWatchpointAddressByIndex(i) != INVALID_NUB_ADDRESS) {
-          addr = GetWatchpointAddressByIndex(i);
-          DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::NotifyException "
-                                            "It is a linked watchpoint; "
-                                            "rewritten to index %d addr 0x%llx",
-                           LoHi[i], (uint64_t)addr);
-        }
-      }
-      if (hw_index != INVALID_NUB_HW_INDEX) {
-        m_watchpoint_did_occur = true;
-        m_watchpoint_hw_index = hw_index;
-        exc.exc_data[1] = addr;
-        // Piggyback the hw_index in the exc.data.
-        exc.exc_data.push_back(hw_index);
-      }
-
-      return true;
-    }
-    break;
-  }
-  return false;
-}
-
-bool DNBArchMachARM::StepNotComplete() {
-  if (m_hw_single_chained_step_addr != INVALID_NUB_ADDRESS) {
-    kern_return_t kret = KERN_INVALID_ARGUMENT;
-    kret = GetGPRState(false);
-    if (kret == KERN_SUCCESS) {
-      if (m_state.context.gpr.__pc == m_hw_single_chained_step_addr) {
-        DNBLogThreadedIf(LOG_STEP, "Need to step some more at 0x%8.8llx",
-                         (uint64_t)m_hw_single_chained_step_addr);
-        return true;
-      }
-    }
-  }
-
-  m_hw_single_chained_step_addr = INVALID_NUB_ADDRESS;
-  return false;
-}
-
-// Set the single step bit in the processor status register.
-kern_return_t DNBArchMachARM::EnableHardwareSingleStep(bool enable) {
-  DNBError err;
-  DNBLogThreadedIf(LOG_STEP, "%s( enable = %d )", __FUNCTION__, enable);
-
-  err = GetGPRState(false);
-
-  if (err.Fail()) {
-    err.LogThreaded("%s: failed to read the GPR registers", __FUNCTION__);
-    return err.Status();
-  }
-
-  err = GetDBGState(false);
-
-  if (err.Fail()) {
-    err.LogThreaded("%s: failed to read the DBG registers", __FUNCTION__);
-    return err.Status();
-  }
-
-// The use of __arm64__ here is not ideal.  If debugserver is running on
-// an armv8 device, regardless of whether it was built for arch arm or arch
-// arm64,
-// it needs to use the MDSCR_EL1 SS bit to single instruction step.
-
-#if defined(__arm64__) || defined(__aarch64__)
-  if (enable) {
-    DNBLogThreadedIf(LOG_STEP,
-                     "%s: Setting MDSCR_EL1 Single Step bit at pc 0x%llx",
-                     __FUNCTION__, (uint64_t)m_state.context.gpr.__pc);
-    m_state.dbg.__mdscr_el1 |=
-        1; // Set bit 0 (single step, SS) in the MDSCR_EL1.
-  } else {
-    DNBLogThreadedIf(LOG_STEP,
-                     "%s: Clearing MDSCR_EL1 Single Step bit at pc 0x%llx",
-                     __FUNCTION__, (uint64_t)m_state.context.gpr.__pc);
-    m_state.dbg.__mdscr_el1 &=
-        ~(1ULL); // Clear bit 0 (single step, SS) in the MDSCR_EL1.
-  }
-#else
-  const uint32_t i = 0;
-  if (enable) {
-    m_hw_single_chained_step_addr = INVALID_NUB_ADDRESS;
-
-    // Save our previous state
-    m_dbg_save = m_state.dbg;
-    // Set a breakpoint that will stop when the PC doesn't match the current
-    // one!
-    m_state.dbg.__bvr[i] =
-        m_state.context.gpr.__pc &
-        0xFFFFFFFCu; // Set the current PC as the breakpoint address
-    m_state.dbg.__bcr[i] = BCR_M_IMVA_MISMATCH | // Stop on address mismatch
-                           S_USER |              // Stop only in user mode
-                           BCR_ENABLE;           // Enable this breakpoint
-    if (m_state.context.gpr.__cpsr & 0x20) {
-      // Thumb breakpoint
-      if (m_state.context.gpr.__pc & 2)
-        m_state.dbg.__bcr[i] |= BAS_IMVA_2_3;
-      else
-        m_state.dbg.__bcr[i] |= BAS_IMVA_0_1;
-
-      uint16_t opcode;
-      if (sizeof(opcode) ==
-          m_thread->Process()->Task().ReadMemory(m_state.context.gpr.__pc,
-                                                 sizeof(opcode), &opcode)) {
-        if (IsThumb32Opcode(opcode)) {
-          // 32 bit thumb opcode...
-          if (m_state.context.gpr.__pc & 2) {
-            // We can't take care of a 32 bit thumb instruction single step
-            // with just IVA mismatching. We will need to chain an extra
-            // hardware single step in order to complete this single step...
-            m_hw_single_chained_step_addr = m_state.context.gpr.__pc + 2;
-          } else {
-            // Extend the number of bits to ignore for the mismatch
-            m_state.dbg.__bcr[i] |= BAS_IMVA_ALL;
-          }
-        }
-      }
-    } else {
-      // ARM breakpoint
-      m_state.dbg.__bcr[i] |= BAS_IMVA_ALL; // Stop when any address bits change
-    }
-
-    DNBLogThreadedIf(LOG_STEP, "%s: BVR%u=0x%8.8x  BCR%u=0x%8.8x", __FUNCTION__,
-                     i, m_state.dbg.__bvr[i], i, m_state.dbg.__bcr[i]);
-
-    for (uint32_t j = i + 1; j < 16; ++j) {
-      // Disable all others
-      m_state.dbg.__bvr[j] = 0;
-      m_state.dbg.__bcr[j] = 0;
-    }
-  } else {
-    // Just restore the state we had before we did single stepping
-    m_state.dbg = m_dbg_save;
-  }
-#endif
-
-  return SetDBGState(false);
-}
-
-// return 1 if bit "BIT" is set in "value"
-static inline uint32_t bit(uint32_t value, uint32_t bit) {
-  return (value >> bit) & 1u;
-}
-
-// return the bitfield "value[msbit:lsbit]".
-static inline uint32_t bits(uint32_t value, uint32_t msbit, uint32_t lsbit) {
-  assert(msbit >= lsbit);
-  uint32_t shift_left = sizeof(value) * 8 - 1 - msbit;
-  value <<=
-      shift_left; // shift anything above the msbit off of the unsigned edge
-  value >>= (shift_left + lsbit); // shift it back again down to the lsbit
-                                  // (including undoing any shift from above)
-  return value;                   // return our result
-}
-
-bool DNBArchMachARM::ConditionPassed(uint8_t condition, uint32_t cpsr) {
-  uint32_t cpsr_n = bit(cpsr, 31); // Negative condition code flag
-  uint32_t cpsr_z = bit(cpsr, 30); // Zero condition code flag
-  uint32_t cpsr_c = bit(cpsr, 29); // Carry condition code flag
-  uint32_t cpsr_v = bit(cpsr, 28); // Overflow condition code flag
-
-  switch (condition) {
-  case COND_EQ: // (0x0)
-    if (cpsr_z == 1)
-      return true;
-    break;
-  case COND_NE: // (0x1)
-    if (cpsr_z == 0)
-      return true;
-    break;
-  case COND_CS: // (0x2)
-    if (cpsr_c == 1)
-      return true;
-    break;
-  case COND_CC: // (0x3)
-    if (cpsr_c == 0)
-      return true;
-    break;
-  case COND_MI: // (0x4)
-    if (cpsr_n == 1)
-      return true;
-    break;
-  case COND_PL: // (0x5)
-    if (cpsr_n == 0)
-      return true;
-    break;
-  case COND_VS: // (0x6)
-    if (cpsr_v == 1)
-      return true;
-    break;
-  case COND_VC: // (0x7)
-    if (cpsr_v == 0)
-      return true;
-    break;
-  case COND_HI: // (0x8)
-    if ((cpsr_c == 1) && (cpsr_z == 0))
-      return true;
-    break;
-  case COND_LS: // (0x9)
-    if ((cpsr_c == 0) || (cpsr_z == 1))
-      return true;
-    break;
-  case COND_GE: // (0xA)
-    if (cpsr_n == cpsr_v)
-      return true;
-    break;
-  case COND_LT: // (0xB)
-    if (cpsr_n != cpsr_v)
-      return true;
-    break;
-  case COND_GT: // (0xC)
-    if ((cpsr_z == 0) && (cpsr_n == cpsr_v))
-      return true;
-    break;
-  case COND_LE: // (0xD)
-    if ((cpsr_z == 1) || (cpsr_n != cpsr_v))
-      return true;
-    break;
-  default:
-    return true;
-    break;
-  }
-
-  return false;
-}
-
-uint32_t DNBArchMachARM::NumSupportedHardwareBreakpoints() {
-  // Set the init value to something that will let us know that we need to
-  // autodetect how many breakpoints are supported dynamically...
-  static uint32_t g_num_supported_hw_breakpoints = UINT_MAX;
-  if (g_num_supported_hw_breakpoints == UINT_MAX) {
-    // Set this to zero in case we can't tell if there are any HW breakpoints
-    g_num_supported_hw_breakpoints = 0;
-
-    size_t len;
-    uint32_t n = 0;
-    len = sizeof(n);
-    if (::sysctlbyname("hw.optional.breakpoint", &n, &len, NULL, 0) == 0) {
-      g_num_supported_hw_breakpoints = n;
-      DNBLogThreadedIf(LOG_THREAD, "hw.optional.breakpoint=%u", n);
-    } else {
-#if !defined(__arm64__) && !defined(__aarch64__)
-      // Read the DBGDIDR to get the number of available hardware breakpoints
-      // However, in some of our current armv7 processors, hardware
-      // breakpoints/watchpoints were not properly connected. So detect those
-      // cases using a field in a sysctl. For now we are using "hw.cpusubtype"
-      // field to distinguish CPU architectures. This is a hack until we can
-      // get <rdar://problem/6372672> fixed, at which point we will switch to
-      // using a different sysctl string that will tell us how many BRPs
-      // are available to us directly without having to read DBGDIDR.
-      uint32_t register_DBGDIDR;
-
-      asm("mrc p14, 0, %0, c0, c0, 0" : "=r"(register_DBGDIDR));
-      uint32_t numBRPs = bits(register_DBGDIDR, 27, 24);
-      // Zero is reserved for the BRP count, so don't increment it if it is zero
-      if (numBRPs > 0)
-        numBRPs++;
-      DNBLogThreadedIf(LOG_THREAD, "DBGDIDR=0x%8.8x (number BRP pairs = %u)",
-                       register_DBGDIDR, numBRPs);
-
-      if (numBRPs > 0) {
-        uint32_t cpusubtype;
-        len = sizeof(cpusubtype);
-        // TODO: remove this hack and change to using hw.optional.xx when
-        // implmented
-        if (::sysctlbyname("hw.cpusubtype", &cpusubtype, &len, NULL, 0) == 0) {
-          DNBLogThreadedIf(LOG_THREAD, "hw.cpusubtype=%d", cpusubtype);
-          if (cpusubtype == CPU_SUBTYPE_ARM_V7)
-            DNBLogThreadedIf(LOG_THREAD, "Hardware breakpoints disabled for "
-                                         "armv7 (rdar://problem/6372672)");
-          else
-            g_num_supported_hw_breakpoints = numBRPs;
-        }
-      }
-#endif
-    }
-  }
-  return g_num_supported_hw_breakpoints;
-}
-
-uint32_t DNBArchMachARM::NumSupportedHardwareWatchpoints() {
-  // Set the init value to something that will let us know that we need to
-  // autodetect how many watchpoints are supported dynamically...
-  static uint32_t g_num_supported_hw_watchpoints = UINT_MAX;
-  if (g_num_supported_hw_watchpoints == UINT_MAX) {
-    // Set this to zero in case we can't tell if there are any HW breakpoints
-    g_num_supported_hw_watchpoints = 0;
-
-    size_t len;
-    uint32_t n = 0;
-    len = sizeof(n);
-    if (::sysctlbyname("hw.optional.watchpoint", &n, &len, NULL, 0) == 0) {
-      g_num_supported_hw_watchpoints = n;
-      DNBLogThreadedIf(LOG_THREAD, "hw.optional.watchpoint=%u", n);
-    } else {
-#if !defined(__arm64__) && !defined(__aarch64__)
-      // Read the DBGDIDR to get the number of available hardware breakpoints
-      // However, in some of our current armv7 processors, hardware
-      // breakpoints/watchpoints were not properly connected. So detect those
-      // cases using a field in a sysctl. For now we are using "hw.cpusubtype"
-      // field to distinguish CPU architectures. This is a hack until we can
-      // get <rdar://problem/6372672> fixed, at which point we will switch to
-      // using a different sysctl string that will tell us how many WRPs
-      // are available to us directly without having to read DBGDIDR.
-
-      uint32_t register_DBGDIDR;
-      asm("mrc p14, 0, %0, c0, c0, 0" : "=r"(register_DBGDIDR));
-      uint32_t numWRPs = bits(register_DBGDIDR, 31, 28) + 1;
-      DNBLogThreadedIf(LOG_THREAD, "DBGDIDR=0x%8.8x (number WRP pairs = %u)",
-                       register_DBGDIDR, numWRPs);
-
-      if (numWRPs > 0) {
-        uint32_t cpusubtype;
-        size_t len;
-        len = sizeof(cpusubtype);
-        // TODO: remove this hack and change to using hw.optional.xx when
-        // implmented
-        if (::sysctlbyname("hw.cpusubtype", &cpusubtype, &len, NULL, 0) == 0) {
-          DNBLogThreadedIf(LOG_THREAD, "hw.cpusubtype=0x%d", cpusubtype);
-
-          if (cpusubtype == CPU_SUBTYPE_ARM_V7)
-            DNBLogThreadedIf(LOG_THREAD, "Hardware watchpoints disabled for "
-                                         "armv7 (rdar://problem/6372672)");
-          else
-            g_num_supported_hw_watchpoints = numWRPs;
-        }
-      }
-#endif
-    }
-  }
-  return g_num_supported_hw_watchpoints;
-}
-
-uint32_t DNBArchMachARM::EnableHardwareBreakpoint(nub_addr_t addr,
-                                                  nub_size_t size,
-                                                  bool also_set_on_task) {
-  // Make sure our address isn't bogus
-  if (addr & 1)
-    return INVALID_NUB_HW_INDEX;
-
-  kern_return_t kret = GetDBGState(false);
-
-  if (kret == KERN_SUCCESS) {
-    const uint32_t num_hw_breakpoints = NumSupportedHardwareBreakpoints();
-    uint32_t i;
-    for (i = 0; i < num_hw_breakpoints; ++i) {
-      if ((m_state.dbg.__bcr[i] & BCR_ENABLE) == 0)
-        break; // We found an available hw breakpoint slot (in i)
-    }
-
-    // See if we found an available hw breakpoint slot above
-    if (i < num_hw_breakpoints) {
-      // Make sure bits 1:0 are clear in our address
-      m_state.dbg.__bvr[i] = addr & ~((nub_addr_t)3);
-
-      if (size == 2 || addr & 2) {
-        uint32_t byte_addr_select = (addr & 2) ? BAS_IMVA_2_3 : BAS_IMVA_0_1;
-
-        // We have a thumb breakpoint
-        // We have an ARM breakpoint
-        m_state.dbg.__bcr[i] =
-            BCR_M_IMVA_MATCH | // Stop on address match
-            byte_addr_select | // Set the correct byte address select so we only
-                               // trigger on the correct opcode
-            S_USER |           // Which modes should this breakpoint stop in?
-            BCR_ENABLE;        // Enable this hardware breakpoint
-        DNBLogThreadedIf(LOG_BREAKPOINTS,
-                         "DNBArchMachARM::EnableHardwareBreakpoint( addr = "
-                         "0x%8.8llx, size = %llu ) - BVR%u/BCR%u = 0x%8.8x / "
-                         "0x%8.8x (Thumb)",
-                         (uint64_t)addr, (uint64_t)size, i, i,
-                         m_state.dbg.__bvr[i], m_state.dbg.__bcr[i]);
-      } else if (size == 4) {
-        // We have an ARM breakpoint
-        m_state.dbg.__bcr[i] =
-            BCR_M_IMVA_MATCH | // Stop on address match
-            BAS_IMVA_ALL | // Stop on any of the four bytes following the IMVA
-            S_USER |       // Which modes should this breakpoint stop in?
-            BCR_ENABLE;    // Enable this hardware breakpoint
-        DNBLogThreadedIf(LOG_BREAKPOINTS,
-                         "DNBArchMachARM::EnableHardwareBreakpoint( addr = "
-                         "0x%8.8llx, size = %llu ) - BVR%u/BCR%u = 0x%8.8x / "
-                         "0x%8.8x (ARM)",
-                         (uint64_t)addr, (uint64_t)size, i, i,
-                         m_state.dbg.__bvr[i], m_state.dbg.__bcr[i]);
-      }
-
-      kret = SetDBGState(false);
-      DNBLogThreadedIf(LOG_BREAKPOINTS, "DNBArchMachARM::"
-                                        "EnableHardwareBreakpoint() "
-                                        "SetDBGState() => 0x%8.8x.",
-                       kret);
-
-      if (kret == KERN_SUCCESS)
-        return i;
-    } else {
-      DNBLogThreadedIf(LOG_BREAKPOINTS,
-                       "DNBArchMachARM::EnableHardwareBreakpoint(addr = "
-                       "0x%8.8llx, size = %llu) => all hardware breakpoint "
-                       "resources are being used.",
-                       (uint64_t)addr, (uint64_t)size);
-    }
-  }
-
-  return INVALID_NUB_HW_INDEX;
-}
-
-bool DNBArchMachARM::DisableHardwareBreakpoint(uint32_t hw_index,
-                                               bool also_set_on_task) {
-  kern_return_t kret = GetDBGState(false);
-
-  const uint32_t num_hw_points = NumSupportedHardwareBreakpoints();
-  if (kret == KERN_SUCCESS) {
-    if (hw_index < num_hw_points) {
-      m_state.dbg.__bcr[hw_index] = 0;
-      DNBLogThreadedIf(LOG_BREAKPOINTS, "DNBArchMachARM::SetHardwareBreakpoint("
-                                        " %u ) - BVR%u = 0x%8.8x  BCR%u = "
-                                        "0x%8.8x",
-                       hw_index, hw_index, m_state.dbg.__bvr[hw_index],
-                       hw_index, m_state.dbg.__bcr[hw_index]);
-
-      kret = SetDBGState(false);
-
-      if (kret == KERN_SUCCESS)
-        return true;
-    }
-  }
-  return false;
-}
-
-// ARM v7 watchpoints may be either word-size or double-word-size.
-// It's implementation defined which they can handle.  It looks like on an
-// armv8 device, armv7 processes can watch dwords.  But on a genuine armv7
-// device I tried, only word watchpoints are supported.
-
-#if defined(__arm64__) || defined(__aarch64__)
-#define WATCHPOINTS_ARE_DWORD 1
-#else
-#undef WATCHPOINTS_ARE_DWORD
-#endif
-
-uint32_t DNBArchMachARM::EnableHardwareWatchpoint(nub_addr_t addr,
-                                                  nub_size_t size, bool read,
-                                                  bool write,
-                                                  bool also_set_on_task) {
-
-  DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::EnableHardwareWatchpoint("
-                                    "addr = 0x%8.8llx, size = %zu, read = %u, "
-                                    "write = %u)",
-                   (uint64_t)addr, size, read, write);
-
-  const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints();
-
-  // Can't watch zero bytes
-  if (size == 0)
-    return INVALID_NUB_HW_INDEX;
-
-  // We must watch for either read or write
-  if (read == false && write == false)
-    return INVALID_NUB_HW_INDEX;
-
-  // Otherwise, can't watch more than 8 bytes per WVR/WCR pair
-  if (size > 8)
-    return INVALID_NUB_HW_INDEX;
-
-// Treat arm watchpoints as having an 8-byte alignment requirement.  You can put
-// a watchpoint on a 4-byte
-// offset address but you can only watch 4 bytes with that watchpoint.
-
-// arm watchpoints on an 8-byte (double word) aligned addr can watch any bytes
-// in that
-// 8-byte long region of memory.  They can watch the 1st byte, the 2nd byte, 3rd
-// byte, etc, or any
-// combination therein by setting the bits in the BAS [12:5] (Byte Address
-// Select) field of
-// the DBGWCRn_EL1 reg for the watchpoint.
-
-// If the MASK [28:24] bits in the DBGWCRn_EL1 allow a single watchpoint to
-// monitor a larger region
-// of memory (16 bytes, 32 bytes, or 2GB) but the Byte Address Select bitfield
-// then selects a larger
-// range of bytes, instead of individual bytes.  See the ARMv8 Debug
-// Architecture manual for details.
-// This implementation does not currently use the MASK bits; the largest single
-// region watched by a single
-// watchpoint right now is 8-bytes.
-
-#if defined(WATCHPOINTS_ARE_DWORD)
-  nub_addr_t aligned_wp_address = addr & ~0x7;
-  uint32_t addr_dword_offset = addr & 0x7;
-  const int max_watchpoint_size = 8;
-#else
-  nub_addr_t aligned_wp_address = addr & ~0x3;
-  uint32_t addr_dword_offset = addr & 0x3;
-  const int max_watchpoint_size = 4;
-#endif
-
-  DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::EnableHardwareWatchpoint "
-                                    "aligned_wp_address is 0x%llx and "
-                                    "addr_dword_offset is 0x%x",
-                   (uint64_t)aligned_wp_address, addr_dword_offset);
-
-  // Do we need to split up this logical watchpoint into two hardware watchpoint
-  // registers?
-  // e.g. a watchpoint of length 4 on address 6.  We need do this with
-  //   one watchpoint on address 0 with bytes 6 & 7 being monitored
-  //   one watchpoint on address 8 with bytes 0, 1, 2, 3 being monitored
-
-  if (addr_dword_offset + size > max_watchpoint_size) {
-    DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::"
-                                      "EnableHardwareWatchpoint(addr = "
-                                      "0x%8.8llx, size = %zu) needs two "
-                                      "hardware watchpoints slots to monitor",
-                     (uint64_t)addr, size);
-    int low_watchpoint_size = max_watchpoint_size - addr_dword_offset;
-    int high_watchpoint_size = addr_dword_offset + size - max_watchpoint_size;
-
-    uint32_t lo = EnableHardwareWatchpoint(addr, low_watchpoint_size, read,
-                                           write, also_set_on_task);
-    if (lo == INVALID_NUB_HW_INDEX)
-      return INVALID_NUB_HW_INDEX;
-    uint32_t hi = EnableHardwareWatchpoint(
-        aligned_wp_address + max_watchpoint_size, high_watchpoint_size, read,
-        write, also_set_on_task);
-    if (hi == INVALID_NUB_HW_INDEX) {
-      DisableHardwareWatchpoint(lo, also_set_on_task);
-      return INVALID_NUB_HW_INDEX;
-    }
-    // Tag this lo->hi mapping in our database.
-    LoHi[lo] = hi;
-    return lo;
-  }
-
-  // At this point
-  //  1 aligned_wp_address is the requested address rounded down to 8-byte
-  //  alignment
-  //  2 addr_dword_offset is the offset into that double word (8-byte) region
-  //  that we are watching
-  //  3 size is the number of bytes within that 8-byte region that we are
-  //  watching
-
-  // Set the Byte Address Selects bits DBGWCRn_EL1 bits [12:5] based on the
-  // above.
-  // The bit shift and negation operation will give us 0b11 for 2, 0b1111 for 4,
-  // etc, up to 0b11111111 for 8.
-  // then we shift those bits left by the offset into this dword that we are
-  // interested in.
-  // e.g. if we are watching bytes 4,5,6,7 in a dword we want a BAS of
-  // 0b11110000.
-  uint32_t byte_address_select = ((1 << size) - 1) << addr_dword_offset;
-
-  // Read the debug state
-  kern_return_t kret = GetDBGState(true);
-
-  if (kret == KERN_SUCCESS) {
-    // Check to make sure we have the needed hardware support
-    uint32_t i = 0;
-
-    for (i = 0; i < num_hw_watchpoints; ++i) {
-      if ((m_state.dbg.__wcr[i] & WCR_ENABLE) == 0)
-        break; // We found an available hw watchpoint slot (in i)
-    }
-
-    // See if we found an available hw watchpoint slot above
-    if (i < num_hw_watchpoints) {
-      // DumpDBGState(m_state.dbg);
-
-      // Clear any previous LoHi joined-watchpoint that may have been in use
-      LoHi[i] = 0;
-
-      // shift our Byte Address Select bits up to the correct bit range for the
-      // DBGWCRn_EL1
-      byte_address_select = byte_address_select << 5;
-
-      // Make sure bits 1:0 are clear in our address
-      m_state.dbg.__wvr[i] = aligned_wp_address;   // DVA (Data Virtual Address)
-      m_state.dbg.__wcr[i] = byte_address_select | // Which bytes that follow
-                                                   // the DVA that we will watch
-                             S_USER |              // Stop only in user mode
-                             (read ? WCR_LOAD : 0) |   // Stop on read access?
-                             (write ? WCR_STORE : 0) | // Stop on write access?
-                             WCR_ENABLE; // Enable this watchpoint;
-
-      DNBLogThreadedIf(
-          LOG_WATCHPOINTS, "DNBArchMachARM::EnableHardwareWatchpoint() adding "
-                           "watchpoint on address 0x%llx with control register "
-                           "value 0x%x",
-          (uint64_t)m_state.dbg.__wvr[i], (uint32_t)m_state.dbg.__wcr[i]);
-
-      // The kernel will set the MDE_ENABLE bit in the MDSCR_EL1 for us
-      // automatically, don't need to do it here.
-
-      kret = SetDBGState(also_set_on_task);
-      // DumpDBGState(m_state.dbg);
-
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::"
-                                        "EnableHardwareWatchpoint() "
-                                        "SetDBGState() => 0x%8.8x.",
-                       kret);
-
-      if (kret == KERN_SUCCESS)
-        return i;
-    } else {
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::"
-                                        "EnableHardwareWatchpoint(): All "
-                                        "hardware resources (%u) are in use.",
-                       num_hw_watchpoints);
-    }
-  }
-  return INVALID_NUB_HW_INDEX;
-}
-
-bool DNBArchMachARM::ReenableHardwareWatchpoint(uint32_t hw_index) {
-  // If this logical watchpoint # is actually implemented using
-  // two hardware watchpoint registers, re-enable both of them.
-
-  if (hw_index < NumSupportedHardwareWatchpoints() && LoHi[hw_index]) {
-    return ReenableHardwareWatchpoint_helper(hw_index) &&
-           ReenableHardwareWatchpoint_helper(LoHi[hw_index]);
-  } else {
-    return ReenableHardwareWatchpoint_helper(hw_index);
-  }
-}
-
-bool DNBArchMachARM::ReenableHardwareWatchpoint_helper(uint32_t hw_index) {
-  kern_return_t kret = GetDBGState(false);
-  if (kret != KERN_SUCCESS)
-    return false;
-  const uint32_t num_hw_points = NumSupportedHardwareWatchpoints();
-  if (hw_index >= num_hw_points)
-    return false;
-
-  m_state.dbg.__wvr[hw_index] = m_disabled_watchpoints[hw_index].addr;
-  m_state.dbg.__wcr[hw_index] = m_disabled_watchpoints[hw_index].control;
-
-  DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::EnableHardwareWatchpoint( "
-                                    "%u ) - WVR%u = 0x%8.8llx  WCR%u = "
-                                    "0x%8.8llx",
-                   hw_index, hw_index, (uint64_t)m_state.dbg.__wvr[hw_index],
-                   hw_index, (uint64_t)m_state.dbg.__wcr[hw_index]);
-
-  // The kernel will set the MDE_ENABLE bit in the MDSCR_EL1 for us
-  // automatically, don't need to do it here.
-
-  kret = SetDBGState(false);
-
-  return (kret == KERN_SUCCESS);
-}
-
-bool DNBArchMachARM::DisableHardwareWatchpoint(uint32_t hw_index,
-                                               bool also_set_on_task) {
-  if (hw_index < NumSupportedHardwareWatchpoints() && LoHi[hw_index]) {
-    return DisableHardwareWatchpoint_helper(hw_index, also_set_on_task) &&
-           DisableHardwareWatchpoint_helper(LoHi[hw_index], also_set_on_task);
-  } else {
-    return DisableHardwareWatchpoint_helper(hw_index, also_set_on_task);
-  }
-}
-
-bool DNBArchMachARM::DisableHardwareWatchpoint_helper(uint32_t hw_index,
-                                                      bool also_set_on_task) {
-  kern_return_t kret = GetDBGState(false);
-  if (kret != KERN_SUCCESS)
-    return false;
-
-  const uint32_t num_hw_points = NumSupportedHardwareWatchpoints();
-  if (hw_index >= num_hw_points)
-    return false;
-
-  m_disabled_watchpoints[hw_index].addr = m_state.dbg.__wvr[hw_index];
-  m_disabled_watchpoints[hw_index].control = m_state.dbg.__wcr[hw_index];
-
-  m_state.dbg.__wvr[hw_index] = 0;
-  m_state.dbg.__wcr[hw_index] = 0;
-  DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::DisableHardwareWatchpoint("
-                                    " %u ) - WVR%u = 0x%8.8llx  WCR%u = "
-                                    "0x%8.8llx",
-                   hw_index, hw_index, (uint64_t)m_state.dbg.__wvr[hw_index],
-                   hw_index, (uint64_t)m_state.dbg.__wcr[hw_index]);
-
-  kret = SetDBGState(also_set_on_task);
-
-  return (kret == KERN_SUCCESS);
-}
-
-// Returns -1 if the trailing bit patterns are not one of:
-// { 0b???1, 0b??10, 0b?100, 0b1000 }.
-static inline int32_t LowestBitSet(uint32_t val) {
-  for (unsigned i = 0; i < 4; ++i) {
-    if (bit(val, i))
-      return i;
-  }
-  return -1;
-}
-
-// Iterate through the debug registers; return the index of the first watchpoint
-// whose address matches.
-// As a side effect, the starting address as understood by the debugger is
-// returned which could be
-// different from 'addr' passed as an in/out argument.
-uint32_t DNBArchMachARM::GetHardwareWatchpointHit(nub_addr_t &addr) {
-  // Read the debug state
-  kern_return_t kret = GetDBGState(true);
-  // DumpDBGState(m_state.dbg);
-  DNBLogThreadedIf(
-      LOG_WATCHPOINTS,
-      "DNBArchMachARM::GetHardwareWatchpointHit() GetDBGState() => 0x%8.8x.",
-      kret);
-  DNBLogThreadedIf(LOG_WATCHPOINTS,
-                   "DNBArchMachARM::GetHardwareWatchpointHit() addr = 0x%llx",
-                   (uint64_t)addr);
-
-// This is the watchpoint value to match against, i.e., word address.
-#if defined(WATCHPOINTS_ARE_DWORD)
-  nub_addr_t wp_val = addr & ~((nub_addr_t)7);
-#else
-  nub_addr_t wp_val = addr & ~((nub_addr_t)3);
-#endif
-  if (kret == KERN_SUCCESS) {
-    DBG &debug_state = m_state.dbg;
-    uint32_t i, num = NumSupportedHardwareWatchpoints();
-    for (i = 0; i < num; ++i) {
-      nub_addr_t wp_addr = GetWatchAddress(debug_state, i);
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM::"
-                                        "GetHardwareWatchpointHit() slot: %u "
-                                        "(addr = 0x%llx).",
-                       i, (uint64_t)wp_addr);
-      if (wp_val == wp_addr) {
-#if defined(WATCHPOINTS_ARE_DWORD)
-        uint32_t byte_mask = bits(debug_state.__wcr[i], 12, 5);
-#else
-        uint32_t byte_mask = bits(debug_state.__wcr[i], 8, 5);
-#endif
-
-        // Sanity check the byte_mask, first.
-        if (LowestBitSet(byte_mask) < 0)
-          continue;
-
-        // Compute the starting address (from the point of view of the
-        // debugger).
-        addr = wp_addr + LowestBitSet(byte_mask);
-        return i;
-      }
-    }
-  }
-  return INVALID_NUB_HW_INDEX;
-}
-
-nub_addr_t DNBArchMachARM::GetWatchpointAddressByIndex(uint32_t hw_index) {
-  kern_return_t kret = GetDBGState(true);
-  if (kret != KERN_SUCCESS)
-    return INVALID_NUB_ADDRESS;
-  const uint32_t num = NumSupportedHardwareWatchpoints();
-  if (hw_index >= num)
-    return INVALID_NUB_ADDRESS;
-  if (IsWatchpointEnabled(m_state.dbg, hw_index))
-    return GetWatchAddress(m_state.dbg, hw_index);
-  return INVALID_NUB_ADDRESS;
-}
-
-bool DNBArchMachARM::IsWatchpointEnabled(const DBG &debug_state,
-                                         uint32_t hw_index) {
-  // Watchpoint Control Registers, bitfield definitions
-  // ...
-  // Bits    Value    Description
-  // [0]     0        Watchpoint disabled
-  //         1        Watchpoint enabled.
-  return (debug_state.__wcr[hw_index] & 1u);
-}
-
-nub_addr_t DNBArchMachARM::GetWatchAddress(const DBG &debug_state,
-                                           uint32_t hw_index) {
-  // Watchpoint Value Registers, bitfield definitions
-  // Bits        Description
-  // [31:2]      Watchpoint value (word address, i.e., 4-byte aligned)
-  // [1:0]       RAZ/SBZP
-  return bits(debug_state.__wvr[hw_index], 31, 0);
-}
-
-// Register information definitions for 32 bit ARMV7.
-enum gpr_regnums {
-  gpr_r0 = 0,
-  gpr_r1,
-  gpr_r2,
-  gpr_r3,
-  gpr_r4,
-  gpr_r5,
-  gpr_r6,
-  gpr_r7,
-  gpr_r8,
-  gpr_r9,
-  gpr_r10,
-  gpr_r11,
-  gpr_r12,
-  gpr_sp,
-  gpr_lr,
-  gpr_pc,
-  gpr_cpsr
-};
-
-enum {
-  vfp_s0 = 0,
-  vfp_s1,
-  vfp_s2,
-  vfp_s3,
-  vfp_s4,
-  vfp_s5,
-  vfp_s6,
-  vfp_s7,
-  vfp_s8,
-  vfp_s9,
-  vfp_s10,
-  vfp_s11,
-  vfp_s12,
-  vfp_s13,
-  vfp_s14,
-  vfp_s15,
-  vfp_s16,
-  vfp_s17,
-  vfp_s18,
-  vfp_s19,
-  vfp_s20,
-  vfp_s21,
-  vfp_s22,
-  vfp_s23,
-  vfp_s24,
-  vfp_s25,
-  vfp_s26,
-  vfp_s27,
-  vfp_s28,
-  vfp_s29,
-  vfp_s30,
-  vfp_s31,
-  vfp_d0,
-  vfp_d1,
-  vfp_d2,
-  vfp_d3,
-  vfp_d4,
-  vfp_d5,
-  vfp_d6,
-  vfp_d7,
-  vfp_d8,
-  vfp_d9,
-  vfp_d10,
-  vfp_d11,
-  vfp_d12,
-  vfp_d13,
-  vfp_d14,
-  vfp_d15,
-  vfp_d16,
-  vfp_d17,
-  vfp_d18,
-  vfp_d19,
-  vfp_d20,
-  vfp_d21,
-  vfp_d22,
-  vfp_d23,
-  vfp_d24,
-  vfp_d25,
-  vfp_d26,
-  vfp_d27,
-  vfp_d28,
-  vfp_d29,
-  vfp_d30,
-  vfp_d31,
-  vfp_q0,
-  vfp_q1,
-  vfp_q2,
-  vfp_q3,
-  vfp_q4,
-  vfp_q5,
-  vfp_q6,
-  vfp_q7,
-  vfp_q8,
-  vfp_q9,
-  vfp_q10,
-  vfp_q11,
-  vfp_q12,
-  vfp_q13,
-  vfp_q14,
-  vfp_q15,
-#if defined(__arm64__) || defined(__aarch64__)
-  vfp_fpsr,
-  vfp_fpcr,
-#else
-  vfp_fpscr
-#endif
-};
-
-enum {
-  exc_exception,
-  exc_fsr,
-  exc_far,
-};
-
-#define GPR_OFFSET_IDX(idx) (offsetof(DNBArchMachARM::GPR, __r[idx]))
-#define GPR_OFFSET_NAME(reg) (offsetof(DNBArchMachARM::GPR, __##reg))
-
-#define EXC_OFFSET(reg)                                                        \
-  (offsetof(DNBArchMachARM::EXC, __##reg) +                                    \
-   offsetof(DNBArchMachARM::Context, exc))
-
-// These macros will auto define the register name, alt name, register size,
-// register offset, encoding, format and native register. This ensures that
-// the register state structures are defined correctly and have the correct
-// sizes and offsets.
-#define DEFINE_GPR_IDX(idx, reg, alt, gen)                                     \
-  {                                                                            \
-    e_regSetGPR, gpr_##reg, #reg, alt, Uint, Hex, 4, GPR_OFFSET_IDX(idx),      \
-        ehframe_##reg, dwarf_##reg, gen, INVALID_NUB_REGNUM, NULL, NULL        \
-  }
-#define DEFINE_GPR_NAME(reg, alt, gen, inval)                                  \
-  {                                                                            \
-    e_regSetGPR, gpr_##reg, #reg, alt, Uint, Hex, 4, GPR_OFFSET_NAME(reg),     \
-        ehframe_##reg, dwarf_##reg, gen, INVALID_NUB_REGNUM, NULL, inval       \
-  }
-
-// In case we are debugging to a debug target that the ability to
-// change into the protected modes with folded registers (ABT, IRQ,
-// FIQ, SYS, USR, etc..), we should invalidate r8-r14 if the CPSR
-// gets modified.
-
-const char *g_invalidate_cpsr[] = {"r8",  "r9", "r10", "r11",
-                                   "r12", "sp", "lr",  NULL};
-
-// General purpose registers
-const DNBRegisterInfo DNBArchMachARM::g_gpr_registers[] = {
-    DEFINE_GPR_IDX(0, r0, "arg1", GENERIC_REGNUM_ARG1),
-    DEFINE_GPR_IDX(1, r1, "arg2", GENERIC_REGNUM_ARG2),
-    DEFINE_GPR_IDX(2, r2, "arg3", GENERIC_REGNUM_ARG3),
-    DEFINE_GPR_IDX(3, r3, "arg4", GENERIC_REGNUM_ARG4),
-    DEFINE_GPR_IDX(4, r4, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(5, r5, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(6, r6, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(7, r7, "fp", GENERIC_REGNUM_FP),
-    DEFINE_GPR_IDX(8, r8, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(9, r9, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(10, r10, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(11, r11, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_IDX(12, r12, NULL, INVALID_NUB_REGNUM),
-    DEFINE_GPR_NAME(sp, "r13", GENERIC_REGNUM_SP, NULL),
-    DEFINE_GPR_NAME(lr, "r14", GENERIC_REGNUM_RA, NULL),
-    DEFINE_GPR_NAME(pc, "r15", GENERIC_REGNUM_PC, NULL),
-    DEFINE_GPR_NAME(cpsr, "flags", GENERIC_REGNUM_FLAGS, g_invalidate_cpsr)};
-
-const char *g_contained_q0[]{"q0", NULL};
-const char *g_contained_q1[]{"q1", NULL};
-const char *g_contained_q2[]{"q2", NULL};
-const char *g_contained_q3[]{"q3", NULL};
-const char *g_contained_q4[]{"q4", NULL};
-const char *g_contained_q5[]{"q5", NULL};
-const char *g_contained_q6[]{"q6", NULL};
-const char *g_contained_q7[]{"q7", NULL};
-const char *g_contained_q8[]{"q8", NULL};
-const char *g_contained_q9[]{"q9", NULL};
-const char *g_contained_q10[]{"q10", NULL};
-const char *g_contained_q11[]{"q11", NULL};
-const char *g_contained_q12[]{"q12", NULL};
-const char *g_contained_q13[]{"q13", NULL};
-const char *g_contained_q14[]{"q14", NULL};
-const char *g_contained_q15[]{"q15", NULL};
-
-const char *g_invalidate_q0[]{"q0", "d0", "d1", "s0", "s1", "s2", "s3", NULL};
-const char *g_invalidate_q1[]{"q1", "d2", "d3", "s4", "s5", "s6", "s7", NULL};
-const char *g_invalidate_q2[]{"q2", "d4", "d5", "s8", "s9", "s10", "s11", NULL};
-const char *g_invalidate_q3[]{"q3",  "d6",  "d7",  "s12",
-                              "s13", "s14", "s15", NULL};
-const char *g_invalidate_q4[]{"q4",  "d8",  "d9",  "s16",
-                              "s17", "s18", "s19", NULL};
-const char *g_invalidate_q5[]{"q5",  "d10", "d11", "s20",
-                              "s21", "s22", "s23", NULL};
-const char *g_invalidate_q6[]{"q6",  "d12", "d13", "s24",
-                              "s25", "s26", "s27", NULL};
-const char *g_invalidate_q7[]{"q7",  "d14", "d15", "s28",
-                              "s29", "s30", "s31", NULL};
-const char *g_invalidate_q8[]{"q8", "d16", "d17", NULL};
-const char *g_invalidate_q9[]{"q9", "d18", "d19", NULL};
-const char *g_invalidate_q10[]{"q10", "d20", "d21", NULL};
-const char *g_invalidate_q11[]{"q11", "d22", "d23", NULL};
-const char *g_invalidate_q12[]{"q12", "d24", "d25", NULL};
-const char *g_invalidate_q13[]{"q13", "d26", "d27", NULL};
-const char *g_invalidate_q14[]{"q14", "d28", "d29", NULL};
-const char *g_invalidate_q15[]{"q15", "d30", "d31", NULL};
-
-#define VFP_S_OFFSET_IDX(idx)                                                  \
-  (((idx) % 4) * 4) // offset into q reg: 0, 4, 8, 12
-#define VFP_D_OFFSET_IDX(idx) (((idx) % 2) * 8) // offset into q reg: 0, 8
-#define VFP_Q_OFFSET_IDX(idx) (VFP_S_OFFSET_IDX((idx)*4))
-
-#define VFP_OFFSET_NAME(reg)                                                   \
-  (offsetof(DNBArchMachARM::FPU, __##reg) +                                    \
-   offsetof(DNBArchMachARM::Context, vfp))
-
-#define FLOAT_FORMAT Float
-
-#define DEFINE_VFP_S_IDX(idx)                                                  \
-  e_regSetVFP, vfp_s##idx, "s" #idx, NULL, IEEE754, FLOAT_FORMAT, 4,           \
-      VFP_S_OFFSET_IDX(idx), INVALID_NUB_REGNUM, dwarf_s##idx,                 \
-      INVALID_NUB_REGNUM, INVALID_NUB_REGNUM
-#define DEFINE_VFP_D_IDX(idx)                                                  \
-  e_regSetVFP, vfp_d##idx, "d" #idx, NULL, IEEE754, FLOAT_FORMAT, 8,           \
-      VFP_D_OFFSET_IDX(idx), INVALID_NUB_REGNUM, dwarf_d##idx,                 \
-      INVALID_NUB_REGNUM, INVALID_NUB_REGNUM
-#define DEFINE_VFP_Q_IDX(idx)                                                  \
-  e_regSetVFP, vfp_q##idx, "q" #idx, NULL, Vector, VectorOfUInt8, 16,          \
-      VFP_Q_OFFSET_IDX(idx), INVALID_NUB_REGNUM, dwarf_q##idx,                 \
-      INVALID_NUB_REGNUM, INVALID_NUB_REGNUM
-
-// Floating point registers
-const DNBRegisterInfo DNBArchMachARM::g_vfp_registers[] = {
-    {DEFINE_VFP_S_IDX(0), g_contained_q0, g_invalidate_q0},
-    {DEFINE_VFP_S_IDX(1), g_contained_q0, g_invalidate_q0},
-    {DEFINE_VFP_S_IDX(2), g_contained_q0, g_invalidate_q0},
-    {DEFINE_VFP_S_IDX(3), g_contained_q0, g_invalidate_q0},
-    {DEFINE_VFP_S_IDX(4), g_contained_q1, g_invalidate_q1},
-    {DEFINE_VFP_S_IDX(5), g_contained_q1, g_invalidate_q1},
-    {DEFINE_VFP_S_IDX(6), g_contained_q1, g_invalidate_q1},
-    {DEFINE_VFP_S_IDX(7), g_contained_q1, g_invalidate_q1},
-    {DEFINE_VFP_S_IDX(8), g_contained_q2, g_invalidate_q2},
-    {DEFINE_VFP_S_IDX(9), g_contained_q2, g_invalidate_q2},
-    {DEFINE_VFP_S_IDX(10), g_contained_q2, g_invalidate_q2},
-    {DEFINE_VFP_S_IDX(11), g_contained_q2, g_invalidate_q2},
-    {DEFINE_VFP_S_IDX(12), g_contained_q3, g_invalidate_q3},
-    {DEFINE_VFP_S_IDX(13), g_contained_q3, g_invalidate_q3},
-    {DEFINE_VFP_S_IDX(14), g_contained_q3, g_invalidate_q3},
-    {DEFINE_VFP_S_IDX(15), g_contained_q3, g_invalidate_q3},
-    {DEFINE_VFP_S_IDX(16), g_contained_q4, g_invalidate_q4},
-    {DEFINE_VFP_S_IDX(17), g_contained_q4, g_invalidate_q4},
-    {DEFINE_VFP_S_IDX(18), g_contained_q4, g_invalidate_q4},
-    {DEFINE_VFP_S_IDX(19), g_contained_q4, g_invalidate_q4},
-    {DEFINE_VFP_S_IDX(20), g_contained_q5, g_invalidate_q5},
-    {DEFINE_VFP_S_IDX(21), g_contained_q5, g_invalidate_q5},
-    {DEFINE_VFP_S_IDX(22), g_contained_q5, g_invalidate_q5},
-    {DEFINE_VFP_S_IDX(23), g_contained_q5, g_invalidate_q5},
-    {DEFINE_VFP_S_IDX(24), g_contained_q6, g_invalidate_q6},
-    {DEFINE_VFP_S_IDX(25), g_contained_q6, g_invalidate_q6},
-    {DEFINE_VFP_S_IDX(26), g_contained_q6, g_invalidate_q6},
-    {DEFINE_VFP_S_IDX(27), g_contained_q6, g_invalidate_q6},
-    {DEFINE_VFP_S_IDX(28), g_contained_q7, g_invalidate_q7},
-    {DEFINE_VFP_S_IDX(29), g_contained_q7, g_invalidate_q7},
-    {DEFINE_VFP_S_IDX(30), g_contained_q7, g_invalidate_q7},
-    {DEFINE_VFP_S_IDX(31), g_contained_q7, g_invalidate_q7},
-
-    {DEFINE_VFP_D_IDX(0), g_contained_q0, g_invalidate_q0},
-    {DEFINE_VFP_D_IDX(1), g_contained_q0, g_invalidate_q0},
-    {DEFINE_VFP_D_IDX(2), g_contained_q1, g_invalidate_q1},
-    {DEFINE_VFP_D_IDX(3), g_contained_q1, g_invalidate_q1},
-    {DEFINE_VFP_D_IDX(4), g_contained_q2, g_invalidate_q2},
-    {DEFINE_VFP_D_IDX(5), g_contained_q2, g_invalidate_q2},
-    {DEFINE_VFP_D_IDX(6), g_contained_q3, g_invalidate_q3},
-    {DEFINE_VFP_D_IDX(7), g_contained_q3, g_invalidate_q3},
-    {DEFINE_VFP_D_IDX(8), g_contained_q4, g_invalidate_q4},
-    {DEFINE_VFP_D_IDX(9), g_contained_q4, g_invalidate_q4},
-    {DEFINE_VFP_D_IDX(10), g_contained_q5, g_invalidate_q5},
-    {DEFINE_VFP_D_IDX(11), g_contained_q5, g_invalidate_q5},
-    {DEFINE_VFP_D_IDX(12), g_contained_q6, g_invalidate_q6},
-    {DEFINE_VFP_D_IDX(13), g_contained_q6, g_invalidate_q6},
-    {DEFINE_VFP_D_IDX(14), g_contained_q7, g_invalidate_q7},
-    {DEFINE_VFP_D_IDX(15), g_contained_q7, g_invalidate_q7},
-    {DEFINE_VFP_D_IDX(16), g_contained_q8, g_invalidate_q8},
-    {DEFINE_VFP_D_IDX(17), g_contained_q8, g_invalidate_q8},
-    {DEFINE_VFP_D_IDX(18), g_contained_q9, g_invalidate_q9},
-    {DEFINE_VFP_D_IDX(19), g_contained_q9, g_invalidate_q9},
-    {DEFINE_VFP_D_IDX(20), g_contained_q10, g_invalidate_q10},
-    {DEFINE_VFP_D_IDX(21), g_contained_q10, g_invalidate_q10},
-    {DEFINE_VFP_D_IDX(22), g_contained_q11, g_invalidate_q11},
-    {DEFINE_VFP_D_IDX(23), g_contained_q11, g_invalidate_q11},
-    {DEFINE_VFP_D_IDX(24), g_contained_q12, g_invalidate_q12},
-    {DEFINE_VFP_D_IDX(25), g_contained_q12, g_invalidate_q12},
-    {DEFINE_VFP_D_IDX(26), g_contained_q13, g_invalidate_q13},
-    {DEFINE_VFP_D_IDX(27), g_contained_q13, g_invalidate_q13},
-    {DEFINE_VFP_D_IDX(28), g_contained_q14, g_invalidate_q14},
-    {DEFINE_VFP_D_IDX(29), g_contained_q14, g_invalidate_q14},
-    {DEFINE_VFP_D_IDX(30), g_contained_q15, g_invalidate_q15},
-    {DEFINE_VFP_D_IDX(31), g_contained_q15, g_invalidate_q15},
-
-    {DEFINE_VFP_Q_IDX(0), NULL, g_invalidate_q0},
-    {DEFINE_VFP_Q_IDX(1), NULL, g_invalidate_q1},
-    {DEFINE_VFP_Q_IDX(2), NULL, g_invalidate_q2},
-    {DEFINE_VFP_Q_IDX(3), NULL, g_invalidate_q3},
-    {DEFINE_VFP_Q_IDX(4), NULL, g_invalidate_q4},
-    {DEFINE_VFP_Q_IDX(5), NULL, g_invalidate_q5},
-    {DEFINE_VFP_Q_IDX(6), NULL, g_invalidate_q6},
-    {DEFINE_VFP_Q_IDX(7), NULL, g_invalidate_q7},
-    {DEFINE_VFP_Q_IDX(8), NULL, g_invalidate_q8},
-    {DEFINE_VFP_Q_IDX(9), NULL, g_invalidate_q9},
-    {DEFINE_VFP_Q_IDX(10), NULL, g_invalidate_q10},
-    {DEFINE_VFP_Q_IDX(11), NULL, g_invalidate_q11},
-    {DEFINE_VFP_Q_IDX(12), NULL, g_invalidate_q12},
-    {DEFINE_VFP_Q_IDX(13), NULL, g_invalidate_q13},
-    {DEFINE_VFP_Q_IDX(14), NULL, g_invalidate_q14},
-    {DEFINE_VFP_Q_IDX(15), NULL, g_invalidate_q15},
-
-#if defined(__arm64__) || defined(__aarch64__)
-    {e_regSetVFP, vfp_fpsr, "fpsr", NULL, Uint, Hex, 4, VFP_OFFSET_NAME(fpsr),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetVFP, vfp_fpcr, "fpcr", NULL, Uint, Hex, 4, VFP_OFFSET_NAME(fpcr),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL}
-#else
-    {e_regSetVFP, vfp_fpscr, "fpscr", NULL, Uint, Hex, 4,
-     VFP_OFFSET_NAME(fpscr), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL}
-#endif
-};
-
-// Exception registers
-
-const DNBRegisterInfo DNBArchMachARM::g_exc_registers[] = {
-    {e_regSetVFP, exc_exception, "exception", NULL, Uint, Hex, 4,
-     EXC_OFFSET(exception), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetVFP, exc_fsr, "fsr", NULL, Uint, Hex, 4, EXC_OFFSET(fsr),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetVFP, exc_far, "far", NULL, Uint, Hex, 4, EXC_OFFSET(far),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL}};
-
-// Number of registers in each register set
-const size_t DNBArchMachARM::k_num_gpr_registers =
-    sizeof(g_gpr_registers) / sizeof(DNBRegisterInfo);
-const size_t DNBArchMachARM::k_num_vfp_registers =
-    sizeof(g_vfp_registers) / sizeof(DNBRegisterInfo);
-const size_t DNBArchMachARM::k_num_exc_registers =
-    sizeof(g_exc_registers) / sizeof(DNBRegisterInfo);
-const size_t DNBArchMachARM::k_num_all_registers =
-    k_num_gpr_registers + k_num_vfp_registers + k_num_exc_registers;
-
-// Register set definitions. The first definitions at register set index
-// of zero is for all registers, followed by other registers sets. The
-// register information for the all register set need not be filled in.
-const DNBRegisterSetInfo DNBArchMachARM::g_reg_sets[] = {
-    {"ARM Registers", NULL, k_num_all_registers},
-    {"General Purpose Registers", g_gpr_registers, k_num_gpr_registers},
-    {"Floating Point Registers", g_vfp_registers, k_num_vfp_registers},
-    {"Exception State Registers", g_exc_registers, k_num_exc_registers}};
-// Total number of register sets for this architecture
-const size_t DNBArchMachARM::k_num_register_sets =
-    sizeof(g_reg_sets) / sizeof(DNBRegisterSetInfo);
-
-const DNBRegisterSetInfo *
-DNBArchMachARM::GetRegisterSetInfo(nub_size_t *num_reg_sets) {
-  *num_reg_sets = k_num_register_sets;
-  return g_reg_sets;
-}
-
-bool DNBArchMachARM::GetRegisterValue(uint32_t set, uint32_t reg,
-                                      DNBRegisterValue *value) {
-  if (set == REGISTER_SET_GENERIC) {
-    switch (reg) {
-    case GENERIC_REGNUM_PC: // Program Counter
-      set = e_regSetGPR;
-      reg = gpr_pc;
-      break;
-
-    case GENERIC_REGNUM_SP: // Stack Pointer
-      set = e_regSetGPR;
-      reg = gpr_sp;
-      break;
-
-    case GENERIC_REGNUM_FP: // Frame Pointer
-      set = e_regSetGPR;
-      reg = gpr_r7; // is this the right reg?
-      break;
-
-    case GENERIC_REGNUM_RA: // Return Address
-      set = e_regSetGPR;
-      reg = gpr_lr;
-      break;
-
-    case GENERIC_REGNUM_FLAGS: // Processor flags register
-      set = e_regSetGPR;
-      reg = gpr_cpsr;
-      break;
-
-    default:
-      return false;
-    }
-  }
-
-  if (GetRegisterState(set, false) != KERN_SUCCESS)
-    return false;
-
-  const DNBRegisterInfo *regInfo = m_thread->GetRegisterInfo(set, reg);
-  if (regInfo) {
-    value->info = *regInfo;
-    switch (set) {
-    case e_regSetGPR:
-      if (reg < k_num_gpr_registers) {
-        value->value.uint32 = m_state.context.gpr.__r[reg];
-        return true;
-      }
-      break;
-
-    case e_regSetVFP:
-      // "reg" is an index into the floating point register set at this point.
-      // We need to translate it up so entry 0 in the fp reg set is the same as
-      // vfp_s0
-      // in the enumerated values for case statement below.
-      if (reg >= vfp_s0 && reg <= vfp_s31) {
-#if defined(__arm64__) || defined(__aarch64__)
-        uint32_t *s_reg =
-            ((uint32_t *)&m_state.context.vfp.__v[0]) + (reg - vfp_s0);
-        memcpy(&value->value.v_uint8, s_reg, 4);
-#else
-        value->value.uint32 = m_state.context.vfp.__r[reg];
-#endif
-        return true;
-      } else if (reg >= vfp_d0 && reg <= vfp_d31) {
-#if defined(__arm64__) || defined(__aarch64__)
-        uint64_t *d_reg =
-            ((uint64_t *)&m_state.context.vfp.__v[0]) + (reg - vfp_d0);
-        memcpy(&value->value.v_uint8, d_reg, 8);
-#else
-        uint32_t d_reg_idx = reg - vfp_d0;
-        uint32_t s_reg_idx = d_reg_idx * 2;
-        value->value.v_sint32[0] = m_state.context.vfp.__r[s_reg_idx + 0];
-        value->value.v_sint32[1] = m_state.context.vfp.__r[s_reg_idx + 1];
-#endif
-        return true;
-      } else if (reg >= vfp_q0 && reg <= vfp_q15) {
-#if defined(__arm64__) || defined(__aarch64__)
-        memcpy(&value->value.v_uint8,
-               (uint8_t *)&m_state.context.vfp.__v[reg - vfp_q0], 16);
-#else
-        uint32_t s_reg_idx = (reg - vfp_q0) * 4;
-        memcpy(&value->value.v_uint8,
-               (uint8_t *)&m_state.context.vfp.__r[s_reg_idx], 16);
-#endif
-        return true;
-      }
-#if defined(__arm64__) || defined(__aarch64__)
-      else if (reg == vfp_fpsr) {
-        value->value.uint32 = m_state.context.vfp.__fpsr;
-        return true;
-      } else if (reg == vfp_fpcr) {
-        value->value.uint32 = m_state.context.vfp.__fpcr;
-        return true;
-      }
-#else
-      else if (reg == vfp_fpscr) {
-        value->value.uint32 = m_state.context.vfp.__fpscr;
-        return true;
-      }
-#endif
-      break;
-
-    case e_regSetEXC:
-      if (reg < k_num_exc_registers) {
-        value->value.uint32 = (&m_state.context.exc.__exception)[reg];
-        return true;
-      }
-      break;
-    }
-  }
-  return false;
-}
-
-bool DNBArchMachARM::SetRegisterValue(uint32_t set, uint32_t reg,
-                                      const DNBRegisterValue *value) {
-  if (set == REGISTER_SET_GENERIC) {
-    switch (reg) {
-    case GENERIC_REGNUM_PC: // Program Counter
-      set = e_regSetGPR;
-      reg = gpr_pc;
-      break;
-
-    case GENERIC_REGNUM_SP: // Stack Pointer
-      set = e_regSetGPR;
-      reg = gpr_sp;
-      break;
-
-    case GENERIC_REGNUM_FP: // Frame Pointer
-      set = e_regSetGPR;
-      reg = gpr_r7;
-      break;
-
-    case GENERIC_REGNUM_RA: // Return Address
-      set = e_regSetGPR;
-      reg = gpr_lr;
-      break;
-
-    case GENERIC_REGNUM_FLAGS: // Processor flags register
-      set = e_regSetGPR;
-      reg = gpr_cpsr;
-      break;
-
-    default:
-      return false;
-    }
-  }
-
-  if (GetRegisterState(set, false) != KERN_SUCCESS)
-    return false;
-
-  bool success = false;
-  const DNBRegisterInfo *regInfo = m_thread->GetRegisterInfo(set, reg);
-  if (regInfo) {
-    switch (set) {
-    case e_regSetGPR:
-      if (reg < k_num_gpr_registers) {
-        m_state.context.gpr.__r[reg] = value->value.uint32;
-        success = true;
-      }
-      break;
-
-    case e_regSetVFP:
-      // "reg" is an index into the floating point register set at this point.
-      // We need to translate it up so entry 0 in the fp reg set is the same as
-      // vfp_s0
-      // in the enumerated values for case statement below.
-      if (reg >= vfp_s0 && reg <= vfp_s31) {
-#if defined(__arm64__) || defined(__aarch64__)
-        uint32_t *s_reg =
-            ((uint32_t *)&m_state.context.vfp.__v[0]) + (reg - vfp_s0);
-        memcpy(s_reg, &value->value.v_uint8, 4);
-#else
-        m_state.context.vfp.__r[reg] = value->value.uint32;
-#endif
-        success = true;
-      } else if (reg >= vfp_d0 && reg <= vfp_d31) {
-#if defined(__arm64__) || defined(__aarch64__)
-        uint64_t *d_reg =
-            ((uint64_t *)&m_state.context.vfp.__v[0]) + (reg - vfp_d0);
-        memcpy(d_reg, &value->value.v_uint8, 8);
-#else
-        uint32_t d_reg_idx = reg - vfp_d0;
-        uint32_t s_reg_idx = d_reg_idx * 2;
-        m_state.context.vfp.__r[s_reg_idx + 0] = value->value.v_sint32[0];
-        m_state.context.vfp.__r[s_reg_idx + 1] = value->value.v_sint32[1];
-#endif
-        success = true;
-      } else if (reg >= vfp_q0 && reg <= vfp_q15) {
-#if defined(__arm64__) || defined(__aarch64__)
-        memcpy((uint8_t *)&m_state.context.vfp.__v[reg - vfp_q0],
-               &value->value.v_uint8, 16);
-#else
-        uint32_t s_reg_idx = (reg - vfp_q0) * 4;
-        memcpy((uint8_t *)&m_state.context.vfp.__r[s_reg_idx],
-               &value->value.v_uint8, 16);
-#endif
-        success = true;
-      }
-#if defined(__arm64__) || defined(__aarch64__)
-      else if (reg == vfp_fpsr) {
-        m_state.context.vfp.__fpsr = value->value.uint32;
-        success = true;
-      } else if (reg == vfp_fpcr) {
-        m_state.context.vfp.__fpcr = value->value.uint32;
-        success = true;
-      }
-#else
-      else if (reg == vfp_fpscr) {
-        m_state.context.vfp.__fpscr = value->value.uint32;
-        success = true;
-      }
-#endif
-      break;
-
-    case e_regSetEXC:
-      if (reg < k_num_exc_registers) {
-        (&m_state.context.exc.__exception)[reg] = value->value.uint32;
-        success = true;
-      }
-      break;
-    }
-  }
-  if (success)
-    return SetRegisterState(set) == KERN_SUCCESS;
-  return false;
-}
-
-kern_return_t DNBArchMachARM::GetRegisterState(int set, bool force) {
-  switch (set) {
-  case e_regSetALL:
-    return GetGPRState(force) | GetVFPState(force) | GetEXCState(force) |
-           GetDBGState(force);
-  case e_regSetGPR:
-    return GetGPRState(force);
-  case e_regSetVFP:
-    return GetVFPState(force);
-  case e_regSetEXC:
-    return GetEXCState(force);
-  case e_regSetDBG:
-    return GetDBGState(force);
-  default:
-    break;
-  }
-  return KERN_INVALID_ARGUMENT;
-}
-
-kern_return_t DNBArchMachARM::SetRegisterState(int set) {
-  // Make sure we have a valid context to set.
-  kern_return_t err = GetRegisterState(set, false);
-  if (err != KERN_SUCCESS)
-    return err;
-
-  switch (set) {
-  case e_regSetALL:
-    return SetGPRState() | SetVFPState() | SetEXCState() | SetDBGState(false);
-  case e_regSetGPR:
-    return SetGPRState();
-  case e_regSetVFP:
-    return SetVFPState();
-  case e_regSetEXC:
-    return SetEXCState();
-  case e_regSetDBG:
-    return SetDBGState(false);
-  default:
-    break;
-  }
-  return KERN_INVALID_ARGUMENT;
-}
-
-bool DNBArchMachARM::RegisterSetStateIsValid(int set) const {
-  return m_state.RegsAreValid(set);
-}
-
-nub_size_t DNBArchMachARM::GetRegisterContext(void *buf, nub_size_t buf_len) {
-  nub_size_t size = sizeof(m_state.context.gpr) + sizeof(m_state.context.vfp) +
-                    sizeof(m_state.context.exc);
-
-  if (buf && buf_len) {
-    if (size > buf_len)
-      size = buf_len;
-
-    bool force = false;
-    if (GetGPRState(force) | GetVFPState(force) | GetEXCState(force))
-      return 0;
-
-    // Copy each struct individually to avoid any padding that might be between
-    // the structs in m_state.context
-    uint8_t *p = (uint8_t *)buf;
-    ::memcpy(p, &m_state.context.gpr, sizeof(m_state.context.gpr));
-    p += sizeof(m_state.context.gpr);
-    ::memcpy(p, &m_state.context.vfp, sizeof(m_state.context.vfp));
-    p += sizeof(m_state.context.vfp);
-    ::memcpy(p, &m_state.context.exc, sizeof(m_state.context.exc));
-    p += sizeof(m_state.context.exc);
-
-    size_t bytes_written = p - (uint8_t *)buf;
-    UNUSED_IF_ASSERT_DISABLED(bytes_written);
-    assert(bytes_written == size);
-  }
-  DNBLogThreadedIf(
-      LOG_THREAD,
-      "DNBArchMachARM::GetRegisterContext (buf = %p, len = %llu) => %llu", buf,
-      (uint64_t)buf_len, (uint64_t)size);
-  // Return the size of the register context even if NULL was passed in
-  return size;
-}
-
-nub_size_t DNBArchMachARM::SetRegisterContext(const void *buf,
-                                              nub_size_t buf_len) {
-  nub_size_t size = sizeof(m_state.context.gpr) + sizeof(m_state.context.vfp) +
-                    sizeof(m_state.context.exc);
-
-  if (buf == NULL || buf_len == 0)
-    size = 0;
-
-  if (size) {
-    if (size > buf_len)
-      size = buf_len;
-
-    // Copy each struct individually to avoid any padding that might be between
-    // the structs in m_state.context
-    uint8_t *p = const_cast<uint8_t*>(reinterpret_cast<const uint8_t *>(buf));
-    ::memcpy(&m_state.context.gpr, p, sizeof(m_state.context.gpr));
-    p += sizeof(m_state.context.gpr);
-    ::memcpy(&m_state.context.vfp, p, sizeof(m_state.context.vfp));
-    p += sizeof(m_state.context.vfp);
-    ::memcpy(&m_state.context.exc, p, sizeof(m_state.context.exc));
-    p += sizeof(m_state.context.exc);
-
-    size_t bytes_written = p - reinterpret_cast<const uint8_t *>(buf);
-    UNUSED_IF_ASSERT_DISABLED(bytes_written);
-    assert(bytes_written == size);
-
-    if (SetGPRState() | SetVFPState() | SetEXCState())
-      return 0;
-  }
-  DNBLogThreadedIf(
-      LOG_THREAD,
-      "DNBArchMachARM::SetRegisterContext (buf = %p, len = %llu) => %llu", buf,
-      (uint64_t)buf_len, (uint64_t)size);
-  return size;
-}
-
-uint32_t DNBArchMachARM::SaveRegisterState() {
-  kern_return_t kret = ::thread_abort_safely(m_thread->MachPortNumber());
-  DNBLogThreadedIf(
-      LOG_THREAD, "thread = 0x%4.4x calling thread_abort_safely (tid) => %u "
-                  "(SetGPRState() for stop_count = %u)",
-      m_thread->MachPortNumber(), kret, m_thread->Process()->StopCount());
-
-  // Always re-read the registers because above we call thread_abort_safely();
-  bool force = true;
-
-  if ((kret = GetGPRState(force)) != KERN_SUCCESS) {
-    DNBLogThreadedIf(LOG_THREAD, "DNBArchMachARM::SaveRegisterState () error: "
-                                 "GPR regs failed to read: %u ",
-                     kret);
-  } else if ((kret = GetVFPState(force)) != KERN_SUCCESS) {
-    DNBLogThreadedIf(LOG_THREAD, "DNBArchMachARM::SaveRegisterState () error: "
-                                 "%s regs failed to read: %u",
-                     "VFP", kret);
-  } else {
-    const uint32_t save_id = GetNextRegisterStateSaveID();
-    m_saved_register_states[save_id] = m_state.context;
-    return save_id;
-  }
-  return UINT32_MAX;
-}
-
-bool DNBArchMachARM::RestoreRegisterState(uint32_t save_id) {
-  SaveRegisterStates::iterator pos = m_saved_register_states.find(save_id);
-  if (pos != m_saved_register_states.end()) {
-    m_state.context.gpr = pos->second.gpr;
-    m_state.context.vfp = pos->second.vfp;
-    kern_return_t kret;
-    bool success = true;
-    if ((kret = SetGPRState()) != KERN_SUCCESS) {
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchMachARM::RestoreRegisterState "
-                                   "(save_id = %u) error: GPR regs failed to "
-                                   "write: %u",
-                       save_id, kret);
-      success = false;
-    } else if ((kret = SetVFPState()) != KERN_SUCCESS) {
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchMachARM::RestoreRegisterState "
-                                   "(save_id = %u) error: %s regs failed to "
-                                   "write: %u",
-                       save_id, "VFP", kret);
-      success = false;
-    }
-    m_saved_register_states.erase(pos);
-    return success;
-  }
-  return false;
-}
-
-#endif // #if defined (__arm__)
diff --git a/lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.h b/lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.h
deleted file mode 100644
index 9c27e988ea052..0000000000000
--- a/lldb/tools/debugserver/source/MacOSX/arm/DNBArchImpl.h
+++ /dev/null
@@ -1,275 +0,0 @@
-//===-- DNBArchImpl.h -------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  Created by Greg Clayton on 6/25/07.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_ARM_DNBARCHIMPL_H
-#define LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_ARM_DNBARCHIMPL_H
-
-#if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
-
-#include "DNBArch.h"
-
-#include <map>
-
-class MachThread;
-
-class DNBArchMachARM : public DNBArchProtocol {
-public:
-  enum { kMaxNumThumbITBreakpoints = 4 };
-
-  DNBArchMachARM(MachThread *thread)
-      : m_thread(thread), m_state(), m_disabled_watchpoints(),
-        m_hw_single_chained_step_addr(INVALID_NUB_ADDRESS),
-        m_last_decode_pc(INVALID_NUB_ADDRESS), m_watchpoint_hw_index(-1),
-        m_watchpoint_did_occur(false),
-        m_watchpoint_resume_single_step_enabled(false),
-        m_saved_register_states() {
-    m_disabled_watchpoints.resize(16);
-    memset(&m_dbg_save, 0, sizeof(m_dbg_save));
-#if defined(USE_ARM_DISASSEMBLER_FRAMEWORK)
-    ThumbStaticsInit(&m_last_decode_thumb);
-#endif
-  }
-
-  virtual ~DNBArchMachARM() {}
-
-  static void Initialize();
-  static const DNBRegisterSetInfo *GetRegisterSetInfo(nub_size_t *num_reg_sets);
-
-  bool GetRegisterValue(uint32_t set, uint32_t reg,
-                        DNBRegisterValue *value) override;
-  bool SetRegisterValue(uint32_t set, uint32_t reg,
-                        const DNBRegisterValue *value) override;
-  nub_size_t GetRegisterContext(void *buf, nub_size_t buf_len) override;
-  nub_size_t SetRegisterContext(const void *buf, nub_size_t buf_len) override;
-  uint32_t SaveRegisterState() override;
-  bool RestoreRegisterState(uint32_t save_id) override;
-
-  kern_return_t GetRegisterState(int set, bool force) override;
-  kern_return_t SetRegisterState(int set) override;
-  bool RegisterSetStateIsValid(int set) const override;
-
-  uint64_t GetPC(uint64_t failValue) override; // Get program counter
-  kern_return_t SetPC(uint64_t value) override;
-  uint64_t GetSP(uint64_t failValue) override; // Get stack pointer
-  void ThreadWillResume() override;
-  bool ThreadDidStop() override;
-  bool NotifyException(MachException::Data &exc) override;
-
-  static DNBArchProtocol *Create(MachThread *thread);
-  static const uint8_t *SoftwareBreakpointOpcode(nub_size_t byte_size);
-  static uint32_t GetCPUType();
-
-  uint32_t NumSupportedHardwareBreakpoints() override;
-  uint32_t NumSupportedHardwareWatchpoints() override;
-  uint32_t EnableHardwareBreakpoint(nub_addr_t addr, nub_size_t size,
-                                    bool also_set_on_task) override;
-  bool DisableHardwareBreakpoint(uint32_t hw_break_index,
-                                 bool also_set_on_task) override;
-
-  uint32_t EnableHardwareWatchpoint(nub_addr_t addr, nub_size_t size, bool read,
-                                    bool write, bool also_set_on_task) override;
-  bool DisableHardwareWatchpoint(uint32_t hw_break_index,
-                                 bool also_set_on_task) override;
-  virtual bool DisableHardwareWatchpoint_helper(uint32_t hw_break_index,
-                                                bool also_set_on_task);
-  virtual bool ReenableHardwareWatchpoint(uint32_t hw_break_index);
-  virtual bool ReenableHardwareWatchpoint_helper(uint32_t hw_break_index);
-
-  virtual bool StepNotComplete() override;
-  uint32_t GetHardwareWatchpointHit(nub_addr_t &addr) override;
-
-#if defined(ARM_DEBUG_STATE32) && (defined(__arm64__) || defined(__aarch64__))
-  typedef arm_debug_state32_t DBG;
-#else
-  typedef arm_debug_state_t DBG;
-#endif
-
-protected:
-  kern_return_t EnableHardwareSingleStep(bool enable);
-  kern_return_t SetSingleStepSoftwareBreakpoints();
-
-  bool ConditionPassed(uint8_t condition, uint32_t cpsr);
-#if defined(USE_ARM_DISASSEMBLER_FRAMEWORK)
-  bool ComputeNextPC(nub_addr_t currentPC,
-                     arm_decoded_instruction_t decodedInstruction,
-                     bool currentPCIsThumb, nub_addr_t *targetPC);
-  arm_error_t DecodeInstructionUsingDisassembler(
-      nub_addr_t curr_pc, uint32_t curr_cpsr,
-      arm_decoded_instruction_t *decodedInstruction,
-      thumb_static_data_t *thumbStaticData, nub_addr_t *next_pc);
-  void DecodeITBlockInstructions(nub_addr_t curr_pc);
-#endif
-  void EvaluateNextInstructionForSoftwareBreakpointSetup(nub_addr_t currentPC,
-                                                         uint32_t cpsr,
-                                                         bool currentPCIsThumb,
-                                                         nub_addr_t *nextPC,
-                                                         bool *nextPCIsThumb);
-
-  enum RegisterSet {
-    e_regSetALL = REGISTER_SET_ALL,
-    e_regSetGPR, // ARM_THREAD_STATE
-    e_regSetVFP, // ARM_VFP_STATE (ARM_NEON_STATE if defined __arm64__)
-    e_regSetEXC, // ARM_EXCEPTION_STATE
-    e_regSetDBG, // ARM_DEBUG_STATE (ARM_DEBUG_STATE32 if defined __arm64__)
-    kNumRegisterSets
-  };
-
-  enum { Read = 0, Write = 1, kNumErrors = 2 };
-
-  typedef arm_thread_state_t GPR;
-#if defined(__arm64__) || defined(__aarch64__)
-  typedef arm_neon_state_t FPU;
-#else
-  typedef arm_vfp_state_t FPU;
-#endif
-  typedef arm_exception_state_t EXC;
-
-  static const DNBRegisterInfo g_gpr_registers[];
-  static const DNBRegisterInfo g_vfp_registers[];
-  static const DNBRegisterInfo g_exc_registers[];
-  static const DNBRegisterSetInfo g_reg_sets[];
-
-  static const size_t k_num_gpr_registers;
-  static const size_t k_num_vfp_registers;
-  static const size_t k_num_exc_registers;
-  static const size_t k_num_all_registers;
-  static const size_t k_num_register_sets;
-
-  struct Context {
-    GPR gpr;
-    FPU vfp;
-    EXC exc;
-  };
-
-  struct State {
-    Context context;
-    DBG dbg;
-    kern_return_t gpr_errs[2]; // Read/Write errors
-    kern_return_t vfp_errs[2]; // Read/Write errors
-    kern_return_t exc_errs[2]; // Read/Write errors
-    kern_return_t dbg_errs[2]; // Read/Write errors
-    State() {
-      uint32_t i;
-      for (i = 0; i < kNumErrors; i++) {
-        gpr_errs[i] = -1;
-        vfp_errs[i] = -1;
-        exc_errs[i] = -1;
-        dbg_errs[i] = -1;
-      }
-    }
-    void InvalidateRegisterSetState(int set) { SetError(set, Read, -1); }
-    kern_return_t GetError(int set, uint32_t err_idx) const {
-      if (err_idx < kNumErrors) {
-        switch (set) {
-        // When getting all errors, just OR all values together to see if
-        // we got any kind of error.
-        case e_regSetALL:
-          return gpr_errs[err_idx] | vfp_errs[err_idx] | exc_errs[err_idx] |
-                 dbg_errs[err_idx];
-        case e_regSetGPR:
-          return gpr_errs[err_idx];
-        case e_regSetVFP:
-          return vfp_errs[err_idx];
-        case e_regSetEXC:
-          return exc_errs[err_idx];
-        case e_regSetDBG:
-          return dbg_errs[err_idx];
-        default:
-          break;
-        }
-      }
-      return -1;
-    }
-    bool SetError(int set, uint32_t err_idx, kern_return_t err) {
-      if (err_idx < kNumErrors) {
-        switch (set) {
-        case e_regSetALL:
-          gpr_errs[err_idx] = err;
-          vfp_errs[err_idx] = err;
-          dbg_errs[err_idx] = err;
-          exc_errs[err_idx] = err;
-          return true;
-
-        case e_regSetGPR:
-          gpr_errs[err_idx] = err;
-          return true;
-
-        case e_regSetVFP:
-          vfp_errs[err_idx] = err;
-          return true;
-
-        case e_regSetEXC:
-          exc_errs[err_idx] = err;
-          return true;
-
-        case e_regSetDBG:
-          dbg_errs[err_idx] = err;
-          return true;
-        default:
-          break;
-        }
-      }
-      return false;
-    }
-    bool RegsAreValid(int set) const {
-      return GetError(set, Read) == KERN_SUCCESS;
-    }
-  };
-
-  kern_return_t GetGPRState(bool force);
-  kern_return_t GetVFPState(bool force);
-  kern_return_t GetEXCState(bool force);
-  kern_return_t GetDBGState(bool force);
-
-  kern_return_t SetGPRState();
-  kern_return_t SetVFPState();
-  kern_return_t SetEXCState();
-  kern_return_t SetDBGState(bool also_set_on_task);
-
-  bool IsWatchpointEnabled(const DBG &debug_state, uint32_t hw_index);
-  nub_addr_t GetWatchpointAddressByIndex(uint32_t hw_index);
-  nub_addr_t GetWatchAddress(const DBG &debug_state, uint32_t hw_index);
-
-  class disabled_watchpoint {
-  public:
-    disabled_watchpoint() {
-      addr = 0;
-      control = 0;
-    }
-    nub_addr_t addr;
-    uint32_t control;
-  };
-
-protected:
-  MachThread *m_thread;
-  State m_state;
-  DBG m_dbg_save;
-
-  // armv8 doesn't keep the disabled watchpoint values in the debug register
-  // context like armv7;
-  // we need to save them aside when we disable them temporarily.
-  std::vector<disabled_watchpoint> m_disabled_watchpoints;
-
-  nub_addr_t m_hw_single_chained_step_addr;
-  nub_addr_t m_last_decode_pc;
-
-  // The following member variables should be updated atomically.
-  int32_t m_watchpoint_hw_index;
-  bool m_watchpoint_did_occur;
-  bool m_watchpoint_resume_single_step_enabled;
-
-  typedef std::map<uint32_t, Context> SaveRegisterStates;
-  SaveRegisterStates m_saved_register_states;
-};
-
-#endif // #if defined (__arm__)
-#endif // LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_ARM_DNBARCHIMPL_H
diff --git a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
index 583417bbb01d1..bdecae81d31a7 100644
--- a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
+++ b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.cpp
@@ -42,11 +42,6 @@
 #define WCR_LOAD ((uint32_t)(1u << 3))
 #define WCR_STORE ((uint32_t)(1u << 4))
 
-// Enable breakpoint, watchpoint, and vector catch debug exceptions.
-// (MDE bit in the MDSCR_EL1 register.  Equivalent to the MDBGen bit in
-// DBGDSCRext in Aarch32)
-#define MDE_ENABLE ((uint32_t)(1u << 15))
-
 // Single instruction step
 // (SS bit in the MDSCR_EL1 register)
 #define SS_ENABLE ((uint32_t)(1u))
@@ -807,8 +802,6 @@ uint32_t DNBArchMachARM64::EnableHardwareBreakpoint(nub_addr_t addr,
                        (uint64_t)m_state.dbg.__bvr[i],
                        (uint32_t)m_state.dbg.__bcr[i]);
 
-      // The kernel will set the MDE_ENABLE bit in the MDSCR_EL1 for us
-      // automatically, don't need to do it here.
       kret = SetDBGState(also_set_on_task);
 
       DNBLogThreadedIf(LOG_WATCHPOINTS,
@@ -830,6 +823,70 @@ uint32_t DNBArchMachARM64::EnableHardwareBreakpoint(nub_addr_t addr,
   return INVALID_NUB_HW_INDEX;
 }
 
+std::vector<DNBArchMachARM64::WatchpointSpec>
+DNBArchMachARM64::AlignRequestedWatchpoint(nub_addr_t requested_addr,
+                                           nub_size_t requested_size) {
+
+  // Can't watch zero bytes
+  if (requested_size == 0)
+    return {};
+
+  // Smallest size we can watch on AArch64 is 8 bytes
+  constexpr nub_size_t min_watchpoint_alignment = 8;
+  nub_size_t aligned_size = std::max(requested_size, min_watchpoint_alignment);
+
+  // AArch64 addresses are 8 bytes.
+  constexpr int addr_byte_size = 8;
+  constexpr int addr_bit_size = addr_byte_size * 8;
+
+  /// Round up \a requested_size to the next power-of-2 size, at least 8
+  /// bytes
+  /// requested_size == 8   -> aligned_size == 8
+  /// requested_size == 9   -> aligned_size == 16
+  /// requested_size == 15  -> aligned_size == 16
+  /// requested_size == 192 -> aligned_size == 256
+  /// Could be `std::bit_ceil(aligned_size)` when we build with C++20?
+  aligned_size = 1ULL << (addr_bit_size - __builtin_clzll(aligned_size - 1));
+
+  nub_addr_t aligned_start = requested_addr & ~(aligned_size - 1);
+  // Does this power-of-2 memory range, aligned to power-of-2, completely
+  // encompass the requested watch region.
+  if (aligned_start + aligned_size >= requested_addr + requested_size) {
+    WatchpointSpec wp;
+    wp.aligned_start = aligned_start;
+    wp.requested_start = requested_addr;
+    wp.aligned_size = aligned_size;
+    wp.requested_size = requested_size;
+    return {{wp}};
+  }
+
+  // We need to split this into two watchpoints, split on the aligned_size
+  // boundary and re-evaluate the alignment of each half.
+  //
+  // requested_addr 48 requested_size 20 -> aligned_size 32
+  //                              aligned_start 32
+  //                              split_addr 64
+  //                              first_requested_addr 48
+  //                              first_requested_size 16
+  //                              second_requested_addr 64
+  //                              second_requested_size 4
+  nub_addr_t split_addr = aligned_start + aligned_size;
+
+  nub_addr_t first_requested_addr = requested_addr;
+  nub_size_t first_requested_size = split_addr - requested_addr;
+  nub_addr_t second_requested_addr = split_addr;
+  nub_size_t second_requested_size = requested_size - first_requested_size;
+
+  std::vector<WatchpointSpec> first_wp =
+      AlignRequestedWatchpoint(first_requested_addr, first_requested_size);
+  std::vector<WatchpointSpec> second_wp =
+      AlignRequestedWatchpoint(second_requested_addr, second_requested_size);
+  if (first_wp.size() != 1 || second_wp.size() != 1)
+    return {};
+
+  return {{first_wp[0], second_wp[0]}};
+}
+
 uint32_t DNBArchMachARM64::EnableHardwareWatchpoint(nub_addr_t addr,
                                                     nub_size_t size, bool read,
                                                     bool write,
@@ -839,91 +896,65 @@ uint32_t DNBArchMachARM64::EnableHardwareWatchpoint(nub_addr_t addr,
                    "0x%8.8llx, size = %zu, read = %u, write = %u)",
                    (uint64_t)addr, size, read, write);
 
-  const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints();
+  std::vector<DNBArchMachARM64::WatchpointSpec> wps =
+      AlignRequestedWatchpoint(addr, size);
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::EnableHardwareWatchpoint() using %zu "
+                   "hardware watchpoints",
+                   wps.size());
 
-  // Can't watch zero bytes
-  if (size == 0)
+  if (wps.size() == 0)
     return INVALID_NUB_HW_INDEX;
 
   // We must watch for either read or write
   if (read == false && write == false)
     return INVALID_NUB_HW_INDEX;
 
-  // Otherwise, can't watch more than 8 bytes per WVR/WCR pair
-  if (size > 8)
-    return INVALID_NUB_HW_INDEX;
+  // Only one hardware watchpoint needed
+  // to implement the user's request.
+  if (wps.size() == 1) {
+    if (wps[0].aligned_size <= 8)
+      return SetBASWatchpoint(wps[0], read, write, also_set_on_task);
+    else
+      return SetMASKWatchpoint(wps[0], read, write, also_set_on_task);
+  }
 
-  // Aarch64 watchpoints are in one of two forms: (1) 1-8 bytes, aligned to
-  // an 8 byte address, or (2) a power-of-two size region of memory; minimum
-  // 8 bytes, maximum 2GB; the starting address must be aligned to that power
-  // of two.
-  //
-  // For (1), 1-8 byte watchpoints, using the Byte Address Selector field in
-  // DBGWCR<n>.BAS.  Any of the bytes may be watched, but if multiple bytes
-  // are watched, the bytes selected must be contiguous.  The start address
-  // watched must be doubleword (8-byte) aligned; if the start address is
-  // word (4-byte) aligned, only 4 bytes can be watched.
-  //
-  // For (2), the MASK field in DBGWCR<n>.MASK is used.
-  //
-  // See the ARM ARM, section "Watchpoint exceptions", and more specifically,
-  // "Watchpoint data address comparisons".
-  //
-  // debugserver today only supports (1) - the Byte Address Selector 1-8 byte
-  // watchpoints that are 8-byte aligned.  To support larger watchpoints,
-  // debugserver would need to interpret the mach exception when the watched
-  // region was hit, see if the address accessed lies within the subset
-  // of the power-of-two region that lldb asked us to watch (v. ARM ARM,
-  // "Determining the memory location that caused a Watchpoint exception"),
-  // and silently resume the inferior (disable watchpoint, stepi, re-enable
-  // watchpoint) if the address lies outside the region that lldb asked us
-  // to watch.
-  //
-  // Alternatively, lldb would need to be prepared for a larger region
-  // being watched than it requested, and silently resume the inferior if
-  // the accessed address is outside the region lldb wants to watch.
-
-  nub_addr_t aligned_wp_address = addr & ~0x7;
-  uint32_t addr_dword_offset = addr & 0x7;
-
-  // Do we need to split up this logical watchpoint into two hardware watchpoint
-  // registers?
-  // e.g. a watchpoint of length 4 on address 6.  We need do this with
-  //   one watchpoint on address 0 with bytes 6 & 7 being monitored
-  //   one watchpoint on address 8 with bytes 0, 1, 2, 3 being monitored
-
-  if (addr_dword_offset + size > 8) {
-    DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM64::"
-                                      "EnableHardwareWatchpoint(addr = "
-                                      "0x%8.8llx, size = %zu) needs two "
-                                      "hardware watchpoints slots to monitor",
-                     (uint64_t)addr, size);
-    int low_watchpoint_size = 8 - addr_dword_offset;
-    int high_watchpoint_size = addr_dword_offset + size - 8;
-
-    uint32_t lo = EnableHardwareWatchpoint(addr, low_watchpoint_size, read,
-                                           write, also_set_on_task);
-    if (lo == INVALID_NUB_HW_INDEX)
-      return INVALID_NUB_HW_INDEX;
-    uint32_t hi =
-        EnableHardwareWatchpoint(aligned_wp_address + 8, high_watchpoint_size,
+  // We have multiple WatchpointSpecs
+
+  std::vector<uint32_t> wp_slots_used;
+  for (size_t i = 0; i < wps.size(); i++) {
+    uint32_t idx =
+        EnableHardwareWatchpoint(wps[i].requested_start, wps[i].requested_size,
                                  read, write, also_set_on_task);
-    if (hi == INVALID_NUB_HW_INDEX) {
-      DisableHardwareWatchpoint(lo, also_set_on_task);
-      return INVALID_NUB_HW_INDEX;
-    }
-    // Tag this lo->hi mapping in our database.
-    LoHi[lo] = hi;
-    return lo;
+    if (idx != INVALID_NUB_HW_INDEX)
+      wp_slots_used.push_back(idx);
+  }
+
+  // Did we fail to set all of the WatchpointSpecs needed
+  // for this user's request?
+  if (wps.size() != wp_slots_used.size()) {
+    for (int wp_slot : wp_slots_used)
+      DisableHardwareWatchpoint(wp_slot, also_set_on_task);
+    return INVALID_NUB_HW_INDEX;
   }
 
-  // At this point
-  //  1 aligned_wp_address is the requested address rounded down to 8-byte
-  //  alignment
-  //  2 addr_dword_offset is the offset into that double word (8-byte) region
-  //  that we are watching
-  //  3 size is the number of bytes within that 8-byte region that we are
-  //  watching
+  LoHi[wp_slots_used[0]] = wp_slots_used[1];
+  return wp_slots_used[0];
+}
+
+uint32_t DNBArchMachARM64::SetBASWatchpoint(DNBArchMachARM64::WatchpointSpec wp,
+                                            bool read, bool write,
+                                            bool also_set_on_task) {
+  const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints();
+
+  nub_addr_t aligned_dword_addr = wp.aligned_start;
+  nub_addr_t watching_offset = wp.requested_start - wp.aligned_start;
+  nub_size_t watching_size = wp.requested_size;
+
+  // If user asks to watch 3 bytes at 0x1005,
+  // aligned_dword_addr 0x1000
+  // watching_offset 5
+  // watching_size 3
 
   // Set the Byte Address Selects bits DBGWCRn_EL1 bits [12:5] based on the
   // above.
@@ -933,66 +964,147 @@ uint32_t DNBArchMachARM64::EnableHardwareWatchpoint(nub_addr_t addr,
   // interested in.
   // e.g. if we are watching bytes 4,5,6,7 in a dword we want a BAS of
   // 0b11110000.
-  uint32_t byte_address_select = ((1 << size) - 1) << addr_dword_offset;
+  uint32_t byte_address_select = ((1 << watching_size) - 1) << watching_offset;
 
   // Read the debug state
   kern_return_t kret = GetDBGState(false);
+  if (kret != KERN_SUCCESS)
+    return INVALID_NUB_HW_INDEX;
 
-  if (kret == KERN_SUCCESS) {
-    // Check to make sure we have the needed hardware support
-    uint32_t i = 0;
+  // Check to make sure we have the needed hardware support
+  uint32_t i = 0;
 
-    for (i = 0; i < num_hw_watchpoints; ++i) {
-      if ((m_state.dbg.__wcr[i] & WCR_ENABLE) == 0)
-        break; // We found an available hw watchpoint slot (in i)
-    }
+  for (i = 0; i < num_hw_watchpoints; ++i) {
+    if ((m_state.dbg.__wcr[i] & WCR_ENABLE) == 0)
+      break; // We found an available hw watchpoint slot
+  }
+  if (i == num_hw_watchpoints) {
+    DNBLogThreadedIf(LOG_WATCHPOINTS,
+                     "DNBArchMachARM64::"
+                     "SetBASWatchpoint(): All "
+                     "hardware resources (%u) are in use.",
+                     num_hw_watchpoints);
+    return INVALID_NUB_HW_INDEX;
+  }
 
-    // See if we found an available hw watchpoint slot above
-    if (i < num_hw_watchpoints) {
-      // DumpDBGState(m_state.dbg);
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::"
+                   "SetBASWatchpoint() "
+                   "set hardware register %d to BAS watchpoint "
+                   "aligned start address 0x%llx, watch region start "
+                   "offset %lld, number of bytes %zu",
+                   i, aligned_dword_addr, watching_offset, watching_size);
+
+  // Clear any previous LoHi joined-watchpoint that may have been in use
+  LoHi[i] = 0;
+
+  // shift our Byte Address Select bits up to the correct bit range for the
+  // DBGWCRn_EL1
+  byte_address_select = byte_address_select << 5;
+
+  // Make sure bits 1:0 are clear in our address
+  m_state.dbg.__wvr[i] = aligned_dword_addr;       // DVA (Data Virtual Address)
+  m_state.dbg.__wcr[i] = byte_address_select |     // Which bytes that follow
+                                                   // the DVA that we will watch
+                         S_USER |                  // Stop only in user mode
+                         (read ? WCR_LOAD : 0) |   // Stop on read access?
+                         (write ? WCR_STORE : 0) | // Stop on write access?
+                         WCR_ENABLE;               // Enable this watchpoint;
 
-      // Clear any previous LoHi joined-watchpoint that may have been in use
-      LoHi[i] = 0;
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::SetBASWatchpoint() "
+                   "adding watchpoint on address 0x%llx with control "
+                   "register value 0x%x",
+                   (uint64_t)m_state.dbg.__wvr[i],
+                   (uint32_t)m_state.dbg.__wcr[i]);
 
-      // shift our Byte Address Select bits up to the correct bit range for the
-      // DBGWCRn_EL1
-      byte_address_select = byte_address_select << 5;
+  kret = SetDBGState(also_set_on_task);
+  // DumpDBGState(m_state.dbg);
 
-      // Make sure bits 1:0 are clear in our address
-      m_state.dbg.__wvr[i] = aligned_wp_address;   // DVA (Data Virtual Address)
-      m_state.dbg.__wcr[i] = byte_address_select | // Which bytes that follow
-                                                   // the DVA that we will watch
-                             S_USER |              // Stop only in user mode
-                             (read ? WCR_LOAD : 0) |   // Stop on read access?
-                             (write ? WCR_STORE : 0) | // Stop on write access?
-                             WCR_ENABLE; // Enable this watchpoint;
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::"
+                   "SetBASWatchpoint() "
+                   "SetDBGState() => 0x%8.8x.",
+                   kret);
 
-      DNBLogThreadedIf(
-          LOG_WATCHPOINTS, "DNBArchMachARM64::EnableHardwareWatchpoint() "
-                           "adding watchpoint on address 0x%llx with control "
-                           "register value 0x%x",
-          (uint64_t)m_state.dbg.__wvr[i], (uint32_t)m_state.dbg.__wcr[i]);
+  if (kret == KERN_SUCCESS)
+    return i;
 
-      // The kernel will set the MDE_ENABLE bit in the MDSCR_EL1 for us
-      // automatically, don't need to do it here.
+  return INVALID_NUB_HW_INDEX;
+}
 
-      kret = SetDBGState(also_set_on_task);
-      // DumpDBGState(m_state.dbg);
+uint32_t
+DNBArchMachARM64::SetMASKWatchpoint(DNBArchMachARM64::WatchpointSpec wp,
+                                    bool read, bool write,
+                                    bool also_set_on_task) {
+  const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints();
 
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM64::"
-                                        "EnableHardwareWatchpoint() "
-                                        "SetDBGState() => 0x%8.8x.",
-                       kret);
+  // Read the debug state
+  kern_return_t kret = GetDBGState(false);
+  if (kret != KERN_SUCCESS)
+    return INVALID_NUB_HW_INDEX;
 
-      if (kret == KERN_SUCCESS)
-        return i;
-    } else {
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM64::"
-                                        "EnableHardwareWatchpoint(): All "
-                                        "hardware resources (%u) are in use.",
-                       num_hw_watchpoints);
-    }
+  // Check to make sure we have the needed hardware support
+  uint32_t i = 0;
+
+  for (i = 0; i < num_hw_watchpoints; ++i) {
+    if ((m_state.dbg.__wcr[i] & WCR_ENABLE) == 0)
+      break; // We found an available hw watchpoint slot
+  }
+  if (i == num_hw_watchpoints) {
+    DNBLogThreadedIf(LOG_WATCHPOINTS,
+                     "DNBArchMachARM64::"
+                     "SetMASKWatchpoint(): All "
+                     "hardware resources (%u) are in use.",
+                     num_hw_watchpoints);
+    return INVALID_NUB_HW_INDEX;
   }
+
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::"
+                   "SetMASKWatchpoint() "
+                   "set hardware register %d to MASK watchpoint "
+                   "aligned start address 0x%llx, aligned size %zu",
+                   i, wp.aligned_start, wp.aligned_size);
+
+  // Clear any previous LoHi joined-watchpoint that may have been in use
+  LoHi[i] = 0;
+
+  // MASK field is the number of low bits that are masked off
+  // when comparing the address with the DBGWVR<n>_EL1 values.
+  // If aligned size is 16, that means we ignore low 4 bits, 0b1111.
+  // popcount(16 - 1) give us the correct value of 4.
+  // 2GB is max watchable region, which is 31 bits (low bits 0x7fffffff
+  // masked off) -- a MASK value of 31.
+  const uint64_t mask = __builtin_popcountl(wp.aligned_size - 1) << 24;
+  // A '0b11111111' BAS value needed for mask watchpoints plus a
+  // nonzero mask value.
+  const uint64_t not_bas_wp = 0xff << 5;
+
+  m_state.dbg.__wvr[i] = wp.aligned_start;
+  m_state.dbg.__wcr[i] = mask | not_bas_wp | S_USER | // Stop only in user mode
+                         (read ? WCR_LOAD : 0) |      // Stop on read access?
+                         (write ? WCR_STORE : 0) |    // Stop on write access?
+                         WCR_ENABLE;                  // Enable this watchpoint;
+
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::SetMASKWatchpoint() "
+                   "adding watchpoint on address 0x%llx with control "
+                   "register value 0x%llx",
+                   (uint64_t)m_state.dbg.__wvr[i],
+                   (uint64_t)m_state.dbg.__wcr[i]);
+
+  kret = SetDBGState(also_set_on_task);
+
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::"
+                   "SetMASKWatchpoint() "
+                   "SetDBGState() => 0x%8.8x.",
+                   kret);
+
+  if (kret == KERN_SUCCESS)
+    return i;
+
   return INVALID_NUB_HW_INDEX;
 }
 
@@ -1020,15 +1132,13 @@ bool DNBArchMachARM64::ReenableHardwareWatchpoint_helper(uint32_t hw_index) {
   m_state.dbg.__wvr[hw_index] = m_disabled_watchpoints[hw_index].addr;
   m_state.dbg.__wcr[hw_index] = m_disabled_watchpoints[hw_index].control;
 
-  DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchMachARM64::"
-                                    "EnableHardwareWatchpoint( %u ) - WVR%u = "
-                                    "0x%8.8llx  WCR%u = 0x%8.8llx",
+  DNBLogThreadedIf(LOG_WATCHPOINTS,
+                   "DNBArchMachARM64::"
+                   "ReenableHardwareWatchpoint_helper( %u ) - WVR%u = "
+                   "0x%8.8llx  WCR%u = 0x%8.8llx",
                    hw_index, hw_index, (uint64_t)m_state.dbg.__wvr[hw_index],
                    hw_index, (uint64_t)m_state.dbg.__wcr[hw_index]);
 
-  // The kernel will set the MDE_ENABLE bit in the MDSCR_EL1 for us
-  // automatically, don't need to do it here.
-
   kret = SetDBGState(false);
 
   return (kret == KERN_SUCCESS);
@@ -1130,30 +1240,61 @@ uint32_t DNBArchMachARM64::GetHardwareWatchpointHit(nub_addr_t &addr) {
     uint32_t i, num = NumSupportedHardwareWatchpoints();
     for (i = 0; i < num; ++i) {
       nub_addr_t wp_addr = GetWatchAddress(debug_state, i);
-      uint32_t byte_mask = bits(debug_state.__wcr[i], 12, 5);
 
       DNBLogThreadedIf(LOG_WATCHPOINTS,
                        "DNBArchImplARM64::"
                        "GetHardwareWatchpointHit() slot: %u "
-                       "(addr = 0x%llx; byte_mask = 0x%x)",
-                       i, static_cast<uint64_t>(wp_addr), byte_mask);
+                       "(addr = 0x%llx, WCR = 0x%llx)",
+                       i, wp_addr, debug_state.__wcr[i]);
 
       if (!IsWatchpointEnabled(debug_state, i))
         continue;
 
-      if (bits(wp_addr, 48, 3) != bits(addr, 48, 3))
-        continue;
+      // DBGWCR<n>EL1.BAS are the bits of the doubleword that are watched
+      // with a BAS watchpoint.
+      uint32_t bas_bits = bits(debug_state.__wcr[i], 12, 5);
+      // DBGWCR<n>EL1.MASK is the number of bits that are masked off the
+      // virtual address when comparing to DBGWVR<n>_EL1.
+      uint32_t mask = bits(debug_state.__wcr[i], 28, 24);
 
-      // Sanity check the byte_mask
-      uint32_t lsb = LowestBitSet(byte_mask);
-      if (lsb < 0)
-        continue;
+      const bool is_bas_watchpoint = mask == 0;
 
-      uint64_t byte_to_match = bits(addr, 2, 0);
+      DNBLogThreadedIf(
+          LOG_WATCHPOINTS,
+          "DNBArchImplARM64::"
+          "GetHardwareWatchpointHit() slot: %u %s",
+          i, is_bas_watchpoint ? "is BAS watchpoint" : "is MASK watchpoint");
+
+      if (is_bas_watchpoint) {
+        if (bits(wp_addr, 48, 3) != bits(addr, 48, 3))
+          continue;
+      } else {
+        if (bits(wp_addr, 48, mask) == bits(addr, 48, mask)) {
+          DNBLogThreadedIf(LOG_WATCHPOINTS,
+                           "DNBArchImplARM64::"
+                           "GetHardwareWatchpointHit() slot: %u matched MASK "
+                           "ignoring %u low bits",
+                           i, mask);
+          return i;
+        }
+      }
 
-      if (byte_mask & (1 << byte_to_match)) {
-        addr = wp_addr + lsb;
-        return i;
+      if (is_bas_watchpoint) {
+        // Sanity check the bas_bits
+        uint32_t lsb = LowestBitSet(bas_bits);
+        if (lsb < 0)
+          continue;
+
+        uint64_t byte_to_match = bits(addr, 2, 0);
+
+        if (bas_bits & (1 << byte_to_match)) {
+          addr = wp_addr + lsb;
+          DNBLogThreadedIf(LOG_WATCHPOINTS,
+                           "DNBArchImplARM64::"
+                           "GetHardwareWatchpointHit() slot: %u matched BAS",
+                           i);
+          return i;
+        }
       }
     }
   }
diff --git a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.h b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.h
index b492fcaca94b5..0ea33d8e1c4c5 100644
--- a/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.h
+++ b/lldb/tools/debugserver/source/MacOSX/arm64/DNBArchImplARM64.h
@@ -13,6 +13,7 @@
 
 #include <mach/thread_status.h>
 #include <map>
+#include <vector>
 
 #if defined(ARM_THREAD_STATE64_COUNT)
 
@@ -35,6 +36,13 @@ class DNBArchMachARM64 : public DNBArchProtocol {
     memset(&m_dbg_save, 0, sizeof(m_dbg_save));
   }
 
+  struct WatchpointSpec {
+    nub_addr_t aligned_start;
+    nub_addr_t requested_start;
+    nub_size_t aligned_size;
+    nub_size_t requested_size;
+  };
+
   virtual ~DNBArchMachARM64() {}
 
   static void Initialize();
@@ -71,8 +79,15 @@ class DNBArchMachARM64 : public DNBArchProtocol {
                                     bool also_set_on_task) override;
   bool DisableHardwareBreakpoint(uint32_t hw_break_index,
                                  bool also_set_on_task) override;
+  std::vector<WatchpointSpec>
+  AlignRequestedWatchpoint(nub_addr_t requested_addr,
+                           nub_size_t requested_size);
   uint32_t EnableHardwareWatchpoint(nub_addr_t addr, nub_size_t size, bool read,
                                     bool write, bool also_set_on_task) override;
+  uint32_t SetBASWatchpoint(WatchpointSpec wp, bool read, bool write,
+                            bool also_set_on_task);
+  uint32_t SetMASKWatchpoint(WatchpointSpec wp, bool read, bool write,
+                             bool also_set_on_task);
   bool DisableHardwareWatchpoint(uint32_t hw_break_index,
                                  bool also_set_on_task) override;
   bool DisableHardwareWatchpoint_helper(uint32_t hw_break_index,
diff --git a/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp b/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp
deleted file mode 100644
index 2b1d360dcae5f..0000000000000
--- a/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp
+++ /dev/null
@@ -1,2532 +0,0 @@
-//===-- DNBArchImplI386.cpp -------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  Created by Greg Clayton on 6/25/07.
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(__i386__) || defined(__x86_64__)
-
-#include <sys/cdefs.h>
-
-#include "DNBLog.h"
-#include "MacOSX/i386/DNBArchImplI386.h"
-#include "MachProcess.h"
-#include "MachThread.h"
-
-extern "C" bool CPUHasAVX(); // Defined over in DNBArchImplX86_64.cpp
-extern "C" bool CPUHasAVX512f(); // Defined over in DNBArchImplX86_64.cpp
-#if defined(LLDB_DEBUGSERVER_RELEASE) || defined(LLDB_DEBUGSERVER_DEBUG)
-enum debugState { debugStateUnknown, debugStateOff, debugStateOn };
-
-static debugState sFPUDebugState = debugStateUnknown;
-static debugState sAVXForceState = debugStateUnknown;
-
-static bool DebugFPURegs() {
-  if (sFPUDebugState == debugStateUnknown) {
-    if (getenv("DNB_DEBUG_FPU_REGS"))
-      sFPUDebugState = debugStateOn;
-    else
-      sFPUDebugState = debugStateOff;
-  }
-
-  return (sFPUDebugState == debugStateOn);
-}
-
-static bool ForceAVXRegs() {
-  if (sFPUDebugState == debugStateUnknown) {
-    if (getenv("DNB_DEBUG_X86_FORCE_AVX_REGS"))
-      sAVXForceState = debugStateOn;
-    else
-      sAVXForceState = debugStateOff;
-  }
-
-  return (sAVXForceState == debugStateOn);
-}
-
-#define DEBUG_FPU_REGS (DebugFPURegs())
-#define FORCE_AVX_REGS (ForceAVXRegs())
-#else
-#define DEBUG_FPU_REGS (0)
-#define FORCE_AVX_REGS (0)
-#endif
-
-enum {
-  gpr_eax = 0,
-  gpr_ebx = 1,
-  gpr_ecx = 2,
-  gpr_edx = 3,
-  gpr_edi = 4,
-  gpr_esi = 5,
-  gpr_ebp = 6,
-  gpr_esp = 7,
-  gpr_ss = 8,
-  gpr_eflags = 9,
-  gpr_eip = 10,
-  gpr_cs = 11,
-  gpr_ds = 12,
-  gpr_es = 13,
-  gpr_fs = 14,
-  gpr_gs = 15,
-  gpr_ax,
-  gpr_bx,
-  gpr_cx,
-  gpr_dx,
-  gpr_di,
-  gpr_si,
-  gpr_bp,
-  gpr_sp,
-  gpr_ah,
-  gpr_bh,
-  gpr_ch,
-  gpr_dh,
-  gpr_al,
-  gpr_bl,
-  gpr_cl,
-  gpr_dl,
-  gpr_dil,
-  gpr_sil,
-  gpr_bpl,
-  gpr_spl,
-  k_num_gpr_regs
-};
-
-enum {
-  fpu_fcw,
-  fpu_fsw,
-  fpu_ftw,
-  fpu_fop,
-  fpu_ip,
-  fpu_cs,
-  fpu_dp,
-  fpu_ds,
-  fpu_mxcsr,
-  fpu_mxcsrmask,
-  fpu_stmm0,
-  fpu_stmm1,
-  fpu_stmm2,
-  fpu_stmm3,
-  fpu_stmm4,
-  fpu_stmm5,
-  fpu_stmm6,
-  fpu_stmm7,
-  fpu_xmm0,
-  fpu_xmm1,
-  fpu_xmm2,
-  fpu_xmm3,
-  fpu_xmm4,
-  fpu_xmm5,
-  fpu_xmm6,
-  fpu_xmm7,
-  fpu_ymm0,
-  fpu_ymm1,
-  fpu_ymm2,
-  fpu_ymm3,
-  fpu_ymm4,
-  fpu_ymm5,
-  fpu_ymm6,
-  fpu_ymm7,
-  fpu_k0,
-  fpu_k1,
-  fpu_k2,
-  fpu_k3,
-  fpu_k4,
-  fpu_k5,
-  fpu_k6,
-  fpu_k7,
-  fpu_zmm0,
-  fpu_zmm1,
-  fpu_zmm2,
-  fpu_zmm3,
-  fpu_zmm4,
-  fpu_zmm5,
-  fpu_zmm6,
-  fpu_zmm7,
-  k_num_fpu_regs,
-
-  // Aliases
-  fpu_fctrl = fpu_fcw,
-  fpu_fstat = fpu_fsw,
-  fpu_ftag = fpu_ftw,
-  fpu_fiseg = fpu_cs,
-  fpu_fioff = fpu_ip,
-  fpu_foseg = fpu_ds,
-  fpu_fooff = fpu_dp
-};
-
-enum {
-  exc_trapno,
-  exc_err,
-  exc_faultvaddr,
-  k_num_exc_regs,
-};
-
-enum {
-  ehframe_eax = 0,
-  ehframe_ecx,
-  ehframe_edx,
-  ehframe_ebx,
-
-  // On i386 Darwin the eh_frame register numbers for ebp and esp are reversed
-  // from DWARF.
-  // It's due to an ancient compiler bug in the output of the eh_frame.
-  // Specifically, on i386 darwin eh_frame, 4 is ebp, 5 is esp.
-  // On i386 darwin debug_frame (and debug_info), 4 is esp, 5 is ebp.
-  ehframe_ebp,
-  ehframe_esp,
-  ehframe_esi,
-  ehframe_edi,
-  ehframe_eip,
-  ehframe_eflags
-};
-
-enum {
-  dwarf_eax = 0,
-  dwarf_ecx,
-  dwarf_edx,
-  dwarf_ebx,
-  dwarf_esp,
-  dwarf_ebp,
-  dwarf_esi,
-  dwarf_edi,
-  dwarf_eip,
-  dwarf_eflags,
-  dwarf_stmm0 = 11,
-  dwarf_stmm1,
-  dwarf_stmm2,
-  dwarf_stmm3,
-  dwarf_stmm4,
-  dwarf_stmm5,
-  dwarf_stmm6,
-  dwarf_stmm7,
-  dwarf_xmm0 = 21,
-  dwarf_xmm1,
-  dwarf_xmm2,
-  dwarf_xmm3,
-  dwarf_xmm4,
-  dwarf_xmm5,
-  dwarf_xmm6,
-  dwarf_xmm7,
-  dwarf_ymm0 = dwarf_xmm0,
-  dwarf_ymm1 = dwarf_xmm1,
-  dwarf_ymm2 = dwarf_xmm2,
-  dwarf_ymm3 = dwarf_xmm3,
-  dwarf_ymm4 = dwarf_xmm4,
-  dwarf_ymm5 = dwarf_xmm5,
-  dwarf_ymm6 = dwarf_xmm6,
-  dwarf_ymm7 = dwarf_xmm7,
-  dwarf_zmm0 = dwarf_xmm0,
-  dwarf_zmm1 = dwarf_xmm1,
-  dwarf_zmm2 = dwarf_xmm2,
-  dwarf_zmm3 = dwarf_xmm3,
-  dwarf_zmm4 = dwarf_xmm4,
-  dwarf_zmm5 = dwarf_xmm5,
-  dwarf_zmm6 = dwarf_xmm6,
-  dwarf_zmm7 = dwarf_xmm7,
-  dwarf_k0 = 118,
-  dwarf_k1,
-  dwarf_k2,
-  dwarf_k3,
-  dwarf_k4,
-  dwarf_k5,
-  dwarf_k6,
-  dwarf_k7,
-};
-
-enum {
-  debugserver_eax = 0,
-  debugserver_ecx = 1,
-  debugserver_edx = 2,
-  debugserver_ebx = 3,
-  debugserver_esp = 4,
-  debugserver_ebp = 5,
-  debugserver_esi = 6,
-  debugserver_edi = 7,
-  debugserver_eip = 8,
-  debugserver_eflags = 9,
-  debugserver_cs = 10,
-  debugserver_ss = 11,
-  debugserver_ds = 12,
-  debugserver_es = 13,
-  debugserver_fs = 14,
-  debugserver_gs = 15,
-  debugserver_stmm0 = 16,
-  debugserver_stmm1 = 17,
-  debugserver_stmm2 = 18,
-  debugserver_stmm3 = 19,
-  debugserver_stmm4 = 20,
-  debugserver_stmm5 = 21,
-  debugserver_stmm6 = 22,
-  debugserver_stmm7 = 23,
-  debugserver_fctrl = 24,
-  debugserver_fcw = debugserver_fctrl,
-  debugserver_fstat = 25,
-  debugserver_fsw = debugserver_fstat,
-  debugserver_ftag = 26,
-  debugserver_ftw = debugserver_ftag,
-  debugserver_fiseg = 27,
-  debugserver_fpu_cs = debugserver_fiseg,
-  debugserver_fioff = 28,
-  debugserver_ip = debugserver_fioff,
-  debugserver_foseg = 29,
-  debugserver_fpu_ds = debugserver_foseg,
-  debugserver_fooff = 30,
-  debugserver_dp = debugserver_fooff,
-  debugserver_fop = 31,
-  debugserver_xmm0 = 32,
-  debugserver_xmm1 = 33,
-  debugserver_xmm2 = 34,
-  debugserver_xmm3 = 35,
-  debugserver_xmm4 = 36,
-  debugserver_xmm5 = 37,
-  debugserver_xmm6 = 38,
-  debugserver_xmm7 = 39,
-  debugserver_mxcsr = 40,
-  debugserver_mm0 = 41,
-  debugserver_mm1 = 42,
-  debugserver_mm2 = 43,
-  debugserver_mm3 = 44,
-  debugserver_mm4 = 45,
-  debugserver_mm5 = 46,
-  debugserver_mm6 = 47,
-  debugserver_mm7 = 48,
-  debugserver_ymm0 = debugserver_xmm0,
-  debugserver_ymm1 = debugserver_xmm1,
-  debugserver_ymm2 = debugserver_xmm2,
-  debugserver_ymm3 = debugserver_xmm3,
-  debugserver_ymm4 = debugserver_xmm4,
-  debugserver_ymm5 = debugserver_xmm5,
-  debugserver_ymm6 = debugserver_xmm6,
-  debugserver_ymm7 = debugserver_xmm7,
-  debugserver_zmm0 = debugserver_xmm0,
-  debugserver_zmm1 = debugserver_xmm1,
-  debugserver_zmm2 = debugserver_xmm2,
-  debugserver_zmm3 = debugserver_xmm3,
-  debugserver_zmm4 = debugserver_xmm4,
-  debugserver_zmm5 = debugserver_xmm5,
-  debugserver_zmm6 = debugserver_xmm6,
-  debugserver_zmm7 = debugserver_xmm7,
-  debugserver_k0 = 118,
-  debugserver_k1 = 119,
-  debugserver_k2 = 120,
-  debugserver_k3 = 121,
-  debugserver_k4 = 122,
-  debugserver_k5 = 123,
-  debugserver_k6 = 124,
-  debugserver_k7 = 125,
-};
-
-uint64_t DNBArchImplI386::GetPC(uint64_t failValue) {
-  // Get program counter
-  if (GetGPRState(false) == KERN_SUCCESS)
-    return m_state.context.gpr.__eip;
-  return failValue;
-}
-
-kern_return_t DNBArchImplI386::SetPC(uint64_t value) {
-  // Get program counter
-  kern_return_t err = GetGPRState(false);
-  if (err == KERN_SUCCESS) {
-    m_state.context.gpr.__eip = static_cast<uint32_t>(value);
-    err = SetGPRState();
-  }
-  return err == KERN_SUCCESS;
-}
-
-uint64_t DNBArchImplI386::GetSP(uint64_t failValue) {
-  // Get stack pointer
-  if (GetGPRState(false) == KERN_SUCCESS)
-    return m_state.context.gpr.__esp;
-  return failValue;
-}
-
-// Uncomment the value below to verify the values in the debugger.
-//#define DEBUG_GPR_VALUES 1    // DO NOT CHECK IN WITH THIS DEFINE ENABLED
-//#define SET_GPR(reg) m_state.context.gpr.__##reg = gpr_##reg
-
-kern_return_t DNBArchImplI386::GetGPRState(bool force) {
-  if (force || m_state.GetError(e_regSetGPR, Read)) {
-#if DEBUG_GPR_VALUES
-    SET_GPR(eax);
-    SET_GPR(ebx);
-    SET_GPR(ecx);
-    SET_GPR(edx);
-    SET_GPR(edi);
-    SET_GPR(esi);
-    SET_GPR(ebp);
-    SET_GPR(esp);
-    SET_GPR(ss);
-    SET_GPR(eflags);
-    SET_GPR(eip);
-    SET_GPR(cs);
-    SET_GPR(ds);
-    SET_GPR(es);
-    SET_GPR(fs);
-    SET_GPR(gs);
-    m_state.SetError(e_regSetGPR, Read, 0);
-#else
-    mach_msg_type_number_t count = e_regSetWordSizeGPR;
-    m_state.SetError(
-        e_regSetGPR, Read,
-        ::thread_get_state(m_thread->MachPortNumber(), __i386_THREAD_STATE,
-                           (thread_state_t)&m_state.context.gpr, &count));
-#endif
-  }
-  return m_state.GetError(e_regSetGPR, Read);
-}
-
-// Uncomment the value below to verify the values in the debugger.
-//#define DEBUG_FPU_VALUES 1    // DO NOT CHECK IN WITH THIS DEFINE ENABLED
-
-kern_return_t DNBArchImplI386::GetFPUState(bool force) {
-  if (force || m_state.GetError(e_regSetFPU, Read)) {
-    if (DEBUG_FPU_REGS) {
-
-      m_state.context.fpu.no_avx.__fpu_reserved[0] = -1;
-      m_state.context.fpu.no_avx.__fpu_reserved[1] = -1;
-      *(uint16_t *)&(m_state.context.fpu.no_avx.__fpu_fcw) = 0x1234;
-      *(uint16_t *)&(m_state.context.fpu.no_avx.__fpu_fsw) = 0x5678;
-      m_state.context.fpu.no_avx.__fpu_ftw = 1;
-      m_state.context.fpu.no_avx.__fpu_rsrv1 = UINT8_MAX;
-      m_state.context.fpu.no_avx.__fpu_fop = 2;
-      m_state.context.fpu.no_avx.__fpu_ip = 3;
-      m_state.context.fpu.no_avx.__fpu_cs = 4;
-      m_state.context.fpu.no_avx.__fpu_rsrv2 = 5;
-      m_state.context.fpu.no_avx.__fpu_dp = 6;
-      m_state.context.fpu.no_avx.__fpu_ds = 7;
-      m_state.context.fpu.no_avx.__fpu_rsrv3 = UINT16_MAX;
-      m_state.context.fpu.no_avx.__fpu_mxcsr = 8;
-      m_state.context.fpu.no_avx.__fpu_mxcsrmask = 9;
-      for (int i = 0; i < 16; ++i) {
-        if (i < 10) {
-          m_state.context.fpu.no_avx.__fpu_stmm0.__mmst_reg[i] = 'a';
-          m_state.context.fpu.no_avx.__fpu_stmm1.__mmst_reg[i] = 'b';
-          m_state.context.fpu.no_avx.__fpu_stmm2.__mmst_reg[i] = 'c';
-          m_state.context.fpu.no_avx.__fpu_stmm3.__mmst_reg[i] = 'd';
-          m_state.context.fpu.no_avx.__fpu_stmm4.__mmst_reg[i] = 'e';
-          m_state.context.fpu.no_avx.__fpu_stmm5.__mmst_reg[i] = 'f';
-          m_state.context.fpu.no_avx.__fpu_stmm6.__mmst_reg[i] = 'g';
-          m_state.context.fpu.no_avx.__fpu_stmm7.__mmst_reg[i] = 'h';
-        } else {
-          m_state.context.fpu.no_avx.__fpu_stmm0.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm1.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm2.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm3.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm4.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm5.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm6.__mmst_reg[i] = INT8_MIN;
-          m_state.context.fpu.no_avx.__fpu_stmm7.__mmst_reg[i] = INT8_MIN;
-        }
-
-        m_state.context.fpu.no_avx.__fpu_xmm0.__xmm_reg[i] = '0';
-        m_state.context.fpu.no_avx.__fpu_xmm1.__xmm_reg[i] = '1';
-        m_state.context.fpu.no_avx.__fpu_xmm2.__xmm_reg[i] = '2';
-        m_state.context.fpu.no_avx.__fpu_xmm3.__xmm_reg[i] = '3';
-        m_state.context.fpu.no_avx.__fpu_xmm4.__xmm_reg[i] = '4';
-        m_state.context.fpu.no_avx.__fpu_xmm5.__xmm_reg[i] = '5';
-        m_state.context.fpu.no_avx.__fpu_xmm6.__xmm_reg[i] = '6';
-        m_state.context.fpu.no_avx.__fpu_xmm7.__xmm_reg[i] = '7';
-      }
-      for (int i = 0; i < sizeof(m_state.context.fpu.no_avx.__fpu_rsrv4); ++i)
-        m_state.context.fpu.no_avx.__fpu_rsrv4[i] = INT8_MIN;
-      m_state.context.fpu.no_avx.__fpu_reserved1 = -1;
-
-      if (CPUHasAVX() || FORCE_AVX_REGS) {
-        for (int i = 0; i < sizeof(m_state.context.fpu.avx.__avx_reserved1);
-             ++i)
-          m_state.context.fpu.avx.__avx_reserved1[i] = INT8_MIN;
-
-        for (int i = 0; i < 16; ++i) {
-          m_state.context.fpu.avx.__fpu_ymmh0.__xmm_reg[i] = '0';
-          m_state.context.fpu.avx.__fpu_ymmh1.__xmm_reg[i] = '1';
-          m_state.context.fpu.avx.__fpu_ymmh2.__xmm_reg[i] = '2';
-          m_state.context.fpu.avx.__fpu_ymmh3.__xmm_reg[i] = '3';
-          m_state.context.fpu.avx.__fpu_ymmh4.__xmm_reg[i] = '4';
-          m_state.context.fpu.avx.__fpu_ymmh5.__xmm_reg[i] = '5';
-          m_state.context.fpu.avx.__fpu_ymmh6.__xmm_reg[i] = '6';
-          m_state.context.fpu.avx.__fpu_ymmh7.__xmm_reg[i] = '7';
-        }
-      }
-      if (CPUHasAVX512f() || FORCE_AVX_REGS) {
-        for (int i = 0; i < 8; ++i) {
-          m_state.context.fpu.avx512f.__fpu_k0.__opmask_reg[i] = '0';
-          m_state.context.fpu.avx512f.__fpu_k1.__opmask_reg[i] = '1';
-          m_state.context.fpu.avx512f.__fpu_k2.__opmask_reg[i] = '2';
-          m_state.context.fpu.avx512f.__fpu_k3.__opmask_reg[i] = '3';
-          m_state.context.fpu.avx512f.__fpu_k4.__opmask_reg[i] = '4';
-          m_state.context.fpu.avx512f.__fpu_k5.__opmask_reg[i] = '5';
-          m_state.context.fpu.avx512f.__fpu_k6.__opmask_reg[i] = '6';
-          m_state.context.fpu.avx512f.__fpu_k7.__opmask_reg[i] = '7';
-        }
-
-        for (int i = 0; i < 32; ++i) {
-          m_state.context.fpu.avx512f.__fpu_zmmh0.__ymm_reg[i] = '0';
-          m_state.context.fpu.avx512f.__fpu_zmmh1.__ymm_reg[i] = '1';
-          m_state.context.fpu.avx512f.__fpu_zmmh2.__ymm_reg[i] = '2';
-          m_state.context.fpu.avx512f.__fpu_zmmh3.__ymm_reg[i] = '3';
-          m_state.context.fpu.avx512f.__fpu_zmmh4.__ymm_reg[i] = '4';
-          m_state.context.fpu.avx512f.__fpu_zmmh5.__ymm_reg[i] = '5';
-          m_state.context.fpu.avx512f.__fpu_zmmh6.__ymm_reg[i] = '6';
-          m_state.context.fpu.avx512f.__fpu_zmmh7.__ymm_reg[i] = '7';
-        }
-      }
-      m_state.SetError(e_regSetFPU, Read, 0);
-    } else {
-      mach_msg_type_number_t count = e_regSetWordSizeFPU;
-      int flavor = __i386_FLOAT_STATE;
-
-      // On a machine with the AVX512 register set, a process only gets a
-      // full AVX512 register context after it uses the AVX512 registers;
-      // if the process has not yet triggered this change, trying to fetch
-      // the AVX512 registers will fail.  Fall through to fetching the AVX
-      // registers.
-      if (CPUHasAVX512f() || FORCE_AVX_REGS) {
-        count = e_regSetWordSizeAVX512f;
-        flavor = __i386_AVX512F_STATE;
-        m_state.SetError(e_regSetFPU, Read,
-                         ::thread_get_state(m_thread->MachPortNumber(), flavor,
-                                            (thread_state_t)&m_state.context.fpu,
-                                            &count));
-        DNBLogThreadedIf(LOG_THREAD,
-                         "::thread_get_state (0x%4.4x, %u, &fpu, %u => 0x%8.8x",
-                         m_thread->MachPortNumber(), flavor, (uint32_t)count,
-                         m_state.GetError(e_regSetFPU, Read));
-        if (m_state.GetError(e_regSetFPU, Read) == KERN_SUCCESS)
-          return m_state.GetError(e_regSetFPU, Read);
-      }
-      if (CPUHasAVX()) {
-        count = e_regSetWordSizeAVX;
-        flavor = __i386_AVX_STATE;
-      }
-      m_state.SetError(e_regSetFPU, Read,
-                       ::thread_get_state(m_thread->MachPortNumber(), flavor,
-                                          (thread_state_t)&m_state.context.fpu,
-                                          &count));
-      DNBLogThreadedIf(LOG_THREAD,
-                       "::thread_get_state (0x%4.4x, %u, &fpu, %u => 0x%8.8x",
-                       m_thread->MachPortNumber(), flavor, (uint32_t)count,
-                       m_state.GetError(e_regSetFPU, Read));
-    }
-  }
-  return m_state.GetError(e_regSetFPU, Read);
-}
-
-kern_return_t DNBArchImplI386::GetEXCState(bool force) {
-  if (force || m_state.GetError(e_regSetEXC, Read)) {
-    mach_msg_type_number_t count = e_regSetWordSizeEXC;
-    m_state.SetError(
-        e_regSetEXC, Read,
-        ::thread_get_state(m_thread->MachPortNumber(), __i386_EXCEPTION_STATE,
-                           (thread_state_t)&m_state.context.exc, &count));
-  }
-  return m_state.GetError(e_regSetEXC, Read);
-}
-
-kern_return_t DNBArchImplI386::SetGPRState() {
-  kern_return_t kret = ::thread_abort_safely(m_thread->MachPortNumber());
-  DNBLogThreadedIf(
-      LOG_THREAD, "thread = 0x%4.4x calling thread_abort_safely (tid) => %u "
-                  "(SetGPRState() for stop_count = %u)",
-      m_thread->MachPortNumber(), kret, m_thread->Process()->StopCount());
-
-  m_state.SetError(e_regSetGPR, Write,
-                   ::thread_set_state(m_thread->MachPortNumber(),
-                                      __i386_THREAD_STATE,
-                                      (thread_state_t)&m_state.context.gpr,
-                                      e_regSetWordSizeGPR));
-  return m_state.GetError(e_regSetGPR, Write);
-}
-
-kern_return_t DNBArchImplI386::SetFPUState() {
-  if (DEBUG_FPU_REGS) {
-    m_state.SetError(e_regSetFPU, Write, 0);
-    return m_state.GetError(e_regSetFPU, Write);
-  } else {
-    int flavor = __i386_FLOAT_STATE;
-    mach_msg_type_number_t count = e_regSetWordSizeFPU;
-    if (CPUHasAVX512f() || FORCE_AVX_REGS) {
-      flavor = __i386_AVX512F_STATE;
-      count = e_regSetWordSizeAVX512f;
-    } else 
-    if (CPUHasAVX()) {
-      flavor = __i386_AVX_STATE;
-      count = e_regSetWordSizeAVX;
-    }
-
-    m_state.SetError(e_regSetFPU, Write,
-                     ::thread_set_state(m_thread->MachPortNumber(), flavor,
-                                        (thread_state_t)&m_state.context.fpu,
-                                        count));
-    return m_state.GetError(e_regSetFPU, Write);
-  }
-}
-
-kern_return_t DNBArchImplI386::SetEXCState() {
-  m_state.SetError(e_regSetEXC, Write,
-                   ::thread_set_state(m_thread->MachPortNumber(),
-                                      __i386_EXCEPTION_STATE,
-                                      (thread_state_t)&m_state.context.exc,
-                                      e_regSetWordSizeEXC));
-  return m_state.GetError(e_regSetEXC, Write);
-}
-
-kern_return_t DNBArchImplI386::GetDBGState(bool force) {
-  if (force || m_state.GetError(e_regSetDBG, Read)) {
-    mach_msg_type_number_t count = e_regSetWordSizeDBG;
-    m_state.SetError(
-        e_regSetDBG, Read,
-        ::thread_get_state(m_thread->MachPortNumber(), __i386_DEBUG_STATE,
-                           (thread_state_t)&m_state.context.dbg, &count));
-  }
-  return m_state.GetError(e_regSetDBG, Read);
-}
-
-kern_return_t DNBArchImplI386::SetDBGState(bool also_set_on_task) {
-  m_state.SetError(e_regSetDBG, Write,
-                   ::thread_set_state(m_thread->MachPortNumber(),
-                                      __i386_DEBUG_STATE,
-                                      (thread_state_t)&m_state.context.dbg,
-                                      e_regSetWordSizeDBG));
-  if (also_set_on_task) {
-    kern_return_t kret = ::task_set_state(
-        m_thread->Process()->Task().TaskPort(), __i386_DEBUG_STATE,
-        (thread_state_t)&m_state.context.dbg, e_regSetWordSizeDBG);
-    if (kret != KERN_SUCCESS)
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchImplI386::SetDBGState failed "
-                                        "to set debug control register state: "
-                                        "0x%8.8x.",
-                       kret);
-  }
-  return m_state.GetError(e_regSetDBG, Write);
-}
-
-void DNBArchImplI386::ThreadWillResume() {
-  // Do we need to step this thread? If so, let the mach thread tell us so.
-  if (m_thread->IsStepping()) {
-    // This is the primary thread, let the arch do anything it needs
-    EnableHardwareSingleStep(true);
-  }
-
-  // Reset the debug status register, if necessary, before we resume.
-  kern_return_t kret = GetDBGState(false);
-  DNBLogThreadedIf(
-      LOG_WATCHPOINTS,
-      "DNBArchImplI386::ThreadWillResume() GetDBGState() => 0x%8.8x.", kret);
-  if (kret != KERN_SUCCESS)
-    return;
-
-  DBG &debug_state = m_state.context.dbg;
-  bool need_reset = false;
-  uint32_t i, num = NumSupportedHardwareWatchpoints();
-  for (i = 0; i < num; ++i)
-    if (IsWatchpointHit(debug_state, i))
-      need_reset = true;
-
-  if (need_reset) {
-    ClearWatchpointHits(debug_state);
-    kret = SetDBGState(false);
-    DNBLogThreadedIf(
-        LOG_WATCHPOINTS,
-        "DNBArchImplI386::ThreadWillResume() SetDBGState() => 0x%8.8x.", kret);
-  }
-}
-
-bool DNBArchImplI386::ThreadDidStop() {
-  bool success = true;
-
-  m_state.InvalidateAllRegisterStates();
-
-  // Are we stepping a single instruction?
-  if (GetGPRState(true) == KERN_SUCCESS) {
-    // We are single stepping, was this the primary thread?
-    if (m_thread->IsStepping()) {
-      // This was the primary thread, we need to clear the trace
-      // bit if so.
-      success = EnableHardwareSingleStep(false) == KERN_SUCCESS;
-    } else {
-      // The MachThread will automatically restore the suspend count
-      // in ThreadDidStop(), so we don't need to do anything here if
-      // we weren't the primary thread the last time
-    }
-  }
-  return success;
-}
-
-bool DNBArchImplI386::NotifyException(MachException::Data &exc) {
-  switch (exc.exc_type) {
-  case EXC_BAD_ACCESS:
-    break;
-  case EXC_BAD_INSTRUCTION:
-    break;
-  case EXC_ARITHMETIC:
-    break;
-  case EXC_EMULATION:
-    break;
-  case EXC_SOFTWARE:
-    break;
-  case EXC_BREAKPOINT:
-    if (exc.exc_data.size() >= 2 && exc.exc_data[0] == 2) {
-      // exc_code = EXC_I386_BPT
-      //
-      nub_addr_t pc = GetPC(INVALID_NUB_ADDRESS);
-      if (pc != INVALID_NUB_ADDRESS && pc > 0) {
-        pc -= 1;
-        // Check for a breakpoint at one byte prior to the current PC value
-        // since the PC will be just past the trap.
-
-        DNBBreakpoint *bp =
-            m_thread->Process()->Breakpoints().FindByAddress(pc);
-        if (bp) {
-          // Backup the PC for i386 since the trap was taken and the PC
-          // is at the address following the single byte trap instruction.
-          if (m_state.context.gpr.__eip > 0) {
-            m_state.context.gpr.__eip = static_cast<uint32_t>(pc);
-            // Write the new PC back out
-            SetGPRState();
-          }
-        }
-        return true;
-      }
-    } else if (exc.exc_data.size() >= 2 && exc.exc_data[0] == 1) {
-      // exc_code = EXC_I386_SGL
-      //
-      // Check whether this corresponds to a watchpoint hit event.
-      // If yes, set the exc_sub_code to the data break address.
-      nub_addr_t addr = 0;
-      uint32_t hw_index = GetHardwareWatchpointHit(addr);
-      if (hw_index != INVALID_NUB_HW_INDEX) {
-        exc.exc_data[1] = addr;
-        // Piggyback the hw_index in the exc.data.
-        exc.exc_data.push_back(hw_index);
-      }
-
-      return true;
-    }
-    break;
-  case EXC_SYSCALL:
-    break;
-  case EXC_MACH_SYSCALL:
-    break;
-  case EXC_RPC_ALERT:
-    break;
-  }
-  return false;
-}
-
-uint32_t DNBArchImplI386::NumSupportedHardwareBreakpoints() {
-  // Available debug address registers: dr0, dr1, dr2, dr3.
-  return 4;
-}
-
-uint32_t DNBArchImplI386::NumSupportedHardwareWatchpoints() {
-  // Available debug address registers: dr0, dr1, dr2, dr3.
-  return 4;
-}
-
-static uint32_t size_and_rw_bits(nub_size_t size, bool read, bool write) {
-  uint32_t rw;
-  if (read) {
-    rw = 0x3; // READ or READ/WRITE
-  } else if (write) {
-    rw = 0x1; // WRITE
-  } else {
-    assert(0 && "read and write cannot both be false");
-  }
-
-  switch (size) {
-  case 1:
-    return rw;
-  case 2:
-    return (0x1 << 2) | rw;
-  case 4:
-    return (0x3 << 2) | rw;
-  case 8:
-    return (0x2 << 2) | rw;
-  }
-  assert(0 && "invalid size, must be one of 1, 2, 4, or 8");
-  return 0;
-}
-
-void DNBArchImplI386::SetWatchpoint(DBG &debug_state, uint32_t hw_index,
-                                    nub_addr_t addr, nub_size_t size, bool read,
-                                    bool write) {
-  // Set both dr7 (debug control register) and dri (debug address register).
-
-  // dr7{7-0} encodes the local/gloabl enable bits:
-  //  global enable --. .-- local enable
-  //                  | |
-  //                  v v
-  //      dr0 -> bits{1-0}
-  //      dr1 -> bits{3-2}
-  //      dr2 -> bits{5-4}
-  //      dr3 -> bits{7-6}
-  //
-  // dr7{31-16} encodes the rw/len bits:
-  //  b_x+3, b_x+2, b_x+1, b_x
-  //      where bits{x+1, x} => rw
-  //            0b00: execute, 0b01: write, 0b11: read-or-write, 0b10: io
-  //            read-or-write (unused)
-  //      and bits{x+3, x+2} => len
-  //            0b00: 1-byte, 0b01: 2-byte, 0b11: 4-byte, 0b10: 8-byte
-  //
-  //      dr0 -> bits{19-16}
-  //      dr1 -> bits{23-20}
-  //      dr2 -> bits{27-24}
-  //      dr3 -> bits{31-28}
-  debug_state.__dr7 |=
-      (1 << (2 * hw_index) |
-       size_and_rw_bits(size, read, write) << (16 + 4 * hw_index));
-  uint32_t addr_32 = addr & 0xffffffff;
-  switch (hw_index) {
-  case 0:
-    debug_state.__dr0 = addr_32;
-    break;
-  case 1:
-    debug_state.__dr1 = addr_32;
-    break;
-  case 2:
-    debug_state.__dr2 = addr_32;
-    break;
-  case 3:
-    debug_state.__dr3 = addr_32;
-    break;
-  default:
-    assert(0 &&
-           "invalid hardware register index, must be one of 0, 1, 2, or 3");
-  }
-  return;
-}
-
-void DNBArchImplI386::SetHardwareBreakpoint(DBG &debug_state, uint32_t hw_index,
-                                            nub_addr_t addr, nub_size_t size) {
-  // Set both dr7 (debug control register) and dri (debug address register).
-
-  // dr7{7-0} encodes the local/gloabl enable bits:
-  //  global enable --. .-- local enable
-  //                  | |
-  //                  v v
-  //      dr0 -> bits{1-0}
-  //      dr1 -> bits{3-2}
-  //      dr2 -> bits{5-4}
-  //      dr3 -> bits{7-6}
-  //
-  // dr7{31-16} encodes the rw/len bits:
-  //  b_x+3, b_x+2, b_x+1, b_x
-  //      where bits{x+1, x} => rw
-  //            0b00: execute, 0b01: write, 0b11: read-or-write, 0b10: io
-  //            read-or-write (unused)
-  //      and bits{x+3, x+2} => len
-  //            0b00: 1-byte, 0b01: 2-byte, 0b11: 4-byte, 0b10: 8-byte
-  //
-  //      dr0 -> bits{19-16}
-  //      dr1 -> bits{23-20}
-  //      dr2 -> bits{27-24}
-  //      dr3 -> bits{31-28}
-  debug_state.__dr7 |= (1 << (2 * hw_index) | 0 << (16 + 4 * hw_index));
-  uint32_t addr_32 = addr & 0xffffffff;
-  switch (hw_index) {
-  case 0:
-    debug_state.__dr0 = addr_32;
-    break;
-  case 1:
-    debug_state.__dr1 = addr_32;
-    break;
-  case 2:
-    debug_state.__dr2 = addr_32;
-    break;
-  case 3:
-    debug_state.__dr3 = addr_32;
-    break;
-  default:
-    assert(0 &&
-           "invalid hardware register index, must be one of 0, 1, 2, or 3");
-  }
-  return;
-}
-
-uint32_t DNBArchImplI386::EnableHardwareBreakpoint(nub_addr_t addr,
-                                                   nub_size_t size,
-                                                   bool also_set_on_task) {
-  DNBLogThreadedIf(LOG_BREAKPOINTS,
-                   "DNBArchImplI386::EnableHardwareBreakpoint( addr = "
-                   "0x%8.8llx, size = %llu )",
-                   (uint64_t)addr, (uint64_t)size);
-
-  const uint32_t num_hw_breakpoints = NumSupportedHardwareBreakpoints();
-  // Read the debug state
-  kern_return_t kret = GetDBGState(false);
-
-  if (kret != KERN_SUCCESS) {
-    return INVALID_NUB_HW_INDEX;
-  }
-
-  // Check to make sure we have the needed hardware support
-  uint32_t i = 0;
-
-  DBG &debug_state = m_state.context.dbg;
-  for (i = 0; i < num_hw_breakpoints; ++i) {
-    if (IsWatchpointVacant(debug_state, i)) {
-      break;
-    }
-  }
-
-  // See if we found an available hw breakpoint slot above
-  if (i < num_hw_breakpoints) {
-    DNBLogThreadedIf(
-        LOG_BREAKPOINTS,
-        "DNBArchImplI386::EnableHardwareBreakpoint( free slot = %u )", i);
-
-    StartTransForHWP();
-
-    // Modify our local copy of the debug state, first.
-    SetHardwareBreakpoint(debug_state, i, addr, size);
-    // Now set the watch point in the inferior.
-    kret = SetDBGState(also_set_on_task);
-
-    DNBLogThreadedIf(LOG_BREAKPOINTS,
-                     "DNBArchImplI386::"
-                     "EnableHardwareBreakpoint() "
-                     "SetDBGState() => 0x%8.8x.",
-                     kret);
-
-    if (kret == KERN_SUCCESS) {
-      DNBLogThreadedIf(
-          LOG_BREAKPOINTS,
-          "DNBArchImplI386::EnableHardwareBreakpoint( enabled at slot = %u)",
-          i);
-      return i;
-    }
-    // Revert to the previous debug state voluntarily.  The transaction
-    // coordinator knows that we have failed.
-    else {
-      m_state.context.dbg = GetDBGCheckpoint();
-    }
-  } else {
-    DNBLogThreadedIf(LOG_BREAKPOINTS,
-                     "DNBArchImplI386::EnableHardwareBreakpoint(addr = "
-                     "0x%8.8llx, size = %llu) => all hardware breakpoint "
-                     "resources are being used.",
-                     (uint64_t)addr, (uint64_t)size);
-  }
-
-  return INVALID_NUB_HW_INDEX;
-}
-
-bool DNBArchImplI386::DisableHardwareBreakpoint(uint32_t hw_index,
-                                                bool also_set_on_task) {
-  kern_return_t kret = GetDBGState(false);
-
-  const uint32_t num_hw_points = NumSupportedHardwareBreakpoints();
-  if (kret == KERN_SUCCESS) {
-    DBG &debug_state = m_state.context.dbg;
-    if (hw_index < num_hw_points &&
-        !IsWatchpointVacant(debug_state, hw_index)) {
-
-      StartTransForHWP();
-
-      // Modify our local copy of the debug state, first.
-      ClearWatchpoint(debug_state, hw_index);
-      // Now disable the watch point in the inferior.
-      kret = SetDBGState(true);
-      DNBLogThreadedIf(LOG_WATCHPOINTS,
-                       "DNBArchImplI386::DisableHardwareBreakpoint( %u )",
-                       hw_index);
-
-      if (kret == KERN_SUCCESS)
-        return true;
-      else // Revert to the previous debug state voluntarily.  The transaction
-           // coordinator knows that we have failed.
-        m_state.context.dbg = GetDBGCheckpoint();
-    }
-  }
-  return false;
-}
-
-void DNBArchImplI386::ClearWatchpoint(DBG &debug_state, uint32_t hw_index) {
-  debug_state.__dr7 &= ~(3 << (2 * hw_index));
-  switch (hw_index) {
-  case 0:
-    debug_state.__dr0 = 0;
-    break;
-  case 1:
-    debug_state.__dr1 = 0;
-    break;
-  case 2:
-    debug_state.__dr2 = 0;
-    break;
-  case 3:
-    debug_state.__dr3 = 0;
-    break;
-  default:
-    assert(0 &&
-           "invalid hardware register index, must be one of 0, 1, 2, or 3");
-  }
-  return;
-}
-
-bool DNBArchImplI386::IsWatchpointVacant(const DBG &debug_state,
-                                         uint32_t hw_index) {
-  // Check dr7 (debug control register) for local/global enable bits:
-  //  global enable --. .-- local enable
-  //                  | |
-  //                  v v
-  //      dr0 -> bits{1-0}
-  //      dr1 -> bits{3-2}
-  //      dr2 -> bits{5-4}
-  //      dr3 -> bits{7-6}
-  return (debug_state.__dr7 & (3 << (2 * hw_index))) == 0;
-}
-
-// Resets local copy of debug status register to wait for the next debug
-// exception.
-void DNBArchImplI386::ClearWatchpointHits(DBG &debug_state) {
-  // See also IsWatchpointHit().
-  debug_state.__dr6 = 0;
-  return;
-}
-
-bool DNBArchImplI386::IsWatchpointHit(const DBG &debug_state,
-                                      uint32_t hw_index) {
-  // Check dr6 (debug status register) whether a watchpoint hits:
-  //          is watchpoint hit?
-  //                  |
-  //                  v
-  //      dr0 -> bits{0}
-  //      dr1 -> bits{1}
-  //      dr2 -> bits{2}
-  //      dr3 -> bits{3}
-  return (debug_state.__dr6 & (1 << hw_index));
-}
-
-nub_addr_t DNBArchImplI386::GetWatchAddress(const DBG &debug_state,
-                                            uint32_t hw_index) {
-  switch (hw_index) {
-  case 0:
-    return debug_state.__dr0;
-  case 1:
-    return debug_state.__dr1;
-  case 2:
-    return debug_state.__dr2;
-  case 3:
-    return debug_state.__dr3;
-  }
-  assert(0 && "invalid hardware register index, must be one of 0, 1, 2, or 3");
-  return 0;
-}
-
-bool DNBArchImplI386::StartTransForHWP() {
-  if (m_2pc_trans_state != Trans_Done && m_2pc_trans_state != Trans_Rolled_Back)
-    DNBLogError("%s inconsistent state detected, expected %d or %d, got: %d",
-                __FUNCTION__, Trans_Done, Trans_Rolled_Back, m_2pc_trans_state);
-  m_2pc_dbg_checkpoint = m_state.context.dbg;
-  m_2pc_trans_state = Trans_Pending;
-  return true;
-}
-bool DNBArchImplI386::RollbackTransForHWP() {
-  m_state.context.dbg = m_2pc_dbg_checkpoint;
-  if (m_2pc_trans_state != Trans_Pending)
-    DNBLogError("%s inconsistent state detected, expected %d, got: %d",
-                __FUNCTION__, Trans_Pending, m_2pc_trans_state);
-  m_2pc_trans_state = Trans_Rolled_Back;
-  kern_return_t kret = SetDBGState(false);
-  DNBLogThreadedIf(
-      LOG_WATCHPOINTS,
-      "DNBArchImplI386::RollbackTransForHWP() SetDBGState() => 0x%8.8x.", kret);
-
-  return kret == KERN_SUCCESS;
-}
-bool DNBArchImplI386::FinishTransForHWP() {
-  m_2pc_trans_state = Trans_Done;
-  return true;
-}
-DNBArchImplI386::DBG DNBArchImplI386::GetDBGCheckpoint() {
-  return m_2pc_dbg_checkpoint;
-}
-
-uint32_t DNBArchImplI386::EnableHardwareWatchpoint(nub_addr_t addr,
-                                                   nub_size_t size, bool read,
-                                                   bool write,
-                                                   bool also_set_on_task) {
-  DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchImplI386::EnableHardwareWatchpoint("
-                                    "addr = 0x%llx, size = %llu, read = %u, "
-                                    "write = %u)",
-                   (uint64_t)addr, (uint64_t)size, read, write);
-
-  const uint32_t num_hw_watchpoints = NumSupportedHardwareWatchpoints();
-
-  // Can only watch 1, 2, 4, or 8 bytes.
-  if (!(size == 1 || size == 2 || size == 4 || size == 8))
-    return INVALID_NUB_HW_INDEX;
-
-  // We must watch for either read or write
-  if (!read && !write)
-    return INVALID_NUB_HW_INDEX;
-
-  // Read the debug state
-  kern_return_t kret = GetDBGState(false);
-
-  if (kret == KERN_SUCCESS) {
-    // Check to make sure we have the needed hardware support
-    uint32_t i = 0;
-
-    DBG &debug_state = m_state.context.dbg;
-    for (i = 0; i < num_hw_watchpoints; ++i) {
-      if (IsWatchpointVacant(debug_state, i))
-        break;
-    }
-
-    // See if we found an available hw breakpoint slot above
-    if (i < num_hw_watchpoints) {
-      StartTransForHWP();
-
-      // Modify our local copy of the debug state, first.
-      SetWatchpoint(debug_state, i, addr, size, read, write);
-      // Now set the watch point in the inferior.
-      kret = SetDBGState(also_set_on_task);
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchImplI386::"
-                                        "EnableHardwareWatchpoint() "
-                                        "SetDBGState() => 0x%8.8x.",
-                       kret);
-
-      if (kret == KERN_SUCCESS)
-        return i;
-      else // Revert to the previous debug state voluntarily.  The transaction
-           // coordinator knows that we have failed.
-        m_state.context.dbg = GetDBGCheckpoint();
-    } else {
-      DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchImplI386::"
-                                        "EnableHardwareWatchpoint(): All "
-                                        "hardware resources (%u) are in use.",
-                       num_hw_watchpoints);
-    }
-  }
-  return INVALID_NUB_HW_INDEX;
-}
-
-bool DNBArchImplI386::DisableHardwareWatchpoint(uint32_t hw_index,
-                                                bool also_set_on_task) {
-  kern_return_t kret = GetDBGState(false);
-
-  const uint32_t num_hw_points = NumSupportedHardwareWatchpoints();
-  if (kret == KERN_SUCCESS) {
-    DBG &debug_state = m_state.context.dbg;
-    if (hw_index < num_hw_points &&
-        !IsWatchpointVacant(debug_state, hw_index)) {
-      StartTransForHWP();
-
-      // Modify our local copy of the debug state, first.
-      ClearWatchpoint(debug_state, hw_index);
-      // Now disable the watch point in the inferior.
-      kret = SetDBGState(also_set_on_task);
-      DNBLogThreadedIf(LOG_WATCHPOINTS,
-                       "DNBArchImplI386::DisableHardwareWatchpoint( %u )",
-                       hw_index);
-
-      if (kret == KERN_SUCCESS)
-        return true;
-      else // Revert to the previous debug state voluntarily.  The transaction
-           // coordinator knows that we have failed.
-        m_state.context.dbg = GetDBGCheckpoint();
-    }
-  }
-  return false;
-}
-
-// Iterate through the debug status register; return the index of the first hit.
-uint32_t DNBArchImplI386::GetHardwareWatchpointHit(nub_addr_t &addr) {
-  // Read the debug state
-  kern_return_t kret = GetDBGState(true);
-  DNBLogThreadedIf(
-      LOG_WATCHPOINTS,
-      "DNBArchImplI386::GetHardwareWatchpointHit() GetDBGState() => 0x%8.8x.",
-      kret);
-  if (kret == KERN_SUCCESS) {
-    DBG &debug_state = m_state.context.dbg;
-    uint32_t i, num = NumSupportedHardwareWatchpoints();
-    for (i = 0; i < num; ++i) {
-      if (IsWatchpointHit(debug_state, i)) {
-        addr = GetWatchAddress(debug_state, i);
-        DNBLogThreadedIf(LOG_WATCHPOINTS, "DNBArchImplI386::"
-                                          "GetHardwareWatchpointHit() found => "
-                                          "%u (addr = 0x%llx).",
-                         i, (uint64_t)addr);
-        return i;
-      }
-    }
-  }
-  return INVALID_NUB_HW_INDEX;
-}
-
-// Set the single step bit in the processor status register.
-kern_return_t DNBArchImplI386::EnableHardwareSingleStep(bool enable) {
-  if (GetGPRState(false) == KERN_SUCCESS) {
-    const uint32_t trace_bit = 0x100u;
-    if (enable)
-      m_state.context.gpr.__eflags |= trace_bit;
-    else
-      m_state.context.gpr.__eflags &= ~trace_bit;
-    return SetGPRState();
-  }
-  return m_state.GetError(e_regSetGPR, Read);
-}
-
-// Register information definitions
-
-#define DEFINE_GPR_PSEUDO_16(reg16, reg32)                                     \
-  {                                                                            \
-    e_regSetGPR, gpr_##reg16, #reg16, NULL, Uint, Hex, 2, 0,                   \
-        INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,            \
-        INVALID_NUB_REGNUM, g_contained_##reg32, g_invalidate_##reg32          \
-  }
-#define DEFINE_GPR_PSEUDO_8H(reg8, reg32)                                      \
-  {                                                                            \
-    e_regSetGPR, gpr_##reg8, #reg8, NULL, Uint, Hex, 1, 1, INVALID_NUB_REGNUM, \
-        INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,            \
-        g_contained_##reg32, g_invalidate_##reg32                              \
-  }
-#define DEFINE_GPR_PSEUDO_8L(reg8, reg32)                                      \
-  {                                                                            \
-    e_regSetGPR, gpr_##reg8, #reg8, NULL, Uint, Hex, 1, 0, INVALID_NUB_REGNUM, \
-        INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,            \
-        g_contained_##reg32, g_invalidate_##reg32                              \
-  }
-
-#define GPR_OFFSET(reg) (offsetof(DNBArchImplI386::GPR, __##reg))
-#define FPU_OFFSET(reg)                                                        \
-  (offsetof(DNBArchImplI386::FPU, __fpu_##reg) +                               \
-   offsetof(DNBArchImplI386::Context, fpu.no_avx))
-#define AVX_OFFSET(reg)                                                        \
-  (offsetof(DNBArchImplI386::AVX, __fpu_##reg) +                               \
-   offsetof(DNBArchImplI386::Context, fpu.avx))
-#define AVX512F_OFFSET(reg)                                                    \
-  (offsetof(DNBArchImplI386::AVX512F, __fpu_##reg) +                           \
-   offsetof(DNBArchImplI386::Context, fpu.avx512f))
-#define EXC_OFFSET(reg)                                                        \
-  (offsetof(DNBArchImplI386::EXC, __##reg) +                                   \
-   offsetof(DNBArchImplI386::Context, exc))
-
-#define GPR_SIZE(reg) (sizeof(((DNBArchImplI386::GPR *)NULL)->__##reg))
-#define FPU_SIZE_UINT(reg) (sizeof(((DNBArchImplI386::FPU *)NULL)->__fpu_##reg))
-#define FPU_SIZE_MMST(reg)                                                     \
-  (sizeof(((DNBArchImplI386::FPU *)NULL)->__fpu_##reg.__mmst_reg))
-#define FPU_SIZE_XMM(reg)                                                      \
-  (sizeof(((DNBArchImplI386::FPU *)NULL)->__fpu_##reg.__xmm_reg))
-#define FPU_SIZE_YMM(reg) (32)
-#define FPU_SIZE_ZMM(reg) (64)
-#define EXC_SIZE(reg) (sizeof(((DNBArchImplI386::EXC *)NULL)->__##reg))
-
-// This does not accurately identify the location of ymm0...7 in
-// Context.fpu.avx.  That is because there is a bunch of padding
-// in Context.fpu.avx that we don't need.  Offset macros lay out
-// the register state that Debugserver transmits to the debugger
-// -- not to interpret the thread_get_state info.
-#define AVX_OFFSET_YMM(n) (AVX_OFFSET(xmm7) + FPU_SIZE_XMM(xmm7) + (32 * n))
-
-// TODO: Test this and come back.
-#define AVX512F_OFFSET_ZMM(n) (AVX_OFFSET_YMM(7) + FPU_SIZE_XMM(xmm7) + (64 * n))
-
-// These macros will auto define the register name, alt name, register size,
-// register offset, encoding, format and native register. This ensures that
-// the register state structures are defined correctly and have the correct
-// sizes and offsets.
-
-const char *g_contained_eax[] = {"eax", NULL};
-const char *g_contained_ebx[] = {"ebx", NULL};
-const char *g_contained_ecx[] = {"ecx", NULL};
-const char *g_contained_edx[] = {"edx", NULL};
-const char *g_contained_edi[] = {"edi", NULL};
-const char *g_contained_esi[] = {"esi", NULL};
-const char *g_contained_ebp[] = {"ebp", NULL};
-const char *g_contained_esp[] = {"esp", NULL};
-
-const char *g_invalidate_eax[] = {"eax", "ax", "ah", "al", NULL};
-const char *g_invalidate_ebx[] = {"ebx", "bx", "bh", "bl", NULL};
-const char *g_invalidate_ecx[] = {"ecx", "cx", "ch", "cl", NULL};
-const char *g_invalidate_edx[] = {"edx", "dx", "dh", "dl", NULL};
-const char *g_invalidate_edi[] = {"edi", "di", "dil", NULL};
-const char *g_invalidate_esi[] = {"esi", "si", "sil", NULL};
-const char *g_invalidate_ebp[] = {"ebp", "bp", "bpl", NULL};
-const char *g_invalidate_esp[] = {"esp", "sp", "spl", NULL};
-
-// General purpose registers for 64 bit
-const DNBRegisterInfo DNBArchImplI386::g_gpr_registers[] = {
-    {e_regSetGPR, gpr_eax, "eax", NULL, Uint, Hex, GPR_SIZE(eax),
-     GPR_OFFSET(eax), ehframe_eax, dwarf_eax, INVALID_NUB_REGNUM,
-     debugserver_eax, NULL, g_invalidate_eax},
-    {e_regSetGPR, gpr_ebx, "ebx", NULL, Uint, Hex, GPR_SIZE(ebx),
-     GPR_OFFSET(ebx), ehframe_ebx, dwarf_ebx, INVALID_NUB_REGNUM,
-     debugserver_ebx, NULL, g_invalidate_ebx},
-    {e_regSetGPR, gpr_ecx, "ecx", NULL, Uint, Hex, GPR_SIZE(ecx),
-     GPR_OFFSET(ecx), ehframe_ecx, dwarf_ecx, INVALID_NUB_REGNUM,
-     debugserver_ecx, NULL, g_invalidate_ecx},
-    {e_regSetGPR, gpr_edx, "edx", NULL, Uint, Hex, GPR_SIZE(edx),
-     GPR_OFFSET(edx), ehframe_edx, dwarf_edx, INVALID_NUB_REGNUM,
-     debugserver_edx, NULL, g_invalidate_edx},
-    {e_regSetGPR, gpr_edi, "edi", NULL, Uint, Hex, GPR_SIZE(edi),
-     GPR_OFFSET(edi), ehframe_edi, dwarf_edi, INVALID_NUB_REGNUM,
-     debugserver_edi, NULL, g_invalidate_edi},
-    {e_regSetGPR, gpr_esi, "esi", NULL, Uint, Hex, GPR_SIZE(esi),
-     GPR_OFFSET(esi), ehframe_esi, dwarf_esi, INVALID_NUB_REGNUM,
-     debugserver_esi, NULL, g_invalidate_esi},
-    {e_regSetGPR, gpr_ebp, "ebp", "fp", Uint, Hex, GPR_SIZE(ebp),
-     GPR_OFFSET(ebp), ehframe_ebp, dwarf_ebp, GENERIC_REGNUM_FP,
-     debugserver_ebp, NULL, g_invalidate_ebp},
-    {e_regSetGPR, gpr_esp, "esp", "sp", Uint, Hex, GPR_SIZE(esp),
-     GPR_OFFSET(esp), ehframe_esp, dwarf_esp, GENERIC_REGNUM_SP,
-     debugserver_esp, NULL, g_invalidate_esp},
-    {e_regSetGPR, gpr_ss, "ss", NULL, Uint, Hex, GPR_SIZE(ss), GPR_OFFSET(ss),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, debugserver_ss,
-     NULL, NULL},
-    {e_regSetGPR, gpr_eflags, "eflags", "flags", Uint, Hex, GPR_SIZE(eflags),
-     GPR_OFFSET(eflags), ehframe_eflags, dwarf_eflags, GENERIC_REGNUM_FLAGS,
-     debugserver_eflags, NULL, NULL},
-    {e_regSetGPR, gpr_eip, "eip", "pc", Uint, Hex, GPR_SIZE(eip),
-     GPR_OFFSET(eip), ehframe_eip, dwarf_eip, GENERIC_REGNUM_PC,
-     debugserver_eip, NULL, NULL},
-    {e_regSetGPR, gpr_cs, "cs", NULL, Uint, Hex, GPR_SIZE(cs), GPR_OFFSET(cs),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, debugserver_cs,
-     NULL, NULL},
-    {e_regSetGPR, gpr_ds, "ds", NULL, Uint, Hex, GPR_SIZE(ds), GPR_OFFSET(ds),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, debugserver_ds,
-     NULL, NULL},
-    {e_regSetGPR, gpr_es, "es", NULL, Uint, Hex, GPR_SIZE(es), GPR_OFFSET(es),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, debugserver_es,
-     NULL, NULL},
-    {e_regSetGPR, gpr_fs, "fs", NULL, Uint, Hex, GPR_SIZE(fs), GPR_OFFSET(fs),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, debugserver_fs,
-     NULL, NULL},
-    {e_regSetGPR, gpr_gs, "gs", NULL, Uint, Hex, GPR_SIZE(gs), GPR_OFFSET(gs),
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, debugserver_gs,
-     NULL, NULL},
-    DEFINE_GPR_PSEUDO_16(ax, eax),
-    DEFINE_GPR_PSEUDO_16(bx, ebx),
-    DEFINE_GPR_PSEUDO_16(cx, ecx),
-    DEFINE_GPR_PSEUDO_16(dx, edx),
-    DEFINE_GPR_PSEUDO_16(di, edi),
-    DEFINE_GPR_PSEUDO_16(si, esi),
-    DEFINE_GPR_PSEUDO_16(bp, ebp),
-    DEFINE_GPR_PSEUDO_16(sp, esp),
-    DEFINE_GPR_PSEUDO_8H(ah, eax),
-    DEFINE_GPR_PSEUDO_8H(bh, ebx),
-    DEFINE_GPR_PSEUDO_8H(ch, ecx),
-    DEFINE_GPR_PSEUDO_8H(dh, edx),
-    DEFINE_GPR_PSEUDO_8L(al, eax),
-    DEFINE_GPR_PSEUDO_8L(bl, ebx),
-    DEFINE_GPR_PSEUDO_8L(cl, ecx),
-    DEFINE_GPR_PSEUDO_8L(dl, edx),
-    DEFINE_GPR_PSEUDO_8L(dil, edi),
-    DEFINE_GPR_PSEUDO_8L(sil, esi),
-    DEFINE_GPR_PSEUDO_8L(bpl, ebp),
-    DEFINE_GPR_PSEUDO_8L(spl, esp)};
-
-const DNBRegisterInfo DNBArchImplI386::g_fpu_registers_no_avx[] = {
-    {e_regSetFPU, fpu_fcw, "fctrl", NULL, Uint, Hex, FPU_SIZE_UINT(fcw),
-     FPU_OFFSET(fcw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_fsw, "fstat", NULL, Uint, Hex, FPU_SIZE_UINT(fsw),
-     FPU_OFFSET(fsw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ftw, "ftag", NULL, Uint, Hex, 2 /* sizeof __fpu_ftw + sizeof __fpu_rsrv1 */,
-     FPU_OFFSET(ftw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_fop, "fop", NULL, Uint, Hex, FPU_SIZE_UINT(fop),
-     FPU_OFFSET(fop), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ip, "fioff", NULL, Uint, Hex, FPU_SIZE_UINT(ip),
-     FPU_OFFSET(ip), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_cs, "fiseg", NULL, Uint, Hex, FPU_SIZE_UINT(cs),
-     FPU_OFFSET(cs), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_dp, "fooff", NULL, Uint, Hex, FPU_SIZE_UINT(dp),
-     FPU_OFFSET(dp), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ds, "foseg", NULL, Uint, Hex, FPU_SIZE_UINT(ds),
-     FPU_OFFSET(ds), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_mxcsr, "mxcsr", NULL, Uint, Hex, FPU_SIZE_UINT(mxcsr),
-     FPU_OFFSET(mxcsr), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_mxcsrmask, "mxcsrmask", NULL, Uint, Hex,
-     FPU_SIZE_UINT(mxcsrmask), FPU_OFFSET(mxcsrmask), INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-
-    {e_regSetFPU, fpu_stmm0, "stmm0", "st0", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm0), FPU_OFFSET(stmm0), INVALID_NUB_REGNUM, dwarf_stmm0,
-     INVALID_NUB_REGNUM, debugserver_stmm0, NULL, NULL},
-    {e_regSetFPU, fpu_stmm1, "stmm1", "st1", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm1), FPU_OFFSET(stmm1), INVALID_NUB_REGNUM, dwarf_stmm1,
-     INVALID_NUB_REGNUM, debugserver_stmm1, NULL, NULL},
-    {e_regSetFPU, fpu_stmm2, "stmm2", "st2", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm2), FPU_OFFSET(stmm2), INVALID_NUB_REGNUM, dwarf_stmm2,
-     INVALID_NUB_REGNUM, debugserver_stmm2, NULL, NULL},
-    {e_regSetFPU, fpu_stmm3, "stmm3", "st3", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm3), FPU_OFFSET(stmm3), INVALID_NUB_REGNUM, dwarf_stmm3,
-     INVALID_NUB_REGNUM, debugserver_stmm3, NULL, NULL},
-    {e_regSetFPU, fpu_stmm4, "stmm4", "st4", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm4), FPU_OFFSET(stmm4), INVALID_NUB_REGNUM, dwarf_stmm4,
-     INVALID_NUB_REGNUM, debugserver_stmm4, NULL, NULL},
-    {e_regSetFPU, fpu_stmm5, "stmm5", "st5", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm5), FPU_OFFSET(stmm5), INVALID_NUB_REGNUM, dwarf_stmm5,
-     INVALID_NUB_REGNUM, debugserver_stmm5, NULL, NULL},
-    {e_regSetFPU, fpu_stmm6, "stmm6", "st6", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm6), FPU_OFFSET(stmm6), INVALID_NUB_REGNUM, dwarf_stmm6,
-     INVALID_NUB_REGNUM, debugserver_stmm6, NULL, NULL},
-    {e_regSetFPU, fpu_stmm7, "stmm7", "st7", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm7), FPU_OFFSET(stmm7), INVALID_NUB_REGNUM, dwarf_stmm7,
-     INVALID_NUB_REGNUM, debugserver_stmm7, NULL, NULL},
-
-    {e_regSetFPU, fpu_xmm0, "xmm0", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm0), FPU_OFFSET(xmm0), INVALID_NUB_REGNUM, dwarf_xmm0,
-     INVALID_NUB_REGNUM, debugserver_xmm0, NULL, NULL},
-    {e_regSetFPU, fpu_xmm1, "xmm1", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm1), FPU_OFFSET(xmm1), INVALID_NUB_REGNUM, dwarf_xmm1,
-     INVALID_NUB_REGNUM, debugserver_xmm1, NULL, NULL},
-    {e_regSetFPU, fpu_xmm2, "xmm2", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm2), FPU_OFFSET(xmm2), INVALID_NUB_REGNUM, dwarf_xmm2,
-     INVALID_NUB_REGNUM, debugserver_xmm2, NULL, NULL},
-    {e_regSetFPU, fpu_xmm3, "xmm3", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm3), FPU_OFFSET(xmm3), INVALID_NUB_REGNUM, dwarf_xmm3,
-     INVALID_NUB_REGNUM, debugserver_xmm3, NULL, NULL},
-    {e_regSetFPU, fpu_xmm4, "xmm4", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm4), FPU_OFFSET(xmm4), INVALID_NUB_REGNUM, dwarf_xmm4,
-     INVALID_NUB_REGNUM, debugserver_xmm4, NULL, NULL},
-    {e_regSetFPU, fpu_xmm5, "xmm5", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm5), FPU_OFFSET(xmm5), INVALID_NUB_REGNUM, dwarf_xmm5,
-     INVALID_NUB_REGNUM, debugserver_xmm5, NULL, NULL},
-    {e_regSetFPU, fpu_xmm6, "xmm6", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm6), FPU_OFFSET(xmm6), INVALID_NUB_REGNUM, dwarf_xmm6,
-     INVALID_NUB_REGNUM, debugserver_xmm6, NULL, NULL},
-    {e_regSetFPU, fpu_xmm7, "xmm7", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm7), FPU_OFFSET(xmm7), INVALID_NUB_REGNUM, dwarf_xmm7,
-     INVALID_NUB_REGNUM, debugserver_xmm7, NULL, NULL}};
-
-static const char *g_contained_ymm0[] = {"ymm0", NULL};
-static const char *g_contained_ymm1[] = {"ymm1", NULL};
-static const char *g_contained_ymm2[] = {"ymm2", NULL};
-static const char *g_contained_ymm3[] = {"ymm3", NULL};
-static const char *g_contained_ymm4[] = {"ymm4", NULL};
-static const char *g_contained_ymm5[] = {"ymm5", NULL};
-static const char *g_contained_ymm6[] = {"ymm6", NULL};
-static const char *g_contained_ymm7[] = {"ymm7", NULL};
-
-const DNBRegisterInfo DNBArchImplI386::g_fpu_registers_avx[] = {
-    {e_regSetFPU, fpu_fcw, "fctrl", NULL, Uint, Hex, FPU_SIZE_UINT(fcw),
-     AVX_OFFSET(fcw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_fsw, "fstat", NULL, Uint, Hex, FPU_SIZE_UINT(fsw),
-     AVX_OFFSET(fsw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ftw, "ftag", NULL, Uint, Hex, 2 /* sizeof __fpu_ftw + sizeof __fpu_rsrv1 */,
-     AVX_OFFSET(ftw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_fop, "fop", NULL, Uint, Hex, FPU_SIZE_UINT(fop),
-     AVX_OFFSET(fop), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ip, "fioff", NULL, Uint, Hex, FPU_SIZE_UINT(ip),
-     AVX_OFFSET(ip), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_cs, "fiseg", NULL, Uint, Hex, FPU_SIZE_UINT(cs),
-     AVX_OFFSET(cs), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_dp, "fooff", NULL, Uint, Hex, FPU_SIZE_UINT(dp),
-     AVX_OFFSET(dp), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ds, "foseg", NULL, Uint, Hex, FPU_SIZE_UINT(ds),
-     AVX_OFFSET(ds), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_mxcsr, "mxcsr", NULL, Uint, Hex, FPU_SIZE_UINT(mxcsr),
-     AVX_OFFSET(mxcsr), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_mxcsrmask, "mxcsrmask", NULL, Uint, Hex,
-     FPU_SIZE_UINT(mxcsrmask), AVX_OFFSET(mxcsrmask), INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-
-    {e_regSetFPU, fpu_stmm0, "stmm0", "st0", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm0), AVX_OFFSET(stmm0), INVALID_NUB_REGNUM, dwarf_stmm0,
-     INVALID_NUB_REGNUM, debugserver_stmm0, NULL, NULL},
-    {e_regSetFPU, fpu_stmm1, "stmm1", "st1", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm1), AVX_OFFSET(stmm1), INVALID_NUB_REGNUM, dwarf_stmm1,
-     INVALID_NUB_REGNUM, debugserver_stmm1, NULL, NULL},
-    {e_regSetFPU, fpu_stmm2, "stmm2", "st2", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm2), AVX_OFFSET(stmm2), INVALID_NUB_REGNUM, dwarf_stmm2,
-     INVALID_NUB_REGNUM, debugserver_stmm2, NULL, NULL},
-    {e_regSetFPU, fpu_stmm3, "stmm3", "st3", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm3), AVX_OFFSET(stmm3), INVALID_NUB_REGNUM, dwarf_stmm3,
-     INVALID_NUB_REGNUM, debugserver_stmm3, NULL, NULL},
-    {e_regSetFPU, fpu_stmm4, "stmm4", "st4", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm4), AVX_OFFSET(stmm4), INVALID_NUB_REGNUM, dwarf_stmm4,
-     INVALID_NUB_REGNUM, debugserver_stmm4, NULL, NULL},
-    {e_regSetFPU, fpu_stmm5, "stmm5", "st5", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm5), AVX_OFFSET(stmm5), INVALID_NUB_REGNUM, dwarf_stmm5,
-     INVALID_NUB_REGNUM, debugserver_stmm5, NULL, NULL},
-    {e_regSetFPU, fpu_stmm6, "stmm6", "st6", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm6), AVX_OFFSET(stmm6), INVALID_NUB_REGNUM, dwarf_stmm6,
-     INVALID_NUB_REGNUM, debugserver_stmm6, NULL, NULL},
-    {e_regSetFPU, fpu_stmm7, "stmm7", "st7", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm7), AVX_OFFSET(stmm7), INVALID_NUB_REGNUM, dwarf_stmm7,
-     INVALID_NUB_REGNUM, debugserver_stmm7, NULL, NULL},
-
-    {e_regSetFPU, fpu_ymm0, "ymm0", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm0), AVX_OFFSET_YMM(0), INVALID_NUB_REGNUM, dwarf_ymm0,
-     INVALID_NUB_REGNUM, debugserver_ymm0, NULL, NULL},
-    {e_regSetFPU, fpu_ymm1, "ymm1", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm1), AVX_OFFSET_YMM(1), INVALID_NUB_REGNUM, dwarf_ymm1,
-     INVALID_NUB_REGNUM, debugserver_ymm1, NULL, NULL},
-    {e_regSetFPU, fpu_ymm2, "ymm2", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm2), AVX_OFFSET_YMM(2), INVALID_NUB_REGNUM, dwarf_ymm2,
-     INVALID_NUB_REGNUM, debugserver_ymm2, NULL, NULL},
-    {e_regSetFPU, fpu_ymm3, "ymm3", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm3), AVX_OFFSET_YMM(3), INVALID_NUB_REGNUM, dwarf_ymm3,
-     INVALID_NUB_REGNUM, debugserver_ymm3, NULL, NULL},
-    {e_regSetFPU, fpu_ymm4, "ymm4", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm4), AVX_OFFSET_YMM(4), INVALID_NUB_REGNUM, dwarf_ymm4,
-     INVALID_NUB_REGNUM, debugserver_ymm4, NULL, NULL},
-    {e_regSetFPU, fpu_ymm5, "ymm5", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm5), AVX_OFFSET_YMM(5), INVALID_NUB_REGNUM, dwarf_ymm5,
-     INVALID_NUB_REGNUM, debugserver_ymm5, NULL, NULL},
-    {e_regSetFPU, fpu_ymm6, "ymm6", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm6), AVX_OFFSET_YMM(6), INVALID_NUB_REGNUM, dwarf_ymm6,
-     INVALID_NUB_REGNUM, debugserver_ymm6, NULL, NULL},
-    {e_regSetFPU, fpu_ymm7, "ymm7", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_YMM(ymm7), AVX_OFFSET_YMM(7), INVALID_NUB_REGNUM, dwarf_ymm7,
-     INVALID_NUB_REGNUM, debugserver_ymm7, NULL, NULL},
-
-    {e_regSetFPU, fpu_xmm0, "xmm0", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm0), 0, INVALID_NUB_REGNUM, dwarf_xmm0, INVALID_NUB_REGNUM,
-     debugserver_xmm0, g_contained_ymm0, NULL},
-    {e_regSetFPU, fpu_xmm1, "xmm1", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm1), 0, INVALID_NUB_REGNUM, dwarf_xmm1, INVALID_NUB_REGNUM,
-     debugserver_xmm1, g_contained_ymm1, NULL},
-    {e_regSetFPU, fpu_xmm2, "xmm2", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm2), 0, INVALID_NUB_REGNUM, dwarf_xmm2, INVALID_NUB_REGNUM,
-     debugserver_xmm2, g_contained_ymm2, NULL},
-    {e_regSetFPU, fpu_xmm3, "xmm3", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm3), 0, INVALID_NUB_REGNUM, dwarf_xmm3, INVALID_NUB_REGNUM,
-     debugserver_xmm3, g_contained_ymm3, NULL},
-    {e_regSetFPU, fpu_xmm4, "xmm4", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm4), 0, INVALID_NUB_REGNUM, dwarf_xmm4, INVALID_NUB_REGNUM,
-     debugserver_xmm4, g_contained_ymm4, NULL},
-    {e_regSetFPU, fpu_xmm5, "xmm5", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm5), 0, INVALID_NUB_REGNUM, dwarf_xmm5, INVALID_NUB_REGNUM,
-     debugserver_xmm5, g_contained_ymm5, NULL},
-    {e_regSetFPU, fpu_xmm6, "xmm6", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm6), 0, INVALID_NUB_REGNUM, dwarf_xmm6, INVALID_NUB_REGNUM,
-     debugserver_xmm6, g_contained_ymm6, NULL},
-    {e_regSetFPU, fpu_xmm7, "xmm7", NULL, Vector, VectorOfUInt8,
-     FPU_SIZE_XMM(xmm7), 0, INVALID_NUB_REGNUM, dwarf_xmm7, INVALID_NUB_REGNUM,
-     debugserver_xmm7, g_contained_ymm7, NULL},
-
-};
-
-
-#define STR(s) #s
-
-#define ZMM_REG_DEF(reg)                                                       \
-  {                                                                            \
-    e_regSetFPU, fpu_zmm##reg, STR(zmm##reg), NULL, Vector, VectorOfUInt8,        \
-        FPU_SIZE_ZMM(zmm##reg), AVX512F_OFFSET_ZMM(reg), INVALID_NUB_REGNUM,   \
-        dwarf_zmm##reg, INVALID_NUB_REGNUM, debugserver_zmm##reg, NULL, NULL   \
-  }
-
-#define YMM_REG_ALIAS(reg)                                                     \
-  {                                                                            \
-    e_regSetFPU, fpu_ymm##reg, STR(ymm##reg), NULL, Vector, VectorOfUInt8,        \
-        FPU_SIZE_YMM(ymm##reg), 0, INVALID_NUB_REGNUM, dwarf_ymm##reg,         \
-        INVALID_NUB_REGNUM, debugserver_ymm##reg, g_contained_zmm##reg, NULL   \
-  }
-
-#define XMM_REG_ALIAS(reg)                                                     \
-  {                                                                            \
-    e_regSetFPU, fpu_xmm##reg, STR(xmm##reg), NULL, Vector, VectorOfUInt8,        \
-        FPU_SIZE_XMM(xmm##reg), 0, INVALID_NUB_REGNUM, dwarf_xmm##reg,         \
-        INVALID_NUB_REGNUM, debugserver_xmm##reg, g_contained_zmm##reg, NULL   \
-  }
-
-#define AVX512_K_REG_DEF(reg)                                                  \
-  {                                                                            \
-    e_regSetFPU, fpu_k##reg, STR(k##reg), NULL, Vector, VectorOfUInt8, 8,      \
-        AVX512F_OFFSET(k##reg), dwarf_k##reg, dwarf_k##reg, -1U,               \
-        debugserver_k##reg, NULL, NULL                                         \
-  }
-
-static const char *g_contained_zmm0[] = {"zmm0", NULL};
-static const char *g_contained_zmm1[] = {"zmm1", NULL};
-static const char *g_contained_zmm2[] = {"zmm2", NULL};
-static const char *g_contained_zmm3[] = {"zmm3", NULL};
-static const char *g_contained_zmm4[] = {"zmm4", NULL};
-static const char *g_contained_zmm5[] = {"zmm5", NULL};
-static const char *g_contained_zmm6[] = {"zmm6", NULL};
-static const char *g_contained_zmm7[] = {"zmm7", NULL};
-
-const DNBRegisterInfo DNBArchImplI386::g_fpu_registers_avx512f[] = {
-    {e_regSetFPU, fpu_fcw, "fctrl", NULL, Uint, Hex, FPU_SIZE_UINT(fcw),
-     AVX_OFFSET(fcw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_fsw, "fstat", NULL, Uint, Hex, FPU_SIZE_UINT(fsw),
-     AVX_OFFSET(fsw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ftw, "ftag", NULL, Uint, Hex, 2 /* sizeof __fpu_ftw + sizeof __fpu_rsrv1 */,
-     FPU_OFFSET(ftw), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_fop, "fop", NULL, Uint, Hex, FPU_SIZE_UINT(fop),
-     AVX_OFFSET(fop), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ip, "fioff", NULL, Uint, Hex, FPU_SIZE_UINT(ip),
-     AVX_OFFSET(ip), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_cs, "fiseg", NULL, Uint, Hex, FPU_SIZE_UINT(cs),
-     AVX_OFFSET(cs), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_dp, "fooff", NULL, Uint, Hex, FPU_SIZE_UINT(dp),
-     AVX_OFFSET(dp), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_ds, "foseg", NULL, Uint, Hex, FPU_SIZE_UINT(ds),
-     AVX_OFFSET(ds), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_mxcsr, "mxcsr", NULL, Uint, Hex, FPU_SIZE_UINT(mxcsr),
-     AVX_OFFSET(mxcsr), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetFPU, fpu_mxcsrmask, "mxcsrmask", NULL, Uint, Hex,
-     FPU_SIZE_UINT(mxcsrmask), AVX_OFFSET(mxcsrmask), INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-
-    {e_regSetFPU, fpu_stmm0, "stmm0", "st0", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm0), AVX_OFFSET(stmm0), INVALID_NUB_REGNUM, dwarf_stmm0,
-     INVALID_NUB_REGNUM, debugserver_stmm0, NULL, NULL},
-    {e_regSetFPU, fpu_stmm1, "stmm1", "st1", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm1), AVX_OFFSET(stmm1), INVALID_NUB_REGNUM, dwarf_stmm1,
-     INVALID_NUB_REGNUM, debugserver_stmm1, NULL, NULL},
-    {e_regSetFPU, fpu_stmm2, "stmm2", "st2", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm2), AVX_OFFSET(stmm2), INVALID_NUB_REGNUM, dwarf_stmm2,
-     INVALID_NUB_REGNUM, debugserver_stmm2, NULL, NULL},
-    {e_regSetFPU, fpu_stmm3, "stmm3", "st3", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm3), AVX_OFFSET(stmm3), INVALID_NUB_REGNUM, dwarf_stmm3,
-     INVALID_NUB_REGNUM, debugserver_stmm3, NULL, NULL},
-    {e_regSetFPU, fpu_stmm4, "stmm4", "st4", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm4), AVX_OFFSET(stmm4), INVALID_NUB_REGNUM, dwarf_stmm4,
-     INVALID_NUB_REGNUM, debugserver_stmm4, NULL, NULL},
-    {e_regSetFPU, fpu_stmm5, "stmm5", "st5", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm5), AVX_OFFSET(stmm5), INVALID_NUB_REGNUM, dwarf_stmm5,
-     INVALID_NUB_REGNUM, debugserver_stmm5, NULL, NULL},
-    {e_regSetFPU, fpu_stmm6, "stmm6", "st6", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm6), AVX_OFFSET(stmm6), INVALID_NUB_REGNUM, dwarf_stmm6,
-     INVALID_NUB_REGNUM, debugserver_stmm6, NULL, NULL},
-    {e_regSetFPU, fpu_stmm7, "stmm7", "st7", Vector, VectorOfUInt8,
-     FPU_SIZE_MMST(stmm7), AVX_OFFSET(stmm7), INVALID_NUB_REGNUM, dwarf_stmm7,
-     INVALID_NUB_REGNUM, debugserver_stmm7, NULL, NULL},
-
-    AVX512_K_REG_DEF(0),
-    AVX512_K_REG_DEF(1),
-    AVX512_K_REG_DEF(2),
-    AVX512_K_REG_DEF(3),
-    AVX512_K_REG_DEF(4),
-    AVX512_K_REG_DEF(5),
-    AVX512_K_REG_DEF(6),
-    AVX512_K_REG_DEF(7),
-
-    ZMM_REG_DEF(0),
-    ZMM_REG_DEF(1),
-    ZMM_REG_DEF(2),
-    ZMM_REG_DEF(3),
-    ZMM_REG_DEF(4),
-    ZMM_REG_DEF(5),
-    ZMM_REG_DEF(6),
-    ZMM_REG_DEF(7),
-
-    YMM_REG_ALIAS(0),
-    YMM_REG_ALIAS(1),
-    YMM_REG_ALIAS(2),
-    YMM_REG_ALIAS(3),
-    YMM_REG_ALIAS(4),
-    YMM_REG_ALIAS(5),
-    YMM_REG_ALIAS(6),
-    YMM_REG_ALIAS(7),
-
-    XMM_REG_ALIAS(0),
-    XMM_REG_ALIAS(1),
-    XMM_REG_ALIAS(2),
-    XMM_REG_ALIAS(3),
-    XMM_REG_ALIAS(4),
-    XMM_REG_ALIAS(5),
-    XMM_REG_ALIAS(6),
-    XMM_REG_ALIAS(7)
-
-};
-
-const DNBRegisterInfo DNBArchImplI386::g_exc_registers[] = {
-    {e_regSetEXC, exc_trapno, "trapno", NULL, Uint, Hex, EXC_SIZE(trapno),
-     EXC_OFFSET(trapno), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetEXC, exc_err, "err", NULL, Uint, Hex, EXC_SIZE(err),
-     EXC_OFFSET(err), INVALID_NUB_REGNUM, INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL},
-    {e_regSetEXC, exc_faultvaddr, "faultvaddr", NULL, Uint, Hex,
-     EXC_SIZE(faultvaddr), EXC_OFFSET(faultvaddr), INVALID_NUB_REGNUM,
-     INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, INVALID_NUB_REGNUM, NULL, NULL}};
-
-// Number of registers in each register set
-const size_t DNBArchImplI386::k_num_gpr_registers =
-    sizeof(g_gpr_registers) / sizeof(DNBRegisterInfo);
-const size_t DNBArchImplI386::k_num_fpu_registers_no_avx =
-    sizeof(g_fpu_registers_no_avx) / sizeof(DNBRegisterInfo);
-const size_t DNBArchImplI386::k_num_fpu_registers_avx =
-    sizeof(g_fpu_registers_avx) / sizeof(DNBRegisterInfo);
-const size_t DNBArchImplI386::k_num_fpu_registers_avx512f =
-    sizeof(g_fpu_registers_avx512f) / sizeof(DNBRegisterInfo);
-const size_t DNBArchImplI386::k_num_exc_registers =
-    sizeof(g_exc_registers) / sizeof(DNBRegisterInfo);
-const size_t DNBArchImplI386::k_num_all_registers_no_avx =
-    k_num_gpr_registers + k_num_fpu_registers_no_avx + k_num_exc_registers;
-const size_t DNBArchImplI386::k_num_all_registers_avx =
-    k_num_gpr_registers + k_num_fpu_registers_avx + k_num_exc_registers;
-const size_t DNBArchImplI386::k_num_all_registers_avx512f =
-    k_num_gpr_registers + k_num_fpu_registers_avx512f + k_num_exc_registers;
-
-// Register set definitions. The first definitions at register set index
-// of zero is for all registers, followed by other registers sets. The
-// register information for the all register set need not be filled in.
-const DNBRegisterSetInfo DNBArchImplI386::g_reg_sets_no_avx[] = {
-    {"i386 Registers", NULL, k_num_all_registers_no_avx},
-    {"General Purpose Registers", g_gpr_registers, k_num_gpr_registers},
-    {"Floating Point Registers", g_fpu_registers_no_avx,
-     k_num_fpu_registers_no_avx},
-    {"Exception State Registers", g_exc_registers, k_num_exc_registers}};
-
-const DNBRegisterSetInfo DNBArchImplI386::g_reg_sets_avx[] = {
-    {"i386 Registers", NULL, k_num_all_registers_avx},
-    {"General Purpose Registers", g_gpr_registers, k_num_gpr_registers},
-    {"Floating Point Registers", g_fpu_registers_avx, k_num_fpu_registers_avx},
-    {"Exception State Registers", g_exc_registers, k_num_exc_registers}};
-
-const DNBRegisterSetInfo DNBArchImplI386::g_reg_sets_avx512f[] = {
-    {"i386 Registers", NULL, k_num_all_registers_avx512f},
-    {"General Purpose Registers", g_gpr_registers, k_num_gpr_registers},
-    {"Floating Point Registers", g_fpu_registers_avx512f,
-    k_num_fpu_registers_avx512f},
-    {"Exception State Registers", g_exc_registers, k_num_exc_registers}};
-
-// Total number of register sets for this architecture
-const size_t DNBArchImplI386::k_num_register_sets =
-    sizeof(g_reg_sets_avx) / sizeof(DNBRegisterSetInfo);
-
-DNBArchProtocol *DNBArchImplI386::Create(MachThread *thread) {
-  DNBArchImplI386 *obj = new DNBArchImplI386(thread);
-  return obj;
-}
-
-const uint8_t *DNBArchImplI386::SoftwareBreakpointOpcode(nub_size_t byte_size) {
-  static const uint8_t g_breakpoint_opcode[] = {0xCC};
-  if (byte_size == 1)
-    return g_breakpoint_opcode;
-  return NULL;
-}
-
-const DNBRegisterSetInfo *
-DNBArchImplI386::GetRegisterSetInfo(nub_size_t *num_reg_sets) {
-  *num_reg_sets = k_num_register_sets;
-  if (CPUHasAVX512f() || FORCE_AVX_REGS)
-    return g_reg_sets_avx512f;
-  if (CPUHasAVX())
-    return g_reg_sets_avx;
-  else
-    return g_reg_sets_no_avx;
-}
-
-void DNBArchImplI386::Initialize() {
-  DNBArchPluginInfo arch_plugin_info = {
-      CPU_TYPE_I386, DNBArchImplI386::Create,
-      DNBArchImplI386::GetRegisterSetInfo,
-      DNBArchImplI386::SoftwareBreakpointOpcode};
-
-  // Register this arch plug-in with the main protocol class
-  DNBArchProtocol::RegisterArchPlugin(arch_plugin_info);
-}
-
-bool DNBArchImplI386::GetRegisterValue(uint32_t set, uint32_t reg,
-                                       DNBRegisterValue *value) {
-  if (set == REGISTER_SET_GENERIC) {
-    switch (reg) {
-    case GENERIC_REGNUM_PC: // Program Counter
-      set = e_regSetGPR;
-      reg = gpr_eip;
-      break;
-
-    case GENERIC_REGNUM_SP: // Stack Pointer
-      set = e_regSetGPR;
-      reg = gpr_esp;
-      break;
-
-    case GENERIC_REGNUM_FP: // Frame Pointer
-      set = e_regSetGPR;
-      reg = gpr_ebp;
-      break;
-
-    case GENERIC_REGNUM_FLAGS: // Processor flags register
-      set = e_regSetGPR;
-      reg = gpr_eflags;
-      break;
-
-    case GENERIC_REGNUM_RA: // Return Address
-    default:
-      return false;
-    }
-  }
-
-  if (GetRegisterState(set, false) != KERN_SUCCESS)
-    return false;
-
-  const DNBRegisterInfo *regInfo = m_thread->GetRegisterInfo(set, reg);
-  if (regInfo) {
-    value->info = *regInfo;
-    switch (set) {
-    case e_regSetGPR:
-      if (reg < k_num_gpr_registers) {
-        value->value.uint32 = ((uint32_t *)(&m_state.context.gpr))[reg];
-        return true;
-      }
-      break;
-
-    case e_regSetFPU:
-      if (reg > fpu_xmm7 && !(CPUHasAVX() || FORCE_AVX_REGS))
-        return false;
-      if (reg > fpu_ymm7 && !(CPUHasAVX512f() || FORCE_AVX_REGS))
-        return false;
-      switch (reg) {
-      case fpu_fcw:
-        value->value.uint16 =
-            *((uint16_t *)(&m_state.context.fpu.no_avx.__fpu_fcw));
-        return true;
-      case fpu_fsw:
-        value->value.uint16 =
-            *((uint16_t *)(&m_state.context.fpu.no_avx.__fpu_fsw));
-        return true;
-      case fpu_ftw:
-        memcpy (&value->value.uint16, &m_state.context.fpu.no_avx.__fpu_ftw, 2);
-        return true;
-      case fpu_fop:
-        value->value.uint16 = m_state.context.fpu.no_avx.__fpu_fop;
-        return true;
-      case fpu_ip:
-        value->value.uint32 = m_state.context.fpu.no_avx.__fpu_ip;
-        return true;
-      case fpu_cs:
-        value->value.uint16 = m_state.context.fpu.no_avx.__fpu_cs;
-        return true;
-      case fpu_dp:
-        value->value.uint32 = m_state.context.fpu.no_avx.__fpu_dp;
-        return true;
-      case fpu_ds:
-        value->value.uint16 = m_state.context.fpu.no_avx.__fpu_ds;
-        return true;
-      case fpu_mxcsr:
-        value->value.uint32 = m_state.context.fpu.no_avx.__fpu_mxcsr;
-        return true;
-      case fpu_mxcsrmask:
-        value->value.uint32 = m_state.context.fpu.no_avx.__fpu_mxcsrmask;
-        return true;
-
-      case fpu_stmm0:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm0.__mmst_reg, 10);
-        return true;
-      case fpu_stmm1:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm1.__mmst_reg, 10);
-        return true;
-      case fpu_stmm2:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm2.__mmst_reg, 10);
-        return true;
-      case fpu_stmm3:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm3.__mmst_reg, 10);
-        return true;
-      case fpu_stmm4:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm4.__mmst_reg, 10);
-        return true;
-      case fpu_stmm5:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm5.__mmst_reg, 10);
-        return true;
-      case fpu_stmm6:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm6.__mmst_reg, 10);
-        return true;
-      case fpu_stmm7:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_stmm7.__mmst_reg, 10);
-        return true;
-
-      case fpu_xmm0:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm0.__xmm_reg, 16);
-        return true;
-      case fpu_xmm1:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm1.__xmm_reg, 16);
-        return true;
-      case fpu_xmm2:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm2.__xmm_reg, 16);
-        return true;
-      case fpu_xmm3:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm3.__xmm_reg, 16);
-        return true;
-      case fpu_xmm4:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm4.__xmm_reg, 16);
-        return true;
-      case fpu_xmm5:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm5.__xmm_reg, 16);
-        return true;
-      case fpu_xmm6:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm6.__xmm_reg, 16);
-        return true;
-      case fpu_xmm7:
-        memcpy(&value->value.uint8,
-               m_state.context.fpu.no_avx.__fpu_xmm7.__xmm_reg, 16);
-        return true;
-
-#define MEMCPY_YMM(n)                                                          \
-  memcpy(&value->value.uint8, m_state.context.fpu.avx.__fpu_xmm##n.__xmm_reg,  \
-         16);                                                                  \
-  memcpy((&value->value.uint8) + 16,                                           \
-         m_state.context.fpu.avx.__fpu_ymmh##n.__xmm_reg, 16);
-      case fpu_ymm0:
-        MEMCPY_YMM(0);
-        return true;
-      case fpu_ymm1:
-        MEMCPY_YMM(1);
-        return true;
-      case fpu_ymm2:
-        MEMCPY_YMM(2);
-        return true;
-      case fpu_ymm3:
-        MEMCPY_YMM(3);
-        return true;
-      case fpu_ymm4:
-        MEMCPY_YMM(4);
-        return true;
-      case fpu_ymm5:
-        MEMCPY_YMM(5);
-        return true;
-      case fpu_ymm6:
-        MEMCPY_YMM(6);
-        return true;
-      case fpu_ymm7:
-        MEMCPY_YMM(7);
-        return true;
-#undef MEMCPY_YMM
-
-      case fpu_k0:
-      case fpu_k1:
-      case fpu_k2:
-      case fpu_k3:
-      case fpu_k4:
-      case fpu_k5:
-      case fpu_k6:
-      case fpu_k7:
-        memcpy((&value->value.uint8),
-               &m_state.context.fpu.avx512f.__fpu_k0 + (reg - fpu_k0), 8);
-        return true;
-      case fpu_zmm0:
-      case fpu_zmm1:
-      case fpu_zmm2:
-      case fpu_zmm3:
-      case fpu_zmm4:
-      case fpu_zmm5:
-      case fpu_zmm6:
-      case fpu_zmm7:
-        memcpy(&value->value.uint8,
-               &m_state.context.fpu.avx512f.__fpu_xmm0 + (reg - fpu_zmm0), 16);
-        memcpy(&value->value.uint8 + 16,
-               &m_state.context.fpu.avx512f.__fpu_ymmh0 + (reg - fpu_zmm0), 16);
-        memcpy(&value->value.uint8 + 32,
-               &m_state.context.fpu.avx512f.__fpu_zmmh0 + (reg - fpu_zmm0), 32);
-        return true;
-      }
-      break;
-
-    case e_regSetEXC:
-      if (reg < k_num_exc_registers) {
-        value->value.uint32 = (&m_state.context.exc.__trapno)[reg];
-        return true;
-      }
-      break;
-    }
-  }
-  return false;
-}
-
-bool DNBArchImplI386::SetRegisterValue(uint32_t set, uint32_t reg,
-                                       const DNBRegisterValue *value) {
-  if (set == REGISTER_SET_GENERIC) {
-    switch (reg) {
-    case GENERIC_REGNUM_PC: // Program Counter
-      set = e_regSetGPR;
-      reg = gpr_eip;
-      break;
-
-    case GENERIC_REGNUM_SP: // Stack Pointer
-      set = e_regSetGPR;
-      reg = gpr_esp;
-      break;
-
-    case GENERIC_REGNUM_FP: // Frame Pointer
-      set = e_regSetGPR;
-      reg = gpr_ebp;
-      break;
-
-    case GENERIC_REGNUM_FLAGS: // Processor flags register
-      set = e_regSetGPR;
-      reg = gpr_eflags;
-      break;
-
-    case GENERIC_REGNUM_RA: // Return Address
-    default:
-      return false;
-    }
-  }
-
-  if (GetRegisterState(set, false) != KERN_SUCCESS)
-    return false;
-
-  bool success = false;
-  const DNBRegisterInfo *regInfo = m_thread->GetRegisterInfo(set, reg);
-  if (regInfo) {
-    switch (set) {
-    case e_regSetGPR:
-      if (reg < k_num_gpr_registers) {
-        ((uint32_t *)(&m_state.context.gpr))[reg] = value->value.uint32;
-        success = true;
-      }
-      break;
-
-    case e_regSetFPU:
-      if (reg > fpu_xmm7 && !(CPUHasAVX() || FORCE_AVX_REGS))
-        return false;
-      if (reg > fpu_ymm7 && !(CPUHasAVX512f() || FORCE_AVX_REGS))
-        return false;
-      switch (reg) {
-      case fpu_fcw:
-        *((uint16_t *)(&m_state.context.fpu.no_avx.__fpu_fcw)) =
-            value->value.uint16;
-        success = true;
-        break;
-      case fpu_fsw:
-        *((uint16_t *)(&m_state.context.fpu.no_avx.__fpu_fsw)) =
-            value->value.uint16;
-        success = true;
-        break;
-      case fpu_ftw:
-        memcpy (&m_state.context.fpu.no_avx.__fpu_ftw, &value->value.uint16, 2);
-        success = true;
-        break;
-      case fpu_fop:
-        m_state.context.fpu.no_avx.__fpu_fop = value->value.uint16;
-        success = true;
-        break;
-      case fpu_ip:
-        m_state.context.fpu.no_avx.__fpu_ip = value->value.uint32;
-        success = true;
-        break;
-      case fpu_cs:
-        m_state.context.fpu.no_avx.__fpu_cs = value->value.uint16;
-        success = true;
-        break;
-      case fpu_dp:
-        m_state.context.fpu.no_avx.__fpu_dp = value->value.uint32;
-        success = true;
-        break;
-      case fpu_ds:
-        m_state.context.fpu.no_avx.__fpu_ds = value->value.uint16;
-        success = true;
-        break;
-      case fpu_mxcsr:
-        m_state.context.fpu.no_avx.__fpu_mxcsr = value->value.uint32;
-        success = true;
-        break;
-      case fpu_mxcsrmask:
-        m_state.context.fpu.no_avx.__fpu_mxcsrmask = value->value.uint32;
-        success = true;
-        break;
-
-      case fpu_stmm0:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm0.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm1:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm1.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm2:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm2.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm3:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm3.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm4:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm4.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm5:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm5.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm6:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm6.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-      case fpu_stmm7:
-        memcpy(m_state.context.fpu.no_avx.__fpu_stmm7.__mmst_reg,
-               &value->value.uint8, 10);
-        success = true;
-        break;
-
-      case fpu_xmm0:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm0.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm1:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm1.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm2:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm2.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm3:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm3.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm4:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm4.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm5:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm5.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm6:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm6.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-      case fpu_xmm7:
-        memcpy(m_state.context.fpu.no_avx.__fpu_xmm7.__xmm_reg,
-               &value->value.uint8, 16);
-        success = true;
-        break;
-
-#define MEMCPY_YMM(n)                                                          \
-  memcpy(m_state.context.fpu.avx.__fpu_xmm##n.__xmm_reg, &value->value.uint8,  \
-         16);                                                                  \
-  memcpy(m_state.context.fpu.avx.__fpu_ymmh##n.__xmm_reg,                      \
-         (&value->value.uint8) + 16, 16);
-      case fpu_ymm0:
-        MEMCPY_YMM(0);
-        return true;
-      case fpu_ymm1:
-        MEMCPY_YMM(1);
-        return true;
-      case fpu_ymm2:
-        MEMCPY_YMM(2);
-        return true;
-      case fpu_ymm3:
-        MEMCPY_YMM(3);
-        return true;
-      case fpu_ymm4:
-        MEMCPY_YMM(4);
-        return true;
-      case fpu_ymm5:
-        MEMCPY_YMM(5);
-        return true;
-      case fpu_ymm6:
-        MEMCPY_YMM(6);
-        return true;
-      case fpu_ymm7:
-        MEMCPY_YMM(7);
-        return true;
-#undef MEMCPY_YMM
-
-      case fpu_k0:
-      case fpu_k1:
-      case fpu_k2:
-      case fpu_k3:
-      case fpu_k4:
-      case fpu_k5:
-      case fpu_k6:
-      case fpu_k7:
-        memcpy(&m_state.context.fpu.avx512f.__fpu_k0 + (reg - fpu_k0),
-               &value->value.uint8, 8);
-        return true;
-      case fpu_zmm0:
-      case fpu_zmm1:
-      case fpu_zmm2:
-      case fpu_zmm3:
-      case fpu_zmm4:
-      case fpu_zmm5:
-      case fpu_zmm6:
-      case fpu_zmm7:
-        memcpy(&m_state.context.fpu.avx512f.__fpu_xmm0 + (reg - fpu_zmm0),
-               &value->value.uint8, 16);
-        memcpy(&m_state.context.fpu.avx512f.__fpu_ymmh0 + (reg - fpu_zmm0),
-               &value->value.uint8 + 16, 16);
-        memcpy(&m_state.context.fpu.avx512f.__fpu_zmmh0 + (reg - fpu_zmm0),
-               &value->value.uint8 + 32, 32);
-        return true;
-      }
-      break;
-
-    case e_regSetEXC:
-      if (reg < k_num_exc_registers) {
-        (&m_state.context.exc.__trapno)[reg] = value->value.uint32;
-        success = true;
-      }
-      break;
-    }
-  }
-
-  if (success)
-    return SetRegisterState(set) == KERN_SUCCESS;
-  return false;
-}
-
-uint32_t DNBArchImplI386::GetRegisterContextSize() {
-  static uint32_t g_cached_size = 0;
-  if (g_cached_size == 0) {
-    if(CPUHasAVX512f() || FORCE_AVX_REGS) {
-      for (size_t i = 0; i < k_num_fpu_registers_avx512f; ++i) {
-        if (g_fpu_registers_avx512f[i].value_regs == NULL)
-          g_cached_size += g_fpu_registers_avx512f[i].size;
-      }
-    } else 
-    if (CPUHasAVX()) {
-      for (size_t i = 0; i < k_num_fpu_registers_avx; ++i) {
-        if (g_fpu_registers_avx[i].value_regs == NULL)
-          g_cached_size += g_fpu_registers_avx[i].size;
-      }
-    } else {
-      for (size_t i = 0; i < k_num_fpu_registers_no_avx; ++i) {
-        if (g_fpu_registers_no_avx[i].value_regs == NULL)
-          g_cached_size += g_fpu_registers_no_avx[i].size;
-      }
-    }
-    DNBLogThreaded("DNBArchImplX86_64::GetRegisterContextSize() - GPR = %zu, "
-                   "FPU = %u, EXC = %zu",
-                   sizeof(GPR), g_cached_size, sizeof(EXC));
-    g_cached_size += sizeof(GPR);
-    g_cached_size += sizeof(EXC);
-    DNBLogThreaded(
-        "DNBArchImplX86_64::GetRegisterContextSize() - GPR + FPU + EXC = %u",
-        g_cached_size);
-  }
-  return g_cached_size;
-}
-
-nub_size_t DNBArchImplI386::GetRegisterContext(void *buf, nub_size_t buf_len) {
-  uint32_t size = GetRegisterContextSize();
-
-  if (buf && buf_len) {
-    if (size > buf_len)
-      size = static_cast<uint32_t>(buf_len);
-
-    bool force = false;
-    kern_return_t kret;
-    if ((kret = GetGPRState(force)) != KERN_SUCCESS) {
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::GetRegisterContext (buf = "
-                                   "%p, len = %llu) error: GPR regs failed to "
-                                   "read: %u ",
-                       buf, (uint64_t)buf_len, kret);
-      size = 0;
-    } else if ((kret = GetFPUState(force)) != KERN_SUCCESS) {
-      DNBLogThreadedIf(
-          LOG_THREAD, "DNBArchImplI386::GetRegisterContext (buf = %p, len = "
-                      "%llu) error: %s regs failed to read: %u",
-          buf, (uint64_t)buf_len, CPUHasAVX() ? "AVX" : "FPU", kret);
-      size = 0;
-    } else if ((kret = GetEXCState(force)) != KERN_SUCCESS) {
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::GetRegisterContext (buf = "
-                                   "%p, len = %llu) error: EXC regs failed to "
-                                   "read: %u",
-                       buf, (uint64_t)buf_len, kret);
-      size = 0;
-    } else {
-      uint8_t *p = (uint8_t *)buf;
-      // Copy the GPR registers
-      memcpy(p, &m_state.context.gpr, sizeof(GPR));
-      p += sizeof(GPR);
-
-      // Walk around the gaps in the FPU regs
-      memcpy(p, &m_state.context.fpu.no_avx.__fpu_fcw, 5);
-      p += 5;
-      memcpy(p, &m_state.context.fpu.no_avx.__fpu_fop, 8);
-      p += 8;
-      memcpy(p, &m_state.context.fpu.no_avx.__fpu_dp, 6);
-      p += 6;
-      memcpy(p, &m_state.context.fpu.no_avx.__fpu_mxcsr, 8);
-      p += 8;
-
-      // Work around the padding between the stmm registers as they are 16
-      // byte structs with 10 bytes of the value in each
-      for (size_t i = 0; i < 8; ++i) {
-        memcpy(p, &m_state.context.fpu.no_avx.__fpu_stmm0 + i, 10);
-        p += 10;
-      }
-
-      if (CPUHasAVX512f() || FORCE_AVX_REGS) {
-        for (size_t i = 0; i < 8; ++i) {
-          memcpy(p, &m_state.context.fpu.avx512f.__fpu_k0 + i, 8);
-          p += 8;
-        }
-      }
-
-      if (CPUHasAVX() || FORCE_AVX_REGS) {
-        // Interleave the XMM and YMMH registers to make the YMM registers
-        for (size_t i = 0; i < 8; ++i) {
-          memcpy(p, &m_state.context.fpu.avx.__fpu_xmm0 + i, 16);
-          p += 16;
-          memcpy(p, &m_state.context.fpu.avx.__fpu_ymmh0 + i, 16);
-          p += 16;
-        }
-        if(CPUHasAVX512f() || FORCE_AVX_REGS) {
-          for (size_t i = 0; i < 8; ++i) {
-            memcpy(p, &m_state.context.fpu.avx512f.__fpu_zmmh0 + i, 32);
-            p += 32;
-          }
-        }
-      } else {
-        // Copy the XMM registers in a single block
-        memcpy(p, &m_state.context.fpu.no_avx.__fpu_xmm0, 8 * 16);
-        p += 8 * 16;
-      }
-
-      // Copy the exception registers
-      memcpy(p, &m_state.context.exc, sizeof(EXC));
-      p += sizeof(EXC);
-
-      // make sure we end up with exactly what we think we should have
-      size_t bytes_written = p - (uint8_t *)buf;
-      UNUSED_IF_ASSERT_DISABLED(bytes_written);
-      assert(bytes_written == size);
-    }
-  }
-  DNBLogThreadedIf(
-      LOG_THREAD,
-      "DNBArchImplI386::GetRegisterContext (buf = %p, len = %llu) => %llu", buf,
-      (uint64_t)buf_len, (uint64_t)size);
-  // Return the size of the register context even if NULL was passed in
-  return size;
-}
-
-nub_size_t DNBArchImplI386::SetRegisterContext(const void *buf,
-                                               nub_size_t buf_len) {
-  nub_size_t size = sizeof(m_state.context);
-  if (buf == NULL || buf_len == 0)
-    size = 0;
-
-  if (size) {
-    if (size > buf_len)
-      size = buf_len;
-
-    const uint8_t *p = (const uint8_t *)buf;
-    // Copy the GPR registers
-    memcpy(&m_state.context.gpr, p, sizeof(GPR));
-    p += sizeof(GPR);
-
-    // Copy fcw through mxcsrmask as there is no padding
-    memcpy(&m_state.context.fpu.no_avx.__fpu_fcw, p, 5);
-    p += 5;
-    memcpy(&m_state.context.fpu.no_avx.__fpu_fop, p, 8);
-    p += 8;
-    memcpy(&m_state.context.fpu.no_avx.__fpu_dp, p, 6);
-    p += 6;
-    memcpy(&m_state.context.fpu.no_avx.__fpu_mxcsr, p, 8);
-    p += 8;
-
-    // Work around the padding between the stmm registers as they are 16
-    // byte structs with 10 bytes of the value in each
-    for (size_t i = 0; i < 8; ++i) {
-      memcpy(&m_state.context.fpu.no_avx.__fpu_stmm0 + i, p, 10);
-      p += 10;
-    }
-
-    if(CPUHasAVX512f() || FORCE_AVX_REGS) {
-      for (size_t i = 0; i < 8; ++i) {
-        memcpy(&m_state.context.fpu.avx512f.__fpu_k0 + i, p, 8);
-        p += 8;
-      }
-    }
-
-    if (CPUHasAVX() || FORCE_AVX_REGS) {
-      // Interleave the XMM and YMMH registers to make the YMM registers
-      for (size_t i = 0; i < 8; ++i) {
-        memcpy(&m_state.context.fpu.avx.__fpu_xmm0 + i, p, 16);
-        p += 16;
-        memcpy(&m_state.context.fpu.avx.__fpu_ymmh0 + i, p, 16);
-        p += 16;
-      }
-
-      if(CPUHasAVX512f() || FORCE_AVX_REGS) {
-        for (size_t i = 0; i < 8; ++i) {
-          memcpy(&m_state.context.fpu.avx512f.__fpu_zmmh0 + i, p, 32);
-          p += 32;
-        }
-      }
-    } else {
-      // Copy the XMM registers in a single block
-      memcpy(&m_state.context.fpu.no_avx.__fpu_xmm0, p, 8 * 16);
-      p += 8 * 16;
-    }
-
-    // Copy the exception registers
-    memcpy(&m_state.context.exc, p, sizeof(EXC));
-    p += sizeof(EXC);
-
-    // make sure we end up with exactly what we think we should have
-    size_t bytes_written = p - (const uint8_t *)buf;
-    UNUSED_IF_ASSERT_DISABLED(bytes_written);
-    assert(bytes_written == size);
-    kern_return_t kret;
-    if ((kret = SetGPRState()) != KERN_SUCCESS)
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::SetRegisterContext (buf = "
-                                   "%p, len = %llu) error: GPR regs failed to "
-                                   "write: %u",
-                       buf, (uint64_t)buf_len, kret);
-    if ((kret = SetFPUState()) != KERN_SUCCESS)
-      DNBLogThreadedIf(
-          LOG_THREAD, "DNBArchImplI386::SetRegisterContext (buf = %p, len = "
-                      "%llu) error: %s regs failed to write: %u",
-          buf, (uint64_t)buf_len, CPUHasAVX() ? "AVX" : "FPU", kret);
-    if ((kret = SetEXCState()) != KERN_SUCCESS)
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::SetRegisterContext (buf = "
-                                   "%p, len = %llu) error: EXP regs failed to "
-                                   "write: %u",
-                       buf, (uint64_t)buf_len, kret);
-  }
-  DNBLogThreadedIf(
-      LOG_THREAD,
-      "DNBArchImplI386::SetRegisterContext (buf = %p, len = %llu) => %llu", buf,
-      (uint64_t)buf_len, (uint64_t)size);
-  return size;
-}
-
-uint32_t DNBArchImplI386::SaveRegisterState() {
-  kern_return_t kret = ::thread_abort_safely(m_thread->MachPortNumber());
-  DNBLogThreadedIf(
-      LOG_THREAD, "thread = 0x%4.4x calling thread_abort_safely (tid) => %u "
-                  "(SetGPRState() for stop_count = %u)",
-      m_thread->MachPortNumber(), kret, m_thread->Process()->StopCount());
-
-  bool force = true;
-
-  if ((kret = GetGPRState(force)) != KERN_SUCCESS) {
-    DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::SaveRegisterState () error: "
-                                 "GPR regs failed to read: %u ",
-                     kret);
-  } else if ((kret = GetFPUState(force)) != KERN_SUCCESS) {
-    DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::SaveRegisterState () error: "
-                                 "%s regs failed to read: %u",
-                     CPUHasAVX() ? "AVX" : "FPU", kret);
-  } else {
-    const uint32_t save_id = GetNextRegisterStateSaveID();
-    m_saved_register_states[save_id] = m_state.context;
-    return save_id;
-  }
-  return 0;
-}
-bool DNBArchImplI386::RestoreRegisterState(uint32_t save_id) {
-  SaveRegisterStates::iterator pos = m_saved_register_states.find(save_id);
-  if (pos != m_saved_register_states.end()) {
-    m_state.context.gpr = pos->second.gpr;
-    m_state.context.fpu = pos->second.fpu;
-    m_state.context.exc = pos->second.exc;
-    m_state.SetError(e_regSetGPR, Read, 0);
-    m_state.SetError(e_regSetFPU, Read, 0);
-    m_state.SetError(e_regSetEXC, Read, 0);
-    kern_return_t kret;
-    bool success = true;
-    if ((kret = SetGPRState()) != KERN_SUCCESS) {
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::RestoreRegisterState "
-                                   "(save_id = %u) error: GPR regs failed to "
-                                   "write: %u",
-                       save_id, kret);
-      success = false;
-    } else if ((kret = SetFPUState()) != KERN_SUCCESS) {
-      DNBLogThreadedIf(LOG_THREAD, "DNBArchImplI386::RestoreRegisterState "
-                                   "(save_id = %u) error: %s regs failed to "
-                                   "write: %u",
-                       save_id, CPUHasAVX() ? "AVX" : "FPU", kret);
-      success = false;
-    }
-    m_saved_register_states.erase(pos);
-    return success;
-  }
-  return false;
-}
-
-kern_return_t DNBArchImplI386::GetRegisterState(int set, bool force) {
-  switch (set) {
-  case e_regSetALL:
-    return GetGPRState(force) | GetFPUState(force) | GetEXCState(force);
-  case e_regSetGPR:
-    return GetGPRState(force);
-  case e_regSetFPU:
-    return GetFPUState(force);
-  case e_regSetEXC:
-    return GetEXCState(force);
-  default:
-    break;
-  }
-  return KERN_INVALID_ARGUMENT;
-}
-
-kern_return_t DNBArchImplI386::SetRegisterState(int set) {
-  // Make sure we have a valid context to set.
-  if (RegisterSetStateIsValid(set)) {
-    switch (set) {
-    case e_regSetALL:
-      return SetGPRState() | SetFPUState() | SetEXCState();
-    case e_regSetGPR:
-      return SetGPRState();
-    case e_regSetFPU:
-      return SetFPUState();
-    case e_regSetEXC:
-      return SetEXCState();
-    default:
-      break;
-    }
-  }
-  return KERN_INVALID_ARGUMENT;
-}
-
-bool DNBArchImplI386::RegisterSetStateIsValid(int set) const {
-  return m_state.RegsAreValid(set);
-}
-
-#endif // #if defined (__i386__)
diff --git a/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.h b/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.h
deleted file mode 100644
index a702ea52e8536..0000000000000
--- a/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.h
+++ /dev/null
@@ -1,246 +0,0 @@
-//===-- DNBArchImplI386.h ---------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  Created by Greg Clayton on 6/25/07.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_I386_DNBARCHIMPLI386_H
-#define LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_I386_DNBARCHIMPLI386_H
-
-#if defined(__i386__) || defined(__x86_64__)
-
-#include "DNBArch.h"
-#include "MachRegisterStatesI386.h"
-
-#include <map>
-
-class MachThread;
-
-class DNBArchImplI386 : public DNBArchProtocol {
-public:
-  DNBArchImplI386(MachThread *thread)
-      : DNBArchProtocol(), m_thread(thread), m_state(), m_2pc_dbg_checkpoint(),
-        m_2pc_trans_state(Trans_Done), m_saved_register_states() {}
-  virtual ~DNBArchImplI386() {}
-
-  static void Initialize();
-
-  bool GetRegisterValue(uint32_t set, uint32_t reg,
-                        DNBRegisterValue *value) override;
-  bool SetRegisterValue(uint32_t set, uint32_t reg,
-                        const DNBRegisterValue *value) override;
-  nub_size_t GetRegisterContext(void *buf, nub_size_t buf_len) override;
-  nub_size_t SetRegisterContext(const void *buf, nub_size_t buf_len) override;
-  uint32_t SaveRegisterState() override;
-  bool RestoreRegisterState(uint32_t save_id) override;
-
-  kern_return_t GetRegisterState(int set, bool force) override;
-  kern_return_t SetRegisterState(int set) override;
-  bool RegisterSetStateIsValid(int set) const override;
-
-  uint64_t GetPC(uint64_t failValue) override; // Get program counter
-  kern_return_t SetPC(uint64_t value) override;
-  uint64_t GetSP(uint64_t failValue) override; // Get stack pointer
-  void ThreadWillResume() override;
-  bool ThreadDidStop() override;
-  bool NotifyException(MachException::Data &exc) override;
-
-  uint32_t NumSupportedHardwareBreakpoints() override;
-  uint32_t NumSupportedHardwareWatchpoints() override;
-  uint32_t EnableHardwareBreakpoint(nub_addr_t addr, nub_size_t size,
-                                    bool also_set_on_task) override;
-  bool DisableHardwareBreakpoint(uint32_t hw_index,
-                                 bool also_set_on_task) override;
-  uint32_t EnableHardwareWatchpoint(nub_addr_t addr, nub_size_t size,
-                                    bool read, bool write,
-                                    bool also_set_on_task) override;
-  bool DisableHardwareWatchpoint(uint32_t hw_break_index,
-                                 bool also_set_on_task) override;
-  uint32_t GetHardwareWatchpointHit(nub_addr_t &addr) override;
-
-protected:
-  kern_return_t EnableHardwareSingleStep(bool enable);
-
-  typedef __i386_thread_state_t GPR;
-  typedef __i386_float_state_t FPU;
-  typedef __i386_exception_state_t EXC;
-  typedef __i386_avx_state_t AVX;
-  typedef __i386_debug_state_t DBG;
-
-  static const DNBRegisterInfo g_gpr_registers[];
-  static const DNBRegisterInfo g_fpu_registers_no_avx[];
-  static const DNBRegisterInfo g_fpu_registers_avx[];
-  static const DNBRegisterInfo g_exc_registers[];
-  static const DNBRegisterSetInfo g_reg_sets_no_avx[];
-  static const DNBRegisterSetInfo g_reg_sets_avx[];
-  static const size_t k_num_gpr_registers;
-  static const size_t k_num_fpu_registers_no_avx;
-  static const size_t k_num_fpu_registers_avx;
-  static const size_t k_num_exc_registers;
-  static const size_t k_num_all_registers_no_avx;
-  static const size_t k_num_all_registers_avx;
-  static const size_t k_num_register_sets;
-
-  typedef __i386_avx512f_state_t AVX512F;
-  static const DNBRegisterInfo g_fpu_registers_avx512f[];
-  static const DNBRegisterSetInfo g_reg_sets_avx512f[];
-  static const size_t k_num_fpu_registers_avx512f;
-  static const size_t k_num_all_registers_avx512f;
-
-  enum RegisterSet {
-    e_regSetALL = REGISTER_SET_ALL,
-    e_regSetGPR,
-    e_regSetFPU,
-    e_regSetEXC,
-    e_regSetDBG,
-    kNumRegisterSets
-  };
-
-  enum RegisterSetWordSize {
-    e_regSetWordSizeGPR = sizeof(GPR) / sizeof(int),
-    e_regSetWordSizeFPU = sizeof(FPU) / sizeof(int),
-    e_regSetWordSizeEXC = sizeof(EXC) / sizeof(int),
-    e_regSetWordSizeAVX = sizeof(AVX) / sizeof(int),
-    e_regSetWordSizeAVX512f = sizeof(AVX512F) / sizeof(int),
-    e_regSetWordSizeDBG = sizeof(DBG) / sizeof(int)
-  };
-
-  enum { Read = 0, Write = 1, kNumErrors = 2 };
-
-  struct Context {
-    GPR gpr;
-    union {
-      FPU no_avx;
-      AVX avx;
-      AVX512F avx512f;
-    } fpu;
-    EXC exc;
-    DBG dbg;
-  };
-
-  struct State {
-    Context context;
-    kern_return_t gpr_errs[2]; // Read/Write errors
-    kern_return_t fpu_errs[2]; // Read/Write errors
-    kern_return_t exc_errs[2]; // Read/Write errors
-    kern_return_t dbg_errs[2]; // Read/Write errors
-
-    State() {
-      uint32_t i;
-      for (i = 0; i < kNumErrors; i++) {
-        gpr_errs[i] = -1;
-        fpu_errs[i] = -1;
-        exc_errs[i] = -1;
-        dbg_errs[i] = -1;
-      }
-    }
-    void InvalidateAllRegisterStates() { SetError(e_regSetALL, Read, -1); }
-    kern_return_t GetError(int flavor, uint32_t err_idx) const {
-      if (err_idx < kNumErrors) {
-        switch (flavor) {
-        // When getting all errors, just OR all values together to see if
-        // we got any kind of error.
-        case e_regSetALL:
-          return gpr_errs[err_idx] | fpu_errs[err_idx] | exc_errs[err_idx];
-        case e_regSetGPR:
-          return gpr_errs[err_idx];
-        case e_regSetFPU:
-          return fpu_errs[err_idx];
-        case e_regSetEXC:
-          return exc_errs[err_idx];
-        case e_regSetDBG:
-          return dbg_errs[err_idx];
-        default:
-          break;
-        }
-      }
-      return -1;
-    }
-    bool SetError(int flavor, uint32_t err_idx, kern_return_t err) {
-      if (err_idx < kNumErrors) {
-        switch (flavor) {
-        case e_regSetALL:
-          gpr_errs[err_idx] = fpu_errs[err_idx] = exc_errs[err_idx] =
-              dbg_errs[err_idx] = err;
-          return true;
-
-        case e_regSetGPR:
-          gpr_errs[err_idx] = err;
-          return true;
-
-        case e_regSetFPU:
-          fpu_errs[err_idx] = err;
-          return true;
-
-        case e_regSetEXC:
-          exc_errs[err_idx] = err;
-          return true;
-
-        case e_regSetDBG:
-          dbg_errs[err_idx] = err;
-          return true;
-
-        default:
-          break;
-        }
-      }
-      return false;
-    }
-    bool RegsAreValid(int flavor) const {
-      return GetError(flavor, Read) == KERN_SUCCESS;
-    }
-  };
-
-  kern_return_t GetGPRState(bool force);
-  kern_return_t GetFPUState(bool force);
-  kern_return_t GetEXCState(bool force);
-  kern_return_t GetDBGState(bool force);
-
-  kern_return_t SetGPRState();
-  kern_return_t SetFPUState();
-  kern_return_t SetEXCState();
-  kern_return_t SetDBGState(bool also_set_on_task);
-
-  static DNBArchProtocol *Create(MachThread *thread);
-
-  static const uint8_t *SoftwareBreakpointOpcode(nub_size_t byte_size);
-
-  static const DNBRegisterSetInfo *GetRegisterSetInfo(nub_size_t *num_reg_sets);
-
-  static uint32_t GetRegisterContextSize();
-
-  static void SetHardwareBreakpoint(DBG &debug_state, uint32_t hw_index,
-                                    nub_addr_t addr, nub_size_t size);
-
-  // Helper functions for watchpoint manipulations.
-  static void SetWatchpoint(DBG &debug_state, uint32_t hw_index,
-                            nub_addr_t addr, nub_size_t size, bool read,
-                            bool write);
-  static void ClearWatchpoint(DBG &debug_state, uint32_t hw_index);
-  static bool IsWatchpointVacant(const DBG &debug_state, uint32_t hw_index);
-  static void ClearWatchpointHits(DBG &debug_state);
-  static bool IsWatchpointHit(const DBG &debug_state, uint32_t hw_index);
-  static nub_addr_t GetWatchAddress(const DBG &debug_state, uint32_t hw_index);
-
-  bool StartTransForHWP() override;
-  bool RollbackTransForHWP() override;
-  bool FinishTransForHWP() override;
-  DBG GetDBGCheckpoint();
-
-  MachThread *m_thread;
-  State m_state;
-  DBG m_2pc_dbg_checkpoint;
-  uint32_t m_2pc_trans_state; // Is transaction of DBG state change: Pedning
-                              // (0), Done (1), or Rolled Back (2)?
-  typedef std::map<uint32_t, Context> SaveRegisterStates;
-  SaveRegisterStates m_saved_register_states;
-};
-
-#endif // #if defined (__i386__) || defined (__x86_64__)
-#endif // LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_I386_DNBARCHIMPLI386_H
diff --git a/lldb/tools/debugserver/source/MacOSX/i386/MachRegisterStatesI386.h b/lldb/tools/debugserver/source/MacOSX/i386/MachRegisterStatesI386.h
deleted file mode 100644
index ba76925369c53..0000000000000
--- a/lldb/tools/debugserver/source/MacOSX/i386/MachRegisterStatesI386.h
+++ /dev/null
@@ -1,241 +0,0 @@
-//===-- MachRegisterStatesI386.h --------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  Created by Sean Callanan on 3/16/11.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_I386_MACHREGISTERSTATESI386_H
-#define LLDB_TOOLS_DEBUGSERVER_SOURCE_MACOSX_I386_MACHREGISTERSTATESI386_H
-
-#include <cinttypes>
-
-#define __i386_THREAD_STATE 1
-#define __i386_FLOAT_STATE 2
-#define __i386_EXCEPTION_STATE 3
-#define __i386_DEBUG_STATE 10
-#define __i386_AVX_STATE 16
-#define __i386_AVX512F_STATE 19
-
-typedef struct {
-  uint32_t __eax;
-  uint32_t __ebx;
-  uint32_t __ecx;
-  uint32_t __edx;
-  uint32_t __edi;
-  uint32_t __esi;
-  uint32_t __ebp;
-  uint32_t __esp;
-  uint32_t __ss;
-  uint32_t __eflags;
-  uint32_t __eip;
-  uint32_t __cs;
-  uint32_t __ds;
-  uint32_t __es;
-  uint32_t __fs;
-  uint32_t __gs;
-} __i386_thread_state_t;
-
-typedef struct {
-  uint16_t __invalid : 1;
-  uint16_t __denorm : 1;
-  uint16_t __zdiv : 1;
-  uint16_t __ovrfl : 1;
-  uint16_t __undfl : 1;
-  uint16_t __precis : 1;
-  uint16_t __PAD1 : 2;
-  uint16_t __pc : 2;
-  uint16_t __rc : 2;
-  uint16_t __PAD2 : 1;
-  uint16_t __PAD3 : 3;
-} __i386_fp_control_t;
-
-typedef struct {
-  uint16_t __invalid : 1;
-  uint16_t __denorm : 1;
-  uint16_t __zdiv : 1;
-  uint16_t __ovrfl : 1;
-  uint16_t __undfl : 1;
-  uint16_t __precis : 1;
-  uint16_t __stkflt : 1;
-  uint16_t __errsumm : 1;
-  uint16_t __c0 : 1;
-  uint16_t __c1 : 1;
-  uint16_t __c2 : 1;
-  uint16_t __tos : 3;
-  uint16_t __c3 : 1;
-  uint16_t __busy : 1;
-} __i386_fp_status_t;
-
-typedef struct {
-  uint8_t __mmst_reg[10];
-  uint8_t __mmst_rsrv[6];
-} __i386_mmst_reg;
-
-typedef struct { uint8_t __xmm_reg[16]; } __i386_xmm_reg;
-
-typedef struct {
-  uint32_t __fpu_reserved[2];
-  __i386_fp_control_t __fpu_fcw;
-  __i386_fp_status_t __fpu_fsw;
-  uint8_t __fpu_ftw;
-  uint8_t __fpu_rsrv1;
-  uint16_t __fpu_fop;
-  uint32_t __fpu_ip;
-  uint16_t __fpu_cs;
-  uint16_t __fpu_rsrv2;
-  uint32_t __fpu_dp;
-  uint16_t __fpu_ds;
-  uint16_t __fpu_rsrv3;
-  uint32_t __fpu_mxcsr;
-  uint32_t __fpu_mxcsrmask;
-  __i386_mmst_reg __fpu_stmm0;
-  __i386_mmst_reg __fpu_stmm1;
-  __i386_mmst_reg __fpu_stmm2;
-  __i386_mmst_reg __fpu_stmm3;
-  __i386_mmst_reg __fpu_stmm4;
-  __i386_mmst_reg __fpu_stmm5;
-  __i386_mmst_reg __fpu_stmm6;
-  __i386_mmst_reg __fpu_stmm7;
-  __i386_xmm_reg __fpu_xmm0;
-  __i386_xmm_reg __fpu_xmm1;
-  __i386_xmm_reg __fpu_xmm2;
-  __i386_xmm_reg __fpu_xmm3;
-  __i386_xmm_reg __fpu_xmm4;
-  __i386_xmm_reg __fpu_xmm5;
-  __i386_xmm_reg __fpu_xmm6;
-  __i386_xmm_reg __fpu_xmm7;
-  uint8_t __fpu_rsrv4[14 * 16];
-  uint32_t __fpu_reserved1;
-} __i386_float_state_t;
-
-typedef struct {
-  uint32_t __fpu_reserved[2];
-  __i386_fp_control_t __fpu_fcw;
-  __i386_fp_status_t __fpu_fsw;
-  uint8_t __fpu_ftw;
-  uint8_t __fpu_rsrv1;
-  uint16_t __fpu_fop;
-  uint32_t __fpu_ip;
-  uint16_t __fpu_cs;
-  uint16_t __fpu_rsrv2;
-  uint32_t __fpu_dp;
-  uint16_t __fpu_ds;
-  uint16_t __fpu_rsrv3;
-  uint32_t __fpu_mxcsr;
-  uint32_t __fpu_mxcsrmask;
-  __i386_mmst_reg __fpu_stmm0;
-  __i386_mmst_reg __fpu_stmm1;
-  __i386_mmst_reg __fpu_stmm2;
-  __i386_mmst_reg __fpu_stmm3;
-  __i386_mmst_reg __fpu_stmm4;
-  __i386_mmst_reg __fpu_stmm5;
-  __i386_mmst_reg __fpu_stmm6;
-  __i386_mmst_reg __fpu_stmm7;
-  __i386_xmm_reg __fpu_xmm0;
-  __i386_xmm_reg __fpu_xmm1;
-  __i386_xmm_reg __fpu_xmm2;
-  __i386_xmm_reg __fpu_xmm3;
-  __i386_xmm_reg __fpu_xmm4;
-  __i386_xmm_reg __fpu_xmm5;
-  __i386_xmm_reg __fpu_xmm6;
-  __i386_xmm_reg __fpu_xmm7;
-  uint8_t __fpu_rsrv4[14 * 16];
-  uint32_t __fpu_reserved1;
-  uint8_t __avx_reserved1[64];
-  __i386_xmm_reg __fpu_ymmh0;
-  __i386_xmm_reg __fpu_ymmh1;
-  __i386_xmm_reg __fpu_ymmh2;
-  __i386_xmm_reg __fpu_ymmh3;
-  __i386_xmm_reg __fpu_ymmh4;
-  __i386_xmm_reg __fpu_ymmh5;
-  __i386_xmm_reg __fpu_ymmh6;
-  __i386_xmm_reg __fpu_ymmh7;
-} __i386_avx_state_t;
-
-typedef struct { uint8_t __ymm_reg[32]; } __i386_ymm_reg;
-typedef struct { uint8_t __opmask_reg[8]; } __i386_opmask_reg;
-
-typedef struct {
-  uint32_t __fpu_reserved[2];
-  __i386_fp_control_t __fpu_fcw;
-  __i386_fp_status_t __fpu_fsw;
-  uint8_t __fpu_ftw;
-  uint8_t __fpu_rsrv1;
-  uint16_t __fpu_fop;
-  uint32_t __fpu_ip;
-  uint16_t __fpu_cs;
-  uint16_t __fpu_rsrv2;
-  uint32_t __fpu_dp;
-  uint16_t __fpu_ds;
-  uint16_t __fpu_rsrv3;
-  uint32_t __fpu_mxcsr;
-  uint32_t __fpu_mxcsrmask;
-  __i386_mmst_reg __fpu_stmm0;
-  __i386_mmst_reg __fpu_stmm1;
-  __i386_mmst_reg __fpu_stmm2;
-  __i386_mmst_reg __fpu_stmm3;
-  __i386_mmst_reg __fpu_stmm4;
-  __i386_mmst_reg __fpu_stmm5;
-  __i386_mmst_reg __fpu_stmm6;
-  __i386_mmst_reg __fpu_stmm7;
-  __i386_xmm_reg __fpu_xmm0;
-  __i386_xmm_reg __fpu_xmm1;
-  __i386_xmm_reg __fpu_xmm2;
-  __i386_xmm_reg __fpu_xmm3;
-  __i386_xmm_reg __fpu_xmm4;
-  __i386_xmm_reg __fpu_xmm5;
-  __i386_xmm_reg __fpu_xmm6;
-  __i386_xmm_reg __fpu_xmm7;
-  uint8_t __fpu_rsrv4[14 * 16];
-  uint32_t __fpu_reserved1;
-  uint8_t __avx_reserved1[64];
-  __i386_xmm_reg __fpu_ymmh0;
-  __i386_xmm_reg __fpu_ymmh1;
-  __i386_xmm_reg __fpu_ymmh2;
-  __i386_xmm_reg __fpu_ymmh3;
-  __i386_xmm_reg __fpu_ymmh4;
-  __i386_xmm_reg __fpu_ymmh5;
-  __i386_xmm_reg __fpu_ymmh6;
-  __i386_xmm_reg __fpu_ymmh7;
-  __i386_opmask_reg __fpu_k0;
-  __i386_opmask_reg __fpu_k1;
-  __i386_opmask_reg __fpu_k2;
-  __i386_opmask_reg __fpu_k3;
-  __i386_opmask_reg __fpu_k4;
-  __i386_opmask_reg __fpu_k5;
-  __i386_opmask_reg __fpu_k6;
-  __i386_opmask_reg __fpu_k7;
-  __i386_ymm_reg __fpu_zmmh0;
-  __i386_ymm_reg __fpu_zmmh1;
-  __i386_ymm_reg __fpu_zmmh2;
-  __i386_ymm_reg __fpu_zmmh3;
-  __i386_ymm_reg __fpu_zmmh4;
-  __i386_ymm_reg __fpu_zmmh5;
-  __i386_ymm_reg __fpu_zmmh6;
-  __i386_ymm_reg __fpu_zmmh7;
-} __i386_avx512f_state_t;
-
-typedef struct {
-  uint32_t __trapno;
-  uint32_t __err;
-  uint32_t __faultvaddr;
-} __i386_exception_state_t;
-
-typedef struct {
-  uint32_t __dr0;
-  uint32_t __dr1;
-  uint32_t __dr2;
-  uint32_t __dr3;
-  uint32_t __dr4;
-  uint32_t __dr5;
-  uint32_t __dr6;
-  uint32_t __dr7;
-} __i386_debug_state_t;
-
-#endif
diff --git a/lldb/tools/lldb-vscode/VSCode.cpp b/lldb/tools/lldb-vscode/VSCode.cpp
index 6cbdbfa3c24ef..ca1a626e514c9 100644
--- a/lldb/tools/lldb-vscode/VSCode.cpp
+++ b/lldb/tools/lldb-vscode/VSCode.cpp
@@ -39,8 +39,10 @@ VSCode::VSCode()
            {"swift_catch", "Swift Catch", lldb::eLanguageTypeSwift},
            {"swift_throw", "Swift Throw", lldb::eLanguageTypeSwift}}),
       focus_tid(LLDB_INVALID_THREAD_ID), sent_terminated_event(false),
-      stop_at_entry(false), is_attach(false), configuration_done_sent(false),
-      reverse_request_seq(0), waiting_for_run_in_terminal(false),
+      stop_at_entry(false), is_attach(false),
+      restarting_process_id(LLDB_INVALID_PROCESS_ID),
+      configuration_done_sent(false), reverse_request_seq(0),
+      waiting_for_run_in_terminal(false),
       progress_event_reporter(
           [&](const ProgressEvent &event) { SendJSON(event.ToJSON()); }) {
   const char *log_file_path = getenv("LLDBVSCODE_LOG");
diff --git a/lldb/tools/lldb-vscode/VSCode.h b/lldb/tools/lldb-vscode/VSCode.h
index aa325038b2e79..48bef18de2a23 100644
--- a/lldb/tools/lldb-vscode/VSCode.h
+++ b/lldb/tools/lldb-vscode/VSCode.h
@@ -15,6 +15,7 @@
 #include <cstdio>
 #include <iosfwd>
 #include <map>
+#include <optional>
 #include <set>
 #include <thread>
 
@@ -141,10 +142,17 @@ struct VSCode {
   std::vector<std::string> exit_commands;
   std::vector<std::string> stop_commands;
   std::vector<std::string> terminate_commands;
+  // A copy of the last LaunchRequest or AttachRequest so we can reuse its
+  // arguments if we get a RestartRequest.
+  std::optional<llvm::json::Object> last_launch_or_attach_request;
   lldb::tid_t focus_tid;
   bool sent_terminated_event;
   bool stop_at_entry;
   bool is_attach;
+  // The process event thread normally responds to process exited events by
+  // shutting down the entire adapter. When we're restarting, we keep the id of
+  // the old process here so we can detect this case and keep running.
+  lldb::pid_t restarting_process_id;
   bool configuration_done_sent;
   uint32_t reverse_request_seq;
   std::map<std::string, RequestCallback> request_handlers;
diff --git a/lldb/tools/lldb-vscode/lldb-vscode.cpp b/lldb/tools/lldb-vscode/lldb-vscode.cpp
index 8257a1ffbe9b7..f56a5368939e2 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode.cpp
+++ b/lldb/tools/lldb-vscode/lldb-vscode.cpp
@@ -458,7 +458,8 @@ void EventThreadFunction() {
             // manually send a stopped event in request_configurationDone(...)
             // so don't send any before then.
             if (g_vsc.configuration_done_sent) {
-              // Only report a stopped event if the process was not restarted.
+              // Only report a stopped event if the process was not
+              // automatically restarted.
               if (!lldb::SBProcess::GetRestartedFromEvent(event)) {
                 SendStdOutStdErr(process);
                 SendThreadStoppedEvent();
@@ -468,14 +469,22 @@ void EventThreadFunction() {
           case lldb::eStateRunning:
             g_vsc.WillContinue();
             break;
-          case lldb::eStateExited: {
-            // Run any exit LLDB commands the user specified in the
-            // launch.json
-            g_vsc.RunExitCommands();
-            SendProcessExitedEvent(process);
-            SendTerminatedEvent();
-            done = true;
-          } break;
+          case lldb::eStateExited:
+            // When restarting, we can get an "exited" event for the process we
+            // just killed with the old PID, or even with no PID. In that case
+            // we don't have to terminate the session.
+            if (process.GetProcessID() == LLDB_INVALID_PROCESS_ID ||
+                process.GetProcessID() == g_vsc.restarting_process_id) {
+              g_vsc.restarting_process_id = LLDB_INVALID_PROCESS_ID;
+            } else {
+              // Run any exit LLDB commands the user specified in the
+              // launch.json
+              g_vsc.RunExitCommands();
+              SendProcessExitedEvent(process);
+              SendTerminatedEvent();
+              done = true;
+            }
+            break;
           }
         } else if ((event_mask & lldb::SBProcess::eBroadcastBitSTDOUT) ||
                    (event_mask & lldb::SBProcess::eBroadcastBitSTDERR)) {
@@ -592,6 +601,7 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
 // }
 void request_attach(const llvm::json::Object &request) {
   g_vsc.is_attach = true;
+  g_vsc.last_launch_or_attach_request = request;
   llvm::json::Object response;
   lldb::SBError error;
   FillResponse(request, response);
@@ -1527,7 +1537,7 @@ void request_initialize(const llvm::json::Object &request) {
   // The debug adapter supports the RestartRequest. In this case a client
   // should not implement 'restart' by terminating and relaunching the adapter
   // but by calling the RestartRequest.
-  body.try_emplace("supportsRestartRequest", false);
+  body.try_emplace("supportsRestartRequest", true);
   // The debug adapter supports 'exceptionOptions' on the
   // setExceptionBreakpoints request.
   body.try_emplace("supportsExceptionOptions", true);
@@ -1622,6 +1632,71 @@ llvm::Error request_runInTerminal(const llvm::json::Object &launch_request) {
                                  error.GetCString());
 }
 
+// Takes a LaunchRequest object and launches the process, also handling
+// runInTerminal if applicable. It doesn't do any of the additional
+// initialization and bookkeeping stuff that is needed for `request_launch`.
+// This way we can reuse the process launching logic for RestartRequest too.
+lldb::SBError LaunchProcess(const llvm::json::Object &request) {
+  lldb::SBError error;
+  auto arguments = request.getObject("arguments");
+  auto launchCommands = GetStrings(arguments, "launchCommands");
+
+  // Instantiate a launch info instance for the target.
+  auto launch_info = g_vsc.target.GetLaunchInfo();
+
+  // Grab the current working directory if there is one and set it in the
+  // launch info.
+  const auto cwd = GetString(arguments, "cwd");
+  if (!cwd.empty())
+    launch_info.SetWorkingDirectory(cwd.data());
+
+  // Extract any extra arguments and append them to our program arguments for
+  // when we launch
+  auto args = GetStrings(arguments, "args");
+  if (!args.empty())
+    launch_info.SetArguments(MakeArgv(args).data(), true);
+
+  // Pass any environment variables along that the user specified.
+  auto envs = GetStrings(arguments, "env");
+  if (!envs.empty())
+    launch_info.SetEnvironmentEntries(MakeArgv(envs).data(), true);
+
+  auto flags = launch_info.GetLaunchFlags();
+
+  if (GetBoolean(arguments, "disableASLR", true))
+    flags |= lldb::eLaunchFlagDisableASLR;
+  if (GetBoolean(arguments, "disableSTDIO", false))
+    flags |= lldb::eLaunchFlagDisableSTDIO;
+  if (GetBoolean(arguments, "shellExpandArguments", false))
+    flags |= lldb::eLaunchFlagShellExpandArguments;
+  const bool detachOnError = GetBoolean(arguments, "detachOnError", false);
+  launch_info.SetDetachOnError(detachOnError);
+  launch_info.SetLaunchFlags(flags | lldb::eLaunchFlagDebug |
+                             lldb::eLaunchFlagStopAtEntry);
+  const uint64_t timeout_seconds = GetUnsigned(arguments, "timeout", 30);
+
+  if (GetBoolean(arguments, "runInTerminal", false)) {
+    if (llvm::Error err = request_runInTerminal(request))
+      error.SetErrorString(llvm::toString(std::move(err)).c_str());
+  } else if (launchCommands.empty()) {
+    // Disable async events so the launch will be successful when we return from
+    // the launch call and the launch will happen synchronously
+    g_vsc.debugger.SetAsync(false);
+    g_vsc.target.Launch(launch_info, error);
+    g_vsc.debugger.SetAsync(true);
+  } else {
+    g_vsc.RunLLDBCommands("Running launchCommands:", launchCommands);
+    // The custom commands might have created a new target so we should use the
+    // selected target after these commands are run.
+    g_vsc.target = g_vsc.debugger.GetSelectedTarget();
+    // Make sure the process is launched and stopped at the entry point before
+    // proceeding as the the launch commands are not run using the synchronous
+    // mode.
+    error = g_vsc.WaitForProcessToStop(timeout_seconds);
+  }
+  return error;
+}
+
 // "LaunchRequest": {
 //   "allOf": [ { "$ref": "#/definitions/Request" }, {
 //     "type": "object",
@@ -1658,8 +1733,8 @@ llvm::Error request_runInTerminal(const llvm::json::Object &launch_request) {
 // }
 void request_launch(const llvm::json::Object &request) {
   g_vsc.is_attach = false;
+  g_vsc.last_launch_or_attach_request = request;
   llvm::json::Object response;
-  lldb::SBError error;
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
   g_vsc.init_commands = GetStrings(arguments, "initCommands");
@@ -1667,12 +1742,10 @@ void request_launch(const llvm::json::Object &request) {
   g_vsc.stop_commands = GetStrings(arguments, "stopCommands");
   g_vsc.exit_commands = GetStrings(arguments, "exitCommands");
   g_vsc.terminate_commands = GetStrings(arguments, "terminateCommands");
-  auto launchCommands = GetStrings(arguments, "launchCommands");
   std::vector<std::string> postRunCommands =
       GetStrings(arguments, "postRunCommands");
   g_vsc.stop_at_entry = GetBoolean(arguments, "stopOnEntry", false);
   const llvm::StringRef debuggerRoot = GetString(arguments, "debuggerRoot");
-  const uint64_t timeout_seconds = GetUnsigned(arguments, "timeout", 30);
 
   // This is a hack for loading DWARF in .o files on Mac where the .o files
   // in the debug map of the main executable have relative paths which require
@@ -1697,76 +1770,27 @@ void request_launch(const llvm::json::Object &request) {
     return;
   }
 
-  // Instantiate a launch info instance for the target.
-  auto launch_info = g_vsc.target.GetLaunchInfo();
-
-  // Grab the current working directory if there is one and set it in the
-  // launch info.
-  const auto cwd = GetString(arguments, "cwd");
-  if (!cwd.empty())
-    launch_info.SetWorkingDirectory(cwd.data());
-
-  // Extract any extra arguments and append them to our program arguments for
-  // when we launch
-  auto args = GetStrings(arguments, "args");
-  if (!args.empty())
-    launch_info.SetArguments(MakeArgv(args).data(), true);
-
-  // Pass any environment variables along that the user specified.
-  auto envs = GetStrings(arguments, "env");
-  if (!envs.empty())
-    launch_info.SetEnvironmentEntries(MakeArgv(envs).data(), true);
-
-  auto flags = launch_info.GetLaunchFlags();
-
-  if (GetBoolean(arguments, "disableASLR", true))
-    flags |= lldb::eLaunchFlagDisableASLR;
-  if (GetBoolean(arguments, "disableSTDIO", false))
-    flags |= lldb::eLaunchFlagDisableSTDIO;
-  if (GetBoolean(arguments, "shellExpandArguments", false))
-    flags |= lldb::eLaunchFlagShellExpandArguments;
-  const bool detatchOnError = GetBoolean(arguments, "detachOnError", false);
-  launch_info.SetDetachOnError(detatchOnError);
-  launch_info.SetLaunchFlags(flags | lldb::eLaunchFlagDebug |
-                             lldb::eLaunchFlagStopAtEntry);
-
   // Run any pre run LLDB commands the user specified in the launch.json
   g_vsc.RunPreRunCommands();
 
-  if (GetBoolean(arguments, "runInTerminal", false)) {
-    if (llvm::Error err = request_runInTerminal(request))
-      error.SetErrorString(llvm::toString(std::move(err)).c_str());
-  } else if (launchCommands.empty()) {
-    // Disable async events so the launch will be successful when we return from
-    // the launch call and the launch will happen synchronously
-    g_vsc.debugger.SetAsync(false);
-    g_vsc.target.Launch(launch_info, error);
-    g_vsc.debugger.SetAsync(true);
-  } else {
-    g_vsc.RunLLDBCommands("Running launchCommands:", launchCommands);
-    // The custom commands might have created a new target so we should use the
-    // selected target after these commands are run.
-    g_vsc.target = g_vsc.debugger.GetSelectedTarget();
-    // Make sure the process is launched and stopped at the entry point before
-    // proceeding as the the launch commands are not run using the synchronous
-    // mode.
-    error = g_vsc.WaitForProcessToStop(timeout_seconds);
-  }
+  status = LaunchProcess(request);
 
-  if (error.Fail()) {
+  if (status.Fail()) {
     response["success"] = llvm::json::Value(false);
-    EmplaceSafeString(response, "message", std::string(error.GetCString()));
+    EmplaceSafeString(response, "message", std::string(status.GetCString()));
   } else {
     g_vsc.RunLLDBCommands("Running postRunCommands:", postRunCommands);
   }
 
   g_vsc.SendJSON(llvm::json::Value(std::move(response)));
 
-  if (g_vsc.is_attach)
-    SendProcessEvent(Attach); // this happens when doing runInTerminal
-  else
-    SendProcessEvent(Launch);
-  g_vsc.SendJSON(llvm::json::Value(CreateEventObject("initialized")));
+  if (!status.Fail()) {
+    if (g_vsc.is_attach)
+      SendProcessEvent(Attach);  // this happens when doing runInTerminal
+    else
+      SendProcessEvent(Launch);
+  }
+  g_vsc.SendJSON(CreateEventObject("initialized"));
 }
 
 // "NextRequest": {
@@ -1867,6 +1891,118 @@ void request_pause(const llvm::json::Object &request) {
   g_vsc.SendJSON(llvm::json::Value(std::move(response)));
 }
 
+
+// "RestartRequest": {
+//   "allOf": [ { "$ref": "#/definitions/Request" }, {
+//     "type": "object",
+//     "description": "Restarts a debug session. Clients should only call this
+//     request if the corresponding capability `supportsRestartRequest` is
+//     true.\nIf the capability is missing or has the value false, a typical
+//     client emulates `restart` by terminating the debug adapter first and then
+//     launching it anew.",
+//     "properties": {
+//       "command": {
+//         "type": "string",
+//         "enum": [ "restart" ]
+//       },
+//       "arguments": {
+//         "$ref": "#/definitions/RestartArguments"
+//       }
+//     },
+//     "required": [ "command" ]
+//   }]
+// },
+// "RestartArguments": {
+//   "type": "object",
+//   "description": "Arguments for `restart` request.",
+//   "properties": {
+//     "arguments": {
+//       "oneOf": [
+//         { "$ref": "#/definitions/LaunchRequestArguments" },
+//         { "$ref": "#/definitions/AttachRequestArguments" }
+//       ],
+//       "description": "The latest version of the `launch` or `attach`
+//       configuration."
+//     }
+//   }
+// },
+// "RestartResponse": {
+//   "allOf": [ { "$ref": "#/definitions/Response" }, {
+//     "type": "object",
+//     "description": "Response to `restart` request. This is just an
+//     acknowledgement, so no body field is required."
+//   }]
+// },
+void request_restart(const llvm::json::Object &request) {
+  llvm::json::Object response;
+  FillResponse(request, response);
+  if (!g_vsc.last_launch_or_attach_request) {
+    response["success"] = llvm::json::Value(false);
+    EmplaceSafeString(response, "message",
+                      "Restart request received but no process was launched.");
+    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+  // Check if we were in a "launch" session or an "attach" session.
+  //
+  // Restarting is not well defined when we started the session by attaching to
+  // an existing process, because we don't know how the process was started, so
+  // we don't support it.
+  //
+  // Note that when using runInTerminal we're technically attached, but it's an
+  // implementation detail. The adapter *did* launch the process in response to
+  // a "launch" command, so we can still stop it and re-run it. This is why we
+  // don't just check `g_vsc.is_attach`.
+  if (GetString(*g_vsc.last_launch_or_attach_request, "command") == "attach") {
+    response["success"] = llvm::json::Value(false);
+    EmplaceSafeString(response, "message",
+                      "Restarting an \"attach\" session is not supported.");
+    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    return;
+  }
+
+  // The optional `arguments` field in RestartRequest can contain an updated
+  // version of the launch arguments. If there's one, use it.
+  auto request_arguments = request.getObject("arguments");
+  if (request_arguments) {
+    llvm::json::Object arguments = *request_arguments;
+    (*g_vsc.last_launch_or_attach_request)["arguments"] =
+        llvm::json::Value(std::move(arguments));
+  }
+
+  // Keep track of the old PID so when we get a "process exited" event from the
+  // killed process we can detect it and not shut down the whole session.
+  lldb::SBProcess process = g_vsc.target.GetProcess();
+  g_vsc.restarting_process_id = process.GetProcessID();
+
+  // Stop the current process if necessary. The logic here is similar to
+  // CommandObjectProcessLaunchOrAttach::StopProcessIfNecessary, except that
+  // we don't ask the user for confirmation.
+  g_vsc.debugger.SetAsync(false);
+  if (process.IsValid()) {
+    lldb::StateType state = process.GetState();
+    if (state != lldb::eStateConnected) {
+      process.Kill();
+    }
+    // Clear the list of thread ids to avoid sending "thread exited" events
+    // for threads of the process we are terminating.
+    g_vsc.thread_ids.clear();
+  }
+  g_vsc.debugger.SetAsync(true);
+  LaunchProcess(*g_vsc.last_launch_or_attach_request);
+
+  // This is normally done after receiving a "configuration done" request.
+  // Because we're restarting, configuration has already happened so we can
+  // continue the process right away.
+  if (g_vsc.stop_at_entry) {
+    SendThreadStoppedEvent();
+  } else {
+    g_vsc.target.GetProcess().Continue();
+  }
+
+  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+}
+
 // "ScopesRequest": {
 //   "allOf": [ { "$ref": "#/definitions/Request" }, {
 //     "type": "object",
@@ -3084,6 +3220,7 @@ void RegisterRequestCallbacks() {
   g_vsc.RegisterRequestCallback("launch", request_launch);
   g_vsc.RegisterRequestCallback("next", request_next);
   g_vsc.RegisterRequestCallback("pause", request_pause);
+  g_vsc.RegisterRequestCallback("restart", request_restart);
   g_vsc.RegisterRequestCallback("scopes", request_scopes);
   g_vsc.RegisterRequestCallback("setBreakpoints", request_setBreakpoints);
   g_vsc.RegisterRequestCallback("setExceptionBreakpoints",
diff --git a/lldb/unittests/Core/FileSpecListTest.cpp b/lldb/unittests/Core/FileSpecListTest.cpp
index d1a8eadde3bfa..d65e7cd2d0586 100644
--- a/lldb/unittests/Core/FileSpecListTest.cpp
+++ b/lldb/unittests/Core/FileSpecListTest.cpp
@@ -8,7 +8,7 @@
 
 #include "gtest/gtest.h"
 
-#include "lldb/Core/FileSpecList.h"
+#include "lldb/Utility/FileSpecList.h"
 
 using namespace lldb_private;
 
diff --git a/lldb/unittests/Interpreter/TestOptionValue.cpp b/lldb/unittests/Interpreter/TestOptionValue.cpp
index 9bdbb22240eae..64adcff243fd9 100644
--- a/lldb/unittests/Interpreter/TestOptionValue.cpp
+++ b/lldb/unittests/Interpreter/TestOptionValue.cpp
@@ -16,6 +16,7 @@ class Callback {
 public:
   virtual void Invoke() const {}
   void operator()() const { Invoke(); }
+
 protected:
   ~Callback() = default;
 };
@@ -101,12 +102,11 @@ class TestProperties : public OptionValueProperties {
   }
 
   OptionValueDictionary *GetDictionary() {
-    return GetPropertyAtIndexAsOptionValueDictionary(nullptr, m_dict_index);
+    return GetPropertyAtIndexAsOptionValueDictionary(m_dict_index);
   }
 
   OptionValueFileSpecList *GetFileList() {
-    return GetPropertyAtIndexAsOptionValueFileSpecList(nullptr, true,
-                                                       m_file_list_index);
+    return GetPropertyAtIndexAsOptionValueFileSpecList(m_file_list_index);
   }
 
 private:
@@ -146,12 +146,12 @@ TEST(TestProperties, DeepCopy) {
   ASSERT_TRUE(dict_copy_ptr->OptionWasSet());
   ASSERT_EQ(dict_copy_ptr->GetNumValues(), 2U);
 
-  auto value_ptr = dict_copy_ptr->GetValueForKey(ConstString("A"));
+  auto value_ptr = dict_copy_ptr->GetValueForKey("A");
   ASSERT_TRUE(value_ptr);
   ASSERT_EQ(value_ptr->GetParent().get(), dict_copy_ptr);
   ASSERT_EQ(value_ptr->GetUInt64Value(), 1U);
 
-  value_ptr = dict_copy_ptr->GetValueForKey(ConstString("B"));
+  value_ptr = dict_copy_ptr->GetValueForKey("B");
   ASSERT_TRUE(value_ptr);
   ASSERT_EQ(value_ptr->GetParent().get(), dict_copy_ptr);
   ASSERT_EQ(value_ptr->GetUInt64Value(), 2U);
@@ -170,5 +170,6 @@ TEST(TestProperties, DeepCopy) {
   dict_copy_ptr->SetValueFromString("C=3", eVarSetOperationAppend);
 
   // Trigger the callback second time.
-  file_list_copy_ptr->SetValueFromString("0 another/path", eVarSetOperationReplace);
+  file_list_copy_ptr->SetValueFromString("0 another/path",
+                                         eVarSetOperationReplace);
 }
diff --git a/lldb/unittests/Language/CMakeLists.txt b/lldb/unittests/Language/CMakeLists.txt
index 3cca831956ab6..6f173568c76e6 100644
--- a/lldb/unittests/Language/CMakeLists.txt
+++ b/lldb/unittests/Language/CMakeLists.txt
@@ -1,3 +1,4 @@
 add_subdirectory(CPlusPlus)
 add_subdirectory(CLanguages)
 add_subdirectory(Highlighting)
+add_subdirectory(ObjC)
diff --git a/lldb/unittests/Language/ObjC/CMakeLists.txt b/lldb/unittests/Language/ObjC/CMakeLists.txt
new file mode 100644
index 0000000000000..82cc847e1045f
--- /dev/null
+++ b/lldb/unittests/Language/ObjC/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_lldb_unittest(LanguageObjCTests
+  ObjCLanguageTest.cpp
+
+  LINK_LIBS
+    lldbPluginObjCLanguage
+)
diff --git a/lldb/unittests/Language/ObjC/ObjCLanguageTest.cpp b/lldb/unittests/Language/ObjC/ObjCLanguageTest.cpp
new file mode 100644
index 0000000000000..e9cb5d1b32143
--- /dev/null
+++ b/lldb/unittests/Language/ObjC/ObjCLanguageTest.cpp
@@ -0,0 +1,114 @@
+//===-- ObjCLanguageTest.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "Plugins/Language/ObjC/ObjCLanguage.h"
+#include "lldb/lldb-enumerations.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <optional>
+
+#include "llvm/ADT/StringRef.h"
+
+using namespace lldb_private;
+
+TEST(ObjCLanguage, MethodNameParsing) {
+  struct TestCase {
+    llvm::StringRef input;
+    llvm::StringRef full_name_sans_category;
+    llvm::StringRef class_name;
+    llvm::StringRef class_name_with_category;
+    llvm::StringRef category;
+    llvm::StringRef selector;
+  };
+
+  TestCase strict_cases[] = {
+      {"-[MyClass mySelector:]", "", "MyClass", "MyClass", "", "mySelector:"},
+      {"+[MyClass mySelector:]", "", "MyClass", "MyClass", "", "mySelector:"},
+      {"-[MyClass(my_category) mySelector:]", "-[MyClass mySelector:]",
+       "MyClass", "MyClass(my_category)", "my_category", "mySelector:"},
+      {"+[MyClass(my_category) mySelector:]", "+[MyClass mySelector:]",
+       "MyClass", "MyClass(my_category)", "my_category", "mySelector:"},
+  };
+
+  TestCase lax_cases[] = {
+      {"[MyClass mySelector:]", "", "MyClass", "MyClass", "", "mySelector:"},
+      {"[MyClass(my_category) mySelector:]", "[MyClass mySelector:]", "MyClass",
+       "MyClass(my_category)", "my_category", "mySelector:"},
+  };
+
+  // First, be strict
+  for (const auto &test : strict_cases) {
+    ObjCLanguage::MethodName method(test.input, /*strict = */ true);
+    EXPECT_TRUE(method.IsValid(/*strict = */ true));
+    EXPECT_EQ(
+        test.full_name_sans_category,
+        method.GetFullNameWithoutCategory(/*empty_if_no_category = */ true)
+            .GetStringRef());
+    EXPECT_EQ(test.class_name, method.GetClassName().GetStringRef());
+    EXPECT_EQ(test.class_name_with_category,
+              method.GetClassNameWithCategory().GetStringRef());
+    EXPECT_EQ(test.category, method.GetCategory().GetStringRef());
+    EXPECT_EQ(test.selector, method.GetSelector().GetStringRef());
+  }
+
+  // We should make sure strict parsing does not accept lax cases
+  for (const auto &test : lax_cases) {
+    ObjCLanguage::MethodName method(test.input, /*strict = */ true);
+    EXPECT_FALSE(method.IsValid(/*strict = */ true));
+  }
+
+  // All strict cases should work when not lax
+  for (const auto &test : strict_cases) {
+    ObjCLanguage::MethodName method(test.input, /*strict = */ false);
+    EXPECT_TRUE(method.IsValid(/*strict = */ false));
+    EXPECT_EQ(
+        test.full_name_sans_category,
+        method.GetFullNameWithoutCategory(/*empty_if_no_category = */ true)
+            .GetStringRef());
+    EXPECT_EQ(test.class_name, method.GetClassName().GetStringRef());
+    EXPECT_EQ(test.class_name_with_category,
+              method.GetClassNameWithCategory().GetStringRef());
+    EXPECT_EQ(test.category, method.GetCategory().GetStringRef());
+    EXPECT_EQ(test.selector, method.GetSelector().GetStringRef());
+  }
+
+  // Make sure non-strict parsing works
+  for (const auto &test : lax_cases) {
+    ObjCLanguage::MethodName method(test.input, /*strict = */ false);
+    EXPECT_TRUE(method.IsValid(/*strict = */ false));
+    EXPECT_EQ(
+        test.full_name_sans_category,
+        method.GetFullNameWithoutCategory(/*empty_if_no_category = */ true)
+            .GetStringRef());
+    EXPECT_EQ(test.class_name, method.GetClassName().GetStringRef());
+    EXPECT_EQ(test.class_name_with_category,
+              method.GetClassNameWithCategory().GetStringRef());
+    EXPECT_EQ(test.category, method.GetCategory().GetStringRef());
+    EXPECT_EQ(test.selector, method.GetSelector().GetStringRef());
+  }
+}
+
+TEST(CPlusPlusLanguage, InvalidMethodNameParsing) {
+  // Tests that we correctly reject malformed function names
+
+  llvm::StringRef test_cases[] = {"+[Uh oh!",
+                                  "-[Definitely not...",
+                                  "[Nice try ] :)",
+                                  "+MaybeIfYouSquintYourEyes]",
+                                  "?[Tricky]",
+                                  "+[]",
+                                  "-[]",
+                                  "[]"};
+
+  for (const auto &name : test_cases) {
+    ObjCLanguage::MethodName strict_method(name, /*strict = */ true);
+    EXPECT_FALSE(strict_method.IsValid(true));
+
+    ObjCLanguage::MethodName lax_method(name, /*strict = */ false);
+    EXPECT_FALSE(lax_method.IsValid(true));
+  }
+}
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
index 59ba0b7b6aeab..2883b39639e1a 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
@@ -107,6 +107,14 @@ bool lldb_private::LLDBSWIGPythonCallThreadPlan(void *implementor,
   return false;
 }
 
+bool 
+lldb_private::LLDBSWIGPythonCallThreadPlan(void *implementor,
+                                                const char *method_name,
+                                                Stream *event_sp,
+                                                bool &got_error) {
+  return false;
+}
+
 python::PythonObject
 lldb_private::LLDBSwigPythonCreateScriptedBreakpointResolver(
     const char *python_class_name, const char *session_dictionary_name,
@@ -139,6 +147,10 @@ void *lldb_private::LLDBSWIGPython_CastPyObjectToSBData(PyObject *data) {
   return nullptr;
 }
 
+void *lldb_private::LLDBSWIGPython_CastPyObjectToSBBreakpoint(PyObject *data) {
+  return nullptr;
+}
+
 void *lldb_private::LLDBSWIGPython_CastPyObjectToSBAttachInfo(PyObject *data) {
   return nullptr;
 }
diff --git a/lldb/unittests/Utility/FileSpecTest.cpp b/lldb/unittests/Utility/FileSpecTest.cpp
index 96f19953c3b74..ffd3d343dc188 100644
--- a/lldb/unittests/Utility/FileSpecTest.cpp
+++ b/lldb/unittests/Utility/FileSpecTest.cpp
@@ -456,7 +456,7 @@ TEST(FileSpecTest, TestFileNameExtensions) {
   FileSpec just_dot = PosixSpec("/tmp/bar.");
 
   EXPECT_TRUE(dylib.GetFileNameExtension() == ".dylib");
-  EXPECT_TRUE(exe.GetFileNameExtension() == ConstString(nullptr));
+  EXPECT_TRUE(exe.GetFileNameExtension() == llvm::StringRef());
   EXPECT_TRUE(dSYM.GetFileNameExtension() == ".dSYM");
   EXPECT_TRUE(just_dot.GetFileNameExtension() == ".");
 
@@ -464,7 +464,7 @@ TEST(FileSpecTest, TestFileNameExtensions) {
   FileSpec win_noext = WindowsSpec("C:\\tmp\\foo");
 
   EXPECT_TRUE(dll.GetFileNameExtension() == ".dll");
-  EXPECT_TRUE(win_noext.GetFileNameExtension() == ConstString(nullptr));
+  EXPECT_TRUE(win_noext.GetFileNameExtension() == llvm::StringRef());
 }
 
 TEST(FileSpecTest, TestFileNameStrippingExtension) {
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index bd23d6657829a..ce6cd1a52d3a1 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -1163,7 +1163,7 @@ add_subdirectory(lib/TableGen)
 
 add_subdirectory(utils/TableGen)
 
-add_subdirectory(include/llvm)
+add_subdirectory(include)
 
 add_subdirectory(lib)
 
@@ -1275,14 +1275,14 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
     )
 
   if (LLVM_INSTALL_MODULEMAPS)
-    install(DIRECTORY include/llvm include/llvm-c
+    install(DIRECTORY include
             DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
             COMPONENT llvm-headers
             FILES_MATCHING
             PATTERN "module.modulemap"
             )
-    install(FILES include/llvm/module.install.modulemap
-            DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/llvm"
+    install(FILES include/module.install.modulemap
+            DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
             COMPONENT llvm-headers
             RENAME "module.extern.modulemap"
             )
diff --git a/llvm/cmake/modules/CrossCompile.cmake b/llvm/cmake/modules/CrossCompile.cmake
index a46a0fd630b0f..7c750f01e0cab 100644
--- a/llvm/cmake/modules/CrossCompile.cmake
+++ b/llvm/cmake/modules/CrossCompile.cmake
@@ -112,11 +112,17 @@ function(build_native_tool target output_path_var)
   endif()
   set(output_path ${output_path}${LLVM_HOST_EXECUTABLE_SUFFIX})
 
+  # Make chain of preceding actions
+  if(CMAKE_GENERATOR MATCHES "Visual Studio")
+    get_property(host_targets GLOBAL PROPERTY ${PROJECT_NAME}_HOST_TARGETS)
+    set_property(GLOBAL APPEND PROPERTY ${PROJECT_NAME}_HOST_TARGETS ${output_path})
+  endif()
+
   llvm_ExternalProject_BuildCmd(build_cmd ${target} ${${PROJECT_NAME}_NATIVE_BUILD}
                                 CONFIGURATION Release)
   add_custom_command(OUTPUT "${output_path}"
                      COMMAND ${build_cmd}
-                     DEPENDS CONFIGURE_${PROJECT_NAME}_NATIVE ${ARG_DEPENDS}
+                     DEPENDS CONFIGURE_${PROJECT_NAME}_NATIVE ${ARG_DEPENDS} ${host_targets}
                      WORKING_DIRECTORY "${${PROJECT_NAME}_NATIVE_BUILD}"
                      COMMENT "Building native ${target}..."
                      USES_TERMINAL)
diff --git a/cmake/Modules/FindLibEdit.cmake b/llvm/cmake/modules/FindLibEdit.cmake
similarity index 100%
rename from cmake/Modules/FindLibEdit.cmake
rename to llvm/cmake/modules/FindLibEdit.cmake
diff --git a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
index 73910b4b083a1..44af6c1462bde 100644
--- a/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
+++ b/llvm/cmake/modules/LLVMExternalProjectUtils.cmake
@@ -165,6 +165,19 @@ function(llvm_ExternalProject_Add name source_dir)
     endforeach()
   endforeach()
 
+  # Populate the non-project-specific passthrough variables
+  foreach(variableName ${LLVM_EXTERNAL_PROJECT_PASSTHROUGH})
+    if(DEFINED ${variableName})
+      if("${${variableName}}" STREQUAL "")
+        set(value "")
+      else()
+        string(REPLACE ";" "|" value "${${variableName}}")
+      endif()
+      list(APPEND PASSTHROUGH_VARIABLES
+        -D${variableName}=${value})
+    endif()
+  endforeach()
+
   if(ARG_USE_TOOLCHAIN AND NOT CMAKE_CROSSCOMPILING)
     if(CLANG_IN_TOOLCHAIN)
       if(is_msvc_target)
@@ -327,8 +340,6 @@ function(llvm_ExternalProject_Add name source_dir)
                -DPACKAGE_VERSION=${PACKAGE_VERSION}
                -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
-               -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
-               -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
                -DCMAKE_EXPORT_COMPILE_COMMANDS=1
                ${cmake_args}
                ${PASSTHROUGH_VARIABLES}
diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index 70c2edb8c833f..7fd6628ef55d3 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -6,6 +6,12 @@ include(LLVMDistributionSupport)
 
 function(tablegen project ofn)
   cmake_parse_arguments(ARG "" "" "DEPENDS;EXTRA_INCLUDES" ${ARGN})
+
+  # Override ${project} with ${project}_TABLEGEN_PROJECT
+  if(NOT "${${project}_TABLEGEN_PROJECT}" STREQUAL "")
+    set(project ${${project}_TABLEGEN_PROJECT})
+  endif()
+
   # Validate calling context.
   if(NOT ${project}_TABLEGEN_EXE)
     message(FATAL_ERROR "${project}_TABLEGEN_EXE not set")
@@ -137,12 +143,6 @@ macro(add_tablegen target project)
   set(${target}_OLD_LLVM_LINK_COMPONENTS ${LLVM_LINK_COMPONENTS})
   set(LLVM_LINK_COMPONENTS ${LLVM_LINK_COMPONENTS} TableGen)
 
-  # CMake doesn't let compilation units depend on their dependent libraries on some generators.
-  if(NOT CMAKE_GENERATOR MATCHES "Ninja" AND NOT XCODE)
-    # FIXME: It leaks to user, callee of add_tablegen.
-    set(LLVM_ENABLE_OBJLIB ON)
-  endif()
-
   add_llvm_executable(${target} DISABLE_LLVM_LINK_LLVM_DYLIB
     ${ADD_TABLEGEN_UNPARSED_ARGUMENTS})
   set(LLVM_LINK_COMPONENTS ${${target}_OLD_LLVM_LINK_COMPONENTS})
@@ -153,8 +153,22 @@ macro(add_tablegen target project)
       set(${project}_TABLEGEN_DEFAULT "${LLVM_NATIVE_TOOL_DIR}/${target}${LLVM_HOST_EXECUTABLE_SUFFIX}")
     endif()
   endif()
-  set(${project}_TABLEGEN "${${project}_TABLEGEN_DEFAULT}" CACHE
+
+  # FIXME: Quick fix to reflect LLVM_TABLEGEN to llvm-min-tblgen
+  if("${target}" STREQUAL "llvm-min-tblgen"
+      AND NOT "${LLVM_TABLEGEN}" STREQUAL ""
+      AND NOT "${LLVM_TABLEGEN}" STREQUAL "llvm-tblgen")
+    set(${project}_TABLEGEN_DEFAULT "${LLVM_TABLEGEN}")
+  endif()
+
+  if(ADD_TABLEGEN_EXPORT)
+    set(${project}_TABLEGEN "${${project}_TABLEGEN_DEFAULT}" CACHE
       STRING "Native TableGen executable. Saves building one when cross-compiling.")
+  else()
+    # Internal tablegen
+    set(${project}_TABLEGEN "${${project}_TABLEGEN_DEFAULT}")
+    set_target_properties(${target} PROPERTIES EXCLUDE_FROM_ALL ON)
+  endif()
 
   # Effective tblgen executable to be used:
   set(${project}_TABLEGEN_EXE ${${project}_TABLEGEN} PARENT_SCOPE)
@@ -168,16 +182,8 @@ macro(add_tablegen target project)
       build_native_tool(${target} ${project}_TABLEGEN_EXE DEPENDS ${target})
       set(${project}_TABLEGEN_EXE ${${project}_TABLEGEN_EXE} PARENT_SCOPE)
 
-      add_custom_target(${project}-tablegen-host DEPENDS ${${project}_TABLEGEN_EXE})
-      set(${project}_TABLEGEN_TARGET ${project}-tablegen-host PARENT_SCOPE)
-
-      # Create an artificial dependency between tablegen projects, because they
-      # compile the same dependencies, thus using the same build folders.
-      # FIXME: A proper fix requires sequentially chaining tablegens.
-      if (NOT ${project} STREQUAL LLVM AND TARGET ${project}-tablegen-host AND
-          TARGET LLVM-tablegen-host)
-        add_dependencies(${project}-tablegen-host LLVM-tablegen-host)
-      endif()
+      add_custom_target(${target}-host DEPENDS ${${project}_TABLEGEN_EXE})
+      set(${project}_TABLEGEN_TARGET ${target}-host PARENT_SCOPE)
 
       # If we're using the host tablegen, and utils were not requested, we have no
       # need to build this tablegen.
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index cb774a0f10db9..5755a60859440 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -675,6 +675,7 @@ supported for the ``amdgcn`` target.
      Private                           5               private     scratch          32      0xFFFFFFFF
      Constant 32-bit                   6               *TODO*                               0x00000000
      Buffer Fat Pointer (experimental) 7               *TODO*
+     Buffer Resource (experimental)    8               *TODO*
      Streamout Registers               128             N/A         GS_REGS
      ================================= =============== =========== ================ ======= ============================
 
@@ -784,6 +785,26 @@ supported for the ``amdgcn`` target.
   model the buffer descriptors used heavily in graphics workloads targeting
   the backend.
 
+  The buffer descriptor used to construct a buffer fat pointer must be *raw*:
+  the stride must be 0, the "add tid" flag bust be 0, the swizzle enable bits
+  must be off, and the extent must be measured in bytes. (On subtargets where
+  bounds checking may be disabled, buffer fat pointers may choose to enable
+  it or not).
+
+**Buffer Resource**
+  The buffer resource is an experimental address space that is currently unsupported
+  in the backend. It exposes a non-integral pointer that will represent a 128-bit
+  buffer descriptor resource.
+
+  Since, in general, a buffer resource supports complex addressing modes that cannot
+  be easily represented in LLVM (such as implicit swizzled access to structured
+  buffers), it is **illegal** to perform non-trivial address computations, such as
+  ``getelementptr`` operations, on buffer resources. They may be passed to
+  AMDGPU buffer intrinsics, and they may be converted to and from ``i128``.
+
+  Casting a buffer resource to a bufer fat pointer is permitted and adds an offset
+  of 0.
+
 **Streamout Registers**
   Dedicated registers used by the GS NGG Streamout Instructions. The register
   file is modelled as a memory in a distinct address space because it is indexed
diff --git a/llvm/docs/AdvancedBuilds.rst b/llvm/docs/AdvancedBuilds.rst
index ba3f3c3e498f4..a172e5da63c44 100644
--- a/llvm/docs/AdvancedBuilds.rst
+++ b/llvm/docs/AdvancedBuilds.rst
@@ -41,7 +41,10 @@ CLANG_ENABLE_BOOTSTRAP.
 
 .. code-block:: console
 
-  $ cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCLANG_ENABLE_BOOTSTRAP=On <path to source>
+  $ cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
+      -DCLANG_ENABLE_BOOTSTRAP=On \
+      -DLLVM_ENABLE_PROJECTS="clang" \
+      <path to source>/llvm
   $ ninja stage2
 
 This command itself isn't terribly useful because it assumes default
@@ -55,7 +58,11 @@ CMake option, each variable separated by a ";". As example:
 
 .. code-block:: console
 
-  $ cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCLANG_ENABLE_BOOTSTRAP=On -DCLANG_BOOTSTRAP_PASSTHROUGH="CMAKE_INSTALL_PREFIX;CMAKE_VERBOSE_MAKEFILE" <path to source>
+  $ cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
+      -DCLANG_ENABLE_BOOTSTRAP=On \
+      -DCLANG_BOOTSTRAP_PASSTHROUGH="CMAKE_INSTALL_PREFIX;CMAKE_VERBOSE_MAKEFILE" \
+      -DLLVM_ENABLE_PROJECTS="clang" \
+      <path to source>/llvm
   $ ninja stage2
 
 CMake options starting by ``BOOTSTRAP_`` will be passed only to the stage2 build.
@@ -88,7 +95,7 @@ You can build an Apple Clang compiler using the following commands:
 
 .. code-block:: console
 
-  $ cmake -G Ninja -C <path to source>/clang/cmake/caches/Apple-stage1.cmake <path to source>
+  $ cmake -G Ninja -C <path to source>/clang/cmake/caches/Apple-stage1.cmake <path to source>/llvm
   $ ninja stage2-distribution
 
 This CMake invocation configures the stage1 host compiler, and sets
@@ -268,7 +275,7 @@ following commands:
 
 .. code-block:: console
 
-  $ cmake -G Ninja -C <path to source>/clang/cmake/caches/3-stage.cmake <path to source>
+  $ cmake -G Ninja -C <path to source>/clang/cmake/caches/3-stage.cmake <path to source>/llvm
   $ cmake --build . --target stage3 --parallel
 
 After the build you can compare the stage2 & stage3 compilers.
diff --git a/llvm/docs/CommandGuide/llvm-mca.rst b/llvm/docs/CommandGuide/llvm-mca.rst
index 04248f68ab420..324e1e6e7bd11 100644
--- a/llvm/docs/CommandGuide/llvm-mca.rst
+++ b/llvm/docs/CommandGuide/llvm-mca.rst
@@ -32,13 +32,13 @@ directly into :program:`llvm-mca` for analysis:
 
 .. code-block:: bash
 
-  $ clang foo.c -O2 -target x86_64-unknown-unknown -S -o - | llvm-mca -mcpu=btver2
+  $ clang foo.c -O2 --target=x86_64 -S -o - | llvm-mca -mcpu=btver2
 
 Or for Intel syntax:
 
 .. code-block:: bash
 
-  $ clang foo.c -O2 -target x86_64-unknown-unknown -mllvm -x86-asm-syntax=intel -S -o - | llvm-mca -mcpu=btver2
+  $ clang foo.c -O2 --target=x86_64 -masm=intel -S -o - | llvm-mca -mcpu=btver2
 
 (:program:`llvm-mca` detects Intel syntax by the presence of an `.intel_syntax`
 directive at the beginning of the input.  By default its output syntax matches
diff --git a/llvm/docs/CommandGuide/tblgen.rst b/llvm/docs/CommandGuide/tblgen.rst
index a7c254901392a..fba4e4380db11 100644
--- a/llvm/docs/CommandGuide/tblgen.rst
+++ b/llvm/docs/CommandGuide/tblgen.rst
@@ -348,6 +348,14 @@ clang-tblgen Options
 
   Generate ``riscv_vector_builtin_cg.inc`` for Clang.
 
+.. option:: -gen-riscv-sifive-vector-builtins
+
+  Generate ``riscv_sifive_vector_builtins.inc`` for Clang.
+
+.. option:: -gen-riscv-sifive-vector-builtin-codegen
+
+  Generate ``riscv_sifive_vector_builtin_cg.inc`` for Clang.
+
 .. option:: -gen-attr-docs
 
   Generate attribute documentation.
diff --git a/llvm/docs/GitHub.rst b/llvm/docs/GitHub.rst
index 06d574be946f5..407a612e544b7 100644
--- a/llvm/docs/GitHub.rst
+++ b/llvm/docs/GitHub.rst
@@ -12,6 +12,14 @@ The LLVM Project uses `GitHub <https://github.com/>`_ for
 This page describes how the LLVM Project users and developers can
 participate in the project using GitHub.
 
+Branches
+========
+Do not create any branches in the llvm/llvm-project repository.  This repository
+is reserved for official project branches only.  We may relax this rule in
+the future if needed to support "stacked" pull request, but in that case only
+branches being used for "stacked" pull requests will be allowed.
+
+
 Releases
 ========
 
diff --git a/llvm/docs/HowToAddABuilder.rst b/llvm/docs/HowToAddABuilder.rst
index e0ac34769fd07..fe3863a3d9418 100644
--- a/llvm/docs/HowToAddABuilder.rst
+++ b/llvm/docs/HowToAddABuilder.rst
@@ -171,7 +171,7 @@ Here are the steps you can follow to do so:
    keep an unstable builder connected to staging indefinitely.
 
 #. (Optional) Once the builder is stable on the staging buildmaster with
-   several days of green history, you can chose to move it to the production
+   several days of green history, you can choose to move it to the production
    buildmaster to enable developer notifications.  Please email `Galina
    Kistanova <mailto:gkistanova@gmail.com>`_ for review and approval.
 
diff --git a/llvm/docs/HowToReleaseLLVM.rst b/llvm/docs/HowToReleaseLLVM.rst
index 0931aa677a359..15397c593d497 100644
--- a/llvm/docs/HowToReleaseLLVM.rst
+++ b/llvm/docs/HowToReleaseLLVM.rst
@@ -115,7 +115,7 @@ Branch the Git trunk using the following procedure:
 
 ::
 
-  $ git tag -a llvmorg-N-init
+  $ git tag -sa llvmorg-N-init
 
 #. Clear the release notes in trunk.
 
@@ -130,10 +130,7 @@ Update LLVM Version
 ^^^^^^^^^^^^^^^^^^^
 
 After creating the LLVM release branch, update the release branches'
-``CMakeLists.txt`` versions from '``X.0.0git``' to '``X.0.0``'.
-
-In addition, the version numbers of all the Bugzilla components must be updated
-for the next release.
+version with the script in ``llvm/utils/release/bump-version.py``.
 
 Tagging the LLVM Release Candidates
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -142,7 +139,7 @@ Tag release candidates:
 
 ::
 
-  $ git tag -a llvmorg-X.Y.Z-rcN
+  $ git tag -sa llvmorg-X.Y.Z-rcN
 
 The Release Manager must supply pre-packaged source tarballs for users.  This can
 be done with the export.sh script in utils/release.
@@ -206,12 +203,12 @@ release is out, all bugs reported will have to go on the next stable release.
 
 The official release managers are:
 
-* Major releases (X.0): Hans Wennborg
-* Stable releases (X.n): Tom Stellard
+* Even releases: Tom Stellard (tstellar@redhat.com)
+* Odd releases: Tobias Hieta (tobias@hieta.se)
 
 The official release testers are volunteered from the community and have
 consistently validated and released binaries for their targets/OSs. To contact
-them, you should post on the `Discourse forums (Project 
+them, you should post on the `Discourse forums (Project
 Infrastructure - Release Testers). <https://discourse.llvm.org/c/infrastructure/release-testers/66>`_
 
 The official testers list is in the file ``RELEASE_TESTERS.TXT``, in the ``LLVM``
@@ -247,7 +244,7 @@ We ask that all LLVM developers test the release in any the following ways:
    architecture.
 
 We also ask that the OS distribution release managers test their packages with
-the first candidate of every release, and report any *new* errors in Bugzilla.
+the first candidate of every release, and report any *new* errors in GitHub.
 If the bug can be reproduced with an unpatched upstream version of the release
 candidate (as opposed to the distribution's own build), the priority should be
 release blocker.
@@ -265,16 +262,10 @@ Reporting Regressions
 ---------------------
 
 Every regression that is found during the tests (as per the criteria above),
-should be filled in a bug in Bugzilla with the priority *release blocker* and
-blocking a specific release.
-
-To help manage all the bugs reported and which ones are blockers or not, a new
-"[meta]" bug should be created and all regressions *blocking* that Meta. Once
-all blockers are done, the Meta can be closed.
+should be filled in a bug in GitHub and added to the release milestone.
 
 If a bug can't be reproduced, or stops being a blocker, it should be removed
-from the Meta and its priority decreased to *normal*. Debugging can continue,
-but on trunk.
+from the Milestone. Debugging can continue, but on trunk.
 
 Backport Requests
 -----------------
@@ -317,13 +308,8 @@ This section describes how to triage bug reports:
    issue.  If all the tests pass, then the pull request can be merged.  If not,
    then add a comment on the issue asking someone to take a look at the failures.
 
-#. Once the pull request has been merged push it to the official release branch:
-
-   ::
-
-      git checkout release/XX.x
-      git pull --ff-only https://github.com/llvm/llvm-project-release-prs release/XX.x
-      git push https://github.com/llvm/llvm-project release/XX.x:release/XX.x
+#. Once the pull request has been merged push it to the official release branch
+   with the script ``llvm/utils/git/sync-release-repo.sh``.
 
    Then add a comment to the issue stating that the fix has been merged along with
    the git hashes from the release branch.  Add the release:merged label to the issue
@@ -360,15 +346,6 @@ Below are the rules regarding patching the release branch:
    ABI compatibility with the previous major release.
 
 
-Merging Patches
-^^^^^^^^^^^^^^^
-
-Use the ``git cherry-pick -x`` command to merge patches to the release branch:
-
-#. ``git cherry-pick -x abcdef0``
-
-#. Run regression tests.
-
 Release Final Tasks
 -------------------
 
@@ -395,7 +372,7 @@ Tag the final release sources:
 
 ::
 
-  $ git tag -a llvmorg-X.Y.Z
+  $ git tag -sa llvmorg-X.Y.Z
   $ git push https://github.com/llvm/llvm-project.git llvmorg-X.Y.Z
 
 Update the LLVM Website
@@ -435,7 +412,7 @@ using this command and add it to the post.
 
 ::
 
-  $ git log --format="[%h %s](https://github.com/llvm/llvm-project/commit/%H)" llvmorg-X.0.N-1..llvmorg-X.0.N
+  $ git log --format="- %aN: [%s (%h)](https://github.com/llvm/llvm-project/commit/%H)" llvmorg-X.0.N-1..llvmorg-X.0.N
 
 Once the release has been announced add a link to the announcement on the llvm
 homepage (from the llvm-www repo) in the "Release Emails" section.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index dc571093e2cb8..e79f23bb33e16 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -370,6 +370,9 @@ added in the future:
       Floating-point registers (XMMs/YMMs) are not preserved and need to be
       saved by the caller.
 
+    - On AArch64 the callee preserve all general purpose registers, except X0-X8
+      and X16-X18.
+
     The idea behind this convention is to support calls to runtime functions
     that have a hot path and a cold path. The hot path is usually a small piece
     of code that doesn't use many registers. The cold path might need to call out to
@@ -404,6 +407,10 @@ added in the future:
       R11. R11 can be used as a scratch register. Furthermore it also preserves
       all floating-point registers (XMMs/YMMs).
 
+    - On AArch64 the callee preserve all general purpose registers, except X0-X8
+      and X16-X18. Furthermore it also preserves lower 128 bits of V8-V31 SIMD -
+      floating point registers.
+
     The idea behind this convention is to support calls to runtime functions
     that don't need to call out to any other functions.
 
@@ -2226,31 +2233,36 @@ example:
     This indicates the denormal (subnormal) handling that may be
     assumed for the default floating-point environment. This is a
     comma separated pair. The elements may be one of ``"ieee"``,
-    ``"preserve-sign"``, or ``"positive-zero"``. The first entry
-    indicates the flushing mode for the result of floating point
-    operations. The second indicates the handling of denormal inputs
+    ``"preserve-sign"``, ``"positive-zero"``, or ``"dynamic"``. The
+    first entry indicates the flushing mode for the result of floating
+    point operations. The second indicates the handling of denormal inputs
     to floating point instructions. For compatibility with older
     bitcode, if the second value is omitted, both input and output
     modes will assume the same mode.
 
-    If this is attribute is not specified, the default is
-    ``"ieee,ieee"``.
+    If this is attribute is not specified, the default is ``"ieee,ieee"``.
 
     If the output mode is ``"preserve-sign"``, or ``"positive-zero"``,
     denormal outputs may be flushed to zero by standard floating-point
     operations. It is not mandated that flushing to zero occurs, but if
     a denormal output is flushed to zero, it must respect the sign
-    mode. Not all targets support all modes. While this indicates the
-    expected floating point mode the function will be executed with,
-    this does not make any attempt to ensure the mode is
-    consistent. User or platform code is expected to set the floating
-    point mode appropriately before function entry.
-
-   If the input mode is ``"preserve-sign"``, or ``"positive-zero"``, a
-   floating-point operation must treat any input denormal value as
-   zero. In some situations, if an instruction does not respect this
-   mode, the input may need to be converted to 0 as if by
-   ``@llvm.canonicalize`` during lowering for correctness.
+    mode. Not all targets support all modes.
+
+    If the mode is ``"dynamic"``, the behavior is derived from the
+    dynamic state of the floating-point environment. Transformations
+    which depend on the behavior of denormal values should not be
+    performed.
+
+    While this indicates the expected floating point mode the function
+    will be executed with, this does not make any attempt to ensure
+    the mode is consistent. User or platform code is expected to set
+    the floating point mode appropriately before function entry.
+
+    If the input mode is ``"preserve-sign"``, or ``"positive-zero"``,
+    a floating-point operation must treat any input denormal value as
+    zero. In some situations, if an instruction does not respect this
+    mode, the input may need to be converted to 0 as if by
+    ``@llvm.canonicalize`` during lowering for correctness.
 
 ``"denormal-fp-math-f32"``
     Same as ``"denormal-fp-math"``, but only controls the behavior of
@@ -5978,28 +5990,32 @@ The current supported opcode vocabulary is limited:
   of the stack is treated as an address. The second stack entry is treated as an
   address space identifier.
 - ``DW_OP_stack_value`` marks a constant value.
-- ``DW_OP_LLVM_entry_value, N`` may only appear in MIR and at the
-  beginning of a ``DIExpression``. In DWARF a ``DBG_VALUE``
-  instruction binding a ``DIExpression(DW_OP_LLVM_entry_value`` to a
-  register is lowered to a ``DW_OP_entry_value [reg]``, pushing the
-  value the register had upon function entry onto the stack.  The next
-  ``(N - 1)`` operations will be part of the ``DW_OP_entry_value``
-  block argument. For example, ``!DIExpression(DW_OP_LLVM_entry_value,
-  1, DW_OP_plus_uconst, 123, DW_OP_stack_value)`` specifies an
-  expression where the entry value of the debug value instruction's
-  value/address operand is pushed to the stack, and is added
-  with 123. Due to framework limitations ``N`` can currently only
-  be 1.
-
-  The operation is introduced by the ``LiveDebugValues`` pass, which
-  applies it only to function parameters that are unmodified
-  throughout the function. Support is limited to simple register
-  location descriptions, or as indirect locations (e.g., when a struct
-  is passed-by-value to a callee via a pointer to a temporary copy
-  made in the caller). The entry value op is also introduced by the
-  ``AsmPrinter`` pass when a call site parameter value
-  (``DW_AT_call_site_parameter_value``) is represented as entry value
-  of the parameter.
+- ``DW_OP_LLVM_entry_value, N`` refers to the value a register had upon
+  function entry. When targeting DWARF, a ``DBG_VALUE(reg, ...,
+  DIExpression(DW_OP_LLVM_entry_value, 1, ...)`` is lowered to
+  ``DW_OP_entry_value [reg], ...``, which pushes the value ``reg`` had upon
+  function entry onto the DWARF expression stack.
+
+  The next ``(N - 1)`` operations will be part of the ``DW_OP_entry_value``
+  block argument. For example, ``!DIExpression(DW_OP_LLVM_entry_value, 1,
+  DW_OP_plus_uconst, 123, DW_OP_stack_value)`` specifies an expression where
+  the entry value of ``reg`` is pushed onto the stack, and is added with 123.
+  Due to framework limitations ``N`` must be 1, in other words,
+  ``DW_OP_entry_value`` always refers to the value/address operand of the
+  instruction.
+
+  Because ``DW_OP_LLVM_entry_value`` is defined in terms of registers, it is
+  only allowed in MIR. The operation is introduced by:
+
+    - ``LiveDebugValues`` pass, which applies it to function parameters that
+      are unmodified throughout the function. Support is limited to simple
+      register location descriptions, or as indirect locations (e.g.,
+      parameters passed-by-value to a callee via a pointer to a temporary copy
+      made in the caller).
+    - ``AsmPrinter`` pass when a call site parameter value
+      (``DW_AT_call_site_parameter_value``) is represented as entry value of
+      the parameter.
+
 - ``DW_OP_LLVM_arg, N`` is used in debug intrinsics that refer to more than one
   value, such as one that calculates the sum of two registers. This is always
   used in combination with an ordered list of values, such that
@@ -10071,7 +10087,7 @@ Arguments:
 The first two operands of a '``shufflevector``' instruction are vectors
 with the same type. The third argument is a shuffle mask vector constant
 whose element type is ``i32``. The mask vector elements must be constant
-integers or ``undef`` values. The result of the instruction is a vector
+integers or ``poison`` values. The result of the instruction is a vector
 whose length is the same as the shuffle mask and whose element type is the
 same as the element type of the first two operands.
 
@@ -10084,15 +10100,15 @@ shuffle mask selects an element from one of the input vectors to copy
 to the result. Non-negative elements in the mask represent an index
 into the concatenated pair of input vectors.
 
-If the shuffle mask is undefined, the result vector is undefined. If
-the shuffle mask selects an undefined element from one of the input
-vectors, the resulting element is undefined. An undefined element
-in the mask vector specifies that the resulting element is undefined.
-An undefined element in the mask vector prevents a poisoned vector
-element from propagating.
+A ``poison`` element in the mask vector specifies that the resulting element
+is ``poison``.
+For backwards-compatibility reasons, LLVM temporarily also accepts ``undef``
+mask elements, which will be interpreted the same way as ``poison`` elements.
+If the shuffle mask selects an ``undef`` element from one of the input
+vectors, the resulting element is ``undef``.
 
 For scalable vectors, the only valid mask values at present are
-``zeroinitializer`` and ``undef``, since we cannot write all indices as
+``zeroinitializer``, ``undef`` and ``poison``, since we cannot write all indices as
 literals for a vector with a length unknown at compile time.
 
 Example:
@@ -10102,9 +10118,9 @@ Example:
 
       <result> = shufflevector <4 x i32> %v1, <4 x i32> %v2,
                               <4 x i32> <i32 0, i32 4, i32 1, i32 5>  ; yields <4 x i32>
-      <result> = shufflevector <4 x i32> %v1, <4 x i32> undef,
+      <result> = shufflevector <4 x i32> %v1, <4 x i32> poison,
                               <4 x i32> <i32 0, i32 1, i32 2, i32 3>  ; yields <4 x i32> - Identity shuffle.
-      <result> = shufflevector <8 x i32> %v1, <8 x i32> undef,
+      <result> = shufflevector <8 x i32> %v1, <8 x i32> poison,
                               <4 x i32> <i32 0, i32 1, i32 2, i32 3>  ; yields <4 x i32>
       <result> = shufflevector <4 x i32> %v1, <4 x i32> %v2,
                               <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7 >  ; yields <8 x i32>
@@ -12013,10 +12029,11 @@ This instruction requires several arguments:
 
 #. The optional ``tail`` and ``musttail`` markers indicate that the optimizers
    should perform tail call optimization. The ``tail`` marker is a hint that
-   `can be ignored <CodeGenerator.html#sibcallopt>`_. The ``musttail`` marker
-   means that the call must be tail call optimized in order for the program to
-   be correct. This is true even in the presence of attributes like
-   "disable-tail-calls". The ``musttail`` marker provides these guarantees:
+   `can be ignored <CodeGenerator.html#tail-call-optimization>`_. The
+   ``musttail`` marker means that the call must be tail call optimized in order
+   for the program to be correct. This is true even in the presence of
+   attributes like "disable-tail-calls". The ``musttail`` marker provides these
+   guarantees:
 
    #. The call will not cause unbounded stack growth if it is part of a
       recursive cycle in the call graph.
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index daec01d5bd2c7..5c4ae89085f16 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -100,18 +100,20 @@ on support follow.
      ``Zbkc``         Supported
      ``Zbkx``         Supported (`See note <#riscv-scalar-crypto-note1>`__)
      ``Zbs``          Supported
-     ``Zdinx``        Assembly Support
+     ``Zdinx``        Assembly Support for RV32. Full support for RV64.
      ``Zfh``          Supported
      ``Zfhmin``       Supported
-     ``Zfinx``        Assembly Support
+     ``Zfinx``        Supported
      ``Zhinx``        Assembly Support
      ``Zhinxmin``     Assembly Support
      ``Zicbom``       Assembly Support
      ``Zicbop``       Assembly Support
      ``Zicboz``       Assembly Support
+     ``Zicntr``       (`See Note <#riscv-i2p1-note>`__)
      ``Zicsr``        (`See Note <#riscv-i2p1-note>`__)
      ``Zifencei``     (`See Note <#riscv-i2p1-note>`__)
      ``Zihintpause``  Assembly Support
+     ``Zihpm``        (`See Note <#riscv-i2p1-note>`__)
      ``Zkn``          Supported
      ``Zknd``         Supported (`See note <#riscv-scalar-crypto-note2>`__)
      ``Zkne``         Supported (`See note <#riscv-scalar-crypto-note2>`__)
@@ -165,8 +167,8 @@ Supported
 
 .. _riscv-i2p1-note:
 
-``zicsr``, ``zifencei``
-  Between versions 2.0 and 2.1 of the base I specification, a backwards incompatible change was made to remove selected instructions and CSRs from the base ISA.  These instructions were grouped into a set of new extensions, but were no longer required by the base ISA.  This change is described in "Preface to Document Version 20190608-Base-Ratified" from the specification document.  LLVM currently implements version 2.1 of the base specification. To maintain compatibility, instructions from these extensions are accepted without being in the ``-march`` string.  LLVM also allows the explicit specification of the extensions in an ``-march`` string.
+``zicntr``, ``zicsr``, ``zifencei``, ``zihpm``
+  Between versions 2.0 and 2.1 of the base I specification, a backwards incompatible change was made to remove selected instructions and CSRs from the base ISA.  These instructions were grouped into a set of new extensions, but were no longer required by the base ISA.  This change is partially described in "Preface to Document Version 20190608-Base-Ratified" from the specification document (the ``zicntr`` and ``zihpm`` bits are not mentioned).  LLVM currently implements version 2.1 of the base specification. To maintain compatibility, instructions from these extensions are accepted without being in the ``-march`` string.  LLVM also allows the explicit specification of the extensions in an ``-march`` string.
 
 Experimental Extensions
 =======================
@@ -175,6 +177,12 @@ LLVM supports (to various degrees) a number of experimental extensions.  All exp
 
 The primary goal of experimental support is to assist in the process of ratification by providing an existence proof of an implementation, and simplifying efforts to validate the value of a proposed extension against large code bases.  Experimental extensions are expected to either transition to ratified status, or be eventually removed.  The decision on whether to accept an experimental extension is currently done on an entirely case by case basis; if you want to propose one, attending the bi-weekly RISC-V sync-up call is strongly advised.
 
+``experimental-smaia``
+  LLVM implements the `Ratification candidate 3 <https://github.com/riscv/riscv-aia/releases/tag/1.0-RC3>`_.
+
+``experimental-ssaia``
+  LLVM implements the `Ratification candidate 3 <https://github.com/riscv/riscv-aia/releases/tag/1.0-RC3>`_.
+
 ``experimental-zca``
   LLVM implements the `1.0.1 draft specification <https://github.com/riscv/riscv-code-size-reduction/releases/tag/v1.0.1>`__.
 
@@ -187,6 +195,9 @@ The primary goal of experimental support is to assist in the process of ratifica
 ``experimental-zcf``
   LLVM implements the `1.0.1 draft specification <https://github.com/riscv/riscv-code-size-reduction/releases/tag/v1.0.1>`__.
 
+``experimental-zcmt``
+  LLVM implements the `1.0.1 draft specification <https://github.com/riscv/riscv-code-size-reduction/releases/tag/v1.0.1>`_.
+
 ``experimental-zfa``
   LLVM implements the `0.2 draft specification <https://github.com/riscv/riscv-isa-manual/releases/download/draft-20230131-c0b298a/zfa-20230414.pdf>`__.
 
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index aef940a63cf09..c764a50f88b22 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -84,6 +84,7 @@ Changes to the AArch64 Backend
 
 * Added Assembly Support for the 2022 A-profile extensions FEAT_GCS (Guarded
   Control Stacks), FEAT_CHK (Check Feature Status), and FEAT_ATS1A.
+* Support for preserve_all calling convention is added.
 
 Changes to the AMDGPU Backend
 -----------------------------
@@ -169,6 +170,11 @@ Changes to the RISC-V Backend
 * Added support for Zfa extension version 0.2.
 * Updated support experimental vector crypto extensions to version 0.5.1 of
   the specification.
+* Removed N extension (User-Level Interrupts) CSR names in the assembler.
+* ``RISCV::parseCPUKind`` and ``RISCV::checkCPUKind`` were merged into a single
+  ``RISCV::parseCPU``. The ``CPUKind`` enum is no longer part of the
+  RISCVTargetParser.h interface. Similar for ``parseTuneCPUkind`` and
+  ``checkTuneCPUKind``.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/docs/SourceLevelDebugging.rst b/llvm/docs/SourceLevelDebugging.rst
index be288f6f7f172..3a90d242dedc8 100644
--- a/llvm/docs/SourceLevelDebugging.rst
+++ b/llvm/docs/SourceLevelDebugging.rst
@@ -436,7 +436,7 @@ trust in the debugger.
 
 Sometimes perfectly preserving variable locations is not possible, often when a
 redundant calculation is optimized out. In such cases, a ``llvm.dbg.value``
-with operand ``undef`` should be used, to terminate earlier variable locations
+with operand ``poison`` should be used, to terminate earlier variable locations
 and let the debugger present ``optimized out`` to the developer. Withholding
 these potentially stale variable values from the developer diminishes the
 amount of available debug information, but increases the reliability of the
@@ -508,7 +508,7 @@ might consider this placement of dbg.values:
 However, this will cause ``!3`` to have the return value of ``@gazonk()`` at
 the same time as ``!1`` has the constant value zero -- a pair of assignments
 that never occurred in the unoptimized program. To avoid this, we must terminate
-the range that ``!1`` has the constant value assignment by inserting an undef
+the range that ``!1`` has the constant value assignment by inserting a poison
 dbg.value before the dbg.value for ``!3``:
 
 .. code-block:: llvm
@@ -517,7 +517,7 @@ dbg.value before the dbg.value for ``!3``:
   entry:
     call @llvm.dbg.value(metadata i32 0, metadata !1, metadata !2)
     %g = call i32 @gazonk()
-    call @llvm.dbg.value(metadata i32 undef, metadata !1, metadata !2)
+    call @llvm.dbg.value(metadata i32 poison, metadata !1, metadata !2)
     call @llvm.dbg.value(metadata i32 %g, metadata !3, metadata !2)
     %addoper = select i1 %cond, i32 11, i32 12
     %plusten = add i32 %bar, %addoper
@@ -526,9 +526,26 @@ dbg.value before the dbg.value for ``!3``:
     ret i32 %toret
   }
 
+There are a few other dbg.value configurations that mean it terminates
+dominating location definitions without adding a new location. The complete
+list is:
+
+* Any location operand is ``poison`` (or ``undef``).
+* Any location operand is an empty metadata tuple (``!{}``) (which cannot
+  occur in a ``!DIArgList``).
+* There are no location operands (empty ``DIArgList``) and the ``DIExpression``
+  is empty.
+
+This class of dbg.value that kills variable locations is called a "kill
+dbg.value" or "kill location", and for legacy reasons the term "undef
+dbg.value" may be used in existing code. The ``DbgVariableIntrinsic`` methods
+``isKillLocation`` and ``setKillLocation`` should be used where possible rather
+than inspecting location operands directly to check or set whether a dbg.value
+is a kill location.
+
 In general, if any dbg.value has its operand optimized out and cannot be
-recovered, then an undef dbg.value is necessary to terminate earlier variable
-locations. Additional undef dbg.values may be necessary when the debugger can
+recovered, then a kill dbg.value is necessary to terminate earlier variable
+locations. Additional kill dbg.values may be necessary when the debugger can
 observe re-ordering of assignments.
 
 How variable location metadata is transformed during CodeGen
diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst
index 9cada3b7aaf67..b4b35c4c16000 100644
--- a/llvm/docs/TableGen/ProgRef.rst
+++ b/llvm/docs/TableGen/ProgRef.rst
@@ -225,9 +225,9 @@ TableGen provides "bang operators" that have a wide variety of uses:
                : !gt          !head        !if          !interleave  !isa
                : !le          !listconcat  !listremove  !listsplat   !logtwo
                : !lt          !mul         !ne          !not         !or
-               : !setdagop    !shl         !size        !sra         !srl
-               : !strconcat   !sub         !subst       !substr      !tail
-               : !tolower     !toupper     !xor
+               : !range       !setdagop    !shl         !size        !sra
+               : !srl         !strconcat   !sub         !subst       !substr
+               : !tail        !tolower     !toupper     !xor
 
 The ``!cond`` operator has a slightly different
 syntax compared to other bang operators, so it is defined separately:
@@ -335,19 +335,25 @@ to an entity of type ``bits<4>``.
    Value: `SimpleValue` `ValueSuffix`*
         :| `Value` "#" [`Value`]
    ValueSuffix: "{" `RangeList` "}"
-              :| "[" `RangeList` "]"
+              :| "[" `SliceElements` "]"
               :| "." `TokIdentifier`
    RangeList: `RangePiece` ("," `RangePiece`)*
    RangePiece: `TokInteger`
              :| `TokInteger` "..." `TokInteger`
              :| `TokInteger` "-" `TokInteger`
              :| `TokInteger` `TokInteger`
+   SliceElements: (`SliceElement` ",")* `SliceElement` ","?
+   SliceElement: `Value`
+               :| `Value` "..." `Value`
+               :| `Value` "-" `Value`
+               :| `Value` `TokInteger`
 
 .. warning::
-  The peculiar last form of :token:`RangePiece` is due to the fact that the
-  "``-``" is included in the :token:`TokInteger`, hence ``1-5`` gets lexed as
-  two consecutive tokens, with values ``1`` and ``-5``, instead of "1", "-",
-  and "5". The use of hyphen as the range punctuation is deprecated.
+  The peculiar last form of :token:`RangePiece` and :token:`SliceElement` is
+  due to the fact that the "``-``" is included in the :token:`TokInteger`,
+  hence ``1-5`` gets lexed as two consecutive tokens, with values ``1`` and
+  ``-5``, instead of "1", "-", and "5".
+  The use of hyphen as the range punctuation is deprecated.
 
 Simple values
 -------------
@@ -505,17 +511,26 @@ primary value. Here are the possible suffixes for some primary *value*.
     The final value is bits 8--15 of the integer *value*. The order of the
     bits can be reversed by specifying ``{15...8}``.
 
-*value*\ ``[4]``
-    The final value is element 4 of the list *value* (note the brackets).
+*value*\ ``[i]``
+    The final value is element `i` of the list *value* (note the brackets).
     In other words, the brackets act as a subscripting operator on the list.
     This is the case only when a single element is specified.
 
+*value*\ ``[i,]``
+    The final value is a list that contains a single element `i` of the list.
+    In short, a list slice with a single element.
+
 *value*\ ``[4...7,17,2...3,4]``
     The final value is a new list that is a slice of the list *value*.
     The new list contains elements 4, 5, 6, 7, 17, 2, 3, and 4.
     Elements may be included multiple times and in any order. This is the result
     only when more than one element is specified.
 
+    *value*\ ``[i,m...n,j,ls]``
+        Each element may be an expression (variables, bang operators).
+        The type of `m` and `n` should be `int`.
+        The type of `i`, `j`, and `ls` should be either `int` or `list<int>`.
+
 *value*\ ``.``\ *field*
     The final value is the value of the specified *field* in the specified
     record *value*.
@@ -1777,6 +1792,15 @@ and non-0 as true.
     result. A logical OR can be performed if all the arguments are either
     0 or 1.
 
+``!range([``\ *a*\ ``,``] *b*\ ``)``
+    This operator produces half-open range sequence ``[a : b)`` as ``list<int>``.
+    *a* is ``0`` by default. ``!range(4)`` is equivalent to ``!range(0, 4)``.
+    The result is `[0, 1, 2, 3]`.
+    If *a* ``>=`` *b*, then the result is `[]<list<int>>`.
+
+``!range(``\ *list*\ ``)``
+    Equivalent to ``!range(0, !size(list))``.
+
 ``!setdagop(``\ *dag*\ ``,`` *op*\ ``)``
     This operator produces a DAG node with the same arguments as *dag*, but with its
     operator replaced with *op*.
diff --git a/llvm/docs/doxygen.cfg.in b/llvm/docs/doxygen.cfg.in
index b19ca6d215807..4f2f02f400bb0 100644
--- a/llvm/docs/doxygen.cfg.in
+++ b/llvm/docs/doxygen.cfg.in
@@ -2091,7 +2091,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2136,7 +2136,7 @@ TEMPLATE_RELATIONS     = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2145,7 +2145,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/llvm/include/CMakeLists.txt b/llvm/include/CMakeLists.txt
new file mode 100644
index 0000000000000..06da8dcc26097
--- /dev/null
+++ b/llvm/include/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_subdirectory(llvm)
+
+# If we're doing an out-of-tree build, copy a module map for generated
+# header files into the build area.
+if (NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
+  configure_file(module.modulemap.build module.modulemap COPYONLY)
+endif (NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 0eaf949c61f48..78dff8a8c274f 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -4093,8 +4093,8 @@ int LLVMGetUndefMaskElem(void);
  * Get the mask value at position Elt in the mask of a ShuffleVector
  * instruction.
  *
- * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is undef
- * at that position.
+ * \Returns the result of \c LLVMGetUndefMaskElem() if the mask value is
+ * poison at that position.
  */
 int LLVMGetMaskValue(LLVMValueRef ShuffleVectorInst, unsigned Elt);
 
diff --git a/llvm/include/llvm-c/module.modulemap b/llvm/include/llvm-c/module.modulemap
deleted file mode 100644
index a456119595c95..0000000000000
--- a/llvm/include/llvm-c/module.modulemap
+++ /dev/null
@@ -1,4 +0,0 @@
-module LLVM_C {
-  umbrella "."
-  module * { export * }
-}
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index 8a680c031d302..3ef6a7cd1b4b5 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -312,6 +312,20 @@ class DenseMapBase : public DebugEpochBase {
       insert(*I);
   }
 
+  /// Returns the value associated to the key in the map if it exists. If it
+  /// does not exist, emplace a default value for the key and returns a
+  /// reference to the newly created value.
+  ValueT &getOrInsertDefault(KeyT &&Key) {
+    return try_emplace(Key).first->second;
+  }
+
+  /// Returns the value associated to the key in the map if it exists. If it
+  /// does not exist, emplace a default value for the key and returns a
+  /// reference to the newly created value.
+  ValueT &getOrInsertDefault(const KeyT &Key) {
+    return try_emplace(Key).first->second;
+  }
+
   bool erase(const KeyT &Val) {
     BucketT *TheBucket;
     if (!LookupBucketFor(Val, TheBucket))
diff --git a/llvm/include/llvm/ADT/FloatingPointMode.h b/llvm/include/llvm/ADT/FloatingPointMode.h
index a7198babc69bf..61e57094fdbb9 100644
--- a/llvm/include/llvm/ADT/FloatingPointMode.h
+++ b/llvm/include/llvm/ADT/FloatingPointMode.h
@@ -80,7 +80,10 @@ struct DenormalMode {
     PreserveSign,
 
     /// Denormals are flushed to positive zero.
-    PositiveZero
+    PositiveZero,
+
+    /// Denormals have unknown treatment.
+    Dynamic
   };
 
   /// Denormal flushing mode for floating point instruction results in the
@@ -101,6 +104,11 @@ struct DenormalMode {
     return DenormalMode(DenormalModeKind::Invalid, DenormalModeKind::Invalid);
   }
 
+  /// Return the assumed default mode for a function without denormal-fp-math.
+  static constexpr DenormalMode getDefault() {
+    return getIEEE();
+  }
+
   static constexpr DenormalMode getIEEE() {
     return DenormalMode(DenormalModeKind::IEEE, DenormalModeKind::IEEE);
   }
@@ -115,6 +123,10 @@ struct DenormalMode {
                         DenormalModeKind::PositiveZero);
   }
 
+  static constexpr DenormalMode getDynamic() {
+    return DenormalMode(DenormalModeKind::Dynamic, DenormalModeKind::Dynamic);
+  }
+
   bool operator==(DenormalMode Other) const {
     return Output == Other.Output && Input == Other.Input;
   }
@@ -144,6 +156,18 @@ struct DenormalMode {
            Output == DenormalModeKind::PositiveZero;
   }
 
+  /// Get the effective denormal mode if the mode if this caller calls into a
+  /// function with \p Callee. This promotes dynamic modes to the mode of the
+  /// caller.
+  DenormalMode mergeCalleeMode(DenormalMode Callee) const {
+    DenormalMode MergedMode = Callee;
+    if (Callee.Input == DenormalMode::Dynamic)
+      MergedMode.Input = Input;
+    if (Callee.Output == DenormalMode::Dynamic)
+      MergedMode.Output = Output;
+    return MergedMode;
+  }
+
   inline void print(raw_ostream &OS) const;
 
   inline std::string str() const {
@@ -164,10 +188,11 @@ inline DenormalMode::DenormalModeKind
 parseDenormalFPAttributeComponent(StringRef Str) {
   // Assume ieee on unspecified attribute.
   return StringSwitch<DenormalMode::DenormalModeKind>(Str)
-    .Cases("", "ieee", DenormalMode::IEEE)
-    .Case("preserve-sign", DenormalMode::PreserveSign)
-    .Case("positive-zero", DenormalMode::PositiveZero)
-    .Default(DenormalMode::Invalid);
+      .Cases("", "ieee", DenormalMode::IEEE)
+      .Case("preserve-sign", DenormalMode::PreserveSign)
+      .Case("positive-zero", DenormalMode::PositiveZero)
+      .Case("dynamic", DenormalMode::Dynamic)
+      .Default(DenormalMode::Invalid);
 }
 
 /// Return the name used for the denormal handling mode used by the the
@@ -180,6 +205,8 @@ inline StringRef denormalModeKindName(DenormalMode::DenormalModeKind Mode) {
     return "preserve-sign";
   case DenormalMode::PositiveZero:
     return "positive-zero";
+  case DenormalMode::Dynamic:
+    return "dynamic";
   default:
     return "";
   }
diff --git a/llvm/include/llvm/ADT/SmallVectorExtras.h b/llvm/include/llvm/ADT/SmallVectorExtras.h
new file mode 100644
index 0000000000000..8d5228025e0e7
--- /dev/null
+++ b/llvm/include/llvm/ADT/SmallVectorExtras.h
@@ -0,0 +1,31 @@
+//===- llvm/ADT/SmallVectorExtras.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines less commonly used SmallVector utilities.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_SMALLVECTOREXTRAS_H
+#define LLVM_ADT_SMALLVECTOREXTRAS_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+/// Map a range to a SmallVector with element types deduced from the mapping.
+template <class ContainerTy, class FuncTy>
+auto map_to_vector(ContainerTy &&C, FuncTy &&F) {
+  return to_vector(
+      map_range(std::forward<ContainerTy>(C), std::forward<FuncTy>(F)));
+}
+
+} // namespace llvm
+
+#endif // LLVM_ADT_SMALLVECTOREXTRAS_H
\ No newline at end of file
diff --git a/llvm/include/llvm/ADT/ilist.h b/llvm/include/llvm/ADT/ilist.h
index 9913b7cccbdda..aed19ccbff7f2 100644
--- a/llvm/include/llvm/ADT/ilist.h
+++ b/llvm/include/llvm/ADT/ilist.h
@@ -92,63 +92,6 @@ struct ilist_traits : public ilist_node_traits<NodeTy> {};
 /// Const traits should never be instantiated.
 template <typename Ty> struct ilist_traits<const Ty> {};
 
-namespace ilist_detail {
-
-template <class T> T &make();
-
-/// Type trait to check for a traits class that has a getNext member (as a
-/// canary for any of the ilist_nextprev_traits API).
-template <class TraitsT, class NodeT> struct HasGetNext {
-  typedef char Yes[1];
-  typedef char No[2];
-  template <size_t N> struct SFINAE {};
-
-  template <class U>
-  static Yes &test(U *I, decltype(I->getNext(&make<NodeT>())) * = nullptr);
-  template <class> static No &test(...);
-
-public:
-  static const bool value = sizeof(test<TraitsT>(nullptr)) == sizeof(Yes);
-};
-
-/// Type trait to check for a traits class that has a createSentinel member (as
-/// a canary for any of the ilist_sentinel_traits API).
-template <class TraitsT> struct HasCreateSentinel {
-  typedef char Yes[1];
-  typedef char No[2];
-
-  template <class U>
-  static Yes &test(U *I, decltype(I->createSentinel()) * = nullptr);
-  template <class> static No &test(...);
-
-public:
-  static const bool value = sizeof(test<TraitsT>(nullptr)) == sizeof(Yes);
-};
-
-/// Type trait to check for a traits class that has a createNode member.
-/// Allocation should be managed in a wrapper class, instead of in
-/// ilist_traits.
-template <class TraitsT, class NodeT> struct HasCreateNode {
-  typedef char Yes[1];
-  typedef char No[2];
-  template <size_t N> struct SFINAE {};
-
-  template <class U>
-  static Yes &test(U *I, decltype(I->createNode(make<NodeT>())) * = 0);
-  template <class> static No &test(...);
-
-public:
-  static const bool value = sizeof(test<TraitsT>(nullptr)) == sizeof(Yes);
-};
-
-template <class TraitsT, class NodeT> struct HasObsoleteCustomization {
-  static const bool value = HasGetNext<TraitsT, NodeT>::value ||
-                            HasCreateSentinel<TraitsT>::value ||
-                            HasCreateNode<TraitsT, NodeT>::value;
-};
-
-} // end namespace ilist_detail
-
 //===----------------------------------------------------------------------===//
 //
 /// A wrapper around an intrusive list with callbacks and non-intrusive
@@ -182,13 +125,6 @@ class iplist_impl : public TraitsT, IntrusiveListT {
       typename base_list_type::const_reverse_iterator const_reverse_iterator;
 
 private:
-  // TODO: Drop this assertion and the transitive type traits anytime after
-  // v4.0 is branched (i.e,. keep them for one release to help out-of-tree code
-  // update).
-  static_assert(
-      !ilist_detail::HasObsoleteCustomization<TraitsT, value_type>::value,
-      "ilist customization points have changed!");
-
   static bool op_less(const_reference L, const_reference R) { return L < R; }
   static bool op_equal(const_reference L, const_reference R) { return L == R; }
 
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 23ec7d6b70ec4..b11981c609ce1 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -99,6 +99,9 @@ Constant *ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
 /// correct sign, otherwise return the original constant. Inputs and outputs to
 /// floating point instructions can have their mode set separately, so the
 /// direction is also needed.
+///
+/// If the calling function's "denormal-fp-math" input mode is "dynamic" for the
+/// floating-point type, returns nullptr for denormal inputs.
 Constant *FlushFPConstant(Constant *Operand, const Instruction *I,
                           bool IsOutput);
 
diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
index d40d51e2d3376..ad137baff5d42 100644
--- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
+++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
@@ -127,7 +127,7 @@ struct IRInstructionData
 
   /// This is only relevant if we are wrapping a CmpInst where we needed to
   /// change the predicate of a compare instruction from a greater than form
-  /// to a less than form.  It is None otherwise.
+  /// to a less than form.  It is std::nullopt otherwise.
   std::optional<CmpInst::Predicate> RevisedPredicate;
 
   /// This is only relevant if we are wrapping a CallInst. If we are requiring
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index ad936d213134c..d8da0b86ec511 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -322,7 +322,9 @@ class InductionDescriptor {
 
   /// Returns true if \p Phi is an induction in the loop \p L. If \p Phi is an
   /// induction, the induction descriptor \p D will contain the data describing
-  /// this induction. If by some other means the caller has a better SCEV
+  /// this induction. Since Induction Phis can only be present inside loop
+  /// headers, the function will assert if it is passed a Phi whose parent is
+  /// not the loop header. If by some other means the caller has a better SCEV
   /// expression for \p Phi than the one returned by the ScalarEvolution
   /// analysis, it can be passed through \p Expr. If the def-use chain
   /// associated with the phi includes casts (that we know we can ignore
diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
index a64e4c3cb7274..77ae60059ce9e 100644
--- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
+++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
@@ -22,34 +22,51 @@ namespace llvm {
 // inline cost, and we define them separately to preserve the original heuristic
 // behavior.
 #define INLINE_COST_FEATURE_ITERATOR(M)                                        \
-  M(SROASavings, "sroa_savings")                                               \
-  M(SROALosses, "sroa_losses")                                                 \
-  M(LoadElimination, "load_elimination")                                       \
-  M(CallPenalty, "call_penalty")                                               \
-  M(CallArgumentSetup, "call_argument_setup")                                  \
-  M(LoadRelativeIntrinsic, "load_relative_intrinsic")                          \
-  M(LoweredCallArgSetup, "lowered_call_arg_setup")                             \
-  M(IndirectCallPenalty, "indirect_call_penalty")                              \
-  M(JumpTablePenalty, "jump_table_penalty")                                    \
-  M(CaseClusterPenalty, "case_cluster_penalty")                                \
-  M(SwitchPenalty, "switch_penalty")                                           \
-  M(UnsimplifiedCommonInstructions, "unsimplified_common_instructions")        \
-  M(NumLoops, "num_loops")                                                     \
-  M(DeadBlocks, "dead_blocks")                                                 \
-  M(SimplifiedInstructions, "simplified_instructions")                         \
-  M(ConstantArgs, "constant_args")                                             \
-  M(ConstantOffsetPtrArgs, "constant_offset_ptr_args")                         \
-  M(CallSiteCost, "callsite_cost")                                             \
-  M(ColdCcPenalty, "cold_cc_penalty")                                          \
-  M(LastCallToStaticBonus, "last_call_to_static_bonus")                        \
-  M(IsMultipleBlocks, "is_multiple_blocks")                                    \
-  M(NestedInlines, "nested_inlines")                                           \
-  M(NestedInlineCostEstimate, "nested_inline_cost_estimate")                   \
-  M(Threshold, "threshold")
+  M(int64_t, {1}, sroa_savings,                                                \
+    "Savings from SROA (scalar replacement of aggregates)")                    \
+  M(int64_t, {1}, sroa_losses,                                                 \
+    "Losses from SROA (scalar replacement of aggregates)")                     \
+  M(int64_t, {1}, load_elimination, "Cost of load elimination in the call")    \
+  M(int64_t, {1}, call_penalty,                                                \
+    "Accumulation of penalty applied to call sites when inlining")             \
+  M(int64_t, {1}, call_argument_setup,                                         \
+    "Accumulation of call argument setup costs")                               \
+  M(int64_t, {1}, load_relative_intrinsic,                                     \
+    "Accumulation of costs of loading relative intrinsics")                    \
+  M(int64_t, {1}, lowered_call_arg_setup,                                      \
+    "Accumulation of cost of lowered call argument setups")                    \
+  M(int64_t, {1}, indirect_call_penalty,                                       \
+    "Accumulation of costs for indirect calls")                                \
+  M(int64_t, {1}, jump_table_penalty, "Accumulation of costs for jump tables") \
+  M(int64_t, {1}, case_cluster_penalty,                                        \
+    "Accumulation of costs for case clusters")                                 \
+  M(int64_t, {1}, switch_penalty,                                              \
+    "Accumulation of costs for switch statements")                             \
+  M(int64_t, {1}, unsimplified_common_instructions,                            \
+    "Costs from unsimplified common instructions")                             \
+  M(int64_t, {1}, num_loops, "Number of loops in the caller")                  \
+  M(int64_t, {1}, dead_blocks, "Number of dead blocks in the caller")          \
+  M(int64_t, {1}, simplified_instructions,                                     \
+    "Number of simplified instructions")                                       \
+  M(int64_t, {1}, constant_args,                                               \
+    "Number of constant arguments in the call site")                           \
+  M(int64_t, {1}, constant_offset_ptr_args,                                    \
+    "Number of constant offset pointer args in the call site")                 \
+  M(int64_t, {1}, callsite_cost, "Estimated cost of the call site")            \
+  M(int64_t, {1}, cold_cc_penalty, "Penalty for a cold calling convention")    \
+  M(int64_t, {1}, last_call_to_static_bonus,                                   \
+    "Bonus for being the last call to static")                                 \
+  M(int64_t, {1}, is_multiple_blocks,                                          \
+    "Boolean; is the Callee multiple blocks")                                  \
+  M(int64_t, {1}, nested_inlines,                                              \
+    "Would the default inliner perfom nested inlining")                        \
+  M(int64_t, {1}, nested_inline_cost_estimate,                                 \
+    "Estimate of the accumulated cost of nested inlines")                      \
+  M(int64_t, {1}, threshold, "Threshold for the heuristic inliner")
 
 // clang-format off
 enum class InlineCostFeatureIndex : size_t {
-#define POPULATE_INDICES(INDEX_NAME, NAME) INDEX_NAME,
+#define POPULATE_INDICES(DTYPE, SHAPE, NAME, DOC) NAME,
   INLINE_COST_FEATURE_ITERATOR(POPULATE_INDICES)
 #undef POPULATE_INDICES
 
@@ -62,15 +79,15 @@ using InlineCostFeatures =
                static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures)>;
 
 constexpr bool isHeuristicInlineCostFeature(InlineCostFeatureIndex Feature) {
-  return Feature != InlineCostFeatureIndex::SROASavings &&
-         Feature != InlineCostFeatureIndex::IsMultipleBlocks &&
-         Feature != InlineCostFeatureIndex::DeadBlocks &&
-         Feature != InlineCostFeatureIndex::SimplifiedInstructions &&
-         Feature != InlineCostFeatureIndex::ConstantArgs &&
-         Feature != InlineCostFeatureIndex::ConstantOffsetPtrArgs &&
-         Feature != InlineCostFeatureIndex::NestedInlines &&
-         Feature != InlineCostFeatureIndex::NestedInlineCostEstimate &&
-         Feature != InlineCostFeatureIndex::Threshold;
+  return Feature != InlineCostFeatureIndex::sroa_savings &&
+         Feature != InlineCostFeatureIndex::is_multiple_blocks &&
+         Feature != InlineCostFeatureIndex::dead_blocks &&
+         Feature != InlineCostFeatureIndex::simplified_instructions &&
+         Feature != InlineCostFeatureIndex::constant_args &&
+         Feature != InlineCostFeatureIndex::constant_offset_ptr_args &&
+         Feature != InlineCostFeatureIndex::nested_inlines &&
+         Feature != InlineCostFeatureIndex::nested_inline_cost_estimate &&
+         Feature != InlineCostFeatureIndex::threshold;
 }
 
 // List of features. Each feature is defined through a triple:
@@ -81,39 +98,37 @@ constexpr bool isHeuristicInlineCostFeature(InlineCostFeatureIndex Feature) {
 // programmatically, and serves as workaround to inability of inserting comments
 // in macros.
 #define INLINE_FEATURE_ITERATOR(M)                                             \
-  M(CalleeBasicBlockCount, "callee_basic_block_count",                         \
+  M(int64_t, {1}, callee_basic_block_count,                                    \
     "number of basic blocks of the callee")                                    \
-  M(CallSiteHeight, "callsite_height",                                         \
+  M(int64_t, {1}, callsite_height,                                             \
     "position of the call site in the original call graph - measured from "    \
     "the farthest SCC")                                                        \
-  M(NodeCount, "node_count",                                                   \
+  M(int64_t, {1}, node_count,                                                  \
     "total current number of defined functions in the module")                 \
-  M(NrCtantParams, "nr_ctant_params",                                          \
+  M(int64_t, {1}, nr_ctant_params,                                             \
     "number of parameters in the call site that are constants")                \
-  M(CostEstimate, "cost_estimate", "total cost estimate (threshold - free)")   \
-  M(EdgeCount, "edge_count", "total number of calls in the module")            \
-  M(CallerUsers, "caller_users",                                               \
+  M(int64_t, {1}, cost_estimate, "total cost estimate (threshold - free)")     \
+  M(int64_t, {1}, edge_count, "total number of calls in the module")           \
+  M(int64_t, {1}, caller_users,                                                \
     "number of module-internal users of the caller, +1 if the caller is "      \
     "exposed externally")                                                      \
-  M(CallerConditionallyExecutedBlocks, "caller_conditionally_executed_blocks", \
+  M(int64_t, {1}, caller_conditionally_executed_blocks,                        \
     "number of blocks reached from a conditional instruction, in the caller")  \
-  M(CallerBasicBlockCount, "caller_basic_block_count",                         \
+  M(int64_t, {1}, caller_basic_block_count,                                    \
     "number of basic blocks in the caller")                                    \
-  M(CalleeConditionallyExecutedBlocks, "callee_conditionally_executed_blocks", \
+  M(int64_t, {1}, callee_conditionally_executed_blocks,                        \
     "number of blocks reached from a conditional instruction, in the callee")  \
-  M(CalleeUsers, "callee_users",                                               \
+  M(int64_t, {1}, callee_users,                                                \
     "number of module-internal users of the callee, +1 if the callee is "      \
     "exposed externally")
 
 // clang-format off
 enum class FeatureIndex : size_t {
+#define POPULATE_INDICES(DTYPE, SHAPE, NAME, COMMENT) NAME,
 // InlineCost features - these must come first
-#define POPULATE_INDICES(INDEX_NAME, NAME) INDEX_NAME,
   INLINE_COST_FEATURE_ITERATOR(POPULATE_INDICES)
-#undef POPULATE_INDICES
 
 // Non-cost features
-#define POPULATE_INDICES(INDEX_NAME, NAME, COMMENT) INDEX_NAME,
   INLINE_FEATURE_ITERATOR(POPULATE_INDICES)
 #undef POPULATE_INDICES
 
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 759619396ccb8..fae22d484a09b 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -730,6 +730,10 @@ const SCEV *replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
 /// to \p PtrToStride and therefore add further predicates to \p PSE.
 /// The \p Assume parameter indicates if we are allowed to make additional
 /// run-time assumptions.
+///
+/// Note that the analysis results are defined if-and-only-if the original
+/// memory access was defined.  If that access was dead, or UB, then the
+/// result of this function is undefined.
 std::optional<int64_t>
 getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
              const Loop *Lp,
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index 763d7f6a8e717..3434630c27cfe 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -164,7 +164,7 @@ class LLVM_EXTERNAL_VISIBILITY Loop : public LoopBase<BasicBlock, Loop> {
     /// - the step instruction of the induction variable can be found
     /// - the final value of the induction variable can be found
     ///
-    /// Else None.
+    /// Else std::nullopt.
     static std::optional<Loop::LoopBounds>
     getBounds(const Loop &L, PHINode &IndVar, ScalarEvolution &SE);
 
diff --git a/llvm/include/llvm/Analysis/MemoryProfileInfo.h b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
index 9a97c76a4dd04..bbade1b337916 100644
--- a/llvm/include/llvm/Analysis/MemoryProfileInfo.h
+++ b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
@@ -24,8 +24,8 @@ namespace llvm {
 namespace memprof {
 
 /// Return the allocation type for a given set of memory profile values.
-AllocationType getAllocType(uint64_t MaxAccessCount, uint64_t MinSize,
-                            uint64_t MinLifetime);
+AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
+                            uint64_t AllocCount, uint64_t TotalLifetime);
 
 /// Build callstack metadata from the provided list of call stack ids. Returns
 /// the resulting metadata node.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index f27cf22639c96..0f281d09eefea 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -791,16 +791,19 @@ class ScalarEvolution {
   bool isLoopBackedgeGuardedByCond(const Loop *L, ICmpInst::Predicate Pred,
                                    const SCEV *LHS, const SCEV *RHS);
 
+  /// A version of getTripCountFromExitCount below which always picks an
+  /// evaluation type which can not result in overflow.
+  const SCEV *getTripCountFromExitCount(const SCEV *ExitCount);
+
   /// Convert from an "exit count" (i.e. "backedge taken count") to a "trip
   /// count".  A "trip count" is the number of times the header of the loop
   /// will execute if an exit is taken after the specified number of backedges
   /// have been taken.  (e.g. TripCount = ExitCount + 1).  Note that the
-  /// expression can overflow if ExitCount = UINT_MAX.  \p Extend controls
-  /// how potential overflow is handled.  If true, a wider result type is
-  /// returned. ex: EC = 255 (i8), TC = 256 (i9).  If false, result unsigned
-  /// wraps with 2s-complement semantics.  ex: EC = 255 (i8), TC = 0 (i8)
-  const SCEV *getTripCountFromExitCount(const SCEV *ExitCount,
-                                        bool Extend = true);
+  /// expression can overflow if ExitCount = UINT_MAX.  If EvalTy is not wide
+  /// enough to hold the result without overflow, result unsigned wraps with
+  /// 2s-complement semantics.  ex: EC = 255 (i8), TC = 0 (i8)
+  const SCEV *getTripCountFromExitCount(const SCEV *ExitCount, Type *EvalTy,
+                                        const Loop *L);
 
   /// Returns the exact trip count of the loop if we can compute it, and
   /// the result is a small constant.  '0' is used to represent an unknown
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 0a1c900c3954b..91848a91c17e6 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -752,7 +752,7 @@ class SCEVRewriteVisitor : public SCEVVisitor<SC, const SCEV *> {
   // a SCEV is referenced by multiple SCEVs. Without memoization, this
   // visit algorithm would have exponential time complexity in the worst
   // case, causing the compiler to hang on certain tests.
-  DenseMap<const SCEV *, const SCEV *> RewriteResults;
+  SmallDenseMap<const SCEV *, const SCEV *> RewriteResults;
 
 public:
   SCEVRewriteVisitor(ScalarEvolution &SE) : SE(SE) {}
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index 72a32a5b37d62..03ac422d3e6b7 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -256,14 +256,14 @@ TLI_DEFINE_ENUM_INTERNAL(Znam)
 TLI_DEFINE_STRING_INTERNAL("_Znam")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long)
 
-/// void *operator new[](unsigned long, hot_cold_t)
-/// Currently this and other operator new interfaces that take a hot_cold_t
+/// void *operator new[](unsigned long, __hot_cold_t)
+/// Currently this and other operator new interfaces that take a __hot_cold_t
 /// hint are supported by the open source version of tcmalloc, see:
 /// https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
-/// and for the definition of the hot_cold_t parameter see:
+/// and for the definition of the __hot_cold_t parameter see:
 /// https://github.com/google/tcmalloc/blob/master/tcmalloc/malloc_extension.h
-TLI_DEFINE_ENUM_INTERNAL(Znam10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_Znam10hot_cold_t")
+TLI_DEFINE_ENUM_INTERNAL(Znam12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_Znam12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Bool)
 
 /// void *operator new[](unsigned long, const std::nothrow_t&);
@@ -271,9 +271,9 @@ TLI_DEFINE_ENUM_INTERNAL(ZnamRKSt9nothrow_t)
 TLI_DEFINE_STRING_INTERNAL("_ZnamRKSt9nothrow_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Ptr)
 
-/// void *operator new[](unsigned long, const std::nothrow_t&, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(ZnamRKSt9nothrow_t10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_ZnamRKSt9nothrow_t10hot_cold_t")
+/// void *operator new[](unsigned long, const std::nothrow_t&, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(ZnamRKSt9nothrow_t12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_ZnamRKSt9nothrow_t12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Ptr, Bool)
 
 /// void *operator new[](unsigned long, std::align_val_t)
@@ -281,9 +281,9 @@ TLI_DEFINE_ENUM_INTERNAL(ZnamSt11align_val_t)
 TLI_DEFINE_STRING_INTERNAL("_ZnamSt11align_val_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long)
 
-/// void *operator new[](unsigned long, std::align_val_t, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(ZnamSt11align_val_t10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_ZnamSt11align_val_t10hot_cold_t")
+/// void *operator new[](unsigned long, std::align_val_t, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(ZnamSt11align_val_t12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_ZnamSt11align_val_t12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long, Bool)
 
 /// void *operator new[](unsigned long, std::align_val_t, const std::nothrow_t&)
@@ -291,9 +291,9 @@ TLI_DEFINE_ENUM_INTERNAL(ZnamSt11align_val_tRKSt9nothrow_t)
 TLI_DEFINE_STRING_INTERNAL("_ZnamSt11align_val_tRKSt9nothrow_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long, Ptr)
 
-/// void *operator new[](unsigned long, std::align_val_t, const std::nothrow_t&, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t")
+/// void *operator new[](unsigned long, std::align_val_t, const std::nothrow_t&, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long, Ptr, Bool)
 
 /// void *operator new(unsigned int);
@@ -321,9 +321,9 @@ TLI_DEFINE_ENUM_INTERNAL(Znwm)
 TLI_DEFINE_STRING_INTERNAL("_Znwm")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long)
 
-/// void *operator new(unsigned long, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(Znwm10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_Znwm10hot_cold_t")
+/// void *operator new(unsigned long, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(Znwm12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_Znwm12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Bool)
 
 /// void *operator new(unsigned long, const std::nothrow_t&);
@@ -331,9 +331,9 @@ TLI_DEFINE_ENUM_INTERNAL(ZnwmRKSt9nothrow_t)
 TLI_DEFINE_STRING_INTERNAL("_ZnwmRKSt9nothrow_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Ptr)
 
-/// void *operator new(unsigned long, const std::nothrow_t&, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(ZnwmRKSt9nothrow_t10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_ZnwmRKSt9nothrow_t10hot_cold_t")
+/// void *operator new(unsigned long, const std::nothrow_t&, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(ZnwmRKSt9nothrow_t12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_ZnwmRKSt9nothrow_t12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Ptr, Bool)
 
 /// void *operator new(unsigned long, std::align_val_t)
@@ -341,9 +341,9 @@ TLI_DEFINE_ENUM_INTERNAL(ZnwmSt11align_val_t)
 TLI_DEFINE_STRING_INTERNAL("_ZnwmSt11align_val_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long)
 
-/// void *operator new(unsigned long, std::align_val_t, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(ZnwmSt11align_val_t10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_ZnwmSt11align_val_t10hot_cold_t")
+/// void *operator new(unsigned long, std::align_val_t, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(ZnwmSt11align_val_t12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_ZnwmSt11align_val_t12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long, Bool)
 
 /// void *operator new(unsigned long, std::align_val_t, const std::nothrow_t&)
@@ -351,9 +351,9 @@ TLI_DEFINE_ENUM_INTERNAL(ZnwmSt11align_val_tRKSt9nothrow_t)
 TLI_DEFINE_STRING_INTERNAL("_ZnwmSt11align_val_tRKSt9nothrow_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long, Ptr)
 
-/// void *operator new(unsigned long, std::align_val_t, const std::nothrow_t&, hot_cold_t)
-TLI_DEFINE_ENUM_INTERNAL(ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t)
-TLI_DEFINE_STRING_INTERNAL("_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t")
+/// void *operator new(unsigned long, std::align_val_t, const std::nothrow_t&, __hot_cold_t)
+TLI_DEFINE_ENUM_INTERNAL(ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t)
+TLI_DEFINE_STRING_INTERNAL("_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t")
 TLI_DEFINE_SIG_INTERNAL(Ptr, Long, Long, Ptr, Bool)
 
 /// double __acos_finite(double x);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index db5ebb7c30b2f..131e1e3f4b91b 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1269,7 +1269,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
           APInt DemandedDstElts =
               APInt::getZero(Shuffle->getShuffleMask().size());
           for (auto I : enumerate(Shuffle->getShuffleMask())) {
-            if (I.value() != UndefMaskElem)
+            if (I.value() != PoisonMaskElem)
               DemandedDstElts.setBit(I.index());
           }
           return TargetTTI->getReplicationShuffleCost(
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index ccf0ccf36ded9..9630a7bc24ca6 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -238,6 +238,10 @@ struct KnownFPClass {
   /// definitely set or false if the sign bit is definitely unset.
   std::optional<bool> SignBit;
 
+  /// Return true if it's known this can never be one of the mask entries.
+  bool isKnownNever(FPClassTest Mask) const {
+    return (KnownFPClasses & Mask) == fcNone;
+  }
 
   bool isUnknown() const {
     return KnownFPClasses == fcAllFlags && !SignBit;
@@ -245,33 +249,43 @@ struct KnownFPClass {
 
   /// Return true if it's known this can never be a nan.
   bool isKnownNeverNaN() const {
-    return (KnownFPClasses & fcNan) == fcNone;
+    return isKnownNever(fcNan);
   }
 
   /// Return true if it's known this can never be an infinity.
   bool isKnownNeverInfinity() const {
-    return (KnownFPClasses & fcInf) == fcNone;
+    return isKnownNever(fcInf);
   }
 
   /// Return true if it's known this can never be +infinity.
   bool isKnownNeverPosInfinity() const {
-    return (KnownFPClasses & fcPosInf) == fcNone;
+    return isKnownNever(fcPosInf);
   }
 
   /// Return true if it's known this can never be -infinity.
   bool isKnownNeverNegInfinity() const {
-    return (KnownFPClasses & fcNegInf) == fcNone;
+    return isKnownNever(fcNegInf);
   }
 
   /// Return true if it's known this can never be a subnormal
   bool isKnownNeverSubnormal() const {
-    return (KnownFPClasses & fcSubnormal) == fcNone;
+    return isKnownNever(fcSubnormal);
+  }
+
+  /// Return true if it's known this can never be a negativesubnormal
+  bool isKnownNeverNegSubnormal() const {
+    return isKnownNever(fcNegSubnormal);
   }
 
   /// Return true if it's known this can never be a zero. This means a literal
   /// [+-]0, and does not include denormal inputs implicitly treated as [+-]0.
   bool isKnownNeverZero() const {
-    return (KnownFPClasses & fcZero) == fcNone;
+    return isKnownNever(fcZero);
+  }
+
+  /// Return true if it's known this can never be a literal negative zero.
+  bool isKnownNeverNegZero() const {
+    return isKnownNever(fcNegZero);
   }
 
   /// Return true if it's know this can never be interpreted as a zero. This
@@ -279,6 +293,11 @@ struct KnownFPClass {
   /// floating-point mode for the function interprets denormals as zero.
   bool isKnownNeverLogicalZero(const Function &F, Type *Ty) const;
 
+  static constexpr FPClassTest OrderedLessThanZeroMask =
+      fcNegSubnormal | fcNegNormal | fcNegInf;
+  static constexpr FPClassTest OrderedGreaterThanZeroMask =
+      fcPosSubnormal | fcPosNormal | fcPosInf;
+
   /// Return true if we can prove that the analyzed floating-point value is
   /// either NaN or never less than -0.0.
   ///
@@ -288,8 +307,18 @@ struct KnownFPClass {
   ///   x > +0 --> true
   ///   x < -0 --> false
   bool cannotBeOrderedLessThanZero() const {
-    const FPClassTest OrderedNegMask = fcNegSubnormal | fcNegNormal | fcNegInf;
-    return (KnownFPClasses & OrderedNegMask) == fcNone;
+  return isKnownNever(OrderedLessThanZeroMask);
+  }
+
+  /// Return true if we can prove that the analyzed floating-point value is
+  /// either NaN or never greater than -0.0.
+  ///      NaN --> true
+  ///       +0 --> true
+  ///       -0 --> true
+  ///   x > +0 --> false
+  ///   x < -0 --> true
+  bool cannotBeOrderedGreaterThanZero() const {
+    return isKnownNever(OrderedGreaterThanZeroMask);
   }
 
   KnownFPClass &operator|=(const KnownFPClass &RHS) {
@@ -311,27 +340,41 @@ struct KnownFPClass {
   }
 
   void fabs() {
-    KnownFPClasses = llvm::fabs(KnownFPClasses);
+    KnownFPClasses &= (fcPositive | fcNan);
     SignBit = false;
   }
 
+  /// Return true if the sign bit must be 0, ignoring the sign of nans.
+  bool signBitIsZeroOrNaN() const {
+    return isKnownNever(fcNegative);
+  }
+
   /// Assume the sign bit is zero.
-  void signBitIsZero() {
-    KnownFPClasses = (KnownFPClasses & fcPositive) |
-                     (KnownFPClasses & fcNan);
+  void signBitMustBeZero() {
+    KnownFPClasses &= (fcPositive | fcNan);
     SignBit = false;
   }
 
   void copysign(const KnownFPClass &Sign) {
-    // Start assuming nothing about the sign.
+    // Don't know anything about the sign of the source. Expand the possible set
+    // to its opposite sign pair.
+    if (KnownFPClasses & fcZero)
+      KnownFPClasses |= fcZero;
+    if (KnownFPClasses & fcSubnormal)
+      KnownFPClasses |= fcSubnormal;
+    if (KnownFPClasses & fcNormal)
+      KnownFPClasses |= fcNormal;
+    if (KnownFPClasses & fcInf)
+      KnownFPClasses |= fcInf;
+
+    // Sign bit is exactly preserved even for nans.
     SignBit = Sign.SignBit;
-    if (!SignBit)
-      return;
 
-    if (*SignBit)
-      KnownFPClasses = KnownFPClasses & fcNegative;
-    else
-      KnownFPClasses = KnownFPClasses & fcPositive;
+    // Clear sign bits based on the input sign mask.
+    if (Sign.isKnownNever(fcPositive | fcNan) || (SignBit && *SignBit))
+      KnownFPClasses &= (fcNegative | fcNan);
+    if (Sign.isKnownNever(fcNegative | fcNan) || (SignBit && !*SignBit))
+      KnownFPClasses &= (fcPositive | fcNan);
   }
 
   void resetAll() { *this = KnownFPClass(); }
@@ -392,6 +435,17 @@ bool CannotBeOrderedLessThanZero(const Value *V, const TargetLibraryInfo *TLI);
 bool isKnownNeverInfinity(const Value *V, const TargetLibraryInfo *TLI,
                           unsigned Depth = 0);
 
+/// Return true if the floating-point value can never contain a NaN or infinity.
+inline bool isKnownNeverInfOrNaN(
+    const Value *V, const DataLayout &DL, const TargetLibraryInfo *TLI,
+    unsigned Depth = 0, AssumptionCache *AC = nullptr,
+    const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr,
+    OptimizationRemarkEmitter *ORE = nullptr, bool UseInstrInfo = true) {
+  KnownFPClass Known = computeKnownFPClass(V, DL, fcInf | fcNan, Depth, TLI, AC,
+                                           CtxI, DT, ORE, UseInstrInfo);
+  return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity();
+}
+
 /// Return true if the floating-point scalar value is not a NaN or if the
 /// floating-point vector value has no NaN elements. Return false if a value
 /// could ever be NaN.
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 4f841c5f55b12..38e6170b6b810 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -434,7 +434,6 @@ enum Kind {
   kw_versions,
   kw_memProf,
   kw_notcold,
-  kw_notcoldandcold,
 
   // GV's with __attribute__((no_sanitize("address"))), or things in
   // -fsanitize-ignorelist when built with ASan.
diff --git a/llvm/include/llvm/BinaryFormat/GOFF.h b/llvm/include/llvm/BinaryFormat/GOFF.h
index 96992414c6ccf..b4ddbabdf1e4e 100644
--- a/llvm/include/llvm/BinaryFormat/GOFF.h
+++ b/llvm/include/llvm/BinaryFormat/GOFF.h
@@ -9,7 +9,8 @@
 // This header contains common, non-processor-specific data structures and
 // constants for the GOFF file format.
 //
-// GOFF specifics can be found in MVS Program Management: Advanced Facilities
+// GOFF specifics can be found in MVS Program Management: Advanced Facilities.
+//
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_BINARYFORMAT_GOFF_H
@@ -18,14 +19,142 @@
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
-
 namespace GOFF {
 
+constexpr uint8_t RecordLength = 80;
+constexpr uint8_t RecordPrefixLength = 3;
+constexpr uint8_t PayloadLength = 77;
+
+// Prefix byte on every record. This indicates GOFF format.
+constexpr uint8_t PTVPrefix = 0x03;
+
+enum RecordType : uint8_t {
+  RT_ESD = 0,
+  RT_TXT = 1,
+  RT_RLD = 2,
+  RT_LEN = 3,
+  RT_END = 4,
+  RT_HDR = 15,
+};
+
+enum ESDSymbolType : uint8_t {
+  ESD_ST_SectionDefinition = 0,
+  ESD_ST_ElementDefinition = 1,
+  ESD_ST_LabelDefinition = 2,
+  ESD_ST_PartReference = 3,
+  ESD_ST_ExternalReference = 4,
+};
+
+enum ESDNameSpaceId : uint8_t {
+  ESD_NS_ProgramManagementBinder = 0,
+  ESD_NS_NormalName = 1,
+  ESD_NS_PseudoRegister = 2,
+  ESD_NS_Parts = 3
+};
+
+enum ESDReserveQwords : uint8_t {
+  ESD_RQ_0 = 0,
+  ESD_RQ_1 = 1,
+  ESD_RQ_2 = 2,
+  ESD_RQ_3 = 3
+};
+
+enum ESDAmode : uint8_t {
+  ESD_AMODE_None = 0,
+  ESD_AMODE_24 = 1,
+  ESD_AMODE_31 = 2,
+  ESD_AMODE_ANY = 3,
+  ESD_AMODE_64 = 4,
+  ESD_AMODE_MIN = 16,
+};
+
+enum ESDRmode : uint8_t {
+  ESD_RMODE_None = 0,
+  ESD_RMODE_24 = 1,
+  ESD_RMODE_31 = 3,
+  ESD_RMODE_64 = 4,
+};
+
+enum ESDTextStyle : uint8_t {
+  ESD_TS_ByteOriented = 0,
+  ESD_TS_Structured = 1,
+  ESD_TS_Unstructured = 2,
+};
+
+enum ESDBindingAlgorithm : uint8_t {
+  ESD_BA_Concatenate = 0,
+  ESD_BA_Merge = 1,
+};
+
+enum ESDTaskingBehavior : uint8_t {
+  ESD_TA_Unspecified = 0,
+  ESD_TA_NonReus = 1,
+  ESD_TA_Reus = 2,
+  ESD_TA_Rent = 3,
+};
+
+enum ESDExecutable : uint8_t {
+  ESD_EXE_Unspecified = 0,
+  ESD_EXE_DATA = 1,
+  ESD_EXE_CODE = 2,
+};
+
+enum ESDDuplicateSymbolSeverity : uint8_t {
+  ESD_DSS_NoWarning = 0,
+  ESD_DSS_Warning = 1,
+  ESD_DSS_Error = 2,
+  ESD_DSS_Reserved = 3,
+};
+
+enum ESDBindingStrength : uint8_t {
+  ESD_BST_Strong = 0,
+  ESD_BST_Weak = 1,
+};
+
+enum ESDLoadingBehavior : uint8_t {
+  ESD_LB_Initial = 0,
+  ESD_LB_Deferred = 1,
+  ESD_LB_NoLoad = 2,
+  ESD_LB_Reserved = 3,
+};
+
+enum ESDBindingScope : uint8_t {
+  ESD_BSC_Unspecified = 0,
+  ESD_BSC_Section = 1,
+  ESD_BSC_Module = 2,
+  ESD_BSC_Library = 3,
+  ESD_BSC_ImportExport = 4,
+};
+
+enum ESDLinkageType : uint8_t { ESD_LT_OS = 0, ESD_LT_XPLink = 1 };
+
+enum ESDAlignment : uint8_t {
+  ESD_ALIGN_Byte = 0,
+  ESD_ALIGN_Halfword = 1,
+  ESD_ALIGN_Fullword = 2,
+  ESD_ALIGN_Doubleword = 3,
+  ESD_ALIGN_Quadword = 4,
+  ESD_ALIGN_32byte = 5,
+  ESD_ALIGN_64byte = 6,
+  ESD_ALIGN_128byte = 7,
+  ESD_ALIGN_256byte = 8,
+  ESD_ALIGN_512byte = 9,
+  ESD_ALIGN_1024byte = 10,
+  ESD_ALIGN_2Kpage = 11,
+  ESD_ALIGN_4Kpage = 12,
+};
+
+enum ENDEntryPointRequest : uint8_t {
+  END_EPR_None = 0,
+  END_EPR_EsdidOffset = 1,
+  END_EPR_ExternalName = 2,
+  END_EPR_Reserved = 3,
+};
+
 // \brief Subsections of the primary C_CODE section in the object file.
 enum SubsectionKind : uint8_t {
   SK_PPA1 = 2,
 };
-
 } // end namespace GOFF
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CMakeLists.txt b/llvm/include/llvm/CMakeLists.txt
index d1dc0a99463ba..ac6b96a68ab92 100644
--- a/llvm/include/llvm/CMakeLists.txt
+++ b/llvm/include/llvm/CMakeLists.txt
@@ -1,10 +1,9 @@
+# Use LLVM_HEADERS for each `tablegen(LLVM)` in subdirectories
+# LLVM_HEADERS_TABLEGEN points to `llvm-min-tblgen`
+set(LLVM_TABLEGEN_PROJECT LLVM_HEADERS)
+
+add_subdirectory(CodeGen)
 add_subdirectory(IR)
 add_subdirectory(Support)
 add_subdirectory(Frontend)
 add_subdirectory(TargetParser)
-
-# If we're doing an out-of-tree build, copy a module map for generated
-# header files into the build area.
-if (NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
-  configure_file(module.modulemap.build module.modulemap COPYONLY)
-endif (NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 04d63db4cf231..1a5e94b8f1085 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -26,6 +26,7 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TargetTransformInfoImpl.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -44,7 +45,6 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
diff --git a/llvm/include/llvm/CodeGen/CMakeLists.txt b/llvm/include/llvm/CodeGen/CMakeLists.txt
new file mode 100644
index 0000000000000..652e1baed1e7c
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/CMakeLists.txt
@@ -0,0 +1,3 @@
+set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/include/llvm/CodeGen/ValueTypes.td)
+tablegen(LLVM GenVT.inc -gen-vt)
+add_public_tablegen_target(vt_gen)
diff --git a/llvm/include/llvm/CodeGen/CostTable.h b/llvm/include/llvm/CodeGen/CostTable.h
index d23f79d94a7aa..ca0cbdda3b114 100644
--- a/llvm/include/llvm/CodeGen/CostTable.h
+++ b/llvm/include/llvm/CodeGen/CostTable.h
@@ -16,7 +16,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/MachineValueType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/CodeGen/FastISel.h b/llvm/include/llvm/CodeGen/FastISel.h
index d25776e5eff16..dc2931b40d352 100644
--- a/llvm/include/llvm/CodeGen/FastISel.h
+++ b/llvm/include/llvm/CodeGen/FastISel.h
@@ -19,13 +19,13 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/InstrTypes.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cstdint>
 #include <utility>
 
diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
index e880b6227ef0b..4c17e8dcc41a6 100644
--- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -35,6 +35,7 @@ namespace llvm {
 class Argument;
 class BasicBlock;
 class BranchProbabilityInfo;
+class DbgDeclareInst;
 class Function;
 class Instruction;
 class MachineFunction;
@@ -187,6 +188,10 @@ class FunctionLoweringInfo {
   /// SelectionDAGISel::PrepareEHLandingPad().
   unsigned ExceptionPointerVirtReg, ExceptionSelectorVirtReg;
 
+  /// Collection of dbg.declare instructions handled after argument
+  /// lowering and before ISel proper.
+  SmallPtrSet<const DbgDeclareInst *, 8> PreprocessedDbgDeclares;
+
   /// set - Initialize this FunctionLoweringInfo with the given Function
   /// and its associated MachineFunction.
   ///
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index de2d6876e6b7b..e0a72a21f425b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -17,14 +17,14 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cstdint>
 #include <functional>
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 1b268933618f7..f6c0764a6426c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -19,8 +19,8 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/Register.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/IR/InstrTypes.h"
 #include <functional>
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
index 8954d5cdcc377..fda84080636b6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
@@ -18,9 +18,9 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/IR/Function.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include <bitset>
 #include <cstddef>
 #include <cstdint>
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h
index 3cacdc99dbf8b..08233dba20411 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h
@@ -16,8 +16,8 @@
 #define LLVM_CODEGEN_GLOBALISEL_LEGACYLEGALIZERINFO_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include <unordered_map>
 
 namespace llvm {
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 50b11a4920bbd..569b86cca6353 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -17,12 +17,12 @@
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include <cassert>
 #include <cstdint>
 #include <tuple>
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index c9941afc8013f..d5c1fd8d0d512 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -17,11 +17,11 @@
 #include "GISelWorkList.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include <cstdint>
 
 namespace llvm {
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 9120d97e3cfb4..b49066d3751c4 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -263,9 +263,9 @@ enum NodeType {
   /// These nodes take two operands of the same value type, and produce two
   /// results.  The first result is the normal add or sub result, the second
   /// result is the carry flag result.
-  /// FIXME: These nodes are deprecated in favor of ADDCARRY and SUBCARRY.
+  /// FIXME: These nodes are deprecated in favor of UADDO_CARRY and USUBO_CARRY.
   /// They are kept around for now to provide a smooth transition path
-  /// toward the use of ADDCARRY/SUBCARRY and will eventually be removed.
+  /// toward the use of UADDO_CARRY/USUBO_CARRY and will eventually be removed.
   ADDC,
   SUBC,
 
@@ -297,11 +297,11 @@ enum NodeType {
   /// it, as the carry is a regular value rather than a glue, which allows
   /// further optimisation.
   ///
-  /// These opcodes are different from [US]{ADD,SUB}O in that ADDCARRY/SUBCARRY
-  /// consume and produce a carry/borrow, whereas [US]{ADD,SUB}O produce an
-  /// overflow.
-  ADDCARRY,
-  SUBCARRY,
+  /// These opcodes are different from [US]{ADD,SUB}O in that
+  /// U{ADD,SUB}O_CARRY consume and produce a carry/borrow, whereas
+  /// [US]{ADD,SUB}O produce an overflow.
+  UADDO_CARRY,
+  USUBO_CARRY,
 
   /// Carry-using overflow-aware nodes for multiple precision addition and
   /// subtraction. These nodes take three operands: The first two are normal lhs
@@ -752,7 +752,7 @@ enum NodeType {
   /// op #2 is a boolean indicating if there is an incoming carry. This
   /// operator checks the result of "LHS - RHS - Carry", and can be used to
   /// compare two wide integers:
-  /// (setcccarry lhshi rhshi (subcarry lhslo rhslo) cc).
+  /// (setcccarry lhshi rhshi (usubo_carry lhslo rhslo) cc).
   /// Only valid for integers.
   SETCCCARRY,
 
diff --git a/llvm/include/llvm/CodeGen/LowLevelType.h b/llvm/include/llvm/CodeGen/LowLevelType.h
index 922f93d2e5985..2924f475ac850 100644
--- a/llvm/include/llvm/CodeGen/LowLevelType.h
+++ b/llvm/include/llvm/CodeGen/LowLevelType.h
@@ -9,37 +9,420 @@
 /// Implement a low-level type suitable for MachineInstr level instruction
 /// selection.
 ///
-/// This provides the CodeGen aspects of LowLevelType, such as Type conversion.
+/// For a type attached to a MachineInstr, we only care about 2 details: total
+/// size and the number of vector lanes (if any). Accordingly, there are 4
+/// possible valid type-kinds:
+///
+///    * `sN` for scalars and aggregates
+///    * `<N x sM>` for vectors, which must have at least 2 elements.
+///    * `pN` for pointers
+///
+/// Other information required for correct selection is expected to be carried
+/// by the opcode, or non-type flags. For example the distinction between G_ADD
+/// and G_FADD for int/float or fast-math flags.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CODEGEN_LOWLEVELTYPE_H
 #define LLVM_CODEGEN_LOWLEVELTYPE_H
 
-#include "llvm/CodeGen/ValueTypes.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
+#include "llvm/Support/Debug.h"
+#include <cassert>
 
 namespace llvm {
 
-class DataLayout;
 class Type;
-struct fltSemantics;
+class raw_ostream;
+
+class LLT {
+public:
+  /// Get a low-level scalar or aggregate "bag of bits".
+  static constexpr LLT scalar(unsigned SizeInBits) {
+    return LLT{/*isPointer=*/false, /*isVector=*/false, /*isScalar=*/true,
+               ElementCount::getFixed(0), SizeInBits,
+               /*AddressSpace=*/0};
+  }
+
+  /// Get a low-level pointer in the given address space.
+  static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits) {
+    assert(SizeInBits > 0 && "invalid pointer size");
+    return LLT{/*isPointer=*/true, /*isVector=*/false, /*isScalar=*/false,
+               ElementCount::getFixed(0), SizeInBits, AddressSpace};
+  }
+
+  /// Get a low-level vector of some number of elements and element width.
+  static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits) {
+    assert(!EC.isScalar() && "invalid number of vector elements");
+    return LLT{/*isPointer=*/false, /*isVector=*/true, /*isScalar=*/false,
+               EC, ScalarSizeInBits, /*AddressSpace=*/0};
+  }
+
+  /// Get a low-level vector of some number of elements and element type.
+  static constexpr LLT vector(ElementCount EC, LLT ScalarTy) {
+    assert(!EC.isScalar() && "invalid number of vector elements");
+    assert(!ScalarTy.isVector() && "invalid vector element type");
+    return LLT{ScalarTy.isPointer(),
+               /*isVector=*/true,
+               /*isScalar=*/false,
+               EC,
+               ScalarTy.getSizeInBits().getFixedValue(),
+               ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0};
+  }
+
+  /// Get a low-level fixed-width vector of some number of elements and element
+  /// width.
+  static constexpr LLT fixed_vector(unsigned NumElements,
+                                    unsigned ScalarSizeInBits) {
+    return vector(ElementCount::getFixed(NumElements), ScalarSizeInBits);
+  }
+
+  /// Get a low-level fixed-width vector of some number of elements and element
+  /// type.
+  static constexpr LLT fixed_vector(unsigned NumElements, LLT ScalarTy) {
+    return vector(ElementCount::getFixed(NumElements), ScalarTy);
+  }
+
+  /// Get a low-level scalable vector of some number of elements and element
+  /// width.
+  static constexpr LLT scalable_vector(unsigned MinNumElements,
+                                       unsigned ScalarSizeInBits) {
+    return vector(ElementCount::getScalable(MinNumElements), ScalarSizeInBits);
+  }
+
+  /// Get a low-level scalable vector of some number of elements and element
+  /// type.
+  static constexpr LLT scalable_vector(unsigned MinNumElements, LLT ScalarTy) {
+    return vector(ElementCount::getScalable(MinNumElements), ScalarTy);
+  }
+
+  static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy) {
+    return EC.isScalar() ? ScalarTy : LLT::vector(EC, ScalarTy);
+  }
+
+  static constexpr LLT scalarOrVector(ElementCount EC, uint64_t ScalarSize) {
+    assert(ScalarSize <= std::numeric_limits<unsigned>::max() &&
+           "Not enough bits in LLT to represent size");
+    return scalarOrVector(EC, LLT::scalar(static_cast<unsigned>(ScalarSize)));
+  }
+
+  explicit constexpr LLT(bool isPointer, bool isVector, bool isScalar,
+                         ElementCount EC, uint64_t SizeInBits,
+                         unsigned AddressSpace)
+      : LLT() {
+    init(isPointer, isVector, isScalar, EC, SizeInBits, AddressSpace);
+  }
+  explicit constexpr LLT()
+      : IsScalar(false), IsPointer(false), IsVector(false), RawData(0) {}
+
+  explicit LLT(MVT VT);
+
+  constexpr bool isValid() const { return IsScalar || RawData != 0; }
+
+  constexpr bool isScalar() const { return IsScalar; }
+
+  constexpr bool isPointer() const {
+    return isValid() && IsPointer && !IsVector;
+  }
+
+  constexpr bool isVector() const { return isValid() && IsVector; }
+
+  /// Returns the number of elements in a vector LLT. Must only be called on
+  /// vector types.
+  constexpr uint16_t getNumElements() const {
+    if (isScalable())
+      llvm::reportInvalidSizeRequest(
+          "Possible incorrect use of LLT::getNumElements() for "
+          "scalable vector. Scalable flag may be dropped, use "
+          "LLT::getElementCount() instead");
+    return getElementCount().getKnownMinValue();
+  }
+
+  /// Returns true if the LLT is a scalable vector. Must only be called on
+  /// vector types.
+  constexpr bool isScalable() const {
+    assert(isVector() && "Expected a vector type");
+    return IsPointer ? getFieldValue(PointerVectorScalableFieldInfo)
+                     : getFieldValue(VectorScalableFieldInfo);
+  }
+
+  constexpr ElementCount getElementCount() const {
+    assert(IsVector && "cannot get number of elements on scalar/aggregate");
+    return ElementCount::get(IsPointer
+                                 ? getFieldValue(PointerVectorElementsFieldInfo)
+                                 : getFieldValue(VectorElementsFieldInfo),
+                             isScalable());
+  }
+
+  /// Returns the total size of the type. Must only be called on sized types.
+  constexpr TypeSize getSizeInBits() const {
+    if (isPointer() || isScalar())
+      return TypeSize::Fixed(getScalarSizeInBits());
+    auto EC = getElementCount();
+    return TypeSize(getScalarSizeInBits() * EC.getKnownMinValue(),
+                    EC.isScalable());
+  }
+
+  /// Returns the total size of the type in bytes, i.e. number of whole bytes
+  /// needed to represent the size in bits. Must only be called on sized types.
+  constexpr TypeSize getSizeInBytes() const {
+    TypeSize BaseSize = getSizeInBits();
+    return {(BaseSize.getKnownMinValue() + 7) / 8, BaseSize.isScalable()};
+  }
 
-/// Construct a low-level type based on an LLVM type.
-LLT getLLTForType(Type &Ty, const DataLayout &DL);
+  constexpr LLT getScalarType() const {
+    return isVector() ? getElementType() : *this;
+  }
 
-/// Get a rough equivalent of an MVT for a given LLT. MVT can't distinguish
-/// pointers, so these will convert to a plain integer.
-MVT getMVTForLLT(LLT Ty);
-EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx);
+  /// If this type is a vector, return a vector with the same number of elements
+  /// but the new element type. Otherwise, return the new element type.
+  constexpr LLT changeElementType(LLT NewEltTy) const {
+    return isVector() ? LLT::vector(getElementCount(), NewEltTy) : NewEltTy;
+  }
+
+  /// If this type is a vector, return a vector with the same number of elements
+  /// but the new element size. Otherwise, return the new element type. Invalid
+  /// for pointer types. For pointer types, use changeElementType.
+  constexpr LLT changeElementSize(unsigned NewEltSize) const {
+    assert(!getScalarType().isPointer() &&
+           "invalid to directly change element size for pointers");
+    return isVector() ? LLT::vector(getElementCount(), NewEltSize)
+                      : LLT::scalar(NewEltSize);
+  }
+
+  /// Return a vector or scalar with the same element type and the new element
+  /// count.
+  constexpr LLT changeElementCount(ElementCount EC) const {
+    return LLT::scalarOrVector(EC, getScalarType());
+  }
+
+  /// Return a type that is \p Factor times smaller. Reduces the number of
+  /// elements if this is a vector, or the bitwidth for scalar/pointers. Does
+  /// not attempt to handle cases that aren't evenly divisible.
+  constexpr LLT divide(int Factor) const {
+    assert(Factor != 1);
+    assert((!isScalar() || getScalarSizeInBits() != 0) &&
+           "cannot divide scalar of size zero");
+    if (isVector()) {
+      assert(getElementCount().isKnownMultipleOf(Factor));
+      return scalarOrVector(getElementCount().divideCoefficientBy(Factor),
+                            getElementType());
+    }
+
+    assert(getScalarSizeInBits() % Factor == 0);
+    return scalar(getScalarSizeInBits() / Factor);
+  }
+
+  /// Produce a vector type that is \p Factor times bigger, preserving the
+  /// element type. For a scalar or pointer, this will produce a new vector with
+  /// \p Factor elements.
+  constexpr LLT multiplyElements(int Factor) const {
+    if (isVector()) {
+      return scalarOrVector(getElementCount().multiplyCoefficientBy(Factor),
+                            getElementType());
+    }
+
+    return fixed_vector(Factor, *this);
+  }
+
+  constexpr bool isByteSized() const {
+    return getSizeInBits().isKnownMultipleOf(8);
+  }
+
+  constexpr unsigned getScalarSizeInBits() const {
+    if (IsScalar)
+      return getFieldValue(ScalarSizeFieldInfo);
+    if (IsVector) {
+      if (!IsPointer)
+        return getFieldValue(VectorSizeFieldInfo);
+      else
+        return getFieldValue(PointerVectorSizeFieldInfo);
+    } else if (IsPointer)
+      return getFieldValue(PointerSizeFieldInfo);
+    else
+      llvm_unreachable("unexpected LLT");
+  }
+
+  constexpr unsigned getAddressSpace() const {
+    assert(RawData != 0 && "Invalid Type");
+    assert(IsPointer && "cannot get address space of non-pointer type");
+    if (!IsVector)
+      return getFieldValue(PointerAddressSpaceFieldInfo);
+    else
+      return getFieldValue(PointerVectorAddressSpaceFieldInfo);
+  }
+
+  /// Returns the vector's element type. Only valid for vector types.
+  constexpr LLT getElementType() const {
+    assert(isVector() && "cannot get element type of scalar/aggregate");
+    if (IsPointer)
+      return pointer(getAddressSpace(), getScalarSizeInBits());
+    else
+      return scalar(getScalarSizeInBits());
+  }
+
+  void print(raw_ostream &OS) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+
+  constexpr bool operator==(const LLT &RHS) const {
+    return IsPointer == RHS.IsPointer && IsVector == RHS.IsVector &&
+           IsScalar == RHS.IsScalar && RHS.RawData == RawData;
+  }
+
+  constexpr bool operator!=(const LLT &RHS) const { return !(*this == RHS); }
+
+  friend struct DenseMapInfo<LLT>;
+  friend class GISelInstProfileBuilder;
+
+private:
+  /// LLT is packed into 64 bits as follows:
+  /// isScalar : 1
+  /// isPointer : 1
+  /// isVector  : 1
+  /// with 61 bits remaining for Kind-specific data, packed in bitfields
+  /// as described below. As there isn't a simple portable way to pack bits
+  /// into bitfields, here the different fields in the packed structure is
+  /// described in static const *Field variables. Each of these variables
+  /// is a 2-element array, with the first element describing the bitfield size
+  /// and the second element describing the bitfield offset.
+  typedef int BitFieldInfo[2];
+  ///
+  /// This is how the bitfields are packed per Kind:
+  /// * Invalid:
+  ///   gets encoded as RawData == 0, as that is an invalid encoding, since for
+  ///   valid encodings, SizeInBits/SizeOfElement must be larger than 0.
+  /// * Non-pointer scalar (isPointer == 0 && isVector == 0):
+  ///   SizeInBits: 32;
+  static const constexpr BitFieldInfo ScalarSizeFieldInfo{32, 0};
+  /// * Pointer (isPointer == 1 && isVector == 0):
+  ///   SizeInBits: 16;
+  ///   AddressSpace: 24;
+  static const constexpr BitFieldInfo PointerSizeFieldInfo{16, 0};
+  static const constexpr BitFieldInfo PointerAddressSpaceFieldInfo{
+      24, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]};
+  static_assert((PointerAddressSpaceFieldInfo[0] +
+                 PointerAddressSpaceFieldInfo[1]) <= 61,
+                "Insufficient bits to encode all data");
+  /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1):
+  ///   NumElements: 16;
+  ///   SizeOfElement: 32;
+  ///   Scalable: 1;
+  static const constexpr BitFieldInfo VectorElementsFieldInfo{16, 0};
+  static const constexpr BitFieldInfo VectorSizeFieldInfo{
+      32, VectorElementsFieldInfo[0] + VectorElementsFieldInfo[1]};
+  static const constexpr BitFieldInfo VectorScalableFieldInfo{
+      1, VectorSizeFieldInfo[0] + VectorSizeFieldInfo[1]};
+  static_assert((VectorSizeFieldInfo[0] + VectorSizeFieldInfo[1]) <= 61,
+                "Insufficient bits to encode all data");
+  /// * Vector-of-pointer (isPointer == 1 && isVector == 1):
+  ///   NumElements: 16;
+  ///   SizeOfElement: 16;
+  ///   AddressSpace: 24;
+  ///   Scalable: 1;
+  static const constexpr BitFieldInfo PointerVectorElementsFieldInfo{16, 0};
+  static const constexpr BitFieldInfo PointerVectorSizeFieldInfo{
+      16,
+      PointerVectorElementsFieldInfo[1] + PointerVectorElementsFieldInfo[0]};
+  static const constexpr BitFieldInfo PointerVectorAddressSpaceFieldInfo{
+      24, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]};
+  static const constexpr BitFieldInfo PointerVectorScalableFieldInfo{
+      1, PointerVectorAddressSpaceFieldInfo[0] +
+             PointerVectorAddressSpaceFieldInfo[1]};
+  static_assert((PointerVectorAddressSpaceFieldInfo[0] +
+                 PointerVectorAddressSpaceFieldInfo[1]) <= 61,
+                "Insufficient bits to encode all data");
+
+  uint64_t IsScalar : 1;
+  uint64_t IsPointer : 1;
+  uint64_t IsVector : 1;
+  uint64_t RawData : 61;
+
+  static constexpr uint64_t getMask(const BitFieldInfo FieldInfo) {
+    const int FieldSizeInBits = FieldInfo[0];
+    return (((uint64_t)1) << FieldSizeInBits) - 1;
+  }
+  static constexpr uint64_t maskAndShift(uint64_t Val, uint64_t Mask,
+                                         uint8_t Shift) {
+    assert(Val <= Mask && "Value too large for field");
+    return (Val & Mask) << Shift;
+  }
+  static constexpr uint64_t maskAndShift(uint64_t Val,
+                                         const BitFieldInfo FieldInfo) {
+    return maskAndShift(Val, getMask(FieldInfo), FieldInfo[1]);
+  }
+
+  constexpr uint64_t getFieldValue(const BitFieldInfo FieldInfo) const {
+    return getMask(FieldInfo) & (RawData >> FieldInfo[1]);
+  }
+
+  constexpr void init(bool IsPointer, bool IsVector, bool IsScalar,
+                      ElementCount EC, uint64_t SizeInBits,
+                      unsigned AddressSpace) {
+    assert(SizeInBits <= std::numeric_limits<unsigned>::max() &&
+           "Not enough bits in LLT to represent size");
+    this->IsPointer = IsPointer;
+    this->IsVector = IsVector;
+    this->IsScalar = IsScalar;
+    if (IsScalar)
+      RawData = maskAndShift(SizeInBits, ScalarSizeFieldInfo);
+    else if (IsVector) {
+      assert(EC.isVector() && "invalid number of vector elements");
+      if (!IsPointer)
+        RawData =
+            maskAndShift(EC.getKnownMinValue(), VectorElementsFieldInfo) |
+            maskAndShift(SizeInBits, VectorSizeFieldInfo) |
+            maskAndShift(EC.isScalable() ? 1 : 0, VectorScalableFieldInfo);
+      else
+        RawData =
+            maskAndShift(EC.getKnownMinValue(),
+                         PointerVectorElementsFieldInfo) |
+            maskAndShift(SizeInBits, PointerVectorSizeFieldInfo) |
+            maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo) |
+            maskAndShift(EC.isScalable() ? 1 : 0,
+                         PointerVectorScalableFieldInfo);
+    } else if (IsPointer)
+      RawData = maskAndShift(SizeInBits, PointerSizeFieldInfo) |
+                maskAndShift(AddressSpace, PointerAddressSpaceFieldInfo);
+    else
+      llvm_unreachable("unexpected LLT configuration");
+  }
+
+public:
+  constexpr uint64_t getUniqueRAWLLTData() const {
+    return ((uint64_t)RawData) << 3 | ((uint64_t)IsScalar) << 2 |
+           ((uint64_t)IsPointer) << 1 | ((uint64_t)IsVector);
+  }
+};
+
+inline raw_ostream& operator<<(raw_ostream &OS, const LLT &Ty) {
+  Ty.print(OS);
+  return OS;
+}
 
-/// Get a rough equivalent of an LLT for a given MVT. LLT does not yet support
-/// scalarable vector types, and will assert if used.
-LLT getLLTForMVT(MVT Ty);
+template<> struct DenseMapInfo<LLT> {
+  static inline LLT getEmptyKey() {
+    LLT Invalid;
+    Invalid.IsPointer = true;
+    return Invalid;
+  }
+  static inline LLT getTombstoneKey() {
+    LLT Invalid;
+    Invalid.IsVector = true;
+    return Invalid;
+  }
+  static inline unsigned getHashValue(const LLT &Ty) {
+    uint64_t Val = Ty.getUniqueRAWLLTData();
+    return DenseMapInfo<uint64_t>::getHashValue(Val);
+  }
+  static bool isEqual(const LLT &LHS, const LLT &RHS) {
+    return LHS == RHS;
+  }
+};
 
-/// Get the appropriate floating point arithmetic semantic based on the bit size
-/// of the given scalar LLT.
-const llvm::fltSemantics &getFltSemanticForLLT(LLT Ty);
 }
 
 #endif // LLVM_CODEGEN_LOWLEVELTYPE_H
diff --git a/llvm/include/llvm/CodeGen/LowLevelTypeUtils.h b/llvm/include/llvm/CodeGen/LowLevelTypeUtils.h
new file mode 100644
index 0000000000000..5cd8e5412df26
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/LowLevelTypeUtils.h
@@ -0,0 +1,45 @@
+//== llvm/CodeGen/LowLevelTypeUtils.h -------------------------- -*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// Implement a low-level type suitable for MachineInstr level instruction
+/// selection.
+///
+/// This provides the CodeGen aspects of LowLevelType, such as Type conversion.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_LOWLEVELTYPEUTILS_H
+#define LLVM_CODEGEN_LOWLEVELTYPEUTILS_H
+
+#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/ValueTypes.h"
+
+namespace llvm {
+
+class DataLayout;
+class Type;
+struct fltSemantics;
+
+/// Construct a low-level type based on an LLVM type.
+LLT getLLTForType(Type &Ty, const DataLayout &DL);
+
+/// Get a rough equivalent of an MVT for a given LLT. MVT can't distinguish
+/// pointers, so these will convert to a plain integer.
+MVT getMVTForLLT(LLT Ty);
+EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx);
+
+/// Get a rough equivalent of an LLT for a given MVT. LLT does not yet support
+/// scalarable vector types, and will assert if used.
+LLT getLLTForMVT(MVT Ty);
+
+/// Get the appropriate floating point arithmetic semantic based on the bit size
+/// of the given scalar LLT.
+const llvm::fltSemantics &getFltSemanticForLLT(LLT Ty);
+}
+
+#endif // LLVM_CODEGEN_LOWLEVELTYPEUTILS_H
diff --git a/llvm/include/llvm/CodeGen/MIRFormatter.h b/llvm/include/llvm/CodeGen/MIRFormatter.h
index a039da1bd5b34..203d965836f65 100644
--- a/llvm/include/llvm/CodeGen/MIRFormatter.h
+++ b/llvm/include/llvm/CodeGen/MIRFormatter.h
@@ -35,7 +35,7 @@ class MIRFormatter {
 
   /// Implement target specific printing for machine operand immediate value, so
   /// that we can have more meaningful mnemonic than a 64-bit integer. Passing
-  /// None to OpIdx means the index is unknown.
+  /// std::nullopt to OpIdx means the index is unknown.
   virtual void printImm(raw_ostream &OS, const MachineInstr &MI,
                         std::optional<unsigned> OpIdx, int64_t Imm) const {
     OS << Imm;
diff --git a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
index 39e70d583710e..6d7c515b10086 100644
--- a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
+++ b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
@@ -175,6 +175,12 @@ enum class MachineCombinerPattern {
   FMADD_XA,
   FMSUB,
   FNMSUB,
+
+  // X86 VNNI
+  DPWSSD,
+
+  FNMADDS,
+  FNMADDD,
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index ad3f919824b3e..411653abf6aeb 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -120,7 +120,7 @@ class MachineInstr
 
   // Operands are allocated by an ArrayRecycler.
   MachineOperand *Operands = nullptr;   // Pointer to the first operand.
-  unsigned NumOperands = 0;             // Number of operands on instruction.
+  uint16_t NumOperands = 0;             // Number of operands on instruction.
 
   uint16_t Flags = 0;                   // Various bits of additional
                                         // information about machine
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index fd5f7f7a115ca..8410f5745aa4c 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -17,12 +17,12 @@
 
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/PointerUnion.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Value.h" // PointerLikeTypeTraits<Value*>
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/DataTypes.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h
index 6089339c7f5ad..5dc0e2918d469 100644
--- a/llvm/include/llvm/CodeGen/MachinePassManager.h
+++ b/llvm/include/llvm/CodeGen/MachinePassManager.h
@@ -134,8 +134,7 @@ class MachineFunctionPassManager
   using Base = PassManager<MachineFunction, MachineFunctionAnalysisManager>;
 
 public:
-  MachineFunctionPassManager(bool DebugLogging = false,
-                             bool RequireCodeGenSCCOrder = false,
+  MachineFunctionPassManager(bool RequireCodeGenSCCOrder = false,
                              bool VerifyMachineFunction = false)
       : RequireCodeGenSCCOrder(RequireCodeGenSCCOrder),
         VerifyMachineFunction(VerifyMachineFunction) {}
diff --git a/llvm/include/llvm/CodeGen/MachineValueType.h b/llvm/include/llvm/CodeGen/MachineValueType.h
new file mode 100644
index 0000000000000..d7dc38c7bd988
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineValueType.h
@@ -0,0 +1,555 @@
+//===- CodeGen/MachineValueType.h - Machine-Level types ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the set of machine-level target independent types which
+// legal values in the code generator use.
+//
+// Constants and properties are defined in ValueTypes.td.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEVALUETYPE_H
+#define LLVM_CODEGEN_MACHINEVALUETYPE_H
+
+#include "llvm/ADT/Sequence.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/TypeSize.h"
+#include <cassert>
+#include <cstdint>
+
+namespace llvm {
+
+  class Type;
+  class raw_ostream;
+
+  /// Machine Value Type. Every type that is supported natively by some
+  /// processor targeted by LLVM occurs here. This means that any legal value
+  /// type can be represented by an MVT.
+  class MVT {
+  public:
+    enum SimpleValueType : uint8_t {
+      // Simple value types that aren't explicitly part of this enumeration
+      // are considered extended value types.
+      INVALID_SIMPLE_VALUE_TYPE = 0,
+
+#define GET_VT_ATTR(Ty, n, sz, Any, Int, FP, Vec, Sc) Ty = n,
+#define GET_VT_RANGES
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_ATTR
+#undef GET_VT_RANGES
+
+      VALUETYPE_SIZE = LAST_VALUETYPE + 1,
+
+      // This is the current maximum for LAST_VALUETYPE.
+      // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
+      // This value must be a multiple of 32.
+      MAX_ALLOWED_VALUETYPE = 224,
+    };
+
+    static_assert(FIRST_VALUETYPE > 0);
+    static_assert(LAST_VALUETYPE < MAX_ALLOWED_VALUETYPE);
+
+    SimpleValueType SimpleTy = INVALID_SIMPLE_VALUE_TYPE;
+
+    constexpr MVT() = default;
+    constexpr MVT(SimpleValueType SVT) : SimpleTy(SVT) {}
+
+    bool operator>(const MVT& S)  const { return SimpleTy >  S.SimpleTy; }
+    bool operator<(const MVT& S)  const { return SimpleTy <  S.SimpleTy; }
+    bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; }
+    bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; }
+    bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; }
+    bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; }
+
+    /// Support for debugging, callable in GDB: VT.dump()
+    void dump() const;
+
+    /// Implement operator<<.
+    void print(raw_ostream &OS) const;
+
+    /// Return true if this is a valid simple valuetype.
+    bool isValid() const {
+      return (SimpleTy >= MVT::FIRST_VALUETYPE &&
+              SimpleTy <= MVT::LAST_VALUETYPE);
+    }
+
+    /// Return true if this is a FP or a vector FP type.
+    bool isFloatingPoint() const {
+      return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE));
+    }
+
+    /// Return true if this is an integer or a vector integer type.
+    bool isInteger() const {
+      return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE));
+    }
+
+    /// Return true if this is an integer, not including vectors.
+    bool isScalarInteger() const {
+      return (SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
+              SimpleTy <= MVT::LAST_INTEGER_VALUETYPE);
+    }
+
+    /// Return true if this is a vector value type.
+    bool isVector() const {
+      return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE &&
+              SimpleTy <= MVT::LAST_VECTOR_VALUETYPE);
+    }
+
+    /// Return true if this is a vector value type where the
+    /// runtime length is machine dependent
+    bool isScalableVector() const {
+      return (SimpleTy >= MVT::FIRST_SCALABLE_VECTOR_VALUETYPE &&
+              SimpleTy <= MVT::LAST_SCALABLE_VECTOR_VALUETYPE);
+    }
+
+    /// Return true if this is a custom target type that has a scalable size.
+    bool isScalableTargetExtVT() const {
+      return SimpleTy == MVT::aarch64svcount;
+    }
+
+    /// Return true if the type is a scalable type.
+    bool isScalableVT() const {
+      return isScalableVector() || isScalableTargetExtVT();
+    }
+
+    bool isFixedLengthVector() const {
+      return (SimpleTy >= MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE &&
+              SimpleTy <= MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE);
+    }
+
+    /// Return true if this is a 16-bit vector type.
+    bool is16BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 16);
+    }
+
+    /// Return true if this is a 32-bit vector type.
+    bool is32BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 32);
+    }
+
+    /// Return true if this is a 64-bit vector type.
+    bool is64BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 64);
+    }
+
+    /// Return true if this is a 128-bit vector type.
+    bool is128BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 128);
+    }
+
+    /// Return true if this is a 256-bit vector type.
+    bool is256BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 256);
+    }
+
+    /// Return true if this is a 512-bit vector type.
+    bool is512BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 512);
+    }
+
+    /// Return true if this is a 1024-bit vector type.
+    bool is1024BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 1024);
+    }
+
+    /// Return true if this is a 2048-bit vector type.
+    bool is2048BitVector() const {
+      return (isFixedLengthVector() && getFixedSizeInBits() == 2048);
+    }
+
+    /// Return true if this is an overloaded type for TableGen.
+    bool isOverloaded() const {
+      switch (SimpleTy) {
+#define GET_VT_ATTR(Ty, n, sz, Any, Int, FP, Vec, Sc)                          \
+  case Ty:                                                                     \
+    return Any;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_ATTR
+      default:
+        return false;
+      }
+    }
+
+    /// Return a vector with the same number of elements as this vector, but
+    /// with the element type converted to an integer type with the same
+    /// bitwidth.
+    MVT changeVectorElementTypeToInteger() const {
+      MVT EltTy = getVectorElementType();
+      MVT IntTy = MVT::getIntegerVT(EltTy.getSizeInBits());
+      MVT VecTy = MVT::getVectorVT(IntTy, getVectorElementCount());
+      assert(VecTy.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
+             "Simple vector VT not representable by simple integer vector VT!");
+      return VecTy;
+    }
+
+    /// Return a VT for a vector type whose attributes match ourselves
+    /// with the exception of the element type that is chosen by the caller.
+    MVT changeVectorElementType(MVT EltVT) const {
+      MVT VecTy = MVT::getVectorVT(EltVT, getVectorElementCount());
+      assert(VecTy.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
+             "Simple vector VT not representable by simple integer vector VT!");
+      return VecTy;
+    }
+
+    /// Return the type converted to an equivalently sized integer or vector
+    /// with integer element type. Similar to changeVectorElementTypeToInteger,
+    /// but also handles scalars.
+    MVT changeTypeToInteger() {
+      if (isVector())
+        return changeVectorElementTypeToInteger();
+      return MVT::getIntegerVT(getSizeInBits());
+    }
+
+    /// Return a VT for a vector type with the same element type but
+    /// half the number of elements.
+    MVT getHalfNumVectorElementsVT() const {
+      MVT EltVT = getVectorElementType();
+      auto EltCnt = getVectorElementCount();
+      assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!");
+      return getVectorVT(EltVT, EltCnt.divideCoefficientBy(2));
+    }
+
+    // Return a VT for a vector type with the same element type but
+    // double the number of elements.
+    MVT getDoubleNumVectorElementsVT() const {
+      MVT EltVT = getVectorElementType();
+      auto EltCnt = getVectorElementCount();
+      return MVT::getVectorVT(EltVT, EltCnt * 2);
+    }
+
+    /// Returns true if the given vector is a power of 2.
+    bool isPow2VectorType() const {
+      unsigned NElts = getVectorMinNumElements();
+      return !(NElts & (NElts - 1));
+    }
+
+    /// Widens the length of the given vector MVT up to the nearest power of 2
+    /// and returns that type.
+    MVT getPow2VectorType() const {
+      if (isPow2VectorType())
+        return *this;
+
+      ElementCount NElts = getVectorElementCount();
+      unsigned NewMinCount = 1 << Log2_32_Ceil(NElts.getKnownMinValue());
+      NElts = ElementCount::get(NewMinCount, NElts.isScalable());
+      return MVT::getVectorVT(getVectorElementType(), NElts);
+    }
+
+    /// If this is a vector, return the element type, otherwise return this.
+    MVT getScalarType() const {
+      return isVector() ? getVectorElementType() : *this;
+    }
+
+    MVT getVectorElementType() const {
+      switch (SimpleTy) {
+      default:
+        llvm_unreachable("Not a vector MVT!");
+
+#define GET_VT_VECATTR(Ty, Sc, nElem, ElTy, ElSz)                              \
+  case Ty:                                                                     \
+    return ElTy;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_VECATTR
+      }
+    }
+
+    /// Given a vector type, return the minimum number of elements it contains.
+    unsigned getVectorMinNumElements() const {
+      switch (SimpleTy) {
+      default:
+        llvm_unreachable("Not a vector MVT!");
+
+#define GET_VT_VECATTR(Ty, Sc, nElem, ElTy, ElSz)                              \
+  case Ty:                                                                     \
+    return nElem;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_VECATTR
+      }
+    }
+
+    ElementCount getVectorElementCount() const {
+      return ElementCount::get(getVectorMinNumElements(), isScalableVector());
+    }
+
+    unsigned getVectorNumElements() const {
+      if (isScalableVector())
+        llvm::reportInvalidSizeRequest(
+            "Possible incorrect use of MVT::getVectorNumElements() for "
+            "scalable vector. Scalable flag may be dropped, use "
+            "MVT::getVectorElementCount() instead");
+      return getVectorMinNumElements();
+    }
+
+    /// Returns the size of the specified MVT in bits.
+    ///
+    /// If the value type is a scalable vector type, the scalable property will
+    /// be set and the runtime size will be a positive integer multiple of the
+    /// base size.
+    TypeSize getSizeInBits() const {
+      switch (SimpleTy) {
+      default:
+        switch (SimpleTy) {
+        default:
+          llvm_unreachable("getSizeInBits called on extended MVT.");
+
+#define GET_VT_ATTR(Ty, N, Sz, Any, Int, FP, Vec, Sc)                          \
+  case Ty:                                                                     \
+    return (Sc ? TypeSize::Scalable(Sz) : TypeSize::Fixed(Sz));
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_ATTR
+        }
+      case Other:
+        llvm_unreachable("Value type is non-standard value, Other.");
+      case iPTR:
+        llvm_unreachable("Value type size is target-dependent. Ask TLI.");
+      case iPTRAny:
+      case iAny:
+      case fAny:
+      case vAny:
+      case Any:
+        llvm_unreachable("Value type is overloaded.");
+      case token:
+        llvm_unreachable("Token type is a sentinel that cannot be used "
+                         "in codegen and has no size");
+      case Metadata:
+        llvm_unreachable("Value type is metadata.");
+      case aarch64svcount: // FIXME: Not in the td.
+        return TypeSize::Scalable(16);
+      }
+    }
+
+    /// Return the size of the specified fixed width value type in bits. The
+    /// function will assert if the type is scalable.
+    uint64_t getFixedSizeInBits() const {
+      return getSizeInBits().getFixedValue();
+    }
+
+    uint64_t getScalarSizeInBits() const {
+      return getScalarType().getSizeInBits().getFixedValue();
+    }
+
+    /// Return the number of bytes overwritten by a store of the specified value
+    /// type.
+    ///
+    /// If the value type is a scalable vector type, the scalable property will
+    /// be set and the runtime size will be a positive integer multiple of the
+    /// base size.
+    TypeSize getStoreSize() const {
+      TypeSize BaseSize = getSizeInBits();
+      return {(BaseSize.getKnownMinValue() + 7) / 8, BaseSize.isScalable()};
+    }
+
+    // Return the number of bytes overwritten by a store of this value type or
+    // this value type's element type in the case of a vector.
+    uint64_t getScalarStoreSize() const {
+      return getScalarType().getStoreSize().getFixedValue();
+    }
+
+    /// Return the number of bits overwritten by a store of the specified value
+    /// type.
+    ///
+    /// If the value type is a scalable vector type, the scalable property will
+    /// be set and the runtime size will be a positive integer multiple of the
+    /// base size.
+    TypeSize getStoreSizeInBits() const {
+      return getStoreSize() * 8;
+    }
+
+    /// Returns true if the number of bits for the type is a multiple of an
+    /// 8-bit byte.
+    bool isByteSized() const { return getSizeInBits().isKnownMultipleOf(8); }
+
+    /// Return true if we know at compile time this has more bits than VT.
+    bool knownBitsGT(MVT VT) const {
+      return TypeSize::isKnownGT(getSizeInBits(), VT.getSizeInBits());
+    }
+
+    /// Return true if we know at compile time this has more than or the same
+    /// bits as VT.
+    bool knownBitsGE(MVT VT) const {
+      return TypeSize::isKnownGE(getSizeInBits(), VT.getSizeInBits());
+    }
+
+    /// Return true if we know at compile time this has fewer bits than VT.
+    bool knownBitsLT(MVT VT) const {
+      return TypeSize::isKnownLT(getSizeInBits(), VT.getSizeInBits());
+    }
+
+    /// Return true if we know at compile time this has fewer than or the same
+    /// bits as VT.
+    bool knownBitsLE(MVT VT) const {
+      return TypeSize::isKnownLE(getSizeInBits(), VT.getSizeInBits());
+    }
+
+    /// Return true if this has more bits than VT.
+    bool bitsGT(MVT VT) const {
+      assert(isScalableVector() == VT.isScalableVector() &&
+             "Comparison between scalable and fixed types");
+      return knownBitsGT(VT);
+    }
+
+    /// Return true if this has no less bits than VT.
+    bool bitsGE(MVT VT) const {
+      assert(isScalableVector() == VT.isScalableVector() &&
+             "Comparison between scalable and fixed types");
+      return knownBitsGE(VT);
+    }
+
+    /// Return true if this has less bits than VT.
+    bool bitsLT(MVT VT) const {
+      assert(isScalableVector() == VT.isScalableVector() &&
+             "Comparison between scalable and fixed types");
+      return knownBitsLT(VT);
+    }
+
+    /// Return true if this has no more bits than VT.
+    bool bitsLE(MVT VT) const {
+      assert(isScalableVector() == VT.isScalableVector() &&
+             "Comparison between scalable and fixed types");
+      return knownBitsLE(VT);
+    }
+
+    static MVT getFloatingPointVT(unsigned BitWidth) {
+#define GET_VT_ATTR(Ty, n, sz, Any, Int, FP, Vec, Sc)                          \
+  if (FP == 3 && sz == BitWidth)                                               \
+    return Ty;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_ATTR
+
+      llvm_unreachable("Bad bit width!");
+    }
+
+    static MVT getIntegerVT(unsigned BitWidth) {
+#define GET_VT_ATTR(Ty, n, sz, Any, Int, FP, Vec, Sc)                          \
+  if (Int == 3 && sz == BitWidth)                                              \
+    return Ty;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_ATTR
+
+      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
+    }
+
+    static MVT getVectorVT(MVT VT, unsigned NumElements) {
+#define GET_VT_VECATTR(Ty, Sc, nElem, ElTy, ElSz)                              \
+  if (!Sc && VT.SimpleTy == ElTy && NumElements == nElem)                      \
+    return Ty;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_VECATTR
+
+      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
+    }
+
+    static MVT getScalableVectorVT(MVT VT, unsigned NumElements) {
+#define GET_VT_VECATTR(Ty, Sc, nElem, ElTy, ElSz)                              \
+  if (Sc && VT.SimpleTy == ElTy && NumElements == nElem)                       \
+    return Ty;
+#include "llvm/CodeGen/GenVT.inc"
+#undef GET_VT_VECATTR
+
+      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
+    }
+
+    static MVT getVectorVT(MVT VT, unsigned NumElements, bool IsScalable) {
+      if (IsScalable)
+        return getScalableVectorVT(VT, NumElements);
+      return getVectorVT(VT, NumElements);
+    }
+
+    static MVT getVectorVT(MVT VT, ElementCount EC) {
+      if (EC.isScalable())
+        return getScalableVectorVT(VT, EC.getKnownMinValue());
+      return getVectorVT(VT, EC.getKnownMinValue());
+    }
+
+    /// Return the value type corresponding to the specified type.  This returns
+    /// all pointers as iPTR.  If HandleUnknown is true, unknown types are
+    /// returned as Other, otherwise they are invalid.
+    static MVT getVT(Type *Ty, bool HandleUnknown = false);
+
+  public:
+    /// SimpleValueType Iteration
+    /// @{
+    static auto all_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_VALUETYPE, MVT::LAST_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto integer_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_INTEGER_VALUETYPE,
+                                MVT::LAST_INTEGER_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto fp_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_FP_VALUETYPE, MVT::LAST_FP_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_VECTOR_VALUETYPE,
+                                MVT::LAST_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto fixedlen_vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE,
+                                MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto scalable_vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_SCALABLE_VECTOR_VALUETYPE,
+                                MVT::LAST_SCALABLE_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto integer_fixedlen_vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
+                                MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto fp_fixedlen_vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE,
+                                MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto integer_scalable_vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
+                                MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+
+    static auto fp_scalable_vector_valuetypes() {
+      return enum_seq_inclusive(MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE,
+                                MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE,
+                                force_iteration_on_noniterable_enum);
+    }
+    /// @}
+  };
+
+  inline raw_ostream &operator<<(raw_ostream &OS, const MVT &VT) {
+    VT.print(OS);
+    return OS;
+  }
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINEVALUETYPE_H
diff --git a/llvm/include/llvm/CodeGen/RegisterBankInfo.h b/llvm/include/llvm/CodeGen/RegisterBankInfo.h
index 2606b04a02060..03bb58fbf4f26 100644
--- a/llvm/include/llvm/CodeGen/RegisterBankInfo.h
+++ b/llvm/include/llvm/CodeGen/RegisterBankInfo.h
@@ -18,9 +18,9 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include <cassert>
 #include <initializer_list>
 #include <memory>
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index a9aeb62c11692..fd185f0601be8 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DebugLoc.h"
@@ -37,7 +38,6 @@
 #include "llvm/Support/ArrayRecycler.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/RecyclingAllocator.h"
 #include <cassert>
 #include <cstdint>
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 620fdfc9f754d..5c0b9860e07df 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -30,6 +30,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Constants.h"
@@ -42,7 +43,6 @@
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/TypeSize.h"
 #include <algorithm>
 #include <cassert>
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index 1333f2d989731..89ea9bcb2a408 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -13,9 +13,9 @@
 #ifndef LLVM_CODEGEN_TARGETCALLINGCONV_H
 #define LLVM_CODEGEN_TARGETCALLINGCONV_H
 
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/Support/Alignment.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
 #include <climits>
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index b4a7bbb4c99ce..7d8265ad413c2 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1223,6 +1223,13 @@ class TargetInstrInfo : public MCInstrInfo {
       SmallVectorImpl<MachineInstr *> &DelInstrs,
       DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const;
 
+  /// When calculate the latency of the root instruction, accumulate the
+  /// latency of the sequence to the root latency.
+  /// \param Root - Instruction that could be combined with one of its operands
+  virtual bool accumulateInstrSeqToRootLatency(MachineInstr &Root) const {
+    return true;
+  }
+
   /// Attempt to reassociate \P Root and \P Prev according to \P Pattern to
   /// reduce critical path length.
   void reassociateOps(MachineInstr &Root, MachineInstr &Prev,
@@ -1966,7 +1973,7 @@ class TargetInstrInfo : public MCInstrInfo {
   }
 
   /// Returns a \p outliner::OutlinedFunction struct containing target-specific
-  /// information for a set of outlining candidates. Returns None if the
+  /// information for a set of outlining candidates. Returns std::nullopt if the
   /// candidates are not suitable for outlining.
   virtual std::optional<outliner::OutlinedFunction> getOutliningCandidateInfo(
       std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 85c5ed2c4a5fe..56a067a7bbae1 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -30,7 +30,8 @@
 #include "llvm/CodeGen/ComplexDeinterleavingPass.h"
 #include "llvm/CodeGen/DAGCombine.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -49,7 +50,6 @@
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include <algorithm>
 #include <cassert>
 #include <climits>
@@ -433,6 +433,13 @@ class TargetLoweringBase {
     return MachineMemOperand::MONone;
   }
 
+  /// This callback is used to inspect load/store SDNode.
+  /// The default implementation does nothing.
+  virtual MachineMemOperand::Flags
+  getTargetMMOFlags(const MemSDNode &Node) const {
+    return MachineMemOperand::MONone;
+  }
+
   MachineMemOperand::Flags
   getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL,
                          AssumptionCache *AC = nullptr,
@@ -672,6 +679,13 @@ class TargetLoweringBase {
     return false;
   }
 
+  /// Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
+  virtual bool
+  areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX,
+                                      const MemSDNode &NodeY) const {
+    return true;
+  }
+
   /// Use bitwise logic to make pairs of compares more efficient. For example:
   /// and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
   /// This should be true when it takes more than one instruction to lower
@@ -2284,11 +2298,11 @@ class TargetLoweringBase {
   /// considered beneficial.
   /// If optimizing for size, expansion is only considered beneficial for upto
   /// 5 multiplies and a divide (if the exponent is negative).
-  bool isBeneficialToExpandPowI(int Exponent, bool OptForSize) const {
+  bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const {
     if (Exponent < 0)
       Exponent = -Exponent;
-    return !OptForSize ||
-           (llvm::popcount((unsigned int)Exponent) + Log2_32(Exponent) < 7);
+    uint64_t E = static_cast<uint64_t>(Exponent);
+    return !OptForSize || (llvm::popcount(E) + Log2_64(E) < 7);
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 5f2793203f5ca..fe34be9a42c14 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -20,11 +20,11 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/MC/LaneBitmask.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Printable.h"
 #include <cassert>
diff --git a/llvm/include/llvm/CodeGen/ValueTypes.h b/llvm/include/llvm/CodeGen/ValueTypes.h
index 0719c742b22cd..747cce781cf7a 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.h
+++ b/llvm/include/llvm/CodeGen/ValueTypes.h
@@ -15,8 +15,8 @@
 #ifndef LLVM_CODEGEN_VALUETYPES_H
 #define LLVM_CODEGEN_VALUETYPES_H
 
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/TypeSize.h"
 #include <cassert>
diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td
index 336b2a49b131e..25f0d385259db 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -14,224 +14,269 @@
 
 class ValueType<int size, int value> {
   string Namespace = "MVT";
+  string LLVMName = NAME;
   int Size = size;
   int Value = value;
+  int nElem = 1;
+  ValueType ElementType = ?;
+  int isOverloaded = false;
+  int isInteger = false;
+  int isFP = false;
+  int isVector = false;
+  int isScalable = false;
 }
 
-def OtherVT : ValueType<0,   1>;  // "Other" value
-def i1      : ValueType<1,   2>;  // One bit boolean value
-def i2      : ValueType<2,   3>;  // 2-bit integer value
-def i4      : ValueType<4,   4>;  // 4-bit integer value
-def i8      : ValueType<8,   5>;  // 8-bit integer value
-def i16     : ValueType<16,  6>;  // 16-bit integer value
-def i32     : ValueType<32,  7>;  // 32-bit integer value
-def i64     : ValueType<64,  8>;  // 64-bit integer value
-def i128    : ValueType<128, 9>;  // 128-bit integer value
-
-def bf16    : ValueType<16,  10>;  // 16-bit brain floating point value
-def f16     : ValueType<16,  11>;  // 16-bit floating point value
-def f32     : ValueType<32,  12>;  // 32-bit floating point value
-def f64     : ValueType<64,  13>;  // 64-bit floating point value
-def f80     : ValueType<80,  14>;  // 80-bit floating point value
-def f128    : ValueType<128, 15>;  // 128-bit floating point value
-def ppcf128 : ValueType<128, 16>;  // PPC 128-bit floating point value
-
-def v1i1    : ValueType<1,    17>;  //    1 x i1 vector value
-def v2i1    : ValueType<2,    18>;  //    2 x i1 vector value
-def v4i1    : ValueType<4,    19>;  //    4 x i1 vector value
-def v8i1    : ValueType<8,    20>;  //    8 x i1 vector value
-def v16i1   : ValueType<16,   21>;  //   16 x i1 vector value
-def v32i1   : ValueType<32,   22>;  //   32 x i1 vector value
-def v64i1   : ValueType<64,   23>;  //   64 x i1 vector value
-def v128i1  : ValueType<128,  24>;  //  128 x i1 vector value
-def v256i1  : ValueType<256,  25>;  //  256 x i1 vector value
-def v512i1  : ValueType<512,  26>;  //  512 x i1 vector value
-def v1024i1 : ValueType<1024, 27>;  // 1024 x i1 vector value
-def v2048i1 : ValueType<2048, 28>;  // 2048 x i1 vector value
-
-def v128i2  : ValueType<256,  29>;   //  128 x i2 vector value
-def v256i2  : ValueType<512,  30>;   //  256 x i2 vector value
-
-def v64i4   : ValueType<256,  31>;   //   64 x i4 vector value
-def v128i4  : ValueType<512,  32>;   //  128 x i4 vector value
-
-def v1i8    : ValueType<8,    33>;  //    1 x i8 vector value
-def v2i8    : ValueType<16,   34>;  //    2 x i8 vector value
-def v4i8    : ValueType<32,   35>;  //    4 x i8 vector value
-def v8i8    : ValueType<64,   36>;  //    8 x i8 vector value
-def v16i8   : ValueType<128,  37>;  //   16 x i8 vector value
-def v32i8   : ValueType<256,  38>;  //   32 x i8 vector value
-def v64i8   : ValueType<512,  39>;  //   64 x i8 vector value
-def v128i8  : ValueType<1024, 40>;  //  128 x i8 vector value
-def v256i8  : ValueType<2048, 41>;  //  256 x i8 vector value
-def v512i8  : ValueType<4096, 42>;  //  512 x i8 vector value
-def v1024i8 : ValueType<8192, 43>;  // 1024 x i8 vector value
-
-def v1i16   : ValueType<16,   44>;  //   1 x i16 vector value
-def v2i16   : ValueType<32,   45>;  //   2 x i16 vector value
-def v3i16   : ValueType<48,   46>;  //   3 x i16 vector value
-def v4i16   : ValueType<64,   47>;  //   4 x i16 vector value
-def v8i16   : ValueType<128,  48>;  //   8 x i16 vector value
-def v16i16  : ValueType<256,  49>;  //  16 x i16 vector value
-def v32i16  : ValueType<512,  50>;  //  32 x i16 vector value
-def v64i16  : ValueType<1024, 51>;  //  64 x i16 vector value
-def v128i16 : ValueType<2048, 52>;  // 128 x i16 vector value
-def v256i16 : ValueType<4096, 53>;  // 256 x i16 vector value
-def v512i16 : ValueType<8192, 54>;  // 512 x i16 vector value
-
-def v1i32    : ValueType<32,    55>;  //    1 x i32 vector value
-def v2i32    : ValueType<64,    56>;  //    2 x i32 vector value
-def v3i32    : ValueType<96,    57>;  //    3 x i32 vector value
-def v4i32    : ValueType<128,   58>;  //    4 x i32 vector value
-def v5i32    : ValueType<160,   59>;  //    5 x i32 vector value
-def v6i32    : ValueType<192,   60>;  //    6 x f32 vector value
-def v7i32    : ValueType<224,   61>;  //    7 x f32 vector value
-def v8i32    : ValueType<256,   62>;  //    8 x i32 vector value
-def v9i32    : ValueType<288,   63>;  //    9 x i32 vector value
-def v10i32   : ValueType<320,   64>;  //   10 x i32 vector value
-def v11i32   : ValueType<352,   65>;  //   11 x i32 vector value
-def v12i32   : ValueType<384,   66>;  //   12 x i32 vector value
-def v16i32   : ValueType<512,   67>;  //   16 x i32 vector value
-def v32i32   : ValueType<1024,  68>;  //   32 x i32 vector value
-def v64i32   : ValueType<2048,  69>;  //   64 x i32 vector value
-def v128i32  : ValueType<4096,  70>;  //  128 x i32 vector value
-def v256i32  : ValueType<8192,  71>;  //  256 x i32 vector value
-def v512i32  : ValueType<16384, 72>;  //  512 x i32 vector value
-def v1024i32 : ValueType<32768, 73>;  // 1024 x i32 vector value
-def v2048i32 : ValueType<65536, 74>;  // 2048 x i32 vector value
-
-def v1i64   : ValueType<64,    75>;  //   1 x i64 vector value
-def v2i64   : ValueType<128,   76>;  //   2 x i64 vector value
-def v3i64   : ValueType<192,   77>;  //   3 x i64 vector value
-def v4i64   : ValueType<256,   78>;  //   4 x i64 vector value
-def v8i64   : ValueType<512,   79>;  //   8 x i64 vector value
-def v16i64  : ValueType<1024,  80>;  //  16 x i64 vector value
-def v32i64  : ValueType<2048,  81>;  //  32 x i64 vector value
-def v64i64  : ValueType<4096,  82>;  //  64 x i64 vector value
-def v128i64 : ValueType<8192,  83>;  // 128 x i64 vector value
-def v256i64 : ValueType<16384, 84>;  // 256 x i64 vector value
-
-def v1i128 : ValueType<128, 85>;  //  1 x i128 vector value
-
-def v1f16    : ValueType<16,     86>;  //    1 x f16 vector value
-def v2f16    : ValueType<32,     87>;  //    2 x f16 vector value
-def v3f16    : ValueType<48,     88>;  //    3 x f16 vector value
-def v4f16    : ValueType<64,     89>;  //    4 x f16 vector value
-def v8f16    : ValueType<128,    90>;  //    8 x f16 vector value
-def v16f16   : ValueType<256,    91>;  //   16 x f16 vector value
-def v32f16   : ValueType<512,    92>;  //   32 x f16 vector value
-def v64f16   : ValueType<1024,   93>;  //   64 x f16 vector value
-def v128f16  : ValueType<2048,   94>;  //  128 x f16 vector value
-def v256f16  : ValueType<4096,   95>;  //  256 x f16 vector value
-def v512f16  : ValueType<8192,   96>;  //  512 x f16 vector value
-
-def v2bf16   : ValueType<32,     97>;  //    2 x bf16 vector value
-def v3bf16   : ValueType<48,     98>;  //    3 x bf16 vector value
-def v4bf16   : ValueType<64,     99>;  //    4 x bf16 vector value
-def v8bf16   : ValueType<128,   100>;  //    8 x bf16 vector value
-def v16bf16  : ValueType<256,   101>;  //   16 x bf16 vector value
-def v32bf16  : ValueType<512,   102>;  //   32 x bf16 vector value
-def v64bf16  : ValueType<1024,  103>;  //   64 x bf16 vector value
-def v128bf16 : ValueType<2048,  104>;  //  128 x bf16 vector value
-
-def v1f32    : ValueType<32,    105>;  //    1 x f32 vector value
-def v2f32    : ValueType<64,    106>;  //    2 x f32 vector value
-def v3f32    : ValueType<96,    107>;  //    3 x f32 vector value
-def v4f32    : ValueType<128,   108>;  //    4 x f32 vector value
-def v5f32    : ValueType<160,   109>;  //    5 x f32 vector value
-def v6f32    : ValueType<192,   110>;  //    6 x f32 vector value
-def v7f32    : ValueType<224,   111>;  //    7 x f32 vector value
-def v8f32    : ValueType<256,   112>;  //    8 x f32 vector value
-def v9f32    : ValueType<288,   113>;  //    9 x f32 vector value
-def v10f32   : ValueType<320,   114>;  //   10 x f32 vector value
-def v11f32   : ValueType<352,   115>;  //   11 x f32 vector value
-def v12f32   : ValueType<384,   116>;  //   12 x f32 vector value
-def v16f32   : ValueType<512,   117>;  //   16 x f32 vector value
-def v32f32   : ValueType<1024,  118>;  //   32 x f32 vector value
-def v64f32   : ValueType<2048,  119>;  //   64 x f32 vector value
-def v128f32  : ValueType<4096,  120>;  //  128 x f32 vector value
-def v256f32  : ValueType<8192,  121>;  //  256 x f32 vector value
-def v512f32  : ValueType<16384, 122>;  //  512 x f32 vector value
-def v1024f32 : ValueType<32768, 123>;  // 1024 x f32 vector value
-def v2048f32 : ValueType<65536, 124>;  // 2048 x f32 vector value
-
-def v1f64    : ValueType<64,    125>;  //    1 x f64 vector value
-def v2f64    : ValueType<128,   126>;  //    2 x f64 vector value
-def v3f64    : ValueType<192,   127>;  //    3 x f64 vector value
-def v4f64    : ValueType<256,   128>;  //    4 x f64 vector value
-def v8f64    : ValueType<512,   129>;  //    8 x f64 vector value
-def v16f64   : ValueType<1024,  130>;  //   16 x f64 vector value
-def v32f64   : ValueType<2048,  131>;  //   32 x f64 vector value
-def v64f64   : ValueType<4096,  132>;  //   64 x f64 vector value
-def v128f64  : ValueType<8192,  133>;  //  128 x f64 vector value
-def v256f64  : ValueType<16384, 134>;  //  256 x f64 vector value
-
-def nxv1i1  : ValueType<1,  135>;  // n x  1 x i1  vector value
-def nxv2i1  : ValueType<2,  136>;  // n x  2 x i1  vector value
-def nxv4i1  : ValueType<4,  137>;  // n x  4 x i1  vector value
-def nxv8i1  : ValueType<8,  138>;  // n x  8 x i1  vector value
-def nxv16i1 : ValueType<16, 139>;  // n x 16 x i1  vector value
-def nxv32i1 : ValueType<32, 140>;  // n x 32 x i1  vector value
-def nxv64i1 : ValueType<64, 141>;  // n x 64 x i1  vector value
-
-def nxv1i8  : ValueType<8,   142>;  // n x  1 x i8  vector value
-def nxv2i8  : ValueType<16,  143>;  // n x  2 x i8  vector value
-def nxv4i8  : ValueType<32,  144>;  // n x  4 x i8  vector value
-def nxv8i8  : ValueType<64,  145>;  // n x  8 x i8  vector value
-def nxv16i8 : ValueType<128, 146>;  // n x 16 x i8  vector value
-def nxv32i8 : ValueType<256, 147>;  // n x 32 x i8  vector value
-def nxv64i8 : ValueType<512, 148>;  // n x 64 x i8  vector value
-
-def nxv1i16  : ValueType<16,  149>;  // n x  1 x i16 vector value
-def nxv2i16  : ValueType<32,  150>;  // n x  2 x i16 vector value
-def nxv4i16  : ValueType<64,  151>;  // n x  4 x i16 vector value
-def nxv8i16  : ValueType<128, 152>;  // n x  8 x i16 vector value
-def nxv16i16 : ValueType<256, 153>;  // n x 16 x i16 vector value
-def nxv32i16 : ValueType<512, 154>;  // n x 32 x i16 vector value
-
-def nxv1i32  : ValueType<32,   155>;  // n x  1 x i32 vector value
-def nxv2i32  : ValueType<64,   156>;  // n x  2 x i32 vector value
-def nxv4i32  : ValueType<128,  157>;  // n x  4 x i32 vector value
-def nxv8i32  : ValueType<256,  158>;  // n x  8 x i32 vector value
-def nxv16i32 : ValueType<512,  159>;  // n x 16 x i32 vector value
-def nxv32i32 : ValueType<1024, 160>;  // n x 32 x i32 vector value
-
-def nxv1i64  : ValueType<64,   161>;  // n x  1 x i64 vector value
-def nxv2i64  : ValueType<128,  162>;  // n x  2 x i64 vector value
-def nxv4i64  : ValueType<256,  163>;  // n x  4 x i64 vector value
-def nxv8i64  : ValueType<512,  164>;  // n x  8 x i64 vector value
-def nxv16i64 : ValueType<1024, 165>;  // n x 16 x i64 vector value
-def nxv32i64 : ValueType<2048, 166>;  // n x 32 x i64 vector value
-
-def nxv1f16  : ValueType<16,  167>;  // n x  1 x  f16 vector value
-def nxv2f16  : ValueType<32,  168>;  // n x  2 x  f16 vector value
-def nxv4f16  : ValueType<64,  169>;  // n x  4 x  f16 vector value
-def nxv8f16  : ValueType<128, 170>;  // n x  8 x  f16 vector value
-def nxv16f16 : ValueType<256, 171>;  // n x 16 x  f16 vector value
-def nxv32f16 : ValueType<512, 172>;  // n x 32 x  f16 vector value
-
-def nxv1bf16 : ValueType<16,  173>;  // n x  1 x bf16 vector value
-def nxv2bf16 : ValueType<32,  174>;  // n x  2 x bf16 vector value
-def nxv4bf16 : ValueType<64,  175>;  // n x  4 x bf16 vector value
-def nxv8bf16 : ValueType<128, 176>;  // n x  8 x bf16 vector value
-def nxv16bf16 : ValueType<256, 177>;  // n x 16 x bf16 vector value
-def nxv32bf16 : ValueType<512, 178>;  // n x 32 x bf16 vector value
-
-def nxv1f32  : ValueType<32,  179>;  // n x  1 x  f32 vector value
-def nxv2f32  : ValueType<64,  180>;  // n x  2 x  f32 vector value
-def nxv4f32  : ValueType<128, 181>;  // n x  4 x  f32 vector value
-def nxv8f32  : ValueType<256, 182>;  // n x  8 x  f32 vector value
-def nxv16f32 : ValueType<512, 183>;  // n x 16 x  f32 vector value
-
-def nxv1f64  : ValueType<64,  184>;  // n x  1 x  f64 vector value
-def nxv2f64  : ValueType<128, 185>;  // n x  2 x  f64 vector value
-def nxv4f64  : ValueType<256, 186>;  // n x  4 x  f64 vector value
-def nxv8f64  : ValueType<512, 187>;  // n x  8 x  f64 vector value
+class VTAny<int value> : ValueType<0, value> {
+  let isOverloaded = true;
+}
+
+class VTInt<int size, int value>
+    : ValueType<size, value> {
+  let isInteger = true;
+}
+
+class VTFP<int size, int value>
+    : ValueType<size, value> {
+  let isFP = true;
+}
+
+class VTVec<int nelem, ValueType elt, int value>
+    : ValueType<!mul(nelem, elt.Size), value> {
+  let nElem = nelem;
+  let ElementType = elt;
+  let isInteger = elt.isInteger;
+  let isFP = elt.isFP;
+  let isVector = true;
+}
+
+class VTScalableVec<int nelem, ValueType elt, int value>
+    : VTVec<nelem, elt, value> {
+  let isScalable = true;
+}
+
+defset list<ValueType> ValueTypes = {
+
+def OtherVT : ValueType<0,   1> {  // "Other" value
+  let LLVMName = "Other";
+}
+
+def i1      : VTInt<1,   2>;  // One bit boolean value
+def i2      : VTInt<2,   3>;  // 2-bit integer value
+def i4      : VTInt<4,   4>;  // 4-bit integer value
+def i8      : VTInt<8,   5>;  // 8-bit integer value
+def i16     : VTInt<16,  6>;  // 16-bit integer value
+def i32     : VTInt<32,  7>;  // 32-bit integer value
+def i64     : VTInt<64,  8>;  // 64-bit integer value
+def i128    : VTInt<128, 9>;  // 128-bit integer value
+
+def bf16    : VTFP<16,  10>;  // 16-bit brain floating point value
+def f16     : VTFP<16,  11>;  // 16-bit floating point value
+def f32     : VTFP<32,  12>;  // 32-bit floating point value
+def f64     : VTFP<64,  13>;  // 64-bit floating point value
+def f80     : VTFP<80,  14>;  // 80-bit floating point value
+def f128    : VTFP<128, 15>;  // 128-bit floating point value
+def ppcf128 : VTFP<128, 16>;  // PPC 128-bit floating point value
+
+def v1i1    : VTVec<1,    i1, 17>;  //    1 x i1 vector value
+def v2i1    : VTVec<2,    i1, 18>;  //    2 x i1 vector value
+def v4i1    : VTVec<4,    i1, 19>;  //    4 x i1 vector value
+def v8i1    : VTVec<8,    i1, 20>;  //    8 x i1 vector value
+def v16i1   : VTVec<16,   i1, 21>;  //   16 x i1 vector value
+def v32i1   : VTVec<32,   i1, 22>;  //   32 x i1 vector value
+def v64i1   : VTVec<64,   i1, 23>;  //   64 x i1 vector value
+def v128i1  : VTVec<128,  i1, 24>;  //  128 x i1 vector value
+def v256i1  : VTVec<256,  i1, 25>;  //  256 x i1 vector value
+def v512i1  : VTVec<512,  i1, 26>;  //  512 x i1 vector value
+def v1024i1 : VTVec<1024, i1, 27>;  // 1024 x i1 vector value
+def v2048i1 : VTVec<2048, i1, 28>;  // 2048 x i1 vector value
+
+def v128i2  : VTVec<128,  i2, 29>;   //  128 x i2 vector value
+def v256i2  : VTVec<256,  i2, 30>;   //  256 x i2 vector value
+
+def v64i4   : VTVec<64,   i4, 31>;   //   64 x i4 vector value
+def v128i4  : VTVec<128,  i4, 32>;   //  128 x i4 vector value
+
+def v1i8    : VTVec<1,    i8, 33>;  //    1 x i8 vector value
+def v2i8    : VTVec<2,    i8, 34>;  //    2 x i8 vector value
+def v4i8    : VTVec<4,    i8, 35>;  //    4 x i8 vector value
+def v8i8    : VTVec<8,    i8, 36>;  //    8 x i8 vector value
+def v16i8   : VTVec<16,   i8, 37>;  //   16 x i8 vector value
+def v32i8   : VTVec<32,   i8, 38>;  //   32 x i8 vector value
+def v64i8   : VTVec<64,   i8, 39>;  //   64 x i8 vector value
+def v128i8  : VTVec<128,  i8, 40>;  //  128 x i8 vector value
+def v256i8  : VTVec<256,  i8, 41>;  //  256 x i8 vector value
+def v512i8  : VTVec<512,  i8, 42>;  //  512 x i8 vector value
+def v1024i8 : VTVec<1024, i8, 43>;  // 1024 x i8 vector value
+
+def v1i16   : VTVec<1,   i16, 44>;  //   1 x i16 vector value
+def v2i16   : VTVec<2,   i16, 45>;  //   2 x i16 vector value
+def v3i16   : VTVec<3,   i16, 46>;  //   3 x i16 vector value
+def v4i16   : VTVec<4,   i16, 47>;  //   4 x i16 vector value
+def v8i16   : VTVec<8,   i16, 48>;  //   8 x i16 vector value
+def v16i16  : VTVec<16,  i16, 49>;  //  16 x i16 vector value
+def v32i16  : VTVec<32,  i16, 50>;  //  32 x i16 vector value
+def v64i16  : VTVec<64,  i16, 51>;  //  64 x i16 vector value
+def v128i16 : VTVec<128, i16, 52>;  // 128 x i16 vector value
+def v256i16 : VTVec<256, i16, 53>;  // 256 x i16 vector value
+def v512i16 : VTVec<512, i16, 54>;  // 512 x i16 vector value
+
+def v1i32    : VTVec<1,    i32, 55>;  //    1 x i32 vector value
+def v2i32    : VTVec<2,    i32, 56>;  //    2 x i32 vector value
+def v3i32    : VTVec<3,    i32, 57>;  //    3 x i32 vector value
+def v4i32    : VTVec<4,    i32, 58>;  //    4 x i32 vector value
+def v5i32    : VTVec<5,    i32, 59>;  //    5 x i32 vector value
+def v6i32    : VTVec<6,    i32, 60>;  //    6 x f32 vector value
+def v7i32    : VTVec<7,    i32, 61>;  //    7 x f32 vector value
+def v8i32    : VTVec<8,    i32, 62>;  //    8 x i32 vector value
+def v9i32    : VTVec<9,    i32, 63>;  //    9 x i32 vector value
+def v10i32   : VTVec<10,   i32, 64>;  //   10 x i32 vector value
+def v11i32   : VTVec<11,   i32, 65>;  //   11 x i32 vector value
+def v12i32   : VTVec<12,   i32, 66>;  //   12 x i32 vector value
+def v16i32   : VTVec<16,   i32, 67>;  //   16 x i32 vector value
+def v32i32   : VTVec<32,   i32, 68>;  //   32 x i32 vector value
+def v64i32   : VTVec<64,   i32, 69>;  //   64 x i32 vector value
+def v128i32  : VTVec<128,  i32, 70>;  //  128 x i32 vector value
+def v256i32  : VTVec<256,  i32, 71>;  //  256 x i32 vector value
+def v512i32  : VTVec<512,  i32, 72>;  //  512 x i32 vector value
+def v1024i32 : VTVec<1024, i32, 73>;  // 1024 x i32 vector value
+def v2048i32 : VTVec<2048, i32, 74>;  // 2048 x i32 vector value
+
+def v1i64   : VTVec<1,   i64, 75>;  //   1 x i64 vector value
+def v2i64   : VTVec<2,   i64, 76>;  //   2 x i64 vector value
+def v3i64   : VTVec<3,   i64, 77>;  //   3 x i64 vector value
+def v4i64   : VTVec<4,   i64, 78>;  //   4 x i64 vector value
+def v8i64   : VTVec<8,   i64, 79>;  //   8 x i64 vector value
+def v16i64  : VTVec<16,  i64, 80>;  //  16 x i64 vector value
+def v32i64  : VTVec<32,  i64, 81>;  //  32 x i64 vector value
+def v64i64  : VTVec<64,  i64, 82>;  //  64 x i64 vector value
+def v128i64 : VTVec<128, i64, 83>;  // 128 x i64 vector value
+def v256i64 : VTVec<256, i64, 84>;  // 256 x i64 vector value
+
+def v1i128  : VTVec<1,  i128, 85>;  //  1 x i128 vector value
+
+def v1f16    : VTVec<1,    f16,  86>;  //    1 x f16 vector value
+def v2f16    : VTVec<2,    f16,  87>;  //    2 x f16 vector value
+def v3f16    : VTVec<3,    f16,  88>;  //    3 x f16 vector value
+def v4f16    : VTVec<4,    f16,  89>;  //    4 x f16 vector value
+def v8f16    : VTVec<8,    f16,  90>;  //    8 x f16 vector value
+def v16f16   : VTVec<16,   f16,  91>;  //   16 x f16 vector value
+def v32f16   : VTVec<32,   f16,  92>;  //   32 x f16 vector value
+def v64f16   : VTVec<64,   f16,  93>;  //   64 x f16 vector value
+def v128f16  : VTVec<128,  f16,  94>;  //  128 x f16 vector value
+def v256f16  : VTVec<256,  f16,  95>;  //  256 x f16 vector value
+def v512f16  : VTVec<512,  f16,  96>;  //  512 x f16 vector value
+
+def v2bf16   : VTVec<2,   bf16,  97>;  //    2 x bf16 vector value
+def v3bf16   : VTVec<3,   bf16,  98>;  //    3 x bf16 vector value
+def v4bf16   : VTVec<4,   bf16,  99>;  //    4 x bf16 vector value
+def v8bf16   : VTVec<8,   bf16, 100>;  //    8 x bf16 vector value
+def v16bf16  : VTVec<16,  bf16, 101>;  //   16 x bf16 vector value
+def v32bf16  : VTVec<32,  bf16, 102>;  //   32 x bf16 vector value
+def v64bf16  : VTVec<64,  bf16, 103>;  //   64 x bf16 vector value
+def v128bf16 : VTVec<128, bf16, 104>;  //  128 x bf16 vector value
+
+def v1f32    : VTVec<1,    f32, 105>;  //    1 x f32 vector value
+def v2f32    : VTVec<2,    f32, 106>;  //    2 x f32 vector value
+def v3f32    : VTVec<3,    f32, 107>;  //    3 x f32 vector value
+def v4f32    : VTVec<4,    f32, 108>;  //    4 x f32 vector value
+def v5f32    : VTVec<5,    f32, 109>;  //    5 x f32 vector value
+def v6f32    : VTVec<6,    f32, 110>;  //    6 x f32 vector value
+def v7f32    : VTVec<7,    f32, 111>;  //    7 x f32 vector value
+def v8f32    : VTVec<8,    f32, 112>;  //    8 x f32 vector value
+def v9f32    : VTVec<9,    f32, 113>;  //    9 x f32 vector value
+def v10f32   : VTVec<10,   f32, 114>;  //   10 x f32 vector value
+def v11f32   : VTVec<11,   f32, 115>;  //   11 x f32 vector value
+def v12f32   : VTVec<12,   f32, 116>;  //   12 x f32 vector value
+def v16f32   : VTVec<16,   f32, 117>;  //   16 x f32 vector value
+def v32f32   : VTVec<32,   f32, 118>;  //   32 x f32 vector value
+def v64f32   : VTVec<64,   f32, 119>;  //   64 x f32 vector value
+def v128f32  : VTVec<128,  f32, 120>;  //  128 x f32 vector value
+def v256f32  : VTVec<256,  f32, 121>;  //  256 x f32 vector value
+def v512f32  : VTVec<512,  f32, 122>;  //  512 x f32 vector value
+def v1024f32 : VTVec<1024, f32, 123>;  // 1024 x f32 vector value
+def v2048f32 : VTVec<2048, f32, 124>;  // 2048 x f32 vector value
+
+def v1f64    : VTVec<1,    f64, 125>;  //    1 x f64 vector value
+def v2f64    : VTVec<2,    f64, 126>;  //    2 x f64 vector value
+def v3f64    : VTVec<3,    f64, 127>;  //    3 x f64 vector value
+def v4f64    : VTVec<4,    f64, 128>;  //    4 x f64 vector value
+def v8f64    : VTVec<8,    f64, 129>;  //    8 x f64 vector value
+def v16f64   : VTVec<16,   f64, 130>;  //   16 x f64 vector value
+def v32f64   : VTVec<32,   f64, 131>;  //   32 x f64 vector value
+def v64f64   : VTVec<64,   f64, 132>;  //   64 x f64 vector value
+def v128f64  : VTVec<128,  f64, 133>;  //  128 x f64 vector value
+def v256f64  : VTVec<256,  f64, 134>;  //  256 x f64 vector value
+
+def nxv1i1  : VTScalableVec<1,  i1, 135>;  // n x  1 x i1  vector value
+def nxv2i1  : VTScalableVec<2,  i1, 136>;  // n x  2 x i1  vector value
+def nxv4i1  : VTScalableVec<4,  i1, 137>;  // n x  4 x i1  vector value
+def nxv8i1  : VTScalableVec<8,  i1, 138>;  // n x  8 x i1  vector value
+def nxv16i1 : VTScalableVec<16, i1, 139>;  // n x 16 x i1  vector value
+def nxv32i1 : VTScalableVec<32, i1, 140>;  // n x 32 x i1  vector value
+def nxv64i1 : VTScalableVec<64, i1, 141>;  // n x 64 x i1  vector value
+
+def nxv1i8  : VTScalableVec<1,  i8, 142>;  // n x  1 x i8  vector value
+def nxv2i8  : VTScalableVec<2,  i8, 143>;  // n x  2 x i8  vector value
+def nxv4i8  : VTScalableVec<4,  i8, 144>;  // n x  4 x i8  vector value
+def nxv8i8  : VTScalableVec<8,  i8, 145>;  // n x  8 x i8  vector value
+def nxv16i8 : VTScalableVec<16, i8, 146>;  // n x 16 x i8  vector value
+def nxv32i8 : VTScalableVec<32, i8, 147>;  // n x 32 x i8  vector value
+def nxv64i8 : VTScalableVec<64, i8, 148>;  // n x 64 x i8  vector value
+
+def nxv1i16  : VTScalableVec<1,  i16, 149>;  // n x  1 x i16 vector value
+def nxv2i16  : VTScalableVec<2,  i16, 150>;  // n x  2 x i16 vector value
+def nxv4i16  : VTScalableVec<4,  i16, 151>;  // n x  4 x i16 vector value
+def nxv8i16  : VTScalableVec<8,  i16, 152>;  // n x  8 x i16 vector value
+def nxv16i16 : VTScalableVec<16, i16, 153>;  // n x 16 x i16 vector value
+def nxv32i16 : VTScalableVec<32, i16, 154>;  // n x 32 x i16 vector value
+
+def nxv1i32  : VTScalableVec<1,  i32, 155>;  // n x  1 x i32 vector value
+def nxv2i32  : VTScalableVec<2,  i32, 156>;  // n x  2 x i32 vector value
+def nxv4i32  : VTScalableVec<4,  i32, 157>;  // n x  4 x i32 vector value
+def nxv8i32  : VTScalableVec<8,  i32, 158>;  // n x  8 x i32 vector value
+def nxv16i32 : VTScalableVec<16, i32, 159>;  // n x 16 x i32 vector value
+def nxv32i32 : VTScalableVec<32, i32, 160>;  // n x 32 x i32 vector value
+
+def nxv1i64  : VTScalableVec<1,  i64, 161>;  // n x  1 x i64 vector value
+def nxv2i64  : VTScalableVec<2,  i64, 162>;  // n x  2 x i64 vector value
+def nxv4i64  : VTScalableVec<4,  i64, 163>;  // n x  4 x i64 vector value
+def nxv8i64  : VTScalableVec<8,  i64, 164>;  // n x  8 x i64 vector value
+def nxv16i64 : VTScalableVec<16, i64, 165>;  // n x 16 x i64 vector value
+def nxv32i64 : VTScalableVec<32, i64, 166>;  // n x 32 x i64 vector value
+
+def nxv1f16  : VTScalableVec<1,  f16, 167>;  // n x  1 x  f16 vector value
+def nxv2f16  : VTScalableVec<2,  f16, 168>;  // n x  2 x  f16 vector value
+def nxv4f16  : VTScalableVec<4,  f16, 169>;  // n x  4 x  f16 vector value
+def nxv8f16  : VTScalableVec<8,  f16, 170>;  // n x  8 x  f16 vector value
+def nxv16f16 : VTScalableVec<16, f16, 171>;  // n x 16 x  f16 vector value
+def nxv32f16 : VTScalableVec<32, f16, 172>;  // n x 32 x  f16 vector value
+
+def nxv1bf16  : VTScalableVec<1,  bf16, 173>;  // n x  1 x bf16 vector value
+def nxv2bf16  : VTScalableVec<2,  bf16, 174>;  // n x  2 x bf16 vector value
+def nxv4bf16  : VTScalableVec<4,  bf16, 175>;  // n x  4 x bf16 vector value
+def nxv8bf16  : VTScalableVec<8,  bf16, 176>;  // n x  8 x bf16 vector value
+def nxv16bf16 : VTScalableVec<16, bf16, 177>;  // n x 16 x bf16 vector value
+def nxv32bf16 : VTScalableVec<32, bf16, 178>;  // n x 32 x bf16 vector value
+
+def nxv1f32  : VTScalableVec<1,  f32, 179>;  // n x  1 x  f32 vector value
+def nxv2f32  : VTScalableVec<2,  f32, 180>;  // n x  2 x  f32 vector value
+def nxv4f32  : VTScalableVec<4,  f32, 181>;  // n x  4 x  f32 vector value
+def nxv8f32  : VTScalableVec<8,  f32, 182>;  // n x  8 x  f32 vector value
+def nxv16f32 : VTScalableVec<16, f32, 183>;  // n x 16 x  f32 vector value
+
+def nxv1f64  : VTScalableVec<1,  f64, 184>;  // n x  1 x  f64 vector value
+def nxv2f64  : VTScalableVec<2,  f64, 185>;  // n x  2 x  f64 vector value
+def nxv4f64  : VTScalableVec<4,  f64, 186>;  // n x  4 x  f64 vector value
+def nxv8f64  : VTScalableVec<8,  f64, 187>;  // n x  8 x  f64 vector value
 
 def x86mmx    : ValueType<64,   188>;  // X86 MMX value
-def FlagVT    : ValueType<0,    189>;  // Pre-RA sched glue
+def FlagVT    : ValueType<0,    189> { // Pre-RA sched glue
+  let LLVMName = "Glue";
+}
 def isVoid    : ValueType<0,    190>;  // Produces no value
-def untyped   : ValueType<8,    191>;  // Produces an untyped value
+def untyped   : ValueType<8,    191> { // Produces an untyped value
+  let LLVMName = "Untyped";
+}
 def funcref   : ValueType<0,    192>;  // WebAssembly's funcref type
 def externref : ValueType<0,    193>;  // WebAssembly's externref type
 def x86amx    : ValueType<8192, 194>;  // X86 AMX value
@@ -241,26 +286,30 @@ def aarch64svcount
 def spirvbuiltin : ValueType<0,  197>; // SPIR-V's builtin type
 
 def token      : ValueType<0, 248>;  // TokenTy
-def MetadataVT : ValueType<0, 249>;  // Metadata
+def MetadataVT : ValueType<0, 249> { // Metadata
+  let LLVMName = "Metadata";
+}
 
 // Pseudo valuetype mapped to the current pointer size to any address space.
 // Should only be used in TableGen.
-def iPTRAny    : ValueType<0, 250>;
+def iPTRAny    : VTAny<250>;
 
 // Pseudo valuetype to represent "vector of any size"
-def vAny       : ValueType<0, 251>;
+def vAny       : VTAny<251>;
 
 // Pseudo valuetype to represent "float of any format"
-def fAny       : ValueType<0, 252>;
+def fAny       : VTAny<252>;
 
 // Pseudo valuetype to represent "integer of any bit width"
-def iAny       : ValueType<0, 253>;
+def iAny       : VTAny<253>;
 
 // Pseudo valuetype mapped to the current pointer size.
 def iPTR       : ValueType<0, 254>;
 
 // Pseudo valuetype to represent "any type of any size".
-def Any        : ValueType<0, 255>;
+def Any        : VTAny<255>;
+
+} // end defset ValueTypes
 
 /// This class is for targets that want to use pointer types in patterns
 /// with the GlobalISelEmitter.  Targets must define their own pointer
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/DWARFLinker.h
index 06a0696c03c93..a090ea43f1d33 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinker.h
+++ b/llvm/include/llvm/DWARFLinker/DWARFLinker.h
@@ -49,17 +49,23 @@ class AddressesMap {
   /// section.
   virtual bool hasValidRelocs() = 0;
 
-  /// Checks that the specified variable \p DIE references live code section.
-  /// Allowed kind of input die: DW_TAG_variable, DW_TAG_constant.
-  /// \returns true and sets Info.InDebugMap if it is the case.
-  virtual bool isLiveVariable(const DWARFDie &DIE,
-                              CompileUnit::DIEInfo &Info) = 0;
-
-  /// Checks that the specified subprogram \p DIE references live code section.
-  /// Allowed kind of input die: DW_TAG_subprogram, DW_TAG_label.
-  /// \returns true and sets Info.InDebugMap if it is the case.
-  virtual bool isLiveSubprogram(const DWARFDie &DIE,
-                                CompileUnit::DIEInfo &Info) = 0;
+  /// Checks that the specified variable \p DIE references the live code
+  /// section and returns the relocation adjustment value (to get the linked
+  /// address this value might be added to the source variable address).
+  /// Allowed kinds of input DIE: DW_TAG_variable, DW_TAG_constant.
+  /// \returns relocation adjustment value or std::nullopt if there is no
+  /// corresponding live address.
+  virtual std::optional<int64_t>
+  getVariableRelocAdjustment(const DWARFDie &DIE) = 0;
+
+  /// Checks that the specified subprogram \p DIE references the live code
+  /// section and returns the relocation adjustment value (to get the linked
+  /// address this value might be added to the source subprogram address).
+  /// Allowed kinds of input DIE: DW_TAG_subprogram, DW_TAG_label.
+  /// \returns relocation adjustment value or std::nullopt if there is no
+  /// corresponding live address.
+  virtual std::optional<int64_t>
+  getSubprogramRelocAdjustment(const DWARFDie &DIE) = 0;
 
   /// Apply the valid relocations to the buffer \p Data, taking into
   /// account that Data is at \p BaseOffset in the .debug_info section.
diff --git a/llvm/include/llvm/Demangle/Demangle.h b/llvm/include/llvm/Demangle/Demangle.h
index 855ab1f57512f..f41fdc2c0cba3 100644
--- a/llvm/include/llvm/Demangle/Demangle.h
+++ b/llvm/include/llvm/Demangle/Demangle.h
@@ -28,8 +28,10 @@ enum : int {
   demangle_success = 0,
 };
 
-char *itaniumDemangle(const char *mangled_name, char *buf, size_t *n,
-                      int *status);
+/// Returns a non-NULL pointer to a NUL-terminated C style string
+/// that should be explicitly freed, if successful. Otherwise, may return
+/// nullptr if mangled_name is not a valid mangling or is nullptr.
+char *itaniumDemangle(const char *mangled_name);
 
 enum MSDemangleFlags {
   MSDF_None = 0,
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index 428f0646d90ee..6eaf31e0dea44 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -2330,17 +2330,14 @@ template <class Float> class FloatLiteralImpl : public Node {
   template<typename Fn> void match(Fn F) const { F(Contents); }
 
   void printLeft(OutputBuffer &OB) const override {
-    const char *first = &*Contents.begin();
-    const char *last = &*Contents.end() + 1;
-
     const size_t N = FloatData<Float>::mangled_size;
-    if (static_cast<std::size_t>(last - first) > N) {
-      last = first + N;
+    if (Contents.size() >= N) {
       union {
         Float value;
         char buf[sizeof(Float)];
       };
-      const char *t = first;
+      const char *t = &*Contents.begin();
+      const char *last = t + N;
       char *e = buf;
       for (; t != last; ++t, ++e) {
         unsigned d1 = isdigit(*t) ? static_cast<unsigned>(*t - '0')
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
index 33c09e1a2a315..b8d83ccaddc60 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h
@@ -107,6 +107,28 @@ enum EdgeKind_aarch64 : Edge::Kind {
   ///     out-of-range error will be returned.
   Branch26PCRel,
 
+  /// A 19-bit PC-relative conditional branch.
+  ///
+  /// Represents a PC-relative conditional branch to a target within +/-1Mb. The
+  /// target must be 32-bit aligned.
+  ///
+  /// Fixup expression:
+  ///   Fixup <- (Target - Fixup + Addend) >> 2 : int19
+  ///
+  /// Notes:
+  ///   The '19' in the name refers to the number operand bits and follows the
+  /// naming convention used by the corresponding ELF relocation.
+  /// Since the low two bits must be zero (because of the 32-bit alignment of
+  /// the target) the operand is effectively a signed 21-bit number.
+  ///
+  ///
+  /// Errors:
+  ///   - The result of the unshifted part of the fixup expression must be
+  ///     32-bit aligned otherwise an alignment error will be returned.
+  ///   - The result of the fixup expression must fit into an int19 otherwise an
+  ///     out-of-range error will be returned.
+  CondBranch19PCRel,
+
   /// A 16-bit slice of the target address (which slice depends on the
   /// instruction at the fixup location).
   ///
@@ -299,6 +321,16 @@ inline bool isLoadStoreImm12(uint32_t Instr) {
   return (Instr & LoadStoreImm12Mask) == 0x39000000;
 }
 
+inline bool isCondBranchImm19(uint32_t Instr) {
+  constexpr uint32_t CondBranchImm19Mask = 0xfe000000;
+  return (Instr & CondBranchImm19Mask) == 0x54000000;
+}
+
+inline bool isCompAndBranchImm19(uint32_t Instr) {
+  constexpr uint32_t CompAndBranchImm19Mask = 0x7e000000;
+  return (Instr & CompAndBranchImm19Mask) == 0x34000000;
+}
+
 // Returns the amount the address operand of LD/ST (imm12)
 // should be shifted right by.
 //
@@ -431,6 +463,24 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
     *(ulittle32_t *)FixupPtr = FixedInstr;
     break;
   }
+  case CondBranch19PCRel: {
+    assert((FixupAddress.getValue() & 0x3) == 0 &&
+           "Conditional branch is not 32-bit aligned");
+    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+    assert((isCondBranchImm19(RawInstr) || isCompAndBranchImm19(RawInstr)) &&
+           "RawInstr is not a conditional branch");
+    int64_t Delta = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
+    if (Delta & 0x3)
+      return make_error<JITLinkError>(
+          "Conditional branch literal target is not 32-bit "
+          "aligned");
+    if (!isInt<21>(Delta))
+      return makeTargetOutOfRangeError(G, B, E);
+    uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;
+    uint32_t FixedInstr = RawInstr | EncodedImm;
+    *(ulittle32_t *)FixupPtr = FixedInstr;
+    break;
+  }
   case Page21: {
     uint64_t TargetPage =
         (E.getTarget().getAddress().getValue() + E.getAddend()) &
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index fe5af4cb0c428..6e94302fab31c 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -295,8 +295,7 @@ def OMPC_From : Clause<"from"> {
 }
 def OMPC_UseDevicePtr : Clause<"use_device_ptr"> {
   let clangClass = "OMPUseDevicePtrClause";
-  let flangClass = "Name";
-  let isValueList = true;
+  let flangClass = "OmpObjectList";
 }
 def OMPC_IsDevicePtr : Clause<"is_device_ptr"> {
   let clangClass = "OMPIsDevicePtrClause";
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index f9ccad0a1d18c..d5d1646fb5a37 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1445,6 +1445,49 @@ class OpenMPIRBuilder {
     bool separateBeginEndCalls() { return SeparateBeginEndCalls; }
   };
 
+  using MapValuesArrayTy = SmallVector<Value *, 4>;
+  using MapFlagsArrayTy = SmallVector<omp::OpenMPOffloadMappingFlags, 4>;
+  using MapNamesArrayTy = SmallVector<Constant *, 4>;
+  using MapDimArrayTy = SmallVector<uint64_t, 4>;
+  using MapNonContiguousArrayTy = SmallVector<MapValuesArrayTy, 4>;
+
+  /// This structure contains combined information generated for mappable
+  /// clauses, including base pointers, pointers, sizes, map types, user-defined
+  /// mappers, and non-contiguous information.
+  struct MapInfosTy {
+    struct StructNonContiguousInfo {
+      bool IsNonContiguous = false;
+      MapDimArrayTy Dims;
+      MapNonContiguousArrayTy Offsets;
+      MapNonContiguousArrayTy Counts;
+      MapNonContiguousArrayTy Strides;
+    };
+    MapValuesArrayTy BasePointers;
+    MapValuesArrayTy Pointers;
+    MapValuesArrayTy Sizes;
+    MapFlagsArrayTy Types;
+    MapNamesArrayTy Names;
+    StructNonContiguousInfo NonContigInfo;
+
+    /// Append arrays in \a CurInfo.
+    void append(MapInfosTy &CurInfo) {
+      BasePointers.append(CurInfo.BasePointers.begin(),
+                          CurInfo.BasePointers.end());
+      Pointers.append(CurInfo.Pointers.begin(), CurInfo.Pointers.end());
+      Sizes.append(CurInfo.Sizes.begin(), CurInfo.Sizes.end());
+      Types.append(CurInfo.Types.begin(), CurInfo.Types.end());
+      Names.append(CurInfo.Names.begin(), CurInfo.Names.end());
+      NonContigInfo.Dims.append(CurInfo.NonContigInfo.Dims.begin(),
+                                CurInfo.NonContigInfo.Dims.end());
+      NonContigInfo.Offsets.append(CurInfo.NonContigInfo.Offsets.begin(),
+                                   CurInfo.NonContigInfo.Offsets.end());
+      NonContigInfo.Counts.append(CurInfo.NonContigInfo.Counts.begin(),
+                                  CurInfo.NonContigInfo.Counts.end());
+      NonContigInfo.Strides.append(CurInfo.NonContigInfo.Strides.begin(),
+                                   CurInfo.NonContigInfo.Strides.end());
+    }
+  };
+
   /// Emit the arguments to be passed to the runtime library based on the
   /// arrays of base pointers, pointers, sizes, map types, and mappers.  If
   /// ForEndCall, emit map types to be passed for the end of the region instead
@@ -1824,6 +1867,27 @@ class OpenMPIRBuilder {
       Value *IfCond, BodyGenCallbackTy ProcessMapOpCB,
       BodyGenCallbackTy BodyGenCB = {});
 
+  using TargetBodyGenCallbackTy = function_ref<InsertPointTy(
+      InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
+
+  /// Generator for '#omp target'
+  ///
+  /// \param Loc where the target data construct was encountered.
+  /// \param CodeGenIP The insertion point where the call to the outlined
+  /// function should be emitted.
+  /// \param EntryInfo The entry information about the function.
+  /// \param NumTeams Number of teams specified in the num_teams clause.
+  /// \param NumThreads Number of teams specified in the thread_limit clause.
+  /// \param Inputs The input values to the region that will be passed.
+  /// as arguments to the outlined function.
+  /// \param BodyGenCB Callback that will generate the region code.
+  InsertPointTy createTarget(const LocationDescription &Loc,
+                             OpenMPIRBuilder::InsertPointTy CodeGenIP,
+                             TargetRegionEntryInfo &EntryInfo, int32_t NumTeams,
+                             int32_t NumThreads,
+                             SmallVectorImpl<Value *> &Inputs,
+                             TargetBodyGenCallbackTy BodyGenCB);
+
   /// Declarations for LLVM-IR types (simple, array, function and structure) are
   /// generated below. Their names are defined and used in OpenMPKinds.def. Here
   /// we provide the declarations, the initializeTypes function will provide the
diff --git a/llvm/include/llvm/FuzzMutate/IRMutator.h b/llvm/include/llvm/FuzzMutate/IRMutator.h
index 960024652d394..e01b91106fec1 100644
--- a/llvm/include/llvm/FuzzMutate/IRMutator.h
+++ b/llvm/include/llvm/FuzzMutate/IRMutator.h
@@ -81,6 +81,7 @@ class InjectorIRStrategy : public IRMutationStrategy {
                                                         RandomIRBuilder &IB);
 
 public:
+  InjectorIRStrategy() : Operations(getDefaultOps()) {}
   InjectorIRStrategy(std::vector<fuzzerop::OpDescriptor> &&Operations)
       : Operations(std::move(Operations)) {}
   static std::vector<fuzzerop::OpDescriptor> getDefaultOps();
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index e72054cfd475e..aba1d718f7f72 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -41,6 +41,9 @@ class TypeAttr<string S, list<AttrProperty> P> : Attr<S, P>;
 /// StringBool attribute.
 class StrBoolAttr<string S> : Attr<S, []>;
 
+/// Arbitrary string attribute.
+class ComplexStrAttr<string S, list<AttrProperty> P> : Attr<S, P>;
+
 /// Target-independent enum attributes.
 
 /// Alignment of parameter (5 bits) stored as log2 of alignment with +1 bias.
@@ -321,6 +324,9 @@ def NoInlineLineTables : StrBoolAttr<"no-inline-line-tables">;
 def ProfileSampleAccurate : StrBoolAttr<"profile-sample-accurate">;
 def UseSampleProfile : StrBoolAttr<"use-sample-profile">;
 
+def DenormalFPMath : ComplexStrAttr<"denormal-fp-math", [FnAttr]>;
+def DenormalFPMathF32 : ComplexStrAttr<"denormal-fp-math-f32", [FnAttr]>;
+
 class CompatRule<string F> {
   // The name of the function called to check the attribute of the caller and
   // callee and decide whether inlining should be allowed. The function's
@@ -340,6 +346,8 @@ def : CompatRule<"isEqual<SafeStackAttr>">;
 def : CompatRule<"isEqual<ShadowCallStackAttr>">;
 def : CompatRule<"isEqual<UseSampleProfileAttr>">;
 def : CompatRule<"isEqual<NoProfileAttr>">;
+def : CompatRule<"checkDenormMode">;
+
 
 class MergeRule<string F> {
   // The name of the function called to merge the attributes of the caller and
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index 86bdc241a5ffe..8b514528daf38 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -654,6 +654,15 @@ class LLVM_EXTERNAL_VISIBILITY Function : public GlobalObject,
   /// function.
   DenormalMode getDenormalMode(const fltSemantics &FPType) const;
 
+  /// Return the representational value of "denormal-fp-math". Code interested
+  /// in the semantics of the function should use getDenormalMode instead.
+  DenormalMode getDenormalModeRaw() const;
+
+  /// Return the representational value of "denormal-fp-math-f32". Code
+  /// interested in the semantics of the function should use getDenormalMode
+  /// instead.
+  DenormalMode getDenormalModeF32Raw() const;
+
   /// copyAttributesFrom - copy all additional attributes (those not needed to
   /// create a Function) from the Function Src to this one.
   void copyAttributesFrom(const Function *Src);
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 980a46073d493..53aff55cc2c4f 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1559,6 +1559,11 @@ class CallBase : public Instruction {
     Attrs = Attrs.removeFnAttribute(getContext(), Kind);
   }
 
+  /// Removes the attribute from the function
+  void removeFnAttr(StringRef Kind) {
+    Attrs = Attrs.removeFnAttribute(getContext(), Kind);
+  }
+
   /// Removes the attribute from the return value
   void removeRetAttr(Attribute::AttrKind Kind) {
     Attrs = Attrs.removeRetAttribute(getContext(), Kind);
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 46202e4570ff4..d7b1568ea2c01 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -2002,7 +2002,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)
 //                           ShuffleVectorInst Class
 //===----------------------------------------------------------------------===//
 
-constexpr int UndefMaskElem = -1;
+constexpr int PoisonMaskElem = -1;
 
 /// This instruction constructs a fixed permutation of two
 /// input vectors.
@@ -2010,7 +2010,7 @@ constexpr int UndefMaskElem = -1;
 /// For each element of the result vector, the shuffle mask selects an element
 /// from one of the input vectors to copy to the result. Non-negative elements
 /// in the mask represent an index into the concatenated pair of input vectors.
-/// UndefMaskElem (-1) specifies that the result element is undefined.
+/// PoisonMaskElem (-1) specifies that the result element is poison.
 ///
 /// For scalable vectors, all the elements of the mask must be 0 or -1. This
 /// requirement may be relaxed in the future.
@@ -2068,16 +2068,16 @@ class ShuffleVectorInst : public Instruction {
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
   /// Return the shuffle mask value of this instruction for the given element
-  /// index. Return UndefMaskElem if the element is undef.
+  /// index. Return PoisonMaskElem if the element is undef.
   int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
 
   /// Convert the input shuffle mask operand to a vector of integers. Undefined
-  /// elements of the mask are returned as UndefMaskElem.
+  /// elements of the mask are returned as PoisonMaskElem.
   static void getShuffleMask(const Constant *Mask,
                              SmallVectorImpl<int> &Result);
 
   /// Return the mask for this instruction as a vector of integers. Undefined
-  /// elements of the mask are returned as UndefMaskElem.
+  /// elements of the mask are returned as PoisonMaskElem.
   void getShuffleMask(SmallVectorImpl<int> &Result) const {
     Result.assign(ShuffleMask.begin(), ShuffleMask.end());
   }
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 9b349c8846480..62bd833198f02 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -251,8 +251,16 @@ class RawLocationWrapper {
   }
   bool hasArgList() const { return isa<DIArgList>(getRawLocation()); }
   bool isKillLocation(const DIExpression *Expression) const {
-    return (getNumVariableLocationOps() == 0 && !Expression->isComplex()) ||
-           any_of(location_ops(), [](Value *V) { return isa<UndefValue>(V); });
+    // Check for "kill" sentinel values.
+    // Non-variadic: empty metadata.
+    if (!hasArgList() && isa<MDNode>(getRawLocation()))
+      return true;
+    // Variadic: empty DIArgList with empty expression.
+    if (getNumVariableLocationOps() == 0 && !Expression->isComplex())
+      return true;
+    // Variadic and non-variadic: Interpret expressions using undef or poison
+    // values as kills.
+    return any_of(location_ops(), [](Value *V) { return isa<UndefValue>(V); });
   }
 
   friend bool operator==(const RawLocationWrapper &A,
@@ -588,8 +596,17 @@ class VPIntrinsic : public IntrinsicInst {
     return getFunctionalOpcodeForVP(getIntrinsicID());
   }
 
+  // Equivalent non-predicated constrained ID
+  std::optional<unsigned> getConstrainedIntrinsicID() const {
+    return getConstrainedIntrinsicIDForVP(getIntrinsicID());
+  }
+
   // Equivalent non-predicated opcode
   static std::optional<unsigned> getFunctionalOpcodeForVP(Intrinsic::ID ID);
+
+  // Equivalent non-predicated constrained ID
+  static std::optional<unsigned>
+  getConstrainedIntrinsicIDForVP(Intrinsic::ID ID);
 };
 
 /// This represents vector predication reduction intrinsics.
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 2b9436d326b79..b8666b52ecec4 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -182,29 +182,234 @@ def ArgKind {
   int MatchType  = 7;
 }
 
+// Encode placeholder.
+// [15:8] is the ID used how to resolve ArgCode.
+
+// (ACIdx << 3) | ArgCode
+class EncAnyType<int ArgCode=0> {
+  int ID = 0x100;
+  int ret = !or(ID, ArgCode);
+}
+
+// (Mapping[Num] << 3) | AK.MatchType
+class EncMatchType<int Num=0> {
+  int ID = 0x200;
+  int ret = !or(ID, Num);
+}
+
+// (Mapping[Num] << 3) | ArgCodes[Mapping[Num]]
+class EncSameWidth<int Num=0> {
+  int ID = 0x300;
+  int ret = !or(ID, Num);
+}
+
+// ACIdx
+class EncNextArgA<int dummy=0> {
+  int ID = 0x400;
+  int ret = !or(ID, dummy);
+}
+
+// Mapping[Num]
+class EncNextArgN<int Num=0> {
+  int ID = 0x500;
+  int ret = !or(ID, Num);
+}
+
+class ResolveArgCode<
+    list<int> Mapping,
+    list<int> ArgCodes,
+    int ACIdx,
+    int ax> {
+  int ah = !and(ax, 0xFF00);
+  int al = !and(ax, 0x00FF);
+  int num = Mapping[al];
+  int ret = !cond(
+    !eq(ah, EncAnyType<>.ID)   : !or(!shl(ACIdx, 3), al),
+    !eq(ah, EncMatchType<>.ID) : !or(!shl(num, 3), ArgKind.MatchType),
+    !eq(ah, EncSameWidth<>.ID) : !or(!shl(num, 3), ArgCodes[num]),
+    !eq(ah, EncNextArgA<>.ID)  : ACIdx,
+    !eq(ah, EncNextArgN<>.ID)  : num,
+    true : al);
+}
+
+//===----------------------------------------------------------------------===//
+// IIT_Info
+//===----------------------------------------------------------------------===//
+
+class IIT_Base<int num> {
+  int Number = num;
+  list<ValueType> VTs = ?;
+}
+
+class IIT_VT<ValueType vt, int num> : IIT_Base<num> {
+  let VTs = [vt];
+}
+
+class IIT_Int<int size, int num> : IIT_Base<num> {
+  let VTs = !filter(vti, ValueTypes,
+    !and(vti.isInteger, !eq(vti.Size, size)));
+}
+
+class IIT_Vec<int nelem, int num> : IIT_Base<num> {
+  let VTs = !filter(vti, ValueTypes,
+    !and(vti.isVector, !eq(vti.nElem, nelem)));
+}
+
+defset list<IIT_Base> IIT_all = {
+def IIT_Done : IIT_Base<    0>;
+def IIT_I1   : IIT_Int<1,   1>;
+def IIT_I8   : IIT_Int<8,   2>;
+def IIT_I16  : IIT_Int<16,  3>;
+def IIT_I32  : IIT_Int<32,  4>;
+def IIT_I64  : IIT_Int<64,  5>;
+def IIT_F16  : IIT_VT<f16,  6>;
+def IIT_F32  : IIT_VT<f32,  7>;
+def IIT_F64  : IIT_VT<f64,  8>;
+def IIT_V2   : IIT_Vec<2,   9>;
+def IIT_V4   : IIT_Vec<4,  10>;
+def IIT_V8   : IIT_Vec<8,  11>;
+def IIT_V16  : IIT_Vec<16, 12>;
+def IIT_V32  : IIT_Vec<32, 13>;
+def IIT_PTR  : IIT_Base<   14>;
+def IIT_ARG  : IIT_Base<   15>;
+
+def IIT_V64 : IIT_Vec<64, 16>;
+def IIT_MMX : IIT_VT<x86mmx, 17>;
+def IIT_TOKEN : IIT_VT<token, 18>;
+def IIT_METADATA : IIT_VT<MetadataVT, 19>;
+def IIT_EMPTYSTRUCT : IIT_VT<OtherVT, 20>;
+def IIT_STRUCT2 : IIT_Base<21>;
+def IIT_STRUCT3 : IIT_Base<22>;
+def IIT_STRUCT4 : IIT_Base<23>;
+def IIT_STRUCT5 : IIT_Base<24>;
+def IIT_EXTEND_ARG : IIT_Base<25>;
+def IIT_TRUNC_ARG : IIT_Base<26>;
+def IIT_ANYPTR : IIT_Base<27>;
+def IIT_V1 : IIT_Vec<1, 28>;
+def IIT_VARARG : IIT_VT<isVoid, 29>;
+def IIT_HALF_VEC_ARG : IIT_Base<30>;
+def IIT_SAME_VEC_WIDTH_ARG : IIT_Base<31>;
+def IIT_PTR_TO_ARG : IIT_Base<32>;
+def IIT_PTR_TO_ELT : IIT_Base<33>;
+def IIT_VEC_OF_ANYPTRS_TO_ELT : IIT_Base<34>;
+def IIT_I128 : IIT_Int<128, 35>;
+def IIT_V512 : IIT_Vec<512, 36>;
+def IIT_V1024 : IIT_Vec<1024, 37>;
+def IIT_STRUCT6 : IIT_Base<38>;
+def IIT_STRUCT7 : IIT_Base<39>;
+def IIT_STRUCT8 : IIT_Base<40>;
+def IIT_F128 : IIT_VT<f128, 41>;
+def IIT_VEC_ELEMENT : IIT_Base<42>;
+def IIT_SCALABLE_VEC : IIT_Base<43>;
+def IIT_SUBDIVIDE2_ARG : IIT_Base<44>;
+def IIT_SUBDIVIDE4_ARG : IIT_Base<45>;
+def IIT_VEC_OF_BITCASTS_TO_INT : IIT_Base<46>;
+def IIT_V128 : IIT_Vec<128, 47>;
+def IIT_BF16 : IIT_VT<bf16, 48>;
+def IIT_STRUCT9 : IIT_Base<49>;
+def IIT_V256 : IIT_Vec<256, 50>;
+def IIT_AMX : IIT_VT<x86amx, 51>;
+def IIT_PPCF128 : IIT_VT<ppcf128, 52>;
+def IIT_V3 : IIT_Vec<3, 53>;
+def IIT_EXTERNREF : IIT_VT<externref, 54>;
+def IIT_FUNCREF : IIT_VT<funcref, 55>;
+def IIT_ANYPTR_TO_ELT : IIT_Base<56>;
+def IIT_I2 : IIT_Int<2, 57>;
+def IIT_I4 : IIT_Int<4, 58>;
+}
+
+defvar IIT_all_FixedTypes = !filter(iit, IIT_all,
+  !or(!isa<IIT_VT>(iit), !isa<IIT_Int>(iit)));
+
+defvar IIT_all_VectorTypes = !filter(iit, IIT_all,
+  !isa<IIT_Vec>(iit));
+
+defvar IIT_RetNumbers = [
+  [IIT_Done.Number],
+  []<int>,
+  [IIT_STRUCT2.Number],
+  [IIT_STRUCT3.Number],
+  [IIT_STRUCT4.Number],
+  [IIT_STRUCT5.Number],
+  [IIT_STRUCT6.Number],
+  [IIT_STRUCT7.Number],
+  [IIT_STRUCT8.Number],
+  [IIT_STRUCT9.Number],
+];
+
 //===----------------------------------------------------------------------===//
 // Types used by intrinsics.
 //===----------------------------------------------------------------------===//
 
 class LLVMType<ValueType vt> {
   ValueType VT = vt;
-  int isAny = false;
+  int isAny = vt.isOverloaded;
+
+  int ArgCode = ?;
+  int Number = ?;
+
+  list<IIT_Base> IITs = !filter(iit, IIT_all_FixedTypes,
+    !not(!empty(!filter(iit_vt, iit.VTs,
+      !eq(iit_vt, !if(vt.isVector, vt.ElementType, vt))))));
+  assert !le(!size(IITs), 1), "Duplicate type";
+
+  list<IIT_Base> IIT_Vecs = !if(vt.isVector,
+    !filter(iit, IIT_all_VectorTypes,
+      !not(!empty(!filter(iit_vt, iit.VTs, !and(
+        !eq(iit_vt.ElementType, vt.ElementType),
+        !eq(iit_vt.nElem, vt.nElem)))))),
+    []);
+  assert !le(!size(IIT_Vecs), 1), "Duplicate type";
+
+  list<int> Sig = !listconcat(
+    !if(vt.isScalable, [IIT_SCALABLE_VEC.Number], []),
+    !foreach(iit, IIT_Vecs, iit.Number),
+    !foreach(iit, IITs,     iit.Number));
+}
+
+class LLVMAnyType<ValueType vt> : LLVMType<vt> {
+  let ArgCode = !cond(
+    !eq(vt, Any)     : ArgKind.Any,
+    !eq(vt, iAny)    : ArgKind.AnyInteger,
+    !eq(vt, fAny)    : ArgKind.AnyFloat,
+    !eq(vt, vAny)    : ArgKind.AnyVector,
+    !eq(vt, iPTRAny) : ArgKind.AnyPointer,
+  );
+  let Sig = [
+    IIT_ARG.Number,
+    EncAnyType<ArgCode>.ret,
+  ];
+
+  assert isAny, "LLVMAnyType.VT should have isOverloaded";
 }
 
 class LLVMQualPointerType<LLVMType elty, int addrspace>
   : LLVMType<iPTR>{
   LLVMType ElTy = elty;
-  int AddrSpace = addrspace;
+  assert !and(!le(0, addrspace), !le(addrspace, 255)),
+    "Address space exceeds 255";
+
+  // D63507: LLVMPointerType<llvm_any_ty>
+  let isAny = elty.isAny;
+
+  let Sig = !listconcat(
+    !if(addrspace, [
+      IIT_ANYPTR.Number,
+      addrspace,
+    ], [
+      IIT_PTR.Number,
+    ]),
+    ElTy.Sig);
 }
 
 class LLVMPointerType<LLVMType elty>
   : LLVMQualPointerType<elty, 0>;
 
 class LLVMAnyPointerType<LLVMType elty>
-  : LLVMType<iPTRAny>{
+  : LLVMAnyType<iPTRAny> {
   LLVMType ElTy = elty;
 
-  let isAny = true;
+  assert isAny, "iPTRAny should have isOverloaded";
 }
 
 // Match the type of another intrinsic parameter.  Number is an index into the
@@ -213,53 +418,75 @@ class LLVMAnyPointerType<LLVMType elty>
 //   Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_anyfloat_ty, LLVMMatchType<0>]>
 // has two overloaded types, the 2nd and 3rd arguments.  LLVMMatchType<0>
 // refers to the first overloaded type, which is the 2nd argument.
-class LLVMMatchType<int num>
+class LLVMMatchType<int num, IIT_Base IIT_Info = IIT_ARG>
   : LLVMType<OtherVT>{
-  int Number = num;
+  let Number = num;
+  let Sig = [
+    IIT_Info.Number,
+    EncMatchType<num>.ret,
+  ];
+}
+
+class LLVMMatchTypeNextArg<int num, IIT_Base IIT_Info>
+  : LLVMMatchType<num, IIT_Info> {
+  let Sig = [
+    IIT_Info.Number,
+    EncNextArgA<>.ret,
+    EncNextArgN<num>.ret,
+  ];
 }
 
 // Match the type of another intrinsic parameter that is expected to be based on
 // an integral type (i.e. either iN or <N x iM>), but change the scalar size to
 // be twice as wide or half as wide as the other type.  This is only useful when
 // the intrinsic is overloaded, so the matched type should be declared as iAny.
-class LLVMExtendedType<int num> : LLVMMatchType<num>;
-class LLVMTruncatedType<int num> : LLVMMatchType<num>;
+class LLVMExtendedType<int num> : LLVMMatchType<num, IIT_EXTEND_ARG>;
+class LLVMTruncatedType<int num> : LLVMMatchType<num, IIT_TRUNC_ARG>;
 
 // Match the scalar/vector of another intrinsic parameter but with a different
 // element type. Either both are scalars or both are vectors with the same
 // number of elements.
 class LLVMScalarOrSameVectorWidth<int idx, LLVMType elty>
-  : LLVMMatchType<idx> {
-  ValueType ElTy = elty.VT;
+  : LLVMMatchType<idx, IIT_SAME_VEC_WIDTH_ARG> {
+  let Sig = !listconcat([
+    IIT_SAME_VEC_WIDTH_ARG.Number,
+    EncSameWidth<idx>.ret,
+  ], elty.Sig);
 }
 
-class LLVMPointerTo<int num> : LLVMMatchType<num>;
-class LLVMPointerToElt<int num> : LLVMMatchType<num>;
-class LLVMAnyPointerToElt<int num> : LLVMMatchType<num>;
-class LLVMVectorOfAnyPointersToElt<int num> : LLVMMatchType<num>;
-class LLVMVectorElementType<int num> : LLVMMatchType<num>;
+class LLVMPointerTo<int num> : LLVMMatchType<num, IIT_PTR_TO_ARG>;
+class LLVMPointerToElt<int num> : LLVMMatchType<num, IIT_PTR_TO_ELT>;
+class LLVMAnyPointerToElt<int num>
+  : LLVMMatchTypeNextArg<num, IIT_ANYPTR_TO_ELT>;
+class LLVMVectorOfAnyPointersToElt<int num>
+  : LLVMMatchTypeNextArg<num, IIT_VEC_OF_ANYPTRS_TO_ELT>;
+class LLVMVectorElementType<int num> : LLVMMatchType<num, IIT_VEC_ELEMENT>;
 
 // Match the type of another intrinsic parameter that is expected to be a
 // vector type, but change the element count to be half as many.
-class LLVMHalfElementsVectorType<int num> : LLVMMatchType<num>;
+class LLVMHalfElementsVectorType<int num>
+  : LLVMMatchType<num, IIT_HALF_VEC_ARG>;
 
 // Match the type of another intrinsic parameter that is expected to be a
 // vector type (i.e. <N x iM>) but with each element subdivided to
 // form a vector with more elements that are smaller than the original.
-class LLVMSubdivide2VectorType<int num> : LLVMMatchType<num>;
-class LLVMSubdivide4VectorType<int num> : LLVMMatchType<num>;
+class LLVMSubdivide2VectorType<int num>
+  : LLVMMatchType<num, IIT_SUBDIVIDE2_ARG>;
+class LLVMSubdivide4VectorType<int num>
+  : LLVMMatchType<num, IIT_SUBDIVIDE4_ARG>;
 
 // Match the element count and bit width of another intrinsic parameter, but
 // change the element type to an integer.
-class LLVMVectorOfBitcastsToInt<int num> : LLVMMatchType<num>;
+class LLVMVectorOfBitcastsToInt<int num>
+  : LLVMMatchType<num, IIT_VEC_OF_BITCASTS_TO_INT>;
 
 def llvm_void_ty       : LLVMType<isVoid>;
-let isAny = true in {
-  def llvm_any_ty        : LLVMType<Any>;
-  def llvm_anyint_ty     : LLVMType<iAny>;
-  def llvm_anyfloat_ty   : LLVMType<fAny>;
-  def llvm_anyvector_ty  : LLVMType<vAny>;
-}
+
+def llvm_any_ty        : LLVMAnyType<Any>;
+def llvm_anyint_ty     : LLVMAnyType<iAny>;
+def llvm_anyfloat_ty   : LLVMAnyType<fAny>;
+def llvm_anyvector_ty  : LLVMAnyType<vAny>;
+
 def llvm_i1_ty         : LLVMType<i1>;
 def llvm_i8_ty         : LLVMType<i8>;
 def llvm_i16_ty        : LLVMType<i16>;
@@ -363,6 +590,60 @@ def llvm_vararg_ty     : LLVMType<isVoid>;   // this means vararg here
 def llvm_externref_ty  : LLVMType<externref>;
 def llvm_funcref_ty    : LLVMType<funcref>;
 
+//===----------------------------------------------------------------------===//
+
+class MakeIdx<list<int> Set> {
+  list<int> IdxsR = !foreach(i, !range(Set),
+    !if(Set[i],
+      !foldl(0, !range(0, i), m, j, !add(m, Set[j])),
+      -1));
+
+  list<int> RIdxsR = !foreach(i, !range(Set),
+    !foldl(-1, !range(Set), m, j,
+      !if(!and(Set[j], !eq(IdxsR[j], i)), j, m)));
+
+  list<int> Idxs  = !foreach(a, IdxsR,  !if(!ge(a, 0), a, ?));
+  list<int> RIdxs = !foreach(a, RIdxsR, !if(!ge(a, 0), a, ?));
+}
+
+class TypeInfoGen<
+    list<LLVMType> RetTypes,
+    list<LLVMType> ParamTypes> {
+  list<LLVMType> AllTypes = !listconcat(RetTypes, ParamTypes);
+
+  // ArgCodes for NextArg -- isAny or MatchTypeNextArg
+  list<int> ACIdxs = MakeIdx<
+    !foreach(ty, AllTypes,
+      !or(ty.isAny, !isa<LLVMMatchTypeNextArg>(ty)))>.Idxs;
+
+  // ArgCodes (only for isAny or MatchTypeNextArg)
+  list<LLVMType> ACTys = !filter(ty, AllTypes,
+    !or(ty.isAny, !isa<LLVMMatchTypeNextArg>(ty)));
+
+  list<int> ArgCodes = !foreach(ty, ACTys, ty.ArgCode);
+
+  // Mappings MatchTypeIdx to ACTys
+  list<int> MappingRIdxs = MakeIdx<
+    !foreach(ty, ACTys, ty.isAny)>.RIdxs;
+
+  // D63507: Exclude LLVMPointerType<llvm_any_ty>
+  bit isOverloaded = !not(!empty(!filter(ty, AllTypes,
+    !isa<LLVMAnyType>(ty))));
+
+  list<LLVMType> Types = !foreach(ty, AllTypes,
+    !if(!isa<LLVMMatchType>(ty), ACTys[MappingRIdxs[ty.Number]], ty));
+
+  list<list<int>> TypeSig = !listconcat(
+    [IIT_RetNumbers[!size(RetTypes)]],
+    !foreach(i, !range(AllTypes),
+      !foreach(a, AllTypes[i].Sig,
+        ResolveArgCode<
+          MappingRIdxs,
+          ArgCodes,
+          ACIdxs[i],
+          a>.ret)));
+}
+
 //===----------------------------------------------------------------------===//
 // Intrinsic Definitions.
 //===----------------------------------------------------------------------===//
@@ -396,6 +677,11 @@ class Intrinsic<list<LLVMType> ret_types,
   bit DisableDefaultAttributes = disable_default_attributes;
 
   bit isTarget = false;
+
+  TypeInfoGen TypeInfo = TypeInfoGen<RetTypes, ParamTypes>;
+  bit isOverloaded = TypeInfo.isOverloaded;
+  list<LLVMType> Types = TypeInfo.Types;
+  list<list<int>> TypeSig = TypeInfo.TypeSig;
 }
 
 // Intrinsic with default attributes (disable_default_attributes = false).
diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h
index 262a148039a94..1a584f2ba599b 100644
--- a/llvm/include/llvm/IR/Metadata.h
+++ b/llvm/include/llvm/IR/Metadata.h
@@ -1274,6 +1274,11 @@ class MDNode : public Metadata {
   template <class NodeTy>
   static void dispatchResetHash(NodeTy *, std::false_type) {}
 
+  /// Merge branch weights from two direct callsites.
+  static MDNode *mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
+                                             const Instruction *AInstr,
+                                             const Instruction *BInstr);
+
 public:
   using op_iterator = const MDOperand *;
   using op_range = iterator_range<op_iterator>;
@@ -1319,6 +1324,11 @@ class MDNode : public Metadata {
   static MDNode *getMostGenericRange(MDNode *A, MDNode *B);
   static MDNode *getMostGenericAliasScope(MDNode *A, MDNode *B);
   static MDNode *getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B);
+  /// Merge !prof metadata from two instructions.
+  /// Currently only implemented with direct callsites with branch weights.
+  static MDNode *getMergedProfMetadata(MDNode *A, MDNode *B,
+                                       const Instruction *AInstr,
+                                       const Instruction *BInstr);
 };
 
 /// Tuple of metadata.
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index c540fa5b4ba14..6fdc378ab5c3b 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -1815,7 +1815,7 @@ class ModuleSummaryIndex {
   void propagateAttributes(const DenseSet<GlobalValue::GUID> &PreservedSymbols);
 
   /// Checks if we can import global variable from another module.
-  bool canImportGlobalVar(GlobalValueSummary *S, bool AnalyzeRefs) const;
+  bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const;
 };
 
 /// GraphTraits definition to build SCC for the index
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index b949f9f590679..61c7a3e501298 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -155,94 +155,104 @@
 #error                                                                         \
     "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!"
 #endif
-#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC)                       \
+#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC, SDOPC)                \
   BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1)                                      \
   VP_PROPERTY_FUNCTIONAL_OPC(IROPC)                                            \
+  VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC)                                          \
   VP_PROPERTY_BINARYOP                                                         \
   END_REGISTER_VP(VPID, VPSD)
 
 // llvm.vp.add(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add)
+HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add, ADD)
 
 // llvm.vp.and(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And)
+HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And, AND)
 
 // llvm.vp.ashr(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr)
+HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr, SRA)
 
 // llvm.vp.lshr(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr)
+HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr, SRL)
 
 // llvm.vp.mul(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul)
+HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul, MUL)
 
 // llvm.vp.or(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or)
+HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or, OR)
 
 // llvm.vp.sdiv(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv)
+HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv, SDIV)
 
 // llvm.vp.shl(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl)
+HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl, SHL)
 
 // llvm.vp.srem(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem)
+HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem, SREM)
 
 // llvm.vp.sub(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub)
+HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub, SUB)
 
 // llvm.vp.udiv(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv)
+HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv, UDIV)
 
 // llvm.vp.urem(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem)
+HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem, UREM)
 
 // llvm.vp.xor(x,y,mask,vlen)
-HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor)
+HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor, XOR)
 
 #undef HELPER_REGISTER_BINARY_INT_VP
 
 // llvm.vp.smin(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_smin, 2, 3, VP_SMIN, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(SMIN)
 END_REGISTER_VP(vp_smin, VP_SMIN)
 
 // llvm.vp.smax(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_smax, 2, 3, VP_SMAX, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(SMAX)
 END_REGISTER_VP(vp_smax, VP_SMAX)
 
 // llvm.vp.umin(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_umin, 2, 3, VP_UMIN, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(UMIN)
 END_REGISTER_VP(vp_umin, VP_UMIN)
 
 // llvm.vp.umax(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_umax, 2, 3, VP_UMAX, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(UMAX)
 END_REGISTER_VP(vp_umax, VP_UMAX)
 
 // llvm.vp.abs(x,is_int_min_poison,mask,vlen)
 BEGIN_REGISTER_VP_INTRINSIC(vp_abs, 2, 3)
 BEGIN_REGISTER_VP_SDNODE(VP_ABS, -1, vp_abs, 1, 2)
 HELPER_MAP_VPID_TO_VPSD(vp_abs, VP_ABS)
+VP_PROPERTY_FUNCTIONAL_SDOPC(ABS)
 END_REGISTER_VP(vp_abs, VP_ABS)
 
 // llvm.vp.bswap(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_bswap, 1, 2, VP_BSWAP, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(BSWAP)
 END_REGISTER_VP(vp_bswap, VP_BSWAP)
 
 // llvm.vp.bitreverse(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_bitreverse, 1, 2, VP_BITREVERSE, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(BITREVERSE)
 END_REGISTER_VP(vp_bitreverse, VP_BITREVERSE)
 
 // llvm.vp.ctpop(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_ctpop, 1, 2, VP_CTPOP, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(CTPOP)
 END_REGISTER_VP(vp_ctpop, VP_CTPOP)
 
 // llvm.vp.ctlz(x,is_zero_poison,mask,vlen)
 BEGIN_REGISTER_VP_INTRINSIC(vp_ctlz, 2, 3)
 BEGIN_REGISTER_VP_SDNODE(VP_CTLZ, -1, vp_ctlz, 1, 2)
+VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ)
 END_REGISTER_VP_SDNODE(VP_CTLZ)
 BEGIN_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF, -1, vp_ctlz_zero_undef, 1, 2)
 END_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF)
@@ -251,6 +261,7 @@ END_REGISTER_VP_INTRINSIC(vp_ctlz)
 // llvm.vp.cttz(x,is_zero_poison,mask,vlen)
 BEGIN_REGISTER_VP_INTRINSIC(vp_cttz, 2, 3)
 BEGIN_REGISTER_VP_SDNODE(VP_CTTZ, -1, vp_cttz, 1, 2)
+VP_PROPERTY_FUNCTIONAL_SDOPC(CTTZ)
 END_REGISTER_VP_SDNODE(VP_CTTZ)
 BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF, -1, vp_cttz_zero_undef, 1, 2)
 END_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF)
@@ -258,10 +269,12 @@ END_REGISTER_VP_INTRINSIC(vp_cttz)
 
 // llvm.vp.fshl(x,y,z,mask,vlen)
 BEGIN_REGISTER_VP(vp_fshl, 3, 4, VP_FSHL, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FSHL)
 END_REGISTER_VP(vp_fshl, VP_FSHL)
 
 // llvm.vp.fshr(x,y,z,mask,vlen)
 BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FSHR)
 END_REGISTER_VP(vp_fshr, VP_FSHR)
 ///// } Integer Arithmetic
 
@@ -301,14 +314,17 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem, FREM)
 // llvm.vp.fneg(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1)
 VP_PROPERTY_FUNCTIONAL_OPC(FNeg)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FNEG)
 END_REGISTER_VP(vp_fneg, VP_FNEG)
 
 // llvm.vp.fabs(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_fabs, 1, 2, VP_FABS, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FABS)
 END_REGISTER_VP(vp_fabs, VP_FABS)
 
 // llvm.vp.sqrt(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_sqrt, 1, 2, VP_SQRT, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FSQRT)
 END_REGISTER_VP(vp_sqrt, VP_SQRT)
 
 // llvm.vp.fma(x,y,z,mask,vlen)
@@ -320,49 +336,60 @@ END_REGISTER_VP(vp_fma, VP_FMA)
 // llvm.vp.fmuladd(x,y,z,mask,vlen)
 BEGIN_REGISTER_VP(vp_fmuladd, 3, 4, VP_FMULADD, -1)
 VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fmuladd)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FMAD)
 END_REGISTER_VP(vp_fmuladd, VP_FMULADD)
 
 // llvm.vp.copysign(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_copysign, 2, 3, VP_FCOPYSIGN, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN)
 END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN)
 
 // llvm.vp.minnum(x, y, mask,vlen)
 BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM)
 END_REGISTER_VP(vp_minnum, VP_FMINNUM)
 
 // llvm.vp.maxnum(x, y, mask,vlen)
 BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1)
 VP_PROPERTY_BINARYOP
+VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM)
 END_REGISTER_VP(vp_maxnum, VP_FMAXNUM)
 
 // llvm.vp.ceil(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FCEIL)
 END_REGISTER_VP(vp_ceil, VP_FCEIL)
 
 // llvm.vp.floor(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_floor, 1, 2, VP_FFLOOR, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FFLOOR)
 END_REGISTER_VP(vp_floor, VP_FFLOOR)
 
 // llvm.vp.round(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_round, 1, 2, VP_FROUND, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FROUND)
 END_REGISTER_VP(vp_round, VP_FROUND)
 
 // llvm.vp.roundeven(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_roundeven, 1, 2, VP_FROUNDEVEN, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FROUNDEVEN)
 END_REGISTER_VP(vp_roundeven, VP_FROUNDEVEN)
 
 // llvm.vp.roundtozero(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_roundtozero, 1, 2, VP_FROUNDTOZERO, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FTRUNC)
 END_REGISTER_VP(vp_roundtozero, VP_FROUNDTOZERO)
 
 // llvm.vp.rint(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_rint, 1, 2, VP_FRINT, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FRINT)
 END_REGISTER_VP(vp_rint, VP_FRINT)
 
 // llvm.vp.nearbyint(x,mask,vlen)
 BEGIN_REGISTER_VP(vp_nearbyint, 1, 2, VP_FNEARBYINT, -1)
+VP_PROPERTY_FUNCTIONAL_SDOPC(FNEARBYINT)
 END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT)
 
 ///// } Floating-Point Arithmetic
@@ -374,30 +401,31 @@ END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT)
 #error                                                                         \
     "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!"
 #endif
-#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, HASROUND)            \
+#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC, HASROUND)     \
   BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1)                             \
   VP_PROPERTY_FUNCTIONAL_OPC(IROPC)                                            \
+  VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC)                                          \
   VP_PROPERTY_CONSTRAINEDFP(HASROUND, 1, experimental_constrained_##OPSUFFIX)  \
   VP_PROPERTY_CASTOP                                                           \
   END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
 
 // llvm.vp.fptoui(x,mask,vlen)
-HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, 0)
+HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, FP_TO_UINT, 0)
 
 // llvm.vp.fptosi(x,mask,vlen)
-HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, 0)
+HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, FP_TO_SINT, 0)
 
 // llvm.vp.uitofp(x,mask,vlen)
-HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, 1)
+HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, UINT_TO_FP, 1)
 
 // llvm.vp.sitofp(x,mask,vlen)
-HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, 1)
+HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, SINT_TO_FP, 1)
 
 // llvm.vp.fptrunc(x,mask,vlen)
-HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, 1)
+HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, FP_ROUND, 1)
 
 // llvm.vp.fpext(x,mask,vlen)
-HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, 0)
+HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, FP_EXTEND, 0)
 
 #undef HELPER_REGISTER_FP_CAST_VP
 
@@ -407,26 +435,33 @@ HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, 0)
 #error                                                                         \
     "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!"
 #endif
-#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC)                     \
+#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC)              \
   BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1)                             \
   VP_PROPERTY_FUNCTIONAL_OPC(IROPC)                                            \
+  VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC)                                          \
   VP_PROPERTY_CASTOP                                                           \
   END_REGISTER_VP(vp_##OPSUFFIX, VPSD)
 
 // llvm.vp.trunc(x,mask,vlen)
-HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc)
+HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc, TRUNCATE)
 
 // llvm.vp.zext(x,mask,vlen)
-HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt)
+HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt, ZERO_EXTEND)
 
 // llvm.vp.sext(x,mask,vlen)
-HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt)
+HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt, SIGN_EXTEND)
 
 // llvm.vp.ptrtoint(x,mask,vlen)
-HELPER_REGISTER_INT_CAST_VP(ptrtoint, VP_PTRTOINT, PtrToInt)
+BEGIN_REGISTER_VP(vp_ptrtoint, 1, 2, VP_PTRTOINT, -1)
+VP_PROPERTY_FUNCTIONAL_OPC(PtrToInt)
+VP_PROPERTY_CASTOP
+END_REGISTER_VP(vp_ptrtoint, VP_PTRTOINT)
 
 // llvm.vp.inttoptr(x,mask,vlen)
-HELPER_REGISTER_INT_CAST_VP(inttoptr, VP_INTTOPTR, IntToPtr)
+BEGIN_REGISTER_VP(vp_inttoptr, 1, 2, VP_INTTOPTR, -1)
+VP_PROPERTY_FUNCTIONAL_OPC(IntToPtr)
+VP_PROPERTY_CASTOP
+END_REGISTER_VP(vp_inttoptr, VP_INTTOPTR)
 
 #undef HELPER_REGISTER_INT_CAST_VP
 
@@ -617,6 +652,7 @@ HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL,
 // llvm.vp.select(cond,on_true,on_false,vlen)
 BEGIN_REGISTER_VP(vp_select, std::nullopt, 3, VP_SELECT, -1)
 VP_PROPERTY_FUNCTIONAL_OPC(Select)
+VP_PROPERTY_FUNCTIONAL_SDOPC(VSELECT)
 END_REGISTER_VP(vp_select, VP_SELECT)
 
 // llvm.vp.merge(cond,on_true,on_false,pivot)
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 9b98484c37cfd..9e9aea5841710 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -54,7 +54,6 @@ void initializeADCELegacyPassPass(PassRegistry&);
 void initializeAddDiscriminatorsLegacyPassPass(PassRegistry&);
 void initializeAlwaysInlinerLegacyPassPass(PassRegistry&);
 void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
-void initializeAssumeSimplifyPassLegacyPassPass(PassRegistry &);
 void initializeAssumeBuilderPassLegacyPassPass(PassRegistry &);
 void initializeAnnotation2MetadataLegacyPass(PassRegistry &);
 void initializeAssumptionCacheTrackerPass(PassRegistry&);
@@ -157,9 +156,7 @@ void initializeIndVarSimplifyLegacyPassPass(PassRegistry&);
 void initializeIndirectBrExpandPassPass(PassRegistry&);
 void initializeInferAddressSpacesPass(PassRegistry&);
 void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&);
-void initializeInjectTLIMappingsLegacyPass(PassRegistry &);
 void initializeInstCountLegacyPassPass(PassRegistry &);
-void initializeInstNamerPass(PassRegistry&);
 void initializeInstSimplifyLegacyPassPass(PassRegistry &);
 void initializeInstructionCombiningPassPass(PassRegistry&);
 void initializeInstructionSelectPass(PassRegistry&);
@@ -258,7 +255,6 @@ void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&);
 void initializeMemorySSAWrapperPassPass(PassRegistry&);
 void initializeMergeICmpsLegacyPassPass(PassRegistry &);
 void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&);
-void initializeMetaRenamerPass(PassRegistry&);
 void initializeModuleDebugInfoLegacyPrinterPass(PassRegistry &);
 void initializeModuleSummaryIndexWrapperPassPass(PassRegistry&);
 void initializeModuloScheduleTestPass(PassRegistry&);
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 97ab7c4a09536..dabc367c80c88 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -78,9 +78,8 @@ namespace lto {
 /// Given the original \p Path to an output file, replace any path
 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
 /// resulting directory if it does not yet exist.
-std::string getThinLTOOutputFile(const std::string &Path,
-                                 const std::string &OldPrefix,
-                                 const std::string &NewPrefix);
+std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
+                                 StringRef NewPrefix);
 
 /// Setup optimization remarks.
 Expected<std::unique_ptr<ToolOutputFile>> setupLLVMOptimizationRemarks(
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index ef1db9d654d26..2b844595765b4 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -137,8 +137,6 @@ namespace {
       (void) llvm::createMergedLoadStoreMotionPass();
       (void) llvm::createGVNPass();
       (void) llvm::createPostDomTree();
-      (void) llvm::createInstructionNamerPass();
-      (void) llvm::createMetaRenamerPass();
       (void) llvm::createMergeICmpsLegacyPass();
       (void) llvm::createExpandLargeDivRemPass();
       (void) llvm::createExpandMemCmpPass();
@@ -164,7 +162,6 @@ namespace {
       (void) llvm::createMustBeExecutedContextPrinter();
       (void)llvm::createScalarizeMaskedMemIntrinLegacyPass();
       (void) llvm::createHardwareLoopsLegacyPass();
-      (void) llvm::createInjectTLIMappingsLegacyPass();
       (void) llvm::createUnifyLoopExitsPass();
       (void) llvm::createFixIrreduciblePass();
       (void)llvm::createSelectOptimizePass();
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 80aa97c315da7..5e1fc738b1dab 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -473,8 +473,7 @@ class MCAssembler {
   /// @{
 
   bool registerSection(MCSection &Section);
-
-  void registerSymbol(const MCSymbol &Symbol, bool *Created = nullptr);
+  bool registerSymbol(const MCSymbol &Symbol);
 
   MutableArrayRef<std::pair<std::string, size_t>> getFileNames() {
     return FileNames;
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 74c3fe5c61f29..26ee1803cb6ef 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -190,7 +190,7 @@ class MCContext {
   SmallString<128> CompilationDir;
 
   /// Prefix replacement map for source file information.
-  std::map<std::string, const std::string, std::greater<>> DebugPrefixMap;
+  SmallVector<std::pair<std::string, std::string>, 0> DebugPrefixMap;
 
   /// The main file name if passed in explicitly.
   std::string MainFileName;
diff --git a/llvm/include/llvm/MC/MCDirectives.h b/llvm/include/llvm/MC/MCDirectives.h
index fcab56ff0a740..b9668a0c994fa 100644
--- a/llvm/include/llvm/MC/MCDirectives.h
+++ b/llvm/include/llvm/MC/MCDirectives.h
@@ -46,7 +46,6 @@ enum MCSymbolAttr {
   MCSA_WeakDefinition,          ///< .weak_definition (MachO)
   MCSA_WeakReference,           ///< .weak_reference (MachO)
   MCSA_WeakDefAutoPrivate,      ///< .weak_def_can_be_hidden (MachO)
-  MCSA_WeakAntiDep,             ///< .weak_anti_dep (COFF)
   MCSA_Memtag,                  ///< .memtag (ELF)
 };
 
diff --git a/llvm/include/llvm/MC/MCInstrAnalysis.h b/llvm/include/llvm/MC/MCInstrAnalysis.h
index 8a6bc371672a0..aca0f4daeeee8 100644
--- a/llvm/include/llvm/MC/MCInstrAnalysis.h
+++ b/llvm/include/llvm/MC/MCInstrAnalysis.h
@@ -168,7 +168,7 @@ class MCInstrAnalysis {
   /// Returns (PLT virtual address, GOT virtual address) pairs for PLT entries.
   virtual std::vector<std::pair<uint64_t, uint64_t>>
   findPltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
-                 uint64_t GotPltSectionVA, const Triple &TargetTriple) const {
+                 const Triple &TargetTriple) const {
     return {};
   }
 };
diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index f59821d48dc3d..8954960e3b8c7 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -102,9 +102,6 @@ class MCSymbol {
   /// This symbol is private extern.
   mutable unsigned IsPrivateExtern : 1;
 
-  /// This symbol is weak external.
-  mutable unsigned IsWeakExternal : 1;
-
   /// LLVM RTTI discriminator. This is actually a SymbolKind enumerator, but is
   /// unsigned to avoid sign extension and achieve better bitpacking with MSVC.
   unsigned Kind : 3;
@@ -164,8 +161,8 @@ class MCSymbol {
   MCSymbol(SymbolKind Kind, const StringMapEntry<bool> *Name, bool isTemporary)
       : IsTemporary(isTemporary), IsRedefinable(false), IsUsed(false),
         IsRegistered(false), IsExternal(false), IsPrivateExtern(false),
-        IsWeakExternal(false), Kind(Kind), IsUsedInReloc(false),
-        SymbolContents(SymContentsUnset), CommonAlignLog2(0), Flags(0) {
+        Kind(Kind), IsUsedInReloc(false), SymbolContents(SymContentsUnset),
+        CommonAlignLog2(0), Flags(0) {
     Offset = 0;
     FragmentAndHasName.setInt(!!Name);
     if (Name)
@@ -397,10 +394,8 @@ class MCSymbol {
 
   MCFragment *getFragment(bool SetUsed = true) const {
     MCFragment *Fragment = FragmentAndHasName.getPointer();
-    if (Fragment || !isVariable() || isWeakExternal())
+    if (Fragment || !isVariable())
       return Fragment;
-    // If the symbol is a non-weak alias, get information about
-    // the aliasee. (Don't try to resolve weak aliases.)
     Fragment = getVariableValue(SetUsed)->findAssociatedFragment();
     FragmentAndHasName.setPointer(Fragment);
     return Fragment;
@@ -412,8 +407,6 @@ class MCSymbol {
   bool isPrivateExtern() const { return IsPrivateExtern; }
   void setPrivateExtern(bool Value) { IsPrivateExtern = Value; }
 
-  bool isWeakExternal() const { return IsWeakExternal; }
-
   /// print - Print the value to the stream \p OS.
   void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
 
diff --git a/llvm/include/llvm/MC/MCSymbolCOFF.h b/llvm/include/llvm/MC/MCSymbolCOFF.h
index 7f077c7986c01..b25993c32db6d 100644
--- a/llvm/include/llvm/MC/MCSymbolCOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolCOFF.h
@@ -23,9 +23,8 @@ class MCSymbolCOFF : public MCSymbol {
     SF_ClassMask = 0x00FF,
     SF_ClassShift = 0,
 
-    SF_SafeSEH = 0x0100,
-    SF_WeakExternalCharacteristicsMask = 0x0E00,
-    SF_WeakExternalCharacteristicsShift = 9,
+    SF_WeakExternal = 0x0100,
+    SF_SafeSEH = 0x0200,
   };
 
 public:
@@ -46,14 +45,11 @@ class MCSymbolCOFF : public MCSymbol {
     modifyFlags(StorageClass << SF_ClassShift, SF_ClassMask);
   }
 
-  COFF::WeakExternalCharacteristics getWeakExternalCharacteristics() const {
-    return static_cast<COFF::WeakExternalCharacteristics>((getFlags() & SF_WeakExternalCharacteristicsMask) >>
-           SF_WeakExternalCharacteristicsShift);
+  bool isWeakExternal() const {
+    return getFlags() & SF_WeakExternal;
   }
-  void setIsWeakExternal(COFF::WeakExternalCharacteristics Characteristics) const {
-    IsWeakExternal = true;
-    modifyFlags(Characteristics << SF_WeakExternalCharacteristicsShift,
-                SF_WeakExternalCharacteristicsMask);
+  void setIsWeakExternal() const {
+    modifyFlags(SF_WeakExternal, SF_WeakExternal);
   }
 
   bool isSafeSEH() const {
diff --git a/llvm/include/llvm/Object/Binary.h b/llvm/include/llvm/Object/Binary.h
index 955b0b0ce2817..ce870e25acafe 100644
--- a/llvm/include/llvm/Object/Binary.h
+++ b/llvm/include/llvm/Object/Binary.h
@@ -69,6 +69,7 @@ class Binary {
     ID_MachO64L, // MachO 64-bit, little endian
     ID_MachO64B, // MachO 64-bit, big endian
 
+    ID_GOFF,
     ID_Wasm,
 
     ID_EndObjects
@@ -145,6 +146,8 @@ class Binary {
     return TypeID == ID_IR;
   }
 
+  bool isGOFF() const { return TypeID == ID_GOFF; }
+
   bool isMinidump() const { return TypeID == ID_Minidump; }
 
   bool isTapiFile() const { return TypeID == ID_TapiFile; }
@@ -164,6 +167,8 @@ class Binary {
       return Triple::MachO;
     if (isELF())
       return Triple::ELF;
+    if (isGOFF())
+      return Triple::GOFF;
     return Triple::UnknownObjectFormat;
   }
 
diff --git a/llvm/include/llvm/Object/GOFF.h b/llvm/include/llvm/Object/GOFF.h
new file mode 100644
index 0000000000000..f4aa04cd99fcf
--- /dev/null
+++ b/llvm/include/llvm/Object/GOFF.h
@@ -0,0 +1,284 @@
+//===- GOFF.h - GOFF object file implementation -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the GOFFObjectFile class.
+// Record classes and derivatives are also declared and implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_GOFF_H
+#define LLVM_OBJECT_GOFF_H
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace object {
+
+/// \brief Represents a GOFF physical record.
+///
+/// Specifies protected member functions to manipulate the record. These should
+/// be called from deriving classes to change values as that record specifies.
+class Record {
+public:
+  static Error getContinuousData(const uint8_t *Record, uint16_t DataLength,
+                                 int DataIndex, SmallString<256> &CompleteData);
+
+  static bool isContinued(const uint8_t *Record) {
+    uint8_t IsContinued;
+    getBits(Record, 1, 7, 1, IsContinued);
+    return IsContinued;
+  }
+
+  static bool isContinuation(const uint8_t *Record) {
+    uint8_t IsContinuation;
+    getBits(Record, 1, 6, 1, IsContinuation);
+    return IsContinuation;
+  }
+
+protected:
+  /// \brief Get bit field of specified byte.
+  ///
+  /// Used to pack bit fields into one byte. Fields are packed left to right.
+  /// Bit index zero is the most significant bit of the byte.
+  ///
+  /// \param ByteIndex index of byte the field is in.
+  /// \param BitIndex index of first bit of field.
+  /// \param Length length of bit field.
+  /// \param Value value of bit field.
+  static void getBits(const uint8_t *Bytes, uint8_t ByteIndex, uint8_t BitIndex,
+                      uint8_t Length, uint8_t &Value) {
+    assert(ByteIndex < GOFF::RecordLength && "Byte index out of bounds!");
+    assert(BitIndex < 8 && "Bit index out of bounds!");
+    assert(Length + BitIndex <= 8 && "Bit length too long!");
+
+    get<uint8_t>(Bytes, ByteIndex, Value);
+    Value = (Value >> (8 - BitIndex - Length)) & ((1 << Length) - 1);
+  }
+
+  template <class T>
+  static void get(const uint8_t *Bytes, uint8_t ByteIndex, T &Value) {
+    assert(ByteIndex + sizeof(T) <= GOFF::RecordLength &&
+           "Byte index out of bounds!");
+    Value = support::endian::read<T, support::big, support::unaligned>(
+        &Bytes[ByteIndex]);
+  }
+};
+
+class HDRRecord : public Record {
+public:
+  static Error getData(const uint8_t *Record, SmallString<256> &CompleteData);
+
+  static uint16_t getPropertyModuleLength(const uint8_t *Record) {
+    uint16_t Length;
+    get<uint16_t>(Record, 52, Length);
+    return Length;
+  }
+};
+
+class ESDRecord : public Record {
+public:
+  /// \brief Number of bytes for name; any more must go in continuation.
+  /// This is the number of bytes that can fit into the data field of an ESD
+  /// record.
+  static const uint8_t ESDMaxUncontinuedNameLength = 8;
+
+  /// \brief Maximum name length for ESD records and continuations.
+  /// This is the number of bytes that can fit into the data field of an ESD
+  /// record AND following continuations. This is limited fundamentally by the
+  /// 16 bit SIGNED length field.
+  static const uint16_t MaxNameLength = 32 * 1024;
+
+public:
+  static Error getData(const uint8_t *Record, SmallString<256> &CompleteData);
+
+  // ESD Get routines.
+  static void getSymbolType(const uint8_t *Record,
+                            GOFF::ESDSymbolType &SymbolType) {
+    uint8_t Value;
+    get<uint8_t>(Record, 3, Value);
+    SymbolType = (GOFF::ESDSymbolType)Value;
+  }
+
+  static void getEsdId(const uint8_t *Record, uint32_t &EsdId) {
+    get<uint32_t>(Record, 4, EsdId);
+  }
+
+  static void getParentEsdId(const uint8_t *Record, uint32_t &EsdId) {
+    get<uint32_t>(Record, 8, EsdId);
+  }
+
+  static void getOffset(const uint8_t *Record, uint32_t &Offset) {
+    get<uint32_t>(Record, 16, Offset);
+  }
+
+  static void getLength(const uint8_t *Record, uint32_t &Length) {
+    get<uint32_t>(Record, 24, Length);
+  }
+
+  static void getNameSpaceId(const uint8_t *Record, GOFF::ESDNameSpaceId &Id) {
+    uint8_t Value;
+    get<uint8_t>(Record, 40, Value);
+    Id = (GOFF::ESDNameSpaceId)Value;
+  }
+
+  static void getFillBytePresent(const uint8_t *Record, bool &Present) {
+    uint8_t Value;
+    getBits(Record, 41, 0, 1, Value);
+    Present = (bool)Value;
+  }
+
+  static void getNameMangled(const uint8_t *Record, bool &Mangled) {
+    uint8_t Value;
+    getBits(Record, 41, 1, 1, Value);
+    Mangled = (bool)Value;
+  }
+
+  static void getRenamable(const uint8_t *Record, bool &Renamable) {
+    uint8_t Value;
+    getBits(Record, 41, 2, 1, Value);
+    Renamable = (bool)Value;
+  }
+
+  static void getRemovable(const uint8_t *Record, bool &Removable) {
+    uint8_t Value;
+    getBits(Record, 41, 3, 1, Value);
+    Removable = (bool)Value;
+  }
+
+  static void getFillByteValue(const uint8_t *Record, uint8_t &Fill) {
+    get<uint8_t>(Record, 42, Fill);
+  }
+
+  static void getAdaEsdId(const uint8_t *Record, uint32_t &EsdId) {
+    get<uint32_t>(Record, 44, EsdId);
+  }
+
+  static void getSortPriority(const uint8_t *Record, uint32_t &Priority) {
+    get<uint32_t>(Record, 48, Priority);
+  }
+
+  static void getAmode(const uint8_t *Record, GOFF::ESDAmode &Amode) {
+    uint8_t Value;
+    get<uint8_t>(Record, 60, Value);
+    Amode = (GOFF::ESDAmode)Value;
+  }
+
+  static void getRmode(const uint8_t *Record, GOFF::ESDRmode &Rmode) {
+    uint8_t Value;
+    get<uint8_t>(Record, 61, Value);
+    Rmode = (GOFF::ESDRmode)Value;
+  }
+
+  static void getTextStyle(const uint8_t *Record, GOFF::ESDTextStyle &Style) {
+    uint8_t Value;
+    getBits(Record, 62, 0, 4, Value);
+    Style = (GOFF::ESDTextStyle)Value;
+  }
+
+  static void getBindingAlgorithm(const uint8_t *Record,
+                                  GOFF::ESDBindingAlgorithm &Algorithm) {
+    uint8_t Value;
+    getBits(Record, 62, 4, 4, Value);
+    Algorithm = (GOFF::ESDBindingAlgorithm)Value;
+  }
+
+  static void getTaskingBehavior(const uint8_t *Record,
+                                 GOFF::ESDTaskingBehavior &TaskingBehavior) {
+    uint8_t Value;
+    getBits(Record, 63, 0, 3, Value);
+    TaskingBehavior = (GOFF::ESDTaskingBehavior)Value;
+  }
+
+  static void getReadOnly(const uint8_t *Record, bool &ReadOnly) {
+    uint8_t Value;
+    getBits(Record, 63, 4, 1, Value);
+    ReadOnly = (bool)Value;
+  }
+
+  static void getExecutable(const uint8_t *Record,
+                            GOFF::ESDExecutable &Executable) {
+    uint8_t Value;
+    getBits(Record, 63, 5, 3, Value);
+    Executable = (GOFF::ESDExecutable)Value;
+  }
+
+  static void getDuplicateSeverity(const uint8_t *Record,
+                                   GOFF::ESDDuplicateSymbolSeverity &DSS) {
+    uint8_t Value;
+    getBits(Record, 64, 2, 2, Value);
+    DSS = (GOFF::ESDDuplicateSymbolSeverity)Value;
+  }
+
+  static void getBindingStrength(const uint8_t *Record,
+                                 GOFF::ESDBindingStrength &Strength) {
+    uint8_t Value;
+    getBits(Record, 64, 4, 4, Value);
+    Strength = (GOFF::ESDBindingStrength)Value;
+  }
+
+  static void getLoadingBehavior(const uint8_t *Record,
+                                 GOFF::ESDLoadingBehavior &Behavior) {
+    uint8_t Value;
+    getBits(Record, 65, 0, 2, Value);
+    Behavior = (GOFF::ESDLoadingBehavior)Value;
+  }
+
+  static void getIndirectReference(const uint8_t *Record, bool &Indirect) {
+    uint8_t Value;
+    getBits(Record, 65, 3, 1, Value);
+    Indirect = (bool)Value;
+  }
+
+  static void getBindingScope(const uint8_t *Record,
+                              GOFF::ESDBindingScope &Scope) {
+    uint8_t Value;
+    getBits(Record, 65, 4, 4, Value);
+    Scope = (GOFF::ESDBindingScope)Value;
+  }
+
+  static void getLinkageType(const uint8_t *Record,
+                             GOFF::ESDLinkageType &Type) {
+    uint8_t Value;
+    getBits(Record, 66, 2, 1, Value);
+    Type = (GOFF::ESDLinkageType)Value;
+  }
+
+  static void getAlignment(const uint8_t *Record,
+                           GOFF::ESDAlignment &Alignment) {
+    uint8_t Value;
+    getBits(Record, 66, 3, 5, Value);
+    Alignment = (GOFF::ESDAlignment)Value;
+  }
+
+  static uint16_t getNameLength(const uint8_t *Record) {
+    uint16_t Length;
+    get<uint16_t>(Record, 70, Length);
+    return Length;
+  }
+};
+
+class ENDRecord : public Record {
+public:
+  static Error getData(const uint8_t *Record, SmallString<256> &CompleteData);
+
+  static uint16_t getNameLength(const uint8_t *Record) {
+    uint16_t Length;
+    get<uint16_t>(Record, 24, Length);
+    return Length;
+  }
+};
+
+} // end namespace object
+} // end namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Object/GOFFObjectFile.h b/llvm/include/llvm/Object/GOFFObjectFile.h
new file mode 100644
index 0000000000000..37b6b1ec659a0
--- /dev/null
+++ b/llvm/include/llvm/Object/GOFFObjectFile.h
@@ -0,0 +1,130 @@
+//===- GOFFObjectFile.h - GOFF object file implementation -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the GOFFObjectFile class.
+// Record classes and derivatives are also declared and implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_GOFFOBJECTFILE_H
+#define LLVM_OBJECT_GOFFOBJECTFILE_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/MC/SubtargetFeature.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/ConvertEBCDIC.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+namespace object {
+
+class GOFFObjectFile : public ObjectFile {
+  IndexedMap<const uint8_t *> EsdPtrs; // Indexed by EsdId.
+
+  mutable DenseMap<uint32_t, std::pair<size_t, std::unique_ptr<char[]>>>
+      EsdNamesCache;
+
+  typedef DataRefImpl SectionEntryImpl;
+  // (EDID, 0)               code, r/o data section
+  // (EDID,PRID)             r/w data section
+  SmallVector<SectionEntryImpl, 256> SectionList;
+  mutable DenseMap<uint32_t, std::string> SectionDataCache;
+
+public:
+  Expected<StringRef> getSymbolName(SymbolRef Symbol) const;
+
+  GOFFObjectFile(MemoryBufferRef Object, Error &Err);
+  static inline bool classof(const Binary *V) { return V->isGOFF(); }
+  section_iterator section_begin() const override;
+  section_iterator section_end() const override;
+
+  uint8_t getBytesInAddress() const override { return 8; }
+
+  StringRef getFileFormatName() const override { return "GOFF-SystemZ"; }
+
+  Triple::ArchType getArch() const override { return Triple::systemz; }
+
+  Expected<SubtargetFeatures> getFeatures() const override { return SubtargetFeatures(); }
+
+  bool isRelocatableObject() const override { return true; }
+
+  void moveSymbolNext(DataRefImpl &Symb) const override;
+  basic_symbol_iterator symbol_begin() const override;
+  basic_symbol_iterator symbol_end() const override;
+
+  bool is64Bit() const override {
+    return true;
+  }
+
+private:
+  // SymbolRef.
+  Expected<StringRef> getSymbolName(DataRefImpl Symb) const override;
+  Expected<uint64_t> getSymbolAddress(DataRefImpl Symb) const override;
+  uint64_t getSymbolValueImpl(DataRefImpl Symb) const override;
+  uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override;
+  Expected<uint32_t> getSymbolFlags(DataRefImpl Symb) const override;
+  Expected<SymbolRef::Type> getSymbolType(DataRefImpl Symb) const override;
+  Expected<section_iterator> getSymbolSection(DataRefImpl Symb) const override;
+
+  const uint8_t *getSymbolEsdRecord(DataRefImpl Symb) const;
+  bool isSymbolUnresolved(DataRefImpl Symb) const;
+  bool isSymbolIndirect(DataRefImpl Symb) const;
+
+  // SectionRef.
+  void moveSectionNext(DataRefImpl &Sec) const override{};
+  virtual Expected<StringRef> getSectionName(DataRefImpl Sec) const override {
+    return StringRef();
+  }
+  uint64_t getSectionAddress(DataRefImpl Sec) const override { return 0; }
+  uint64_t getSectionSize(DataRefImpl Sec) const override { return 0; }
+  virtual Expected<ArrayRef<uint8_t>>
+  getSectionContents(DataRefImpl Sec) const override {
+    return ArrayRef<uint8_t>();
+  }
+  uint64_t getSectionIndex(DataRefImpl Sec) const override { return 0; }
+  uint64_t getSectionAlignment(DataRefImpl Sec) const override { return 0; }
+  bool isSectionCompressed(DataRefImpl Sec) const override { return false; }
+  bool isSectionText(DataRefImpl Sec) const override { return false; }
+  bool isSectionData(DataRefImpl Sec) const override { return false; }
+  bool isSectionBSS(DataRefImpl Sec) const override { return false; }
+  bool isSectionVirtual(DataRefImpl Sec) const override { return false; }
+  relocation_iterator section_rel_begin(DataRefImpl Sec) const override {
+    return relocation_iterator(RelocationRef(Sec, this));
+  }
+  relocation_iterator section_rel_end(DataRefImpl Sec) const override {
+    return relocation_iterator(RelocationRef(Sec, this));
+  }
+
+  const uint8_t *getSectionEdEsdRecord(DataRefImpl &Sec) const;
+  const uint8_t *getSectionPrEsdRecord(DataRefImpl &Sec) const;
+  const uint8_t *getSectionEdEsdRecord(uint32_t SectionIndex) const;
+  const uint8_t *getSectionPrEsdRecord(uint32_t SectionIndex) const;
+
+  // RelocationRef.
+  void moveRelocationNext(DataRefImpl &Rel) const override{};
+  uint64_t getRelocationOffset(DataRefImpl Rel) const override { return 0; }
+  symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override {
+    DataRefImpl Temp;
+    return basic_symbol_iterator(SymbolRef(Temp, this));
+  }
+  uint64_t getRelocationType(DataRefImpl Rel) const override { return 0; }
+  void getRelocationTypeName(DataRefImpl Rel,
+                             SmallVectorImpl<char> &Result) const override{};
+};
+
+} // namespace object
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h
index e7a8fcd156f2b..d2abffdc89069 100644
--- a/llvm/include/llvm/Object/MachO.h
+++ b/llvm/include/llvm/Object/MachO.h
@@ -717,7 +717,7 @@ class MachOObjectFile : public ObjectFile {
   ArrayRef<uint8_t> getDyldInfoLazyBindOpcodes() const;
   ArrayRef<uint8_t> getDyldInfoExportsTrie() const;
 
-  /// If the optional is None, no header was found, but the object was
+  /// If the optional is std::nullopt, no header was found, but the object was
   /// well-formed.
   Expected<std::optional<MachO::dyld_chained_fixups_header>>
   getChainedFixupsHeader() const;
diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h
index f8c2eea2fcaba..684e7b500478b 100644
--- a/llvm/include/llvm/Object/ObjectFile.h
+++ b/llvm/include/llvm/Object/ObjectFile.h
@@ -392,6 +392,9 @@ class ObjectFile : public SymbolicFile {
                         uint32_t UniversalCputype = 0,
                         uint32_t UniversalIndex = 0);
 
+  static Expected<std::unique_ptr<ObjectFile>>
+  createGOFFObjectFile(MemoryBufferRef Object);
+
   static Expected<std::unique_ptr<WasmObjectFile>>
   createWasmObjectFile(MemoryBufferRef Object);
 };
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 2a9da23f4f02f..ef01070832dee 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -330,7 +330,8 @@ enum class instrprof_error {
   compress_failed,
   uncompress_failed,
   empty_raw_profile,
-  zlib_unavailable
+  zlib_unavailable,
+  raw_profile_version_mismatch
 };
 
 /// An ordered list of functions identified by their NameRef found in
@@ -362,15 +363,18 @@ class InstrProfError : public ErrorInfo<InstrProfError> {
   instrprof_error get() const { return Err; }
   const std::string &getMessage() const { return Msg; }
 
-  /// Consume an Error and return the raw enum value contained within it. The
-  /// Error must either be a success value, or contain a single InstrProfError.
-  static instrprof_error take(Error E) {
+  /// Consume an Error and return the raw enum value contained within it, and
+  /// the optional error message. The Error must either be a success value, or
+  /// contain a single InstrProfError.
+  static std::pair<instrprof_error, std::string> take(Error E) {
     auto Err = instrprof_error::success;
-    handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) {
+    std::string Msg = "";
+    handleAllErrors(std::move(E), [&Err, &Msg](const InstrProfError &IPE) {
       assert(Err == instrprof_error::success && "Multiple errors encountered");
       Err = IPE.get();
+      Msg = IPE.getMessage();
     });
-    return Err;
+    return {Err, Msg};
   }
 
   static char ID;
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index ddab9a1e49b3d..b8456eec1e9fd 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -90,7 +90,7 @@ namespace sampleprof {
 enum SampleProfileFormat {
   SPF_None = 0,
   SPF_Text = 0x1,
-  SPF_Compact_Binary = 0x2,
+  SPF_Compact_Binary = 0x2, // Deprecated
   SPF_GCC = 0x3,
   SPF_Ext_Binary = 0x4,
   SPF_Binary = 0xff
@@ -723,6 +723,8 @@ using BodySampleMap = std::map<LineLocation, SampleRecord>;
 // memory, which is *very* significant for large profiles.
 using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
 using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
+using LocToLocMap =
+    std::unordered_map<LineLocation, LineLocation, LineLocationHash>;
 
 /// Representation of the samples collected for a function.
 ///
@@ -842,12 +844,26 @@ class FunctionSamples {
     }
   }
 
+  // Query the stale profile matching results and remap the location.
+  const LineLocation &mapIRLocToProfileLoc(const LineLocation &IRLoc) const {
+    // There is no remapping if the profile is not stale or the matching gives
+    // the same location.
+    if (!IRToProfileLocationMap)
+      return IRLoc;
+    const auto &ProfileLoc = IRToProfileLocationMap->find(IRLoc);
+    if (ProfileLoc != IRToProfileLocationMap->end())
+      return ProfileLoc->second;
+    else
+      return IRLoc;
+  }
+
   /// Return the number of samples collected at the given location.
   /// Each location is specified by \p LineOffset and \p Discriminator.
   /// If the location is not found in profile, return error.
   ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset,
                                   uint32_t Discriminator) const {
-    const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
+    const auto &ret = BodySamples.find(
+        mapIRLocToProfileLoc(LineLocation(LineOffset, Discriminator)));
     if (ret == BodySamples.end())
       return std::error_code();
     return ret->second.getSamples();
@@ -858,7 +874,8 @@ class FunctionSamples {
   /// If the location is not found in profile, return error.
   ErrorOr<SampleRecord::CallTargetMap>
   findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const {
-    const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
+    const auto &ret = BodySamples.find(
+        mapIRLocToProfileLoc(LineLocation(LineOffset, Discriminator)));
     if (ret == BodySamples.end())
       return std::error_code();
     return ret->second.getCallTargets();
@@ -868,7 +885,7 @@ class FunctionSamples {
   /// CallSite. If the location is not found in profile, return error.
   ErrorOr<SampleRecord::CallTargetMap>
   findCallTargetMapAt(const LineLocation &CallSite) const {
-    const auto &Ret = BodySamples.find(CallSite);
+    const auto &Ret = BodySamples.find(mapIRLocToProfileLoc(CallSite));
     if (Ret == BodySamples.end())
       return std::error_code();
     return Ret->second.getCallTargets();
@@ -876,13 +893,13 @@ class FunctionSamples {
 
   /// Return the function samples at the given callsite location.
   FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
-    return CallsiteSamples[Loc];
+    return CallsiteSamples[mapIRLocToProfileLoc(Loc)];
   }
 
   /// Returns the FunctionSamplesMap at the given \p Loc.
   const FunctionSamplesMap *
   findFunctionSamplesMapAt(const LineLocation &Loc) const {
-    auto iter = CallsiteSamples.find(Loc);
+    auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
     if (iter == CallsiteSamples.end())
       return nullptr;
     return &iter->second;
@@ -1046,6 +1063,11 @@ class FunctionSamples {
 
   uint64_t getFunctionHash() const { return FunctionHash; }
 
+  void setIRToProfileLocationMap(const LocToLocMap *LTLM) {
+    assert(IRToProfileLocationMap == nullptr && "this should be set only once");
+    IRToProfileLocationMap = LTLM;
+  }
+
   /// Return the canonical name for a function, taking into account
   /// suffix elision policy attributes.
   static StringRef getCanonicalFnName(const Function &F) {
@@ -1229,6 +1251,25 @@ class FunctionSamples {
   /// in the call to bar() at line offset 1, the other for all the samples
   /// collected in the call to baz() at line offset 8.
   CallsiteSampleMap CallsiteSamples;
+
+  /// IR to profile location map generated by stale profile matching.
+  ///
+  /// Each entry is a mapping from the location on current build to the matched
+  /// location in the "stale" profile. For example:
+  ///   Profiled source code:
+  ///      void foo() {
+  ///   1    bar();
+  ///      }
+  ///
+  ///   Current source code:
+  ///      void foo() {
+  ///   1    // Code change
+  ///   2    bar();
+  ///      }
+  /// Supposing the stale profile matching algorithm generated the mapping [2 ->
+  /// 1], the profile query using the location of bar on the IR which is 2 will
+  /// be remapped to 1 and find the location of bar in the profile.
+  const LocToLocMap *IRToProfileLocationMap = nullptr;
 };
 
 raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index d6f8c27fbe6b9..3b657b3ab4621 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -811,41 +811,6 @@ class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
   static bool hasFormat(const MemoryBuffer &Buffer);
 };
 
-class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary {
-private:
-  /// Function name table.
-  std::vector<std::string> NameTable;
-  /// The table mapping from function name to the offset of its FunctionSample
-  /// towards file start.
-  DenseMap<StringRef, uint64_t> FuncOffsetTable;
-  /// The set containing the functions to use when compiling a module.
-  DenseSet<StringRef> FuncsToUse;
-  std::error_code verifySPMagic(uint64_t Magic) override;
-  std::error_code readNameTable() override;
-  /// Read a string indirectly via the name table.
-  ErrorOr<StringRef> readStringFromTable() override;
-  std::error_code readHeader() override;
-  std::error_code readFuncOffsetTable();
-
-public:
-  SampleProfileReaderCompactBinary(std::unique_ptr<MemoryBuffer> B,
-                                   LLVMContext &C)
-      : SampleProfileReaderBinary(std::move(B), C, SPF_Compact_Binary) {}
-
-  /// \brief Return true if \p Buffer is in the format supported by this class.
-  static bool hasFormat(const MemoryBuffer &Buffer);
-
-  /// Read samples only for functions to use.
-  std::error_code readImpl() override;
-
-  /// Collect functions with definitions in Module M. Return true if
-  /// the reader has been given a module.
-  bool collectFuncsFromModule() override;
-
-  /// Return whether names in the profile are all MD5 numbers.
-  bool useMD5() override { return true; }
-};
-
 using InlineCallStack = SmallVector<FunctionSamples *, 10>;
 
 // Supported histogram types in GCC.  Currently, we only need support for
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 4edb3b049e704..1f19283ea1dd0 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -414,55 +414,6 @@ class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase {
   }
 };
 
-// CompactBinary is a compact format of binary profile which both reduces
-// the profile size and the load time needed when compiling. It has two
-// major difference with Binary format.
-// 1. It represents all the strings in name table using md5 hash.
-// 2. It saves a function offset table which maps function name index to
-// the offset of its function profile to the start of the binary profile,
-// so by using the function offset table, for those function profiles which
-// will not be needed when compiling a module, the profile reader does't
-// have to read them and it saves compile time if the profile size is huge.
-// The layout of the compact format is shown as follows:
-//
-//    Part1: Profile header, the same as binary format, containing magic
-//           number, version, summary, name table...
-//    Part2: Function Offset Table Offset, which saves the position of
-//           Part4.
-//    Part3: Function profile collection
-//             function1 profile start
-//                 ....
-//             function2 profile start
-//                 ....
-//             function3 profile start
-//                 ....
-//                ......
-//    Part4: Function Offset Table
-//             function1 name index --> function1 profile start
-//             function2 name index --> function2 profile start
-//             function3 name index --> function3 profile start
-//
-// We need Part2 because profile reader can use it to find out and read
-// function offset table without reading Part3 first.
-class SampleProfileWriterCompactBinary : public SampleProfileWriterBinary {
-  using SampleProfileWriterBinary::SampleProfileWriterBinary;
-
-public:
-  std::error_code writeSample(const FunctionSamples &S) override;
-  std::error_code write(const SampleProfileMap &ProfileMap) override;
-
-protected:
-  /// The table mapping from function name to the offset of its FunctionSample
-  /// towards profile start.
-  MapVector<StringRef, uint64_t> FuncOffsetTable;
-  /// The offset of the slot to be filled with the offset of FuncOffsetTable
-  /// towards profile start.
-  uint64_t TableOffset;
-  std::error_code writeNameTable() override;
-  std::error_code writeHeader(const SampleProfileMap &ProfileMap) override;
-  std::error_code writeFuncOffsetTable();
-};
-
 } // end namespace sampleprof
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Support/CachePruning.h b/llvm/include/llvm/Support/CachePruning.h
index 30b5c08942c56..17e148830a73f 100644
--- a/llvm/include/llvm/Support/CachePruning.h
+++ b/llvm/include/llvm/Support/CachePruning.h
@@ -28,8 +28,8 @@ class StringRef;
 struct CachePruningPolicy {
   /// The pruning interval. This is intended to be used to avoid scanning the
   /// directory too often. It does not impact the decision of which file to
-  /// prune. A value of 0 forces the scan to occur. A value of None disables
-  /// pruning.
+  /// prune. A value of 0 forces the scan to occur. A value of std::nullopt
+  /// disables pruning.
   std::optional<std::chrono::seconds> Interval = std::chrono::seconds(1200);
 
   /// The expiration for a file. When a file hasn't been accessed for Expiration
diff --git a/llvm/include/llvm/Support/ConvertEBCDIC.h b/llvm/include/llvm/Support/ConvertEBCDIC.h
new file mode 100644
index 0000000000000..ea761b31e022b
--- /dev/null
+++ b/llvm/include/llvm/Support/ConvertEBCDIC.h
@@ -0,0 +1,28 @@
+//===--- ConvertEBCDIC.h - UTF8/EBCDIC CharSet Conversion -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides utility functions for converting between EBCDIC-1047 and
+/// UTF-8.
+///
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include <system_error>
+
+namespace llvm {
+namespace ConverterEBCDIC {
+std::error_code convertToEBCDIC(StringRef Source,
+                                SmallVectorImpl<char> &Result);
+
+void convertToUTF8(StringRef Source, SmallVectorImpl<char> &Result);
+
+} // namespace ConverterEBCDIC
+} // namespace llvm
diff --git a/llvm/include/llvm/Support/Format.h b/llvm/include/llvm/Support/Format.h
index c22c941ae06ef..3b93a0f206523 100644
--- a/llvm/include/llvm/Support/Format.h
+++ b/llvm/include/llvm/Support/Format.h
@@ -215,7 +215,8 @@ inline FormattedNumber format_decimal(int64_t N, unsigned Width) {
 class FormattedBytes {
   ArrayRef<uint8_t> Bytes;
 
-  // If not None, display offsets for each line relative to starting value.
+  // If not std::nullopt, display offsets for each line relative to starting
+  // value.
   std::optional<uint64_t> FirstByteOffset;
   uint32_t IndentLevel;  // Number of characters to indent each line.
   uint32_t NumPerLine;   // Number of bytes to show per line.
diff --git a/llvm/include/llvm/Support/LowLevelTypeImpl.h b/llvm/include/llvm/Support/LowLevelTypeImpl.h
deleted file mode 100644
index f33c1ecde0394..0000000000000
--- a/llvm/include/llvm/Support/LowLevelTypeImpl.h
+++ /dev/null
@@ -1,431 +0,0 @@
-//== llvm/Support/LowLevelTypeImpl.h --------------------------- -*- C++ -*-==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-/// \file
-/// Implement a low-level type suitable for MachineInstr level instruction
-/// selection.
-///
-/// For a type attached to a MachineInstr, we only care about 2 details: total
-/// size and the number of vector lanes (if any). Accordingly, there are 4
-/// possible valid type-kinds:
-///
-///    * `sN` for scalars and aggregates
-///    * `<N x sM>` for vectors, which must have at least 2 elements.
-///    * `pN` for pointers
-///
-/// Other information required for correct selection is expected to be carried
-/// by the opcode, or non-type flags. For example the distinction between G_ADD
-/// and G_FADD for int/float or fast-math flags.
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_LOWLEVELTYPEIMPL_H
-#define LLVM_SUPPORT_LOWLEVELTYPEIMPL_H
-
-#include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/MachineValueType.h"
-#include <cassert>
-
-namespace llvm {
-
-class Type;
-class raw_ostream;
-
-class LLT {
-public:
-  /// Get a low-level scalar or aggregate "bag of bits".
-  static constexpr LLT scalar(unsigned SizeInBits) {
-    return LLT{/*isPointer=*/false, /*isVector=*/false, /*isScalar=*/true,
-               ElementCount::getFixed(0), SizeInBits,
-               /*AddressSpace=*/0};
-  }
-
-  /// Get a low-level pointer in the given address space.
-  static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits) {
-    assert(SizeInBits > 0 && "invalid pointer size");
-    return LLT{/*isPointer=*/true, /*isVector=*/false, /*isScalar=*/false,
-               ElementCount::getFixed(0), SizeInBits, AddressSpace};
-  }
-
-  /// Get a low-level vector of some number of elements and element width.
-  static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits) {
-    assert(!EC.isScalar() && "invalid number of vector elements");
-    return LLT{/*isPointer=*/false, /*isVector=*/true, /*isScalar=*/false,
-               EC, ScalarSizeInBits, /*AddressSpace=*/0};
-  }
-
-  /// Get a low-level vector of some number of elements and element type.
-  static constexpr LLT vector(ElementCount EC, LLT ScalarTy) {
-    assert(!EC.isScalar() && "invalid number of vector elements");
-    assert(!ScalarTy.isVector() && "invalid vector element type");
-    return LLT{ScalarTy.isPointer(),
-               /*isVector=*/true,
-               /*isScalar=*/false,
-               EC,
-               ScalarTy.getSizeInBits().getFixedValue(),
-               ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0};
-  }
-
-  /// Get a low-level fixed-width vector of some number of elements and element
-  /// width.
-  static constexpr LLT fixed_vector(unsigned NumElements,
-                                    unsigned ScalarSizeInBits) {
-    return vector(ElementCount::getFixed(NumElements), ScalarSizeInBits);
-  }
-
-  /// Get a low-level fixed-width vector of some number of elements and element
-  /// type.
-  static constexpr LLT fixed_vector(unsigned NumElements, LLT ScalarTy) {
-    return vector(ElementCount::getFixed(NumElements), ScalarTy);
-  }
-
-  /// Get a low-level scalable vector of some number of elements and element
-  /// width.
-  static constexpr LLT scalable_vector(unsigned MinNumElements,
-                                       unsigned ScalarSizeInBits) {
-    return vector(ElementCount::getScalable(MinNumElements), ScalarSizeInBits);
-  }
-
-  /// Get a low-level scalable vector of some number of elements and element
-  /// type.
-  static constexpr LLT scalable_vector(unsigned MinNumElements, LLT ScalarTy) {
-    return vector(ElementCount::getScalable(MinNumElements), ScalarTy);
-  }
-
-  static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy) {
-    return EC.isScalar() ? ScalarTy : LLT::vector(EC, ScalarTy);
-  }
-
-  static constexpr LLT scalarOrVector(ElementCount EC, uint64_t ScalarSize) {
-    assert(ScalarSize <= std::numeric_limits<unsigned>::max() &&
-           "Not enough bits in LLT to represent size");
-    return scalarOrVector(EC, LLT::scalar(static_cast<unsigned>(ScalarSize)));
-  }
-
-  explicit constexpr LLT(bool isPointer, bool isVector, bool isScalar,
-                         ElementCount EC, uint64_t SizeInBits,
-                         unsigned AddressSpace)
-      : LLT() {
-    init(isPointer, isVector, isScalar, EC, SizeInBits, AddressSpace);
-  }
-  explicit constexpr LLT()
-      : IsScalar(false), IsPointer(false), IsVector(false), RawData(0) {}
-
-  explicit LLT(MVT VT);
-
-  constexpr bool isValid() const { return IsScalar || RawData != 0; }
-
-  constexpr bool isScalar() const { return IsScalar; }
-
-  constexpr bool isPointer() const {
-    return isValid() && IsPointer && !IsVector;
-  }
-
-  constexpr bool isVector() const { return isValid() && IsVector; }
-
-  /// Returns the number of elements in a vector LLT. Must only be called on
-  /// vector types.
-  constexpr uint16_t getNumElements() const {
-    if (isScalable())
-      llvm::reportInvalidSizeRequest(
-          "Possible incorrect use of LLT::getNumElements() for "
-          "scalable vector. Scalable flag may be dropped, use "
-          "LLT::getElementCount() instead");
-    return getElementCount().getKnownMinValue();
-  }
-
-  /// Returns true if the LLT is a scalable vector. Must only be called on
-  /// vector types.
-  constexpr bool isScalable() const {
-    assert(isVector() && "Expected a vector type");
-    return IsPointer ? getFieldValue(PointerVectorScalableFieldInfo)
-                     : getFieldValue(VectorScalableFieldInfo);
-  }
-
-  constexpr ElementCount getElementCount() const {
-    assert(IsVector && "cannot get number of elements on scalar/aggregate");
-    return ElementCount::get(IsPointer
-                                 ? getFieldValue(PointerVectorElementsFieldInfo)
-                                 : getFieldValue(VectorElementsFieldInfo),
-                             isScalable());
-  }
-
-  /// Returns the total size of the type. Must only be called on sized types.
-  constexpr TypeSize getSizeInBits() const {
-    if (isPointer() || isScalar())
-      return TypeSize::Fixed(getScalarSizeInBits());
-    auto EC = getElementCount();
-    return TypeSize(getScalarSizeInBits() * EC.getKnownMinValue(),
-                    EC.isScalable());
-  }
-
-  /// Returns the total size of the type in bytes, i.e. number of whole bytes
-  /// needed to represent the size in bits. Must only be called on sized types.
-  constexpr TypeSize getSizeInBytes() const {
-    TypeSize BaseSize = getSizeInBits();
-    return {(BaseSize.getKnownMinValue() + 7) / 8, BaseSize.isScalable()};
-  }
-
-  constexpr LLT getScalarType() const {
-    return isVector() ? getElementType() : *this;
-  }
-
-  /// If this type is a vector, return a vector with the same number of elements
-  /// but the new element type. Otherwise, return the new element type.
-  constexpr LLT changeElementType(LLT NewEltTy) const {
-    return isVector() ? LLT::vector(getElementCount(), NewEltTy) : NewEltTy;
-  }
-
-  /// If this type is a vector, return a vector with the same number of elements
-  /// but the new element size. Otherwise, return the new element type. Invalid
-  /// for pointer types. For pointer types, use changeElementType.
-  constexpr LLT changeElementSize(unsigned NewEltSize) const {
-    assert(!getScalarType().isPointer() &&
-           "invalid to directly change element size for pointers");
-    return isVector() ? LLT::vector(getElementCount(), NewEltSize)
-                      : LLT::scalar(NewEltSize);
-  }
-
-  /// Return a vector or scalar with the same element type and the new element
-  /// count.
-  constexpr LLT changeElementCount(ElementCount EC) const {
-    return LLT::scalarOrVector(EC, getScalarType());
-  }
-
-  /// Return a type that is \p Factor times smaller. Reduces the number of
-  /// elements if this is a vector, or the bitwidth for scalar/pointers. Does
-  /// not attempt to handle cases that aren't evenly divisible.
-  constexpr LLT divide(int Factor) const {
-    assert(Factor != 1);
-    assert((!isScalar() || getScalarSizeInBits() != 0) &&
-           "cannot divide scalar of size zero");
-    if (isVector()) {
-      assert(getElementCount().isKnownMultipleOf(Factor));
-      return scalarOrVector(getElementCount().divideCoefficientBy(Factor),
-                            getElementType());
-    }
-
-    assert(getScalarSizeInBits() % Factor == 0);
-    return scalar(getScalarSizeInBits() / Factor);
-  }
-
-  /// Produce a vector type that is \p Factor times bigger, preserving the
-  /// element type. For a scalar or pointer, this will produce a new vector with
-  /// \p Factor elements.
-  constexpr LLT multiplyElements(int Factor) const {
-    if (isVector()) {
-      return scalarOrVector(getElementCount().multiplyCoefficientBy(Factor),
-                            getElementType());
-    }
-
-    return fixed_vector(Factor, *this);
-  }
-
-  constexpr bool isByteSized() const {
-    return getSizeInBits().isKnownMultipleOf(8);
-  }
-
-  constexpr unsigned getScalarSizeInBits() const {
-    if (IsScalar)
-      return getFieldValue(ScalarSizeFieldInfo);
-    if (IsVector) {
-      if (!IsPointer)
-        return getFieldValue(VectorSizeFieldInfo);
-      else
-        return getFieldValue(PointerVectorSizeFieldInfo);
-    } else if (IsPointer)
-      return getFieldValue(PointerSizeFieldInfo);
-    else
-      llvm_unreachable("unexpected LLT");
-  }
-
-  constexpr unsigned getAddressSpace() const {
-    assert(RawData != 0 && "Invalid Type");
-    assert(IsPointer && "cannot get address space of non-pointer type");
-    if (!IsVector)
-      return getFieldValue(PointerAddressSpaceFieldInfo);
-    else
-      return getFieldValue(PointerVectorAddressSpaceFieldInfo);
-  }
-
-  /// Returns the vector's element type. Only valid for vector types.
-  constexpr LLT getElementType() const {
-    assert(isVector() && "cannot get element type of scalar/aggregate");
-    if (IsPointer)
-      return pointer(getAddressSpace(), getScalarSizeInBits());
-    else
-      return scalar(getScalarSizeInBits());
-  }
-
-  void print(raw_ostream &OS) const;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  LLVM_DUMP_METHOD void dump() const {
-    print(dbgs());
-    dbgs() << '\n';
-  }
-#endif
-
-  constexpr bool operator==(const LLT &RHS) const {
-    return IsPointer == RHS.IsPointer && IsVector == RHS.IsVector &&
-           IsScalar == RHS.IsScalar && RHS.RawData == RawData;
-  }
-
-  constexpr bool operator!=(const LLT &RHS) const { return !(*this == RHS); }
-
-  friend struct DenseMapInfo<LLT>;
-  friend class GISelInstProfileBuilder;
-
-private:
-  /// LLT is packed into 64 bits as follows:
-  /// isScalar : 1
-  /// isPointer : 1
-  /// isVector  : 1
-  /// with 61 bits remaining for Kind-specific data, packed in bitfields
-  /// as described below. As there isn't a simple portable way to pack bits
-  /// into bitfields, here the different fields in the packed structure is
-  /// described in static const *Field variables. Each of these variables
-  /// is a 2-element array, with the first element describing the bitfield size
-  /// and the second element describing the bitfield offset.
-  typedef int BitFieldInfo[2];
-  ///
-  /// This is how the bitfields are packed per Kind:
-  /// * Invalid:
-  ///   gets encoded as RawData == 0, as that is an invalid encoding, since for
-  ///   valid encodings, SizeInBits/SizeOfElement must be larger than 0.
-  /// * Non-pointer scalar (isPointer == 0 && isVector == 0):
-  ///   SizeInBits: 32;
-  static const constexpr BitFieldInfo ScalarSizeFieldInfo{32, 0};
-  /// * Pointer (isPointer == 1 && isVector == 0):
-  ///   SizeInBits: 16;
-  ///   AddressSpace: 24;
-  static const constexpr BitFieldInfo PointerSizeFieldInfo{16, 0};
-  static const constexpr BitFieldInfo PointerAddressSpaceFieldInfo{
-      24, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]};
-  static_assert((PointerAddressSpaceFieldInfo[0] +
-                 PointerAddressSpaceFieldInfo[1]) <= 61,
-                "Insufficient bits to encode all data");
-  /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1):
-  ///   NumElements: 16;
-  ///   SizeOfElement: 32;
-  ///   Scalable: 1;
-  static const constexpr BitFieldInfo VectorElementsFieldInfo{16, 0};
-  static const constexpr BitFieldInfo VectorSizeFieldInfo{
-      32, VectorElementsFieldInfo[0] + VectorElementsFieldInfo[1]};
-  static const constexpr BitFieldInfo VectorScalableFieldInfo{
-      1, VectorSizeFieldInfo[0] + VectorSizeFieldInfo[1]};
-  static_assert((VectorSizeFieldInfo[0] + VectorSizeFieldInfo[1]) <= 61,
-                "Insufficient bits to encode all data");
-  /// * Vector-of-pointer (isPointer == 1 && isVector == 1):
-  ///   NumElements: 16;
-  ///   SizeOfElement: 16;
-  ///   AddressSpace: 24;
-  ///   Scalable: 1;
-  static const constexpr BitFieldInfo PointerVectorElementsFieldInfo{16, 0};
-  static const constexpr BitFieldInfo PointerVectorSizeFieldInfo{
-      16,
-      PointerVectorElementsFieldInfo[1] + PointerVectorElementsFieldInfo[0]};
-  static const constexpr BitFieldInfo PointerVectorAddressSpaceFieldInfo{
-      24, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]};
-  static const constexpr BitFieldInfo PointerVectorScalableFieldInfo{
-      1, PointerVectorAddressSpaceFieldInfo[0] +
-             PointerVectorAddressSpaceFieldInfo[1]};
-  static_assert((PointerVectorAddressSpaceFieldInfo[0] +
-                 PointerVectorAddressSpaceFieldInfo[1]) <= 61,
-                "Insufficient bits to encode all data");
-
-  uint64_t IsScalar : 1;
-  uint64_t IsPointer : 1;
-  uint64_t IsVector : 1;
-  uint64_t RawData : 61;
-
-  static constexpr uint64_t getMask(const BitFieldInfo FieldInfo) {
-    const int FieldSizeInBits = FieldInfo[0];
-    return (((uint64_t)1) << FieldSizeInBits) - 1;
-  }
-  static constexpr uint64_t maskAndShift(uint64_t Val, uint64_t Mask,
-                                         uint8_t Shift) {
-    assert(Val <= Mask && "Value too large for field");
-    return (Val & Mask) << Shift;
-  }
-  static constexpr uint64_t maskAndShift(uint64_t Val,
-                                         const BitFieldInfo FieldInfo) {
-    return maskAndShift(Val, getMask(FieldInfo), FieldInfo[1]);
-  }
-
-  constexpr uint64_t getFieldValue(const BitFieldInfo FieldInfo) const {
-    return getMask(FieldInfo) & (RawData >> FieldInfo[1]);
-  }
-
-  constexpr void init(bool IsPointer, bool IsVector, bool IsScalar,
-                      ElementCount EC, uint64_t SizeInBits,
-                      unsigned AddressSpace) {
-    assert(SizeInBits <= std::numeric_limits<unsigned>::max() &&
-           "Not enough bits in LLT to represent size");
-    this->IsPointer = IsPointer;
-    this->IsVector = IsVector;
-    this->IsScalar = IsScalar;
-    if (IsScalar)
-      RawData = maskAndShift(SizeInBits, ScalarSizeFieldInfo);
-    else if (IsVector) {
-      assert(EC.isVector() && "invalid number of vector elements");
-      if (!IsPointer)
-        RawData =
-            maskAndShift(EC.getKnownMinValue(), VectorElementsFieldInfo) |
-            maskAndShift(SizeInBits, VectorSizeFieldInfo) |
-            maskAndShift(EC.isScalable() ? 1 : 0, VectorScalableFieldInfo);
-      else
-        RawData =
-            maskAndShift(EC.getKnownMinValue(),
-                         PointerVectorElementsFieldInfo) |
-            maskAndShift(SizeInBits, PointerVectorSizeFieldInfo) |
-            maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo) |
-            maskAndShift(EC.isScalable() ? 1 : 0,
-                         PointerVectorScalableFieldInfo);
-    } else if (IsPointer)
-      RawData = maskAndShift(SizeInBits, PointerSizeFieldInfo) |
-                maskAndShift(AddressSpace, PointerAddressSpaceFieldInfo);
-    else
-      llvm_unreachable("unexpected LLT configuration");
-  }
-
-public:
-  constexpr uint64_t getUniqueRAWLLTData() const {
-    return ((uint64_t)RawData) << 3 | ((uint64_t)IsScalar) << 2 |
-           ((uint64_t)IsPointer) << 1 | ((uint64_t)IsVector);
-  }
-};
-
-inline raw_ostream& operator<<(raw_ostream &OS, const LLT &Ty) {
-  Ty.print(OS);
-  return OS;
-}
-
-template<> struct DenseMapInfo<LLT> {
-  static inline LLT getEmptyKey() {
-    LLT Invalid;
-    Invalid.IsPointer = true;
-    return Invalid;
-  }
-  static inline LLT getTombstoneKey() {
-    LLT Invalid;
-    Invalid.IsVector = true;
-    return Invalid;
-  }
-  static inline unsigned getHashValue(const LLT &Ty) {
-    uint64_t Val = Ty.getUniqueRAWLLTData();
-    return DenseMapInfo<uint64_t>::getHashValue(Val);
-  }
-  static bool isEqual(const LLT &LHS, const LLT &RHS) {
-    return LHS == RHS;
-  }
-};
-
-}
-
-#endif // LLVM_SUPPORT_LOWLEVELTYPEIMPL_H
diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h
deleted file mode 100644
index 2a2ab24e4f2cc..0000000000000
--- a/llvm/include/llvm/Support/MachineValueType.h
+++ /dev/null
@@ -1,1610 +0,0 @@
-//===- Support/MachineValueType.h - Machine-Level types ---------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the set of machine-level target independent types which
-// legal values in the code generator use.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_MACHINEVALUETYPE_H
-#define LLVM_SUPPORT_MACHINEVALUETYPE_H
-
-#include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/iterator_range.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/TypeSize.h"
-#include <cassert>
-
-namespace llvm {
-
-  class Type;
-
-  /// Machine Value Type. Every type that is supported natively by some
-  /// processor targeted by LLVM occurs here. This means that any legal value
-  /// type can be represented by an MVT.
-  class MVT {
-  public:
-    enum SimpleValueType : uint8_t {
-      // clang-format off
-
-      // Simple value types that aren't explicitly part of this enumeration
-      // are considered extended value types.
-      INVALID_SIMPLE_VALUE_TYPE = 0,
-
-      // If you change this numbering, you must change the values in
-      // ValueTypes.td as well!
-      Other          =   1,   // This is a non-standard value
-      i1             =   2,   // This is a 1 bit integer value
-      i2             =   3,   // This is a 2 bit integer value
-      i4             =   4,   // This is a 4 bit integer value
-      i8             =   5,   // This is an 8 bit integer value
-      i16            =   6,   // This is a 16 bit integer value
-      i32            =   7,   // This is a 32 bit integer value
-      i64            =   8,   // This is a 64 bit integer value
-      i128           =   9,   // This is a 128 bit integer value
-
-      FIRST_INTEGER_VALUETYPE = i1,
-      LAST_INTEGER_VALUETYPE  = i128,
-
-      bf16           =  10,   // This is a 16 bit brain floating point value
-      f16            =  11,   // This is a 16 bit floating point value
-      f32            =  12,   // This is a 32 bit floating point value
-      f64            =  13,   // This is a 64 bit floating point value
-      f80            =  14,   // This is a 80 bit floating point value
-      f128           =  15,   // This is a 128 bit floating point value
-      ppcf128        =  16,   // This is a PPC 128-bit floating point value
-
-      FIRST_FP_VALUETYPE = bf16,
-      LAST_FP_VALUETYPE  = ppcf128,
-
-      v1i1           =  17,   //    1 x i1
-      v2i1           =  18,   //    2 x i1
-      v4i1           =  19,   //    4 x i1
-      v8i1           =  20,   //    8 x i1
-      v16i1          =  21,   //   16 x i1
-      v32i1          =  22,   //   32 x i1
-      v64i1          =  23,   //   64 x i1
-      v128i1         =  24,   //  128 x i1
-      v256i1         =  25,   //  256 x i1
-      v512i1         =  26,   //  512 x i1
-      v1024i1        =  27,   // 1024 x i1
-      v2048i1        =  28,   // 2048 x i1
-
-      v128i2         =  29,   //  128 x i2
-      v256i2         =  30,   //  256 x i2
-
-      v64i4          =  31,   //   64 x i4
-      v128i4         =  32,   //  128 x i4
-
-      v1i8           =  33,   //    1 x i8
-      v2i8           =  34,   //    2 x i8
-      v4i8           =  35,   //    4 x i8
-      v8i8           =  36,   //    8 x i8
-      v16i8          =  37,   //   16 x i8
-      v32i8          =  38,   //   32 x i8
-      v64i8          =  39,   //   64 x i8
-      v128i8         =  40,   //  128 x i8
-      v256i8         =  41,   //  256 x i8
-      v512i8         =  42,   //  512 x i8
-      v1024i8        =  43,   // 1024 x i8
-
-      v1i16          =  44,   //   1 x i16
-      v2i16          =  45,   //   2 x i16
-      v3i16          =  46,   //   3 x i16
-      v4i16          =  47,   //   4 x i16
-      v8i16          =  48,   //   8 x i16
-      v16i16         =  49,   //  16 x i16
-      v32i16         =  50,   //  32 x i16
-      v64i16         =  51,   //  64 x i16
-      v128i16        =  52,   // 128 x i16
-      v256i16        =  53,   // 256 x i16
-      v512i16        =  54,   // 512 x i16
-
-      v1i32          =  55,   //    1 x i32
-      v2i32          =  56,   //    2 x i32
-      v3i32          =  57,   //    3 x i32
-      v4i32          =  58,   //    4 x i32
-      v5i32          =  59,   //    5 x i32
-      v6i32          =  60,   //    6 x i32
-      v7i32          =  61,   //    7 x i32
-      v8i32          =  62,   //    8 x i32
-      v9i32          =  63,   //    9 x i32
-      v10i32         =  64,   //   10 x i32
-      v11i32         =  65,   //   11 x i32
-      v12i32         =  66,   //   12 x i32
-      v16i32         =  67,   //   16 x i32
-      v32i32         =  68,   //   32 x i32
-      v64i32         =  69,   //   64 x i32
-      v128i32        =  70,   //  128 x i32
-      v256i32        =  71,   //  256 x i32
-      v512i32        =  72,   //  512 x i32
-      v1024i32       =  73,   // 1024 x i32
-      v2048i32       =  74,   // 2048 x i32
-
-      v1i64          =  75,   //   1 x i64
-      v2i64          =  76,   //   2 x i64
-      v3i64          =  77,   //   3 x i64
-      v4i64          =  78,   //   4 x i64
-      v8i64          =  79,   //   8 x i64
-      v16i64         =  80,   //  16 x i64
-      v32i64         =  81,   //  32 x i64
-      v64i64         =  82,   //  64 x i64
-      v128i64        =  83,   // 128 x i64
-      v256i64        =  84,   // 256 x i64
-
-      v1i128         =  85,   //  1 x i128
-
-      FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
-      LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i128,
-
-      v1f16          =  86,   //    1 x f16
-      v2f16          =  87,   //    2 x f16
-      v3f16          =  88,   //    3 x f16
-      v4f16          =  89,   //    4 x f16
-      v8f16          =  90,   //    8 x f16
-      v16f16         =  91,   //   16 x f16
-      v32f16         =  92,   //   32 x f16
-      v64f16         =  93,   //   64 x f16
-      v128f16        =  94,   //  128 x f16
-      v256f16        =  95,   //  256 x f16
-      v512f16        =  96,   //  512 x f16
-
-      v2bf16         =  97,   //    2 x bf16
-      v3bf16         =  98,   //    3 x bf16
-      v4bf16         =  99,   //    4 x bf16
-      v8bf16         = 100,   //    8 x bf16
-      v16bf16        = 101,   //   16 x bf16
-      v32bf16        = 102,   //   32 x bf16
-      v64bf16        = 103,   //   64 x bf16
-      v128bf16       = 104,   //  128 x bf16
-
-      v1f32          = 105,   //    1 x f32
-      v2f32          = 106,   //    2 x f32
-      v3f32          = 107,   //    3 x f32
-      v4f32          = 108,   //    4 x f32
-      v5f32          = 109,   //    5 x f32
-      v6f32          = 110,   //    6 x f32
-      v7f32          = 111,   //    7 x f32
-      v8f32          = 112,   //    8 x f32
-      v9f32          = 113,   //    9 x f32
-      v10f32         = 114,   //   10 x f32
-      v11f32         = 115,   //   11 x f32
-      v12f32         = 116,   //   12 x f32
-      v16f32         = 117,   //   16 x f32
-
-      v32f32         = 118,   //   32 x f32
-      v64f32         = 119,   //   64 x f32
-      v128f32        = 120,   //  128 x f32
-      v256f32        = 121,   //  256 x f32
-      v512f32        = 122,   //  512 x f32
-      v1024f32       = 123,   // 1024 x f32
-      v2048f32       = 124,   // 2048 x f32
-
-      v1f64          = 125,   //    1 x f64
-      v2f64          = 126,   //    2 x f64
-      v3f64          = 127,   //    3 x f64
-      v4f64          = 128,   //    4 x f64
-      v8f64          = 129,   //    8 x f64
-      v16f64         = 130,   //   16 x f64
-      v32f64         = 131,   //   32 x f64
-      v64f64         = 132,   //   64 x f64
-      v128f64        = 133,   //  128 x f64
-      v256f64        = 134,   //  256 x f64
-
-      FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE = v1f16,
-      LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v256f64,
-
-      FIRST_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
-      LAST_FIXEDLEN_VECTOR_VALUETYPE = v256f64,
-
-      nxv1i1         = 135,   // n x  1 x i1
-      nxv2i1         = 136,   // n x  2 x i1
-      nxv4i1         = 137,   // n x  4 x i1
-      nxv8i1         = 138,   // n x  8 x i1
-      nxv16i1        = 139,   // n x 16 x i1
-      nxv32i1        = 140,   // n x 32 x i1
-      nxv64i1        = 141,   // n x 64 x i1
-
-      nxv1i8         = 142,   // n x  1 x i8
-      nxv2i8         = 143,   // n x  2 x i8
-      nxv4i8         = 144,   // n x  4 x i8
-      nxv8i8         = 145,   // n x  8 x i8
-      nxv16i8        = 146,   // n x 16 x i8
-      nxv32i8        = 147,   // n x 32 x i8
-      nxv64i8        = 148,   // n x 64 x i8
-
-      nxv1i16        = 149,  // n x  1 x i16
-      nxv2i16        = 150,  // n x  2 x i16
-      nxv4i16        = 151,  // n x  4 x i16
-      nxv8i16        = 152,  // n x  8 x i16
-      nxv16i16       = 153,  // n x 16 x i16
-      nxv32i16       = 154,  // n x 32 x i16
-
-      nxv1i32        = 155,  // n x  1 x i32
-      nxv2i32        = 156,  // n x  2 x i32
-      nxv4i32        = 157,  // n x  4 x i32
-      nxv8i32        = 158,  // n x  8 x i32
-      nxv16i32       = 159,  // n x 16 x i32
-      nxv32i32       = 160,  // n x 32 x i32
-
-      nxv1i64        = 161,  // n x  1 x i64
-      nxv2i64        = 162,  // n x  2 x i64
-      nxv4i64        = 163,  // n x  4 x i64
-      nxv8i64        = 164,  // n x  8 x i64
-      nxv16i64       = 165,  // n x 16 x i64
-      nxv32i64       = 166,  // n x 32 x i64
-
-      FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
-      LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv32i64,
-
-      nxv1f16        = 167,  // n x  1 x f16
-      nxv2f16        = 168,  // n x  2 x f16
-      nxv4f16        = 169,  // n x  4 x f16
-      nxv8f16        = 170,  // n x  8 x f16
-      nxv16f16       = 171,  // n x 16 x f16
-      nxv32f16       = 172,  // n x 32 x f16
-
-      nxv1bf16       = 173,  // n x  1 x bf16
-      nxv2bf16       = 174,  // n x  2 x bf16
-      nxv4bf16       = 175,  // n x  4 x bf16
-      nxv8bf16       = 176,  // n x  8 x bf16
-      nxv16bf16      = 177,  // n x 16 x bf16
-      nxv32bf16      = 178,  // n x 32 x bf16
-
-      nxv1f32        = 179,  // n x  1 x f32
-      nxv2f32        = 180,  // n x  2 x f32
-      nxv4f32        = 181,  // n x  4 x f32
-      nxv8f32        = 182,  // n x  8 x f32
-      nxv16f32       = 183,  // n x 16 x f32
-
-      nxv1f64        = 184,  // n x  1 x f64
-      nxv2f64        = 185,  // n x  2 x f64
-      nxv4f64        = 186,  // n x  4 x f64
-      nxv8f64        = 187,  // n x  8 x f64
-
-      FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv1f16,
-      LAST_FP_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
-
-      FIRST_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
-      LAST_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
-
-      FIRST_VECTOR_VALUETYPE = v1i1,
-      LAST_VECTOR_VALUETYPE  = nxv8f64,
-
-      x86mmx         = 188,    // This is an X86 MMX value
-
-      Glue           = 189,    // This glues nodes together during pre-RA sched
-
-      isVoid         = 190,    // This has no value
-
-      Untyped        = 191,    // This value takes a register, but has
-                               // unspecified type.  The register class
-                               // will be determined by the opcode.
-
-      funcref        = 192,    // WebAssembly's funcref type
-      externref      = 193,    // WebAssembly's externref type
-      x86amx         = 194,    // This is an X86 AMX value
-      i64x8          = 195,    // 8 Consecutive GPRs (AArch64)
-      aarch64svcount = 196,    // AArch64 predicate-as-counter
-      spirvbuiltin   = 197,    // SPIR-V's builtin type
-
-      FIRST_VALUETYPE =  1,    // This is always the beginning of the list.
-      LAST_VALUETYPE = spirvbuiltin, // This always remains at the end of the list.
-      VALUETYPE_SIZE = LAST_VALUETYPE + 1,
-
-      // This is the current maximum for LAST_VALUETYPE.
-      // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
-      // This value must be a multiple of 32.
-      MAX_ALLOWED_VALUETYPE = 224,
-
-      // A value of type llvm::TokenTy
-      token          = 248,
-
-      // This is MDNode or MDString.
-      Metadata       = 249,
-
-      // An int value the size of the pointer of the current
-      // target to any address space. This must only be used internal to
-      // tblgen. Other than for overloading, we treat iPTRAny the same as iPTR.
-      iPTRAny        = 250,
-
-      // A vector with any length and element size. This is used
-      // for intrinsics that have overloadings based on vector types.
-      // This is only for tblgen's consumption!
-      vAny           = 251,
-
-      // Any floating-point or vector floating-point value. This is used
-      // for intrinsics that have overloadings based on floating-point types.
-      // This is only for tblgen's consumption!
-      fAny           = 252,
-
-      // An integer or vector integer value of any bit width. This is
-      // used for intrinsics that have overloadings based on integer bit widths.
-      // This is only for tblgen's consumption!
-      iAny           = 253,
-
-      // An int value the size of the pointer of the current
-      // target.  This should only be used internal to tblgen!
-      iPTR           = 254,
-
-      // Any type. This is used for intrinsics that have overloadings.
-      // This is only for tblgen's consumption!
-      Any            = 255
-
-      // clang-format on
-    };
-
-    SimpleValueType SimpleTy = INVALID_SIMPLE_VALUE_TYPE;
-
-    constexpr MVT() = default;
-    constexpr MVT(SimpleValueType SVT) : SimpleTy(SVT) {}
-
-    bool operator>(const MVT& S)  const { return SimpleTy >  S.SimpleTy; }
-    bool operator<(const MVT& S)  const { return SimpleTy <  S.SimpleTy; }
-    bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; }
-    bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; }
-    bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; }
-    bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; }
-
-    /// Support for debugging, callable in GDB: VT.dump()
-    void dump() const;
-
-    /// Implement operator<<.
-    void print(raw_ostream &OS) const;
-
-    /// Return true if this is a valid simple valuetype.
-    bool isValid() const {
-      return (SimpleTy >= MVT::FIRST_VALUETYPE &&
-              SimpleTy <= MVT::LAST_VALUETYPE);
-    }
-
-    /// Return true if this is a FP or a vector FP type.
-    bool isFloatingPoint() const {
-      return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE));
-    }
-
-    /// Return true if this is an integer or a vector integer type.
-    bool isInteger() const {
-      return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE));
-    }
-
-    /// Return true if this is an integer, not including vectors.
-    bool isScalarInteger() const {
-      return (SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
-              SimpleTy <= MVT::LAST_INTEGER_VALUETYPE);
-    }
-
-    /// Return true if this is a vector value type.
-    bool isVector() const {
-      return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_VECTOR_VALUETYPE);
-    }
-
-    /// Return true if this is a vector value type where the
-    /// runtime length is machine dependent
-    bool isScalableVector() const {
-      return (SimpleTy >= MVT::FIRST_SCALABLE_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_SCALABLE_VECTOR_VALUETYPE);
-    }
-
-    /// Return true if this is a custom target type that has a scalable size.
-    bool isScalableTargetExtVT() const {
-      return SimpleTy == MVT::aarch64svcount;
-    }
-
-    /// Return true if the type is a scalable type.
-    bool isScalableVT() const {
-      return isScalableVector() || isScalableTargetExtVT();
-    }
-
-    bool isFixedLengthVector() const {
-      return (SimpleTy >= MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE);
-    }
-
-    /// Return true if this is a 16-bit vector type.
-    bool is16BitVector() const {
-      return (SimpleTy == MVT::v2i8  || SimpleTy == MVT::v1i16 ||
-              SimpleTy == MVT::v16i1 || SimpleTy == MVT::v1f16);
-    }
-
-    /// Return true if this is a 32-bit vector type.
-    bool is32BitVector() const {
-      return (SimpleTy == MVT::v32i1 || SimpleTy == MVT::v4i8   ||
-              SimpleTy == MVT::v2i16 || SimpleTy == MVT::v1i32  ||
-              SimpleTy == MVT::v2f16 || SimpleTy == MVT::v2bf16 ||
-              SimpleTy == MVT::v1f32);
-    }
-
-    /// Return true if this is a 64-bit vector type.
-    bool is64BitVector() const {
-      return (SimpleTy == MVT::v64i1  || SimpleTy == MVT::v8i8  ||
-              SimpleTy == MVT::v4i16  || SimpleTy == MVT::v2i32 ||
-              SimpleTy == MVT::v1i64  || SimpleTy == MVT::v4f16 ||
-              SimpleTy == MVT::v4bf16 ||SimpleTy == MVT::v2f32  ||
-              SimpleTy == MVT::v1f64);
-    }
-
-    /// Return true if this is a 128-bit vector type.
-    bool is128BitVector() const {
-      return (SimpleTy == MVT::v128i1 || SimpleTy == MVT::v16i8  ||
-              SimpleTy == MVT::v8i16  || SimpleTy == MVT::v4i32  ||
-              SimpleTy == MVT::v2i64  || SimpleTy == MVT::v1i128 ||
-              SimpleTy == MVT::v8f16  || SimpleTy == MVT::v8bf16 ||
-              SimpleTy == MVT::v4f32  || SimpleTy == MVT::v2f64);
-    }
-
-    /// Return true if this is a 256-bit vector type.
-    bool is256BitVector() const {
-      return (SimpleTy == MVT::v16f16 || SimpleTy == MVT::v16bf16 ||
-              SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64 ||
-              SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 ||
-              SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64 ||
-              SimpleTy == MVT::v256i1 || SimpleTy == MVT::v128i2 ||
-              SimpleTy == MVT::v64i4);
-    }
-
-    /// Return true if this is a 512-bit vector type.
-    bool is512BitVector() const {
-      return (SimpleTy == MVT::v32f16 || SimpleTy == MVT::v32bf16 ||
-              SimpleTy == MVT::v16f32 || SimpleTy == MVT::v8f64 ||
-              SimpleTy == MVT::v512i1 || SimpleTy == MVT::v256i2 ||
-              SimpleTy == MVT::v128i4 || SimpleTy == MVT::v64i8 ||
-              SimpleTy == MVT::v32i16 || SimpleTy == MVT::v16i32 ||
-              SimpleTy == MVT::v8i64);
-    }
-
-    /// Return true if this is a 1024-bit vector type.
-    bool is1024BitVector() const {
-      return (SimpleTy == MVT::v1024i1 || SimpleTy == MVT::v128i8 ||
-              SimpleTy == MVT::v64i16  || SimpleTy == MVT::v32i32 ||
-              SimpleTy == MVT::v16i64  || SimpleTy == MVT::v64f16 ||
-              SimpleTy == MVT::v32f32  || SimpleTy == MVT::v16f64 ||
-              SimpleTy == MVT::v64bf16);
-    }
-
-    /// Return true if this is a 2048-bit vector type.
-    bool is2048BitVector() const {
-      return (SimpleTy == MVT::v256i8  || SimpleTy == MVT::v128i16 ||
-              SimpleTy == MVT::v64i32  || SimpleTy == MVT::v32i64  ||
-              SimpleTy == MVT::v128f16 || SimpleTy == MVT::v64f32  ||
-              SimpleTy == MVT::v32f64  || SimpleTy == MVT::v128bf16 ||
-              SimpleTy == MVT::v2048i1);
-    }
-
-    /// Return true if this is an overloaded type for TableGen.
-    bool isOverloaded() const {
-      return (SimpleTy == MVT::Any || SimpleTy == MVT::iAny ||
-              SimpleTy == MVT::fAny || SimpleTy == MVT::vAny ||
-              SimpleTy == MVT::iPTRAny);
-    }
-
-    /// Return a vector with the same number of elements as this vector, but
-    /// with the element type converted to an integer type with the same
-    /// bitwidth.
-    MVT changeVectorElementTypeToInteger() const {
-      MVT EltTy = getVectorElementType();
-      MVT IntTy = MVT::getIntegerVT(EltTy.getSizeInBits());
-      MVT VecTy = MVT::getVectorVT(IntTy, getVectorElementCount());
-      assert(VecTy.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
-             "Simple vector VT not representable by simple integer vector VT!");
-      return VecTy;
-    }
-
-    /// Return a VT for a vector type whose attributes match ourselves
-    /// with the exception of the element type that is chosen by the caller.
-    MVT changeVectorElementType(MVT EltVT) const {
-      MVT VecTy = MVT::getVectorVT(EltVT, getVectorElementCount());
-      assert(VecTy.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
-             "Simple vector VT not representable by simple integer vector VT!");
-      return VecTy;
-    }
-
-    /// Return the type converted to an equivalently sized integer or vector
-    /// with integer element type. Similar to changeVectorElementTypeToInteger,
-    /// but also handles scalars.
-    MVT changeTypeToInteger() {
-      if (isVector())
-        return changeVectorElementTypeToInteger();
-      return MVT::getIntegerVT(getSizeInBits());
-    }
-
-    /// Return a VT for a vector type with the same element type but
-    /// half the number of elements.
-    MVT getHalfNumVectorElementsVT() const {
-      MVT EltVT = getVectorElementType();
-      auto EltCnt = getVectorElementCount();
-      assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!");
-      return getVectorVT(EltVT, EltCnt.divideCoefficientBy(2));
-    }
-
-    // Return a VT for a vector type with the same element type but
-    // double the number of elements.
-    MVT getDoubleNumVectorElementsVT() const {
-      MVT EltVT = getVectorElementType();
-      auto EltCnt = getVectorElementCount();
-      return MVT::getVectorVT(EltVT, EltCnt * 2);
-    }
-
-    /// Returns true if the given vector is a power of 2.
-    bool isPow2VectorType() const {
-      unsigned NElts = getVectorMinNumElements();
-      return !(NElts & (NElts - 1));
-    }
-
-    /// Widens the length of the given vector MVT up to the nearest power of 2
-    /// and returns that type.
-    MVT getPow2VectorType() const {
-      if (isPow2VectorType())
-        return *this;
-
-      ElementCount NElts = getVectorElementCount();
-      unsigned NewMinCount = 1 << Log2_32_Ceil(NElts.getKnownMinValue());
-      NElts = ElementCount::get(NewMinCount, NElts.isScalable());
-      return MVT::getVectorVT(getVectorElementType(), NElts);
-    }
-
-    /// If this is a vector, return the element type, otherwise return this.
-    MVT getScalarType() const {
-      return isVector() ? getVectorElementType() : *this;
-    }
-
-    MVT getVectorElementType() const {
-      // clang-format off
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("Not a vector MVT!");
-      case v1i1:
-      case v2i1:
-      case v4i1:
-      case v8i1:
-      case v16i1:
-      case v32i1:
-      case v64i1:
-      case v128i1:
-      case v256i1:
-      case v512i1:
-      case v1024i1:
-      case v2048i1:
-      case nxv1i1:
-      case nxv2i1:
-      case nxv4i1:
-      case nxv8i1:
-      case nxv16i1:
-      case nxv32i1:
-      case nxv64i1: return i1;
-      case v128i2:
-      case v256i2: return i2;
-      case v64i4:
-      case v128i4: return i4;
-      case v1i8:
-      case v2i8:
-      case v4i8:
-      case v8i8:
-      case v16i8:
-      case v32i8:
-      case v64i8:
-      case v128i8:
-      case v256i8:
-      case v512i8:
-      case v1024i8:
-      case nxv1i8:
-      case nxv2i8:
-      case nxv4i8:
-      case nxv8i8:
-      case nxv16i8:
-      case nxv32i8:
-      case nxv64i8: return i8;
-      case v1i16:
-      case v2i16:
-      case v3i16:
-      case v4i16:
-      case v8i16:
-      case v16i16:
-      case v32i16:
-      case v64i16:
-      case v128i16:
-      case v256i16:
-      case v512i16:
-      case nxv1i16:
-      case nxv2i16:
-      case nxv4i16:
-      case nxv8i16:
-      case nxv16i16:
-      case nxv32i16: return i16;
-      case v1i32:
-      case v2i32:
-      case v3i32:
-      case v4i32:
-      case v5i32:
-      case v6i32:
-      case v7i32:
-      case v8i32:
-      case v9i32:
-      case v10i32:
-      case v11i32:
-      case v12i32:
-      case v16i32:
-      case v32i32:
-      case v64i32:
-      case v128i32:
-      case v256i32:
-      case v512i32:
-      case v1024i32:
-      case v2048i32:
-      case nxv1i32:
-      case nxv2i32:
-      case nxv4i32:
-      case nxv8i32:
-      case nxv16i32:
-      case nxv32i32: return i32;
-      case v1i64:
-      case v2i64:
-      case v3i64:
-      case v4i64:
-      case v8i64:
-      case v16i64:
-      case v32i64:
-      case v64i64:
-      case v128i64:
-      case v256i64:
-      case nxv1i64:
-      case nxv2i64:
-      case nxv4i64:
-      case nxv8i64:
-      case nxv16i64:
-      case nxv32i64: return i64;
-      case v1i128: return i128;
-      case v1f16:
-      case v2f16:
-      case v3f16:
-      case v4f16:
-      case v8f16:
-      case v16f16:
-      case v32f16:
-      case v64f16:
-      case v128f16:
-      case v256f16:
-      case v512f16:
-      case nxv1f16:
-      case nxv2f16:
-      case nxv4f16:
-      case nxv8f16:
-      case nxv16f16:
-      case nxv32f16: return f16;
-      case v2bf16:
-      case v3bf16:
-      case v4bf16:
-      case v8bf16:
-      case v16bf16:
-      case v32bf16:
-      case v64bf16:
-      case v128bf16:
-      case nxv1bf16:
-      case nxv2bf16:
-      case nxv4bf16:
-      case nxv8bf16:
-      case nxv16bf16:
-      case nxv32bf16: return bf16;
-      case v1f32:
-      case v2f32:
-      case v3f32:
-      case v4f32:
-      case v5f32:
-      case v6f32:
-      case v7f32:
-      case v8f32:
-      case v9f32:
-      case v10f32:
-      case v11f32:
-      case v12f32:
-      case v16f32:
-      case v32f32:
-      case v64f32:
-      case v128f32:
-      case v256f32:
-      case v512f32:
-      case v1024f32:
-      case v2048f32:
-      case nxv1f32:
-      case nxv2f32:
-      case nxv4f32:
-      case nxv8f32:
-      case nxv16f32: return f32;
-      case v1f64:
-      case v2f64:
-      case v3f64:
-      case v4f64:
-      case v8f64:
-      case v16f64:
-      case v32f64:
-      case v64f64:
-      case v128f64:
-      case v256f64:
-      case nxv1f64:
-      case nxv2f64:
-      case nxv4f64:
-      case nxv8f64: return f64;
-      }
-      // clang-format on
-    }
-
-    /// Given a vector type, return the minimum number of elements it contains.
-    unsigned getVectorMinNumElements() const {
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("Not a vector MVT!");
-      case v2048i1:
-      case v2048i32:
-      case v2048f32: return 2048;
-      case v1024i1:
-      case v1024i8:
-      case v1024i32:
-      case v1024f32: return 1024;
-      case v512i1:
-      case v512i8:
-      case v512i16:
-      case v512i32:
-      case v512f16:
-      case v512f32: return 512;
-      case v256i1:
-      case v256i2:
-      case v256i8:
-      case v256i16:
-      case v256f16:
-      case v256i32:
-      case v256i64:
-      case v256f32:
-      case v256f64: return 256;
-      case v128i1:
-      case v128i2:
-      case v128i4:
-      case v128i8:
-      case v128i16:
-      case v128i32:
-      case v128i64:
-      case v128f16:
-      case v128bf16:
-      case v128f32:
-      case v128f64: return 128;
-      case v64i1:
-      case v64i4:
-      case v64i8:
-      case v64i16:
-      case v64i32:
-      case v64i64:
-      case v64f16:
-      case v64bf16:
-      case v64f32:
-      case v64f64:
-      case nxv64i1:
-      case nxv64i8: return 64;
-      case v32i1:
-      case v32i8:
-      case v32i16:
-      case v32i32:
-      case v32i64:
-      case v32f16:
-      case v32bf16:
-      case v32f32:
-      case v32f64:
-      case nxv32i1:
-      case nxv32i8:
-      case nxv32i16:
-      case nxv32i32:
-      case nxv32i64:
-      case nxv32f16:
-      case nxv32bf16: return 32;
-      case v16i1:
-      case v16i8:
-      case v16i16:
-      case v16i32:
-      case v16i64:
-      case v16f16:
-      case v16bf16:
-      case v16f32:
-      case v16f64:
-      case nxv16i1:
-      case nxv16i8:
-      case nxv16i16:
-      case nxv16i32:
-      case nxv16i64:
-      case nxv16f16:
-      case nxv16bf16:
-      case nxv16f32: return 16;
-      case v12i32:
-      case v12f32: return 12;
-      case v11i32:
-      case v11f32: return 11;
-      case v10i32:
-      case v10f32: return 10;
-      case v9i32:
-      case v9f32: return 9;
-      case v8i1:
-      case v8i8:
-      case v8i16:
-      case v8i32:
-      case v8i64:
-      case v8f16:
-      case v8bf16:
-      case v8f32:
-      case v8f64:
-      case nxv8i1:
-      case nxv8i8:
-      case nxv8i16:
-      case nxv8i32:
-      case nxv8i64:
-      case nxv8f16:
-      case nxv8bf16:
-      case nxv8f32:
-      case nxv8f64: return 8;
-      case v7i32:
-      case v7f32: return 7;
-      case v6i32:
-      case v6f32: return 6;
-      case v5i32:
-      case v5f32: return 5;
-      case v4i1:
-      case v4i8:
-      case v4i16:
-      case v4i32:
-      case v4i64:
-      case v4f16:
-      case v4bf16:
-      case v4f32:
-      case v4f64:
-      case nxv4i1:
-      case nxv4i8:
-      case nxv4i16:
-      case nxv4i32:
-      case nxv4i64:
-      case nxv4f16:
-      case nxv4bf16:
-      case nxv4f32:
-      case nxv4f64: return 4;
-      case v3i16:
-      case v3i32:
-      case v3i64:
-      case v3f16:
-      case v3bf16:
-      case v3f32:
-      case v3f64: return 3;
-      case v2i1:
-      case v2i8:
-      case v2i16:
-      case v2i32:
-      case v2i64:
-      case v2f16:
-      case v2bf16:
-      case v2f32:
-      case v2f64:
-      case nxv2i1:
-      case nxv2i8:
-      case nxv2i16:
-      case nxv2i32:
-      case nxv2i64:
-      case nxv2f16:
-      case nxv2bf16:
-      case nxv2f32:
-      case nxv2f64: return 2;
-      case v1i1:
-      case v1i8:
-      case v1i16:
-      case v1i32:
-      case v1i64:
-      case v1i128:
-      case v1f16:
-      case v1f32:
-      case v1f64:
-      case nxv1i1:
-      case nxv1i8:
-      case nxv1i16:
-      case nxv1i32:
-      case nxv1i64:
-      case nxv1f16:
-      case nxv1bf16:
-      case nxv1f32:
-      case nxv1f64: return 1;
-      }
-    }
-
-    ElementCount getVectorElementCount() const {
-      return ElementCount::get(getVectorMinNumElements(), isScalableVector());
-    }
-
-    unsigned getVectorNumElements() const {
-      if (isScalableVector())
-        llvm::reportInvalidSizeRequest(
-            "Possible incorrect use of MVT::getVectorNumElements() for "
-            "scalable vector. Scalable flag may be dropped, use "
-            "MVT::getVectorElementCount() instead");
-      return getVectorMinNumElements();
-    }
-
-    /// Returns the size of the specified MVT in bits.
-    ///
-    /// If the value type is a scalable vector type, the scalable property will
-    /// be set and the runtime size will be a positive integer multiple of the
-    /// base size.
-    TypeSize getSizeInBits() const {
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("getSizeInBits called on extended MVT.");
-      case Other:
-        llvm_unreachable("Value type is non-standard value, Other.");
-      case iPTR:
-        llvm_unreachable("Value type size is target-dependent. Ask TLI.");
-      case iPTRAny:
-      case iAny:
-      case fAny:
-      case vAny:
-      case Any:
-        llvm_unreachable("Value type is overloaded.");
-      case token:
-        llvm_unreachable("Token type is a sentinel that cannot be used "
-                         "in codegen and has no size");
-      case Metadata:
-        llvm_unreachable("Value type is metadata.");
-      case i1:
-      case v1i1: return TypeSize::Fixed(1);
-      case nxv1i1: return TypeSize::Scalable(1);
-      case i2:
-      case v2i1: return TypeSize::Fixed(2);
-      case nxv2i1: return TypeSize::Scalable(2);
-      case i4:
-      case v4i1: return TypeSize::Fixed(4);
-      case nxv4i1: return TypeSize::Scalable(4);
-      case i8  :
-      case v1i8:
-      case v8i1: return TypeSize::Fixed(8);
-      case nxv1i8:
-      case nxv8i1: return TypeSize::Scalable(8);
-      case i16 :
-      case f16:
-      case bf16:
-      case v16i1:
-      case v2i8:
-      case v1i16:
-      case v1f16: return TypeSize::Fixed(16);
-      case aarch64svcount:
-      case nxv16i1:
-      case nxv2i8:
-      case nxv1i16:
-      case nxv1bf16:
-      case nxv1f16: return TypeSize::Scalable(16);
-      case f32 :
-      case i32 :
-      case v32i1:
-      case v4i8:
-      case v2i16:
-      case v2f16:
-      case v2bf16:
-      case v1f32:
-      case v1i32: return TypeSize::Fixed(32);
-      case nxv32i1:
-      case nxv4i8:
-      case nxv2i16:
-      case nxv1i32:
-      case nxv2f16:
-      case nxv2bf16:
-      case nxv1f32: return TypeSize::Scalable(32);
-      case v3i16:
-      case v3f16:
-      case v3bf16: return TypeSize::Fixed(48);
-      case x86mmx:
-      case f64 :
-      case i64 :
-      case v64i1:
-      case v8i8:
-      case v4i16:
-      case v2i32:
-      case v1i64:
-      case v4f16:
-      case v4bf16:
-      case v2f32:
-      case v1f64: return TypeSize::Fixed(64);
-      case nxv64i1:
-      case nxv8i8:
-      case nxv4i16:
-      case nxv2i32:
-      case nxv1i64:
-      case nxv4f16:
-      case nxv4bf16:
-      case nxv2f32:
-      case nxv1f64: return TypeSize::Scalable(64);
-      case f80 :  return TypeSize::Fixed(80);
-      case v3i32:
-      case v3f32: return TypeSize::Fixed(96);
-      case f128:
-      case ppcf128:
-      case i128:
-      case v128i1:
-      case v16i8:
-      case v8i16:
-      case v4i32:
-      case v2i64:
-      case v1i128:
-      case v8f16:
-      case v8bf16:
-      case v4f32:
-      case v2f64: return TypeSize::Fixed(128);
-      case nxv16i8:
-      case nxv8i16:
-      case nxv4i32:
-      case nxv2i64:
-      case nxv8f16:
-      case nxv8bf16:
-      case nxv4f32:
-      case nxv2f64: return TypeSize::Scalable(128);
-      case v5i32:
-      case v5f32: return TypeSize::Fixed(160);
-      case v6i32:
-      case v3i64:
-      case v6f32:
-      case v3f64: return TypeSize::Fixed(192);
-      case v7i32:
-      case v7f32: return TypeSize::Fixed(224);
-      case v256i1:
-      case v128i2:
-      case v64i4:
-      case v32i8:
-      case v16i16:
-      case v8i32:
-      case v4i64:
-      case v16f16:
-      case v16bf16:
-      case v8f32:
-      case v4f64: return TypeSize::Fixed(256);
-      case nxv32i8:
-      case nxv16i16:
-      case nxv8i32:
-      case nxv4i64:
-      case nxv16f16:
-      case nxv16bf16:
-      case nxv8f32:
-      case nxv4f64: return TypeSize::Scalable(256);
-      case v9i32:
-      case v9f32: return TypeSize::Fixed(288);
-      case v10i32:
-      case v10f32: return TypeSize::Fixed(320);
-      case v11i32:
-      case v11f32: return TypeSize::Fixed(352);
-      case v12i32:
-      case v12f32: return TypeSize::Fixed(384);
-      case i64x8:
-      case v512i1:
-      case v256i2:
-      case v128i4:
-      case v64i8:
-      case v32i16:
-      case v16i32:
-      case v8i64:
-      case v32f16:
-      case v32bf16:
-      case v16f32:
-      case v8f64: return TypeSize::Fixed(512);
-      case nxv64i8:
-      case nxv32i16:
-      case nxv16i32:
-      case nxv8i64:
-      case nxv32f16:
-      case nxv32bf16:
-      case nxv16f32:
-      case nxv8f64: return TypeSize::Scalable(512);
-      case v1024i1:
-      case v128i8:
-      case v64i16:
-      case v32i32:
-      case v16i64:
-      case v64f16:
-      case v64bf16:
-      case v32f32:
-      case v16f64: return TypeSize::Fixed(1024);
-      case nxv32i32:
-      case nxv16i64: return TypeSize::Scalable(1024);
-      case v2048i1:
-      case v256i8:
-      case v128i16:
-      case v64i32:
-      case v32i64:
-      case v128f16:
-      case v128bf16:
-      case v64f32:
-      case v32f64: return TypeSize::Fixed(2048);
-      case nxv32i64: return TypeSize::Scalable(2048);
-      case v512i8:
-      case v256i16:
-      case v128i32:
-      case v64i64:
-      case v256f16:
-      case v128f32:
-      case v64f64:  return TypeSize::Fixed(4096);
-      case v1024i8:
-      case v512i16:
-      case v256i32:
-      case v128i64:
-      case v512f16:
-      case v256f32:
-      case x86amx:
-      case v128f64:  return TypeSize::Fixed(8192);
-      case v512i32:
-      case v256i64:
-      case v512f32:
-      case v256f64:  return TypeSize::Fixed(16384);
-      case v1024i32:
-      case v1024f32:  return TypeSize::Fixed(32768);
-      case v2048i32:
-      case v2048f32:  return TypeSize::Fixed(65536);
-      case funcref:
-      case externref:
-      case spirvbuiltin:
-        return TypeSize::Fixed(0); // opaque type
-      }
-    }
-
-    /// Return the size of the specified fixed width value type in bits. The
-    /// function will assert if the type is scalable.
-    uint64_t getFixedSizeInBits() const {
-      return getSizeInBits().getFixedValue();
-    }
-
-    uint64_t getScalarSizeInBits() const {
-      return getScalarType().getSizeInBits().getFixedValue();
-    }
-
-    /// Return the number of bytes overwritten by a store of the specified value
-    /// type.
-    ///
-    /// If the value type is a scalable vector type, the scalable property will
-    /// be set and the runtime size will be a positive integer multiple of the
-    /// base size.
-    TypeSize getStoreSize() const {
-      TypeSize BaseSize = getSizeInBits();
-      return {(BaseSize.getKnownMinValue() + 7) / 8, BaseSize.isScalable()};
-    }
-
-    // Return the number of bytes overwritten by a store of this value type or
-    // this value type's element type in the case of a vector.
-    uint64_t getScalarStoreSize() const {
-      return getScalarType().getStoreSize().getFixedValue();
-    }
-
-    /// Return the number of bits overwritten by a store of the specified value
-    /// type.
-    ///
-    /// If the value type is a scalable vector type, the scalable property will
-    /// be set and the runtime size will be a positive integer multiple of the
-    /// base size.
-    TypeSize getStoreSizeInBits() const {
-      return getStoreSize() * 8;
-    }
-
-    /// Returns true if the number of bits for the type is a multiple of an
-    /// 8-bit byte.
-    bool isByteSized() const { return getSizeInBits().isKnownMultipleOf(8); }
-
-    /// Return true if we know at compile time this has more bits than VT.
-    bool knownBitsGT(MVT VT) const {
-      return TypeSize::isKnownGT(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if we know at compile time this has more than or the same
-    /// bits as VT.
-    bool knownBitsGE(MVT VT) const {
-      return TypeSize::isKnownGE(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if we know at compile time this has fewer bits than VT.
-    bool knownBitsLT(MVT VT) const {
-      return TypeSize::isKnownLT(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if we know at compile time this has fewer than or the same
-    /// bits as VT.
-    bool knownBitsLE(MVT VT) const {
-      return TypeSize::isKnownLE(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if this has more bits than VT.
-    bool bitsGT(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsGT(VT);
-    }
-
-    /// Return true if this has no less bits than VT.
-    bool bitsGE(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsGE(VT);
-    }
-
-    /// Return true if this has less bits than VT.
-    bool bitsLT(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsLT(VT);
-    }
-
-    /// Return true if this has no more bits than VT.
-    bool bitsLE(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsLE(VT);
-    }
-
-    static MVT getFloatingPointVT(unsigned BitWidth) {
-      switch (BitWidth) {
-      default:
-        llvm_unreachable("Bad bit width!");
-      case 16:
-        return MVT::f16;
-      case 32:
-        return MVT::f32;
-      case 64:
-        return MVT::f64;
-      case 80:
-        return MVT::f80;
-      case 128:
-        return MVT::f128;
-      }
-    }
-
-    static MVT getIntegerVT(unsigned BitWidth) {
-      switch (BitWidth) {
-      default:
-        return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-      case 1:
-        return MVT::i1;
-      case 2:
-        return MVT::i2;
-      case 4:
-        return MVT::i4;
-      case 8:
-        return MVT::i8;
-      case 16:
-        return MVT::i16;
-      case 32:
-        return MVT::i32;
-      case 64:
-        return MVT::i64;
-      case 128:
-        return MVT::i128;
-      }
-    }
-
-    static MVT getVectorVT(MVT VT, unsigned NumElements) {
-      // clang-format off
-      switch (VT.SimpleTy) {
-      default:
-        break;
-      case MVT::i1:
-        if (NumElements == 1)    return MVT::v1i1;
-        if (NumElements == 2)    return MVT::v2i1;
-        if (NumElements == 4)    return MVT::v4i1;
-        if (NumElements == 8)    return MVT::v8i1;
-        if (NumElements == 16)   return MVT::v16i1;
-        if (NumElements == 32)   return MVT::v32i1;
-        if (NumElements == 64)   return MVT::v64i1;
-        if (NumElements == 128)  return MVT::v128i1;
-        if (NumElements == 256)  return MVT::v256i1;
-        if (NumElements == 512)  return MVT::v512i1;
-        if (NumElements == 1024) return MVT::v1024i1;
-        if (NumElements == 2048) return MVT::v2048i1;
-        break;
-      case MVT::i2:
-        if (NumElements == 128) return MVT::v128i2;
-        if (NumElements == 256) return MVT::v256i2;
-        break;
-      case MVT::i4:
-        if (NumElements == 64)  return MVT::v64i4;
-        if (NumElements == 128) return MVT::v128i4;
-        break;
-      case MVT::i8:
-        if (NumElements == 1)   return MVT::v1i8;
-        if (NumElements == 2)   return MVT::v2i8;
-        if (NumElements == 4)   return MVT::v4i8;
-        if (NumElements == 8)   return MVT::v8i8;
-        if (NumElements == 16)  return MVT::v16i8;
-        if (NumElements == 32)  return MVT::v32i8;
-        if (NumElements == 64)  return MVT::v64i8;
-        if (NumElements == 128) return MVT::v128i8;
-        if (NumElements == 256) return MVT::v256i8;
-        if (NumElements == 512) return MVT::v512i8;
-        if (NumElements == 1024) return MVT::v1024i8;
-        break;
-      case MVT::i16:
-        if (NumElements == 1)   return MVT::v1i16;
-        if (NumElements == 2)   return MVT::v2i16;
-        if (NumElements == 3)   return MVT::v3i16;
-        if (NumElements == 4)   return MVT::v4i16;
-        if (NumElements == 8)   return MVT::v8i16;
-        if (NumElements == 16)  return MVT::v16i16;
-        if (NumElements == 32)  return MVT::v32i16;
-        if (NumElements == 64)  return MVT::v64i16;
-        if (NumElements == 128) return MVT::v128i16;
-        if (NumElements == 256) return MVT::v256i16;
-        if (NumElements == 512) return MVT::v512i16;
-        break;
-      case MVT::i32:
-        if (NumElements == 1)    return MVT::v1i32;
-        if (NumElements == 2)    return MVT::v2i32;
-        if (NumElements == 3)    return MVT::v3i32;
-        if (NumElements == 4)    return MVT::v4i32;
-        if (NumElements == 5)    return MVT::v5i32;
-        if (NumElements == 6)    return MVT::v6i32;
-        if (NumElements == 7)    return MVT::v7i32;
-        if (NumElements == 8)    return MVT::v8i32;
-        if (NumElements == 9)    return MVT::v9i32;
-        if (NumElements == 10)   return MVT::v10i32;
-        if (NumElements == 11)   return MVT::v11i32;
-        if (NumElements == 12)   return MVT::v12i32;
-        if (NumElements == 16)   return MVT::v16i32;
-        if (NumElements == 32)   return MVT::v32i32;
-        if (NumElements == 64)   return MVT::v64i32;
-        if (NumElements == 128)  return MVT::v128i32;
-        if (NumElements == 256)  return MVT::v256i32;
-        if (NumElements == 512)  return MVT::v512i32;
-        if (NumElements == 1024) return MVT::v1024i32;
-        if (NumElements == 2048) return MVT::v2048i32;
-        break;
-      case MVT::i64:
-        if (NumElements == 1)  return MVT::v1i64;
-        if (NumElements == 2)  return MVT::v2i64;
-        if (NumElements == 3)  return MVT::v3i64;
-        if (NumElements == 4)  return MVT::v4i64;
-        if (NumElements == 8)  return MVT::v8i64;
-        if (NumElements == 16) return MVT::v16i64;
-        if (NumElements == 32) return MVT::v32i64;
-        if (NumElements == 64) return MVT::v64i64;
-        if (NumElements == 128) return MVT::v128i64;
-        if (NumElements == 256) return MVT::v256i64;
-        break;
-      case MVT::i128:
-        if (NumElements == 1)  return MVT::v1i128;
-        break;
-      case MVT::f16:
-        if (NumElements == 1)   return MVT::v1f16;
-        if (NumElements == 2)   return MVT::v2f16;
-        if (NumElements == 3)   return MVT::v3f16;
-        if (NumElements == 4)   return MVT::v4f16;
-        if (NumElements == 8)   return MVT::v8f16;
-        if (NumElements == 16)  return MVT::v16f16;
-        if (NumElements == 32)  return MVT::v32f16;
-        if (NumElements == 64)  return MVT::v64f16;
-        if (NumElements == 128) return MVT::v128f16;
-        if (NumElements == 256) return MVT::v256f16;
-        if (NumElements == 512) return MVT::v512f16;
-        break;
-      case MVT::bf16:
-        if (NumElements == 2)   return MVT::v2bf16;
-        if (NumElements == 3)   return MVT::v3bf16;
-        if (NumElements == 4)   return MVT::v4bf16;
-        if (NumElements == 8)   return MVT::v8bf16;
-        if (NumElements == 16)  return MVT::v16bf16;
-        if (NumElements == 32)  return MVT::v32bf16;
-        if (NumElements == 64)  return MVT::v64bf16;
-        if (NumElements == 128) return MVT::v128bf16;
-        break;
-      case MVT::f32:
-        if (NumElements == 1)    return MVT::v1f32;
-        if (NumElements == 2)    return MVT::v2f32;
-        if (NumElements == 3)    return MVT::v3f32;
-        if (NumElements == 4)    return MVT::v4f32;
-        if (NumElements == 5)    return MVT::v5f32;
-        if (NumElements == 6)    return MVT::v6f32;
-        if (NumElements == 7)    return MVT::v7f32;
-        if (NumElements == 8)    return MVT::v8f32;
-        if (NumElements == 9)    return MVT::v9f32;
-        if (NumElements == 10)   return MVT::v10f32;
-        if (NumElements == 11)   return MVT::v11f32;
-        if (NumElements == 12)   return MVT::v12f32;
-        if (NumElements == 16)   return MVT::v16f32;
-        if (NumElements == 32)   return MVT::v32f32;
-        if (NumElements == 64)   return MVT::v64f32;
-        if (NumElements == 128)  return MVT::v128f32;
-        if (NumElements == 256)  return MVT::v256f32;
-        if (NumElements == 512)  return MVT::v512f32;
-        if (NumElements == 1024) return MVT::v1024f32;
-        if (NumElements == 2048) return MVT::v2048f32;
-        break;
-      case MVT::f64:
-        if (NumElements == 1)  return MVT::v1f64;
-        if (NumElements == 2)  return MVT::v2f64;
-        if (NumElements == 3)  return MVT::v3f64;
-        if (NumElements == 4)  return MVT::v4f64;
-        if (NumElements == 8)  return MVT::v8f64;
-        if (NumElements == 16) return MVT::v16f64;
-        if (NumElements == 32) return MVT::v32f64;
-        if (NumElements == 64) return MVT::v64f64;
-        if (NumElements == 128) return MVT::v128f64;
-        if (NumElements == 256) return MVT::v256f64;
-        break;
-      }
-      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-      // clang-format on
-    }
-
-    static MVT getScalableVectorVT(MVT VT, unsigned NumElements) {
-      switch(VT.SimpleTy) {
-        default:
-          break;
-        case MVT::i1:
-          if (NumElements == 1)  return MVT::nxv1i1;
-          if (NumElements == 2)  return MVT::nxv2i1;
-          if (NumElements == 4)  return MVT::nxv4i1;
-          if (NumElements == 8)  return MVT::nxv8i1;
-          if (NumElements == 16) return MVT::nxv16i1;
-          if (NumElements == 32) return MVT::nxv32i1;
-          if (NumElements == 64) return MVT::nxv64i1;
-          break;
-        case MVT::i8:
-          if (NumElements == 1)  return MVT::nxv1i8;
-          if (NumElements == 2)  return MVT::nxv2i8;
-          if (NumElements == 4)  return MVT::nxv4i8;
-          if (NumElements == 8)  return MVT::nxv8i8;
-          if (NumElements == 16) return MVT::nxv16i8;
-          if (NumElements == 32) return MVT::nxv32i8;
-          if (NumElements == 64) return MVT::nxv64i8;
-          break;
-        case MVT::i16:
-          if (NumElements == 1)  return MVT::nxv1i16;
-          if (NumElements == 2)  return MVT::nxv2i16;
-          if (NumElements == 4)  return MVT::nxv4i16;
-          if (NumElements == 8)  return MVT::nxv8i16;
-          if (NumElements == 16) return MVT::nxv16i16;
-          if (NumElements == 32) return MVT::nxv32i16;
-          break;
-        case MVT::i32:
-          if (NumElements == 1)  return MVT::nxv1i32;
-          if (NumElements == 2)  return MVT::nxv2i32;
-          if (NumElements == 4)  return MVT::nxv4i32;
-          if (NumElements == 8)  return MVT::nxv8i32;
-          if (NumElements == 16) return MVT::nxv16i32;
-          if (NumElements == 32) return MVT::nxv32i32;
-          break;
-        case MVT::i64:
-          if (NumElements == 1)  return MVT::nxv1i64;
-          if (NumElements == 2)  return MVT::nxv2i64;
-          if (NumElements == 4)  return MVT::nxv4i64;
-          if (NumElements == 8)  return MVT::nxv8i64;
-          if (NumElements == 16) return MVT::nxv16i64;
-          if (NumElements == 32) return MVT::nxv32i64;
-          break;
-        case MVT::f16:
-          if (NumElements == 1)  return MVT::nxv1f16;
-          if (NumElements == 2)  return MVT::nxv2f16;
-          if (NumElements == 4)  return MVT::nxv4f16;
-          if (NumElements == 8)  return MVT::nxv8f16;
-          if (NumElements == 16)  return MVT::nxv16f16;
-          if (NumElements == 32)  return MVT::nxv32f16;
-          break;
-        case MVT::bf16:
-          if (NumElements == 1)  return MVT::nxv1bf16;
-          if (NumElements == 2)  return MVT::nxv2bf16;
-          if (NumElements == 4)  return MVT::nxv4bf16;
-          if (NumElements == 8)  return MVT::nxv8bf16;
-          if (NumElements == 16)  return MVT::nxv16bf16;
-          if (NumElements == 32)  return MVT::nxv32bf16;
-          break;
-        case MVT::f32:
-          if (NumElements == 1)  return MVT::nxv1f32;
-          if (NumElements == 2)  return MVT::nxv2f32;
-          if (NumElements == 4)  return MVT::nxv4f32;
-          if (NumElements == 8)  return MVT::nxv8f32;
-          if (NumElements == 16) return MVT::nxv16f32;
-          break;
-        case MVT::f64:
-          if (NumElements == 1)  return MVT::nxv1f64;
-          if (NumElements == 2)  return MVT::nxv2f64;
-          if (NumElements == 4)  return MVT::nxv4f64;
-          if (NumElements == 8)  return MVT::nxv8f64;
-          break;
-      }
-      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-    }
-
-    static MVT getVectorVT(MVT VT, unsigned NumElements, bool IsScalable) {
-      if (IsScalable)
-        return getScalableVectorVT(VT, NumElements);
-      return getVectorVT(VT, NumElements);
-    }
-
-    static MVT getVectorVT(MVT VT, ElementCount EC) {
-      if (EC.isScalable())
-        return getScalableVectorVT(VT, EC.getKnownMinValue());
-      return getVectorVT(VT, EC.getKnownMinValue());
-    }
-
-    /// Return the value type corresponding to the specified type.  This returns
-    /// all pointers as iPTR.  If HandleUnknown is true, unknown types are
-    /// returned as Other, otherwise they are invalid.
-    static MVT getVT(Type *Ty, bool HandleUnknown = false);
-
-  public:
-    /// SimpleValueType Iteration
-    /// @{
-    static auto all_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_VALUETYPE, MVT::LAST_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto integer_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_INTEGER_VALUETYPE,
-                                MVT::LAST_INTEGER_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fp_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FP_VALUETYPE, MVT::LAST_FP_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_VECTOR_VALUETYPE,
-                                MVT::LAST_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fixedlen_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE,
-                                MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto scalable_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_SCALABLE_VECTOR_VALUETYPE,
-                                MVT::LAST_SCALABLE_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto integer_fixedlen_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
-                                MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fp_fixedlen_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE,
-                                MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto integer_scalable_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
-                                MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fp_scalable_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE,
-                                MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-    /// @}
-  };
-
-  inline raw_ostream &operator<<(raw_ostream &OS, const MVT &VT) {
-    VT.print(OS);
-    return OS;
-  }
-
-} // end namespace llvm
-
-#endif // LLVM_SUPPORT_MACHINEVALUETYPE_H
diff --git a/llvm/include/llvm/Support/Parallel.h b/llvm/include/llvm/Support/Parallel.h
index 219197c4eb296..75e7e8d597c44 100644
--- a/llvm/include/llvm/Support/Parallel.h
+++ b/llvm/include/llvm/Support/Parallel.h
@@ -30,6 +30,14 @@ namespace parallel {
 extern ThreadPoolStrategy strategy;
 
 #if LLVM_ENABLE_THREADS
+#define GET_THREAD_INDEX_IMPL                                                  \
+  if (parallel::strategy.ThreadsRequested == 1)                                \
+    return 0;                                                                  \
+  assert((threadIndex != UINT_MAX) &&                                          \
+         "getThreadIndex() must be called from a thread created by "           \
+         "ThreadPoolExecutor");                                                \
+  return threadIndex;
+
 #ifdef _WIN32
 // Direct access to thread_local variables from a different DLL isn't
 // possible with Windows Native TLS.
@@ -38,7 +46,7 @@ unsigned getThreadIndex();
 // Don't access this directly, use the getThreadIndex wrapper.
 extern thread_local unsigned threadIndex;
 
-inline unsigned getThreadIndex() { return threadIndex; }
+inline unsigned getThreadIndex() { GET_THREAD_INDEX_IMPL; }
 #endif
 #else
 inline unsigned getThreadIndex() { return 0; }
@@ -84,26 +92,15 @@ class TaskGroup {
   ~TaskGroup();
 
   // Spawn a task, but does not wait for it to finish.
-  void spawn(std::function<void()> f);
-
-  // Similar to spawn, but execute the task immediately when ThreadsRequested ==
-  // 1. The difference is to give the following pattern a more intuitive order
-  // when single threading is requested.
-  //
-  // for (size_t begin = 0, i = 0, taskSize = 0;;) {
-  //   taskSize += ...
-  //   bool done = ++i == end;
-  //   if (done || taskSize >= taskSizeLimit) {
-  //     tg.execute([=] { fn(begin, i); });
-  //     if (done)
-  //       break;
-  //     begin = i;
-  //     taskSize = 0;
-  //   }
-  // }
-  void execute(std::function<void()> f);
+  // Tasks marked with \p Sequential will be executed
+  // exactly in the order which they were spawned.
+  // Note: Sequential tasks may be executed on different
+  // threads, but strictly in sequential order.
+  void spawn(std::function<void()> f, bool Sequential = false);
 
   void sync() const { L.sync(); }
+
+  bool isParallel() const { return Parallel; }
 };
 
 namespace detail {
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index b4f06a9adcfc5..adb8e8cae64bb 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -314,7 +314,6 @@ class Init {
     IK_AnonymousNameInit,
     IK_StringInit,
     IK_VarInit,
-    IK_VarListElementInit,
     IK_VarBitInit,
     IK_VarDefInit,
     IK_LastTypedInit,
@@ -384,14 +383,6 @@ class Init {
     return nullptr;
   }
 
-  /// This function is used to implement the list slice
-  /// selection operator.  Given a value, it selects the specified list
-  /// elements, returning them as a new \p Init of type \p list. If it
-  /// is not legal to use the slice operator, null is returned.
-  virtual Init *convertInitListSlice(ArrayRef<unsigned> Elements) const {
-    return nullptr;
-  }
-
   /// This function is used to implement the FieldInit class.
   /// Implementors of this method should return the type of the named
   /// field if they are of type record.
@@ -443,7 +434,6 @@ class TypedInit : public Init {
   Init *convertInitializerTo(RecTy *Ty) const override;
 
   Init *convertInitializerBitRange(ArrayRef<unsigned> Bits) const override;
-  Init *convertInitListSlice(ArrayRef<unsigned> Elements) const override;
 
   /// This method is used to implement the FieldInit class.
   /// Implementors of this method should return the type of the named field if
@@ -724,8 +714,6 @@ class ListInit final : public TypedInit, public FoldingSetNode,
 
   Record *getElementAsRecord(unsigned i) const;
 
-  Init *convertInitListSlice(ArrayRef<unsigned> Elements) const override;
-
   Init *convertInitializerTo(RecTy *Ty) const override;
 
   /// This method is used by classes that refer to other
@@ -857,6 +845,10 @@ class BinOpInit : public OpInit, public FoldingSetNode {
     LISTCONCAT,
     LISTSPLAT,
     LISTREMOVE,
+    LISTELEM,
+    LISTSLICE,
+    RANGE,
+    RANGEC,
     STRCONCAT,
     INTERLEAVE,
     CONCAT,
@@ -1238,39 +1230,6 @@ class VarBitInit final : public TypedInit {
   }
 };
 
-/// List[4] - Represent access to one element of a var or
-/// field.
-class VarListElementInit : public TypedInit {
-  TypedInit *TI;
-  unsigned Element;
-
-  VarListElementInit(TypedInit *T, unsigned E)
-      : TypedInit(IK_VarListElementInit,
-                  cast<ListRecTy>(T->getType())->getElementType()),
-        TI(T), Element(E) {
-    assert(T->getType() && isa<ListRecTy>(T->getType()) &&
-           "Illegal VarBitInit expression!");
-  }
-
-public:
-  VarListElementInit(const VarListElementInit &) = delete;
-  VarListElementInit &operator=(const VarListElementInit &) = delete;
-
-  static bool classof(const Init *I) {
-    return I->getKind() == IK_VarListElementInit;
-  }
-
-  static VarListElementInit *get(TypedInit *T, unsigned E);
-
-  TypedInit *getVariable() const { return TI; }
-  unsigned getElementNum() const { return Element; }
-
-  std::string getAsString() const override;
-  Init *resolveReferences(Resolver &R) const override;
-
-  Init *getBit(unsigned Bit) const override;
-};
-
 /// AL - Represent a reference to a 'def' in the description
 class DefInit : public TypedInit {
   friend class Record;
diff --git a/llvm/include/llvm/TargetParser/RISCVTargetParser.h b/llvm/include/llvm/TargetParser/RISCVTargetParser.h
index 993f653455a0d..a4cb7988eb398 100644
--- a/llvm/include/llvm/TargetParser/RISCVTargetParser.h
+++ b/llvm/include/llvm/TargetParser/RISCVTargetParser.h
@@ -26,16 +26,8 @@ namespace RISCV {
 // We use 64 bits as the known part in the scalable vector types.
 static constexpr unsigned RVVBitsPerBlock = 64;
 
-enum CPUKind : unsigned {
-#define PROC(ENUM, NAME, DEFAULT_MARCH) CK_##ENUM,
-#define TUNE_PROC(ENUM, NAME) CK_##ENUM,
-#include "llvm/TargetParser/RISCVTargetParserDef.inc"
-};
-
-bool checkCPUKind(CPUKind Kind, bool IsRV64);
-bool checkTuneCPUKind(CPUKind Kind, bool IsRV64);
-CPUKind parseCPUKind(StringRef CPU);
-CPUKind parseTuneCPUKind(StringRef CPU, bool IsRV64);
+bool parseCPU(StringRef CPU, bool IsRV64);
+bool parseTuneCPU(StringRef CPU, bool IsRV64);
 StringRef getMArchFromMcpu(StringRef CPU);
 void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64);
 void fillValidTuneCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64);
diff --git a/llvm/lib/Transforms/Instrumentation/CFGMST.h b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
similarity index 97%
rename from llvm/lib/Transforms/Instrumentation/CFGMST.h
rename to llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
index 2abe8d12de3c0..4d31898bb3147 100644
--- a/llvm/lib/Transforms/Instrumentation/CFGMST.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIB_TRANSFORMS_INSTRUMENTATION_CFGMST_H
-#define LLVM_LIB_TRANSFORMS_INSTRUMENTATION_CFGMST_H
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_CFGMST_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_CFGMST_H
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
@@ -100,7 +100,7 @@ template <class Edge, class BBInfo> class CFGMST {
   void buildEdges() {
     LLVM_DEBUG(dbgs() << "Build Edge on " << F.getName() << "\n");
 
-    const BasicBlock *Entry = &(F.getEntryBlock());
+    BasicBlock *Entry = &(F.getEntryBlock());
     uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2);
     // If we want to instrument the entry count, lower the weight to 0.
     if (InstrumentFuncEntry)
@@ -257,7 +257,7 @@ template <class Edge, class BBInfo> class CFGMST {
   }
 
   // Add an edge to AllEdges with weight W.
-  Edge &addEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W) {
+  Edge &addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W) {
     uint32_t Index = BBInfos.size();
     auto Iter = BBInfos.end();
     bool Inserted;
@@ -300,4 +300,4 @@ template <class Edge, class BBInfo> class CFGMST {
 
 #undef DEBUG_TYPE // "cfgmst"
 
-#endif // LLVM_LIB_TRANSFORMS_INSTRUMENTATION_CFGMST_H
+#endif // LLVM_TRANSFORMS_INSTRUMENTATION_CFGMST_H
diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h
index 3c07c53becf56..c89d51068be28 100644
--- a/llvm/include/llvm/Transforms/Utils.h
+++ b/llvm/include/llvm/Transforms/Utils.h
@@ -20,11 +20,6 @@ class ModulePass;
 class FunctionPass;
 class Pass;
 
-//===----------------------------------------------------------------------===//
-// createMetaRenamerPass - Rename everything with metasyntatic names.
-//
-ModulePass *createMetaRenamerPass();
-
 //===----------------------------------------------------------------------===//
 //
 // LowerInvoke - This pass removes invoke instructions, converting them to call
@@ -33,13 +28,6 @@ ModulePass *createMetaRenamerPass();
 FunctionPass *createLowerInvokePass();
 extern char &LowerInvokePassID;
 
-//===----------------------------------------------------------------------===//
-//
-// InstructionNamer - Give any unnamed non-void instructions "tmp" names.
-//
-FunctionPass *createInstructionNamerPass();
-extern char &InstructionNamerID;
-
 //===----------------------------------------------------------------------===//
 //
 // LowerSwitch - This pass converts SwitchInst instructions into a sequence of
@@ -100,13 +88,6 @@ FunctionPass *createPromoteMemoryToRegisterPass(bool IsForced = false);
 Pass *createLoopSimplifyPass();
 extern char &LoopSimplifyID;
 
-//===----------------------------------------------------------------------===//
-//
-// InjectTLIMappingsLegacy - populates the VFABI attribute with the
-// scalar-to-vector mappings from the TargetLibraryInfo.
-//
-FunctionPass *createInjectTLIMappingsLegacyPass();
-
 //===----------------------------------------------------------------------===//
 //
 // UnifyLoopExits - For each loop, creates a new block N such that all exiting
@@ -122,13 +103,6 @@ FunctionPass *createUnifyLoopExitsPass();
 //
 FunctionPass *createFixIrreduciblePass();
 
-//===----------------------------------------------------------------------===//
-//
-// AssumeSimplify - remove redundant assumes and merge assumes in the same
-// BasicBlock when possible.
-//
-FunctionPass *createAssumeSimplifyPass();
-
 //===----------------------------------------------------------------------===//
 //
 // CanonicalizeFreezeInLoops - Canonicalize freeze instructions in loops so they
diff --git a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h
index d2ce0c5d3988c..b6c402402b17a 100644
--- a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h
+++ b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h
@@ -14,8 +14,6 @@
 #define LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H
 
 #include "llvm/IR/PassManager.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
 
 namespace llvm {
 class Function;
@@ -24,16 +22,5 @@ class InjectTLIMappings : public PassInfoMixin<InjectTLIMappings> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
 
-// Legacy pass
-class InjectTLIMappingsLegacy : public FunctionPass {
-public:
-  static char ID;
-  InjectTLIMappingsLegacy() : FunctionPass(ID) {
-    initializeInjectTLIMappingsLegacyPass(*PassRegistry::getPassRegistry());
-  }
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  bool runOnFunction(Function &F) override;
-};
-
 } // End namespace llvm
 #endif // LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 8921b24bc4f50..648fcb7260df9 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -78,7 +78,7 @@ bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
 /// they still do not have any uses). Otherwise the PHIs are directly removed.
 bool formLCSSAForInstructions(
     SmallVectorImpl<Instruction *> &Worklist, const DominatorTree &DT,
-    const LoopInfo &LI, ScalarEvolution *SE, IRBuilderBase &Builder,
+    const LoopInfo &LI, IRBuilderBase &Builder,
     SmallVectorImpl<PHINode *> *PHIsToRemove = nullptr);
 
 /// Put loop into LCSSA form.
@@ -88,25 +88,21 @@ bool formLCSSAForInstructions(
 /// the loop are rewritten to use this node. Sub-loops must be in LCSSA form
 /// already.
 ///
-/// LoopInfo and DominatorTree are required and preserved.
-///
-/// If ScalarEvolution is passed in, it will be preserved.
+/// LoopInfo and DominatorTree are required and preserved. ScalarEvolution is
+/// preserved.
 ///
 /// Returns true if any modifications are made to the loop.
-bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
-               ScalarEvolution *SE);
+bool formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI);
 
 /// Put a loop nest into LCSSA form.
 ///
 /// This recursively forms LCSSA for a loop nest.
 ///
-/// LoopInfo and DominatorTree are required and preserved.
-///
-/// If ScalarEvolution is passed in, it will be preserved.
+/// LoopInfo and DominatorTree are required and preserved. ScalarEvolution is
+/// preserved.
 ///
 /// Returns true if any modifications are made to the loop.
-bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
-                          ScalarEvolution *SE);
+bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI);
 
 /// Flags controlling how much is checked when sinking or hoisting
 /// instructions.  The number of memory access in the loop (and whether there
diff --git a/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h b/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h
new file mode 100644
index 0000000000000..980b55f46f114
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h
@@ -0,0 +1,29 @@
+//===- MoveAutoInit.h - Move insts marked as auto-init Pass --*- C++ -*-======//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass moves instructions marked as auto-init closer to their use if
+// profitable, generally because it moves them under a guard, potentially
+// skipping the overhead of the auto-init under some execution paths.
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H
+#define LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class MoveAutoInitPass : public PassInfoMixin<MoveAutoInitPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 0ba4f59ab85ba..8cd9781831a45 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -119,12 +119,12 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
   /// Try to find horizontal reduction or otherwise, collect instructions
   /// for postponed vectorization attempts.
   /// \a P if not null designates phi node the reduction is fed into
-  /// (with reduction operators \a V or one of its operands, in a basic block
+  /// (with reduction operators \a Root or one of its operands, in a basic block
   /// \a BB).
   /// \returns true if a horizontal reduction was matched and reduced.
   /// \returns false if \a V is null or not an instruction,
   /// or a horizontal reduction was not matched or not possible.
-  bool vectorizeHorReduction(PHINode *P, Value *V, BasicBlock *BB,
+  bool vectorizeHorReduction(PHINode *P, Instruction *Root, BasicBlock *BB,
                              slpvectorizer::BoUpSLP &R,
                              TargetTransformInfo *TTI,
                              SmallVectorImpl<WeakTrackingVH> &PostponedInsts);
@@ -132,7 +132,7 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
   /// Make an attempt to vectorize reduction and then try to vectorize
   /// postponed binary operations.
   /// \returns true on any successfull vectorization.
-  bool vectorizeRootInstruction(PHINode *P, Value *V, BasicBlock *BB,
+  bool vectorizeRootInstruction(PHINode *P, Instruction *Root, BasicBlock *BB,
                                 slpvectorizer::BoUpSLP &R,
                                 TargetTransformInfo *TTI);
 
diff --git a/llvm/include/llvm/module.install.modulemap b/llvm/include/llvm/module.install.modulemap
deleted file mode 100644
index 1be59215cf199..0000000000000
--- a/llvm/include/llvm/module.install.modulemap
+++ /dev/null
@@ -1,31 +0,0 @@
-
-module LLVM_Extern_Config_Def {
- textual header "Config/AsmParsers.def"
- textual header "Config/AsmPrinters.def"
- textual header "Config/Disassemblers.def"
- textual header "Config/Targets.def"
- export *
-}
-
-module LLVM_Extern_IR_Attributes_Gen {
-  textual header "IR/Attributes.gen"
-  textual header "IR/Attributes.inc"
-}
-
-module LLVM_Extern_IR_Intrinsics_Gen {
-  textual header "IR/Intrinsics.gen"
-  textual header "IR/Intrinsics.inc"
-}
-
-module LLVM_Extern_IR_Intrinsics_Enum {
-  textual header "IR/IntrinsicEnums.inc"
-}
-
-module LLVM_Extern_Utils_DataTypes {
-  header "Support/DataTypes.h"
-  export *
-}
-
-module LLVM_Extern_TargetParser_Gen {
-  textual header "TargetParser/RISCVTargetParserDef.inc"
-}
diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap
deleted file mode 100644
index ec986cb8c3d29..0000000000000
--- a/llvm/include/llvm/module.modulemap
+++ /dev/null
@@ -1,476 +0,0 @@
-module LLVM_Analysis {
-  requires cplusplus
-  umbrella "Analysis"
-  module * { export * }
-
-  // This is intended for (repeated) textual inclusion.
-  textual header "Analysis/ScalarFuncs.def"
-  textual header "Analysis/TargetLibraryInfo.def"
-  textual header "Analysis/VecFuncs.def"
-}
-
-module LLVM_AsmParser {
-  requires cplusplus
-  umbrella "AsmParser"
-  module * { export * }
-}
-
-// A module covering CodeGen/ and Target/. These are intertwined
-// and codependent, and thus notionally form a single module.
-module LLVM_Backend {
-  requires cplusplus
-
-  module CodeGen {
-    umbrella "CodeGen"
-    module * { export * }
-
-    // Exclude these; they're intended to be included into only a single
-    // translation unit (or none) and aren't part of this module.
-    exclude header "CodeGen/LinkAllAsmWriterComponents.h"
-    exclude header "CodeGen/LinkAllCodegenComponents.h"
-
-    exclude header "CodeGen/CodeGenPassBuilder.h"
-
-    // These are intended for (repeated) textual inclusion.
-    textual header "CodeGen/DIEValue.def"
-    textual header "CodeGen/MachinePassRegistry.def"
-  }
-}
-
-// FIXME: Make this as a submodule of LLVM_Backend again.
-//        Doing so causes a linker error in clang-format.
-module LLVM_Backend_Target {
-  umbrella "Target"
-  module * { export * }
-}
-
-module LLVM_Bitcode {
- requires cplusplus
- umbrella "Bitcode"
- module * { export * }
-}
-
-module LLVM_Bitstream {
- requires cplusplus
- umbrella "Bitstream"
- module * { export * }
-}
-
-module LLVM_BinaryFormat {
-    requires cplusplus
-    umbrella "BinaryFormat" module * { export * }
-    textual header "BinaryFormat/Dwarf.def"
-    textual header "BinaryFormat/DXContainerConstants.def"
-    textual header "BinaryFormat/DynamicTags.def"
-    textual header "BinaryFormat/MachO.def"
-    textual header "BinaryFormat/MinidumpConstants.def"
-    textual header "BinaryFormat/Swift.def"
-    textual header "BinaryFormat/ELFRelocs/AArch64.def"
-    textual header "BinaryFormat/ELFRelocs/AMDGPU.def"
-    textual header "BinaryFormat/ELFRelocs/ARM.def"
-    textual header "BinaryFormat/ELFRelocs/ARC.def"
-    textual header "BinaryFormat/ELFRelocs/AVR.def"
-    textual header "BinaryFormat/ELFRelocs/BPF.def"
-    textual header "BinaryFormat/ELFRelocs/CSKY.def"
-    textual header "BinaryFormat/ELFRelocs/Hexagon.def"
-    textual header "BinaryFormat/ELFRelocs/i386.def"
-    textual header "BinaryFormat/ELFRelocs/Lanai.def"
-    textual header "BinaryFormat/ELFRelocs/LoongArch.def"
-    textual header "BinaryFormat/ELFRelocs/M68k.def"
-    textual header "BinaryFormat/ELFRelocs/Mips.def"
-    textual header "BinaryFormat/ELFRelocs/MSP430.def"
-    textual header "BinaryFormat/ELFRelocs/PowerPC64.def"
-    textual header "BinaryFormat/ELFRelocs/PowerPC.def"
-    textual header "BinaryFormat/ELFRelocs/RISCV.def"
-    textual header "BinaryFormat/ELFRelocs/Sparc.def"
-    textual header "BinaryFormat/ELFRelocs/SystemZ.def"
-    textual header "BinaryFormat/ELFRelocs/VE.def"
-    textual header "BinaryFormat/ELFRelocs/x86_64.def"
-    textual header "BinaryFormat/ELFRelocs/Xtensa.def"
-    textual header "BinaryFormat/WasmRelocs.def"
-    textual header "BinaryFormat/MsgPack.def"
-}
-
-module LLVM_Config {
-  requires cplusplus
-  umbrella "Config"
-  extern module LLVM_Extern_Config_Def "module.extern.modulemap"
-  module * { export * }
-}
-
-module LLVM_DebugInfo {
-  requires cplusplus
-  module DIContext { header "DebugInfo/DIContext.h" export * }
-}
-
-module LLVM_DebugInfo_DWARF {
-  requires cplusplus
-
-  umbrella "DebugInfo/DWARF"
-  module * { export * }
-}
-
-module LLVM_DebugInfo_PDB {
-  requires cplusplus
-
-  umbrella "DebugInfo/PDB"
-  module * { export * }
-
-  // Separate out this subdirectory; it's an optional component that depends on
-  // a separate library which might not be available.
-  //
-  // FIXME: There should be a better way to specify this.
-  exclude header "DebugInfo/PDB/DIA/DIADataStream.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumDebugStreams.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumFrameData.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumInjectedSources.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumLineNumbers.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumSectionContribs.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumSourceFiles.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumSymbols.h"
-  exclude header "DebugInfo/PDB/DIA/DIAEnumTables.h"
-  exclude header "DebugInfo/PDB/DIA/DIAError.h"
-  exclude header "DebugInfo/PDB/DIA/DIAFrameData.h"
-  exclude header "DebugInfo/PDB/DIA/DIAInjectedSource.h"
-  exclude header "DebugInfo/PDB/DIA/DIALineNumber.h"
-  exclude header "DebugInfo/PDB/DIA/DIARawSymbol.h"
-  exclude header "DebugInfo/PDB/DIA/DIASectionContrib.h"
-  exclude header "DebugInfo/PDB/DIA/DIASession.h"
-  exclude header "DebugInfo/PDB/DIA/DIASourceFile.h"
-  exclude header "DebugInfo/PDB/DIA/DIASupport.h"
-  exclude header "DebugInfo/PDB/DIA/DIATable.h"
-  exclude header "DebugInfo/PDB/DIA/DIAUtils.h"
-}
-
-module LLVM_DebugInfo_PDB_DIA {
-  requires cplusplus
-
-  umbrella "DebugInfo/PDB/DIA"
-  module * { export * }
-}
-
-module LLVM_DebugInfo_MSF {
-  requires cplusplus
-
-  umbrella "DebugInfo/MSF"
-  module * { export * }
-}
-
-module LLVM_DebugInfo_CodeView {
-  requires cplusplus
-
-  umbrella "DebugInfo/CodeView"
-  module * { export * }
-
-  // These are intended for (repeated) textual inclusion.
-  textual header "DebugInfo/CodeView/CodeViewRegisters.def"
-  textual header "DebugInfo/CodeView/CodeViewTypes.def"
-  textual header "DebugInfo/CodeView/CodeViewSymbols.def"
-}
-
-module LLVM_DWARFLinker {
-  requires cplusplus
-
-  umbrella "DWARFLinker"
-  module * { export * }
-}
-
-module LLVM_ExecutionEngine {
-  requires cplusplus
-
-  umbrella "ExecutionEngine"
-  module * { export * }
-
-  // Exclude this; it's an optional component of the ExecutionEngine.
-  exclude header "ExecutionEngine/OProfileWrapper.h"
-
-  // Exclude these; they're intended to be included into only a single
-  // translation unit (or none) and aren't part of this module.
-  exclude header "ExecutionEngine/MCJIT.h"
-  exclude header "ExecutionEngine/Interpreter.h"
-
-  // Exclude headers from LLVM_OrcSupport.
-  exclude header "ExecutionEngine/Orc/Shared/OrcError.h"
-}
-
-module LLVM_FileCheck {
-  requires cplusplus
-
-  umbrella "FileCheck"
-  module * { export * }
-}
-
-module LLVM_Frontend_OpenMP {
-  requires cplusplus
-
-  umbrella "Frontend/OpenMP"
-  module * { export * }
-
-  exclude header "Frontend/OpenMP/OMPKinds.def"
-}
-
-// Orc utilities that don't depend only on Support (not ExecutionEngine or
-// IR). This is a workaround for ExecutionEngine's broken layering, and will
-// be removed in the future.
-module LLVM_OrcSupport {
-  requires cplusplus
-
-  header "ExecutionEngine/Orc/Shared/OrcError.h"
-
-  export *
-}
-
-module LLVM_Pass {
-  module Pass {
-    // PassSupport.h and PassAnalysisSupport.h are made available only through
-    // Pass.h.
-    header "Pass.h"
-    textual header "PassSupport.h"
-    textual header "PassAnalysisSupport.h"
-    export *
-  }
-
-  module PassRegistry { header "PassRegistry.h" export * }
-  module InitializePasses { header "InitializePasses.h" export * }
-}
-
-module LLVM_intrinsic_gen {
-  requires cplusplus
-
-  // Delay building the modules containing dependencies to Attributes.h and
-  // Intrinsics.h because they need to be generated by tablegen first.
-
-  // Attributes.h
-  module IR_Argument { header "IR/Argument.h" export * }
-  module IR_Attributes {
-    header "IR/Attributes.h"
-    extern module LLVM_Extern_IR_Attributes_Gen "module.extern.modulemap"
-    export *
-  }
-  module IR_AbstractCallSite { header "IR/AbstractCallSite.h" export * }
-  module IR_ConstantFold { header "IR/ConstantFold.h" export * }
-  module IR_ConstantFolder { header "IR/ConstantFolder.h" export * }
-  module IR_GlobalVariable { header "IR/GlobalVariable.h" export * }
-  module IR_NoFolder { header "IR/NoFolder.h" export * }
-  module IRBuilderFolder { header "IR/IRBuilderFolder.h" export * }
-  module IR_Module { header "IR/Module.h" export * }
-  module IR_ModuleSummaryIndex { header "IR/ModuleSummaryIndex.h" export * }
-  module IR_ModuleSummaryIndexYAML { header "IR/ModuleSummaryIndexYAML.h" export * }
-  module IR_Function { header "IR/Function.h" export * }
-  module IR_InstrTypes { header "IR/InstrTypes.h" export * }
-  module IR_Instructions { header "IR/Instructions.h" export * }
-  module IR_TypeFinder { header "IR/TypeFinder.h" export * }
-  module IR_VectorBuilder { header "IR/VectorBuilder.h" export * }
-
-
-  // Intrinsics.h
-  module IR_CFG { header "IR/CFG.h" export * }
-  module IR_ConstantRange { header "IR/ConstantRange.h" export * }
-  module IR_Dominators { header "IR/Dominators.h" export * }
-  module IR_FixedPointBuilder { header "IR/FixedPointBuilder.h" export * }
-  module Analysis_PostDominators { header "Analysis/PostDominators.h" export * }
-  module Analysis_DomTreeUpdater { header "Analysis/DomTreeUpdater.h" export * }
-  module IR_IRBuilder { header "IR/IRBuilder.h" export * }
-  module IR_IRPrintingPasses { header "IR/IRPrintingPasses.h" export * }
-  module IR_MatrixBuilder { header "IR/MatrixBuilder.h" export * }
-  module IR_PassManager { header "IR/PassManager.h" export * }
-  module IR_PassManagerImpl { header "IR/PassManagerImpl.h" export * }
-  module IR_PredIteratorCache { header "IR/PredIteratorCache.h" export * }
-  module IR_Verifier { header "IR/Verifier.h" export * }
-  module IR_InstIterator { header "IR/InstIterator.h" export * }
-  module IR_InstVisitor { header "IR/InstVisitor.h" export * }
-  module IR_Intrinsics {
-    header "IR/Intrinsics.h"
-    extern module LLVM_Extern_IR_Intricsics_Gen "module.extern.modulemap"
-    extern module LLVM_Extern_IR_Intrinsics_Enum "module.extern.modulemap"
-    export *
-  }
-  module IR_IntrinsicInst { header "IR/IntrinsicInst.h" export * }
-  module IR_PatternMatch { header "IR/PatternMatch.h" export * }
-  module IR_SafepointIRVerifier { header "IR/SafepointIRVerifier.h" export * }
-  module IR_Statepoint { header "IR/Statepoint.h" export * }
-  module IR_DebugInfo { header "IR/DebugInfo.h" export * }
-
-  export *
-}
-
-module LLVM_IR {
-  requires cplusplus
-
-  umbrella "IR"
-  module * { export * }
-
-  // These are intended for (repeated) textual inclusion.
-  textual header "IR/ConstrainedOps.def"
-  textual header "IR/DebugInfoFlags.def"
-  textual header "IR/Instruction.def"
-  textual header "IR/Metadata.def"
-  textual header "IR/FixedMetadataKinds.def"
-  textual header "IR/Value.def"
-  textual header "IR/VPIntrinsics.def"
-  textual header "IR/RuntimeLibcalls.def"
-}
-
-module LLVM_IRReader {
-  requires cplusplus
-  umbrella "IRReader"
-  module * { export * }
-}
-
-module LLVM_LineEditor {
-  requires cplusplus
-  umbrella "LineEditor"
-  module * { export * }
-}
-
-module LLVM_LTO {
-  requires cplusplus
-  umbrella "LTO"
-  module * { export * }
-}
-
-module LLVM_MC {
-  requires cplusplus
-
-  umbrella "MC"
-  module * { export * }
-}
-
-// Used by llvm-tblgen
-module LLVM_MC_TableGen {
-  requires cplusplus
-  module MC_LaneBitmask { header "MC/LaneBitmask.h" export * }
-  module MC_InstrItineraries { header "MC/MCInstrItineraries.h" export * }
-  module MC_Schedule { header "MC/MCSchedule.h" export * }
-  module MC_SubtargetFeature { header "MC/SubtargetFeature.h" export * }
-}
-
-module LLVM_Object {
-  requires cplusplus
-  umbrella "Object"
-  module * { export * }
-}
-
-module LLVM_Option {
-  requires cplusplus
-  umbrella "Option"
-  module * { export * }
-}
-
-module LLVM_ProfileData {
-  requires cplusplus
-
-  umbrella "ProfileData"
-  module * { export * }
-
-  textual header "ProfileData/InstrProfData.inc"
-  textual header "ProfileData/MemProfData.inc"
-  textual header "ProfileData/MIBEntryDef.inc"
-}
-
-// FIXME: Mislayered?
-module LLVM_Support_TargetRegistry {
-  requires cplusplus
-  header "Support/TargetRegistry.h"
-  export *
-}
-
-module LLVM_TableGen {
-  requires cplusplus
-  umbrella "TableGen"
-  module * { export * }
-}
-
-module LLVM_Transforms {
-  requires cplusplus
-  umbrella "Transforms"
-
-  module * { export * }
-
-  // Requires DEBUG_TYPE to be defined by including file.
-  exclude header "Transforms/Utils/InstructionWorklist.h"
-}
-
-extern module LLVM_Extern_Utils_DataTypes "module.extern.modulemap"
-
-// Build the module with the tablegen-generated files needed by the
-// TargetParser module before building the TargetParser module itself.
-module TargetParserGen {
-  module RISCVTargetParserDef {
-    header "TargetParser/RISCVTargetParser.h"
-    extern module LLVM_Extern_TargetParser_Gen "module.extern.modulemap"
-    export *
-  }
-}
-
-// A module covering ADT/ and Support/. These are intertwined and
-// codependent, and notionally form a single module.
-module LLVM_Utils {
-  module ADT {
-    requires cplusplus
-
-    umbrella "ADT"
-    module * { export * }
-  }
-
-  module Demangle {
-    requires cplusplus
-
-    umbrella "Demangle"
-    module * { export * }
-
-    textual header "Demangle/ItaniumNodes.def"
-  }
-
-  module Support {
-    requires cplusplus
-
-    umbrella "Support"
-    module * { export * }
-
-    // Exclude this; deprecated.
-    exclude header "Support/Host.h"
-
-    // Exclude this; it should only be used on Windows.
-    exclude header "Support/Windows/WindowsSupport.h"
-
-    // Exclude these; they are fundamentally non-modular.
-    exclude header "Support/PluginLoader.h"
-    exclude header "Support/Solaris/sys/regset.h"
-    textual header "Support/TargetOpcodes.def"
-
-  }
-
-  module TargetParser {
-    requires cplusplus
-
-    umbrella "TargetParser"
-    module * { export * }
-
-    // These are intended for textual inclusion.
-    textual header "TargetParser/ARMTargetParser.def"
-    textual header "TargetParser/CSKYTargetParser.def"
-    textual header "TargetParser/X86TargetParser.def"
-    textual header "TargetParser/LoongArchTargetParser.def"
-  }
-
-  // This part of the module is usable from both C and C++ code.
-  module ConvertUTF {
-    header "Support/ConvertUTF.h"
-    export *
-  }
-}
-
-// This is used for a $src == $build compilation. Otherwise we use
-// LLVM_Support_DataTypes_Build, defined in a module map that is
-// copied into the build area.
-module LLVM_Support_DataTypes_Src {
-  header "llvm/Support/DataTypes.h"
-  export *
-}
-
-module LLVM_WindowsManifest {
-  requires cplusplus
-  umbrella "WindowsManifest"
-  module * { export * }
-}
diff --git a/llvm/include/llvm/module.extern.modulemap b/llvm/include/module.extern.modulemap
similarity index 85%
rename from llvm/include/llvm/module.extern.modulemap
rename to llvm/include/module.extern.modulemap
index 8e726a3957cc2..c69d2764e915e 100644
--- a/llvm/include/llvm/module.extern.modulemap
+++ b/llvm/include/module.extern.modulemap
@@ -1,3 +1,4 @@
+module LLVM_Extern_CodeGenTypes_Gen {}
 module LLVM_Extern_Config_Def {}
 module LLVM_Extern_IR_Attributes_Gen {}
 module LLVM_Extern_IR_Intrinsics_Gen {}
diff --git a/llvm/include/module.install.modulemap b/llvm/include/module.install.modulemap
new file mode 100644
index 0000000000000..f7302830f561d
--- /dev/null
+++ b/llvm/include/module.install.modulemap
@@ -0,0 +1,35 @@
+
+module LLVM_Extern_CodeGenTypes_Gen {
+  textual header "llvm/CodeGen/GenVT.inc"
+}
+
+module LLVM_Extern_Config_Def {
+  textual header "llvm/Config/AsmParsers.def"
+  textual header "llvm/Config/AsmPrinters.def"
+  textual header "llvm/Config/Disassemblers.def"
+  textual header "llvm/Config/Targets.def"
+  export *
+}
+
+module LLVM_Extern_IR_Attributes_Gen {
+  textual header "llvm/IR/Attributes.gen"
+  textual header "llvm/IR/Attributes.inc"
+}
+
+module LLVM_Extern_IR_Intrinsics_Gen {
+  textual header "llvm/IR/Intrinsics.gen"
+  textual header "llvm/IR/Intrinsics.inc"
+}
+
+module LLVM_Extern_IR_Intrinsics_Enum {
+  textual header "llvm/IR/IntrinsicEnums.inc"
+}
+
+module LLVM_Extern_Utils_DataTypes {
+  header "llvm/Support/DataTypes.h"
+  export *
+}
+
+module LLVM_Extern_TargetParser_Gen {
+  textual header "llvm/TargetParser/RISCVTargetParserDef.inc"
+}
diff --git a/llvm/include/module.modulemap b/llvm/include/module.modulemap
new file mode 100644
index 0000000000000..4c2ba437edb9f
--- /dev/null
+++ b/llvm/include/module.modulemap
@@ -0,0 +1,428 @@
+module LLVM_C {
+  umbrella "llvm-c"
+  module * { export * }
+}
+
+module LLVM_Analysis {
+  requires cplusplus
+  umbrella "llvm/Analysis"
+  module * { export * }
+
+  // This is intended for (repeated) textual inclusion.
+  textual header "llvm/Analysis/ScalarFuncs.def"
+  textual header "llvm/Analysis/TargetLibraryInfo.def"
+  textual header "llvm/Analysis/VecFuncs.def"
+}
+
+module LLVM_AsmParser {
+  requires cplusplus
+  umbrella "llvm/AsmParser"
+  module * { export * }
+}
+
+module LLVM_CodeGenTypes {
+  requires cplusplus
+
+  module LLT {
+    header "llvm/CodeGen/LowLevelType.h" export *
+  }
+  module MVT {
+    header "llvm/CodeGen/MachineValueType.h" export *
+    extern module LLVM_Extern_CodeGenTypes_Gen "module.extern.modulemap"
+  }
+}
+
+// A module covering CodeGen/ and Target/. These are intertwined
+// and codependent, and thus notionally form a single module.
+module LLVM_Backend {
+  requires cplusplus
+
+  module CodeGen {
+    umbrella "llvm/CodeGen"
+    module * { export * }
+
+    // Exclude these; they're intended to be included into only a single
+    // translation unit (or none) and aren't part of this module.
+    exclude header "llvm/CodeGen/LinkAllAsmWriterComponents.h"
+    exclude header "llvm/CodeGen/LinkAllCodegenComponents.h"
+
+    exclude header "llvm/CodeGen/CodeGenPassBuilder.h"
+
+    // These are intended for (repeated) textual inclusion.
+    textual header "llvm/CodeGen/DIEValue.def"
+    textual header "llvm/CodeGen/MachinePassRegistry.def"
+  }
+}
+
+// FIXME: Make this as a submodule of LLVM_Backend again.
+//        Doing so causes a linker error in clang-format.
+module LLVM_Backend_Target {
+  umbrella "llvm/Target"
+  module * { export * }
+}
+
+module LLVM_Bitcode {
+ requires cplusplus
+ umbrella "llvm/Bitcode"
+ module * { export * }
+}
+
+module LLVM_Bitstream {
+ requires cplusplus
+ umbrella "llvm/Bitstream"
+ module * { export * }
+}
+
+module LLVM_BinaryFormat {
+    requires cplusplus
+    umbrella "llvm/BinaryFormat" module * { export * }
+    textual header "llvm/BinaryFormat/Dwarf.def"
+    textual header "llvm/BinaryFormat/DXContainerConstants.def"
+    textual header "llvm/BinaryFormat/DynamicTags.def"
+    textual header "llvm/BinaryFormat/MachO.def"
+    textual header "llvm/BinaryFormat/MinidumpConstants.def"
+    textual header "llvm/BinaryFormat/Swift.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/AArch64.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/AMDGPU.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/ARM.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/ARC.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/AVR.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/BPF.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/CSKY.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/Hexagon.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/i386.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/Lanai.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/LoongArch.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/M68k.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/Mips.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/MSP430.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/PowerPC64.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/PowerPC.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/RISCV.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/Sparc.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/SystemZ.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/VE.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/x86_64.def"
+    textual header "llvm/BinaryFormat/ELFRelocs/Xtensa.def"
+    textual header "llvm/BinaryFormat/WasmRelocs.def"
+    textual header "llvm/BinaryFormat/MsgPack.def"
+}
+
+module LLVM_Config {
+  requires cplusplus
+  umbrella "llvm/Config"
+  extern module LLVM_Extern_Config_Def "module.extern.modulemap"
+  module * { export * }
+}
+
+module LLVM_DebugInfo {
+  requires cplusplus
+  module DIContext { header "llvm/DebugInfo/DIContext.h" export * }
+}
+
+module LLVM_DebugInfo_DWARF {
+  requires cplusplus
+
+  umbrella "llvm/DebugInfo/DWARF"
+  module * { export * }
+}
+
+module LLVM_DebugInfo_PDB {
+  requires cplusplus
+
+  umbrella "llvm/DebugInfo/PDB"
+  module * { export * }
+
+  // Separate out this subdirectory; it's an optional component that depends on
+  // a separate library which might not be available.
+  //
+  // FIXME: There should be a better way to specify this.
+  exclude header "llvm/DebugInfo/PDB/DIA/DIADataStream.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAEnumTables.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAError.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAFrameData.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIALineNumber.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIARawSymbol.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIASectionContrib.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIASession.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIASourceFile.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIASupport.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIATable.h"
+  exclude header "llvm/DebugInfo/PDB/DIA/DIAUtils.h"
+}
+
+module LLVM_DebugInfo_PDB_DIA {
+  requires cplusplus
+
+  umbrella "llvm/DebugInfo/PDB/DIA"
+  module * { export * }
+}
+
+module LLVM_DebugInfo_MSF {
+  requires cplusplus
+
+  umbrella "llvm/DebugInfo/MSF"
+  module * { export * }
+}
+
+module LLVM_DebugInfo_CodeView {
+  requires cplusplus
+
+  umbrella "llvm/DebugInfo/CodeView"
+  module * { export * }
+
+  // These are intended for (repeated) textual inclusion.
+  textual header "llvm/DebugInfo/CodeView/CodeViewRegisters.def"
+  textual header "llvm/DebugInfo/CodeView/CodeViewTypes.def"
+  textual header "llvm/DebugInfo/CodeView/CodeViewSymbols.def"
+}
+
+module LLVM_DWARFLinker {
+  requires cplusplus
+
+  umbrella "llvm/DWARFLinker"
+  module * { export * }
+}
+
+module LLVM_ExecutionEngine {
+  requires cplusplus
+
+  umbrella "llvm/ExecutionEngine"
+  module * { export * }
+
+  // Exclude this; it's an optional component of the ExecutionEngine.
+  exclude header "llvm/ExecutionEngine/OProfileWrapper.h"
+
+  // Exclude these; they're intended to be included into only a single
+  // translation unit (or none) and aren't part of this module.
+  exclude header "llvm/ExecutionEngine/MCJIT.h"
+  exclude header "llvm/ExecutionEngine/Interpreter.h"
+
+  // Exclude headers from LLVM_OrcSupport.
+  exclude header "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
+}
+
+module LLVM_FileCheck {
+  requires cplusplus
+
+  umbrella "llvm/FileCheck"
+  module * { export * }
+}
+
+module LLVM_Frontend_OpenMP {
+  requires cplusplus
+
+  umbrella "llvm/Frontend/OpenMP"
+  module * { export * }
+
+  exclude header "llvm/Frontend/OpenMP/OMPKinds.def"
+}
+
+// Orc utilities that don't depend only on Support (not ExecutionEngine or
+// IR). This is a workaround for ExecutionEngine's broken layering, and will
+// be removed in the future.
+module LLVM_OrcSupport {
+  requires cplusplus
+
+  header "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
+
+  export *
+}
+
+module LLVM_Pass {
+  module Pass {
+    // PassSupport.h and PassAnalysisSupport.h are made available only through
+    // Pass.h.
+    header "llvm/Pass.h"
+    textual header "llvm/PassSupport.h"
+    textual header "llvm/PassAnalysisSupport.h"
+    export *
+  }
+
+  module PassRegistry { header "llvm/PassRegistry.h" export * }
+  module InitializePasses { header "llvm/InitializePasses.h" export * }
+}
+
+module LLVM_IR {
+  requires cplusplus
+
+  umbrella "llvm/IR"
+  module * { export * }
+
+  extern module LLVM_Extern_IR_Attributes_Gen "module.extern.modulemap"
+  extern module LLVM_Extern_IR_Intrinsics_Gen "module.extern.modulemap"
+  extern module LLVM_Extern_IR_Intrinsics_Enum "module.extern.modulemap"
+
+  // These are intended for (repeated) textual inclusion.
+  textual header "llvm/IR/ConstrainedOps.def"
+  textual header "llvm/IR/DebugInfoFlags.def"
+  textual header "llvm/IR/Instruction.def"
+  textual header "llvm/IR/Metadata.def"
+  textual header "llvm/IR/FixedMetadataKinds.def"
+  textual header "llvm/IR/Value.def"
+  textual header "llvm/IR/VPIntrinsics.def"
+  textual header "llvm/IR/RuntimeLibcalls.def"
+}
+
+module LLVM_IRReader {
+  requires cplusplus
+  umbrella "llvm/IRReader"
+  module * { export * }
+}
+
+module LLVM_LineEditor {
+  requires cplusplus
+  umbrella "llvm/LineEditor"
+  module * { export * }
+}
+
+module LLVM_LTO {
+  requires cplusplus
+  umbrella "llvm/LTO"
+  module * { export * }
+}
+
+module LLVM_MC {
+  requires cplusplus
+
+  umbrella "llvm/MC"
+  module * { export * }
+}
+
+module LLVM_Object {
+  requires cplusplus
+  umbrella "llvm/Object"
+  module * { export * }
+}
+
+module LLVM_Option {
+  requires cplusplus
+  umbrella "llvm/Option"
+  module * { export * }
+}
+
+module LLVM_ProfileData {
+  requires cplusplus
+
+  umbrella "llvm/ProfileData"
+  module * { export * }
+
+  textual header "llvm/ProfileData/InstrProfData.inc"
+  textual header "llvm/ProfileData/MemProfData.inc"
+  textual header "llvm/ProfileData/MIBEntryDef.inc"
+}
+
+// FIXME: Mislayered?
+module LLVM_Support_TargetRegistry {
+  requires cplusplus
+  header "llvm/Support/TargetRegistry.h"
+  export *
+}
+
+module LLVM_TableGen {
+  requires cplusplus
+  umbrella "llvm/TableGen"
+  module * { export * }
+}
+
+module LLVM_Transforms {
+  requires cplusplus
+  umbrella "llvm/Transforms"
+
+  module * { export * }
+
+  // Requires DEBUG_TYPE to be defined by including file.
+  exclude header "llvm/Transforms/Utils/InstructionWorklist.h"
+}
+
+extern module LLVM_Extern_Utils_DataTypes "module.extern.modulemap"
+
+// Build the module with the tablegen-generated files needed by the
+// TargetParser module before building the TargetParser module itself.
+module TargetParserGen {
+  module RISCVTargetParserDef {
+    header "llvm/TargetParser/RISCVTargetParser.h"
+    extern module LLVM_Extern_TargetParser_Gen "module.extern.modulemap"
+    export *
+  }
+}
+
+// A module covering ADT/ and Support/. These are intertwined and
+// codependent, and notionally form a single module.
+module LLVM_Utils {
+  module ADT {
+    requires cplusplus
+
+    umbrella "llvm/ADT"
+    module * { export * }
+  }
+
+  module Demangle {
+    requires cplusplus
+
+    umbrella "llvm/Demangle"
+    module * { export * }
+
+    textual header "llvm/Demangle/ItaniumNodes.def"
+  }
+
+  module Support {
+    requires cplusplus
+
+    umbrella "llvm/Support"
+    module * { export * }
+
+    // Exclude this; deprecated.
+    exclude header "llvm/Support/Host.h"
+
+    // Exclude this; it should only be used on Windows.
+    exclude header "llvm/Support/Windows/WindowsSupport.h"
+
+    // Exclude these; they are fundamentally non-modular.
+    exclude header "llvm/Support/PluginLoader.h"
+    exclude header "llvm/Support/Solaris/sys/regset.h"
+    textual header "llvm/Support/TargetOpcodes.def"
+
+  }
+
+  module TargetParser {
+    requires cplusplus
+
+    umbrella "llvm/TargetParser"
+    module * { export * }
+
+    // These are intended for textual inclusion.
+    textual header "llvm/TargetParser/ARMTargetParser.def"
+    textual header "llvm/TargetParser/CSKYTargetParser.def"
+    textual header "llvm/TargetParser/X86TargetParser.def"
+    textual header "llvm/TargetParser/LoongArchTargetParser.def"
+  }
+
+  // This part of the module is usable from both C and C++ code.
+  module ConvertUTF {
+    header "llvm/Support/ConvertUTF.h"
+    export *
+  }
+}
+
+// This is used for a $src == $build compilation. Otherwise we use
+// LLVM_Support_DataTypes_Build, defined in a module map that is
+// copied into the build area.
+module LLVM_Support_DataTypes_Src {
+  header "llvm/Support/DataTypes.h"
+  export *
+}
+
+module LLVM_WindowsManifest {
+  requires cplusplus
+  umbrella "llvm/WindowsManifest"
+  module * { export * }
+}
diff --git a/llvm/include/llvm/module.modulemap.build b/llvm/include/module.modulemap.build
similarity index 64%
rename from llvm/include/llvm/module.modulemap.build
rename to llvm/include/module.modulemap.build
index 162a262a00a78..2a5b23f2a4128 100644
--- a/llvm/include/llvm/module.modulemap.build
+++ b/llvm/include/module.modulemap.build
@@ -1,13 +1,13 @@
 // This is copied into the build area for a $src != $build compilation.
 module LLVM_Support_DataTypes {
-  header "Support/DataTypes.h"
+  header "llvm/Support/DataTypes.h"
   export *
 }
 module LLVM_Config_ABI_Breaking {
-  header "Config/abi-breaking.h"
+  header "llvm/Config/abi-breaking.h"
   export *
 }
 module LLVM_Config_Config {
-  header "Config/llvm-config.h"
+  header "llvm/Config/llvm-config.h"
   export *
 }
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index f085b16c2622c..ce6f334cdc173 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1326,7 +1326,11 @@ Constant *llvm::ConstantFoldCompareInstOperands(
   // Flush any denormal constant float input according to denormal handling
   // mode.
   Ops0 = FlushFPConstant(Ops0, I, /* IsOutput */ false);
+  if (!Ops0)
+    return nullptr;
   Ops1 = FlushFPConstant(Ops1, I, /* IsOutput */ false);
+  if (!Ops1)
+    return nullptr;
 
   return ConstantExpr::getCompare(Predicate, Ops0, Ops1);
 }
@@ -1361,6 +1365,10 @@ Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I,
     return Operand;
 
   const APFloat &APF = CFP->getValueAPF();
+  // TODO: Should this canonicalize nans?
+  if (!APF.isDenormal())
+    return Operand;
+
   Type *Ty = CFP->getType();
   DenormalMode DenormMode =
       I->getFunction()->getDenormalMode(Ty->getFltSemantics());
@@ -1369,7 +1377,8 @@ Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I,
   switch (Mode) {
   default:
     llvm_unreachable("unknown denormal mode");
-    return Operand;
+  case DenormalMode::Dynamic:
+    return nullptr;
   case DenormalMode::IEEE:
     return Operand;
   case DenormalMode::PreserveSign:
@@ -1395,7 +1404,11 @@ Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
   if (Instruction::isBinaryOp(Opcode)) {
     // Flush denormal inputs if needed.
     Constant *Op0 = FlushFPConstant(LHS, I, /* IsOutput */ false);
+    if (!Op0)
+      return nullptr;
     Constant *Op1 = FlushFPConstant(RHS, I, /* IsOutput */ false);
+    if (!Op1)
+      return nullptr;
 
     // Calculate constant result.
     Constant *C = ConstantFoldBinaryOpOperands(Opcode, Op0, Op1, DL);
@@ -1969,13 +1982,26 @@ static Constant *constantFoldCanonicalize(const Type *Ty, const CallBase *CI,
   if (Src.isDenormal() && CI->getParent() && CI->getFunction()) {
     DenormalMode DenormMode =
         CI->getFunction()->getDenormalMode(Src.getSemantics());
+
+    // TODO: Should allow folding for pure IEEE.
     if (DenormMode == DenormalMode::getIEEE())
       return nullptr;
 
+    if (DenormMode == DenormalMode::getDynamic())
+      return nullptr;
+
+    // If we know if either input or output is flushed, we can fold.
+    if ((DenormMode.Input == DenormalMode::Dynamic &&
+         DenormMode.Output == DenormalMode::IEEE) ||
+        (DenormMode.Input == DenormalMode::IEEE &&
+         DenormMode.Output == DenormalMode::Dynamic))
+      return nullptr;
+
     bool IsPositive =
         (!Src.isNegative() || DenormMode.Input == DenormalMode::PositiveZero ||
          (DenormMode.Output == DenormalMode::PositiveZero &&
           DenormMode.Input == DenormalMode::IEEE));
+
     return ConstantFP::get(CI->getContext(),
                            APFloat::getZero(Src.getSemantics(), !IsPositive));
   }
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 30cb3adc8b809..3216fad8a8ad1 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1484,6 +1484,12 @@ bool InductionDescriptor::isInductionPHI(
     return false;
   }
 
+  // This function assumes that InductionPhi is called only on Phi nodes
+  // present inside loop headers. Check for the same, and throw an assert if
+  // the current Phi is not present inside the loop header.
+  assert(Phi->getParent() == AR->getLoop()->getHeader()
+    && "Invalid Phi node, not present in loop header");
+
   Value *StartValue =
       Phi->getIncomingValueForBlock(AR->getLoop()->getLoopPreheader());
 
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 6877e44813f56..09d4b81ad6c40 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1108,31 +1108,31 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
     if (CostIt == SROACosts.end())
       return;
 
-    increment(InlineCostFeatureIndex::SROALosses, CostIt->second);
+    increment(InlineCostFeatureIndex::sroa_losses, CostIt->second);
     SROACostSavingOpportunities -= CostIt->second;
     SROACosts.erase(CostIt);
   }
 
   void onDisableLoadElimination() override {
-    set(InlineCostFeatureIndex::LoadElimination, 1);
+    set(InlineCostFeatureIndex::load_elimination, 1);
   }
 
   void onCallPenalty() override {
-    increment(InlineCostFeatureIndex::CallPenalty, CallPenalty);
+    increment(InlineCostFeatureIndex::call_penalty, CallPenalty);
   }
 
   void onCallArgumentSetup(const CallBase &Call) override {
-    increment(InlineCostFeatureIndex::CallArgumentSetup,
+    increment(InlineCostFeatureIndex::call_argument_setup,
               Call.arg_size() * InstrCost);
   }
 
   void onLoadRelativeIntrinsic() override {
-    increment(InlineCostFeatureIndex::LoadRelativeIntrinsic, 3 * InstrCost);
+    increment(InlineCostFeatureIndex::load_relative_intrinsic, 3 * InstrCost);
   }
 
   void onLoweredCall(Function *F, CallBase &Call,
                      bool IsIndirectCall) override {
-    increment(InlineCostFeatureIndex::LoweredCallArgSetup,
+    increment(InlineCostFeatureIndex::lowered_call_arg_setup,
               Call.arg_size() * InstrCost);
 
     if (IsIndirectCall) {
@@ -1153,9 +1153,9 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
                                 GetAssumptionCache, GetBFI, PSI, ORE, false,
                                 true);
       if (CA.analyze().isSuccess()) {
-        increment(InlineCostFeatureIndex::NestedInlineCostEstimate,
+        increment(InlineCostFeatureIndex::nested_inline_cost_estimate,
                   CA.getCost());
-        increment(InlineCostFeatureIndex::NestedInlines, 1);
+        increment(InlineCostFeatureIndex::nested_inlines, 1);
       }
     } else {
       onCallPenalty();
@@ -1168,12 +1168,12 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
     if (JumpTableSize) {
       int64_t JTCost = static_cast<int64_t>(JumpTableSize) * InstrCost +
                        JTCostMultiplier * InstrCost;
-      increment(InlineCostFeatureIndex::JumpTablePenalty, JTCost);
+      increment(InlineCostFeatureIndex::jump_table_penalty, JTCost);
       return;
     }
 
     if (NumCaseCluster <= 3) {
-      increment(InlineCostFeatureIndex::CaseClusterPenalty,
+      increment(InlineCostFeatureIndex::case_cluster_penalty,
                 NumCaseCluster * CaseClusterCostMultiplier * InstrCost);
       return;
     }
@@ -1183,11 +1183,11 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
 
     int64_t SwitchCost =
         ExpectedNumberOfCompare * SwitchCostMultiplier * InstrCost;
-    increment(InlineCostFeatureIndex::SwitchPenalty, SwitchCost);
+    increment(InlineCostFeatureIndex::switch_penalty, SwitchCost);
   }
 
   void onMissedSimplification() override {
-    increment(InlineCostFeatureIndex::UnsimplifiedCommonInstructions,
+    increment(InlineCostFeatureIndex::unsimplified_common_instructions,
               InstrCost);
   }
 
@@ -1199,7 +1199,7 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
 
   void onBlockAnalyzed(const BasicBlock *BB) override {
     if (BB->getTerminator()->getNumSuccessors() > 1)
-      set(InlineCostFeatureIndex::IsMultipleBlocks, 1);
+      set(InlineCostFeatureIndex::is_multiple_blocks, 1);
     Threshold -= SingleBBBonus;
   }
 
@@ -1212,24 +1212,24 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
         // Ignore loops that will not be executed
         if (DeadBlocks.count(L->getHeader()))
           continue;
-        increment(InlineCostFeatureIndex::NumLoops,
+        increment(InlineCostFeatureIndex::num_loops,
                   InlineConstants::LoopPenalty);
       }
     }
-    set(InlineCostFeatureIndex::DeadBlocks, DeadBlocks.size());
-    set(InlineCostFeatureIndex::SimplifiedInstructions,
+    set(InlineCostFeatureIndex::dead_blocks, DeadBlocks.size());
+    set(InlineCostFeatureIndex::simplified_instructions,
         NumInstructionsSimplified);
-    set(InlineCostFeatureIndex::ConstantArgs, NumConstantArgs);
-    set(InlineCostFeatureIndex::ConstantOffsetPtrArgs,
+    set(InlineCostFeatureIndex::constant_args, NumConstantArgs);
+    set(InlineCostFeatureIndex::constant_offset_ptr_args,
         NumConstantOffsetPtrArgs);
-    set(InlineCostFeatureIndex::SROASavings, SROACostSavingOpportunities);
+    set(InlineCostFeatureIndex::sroa_savings, SROACostSavingOpportunities);
 
     if (NumVectorInstructions <= NumInstructions / 10)
       Threshold -= VectorBonus;
     else if (NumVectorInstructions <= NumInstructions / 2)
       Threshold -= VectorBonus / 2;
 
-    set(InlineCostFeatureIndex::Threshold, Threshold);
+    set(InlineCostFeatureIndex::threshold, Threshold);
 
     return InlineResult::success();
   }
@@ -1237,17 +1237,17 @@ class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
   bool shouldStop() override { return false; }
 
   void onLoadEliminationOpportunity() override {
-    increment(InlineCostFeatureIndex::LoadElimination, 1);
+    increment(InlineCostFeatureIndex::load_elimination, 1);
   }
 
   InlineResult onAnalysisStart() override {
-    increment(InlineCostFeatureIndex::CallSiteCost,
+    increment(InlineCostFeatureIndex::callsite_cost,
               -1 * getCallsiteCost(this->CandidateCall, DL));
 
-    set(InlineCostFeatureIndex::ColdCcPenalty,
+    set(InlineCostFeatureIndex::cold_cc_penalty,
         (F.getCallingConv() == CallingConv::Cold));
 
-    set(InlineCostFeatureIndex::LastCallToStaticBonus,
+    set(InlineCostFeatureIndex::last_call_to_static_bonus,
         isSoleCallToLocalFunction(CandidateCall, F));
 
     // FIXME: we shouldn't repeat this logic in both the Features and Cost
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index ab0c33ca9cc7e..75fa94fb02165 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4210,6 +4210,11 @@ static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
   if (!I || !is_contained(I->operands(), Op))
     return nullptr;
 
+  // The arguments of a phi node might refer to a value from a previous
+  // cycle iteration.
+  if (isa<PHINode>(I))
+    return nullptr;
+
   if (Op->getType()->isVectorTy()) {
     // For vector types, the simplification must hold per-lane, so forbid
     // potentially cross-lane operations like shufflevector.
@@ -4431,6 +4436,24 @@ static Value *simplifySelectWithFakeICmpEq(Value *CmpLHS, Value *CmpRHS,
                                Pred == ICmpInst::ICMP_EQ);
 }
 
+/// Try to simplify a select instruction when its condition operand is an
+/// integer equality comparison.
+static Value *simplifySelectWithICmpEq(Value *CmpLHS, Value *CmpRHS,
+                                       Value *TrueVal, Value *FalseVal,
+                                       const SimplifyQuery &Q,
+                                       unsigned MaxRecurse) {
+  if (simplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, Q,
+                             /* AllowRefinement */ false,
+                             MaxRecurse) == TrueVal)
+    return FalseVal;
+  if (simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, Q,
+                             /* AllowRefinement */ true,
+                             MaxRecurse) == FalseVal)
+    return FalseVal;
+
+  return nullptr;
+}
+
 /// Try to simplify a select instruction when its condition operand is an
 /// integer comparison.
 static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
@@ -4519,35 +4542,38 @@ static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal,
   // the arms of the select. See if substituting this value into the arm and
   // simplifying the result yields the same value as the other arm.
   if (Pred == ICmpInst::ICMP_EQ) {
-    if (simplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, Q,
-                               /* AllowRefinement */ false,
-                               MaxRecurse) == TrueVal ||
-        simplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, Q,
-                               /* AllowRefinement */ false,
-                               MaxRecurse) == TrueVal)
-      return FalseVal;
-    if (simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, Q,
-                               /* AllowRefinement */ true,
-                               MaxRecurse) == FalseVal ||
-        simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, Q,
-                               /* AllowRefinement */ true,
-                               MaxRecurse) == FalseVal)
-      return FalseVal;
-  }
+    if (Value *V = simplifySelectWithICmpEq(CmpLHS, CmpRHS, TrueVal, FalseVal,
+                                            Q, MaxRecurse))
+      return V;
+    if (Value *V = simplifySelectWithICmpEq(CmpRHS, CmpLHS, TrueVal, FalseVal,
+                                            Q, MaxRecurse))
+      return V;
 
-  if (Pred == ICmpInst::Predicate::ICMP_EQ) {
     Value *X;
     Value *Y;
-    // select(X | Y == 0, X or Y, X | Y) -> X | Y
-    if (match(CondVal, m_ICmp(Pred, m_Specific(FalseVal), m_Zero())) &&
-        match(FalseVal, m_Or(m_Value(X), m_Value(Y))) &&
-        (TrueVal == X || TrueVal == Y))
-      return FalseVal;
-    // select(X & Y == -1, X or Y, X & Y) -> X & Y
-    if (match(CondVal, m_ICmp(Pred, m_Specific(FalseVal), m_AllOnes())) &&
-        match(FalseVal, m_And(m_Value(X), m_Value(Y))) &&
-        (TrueVal == X || TrueVal == Y))
-      return FalseVal;
+    // select((X | Y) == 0 ?  X : 0) --> 0 (commuted 2 ways)
+    if (match(CmpLHS, m_Or(m_Value(X), m_Value(Y))) &&
+        match(CmpRHS, m_Zero())) {
+      // (X | Y) == 0 implies X == 0 and Y == 0.
+      if (Value *V = simplifySelectWithICmpEq(X, CmpRHS, TrueVal, FalseVal, Q,
+                                              MaxRecurse))
+        return V;
+      if (Value *V = simplifySelectWithICmpEq(Y, CmpRHS, TrueVal, FalseVal, Q,
+                                              MaxRecurse))
+        return V;
+    }
+
+    // select((X & Y) == -1 ?  X : -1) --> -1 (commuted 2 ways)
+    if (match(CmpLHS, m_And(m_Value(X), m_Value(Y))) &&
+        match(CmpRHS, m_AllOnes())) {
+      // (X & Y) == -1 implies X == -1 and Y == -1.
+      if (Value *V = simplifySelectWithICmpEq(X, CmpRHS, TrueVal, FalseVal, Q,
+                                              MaxRecurse))
+        return V;
+      if (Value *V = simplifySelectWithICmpEq(Y, CmpRHS, TrueVal, FalseVal, Q,
+                                              MaxRecurse))
+        return V;
+    }
   }
 
   return nullptr;
@@ -5209,8 +5235,8 @@ static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1,
                                         ArrayRef<int> Mask, Type *RetTy,
                                         const SimplifyQuery &Q,
                                         unsigned MaxRecurse) {
-  if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
-    return UndefValue::get(RetTy);
+  if (all_of(Mask, [](int Elem) { return Elem == PoisonMaskElem; }))
+    return PoisonValue::get(RetTy);
 
   auto *InVecTy = cast<VectorType>(Op0->getType());
   unsigned MaskNumElts = Mask.size();
@@ -5275,11 +5301,11 @@ static Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1,
         })) {
       assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");
 
-      // Shuffle mask undefs become undefined constant result elements.
+      // Shuffle mask poisons become poison constant result elements.
       SmallVector<Constant *, 16> VecC(MaskNumElts, C);
       for (unsigned i = 0; i != MaskNumElts; ++i)
         if (Indices[i] == -1)
-          VecC[i] = UndefValue::get(C->getType());
+          VecC[i] = PoisonValue::get(C->getType());
       return ConstantVector::get(VecC);
     }
   }
@@ -6093,6 +6119,51 @@ static Value *foldMinMaxSharedOp(Intrinsic::ID IID, Value *Op0, Value *Op1) {
   return nullptr;
 }
 
+/// Given a min/max intrinsic, see if it can be removed based on having an
+/// operand that is another min/max intrinsic with shared operand(s). The caller
+/// is expected to swap the operand arguments to handle commutation.
+static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
+                                         Value *Op1) {
+  assert((IID == Intrinsic::maxnum || IID == Intrinsic::minnum ||
+          IID == Intrinsic::maximum || IID == Intrinsic::minimum) &&
+         "Unsupported intrinsic");
+
+  auto *M0 = dyn_cast<IntrinsicInst>(Op0);
+  // If Op0 is not the same intrinsic as IID, do not process.
+  // This is a difference with integer min/max handling. We do not process the
+  // case like max(min(X,Y),min(X,Y)) => min(X,Y). But it can be handled by GVN.
+  if (!M0 || M0->getIntrinsicID() != IID)
+    return nullptr;
+  Value *X0 = M0->getOperand(0);
+  Value *Y0 = M0->getOperand(1);
+  // Simple case, m(m(X,Y), X) => m(X, Y)
+  //              m(m(X,Y), Y) => m(X, Y)
+  // For minimum/maximum, X is NaN => m(NaN, Y) == NaN and m(NaN, NaN) == NaN.
+  // For minimum/maximum, Y is NaN => m(X, NaN) == NaN  and m(NaN, NaN) == NaN.
+  // For minnum/maxnum, X is NaN => m(NaN, Y) == Y and m(Y, Y) == Y.
+  // For minnum/maxnum, Y is NaN => m(X, NaN) == X and m(X, NaN) == X.
+  if (X0 == Op1 || Y0 == Op1)
+    return M0;
+
+  auto *M1 = dyn_cast<IntrinsicInst>(Op1);
+  if (!M1)
+    return nullptr;
+  Value *X1 = M1->getOperand(0);
+  Value *Y1 = M1->getOperand(1);
+  Intrinsic::ID IID1 = M1->getIntrinsicID();
+  // we have a case m(m(X,Y),m'(X,Y)) taking into account m' is commutative.
+  // if m' is m or inversion of m => m(m(X,Y),m'(X,Y)) == m(X,Y).
+  // For minimum/maximum, X is NaN => m(NaN,Y) == m'(NaN, Y) == NaN.
+  // For minimum/maximum, Y is NaN => m(X,NaN) == m'(X, NaN) == NaN.
+  // For minnum/maxnum, X is NaN => m(NaN,Y) == m'(NaN, Y) == Y.
+  // For minnum/maxnum, Y is NaN => m(X,NaN) == m'(X, NaN) == X.
+  if ((X0 == X1 && Y0 == Y1) || (X0 == Y1 && Y0 == X1))
+    if (IID1 == IID || getInverseMinMaxIntrinsic(IID1) == IID)
+      return M0;
+
+  return nullptr;
+}
+
 static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
                                       const SimplifyQuery &Q) {
   Intrinsic::ID IID = F->getIntrinsicID();
@@ -6332,14 +6403,10 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
 
     // Min/max of the same operation with common operand:
     // m(m(X, Y)), X --> m(X, Y) (4 commuted variants)
-    if (auto *M0 = dyn_cast<IntrinsicInst>(Op0))
-      if (M0->getIntrinsicID() == IID &&
-          (M0->getOperand(0) == Op1 || M0->getOperand(1) == Op1))
-        return Op0;
-    if (auto *M1 = dyn_cast<IntrinsicInst>(Op1))
-      if (M1->getIntrinsicID() == IID &&
-          (M1->getOperand(0) == Op0 || M1->getOperand(1) == Op0))
-        return Op1;
+    if (Value *V = foldMinimumMaximumSharedOp(IID, Op0, Op1))
+      return V;
+    if (Value *V = foldMinimumMaximumSharedOp(IID, Op1, Op0))
+      return V;
 
     break;
   }
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 3491bcefdcbf7..789c2902369b9 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -162,7 +162,7 @@ namespace {
     struct BlockCacheEntry {
       SmallDenseMap<AssertingVH<Value>, ValueLatticeElement, 4> LatticeElements;
       SmallDenseSet<AssertingVH<Value>, 4> OverDefined;
-      // None indicates that the nonnull pointers for this basic block
+      // std::nullopt indicates that the nonnull pointers for this basic block
       // block have not been computed yet.
       std::optional<NonNullPointerSet> NonNullPointers;
     };
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 623cd22e32951..351e09094d487 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -137,6 +137,11 @@ static cl::opt<unsigned> MaxForkedSCEVDepth(
     cl::desc("Maximum recursion depth when finding forked SCEVs (default = 5)"),
     cl::init(5));
 
+static cl::opt<bool> SpeculateUnitStride(
+    "laa-speculate-unit-stride", cl::Hidden,
+    cl::desc("Speculate that non-constant strides are unit in LAA"),
+    cl::init(true));
+
 bool VectorizerParams::isInterleaveForced() {
   return ::VectorizationInterleave.getNumOccurrences() > 0;
 }
@@ -1312,20 +1317,18 @@ void AccessAnalysis::processMemAccesses() {
   }
 }
 
-static bool isInBoundsGep(Value *Ptr) {
-  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr))
-    return GEP->isInBounds();
-  return false;
-}
-
 /// Return true if an AddRec pointer \p Ptr is unsigned non-wrapping,
 /// i.e. monotonically increasing/decreasing.
 static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR,
                            PredicatedScalarEvolution &PSE, const Loop *L) {
+
   // FIXME: This should probably only return true for NUW.
   if (AR->getNoWrapFlags(SCEV::NoWrapMask))
     return true;
 
+  if (PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW))
+    return true;
+
   // Scalar evolution does not propagate the non-wrapping flags to values that
   // are derived from a non-wrapping induction variable because non-wrapping
   // could be flow-sensitive.
@@ -1400,35 +1403,6 @@ std::optional<int64_t> llvm::getPtrStride(PredicatedScalarEvolution &PSE,
     return std::nullopt;
   }
 
-  // The address calculation must not wrap. Otherwise, a dependence could be
-  // inverted.
-  // An inbounds getelementptr that is a AddRec with a unit stride
-  // cannot wrap per definition. The unit stride requirement is checked later.
-  // An getelementptr without an inbounds attribute and unit stride would have
-  // to access the pointer value "0" which is undefined behavior in address
-  // space 0, therefore we can also vectorize this case.
-  unsigned AddrSpace = Ty->getPointerAddressSpace();
-  bool IsInBoundsGEP = isInBoundsGep(Ptr);
-  bool IsNoWrapAddRec = !ShouldCheckWrap ||
-    PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW) ||
-    isNoWrapAddRec(Ptr, AR, PSE, Lp);
-  if (!IsNoWrapAddRec && !IsInBoundsGEP &&
-      NullPointerIsDefined(Lp->getHeader()->getParent(), AddrSpace)) {
-    if (Assume) {
-      PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
-      IsNoWrapAddRec = true;
-      LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap in the address space:\n"
-                        << "LAA:   Pointer: " << *Ptr << "\n"
-                        << "LAA:   SCEV: " << *AR << "\n"
-                        << "LAA:   Added an overflow assumption\n");
-    } else {
-      LLVM_DEBUG(
-          dbgs() << "LAA: Bad stride - Pointer may wrap in the address space "
-                 << *Ptr << " SCEV: " << *AR << "\n");
-      return std::nullopt;
-    }
-  }
-
   // Check the step is constant.
   const SCEV *Step = AR->getStepRecurrence(*PSE.getSE());
 
@@ -1457,25 +1431,42 @@ std::optional<int64_t> llvm::getPtrStride(PredicatedScalarEvolution &PSE,
   if (Rem)
     return std::nullopt;
 
-  // If the SCEV could wrap but we have an inbounds gep with a unit stride we
-  // know we can't "wrap around the address space". In case of address space
-  // zero we know that this won't happen without triggering undefined behavior.
-  if (!IsNoWrapAddRec && Stride != 1 && Stride != -1 &&
-      (IsInBoundsGEP || !NullPointerIsDefined(Lp->getHeader()->getParent(),
-                                              AddrSpace))) {
-    if (Assume) {
-      // We can avoid this case by adding a run-time check.
-      LLVM_DEBUG(dbgs() << "LAA: Non unit strided pointer which is not either "
-                        << "inbounds or in address space 0 may wrap:\n"
-                        << "LAA:   Pointer: " << *Ptr << "\n"
-                        << "LAA:   SCEV: " << *AR << "\n"
-                        << "LAA:   Added an overflow assumption\n");
-      PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
-    } else
-      return std::nullopt;
-  }
+  if (!ShouldCheckWrap)
+    return Stride;
 
-  return Stride;
+  // The address calculation must not wrap. Otherwise, a dependence could be
+  // inverted.
+  if (isNoWrapAddRec(Ptr, AR, PSE, Lp))
+    return Stride;
+
+  // An inbounds getelementptr that is a AddRec with a unit stride
+  // cannot wrap per definition.  If it did, the result would be poison
+  // and any memory access dependent on it would be immediate UB
+  // when executed.
+  if (auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
+      GEP && GEP->isInBounds() && (Stride == 1 || Stride == -1))
+    return Stride;
+
+  // If the null pointer is undefined, then a access sequence which would
+  // otherwise access it can be assumed not to unsigned wrap.  Note that this
+  // assumes the object in memory is aligned to the natural alignment.
+  unsigned AddrSpace = Ty->getPointerAddressSpace();
+  if (!NullPointerIsDefined(Lp->getHeader()->getParent(), AddrSpace) &&
+      (Stride == 1 || Stride == -1))
+    return Stride;
+
+  if (Assume) {
+    PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
+    LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n"
+                      << "LAA:   Pointer: " << *Ptr << "\n"
+                      << "LAA:   SCEV: " << *AR << "\n"
+                      << "LAA:   Added an overflow assumption\n");
+    return Stride;
+  }
+  LLVM_DEBUG(
+      dbgs() << "LAA: Bad stride - Pointer may wrap in the address space "
+             << *Ptr << " SCEV: " << *AR << "\n");
+  return std::nullopt;
 }
 
 std::optional<int> llvm::getPointersDiff(Type *ElemTyA, Value *PtrA,
@@ -2642,6 +2633,11 @@ static Value *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp) {
   if (!S)
     return nullptr;
 
+  // If the pointer is invariant then there is no stride and it makes no
+  // sense to add it here.
+  if (Lp != S->getLoop())
+    return nullptr;
+
   V = S->getStepRecurrence(*SE);
   if (!V)
     return nullptr;
@@ -2703,6 +2699,11 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
                        "versioning:");
   LLVM_DEBUG(dbgs() << "  Ptr: " << *Ptr << " Stride: " << *Stride << "\n");
 
+  if (!SpeculateUnitStride) {
+    LLVM_DEBUG(dbgs() << "  Chose not to due to -laa-speculate-unit-stride\n");
+    return;
+  }
+
   // Avoid adding the "Stride == 1" predicate when we know that
   // Stride >= Trip-Count. Such a predicate will effectively optimize a single
   // or zero iteration loop, as Trip-Count <= Stride == 1.
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
index 49d5deea8fc91..0660a9993b6dd 100644
--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -93,13 +93,11 @@ static cl::opt<bool> KeepFPICache(
 
 // clang-format off
 const std::vector<TensorSpec> llvm::FeatureMap{
-#define POPULATE_NAMES(_, NAME) TensorSpec::createSpec<int64_t>(NAME, {1} ),
+#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
 // InlineCost features - these must come first
   INLINE_COST_FEATURE_ITERATOR(POPULATE_NAMES)
-#undef POPULATE_NAMES
 
 // Non-cost features
-#define POPULATE_NAMES(_, NAME, __) TensorSpec::createSpec<int64_t>(NAME, {1} ),
   INLINE_FEATURE_ITERATOR(POPULATE_NAMES)
 #undef POPULATE_NAMES
 };
@@ -383,26 +381,27 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
   auto &CallerBefore = getCachedFPI(Caller);
   auto &CalleeBefore = getCachedFPI(Callee);
 
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeBasicBlockCount) =
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::callee_basic_block_count) =
       CalleeBefore.BasicBlockCount;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::CallSiteHeight) =
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::callsite_height) =
       getInitialFunctionLevel(Caller);
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::NodeCount) = NodeCount;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::NrCtantParams) = NrCtantParams;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::EdgeCount) = EdgeCount;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::CallerUsers) =
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::node_count) = NodeCount;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::nr_ctant_params) =
+      NrCtantParams;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::edge_count) = EdgeCount;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::caller_users) =
       CallerBefore.Uses;
   *ModelRunner->getTensor<int64_t>(
-      FeatureIndex::CallerConditionallyExecutedBlocks) =
+      FeatureIndex::caller_conditionally_executed_blocks) =
       CallerBefore.BlocksReachedFromConditionalInstruction;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::CallerBasicBlockCount) =
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::caller_basic_block_count) =
       CallerBefore.BasicBlockCount;
   *ModelRunner->getTensor<int64_t>(
-      FeatureIndex::CalleeConditionallyExecutedBlocks) =
+      FeatureIndex::callee_conditionally_executed_blocks) =
       CalleeBefore.BlocksReachedFromConditionalInstruction;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeUsers) =
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::callee_users) =
       CalleeBefore.Uses;
-  *ModelRunner->getTensor<int64_t>(FeatureIndex::CostEstimate) = CostEstimate;
+  *ModelRunner->getTensor<int64_t>(FeatureIndex::cost_estimate) = CostEstimate;
 
   // Add the cost features
   for (size_t I = 0;
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 116a9222a241a..9fabca2e04206 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -115,25 +115,25 @@ static const std::pair<LibFunc, AllocFnsTy> AllocationFnData[] = {
     {LibFunc_ZnwjSt11align_val_t,               {OpNewLike,        2,  0, -1,  1, MallocFamily::CPPNewAligned}},      // new(unsigned int, align_val_t)
     {LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t, {MallocLike,       3,  0, -1,  1, MallocFamily::CPPNewAligned}},      // new(unsigned int, align_val_t, nothrow)
     {LibFunc_Znwm,                              {OpNewLike,        1,  0, -1, -1, MallocFamily::CPPNew}},             // new(unsigned long)
-    {LibFunc_Znwm10hot_cold_t,                  {OpNewLike,        2, 0,  -1, -1, MallocFamily::CPPNew}},             // new(unsigned long, hot_cold_t)
+    {LibFunc_Znwm12__hot_cold_t,                  {OpNewLike,        2, 0,  -1, -1, MallocFamily::CPPNew}},             // new(unsigned long, __hot_cold_t)
     {LibFunc_ZnwmRKSt9nothrow_t,                {MallocLike,       2,  0, -1, -1, MallocFamily::CPPNew}},             // new(unsigned long, nothrow)
-    {LibFunc_ZnwmRKSt9nothrow_t10hot_cold_t,      {MallocLike,       3, 0,  -1, -1, MallocFamily::CPPNew}},             // new(unsigned long, nothrow, hot_cold_t)
+    {LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,      {MallocLike,       3, 0,  -1, -1, MallocFamily::CPPNew}},             // new(unsigned long, nothrow, __hot_cold_t)
     {LibFunc_ZnwmSt11align_val_t,               {OpNewLike,        2,  0, -1,  1, MallocFamily::CPPNewAligned}},      // new(unsigned long, align_val_t)
-    {LibFunc_ZnwmSt11align_val_t10hot_cold_t,   {OpNewLike,        3, 0,  -1, 1, MallocFamily::CPPNewAligned}},       // new(unsigned long, align_val_t, hot_cold_t)
+    {LibFunc_ZnwmSt11align_val_t12__hot_cold_t,   {OpNewLike,        3, 0,  -1, 1, MallocFamily::CPPNewAligned}},       // new(unsigned long, align_val_t, __hot_cold_t)
     {LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t, {MallocLike,       3,  0, -1,  1, MallocFamily::CPPNewAligned}},      // new(unsigned long, align_val_t, nothrow)
-    {LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t, {MallocLike,  4, 0,  -1, 1, MallocFamily::CPPNewAligned}},            // new(unsigned long, align_val_t, nothrow, hot_cold_t)
+    {LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, {MallocLike,  4, 0,  -1, 1, MallocFamily::CPPNewAligned}},            // new(unsigned long, align_val_t, nothrow, __hot_cold_t)
     {LibFunc_Znaj,                              {OpNewLike,        1,  0, -1, -1, MallocFamily::CPPNewArray}},        // new[](unsigned int)
     {LibFunc_ZnajRKSt9nothrow_t,                {MallocLike,       2,  0, -1, -1, MallocFamily::CPPNewArray}},        // new[](unsigned int, nothrow)
     {LibFunc_ZnajSt11align_val_t,               {OpNewLike,        2,  0, -1,  1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned int, align_val_t)
     {LibFunc_ZnajSt11align_val_tRKSt9nothrow_t, {MallocLike,       3,  0, -1,  1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned int, align_val_t, nothrow)
     {LibFunc_Znam,                              {OpNewLike,        1,  0, -1, -1, MallocFamily::CPPNewArray}},        // new[](unsigned long)
-    {LibFunc_Znam10hot_cold_t,                  {OpNewLike,        2, 0,  -1, -1, MallocFamily::CPPNew}},             // new[](unsigned long, hot_cold_t)
+    {LibFunc_Znam12__hot_cold_t,                  {OpNewLike,        2, 0,  -1, -1, MallocFamily::CPPNew}},             // new[](unsigned long, __hot_cold_t)
     {LibFunc_ZnamRKSt9nothrow_t,                {MallocLike,       2,  0, -1, -1, MallocFamily::CPPNewArray}},        // new[](unsigned long, nothrow)
-    {LibFunc_ZnamRKSt9nothrow_t10hot_cold_t,      {MallocLike,       3, 0,  -1, -1, MallocFamily::CPPNew}},             // new[](unsigned long, nothrow, hot_cold_t)
+    {LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,      {MallocLike,       3, 0,  -1, -1, MallocFamily::CPPNew}},             // new[](unsigned long, nothrow, __hot_cold_t)
     {LibFunc_ZnamSt11align_val_t,               {OpNewLike,        2,  0, -1,  1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned long, align_val_t)
-    {LibFunc_ZnamSt11align_val_t10hot_cold_t,   {OpNewLike,        3, 0,  -1, 1, MallocFamily::CPPNewAligned}},       // new[](unsigned long, align_val_t, hot_cold_t)
+    {LibFunc_ZnamSt11align_val_t12__hot_cold_t,   {OpNewLike,        3, 0,  -1, 1, MallocFamily::CPPNewAligned}},       // new[](unsigned long, align_val_t, __hot_cold_t)
     {LibFunc_ZnamSt11align_val_tRKSt9nothrow_t, {MallocLike,       3,  0, -1,  1, MallocFamily::CPPNewArrayAligned}}, // new[](unsigned long, align_val_t, nothrow)
-    {LibFunc_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t, {MallocLike,  4, 0,  -1, 1, MallocFamily::CPPNewAligned}},            // new[](unsigned long, align_val_t, nothrow, hot_cold_t)
+    {LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, {MallocLike,  4, 0,  -1, 1, MallocFamily::CPPNewAligned}},            // new[](unsigned long, align_val_t, nothrow, __hot_cold_t)
     {LibFunc_msvc_new_int,                      {OpNewLike,        1,  0, -1, -1, MallocFamily::MSVCNew}},            // new(unsigned int)
     {LibFunc_msvc_new_int_nothrow,              {MallocLike,       2,  0, -1, -1, MallocFamily::MSVCNew}},            // new(unsigned int, nothrow)
     {LibFunc_msvc_new_longlong,                 {OpNewLike,        1,  0, -1, -1, MallocFamily::MSVCNew}},            // new(unsigned long long)
diff --git a/llvm/lib/Analysis/MemoryProfileInfo.cpp b/llvm/lib/Analysis/MemoryProfileInfo.cpp
index e7284da2163b8..366b21be1fbce 100644
--- a/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -18,25 +18,31 @@ using namespace llvm::memprof;
 
 #define DEBUG_TYPE "memory-profile-info"
 
-// Upper bound on accesses per byte for marking an allocation cold.
-cl::opt<float> MemProfAccessesPerByteColdThreshold(
-    "memprof-accesses-per-byte-cold-threshold", cl::init(10.0), cl::Hidden,
-    cl::desc("The threshold the accesses per byte must be under to consider "
-             "an allocation cold"));
+// Upper bound on lifetime access density (accesses per byte per lifetime sec)
+// for marking an allocation cold.
+cl::opt<float> MemProfLifetimeAccessDensityColdThreshold(
+    "memprof-lifetime-access-density-cold-threshold", cl::init(0.05),
+    cl::Hidden,
+    cl::desc("The threshold the lifetime access density (accesses per byte per "
+             "lifetime sec) must be under to consider an allocation cold"));
 
 // Lower bound on lifetime to mark an allocation cold (in addition to accesses
-// per byte above). This is to avoid pessimizing short lived objects.
-cl::opt<unsigned> MemProfMinLifetimeColdThreshold(
-    "memprof-min-lifetime-cold-threshold", cl::init(200), cl::Hidden,
-    cl::desc("The minimum lifetime (s) for an allocation to be considered "
+// per byte per sec above). This is to avoid pessimizing short lived objects.
+cl::opt<unsigned> MemProfAveLifetimeColdThreshold(
+    "memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden,
+    cl::desc("The average lifetime (s) for an allocation to be considered "
              "cold"));
 
-AllocationType llvm::memprof::getAllocType(uint64_t MaxAccessCount,
-                                           uint64_t MinSize,
-                                           uint64_t MinLifetime) {
-  if (((float)MaxAccessCount) / MinSize < MemProfAccessesPerByteColdThreshold &&
-      // MinLifetime is expected to be in ms, so convert the threshold to ms.
-      MinLifetime >= MemProfMinLifetimeColdThreshold * 1000)
+AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity,
+                                           uint64_t AllocCount,
+                                           uint64_t TotalLifetime) {
+  // The access densities are multiplied by 100 to hold 2 decimal places of
+  // precision, so need to divide by 100.
+  if (((float)TotalLifetimeAccessDensity) / AllocCount / 100 <
+          MemProfLifetimeAccessDensityColdThreshold
+      // Lifetime is expected to be in ms, so convert the threshold to ms.
+      && ((float)TotalLifetime) / AllocCount >=
+             MemProfAveLifetimeColdThreshold * 1000)
     return AllocationType::Cold;
   return AllocationType::NotCold;
 }
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index f12013f4ff620..ba3a6092b62cc 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -76,6 +76,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -2340,25 +2341,42 @@ bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
   // Can we use context to prove the fact we need?
   if (!CtxI)
     return false;
-  // We can prove that add(x, constant) doesn't wrap if isKnownPredicateAt can
-  // guarantee that x <= max_int - constant at the given context.
-  // TODO: Support other operations.
-  if (BinOp != Instruction::Add)
+  // TODO: Support mul.
+  if (BinOp == Instruction::Mul)
     return false;
   auto *RHSC = dyn_cast<SCEVConstant>(RHS);
   // TODO: Lift this limitation.
   if (!RHSC)
     return false;
   APInt C = RHSC->getAPInt();
-  // TODO: Also lift this limitation.
-  if (Signed && C.isNegative())
-    return false;
   unsigned NumBits = C.getBitWidth();
-  APInt Max =
-      Signed ? APInt::getSignedMaxValue(NumBits) : APInt::getMaxValue(NumBits);
-  APInt Limit = Max - C;
+  bool IsSub = (BinOp == Instruction::Sub);
+  bool IsNegativeConst = (Signed && C.isNegative());
+  // Compute the direction and magnitude by which we need to check overflow.
+  bool OverflowDown = IsSub ^ IsNegativeConst;
+  APInt Magnitude = C;
+  if (IsNegativeConst) {
+    if (C == APInt::getSignedMinValue(NumBits))
+      // TODO: SINT_MIN on inversion gives the same negative value, we don't
+      // want to deal with that.
+      return false;
+    Magnitude = -C;
+  }
+
   ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
-  return isKnownPredicateAt(Pred, LHS, getConstant(Limit), CtxI);
+  if (OverflowDown) {
+    // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
+    APInt Min = Signed ? APInt::getSignedMinValue(NumBits)
+                       : APInt::getMinValue(NumBits);
+    APInt Limit = Min + Magnitude;
+    return isKnownPredicateAt(Pred, getConstant(Limit), LHS, CtxI);
+  } else {
+    // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
+    APInt Max = Signed ? APInt::getSignedMaxValue(NumBits)
+                       : APInt::getMaxValue(NumBits);
+    APInt Limit = Max - Magnitude;
+    return isKnownPredicateAt(Pred, LHS, getConstant(Limit), CtxI);
+  }
 }
 
 std::optional<SCEV::NoWrapFlags>
@@ -4473,6 +4491,18 @@ void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
   }
 }
 
+/// Determine whether this instruction is either not SCEVable or will always
+/// produce a SCEVUnknown. We do not have to walk past such instructions when
+/// invalidating.
+static bool isAlwaysUnknown(const Instruction *I) {
+  switch (I->getOpcode()) {
+  case Instruction::Load:
+    return true;
+  default:
+    return false;
+  }
+}
+
 /// Return an existing SCEV if it exists, otherwise analyze the expression and
 /// create a new one.
 const SCEV *ScalarEvolution::getSCEV(Value *V) {
@@ -4480,7 +4510,11 @@ const SCEV *ScalarEvolution::getSCEV(Value *V) {
 
   if (const SCEV *S = getExistingSCEV(V))
     return S;
-  return createSCEVIter(V);
+  const SCEV *S = createSCEVIter(V);
+  assert((!isa<Instruction>(V) || !isAlwaysUnknown(cast<Instruction>(V)) ||
+          isa<SCEVUnknown>(S)) &&
+         "isAlwaysUnknown() instruction is not SCEVUnknown");
+  return S;
 }
 
 const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
@@ -4781,6 +4815,8 @@ static void PushDefUseChildren(Instruction *I,
   // Push the def-use children onto the Worklist stack.
   for (User *U : I->users()) {
     auto *UserInsn = cast<Instruction>(U);
+    if (isAlwaysUnknown(UserInsn))
+      continue;
     if (Visited.insert(UserInsn).second)
       Worklist.push_back(UserInsn);
   }
@@ -5802,9 +5838,7 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
           // indices form a positive value.
           if (GEP->isInBounds() && GEP->getOperand(0) == PN) {
             Flags = setFlags(Flags, SCEV::FlagNW);
-
-            const SCEV *Ptr = getSCEV(GEP->getPointerOperand());
-            if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr)))
+            if (isKnownPositive(Accum))
               Flags = setFlags(Flags, SCEV::FlagNUW);
           }
 
@@ -5873,90 +5907,6 @@ const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
   return nullptr;
 }
 
-// Checks if the SCEV S is available at BB.  S is considered available at BB
-// if S can be materialized at BB without introducing a fault.
-static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S,
-                               BasicBlock *BB) {
-  struct CheckAvailable {
-    bool TraversalDone = false;
-    bool Available = true;
-
-    const Loop *L = nullptr;  // The loop BB is in (can be nullptr)
-    BasicBlock *BB = nullptr;
-    DominatorTree &DT;
-
-    CheckAvailable(const Loop *L, BasicBlock *BB, DominatorTree &DT)
-      : L(L), BB(BB), DT(DT) {}
-
-    bool setUnavailable() {
-      TraversalDone = true;
-      Available = false;
-      return false;
-    }
-
-    bool follow(const SCEV *S) {
-      switch (S->getSCEVType()) {
-      case scConstant:
-      case scVScale:
-      case scPtrToInt:
-      case scTruncate:
-      case scZeroExtend:
-      case scSignExtend:
-      case scAddExpr:
-      case scMulExpr:
-      case scUMaxExpr:
-      case scSMaxExpr:
-      case scUMinExpr:
-      case scSMinExpr:
-      case scSequentialUMinExpr:
-        // These expressions are available if their operand(s) is/are.
-        return true;
-
-      case scAddRecExpr: {
-        // We allow add recurrences that are on the loop BB is in, or some
-        // outer loop.  This guarantees availability because the value of the
-        // add recurrence at BB is simply the "current" value of the induction
-        // variable.  We can relax this in the future; for instance an add
-        // recurrence on a sibling dominating loop is also available at BB.
-        const auto *ARLoop = cast<SCEVAddRecExpr>(S)->getLoop();
-        if (L && (ARLoop == L || ARLoop->contains(L)))
-          return true;
-
-        return setUnavailable();
-      }
-
-      case scUnknown: {
-        // For SCEVUnknown, we check for simple dominance.
-        const auto *SU = cast<SCEVUnknown>(S);
-        Value *V = SU->getValue();
-
-        if (isa<Argument>(V))
-          return false;
-
-        if (isa<Instruction>(V) && DT.dominates(cast<Instruction>(V), BB))
-          return false;
-
-        return setUnavailable();
-      }
-
-      case scUDivExpr:
-      case scCouldNotCompute:
-        // We do not try to smart about these at all.
-        return setUnavailable();
-      }
-      llvm_unreachable("Unknown SCEV kind!");
-    }
-
-    bool isDone() { return TraversalDone; }
-  };
-
-  CheckAvailable CA(L, BB, DT);
-  SCEVTraversal<CheckAvailable> ST(CA);
-
-  ST.visitAll(S);
-  return CA.Available;
-}
-
 // Try to match a control flow sequence that branches out at BI and merges back
 // at Merge into a "C ? LHS : RHS" select pattern.  Return true on a successful
 // match.
@@ -5994,13 +5944,6 @@ const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
   auto IsReachable =
       [&](BasicBlock *BB) { return DT.isReachableFromEntry(BB); };
   if (PN->getNumIncomingValues() == 2 && all_of(PN->blocks(), IsReachable)) {
-    const Loop *L = LI.getLoopFor(PN->getParent());
-
-    // We don't want to break LCSSA, even in a SCEV expression tree.
-    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
-      if (LI.getLoopFor(PN->getIncomingBlock(i)) != L)
-        return nullptr;
-
     // Try to match
     //
     //  br %cond, label %left, label %right
@@ -6021,8 +5964,8 @@ const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
 
     if (BI && BI->isConditional() &&
         BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS) &&
-        IsAvailableOnEntry(L, DT, getSCEV(LHS), PN->getParent()) &&
-        IsAvailableOnEntry(L, DT, getSCEV(RHS), PN->getParent()))
+        properlyDominates(getSCEV(LHS), PN->getParent()) &&
+        properlyDominates(getSCEV(RHS), PN->getParent()))
       return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
   }
 
@@ -6033,12 +5976,12 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
   if (const SCEV *S = createAddRecFromPHI(PN))
     return S;
 
-  if (const SCEV *S = createNodeFromSelectLikePHI(PN))
-    return S;
-
   if (Value *V = simplifyInstruction(PN, {getDataLayout(), &TLI, &DT, &AC}))
     return getSCEV(V);
 
+  if (const SCEV *S = createNodeFromSelectLikePHI(PN))
+    return S;
+
   // If it's not a loop phi, we can't handle it yet.
   return getUnknown(PN);
 }
@@ -6829,6 +6772,31 @@ const ConstantRange &ScalarEvolution::getRangeRef(
                         APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1),
           RangeType);
 
+    if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
+      // Strengthen the range if the underlying IR value is a global using the
+      // size of the global.
+      ObjectSizeOpts Opts;
+      Opts.RoundToAlign = false;
+      Opts.NullIsUnknownSize = true;
+      uint64_t ObjSize;
+      auto *GV = dyn_cast<GlobalVariable>(U->getValue());
+      if (GV && getObjectSize(U->getValue(), ObjSize, DL, &TLI, Opts) &&
+          ObjSize > 1) {
+        // The highest address the object can start is ObjSize bytes before the
+        // end (unsigned max value). If this value is not a multiple of the
+        // alignment, the last possible start value is the next lowest multiple
+        // of the alignment. Note: The computations below cannot overflow,
+        // because if they would there's no possible start address for the
+        // object.
+        APInt MaxVal = APInt::getMaxValue(BitWidth) - APInt(BitWidth, ObjSize);
+        uint64_t Align = GV->getAlign().valueOrOne().value();
+        uint64_t Rem = MaxVal.urem(Align);
+        MaxVal -= APInt(BitWidth, Rem);
+        ConservativeResult = ConservativeResult.intersectWith(
+            {ConservativeResult.getUnsignedMin(), MaxVal + 1}, RangeType);
+      }
+    }
+
     // A range of Phi is a subset of union of all ranges of its input.
     if (PHINode *Phi = dyn_cast<PHINode>(U->getValue())) {
       // Make sure that we do not run over cycled Phis.
@@ -7425,18 +7393,18 @@ ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
         auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
                                    dyn_cast<Instruction>(V));
         if (!NewBO ||
-            (U->getOpcode() == Instruction::Add &&
+            (BO->Opcode == Instruction::Add &&
              (NewBO->Opcode != Instruction::Add &&
               NewBO->Opcode != Instruction::Sub)) ||
-            (U->getOpcode() == Instruction::Mul &&
+            (BO->Opcode == Instruction::Mul &&
              NewBO->Opcode != Instruction::Mul)) {
           Ops.push_back(BO->LHS);
           break;
         }
         // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
         // requires a SCEV for the LHS.
-        if (NewBO->Op && (NewBO->IsNSW || NewBO->IsNUW)) {
-          auto *I = dyn_cast<Instruction>(NewBO->Op);
+        if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
+          auto *I = dyn_cast<Instruction>(BO->Op);
           if (I && programUndefinedIfPoison(I)) {
             Ops.push_back(BO->LHS);
             break;
@@ -7458,7 +7426,7 @@ ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
       break;
     case Instruction::And:
     case Instruction::Or:
-      if (!IsConstArg && BO->LHS->getType()->isIntegerTy(1))
+      if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(1))
         return nullptr;
       break;
     case Instruction::LShr:
@@ -8020,27 +7988,45 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
 //                   Iteration Count Computation Code
 //
 
-const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
-                                                       bool Extend) {
+const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
   if (isa<SCEVCouldNotCompute>(ExitCount))
     return getCouldNotCompute();
 
   auto *ExitCountType = ExitCount->getType();
   assert(ExitCountType->isIntegerTy());
+  auto *EvalTy = Type::getIntNTy(ExitCountType->getContext(),
+                                 1 + ExitCountType->getScalarSizeInBits());
+  return getTripCountFromExitCount(ExitCount, EvalTy, nullptr);
+}
+
+const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
+                                                       Type *EvalTy,
+                                                       const Loop *L) {
+  if (isa<SCEVCouldNotCompute>(ExitCount))
+    return getCouldNotCompute();
 
-  if (!Extend)
-    return getAddExpr(ExitCount, getOne(ExitCountType));
+  unsigned ExitCountSize = getTypeSizeInBits(ExitCount->getType());
+  unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
 
-  ConstantRange ExitCountRange =
+  auto CanAddOneWithoutOverflow = [&]() {
+    ConstantRange ExitCountRange =
       getRangeRef(ExitCount, RangeSignHint::HINT_RANGE_UNSIGNED);
-  if (!ExitCountRange.contains(
-          APInt::getMaxValue(ExitCountRange.getBitWidth())))
-    return getAddExpr(ExitCount, getOne(ExitCountType));
+    if (!ExitCountRange.contains(APInt::getMaxValue(ExitCountSize)))
+      return true;
+
+    return L && isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, ExitCount,
+                                         getMinusOne(ExitCount->getType()));
+  };
 
-  auto *WiderType = Type::getIntNTy(ExitCountType->getContext(),
-                                    1 + ExitCountType->getScalarSizeInBits());
-  return getAddExpr(getNoopOrZeroExtend(ExitCount, WiderType),
-                    getOne(WiderType));
+  // If we need to zero extend the backedge count, check if we can add one to
+  // it prior to zero extending without overflow. Provided this is safe, it
+  // allows better simplification of the +1.
+  if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
+    return getZeroExtendExpr(
+        getAddExpr(ExitCount, getOne(ExitCount->getType())), EvalTy);
+
+  // Get the total trip count from the count by adding 1.  This may wrap.
+  return getAddExpr(getTruncateOrZeroExtend(ExitCount, EvalTy), getOne(EvalTy));
 }
 
 static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
@@ -8423,6 +8409,8 @@ void ScalarEvolution::visitAndClearUsers(
     SmallVectorImpl<const SCEV *> &ToForget) {
   while (!Worklist.empty()) {
     Instruction *I = Worklist.pop_back_val();
+    if (!isSCEVable(I->getType()))
+      continue;
 
     ValueExprMapType::iterator It =
         ValueExprMap.find_as(static_cast<Value *>(I));
@@ -9991,17 +9979,6 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
             return getSCEV(RV);
         }
       }
-
-      // If there is a single-input Phi, evaluate it at our scope. If we can
-      // prove that this replacement does not break LCSSA form, use new value.
-      if (PN->getNumOperands() == 1) {
-        const SCEV *Input = getSCEV(PN->getOperand(0));
-        const SCEV *InputAtScope = getSCEVAtScope(Input, L);
-        // TODO: We can generalize it using LI.replacementPreservesLCSSAForm,
-        // for the simplest case just support constants.
-        if (isa<SCEVConstant>(InputAtScope))
-          return InputAtScope;
-      }
     }
 
     // Okay, this is an expression that we cannot symbolically evaluate
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 00ed1edd3a881..0376c011fb76a 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -474,7 +474,7 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_ZnajSt11align_val_tRKSt9nothrow_t);
     TLI.setUnavailable(LibFunc_Znam);
     TLI.setUnavailable(LibFunc_ZnamRKSt9nothrow_t);
-    TLI.setUnavailable(LibFunc_ZnamRKSt9nothrow_t10hot_cold_t);
+    TLI.setUnavailable(LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t);
     TLI.setUnavailable(LibFunc_ZnamSt11align_val_t);
     TLI.setUnavailable(LibFunc_ZnamSt11align_val_tRKSt9nothrow_t);
     TLI.setUnavailable(LibFunc_Znwj);
@@ -483,15 +483,15 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_ZnwjSt11align_val_tRKSt9nothrow_t);
     TLI.setUnavailable(LibFunc_Znwm);
     TLI.setUnavailable(LibFunc_ZnwmRKSt9nothrow_t);
-    TLI.setUnavailable(LibFunc_ZnwmRKSt9nothrow_t10hot_cold_t);
+    TLI.setUnavailable(LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t);
     TLI.setUnavailable(LibFunc_ZnwmSt11align_val_t);
     TLI.setUnavailable(LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t);
-    TLI.setUnavailable(LibFunc_Znwm10hot_cold_t);
-    TLI.setUnavailable(LibFunc_ZnwmSt11align_val_t10hot_cold_t);
-    TLI.setUnavailable(LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t);
-    TLI.setUnavailable(LibFunc_Znam10hot_cold_t);
-    TLI.setUnavailable(LibFunc_ZnamSt11align_val_t10hot_cold_t);
-    TLI.setUnavailable(LibFunc_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t);
+    TLI.setUnavailable(LibFunc_Znwm12__hot_cold_t);
+    TLI.setUnavailable(LibFunc_ZnwmSt11align_val_t12__hot_cold_t);
+    TLI.setUnavailable(LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t);
+    TLI.setUnavailable(LibFunc_Znam12__hot_cold_t);
+    TLI.setUnavailable(LibFunc_ZnamSt11align_val_t12__hot_cold_t);
+    TLI.setUnavailable(LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t);
   } else {
     // Not MSVC, assume it's Itanium.
     TLI.setUnavailable(LibFunc_msvc_new_int);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index c3e0eb7cc546f..f5bf29d169826 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -641,6 +641,250 @@ static bool isKnownNonZeroFromAssume(const Value *V, const Query &Q) {
   return false;
 }
 
+static void computeKnownBitsFromCmp(const Value *V, const ICmpInst *Cmp,
+                                    KnownBits &Known, unsigned Depth,
+                                    const Query &Q) {
+  unsigned BitWidth = Known.getBitWidth();
+  // We are attempting to compute known bits for the operands of an assume.
+  // Do not try to use other assumptions for those recursive calls because
+  // that can lead to mutual recursion and a compile-time explosion.
+  // An example of the mutual recursion: computeKnownBits can call
+  // isKnownNonZero which calls computeKnownBitsFromAssume (this function)
+  // and so on.
+  Query QueryNoAC = Q;
+  QueryNoAC.AC = nullptr;
+
+  // Note that ptrtoint may change the bitwidth.
+  Value *A, *B;
+  auto m_V = m_CombineOr(m_Specific(V), m_PtrToInt(m_Specific(V)));
+
+  CmpInst::Predicate Pred;
+  uint64_t C;
+  switch (Cmp->getPredicate()) {
+  default:
+    break;
+  case ICmpInst::ICMP_EQ:
+    // assume(v = a)
+    if (match(Cmp, m_c_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      Known.Zero |= RHSKnown.Zero;
+      Known.One |= RHSKnown.One;
+      // assume(v & b = a)
+    } else if (match(Cmp,
+                     m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      KnownBits MaskKnown =
+          computeKnownBits(B, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in the mask that are known to be one, we can propagate
+      // known bits from the RHS to V.
+      Known.Zero |= RHSKnown.Zero & MaskKnown.One;
+      Known.One |= RHSKnown.One & MaskKnown.One;
+      // assume(~(v & b) = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))),
+                                   m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      KnownBits MaskKnown =
+          computeKnownBits(B, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in the mask that are known to be one, we can propagate
+      // inverted known bits from the RHS to V.
+      Known.Zero |= RHSKnown.One & MaskKnown.One;
+      Known.One |= RHSKnown.Zero & MaskKnown.One;
+      // assume(v | b = a)
+    } else if (match(Cmp,
+                     m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      KnownBits BKnown =
+          computeKnownBits(B, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in B that are known to be zero, we can propagate known
+      // bits from the RHS to V.
+      Known.Zero |= RHSKnown.Zero & BKnown.Zero;
+      Known.One |= RHSKnown.One & BKnown.Zero;
+      // assume(~(v | b) = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))),
+                                   m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      KnownBits BKnown =
+          computeKnownBits(B, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in B that are known to be zero, we can propagate
+      // inverted known bits from the RHS to V.
+      Known.Zero |= RHSKnown.One & BKnown.Zero;
+      Known.One |= RHSKnown.Zero & BKnown.Zero;
+      // assume(v ^ b = a)
+    } else if (match(Cmp,
+                     m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      KnownBits BKnown =
+          computeKnownBits(B, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in B that are known to be zero, we can propagate known
+      // bits from the RHS to V. For those bits in B that are known to be one,
+      // we can propagate inverted known bits from the RHS to V.
+      Known.Zero |= RHSKnown.Zero & BKnown.Zero;
+      Known.One |= RHSKnown.One & BKnown.Zero;
+      Known.Zero |= RHSKnown.One & BKnown.One;
+      Known.One |= RHSKnown.Zero & BKnown.One;
+      // assume(~(v ^ b) = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))),
+                                   m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      KnownBits BKnown =
+          computeKnownBits(B, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in B that are known to be zero, we can propagate
+      // inverted known bits from the RHS to V. For those bits in B that are
+      // known to be one, we can propagate known bits from the RHS to V.
+      Known.Zero |= RHSKnown.One & BKnown.Zero;
+      Known.One |= RHSKnown.Zero & BKnown.Zero;
+      Known.Zero |= RHSKnown.Zero & BKnown.One;
+      Known.One |= RHSKnown.One & BKnown.One;
+      // assume(v << c = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)),
+                                   m_Value(A))) &&
+               C < BitWidth) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // For those bits in RHS that are known, we can propagate them to known
+      // bits in V shifted to the right by C.
+      RHSKnown.Zero.lshrInPlace(C);
+      Known.Zero |= RHSKnown.Zero;
+      RHSKnown.One.lshrInPlace(C);
+      Known.One |= RHSKnown.One;
+      // assume(~(v << c) = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))),
+                                   m_Value(A))) &&
+               C < BitWidth) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      // For those bits in RHS that are known, we can propagate them inverted
+      // to known bits in V shifted to the right by C.
+      RHSKnown.One.lshrInPlace(C);
+      Known.Zero |= RHSKnown.One;
+      RHSKnown.Zero.lshrInPlace(C);
+      Known.One |= RHSKnown.Zero;
+      // assume(v >> c = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Shr(m_V, m_ConstantInt(C)),
+                                   m_Value(A))) &&
+               C < BitWidth) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      // For those bits in RHS that are known, we can propagate them to known
+      // bits in V shifted to the right by C.
+      Known.Zero |= RHSKnown.Zero << C;
+      Known.One |= RHSKnown.One << C;
+      // assume(~(v >> c) = a)
+    } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shr(m_V, m_ConstantInt(C))),
+                                   m_Value(A))) &&
+               C < BitWidth) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+      // For those bits in RHS that are known, we can propagate them inverted
+      // to known bits in V shifted to the right by C.
+      Known.Zero |= RHSKnown.One << C;
+      Known.One |= RHSKnown.Zero << C;
+    }
+    break;
+  case ICmpInst::ICMP_SGE:
+    // assume(v >=_s c) where c is non-negative
+    if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      if (RHSKnown.isNonNegative()) {
+        // We know that the sign bit is zero.
+        Known.makeNonNegative();
+      }
+    }
+    break;
+  case ICmpInst::ICMP_SGT:
+    // assume(v >_s c) where c is at least -1.
+    if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      if (RHSKnown.isAllOnes() || RHSKnown.isNonNegative()) {
+        // We know that the sign bit is zero.
+        Known.makeNonNegative();
+      }
+    }
+    break;
+  case ICmpInst::ICMP_SLE:
+    // assume(v <=_s c) where c is negative
+    if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      if (RHSKnown.isNegative()) {
+        // We know that the sign bit is one.
+        Known.makeNegative();
+      }
+    }
+    break;
+  case ICmpInst::ICMP_SLT:
+    // assume(v <_s c) where c is non-positive
+    if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      if (RHSKnown.isZero() || RHSKnown.isNegative()) {
+        // We know that the sign bit is one.
+        Known.makeNegative();
+      }
+    }
+    break;
+  case ICmpInst::ICMP_ULE:
+    // assume(v <=_u c)
+    if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // Whatever high bits in c are zero are known to be zero.
+      Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros());
+    }
+    break;
+  case ICmpInst::ICMP_ULT:
+    // assume(v <_u c)
+    if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A)))) {
+      KnownBits RHSKnown =
+          computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
+
+      // If the RHS is known zero, then this assumption must be wrong (nothing
+      // is unsigned less than zero). Signal a conflict and get out of here.
+      if (RHSKnown.isZero()) {
+        Known.Zero.setAllBits();
+        Known.One.setAllBits();
+        break;
+      }
+
+      // Whatever high bits in c are zero are known to be zero (if c is a power
+      // of 2, then one more).
+      if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, QueryNoAC))
+        Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros() + 1);
+      else
+        Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros());
+    }
+    break;
+  case ICmpInst::ICMP_NE: {
+    // assume (v & b != 0) where b is a power of 2
+    const APInt *BPow2;
+    if (match(Cmp, m_ICmp(Pred, m_c_And(m_V, m_Power2(BPow2)), m_Zero()))) {
+      Known.One |= BPow2->zextOrTrunc(BitWidth);
+    }
+  } break;
+  }
+}
+
 static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known,
                                        unsigned Depth, const Query &Q) {
   // Use of assumptions is context-sensitive. If we don't have a context, we
@@ -653,7 +897,7 @@ static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known,
   // Refine Known set if the pointer alignment is set by assume bundles.
   if (V->getType()->isPointerTy()) {
     if (RetainedKnowledge RK = getKnowledgeValidInContext(
-            V, {Attribute::Alignment}, Q.CxtI, Q.DT, Q.AC)) {
+            V, { Attribute::Alignment }, Q.CxtI, Q.DT, Q.AC)) {
       if (isPowerOf2_64(RK.ArgValue))
         Known.Zero.setLowBits(Log2_64(RK.ArgValue));
     }
@@ -698,258 +942,10 @@ static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known,
     if (!Cmp)
       continue;
 
-    // We are attempting to compute known bits for the operands of an assume.
-    // Do not try to use other assumptions for those recursive calls because
-    // that can lead to mutual recursion and a compile-time explosion.
-    // An example of the mutual recursion: computeKnownBits can call
-    // isKnownNonZero which calls computeKnownBitsFromAssume (this function)
-    // and so on.
-    Query QueryNoAC = Q;
-    QueryNoAC.AC = nullptr;
-
-    // Note that ptrtoint may change the bitwidth.
-    Value *A, *B;
-    auto m_V = m_CombineOr(m_Specific(V), m_PtrToInt(m_Specific(V)));
-
-    CmpInst::Predicate Pred;
-    uint64_t C;
-    switch (Cmp->getPredicate()) {
-    default:
-      break;
-    case ICmpInst::ICMP_EQ:
-      // assume(v = a)
-      if (match(Cmp, m_c_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        Known.Zero |= RHSKnown.Zero;
-        Known.One  |= RHSKnown.One;
-      // assume(v & b = a)
-      } else if (match(Cmp,
-                       m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        KnownBits MaskKnown =
-            computeKnownBits(B, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in the mask that are known to be one, we can propagate
-        // known bits from the RHS to V.
-        Known.Zero |= RHSKnown.Zero & MaskKnown.One;
-        Known.One  |= RHSKnown.One  & MaskKnown.One;
-      // assume(~(v & b) = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        KnownBits MaskKnown =
-            computeKnownBits(B, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in the mask that are known to be one, we can propagate
-        // inverted known bits from the RHS to V.
-        Known.Zero |= RHSKnown.One  & MaskKnown.One;
-        Known.One  |= RHSKnown.Zero & MaskKnown.One;
-      // assume(v | b = a)
-      } else if (match(Cmp,
-                       m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        KnownBits BKnown =
-            computeKnownBits(B, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in B that are known to be zero, we can propagate known
-        // bits from the RHS to V.
-        Known.Zero |= RHSKnown.Zero & BKnown.Zero;
-        Known.One  |= RHSKnown.One  & BKnown.Zero;
-      // assume(~(v | b) = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        KnownBits BKnown =
-            computeKnownBits(B, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in B that are known to be zero, we can propagate
-        // inverted known bits from the RHS to V.
-        Known.Zero |= RHSKnown.One  & BKnown.Zero;
-        Known.One  |= RHSKnown.Zero & BKnown.Zero;
-      // assume(v ^ b = a)
-      } else if (match(Cmp,
-                       m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        KnownBits BKnown =
-            computeKnownBits(B, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in B that are known to be zero, we can propagate known
-        // bits from the RHS to V. For those bits in B that are known to be one,
-        // we can propagate inverted known bits from the RHS to V.
-        Known.Zero |= RHSKnown.Zero & BKnown.Zero;
-        Known.One  |= RHSKnown.One  & BKnown.Zero;
-        Known.Zero |= RHSKnown.One  & BKnown.One;
-        Known.One  |= RHSKnown.Zero & BKnown.One;
-      // assume(~(v ^ b) = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        KnownBits BKnown =
-            computeKnownBits(B, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in B that are known to be zero, we can propagate
-        // inverted known bits from the RHS to V. For those bits in B that are
-        // known to be one, we can propagate known bits from the RHS to V.
-        Known.Zero |= RHSKnown.One  & BKnown.Zero;
-        Known.One  |= RHSKnown.Zero & BKnown.Zero;
-        Known.Zero |= RHSKnown.Zero & BKnown.One;
-        Known.One  |= RHSKnown.One  & BKnown.One;
-      // assume(v << c = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // For those bits in RHS that are known, we can propagate them to known
-        // bits in V shifted to the right by C.
-        RHSKnown.Zero.lshrInPlace(C);
-        Known.Zero |= RHSKnown.Zero;
-        RHSKnown.One.lshrInPlace(C);
-        Known.One  |= RHSKnown.One;
-      // assume(~(v << c) = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        // For those bits in RHS that are known, we can propagate them inverted
-        // to known bits in V shifted to the right by C.
-        RHSKnown.One.lshrInPlace(C);
-        Known.Zero |= RHSKnown.One;
-        RHSKnown.Zero.lshrInPlace(C);
-        Known.One  |= RHSKnown.Zero;
-      // assume(v >> c = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Shr(m_V, m_ConstantInt(C)),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        // For those bits in RHS that are known, we can propagate them to known
-        // bits in V shifted to the right by C.
-        Known.Zero |= RHSKnown.Zero << C;
-        Known.One  |= RHSKnown.One  << C;
-      // assume(~(v >> c) = a)
-      } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shr(m_V, m_ConstantInt(C))),
-                                     m_Value(A))) &&
-                 isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-        // For those bits in RHS that are known, we can propagate them inverted
-        // to known bits in V shifted to the right by C.
-        Known.Zero |= RHSKnown.One  << C;
-        Known.One  |= RHSKnown.Zero << C;
-      }
-      break;
-    case ICmpInst::ICMP_SGE:
-      // assume(v >=_s c) where c is non-negative
-      if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        if (RHSKnown.isNonNegative()) {
-          // We know that the sign bit is zero.
-          Known.makeNonNegative();
-        }
-      }
-      break;
-    case ICmpInst::ICMP_SGT:
-      // assume(v >_s c) where c is at least -1.
-      if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        if (RHSKnown.isAllOnes() || RHSKnown.isNonNegative()) {
-          // We know that the sign bit is zero.
-          Known.makeNonNegative();
-        }
-      }
-      break;
-    case ICmpInst::ICMP_SLE:
-      // assume(v <=_s c) where c is negative
-      if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth + 1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        if (RHSKnown.isNegative()) {
-          // We know that the sign bit is one.
-          Known.makeNegative();
-        }
-      }
-      break;
-    case ICmpInst::ICMP_SLT:
-      // assume(v <_s c) where c is non-positive
-      if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        if (RHSKnown.isZero() || RHSKnown.isNegative()) {
-          // We know that the sign bit is one.
-          Known.makeNegative();
-        }
-      }
-      break;
-    case ICmpInst::ICMP_ULE:
-      // assume(v <=_u c)
-      if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // Whatever high bits in c are zero are known to be zero.
-        Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros());
-      }
-      break;
-    case ICmpInst::ICMP_ULT:
-      // assume(v <_u c)
-      if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        KnownBits RHSKnown =
-            computeKnownBits(A, Depth+1, QueryNoAC).anyextOrTrunc(BitWidth);
-
-        // If the RHS is known zero, then this assumption must be wrong (nothing
-        // is unsigned less than zero). Signal a conflict and get out of here.
-        if (RHSKnown.isZero()) {
-          Known.Zero.setAllBits();
-          Known.One.setAllBits();
-          break;
-        }
+    if (!isValidAssumeForContext(I, Q.CxtI, Q.DT))
+      continue;
 
-        // Whatever high bits in c are zero are known to be zero (if c is a power
-        // of 2, then one more).
-        if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, QueryNoAC))
-          Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros() + 1);
-        else
-          Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros());
-      }
-      break;
-    case ICmpInst::ICMP_NE: {
-      // assume (v & b != 0) where b is a power of 2
-      const APInt *BPow2;
-      if (match(Cmp, m_ICmp(Pred, m_c_And(m_V, m_Power2(BPow2)), m_Zero())) &&
-          isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
-        Known.One |= BPow2->zextOrTrunc(BitWidth);
-      }
-    } break;
-    }
+    computeKnownBitsFromCmp(V, Cmp, Known, Depth, Q);
   }
 
   // If assumptions conflict with each other or previous known bits, then we
@@ -2512,6 +2508,63 @@ static bool isNonZeroRecurrence(const PHINode *PN) {
   }
 }
 
+static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
+                         const Query &Q, unsigned BitWidth, Value *X, Value *Y,
+                         bool NSW) {
+  KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q);
+  KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q);
+
+  // If X and Y are both non-negative (as signed values) then their sum is not
+  // zero unless both X and Y are zero.
+  if (XKnown.isNonNegative() && YKnown.isNonNegative())
+    if (isKnownNonZero(Y, DemandedElts, Depth, Q) ||
+        isKnownNonZero(X, DemandedElts, Depth, Q))
+      return true;
+
+  // If X and Y are both negative (as signed values) then their sum is not
+  // zero unless both X and Y equal INT_MIN.
+  if (XKnown.isNegative() && YKnown.isNegative()) {
+    APInt Mask = APInt::getSignedMaxValue(BitWidth);
+    // The sign bit of X is set.  If some other bit is set then X is not equal
+    // to INT_MIN.
+    if (XKnown.One.intersects(Mask))
+      return true;
+    // The sign bit of Y is set.  If some other bit is set then Y is not equal
+    // to INT_MIN.
+    if (YKnown.One.intersects(Mask))
+      return true;
+  }
+
+  // The sum of a non-negative number and a power of two is not zero.
+  if (XKnown.isNonNegative() &&
+      isKnownToBeAPowerOfTwo(Y, /*OrZero*/ false, Depth, Q))
+    return true;
+  if (YKnown.isNonNegative() &&
+      isKnownToBeAPowerOfTwo(X, /*OrZero*/ false, Depth, Q))
+    return true;
+
+  return KnownBits::computeForAddSub(/*Add*/ true, NSW, XKnown, YKnown)
+      .isNonZero();
+}
+
+static bool isNonZeroSub(const APInt &DemandedElts, unsigned Depth,
+                         const Query &Q, unsigned BitWidth, Value *X,
+                         Value *Y) {
+  if (auto *C = dyn_cast<Constant>(X))
+    if (C->isNullValue() && isKnownNonZero(Y, DemandedElts, Depth, Q))
+      return true;
+
+  KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q);
+  if (XKnown.isUnknown())
+    return false;
+  KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q);
+  // If X != Y then X - Y is non zero.
+  std::optional<bool> ne = KnownBits::ne(XKnown, YKnown);
+  // If we are unable to compute if X != Y, we won't be able to do anything
+  // computing the knownbits of the sub expression so just return here.
+  return ne && *ne;
+}
+
 static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
                            unsigned Depth, const Query &Q,
                            const KnownBits &KnownVal) {
@@ -2684,10 +2737,39 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
     if (I->getType()->isPointerTy())
       return isGEPKnownNonNull(cast<GEPOperator>(I), Depth, Q);
     break;
-  case Instruction::BitCast:
-    if (I->getType()->isPointerTy())
+  case Instruction::BitCast: {
+    // We need to be a bit careful here. We can only peek through the bitcast
+    // if the scalar size of elements in the operand are smaller than and a
+    // multiple of the size they are casting too. Take three cases:
+    //
+    // 1) Unsafe:
+    //        bitcast <2 x i16> %NonZero to <4 x i8>
+    //
+    //    %NonZero can have 2 non-zero i16 elements, but isKnownNonZero on a
+    //    <4 x i8> requires that all 4 i8 elements be non-zero which isn't
+    //    guranteed (imagine just sign bit set in the 2 i16 elements).
+    //
+    // 2) Unsafe:
+    //        bitcast <4 x i3> %NonZero to <3 x i4>
+    //
+    //    Even though the scalar size of the src (`i3`) is smaller than the
+    //    scalar size of the dst `i4`, because `i3` is not a multiple of `i4`
+    //    its possible for the `3 x i4` elements to be zero because there are
+    //    some elements in the destination that don't contain any full src
+    //    element.
+    //
+    // 3) Safe:
+    //        bitcast <4 x i8> %NonZero to <2 x i16>
+    //
+    //    This is always safe as non-zero in the 4 i8 elements implies
+    //    non-zero in the combination of any two adjacent ones. Since i8 is a
+    //    multiple of i16, each i16 is guranteed to have 2 full i8 elements.
+    //    This all implies the 2 i16 elements are non-zero.
+    Type *FromTy = I->getOperand(0)->getType();
+    if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
+        (BitWidth % getBitWidth(FromTy->getScalarType(), Q.DL)) == 0)
       return isKnownNonZero(I->getOperand(0), Depth, Q);
-    break;
+  } break;
   case Instruction::IntToPtr:
     // Note that we have to take special care to avoid looking through
     // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
@@ -2706,15 +2788,12 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
       return isKnownNonZero(I->getOperand(0), Depth, Q);
     break;
   case Instruction::Sub:
-    if (auto *C = dyn_cast<Constant>(I->getOperand(0)))
-      if (C->isNullValue() &&
-          isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q))
-        return true;
-    break;
+    return isNonZeroSub(DemandedElts, Depth, Q, BitWidth, I->getOperand(0),
+                        I->getOperand(1));
   case Instruction::Or:
     // X | Y != 0 if X != 0 or Y != 0.
-    return isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q) ||
-           isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q);
+    return isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q) ||
+           isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q);
   case Instruction::SExt:
   case Instruction::ZExt:
     // ext X != 0 if X != 0.
@@ -2753,56 +2832,60 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
   }
   case Instruction::UDiv:
   case Instruction::SDiv:
+    // X / Y
     // div exact can only produce a zero if the dividend is zero.
     if (cast<PossiblyExactOperator>(I)->isExact())
       return isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q);
+    if (I->getOpcode() == Instruction::UDiv) {
+      std::optional<bool> XUgeY;
+      KnownBits XKnown =
+          computeKnownBits(I->getOperand(0), DemandedElts, Depth, Q);
+      if (!XKnown.isUnknown()) {
+        KnownBits YKnown =
+            computeKnownBits(I->getOperand(1), DemandedElts, Depth, Q);
+        // If X u>= Y then div is non zero (0/0 is UB).
+        XUgeY = KnownBits::uge(XKnown, YKnown);
+      }
+      // If X is total unknown or X u< Y we won't be able to prove non-zero
+      // with compute known bits so just return early.
+      return XUgeY && *XUgeY;
+    }
     break;
   case Instruction::Add: {
     // X + Y.
-    KnownBits XKnown =
-        computeKnownBits(I->getOperand(0), DemandedElts, Depth, Q);
-    KnownBits YKnown =
-        computeKnownBits(I->getOperand(1), DemandedElts, Depth, Q);
-
-    // If X and Y are both non-negative (as signed values) then their sum is not
-    // zero unless both X and Y are zero.
-    if (XKnown.isNonNegative() && YKnown.isNonNegative())
-      if (isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q) ||
-          isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q))
-        return true;
 
-    // If X and Y are both negative (as signed values) then their sum is not
-    // zero unless both X and Y equal INT_MIN.
-    if (XKnown.isNegative() && YKnown.isNegative()) {
-      APInt Mask = APInt::getSignedMaxValue(BitWidth);
-      // The sign bit of X is set.  If some other bit is set then X is not equal
-      // to INT_MIN.
-      if (XKnown.One.intersects(Mask))
-        return true;
-      // The sign bit of Y is set.  If some other bit is set then Y is not equal
-      // to INT_MIN.
-      if (YKnown.One.intersects(Mask))
-        return true;
-    }
+    // If Add has nuw wrap flag, then if either X or Y is non-zero the result is
+    // non-zero.
+    auto *BO = cast<OverflowingBinaryOperator>(V);
+    if (Q.IIQ.hasNoUnsignedWrap(BO))
+      return isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q) ||
+             isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q);
 
-    // The sum of a non-negative number and a power of two is not zero.
-    if (XKnown.isNonNegative() &&
-        isKnownToBeAPowerOfTwo(I->getOperand(1), /*OrZero*/ false, Depth, Q))
-      return true;
-    if (YKnown.isNonNegative() &&
-        isKnownToBeAPowerOfTwo(I->getOperand(0), /*OrZero*/ false, Depth, Q))
-      return true;
-    break;
+    return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth, I->getOperand(0),
+                        I->getOperand(1), Q.IIQ.hasNoSignedWrap(BO));
   }
   case Instruction::Mul: {
     // If X and Y are non-zero then so is X * Y as long as the multiplication
     // does not overflow.
     const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(V);
-    if ((Q.IIQ.hasNoSignedWrap(BO) || Q.IIQ.hasNoUnsignedWrap(BO)) &&
-        isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q) &&
-        isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q))
-      return true;
-    break;
+    if (Q.IIQ.hasNoSignedWrap(BO) || Q.IIQ.hasNoUnsignedWrap(BO))
+      return isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q) &&
+             isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q);
+
+    // If either X or Y is odd, then if the other is non-zero the result can't
+    // be zero.
+    KnownBits XKnown =
+        computeKnownBits(I->getOperand(0), DemandedElts, Depth, Q);
+    if (XKnown.One[0])
+      return isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q);
+
+    KnownBits YKnown =
+        computeKnownBits(I->getOperand(1), DemandedElts, Depth, Q);
+    if (YKnown.One[0])
+      return XKnown.isNonZero() ||
+             isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q);
+
+    return KnownBits::mul(XKnown, YKnown).isNonZero();
   }
   case Instruction::Select:
     // (C ? X : Y) != 0 if X != 0 and Y != 0.
@@ -2846,17 +2929,58 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
   case Instruction::Call:
     if (auto *II = dyn_cast<IntrinsicInst>(I)) {
       switch (II->getIntrinsicID()) {
+      case Intrinsic::sshl_sat:
+      case Intrinsic::ushl_sat:
       case Intrinsic::abs:
       case Intrinsic::bitreverse:
       case Intrinsic::bswap:
       case Intrinsic::ctpop:
-        if (isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q))
-          return true;
-        break;
+        return isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q);
+      case Intrinsic::ssub_sat:
+        return isNonZeroSub(DemandedElts, Depth, Q, BitWidth,
+                            II->getArgOperand(0), II->getArgOperand(1));
+      case Intrinsic::sadd_sat:
+        return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth,
+                            II->getArgOperand(0), II->getArgOperand(1),
+                            /*NSW*/ true);
+      case Intrinsic::umax:
       case Intrinsic::uadd_sat:
-        if (isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q) ||
-            isKnownNonZero(II->getArgOperand(1), DemandedElts, Depth, Q))
+        return isKnownNonZero(II->getArgOperand(1), DemandedElts, Depth, Q) ||
+               isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q);
+      case Intrinsic::smin:
+      case Intrinsic::smax: {
+        auto KnownOpImpliesNonZero = [&](const KnownBits &K) {
+          return II->getIntrinsicID() == Intrinsic::smin
+                     ? K.isNegative()
+                     : K.isStrictlyPositive();
+        };
+        KnownBits XKnown =
+            computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q);
+        if (KnownOpImpliesNonZero(XKnown))
+          return true;
+        KnownBits YKnown =
+            computeKnownBits(II->getArgOperand(1), DemandedElts, Depth, Q);
+        if (KnownOpImpliesNonZero(YKnown))
+          return true;
+
+        if (XKnown.isNonZero() && YKnown.isNonZero())
           return true;
+      }
+        [[fallthrough]];
+      case Intrinsic::umin:
+        return isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q) &&
+               isKnownNonZero(II->getArgOperand(1), DemandedElts, Depth, Q);
+      case Intrinsic::cttz:
+        return computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q)
+            .Zero[0];
+      case Intrinsic::ctlz:
+        return computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q)
+            .isNonNegative();
+      case Intrinsic::fshr:
+      case Intrinsic::fshl:
+        // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
+        if (II->getArgOperand(0) == II->getArgOperand(1))
+          return isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q);
         break;
       case Intrinsic::vscale:
         return true;
@@ -4420,8 +4544,28 @@ static void computeKnownFPClass(const Value *V, KnownFPClass &Known,
   computeKnownFPClass(V, DemandedElts, InterestedClasses, Known, Depth, Q, TLI);
 }
 
+static void computeKnownFPClassForFPTrunc(const Operator *Op,
+                                          const APInt &DemandedElts,
+                                          FPClassTest InterestedClasses,
+                                          KnownFPClass &Known, unsigned Depth,
+                                          const Query &Q,
+                                          const TargetLibraryInfo *TLI) {
+  if ((InterestedClasses & fcNan) == fcNone)
+    return;
+
+  KnownFPClass KnownSrc;
+  computeKnownFPClass(Op->getOperand(0), DemandedElts, InterestedClasses,
+                      KnownSrc, Depth + 1, Q, TLI);
+  if (KnownSrc.isKnownNeverNaN())
+    Known.knownNot(fcNan);
+
+  // Infinity needs a range check.
+  // TODO: Sign bit should be preserved
+}
+
 // TODO: Merge implementations of isKnownNeverNaN, isKnownNeverInfinity,
 // CannotBeNegativeZero, cannotBeOrderedLessThanZero into here.
+
 void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
                          FPClassTest InterestedClasses, KnownFPClass &Known,
                          unsigned Depth, const Query &Q,
@@ -4505,7 +4649,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
   if (Depth == MaxAnalysisRecursionDepth)
     return;
 
-  switch (Op->getOpcode()) {
+  const unsigned Opc = Op->getOpcode();
+  switch (Opc) {
   case Instruction::FNeg: {
     computeKnownFPClass(Op->getOperand(0), DemandedElts, InterestedClasses,
                         Known, Depth + 1, Q, TLI);
@@ -4525,11 +4670,17 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Op)) {
       const Intrinsic::ID IID = II->getIntrinsicID();
       switch (IID) {
-      case Intrinsic::fabs:
-        computeKnownFPClass(II->getArgOperand(0), DemandedElts,
-                            InterestedClasses, Known, Depth + 1, Q, TLI);
+      case Intrinsic::fabs: {
+        if ((InterestedClasses & (fcNan | fcPositive)) != fcNone) {
+          // If we only care about the sign bit we don't need to inspect the
+          // operand.
+          computeKnownFPClass(II->getArgOperand(0), DemandedElts,
+                              InterestedClasses, Known, Depth + 1, Q, TLI);
+        }
+
         Known.fabs();
         break;
+      }
       case Intrinsic::copysign: {
         KnownFPClass KnownSign;
 
@@ -4540,6 +4691,27 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
         Known.copysign(KnownSign);
         break;
       }
+      case Intrinsic::fma:
+      case Intrinsic::fmuladd: {
+        if ((InterestedClasses & fcNegative) == fcNone)
+          break;
+
+        if (II->getArgOperand(0) != II->getArgOperand(1))
+          break;
+
+        // The multiply cannot be -0 and therefore the add can't be -0
+        Known.knownNot(fcNegZero);
+
+        // x * x + y is non-negative if y is non-negative.
+        KnownFPClass KnownAddend;
+        computeKnownFPClass(II->getArgOperand(2), DemandedElts,
+                            InterestedClasses, KnownAddend, Depth + 1, Q, TLI);
+
+        // TODO: Known sign bit with no nans
+        if (KnownAddend.cannotBeOrderedLessThanZero())
+          Known.knownNot(fcNegative);
+        break;
+      }
       case Intrinsic::sin:
       case Intrinsic::cos: {
         // Return NaN on infinite inputs.
@@ -4551,6 +4723,74 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
           Known.knownNot(fcNan);
         break;
       }
+
+      case Intrinsic::maxnum:
+      case Intrinsic::minnum:
+      case Intrinsic::minimum:
+      case Intrinsic::maximum: {
+        KnownFPClass KnownLHS, KnownRHS;
+        computeKnownFPClass(II->getArgOperand(0), DemandedElts,
+                            InterestedClasses, KnownLHS, Depth + 1, Q, TLI);
+        computeKnownFPClass(II->getArgOperand(1), DemandedElts,
+                            InterestedClasses, KnownRHS, Depth + 1, Q, TLI);
+
+        bool NeverNaN =
+            KnownLHS.isKnownNeverNaN() || KnownRHS.isKnownNeverNaN();
+        Known = KnownLHS | KnownRHS;
+
+        // If either operand is not NaN, the result is not NaN.
+        if (NeverNaN && (IID == Intrinsic::minnum || IID == Intrinsic::maxnum))
+          Known.knownNot(fcNan);
+
+        if (IID == Intrinsic::maxnum) {
+          // If at least one operand is known to be positive, the result must be
+          // positive.
+          if ((KnownLHS.cannotBeOrderedLessThanZero() &&
+               KnownLHS.isKnownNeverNaN()) ||
+              (KnownRHS.cannotBeOrderedLessThanZero() &&
+               KnownRHS.isKnownNeverNaN()))
+            Known.knownNot(KnownFPClass::OrderedLessThanZeroMask);
+        } else if (IID == Intrinsic::maximum) {
+          // If at least one operand is known to be positive, the result must be
+          // positive.
+          if (KnownLHS.cannotBeOrderedLessThanZero() ||
+              KnownRHS.cannotBeOrderedLessThanZero())
+            Known.knownNot(KnownFPClass::OrderedLessThanZeroMask);
+        } else if (IID == Intrinsic::minnum) {
+          // If at least one operand is known to be negative, the result must be
+          // negative.
+          if ((KnownLHS.cannotBeOrderedGreaterThanZero() &&
+               KnownLHS.isKnownNeverNaN()) ||
+              (KnownRHS.cannotBeOrderedGreaterThanZero() &&
+               KnownRHS.isKnownNeverNaN()))
+            Known.knownNot(KnownFPClass::OrderedGreaterThanZeroMask);
+        } else {
+          // If at least one operand is known to be negative, the result must be
+          // negative.
+          if (KnownLHS.cannotBeOrderedGreaterThanZero() ||
+              KnownRHS.cannotBeOrderedGreaterThanZero())
+            Known.knownNot(KnownFPClass::OrderedGreaterThanZeroMask);
+          }
+
+        // Fixup zero handling if denormals could be returned as a zero.
+        //
+        // As there's no spec for denormal flushing, be conservative with the
+        // treatment of denormals that could be flushed to zero. For older
+        // subtargets on AMDGPU the min/max instructions would not flush the
+        // output and return the original value.
+        //
+        // TODO: This could be refined based on the sign
+        if ((Known.KnownFPClasses & fcZero) != fcNone &&
+            !Known.isKnownNeverSubnormal()) {
+          const Function *Parent = II->getFunction();
+          DenormalMode Mode = Parent->getDenormalMode(
+              II->getType()->getScalarType()->getFltSemantics());
+          if (Mode != DenormalMode::getIEEE())
+            Known.KnownFPClasses |= fcZero;
+        }
+
+        break;
+      }
       case Intrinsic::canonicalize: {
         computeKnownFPClass(II->getArgOperand(0), DemandedElts,
                             InterestedClasses, Known, Depth + 1, Q, TLI);
@@ -4573,8 +4813,13 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
       }
       case Intrinsic::trunc: {
         KnownFPClass KnownSrc;
-        computeKnownFPClass(II->getArgOperand(0), DemandedElts,
-                            InterestedClasses, KnownSrc, Depth + 1, Q, TLI);
+
+        FPClassTest InterestedSrcs = InterestedClasses;
+        if (InterestedClasses & fcZero)
+          InterestedClasses |= fcNormal | fcSubnormal;
+
+        computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedSrcs,
+                            KnownSrc, Depth + 1, Q, TLI);
 
         // Integer results cannot be subnormal.
         Known.knownNot(fcSubnormal);
@@ -4588,6 +4833,107 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
         // Non-constrained intrinsics do not guarantee signaling nan quieting.
         if (KnownSrc.isKnownNeverNaN())
           Known.knownNot(fcNan);
+
+        if (KnownSrc.isKnownNever(fcPosNormal))
+          Known.knownNot(fcPosNormal);
+
+        if (KnownSrc.isKnownNever(fcNegNormal))
+          Known.knownNot(fcNegNormal);
+
+        if (KnownSrc.isKnownNever(fcPosZero | fcPosSubnormal | fcPosNormal))
+          Known.knownNot(fcPosZero);
+
+        if (KnownSrc.isKnownNever(fcNegZero | fcNegSubnormal | fcNegNormal))
+          Known.knownNot(fcNegZero);
+
+        // Sign should be preserved
+        Known.SignBit = KnownSrc.SignBit;
+        break;
+      }
+      case Intrinsic::exp:
+      case Intrinsic::exp2: {
+        Known.knownNot(fcNegative);
+        if ((InterestedClasses & fcNan) == fcNone)
+          break;
+
+        KnownFPClass KnownSrc;
+        computeKnownFPClass(II->getArgOperand(0), DemandedElts,
+                            InterestedClasses, KnownSrc, Depth + 1, Q, TLI);
+        if (KnownSrc.isKnownNeverNaN()) {
+          Known.knownNot(fcNan);
+          Known.SignBit = false;
+        }
+
+        break;
+      }
+      case Intrinsic::fptrunc_round: {
+        computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses,
+                                      Known, Depth, Q, TLI);
+        break;
+      }
+      case Intrinsic::log:
+      case Intrinsic::log10:
+      case Intrinsic::log2:
+      case Intrinsic::experimental_constrained_log:
+      case Intrinsic::experimental_constrained_log10:
+      case Intrinsic::experimental_constrained_log2: {
+        // log(+inf) -> +inf
+        // log([+-]0.0) -> -inf
+        // log(-inf) -> nan
+        // log(-x) -> nan
+        if ((InterestedClasses & (fcNan | fcInf)) == fcNone)
+          break;
+
+        FPClassTest InterestedSrcs = InterestedClasses;
+        if ((InterestedClasses & fcNegInf) != fcNone)
+          InterestedSrcs |= fcZero | fcSubnormal;
+        if ((InterestedClasses & fcNan) != fcNone)
+          InterestedSrcs |= fcNan | (fcNegative & ~fcNan);
+
+        KnownFPClass KnownSrc;
+        computeKnownFPClass(II->getArgOperand(0), DemandedElts, InterestedSrcs,
+                            KnownSrc, Depth + 1, Q, TLI);
+
+        if (KnownSrc.isKnownNeverPosInfinity())
+          Known.knownNot(fcPosInf);
+
+        if (KnownSrc.isKnownNeverNaN() &&
+            KnownSrc.cannotBeOrderedLessThanZero())
+          Known.knownNot(fcNan);
+
+        if (KnownSrc.isKnownNeverLogicalZero(*II->getFunction(), II->getType()))
+          Known.knownNot(fcNegInf);
+
+        break;
+      }
+      case Intrinsic::powi: {
+        if ((InterestedClasses & fcNegative) == fcNone)
+          break;
+
+        const Value *Exp = II->getArgOperand(1);
+        unsigned BitWidth =
+            Exp->getType()->getScalarType()->getIntegerBitWidth();
+        KnownBits ExponentKnownBits(BitWidth);
+        computeKnownBits(Exp, DemandedElts, ExponentKnownBits, Depth + 1, Q);
+
+        if (ExponentKnownBits.Zero[0]) { // Is even
+          Known.knownNot(fcNegative);
+          break;
+        }
+
+        // Given that exp is an integer, here are the
+        // ways that pow can return a negative value:
+        //
+        //   pow(-x, exp)   --> negative if exp is odd and x is negative.
+        //   pow(-0, exp)   --> -inf if exp is negative odd.
+        //   pow(-0, exp)   --> -0 if exp is positive odd.
+        //   pow(-inf, exp) --> -0 if exp is negative odd.
+        //   pow(-inf, exp) --> -inf if exp is positive odd.
+        KnownFPClass KnownSrc;
+        computeKnownFPClass(II->getArgOperand(0), DemandedElts, fcNegative,
+                            KnownSrc, Depth + 1, Q, TLI);
+        if (KnownSrc.isKnownNever(fcNegative))
+          Known.knownNot(fcNegative);
         break;
       }
       case Intrinsic::arithmetic_fence: {
@@ -4607,7 +4953,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
         Known.knownNot(fcSubnormal);
 
         if (IID == Intrinsic::experimental_constrained_uitofp)
-          Known.signBitIsZero();
+          Known.signBitMustBeZero();
 
         // TODO: Copy inf handling from instructions
         break;
@@ -4638,6 +4984,13 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     break;
   }
   case Instruction::FMul: {
+    // X * X is always non-negative or a NaN.
+    if (Op->getOperand(0) == Op->getOperand(1))
+      Known.knownNot(fcNegative);
+
+    if ((InterestedClasses & fcNan) != fcNan)
+      break;
+
     KnownFPClass KnownLHS, KnownRHS;
     computeKnownFPClass(Op->getOperand(1), DemandedElts,
                         fcNan | fcInf | fcZero | fcSubnormal, KnownRHS,
@@ -4663,36 +5016,76 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
 
     break;
   }
-  case Instruction::FDiv: {
+  case Instruction::FDiv:
+  case Instruction::FRem: {
+    if (Op->getOperand(0) == Op->getOperand(1)) {
+      // TODO: Could filter out snan if we inspect the operand
+      if (Op->getOpcode() == Instruction::FDiv) {
+        // X / X is always exactly 1.0 or a NaN.
+        Known.KnownFPClasses = fcNan | fcPosNormal;
+      } else {
+        // X % X is always exactly [+-]0.0 or a NaN.
+        Known.KnownFPClasses = fcNan | fcZero;
+      }
+
+      break;
+    }
+
     const bool WantNan = (InterestedClasses & fcNan) != fcNone;
-    if (!WantNan)
+    const bool WantNegative = (InterestedClasses & fcNegative) != fcNone;
+    const bool WantPositive =
+        Opc == Instruction::FRem && (InterestedClasses & fcPositive) != fcNone;
+    if (!WantNan && !WantNegative && !WantPositive)
       break;
 
-    // TODO: FRem
     KnownFPClass KnownLHS, KnownRHS;
 
     computeKnownFPClass(Op->getOperand(1), DemandedElts,
-                        fcNan | fcInf | fcZero | fcSubnormal, KnownRHS,
+                        fcNan | fcInf | fcZero | fcNegative, KnownRHS,
                         Depth + 1, Q, TLI);
 
-    bool KnowSomethingUseful = KnownRHS.isKnownNeverNaN() ||
-                               KnownRHS.isKnownNeverInfinity() ||
-                               KnownRHS.isKnownNeverZero();
+    bool KnowSomethingUseful =
+        KnownRHS.isKnownNeverNaN() || KnownRHS.isKnownNever(fcNegative);
+
+    if (KnowSomethingUseful || WantPositive) {
+      const FPClassTest InterestedLHS =
+          WantPositive ? fcAllFlags
+                       : fcNan | fcInf | fcZero | fcSubnormal | fcNegative;
 
-    if (KnowSomethingUseful) {
       computeKnownFPClass(Op->getOperand(0), DemandedElts,
-                          fcNan | fcInf | fcZero, KnownLHS, Depth + 1, Q, TLI);
+                          InterestedClasses & InterestedLHS, KnownLHS,
+                          Depth + 1, Q, TLI);
     }
 
     const Function *F = cast<Instruction>(Op)->getFunction();
 
-    // Only 0/0, Inf/Inf, Inf REM x and x REM 0 produce NaN.
-    // TODO: Track sign bit.
-    if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
-        (KnownLHS.isKnownNeverInfinity() || KnownRHS.isKnownNeverInfinity()) &&
-        (KnownLHS.isKnownNeverLogicalZero(*F, Op->getType()) ||
-         KnownRHS.isKnownNeverLogicalZero(*F, Op->getType()))) {
-      Known.knownNot(fcNan);
+    if (Op->getOpcode() == Instruction::FDiv) {
+      // Only 0/0, Inf/Inf produce NaN.
+      if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
+          (KnownLHS.isKnownNeverInfinity() ||
+           KnownRHS.isKnownNeverInfinity()) &&
+          (KnownLHS.isKnownNeverLogicalZero(*F, Op->getType()) ||
+           KnownRHS.isKnownNeverLogicalZero(*F, Op->getType()))) {
+        Known.knownNot(fcNan);
+      }
+
+      // X / -0.0 is -Inf (or NaN).
+      // +X / +X is +X
+      if (KnownLHS.isKnownNever(fcNegative) && KnownRHS.isKnownNever(fcNegative))
+        Known.knownNot(fcNegative);
+    } else {
+      // Inf REM x and x REM 0 produce NaN.
+      if (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN() &&
+          KnownLHS.isKnownNeverInfinity() &&
+          KnownRHS.isKnownNeverLogicalZero(*F, Op->getType())) {
+        Known.knownNot(fcNan);
+      }
+
+      // The sign for frem is the same as the first operand.
+      if (KnownLHS.isKnownNever(fcNegative))
+        Known.knownNot(fcNegative);
+      if (KnownLHS.isKnownNever(fcPositive))
+        Known.knownNot(fcPositive);
     }
 
     break;
@@ -4717,16 +5110,8 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     break;
   }
   case Instruction::FPTrunc: {
-    if ((InterestedClasses & fcNan) == fcNone)
-      break;
-
-    KnownFPClass KnownSrc;
-    computeKnownFPClass(Op->getOperand(0), DemandedElts,
-                        InterestedClasses, KnownSrc, Depth + 1, Q, TLI);
-    if (KnownSrc.isKnownNeverNaN())
-      Known.knownNot(fcNan);
-
-    // Infinity needs a range check.
+    computeKnownFPClassForFPTrunc(Op, DemandedElts, InterestedClasses, Known,
+                                  Depth, Q, TLI);
     break;
   }
   case Instruction::SIToFP:
@@ -4740,7 +5125,7 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     // sitofp and uitofp turn into +0.0 for zero.
     Known.knownNot(fcNegZero);
     if (Op->getOpcode() == Instruction::UIToFP)
-      Known.signBitIsZero();
+      Known.signBitMustBeZero();
 
     if (InterestedClasses & fcInf) {
       // Get width of largest magnitude integer (remove a bit if signed).
@@ -5712,31 +6097,34 @@ bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
   default:
     return true;
   case Instruction::UDiv:
-  case Instruction::URem: {
-    // x / y is undefined if y == 0.
-    const APInt *V;
-    if (match(Inst->getOperand(1), m_APInt(V)))
-      return *V != 0;
-    return false;
-  }
+  case Instruction::URem:
   case Instruction::SDiv:
   case Instruction::SRem: {
-    // x / y is undefined if y == 0 or x == INT_MIN and y == -1
-    const APInt *Numerator, *Denominator;
-    if (!match(Inst->getOperand(1), m_APInt(Denominator)))
-      return false;
-    // We cannot hoist this division if the denominator is 0.
-    if (*Denominator == 0)
+    // x / y is undefined if y == 0 or y is poison.
+    const DataLayout &DL = Inst->getModule()->getDataLayout();
+    if (!isGuaranteedNotToBePoison(Inst->getOperand(1), AC, CtxI, DT) ||
+        !isKnownNonZero(Inst->getOperand(1), DL, /*Depth*/ 0, AC, CtxI, DT))
       return false;
+
+    // Unsigned case only needs to avoid denominator == 0 or poison.
+    if (Opcode == Instruction::UDiv || Opcode == Instruction::URem)
+      return true;
+
+    // x s/ y is also undefined if x == INT_MIN and y == -1
+    KnownBits KnownDenominator =
+        computeKnownBits(Inst->getOperand(1), DL, /*Depth*/ 0, AC, CtxI, DT);
+
     // It's safe to hoist if the denominator is not 0 or -1.
-    if (!Denominator->isAllOnes())
+    if (!KnownDenominator.Zero.isZero())
       return true;
-    // At this point we know that the denominator is -1.  It is safe to hoist as
-    // long we know that the numerator is not INT_MIN.
-    if (match(Inst->getOperand(0), m_APInt(Numerator)))
-      return !Numerator->isMinSignedValue();
-    // The numerator *might* be MinSignedValue.
-    return false;
+
+    // At this point denominator may be -1.  It is safe to hoist as
+    // long we know that the numerator is neither poison nor INT_MIN.
+    if (!isGuaranteedNotToBePoison(Inst->getOperand(0), AC, CtxI, DT))
+      return false;
+    KnownBits KnownNumerator =
+        computeKnownBits(Inst->getOperand(0), DL, /*Depth*/ 0, AC, CtxI, DT);
+    return !KnownNumerator.getSignedMinValue().isMinSignedValue();
   }
   case Instruction::Load: {
     const LoadInst *LI = dyn_cast<LoadInst>(Inst);
@@ -6224,7 +6612,7 @@ static bool canCreateUndefOrPoison(const Operator *Op, bool PoisonOnly,
     ArrayRef<int> Mask = isa<ConstantExpr>(Op)
                              ? cast<ConstantExpr>(Op)->getShuffleMask()
                              : cast<ShuffleVectorInst>(Op)->getShuffleMask();
-    return is_contained(Mask, UndefMaskElem);
+    return is_contained(Mask, PoisonMaskElem);
   }
   case Instruction::FNeg:
   case Instruction::PHI:
@@ -6337,7 +6725,9 @@ static bool isGuaranteedNotToBeUndefOrPoison(const Value *V,
     return false;
 
   if (const auto *A = dyn_cast<Argument>(V)) {
-    if (A->hasAttribute(Attribute::NoUndef))
+    if (A->hasAttribute(Attribute::NoUndef) ||
+        A->hasAttribute(Attribute::Dereferenceable) ||
+        A->hasAttribute(Attribute::DereferenceableOrNull))
       return true;
   }
 
@@ -6470,7 +6860,7 @@ bool llvm::isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC,
   return ::isGuaranteedNotToBeUndefOrPoison(V, AC, CtxI, DT, Depth, true);
 }
 
-/// Return true if undefined behavior would provable be executed on the path to
+/// Return true if undefined behavior would provably be executed on the path to
 /// OnPathTo if Root produced a posion result.  Note that this doesn't say
 /// anything about whether OnPathTo is actually executed or whether Root is
 /// actually poison.  This can be used to assess whether a new use of Root can
@@ -6804,6 +7194,15 @@ static bool programUndefinedIfUndefOrPoison(const Value *V,
           break;
         }
       }
+
+      // Special handling for select, which returns poison if its operand 0 is
+      // poison (handled in the loop above) *or* if both its true/false operands
+      // are poison (handled here).
+      if (I.getOpcode() == Instruction::Select &&
+          YieldsPoison.count(I.getOperand(1)) &&
+          YieldsPoison.count(I.getOperand(2))) {
+        YieldsPoison.insert(&I);
+      }
     }
 
     BB = BB->getSingleSuccessor();
@@ -7540,6 +7939,12 @@ Intrinsic::ID llvm::getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID) {
   case Intrinsic::smin: return Intrinsic::smax;
   case Intrinsic::umax: return Intrinsic::umin;
   case Intrinsic::umin: return Intrinsic::umax;
+  // Please note that next four intrinsics may produce the same result for
+  // original and inverted case even if X != Y due to NaN is handled specially.
+  case Intrinsic::maximum: return Intrinsic::minimum;
+  case Intrinsic::minimum: return Intrinsic::maximum;
+  case Intrinsic::maxnum: return Intrinsic::minnum;
+  case Intrinsic::minnum: return Intrinsic::maxnum;
   default: llvm_unreachable("Unexpected intrinsic");
   }
 }
@@ -7735,12 +8140,26 @@ isImpliedCondOperands(CmpInst::Predicate Pred, const Value *ALHS,
       return true;
     return std::nullopt;
 
+  case CmpInst::ICMP_SGT:
+  case CmpInst::ICMP_SGE:
+    if (isTruePredicate(CmpInst::ICMP_SLE, ALHS, BLHS, DL, Depth) &&
+        isTruePredicate(CmpInst::ICMP_SLE, BRHS, ARHS, DL, Depth))
+      return true;
+    return std::nullopt;
+
   case CmpInst::ICMP_ULT:
   case CmpInst::ICMP_ULE:
     if (isTruePredicate(CmpInst::ICMP_ULE, BLHS, ALHS, DL, Depth) &&
         isTruePredicate(CmpInst::ICMP_ULE, ARHS, BRHS, DL, Depth))
       return true;
     return std::nullopt;
+
+  case CmpInst::ICMP_UGT:
+  case CmpInst::ICMP_UGE:
+    if (isTruePredicate(CmpInst::ICMP_ULE, ALHS, BLHS, DL, Depth) &&
+        isTruePredicate(CmpInst::ICMP_ULE, BRHS, ARHS, DL, Depth))
+      return true;
+    return std::nullopt;
   }
 }
 
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index aadecfb9e8180..423ef670bb3c2 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -444,13 +444,13 @@ void llvm::processShuffleMasks(
       int Idx = I * SzDest + K;
       if (Idx == Sz)
         break;
-      if (Mask[Idx] >= Sz || Mask[Idx] == UndefMaskElem)
+      if (Mask[Idx] >= Sz || Mask[Idx] == PoisonMaskElem)
         continue;
       int SrcRegIdx = Mask[Idx] / SzSrc;
       // Add a cost of PermuteTwoSrc for each new source register permute,
       // if we have more than one source registers.
       if (RegMasks[SrcRegIdx].empty())
-        RegMasks[SrcRegIdx].assign(SzDest, UndefMaskElem);
+        RegMasks[SrcRegIdx].assign(SzDest, PoisonMaskElem);
       RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc;
     }
   }
@@ -482,8 +482,8 @@ void llvm::processShuffleMasks(
       auto &&CombineMasks = [](MutableArrayRef<int> FirstMask,
                                ArrayRef<int> SecondMask) {
         for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) {
-          if (SecondMask[Idx] != UndefMaskElem) {
-            assert(FirstMask[Idx] == UndefMaskElem &&
+          if (SecondMask[Idx] != PoisonMaskElem) {
+            assert(FirstMask[Idx] == PoisonMaskElem &&
                    "Expected undefined mask element.");
             FirstMask[Idx] = SecondMask[Idx] + VF;
           }
@@ -491,7 +491,7 @@ void llvm::processShuffleMasks(
       };
       auto &&NormalizeMask = [](MutableArrayRef<int> Mask) {
         for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) {
-          if (Mask[Idx] != UndefMaskElem)
+          if (Mask[Idx] != PoisonMaskElem)
             Mask[Idx] = Idx;
         }
       };
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index f24c4101e8374..6525dfdde7056 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -799,7 +799,6 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(versions);
   KEYWORD(memProf);
   KEYWORD(notcold);
-  KEYWORD(notcoldandcold);
 
 #undef KEYWORD
 
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 8ee1b665c9b76..d64a26776b76e 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -9979,7 +9979,7 @@ bool LLParser::parseMemProfs(std::vector<MIBInfo> &MIBs) {
 }
 
 /// AllocType
-///   := ('none'|'notcold'|'cold'|'notcoldandcold')
+///   := ('none'|'notcold'|'cold')
 bool LLParser::parseAllocType(uint8_t &AllocType) {
   switch (Lex.getKind()) {
   case lltok::kw_none:
@@ -9991,10 +9991,6 @@ bool LLParser::parseAllocType(uint8_t &AllocType) {
   case lltok::kw_cold:
     AllocType = (uint8_t)AllocationType::Cold;
     break;
-  case lltok::kw_notcoldandcold:
-    AllocType =
-        (uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold;
-    break;
   default:
     return error(Lex.getLoc(), "invalid alloc type");
   }
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index bf05ad6757f4b..c9c437282b851 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -31,7 +32,6 @@
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <utility>
diff --git a/llvm/lib/CodeGen/AsmPrinter/AddressPool.cpp b/llvm/lib/CodeGen/AsmPrinter/AddressPool.cpp
index 32d8dc7935103..00ee4e1b47a82 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AddressPool.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AddressPool.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include <utility>
@@ -24,7 +25,7 @@ unsigned AddressPool::getIndex(const MCSymbol *Sym, bool TLS) {
 }
 
 MCSymbol *AddressPool::emitHeader(AsmPrinter &Asm, MCSection *Section) {
-  static const uint8_t AddrSize = Asm.getDataLayout().getPointerSize();
+  static const uint8_t AddrSize = Asm.MAI->getCodePointerSize();
 
   MCSymbol *EndLabel =
       Asm.emitDwarfUnitLength("debug_addr", "Length of contribution");
@@ -65,7 +66,7 @@ void AddressPool::emit(AsmPrinter &Asm, MCSection *AddrSection) {
             : MCSymbolRefExpr::create(I.first, Asm.OutContext);
 
   for (const MCExpr *Entry : Entries)
-    Asm.OutStreamer->emitValue(Entry, Asm.getDataLayout().getPointerSize());
+    Asm.OutStreamer->emitValue(Entry, Asm.MAI->getCodePointerSize());
 
   if (EndLabel)
     Asm.OutStreamer->emitLabel(EndLabel);
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 0a803d8a1405b..e7d67e1cdfe44 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -4144,7 +4144,7 @@ unsigned int AsmPrinter::getDwarfOffsetByteSize() const {
 }
 
 dwarf::FormParams AsmPrinter::getDwarfFormParams() const {
-  return {getDwarfVersion(), uint8_t(getPointerSize()),
+  return {getDwarfVersion(), uint8_t(MAI->getCodePointerSize()),
           OutStreamer->getContext().getDwarfFormat(),
           doesDwarfUseRelocationsAcrossSections()};
 }
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index e330c57d341a7..1d5ddef173383 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -108,7 +108,7 @@ unsigned AsmPrinter::GetSizeOfEncodedValue(unsigned Encoding) const {
   default:
     llvm_unreachable("Invalid encoded value.");
   case dwarf::DW_EH_PE_absptr:
-    return MF->getDataLayout().getPointerSize();
+    return MAI->getCodePointerSize();
   case dwarf::DW_EH_PE_udata2:
     return 2;
   case dwarf::DW_EH_PE_udata4:
diff --git a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
index 4e0b7a919445b..0fe4b905831ff 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
+++ b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
@@ -34,6 +34,7 @@ add_llvm_component_library(LLVMAsmPrinter
   Analysis
   BinaryFormat
   CodeGen
+  CodeGenTypes
   Core
   DebugInfoCodeView
   DebugInfoDWARF
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index 3494ae0490a1f..8a1218ae6d200 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -1160,7 +1160,14 @@ void CodeViewDebug::emitDebugInfoForFunction(const Function *GV,
     OS.AddComment("Function section index");
     OS.emitCOFFSectionIndex(Fn);
     OS.AddComment("Flags");
-    OS.emitInt8(0);
+    ProcSymFlags ProcFlags = ProcSymFlags::HasOptimizedDebugInfo;
+    if (FI.HasFramePointer)
+      ProcFlags |= ProcSymFlags::HasFP;
+    if (GV->hasFnAttribute(Attribute::NoReturn))
+      ProcFlags |= ProcSymFlags::IsNoReturn;
+    if (GV->hasFnAttribute(Attribute::NoInline))
+      ProcFlags |= ProcSymFlags::IsNoInline;
+    OS.emitInt8(static_cast<uint8_t>(ProcFlags));
     // Emit the function display name as a null-terminated string.
     OS.AddComment("Function name");
     // Truncate the name so we won't overflow the record length field.
@@ -1478,6 +1485,7 @@ void CodeViewDebug::beginFunctionImpl(const MachineFunction *MF) {
       CurFn->EncodedLocalFramePtrReg = EncodedFramePtrReg::StackPtr;
       CurFn->EncodedParamFramePtrReg = EncodedFramePtrReg::StackPtr;
     } else {
+      CurFn->HasFramePointer = true;
       // If there is an FP, parameters are always relative to it.
       CurFn->EncodedParamFramePtrReg = EncodedFramePtrReg::FramePtr;
       if (CurFn->HasStackRealignment) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index 495822a6e6532..29445b31e7e74 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -191,6 +191,8 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
     bool HasStackRealignment = false;
 
     bool HaveLineInfo = false;
+
+    bool HasFramePointer = false;
   };
   FunctionInfo *CurFn = nullptr;
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DebugLocEntry.h b/llvm/lib/CodeGen/AsmPrinter/DebugLocEntry.h
index 2008aa39ff87f..726aba18bb804 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DebugLocEntry.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DebugLocEntry.h
@@ -126,7 +126,7 @@ class DbgValueLoc {
       : Expression(Expr), ValueLocEntries(Locs.begin(), Locs.end()),
         IsVariadic(IsVariadic) {
 #ifndef NDEBUG
-    assert(cast<DIExpression>(Expr)->isValid() ||
+    assert(Expr->isValid() ||
            !any_of(Locs, [](auto LE) { return LE.isLocation(); }));
     if (!IsVariadic) {
       assert(ValueLocEntries.size() == 1);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index f0c24f2580088..dd5dbf423fb6d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/GlobalVariable.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
@@ -266,7 +267,7 @@ void DwarfCompileUnit::addLocationAttribute(
       // 16-bit platforms like MSP430 and AVR take this path, so sink this
       // assert to platforms that use it.
       auto GetPointerSizedFormAndOp = [this]() {
-        unsigned PointerSize = Asm->getDataLayout().getPointerSize();
+        unsigned PointerSize = Asm->MAI->getCodePointerSize();
         assert((PointerSize == 4 || PointerSize == 8) &&
                "Add support for other sizes if necessary");
         struct FormAndOp {
diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp
index cd094d6d89c50..5ef850d09d925 100644
--- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp
+++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp
@@ -2011,16 +2011,17 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares(
     }
   }
 
-  // Sort the fragment map for each DebugAggregate in non-descending
-  // order of fragment size. Assert no entries are duplicates.
+  // Sort the fragment map for each DebugAggregate in ascending
+  // order of fragment size - there should be no duplicates.
   for (auto &Pair : FragmentMap) {
     SmallVector<DebugVariable, 8> &Frags = Pair.second;
-    std::sort(
-        Frags.begin(), Frags.end(), [](DebugVariable Next, DebugVariable Elmt) {
-          assert(!(Elmt.getFragmentOrDefault() == Next.getFragmentOrDefault()));
-          return Elmt.getFragmentOrDefault().SizeInBits >
-                 Next.getFragmentOrDefault().SizeInBits;
-        });
+    std::sort(Frags.begin(), Frags.end(),
+              [](const DebugVariable &Next, const DebugVariable &Elmt) {
+                return Elmt.getFragmentOrDefault().SizeInBits >
+                       Next.getFragmentOrDefault().SizeInBits;
+              });
+    // Check for duplicates.
+    assert(std::adjacent_find(Frags.begin(), Frags.end()) == Frags.end());
   }
 
   // Build the map.
@@ -2193,8 +2194,7 @@ bool AssignmentTrackingLowering::run(FunctionVarLocsBuilder *FnVarLocsBuilder) {
       //
       // Unless we've already done so, create the single location def now.
       if (AlwaysStackHomed.insert(Aggr).second) {
-        assert(!VarLoc.Values.hasArgList() &&
-               isa<AllocaInst>(VarLoc.Values.getVariableLocationOp(0)));
+        assert(!VarLoc.Values.hasArgList());
         // TODO: When more complex cases are handled VarLoc.Expr should be
         // built appropriately rather than always using an empty DIExpression.
         // The assert below is a reminder.
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 1b78f5097d073..a5302a66d21e5 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -860,6 +860,14 @@ void BranchFolder::mergeCommonTails(unsigned commonTailIndex) {
       for (Register Reg : NewLiveIns) {
         if (!LiveRegs.available(*MRI, Reg))
           continue;
+
+        // Skip the register if we are about to add one of its super registers.
+        // TODO: Common this up with the same logic in addLineIns().
+        if (any_of(TRI->superregs(Reg), [&](MCPhysReg SReg) {
+              return NewLiveIns.contains(SReg) && !MRI->isReserved(SReg);
+            }))
+          continue;
+
         DebugLoc DL;
         BuildMI(*Pred, InsertBefore, DL, TII->get(TargetOpcode::IMPLICIT_DEF),
                 Reg);
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 7a3a82b10c5c8..4c4a0ea24cde8 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -22,6 +22,19 @@ if (DEFINED LLVM_HAVE_TF_AOT OR LLVM_HAVE_TFLITE)
   endif()
 endif()
 
+# This provides the implementation of MVT and LLT.
+# Be careful to append deps on this, since Targets' tablegens depend on this.
+add_llvm_component_library(LLVMCodeGenTypes
+  LowLevelType.cpp
+  PARTIAL_SOURCES_INTENDED
+
+  DEPENDS
+  vt_gen
+
+  LINK_COMPONENTS
+  Support
+  )
+
 add_llvm_component_library(LLVMCodeGen
   AggressiveAntiDepBreaker.cpp
   AllocationOrder.cpp
@@ -99,7 +112,7 @@ add_llvm_component_library(LLVMCodeGen
   LLVMTargetMachine.cpp
   LocalStackSlotAllocation.cpp
   LoopTraversal.cpp
-  LowLevelType.cpp
+  LowLevelTypeUtils.cpp
   LowerEmuTLS.cpp
   MachineBasicBlock.cpp
   MachineBlockFrequencyInfo.cpp
@@ -261,6 +274,7 @@ add_llvm_component_library(LLVMCodeGen
   Analysis
   BitReader
   BitWriter
+  CodeGenTypes
   Core
   MC
   ObjCARC
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 96fe94f8e02a1..f9e946b9de3a3 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -33,6 +33,7 @@
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
@@ -82,7 +83,6 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -268,10 +268,6 @@ static cl::opt<unsigned>
     MaxAddressUsersToScan("cgp-max-address-users-to-scan", cl::init(100),
                           cl::Hidden,
                           cl::desc("Max number of address users to look at"));
-
-static cl::opt<unsigned> MaxLoopInvUsersToScan(
-    "cgp-max-loop-inv-users-to-scan", cl::init(20), cl::Hidden,
-    cl::desc("Max number of loop invariant users to look at"));
 namespace {
 
 enum ExtType {
@@ -5064,19 +5060,6 @@ static bool FindAllMemoryUses(
                            PSI, BFI, SeenInsts);
 }
 
-static bool isUsedInLoop(const Value *V, const Loop *L) {
-  unsigned N = 0;
-
-  for (const Use &U : V->uses()) {
-    if (++N > MaxLoopInvUsersToScan)
-      break;
-    const Instruction *UserI = cast<Instruction>(U.getUser());
-    if (L->contains(UserI->getParent()))
-      return true;
-  }
-
-  return false;
-}
 
 /// Return true if Val is already known to be live at the use site that we're
 /// folding it into. If so, there is no cost to include it in the addressing
@@ -5100,17 +5083,10 @@ bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,
     if (AI->isStaticAlloca())
       return true;
 
-  // If the value is loop invariant and is used in the loop which contains the
-  // memory instruction, it's live.
-  BasicBlock *BB = MemoryInst->getParent();
-  if (Loop *L = LI.getLoopFor(BB);
-      L && L->isLoopInvariant(Val) && isUsedInLoop(Val, L))
-    return true;
-
   // Check to see if this value is already used in the memory instruction's
   // block.  If so, it's already live into the block at the very least, so we
   // can reasonably fold it.
-  return Val->isUsedInBasicBlock(BB);
+  return Val->isUsedInBasicBlock(MemoryInst->getParent());
 }
 
 /// It is possible for the addressing mode of the machine to fold the specified
@@ -6912,9 +6888,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
     return false;
 
   TargetLowering::SelectSupportKind SelectKind;
-  if (VectorCond)
-    SelectKind = TargetLowering::VectorMaskSelect;
-  else if (SI->getType()->isVectorTy())
+  if (SI->getType()->isVectorTy())
     SelectKind = TargetLowering::ScalarCondVectorVal;
   else
     SelectKind = TargetLowering::ScalarValSelect;
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index 6dafe23955c62..35263eabfc4f9 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -241,14 +241,15 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
       cl::init(false));
   CGBINDOPT(EnableNoTrappingFPMath);
 
-  static const auto DenormFlagEnumOptions =
-  cl::values(clEnumValN(DenormalMode::IEEE, "ieee",
-                        "IEEE 754 denormal numbers"),
-             clEnumValN(DenormalMode::PreserveSign, "preserve-sign",
-                        "the sign of a  flushed-to-zero number is preserved "
-                        "in the sign of 0"),
-             clEnumValN(DenormalMode::PositiveZero, "positive-zero",
-                        "denormals are flushed to positive zero"));
+  static const auto DenormFlagEnumOptions = cl::values(
+      clEnumValN(DenormalMode::IEEE, "ieee", "IEEE 754 denormal numbers"),
+      clEnumValN(DenormalMode::PreserveSign, "preserve-sign",
+                 "the sign of a  flushed-to-zero number is preserved "
+                 "in the sign of 0"),
+      clEnumValN(DenormalMode::PositiveZero, "positive-zero",
+                 "denormals are flushed to positive zero"),
+      clEnumValN(DenormalMode::Dynamic, "dynamic",
+                 "denormals have unknown treatment"));
 
   // FIXME: Doesn't have way to specify separate input and output modes.
   static cl::opt<DenormalMode::DenormalModeKind> DenormalFPMath(
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index a0dfff7270595..86c3d96b1bf08 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -343,8 +343,11 @@ bool SSAIfConv::canPredicateInstrs(MachineBasicBlock *MBB) {
 // Apply predicate to all instructions in the machine block.
 void SSAIfConv::PredicateBlock(MachineBasicBlock *MBB, bool ReversePredicate) {
   auto Condition = Cond;
-  if (ReversePredicate)
-    TII->reverseBranchCondition(Condition);
+  if (ReversePredicate) {
+    bool CanRevCond = !TII->reverseBranchCondition(Condition);
+    assert(CanRevCond && "Reversed predicate is not supported");
+    (void)CanRevCond;
+  }
   // Terminators don't need to be predicated as they will be removed.
   for (MachineBasicBlock::iterator I = MBB->begin(),
                                    E = MBB->getFirstTerminator();
diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index 5ee76ff567fb7..9807be0bea39e 100644
--- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -171,6 +171,10 @@ struct CachingVPExpander {
   Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
                                            VPIntrinsic &PI);
 
+  /// Lower this VP fp call to a unpredicated fp call.
+  Value *expandPredicationToFPCall(IRBuilder<> &Builder, VPIntrinsic &PI,
+                                   unsigned UnpredicatedIntrinsicID);
+
   /// Lower this VP reduction to a call to an unpredicated reduction intrinsic.
   Value *expandPredicationInReduction(IRBuilder<> &Builder,
                                       VPReductionIntrinsic &PI);
@@ -271,6 +275,38 @@ CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
   return NewBinOp;
 }
 
+Value *CachingVPExpander::expandPredicationToFPCall(
+    IRBuilder<> &Builder, VPIntrinsic &VPI, unsigned UnpredicatedIntrinsicID) {
+  assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) &&
+         "Implicitly dropping %evl in non-speculatable operator!");
+
+  switch (UnpredicatedIntrinsicID) {
+  case Intrinsic::fabs:
+  case Intrinsic::sqrt: {
+    Value *Op0 = VPI.getOperand(0);
+    Function *Fn = Intrinsic::getDeclaration(
+        VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
+    Value *NewOp = Builder.CreateCall(Fn, {Op0}, VPI.getName());
+    replaceOperation(*NewOp, VPI);
+    return NewOp;
+  }
+  case Intrinsic::experimental_constrained_fma:
+  case Intrinsic::experimental_constrained_fmuladd: {
+    Value *Op0 = VPI.getOperand(0);
+    Value *Op1 = VPI.getOperand(1);
+    Value *Op2 = VPI.getOperand(2);
+    Function *Fn = Intrinsic::getDeclaration(
+        VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
+    Value *NewOp =
+        Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName());
+    replaceOperation(*NewOp, VPI);
+    return NewOp;
+  }
+  }
+
+  return nullptr;
+}
+
 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
                                          Type *EltTy) {
   bool Negative = false;
@@ -565,6 +601,15 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
   switch (VPI.getIntrinsicID()) {
   default:
     break;
+  case Intrinsic::vp_fneg: {
+    Value *NewNegOp = Builder.CreateFNeg(VPI.getOperand(0), VPI.getName());
+    replaceOperation(*NewNegOp, VPI);
+    return NewNegOp;  
+  }
+  case Intrinsic::vp_fabs:
+    return expandPredicationToFPCall(Builder, VPI, Intrinsic::fabs);
+  case Intrinsic::vp_sqrt:
+    return expandPredicationToFPCall(Builder, VPI, Intrinsic::sqrt);
   case Intrinsic::vp_load:
   case Intrinsic::vp_store:
   case Intrinsic::vp_gather:
@@ -572,6 +617,10 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
     return expandPredicationInMemoryIntrinsic(Builder, VPI);
   }
 
+  if (auto CID = VPI.getConstrainedIntrinsicID())
+    if (Value *Call = expandPredicationToFPCall(Builder, VPI, *CID))
+      return Call;
+
   return &VPI;
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
index 005de9bf125d3..09e2abbe89c63 100644
--- a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
+++ b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt
@@ -33,6 +33,7 @@ add_llvm_component_library(LLVMGlobalISel
   LINK_COMPONENTS
   Analysis
   CodeGen
+  CodeGenTypes
   Core
   MC
   SelectionDAG
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 733e23440a984..0ddec2599803c 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -16,7 +16,7 @@
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineInstr.h"
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 50d2b214a5813..94fd394da9aba 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -74,7 +75,6 @@
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index e0357c50e555a..3925611f14858 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -391,10 +391,12 @@ bool InlineAsmLowering::lowerInlineAsm(
         Inst.addReg(SourceRegs[0]);
       } else {
         // Otherwise, this outputs to a register (directly for C_Register /
-        // C_RegisterClass. Find a register that we can use.
+        // C_RegisterClass/C_Other.
         assert(OpInfo.ConstraintType == TargetLowering::C_Register ||
-               OpInfo.ConstraintType == TargetLowering::C_RegisterClass);
+               OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
+               OpInfo.ConstraintType == TargetLowering::C_Other);
 
+        // Find a register that we can use.
         if (OpInfo.Regs.empty()) {
           LLVM_DEBUG(dbgs()
                      << "Couldn't allocate output register for constraint\n");
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index c61ed539a543a..5c5f843764a2d 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4835,6 +4835,7 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_EXTRACT_VECTOR_ELT:
   case TargetOpcode::G_EXTRACT:
     if (TypeIdx != 1)
       return UnableToLegalize;
@@ -4843,6 +4844,7 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
     Observer.changedInstr(MI);
     return Legalized;
   case TargetOpcode::G_INSERT:
+  case TargetOpcode::G_INSERT_VECTOR_ELT:
   case TargetOpcode::G_FREEZE:
   case TargetOpcode::G_FNEG:
   case TargetOpcode::G_FABS:
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 4b6c3a1567093..1f2e481c63e0b 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -21,7 +22,6 @@
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include <algorithm>
 
 using namespace llvm;
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 12ae12ab827ad..49f40495d6fc9 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -20,7 +20,7 @@
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
diff --git a/llvm/lib/CodeGen/ImplicitNullChecks.cpp b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
index fa493af0eea72..ae002e074add2 100644
--- a/llvm/lib/CodeGen/ImplicitNullChecks.cpp
+++ b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
@@ -94,7 +94,7 @@ class ImplicitNullChecks : public MachineFunctionPass {
     /// computeDependence).
     bool CanReorder;
 
-    /// If non-None, then an instruction in \p Insts that also must be
+    /// If non-std::nullopt, then an instruction in \p Insts that also must be
     /// hoisted.
     std::optional<ArrayRef<MachineInstr *>::iterator> PotentialDependence;
 
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
index cb0008a5ce6ed..772aa4079a6d8 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
@@ -1197,7 +1197,7 @@ class InstrRefBasedLDV : public LDVImpl {
 
   /// For an instruction reference given by \p InstNo and \p OpNo in instruction
   /// \p MI returns the Value pointed to by that instruction reference if any
-  /// exists, otherwise returns None.
+  /// exists, otherwise returns std::nullopt.
   std::optional<ValueIDNum> getValueForInstrRef(unsigned InstNo, unsigned OpNo,
                                                 MachineInstr &MI,
                                                 const ValueTable *MLiveOuts,
diff --git a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
index 959fea6ccb065..c5942aeffdbbf 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
@@ -1116,7 +1116,7 @@ VarLocBasedLDV::~VarLocBasedLDV() = default;
 /// location, erase the variable from the Vars set.
 void VarLocBasedLDV::OpenRangesSet::erase(const VarLoc &VL) {
   // Erasure helper.
-  auto DoErase = [VL, this](DebugVariable VarToErase) {
+  auto DoErase = [&VL, this](DebugVariable VarToErase) {
     auto *EraseFrom = VL.isEntryBackupLoc() ? &EntryValuesBackupVars : &Vars;
     auto It = EraseFrom->find(VarToErase);
     if (It != EraseFrom->end()) {
diff --git a/llvm/lib/CodeGen/LowLevelType.cpp b/llvm/lib/CodeGen/LowLevelType.cpp
index 539f56fbc5a96..24c30b756737b 100644
--- a/llvm/lib/CodeGen/LowLevelType.cpp
+++ b/llvm/lib/CodeGen/LowLevelType.cpp
@@ -12,74 +12,55 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/LowLevelType.h"
-#include "llvm/ADT/APFloat.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DerivedTypes.h"
+#include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-LLT llvm::getLLTForType(Type &Ty, const DataLayout &DL) {
-  if (auto VTy = dyn_cast<VectorType>(&Ty)) {
-    auto EC = VTy->getElementCount();
-    LLT ScalarTy = getLLTForType(*VTy->getElementType(), DL);
-    if (EC.isScalar())
-      return ScalarTy;
-    return LLT::vector(EC, ScalarTy);
-  }
-
-  if (auto PTy = dyn_cast<PointerType>(&Ty)) {
-    unsigned AddrSpace = PTy->getAddressSpace();
-    return LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
-  }
-
-  if (Ty.isSized() && !Ty.isScalableTargetExtTy()) {
+LLT::LLT(MVT VT) {
+  if (VT.isVector()) {
+    bool asVector = VT.getVectorMinNumElements() > 1;
+    init(/*IsPointer=*/false, asVector, /*IsScalar=*/!asVector,
+         VT.getVectorElementCount(), VT.getVectorElementType().getSizeInBits(),
+         /*AddressSpace=*/0);
+  } else if (VT.isValid() && !VT.isScalableTargetExtVT()) {
     // Aggregates are no different from real scalars as far as GlobalISel is
     // concerned.
-    auto SizeInBits = DL.getTypeSizeInBits(&Ty);
-    assert(SizeInBits != 0 && "invalid zero-sized type");
-    return LLT::scalar(SizeInBits);
+    init(/*IsPointer=*/false, /*IsVector=*/false, /*IsScalar=*/true,
+         ElementCount::getFixed(0), VT.getSizeInBits(), /*AddressSpace=*/0);
+  } else {
+    IsScalar = false;
+    IsPointer = false;
+    IsVector = false;
+    RawData = 0;
   }
-
-  return LLT();
 }
 
-MVT llvm::getMVTForLLT(LLT Ty) {
-  if (!Ty.isVector())
-    return MVT::getIntegerVT(Ty.getSizeInBits());
-
-  return MVT::getVectorVT(
-      MVT::getIntegerVT(Ty.getElementType().getSizeInBits()),
-      Ty.getNumElements());
+void LLT::print(raw_ostream &OS) const {
+  if (isVector()) {
+    OS << "<";
+    OS << getElementCount() << " x " << getElementType() << ">";
+  } else if (isPointer())
+    OS << "p" << getAddressSpace();
+  else if (isValid()) {
+    assert(isScalar() && "unexpected type");
+    OS << "s" << getScalarSizeInBits();
+  } else
+    OS << "LLT_invalid";
 }
 
-EVT llvm::getApproximateEVTForLLT(LLT Ty, const DataLayout &DL,
-                                  LLVMContext &Ctx) {
-  if (Ty.isVector()) {
-    EVT EltVT = getApproximateEVTForLLT(Ty.getElementType(), DL, Ctx);
-    return EVT::getVectorVT(Ctx, EltVT, Ty.getElementCount());
-  }
-
-  return EVT::getIntegerVT(Ctx, Ty.getSizeInBits());
-}
-
-LLT llvm::getLLTForMVT(MVT Ty) {
-  if (!Ty.isVector())
-    return LLT::scalar(Ty.getSizeInBits());
-
-  return LLT::scalarOrVector(Ty.getVectorElementCount(),
-                             Ty.getVectorElementType().getSizeInBits());
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void LLT::dump() const {
+  print(dbgs());
+  dbgs() << '\n';
 }
+#endif
 
-const llvm::fltSemantics &llvm::getFltSemanticForLLT(LLT Ty) {
-  assert(Ty.isScalar() && "Expected a scalar type.");
-  switch (Ty.getSizeInBits()) {
-  case 16:
-    return APFloat::IEEEhalf();
-  case 32:
-    return APFloat::IEEEsingle();
-  case 64:
-    return APFloat::IEEEdouble();
-  case 128:
-    return APFloat::IEEEquad();
-  }
-  llvm_unreachable("Invalid FP type size.");
-}
+const constexpr LLT::BitFieldInfo LLT::ScalarSizeFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerSizeFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerAddressSpaceFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::VectorElementsFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::VectorScalableFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::VectorSizeFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerVectorElementsFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerVectorScalableFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerVectorSizeFieldInfo;
+const constexpr LLT::BitFieldInfo LLT::PointerVectorAddressSpaceFieldInfo;
diff --git a/llvm/lib/CodeGen/LowLevelTypeUtils.cpp b/llvm/lib/CodeGen/LowLevelTypeUtils.cpp
new file mode 100644
index 0000000000000..bc2ea3f05b6da
--- /dev/null
+++ b/llvm/lib/CodeGen/LowLevelTypeUtils.cpp
@@ -0,0 +1,85 @@
+//===-- llvm/CodeGen/LowLevelTypeUtils.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file This file implements the more header-heavy bits of the LLT class to
+/// avoid polluting users' namespaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+using namespace llvm;
+
+LLT llvm::getLLTForType(Type &Ty, const DataLayout &DL) {
+  if (auto VTy = dyn_cast<VectorType>(&Ty)) {
+    auto EC = VTy->getElementCount();
+    LLT ScalarTy = getLLTForType(*VTy->getElementType(), DL);
+    if (EC.isScalar())
+      return ScalarTy;
+    return LLT::vector(EC, ScalarTy);
+  }
+
+  if (auto PTy = dyn_cast<PointerType>(&Ty)) {
+    unsigned AddrSpace = PTy->getAddressSpace();
+    return LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
+  }
+
+  if (Ty.isSized() && !Ty.isScalableTargetExtTy()) {
+    // Aggregates are no different from real scalars as far as GlobalISel is
+    // concerned.
+    auto SizeInBits = DL.getTypeSizeInBits(&Ty);
+    assert(SizeInBits != 0 && "invalid zero-sized type");
+    return LLT::scalar(SizeInBits);
+  }
+
+  return LLT();
+}
+
+MVT llvm::getMVTForLLT(LLT Ty) {
+  if (!Ty.isVector())
+    return MVT::getIntegerVT(Ty.getSizeInBits());
+
+  return MVT::getVectorVT(
+      MVT::getIntegerVT(Ty.getElementType().getSizeInBits()),
+      Ty.getNumElements());
+}
+
+EVT llvm::getApproximateEVTForLLT(LLT Ty, const DataLayout &DL,
+                                  LLVMContext &Ctx) {
+  if (Ty.isVector()) {
+    EVT EltVT = getApproximateEVTForLLT(Ty.getElementType(), DL, Ctx);
+    return EVT::getVectorVT(Ctx, EltVT, Ty.getElementCount());
+  }
+
+  return EVT::getIntegerVT(Ctx, Ty.getSizeInBits());
+}
+
+LLT llvm::getLLTForMVT(MVT Ty) {
+  if (!Ty.isVector())
+    return LLT::scalar(Ty.getSizeInBits());
+
+  return LLT::scalarOrVector(Ty.getVectorElementCount(),
+                             Ty.getVectorElementType().getSizeInBits());
+}
+
+const llvm::fltSemantics &llvm::getFltSemanticForLLT(LLT Ty) {
+  assert(Ty.isScalar() && "Expected a scalar type.");
+  switch (Ty.getSizeInBits()) {
+  case 16:
+    return APFloat::IEEEhalf();
+  case 32:
+    return APFloat::IEEEsingle();
+  case 64:
+    return APFloat::IEEEdouble();
+  case 128:
+    return APFloat::IEEEquad();
+  }
+  llvm_unreachable("Invalid FP type size.");
+}
diff --git a/llvm/lib/CodeGen/MIRParser/CMakeLists.txt b/llvm/lib/CodeGen/MIRParser/CMakeLists.txt
index eba19e237b52f..8e85c0476c7ad 100644
--- a/llvm/lib/CodeGen/MIRParser/CMakeLists.txt
+++ b/llvm/lib/CodeGen/MIRParser/CMakeLists.txt
@@ -13,6 +13,7 @@ add_llvm_component_library(LLVMMIRParser
   AsmParser
   BinaryFormat
   CodeGen
+  CodeGenTypes
   Core
   MC
   Support
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index d38e0ba365802..34fb64ea66ce3 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -24,6 +24,7 @@
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/AsmParser/SlotMapping.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MIRFormatter.h"
 #include "llvm/CodeGen/MIRPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -62,7 +63,6 @@
 #include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 4fffd628f1e6e..5488824306261 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -130,6 +130,16 @@ class MIRParserImpl {
                                 const yaml::StringValue &RegisterSource,
                                 bool IsRestored, int FrameIdx);
 
+  struct VarExprLoc {
+    DILocalVariable *DIVar = nullptr;
+    DIExpression *DIExpr = nullptr;
+    DILocation *DILoc = nullptr;
+  };
+
+  std::optional<VarExprLoc> parseVarExprLoc(PerFunctionMIParsingState &PFS,
+                                            const yaml::StringValue &VarStr,
+                                            const yaml::StringValue &ExprStr,
+                                            const yaml::StringValue &LocStr);
   template <typename T>
   bool parseStackObjectsDebugInfo(PerFunctionMIParsingState &PFS,
                                   const T &Object,
@@ -888,26 +898,37 @@ static bool typecheckMDNode(T *&Result, MDNode *Node,
   return false;
 }
 
-template <typename T>
-bool MIRParserImpl::parseStackObjectsDebugInfo(PerFunctionMIParsingState &PFS,
-    const T &Object, int FrameIdx) {
-  // Debug information can only be attached to stack objects; Fixed stack
-  // objects aren't supported.
-  MDNode *Var = nullptr, *Expr = nullptr, *Loc = nullptr;
-  if (parseMDNode(PFS, Var, Object.DebugVar) ||
-      parseMDNode(PFS, Expr, Object.DebugExpr) ||
-      parseMDNode(PFS, Loc, Object.DebugLoc))
-    return true;
-  if (!Var && !Expr && !Loc)
-    return false;
+std::optional<MIRParserImpl::VarExprLoc> MIRParserImpl::parseVarExprLoc(
+    PerFunctionMIParsingState &PFS, const yaml::StringValue &VarStr,
+    const yaml::StringValue &ExprStr, const yaml::StringValue &LocStr) {
+  MDNode *Var = nullptr;
+  MDNode *Expr = nullptr;
+  MDNode *Loc = nullptr;
+  if (parseMDNode(PFS, Var, VarStr) || parseMDNode(PFS, Expr, ExprStr) ||
+      parseMDNode(PFS, Loc, LocStr))
+    return std::nullopt;
   DILocalVariable *DIVar = nullptr;
   DIExpression *DIExpr = nullptr;
   DILocation *DILoc = nullptr;
-  if (typecheckMDNode(DIVar, Var, Object.DebugVar, "DILocalVariable", *this) ||
-      typecheckMDNode(DIExpr, Expr, Object.DebugExpr, "DIExpression", *this) ||
-      typecheckMDNode(DILoc, Loc, Object.DebugLoc, "DILocation", *this))
+  if (typecheckMDNode(DIVar, Var, VarStr, "DILocalVariable", *this) ||
+      typecheckMDNode(DIExpr, Expr, ExprStr, "DIExpression", *this) ||
+      typecheckMDNode(DILoc, Loc, LocStr, "DILocation", *this))
+    return std::nullopt;
+  return VarExprLoc{DIVar, DIExpr, DILoc};
+}
+
+template <typename T>
+bool MIRParserImpl::parseStackObjectsDebugInfo(PerFunctionMIParsingState &PFS,
+                                               const T &Object, int FrameIdx) {
+  std::optional<VarExprLoc> MaybeInfo =
+      parseVarExprLoc(PFS, Object.DebugVar, Object.DebugExpr, Object.DebugLoc);
+  if (!MaybeInfo)
     return true;
-  PFS.MF.setVariableDbgInfo(DIVar, DIExpr, FrameIdx, DILoc);
+  // Debug information can only be attached to stack objects; Fixed stack
+  // objects aren't supported.
+  if (MaybeInfo->DIVar || MaybeInfo->DIExpr || MaybeInfo->DILoc)
+    PFS.MF.setVariableDbgInfo(MaybeInfo->DIVar, MaybeInfo->DIExpr, FrameIdx,
+                              MaybeInfo->DILoc);
   return false;
 }
 
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index e2031e12edbea..4b2fdfb356cad 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MIRYamlMapping.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -47,7 +48,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index bb4d1316dd252..ec5eb71205cf1 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -149,7 +149,8 @@ void MachineCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
-MachineInstr *MachineCombiner::getOperandDef(const MachineOperand &MO) {
+MachineInstr *
+MachineCombiner::getOperandDef(const MachineOperand &MO) {
   MachineInstr *DefInstr = nullptr;
   // We need a virtual register definition.
   if (MO.isReg() && MO.getReg().isVirtual())
@@ -404,8 +405,13 @@ bool MachineCombiner::improvesCriticalPathLen(
 
   // Account for the latency of the inserted and deleted instructions by
   unsigned NewRootLatency, RootLatency;
-  std::tie(NewRootLatency, RootLatency) =
-      getLatenciesForInstrSequences(*Root, InsInstrs, DelInstrs, BlockTrace);
+  if (TII->accumulateInstrSeqToRootLatency(*Root)) {
+    std::tie(NewRootLatency, RootLatency) =
+        getLatenciesForInstrSequences(*Root, InsInstrs, DelInstrs, BlockTrace);
+  } else {
+    NewRootLatency = TSchedModel.computeInstrLatency(InsInstrs.back());
+    RootLatency = TSchedModel.computeInstrLatency(Root);
+  }
 
   unsigned RootSlack = BlockTrace.getInstrSlack(*Root);
   unsigned NewCycleCount = NewRootDepth + NewRootLatency;
@@ -496,8 +502,8 @@ bool MachineCombiner::preservesResourceLen(
 ///                          otherwise invalidate the trace
 static void insertDeleteInstructions(
     MachineBasicBlock *MBB, MachineInstr &MI,
-    SmallVector<MachineInstr *, 16> InsInstrs,
-    SmallVector<MachineInstr *, 16> DelInstrs,
+    SmallVectorImpl<MachineInstr *> &InsInstrs,
+    SmallVectorImpl<MachineInstr *> &DelInstrs,
     MachineTraceMetrics::Ensemble *TraceEnsemble,
     SparseSet<LiveRegUnit> &RegUnits, const TargetInstrInfo *TII,
     MachineCombinerPattern Pattern, bool IncrementalUpdate) {
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 5617cb0369e0e..0b5e3e8335282 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -50,7 +51,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormattedStream.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include <algorithm>
@@ -192,6 +192,7 @@ static void moveOperands(MachineOperand *Dst, MachineOperand *Src,
 /// an explicit operand it is added at the end of the explicit operand list
 /// (before the first implicit operand).
 void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) {
+  assert(NumOperands < USHRT_MAX && "Cannot add more operands.");
   assert(MCID && "Cannot add operands before providing an instr descriptor");
 
   // Check if we're adding one of our existing operands.
diff --git a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
index f69febadcd77f..2dc289014200d 100644
--- a/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
+++ b/llvm/lib/CodeGen/MachineLateInstrsCleanup.cpp
@@ -175,7 +175,8 @@ bool MachineLateInstrsCleanup::processBlock(MachineBasicBlock *MBB) {
   Reg2DefMap &MBBDefs = RegDefs[MBB->getNumber()];
 
   // Find reusable definitions in the predecessor(s).
-  if (!MBB->pred_empty() && !MBB->isEHPad()) {
+  if (!MBB->pred_empty() && !MBB->isEHPad() &&
+      !MBB->isInlineAsmBrIndirectTarget()) {
     MachineBasicBlock *FirstPred = *MBB->pred_begin();
     for (auto [Reg, DefMI] : RegDefs[FirstPred->getNumber()])
       if (llvm::all_of(
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 30261fa5eb889..3fa35d7c43320 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -32,6 +32,7 @@
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachinePassRegistry.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/RegisterPressure.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
@@ -56,7 +57,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/GraphWriter.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index fb78c4ce1242e..d6a323db250e6 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -37,6 +37,7 @@
 #include "llvm/CodeGen/LiveRangeCalc.h"
 #include "llvm/CodeGen/LiveStacks.h"
 #include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -71,7 +72,6 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ModRef.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index f60e17679f653..17df5d6db90b3 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -106,7 +106,7 @@ namespace {
       }
     };
 
-    using LiveRegMap = SparseSet<LiveReg>;
+    using LiveRegMap = SparseSet<LiveReg, identity<unsigned>, uint16_t>;
     /// This map contains entries for each virtual register that is currently
     /// available in a physical register.
     LiveRegMap LiveVirtRegs;
diff --git a/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt b/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt
index 9617ff3187dfc..cbfbfa3a321bc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt
+++ b/llvm/lib/CodeGen/SelectionDAG/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_component_library(LLVMSelectionDAG
   LINK_COMPONENTS
   Analysis
   CodeGen
+  CodeGenTypes
   Core
   MC
   Support
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ade34f70c9768..b489321365d55 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -36,6 +36,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
@@ -58,7 +59,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -423,11 +423,12 @@ namespace {
     SDValue visitSUBC(SDNode *N);
     SDValue visitSUBO(SDNode *N);
     SDValue visitADDE(SDNode *N);
-    SDValue visitADDCARRY(SDNode *N);
+    SDValue visitUADDO_CARRY(SDNode *N);
     SDValue visitSADDO_CARRY(SDNode *N);
-    SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
+    SDValue visitUADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
+                                 SDNode *N);
     SDValue visitSUBE(SDNode *N);
-    SDValue visitSUBCARRY(SDNode *N);
+    SDValue visitUSUBO_CARRY(SDNode *N);
     SDValue visitSSUBO_CARRY(SDNode *N);
     SDValue visitMUL(SDNode *N);
     SDValue visitMULFIX(SDNode *N);
@@ -484,6 +485,7 @@ namespace {
     SDValue visitBUILD_PAIR(SDNode *N);
     SDValue visitFADD(SDNode *N);
     SDValue visitVP_FADD(SDNode *N);
+    SDValue visitVP_FSUB(SDNode *N);
     SDValue visitSTRICT_FADD(SDNode *N);
     SDValue visitFSUB(SDNode *N);
     SDValue visitFMUL(SDNode *N);
@@ -538,6 +540,7 @@ namespace {
 
     template <class MatchContextClass>
     SDValue visitFADDForFMACombine(SDNode *N);
+    template <class MatchContextClass>
     SDValue visitFSUBForFMACombine(SDNode *N);
     SDValue visitFMULForFMADistributiveCombine(SDNode *N);
 
@@ -1876,10 +1879,10 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::SSUBO:
   case ISD::USUBO:              return visitSUBO(N);
   case ISD::ADDE:               return visitADDE(N);
-  case ISD::ADDCARRY:           return visitADDCARRY(N);
+  case ISD::UADDO_CARRY:        return visitUADDO_CARRY(N);
   case ISD::SADDO_CARRY:        return visitSADDO_CARRY(N);
   case ISD::SUBE:               return visitSUBE(N);
-  case ISD::SUBCARRY:           return visitSUBCARRY(N);
+  case ISD::USUBO_CARRY:        return visitUSUBO_CARRY(N);
   case ISD::SSUBO_CARRY:        return visitSSUBO_CARRY(N);
   case ISD::SMULFIX:
   case ISD::SMULFIXSAT:
@@ -2976,7 +2979,7 @@ static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
   if (V.getResNo() != 1)
     return SDValue();
 
-  if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
+  if (V.getOpcode() != ISD::UADDO_CARRY && V.getOpcode() != ISD::USUBO_CARRY &&
       V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
     return SDValue();
 
@@ -3098,16 +3101,16 @@ SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
     }
   }
 
-  // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
-  if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
+  // (add X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
+  if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1)) &&
       N1.getResNo() == 0)
-    return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
+    return DAG.getNode(ISD::UADDO_CARRY, DL, N1->getVTList(),
                        N0, N1.getOperand(0), N1.getOperand(2));
 
-  // (add X, Carry) -> (addcarry X, 0, Carry)
-  if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
+  // (add X, Carry) -> (uaddo_carry X, 0, Carry)
+  if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT))
     if (SDValue Carry = getAsCarry(TLI, N1))
-      return DAG.getNode(ISD::ADDCARRY, DL,
+      return DAG.getNode(ISD::UADDO_CARRY, DL,
                          DAG.getVTList(VT, Carry.getValueType()), N0,
                          DAG.getConstant(0, DL, VT), Carry);
 
@@ -3238,20 +3241,20 @@ SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
   if (VT.isVector())
     return SDValue();
 
-  // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
+  // (uaddo X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
   // If Y + 1 cannot overflow.
-  if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
+  if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1))) {
     SDValue Y = N1.getOperand(0);
     SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
     if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
-      return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
+      return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y,
                          N1.getOperand(2));
   }
 
-  // (uaddo X, Carry) -> (addcarry X, 0, Carry)
-  if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
+  // (uaddo X, Carry) -> (uaddo_carry X, 0, Carry)
+  if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT))
     if (SDValue Carry = getAsCarry(TLI, N1))
-      return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
+      return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0,
                          DAG.getConstant(0, SDLoc(N), VT), Carry);
 
   return SDValue();
@@ -3276,7 +3279,7 @@ SDValue DAGCombiner::visitADDE(SDNode *N) {
   return SDValue();
 }
 
-SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
+SDValue DAGCombiner::visitUADDO_CARRY(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDValue CarryIn = N->getOperand(2);
@@ -3286,16 +3289,16 @@ SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
   ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
   if (N0C && !N1C)
-    return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
+    return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
 
-  // fold (addcarry x, y, false) -> (uaddo x, y)
+  // fold (uaddo_carry x, y, false) -> (uaddo x, y)
   if (isNullConstant(CarryIn)) {
     if (!LegalOperations ||
         TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
       return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
   }
 
-  // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
+  // fold (uaddo_carry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
   if (isNullConstant(N0) && isNullConstant(N1)) {
     EVT VT = N0.getValueType();
     EVT CarryVT = CarryIn.getValueType();
@@ -3306,20 +3309,20 @@ SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
                      DAG.getConstant(0, DL, CarryVT));
   }
 
-  if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
+  if (SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn, N))
     return Combined;
 
-  if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
+  if (SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn, N))
     return Combined;
 
   // We want to avoid useless duplication.
-  // TODO: This is done automatically for binary operations. As ADDCARRY is
+  // TODO: This is done automatically for binary operations. As UADDO_CARRY is
   // not a binary operation, this is not really possible to leverage this
   // existing mechanism for it. However, if more operations require the same
   // deduplication logic, then it may be worth generalize.
   SDValue Ops[] = {N1, N0, CarryIn};
   SDNode *CSENode =
-      DAG.getNodeIfExists(ISD::ADDCARRY, N->getVTList(), Ops, N->getFlags());
+      DAG.getNodeIfExists(ISD::UADDO_CARRY, N->getVTList(), Ops, N->getFlags());
   if (CSENode)
     return SDValue(CSENode, 0);
 
@@ -3351,28 +3354,29 @@ SDValue DAGCombiner::visitSADDO_CARRY(SDNode *N) {
 /**
  * If we are facing some sort of diamond carry propapagtion pattern try to
  * break it up to generate something like:
- *   (addcarry X, 0, (addcarry A, B, Z):Carry)
+ *   (uaddo_carry X, 0, (uaddo_carry A, B, Z):Carry)
  *
  * The end result is usually an increase in operation required, but because the
  * carry is now linearized, other transforms can kick in and optimize the DAG.
  *
  * Patterns typically look something like
- *            (uaddo A, B)
- *             /       \
- *          Carry      Sum
- *            |          \
- *            | (addcarry *, 0, Z)
- *            |       /
- *             \   Carry
- *              |   /
- * (addcarry X, *, *)
+ *                (uaddo A, B)
+ *                /          \
+ *             Carry         Sum
+ *               |             \
+ *               | (uaddo_carry *, 0, Z)
+ *               |       /
+ *                \   Carry
+ *                 |   /
+ * (uaddo_carry X, *, *)
  *
  * But numerous variation exist. Our goal is to identify A, B, X and Z and
  * produce a combine with a single path for carry propagation.
  */
-static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
-                                      SDValue X, SDValue Carry0, SDValue Carry1,
-                                      SDNode *N) {
+static SDValue combineUADDO_CARRYDiamond(DAGCombiner &Combiner,
+                                         SelectionDAG &DAG, SDValue X,
+                                         SDValue Carry0, SDValue Carry1,
+                                         SDNode *N) {
   if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
     return SDValue();
   if (Carry1.getOpcode() != ISD::UADDO)
@@ -3382,9 +3386,9 @@ static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
 
   /**
    * First look for a suitable Z. It will present itself in the form of
-   * (addcarry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
+   * (uaddo_carry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
    */
-  if (Carry0.getOpcode() == ISD::ADDCARRY &&
+  if (Carry0.getOpcode() == ISD::UADDO_CARRY &&
       isNullConstant(Carry0.getOperand(1))) {
     Z = Carry0.getOperand(2);
   } else if (Carry0.getOpcode() == ISD::UADDO &&
@@ -3399,26 +3403,27 @@ static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
 
   auto cancelDiamond = [&](SDValue A,SDValue B) {
     SDLoc DL(N);
-    SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
+    SDValue NewY =
+        DAG.getNode(ISD::UADDO_CARRY, DL, Carry0->getVTList(), A, B, Z);
     Combiner.AddToWorklist(NewY.getNode());
-    return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X,
+    return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), X,
                        DAG.getConstant(0, DL, X.getValueType()),
                        NewY.getValue(1));
   };
 
   /**
-   *      (uaddo A, B)
-   *           |
-   *          Sum
-   *           |
-   * (addcarry *, 0, Z)
+   *         (uaddo A, B)
+   *              |
+   *             Sum
+   *              |
+   * (uaddo_carry *, 0, Z)
    */
   if (Carry0.getOperand(0) == Carry1.getValue(0)) {
     return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1));
   }
 
   /**
-   * (addcarry A, 0, Z)
+   * (uaddo_carry A, 0, Z)
    *         |
    *        Sum
    *         |
@@ -3455,12 +3460,12 @@ static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
 //                  |   /
 //   CarryOut = (or *, *)
 //
-// And generate ADDCARRY (or SUBCARRY) with two result values:
+// And generate UADDO_CARRY (or USUBO_CARRY) with two result values:
 //
-//    {AddCarrySum, CarryOut} = (addcarry A, B, CarryIn)
+//    {AddCarrySum, CarryOut} = (uaddo_carry A, B, CarryIn)
 //
-// Our goal is to identify A, B, and CarryIn and produce ADDCARRY/SUBCARRY with
-// a single path for carry/borrow out propagation:
+// Our goal is to identify A, B, and CarryIn and produce UADDO_CARRY/USUBO_CARRY
+// with a single path for carry/borrow out propagation.
 static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
                                    SDValue N0, SDValue N1, SDNode *N) {
   SDValue Carry0 = getAsCarry(TLI, N0);
@@ -3493,7 +3498,7 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
     return SDValue();
   SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
 
-  unsigned NewOp = Opcode == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
+  unsigned NewOp = Opcode == ISD::UADDO ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
   if (!TLI.isOperationLegalOrCustom(NewOp, Carry0.getValue(0).getValueType()))
     return SDValue();
 
@@ -3529,39 +3534,40 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
   return Merged.getValue(1);
 }
 
-SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
-                                       SDNode *N) {
-  // fold (addcarry (xor a, -1), b, c) -> (subcarry b, a, !c) and flip carry.
+SDValue DAGCombiner::visitUADDO_CARRYLike(SDValue N0, SDValue N1,
+                                          SDValue CarryIn, SDNode *N) {
+  // fold (uaddo_carry (xor a, -1), b, c) -> (usubo_carry b, a, !c) and flip
+  // carry.
   if (isBitwiseNot(N0))
     if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
       SDLoc DL(N);
-      SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
+      SDValue Sub = DAG.getNode(ISD::USUBO_CARRY, DL, N->getVTList(), N1,
                                 N0.getOperand(0), NotC);
       return CombineTo(
           N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
     }
 
   // Iff the flag result is dead:
-  // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
+  // (uaddo_carry (add|uaddo X, Y), 0, Carry) -> (uaddo_carry X, Y, Carry)
   // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
   // or the dependency between the instructions.
   if ((N0.getOpcode() == ISD::ADD ||
        (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
         N0.getValue(1) != CarryIn)) &&
       isNullConstant(N1) && !N->hasAnyUseOfValue(1))
-    return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
+    return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(),
                        N0.getOperand(0), N0.getOperand(1), CarryIn);
 
   /**
-   * When one of the addcarry argument is itself a carry, we may be facing
+   * When one of the uaddo_carry argument is itself a carry, we may be facing
    * a diamond carry propagation. In which case we try to transform the DAG
    * to ensure linear carry propagation if that is possible.
    */
   if (auto Y = getAsCarry(TLI, N1)) {
     // Because both are carries, Y and Z can be swapped.
-    if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
+    if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
       return R;
-    if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
+    if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
       return R;
   }
 
@@ -3985,19 +3991,19 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
       return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
   }
 
-  // (sub (subcarry X, 0, Carry), Y) -> (subcarry X, Y, Carry)
-  if (N0.getOpcode() == ISD::SUBCARRY && isNullConstant(N0.getOperand(1)) &&
+  // (sub (usubo_carry X, 0, Carry), Y) -> (usubo_carry X, Y, Carry)
+  if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(N0.getOperand(1)) &&
       N0.getResNo() == 0 && N0.hasOneUse())
-    return DAG.getNode(ISD::SUBCARRY, DL, N0->getVTList(),
+    return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(),
                        N0.getOperand(0), N1, N0.getOperand(2));
 
-  if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) {
-    // (sub Carry, X)  ->  (addcarry (sub 0, X), 0, Carry)
+  if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) {
+    // (sub Carry, X)  ->  (uaddo_carry (sub 0, X), 0, Carry)
     if (SDValue Carry = getAsCarry(TLI, N0)) {
       SDValue X = N1;
       SDValue Zero = DAG.getConstant(0, DL, VT);
       SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
-      return DAG.getNode(ISD::ADDCARRY, DL,
+      return DAG.getNode(ISD::UADDO_CARRY, DL,
                          DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
                          Carry);
     }
@@ -4149,12 +4155,12 @@ SDValue DAGCombiner::visitSUBE(SDNode *N) {
   return SDValue();
 }
 
-SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
+SDValue DAGCombiner::visitUSUBO_CARRY(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDValue CarryIn = N->getOperand(2);
 
-  // fold (subcarry x, y, false) -> (usubo x, y)
+  // fold (usubo_carry x, y, false) -> (usubo x, y)
   if (isNullConstant(CarryIn)) {
     if (!LegalOperations ||
         TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
@@ -14549,12 +14555,13 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
     }
     break;
   case ISD::ADDE:
-  case ISD::ADDCARRY:
+  case ISD::UADDO_CARRY:
     // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
-    // (trunc addcarry(X, Y, Carry)) -> (addcarry trunc(X), trunc(Y), Carry)
+    // (trunc uaddo_carry(X, Y, Carry)) ->
+    //     (uaddo_carry trunc(X), trunc(Y), Carry)
     // When the adde's carry is not used.
-    // We only do for addcarry before legalize operation
-    if (((!LegalOperations && N0.getOpcode() == ISD::ADDCARRY) ||
+    // We only do for uaddo_carry before legalize operation
+    if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) ||
          TLI.isOperationLegal(N0.getOpcode(), VT)) &&
         N0.hasOneUse() && !N0->hasAnyUseOfValue(1)) {
       SDLoc DL(N);
@@ -15226,10 +15233,10 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
     SDValue TmpFMA = FMA;
     while (E && isFusedOp(TmpFMA) && TmpFMA.hasOneUse()) {
       SDValue FMul = TmpFMA->getOperand(2);
-      if (FMul.getOpcode() == ISD::FMUL && FMul.hasOneUse()) {
+      if (matcher.match(FMul, ISD::FMUL) && FMul.hasOneUse()) {
         SDValue C = FMul.getOperand(0);
         SDValue D = FMul.getOperand(1);
-        SDValue CDE = DAG.getNode(PreferredFusedOpcode, SL, VT, C, D, E);
+        SDValue CDE = matcher.getNode(PreferredFusedOpcode, SL, VT, C, D, E);
         DAG.ReplaceAllUsesOfValueWith(FMul, CDE);
         // Replacing the inner FMul could cause the outer FMA to be simplified
         // away.
@@ -15364,20 +15371,26 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
 }
 
 /// Try to perform FMA combining on a given FSUB node.
+template <class MatchContextClass>
 SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   EVT VT = N->getValueType(0);
   SDLoc SL(N);
-
+  MatchContextClass matcher(DAG, TLI, N);
   const TargetOptions &Options = DAG.getTarget().Options;
+
+  bool UseVP = std::is_same_v<MatchContextClass, VPMatchContext>;
+
   // Floating-point multiply-add with intermediate rounding.
-  bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
+  // FIXME: Make isFMADLegal have specific behavior when using VPMatchContext.
+  // FIXME: Add VP_FMAD opcode.
+  bool HasFMAD = !UseVP && (LegalOperations && TLI.isFMADLegal(DAG, N));
 
   // Floating-point multiply-add without intermediate rounding.
   bool HasFMA =
       TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) &&
-      (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT));
+      (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT));
 
   // No valid opcode, do not combine.
   if (!HasFMAD && !HasFMA)
@@ -15401,8 +15414,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
 
   // Is the node an FMUL and contractable either due to global flags or
   // SDNodeFlags.
-  auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
-    if (N.getOpcode() != ISD::FMUL)
+  auto isContractableFMUL = [AllowFusionGlobally, &matcher](SDValue N) {
+    if (!matcher.match(N, ISD::FMUL))
       return false;
     return AllowFusionGlobally || N->getFlags().hasAllowContract();
   };
@@ -15410,8 +15423,9 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
   auto tryToFoldXYSubZ = [&](SDValue XY, SDValue Z) {
     if (isContractableFMUL(XY) && (Aggressive || XY->hasOneUse())) {
-      return DAG.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0),
-                         XY.getOperand(1), DAG.getNode(ISD::FNEG, SL, VT, Z));
+      return matcher.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0),
+                             XY.getOperand(1),
+                             matcher.getNode(ISD::FNEG, SL, VT, Z));
     }
     return SDValue();
   };
@@ -15420,9 +15434,10 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   // Note: Commutes FSUB operands.
   auto tryToFoldXSubYZ = [&](SDValue X, SDValue YZ) {
     if (isContractableFMUL(YZ) && (Aggressive || YZ->hasOneUse())) {
-      return DAG.getNode(PreferredFusedOpcode, SL, VT,
-                         DAG.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)),
-                         YZ.getOperand(1), X);
+      return matcher.getNode(
+          PreferredFusedOpcode, SL, VT,
+          matcher.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)),
+          YZ.getOperand(1), X);
     }
     return SDValue();
   };
@@ -15447,44 +15462,46 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   }
 
   // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
-  if (N0.getOpcode() == ISD::FNEG && isContractableFMUL(N0.getOperand(0)) &&
+  if (matcher.match(N0, ISD::FNEG) && isContractableFMUL(N0.getOperand(0)) &&
       (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) {
     SDValue N00 = N0.getOperand(0).getOperand(0);
     SDValue N01 = N0.getOperand(0).getOperand(1);
-    return DAG.getNode(PreferredFusedOpcode, SL, VT,
-                       DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
-                       DAG.getNode(ISD::FNEG, SL, VT, N1));
+    return matcher.getNode(PreferredFusedOpcode, SL, VT,
+                           matcher.getNode(ISD::FNEG, SL, VT, N00), N01,
+                           matcher.getNode(ISD::FNEG, SL, VT, N1));
   }
 
   // Look through FP_EXTEND nodes to do more combining.
 
   // fold (fsub (fpext (fmul x, y)), z)
   //   -> (fma (fpext x), (fpext y), (fneg z))
-  if (N0.getOpcode() == ISD::FP_EXTEND) {
+  if (matcher.match(N0, ISD::FP_EXTEND)) {
     SDValue N00 = N0.getOperand(0);
     if (isContractableFMUL(N00) &&
         TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                             N00.getValueType())) {
-      return DAG.getNode(PreferredFusedOpcode, SL, VT,
-                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
-                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
-                         DAG.getNode(ISD::FNEG, SL, VT, N1));
+      return matcher.getNode(
+          PreferredFusedOpcode, SL, VT,
+          matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
+          matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
+          matcher.getNode(ISD::FNEG, SL, VT, N1));
     }
   }
 
   // fold (fsub x, (fpext (fmul y, z)))
   //   -> (fma (fneg (fpext y)), (fpext z), x)
   // Note: Commutes FSUB operands.
-  if (N1.getOpcode() == ISD::FP_EXTEND) {
+  if (matcher.match(N1, ISD::FP_EXTEND)) {
     SDValue N10 = N1.getOperand(0);
     if (isContractableFMUL(N10) &&
         TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                             N10.getValueType())) {
-      return DAG.getNode(
+      return matcher.getNode(
           PreferredFusedOpcode, SL, VT,
-          DAG.getNode(ISD::FNEG, SL, VT,
-                      DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))),
-          DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0);
+          matcher.getNode(
+              ISD::FNEG, SL, VT,
+              matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))),
+          matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0);
     }
   }
 
@@ -15494,19 +15511,20 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
   // orthogonal flags -fp-contract=fast and -enable-unsafe-fp-math prevent
   // from implementing the canonicalization in visitFSUB.
-  if (N0.getOpcode() == ISD::FP_EXTEND) {
+  if (matcher.match(N0, ISD::FP_EXTEND)) {
     SDValue N00 = N0.getOperand(0);
-    if (N00.getOpcode() == ISD::FNEG) {
+    if (matcher.match(N00, ISD::FNEG)) {
       SDValue N000 = N00.getOperand(0);
       if (isContractableFMUL(N000) &&
           TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                               N00.getValueType())) {
-        return DAG.getNode(
+        return matcher.getNode(
             ISD::FNEG, SL, VT,
-            DAG.getNode(PreferredFusedOpcode, SL, VT,
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
-                        N1));
+            matcher.getNode(
+                PreferredFusedOpcode, SL, VT,
+                matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
+                matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
+                N1));
       }
     }
   }
@@ -15517,24 +15535,25 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
   // orthogonal flags -fp-contract=fast and -enable-unsafe-fp-math prevent
   // from implementing the canonicalization in visitFSUB.
-  if (N0.getOpcode() == ISD::FNEG) {
+  if (matcher.match(N0, ISD::FNEG)) {
     SDValue N00 = N0.getOperand(0);
-    if (N00.getOpcode() == ISD::FP_EXTEND) {
+    if (matcher.match(N00, ISD::FP_EXTEND)) {
       SDValue N000 = N00.getOperand(0);
       if (isContractableFMUL(N000) &&
           TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                               N000.getValueType())) {
-        return DAG.getNode(
+        return matcher.getNode(
             ISD::FNEG, SL, VT,
-            DAG.getNode(PreferredFusedOpcode, SL, VT,
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
-                        N1));
+            matcher.getNode(
+                PreferredFusedOpcode, SL, VT,
+                matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
+                matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
+                N1));
       }
     }
   }
 
-  auto isReassociable = [Options](SDNode *N) {
+  auto isReassociable = [&Options](SDNode *N) {
     return Options.UnsafeFPMath || N->getFlags().hasAllowReassociation();
   };
 
@@ -15544,8 +15563,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   };
 
   auto isFusedOp = [&](SDValue N) {
-    unsigned Opcode = N.getOpcode();
-    return Opcode == ISD::FMA || Opcode == ISD::FMAD;
+    return matcher.match(N, ISD::FMA) || matcher.match(N, ISD::FMAD);
   };
 
   // More folding opportunities when target permits.
@@ -15556,12 +15574,12 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     if (CanFuse && isFusedOp(N0) &&
         isContractableAndReassociableFMUL(N0.getOperand(2)) &&
         N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) {
-      return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0),
-                         N0.getOperand(1),
-                         DAG.getNode(PreferredFusedOpcode, SL, VT,
-                                     N0.getOperand(2).getOperand(0),
-                                     N0.getOperand(2).getOperand(1),
-                                     DAG.getNode(ISD::FNEG, SL, VT, N1)));
+      return matcher.getNode(
+          PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1),
+          matcher.getNode(PreferredFusedOpcode, SL, VT,
+                          N0.getOperand(2).getOperand(0),
+                          N0.getOperand(2).getOperand(1),
+                          matcher.getNode(ISD::FNEG, SL, VT, N1)));
     }
 
     // fold (fsub x, (fma y, z, (fmul u, v)))
@@ -15571,29 +15589,30 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
         N1->hasOneUse() && NoSignedZero) {
       SDValue N20 = N1.getOperand(2).getOperand(0);
       SDValue N21 = N1.getOperand(2).getOperand(1);
-      return DAG.getNode(
+      return matcher.getNode(
           PreferredFusedOpcode, SL, VT,
-          DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1),
-          DAG.getNode(PreferredFusedOpcode, SL, VT,
-                      DAG.getNode(ISD::FNEG, SL, VT, N20), N21, N0));
+          matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)),
+          N1.getOperand(1),
+          matcher.getNode(PreferredFusedOpcode, SL, VT,
+                          matcher.getNode(ISD::FNEG, SL, VT, N20), N21, N0));
     }
 
     // fold (fsub (fma x, y, (fpext (fmul u, v))), z)
     //   -> (fma x, y (fma (fpext u), (fpext v), (fneg z)))
     if (isFusedOp(N0) && N0->hasOneUse()) {
       SDValue N02 = N0.getOperand(2);
-      if (N02.getOpcode() == ISD::FP_EXTEND) {
+      if (matcher.match(N02, ISD::FP_EXTEND)) {
         SDValue N020 = N02.getOperand(0);
         if (isContractableAndReassociableFMUL(N020) &&
             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                 N020.getValueType())) {
-          return DAG.getNode(
+          return matcher.getNode(
               PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1),
-              DAG.getNode(
+              matcher.getNode(
                   PreferredFusedOpcode, SL, VT,
-                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)),
-                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)),
-                  DAG.getNode(ISD::FNEG, SL, VT, N1)));
+                  matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)),
+                  matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)),
+                  matcher.getNode(ISD::FNEG, SL, VT, N1)));
         }
       }
     }
@@ -15604,29 +15623,29 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     // FIXME: This turns two single-precision and one double-precision
     // operation into two double-precision operations, which might not be
     // interesting for all targets, especially GPUs.
-    if (N0.getOpcode() == ISD::FP_EXTEND) {
+    if (matcher.match(N0, ISD::FP_EXTEND)) {
       SDValue N00 = N0.getOperand(0);
       if (isFusedOp(N00)) {
         SDValue N002 = N00.getOperand(2);
         if (isContractableAndReassociableFMUL(N002) &&
             TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                 N00.getValueType())) {
-          return DAG.getNode(
+          return matcher.getNode(
               PreferredFusedOpcode, SL, VT,
-              DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
-              DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
-              DAG.getNode(
+              matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
+              matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
+              matcher.getNode(
                   PreferredFusedOpcode, SL, VT,
-                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)),
-                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)),
-                  DAG.getNode(ISD::FNEG, SL, VT, N1)));
+                  matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)),
+                  matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)),
+                  matcher.getNode(ISD::FNEG, SL, VT, N1)));
         }
       }
     }
 
     // fold (fsub x, (fma y, z, (fpext (fmul u, v))))
     //   -> (fma (fneg y), z, (fma (fneg (fpext u)), (fpext v), x))
-    if (isFusedOp(N1) && N1.getOperand(2).getOpcode() == ISD::FP_EXTEND &&
+    if (isFusedOp(N1) && matcher.match(N1.getOperand(2), ISD::FP_EXTEND) &&
         N1->hasOneUse()) {
       SDValue N120 = N1.getOperand(2).getOperand(0);
       if (isContractableAndReassociableFMUL(N120) &&
@@ -15634,13 +15653,15 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
                               N120.getValueType())) {
         SDValue N1200 = N120.getOperand(0);
         SDValue N1201 = N120.getOperand(1);
-        return DAG.getNode(
+        return matcher.getNode(
             PreferredFusedOpcode, SL, VT,
-            DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1),
-            DAG.getNode(PreferredFusedOpcode, SL, VT,
-                        DAG.getNode(ISD::FNEG, SL, VT,
-                                    DAG.getNode(ISD::FP_EXTEND, SL, VT, N1200)),
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0));
+            matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)),
+            N1.getOperand(1),
+            matcher.getNode(
+                PreferredFusedOpcode, SL, VT,
+                matcher.getNode(ISD::FNEG, SL, VT,
+                                matcher.getNode(ISD::FP_EXTEND, SL, VT, N1200)),
+                matcher.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0));
       }
     }
 
@@ -15650,7 +15671,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     // FIXME: This turns two single-precision and one double-precision
     // operation into two double-precision operations, which might not be
     // interesting for all targets, especially GPUs.
-    if (N1.getOpcode() == ISD::FP_EXTEND && isFusedOp(N1.getOperand(0))) {
+    if (matcher.match(N1, ISD::FP_EXTEND) && isFusedOp(N1.getOperand(0))) {
       SDValue CvtSrc = N1.getOperand(0);
       SDValue N100 = CvtSrc.getOperand(0);
       SDValue N101 = CvtSrc.getOperand(1);
@@ -15660,15 +15681,16 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
                               CvtSrc.getValueType())) {
         SDValue N1020 = N102.getOperand(0);
         SDValue N1021 = N102.getOperand(1);
-        return DAG.getNode(
+        return matcher.getNode(
             PreferredFusedOpcode, SL, VT,
-            DAG.getNode(ISD::FNEG, SL, VT,
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N100)),
-            DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
-            DAG.getNode(PreferredFusedOpcode, SL, VT,
-                        DAG.getNode(ISD::FNEG, SL, VT,
-                                    DAG.getNode(ISD::FP_EXTEND, SL, VT, N1020)),
-                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0));
+            matcher.getNode(ISD::FNEG, SL, VT,
+                            matcher.getNode(ISD::FP_EXTEND, SL, VT, N100)),
+            matcher.getNode(ISD::FP_EXTEND, SL, VT, N101),
+            matcher.getNode(
+                PreferredFusedOpcode, SL, VT,
+                matcher.getNode(ISD::FNEG, SL, VT,
+                                matcher.getNode(ISD::FP_EXTEND, SL, VT, N1020)),
+                matcher.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0));
       }
     }
   }
@@ -16077,7 +16099,7 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
     return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1);
 
   // FSUB -> FMA combines:
-  if (SDValue Fused = visitFSUBForFMACombine(N)) {
+  if (SDValue Fused = visitFSUBForFMACombine<EmptyMatchContext>(N)) {
     AddToWorklist(Fused.getNode());
     return Fused;
   }
@@ -19412,6 +19434,8 @@ void DAGCombiner::getStoreMergeCandidates(
     // Don't mix temporal stores with non-temporal stores.
     if (St->isNonTemporal() != Other->isNonTemporal())
       return false;
+    if (!TLI.areTwoSDNodeTargetMMOFlagsMergeable(*St, *Other))
+      return false;
     SDValue OtherBC = peekThroughBitcasts(Other->getValue());
     // Allow merging constants of different types as integers.
     bool NoTypeMatch = (MemVT.isInteger()) ? !MemVT.bitsEq(Other->getMemoryVT())
@@ -19437,6 +19461,9 @@ void DAGCombiner::getStoreMergeCandidates(
       // Don't mix temporal loads with non-temporal loads.
       if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
         return false;
+      if (!TLI.areTwoSDNodeTargetMMOFlagsMergeable(*cast<LoadSDNode>(Val),
+                                                   *OtherLd))
+        return false;
       if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
         return false;
       break;
@@ -20061,10 +20088,14 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
     if (IsNonTemporalLoad)
       LdMMOFlags |= MachineMemOperand::MONonTemporal;
 
+    LdMMOFlags |= TLI.getTargetMMOFlags(*FirstLoad);
+
     MachineMemOperand::Flags StMMOFlags = IsNonTemporalStore
                                               ? MachineMemOperand::MONonTemporal
                                               : MachineMemOperand::MONone;
 
+    StMMOFlags |= TLI.getTargetMMOFlags(*StoreNodes[0].MemNode);
+
     SDValue NewLoad, NewStore;
     if (UseVectorTy || !DoIntegerTruncate) {
       NewLoad = DAG.getLoad(
@@ -25462,6 +25493,17 @@ SDValue DAGCombiner::visitVECREDUCE(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitVP_FSUB(SDNode *N) {
+  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
+
+  // FSUB -> FMA combines:
+  if (SDValue Fused = visitFSUBForFMACombine<VPMatchContext>(N)) {
+    AddToWorklist(Fused.getNode());
+    return Fused;
+  }
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitVPOp(SDNode *N) {
 
   if (N->getOpcode() == ISD::VP_GATHER)
@@ -25487,6 +25529,8 @@ SDValue DAGCombiner::visitVPOp(SDNode *N) {
     switch (N->getOpcode()) {
     case ISD::VP_FADD:
       return visitVP_FADD(N);
+    case ISD::VP_FSUB:
+      return visitVP_FSUB(N);
     }
     return SDValue();
   }
@@ -26687,7 +26731,7 @@ SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags,
                           Reciprocal)) {
     AddToWorklist(Est.getNode());
 
-    if (Iterations)
+    if (Iterations > 0)
       Est = UseOneConstNR
             ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal)
             : buildSqrtNRTwoConst(Op, Est, Iterations, Flags, Reciprocal);
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 5f7e3fe8b1665..3e6b6798dd230 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -59,6 +59,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -95,7 +96,6 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -1208,6 +1208,9 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
       return true;
     }
 
+    if (FuncInfo.PreprocessedDbgDeclares.contains(DI))
+      return true;
+
     const Value *Address = DI->getAddress();
     if (!Address || isa<UndefValue>(Address)) {
       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI
@@ -1215,13 +1218,6 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
       return true;
     }
 
-    // Byval arguments with frame indices were already handled after argument
-    // lowering and before isel.
-    const auto *Arg =
-        dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
-    if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
-      return true;
-
     std::optional<MachineOperand> Op;
     if (Register Reg = lookUpRegForValue(Address))
       Op = MachineOperand::CreateReg(Reg, false);
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index df236c503e4b1..f5569ffdac3a8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -348,6 +348,7 @@ void FunctionLoweringInfo::clear() {
   StatepointStackSlots.clear();
   StatepointRelocationMaps.clear();
   PreferredExtendType.clear();
+  PreprocessedDbgDeclares.clear();
 }
 
 /// CreateReg - Allocate a single virtual register for the given type.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 8199e5d5a9c18..7a370db299e46 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -41,7 +42,6 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -3477,13 +3477,13 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     // if we were allowed to generate libcalls to division functions of illegal
     // type. But we cannot do that.
     llvm_unreachable("Cannot expand DIVFIX!");
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY: {
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY: {
     SDValue LHS = Node->getOperand(0);
     SDValue RHS = Node->getOperand(1);
     SDValue Carry = Node->getOperand(2);
 
-    bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
+    bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
 
     // Initial add of the 2 operands.
     unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 4dab62e348233..b80eeedbdf737 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -198,8 +198,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
 
   case ISD::ADDE:
   case ISD::SUBE:
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
 
   case ISD::SADDO_CARRY:
   case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
@@ -1452,23 +1452,24 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
   return Res;
 }
 
-// Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
+// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
-// the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
-SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
+// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
+SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
+                                                       unsigned ResNo) {
   if (ResNo == 1)
     return PromoteIntRes_Overflow(N);
 
   // We need to sign-extend the operands so the carry value computed by the
   // wide operation will be equivalent to the carry value computed by the
   // narrow operation.
-  // An ADDCARRY can generate carry only if any of the operands has its
+  // An UADDO_CARRY can generate carry only if any of the operands has its
   // most significant bit set. Sign extension propagates the most significant
   // bit into the higher bits which means the extra bit that the narrow
   // addition would need (i.e. the carry) will be propagated through the higher
   // bits of the wide addition.
-  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
-  // preserved by sign extension.
+  // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
+  // be preserved by sign extension.
   SDValue LHS = SExtPromotedInteger(N->getOperand(0));
   SDValue RHS = SExtPromotedInteger(N->getOperand(1));
 
@@ -1697,8 +1698,8 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
 
   case ISD::SADDO_CARRY:
   case ISD::SSUBO_CARRY:
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break;
 
   case ISD::FRAMEADDR:
   case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
@@ -2163,7 +2164,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
   return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
 }
 
-SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
+SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) {
   assert(OpNo == 2 && "Don't know how to promote this operand!");
 
   SDValue LHS = N->getOperand(0);
@@ -2547,8 +2548,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::ADDE:
   case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
 
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
 
   case ISD::SADDO_CARRY:
   case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
@@ -2966,7 +2967,7 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
   SDValue HiOps[3] = { LHSH, RHSH };
 
   bool HasOpCarry = TLI.isOperationLegalOrCustom(
-      N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
+      N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
       TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
   if (HasOpCarry) {
     SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
@@ -2975,13 +2976,13 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
       HiOps[2] = Lo.getValue(1);
       Hi = DAG.computeKnownBits(HiOps[2]).isZero()
                ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2))
-               : DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
+               : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps);
     } else {
       Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
       HiOps[2] = Lo.getValue(1);
       Hi = DAG.computeKnownBits(HiOps[2]).isZero()
                ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2))
-               : DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
+               : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps);
     }
     return;
   }
@@ -3153,12 +3154,12 @@ void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
   ISD::CondCode Cond;
   switch(N->getOpcode()) {
     case ISD::UADDO:
-      CarryOp = ISD::ADDCARRY;
+      CarryOp = ISD::UADDO_CARRY;
       NoCarryOp = ISD::ADD;
       Cond = ISD::SETULT;
       break;
     case ISD::USUBO:
-      CarryOp = ISD::SUBCARRY;
+      CarryOp = ISD::USUBO_CARRY;
       NoCarryOp = ISD::SUB;
       Cond = ISD::SETUGT;
       break;
@@ -3212,8 +3213,8 @@ void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
   ReplaceValueWith(SDValue(N, 1), Ovf);
 }
 
-void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
-                                                SDValue &Lo, SDValue &Hi) {
+void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
+                                                   SDValue &Hi) {
   // Expand the subcomponents.
   SDValue LHSL, LHSH, RHSL, RHSH;
   SDLoc dl(N);
@@ -3242,8 +3243,8 @@ void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
   SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
 
   // We need to use an unsigned carry op for the lo part.
-  unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
-                                                        : ISD::SUBCARRY;
+  unsigned CarryOp =
+      N->getOpcode() == ISD::SADDO_CARRY ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
   Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
   Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
 
@@ -3373,14 +3374,14 @@ void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
     return;
   }
 
-  // If we have SUBCARRY, use the expanded form of the sra+xor+sub sequence we
-  // use in LegalizeDAG. The SUB part of the expansion is based on
-  // ExpandIntRes_ADDSUB which also uses SUBCARRY/USUBO after checking that
-  // SUBCARRY is LegalOrCustom. Each of the pieces here can be further expanded
-  // if needed. Shift expansion has a special case for filling with sign bits
-  // so that we will only end up with one SRA.
+  // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
+  // we use in LegalizeDAG. The SUB part of the expansion is based on
+  // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
+  // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
+  // expanded if needed. Shift expansion has a special case for filling with
+  // sign bits so that we will only end up with one SRA.
   bool HasSubCarry = TLI.isOperationLegalOrCustom(
-      ISD::SUBCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
+      ISD::USUBO_CARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
   if (HasSubCarry) {
     SDValue Sign = DAG.getNode(
         ISD::SRA, dl, NVT, Hi,
@@ -3389,7 +3390,7 @@ void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
     Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
     Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
     Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
-    Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
+    Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
     return;
   }
 
@@ -5153,9 +5154,10 @@ SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
   GetExpandedInteger(LHS, LHSLo, LHSHi);
   GetExpandedInteger(RHS, RHSLo, RHSHi);
 
-  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
+  // Expand to a USUBO_CARRY for the low part and a SETCCCARRY for the high.
   SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
-  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
+  SDValue LowCmp =
+      DAG.getNode(ISD::USUBO_CARRY, dl, VTList, LHSLo, RHSLo, Carry);
   return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
                      LowCmp.getValue(1), Cond);
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index f5cce12a39299..f4896cd1f09ad 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -347,7 +347,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntRes_SRL(SDNode *N);
   SDValue PromoteIntRes_TRUNCATE(SDNode *N);
   SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
-  SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
+  SDValue PromoteIntRes_UADDSUBO_CARRY(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_SADDSUBO_CARRY(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_UNDEF(SDNode *N);
   SDValue PromoteIntRes_VAARG(SDNode *N);
@@ -396,7 +396,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
-  SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
+  SDValue PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
   SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FIX(SDNode *N);
@@ -449,7 +449,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   void ExpandIntRes_ADDSUB            (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_ADDSUBC           (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_ADDSUBE           (SDNode *N, SDValue &Lo, SDValue &Hi);
-  void ExpandIntRes_ADDSUBCARRY       (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandIntRes_UADDSUBO_CARRY    (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_SADDSUBO_CARRY    (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_BITREVERSE        (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_BSWAP             (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index e245b3cb4c6d6..5d5157142f0ac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -29,6 +29,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -38,7 +39,6 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -296,7 +296,17 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
     if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
         Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
       ValVT = Node->getOperand(1).getValueType();
-    Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
+    if (Op.getOpcode() == ISD::STRICT_FSETCC ||
+        Op.getOpcode() == ISD::STRICT_FSETCCS) {
+      MVT OpVT = Node->getOperand(1).getSimpleValueType();
+      ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get();
+      Action = TLI.getCondCodeAction(CCCode, OpVT);
+      if (Action == TargetLowering::Legal)
+        Action =
+            TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
+    } else {
+      Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
+    }
     // If we're asked to expand a strict vector floating-point operation,
     // by default we're going to simply unroll it.  That is usually the
     // best approach, except in the case where the resulting strict (scalar)
@@ -1304,11 +1314,11 @@ SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
     return DAG.UnrollVectorOp(Node);
 
   SDValue Ones = DAG.getAllOnesConstant(DL, VT);
-  SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Mask, EVL);
+  SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL);
 
-  Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Mask, EVL);
-  Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Mask, EVL);
-  return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Mask, EVL);
+  Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL);
+  Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL);
+  return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL);
 }
 
 SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
@@ -1516,39 +1526,54 @@ void VectorLegalizer::ExpandSETCC(SDNode *Node,
                                   SmallVectorImpl<SDValue> &Results) {
   bool NeedInvert = false;
   bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
-  SDLoc dl(Node);
-  MVT OpVT = Node->getOperand(0).getSimpleValueType();
-  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
+  bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
+                  Node->getOpcode() == ISD::STRICT_FSETCCS;
+  bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
+  unsigned Offset = IsStrict ? 1 : 0;
+
+  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
+  SDValue LHS = Node->getOperand(0 + Offset);
+  SDValue RHS = Node->getOperand(1 + Offset);
+  SDValue CC = Node->getOperand(2 + Offset);
+
+  MVT OpVT = LHS.getSimpleValueType();
+  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
 
   if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
+    if (IsStrict) {
+      UnrollStrictFPOp(Node, Results);
+      return;
+    }
     Results.push_back(UnrollVSETCC(Node));
     return;
   }
 
-  SDValue Chain;
-  SDValue LHS = Node->getOperand(0);
-  SDValue RHS = Node->getOperand(1);
-  SDValue CC = Node->getOperand(2);
   SDValue Mask, EVL;
   if (IsVP) {
-    Mask = Node->getOperand(3);
-    EVL = Node->getOperand(4);
+    Mask = Node->getOperand(3 + Offset);
+    EVL = Node->getOperand(4 + Offset);
   }
 
+  SDLoc dl(Node);
   bool Legalized =
       TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
-                                EVL, NeedInvert, dl, Chain);
+                                EVL, NeedInvert, dl, Chain, IsSignaling);
 
   if (Legalized) {
     // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
     // condition code, create a new SETCC node.
     if (CC.getNode()) {
-      if (!IsVP)
-        LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
-                          Node->getFlags());
-      else
+      if (IsStrict) {
+        LHS = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
+                          {Chain, LHS, RHS, CC}, Node->getFlags());
+        Chain = LHS.getValue(1);
+      } else if (IsVP) {
         LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
                           {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
+      } else {
+        LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
+                          Node->getFlags());
+      }
     }
 
     // If we expanded the SETCC by inverting the condition code, then wrap
@@ -1560,6 +1585,8 @@ void VectorLegalizer::ExpandSETCC(SDNode *Node,
         LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
     }
   } else {
+    assert(!IsStrict && "Don't know how to expand for strict nodes.");
+
     // Otherwise, SETCC for the given comparison type must be completely
     // illegal; expand it into a SELECT_CC.
     EVT VT = Node->getValueType(0);
@@ -1571,6 +1598,8 @@ void VectorLegalizer::ExpandSETCC(SDNode *Node,
   }
 
   Results.push_back(LHS);
+  if (IsStrict)
+    Results.push_back(Chain);
 }
 
 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
@@ -1618,6 +1647,12 @@ void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
     return;
   }
 
+  if (Node->getOpcode() == ISD::STRICT_FSETCC ||
+      Node->getOpcode() == ISD::STRICT_FSETCCS) {
+    ExpandSETCC(Node, Results);
+    return;
+  }
+
   UnrollStrictFPOp(Node, Results);
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 46795fd8b627e..7762aeef114c5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2383,7 +2383,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
     EVT EltVT = NewVT.getVectorElementType();
     SmallVector<SDValue> Ops(NewElts, DAG.getUNDEF(EltVT));
     for (unsigned I = 0; I < NewElts; ++I) {
-      if (Mask[I] == UndefMaskElem)
+      if (Mask[I] == PoisonMaskElem)
         continue;
       unsigned Idx = Mask[I];
       if (Idx >= NewElts)
@@ -2423,11 +2423,11 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
       // Use shuffles operands instead of shuffles themselves.
       // 1. Adjust mask.
       for (int &Idx : Mask) {
-        if (Idx == UndefMaskElem)
+        if (Idx == PoisonMaskElem)
           continue;
         unsigned SrcRegIdx = Idx / NewElts;
         if (Inputs[SrcRegIdx].isUndef()) {
-          Idx = UndefMaskElem;
+          Idx = PoisonMaskElem;
           continue;
         }
         auto *Shuffle =
@@ -2435,8 +2435,8 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
         if (!Shuffle || !is_contained(P.second, SrcRegIdx))
           continue;
         int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
-        if (MaskElt == UndefMaskElem) {
-          Idx = UndefMaskElem;
+        if (MaskElt == PoisonMaskElem) {
+          Idx = PoisonMaskElem;
           continue;
         }
         Idx = MaskElt % NewElts +
@@ -2455,11 +2455,11 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
     // Check if any concat_vectors can be simplified.
     SmallBitVector UsedSubVector(2 * std::size(Inputs));
     for (int &Idx : Mask) {
-      if (Idx == UndefMaskElem)
+      if (Idx == PoisonMaskElem)
         continue;
       unsigned SrcRegIdx = Idx / NewElts;
       if (Inputs[SrcRegIdx].isUndef()) {
-        Idx = UndefMaskElem;
+        Idx = PoisonMaskElem;
         continue;
       }
       TargetLowering::LegalizeTypeAction TypeAction =
@@ -2489,7 +2489,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
       if (!Pairs.empty() && Pairs.front().size() > 1) {
         // Adjust mask.
         for (int &Idx : Mask) {
-          if (Idx == UndefMaskElem)
+          if (Idx == PoisonMaskElem)
             continue;
           unsigned SrcRegIdx = Idx / NewElts;
           auto *It = find_if(
@@ -2531,14 +2531,14 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
                    !Shuffle->getOperand(1).isUndef()) {
           // Find the only used operand, if possible.
           for (int &Idx : Mask) {
-            if (Idx == UndefMaskElem)
+            if (Idx == PoisonMaskElem)
               continue;
             unsigned SrcRegIdx = Idx / NewElts;
             if (SrcRegIdx != I)
               continue;
             int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
-            if (MaskElt == UndefMaskElem) {
-              Idx = UndefMaskElem;
+            if (MaskElt == PoisonMaskElem) {
+              Idx = PoisonMaskElem;
               continue;
             }
             int OpIdx = MaskElt / NewElts;
@@ -2564,14 +2564,14 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
             // Found that operand is used already.
             // 1. Fix the mask for the reused operand.
             for (int &Idx : Mask) {
-              if (Idx == UndefMaskElem)
+              if (Idx == PoisonMaskElem)
                 continue;
               unsigned SrcRegIdx = Idx / NewElts;
               if (SrcRegIdx != I)
                 continue;
               int MaskElt = Shuffle->getMaskElt(Idx % NewElts);
-              if (MaskElt == UndefMaskElem) {
-                Idx = UndefMaskElem;
+              if (MaskElt == PoisonMaskElem) {
+                Idx = PoisonMaskElem;
                 continue;
               }
               int MaskIdx = MaskElt / NewElts;
@@ -2588,7 +2588,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
           Inputs[I] = Shuffle->getOperand(Op);
           // Adjust mask.
           for (int &Idx : Mask) {
-            if (Idx == UndefMaskElem)
+            if (Idx == PoisonMaskElem)
               continue;
             unsigned SrcRegIdx = Idx / NewElts;
             if (SrcRegIdx != I)
@@ -2622,11 +2622,11 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
       auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
       unsigned ConstNum = UniqueConstantVec.size();
       for (int &Idx : Mask) {
-        if (Idx == UndefMaskElem)
+        if (Idx == PoisonMaskElem)
           continue;
         unsigned SrcRegIdx = Idx / NewElts;
         if (Inputs[SrcRegIdx].isUndef()) {
-          Idx = UndefMaskElem;
+          Idx = PoisonMaskElem;
           continue;
         }
         const auto It = find(UniqueConstantVec, Inputs[SrcRegIdx]);
@@ -2655,7 +2655,7 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
     // Build a shuffle mask for the output, discovering on the fly which
     // input vectors to use as shuffle operands.
     unsigned FirstMaskIdx = High * NewElts;
-    SmallVector<int> Mask(NewElts * std::size(Inputs), UndefMaskElem);
+    SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
     copy(ArrayRef(OrigMask).slice(FirstMaskIdx, NewElts), Mask.begin());
     assert(!Output && "Expected default initialized initial value.");
     TryPeekThroughShufflesInputs(Mask);
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 78ac0c27d7205..458f50c54824e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
@@ -45,7 +46,6 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index 99bbaeb191821..439ccfdc32758 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -16,10 +16,10 @@
 
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cassert>
 #include <string>
 #include <vector>
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index bdbd2f899c8b4..faae1cfc0a346 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -35,6 +35,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -61,7 +62,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/raw_ostream.h"
@@ -3381,7 +3381,6 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
     Known = KnownBits::ashr(Known, Known2);
-    // TODO: Add minimum shift high known sign bits.
     break;
   case ISD::FSHL:
   case ISD::FSHR:
@@ -3637,7 +3636,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     break;
   case ISD::USUBO:
   case ISD::SSUBO:
-  case ISD::SUBCARRY:
+  case ISD::USUBO_CARRY:
   case ISD::SSUBO_CARRY:
     if (Op.getResNo() == 1) {
       // If we know the result of a setcc has the top bits zero, use this info.
@@ -3654,7 +3653,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
            "We only compute knownbits for the difference here.");
 
     // TODO: Compute influence of the carry operand.
-    if (Opcode == ISD::SUBCARRY || Opcode == ISD::SSUBO_CARRY)
+    if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY)
       break;
 
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
@@ -3665,7 +3664,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
   }
   case ISD::UADDO:
   case ISD::SADDO:
-  case ISD::ADDCARRY:
+  case ISD::UADDO_CARRY:
   case ISD::SADDO_CARRY:
     if (Op.getResNo() == 1) {
       // If we know the result of a setcc has the top bits zero, use this info.
@@ -3681,12 +3680,12 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
   case ISD::ADDE: {
     assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
 
-    // With ADDE and ADDCARRY, a carry bit may be added in.
+    // With ADDE and UADDO_CARRY, a carry bit may be added in.
     KnownBits Carry(1);
     if (Opcode == ISD::ADDE)
       // Can't track carry from glue, set carry to unknown.
       Carry.resetAll();
-    else if (Opcode == ISD::ADDCARRY || Opcode == ISD::SADDO_CARRY)
+    else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY)
       // TODO: Compute known bits for the carry operand. Not sure if it is worth
       // the trouble (how often will we find a known carry bit). And I haven't
       // tested this very much yet, but something like this might work:
@@ -4022,10 +4021,7 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
 
   // More could be done here, though the above checks are enough
   // to handle some common cases.
-
-  // Fall back to computeKnownBits to catch other known cases.
-  KnownBits Known = computeKnownBits(Val);
-  return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
+  return false;
 }
 
 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
@@ -4272,11 +4268,11 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
   case ISD::SADDO:
   case ISD::UADDO:
   case ISD::SADDO_CARRY:
-  case ISD::ADDCARRY:
+  case ISD::UADDO_CARRY:
   case ISD::SSUBO:
   case ISD::USUBO:
   case ISD::SSUBO_CARRY:
-  case ISD::SUBCARRY:
+  case ISD::USUBO_CARRY:
   case ISD::SMULO:
   case ISD::UMULO:
     if (Op.getResNo() != 1)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 329fde6401818..f4ca34ebfd582 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1405,6 +1405,17 @@ void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
                     << "\n");
 }
 
+void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
+                                               DIExpression *Expr,
+                                               DebugLoc DbgLoc,
+                                               unsigned Order) {
+  Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context));
+  DIExpression *NewExpr =
+      const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
+  handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
+                   /*IsVariadic*/ false);
+}
+
 bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
                                            DILocalVariable *Var,
                                            DIExpression *Expr, DebugLoc DbgLoc,
@@ -6114,12 +6125,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     return;
   }
   case Intrinsic::dbg_declare: {
+    const auto &DI = cast<DbgDeclareInst>(I);
     // Debug intrinsics are handled separately in assignment tracking mode.
-    if (AssignmentTrackingEnabled)
+    // Some intrinsics are handled right after Argument lowering.
+    if (AssignmentTrackingEnabled ||
+        FuncInfo.PreprocessedDbgDeclares.count(&DI))
       return;
     // Assume dbg.declare can not currently use DIArgList, i.e.
     // it is non-variadic.
-    const auto &DI = cast<DbgVariableIntrinsic>(I);
     assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
     DILocalVariable *Variable = DI.getVariable();
     DIExpression *Expression = DI.getExpression();
@@ -6138,29 +6151,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
 
     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
 
-    // Check if this variable can be described by a frame index, typically
-    // either as a static alloca or a byval parameter.
-    int FI = std::numeric_limits<int>::max();
-    if (const auto *AI =
-            dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
-      if (AI->isStaticAlloca()) {
-        auto I = FuncInfo.StaticAllocaMap.find(AI);
-        if (I != FuncInfo.StaticAllocaMap.end())
-          FI = I->second;
-      }
-    } else if (const auto *Arg = dyn_cast<Argument>(
-                   Address->stripInBoundsConstantOffsets())) {
-      FI = FuncInfo.getArgumentFrameIndex(Arg);
-    }
-
-    // llvm.dbg.declare is handled as a frame index in the MachineFunction
-    // variable table.
-    if (FI != std::numeric_limits<int>::max()) {
-      LLVM_DEBUG(dbgs() << "Skipping " << DI
-                        << " (variable info stashed in MF side table)\n");
-      return;
-    }
-
     SDValue &N = NodeMap[Address];
     if (!N.getNode() && isa<Argument>(Address))
       // Check unused arguments map.
@@ -6226,11 +6216,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     DILocalVariable *Variable = DI.getVariable();
     DIExpression *Expression = DI.getExpression();
     dropDanglingDebugInfo(Variable, Expression);
-    SmallVector<Value *, 4> Values(DI.getValues());
-    if (Values.empty())
+
+    if (DI.isKillLocation()) {
+      handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
       return;
+    }
 
-    if (llvm::is_contained(Values, nullptr))
+    SmallVector<Value *, 4> Values(DI.getValues());
+    if (Values.empty())
       return;
 
     bool IsVariadic = DI.hasArgList();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 45685f59e1631..f2496f24973a7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
 #include "llvm/CodeGen/CodeGenCommonISel.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/SwitchLoweringUtils.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -30,7 +31,6 @@
 #include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -376,6 +376,10 @@ class SelectionDAGBuilder {
                         DIExpression *Expr, DebugLoc DbgLoc, unsigned Order,
                         bool IsVariadic);
 
+  /// Create a record for a kill location debug intrinsic.
+  void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr,
+                            DebugLoc DbgLoc, unsigned Order);
+
   /// Evict any dangling debug information, attempting to salvage it first.
   void resolveOrClearDbgInfo();
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index f6d4ad1b46e90..1dd5e6cb057f3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -40,7 +41,6 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/Printable.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
@@ -309,7 +309,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::CARRY_FALSE:                return "carry_false";
   case ISD::ADDC:                       return "addc";
   case ISD::ADDE:                       return "adde";
-  case ISD::ADDCARRY:                   return "addcarry";
+  case ISD::UADDO_CARRY:                return "uaddo_carry";
   case ISD::SADDO_CARRY:                return "saddo_carry";
   case ISD::SADDO:                      return "saddo";
   case ISD::UADDO:                      return "uaddo";
@@ -319,7 +319,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::UMULO:                      return "umulo";
   case ISD::SUBC:                       return "subc";
   case ISD::SUBE:                       return "sube";
-  case ISD::SUBCARRY:                   return "subcarry";
+  case ISD::USUBO_CARRY:                return "usubo_carry";
   case ISD::SSUBO_CARRY:                return "ssubo_carry";
   case ISD::SHL_PARTS:                  return "shl_parts";
   case ISD::SRA_PARTS:                  return "sra_parts";
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 563f14c9ce188..36611b117296a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -48,6 +48,7 @@
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachinePassRegistry.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -92,7 +93,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
@@ -1342,13 +1342,13 @@ static bool isFoldedOrDeadInstruction(const Instruction *I,
          !FuncInfo.isExportedInst(I); // Exported instrs must be computed.
 }
 
-static void processDbgDeclare(FunctionLoweringInfo &FuncInfo,
+static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo,
                               const Value *Address, DIExpression *Expr,
                               DILocalVariable *Var, DebugLoc DbgLoc) {
   if (!Address) {
     LLVM_DEBUG(dbgs() << "processDbgDeclares skipping " << *Var
                       << " (bad address)\n");
-    return;
+    return false;
   }
   MachineFunction *MF = FuncInfo.MF;
   const DataLayout &DL = MF->getDataLayout();
@@ -1373,7 +1373,7 @@ static void processDbgDeclare(FunctionLoweringInfo &FuncInfo,
     FI = FuncInfo.getArgumentFrameIndex(Arg);
 
   if (FI == std::numeric_limits<int>::max())
-    return;
+    return false;
 
   if (Offset.getBoolValue())
     Expr = DIExpression::prepend(Expr, DIExpression::ApplyOffset,
@@ -1383,18 +1383,17 @@ static void processDbgDeclare(FunctionLoweringInfo &FuncInfo,
                     << ", Expr=" << *Expr << ",  FI=" << FI
                     << ", DbgLoc=" << DbgLoc << "\n");
   MF->setVariableDbgInfo(Var, Expr, FI, DbgLoc);
+  return true;
 }
 
 /// Collect llvm.dbg.declare information. This is done after argument lowering
 /// in case the declarations refer to arguments.
 static void processDbgDeclares(FunctionLoweringInfo &FuncInfo) {
-  for (const BasicBlock &BB : *FuncInfo.Fn) {
-    for (const Instruction &I : BB) {
-      if (const DbgDeclareInst *DI = dyn_cast<DbgDeclareInst>(&I)) {
-        processDbgDeclare(FuncInfo, DI->getAddress(), DI->getExpression(),
-                          DI->getVariable(), DI->getDebugLoc());
-      }
-    }
+  for (const auto &I : instructions(*FuncInfo.Fn)) {
+    const auto *DI = dyn_cast<DbgDeclareInst>(&I);
+    if (DI && processDbgDeclare(FuncInfo, DI->getAddress(), DI->getExpression(),
+                                DI->getVariable(), DI->getDebugLoc()))
+      FuncInfo.PreprocessedDbgDeclares.insert(DI);
   }
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 24e5824063d06..5984a99e8aee8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -42,7 +43,6 @@
 #include "llvm/IR/Type.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include <cassert>
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 5a295a7957dae..b18f2d7e9aa71 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6703,9 +6703,9 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
   // NOTE: we avoid letting illegal types through even if we're before legalize
   // ops – legalization has a hard time producing good code for the code that
   // follows.
-  if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
+  if (!isOperationLegalOrCustom(ISD::SETCC, SETCCVT) ||
       !isOperationLegalOrCustom(ISD::AND, VT) ||
-      !isOperationLegalOrCustom(Cond, VT) ||
+      !isCondCodeLegalOrCustom(Cond, VT.getSimpleVT()) ||
       !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))
     return SDValue();
 
@@ -6755,20 +6755,23 @@ SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
   EVT VT = Op.getValueType();
   EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
   SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
-  // Testing it with denormal inputs to avoid wrong estimate.
-  if (Mode.Input == DenormalMode::IEEE) {
-    // This is specifically a check for the handling of denormal inputs,
-    // not the result.
 
-    // Test = fabs(X) < SmallestNormal
-    const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
-    APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
-    SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
-    SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
-    return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
+  // This is specifically a check for the handling of denormal inputs, not the
+  // result.
+  if (Mode.Input == DenormalMode::PreserveSign ||
+      Mode.Input == DenormalMode::PositiveZero) {
+    // Test = X == 0.0
+    return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
   }
-  // Test = X == 0.0
-  return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
+
+  // Testing it with denormal inputs to avoid wrong estimate.
+  //
+  // Test = fabs(X) < SmallestNormal
+  const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
+  APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
+  SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
+  SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
+  return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
 }
 
 SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
@@ -7219,7 +7222,7 @@ bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
     Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
                        Merge(Lo, Hi));
   else
-    Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
+    Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next,
                        Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
 
   SDValue Carry = Next.getValue(1);
@@ -7233,7 +7236,7 @@ bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
     Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
                      Carry);
   else
-    Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
+    Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
                      Zero, Carry);
 
   Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
@@ -7375,13 +7378,13 @@ bool TargetLowering::expandDIVREMByConstant(SDNode *N,
                        DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl));
     }
 
-    // Use addcarry if we can, otherwise use a compare to detect overflow.
+    // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
     EVT SetCCType =
         getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), HiLoVT);
-    if (isOperationLegalOrCustom(ISD::ADDCARRY, HiLoVT)) {
+    if (isOperationLegalOrCustom(ISD::UADDO_CARRY, HiLoVT)) {
       SDVTList VTList = DAG.getVTList(HiLoVT, SetCCType);
       Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH);
-      Sum = DAG.getNode(ISD::ADDCARRY, dl, VTList, Sum,
+      Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum,
                         DAG.getConstant(0, dl, HiLoVT), Sum.getValue(1));
     } else {
       Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH);
@@ -7991,7 +7994,9 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
   // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
   // instead if there are no NaNs and there can't be an incompatible zero
   // compare: at least one operand isn't +/-0, or there are no signed-zeros.
-  if (Node->getFlags().hasNoNaNs() &&
+  if ((Node->getFlags().hasNoNaNs() ||
+       (DAG.isKnownNeverNaN(Node->getOperand(0)) &&
+        DAG.isKnownNeverNaN(Node->getOperand(1)))) &&
       (Node->getFlags().hasNoSignedZeros() ||
        DAG.isKnownNeverZeroFloat(Node->getOperand(0)) ||
        DAG.isKnownNeverZeroFloat(Node->getOperand(1)))) {
@@ -9932,8 +9937,8 @@ void TargetLowering::expandUADDSUBO(
   SDValue RHS = Node->getOperand(1);
   bool IsAdd = Node->getOpcode() == ISD::UADDO;
 
-  // If ADD/SUBCARRY is legal, use that instead.
-  unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
+  // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
+  unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
   if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
     SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
     SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
@@ -10316,8 +10321,10 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
 
     // Otherwise, select 0 if Src is NaN.
     SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
-    return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
-                           ISD::CondCode::SETUO);
+    EVT SetCCVT =
+        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
+    SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
+    return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt);
   }
 
   SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
@@ -10331,13 +10338,16 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
 
   SDValue Select = FpToInt;
 
+  EVT SetCCVT =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
+
   // If Src ULT MinFloat, select MinInt. In particular, this also selects
   // MinInt if Src is NaN.
-  Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,
-                           ISD::CondCode::SETULT);
+  SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT);
+  Select = DAG.getSelect(dl, DstVT, ULT, MinIntNode, Select);
   // If Src OGT MaxFloat, select MaxInt.
-  Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
-                           ISD::CondCode::SETOGT);
+  SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT);
+  Select = DAG.getSelect(dl, DstVT, OGT, MaxIntNode, Select);
 
   // In the unsigned case we are done, because we mapped NaN to MinInt, which
   // is already zero.
@@ -10346,7 +10356,8 @@ SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
 
   // Otherwise, select 0 if Src is NaN.
   SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
-  return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
+  SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
+  return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, Select);
 }
 
 SDValue TargetLowering::expandVectorSplice(SDNode *Node,
diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp
index 292a37b8264cb..b219b83bbc2fe 100644
--- a/llvm/lib/CodeGen/ShrinkWrap.cpp
+++ b/llvm/lib/CodeGen/ShrinkWrap.cpp
@@ -53,6 +53,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MachineDominators.h"
@@ -259,13 +260,30 @@ INITIALIZE_PASS_END(ShrinkWrap, DEBUG_TYPE, "Shrink Wrap Pass", false, false)
 
 bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr &MI,
                                  RegScavenger *RS) const {
+  /// Check if \p Op is known to access an address not on the function's stack .
+  /// At the moment, accesses where the underlying object is a global or a
+  /// function argument are considered non-stack accesses. Note that the
+  /// caller's stack may get accessed when passing an argument via the stack,
+  /// but not the stack of the current function.
+  ///
+  auto IsKnownNonStackPtr = [](MachineMemOperand *Op) {
+    if (Op->getValue()) {
+      const Value *UO = getUnderlyingObject(Op->getValue());
+      if (!UO)
+        return false;
+      if (auto *Arg = dyn_cast<Argument>(UO))
+        return !Arg->hasPassPointeeByValueCopyAttr();
+      return isa<GlobalValue>(UO);
+    }
+    return false;
+  };
   // This prevents premature stack popping when occurs a indirect stack
-  // access. It is overly aggressive for the moment.
-  // TODO: - Obvious non-stack loads and store, such as global values,
-  //         are known to not access the stack.
+  // access.  It is overly aggressive for the moment.
+  // TODO:
   //       - Further, data dependency and alias analysis can validate
   //         that load and stores never derive from the stack pointer.
-  if (MI.mayLoadOrStore())
+  if (MI.mayLoadOrStore() && (MI.isCall() || MI.hasUnmodeledSideEffects() ||
+                              !all_of(MI.memoperands(), IsKnownNonStackPtr)))
     return true;
 
   if (MI.getOpcode() == FrameSetupOpcode ||
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index 93c2252978391..5ed67bd0a121e 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -427,7 +427,13 @@ void TailDuplicator::duplicateInstruction(
           } else {
             // For mapped registers that do not have sub-registers, simply
             // restrict their class to match the original one.
-            ConstrRC = MRI->constrainRegClass(VI->second.Reg, OrigRC);
+
+            // We don't want debug instructions affecting the resulting code so
+            // if we're cloning a debug instruction then just use MappedRC
+            // rather than constraining the register class further.
+            ConstrRC = NewMI.isDebugInstr()
+                           ? MappedRC
+                           : MRI->constrainRegClass(VI->second.Reg, OrigRC);
           }
 
           if (ConstrRC) {
@@ -1013,13 +1019,11 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
 
     DenseMap<Register, RegSubRegPair> LocalVRMap;
     SmallVector<std::pair<Register, RegSubRegPair>, 4> CopyInfos;
-    MachineBasicBlock::iterator I = TailBB->begin();
     // Process PHI instructions first.
-    while (I != TailBB->end() && I->isPHI()) {
+    for (MachineInstr &MI : make_early_inc_range(TailBB->phis())) {
       // Replace the uses of the def of the PHI with the register coming
       // from PredBB.
-      MachineInstr *MI = &*I++;
-      processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
+      processPHI(&MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
     }
     appendCopies(PredBB, CopyInfos, Copies);
   }
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 9d6c9a6768d5c..c3a79933136b6 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -1343,11 +1343,7 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
     if (Reg == DestReg)
       return ParamLoadedValue(*DestSrc->Source, Expr);
 
-    // Cases where super- or sub-registers needs to be described should
-    // be handled by the target's hook implementation.
-    assert(!TRI->isSuperOrSubRegisterEq(Reg, DestReg) &&
-           "TargetInstrInfo::describeLoadedValue can't describe super- or "
-           "sub-regs for copy instructions");
+    // If the target's hook couldn't describe this copy, give up.
     return std::nullopt;
   } else if (auto RegImm = isAddImmediate(MI, Reg)) {
     Register SrcReg = RegImm->Reg;
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 6d006ff6081b4..22472d422ffed 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -28,6 +28,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/StackMaps.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -48,7 +49,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -821,8 +821,8 @@ void TargetLoweringBase::initActions() {
                         ISD::SMULO, ISD::UMULO},
                        VT, Expand);
 
-    // ADDCARRY operations default to expand
-    setOperationAction({ISD::ADDCARRY, ISD::SUBCARRY, ISD::SETCCCARRY,
+    // Carry-using overflow operations default to expand.
+    setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY, ISD::SETCCCARRY,
                         ISD::SADDO_CARRY, ISD::SSUBO_CARRY},
                        VT, Expand);
 
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index c81b6bb623b96..db1a4369e1291 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -1202,7 +1202,12 @@ void TargetLoweringObjectFileMachO::Initialize(MCContext &Ctx,
 
 MCSection *TargetLoweringObjectFileMachO::getStaticDtorSection(
     unsigned Priority, const MCSymbol *KeySym) const {
-  report_fatal_error("@llvm.global_dtors should have been lowered already");
+  return StaticDtorSection;
+  // In userspace, we lower global destructors via atexit(), but kernel/kext
+  // environments do not provide this function so we still need to support the
+  // legacy way here.
+  // See the -disable-atexit-based-global-dtor-lowering CodeGen flag for more
+  // context.
 }
 
 void TargetLoweringObjectFileMachO::emitModuleMetadata(MCStreamer &Streamer,
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 32fa6cc17d53f..86490c0d6417d 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -100,6 +100,9 @@ static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
     cl::desc("Disable Copy Propagation pass"));
 static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
     cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
+static cl::opt<bool> DisableAtExitBasedGlobalDtorLowering(
+    "disable-atexit-based-global-dtor-lowering", cl::Hidden,
+    cl::desc("For MachO, disable atexit()-based global destructor lowering"));
 static cl::opt<bool> EnableImplicitNullChecks(
     "enable-implicit-null-checks",
     cl::desc("Fold null checks into faulting memory operations"),
@@ -878,7 +881,8 @@ void TargetPassConfig::addIRPasses() {
 
   // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with
   // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func.
-  if (TM->getTargetTriple().isOSBinFormatMachO())
+  if (TM->getTargetTriple().isOSBinFormatMachO() &&
+      !DisableAtExitBasedGlobalDtorLowering)
     addPass(createLowerGlobalDtorsLegacyPass());
 
   // Make sure that no unreachable blocks are instruction selected.
diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
index 3357be52f2aa7..77d2dfcf2323b 100644
--- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -33,7 +34,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Printable.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/lib/CodeGen/UnreachableBlockElim.cpp b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
index 5e8514f525e9d..f17450d264ba0 100644
--- a/llvm/lib/CodeGen/UnreachableBlockElim.cpp
+++ b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
@@ -120,16 +120,14 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
       while (BB.succ_begin() != BB.succ_end()) {
         MachineBasicBlock* succ = *BB.succ_begin();
 
-        MachineBasicBlock::iterator start = succ->begin();
-        while (start != succ->end() && start->isPHI()) {
-          for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
-            if (start->getOperand(i).isMBB() &&
-                start->getOperand(i).getMBB() == &BB) {
-              start->removeOperand(i);
-              start->removeOperand(i-1);
+        for (MachineInstr &Phi : succ->phis()) {
+          for (unsigned i = Phi.getNumOperands() - 1; i >= 2; i -= 2) {
+            if (Phi.getOperand(i).isMBB() &&
+                Phi.getOperand(i).getMBB() == &BB) {
+              Phi.removeOperand(i);
+              Phi.removeOperand(i - 1);
             }
-
-          start++;
+          }
         }
 
         BB.removeSuccessor(BB.succ_begin());
@@ -152,18 +150,18 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
     // Prune unneeded PHI entries.
     SmallPtrSet<MachineBasicBlock*, 8> preds(BB.pred_begin(),
                                              BB.pred_end());
-    MachineBasicBlock::iterator phi = BB.begin();
-    while (phi != BB.end() && phi->isPHI()) {
-      for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
-        if (!preds.count(phi->getOperand(i).getMBB())) {
-          phi->removeOperand(i);
-          phi->removeOperand(i-1);
+    for (MachineInstr &Phi : make_early_inc_range(BB.phis())) {
+      for (unsigned i = Phi.getNumOperands() - 1; i >= 2; i -= 2) {
+        if (!preds.count(Phi.getOperand(i).getMBB())) {
+          Phi.removeOperand(i);
+          Phi.removeOperand(i - 1);
           ModifiedPHI = true;
         }
+      }
 
-      if (phi->getNumOperands() == 3) {
-        const MachineOperand &Input = phi->getOperand(1);
-        const MachineOperand &Output = phi->getOperand(0);
+      if (Phi.getNumOperands() == 3) {
+        const MachineOperand &Input = Phi.getOperand(1);
+        const MachineOperand &Output = Phi.getOperand(0);
         Register InputReg = Input.getReg();
         Register OutputReg = Output.getReg();
         assert(Output.getSubReg() == 0 && "Cannot have output subregister");
@@ -182,16 +180,13 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
             // insert a COPY instead of simply replacing the output
             // with the input.
             const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo();
-            BuildMI(BB, BB.getFirstNonPHI(), phi->getDebugLoc(),
+            BuildMI(BB, BB.getFirstNonPHI(), Phi.getDebugLoc(),
                     TII->get(TargetOpcode::COPY), OutputReg)
                 .addReg(InputReg, getRegState(Input), InputSub);
           }
-          phi++->eraseFromParent();
+          Phi.eraseFromParent();
         }
-        continue;
       }
-
-      ++phi;
     }
   }
 
diff --git a/llvm/lib/DWARFLinker/CMakeLists.txt b/llvm/lib/DWARFLinker/CMakeLists.txt
index 913fcba35b146..f720c5e844b36 100644
--- a/llvm/lib/DWARFLinker/CMakeLists.txt
+++ b/llvm/lib/DWARFLinker/CMakeLists.txt
@@ -14,6 +14,7 @@ add_llvm_component_library(LLVMDWARFLinker
   AsmPrinter
   BinaryFormat
   CodeGen
+  CodeGenTypes
   DebugInfoDWARF
   MC
   Object
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index 94a711a10fefa..3c6495a12a9bb 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -439,9 +439,16 @@ unsigned DWARFLinker::shouldKeepVariableDIE(AddressesMap &RelocMgr,
   // if the variable has a valid relocation, so that the DIEInfo is filled.
   // However, we don't want a static variable in a function to force us to keep
   // the enclosing function, unless requested explicitly.
-  const bool HasLiveMemoryLocation = RelocMgr.isLiveVariable(DIE, MyInfo);
-  if (!HasLiveMemoryLocation || ((Flags & TF_InFunctionScope) &&
-                                 !LLVM_UNLIKELY(Options.KeepFunctionForStatic)))
+  std::optional<int64_t> RelocAdjustment =
+      RelocMgr.getVariableRelocAdjustment(DIE);
+
+  if (RelocAdjustment) {
+    MyInfo.AddrAdjust = *RelocAdjustment;
+    MyInfo.InDebugMap = true;
+  }
+
+  if (!RelocAdjustment || ((Flags & TF_InFunctionScope) &&
+                           !LLVM_UNLIKELY(Options.KeepFunctionForStatic)))
     return Flags;
 
   if (Options.Verbose) {
@@ -468,9 +475,14 @@ unsigned DWARFLinker::shouldKeepSubprogramDIE(
     return Flags;
 
   assert(LowPc && "low_pc attribute is not an address.");
-  if (!RelocMgr.isLiveSubprogram(DIE, MyInfo))
+  std::optional<int64_t> RelocAdjustment =
+      RelocMgr.getSubprogramRelocAdjustment(DIE);
+  if (!RelocAdjustment)
     return Flags;
 
+  MyInfo.AddrAdjust = *RelocAdjustment;
+  MyInfo.InDebugMap = true;
+
   if (Options.Verbose) {
     outs() << "Keeping subprogram DIE:";
     DIDumpOptions DumpOpts;
diff --git a/llvm/lib/Demangle/Demangle.cpp b/llvm/lib/Demangle/Demangle.cpp
index 68eddde056021..87603603cb4ac 100644
--- a/llvm/lib/Demangle/Demangle.cpp
+++ b/llvm/lib/Demangle/Demangle.cpp
@@ -48,7 +48,7 @@ std::string llvm::demangle(const std::string &MangledName) {
 bool llvm::nonMicrosoftDemangle(const char *MangledName, std::string &Result) {
   char *Demangled = nullptr;
   if (isItaniumEncoding(MangledName))
-    Demangled = itaniumDemangle(MangledName, nullptr, nullptr, nullptr);
+    Demangled = itaniumDemangle(MangledName);
   else if (isRustEncoding(MangledName))
     Demangled = rustDemangle(MangledName);
   else if (isDLangEncoding(MangledName))
diff --git a/llvm/lib/Demangle/ItaniumDemangle.cpp b/llvm/lib/Demangle/ItaniumDemangle.cpp
index 7a7bcd36fc689..b1a8832f1b32a 100644
--- a/llvm/lib/Demangle/ItaniumDemangle.cpp
+++ b/llvm/lib/Demangle/ItaniumDemangle.cpp
@@ -365,33 +365,20 @@ class DefaultAllocator {
 
 using Demangler = itanium_demangle::ManglingParser<DefaultAllocator>;
 
-char *llvm::itaniumDemangle(const char *MangledName, char *Buf,
-                            size_t *N, int *Status) {
-  if (MangledName == nullptr || (Buf != nullptr && N == nullptr)) {
-    if (Status)
-      *Status = demangle_invalid_args;
+char *llvm::itaniumDemangle(const char *MangledName) {
+  if (!MangledName)
     return nullptr;
-  }
 
-  int InternalStatus = demangle_success;
   Demangler Parser(MangledName, MangledName + std::strlen(MangledName));
   Node *AST = Parser.parse();
+  if (!AST)
+    return nullptr;
 
-  if (AST == nullptr)
-    InternalStatus = demangle_invalid_mangled_name;
-  else {
-    OutputBuffer OB(Buf, N);
-    assert(Parser.ForwardTemplateRefs.empty());
-    AST->print(OB);
-    OB += '\0';
-    if (N != nullptr)
-      *N = OB.getCurrentPosition();
-    Buf = OB.getBuffer();
-  }
-
-  if (Status)
-    *Status = InternalStatus;
-  return InternalStatus == demangle_success ? Buf : nullptr;
+  OutputBuffer OB;
+  assert(Parser.ForwardTemplateRefs.empty());
+  AST->print(OB);
+  OB += '\0';
+  return OB.getBuffer();
 }
 
 ItaniumPartialDemangler::ItaniumPartialDemangler()
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index ab47f86859dd6..9df1126277e9a 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -58,6 +58,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
     ELFMovwAbsG1,
     ELFMovwAbsG2,
     ELFMovwAbsG3,
+    ELFCondBr19,
     ELFAbs32,
     ELFAbs64,
     ELFPrel32,
@@ -99,6 +100,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       return ELFMovwAbsG2;
     case ELF::R_AARCH64_MOVW_UABS_G3:
       return ELFMovwAbsG3;
+    case ELF::R_AARCH64_CONDBR19:
+      return ELFCondBr19;
     case ELF::R_AARCH64_ABS32:
       return ELFAbs32;
     case ELF::R_AARCH64_ABS64:
@@ -287,6 +290,16 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       Kind = aarch64::MoveWide16;
       break;
     }
+    case ELFCondBr19: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isCondBranchImm19(Instr) &&
+          !aarch64::isCompAndBranchImm19(Instr))
+        return make_error<JITLinkError>("R_AARCH64_CONDBR19 target is not a "
+                                        "conditional branch instruction");
+
+      Kind = aarch64::CondBranch19PCRel;
+      break;
+    }
     case ELFAbs32: {
       Kind = aarch64::Pointer32;
       break;
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index 1011fa81f750f..587b87d1d02f1 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -47,6 +47,8 @@ const char *getEdgeKindName(Edge::Kind R) {
     return "MoveWide16";
   case LDRLiteral19:
     return "LDRLiteral19";
+  case CondBranch19PCRel:
+    return "CondBranch19PCRel";
   case Page21:
     return "Page21";
   case PageOffset12:
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index c6eb12012a1e1..23a992da663df 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -908,7 +908,8 @@ Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections(
   // address in the first graph that it appears in, then verify and discard it
   // in all subsequent graphs. In this pass we preserve unconditionally -- we'll
   // manually throw it away in the processObjCImageInfo pass.
-  if (auto *ObjCImageInfoSec = G.findSectionByName("__DATA,__objc_imageinfo")) {
+  if (auto *ObjCImageInfoSec =
+          G.findSectionByName(MachOObjCImageInfoSectionName)) {
     if (ObjCImageInfoSec->blocks_size() != 1)
       return make_error<StringError>(
           "In " + G.getName() +
@@ -916,6 +917,13 @@ Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections(
           inconvertibleErrorCode());
     G.addAnonymousSymbol(**ObjCImageInfoSec->blocks().begin(), 0, 0, false,
                          true);
+
+    for (auto *B : ObjCImageInfoSec->blocks())
+      if (!B->edges_empty())
+        return make_error<StringError>("In " + G.getName() + ", " +
+                                           MachOObjCImageInfoSectionName +
+                                           " contains references to symbols",
+                                       inconvertibleErrorCode());
   }
 
   // Init sections are important: We need to preserve them and so that their
@@ -923,6 +931,11 @@ Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections(
   // registerObjectPlatformSections.
   JITLinkSymbolSet InitSectionSymbols;
   for (auto &InitSectionName : MachOInitSectionNames) {
+    // Skip ObjCImageInfo -- this shouldn't have any dependencies, and we may
+    // remove it later.
+    if (InitSectionName == MachOObjCImageInfoSectionName)
+      continue;
+
     // Skip non-init sections.
     auto *InitSection = G.findSectionByName(InitSectionName);
     if (!InitSection)
diff --git a/llvm/lib/FileCheck/FileCheckImpl.h b/llvm/lib/FileCheck/FileCheckImpl.h
index fd3568e7a5b06..3a3f4ecdfb90b 100644
--- a/llvm/lib/FileCheck/FileCheckImpl.h
+++ b/llvm/lib/FileCheck/FileCheckImpl.h
@@ -269,7 +269,7 @@ class NumericVariable {
   std::optional<ExpressionValue> Value;
 
   /// The input buffer's string from which Value was parsed, or std::nullopt.
-  /// See comments on getStringValue for a discussion of the None case.
+  /// See comments on getStringValue for a discussion of the std::nullopt case.
   std::optional<StringRef> StrValue;
 
   /// Line number where this variable is defined, or std::nullopt if defined
@@ -280,7 +280,7 @@ class NumericVariable {
 public:
   /// Constructor for a variable \p Name with implicit format \p ImplicitFormat
   /// defined at line \p DefLineNumber or defined before input is parsed if
-  /// \p DefLineNumber is None.
+  /// \p DefLineNumber is std::nullopt.
   explicit NumericVariable(StringRef Name, ExpressionFormat ImplicitFormat,
                            std::optional<size_t> DefLineNumber = std::nullopt)
       : Name(Name), ImplicitFormat(ImplicitFormat),
@@ -304,7 +304,7 @@ class NumericVariable {
 
   /// Sets value of this numeric variable to \p NewValue, and sets the input
   /// buffer string from which it was parsed to \p NewStrValue.  See comments on
-  /// getStringValue for a discussion of when the latter can be None.
+  /// getStringValue for a discussion of when the latter can be std::nullopt.
   void setValue(ExpressionValue NewValue,
                 std::optional<StringRef> NewStrValue = std::nullopt) {
     Value = NewValue;
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index cf329e40f5a44..0c3d15dd66fad 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4111,6 +4111,88 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
   return Builder.saveIP();
 }
 
+static Function *
+createOutlinedFunction(IRBuilderBase &Builder, StringRef FuncName,
+                       SmallVectorImpl<Value *> &Inputs,
+                       OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc) {
+  SmallVector<Type *> ParameterTypes;
+  for (auto &Arg : Inputs)
+    ParameterTypes.push_back(Arg->getType());
+
+  auto FuncType = FunctionType::get(Builder.getVoidTy(), ParameterTypes,
+                                    /*isVarArg*/ false);
+  auto Func = Function::Create(FuncType, GlobalValue::InternalLinkage, FuncName,
+                               Builder.GetInsertBlock()->getModule());
+
+  // Save insert point.
+  auto OldInsertPoint = Builder.saveIP();
+
+  // Generate the region into the function.
+  BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
+  Builder.SetInsertPoint(EntryBB);
+  Builder.restoreIP(CBFunc(Builder.saveIP(), Builder.saveIP()));
+
+  // Insert return instruction.
+  Builder.CreateRetVoid();
+
+  // Rewrite uses of input valus to parameters.
+  for (auto InArg : zip(Inputs, Func->args())) {
+    Value *Input = std::get<0>(InArg);
+    Argument &Arg = std::get<1>(InArg);
+
+    // Collect all the instructions
+    for (User *User : make_early_inc_range(Input->users()))
+      if (auto Instr = dyn_cast<Instruction>(User))
+        if (Instr->getFunction() == Func)
+          Instr->replaceUsesOfWith(Input, &Arg);
+  }
+
+  // Restore insert point.
+  Builder.restoreIP(OldInsertPoint);
+
+  return Func;
+}
+
+static void
+emitTargetOutlinedFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
+                           TargetRegionEntryInfo &EntryInfo,
+                           Function *&OutlinedFn, int32_t NumTeams,
+                           int32_t NumThreads, SmallVectorImpl<Value *> &Inputs,
+                           OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc) {
+
+  OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
+      [&Builder, &Inputs, &CBFunc](StringRef EntryFnName) {
+        return createOutlinedFunction(Builder, EntryFnName, Inputs, CBFunc);
+      };
+
+  Constant *OutlinedFnID;
+  OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateOutlinedFunction,
+                                      NumTeams, NumThreads, true, OutlinedFn,
+                                      OutlinedFnID);
+}
+
+static void emitTargetCall(IRBuilderBase &Builder, Function *OutlinedFn,
+                           SmallVectorImpl<Value *> &Args) {
+  // TODO: Add kernel launch call when device codegen is supported.
+  Builder.CreateCall(OutlinedFn, Args);
+}
+
+OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
+    const LocationDescription &Loc, OpenMPIRBuilder::InsertPointTy CodeGenIP,
+    TargetRegionEntryInfo &EntryInfo, int32_t NumTeams, int32_t NumThreads,
+    SmallVectorImpl<Value *> &Args, TargetBodyGenCallbackTy CBFunc) {
+  if (!updateToLocation(Loc))
+    return InsertPointTy();
+
+  Builder.restoreIP(CodeGenIP);
+
+  Function *OutlinedFn;
+  emitTargetOutlinedFunction(*this, Builder, EntryInfo, OutlinedFn, NumTeams,
+                             NumThreads, Args, CBFunc);
+  emitTargetCall(Builder, OutlinedFn, Args);
+  return Builder.saveIP();
+}
+
 std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
                                                    StringRef FirstSeparator,
                                                    StringRef Separator) {
diff --git a/llvm/lib/FuzzMutate/IRMutator.cpp b/llvm/lib/FuzzMutate/IRMutator.cpp
index 2d6c83e49a81c..37a26a8397f2f 100644
--- a/llvm/lib/FuzzMutate/IRMutator.cpp
+++ b/llvm/lib/FuzzMutate/IRMutator.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Transforms/Scalar/DCE.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include <map>
 #include <optional>
 
 using namespace llvm;
@@ -45,7 +46,10 @@ void IRMutationStrategy::mutate(Module &M, RandomIRBuilder &IB) {
 }
 
 void IRMutationStrategy::mutate(Function &F, RandomIRBuilder &IB) {
-  mutate(*makeSampler(IB.Rand, make_pointer_range(F)).getSelection(), IB);
+  auto Range = make_filter_range(make_pointer_range(F),
+                                 [](BasicBlock *BB) { return !BB->isEHPad(); });
+
+  mutate(*makeSampler(IB.Rand, Range).getSelection(), IB);
 }
 
 void IRMutationStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
@@ -566,56 +570,66 @@ void SinkInstructionStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
 }
 
 void ShuffleBlockStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
-
-  SmallPtrSet<Instruction *, 8> AliveInsts;
+  // A deterministic alternative to SmallPtrSet with the same lookup
+  // performance.
+  std::map<size_t, Instruction *> AliveInsts;
+  std::map<Instruction *, size_t> AliveInstsLookup;
+  size_t InsertIdx = 0;
   for (auto &I : make_early_inc_range(make_range(
            BB.getFirstInsertionPt(), BB.getTerminator()->getIterator()))) {
     // First gather all instructions that can be shuffled. Don't take
     // terminator.
-    AliveInsts.insert(&I);
+    AliveInsts.insert({InsertIdx, &I});
+    AliveInstsLookup.insert({&I, InsertIdx++});
     // Then remove these instructions from the block
     I.removeFromParent();
   }
 
   // Shuffle these instructions using topological sort.
-  // Returns true if all current instruction's dependencies in this block have
+  // Returns false if all current instruction's dependencies in this block have
   // been shuffled. If so, this instruction can be shuffled too.
-  auto hasAliveParent = [&AliveInsts](Instruction *I) {
-    for (Value *O : I->operands()) {
+  auto hasAliveParent = [&AliveInsts, &AliveInstsLookup](size_t Index) {
+    for (Value *O : AliveInsts[Index]->operands()) {
       Instruction *P = dyn_cast<Instruction>(O);
-      if (P && AliveInsts.count(P))
+      if (P && AliveInstsLookup.count(P))
         return true;
     }
     return false;
   };
   // Get all alive instructions that depend on the current instruction.
-  auto getAliveChildren = [&AliveInsts](Instruction *I) {
-    SmallPtrSet<Instruction *, 4> Children;
+  // Takes Instruction* instead of index because the instruction is already
+  // shuffled.
+  auto getAliveChildren = [&AliveInstsLookup](Instruction *I) {
+    SmallSetVector<size_t, 8> Children;
     for (Value *U : I->users()) {
       Instruction *P = dyn_cast<Instruction>(U);
-      if (P && AliveInsts.count(P))
-        Children.insert(P);
+      if (P && AliveInstsLookup.count(P))
+        Children.insert(AliveInstsLookup[P]);
     }
     return Children;
   };
-  SmallPtrSet<Instruction *, 8> Roots;
+  SmallSet<size_t, 8> RootIndices;
   SmallVector<Instruction *, 8> Insts;
-  for (Instruction *I : AliveInsts) {
-    if (!hasAliveParent(I))
-      Roots.insert(I);
+  for (const auto &[Index, Inst] : AliveInsts) {
+    if (!hasAliveParent(Index))
+      RootIndices.insert(Index);
   }
   // Topological sort by randomly selecting a node without a parent, or root.
-  while (!Roots.empty()) {
-    auto RS = makeSampler<Instruction *>(IB.Rand);
-    for (Instruction *Root : Roots)
-      RS.sample(Root, 1);
-    Instruction *Root = RS.getSelection();
-    Roots.erase(Root);
-    AliveInsts.erase(Root);
+  while (!RootIndices.empty()) {
+    auto RS = makeSampler<size_t>(IB.Rand);
+    for (size_t RootIdx : RootIndices)
+      RS.sample(RootIdx, 1);
+    size_t RootIdx = RS.getSelection();
+
+    RootIndices.erase(RootIdx);
+    Instruction *Root = AliveInsts[RootIdx];
+    AliveInsts.erase(RootIdx);
+    AliveInstsLookup.erase(Root);
     Insts.push_back(Root);
-    for (Instruction *Child : getAliveChildren(Root)) {
+
+    for (size_t Child : getAliveChildren(Root)) {
       if (!hasAliveParent(Child)) {
-        Roots.insert(Child);
+        RootIndices.insert(Child);
       }
     }
   }
diff --git a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
index 4359ae6c4616b..fff5cfe2ba6ef 100644
--- a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
+++ b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
@@ -262,8 +262,15 @@ static bool isCompatibleReplacement(const Instruction *I, const Use &Operand,
   case Instruction::Call:
   case Instruction::Invoke:
   case Instruction::CallBr: {
-    const CallBase *II = cast<CallBase>(I);
-    const Function *Callee = II->getCalledFunction();
+    const Function *Callee = cast<CallBase>(I)->getCalledFunction();
+    // If it's an indirect call, give up.
+    if (!Callee)
+      return false;
+    // If callee is not an intrinsic, operand 0 is the function to be called.
+    // Since we cannot assume that the replacement is a function pointer,
+    // we give up.
+    if (!Callee->getIntrinsicID() && OperandNo == 0)
+      return false;
     return !Callee->hasParamAttribute(OperandNo, Attribute::ImmArg);
   }
   default:
@@ -309,7 +316,7 @@ Instruction *RandomIRBuilder::connectToSink(BasicBlock &BB,
       std::shuffle(Dominators.begin(), Dominators.end(), Rand);
       for (BasicBlock *Dom : Dominators) {
         for (Instruction &I : *Dom) {
-          if (PointerType *PtrTy = dyn_cast<PointerType>(I.getType()))
+          if (isa<PointerType>(I.getType()))
             return new StoreInst(V, &I, Insts.back());
         }
       }
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index d6e36927e73ef..3e19b3c558c7d 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -425,8 +425,8 @@ static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
   bool FirstElt = true;
   if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
     Out << "zeroinitializer";
-  } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
-    Out << "undef";
+  } else if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) {
+    Out << "poison";
   } else {
     Out << "<";
     for (int Elt : Mask) {
@@ -435,8 +435,8 @@ static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
       else
         Out << ", ";
       Out << "i32 ";
-      if (Elt == UndefMaskElem)
-        Out << "undef";
+      if (Elt == PoisonMaskElem)
+        Out << "poison";
       else
         Out << Elt;
     }
@@ -3210,10 +3210,7 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) {
     printTypeIdInfo(*TIdInfo);
 
   // The AllocationType identifiers capture the profiled context behavior
-  // reaching a specific static allocation site (possibly cloned). Thus
-  // "notcoldandcold" implies there are multiple contexts which reach this site,
-  // some of which are cold and some of which are not, and that need to
-  // disambiguate via cloning or other context identification.
+  // reaching a specific static allocation site (possibly cloned).
   auto AllocTypeName = [](uint8_t Type) -> const char * {
     switch (Type) {
     case (uint8_t)AllocationType::None:
@@ -3222,8 +3219,6 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) {
       return "notcold";
     case (uint8_t)AllocationType::Cold:
       return "cold";
-    case (uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold:
-      return "notcoldandcold";
     }
     llvm_unreachable("Unexpected alloc type");
   };
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 57944c35836a2..c1c4c0fc81a52 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -2001,6 +2001,37 @@ AttributeMask AttributeFuncs::getUBImplyingAttributes() {
   return AM;
 }
 
+/// Callees with dynamic denormal modes are compatible with any caller mode.
+static bool denormModeCompatible(DenormalMode CallerMode,
+                                 DenormalMode CalleeMode) {
+  if (CallerMode == CalleeMode || CalleeMode == DenormalMode::getDynamic())
+    return true;
+
+  // If they don't exactly match, it's OK if the mismatched component is
+  // dynamic.
+  if (CalleeMode.Input == CallerMode.Input &&
+      CalleeMode.Output == DenormalMode::Dynamic)
+    return true;
+
+  if (CalleeMode.Output == CallerMode.Output &&
+      CalleeMode.Input == DenormalMode::Dynamic)
+    return true;
+  return false;
+}
+
+static bool checkDenormMode(const Function &Caller, const Function &Callee) {
+  DenormalMode CallerMode = Caller.getDenormalModeRaw();
+  DenormalMode CalleeMode = Callee.getDenormalModeRaw();
+
+  if (denormModeCompatible(CallerMode, CalleeMode)) {
+    DenormalMode CallerModeF32 = Caller.getDenormalModeF32Raw();
+    DenormalMode CalleeModeF32 = Callee.getDenormalModeF32Raw();
+    return denormModeCompatible(CallerModeF32, CalleeModeF32);
+  }
+
+  return false;
+}
+
 template<typename AttrClass>
 static bool isEqual(const Function &Caller, const Function &Callee) {
   return Caller.getFnAttribute(AttrClass::getKind()) ==
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 5c74acc0fd871..92153b05c6ea7 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/AutoUpgrade.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/Constants.h"
@@ -4963,9 +4964,10 @@ MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
 
 std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
   Triple T(TT);
-  // For AMDGPU we uprgrade older DataLayouts to include the default globals
-  // address space of 1.
-  if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) {
+  // The only data layout upgrades needed for pre-GCN are setting the address
+  // space of globals to 1.
+  if (T.isAMDGPU() && !T.isAMDGCN() && !DL.contains("-G") &&
+      !DL.startswith("G")) {
     return DL.empty() ? std::string("G1") : (DL + "-G1").str();
   }
 
@@ -4978,6 +4980,31 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
   }
 
   std::string Res = DL.str();
+  // AMDGCN data layout upgrades.
+  if (T.isAMDGCN()) {
+    // Define address spaces for constants.
+    if (!DL.contains("-G") && !DL.starts_with("G"))
+      Res.append(Res.empty() ? "G1" : "-G1");
+
+    // Add missing non-integral declarations.
+    // This goes before adding new address spaces to prevent incoherent string
+    // values.
+    if (!DL.contains("-ni") && !DL.startswith("ni"))
+      Res.append("-ni:7:8");
+    // Update ni:7 to ni:7:8.
+    if (DL.ends_with("ni:7"))
+      Res.append(":8");
+
+    // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
+    // resources) An empty data layout has already been upgraded to G1 by now.
+    if (!DL.contains("-p7") && !DL.startswith("p7"))
+      Res.append("-p7:160:256:256:32");
+    if (!DL.contains("-p8") && !DL.startswith("p8"))
+      Res.append("-p8:128:128");
+
+    return Res;
+  }
+
   if (!T.isX86())
     return Res;
 
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 59131a4264fc5..101e1de31b2b1 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -711,7 +711,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
   Type *EltTy = V1VTy->getElementType();
 
   // Undefined shuffle mask -> undefined value.
-  if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+  if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) {
     return UndefValue::get(VectorType::get(EltTy, MaskEltCount));
   }
 
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 4821ff56fe457..4aee5ce481fe0 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -3949,7 +3949,7 @@ int LLVMGetMaskValue(LLVMValueRef SVInst, unsigned Elt) {
   return I->getMaskValue(Elt);
 }
 
-int LLVMGetUndefMaskElem(void) { return UndefMaskElem; }
+int LLVMGetUndefMaskElem(void) { return PoisonMaskElem; }
 
 LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) {
   Value *P = unwrap(AtomicInst);
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 665c8cd631808..76feecae79b30 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -1919,7 +1919,9 @@ bool at::calculateFragmentIntersect(
 /// Return std::nullopt if any properties are not constants.
 static std::optional<AssignmentInfo>
 getAssignmentInfoImpl(const DataLayout &DL, const Value *StoreDest,
-                      uint64_t SizeInBits) {
+                      TypeSize SizeInBits) {
+  if (SizeInBits.isScalable())
+    return std::nullopt;
   APInt GEPOffset(DL.getIndexTypeSizeInBits(StoreDest->getType()), 0);
   const Value *Base = StoreDest->stripAndAccumulateConstantOffsets(
       DL, GEPOffset, /*AllowNonInbounds*/ true);
@@ -1941,19 +1943,18 @@ std::optional<AssignmentInfo> at::getAssignmentInfo(const DataLayout &DL,
     // We can't use a non-const size, bail.
     return std::nullopt;
   uint64_t SizeInBits = 8 * ConstLengthInBytes->getZExtValue();
-  return getAssignmentInfoImpl(DL, StoreDest, SizeInBits);
+  return getAssignmentInfoImpl(DL, StoreDest, TypeSize::getFixed(SizeInBits));
 }
 
 std::optional<AssignmentInfo> at::getAssignmentInfo(const DataLayout &DL,
                                                     const StoreInst *SI) {
-  const Value *StoreDest = SI->getPointerOperand();
-  uint64_t SizeInBits = DL.getTypeSizeInBits(SI->getValueOperand()->getType());
-  return getAssignmentInfoImpl(DL, StoreDest, SizeInBits);
+  TypeSize SizeInBits = DL.getTypeSizeInBits(SI->getValueOperand()->getType());
+  return getAssignmentInfoImpl(DL, SI->getPointerOperand(), SizeInBits);
 }
 
 std::optional<AssignmentInfo> at::getAssignmentInfo(const DataLayout &DL,
                                                     const AllocaInst *AI) {
-  uint64_t SizeInBits = DL.getTypeSizeInBits(AI->getAllocatedType());
+  TypeSize SizeInBits = DL.getTypeSizeInBits(AI->getAllocatedType());
   return getAssignmentInfoImpl(DL, AI, SizeInBits);
 }
 
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index a5da64e59e1d8..667cca3fb5cfd 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -2093,7 +2093,7 @@ void DIArgList::handleChangedOperand(void *Ref, Metadata *New) {
       if (NewVM)
         VM = NewVM;
       else
-        VM = ValueAsMetadata::get(UndefValue::get(VM->getValue()->getType()));
+        VM = ValueAsMetadata::get(PoisonValue::get(VM->getValue()->getType()));
     }
   }
   if (Uniq) {
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 3a214cc912093..300e0227b66f8 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -702,17 +702,30 @@ void Function::addDereferenceableOrNullParamAttr(unsigned ArgNo,
 
 DenormalMode Function::getDenormalMode(const fltSemantics &FPType) const {
   if (&FPType == &APFloat::IEEEsingle()) {
-    Attribute Attr = getFnAttribute("denormal-fp-math-f32");
-    StringRef Val = Attr.getValueAsString();
-    if (!Val.empty())
-      return parseDenormalFPAttribute(Val);
-
+    DenormalMode Mode = getDenormalModeF32Raw();
     // If the f32 variant of the attribute isn't specified, try to use the
     // generic one.
+    if (Mode.isValid())
+      return Mode;
   }
 
+  return getDenormalModeRaw();
+}
+
+DenormalMode Function::getDenormalModeRaw() const {
   Attribute Attr = getFnAttribute("denormal-fp-math");
-  return parseDenormalFPAttribute(Attr.getValueAsString());
+  StringRef Val = Attr.getValueAsString();
+  return parseDenormalFPAttribute(Val);
+}
+
+DenormalMode Function::getDenormalModeF32Raw() const {
+  Attribute Attr = getFnAttribute("denormal-fp-math-f32");
+  if (Attr.isValid()) {
+    StringRef Val = Attr.getValueAsString();
+    return parseDenormalFPAttribute(Val);
+  }
+
+  return DenormalMode::getInvalid();
 }
 
 const std::string &Function::getGC() const {
@@ -1023,70 +1036,11 @@ std::string Intrinsic::getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys) {
 /// IIT_Info - These are enumerators that describe the entries returned by the
 /// getIntrinsicInfoTableEntries function.
 ///
-/// NOTE: This must be kept in synch with the copy in TblGen/IntrinsicEmitter!
+/// Defined in Intrinsics.td.
 enum IIT_Info {
-  // Common values should be encoded with 0-15.
-  IIT_Done = 0,
-  IIT_I1 = 1,
-  IIT_I8 = 2,
-  IIT_I16 = 3,
-  IIT_I32 = 4,
-  IIT_I64 = 5,
-  IIT_F16 = 6,
-  IIT_F32 = 7,
-  IIT_F64 = 8,
-  IIT_V2 = 9,
-  IIT_V4 = 10,
-  IIT_V8 = 11,
-  IIT_V16 = 12,
-  IIT_V32 = 13,
-  IIT_PTR = 14,
-  IIT_ARG = 15,
-
-  // Values from 16+ are only encodable with the inefficient encoding.
-  IIT_V64 = 16,
-  IIT_MMX = 17,
-  IIT_TOKEN = 18,
-  IIT_METADATA = 19,
-  IIT_EMPTYSTRUCT = 20,
-  IIT_STRUCT2 = 21,
-  IIT_STRUCT3 = 22,
-  IIT_STRUCT4 = 23,
-  IIT_STRUCT5 = 24,
-  IIT_EXTEND_ARG = 25,
-  IIT_TRUNC_ARG = 26,
-  IIT_ANYPTR = 27,
-  IIT_V1 = 28,
-  IIT_VARARG = 29,
-  IIT_HALF_VEC_ARG = 30,
-  IIT_SAME_VEC_WIDTH_ARG = 31,
-  IIT_PTR_TO_ARG = 32,
-  IIT_PTR_TO_ELT = 33,
-  IIT_VEC_OF_ANYPTRS_TO_ELT = 34,
-  IIT_I128 = 35,
-  IIT_V512 = 36,
-  IIT_V1024 = 37,
-  IIT_STRUCT6 = 38,
-  IIT_STRUCT7 = 39,
-  IIT_STRUCT8 = 40,
-  IIT_F128 = 41,
-  IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43,
-  IIT_SUBDIVIDE2_ARG = 44,
-  IIT_SUBDIVIDE4_ARG = 45,
-  IIT_VEC_OF_BITCASTS_TO_INT = 46,
-  IIT_V128 = 47,
-  IIT_BF16 = 48,
-  IIT_STRUCT9 = 49,
-  IIT_V256 = 50,
-  IIT_AMX = 51,
-  IIT_PPCF128 = 52,
-  IIT_V3 = 53,
-  IIT_EXTERNREF = 54,
-  IIT_FUNCREF = 55,
-  IIT_ANYPTR_TO_ELT = 56,
-  IIT_I2 = 57,
-  IIT_I4 = 58,
+#define GET_INTRINSIC_IITINFO
+#include "llvm/IR/IntrinsicImpl.inc"
+#undef GET_INTRINSIC_IITINFO
 };
 
 static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index c72fdc8ba75e3..3983fd8ed7ff8 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2164,8 +2164,8 @@ void ShuffleVectorInst::commute() {
   SmallVector<int, 16> NewMask(NumMaskElts);
   for (int i = 0; i != NumMaskElts; ++i) {
     int MaskElt = getMaskValue(i);
-    if (MaskElt == UndefMaskElem) {
-      NewMask[i] = UndefMaskElem;
+    if (MaskElt == PoisonMaskElem) {
+      NewMask[i] = PoisonMaskElem;
       continue;
     }
     assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
@@ -2186,11 +2186,11 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
   int V1Size =
       cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
   for (int Elem : Mask)
-    if (Elem != UndefMaskElem && Elem >= V1Size * 2)
+    if (Elem != PoisonMaskElem && Elem >= V1Size * 2)
       return false;
 
   if (isa<ScalableVectorType>(V1->getType()))
-    if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !all_equal(Mask))
+    if ((Mask[0] != 0 && Mask[0] != PoisonMaskElem) || !all_equal(Mask))
       return false;
 
   return true;
@@ -2289,7 +2289,7 @@ Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
   }
   SmallVector<Constant *, 16> MaskConst;
   for (int Elem : Mask) {
-    if (Elem == UndefMaskElem)
+    if (Elem == PoisonMaskElem)
       MaskConst.push_back(UndefValue::get(Int32Ty));
     else
       MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
@@ -2627,7 +2627,7 @@ static bool isReplicationMaskWithParams(ArrayRef<int> Mask,
            "Run out of mask?");
     Mask = Mask.drop_front(ReplicationFactor);
     if (!all_of(CurrSubMask, [CurrElt](int MaskElt) {
-          return MaskElt == UndefMaskElem || MaskElt == CurrElt;
+          return MaskElt == PoisonMaskElem || MaskElt == CurrElt;
         }))
       return false;
   }
@@ -2639,7 +2639,7 @@ static bool isReplicationMaskWithParams(ArrayRef<int> Mask,
 bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask,
                                           int &ReplicationFactor, int &VF) {
   // undef-less case is trivial.
-  if (!llvm::is_contained(Mask, UndefMaskElem)) {
+  if (!llvm::is_contained(Mask, PoisonMaskElem)) {
     ReplicationFactor =
         Mask.take_while([](int MaskElt) { return MaskElt == 0; }).size();
     if (ReplicationFactor == 0 || Mask.size() % ReplicationFactor != 0)
@@ -2657,7 +2657,7 @@ bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask,
   // Before doing that, let's perform basic correctness checking first.
   int Largest = -1;
   for (int MaskElt : Mask) {
-    if (MaskElt == UndefMaskElem)
+    if (MaskElt == PoisonMaskElem)
       continue;
     // Elements must be in non-decreasing order.
     if (MaskElt < Largest)
@@ -2703,11 +2703,11 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF) {
     return false;
   for (unsigned K = 0, Sz = Mask.size(); K < Sz; K += VF) {
     ArrayRef<int> SubMask = Mask.slice(K, VF);
-    if (all_of(SubMask, [](int Idx) { return Idx == UndefMaskElem; }))
+    if (all_of(SubMask, [](int Idx) { return Idx == PoisonMaskElem; }))
       continue;
     SmallBitVector Used(VF, false);
     for_each(SubMask, [&Used, VF](int Idx) {
-      if (Idx != UndefMaskElem && Idx < VF)
+      if (Idx != PoisonMaskElem && Idx < VF)
         Used.set(Idx);
     });
     if (!Used.all())
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 9270d708a620a..36d56699c64e9 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -529,6 +529,20 @@ VPIntrinsic::getFunctionalOpcodeForVP(Intrinsic::ID ID) {
   return std::nullopt;
 }
 
+// Equivalent non-predicated constrained intrinsic
+std::optional<unsigned>
+VPIntrinsic::getConstrainedIntrinsicIDForVP(Intrinsic::ID ID) {
+  switch (ID) {
+  default:
+    break;
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
+#define VP_PROPERTY_CONSTRAINEDFP(HASRND, HASEXCEPT, CID) return Intrinsic::CID;
+#define END_REGISTER_VP_INTRINSIC(VPID) break;
+#include "llvm/IR/VPIntrinsics.def"
+  }
+  return std::nullopt;
+}
+
 Intrinsic::ID VPIntrinsic::getForOpcode(unsigned IROPC) {
   switch (IROPC) {
   default:
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index cfcfcd762fdc3..6ffeec1f21d33 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -1072,6 +1072,70 @@ MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
   return B;
 }
 
+// Call instructions with branch weights are only used in SamplePGO as
+// documented in
+/// https://llvm.org/docs/BranchWeightMetadata.html#callinst).
+MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
+                                            const Instruction *AInstr,
+                                            const Instruction *BInstr) {
+  assert(A && B && AInstr && BInstr && "Caller should guarantee");
+  auto &Ctx = AInstr->getContext();
+  MDBuilder MDHelper(Ctx);
+
+  // LLVM IR verifier verifies !prof metadata has at least 2 operands.
+  assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 &&
+         "!prof annotations should have no less than 2 operands");
+  MDString *AMDS = dyn_cast<MDString>(A->getOperand(0));
+  MDString *BMDS = dyn_cast<MDString>(B->getOperand(0));
+  // LLVM IR verfier verifies first operand is MDString.
+  assert(AMDS != nullptr && BMDS != nullptr &&
+         "first operand should be a non-null MDString");
+  StringRef AProfName = AMDS->getString();
+  StringRef BProfName = BMDS->getString();
+  if (AProfName.equals("branch_weights") &&
+      BProfName.equals("branch_weights")) {
+    ConstantInt *AInstrWeight =
+        mdconst::dyn_extract<ConstantInt>(A->getOperand(1));
+    ConstantInt *BInstrWeight =
+        mdconst::dyn_extract<ConstantInt>(B->getOperand(1));
+    assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
+    return MDNode::get(Ctx,
+                       {MDHelper.createString("branch_weights"),
+                        MDHelper.createConstant(ConstantInt::get(
+                            Type::getInt64Ty(Ctx),
+                            SaturatingAdd(AInstrWeight->getZExtValue(),
+                                          BInstrWeight->getZExtValue())))});
+  }
+  return nullptr;
+}
+
+// Pass in both instructions and nodes. Instruction information (e.g.,
+// instruction type) helps interpret profiles and make implementation clearer.
+MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
+                                      const Instruction *AInstr,
+                                      const Instruction *BInstr) {
+  if (!(A && B)) {
+    return A ? A : B;
+  }
+
+  assert(AInstr->getMetadata(LLVMContext::MD_prof) == A &&
+         "Caller should guarantee");
+  assert(BInstr->getMetadata(LLVMContext::MD_prof) == B &&
+         "Caller should guarantee");
+
+  const CallInst *ACall = dyn_cast<CallInst>(AInstr);
+  const CallInst *BCall = dyn_cast<CallInst>(BInstr);
+
+  // Both ACall and BCall are direct callsites.
+  if (ACall && BCall && ACall->getCalledFunction() &&
+      BCall->getCalledFunction())
+    return mergeDirectCallProfMetadata(A, B, AInstr, BInstr);
+
+  // The rest of the cases are not implemented but could be added
+  // when there are use cases.
+  return nullptr;
+}
+
 static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
   return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
 }
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 2d1440756a95c..99e5044bdb9b1 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -317,7 +317,7 @@ void ModuleSummaryIndex::propagateAttributes(
           }
 }
 
-bool ModuleSummaryIndex::canImportGlobalVar(GlobalValueSummary *S,
+bool ModuleSummaryIndex::canImportGlobalVar(const GlobalValueSummary *S,
                                             bool AnalyzeRefs) const {
   auto HasRefsPreventingImport = [this](const GlobalVarSummary *GVS) {
     // We don't analyze GV references during attribute propagation, so
diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp
index 077f8b6562f5e..8fcd337c6a0d7 100644
--- a/llvm/lib/IR/StructuralHash.cpp
+++ b/llvm/lib/IR/StructuralHash.cpp
@@ -19,19 +19,19 @@ namespace {
 // llvm/lib/Transforms/Utils/FunctionComparator.cpp
 
 class StructuralHashImpl {
-  uint64_t Hash = 0x6acaa36bef8325c5ULL;
+  hash_code Hash;
 
-  void update(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
+  template <typename T> void hash(const T &V) { Hash = hash_combine(Hash, V); }
 
 public:
-  StructuralHashImpl() = default;
+  StructuralHashImpl() : Hash(4) {}
 
   void update(const Function &F) {
     if (F.empty())
       return;
 
-    update(F.isVarArg());
-    update(F.arg_size());
+    hash(F.isVarArg());
+    hash(F.arg_size());
 
     SmallVector<const BasicBlock *, 8> BBs;
     SmallPtrSet<const BasicBlock *, 16> VisitedBBs;
@@ -40,9 +40,9 @@ class StructuralHashImpl {
     VisitedBBs.insert(BBs[0]);
     while (!BBs.empty()) {
       const BasicBlock *BB = BBs.pop_back_val();
-      update(45798); // Block header
+      hash(45798); // Block header
       for (auto &Inst : *BB)
-        update(Inst.getOpcode());
+        hash(Inst.getOpcode());
 
       const Instruction *Term = BB->getTerminator();
       for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
diff --git a/llvm/lib/LTO/CMakeLists.txt b/llvm/lib/LTO/CMakeLists.txt
index 6617d27d73e84..9872a576af102 100644
--- a/llvm/lib/LTO/CMakeLists.txt
+++ b/llvm/lib/LTO/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_component_library(LLVMLTO
   BitReader
   BitWriter
   CodeGen
+  CodeGenTypes
   Core
   Extensions
   IPO
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 15b6e22a683c1..fee09fcd76bdb 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1404,11 +1404,10 @@ ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism,
 // Given the original \p Path to an output file, replace any path
 // prefix matching \p OldPrefix with \p NewPrefix. Also, create the
 // resulting directory if it does not yet exist.
-std::string lto::getThinLTOOutputFile(const std::string &Path,
-                                      const std::string &OldPrefix,
-                                      const std::string &NewPrefix) {
+std::string lto::getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
+                                      StringRef NewPrefix) {
   if (OldPrefix.empty() && NewPrefix.empty())
-    return Path;
+    return std::string(Path);
   SmallString<128> NewPath(Path);
   llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
   StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
@@ -1447,13 +1446,13 @@ class WriteIndexesThinBackend : public ThinBackendProc {
       MapVector<StringRef, BitcodeModule> &ModuleMap) override {
     StringRef ModulePath = BM.getModuleIdentifier();
     std::string NewModulePath =
-        getThinLTOOutputFile(std::string(ModulePath), OldPrefix, NewPrefix);
+        getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
 
     if (LinkedObjectsFile) {
       std::string ObjectPrefix =
           NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
-      std::string LinkedObjectsFilePath = getThinLTOOutputFile(
-          std::string(ModulePath), OldPrefix, ObjectPrefix);
+      std::string LinkedObjectsFilePath =
+          getThinLTOOutputFile(ModulePath, OldPrefix, ObjectPrefix);
       *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
     }
 
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 6f21566a50f25..c11d28b746f60 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -772,9 +772,6 @@ bool MCAsmStreamer::emitSymbolAttribute(MCSymbol *Symbol,
   case MCSA_Memtag:
     OS << "\t.memtag\t";
     break;
-  case MCSA_WeakAntiDep:
-    OS << "\t.weak_anti_dep\t";
-    break;
   }
 
   Symbol->print(OS, MAI);
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 179fe23abc768..8d87bcb5dc202 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -464,14 +464,13 @@ void MCAsmLayout::layoutFragment(MCFragment *F) {
   }
 }
 
-void MCAssembler::registerSymbol(const MCSymbol &Symbol, bool *Created) {
-  bool New = !Symbol.isRegistered();
-  if (Created)
-    *Created = New;
-  if (New) {
+bool MCAssembler::registerSymbol(const MCSymbol &Symbol) {
+  bool Changed = !Symbol.isRegistered();
+  if (Changed) {
     Symbol.setIsRegistered(true);
     Symbols.push_back(&Symbol);
   }
+  return Changed;
 }
 
 void MCAssembler::writeFragmentPadding(raw_ostream &OS,
@@ -991,18 +990,11 @@ bool MCAssembler::relaxInstruction(MCAsmLayout &Layout,
   getBackend().relaxInstruction(Relaxed, *F.getSubtargetInfo());
 
   // Encode the new instruction.
-  //
-  // FIXME-PERF: If it matters, we could let the target do this. It can
-  // probably do so more efficiently in many cases.
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  getEmitter().encodeInstruction(Relaxed, Code, Fixups, *F.getSubtargetInfo());
-
-  // Update the fragment.
   F.setInst(Relaxed);
-  F.getContents() = Code;
-  F.getFixups() = Fixups;
-
+  F.getFixups().clear();
+  F.getContents().clear();
+  getEmitter().encodeInstruction(Relaxed, F.getContents(), F.getFixups(),
+                                 *F.getSubtargetInfo());
   return true;
 }
 
diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp
index aec8fc89acb23..a27ef64bec0fa 100644
--- a/llvm/lib/MC/MCCodeView.cpp
+++ b/llvm/lib/MC/MCCodeView.cpp
@@ -649,8 +649,6 @@ void CodeViewContext::encodeDefRange(MCAsmLayout &Layout,
       const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(RangeBegin, Ctx);
       const MCBinaryExpr *BE =
           MCBinaryExpr::createAdd(SRE, MCConstantExpr::create(Bias, Ctx), Ctx);
-      MCValue Res;
-      BE->evaluateAsRelocatable(Res, &Layout, /*Fixup=*/nullptr);
 
       // Each record begins with a 2-byte number indicating how large the record
       // is.
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 80a036a950bc9..6ec1a9ae8444d 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -881,11 +881,11 @@ MCSubtargetInfo &MCContext::getSubtargetCopy(const MCSubtargetInfo &STI) {
 
 void MCContext::addDebugPrefixMapEntry(const std::string &From,
                                        const std::string &To) {
-  DebugPrefixMap.insert(std::make_pair(From, To));
+  DebugPrefixMap.emplace_back(From, To);
 }
 
 void MCContext::remapDebugPath(SmallVectorImpl<char> &Path) {
-  for (const auto &[From, To] : DebugPrefixMap)
+  for (const auto &[From, To] : llvm::reverse(DebugPrefixMap))
     if (llvm::sys::path::replace_path_prefix(Path, From, To))
       break;
 }
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 30b8aadbf63c3..66984ce08ad52 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -1876,7 +1876,11 @@ void MCDwarfFrameEmitter::Emit(MCObjectStreamer &Streamer, MCAsmBackend *MAB,
     }
   }
 
-  if (!NeedsEHFrameSection) return;
+  // Compact unwind information can be emitted in the eh_frame section or the
+  // debug_frame section. Skip emitting FDEs and CIEs when the compact unwind
+  // doesn't need an eh_frame section and the emission location is the eh_frame
+  // section.
+  if (!NeedsEHFrameSection && IsEH) return;
 
   MCSection &Section =
       IsEH ? *const_cast<MCObjectFileInfo *>(MOFI)->getEHFrameSection()
@@ -1903,9 +1907,13 @@ void MCDwarfFrameEmitter::Emit(MCObjectStreamer &Streamer, MCAsmBackend *MAB,
     const MCDwarfFrameInfo &Frame = *I;
     ++I;
     if (CanOmitDwarf && Frame.CompactUnwindEncoding !=
-          MOFI->getCompactUnwindDwarfEHFrameOnly())
-      // Don't generate an EH frame if we don't need one. I.e., it's taken care
-      // of by the compact unwind encoding.
+          MOFI->getCompactUnwindDwarfEHFrameOnly() && IsEH)
+      // CIEs and FDEs can be emitted in either the eh_frame section or the
+      // debug_frame section, on some platforms (e.g. AArch64) the target object
+      // file supports emitting a compact_unwind section without an associated
+      // eh_frame section. If the eh_frame section is not needed, and the
+      // location where the CIEs and FDEs are to be emitted is the eh_frame
+      // section, do not emit anything.
       continue;
 
     CIEKey Key(Frame);
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 303eb16011d17..8dd002fba0e1a 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -216,7 +216,6 @@ bool MCELFStreamer::emitSymbolAttribute(MCSymbol *S, MCSymbolAttr Attribute) {
   case MCSA_Invalid:
   case MCSA_IndirectSymbol:
   case MCSA_Exported:
-  case MCSA_WeakAntiDep:
     return false;
 
   case MCSA_NoDeadStrip:
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index 68cd88f822066..04682fdc7ecec 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -761,9 +761,6 @@ bool MCExpr::evaluateAsValue(MCValue &Res, const MCAsmLayout &Layout) const {
 }
 
 static bool canExpand(const MCSymbol &Sym, bool InSet) {
-  if (Sym.isWeakExternal())
-    return false;
-
   const MCExpr *Expr = Sym.getVariableValue();
   const auto *Inner = dyn_cast<MCSymbolRefExpr>(Expr);
   if (Inner) {
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 6711ff9001a32..82cfcc3ac76cc 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -358,7 +358,6 @@ bool MCMachOStreamer::emitSymbolAttribute(MCSymbol *Sym,
   case MCSA_LGlobal:
   case MCSA_Exported:
   case MCSA_Memtag:
-  case MCSA_WeakAntiDep:
     return false;
 
   case MCSA_Global:
@@ -535,9 +534,7 @@ void MCMachOStreamer::finishImpl() {
 
 void MCMachOStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *&SRE) {
   const MCSymbol *S = &SRE->getSymbol();
-  bool Created;
-  getAssembler().registerSymbol(*S, &Created);
-  if (Created)
+  if (getAssembler().registerSymbol(*S))
     S->setExternal(true);
 }
 
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 254c747aca050..c49e51332b126 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -6371,7 +6371,7 @@ static bool isSymbolUsedInExpression(const MCSymbol *Sym, const MCExpr *Value) {
   case MCExpr::SymbolRef: {
     const MCSymbol &S =
         static_cast<const MCSymbolRefExpr *>(Value)->getSymbol();
-    if (S.isVariable() && !S.isWeakExternal())
+    if (S.isVariable())
       return isSymbolUsedInExpression(Sym, S.getVariableValue());
     return &S == Sym;
   }
diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index 3f88257d37db9..fc3d1e94e49f7 100644
--- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -67,7 +67,6 @@ class COFFAsmParser : public MCAsmParserExtension {
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveLinkOnce>(".linkonce");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveRVA>(".rva");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveSymbolAttribute>(".weak");
-    addDirectiveHandler<&COFFAsmParser::ParseDirectiveSymbolAttribute>(".weak_anti_dep");
     addDirectiveHandler<&COFFAsmParser::ParseDirectiveCGProfile>(".cg_profile");
 
     // Win64 EH directives.
@@ -282,7 +281,6 @@ bool COFFAsmParser::ParseSectionFlags(StringRef SectionName,
 bool COFFAsmParser::ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc) {
   MCSymbolAttr Attr = StringSwitch<MCSymbolAttr>(Directive)
     .Case(".weak", MCSA_Weak)
-    .Case(".weak_anti_dep", MCSA_WeakAntiDep)
     .Default(MCSA_Invalid);
   assert(Attr != MCSA_Invalid && "unexpected symbol attribute directive!");
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
diff --git a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
index 67311f1608f55..7c390041b3698 100644
--- a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
@@ -1130,7 +1130,7 @@ bool DarwinAsmParser::parseVersionMin(StringRef Directive, SMLoc Loc,
   if (isSDKVersionToken(getLexer().getTok()) && parseSDKVersion(SDKVersion))
     return true;
 
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(Twine(" in '") + Directive + "' directive");
 
   Triple::OSType ExpectedOS = getOSTypeFromMCVM(Type);
@@ -1191,7 +1191,7 @@ bool DarwinAsmParser::parseBuildVersion(StringRef Directive, SMLoc Loc) {
   if (isSDKVersionToken(getLexer().getTok()) && parseSDKVersion(SDKVersion))
     return true;
 
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(" in '.build_version' directive");
 
   Triple::OSType ExpectedOS
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index 211a10eba792a..d522b77c25874 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -4537,7 +4537,7 @@ bool MasmParser::parseDirectiveStruct(StringRef Directive,
                                      "' directive; expected none or NONUNIQUE");
   }
 
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
 
   StructInProgress.emplace_back(Name, DirKind == DK_UNION, AlignmentValue);
@@ -4559,7 +4559,7 @@ bool MasmParser::parseDirectiveNestedStruct(StringRef Directive,
     Name = getTok().getIdentifier();
     parseToken(AsmToken::Identifier);
   }
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(" in '" + Twine(Directive) + "' directive");
 
   // Reserve space to ensure Alignment doesn't get invalidated when
@@ -4585,7 +4585,7 @@ bool MasmParser::parseDirectiveEnds(StringRef Name, SMLoc NameLoc) {
       Structure.Size, std::min(Structure.Alignment, Structure.AlignmentSize));
   Structs[Name.lower()] = Structure;
 
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(" in ENDS directive");
 
   return false;
@@ -4597,7 +4597,7 @@ bool MasmParser::parseDirectiveNestedEnds() {
   if (StructInProgress.size() == 1)
     return TokError("missing name in top-level ENDS directive");
 
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(" in nested ENDS directive");
 
   StructInfo Structure = StructInProgress.pop_back_val();
@@ -4669,7 +4669,7 @@ bool MasmParser::parseDirectiveOrg() {
   SMLoc OffsetLoc = Lexer.getLoc();
   if (checkForValidSection() || parseExpression(Offset))
     return true;
-  if (parseToken(AsmToken::EndOfStatement))
+  if (parseEOL())
     return addErrorSuffix(" in 'org' directive");
 
   if (StructInProgress.empty()) {
@@ -4738,10 +4738,9 @@ bool MasmParser::parseDirectiveAlign() {
   if (getTok().is(AsmToken::EndOfStatement)) {
     return Warning(AlignmentLoc,
                    "align directive with no operand is ignored") &&
-           parseToken(AsmToken::EndOfStatement);
+           parseEOL();
   }
-  if (parseAbsoluteExpression(Alignment) ||
-      parseToken(AsmToken::EndOfStatement))
+  if (parseAbsoluteExpression(Alignment) || parseEOL())
     return addErrorSuffix(" in align directive");
 
   // Always emit an alignment here even if we throw an error.
@@ -4764,7 +4763,7 @@ bool MasmParser::parseDirectiveAlign() {
 /// parseDirectiveEven
 ///  ::= even
 bool MasmParser::parseDirectiveEven() {
-  if (parseToken(AsmToken::EndOfStatement) || emitAlignTo(2))
+  if (parseEOL() || emitAlignTo(2))
     return addErrorSuffix(" in even directive");
 
   return false;
@@ -5482,7 +5481,7 @@ bool MasmParser::parseDirectiveCFIStartProc() {
   if (!parseOptionalToken(AsmToken::EndOfStatement)) {
     if (check(parseIdentifier(Simple) || Simple != "simple",
               "unexpected token") ||
-        parseToken(AsmToken::EndOfStatement))
+        parseEOL())
       return addErrorSuffix(" in '.cfi_startproc' directive");
   }
 
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 4dd3163fd399d..8f29153783066 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -219,7 +219,8 @@ void MCStreamer::emitGPRel32Value(const MCExpr *Value) {
 /// Emit NumBytes bytes worth of the value specified by FillValue.
 /// This implements directives such as '.space'.
 void MCStreamer::emitFill(uint64_t NumBytes, uint8_t FillValue) {
-  emitFill(*MCConstantExpr::create(NumBytes, getContext()), FillValue);
+  if (NumBytes)
+    emitFill(*MCConstantExpr::create(NumBytes, getContext()), FillValue);
 }
 
 void llvm::MCStreamer::emitNops(int64_t NumBytes, int64_t ControlledNopLen,
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index ad0b1c157278a..36b29b792420f 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -115,11 +115,7 @@ bool MCWinCOFFStreamer::emitSymbolAttribute(MCSymbol *S,
   default: return false;
   case MCSA_WeakReference:
   case MCSA_Weak:
-    Symbol->setIsWeakExternal(COFF::IMAGE_WEAK_EXTERN_SEARCH_ALIAS);
-    Symbol->setExternal(true);
-    break;
-  case MCSA_WeakAntiDep:
-    Symbol->setIsWeakExternal(COFF::IMAGE_WEAK_EXTERN_ANTI_DEPENDENCY);
+    Symbol->setIsWeakExternal();
     Symbol->setExternal(true);
     break;
   case MCSA_Global:
@@ -348,9 +344,7 @@ void MCWinCOFFStreamer::emitCGProfileEntry(const MCSymbolRefExpr *From,
 
 void MCWinCOFFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *&SRE) {
   const MCSymbol *S = &SRE->getSymbol();
-  bool Created;
-  getAssembler().registerSymbol(*S, &Created);
-  if (Created)
+  if (getAssembler().registerSymbol(*S))
     cast<MCSymbolCOFF>(S)->setExternal(true);
 }
 
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 446d1372fa669..6b263df92cbeb 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -531,9 +531,7 @@ void MachObjectWriter::bindIndirectSymbols(MCAssembler &Asm) {
     // Set the symbol type to undefined lazy, but only on construction.
     //
     // FIXME: Do not hardcode.
-    bool Created;
-    Asm.registerSymbol(*it->Symbol, &Created);
-    if (Created)
+    if (Asm.registerSymbol(*it->Symbol))
       cast<MCSymbolMachO>(it->Symbol)->setReferenceTypeUndefinedLazy(true);
   }
 }
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index c18a963280911..07de8cdb98ac7 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -414,9 +414,9 @@ void WinCOFFObjectWriter::DefineSymbol(const MCSymbol &MCSym,
     Sym->Aux.resize(1);
     memset(&Sym->Aux[0], 0, sizeof(Sym->Aux[0]));
     Sym->Aux[0].AuxType = ATWeakExternal;
-    Sym->Aux[0].Aux.WeakExternal.TagIndex = 0; // Filled in later
+    Sym->Aux[0].Aux.WeakExternal.TagIndex = 0;
     Sym->Aux[0].Aux.WeakExternal.Characteristics =
-        cast<MCSymbolCOFF>(MCSym).getWeakExternalCharacteristics();
+        COFF::IMAGE_WEAK_EXTERN_SEARCH_ALIAS;
   } else {
     if (!Base)
       Sym->Data.SectionNumber = COFF::IMAGE_SYM_ABSOLUTE;
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index e9b88115195f4..1a540e261b2af 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -76,9 +76,11 @@ object::Archive::Kind NewArchiveMember::detectKindFromObject() const {
     if (auto ObjOrErr = object::SymbolicFile::createSymbolicFile(
             MemBufferRef, file_magic::bitcode, &Context)) {
       auto &IRObject = cast<object::IRObjectFile>(**ObjOrErr);
-      return Triple(IRObject.getTargetTriple()).isOSDarwin()
+      auto TargetTriple = Triple(IRObject.getTargetTriple());
+      return TargetTriple.isOSDarwin()
                  ? object::Archive::K_DARWIN
-                 : object::Archive::K_GNU;
+                 : (TargetTriple.isOSAIX() ? object::Archive::K_AIXBIG
+                                           : object::Archive::K_GNU);
     } else {
       // Squelch the error in case this was not a SymbolicFile.
       consumeError(ObjOrErr.takeError());
diff --git a/llvm/lib/Object/CMakeLists.txt b/llvm/lib/Object/CMakeLists.txt
index c9a93f398a9f2..bfb420e57a7f4 100644
--- a/llvm/lib/Object/CMakeLists.txt
+++ b/llvm/lib/Object/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_component_library(LLVMObject
   ELF.cpp
   ELFObjectFile.cpp
   Error.cpp
+  GOFFObjectFile.cpp
   FaultMapParser.cpp
   IRObjectFile.cpp
   IRSymtab.cpp
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index 8149df5ebbfa5..be5886258281b 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -624,7 +624,8 @@ ELFObjectFileBase::getPltAddresses() const {
       T->createMCInstrAnalysis(MII.get()));
   if (!MIA)
     return {};
-  std::optional<SectionRef> Plt, RelaPlt, GotPlt;
+  std::optional<SectionRef> Plt, RelaPlt;
+  uint64_t GotBaseVA = 0;
   for (const SectionRef &Section : sections()) {
     Expected<StringRef> NameOrErr = Section.getName();
     if (!NameOrErr) {
@@ -638,22 +639,27 @@ ELFObjectFileBase::getPltAddresses() const {
     else if (Name == ".rela.plt" || Name == ".rel.plt")
       RelaPlt = Section;
     else if (Name == ".got.plt")
-      GotPlt = Section;
+      GotBaseVA = Section.getAddress();
   }
-  if (!Plt || !RelaPlt || !GotPlt)
+  if (!Plt || !RelaPlt)
     return {};
   Expected<StringRef> PltContents = Plt->getContents();
   if (!PltContents) {
     consumeError(PltContents.takeError());
     return {};
   }
-  auto PltEntries = MIA->findPltEntries(Plt->getAddress(),
-                                        arrayRefFromStringRef(*PltContents),
-                                        GotPlt->getAddress(), Triple);
+  auto PltEntries = MIA->findPltEntries(
+      Plt->getAddress(), arrayRefFromStringRef(*PltContents), Triple);
+
   // Build a map from GOT entry virtual address to PLT entry virtual address.
   DenseMap<uint64_t, uint64_t> GotToPlt;
-  for (const auto &Entry : PltEntries)
-    GotToPlt.insert(std::make_pair(Entry.second, Entry.first));
+  for (auto [Plt, GotPltEntry] : PltEntries) {
+    // An x86-32 PIC PLT uses jmp DWORD PTR [ebx-offset]. Add
+    // _GLOBAL_OFFSET_TABLE_ (EBX) to get the .got.plt (or .got) entry address.
+    if (static_cast<int64_t>(GotPltEntry) < 0 && getEMachine() == ELF::EM_386)
+      GotPltEntry = ~GotPltEntry + GotBaseVA;
+    GotToPlt.insert(std::make_pair(GotPltEntry, Plt));
+  }
   // Find the relocations in the dynamic relocation table that point to
   // locations in the GOT for which we know the corresponding PLT entry.
   std::vector<std::pair<std::optional<DataRefImpl>, uint64_t>> Result;
diff --git a/llvm/lib/Object/GOFFObjectFile.cpp b/llvm/lib/Object/GOFFObjectFile.cpp
new file mode 100644
index 0000000000000..76a13559ebfe3
--- /dev/null
+++ b/llvm/lib/Object/GOFFObjectFile.cpp
@@ -0,0 +1,483 @@
+//===- GOFFObjectFile.cpp - GOFF object file implementation -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of the GOFFObjectFile class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/GOFFObjectFile.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/Object/GOFF.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/raw_ostream.h"
+
+#ifndef DEBUG_TYPE
+#define DEBUG_TYPE "goff"
+#endif
+
+using namespace llvm::object;
+using namespace llvm;
+
+Expected<std::unique_ptr<ObjectFile>>
+ObjectFile::createGOFFObjectFile(MemoryBufferRef Object) {
+  Error Err = Error::success();
+  std::unique_ptr<GOFFObjectFile> Ret(new GOFFObjectFile(Object, Err));
+  if (Err)
+    return std::move(Err);
+  return std::move(Ret);
+}
+
+GOFFObjectFile::GOFFObjectFile(MemoryBufferRef Object, Error &Err)
+    : ObjectFile(Binary::ID_GOFF, Object) {
+  ErrorAsOutParameter ErrAsOutParam(&Err);
+  // Object file isn't the right size, bail out early.
+  if ((Object.getBufferSize() % GOFF::RecordLength) != 0) {
+    Err = createStringError(
+        object_error::unexpected_eof,
+        "object file is not the right size. Must be a multiple "
+        "of 80 bytes, but is " +
+            std::to_string(Object.getBufferSize()) + " bytes");
+    return;
+  }
+  // Object file doesn't start/end with HDR/END records.
+  // Bail out early.
+  if (Object.getBufferSize() != 0) {
+    if ((base()[1] & 0xF0) >> 4 != GOFF::RT_HDR) {
+      Err = createStringError(object_error::parse_failed,
+                              "object file must start with HDR record");
+      return;
+    }
+    if ((base()[Object.getBufferSize() - GOFF::RecordLength + 1] & 0xF0) >> 4 !=
+        GOFF::RT_END) {
+      Err = createStringError(object_error::parse_failed,
+                              "object file must end with END record");
+      return;
+    }
+  }
+
+  SectionEntryImpl DummySection;
+  SectionList.emplace_back(DummySection); // Dummy entry at index 0.
+
+  uint8_t PrevRecordType = 0;
+  uint8_t PrevContinuationBits = 0;
+  const uint8_t *End = reinterpret_cast<const uint8_t *>(Data.getBufferEnd());
+  for (const uint8_t *I = base(); I < End; I += GOFF::RecordLength) {
+    uint8_t RecordType = (I[1] & 0xF0) >> 4;
+    bool IsContinuation = I[1] & 0x02;
+    bool PrevWasContinued = PrevContinuationBits & 0x01;
+    size_t RecordNum = (I - base()) / GOFF::RecordLength;
+
+    // If the previous record was continued, the current record should be a
+    // continuation.
+    if (PrevWasContinued && !IsContinuation) {
+      if (PrevRecordType == RecordType) {
+        Err = createStringError(object_error::parse_failed,
+                                "record " + std::to_string(RecordNum) +
+                                    " is not a continuation record but the "
+                                    "preceding record is continued");
+        return;
+      }
+    }
+    // Don't parse continuations records, only parse initial record.
+    if (IsContinuation) {
+      if (RecordType != PrevRecordType) {
+        Err = createStringError(object_error::parse_failed,
+                                "record " + std::to_string(RecordNum) +
+                                    " is a continuation record that does not "
+                                    "match the type of the previous record");
+        return;
+      }
+      if (!PrevWasContinued) {
+        Err = createStringError(object_error::parse_failed,
+                                "record " + std::to_string(RecordNum) +
+                                    " is a continuation record that is not "
+                                    "preceded by a continued record");
+        return;
+      }
+      PrevRecordType = RecordType;
+      PrevContinuationBits = I[1] & 0x03;
+      continue;
+    }
+
+#ifndef NDEBUG
+    for (size_t J = 0; J < GOFF::RecordLength; ++J) {
+      const uint8_t *P = I + J;
+      if (J % 8 == 0)
+        dbgs() << "  ";
+
+      dbgs() << format("%02hhX", *P);
+    }
+#endif
+    switch (RecordType) {
+    case GOFF::RT_ESD: {
+      // Save ESD record.
+      uint32_t EsdId;
+      ESDRecord::getEsdId(I, EsdId);
+      EsdPtrs.grow(EsdId);
+      EsdPtrs[EsdId] = I;
+
+      // Determine and save the "sections" in GOFF.
+      // A section is saved as a tuple of the form
+      // case (1): (ED,child PR)
+      //    - where the PR must have non-zero length.
+      // case (2a) (ED,0)
+      //   - where the ED is of non-zero length.
+      // case (2b) (ED,0)
+      //   - where the ED is zero length but
+      //     contains a label (LD).
+      GOFF::ESDSymbolType SymbolType;
+      ESDRecord::getSymbolType(I, SymbolType);
+      SectionEntryImpl Section;
+      uint32_t Length;
+      ESDRecord::getLength(I, Length);
+      if (SymbolType == GOFF::ESD_ST_ElementDefinition) {
+        // case (2a)
+        if (Length != 0) {
+          Section.d.a = EsdId;
+          SectionList.emplace_back(Section);
+        }
+      } else if (SymbolType == GOFF::ESD_ST_PartReference) {
+        // case (1)
+        if (Length != 0) {
+          uint32_t SymEdId;
+          ESDRecord::getParentEsdId(I, SymEdId);
+          Section.d.a = SymEdId;
+          Section.d.b = EsdId;
+          SectionList.emplace_back(Section);
+        }
+      } else if (SymbolType == GOFF::ESD_ST_LabelDefinition) {
+        // case (2b)
+        uint32_t SymEdId;
+        ESDRecord::getParentEsdId(I, SymEdId);
+        const uint8_t *SymEdRecord = EsdPtrs[SymEdId];
+        uint32_t EdLength;
+        ESDRecord::getLength(SymEdRecord, EdLength);
+        if (!EdLength) { // [ EDID, PRID ]
+          // LD child of a zero length parent ED.
+          // Add the section ED which was previously ignored.
+          Section.d.a = SymEdId;
+          SectionList.emplace_back(Section);
+        }
+      }
+      LLVM_DEBUG(dbgs() << "  --  ESD " << EsdId << "\n");
+      break;
+    }
+    case GOFF::RT_END:
+      LLVM_DEBUG(dbgs() << "  --  END (GOFF record type) unhandled\n");
+      break;
+    case GOFF::RT_HDR:
+      LLVM_DEBUG(dbgs() << "  --  HDR (GOFF record type) unhandled\n");
+      break;
+    default:
+      llvm_unreachable("Unknown record type");
+    }
+    PrevRecordType = RecordType;
+    PrevContinuationBits = I[1] & 0x03;
+  }
+}
+
+const uint8_t *GOFFObjectFile::getSymbolEsdRecord(DataRefImpl Symb) const {
+  const uint8_t *EsdRecord = EsdPtrs[Symb.d.a];
+  return EsdRecord;
+}
+
+Expected<StringRef> GOFFObjectFile::getSymbolName(DataRefImpl Symb) const {
+  if (EsdNamesCache.count(Symb.d.a)) {
+    auto &StrPtr = EsdNamesCache[Symb.d.a];
+    return StringRef(StrPtr.second.get(), StrPtr.first);
+  }
+
+  SmallString<256> SymbolName;
+  if (auto Err = ESDRecord::getData(getSymbolEsdRecord(Symb), SymbolName))
+    return std::move(Err);
+
+  SmallString<256> SymbolNameConverted;
+  ConverterEBCDIC::convertToUTF8(SymbolName, SymbolNameConverted);
+
+  size_t Size = SymbolNameConverted.size();
+  auto StrPtr = std::make_pair(Size, std::make_unique<char[]>(Size));
+  char *Buf = StrPtr.second.get();
+  memcpy(Buf, SymbolNameConverted.data(), Size);
+  EsdNamesCache[Symb.d.a] = std::move(StrPtr);
+  return StringRef(Buf, Size);
+}
+
+Expected<StringRef> GOFFObjectFile::getSymbolName(SymbolRef Symbol) const {
+  return getSymbolName(Symbol.getRawDataRefImpl());
+}
+
+Expected<uint64_t> GOFFObjectFile::getSymbolAddress(DataRefImpl Symb) const {
+  uint32_t Offset;
+  const uint8_t *EsdRecord = getSymbolEsdRecord(Symb);
+  ESDRecord::getOffset(EsdRecord, Offset);
+  return static_cast<uint64_t>(Offset);
+}
+
+uint64_t GOFFObjectFile::getSymbolValueImpl(DataRefImpl Symb) const {
+  uint32_t Offset;
+  const uint8_t *EsdRecord = getSymbolEsdRecord(Symb);
+  ESDRecord::getOffset(EsdRecord, Offset);
+  return static_cast<uint64_t>(Offset);
+}
+
+uint64_t GOFFObjectFile::getCommonSymbolSizeImpl(DataRefImpl Symb) const {
+  return 0;
+}
+
+bool GOFFObjectFile::isSymbolUnresolved(DataRefImpl Symb) const {
+  const uint8_t *Record = getSymbolEsdRecord(Symb);
+  GOFF::ESDSymbolType SymbolType;
+  ESDRecord::getSymbolType(Record, SymbolType);
+
+  if (SymbolType == GOFF::ESD_ST_ExternalReference)
+    return true;
+  if (SymbolType == GOFF::ESD_ST_PartReference) {
+    uint32_t Length;
+    ESDRecord::getLength(Record, Length);
+    if (Length == 0)
+      return true;
+  }
+  return false;
+}
+
+bool GOFFObjectFile::isSymbolIndirect(DataRefImpl Symb) const {
+  const uint8_t *Record = getSymbolEsdRecord(Symb);
+  bool Indirect;
+  ESDRecord::getIndirectReference(Record, Indirect);
+  return Indirect;
+}
+
+Expected<uint32_t> GOFFObjectFile::getSymbolFlags(DataRefImpl Symb) const {
+  uint32_t Flags = 0;
+  if (isSymbolUnresolved(Symb))
+    Flags |= SymbolRef::SF_Undefined;
+
+  const uint8_t *Record = getSymbolEsdRecord(Symb);
+
+  GOFF::ESDBindingStrength BindingStrength;
+  ESDRecord::getBindingStrength(Record, BindingStrength);
+  if (BindingStrength == GOFF::ESD_BST_Weak)
+    Flags |= SymbolRef::SF_Weak;
+
+  GOFF::ESDBindingScope BindingScope;
+  ESDRecord::getBindingScope(Record, BindingScope);
+
+  if (BindingScope != GOFF::ESD_BSC_Section) {
+    Expected<StringRef> Name = getSymbolName(Symb);
+    if (Name && *Name != " ") { // Blank name is local.
+      Flags |= SymbolRef::SF_Global;
+      if (BindingScope == GOFF::ESD_BSC_ImportExport)
+        Flags |= SymbolRef::SF_Exported;
+      else if (!(Flags & SymbolRef::SF_Undefined))
+        Flags |= SymbolRef::SF_Hidden;
+    }
+  }
+
+  return Flags;
+}
+
+Expected<SymbolRef::Type>
+GOFFObjectFile::getSymbolType(DataRefImpl Symb) const {
+  const uint8_t *Record = getSymbolEsdRecord(Symb);
+  GOFF::ESDSymbolType SymbolType;
+  ESDRecord::getSymbolType(Record, SymbolType);
+  GOFF::ESDExecutable Executable;
+  ESDRecord::getExecutable(Record, Executable);
+
+  if (SymbolType != GOFF::ESD_ST_SectionDefinition &&
+      SymbolType != GOFF::ESD_ST_ElementDefinition &&
+      SymbolType != GOFF::ESD_ST_LabelDefinition &&
+      SymbolType != GOFF::ESD_ST_PartReference &&
+      SymbolType != GOFF::ESD_ST_ExternalReference) {
+    uint32_t EsdId;
+    ESDRecord::getEsdId(Record, EsdId);
+    return createStringError(llvm::errc::invalid_argument,
+                             "ESD record %" PRIu32
+                             " has invalid symbol type 0x%02" PRIX8,
+                             EsdId, SymbolType);
+  }
+  switch (SymbolType) {
+  case GOFF::ESD_ST_SectionDefinition:
+  case GOFF::ESD_ST_ElementDefinition:
+    return SymbolRef::ST_Other;
+  case GOFF::ESD_ST_LabelDefinition:
+  case GOFF::ESD_ST_PartReference:
+  case GOFF::ESD_ST_ExternalReference:
+    if (Executable != GOFF::ESD_EXE_CODE && Executable != GOFF::ESD_EXE_DATA &&
+        Executable != GOFF::ESD_EXE_Unspecified) {
+      uint32_t EsdId;
+      ESDRecord::getEsdId(Record, EsdId);
+      return createStringError(llvm::errc::invalid_argument,
+                               "ESD record %" PRIu32
+                               " has unknown Executable type 0x%02X",
+                               EsdId, Executable);
+    }
+    switch (Executable) {
+    case GOFF::ESD_EXE_CODE:
+      return SymbolRef::ST_Function;
+    case GOFF::ESD_EXE_DATA:
+      return SymbolRef::ST_Data;
+    case GOFF::ESD_EXE_Unspecified:
+      return SymbolRef::ST_Unknown;
+    }
+    llvm_unreachable("Unhandled ESDExecutable");
+  }
+  llvm_unreachable("Unhandled ESDSymbolType");
+}
+
+Expected<section_iterator>
+GOFFObjectFile::getSymbolSection(DataRefImpl Symb) const {
+  DataRefImpl Sec;
+
+  if (isSymbolUnresolved(Symb))
+    return section_iterator(SectionRef(Sec, this));
+
+  const uint8_t *SymEsdRecord = EsdPtrs[Symb.d.a];
+  uint32_t SymEdId;
+  ESDRecord::getParentEsdId(SymEsdRecord, SymEdId);
+  const uint8_t *SymEdRecord = EsdPtrs[SymEdId];
+
+  for (size_t I = 0, E = SectionList.size(); I < E; ++I) {
+    bool Found;
+    const uint8_t *SectionPrRecord = getSectionPrEsdRecord(I);
+    if (SectionPrRecord) {
+      Found = SymEsdRecord == SectionPrRecord;
+    } else {
+      const uint8_t *SectionEdRecord = getSectionEdEsdRecord(I);
+      Found = SymEdRecord == SectionEdRecord;
+    }
+
+    if (Found) {
+      Sec.d.a = I;
+      return section_iterator(SectionRef(Sec, this));
+    }
+  }
+  return createStringError(llvm::errc::invalid_argument,
+                           "symbol with ESD id " + std::to_string(Symb.d.a) +
+                               " refers to invalid section with ESD id " +
+                               std::to_string(SymEdId));
+}
+
+const uint8_t *GOFFObjectFile::getSectionEdEsdRecord(DataRefImpl &Sec) const {
+  SectionEntryImpl EsdIds = SectionList[Sec.d.a];
+  const uint8_t *EsdRecord = EsdPtrs[EsdIds.d.a];
+  return EsdRecord;
+}
+
+const uint8_t *GOFFObjectFile::getSectionPrEsdRecord(DataRefImpl &Sec) const {
+  SectionEntryImpl EsdIds = SectionList[Sec.d.a];
+  const uint8_t *EsdRecord = nullptr;
+  if (EsdIds.d.b)
+    EsdRecord = EsdPtrs[EsdIds.d.b];
+  return EsdRecord;
+}
+
+const uint8_t *
+GOFFObjectFile::getSectionEdEsdRecord(uint32_t SectionIndex) const {
+  DataRefImpl Sec;
+  Sec.d.a = SectionIndex;
+  const uint8_t *EsdRecord = getSectionEdEsdRecord(Sec);
+  return EsdRecord;
+}
+
+const uint8_t *
+GOFFObjectFile::getSectionPrEsdRecord(uint32_t SectionIndex) const {
+  DataRefImpl Sec;
+  Sec.d.a = SectionIndex;
+  const uint8_t *EsdRecord = getSectionPrEsdRecord(Sec);
+  return EsdRecord;
+}
+
+section_iterator GOFFObjectFile::section_begin() const {
+  DataRefImpl Sec;
+  moveSectionNext(Sec);
+  return section_iterator(SectionRef(Sec, this));
+}
+
+section_iterator GOFFObjectFile::section_end() const {
+  DataRefImpl Sec;
+  return section_iterator(SectionRef(Sec, this));
+}
+
+void GOFFObjectFile::moveSymbolNext(DataRefImpl &Symb) const {
+  for (uint32_t I = Symb.d.a + 1, E = EsdPtrs.size(); I < E; ++I) {
+    if (EsdPtrs[I]) {
+      const uint8_t *EsdRecord = EsdPtrs[I];
+      GOFF::ESDSymbolType SymbolType;
+      ESDRecord::getSymbolType(EsdRecord, SymbolType);
+      // Skip EDs - i.e. section symbols.
+      bool IgnoreSpecialGOFFSymbols = true;
+      bool SkipSymbol = ((SymbolType == GOFF::ESD_ST_ElementDefinition) ||
+                         (SymbolType == GOFF::ESD_ST_SectionDefinition)) &&
+                        IgnoreSpecialGOFFSymbols;
+      if (!SkipSymbol) {
+        Symb.d.a = I;
+        return;
+      }
+    }
+  }
+  Symb.d.a = 0;
+}
+
+basic_symbol_iterator GOFFObjectFile::symbol_begin() const {
+  DataRefImpl Symb;
+  moveSymbolNext(Symb);
+  return basic_symbol_iterator(SymbolRef(Symb, this));
+}
+
+basic_symbol_iterator GOFFObjectFile::symbol_end() const {
+  DataRefImpl Symb;
+  return basic_symbol_iterator(SymbolRef(Symb, this));
+}
+
+Error Record::getContinuousData(const uint8_t *Record, uint16_t DataLength,
+                                int DataIndex, SmallString<256> &CompleteData) {
+  // First record.
+  const uint8_t *Slice = Record + DataIndex;
+  size_t SliceLength =
+      std::min(DataLength, (uint16_t)(GOFF::RecordLength - DataIndex));
+  CompleteData.append(Slice, Slice + SliceLength);
+  DataLength -= SliceLength;
+  Slice += SliceLength;
+
+  // Continuation records.
+  for (; DataLength > 0;
+       DataLength -= SliceLength, Slice += GOFF::PayloadLength) {
+    // Slice points to the start of the new record.
+    // Check that this block is a Continuation.
+    assert(Record::isContinuation(Slice) && "Continuation bit must be set");
+    // Check that the last Continuation is terminated correctly.
+    if (DataLength <= 77 && Record::isContinued(Slice))
+      return createStringError(object_error::parse_failed,
+                               "continued bit should not be set");
+
+    SliceLength = std::min(DataLength, (uint16_t)GOFF::PayloadLength);
+    Slice += GOFF::RecordPrefixLength;
+    CompleteData.append(Slice, Slice + SliceLength);
+  }
+  return Error::success();
+}
+
+Error HDRRecord::getData(const uint8_t *Record,
+                         SmallString<256> &CompleteData) {
+  uint16_t Length = getPropertyModuleLength(Record);
+  return getContinuousData(Record, Length, 60, CompleteData);
+}
+
+Error ESDRecord::getData(const uint8_t *Record,
+                         SmallString<256> &CompleteData) {
+  uint16_t DataSize = getNameLength(Record);
+  return getContinuousData(Record, DataSize, 72, CompleteData);
+}
+
+Error ENDRecord::getData(const uint8_t *Record,
+                         SmallString<256> &CompleteData) {
+  uint16_t Length = getNameLength(Record);
+  return getContinuousData(Record, Length, 26, CompleteData);
+}
diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp
index 56a1d09097d42..c953a5692cbb4 100644
--- a/llvm/lib/Object/ObjectFile.cpp
+++ b/llvm/lib/Object/ObjectFile.cpp
@@ -130,6 +130,10 @@ Triple ObjectFile::makeTriple() const {
     TheTriple.setOS(Triple::AIX);
     TheTriple.setObjectFormat(Triple::XCOFF);
   }
+  else if (isGOFF()) {
+    TheTriple.setOS(Triple::ZOS);
+    TheTriple.setObjectFormat(Triple::GOFF);
+  }
 
   return TheTriple;
 }
diff --git a/llvm/lib/ObjectYAML/MachOEmitter.cpp b/llvm/lib/ObjectYAML/MachOEmitter.cpp
index 54983f0c260de..0a64f7285fd09 100644
--- a/llvm/lib/ObjectYAML/MachOEmitter.cpp
+++ b/llvm/lib/ObjectYAML/MachOEmitter.cpp
@@ -105,7 +105,7 @@ void MachOWriter::writeHeader(raw_ostream &OS) {
 }
 
 template <typename SectionType>
-SectionType constructSection(MachOYAML::Section Sec) {
+SectionType constructSection(const MachOYAML::Section &Sec) {
   SectionType TempSec;
   memcpy(reinterpret_cast<void *>(&TempSec.sectname[0]), &Sec.sectname[0], 16);
   memcpy(reinterpret_cast<void *>(&TempSec.segname[0]), &Sec.segname[0], 16);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index a7d65aa8f5088..55fc78d7f6cb0 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -245,6 +245,7 @@
 #include "llvm/Transforms/Utils/LowerSwitch.h"
 #include "llvm/Transforms/Utils/Mem2Reg.h"
 #include "llvm/Transforms/Utils/MetaRenamer.h"
+#include "llvm/Transforms/Utils/MoveAutoInit.h"
 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 #include "llvm/Transforms/Utils/PredicateInfo.h"
 #include "llvm/Transforms/Utils/RelLookupTableConverter.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 4407ad0131e12..2b392f128243f 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -124,6 +124,7 @@
 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 #include "llvm/Transforms/Utils/Mem2Reg.h"
+#include "llvm/Transforms/Utils/MoveAutoInit.h"
 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 #include "llvm/Transforms/Utils/RelLookupTableConverter.h"
 #include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
@@ -696,6 +697,8 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
   FPM.addPass(MemCpyOptPass());
 
   FPM.addPass(DSEPass());
+  FPM.addPass(MoveAutoInitPass());
+
   FPM.addPass(createFunctionToLoopPassAdaptor(
       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
                /*AllowSpeculation=*/true),
@@ -1809,6 +1812,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
 
   // Nuke dead stores.
   MainFPM.addPass(DSEPass());
+  MainFPM.addPass(MoveAutoInitPass());
   MainFPM.addPass(MergedLoadStoreMotionPass());
 
   LoopPassManager LPM;
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 891221d534526..2c760adc1a409 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -331,6 +331,7 @@ FUNCTION_PASS("mem2reg", PromotePass())
 FUNCTION_PASS("memcpyopt", MemCpyOptPass())
 FUNCTION_PASS("mergeicmps", MergeICmpsPass())
 FUNCTION_PASS("mergereturn", UnifyFunctionExitNodesPass())
+FUNCTION_PASS("move-auto-init", MoveAutoInitPass())
 FUNCTION_PASS("nary-reassociate", NaryReassociatePass())
 FUNCTION_PASS("newgvn", NewGVNPass())
 FUNCTION_PASS("jump-threading", JumpThreadingPass())
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index d6aec276838e6..849ee80bfaa33 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -249,7 +249,7 @@ Error CoverageMapping::loadFunctionRecord(
   std::vector<uint64_t> Counts;
   if (Error E = ProfileReader.getFunctionCounts(Record.FunctionName,
                                                 Record.FunctionHash, Counts)) {
-    instrprof_error IPE = InstrProfError::take(std::move(E));
+    instrprof_error IPE = std::get<0>(InstrProfError::take(std::move(E)));
     if (IPE == instrprof_error::hash_mismatch) {
       FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
                                       Record.FunctionHash);
diff --git a/llvm/lib/ProfileData/GCOV.cpp b/llvm/lib/ProfileData/GCOV.cpp
index 5af92799e05ef..e922f8cff9e60 100644
--- a/llvm/lib/ProfileData/GCOV.cpp
+++ b/llvm/lib/ProfileData/GCOV.cpp
@@ -337,10 +337,8 @@ StringRef GCOVFunction::getName(bool demangle) const {
   if (demangled.empty()) {
     do {
       if (Name.startswith("_Z")) {
-        int status = 0;
         // Name is guaranteed to be NUL-terminated.
-        char *res = itaniumDemangle(Name.data(), nullptr, nullptr, &status);
-        if (status == 0) {
+        if (char *res = itaniumDemangle(Name.data())) {
           demangled = res;
           free(res);
           break;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index d8dbcf1ebbe9a..1dd1ce4b3e50a 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -153,6 +153,9 @@ static std::string getInstrProfErrString(instrprof_error Err,
     OS << "profile uses zlib compression but the profile reader was built "
           "without zlib support";
     break;
+  case instrprof_error::raw_profile_version_mismatch:
+    OS << "raw profile version mismatch";
+    break;
   }
 
   // If optional error message is not empty, append it to the message.
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 47c0a57a3da92..4160f7e6dfd55 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -532,7 +532,13 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
     const RawInstrProf::Header &Header) {
   Version = swap(Header.Version);
   if (GET_VERSION(Version) != RawInstrProf::Version)
-    return error(instrprof_error::unsupported_version);
+    return error(instrprof_error::raw_profile_version_mismatch,
+                 ("Profile uses raw profile format version = " +
+                  Twine(GET_VERSION(Version)) +
+                  "; expected version = " + Twine(RawInstrProf::Version) +
+                  "\nPLEASE update this tool to version in the raw profile, or "
+                  "regenerate raw profile with expected version.")
+                     .str());
   if (useDebugInfoCorrelate() && !Correlator)
     return error(instrprof_error::missing_debug_info_for_correlation);
   if (!useDebugInfoCorrelate() && Correlator)
@@ -1199,7 +1205,8 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
 
   std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
   if (Error E = Index->populateSymtab(*NewSymtab)) {
-    consumeError(error(InstrProfError::take(std::move(E))));
+    auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
+    consumeError(error(ErrCode, Msg));
   }
 
   Symtab = std::move(NewSymtab);
diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp
index 7c519ed492e45..dbfb472f211ed 100644
--- a/llvm/lib/ProfileData/RawMemProfReader.cpp
+++ b/llvm/lib/ProfileData/RawMemProfReader.cpp
@@ -282,8 +282,8 @@ Error RawMemProfReader::initialize(std::unique_ptr<MemoryBuffer> DataBuffer) {
   }
 
   // Check whether the profiled binary was built with position independent code
-  // (PIC). For now we provide a error message until symbolization support
-  // is added for pic.
+  // (PIC). Perform sanity checks for assumptions we rely on to simplify
+  // symbolization.
   auto* Elf64LEObject = llvm::cast<llvm::object::ELF64LEObjectFile>(ElfObject);
   const llvm::object::ELF64LEFile& ElfFile = Elf64LEObject->getELFFile();
   auto PHdrsOr = ElfFile.program_headers();
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 3c60ef436db91..fdae8a011e712 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -291,7 +291,7 @@ const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
   std::string CalleeGUID;
   CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
 
-  auto iter = CallsiteSamples.find(Loc);
+  auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
   if (iter == CallsiteSamples.end())
     return nullptr;
   auto FS = iter->second.find(CalleeName);
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index dd23a4029ba9e..6f5dedb26b1aa 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -51,7 +51,7 @@ using namespace sampleprof;
 #define DEBUG_TYPE "samplepgo-reader"
 
 // This internal option specifies if the profile uses FS discriminators.
-// It only applies to text, binary and compact binary format profiles.
+// It only applies to text, and binary format profiles.
 // For ext-binary format profiles, the flag is set in the summary.
 static cl::opt<bool> ProfileIsFSDisciminator(
     "profile-isfs", cl::Hidden, cl::init(false),
@@ -568,14 +568,6 @@ ErrorOr<StringRef> SampleProfileReaderExtBinaryBase::readStringFromTable() {
   return SR;
 }
 
-ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() {
-  auto Idx = readStringIndex(NameTable);
-  if (std::error_code EC = Idx.getError())
-    return EC;
-
-  return StringRef(NameTable[*Idx]);
-}
-
 std::error_code
 SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
   auto NumSamples = readNumber<uint64_t>();
@@ -1012,40 +1004,6 @@ std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
   return sampleprof_error::success;
 }
 
-std::error_code SampleProfileReaderCompactBinary::readImpl() {
-  // Collect functions used by current module if the Reader has been
-  // given a module.
-  bool LoadFuncsToBeUsed = collectFuncsFromModule();
-  ProfileIsFS = ProfileIsFSDisciminator;
-  FunctionSamples::ProfileIsFS = ProfileIsFS;
-  std::vector<uint64_t> OffsetsToUse;
-  if (!LoadFuncsToBeUsed) {
-    // load all the function profiles.
-    for (auto FuncEntry : FuncOffsetTable) {
-      OffsetsToUse.push_back(FuncEntry.second);
-    }
-  } else {
-    // load function profiles on demand.
-    for (auto Name : FuncsToUse) {
-      auto GUID = std::to_string(MD5Hash(Name));
-      auto iter = FuncOffsetTable.find(StringRef(GUID));
-      if (iter == FuncOffsetTable.end())
-        continue;
-      OffsetsToUse.push_back(iter->second);
-    }
-  }
-
-  for (auto Offset : OffsetsToUse) {
-    const uint8_t *SavedData = Data;
-    if (std::error_code EC = readFuncProfile(
-            reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
-            Offset))
-      return EC;
-    Data = SavedData;
-  }
-  return sampleprof_error::success;
-}
-
 std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
   if (Magic == SPMagic())
     return sampleprof_error::success;
@@ -1058,13 +1016,6 @@ std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
   return sampleprof_error::bad_magic;
 }
 
-std::error_code
-SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) {
-  if (Magic == SPMagic(SPF_Compact_Binary))
-    return sampleprof_error::success;
-  return sampleprof_error::bad_magic;
-}
-
 std::error_code SampleProfileReaderBinary::readNameTable() {
   auto Size = readNumber<size_t>();
   if (std::error_code EC = Size.getError())
@@ -1234,20 +1185,6 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
   return sampleprof_error::success;
 }
 
-std::error_code SampleProfileReaderCompactBinary::readNameTable() {
-  auto Size = readNumber<uint64_t>();
-  if (std::error_code EC = Size.getError())
-    return EC;
-  NameTable.reserve(*Size);
-  for (uint64_t I = 0; I < *Size; ++I) {
-    auto FID = readNumber<uint64_t>();
-    if (std::error_code EC = FID.getError())
-      return EC;
-    NameTable.push_back(std::to_string(*FID));
-  }
-  return sampleprof_error::success;
-}
-
 std::error_code
 SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
   SecHdrTableEntry Entry;
@@ -1427,54 +1364,6 @@ std::error_code SampleProfileReaderBinary::readHeader() {
   return sampleprof_error::success;
 }
 
-std::error_code SampleProfileReaderCompactBinary::readHeader() {
-  SampleProfileReaderBinary::readHeader();
-  if (std::error_code EC = readFuncOffsetTable())
-    return EC;
-  return sampleprof_error::success;
-}
-
-std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {
-  auto TableOffset = readUnencodedNumber<uint64_t>();
-  if (std::error_code EC = TableOffset.getError())
-    return EC;
-
-  const uint8_t *SavedData = Data;
-  const uint8_t *TableStart =
-      reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
-      *TableOffset;
-  Data = TableStart;
-
-  auto Size = readNumber<uint64_t>();
-  if (std::error_code EC = Size.getError())
-    return EC;
-
-  FuncOffsetTable.reserve(*Size);
-  for (uint64_t I = 0; I < *Size; ++I) {
-    auto FName(readStringFromTable());
-    if (std::error_code EC = FName.getError())
-      return EC;
-
-    auto Offset = readNumber<uint64_t>();
-    if (std::error_code EC = Offset.getError())
-      return EC;
-
-    FuncOffsetTable[*FName] = *Offset;
-  }
-  End = TableStart;
-  Data = SavedData;
-  return sampleprof_error::success;
-}
-
-bool SampleProfileReaderCompactBinary::collectFuncsFromModule() {
-  if (!M)
-    return false;
-  FuncsToUse.clear();
-  for (auto &F : *M)
-    FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));
-  return true;
-}
-
 std::error_code SampleProfileReaderBinary::readSummaryEntry(
     std::vector<ProfileSummaryEntry> &Entries) {
   auto Cutoff = readNumber<uint64_t>();
@@ -1545,13 +1434,6 @@ bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
   return Magic == SPMagic(SPF_Ext_Binary);
 }
 
-bool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) {
-  const uint8_t *Data =
-      reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
-  uint64_t Magic = decodeULEB128(Data);
-  return Magic == SPMagic(SPF_Compact_Binary);
-}
-
 std::error_code SampleProfileReaderGCC::skipNextWord() {
   uint32_t dummy;
   if (!GcovBuffer.readInt(dummy))
@@ -1803,7 +1685,7 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
     Ctx.diagnose(DiagnosticInfoSampleProfile(
         Reader.getBuffer()->getBufferIdentifier(),
         "Profile data remapping cannot be applied to profile data "
-        "in compact format (original mangled names are not available).",
+        "using MD5 names (original mangled names are not available).",
         DS_Warning));
     return;
   }
@@ -1934,8 +1816,6 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
     Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));
   else if (SampleProfileReaderExtBinary::hasFormat(*B))
     Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));
-  else if (SampleProfileReaderCompactBinary::hasFormat(*B))
-    Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C));
   else if (SampleProfileReaderGCC::hasFormat(*B))
     Reader.reset(new SampleProfileReaderGCC(std::move(B), C));
   else if (SampleProfileReaderText::hasFormat(*B))
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index bce858a99a819..0873093ad426c 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -557,15 +557,6 @@ std::error_code SampleProfileWriterExtBinary::writeSections(
   return EC;
 }
 
-std::error_code
-SampleProfileWriterCompactBinary::write(const SampleProfileMap &ProfileMap) {
-  if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
-    return EC;
-  if (std::error_code EC = writeFuncOffsetTable())
-    return EC;
-  return sampleprof_error::success;
-}
-
 /// Write samples to a text file.
 ///
 /// Note: it may be tempting to implement this in terms of
@@ -712,41 +703,6 @@ std::error_code SampleProfileWriterBinary::writeNameTable() {
   return sampleprof_error::success;
 }
 
-std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
-  auto &OS = *OutputStream;
-
-  // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
-  uint64_t FuncOffsetTableStart = OS.tell();
-  support::endian::SeekableWriter Writer(static_cast<raw_pwrite_stream &>(OS),
-                                         support::little);
-  Writer.pwrite(FuncOffsetTableStart, TableOffset);
-
-  // Write out the table size.
-  encodeULEB128(FuncOffsetTable.size(), OS);
-
-  // Write out FuncOffsetTable.
-  for (auto Entry : FuncOffsetTable) {
-    if (std::error_code EC = writeNameIdx(Entry.first))
-      return EC;
-    encodeULEB128(Entry.second, OS);
-  }
-  FuncOffsetTable.clear();
-  return sampleprof_error::success;
-}
-
-std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
-  auto &OS = *OutputStream;
-  std::set<StringRef> V;
-  stablizeNameTable(NameTable, V);
-
-  // Write out the name table.
-  encodeULEB128(NameTable.size(), OS);
-  for (auto N : V) {
-    encodeULEB128(MD5Hash(N), OS);
-  }
-  return sampleprof_error::success;
-}
-
 std::error_code
 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
   auto &OS = *OutputStream;
@@ -848,19 +804,6 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
   return sampleprof_error::success;
 }
 
-std::error_code SampleProfileWriterCompactBinary::writeHeader(
-    const SampleProfileMap &ProfileMap) {
-  support::endian::Writer Writer(*OutputStream, support::little);
-  if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
-    return EC;
-
-  // Reserve a slot for the offset of function offset table. The slot will
-  // be populated with the offset of FuncOffsetTable later.
-  TableOffset = OutputStream->tell();
-  Writer.write(static_cast<uint64_t>(-2));
-  return sampleprof_error::success;
-}
-
 std::error_code SampleProfileWriterBinary::writeSummary() {
   auto &OS = *OutputStream;
   encodeULEB128(Summary->getTotalCount(), OS);
@@ -930,15 +873,6 @@ SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
   return writeBody(S);
 }
 
-std::error_code
-SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {
-  uint64_t Offset = OutputStream->tell();
-  StringRef Name = S.getName();
-  FuncOffsetTable[Name] = Offset;
-  encodeULEB128(S.getHeadSamples(), *OutputStream);
-  return writeBody(S);
-}
-
 /// Create a sample profile file writer based on the specified format.
 ///
 /// \param Filename The file to create.
@@ -950,8 +884,7 @@ ErrorOr<std::unique_ptr<SampleProfileWriter>>
 SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
   std::error_code EC;
   std::unique_ptr<raw_ostream> OS;
-  if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
-      Format == SPF_Compact_Binary)
+  if (Format == SPF_Binary || Format == SPF_Ext_Binary)
     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
   else
     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
@@ -976,15 +909,13 @@ SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
 
   // Currently only Text and Extended Binary format are supported for CSSPGO.
   if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&
-      (Format == SPF_Binary || Format == SPF_Compact_Binary))
+      Format == SPF_Binary)
     return sampleprof_error::unsupported_writing_format;
 
   if (Format == SPF_Binary)
     Writer.reset(new SampleProfileWriterRawBinary(OS));
   else if (Format == SPF_Ext_Binary)
     Writer.reset(new SampleProfileWriterExtBinary(OS));
-  else if (Format == SPF_Compact_Binary)
-    Writer.reset(new SampleProfileWriterCompactBinary(OS));
   else if (Format == SPF_Text)
     Writer.reset(new SampleProfileWriterText(OS));
   else if (Format == SPF_GCC)
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 2270575571cf7..b000c436670e1 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -146,6 +146,7 @@ add_llvm_component_library(LLVMSupport
   Compression.cpp
   CRC.cpp
   ConvertUTF.cpp
+  ConvertEBCDIC.cpp
   ConvertUTFWrapper.cpp
   CrashRecoveryContext.cpp
   CSKYAttributes.cpp
@@ -182,7 +183,6 @@ add_llvm_component_library(LLVMSupport
   LineIterator.cpp
   Locale.cpp
   LockFileManager.cpp
-  LowLevelType.cpp
   ManagedStatic.cpp
   MathExtras.cpp
   MemAlloc.cpp
diff --git a/llvm/lib/Support/ConvertEBCDIC.cpp b/llvm/lib/Support/ConvertEBCDIC.cpp
new file mode 100644
index 0000000000000..08eeaa52a6c97
--- /dev/null
+++ b/llvm/lib/Support/ConvertEBCDIC.cpp
@@ -0,0 +1,123 @@
+//===--- ConvertEBCDIC.cpp - UTF8/EBCDIC CharSet Conversion -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides utility functions for converting between EBCDIC-1047 and
+/// UTF-8.
+///
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/ConvertEBCDIC.h"
+
+using namespace llvm;
+
+static const unsigned char ISO88591ToIBM1047[256] = {
+    0x00, 0x01, 0x02, 0x03, 0x37, 0x2d, 0x2e, 0x2f, 0x16, 0x05, 0x15, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x3c, 0x3d, 0x32, 0x26,
+    0x18, 0x19, 0x3f, 0x27, 0x1c, 0x1d, 0x1e, 0x1f, 0x40, 0x5a, 0x7f, 0x7b,
+    0x5b, 0x6c, 0x50, 0x7d, 0x4d, 0x5d, 0x5c, 0x4e, 0x6b, 0x60, 0x4b, 0x61,
+    0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0x7a, 0x5e,
+    0x4c, 0x7e, 0x6e, 0x6f, 0x7c, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+    0xc8, 0xc9, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xe2,
+    0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xad, 0xe0, 0xbd, 0x5f, 0x6d,
+    0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92,
+    0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6,
+    0xa7, 0xa8, 0xa9, 0xc0, 0x4f, 0xd0, 0xa1, 0x07, 0x20, 0x21, 0x22, 0x23,
+    0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x09, 0x0a, 0x1b,
+    0x30, 0x31, 0x1a, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3a, 0x3b,
+    0x04, 0x14, 0x3e, 0xff, 0x41, 0xaa, 0x4a, 0xb1, 0x9f, 0xb2, 0x6a, 0xb5,
+    0xbb, 0xb4, 0x9a, 0x8a, 0xb0, 0xca, 0xaf, 0xbc, 0x90, 0x8f, 0xea, 0xfa,
+    0xbe, 0xa0, 0xb6, 0xb3, 0x9d, 0xda, 0x9b, 0x8b, 0xb7, 0xb8, 0xb9, 0xab,
+    0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9e, 0x68, 0x74, 0x71, 0x72, 0x73,
+    0x78, 0x75, 0x76, 0x77, 0xac, 0x69, 0xed, 0xee, 0xeb, 0xef, 0xec, 0xbf,
+    0x80, 0xfd, 0xfe, 0xfb, 0xfc, 0xba, 0xae, 0x59, 0x44, 0x45, 0x42, 0x46,
+    0x43, 0x47, 0x9c, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57,
+    0x8c, 0x49, 0xcd, 0xce, 0xcb, 0xcf, 0xcc, 0xe1, 0x70, 0xdd, 0xde, 0xdb,
+    0xdc, 0x8d, 0x8e, 0xdf};
+
+static const unsigned char IBM1047ToISO88591[256] = {
+    0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d, 0x8e, 0x0b,
+    0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x9d, 0x0a, 0x08, 0x87,
+    0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d, 0x1e, 0x1f, 0x80, 0x81, 0x82, 0x83,
+    0x84, 0x85, 0x17, 0x1b, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07,
+    0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b,
+    0x14, 0x15, 0x9e, 0x1a, 0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1, 0xe3, 0xe5,
+    0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, 0x26, 0xe9, 0xea, 0xeb,
+    0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x5e,
+    0x2d, 0x2f, 0xc2, 0xc4, 0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c,
+    0x25, 0x5f, 0x3e, 0x3f, 0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf,
+    0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, 0xd8, 0x61, 0x62, 0x63,
+    0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1,
+    0xb0, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba,
+    0xe6, 0xb8, 0xc6, 0xa4, 0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+    0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0x5b, 0xde, 0xae, 0xac, 0xa3, 0xa5, 0xb7,
+    0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0xdd, 0xa8, 0xaf, 0x5d, 0xb4, 0xd7,
+    0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xad, 0xf4,
+    0xf6, 0xf2, 0xf3, 0xf5, 0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50,
+    0x51, 0x52, 0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff, 0x5c, 0xf7, 0x53, 0x54,
+    0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2, 0xd3, 0xd5,
+    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xb3, 0xdb,
+    0xdc, 0xd9, 0xda, 0x9f};
+
+std::error_code
+ConverterEBCDIC::convertToEBCDIC(StringRef Source,
+                                 SmallVectorImpl<char> &Result) {
+  assert(Result.empty() && "Result must be empty!");
+  const unsigned char *Table = ISO88591ToIBM1047;
+  const unsigned char *Ptr =
+      reinterpret_cast<const unsigned char *>(Source.data());
+  size_t Length = Source.size();
+  Result.reserve(Length);
+  while (Length--) {
+    unsigned char Ch = *Ptr++;
+    // Handle UTF-8 2-byte-sequences in input.
+    if (Ch >= 128) {
+      // Only two-byte sequences can be decoded.
+      if (Ch != 0xc2 && Ch != 0xc3)
+        return std::make_error_code(std::errc::illegal_byte_sequence);
+      // Is buffer truncated?
+      if (!Length)
+        return std::make_error_code(std::errc::invalid_argument);
+      unsigned char Ch2 = *Ptr++;
+      // Is second byte well-formed?
+      if ((Ch2 & 0xc0) != 0x80)
+        return std::make_error_code(std::errc::illegal_byte_sequence);
+      Ch = Ch2 | (Ch << 6);
+      Length--;
+    }
+    // Translate the character.
+    Ch = Table[Ch];
+    Result.push_back(static_cast<char>(Ch));
+  }
+  return std::error_code();
+}
+
+void ConverterEBCDIC::convertToUTF8(StringRef Source,
+                                    SmallVectorImpl<char> &Result) {
+  assert(Result.empty() && "Result must be empty!");
+
+  const unsigned char *Table = IBM1047ToISO88591;
+  const unsigned char *Ptr =
+      reinterpret_cast<const unsigned char *>(Source.data());
+  size_t Length = Source.size();
+  Result.reserve(Length);
+  while (Length--) {
+    unsigned char Ch = *Ptr++;
+    // Translate the character.
+    Ch = Table[Ch];
+    // Handle UTF-8 2-byte-sequences in output.
+    if (Ch >= 128) {
+      // First byte prefixed with either 0xc2 or 0xc3.
+      Result.push_back(static_cast<char>(0xc0 | (Ch >> 6)));
+      // Second byte is either the same as the ASCII byte or ASCII byte -64.
+      Ch = Ch & 0xbf;
+    }
+    Result.push_back(static_cast<char>(Ch));
+  }
+}
diff --git a/llvm/lib/Support/LowLevelType.cpp b/llvm/lib/Support/LowLevelType.cpp
deleted file mode 100644
index e394f98fa62e3..0000000000000
--- a/llvm/lib/Support/LowLevelType.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- llvm/Support/LowLevelType.cpp -------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file This file implements the more header-heavy bits of the LLT class to
-/// avoid polluting users' namespaces.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/LowLevelTypeImpl.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace llvm;
-
-LLT::LLT(MVT VT) {
-  if (VT.isVector()) {
-    bool asVector = VT.getVectorMinNumElements() > 1;
-    init(/*IsPointer=*/false, asVector, /*IsScalar=*/!asVector,
-         VT.getVectorElementCount(), VT.getVectorElementType().getSizeInBits(),
-         /*AddressSpace=*/0);
-  } else if (VT.isValid() && !VT.isScalableTargetExtVT()) {
-    // Aggregates are no different from real scalars as far as GlobalISel is
-    // concerned.
-    init(/*IsPointer=*/false, /*IsVector=*/false, /*IsScalar=*/true,
-         ElementCount::getFixed(0), VT.getSizeInBits(), /*AddressSpace=*/0);
-  } else {
-    IsScalar = false;
-    IsPointer = false;
-    IsVector = false;
-    RawData = 0;
-  }
-}
-
-void LLT::print(raw_ostream &OS) const {
-  if (isVector()) {
-    OS << "<";
-    OS << getElementCount() << " x " << getElementType() << ">";
-  } else if (isPointer())
-    OS << "p" << getAddressSpace();
-  else if (isValid()) {
-    assert(isScalar() && "unexpected type");
-    OS << "s" << getScalarSizeInBits();
-  } else
-    OS << "LLT_invalid";
-}
-
-const constexpr LLT::BitFieldInfo LLT::ScalarSizeFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::PointerSizeFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::PointerAddressSpaceFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::VectorElementsFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::VectorScalableFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::VectorSizeFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::PointerVectorElementsFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::PointerVectorScalableFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::PointerVectorSizeFieldInfo;
-const constexpr LLT::BitFieldInfo LLT::PointerVectorAddressSpaceFieldInfo;
diff --git a/llvm/lib/Support/Parallel.cpp b/llvm/lib/Support/Parallel.cpp
index 0a1d82efe768f..f54479069cf67 100644
--- a/llvm/lib/Support/Parallel.cpp
+++ b/llvm/lib/Support/Parallel.cpp
@@ -12,8 +12,8 @@
 #include "llvm/Support/Threading.h"
 
 #include <atomic>
+#include <deque>
 #include <future>
-#include <stack>
 #include <thread>
 #include <vector>
 
@@ -24,11 +24,11 @@ namespace parallel {
 #if LLVM_ENABLE_THREADS
 
 #ifdef _WIN32
-static thread_local unsigned threadIndex;
+static thread_local unsigned threadIndex = UINT_MAX;
 
-unsigned getThreadIndex() { return threadIndex; }
+unsigned getThreadIndex() { GET_THREAD_INDEX_IMPL; }
 #else
-thread_local unsigned threadIndex;
+thread_local unsigned threadIndex = UINT_MAX;
 #endif
 
 namespace detail {
@@ -39,7 +39,7 @@ namespace {
 class Executor {
 public:
   virtual ~Executor() = default;
-  virtual void add(std::function<void()> func) = 0;
+  virtual void add(std::function<void()> func, bool Sequential = false) = 0;
 
   static Executor *getDefaultExecutor();
 };
@@ -97,32 +97,54 @@ class ThreadPoolExecutor : public Executor {
     static void call(void *Ptr) { ((ThreadPoolExecutor *)Ptr)->stop(); }
   };
 
-  void add(std::function<void()> F) override {
+  void add(std::function<void()> F, bool Sequential = false) override {
     {
       std::lock_guard<std::mutex> Lock(Mutex);
-      WorkStack.push(std::move(F));
+      if (Sequential)
+        WorkQueueSequential.emplace_front(std::move(F));
+      else
+        WorkQueue.emplace_back(std::move(F));
     }
     Cond.notify_one();
   }
 
 private:
+  bool hasSequentialTasks() const {
+    return !WorkQueueSequential.empty() && !SequentialQueueIsLocked;
+  }
+
+  bool hasGeneralTasks() const { return !WorkQueue.empty(); }
+
   void work(ThreadPoolStrategy S, unsigned ThreadID) {
     threadIndex = ThreadID;
     S.apply_thread_strategy(ThreadID);
     while (true) {
       std::unique_lock<std::mutex> Lock(Mutex);
-      Cond.wait(Lock, [&] { return Stop || !WorkStack.empty(); });
+      Cond.wait(Lock, [&] {
+        return Stop || hasGeneralTasks() || hasSequentialTasks();
+      });
       if (Stop)
         break;
-      auto Task = std::move(WorkStack.top());
-      WorkStack.pop();
+      bool Sequential = hasSequentialTasks();
+      if (Sequential)
+        SequentialQueueIsLocked = true;
+      else
+        assert(hasGeneralTasks());
+
+      auto &Queue = Sequential ? WorkQueueSequential : WorkQueue;
+      auto Task = std::move(Queue.back());
+      Queue.pop_back();
       Lock.unlock();
       Task();
+      if (Sequential)
+        SequentialQueueIsLocked = false;
     }
   }
 
   std::atomic<bool> Stop{false};
-  std::stack<std::function<void()>> WorkStack;
+  std::atomic<bool> SequentialQueueIsLocked{false};
+  std::deque<std::function<void()>> WorkQueue;
+  std::deque<std::function<void()>> WorkQueueSequential;
   std::mutex Mutex;
   std::condition_variable Cond;
   std::promise<void> ThreadsCreated;
@@ -158,52 +180,47 @@ Executor *Executor::getDefaultExecutor() {
 } // namespace detail
 #endif
 
-static std::atomic<int> TaskGroupInstances;
-
 // Latch::sync() called by the dtor may cause one thread to block. If is a dead
 // lock if all threads in the default executor are blocked. To prevent the dead
-// lock, only allow the first TaskGroup to run tasks parallelly. In the scenario
+// lock, only allow the root TaskGroup to run tasks parallelly. In the scenario
 // of nested parallel_for_each(), only the outermost one runs parallelly.
-TaskGroup::TaskGroup() : Parallel(TaskGroupInstances++ == 0) {}
+TaskGroup::TaskGroup()
+#if LLVM_ENABLE_THREADS
+    : Parallel((parallel::strategy.ThreadsRequested != 1) &&
+               (threadIndex == UINT_MAX)) {}
+#else
+    : Parallel(false) {}
+#endif
 TaskGroup::~TaskGroup() {
   // We must ensure that all the workloads have finished before decrementing the
   // instances count.
   L.sync();
-  --TaskGroupInstances;
 }
 
-void TaskGroup::spawn(std::function<void()> F) {
+void TaskGroup::spawn(std::function<void()> F, bool Sequential) {
 #if LLVM_ENABLE_THREADS
   if (Parallel) {
     L.inc();
-    detail::Executor::getDefaultExecutor()->add([&, F = std::move(F)] {
-      F();
-      L.dec();
-    });
+    detail::Executor::getDefaultExecutor()->add(
+        [&, F = std::move(F)] {
+          F();
+          L.dec();
+        },
+        Sequential);
     return;
   }
 #endif
   F();
 }
 
-void TaskGroup::execute(std::function<void()> F) {
-  if (parallel::strategy.ThreadsRequested == 1)
-    F();
-  else
-    spawn(F);
-}
 } // namespace parallel
 } // namespace llvm
 
 void llvm::parallelFor(size_t Begin, size_t End,
                        llvm::function_ref<void(size_t)> Fn) {
-  // If we have zero or one items, then do not incur the overhead of spinning up
-  // a task group.  They are surprisingly expensive, and because they do not
-  // support nested parallelism, a single entry task group can block parallel
-  // execution underneath them.
 #if LLVM_ENABLE_THREADS
-  auto NumItems = End - Begin;
-  if (NumItems > 1 && parallel::strategy.ThreadsRequested != 1) {
+  if (parallel::strategy.ThreadsRequested != 1) {
+    auto NumItems = End - Begin;
     // Limit the number of tasks to MaxTasksPerGroup to limit job scheduling
     // overhead on large inputs.
     auto TaskSize = NumItems / parallel::detail::MaxTasksPerGroup;
diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp
index 40cb45c7dd126..d25dc2b0381d1 100644
--- a/llvm/lib/Support/RISCVISAInfo.cpp
+++ b/llvm/lib/Support/RISCVISAInfo.cpp
@@ -107,8 +107,10 @@ static const RISCVSupportedExtension SupportedExtensions[] = {
     {"zicbom", RISCVExtensionVersion{1, 0}},
     {"zicboz", RISCVExtensionVersion{1, 0}},
     {"zicbop", RISCVExtensionVersion{1, 0}},
+    {"zicntr", RISCVExtensionVersion{1, 0}},
     {"zicsr", RISCVExtensionVersion{2, 0}},
     {"zifencei", RISCVExtensionVersion{2, 0}},
+    {"zihpm", RISCVExtensionVersion{1, 0}},
 
     {"zawrs", RISCVExtensionVersion{1, 0}},
 
@@ -133,12 +135,16 @@ static const RISCVSupportedExtension SupportedExtensions[] = {
 };
 
 static const RISCVSupportedExtension SupportedExperimentalExtensions[] = {
+    {"smaia", RISCVExtensionVersion{1, 0}},
+    {"ssaia", RISCVExtensionVersion{1, 0}},
+
     {"zihintntl", RISCVExtensionVersion{0, 2}},
 
     {"zca", RISCVExtensionVersion{1, 0}},
     {"zcb", RISCVExtensionVersion{1, 0}},
     {"zcd", RISCVExtensionVersion{1, 0}},
     {"zcf", RISCVExtensionVersion{1, 0}},
+    {"zcmt", RISCVExtensionVersion{1, 0}},
     {"zfa", RISCVExtensionVersion{0, 2}},
     {"zicond", RISCVExtensionVersion{1, 0}},
     {"zvfh", RISCVExtensionVersion{0, 1}},
@@ -687,8 +693,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   Exts = Exts.drop_front(ConsumeLength);
   Exts.consume_front("_");
 
-  // TODO: Use version number when setting target features
-
   auto StdExtsItr = StdExts.begin();
   auto StdExtsEnd = StdExts.end();
   auto GoToNextExt = [](StringRef::iterator &I, unsigned ConsumeLength) {
@@ -736,7 +740,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
     }
 
     // The order is OK, then push it into features.
-    // TODO: Use version number when setting target features
     // Currently LLVM supports only "mafdcvh".
     if (!isSupportedExtension(StringRef(&C, 1))) {
       if (IgnoreUnknown) {
@@ -849,6 +852,7 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
 }
 
 Error RISCVISAInfo::checkDependency() {
+  bool HasC = Exts.count("c") != 0;
   bool HasD = Exts.count("d") != 0;
   bool HasF = Exts.count("f") != 0;
   bool HasZfinx = Exts.count("zfinx") != 0;
@@ -857,6 +861,8 @@ Error RISCVISAInfo::checkDependency() {
   bool HasZve32f = Exts.count("zve32f") != 0;
   bool HasZve64d = Exts.count("zve64d") != 0;
   bool HasZvl = MinVLen != 0;
+  bool HasZcmt = Exts.count("zcmt") != 0;
+  bool HasZcd = Exts.count("zcd") != 0;
 
   if (HasF && HasZfinx)
     return createStringError(errc::invalid_argument,
@@ -906,6 +912,16 @@ Error RISCVISAInfo::checkDependency() {
         errc::invalid_argument,
         "'zvknhb' requires 'v' or 'zve64*' extension to also be specified");
 
+  if (HasZcmt && HasD && HasC)
+    return createStringError(
+        errc::invalid_argument,
+        "'zcmt' is incompatible with 'c' extension when 'd' extension is set");
+
+  if (HasZcmt && HasD && HasZcd)
+    return createStringError(errc::invalid_argument,
+                             "'zcmt' is incompatible with 'zcd' extension when "
+                             "'d' extension is set");
+
   // Additional dependency checks.
   // TODO: The 'q' extension requires rv64.
   // TODO: It is illegal to specify 'e' extensions with 'f' and 'd'.
@@ -919,6 +935,7 @@ static const char *ImpliedExtsV[] = {"zvl128b", "zve64d", "f", "d"};
 static const char *ImpliedExtsXTHeadVdot[] = {"v"};
 static const char *ImpliedExtsXsfvcp[] = {"zve32x"};
 static const char *ImpliedExtsZcb[] = {"zca"};
+static const char *ImpliedExtsZcmt[] = {"zca"};
 static const char *ImpliedExtsZdinx[] = {"zfinx"};
 static const char *ImpliedExtsZfa[] = {"f"};
 static const char *ImpliedExtsZfh[] = {"f"};
@@ -926,6 +943,8 @@ static const char *ImpliedExtsZfhmin[] = {"f"};
 static const char *ImpliedExtsZfinx[] = {"zicsr"};
 static const char *ImpliedExtsZhinx[] = {"zfinx"};
 static const char *ImpliedExtsZhinxmin[] = {"zfinx"};
+static const char *ImpliedExtsZicntr[] = {"zicsr"};
+static const char *ImpliedExtsZihpm[] = {"zicsr"};
 static const char *ImpliedExtsZk[] = {"zkn", "zkt", "zkr"};
 static const char *ImpliedExtsZkn[] = {"zbkb", "zbkc", "zbkx",
                                        "zkne", "zknd", "zknh"};
@@ -974,6 +993,7 @@ static constexpr ImpliedExtsEntry ImpliedExts[] = {
     {{"xsfvcp"}, {ImpliedExtsXsfvcp}},
     {{"xtheadvdot"}, {ImpliedExtsXTHeadVdot}},
     {{"zcb"}, {ImpliedExtsZcb}},
+    {{"zcmt"}, {ImpliedExtsZcmt}},
     {{"zdinx"}, {ImpliedExtsZdinx}},
     {{"zfa"}, {ImpliedExtsZfa}},
     {{"zfh"}, {ImpliedExtsZfh}},
@@ -981,6 +1001,8 @@ static constexpr ImpliedExtsEntry ImpliedExts[] = {
     {{"zfinx"}, {ImpliedExtsZfinx}},
     {{"zhinx"}, {ImpliedExtsZhinx}},
     {{"zhinxmin"}, {ImpliedExtsZhinxmin}},
+    {{"zicntr"}, {ImpliedExtsZicntr}},
+    {{"zihpm"}, {ImpliedExtsZihpm}},
     {{"zk"}, {ImpliedExtsZk}},
     {{"zkn"}, {ImpliedExtsZkn}},
     {{"zks"}, {ImpliedExtsZks}},
diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc
index d2842d51c3cf0..fcf5701afcfd4 100644
--- a/llvm/lib/Support/Unix/Signals.inc
+++ b/llvm/lib/Support/Unix/Signals.inc
@@ -652,13 +652,12 @@ void llvm::sys::PrintStackTrace(raw_ostream &OS, int Depth) {
 
     if (dlinfo.dli_sname != nullptr) {
       OS << ' ';
-      int res;
-      char *d = itaniumDemangle(dlinfo.dli_sname, nullptr, nullptr, &res);
-      if (!d)
-        OS << dlinfo.dli_sname;
-      else
+      if (char *d = itaniumDemangle(dlinfo.dli_sname)) {
         OS << d;
-      free(d);
+        free(d);
+      } else {
+        OS << dlinfo.dli_sname;
+      }
 
       OS << format(" + %tu", (static_cast<const char *>(StackTrace[i]) -
                               static_cast<const char *>(dlinfo.dli_saddr)));
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index a167e0a76795a..bcf5389db6a0e 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -257,12 +257,12 @@ class RealFileSystem : public FileSystem {
   explicit RealFileSystem(bool LinkCWDToProcess) {
     if (!LinkCWDToProcess) {
       SmallString<128> PWD, RealPWD;
-      if (llvm::sys::fs::current_path(PWD))
-        return; // Awful, but nothing to do here.
-      if (llvm::sys::fs::real_path(PWD, RealPWD))
-        WD = {PWD, PWD};
+      if (std::error_code EC = llvm::sys::fs::current_path(PWD))
+        WD = EC;
+      else if (llvm::sys::fs::real_path(PWD, RealPWD))
+        WD = WorkingDirectory{PWD, PWD};
       else
-        WD = {PWD, RealPWD};
+        WD = WorkingDirectory{PWD, RealPWD};
     }
   }
 
@@ -284,10 +284,10 @@ class RealFileSystem : public FileSystem {
   // If this FS has its own working dir, use it to make Path absolute.
   // The returned twine is safe to use as long as both Storage and Path live.
   Twine adjustPath(const Twine &Path, SmallVectorImpl<char> &Storage) const {
-    if (!WD)
+    if (!WD || !*WD)
       return Path;
     Path.toVector(Storage);
-    sys::fs::make_absolute(WD->Resolved, Storage);
+    sys::fs::make_absolute(WD->get().Resolved, Storage);
     return Storage;
   }
 
@@ -297,7 +297,7 @@ class RealFileSystem : public FileSystem {
     // The current working directory, with links resolved. (readlink .).
     SmallString<128> Resolved;
   };
-  std::optional<WorkingDirectory> WD;
+  std::optional<llvm::ErrorOr<WorkingDirectory>> WD;
 };
 
 } // namespace
@@ -323,8 +323,10 @@ RealFileSystem::openFileForRead(const Twine &Name) {
 }
 
 llvm::ErrorOr<std::string> RealFileSystem::getCurrentWorkingDirectory() const {
+  if (WD && *WD)
+    return std::string(WD->get().Specified.str());
   if (WD)
-    return std::string(WD->Specified.str());
+    return WD->getError();
 
   SmallString<128> Dir;
   if (std::error_code EC = llvm::sys::fs::current_path(Dir))
@@ -345,7 +347,7 @@ std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
     return std::make_error_code(std::errc::not_a_directory);
   if (auto Err = llvm::sys::fs::real_path(Absolute, Resolved))
     return Err;
-  WD = {Absolute, Resolved};
+  WD = WorkingDirectory{Absolute, Resolved};
   return std::error_code();
 }
 
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index df1a4d47ff96e..da64d685a97f6 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -83,8 +83,6 @@ struct RecordKeeperImpl {
   FoldingSet<ExistsOpInit> TheExistsOpInitPool;
   DenseMap<std::pair<RecTy *, Init *>, VarInit *> TheVarInitPool;
   DenseMap<std::pair<TypedInit *, unsigned>, VarBitInit *> TheVarBitInitPool;
-  DenseMap<std::pair<TypedInit *, unsigned>, VarListElementInit *>
-      TheVarListElementInitPool;
   FoldingSet<VarDefInit> TheVarDefInitPool;
   DenseMap<std::pair<Init *, StringInit *>, FieldInit *> TheFieldInitPool;
   FoldingSet<CondOpInit> TheCondOpInitPool;
@@ -670,23 +668,6 @@ Init *ListInit::convertInitializerTo(RecTy *Ty) const {
   return nullptr;
 }
 
-Init *ListInit::convertInitListSlice(ArrayRef<unsigned> Elements) const {
-  if (Elements.size() == 1) {
-    if (Elements[0] >= size())
-      return nullptr;
-    return getElement(Elements[0]);
-  }
-
-  SmallVector<Init*, 8> Vals;
-  Vals.reserve(Elements.size());
-  for (unsigned Element : Elements) {
-    if (Element >= size())
-      return nullptr;
-    Vals.push_back(getElement(Element));
-  }
-  return ListInit::get(Vals, getElementType());
-}
-
 Record *ListInit::getElementAsRecord(unsigned i) const {
   assert(i < NumValues && "List element index out of range!");
   DefInit *DI = dyn_cast<DefInit>(getElement(i));
@@ -1200,6 +1181,67 @@ std::optional<bool> BinOpInit::CompareInit(unsigned Opc, Init *LHS, Init *RHS) c
     }
     break;
   }
+  case LISTELEM: {
+    auto *TheList = dyn_cast<ListInit>(LHS);
+    auto *Idx = dyn_cast<IntInit>(RHS);
+    if (!TheList || !Idx)
+      break;
+    auto i = Idx->getValue();
+    if (i < 0 || i >= (ssize_t)TheList->size())
+      break;
+    return TheList->getElement(i);
+  }
+  case LISTSLICE: {
+    auto *TheList = dyn_cast<ListInit>(LHS);
+    auto *SliceIdxs = dyn_cast<ListInit>(RHS);
+    if (!TheList || !SliceIdxs)
+      break;
+    SmallVector<Init *, 8> Args;
+    Args.reserve(SliceIdxs->size());
+    for (auto *I : *SliceIdxs) {
+      auto *II = dyn_cast<IntInit>(I);
+      if (!II)
+        goto unresolved;
+      auto i = II->getValue();
+      if (i < 0 || i >= (ssize_t)TheList->size())
+        goto unresolved;
+      Args.push_back(TheList->getElement(i));
+    }
+    return ListInit::get(Args, TheList->getElementType());
+  }
+  case RANGE:
+  case RANGEC: {
+    auto *LHSi = dyn_cast<IntInit>(LHS);
+    auto *RHSi = dyn_cast<IntInit>(RHS);
+    if (!LHSi || !RHSi)
+      break;
+
+    auto Start = LHSi->getValue();
+    auto End = RHSi->getValue();
+    SmallVector<Init *, 8> Args;
+    if (getOpcode() == RANGEC) {
+      // Closed interval
+      if (Start <= End) {
+        // Ascending order
+        Args.reserve(End - Start + 1);
+        for (auto i = Start; i <= End; ++i)
+          Args.push_back(IntInit::get(getRecordKeeper(), i));
+      } else {
+        // Descending order
+        Args.reserve(Start - End + 1);
+        for (auto i = Start; i >= End; --i)
+          Args.push_back(IntInit::get(getRecordKeeper(), i));
+      }
+    } else if (Start < End) {
+      // Half-open interval (excludes `End`)
+      Args.reserve(End - Start);
+      for (auto i = Start; i < End; ++i)
+        Args.push_back(IntInit::get(getRecordKeeper(), i));
+    } else {
+      // Empty set
+    }
+    return ListInit::get(Args, LHSi->getType());
+  }
   case STRCONCAT: {
     StringInit *LHSs = dyn_cast<StringInit>(LHS);
     StringInit *RHSs = dyn_cast<StringInit>(RHS);
@@ -1289,6 +1331,7 @@ std::optional<bool> BinOpInit::CompareInit(unsigned Opc, Init *LHS, Init *RHS) c
     break;
   }
   }
+unresolved:
   return const_cast<BinOpInit *>(this);
 }
 
@@ -1305,6 +1348,11 @@ Init *BinOpInit::resolveReferences(Resolver &R) const {
 std::string BinOpInit::getAsString() const {
   std::string Result;
   switch (getOpcode()) {
+  case LISTELEM:
+  case LISTSLICE:
+    return LHS->getAsString() + "[" + RHS->getAsString() + "]";
+  case RANGEC:
+    return LHS->getAsString() + "..." + RHS->getAsString();
   case CONCAT: Result = "!con"; break;
   case ADD: Result = "!add"; break;
   case SUB: Result = "!sub"; break;
@@ -1325,6 +1373,7 @@ std::string BinOpInit::getAsString() const {
   case LISTCONCAT: Result = "!listconcat"; break;
   case LISTSPLAT: Result = "!listsplat"; break;
   case LISTREMOVE: Result = "!listremove"; break;
+  case RANGE: Result = "!range"; break;
   case STRCONCAT: Result = "!strconcat"; break;
   case INTERLEAVE: Result = "!interleave"; break;
   case SETDAGOP: Result = "!setdagop"; break;
@@ -1888,22 +1937,6 @@ Init *TypedInit::getCastTo(RecTy *Ty) const {
       ->Fold(nullptr);
 }
 
-Init *TypedInit::convertInitListSlice(ArrayRef<unsigned> Elements) const {
-  ListRecTy *T = dyn_cast<ListRecTy>(getType());
-  if (!T) return nullptr;  // Cannot subscript a non-list variable.
-
-  if (Elements.size() == 1)
-    return VarListElementInit::get(const_cast<TypedInit *>(this), Elements[0]);
-
-  SmallVector<Init*, 8> ListInits;
-  ListInits.reserve(Elements.size());
-  for (unsigned Element : Elements)
-    ListInits.push_back(VarListElementInit::get(const_cast<TypedInit *>(this),
-                                                Element));
-  return ListInit::get(ListInits, T->getElementType());
-}
-
-
 VarInit *VarInit::get(StringRef VN, RecTy *T) {
   Init *Value = StringInit::get(T->getRecordKeeper(), VN);
   return VarInit::get(Value, T);
@@ -1954,37 +1987,6 @@ Init *VarBitInit::resolveReferences(Resolver &R) const {
   return const_cast<VarBitInit*>(this);
 }
 
-VarListElementInit *VarListElementInit::get(TypedInit *T, unsigned E) {
-  detail::RecordKeeperImpl &RK = T->getRecordKeeper().getImpl();
-  VarListElementInit *&I = RK.TheVarListElementInitPool[std::make_pair(T, E)];
-  if (!I)
-    I = new (RK.Allocator) VarListElementInit(T, E);
-  return I;
-}
-
-std::string VarListElementInit::getAsString() const {
-  return TI->getAsString() + "[" + utostr(Element) + "]";
-}
-
-Init *VarListElementInit::resolveReferences(Resolver &R) const {
-  Init *NewTI = TI->resolveReferences(R);
-  if (ListInit *List = dyn_cast<ListInit>(NewTI)) {
-    // Leave out-of-bounds array references as-is. This can happen without
-    // being an error, e.g. in the untaken "branch" of an !if expression.
-    if (getElementNum() < List->size())
-      return List->getElement(getElementNum());
-  }
-  if (NewTI != TI && isa<TypedInit>(NewTI))
-    return VarListElementInit::get(cast<TypedInit>(NewTI), getElementNum());
-  return const_cast<VarListElementInit *>(this);
-}
-
-Init *VarListElementInit::getBit(unsigned Bit) const {
-  if (getType() == BitRecTy::get(getRecordKeeper()))
-    return const_cast<VarListElementInit*>(this);
-  return VarBitInit::get(const_cast<VarListElementInit*>(this), Bit);
-}
-
 DefInit::DefInit(Record *D)
     : TypedInit(IK_DefInit, D->getType()), Def(D) {}
 
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index ba63d0b671ce6..24ec4031a7eda 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -585,6 +585,7 @@ tgtok::TokKind TGLexer::LexExclaim() {
           .Case("listconcat", tgtok::XListConcat)
           .Case("listsplat", tgtok::XListSplat)
           .Case("listremove", tgtok::XListRemove)
+          .Case("range", tgtok::XRange)
           .Case("strconcat", tgtok::XStrConcat)
           .Case("interleave", tgtok::XInterleave)
           .Case("substr", tgtok::XSubstr)
diff --git a/llvm/lib/TableGen/TGLexer.h b/llvm/lib/TableGen/TGLexer.h
index 93e28a4417a47..35976c666d122 100644
--- a/llvm/lib/TableGen/TGLexer.h
+++ b/llvm/lib/TableGen/TGLexer.h
@@ -56,7 +56,7 @@ enum TokKind {
     XSHL, XListConcat, XListSplat, XStrConcat, XInterleave, XSubstr, XFind,
     XCast, XSubst, XForEach, XFilter, XFoldl, XHead, XTail, XSize, XEmpty, XIf,
     XCond, XEq, XIsA, XDag, XNe, XLe, XLt, XGe, XGt, XSetDagOp, XGetDagOp,
-    XExists, XListRemove,  XToLower, XToUpper,
+    XExists, XListRemove, XToLower, XToUpper, XRange,
 
   // Boolean literals.
     TrueVal, FalseVal,
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 14c529caa5947..ecd19b98e1b27 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -709,6 +709,148 @@ ParseSubMultiClassReference(MultiClass *CurMC) {
   return Result;
 }
 
+/// ParseSliceElement - Parse subscript or range
+///
+///  SliceElement  ::= Value<list<int>>
+///  SliceElement  ::= Value<int>
+///  SliceElement  ::= Value<int> '...' Value<int>
+///  SliceElement  ::= Value<int> '-' Value<int> (deprecated)
+///  SliceElement  ::= Value<int> INTVAL(Negative; deprecated)
+///
+/// SliceElement is either IntRecTy, ListRecTy, or nullptr
+///
+TypedInit *TGParser::ParseSliceElement(Record *CurRec) {
+  auto LHSLoc = Lex.getLoc();
+  auto *CurVal = ParseValue(CurRec);
+  if (!CurVal)
+    return nullptr;
+  auto *LHS = cast<TypedInit>(CurVal);
+
+  TypedInit *RHS = nullptr;
+  switch (Lex.getCode()) {
+  case tgtok::dotdotdot:
+  case tgtok::minus: { // Deprecated
+    Lex.Lex();         // eat
+    auto RHSLoc = Lex.getLoc();
+    CurVal = ParseValue(CurRec);
+    if (!CurVal)
+      return nullptr;
+    RHS = cast<TypedInit>(CurVal);
+    if (!isa<IntRecTy>(RHS->getType())) {
+      Error(RHSLoc,
+            "expected int...int, got " + Twine(RHS->getType()->getAsString()));
+      return nullptr;
+    }
+    break;
+  }
+  case tgtok::IntVal: { // Deprecated "-num"
+    auto i = -Lex.getCurIntVal();
+    if (i < 0) {
+      TokError("invalid range, cannot be negative");
+      return nullptr;
+    }
+    RHS = IntInit::get(Records, i);
+    Lex.Lex(); // eat IntVal
+    break;
+  }
+  default: // Single value (IntRecTy or ListRecTy)
+    return LHS;
+  }
+
+  assert(RHS);
+  assert(isa<IntRecTy>(RHS->getType()));
+
+  // Closed-interval range <LHS:IntRecTy>...<RHS:IntRecTy>
+  if (!isa<IntRecTy>(LHS->getType())) {
+    Error(LHSLoc,
+          "expected int...int, got " + Twine(LHS->getType()->getAsString()));
+    return nullptr;
+  }
+
+  return cast<TypedInit>(BinOpInit::get(BinOpInit::RANGEC, LHS, RHS,
+                                        IntRecTy::get(Records)->getListTy())
+                             ->Fold(CurRec));
+}
+
+/// ParseSliceElements - Parse subscripts in square brackets.
+///
+///  SliceElements ::= ( SliceElement ',' )* SliceElement ','?
+///
+/// SliceElement is either IntRecTy, ListRecTy, or nullptr
+///
+/// Returns ListRecTy by defaut.
+/// Returns IntRecTy if;
+///  - Single=true
+///  - SliceElements is Value<int> w/o trailing comma
+///
+TypedInit *TGParser::ParseSliceElements(Record *CurRec, bool Single) {
+  TypedInit *CurVal;
+  SmallVector<Init *, 2> Elems;       // int
+  SmallVector<TypedInit *, 2> Slices; // list<int>
+
+  auto FlushElems = [&] {
+    if (!Elems.empty()) {
+      Slices.push_back(ListInit::get(Elems, IntRecTy::get(Records)));
+      Elems.clear();
+    }
+  };
+
+  do {
+    auto LHSLoc = Lex.getLoc();
+    CurVal = ParseSliceElement(CurRec);
+    if (!CurVal)
+      return nullptr;
+    auto *CurValTy = CurVal->getType();
+
+    if (auto *ListValTy = dyn_cast<ListRecTy>(CurValTy)) {
+      if (!isa<IntRecTy>(ListValTy->getElementType())) {
+        Error(LHSLoc,
+              "expected list<int>, got " + Twine(ListValTy->getAsString()));
+        return nullptr;
+      }
+
+      FlushElems();
+      Slices.push_back(CurVal);
+      Single = false;
+      CurVal = nullptr;
+    } else if (!isa<IntRecTy>(CurValTy)) {
+      Error(LHSLoc,
+            "unhandled type " + Twine(CurValTy->getAsString()) + " in range");
+      return nullptr;
+    }
+
+    if (Lex.getCode() != tgtok::comma)
+      break;
+
+    Lex.Lex(); // eat comma
+
+    // `[i,]` is not LISTELEM but LISTSLICE
+    Single = false;
+    if (CurVal)
+      Elems.push_back(CurVal);
+    CurVal = nullptr;
+  } while (Lex.getCode() != tgtok::r_square);
+
+  if (CurVal) {
+    // LISTELEM
+    if (Single)
+      return CurVal;
+
+    Elems.push_back(CurVal);
+  }
+
+  FlushElems();
+
+  // Concatenate lists in Slices
+  TypedInit *Result = nullptr;
+  for (auto *Slice : Slices) {
+    Result = (Result ? cast<TypedInit>(BinOpInit::getListConcat(Result, Slice))
+                     : Slice);
+  }
+
+  return Result;
+}
+
 /// ParseRangePiece - Parse a bit/value range.
 ///   RangePiece ::= INTVAL
 ///   RangePiece ::= INTVAL '...' INTVAL
@@ -1217,6 +1359,7 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
   case tgtok::XListConcat:
   case tgtok::XListSplat:
   case tgtok::XListRemove:
+  case tgtok::XRange:
   case tgtok::XStrConcat:
   case tgtok::XInterleave:
   case tgtok::XSetDagOp: { // Value ::= !binop '(' Value ',' Value ')'
@@ -1247,6 +1390,7 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
     case tgtok::XListConcat: Code = BinOpInit::LISTCONCAT; break;
     case tgtok::XListSplat:  Code = BinOpInit::LISTSPLAT; break;
     case tgtok::XListRemove: Code = BinOpInit::LISTREMOVE; break;
+    case tgtok::XRange:      Code = BinOpInit::RANGE; break;
     case tgtok::XStrConcat:  Code = BinOpInit::STRCONCAT; break;
     case tgtok::XInterleave: Code = BinOpInit::INTERLEAVE; break;
     case tgtok::XSetDagOp:   Code = BinOpInit::SETDAGOP; break;
@@ -1295,6 +1439,10 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
       // We don't know the list type until we parse the first argument.
       ArgType = ItemType;
       break;
+    case tgtok::XRange:
+      Type = IntRecTy::get(Records)->getListTy();
+      // ArgType may be either Int or List.
+      break;
     case tgtok::XStrConcat:
       Type = StringRecTy::get(Records);
       ArgType = StringRecTy::get(Records);
@@ -1379,6 +1527,27 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
             return nullptr;
           }
           break;
+        case BinOpInit::RANGE:
+          if (InitList.size() == 1) {
+            if (isa<ListRecTy>(ArgType)) {
+              ArgType = nullptr; // Detect error if 2nd arg were present.
+            } else if (isa<IntRecTy>(ArgType)) {
+              // Assume 2nd arg should be IntRecTy
+            } else {
+              Error(InitLoc,
+                    Twine("expected list or int, got value of type '") +
+                        ArgType->getAsString() + "'");
+              return nullptr;
+            }
+          } else {
+            // Don't come here unless 1st arg is ListRecTy.
+            assert(isa<ListRecTy>(cast<TypedInit>(InitList[0])->getType()));
+            Error(InitLoc,
+                  Twine("expected one list, got extra value of type '") +
+                      ArgType->getAsString() + "'");
+            return nullptr;
+          }
+          break;
         case BinOpInit::EQ:
         case BinOpInit::NE:
           if (!ArgType->typeIsConvertibleTo(IntRecTy::get(Records)) &&
@@ -1477,6 +1646,37 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
     if (Code == BinOpInit::LISTREMOVE)
       Type = ArgType;
 
+    if (Code == BinOpInit::RANGE) {
+      Init *LHS, *RHS;
+      auto ArgCount = InitList.size();
+      assert(ArgCount >= 1);
+      auto *Arg0 = cast<TypedInit>(InitList[0]);
+      auto *Arg0Ty = Arg0->getType();
+      if (ArgCount == 1) {
+        if (isa<ListRecTy>(Arg0Ty)) {
+          // (0, !size(arg))
+          LHS = IntInit::get(Records, 0);
+          RHS = UnOpInit::get(UnOpInit::SIZE, Arg0, IntRecTy::get(Records))
+                    ->Fold(CurRec);
+        } else {
+          assert(isa<IntRecTy>(Arg0Ty));
+          // (0, arg)
+          LHS = IntInit::get(Records, 0);
+          RHS = Arg0;
+        }
+      } else if (ArgCount == 2) {
+        assert(isa<IntRecTy>(Arg0Ty));
+        auto *Arg1 = cast<TypedInit>(InitList[1]);
+        assert(isa<IntRecTy>(Arg1->getType()));
+        LHS = Arg0;
+        RHS = Arg1;
+      } else {
+        Error(OpLoc, "expected at most two values of integer");
+        return nullptr;
+      }
+      return BinOpInit::get(Code, LHS, RHS, Type)->Fold(CurRec);
+    }
+
     // We allow multiple operands to associative operators like !strconcat as
     // shorthand for nesting them.
     if (Code == BinOpInit::STRCONCAT || Code == BinOpInit::LISTCONCAT ||
@@ -2208,6 +2408,8 @@ Init *TGParser::ParseOperationCond(Record *CurRec, RecTy *ItemType) {
 ///   SimpleValue ::= LISTCONCATTOK '(' Value ',' Value ')'
 ///   SimpleValue ::= LISTSPLATTOK '(' Value ',' Value ')'
 ///   SimpleValue ::= LISTREMOVETOK '(' Value ',' Value ')'
+///   SimpleValue ::= RANGE '(' Value ')'
+///   SimpleValue ::= RANGE '(' Value ',' Value ')'
 ///   SimpleValue ::= STRCONCATTOK '(' Value ',' Value ')'
 ///   SimpleValue ::= COND '(' [Value ':' Value,]+ ')'
 ///
@@ -2510,6 +2712,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
   case tgtok::XListConcat:
   case tgtok::XListSplat:
   case tgtok::XListRemove:
+  case tgtok::XRange:
   case tgtok::XStrConcat:
   case tgtok::XInterleave:
   case tgtok::XSetDagOp: // Value ::= !binop '(' Value ',' Value ')'
@@ -2532,10 +2735,11 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
 ///
 ///   Value       ::= SimpleValue ValueSuffix*
 ///   ValueSuffix ::= '{' BitList '}'
-///   ValueSuffix ::= '[' BitList ']'
+///   ValueSuffix ::= '[' SliceElements ']'
 ///   ValueSuffix ::= '.' ID
 ///
 Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
+  SMLoc LHSLoc = Lex.getLoc();
   Init *Result = ParseSimpleValue(CurRec, ItemType, Mode);
   if (!Result) return nullptr;
 
@@ -2570,18 +2774,35 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
       break;
     }
     case tgtok::l_square: {
-      SMLoc SquareLoc = Lex.getLoc();
-      Lex.Lex(); // eat the '['
-      SmallVector<unsigned, 16> Ranges;
-      ParseRangeList(Ranges);
-      if (Ranges.empty()) return nullptr;
+      auto *LHS = dyn_cast<TypedInit>(Result);
+      if (!LHS) {
+        Error(LHSLoc, "Invalid value, list expected");
+        return nullptr;
+      }
 
-      Result = Result->convertInitListSlice(Ranges);
-      if (!Result) {
-        Error(SquareLoc, "Invalid range for list slice");
+      auto *LHSTy = dyn_cast<ListRecTy>(LHS->getType());
+      if (!LHSTy) {
+        Error(LHSLoc, "Type '" + Twine(LHS->getType()->getAsString()) +
+                          "' is invalid, list expected");
+        return nullptr;
+      }
+
+      Lex.Lex(); // eat the '['
+      TypedInit *RHS = ParseSliceElements(CurRec, /*Single=*/true);
+      if (!RHS)
         return nullptr;
+
+      if (isa<ListRecTy>(RHS->getType())) {
+        Result =
+            BinOpInit::get(BinOpInit::LISTSLICE, LHS, RHS, LHSTy)->Fold(CurRec);
+      } else {
+        Result = BinOpInit::get(BinOpInit::LISTELEM, LHS, RHS,
+                                LHSTy->getElementType())
+                     ->Fold(CurRec);
       }
 
+      assert(Result);
+
       // Eat the ']'.
       if (!consume(tgtok::r_square)) {
         TokError("expected ']' at end of list slice");
diff --git a/llvm/lib/TableGen/TGParser.h b/llvm/lib/TableGen/TGParser.h
index 63a8db0f4f6bc..4569b22151fdc 100644
--- a/llvm/lib/TableGen/TGParser.h
+++ b/llvm/lib/TableGen/TGParser.h
@@ -265,6 +265,8 @@ class TGParser {
       Record *CurRec);
   bool ParseOptionalRangeList(SmallVectorImpl<unsigned> &Ranges);
   bool ParseOptionalBitList(SmallVectorImpl<unsigned> &Ranges);
+  TypedInit *ParseSliceElement(Record *CurRec);
+  TypedInit *ParseSliceElements(Record *CurRec, bool Single = false);
   void ParseRangeList(SmallVectorImpl<unsigned> &Result);
   bool ParseRangePiece(SmallVectorImpl<unsigned> &Ranges,
                        TypedInit *FirstItem = nullptr);
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 1c9a790c6d72f..9e433db8f23a6 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -531,14 +531,14 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
 
     if (HasMatchAllTag) {
       OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
-                                       .addReg(AArch64::X16)
+                                       .addReg(AArch64::X17)
                                        .addReg(Reg)
                                        .addImm(56)
                                        .addImm(63),
                                    *STI);
       OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
                                        .addReg(AArch64::XZR)
-                                       .addReg(AArch64::X16)
+                                       .addReg(AArch64::X17)
                                        .addImm(MatchAllTag)
                                        .addImm(0),
                                    *STI);
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.td b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
index 853975c6193d9..ce087be79202e 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.td
@@ -489,6 +489,9 @@ def CSR_AArch64_NoRegs : CalleeSavedRegs<(add)>;
 def CSR_AArch64_RT_MostRegs :  CalleeSavedRegs<(add CSR_AArch64_AAPCS,
                                                 (sequence "X%u", 9, 15))>;
 
+def CSR_AArch64_RT_AllRegs :  CalleeSavedRegs<(add CSR_AArch64_RT_MostRegs,
+                                                (sequence "Q%u", 8, 31))>;
+
 def CSR_AArch64_StackProbe_Windows
     : CalleeSavedRegs<(add (sequence "X%u", 0, 15),
                            (sequence "X%u", 18, 28), FP, SP,
@@ -551,6 +554,9 @@ def CSR_Darwin_AArch64_CXX_TLS_ViaCopy
 def CSR_Darwin_AArch64_RT_MostRegs
     : CalleeSavedRegs<(add CSR_Darwin_AArch64_AAPCS, (sequence "X%u", 9, 15))>;
 
+def CSR_Darwin_AArch64_RT_AllRegs
+    : CalleeSavedRegs<(add CSR_Darwin_AArch64_RT_MostRegs, (sequence "Q%u", 8, 31))>;
+
 // Variants of the standard calling conventions for shadow call stack.
 // These all preserve x18 in addition to any other registers.
 def CSR_AArch64_NoRegs_SCS
@@ -561,6 +567,8 @@ def CSR_AArch64_AAPCS_SwiftError_SCS
     : CalleeSavedRegs<(add CSR_AArch64_AAPCS_SwiftError, X18)>;
 def CSR_AArch64_RT_MostRegs_SCS
     : CalleeSavedRegs<(add CSR_AArch64_RT_MostRegs, X18)>;
+def CSR_AArch64_RT_AllRegs_SCS
+    : CalleeSavedRegs<(add CSR_AArch64_RT_AllRegs, X18)>;
 def CSR_AArch64_AAVPCS_SCS
     : CalleeSavedRegs<(add CSR_AArch64_AAVPCS, X18)>;
 def CSR_AArch64_SVE_AAPCS_SCS
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 8575aacf78ff7..f805adc198156 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -35,6 +35,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Argument.h"
@@ -53,6 +54,7 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
@@ -65,7 +67,6 @@
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include <algorithm>
 #include <cassert>
@@ -272,7 +273,7 @@ class AArch64FastISel final : public FastISel {
   CCAssignFn *CCAssignFnForCall(CallingConv::ID CC) const;
   bool processCallArgs(CallLoweringInfo &CLI, SmallVectorImpl<MVT> &ArgVTs,
                        unsigned &NumBytes);
-  bool finishCall(CallLoweringInfo &CLI, MVT RetVT, unsigned NumBytes);
+  bool finishCall(CallLoweringInfo &CLI, unsigned NumBytes);
 
 public:
   // Backend specific FastISel code.
@@ -3102,8 +3103,7 @@ bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
   return true;
 }
 
-bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,
-                                 unsigned NumBytes) {
+bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, unsigned NumBytes) {
   CallingConv::ID CC = CLI.CallConv;
 
   // Issue CALLSEQ_END
@@ -3111,33 +3111,31 @@ bool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackUp))
     .addImm(NumBytes).addImm(0);
 
-  // Now the return value.
-  if (RetVT != MVT::isVoid) {
-    SmallVector<CCValAssign, 16> RVLocs;
-    CCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC));
+  // Now the return values.
+  SmallVector<CCValAssign, 16> RVLocs;
+  CCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context);
+  CCInfo.AnalyzeCallResult(CLI.Ins, CCAssignFnForCall(CC));
 
-    // Only handle a single return value.
-    if (RVLocs.size() != 1)
-      return false;
-
-    // Copy all of the result registers out of their specified physreg.
-    MVT CopyVT = RVLocs[0].getValVT();
+  Register ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
+  for (unsigned i = 0; i != RVLocs.size(); ++i) {
+    CCValAssign &VA = RVLocs[i];
+    MVT CopyVT = VA.getValVT();
+    unsigned CopyReg = ResultReg + i;
 
     // TODO: Handle big-endian results
     if (CopyVT.isVector() && !Subtarget->isLittleEndian())
       return false;
 
-    Register ResultReg = createResultReg(TLI.getRegClassFor(CopyVT));
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
-            TII.get(TargetOpcode::COPY), ResultReg)
-        .addReg(RVLocs[0].getLocReg());
-    CLI.InRegs.push_back(RVLocs[0].getLocReg());
-
-    CLI.ResultReg = ResultReg;
-    CLI.NumResultRegs = 1;
+    // Copy result out of their specified physreg.
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
+            CopyReg)
+        .addReg(VA.getLocReg());
+    CLI.InRegs.push_back(VA.getLocReg());
   }
 
+  CLI.ResultReg = ResultReg;
+  CLI.NumResultRegs = RVLocs.size();
+
   return true;
 }
 
@@ -3185,13 +3183,6 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   if (IsVarArg)
     return false;
 
-  // FIXME: Only handle *simple* calls for now.
-  MVT RetVT;
-  if (CLI.RetTy->isVoidTy())
-    RetVT = MVT::isVoid;
-  else if (!isTypeLegal(CLI.RetTy, RetVT))
-    return false;
-
   for (auto Flag : CLI.OutFlags)
     if (Flag.isInReg() || Flag.isSRet() || Flag.isNest() || Flag.isByVal() ||
         Flag.isSwiftSelf() || Flag.isSwiftAsync() || Flag.isSwiftError())
@@ -3287,7 +3278,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   CLI.Call = MIB;
 
   // Finish off the call including any return values.
-  return finishCall(CLI, RetVT, NumBytes);
+  return finishCall(CLI, NumBytes);
 }
 
 bool AArch64FastISel::isMemCpySmall(uint64_t Len, MaybeAlign Alignment) {
@@ -3786,6 +3777,57 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     updateValueMap(II, ResultReg1, 2);
     return true;
   }
+  case Intrinsic::aarch64_crc32b:
+  case Intrinsic::aarch64_crc32h:
+  case Intrinsic::aarch64_crc32w:
+  case Intrinsic::aarch64_crc32x:
+  case Intrinsic::aarch64_crc32cb:
+  case Intrinsic::aarch64_crc32ch:
+  case Intrinsic::aarch64_crc32cw:
+  case Intrinsic::aarch64_crc32cx: {
+    if (!Subtarget->hasCRC())
+      return false;
+
+    unsigned Opc;
+    switch (II->getIntrinsicID()) {
+    default:
+      llvm_unreachable("Unexpected intrinsic!");
+    case Intrinsic::aarch64_crc32b:
+      Opc = AArch64::CRC32Brr;
+      break;
+    case Intrinsic::aarch64_crc32h:
+      Opc = AArch64::CRC32Hrr;
+      break;
+    case Intrinsic::aarch64_crc32w:
+      Opc = AArch64::CRC32Wrr;
+      break;
+    case Intrinsic::aarch64_crc32x:
+      Opc = AArch64::CRC32Xrr;
+      break;
+    case Intrinsic::aarch64_crc32cb:
+      Opc = AArch64::CRC32CBrr;
+      break;
+    case Intrinsic::aarch64_crc32ch:
+      Opc = AArch64::CRC32CHrr;
+      break;
+    case Intrinsic::aarch64_crc32cw:
+      Opc = AArch64::CRC32CWrr;
+      break;
+    case Intrinsic::aarch64_crc32cx:
+      Opc = AArch64::CRC32CXrr;
+      break;
+    }
+
+    Register LHSReg = getRegForValue(II->getArgOperand(0));
+    Register RHSReg = getRegForValue(II->getArgOperand(1));
+    if (!LHSReg || !RHSReg)
+      return false;
+
+    Register ResultReg =
+        fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, LHSReg, RHSReg);
+    updateValueMap(II, ResultReg);
+    return true;
+  }
   }
   return false;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 33b586e2c1b40..72d69a8cf0a65 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2541,8 +2541,8 @@ static void computeCalleeSaveRegisterPairs(
   // MachO's compact unwind format relies on all registers being stored in
   // pairs.
   assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
-          CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 ||
-          (Count & 1) == 0) &&
+          CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
+          CC == CallingConv::Win64 || (Count & 1) == 0) &&
          "Odd number of callee-saved regs to spill!");
   int ByteOffset = AFI->getCalleeSavedStackSize();
   int StackFillDir = -1;
@@ -2628,7 +2628,8 @@ static void computeCalleeSaveRegisterPairs(
     // MachO's compact unwind format relies on all registers being stored in
     // adjacent register pairs.
     assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
-            CC == CallingConv::CXX_FAST_TLS || CC == CallingConv::Win64 ||
+            CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
+            CC == CallingConv::Win64 ||
             (RPI.isPaired() &&
              ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
               RPI.Reg1 + 1 == RPI.Reg2))) &&
@@ -3394,7 +3395,8 @@ class TagStoreEdit {
   Register FrameReg;
   StackOffset FrameRegOffset;
   int64_t Size;
-  // If not None, move FrameReg to (FrameReg + FrameRegUpdate) at the end.
+  // If not std::nullopt, move FrameReg to (FrameReg + FrameRegUpdate) at the
+  // end.
   std::optional<int64_t> FrameRegUpdate;
   // MIFlags for any FrameReg updating instructions.
   unsigned FrameRegUpdateFlags;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index cb54ee082e341..f55e269e4dd6a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -44,6 +44,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -80,7 +81,6 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -645,10 +645,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::UMULO, MVT::i32, Custom);
   setOperationAction(ISD::UMULO, MVT::i64, Custom);
 
-  setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
-  setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
-  setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
-  setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
+  setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom);
+  setOperationAction(ISD::UADDO_CARRY, MVT::i64, Custom);
+  setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom);
+  setOperationAction(ISD::USUBO_CARRY, MVT::i64, Custom);
   setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
   setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom);
   setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
@@ -1001,6 +1001,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   setTargetDAGCombine(ISD::CTLZ);
 
+  setTargetDAGCombine(ISD::VECREDUCE_AND);
+  setTargetDAGCombine(ISD::VECREDUCE_OR);
+  setTargetDAGCombine(ISD::VECREDUCE_XOR);
+
   // In case of strict alignment, avoid an excessive number of byte wide stores.
   MaxStoresPerMemsetOptSize = 8;
   MaxStoresPerMemset =
@@ -1165,8 +1169,14 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
       setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
       setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
     }
     setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
+    setOperationAction(ISD::VECREDUCE_AND, MVT::v2i64, Custom);
+    setOperationAction(ISD::VECREDUCE_OR, MVT::v2i64, Custom);
+    setOperationAction(ISD::VECREDUCE_XOR, MVT::v2i64, Custom);
 
     setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
     setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
@@ -2141,6 +2151,38 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode(
   }
 }
 
+unsigned AArch64TargetLowering::ComputeNumSignBitsForTargetNode(
+    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
+    unsigned Depth) const {
+  EVT VT = Op.getValueType();
+  unsigned VTBits = VT.getScalarSizeInBits();
+  unsigned Opcode = Op.getOpcode();
+  switch (Opcode) {
+    case AArch64ISD::CMEQ:
+    case AArch64ISD::CMGE:
+    case AArch64ISD::CMGT:
+    case AArch64ISD::CMHI:
+    case AArch64ISD::CMHS:
+    case AArch64ISD::FCMEQ:
+    case AArch64ISD::FCMGE:
+    case AArch64ISD::FCMGT:
+    case AArch64ISD::CMEQz:
+    case AArch64ISD::CMGEz:
+    case AArch64ISD::CMGTz:
+    case AArch64ISD::CMLEz:
+    case AArch64ISD::CMLTz:
+    case AArch64ISD::FCMEQz:
+    case AArch64ISD::FCMGEz:
+    case AArch64ISD::FCMGTz:
+    case AArch64ISD::FCMLEz:
+    case AArch64ISD::FCMLTz:
+      // Compares return either 0 or all-ones
+      return VTBits;
+  }
+
+  return 1;
+}
+
 MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
                                                   EVT) const {
   return MVT::i64;
@@ -3683,8 +3725,8 @@ static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG) {
 
 // This lowering is inefficient, but it will get cleaned up by
 // `foldOverflowCheck`
-static SDValue lowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG, unsigned Opcode,
-                                bool IsSigned) {
+static SDValue lowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
+                                  unsigned Opcode, bool IsSigned) {
   EVT VT0 = Op.getValue(0).getValueType();
   EVT VT1 = Op.getValue(1).getValueType();
 
@@ -4534,8 +4576,8 @@ SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
 
   // If SVE is available then i64 vector multiplications can also be made legal.
-  bool OverrideNEON = VT == MVT::v2i64 || VT == MVT::v1i64 ||
-                      Subtarget->forceStreamingCompatibleSVE();
+  bool OverrideNEON =
+      VT == MVT::v1i64 || Subtarget->forceStreamingCompatibleSVE();
 
   if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED);
@@ -4551,10 +4593,14 @@ SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
   unsigned NewOpc = selectUmullSmull(N0, N1, DAG, DL, isMLA);
 
   if (!NewOpc) {
-    if (VT == MVT::v2i64)
+    if (VT == MVT::v2i64) {
+      // If SVE is available then i64 vector multiplications can also be made
+      // legal.
+      if (Subtarget->hasSVE())
+        return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED);
       // Fall through to expand this.  It is not legal.
       return SDValue();
-    else
+    } else
       // Other vector multiplications are legal.
       return Op;
   }
@@ -5756,14 +5802,14 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerVACOPY(Op, DAG);
   case ISD::VAARG:
     return LowerVAARG(Op, DAG);
-  case ISD::ADDCARRY:
-    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::ADCS, false /*unsigned*/);
-  case ISD::SUBCARRY:
-    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::SBCS, false /*unsigned*/);
+  case ISD::UADDO_CARRY:
+    return lowerADDSUBO_CARRY(Op, DAG, AArch64ISD::ADCS, false /*unsigned*/);
+  case ISD::USUBO_CARRY:
+    return lowerADDSUBO_CARRY(Op, DAG, AArch64ISD::SBCS, false /*unsigned*/);
   case ISD::SADDO_CARRY:
-    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::ADCS, true /*signed*/);
+    return lowerADDSUBO_CARRY(Op, DAG, AArch64ISD::ADCS, true /*signed*/);
   case ISD::SSUBO_CARRY:
-    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::SBCS, true /*signed*/);
+    return lowerADDSUBO_CARRY(Op, DAG, AArch64ISD::SBCS, true /*signed*/);
   case ISD::SADDO:
   case ISD::UADDO:
   case ISD::SSUBO:
@@ -6120,6 +6166,7 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
   case CallingConv::C:
   case CallingConv::Fast:
   case CallingConv::PreserveMost:
+  case CallingConv::PreserveAll:
   case CallingConv::CXX_FAST_TLS:
   case CallingConv::Swift:
   case CallingConv::SwiftTail:
@@ -6749,6 +6796,7 @@ static bool mayTailCallThisCC(CallingConv::ID CC) {
   case CallingConv::C:
   case CallingConv::AArch64_SVE_VectorCall:
   case CallingConv::PreserveMost:
+  case CallingConv::PreserveAll:
   case CallingConv::Swift:
   case CallingConv::SwiftTail:
   case CallingConv::Tail:
@@ -9864,6 +9912,72 @@ static PredicateConstraint parsePredicateConstraint(StringRef Constraint) {
   return P;
 }
 
+// The set of cc code supported is from
+// https://gcc.gnu.org/onlinedocs/gcc/Extended-Asm.html#Flag-Output-Operands
+static AArch64CC::CondCode parseConstraintCode(llvm::StringRef Constraint) {
+  AArch64CC::CondCode Cond = StringSwitch<AArch64CC::CondCode>(Constraint)
+                                 .Case("{@cchi}", AArch64CC::HI)
+                                 .Case("{@cccs}", AArch64CC::HS)
+                                 .Case("{@cclo}", AArch64CC::LO)
+                                 .Case("{@ccls}", AArch64CC::LS)
+                                 .Case("{@cccc}", AArch64CC::LO)
+                                 .Case("{@cceq}", AArch64CC::EQ)
+                                 .Case("{@ccgt}", AArch64CC::GT)
+                                 .Case("{@ccge}", AArch64CC::GE)
+                                 .Case("{@cclt}", AArch64CC::LT)
+                                 .Case("{@ccle}", AArch64CC::LE)
+                                 .Case("{@cchs}", AArch64CC::HS)
+                                 .Case("{@ccne}", AArch64CC::NE)
+                                 .Case("{@ccvc}", AArch64CC::VC)
+                                 .Case("{@ccpl}", AArch64CC::PL)
+                                 .Case("{@ccvs}", AArch64CC::VS)
+                                 .Case("{@ccmi}", AArch64CC::MI)
+                                 .Default(AArch64CC::Invalid);
+  return Cond;
+}
+
+/// Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR,
+/// WZR, invert(<cond>)'.
+static SDValue getSETCC(AArch64CC::CondCode CC, SDValue NZCV, const SDLoc &DL,
+                        SelectionDAG &DAG) {
+  return DAG.getNode(
+      AArch64ISD::CSINC, DL, MVT::i32, DAG.getConstant(0, DL, MVT::i32),
+      DAG.getConstant(0, DL, MVT::i32),
+      DAG.getConstant(getInvertedCondCode(CC), DL, MVT::i32), NZCV);
+}
+
+// Lower @cc flag output via getSETCC.
+SDValue AArch64TargetLowering::LowerAsmOutputForConstraint(
+    SDValue &Chain, SDValue &Glue, const SDLoc &DL,
+    const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
+  AArch64CC::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
+  if (Cond == AArch64CC::Invalid)
+    return SDValue();
+  // The output variable should be a scalar integer.
+  if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
+      OpInfo.ConstraintVT.getSizeInBits() < 8)
+    report_fatal_error("Flag output operand is of invalid type");
+
+  // Get NZCV register. Only update chain when copyfrom is glued.
+  if (Glue.getNode()) {
+    Glue = DAG.getCopyFromReg(Chain, DL, AArch64::NZCV, MVT::i32, Glue);
+    Chain = Glue.getValue(1);
+  } else
+    Glue = DAG.getCopyFromReg(Chain, DL, AArch64::NZCV, MVT::i32);
+  // Extract CC code.
+  SDValue CC = getSETCC(Cond, Glue, DL, DAG);
+
+  SDValue Result;
+
+  // Truncate or ZERO_EXTEND based on value types.
+  if (OpInfo.ConstraintVT.getSizeInBits() <= 32)
+    Result = DAG.getNode(ISD::TRUNCATE, DL, OpInfo.ConstraintVT, CC);
+  else
+    Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
+
+  return Result;
+}
+
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 AArch64TargetLowering::ConstraintType
@@ -9896,6 +10010,8 @@ AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
   } else if (parsePredicateConstraint(Constraint) !=
              PredicateConstraint::Invalid)
       return C_RegisterClass;
+  else if (parseConstraintCode(Constraint) != AArch64CC::Invalid)
+    return C_Other;
   return TargetLowering::getConstraintType(Constraint);
 }
 
@@ -9993,7 +10109,8 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
                         : std::make_pair(0U, &AArch64::PPRRegClass);
     }
   }
-  if (StringRef("{cc}").equals_insensitive(Constraint))
+  if (StringRef("{cc}").equals_insensitive(Constraint) ||
+      parseConstraintCode(Constraint) != AArch64CC::Invalid)
     return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
 
   // Use the default implementation in TargetLowering to convert the register
@@ -13199,6 +13316,106 @@ static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
                      DAG.getConstant(0, DL, MVT::i64));
 }
 
+static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
+                                      SDLoc DL, SelectionDAG &DAG) {
+  unsigned ScalarOpcode;
+  switch (Opcode) {
+  case ISD::VECREDUCE_AND:
+    ScalarOpcode = ISD::AND;
+    break;
+  case ISD::VECREDUCE_OR:
+    ScalarOpcode = ISD::OR;
+    break;
+  case ISD::VECREDUCE_XOR:
+    ScalarOpcode = ISD::XOR;
+    break;
+  default:
+    llvm_unreachable("Expected bitwise vector reduction");
+    return SDValue();
+  }
+
+  EVT VecVT = Vec.getValueType();
+  assert(VecVT.isFixedLengthVector() && VecVT.isPow2VectorType() &&
+         "Expected power-of-2 length vector");
+
+  EVT ElemVT = VecVT.getVectorElementType();
+
+  SDValue Result;
+  unsigned NumElems = VecVT.getVectorNumElements();
+
+  // Special case for boolean reductions
+  if (ElemVT == MVT::i1) {
+    // Split large vectors into smaller ones
+    if (NumElems > 16) {
+      SDValue Lo, Hi;
+      std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
+      EVT HalfVT = Lo.getValueType();
+      SDValue HalfVec = DAG.getNode(ScalarOpcode, DL, HalfVT, Lo, Hi);
+      return getVectorBitwiseReduce(Opcode, HalfVec, VT, DL, DAG);
+    }
+
+    // Vectors that are less than 64 bits get widened to neatly fit a 64 bit
+    // register, so e.g. <4 x i1> gets lowered to <4 x i16>. Sign extending to
+    // this element size leads to the best codegen, since e.g. setcc results
+    // might need to be truncated otherwise.
+    EVT ExtendedVT = MVT::getIntegerVT(std::max(64u / NumElems, 8u));
+
+    // any_ext doesn't work with umin/umax, so only use it for uadd.
+    unsigned ExtendOp =
+        ScalarOpcode == ISD::XOR ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
+    SDValue Extended = DAG.getNode(
+        ExtendOp, DL, VecVT.changeVectorElementType(ExtendedVT), Vec);
+    switch (ScalarOpcode) {
+    case ISD::AND:
+      Result = DAG.getNode(ISD::VECREDUCE_UMIN, DL, ExtendedVT, Extended);
+      break;
+    case ISD::OR:
+      Result = DAG.getNode(ISD::VECREDUCE_UMAX, DL, ExtendedVT, Extended);
+      break;
+    case ISD::XOR:
+      Result = DAG.getNode(ISD::VECREDUCE_ADD, DL, ExtendedVT, Extended);
+      break;
+    default:
+      llvm_unreachable("Unexpected Opcode");
+    }
+
+    Result = DAG.getAnyExtOrTrunc(Result, DL, MVT::i1);
+  } else {
+    // Iteratively split the vector in half and combine using the bitwise
+    // operation until it fits in a 64 bit register.
+    while (VecVT.getSizeInBits() > 64) {
+      SDValue Lo, Hi;
+      std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
+      VecVT = Lo.getValueType();
+      NumElems = VecVT.getVectorNumElements();
+      Vec = DAG.getNode(ScalarOpcode, DL, VecVT, Lo, Hi);
+    }
+
+    EVT ScalarVT = EVT::getIntegerVT(*DAG.getContext(), VecVT.getSizeInBits());
+
+    // Do the remaining work on a scalar since it allows the code generator to
+    // combine the shift and bitwise operation into one instruction and since
+    // integer instructions can have higher throughput than vector instructions.
+    SDValue Scalar = DAG.getBitcast(ScalarVT, Vec);
+
+    // Iteratively combine the lower and upper halves of the scalar using the
+    // bitwise operation, halving the relevant region of the scalar in each
+    // iteration, until the relevant region is just one element of the original
+    // vector.
+    for (unsigned Shift = NumElems / 2; Shift > 0; Shift /= 2) {
+      SDValue ShiftAmount =
+          DAG.getConstant(Shift * ElemVT.getSizeInBits(), DL, MVT::i64);
+      SDValue Shifted =
+          DAG.getNode(ISD::SRL, DL, ScalarVT, Scalar, ShiftAmount);
+      Scalar = DAG.getNode(ScalarOpcode, DL, ScalarVT, Scalar, Shifted);
+    }
+
+    Result = DAG.getAnyExtOrTrunc(Scalar, DL, ElemVT);
+  }
+
+  return DAG.getAnyExtOrTrunc(Result, DL, VT);
+}
+
 SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
@@ -13250,6 +13467,11 @@ SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
   // Lower NEON reductions.
   SDLoc dl(Op);
   switch (Op.getOpcode()) {
+  case ISD::VECREDUCE_AND:
+  case ISD::VECREDUCE_OR:
+  case ISD::VECREDUCE_XOR:
+    return getVectorBitwiseReduce(Op.getOpcode(), Op.getOperand(0),
+                                  Op.getValueType(), dl, DAG);
   case ISD::VECREDUCE_ADD:
     return getReductionSDNode(AArch64ISD::UADDV, dl, Op, DAG);
   case ISD::VECREDUCE_SMAX:
@@ -14671,9 +14893,9 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
   auto Mask = SVI->getShuffleMask();
 
   // Sanity check if all the indices are NOT in range.
-  // If mask is `undef` or `poison`, `Mask` may be a vector of -1s.
-  // If all of them are `undef`, OOB read will happen later.
-  if (llvm::all_of(Mask, [](int Idx) { return Idx == UndefMaskElem; })) {
+  // If mask is `poison`, `Mask` may be a vector of -1s.
+  // If all of them are `poison`, OOB read will happen later.
+  if (llvm::all_of(Mask, [](int Idx) { return Idx == PoisonMaskElem; })) {
     return false;
   }
   // A 64bit st2 which does not start at element 0 will involved adding extra
@@ -16962,6 +17184,51 @@ static SDValue performConcatVectorsCombine(SDNode *N,
     }
   }
 
+  auto IsRSHRN = [](SDValue Shr) {
+    if (Shr.getOpcode() != AArch64ISD::VLSHR)
+      return false;
+    SDValue Op = Shr.getOperand(0);
+    EVT VT = Op.getValueType();
+    unsigned ShtAmt = Shr.getConstantOperandVal(1);
+    if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
+      return false;
+
+    APInt Imm;
+    if (Op.getOperand(1).getOpcode() == AArch64ISD::MOVIshift)
+      Imm = APInt(VT.getScalarSizeInBits(),
+                  Op.getOperand(1).getConstantOperandVal(0)
+                      << Op.getOperand(1).getConstantOperandVal(1));
+    else if (Op.getOperand(1).getOpcode() == AArch64ISD::DUP &&
+             isa<ConstantSDNode>(Op.getOperand(1).getOperand(0)))
+      Imm = APInt(VT.getScalarSizeInBits(),
+                  Op.getOperand(1).getConstantOperandVal(0));
+    else
+      return false;
+
+    if (Imm != 1ULL << (ShtAmt - 1))
+      return false;
+    return true;
+  };
+
+  // concat(rshrn(x), rshrn(y)) -> rshrn(concat(x, y))
+  if (N->getNumOperands() == 2 && IsRSHRN(N0) &&
+      ((IsRSHRN(N1) &&
+        N0.getConstantOperandVal(1) == N1.getConstantOperandVal(1)) ||
+       N1.isUndef())) {
+    SDValue X = N0.getOperand(0).getOperand(0);
+    SDValue Y = N1.isUndef() ? DAG.getUNDEF(X.getValueType())
+                             : N1.getOperand(0).getOperand(0);
+    EVT BVT =
+        X.getValueType().getDoubleNumVectorElementsVT(*DCI.DAG.getContext());
+    SDValue CC = DAG.getNode(ISD::CONCAT_VECTORS, dl, BVT, X, Y);
+    SDValue Add = DAG.getNode(
+        ISD::ADD, dl, BVT, CC,
+        DAG.getConstant(1ULL << (N0.getConstantOperandVal(1) - 1), dl, BVT));
+    SDValue Shr =
+        DAG.getNode(AArch64ISD::VLSHR, dl, BVT, Add, N0.getOperand(1));
+    return Shr;
+  }
+
   // concat(zip1(a, b), zip2(a, b)) is zip1(a, b)
   if (N->getNumOperands() == 2 && N0Opc == AArch64ISD::ZIP1 &&
       N1Opc == AArch64ISD::ZIP2 && N0.getOperand(0) == N1.getOperand(0) &&
@@ -19403,6 +19670,13 @@ static SDValue performVectorShiftCombine(SDNode *N,
   unsigned ShiftImm = N->getConstantOperandVal(1);
   assert(OpScalarSize > ShiftImm && "Invalid shift imm");
 
+  // Remove sign_extend_inreg (ashr(shl(x)) based on the number of sign bits.
+  if (N->getOpcode() == AArch64ISD::VASHR &&
+      Op.getOpcode() == AArch64ISD::VSHL &&
+      N->getOperand(1) == Op.getOperand(1))
+    if (DCI.DAG.ComputeNumSignBits(Op.getOperand(0)) > ShiftImm)
+      return Op.getOperand(0);
+
   APInt ShiftedOutBits = APInt::getLowBitsSet(OpScalarSize, ShiftImm);
   APInt DemandedMask = ~ShiftedOutBits;
 
@@ -19702,20 +19976,25 @@ static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) {
 static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
   SDLoc DL(N);
   SDValue ComparisonResult(N, 0);
-  EVT BoolVecVT = ComparisonResult.getValueType();
-  assert(BoolVecVT.isVector() && "Must be a vector type");
+  EVT VecVT = ComparisonResult.getValueType();
+  assert(VecVT.isVector() && "Must be a vector type");
 
-  unsigned NumElts = BoolVecVT.getVectorNumElements();
+  unsigned NumElts = VecVT.getVectorNumElements();
   if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
     return SDValue();
 
+  if (VecVT.getVectorElementType() != MVT::i1 &&
+      !DAG.getTargetLoweringInfo().isTypeLegal(VecVT))
+    return SDValue();
+
   // If we can find the original types to work on instead of a vector of i1,
   // we can avoid extend/extract conversion instructions.
-  EVT VecVT = tryGetOriginalBoolVectorType(ComparisonResult);
-  if (!VecVT.isSimple()) {
-    unsigned BitsPerElement = std::max(64 / NumElts, 8u); // min. 64-bit vector
-    VecVT =
-        BoolVecVT.changeVectorElementType(MVT::getIntegerVT(BitsPerElement));
+  if (VecVT.getVectorElementType() == MVT::i1) {
+    VecVT = tryGetOriginalBoolVectorType(ComparisonResult);
+    if (!VecVT.isSimple()) {
+      unsigned BitsPerElement = std::max(64 / NumElts, 8u); // >= 64-bit vector
+      VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts);
+    }
   }
   VecVT = VecVT.changeVectorElementTypeToInteger();
 
@@ -19776,6 +20055,37 @@ static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
   return DAG.getNode(ISD::VECREDUCE_ADD, DL, ResultVT, RepresentativeBits);
 }
 
+static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG,
+                                                 StoreSDNode *Store) {
+  if (!Store->isTruncatingStore())
+    return SDValue();
+
+  SDLoc DL(Store);
+  SDValue VecOp = Store->getValue();
+  EVT VT = VecOp.getValueType();
+  EVT MemVT = Store->getMemoryVT();
+
+  if (!MemVT.isVector() || !VT.isVector() ||
+      MemVT.getVectorElementType() != MVT::i1)
+    return SDValue();
+
+  // If we are storing a vector that we are currently building, let
+  // `scalarizeVectorStore()` handle this more efficiently.
+  if (VecOp.getOpcode() == ISD::BUILD_VECTOR)
+    return SDValue();
+
+  VecOp = DAG.getNode(ISD::TRUNCATE, DL, MemVT, VecOp);
+  SDValue VectorBits = vectorToScalarBitmask(VecOp.getNode(), DAG);
+  if (!VectorBits)
+    return SDValue();
+
+  EVT StoreVT =
+      EVT::getIntegerVT(*DAG.getContext(), MemVT.getStoreSizeInBits());
+  SDValue ExtendedBits = DAG.getZExtOrTrunc(VectorBits, DL, StoreVT);
+  return DAG.getStore(Store->getChain(), DL, ExtendedBits, Store->getBasePtr(),
+                      Store->getMemOperand());
+}
+
 static SDValue performSTORECombine(SDNode *N,
                                    TargetLowering::DAGCombinerInfo &DCI,
                                    SelectionDAG &DAG,
@@ -19814,6 +20124,9 @@ static SDValue performSTORECombine(SDNode *N,
   if (SDValue Store = foldTruncStoreOfExt(DAG, N))
     return Store;
 
+  if (SDValue Store = combineBoolVectorAndTruncateStore(DAG, ST))
+    return Store;
+
   return SDValue();
 }
 
@@ -20694,6 +21007,22 @@ static SDValue tryToWidenSetCCOperands(SDNode *Op, SelectionDAG &DAG) {
                      Op0ExtV, Op1ExtV, Op->getOperand(2));
 }
 
+static SDValue
+performVecReduceBitwiseCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+                               SelectionDAG &DAG) {
+  SDValue Vec = N->getOperand(0);
+  if (DCI.isBeforeLegalize() &&
+      Vec.getValueType().getVectorElementType() == MVT::i1 &&
+      Vec.getValueType().isFixedLengthVector() &&
+      Vec.getValueType().isPow2VectorType()) {
+    SDLoc DL(N);
+    return getVectorBitwiseReduce(N->getOpcode(), Vec, N->getValueType(0), DL,
+                                  DAG);
+  }
+
+  return SDValue();
+}
+
 static SDValue performSETCCCombine(SDNode *N,
                                    TargetLowering::DAGCombinerInfo &DCI,
                                    SelectionDAG &DAG) {
@@ -21862,6 +22191,10 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
   default:
     LLVM_DEBUG(dbgs() << "Custom combining: skipping\n");
     break;
+  case ISD::VECREDUCE_AND:
+  case ISD::VECREDUCE_OR:
+  case ISD::VECREDUCE_XOR:
+    return performVecReduceBitwiseCombine(N, DCI, DAG);
   case ISD::ADD:
   case ISD::SUB:
     return performAddSubCombine(N, DCI, DAG);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 82653d473f152..2df9fc869eb05 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -535,6 +535,11 @@ class AArch64TargetLowering : public TargetLowering {
                                      const SelectionDAG &DAG,
                                      unsigned Depth = 0) const override;
 
+  unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
+                                           const APInt &DemandedElts,
+                                           const SelectionDAG &DAG,
+                                           unsigned Depth) const override;
+
   MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override {
     // Returning i64 unconditionally here (i.e. even for ILP32) means that the
     // *DAG* representation of pointers will always be 64-bits. They will be
@@ -1165,6 +1170,12 @@ class AArch64TargetLowering : public TargetLowering {
     return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
   }
 
+  /// Handle Lowering flag assembly outputs.
+  SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
+                                      const SDLoc &DL,
+                                      const AsmOperandInfo &Constraint,
+                                      SelectionDAG &DAG) const override;
+
   bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const override;
   bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const override;
   bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 2bd0c1d782c2f..ff2050a26c645 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -5409,6 +5409,41 @@ static bool getFMULPatterns(MachineInstr &Root,
   return Found;
 }
 
+static bool getFNEGPatterns(MachineInstr &Root,
+                            SmallVectorImpl<MachineCombinerPattern> &Patterns) {
+  unsigned Opc = Root.getOpcode();
+  MachineBasicBlock &MBB = *Root.getParent();
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+  bool Found = false;
+
+  auto Match = [&](unsigned Opcode, MachineCombinerPattern Pattern) -> bool {
+    MachineOperand &MO = Root.getOperand(1);
+    MachineInstr *MI = MRI.getUniqueVRegDef(MO.getReg());
+    if ((MI->getOpcode() == Opcode) &&
+        Root.getFlag(MachineInstr::MIFlag::FmContract) &&
+        Root.getFlag(MachineInstr::MIFlag::FmNsz) &&
+        MI->getFlag(MachineInstr::MIFlag::FmContract) &&
+        MI->getFlag(MachineInstr::MIFlag::FmNsz)) {
+      Patterns.push_back(Pattern);
+      return true;
+    }
+    return false;
+  };
+
+  switch (Opc) {
+  default:
+    return false;
+  case AArch64::FNEGDr:
+    Found |= Match(AArch64::FMADDDrrr, MachineCombinerPattern::FNMADDD);
+    break;
+  case AArch64::FNEGSr:
+    Found |= Match(AArch64::FMADDSrrr, MachineCombinerPattern::FNMADDS);
+    break;
+  }
+
+  return Found;
+}
+
 /// Return true when a code sequence can improve throughput. It
 /// should be called only for instructions in loops.
 /// \param Pattern - combiner pattern
@@ -5578,6 +5613,8 @@ bool AArch64InstrInfo::getMachineCombinerPatterns(
     return true;
   if (getFMAPatterns(Root, Patterns))
     return true;
+  if (getFNEGPatterns(Root, Patterns))
+    return true;
 
   // Other patterns
   if (getMiscPatterns(Root, Patterns))
@@ -5668,6 +5705,39 @@ genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
   return MUL;
 }
 
+static MachineInstr *
+genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
+               const TargetInstrInfo *TII, MachineInstr &Root,
+               SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned Opc,
+               const TargetRegisterClass *RC) {
+  MachineInstr *MAD = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
+  Register ResultReg = Root.getOperand(0).getReg();
+  Register SrcReg0 = MAD->getOperand(1).getReg();
+  Register SrcReg1 = MAD->getOperand(2).getReg();
+  Register SrcReg2 = MAD->getOperand(3).getReg();
+  bool Src0IsKill = MAD->getOperand(1).isKill();
+  bool Src1IsKill = MAD->getOperand(2).isKill();
+  bool Src2IsKill = MAD->getOperand(3).isKill();
+
+  if (ResultReg.isVirtual())
+    MRI.constrainRegClass(ResultReg, RC);
+  if (SrcReg0.isVirtual())
+    MRI.constrainRegClass(SrcReg0, RC);
+  if (SrcReg1.isVirtual())
+    MRI.constrainRegClass(SrcReg1, RC);
+  if (SrcReg2.isVirtual())
+    MRI.constrainRegClass(SrcReg2, RC);
+
+  MachineInstrBuilder MIB =
+      BuildMI(MF, MIMetadata(Root), TII->get(Opc), ResultReg)
+          .addReg(SrcReg0, getKillRegState(Src0IsKill))
+          .addReg(SrcReg1, getKillRegState(Src1IsKill))
+          .addReg(SrcReg2, getKillRegState(Src2IsKill));
+  InsInstrs.push_back(MIB);
+
+  return MAD;
+}
+
 /// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
 static MachineInstr *
 genIndexedMultiply(MachineInstr &Root,
@@ -5894,6 +5964,7 @@ void AArch64InstrInfo::genAlternativeCodeSequence(
   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 
   MachineInstr *MUL = nullptr;
+  MachineInstr *MAD = nullptr;
   const TargetRegisterClass *RC;
   unsigned Opc;
   switch (Pattern) {
@@ -6800,6 +6871,20 @@ void AArch64InstrInfo::genAlternativeCodeSequence(
                        &AArch64::FPR128_loRegClass, MRI);
     break;
   }
+
+  case MachineCombinerPattern::FNMADDS: {
+    Opc = AArch64::FNMADDSrrr;
+    RC = &AArch64::FPR32RegClass;
+    MAD = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs, Opc, RC);
+    break;
+  }
+  case MachineCombinerPattern::FNMADDD: {
+    Opc = AArch64::FNMADDDrrr;
+    RC = &AArch64::FPR64RegClass;
+    MAD = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs, Opc, RC);
+    break;
+  }
+
   } // end switch (Pattern)
   // Record MUL and ADD/SUB for deletion
   if (MUL)
@@ -6811,6 +6896,8 @@ void AArch64InstrInfo::genAlternativeCodeSequence(
   uint16_t Flags = Root.getFlags();
   if (MUL)
     Flags = Root.mergeFlagsWith(*MUL);
+  if (MAD)
+    Flags = Root.mergeFlagsWith(*MAD);
   for (auto *MI : InsInstrs)
     MI->setFlags(Flags);
 }
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 8a8c14fd057f2..f0bdce6938536 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -112,6 +112,8 @@ AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     return CSR_AArch64_AAPCS_SwiftTail_SaveList;
   if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
     return CSR_AArch64_RT_MostRegs_SaveList;
+  if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll)
+    return CSR_AArch64_RT_AllRegs_SaveList;
   if (MF->getFunction().getCallingConv() == CallingConv::Win64)
     // This is for OSes other than Windows; Windows is a separate case further
     // above.
@@ -160,6 +162,8 @@ AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const {
     return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;
   if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
     return CSR_Darwin_AArch64_RT_MostRegs_SaveList;
+  if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll)
+    return CSR_Darwin_AArch64_RT_AllRegs_SaveList;
   if (MF->getFunction().getCallingConv() == CallingConv::Win64)
     return CSR_Darwin_AArch64_AAPCS_Win64_SaveList;
   return CSR_Darwin_AArch64_AAPCS_SaveList;
@@ -237,6 +241,8 @@ AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
     return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;
   if (CC == CallingConv::PreserveMost)
     return CSR_Darwin_AArch64_RT_MostRegs_RegMask;
+  if (CC == CallingConv::PreserveAll)
+    return CSR_Darwin_AArch64_RT_AllRegs_RegMask;
   return CSR_Darwin_AArch64_AAPCS_RegMask;
 }
 
@@ -281,6 +287,10 @@ AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
   if (CC == CallingConv::PreserveMost)
     return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask
                : CSR_AArch64_RT_MostRegs_RegMask;
+  else if (CC == CallingConv::PreserveAll)
+    return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask
+               : CSR_AArch64_RT_AllRegs_RegMask;
+
   else
     return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;
 }
@@ -547,6 +557,7 @@ bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
   case CallingConv::C:
   case CallingConv::Fast:
   case CallingConv::PreserveMost:
+  case CallingConv::PreserveAll:
   case CallingConv::CXX_FAST_TLS:
   case CallingConv::Swift:
   case CallingConv::SwiftTail:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 6e3fd7358bb3d..80a3f13f304b0 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3265,9 +3265,9 @@ InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
       unsigned NumSources = 0;
       for (unsigned E = 0; E < LTNumElts; E++) {
         int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E]
-                                                      : UndefMaskElem;
+                                                      : PoisonMaskElem;
         if (MaskElt < 0) {
-          NMask.push_back(UndefMaskElem);
+          NMask.push_back(PoisonMaskElem);
           continue;
         }
 
diff --git a/llvm/lib/Target/AArch64/AsmParser/CMakeLists.txt b/llvm/lib/Target/AArch64/AsmParser/CMakeLists.txt
index deb5fca02f7fe..3c2109450879b 100644
--- a/llvm/lib/Target/AArch64/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/AsmParser/CMakeLists.txt
@@ -7,6 +7,7 @@ add_llvm_component_library(LLVMAArch64AsmParser
   AArch64Desc
   AArch64Info
   AArch64Utils
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 465936db571a2..0dfcc080878a1 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -99,6 +99,7 @@ add_llvm_target(AArch64CodeGen
   AsmPrinter
   CFGuard
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   MC
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 2855211258966..d09ecdd29e26c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -33,6 +33,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -41,7 +42,6 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
-#include "llvm/Support/MachineValueType.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -769,6 +769,7 @@ static bool mayTailCallThisCC(CallingConv::ID CC) {
   switch (CC) {
   case CallingConv::C:
   case CallingConv::PreserveMost:
+  case CallingConv::PreserveAll:
   case CallingConv::Swift:
   case CallingConv::SwiftTail:
   case CallingConv::Tail:
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index ebe870d870f52..0dbfb4c743b0c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineOperand.h"
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index 05a6f8182fe2a..c4d470f278770 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -427,7 +427,6 @@ class AArch64MCInstrAnalysis : public MCInstrAnalysis {
 
   std::vector<std::pair<uint64_t, uint64_t>>
   findPltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
-                 uint64_t GotPltSectionVA,
                  const Triple &TargetTriple) const override {
     // Do a lightweight parsing of PLT entries.
     std::vector<std::pair<uint64_t, uint64_t>> Result;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
index b7a770580e8b6..7f220657e45f8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
@@ -16,6 +16,7 @@ add_llvm_component_library(LLVMAArch64Desc
   AArch64Info
   AArch64Utils
   BinaryFormat
+  CodeGenTypes
   MC
   Support
   TargetParser
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h b/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h
index 2cdfff90b397d..20ef2403282dd 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h
+++ b/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h
@@ -30,7 +30,7 @@ class SMEAttrs {
     Normal = 0,
     SM_Enabled = 1 << 0,    // aarch64_pstate_sm_enabled
     SM_Compatible = 1 << 1, // aarch64_pstate_sm_compatible
-    SM_Body = 1 << 2,       // aarch64_pstate_sm_locally
+    SM_Body = 1 << 2,       // aarch64_pstate_sm_body
     ZA_Shared = 1 << 3,     // aarch64_pstate_sm_shared
     ZA_New = 1 << 4,        // aarch64_pstate_sm_new
     ZA_Preserved = 1 << 5,  // aarch64_pstate_sm_preserved
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index b1a94b78ec618..ae0403bb0533d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -370,7 +370,7 @@ enum TargetIndex {
 namespace AMDGPUAS {
 enum : unsigned {
   // The maximum value for flat, generic, local, private, constant and region.
-  MAX_AMDGPU_ADDRESS = 7,
+  MAX_AMDGPU_ADDRESS = 8,
 
   FLAT_ADDRESS = 0,   ///< Address space for flat memory.
   GLOBAL_ADDRESS = 1, ///< Address space for global memory (RAT0, VTX0).
@@ -383,6 +383,9 @@ enum : unsigned {
   CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory.
 
   BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers.
+                          ///< Not used in backend.
+
+  BUFFER_RESOURCE = 8, ///< Address space for 128-bit buffer resources.
 
   /// Internal address spaces. Can be freely renumbered.
   STREAMOUT_REGISTER = 128, ///< Address space for GS NGG Streamout registers.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
index 8155c895e3669..4f90f21de02ad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
@@ -47,25 +47,28 @@ void AMDGPUAAWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
 }
 
 static AliasResult getAliasResult(unsigned AS1, unsigned AS2) {
-  static_assert(AMDGPUAS::MAX_AMDGPU_ADDRESS <= 7, "Addr space out of range");
+  static_assert(AMDGPUAS::MAX_AMDGPU_ADDRESS <= 8, "Addr space out of range");
 
   if (AS1 > AMDGPUAS::MAX_AMDGPU_ADDRESS || AS2 > AMDGPUAS::MAX_AMDGPU_ADDRESS)
     return AliasResult::MayAlias;
 
 #define ASMay AliasResult::MayAlias
 #define ASNo AliasResult::NoAlias
-  // This array is indexed by address space value enum elements 0 ... to 7
-  static const AliasResult ASAliasRules[8][8] = {
-    /*                    Flat    Global Region Group  Constant Private Const32 Buf Fat Ptr */
-    /* Flat     */        {ASMay, ASMay, ASNo,  ASMay, ASMay,   ASMay,  ASMay,  ASMay},
-    /* Global   */        {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASMay,  ASMay},
-    /* Region   */        {ASNo,  ASNo,  ASMay, ASNo,  ASNo,    ASNo,   ASNo,   ASNo},
-    /* Group    */        {ASMay, ASNo,  ASNo,  ASMay, ASNo,    ASNo,   ASNo,   ASNo},
-    /* Constant */        {ASMay, ASMay, ASNo,  ASNo,  ASNo,    ASNo,   ASMay,  ASMay},
-    /* Private  */        {ASMay, ASNo,  ASNo,  ASNo,  ASNo,    ASMay,  ASNo,   ASNo},
-    /* Constant 32-bit */ {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASNo,   ASMay},
-    /* Buffer Fat Ptr  */ {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASMay,  ASMay}
+  // This array is indexed by address space value enum elements 0 ... to 8
+  // clang-format off
+  static const AliasResult ASAliasRules[9][9] = {
+    /*                    Flat    Global Region Group  Constant Private Const32 BufFatPtr BufRsrc */
+    /* Flat     */        {ASMay, ASMay, ASNo,  ASMay, ASMay,   ASMay,  ASMay,  ASMay,    ASMay},
+    /* Global   */        {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASMay,  ASMay,    ASMay},
+    /* Region   */        {ASNo,  ASNo,  ASMay, ASNo,  ASNo,    ASNo,   ASNo,   ASNo,     ASNo},
+    /* Group    */        {ASMay, ASNo,  ASNo,  ASMay, ASNo,    ASNo,   ASNo,   ASNo,     ASNo},
+    /* Constant */        {ASMay, ASMay, ASNo,  ASNo,  ASNo,    ASNo,   ASMay,  ASMay,    ASMay},
+    /* Private  */        {ASMay, ASNo,  ASNo,  ASNo,  ASNo,    ASMay,  ASNo,   ASNo,     ASNo},
+    /* Constant 32-bit */ {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASNo,   ASMay,    ASMay},
+    /* Buffer Fat Ptr  */ {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASMay,  ASMay,    ASMay},
+    /* Buffer Resource */ {ASMay, ASMay, ASNo,  ASNo,  ASMay,   ASNo,   ASMay,  ASMay,    ASMay},
   };
+  // clang-format on
 #undef ASMay
 #undef ASNo
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index da647e53bd264..09207319acee4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -958,10 +958,14 @@ getAssignFnsForCC(CallingConv::ID CC, const SITargetLowering &TLI) {
 }
 
 static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
-                              bool IsTailCall) {
+                              bool IsTailCall, CallingConv::ID CC) {
   assert(!(IsIndirect && IsTailCall) && "Indirect calls can't be tail calls, "
                                         "because the address can be divergent");
-  return IsTailCall ? AMDGPU::SI_TCRETURN : AMDGPU::G_SI_CALL;
+  if (!IsTailCall)
+    return AMDGPU::G_SI_CALL;
+
+  return CC == CallingConv::AMDGPU_Gfx ? AMDGPU::SI_TCRETURN_GFX :
+                                         AMDGPU::SI_TCRETURN;
 }
 
 // Add operands to call instruction to track the callee.
@@ -1186,7 +1190,7 @@ bool AMDGPUCallLowering::lowerTailCall(
   if (!IsSibCall)
     CallSeqStart = MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP);
 
-  unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
+  unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true, CalleeCC);
   auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
   if (!addCallTargetOperands(MIB, MIRBuilder, Info))
     return false;
@@ -1350,7 +1354,7 @@ bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   // Create a temporarily-floating call instruction so we can add the implicit
   // uses of arg registers.
-  unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
+  unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false, Info.CallConv);
 
   auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
   MIB.addDef(TRI->getReturnAddressReg(MF));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 3d5f3354196a3..bdf6c425f8f7a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1343,9 +1343,7 @@ bool AMDGPUCodeGenPrepare::visitLoadInst(LoadInst &I) {
     Builder.SetCurrentDebugLocation(I.getDebugLoc());
 
     Type *I32Ty = Builder.getInt32Ty();
-    Type *PT = PointerType::get(I32Ty, I.getPointerAddressSpace());
-    Value *BitCast= Builder.CreateBitCast(I.getPointerOperand(), PT);
-    LoadInst *WidenLoad = Builder.CreateLoad(I32Ty, BitCast);
+    LoadInst *WidenLoad = Builder.CreateLoad(I32Ty, I.getPointerOperand());
     WidenLoad->copyMetadata(I);
 
     // If we have range metadata, we need to convert the type, and not make
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCtorDtorLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCtorDtorLowering.cpp
index 39736aed9969f..ceb7dfcf12f67 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCtorDtorLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCtorDtorLowering.cpp
@@ -31,6 +31,8 @@ static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
   StringRef InitOrFiniKernelName = "amdgcn.device.init";
   if (!IsCtor)
     InitOrFiniKernelName = "amdgcn.device.fini";
+  if (Function *F = M.getFunction(InitOrFiniKernelName))
+    return F;
 
   Function *InitOrFiniKernel = Function::createWithDefaultAttr(
       FunctionType::get(Type::getVoidTy(M.getContext()), false),
@@ -63,12 +65,18 @@ static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
 
   for (Value *V : GA->operands()) {
     auto *CS = cast<ConstantStruct>(V);
-    IRB.CreateCall(ConstructorTy, CS->getOperand(1));
+    bool AlreadyRegistered =
+        llvm::any_of(CS->getOperand(1)->uses(), [=](Use &U) {
+          if (auto *CB = dyn_cast<CallBase>(U.getUser()))
+            if (CB->getCaller() == InitOrFiniKernel)
+              return true;
+          return false;
+        });
+    if (!AlreadyRegistered)
+      IRB.CreateCall(ConstructorTy, CS->getOperand(1));
   }
 
   appendToUsed(M, {InitOrFiniKernel});
-
-  GV->eraseFromParent();
   return true;
 }
 
@@ -83,9 +91,7 @@ class AMDGPUCtorDtorLoweringLegacy final : public ModulePass {
 public:
   static char ID;
   AMDGPUCtorDtorLoweringLegacy() : ModulePass(ID) {}
-  bool runOnModule(Module &M) override {
-    return lowerCtorsAndDtors(M);
-  }
+  bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
 };
 
 } // End anonymous namespace
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
index 2ffc8b2a3a7b8..09930dc9612cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp
@@ -10,8 +10,8 @@
 #include "GCNSubtarget.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 
 using namespace llvm;
 using namespace MIPatternMatch;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 9e3477cf9c272..b50687b643951 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -527,8 +527,8 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
     SelectADD_SUB_I64(N);
     return;
   }
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY:
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:
     if (N->getValueType(0) != MVT::i32)
       break;
 
@@ -808,7 +808,7 @@ SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
   return SDValue(Mov, 0);
 }
 
-// FIXME: Should only handle addcarry/subcarry
+// FIXME: Should only handle uaddo_carry/usubo_carry
 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
   SDLoc DL(N);
   SDValue LHS = N->getOperand(0);
@@ -885,15 +885,15 @@ void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
   SDValue CI = N->getOperand(2);
 
   if (N->isDivergent()) {
-    unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
-                                                   : AMDGPU::V_SUBB_U32_e64;
+    unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::V_ADDC_U32_e64
+                                                      : AMDGPU::V_SUBB_U32_e64;
     CurDAG->SelectNodeTo(
         N, Opc, N->getVTList(),
         {LHS, RHS, CI,
          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
   } else {
-    unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO
-                                                   : AMDGPU::S_SUB_CO_PSEUDO;
+    unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::S_ADD_CO_PSEUDO
+                                                      : AMDGPU::S_SUB_CO_PSEUDO;
     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
   }
 }
@@ -908,8 +908,8 @@ void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
        ++UI)
     if (UI.getUse().getResNo() == 1) {
-      if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) ||
-          (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) {
+      if ((IsAdd && (UI->getOpcode() != ISD::UADDO_CARRY)) ||
+          (!IsAdd && (UI->getOpcode() != ISD::USUBO_CARRY))) {
         IsVALU = true;
         break;
       }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index ba0812537fba4..e7a7b205b247f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -16,7 +16,6 @@
 #include "AMDGPU.h"
 #include "AMDGPUInstrInfo.h"
 #include "AMDGPUMachineFunction.h"
-#include "GCNSubtarget.h"
 #include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -1424,32 +1423,42 @@ SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
 
 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
                                                      SelectionDAG &DAG) const {
-
+  SDLoc SL(Op);
   SmallVector<SDValue, 8> Args;
   unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
   EVT VT = Op.getValueType();
   EVT SrcVT = Op.getOperand(0).getValueType();
 
-  // For these types, we have some TableGen patterns except if the index is 1
-  if (((SrcVT == MVT::v4f16 && VT == MVT::v2f16) ||
-       (SrcVT == MVT::v4i16 && VT == MVT::v2i16)) &&
-      Start != 1)
-    return Op;
+  if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
+    unsigned NumElt = VT.getVectorNumElements();
+    unsigned NumSrcElt = SrcVT.getVectorNumElements();
+    assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
 
-  if (((SrcVT == MVT::v8f16 && VT == MVT::v4f16) ||
-       (SrcVT == MVT::v8i16 && VT == MVT::v4i16)) &&
-      (Start == 0 || Start == 4))
-    return Op;
+    // We have some TableGen patterns for when the extracted vector is exactly
+    // the low or high half of the operand.
+    if ((NumSrcElt == 2 * NumElt) && (Start == 0 || Start == NumElt))
+      return Op;
 
-  if (((SrcVT == MVT::v16f16 && VT == MVT::v8f16) ||
-       (SrcVT == MVT::v16i16 && VT == MVT::v8i16)) &&
-      (Start == 0 || Start == 8))
-    return Op;
+    // Extract 32-bit registers at a time.
+    EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumSrcElt / 2);
+    EVT NewVT = NumElt == 2
+                    ? MVT::i32
+                    : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElt / 2);
+    SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0));
+
+    DAG.ExtractVectorElements(Tmp, Args, Start / 2, NumElt / 2);
+    if (NumElt == 2)
+      Tmp = Args[0];
+    else
+      Tmp = DAG.getBuildVector(NewVT, SL, Args);
+
+    return DAG.getNode(ISD::BITCAST, SL, VT, Tmp);
+  }
 
   DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
                             VT.getVectorNumElements());
 
-  return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
+  return DAG.getBuildVector(Op.getValueType(), SL, Args);
 }
 
 // TODO: Handle fabs too
@@ -1964,9 +1973,9 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
     SDValue Mulhi1_Lo, Mulhi1_Hi;
     std::tie(Mulhi1_Lo, Mulhi1_Hi) =
         DAG.SplitScalar(Mulhi1, DL, HalfVT, HalfVT);
-    SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
+    SDValue Add1_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Lo,
                                   Mulhi1_Lo, Zero1);
-    SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
+    SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi,
                                   Mulhi1_Hi, Add1_Lo.getValue(1));
     SDValue Add1 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
@@ -1977,9 +1986,9 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
     SDValue Mulhi2_Lo, Mulhi2_Hi;
     std::tie(Mulhi2_Lo, Mulhi2_Hi) =
         DAG.SplitScalar(Mulhi2, DL, HalfVT, HalfVT);
-    SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
+    SDValue Add2_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Lo,
                                   Mulhi2_Lo, Zero1);
-    SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Hi,
+    SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi,
                                   Mulhi2_Hi, Add2_Lo.getValue(1));
     SDValue Add2 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
@@ -1990,9 +1999,9 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
 
     SDValue Mul3_Lo, Mul3_Hi;
     std::tie(Mul3_Lo, Mul3_Hi) = DAG.SplitScalar(Mul3, DL, HalfVT, HalfVT);
-    SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
+    SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo,
                                   Mul3_Lo, Zero1);
-    SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
+    SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi,
                                   Mul3_Hi, Sub1_Lo.getValue(1));
     SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
     SDValue Sub1 = DAG.getBitcast(VT,
@@ -2010,11 +2019,11 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
     // potential endif to substitute PHIs.
 
     // if C3 != 0 ...
-    SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
+    SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo,
                                   RHS_Lo, Zero1);
-    SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
+    SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi,
                                   RHS_Hi, Sub1_Lo.getValue(1));
-    SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
+    SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
                                   Zero, Sub2_Lo.getValue(1));
     SDValue Sub2 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
@@ -2030,11 +2039,11 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
     // if (C6 != 0)
     SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
 
-    SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
+    SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo,
                                   RHS_Lo, Zero1);
-    SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
+    SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
                                   RHS_Hi, Sub2_Lo.getValue(1));
-    SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
+    SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi,
                                   Zero, Sub3_Lo.getValue(1));
     SDValue Sub3 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
@@ -4588,6 +4597,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(LOOP)
   NODE_NAME_CASE(CALL)
   NODE_NAME_CASE(TC_RETURN)
+  NODE_NAME_CASE(TC_RETURN_GFX)
   NODE_NAME_CASE(TRAP)
   NODE_NAME_CASE(RET_GLUE)
   NODE_NAME_CASE(RETURN_TO_EPILOG)
@@ -4905,22 +4915,6 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
   case ISD::INTRINSIC_WO_CHAIN: {
     unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
     switch (IID) {
-    case Intrinsic::amdgcn_mbcnt_lo:
-    case Intrinsic::amdgcn_mbcnt_hi: {
-      const GCNSubtarget &ST =
-          DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
-      // These return at most the (wavefront size - 1) + src1
-      // As long as src1 is an immediate we can calc known bits
-      KnownBits Src1Known = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
-      unsigned Src1ValBits = Src1Known.countMaxActiveBits();
-      unsigned MaxActiveBits = std::max(Src1ValBits, ST.getWavefrontSizeLog2());
-      // Cater for potential carry
-      MaxActiveBits += Src1ValBits ? 1 : 0;
-      unsigned Size = Op.getValueType().getSizeInBits();
-      if (MaxActiveBits < Size)
-        Known.Zero.setHighBits(Size - MaxActiveBits);
-      break;
-    }
     case Intrinsic::amdgcn_workitem_id_x:
     case Intrinsic::amdgcn_workitem_id_y:
     case Intrinsic::amdgcn_workitem_id_z: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index c20c7a72cbed6..84dfc376b7971 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -367,6 +367,7 @@ enum NodeType : unsigned {
   // Function call.
   CALL,
   TC_RETURN,
+  TC_RETURN_GFX,
   TRAP,
 
   // Masked control flow nodes.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 44ad131bd7eff..48a6fde657094 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -328,7 +328,8 @@ simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
       });
 }
 
-bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Value *Op0, const Value *Op1,
+bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
+                                           const Value *Op0, const Value *Op1,
                                            InstCombiner &IC) const {
   // The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
   // infinity, gives +0.0. If we can prove we don't have one of the special
@@ -340,9 +341,14 @@ bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Value *Op0, const Value *Op1,
     // One operand is not zero or infinity or NaN.
     return true;
   }
+
   auto *TLI = &IC.getTargetLibraryInfo();
-  if (isKnownNeverInfinity(Op0, TLI) && isKnownNeverNaN(Op0, TLI) &&
-      isKnownNeverInfinity(Op1, TLI) && isKnownNeverNaN(Op1, TLI)) {
+  if (isKnownNeverInfOrNaN(Op0, IC.getDataLayout(), TLI, 0,
+                           &IC.getAssumptionCache(), &I, &IC.getDominatorTree(),
+                           &IC.getOptimizationRemarkEmitter()) &&
+      isKnownNeverInfOrNaN(Op1, IC.getDataLayout(), TLI, 0,
+                           &IC.getAssumptionCache(), &I, &IC.getDominatorTree(),
+                           &IC.getOptimizationRemarkEmitter())) {
     // Neither operand is infinity or NaN.
     return true;
   }
@@ -1005,7 +1011,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 
     // If we can prove we don't have one of the special cases then we can use a
     // normal fmul instruction instead.
-    if (canSimplifyLegacyMulToMul(Op0, Op1, IC)) {
+    if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
       auto *FMul = IC.Builder.CreateFMulFMF(Op0, Op1, &II);
       FMul->takeName(&II);
       return IC.replaceInstUsesWith(II, FMul);
@@ -1032,7 +1038,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
 
     // If we can prove we don't have one of the special cases then we can use a
     // normal fma instead.
-    if (canSimplifyLegacyMulToMul(Op0, Op1, IC)) {
+    if (canSimplifyLegacyMulToMul(II, Op0, Op1, IC)) {
       II.setCalledOperand(Intrinsic::getDeclaration(
           II.getModule(), Intrinsic::fma, II.getType()));
       return &II;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
index fbde546cf07e5..d1e19e69cce35 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
@@ -85,9 +85,16 @@ def AMDGPUcall : SDNode<"AMDGPUISD::CALL",
   SDNPVariadic]
 >;
 
-def AMDGPUtc_return: SDNode<"AMDGPUISD::TC_RETURN",
-  SDTypeProfile<0, 3, [SDTCisPtrTy<0>]>,
-  [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]
+def AMDGPUTCReturnTP : SDTypeProfile<0, 3, [
+  SDTCisPtrTy<0>
+]>;
+
+def AMDGPUtc_return: SDNode<"AMDGPUISD::TC_RETURN", AMDGPUTCReturnTP,
+[SDNPHasChain, SDNPOptInGlue, SDNPVariadic]
+>;
+
+def AMDGPUtc_return_gfx: SDNode<"AMDGPUISD::TC_RETURN_GFX", AMDGPUTCReturnTP,
+[SDNPHasChain, SDNPOptInGlue, SDNPVariadic]
 >;
 
 def AMDGPUtrap : SDNode<"AMDGPUISD::TRAP",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index 2fc68a1423969..fb7148ba10ac1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -156,18 +156,14 @@ bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
   IRBuilder<> IRB(&LI);
   IRB.SetCurrentDebugLocation(LI.getDebugLoc());
 
-  unsigned AS = LI.getPointerAddressSpace();
-  unsigned LdBits = DL->getTypeStoreSize(LI.getType()) * 8;
+  unsigned LdBits = DL->getTypeStoreSizeInBits(LI.getType());
   auto IntNTy = Type::getIntNTy(LI.getContext(), LdBits);
 
-  PointerType *Int32PtrTy = Type::getInt32PtrTy(LI.getContext(), AS);
-  PointerType *Int8PtrTy = Type::getInt8PtrTy(LI.getContext(), AS);
-  auto *NewPtr = IRB.CreateBitCast(
-      IRB.CreateConstGEP1_64(
-          IRB.getInt8Ty(),
-          IRB.CreatePointerBitCastOrAddrSpaceCast(Base, Int8PtrTy),
-          Offset - Adjust),
-      Int32PtrTy);
+  auto *NewPtr = IRB.CreateConstGEP1_64(
+      IRB.getInt8Ty(),
+      IRB.CreateAddrSpaceCast(Base, LI.getPointerOperand()->getType()),
+      Offset - Adjust);
+
   LoadInst *NewLd = IRB.CreateAlignedLoad(IRB.getInt32Ty(), NewPtr, Align(4));
   NewLd->copyMetadata(LI);
   NewLd->setMetadata(LLVMContext::MD_range, nullptr);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 7cb8a95c983ab..296cc34ef1fc4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -131,6 +131,28 @@ static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
   };
 }
 
+// Increase the number of vector elements to reach the next legal RegClass.
+static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    const unsigned NumElts = Ty.getNumElements();
+    const unsigned EltSize = Ty.getElementType().getSizeInBits();
+    const unsigned MaxNumElts = MaxRegisterSize / EltSize;
+
+    assert(EltSize == 32 || EltSize == 64);
+    assert(Ty.getSizeInBits() < MaxRegisterSize);
+
+    unsigned NewNumElts;
+    // Find the nearest legal RegClass that is larger than the current type.
+    for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
+      if (SIRegisterInfo::getSGPRClassForBitWidth(NewNumElts * EltSize))
+        break;
+    }
+
+    return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltSize));
+  };
+}
+
 static LLT getBitcastRegisterType(const LLT Ty) {
   const unsigned Size = Ty.getSizeInBits();
 
@@ -215,6 +237,15 @@ static LegalityPredicate isRegisterType(unsigned TypeIdx) {
   };
 }
 
+// RegisterType that doesn't have a corresponding RegClass.
+static LegalityPredicate isIllegalRegisterType(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    LLT Ty = Query.Types[TypeIdx];
+    return isRegisterType(Ty) &&
+           !SIRegisterInfo::getSGPRClassForBitWidth(Ty.getSizeInBits());
+  };
+}
+
 static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
     const LLT QueryTy = Query.Types[TypeIdx];
@@ -1455,10 +1486,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
           const LLT VecTy = Query.Types[VecTypeIdx];
           const LLT IdxTy = Query.Types[IdxTypeIdx];
           const unsigned EltSize = EltTy.getSizeInBits();
+          const bool isLegalVecType =
+              !!SIRegisterInfo::getSGPRClassForBitWidth(VecTy.getSizeInBits());
           return (EltSize == 32 || EltSize == 64) &&
                   VecTy.getSizeInBits() % 32 == 0 &&
                   VecTy.getSizeInBits() <= MaxRegisterSize &&
-                  IdxTy.getSizeInBits() == 32;
+                  IdxTy.getSizeInBits() == 32 &&
+                  isLegalVecType;
         })
       .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx), scalarOrEltNarrowerThan(VecTypeIdx, 32)),
                  bitcastToVectorElement32(VecTypeIdx))
@@ -1484,6 +1518,9 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .clampScalar(IdxTypeIdx, S32, S32)
       .clampMaxNumElements(VecTypeIdx, S32, 32)
       // TODO: Clamp elements for 64-bit vectors?
+      .moreElementsIf(
+        isIllegalRegisterType(VecTypeIdx),
+        moreElementsToNextExistingRegClass(VecTypeIdx))
       // It should only be necessary with variable indexes.
       // As a last resort, lower to the stack
       .lower();
@@ -1538,7 +1575,10 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .legalForCartesianProduct(AllS64Vectors, {S64})
     .clampNumElements(0, V16S32, V32S32)
     .clampNumElements(0, V2S64, V16S64)
-    .fewerElementsIf(isWideVec16(0), changeTo(0, V2S16));
+    .fewerElementsIf(isWideVec16(0), changeTo(0, V2S16))
+    .moreElementsIf(
+      isIllegalRegisterType(0),
+      moreElementsToNextExistingRegClass(0));
 
   if (ST.hasScalarPackInsts()) {
     BuildVector
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index f3ff9b7535858..eb57203c6aee0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -86,7 +86,6 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
   KernArgSegment->addRetAttr(
       Attribute::getWithDereferenceableBytes(Ctx, TotalKernArgSize));
 
-  unsigned AS = KernArgSegment->getType()->getPointerAddressSpace();
   uint64_t ExplicitArgOffset = 0;
 
   for (Argument &Arg : F.args()) {
@@ -111,8 +110,8 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
           Builder.getInt8Ty(), KernArgSegment, EltOffset,
           Arg.getName() + ".byval.kernarg.offset");
 
-      Value *CastOffsetPtr = Builder.CreatePointerBitCastOrAddrSpaceCast(
-          ArgOffsetPtr, Arg.getType());
+      Value *CastOffsetPtr =
+          Builder.CreateAddrSpaceCast(ArgOffsetPtr, Arg.getType());
       Arg.replaceAllUsesWith(CastOffsetPtr);
       continue;
     }
@@ -170,8 +169,6 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
       AdjustedArgTy = V4Ty;
     }
 
-    ArgPtr = Builder.CreateBitCast(ArgPtr, AdjustedArgTy->getPointerTo(AS),
-                                   ArgPtr->getName() + ".cast");
     LoadInst *Load =
         Builder.CreateAlignedLoad(AdjustedArgTy, ArgPtr, AdjustedAlign);
     Load->setMetadata(LLVMContext::MD_invariant_load, MDNode::get(Ctx, {}));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index 9ecad3be663e4..508517f754d18 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -134,7 +134,8 @@ void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
     OutMI.addOperand(Dest);
     OutMI.addOperand(Src);
     return;
-  } else if (Opcode == AMDGPU::SI_TCRETURN) {
+  } else if (Opcode == AMDGPU::SI_TCRETURN ||
+             Opcode == AMDGPU::SI_TCRETURN_GFX) {
     // TODO: How to use branch immediate and avoid register+add?
     Opcode = AMDGPU::S_SETPC_B64;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index 3ff3546f4f92d..9205770c8ead0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -377,19 +377,12 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
     if (!OutArgIndexes.count(Arg.getArgNo()))
       continue;
 
-    PointerType *ArgType = cast<PointerType>(Arg.getType());
-
     Type *EltTy = OutArgIndexes[Arg.getArgNo()];
     const auto Align =
         DL->getValueOrABITypeAlignment(Arg.getParamAlign(), EltTy);
 
     Value *Val = B.CreateExtractValue(StubCall, RetIdx++);
-    Type *PtrTy = Val->getType()->getPointerTo(ArgType->getAddressSpace());
-
-    // We can peek through bitcasts, so the type may not match.
-    Value *PtrVal = B.CreateBitCast(&Arg, PtrTy);
-
-    B.CreateAlignedStore(Val, PtrVal, Align);
+    B.CreateAlignedStore(Val, &Arg, Align);
   }
 
   if (!RetTy->isVoidTy()) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 94c3cc30a11e7..9da225366745f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -508,11 +508,15 @@ static StringRef computeDataLayout(const Triple &TT) {
   }
 
   // 32-bit private, local, and region pointers. 64-bit global, constant and
-  // flat, non-integral buffer fat pointers.
+  // flat. 160-bit non-integral fat buffer pointers that include a 128-bit
+  // buffer descriptor and a 32-bit offset, which are indexed by 32-bit values
+  // (address space 7), and 128-bit non-integral buffer resourcees (address
+  // space 8) which cannot be non-trivilally accessed by LLVM memory operations
+  // like getelementptr.
   return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
-         "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
-         "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1"
-         "-ni:7";
+         "-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:"
+         "128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-"
+         "G1-ni:7:8";
 }
 
 LLVM_READNONE
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index d5f2a11905aac..291a5a40985a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -355,7 +355,8 @@ unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
   if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
       AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
       AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
-      AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER) {
+      AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
+      AddrSpace == AMDGPUAS::BUFFER_RESOURCE) {
     return 512;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 972ea8cf52ea0..cdd76861335fb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -209,8 +209,8 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
                                           Value *NewV) const;
 
-  bool canSimplifyLegacyMulToMul(const Value *Op0, const Value *Op1,
-                                 InstCombiner &IC) const;
+  bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0,
+                                 const Value *Op1, InstCombiner &IC) const;
   std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
                                                     IntrinsicInst &II) const;
   std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index ff781a02f5536..084aeedac11d3 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -21,6 +21,7 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
@@ -35,7 +36,6 @@
 #include "llvm/Support/AMDGPUMetadata.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/TargetParser/TargetParser.h"
 #include <optional>
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/CMakeLists.txt b/llvm/lib/Target/AMDGPU/AsmParser/CMakeLists.txt
index 1b96c7571d7b7..756a8e9f8f692 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/AsmParser/CMakeLists.txt
@@ -5,6 +5,7 @@ add_llvm_component_library(LLVMAMDGPUAsmParser
   AMDGPUDesc
   AMDGPUInfo
   AMDGPUUtils
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index e1c778d11697f..9ac805cfd84b9 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -172,6 +172,7 @@ add_llvm_target(AMDGPUCodeGen
   AsmPrinter
   BinaryFormat
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   IPO
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 9fcc8d5480def..6ad59e026003e 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -413,7 +413,6 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                                                 ArrayRef<uint8_t> Bytes_,
                                                 uint64_t Address,
                                                 raw_ostream &CS) const {
-  CommentStream = &CS;
   bool IsSDWA = false;
 
   unsigned MaxInstBytesNum = std::min((size_t)TargetMaxInstBytes, Bytes_.size());
@@ -428,13 +427,11 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
     // encodings
     if (isGFX11Plus() && Bytes.size() >= 12 ) {
       DecoderUInt128 DecW = eat12Bytes(Bytes);
-      Res = tryDecodeInst(DecoderTableDPP8GFX1196, MI, DecW,
-                                          Address);
+      Res = tryDecodeInst(DecoderTableDPP8GFX1196, MI, DecW, Address, CS);
       if (Res && convertDPP8Inst(MI) == MCDisassembler::Success)
         break;
       MI = MCInst(); // clear
-      Res = tryDecodeInst(DecoderTableDPPGFX1196, MI, DecW,
-                                          Address);
+      Res = tryDecodeInst(DecoderTableDPPGFX1196, MI, DecW, Address, CS);
       if (Res) {
         if (MCII->get(MI.getOpcode()).TSFlags & SIInstrFlags::VOP3P)
           convertVOP3PDPPInst(MI);
@@ -446,7 +443,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
         }
         break;
       }
-      Res = tryDecodeInst(DecoderTableGFX1196, MI, DecW, Address);
+      Res = tryDecodeInst(DecoderTableGFX1196, MI, DecW, Address, CS);
       if (Res)
         break;
     }
@@ -457,7 +454,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
       const uint64_t QW = eatBytes<uint64_t>(Bytes);
 
       if (STI.hasFeature(AMDGPU::FeatureGFX10_BEncoding)) {
-        Res = tryDecodeInst(DecoderTableGFX10_B64, MI, QW, Address);
+        Res = tryDecodeInst(DecoderTableGFX10_B64, MI, QW, Address, CS);
         if (Res) {
           if (AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dpp8)
               == -1)
@@ -468,37 +465,37 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
         }
       }
 
-      Res = tryDecodeInst(DecoderTableDPP864, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableDPP864, MI, QW, Address, CS);
       if (Res && convertDPP8Inst(MI) == MCDisassembler::Success)
         break;
       MI = MCInst(); // clear
 
-      Res = tryDecodeInst(DecoderTableDPP8GFX1164, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableDPP8GFX1164, MI, QW, Address, CS);
       if (Res && convertDPP8Inst(MI) == MCDisassembler::Success)
         break;
       MI = MCInst(); // clear
 
-      Res = tryDecodeInst(DecoderTableDPP64, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableDPP64, MI, QW, Address, CS);
       if (Res) break;
 
-      Res = tryDecodeInst(DecoderTableDPPGFX1164, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableDPPGFX1164, MI, QW, Address, CS);
       if (Res) {
         if (MCII->get(MI.getOpcode()).TSFlags & SIInstrFlags::VOPC)
           convertVOPCDPPInst(MI);
         break;
       }
 
-      Res = tryDecodeInst(DecoderTableSDWA64, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableSDWA64, MI, QW, Address, CS);
       if (Res) { IsSDWA = true;  break; }
 
-      Res = tryDecodeInst(DecoderTableSDWA964, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableSDWA964, MI, QW, Address, CS);
       if (Res) { IsSDWA = true;  break; }
 
-      Res = tryDecodeInst(DecoderTableSDWA1064, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableSDWA1064, MI, QW, Address, CS);
       if (Res) { IsSDWA = true;  break; }
 
       if (STI.hasFeature(AMDGPU::FeatureUnpackedD16VMem)) {
-        Res = tryDecodeInst(DecoderTableGFX80_UNPACKED64, MI, QW, Address);
+        Res = tryDecodeInst(DecoderTableGFX80_UNPACKED64, MI, QW, Address, CS);
         if (Res)
           break;
       }
@@ -507,7 +504,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
       // v_mad_mixhi_f16 for FMA variants. Try to decode using this special
       // table first so we print the correct name.
       if (STI.hasFeature(AMDGPU::FeatureFmaMixInsts)) {
-        Res = tryDecodeInst(DecoderTableGFX9_DL64, MI, QW, Address);
+        Res = tryDecodeInst(DecoderTableGFX9_DL64, MI, QW, Address, CS);
         if (Res)
           break;
       }
@@ -519,64 +516,64 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
     // Try decode 32-bit instruction
     if (Bytes.size() < 4) break;
     const uint32_t DW = eatBytes<uint32_t>(Bytes);
-    Res = tryDecodeInst(DecoderTableGFX832, MI, DW, Address);
+    Res = tryDecodeInst(DecoderTableGFX832, MI, DW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableAMDGPU32, MI, DW, Address);
+    Res = tryDecodeInst(DecoderTableAMDGPU32, MI, DW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableGFX932, MI, DW, Address);
+    Res = tryDecodeInst(DecoderTableGFX932, MI, DW, Address, CS);
     if (Res) break;
 
     if (STI.hasFeature(AMDGPU::FeatureGFX90AInsts)) {
-      Res = tryDecodeInst(DecoderTableGFX90A32, MI, DW, Address);
+      Res = tryDecodeInst(DecoderTableGFX90A32, MI, DW, Address, CS);
       if (Res)
         break;
     }
 
     if (STI.hasFeature(AMDGPU::FeatureGFX10_BEncoding)) {
-      Res = tryDecodeInst(DecoderTableGFX10_B32, MI, DW, Address);
+      Res = tryDecodeInst(DecoderTableGFX10_B32, MI, DW, Address, CS);
       if (Res) break;
     }
 
-    Res = tryDecodeInst(DecoderTableGFX1032, MI, DW, Address);
+    Res = tryDecodeInst(DecoderTableGFX1032, MI, DW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableGFX1132, MI, DW, Address);
+    Res = tryDecodeInst(DecoderTableGFX1132, MI, DW, Address, CS);
     if (Res) break;
 
     if (Bytes.size() < 4) break;
     const uint64_t QW = ((uint64_t)eatBytes<uint32_t>(Bytes) << 32) | DW;
 
     if (STI.hasFeature(AMDGPU::FeatureGFX940Insts)) {
-      Res = tryDecodeInst(DecoderTableGFX94064, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableGFX94064, MI, QW, Address, CS);
       if (Res)
         break;
     }
 
     if (STI.hasFeature(AMDGPU::FeatureGFX90AInsts)) {
-      Res = tryDecodeInst(DecoderTableGFX90A64, MI, QW, Address);
+      Res = tryDecodeInst(DecoderTableGFX90A64, MI, QW, Address, CS);
       if (Res)
         break;
     }
 
-    Res = tryDecodeInst(DecoderTableGFX864, MI, QW, Address);
+    Res = tryDecodeInst(DecoderTableGFX864, MI, QW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableAMDGPU64, MI, QW, Address);
+    Res = tryDecodeInst(DecoderTableAMDGPU64, MI, QW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableGFX964, MI, QW, Address);
+    Res = tryDecodeInst(DecoderTableGFX964, MI, QW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableGFX1064, MI, QW, Address);
+    Res = tryDecodeInst(DecoderTableGFX1064, MI, QW, Address, CS);
     if (Res) break;
 
-    Res = tryDecodeInst(DecoderTableGFX1164, MI, QW, Address);
+    Res = tryDecodeInst(DecoderTableGFX1164, MI, QW, Address, CS);
     if (Res)
       break;
 
-    Res = tryDecodeInst(DecoderTableWMMAGFX1164, MI, QW, Address);
+    Res = tryDecodeInst(DecoderTableWMMAGFX1164, MI, QW, Address, CS);
   } while (false);
 
   if (Res && AMDGPU::isMAC(MI.getOpcode())) {
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
index 5b9a1551a989e..490eb14c3396f 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
@@ -115,14 +115,25 @@ class AMDGPUDisassembler : public MCDisassembler {
 
   template <typename InsnType>
   DecodeStatus tryDecodeInst(const uint8_t *Table, MCInst &MI, InsnType Inst,
-                             uint64_t Address) const {
+                             uint64_t Address, raw_ostream &Comments) const {
     assert(MI.getOpcode() == 0);
     assert(MI.getNumOperands() == 0);
     MCInst TmpInst;
     HasLiteral = false;
     const auto SavedBytes = Bytes;
-    if (decodeInstruction(Table, TmpInst, Inst, Address, this, STI)) {
+
+    SmallString<64> LocalComments;
+    raw_svector_ostream LocalCommentStream(LocalComments);
+    CommentStream = &LocalCommentStream;
+
+    DecodeStatus Res =
+        decodeInstruction(Table, TmpInst, Inst, Address, this, STI);
+
+    CommentStream = nullptr;
+
+    if (Res != Fail) {
       MI = TmpInst;
+      Comments << LocalComments;
       return MCDisassembler::Success;
     }
     Bytes = SavedBytes;
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Disassembler/CMakeLists.txt
index 33591ec211fe0..012f9e754d16b 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Disassembler/CMakeLists.txt
@@ -7,6 +7,7 @@ add_llvm_component_library(LLVMAMDGPUDisassembler
   AMDGPUDesc
   AMDGPUInfo
   AMDGPUUtils
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 7f7d6e56874c2..32fd64f1140dd 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -872,7 +872,8 @@ void GCNSchedStage::setupNewBlock() {
   DAG.startBlock(CurrentMBB);
   // Get real RP for the region if it hasn't be calculated before. After the
   // initial schedule stage real RP will be collected after scheduling.
-  if (StageID == GCNSchedStageID::OccInitialSchedule)
+  if (StageID == GCNSchedStageID::OccInitialSchedule ||
+      StageID == GCNSchedStageID::ILPInitialSchedule)
     DAG.computeBlockPressure(RegionIdx, CurrentMBB);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/MCA/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCA/CMakeLists.txt
index fcad1d4a35199..f640392ca42a0 100644
--- a/llvm/lib/Target/AMDGPU/MCA/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/MCA/CMakeLists.txt
@@ -5,6 +5,7 @@ add_llvm_component_library(LLVMAMDGPUTargetMCA
   AMDGPUDesc
   AMDGPUInfo
   AMDGPUUtils
+  CodeGenTypes
   MC
   MCA
   MCParser
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
index 7eb5008cfbfe4..25d5ae4ea0b4d 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
@@ -16,6 +16,7 @@ add_llvm_component_library(LLVMAMDGPUDesc
   AMDGPUInfo
   AMDGPUUtils
   BinaryFormat
+  CodeGenTypes
   Core
   MC
   Support
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 57afcbc255f63..0420cf0474832 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -241,14 +241,10 @@ class NSAHelper {
   int NSA;
 }
 
-// This class used to use !foldl to memoize the AddrAsmNames list.
-// It turned out that that was much slower than using !filter.
 class MIMGNSAHelper<int num_addrs,
                     list<RegisterClass> addr_types=!listsplat(VGPR_32, num_addrs)>
   : NSAHelper<> {
-  list<string> AddrAsmNames =
-    !foreach(i, !filter(i, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
-                        !lt(i, num_addrs)), "vaddr" # i);
+  list<string> AddrAsmNames = !foreach(i, !range(num_addrs), "vaddr" # i);
   let AddrIns = !dag(ins, addr_types, AddrAsmNames);
   let AddrAsm = "[$" # !interleave(AddrAsmNames, ", $") # "]";
 
@@ -267,9 +263,7 @@ class PartialNSAHelper<int num_addrs, int max_addr, RegisterClass LastAddrRC>
       !listsplat(VGPR_32, num_addrs));
 
   int VAddrCount = !if(!gt(num_addrs, max_addr), max_addr, num_addrs);
-  list<string> AddrAsmNames =
-    !foreach(i, !filter(i, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
-             !lt(i, VAddrCount)), "vaddr" # i);
+  list<string> AddrAsmNames =  !foreach(i, !range(VAddrCount), "vaddr" # i);
 
   let AddrIns = !dag(ins, addr_types, AddrAsmNames);
   let AddrAsm = "[$" # !interleave(AddrAsmNames, ", $") # "]";
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 87fbefd36ad2f..cbbb2f6b9f40b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -244,13 +244,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
 
   setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Legal);
 
-  setOperationAction({ISD::ADDCARRY, ISD::SUBCARRY}, MVT::i32, Legal);
+  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i32, Legal);
 
   setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64,
                      Expand);
 
 #if 0
-  setOperationAction({ISD::ADDCARRY, ISD::SUBCARRY}, MVT::i64, Legal);
+  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal);
 #endif
 
   // We only support LOAD/STORE and vector manipulation ops for vectors
@@ -744,9 +744,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
                      Custom);
 
   setTargetDAGCombine({ISD::ADD,
-                       ISD::ADDCARRY,
+                       ISD::UADDO_CARRY,
                        ISD::SUB,
-                       ISD::SUBCARRY,
+                       ISD::USUBO_CARRY,
                        ISD::FADD,
                        ISD::FSUB,
                        ISD::FMINNUM,
@@ -996,7 +996,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
       return false;
 
     // TODO: Should images get their own address space?
-    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_FAT_POINTER;
+    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
 
     if (RsrcIntr->IsImage)
       Info.align.reset();
@@ -1086,7 +1086,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   case Intrinsic::amdgcn_buffer_atomic_fadd: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
-    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_FAT_POINTER;
+    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
     Info.align.reset();
     Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
@@ -1133,7 +1133,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT?
 
-    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_FAT_POINTER;
+    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
     Info.align.reset();
     Info.flags |= MachineMemOperand::MOLoad |
                   MachineMemOperand::MODereferenceable;
@@ -1325,7 +1325,7 @@ bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
 
   if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
       AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
-      AS == AMDGPUAS::BUFFER_FAT_POINTER) {
+      AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE) {
     // If the offset isn't a multiple of 4, it probably isn't going to be
     // correctly aligned.
     // FIXME: Can we get the real alignment here?
@@ -3396,7 +3396,9 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   // actual call instruction.
   if (IsTailCall) {
     MFI.setHasTailCall();
-    return DAG.getNode(AMDGPUISD::TC_RETURN, DL, NodeTys, Ops);
+    unsigned OPC = CallConv == CallingConv::AMDGPU_Gfx ?
+                   AMDGPUISD::TC_RETURN_GFX : AMDGPUISD::TC_RETURN;
+    return DAG.getNode(OPC, DL, NodeTys, Ops);
   }
 
   // Returns a chain and a flag for retval copy to use.
@@ -5760,6 +5762,35 @@ SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
   unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
   SDLoc SL(Op);
 
+  if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
+    // Insert 32-bit registers at a time.
+    assert(InsNumElts % 2 == 0 && "expect legal vector types");
+
+    unsigned VecNumElts = VecVT.getVectorNumElements();
+    EVT NewVecVT =
+        EVT::getVectorVT(*DAG.getContext(), MVT::i32, VecNumElts / 2);
+    EVT NewInsVT = InsNumElts == 2 ? MVT::i32
+                                   : EVT::getVectorVT(*DAG.getContext(),
+                                                      MVT::i32, InsNumElts / 2);
+
+    Vec = DAG.getNode(ISD::BITCAST, SL, NewVecVT, Vec);
+    Ins = DAG.getNode(ISD::BITCAST, SL, NewInsVT, Ins);
+
+    for (unsigned I = 0; I != InsNumElts / 2; ++I) {
+      SDValue Elt;
+      if (InsNumElts == 2) {
+        Elt = Ins;
+      } else {
+        Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins,
+                          DAG.getConstant(I, SL, MVT::i32));
+      }
+      Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NewVecVT, Vec, Elt,
+                        DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32));
+    }
+
+    return DAG.getNode(ISD::BITCAST, SL, VecVT, Vec);
+  }
+
   for (unsigned I = 0; I != InsNumElts; ++I) {
     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
                               DAG.getConstant(I, SL, MVT::i32));
@@ -11266,11 +11297,11 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
   if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
     return SDValue();
 
-  // add x, zext (setcc) => addcarry x, 0, setcc
-  // add x, sext (setcc) => subcarry x, 0, setcc
+  // add x, zext (setcc) => uaddo_carry x, 0, setcc
+  // add x, sext (setcc) => usubo_carry x, 0, setcc
   unsigned Opc = LHS.getOpcode();
   if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
-      Opc == ISD::ANY_EXTEND || Opc == ISD::ADDCARRY)
+      Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
     std::swap(RHS, LHS);
 
   Opc = RHS.getOpcode();
@@ -11286,15 +11317,15 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
       break;
     SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1);
     SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
-    Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::SUBCARRY : ISD::ADDCARRY;
+    Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
     return DAG.getNode(Opc, SL, VTList, Args);
   }
-  case ISD::ADDCARRY: {
-    // add x, (addcarry y, 0, cc) => addcarry x, y, cc
+  case ISD::UADDO_CARRY: {
+    // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
     if (!isNullConstant(RHS.getOperand(1)))
       break;
     SDValue Args[] = { LHS, RHS.getOperand(0), RHS.getOperand(2) };
-    return DAG.getNode(ISD::ADDCARRY, SDLoc(N), RHS->getVTList(), Args);
+    return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), RHS->getVTList(), Args);
   }
   }
   return SDValue();
@@ -11312,8 +11343,8 @@ SDValue SITargetLowering::performSubCombine(SDNode *N,
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
 
-  // sub x, zext (setcc) => subcarry x, 0, setcc
-  // sub x, sext (setcc) => addcarry x, 0, setcc
+  // sub x, zext (setcc) => usubo_carry x, 0, setcc
+  // sub x, sext (setcc) => uaddo_carry x, 0, setcc
   unsigned Opc = RHS.getOpcode();
   switch (Opc) {
   default: break;
@@ -11327,18 +11358,18 @@ SDValue SITargetLowering::performSubCombine(SDNode *N,
       break;
     SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1);
     SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond };
-    Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::ADDCARRY : ISD::SUBCARRY;
+    Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
     return DAG.getNode(Opc, SL, VTList, Args);
   }
   }
 
-  if (LHS.getOpcode() == ISD::SUBCARRY) {
-    // sub (subcarry x, 0, cc), y => subcarry x, y, cc
+  if (LHS.getOpcode() == ISD::USUBO_CARRY) {
+    // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
     auto C = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
     if (!C || !C->isZero())
       return SDValue();
     SDValue Args[] = { LHS.getOperand(0), RHS, LHS.getOperand(2) };
-    return DAG.getNode(ISD::SUBCARRY, SDLoc(N), LHS->getVTList(), Args);
+    return DAG.getNode(ISD::USUBO_CARRY, SDLoc(N), LHS->getVTList(), Args);
   }
   return SDValue();
 }
@@ -11355,12 +11386,12 @@ SDValue SITargetLowering::performAddCarrySubCarryCombine(SDNode *N,
   SelectionDAG &DAG = DCI.DAG;
   SDValue LHS = N->getOperand(0);
 
-  // addcarry (add x, y), 0, cc => addcarry x, y, cc
-  // subcarry (sub x, y), 0, cc => subcarry x, y, cc
+  // uaddo_carry (add x, y), 0, cc => uaddo_carry x, y, cc
+  // usubo_carry (sub x, y), 0, cc => usubo_carry x, y, cc
   unsigned LHSOpc = LHS.getOpcode();
   unsigned Opc = N->getOpcode();
-  if ((LHSOpc == ISD::ADD && Opc == ISD::ADDCARRY) ||
-      (LHSOpc == ISD::SUB && Opc == ISD::SUBCARRY)) {
+  if ((LHSOpc == ISD::ADD && Opc == ISD::UADDO_CARRY) ||
+      (LHSOpc == ISD::SUB && Opc == ISD::USUBO_CARRY)) {
     SDValue Args[] = { LHS.getOperand(0), LHS.getOperand(1), N->getOperand(2) };
     return DAG.getNode(Opc, SDLoc(N), N->getVTList(), Args);
   }
@@ -11712,8 +11743,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
     return performAddCombine(N, DCI);
   case ISD::SUB:
     return performSubCombine(N, DCI);
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY:
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:
     return performAddCarrySubCarryCombine(N, DCI);
   case ISD::FADD:
     return performFAddCombine(N, DCI);
@@ -12706,6 +12737,41 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
   TargetLoweringBase::finalizeLowering(MF);
 }
 
+void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
+                                                     KnownBits &Known,
+                                                     const APInt &DemandedElts,
+                                                     const SelectionDAG &DAG,
+                                                     unsigned Depth) const {
+  Known.resetAll();
+  unsigned Opc = Op.getOpcode();
+  switch (Opc) {
+  case ISD::INTRINSIC_WO_CHAIN: {
+    unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+    switch (IID) {
+    case Intrinsic::amdgcn_mbcnt_lo:
+    case Intrinsic::amdgcn_mbcnt_hi: {
+      const GCNSubtarget &ST =
+          DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
+      // These return at most the (wavefront size - 1) + src1
+      // As long as src1 is an immediate we can calc known bits
+      KnownBits Src1Known = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
+      unsigned Src1ValBits = Src1Known.countMaxActiveBits();
+      unsigned MaxActiveBits = std::max(Src1ValBits, ST.getWavefrontSizeLog2());
+      // Cater for potential carry
+      MaxActiveBits += Src1ValBits ? 1 : 0;
+      unsigned Size = Op.getValueType().getSizeInBits();
+      if (MaxActiveBits < Size)
+        Known.Zero.setHighBits(Size - MaxActiveBits);
+      return;
+    }
+    }
+    break;
+  }
+  }
+  return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
+      Op, Known, DemandedElts, DAG, Depth);
+}
+
 void SITargetLowering::computeKnownBitsForFrameIndex(
   const int FI, KnownBits &Known, const MachineFunction &MF) const {
   TargetLowering::computeKnownBitsForFrameIndex(FI, Known, MF);
@@ -13314,37 +13380,36 @@ void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
   assert(AI->getOperation() == AtomicRMWInst::FAdd &&
          "only fadd is supported for now");
 
-  // Given: atomicrmw fadd float* %addr, float %val ordering
+  // Given: atomicrmw fadd ptr %addr, float %val ordering
   //
   // With this expansion we produce the following code:
   //   [...]
-  //   %int8ptr = bitcast float* %addr to i8*
   //   br label %atomicrmw.check.shared
   //
   // atomicrmw.check.shared:
-  //   %is.shared = call i1 @llvm.amdgcn.is.shared(i8* %int8ptr)
+  //   %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
   //   br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
   //
   // atomicrmw.shared:
-  //   %cast.shared = addrspacecast float* %addr to float addrspace(3)*
-  //   %loaded.shared = atomicrmw fadd float addrspace(3)* %cast.shared,
+  //   %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
+  //   %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
   //                                   float %val ordering
   //   br label %atomicrmw.phi
   //
   // atomicrmw.check.private:
-  //   %is.private = call i1 @llvm.amdgcn.is.private(i8* %int8ptr)
+  //   %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
   //   br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
   //
   // atomicrmw.private:
-  //   %cast.private = addrspacecast float* %addr to float addrspace(5)*
-  //   %loaded.private = load float, float addrspace(5)* %cast.private
+  //   %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
+  //   %loaded.private = load float, ptr addrspace(5) %cast.private
   //   %val.new = fadd float %loaded.private, %val
-  //   store float %val.new, float addrspace(5)* %cast.private
+  //   store float %val.new, ptr addrspace(5) %cast.private
   //   br label %atomicrmw.phi
   //
   // atomicrmw.global:
-  //   %cast.global = addrspacecast float* %addr to float addrspace(1)*
-  //   %loaded.global = atomicrmw fadd float addrspace(1)* %cast.global,
+  //   %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
+  //   %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
   //                                   float %val ordering
   //   br label %atomicrmw.phi
   //
@@ -13393,12 +13458,11 @@ void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
 
   std::prev(BB->end())->eraseFromParent();
   Builder.SetInsertPoint(BB);
-  Value *Int8Ptr = Builder.CreateBitCast(Addr, Builder.getInt8PtrTy());
   Builder.CreateBr(CheckSharedBB);
 
   Builder.SetInsertPoint(CheckSharedBB);
   CallInst *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared, {},
-                                               {Int8Ptr}, nullptr, "is.shared");
+                                               {Addr}, nullptr, "is.shared");
   Builder.CreateCondBr(IsShared, SharedBB, CheckPrivateBB);
 
   Builder.SetInsertPoint(SharedBB);
@@ -13410,7 +13474,7 @@ void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
 
   Builder.SetInsertPoint(CheckPrivateBB);
   CallInst *IsPrivate = Builder.CreateIntrinsic(
-      Intrinsic::amdgcn_is_private, {}, {Int8Ptr}, nullptr, "is.private");
+      Intrinsic::amdgcn_is_private, {}, {Addr}, nullptr, "is.private");
   Builder.CreateCondBr(IsPrivate, PrivateBB, GlobalBB);
 
   Builder.SetInsertPoint(PrivateBB);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index b12d06c7f24cc..7e4b431ea6945 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -452,6 +452,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   void finalizeLowering(MachineFunction &MF) const override;
 
+  void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
+                                     const APInt &DemandedElts,
+                                     const SelectionDAG &DAG,
+                                     unsigned Depth = 0) const override;
   void computeKnownBitsForFrameIndex(int FrameIdx,
                                      KnownBits &Known,
                                      const MachineFunction &MF) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 25aecfddb7cb5..a6e11e3194e46 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -329,8 +329,8 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       const MachineOperand *Offset1Op =
           getNamedOperand(LdSt, AMDGPU::OpName::offset1);
 
-      unsigned Offset0 = Offset0Op->getImm();
-      unsigned Offset1 = Offset1Op->getImm();
+      unsigned Offset0 = Offset0Op->getImm() & 0xff;
+      unsigned Offset1 = Offset1Op->getImm() & 0xff;
       if (Offset0 + 1 != Offset1)
         return false;
 
@@ -5599,13 +5599,12 @@ void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
 }
 
 // Emit the actual waterfall loop, executing the wrapped instruction for each
-// unique value of \p Rsrc across all lanes. In the best case we execute 1
+// unique value of \p ScalarOps across all lanes. In the best case we execute 1
 // iteration, in the worst case we execute 64 (once per lane).
-static void
-emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
-                          MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
-                          MachineBasicBlock &BodyBB, const DebugLoc &DL,
-                          MachineOperand &Rsrc) {
+static void emitLoadScalarOpsFromVGPRLoop(
+    const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB,
+    MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
+    ArrayRef<MachineOperand *> ScalarOps) {
   MachineFunction &MF = *OrigBB.getParent();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
@@ -5623,73 +5622,106 @@ emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
   SmallVector<Register, 8> ReadlanePieces;
   Register CondReg;
 
-  Register VRsrc = Rsrc.getReg();
-  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
+  for (MachineOperand *ScalarOp : ScalarOps) {
+    unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
+    unsigned NumSubRegs = RegSize / 32;
+    Register VScalarOp = ScalarOp->getReg();
 
-  unsigned RegSize = TRI->getRegSizeInBits(Rsrc.getReg(), MRI);
-  unsigned NumSubRegs =  RegSize / 32;
-  assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && "Unhandled register size");
+    if (NumSubRegs == 1) {
+      Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
 
-  for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
+      BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
+          .addReg(VScalarOp);
 
-    Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
-    Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
+      Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
 
-    // Read the next variant <- also loop target.
-    BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
-            .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx));
+      BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
+          .addReg(CurReg)
+          .addReg(VScalarOp);
 
-    // Read the next variant <- also loop target.
-    BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
-            .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx + 1));
+      // Combine the comparison results with AND.
+      if (!CondReg) // First.
+        CondReg = NewCondReg;
+      else { // If not the first, we create an AND.
+        Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+        BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg)
+            .addReg(CondReg)
+            .addReg(NewCondReg);
+        CondReg = AndReg;
+      }
+
+      // Update ScalarOp operand to use the SGPR ScalarOp.
+      ScalarOp->setReg(CurReg);
+      ScalarOp->setIsKill();
+    } else {
+      unsigned VScalarOpUndef = getUndefRegState(ScalarOp->isUndef());
+      assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
+             "Unhandled register size");
+
+      for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
+        Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
+        Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
 
-    ReadlanePieces.push_back(CurRegLo);
-    ReadlanePieces.push_back(CurRegHi);
+        // Read the next variant <- also loop target.
+        BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
+            .addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx));
 
-    // Comparison is to be done as 64-bit.
-    Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
-    BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
+        // Read the next variant <- also loop target.
+        BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
+            .addReg(VScalarOp, VScalarOpUndef,
+                    TRI->getSubRegFromChannel(Idx + 1));
+
+        ReadlanePieces.push_back(CurRegLo);
+        ReadlanePieces.push_back(CurRegHi);
+
+        // Comparison is to be done as 64-bit.
+        Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
+        BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
             .addReg(CurRegLo)
             .addImm(AMDGPU::sub0)
             .addReg(CurRegHi)
             .addImm(AMDGPU::sub1);
 
-    Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
-    auto Cmp =
-        BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
-            .addReg(CurReg);
-    if (NumSubRegs <= 2)
-      Cmp.addReg(VRsrc);
-    else
-      Cmp.addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx, 2));
-
-    // Combine the comparison results with AND.
-    if (!CondReg) // First.
-      CondReg = NewCondReg;
-    else { // If not the first, we create an AND.
-      Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
-      BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg)
+        Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
+        auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
+                           NewCondReg)
+                       .addReg(CurReg);
+        if (NumSubRegs <= 2)
+          Cmp.addReg(VScalarOp);
+        else
+          Cmp.addReg(VScalarOp, VScalarOpUndef,
+                     TRI->getSubRegFromChannel(Idx, 2));
+
+        // Combine the comparison results with AND.
+        if (!CondReg) // First.
+          CondReg = NewCondReg;
+        else { // If not the first, we create an AND.
+          Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
+          BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg)
               .addReg(CondReg)
               .addReg(NewCondReg);
-      CondReg = AndReg;
-    }
-  } // End for loop.
-
-  auto SRsrcRC = TRI->getEquivalentSGPRClass(MRI.getRegClass(VRsrc));
-  Register SRsrc = MRI.createVirtualRegister(SRsrcRC);
+          CondReg = AndReg;
+        }
+      } // End for loop.
+
+      auto SScalarOpRC =
+          TRI->getEquivalentSGPRClass(MRI.getRegClass(VScalarOp));
+      Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC);
+
+      // Build scalar ScalarOp.
+      auto Merge =
+          BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
+      unsigned Channel = 0;
+      for (Register Piece : ReadlanePieces) {
+        Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++));
+      }
 
-  // Build scalar Rsrc.
-  auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc);
-  unsigned Channel = 0;
-  for (Register Piece : ReadlanePieces) {
-    Merge.addReg(Piece)
-         .addImm(TRI->getSubRegFromChannel(Channel++));
+      // Update ScalarOp operand to use the SGPR ScalarOp.
+      ScalarOp->setReg(SScalarOp);
+      ScalarOp->setIsKill();
+    }
   }
 
-  // Update Rsrc operand to use the SGPR Rsrc.
-  Rsrc.setReg(SRsrc);
-  Rsrc.setIsKill();
-
   Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
   MRI.setSimpleHint(SaveExec, CondReg);
 
@@ -5708,14 +5740,15 @@ emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
   BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
 }
 
-// Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
+// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
 // with SGPRs by iterating over all unique values across all lanes.
 // Returns the loop basic block that now contains \p MI.
 static MachineBasicBlock *
-loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
-                  MachineOperand &Rsrc, MachineDominatorTree *MDT,
-                  MachineBasicBlock::iterator Begin = nullptr,
-                  MachineBasicBlock::iterator End = nullptr) {
+loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
+                               ArrayRef<MachineOperand *> ScalarOps,
+                               MachineDominatorTree *MDT,
+                               MachineBasicBlock::iterator Begin = nullptr,
+                               MachineBasicBlock::iterator End = nullptr) {
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
@@ -5788,7 +5821,7 @@ loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
     }
   }
 
-  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, Rsrc);
+  emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps);
 
   // Restore the EXEC mask
   MachineBasicBlock::iterator First = RemainderBB->begin();
@@ -5985,11 +6018,11 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
                      (isMUBUF(MI) || isMTBUF(MI)))) {
     MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
     if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg())))
-      CreatedBB = loadSRsrcFromVGPR(*this, MI, *SRsrc, MDT);
+      CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SRsrc}, MDT);
 
     MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
     if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg())))
-      CreatedBB = loadSRsrcFromVGPR(*this, MI, *SSamp, MDT);
+      CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SSamp}, MDT);
 
     return CreatedBB;
   }
@@ -6017,25 +6050,39 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
       while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() &&
              MI.definesRegister(End->getOperand(1).getReg()))
         ++End;
-      CreatedBB = loadSRsrcFromVGPR(*this, MI, *Dest, MDT, Start, End);
+      CreatedBB =
+          loadMBUFScalarOperandsFromVGPR(*this, MI, {Dest}, MDT, Start, End);
+    }
+  }
+
+  // Legalize MUBUF instructions.
+  bool isSoffsetLegal = true;
+  int SoffsetIdx =
+      AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset);
+  if (SoffsetIdx != -1) {
+    MachineOperand *Soffset = &MI.getOperand(SoffsetIdx);
+    if (Soffset->isReg() &&
+        !RI.isSGPRClass(MRI.getRegClass(Soffset->getReg()))) {
+      isSoffsetLegal = false;
     }
   }
 
-  // Legalize MUBUF* instructions.
+  bool isRsrcLegal = true;
   int RsrcIdx =
       AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
   if (RsrcIdx != -1) {
-    // We have an MUBUF instruction
     MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
-    unsigned RsrcRC = get(MI.getOpcode()).operands()[RsrcIdx].RegClass;
-    if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
-                             RI.getRegClass(RsrcRC))) {
-      // The operands are legal.
-      // FIXME: We may need to legalize operands besides srsrc.
-      return CreatedBB;
+    if (Rsrc->isReg() && !RI.isSGPRClass(MRI.getRegClass(Rsrc->getReg()))) {
+      isRsrcLegal = false;
     }
+  }
 
-    // Legalize a VGPR Rsrc.
+  // The operands are legal.
+  if (isRsrcLegal && isSoffsetLegal)
+    return CreatedBB;
+
+  if (!isRsrcLegal) {
+    // Legalize a VGPR Rsrc
     //
     // If the instruction is _ADDR64, we can avoid a waterfall by extracting
     // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
@@ -6048,6 +6095,7 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
     // Otherwise we are on non-ADDR64 hardware, and/or we have
     // idxen/offen/bothen and we fall back to a waterfall loop.
 
+    MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
     MachineBasicBlock &MBB = *MI.getParent();
 
     MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
@@ -6157,12 +6205,24 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
           .addReg(RsrcPtr, 0, AMDGPU::sub1)
           .addImm(AMDGPU::sub1);
     } else {
-      // This is another variant; legalize Rsrc with waterfall loop from VGPRs
-      // to SGPRs.
-      CreatedBB = loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
+      // Legalize a VGPR Rsrc and soffset together.
+      if (!isSoffsetLegal) {
+        MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
+        CreatedBB =
+            loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc, Soffset}, MDT);
+        return CreatedBB;
+      }
+      CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc}, MDT);
       return CreatedBB;
     }
   }
+
+  // Legalize a VGPR soffset.
+  if (!isSoffsetLegal) {
+    MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
+    CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Soffset}, MDT);
+    return CreatedBB;
+  }
   return CreatedBB;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 38210b75e79a1..fb267f0260422 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -598,10 +598,9 @@ def SI_CALL : SPseudoInstSI <
   let isConvergent = 1;
 }
 
-// Tail call handling pseudo
-def SI_TCRETURN : SPseudoInstSI <(outs),
-  (ins CCR_SGPR_64:$src0, unknown:$callee, i32imm:$fpdiff),
-  [(AMDGPUtc_return i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> {
+class SI_TCRETURN_Pseudo<RegisterClass rc, SDNode sd> : SPseudoInstSI <(outs),
+  (ins rc:$src0, unknown:$callee, i32imm:$fpdiff),
+  [(sd i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> {
   let Size = 4;
   let FixedSize = 1;
   let isCall = 1;
@@ -614,12 +613,22 @@ def SI_TCRETURN : SPseudoInstSI <(outs),
   let isConvergent = 1;
 }
 
+// Tail call handling pseudo
+def SI_TCRETURN :     SI_TCRETURN_Pseudo<CCR_SGPR_64, AMDGPUtc_return>;
+def SI_TCRETURN_GFX : SI_TCRETURN_Pseudo<Gfx_CCR_SGPR_64, AMDGPUtc_return_gfx>;
+
 // Handle selecting indirect tail calls
 def : GCNPat<
   (AMDGPUtc_return i64:$src0, (i64 0), (i32 timm:$fpdiff)),
   (SI_TCRETURN CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff)
 >;
 
+// Handle selecting indirect tail calls for AMDGPU_gfx
+def : GCNPat<
+  (AMDGPUtc_return_gfx i64:$src0, (i64 0), (i32 timm:$fpdiff)),
+  (SI_TCRETURN_GFX Gfx_CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff)
+>;
+
 def ADJCALLSTACKUP : SPseudoInstSI<
   (outs), (ins i32imm:$amt0, i32imm:$amt1),
   [(callseq_start timm:$amt0, timm:$amt1)],
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 507916007becd..a0f782c6b9f99 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -331,7 +331,6 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
 
   switch (Opc) {
   case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
   case AMDGPU::S_LOAD_DWORD_IMM:
   case AMDGPU::GLOBAL_LOAD_DWORD:
@@ -342,7 +341,6 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::FLAT_STORE_DWORD:
     return 1;
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
   case AMDGPU::S_LOAD_DWORDX2_IMM:
   case AMDGPU::GLOBAL_LOAD_DWORDX2:
@@ -360,7 +358,6 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::FLAT_STORE_DWORDX3:
     return 3;
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
   case AMDGPU::S_LOAD_DWORDX4_IMM:
   case AMDGPU::GLOBAL_LOAD_DWORDX4:
@@ -371,7 +368,6 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::FLAT_STORE_DWORDX4:
     return 4;
   case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
   case AMDGPU::S_LOAD_DWORDX8_IMM:
     return 8;
@@ -450,12 +446,6 @@ static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
     return S_BUFFER_LOAD_IMM;
-  // For the purposes of this optimization SGPR variants of buffer loads
-  // are considered to be zero-offsetted SGPR_IMM loads.
-  case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
@@ -537,12 +527,6 @@ static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
     return AMDGPU::S_BUFFER_LOAD_DWORD_IMM;
-  // For the purposes of this optimization SGPR variants of buffer loads
-  // are considered to be zero-offsetted SGPR_IMM loads.
-  case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
@@ -645,10 +629,6 @@ static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
   switch (Opc) {
   default:
     return Result;
-  case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
   case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
@@ -744,7 +724,7 @@ void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
     Offset = 0;
   } else {
     int OffsetIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::offset);
-    Offset = OffsetIdx == -1 ? 0 : I->getOperand(OffsetIdx).getImm();
+    Offset = I->getOperand(OffsetIdx).getImm();
   }
 
   if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE)
@@ -1387,10 +1367,7 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeSMemLoadImmPair(
           .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase));
   if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM)
     New.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset));
-  // For convenience, when SGPR_IMM buffer loads are merged into a
-  // zero-offset load, we generate its SGPR variant.
-  if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::offset))
-    New.addImm(MergedOffset);
+  New.addImm(MergedOffset);
   New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
 
   std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired);
@@ -1701,14 +1678,11 @@ unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
     default:
       return 0;
     case 2:
-      return CI.Offset == 0 ? AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR
-                            : AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM;
+      return AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM;
     case 4:
-      return CI.Offset == 0 ? AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR
-                            : AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM;
+      return AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM;
     case 8:
-      return CI.Offset == 0 ? AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR
-                            : AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
+      return AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
     }
   case S_LOAD_IMM:
     switch (Width) {
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 0e2bf9dad64f8..9607092c70afe 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -427,6 +427,8 @@ void SILowerControlFlow::emitLoop(MachineInstr &MI) {
       BuildMI(MBB, &MI, DL, TII->get(Andn2TermOpc), Exec)
           .addReg(Exec)
           .add(MI.getOperand(0));
+  if (LV)
+    LV->replaceKillInstruction(MI.getOperand(0).getReg(), MI, *AndN2);
 
   auto BranchPt = skipToUncondBrOrEnd(MBB, MI.getIterator());
   MachineInstr *Branch =
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h b/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h
index c834a83047fc0..6693bd5150600 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h
+++ b/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h
@@ -34,9 +34,11 @@ struct SIModeRegisterDefaults {
   /// and f16/v2f16 instructions.
   DenormalMode FP64FP16Denormals;
 
-  SIModeRegisterDefaults()
-      : IEEE(true), DX10Clamp(true), FP32Denormals(DenormalMode::getIEEE()),
-        FP64FP16Denormals(DenormalMode::getIEEE()) {}
+  SIModeRegisterDefaults() :
+    IEEE(true),
+    DX10Clamp(true),
+    FP32Denormals(DenormalMode::getIEEE()),
+    FP64FP16Denormals(DenormalMode::getIEEE()) {}
 
   SIModeRegisterDefaults(const Function &F);
 
@@ -84,35 +86,10 @@ struct SIModeRegisterDefaults {
     return FP_DENORM_FLUSH_NONE;
   }
 
-  /// Returns true if a flag is compatible if it's enabled in the callee, but
-  /// disabled in the caller.
-  static bool oneWayCompatible(bool CallerMode, bool CalleeMode) {
-    return CallerMode == CalleeMode || (!CallerMode && CalleeMode);
-  }
-
   // FIXME: Inlining should be OK for dx10-clamp, since the caller's mode should
   // be able to override.
   bool isInlineCompatible(SIModeRegisterDefaults CalleeMode) const {
-    if (DX10Clamp != CalleeMode.DX10Clamp)
-      return false;
-    if (IEEE != CalleeMode.IEEE)
-      return false;
-
-    // Allow inlining denormals enabled into denormals flushed functions.
-    return oneWayCompatible(FP64FP16Denormals.Input !=
-                                DenormalMode::PreserveSign,
-                            CalleeMode.FP64FP16Denormals.Input !=
-                                DenormalMode::PreserveSign) &&
-           oneWayCompatible(FP64FP16Denormals.Output !=
-                                DenormalMode::PreserveSign,
-                            CalleeMode.FP64FP16Denormals.Output !=
-                                DenormalMode::PreserveSign) &&
-           oneWayCompatible(FP32Denormals.Input != DenormalMode::PreserveSign,
-                            CalleeMode.FP32Denormals.Input !=
-                                DenormalMode::PreserveSign) &&
-           oneWayCompatible(FP32Denormals.Output != DenormalMode::PreserveSign,
-                            CalleeMode.FP32Denormals.Output !=
-                                DenormalMode::PreserveSign);
+    return DX10Clamp == CalleeMode.DX10Clamp && IEEE == CalleeMode.IEEE;
   }
 };
 
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
index ae2c10116de85..126f56100c64a 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
@@ -522,8 +522,15 @@ void SIOptimizeVGPRLiveRange::optimizeLiveRange(
     auto *UseBlock = UseMI->getParent();
     // Replace uses in Endif block
     if (UseBlock == Endif) {
-      assert(UseMI->isPHI() && "Uses should be PHI in Endif block");
-      O.setReg(NewReg);
+      if (UseMI->isPHI()) {
+        O.setReg(NewReg);
+      } else {
+        // DetectDeadLanes may mark register uses as undef without removing
+        // them, in which case a non-phi instruction using the original register
+        // may exist in the Endif block even though the register is not live
+        // into it.
+        assert(!O.readsReg());
+      }
       continue;
     }
 
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 9d94344b752ac..5999fc94e934c 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -2501,31 +2501,31 @@ StringRef SIRegisterInfo::getRegAsmName(MCRegister Reg) const {
 
 static const TargetRegisterClass *
 getAnyVGPRClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::VReg_64RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::VReg_96RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::VReg_128RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::VReg_160RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::VReg_192RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::VReg_224RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::VReg_256RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::VReg_288RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::VReg_320RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::VReg_352RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::VReg_384RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::VReg_512RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::VReg_1024RegClass;
 
   return nullptr;
@@ -2533,31 +2533,31 @@ getAnyVGPRClassForBitWidth(unsigned BitWidth) {
 
 static const TargetRegisterClass *
 getAlignedVGPRClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::VReg_64_Align2RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::VReg_96_Align2RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::VReg_128_Align2RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::VReg_160_Align2RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::VReg_192_Align2RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::VReg_224_Align2RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::VReg_256_Align2RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::VReg_288_Align2RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::VReg_320_Align2RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::VReg_352_Align2RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::VReg_384_Align2RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::VReg_512_Align2RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::VReg_1024_Align2RegClass;
 
   return nullptr;
@@ -2567,9 +2567,9 @@ const TargetRegisterClass *
 SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) const {
   if (BitWidth == 1)
     return &AMDGPU::VReg_1RegClass;
-  if (BitWidth <= 16)
+  if (BitWidth == 16)
     return &AMDGPU::VGPR_LO16RegClass;
-  if (BitWidth <= 32)
+  if (BitWidth == 32)
     return &AMDGPU::VGPR_32RegClass;
   return ST.needsAlignedVGPRs() ? getAlignedVGPRClassForBitWidth(BitWidth)
                                 : getAnyVGPRClassForBitWidth(BitWidth);
@@ -2577,31 +2577,31 @@ SIRegisterInfo::getVGPRClassForBitWidth(unsigned BitWidth) const {
 
 static const TargetRegisterClass *
 getAnyAGPRClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::AReg_64RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::AReg_96RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::AReg_128RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::AReg_160RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::AReg_192RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::AReg_224RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::AReg_256RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::AReg_288RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::AReg_320RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::AReg_352RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::AReg_384RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::AReg_512RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::AReg_1024RegClass;
 
   return nullptr;
@@ -2609,31 +2609,31 @@ getAnyAGPRClassForBitWidth(unsigned BitWidth) {
 
 static const TargetRegisterClass *
 getAlignedAGPRClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::AReg_64_Align2RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::AReg_96_Align2RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::AReg_128_Align2RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::AReg_160_Align2RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::AReg_192_Align2RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::AReg_224_Align2RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::AReg_256_Align2RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::AReg_288_Align2RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::AReg_320_Align2RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::AReg_352_Align2RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::AReg_384_Align2RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::AReg_512_Align2RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::AReg_1024_Align2RegClass;
 
   return nullptr;
@@ -2641,9 +2641,9 @@ getAlignedAGPRClassForBitWidth(unsigned BitWidth) {
 
 const TargetRegisterClass *
 SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) const {
-  if (BitWidth <= 16)
+  if (BitWidth == 16)
     return &AMDGPU::AGPR_LO16RegClass;
-  if (BitWidth <= 32)
+  if (BitWidth == 32)
     return &AMDGPU::AGPR_32RegClass;
   return ST.needsAlignedVGPRs() ? getAlignedAGPRClassForBitWidth(BitWidth)
                                 : getAnyAGPRClassForBitWidth(BitWidth);
@@ -2651,31 +2651,31 @@ SIRegisterInfo::getAGPRClassForBitWidth(unsigned BitWidth) const {
 
 static const TargetRegisterClass *
 getAnyVectorSuperClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::AV_64RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::AV_96RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::AV_128RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::AV_160RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::AV_192RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::AV_224RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::AV_256RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::AV_288RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::AV_320RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::AV_352RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::AV_384RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::AV_512RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::AV_1024RegClass;
 
   return nullptr;
@@ -2683,31 +2683,31 @@ getAnyVectorSuperClassForBitWidth(unsigned BitWidth) {
 
 static const TargetRegisterClass *
 getAlignedVectorSuperClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::AV_64_Align2RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::AV_96_Align2RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::AV_128_Align2RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::AV_160_Align2RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::AV_192_Align2RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::AV_224_Align2RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::AV_256_Align2RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::AV_288_Align2RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::AV_320_Align2RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::AV_352_Align2RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::AV_384_Align2RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::AV_512_Align2RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::AV_1024_Align2RegClass;
 
   return nullptr;
@@ -2715,9 +2715,9 @@ getAlignedVectorSuperClassForBitWidth(unsigned BitWidth) {
 
 const TargetRegisterClass *
 SIRegisterInfo::getVectorSuperClassForBitWidth(unsigned BitWidth) const {
-  if (BitWidth <= 16)
+  if (BitWidth == 16)
     return &AMDGPU::VGPR_LO16RegClass;
-  if (BitWidth <= 32)
+  if (BitWidth == 32)
     return &AMDGPU::AV_32RegClass;
   return ST.needsAlignedVGPRs()
              ? getAlignedVectorSuperClassForBitWidth(BitWidth)
@@ -2726,35 +2726,35 @@ SIRegisterInfo::getVectorSuperClassForBitWidth(unsigned BitWidth) const {
 
 const TargetRegisterClass *
 SIRegisterInfo::getSGPRClassForBitWidth(unsigned BitWidth) {
-  if (BitWidth <= 16)
+  if (BitWidth == 16)
     return &AMDGPU::SGPR_LO16RegClass;
-  if (BitWidth <= 32)
+  if (BitWidth == 32)
     return &AMDGPU::SReg_32RegClass;
-  if (BitWidth <= 64)
+  if (BitWidth == 64)
     return &AMDGPU::SReg_64RegClass;
-  if (BitWidth <= 96)
+  if (BitWidth == 96)
     return &AMDGPU::SGPR_96RegClass;
-  if (BitWidth <= 128)
+  if (BitWidth == 128)
     return &AMDGPU::SGPR_128RegClass;
-  if (BitWidth <= 160)
+  if (BitWidth == 160)
     return &AMDGPU::SGPR_160RegClass;
-  if (BitWidth <= 192)
+  if (BitWidth == 192)
     return &AMDGPU::SGPR_192RegClass;
-  if (BitWidth <= 224)
+  if (BitWidth == 224)
     return &AMDGPU::SGPR_224RegClass;
-  if (BitWidth <= 256)
+  if (BitWidth == 256)
     return &AMDGPU::SGPR_256RegClass;
-  if (BitWidth <= 288)
+  if (BitWidth == 288)
     return &AMDGPU::SGPR_288RegClass;
-  if (BitWidth <= 320)
+  if (BitWidth == 320)
     return &AMDGPU::SGPR_320RegClass;
-  if (BitWidth <= 352)
+  if (BitWidth == 352)
     return &AMDGPU::SGPR_352RegClass;
-  if (BitWidth <= 384)
+  if (BitWidth == 384)
     return &AMDGPU::SGPR_384RegClass;
-  if (BitWidth <= 512)
+  if (BitWidth == 512)
     return &AMDGPU::SGPR_512RegClass;
-  if (BitWidth <= 1024)
+  if (BitWidth == 1024)
     return &AMDGPU::SGPR_1024RegClass;
 
   return nullptr;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index 060f63158b53c..5c2e88727296c 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -10,16 +10,6 @@
 //  Subregister declarations
 //===----------------------------------------------------------------------===//
 
-class Indexes<int N> {
-  list<int> all = [0,   1,  2,  3,  4,  5,  6 , 7,
-                   8,   9, 10, 11, 12, 13, 14, 15,
-                   16, 17, 18, 19, 20, 21, 22, 23,
-                   24, 25, 26, 27, 28, 29, 30, 31];
-
-  // Returns list of indexes [0..N)
-  list<int> slice = !filter(i, all, !lt(i, N));
-}
-
 let Namespace = "AMDGPU" in {
 
 def lo16 : SubRegIndex<16, 0>;
@@ -35,13 +25,11 @@ foreach Index = 1...31 in {
 }
 
 foreach Size = {2...6,8,16} in {
-  foreach Index = Indexes<!sub(33, Size)>.slice in {
-    def !interleave(!foreach(cur, Indexes<Size>.slice, "sub"#!add(cur, Index)),
-                    "_") :
+  foreach Index = !range(!sub(33, Size)) in {
+    def !interleave(!foreach(cur, !range(Size), "sub"#!add(cur, Index)), "_") :
       SubRegIndex<!mul(Size, 32), !shl(Index, 5)> {
       let CoveringSubRegIndices =
-        !foreach(cur, Indexes<Size>.slice,
-                 !cast<SubRegIndex>(sub#!add(cur, Index)));
+        !foreach(cur, !range(Size), !cast<SubRegIndex>(sub#!add(cur, Index)));
     }
   }
 }
@@ -817,12 +805,19 @@ def SGPR_64 : SIRegisterClass<"AMDGPU", [v2i32, i64, v2f32, f64, v4i16, v4f16],
   let HasSGPR = 1;
 }
 
-// CCR (call clobbered registers) SGPR 64-bit registers, currently used for tail call
-// return address only.
-def CCR_SGPR_64 : SIRegisterClass<"AMDGPU", SGPR_64.RegTypes, 32, (add (trunc SGPR_64, 16))> {
+// CCR (call clobbered registers) SGPR 64-bit registers
+def CCR_SGPR_64 : SIRegisterClass<"AMDGPU", SGPR_64.RegTypes, 32, (add (trunc SGPR_64, 15))> {
   let CopyCost = SGPR_64.CopyCost;
   let AllocationPriority = SGPR_64.AllocationPriority;
-  let HasSGPR = SGPR_64.HasSGPR;
+  let HasSGPR = 1;
+}
+
+// Call clobbered 64-bit SGPRs for AMDGPU_Gfx CC
+def Gfx_CCR_SGPR_64 : SIRegisterClass<"AMDGPU", SGPR_64.RegTypes, 32,
+                                (add (trunc (shl SGPR_64, 18), 14))> { // s[36:37]-s[s62:63]
+  let CopyCost = SGPR_64.CopyCost;
+  let AllocationPriority = SGPR_64.AllocationPriority;
+  let HasSGPR = 1;
 }
 
 def TTMP_64 : SIRegisterClass<"AMDGPU", [v2i32, i64, f64, v4i16, v4f16], 32,
diff --git a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
index 2d4a3f22e8c4f..19d3b690b1315 100644
--- a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -8,6 +8,7 @@ add_llvm_component_library(LLVMAMDGPUUtils
   LINK_COMPONENTS
   Analysis
   BinaryFormat
+  CodeGenTypes
   Core
   MC
   Support
diff --git a/llvm/lib/Target/ARC/CMakeLists.txt b/llvm/lib/Target/ARC/CMakeLists.txt
index f63d088d6c0ac..9f3c1787c5635 100644
--- a/llvm/lib/Target/ARC/CMakeLists.txt
+++ b/llvm/lib/Target/ARC/CMakeLists.txt
@@ -28,9 +28,12 @@ add_llvm_target(ARCCodeGen
   ARCTargetMachine.cpp
 
   LINK_COMPONENTS
+  ARCDesc
+  ARCInfo
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   SelectionDAG
@@ -38,12 +41,10 @@ add_llvm_target(ARCCodeGen
   Target
   TargetParser
   TransformUtils
-  ARCDesc
-  ARCInfo
 
   ADD_TO_COMPONENT
   ARC
-)
+  )
 
 add_subdirectory(Disassembler)
 add_subdirectory(MCTargetDesc)
diff --git a/llvm/lib/Target/ARC/Disassembler/CMakeLists.txt b/llvm/lib/Target/ARC/Disassembler/CMakeLists.txt
index 3e68a42f10e07..1e634795c422d 100644
--- a/llvm/lib/Target/ARC/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/ARC/Disassembler/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMARCDisassembler
 
   LINK_COMPONENTS
   ARCInfo
+  CodeGenTypes
   MCDisassembler
   Support
 
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.cpp b/llvm/lib/Target/ARM/ARMCallLowering.cpp
index 76b99403a2f8f..b2ffa9df3a81b 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMCallLowering.cpp
@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -30,6 +31,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -40,8 +42,6 @@
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
-#include "llvm/Support/MachineValueType.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 60a6e9ade9234..98716accd9e15 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -40,6 +40,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -72,7 +73,6 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 643313daf9288..4496d4928ebe9 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -3168,12 +3168,12 @@ void ARMFrameLowering::adjustForSegmentedStacks(
       .addReg(ScratchReg1)
       .add(predOps(ARMCC::AL));
 
-  // This jump is taken if StackLimit < SP - stack required.
+  // This jump is taken if StackLimit <= SP - stack required.
   Opcode = Thumb ? ARM::tBcc : ARM::Bcc;
-  BuildMI(GetMBB, DL, TII.get(Opcode)).addMBB(PostStackMBB)
-       .addImm(ARMCC::LO)
-       .addReg(ARM::CPSR);
-
+  BuildMI(GetMBB, DL, TII.get(Opcode))
+      .addMBB(PostStackMBB)
+      .addImm(ARMCC::LS)
+      .addReg(ARM::CPSR);
 
   // Calling __morestack(StackSize, Size of stack arguments).
   // __morestack knows that the stack size requested is in SR0(r4)
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 49a4025cae38c..0bbd4dde69ae7 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -52,6 +52,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
@@ -98,7 +99,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
@@ -1132,8 +1132,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::SSUBO, MVT::i32, Custom);
   setOperationAction(ISD::USUBO, MVT::i32, Custom);
 
-  setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
-  setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
+  setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom);
+  setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom);
   if (Subtarget->hasDSP()) {
     setOperationAction(ISD::SADDSAT, MVT::i8, Custom);
     setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
@@ -2081,6 +2081,8 @@ ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
     return CC;
   case CallingConv::PreserveMost:
     return CallingConv::PreserveMost;
+  case CallingConv::PreserveAll:
+    return CallingConv::PreserveAll;
   case CallingConv::ARM_AAPCS_VFP:
   case CallingConv::Swift:
   case CallingConv::SwiftTail:
@@ -2139,6 +2141,8 @@ CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
     return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
   case CallingConv::PreserveMost:
     return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
+  case CallingConv::PreserveAll:
+    return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
   case CallingConv::CFGuard_Check:
     return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
   }
@@ -6884,7 +6888,7 @@ static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
 
   assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
 
-  // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we
+  // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
   // have to invert the carry first.
   Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
                       DAG.getConstant(1, DL, MVT::i32), Carry);
@@ -9787,7 +9791,7 @@ static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
   return N0;
 }
 
-static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
+static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
   SDNode *N = Op.getNode();
   EVT VT = N->getValueType(0);
   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
@@ -9797,7 +9801,7 @@ static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
   SDLoc DL(Op);
 
   SDValue Result;
-  if (Op.getOpcode() == ISD::ADDCARRY) {
+  if (Op.getOpcode() == ISD::UADDO_CARRY) {
     // This converts the boolean value carry into the carry flag.
     Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
 
@@ -9808,7 +9812,7 @@ static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
     // Now convert the carry flag into a boolean value.
     Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
   } else {
-    // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we
+    // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
     // have to invert the carry first.
     Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
                         DAG.getConstant(1, DL, MVT::i32), Carry);
@@ -9822,7 +9826,7 @@ static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
     // Now convert the carry flag into a boolean value.
     Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
     // But the carry returned by ARMISD::SUBE is not a borrow as expected
-    // by ISD::SUBCARRY, so compute 1 - C.
+    // by ISD::USUBO_CARRY, so compute 1 - C.
     Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
                         DAG.getConstant(1, DL, MVT::i32), Carry);
   }
@@ -10540,8 +10544,9 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     if (Subtarget->isTargetWindows() && !Op.getValueType().isVector())
       return LowerDIV_Windows(Op, DAG, /* Signed */ false);
     return LowerUDIV(Op, DAG, Subtarget);
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY:      return LowerADDSUBCARRY(Op, DAG);
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:
+    return LowerUADDSUBO_CARRY(Op, DAG);
   case ISD::SADDO:
   case ISD::SSUBO:
     return LowerSignedALUO(Op, DAG);
@@ -13787,6 +13792,11 @@ bool ARMTargetLowering::shouldFoldConstantShiftPairToMask(
   return false;
 }
 
+bool ARMTargetLowering::shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
+                                                             EVT VT) const {
+  return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT);
+}
+
 bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
   if (!Subtarget->hasNEON()) {
     if (Subtarget->isThumb1Only())
@@ -18364,23 +18374,23 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
                           DAG.getConstant(5, dl, MVT::i32));
       } else {
         // CMOV 0, 1, ==, (CMPZ x, y) ->
-        //     (ADDCARRY (SUB x, y), t:0, t:1)
-        // where t = (SUBCARRY 0, (SUB x, y), 0)
+        //     (UADDO_CARRY (SUB x, y), t:0, t:1)
+        // where t = (USUBO_CARRY 0, (SUB x, y), 0)
         //
-        // The SUBCARRY computes 0 - (x - y) and this will give a borrow when
+        // The USUBO_CARRY computes 0 - (x - y) and this will give a borrow when
         // x != y. In other words, a carry C == 1 when x == y, C == 0
         // otherwise.
-        // The final ADDCARRY computes
+        // The final UADDO_CARRY computes
         //     x - y + (0 - (x - y)) + C == C
         SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
         SDVTList VTs = DAG.getVTList(VT, MVT::i32);
         SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
-        // ISD::SUBCARRY returns a borrow but we want the carry here
+        // ISD::USUBO_CARRY returns a borrow but we want the carry here
         // actually.
         SDValue Carry =
             DAG.getNode(ISD::SUB, dl, MVT::i32,
                         DAG.getConstant(1, dl, MVT::i32), Neg.getValue(1));
-        Res = DAG.getNode(ISD::ADDCARRY, dl, VTs, Sub, Neg, Carry);
+        Res = DAG.getNode(ISD::UADDO_CARRY, dl, VTs, Sub, Neg, Carry);
       }
     } else if (CC == ARMCC::NE && !isNullConstant(RHS) &&
                (!Subtarget->isThumb1Only() || isPowerOf2Constant(TrueVal))) {
@@ -18415,14 +18425,14 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   // (z == 2 ^ K).
   // CMOV (SUBS x, y), z, !=, (SUBS x, y):1 ->
   // t1 = (USUBO (SUB x, y), 1)
-  // t2 = (SUBCARRY (SUB x, y), t1:0, t1:1)
+  // t2 = (USUBO_CARRY (SUB x, y), t1:0, t1:1)
   // Result = if K != 0 then (SHL t2:0, K) else t2:0
   //
   // This also handles the special case of comparing against zero; it's
   // essentially, the same pattern, except there's no SUBS:
   // CMOV x, z, !=, (CMPZ x, 0) ->
   // t1 = (USUBO x, 1)
-  // t2 = (SUBCARRY x, t1:0, t1:1)
+  // t2 = (USUBO_CARRY x, t1:0, t1:1)
   // Result = if K != 0 then (SHL t2:0, K) else t2:0
   const APInt *TrueConst;
   if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
@@ -18435,7 +18445,8 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
     if (ShiftAmount)
       TrueVal = DAG.getConstant(1, dl, VT);
     SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
-    Res = DAG.getNode(ISD::SUBCARRY, dl, VTs, FalseVal, Subc, Subc.getValue(1));
+    Res = DAG.getNode(ISD::USUBO_CARRY, dl, VTs, FalseVal, Subc,
+                      Subc.getValue(1));
 
     if (ShiftAmount)
       Res = DAG.getNode(ISD::SHL, dl, VT, Res,
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 2db55a5e9f42e..207bb54e791a9 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -20,6 +20,7 @@
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -29,7 +30,6 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/CodeGen.h"
-#include "llvm/Support/MachineValueType.h"
 #include <optional>
 #include <utility>
 
@@ -738,6 +738,9 @@ class VectorType;
     bool shouldFoldConstantShiftPairToMask(const SDNode *N,
                                            CombineLevel Level) const override;
 
+    bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
+                                              EVT VT) const override;
+
     bool preferIncOfAddToSubOfNot(EVT VT) const override;
 
     bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index d4a70cddd32fc..fa25c571a9bd5 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -3192,19 +3192,18 @@ defm MVE_VQRSHL_by_vec : mve_shift_by_vec_multi<"vqrshl", 0b1, 0b1>;
 defm MVE_VRSHL_by_vec  : mve_shift_by_vec_multi<"vrshl",  0b0, 0b1>;
 
 let Predicates = [HasMVEInt] in {
-  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
-            (v4i32 (MVE_VSHL_by_vecu32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
-  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
-            (v8i16 (MVE_VSHL_by_vecu16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
-  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
-            (v16i8 (MVE_VSHL_by_vecu8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
-
-  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
-            (v4i32 (MVE_VSHL_by_vecs32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
-  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
-            (v8i16 (MVE_VSHL_by_vecs16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
-  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
-            (v16i8 (MVE_VSHL_by_vecs8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
+  defm : MVE_TwoOpPattern<MVE_v16i8, ARMvshlu, int_arm_mve_vshl_vector_predicated,
+                          (? (i32 0), (i32 0), (i32 1)), MVE_VSHL_by_vecu8, null_frag>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, ARMvshlu, int_arm_mve_vshl_vector_predicated,
+                          (? (i32 0), (i32 0), (i32 1)), MVE_VSHL_by_vecu16, null_frag>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, ARMvshlu, int_arm_mve_vshl_vector_predicated,
+                          (? (i32 0), (i32 0), (i32 1)), MVE_VSHL_by_vecu32, null_frag>;
+  defm : MVE_TwoOpPattern<MVE_v16i8, ARMvshls, int_arm_mve_vshl_vector_predicated,
+                          (? (i32 0), (i32 0), (i32 0)), MVE_VSHL_by_vecs8, null_frag>;
+  defm : MVE_TwoOpPattern<MVE_v8i16, ARMvshls, int_arm_mve_vshl_vector_predicated,
+                          (? (i32 0), (i32 0), (i32 0)), MVE_VSHL_by_vecs16, null_frag>;
+  defm : MVE_TwoOpPattern<MVE_v4i32, ARMvshls, int_arm_mve_vshl_vector_predicated,
+                          (? (i32 0), (i32 0), (i32 0)), MVE_VSHL_by_vecs32, null_frag>;
 }
 
 class MVE_shift_with_imm<string iname, string suffix, dag oops, dag iops,
diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
index 52b6b6f3bcf72..3ffde86ce1bb2 100644
--- a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
@@ -15,7 +15,7 @@
 #include "ARMSubtarget.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 0a38f5633ae3b..eb2f6cd87afc2 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -2172,10 +2172,10 @@ namespace {
                           unsigned &NewOpc, Register &EvenReg, Register &OddReg,
                           Register &BaseReg, int &Offset, Register &PredReg,
                           ARMCC::CondCodes &Pred, bool &isT2);
-    bool RescheduleOps(MachineBasicBlock *MBB,
-                       SmallVectorImpl<MachineInstr *> &Ops,
-                       unsigned Base, bool isLd,
-                       DenseMap<MachineInstr*, unsigned> &MI2LocMap);
+    bool RescheduleOps(
+        MachineBasicBlock *MBB, SmallVectorImpl<MachineInstr *> &Ops,
+        unsigned Base, bool isLd, DenseMap<MachineInstr *, unsigned> &MI2LocMap,
+        SmallDenseMap<Register, SmallVector<MachineInstr *>, 8> &RegisterMap);
     bool RescheduleLoadStoreInstrs(MachineBasicBlock *MBB);
     bool DistributeIncrements();
     bool DistributeIncrements(Register Base);
@@ -2324,10 +2324,10 @@ bool ARMPreAllocLoadStoreOpt::CanFormLdStDWord(
   return true;
 }
 
-bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
-                                 SmallVectorImpl<MachineInstr *> &Ops,
-                                 unsigned Base, bool isLd,
-                                 DenseMap<MachineInstr*, unsigned> &MI2LocMap) {
+bool ARMPreAllocLoadStoreOpt::RescheduleOps(
+    MachineBasicBlock *MBB, SmallVectorImpl<MachineInstr *> &Ops, unsigned Base,
+    bool isLd, DenseMap<MachineInstr *, unsigned> &MI2LocMap,
+    SmallDenseMap<Register, SmallVector<MachineInstr *>, 8> &RegisterMap) {
   bool RetVal = false;
 
   // Sort by offset (in reverse order).
@@ -2476,6 +2476,12 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
         } else {
           for (unsigned i = 0; i != NumMove; ++i) {
             MachineInstr *Op = Ops.pop_back_val();
+            if (isLd) {
+              // Populate RegisterMap with all Registers defined by loads.
+              Register Reg = Op->getOperand(0).getReg();
+              RegisterMap[Reg];
+            }
+
             MBB->splice(InsertPos, MBB, Op);
           }
         }
@@ -2489,6 +2495,44 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
   return RetVal;
 }
 
+static void forEachDbgRegOperand(MachineInstr *MI,
+                                 std::function<void(MachineOperand &)> Fn) {
+  if (MI->isNonListDebugValue()) {
+    auto &Op = MI->getOperand(0);
+    if (Op.isReg())
+      Fn(Op);
+  } else {
+    for (unsigned I = 2; I < MI->getNumOperands(); I++) {
+      auto &Op = MI->getOperand(I);
+      if (Op.isReg())
+        Fn(Op);
+    }
+  }
+}
+
+// Update the RegisterMap with the instruction that was moved because a
+// DBG_VALUE_LIST may need to be moved again.
+static void updateRegisterMapForDbgValueListAfterMove(
+    SmallDenseMap<Register, SmallVector<MachineInstr *>, 8> &RegisterMap,
+    MachineInstr *DbgValueListInstr, MachineInstr *InstrToReplace) {
+
+  forEachDbgRegOperand(DbgValueListInstr, [&](MachineOperand &Op) {
+    auto RegIt = RegisterMap.find(Op.getReg());
+    if (RegIt == RegisterMap.end())
+      return;
+    auto &InstrVec = RegIt->getSecond();
+    for (unsigned I = 0; I < InstrVec.size(); I++)
+      if (InstrVec[I] == InstrToReplace)
+        InstrVec[I] = DbgValueListInstr;
+  });
+}
+
+static DebugVariable createDebugVariableFromMachineInstr(MachineInstr *MI) {
+  auto DbgVar = DebugVariable(MI->getDebugVariable(), MI->getDebugExpression(),
+                              MI->getDebugLoc()->getInlinedAt());
+  return DbgVar;
+}
+
 bool
 ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
   bool RetVal = false;
@@ -2501,6 +2545,10 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
   Base2InstMap Base2StsMap;
   BaseVec LdBases;
   BaseVec StBases;
+  // This map is used to track the relationship between the virtual
+  // register that is the result of a load that is moved and the DBG_VALUE
+  // MachineInstr pointer that uses that virtual register.
+  SmallDenseMap<Register, SmallVector<MachineInstr *>, 8> RegisterMap;
 
   unsigned Loc = 0;
   MachineBasicBlock::iterator MBBI = MBB->begin();
@@ -2563,7 +2611,7 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
       unsigned Base = LdBases[i];
       SmallVectorImpl<MachineInstr *> &Lds = Base2LdsMap[Base];
       if (Lds.size() > 1)
-        RetVal |= RescheduleOps(MBB, Lds, Base, true, MI2LocMap);
+        RetVal |= RescheduleOps(MBB, Lds, Base, true, MI2LocMap, RegisterMap);
     }
 
     // Re-schedule stores.
@@ -2571,7 +2619,7 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
       unsigned Base = StBases[i];
       SmallVectorImpl<MachineInstr *> &Sts = Base2StsMap[Base];
       if (Sts.size() > 1)
-        RetVal |= RescheduleOps(MBB, Sts, Base, false, MI2LocMap);
+        RetVal |= RescheduleOps(MBB, Sts, Base, false, MI2LocMap, RegisterMap);
     }
 
     if (MBBI != E) {
@@ -2582,6 +2630,255 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
     }
   }
 
+  // Reschedule DBG_VALUEs to match any loads that were moved. When a load is
+  // sunk beyond a DBG_VALUE that is referring to it, the DBG_VALUE becomes a
+  // use-before-def, resulting in a loss of debug info.
+
+  // Example:
+  // Before the Pre Register Allocation Load Store Pass
+  // inst_a
+  // %2 = ld ...
+  // inst_b
+  // DBG_VALUE %2, "x", ...
+  // %3 = ld ...
+
+  // After the Pass:
+  // inst_a
+  // inst_b
+  // DBG_VALUE %2, "x", ...
+  // %2 = ld ...
+  // %3 = ld ...
+
+  // The code below addresses this by moving the DBG_VALUE to the position
+  // immediately after the load.
+
+  // Example:
+  // After the code below:
+  // inst_a
+  // inst_b
+  // %2 = ld ...
+  // DBG_VALUE %2, "x", ...
+  // %3 = ld ...
+
+  // The algorithm works in two phases: First RescheduleOps() populates the
+  // RegisterMap with registers that were moved as keys, there is no value
+  // inserted. In the next phase, every MachineInstr in a basic block is
+  // iterated over. If it is a valid DBG_VALUE or DBG_VALUE_LIST and it uses one
+  // or more registers in the RegisterMap, the RegisterMap and InstrMap are
+  // populated with the MachineInstr. If the DBG_VALUE or DBG_VALUE_LIST
+  // describes debug information for a variable that already exists in the
+  // DbgValueSinkCandidates, the MachineInstr in the DbgValueSinkCandidates must
+  // be set to undef. If the current MachineInstr is a load that was moved,
+  // undef the corresponding DBG_VALUE or DBG_VALUE_LIST and clone it to below
+  // the load.
+
+  // To illustrate the above algorithm visually let's take this example.
+
+  // Before the Pre Register Allocation Load Store Pass:
+  // %2 = ld ...
+  // DBG_VALUE %2, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // %3 = ld ...
+  // DBG_VALUE %3, A, ..., # Z
+  // %4 = ld ...
+
+  // After Pre Register Allocation Load Store Pass:
+  // DBG_VALUE %2, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE %3, A, ..., # Z
+  // %2 = ld ...
+  // %3 = ld ...
+  // %4 = ld ...
+
+  // The algorithm below does the following:
+
+  // In the beginning, the RegisterMap will have been populated with the virtual
+  // registers %2, and %3, the DbgValueSinkCandidates and the InstrMap will be
+  // empty. DbgValueSinkCandidates = {}, RegisterMap = {2 -> {}, 3 -> {}},
+  // InstrMap {}
+  // -> DBG_VALUE %2, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE %3, A, ..., # Z
+  // %2 = ld ...
+  // %3 = ld ...
+  // %4 = ld ...
+
+  // After the first DBG_VALUE (denoted with an X) is processed, the
+  // DbgValueSinkCandidates and InstrMap will be populated and the RegisterMap
+  // entry for %2 will be populated as well. DbgValueSinkCandidates = {A -> X},
+  // RegisterMap = {2 -> {X}, 3 -> {}}, InstrMap {X -> 2}
+  // DBG_VALUE %2, A, .... # X
+  // -> DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE %3, A, ..., # Z
+  // %2 = ld ...
+  // %3 = ld ...
+  // %4 = ld ...
+
+  // After the DBG_VALUE Y is processed, the DbgValueSinkCandidates is updated
+  // to now hold Y for A and the RegisterMap is also updated to remove X from
+  // %2, this is because both X and Y describe the same debug variable A. X is
+  // also updated to have a $noreg as the first operand.
+  // DbgValueSinkCandidates = {A -> {Y}}, RegisterMap = {2 -> {}, 3 -> {}},
+  // InstrMap = {X-> 2}
+  // DBG_VALUE $noreg, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // -> DBG_VALUE %3, A, ..., # Z
+  // %2 = ld ...
+  // %3 = ld ...
+  // %4 = ld ...
+
+  // After DBG_VALUE Z is processed, the DbgValueSinkCandidates is updated to
+  // hold Z fr A, the RegisterMap is updated to hold Z for %3, and the InstrMap
+  // is updated to have Z mapped to %3. This is again because Z describes the
+  // debug variable A, Y is not updated to have $noreg as first operand because
+  // its first operand is an immediate, not a register.
+  // DbgValueSinkCandidates = {A -> {Z}}, RegisterMap = {2 -> {}, 3 -> {Z}},
+  // InstrMap = {X -> 2, Z -> 3}
+  // DBG_VALUE $noreg, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE %3, A, ..., # Z
+  // -> %2 = ld ...
+  // %3 = ld ...
+  // %4 = ld ...
+
+  // Nothing happens here since the RegisterMap for %2 contains no value.
+  // DbgValueSinkCandidates = {A -> {Z}}, RegisterMap = {2 -> {}, 3 -> {Z}},
+  // InstrMap = {X -> 2, Z -> 3}
+  // DBG_VALUE $noreg, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE %3, A, ..., # Z
+  // %2 = ld ...
+  // -> %3 = ld ...
+  // %4 = ld ...
+
+  // Since the RegisterMap contains Z as a value for %3, the MachineInstr
+  // pointer Z is copied to come after the load for %3 and the old Z's first
+  // operand is changed to $noreg the Basic Block iterator is moved to after the
+  // DBG_VALUE Z's new position.
+  // DbgValueSinkCandidates = {A -> {Z}}, RegisterMap = {2 -> {}, 3 -> {Z}},
+  // InstrMap = {X -> 2, Z -> 3}
+  // DBG_VALUE $noreg, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE $noreg, A, ..., # Old Z
+  // %2 = ld ...
+  // %3 = ld ...
+  // DBG_VALUE %3, A, ..., # Z
+  // -> %4 = ld ...
+
+  // Nothing happens for %4 and the algorithm exits having processed the entire
+  // Basic Block.
+  // DbgValueSinkCandidates = {A -> {Z}}, RegisterMap = {2 -> {}, 3 -> {Z}},
+  // InstrMap = {X -> 2, Z -> 3}
+  // DBG_VALUE $noreg, A, .... # X
+  // DBG_VALUE 0, A, ... # Y
+  // DBG_VALUE $noreg, A, ..., # Old Z
+  // %2 = ld ...
+  // %3 = ld ...
+  // DBG_VALUE %3, A, ..., # Z
+  // %4 = ld ...
+
+  // This map is used to track the relationship between
+  // a Debug Variable and the DBG_VALUE MachineInstr pointer that describes the
+  // debug information for that Debug Variable.
+  SmallDenseMap<DebugVariable, MachineInstr *, 8> DbgValueSinkCandidates;
+  // This map is used to track the relationship between a DBG_VALUE or
+  // DBG_VALUE_LIST MachineInstr pointer and Registers that it uses.
+  SmallDenseMap<MachineInstr *, SmallVector<Register>, 8> InstrMap;
+  for (MBBI = MBB->begin(), E = MBB->end(); MBBI != E; ++MBBI) {
+    MachineInstr &MI = *MBBI;
+
+    auto PopulateRegisterAndInstrMapForDebugInstr = [&](Register Reg) {
+      auto RegIt = RegisterMap.find(Reg);
+      if (RegIt == RegisterMap.end())
+        return;
+      auto &InstrVec = RegIt->getSecond();
+      InstrVec.push_back(&MI);
+      InstrMap[&MI].push_back(Reg);
+    };
+
+    if (MI.isDebugValue()) {
+      assert(MI.getDebugVariable() &&
+             "DBG_VALUE or DBG_VALUE_LIST must contain a DILocalVariable");
+
+      auto DbgVar = createDebugVariableFromMachineInstr(&MI);
+      // If the first operand is a register and it exists in the RegisterMap, we
+      // know this is a DBG_VALUE that uses the result of a load that was moved,
+      // and is therefore a candidate to also be moved, add it to the
+      // RegisterMap and InstrMap.
+      forEachDbgRegOperand(&MI, [&](MachineOperand &Op) {
+        PopulateRegisterAndInstrMapForDebugInstr(Op.getReg());
+      });
+
+      // If the current DBG_VALUE describes the same variable as one of the
+      // in-flight DBG_VALUEs, remove the candidate from the list and set it to
+      // undef. Moving one DBG_VALUE past another would result in the variable's
+      // value going back in time when stepping through the block in the
+      // debugger.
+      auto InstrIt = DbgValueSinkCandidates.find(DbgVar);
+      if (InstrIt != DbgValueSinkCandidates.end()) {
+        auto *Instr = InstrIt->getSecond();
+        auto RegIt = InstrMap.find(Instr);
+        if (RegIt != InstrMap.end()) {
+          const auto &RegVec = RegIt->getSecond();
+          // For every Register in the RegVec, remove the MachineInstr in the
+          // RegisterMap that describes the DbgVar.
+          for (auto &Reg : RegVec) {
+            auto RegIt = RegisterMap.find(Reg);
+            if (RegIt == RegisterMap.end())
+              continue;
+            auto &InstrVec = RegIt->getSecond();
+            auto IsDbgVar = [&](MachineInstr *I) -> bool {
+              auto Var = createDebugVariableFromMachineInstr(I);
+              return Var == DbgVar;
+            };
+
+            InstrVec.erase(
+                std::remove_if(InstrVec.begin(), InstrVec.end(), IsDbgVar),
+                InstrVec.end());
+          }
+          forEachDbgRegOperand(Instr,
+                               [&](MachineOperand &Op) { Op.setReg(0); });
+        }
+      }
+      DbgValueSinkCandidates[DbgVar] = &MI;
+    } else {
+      // If the first operand of a load matches with a DBG_VALUE in RegisterMap,
+      // then move that DBG_VALUE to below the load.
+      auto Opc = MI.getOpcode();
+      if (!isLoadSingle(Opc))
+        continue;
+      auto Reg = MI.getOperand(0).getReg();
+      auto RegIt = RegisterMap.find(Reg);
+      if (RegIt == RegisterMap.end())
+        continue;
+      auto &DbgInstrVec = RegIt->getSecond();
+      if (!DbgInstrVec.size())
+        continue;
+      for (auto *DbgInstr : DbgInstrVec) {
+        MachineBasicBlock::iterator InsertPos = std::next(MBBI);
+        auto *ClonedMI = MI.getMF()->CloneMachineInstr(DbgInstr);
+        MBB->insert(InsertPos, ClonedMI);
+        MBBI++;
+        //  Erase the entry into the DbgValueSinkCandidates for the DBG_VALUE
+        //  that was moved.
+        auto DbgVar = createDebugVariableFromMachineInstr(DbgInstr);
+        auto DbgIt = DbgValueSinkCandidates.find(DbgVar);
+        // If the instruction is a DBG_VALUE_LIST, it may have already been
+        // erased from the DbgValueSinkCandidates. Only erase if it exists in
+        // the DbgValueSinkCandidates.
+        if (DbgIt != DbgValueSinkCandidates.end())
+          DbgValueSinkCandidates.erase(DbgIt);
+        // Zero out original dbg instr
+        forEachDbgRegOperand(DbgInstr,
+                             [&](MachineOperand &Op) { Op.setReg(0); });
+        // Update RegisterMap with ClonedMI because it might have to be moved
+        // again.
+        if (DbgInstr->isDebugValueList())
+          updateRegisterMapForDbgValueListAfterMove(RegisterMap, ClonedMI,
+                                                    DbgInstr);
+      }
+    }
+  }
   return RetVal;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 08b040ba092eb..d7f718920fedb 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/CodeGen/CostTable.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DataLayout.h"
@@ -28,7 +29,6 @@
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index ace2891c9deea..bde60372be6d0 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -11957,7 +11957,7 @@ bool ARMAsmParser::parseDirectiveSetFP(SMLoc L) {
     Offset = CE->getValue();
   }
 
-  if (Parser.parseToken(AsmToken::EndOfStatement))
+  if (Parser.parseEOL())
     return true;
 
   getTargetStreamer().emitSetFP(static_cast<unsigned>(FPReg),
diff --git a/llvm/lib/Target/ARM/AsmParser/CMakeLists.txt b/llvm/lib/Target/ARM/AsmParser/CMakeLists.txt
index 424d8550aa09f..903ace64ef7ef 100644
--- a/llvm/lib/Target/ARM/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/ARM/AsmParser/CMakeLists.txt
@@ -5,6 +5,7 @@ add_llvm_component_library(LLVMARMAsmParser
   ARMDesc
   ARMInfo
   ARMUtils
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/lib/Target/ARM/CMakeLists.txt b/llvm/lib/Target/ARM/CMakeLists.txt
index fa7a7471b7926..3d6af28b43753 100644
--- a/llvm/lib/Target/ARM/CMakeLists.txt
+++ b/llvm/lib/Target/ARM/CMakeLists.txt
@@ -77,6 +77,7 @@ add_llvm_target(ARMCodeGen
   AsmPrinter
   CFGuard
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   IPO
diff --git a/llvm/lib/Target/ARM/Disassembler/CMakeLists.txt b/llvm/lib/Target/ARM/Disassembler/CMakeLists.txt
index 5d20172bf36ab..e4c38eef025f0 100644
--- a/llvm/lib/Target/ARM/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/ARM/Disassembler/CMakeLists.txt
@@ -5,6 +5,7 @@ add_llvm_component_library(LLVMARMDisassembler
   ARMDesc
   ARMInfo
   ARMUtils
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
index d84dd962d8b5b..687d9a9e918db 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/ARM/MCTargetDesc/CMakeLists.txt
@@ -18,6 +18,7 @@ add_llvm_component_library(LLVMARMDesc
   ARMInfo
   ARMUtils
   BinaryFormat
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/AVR/AsmParser/CMakeLists.txt b/llvm/lib/Target/AVR/AsmParser/CMakeLists.txt
index a72c2ad2a1745..c1616f74aabdc 100644
--- a/llvm/lib/Target/AVR/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/AVR/AsmParser/CMakeLists.txt
@@ -4,6 +4,7 @@ add_llvm_component_library(LLVMAVRAsmParser
   LINK_COMPONENTS
   AVRDesc
   AVRInfo
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/lib/Target/AVR/CMakeLists.txt b/llvm/lib/Target/AVR/CMakeLists.txt
index 3c87c6f499b60..817ba739d8418 100644
--- a/llvm/lib/Target/AVR/CMakeLists.txt
+++ b/llvm/lib/Target/AVR/CMakeLists.txt
@@ -36,6 +36,7 @@ add_llvm_target(AVRCodeGen
   AVRInfo
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   SelectionDAG
diff --git a/llvm/lib/Target/AVR/Disassembler/CMakeLists.txt b/llvm/lib/Target/AVR/Disassembler/CMakeLists.txt
index 37b35f317aa1c..15dedae80c648 100644
--- a/llvm/lib/Target/AVR/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/AVR/Disassembler/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMAVRDisassembler
 
   LINK_COMPONENTS
   AVRInfo
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index 443cbb91cb684..f4a8fa3674cd3 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -41,6 +41,7 @@ add_llvm_target(BPFCodeGen
   BPFDesc
   BPFInfo
   CodeGen
+  CodeGenTypes
   Core
   IPO
   MC
diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
index 5198399a4ebf7..2b1e2e7c366e8 100644
--- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
+++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
@@ -61,7 +61,8 @@ class CSKYAsmParser : public MCTargetAsmParser {
                                       unsigned Kind) override;
 
   bool generateImmOutOfRangeError(OperandVector &Operands, uint64_t ErrorInfo,
-                                  int64_t Lower, int64_t Upper, Twine Msg);
+                                  int64_t Lower, int64_t Upper,
+                                  const Twine &Msg);
 
   SMLoc getLoc() const { return getParser().getTok().getLoc(); }
 
@@ -650,7 +651,7 @@ static std::string CSKYMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS,
 
 bool CSKYAsmParser::generateImmOutOfRangeError(
     OperandVector &Operands, uint64_t ErrorInfo, int64_t Lower, int64_t Upper,
-    Twine Msg = "immediate must be an integer in the range") {
+    const Twine &Msg = "immediate must be an integer in the range") {
   SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc();
   return Error(ErrorLoc, Msg + " [" + Twine(Lower) + ", " + Twine(Upper) + "]");
 }
diff --git a/llvm/lib/Target/CSKY/CMakeLists.txt b/llvm/lib/Target/CSKY/CMakeLists.txt
index 7f04cc4a21fe1..cdce80591a2fd 100644
--- a/llvm/lib/Target/CSKY/CMakeLists.txt
+++ b/llvm/lib/Target/CSKY/CMakeLists.txt
@@ -33,10 +33,11 @@ add_llvm_target(CSKYCodeGen
   LINK_COMPONENTS
   Analysis
   AsmPrinter
-  Core
-  CodeGen
   CSKYDesc
   CSKYInfo
+  CodeGen
+  CodeGenTypes
+  Core
   MC
   SelectionDAG
   Support
diff --git a/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp b/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
index 09c2d5161aba0..702053e02332e 100644
--- a/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp
@@ -74,10 +74,10 @@ void CSKYDAGToDAGISel::Select(SDNode *N) {
   switch (Opcode) {
   default:
     break;
-  case ISD::ADDCARRY:
+  case ISD::UADDO_CARRY:
     IsSelected = selectAddCarry(N);
     break;
-  case ISD::SUBCARRY:
+  case ISD::USUBO_CARRY:
     IsSelected = selectSubCarry(N);
     break;
   case ISD::GLOBAL_OFFSET_TABLE: {
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index a08ba27f6ec36..487b9d9d1388b 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -51,8 +51,8 @@ CSKYTargetLowering::CSKYTargetLowering(const TargetMachine &TM,
       addRegisterClass(MVT::f64, &CSKY::FPR64RegClass);
   }
 
-  setOperationAction(ISD::ADDCARRY, MVT::i32, Legal);
-  setOperationAction(ISD::SUBCARRY, MVT::i32, Legal);
+  setOperationAction(ISD::UADDO_CARRY, MVT::i32, Legal);
+  setOperationAction(ISD::USUBO_CARRY, MVT::i32, Legal);
   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
 
   setOperationAction(ISD::SREM, MVT::i32, Expand);
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
index da134c4e056ad..73b4429349ea8 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_component_library(LLVMCSKYDesc
 
   LINK_COMPONENTS
   CSKYInfo
+  CodeGenTypes
   MC
   Support
   TargetParser
diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index dbee3de2c59f2..bf93280779bf8 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -32,23 +32,24 @@ add_llvm_target(DirectXCodeGen
   LINK_COMPONENTS
   Analysis
   AsmPrinter
-  Core
   CodeGen
-  SelectionDAG
-  Support
-  TargetParser
+  CodeGenTypes
+  Core
+  DXILBitWriter
   DirectXDesc
   DirectXInfo
   DirectXPointerTypeAnalysis
-  DXILBitWriter
   FrontendHLSL
   MC
+  SelectionDAG
+  Support
   Target
+  TargetParser
   TransformUtils
 
   ADD_TO_COMPONENT
   DirectX
-)
+  )
 
 add_subdirectory(MCTargetDesc)
 add_subdirectory(TargetInfo)
diff --git a/llvm/lib/Target/Hexagon/CMakeLists.txt b/llvm/lib/Target/Hexagon/CMakeLists.txt
index e36f5be7ecc23..76f99b4d3ec58 100644
--- a/llvm/lib/Target/Hexagon/CMakeLists.txt
+++ b/llvm/lib/Target/Hexagon/CMakeLists.txt
@@ -72,6 +72,7 @@ add_llvm_target(HexagonCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   HexagonAsmParser
   HexagonDesc
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 1838b95f2b79a..f826f613b2846 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1545,15 +1545,15 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
   // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
   // but they only operate on i64.
   for (MVT VT : MVT::integer_valuetypes()) {
-    setOperationAction(ISD::UADDO,    VT, Custom);
-    setOperationAction(ISD::USUBO,    VT, Custom);
-    setOperationAction(ISD::SADDO,    VT, Expand);
-    setOperationAction(ISD::SSUBO,    VT, Expand);
-    setOperationAction(ISD::ADDCARRY, VT, Expand);
-    setOperationAction(ISD::SUBCARRY, VT, Expand);
+    setOperationAction(ISD::UADDO, VT, Custom);
+    setOperationAction(ISD::USUBO, VT, Custom);
+    setOperationAction(ISD::SADDO, VT, Expand);
+    setOperationAction(ISD::SSUBO, VT, Expand);
+    setOperationAction(ISD::UADDO_CARRY, VT, Expand);
+    setOperationAction(ISD::USUBO_CARRY, VT, Expand);
   }
-  setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
-  setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
+  setOperationAction(ISD::UADDO_CARRY, MVT::i64, Custom);
+  setOperationAction(ISD::USUBO_CARRY, MVT::i64, Custom);
 
   setOperationAction(ISD::CTLZ, MVT::i8,  Promote);
   setOperationAction(ISD::CTLZ, MVT::i16, Promote);
@@ -3263,13 +3263,13 @@ HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
-SDValue
-HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const {
+SDValue HexagonTargetLowering::LowerUAddSubOCarry(SDValue Op,
+                                                  SelectionDAG &DAG) const {
   const SDLoc &dl(Op);
   unsigned Opc = Op.getOpcode();
   SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
 
-  if (Opc == ISD::ADDCARRY)
+  if (Opc == ISD::UADDO_CARRY)
     return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),
                        { X, Y, C });
 
@@ -3342,8 +3342,8 @@ HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     case ISD::STORE:                return LowerStore(Op, DAG);
     case ISD::UADDO:
     case ISD::USUBO:                return LowerUAddSubO(Op, DAG);
-    case ISD::ADDCARRY:
-    case ISD::SUBCARRY:             return LowerAddSubCarry(Op, DAG);
+    case ISD::UADDO_CARRY:
+    case ISD::USUBO_CARRY:          return LowerUAddSubOCarry(Op, DAG);
     case ISD::SRA:
     case ISD::SHL:
     case ISD::SRL:                  return LowerVECTOR_SHIFT(Op, DAG);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index fc7072d14ccdd..f2ad518665283 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -18,12 +18,12 @@
 #include "MCTargetDesc/HexagonMCTargetDesc.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/InlineAsm.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cstdint>
 #include <utility>
 
@@ -200,7 +200,7 @@ class HexagonTargetLowering : public TargetLowering {
   SDValue LowerStore(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const;
-  SDValue LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerUAddSubOCarry(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 4fd7491ac0a32..05daa3bfff543 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -33,6 +33,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
@@ -48,7 +49,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 7c9a9f7918a12..0bc0877f6e706 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -17,9 +17,9 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cstdint>
 #include <vector>
 
diff --git a/llvm/lib/Target/Lanai/AsmParser/CMakeLists.txt b/llvm/lib/Target/Lanai/AsmParser/CMakeLists.txt
index ed5acca9bfba1..40bbe1dac8fb3 100644
--- a/llvm/lib/Target/Lanai/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/Lanai/AsmParser/CMakeLists.txt
@@ -4,6 +4,7 @@ add_llvm_component_library(LLVMLanaiAsmParser
   LanaiAsmParser.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   LanaiDesc
   LanaiInfo
   MC
diff --git a/llvm/lib/Target/Lanai/CMakeLists.txt b/llvm/lib/Target/Lanai/CMakeLists.txt
index 8035ec84e0d02..16d5f727043fe 100644
--- a/llvm/lib/Target/Lanai/CMakeLists.txt
+++ b/llvm/lib/Target/Lanai/CMakeLists.txt
@@ -34,6 +34,7 @@ add_llvm_target(LanaiCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   LanaiAsmParser
   LanaiDesc
diff --git a/llvm/lib/Target/Lanai/Disassembler/CMakeLists.txt b/llvm/lib/Target/Lanai/Disassembler/CMakeLists.txt
index cc5af1466ba4b..46c63675fa3ed 100644
--- a/llvm/lib/Target/Lanai/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/Lanai/Disassembler/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMLanaiDisassembler
   LanaiDisassembler.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   LanaiDesc
   LanaiInfo
   MC
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index ed68602024259..4c58173518c86 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -42,7 +43,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index 9d6d981bb9082..164f8a2f7e648 100644
--- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -64,7 +64,8 @@ class LoongArchAsmParser : public MCTargetAsmParser {
                                       unsigned Kind) override;
 
   bool generateImmOutOfRangeError(OperandVector &Operands, uint64_t ErrorInfo,
-                                  int64_t Lower, int64_t Upper, Twine Msg);
+                                  int64_t Lower, int64_t Upper,
+                                  const Twine &Msg);
 
   /// Helper for processing MC instructions that have been successfully matched
   /// by MatchAndEmitInstruction.
@@ -1242,7 +1243,7 @@ LoongArchAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
 
 bool LoongArchAsmParser::generateImmOutOfRangeError(
     OperandVector &Operands, uint64_t ErrorInfo, int64_t Lower, int64_t Upper,
-    Twine Msg = "immediate must be an integer in the range") {
+    const Twine &Msg = "immediate must be an integer in the range") {
   SMLoc ErrorLoc = ((LoongArchOperand &)*Operands[ErrorInfo]).getStartLoc();
   return Error(ErrorLoc, Msg + " [" + Twine(Lower) + ", " + Twine(Upper) + "]");
 }
diff --git a/llvm/lib/Target/LoongArch/CMakeLists.txt b/llvm/lib/Target/LoongArch/CMakeLists.txt
index 9b2f1e97cc7ec..54bd73112d461 100644
--- a/llvm/lib/Target/LoongArch/CMakeLists.txt
+++ b/llvm/lib/Target/LoongArch/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_target(LoongArchCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   LoongArchDesc
   LoongArchInfo
diff --git a/llvm/lib/Target/M68k/AsmParser/CMakeLists.txt b/llvm/lib/Target/M68k/AsmParser/CMakeLists.txt
index a213dabcef148..67ae6abfee1fc 100644
--- a/llvm/lib/Target/M68k/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/M68k/AsmParser/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMM68kAsmParser
   M68kAsmParser.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   M68kCodeGen
   M68kInfo
   MC
diff --git a/llvm/lib/Target/M68k/CMakeLists.txt b/llvm/lib/Target/M68k/CMakeLists.txt
index 86f785f301aee..1661dccece3dd 100644
--- a/llvm/lib/Target/M68k/CMakeLists.txt
+++ b/llvm/lib/Target/M68k/CMakeLists.txt
@@ -40,18 +40,19 @@ add_llvm_target(M68kCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
+  M68kDesc
+  M68kInfo
   MC
   SelectionDAG
   Support
   Target
-  M68kDesc
-  M68kInfo
 
   ADD_TO_COMPONENT
   M68k
-)
+  )
 
 add_subdirectory(TargetInfo)
 add_subdirectory(MCTargetDesc)
diff --git a/llvm/lib/Target/M68k/Disassembler/CMakeLists.txt b/llvm/lib/Target/M68k/Disassembler/CMakeLists.txt
index 2b8e676fee1bd..3ce486a76cb02 100644
--- a/llvm/lib/Target/M68k/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/M68k/Disassembler/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMM68kDisassembler
   M68kDisassembler.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   M68kDesc
   M68kInfo
   MCDisassembler
diff --git a/llvm/lib/Target/M68k/M68kExpandPseudo.cpp b/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
index 5383d31079554..09e72e20cbe51 100644
--- a/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
+++ b/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
@@ -162,6 +162,16 @@ bool M68kExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     return TII->ExpandMOVSZX_RM(MIB, false, TII->get(M68k::MOV16rf), MVT::i32,
                                 MVT::i16);
 
+  case M68k::MOVZXd16q8:
+    return TII->ExpandMOVSZX_RM(MIB, false, TII->get(M68k::MOV8dq), MVT::i16,
+                                MVT::i8);
+  case M68k::MOVZXd32q8:
+    return TII->ExpandMOVSZX_RM(MIB, false, TII->get(M68k::MOV8dq), MVT::i32,
+                                MVT::i8);
+  case M68k::MOVZXd32q16:
+    return TII->ExpandMOVSZX_RM(MIB, false, TII->get(M68k::MOV16dq), MVT::i32,
+                                MVT::i16);
+
   case M68k::MOV8cd:
     return TII->ExpandCCR(MIB, /*IsToCCR=*/true);
   case M68k::MOV8dc:
diff --git a/llvm/lib/Target/M68k/M68kInstrData.td b/llvm/lib/Target/M68k/M68kInstrData.td
index 00fcb3db8eb68..e6d4471f7aab6 100644
--- a/llvm/lib/Target/M68k/M68kInstrData.td
+++ b/llvm/lib/Target/M68k/M68kInstrData.td
@@ -525,6 +525,10 @@ foreach EXT = ["S", "Z"] in {
       def MOV#EXT#Xd32f8   : MxPseudoMove_RM<MxType32d,  MxType8.FOp>;
       def MOV#EXT#Xd32f16  : MxPseudoMove_RM<MxType32d, MxType16.FOp>;
 
+      def MOV#EXT#Xd16q8   : MxPseudoMove_RM<MxType16d,  MxType8.QOp>;
+      def MOV#EXT#Xd32q8   : MxPseudoMove_RM<MxType32d,  MxType8.QOp>;
+      def MOV#EXT#Xd32q16  : MxPseudoMove_RM<MxType32d,  MxType16.QOp>;
+
     }
   }
 }
@@ -572,18 +576,21 @@ def: Pat<(MxZExtLoadi16i8 MxCP_ARID:$src),
           (EXTRACT_SUBREG (MOVZXd32p8 MxARID8:$src), MxSubRegIndex16Lo)>;
 def: Pat<(MxZExtLoadi16i8 MxCP_ARII:$src),
           (EXTRACT_SUBREG (MOVZXd32f8 MxARII8:$src), MxSubRegIndex16Lo)>;
+def: Pat<(MxZExtLoadi16i8 MxCP_PCD :$src), (MOVZXd16q8 MxPCD8 :$src)>;
 
 // i32 <- zext i8
 def: Pat<(i32 (zext i8:$src)), (MOVZXd32d8 MxDRD8:$src)>;
 def: Pat<(MxZExtLoadi32i8 MxCP_ARI :$src), (MOVZXd32j8 MxARI8 :$src)>;
 def: Pat<(MxZExtLoadi32i8 MxCP_ARID:$src), (MOVZXd32p8 MxARID8:$src)>;
 def: Pat<(MxZExtLoadi32i8 MxCP_ARII:$src), (MOVZXd32f8 MxARII8:$src)>;
+def: Pat<(MxZExtLoadi32i8 MxCP_PCD :$src), (MOVZXd32q8 MxPCD8 :$src)>;
 
 // i32 <- zext i16
 def: Pat<(i32 (zext i16:$src)), (MOVZXd32d16 MxDRD16:$src)>;
 def: Pat<(MxZExtLoadi32i16 MxCP_ARI :$src), (MOVZXd32j16 MxARI16 :$src)>;
 def: Pat<(MxZExtLoadi32i16 MxCP_ARID:$src), (MOVZXd32p16 MxARID16:$src)>;
 def: Pat<(MxZExtLoadi32i16 MxCP_ARII:$src), (MOVZXd32f16 MxARII16:$src)>;
+def: Pat<(MxZExtLoadi32i16 MxCP_PCD :$src), (MOVZXd32q16 MxPCD16 :$src)>;
 
 // i16 <- anyext i8
 def: Pat<(i16 (anyext i8:$src)),
diff --git a/llvm/lib/Target/M68k/M68kMachineFunction.h b/llvm/lib/Target/M68k/M68kMachineFunction.h
index 188265079cd16..e40069e7afdba 100644
--- a/llvm/lib/Target/M68k/M68kMachineFunction.h
+++ b/llvm/lib/Target/M68k/M68kMachineFunction.h
@@ -16,7 +16,7 @@
 
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/Support/MachineValueType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 
 namespace llvm {
 
diff --git a/llvm/lib/Target/MSP430/AsmParser/CMakeLists.txt b/llvm/lib/Target/MSP430/AsmParser/CMakeLists.txt
index 55f2687188463..c00dfa838b090 100644
--- a/llvm/lib/Target/MSP430/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/MSP430/AsmParser/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMMSP430AsmParser
   MSP430AsmParser.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCParser
   MSP430Desc
diff --git a/llvm/lib/Target/MSP430/CMakeLists.txt b/llvm/lib/Target/MSP430/CMakeLists.txt
index 5583482296f41..1dfe8635acb32 100644
--- a/llvm/lib/Target/MSP430/CMakeLists.txt
+++ b/llvm/lib/Target/MSP430/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_target(MSP430CodeGen
   LINK_COMPONENTS
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   MSP430Desc
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
index d04bb406b9bb8..386b5abe38017 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
@@ -16,7 +16,11 @@ using namespace llvm;
 void MSP430MCAsmInfo::anchor() { }
 
 MSP430MCAsmInfo::MSP430MCAsmInfo(const Triple &TT) {
-  CodePointerSize = 2;
+  // Since MSP430-GCC already generates 32-bit DWARF information, we will
+  // also store 16-bit pointers as 32-bit pointers in DWARF, because using
+  // 32-bit DWARF pointers is already a working and tested path for LLDB
+  // as well.
+  CodePointerSize = 4;
   CalleeSaveStackSlotSize = 2;
 
   CommentString = ";";
diff --git a/llvm/lib/Target/Mips/CMakeLists.txt b/llvm/lib/Target/Mips/CMakeLists.txt
index 92da4b095b372..57f2e01b1c60d 100644
--- a/llvm/lib/Target/Mips/CMakeLists.txt
+++ b/llvm/lib/Target/Mips/CMakeLists.txt
@@ -69,6 +69,7 @@ add_llvm_target(MipsCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   MC
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
index 89050e209c45e..97a6f886d114e 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
@@ -14,6 +14,7 @@ add_llvm_component_library(LLVMMipsDesc
   MipsTargetStreamer.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MipsInfo
   Support
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
index 933077beb0c21..3e0d51ef887c4 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
@@ -9,10 +9,10 @@
 #include "MipsABIInfo.h"
 #include "MipsRegisterInfo.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 263f25a41f174..ccea724abbe2a 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -35,6 +35,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -63,7 +64,6 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 02c10dd13a027..bcb78a4d16342 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -40,6 +40,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -63,7 +64,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h
index 4a1873dd1b452..8614c4d3abe5a 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.h
+++ b/llvm/lib/Target/Mips/MipsISelLowering.h
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -28,7 +29,6 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Type.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Target/TargetMachine.h"
 #include <algorithm>
 #include <cassert>
diff --git a/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp b/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp
index 96fd9abfa19f0..ef847adbebc1c 100644
--- a/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp
+++ b/llvm/lib/Target/Mips/MipsOptimizePICCall.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -33,7 +34,6 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/RecyclingAllocator.h"
 #include <cassert>
 #include <utility>
diff --git a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp
index d7d28e7c46fbf..b84f304373f66 100644
--- a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -38,7 +39,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/lib/Target/Mips/MipsSEISelLowering.h b/llvm/lib/Target/Mips/MipsSEISelLowering.h
index 9714e976a9d38..34f221e2fbf24 100644
--- a/llvm/lib/Target/Mips/MipsSEISelLowering.h
+++ b/llvm/lib/Target/Mips/MipsSEISelLowering.h
@@ -14,8 +14,8 @@
 #define LLVM_LIB_TARGET_MIPS_MIPSSEISELLOWERING_H
 
 #include "MipsISelLowering.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
-#include "llvm/Support/MachineValueType.h"
 
 namespace llvm {
 
diff --git a/llvm/lib/Target/NVPTX/CMakeLists.txt b/llvm/lib/Target/NVPTX/CMakeLists.txt
index 723ab6227cc1d..6b35c2a0444f2 100644
--- a/llvm/lib/Target/NVPTX/CMakeLists.txt
+++ b/llvm/lib/Target/NVPTX/CMakeLists.txt
@@ -37,6 +37,7 @@ set(NVPTXCodeGen_sources
   NVVMIntrRange.cpp
   NVVMReflect.cpp
   NVPTXProxyRegErasure.cpp
+  NVPTXCtorDtorLowering.cpp
   )
 
 add_llvm_target(NVPTXCodeGen
@@ -46,6 +47,7 @@ add_llvm_target(NVPTXCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   IPO
   MC
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 521a7843b1142..9e8a2761010c4 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -39,6 +39,7 @@ FunctionPass *createNVPTXISelDag(NVPTXTargetMachine &TM,
                                  llvm::CodeGenOpt::Level OptLevel);
 ModulePass *createNVPTXAssignValidGlobalNamesPass();
 ModulePass *createGenericToNVVMLegacyPass();
+ModulePass *createNVPTXCtorDtorLoweringLegacyPass();
 FunctionPass *createNVVMIntrRangePass(unsigned int SmVersion);
 FunctionPass *createNVVMReflectPass(unsigned int SmVersion);
 MachineFunctionPass *createNVPTXPrologEpilogPass();
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index aea4be0291fa1..5fce6d67f3067 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -44,6 +44,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Attributes.h"
@@ -74,7 +75,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/NativeFormatting.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
@@ -92,6 +92,11 @@
 
 using namespace llvm;
 
+static cl::opt<bool>
+    LowerCtorDtor("nvptx-lower-global-ctor-dtor",
+                  cl::desc("Lower GPU ctor / dtors to globals on the device."),
+                  cl::init(false), cl::Hidden);
+
 #define DEPOTNAME "__local_depot"
 
 /// DiscoverDependentGlobals - Return a set of GlobalVariables on which \p V
@@ -788,12 +793,14 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
     report_fatal_error("Module has aliases, which NVPTX does not support.");
     return true; // error
   }
-  if (!isEmptyXXStructor(M.getNamedGlobal("llvm.global_ctors"))) {
+  if (!isEmptyXXStructor(M.getNamedGlobal("llvm.global_ctors")) &&
+      !LowerCtorDtor) {
     report_fatal_error(
         "Module has a nontrivial global ctor, which NVPTX does not support.");
     return true;  // error
   }
-  if (!isEmptyXXStructor(M.getNamedGlobal("llvm.global_dtors"))) {
+  if (!isEmptyXXStructor(M.getNamedGlobal("llvm.global_dtors")) &&
+      !LowerCtorDtor) {
     report_fatal_error(
         "Module has a nontrivial global dtor, which NVPTX does not support.");
     return true;  // error
diff --git a/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp
new file mode 100644
index 0000000000000..51d0a1304951a
--- /dev/null
+++ b/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp
@@ -0,0 +1,116 @@
+//===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This pass creates a unified init and fini kernel with the required metadata
+//===----------------------------------------------------------------------===//
+
+#include "NVPTXCtorDtorLowering.h"
+#include "NVPTX.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "nvptx-lower-ctor-dtor"
+
+static cl::opt<std::string>
+    GlobalStr("nvptx-lower-global-ctor-dtor-id",
+              cl::desc("Override unique ID of ctor/dtor globals."),
+              cl::init(""), cl::Hidden);
+
+namespace {
+
+static std::string getHash(StringRef Str) {
+  llvm::MD5 Hasher;
+  llvm::MD5::MD5Result Hash;
+  Hasher.update(Str);
+  Hasher.final(Hash);
+  return llvm::utohexstr(Hash.low(), /*LowerCase=*/true);
+}
+
+static bool createInitOrFiniGlobls(Module &M, StringRef GlobalName,
+                                   bool IsCtor) {
+  GlobalVariable *GV = M.getGlobalVariable(GlobalName);
+  if (!GV || !GV->hasInitializer())
+    return false;
+  ConstantArray *GA = dyn_cast<ConstantArray>(GV->getInitializer());
+  if (!GA || GA->getNumOperands() == 0)
+    return false;
+
+  // NVPTX has no way to emit variables at specific sections or support for
+  // the traditional constructor sections. Instead, we emit mangled global
+  // names so the runtime can build the list manually.
+  for (Value *V : GA->operands()) {
+    auto *CS = cast<ConstantStruct>(V);
+    auto *F = cast<Constant>(CS->getOperand(1));
+    uint64_t Priority = cast<ConstantInt>(CS->getOperand(0))->getSExtValue();
+    std::string PriorityStr = "." + std::to_string(Priority);
+    // We append a semi-unique hash and the priority to the global name.
+    std::string GlobalID =
+        !GlobalStr.empty() ? GlobalStr : getHash(M.getSourceFileName());
+    std::string NameStr =
+        ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
+         F->getName() + "_" + GlobalID + "_" + std::to_string(Priority))
+            .str();
+    // PTX does not support exported names with '.' in them.
+    llvm::transform(NameStr, NameStr.begin(),
+                    [](char c) { return c == '.' ? '_' : c; });
+
+    auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
+                                  GlobalValue::ExternalLinkage, F, NameStr,
+                                  nullptr, GlobalValue::NotThreadLocal,
+                                  /*AddressSpace=*/4);
+    // This isn't respected by Nvidia, simply put here for clarity.
+    GV->setSection(IsCtor ? ".init_array" + PriorityStr
+                          : ".fini_array" + PriorityStr);
+    GV->setVisibility(GlobalVariable::ProtectedVisibility);
+    appendToUsed(M, {GV});
+  }
+
+  GV->eraseFromParent();
+  return true;
+}
+
+static bool lowerCtorsAndDtors(Module &M) {
+  bool Modified = false;
+  Modified |= createInitOrFiniGlobls(M, "llvm.global_ctors", /*IsCtor =*/true);
+  Modified |= createInitOrFiniGlobls(M, "llvm.global_dtors", /*IsCtor =*/false);
+  return Modified;
+}
+
+class NVPTXCtorDtorLoweringLegacy final : public ModulePass {
+public:
+  static char ID;
+  NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {}
+  bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
+};
+
+} // End anonymous namespace
+
+PreservedAnalyses NVPTXCtorDtorLoweringPass::run(Module &M,
+                                                 ModuleAnalysisManager &AM) {
+  return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
+                               : PreservedAnalyses::all();
+}
+
+char NVPTXCtorDtorLoweringLegacy::ID = 0;
+char &llvm::NVPTXCtorDtorLoweringLegacyPassID = NVPTXCtorDtorLoweringLegacy::ID;
+INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE,
+                "Lower ctors and dtors for NVPTX", false, false)
+
+ModulePass *llvm::createNVPTXCtorDtorLoweringLegacyPass() {
+  return new NVPTXCtorDtorLoweringLegacy();
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.h b/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.h
new file mode 100644
index 0000000000000..c03fe97f1a782
--- /dev/null
+++ b/llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.h
@@ -0,0 +1,30 @@
+//===-- NVPTXCtorDtorLowering.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_NVPTX_NVPTXCTORDTORLOWERING_H
+#define LLVM_LIB_TARGET_NVPTX_NVPTXCTORDTORLOWERING_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+class Module;
+class PassRegistry;
+
+extern char &NVPTXCtorDtorLoweringLegacyPassID;
+extern void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &);
+
+/// Lower llvm.global_ctors and llvm.global_dtors to special kernels.
+class NVPTXCtorDtorLoweringPass
+    : public PassInfoMixin<NVPTXCtorDtorLoweringPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_NVPTX_NVPTXCTORDTORLOWERING_H
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index ec96e3c2d7d26..692ec58bdedff 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -25,6 +25,7 @@
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
@@ -48,7 +49,6 @@
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -89,6 +89,12 @@ static cl::opt<bool> UsePrecSqrtF32(
     cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
     cl::init(true));
 
+static cl::opt<bool> ForceMinByValParamAlign(
+    "nvptx-force-min-byval-param-align", cl::Hidden,
+    cl::desc("NVPTX Specific: force 4-byte minimal alignment for byval"
+             " params of device functions."),
+    cl::init(false));
+
 int NVPTXTargetLowering::getDivF32Level() const {
   if (UsePrecDivF32.getNumOccurrences() > 0) {
     // If nvptx-prec-div32=N is used on the command-line, always honor it
@@ -4502,16 +4508,17 @@ Align NVPTXTargetLowering::getFunctionByValParamAlign(
   if (F)
     ArgAlign = std::max(ArgAlign, getFunctionParamOptimizedAlign(F, ArgTy, DL));
 
-  // Work around a bug in ptxas. When PTX code takes address of
+  // Old ptx versions have a bug. When PTX code takes address of
   // byval parameter with alignment < 4, ptxas generates code to
   // spill argument into memory. Alas on sm_50+ ptxas generates
   // SASS code that fails with misaligned access. To work around
   // the problem, make sure that we align byval parameters by at
-  // least 4.
-  // TODO: this will need to be undone when we get to support multi-TU
-  // device-side compilation as it breaks ABI compatibility with nvcc.
-  // Hopefully ptxas bug is fixed by then.
-  ArgAlign = std::max(ArgAlign, Align(4));
+  // least 4. This bug seems to be fixed at least starting from
+  // ptxas > 9.0.
+  // TODO: remove this after verifying the bug is not reproduced
+  // on non-deprecated ptxas versions.
+  if (ForceMinByValParamAlign)
+    ArgAlign = std::max(ArgAlign, Align(4));
 
   return ArgAlign;
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 1c35740104add..79d8c7122766a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -15,6 +15,7 @@
 #include "NVPTXAliasAnalysis.h"
 #include "NVPTXAllocaHoisting.h"
 #include "NVPTXAtomicLower.h"
+#include "NVPTXCtorDtorLowering.h"
 #include "NVPTXLowerAggrCopies.h"
 #include "NVPTXMachineFunctionInfo.h"
 #include "NVPTXTargetObjectFile.h"
@@ -68,8 +69,10 @@ void initializeGenericToNVVMLegacyPassPass(PassRegistry &);
 void initializeNVPTXAllocaHoistingPass(PassRegistry &);
 void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry&);
 void initializeNVPTXAtomicLowerPass(PassRegistry &);
+void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &);
 void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
 void initializeNVPTXLowerAllocaPass(PassRegistry &);
+void initializeNVPTXCtorDtorLoweringLegacyPass(PassRegistry &);
 void initializeNVPTXLowerArgsPass(PassRegistry &);
 void initializeNVPTXProxyRegErasurePass(PassRegistry &);
 void initializeNVVMIntrRangePass(PassRegistry &);
@@ -95,6 +98,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() {
   initializeNVPTXAtomicLowerPass(PR);
   initializeNVPTXLowerArgsPass(PR);
   initializeNVPTXLowerAllocaPass(PR);
+  initializeNVPTXCtorDtorLoweringLegacyPass(PR);
   initializeNVPTXLowerAggrCopiesPass(PR);
   initializeNVPTXProxyRegErasurePass(PR);
   initializeNVPTXDAGToDAGISelPass(PR);
@@ -249,6 +253,10 @@ void NVPTXTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
   PB.registerPipelineParsingCallback(
       [](StringRef PassName, ModulePassManager &PM,
          ArrayRef<PassBuilder::PipelineElement>) {
+        if (PassName == "nvptx-lower-ctor-dtor") {
+          PM.addPass(NVPTXCtorDtorLoweringPass());
+          return true;
+        }
         if (PassName == "generic-to-nvvm") {
           PM.addPass(GenericToNVVMPass());
           return true;
@@ -369,6 +377,7 @@ void NVPTXPassConfig::addIRPasses() {
   }
 
   addPass(createAtomicExpandPass());
+  addPass(createNVPTXCtorDtorLoweringLegacyPass());
 
   // === LSR and other generic IR passes ===
   TargetPassConfig::addIRPasses();
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 42cf674632f46..3f832df50fa3b 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -282,6 +282,11 @@ struct PPCOperand : public MCParsedAsmOperand {
     return (unsigned) Imm.Val;
   }
 
+  unsigned getFpReg() const {
+    assert(isEvenRegNumber() && "Invalid access!");
+    return (unsigned)(Imm.Val >> 1);
+  }
+
   unsigned getVSReg() const {
     assert(isVSRegNumber() && "Invalid access!");
     return (unsigned) Imm.Val;
@@ -502,6 +507,11 @@ struct PPCOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createReg(FRegs[getReg()]));
   }
 
+  void addRegFpRCOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::createReg(FpRegs[getFpReg()]));
+  }
+
   void addRegVFRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     Inst.addOperand(MCOperand::createReg(VFRegs[getReg()]));
diff --git a/llvm/lib/Target/PowerPC/CMakeLists.txt b/llvm/lib/Target/PowerPC/CMakeLists.txt
index dac5964c67ead..c0d3adc0c585b 100644
--- a/llvm/lib/Target/PowerPC/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/CMakeLists.txt
@@ -66,6 +66,7 @@ add_llvm_target(PowerPCCodeGen
   AsmPrinter
   BinaryFormat
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   MC
diff --git a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 9faad9b742097..3d9a9ccb18659 100644
--- a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -112,6 +112,14 @@ static DecodeStatus DecodeF8RCRegisterClass(MCInst &Inst, uint64_t RegNo,
   return decodeRegisterClass(Inst, RegNo, FRegs);
 }
 
+static DecodeStatus DecodeFpRCRegisterClass(MCInst &Inst, uint64_t RegNo,
+                                            uint64_t Address,
+                                            const MCDisassembler *Decoder) {
+  assert(RegNo <= 30 && "Expecting a register number no more than 30.");
+  assert((RegNo & 1) == 0 && "Expecting an even register number.");
+  return decodeRegisterClass(Inst, RegNo >> 1, FpRegs);
+}
+
 static DecodeStatus DecodeVFRCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const MCDisassembler *Decoder) {
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
index 2fba5ab07d7c5..94b3c2f9abb17 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
@@ -13,6 +13,7 @@ add_llvm_component_library(LLVMPowerPCDesc
 
   LINK_COMPONENTS
   BinaryFormat
+  CodeGenTypes
   MC
   PowerPCInfo
   Support
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 3b9d5bc8f64dc..86ca1386fed9a 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -136,6 +136,12 @@ static inline bool isRunOfOnes64(uint64_t Val, unsigned &MB, unsigned &ME) {
         X##22, X##23, X##24, X##25, X##26, X##27, X##28, X##29, X##30, X##31   \
   }
 
+#define PPC_REGS_EVEN0_30(X)                                                   \
+  {                                                                            \
+    X##0, X##2, X##4, X##6, X##8, X##10, X##12, X##14, X##16, X##18, X##20,    \
+        X##22, X##24, X##26, X##28, X##30                                      \
+  }
+
 #define PPC_REGS0_63(X)                                                        \
   {                                                                            \
     X##0, X##1, X##2, X##3, X##4, X##5, X##6, X##7, X##8, X##9, X##10, X##11,  \
@@ -178,41 +184,34 @@ static inline bool isRunOfOnes64(uint64_t Val, unsigned &MB, unsigned &ME) {
 
 using llvm::MCPhysReg;
 
-#define DEFINE_PPC_REGCLASSES \
-  static const MCPhysReg RRegs[32] = PPC_REGS0_31(PPC::R); \
-  static const MCPhysReg XRegs[32] = PPC_REGS0_31(PPC::X); \
-  static const MCPhysReg FRegs[32] = PPC_REGS0_31(PPC::F); \
-  static const MCPhysReg VSRpRegs[32] = PPC_REGS0_31(PPC::VSRp); \
-  static const MCPhysReg SPERegs[32] = PPC_REGS0_31(PPC::S); \
-  static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF); \
-  static const MCPhysReg VRegs[32] = PPC_REGS0_31(PPC::V); \
-  static const MCPhysReg RRegsNoR0[32] = \
-    PPC_REGS_NO0_31(PPC::ZERO, PPC::R); \
-  static const MCPhysReg XRegsNoX0[32] = \
-    PPC_REGS_NO0_31(PPC::ZERO8, PPC::X); \
-  static const MCPhysReg VSRegs[64] = \
-    PPC_REGS_LO_HI(PPC::VSL, PPC::V); \
-  static const MCPhysReg VSFRegs[64] = \
-    PPC_REGS_LO_HI(PPC::F, PPC::VF); \
-  static const MCPhysReg VSSRegs[64] = \
-    PPC_REGS_LO_HI(PPC::F, PPC::VF); \
-  static const MCPhysReg CRBITRegs[32] = { \
-    PPC::CR0LT, PPC::CR0GT, PPC::CR0EQ, PPC::CR0UN, \
-    PPC::CR1LT, PPC::CR1GT, PPC::CR1EQ, PPC::CR1UN, \
-    PPC::CR2LT, PPC::CR2GT, PPC::CR2EQ, PPC::CR2UN, \
-    PPC::CR3LT, PPC::CR3GT, PPC::CR3EQ, PPC::CR3UN, \
-    PPC::CR4LT, PPC::CR4GT, PPC::CR4EQ, PPC::CR4UN, \
-    PPC::CR5LT, PPC::CR5GT, PPC::CR5EQ, PPC::CR5UN, \
-    PPC::CR6LT, PPC::CR6GT, PPC::CR6EQ, PPC::CR6UN, \
-    PPC::CR7LT, PPC::CR7GT, PPC::CR7EQ, PPC::CR7UN}; \
-  static const MCPhysReg CRRegs[8] = PPC_REGS0_7(PPC::CR); \
-  static const MCPhysReg ACCRegs[8] = PPC_REGS0_7(PPC::ACC); \
-  static const MCPhysReg WACCRegs[8] = PPC_REGS0_7(PPC::WACC); \
-  static const MCPhysReg WACC_HIRegs[8] = PPC_REGS0_7(PPC::WACC_HI); \
-  static const MCPhysReg DMRROWpRegs[32] = PPC_REGS0_31(PPC::DMRROWp); \
-  static const MCPhysReg DMRROWRegs[64] = PPC_REGS0_63(PPC::DMRROW); \
-  static const MCPhysReg DMRRegs[8] = PPC_REGS0_7(PPC::DMR); \
+#define DEFINE_PPC_REGCLASSES                                                  \
+  static const MCPhysReg RRegs[32] = PPC_REGS0_31(PPC::R);                     \
+  static const MCPhysReg XRegs[32] = PPC_REGS0_31(PPC::X);                     \
+  static const MCPhysReg FRegs[32] = PPC_REGS0_31(PPC::F);                     \
+  static const MCPhysReg FpRegs[16] = PPC_REGS_EVEN0_30(PPC::Fpair);           \
+  static const MCPhysReg VSRpRegs[32] = PPC_REGS0_31(PPC::VSRp);               \
+  static const MCPhysReg SPERegs[32] = PPC_REGS0_31(PPC::S);                   \
+  static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF);                   \
+  static const MCPhysReg VRegs[32] = PPC_REGS0_31(PPC::V);                     \
+  static const MCPhysReg RRegsNoR0[32] = PPC_REGS_NO0_31(PPC::ZERO, PPC::R);   \
+  static const MCPhysReg XRegsNoX0[32] = PPC_REGS_NO0_31(PPC::ZERO8, PPC::X);  \
+  static const MCPhysReg VSRegs[64] = PPC_REGS_LO_HI(PPC::VSL, PPC::V);        \
+  static const MCPhysReg VSFRegs[64] = PPC_REGS_LO_HI(PPC::F, PPC::VF);        \
+  static const MCPhysReg VSSRegs[64] = PPC_REGS_LO_HI(PPC::F, PPC::VF);        \
+  static const MCPhysReg CRBITRegs[32] = {                                     \
+      PPC::CR0LT, PPC::CR0GT, PPC::CR0EQ, PPC::CR0UN, PPC::CR1LT, PPC::CR1GT,  \
+      PPC::CR1EQ, PPC::CR1UN, PPC::CR2LT, PPC::CR2GT, PPC::CR2EQ, PPC::CR2UN,  \
+      PPC::CR3LT, PPC::CR3GT, PPC::CR3EQ, PPC::CR3UN, PPC::CR4LT, PPC::CR4GT,  \
+      PPC::CR4EQ, PPC::CR4UN, PPC::CR5LT, PPC::CR5GT, PPC::CR5EQ, PPC::CR5UN,  \
+      PPC::CR6LT, PPC::CR6GT, PPC::CR6EQ, PPC::CR6UN, PPC::CR7LT, PPC::CR7GT,  \
+      PPC::CR7EQ, PPC::CR7UN};                                                 \
+  static const MCPhysReg CRRegs[8] = PPC_REGS0_7(PPC::CR);                     \
+  static const MCPhysReg ACCRegs[8] = PPC_REGS0_7(PPC::ACC);                   \
+  static const MCPhysReg WACCRegs[8] = PPC_REGS0_7(PPC::WACC);                 \
+  static const MCPhysReg WACC_HIRegs[8] = PPC_REGS0_7(PPC::WACC_HI);           \
+  static const MCPhysReg DMRROWpRegs[32] = PPC_REGS0_31(PPC::DMRROWp);         \
+  static const MCPhysReg DMRROWRegs[64] = PPC_REGS0_63(PPC::DMRROW);           \
+  static const MCPhysReg DMRRegs[8] = PPC_REGS0_7(PPC::DMR);                   \
   static const MCPhysReg DMRpRegs[4] = PPC_REGS0_3(PPC::DMRp);
 
-
 #endif // LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCMCTARGETDESC_H
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 291c8c40bea02..c517f1b722700 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -32,6 +32,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -53,7 +54,6 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index debb29ad785da..f0dfecfb69582 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -46,6 +46,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -85,7 +86,6 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/KnownBits.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index da5c548782f3a..1ff84fdf4b573 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -18,6 +18,7 @@
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -28,7 +29,6 @@
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Type.h"
-#include "llvm/Support/MachineValueType.h"
 #include <optional>
 #include <utility>
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrDFP.td b/llvm/lib/Target/PowerPC/PPCInstrDFP.td
new file mode 100644
index 0000000000000..be13ed9b69c5b
--- /dev/null
+++ b/llvm/lib/Target/PowerPC/PPCInstrDFP.td
@@ -0,0 +1,28 @@
+//===-- PPCInstrDFP.td - PowerPC Decimal Floating Point ----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the PowerPC Decimal Floating Point (DFP) instructions.
+//
+//===----------------------------------------------------------------------===//
+
+// We provide no scheduling info for the DFP instructions.
+// While they are not pseudo instructions we don't intend on scheduling them.
+let hasNoSchedulingInfo = 1 in {
+defm DADD : XForm_28r<59, 2, (outs f8rc:$RST), (ins f8rc:$RA, f8rc:$RB),
+                      "dadd",  "$RST, $RA, $RB", IIC_FPGeneral, []>;
+
+defm DADDQ : XForm_28r<63, 2, (outs fpairrc:$RST), (ins fpairrc:$RA, fpairrc:$RB),
+                       "daddq",  "$RST, $RA, $RB", IIC_FPGeneral, []>;
+
+defm DSUB : XForm_28r<59, 514, (outs f8rc:$RST), (ins f8rc:$RA, f8rc:$RB),
+                      "dsub",  "$RST, $RA, $RB", IIC_FPGeneral, []>;
+
+defm DSUBQ : XForm_28r<63, 514, (outs fpairrc:$RST), (ins fpairrc:$RA, fpairrc:$RB),
+                       "dsubq",  "$RST, $RA, $RB", IIC_FPGeneral, []>;
+}
+
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index bcd87eb599ef4..8681f3eec38ff 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -3321,6 +3321,7 @@ include "PPCInstrP10.td"
 include "PPCInstrFutureMMA.td"
 include "PPCInstrFuture.td"
 include "PPCInstrMMA.td"
+include "PPCInstrDFP.td"
 
 // Patterns for arithmetic i1 operations.
 def : Pat<(add i1:$a, i1:$b),
diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
index b9cb0a29a9511..f1d1d2f3757f6 100644
--- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -1315,7 +1315,7 @@ static bool eligibleForCompareElimination(MachineBasicBlock &MBB,
     if (isEligibleBB(*Pred1MBB) && isEligibleForMoveCmp(*Pred2MBB)) {
       // We assume Pred1MBB is the BB containing the compare to be merged and
       // Pred2MBB is the BB to which we will append a compare instruction.
-      // Hence we can proceed as is.
+      // Proceed as is if Pred1MBB is different from MBB.
     }
     else if (isEligibleBB(*Pred2MBB) && isEligibleForMoveCmp(*Pred1MBB)) {
       // We need to swap Pred1MBB and Pred2MBB to canonicalize.
@@ -1323,6 +1323,9 @@ static bool eligibleForCompareElimination(MachineBasicBlock &MBB,
     }
     else return false;
 
+    if (Pred1MBB == &MBB)
+      return false;
+
     // Here, Pred2MBB is the BB to which we need to append a compare inst.
     // We cannot move the compare instruction if operands are not available
     // in Pred2MBB (i.e. defined in MBB by an instruction other than PHI).
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
index 781f6255dc72b..2a2fe25989367 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -179,9 +179,12 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
       case 'a':
         if (RegName[1] == 'c' && RegName[2] == 'c')
           return RegName + 3;
-      break;
-      case 'r':
+        break;
       case 'f':
+        if (RegName[1] == 'p')
+          return RegName + 2;
+        LLVM_FALLTHROUGH;
+      case 'r':
       case 'v':
         if (RegName[1] == 's') {
           if (RegName[2] == 'p')
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index ea16392b89a67..a8932a2be2cc8 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -20,6 +20,8 @@ def sub_vsx0 : SubRegIndex<128>;
 def sub_vsx1 : SubRegIndex<128, 128>;
 def sub_gp8_x0 : SubRegIndex<64>;
 def sub_gp8_x1 : SubRegIndex<64, 64>;
+def sub_fp0 : SubRegIndex<64>;
+def sub_fp1 : SubRegIndex<64, 64>;
 }
 
 
@@ -58,6 +60,15 @@ class FPR<bits<5> num, string n> : PPCReg<n> {
   let HWEncoding{4-0} = num;
 }
 
+// FPPair - A pair of 64-bit floating-point registers.
+class FPPair<string n, bits<5> EvenIndex> : PPCReg<n> {
+  assert !eq(EvenIndex{0}, 0), "Index should be even.";
+  let HWEncoding{4-0} = EvenIndex;
+  let SubRegs = [!cast<FPR>("F"#EvenIndex), !cast<FPR>("F"#!add(EvenIndex, 1))];
+  let DwarfNumbers = [-1, -1];
+  let SubRegIndices = [sub_fp0, sub_fp1];
+}
+
 // VF - One of the 32 64-bit floating-point subregisters of the vector
 // registers (used by VSX).
 class VF<bits<5> num, string n> : PPCReg<n> {
@@ -136,6 +147,11 @@ foreach Index = 0-31 in {
                 DwarfRegNum<[!add(Index, 32), !add(Index, 32)]>;
 }
 
+// Floating-point pair registers
+foreach Index = { 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 } in {
+  def Fpair#Index : FPPair<"fp"#Index, Index>;
+}
+
 // 64-bit Floating-point subregisters of Altivec registers
 // Note: the register names are v0-v31 or vs32-vs63 depending on the use.
 //       Custom C++ code is used to produce the correct name and encoding.
@@ -349,6 +365,21 @@ def F8RC : RegisterClass<"PPC", [f64], 64, (add (sequence "F%u", 0, 13),
                                                 (sequence "F%u", 31, 14))>;
 def F4RC : RegisterClass<"PPC", [f32], 32, (add F8RC)>;
 
+// Floating point pair registers.
+// Note that the type used for this register class is ppcf128. This is not
+// completely correct. However, since we are not pattern matching any
+// instructions for these registers and we are not register allocating or
+// scheduling any of these instructions it should be safe to do this.
+// The reason we didn't use the correct type (Decimal Floating Point) is that
+// at the time of this implementation the correct type was not available.
+def FpRC :
+  RegisterClass<"PPC", [ppcf128], 128,
+                (add Fpair0, Fpair2, Fpair4, Fpair6, Fpair8, Fpair10, Fpair12,
+                     Fpair14, Fpair16, Fpair18, Fpair20, Fpair22, Fpair24,
+                     Fpair26, Fpair28, Fpair30)> {
+  let Size = 128;
+}
+
 def VRRC : RegisterClass<"PPC",
                          [v16i8,v8i16,v4i32,v2i64,v1i128,v4f32,v2f64, f128],
                          128,
@@ -504,6 +535,12 @@ def PPCRegF4RCAsmOperand : AsmOperandClass {
 def f4rc : RegisterOperand<F4RC> {
   let ParserMatchClass = PPCRegF4RCAsmOperand;
 }
+def PPCRegFpRCAsmOperand : AsmOperandClass {
+  let Name = "RegFpRC"; let PredicateMethod = "isEvenRegNumber";
+}
+def fpairrc : RegisterOperand<FpRC> {
+  let ParserMatchClass = PPCRegFpRCAsmOperand;
+}
 def PPCRegVRRCAsmOperand : AsmOperandClass {
   let Name = "RegVRRC"; let PredicateMethod = "isRegNumber";
 }
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index df7947f2d8818..4731892ca99c0 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -85,7 +85,10 @@ class RISCVAsmParser : public MCTargetAsmParser {
   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
 
   bool generateImmOutOfRangeError(OperandVector &Operands, uint64_t ErrorInfo,
-                                  int64_t Lower, int64_t Upper, Twine Msg);
+                                  int64_t Lower, int64_t Upper,
+                                  const Twine &Msg);
+  bool generateImmOutOfRangeError(SMLoc ErrorLoc, int64_t Lower, int64_t Upper,
+                                  const Twine &Msg);
 
   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                OperandVector &Operands, MCStreamer &Out,
@@ -593,6 +596,16 @@ struct RISCVOperand final : public MCParsedAsmOperand {
   bool isUImm7() const { return IsUImm<7>(); }
   bool isUImm8() const { return IsUImm<8>(); }
 
+  bool isUImm8GE32() const {
+    int64_t Imm;
+    RISCVMCExpr::VariantKind VK = RISCVMCExpr::VK_RISCV_None;
+    if (!isImm())
+      return false;
+    bool IsConstantImm = evaluateConstantImm(getImm(), Imm, VK);
+    return IsConstantImm && isUInt<8>(Imm) && Imm >= 32 &&
+           VK == RISCVMCExpr::VK_RISCV_None;
+  }
+
   bool isRnumArg() const {
     int64_t Imm;
     RISCVMCExpr::VariantKind VK = RISCVMCExpr::VK_RISCV_None;
@@ -1166,11 +1179,17 @@ unsigned RISCVAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
   return Match_Success;
 }
 
+bool RISCVAsmParser::generateImmOutOfRangeError(
+    SMLoc ErrorLoc, int64_t Lower, int64_t Upper,
+    const Twine &Msg = "immediate must be an integer in the range") {
+  return Error(ErrorLoc, Msg + " [" + Twine(Lower) + ", " + Twine(Upper) + "]");
+}
+
 bool RISCVAsmParser::generateImmOutOfRangeError(
     OperandVector &Operands, uint64_t ErrorInfo, int64_t Lower, int64_t Upper,
-    Twine Msg = "immediate must be an integer in the range") {
+    const Twine &Msg = "immediate must be an integer in the range") {
   SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
-  return Error(ErrorLoc, Msg + " [" + Twine(Lower) + ", " + Twine(Upper) + "]");
+  return generateImmOutOfRangeError(ErrorLoc, Lower, Upper, Msg);
 }
 
 bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
@@ -1282,6 +1301,8 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 7) - 1);
   case Match_InvalidUImm8:
     return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 8) - 1);
+  case Match_InvalidUImm8GE32:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, 32, (1 << 8) - 1);
   case Match_InvalidSImm5:
     return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 4),
                                       (1 << 4) - 1);
@@ -1562,8 +1583,9 @@ RISCVAsmParser::parseInsnDirectiveOpcode(OperandVector &Operands) {
     break;
   }
 
-  Error(S, "opcode must be a valid opcode name or an immediate in the range "
-           "[0, 127]");
+  generateImmOutOfRangeError(S, 0, 127,
+                             "opcode must be a valid opcode name or an "
+                             "immediate in the range");
   return MatchOperand_ParseFail;
 }
 
@@ -1623,8 +1645,9 @@ RISCVAsmParser::parseInsnCDirectiveOpcode(OperandVector &Operands) {
   }
   }
 
-  Error(S, "opcode must be a valid opcode name or an immediate in the range "
-           "[0, 2]");
+  generateImmOutOfRangeError(S, 0, 2,
+                             "opcode must be a valid opcode name or an "
+                             "immediate in the range");
   return MatchOperand_ParseFail;
 }
 
@@ -1659,8 +1682,7 @@ RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
       }
     }
 
-    Twine Msg = "immediate must be an integer in the range";
-    Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 12) - 1) + "]");
+    generateImmOutOfRangeError(S, 0, (1 << 12) - 1);
     return MatchOperand_ParseFail;
   }
   case AsmToken::Identifier: {
@@ -1687,15 +1709,14 @@ RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
       return MatchOperand_Success;
     }
 
-    Twine Msg = "operand must be a valid system register name "
-                "or an integer in the range";
-    Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 12) - 1) + "]");
+    generateImmOutOfRangeError(S, 0, (1 << 12) - 1,
+                               "operand must be a valid system register name "
+                               "or an integer in the range");
     return MatchOperand_ParseFail;
   }
   case AsmToken::Percent: {
     // Discard operand with modifier.
-    Twine Msg = "immediate must be an integer in the range";
-    Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 12) - 1) + "]");
+    generateImmOutOfRangeError(S, 0, (1 << 12) - 1);
     return MatchOperand_ParseFail;
   }
   }
@@ -1793,12 +1814,8 @@ RISCVAsmParser::parseOperandWithModifier(OperandVector &Operands) {
   SMLoc S = getLoc();
   SMLoc E;
 
-  if (getLexer().getKind() != AsmToken::Percent) {
-    Error(getLoc(), "expected '%' for operand modifier");
+  if (parseToken(AsmToken::Percent, "expected '%' for operand modifier"))
     return MatchOperand_ParseFail;
-  }
-
-  getParser().Lex(); // Eat '%'
 
   if (getLexer().getKind() != AsmToken::Identifier) {
     Error(getLoc(), "expected valid identifier for operand modifier");
@@ -1812,11 +1829,8 @@ RISCVAsmParser::parseOperandWithModifier(OperandVector &Operands) {
   }
 
   getParser().Lex(); // Eat the identifier
-  if (getLexer().getKind() != AsmToken::LParen) {
-    Error(getLoc(), "expected '('");
+  if (parseToken(AsmToken::LParen, "expected '('"))
     return MatchOperand_ParseFail;
-  }
-  getParser().Lex(); // Eat '('
 
   const MCExpr *SubExpr;
   if (getParser().parseParenExpression(SubExpr, E)) {
@@ -2150,12 +2164,8 @@ OperandMatchResultTy RISCVAsmParser::parseFenceArg(OperandVector &Operands) {
 
 OperandMatchResultTy
 RISCVAsmParser::parseMemOpBaseReg(OperandVector &Operands) {
-  if (getLexer().isNot(AsmToken::LParen)) {
-    Error(getLoc(), "expected '('");
+  if (parseToken(AsmToken::LParen, "expected '('"))
     return MatchOperand_ParseFail;
-  }
-
-  getParser().Lex(); // Eat '('
   Operands.push_back(RISCVOperand::createToken("(", getLoc()));
 
   if (parseRegister(Operands) != MatchOperand_Success) {
@@ -2163,12 +2173,8 @@ RISCVAsmParser::parseMemOpBaseReg(OperandVector &Operands) {
     return MatchOperand_ParseFail;
   }
 
-  if (getLexer().isNot(AsmToken::RParen)) {
-    Error(getLoc(), "expected ')'");
+  if (parseToken(AsmToken::RParen, "expected ')'"))
     return MatchOperand_ParseFail;
-  }
-
-  getParser().Lex(); // Eat ')'
   Operands.push_back(RISCVOperand::createToken(")", getLoc()));
 
   return MatchOperand_Success;
@@ -2214,23 +2220,18 @@ RISCVAsmParser::parseZeroOffsetMemOp(OperandVector &Operands) {
                                 ImmStart, ImmEnd, isRV64());
   }
 
-  if (getLexer().isNot(AsmToken::LParen)) {
-    Error(getLoc(), OptionalImmOp ? "expected '(' after optional integer offset"
-                                  : "expected '(' or optional integer offset");
+  if (parseToken(AsmToken::LParen,
+                 OptionalImmOp ? "expected '(' after optional integer offset"
+                               : "expected '(' or optional integer offset"))
     return MatchOperand_ParseFail;
-  }
-  getParser().Lex(); // Eat '('
 
   if (parseRegister(Operands) != MatchOperand_Success) {
     Error(getLoc(), "expected register");
     return MatchOperand_ParseFail;
   }
 
-  if (getLexer().isNot(AsmToken::RParen)) {
-    Error(getLoc(), "expected ')'");
+  if (parseToken(AsmToken::RParen, "expected ')'"))
     return MatchOperand_ParseFail;
-  }
-  getParser().Lex(); // Eat ')'
 
   // Deferred Handling of non-zero offsets. This makes the error messages nicer.
   if (OptionalImmOp && !OptionalImmOp->isImmZero()) {
@@ -2313,13 +2314,10 @@ bool RISCVAsmParser::ParseInstruction(ParseInstructionInfo &Info,
       return true;
   }
 
-  if (getLexer().isNot(AsmToken::EndOfStatement)) {
-    SMLoc Loc = getLexer().getLoc();
+  if (getParser().parseEOL("unexpected token")) {
     getParser().eatToEndOfStatement();
-    return Error(Loc, "unexpected token");
+    return true;
   }
-
-  getParser().Lex(); // Consume the EndOfStatement.
   return false;
 }
 
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index a005a2fb57758..5ef191c5eba1d 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -26,7 +26,6 @@ add_llvm_target(RISCVCodeGen
   RISCVExpandPseudoInsts.cpp
   RISCVFrameLowering.cpp
   RISCVGatherScatterLowering.cpp
-  RISCVInsertNTLHInsts.cpp
   RISCVInsertVSETVLI.cpp
   RISCVInstrInfo.cpp
   RISCVISelDAGToDAG.cpp
@@ -52,6 +51,7 @@ add_llvm_target(RISCVCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   IPO
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 2121a0e9feabc..7c7aad05fa929 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -258,8 +258,7 @@ static DecodeStatus decodeVMaskReg(MCInst &Inst, uint64_t RegNo,
 
 // Add implied SP operand for instructions *SP compressed instructions. The SP
 // operand isn't explicitly encoded in the instruction.
-static void addImplySP(MCInst &Inst, int64_t Address,
-                       const MCDisassembler *Decoder) {
+static void addImplySP(MCInst &Inst) {
   if (Inst.getOpcode() == RISCV::C_LWSP || Inst.getOpcode() == RISCV::C_SWSP ||
       Inst.getOpcode() == RISCV::C_LDSP || Inst.getOpcode() == RISCV::C_SDSP ||
       Inst.getOpcode() == RISCV::C_FLWSP ||
@@ -267,11 +266,11 @@ static void addImplySP(MCInst &Inst, int64_t Address,
       Inst.getOpcode() == RISCV::C_FLDSP ||
       Inst.getOpcode() == RISCV::C_FSDSP ||
       Inst.getOpcode() == RISCV::C_ADDI4SPN) {
-    DecodeGPRRegisterClass(Inst, 2, Address, Decoder);
+    Inst.addOperand(MCOperand::createReg(RISCV::X2));
   }
   if (Inst.getOpcode() == RISCV::C_ADDI16SP) {
-    DecodeGPRRegisterClass(Inst, 2, Address, Decoder);
-    DecodeGPRRegisterClass(Inst, 2, Address, Decoder);
+    Inst.addOperand(MCOperand::createReg(RISCV::X2));
+    Inst.addOperand(MCOperand::createReg(RISCV::X2));
   }
 }
 
@@ -280,7 +279,7 @@ static DecodeStatus decodeUImmOperand(MCInst &Inst, uint32_t Imm,
                                       int64_t Address,
                                       const MCDisassembler *Decoder) {
   assert(isUInt<N>(Imm) && "Invalid immediate");
-  addImplySP(Inst, Address, Decoder);
+  addImplySP(Inst);
   Inst.addOperand(MCOperand::createImm(Imm));
   return MCDisassembler::Success;
 }
@@ -299,7 +298,7 @@ static DecodeStatus decodeSImmOperand(MCInst &Inst, uint32_t Imm,
                                       int64_t Address,
                                       const MCDisassembler *Decoder) {
   assert(isUInt<N>(Imm) && "Invalid immediate");
-  addImplySP(Inst, Address, Decoder);
+  addImplySP(Inst);
   // Sign-extend the number in the bottom N bits of Imm
   Inst.addOperand(MCOperand::createImm(SignExtend64<N>(Imm)));
   return MCDisassembler::Success;
diff --git a/llvm/lib/Target/RISCV/MCA/CMakeLists.txt b/llvm/lib/Target/RISCV/MCA/CMakeLists.txt
index 5740faaaa6028..036cd4e0034b2 100644
--- a/llvm/lib/Target/RISCV/MCA/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/MCA/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMRISCVTargetMCA
   RISCVCustomBehaviour.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCA
   MCParser
diff --git a/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp b/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp
index 989fde02592a1..ada5905681d2f 100644
--- a/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp
+++ b/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp
@@ -81,7 +81,7 @@ RISCVInstrumentManager::createInstrument(llvm::StringRef Desc,
                       << '\n');
     return nullptr;
   }
-  if (RISCVLMULInstrument::isDataValid(Data)) {
+  if (!RISCVLMULInstrument::isDataValid(Data)) {
     LLVM_DEBUG(dbgs() << "RVCB: Bad data for instrument kind " << Desc << ": "
                       << Data << '\n');
     return nullptr;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
index aa0664f6faae0..deae80d34593b 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
@@ -36,7 +36,7 @@ namespace RISCVInsnOpcode {
 } // namespace RISCVInsnOpcode
 
 namespace RISCVABI {
-ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits,
+ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
                      StringRef ABIName) {
   auto TargetABI = getTargetABI(ABIName);
   bool IsRV64 = TT.isArch64Bit();
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index f60726ec69078..c2a1596e0940e 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -242,6 +242,7 @@ enum OperandType : unsigned {
   OPERAND_UIMM8_LSB00,
   OPERAND_UIMM8,
   OPERAND_UIMM8_LSB000,
+  OPERAND_UIMM8_GE32,
   OPERAND_UIMM9_LSB000,
   OPERAND_UIMM10_LSB00_NONZERO,
   OPERAND_UIMM12,
@@ -410,7 +411,7 @@ enum ABI {
 
 // Returns the target ABI, or else a StringError if the requested ABIName is
 // not supported for the given TT and FeatureBits combination.
-ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits,
+ABI computeTargetABI(const Triple &TT, const FeatureBitset &FeatureBits,
                      StringRef ABIName);
 
 ABI getTargetABI(StringRef ABIName);
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index e0cf1cd3de6e4..f4f378ebbacb1 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -62,9 +62,6 @@ void initializeRISCVPreRAExpandPseudoPass(PassRegistry &);
 FunctionPass *createRISCVExpandAtomicPseudoPass();
 void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
 
-FunctionPass *createRISCVInsertNTLHInstsPass();
-void initializeRISCVInsertNTLHInstsPass(PassRegistry &);
-
 FunctionPass *createRISCVInsertVSETVLIPass();
 void initializeRISCVInsertVSETVLIPass(PassRegistry &);
 
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index efb45ea5ac01e..30aed87bb67db 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -85,6 +85,8 @@ class RISCVAsmPrinter : public AsmPrinter {
 
 private:
   void emitAttributes();
+
+  void emitNTLHint(const MachineInstr *MI);
 };
 }
 
@@ -100,10 +102,44 @@ void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
 // instructions) auto-generated.
 #include "RISCVGenMCPseudoLowering.inc"
 
+// If the target supports Zihintnthl and the instruction has a nontemporal
+// MachineMemOperand, emit an NTLH hint instruction before it.
+void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
+  if (!STI->hasStdExtZihintntl())
+    return;
+
+  if (MI->memoperands_empty())
+    return;
+
+  MachineMemOperand *MMO = *(MI->memoperands_begin());
+  if (!MMO->isNonTemporal())
+    return;
+
+  unsigned NontemporalMode = 0;
+  if (MMO->getFlags() & MONontemporalBit0)
+    NontemporalMode += 0b1;
+  if (MMO->getFlags() & MONontemporalBit1)
+    NontemporalMode += 0b10;
+
+  MCInst Hint;
+  if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs())
+    Hint.setOpcode(RISCV::C_ADD_HINT);
+  else
+    Hint.setOpcode(RISCV::ADD);
+
+  Hint.addOperand(MCOperand::createReg(RISCV::X0));
+  Hint.addOperand(MCOperand::createReg(RISCV::X0));
+  Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode));
+
+  EmitToStreamer(*OutStreamer, Hint);
+}
+
 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
   RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
                                         getSubtargetInfo().getFeatureBits());
 
+  emitNTLHint(MI);
+
   // Do any auto-generated pseudo lowerings.
   if (emitPseudoExpansionLowering(*OutStreamer, MI))
     return;
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index cc17184ba0168..ca17c68075520 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -86,6 +86,16 @@ def HasStdExtZifencei : Predicate<"Subtarget->hasStdExtZifencei()">,
                                    AssemblerPredicate<(all_of FeatureStdExtZifencei),
                                    "'Zifencei' (fence.i)">;
 
+def FeatureStdExtZicntr
+    : SubtargetFeature<"zicntr", "HasStdExtZicntr", "true",
+                       "'zicntr' (Base Counters and Timers)",
+                       [FeatureStdExtZicsr]>;
+
+def FeatureStdExtZihpm
+    : SubtargetFeature<"zihpm", "HasStdExtZihpm", "true",
+                       "'zihpm' (Hardware Performance Counters)",
+                       [FeatureStdExtZicsr]>;
+
 def FeatureStdExtZfhmin
     : SubtargetFeature<"zfhmin", "HasStdExtZfhmin", "true",
                        "'Zfhmin' (Half-Precision Floating-Point Minimal)",
@@ -351,6 +361,14 @@ def HasStdExtCOrZcf
                                    "'C' (Compressed Instructions) or "
                                    "'Zcf' (Compressed Single-Precision Floating-Point Instructions)">;
 
+def FeatureStdExtZcmt
+    : SubtargetFeature<"experimental-zcmt", "HasStdExtZcmt", "true",
+                       "'Zcmt' (table jump instuctions for code-size reduction)", 
+                       [FeatureStdExtZca, FeatureStdExtZicsr]>; 
+def HasStdExtZcmt : Predicate<"Subtarget->hasStdExtZcmt()">,
+                           AssemblerPredicate<(all_of FeatureStdExtZcmt),
+                           "'Zcmt' (table jump instuctions for code-size reduction)">;
+
 def FeatureNoRVCHints
     : SubtargetFeature<"no-rvc-hints", "EnableRVCHintInstrs", "false",
                        "Disable RVC Hint Instructions.">;
@@ -566,6 +584,19 @@ def HasStdExtZicond : Predicate<"Subtarget->hasStdExtZicond()">,
                                 AssemblerPredicate<(all_of FeatureStdExtZicond),
                                 "'Zicond' (Integer Conditional Operations)">;
 
+def FeatureStdExtSmaia
+    : SubtargetFeature<"experimental-smaia", "HasStdExtSmaia", "true",
+                       "'Smaia' (Smaia encompasses all added CSRs and all "
+                       "modifications to interrupt response behavior that the "
+                       "AIA specifies for a hart, over all privilege levels.)",
+                       []>;
+
+def FeatureStdExtSsaia
+    : SubtargetFeature<"experimental-ssaia", "HasStdExtSsaia", "true",
+                       "'Ssaia' (Ssaia is essentially the same as Smaia except "
+                       "excluding the machine-level CSRs and behavior not "
+                       "directly visible to supervisor level.)", []>;
+
 //===----------------------------------------------------------------------===//
 // Vendor extensions
 //===----------------------------------------------------------------------===//
@@ -707,6 +738,11 @@ def FeatureUnalignedScalarMem
                       "true", "Has reasonably performant unaligned scalar "
                       "loads and stores">;
 
+def FeatureUnalignedVectorMem
+   : SubtargetFeature<"unaligned-vector-mem", "EnableUnalignedVectorMem",
+                      "true", "Has reasonably performant unaligned vector "
+                      "loads and stores">;
+
 def TuneNoOptimizedZeroStrideLoad
    : SubtargetFeature<"no-optimized-zero-stride-load", "HasOptimizedZeroStrideLoad",
                       "false", "Hasn't optimized (perform fewer memory operations)"
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 790b50d6163c4..ebb60f83addac 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -52,18 +52,18 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
   bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
   int64_t SlotSize = STI.getXLen() / 8;
   // Store return address to shadow call stack
-  // s[w|d]  ra, 0(gp)
   // addi    gp, gp, [4|8]
-  BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
-      .addReg(RAReg)
-      .addReg(SCSPReg)
-      .addImm(0)
-      .setMIFlag(MachineInstr::FrameSetup);
+  // s[w|d]  ra, -[4|8](gp)
   BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
       .addReg(SCSPReg, RegState::Define)
       .addReg(SCSPReg)
       .addImm(SlotSize)
       .setMIFlag(MachineInstr::FrameSetup);
+  BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
+      .addReg(RAReg)
+      .addReg(SCSPReg)
+      .addImm(-SlotSize)
+      .setMIFlag(MachineInstr::FrameSetup);
 
   // Emit a CFI instruction that causes SlotSize to be subtracted from the value
   // of the shadow stack pointer when unwinding past this frame.
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index d40ad25e28757..e844e1b11271b 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -63,8 +63,6 @@ class RISCVGatherScatterLowering : public FunctionPass {
   }
 
 private:
-  bool isLegalTypeAndAlignment(Type *DataType, Value *AlignOp);
-
   bool tryCreateStridedLoadStore(IntrinsicInst *II, Type *DataType, Value *Ptr,
                                  Value *AlignOp);
 
@@ -87,24 +85,6 @@ FunctionPass *llvm::createRISCVGatherScatterLoweringPass() {
   return new RISCVGatherScatterLowering();
 }
 
-bool RISCVGatherScatterLowering::isLegalTypeAndAlignment(Type *DataType,
-                                                         Value *AlignOp) {
-  Type *ScalarType = DataType->getScalarType();
-  if (!TLI->isLegalElementTypeForRVV(ScalarType))
-    return false;
-
-  MaybeAlign MA = cast<ConstantInt>(AlignOp)->getMaybeAlignValue();
-  if (MA && MA->value() < DL->getTypeStoreSize(ScalarType).getFixedValue())
-    return false;
-
-  // FIXME: Let the backend type legalize by splitting/widening?
-  EVT DataVT = TLI->getValueType(*DL, DataType);
-  if (!TLI->isTypeLegal(DataVT))
-    return false;
-
-  return true;
-}
-
 // TODO: Should we consider the mask when looking for a stride?
 static std::pair<Value *, Value *> matchStridedConstant(Constant *StartC) {
   if (!isa<FixedVectorType>(StartC->getType()))
@@ -464,7 +444,13 @@ bool RISCVGatherScatterLowering::tryCreateStridedLoadStore(IntrinsicInst *II,
                                                            Value *Ptr,
                                                            Value *AlignOp) {
   // Make sure the operation will be supported by the backend.
-  if (!isLegalTypeAndAlignment(DataType, AlignOp))
+  MaybeAlign MA = cast<ConstantInt>(AlignOp)->getMaybeAlignValue();
+  EVT DataTypeVT = TLI->getValueType(*DL, DataType);
+  if (!MA || !TLI->isLegalStridedLoadStore(DataTypeVT, *MA))
+    return false;
+
+  // FIXME: Let the backend type legalize by splitting/widening?
+  if (!TLI->isTypeLegal(DataTypeVT))
     return false;
 
   // Pointer should be a GEP.
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 8c303a48b98eb..b77c8e2c7a316 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -884,13 +884,17 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       Opc = RISCV::FMV_H_X;
       break;
     case MVT::f32:
-      Opc = RISCV::FMV_W_X;
+      Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
       break;
     case MVT::f64:
       // For RV32, we can't move from a GPR, we need to convert instead. This
       // should only happen for +0.0 and -0.0.
       assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
-      Opc = Subtarget->is64Bit() ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
+      bool HasZdinx = Subtarget->hasStdExtZdinx();
+      if (Subtarget->is64Bit())
+        Opc = HasZdinx ? RISCV::COPY : RISCV::FMV_D_X;
+      else
+        Opc = RISCV::FCVT_D_W;
       break;
     }
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index d23d4962eb833..ae147aa434baf 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -120,6 +120,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
   if (Subtarget.hasStdExtD())
     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
+  if (Subtarget.hasStdExtZfinx())
+    addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
+  if (Subtarget.hasStdExtZdinx()) {
+    if (Subtarget.is64Bit())
+      addRegisterClass(MVT::f64, &RISCV::GPRF64RegClass);
+  }
 
   static const MVT::SimpleValueType BoolVecVTs[] = {
       MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
@@ -359,6 +365,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(FPRndMode, MVT::f16,
                          Subtarget.hasStdExtZfa() ? Legal : Custom);
       setOperationAction(ISD::SELECT, MVT::f16, Custom);
+      setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
     } else {
       static const unsigned ZfhminPromoteOps[] = {
           ISD::FMINNUM,      ISD::FMAXNUM,       ISD::FADD,
@@ -406,7 +413,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FPOWI, MVT::i32, Custom);
   }
 
-  if (Subtarget.hasStdExtF()) {
+  if (Subtarget.hasStdExtFOrZfinx()) {
     setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
     setOperationAction(FPRndMode, MVT::f32,
                        Subtarget.hasStdExtZfa() ? Legal : Custom);
@@ -417,15 +424,16 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(FPOpToExpand, MVT::f32, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
+    setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
 
     if (Subtarget.hasStdExtZfa())
       setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
   }
 
-  if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
+  if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
 
-  if (Subtarget.hasStdExtD()) {
+  if (Subtarget.hasStdExtDOrZdinx()) {
     setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
 
     if (Subtarget.hasStdExtZfa()) {
@@ -450,6 +458,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(FPOpToExpand, MVT::f64, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
+    setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
   }
 
   if (Subtarget.is64Bit()) {
@@ -459,7 +468,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::LROUND, MVT::i32, Custom);
   }
 
-  if (Subtarget.hasStdExtF()) {
+  if (Subtarget.hasStdExtFOrZfinx()) {
     setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
                        Custom);
 
@@ -632,6 +641,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
       setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
 
+      setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, VT, Custom);
+
       setOperationPromotedToType(
           ISD::VECTOR_SPLICE, VT,
           MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
@@ -817,7 +828,11 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                           ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
                          VT, Legal);
-      setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, VT, Custom);
+      setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
+                          ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
+                          ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
+                          ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
+                         VT, Custom);
     };
 
     // Sets common extload/truncstore actions on RVV floating-point vector
@@ -887,8 +902,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         setOperationAction(ISD::SETCC, VT, Custom);
 
-        setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, VT,
-                           Legal);
+        if (VT.getVectorElementType() == MVT::i1)
+          setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, VT,
+                             Custom);
 
         setOperationAction(ISD::SELECT, VT, Custom);
 
@@ -919,6 +935,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                            Custom);
 
+        setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
+
         // Operations below are different for between masks and other vectors.
         if (VT.getVectorElementType() == MVT::i1) {
           setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND,
@@ -941,8 +959,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
           setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
         }
 
-        setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
-
         setOperationAction(
             {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
 
@@ -1047,12 +1063,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
                            Custom);
-        setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB,
-                            ISD::STRICT_FMUL, ISD::STRICT_FDIV,
-                            ISD::STRICT_FSQRT, ISD::STRICT_FMA},
-                           VT, Custom);
-        setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, VT,
-                           Custom);
+        setOperationAction(
+            {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
+             ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
+             ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
+             ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
+             ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
+            VT, Custom);
       }
 
       // Custom-legalize bitcasts from fixed-length vectors to scalar types.
@@ -1060,9 +1077,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                          Custom);
       if (Subtarget.hasStdExtZfhOrZfhmin())
         setOperationAction(ISD::BITCAST, MVT::f16, Custom);
-      if (Subtarget.hasStdExtF())
+      if (Subtarget.hasStdExtFOrZfinx())
         setOperationAction(ISD::BITCAST, MVT::f32, Custom);
-      if (Subtarget.hasStdExtD())
+      if (Subtarget.hasStdExtDOrZdinx())
         setOperationAction(ISD::BITCAST, MVT::f64, Custom);
     }
   }
@@ -1111,7 +1128,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   if (Subtarget.is64Bit())
     setTargetDAGCombine(ISD::SRA);
 
-  if (Subtarget.hasStdExtF())
+  if (Subtarget.hasStdExtFOrZfinx())
     setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM});
 
   if (Subtarget.hasStdExtZbb())
@@ -1124,7 +1141,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine(ISD::BITREVERSE);
   if (Subtarget.hasStdExtZfhOrZfhmin())
     setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
-  if (Subtarget.hasStdExtF())
+  if (Subtarget.hasStdExtFOrZfinx())
     setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                          ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
   if (Subtarget.hasVInstructions())
@@ -1139,6 +1156,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
   setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
+
+  // Disable strict node mutation.
+  IsStrictFPEnabled = true;
 }
 
 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
@@ -1677,6 +1697,8 @@ bool RISCVTargetLowering::canSplatOperand(Instruction *I, int Operand) const {
   case Intrinsic::vp_xor:
   case Intrinsic::vp_fadd:
   case Intrinsic::vp_fmul:
+  case Intrinsic::vp_icmp:
+  case Intrinsic::vp_fcmp:
     // These intrinsics have 'vr' versions.
   case Intrinsic::vp_sub:
   case Intrinsic::vp_fsub:
@@ -1781,9 +1803,9 @@ bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
   if (VT == MVT::f16)
     IsLegalVT = Subtarget.hasStdExtZfhOrZfhmin();
   else if (VT == MVT::f32)
-    IsLegalVT = Subtarget.hasStdExtF();
+    IsLegalVT = Subtarget.hasStdExtFOrZfinx();
   else if (VT == MVT::f64)
-    IsLegalVT = Subtarget.hasStdExtD();
+    IsLegalVT = Subtarget.hasStdExtDOrZdinx();
 
   if (!IsLegalVT)
     return false;
@@ -1845,7 +1867,7 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                       EVT VT) const {
   // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
   // We might still end up using a GPR but that will be decided based on ABI.
-  if (VT == MVT::f16 && Subtarget.hasStdExtF() &&
+  if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
       !Subtarget.hasStdExtZfhOrZfhmin())
     return MVT::f32;
 
@@ -1857,7 +1879,7 @@ unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context
                                                            EVT VT) const {
   // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
   // We might still end up using a GPR but that will be decided based on ABI.
-  if (VT == MVT::f16 && Subtarget.hasStdExtF() &&
+  if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
       !Subtarget.hasStdExtZfhOrZfhmin())
     return 1;
 
@@ -2046,27 +2068,30 @@ bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
          (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
 }
 
-bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
-  if (ScalarTy->isPointerTy())
+bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
+  if (!ScalarTy.isSimple())
+    return false;
+  switch (ScalarTy.getSimpleVT().SimpleTy) {
+  case MVT::iPTR:
     return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
-
-  if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
-      ScalarTy->isIntegerTy(32))
+  case MVT::i8:
+  case MVT::i16:
+  case MVT::i32:
     return true;
-
-  if (ScalarTy->isIntegerTy(64))
+  case MVT::i64:
     return Subtarget.hasVInstructionsI64();
-
-  if (ScalarTy->isHalfTy())
+  case MVT::f16:
     return Subtarget.hasVInstructionsF16();
-  if (ScalarTy->isFloatTy())
+  case MVT::f32:
     return Subtarget.hasVInstructionsF32();
-  if (ScalarTy->isDoubleTy())
+  case MVT::f64:
     return Subtarget.hasVInstructionsF64();
-
-  return false;
+  default:
+    return false;
+  }
 }
 
+
 unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
   return NumRepeatedDivisors;
 }
@@ -2403,18 +2428,23 @@ static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
 static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
   switch (Opc) {
   case ISD::FROUNDEVEN:
+  case ISD::STRICT_FROUNDEVEN:
   case ISD::VP_FROUNDEVEN:
     return RISCVFPRndMode::RNE;
   case ISD::FTRUNC:
+  case ISD::STRICT_FTRUNC:
   case ISD::VP_FROUNDTOZERO:
     return RISCVFPRndMode::RTZ;
   case ISD::FFLOOR:
+  case ISD::STRICT_FFLOOR:
   case ISD::VP_FFLOOR:
     return RISCVFPRndMode::RDN;
   case ISD::FCEIL:
+  case ISD::STRICT_FCEIL:
   case ISD::VP_FCEIL:
     return RISCVFPRndMode::RUP;
   case ISD::FROUND:
+  case ISD::STRICT_FROUND:
   case ISD::VP_FROUND:
     return RISCVFPRndMode::RMM;
   case ISD::FRINT:
@@ -2531,6 +2561,110 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
   return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
 }
 
+// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
+// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
+// qNan and coverting the new source to integer and back to FP.
+static SDValue
+lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
+                                            const RISCVSubtarget &Subtarget) {
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  SDValue Chain = Op.getOperand(0);
+  SDValue Src = Op.getOperand(1);
+
+  MVT ContainerVT = VT;
+  if (VT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
+    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
+  }
+
+  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
+
+  // Freeze the source since we are increasing the number of uses.
+  Src = DAG.getFreeze(Src);
+
+  // Covert sNan to qNan by executing x + x for all unordered elemenet x in Src.
+  MVT MaskVT = Mask.getSimpleValueType();
+  SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
+                                DAG.getVTList(MaskVT, MVT::Other),
+                                {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
+                                 DAG.getUNDEF(MaskVT), Mask, VL});
+  Chain = Unorder.getValue(1);
+  Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
+                    DAG.getVTList(ContainerVT, MVT::Other),
+                    {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
+  Chain = Src.getValue(1);
+
+  // We do the conversion on the absolute value and fix the sign at the end.
+  SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
+
+  // Determine the largest integer that can be represented exactly. This and
+  // values larger than it don't have any fractional bits so don't need to
+  // be converted.
+  const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT);
+  unsigned Precision = APFloat::semanticsPrecision(FltSem);
+  APFloat MaxVal = APFloat(FltSem);
+  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
+                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
+  SDValue MaxValNode =
+      DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
+  SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
+                                    DAG.getUNDEF(ContainerVT), MaxValNode, VL);
+
+  // If abs(Src) was larger than MaxVal or nan, keep it.
+  Mask = DAG.getNode(
+      RISCVISD::SETCC_VL, DL, MaskVT,
+      {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
+
+  // Truncate to integer and convert back to FP.
+  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
+  MVT XLenVT = Subtarget.getXLenVT();
+  SDValue Truncated;
+
+  switch (Op.getOpcode()) {
+  default:
+    llvm_unreachable("Unexpected opcode");
+  case ISD::STRICT_FCEIL:
+  case ISD::STRICT_FFLOOR:
+  case ISD::STRICT_FROUND:
+  case ISD::STRICT_FROUNDEVEN: {
+    RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
+    assert(FRM != RISCVFPRndMode::Invalid);
+    Truncated = DAG.getNode(
+        RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
+        {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
+    break;
+  }
+  case ISD::STRICT_FTRUNC:
+    Truncated =
+        DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
+                    DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
+    break;
+  case ISD::STRICT_FNEARBYINT:
+    Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
+                            DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
+                            Mask, VL);
+    break;
+  }
+  Chain = Truncated.getValue(1);
+
+  // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
+  if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
+    Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
+                            DAG.getVTList(ContainerVT, MVT::Other), Chain,
+                            Truncated, Mask, VL);
+    Chain = Truncated.getValue(1);
+  }
+
+  // Restore the original sign so that -0.0 is preserved.
+  Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
+                          Src, Src, Mask, VL);
+
+  if (VT.isFixedLengthVector())
+    Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
+  return DAG.getMergeValues({Truncated, Chain}, DL);
+}
+
 static SDValue
 lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
@@ -2559,6 +2693,30 @@ lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                      DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
 }
 
+static SDValue
+getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, SDLoc DL,
+              EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask,
+              SDValue VL,
+              unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
+  if (Merge.isUndef())
+    Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
+  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
+  SDValue Ops[] = {Merge, Op, Offset, Mask, VL, PolicyOp};
+  return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
+}
+
+static SDValue
+getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, SDLoc DL,
+            EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask,
+            SDValue VL,
+            unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
+  if (Merge.isUndef())
+    Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
+  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
+  SDValue Ops[] = {Merge, Op, Offset, Mask, VL, PolicyOp};
+  return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
+}
+
 struct VIDSequence {
   int64_t StepNumerator;
   unsigned StepDenominator;
@@ -3009,9 +3167,9 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
 
     ValueCounts.insert(std::make_pair(V, 0));
     unsigned &Count = ValueCounts[V];
-
-    if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
-      NumScalarLoads += !CFP->isExactlyValue(+0.0);
+    if (0 == Count)
+      if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
+        NumScalarLoads += !CFP->isExactlyValue(+0.0);
 
     // Is this value dominant? In case of a tie, prefer the highest element as
     // it's cheaper to insert near the beginning of a vector than it is at the
@@ -3029,6 +3187,7 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   // Don't perform this optimization when optimizing for size, since
   // materializing elements and inserting them tends to cause code bloat.
   if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
+      (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
       ((MostCommonCount > DominantValueCountThreshold) ||
        (ValueCounts.size() <= Log2_32(NumDefElts)))) {
     // Start by splatting the most common element.
@@ -3060,7 +3219,43 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
     return Vec;
   }
 
-  return SDValue();
+  // For constant vectors, use generic constant pool lowering.  Otherwise,
+  // we'd have to materialize constants in GPRs just to move them into the
+  // vector.
+  if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()))
+    return SDValue();
+
+  // We can use a series of vslide1down instructions to move values in GPRs
+  // into the appropriate place in the result vector.  We use slide1down
+  // to avoid the register group overlap constraint of vslide1up.
+  if (VT.isFloatingPoint())
+    // TODO: Use vfslide1down.
+    return SDValue();
+
+  const unsigned Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
+
+  SDValue Vec = DAG.getUNDEF(ContainerVT);
+  unsigned UndefCount = 0;
+  for (const SDValue &V : Op->ops()) {
+    if (V.isUndef()) {
+      UndefCount++;
+      continue;
+    }
+    if (UndefCount) {
+      const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
+      Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
+                          Vec, Offset, Mask, VL, Policy);
+      UndefCount = 0;
+    }
+    Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, ContainerVT,
+                      DAG.getUNDEF(ContainerVT), Vec, V, Mask, VL);
+  }
+  if (UndefCount) {
+    const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
+    Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
+                        Vec, Offset, Mask, VL, Policy);
+  }
+  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
 }
 
 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
@@ -3269,9 +3464,9 @@ static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1,
 
 /// Is this shuffle interleaving contiguous elements from one vector into the
 /// even elements and contiguous elements from another vector into the odd
-/// elements. \p Src1 will contain the element that should be in the first even
-/// element. \p Src2 will contain the element that should be in the first odd
-/// element. These can be the first element in a source or the element half
+/// elements. \p EvenSrc will contain the element that should be in the first
+/// even element. \p OddSrc will contain the element that should be in the first
+/// odd element. These can be the first element in a source or the element half
 /// way through the source.
 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
                                 int &OddSrc, const RISCVSubtarget &Subtarget) {
@@ -3280,7 +3475,8 @@ static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
     return false;
 
   int Size = Mask.size();
-  assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
+  int NumElts = VT.getVectorNumElements();
+  assert(Size == (int)NumElts && "Unexpected mask size");
 
   SmallVector<unsigned, 2> StartIndexes;
   if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
@@ -3293,7 +3489,14 @@ static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
   if (EvenSrc != 0 && OddSrc != 0)
     return false;
 
-  return true;
+  // Subvectors will be subtracted from either at the start of the two input
+  // vectors, or at the start and middle of the first vector if it's an unary
+  // interleave.
+  // In both cases, HalfNumElts will be extracted.
+  // So make sure that EvenSrc/OddSrc are within range.
+  int HalfNumElts = NumElts / 2;
+  return (((EvenSrc % NumElts) + HalfNumElts) <= NumElts) &&
+         (((OddSrc % NumElts) + HalfNumElts) <= NumElts);
 }
 
 /// Match shuffles that concatenate two vectors, rotate the concatenation,
@@ -3421,30 +3624,6 @@ static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src,
   return Res;
 }
 
-static SDValue
-getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, SDLoc DL,
-              EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask,
-              SDValue VL,
-              unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
-  if (Merge.isUndef())
-    Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
-  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
-  SDValue Ops[] = {Merge, Op, Offset, Mask, VL, PolicyOp};
-  return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
-}
-
-static SDValue
-getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, SDLoc DL,
-            EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask,
-            SDValue VL,
-            unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
-  if (Merge.isUndef())
-    Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
-  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
-  SDValue Ops[] = {Merge, Op, Offset, Mask, VL, PolicyOp};
-  return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
-}
-
 // Lower the following shuffle to vslidedown.
 // a)
 // t49: v8i8 = extract_subvector t13, Constant:i64<0>
@@ -3651,6 +3830,17 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
   unsigned NumElts = VT.getVectorNumElements();
   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
 
+  // Promote i1 shuffle to i8 shuffle.
+  if (VT.getVectorElementType() == MVT::i1) {
+    MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
+    V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
+    V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
+                      : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
+    SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
+    return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
+                        ISD::SETNE);
+  }
+
   MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
 
   auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
@@ -3956,6 +4146,10 @@ bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
 
   MVT SVT = VT.getSimpleVT();
 
+  // Not for i1 vectors.
+  if (SVT.getScalarType() == MVT::i1)
+    return false;
+
   int Dummy1, Dummy2;
   return (isElementRotate(Dummy1, Dummy2, M) > 0) ||
          isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
@@ -4160,6 +4354,42 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
   return Op;
 }
 
+static SDValue LowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG,
+                               const RISCVSubtarget &Subtarget) {
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+  auto CNode = cast<ConstantSDNode>(Op.getOperand(1));
+  unsigned Check = CNode->getZExtValue();
+  unsigned TDCMask = 0;
+  if (Check & fcSNan)
+    TDCMask |= RISCV::FPMASK_Signaling_NaN;
+  if (Check & fcQNan)
+    TDCMask |= RISCV::FPMASK_Quiet_NaN;
+  if (Check & fcPosInf)
+    TDCMask |= RISCV::FPMASK_Positive_Infinity;
+  if (Check & fcNegInf)
+    TDCMask |= RISCV::FPMASK_Negative_Infinity;
+  if (Check & fcPosNormal)
+    TDCMask |= RISCV::FPMASK_Positive_Normal;
+  if (Check & fcNegNormal)
+    TDCMask |= RISCV::FPMASK_Negative_Normal;
+  if (Check & fcPosSubnormal)
+    TDCMask |= RISCV::FPMASK_Positive_Subnormal;
+  if (Check & fcNegSubnormal)
+    TDCMask |= RISCV::FPMASK_Negative_Subnormal;
+  if (Check & fcPosZero)
+    TDCMask |= RISCV::FPMASK_Positive_Zero;
+  if (Check & fcNegZero)
+    TDCMask |= RISCV::FPMASK_Negative_Zero;
+
+  SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
+  SDValue FPCLASS = DAG.getNode(RISCVISD::FPCLASS, DL, VT, Op.getOperand(0));
+  SDValue AND = DAG.getNode(ISD::AND, DL, VT, FPCLASS, TDCMaskV);
+  return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, XLenVT),
+                      ISD::CondCode::SETNE);
+}
+
 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                             SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
@@ -4208,7 +4438,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
       return FPConv;
     }
     if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
-        Subtarget.hasStdExtF()) {
+        Subtarget.hasStdExtFOrZfinx()) {
       SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
       SDValue FPConv =
           DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
@@ -4271,6 +4501,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return LowerINTRINSIC_W_CHAIN(Op, DAG);
   case ISD::INTRINSIC_VOID:
     return LowerINTRINSIC_VOID(Op, DAG);
+  case ISD::IS_FPCLASS:
+    return LowerIS_FPCLASS(Op, DAG, Subtarget);
   case ISD::BITREVERSE: {
     MVT VT = Op.getSimpleValueType();
     SDLoc DL(Op);
@@ -4622,7 +4854,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::SELECT_CC: {
     // This occurs because we custom legalize SETGT and SETUGT for setcc. That
     // causes LegalizeDAG to think we need to custom legalize select_cc. Expand
-    // into separate SETCC+SELECT_CC just like LegalizeDAG.
+    // into separate SETCC+SELECT just like LegalizeDAG.
     SDValue Tmp1 = Op.getOperand(0);
     SDValue Tmp2 = Op.getOperand(1);
     SDValue True = Op.getOperand(2);
@@ -4783,6 +5015,14 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::STRICT_FSETCC:
   case ISD::STRICT_FSETCCS:
     return lowerVectorStrictFSetcc(Op, DAG);
+  case ISD::STRICT_FCEIL:
+  case ISD::STRICT_FRINT:
+  case ISD::STRICT_FFLOOR:
+  case ISD::STRICT_FTRUNC:
+  case ISD::STRICT_FNEARBYINT:
+  case ISD::STRICT_FROUND:
+  case ISD::STRICT_FROUNDEVEN:
+    return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
   case ISD::MGATHER:
   case ISD::VP_GATHER:
     return lowerMaskedGather(Op, DAG);
@@ -5108,20 +5348,13 @@ SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
   return Addr;
 }
 
-SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
-  SDValue CondV = Op.getOperand(0);
-  SDValue TrueV = Op.getOperand(1);
-  SDValue FalseV = Op.getOperand(2);
-  SDLoc DL(Op);
-  MVT VT = Op.getSimpleValueType();
-  MVT XLenVT = Subtarget.getXLenVT();
-
-  // Lower vector SELECTs to VSELECTs by splatting the condition.
-  if (VT.isVector()) {
-    MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
-    SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV);
-    return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
-  }
+static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG,
+                                    const RISCVSubtarget &Subtarget) {
+  SDValue CondV = N->getOperand(0);
+  SDValue TrueV = N->getOperand(1);
+  SDValue FalseV = N->getOperand(2);
+  MVT VT = N->getSimpleValueType(0);
+  SDLoc DL(N);
 
   if (!Subtarget.hasShortForwardBranchOpt()) {
     // (select c, -1, y) -> -c | y
@@ -5149,6 +5382,27 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     }
   }
 
+  return SDValue();
+}
+
+SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
+  SDValue CondV = Op.getOperand(0);
+  SDValue TrueV = Op.getOperand(1);
+  SDValue FalseV = Op.getOperand(2);
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  MVT XLenVT = Subtarget.getXLenVT();
+
+  // Lower vector SELECTs to VSELECTs by splatting the condition.
+  if (VT.isVector()) {
+    MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
+    SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV);
+    return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
+  }
+
+  if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget))
+    return V;
+
   // If the condition is not an integer SETCC which operates on XLenVT, we need
   // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
   // (select condv, truev, falsev)
@@ -5924,7 +6178,7 @@ SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
       SDValue Vfirst =
           DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL);
       return DAG.getSetCC(DL, XLenVT, Vfirst, DAG.getConstant(0, DL, XLenVT),
-                          ISD::SETEQ);
+                          ISD::SETNE);
     }
     if (VecVT.isFixedLengthVector()) {
       unsigned NumElts = VecVT.getVectorNumElements();
@@ -6590,7 +6844,7 @@ SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
   return DAG.getNode(BaseOpc, DL, XLenVT, SetCC, Op.getOperand(0));
 }
 
-static bool hasNonZeroAVL(SDValue AVL) {
+static bool isNonZeroAVL(SDValue AVL) {
   auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
   auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
   return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
@@ -6606,7 +6860,7 @@ static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT,
   const MVT VecVT = Vec.getSimpleValueType();
   const MVT M1VT = getLMUL1VT(VecVT);
   const MVT XLenVT = Subtarget.getXLenVT();
-  const bool NonZeroAVL = hasNonZeroAVL(VL);
+  const bool NonZeroAVL = isNonZeroAVL(VL);
 
   // The reduction needs an LMUL1 input; do the splat at either LMUL1
   // or the original VT if fractional.
@@ -7077,9 +7331,9 @@ static SDValue widenVectorOpsToi8(SDValue N, SDLoc &DL, SelectionDAG &DAG) {
   SDValue WideN = DAG.getNode(N.getOpcode(), DL, VTs, WideOps);
   SmallVector<SDValue, 4> TruncVals;
   for (unsigned I = 0; I < NumVals; I++) {
-    TruncVals.push_back(DAG.getNode(ISD::TRUNCATE, DL,
-                                    N->getSimpleValueType(I),
-                                    SDValue(WideN.getNode(), I)));
+    TruncVals.push_back(
+        DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I),
+                     DAG.getConstant(0, DL, WideVT), ISD::SETNE));
   }
 
   if (TruncVals.size() > 1)
@@ -7595,10 +7849,12 @@ SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op,
                                  Op2, OLECCVal);
       SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2,
                                  Op1, OLECCVal);
-      SDValue And = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
-      SDValue OutChain =
-          DAG.getMergeValues({Tmp1.getValue(1), Tmp2.getValue(1)}, DL);
-      return DAG.getMergeValues({And, OutChain}, DL);
+      SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
+                                     Tmp1.getValue(1), Tmp2.getValue(1));
+      // Tmp1 and Tmp2 might be the same node.
+      if (Tmp1 != Tmp2)
+        Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
+      return DAG.getMergeValues({Tmp1, OutChain}, DL);
     }
 
     // Expand (strict_fsetccs x, y, une) to (not (strict_fsetccs x, y, oeq))
@@ -8886,7 +9142,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
       SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
     } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
-               Subtarget.hasStdExtF()) {
+               Subtarget.hasStdExtFOrZfinx()) {
       SDValue FPConv =
           DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
@@ -9120,13 +9376,14 @@ static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
 
   SDValue Extract = N->getOperand(ReduceIdx);
   SDValue Reduce = Extract.getOperand(0);
-  if (!Reduce.hasOneUse())
+  if (!Extract.hasOneUse() || !Reduce.hasOneUse())
     return SDValue();
 
   SDValue ScalarV = Reduce.getOperand(2);
   EVT ScalarVT = ScalarV.getValueType();
   if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
-      ScalarV.getOperand(0)->isUndef())
+      ScalarV.getOperand(0)->isUndef() &&
+      isNullConstant(ScalarV.getOperand(2)))
     ScalarV = ScalarV.getOperand(1);
 
   // Make sure that ScalarV is a splat with VL=1.
@@ -9135,7 +9392,7 @@ static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
       ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
     return SDValue();
 
-  if (!hasNonZeroAVL(ScalarV.getOperand(2)))
+  if (!isNonZeroAVL(ScalarV.getOperand(2)))
     return SDValue();
 
   // Check the scalar of ScalarV is neutral element
@@ -9144,7 +9401,9 @@ static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
                          0))
     return SDValue();
 
-  if (!ScalarV.hasOneUse())
+  // If the AVL is zero, operand 0 will be returned. So it's not safe to fold.
+  // FIXME: We might be able to improve this if operand 0 is undef.
+  if (!isNonZeroAVL(Reduce.getOperand(5)))
     return SDValue();
 
   SDValue NewStart = N->getOperand(1 - ReduceIdx);
@@ -9356,8 +9615,31 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
 }
 
+// Try to turn (add (xor (setcc X, Y), 1) -1) into (neg (setcc X, Y)).
+static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  // RHS should be -1.
+  if (!isAllOnesConstant(N1))
+    return SDValue();
+
+  // Look for an (xor (setcc X, Y), 1).
+  if (N0.getOpcode() != ISD::XOR || !isOneConstant(N0.getOperand(1)) ||
+      N0.getOperand(0).getOpcode() != ISD::SETCC)
+    return SDValue();
+
+  // Emit a negate of the setcc.
+  return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
+                     N0.getOperand(0));
+}
+
 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
+  if (SDValue V = combineAddOfBooleanXor(N, DAG))
+    return V;
   if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
     return V;
   if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
@@ -9573,7 +9855,7 @@ class CmpOpInfo {
   }
 
   // Creates infos about comparison operations CmpOp0 and CmpOp1.
-  // If there is no common operand returns None. Otherwise, returns
+  // If there is no common operand returns std::nullopt. Otherwise, returns
   // correspondence info about comparison operations.
   static std::optional<std::pair<CmpOpInfo, CmpOpInfo>>
   getInfoAbout(SDValue const &CmpOp0, SDValue const &CmpOp1) {
@@ -9596,7 +9878,7 @@ class CmpOpInfo {
   // Returns differ operand.
   SDValue const &getDOp() const { return operator[](DifferPos); }
 
-  // Returns consition code of comparison operation.
+  // Returns condition code of comparison operation.
   ISD::CondCode getCondCode() const { return CCode; }
 };
 } // namespace
@@ -9941,7 +10223,7 @@ struct NodeExtensionHelper {
   }
 
   /// Get or create a value that can feed \p Root with the given extension \p
-  /// SExt. If \p SExt is None, this returns the source of this operand.
+  /// SExt. If \p SExt is std::nullopt, this returns the source of this operand.
   /// \see ::getSource().
   SDValue getOrCreateExtendedOp(const SDNode *Root, SelectionDAG &DAG,
                                 std::optional<bool> SExt) const {
@@ -11238,6 +11520,10 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
   if (!TLI.isTypeLegal(WideVecVT))
     return SDValue();
 
+  // Check that the operation is legal
+  if (!TLI.isLegalStridedLoadStore(WideVecVT, Align))
+    return SDValue();
+
   MVT ContainerVT = TLI.getContainerForFixedLengthVector(WideVecVT);
   SDValue VL =
       getDefaultVLOps(WideVecVT, ContainerVT, DL, DAG, Subtarget).second;
@@ -11265,12 +11551,6 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
       BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
       Align);
 
-  // Can't do the combine if the common alignment isn't naturally aligned with
-  // the new element type
-  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
-                                          DAG.getDataLayout(), WideVecVT, *MMO))
-    return SDValue();
-
   SDValue StridedLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs,
                                                 Ops, WideVecVT, MMO);
   for (SDValue Ld : N->ops())
@@ -12405,7 +12685,9 @@ static bool isSelectPseudo(MachineInstr &MI) {
   case RISCV::Select_GPR_Using_CC_GPR:
   case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
+  case RISCV::Select_FPR32INX_Using_CC_GPR:
   case RISCV::Select_FPR64_Using_CC_GPR:
+  case RISCV::Select_FPR64INX_Using_CC_GPR:
     return true;
   }
 }
@@ -12790,6 +13072,14 @@ static MachineBasicBlock *emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB,
     FSGNJXOpc = RISCV::FSGNJX_S;
     RC = &RISCV::FPR32RegClass;
     break;
+  case RISCV::PseudoFROUND_S_INX:
+    CmpOpc = RISCV::FLT_S_INX;
+    F2IOpc = RISCV::FCVT_W_S_INX;
+    I2FOpc = RISCV::FCVT_S_W_INX;
+    FSGNJOpc = RISCV::FSGNJ_S_INX;
+    FSGNJXOpc = RISCV::FSGNJX_S_INX;
+    RC = &RISCV::GPRF32RegClass;
+    break;
   case RISCV::PseudoFROUND_D:
     assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
     CmpOpc = RISCV::FLT_D;
@@ -12799,6 +13089,15 @@ static MachineBasicBlock *emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB,
     FSGNJXOpc = RISCV::FSGNJX_D;
     RC = &RISCV::FPR64RegClass;
     break;
+  case RISCV::PseudoFROUND_D_INX:
+    assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
+    CmpOpc = RISCV::FLT_D_INX;
+    F2IOpc = RISCV::FCVT_L_D_INX;
+    I2FOpc = RISCV::FCVT_D_L_INX;
+    FSGNJOpc = RISCV::FSGNJ_D_INX;
+    FSGNJXOpc = RISCV::FSGNJX_D_INX;
+    RC = &RISCV::GPRF64RegClass;
+    break;
   }
 
   const BasicBlock *BB = MBB->getBasicBlock();
@@ -12887,7 +13186,9 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case RISCV::Select_GPR_Using_CC_GPR:
   case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
+  case RISCV::Select_FPR32INX_Using_CC_GPR:
   case RISCV::Select_FPR64_Using_CC_GPR:
+  case RISCV::Select_FPR64INX_Using_CC_GPR:
     return emitSelectPseudo(MI, BB, Subtarget);
   case RISCV::BuildPairF64Pseudo:
     return emitBuildPairF64Pseudo(MI, BB, Subtarget);
@@ -12899,12 +13200,20 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     return emitQuietFCMP(MI, BB, RISCV::FLT_H, RISCV::FEQ_H, Subtarget);
   case RISCV::PseudoQuietFLE_S:
     return emitQuietFCMP(MI, BB, RISCV::FLE_S, RISCV::FEQ_S, Subtarget);
+  case RISCV::PseudoQuietFLE_S_INX:
+    return emitQuietFCMP(MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
   case RISCV::PseudoQuietFLT_S:
     return emitQuietFCMP(MI, BB, RISCV::FLT_S, RISCV::FEQ_S, Subtarget);
+  case RISCV::PseudoQuietFLT_S_INX:
+    return emitQuietFCMP(MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
   case RISCV::PseudoQuietFLE_D:
     return emitQuietFCMP(MI, BB, RISCV::FLE_D, RISCV::FEQ_D, Subtarget);
+  case RISCV::PseudoQuietFLE_D_INX:
+    return emitQuietFCMP(MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
   case RISCV::PseudoQuietFLT_D:
     return emitQuietFCMP(MI, BB, RISCV::FLT_D, RISCV::FEQ_D, Subtarget);
+  case RISCV::PseudoQuietFLT_D_INX:
+    return emitQuietFCMP(MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
 
     // =========================================================================
     // VFCVT
@@ -13086,7 +13395,9 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                                      RISCV::PseudoVFCVT_F_X_V_MF4_MASK);
   case RISCV::PseudoFROUND_H:
   case RISCV::PseudoFROUND_S:
+  case RISCV::PseudoFROUND_S_INX:
   case RISCV::PseudoFROUND_D:
+  case RISCV::PseudoFROUND_D_INX:
     return emitFROUND(MI, BB, Subtarget);
   }
 }
@@ -14465,9 +14776,7 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     StringRef Kind =
       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
 
-    if (Kind == "user")
-      RetOpc = RISCVISD::URET_GLUE;
-    else if (Kind == "supervisor")
+    if (Kind == "supervisor")
       RetOpc = RISCVISD::SRET_GLUE;
     else
       RetOpc = RISCVISD::MRET_GLUE;
@@ -14541,7 +14850,6 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case RISCVISD::FIRST_NUMBER:
     break;
   NODE_NAME_CASE(RET_GLUE)
-  NODE_NAME_CASE(URET_GLUE)
   NODE_NAME_CASE(SRET_GLUE)
   NODE_NAME_CASE(MRET_GLUE)
   NODE_NAME_CASE(CALL)
@@ -14581,6 +14889,7 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(STRICT_FCVT_W_RV64)
   NODE_NAME_CASE(STRICT_FCVT_WU_RV64)
   NODE_NAME_CASE(FROUND)
+  NODE_NAME_CASE(FPCLASS)
   NODE_NAME_CASE(READ_CYCLE_WIDE)
   NODE_NAME_CASE(BREV8)
   NODE_NAME_CASE(ORC_B)
@@ -14681,10 +14990,12 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(STRICT_VFNCVT_ROD_VL)
   NODE_NAME_CASE(STRICT_SINT_TO_FP_VL)
   NODE_NAME_CASE(STRICT_UINT_TO_FP_VL)
+  NODE_NAME_CASE(STRICT_VFCVT_RM_X_F_VL)
   NODE_NAME_CASE(STRICT_VFCVT_RTZ_X_F_VL)
   NODE_NAME_CASE(STRICT_VFCVT_RTZ_XU_F_VL)
   NODE_NAME_CASE(STRICT_FSETCC_VL)
   NODE_NAME_CASE(STRICT_FSETCCS_VL)
+  NODE_NAME_CASE(STRICT_VFROUND_NOEXCEPT_VL)
   NODE_NAME_CASE(VWMUL_VL)
   NODE_NAME_CASE(VWMULU_VL)
   NODE_NAME_CASE(VWMULSU_VL)
@@ -15336,9 +15647,9 @@ bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
     return VT.isVector() ? Subtarget.hasVInstructionsF16()
                          : Subtarget.hasStdExtZfh();
   case MVT::f32:
-    return Subtarget.hasStdExtF();
+    return Subtarget.hasStdExtFOrZfinx();
   case MVT::f64:
-    return Subtarget.hasStdExtD();
+    return Subtarget.hasStdExtDOrZdinx();
   default:
     break;
   }
@@ -15449,7 +15760,13 @@ bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
     return true;
   }
 
-  return false;
+  // Note: We lower an unmasked unaligned vector access to an equally sized
+  // e8 element type access.  Given this, we effectively support all unmasked
+  // misaligned accesses.  TODO: Work through the codegen implications of
+  // allowing such accesses to be formed, and considered fast.
+  if (Fast)
+    *Fast = 0;
+  return Subtarget.enableUnalignedVectorMem();
 }
 
 bool RISCVTargetLowering::splitValueIntoRegisterParts(
@@ -15591,12 +15908,14 @@ bool RISCVTargetLowering::isLegalInterleavedAccessType(
     FixedVectorType *VTy, unsigned Factor, const DataLayout &DL) const {
   if (!Subtarget.useRVVForFixedLengthVectors())
     return false;
-  if (!isLegalElementTypeForRVV(VTy->getElementType()))
-    return false;
   EVT VT = getValueType(DL, VTy);
   // Don't lower vlseg/vsseg for fixed length vector types that can't be split.
   if (!isTypeLegal(VT))
     return false;
+
+  if (!isLegalElementTypeForRVV(VT.getScalarType()))
+    return false;
+
   // Sometimes the interleaved access pass picks up splats as interleaves of one
   // element. Don't lower these.
   if (VTy->getNumElements() < 2)
@@ -15610,6 +15929,26 @@ bool RISCVTargetLowering::isLegalInterleavedAccessType(
   return Factor * LMUL <= 8;
 }
 
+bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
+                                                  Align Alignment) const {
+  if (!Subtarget.hasVInstructions())
+    return false;
+
+  // Only support fixed vectors if we know the minimum vector size.
+  if (DataType.isFixedLengthVector() && !Subtarget.useRVVForFixedLengthVectors())
+    return false;
+
+  EVT ScalarType = DataType.getScalarType();
+  if (!isLegalElementTypeForRVV(ScalarType))
+    return false;
+
+  if (!Subtarget.enableUnalignedVectorMem() &&
+      Alignment < ScalarType.getStoreSize())
+    return false;
+
+  return true;
+}
+
 /// Lower an interleaved load into a vlsegN intrinsic.
 ///
 /// E.g. Lower an interleaved load (Factor = 2):
@@ -15618,7 +15957,7 @@ bool RISCVTargetLowering::isLegalInterleavedAccessType(
 /// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7>  ; Extract odd elements
 ///
 /// Into:
-/// %ld2 = { <4 x i32>, <4 x i32> } call llvm.riscv.vlseg.v4i32.p0.i64(
+/// %ld2 = { <4 x i32>, <4 x i32> } call llvm.riscv.seg2.load.v4i32.p0.i64(
 ///                                        %ptr, i64 4)
 /// %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
 /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
@@ -15667,8 +16006,8 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
 /// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
 /// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
 /// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
-/// call void llvm.riscv.vsseg3.v4i32.p0.i64(%sub.v0, %sub.v1, %sub.v2,
-///                                          %ptr, i32 4)
+/// call void llvm.riscv.seg3.store.v4i32.p0.i64(%sub.v0, %sub.v1, %sub.v2,
+///                                              %ptr, i32 4)
 ///
 /// Note that the new shufflevectors will be removed and we'll only generate one
 /// vsseg3 instruction in CodeGen.
@@ -15735,6 +16074,57 @@ RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
   return Reg;
 }
 
+MachineMemOperand::Flags
+RISCVTargetLowering::getTargetMMOFlags(const Instruction &I) const {
+  const MDNode *NontemporalInfo = I.getMetadata(LLVMContext::MD_nontemporal);
+
+  if (NontemporalInfo == nullptr)
+    return MachineMemOperand::MONone;
+
+  // 1 for default value work as __RISCV_NTLH_ALL
+  // 2 -> __RISCV_NTLH_INNERMOST_PRIVATE
+  // 3 -> __RISCV_NTLH_ALL_PRIVATE
+  // 4 -> __RISCV_NTLH_INNERMOST_SHARED
+  // 5 -> __RISCV_NTLH_ALL
+  int NontemporalLevel = 5;
+  const MDNode *RISCVNontemporalInfo =
+      I.getMetadata("riscv-nontemporal-domain");
+  if (RISCVNontemporalInfo != nullptr)
+    NontemporalLevel =
+        cast<ConstantInt>(
+            cast<ConstantAsMetadata>(RISCVNontemporalInfo->getOperand(0))
+                ->getValue())
+            ->getZExtValue();
+
+  assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
+         "RISC-V target doesn't support this non-temporal domain.");
+
+  NontemporalLevel -= 2;
+  MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
+  if (NontemporalLevel & 0b1)
+    Flags |= MONontemporalBit0;
+  if (NontemporalLevel & 0b10)
+    Flags |= MONontemporalBit1;
+
+  return Flags;
+}
+
+MachineMemOperand::Flags
+RISCVTargetLowering::getTargetMMOFlags(const MemSDNode &Node) const {
+
+  MachineMemOperand::Flags NodeFlags = Node.getMemOperand()->getFlags();
+  MachineMemOperand::Flags TargetFlags = MachineMemOperand::MONone;
+  TargetFlags |= (NodeFlags & MONontemporalBit0);
+  TargetFlags |= (NodeFlags & MONontemporalBit1);
+
+  return TargetFlags;
+}
+
+bool RISCVTargetLowering::areTwoSDNodeTargetMMOFlagsMergeable(
+    const MemSDNode &NodeX, const MemSDNode &NodeY) const {
+  return getTargetMMOFlags(NodeX) == getTargetMMOFlags(NodeY);
+}
+
 namespace llvm::RISCVVIntrinsicsTable {
 
 #define GET_RISCVVIntrinsicsTable_IMPL
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index de0e970108b81..c020b27edf3e3 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -28,7 +28,6 @@ namespace RISCVISD {
 enum NodeType : unsigned {
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
   RET_GLUE,
-  URET_GLUE,
   SRET_GLUE,
   MRET_GLUE,
   CALL,
@@ -119,6 +118,7 @@ enum NodeType : unsigned {
   // inserter.
   FROUND,
 
+  FPCLASS,
   // READ_CYCLE_WIDE - A read of the 64-bit cycle CSR on a 32-bit target
   // (returns (Lo, Hi)). It takes a chain operand.
   READ_CYCLE_WIDE,
@@ -344,10 +344,12 @@ enum NodeType : unsigned {
   STRICT_VFNCVT_ROD_VL,
   STRICT_SINT_TO_FP_VL,
   STRICT_UINT_TO_FP_VL,
+  STRICT_VFCVT_RM_X_F_VL,
   STRICT_VFCVT_RTZ_X_F_VL,
   STRICT_VFCVT_RTZ_XU_F_VL,
   STRICT_FSETCC_VL,
   STRICT_FSETCCS_VL,
+  STRICT_VFROUND_NOEXCEPT_VL,
 
   // WARNING: Do not add anything in the end unless you want the node to
   // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
@@ -482,6 +484,16 @@ class RISCVTargetLowering : public TargetLowering {
   // This method returns the name of a target specific DAG node.
   const char *getTargetNodeName(unsigned Opcode) const override;
 
+  MachineMemOperand::Flags
+  getTargetMMOFlags(const Instruction &I) const override;
+
+  MachineMemOperand::Flags
+  getTargetMMOFlags(const MemSDNode &Node) const override;
+
+  bool
+  areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX,
+                                      const MemSDNode &NodeY) const override;
+
   ConstraintType getConstraintType(StringRef Constraint) const override;
 
   unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override;
@@ -654,7 +666,7 @@ class RISCVTargetLowering : public TargetLowering {
 
   bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const override;
 
-  bool isLegalElementTypeForRVV(Type *ScalarTy) const;
+  bool isLegalElementTypeForRVV(EVT ScalarTy) const;
 
   bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
 
@@ -692,6 +704,10 @@ class RISCVTargetLowering : public TargetLowering {
   bool isLegalInterleavedAccessType(FixedVectorType *, unsigned Factor,
                                     const DataLayout &) const;
 
+  /// Return true if a stride load store of the given result type and
+  /// alignment is legal.
+  bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const;
+
   unsigned getMaxSupportedInterleaveFactor() const override { return 8; }
 
   bool lowerInterleavedLoad(LoadInst *LI,
diff --git a/llvm/lib/Target/RISCV/RISCVInsertNTLHInsts.cpp b/llvm/lib/Target/RISCV/RISCVInsertNTLHInsts.cpp
deleted file mode 100644
index 209438a6165bb..0000000000000
--- a/llvm/lib/Target/RISCV/RISCVInsertNTLHInsts.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-//===-- RISCVInsertNTLHInsts.cpp - Insert NTLH extension instrution -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a function pass that inserts non-temporal hint
-// instructions where needed.
-//
-// It checks the MachineMemOperand of all MachineInstr.
-// If the instruction has a MachineMemOperand and isNontemporal is true,
-// then ntlh instruction is inserted before it.
-//
-//===----------------------------------------------------------------------===//
-
-#include "RISCV.h"
-#include "RISCVInstrInfo.h"
-#include "RISCVTargetMachine.h"
-
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-
-using namespace llvm;
-
-#define RISCV_INSERT_NTLH_INSTS_NAME "RISC-V insert NTLH instruction pass"
-
-namespace {
-
-class RISCVInsertNTLHInsts : public MachineFunctionPass {
-public:
-  const RISCVInstrInfo *TII;
-  static char ID;
-
-  RISCVInsertNTLHInsts() : MachineFunctionPass(ID) {
-    initializeRISCVInsertNTLHInstsPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  StringRef getPassName() const override {
-    return RISCV_INSERT_NTLH_INSTS_NAME;
-  }
-};
-
-} // end of anonymous namespace
-
-char RISCVInsertNTLHInsts::ID = 0;
-
-bool RISCVInsertNTLHInsts::runOnMachineFunction(MachineFunction &MF) {
-  const auto &ST = MF.getSubtarget<RISCVSubtarget>();
-  TII = ST.getInstrInfo();
-
-  if (!ST.hasStdExtZihintntl())
-    return false;
-
-  bool Changed = false;
-  for (auto &MBB : MF) {
-    for (auto &MBBI : MBB) {
-      if (MBBI.memoperands_empty())
-        continue;
-      MachineMemOperand *MMO = *(MBBI.memoperands_begin());
-      if (MMO->isNonTemporal()) {
-        DebugLoc DL = MBBI.getDebugLoc();
-        if (ST.hasStdExtCOrZca() && ST.enableRVCHintInstrs())
-          BuildMI(MBB, MBBI, DL, TII->get(RISCV::PseudoCNTLALL));
-        else
-          BuildMI(MBB, MBBI, DL, TII->get(RISCV::PseudoNTLALL));
-        Changed = true;
-      }
-    }
-  }
-
-  return Changed;
-}
-
-INITIALIZE_PASS(RISCVInsertNTLHInsts, "riscv-insert-ntlh-insts",
-                RISCV_INSERT_NTLH_INSTS_NAME, false, false)
-
-namespace llvm {
-
-FunctionPass *createRISCVInsertNTLHInstsPass() {
-  return new RISCVInsertNTLHInsts();
-}
-
-} // end of namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 76d0974e70371..faffe3d12fa39 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -85,6 +85,18 @@ static bool isScalarMoveInstr(const MachineInstr &MI) {
   }
 }
 
+static bool isVSlideInstr(const MachineInstr &MI) {
+  switch (getRVVMCOpcode(MI.getOpcode())) {
+  default:
+    return false;
+  case RISCV::VSLIDEDOWN_VX:
+  case RISCV::VSLIDEDOWN_VI:
+  case RISCV::VSLIDEUP_VX:
+  case RISCV::VSLIDEUP_VI:
+    return true;
+  }
+}
+
 /// Get the EEW for a load or store instruction.  Return std::nullopt if MI is
 /// not a load or store which ignores SEW.
 static std::optional<unsigned> getEEWForLoadStore(const MachineInstr &MI) {
@@ -818,6 +830,11 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
       .addImm(Info.encodeVTYPE());
 }
 
+static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL) {
+  auto [LMul, Fractional] = RISCVVType::decodeVLMUL(LMUL);
+  return Fractional || LMul == 1;
+}
+
 /// Return true if a VSETVLI is required to transition from CurInfo to Require
 /// before MI.
 bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
@@ -845,6 +862,27 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
     }
   }
 
+  // A slidedown/slideup with a VL of 1 whose destination is an IMPLICIT_DEF
+  // can use any VL/SEW combination which writes at least the first element.
+  // Notes:
+  // * VL=1 is special only because we have existing support for zero vs
+  //   non-zero VL.  We could generalize this if we had a VL > C predicate.
+  // * The LMUL1 restriction is for machines whose latency may depend on VL.
+  // * As above, this is only legal for IMPLICIT_DEF, not TA.
+  if (isVSlideInstr(MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
+      isLMUL1OrSmaller(CurInfo.getVLMUL())) {
+    auto *VRegDef = MRI->getVRegDef(MI.getOperand(1).getReg());
+    if (VRegDef && VRegDef->isImplicitDef() &&
+        CurInfo.getSEW() >= Require.getSEW()) {
+      Used.VLAny = false;
+      Used.VLZeroness = true;
+      Used.SEW = false;
+      Used.LMUL = false;
+      Used.SEWLMULRatio = false;
+      Used.TailPolicy = false;
+    }
+  }
+
   if (CurInfo.isCompatible(Used, Require))
     return false;
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index 2f3c6a6e5ccff..6d9094e914b33 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -226,7 +226,7 @@ class Pseudo<dag outs, dag ins, list<dag> pattern, string opcodestr = "", string
   let isCodeGenOnly = 1;
 }
 
-class PseudoQuietFCMP<RegisterClass Ty>
+class PseudoQuietFCMP<DAGOperand Ty>
     : Pseudo<(outs GPR:$rd), (ins Ty:$rs1, Ty:$rs2), []> {
   let hasSideEffects = 1;
   let mayLoad = 0;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 0f96d3c19da4e..b36d166f0a51b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -1271,6 +1271,8 @@ bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
   case RISCV::FSGNJ_D:
   case RISCV::FSGNJ_S:
   case RISCV::FSGNJ_H:
+  case RISCV::FSGNJ_D_INX:
+  case RISCV::FSGNJ_S_INX:
     // The canonical floating-point move is fsgnj rd, rs, rs.
     return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
            MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
@@ -1300,6 +1302,8 @@ RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
   case RISCV::FSGNJ_D:
   case RISCV::FSGNJ_S:
   case RISCV::FSGNJ_H:
+  case RISCV::FSGNJ_D_INX:
+  case RISCV::FSGNJ_S_INX:
     // The canonical floating-point move is fsgnj rd, rs, rs.
     if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
         MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
@@ -1704,6 +1708,9 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_UIMM8_LSB000:
           Ok = isShiftedUInt<5, 3>(Imm);
           break;
+        case RISCVOp::OPERAND_UIMM8_GE32:
+          Ok = isUInt<8>(Imm) && Imm >= 32;
+          break;
         case RISCVOp::OPERAND_UIMM9_LSB000:
           Ok = isShiftedUInt<6, 3>(Imm);
           break;
@@ -2628,6 +2635,14 @@ void RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF,
   }
 }
 
+ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
+RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
+  static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
+      {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
+       {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
+  return ArrayRef(TargetFlags);
+}
+
 // Returns true if this is the sext.w pattern, addiw rd, rs1, 0.
 bool RISCV::isSEXT_W(const MachineInstr &MI) {
   return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 01f112a386d08..8f0f164263be9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -25,6 +25,11 @@ namespace llvm {
 
 class RISCVSubtarget;
 
+static const MachineMemOperand::Flags MONontemporalBit0 =
+    MachineMemOperand::MOTargetFlag1;
+static const MachineMemOperand::Flags MONontemporalBit1 =
+    MachineMemOperand::MOTargetFlag2;
+
 namespace RISCVCC {
 
 enum CondCode {
@@ -227,6 +232,9 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
 
   std::optional<unsigned> getInverseOpcode(unsigned Opcode) const override;
 
+  ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
+  getSerializableMachineMemOperandTargetFlags() const override;
+
 protected:
   const RISCVSubtarget &STI;
 };
@@ -257,6 +265,17 @@ bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2);
 // Special immediate for AVL operand of V pseudo instructions to indicate VLMax.
 static constexpr int64_t VLMaxSentinel = -1LL;
 
+// Mask assignments for floating-point
+static constexpr unsigned FPMASK_Negative_Infinity = 0x001;
+static constexpr unsigned FPMASK_Negative_Normal = 0x002;
+static constexpr unsigned FPMASK_Negative_Subnormal = 0x004;
+static constexpr unsigned FPMASK_Negative_Zero = 0x008;
+static constexpr unsigned FPMASK_Positive_Zero = 0x010;
+static constexpr unsigned FPMASK_Positive_Subnormal = 0x020;
+static constexpr unsigned FPMASK_Positive_Normal = 0x040;
+static constexpr unsigned FPMASK_Positive_Infinity = 0x080;
+static constexpr unsigned FPMASK_Signaling_NaN = 0x100;
+static constexpr unsigned FPMASK_Quiet_NaN = 0x200;
 } // namespace RISCV
 
 namespace RISCVVPseudosTable {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index b125ed5e8352a..9e971f6988722 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -57,8 +57,6 @@ def riscv_call      : SDNode<"RISCVISD::CALL", SDT_RISCVCall,
                               SDNPVariadic]>;
 def riscv_ret_glue  : SDNode<"RISCVISD::RET_GLUE", SDTNone,
                              [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
-def riscv_uret_glue : SDNode<"RISCVISD::URET_GLUE", SDTNone,
-                             [SDNPHasChain, SDNPOptInGlue]>;
 def riscv_sret_glue : SDNode<"RISCVISD::SRET_GLUE", SDTNone,
                              [SDNPHasChain, SDNPOptInGlue]>;
 def riscv_mret_glue : SDNode<"RISCVISD::MRET_GLUE", SDTNone,
@@ -411,10 +409,6 @@ def ixlenimm_li : Operand<XLenVT> {
 
 // Standalone (codegen-only) immleaf patterns.
 
-// A 12-bit signed immediate plus one where the imm range will be [-2047, 2048].
-def simm12_plus1 : ImmLeaf<XLenVT,
-  [{return (isInt<12>(Imm) && Imm != -2048) || Imm == 2048;}]>;
-
 // A 6-bit constant greater than 32.
 def uimm6gt32 : ImmLeaf<XLenVT, [{
   return isUInt<6>(Imm) && Imm > 32;
@@ -822,12 +816,6 @@ def SRAW  : ALUW_rr<0b0100000, 0b101, "sraw">,
 //===----------------------------------------------------------------------===//
 
 let isBarrier = 1, isReturn = 1, isTerminator = 1 in {
-def URET : Priv<"uret", 0b0000000>, Sched<[]> {
-  let rd = 0;
-  let rs1 = 0;
-  let rs2 = 0b00010;
-}
-
 def SRET : Priv<"sret", 0b0001000>, Sched<[]> {
   let rd = 0;
   let rs1 = 0;
@@ -1446,7 +1434,7 @@ def PseudoCCSUBW : Pseudo<(outs GPR:$dst),
                    Sched<[WriteSFB, ReadSFB, ReadSFB, ReadSFB, ReadSFB, ReadSFB]>;
 }
 
-multiclass SelectCC_GPR_rrirr<RegisterClass valty> {
+multiclass SelectCC_GPR_rrirr<DAGOperand valty> {
   let usesCustomInserter = 1 in
   def _Using_CC_GPR : Pseudo<(outs valty:$dst),
                              (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
@@ -1562,7 +1550,6 @@ def PseudoCALL : Pseudo<(outs), (ins call_symbol:$func), []>,
 def : Pat<(riscv_call tglobaladdr:$func), (PseudoCALL tglobaladdr:$func)>;
 def : Pat<(riscv_call texternalsym:$func), (PseudoCALL texternalsym:$func)>;
 
-def : Pat<(riscv_uret_glue), (URET X0, X0)>;
 def : Pat<(riscv_sret_glue), (SRET X0, X0)>;
 def : Pat<(riscv_mret_glue), (MRET X0, X0)>;
 
@@ -1920,7 +1907,6 @@ include "RISCVInstrInfoZfa.td"
 include "RISCVInstrInfoZfh.td"
 include "RISCVInstrInfoZicbo.td"
 include "RISCVInstrInfoZicond.td"
-include "RISCVInstrInfoZihintntl.td"
 
 //===----------------------------------------------------------------------===//
 // Vendor extensions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
index 961f5e9ebb3ae..45aa1e5d6fddb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -80,6 +80,8 @@ def XD_IN32X : ExtInfo_rr<Zdinx32Ext, GPR,        FPR64IN32X>;
 def XD_64    : ExtInfo_rr<D64Ext,     GPR,        FPR64>;
 
 defvar DINX    = [D,     D_INX,  D_IN32X];
+// TODO: Remove DIN64X when Zdinx for RV32 supported
+defvar DIN64X  = [D,     D_INX];
 defvar DDINX   = [DD,    DD_INX, DD_IN32X];
 defvar DXINX   = [DX,    DX_INX, DX_IN32X];
 defvar DFINX   = [DF,    DF_INX, DF_IN32X];
@@ -218,6 +220,10 @@ def : InstAlias<"fgt.d $rd, $rs, $rt",
                 (FLT_D_INX GPR:$rd, FPR64INX:$rt, FPR64INX:$rs), 0>;
 def : InstAlias<"fge.d $rd, $rs, $rt",
                 (FLE_D_INX GPR:$rd, FPR64INX:$rt, FPR64INX:$rs), 0>;
+let usesCustomInserter = 1 in {
+def PseudoQuietFLE_D_INX : PseudoQuietFCMP<FPR64INX>;
+def PseudoQuietFLT_D_INX : PseudoQuietFCMP<FPR64INX>;
+}
 } // Predicates = [HasStdExtZdinx, IsRV64]
 
 let Predicates = [HasStdExtZdinx, IsRV32] in {
@@ -241,22 +247,34 @@ let Predicates = [HasStdExtD] in {
 // f64 -> f32, f32 -> f64
 def : Pat<(any_fpround FPR64:$rs1), (FCVT_S_D FPR64:$rs1, FRM_DYN)>;
 def : Pat<(any_fpextend FPR32:$rs1), (FCVT_D_S FPR32:$rs1)>;
+} // Predicates = [HasStdExtD]
+
+let Predicates = [HasStdExtZdinx, IsRV64] in {
+/// Float conversion operations
+
+// f64 -> f32, f32 -> f64
+def : Pat<(any_fpround FPR64INX:$rs1), (FCVT_S_D_INX FPR64INX:$rs1, FRM_DYN)>;
+def : Pat<(any_fpextend FPR32INX:$rs1), (FCVT_D_S_INX FPR32INX:$rs1)>;
+} // Predicates = [HasStdExtZdinx, IsRV64]
 
 // [u]int<->double conversion patterns must be gated on IsRV32 or IsRV64, so
 // are defined later.
 
 /// Float arithmetic operations
 
-def : PatFprFprDynFrm<any_fadd, FADD_D, FPR64>;
-def : PatFprFprDynFrm<any_fsub, FSUB_D, FPR64>;
-def : PatFprFprDynFrm<any_fmul, FMUL_D, FPR64>;
-def : PatFprFprDynFrm<any_fdiv, FDIV_D, FPR64>;
+defm : PatFprFprDynFrm_m<any_fadd, FADD_D, DINX>;
+defm : PatFprFprDynFrm_m<any_fsub, FSUB_D, DINX>;
+defm : PatFprFprDynFrm_m<any_fmul, FMUL_D, DINX>;
+defm : PatFprFprDynFrm_m<any_fdiv, FDIV_D, DINX>;
 
+let Predicates = [HasStdExtD] in {
 def : Pat<(any_fsqrt FPR64:$rs1), (FSQRT_D FPR64:$rs1, FRM_DYN)>;
 
 def : Pat<(fneg FPR64:$rs1), (FSGNJN_D $rs1, $rs1)>;
 def : Pat<(fabs FPR64:$rs1), (FSGNJX_D $rs1, $rs1)>;
 
+def : Pat<(riscv_fpclass FPR64:$rs1), (FCLASS_D $rs1)>;
+
 def : PatFprFpr<fcopysign, FSGNJ_D, FPR64>;
 def : Pat<(fcopysign FPR64:$rs1, (fneg FPR64:$rs2)), (FSGNJN_D $rs1, $rs2)>;
 def : Pat<(fcopysign FPR64:$rs1, FPR32:$rs2), (FSGNJ_D $rs1, (FCVT_D_S $rs2))>;
@@ -282,25 +300,64 @@ def : Pat<(any_fma (fneg FPR64:$rs1), FPR64:$rs2, (fneg FPR64:$rs3)),
 // fnmadd: -(rs1 * rs2 + rs3) (the nsz flag on the FMA)
 def : Pat<(fneg (any_fma_nsz FPR64:$rs1, FPR64:$rs2, FPR64:$rs3)),
           (FNMADD_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, FRM_DYN)>;
+} // Predicates = [HasStdExtD]
+
+let Predicates = [HasStdExtZdinx, IsRV64] in {
+def : Pat<(any_fsqrt FPR64INX:$rs1), (FSQRT_D_INX FPR64INX:$rs1, FRM_DYN)>;
+
+def : Pat<(fneg FPR64INX:$rs1), (FSGNJN_D_INX $rs1, $rs1)>;
+def : Pat<(fabs FPR64INX:$rs1), (FSGNJX_D_INX $rs1, $rs1)>;
+
+def : Pat<(riscv_fpclass FPR64INX:$rs1), (FCLASS_D_INX $rs1)>;
+
+def : PatFprFpr<fcopysign, FSGNJ_D_INX, FPR64INX>;
+def : Pat<(fcopysign FPR64INX:$rs1, (fneg FPR64INX:$rs2)),
+          (FSGNJN_D_INX $rs1, $rs2)>;
+def : Pat<(fcopysign FPR64INX:$rs1, FPR32INX:$rs2),
+          (FSGNJ_D_INX $rs1, (FCVT_D_S_INX $rs2))>;
+def : Pat<(fcopysign FPR32INX:$rs1, FPR64INX:$rs2),
+          (FSGNJ_S_INX $rs1, (FCVT_S_D_INX $rs2, FRM_DYN))>;
+
+// fmadd: rs1 * rs2 + rs3
+def : Pat<(any_fma FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3),
+          (FMADD_D_INX $rs1, $rs2, $rs3, FRM_DYN)>;
+
+// fmsub: rs1 * rs2 - rs3
+def : Pat<(any_fma FPR64INX:$rs1, FPR64INX:$rs2, (fneg FPR64INX:$rs3)),
+          (FMSUB_D_INX FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3, FRM_DYN)>;
+
+// fnmsub: -rs1 * rs2 + rs3
+def : Pat<(any_fma (fneg FPR64INX:$rs1), FPR64INX:$rs2, FPR64INX:$rs3),
+          (FNMSUB_D_INX FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3, FRM_DYN)>;
+
+// fnmadd: -rs1 * rs2 - rs3
+def : Pat<(any_fma (fneg FPR64INX:$rs1), FPR64INX:$rs2, (fneg FPR64INX:$rs3)),
+          (FNMADD_D_INX FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3, FRM_DYN)>;
+
+// fnmadd: -(rs1 * rs2 + rs3) (the nsz flag on the FMA)
+def : Pat<(fneg (any_fma_nsz FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3)),
+          (FNMADD_D_INX FPR64INX:$rs1, FPR64INX:$rs2, FPR64INX:$rs3, FRM_DYN)>;
+} // Predicates = [HasStdExtZdinx, IsRV64]
 
 // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches
 // LLVM's fminnum and fmaxnum.
 // <https://github.com/riscv/riscv-isa-manual/commit/cd20cee7efd9bac7c5aa127ec3b451749d2b3cce>.
-def : PatFprFpr<fminnum, FMIN_D, FPR64>;
-def : PatFprFpr<fmaxnum, FMAX_D, FPR64>;
+defm : PatFprFpr_m<fminnum, FMIN_D, DINX>;
+defm : PatFprFpr_m<fmaxnum, FMAX_D, DINX>;
 
 /// Setcc
 // FIXME: SETEQ/SETLT/SETLE imply nonans, can we pick better instructions for
 // strict versions of those.
 
 // Match non-signaling FEQ_D
-def : PatSetCC<FPR64, any_fsetcc, SETEQ, FEQ_D>;
-def : PatSetCC<FPR64, any_fsetcc, SETOEQ, FEQ_D>;
-def : PatSetCC<FPR64, strict_fsetcc, SETLT, PseudoQuietFLT_D>;
-def : PatSetCC<FPR64, strict_fsetcc, SETOLT, PseudoQuietFLT_D>;
-def : PatSetCC<FPR64, strict_fsetcc, SETLE, PseudoQuietFLE_D>;
-def : PatSetCC<FPR64, strict_fsetcc, SETOLE, PseudoQuietFLE_D>;
+defm : PatSetCC_m<any_fsetcc,    SETEQ,  FEQ_D,            DINX>;
+defm : PatSetCC_m<any_fsetcc,    SETOEQ, FEQ_D,            DINX>;
+defm : PatSetCC_m<strict_fsetcc, SETLT,  PseudoQuietFLT_D, DIN64X>;
+defm : PatSetCC_m<strict_fsetcc, SETOLT, PseudoQuietFLT_D, DIN64X>;
+defm : PatSetCC_m<strict_fsetcc, SETLE,  PseudoQuietFLE_D, DIN64X>;
+defm : PatSetCC_m<strict_fsetcc, SETOLE, PseudoQuietFLE_D, DIN64X>;
 
+let Predicates = [HasStdExtD] in {
 // Match signaling FEQ_D
 def : Pat<(strict_fsetccs FPR64:$rs1, FPR64:$rs2, SETEQ),
           (AND (FLE_D $rs1, $rs2),
@@ -318,7 +375,29 @@ def : PatSetCC<FPR64, any_fsetccs, SETLT, FLT_D>;
 def : PatSetCC<FPR64, any_fsetccs, SETOLT, FLT_D>;
 def : PatSetCC<FPR64, any_fsetccs, SETLE, FLE_D>;
 def : PatSetCC<FPR64, any_fsetccs, SETOLE, FLE_D>;
+} // Predicates = [HasStdExtD]
 
+let Predicates = [HasStdExtZdinx, IsRV64] in {
+// Match signaling FEQ_D
+def : Pat<(strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs2, SETEQ),
+          (AND (FLE_D_INX $rs1, $rs2),
+               (FLE_D_INX $rs2, $rs1))>;
+def : Pat<(strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs2, SETOEQ),
+          (AND (FLE_D_INX $rs1, $rs2),
+               (FLE_D_INX $rs2, $rs1))>;
+// If both operands are the same, use a single FLE.
+def : Pat<(strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs1, SETEQ),
+          (FLE_D_INX $rs1, $rs1)>;
+def : Pat<(strict_fsetccs FPR64INX:$rs1, FPR64INX:$rs1, SETOEQ),
+          (FLE_D_INX $rs1, $rs1)>;
+
+def : PatSetCC<FPR64INX, any_fsetccs, SETLT,  FLT_D_INX>;
+def : PatSetCC<FPR64INX, any_fsetccs, SETOLT, FLT_D_INX>;
+def : PatSetCC<FPR64INX, any_fsetccs, SETLE,  FLE_D_INX>;
+def : PatSetCC<FPR64INX, any_fsetccs, SETOLE, FLE_D_INX>;
+} // Predicates = [HasStdExtZdinx, IsRV64]
+
+let Predicates = [HasStdExtD] in {
 defm Select_FPR64 : SelectCC_GPR_rrirr<FPR64>;
 
 def PseudoFROUND_D : PseudoFROUND<FPR64>;
@@ -347,6 +426,20 @@ def SplitF64Pseudo
 
 } // Predicates = [HasStdExtD]
 
+let Predicates = [HasStdExtZdinx, IsRV64] in {
+defm Select_FPR64INX : SelectCC_GPR_rrirr<FPR64INX>;
+
+def PseudoFROUND_D_INX : PseudoFROUND<FPR64INX>;
+
+/// Loads
+def : Pat<(f64 (load (AddrRegImm GPR:$rs1, simm12:$imm12))),
+          (COPY_TO_REGCLASS (LD GPR:$rs1, simm12:$imm12), GPRF64)>;
+
+/// Stores
+def : Pat<(store (f64 FPR64INX:$rs2), (AddrRegImm GPR:$rs1, simm12:$imm12)),
+          (SD (COPY_TO_REGCLASS FPR64INX:$rs2, GPR), GPR:$rs1, simm12:$imm12)>;
+} // Predicates = [HasStdExtZdinx, IsRV64]
+
 let Predicates = [HasStdExtD, IsRV32] in {
 
 // double->[u]int. Round-to-zero must be used.
@@ -404,3 +497,40 @@ def : Pat<(i64 (any_llround FPR64:$rs1)), (FCVT_L_D $rs1, FRM_RMM)>;
 def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_D_L GPR:$rs1, FRM_DYN)>;
 def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_D_LU GPR:$rs1, FRM_DYN)>;
 } // Predicates = [HasStdExtD, IsRV64]
+
+let Predicates = [HasStdExtZdinx, IsRV64] in {
+
+// Moves (no conversion)
+def : Pat<(f64 (bitconvert (i64 GPR:$rs1))), (COPY_TO_REGCLASS GPR:$rs1, GPRF64)>;
+def : Pat<(i64 (bitconvert FPR64INX:$rs1)), (COPY_TO_REGCLASS FPR64INX:$rs1, GPR)>;
+
+// Use target specific isd nodes to help us remember the result is sign
+// extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
+// duplicated if it has another user that didn't need the sign_extend.
+def : Pat<(riscv_any_fcvt_w_rv64 FPR64INX:$rs1, timm:$frm),  (FCVT_W_D_INX $rs1, timm:$frm)>;
+def : Pat<(riscv_any_fcvt_wu_rv64 FPR64INX:$rs1, timm:$frm), (FCVT_WU_D_INX $rs1, timm:$frm)>;
+
+// [u]int32->fp
+def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_D_W_INX $rs1)>;
+def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_D_WU_INX $rs1)>;
+
+// Saturating double->[u]int64.
+def : Pat<(i64 (riscv_fcvt_x FPR64INX:$rs1, timm:$frm)), (FCVT_L_D_INX $rs1, timm:$frm)>;
+def : Pat<(i64 (riscv_fcvt_xu FPR64INX:$rs1, timm:$frm)), (FCVT_LU_D_INX $rs1, timm:$frm)>;
+
+// double->[u]int64. Round-to-zero must be used.
+def : Pat<(i64 (any_fp_to_sint FPR64INX:$rs1)), (FCVT_L_D_INX FPR64INX:$rs1, FRM_RTZ)>;
+def : Pat<(i64 (any_fp_to_uint FPR64INX:$rs1)), (FCVT_LU_D_INX FPR64INX:$rs1, FRM_RTZ)>;
+
+// double->int64 with current rounding mode.
+def : Pat<(i64 (any_lrint FPR64INX:$rs1)), (FCVT_L_D_INX $rs1, FRM_DYN)>;
+def : Pat<(i64 (any_llrint FPR64INX:$rs1)), (FCVT_L_D_INX $rs1, FRM_DYN)>;
+
+// double->int64 rounded to nearest with ties rounded away from zero.
+def : Pat<(i64 (any_lround FPR64INX:$rs1)), (FCVT_L_D_INX $rs1, FRM_RMM)>;
+def : Pat<(i64 (any_llround FPR64INX:$rs1)), (FCVT_L_D_INX $rs1, FRM_RMM)>;
+
+// [u]int64->fp. Match GCC and default to using dynamic rounding mode.
+def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_D_L_INX GPR:$rs1, FRM_DYN)>;
+def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_D_LU_INX GPR:$rs1, FRM_DYN)>;
+} // Predicates = [HasStdExtZdinx, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index aa5d0afd69cfe..0ac5d5177a704 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -29,6 +29,11 @@ def SDT_RISCVFCVT_X
 def SDT_RISCVFROUND
     : SDTypeProfile<1, 3, [SDTCisFP<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>,
                            SDTCisVT<3, XLenVT>]>;
+def SDT_RISCVFPCLASS
+    : SDTypeProfile<1, 1, [SDTCisVT<0, XLenVT>, SDTCisFP<1>]>;
+
+def riscv_fpclass
+    : SDNode<"RISCVISD::FPCLASS", SDT_RISCVFPCLASS>;
 
 def riscv_fround
     : SDNode<"RISCVISD::FROUND", SDT_RISCVFROUND>;
@@ -156,7 +161,7 @@ def frmarg : Operand<XLenVT> {
 //===----------------------------------------------------------------------===//
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
-class FPLoad_r<bits<3> funct3, string opcodestr, RegisterClass rty,
+class FPLoad_r<bits<3> funct3, string opcodestr, DAGOperand rty,
                SchedWrite sw>
     : RVInstI<funct3, OPC_LOAD_FP, (outs rty:$rd),
               (ins GPRMem:$rs1, simm12:$imm12),
@@ -164,7 +169,7 @@ class FPLoad_r<bits<3> funct3, string opcodestr, RegisterClass rty,
       Sched<[sw, ReadFMemBase]>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
-class FPStore_r<bits<3> funct3, string opcodestr, RegisterClass rty,
+class FPStore_r<bits<3> funct3, string opcodestr, DAGOperand rty,
                 SchedWrite sw>
     : RVInstS<funct3, OPC_STORE_FP, (outs),
               (ins rty:$rs2, GPRMem:$rs1, simm12:$imm12),
@@ -263,7 +268,7 @@ multiclass FPCmp_rr_m<bits<7> funct7, bits<3> funct3, string opcodestr,
     def Ext.Suffix : FPCmp_rr<funct7, funct3, opcodestr, Ext.Reg, Commutable>;
 }
 
-class PseudoFROUND<RegisterClass Ty>
+class PseudoFROUND<DAGOperand Ty>
     : Pseudo<(outs Ty:$rd), (ins Ty:$rs1, Ty:$rs2, ixlenimm:$rm),
       [(set Ty:$rd, (riscv_fround Ty:$rs1, Ty:$rs2, timm:$rm))]> {
   let hasSideEffects = 0;
@@ -428,6 +433,10 @@ def : InstAlias<"fgt.s $rd, $rs, $rt",
                 (FLT_S_INX GPR:$rd, FPR32INX:$rt, FPR32INX:$rs), 0>;
 def : InstAlias<"fge.s $rd, $rs, $rt",
                 (FLE_S_INX GPR:$rd, FPR32INX:$rt, FPR32INX:$rs), 0>;
+let usesCustomInserter = 1 in {
+def PseudoQuietFLE_S_INX : PseudoQuietFCMP<FPR32INX>;
+def PseudoQuietFLT_S_INX : PseudoQuietFCMP<FPR32INX>;
+}
 } // Predicates = [HasStdExtZfinx]
 
 //===----------------------------------------------------------------------===//
@@ -445,18 +454,39 @@ defvar FRM_DYN = 0b111;
 def fpimm0    : PatLeaf<(fpimm), [{ return N->isExactlyValue(+0.0); }]>;
 
 /// Generic pattern classes
-class PatSetCC<RegisterClass Ty, SDPatternOperator OpNode, CondCode Cond, RVInst Inst>
+class PatSetCC<DAGOperand Ty, SDPatternOperator OpNode, CondCode Cond,
+               RVInst Inst>
     : Pat<(OpNode Ty:$rs1, Ty:$rs2, Cond), (Inst $rs1, $rs2)>;
+multiclass PatSetCC_m<SDPatternOperator OpNode, CondCode Cond,
+                      RVInst Inst, list<ExtInfo_r> Exts> {
+  foreach Ext = Exts in
+    let Predicates = Ext.Predicates in
+    def Ext.Suffix : PatSetCC<Ext.Reg, OpNode, Cond,
+                              !cast<RVInst>(Inst#Ext.Suffix)>;
+}
 
 class PatFprFpr<SDPatternOperator OpNode, RVInstR Inst,
-                RegisterClass RegTy>
+                DAGOperand RegTy>
     : Pat<(OpNode RegTy:$rs1, RegTy:$rs2), (Inst $rs1, $rs2)>;
+multiclass PatFprFpr_m<SDPatternOperator OpNode, RVInstR Inst,
+                       list<ExtInfo_r> Exts> {
+  foreach Ext = Exts in
+    let Predicates = Ext.Predicates in
+    def Ext.Suffix : PatFprFpr<OpNode, !cast<RVInstR>(Inst#Ext.Suffix),
+                               Ext.Reg>;
+}
 
 class PatFprFprDynFrm<SDPatternOperator OpNode, RVInstRFrm Inst,
-                      RegisterClass RegTy>
+                      DAGOperand RegTy>
     : Pat<(OpNode RegTy:$rs1, RegTy:$rs2), (Inst $rs1, $rs2, FRM_DYN)>;
-
-let Predicates = [HasStdExtF] in {
+multiclass PatFprFprDynFrm_m<SDPatternOperator OpNode, RVInstRFrm Inst,
+                             list<ExtInfo_r> Exts> {
+  foreach Ext = Exts in
+    let Predicates = Ext.Predicates in
+    def Ext.Suffix : PatFprFprDynFrm<OpNode,
+                                     !cast<RVInstRFrm>(Inst#Ext.Suffix),
+                                     Ext.Reg>;
+}
 
 /// Float conversion operations
 
@@ -464,18 +494,32 @@ let Predicates = [HasStdExtF] in {
 // are defined later.
 
 /// Float arithmetic operations
+defm : PatFprFprDynFrm_m<any_fadd, FADD_S, FINX>;
+defm : PatFprFprDynFrm_m<any_fsub, FSUB_S, FINX>;
+defm : PatFprFprDynFrm_m<any_fmul, FMUL_S, FINX>;
+defm : PatFprFprDynFrm_m<any_fdiv, FDIV_S, FINX>;
 
-def : PatFprFprDynFrm<any_fadd, FADD_S, FPR32>;
-def : PatFprFprDynFrm<any_fsub, FSUB_S, FPR32>;
-def : PatFprFprDynFrm<any_fmul, FMUL_S, FPR32>;
-def : PatFprFprDynFrm<any_fdiv, FDIV_S, FPR32>;
-
+let Predicates = [HasStdExtF] in {
 def : Pat<(any_fsqrt FPR32:$rs1), (FSQRT_S FPR32:$rs1, FRM_DYN)>;
 
 def : Pat<(fneg FPR32:$rs1), (FSGNJN_S $rs1, $rs1)>;
 def : Pat<(fabs FPR32:$rs1), (FSGNJX_S $rs1, $rs1)>;
 
-def : PatFprFpr<fcopysign, FSGNJ_S, FPR32>;
+def : Pat<(riscv_fpclass FPR32:$rs1), (FCLASS_S $rs1)>;
+} // Predicates = [HasStdExtF]
+
+let Predicates = [HasStdExtZfinx] in {
+def : Pat<(any_fsqrt FPR32INX:$rs1), (FSQRT_S_INX FPR32INX:$rs1, FRM_DYN)>;
+
+def : Pat<(fneg FPR32INX:$rs1), (FSGNJN_S_INX $rs1, $rs1)>;
+def : Pat<(fabs FPR32INX:$rs1), (FSGNJX_S_INX $rs1, $rs1)>;
+
+def : Pat<(riscv_fpclass FPR32INX:$rs1), (FCLASS_S_INX $rs1)>;
+} // Predicates = [HasStdExtZfinx]
+
+defm : PatFprFpr_m<fcopysign, FSGNJ_S, FINX>;
+
+let Predicates = [HasStdExtF] in {
 def : Pat<(fcopysign FPR32:$rs1, (fneg FPR32:$rs2)), (FSGNJN_S $rs1, $rs2)>;
 
 // fmadd: rs1 * rs2 + rs3
@@ -497,25 +541,51 @@ def : Pat<(any_fma (fneg FPR32:$rs1), FPR32:$rs2, (fneg FPR32:$rs3)),
 // fnmadd: -(rs1 * rs2 + rs3) (the nsz flag on the FMA)
 def : Pat<(fneg (any_fma_nsz FPR32:$rs1, FPR32:$rs2, FPR32:$rs3)),
           (FNMADD_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, FRM_DYN)>;
+} // Predicates = [HasStdExtF]
+
+let Predicates = [HasStdExtZfinx] in {
+def : Pat<(fcopysign FPR32INX:$rs1, (fneg FPR32INX:$rs2)), (FSGNJN_S_INX $rs1, $rs2)>;
+
+// fmadd: rs1 * rs2 + rs3
+def : Pat<(any_fma FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3),
+          (FMADD_S_INX $rs1, $rs2, $rs3, FRM_DYN)>;
+
+// fmsub: rs1 * rs2 - rs3
+def : Pat<(any_fma FPR32INX:$rs1, FPR32INX:$rs2, (fneg FPR32INX:$rs3)),
+          (FMSUB_S_INX FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3, FRM_DYN)>;
+
+// fnmsub: -rs1 * rs2 + rs3
+def : Pat<(any_fma (fneg FPR32INX:$rs1), FPR32INX:$rs2, FPR32INX:$rs3),
+          (FNMSUB_S_INX FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3, FRM_DYN)>;
+
+// fnmadd: -rs1 * rs2 - rs3
+def : Pat<(any_fma (fneg FPR32INX:$rs1), FPR32INX:$rs2, (fneg FPR32INX:$rs3)),
+          (FNMADD_S_INX FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3, FRM_DYN)>;
+
+// fnmadd: -(rs1 * rs2 + rs3) (the nsz flag on the FMA)
+def : Pat<(fneg (any_fma_nsz FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3)),
+          (FNMADD_S_INX FPR32INX:$rs1, FPR32INX:$rs2, FPR32INX:$rs3, FRM_DYN)>;
+} // Predicates = [HasStdExtZfinx]
 
 // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches
 // LLVM's fminnum and fmaxnum
 // <https://github.com/riscv/riscv-isa-manual/commit/cd20cee7efd9bac7c5aa127ec3b451749d2b3cce>.
-def : PatFprFpr<fminnum, FMIN_S, FPR32>;
-def : PatFprFpr<fmaxnum, FMAX_S, FPR32>;
+defm : PatFprFpr_m<fminnum, FMIN_S, FINX>;
+defm : PatFprFpr_m<fmaxnum, FMAX_S, FINX>;
 
 /// Setcc
 // FIXME: SETEQ/SETLT/SETLE imply nonans, can we pick better instructions for
 // strict versions of those.
 
 // Match non-signaling FEQ_S
-def : PatSetCC<FPR32, any_fsetcc, SETEQ, FEQ_S>;
-def : PatSetCC<FPR32, any_fsetcc, SETOEQ, FEQ_S>;
-def : PatSetCC<FPR32, strict_fsetcc, SETLT, PseudoQuietFLT_S>;
-def : PatSetCC<FPR32, strict_fsetcc, SETOLT, PseudoQuietFLT_S>;
-def : PatSetCC<FPR32, strict_fsetcc, SETLE, PseudoQuietFLE_S>;
-def : PatSetCC<FPR32, strict_fsetcc, SETOLE, PseudoQuietFLE_S>;
+defm : PatSetCC_m<any_fsetcc,    SETEQ,  FEQ_S,            FINX>;
+defm : PatSetCC_m<any_fsetcc,    SETOEQ, FEQ_S,            FINX>;
+defm : PatSetCC_m<strict_fsetcc, SETLT,  PseudoQuietFLT_S, FINX>;
+defm : PatSetCC_m<strict_fsetcc, SETOLT, PseudoQuietFLT_S, FINX>;
+defm : PatSetCC_m<strict_fsetcc, SETLE,  PseudoQuietFLE_S, FINX>;
+defm : PatSetCC_m<strict_fsetcc, SETOLE, PseudoQuietFLE_S, FINX>;
 
+let Predicates = [HasStdExtF] in {
 // Match signaling FEQ_S
 def : Pat<(strict_fsetccs FPR32:$rs1, FPR32:$rs2, SETEQ),
           (AND (FLE_S $rs1, $rs2),
@@ -528,12 +598,29 @@ def : Pat<(strict_fsetccs FPR32:$rs1, FPR32:$rs1, SETEQ),
           (FLE_S $rs1, $rs1)>;
 def : Pat<(strict_fsetccs FPR32:$rs1, FPR32:$rs1, SETOEQ),
           (FLE_S $rs1, $rs1)>;
+} // Predicates = [HasStdExtF]
+
+let Predicates = [HasStdExtZfinx] in {
+// Match signaling FEQ_S
+def : Pat<(strict_fsetccs FPR32INX:$rs1, FPR32INX:$rs2, SETEQ),
+          (AND (FLE_S_INX $rs1, $rs2),
+               (FLE_S_INX $rs2, $rs1))>;
+def : Pat<(strict_fsetccs FPR32INX:$rs1, FPR32INX:$rs2, SETOEQ),
+          (AND (FLE_S_INX $rs1, $rs2),
+               (FLE_S_INX $rs2, $rs1))>;
+// If both operands are the same, use a single FLE.
+def : Pat<(strict_fsetccs FPR32INX:$rs1, FPR32INX:$rs1, SETEQ),
+          (FLE_S_INX $rs1, $rs1)>;
+def : Pat<(strict_fsetccs FPR32INX:$rs1, FPR32INX:$rs1, SETOEQ),
+          (FLE_S_INX $rs1, $rs1)>;
+} // Predicates = [HasStdExtZfinx]
 
-def : PatSetCC<FPR32, any_fsetccs, SETLT, FLT_S>;
-def : PatSetCC<FPR32, any_fsetccs, SETOLT, FLT_S>;
-def : PatSetCC<FPR32, any_fsetccs, SETLE, FLE_S>;
-def : PatSetCC<FPR32, any_fsetccs, SETOLE, FLE_S>;
+defm : PatSetCC_m<any_fsetccs, SETLT,  FLT_S, FINX>;
+defm : PatSetCC_m<any_fsetccs, SETOLT, FLT_S, FINX>;
+defm : PatSetCC_m<any_fsetccs, SETLE,  FLE_S, FINX>;
+defm : PatSetCC_m<any_fsetccs, SETOLE, FLE_S, FINX>;
 
+let Predicates = [HasStdExtF] in {
 defm Select_FPR32 : SelectCC_GPR_rrirr<FPR32>;
 
 def PseudoFROUND_S : PseudoFROUND<FPR32>;
@@ -548,11 +635,33 @@ defm : StPat<store, FSW, FPR32, f32>;
 
 } // Predicates = [HasStdExtF]
 
+let Predicates = [HasStdExtZfinx] in {
+defm Select_FPR32INX : SelectCC_GPR_rrirr<FPR32INX>;
+
+def PseudoFROUND_S_INX : PseudoFROUND<FPR32INX>;
+
+/// Loads
+def : Pat<(f32 (load (AddrRegImm GPR:$rs1, simm12:$imm12))),
+          (COPY_TO_REGCLASS (LW GPR:$rs1, simm12:$imm12), GPRF32)>;
+
+/// Stores
+def : Pat<(store (f32 FPR32INX:$rs2), (AddrRegImm GPR:$rs1, simm12:$imm12)),
+          (SW (COPY_TO_REGCLASS FPR32INX:$rs2, GPR), GPR:$rs1, simm12:$imm12)>;
+} // Predicates = [HasStdExtZfinx]
+
 let Predicates = [HasStdExtF, IsRV32] in {
 // Moves (no conversion)
 def : Pat<(bitconvert (i32 GPR:$rs1)), (FMV_W_X GPR:$rs1)>;
 def : Pat<(i32 (bitconvert FPR32:$rs1)), (FMV_X_W FPR32:$rs1)>;
+} // Predicates = [HasStdExtF, IsRV32]
+
+let Predicates = [HasStdExtZfinx, IsRV32] in {
+// Moves (no conversion)
+def : Pat<(f32 (bitconvert (i32 GPR:$rs1))), (COPY_TO_REGCLASS GPR:$rs1, GPRF32)>;
+def : Pat<(i32 (bitconvert FPR32INX:$rs1)), (COPY_TO_REGCLASS FPR32INX:$rs1, GPR)>;
+} // Predicates = [HasStdExtZfinx, IsRV32]
 
+let Predicates = [HasStdExtF, IsRV32] in {
 // float->[u]int. Round-to-zero must be used.
 def : Pat<(i32 (any_fp_to_sint FPR32:$rs1)), (FCVT_W_S $rs1, FRM_RTZ)>;
 def : Pat<(i32 (any_fp_to_uint FPR32:$rs1)), (FCVT_WU_S $rs1, FRM_RTZ)>;
@@ -572,12 +681,30 @@ def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_S_W $rs1, FRM_DYN)>;
 def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_S_WU $rs1, FRM_DYN)>;
 } // Predicates = [HasStdExtF, IsRV32]
 
+let Predicates = [HasStdExtZfinx, IsRV32] in {
+// float->[u]int. Round-to-zero must be used.
+def : Pat<(i32 (any_fp_to_sint FPR32INX:$rs1)), (FCVT_W_S_INX $rs1, FRM_RTZ)>;
+def : Pat<(i32 (any_fp_to_uint FPR32INX:$rs1)), (FCVT_WU_S_INX $rs1, FRM_RTZ)>;
+
+// Saturating float->[u]int32.
+def : Pat<(i32 (riscv_fcvt_x FPR32INX:$rs1, timm:$frm)), (FCVT_W_S_INX $rs1, timm:$frm)>;
+def : Pat<(i32 (riscv_fcvt_xu FPR32INX:$rs1, timm:$frm)), (FCVT_WU_S_INX $rs1, timm:$frm)>;
+
+// float->int32 with current rounding mode.
+def : Pat<(i32 (any_lrint FPR32INX:$rs1)), (FCVT_W_S_INX $rs1, FRM_DYN)>;
+
+// float->int32 rounded to nearest with ties rounded away from zero.
+def : Pat<(i32 (any_lround FPR32INX:$rs1)), (FCVT_W_S_INX $rs1, FRM_RMM)>;
+
+// [u]int->float. Match GCC and default to using dynamic rounding mode.
+def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_S_W_INX $rs1, FRM_DYN)>;
+def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_S_WU_INX $rs1, FRM_DYN)>;
+} // Predicates = [HasStdExtZfinx, IsRV32]
+
 let Predicates = [HasStdExtF, IsRV64] in {
 // Moves (no conversion)
 def : Pat<(riscv_fmv_w_x_rv64 GPR:$src), (FMV_W_X GPR:$src)>;
 def : Pat<(riscv_fmv_x_anyextw_rv64 FPR32:$src), (FMV_X_W FPR32:$src)>;
-def : Pat<(sext_inreg (riscv_fmv_x_anyextw_rv64 FPR32:$src), i32),
-          (FMV_X_W FPR32:$src)>;
 
 // Use target specific isd nodes to help us remember the result is sign
 // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
@@ -607,3 +734,37 @@ def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_S_WU $rs1, FRM_
 def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_S_L $rs1, FRM_DYN)>;
 def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU $rs1, FRM_DYN)>;
 } // Predicates = [HasStdExtF, IsRV64]
+
+let Predicates = [HasStdExtZfinx, IsRV64] in {
+// Moves (no conversion)
+def : Pat<(riscv_fmv_w_x_rv64 GPR:$src), (COPY_TO_REGCLASS GPR:$src, GPRF32)>;
+def : Pat<(riscv_fmv_x_anyextw_rv64 GPRF32:$src), (COPY_TO_REGCLASS GPRF32:$src, GPR)>;
+
+// Use target specific isd nodes to help us remember the result is sign
+// extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be
+// duplicated if it has another user that didn't need the sign_extend.
+def : Pat<(riscv_any_fcvt_w_rv64 FPR32INX:$rs1, timm:$frm),  (FCVT_W_S_INX $rs1, timm:$frm)>;
+def : Pat<(riscv_any_fcvt_wu_rv64 FPR32INX:$rs1, timm:$frm), (FCVT_WU_S_INX $rs1, timm:$frm)>;
+
+// float->[u]int64. Round-to-zero must be used.
+def : Pat<(i64 (any_fp_to_sint FPR32INX:$rs1)), (FCVT_L_S_INX $rs1, FRM_RTZ)>;
+def : Pat<(i64 (any_fp_to_uint FPR32INX:$rs1)), (FCVT_LU_S_INX $rs1, FRM_RTZ)>;
+
+// Saturating float->[u]int64.
+def : Pat<(i64 (riscv_fcvt_x FPR32INX:$rs1, timm:$frm)), (FCVT_L_S_INX $rs1, timm:$frm)>;
+def : Pat<(i64 (riscv_fcvt_xu FPR32INX:$rs1, timm:$frm)), (FCVT_LU_S_INX $rs1, timm:$frm)>;
+
+// float->int64 with current rounding mode.
+def : Pat<(i64 (any_lrint FPR32INX:$rs1)), (FCVT_L_S_INX $rs1, FRM_DYN)>;
+def : Pat<(i64 (any_llrint FPR32INX:$rs1)), (FCVT_L_S_INX $rs1, FRM_DYN)>;
+
+// float->int64 rounded to neartest with ties rounded away from zero.
+def : Pat<(i64 (any_lround FPR32INX:$rs1)), (FCVT_L_S_INX $rs1, FRM_DYN)>;
+def : Pat<(i64 (any_llround FPR32INX:$rs1)), (FCVT_L_S_INX $rs1, FRM_DYN)>;
+
+// [u]int->fp. Match GCC and default to using dynamic rounding mode.
+def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_S_W_INX $rs1, FRM_DYN)>;
+def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_S_WU_INX $rs1, FRM_DYN)>;
+def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_S_L_INX $rs1, FRM_DYN)>;
+def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU_INX $rs1, FRM_DYN)>;
+} // Predicates = [HasStdExtZfinx, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 70ccdbf6e814d..3fd98e3d0bbf2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -801,7 +801,7 @@ defm : VPatIntegerSetCCSDNode_VI<"PseudoVMSGT",  SETGT>;
 defm : VPatIntegerSetCCSDNode_VI<"PseudoVMSGTU", SETUGT>;
 
 defm : VPatIntegerSetCCSDNode_VIPlus1<"PseudoVMSLE", SETLT,
-                                      SplatPat_simm5_plus1_nonzero>;
+                                      SplatPat_simm5_plus1>;
 defm : VPatIntegerSetCCSDNode_VIPlus1<"PseudoVMSLEU", SETULT,
                                       SplatPat_simm5_plus1_nonzero>;
 defm : VPatIntegerSetCCSDNode_VIPlus1<"PseudoVMSGT", SETGE,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 83e90ea75bc31..3887e0f401b2f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -219,9 +219,13 @@ def riscv_vfcvt_rm_x_f_vl : SDNode<"RISCVISD::VFCVT_RM_X_F_VL", SDT_RISCVFP2IOp_
 def riscv_vfcvt_rtz_xu_f_vl : SDNode<"RISCVISD::VFCVT_RTZ_XU_F_VL", SDT_RISCVFP2IOp_VL>;
 def riscv_vfcvt_rtz_x_f_vl  : SDNode<"RISCVISD::VFCVT_RTZ_X_F_VL",  SDT_RISCVFP2IOp_VL>;
 
+def riscv_strict_vfcvt_rm_x_f_vl : SDNode<"RISCVISD::STRICT_VFCVT_RM_X_F_VL", SDT_RISCVFP2IOp_RM_VL, [SDNPHasChain]>;
 def riscv_strict_vfcvt_rtz_xu_f_vl : SDNode<"RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL", SDT_RISCVFP2IOp_VL, [SDNPHasChain]>;
 def riscv_strict_vfcvt_rtz_x_f_vl  : SDNode<"RISCVISD::STRICT_VFCVT_RTZ_X_F_VL",  SDT_RISCVFP2IOp_VL, [SDNPHasChain]>;
 
+def any_riscv_vfcvt_rm_x_f_vl : PatFrags<(ops node:$src, node:$mask, node:$vl, node:$rm),
+                            [(riscv_vfcvt_rm_x_f_vl node:$src, node:$mask, node:$vl, node:$rm),
+                             (riscv_strict_vfcvt_rm_x_f_vl node:$src, node:$mask, node:$vl, node:$rm)]>;
 def any_riscv_vfcvt_rtz_xu_f_vl : PatFrags<(ops node:$src, node:$mask, node:$vl),
                             [(riscv_vfcvt_rtz_xu_f_vl node:$src, node:$mask, node:$vl),
                              (riscv_strict_vfcvt_rtz_xu_f_vl node:$src, node:$mask, node:$vl)]>;
@@ -246,6 +250,11 @@ def any_riscv_uint_to_fp_vl : PatFrags<(ops node:$src, node:$mask, node:$vl),
                              (riscv_strict_uint_to_fp_vl node:$src, node:$mask, node:$vl)]>;
 
 def riscv_vfround_noexcept_vl: SDNode<"RISCVISD::VFROUND_NOEXCEPT_VL", SDT_RISCVFPUnOp_VL>;
+def riscv_strict_vfround_noexcept_vl: SDNode<"RISCVISD::STRICT_VFROUND_NOEXCEPT_VL", SDT_RISCVFPUnOp_VL, [SDNPHasChain]>;
+
+def any_riscv_vfround_noexcept_vl : PatFrags<(ops node:$src, node:$mask, node:$vl),
+                        [(riscv_vfround_noexcept_vl node:$src, node:$mask, node:$vl),
+                         (riscv_strict_vfround_noexcept_vl node:$src, node:$mask, node:$vl)]>;
 
 def riscv_setcc_vl : SDNode<"RISCVISD::SETCC_VL", SDT_RISCVSETCCOP_VL>;
 def riscv_strict_fsetcc_vl : SDNode<"RISCVISD::STRICT_FSETCC_VL", SDT_RISCVSETCCOP_VL, [SDNPHasChain]>;
@@ -986,7 +995,7 @@ multiclass VPatConvertFP2IVL_V<SDPatternOperator vop, string instruction_name> {
   }
 }
 
-multiclass VPatConvertFP2I_RM_VL_V<SDNode vop, string instruction_name> {
+multiclass VPatConvertFP2I_RM_VL_V<SDPatternOperator vop, string instruction_name> {
   foreach fvti = AllFloatVectors in {
     defvar ivti = GetIntVTypeInfo<fvti>.Vti;
     def : Pat<(ivti.Vector (vop (fvti.Vector fvti.RegClass:$rs1),
@@ -1679,7 +1688,7 @@ foreach vti = AllIntegerVectors in {
   defm : VPatIntegerSetCCVL_VI_Swappable<vti, "PseudoVMSGTU", SETUGT, SETULT>;
 
   defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSLE",  SETLT, SETGT,
-                                              SplatPat_simm5_plus1_nonzero>;
+                                              SplatPat_simm5_plus1>;
   defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSLEU", SETULT, SETUGT,
                                               SplatPat_simm5_plus1_nonzero>;
   defm : VPatIntegerSetCCVL_VIPlus1_Swappable<vti, "PseudoVMSGT",  SETGE, SETLE,
@@ -1927,8 +1936,8 @@ foreach vti = AllFloatVectors in {
                  TAIL_AGNOSTIC)>;
 
   // Rounding without exception to implement nearbyint.
-  def : Pat<(riscv_vfround_noexcept_vl (vti.Vector vti.RegClass:$rs1),
-                                       (vti.Mask V0), VLOpFrag),
+  def : Pat<(any_riscv_vfround_noexcept_vl (vti.Vector vti.RegClass:$rs1),
+                                           (vti.Mask V0), VLOpFrag),
             (!cast<Instruction>("PseudoVFROUND_NOEXCEPT_V_" # vti.LMul.MX #"_MASK")
                   (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1,
                   (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>;
@@ -2032,7 +2041,7 @@ foreach fvti = AllFloatVectors in {
   defm : VPatConvertFP2IVL_V<riscv_vfcvt_xu_f_vl, "PseudoVFCVT_XU_F_V">;
   defm : VPatConvertFP2IVL_V<riscv_vfcvt_x_f_vl, "PseudoVFCVT_X_F_V">;
   defm : VPatConvertFP2I_RM_VL_V<riscv_vfcvt_rm_xu_f_vl, "PseudoVFCVT_RM_XU_F_V">;
-  defm : VPatConvertFP2I_RM_VL_V<riscv_vfcvt_rm_x_f_vl, "PseudoVFCVT_RM_X_F_V">;
+  defm : VPatConvertFP2I_RM_VL_V<any_riscv_vfcvt_rm_x_f_vl, "PseudoVFCVT_RM_X_F_V">;
 
   defm : VPatConvertFP2IVL_V<any_riscv_vfcvt_rtz_xu_f_vl, "PseudoVFCVT_RTZ_XU_F_V">;
   defm : VPatConvertFP2IVL_V<any_riscv_vfcvt_rtz_x_f_vl, "PseudoVFCVT_RTZ_X_F_V">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
index f43c180886cdb..7d701883e4b5a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
@@ -32,6 +32,13 @@ def uimm2_lsb0 : Operand<XLenVT>,
   }];
 }
 
+def uimm8ge32 : Operand<XLenVT> {
+  let ParserMatchClass = UImmAsmOperand<8, "GE32">;
+  let DecoderMethod = "decodeUImmOperand<8>";
+  let OperandType = "OPERAND_UIMM8_GE32";
+  let OperandNamespace = "RISCVOp";
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction Class Templates
 //===----------------------------------------------------------------------===//
@@ -125,6 +132,26 @@ def C_SH : CStoreH_rri<0b100011, 0b1, "c.sh">,
            Sched<[WriteSTH, ReadStoreData, ReadMemBase]>;
 }
 
+let Predicates = [HasStdExtZcmt],
+hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+def CM_JT : RVInst16CJ<0b101, 0b10, (outs), (ins uimm5:$index),
+                       "cm.jt", "$index">{
+  bits<5> index;
+
+  let Inst{12-7} = 0b000000;
+  let Inst{6-2} = index;
+}
+
+def CM_JALT : RVInst16CJ<0b101, 0b10, (outs), (ins uimm8ge32:$index),
+                         "cm.jalt", "$index">{
+  bits<8> index;
+
+  let Inst{12-10} = 0b000;
+  let Inst{9-2} = index;
+}
+} // Predicates = [HasStdExtZcmt]
+
+
 let Predicates = [HasStdExtZcb, HasStdExtMOrZmmul] in{
 def : CompressPat<(MUL GPRC:$rs1, GPRC:$rs1, GPRC:$rs2),
                   (C_MUL GPRC:$rs1, GPRC:$rs2)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
index fb2aa1bf0ef75..81333172c3126 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
@@ -94,9 +94,9 @@ def FMAXM_S: FPALU_rr<0b0010100, 0b011, "fmaxm.s", FPR32, /*Commutable*/ 1>;
 }
 
 def FROUND_S : FPUnaryOp_r_frm<0b0100000, 0b00100, FPR32, FPR32, "fround.s">,
-               Sched<[WriteFCvtF32ToF32, ReadFCvtF32ToF32]>;
+               Sched<[WriteFRoundF32, ReadFRoundF32]>;
 def FROUNDNX_S : FPUnaryOp_r_frm<0b0100000, 0b00101, FPR32, FPR32, "froundnx.s">,
-                 Sched<[WriteFCvtF32ToF32, ReadFCvtF32ToF32]>;
+                 Sched<[WriteFRoundF32, ReadFRoundF32]>;
 
 let SchedRW = [WriteFCmp32, ReadFCmp32, ReadFCmp32] in {
 def FLTQ_S : FPCmp_rr<0b1010000, 0b101, "fltq.s", FPR32>;
@@ -116,9 +116,9 @@ def FMAXM_D: FPALU_rr<0b0010101, 0b011, "fmaxm.d", FPR64, /*Commutable*/ 1>;
 }
 
 def FROUND_D : FPUnaryOp_r_frm<0b0100001, 0b00100, FPR64, FPR64, "fround.d">,
-               Sched<[WriteFCvtF64ToF64, ReadFCvtF64ToF64]>;
+               Sched<[WriteFRoundF64, ReadFRoundF64]>;
 def FROUNDNX_D : FPUnaryOp_r_frm<0b0100001, 0b00101, FPR64, FPR64, "froundnx.d">,
-                 Sched<[WriteFCvtF64ToF64, ReadFCvtF64ToF64]>;
+                 Sched<[WriteFRoundF64, ReadFRoundF64]>;
 
 def FCVTMOD_W_D
     : FPUnaryOp_r_rtz<0b1100001, 0b01000, GPR, FPR64, "fcvtmod.w.d">,
@@ -157,9 +157,9 @@ def FMAXM_H: FPALU_rr<0b0010110, 0b011, "fmaxm.h", FPR16, /*Commutable*/ 1>;
 }
 
 def FROUND_H : FPUnaryOp_r_frm<0b0100010, 0b00100, FPR16, FPR16, "fround.h">,
-               Sched<[WriteFCvtF16ToF16, ReadFCvtF16ToF16]>;
+               Sched<[WriteFRoundF16, ReadFRoundF16]>;
 def FROUNDNX_H : FPUnaryOp_r_frm<0b0100010, 0b00101, FPR16, FPR16, "froundnx.h">,
-                 Sched<[WriteFCvtF16ToF16, ReadFCvtF16ToF16]>;
+                 Sched<[WriteFRoundF16, ReadFRoundF16]>;
 
 let SchedRW = [WriteFCmp16, ReadFCmp16, ReadFCmp16] in {
 def FLTQ_H : FPCmp_rr<0b1010010, 0b101, "fltq.h", FPR16>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
index 5bda568c3cd0b..8bcc1d713a902 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -252,6 +252,8 @@ def : Pat<(any_fsqrt FPR16:$rs1), (FSQRT_H FPR16:$rs1, FRM_DYN)>;
 def : Pat<(fneg FPR16:$rs1), (FSGNJN_H $rs1, $rs1)>;
 def : Pat<(fabs FPR16:$rs1), (FSGNJX_H $rs1, $rs1)>;
 
+def : Pat<(riscv_fpclass FPR16:$rs1), (FCLASS_H $rs1)>;
+
 def : PatFprFpr<fcopysign, FSGNJ_H, FPR16>;
 def : Pat<(fcopysign FPR16:$rs1, (fneg FPR16:$rs2)), (FSGNJN_H $rs1, $rs2)>;
 def : Pat<(fcopysign FPR16:$rs1, FPR32:$rs2),
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZihintntl.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZihintntl.td
deleted file mode 100644
index ecc5ddedee007..0000000000000
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZihintntl.td
+++ /dev/null
@@ -1,22 +0,0 @@
-//===RISCVInstrInfoZihintntl.td - 'Zihintntl' instructions -*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// This file describes the RISC-V instructions from Non-Temporal Locality 
-/// Hints extension document (zihintntl).
-///
-//===----------------------------------------------------------------------===//
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 4 in {
-  def PseudoNTLALL :  Pseudo<(outs), (ins), [], "ntl.all">, 
-                             PseudoInstExpansion<(ADD X0, X0, X5)>;
-}
-
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 2 in {
-  def PseudoCNTLALL :  Pseudo<(outs), (ins), [], "c.ntl.all">,
-                              PseudoInstExpansion<(C_ADD_HINT X0, X0, X5)>;
-}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 1e27e4306b840..ea7240af691cd 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -138,6 +138,7 @@ let Predicates = [HasStdExtZvbb] in {
   def  VREV8_V  : VALUVs2<0b010010, 0b01001, OPMVV, "vrev8.v">;
   defm VROL_V   : VALU_IV_V_X<"vrol", 0b010101>;
   defm VROR_V   : VROR_IV_V_X_I<"vror", 0b010100>;
+  let Constraints = "@earlyclobber $vd", RVVConstraint = WidenV in
   defm VWSLL_V  : VALU_IV_V_X_I<"vwsll", 0b110101, uimm5>;
 } // Predicates = [HasStdExtZvbb]
 
diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket.td b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
index 79130e63a36fc..a92c0770bfadd 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedRocket.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
@@ -18,9 +18,9 @@ def RocketModel : SchedMachineModel {
   let MispredictPenalty = 3;
   let CompleteModel = false;
   let UnsupportedFeatures = [HasStdExtZbkb, HasStdExtZbkc, HasStdExtZbkx,
-                             HasStdExtZknd, HasStdExtZkne, HasStdExtZknh,
-                             HasStdExtZksed, HasStdExtZksh, HasStdExtZkr,
-                             HasVInstructions, HasVInstructionsI64];
+                             HasStdExtZcmt, HasStdExtZknd, HasStdExtZkne,
+                             HasStdExtZknh, HasStdExtZksed, HasStdExtZksh,
+                             HasStdExtZkr, HasVInstructions, HasVInstructionsI64];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 46c8a0bbf63a7..28454563b99a8 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -16,9 +16,9 @@ def SiFive7Model : SchedMachineModel {
   let MispredictPenalty = 3;
   let CompleteModel = 0;
   let UnsupportedFeatures = [HasStdExtZbkb, HasStdExtZbkc, HasStdExtZbkx,
-                             HasStdExtZknd, HasStdExtZkne, HasStdExtZknh,
-                             HasStdExtZksed, HasStdExtZksh, HasStdExtZkr,
-                             HasVInstructions];
+                             HasStdExtZcmt, HasStdExtZknd, HasStdExtZkne,
+                             HasStdExtZknh, HasStdExtZksed, HasStdExtZksh,
+                             HasStdExtZkr, HasVInstructions];
 }
 
 // The SiFive7 microarchitecture has two pipelines: A and B.
diff --git a/llvm/lib/Target/RISCV/RISCVSchedule.td b/llvm/lib/Target/RISCV/RISCVSchedule.td
index c345d835b53eb..62244a99a90d5 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedule.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedule.td
@@ -76,9 +76,11 @@ def WriteFCvtF16ToF32  : SchedWrite;
 def WriteFCvtF32ToF16  : SchedWrite;
 def WriteFCvtF16ToF64  : SchedWrite;
 def WriteFCvtF64ToF16  : SchedWrite;
-def WriteFCvtF32ToF32  : SchedWrite;
-def WriteFCvtF64ToF64  : SchedWrite;
-def WriteFCvtF16ToF16  : SchedWrite;
+
+// Zfa found instructions.
+def WriteFRoundF32     : SchedWrite;
+def WriteFRoundF64     : SchedWrite;
+def WriteFRoundF16     : SchedWrite;
 
 def WriteFClass16   : SchedWrite;    // 16-bit floating point classify
 def WriteFClass32   : SchedWrite;    // 32-bit floating point classify
@@ -189,9 +191,9 @@ def ReadFCvtF16ToF32     : SchedRead;
 def ReadFCvtF32ToF16     : SchedRead;
 def ReadFCvtF16ToF64     : SchedRead;
 def ReadFCvtF64ToF16     : SchedRead;
-def ReadFCvtF32ToF32     : SchedRead;
-def ReadFCvtF64ToF64     : SchedRead;
-def ReadFCvtF16ToF16     : SchedRead;
+def ReadFRoundF16        : SchedRead;
+def ReadFRoundF32        : SchedRead;
+def ReadFRoundF64        : SchedRead;
 def ReadFClass16         : SchedRead;
 def ReadFClass32         : SchedRead;
 def ReadFClass64         : SchedRead;
@@ -252,16 +254,16 @@ def : ReadAdvance<ReadSFB, 0>;
 
 multiclass UnsupportedSchedZfa {
 let Unsupported = true in {
-def : WriteRes<WriteFCvtF32ToF32, []>;
-def : WriteRes<WriteFCvtF64ToF64, []>;
-def : WriteRes<WriteFCvtF16ToF16, []>;
+def : WriteRes<WriteFRoundF16, []>;
+def : WriteRes<WriteFRoundF32, []>;
+def : WriteRes<WriteFRoundF64, []>;
 def : WriteRes<WriteFLI16, []>;
 def : WriteRes<WriteFLI32, []>;
 def : WriteRes<WriteFLI64, []>;
 
-def : ReadAdvance<ReadFCvtF32ToF32, 0>;
-def : ReadAdvance<ReadFCvtF64ToF64, 0>;
-def : ReadAdvance<ReadFCvtF16ToF16, 0>;
+def : ReadAdvance<ReadFRoundF32, 0>;
+def : ReadAdvance<ReadFRoundF64, 0>;
+def : ReadAdvance<ReadFRoundF16, 0>;
 } // Unsupported = true
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index ccc6f34ce09ac..576b185e74730 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -113,6 +113,8 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
 
   bool hasStdExtCOrZca() const { return HasStdExtC || HasStdExtZca; }
   bool hasStdExtZvl() const { return ZvlLen != 0; }
+  bool hasStdExtFOrZfinx() const { return HasStdExtF || HasStdExtZfinx; }
+  bool hasStdExtDOrZdinx() const { return HasStdExtD || HasStdExtZdinx; }
   bool hasStdExtZfhOrZfhmin() const { return HasStdExtZfh || HasStdExtZfhmin; }
   bool is64Bit() const { return IsRV64; }
   MVT getXLenVT() const { return XLenVT; }
diff --git a/llvm/lib/Target/RISCV/RISCVSystemOperands.td b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
index 524ed0b45ec10..6c9bb3bbd6ec7 100644
--- a/llvm/lib/Target/RISCV/RISCVSystemOperands.td
+++ b/llvm/lib/Target/RISCV/RISCVSystemOperands.td
@@ -67,26 +67,9 @@ def lookupSysRegByDeprecatedName : SearchIndex {
 }
 
 // The following CSR encodings match those given in Tables 2.2,
-// 2.3, 2.4 and  2.5 in the RISC-V Instruction Set Manual
+// 2.3, 2.4, 2.5 and 2.6 in the RISC-V Instruction Set Manual
 // Volume II: Privileged Architecture.
 
-//===----------------------------------------------------------------------===//
-// User Trap Setup
-//===----------------------------------------------------------------------===//
-def : SysReg<"ustatus", 0x000>;
-def : SysReg<"uie", 0x004>;
-def : SysReg<"utvec", 0x005>;
-
-//===----------------------------------------------------------------------===//
-// User Trap Handling
-//===----------------------------------------------------------------------===//
-def : SysReg<"uscratch", 0x040>;
-def : SysReg<"uepc", 0x041>;
-def : SysReg<"ucause", 0x042>;
-let DeprecatedName = "ubadaddr" in
-def : SysReg<"utval", 0x043>;
-def : SysReg<"uip", 0x044>;
-
 //===----------------------------------------------------------------------===//
 // User Floating-Point CSRs
 //===----------------------------------------------------------------------===//
@@ -120,8 +103,6 @@ foreach i = 3...31 in
 // Supervisor Trap Setup
 //===----------------------------------------------------------------------===//
 def : SysReg<"sstatus", 0x100>;
-def : SysReg<"sedeleg", 0x102>;
-def : SysReg<"sideleg", 0x103>;
 def : SysReg<"sie", 0x104>;
 def : SysReg<"stvec", 0x105>;
 def : SysReg<"scounteren", 0x106>;
@@ -378,3 +359,56 @@ foreach i = 0...3 in {
 //===-----------------------------------------------
 
 def SEED : SysReg<"seed", 0x015>;
+
+//===-----------------------------------------------
+// Advanced Interrupt Architecture
+//===-----------------------------------------------
+
+// Machine-level CSRs
+def : SysReg<"miselect", 0x350>;
+def : SysReg<"mireg", 0x351>;
+def : SysReg<"mtopei", 0x35C>;
+def : SysReg<"mtopi", 0xFB0>;
+def : SysReg<"mvien", 0x308>;
+def : SysReg<"mvip", 0x309>;
+let isRV32Only = 1 in {
+def : SysReg<"midelegh", 0x313>;
+def : SysReg<"mieh", 0x314>;
+def : SysReg<"mvienh", 0x318>;
+def : SysReg<"mviph", 0x319>;
+def : SysReg<"miph", 0x354>;
+} // isRV32Only
+
+// Supervisor-level CSRs
+def : SysReg<"siselect", 0x150>;
+def : SysReg<"sireg", 0x151>;
+def : SysReg<"stopei", 0x15C>;
+def : SysReg<"stopi", 0xDB0>;
+let isRV32Only = 1 in {
+def : SysReg<"sieh", 0x114>;
+def : SysReg<"siph", 0x154>;
+} // isRV32Only
+
+// Hypervisor and VS CSRs
+def : SysReg<"hvien", 0x608>;
+def : SysReg<"hvictl", 0x609>;
+def : SysReg<"hviprio1", 0x646>;
+def : SysReg<"hviprio2", 0x647>;
+def : SysReg<"vsiselect", 0x250>;
+def : SysReg<"vsireg", 0x251>;
+def : SysReg<"vstopei", 0x25C>;
+def : SysReg<"vstopi", 0xEB0>;
+let isRV32Only = 1 in {
+def : SysReg<"hidelegh", 0x613>;
+def : SysReg<"hvienh", 0x618>;
+def : SysReg<"hviph", 0x655>;
+def : SysReg<"hviprio1h", 0x656>;
+def : SysReg<"hviprio2h", 0x657>;
+def : SysReg<"vsieh", 0x214>;
+def : SysReg<"vsiph", 0x254>;
+} // isRV32Only
+
+// Jump Vector Table CSR
+//===-----------------------------------------------
+
+def : SysReg<"jvt", 0x017>;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 75a9ed7634f11..5943847640569 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -83,7 +83,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVOptWInstrsPass(*PR);
   initializeRISCVPreRAExpandPseudoPass(*PR);
   initializeRISCVExpandPseudoPass(*PR);
-  initializeRISCVInsertNTLHInstsPass(*PR);
   initializeRISCVInsertVSETVLIPass(*PR);
   initializeRISCVDAGToDAGISelPass(*PR);
   initializeRISCVInitUndefPass(*PR);
@@ -349,7 +348,6 @@ void RISCVPassConfig::addPreEmitPass() {
 
 void RISCVPassConfig::addPreEmitPass2() {
   addPass(createRISCVExpandPseudoPass());
-  addPass(createRISCVInsertNTLHInstsPass());
 
   // Schedule the expansion of AMOs at the last possible moment, avoiding the
   // possibility for other passes to break the requirements for forward
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 0eb79122070fa..5319bff98f700 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -179,22 +179,25 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
       const Instruction *CxtI = nullptr);
 
   bool isElementTypeLegalForScalableVector(Type *Ty) const {
-    return TLI->isLegalElementTypeForRVV(Ty);
+    return TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, Ty));
   }
 
   bool isLegalMaskedLoadStore(Type *DataType, Align Alignment) {
     if (!ST->hasVInstructions())
       return false;
 
+    EVT DataTypeVT = TLI->getValueType(DL, DataType);
+
     // Only support fixed vectors if we know the minimum vector size.
-    if (isa<FixedVectorType>(DataType) && !ST->useRVVForFixedLengthVectors())
+    if (DataTypeVT.isFixedLengthVector() && !ST->useRVVForFixedLengthVectors())
       return false;
 
-    if (Alignment <
-        DL.getTypeStoreSize(DataType->getScalarType()).getFixedValue())
+    EVT ElemType = DataTypeVT.getScalarType();
+    if (!ST->enableUnalignedVectorMem() && Alignment < ElemType.getStoreSize())
       return false;
 
-    return TLI->isLegalElementTypeForRVV(DataType->getScalarType());
+    return TLI->isLegalElementTypeForRVV(ElemType);
+
   }
 
   bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
@@ -208,15 +211,17 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     if (!ST->hasVInstructions())
       return false;
 
+    EVT DataTypeVT = TLI->getValueType(DL, DataType);
+
     // Only support fixed vectors if we know the minimum vector size.
-    if (isa<FixedVectorType>(DataType) && !ST->useRVVForFixedLengthVectors())
+    if (DataTypeVT.isFixedLengthVector() && !ST->useRVVForFixedLengthVectors())
       return false;
 
-    if (Alignment <
-        DL.getTypeStoreSize(DataType->getScalarType()).getFixedValue())
+    EVT ElemType = DataTypeVT.getScalarType();
+    if (!ST->enableUnalignedVectorMem() && Alignment < ElemType.getStoreSize())
       return false;
 
-    return TLI->isLegalElementTypeForRVV(DataType->getScalarType());
+    return TLI->isLegalElementTypeForRVV(ElemType);
   }
 
   bool isLegalMaskedGather(Type *DataType, Align Alignment) {
@@ -260,7 +265,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
       return true;
 
     Type *Ty = RdxDesc.getRecurrenceType();
-    if (!TLI->isLegalElementTypeForRVV(Ty))
+    if (!TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, Ty)))
       return false;
 
     switch (RdxDesc.getRecurrenceKind()) {
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index bcecf141ecdd4..ab9aa20809103 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -39,6 +39,7 @@ add_llvm_target(SPIRVCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   Demangle
   GlobalISel
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/SPIRV/MCTargetDesc/CMakeLists.txt
index cf3e9011ac129..580db513245fd 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMSPIRVDesc
   SPIRVInstPrinter.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   SPIRVInfo
   Support
diff --git a/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp b/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp
index 69681fd2a54ed..3a51e29dcf161 100644
--- a/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVRegularizer.cpp
@@ -171,9 +171,9 @@ void SPIRVRegularizer::visitCallInst(CallInst &CI) {
     return;
 
   auto MangledName = F->getName();
-  size_t n;
-  int status;
-  char *NameStr = itaniumDemangle(F->getName().data(), nullptr, &n, &status);
+  char *NameStr = itaniumDemangle(F->getName().data());
+  if (!NameStr)
+    return;
   StringRef DemangledName(NameStr);
 
   // TODO: add support for other builtins.
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 514dc6497c470..f4f3cdce1ac3c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -293,16 +293,11 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
     return std::string();
 
   // Try to use the itanium demangler.
-  size_t n;
-  int Status;
-  char *DemangledName = itaniumDemangle(Name.data(), nullptr, &n, &Status);
-
-  if (Status == demangle_success) {
+  if (char *DemangledName = itaniumDemangle(Name.data())) {
     std::string Result = DemangledName;
     free(DemangledName);
     return Result;
   }
-  free(DemangledName);
   // Otherwise use simple demangling to return the function name.
   if (IsNonMangledOCL || IsNonMangledSPIRV)
     return Name.str();
diff --git a/llvm/lib/Target/Sparc/CMakeLists.txt b/llvm/lib/Target/Sparc/CMakeLists.txt
index c32840a182dea..6195843a77ce6 100644
--- a/llvm/lib/Target/Sparc/CMakeLists.txt
+++ b/llvm/lib/Target/Sparc/CMakeLists.txt
@@ -32,6 +32,7 @@ add_llvm_target(SparcCodeGen
   LINK_COMPONENTS
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   SelectionDAG
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index aa89488bbb62a..2c0696e8048b5 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -41,11 +41,14 @@ static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
   case Sparc::fixup_sparc_br19:
     return (Value >> 2) & 0x7ffff;
 
-  case Sparc::fixup_sparc_br16_2:
-    return (Value >> 2) & 0xc000;
-
-  case Sparc::fixup_sparc_br16_14:
-    return (Value >> 2) & 0x3fff;
+  case Sparc::fixup_sparc_br16: {
+    // A.3 Branch on Integer Register with Prediction (BPr)
+    // Inst{21-20} = d16hi;
+    // Inst{13-0}  = d16lo;
+    unsigned d16hi = (Value >> 16) & 0x3;
+    unsigned d16lo = (Value >> 2) & 0x3fff;
+    return (d16hi << 20) | d16lo;
+  }
 
   case Sparc::fixup_sparc_hix22:
     return (~Value >> 10) & 0x3fffff;
@@ -164,8 +167,7 @@ namespace {
         { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_br22",      10,     22,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_br19",      13,     19,  MCFixupKindInfo::FKF_IsPCRel },
-        { "fixup_sparc_br16_2",    10,      2,  MCFixupKindInfo::FKF_IsPCRel },
-        { "fixup_sparc_br16_14",   18,     14,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_br16",       0,     32,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_13",        19,     13,  0 },
         { "fixup_sparc_hi22",      10,     22,  0 },
         { "fixup_sparc_lo10",      22,     10,  0 },
@@ -211,8 +213,7 @@ namespace {
         { "fixup_sparc_call30",     0,     30,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_br22",       0,     22,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_br19",       0,     19,  MCFixupKindInfo::FKF_IsPCRel },
-        { "fixup_sparc_br16_2",    20,      2,  MCFixupKindInfo::FKF_IsPCRel },
-        { "fixup_sparc_br16_14",    0,     14,  MCFixupKindInfo::FKF_IsPCRel },
+        { "fixup_sparc_br16",      32,      0,  MCFixupKindInfo::FKF_IsPCRel },
         { "fixup_sparc_13",         0,     13,  0 },
         { "fixup_sparc_hi22",       0,     22,  0 },
         { "fixup_sparc_lo10",       0,     10,  0 },
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
index 9c50c41f6bf2e..c48beab012292 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
@@ -62,6 +62,8 @@ unsigned SparcELFObjectWriter::getRelocType(MCContext &Ctx,
     case Sparc::fixup_sparc_call30:  return ELF::R_SPARC_WDISP30;
     case Sparc::fixup_sparc_br22:    return ELF::R_SPARC_WDISP22;
     case Sparc::fixup_sparc_br19:    return ELF::R_SPARC_WDISP19;
+    case Sparc::fixup_sparc_br16:
+      return ELF::R_SPARC_WDISP16;
     case Sparc::fixup_sparc_pc22:    return ELF::R_SPARC_PC22;
     case Sparc::fixup_sparc_pc10:    return ELF::R_SPARC_PC10;
     case Sparc::fixup_sparc_wplt30:  return ELF::R_SPARC_WPLT30;
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h b/llvm/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h
index 701d8513e6579..3b91326589894 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h
@@ -26,8 +26,7 @@ namespace llvm {
       fixup_sparc_br19,
 
       /// fixup_sparc_bpr  - 16-bit fixup for bpr
-      fixup_sparc_br16_2,
-      fixup_sparc_br16_14,
+      fixup_sparc_br16,
 
       /// fixup_sparc_13 - 13-bit fixup
       fixup_sparc_13,
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
index b2e784a0c2545..51a6732d05c6d 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp
@@ -197,6 +197,10 @@ void SparcInstPrinter::printCCOperand(const MCInst *MI, int opNum,
     // Make sure CC is a cp conditional flag.
     CC = (CC < SPCC::CPCC_BEGIN) ? (CC + SPCC::CPCC_BEGIN) : CC;
     break;
+  case SP::BPR:
+  case SP::BPRA:
+  case SP::BPRNT:
+  case SP::BPRANT:
   case SP::MOVRri:
   case SP::MOVRrr:
   case SP::FMOVRS:
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index ee460002fc586..e69319fb9e264 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -235,10 +235,8 @@ getBranchOnRegTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)Sparc::fixup_sparc_br16_2));
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)Sparc::fixup_sparc_br16_14));
+  Fixups.push_back(
+      MCFixup::create(0, MO.getExpr(), (MCFixupKind)Sparc::fixup_sparc_br16));
 
   return 0;
 }
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 660c141369364..13e18734c9179 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1977,6 +1977,8 @@ const char *SparcTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case SPISD::BRFCC:           return "SPISD::BRFCC";
   case SPISD::BRFCC_V9:
     return "SPISD::BRFCC_V9";
+  case SPISD::BR_REG:
+    return "SPISD::BR_REG";
   case SPISD::SELECT_ICC:      return "SPISD::SELECT_ICC";
   case SPISD::SELECT_XCC:      return "SPISD::SELECT_XCC";
   case SPISD::SELECT_FCC:      return "SPISD::SELECT_FCC";
@@ -2582,7 +2584,7 @@ static SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG,
 
 static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
                           const SparcTargetLowering &TLI, bool hasHardQuad,
-                          bool isV9) {
+                          bool isV9, bool is64Bit) {
   SDValue Chain = Op.getOperand(0);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
   SDValue LHS = Op.getOperand(2);
@@ -2599,6 +2601,15 @@ static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
   // Get the condition flag.
   SDValue CompareFlag;
   if (LHS.getValueType().isInteger()) {
+    // On V9 processors running in 64-bit mode, if CC compares two `i64`s
+    // and the RHS is zero we might be able to use a specialized branch.
+    const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
+    if (is64Bit && isV9 && LHS.getValueType() == MVT::i64 && RHSC &&
+        RHSC->isZero() && !ISD::isUnsignedIntSetCC(CC))
+      return DAG.getNode(SPISD::BR_REG, dl, MVT::Other, Chain, Dest,
+                         DAG.getConstant(intCondCCodeToRcond(CC), dl, MVT::i32),
+                         LHS);
+
     CompareFlag = DAG.getNode(SPISD::CMPICC, dl, MVT::Glue, LHS, RHS);
     if (SPCC == ~0U) SPCC = IntCondCCodeToICC(CC);
     if (isV9)
@@ -3213,7 +3224,7 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::UINT_TO_FP:         return LowerUINT_TO_FP(Op, DAG, *this,
                                                        hasHardQuad);
   case ISD::BR_CC:
-    return LowerBR_CC(Op, DAG, *this, hasHardQuad, isV9);
+    return LowerBR_CC(Op, DAG, *this, hasHardQuad, isV9, is64Bit);
   case ISD::SELECT_CC:
     return LowerSELECT_CC(Op, DAG, *this, hasHardQuad, isV9, is64Bit);
   case ISD::VASTART:            return LowerVASTART(Op, DAG, *this);
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.h b/llvm/lib/Target/Sparc/SparcISelLowering.h
index 0af20271f1b23..5504dcd464fb2 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.h
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.h
@@ -31,6 +31,7 @@ namespace llvm {
     BPXCC,    // Branch to dest on xcc condition, with prediction (64-bit only).
     BRFCC,    // Branch to dest on fcc condition
     BRFCC_V9, // Branch to dest on fcc condition (v9 variant).
+    BR_REG,   // Branch to dest using the comparison of a register with zero.
     SELECT_ICC, // Select between two values using the current ICC flags.
     SELECT_XCC, // Select between two values using the current XCC flags.
     SELECT_FCC, // Select between two values using the current FCC flags.
diff --git a/llvm/lib/Target/Sparc/SparcInstr64Bit.td b/llvm/lib/Target/Sparc/SparcInstr64Bit.td
index 77f203fd0d680..0a64794874188 100644
--- a/llvm/lib/Target/Sparc/SparcInstr64Bit.td
+++ b/llvm/lib/Target/Sparc/SparcInstr64Bit.td
@@ -356,15 +356,15 @@ def FMOVQ_XCC : F4_3<0b110101, 0b000011, (outs QFPRegs:$rd),
 
 // Branch On integer register with Prediction (BPr).
 let isBranch = 1, isTerminator = 1, hasDelaySlot = 1 in
-multiclass BranchOnReg<bits<3> cond, string OpcStr> {
-  def napt : F2_4<cond, 0, 1, (outs), (ins I64Regs:$rs1, bprtarget16:$imm16),
-             !strconcat(OpcStr, " $rs1, $imm16"), []>;
-  def apt  : F2_4<cond, 1, 1, (outs), (ins I64Regs:$rs1, bprtarget16:$imm16),
-             !strconcat(OpcStr, ",a $rs1, $imm16"), []>;
-  def napn  : F2_4<cond, 0, 0, (outs), (ins I64Regs:$rs1, bprtarget16:$imm16),
-             !strconcat(OpcStr, ",pn $rs1, $imm16"), []>;
-  def apn : F2_4<cond, 1, 0, (outs), (ins I64Regs:$rs1, bprtarget16:$imm16),
-             !strconcat(OpcStr, ",a,pn $rs1, $imm16"), []>;
+multiclass BranchOnReg<list<dag> CCPattern> {
+  def R    : F2_4<0, 1, (outs), (ins bprtarget16:$imm16, RegCCOp:$rcond, I64Regs:$rs1),
+             "br$rcond $rs1, $imm16", CCPattern>;
+  def RA   : F2_4<1, 1, (outs), (ins bprtarget16:$imm16, RegCCOp:$rcond, I64Regs:$rs1),
+             "br$rcond,a $rs1, $imm16", []>;
+  def RNT  : F2_4<0, 0, (outs), (ins bprtarget16:$imm16, RegCCOp:$rcond, I64Regs:$rs1),
+             "br$rcond,pn $rs1, $imm16", []>;
+  def RANT : F2_4<1, 0, (outs), (ins bprtarget16:$imm16, RegCCOp:$rcond, I64Regs:$rs1),
+             "br$rcond,a,pn $rs1, $imm16", []>;
 }
 
 multiclass bpr_alias<string OpcStr, Instruction NAPT, Instruction APT> {
@@ -374,19 +374,8 @@ multiclass bpr_alias<string OpcStr, Instruction NAPT, Instruction APT> {
                   (APT I64Regs:$rs1, bprtarget16:$imm16), 0>;
 }
 
-defm BPZ   : BranchOnReg<0b001, "brz">;
-defm BPLEZ : BranchOnReg<0b010, "brlez">;
-defm BPLZ  : BranchOnReg<0b011, "brlz">;
-defm BPNZ  : BranchOnReg<0b101, "brnz">;
-defm BPGZ  : BranchOnReg<0b110, "brgz">;
-defm BPGEZ : BranchOnReg<0b111, "brgez">;
-
-defm : bpr_alias<"brz",   BPZnapt,   BPZapt  >;
-defm : bpr_alias<"brlez", BPLEZnapt, BPLEZapt>;
-defm : bpr_alias<"brlz",  BPLZnapt,  BPLZapt >;
-defm : bpr_alias<"brnz",  BPNZnapt,  BPNZapt >;
-defm : bpr_alias<"brgz",  BPGZnapt,  BPGZapt >;
-defm : bpr_alias<"brgez", BPGEZnapt, BPGEZapt>;
+let Predicates = [Is64Bit] in
+  defm BP : BranchOnReg<[(SPbrreg bb:$imm16, imm:$rcond, i64:$rs1)]>;
 
 // Move integer register on register condition (MOVr).
 let Predicates = [Is64Bit], Constraints = "$f = $rd" in {
diff --git a/llvm/lib/Target/Sparc/SparcInstrAliases.td b/llvm/lib/Target/Sparc/SparcInstrAliases.td
index f10021321406c..01c3696cc7bca 100644
--- a/llvm/lib/Target/Sparc/SparcInstrAliases.td
+++ b/llvm/lib/Target/Sparc/SparcInstrAliases.td
@@ -295,6 +295,36 @@ multiclass cp_cond_alias<string cond, int condVal> {
 
 // Instruction aliases for register conditional branches and moves.
 multiclass reg_cond_alias<string rcond, int condVal> {
+  // br<rcond> $rs1, $imm
+  def : InstAlias<!strconcat(!strconcat("br", rcond), " $rs1, $imm"),
+                  (BPR bprtarget16:$imm, condVal, I64Regs:$rs1)>,
+                  Requires<[Is64Bit]>;
+
+  // br<rcond>,pt $rs1, $imm
+  def : InstAlias<!strconcat(!strconcat("br", rcond), ",pt $rs1, $imm"),
+                  (BPR bprtarget16:$imm, condVal, I64Regs:$rs1)>,
+                  Requires<[Is64Bit]>;
+
+  // br<rcond>,pn $rs1, $imm
+  def : InstAlias<!strconcat(!strconcat("br", rcond), ",pn $rs1, $imm"),
+                  (BPRNT bprtarget16:$imm, condVal, I64Regs:$rs1)>,
+                  Requires<[Is64Bit]>;
+
+  // br<rcond>,a $rs1, $imm
+  def : InstAlias<!strconcat(!strconcat("br", rcond), ",a $rs1, $imm"),
+                  (BPRA bprtarget16:$imm, condVal, I64Regs:$rs1)>,
+                  Requires<[Is64Bit]>;
+
+  // br<rcond>,a,pt $rs1, $imm
+  def : InstAlias<!strconcat(!strconcat("br", rcond), ",a,pt $rs1, $imm"),
+                  (BPRA bprtarget16:$imm, condVal, I64Regs:$rs1)>,
+                  Requires<[Is64Bit]>;
+
+  // br<rcond>,a,pn $rs1, $imm
+  def : InstAlias<!strconcat(!strconcat("br", rcond), ",a,pn $rs1, $imm"),
+                  (BPRANT bprtarget16:$imm, condVal, I64Regs:$rs1)>,
+                  Requires<[Is64Bit]>;
+
   defm : regcond_mov_alias<rcond, condVal,
                             MOVRrr, MOVRri,
                             FMOVRS, FMOVRD, FMOVRQ>,
diff --git a/llvm/lib/Target/Sparc/SparcInstrFormats.td b/llvm/lib/Target/Sparc/SparcInstrFormats.td
index 522dcd96a1125..c67b591ab98ac 100644
--- a/llvm/lib/Target/Sparc/SparcInstrFormats.td
+++ b/llvm/lib/Target/Sparc/SparcInstrFormats.td
@@ -83,17 +83,18 @@ class F2_3<bits<3> op2Val, bit annul, bit pred,
   let Inst{18-0}  = imm19;
 }
 
-class F2_4<bits<3> cond, bit annul, bit pred, dag outs, dag ins,
+class F2_4<bit annul, bit pred, dag outs, dag ins,
            string asmstr, list<dag> pattern, InstrItinClass itin = NoItinerary>
    : InstSP<outs, ins, asmstr, pattern, itin> {
   bits<16> imm16;
   bits<5>  rs1;
+  bits<3>  rcond;
 
   let op          = 0;    // op = 0
 
   let Inst{29}    = annul;
   let Inst{28}    = 0;
-  let Inst{27-25} = cond;
+  let Inst{27-25} = rcond;
   let Inst{24-22} = 0b011;
   let Inst{21-20} = imm16{15-14};
   let Inst{19}    = pred;
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
index a3a09a36f1dd4..90662cd87dcf1 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -32,6 +32,10 @@ static cl::opt<unsigned> BPccDisplacementBits(
     "sparc-bpcc-offset-bits", cl::Hidden, cl::init(19),
     cl::desc("Restrict range of BPcc/FBPfcc instructions (DEBUG)"));
 
+static cl::opt<unsigned>
+    BPrDisplacementBits("sparc-bpr-offset-bits", cl::Hidden, cl::init(16),
+                        cl::desc("Restrict range of BPr instructions (DEBUG)"));
+
 // Pin the vtable to this file.
 void SparcInstrInfo::anchor() {}
 
@@ -166,6 +170,11 @@ static bool isI64CondBranchOpcode(int Opc) {
          Opc == SP::BPXCCANT;
 }
 
+static bool isRegCondBranchOpcode(int Opc) {
+  return Opc == SP::BPR || Opc == SP::BPRA || Opc == SP::BPRNT ||
+         Opc == SP::BPRANT;
+}
+
 static bool isFCondBranchOpcode(int Opc) {
   return Opc == SP::FBCOND || Opc == SP::FBCONDA || Opc == SP::FBCOND_V9 ||
          Opc == SP::FBCONDA_V9;
@@ -173,7 +182,7 @@ static bool isFCondBranchOpcode(int Opc) {
 
 static bool isCondBranchOpcode(int Opc) {
   return isI32CondBranchOpcode(Opc) || isI64CondBranchOpcode(Opc) ||
-         isFCondBranchOpcode(Opc);
+         isRegCondBranchOpcode(Opc) || isFCondBranchOpcode(Opc);
 }
 
 static bool isIndirectBranchOpcode(int Opc) {
@@ -190,6 +199,13 @@ static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
   Cond.push_back(MachineOperand::CreateImm(Opc));
   Cond.push_back(MachineOperand::CreateImm(CC));
 
+  // Branch on register contents need another argument to indicate
+  // the register it branches on.
+  if (isRegCondBranchOpcode(Opc)) {
+      Register Reg = LastInst->getOperand(2).getReg();
+      Cond.push_back(MachineOperand::CreateReg(Reg, false));
+  }
+
   Target = LastInst->getOperand(0).getMBB();
 }
 
@@ -217,6 +233,10 @@ SparcInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
   case SP::BPFCCANT:
   case SP::FBCOND_V9:
   case SP::FBCONDA_V9:
+  case SP::BPR:
+  case SP::BPRA:
+  case SP::BPRNT:
+  case SP::BPRANT:
       return MI.getOperand(0).getMBB();
   }
 }
@@ -311,8 +331,8 @@ unsigned SparcInstrInfo::insertBranch(MachineBasicBlock &MBB,
                                       const DebugLoc &DL,
                                       int *BytesAdded) const {
   assert(TBB && "insertBranch must not be told to insert a fallthrough");
-  assert((Cond.size() <= 2) &&
-         "Sparc branch conditions should have at most two components!");
+  assert((Cond.size() <= 3) &&
+         "Sparc branch conditions should have at most three components!");
 
   if (Cond.empty()) {
     assert(!FBB && "Unconditional branch with multiple successors!");
@@ -325,7 +345,12 @@ unsigned SparcInstrInfo::insertBranch(MachineBasicBlock &MBB,
   // Conditional branch
   unsigned Opc = Cond[0].getImm();
   unsigned CC = Cond[1].getImm();
-  BuildMI(&MBB, DL, get(Opc)).addMBB(TBB).addImm(CC);
+  if (isRegCondBranchOpcode(Opc)) {
+    Register Reg = Cond[2].getReg();
+    BuildMI(&MBB, DL, get(Opc)).addMBB(TBB).addImm(CC).addReg(Reg);
+  } else {
+    BuildMI(&MBB, DL, get(Opc)).addMBB(TBB).addImm(CC);
+  }
 
   if (!FBB) {
     if (BytesAdded)
@@ -367,7 +392,7 @@ unsigned SparcInstrInfo::removeBranch(MachineBasicBlock &MBB,
 
 bool SparcInstrInfo::reverseBranchCondition(
     SmallVectorImpl<MachineOperand> &Cond) const {
-  assert(Cond.size() <= 2);
+  assert(Cond.size() <= 3);
   SPCC::CondCodes CC = static_cast<SPCC::CondCodes>(Cond[1].getImm());
   Cond[1].setImm(GetOppositeBranchCondition(CC));
   return false;
@@ -399,6 +424,12 @@ bool SparcInstrInfo::isBranchOffsetInRange(unsigned BranchOpc,
   case SP::FBCOND_V9:
   case SP::FBCONDA_V9:
     return isIntN(BPccDisplacementBits, Offset >> 2);
+
+  case SP::BPR:
+  case SP::BPRA:
+  case SP::BPRNT:
+  case SP::BPRANT:
+    return isIntN(BPrDisplacementBits, Offset >> 2);
   }
 
   llvm_unreachable("Unknown branch instruction!");
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td
index c10e4a0efa339..a55b7c3f5f18b 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.td
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td
@@ -224,6 +224,8 @@ def SDTSPcmpfcc :
 SDTypeProfile<0, 2, [SDTCisFP<0>, SDTCisSameAs<0, 1>]>;
 def SDTSPbrcc :
 SDTypeProfile<0, 2, [SDTCisVT<0, OtherVT>, SDTCisVT<1, i32>]>;
+def SDTSPbrreg :
+SDTypeProfile<0, 3, [SDTCisVT<0, OtherVT>, SDTCisVT<1, i32>, SDTCisVT<2, i64>]>;
 def SDTSPselectcc :
 SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisVT<3, i32>]>;
 def SDTSPselectreg :
@@ -253,6 +255,7 @@ def SPbpicc : SDNode<"SPISD::BPICC", SDTSPbrcc, [SDNPHasChain, SDNPInGlue]>;
 def SPbpxcc : SDNode<"SPISD::BPXCC", SDTSPbrcc, [SDNPHasChain, SDNPInGlue]>;
 def SPbrfcc : SDNode<"SPISD::BRFCC", SDTSPbrcc, [SDNPHasChain, SDNPInGlue]>;
 def SPbrfccv9 : SDNode<"SPISD::BRFCC_V9", SDTSPbrcc, [SDNPHasChain, SDNPInGlue]>;
+def SPbrreg : SDNode<"SPISD::BR_REG", SDTSPbrreg, [SDNPHasChain, SDNPInGlue]>;
 
 def SPhi    : SDNode<"SPISD::Hi", SDTIntUnaryOp>;
 def SPlo    : SDNode<"SPISD::Lo", SDTIntUnaryOp>;
diff --git a/llvm/lib/Target/SystemZ/CMakeLists.txt b/llvm/lib/Target/SystemZ/CMakeLists.txt
index ba845a74949fa..0776cfcaf21b3 100644
--- a/llvm/lib/Target/SystemZ/CMakeLists.txt
+++ b/llvm/lib/Target/SystemZ/CMakeLists.txt
@@ -43,6 +43,7 @@ add_llvm_target(SystemZCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   Scalar
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
index ea6f5b7cb27d8..055482ee31b05 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/CMakeLists.txt
@@ -7,6 +7,7 @@ add_llvm_component_library(LLVMSystemZDesc
   SystemZMCTargetDesc.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   Support
   SystemZInfo
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index a07a11a7ce930..256c0fa036b4d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -186,8 +186,8 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::USUBO, VT, Custom);
 
       // Support carry in as value rather than glue.
-      setOperationAction(ISD::ADDCARRY, VT, Custom);
-      setOperationAction(ISD::SUBCARRY, VT, Custom);
+      setOperationAction(ISD::UADDO_CARRY, VT, Custom);
+      setOperationAction(ISD::USUBO_CARRY, VT, Custom);
 
       // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and
       // stores, putting a serialization instruction after the stores.
@@ -3995,20 +3995,20 @@ SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
 }
 
 static bool isAddCarryChain(SDValue Carry) {
-  while (Carry.getOpcode() == ISD::ADDCARRY)
+  while (Carry.getOpcode() == ISD::UADDO_CARRY)
     Carry = Carry.getOperand(2);
   return Carry.getOpcode() == ISD::UADDO;
 }
 
 static bool isSubBorrowChain(SDValue Carry) {
-  while (Carry.getOpcode() == ISD::SUBCARRY)
+  while (Carry.getOpcode() == ISD::USUBO_CARRY)
     Carry = Carry.getOperand(2);
   return Carry.getOpcode() == ISD::USUBO;
 }
 
-// Lower ADDCARRY/SUBCARRY nodes.
-SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
-                                                SelectionDAG &DAG) const {
+// Lower UADDO_CARRY/USUBO_CARRY nodes.
+SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
+                                                   SelectionDAG &DAG) const {
 
   SDNode *N = Op.getNode();
   MVT VT = N->getSimpleValueType(0);
@@ -4027,7 +4027,7 @@ SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
 
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Unknown instruction!");
-  case ISD::ADDCARRY:
+  case ISD::UADDO_CARRY:
     if (!isAddCarryChain(Carry))
       return SDValue();
 
@@ -4035,7 +4035,7 @@ SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
     CCValid = SystemZ::CCMASK_LOGICAL;
     CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
     break;
-  case ISD::SUBCARRY:
+  case ISD::USUBO_CARRY:
     if (!isSubBorrowChain(Carry))
       return SDValue();
 
@@ -5741,9 +5741,9 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
   case ISD::UADDO:
   case ISD::USUBO:
     return lowerXALUO(Op, DAG);
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY:
-    return lowerADDSUBCARRY(Op, DAG);
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:
+    return lowerUADDSUBO_CARRY(Op, DAG);
   case ISD::OR:
     return lowerOR(Op, DAG);
   case ISD::CTPOP:
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 8c98eb80f9ee0..7f6e25ccf6f7a 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -658,7 +658,7 @@ class SystemZTargetLowering : public TargetLowering {
   SDValue lowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerXALUO(SDValue Op, SelectionDAG &DAG) const;
-  SDValue lowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/VE/AsmParser/CMakeLists.txt b/llvm/lib/Target/VE/AsmParser/CMakeLists.txt
index 825c75c8e942e..1312ae3ea1035 100644
--- a/llvm/lib/Target/VE/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/VE/AsmParser/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMVEAsmParser
   VEAsmParser.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/lib/Target/VE/CMakeLists.txt b/llvm/lib/Target/VE/CMakeLists.txt
index d14e1a596e480..45f8293cfd22f 100644
--- a/llvm/lib/Target/VE/CMakeLists.txt
+++ b/llvm/lib/Target/VE/CMakeLists.txt
@@ -32,6 +32,7 @@ add_llvm_target(VECodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   SelectionDAG
diff --git a/llvm/lib/Target/VE/Disassembler/CMakeLists.txt b/llvm/lib/Target/VE/Disassembler/CMakeLists.txt
index 94580e19df264..52b86b8149de4 100644
--- a/llvm/lib/Target/VE/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/VE/Disassembler/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMVEDisassembler
   VEDisassembler.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt
index 314d8450b13cd..34a9e2044f309 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMVEDesc
   VETargetStreamer.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   Support
   TargetParser
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
index 5a562d77f941b..6611e4c42eb2b 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCTargetDesc.cpp
@@ -73,6 +73,10 @@ static MCTargetStreamer *createTargetAsmStreamer(MCStreamer &S,
   return new VETargetAsmStreamer(S, OS);
 }
 
+static MCTargetStreamer *createNullTargetStreamer(MCStreamer &S) {
+  return new VETargetStreamer(S);
+}
+
 static MCInstPrinter *createVEMCInstPrinter(const Triple &T,
                                             unsigned SyntaxVariant,
                                             const MCAsmInfo &MAI,
@@ -108,6 +112,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeVETargetMC() {
     // Register the asm streamer.
     TargetRegistry::RegisterAsmTargetStreamer(*T, createTargetAsmStreamer);
 
+    // Register the null streamer.
+    TargetRegistry::RegisterNullTargetStreamer(*T, createNullTargetStreamer);
+
     // Register the MCInstPrinter
     TargetRegistry::RegisterMCInstPrinter(*T, createVEMCInstPrinter);
   }
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VETargetStreamer.h b/llvm/lib/Target/VE/MCTargetDesc/VETargetStreamer.h
index 1704e0b585591..ab85c29aaaf50 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VETargetStreamer.h
+++ b/llvm/lib/Target/VE/MCTargetDesc/VETargetStreamer.h
@@ -20,9 +20,9 @@ class VETargetStreamer : public MCTargetStreamer {
 public:
   VETargetStreamer(MCStreamer &S);
   /// Emit ".register <reg>, #ignore".
-  virtual void emitVERegisterIgnore(unsigned reg) = 0;
+  virtual void emitVERegisterIgnore(unsigned reg){};
   /// Emit ".register <reg>, #scratch".
-  virtual void emitVERegisterScratch(unsigned reg) = 0;
+  virtual void emitVERegisterScratch(unsigned reg){};
 };
 
 // This part is for ascii assembly output
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/CMakeLists.txt b/llvm/lib/Target/WebAssembly/AsmParser/CMakeLists.txt
index 52a64007c717a..95546f9559597 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/CMakeLists.txt
+++ b/llvm/lib/Target/WebAssembly/AsmParser/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMWebAssemblyAsmParser
   WebAssemblyAsmTypeCheck.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/lib/Target/WebAssembly/CMakeLists.txt b/llvm/lib/Target/WebAssembly/CMakeLists.txt
index afdb00ba1653c..e4476a37ca16e 100644
--- a/llvm/lib/Target/WebAssembly/CMakeLists.txt
+++ b/llvm/lib/Target/WebAssembly/CMakeLists.txt
@@ -66,6 +66,7 @@ add_llvm_target(WebAssemblyCodeGen
   AsmPrinter
   BinaryFormat
   CodeGen
+  CodeGenTypes
   Core
   MC
   Scalar
diff --git a/llvm/lib/Target/WebAssembly/Disassembler/CMakeLists.txt b/llvm/lib/Target/WebAssembly/Disassembler/CMakeLists.txt
index c7793ae3089d2..6a32e525adcc7 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/WebAssembly/Disassembler/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMWebAssemblyDisassembler
   WebAssemblyDisassembler.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt
index 1c83a1e9a7dd5..ccecb0c149188 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMWebAssemblyDesc
   WebAssemblyWasmObjectWriter.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   Support
   TargetParser
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
index c81c3a3d9ffaa..51e4d3656ba4a 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h
@@ -16,8 +16,8 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/BinaryFormat/Wasm.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/MC/MCInstPrinter.h"
-#include "llvm/Support/MachineValueType.h"
 
 namespace llvm {
 
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
index 522f6356c28b6..72d36a251a917 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
@@ -16,8 +16,8 @@
 #define LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYTARGETSTREAMER_H
 
 #include "llvm/BinaryFormat/Wasm.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/Support/MachineValueType.h"
 
 namespace llvm {
 
diff --git a/llvm/lib/Target/WebAssembly/Utils/CMakeLists.txt b/llvm/lib/Target/WebAssembly/Utils/CMakeLists.txt
index 95c4d8fc40e26..cc685791cf521 100644
--- a/llvm/lib/Target/WebAssembly/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/WebAssembly/Utils/CMakeLists.txt
@@ -4,6 +4,7 @@ add_llvm_component_library(LLVMWebAssemblyUtils
 
   LINK_COMPONENTS
   CodeGen
+  CodeGenTypes
   Core
   MC
   Support
diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
index 79fc677456c35..ebf3849cd7bed 100644
--- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
+++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
@@ -17,9 +17,9 @@
 
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "llvm/BinaryFormat/Wasm.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/MC/MCSymbolWasm.h"
-#include "llvm/Support/MachineValueType.h"
 
 namespace llvm {
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index e8cea6c0f74a3..e0e6e8deb4d5e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -376,7 +376,7 @@ multiclass SIMDStoreLane<Vec vec, bits<32> simdop> {
            [], name#"\t${off}(${addr})$p2align, $vec, $idx",
            name#"\t$off$p2align, $idx", simdop>;
   defm STORE_LANE_#vec#_A64 :
-    SIMD_I<(outs V128:$dst),
+    SIMD_I<(outs),
            (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx,
                 I64:$addr, V128:$vec),
            (outs), (ins P2Align:$p2align, offset64_op:$off, vec_i8imm_op:$idx),
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 56a82f9144a98..2c18966ac0fce 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -1748,12 +1748,6 @@ bool X86AsmParser::CreateMemForMSInlineAsm(
   // If we found a decl other than a VarDecl, then assume it is a FuncDecl or
   // some other label reference.
   if (Info.isKind(InlineAsmIdentifierInfo::IK_Label)) {
-    // Insert an explicit size if the user didn't have one.
-    if (!Size) {
-      Size = getPointerWidth();
-      InstInfo->AsmRewrites->emplace_back(AOK_SizeDirective, Start,
-                                          /*Len=*/0, Size);
-    }
     // Create an absolute memory reference in order to match against
     // instructions taking a PC relative operand.
     Operands.push_back(X86Operand::CreateMem(getPointerWidth(), Disp, Start,
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 7c167ad280657..8388ddd567c30 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -95,6 +95,7 @@ add_llvm_target(X86CodeGen ${sources}
   AsmPrinter
   CFGuard
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   Instrumentation
diff --git a/llvm/lib/Target/X86/MCA/CMakeLists.txt b/llvm/lib/Target/X86/MCA/CMakeLists.txt
index d03dabb1f27ad..e925c68318439 100644
--- a/llvm/lib/Target/X86/MCA/CMakeLists.txt
+++ b/llvm/lib/Target/X86/MCA/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMX86TargetMCA
   X86CustomBehaviour.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCA
   MCParser
diff --git a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
index d021ff68371f4..4db4721bfe296 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
@@ -18,6 +18,7 @@ add_llvm_component_library(LLVMX86Desc
 
   LINK_COMPONENTS
   BinaryFormat
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 04a918bd93b5e..160384b8ffe75 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -501,7 +501,6 @@ class X86MCInstrAnalysis : public MCInstrAnalysis {
                             APInt &Mask) const override;
   std::vector<std::pair<uint64_t, uint64_t>>
   findPltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
-                 uint64_t GotSectionVA,
                  const Triple &TargetTriple) const override;
 
   bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
@@ -570,8 +569,7 @@ bool X86MCInstrAnalysis::clearsSuperRegisters(const MCRegisterInfo &MRI,
 }
 
 static std::vector<std::pair<uint64_t, uint64_t>>
-findX86PltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
-                  uint64_t GotPltSectionVA) {
+findX86PltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents) {
   // Do a lightweight parsing of PLT entries.
   std::vector<std::pair<uint64_t, uint64_t>> Result;
   for (uint64_t Byte = 0, End = PltContents.size(); Byte + 6 < End; ) {
@@ -580,8 +578,7 @@ findX86PltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
       // The jmp instruction at the beginning of each PLT entry jumps to the
       // address of the base of the .got.plt section plus the immediate.
       uint32_t Imm = support::endian::read32le(PltContents.data() + Byte + 2);
-      Result.push_back(
-          std::make_pair(PltSectionVA + Byte, GotPltSectionVA + Imm));
+      Result.emplace_back(PltSectionVA + Byte, ~static_cast<uint64_t>(Imm));
       Byte += 6;
     } else if (PltContents[Byte] == 0xff && PltContents[Byte + 1] == 0x25) {
       // The jmp instruction at the beginning of each PLT entry jumps to the
@@ -614,17 +611,18 @@ findX86_64PltEntries(uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents) {
   return Result;
 }
 
-std::vector<std::pair<uint64_t, uint64_t>> X86MCInstrAnalysis::findPltEntries(
-    uint64_t PltSectionVA, ArrayRef<uint8_t> PltContents,
-    uint64_t GotPltSectionVA, const Triple &TargetTriple) const {
+std::vector<std::pair<uint64_t, uint64_t>>
+X86MCInstrAnalysis::findPltEntries(uint64_t PltSectionVA,
+                                   ArrayRef<uint8_t> PltContents,
+                                   const Triple &TargetTriple) const {
   switch (TargetTriple.getArch()) {
-    case Triple::x86:
-      return findX86PltEntries(PltSectionVA, PltContents, GotPltSectionVA);
-    case Triple::x86_64:
-      return findX86_64PltEntries(PltSectionVA, PltContents);
-    default:
-      return {};
-    }
+  case Triple::x86:
+    return findX86PltEntries(PltSectionVA, PltContents);
+  case Triple::x86_64:
+    return findX86_64PltEntries(PltSectionVA, PltContents);
+  default:
+    return {};
+  }
 }
 
 bool X86MCInstrAnalysis::evaluateBranch(const MCInst &Inst, uint64_t Addr,
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 4e725d7829809..ff1e1eeebc1dd 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -23,6 +23,7 @@
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/InlineAsm.h"
@@ -42,7 +43,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Target/X86/X86CallLowering.cpp b/llvm/lib/Target/X86/X86CallLowering.cpp
index 919f4f9e119b8..fc0f4b1dee81b 100644
--- a/llvm/lib/Target/X86/X86CallLowering.cpp
+++ b/llvm/lib/Target/X86/X86CallLowering.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -33,6 +34,7 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
@@ -41,8 +43,6 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cassert>
 #include <cstdint>
 
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index cd6bf0c3729db..e2149951aff0e 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -646,7 +646,7 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
     case X86ISD::XOR:
     case X86ISD::OR:
     case ISD::ADD:
-    case ISD::ADDCARRY:
+    case ISD::UADDO_CARRY:
     case ISD::AND:
     case ISD::OR:
     case ISD::XOR: {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7ec6175712a4c..0b1b9768f32e3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1002,6 +1002,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                     : &X86::VR128RegClass);
 
+    setOperationAction(ISD::FMAXIMUM,           MVT::f32, Custom);
+    setOperationAction(ISD::FMINIMUM,           MVT::f32, Custom);
+
     setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);
     setOperationAction(ISD::FABS,               MVT::v4f32, Custom);
     setOperationAction(ISD::FCOPYSIGN,          MVT::v4f32, Custom);
@@ -1038,6 +1041,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                     : &X86::VR128RegClass);
 
+    setOperationAction(ISD::FMAXIMUM,           MVT::f64, Custom);
+    setOperationAction(ISD::FMINIMUM,           MVT::f64, Custom);
+
     for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
                      MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
       setOperationAction(ISD::SDIV, VT, Custom);
@@ -1079,6 +1085,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
 
     setOperationAction(ISD::ABDU,               MVT::v16i8, Custom);
+    setOperationAction(ISD::ABDU,               MVT::v8i16, Custom);
     setOperationAction(ISD::ABDS,               MVT::v8i16, Custom);
 
     setOperationAction(ISD::UADDSAT,            MVT::v16i8, Legal);
@@ -1679,7 +1686,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
   }
 
-  // This block controls legalization for 512-bit operations with 32/64 bit
+  // This block controls legalization for 512-bit operations with 8/16/32/64 bit
   // elements. 512-bits can be disabled based on prefer-vector-width and
   // required-vector-width function attributes.
   if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
@@ -2026,8 +2033,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   }
 
   // This block control legalization of v32i1/v64i1 which are available with
-  // AVX512BW. 512-bit v32i16 and v64i8 vector legalization is controlled with
-  // useBWIRegs.
+  // AVX512BW..
   if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
     addRegisterClass(MVT::v32i1,  &X86::VK32RegClass);
     addRegisterClass(MVT::v64i1,  &X86::VK64RegClass);
@@ -2124,6 +2130,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::STRICT_FROUNDEVEN,    MVT::f16, Legal);
     setOperationAction(ISD::FP_ROUND,             MVT::f16, Custom);
     setOperationAction(ISD::STRICT_FP_ROUND,      MVT::f16, Custom);
+    setOperationAction(ISD::FMAXIMUM,             MVT::f16, Custom);
+    setOperationAction(ISD::FMINIMUM,             MVT::f16, Custom);
     setOperationAction(ISD::FP_EXTEND,            MVT::f32, Legal);
     setOperationAction(ISD::STRICT_FP_EXTEND,     MVT::f32, Legal);
 
@@ -2331,8 +2339,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::UMULO, VT, Custom);
 
     // Support carry in as value rather than glue.
-    setOperationAction(ISD::ADDCARRY, VT, Custom);
-    setOperationAction(ISD::SUBCARRY, VT, Custom);
+    setOperationAction(ISD::UADDO_CARRY, VT, Custom);
+    setOperationAction(ISD::USUBO_CARRY, VT, Custom);
     setOperationAction(ISD::SETCCCARRY, VT, Custom);
     setOperationAction(ISD::SADDO_CARRY, VT, Custom);
     setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
@@ -11231,6 +11239,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   APInt ZeroMask = APInt::getZero(NumElems);
   APInt NonZeroMask = APInt::getZero(NumElems);
   bool IsAllConstants = true;
+  bool OneUseFrozenUndefs = true;
   SmallSet<SDValue, 8> Values;
   unsigned NumConstants = NumElems;
   for (unsigned i = 0; i < NumElems; ++i) {
@@ -11239,7 +11248,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
       UndefMask.setBit(i);
       continue;
     }
-    if (Elt.getOpcode() == ISD::FREEZE && Elt.getOperand(0).isUndef()) {
+    if (ISD::isFreezeUndef(Elt.getNode())) {
+      OneUseFrozenUndefs = OneUseFrozenUndefs && Elt->hasOneUse();
       FrozenUndefMask.setBit(i);
       continue;
     }
@@ -11259,10 +11269,14 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   if (UndefMask.isAllOnes())
     return DAG.getUNDEF(VT);
 
-  // All undef/freeze(undef) vector. Return an FREEZE UNDEF.
-  if ((UndefMask | FrozenUndefMask).isAllOnes())
+  // All undef/freeze(undef) vector. Return a FREEZE UNDEF.
+  if (OneUseFrozenUndefs && (UndefMask | FrozenUndefMask).isAllOnes())
     return DAG.getFreeze(DAG.getUNDEF(VT));
 
+  // All undef/freeze(undef)/zero vector. Return a zero vector.
+  if ((UndefMask | FrozenUndefMask | ZeroMask).isAllOnes())
+    return getZeroVector(VT, Subtarget, DAG, dl);
+
   // If we have multiple FREEZE-UNDEF operands, we are likely going to end up
   // lowering into a suboptimal insertion sequence. Instead, thaw the UNDEF in
   // our source BUILD_VECTOR, create another FREEZE-UNDEF splat BUILD_VECTOR,
@@ -11657,8 +11671,13 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
     SDValue SubVec = Op.getOperand(i);
     if (SubVec.isUndef())
       continue;
-    if (ISD::isFreezeUndef(SubVec.getNode()) && SubVec.hasOneUse())
-      ++NumFreezeUndef;
+    if (ISD::isFreezeUndef(SubVec.getNode())) {
+        // If the freeze(undef) has multiple uses then we must fold to zero.
+        if (SubVec.hasOneUse())
+          ++NumFreezeUndef;
+        else
+          ++NumZero;
+    }
     else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
       ++NumZero;
     else {
@@ -13332,12 +13351,20 @@ static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
   SmallVector<int, 32> PermuteMask(NumElts, -1);
   for (int Elt = 0; Elt != NumElts; ++Elt) {
     int M = Mask[Elt];
+    if (M < 0)
+      continue;
+    int NormM = M;
     if (NumElts <= M)
-      PermuteMask[Elt] = NumLaneElts * ((M - NumElts) / NumLaneElts) +
-                         (2 * (M % NumHalfLaneElts)) + 1;
-    else if (0 <= M)
-      PermuteMask[Elt] =
-          NumLaneElts * (M / NumLaneElts) + (2 * (M % NumHalfLaneElts));
+      NormM -= NumElts;
+    bool IsFirstOp = M < NumElts;
+    int BaseMaskElt =
+        NumLaneElts * (NormM / NumLaneElts) + (2 * (NormM % NumHalfLaneElts));
+    if ((IsFirstOp && V1 == Ops[0]) || (!IsFirstOp && V2 == Ops[0]))
+      PermuteMask[Elt] = BaseMaskElt;
+    else if ((IsFirstOp && V1 == Ops[1]) || (!IsFirstOp && V2 == Ops[1]))
+      PermuteMask[Elt] = BaseMaskElt + 1;
+    assert(PermuteMask[Elt] != -1 &&
+           "Input mask element is defined but failed to assign permute mask");
   }
 
   unsigned UnpckOp = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
@@ -30198,6 +30225,136 @@ static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
   return SDValue();
 }
 
+static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
+                                      SelectionDAG &DAG) {
+  assert((Op.getOpcode() == ISD::FMAXIMUM || Op.getOpcode() == ISD::FMINIMUM) &&
+         "Expected FMAXIMUM or FMINIMUM opcode");
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT VT = Op.getValueType();
+  SDValue X = Op.getOperand(0);
+  SDValue Y = Op.getOperand(1);
+  SDLoc DL(Op);
+  uint64_t SizeInBits = VT.getFixedSizeInBits();
+  APInt PreferredZero = APInt::getZero(SizeInBits);
+  EVT IVT = MVT::getIntegerVT(SizeInBits);
+  X86ISD::NodeType MinMaxOp;
+  if (Op.getOpcode() == ISD::FMAXIMUM) {
+    MinMaxOp = X86ISD::FMAX;
+  } else {
+    PreferredZero.setSignBit();
+    MinMaxOp = X86ISD::FMIN;
+  }
+  EVT SetCCType =
+      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+
+  // The tables below show the expected result of Max in cases of NaN and
+  // signed zeros.
+  //
+  //                 Y                       Y
+  //             Num   xNaN              +0     -0
+  //          ---------------         ---------------
+  //     Num  |  Max |   Y  |     +0  |  +0  |  +0  |
+  // X        ---------------  X      ---------------
+  //    xNaN  |   X  |  X/Y |     -0  |  +0  |  -0  |
+  //          ---------------         ---------------
+  //
+  // It is achieved by means of FMAX/FMIN with preliminary checks and operand
+  // reordering.
+  //
+  // We check if any of operands is NaN and return NaN. Then we check if any of
+  // operands is zero or negative zero (for fmaximum and fminimum respectively)
+  // to ensure the correct zero is returned.
+  auto IsPreferredZero = [PreferredZero](SDValue Op) {
+    Op = peekThroughBitcasts(Op);
+    if (auto *CstOp = dyn_cast<ConstantFPSDNode>(Op))
+      return CstOp->getValueAPF().bitcastToAPInt() == PreferredZero;
+    if (auto *CstOp = dyn_cast<ConstantSDNode>(Op))
+      return CstOp->getAPIntValue() == PreferredZero;
+    return false;
+  };
+
+  bool IsXNeverNaN = DAG.isKnownNeverNaN(X);
+  bool IsYNeverNaN = DAG.isKnownNeverNaN(Y);
+  bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath ||
+                          Op->getFlags().hasNoSignedZeros() ||
+                          DAG.isKnownNeverZeroFloat(X) ||
+                          DAG.isKnownNeverZeroFloat(Y);
+  SDValue NewX, NewY;
+  if (IgnoreSignedZero || IsPreferredZero(Y)) {
+    // Operands are already in right order or order does not matter.
+    NewX = X;
+    NewY = Y;
+  } else if (IsPreferredZero(X)) {
+    NewX = Y;
+    NewY = X;
+  } else if ((VT == MVT::f16 || Subtarget.hasDQI()) &&
+             (Op->getFlags().hasNoNaNs() || IsXNeverNaN || IsYNeverNaN)) {
+    if (IsXNeverNaN)
+      std::swap(X, Y);
+    // VFPCLASSS consumes a vector type. So provide a minimal one corresponded
+    // xmm register.
+    MVT VectorType = MVT::getVectorVT(VT.getSimpleVT(), 128 / SizeInBits);
+    SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X);
+    // Bits of classes:
+    // Bits  Imm8[0] Imm8[1] Imm8[2] Imm8[3] Imm8[4]  Imm8[5]  Imm8[6] Imm8[7]
+    // Class    QNAN PosZero NegZero  PosINF  NegINF Denormal Negative    SNAN
+    SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101,
+                                        DL, MVT::i32);
+    SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm);
+    SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
+                              DAG.getConstant(0, DL, MVT::v8i1), IsNanZero,
+                              DAG.getIntPtrConstant(0, DL));
+    SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins);
+    NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X);
+    NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y);
+    return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
+  } else {
+    SDValue IsXSigned;
+    if (Subtarget.is64Bit() || VT != MVT::f64) {
+      SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
+      SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
+      IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT);
+    } else {
+      assert(VT == MVT::f64);
+      SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
+                                DAG.getConstantFP(0, DL, MVT::v2f64), X,
+                                DAG.getIntPtrConstant(0, DL));
+      SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
+      SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
+                               DAG.getIntPtrConstant(1, DL));
+      Hi = DAG.getBitcast(MVT::i32, Hi);
+      SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
+      EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(),
+                                             *DAG.getContext(), MVT::i32);
+      IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT);
+    }
+    if (MinMaxOp == X86ISD::FMAX) {
+      NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y);
+      NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X);
+    } else {
+      NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X);
+      NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y);
+    }
+  }
+
+  bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath ||
+                   Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN);
+
+  // If we did no ordering operands for singed zero handling and we need
+  // to process NaN and we know that the second operand is not NaN then put
+  // it in first operand and we will not need to post handle NaN after max/min.
+  if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
+    std::swap(NewX, NewY);
+
+  SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
+
+  if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
+    return MinMax;
+
+  SDValue IsNaN = DAG.getSetCC(DL, SetCCType, NewX, NewX, ISD::SETUO);
+  return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax);
+}
+
 static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
                         SelectionDAG &DAG) {
   MVT VT = Op.getSimpleValueType();
@@ -30226,6 +30383,12 @@ static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
     return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
   }
 
+  // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
+  if (!IsSigned && TLI.isOperationLegal(ISD::USUBSAT, VT))
+    return DAG.getNode(ISD::OR, dl, VT,
+                       DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS),
+                       DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS));
+
   // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
   // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
   EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
@@ -33339,7 +33502,7 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
   return Swap.getValue(1);
 }
 
-static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
+static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
   SDNode *N = Op.getNode();
   MVT VT = N->getSimpleValueType(0);
   unsigned Opc = Op.getOpcode();
@@ -33357,7 +33520,7 @@ static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
                       Carry, DAG.getAllOnesConstant(DL, CarryVT));
 
-  bool IsAdd = Opc == ISD::ADDCARRY || Opc == ISD::SADDO_CARRY;
+  bool IsAdd = Opc == ISD::UADDO_CARRY || Opc == ISD::SADDO_CARRY;
   SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
                             Op.getOperand(0), Op.getOperand(1),
                             Carry.getValue(1));
@@ -33932,8 +34095,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::BITCAST:            return LowerBITCAST(Op, Subtarget, DAG);
   case ISD::SADDO_CARRY:
   case ISD::SSUBO_CARRY:
-  case ISD::ADDCARRY:
-  case ISD::SUBCARRY:           return LowerADDSUBCARRY(Op, DAG);
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:        return LowerADDSUBO_CARRY(Op, DAG);
   case ISD::ADD:
   case ISD::SUB:                return lowerAddSub(Op, DAG, Subtarget);
   case ISD::UADDSAT:
@@ -33944,6 +34107,9 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SMIN:
   case ISD::UMAX:
   case ISD::UMIN:               return LowerMINMAX(Op, Subtarget, DAG);
+  case ISD::FMINIMUM:
+  case ISD::FMAXIMUM:
+    return LowerFMINIMUM_FMAXIMUM(Op, Subtarget, DAG);
   case ISD::ABS:                return LowerABS(Op, Subtarget, DAG);
   case ISD::ABDS:
   case ISD::ABDU:               return LowerABD(Op, Subtarget, DAG);
@@ -38576,6 +38742,23 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     Known.Zero.setBitsFrom(16);
     break;
   }
+  case X86ISD::PCMPGT:
+  case X86ISD::PCMPEQ: {
+    KnownBits KnownLhs =
+        DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    KnownBits KnownRhs =
+        DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    std::optional<bool> Res = Opc == X86ISD::PCMPEQ
+                                  ? KnownBits::eq(KnownLhs, KnownRhs)
+                                  : KnownBits::sgt(KnownLhs, KnownRhs);
+    if (Res) {
+      if (*Res)
+        Known.setAllOnes();
+      else
+        Known.setAllZero();
+    }
+    break;
+  }
   case X86ISD::PMULUDQ: {
     KnownBits Known2;
     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 004104c860fd7..560115ac69ba4 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineCombinerPattern.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -9749,5 +9750,141 @@ X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
   return It;
 }
 
+bool X86InstrInfo::getMachineCombinerPatterns(
+    MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
+    bool DoRegPressureReduce) const {
+  unsigned Opc = Root.getOpcode();
+  switch (Opc) {
+  default:
+    return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
+                                                       DoRegPressureReduce);
+  case X86::VPDPWSSDrr:
+  case X86::VPDPWSSDrm:
+  case X86::VPDPWSSDYrr:
+  case X86::VPDPWSSDYrm: {
+    Patterns.push_back(MachineCombinerPattern::DPWSSD);
+    return true;
+  }
+  case X86::VPDPWSSDZ128r:
+  case X86::VPDPWSSDZ128m:
+  case X86::VPDPWSSDZ256r:
+  case X86::VPDPWSSDZ256m:
+  case X86::VPDPWSSDZr:
+  case X86::VPDPWSSDZm: {
+    if (Subtarget.hasBWI())
+      Patterns.push_back(MachineCombinerPattern::DPWSSD);
+    return true;
+  }
+  }
+}
+
+static void
+genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII,
+                             SmallVectorImpl<MachineInstr *> &InsInstrs,
+                             SmallVectorImpl<MachineInstr *> &DelInstrs,
+                             DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) {
+  MachineFunction *MF = Root.getMF();
+  MachineRegisterInfo &RegInfo = MF->getRegInfo();
+
+  unsigned Opc = Root.getOpcode();
+  unsigned AddOpc = 0;
+  unsigned MaddOpc = 0;
+  switch (Opc) {
+  default:
+    assert(false && "It should not reach here");
+    break;
+  // vpdpwssd xmm2,xmm3,xmm1
+  // -->
+  // vpmaddwd xmm3,xmm3,xmm1
+  // vpaddd xmm2,xmm2,xmm3
+  case X86::VPDPWSSDrr:
+    MaddOpc = X86::VPMADDWDrr;
+    AddOpc = X86::VPADDDrr;
+    break;
+  case X86::VPDPWSSDrm:
+    MaddOpc = X86::VPMADDWDrm;
+    AddOpc = X86::VPADDDrr;
+    break;
+  case X86::VPDPWSSDZ128r:
+    MaddOpc = X86::VPMADDWDZ128rr;
+    AddOpc = X86::VPADDDZ128rr;
+    break;
+  case X86::VPDPWSSDZ128m:
+    MaddOpc = X86::VPMADDWDZ128rm;
+    AddOpc = X86::VPADDDZ128rr;
+    break;
+  // vpdpwssd ymm2,ymm3,ymm1
+  // -->
+  // vpmaddwd ymm3,ymm3,ymm1
+  // vpaddd ymm2,ymm2,ymm3
+  case X86::VPDPWSSDYrr:
+    MaddOpc = X86::VPMADDWDYrr;
+    AddOpc = X86::VPADDDYrr;
+    break;
+  case X86::VPDPWSSDYrm:
+    MaddOpc = X86::VPMADDWDYrm;
+    AddOpc = X86::VPADDDYrr;
+    break;
+  case X86::VPDPWSSDZ256r:
+    MaddOpc = X86::VPMADDWDZ256rr;
+    AddOpc = X86::VPADDDZ256rr;
+    break;
+  case X86::VPDPWSSDZ256m:
+    MaddOpc = X86::VPMADDWDZ256rm;
+    AddOpc = X86::VPADDDZ256rr;
+    break;
+  // vpdpwssd zmm2,zmm3,zmm1
+  // -->
+  // vpmaddwd zmm3,zmm3,zmm1
+  // vpaddd zmm2,zmm2,zmm3
+  case X86::VPDPWSSDZr:
+    MaddOpc = X86::VPMADDWDZrr;
+    AddOpc = X86::VPADDDZrr;
+    break;
+  case X86::VPDPWSSDZm:
+    MaddOpc = X86::VPMADDWDZrm;
+    AddOpc = X86::VPADDDZrr;
+    break;
+  }
+  // Create vpmaddwd.
+  const TargetRegisterClass *RC =
+      RegInfo.getRegClass(Root.getOperand(0).getReg());
+  Register NewReg = RegInfo.createVirtualRegister(RC);
+  MachineInstr *Madd = Root.getMF()->CloneMachineInstr(&Root);
+  Madd->setDesc(TII.get(MaddOpc));
+  Madd->untieRegOperand(1);
+  Madd->removeOperand(1);
+  Madd->getOperand(0).setReg(NewReg);
+  InstrIdxForVirtReg.insert(std::make_pair(NewReg, 0));
+  // Create vpaddd.
+  Register DstReg = Root.getOperand(0).getReg();
+  bool IsKill = Root.getOperand(1).isKill();
+  MachineInstr *Add =
+      BuildMI(*MF, MIMetadata(Root), TII.get(AddOpc), DstReg)
+          .addReg(Root.getOperand(1).getReg(), getKillRegState(IsKill))
+          .addReg(Madd->getOperand(0).getReg(), getKillRegState(true));
+  InsInstrs.push_back(Madd);
+  InsInstrs.push_back(Add);
+  DelInstrs.push_back(&Root);
+}
+
+void X86InstrInfo::genAlternativeCodeSequence(
+    MachineInstr &Root, MachineCombinerPattern Pattern,
+    SmallVectorImpl<MachineInstr *> &InsInstrs,
+    SmallVectorImpl<MachineInstr *> &DelInstrs,
+    DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
+  switch (Pattern) {
+  default:
+    // Reassociate instructions.
+    TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
+                                                DelInstrs, InstrIdxForVirtReg);
+    return;
+  case MachineCombinerPattern::DPWSSD:
+    genAlternativeDpCodeSequence(Root, *this, InsInstrs, DelInstrs,
+                                 InstrIdxForVirtReg);
+    return;
+  }
+}
+
 #define GET_INSTRINFO_HELPERS
 #include "X86GenInstrInfo.inc"
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index f8016b3a98ef4..4d1950e68f311 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -602,6 +602,34 @@ class X86InstrInfo final : public X86GenInstrInfo {
   std::optional<DestSourcePair>
   isCopyInstrImpl(const MachineInstr &MI) const override;
 
+  /// Return true when there is potentially a faster code sequence for an
+  /// instruction chain ending in \p Root. All potential patterns are listed in
+  /// the \p Pattern vector. Pattern should be sorted in priority order since
+  /// the pattern evaluator stops checking as soon as it finds a faster
+  /// sequence.
+  bool
+  getMachineCombinerPatterns(MachineInstr &Root,
+                             SmallVectorImpl<MachineCombinerPattern> &Patterns,
+                             bool DoRegPressureReduce) const override;
+
+  /// When getMachineCombinerPatterns() finds potential patterns,
+  /// this function generates the instructions that could replace the
+  /// original code sequence.
+  void genAlternativeCodeSequence(
+      MachineInstr &Root, MachineCombinerPattern Pattern,
+      SmallVectorImpl<MachineInstr *> &InsInstrs,
+      SmallVectorImpl<MachineInstr *> &DelInstrs,
+      DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const override;
+
+  /// When calculate the latency of the root instruction, accumulate the
+  /// latency of the sequence to the root latency.
+  /// \param Root - Instruction that could be combined with one of its operands
+  /// For X86 instruction (vpmaddwd + vpmaddwd) -> vpdpwssd, the vpmaddwd
+  /// is not in the critical path, so the root latency only include vpmaddwd.
+  bool accumulateInstrSeqToRootLatency(MachineInstr &Root) const override {
+    return false;
+  }
+
 private:
   /// This is a helper for convertToThreeAddress for 8 and 16-bit instructions.
   /// We use 32-bit LEA to form 3-address code by promoting to a 32-bit
diff --git a/llvm/lib/Target/X86/X86InstructionSelector.cpp b/llvm/lib/Target/X86/X86InstructionSelector.cpp
index 10637dc05d84f..d7b2d0a61fbc8 100644
--- a/llvm/lib/Target/X86/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/X86InstructionSelector.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -40,7 +41,6 @@
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
diff --git a/llvm/lib/Target/X86/X86InterleavedAccess.cpp b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
index df4437397f4ca..bee76b11414aa 100644
--- a/llvm/lib/Target/X86/X86InterleavedAccess.cpp
+++ b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/VectorUtils.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -28,7 +29,6 @@
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/MachineValueType.h"
 #include <algorithm>
 #include <cassert>
 #include <cmath>
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 7fe4c88b79f7d..a382db493fd4a 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -43,12 +43,10 @@ using namespace llvm;
 #define DEBUG_TYPE "tile-pre-config"
 
 static void emitErrorMsg(MachineFunction &MF) {
-  SmallString<32> Str;
-  Twine ErrorMsg =
-      MF.getName() +
-      ": Failed to config tile register, please define the shape earlier";
   LLVMContext &Context = MF.getMMI().getModule()->getContext();
-  Context.emitError(ErrorMsg);
+  Context.emitError(
+      MF.getName() +
+      ": Failed to config tile register, please define the shape earlier");
 }
 
 namespace {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 784c80301991c..e2de1db698d6a 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1612,7 +1612,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
             LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
         unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
         unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
-        SmallVector<int> NormalizedMask(NormalizedVF, UndefMaskElem);
+        SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
         copy(Mask, NormalizedMask.begin());
         unsigned PrevSrcReg = 0;
         ArrayRef<int> PrevRegMask;
@@ -1634,7 +1634,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
                 return;
               }
               if (SrcReg != DestReg &&
-                  any_of(RegMask, [](int I) { return I != UndefMaskElem; })) {
+                  any_of(RegMask, [](int I) { return I != PoisonMaskElem; })) {
                 // Just a copy of the source register.
                 Cost += TTI::TCC_Basic;
               }
diff --git a/llvm/lib/Target/XCore/CMakeLists.txt b/llvm/lib/Target/XCore/CMakeLists.txt
index 2ae8633bca512..447f5c5452885 100644
--- a/llvm/lib/Target/XCore/CMakeLists.txt
+++ b/llvm/lib/Target/XCore/CMakeLists.txt
@@ -32,6 +32,7 @@ add_llvm_target(XCoreCodeGen
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   MC
   SelectionDAG
diff --git a/llvm/lib/Target/XCore/Disassembler/CMakeLists.txt b/llvm/lib/Target/XCore/Disassembler/CMakeLists.txt
index d35ff535e068f..f472e21536112 100644
--- a/llvm/lib/Target/XCore/Disassembler/CMakeLists.txt
+++ b/llvm/lib/Target/XCore/Disassembler/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMXCoreDisassembler
   XCoreDisassembler.cpp
 
   LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCDisassembler
   Support
diff --git a/llvm/lib/TargetParser/RISCVTargetParser.cpp b/llvm/lib/TargetParser/RISCVTargetParser.cpp
index ac726cdbb797c..30a1023c06732 100644
--- a/llvm/lib/TargetParser/RISCVTargetParser.cpp
+++ b/llvm/lib/TargetParser/RISCVTargetParser.cpp
@@ -19,66 +19,70 @@
 namespace llvm {
 namespace RISCV {
 
+enum CPUKind : unsigned {
+#define PROC(ENUM, NAME, DEFAULT_MARCH) CK_##ENUM,
+#define TUNE_PROC(ENUM, NAME) CK_##ENUM,
+#include "llvm/TargetParser/RISCVTargetParserDef.inc"
+};
+
 struct CPUInfo {
   StringLiteral Name;
-  CPUKind Kind;
   StringLiteral DefaultMarch;
-  bool isInvalid() const { return DefaultMarch.empty(); }
   bool is64Bit() const { return DefaultMarch.starts_with("rv64"); }
 };
 
 constexpr CPUInfo RISCVCPUInfo[] = {
 #define PROC(ENUM, NAME, DEFAULT_MARCH)                              \
-  {NAME, CK_##ENUM, DEFAULT_MARCH},
+  {NAME, DEFAULT_MARCH},
 #include "llvm/TargetParser/RISCVTargetParserDef.inc"
 };
 
-bool checkCPUKind(CPUKind Kind, bool IsRV64) {
-  if (Kind == CK_INVALID)
-    return false;
-  return RISCVCPUInfo[static_cast<unsigned>(Kind)].is64Bit() == IsRV64;
+static const CPUInfo *getCPUInfoByName(StringRef CPU) {
+  for (auto &C : RISCVCPUInfo)
+    if (C.Name == CPU)
+      return &C;
+  return nullptr;
 }
 
-bool checkTuneCPUKind(CPUKind Kind, bool IsRV64) {
-  if (Kind == CK_INVALID)
-    return false;
-#define TUNE_PROC(ENUM, NAME)                                                  \
-  if (Kind == CK_##ENUM)                                                       \
-    return true;
-#include "llvm/TargetParser/RISCVTargetParserDef.inc"
-  return RISCVCPUInfo[static_cast<unsigned>(Kind)].is64Bit() == IsRV64;
-}
+bool parseCPU(StringRef CPU, bool IsRV64) {
+  const CPUInfo *Info = getCPUInfoByName(CPU);
 
-CPUKind parseCPUKind(StringRef CPU) {
-  return llvm::StringSwitch<CPUKind>(CPU)
-#define PROC(ENUM, NAME, DEFAULT_MARCH) .Case(NAME, CK_##ENUM)
-#include "llvm/TargetParser/RISCVTargetParserDef.inc"
-      .Default(CK_INVALID);
+  if (!Info)
+    return false;
+  return Info->is64Bit() == IsRV64;
 }
 
-CPUKind parseTuneCPUKind(StringRef TuneCPU, bool IsRV64) {
-  return llvm::StringSwitch<CPUKind>(TuneCPU)
-#define PROC(ENUM, NAME, DEFAULT_MARCH) .Case(NAME, CK_##ENUM)
+bool parseTuneCPU(StringRef TuneCPU, bool IsRV64) {
+  std::optional<CPUKind> Kind =
+      llvm::StringSwitch<std::optional<CPUKind>>(TuneCPU)
 #define TUNE_PROC(ENUM, NAME) .Case(NAME, CK_##ENUM)
-#include "llvm/TargetParser/RISCVTargetParserDef.inc"
-      .Default(CK_INVALID);
+  #include "llvm/TargetParser/RISCVTargetParserDef.inc"
+      .Default(std::nullopt);
+
+  if (Kind.has_value())
+    return true;
+
+  // Fallback to parsing as a CPU.
+  return parseCPU(TuneCPU, IsRV64);
 }
 
 StringRef getMArchFromMcpu(StringRef CPU) {
-  CPUKind Kind = parseCPUKind(CPU);
-  return RISCVCPUInfo[static_cast<unsigned>(Kind)].DefaultMarch;
+  const CPUInfo *Info = getCPUInfoByName(CPU);
+  if (!Info)
+    return "";
+  return Info->DefaultMarch;
 }
 
 void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64) {
   for (const auto &C : RISCVCPUInfo) {
-    if (C.Kind != CK_INVALID && IsRV64 == C.is64Bit())
+    if (IsRV64 == C.is64Bit())
       Values.emplace_back(C.Name);
   }
 }
 
 void fillValidTuneCPUArchList(SmallVectorImpl<StringRef> &Values, bool IsRV64) {
   for (const auto &C : RISCVCPUInfo) {
-    if (C.Kind != CK_INVALID && IsRV64 == C.is64Bit())
+    if (IsRV64 == C.is64Bit())
       Values.emplace_back(C.Name);
   }
 #define TUNE_PROC(ENUM, NAME) Values.emplace_back(StringRef(NAME));
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 0b8f853d9df23..3c53c7adb29c4 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -611,7 +611,7 @@ struct LoadOps {
   LoadInst *RootInsert = nullptr;
   bool FoundRoot = false;
   uint64_t LoadSize = 0;
-  Value *Shift = nullptr;
+  const APInt *Shift = nullptr;
   Type *ZextType;
   AAMDNodes AATags;
 };
@@ -621,7 +621,7 @@ struct LoadOps {
 // (ZExt(L1) << shift1) | ZExt(L2) -> ZExt(L3)
 static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
                                AliasAnalysis &AA) {
-  Value *ShAmt2 = nullptr;
+  const APInt *ShAmt2 = nullptr;
   Value *X;
   Instruction *L1, *L2;
 
@@ -629,7 +629,7 @@ static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
   if (match(V, m_OneUse(m_c_Or(
                    m_Value(X),
                    m_OneUse(m_Shl(m_OneUse(m_ZExt(m_OneUse(m_Instruction(L2)))),
-                                  m_Value(ShAmt2)))))) ||
+                                  m_APInt(ShAmt2)))))) ||
       match(V, m_OneUse(m_Or(m_Value(X),
                              m_OneUse(m_ZExt(m_OneUse(m_Instruction(L2)))))))) {
     if (!foldLoadsRecursive(X, LOps, DL, AA) && LOps.FoundRoot)
@@ -640,11 +640,11 @@ static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
 
   // Check if the pattern has loads
   LoadInst *LI1 = LOps.Root;
-  Value *ShAmt1 = LOps.Shift;
+  const APInt *ShAmt1 = LOps.Shift;
   if (LOps.FoundRoot == false &&
       (match(X, m_OneUse(m_ZExt(m_Instruction(L1)))) ||
        match(X, m_OneUse(m_Shl(m_OneUse(m_ZExt(m_OneUse(m_Instruction(L1)))),
-                               m_Value(ShAmt1)))))) {
+                               m_APInt(ShAmt1)))))) {
     LI1 = dyn_cast<LoadInst>(L1);
   }
   LoadInst *LI2 = dyn_cast<LoadInst>(L2);
@@ -719,12 +719,11 @@ static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
     std::swap(ShAmt1, ShAmt2);
 
   // Find Shifts values.
-  const APInt *Temp;
   uint64_t Shift1 = 0, Shift2 = 0;
-  if (ShAmt1 && match(ShAmt1, m_APInt(Temp)))
-    Shift1 = Temp->getZExtValue();
-  if (ShAmt2 && match(ShAmt2, m_APInt(Temp)))
-    Shift2 = Temp->getZExtValue();
+  if (ShAmt1)
+    Shift1 = ShAmt1->getZExtValue();
+  if (ShAmt2)
+    Shift2 = ShAmt2->getZExtValue();
 
   // First load is always LI1. This is where we put the new load.
   // Use the merged load size available from LI1 for forward loads.
@@ -816,7 +815,7 @@ static bool foldConsecutiveLoads(Instruction &I, const DataLayout &DL,
   // Check if shift needed. We need to shift with the amount of load1
   // shift if not zero.
   if (LOps.Shift)
-    NewOp = Builder.CreateShl(NewOp, LOps.Shift);
+    NewOp = Builder.CreateShl(NewOp, ConstantInt::get(I.getContext(), *LOps.Shift));
   I.replaceAllUsesWith(NewOp);
 
   return true;
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 8c130be0092fd..c81f1e7f21fc8 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -89,7 +89,7 @@ class BlockToIndexMapping {
 //             crosses a suspend point.
 //
 namespace {
-struct SuspendCrossingInfo {
+class SuspendCrossingInfo {
   BlockToIndexMapping Mapping;
 
   struct BlockData {
@@ -98,18 +98,26 @@ struct SuspendCrossingInfo {
     bool Suspend = false;
     bool End = false;
     bool KillLoop = false;
+    bool Changed = false;
   };
   SmallVector<BlockData, SmallVectorThreshold> Block;
 
-  iterator_range<succ_iterator> successors(BlockData const &BD) const {
+  iterator_range<pred_iterator> predecessors(BlockData const &BD) const {
     BasicBlock *BB = Mapping.indexToBlock(&BD - &Block[0]);
-    return llvm::successors(BB);
+    return llvm::predecessors(BB);
   }
 
   BlockData &getBlockData(BasicBlock *BB) {
     return Block[Mapping.blockToIndex(BB)];
   }
 
+  /// Compute the BlockData for the current function in one iteration.
+  /// Returns whether the BlockData changes in this iteration.
+  /// Initialize - Whether this is the first iteration, we can optimize
+  /// the initial case a little bit by manual loop switch.
+  template <bool Initialize = false> bool computeBlockData();
+
+public:
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const;
   void dump(StringRef Label, BitVector const &BV) const;
@@ -215,6 +223,72 @@ LLVM_DUMP_METHOD void SuspendCrossingInfo::dump() const {
 }
 #endif
 
+template <bool Initialize> bool SuspendCrossingInfo::computeBlockData() {
+  const size_t N = Mapping.size();
+  bool Changed = false;
+
+  for (size_t I = 0; I < N; ++I) {
+    auto &B = Block[I];
+
+    // We don't need to count the predecessors when initialization.
+    if constexpr (!Initialize)
+      // If all the predecessors of the current Block don't change,
+      // the BlockData for the current block must not change too.
+      if (all_of(predecessors(B), [this](BasicBlock *BB) {
+            return !Block[Mapping.blockToIndex(BB)].Changed;
+          })) {
+        B.Changed = false;
+        continue;
+      }
+
+    // Saved Consumes and Kills bitsets so that it is easy to see
+    // if anything changed after propagation.
+    auto SavedConsumes = B.Consumes;
+    auto SavedKills = B.Kills;
+
+    for (BasicBlock *PI : predecessors(B)) {
+      auto PrevNo = Mapping.blockToIndex(PI);
+      auto &P = Block[PrevNo];
+
+      // Propagate Kills and Consumes from predecessors into B.
+      B.Consumes |= P.Consumes;
+      B.Kills |= P.Kills;
+
+      // If block P is a suspend block, it should propagate kills into block
+      // B for every block P consumes.
+      if (P.Suspend)
+        B.Kills |= P.Consumes;
+    }
+
+    if (B.Suspend) {
+      // If block S is a suspend block, it should kill all of the blocks it
+      // consumes.
+      B.Kills |= B.Consumes;
+    } else if (B.End) {
+      // If block B is an end block, it should not propagate kills as the
+      // blocks following coro.end() are reached during initial invocation
+      // of the coroutine while all the data are still available on the
+      // stack or in the registers.
+      B.Kills.reset();
+    } else {
+      // This is reached when B block it not Suspend nor coro.end and it
+      // need to make sure that it is not in the kill set.
+      B.KillLoop |= B.Kills[I];
+      B.Kills.reset(I);
+    }
+
+    if constexpr (!Initialize) {
+      B.Changed = (B.Kills != SavedKills) || (B.Consumes != SavedConsumes);
+      Changed |= B.Changed;
+    }
+  }
+
+  if constexpr (Initialize)
+    return true;
+
+  return Changed;
+}
+
 SuspendCrossingInfo::SuspendCrossingInfo(Function &F, coro::Shape &Shape)
     : Mapping(F) {
   const size_t N = Mapping.size();
@@ -226,6 +300,7 @@ SuspendCrossingInfo::SuspendCrossingInfo(Function &F, coro::Shape &Shape)
     B.Consumes.resize(N);
     B.Kills.resize(N);
     B.Consumes.set(I);
+    B.Changed = true;
   }
 
   // Mark all CoroEnd Blocks. We do not propagate Kills beyond coro.ends as
@@ -250,71 +325,11 @@ SuspendCrossingInfo::SuspendCrossingInfo(Function &F, coro::Shape &Shape)
       markSuspendBlock(Save);
   }
 
-  // Iterate propagating consumes and kills until they stop changing.
-  int Iteration = 0;
-  (void)Iteration;
+  computeBlockData</*Initialize=*/true>();
 
-  bool Changed;
-  do {
-    LLVM_DEBUG(dbgs() << "iteration " << ++Iteration);
-    LLVM_DEBUG(dbgs() << "==============\n");
-
-    Changed = false;
-    for (size_t I = 0; I < N; ++I) {
-      auto &B = Block[I];
-      for (BasicBlock *SI : successors(B)) {
-
-        auto SuccNo = Mapping.blockToIndex(SI);
-
-        // Saved Consumes and Kills bitsets so that it is easy to see
-        // if anything changed after propagation.
-        auto &S = Block[SuccNo];
-        auto SavedConsumes = S.Consumes;
-        auto SavedKills = S.Kills;
-
-        // Propagate Kills and Consumes from block B into its successor S.
-        S.Consumes |= B.Consumes;
-        S.Kills |= B.Kills;
-
-        // If block B is a suspend block, it should propagate kills into the
-        // its successor for every block B consumes.
-        if (B.Suspend) {
-          S.Kills |= B.Consumes;
-        }
-        if (S.Suspend) {
-          // If block S is a suspend block, it should kill all of the blocks it
-          // consumes.
-          S.Kills |= S.Consumes;
-        } else if (S.End) {
-          // If block S is an end block, it should not propagate kills as the
-          // blocks following coro.end() are reached during initial invocation
-          // of the coroutine while all the data are still available on the
-          // stack or in the registers.
-          S.Kills.reset();
-        } else {
-          // This is reached when S block it not Suspend nor coro.end and it
-          // need to make sure that it is not in the kill set.
-          S.KillLoop |= S.Kills[SuccNo];
-          S.Kills.reset(SuccNo);
-        }
-
-        // See if anything changed.
-        Changed |= (S.Kills != SavedKills) || (S.Consumes != SavedConsumes);
+  while (computeBlockData())
+    ;
 
-        if (S.Kills != SavedKills) {
-          LLVM_DEBUG(dbgs() << "\nblock " << I << " follower " << SI->getName()
-                            << "\n");
-          LLVM_DEBUG(dump("S.Kills", S.Kills));
-          LLVM_DEBUG(dump("SavedKills", SavedKills));
-        }
-        if (S.Consumes != SavedConsumes) {
-          LLVM_DEBUG(dbgs() << "\nblock " << I << " follower " << SI << "\n");
-          LLVM_DEBUG(dump("S.Consume", S.Consumes));
-          LLVM_DEBUG(dump("SavedCons", SavedConsumes));
-        }
-      }
-    }
-  } while (Changed);
   LLVM_DEBUG(dump());
 }
 
@@ -1714,7 +1729,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
   StructType *FrameTy = Shape.FrameTy;
   Value *FramePtr = Shape.FramePtr;
   DominatorTree DT(*F);
-  SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache;
+  SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
 
   // Create a GEP with the given index into the coroutine frame for the original
   // value Orig. Appends an extra 0 index for array-allocas, preserving the
@@ -1873,7 +1888,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
                              &*Builder.GetInsertPoint());
           // This dbg.declare is for the main function entry point.  It
           // will be deleted in all coro-split functions.
-          coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
+          coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame);
         }
       }
 
@@ -2815,7 +2830,7 @@ static void collectFrameAlloca(AllocaInst *AI, coro::Shape &Shape,
 }
 
 void coro::salvageDebugInfo(
-    SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> &DbgPtrAllocaCache,
+    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableIntrinsic *DVI, bool OptimizeFrame) {
   Function *F = DVI->getFunction();
   IRBuilder<> Builder(F->getContext());
@@ -2870,8 +2885,8 @@ void coro::salvageDebugInfo(
   // Avoid to create the alloca would be eliminated by optimization
   // passes and the corresponding dbg.declares would be invalid.
   if (!OptimizeFrame)
-    if (auto *Arg = dyn_cast<llvm::Argument>(Storage)) {
-      auto &Cached = DbgPtrAllocaCache[Storage];
+    if (auto *Arg = dyn_cast<Argument>(Storage)) {
+      auto &Cached = ArgToAllocaMap[Arg];
       if (!Cached) {
         Cached = Builder.CreateAlloca(Storage->getType(), 0, nullptr,
                                       Arg->getName() + ".debug");
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index 9af853a163535..372090356e245 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -31,7 +31,7 @@ void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
 /// If the frame pointer is an Argument, store it into an alloca if
 /// OptimizeFrame is false.
 void salvageDebugInfo(
-    SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> &DbgPtrAllocaCache,
+    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
     DbgVariableIntrinsic *DVI, bool OptimizeFrame);
 
 // Keeps data and helper functions for lowering coroutine intrinsics.
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 152dedd7775bf..e3c4e94cf4d37 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -697,9 +697,9 @@ void CoroCloner::replaceSwiftErrorOps() {
 void CoroCloner::salvageDebugInfo() {
   SmallVector<DbgVariableIntrinsic *, 8> Worklist =
       collectDbgVariableIntrinsics(*NewF);
-  SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache;
+  SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
   for (DbgVariableIntrinsic *DVI : Worklist)
-    coro::salvageDebugInfo(DbgPtrAllocaCache, DVI, Shape.OptimizeFrame);
+    coro::salvageDebugInfo(ArgToAllocaMap, DVI, Shape.OptimizeFrame);
 
   // Remove all salvaged dbg.declare intrinsics that became
   // either unreachable or stale due to the CoroSplit transformation.
@@ -1981,9 +1981,9 @@ splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
   // Salvage debug intrinsics that point into the coroutine frame in the
   // original function. The Cloner has already salvaged debug info in the new
   // coroutine funclets.
-  SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache;
+  SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
   for (auto *DDI : collectDbgVariableIntrinsics(F))
-    coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
+    coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame);
 
   return Shape;
 }
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 8c6c0728097ca..6d0cd21cbe92f 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -764,6 +764,15 @@ static Function *promoteArguments(Function *F, FunctionAnalysisManager &FAM,
     if (CB->isMustTailCall())
       return nullptr;
 
+    // If the caller is marked minsize, this transformation may increase code
+    // size. We assume that there is more than one call to this function since
+    // otherwise this function would be inlined or is dead.
+    // TODO: compare the number of loads/stores removed from the function with
+    // the number of introduced loads in callees to see if this is profitable
+    // code-size-wise.
+    if (CB->getFunction()->hasMinSize())
+      return nullptr;
+
     if (CB->getFunction() == F)
       IsRecursive = true;
   }
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 6eda254244fdb..1c7441a5ae851 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -7241,7 +7241,8 @@ struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
   }
 
   /// Identify the type we can chose for a private copy of the underlying
-  /// argument. None means it is not clear yet, nullptr means there is none.
+  /// argument. std::nullopt means it is not clear yet, nullptr means there is
+  /// none.
   virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
 
   /// Return a privatizable type that encloses both T0 and T1.
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index ee19fcb889646..10718bb077552 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -252,87 +252,115 @@ selectCallee(const ModuleSummaryIndex &Index,
 
 namespace {
 
-using EdgeInfo =
-    std::tuple<const GlobalValueSummary *, unsigned /* Threshold */>;
+using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */>;
 
 } // anonymous namespace
 
-static bool shouldImportGlobal(
-    const ValueInfo &VI, const GVSummaryMapTy &DefinedGVSummaries,
-    function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
-        isPrevailing) {
-  const auto &GVS = DefinedGVSummaries.find(VI.getGUID());
-  if (GVS == DefinedGVSummaries.end())
-    return true;
-  // We should not skip import if the module contains a non-prevailing
-  // definition with interposable linkage type. This is required for correctness
-  // in the situation where there is a prevailing def available for import and
-  // marked read-only. In this case, the non-prevailing def will be converted to
-  // a declaration, while the prevailing one becomes internal, thus no
-  // definitions will be available for linking. In order to prevent undefined
-  // symbol link error, the prevailing definition must be imported.
-  // FIXME: Consider adding a check that the suitable prevailing definition
-  // exists and marked read-only.
-  if (VI.getSummaryList().size() > 1 &&
-      GlobalValue::isInterposableLinkage(GVS->second->linkage()) &&
-      !isPrevailing(VI.getGUID(), GVS->second))
-    return true;
-
-  return false;
-}
+/// Import globals referenced by a function or other globals that are being
+/// imported, if importing such global is possible.
+class GlobalsImporter final {
+  const ModuleSummaryIndex &Index;
+  const GVSummaryMapTy &DefinedGVSummaries;
+  function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
+      IsPrevailing;
+  FunctionImporter::ImportMapTy &ImportList;
+  StringMap<FunctionImporter::ExportSetTy> *const ExportLists;
+
+  bool shouldImportGlobal(const ValueInfo &VI) {
+    const auto &GVS = DefinedGVSummaries.find(VI.getGUID());
+    if (GVS == DefinedGVSummaries.end())
+      return true;
+    // We should not skip import if the module contains a non-prevailing
+    // definition with interposable linkage type. This is required for
+    // correctness in the situation where there is a prevailing def available
+    // for import and marked read-only. In this case, the non-prevailing def
+    // will be converted to a declaration, while the prevailing one becomes
+    // internal, thus no definitions will be available for linking. In order to
+    // prevent undefined symbol link error, the prevailing definition must be
+    // imported.
+    // FIXME: Consider adding a check that the suitable prevailing definition
+    // exists and marked read-only.
+    if (VI.getSummaryList().size() > 1 &&
+        GlobalValue::isInterposableLinkage(GVS->second->linkage()) &&
+        !IsPrevailing(VI.getGUID(), GVS->second))
+      return true;
 
-static void computeImportForReferencedGlobals(
-    const GlobalValueSummary &Summary, const ModuleSummaryIndex &Index,
-    const GVSummaryMapTy &DefinedGVSummaries,
-    function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
-        isPrevailing,
-    SmallVectorImpl<EdgeInfo> &Worklist,
-    FunctionImporter::ImportMapTy &ImportList,
-    StringMap<FunctionImporter::ExportSetTy> *ExportLists) {
-  for (const auto &VI : Summary.refs()) {
-    if (!shouldImportGlobal(VI, DefinedGVSummaries, isPrevailing)) {
-      LLVM_DEBUG(
-          dbgs() << "Ref ignored! Target already in destination module.\n");
-      continue;
-    }
+    return false;
+  }
 
-    LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
-
-    // If this is a local variable, make sure we import the copy
-    // in the caller's module. The only time a local variable can
-    // share an entry in the index is if there is a local with the same name
-    // in another module that had the same source file name (in a different
-    // directory), where each was compiled in their own directory so there
-    // was not distinguishing path.
-    auto LocalNotInModule = [&](const GlobalValueSummary *RefSummary) -> bool {
-      return GlobalValue::isLocalLinkage(RefSummary->linkage()) &&
-             RefSummary->modulePath() != Summary.modulePath();
-    };
+  void
+  onImportingSummaryImpl(const GlobalValueSummary &Summary,
+                         SmallVectorImpl<const GlobalVarSummary *> &Worklist) {
+    for (const auto &VI : Summary.refs()) {
+      if (!shouldImportGlobal(VI)) {
+        LLVM_DEBUG(
+            dbgs() << "Ref ignored! Target already in destination module.\n");
+        continue;
+      }
 
-    for (const auto &RefSummary : VI.getSummaryList())
-      if (isa<GlobalVarSummary>(RefSummary.get()) &&
-          Index.canImportGlobalVar(RefSummary.get(), /* AnalyzeRefs */ true) &&
-          !LocalNotInModule(RefSummary.get())) {
+      LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
+
+      // If this is a local variable, make sure we import the copy
+      // in the caller's module. The only time a local variable can
+      // share an entry in the index is if there is a local with the same name
+      // in another module that had the same source file name (in a different
+      // directory), where each was compiled in their own directory so there
+      // was not distinguishing path.
+      auto LocalNotInModule =
+          [&](const GlobalValueSummary *RefSummary) -> bool {
+        return GlobalValue::isLocalLinkage(RefSummary->linkage()) &&
+               RefSummary->modulePath() != Summary.modulePath();
+      };
+
+      for (const auto &RefSummary : VI.getSummaryList()) {
+        const auto *GVS = dyn_cast<GlobalVarSummary>(RefSummary.get());
+        // Functions could be referenced by global vars - e.g. a vtable; but we
+        // don't currently imagine a reason those would be imported here, rather
+        // than as part of the logic deciding which functions to import (i.e.
+        // based on profile information). Should we decide to handle them here,
+        // we can refactor accordingly at that time.
+        if (!GVS || !Index.canImportGlobalVar(GVS, /* AnalyzeRefs */ true) ||
+            LocalNotInModule(GVS))
+          continue;
         auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID());
         // Only update stat and exports if we haven't already imported this
         // variable.
         if (!ILI.second)
           break;
         NumImportedGlobalVarsThinLink++;
-        // Any references made by this variable will be marked exported later,
-        // in ComputeCrossModuleImport, after import decisions are complete,
-        // which is more efficient than adding them here.
+        // Any references made by this variable will be marked exported
+        // later, in ComputeCrossModuleImport, after import decisions are
+        // complete, which is more efficient than adding them here.
         if (ExportLists)
           (*ExportLists)[RefSummary->modulePath()].insert(VI);
 
         // If variable is not writeonly we attempt to recursively analyze
         // its references in order to import referenced constants.
-        if (!Index.isWriteOnly(cast<GlobalVarSummary>(RefSummary.get())))
-          Worklist.emplace_back(RefSummary.get(), 0);
+        if (!Index.isWriteOnly(GVS))
+          Worklist.emplace_back(GVS);
         break;
       }
+    }
   }
-}
+
+public:
+  GlobalsImporter(
+      const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries,
+      function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
+          IsPrevailing,
+      FunctionImporter::ImportMapTy &ImportList,
+      StringMap<FunctionImporter::ExportSetTy> *ExportLists)
+      : Index(Index), DefinedGVSummaries(DefinedGVSummaries),
+        IsPrevailing(IsPrevailing), ImportList(ImportList),
+        ExportLists(ExportLists) {}
+
+  void onImportingSummary(const GlobalValueSummary &Summary) {
+    SmallVector<const GlobalVarSummary *, 128> Worklist;
+    onImportingSummaryImpl(Summary, Worklist);
+    while (!Worklist.empty())
+      onImportingSummaryImpl(*Worklist.pop_back_val(), Worklist);
+  }
+};
 
 static const char *
 getFailureName(FunctionImporter::ImportFailureReason Reason) {
@@ -365,13 +393,11 @@ static void computeImportForFunction(
     const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries,
     function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
         isPrevailing,
-    SmallVectorImpl<EdgeInfo> &Worklist,
+    SmallVectorImpl<EdgeInfo> &Worklist, GlobalsImporter &GVImporter,
     FunctionImporter::ImportMapTy &ImportList,
     StringMap<FunctionImporter::ExportSetTy> *ExportLists,
     FunctionImporter::ImportThresholdsTy &ImportThresholds) {
-  computeImportForReferencedGlobals(Summary, Index, DefinedGVSummaries,
-                                    isPrevailing, Worklist, ImportList,
-                                    ExportLists);
+  GVImporter.onImportingSummary(Summary);
   static int ImportCount = 0;
   for (const auto &Edge : Summary.calls()) {
     ValueInfo VI = Edge.first;
@@ -546,6 +572,8 @@ static void ComputeImportForModule(
   // Worklist contains the list of function imported in this module, for which
   // we will analyse the callees and may import further down the callgraph.
   SmallVector<EdgeInfo, 128> Worklist;
+  GlobalsImporter GVI(Index, DefinedGVSummaries, isPrevailing, ImportList,
+                      ExportLists);
   FunctionImporter::ImportThresholdsTy ImportThresholds;
 
   // Populate the worklist with the import for the functions in the current
@@ -567,7 +595,7 @@ static void ComputeImportForModule(
       continue;
     LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
     computeImportForFunction(*FuncSummary, Index, ImportInstrLimit,
-                             DefinedGVSummaries, isPrevailing, Worklist,
+                             DefinedGVSummaries, isPrevailing, Worklist, GVI,
                              ImportList, ExportLists, ImportThresholds);
   }
 
@@ -579,12 +607,8 @@ static void ComputeImportForModule(
 
     if (auto *FS = dyn_cast<FunctionSummary>(Summary))
       computeImportForFunction(*FS, Index, Threshold, DefinedGVSummaries,
-                               isPrevailing, Worklist, ImportList, ExportLists,
-                               ImportThresholds);
-    else
-      computeImportForReferencedGlobals(*Summary, Index, DefinedGVSummaries,
-                                        isPrevailing, Worklist, ImportList,
-                                        ExportLists);
+                               isPrevailing, Worklist, GVI, ImportList,
+                               ExportLists, ImportThresholds);
   }
 
   // Print stats about functions considered but rejected for importing
@@ -654,17 +678,23 @@ checkVariableImport(const ModuleSummaryIndex &Index,
   // Checks that all GUIDs of read/writeonly vars we see in export lists
   // are also in the import lists. Otherwise we my face linker undefs,
   // because readonly and writeonly vars are internalized in their
-  // source modules.
-  auto IsReadOrWriteOnlyVar = [&](StringRef ModulePath, const ValueInfo &VI) {
+  // source modules. The exception would be if it has a linkage type indicating
+  // that there may have been a copy existing in the importing module (e.g.
+  // linkonce_odr). In that case we cannot accurately do this checking.
+  auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,
+                                                  const ValueInfo &VI) {
     auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
         Index.findSummaryInModule(VI, ModulePath));
-    return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS));
+    return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
+           !(GVS->linkage() == GlobalValue::AvailableExternallyLinkage ||
+             GVS->linkage() == GlobalValue::WeakODRLinkage ||
+             GVS->linkage() == GlobalValue::LinkOnceODRLinkage);
   };
 
   for (auto &ExportPerModule : ExportLists)
     for (auto &VI : ExportPerModule.second)
       if (!FlattenedImports.count(VI.getGUID()) &&
-          IsReadOrWriteOnlyVar(ExportPerModule.first(), VI))
+          IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first(), VI))
         return false;
 
   return true;
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 379211316b95a..3549eec4853c9 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -53,6 +53,7 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueLattice.h"
 #include "llvm/Analysis/ValueLatticeUtils.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Transforms/Scalar/SCCP.h"
 #include "llvm/Transforms/Utils/Cloning.h"
@@ -732,22 +733,19 @@ Constant *FunctionSpecializer::getCandidateConstant(Value *V) {
   if (isa<PoisonValue>(V))
     return nullptr;
 
-  // TrackValueOfGlobalVariable only tracks scalar global variables.
-  if (auto *GV = dyn_cast<GlobalVariable>(V)) {
-    // Check if we want to specialize on the address of non-constant
-    // global values.
-    if (!GV->isConstant() && !SpecializeOnAddress)
-      return nullptr;
-
-    if (!GV->getValueType()->isSingleValueType())
-      return nullptr;
-  }
-
   // Select for possible specialisation values that are constants or
   // are deduced to be constants or constant ranges with a single element.
   Constant *C = dyn_cast<Constant>(V);
   if (!C)
     C = Solver.getConstantOrNull(V);
+
+  // Don't specialize on (anything derived from) the address of a non-constant
+  // global variable, unless explicitly enabled.
+  if (C && C->getType()->isPointerTy() && !C->isNullValue())
+    if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(C));
+        GV && !(GV->isConstant() || SpecializeOnAddress))
+      return nullptr;
+
   return C;
 }
 
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 96162509289b8..bae94fd763d08 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -1145,9 +1145,6 @@ optimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,
           nullptr /* F */,
           GV->getInitializer()->getType()->getPointerAddressSpace())) {
     if (Constant *SOVC = dyn_cast<Constant>(StoredOnceVal)) {
-      if (GV->getInitializer()->getType() != SOVC->getType())
-        SOVC = ConstantExpr::getBitCast(SOVC, GV->getInitializer()->getType());
-
       // Optimize away any trapping uses of the loaded value.
       if (OptimizeAwayTrappingUsesOfLoads(GV, SOVC, DL, GetTLI))
         return true;
@@ -2113,15 +2110,22 @@ static void setUsedInitializer(GlobalVariable &V,
     return;
   }
 
+  // Get address space of pointers in the array of pointers.
+  const Type *UsedArrayType = V.getValueType();
+  const auto *VAT = cast<ArrayType>(UsedArrayType);
+  const auto *VEPT = cast<PointerType>(VAT->getArrayElementType());
+
   // Type of pointer to the array of pointers.
-  PointerType *Int8PtrTy = Type::getInt8PtrTy(V.getContext(), 0);
+  PointerType *Int8PtrTy =
+      Type::getInt8PtrTy(V.getContext(), VEPT->getAddressSpace());
 
   SmallVector<Constant *, 8> UsedArray;
   for (GlobalValue *GV : Init) {
-    Constant *Cast
-      = ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, Int8PtrTy);
+    Constant *Cast =
+        ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, Int8PtrTy);
     UsedArray.push_back(Cast);
   }
+
   // Sort to get deterministic order.
   array_pod_sort(UsedArray.begin(), UsedArray.end(), compareNames);
   ArrayType *ATy = ArrayType::get(Int8PtrTy, UsedArray.size());
@@ -2212,22 +2216,11 @@ static bool hasUseOtherThanLLVMUsed(GlobalAlias &GA, const LLVMUsed &U) {
   return !U.usedCount(&GA) && !U.compilerUsedCount(&GA);
 }
 
-static bool hasMoreThanOneUseOtherThanLLVMUsed(GlobalValue &V,
-                                               const LLVMUsed &U) {
-  unsigned N = 2;
-  assert((!U.usedCount(&V) || !U.compilerUsedCount(&V)) &&
-         "We should have removed the duplicated "
-         "element from llvm.compiler.used");
-  if (U.usedCount(&V) || U.compilerUsedCount(&V))
-    ++N;
-  return V.hasNUsesOrMore(N);
-}
-
-static bool mayHaveOtherReferences(GlobalAlias &GA, const LLVMUsed &U) {
-  if (!GA.hasLocalLinkage())
+static bool mayHaveOtherReferences(GlobalValue &GV, const LLVMUsed &U) {
+  if (!GV.hasLocalLinkage())
     return true;
 
-  return U.usedCount(&GA) || U.compilerUsedCount(&GA);
+  return U.usedCount(&GV) || U.compilerUsedCount(&GV);
 }
 
 static bool hasUsesToReplace(GlobalAlias &GA, const LLVMUsed &U,
@@ -2241,21 +2234,16 @@ static bool hasUsesToReplace(GlobalAlias &GA, const LLVMUsed &U,
   if (!mayHaveOtherReferences(GA, U))
     return Ret;
 
-  // If the aliasee has internal linkage, give it the name and linkage
-  // of the alias, and delete the alias.  This turns:
+  // If the aliasee has internal linkage and no other references (e.g.,
+  // @llvm.used, @llvm.compiler.used), give it the name and linkage of the
+  // alias, and delete the alias. This turns:
   //   define internal ... @f(...)
   //   @a = alias ... @f
   // into:
   //   define ... @a(...)
   Constant *Aliasee = GA.getAliasee();
   GlobalValue *Target = cast<GlobalValue>(Aliasee->stripPointerCasts());
-  if (!Target->hasLocalLinkage())
-    return Ret;
-
-  // Do not perform the transform if multiple aliases potentially target the
-  // aliasee. This check also ensures that it is safe to replace the section
-  // and other attributes of the aliasee with those of the alias.
-  if (hasMoreThanOneUseOtherThanLLVMUsed(*Target, U))
+  if (mayHaveOtherReferences(*Target, U))
     return Ret;
 
   RenameTarget = true;
diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp
index 2174827d4e04a..5e2a23b9e62df 100644
--- a/llvm/lib/Transforms/IPO/SCCP.cpp
+++ b/llvm/lib/Transforms/IPO/SCCP.cpp
@@ -366,8 +366,8 @@ static bool runIPSCCP(
     while (!GV->use_empty()) {
       StoreInst *SI = cast<StoreInst>(GV->user_back());
       SI->eraseFromParent();
-      MadeChanges = true;
     }
+    MadeChanges = true;
     M.eraseGlobalVariable(GV);
     ++NumGlobalConst;
   }
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 6c430aa5988c6..25ebff8ac1681 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -128,6 +128,11 @@ static cl::opt<std::string> SampleProfileRemappingFile(
     "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
     cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
 
+static cl::opt<bool> SalvageStaleProfile(
+    "salvage-stale-profile", cl::Hidden, cl::init(false),
+    cl::desc("Salvage stale profile by fuzzy matching and use the remapped "
+             "location for sample profile query."));
+
 static cl::opt<bool> ReportProfileStaleness(
     "report-profile-staleness", cl::Hidden, cl::init(false),
     cl::desc("Compute and report stale profile statistical metrics."));
@@ -438,6 +443,10 @@ class SampleProfileMatcher {
   SampleProfileReader &Reader;
   const PseudoProbeManager *ProbeManager;
   SampleProfileMap FlattenedProfiles;
+  // For each function, the matcher generates a map, of which each entry is a
+  // mapping from the source location of current build to the source location in
+  // the profile.
+  StringMap<LocToLocMap> FuncMappings;
 
   // Profile mismatching statstics.
   uint64_t TotalProfiledCallsites = 0;
@@ -458,7 +467,9 @@ class SampleProfileMatcher {
                                        FunctionSamples::ProfileIsCS);
     }
   }
+  void runOnModule();
 
+private:
   FunctionSamples *getFlattenedSamplesFor(const Function &F) {
     StringRef CanonFName = FunctionSamples::getCanonicalFnName(F);
     auto It = FlattenedProfiles.find(CanonFName);
@@ -466,9 +477,27 @@ class SampleProfileMatcher {
       return &It->second;
     return nullptr;
   }
-
-  void detectProfileMismatch();
-  void detectProfileMismatch(const Function &F, const FunctionSamples &FS);
+  void runOnFunction(const Function &F, const FunctionSamples &FS);
+  void countProfileMismatches(
+      const FunctionSamples &FS,
+      const std::unordered_set<LineLocation, LineLocationHash>
+          &MatchedCallsiteLocs,
+      uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites);
+
+  LocToLocMap &getIRToProfileLocationMap(const Function &F) {
+    auto Ret = FuncMappings.try_emplace(
+        FunctionSamples::getCanonicalFnName(F.getName()), LocToLocMap());
+    return Ret.first->second;
+  }
+  void distributeIRToProfileLocationMap();
+  void distributeIRToProfileLocationMap(FunctionSamples &FS);
+  void populateProfileCallsites(
+      const FunctionSamples &FS,
+      StringMap<std::set<LineLocation>> &CalleeToCallsitesMap);
+  void runStaleProfileMatching(
+      const std::map<LineLocation, StringRef> &IRLocations,
+      StringMap<std::set<LineLocation>> &CalleeToCallsitesMap,
+      LocToLocMap &IRToProfileLocationMap);
 };
 
 /// Sample profile pass.
@@ -1810,9 +1839,10 @@ bool SampleProfileLoader::emitAnnotations(Function &F) {
     if (!ProbeManager->profileIsValid(F, *Samples)) {
       LLVM_DEBUG(
           dbgs() << "Profile is invalid due to CFG mismatch for Function "
-                 << F.getName());
+                 << F.getName() << "\n");
       ++NumMismatchedProfile;
-      return false;
+      if (!SalvageStaleProfile)
+        return false;
     }
     ++NumMatchedProfile;
   } else {
@@ -2071,7 +2101,8 @@ bool SampleProfileLoader::doInitialization(Module &M,
     }
   }
 
-  if (ReportProfileStaleness || PersistProfileStaleness) {
+  if (ReportProfileStaleness || PersistProfileStaleness ||
+      SalvageStaleProfile) {
     MatchingManager =
         std::make_unique<SampleProfileMatcher>(M, *Reader, ProbeManager.get());
   }
@@ -2079,8 +2110,167 @@ bool SampleProfileLoader::doInitialization(Module &M,
   return true;
 }
 
-void SampleProfileMatcher::detectProfileMismatch(const Function &F,
-                                                 const FunctionSamples &FS) {
+void SampleProfileMatcher::countProfileMismatches(
+    const FunctionSamples &FS,
+    const std::unordered_set<LineLocation, LineLocationHash>
+        &MatchedCallsiteLocs,
+    uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites) {
+
+  auto isInvalidLineOffset = [](uint32_t LineOffset) {
+    return LineOffset & 0x8000;
+  };
+
+  // Check if there are any callsites in the profile that does not match to any
+  // IR callsites, those callsite samples will be discarded.
+  for (auto &I : FS.getBodySamples()) {
+    const LineLocation &Loc = I.first;
+    if (isInvalidLineOffset(Loc.LineOffset))
+      continue;
+
+    uint64_t Count = I.second.getSamples();
+    if (!I.second.getCallTargets().empty()) {
+      TotalCallsiteSamples += Count;
+      FuncProfiledCallsites++;
+      if (!MatchedCallsiteLocs.count(Loc)) {
+        MismatchedCallsiteSamples += Count;
+        FuncMismatchedCallsites++;
+      }
+    }
+  }
+
+  for (auto &I : FS.getCallsiteSamples()) {
+    const LineLocation &Loc = I.first;
+    if (isInvalidLineOffset(Loc.LineOffset))
+      continue;
+
+    uint64_t Count = 0;
+    for (auto &FM : I.second) {
+      Count += FM.second.getHeadSamplesEstimate();
+    }
+    TotalCallsiteSamples += Count;
+    FuncProfiledCallsites++;
+    if (!MatchedCallsiteLocs.count(Loc)) {
+      MismatchedCallsiteSamples += Count;
+      FuncMismatchedCallsites++;
+    }
+  }
+}
+
+// Populate the anchors(direct callee name) from profile.
+void SampleProfileMatcher::populateProfileCallsites(
+    const FunctionSamples &FS,
+    StringMap<std::set<LineLocation>> &CalleeToCallsitesMap) {
+  for (const auto &I : FS.getBodySamples()) {
+    const auto &Loc = I.first;
+    const auto &CTM = I.second.getCallTargets();
+    // Filter out possible indirect calls, use direct callee name as anchor.
+    if (CTM.size() == 1) {
+      StringRef CalleeName = CTM.begin()->first();
+      const auto &Candidates = CalleeToCallsitesMap.try_emplace(
+          CalleeName, std::set<LineLocation>());
+      Candidates.first->second.insert(Loc);
+    }
+  }
+
+  for (const auto &I : FS.getCallsiteSamples()) {
+    const LineLocation &Loc = I.first;
+    const auto &CalleeMap = I.second;
+    // Filter out possible indirect calls, use direct callee name as anchor.
+    if (CalleeMap.size() == 1) {
+      StringRef CalleeName = CalleeMap.begin()->first;
+      const auto &Candidates = CalleeToCallsitesMap.try_emplace(
+          CalleeName, std::set<LineLocation>());
+      Candidates.first->second.insert(Loc);
+    }
+  }
+}
+
+// Call target name anchor based profile fuzzy matching.
+// Input:
+// For IR locations, the anchor is the callee name of direct callsite; For
+// profile locations, it's the call target name for BodySamples or inlinee's
+// profile name for CallsiteSamples.
+// Matching heuristic:
+// First match all the anchors in lexical order, then split the non-anchor
+// locations between the two anchors evenly, first half are matched based on the
+// start anchor, second half are matched based on the end anchor.
+// For example, given:
+// IR locations:      [1, 2(foo), 3, 5, 6(bar), 7]
+// Profile locations: [1, 2, 3(foo), 4, 7, 8(bar), 9]
+// The matching gives:
+//   [1,    2(foo), 3,  5,  6(bar), 7]
+//    |     |       |   |     |     |
+//   [1, 2, 3(foo), 4,  7,  8(bar), 9]
+// The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9].
+void SampleProfileMatcher::runStaleProfileMatching(
+    const std::map<LineLocation, StringRef> &IRLocations,
+    StringMap<std::set<LineLocation>> &CalleeToCallsitesMap,
+    LocToLocMap &IRToProfileLocationMap) {
+  assert(IRToProfileLocationMap.empty() &&
+         "Run stale profile matching only once per function");
+
+  auto InsertMatching = [&](const LineLocation &From, const LineLocation &To) {
+    // Skip the unchanged location mapping to save memory.
+    if (From != To)
+      IRToProfileLocationMap.insert({From, To});
+  };
+
+  // Use function's beginning location as the initial anchor.
+  int32_t LocationDelta = 0;
+  SmallVector<LineLocation> LastMatchedNonAnchors;
+
+  for (const auto &IR : IRLocations) {
+    const auto &Loc = IR.first;
+    StringRef CalleeName = IR.second;
+    bool IsMatchedAnchor = false;
+    // Match the anchor location in lexical order.
+    if (!CalleeName.empty()) {
+      auto ProfileAnchors = CalleeToCallsitesMap.find(CalleeName);
+      if (ProfileAnchors != CalleeToCallsitesMap.end() &&
+          !ProfileAnchors->second.empty()) {
+        auto CI = ProfileAnchors->second.begin();
+        const auto Candidate = *CI;
+        ProfileAnchors->second.erase(CI);
+        InsertMatching(Loc, Candidate);
+        LLVM_DEBUG(dbgs() << "Callsite with callee:" << CalleeName
+                          << " is matched from " << Loc << " to " << Candidate
+                          << "\n");
+        LocationDelta = Candidate.LineOffset - Loc.LineOffset;
+
+        // Match backwards for non-anchor locations.
+        // The locations in LastMatchedNonAnchors have been matched forwards
+        // based on the previous anchor, spilt it evenly and overwrite the
+        // second half based on the current anchor.
+        for (size_t I = (LastMatchedNonAnchors.size() + 1) / 2;
+             I < LastMatchedNonAnchors.size(); I++) {
+          const auto &L = LastMatchedNonAnchors[I];
+          uint32_t CandidateLineOffset = L.LineOffset + LocationDelta;
+          LineLocation Candidate(CandidateLineOffset, L.Discriminator);
+          InsertMatching(L, Candidate);
+          LLVM_DEBUG(dbgs() << "Location is rematched backwards from " << L
+                            << " to " << Candidate << "\n");
+        }
+
+        IsMatchedAnchor = true;
+        LastMatchedNonAnchors.clear();
+      }
+    }
+
+    // Match forwards for non-anchor locations.
+    if (!IsMatchedAnchor) {
+      uint32_t CandidateLineOffset = Loc.LineOffset + LocationDelta;
+      LineLocation Candidate(CandidateLineOffset, Loc.Discriminator);
+      InsertMatching(Loc, Candidate);
+      LLVM_DEBUG(dbgs() << "Location is matched from " << Loc << " to "
+                        << Candidate << "\n");
+      LastMatchedNonAnchors.emplace_back(Loc);
+    }
+  }
+}
+
+void SampleProfileMatcher::runOnFunction(const Function &F,
+                                         const FunctionSamples &FS) {
+  bool IsFuncHashMismatch = false;
   if (FunctionSamples::ProfileIsProbeBased) {
     uint64_t Count = FS.getTotalSamples();
     TotalFuncHashSamples += Count;
@@ -2088,16 +2278,24 @@ void SampleProfileMatcher::detectProfileMismatch(const Function &F,
     if (!ProbeManager->profileIsValid(F, FS)) {
       MismatchedFuncHashSamples += Count;
       NumMismatchedFuncHash++;
-      return;
+      IsFuncHashMismatch = true;
     }
   }
 
   std::unordered_set<LineLocation, LineLocationHash> MatchedCallsiteLocs;
+  // The value of the map is the name of direct callsite and use empty StringRef
+  // for non-direct-call site.
+  std::map<LineLocation, StringRef> IRLocations;
 
-  // Go through all the callsites on the IR and flag the callsite if the target
-  // name is the same as the one in the profile.
+  // Extract profile matching anchors and profile mismatch metrics in the IR.
   for (auto &BB : F) {
     for (auto &I : BB) {
+      // TODO: Support line-number based location(AutoFDO).
+      if (FunctionSamples::ProfileIsProbeBased && isa<PseudoProbeInst>(&I)) {
+        if (std::optional<PseudoProbe> Probe = extractProbe(I))
+          IRLocations.emplace(LineLocation(Probe->Id, 0), StringRef());
+      }
+
       if (!isa<CallBase>(&I) || isa<IntrinsicInst>(&I))
         continue;
 
@@ -2109,6 +2307,17 @@ void SampleProfileMatcher::detectProfileMismatch(const Function &F,
         if (Function *Callee = CB->getCalledFunction())
           CalleeName = FunctionSamples::getCanonicalFnName(Callee->getName());
 
+        // Force to overwrite the callee name in case any non-call location was
+        // written before.
+        auto R = IRLocations.emplace(IRCallsite, CalleeName);
+        R.first->second = CalleeName;
+        assert((!FunctionSamples::ProfileIsProbeBased || R.second ||
+                R.first->second == CalleeName) &&
+               "Overwrite non-call or different callee name location for "
+               "pseudo probe callsite");
+
+        // Go through all the callsites on the IR and flag the callsite if the
+        // target name is the same as the one in the profile.
         const auto CTM = FS.findCallTargetMapAt(IRCallsite);
         const auto CallsiteFS = FS.findFunctionSamplesMapAt(IRCallsite);
 
@@ -2130,47 +2339,40 @@ void SampleProfileMatcher::detectProfileMismatch(const Function &F,
     }
   }
 
-  auto isInvalidLineOffset = [](uint32_t LineOffset) {
-    return LineOffset & 0x8000;
-  };
+  // Detect profile mismatch for profile staleness metrics report.
+  if (ReportProfileStaleness || PersistProfileStaleness) {
+    uint64_t FuncMismatchedCallsites = 0;
+    uint64_t FuncProfiledCallsites = 0;
+    countProfileMismatches(FS, MatchedCallsiteLocs, FuncMismatchedCallsites,
+                           FuncProfiledCallsites);
+    TotalProfiledCallsites += FuncProfiledCallsites;
+    NumMismatchedCallsites += FuncMismatchedCallsites;
+    LLVM_DEBUG({
+      if (FunctionSamples::ProfileIsProbeBased && !IsFuncHashMismatch &&
+          FuncMismatchedCallsites)
+        dbgs() << "Function checksum is matched but there are "
+               << FuncMismatchedCallsites << "/" << FuncProfiledCallsites
+               << " mismatched callsites.\n";
+    });
+  }
 
-  // Check if there are any callsites in the profile that does not match to any
-  // IR callsites, those callsite samples will be discarded.
-  for (auto &I : FS.getBodySamples()) {
-    const LineLocation &Loc = I.first;
-    if (isInvalidLineOffset(Loc.LineOffset))
-      continue;
+  if (IsFuncHashMismatch && SalvageStaleProfile) {
+    LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()
+                      << "\n");
 
-    uint64_t Count = I.second.getSamples();
-    if (!I.second.getCallTargets().empty()) {
-      TotalCallsiteSamples += Count;
-      TotalProfiledCallsites++;
-      if (!MatchedCallsiteLocs.count(Loc)) {
-        MismatchedCallsiteSamples += Count;
-        NumMismatchedCallsites++;
-      }
-    }
-  }
+    StringMap<std::set<LineLocation>> CalleeToCallsitesMap;
+    populateProfileCallsites(FS, CalleeToCallsitesMap);
 
-  for (auto &I : FS.getCallsiteSamples()) {
-    const LineLocation &Loc = I.first;
-    if (isInvalidLineOffset(Loc.LineOffset))
-      continue;
+    // The matching result will be saved to IRToProfileLocationMap, create a new
+    // map for each function.
+    auto &IRToProfileLocationMap = getIRToProfileLocationMap(F);
 
-    uint64_t Count = 0;
-    for (auto &FM : I.second) {
-      Count += FM.second.getHeadSamplesEstimate();
-    }
-    TotalCallsiteSamples += Count;
-    TotalProfiledCallsites++;
-    if (!MatchedCallsiteLocs.count(Loc)) {
-      MismatchedCallsiteSamples += Count;
-      NumMismatchedCallsites++;
-    }
+    runStaleProfileMatching(IRLocations, CalleeToCallsitesMap,
+                            IRToProfileLocationMap);
   }
 }
 
-void SampleProfileMatcher::detectProfileMismatch() {
+void SampleProfileMatcher::runOnModule() {
   for (auto &F : M) {
     if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile"))
       continue;
@@ -2181,8 +2383,10 @@ void SampleProfileMatcher::detectProfileMismatch() {
       FS = Reader.getSamplesFor(F);
     if (!FS)
       continue;
-    detectProfileMismatch(F, *FS);
+    runOnFunction(F, *FS);
   }
+  if (SalvageStaleProfile)
+    distributeIRToProfileLocationMap();
 
   if (ReportProfileStaleness) {
     if (FunctionSamples::ProfileIsProbeBased) {
@@ -2225,6 +2429,28 @@ void SampleProfileMatcher::detectProfileMismatch() {
   }
 }
 
+void SampleProfileMatcher::distributeIRToProfileLocationMap(
+    FunctionSamples &FS) {
+  const auto ProfileMappings = FuncMappings.find(FS.getName());
+  if (ProfileMappings != FuncMappings.end()) {
+    FS.setIRToProfileLocationMap(&(ProfileMappings->second));
+  }
+
+  for (auto &Inlinees : FS.getCallsiteSamples()) {
+    for (auto FS : Inlinees.second) {
+      distributeIRToProfileLocationMap(FS.second);
+    }
+  }
+}
+
+// Use a central place to distribute the matching results. Outlined and inlined
+// profile with the function name will be set to the same pointer.
+void SampleProfileMatcher::distributeIRToProfileLocationMap() {
+  for (auto &I : Reader.getProfiles()) {
+    distributeIRToProfileLocationMap(I.second);
+  }
+}
+
 bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
                                       ProfileSummaryInfo *_PSI,
                                       LazyCallGraph &CG) {
@@ -2270,8 +2496,10 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
   assert(SymbolMap.count(StringRef()) == 0 &&
          "No empty StringRef should be added in SymbolMap");
 
-  if (ReportProfileStaleness || PersistProfileStaleness)
-    MatchingManager->detectProfileMismatch();
+  if (ReportProfileStaleness || PersistProfileStaleness ||
+      SalvageStaleProfile) {
+    MatchingManager->runOnModule();
+  }
 
   bool retval = false;
   for (auto *F : buildFunctionOrder(M, CG)) {
diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index 1602a63e2f67f..4b36c7bce6887 100644
--- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -508,15 +508,17 @@ bool hasTypeMetadata(Module &M) {
   return false;
 }
 
-void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
+bool writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
                          function_ref<AAResults &(Function &)> AARGetter,
                          Module &M, const ModuleSummaryIndex *Index) {
   std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr;
   // See if this module has any type metadata. If so, we try to split it
   // or at least promote type ids to enable WPD.
   if (hasTypeMetadata(M)) {
-    if (enableSplitLTOUnit(M))
-      return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
+    if (enableSplitLTOUnit(M)) {
+      splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
+      return true;
+    }
     // Promote type ids as needed for index-based WPD.
     std::string ModuleId = getUniqueModuleId(&M);
     if (!ModuleId.empty()) {
@@ -549,6 +551,7 @@ void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
   // given OS.
   if (ThinLinkOS && Index)
     writeThinLinkBitcodeToFile(M, *ThinLinkOS, *Index, ModHash);
+  return false;
 }
 
 } // anonymous namespace
@@ -557,10 +560,11 @@ PreservedAnalyses
 llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
   FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
-  writeThinLTOBitcode(OS, ThinLinkOS,
-                      [&FAM](Function &F) -> AAResults & {
-                        return FAM.getResult<AAManager>(F);
-                      },
-                      M, &AM.getResult<ModuleSummaryIndexAnalysis>(M));
-  return PreservedAnalyses::all();
+  bool Changed = writeThinLTOBitcode(
+      OS, ThinLinkOS,
+      [&FAM](Function &F) -> AAResults & {
+        return FAM.getResult<AAManager>(F);
+      },
+      M, &AM.getResult<ModuleSummaryIndexAnalysis>(M));
+  return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 28bb85942d686..fb67e9f260b92 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -928,29 +928,23 @@ Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {
     return replaceInstUsesWith(II, FCmp);
   }
 
-  // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
-  if ((Mask & fcNan) && isKnownNeverNaN(Src0, &getTargetLibraryInfo())) {
-    II.setArgOperand(1, ConstantInt::get(Src1->getType(), Mask & ~fcNan));
-    return &II;
-  }
-
-  // fp_class (nnan x), ~(qnan|snan) -> true
-  if (Mask == (~fcNan & fcAllFlags) &&
-      isKnownNeverNaN(Src0, &getTargetLibraryInfo())) {
-    return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
-  }
+  KnownFPClass Known = computeKnownFPClass(
+      Src0, DL, Mask, 0, &getTargetLibraryInfo(), &AC, &II, &DT, &ORE);
 
+  // Clear test bits we know must be false from the source value.
+  // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
   // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
-  if ((Mask & fcInf) && isKnownNeverInfinity(Src0, &getTargetLibraryInfo())) {
-    II.setArgOperand(1, ConstantInt::get(Src1->getType(), Mask & ~fcInf));
+  if ((Mask & Known.KnownFPClasses) != Mask) {
+    II.setArgOperand(
+        1, ConstantInt::get(Src1->getType(), Mask & Known.KnownFPClasses));
     return &II;
   }
 
+  // If none of the tests which can return false are possible, fold to true.
+  // fp_class (nnan x), ~(qnan|snan) -> true
   // fp_class (ninf x), ~(ninf|pinf) -> true
-  if (Mask == (~fcInf & fcAllFlags) &&
-      isKnownNeverInfinity(Src0, &getTargetLibraryInfo())) {
+  if (Mask == Known.KnownFPClasses)
     return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
-  }
 
   return nullptr;
 }
@@ -2734,7 +2728,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       for (i = 0; i != SubVecNumElts; ++i)
         WidenMask.push_back(i);
       for (; i != VecNumElts; ++i)
-        WidenMask.push_back(UndefMaskElem);
+        WidenMask.push_back(PoisonMaskElem);
 
       Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask);
 
@@ -3029,7 +3023,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     int Sz = Mask.size();
     SmallBitVector UsedIndices(Sz);
     for (int Idx : Mask) {
-      if (Idx == UndefMaskElem || UsedIndices.test(Idx))
+      if (Idx == PoisonMaskElem || UsedIndices.test(Idx))
         break;
       UsedIndices.set(Idx);
     }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index bd3000ada1f38..1815fc858c679 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -3384,15 +3384,22 @@ Instruction *InstCombinerImpl::foldICmpEqIntrinsicWithConstant(
     }
     break;
 
+  case Intrinsic::umax:
   case Intrinsic::uadd_sat: {
     // uadd.sat(a, b) == 0  ->  (a | b) == 0
-    if (C.isZero()) {
+    // umax(a, b) == 0  ->  (a | b) == 0
+    if (C.isZero() && II->hasOneUse()) {
       Value *Or = Builder.CreateOr(II->getArgOperand(0), II->getArgOperand(1));
       return new ICmpInst(Pred, Or, Constant::getNullValue(Ty));
     }
     break;
   }
 
+  case Intrinsic::ssub_sat:
+    // ssub.sat(a, b) == 0 -> a == b
+    if (C.isZero())
+      return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
+    break;
   case Intrinsic::usub_sat: {
     // usub.sat(a, b) == 0  ->  a <= b
     if (C.isZero()) {
@@ -3591,6 +3598,21 @@ Instruction *InstCombinerImpl::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
     }
     break;
   }
+  case Intrinsic::ssub_sat:
+    // ssub.sat(a, b) spred 0 -> a spred b
+    if (ICmpInst::isSigned(Pred)) {
+      if (C.isZero())
+        return new ICmpInst(Pred, II->getArgOperand(0), II->getArgOperand(1));
+      // X s<= 0 is cannonicalized to X s< 1
+      if (Pred == ICmpInst::ICMP_SLT && C.isOne())
+        return new ICmpInst(ICmpInst::ICMP_SLE, II->getArgOperand(0),
+                            II->getArgOperand(1));
+      // X s>= 0 is cannonicalized to X s> -1
+      if (Pred == ICmpInst::ICMP_SGT && C.isAllOnes())
+        return new ICmpInst(ICmpInst::ICMP_SGE, II->getArgOperand(0),
+                            II->getArgOperand(1));
+    }
+    break;
   default:
     break;
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 19f2d2fde6fea..f6912f9a3cb84 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1920,8 +1920,10 @@ Instruction *InstCombinerImpl::visitURem(BinaryOperator &I) {
   // urem Op0, (sext i1 X) --> (Op0 == -1) ? 0 : Op0
   Value *X;
   if (match(Op1, m_SExt(m_Value(X))) && X->getType()->isIntOrIntVectorTy(1)) {
-    Value *Cmp = Builder.CreateICmpEQ(Op0, ConstantInt::getAllOnesValue(Ty));
-    return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), Op0);
+    Value *FrozenOp0 = Builder.CreateFreeze(Op0, Op0->getName() + ".frozen");
+    Value *Cmp =
+        Builder.CreateICmpEQ(FrozenOp0, ConstantInt::getAllOnesValue(Ty));
+    return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), FrozenOp0);
   }
 
   // For "(X + 1) % Op1" and if (X u< Op1) => (X + 1) == Op1 ? 0 : X + 1 .
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 0a746e25b31a4..be1571da4afea 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -2496,7 +2496,7 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
   // in the case of a shuffle with no undefined mask elements.
   ArrayRef<int> Mask;
   if (match(TVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) &&
-      !is_contained(Mask, UndefMaskElem) &&
+      !is_contained(Mask, PoisonMaskElem) &&
       cast<ShuffleVectorInst>(TVal)->isSelect()) {
     if (X == FVal) {
       // select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X)
@@ -2510,7 +2510,7 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
     }
   }
   if (match(FVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) &&
-      !is_contained(Mask, UndefMaskElem) &&
+      !is_contained(Mask, PoisonMaskElem) &&
       cast<ShuffleVectorInst>(FVal)->isSelect()) {
     if (X == TVal) {
       // select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y)
@@ -3291,79 +3291,6 @@ static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder) {
                                 Masked);
 }
 
-// Transform:
-//
-//   1 << (C - ctlz(X >> 1))
-//
-// into
-//
-//   (1 << (C - 1)) >> ctlz(X)
-//
-// The caller must guarantee that X is nonzero.
-//
-// TODO: Relax the requirement that X be nonzero.  We just need to require X to
-// be nonzero or the second argument of CTLZ to be true (that is, returning
-// poison on zero).
-static Instruction *foldBitFloorNonzero(Value *N, Value *X,
-                                        InstCombiner::BuilderTy &Builder) {
-  Type *NType = N->getType();
-  unsigned BitWidth = NType->getScalarSizeInBits();
-
-  // Match C - ctlz(X >> 1), where C is in (0, BitWidth].
-  // TODO: Handle C in [0, BitWidth] (with 0 included in the range), in which
-  // case 1 << C - ctlz(X >> 1) is equivalent to
-  // (1 << ((C - 1) & (BitWidth - 1))) >> ctlz(X).
-  const APInt *C = nullptr;
-  Value *CTLZ;
-  if (!match(N, m_OneUse(m_Shl(m_One(),
-                               m_OneUse(m_Sub(m_APInt(C), m_Value(CTLZ)))))) ||
-      !(C->ugt(0) && C->ule(BitWidth)) ||
-      !match(CTLZ, m_OneUse(m_Intrinsic<Intrinsic::ctlz>(
-                       m_OneUse(m_LShr(m_Specific(X), m_One())), m_Zero()))))
-    return nullptr;
-
-  APInt ShiftedBit = APInt::getOneBitSet(BitWidth, C->getZExtValue() - 1);
-
-  // Build ShiftedBit >> CTLZ.
-  Value *NewCTLZ =
-      Builder.CreateIntrinsic(Intrinsic::ctlz, {CTLZ->getType()},
-                              {X, cast<Instruction>(CTLZ)->getOperand(1)});
-  auto *Shift = cast<Instruction>(
-      Builder.CreateLShr(ConstantInt::get(NType, ShiftedBit), NewCTLZ));
-  Shift->setIsExact();
-  return Shift;
-}
-
-// Transform:
-//
-//   X == 0 ? 0 : (1 << (C1 - ctlz(X >> 1)))
-//
-// into
-//
-//   X == 0 ? 0 : (C2 >> ctlz(X))
-//
-// where C2 is computed by foldBitFloorNonzero based on C1.  The caller is
-// responsible for replacing one of the select operands.
-static Instruction *foldBitFloor(SelectInst &SI,
-                                 InstCombiner::BuilderTy &Builder) {
-  Value *TrueVal = SI.getTrueValue();
-  Value *FalseVal = SI.getFalseValue();
-
-  ICmpInst::Predicate Pred;
-  Value *Cond0;
-  if (!match(SI.getCondition(), m_ICmp(Pred, m_Value(Cond0), m_Zero())) ||
-      !ICmpInst::isEquality(Pred))
-    return nullptr;
-
-  if (Pred == ICmpInst::ICMP_NE)
-    std::swap(TrueVal, FalseVal);
-
-  if (!match(TrueVal, m_Zero()))
-    return nullptr;
-
-  return foldBitFloorNonzero(FalseVal, Cond0, Builder);
-}
-
 Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   Value *CondVal = SI.getCondition();
   Value *TrueVal = SI.getTrueValue();
@@ -3794,8 +3721,5 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   if (Instruction *I = foldBitCeil(SI, Builder))
     return I;
 
-  if (Instruction *I = foldBitFloor(SI, Builder))
-    return replaceOperand(SI, match(SI.getTrueValue(), m_Zero()) ? 2 : 1, I);
-
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 7195edc5d282a..29243dc89e34e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1553,7 +1553,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
       SmallVector<int, 16> Elts;
       for (unsigned i = 0; i < VWidth; ++i) {
         if (UndefElts[i])
-          Elts.push_back(UndefMaskElem);
+          Elts.push_back(PoisonMaskElem);
         else
           Elts.push_back(Shuffle->getMaskValue(i));
       }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 5851e76454334..db452331c1011 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -1344,7 +1344,7 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) {
       // (demanded elements analysis may unset it later).
       return nullptr;
     } else {
-      assert(OldMask[i] == UndefMaskElem &&
+      assert(OldMask[i] == PoisonMaskElem &&
              "Unexpected shuffle mask element for identity shuffle");
       NewMask[i] = IdxC;
     }
@@ -2145,7 +2145,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
                                 ConstantExpr::getShuffleVector(IdC, C, Mask);
 
   bool MightCreatePoisonOrUB =
-      is_contained(Mask, UndefMaskElem) &&
+      is_contained(Mask, PoisonMaskElem) &&
       (Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
   if (MightCreatePoisonOrUB)
     NewC = InstCombiner::getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -2159,7 +2159,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
   // An undef shuffle mask element may propagate as an undef constant element in
   // the new binop. That would produce poison where the original code might not.
   // If we already made a safe constant, then there's no danger.
-  if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+  if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB)
     NewBO->dropPoisonGeneratingFlags();
   return NewBO;
 }
@@ -2194,7 +2194,7 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
       cast<FixedVectorType>(Shuf.getType())->getNumElements();
   SmallVector<int, 16> NewMask(NumMaskElts, 0);
   for (unsigned i = 0; i != NumMaskElts; ++i)
-    if (Mask[i] == UndefMaskElem)
+    if (Mask[i] == PoisonMaskElem)
       NewMask[i] = Mask[i];
 
   return new ShuffleVectorInst(NewIns, NewMask);
@@ -2279,7 +2279,7 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
   // mask element, the result is undefined, but it is not poison or undefined
   // behavior. That is not necessarily true for div/rem/shift.
   bool MightCreatePoisonOrUB =
-      is_contained(Mask, UndefMaskElem) &&
+      is_contained(Mask, PoisonMaskElem) &&
       (Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
   if (MightCreatePoisonOrUB)
     NewC = InstCombiner::getSafeVectorConstantForBinop(BOpc, NewC,
@@ -2330,7 +2330,7 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
     NewI->andIRFlags(B1);
     if (DropNSW)
       NewI->setHasNoSignedWrap(false);
-    if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+    if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB)
       NewI->dropPoisonGeneratingFlags();
   }
   return replaceInstUsesWith(Shuf, NewBO);
@@ -2366,7 +2366,7 @@ static Instruction *foldTruncShuffle(ShuffleVectorInst &Shuf,
       SrcType->getScalarSizeInBits() / DestType->getScalarSizeInBits();
   ArrayRef<int> Mask = Shuf.getShuffleMask();
   for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
-    if (Mask[i] == UndefMaskElem)
+    if (Mask[i] == PoisonMaskElem)
       continue;
     uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio;
     assert(LSBIndex <= INT32_MAX && "Overflowed 32-bits");
@@ -2552,7 +2552,7 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
   for (unsigned i = 0; i != NumElts; ++i) {
     int ExtractMaskElt = Shuf.getMaskValue(i);
     int MaskElt = Mask[i];
-    NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
+    NewMask[i] = ExtractMaskElt == PoisonMaskElem ? ExtractMaskElt : MaskElt;
   }
   return new ShuffleVectorInst(X, Y, NewMask);
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 66bf709dc5391..f0a7a52d9121e 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1040,7 +1040,8 @@ static Constant *constantFoldOperationIntoSelectOperand(Instruction &I,
                                        : SI->getFalseValue());
     } else if (match(SI->getCondition(),
                      m_ICmp(Pred, m_Specific(Op), m_Constant(C))) &&
-               Pred == (IsTrueArm ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
+               Pred == (IsTrueArm ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE) &&
+               isGuaranteedNotToBeUndefOrPoison(C)) {
       // Pass
     } else {
       C = dyn_cast<Constant>(Op);
@@ -1194,6 +1195,7 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
   PHINode *NewPN = PHINode::Create(I.getType(), PN->getNumIncomingValues());
   InsertNewInstBefore(NewPN, *PN);
   NewPN->takeName(PN);
+  NewPN->setDebugLoc(PN->getDebugLoc());
 
   // If we are going to have to insert a new computation, do so right before the
   // predecessor's terminator.
@@ -1222,6 +1224,10 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
     replaceInstUsesWith(*User, NewPN);
     eraseInstFromFunction(*User);
   }
+
+  replaceAllDbgUsesWith(const_cast<PHINode &>(*PN),
+                        const_cast<PHINode &>(*NewPN),
+                        const_cast<PHINode &>(*PN), DT);
   return replaceInstUsesWith(I, NewPN);
 }
 
@@ -1702,9 +1708,9 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
     // TODO: Allow arbitrary shuffles by shuffling after binop?
     //       That might be legal, but we have to deal with poison.
     if (LShuf->isSelect() &&
-        !is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
+        !is_contained(LShuf->getShuffleMask(), PoisonMaskElem) &&
         RShuf->isSelect() &&
-        !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
+        !is_contained(RShuf->getShuffleMask(), PoisonMaskElem)) {
       // Example:
       // LHS = shuffle V1, V2, <0, 5, 6, 3>
       // RHS = shuffle V2, V1, <0, 5, 6, 3>
@@ -1945,13 +1951,6 @@ Instruction *InstCombinerImpl::visitGEPOfGEP(GetElementPtrInst &GEP,
   if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
     return nullptr;
 
-  // Note that if our source is a gep chain itself then we wait for that
-  // chain to be resolved before we perform this transformation.  This
-  // avoids us creating a TON of code in some cases.
-  if (auto *SrcGEP = dyn_cast<GEPOperator>(Src->getOperand(0)))
-    if (SrcGEP->getNumOperands() == 2 && shouldMergeGEPs(*Src, *SrcGEP))
-      return nullptr;   // Wait until our source is folded to completion.
-
   // For constant GEPs, use a more general offset-based folding approach.
   Type *PtrTy = Src->getType()->getScalarType();
   if (GEP.hasAllConstantIndices() &&
diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 332dd2eba8803..acb9e8e3a775b 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -13,7 +13,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CFGMST.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
@@ -38,6 +37,7 @@
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Instrumentation/CFGMST.h"
 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <algorithm>
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index f98cb67481154..55ee31b91318d 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1,4 +1,4 @@
-//===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
+//===- HWAddressSanitizer.cpp - memory access error detector --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -136,14 +136,6 @@ static cl::opt<bool>
                     cl::desc("detect use after scope within function"),
                     cl::Hidden, cl::init(false));
 
-static cl::opt<bool> ClUARRetagToZero(
-    "hwasan-uar-retag-to-zero",
-    cl::desc("Clear alloca tags before returning from the function to allow "
-             "non-instrumented and instrumented function calls mix. When set "
-             "to false, allocas are retagged before returning from the "
-             "function to detect use after return."),
-    cl::Hidden, cl::init(true));
-
 static cl::opt<bool> ClGenerateTagsWithCalls(
     "hwasan-generate-tags-with-calls",
     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
@@ -247,7 +239,9 @@ bool shouldInstrumentStack(const Triple &TargetTriple) {
 }
 
 bool shouldInstrumentWithCalls(const Triple &TargetTriple) {
-  return ClInstrumentWithCalls || TargetTriple.getArch() == Triple::x86_64;
+  return ClInstrumentWithCalls.getNumOccurrences()
+             ? ClInstrumentWithCalls
+             : TargetTriple.getArch() == Triple::x86_64;
 }
 
 bool mightUseStackSafetyAnalysis(bool DisableOptimization) {
@@ -313,16 +307,15 @@ class HWAddressSanitizer {
   void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
-  bool instrumentStack(memtag::StackInfo &Info, Value *StackTag,
+  bool instrumentStack(memtag::StackInfo &Info, Value *StackTag, Value *UARTag,
                        const DominatorTree &DT, const PostDominatorTree &PDT,
                        const LoopInfo &LI);
   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
   Value *getNextTagWithCall(IRBuilder<> &IRB);
   Value *getStackBaseTag(IRBuilder<> &IRB);
-  Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
-                      unsigned AllocaNo);
-  Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
+  Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
+  Value *getUARTag(IRBuilder<> &IRB);
 
   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
   Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
@@ -995,14 +988,15 @@ void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
   if (!UseShortGranules)
     Size = AlignedSize;
 
-  Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
+  Tag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
   if (InstrumentWithCalls) {
     IRB.CreateCall(HwasanTagMemoryFunc,
-                   {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
+                   {IRB.CreatePointerCast(AI, Int8PtrTy), Tag,
                     ConstantInt::get(IntptrTy, AlignedSize)});
   } else {
     size_t ShadowSize = Size >> Mapping.Scale;
-    Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
+    Value *AddrLong = untagPointer(IRB, IRB.CreatePointerCast(AI, IntptrTy));
+    Value *ShadowPtr = memToShadow(AddrLong, IRB);
     // If this memset is not inlined, it will be intercepted in the hwasan
     // runtime library. That's OK, because the interceptor skips the checks if
     // the address is in the shadow region.
@@ -1010,14 +1004,14 @@ void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
     // llvm.memset right here into either a sequence of stores, or a call to
     // hwasan_tag_memory.
     if (ShadowSize)
-      IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align(1));
+      IRB.CreateMemSet(ShadowPtr, Tag, ShadowSize, Align(1));
     if (Size != AlignedSize) {
       const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
       IRB.CreateStore(ConstantInt::get(Int8Ty, SizeRemainder),
                       IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
-      IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
-                                   Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
-                                   AlignedSize - 1));
+      IRB.CreateStore(Tag, IRB.CreateConstGEP1_32(
+                               Int8Ty, IRB.CreatePointerCast(AI, Int8PtrTy),
+                               AlignedSize - 1));
     }
   }
 }
@@ -1036,21 +1030,18 @@ unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
   // mask allocated (temporally) nearby. The program that generated this list
   // can be found at:
   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
-  static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
-                                 48, 16,  120, 248, 56,  24,  8,   124, 252,
-                                 60, 28,  12,  4,   126, 254, 62,  30,  14,
-                                 6,  2,   127, 63,  31,  15,  7,   3,   1};
+  static const unsigned FastMasks[] = {
+      0,   128, 64, 192, 32,  96,  224, 112, 240, 48, 16,  120,
+      248, 56,  24, 8,   124, 252, 60,  28,  12,  4,  126, 254,
+      62,  30,  14, 6,   2,   127, 63,  31,  15,  7,  3,   1};
   return FastMasks[AllocaNo % std::size(FastMasks)];
 }
 
 Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
-  if (TargetTriple.getArch() == Triple::x86_64) {
-    Constant *TagMask = ConstantInt::get(IntptrTy, TagMaskByte);
-    Value *NewTag = IRB.CreateAnd(OldTag, TagMask);
-    return NewTag;
-  }
-  // aarch64 uses 8-bit tags, so no mask is needed.
-  return OldTag;
+  if (TagMaskByte == 0xFF)
+    return OldTag; // No need to clear the tag byte.
+  return IRB.CreateAnd(OldTag,
+                       ConstantInt::get(OldTag->getType(), TagMaskByte));
 }
 
 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
@@ -1059,7 +1050,7 @@ Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
 
 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
   if (ClGenerateTagsWithCalls)
-    return getNextTagWithCall(IRB);
+    return nullptr;
   if (StackBaseTag)
     return StackBaseTag;
   // Extract some entropy from the stack pointer for the tags.
@@ -1074,19 +1065,20 @@ Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
 }
 
 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
-                                        AllocaInst *AI, unsigned AllocaNo) {
+                                        unsigned AllocaNo) {
   if (ClGenerateTagsWithCalls)
     return getNextTagWithCall(IRB);
-  return IRB.CreateXor(StackTag,
-                       ConstantInt::get(IntptrTy, retagMask(AllocaNo)));
+  return IRB.CreateXor(
+      StackTag, ConstantInt::get(StackTag->getType(), retagMask(AllocaNo)));
 }
 
-Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
-  if (ClUARRetagToZero)
-    return ConstantInt::get(IntptrTy, 0);
-  if (ClGenerateTagsWithCalls)
-    return getNextTagWithCall(IRB);
-  return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, TagMaskByte));
+Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
+  Value *StackPointerLong = getSP(IRB);
+  Value *UARTag =
+      applyTagMask(IRB, IRB.CreateLShr(StackPointerLong, PointerTagShift));
+
+  UARTag->setName("hwasan.uar.tag");
+  return UARTag;
 }
 
 // Add a tag to an address.
@@ -1116,12 +1108,12 @@ Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
     // Kernel addresses have 0xFF in the most significant byte.
     UntaggedPtrLong =
         IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(),
-                                               0xFFULL << PointerTagShift));
+                                               TagMaskByte << PointerTagShift));
   } else {
     // Userspace addresses have 0x00.
-    UntaggedPtrLong =
-        IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
-                                                ~(0xFFULL << PointerTagShift)));
+    UntaggedPtrLong = IRB.CreateAnd(
+        PtrLong, ConstantInt::get(PtrLong->getType(),
+                                  ~(TagMaskByte << PointerTagShift)));
   }
   return UntaggedPtrLong;
 }
@@ -1292,7 +1284,7 @@ static bool isLifetimeIntrinsic(Value *V) {
 }
 
 bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
-                                         Value *StackTag,
+                                         Value *StackTag, Value *UARTag,
                                          const DominatorTree &DT,
                                          const PostDominatorTree &PDT,
                                          const LoopInfo &LI) {
@@ -1310,9 +1302,10 @@ bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
     IRBuilder<> IRB(AI->getNextNode());
 
     // Replace uses of the alloca with tagged address.
-    Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
+    Value *Tag = getAllocaTag(IRB, StackTag, N);
     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
-    Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
+    Value *AINoTagLong = untagPointer(IRB, AILong);
+    Value *Replacement = tagPointer(IRB, AI->getType(), AINoTagLong, Tag);
     std::string Name =
         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
     Replacement->setName(Name + ".hwasan");
@@ -1339,7 +1332,7 @@ bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
     llvm::for_each(Info.LifetimeStart, HandleLifetime);
     llvm::for_each(Info.LifetimeEnd, HandleLifetime);
 
-    AI->replaceUsesWithIf(Replacement, [AICast, AILong](Use &U) {
+    AI->replaceUsesWithIf(Replacement, [AICast, AILong](const Use &U) {
       auto *User = U.getUser();
       return User != AILong && User != AICast && !isLifetimeIntrinsic(User);
     });
@@ -1358,9 +1351,8 @@ bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
 
     auto TagEnd = [&](Instruction *Node) {
       IRB.SetInsertPoint(Node);
-      Value *UARTag = getUARTag(IRB, StackTag);
       // When untagging, use the `AlignedSize` because we need to set the tags
-      // for the entire alloca to zero. If we used `Size` here, we would
+      // for the entire alloca to original. If we used `Size` here, we would
       // keep the last granule tagged, and store zero in the last byte of the
       // last granule, due to how short granules are implemented.
       tagAlloca(IRB, AI, UARTag, AlignedSize);
@@ -1462,9 +1454,9 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
     const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
     const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(F);
     const LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
-    Value *StackTag =
-        ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
-    instrumentStack(SInfo, StackTag, DT, PDT, LI);
+    Value *StackTag = getStackBaseTag(EntryIRB);
+    Value *UARTag = getUARTag(EntryIRB);
+    instrumentStack(SInfo, StackTag, UARTag, DT, PDT, LI);
   }
 
   // If we split the entry block, move any allocas that were originally in the
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 93667c9482921..3341615219185 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -122,6 +122,10 @@
 ///    Arbitrary sized accesses are handled with:
 ///      __msan_metadata_ptr_for_load_n(ptr, size)
 ///      __msan_metadata_ptr_for_store_n(ptr, size);
+///    Note that the sanitizer code has to deal with how shadow/origin pairs
+///    returned by the these functions are represented in different ABIs. In
+///    the X86_64 ABI they are returned in RDX:RAX, and in the SystemZ ABI they
+///    are written to memory pointed to by a hidden parameter.
 ///  - TLS variables are stored in a single per-task struct. A call to a
 ///    function __msan_get_context_state() returning a pointer to that struct
 ///    is inserted into every instrumented function before the entry block;
@@ -135,7 +139,7 @@
 /// Also, KMSAN currently ignores uninitialized memory passed into inline asm
 /// calls, making sure we're on the safe side wrt. possible false positives.
 ///
-///  KernelMemorySanitizer only supports X86_64 at the moment.
+///  KernelMemorySanitizer only supports X86_64 and SystemZ at the moment.
 ///
 //
 // FIXME: This sanitizer does not yet handle scalable vectors
@@ -543,6 +547,10 @@ class MemorySanitizer {
   void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
   void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
 
+  template <typename... ArgsTy>
+  FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
+                                                 ArgsTy... Args);
+
   /// True if we're compiling the Linux kernel.
   bool CompileKernel;
   /// Track origins (allocation points) of uninitialized values.
@@ -550,6 +558,7 @@ class MemorySanitizer {
   bool Recover;
   bool EagerChecks;
 
+  Triple TargetTriple;
   LLVMContext *C;
   Type *IntptrTy;
   Type *OriginTy;
@@ -620,13 +629,18 @@ class MemorySanitizer {
   /// Functions for poisoning/unpoisoning local variables
   FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
 
-  /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
-  /// pointers.
+  /// Pair of shadow/origin pointers.
+  Type *MsanMetadata;
+
+  /// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata.
   FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
   FunctionCallee MsanMetadataPtrForLoad_1_8[4];
   FunctionCallee MsanMetadataPtrForStore_1_8[4];
   FunctionCallee MsanInstrumentAsmStoreFn;
 
+  /// Storage for return values of the MsanMetadataPtrXxx functions.
+  Value *MsanMetadataAlloca;
+
   /// Helper to choose between different MsanMetadataPtrXxx().
   FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
 
@@ -729,6 +743,21 @@ static GlobalVariable *createPrivateConstGlobalForString(Module &M,
                             GlobalValue::PrivateLinkage, StrConst, "");
 }
 
+template <typename... ArgsTy>
+FunctionCallee
+MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
+                                                 ArgsTy... Args) {
+  if (TargetTriple.getArch() == Triple::systemz) {
+    // SystemZ ABI: shadow/origin pair is returned via a hidden parameter.
+    return M.getOrInsertFunction(Name, Type::getVoidTy(*C),
+                                 PointerType::get(MsanMetadata, 0),
+                                 std::forward<ArgsTy>(Args)...);
+  }
+
+  return M.getOrInsertFunction(Name, MsanMetadata,
+                               std::forward<ArgsTy>(Args)...);
+}
+
 /// Create KMSAN API callbacks.
 void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
   IRBuilder<> IRB(*C);
@@ -758,25 +787,25 @@ void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
   MsanGetContextStateFn = M.getOrInsertFunction(
       "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
 
-  Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
-                                PointerType::get(IRB.getInt32Ty(), 0));
+  MsanMetadata = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
+                                 PointerType::get(IRB.getInt32Ty(), 0));
 
   for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
     std::string name_load =
         "__msan_metadata_ptr_for_load_" + std::to_string(size);
     std::string name_store =
         "__msan_metadata_ptr_for_store_" + std::to_string(size);
-    MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
-        name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
-    MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
-        name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
+    MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
+        M, name_load, PointerType::get(IRB.getInt8Ty(), 0));
+    MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
+        M, name_store, PointerType::get(IRB.getInt8Ty(), 0));
   }
 
-  MsanMetadataPtrForLoadN = M.getOrInsertFunction(
-      "__msan_metadata_ptr_for_load_n", RetTy,
-      PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
-  MsanMetadataPtrForStoreN = M.getOrInsertFunction(
-      "__msan_metadata_ptr_for_store_n", RetTy,
+  MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
+      M, "__msan_metadata_ptr_for_load_n", PointerType::get(IRB.getInt8Ty(), 0),
+      IRB.getInt64Ty());
+  MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
+      M, "__msan_metadata_ptr_for_store_n",
       PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
 
   // Functions for poisoning and unpoisoning memory.
@@ -927,6 +956,8 @@ FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
 void MemorySanitizer::initializeModule(Module &M) {
   auto &DL = M.getDataLayout();
 
+  TargetTriple = Triple(M.getTargetTriple());
+
   bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
   bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
   // Check the overrides first
@@ -937,7 +968,6 @@ void MemorySanitizer::initializeModule(Module &M) {
     CustomMapParams.OriginBase = ClOriginBase;
     MapParams = &CustomMapParams;
   } else {
-    Triple TargetTriple(M.getTargetTriple());
     switch (TargetTriple.getOS()) {
     case Triple::FreeBSD:
       switch (TargetTriple.getArch()) {
@@ -1464,6 +1494,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     MS.RetvalOriginTLS =
         IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
                       {Zero, IRB.getInt32(6)}, "retval_origin");
+    if (MS.TargetTriple.getArch() == Triple::systemz)
+      MS.MsanMetadataAlloca = IRB.CreateAlloca(MS.MsanMetadata, 0u);
   }
 
   /// Add MemorySanitizer instrumentation to a function.
@@ -1696,6 +1728,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     return std::make_pair(ShadowPtr, OriginPtr);
   }
 
+  template <typename... ArgsTy>
+  Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee,
+                            ArgsTy... Args) {
+    if (MS.TargetTriple.getArch() == Triple::systemz) {
+      IRB.CreateCall(Callee,
+                     {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
+      return IRB.CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
+    }
+
+    return IRB.CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
+  }
+
   std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
                                                             IRBuilder<> &IRB,
                                                             Type *ShadowTy,
@@ -1708,12 +1752,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Value *AddrCast =
         IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
     if (Getter) {
-      ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
+      ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
     } else {
       Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
-      ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
-                                                : MS.MsanMetadataPtrForLoadN,
-                                        {AddrCast, SizeVal});
+      ShadowOriginPtrs = createMetadataCall(
+          IRB,
+          isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
+          AddrCast, SizeVal);
     }
     Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
     ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
@@ -3657,11 +3702,21 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, getOrigin(&I, 0));
   }
 
+  void handleIsFpClass(IntrinsicInst &I) {
+    IRBuilder<> IRB(&I);
+    Value *Shadow = getShadow(&I, 0);
+    setShadow(&I, IRB.CreateICmpNE(Shadow, getCleanShadow(Shadow)));
+    setOrigin(&I, getOrigin(&I, 0));
+  }
+
   void visitIntrinsicInst(IntrinsicInst &I) {
     switch (I.getIntrinsicID()) {
     case Intrinsic::abs:
       handleAbsIntrinsic(I);
       break;
+    case Intrinsic::is_fpclass:
+      handleIsFpClass(I);
+      break;
     case Intrinsic::lifetime_start:
       handleLifetimeStart(I);
       break;
@@ -5684,11 +5739,15 @@ struct VarArgSystemZHelper : public VarArgHelper {
         MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment,
                                /*isStore*/ true);
     // TODO(iii): copy only fragments filled by visitCallBase()
+    // TODO(iii): support packed-stack && !use-soft-float
+    // For use-soft-float functions, it is enough to copy just the GPRs.
+    unsigned RegSaveAreaSize =
+        IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
     IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
-                     SystemZRegSaveAreaSize);
+                     RegSaveAreaSize);
     if (MS.TrackOrigins)
       IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
-                       Alignment, SystemZRegSaveAreaSize);
+                       Alignment, RegSaveAreaSize);
   }
 
   void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index df0dd0ae1c17b..2aea2a66cbb02 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -48,7 +48,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
-#include "CFGMST.h"
 #include "ValueProfileCollector.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -114,6 +113,7 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/BlockCoverageInference.h"
+#include "llvm/Transforms/Instrumentation/CFGMST.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/MisExpect.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -440,32 +440,32 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
   SelectInstVisitor(Function &Func, bool HasSingleByteCoverage)
       : F(Func), HasSingleByteCoverage(HasSingleByteCoverage) {}
 
-  void countSelects(Function &Func) {
+  void countSelects() {
     NSIs = 0;
     Mode = VM_counting;
-    visit(Func);
+    visit(F);
   }
 
   // Visit the IR stream and instrument all select instructions. \p
   // Ind is a pointer to the counter index variable; \p TotalNC
   // is the total number of counters; \p FNV is the pointer to the
   // PGO function name var; \p FHash is the function hash.
-  void instrumentSelects(Function &Func, unsigned *Ind, unsigned TotalNC,
-                         GlobalVariable *FNV, uint64_t FHash) {
+  void instrumentSelects(unsigned *Ind, unsigned TotalNC, GlobalVariable *FNV,
+                         uint64_t FHash) {
     Mode = VM_instrument;
     CurCtrIdx = Ind;
     TotalNumCtrs = TotalNC;
     FuncHash = FHash;
     FuncNameVar = FNV;
-    visit(Func);
+    visit(F);
   }
 
   // Visit the IR stream and annotate all select instructions.
-  void annotateSelects(Function &Func, PGOUseFunc *UF, unsigned *Ind) {
+  void annotateSelects(PGOUseFunc *UF, unsigned *Ind) {
     Mode = VM_annotate;
     UseFunc = UF;
     CurCtrIdx = Ind;
-    visit(Func);
+    visit(F);
   }
 
   void instrumentOneSelectInst(SelectInst &SI);
@@ -479,52 +479,41 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
   unsigned getNumOfSelectInsts() const { return NSIs; }
 };
 
-} // end anonymous namespace
-
-namespace {
-
-/// An MST based instrumentation for PGO
-///
-/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
-/// in the function level.
+/// This class implements the CFG edges for the Minimum Spanning Tree (MST)
+/// based instrumentation.
+/// Note that the CFG can be a multi-graph. So there might be multiple edges
+/// with the same SrcBB and DestBB.
 struct PGOEdge {
-  // This class implements the CFG edges. Note the CFG can be a multi-graph.
-  // So there might be multiple edges with same SrcBB and DestBB.
-  const BasicBlock *SrcBB;
-  const BasicBlock *DestBB;
+  BasicBlock *SrcBB;
+  BasicBlock *DestBB;
   uint64_t Weight;
   bool InMST = false;
   bool Removed = false;
   bool IsCritical = false;
 
-  PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
+  PGOEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W = 1)
       : SrcBB(Src), DestBB(Dest), Weight(W) {}
 
-  // Return the information string of an edge.
+  /// Return the information string of an edge.
   std::string infoString() const {
     return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
-            (IsCritical ? "c" : " ") + "  W=" + Twine(Weight)).str();
+            (IsCritical ? "c" : " ") + "  W=" + Twine(Weight))
+        .str();
   }
 };
 
-// This class stores the auxiliary information for each BB.
-struct BBInfo {
-  BBInfo *Group;
+/// This class stores the auxiliary information for each BB in the MST.
+struct PGOBBInfo {
+  PGOBBInfo *Group;
   uint32_t Index;
   uint32_t Rank = 0;
 
-  BBInfo(unsigned IX) : Group(this), Index(IX) {}
+  PGOBBInfo(unsigned IX) : Group(this), Index(IX) {}
 
-  // Return the information string of this object.
+  /// Return the information string of this object.
   std::string infoString() const {
     return (Twine("Index=") + Twine(Index)).str();
   }
-
-  // Empty function -- only applicable to UseBBInfo.
-  void addOutEdge(PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {}
-
-  // Empty function -- only applicable to UseBBInfo.
-  void addInEdge(PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {}
 };
 
 // This class implements the CFG edges. Note the CFG can be a multi-graph.
@@ -581,9 +570,9 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
   BBInfo *findBBInfo(const BasicBlock *BB) const { return MST.findBBInfo(BB); }
 
   // Dump edges and BB information.
-  void dumpInfo(std::string Str = "") const {
-    MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName + " Hash: " +
-                              Twine(FunctionHash) + "\t" + Str);
+  void dumpInfo(StringRef Str = "") const {
+    MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName +
+                              " Hash: " + Twine(FunctionHash) + "\t" + Str);
   }
 
   FuncPGOInstrumentation(
@@ -600,7 +589,7 @@ template <class Edge, class BBInfo> class FuncPGOInstrumentation {
     if (BCI && PGOViewBlockCoverageGraph)
       BCI->viewBlockCoverageGraph();
     // This should be done before CFG hash computation.
-    SIVisitor.countSelects(Func);
+    SIVisitor.countSelects();
     ValueSites[IPVK_MemOPSize] = VPC.get(IPVK_MemOPSize);
     if (!IsCS) {
       NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
@@ -765,8 +754,8 @@ void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
   }
 }
 
-// Collect all the BBs that will be instruments and return them in
-// InstrumentBBs and setup InEdges/OutEdge for UseBBInfo.
+/// Collect all the BBs that will be instruments and add them to
+/// `InstrumentBBs`.
 template <class Edge, class BBInfo>
 void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
     std::vector<BasicBlock *> &InstrumentBBs) {
@@ -788,18 +777,6 @@ void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
     if (InstrBB)
       InstrumentBBs.push_back(InstrBB);
   }
-
-  // Set up InEdges/OutEdges for all BBs.
-  for (auto &E : MST.AllEdges) {
-    if (E->Removed)
-      continue;
-    const BasicBlock *SrcBB = E->SrcBB;
-    const BasicBlock *DestBB = E->DestBB;
-    BBInfo &SrcInfo = getBBInfo(SrcBB);
-    BBInfo &DestInfo = getBBInfo(DestBB);
-    SrcInfo.addOutEdge(E.get());
-    DestInfo.addInEdge(E.get());
-  }
 }
 
 // Given a CFG E to be instrumented, find which BB to place the instrumented
@@ -809,8 +786,8 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
   if (E->InMST || E->Removed)
     return nullptr;
 
-  BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
-  BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
+  BasicBlock *SrcBB = E->SrcBB;
+  BasicBlock *DestBB = E->DestBB;
   // For a fake edge, instrument the real BB.
   if (SrcBB == nullptr)
     return DestBB;
@@ -905,7 +882,7 @@ static void instrumentOneFunc(
     SplitIndirectBrCriticalEdges(F, /*IgnoreBlocksWithoutPHI=*/false, BPI, BFI);
   }
 
-  FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(
+  FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
       F, TLI, ComdatMembers, true, BPI, BFI, IsCS, PGOInstrumentEntry,
       PGOBlockCoverage);
 
@@ -956,7 +933,7 @@ static void instrumentOneFunc(
   }
 
   // Now instrument select instructions:
-  FuncInfo.SIVisitor.instrumentSelects(F, &I, NumCounters, FuncInfo.FuncNameVar,
+  FuncInfo.SIVisitor.instrumentSelects(&I, NumCounters, FuncInfo.FuncNameVar,
                                        FuncInfo.FunctionHash);
   assert(I == NumCounters);
 
@@ -1011,12 +988,11 @@ namespace {
 
 // This class represents a CFG edge in profile use compilation.
 struct PGOUseEdge : public PGOEdge {
+  using PGOEdge::PGOEdge;
+
   bool CountValid = false;
   uint64_t CountValue = 0;
 
-  PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
-      : PGOEdge(Src, Dest, W) {}
-
   // Set edge count value
   void setEdgeCount(uint64_t Value) {
     CountValue = Value;
@@ -1035,7 +1011,7 @@ struct PGOUseEdge : public PGOEdge {
 using DirectEdges = SmallVector<PGOUseEdge *, 2>;
 
 // This class stores the auxiliary information for each BB.
-struct UseBBInfo : public BBInfo {
+struct PGOUseBBInfo : public PGOBBInfo {
   uint64_t CountValue = 0;
   bool CountValid;
   int32_t UnknownCountInEdge = 0;
@@ -1043,10 +1019,7 @@ struct UseBBInfo : public BBInfo {
   DirectEdges InEdges;
   DirectEdges OutEdges;
 
-  UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {}
-
-  UseBBInfo(unsigned IX, uint64_t C)
-      : BBInfo(IX), CountValue(C), CountValid(true) {}
+  PGOUseBBInfo(unsigned IX) : PGOBBInfo(IX), CountValid(false) {}
 
   // Set the profile count value for this BB.
   void setBBInfoCount(uint64_t Value) {
@@ -1057,8 +1030,9 @@ struct UseBBInfo : public BBInfo {
   // Return the information string of this object.
   std::string infoString() const {
     if (!CountValid)
-      return BBInfo::infoString();
-    return (Twine(BBInfo::infoString()) + "  Count=" + Twine(CountValue)).str();
+      return PGOBBInfo::infoString();
+    return (Twine(PGOBBInfo::infoString()) + "  Count=" + Twine(CountValue))
+        .str();
   }
 
   // Add an OutEdge and update the edge count.
@@ -1141,22 +1115,21 @@ class PGOUseFunc {
   InstrProfRecord &getProfileRecord() { return ProfileRecord; }
 
   // Return the auxiliary BB information.
-  UseBBInfo &getBBInfo(const BasicBlock *BB) const {
+  PGOUseBBInfo &getBBInfo(const BasicBlock *BB) const {
     return FuncInfo.getBBInfo(BB);
   }
 
   // Return the auxiliary BB information if available.
-  UseBBInfo *findBBInfo(const BasicBlock *BB) const {
+  PGOUseBBInfo *findBBInfo(const BasicBlock *BB) const {
     return FuncInfo.findBBInfo(BB);
   }
 
   Function &getFunc() const { return F; }
 
-  void dumpInfo(std::string Str = "") const {
-    FuncInfo.dumpInfo(Str);
-  }
+  void dumpInfo(StringRef Str = "") const { FuncInfo.dumpInfo(Str); }
 
   uint64_t getProgramMaxCount() const { return ProgramMaxCount; }
+
 private:
   Function &F;
   Module *M;
@@ -1164,7 +1137,7 @@ class PGOUseFunc {
   ProfileSummaryInfo *PSI;
 
   // This member stores the shared information with class PGOGenFunc.
-  FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
+  FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;
 
   // The maximum count value in the profile. This is only used in PGO use
   // compilation.
@@ -1192,9 +1165,6 @@ class PGOUseFunc {
   // one unknown edge.
   void setEdgeCount(DirectEdges &Edges, uint64_t Value);
 
-  // Return FuncName string;
-  std::string getFuncName() const { return FuncInfo.FuncName; }
-
   // Set the hot/cold inline hints based on the count values.
   // FIXME: This function should be removed once the functionality in
   // the inliner is implemented.
@@ -1208,6 +1178,24 @@ class PGOUseFunc {
 
 } // end anonymous namespace
 
+/// Set up InEdges/OutEdges for all BBs in the MST.
+static void
+setupBBInfoEdges(FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> &FuncInfo) {
+  // This is not required when there is block coverage inference.
+  if (FuncInfo.BCI)
+    return;
+  for (auto &E : FuncInfo.MST.AllEdges) {
+    if (E->Removed)
+      continue;
+    const BasicBlock *SrcBB = E->SrcBB;
+    const BasicBlock *DestBB = E->DestBB;
+    PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);
+    PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);
+    SrcInfo.addOutEdge(E.get());
+    DestInfo.addInEdge(E.get());
+  }
+}
+
 // Visit all the edges and assign the count value for the instrumented
 // edges and the BB. Return false on error.
 bool PGOUseFunc::setInstrumentedCounts(
@@ -1215,6 +1203,9 @@ bool PGOUseFunc::setInstrumentedCounts(
 
   std::vector<BasicBlock *> InstrumentBBs;
   FuncInfo.getInstrumentBBs(InstrumentBBs);
+
+  setupBBInfoEdges(FuncInfo);
+
   unsigned NumCounters =
       InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
   // The number of counters here should match the number of counters
@@ -1228,7 +1219,7 @@ bool PGOUseFunc::setInstrumentedCounts(
   uint32_t I = 0;
   for (BasicBlock *InstrBB : InstrumentBBs) {
     uint64_t CountValue = CountFromProfile[I++];
-    UseBBInfo &Info = getBBInfo(InstrBB);
+    PGOUseBBInfo &Info = getBBInfo(InstrBB);
     // If we reach here, we know that we have some nonzero count
     // values in this function. The entry count should not be 0.
     // Fix it if necessary.
@@ -1253,7 +1244,7 @@ bool PGOUseFunc::setInstrumentedCounts(
     if (E->Removed || E->InMST)
       continue;
     const BasicBlock *SrcBB = E->SrcBB;
-    UseBBInfo &SrcInfo = getBBInfo(SrcBB);
+    PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);
 
     // If only one out-edge, the edge profile count should be the same as BB
     // profile count.
@@ -1261,7 +1252,7 @@ bool PGOUseFunc::setInstrumentedCounts(
       setEdgeCount(E.get(), SrcInfo.CountValue);
     else {
       const BasicBlock *DestBB = E->DestBB;
-      UseBBInfo &DestInfo = getBBInfo(DestBB);
+      PGOUseBBInfo &DestInfo = getBBInfo(DestBB);
       // If only one in-edge, the edge profile count should be the same as BB
       // profile count.
       if (DestInfo.CountValid && DestInfo.InEdges.size() == 1)
@@ -1292,8 +1283,7 @@ void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
 }
 
 // Emit function metadata indicating PGO profile mismatch.
-static void annotateFunctionWithHashMismatch(Function &F,
-                                             LLVMContext &ctx) {
+static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx) {
   const char MetadataName[] = "instr_prof_hash_mismatch";
   SmallVector<Metadata *, 2> Names;
   // If this metadata already exists, ignore.
@@ -1301,7 +1291,7 @@ static void annotateFunctionWithHashMismatch(Function &F,
   if (Existing) {
     MDTuple *Tuple = cast<MDTuple>(Existing);
     for (const auto &N : Tuple->operands()) {
-      if (cast<MDString>(N.get())->getString() ==  MetadataName)
+      if (cast<MDString>(N.get())->getString() == MetadataName)
         return;
       Names.push_back(N.get());
     }
@@ -1340,9 +1330,9 @@ static void addCallStack(CallStackTrie &AllocTrie,
   SmallVector<uint64_t> StackIds;
   for (const auto &StackFrame : AllocInfo->CallStack)
     StackIds.push_back(computeStackId(StackFrame));
-  auto AllocType = getAllocType(AllocInfo->Info.getMaxAccessCount(),
-                                AllocInfo->Info.getMinSize(),
-                                AllocInfo->Info.getMinLifetime());
+  auto AllocType = getAllocType(AllocInfo->Info.getTotalLifetimeAccessDensity(),
+                                AllocInfo->Info.getAllocCount(),
+                                AllocInfo->Info.getTotalLifetime());
   AllocTrie.addCallStack(AllocType, StackIds);
 }
 
@@ -1644,8 +1634,9 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
         dbgs() << "Inconsistent number of counts, skipping this function");
     Ctx.diagnose(DiagnosticInfoPGOProfile(
         M->getName().data(),
-        Twine("Inconsistent number of counts in ") + F.getName().str()
-        + Twine(": the profile may be stale or there is a function name collision."),
+        Twine("Inconsistent number of counts in ") + F.getName().str() +
+            Twine(": the profile may be stale or there is a function name "
+                  "collision."),
         DS_Warning));
     return false;
   }
@@ -1772,7 +1763,7 @@ void PGOUseFunc::populateCounters() {
     // For efficient traversal, it's better to start from the end as most
     // of the instrumented edges are at the end.
     for (auto &BB : reverse(F)) {
-      UseBBInfo *Count = findBBInfo(&BB);
+      PGOUseBBInfo *Count = findBBInfo(&BB);
       if (Count == nullptr)
         continue;
       if (!Count->CountValid) {
@@ -1811,7 +1802,7 @@ void PGOUseFunc::populateCounters() {
   }
 
   LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");
-  (void) NumPasses;
+  (void)NumPasses;
 #ifndef NDEBUG
   // Assert every BB has a valid counter.
   for (auto &BB : F) {
@@ -1837,7 +1828,7 @@ void PGOUseFunc::populateCounters() {
   markFunctionAttributes(FuncEntryCount, FuncMaxCount);
 
   // Now annotate select instructions
-  FuncInfo.SIVisitor.annotateSelects(F, this, &CountPosition);
+  FuncInfo.SIVisitor.annotateSelects(this, &CountPosition);
   assert(CountPosition == ProfileCountSize);
 
   LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile."));
@@ -1861,7 +1852,7 @@ void PGOUseFunc::setBranchWeights() {
       continue;
 
     // We have a non-zero Branch BB.
-    const UseBBInfo &BBCountInfo = getBBInfo(&BB);
+    const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
     unsigned Size = BBCountInfo.OutEdges.size();
     SmallVector<uint64_t, 2> EdgeCounts(Size, 0);
     uint64_t MaxCount = 0;
@@ -1886,11 +1877,11 @@ void PGOUseFunc::setBranchWeights() {
       // when there is no exit block and the code exits via a noreturn function.
       auto &Ctx = M->getContext();
       Ctx.diagnose(DiagnosticInfoPGOProfile(
-        M->getName().data(),
-        Twine("Profile in ") + F.getName().str() +
-            Twine(" partially ignored") +
-            Twine(", possibly due to the lack of a return path."),
-        DS_Warning));
+          M->getName().data(),
+          Twine("Profile in ") + F.getName().str() +
+              Twine(" partially ignored") +
+              Twine(", possibly due to the lack of a return path."),
+          DS_Warning));
     }
   }
 }
@@ -1912,7 +1903,7 @@ void PGOUseFunc::annotateIrrLoopHeaderWeights() {
     // duplication.
     if (BFI->isIrrLoopHeader(&BB) || isIndirectBrTarget(&BB)) {
       Instruction *TI = BB.getTerminator();
-      const UseBBInfo &BBCountInfo = getBBInfo(&BB);
+      const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
       setIrrLoopHeaderMetadata(M, TI, BBCountInfo.CountValue);
     }
   }
@@ -1995,8 +1986,8 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) {
     Ctx.diagnose(DiagnosticInfoPGOProfile(
         M->getName().data(),
         Twine("Inconsistent number of value sites for ") +
-            Twine(ValueProfKindDescr[Kind]) +
-            Twine(" profiling in \"") + F.getName().str() +
+            Twine(ValueProfKindDescr[Kind]) + Twine(" profiling in \"") +
+            F.getName().str() +
             Twine("\", possibly due to the use of a stale profile."),
         DS_Warning));
     return;
@@ -2092,7 +2083,10 @@ PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) {
   // The variable in a comdat may be discarded by LTO. Ensure the declaration
   // will be retained.
   appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true));
-  return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserve<FunctionAnalysisManagerModuleProxy>();
+  PA.preserveSet<AllAnalysesOn<Function>>();
+  return PA;
 }
 
 PreservedAnalyses PGOInstrumentationGen::run(Module &M,
@@ -2171,7 +2165,7 @@ static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI,
   BlockFrequencyInfo NBFI(F, NBPI, LI);
   //  bool PrintFunc = false;
   bool HotBBOnly = PGOVerifyHotBFI;
-  std::string Msg;
+  StringRef Msg;
   OptimizationRemarkEmitter ORE(&F);
 
   unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
@@ -2471,7 +2465,7 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M,
 
 static std::string getSimpleNodeName(const BasicBlock *Node) {
   if (!Node->getName().empty())
-    return std::string(Node->getName());
+    return Node->getName().str();
 
   std::string SimpleNodeName;
   raw_string_ostream OS(SimpleNodeName);
@@ -2480,8 +2474,7 @@ static std::string getSimpleNodeName(const BasicBlock *Node) {
 }
 
 void llvm::setProfMetadata(Module *M, Instruction *TI,
-                           ArrayRef<uint64_t> EdgeCounts,
-                           uint64_t MaxCount) {
+                           ArrayRef<uint64_t> EdgeCounts, uint64_t MaxCount) {
   MDBuilder MDB(M->getContext());
   assert(MaxCount > 0 && "Bad max count");
   uint64_t Scale = calculateCountScale(MaxCount);
@@ -2570,7 +2563,7 @@ template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
     raw_string_ostream OS(Result);
 
     OS << getSimpleNodeName(Node) << ":\\l";
-    UseBBInfo *BI = Graph->findBBInfo(Node);
+    PGOUseBBInfo *BI = Graph->findBBInfo(Node);
     OS << "Count : ";
     if (BI && BI->CountValid)
       OS << BI->CountValue << "\\l";
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index ba7a590ec2b67..71d3fda429bf7 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -327,6 +327,13 @@ static Decomposition decompose(Value *V,
     if (match(V, m_NSWAdd(m_Value(Op0), m_Value(Op1))))
       return MergeResults(Op0, Op1, IsSigned);
 
+    ConstantInt *CI;
+    if (match(V, m_NSWMul(m_Value(Op0), m_ConstantInt(CI)))) {
+      auto Result = decompose(Op0, Preconditions, IsSigned, DL);
+      Result.mul(CI->getSExtValue());
+      return Result;
+    }
+
     return V;
   }
 
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 9c745a28fe993..3ac315a4f9fa2 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -318,6 +318,14 @@ static unsigned getHashValueImpl(SimpleValue Val) {
     return hash_combine(GCR->getOpcode(), GCR->getOperand(0),
                         GCR->getBasePtr(), GCR->getDerivedPtr());
 
+  // Don't CSE convergent calls in different basic blocks, because they
+  // implicitly depend on the set of threads that is currently executing.
+  if (CallInst *CI = dyn_cast<CallInst>(Inst); CI && CI->isConvergent()) {
+    return hash_combine(
+        Inst->getOpcode(), Inst->getParent(),
+        hash_combine_range(Inst->value_op_begin(), Inst->value_op_end()));
+  }
+
   // Mix in the opcode.
   return hash_combine(
       Inst->getOpcode(),
@@ -344,8 +352,16 @@ static bool isEqualImpl(SimpleValue LHS, SimpleValue RHS) {
 
   if (LHSI->getOpcode() != RHSI->getOpcode())
     return false;
-  if (LHSI->isIdenticalToWhenDefined(RHSI))
+  if (LHSI->isIdenticalToWhenDefined(RHSI)) {
+    // Convergent calls implicitly depend on the set of threads that is
+    // currently executing, so conservatively return false if they are in
+    // different basic blocks.
+    if (CallInst *CI = dyn_cast<CallInst>(LHSI);
+        CI && CI->isConvergent() && LHSI->getParent() != RHSI->getParent())
+      return false;
+
     return true;
+  }
 
   // If we're not strictly identical, we still might be a commutable instruction
   if (BinaryOperator *LHSBinOp = dyn_cast<BinaryOperator>(LHSI)) {
@@ -578,12 +594,13 @@ class EarlyCSE {
     unsigned Generation = 0;
     int MatchingId = -1;
     bool IsAtomic = false;
+    bool IsLoad = false;
 
     LoadValue() = default;
     LoadValue(Instruction *Inst, unsigned Generation, unsigned MatchingId,
-              bool IsAtomic)
+              bool IsAtomic, bool IsLoad)
         : DefInst(Inst), Generation(Generation), MatchingId(MatchingId),
-          IsAtomic(IsAtomic) {}
+          IsAtomic(IsAtomic), IsLoad(IsLoad) {}
   };
 
   using LoadMapAllocator =
@@ -1476,8 +1493,9 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
           LLVM_DEBUG(dbgs() << "Skipping due to debug counter\n");
           continue;
         }
-        if (auto *I = dyn_cast<Instruction>(Op))
-          combineMetadataForCSE(I, &Inst, false);
+        if (InVal.IsLoad)
+          if (auto *I = dyn_cast<Instruction>(Op))
+            combineMetadataForCSE(I, &Inst, false);
         if (!Inst.use_empty())
           Inst.replaceAllUsesWith(Op);
         salvageKnowledge(&Inst, &AC);
@@ -1492,7 +1510,8 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
       AvailableLoads.insert(MemInst.getPointerOperand(),
                             LoadValue(&Inst, CurrentGeneration,
                                       MemInst.getMatchingId(),
-                                      MemInst.isAtomic()));
+                                      MemInst.isAtomic(),
+                                      MemInst.isLoad()));
       LastStore = nullptr;
       continue;
     }
@@ -1616,7 +1635,8 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         AvailableLoads.insert(MemInst.getPointerOperand(),
                               LoadValue(&Inst, CurrentGeneration,
                                         MemInst.getMatchingId(),
-                                        MemInst.isAtomic()));
+                                        MemInst.isAtomic(),
+                                        MemInst.isLoad()));
 
         // Remember that this was the last unordered store we saw for DSE. We
         // don't yet handle DSE on ordered or volatile stores since we don't
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 6223d97c7679c..b21adc968fef2 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -1530,7 +1530,7 @@ bool LoopConstrainer::run() {
 
   // This function canonicalizes the loop into Loop-Simplify and LCSSA forms.
   auto CanonicalizeLoop = [&] (Loop *L, bool IsOriginalLoop) {
-    formLCSSARecursively(*L, DT, &LI, &SE);
+    formLCSSARecursively(*L, DT, &LI);
     simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, true);
     // Pre/post loops are slow paths, we do not need to perform any loop
     // optimizations on them.
@@ -1759,7 +1759,7 @@ PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
     for (const auto &L : LI) {
       CFGChanged |= simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr,
                                  /*PreserveLCSSA=*/false);
-      Changed |= formLCSSARecursively(*L, DT, &LI, &SE);
+      Changed |= formLCSSARecursively(*L, DT, &LI);
     }
     Changed |= CFGChanged;
 
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index b5298cfd258bd..f184ff9aeea23 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -44,7 +44,6 @@
 #include "llvm/Analysis/AliasSetTracker.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CaptureTracking.h"
-#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/GuardUtils.h"
 #include "llvm/Analysis/LazyBlockFrequencyInfo.h"
 #include "llvm/Analysis/Loads.h"
@@ -150,10 +149,10 @@ cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
              "enable memory promotion."));
 
 static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
-static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
-                                  const LoopSafetyInfo *SafetyInfo,
-                                  TargetTransformInfo *TTI, bool &FreeInLoop,
-                                  bool LoopNestMode);
+static bool isNotUsedOrFoldableInLoop(const Instruction &I, const Loop *CurLoop,
+                                      const LoopSafetyInfo *SafetyInfo,
+                                      TargetTransformInfo *TTI,
+                                      bool &FoldableInLoop, bool LoopNestMode);
 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                   BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
                   MemorySSAUpdater &MSSAU, ScalarEvolution *SE,
@@ -512,7 +511,7 @@ bool LoopInvariantCodeMotion::runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI,
       // SSAUpdater strategy during promotion that was LCSSA aware and reformed
       // it as it went.
       if (Promoted)
-        formLCSSARecursively(*L, *DT, LI, SE);
+        formLCSSARecursively(*L, *DT, LI);
 
       Changed |= Promoted;
     }
@@ -583,14 +582,15 @@ bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
       // outside of the loop.  In this case, it doesn't even matter if the
       // operands of the instruction are loop invariant.
       //
-      bool FreeInLoop = false;
+      bool FoldableInLoop = false;
       bool LoopNestMode = OutermostLoop != nullptr;
       if (!I.mayHaveSideEffects() &&
-          isNotUsedOrFreeInLoop(I, LoopNestMode ? OutermostLoop : CurLoop,
-                                SafetyInfo, TTI, FreeInLoop, LoopNestMode) &&
+          isNotUsedOrFoldableInLoop(I, LoopNestMode ? OutermostLoop : CurLoop,
+                                    SafetyInfo, TTI, FoldableInLoop,
+                                    LoopNestMode) &&
           canSinkOrHoistInst(I, AA, DT, CurLoop, MSSAU, true, Flags, ORE)) {
         if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE)) {
-          if (!FreeInLoop) {
+          if (!FoldableInLoop) {
             ++II;
             salvageDebugInfo(I);
             eraseInstruction(I, *SafetyInfo, MSSAU);
@@ -883,6 +883,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
   LoopBlocksRPO Worklist(CurLoop);
   Worklist.perform(LI);
   bool Changed = false;
+  BasicBlock *Preheader = CurLoop->getLoopPreheader();
   for (BasicBlock *BB : Worklist) {
     // Only need to process the contents of this block if it is not part of a
     // subloop (which would already have been processed).
@@ -890,21 +891,6 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
       continue;
 
     for (Instruction &I : llvm::make_early_inc_range(*BB)) {
-      // Try constant folding this instruction.  If all the operands are
-      // constants, it is technically hoistable, but it would be better to
-      // just fold it.
-      if (Constant *C = ConstantFoldInstruction(
-              &I, I.getModule()->getDataLayout(), TLI)) {
-        LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << "  --> " << *C
-                          << '\n');
-        // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
-        I.replaceAllUsesWith(C);
-        if (isInstructionTriviallyDead(&I, TLI))
-          eraseInstruction(I, *SafetyInfo, MSSAU);
-        Changed = true;
-        continue;
-      }
-
       // Try hoisting the instruction out to the preheader.  We can only do
       // this if all of the operands of the instruction are loop invariant and
       // if it is safe to hoist the instruction. We also check block frequency
@@ -916,8 +902,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
           canSinkOrHoistInst(I, AA, DT, CurLoop, MSSAU, true, Flags, ORE) &&
           isSafeToExecuteUnconditionally(
               I, DT, TLI, CurLoop, SafetyInfo, ORE,
-              CurLoop->getLoopPreheader()->getTerminator(), AC,
-              AllowSpeculation)) {
+              Preheader->getTerminator(), AC, AllowSpeculation)) {
         hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
               MSSAU, SE, ORE);
         HoistedInstructions.push_back(&I);
@@ -1354,13 +1339,12 @@ static bool isTriviallyReplaceablePHI(const PHINode &PN, const Instruction &I) {
   return true;
 }
 
-/// Return true if the instruction is free in the loop.
-static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
+/// Return true if the instruction is foldable in the loop.
+static bool isFoldableInLoop(const Instruction &I, const Loop *CurLoop,
                          const TargetTransformInfo *TTI) {
-  InstructionCost CostI =
-      TTI->getInstructionCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
-
   if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
+    InstructionCost CostI =
+        TTI->getInstructionCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
     if (CostI != TargetTransformInfo::TCC_Free)
       return false;
     // For a GEP, we cannot simply use getInstructionCost because currently
@@ -1377,7 +1361,7 @@ static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
     return true;
   }
 
-  return CostI == TargetTransformInfo::TCC_Free;
+  return false;
 }
 
 /// Return true if the only users of this instruction are outside of
@@ -1386,12 +1370,12 @@ static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
 ///
 /// We also return true if the instruction could be folded away in lowering.
 /// (e.g.,  a GEP can be folded into a load as an addressing mode in the loop).
-static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
-                                  const LoopSafetyInfo *SafetyInfo,
-                                  TargetTransformInfo *TTI, bool &FreeInLoop,
-                                  bool LoopNestMode) {
+static bool isNotUsedOrFoldableInLoop(const Instruction &I, const Loop *CurLoop,
+                                      const LoopSafetyInfo *SafetyInfo,
+                                      TargetTransformInfo *TTI,
+                                      bool &FoldableInLoop, bool LoopNestMode) {
   const auto &BlockColors = SafetyInfo->getBlockColors();
-  bool IsFree = isFreeInLoop(I, CurLoop, TTI);
+  bool IsFoldable = isFoldableInLoop(I, CurLoop, TTI);
   for (const User *U : I.users()) {
     const Instruction *UI = cast<Instruction>(U);
     if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
@@ -1418,8 +1402,8 @@ static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
     }
 
     if (CurLoop->contains(UI)) {
-      if (IsFree) {
-        FreeInLoop = true;
+      if (IsFoldable) {
+        FoldableInLoop = true;
         continue;
       }
       return false;
@@ -1518,7 +1502,7 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest,
           MSSAU.getMemorySSA()->getMemoryAccess(&I)))
     MSSAU.moveToPlace(OldMemAcc, Dest.getParent(), MemorySSA::BeforeTerminator);
   if (SE)
-    SE->forgetValue(&I);
+    SE->forgetBlockAndLoopDispositions(&I);
 }
 
 static Instruction *sinkThroughTriviallyReplaceablePHI(
@@ -2421,19 +2405,13 @@ bool pointerInvalidatedByBlock(BasicBlock &BB, MemorySSA &MSSA, MemoryUse &MU) {
 static bool hoistMinMax(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo,
                         MemorySSAUpdater &MSSAU) {
   bool Inverse = false;
-  bool IsLogical = false;
   using namespace PatternMatch;
   Value *Cond1, *Cond2;
-  if (match(&I, m_Or(m_Value(Cond1), m_Value(Cond2))))
+  if (match(&I, m_LogicalOr(m_Value(Cond1), m_Value(Cond2)))) {
     Inverse = true;
-  else if (match(&I, m_LogicalOr(m_Value(Cond1), m_Value(Cond2)))) {
-    Inverse = true;
-    IsLogical = true;
-  } else if (match(&I, m_And(m_Value(Cond1), m_Value(Cond2)))) {
+  } else if (match(&I, m_LogicalAnd(m_Value(Cond1), m_Value(Cond2)))) {
     // Do nothing
-  } else if (match(&I, m_LogicalAnd(m_Value(Cond1), m_Value(Cond2))))
-    IsLogical = true;
-  else
+  } else
     return false;
 
   auto MatchICmpAgainstInvariant = [&](Value *C, ICmpInst::Predicate &P,
@@ -2477,7 +2455,7 @@ static bool hoistMinMax(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo,
   // before (if it was a non-taken input of logical and/or instruction). If it
   // was poison, we need to freeze it. Note that no new use for LHS and RHS1 are
   // introduced, so they don't need this.
-  if (IsLogical)
+  if (isa<SelectInst>(I))
     RHS2 = Builder.CreateFreeze(RHS2, RHS2->getName() + ".fr");
   Value *NewRHS = Builder.CreateBinaryIntrinsic(
       id, RHS1, RHS2, nullptr, StringRef("invariant.") +
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index 591f30c7d2264..edc8a4956dd1c 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -315,12 +315,12 @@ static bool verifyTripCount(Value *RHS, Loop *L,
     return false;
   }
 
-  // The Extend=false flag is used for getTripCountFromExitCount as we want
-  // to verify and match it with the pattern matched tripcount. Please note
-  // that overflow checks are performed in checkOverflow, but are first tried
-  // to avoid by widening the IV.
+  // Evaluating in the trip count's type can not overflow here as the overflow
+  // checks are performed in checkOverflow, but are first tried to avoid by
+  // widening the IV.
   const SCEV *SCEVTripCount =
-      SE->getTripCountFromExitCount(BackedgeTakenCount, /*Extend=*/false);
+    SE->getTripCountFromExitCount(BackedgeTakenCount,
+                                  BackedgeTakenCount->getType(), L);
 
   const SCEV *SCEVRHS = SE->getSCEV(RHS);
   if (SCEVRHS == SCEVTripCount)
@@ -333,7 +333,8 @@ static bool verifyTripCount(Value *RHS, Loop *L,
       // Find the extended backedge taken count and extended trip count using
       // SCEV. One of these should now match the RHS of the compare.
       BackedgeTCExt = SE->getZeroExtendExpr(BackedgeTakenCount, RHS->getType());
-      SCEVTripCountExt = SE->getTripCountFromExitCount(BackedgeTCExt, false);
+      SCEVTripCountExt = SE->getTripCountFromExitCount(BackedgeTCExt,
+                                                       RHS->getType(), L);
       if (SCEVRHS != BackedgeTCExt && SCEVRHS != SCEVTripCountExt) {
         LLVM_DEBUG(dbgs() << "Could not find valid trip count\n");
         return false;
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 2c999d7210186..bb0099e409a9d 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -983,33 +983,6 @@ static const SCEV *getStartForNegStride(const SCEV *Start, const SCEV *BECount,
   return SE->getMinusSCEV(Start, Index);
 }
 
-/// Compute trip count from the backedge taken count.
-static const SCEV *getTripCount(const SCEV *BECount, Type *IntPtr,
-                                Loop *CurLoop, const DataLayout *DL,
-                                ScalarEvolution *SE) {
-  const SCEV *TripCountS = nullptr;
-  // The # stored bytes is (BECount+1).  Expand the trip count out to
-  // pointer size if it isn't already.
-  //
-  // If we're going to need to zero extend the BE count, check if we can add
-  // one to it prior to zero extending without overflow. Provided this is safe,
-  // it allows better simplification of the +1.
-  if (DL->getTypeSizeInBits(BECount->getType()) <
-          DL->getTypeSizeInBits(IntPtr) &&
-      SE->isLoopEntryGuardedByCond(
-          CurLoop, ICmpInst::ICMP_NE, BECount,
-          SE->getMinusOne(BECount->getType()))) {
-    TripCountS = SE->getZeroExtendExpr(
-        SE->getAddExpr(BECount, SE->getOne(BECount->getType())),
-        IntPtr);
-  } else {
-    TripCountS = SE->getAddExpr(SE->getTruncateOrZeroExtend(BECount, IntPtr),
-                                SE->getOne(IntPtr));
-  }
-
-  return TripCountS;
-}
-
 /// Compute the number of bytes as a SCEV from the backedge taken count.
 ///
 /// This also maps the SCEV into the provided type and tries to handle the
@@ -1017,8 +990,8 @@ static const SCEV *getTripCount(const SCEV *BECount, Type *IntPtr,
 static const SCEV *getNumBytes(const SCEV *BECount, Type *IntPtr,
                                const SCEV *StoreSizeSCEV, Loop *CurLoop,
                                const DataLayout *DL, ScalarEvolution *SE) {
-  const SCEV *TripCountSCEV = getTripCount(BECount, IntPtr, CurLoop, DL, SE);
-
+  const SCEV *TripCountSCEV =
+      SE->getTripCountFromExitCount(BECount, IntPtr, CurLoop);
   return SE->getMulExpr(TripCountSCEV,
                         SE->getTruncateOrZeroExtend(StoreSizeSCEV, IntPtr),
                         SCEV::FlagNUW);
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 8f53d6c528d92..ba2cad6591d0f 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -532,7 +532,7 @@ struct LoopInterchange {
     LLVM_DEBUG(dbgs() << "Loops interchanged.\n");
     LoopsInterchanged++;
 
-    llvm::formLCSSARecursively(*OuterLoop, *DT, LI, SE);
+    llvm::formLCSSARecursively(*OuterLoop, *DT, LI);
     return true;
   }
 };
@@ -1691,7 +1691,7 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
   for (Instruction &I :
        make_range(OuterLoopHeader->begin(), std::prev(OuterLoopHeader->end())))
     MayNeedLCSSAPhis.push_back(&I);
-  formLCSSAForInstructions(MayNeedLCSSAPhis, *DT, *LI, SE, Builder);
+  formLCSSAForInstructions(MayNeedLCSSAPhis, *DT, *LI, Builder);
 
   return true;
 }
diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
index 8d59fdff9236f..854bef03481f9 100644
--- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
@@ -417,7 +417,7 @@ class ConstantTerminatorFoldingImpl {
         else
           DTU.applyUpdates(DTUpdates);
         DTUpdates.clear();
-        formLCSSARecursively(*FixLCSSALoop, DT, &LI, &SE);
+        formLCSSARecursively(*FixLCSSALoop, DT, &LI);
         SE.forgetBlockAndLoopDispositions();
       }
     }
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 822b74d9ee2d6..1dd7153f0cb8f 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -5660,7 +5660,7 @@ void LSRInstance::RewriteForPHI(
     }
 
   IRBuilder<> Builder(L->getHeader()->getContext());
-  formLCSSAForInstructions(InsertedNonLCSSAInsts, DT, LI, &SE, Builder);
+  formLCSSAForInstructions(InsertedNonLCSSAInsts, DT, LI, Builder);
 }
 
 /// Emit instructions for the leading candidate expression for this LSRUse (this
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 70b9ff33c5d55..8ea3a7e8c7b64 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1602,7 +1602,7 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
   for (const auto &L : LI) {
     Changed |=
         simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */);
-    Changed |= formLCSSARecursively(*L, DT, &LI, &SE);
+    Changed |= formLCSSARecursively(*L, DT, &LI);
   }
 
   // Add the loop nests in the reverse order of LoopInfo. See method
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 1c8e4e3512dca..77a19567feef6 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -658,7 +658,7 @@ static void scalarizeMaskedExpandLoad(const DataLayout &DL, CallInst *CI,
   if (isConstantIntVector(Mask)) {
     unsigned MemIndex = 0;
     VResult = PoisonValue::get(VecType);
-    SmallVector<int, 16> ShuffleMask(VectorWidth, UndefMaskElem);
+    SmallVector<int, 16> ShuffleMask(VectorWidth, PoisonMaskElem);
     for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
       Value *InsertElt;
       if (cast<Constant>(Mask)->getAggregateElement(Idx)->isNullValue()) {
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 4f5ae7f0ced97..b303edbb7a163 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -521,6 +521,12 @@ bool ConstantOffsetExtractor::CanTraceInto(bool SignExtended,
       !haveNoCommonBitsSet(LHS, RHS, DL, nullptr, BO, DT))
     return false;
 
+  // FIXME: We don't currently support constants from the RHS of subs,
+  // when we are zero-extended, because we need a way to zero-extended
+  // them before they are negated.
+  if (ZeroExtended && !SignExtended && BO->getOpcode() == Instruction::Sub)
+    return false;
+
   // In addition, tracing into BO requires that its surrounding s/zext (if
   // any) is distributable to both operands.
   //
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 68e15de3a89df..461272f1ee86d 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -460,7 +460,7 @@ static void hoistLoopToNewParent(Loop &L, BasicBlock &Preheader,
     // Because we just hoisted a loop out of this one, we have essentially
     // created new exit paths from it. That means we need to form LCSSA PHI
     // nodes for values used in the no-longer-nested loop.
-    formLCSSA(*OldContainingL, DT, &LI, SE);
+    formLCSSA(*OldContainingL, DT, &LI);
 
     // We shouldn't need to form dedicated exits because the exit introduced
     // here is the (just split by unswitching) preheader. However, after trivial
@@ -2551,7 +2551,7 @@ static void unswitchNontrivialInvariants(
     // First build LCSSA for this loop so that we can preserve it when
     // forming dedicated exits. We don't want to perturb some other loop's
     // LCSSA while doing that CFG edit.
-    formLCSSA(UpdateL, DT, &LI, SE);
+    formLCSSA(UpdateL, DT, &LI);
 
     // For loops reached by this loop's original exit blocks we may
     // introduced new, non-dedicated exits. At least try to re-form dedicated
diff --git a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
index 7eb615e90b177..45cf98e65a5af 100644
--- a/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
+++ b/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp
@@ -574,45 +574,6 @@ PreservedAnalyses AssumeSimplifyPass::run(Function &F,
   return PA;
 }
 
-namespace {
-class AssumeSimplifyPassLegacyPass : public FunctionPass {
-public:
-  static char ID;
-
-  AssumeSimplifyPassLegacyPass() : FunctionPass(ID) {
-    initializeAssumeSimplifyPassLegacyPassPass(
-        *PassRegistry::getPassRegistry());
-  }
-  bool runOnFunction(Function &F) override {
-    if (skipFunction(F) || !EnableKnowledgeRetention)
-      return false;
-    AssumptionCache &AC =
-        getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
-    DominatorTreeWrapperPass *DTWP =
-        getAnalysisIfAvailable<DominatorTreeWrapperPass>();
-    return simplifyAssumes(F, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<AssumptionCacheTracker>();
-
-    AU.setPreservesAll();
-  }
-};
-} // namespace
-
-char AssumeSimplifyPassLegacyPass::ID = 0;
-
-INITIALIZE_PASS_BEGIN(AssumeSimplifyPassLegacyPass, "assume-simplify",
-                      "Assume Simplify", false, false)
-INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
-INITIALIZE_PASS_END(AssumeSimplifyPassLegacyPass, "assume-simplify",
-                    "Assume Simplify", false, false)
-
-FunctionPass *llvm::createAssumeSimplifyPass() {
-  return new AssumeSimplifyPassLegacyPass();
-}
-
 PreservedAnalyses AssumeBuilderPass::run(Function &F,
                                          FunctionAnalysisManager &AM) {
   AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 7509c482ee32b..5a66100ca9ea6 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -600,8 +600,8 @@ bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
   unsigned Depth = 0;
   while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
          VisitedBlocks.insert(BB).second) {
-    if (BB->getTerminatingDeoptimizeCall() ||
-        isa<UnreachableInst>(BB->getTerminator()))
+    if (isa<UnreachableInst>(BB->getTerminator()) ||
+        BB->getTerminatingDeoptimizeCall())
       return true;
     BB = BB->getUniqueSuccessor();
   }
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 6663ce6c4b0ac..a870071f3f641 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -58,6 +58,7 @@ add_llvm_component_library(LLVMTransformUtils
   MetaRenamer.cpp
   MisExpect.cpp
   ModuleUtils.cpp
+  MoveAutoInit.cpp
   NameAnonGlobals.cpp
   PredicateInfo.cpp
   PromoteMemoryToRegister.cpp
diff --git a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
index 2ccd2ab1641db..d424ebbef99d1 100644
--- a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
+++ b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
@@ -80,6 +80,13 @@ static void insertCall(Function &CurFn, StringRef Func,
 }
 
 static bool runOnFunction(Function &F, bool PostInlining) {
+  // The asm in a naked function may reasonably expect the argument registers
+  // and the return address register (if present) to be live. An inserted
+  // function call will clobber these registers. Simply skip naked functions for
+  // all targets.
+  if (F.hasFnAttribute(Attribute::Naked))
+    return false;
+
   StringRef EntryAttr = PostInlining ? "instrument-function-entry-inlined"
                                      : "instrument-function-entry";
 
diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
index 156926039c9f9..a35f83eb75a3a 100644
--- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
+++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
@@ -142,38 +142,3 @@ PreservedAnalyses InjectTLIMappings::run(Function &F,
   // Even if the pass adds IR attributes, the analyses are preserved.
   return PreservedAnalyses::all();
 }
-
-////////////////////////////////////////////////////////////////////////////////
-// Legacy PM Implementation.
-////////////////////////////////////////////////////////////////////////////////
-bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
-  const TargetLibraryInfo &TLI =
-      getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-  return runImpl(TLI, F);
-}
-
-void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.setPreservesCFG();
-  AU.addRequired<TargetLibraryInfoWrapperPass>();
-  AU.addPreserved<TargetLibraryInfoWrapperPass>();
-  AU.addPreserved<ScalarEvolutionWrapperPass>();
-  AU.addPreserved<AAResultsWrapperPass>();
-  AU.addPreserved<LoopAccessLegacyAnalysis>();
-  AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
-  AU.addPreserved<GlobalsAAWrapperPass>();
-}
-
-////////////////////////////////////////////////////////////////////////////////
-// Legacy Pass manager initialization
-////////////////////////////////////////////////////////////////////////////////
-char InjectTLIMappingsLegacy::ID = 0;
-
-INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
-                      "Inject TLI Mappings", false, false)
-INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
-INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
-                    false, false)
-
-FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
-  return new InjectTLIMappingsLegacy();
-}
diff --git a/llvm/lib/Transforms/Utils/InstructionNamer.cpp b/llvm/lib/Transforms/Utils/InstructionNamer.cpp
index f3499c9c8aed9..3ae570cfeb776 100644
--- a/llvm/lib/Transforms/Utils/InstructionNamer.cpp
+++ b/llvm/lib/Transforms/Utils/InstructionNamer.cpp
@@ -17,9 +17,6 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/Utils.h"
 
 using namespace llvm;
 
@@ -41,35 +38,7 @@ void nameInstructions(Function &F) {
   }
 }
 
-struct InstNamer : public FunctionPass {
-  static char ID; // Pass identification, replacement for typeid
-  InstNamer() : FunctionPass(ID) {
-    initializeInstNamerPass(*PassRegistry::getPassRegistry());
-  }
-
-  void getAnalysisUsage(AnalysisUsage &Info) const override {
-    Info.setPreservesAll();
-  }
-
-  bool runOnFunction(Function &F) override {
-    nameInstructions(F);
-    return true;
-  }
-};
-
-  char InstNamer::ID = 0;
-  } // namespace
-
-INITIALIZE_PASS(InstNamer, "instnamer",
-                "Assign names to anonymous instructions", false, false)
-char &llvm::InstructionNamerID = InstNamer::ID;
-//===----------------------------------------------------------------------===//
-//
-// InstructionNamer - Give any unnamed non-void instructions "tmp" names.
-//
-FunctionPass *llvm::createInstructionNamerPass() {
-  return new InstNamer();
-}
+} // namespace
 
 PreservedAnalyses InstructionNamerPass::run(Function &F,
                                             FunctionAnalysisManager &FAM) {
diff --git a/llvm/lib/Transforms/Utils/LCSSA.cpp b/llvm/lib/Transforms/Utils/LCSSA.cpp
index af79dc456ea6d..a0a3186523170 100644
--- a/llvm/lib/Transforms/Utils/LCSSA.cpp
+++ b/llvm/lib/Transforms/Utils/LCSSA.cpp
@@ -77,7 +77,7 @@ static bool isExitBlock(BasicBlock *BB,
 /// rewrite the uses.
 bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
                                     const DominatorTree &DT, const LoopInfo &LI,
-                                    ScalarEvolution *SE, IRBuilderBase &Builder,
+                                    IRBuilderBase &Builder,
                                     SmallVectorImpl<PHINode *> *PHIsToRemove) {
   SmallVector<Use *, 16> UsesToRewrite;
   SmallSetVector<PHINode *, 16> LocalPHIsToRemove;
@@ -150,11 +150,6 @@ bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
     SSAUpdater SSAUpdate(&InsertedPHIs);
     SSAUpdate.Initialize(I->getType(), I->getName());
 
-    // Force re-computation of I, as some users now need to use the new PHI
-    // node.
-    if (SE)
-      SE->forgetValue(I);
-
     // Insert the LCSSA phi's into all of the exit blocks dominated by the
     // value, and add them to the Phi's map.
     for (BasicBlock *ExitBB : ExitBlocks) {
@@ -338,8 +333,7 @@ static void computeBlocksDominatingExits(
   }
 }
 
-bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
-                     ScalarEvolution *SE) {
+bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI) {
   bool Changed = false;
 
 #ifdef EXPENSIVE_CHECKS
@@ -393,13 +387,7 @@ bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
   }
 
   IRBuilder<> Builder(L.getHeader()->getContext());
-  Changed = formLCSSAForInstructions(Worklist, DT, *LI, SE, Builder);
-
-  // If we modified the code, remove any caches about the loop from SCEV to
-  // avoid dangling entries.
-  // FIXME: This is a big hammer, can we clear the cache more selectively?
-  if (SE && Changed)
-    SE->forgetLoop(&L);
+  Changed = formLCSSAForInstructions(Worklist, DT, *LI, Builder);
 
   assert(L.isLCSSAForm(DT));
 
@@ -408,23 +396,22 @@ bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI,
 
 /// Process a loop nest depth first.
 bool llvm::formLCSSARecursively(Loop &L, const DominatorTree &DT,
-                                const LoopInfo *LI, ScalarEvolution *SE) {
+                                const LoopInfo *LI) {
   bool Changed = false;
 
   // Recurse depth-first through inner loops.
   for (Loop *SubLoop : L.getSubLoops())
-    Changed |= formLCSSARecursively(*SubLoop, DT, LI, SE);
+    Changed |= formLCSSARecursively(*SubLoop, DT, LI);
 
-  Changed |= formLCSSA(L, DT, LI, SE);
+  Changed |= formLCSSA(L, DT, LI);
   return Changed;
 }
 
 /// Process all loops in the function, inner-most out.
-static bool formLCSSAOnAllLoops(const LoopInfo *LI, const DominatorTree &DT,
-                                ScalarEvolution *SE) {
+static bool formLCSSAOnAllLoops(const LoopInfo *LI, const DominatorTree &DT) {
   bool Changed = false;
   for (const auto &L : *LI)
-    Changed |= formLCSSARecursively(*L, DT, LI, SE);
+    Changed |= formLCSSARecursively(*L, DT, LI);
   return Changed;
 }
 
@@ -438,7 +425,6 @@ struct LCSSAWrapperPass : public FunctionPass {
   // Cached analysis information for the current function.
   DominatorTree *DT;
   LoopInfo *LI;
-  ScalarEvolution *SE;
 
   bool runOnFunction(Function &F) override;
   void verifyAnalysis() const override {
@@ -495,17 +481,13 @@ char &llvm::LCSSAID = LCSSAWrapperPass::ID;
 bool LCSSAWrapperPass::runOnFunction(Function &F) {
   LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-  auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
-  SE = SEWP ? &SEWP->getSE() : nullptr;
-
-  return formLCSSAOnAllLoops(LI, *DT, SE);
+  return formLCSSAOnAllLoops(LI, *DT);
 }
 
 PreservedAnalyses LCSSAPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &LI = AM.getResult<LoopAnalysis>(F);
   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
-  auto *SE = AM.getCachedResult<ScalarEvolutionAnalysis>(F);
-  if (!formLCSSAOnAllLoops(&LI, DT, SE))
+  if (!formLCSSAOnAllLoops(&LI, DT))
     return PreservedAnalyses::all();
 
   PreservedAnalyses PA;
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index d8036f7299a59..0f5d2ce841f19 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -424,18 +424,10 @@ bool llvm::wouldInstructionBeTriviallyDead(Instruction *I,
   if (I->isEHPad())
     return false;
 
-  // We don't want debug info removed by anything this general, unless
-  // debug info is empty.
-  if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
-    if (DDI->getAddress())
-      return false;
-    return true;
-  }
-  if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
-    if (DVI->hasArgList() || DVI->getValue(0))
-      return false;
-    return true;
-  }
+  // We don't want debug info removed by anything this general.
+  if (isa<DbgVariableIntrinsic>(I))
+    return false;
+
   if (DbgLabelInst *DLI = dyn_cast<DbgLabelInst>(I)) {
     if (DLI->getLabel())
       return false;
@@ -2717,6 +2709,10 @@ void llvm::combineMetadata(Instruction *K, const Instruction *J,
         // Preserve !nontemporal if it is present on both instructions.
         K->setMetadata(Kind, JMD);
         break;
+      case LLVMContext::MD_prof:
+        if (DoesKMove)
+          K->setMetadata(Kind, MDNode::getMergedProfMetadata(KMD, JMD, K, J));
+        break;
     }
   }
   // Set !invariant.group from J if J has it. If both instructions have it
@@ -2745,6 +2741,7 @@ void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
                          LLVMContext::MD_dereferenceable_or_null,
                          LLVMContext::MD_access_group,
                          LLVMContext::MD_preserve_access_index,
+                         LLVMContext::MD_prof,
                          LLVMContext::MD_nontemporal,
                          LLVMContext::MD_noundef};
   combineMetadata(K, J, KnownIDs, KDominatesJ);
diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index 87a0e54e2704e..1b36a29bdf772 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -340,7 +340,7 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
     // We don't need to form LCSSA recursively, because there cannot be uses
     // inside a newly created loop of defs from inner loops as those would
     // already be a use of an LCSSA phi node.
-    formLCSSA(*L, *DT, LI, SE);
+    formLCSSA(*L, *DT, LI);
 
     assert(NewOuter->isRecursivelyLCSSAForm(*DT, *LI) &&
            "LCSSA is broken after separating nested loops!");
@@ -693,12 +693,6 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
     }
   }
 
-  // Changing exit conditions for blocks may affect exit counts of this loop and
-  // any of its paretns, so we must invalidate the entire subtree if we've made
-  // any changes.
-  if (Changed && SE)
-    SE->forgetTopmostLoop(L);
-
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
 
@@ -737,6 +731,13 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
     Changed |= simplifyOneLoop(Worklist.pop_back_val(), Worklist, DT, LI, SE,
                                AC, MSSAU, PreserveLCSSA);
 
+  // Changing exit conditions for blocks may affect exit counts of this loop and
+  // any of its parents, so we must invalidate the entire subtree if we've made
+  // any changes. Do this here rather than in simplifyOneLoop() as the top-most
+  // loop is going to be the same for all child loops.
+  if (Changed && SE)
+    SE->forgetTopmostLoop(L);
+
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 916c026928235..ee6fd695d6653 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -876,7 +876,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
         while (FixLCSSALoop->getParentLoop() != LatchLoop)
           FixLCSSALoop = FixLCSSALoop->getParentLoop();
 
-      formLCSSARecursively(*FixLCSSALoop, *DT, LI, SE);
+      formLCSSARecursively(*FixLCSSALoop, *DT, LI);
     } else if (PreserveLCSSA) {
       assert(OuterL->isLCSSAForm(*DT) &&
              "Loops should be in LCSSA form after loop-unroll.");
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 90c9396e11bd6..2e644d79421c4 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -769,7 +769,7 @@ void llvm::breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
   // exit blocks.  If that happened, we need to rebuild LCSSA on the outermost
   // loop which might have a had a block removed.
   if (OutermostLoop != L)
-    formLCSSARecursively(*OutermostLoop, DT, &LI, &SE);
+    formLCSSARecursively(*OutermostLoop, DT, &LI);
 }
 
 
diff --git a/llvm/lib/Transforms/Utils/MetaRenamer.cpp b/llvm/lib/Transforms/Utils/MetaRenamer.cpp
index 0ab750ad43cd1..06ccbf888aa30 100644
--- a/llvm/lib/Transforms/Utils/MetaRenamer.cpp
+++ b/llvm/lib/Transforms/Utils/MetaRenamer.cpp
@@ -30,10 +30,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/TypeFinder.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Transforms/Utils.h"
 
 using namespace llvm;
 
@@ -199,46 +196,8 @@ void MetaRename(Module &M,
   }
 }
 
-struct MetaRenamer : public ModulePass {
-  // Pass identification, replacement for typeid
-  static char ID;
-
-  MetaRenamer() : ModulePass(ID) {
-    initializeMetaRenamerPass(*PassRegistry::getPassRegistry());
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<TargetLibraryInfoWrapperPass>();
-    AU.setPreservesAll();
-  }
-
-  bool runOnModule(Module &M) override {
-    auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
-      return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-    };
-    MetaRename(M, GetTLI);
-    return true;
-  }
-};
-
 } // end anonymous namespace
 
-char MetaRenamer::ID = 0;
-
-INITIALIZE_PASS_BEGIN(MetaRenamer, "metarenamer",
-                      "Assign new names to everything", false, false)
-INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
-INITIALIZE_PASS_END(MetaRenamer, "metarenamer",
-                    "Assign new names to everything", false, false)
-
-//===----------------------------------------------------------------------===//
-//
-// MetaRenamer - Rename everything with metasyntactic names.
-//
-ModulePass *llvm::createMetaRenamerPass() {
-  return new MetaRenamer();
-}
-
 PreservedAnalyses MetaRenamerPass::run(Module &M, ModuleAnalysisManager &AM) {
   FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
diff --git a/llvm/lib/Transforms/Utils/MoveAutoInit.cpp b/llvm/lib/Transforms/Utils/MoveAutoInit.cpp
new file mode 100644
index 0000000000000..21249bd2cf83c
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/MoveAutoInit.cpp
@@ -0,0 +1,233 @@
+//===-- MoveAutoInit.cpp - move auto-init inst closer to their use site----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass moves instruction maked as auto-init closer to the basic block that
+// use it, eventually removing it from some control path of the function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/MoveAutoInit.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "move-auto-init"
+
+STATISTIC(NumMoved, "Number of instructions moved");
+
+static cl::opt<unsigned> MoveAutoInitThreshold(
+    "move-auto-init-threshold", cl::Hidden, cl::init(128),
+    cl::desc("Maximum instructions to analyze per moved initialization"));
+
+static bool hasAutoInitMetadata(const Instruction &I) {
+  return I.hasMetadata(LLVMContext::MD_annotation) &&
+         any_of(I.getMetadata(LLVMContext::MD_annotation)->operands(),
+                [](const MDOperand &Op) {
+                  return cast<MDString>(Op.get())->getString() == "auto-init";
+                });
+}
+
+static std::optional<MemoryLocation> writeToAlloca(const Instruction &I) {
+  MemoryLocation ML;
+  if (auto *MI = dyn_cast<MemIntrinsic>(&I))
+    ML = MemoryLocation::getForDest(MI);
+  else if (auto *SI = dyn_cast<StoreInst>(&I))
+    ML = MemoryLocation::get(SI);
+  else
+    assert(false && "memory location set");
+
+  if (isa<AllocaInst>(getUnderlyingObject(ML.Ptr)))
+    return ML;
+  else
+    return {};
+}
+
+/// Finds a BasicBlock in the CFG where instruction `I` can be moved to while
+/// not changing the Memory SSA ordering and being guarded by at least one
+/// condition.
+static BasicBlock *usersDominator(const MemoryLocation &ML, Instruction *I,
+                                  DominatorTree &DT, MemorySSA &MSSA) {
+  BasicBlock *CurrentDominator = nullptr;
+  MemoryUseOrDef &IMA = *MSSA.getMemoryAccess(I);
+  BatchAAResults AA(MSSA.getAA());
+
+  SmallPtrSet<MemoryAccess *, 8> Visited;
+
+  auto AsMemoryAccess = [](User *U) { return cast<MemoryAccess>(U); };
+  SmallVector<MemoryAccess *> WorkList(map_range(IMA.users(), AsMemoryAccess));
+
+  while (!WorkList.empty()) {
+    MemoryAccess *MA = WorkList.pop_back_val();
+    if (!Visited.insert(MA).second)
+      continue;
+
+    if (Visited.size() > MoveAutoInitThreshold)
+      return nullptr;
+
+    bool FoundClobberingUser = false;
+    if (auto *M = dyn_cast<MemoryUseOrDef>(MA)) {
+      Instruction *MI = M->getMemoryInst();
+
+      // If this memory instruction may not clobber `I`, we can skip it.
+      // LifetimeEnd is a valid user, but we do not want it in the user
+      // dominator.
+      if (AA.getModRefInfo(MI, ML) != ModRefInfo::NoModRef &&
+          !MI->isLifetimeStartOrEnd() && MI != I) {
+        FoundClobberingUser = true;
+        CurrentDominator = CurrentDominator
+                               ? DT.findNearestCommonDominator(CurrentDominator,
+                                                               MI->getParent())
+                               : MI->getParent();
+      }
+    }
+    if (!FoundClobberingUser) {
+      auto UsersAsMemoryAccesses = map_range(MA->users(), AsMemoryAccess);
+      append_range(WorkList, UsersAsMemoryAccesses);
+    }
+  }
+  return CurrentDominator;
+}
+
+static bool runMoveAutoInit(Function &F, DominatorTree &DT, MemorySSA &MSSA) {
+  BasicBlock &EntryBB = F.getEntryBlock();
+  SmallVector<std::pair<Instruction *, BasicBlock *>> JobList;
+
+  //
+  // Compute movable instructions.
+  //
+  for (Instruction &I : EntryBB) {
+    if (!hasAutoInitMetadata(I))
+      continue;
+
+    std::optional<MemoryLocation> ML = writeToAlloca(I);
+    if (!ML)
+      continue;
+
+    if (I.isVolatile())
+      continue;
+
+    BasicBlock *UsersDominator = usersDominator(ML.value(), &I, DT, MSSA);
+    if (!UsersDominator)
+      continue;
+
+    if (UsersDominator == &EntryBB)
+      continue;
+
+    // Traverse the CFG to detect cycles `UsersDominator` would be part of.
+    SmallPtrSet<BasicBlock *, 8> TransitiveSuccessors;
+    SmallVector<BasicBlock *> WorkList(successors(UsersDominator));
+    bool HasCycle = false;
+    while (!WorkList.empty()) {
+      BasicBlock *CurrBB = WorkList.pop_back_val();
+      if (CurrBB == UsersDominator)
+        // No early exit because we want to compute the full set of transitive
+        // successors.
+        HasCycle = true;
+      for (BasicBlock *Successor : successors(CurrBB)) {
+        if (!TransitiveSuccessors.insert(Successor).second)
+          continue;
+        WorkList.push_back(Successor);
+      }
+    }
+
+    // Don't insert if that could create multiple execution of I,
+    // but we can insert it in the non back-edge predecessors, if it exists.
+    if (HasCycle) {
+      BasicBlock *UsersDominatorHead = UsersDominator;
+      while (BasicBlock *UniquePredecessor =
+                 UsersDominatorHead->getUniquePredecessor())
+        UsersDominatorHead = UniquePredecessor;
+
+      if (UsersDominatorHead == &EntryBB)
+        continue;
+
+      BasicBlock *DominatingPredecessor = nullptr;
+      for (BasicBlock *Pred : predecessors(UsersDominatorHead)) {
+        // If one of the predecessor of the dominator also transitively is a
+        // successor, moving to the dominator would do the inverse of loop
+        // hoisting, and we don't want that.
+        if (TransitiveSuccessors.count(Pred))
+          continue;
+
+        DominatingPredecessor =
+            DominatingPredecessor
+                ? DT.findNearestCommonDominator(DominatingPredecessor, Pred)
+                : Pred;
+      }
+
+      if (!DominatingPredecessor || DominatingPredecessor == &EntryBB)
+        continue;
+
+      UsersDominator = DominatingPredecessor;
+    }
+
+    // CatchSwitchInst blocks can only have one instruction, so they are not
+    // good candidates for insertion.
+    while (isa<CatchSwitchInst>(UsersDominator->getFirstInsertionPt())) {
+      for (BasicBlock *Pred : predecessors(UsersDominator))
+        UsersDominator = DT.findNearestCommonDominator(UsersDominator, Pred);
+    }
+
+    // We finally found a place where I can be moved while not introducing extra
+    // execution, and guarded by at least one condition.
+    if (UsersDominator != &EntryBB)
+      JobList.emplace_back(&I, UsersDominator);
+  }
+
+  //
+  // Perform the actual substitution.
+  //
+  if (JobList.empty())
+    return false;
+
+  MemorySSAUpdater MSSAU(&MSSA);
+
+  // Reverse insertion to respect relative order between instructions:
+  // if two instructions are moved from the same BB to the same BB, we insert
+  // the second one in the front, then the first on top of it.
+  for (auto &Job : reverse(JobList)) {
+    Job.first->moveBefore(&*Job.second->getFirstInsertionPt());
+    MSSAU.moveToPlace(MSSA.getMemoryAccess(Job.first), Job.first->getParent(),
+                      MemorySSA::InsertionPlace::Beginning);
+  }
+
+  if (VerifyMemorySSA)
+    MSSA.verifyMemorySSA();
+
+  NumMoved += JobList.size();
+
+  return true;
+}
+
+PreservedAnalyses MoveAutoInitPass::run(Function &F,
+                                        FunctionAnalysisManager &AM) {
+
+  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
+  if (!runMoveAutoInit(F, DT, MSSA))
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA;
+  PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<MemorySSAAnalysis>();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index b83700cc258d0..14604df17841e 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -2558,7 +2558,7 @@ Value *SCEVExpander::fixupLCSSAFormFor(Value *V) {
   SmallVector<Instruction *, 1> ToUpdate;
   ToUpdate.push_back(DefI);
   SmallVector<PHINode *, 16> PHIsToRemove;
-  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, &SE, Builder, &PHIsToRemove);
+  formLCSSAForInstructions(ToUpdate, SE.DT, SE.LI, Builder, &PHIsToRemove);
   for (PHINode *PN : PHIsToRemove) {
     if (!PN->use_empty())
       continue;
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index cbfd7b7fe8600..2dfc16fecc880 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -271,10 +271,8 @@ class SimplifyCFGOpt {
   bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
                                              IRBuilder<> &Builder);
 
-  bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI,
-                             bool EqTermsOnly);
-  bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
-                              const TargetTransformInfo &TTI);
+  bool HoistThenElseCodeToIf(BranchInst *BI, bool EqTermsOnly);
+  bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB);
   bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
                                   BasicBlock *TrueBB, BasicBlock *FalseBB,
                                   uint32_t TrueWeight, uint32_t FalseWeight);
@@ -1458,7 +1456,7 @@ static bool isSafeToHoistInstr(Instruction *I, unsigned Flags) {
   // If we have seen an instruction with side effects, it's unsafe to reorder an
   // instruction which reads memory or itself has side effects.
   if ((Flags & SkipSideEffect) &&
-      (I->mayReadFromMemory() || I->mayHaveSideEffects()))
+      (I->mayReadFromMemory() || I->mayHaveSideEffects() || isa<AllocaInst>(I)))
     return false;
 
   // Reordering across an instruction which does not necessarily transfer
@@ -1486,14 +1484,43 @@ static bool isSafeToHoistInstr(Instruction *I, unsigned Flags) {
 
 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
 
+/// Helper function for HoistThenElseCodeToIf. Return true if identical
+/// instructions \p I1 and \p I2 can and should be hoisted.
+static bool shouldHoistCommonInstructions(Instruction *I1, Instruction *I2,
+                                          const TargetTransformInfo &TTI) {
+  // If we're going to hoist a call, make sure that the two instructions
+  // we're commoning/hoisting are both marked with musttail, or neither of
+  // them is marked as such. Otherwise, we might end up in a situation where
+  // we hoist from a block where the terminator is a `ret` to a block where
+  // the terminator is a `br`, and `musttail` calls expect to be followed by
+  // a return.
+  auto *C1 = dyn_cast<CallInst>(I1);
+  auto *C2 = dyn_cast<CallInst>(I2);
+  if (C1 && C2)
+    if (C1->isMustTailCall() != C2->isMustTailCall())
+      return false;
+
+  if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
+    return false;
+
+  // If any of the two call sites has nomerge or convergent attribute, stop
+  // hoisting.
+  if (const auto *CB1 = dyn_cast<CallBase>(I1))
+    if (CB1->cannotMerge() || CB1->isConvergent())
+      return false;
+  if (const auto *CB2 = dyn_cast<CallBase>(I2))
+    if (CB2->cannotMerge() || CB2->isConvergent())
+      return false;
+
+  return true;
+}
+
 /// Given a conditional branch that goes to BB1 and BB2, hoist any common code
 /// in the two blocks up into the branch block. The caller of this function
 /// guarantees that BI's block dominates BB1 and BB2. If EqTermsOnly is given,
 /// only perform hoisting in case both blocks only contain a terminator. In that
 /// case, only the original BI will be replaced and selects for PHIs are added.
-bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI,
-                                           const TargetTransformInfo &TTI,
-                                           bool EqTermsOnly) {
+bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI, bool EqTermsOnly) {
   // This does very trivial matching, with limited scanning, to find identical
   // instructions in the two blocks.  In particular, we don't want to get into
   // O(M*N) situations here where M and N are the sizes of BB1 and BB2.  As
@@ -1568,38 +1595,13 @@ bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI,
       goto HoistTerminator;
     }
 
-    if (I1->isIdenticalToWhenDefined(I2)) {
-      // Even if the instructions are identical, it may not be safe to hoist
-      // them if we have skipped over instructions with side effects or their
-      // operands weren't hoisted.
-      if (!isSafeToHoistInstr(I1, SkipFlagsBB1) ||
-          !isSafeToHoistInstr(I2, SkipFlagsBB2))
-        return Changed;
-
-      // If we're going to hoist a call, make sure that the two instructions
-      // we're commoning/hoisting are both marked with musttail, or neither of
-      // them is marked as such. Otherwise, we might end up in a situation where
-      // we hoist from a block where the terminator is a `ret` to a block where
-      // the terminator is a `br`, and `musttail` calls expect to be followed by
-      // a return.
-      auto *C1 = dyn_cast<CallInst>(I1);
-      auto *C2 = dyn_cast<CallInst>(I2);
-      if (C1 && C2)
-        if (C1->isMustTailCall() != C2->isMustTailCall())
-          return Changed;
-
-      if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
-        return Changed;
-
-      // If any of the two call sites has nomerge or convergent attribute, stop
-      // hoisting.
-      if (const auto *CB1 = dyn_cast<CallBase>(I1))
-        if (CB1->cannotMerge() || CB1->isConvergent())
-          return Changed;
-      if (const auto *CB2 = dyn_cast<CallBase>(I2))
-        if (CB2->cannotMerge() || CB2->isConvergent())
-          return Changed;
-
+    if (I1->isIdenticalToWhenDefined(I2) &&
+        // Even if the instructions are identical, it may not be safe to hoist
+        // them if we have skipped over instructions with side effects or their
+        // operands weren't hoisted.
+        isSafeToHoistInstr(I1, SkipFlagsBB1) &&
+        isSafeToHoistInstr(I2, SkipFlagsBB2) &&
+        shouldHoistCommonInstructions(I1, I2, TTI)) {
       if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) {
         assert(isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2));
         // The debug location is an integral part of a debug info intrinsic
@@ -2838,8 +2840,8 @@ static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB,
 /// \endcode
 ///
 /// \returns true if the conditional block is removed.
-bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
-                                            const TargetTransformInfo &TTI) {
+bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI,
+                                            BasicBlock *ThenBB) {
   // Be conservative for now. FP select instruction can often be expensive.
   Value *BrCond = BI->getCondition();
   if (isa<FCmpInst>(BrCond))
@@ -7024,8 +7026,7 @@ bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
   // can hoist it up to the branching block.
   if (BI->getSuccessor(0)->getSinglePredecessor()) {
     if (BI->getSuccessor(1)->getSinglePredecessor()) {
-      if (HoistCommon &&
-          HoistThenElseCodeToIf(BI, TTI, !Options.HoistCommonInsts))
+      if (HoistCommon && HoistThenElseCodeToIf(BI, !Options.HoistCommonInsts))
         return requestResimplify();
     } else {
       // If Successor #1 has multiple preds, we may be able to conditionally
@@ -7033,7 +7034,7 @@ bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
       Instruction *Succ0TI = BI->getSuccessor(0)->getTerminator();
       if (Succ0TI->getNumSuccessors() == 1 &&
           Succ0TI->getSuccessor(0) == BI->getSuccessor(1))
-        if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), TTI))
+        if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0)))
           return requestResimplify();
     }
   } else if (BI->getSuccessor(1)->getSinglePredecessor()) {
@@ -7042,7 +7043,7 @@ bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
     Instruction *Succ1TI = BI->getSuccessor(1)->getTerminator();
     if (Succ1TI->getNumSuccessors() == 1 &&
         Succ1TI->getSuccessor(0) == BI->getSuccessor(0))
-      if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), TTI))
+      if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1)))
         return requestResimplify();
   }
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index d40eecaaf58f1..65be1243cadab 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1661,7 +1661,7 @@ Value *LibCallSimplifier::optimizeRealloc(CallInst *CI, IRBuilderBase &B) {
 }
 
 // When enabled, replace operator new() calls marked with a hot or cold memprof
-// attribute with an operator new() call that takes a hot_cold_t parameter.
+// attribute with an operator new() call that takes a __hot_cold_t parameter.
 // Currently this is supported by the open source version of tcmalloc, see:
 // https://github.com/google/tcmalloc/blob/master/tcmalloc/new_extension.h
 Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
@@ -1680,34 +1680,34 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
   switch (Func) {
   case LibFunc_Znwm:
     return emitHotColdNew(CI->getArgOperand(0), B, TLI,
-                          LibFunc_Znwm10hot_cold_t, HotCold);
+                          LibFunc_Znwm12__hot_cold_t, HotCold);
   case LibFunc_Znam:
     return emitHotColdNew(CI->getArgOperand(0), B, TLI,
-                          LibFunc_Znam10hot_cold_t, HotCold);
+                          LibFunc_Znam12__hot_cold_t, HotCold);
   case LibFunc_ZnwmRKSt9nothrow_t:
     return emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnwmRKSt9nothrow_t10hot_cold_t,
+                                 TLI, LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
                                  HotCold);
   case LibFunc_ZnamRKSt9nothrow_t:
     return emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnamRKSt9nothrow_t10hot_cold_t,
+                                 TLI, LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
                                  HotCold);
   case LibFunc_ZnwmSt11align_val_t:
     return emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnwmSt11align_val_t10hot_cold_t,
+                                 TLI, LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
                                  HotCold);
   case LibFunc_ZnamSt11align_val_t:
     return emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
-                                 TLI, LibFunc_ZnamSt11align_val_t10hot_cold_t,
+                                 TLI, LibFunc_ZnamSt11align_val_t12__hot_cold_t,
                                  HotCold);
   case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
     return emitHotColdNewAlignedNoThrow(
         CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-        TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t, HotCold);
+        TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
     return emitHotColdNewAlignedNoThrow(
         CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
-        TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t, HotCold);
+        TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
   default:
     return nullptr;
   }
diff --git a/llvm/lib/Transforms/Utils/Utils.cpp b/llvm/lib/Transforms/Utils/Utils.cpp
index 8a4cb659e7471..401bbe2dbea41 100644
--- a/llvm/lib/Transforms/Utils/Utils.cpp
+++ b/llvm/lib/Transforms/Utils/Utils.cpp
@@ -22,11 +22,9 @@ using namespace llvm;
 /// library.
 void llvm::initializeTransformUtils(PassRegistry &Registry) {
   initializeAddDiscriminatorsLegacyPassPass(Registry);
-  initializeAssumeSimplifyPassLegacyPassPass(Registry);
   initializeAssumeBuilderPassLegacyPassPass(Registry);
   initializeBreakCriticalEdgesPass(Registry);
   initializeCanonicalizeFreezeInLoopsPass(Registry);
-  initializeInstNamerPass(Registry);
   initializeLCSSAWrapperPassPass(Registry);
   initializeLoopSimplifyPass(Registry);
   initializeLowerGlobalDtorsLegacyPassPass(Registry);
@@ -34,10 +32,8 @@ void llvm::initializeTransformUtils(PassRegistry &Registry) {
   initializeLowerSwitchLegacyPassPass(Registry);
   initializePromoteLegacyPassPass(Registry);
   initializeUnifyFunctionExitNodesLegacyPassPass(Registry);
-  initializeMetaRenamerPass(Registry);
   initializeStripGCRelocatesLegacyPass(Registry);
   initializePredicateInfoPrinterLegacyPassPass(Registry);
-  initializeInjectTLIMappingsLegacyPass(Registry);
   initializeFixIrreduciblePass(Registry);
   initializeUnifyLoopExitsLegacyPassPass(Registry);
 }
diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index 6fd608737ec40..cad523ce9e3eb 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -529,6 +529,8 @@ Value *Mapper::mapValue(const Value *V) {
     return getVM()[V] = UndefValue::get(NewTy);
   if (isa<ConstantAggregateZero>(C))
     return getVM()[V] = ConstantAggregateZero::get(NewTy);
+  if (isa<ConstantTargetNone>(C))
+    return getVM()[V] = Constant::getNullValue(NewTy);
   assert(isa<ConstantPointerNull>(C));
   return getVM()[V] = ConstantPointerNull::get(cast<PointerType>(NewTy));
 }
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 317e31107ce22..3a35f3b754743 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -319,8 +319,8 @@ class LoopVectorizationPlanner {
   void printPlans(raw_ostream &O);
 #endif
 
-  /// Look through the existing plans and return true if we have one with all
-  /// the vectorization factors in question.
+  /// Look through the existing plans and return true if we have one with
+  /// vectorization factor \p VF.
   bool hasPlanWithVF(ElementCount VF) const {
     return any_of(VPlans,
                   [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 254af15523cc4..71d494d4797fc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -564,6 +564,14 @@ class InnerLoopVectorizer {
       ArrayRef<BasicBlock *> BypassBlocks,
       std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
 
+  /// Returns the original loop trip count.
+  Value *getTripCount() const { return TripCount; }
+
+  /// Used to set the trip count after ILV's construction and after the
+  /// preheader block has been executed. Note that this always holds the trip
+  /// count of the original loop for both main loop and epilogue vectorization.
+  void setTripCount(Value *TC) { TripCount = TC; }
+
 protected:
   friend class LoopVectorizationPlanner;
 
@@ -580,7 +588,7 @@ class InnerLoopVectorizer {
   void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
                     Value *VectorTripCount, Value *EndValue,
                     BasicBlock *MiddleBlock, BasicBlock *VectorHeader,
-                    VPlan &Plan);
+                    VPlan &Plan, VPTransformState &State);
 
   /// Handle all cross-iteration phis in the header.
   void fixCrossIterationPHIs(VPTransformState &State);
@@ -605,9 +613,6 @@ class InnerLoopVectorizer {
   /// represented as.
   void truncateToMinimalBitwidths(VPTransformState &State);
 
-  /// Returns (and creates if needed) the original loop trip count.
-  Value *getOrCreateTripCount(BasicBlock *InsertBlock);
-
   /// Returns (and creates if needed) the trip count of the widened loop.
   Value *getOrCreateVectorTripCount(BasicBlock *InsertBlock);
 
@@ -987,35 +992,7 @@ const SCEV *createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE,
   assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && "Invalid loop count");
 
   ScalarEvolution &SE = *PSE.getSE();
-
-  unsigned BackEdgeSize = SE.getTypeSizeInBits(BackedgeTakenCount->getType());
-  unsigned IdxSize = IdxTy->getPrimitiveSizeInBits();
-
-  // If we need to need to zero extend the backedge count, check if we can
-  // add one to it prior to zero extending without overflow. Provided this is
-  // safe, it allows better simplification of the +1.
-  if (OrigLoop && BackEdgeSize < IdxSize &&
-      SE.isLoopEntryGuardedByCond(
-          OrigLoop, ICmpInst::ICMP_NE, BackedgeTakenCount,
-          SE.getMinusOne(BackedgeTakenCount->getType()))) {
-    return SE.getZeroExtendExpr(
-        SE.getAddExpr(BackedgeTakenCount,
-                      SE.getOne(BackedgeTakenCount->getType())),
-        IdxTy);
-  }
-
-  // The exit count might have the type of i64 while the phi is i32. This can
-  // happen if we have an induction variable that is sign extended before the
-  // compare. The only way that we get a backedge taken count is that the
-  // induction variable was signed and as such will not overflow. In such a case
-  // truncation is legal.
-  if (BackEdgeSize > IdxSize)
-    BackedgeTakenCount = SE.getTruncateOrNoop(BackedgeTakenCount, IdxTy);
-  BackedgeTakenCount = SE.getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
-
-  // Get the total trip count from the count by adding 1.
-  return SE.getAddExpr(BackedgeTakenCount,
-                       SE.getOne(BackedgeTakenCount->getType()));
+  return SE.getTripCountFromExitCount(BackedgeTakenCount, IdxTy, OrigLoop);
 }
 
 static Value *getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy,
@@ -1221,12 +1198,13 @@ class LoopVectorizationCostModel {
   bool runtimeChecksRequired();
 
   /// \return The most profitable vectorization factor and the cost of that VF.
-  /// This method checks every VF in \p CandidateVFs. If UserVF is not ZERO
-  /// then this vectorization factor will be selected if vectorization is
-  /// possible.
+  /// This method checks every VF in \p CandidateVFs.
   VectorizationFactor
   selectVectorizationFactor(const ElementCountSet &CandidateVFs);
 
+  /// \return The most profitable vectorization factor and the cost of that VF
+  /// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if
+  /// epilogue vectorization is not supported for the loop.
   VectorizationFactor
   selectEpilogueVectorizationFactor(const ElementCount MaxVF,
                                     const LoopVectorizationPlanner &LVP);
@@ -1869,10 +1847,9 @@ class LoopVectorizationCostModel {
         Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
   }
 
-  /// Determines if we have the infrastructure to vectorize loop \p L and its
+  /// Determines if we have the infrastructure to vectorize the loop and its
   /// epilogue, assuming the main loop is vectorized by \p VF.
-  bool isCandidateForEpilogueVectorization(const Loop &L,
-                                           const ElementCount VF) const;
+  bool isCandidateForEpilogueVectorization(const ElementCount VF) const;
 
   /// Returns true if epilogue vectorization is considered profitable, and
   /// false otherwise.
@@ -2897,41 +2874,12 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
     PredicatedInstructions.push_back(Cloned);
 }
 
-Value *InnerLoopVectorizer::getOrCreateTripCount(BasicBlock *InsertBlock) {
-  if (TripCount)
-    return TripCount;
-
-  assert(InsertBlock);
-  IRBuilder<> Builder(InsertBlock->getTerminator());
-  // Find the loop boundaries.
-  Type *IdxTy = Legal->getWidestInductionType();
-  assert(IdxTy && "No type for induction");
-  const SCEV *ExitCount = createTripCountSCEV(IdxTy, PSE, OrigLoop);
-
-  const DataLayout &DL = InsertBlock->getModule()->getDataLayout();
-
-  // Expand the trip count and place the new instructions in the preheader.
-  // Notice that the pre-header does not change, only the loop body.
-  SCEVExpander Exp(*PSE.getSE(), DL, "induction");
-
-  // Count holds the overall loop count (N).
-  TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
-                                InsertBlock->getTerminator());
-
-  if (TripCount->getType()->isPointerTy())
-    TripCount =
-        CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
-                                    InsertBlock->getTerminator());
-
-  return TripCount;
-}
-
 Value *
 InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
   if (VectorTripCount)
     return VectorTripCount;
 
-  Value *TC = getOrCreateTripCount(InsertBlock);
+  Value *TC = getTripCount();
   IRBuilder<> Builder(InsertBlock->getTerminator());
 
   Type *Ty = TC->getType();
@@ -3009,7 +2957,7 @@ Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
 }
 
 void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
-  Value *Count = getOrCreateTripCount(LoopVectorPreHeader);
+  Value *Count = getTripCount();
   // Reuse existing vector loop preheader for TC checks.
   // Note that new preheader block is generated for vector loop.
   BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
@@ -3269,7 +3217,7 @@ void InnerLoopVectorizer::createInductionResumeValues(
 
 BasicBlock *InnerLoopVectorizer::completeLoopSkeleton() {
   // The trip counts should be cached by now.
-  Value *Count = getOrCreateTripCount(LoopVectorPreHeader);
+  Value *Count = getTripCount();
   Value *VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
 
   auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
@@ -3309,8 +3257,9 @@ InnerLoopVectorizer::createVectorizedLoopSkeleton() {
    the vectorized instructions while the old loop will continue to run the
    scalar remainder.
 
-       [ ] <-- loop iteration number check.
-    /   |
+       [ ] <-- old preheader - loop iteration number check and SCEVs in Plan's
+     /  |      preheader are expanded here. Eventually all required SCEV
+    /   |      expansion should happen here.
    /    v
   |    [ ] <-- vector loop bypass (may consist of multiple blocks).
   |  /  |
@@ -3370,7 +3319,8 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
                                        const InductionDescriptor &II,
                                        Value *VectorTripCount, Value *EndValue,
                                        BasicBlock *MiddleBlock,
-                                       BasicBlock *VectorHeader, VPlan &Plan) {
+                                       BasicBlock *VectorHeader, VPlan &Plan,
+                                       VPTransformState &State) {
   // There are two kinds of external IV usages - those that use the value
   // computed in the last iteration (the PHI) and those that use the penultimate
   // value (the value that feeds into the phi from the loop latch).
@@ -3398,7 +3348,6 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
     auto *UI = cast<Instruction>(U);
     if (!OrigLoop->contains(UI)) {
       assert(isa<PHINode>(UI) && "Expected LCSSA form");
-
       IRBuilder<> B(MiddleBlock->getTerminator());
 
       // Fast-math-flags propagate from the original induction instruction.
@@ -3408,8 +3357,11 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
       Value *CountMinusOne = B.CreateSub(
           VectorTripCount, ConstantInt::get(VectorTripCount->getType(), 1));
       CountMinusOne->setName("cmo");
-      Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(),
-                                    VectorHeader->getTerminator());
+
+      VPValue *StepVPV = Plan.getSCEVExpansion(II.getStep());
+      assert(StepVPV && "step must have been expanded during VPlan execution");
+      Value *Step = StepVPV->isLiveIn() ? StepVPV->getLiveInIRValue()
+                                        : State.get(StepVPV, {0, 0});
       Value *Escape =
           emitTransformedIndex(B, CountMinusOne, II.getStartValue(), Step, II);
       Escape->setName("ind.escape");
@@ -3768,7 +3720,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
       fixupIVUsers(Entry.first, Entry.second,
                    getOrCreateVectorTripCount(VectorLoop->getLoopPreheader()),
                    IVEndValues[Entry.first], LoopMiddleBlock,
-                   VectorLoop->getHeader(), Plan);
+                   VectorLoop->getHeader(), Plan, State);
   }
 
   // Fix LCSSA phis not already fixed earlier. Extracts may need to be generated
@@ -5394,17 +5346,25 @@ bool LoopVectorizationCostModel::isMoreProfitable(
 
   unsigned MaxTripCount = PSE.getSE()->getSmallConstantMaxTripCount(TheLoop);
 
-  if (!A.Width.isScalable() && !B.Width.isScalable() && foldTailByMasking() &&
-      MaxTripCount) {
-    // If we are folding the tail and the trip count is a known (possibly small)
-    // constant, the trip count will be rounded up to an integer number of
-    // iterations. The total cost will be PerIterationCost*ceil(TripCount/VF),
-    // which we compare directly. When not folding the tail, the total cost will
-    // be PerIterationCost*floor(TC/VF) + Scalar remainder cost, and so is
-    // approximated with the per-lane cost below instead of using the tripcount
-    // as here.
-    auto RTCostA = CostA * divideCeil(MaxTripCount, A.Width.getFixedValue());
-    auto RTCostB = CostB * divideCeil(MaxTripCount, B.Width.getFixedValue());
+  if (!A.Width.isScalable() && !B.Width.isScalable() && MaxTripCount) {
+    // If the trip count is a known (possibly small) constant, the trip count
+    // will be rounded up to an integer number of iterations under
+    // FoldTailByMasking. The total cost in that case will be
+    // VecCost*ceil(TripCount/VF). When not folding the tail, the total
+    // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
+    // some extra overheads, but for the purpose of comparing the costs of
+    // different VFs we can use this to compare the total loop-body cost
+    // expected after vectorization.
+    auto GetCostForTC = [MaxTripCount, this](unsigned VF,
+                                             InstructionCost VectorCost,
+                                             InstructionCost ScalarCost) {
+      return foldTailByMasking() ? VectorCost * divideCeil(MaxTripCount, VF)
+                                 : VectorCost * (MaxTripCount / VF) +
+                                       ScalarCost * (MaxTripCount % VF);
+    };
+    auto RTCostA = GetCostForTC(A.Width.getFixedValue(), CostA, A.ScalarCost);
+    auto RTCostB = GetCostForTC(B.Width.getFixedValue(), CostB, B.ScalarCost);
+
     return RTCostA < RTCostB;
   }
 
@@ -5572,10 +5532,10 @@ VectorizationFactor LoopVectorizationCostModel::selectVectorizationFactor(
 }
 
 bool LoopVectorizationCostModel::isCandidateForEpilogueVectorization(
-    const Loop &L, ElementCount VF) const {
+    ElementCount VF) const {
   // Cross iteration phis such as reductions need special handling and are
   // currently unsupported.
-  if (any_of(L.getHeader()->phis(),
+  if (any_of(TheLoop->getHeader()->phis(),
              [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
     return false;
 
@@ -5583,20 +5543,21 @@ bool LoopVectorizationCostModel::isCandidateForEpilogueVectorization(
   // currently unsupported.
   for (const auto &Entry : Legal->getInductionVars()) {
     // Look for uses of the value of the induction at the last iteration.
-    Value *PostInc = Entry.first->getIncomingValueForBlock(L.getLoopLatch());
+    Value *PostInc =
+        Entry.first->getIncomingValueForBlock(TheLoop->getLoopLatch());
     for (User *U : PostInc->users())
-      if (!L.contains(cast<Instruction>(U)))
+      if (!TheLoop->contains(cast<Instruction>(U)))
         return false;
     // Look for uses of penultimate value of the induction.
     for (User *U : Entry.first->users())
-      if (!L.contains(cast<Instruction>(U)))
+      if (!TheLoop->contains(cast<Instruction>(U)))
         return false;
   }
 
   // Epilogue vectorization code has not been auditted to ensure it handles
   // non-latch exits properly.  It may be fine, but it needs auditted and
   // tested.
-  if (L.getExitingBlock() != L.getLoopLatch())
+  if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch())
     return false;
 
   return true;
@@ -5643,7 +5604,7 @@ LoopVectorizationCostModel::selectEpilogueVectorizationFactor(
 
   // Not really a cost consideration, but check for unsupported cases here to
   // simplify the logic.
-  if (!isCandidateForEpilogueVectorization(*TheLoop, MainLoopVF)) {
+  if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
     LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
                          "is not a supported candidate.\n");
     return Result;
@@ -7576,6 +7537,9 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
 std::optional<VectorizationFactor>
 LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
   assert(OrigLoop->isInnermost() && "Inner loop expected.");
+  CM.collectValuesToIgnore();
+  CM.collectElementTypesForWidening();
+
   FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
   if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
     return std::nullopt;
@@ -7717,23 +7681,27 @@ void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF,
   LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << BestVF << ", UF=" << BestUF
                     << '\n');
 
-  // Workaround!  Compute the trip count of the original loop and cache it
-  // before we start modifying the CFG.  This code has a systemic problem
-  // wherein it tries to run analysis over partially constructed IR; this is
-  // wrong, and not simply for SCEV.  The trip count of the original loop
-  // simply happens to be prone to hitting this in practice.  In theory, we
-  // can hit the same issue for any SCEV, or ValueTracking query done during
-  // mutation.  See PR49900.
-  ILV.getOrCreateTripCount(OrigLoop->getLoopPreheader());
-
   if (!IsEpilogueVectorization)
     VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
 
   // Perform the actual loop transformation.
+  VPTransformState State{BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan};
+
+  // 0. Generate SCEV-dependent code into the preheader, including TripCount,
+  // before making any changes to the CFG.
+  if (!BestVPlan.getPreheader()->empty()) {
+    State.CFG.PrevBB = OrigLoop->getLoopPreheader();
+    State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator());
+    BestVPlan.getPreheader()->execute(&State);
+  }
+  if (!ILV.getTripCount())
+    ILV.setTripCount(State.get(BestVPlan.getTripCount(), {0, 0}));
+  else
+    assert(IsEpilogueVectorization && "should only re-use the existing trip "
+                                      "count during epilogue vectorization");
 
   // 1. Set up the skeleton for vectorization, including vector pre-header and
   // middle block. The vector loop is created during VPlan execution.
-  VPTransformState State{BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan};
   Value *CanonicalIVStartValue;
   std::tie(State.CFG.PrevBB, CanonicalIVStartValue) =
       ILV.createVectorizedLoopSkeleton();
@@ -7741,6 +7709,7 @@ void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF,
   // Only use noalias metadata when using memory checks guaranteeing no overlap
   // across all iterations.
   const LoopAccessInfo *LAI = ILV.Legal->getLAI();
+  std::unique_ptr<LoopVersioning> LVer = nullptr;
   if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
       !LAI->getRuntimePointerChecking()->getDiffChecks()) {
 
@@ -7748,9 +7717,10 @@ void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF,
     //  still use it to add the noalias metadata.
     //  TODO: Find a better way to re-use LoopVersioning functionality to add
     //        metadata.
-    State.LVer = std::make_unique<LoopVersioning>(
+    LVer = std::make_unique<LoopVersioning>(
         *LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
         PSE.getSE());
+    State.LVer = &*LVer;
     State.LVer->prepareNoAliasMetadata();
   }
 
@@ -7767,10 +7737,9 @@ void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF,
   //===------------------------------------------------===//
 
   // 2. Copy and widen instructions from the old loop into the new loop.
-  BestVPlan.prepareToExecute(ILV.getOrCreateTripCount(nullptr),
-                             ILV.getOrCreateVectorTripCount(nullptr),
-                             CanonicalIVStartValue, State,
-                             IsEpilogueVectorization);
+  BestVPlan.prepareToExecute(
+      ILV.getTripCount(), ILV.getOrCreateVectorTripCount(nullptr),
+      CanonicalIVStartValue, State, IsEpilogueVectorization);
 
   BestVPlan.execute(&State);
 
@@ -7885,7 +7854,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
   assert(Bypass && "Expected valid bypass basic block.");
   ElementCount VFactor = ForEpilogue ? EPI.EpilogueVF : VF;
   unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF;
-  Value *Count = getOrCreateTripCount(LoopVectorPreHeader);
+  Value *Count = getTripCount();
   // Reuse existing vector loop preheader for TC checks.
   // Note that new preheader block is generated for vector loop.
   BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
@@ -8204,7 +8173,7 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlan &Plan) {
     VPBuilder::InsertPointGuard Guard(Builder);
     Builder.setInsertPoint(HeaderVPBB, NewInsertionPoint);
     if (useActiveLaneMask(TFStyle)) {
-      VPValue *TC = Plan.getOrCreateTripCount();
+      VPValue *TC = Plan.getTripCount();
       BlockMask = Builder.createNaryOp(VPInstruction::ActiveLaneMask, {IV, TC},
                                        nullptr, "active.lane.mask");
     } else {
@@ -8768,7 +8737,8 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL,
   VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
   if (useActiveLaneMaskForControlFlow(Style)) {
     // Create the active lane mask instruction in the vplan preheader.
-    VPBasicBlock *Preheader = Plan.getEntry()->getEntryBasicBlock();
+    VPBasicBlock *VecPreheader =
+        cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSinglePredecessor());
 
     // We can't use StartV directly in the ActiveLaneMask VPInstruction, since
     // we have to take unrolling into account. Each part needs to start at
@@ -8777,10 +8747,10 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL,
         new VPInstruction(HasNUW ? VPInstruction::CanonicalIVIncrementForPartNUW
                                  : VPInstruction::CanonicalIVIncrementForPart,
                           {StartV}, DL, "index.part.next");
-    Preheader->appendRecipe(CanonicalIVIncrementParts);
+    VecPreheader->appendRecipe(CanonicalIVIncrementParts);
 
     // Create the ActiveLaneMask instruction using the correct start values.
-    VPValue *TC = Plan.getOrCreateTripCount();
+    VPValue *TC = Plan.getTripCount();
 
     VPValue *TripCount, *IncrementValue;
     if (Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck) {
@@ -8789,7 +8759,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL,
       // done after the active.lane.mask intrinsic is called.
       auto *TCMinusVF =
           new VPInstruction(VPInstruction::CalculateTripCountMinusVF, {TC}, DL);
-      Preheader->appendRecipe(TCMinusVF);
+      VecPreheader->appendRecipe(TCMinusVF);
       IncrementValue = CanonicalIVPHI;
       TripCount = TCMinusVF;
     } else {
@@ -8804,7 +8774,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL,
     auto *EntryALM = new VPInstruction(VPInstruction::ActiveLaneMask,
                                        {CanonicalIVIncrementParts, TC}, DL,
                                        "active.lane.mask.entry");
-    Preheader->appendRecipe(EntryALM);
+    VecPreheader->appendRecipe(EntryALM);
 
     // Now create the ActiveLaneMaskPhi recipe in the main loop using the
     // preheader ActiveLaneMask instruction.
@@ -8922,17 +8892,19 @@ std::optional<VPlanPtr> LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   // visit each basic block after having visited its predecessor basic blocks.
   // ---------------------------------------------------------------------------
 
-  // Create initial VPlan skeleton, starting with a block for the pre-header,
-  // followed by a region for the vector loop, followed by the middle block. The
-  // skeleton vector loop region contains a header and latch block.
-  VPBasicBlock *Preheader = new VPBasicBlock("vector.ph");
-  auto Plan = std::make_unique<VPlan>(Preheader);
-
+  // Create initial VPlan skeleton, having a basic block for the pre-header
+  // which contains SCEV expansions that need to happen before the CFG is
+  // modified; a basic block for the vector pre-header, followed by a region for
+  // the vector loop, followed by the middle basic block. The skeleton vector
+  // loop region contains a header and latch basic blocks.
+  VPlanPtr Plan = VPlan::createInitialVPlan(
+      createTripCountSCEV(Legal->getWidestInductionType(), PSE, OrigLoop),
+      *PSE.getSE());
   VPBasicBlock *HeaderVPBB = new VPBasicBlock("vector.body");
   VPBasicBlock *LatchVPBB = new VPBasicBlock("vector.latch");
   VPBlockUtils::insertBlockAfter(LatchVPBB, HeaderVPBB);
   auto *TopRegion = new VPRegionBlock(HeaderVPBB, LatchVPBB, "vector loop");
-  VPBlockUtils::insertBlockAfter(TopRegion, Preheader);
+  VPBlockUtils::insertBlockAfter(TopRegion, Plan->getEntry());
   VPBasicBlock *MiddleVPBB = new VPBasicBlock("middle.block");
   VPBlockUtils::insertBlockAfter(MiddleVPBB, TopRegion);
 
@@ -9120,7 +9092,9 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
   assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
 
   // Create new empty VPlan
-  auto Plan = std::make_unique<VPlan>();
+  auto Plan = VPlan::createInitialVPlan(
+      createTripCountSCEV(Legal->getWidestInductionType(), PSE, OrigLoop),
+      *PSE.getSE());
 
   // Build hierarchical CFG
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
@@ -9129,11 +9103,10 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
   for (ElementCount VF : Range)
     Plan->addVF(VF);
 
-  SmallPtrSet<Instruction *, 1> DeadInstructions;
   VPlanTransforms::VPInstructionsToVPRecipes(
       Plan,
       [this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
-      DeadInstructions, *PSE.getSE(), *TLI);
+      *PSE.getSE(), *TLI);
 
   // Remove the existing terminator of the exiting block of the top-most region.
   // A BranchOnCount will be added instead when adding the canonical IV recipes.
@@ -9623,7 +9596,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
 
   // A store of a loop varying value to a loop invariant address only
   // needs only the last copy of the store.
-  if (isa<StoreInst>(UI) && !getOperand(1)->hasDefiningRecipe()) {
+  if (isa<StoreInst>(UI) && getOperand(1)->isLiveIn()) {
     auto Lane = VPLane::getLastLaneForVF(State.VF);
     State.ILV->scalarizeInstruction(UI, this, VPIteration(State.UF - 1, Lane),
                                     State);
@@ -9842,9 +9815,11 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part) {
   unsigned LastLane = IsUniform ? 0 : VF.getKnownMinValue() - 1;
   // Check if there is a scalar value for the selected lane.
   if (!hasScalarValue(Def, {Part, LastLane})) {
-    // At the moment, VPWidenIntOrFpInductionRecipes and VPScalarIVStepsRecipes can also be uniform.
+    // At the moment, VPWidenIntOrFpInductionRecipes, VPScalarIVStepsRecipes and
+    // VPExpandSCEVRecipes can also be uniform.
     assert((isa<VPWidenIntOrFpInductionRecipe>(Def->getDefiningRecipe()) ||
-            isa<VPScalarIVStepsRecipe>(Def->getDefiningRecipe())) &&
+            isa<VPScalarIVStepsRecipe>(Def->getDefiningRecipe()) ||
+            isa<VPExpandSCEVRecipe>(Def->getDefiningRecipe())) &&
            "unexpected recipe found to be invariant");
     IsUniform = true;
     LastLane = 0;
@@ -10251,9 +10226,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   // Use the cost model.
   LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
                                 F, &Hints, IAI);
-  CM.collectValuesToIgnore();
-  CM.collectElementTypesForWidening();
-
   // Use the planner for vectorization.
   LoopVectorizationPlanner LVP(L, LI, TLI, TTI, &LVL, CM, IAI, PSE, Hints, ORE);
 
@@ -10434,6 +10406,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
         Header->setName("vec.epilog.vector.body");
 
+        // Re-use the trip count expanded for the main loop, as skeleton
+        // creation needs it as a value that dominates both the scalar and
+        // vector epilogue loops
+        EpilogILV.setTripCount(MainILV.getTripCount());
+        if (auto *R = BestEpiPlan.getTripCount()->getDefiningRecipe()) {
+          assert(BestEpiPlan.getTripCount()->getNumUsers() == 0 &&
+                 "trip count VPValue cannot be used in epilogue plan");
+          R->eraseFromParent();
+        }
+
         // Ensure that the start values for any VPWidenIntOrFpInductionRecipe,
         // VPWidenPointerInductionRecipe and VPReductionPHIRecipes are updated
         // before vectorizing the epilogue loop.
@@ -10576,7 +10558,7 @@ LoopVectorizeResult LoopVectorizePass::runImpl(
 
     // For the inner loops we actually process, form LCSSA to simplify the
     // transform.
-    Changed |= formLCSSARecursively(*L, *DT, LI, SE);
+    Changed |= formLCSSARecursively(*L, *DT, LI);
 
     Changed |= CFGChanged |= processLoop(L);
 
@@ -10628,6 +10610,11 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
     if (!EnableVPlanNativePath) {
       PA.preserve<LoopAnalysis>();
       PA.preserve<DominatorTreeAnalysis>();
+      PA.preserve<ScalarEvolutionAnalysis>();
+
+#ifdef EXPENSIVE_CHECKS
+      SE.verify();
+#endif
     }
 
     if (Result.MadeCFGChange) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8b6fa6883163c..99bacd0d334f7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -349,7 +349,7 @@ static SmallBitVector buildUseMask(int VF, ArrayRef<int> Mask,
                                    UseMask MaskArg) {
   SmallBitVector UseMask(VF, true);
   for (auto [Idx, Value] : enumerate(Mask)) {
-    if (Value == UndefMaskElem) {
+    if (Value == PoisonMaskElem) {
       if (MaskArg == UseMask::UndefsAsMask)
         UseMask.reset(Idx);
       continue;
@@ -467,7 +467,7 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
   Value *Vec2 = nullptr;
   enum ShuffleMode { Unknown, Select, Permute };
   ShuffleMode CommonShuffleMode = Unknown;
-  Mask.assign(VL.size(), UndefMaskElem);
+  Mask.assign(VL.size(), PoisonMaskElem);
   for (unsigned I = 0, E = VL.size(); I < E; ++I) {
     // Undef can be represented as an undef element in a vector.
     if (isa<UndefValue>(VL[I]))
@@ -644,7 +644,7 @@ tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
         !isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
         is_contained(UndefVectorExtracts, I))
       continue;
-    if (Mask[I] == UndefMaskElem && !isa<PoisonValue>(GatheredExtracts[I]))
+    if (Mask[I] == PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[I]))
       std::swap(VL[I], GatheredExtracts[I]);
   }
   return Res;
@@ -958,12 +958,12 @@ static void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask,
     Mask.append(SubMask.begin(), SubMask.end());
     return;
   }
-  SmallVector<int> NewMask(SubMask.size(), UndefMaskElem);
+  SmallVector<int> NewMask(SubMask.size(), PoisonMaskElem);
   int TermValue = std::min(Mask.size(), SubMask.size());
   for (int I = 0, E = SubMask.size(); I < E; ++I) {
     if ((!ExtendingManyInputs &&
          (SubMask[I] >= TermValue || Mask[SubMask[I]] >= TermValue)) ||
-        SubMask[I] == UndefMaskElem)
+        SubMask[I] == PoisonMaskElem)
       continue;
     NewMask[I] = Mask[SubMask[I]];
   }
@@ -1010,7 +1010,7 @@ static void inversePermutation(ArrayRef<unsigned> Indices,
                                SmallVectorImpl<int> &Mask) {
   Mask.clear();
   const unsigned E = Indices.size();
-  Mask.resize(E, UndefMaskElem);
+  Mask.resize(E, PoisonMaskElem);
   for (unsigned I = 0; I < E; ++I)
     Mask[Indices[I]] = I;
 }
@@ -1023,7 +1023,7 @@ static void reorderScalars(SmallVectorImpl<Value *> &Scalars,
                             UndefValue::get(Scalars.front()->getType()));
   Prev.swap(Scalars);
   for (unsigned I = 0, E = Prev.size(); I < E; ++I)
-    if (Mask[I] != UndefMaskElem)
+    if (Mask[I] != PoisonMaskElem)
       Scalars[Mask[I]] = Prev[I];
 }
 
@@ -2538,8 +2538,8 @@ class BoUpSLP {
                std::equal(VL.begin(), VL.end(), Mask.begin(),
                           [Scalars](Value *V, int Idx) {
                             return (isa<UndefValue>(V) &&
-                                    Idx == UndefMaskElem) ||
-                                   (Idx != UndefMaskElem && V == Scalars[Idx]);
+                                    Idx == PoisonMaskElem) ||
+                                   (Idx != PoisonMaskElem && V == Scalars[Idx]);
                           });
       };
       if (!ReorderIndices.empty()) {
@@ -2600,7 +2600,7 @@ class BoUpSLP {
     ValueList Scalars;
 
     /// The Scalars are vectorized into this value. It is initialized to Null.
-    Value *VectorizedValue = nullptr;
+    WeakTrackingVH VectorizedValue = nullptr;
 
     /// Do we need to gather this sequence or vectorize it
     /// (either with vector instruction or with scatter/gather
@@ -3719,7 +3719,7 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
   SmallVector<int> Prev(Reuses.begin(), Reuses.end());
   Prev.swap(Reuses);
   for (unsigned I = 0, E = Prev.size(); I < E; ++I)
-    if (Mask[I] != UndefMaskElem)
+    if (Mask[I] != PoisonMaskElem)
       Reuses[Mask[I]] = Prev[I];
 }
 
@@ -3743,7 +3743,7 @@ static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
   }
   Order.assign(Mask.size(), Mask.size());
   for (unsigned I = 0, E = Mask.size(); I < E; ++I)
-    if (MaskOrder[I] != UndefMaskElem)
+    if (MaskOrder[I] != PoisonMaskElem)
       Order[MaskOrder[I]] = I;
   fixupOrderingIndices(Order);
 }
@@ -4078,14 +4078,14 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
           std::optional<unsigned> Idx = getExtractIndex(cast<Instruction>(V));
           return Idx && *Idx < Sz;
         })) {
-      SmallVector<int> ReorderMask(Sz, UndefMaskElem);
+      SmallVector<int> ReorderMask(Sz, PoisonMaskElem);
       if (TE.ReorderIndices.empty())
         std::iota(ReorderMask.begin(), ReorderMask.end(), 0);
       else
         inversePermutation(TE.ReorderIndices, ReorderMask);
       for (unsigned I = 0; I < VF; ++I) {
         int &Idx = ReusedMask[I];
-        if (Idx == UndefMaskElem)
+        if (Idx == PoisonMaskElem)
           continue;
         Value *V = TE.Scalars[ReorderMask[Idx]];
         std::optional<unsigned> EI = getExtractIndex(cast<Instruction>(V));
@@ -4100,7 +4100,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
     for (unsigned K = 0; K < VF; K += Sz) {
       OrdersType CurrentOrder(TE.ReorderIndices);
       SmallVector<int> SubMask{ArrayRef(ReusedMask).slice(K, Sz)};
-      if (SubMask.front() == UndefMaskElem)
+      if (SubMask.front() == PoisonMaskElem)
         std::iota(SubMask.begin(), SubMask.end(), 0);
       reorderOrder(CurrentOrder, SubMask);
       transform(CurrentOrder, It, [K](unsigned Pos) { return Pos + K; });
@@ -4437,7 +4437,7 @@ void BoUpSLP::reorderTopToBottom() {
         unsigned E = Order.size();
         OrdersType CurrentOrder(E, E);
         transform(Mask, CurrentOrder.begin(), [E](int Idx) {
-          return Idx == UndefMaskElem ? E : static_cast<unsigned>(Idx);
+          return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
         });
         fixupOrderingIndices(CurrentOrder);
         ++OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second;
@@ -4462,10 +4462,10 @@ void BoUpSLP::reorderTopToBottom() {
       continue;
     SmallVector<int> Mask;
     inversePermutation(BestOrder, Mask);
-    SmallVector<int> MaskOrder(BestOrder.size(), UndefMaskElem);
+    SmallVector<int> MaskOrder(BestOrder.size(), PoisonMaskElem);
     unsigned E = BestOrder.size();
     transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
-      return I < E ? static_cast<int>(I) : UndefMaskElem;
+      return I < E ? static_cast<int>(I) : PoisonMaskElem;
     });
     // Do an actual reordering, if profitable.
     for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
@@ -4676,7 +4676,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
           unsigned E = Order.size();
           OrdersType CurrentOrder(E, E);
           transform(Mask, CurrentOrder.begin(), [E](int Idx) {
-            return Idx == UndefMaskElem ? E : static_cast<unsigned>(Idx);
+            return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
           });
           fixupOrderingIndices(CurrentOrder);
           OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second +=
@@ -4755,10 +4755,10 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
       VisitedOps.clear();
       SmallVector<int> Mask;
       inversePermutation(BestOrder, Mask);
-      SmallVector<int> MaskOrder(BestOrder.size(), UndefMaskElem);
+      SmallVector<int> MaskOrder(BestOrder.size(), PoisonMaskElem);
       unsigned E = BestOrder.size();
       transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
-        return I < E ? static_cast<int>(I) : UndefMaskElem;
+        return I < E ? static_cast<int>(I) : PoisonMaskElem;
       });
       for (const std::pair<unsigned, TreeEntry *> &Op : Data.second) {
         TreeEntry *TE = Op.second;
@@ -5167,7 +5167,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     for (Value *V : VL) {
       if (isConstant(V)) {
         ReuseShuffleIndicies.emplace_back(
-            isa<UndefValue>(V) ? UndefMaskElem : UniqueValues.size());
+            isa<UndefValue>(V) ? PoisonMaskElem : UniqueValues.size());
         UniqueValues.emplace_back(V);
         continue;
       }
@@ -6298,7 +6298,7 @@ buildShuffleEntryMask(ArrayRef<Value *> VL, ArrayRef<unsigned> ReorderIndices,
                       SmallVectorImpl<Value *> *OpScalars = nullptr,
                       SmallVectorImpl<Value *> *AltScalars = nullptr) {
   unsigned Sz = VL.size();
-  Mask.assign(Sz, UndefMaskElem);
+  Mask.assign(Sz, PoisonMaskElem);
   SmallVector<int> OrderMask;
   if (!ReorderIndices.empty())
     inversePermutation(ReorderIndices, OrderMask);
@@ -6318,9 +6318,9 @@ buildShuffleEntryMask(ArrayRef<Value *> VL, ArrayRef<unsigned> ReorderIndices,
     }
   }
   if (!ReusesIndices.empty()) {
-    SmallVector<int> NewMask(ReusesIndices.size(), UndefMaskElem);
+    SmallVector<int> NewMask(ReusesIndices.size(), PoisonMaskElem);
     transform(ReusesIndices, NewMask.begin(), [&Mask](int Idx) {
-      return Idx != UndefMaskElem ? Mask[Idx] : UndefMaskElem;
+      return Idx != PoisonMaskElem ? Mask[Idx] : PoisonMaskElem;
     });
     Mask.swap(NewMask);
   }
@@ -6440,13 +6440,13 @@ class BaseShuffleAnalysis {
   static void combineMasks(unsigned LocalVF, SmallVectorImpl<int> &Mask,
                            ArrayRef<int> ExtMask) {
     unsigned VF = Mask.size();
-    SmallVector<int> NewMask(ExtMask.size(), UndefMaskElem);
+    SmallVector<int> NewMask(ExtMask.size(), PoisonMaskElem);
     for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) {
-      if (ExtMask[I] == UndefMaskElem)
+      if (ExtMask[I] == PoisonMaskElem)
         continue;
       int MaskedIdx = Mask[ExtMask[I] % VF];
       NewMask[I] =
-          MaskedIdx == UndefMaskElem ? UndefMaskElem : MaskedIdx % LocalVF;
+          MaskedIdx == PoisonMaskElem ? PoisonMaskElem : MaskedIdx % LocalVF;
     }
     Mask.swap(NewMask);
   }
@@ -6533,9 +6533,9 @@ class BaseShuffleAnalysis {
       if (auto *SVOpTy =
               dyn_cast<FixedVectorType>(SV->getOperand(0)->getType()))
         LocalVF = SVOpTy->getNumElements();
-      SmallVector<int> ExtMask(Mask.size(), UndefMaskElem);
+      SmallVector<int> ExtMask(Mask.size(), PoisonMaskElem);
       for (auto [Idx, I] : enumerate(Mask)) {
-        if (I == UndefMaskElem ||
+        if (I == PoisonMaskElem ||
             static_cast<unsigned>(I) >= SV->getShuffleMask().size())
           continue;
         ExtMask[Idx] = SV->getMaskValue(I);
@@ -6551,11 +6551,11 @@ class BaseShuffleAnalysis {
       if (!IsOp1Undef && !IsOp2Undef) {
         // Update mask and mark undef elems.
         for (int &I : Mask) {
-          if (I == UndefMaskElem)
+          if (I == PoisonMaskElem)
             continue;
           if (SV->getMaskValue(I % SV->getShuffleMask().size()) ==
-              UndefMaskElem)
-            I = UndefMaskElem;
+              PoisonMaskElem)
+            I = PoisonMaskElem;
         }
         break;
       }
@@ -6577,8 +6577,8 @@ class BaseShuffleAnalysis {
                "Expected masks of same sizes.");
         // Clear known poison elements.
         for (auto [I, Idx] : enumerate(Mask))
-          if (Idx == UndefMaskElem)
-            IdentityMask[I] = UndefMaskElem;
+          if (Idx == PoisonMaskElem)
+            IdentityMask[I] = PoisonMaskElem;
         Mask.swap(IdentityMask);
         auto *Shuffle = dyn_cast<ShuffleVectorInst>(V);
         return SinglePermute &&
@@ -6598,9 +6598,9 @@ class BaseShuffleAnalysis {
   /// Smart shuffle instruction emission, walks through shuffles trees and
   /// tries to find the best matching vector for the actual shuffle
   /// instruction.
-  template <typename ShuffleBuilderTy>
-  static Value *createShuffle(Value *V1, Value *V2, ArrayRef<int> Mask,
-                              ShuffleBuilderTy &Builder) {
+  template <typename T, typename ShuffleBuilderTy>
+  static T createShuffle(Value *V1, Value *V2, ArrayRef<int> Mask,
+                         ShuffleBuilderTy &Builder) {
     assert(V1 && "Expected at least one vector value.");
     if (V2)
       Builder.resizeToMatch(V1, V2);
@@ -6614,8 +6614,8 @@ class BaseShuffleAnalysis {
       Value *Op2 = V2;
       int VF =
           cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
-      SmallVector<int> CombinedMask1(Mask.size(), UndefMaskElem);
-      SmallVector<int> CombinedMask2(Mask.size(), UndefMaskElem);
+      SmallVector<int> CombinedMask1(Mask.size(), PoisonMaskElem);
+      SmallVector<int> CombinedMask2(Mask.size(), PoisonMaskElem);
       for (int I = 0, E = Mask.size(); I < E; ++I) {
         if (Mask[I] < VF)
           CombinedMask1[I] = Mask[I];
@@ -6633,9 +6633,9 @@ class BaseShuffleAnalysis {
         // again.
         if (auto *SV1 = dyn_cast<ShuffleVectorInst>(Op1))
           if (auto *SV2 = dyn_cast<ShuffleVectorInst>(Op2)) {
-            SmallVector<int> ExtMask1(Mask.size(), UndefMaskElem);
+            SmallVector<int> ExtMask1(Mask.size(), PoisonMaskElem);
             for (auto [Idx, I] : enumerate(CombinedMask1)) {
-                if (I == UndefMaskElem)
+                if (I == PoisonMaskElem)
                 continue;
                 ExtMask1[Idx] = SV1->getMaskValue(I);
             }
@@ -6643,9 +6643,9 @@ class BaseShuffleAnalysis {
                 cast<FixedVectorType>(SV1->getOperand(1)->getType())
                     ->getNumElements(),
                 ExtMask1, UseMask::SecondArg);
-            SmallVector<int> ExtMask2(CombinedMask2.size(), UndefMaskElem);
+            SmallVector<int> ExtMask2(CombinedMask2.size(), PoisonMaskElem);
             for (auto [Idx, I] : enumerate(CombinedMask2)) {
-                if (I == UndefMaskElem)
+                if (I == PoisonMaskElem)
                 continue;
                 ExtMask2[Idx] = SV2->getMaskValue(I);
             }
@@ -6685,8 +6685,8 @@ class BaseShuffleAnalysis {
                         ->getElementCount()
                         .getKnownMinValue());
       for (int I = 0, E = Mask.size(); I < E; ++I) {
-        if (CombinedMask2[I] != UndefMaskElem) {
-          assert(CombinedMask1[I] == UndefMaskElem &&
+        if (CombinedMask2[I] != PoisonMaskElem) {
+          assert(CombinedMask1[I] == PoisonMaskElem &&
                  "Expected undefined mask element");
           CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF);
         }
@@ -6699,21 +6699,21 @@ class BaseShuffleAnalysis {
             isa<ShuffleVectorInst>(Op1) &&
             cast<ShuffleVectorInst>(Op1)->getShuffleMask() ==
                 ArrayRef(CombinedMask1))))
-        return Op1;
+        return Builder.createIdentity(Op1);
       return Builder.createShuffleVector(
           Op1, Op1 == Op2 ? PoisonValue::get(Op1->getType()) : Op2,
           CombinedMask1);
     }
     if (isa<PoisonValue>(V1))
-      return PoisonValue::get(FixedVectorType::get(
-          cast<VectorType>(V1->getType())->getElementType(), Mask.size()));
+      return Builder.createPoison(
+          cast<VectorType>(V1->getType())->getElementType(), Mask.size());
     SmallVector<int> NewMask(Mask.begin(), Mask.end());
     bool IsIdentity = peekThroughShuffles(V1, NewMask, /*SinglePermute=*/true);
     assert(V1 && "Expected non-null value after looking through shuffles.");
 
     if (!IsIdentity)
       return Builder.createShuffleVector(V1, NewMask);
-    return V1;
+    return Builder.createIdentity(V1);
   }
 };
 } // namespace
@@ -6727,7 +6727,7 @@ class BaseShuffleAnalysis {
 class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
   bool IsFinalized = false;
   SmallVector<int> CommonMask;
-  SmallVector<Value *, 2> InVectors;
+  SmallVector<PointerUnion<Value *, const TreeEntry *> , 2> InVectors;
   const TargetTransformInfo &TTI;
   InstructionCost Cost = 0;
   ArrayRef<Value *> VectorizedVals;
@@ -6880,25 +6880,25 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     // Process extracts in blocks of EltsPerVector to check if the source vector
     // operand can be re-used directly. If not, add the cost of creating a
     // shuffle to extract the values into a vector register.
-    SmallVector<int> RegMask(EltsPerVector, UndefMaskElem);
+    SmallVector<int> RegMask(EltsPerVector, PoisonMaskElem);
     for (auto *V : VL) {
       ++Idx;
 
       // Reached the start of a new vector registers.
       if (Idx % EltsPerVector == 0) {
-        RegMask.assign(EltsPerVector, UndefMaskElem);
+        RegMask.assign(EltsPerVector, PoisonMaskElem);
         AllConsecutive = true;
         continue;
       }
 
       // Need to exclude undefs from analysis.
-      if (isa<UndefValue>(V) || Mask[Idx] == UndefMaskElem)
+      if (isa<UndefValue>(V) || Mask[Idx] == PoisonMaskElem)
         continue;
 
       // Check all extracts for a vector register on the target directly
       // extract values in order.
       unsigned CurrentIdx = *getExtractIndex(cast<Instruction>(V));
-      if (!isa<UndefValue>(VL[Idx - 1]) && Mask[Idx - 1] != UndefMaskElem) {
+      if (!isa<UndefValue>(VL[Idx - 1]) && Mask[Idx - 1] != PoisonMaskElem) {
         unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1]));
         AllConsecutive &= PrevIdx + 1 == CurrentIdx &&
                           CurrentIdx % EltsPerVector == Idx % EltsPerVector;
@@ -6923,6 +6923,90 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     return Cost;
   }
 
+  class ShuffleCostBuilder {
+    const TargetTransformInfo &TTI;
+
+    static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
+      int Limit = 2 * VF;
+      return Mask.empty() ||
+             (VF == Mask.size() &&
+              all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) &&
+              ShuffleVectorInst::isIdentityMask(Mask));
+    }
+
+  public:
+    ShuffleCostBuilder(const TargetTransformInfo &TTI) : TTI(TTI) {}
+    ~ShuffleCostBuilder() = default;
+    InstructionCost createShuffleVector(Value *V1, Value *,
+                                        ArrayRef<int> Mask) const {
+      // Empty mask or identity mask are free.
+      unsigned VF =
+          cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
+      if (isEmptyOrIdentity(Mask, VF))
+        return TTI::TCC_Free;
+      return TTI.getShuffleCost(
+          TTI::SK_PermuteTwoSrc,
+          FixedVectorType::get(
+              cast<VectorType>(V1->getType())->getElementType(), Mask.size()),
+          Mask);
+    }
+    InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask) const {
+      // Empty mask or identity mask are free.
+      if (isEmptyOrIdentity(Mask, Mask.size()))
+        return TTI::TCC_Free;
+      return TTI.getShuffleCost(
+          TTI::SK_PermuteSingleSrc,
+          FixedVectorType::get(
+              cast<VectorType>(V1->getType())->getElementType(), Mask.size()),
+          Mask);
+    }
+    InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
+    InstructionCost createPoison(Type *Ty, unsigned VF) const {
+      return TTI::TCC_Free;
+    }
+    void resizeToMatch(Value *&, Value *&) const {}
+  };
+
+  /// Smart shuffle instruction emission, walks through shuffles trees and
+  /// tries to find the best matching vector for the actual shuffle
+  /// instruction.
+  InstructionCost
+  createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1,
+                const PointerUnion<Value *, const TreeEntry *> &P2,
+                ArrayRef<int> Mask) {
+    ShuffleCostBuilder Builder(TTI);
+    Value *V1 = P1.dyn_cast<Value *>(), *V2 = P2.dyn_cast<Value *>();
+    unsigned CommonVF = 0;
+    if (!V1) {
+      const TreeEntry *E = P1.get<const TreeEntry *>();
+      unsigned VF = E->getVectorFactor();
+      if (V2) {
+        unsigned V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
+        if (V2VF != VF && V2VF == E->Scalars.size())
+          VF = E->Scalars.size();
+      } else if (!P2.isNull()) {
+        const TreeEntry *E2 = P2.get<const TreeEntry *>();
+        if (E->Scalars.size() == E2->Scalars.size())
+          CommonVF = VF = E->Scalars.size();
+      }
+      V1 = Constant::getNullValue(
+          FixedVectorType::get(E->Scalars.front()->getType(), VF));
+    }
+    if (!V2 && !P2.isNull()) {
+      const TreeEntry *E = P2.get<const TreeEntry *>();
+      unsigned VF = E->getVectorFactor();
+      unsigned V1VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+      if (!CommonVF && V1VF == E->Scalars.size())
+        CommonVF = E->Scalars.size();
+      if (CommonVF)
+        VF = CommonVF;
+      V2 = Constant::getNullValue(
+          FixedVectorType::get(E->Scalars.front()->getType(), VF));
+    }
+    return BaseShuffleAnalysis::createShuffle<InstructionCost>(V1, V2, Mask,
+                                                               Builder);
+  }
+
 public:
   ShuffleCostEstimator(TargetTransformInfo &TTI,
                        ArrayRef<Value *> VectorizedVals, BoUpSLP &R,
@@ -6939,13 +7023,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     // If the resulting type is scalarized, do not adjust the cost.
     unsigned VecNumParts = TTI.getNumberOfParts(VecTy);
     if (VecNumParts == VecTy->getNumElements()) {
-      InVectors.assign(1, Constant::getNullValue(VecTy));
+      InVectors.assign(1, E);
       return nullptr;
     }
     DenseMap<Value *, int> ExtractVectorsTys;
     for (auto [I, V] : enumerate(VL)) {
       // Ignore non-extractelement scalars.
-      if (isa<UndefValue>(V) || (!Mask.empty() && Mask[I] == UndefMaskElem))
+      if (isa<UndefValue>(V) || (!Mask.empty() && Mask[I] == PoisonMaskElem))
         continue;
       // If all users of instruction are going to be vectorized and this
       // instruction itself is not going to be vectorized, consider this
@@ -7023,26 +7107,22 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     // into a vector and can be represented as a permutation elements in a
     // single input vector or of 2 input vectors.
     Cost += computeExtractCost(VL, Mask, ShuffleKind);
-    InVectors.assign(1, Constant::getNullValue(VecTy));
+    InVectors.assign(1, E);
     return VecBase;
   }
   void add(const TreeEntry *E1, const TreeEntry *E2, ArrayRef<int> Mask) {
     CommonMask.assign(Mask.begin(), Mask.end());
-    InVectors.assign(
-        2, Constant::getNullValue(FixedVectorType::get(
-               E1->Scalars.front()->getType(),
-               std::max(E1->getVectorFactor(), E2->getVectorFactor()))));
+    InVectors.assign({E1, E2});
   }
   void add(const TreeEntry *E1, ArrayRef<int> Mask) {
     CommonMask.assign(Mask.begin(), Mask.end());
-    InVectors.assign(
-        1, Constant::getNullValue(FixedVectorType::get(
-               E1->Scalars.front()->getType(), E1->getVectorFactor())));
+    InVectors.assign(1, E1);
   }
   void gather(ArrayRef<Value *> VL, Value *Root = nullptr) {
     Cost += getBuildVectorCost(VL, Root);
     if (!Root) {
       assert(InVectors.empty() && "Unexpected input vectors for buildvector.");
+      // FIXME: Need to find a way to avoid use of getNullValue here.
       InVectors.assign(1, Constant::getNullValue(FixedVectorType::get(
                               VL.front()->getType(), VL.size())));
     }
@@ -7058,13 +7138,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
         ShuffleVectorInst::isIdentityMask(CommonMask))
       return Cost;
     return Cost +
-           TTI.getShuffleCost(InVectors.size() == 2 ? TTI::SK_PermuteTwoSrc
-                                                    : TTI::SK_PermuteSingleSrc,
-                              FixedVectorType::get(
-                                  cast<VectorType>(InVectors.front()->getType())
-                                      ->getElementType(),
-                                  CommonMask.size()),
-                              CommonMask);
+           createShuffle(InVectors.front(),
+                         InVectors.size() == 2 ? InVectors.back() : nullptr,
+                         CommonMask);
   }
 
   ~ShuffleCostEstimator() {
@@ -7167,7 +7243,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       }
       // Remove shuffled elements from list of gathers.
       for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
-        if (Mask[I] != UndefMaskElem)
+        if (Mask[I] != PoisonMaskElem)
           GatheredScalars[I] = PoisonValue::get(ScalarTy);
       }
       LLVM_DEBUG(dbgs() << "SLP: shuffled " << Entries.size()
@@ -7398,7 +7474,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
 
     unsigned NumOfParts = TTI->getNumberOfParts(SrcVecTy);
 
-    SmallVector<int> InsertMask(NumElts, UndefMaskElem);
+    SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
     unsigned OffsetBeg = *getInsertIndex(VL.front());
     unsigned OffsetEnd = OffsetBeg;
     InsertMask[OffsetBeg] = 0;
@@ -7435,13 +7511,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
     SmallVector<int> Mask;
     if (!E->ReorderIndices.empty()) {
       inversePermutation(E->ReorderIndices, Mask);
-      Mask.append(InsertVecSz - Mask.size(), UndefMaskElem);
+      Mask.append(InsertVecSz - Mask.size(), PoisonMaskElem);
     } else {
-      Mask.assign(VecSz, UndefMaskElem);
+      Mask.assign(VecSz, PoisonMaskElem);
       std::iota(Mask.begin(), std::next(Mask.begin(), InsertVecSz), 0);
     }
     bool IsIdentity = true;
-    SmallVector<int> PrevMask(InsertVecSz, UndefMaskElem);
+    SmallVector<int> PrevMask(InsertVecSz, PoisonMaskElem);
     Mask.swap(PrevMask);
     for (unsigned I = 0; I < NumScalars; ++I) {
       unsigned InsertIdx = *getInsertIndex(VL[PrevMask[I]]);
@@ -7484,14 +7560,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
                                     InsertVecTy);
       } else {
         for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I)
-          Mask[I] = InMask.test(I) ? UndefMaskElem : I;
+          Mask[I] = InMask.test(I) ? PoisonMaskElem : I;
         for (unsigned I = OffsetBeg - Offset, End = OffsetEnd - Offset;
              I <= End; ++I)
-          if (Mask[I] != UndefMaskElem)
+          if (Mask[I] != PoisonMaskElem)
             Mask[I] = I + VecSz;
         for (unsigned I = OffsetEnd + 1 - Offset; I < VecSz; ++I)
           Mask[I] =
-              ((I >= InMask.size()) || InMask.test(I)) ? UndefMaskElem : I;
+              ((I >= InMask.size()) || InMask.test(I)) ? PoisonMaskElem : I;
         Cost += TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
       }
     }
@@ -8133,8 +8209,8 @@ static T *performExtractsShuffleAction(
         ResizeAction(ShuffleMask.begin()->first, Mask, /*ForSingleMask=*/false);
     SmallBitVector IsBasePoison = isUndefVector<true>(Base, UseMask);
     for (unsigned Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) {
-      if (Mask[Idx] == UndefMaskElem)
-        Mask[Idx] = IsBasePoison.test(Idx) ? UndefMaskElem : Idx;
+      if (Mask[Idx] == PoisonMaskElem)
+        Mask[Idx] = IsBasePoison.test(Idx) ? PoisonMaskElem : Idx;
       else
         Mask[Idx] = (Res.second ? Idx : Mask[Idx]) + VF;
     }
@@ -8163,8 +8239,8 @@ static T *performExtractsShuffleAction(
       // can shuffle them directly.
       ArrayRef<int> SecMask = VMIt->second;
       for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
-        if (SecMask[I] != UndefMaskElem) {
-          assert(Mask[I] == UndefMaskElem && "Multiple uses of scalars.");
+        if (SecMask[I] != PoisonMaskElem) {
+          assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars.");
           Mask[I] = SecMask[I] + Vec1VF;
         }
       }
@@ -8177,12 +8253,12 @@ static T *performExtractsShuffleAction(
           ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false);
       ArrayRef<int> SecMask = VMIt->second;
       for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
-        if (Mask[I] != UndefMaskElem) {
-          assert(SecMask[I] == UndefMaskElem && "Multiple uses of scalars.");
+        if (Mask[I] != PoisonMaskElem) {
+          assert(SecMask[I] == PoisonMaskElem && "Multiple uses of scalars.");
           if (Res1.second)
             Mask[I] = I;
-        } else if (SecMask[I] != UndefMaskElem) {
-          assert(Mask[I] == UndefMaskElem && "Multiple uses of scalars.");
+        } else if (SecMask[I] != PoisonMaskElem) {
+          assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars.");
           Mask[I] = (Res2.second ? I : SecMask[I]) + VF;
         }
       }
@@ -8199,11 +8275,11 @@ static T *performExtractsShuffleAction(
         ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false);
     ArrayRef<int> SecMask = VMIt->second;
     for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
-      if (SecMask[I] != UndefMaskElem) {
-        assert((Mask[I] == UndefMaskElem || IsBaseNotUndef) &&
+      if (SecMask[I] != PoisonMaskElem) {
+        assert((Mask[I] == PoisonMaskElem || IsBaseNotUndef) &&
                "Multiple uses of scalars.");
         Mask[I] = (Res.second ? I : SecMask[I]) + VF;
-      } else if (Mask[I] != UndefMaskElem) {
+      } else if (Mask[I] != PoisonMaskElem) {
         Mask[I] = I;
       }
     }
@@ -8298,7 +8374,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
             (void)ShuffleMasks.emplace_back();
             SmallVectorImpl<int> &Mask = ShuffleMasks.back()[ScalarTE];
             if (Mask.empty())
-              Mask.assign(FTy->getNumElements(), UndefMaskElem);
+              Mask.assign(FTy->getNumElements(), PoisonMaskElem);
             // Find the insertvector, vectorized in tree, if any.
             Value *Base = VU;
             while (auto *IEBase = dyn_cast<InsertElementInst>(Base)) {
@@ -8312,7 +8388,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
                 do {
                   IEBase = cast<InsertElementInst>(Base);
                   int Idx = *getInsertIndex(IEBase);
-                  assert(Mask[Idx] == UndefMaskElem &&
+                  assert(Mask[Idx] == PoisonMaskElem &&
                          "InsertElementInstruction used already.");
                   Mask[Idx] = Idx;
                   Base = IEBase->getOperand(0);
@@ -8332,7 +8408,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
           int InIdx = *InsertIdx;
           SmallVectorImpl<int> &Mask = ShuffleMasks[VecId][ScalarTE];
           if (Mask.empty())
-            Mask.assign(FTy->getNumElements(), UndefMaskElem);
+            Mask.assign(FTy->getNumElements(), PoisonMaskElem);
           Mask[InIdx] = EU.Lane;
           DemandedElts[VecId].setBit(InIdx);
           continue;
@@ -8371,7 +8447,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
          (all_of(Mask,
                  [VF](int Idx) { return Idx < 2 * static_cast<int>(VF); }) &&
           !ShuffleVectorInst::isIdentityMask(Mask)))) {
-      SmallVector<int> OrigMask(VecVF, UndefMaskElem);
+      SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
       std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
                 OrigMask.begin());
       C = TTI->getShuffleCost(
@@ -8457,7 +8533,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
   // No need to check for the topmost gather node.
   if (TE == VectorizableTree.front().get())
     return std::nullopt;
-  Mask.assign(VL.size(), UndefMaskElem);
+  Mask.assign(VL.size(), PoisonMaskElem);
   assert(TE->UserTreeIndices.size() == 1 &&
          "Expected only single user of the gather node.");
   // TODO: currently checking only for Scalars in the tree entry, need to count
@@ -8620,7 +8696,7 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
       // Clear undef scalars.
       for (int I = 0, Sz = VL.size(); I < Sz; ++I)
         if (isa<PoisonValue>(VL[I]))
-          Mask[I] = UndefMaskElem;
+          Mask[I] = PoisonMaskElem;
       return TargetTransformInfo::SK_PermuteSingleSrc;
     }
     // No perfect match, just shuffle, so choose the first tree node from the
@@ -8875,9 +8951,11 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
           LastInst = I;
         continue;
       }
-      assert(isVectorLikeInstWithConstOps(LastInst) &&
-             isVectorLikeInstWithConstOps(I) &&
-             "Expected vector-like insts only.");
+      assert(((E->getOpcode() == Instruction::GetElementPtr &&
+               !isa<GetElementPtrInst>(I)) ||
+              (isVectorLikeInstWithConstOps(LastInst) &&
+               isVectorLikeInstWithConstOps(I))) &&
+             "Expected vector-like or non-GEP in GEP node insts only.");
       if (!DT->isReachableFromEntry(LastInst->getParent())) {
         LastInst = I;
         continue;
@@ -8909,9 +8987,11 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
           FirstInst = I;
         continue;
       }
-      assert(isVectorLikeInstWithConstOps(FirstInst) &&
-             isVectorLikeInstWithConstOps(I) &&
-             "Expected vector-like insts only.");
+      assert(((E->getOpcode() == Instruction::GetElementPtr &&
+              !isa<GetElementPtrInst>(I)) ||
+             (isVectorLikeInstWithConstOps(FirstInst) &&
+              isVectorLikeInstWithConstOps(I))) &&
+                 "Expected vector-like or non-GEP in GEP node insts only.");
       if (!DT->isReachableFromEntry(FirstInst->getParent())) {
         FirstInst = I;
         continue;
@@ -8937,7 +9017,12 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
       (doesNotNeedToSchedule(E->Scalars) ||
        all_of(E->Scalars, isVectorLikeInstWithConstOps))) {
     Instruction *InsertInst;
-    if (all_of(E->Scalars, [](Value *V) {
+    if ((E->getOpcode() == Instruction::GetElementPtr &&
+         any_of(E->Scalars,
+                [](Value *V) {
+                  return !isa<GetElementPtrInst>(V) && isa<Instruction>(V);
+                })) ||
+        all_of(E->Scalars, [](Value *V) {
           return !isVectorLikeInstWithConstOps(V) && isUsedOutsideBlock(V);
         }))
       InsertInst = FindLastInst();
@@ -9067,7 +9152,7 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root) {
       if (isa<PoisonValue>(VL[I]))
         continue;
       if (auto *SV = dyn_cast<ShuffleVectorInst>(Root)) {
-        if (SV->getMaskValue(I) == UndefMaskElem)
+        if (SV->getMaskValue(I) == PoisonMaskElem)
           continue;
       }
     }
@@ -9169,6 +9254,10 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       }
       return Vec;
     }
+    Value *createIdentity(Value *V) { return V; }
+    Value *createPoison(Type *Ty, unsigned VF) {
+      return PoisonValue::get(FixedVectorType::get(Ty, VF));
+    }
     /// Resizes 2 input vector to match the sizes, if the they are not equal
     /// yet. The smallest vector is resized to the size of the larger vector.
     void resizeToMatch(Value *&V1, Value *&V2) {
@@ -9178,7 +9267,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       int V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
       int VF = std::max(V1VF, V2VF);
       int MinVF = std::min(V1VF, V2VF);
-      SmallVector<int> IdentityMask(VF, UndefMaskElem);
+      SmallVector<int> IdentityMask(VF, PoisonMaskElem);
       std::iota(IdentityMask.begin(), std::next(IdentityMask.begin(), MinVF),
                 0);
       Value *&Op = MinVF == V1VF ? V1 : V2;
@@ -9201,7 +9290,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
     assert(V1 && "Expected at least one vector value.");
     ShuffleIRBuilder ShuffleBuilder(Builder, R.GatherShuffleExtractSeq,
                                     R.CSEBlocks);
-    return BaseShuffleAnalysis::createShuffle(V1, V2, Mask, ShuffleBuilder);
+    return BaseShuffleAnalysis::createShuffle<Value *>(V1, V2, Mask,
+                                                       ShuffleBuilder);
   }
 
   /// Transforms mask \p CommonMask per given \p Mask to make proper set after
@@ -9209,7 +9299,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
   static void transformMaskAfterShuffle(MutableArrayRef<int> CommonMask,
                                         ArrayRef<int> Mask) {
     for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
-      if (Mask[Idx] != UndefMaskElem)
+      if (Mask[Idx] != PoisonMaskElem)
         CommonMask[Idx] = Idx;
   }
 
@@ -9222,7 +9312,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
     Value *VecBase = nullptr;
     for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
       int Idx = Mask[I];
-      if (Idx == UndefMaskElem)
+      if (Idx == PoisonMaskElem)
         continue;
       auto *EI = cast<ExtractElementInst>(E->Scalars[I]);
       VecBase = EI->getVectorOperand();
@@ -9270,7 +9360,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
     }
     V1 = createShuffle(V1, V2, Mask);
     for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
-      if (Mask[Idx] != UndefMaskElem)
+      if (Mask[Idx] != PoisonMaskElem)
         CommonMask[Idx] = Idx + Sz;
     InVectors.front() = Vec;
     if (InVectors.size() == 2)
@@ -9283,7 +9373,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
     if (InVectors.empty()) {
       if (!isa<FixedVectorType>(V1->getType())) {
         V1 = createShuffle(V1, nullptr, CommonMask);
-        CommonMask.assign(Mask.size(), UndefMaskElem);
+        CommonMask.assign(Mask.size(), PoisonMaskElem);
         transformMaskAfterShuffle(CommonMask, Mask);
       }
       InVectors.push_back(V1);
@@ -9305,7 +9395,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
           transformMaskAfterShuffle(CommonMask, CommonMask);
         }
         for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
-          if (CommonMask[Idx] == UndefMaskElem && Mask[Idx] != UndefMaskElem)
+          if (CommonMask[Idx] == PoisonMaskElem && Mask[Idx] != PoisonMaskElem)
             CommonMask[Idx] =
                 V->getType() != V1->getType()
                     ? Idx + Sz
@@ -9323,7 +9413,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       // Check if second vector is required if the used elements are already
       // used from the first one.
       for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
-        if (Mask[Idx] != UndefMaskElem && CommonMask[Idx] == UndefMaskElem) {
+        if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) {
           InVectors.push_back(V1);
           break;
         }
@@ -9332,7 +9422,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
     if (auto *FTy = dyn_cast<FixedVectorType>(V1->getType()))
       VF = FTy->getNumElements();
     for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
-      if (Mask[Idx] != UndefMaskElem && CommonMask[Idx] == UndefMaskElem)
+      if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
         CommonMask[Idx] = Mask[Idx] + (It == InVectors.begin() ? 0 : VF);
   }
   /// Adds another one input vector and the mask for the shuffling.
@@ -9357,13 +9447,13 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
         Vec = createShuffle(Vec, nullptr, CommonMask);
       }
       for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
-        if (CommonMask[Idx] != UndefMaskElem)
+        if (CommonMask[Idx] != PoisonMaskElem)
           CommonMask[Idx] = Idx;
       assert(VF > 0 &&
              "Expected vector length for the final value before action.");
       unsigned VecVF = cast<FixedVectorType>(Vec->getType())->getNumElements();
       if (VecVF < VF) {
-        SmallVector<int> ResizeMask(VF, UndefMaskElem);
+        SmallVector<int> ResizeMask(VF, PoisonMaskElem);
         std::iota(ResizeMask.begin(), std::next(ResizeMask.begin(), VecVF), 0);
         Vec = createShuffle(Vec, nullptr, ResizeMask);
       }
@@ -9374,9 +9464,9 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
       if (CommonMask.empty()) {
         CommonMask.assign(ExtMask.begin(), ExtMask.end());
       } else {
-        SmallVector<int> NewMask(ExtMask.size(), UndefMaskElem);
+        SmallVector<int> NewMask(ExtMask.size(), PoisonMaskElem);
         for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) {
-          if (ExtMask[I] == UndefMaskElem)
+          if (ExtMask[I] == PoisonMaskElem)
             continue;
           NewMask[I] = CommonMask[ExtMask[I]];
         }
@@ -9447,18 +9537,18 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
           // ... (use %2)
           // %shuffle = shuffle <2 x> %2, poison, <2 x> {2, 0}
           // br %block
-          SmallVector<int> UniqueIdxs(VF, UndefMaskElem);
+          SmallVector<int> UniqueIdxs(VF, PoisonMaskElem);
           SmallSet<int, 4> UsedIdxs;
           int Pos = 0;
           for (int Idx : VE->ReuseShuffleIndices) {
-            if (Idx != static_cast<int>(VF) && Idx != UndefMaskElem &&
+            if (Idx != static_cast<int>(VF) && Idx != PoisonMaskElem &&
                 UsedIdxs.insert(Idx).second)
               UniqueIdxs[Idx] = Pos;
             ++Pos;
           }
           assert(VF >= UsedIdxs.size() && "Expected vectorization factor "
                                           "less than original vector size.");
-          UniqueIdxs.append(VF - UsedIdxs.size(), UndefMaskElem);
+          UniqueIdxs.append(VF - UsedIdxs.size(), PoisonMaskElem);
           V = FinalShuffle(V, UniqueIdxs);
         } else {
           assert(VF < cast<FixedVectorType>(V->getType())->getNumElements() &&
@@ -9541,7 +9631,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
     if (It == VectorizableTree.end())
       return false;
     unsigned I =
-        *find_if_not(Mask, [](int Idx) { return Idx == UndefMaskElem; });
+        *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
     int Sz = Mask.size();
     if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) &&
         ShuffleVectorInst::isIdentityMask(Mask))
@@ -9601,10 +9691,10 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
         // Restore the mask for previous partially matched values.
         for (auto [I, V] : enumerate(E->Scalars)) {
           if (isa<PoisonValue>(V)) {
-            Mask[I] = UndefMaskElem;
+            Mask[I] = PoisonMaskElem;
             continue;
           }
-          if (Mask[I] == UndefMaskElem)
+          if (Mask[I] == PoisonMaskElem)
             Mask[I] = Entries.front()->findLaneForValue(V);
         }
         ShuffleBuilder.add(Entries.front()->VectorizedValue, Mask);
@@ -9620,7 +9710,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
       }
       // Remove shuffled elements from list of gathers.
       for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
-        if (Mask[I] != UndefMaskElem)
+        if (Mask[I] != PoisonMaskElem)
           GatheredScalars[I] = PoisonValue::get(ScalarTy);
       }
     }
@@ -9667,7 +9757,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
     if (NumNonConsts == 1) {
       // Restore single insert element.
       if (IsSplat) {
-        ReuseMask.assign(VF, UndefMaskElem);
+        ReuseMask.assign(VF, PoisonMaskElem);
         std::swap(Scalars.front(), Scalars[SinglePos]);
         if (!UndefPos.empty() && UndefPos.front() == 0)
           Scalars.front() = UndefValue::get(ScalarTy);
@@ -9706,7 +9796,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
         // Replace undefs by the poisons, emit broadcast and then emit
         // freeze.
         for_each(UndefPos, [&](int I) {
-          ReuseMask[I] = UndefMaskElem;
+          ReuseMask[I] = PoisonMaskElem;
           if (isa<UndefValue>(Scalars[I]))
             Scalars[I] = PoisonValue::get(ScalarTy);
         });
@@ -9724,9 +9814,9 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
       // Find input vectors.
       Value *Vec2 = nullptr;
       for (unsigned I = 0, Sz = ExtractMask.size(); I < Sz; ++I) {
-        if (ExtractMask[I] == UndefMaskElem ||
-            (!Mask.empty() && Mask[I] != UndefMaskElem)) {
-          ExtractMask[I] = UndefMaskElem;
+        if (ExtractMask[I] == PoisonMaskElem ||
+            (!Mask.empty() && Mask[I] != PoisonMaskElem)) {
+          ExtractMask[I] = PoisonMaskElem;
           continue;
         }
         if (isa<UndefValue>(E->Scalars[I]))
@@ -9813,7 +9903,7 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
     }
     // Generate constants for final shuffle and build a mask for them.
     if (!all_of(GatheredScalars, PoisonValue::classof)) {
-      SmallVector<int> BVMask(GatheredScalars.size(), UndefMaskElem);
+      SmallVector<int> BVMask(GatheredScalars.size(), PoisonMaskElem);
       TryPackScalars(GatheredScalars, BVMask, /*IsRootPoison=*/true);
       Value *BV = gather(GatheredScalars);
       ShuffleBuilder.add(BV, BVMask);
@@ -9833,14 +9923,14 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
           });
   } else if (!allConstant(GatheredScalars)) {
     // Gather unique scalars and all constants.
-    SmallVector<int> ReuseMask(GatheredScalars.size(), UndefMaskElem);
+    SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
     TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
     Vec = gather(GatheredScalars);
     ShuffleBuilder.add(Vec, ReuseMask);
     Vec = ShuffleBuilder.finalize(E->ReuseShuffleIndices);
   } else {
     // Gather all constants.
-    SmallVector<int> Mask(E->Scalars.size(), UndefMaskElem);
+    SmallVector<int> Mask(E->Scalars.size(), PoisonMaskElem);
     for (auto [I, V] : enumerate(E->Scalars)) {
       if (!isa<PoisonValue>(V))
         Mask[I] = I;
@@ -9984,14 +10074,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       SmallVector<int> Mask;
       if (!E->ReorderIndices.empty()) {
         inversePermutation(E->ReorderIndices, Mask);
-        Mask.append(NumElts - NumScalars, UndefMaskElem);
+        Mask.append(NumElts - NumScalars, PoisonMaskElem);
       } else {
-        Mask.assign(NumElts, UndefMaskElem);
+        Mask.assign(NumElts, PoisonMaskElem);
         std::iota(Mask.begin(), std::next(Mask.begin(), NumScalars), 0);
       }
       // Create InsertVector shuffle if necessary
       bool IsIdentity = true;
-      SmallVector<int> PrevMask(NumElts, UndefMaskElem);
+      SmallVector<int> PrevMask(NumElts, PoisonMaskElem);
       Mask.swap(PrevMask);
       for (unsigned I = 0; I < NumScalars; ++I) {
         Value *Scalar = E->Scalars[PrevMask[I]];
@@ -10007,9 +10097,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         }
       }
 
-      SmallVector<int> InsertMask(NumElts, UndefMaskElem);
+      SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
       for (unsigned I = 0; I < NumElts; I++) {
-        if (Mask[I] != UndefMaskElem)
+        if (Mask[I] != PoisonMaskElem)
           InsertMask[Offset + I] = I;
       }
       SmallBitVector UseMask =
@@ -10024,7 +10114,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
               isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
           if (!IsFirstPoison.all()) {
             for (unsigned I = 0; I < NumElts; I++) {
-              if (InsertMask[I] == UndefMaskElem && !IsFirstPoison.test(I))
+              if (InsertMask[I] == PoisonMaskElem && !IsFirstPoison.test(I))
                 InsertMask[I] = I + NumElts;
             }
             }
@@ -10042,8 +10132,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
           SmallBitVector IsFirstPoison =
               isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
           for (unsigned I = 0; I < NumElts; I++) {
-            if (InsertMask[I] == UndefMaskElem)
-              InsertMask[I] = IsFirstPoison.test(I) ? UndefMaskElem : I;
+            if (InsertMask[I] == PoisonMaskElem)
+              InsertMask[I] = IsFirstPoison.test(I) ? PoisonMaskElem : I;
             else
               InsertMask[I] += NumElts;
           }
@@ -10705,7 +10795,7 @@ Value *BoUpSLP::vectorizeTree(
                              ShuffledInserts.size() - 1);
               SmallVectorImpl<int> &Mask = It->ValueMasks[Vec];
               if (Mask.empty())
-                Mask.assign(FTy->getNumElements(), UndefMaskElem);
+                Mask.assign(FTy->getNumElements(), PoisonMaskElem);
               // Find the insertvector, vectorized in tree, if any.
               Value *Base = VU;
               while (auto *IEBase = dyn_cast<InsertElementInst>(Base)) {
@@ -10718,7 +10808,7 @@ Value *BoUpSLP::vectorizeTree(
                   do {
                     IEBase = cast<InsertElementInst>(Base);
                     int IEIdx = *getInsertIndex(IEBase);
-                    assert(Mask[Idx] == UndefMaskElem &&
+                    assert(Mask[Idx] == PoisonMaskElem &&
                            "InsertElementInstruction used already.");
                     Mask[IEIdx] = IEIdx;
                     Base = IEBase->getOperand(0);
@@ -10736,7 +10826,7 @@ Value *BoUpSLP::vectorizeTree(
             }
             SmallVectorImpl<int> &Mask = It->ValueMasks[Vec];
             if (Mask.empty())
-              Mask.assign(FTy->getNumElements(), UndefMaskElem);
+              Mask.assign(FTy->getNumElements(), PoisonMaskElem);
             Mask[Idx] = ExternalUse.Lane;
             It->InsertElements.push_back(cast<InsertElementInst>(User));
             continue;
@@ -10778,8 +10868,8 @@ Value *BoUpSLP::vectorizeTree(
   }
 
   auto CreateShuffle = [&](Value *V1, Value *V2, ArrayRef<int> Mask) {
-    SmallVector<int> CombinedMask1(Mask.size(), UndefMaskElem);
-    SmallVector<int> CombinedMask2(Mask.size(), UndefMaskElem);
+    SmallVector<int> CombinedMask1(Mask.size(), PoisonMaskElem);
+    SmallVector<int> CombinedMask2(Mask.size(), PoisonMaskElem);
     int VF = cast<FixedVectorType>(V1->getType())->getNumElements();
     for (int I = 0, E = Mask.size(); I < E; ++I) {
       if (Mask[I] < VF)
@@ -10804,9 +10894,9 @@ Value *BoUpSLP::vectorizeTree(
         return std::make_pair(Vec, true);
       }
       if (!ForSingleMask) {
-        SmallVector<int> ResizeMask(VF, UndefMaskElem);
+        SmallVector<int> ResizeMask(VF, PoisonMaskElem);
         for (unsigned I = 0; I < VF; ++I) {
-          if (Mask[I] != UndefMaskElem)
+          if (Mask[I] != PoisonMaskElem)
             ResizeMask[Mask[I]] = Mask[I];
         }
         Vec = CreateShuffle(Vec, nullptr, ResizeMask);
@@ -11009,14 +11099,14 @@ void BoUpSLP::optimizeGatherSequence() {
     // registers.
     unsigned LastUndefsCnt = 0;
     for (int I = 0, E = NewMask.size(); I < E; ++I) {
-      if (SM1[I] == UndefMaskElem)
+      if (SM1[I] == PoisonMaskElem)
         ++LastUndefsCnt;
       else
         LastUndefsCnt = 0;
-      if (NewMask[I] != UndefMaskElem && SM1[I] != UndefMaskElem &&
+      if (NewMask[I] != PoisonMaskElem && SM1[I] != PoisonMaskElem &&
           NewMask[I] != SM1[I])
         return false;
-      if (NewMask[I] == UndefMaskElem)
+      if (NewMask[I] == PoisonMaskElem)
         NewMask[I] = SM1[I];
     }
     // Check if the last undefs actually change the final number of used vector
@@ -12642,6 +12732,7 @@ class HorizontalReduction {
     return Op;
   }
 
+public:
   static RecurKind getRdxKind(Value *V) {
     auto *I = dyn_cast<Instruction>(V);
     if (!I)
@@ -12744,6 +12835,7 @@ class HorizontalReduction {
     return isCmpSelMinMax(I) ? 1 : 0;
   }
 
+private:
   /// Total number of operands in the reduction operation.
   static unsigned getNumberOfOperands(Instruction *I) {
     return isCmpSelMinMax(I) ? 3 : 2;
@@ -12792,17 +12884,6 @@ class HorizontalReduction {
     }
   }
 
-  static Value *getLHS(RecurKind Kind, Instruction *I) {
-    if (Kind == RecurKind::None)
-      return nullptr;
-    return I->getOperand(getFirstOperandIndex(I));
-  }
-  static Value *getRHS(RecurKind Kind, Instruction *I) {
-    if (Kind == RecurKind::None)
-      return nullptr;
-    return I->getOperand(getFirstOperandIndex(I) + 1);
-  }
-
   static bool isGoodForReduction(ArrayRef<Value *> Data) {
     int Sz = Data.size();
     auto *I = dyn_cast<Instruction>(Data.front());
@@ -12814,65 +12895,39 @@ class HorizontalReduction {
   HorizontalReduction() = default;
 
   /// Try to find a reduction tree.
-  bool matchAssociativeReduction(PHINode *Phi, Instruction *Inst,
+  bool matchAssociativeReduction(BoUpSLP &R, Instruction *Root,
                                  ScalarEvolution &SE, const DataLayout &DL,
                                  const TargetLibraryInfo &TLI) {
-    assert((!Phi || is_contained(Phi->operands(), Inst)) &&
-           "Phi needs to use the binary operator");
-    assert((isa<BinaryOperator>(Inst) || isa<SelectInst>(Inst) ||
-            isa<IntrinsicInst>(Inst)) &&
-           "Expected binop, select, or intrinsic for reduction matching");
-    RdxKind = getRdxKind(Inst);
-
-    // We could have a initial reductions that is not an add.
-    //  r *= v1 + v2 + v3 + v4
-    // In such a case start looking for a tree rooted in the first '+'.
-    if (Phi) {
-      if (getLHS(RdxKind, Inst) == Phi) {
-        Phi = nullptr;
-        Inst = dyn_cast<Instruction>(getRHS(RdxKind, Inst));
-        if (!Inst)
-          return false;
-        RdxKind = getRdxKind(Inst);
-      } else if (getRHS(RdxKind, Inst) == Phi) {
-        Phi = nullptr;
-        Inst = dyn_cast<Instruction>(getLHS(RdxKind, Inst));
-        if (!Inst)
-          return false;
-        RdxKind = getRdxKind(Inst);
-      }
-    }
-
-    if (!isVectorizable(RdxKind, Inst))
+    RdxKind = HorizontalReduction::getRdxKind(Root);
+    if (!isVectorizable(RdxKind, Root))
       return false;
 
     // Analyze "regular" integer/FP types for reductions - no target-specific
     // types or pointers.
-    Type *Ty = Inst->getType();
+    Type *Ty = Root->getType();
     if (!isValidElementType(Ty) || Ty->isPointerTy())
       return false;
 
     // Though the ultimate reduction may have multiple uses, its condition must
     // have only single use.
-    if (auto *Sel = dyn_cast<SelectInst>(Inst))
+    if (auto *Sel = dyn_cast<SelectInst>(Root))
       if (!Sel->getCondition()->hasOneUse())
         return false;
 
-    ReductionRoot = Inst;
+    ReductionRoot = Root;
 
     // Iterate through all the operands of the possible reduction tree and
     // gather all the reduced values, sorting them by their value id.
-    BasicBlock *BB = Inst->getParent();
-    bool IsCmpSelMinMax = isCmpSelMinMax(Inst);
-    SmallVector<Instruction *> Worklist(1, Inst);
+    BasicBlock *BB = Root->getParent();
+    bool IsCmpSelMinMax = isCmpSelMinMax(Root);
+    SmallVector<Instruction *> Worklist(1, Root);
     // Checks if the operands of the \p TreeN instruction are also reduction
     // operations or should be treated as reduced values or an extra argument,
     // which is not part of the reduction.
-    auto &&CheckOperands = [this, IsCmpSelMinMax,
-                            BB](Instruction *TreeN,
-                                SmallVectorImpl<Value *> &ExtraArgs,
-                                SmallVectorImpl<Value *> &PossibleReducedVals,
-                                SmallVectorImpl<Instruction *> &ReductionOps) {
+    auto CheckOperands = [&](Instruction *TreeN,
+                             SmallVectorImpl<Value *> &ExtraArgs,
+                             SmallVectorImpl<Value *> &PossibleReducedVals,
+                             SmallVectorImpl<Instruction *> &ReductionOps) {
       for (int I = getFirstOperandIndex(TreeN),
                End = getNumberOfOperands(TreeN);
            I < End; ++I) {
@@ -12887,10 +12942,14 @@ class HorizontalReduction {
         }
         // If the edge is not an instruction, or it is different from the main
         // reduction opcode or has too many uses - possible reduced value.
+        // Also, do not try to reduce const values, if the operation is not
+        // foldable.
         if (!EdgeInst || getRdxKind(EdgeInst) != RdxKind ||
             IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
             !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
-            !isVectorizable(RdxKind, EdgeInst)) {
+            !isVectorizable(RdxKind, EdgeInst) ||
+            (R.isAnalyzedReductionRoot(EdgeInst) &&
+             all_of(EdgeInst->operands(), Constant::classof))) {
           PossibleReducedVals.push_back(EdgeVal);
           continue;
         }
@@ -12904,10 +12963,43 @@ class HorizontalReduction {
     // instructions (grouping them by the predicate).
     MapVector<size_t, MapVector<size_t, MapVector<Value *, unsigned>>>
         PossibleReducedVals;
-    initReductionOps(Inst);
+    initReductionOps(Root);
     DenseMap<Value *, SmallVector<LoadInst *>> LoadsMap;
     SmallSet<size_t, 2> LoadKeyUsed;
     SmallPtrSet<Value *, 4> DoNotReverseVals;
+
+    auto GenerateLoadsSubkey = [&](size_t Key, LoadInst *LI) {
+      Value *Ptr = getUnderlyingObject(LI->getPointerOperand());
+      if (LoadKeyUsed.contains(Key)) {
+        auto LIt = LoadsMap.find(Ptr);
+        if (LIt != LoadsMap.end()) {
+          for (LoadInst *RLI : LIt->second) {
+            if (getPointersDiff(RLI->getType(), RLI->getPointerOperand(),
+                                LI->getType(), LI->getPointerOperand(), DL, SE,
+                                /*StrictCheck=*/true))
+              return hash_value(RLI->getPointerOperand());
+          }
+          for (LoadInst *RLI : LIt->second) {
+            if (arePointersCompatible(RLI->getPointerOperand(),
+                                      LI->getPointerOperand(), TLI)) {
+              hash_code SubKey = hash_value(RLI->getPointerOperand());
+              DoNotReverseVals.insert(RLI);
+              return SubKey;
+            }
+          }
+          if (LIt->second.size() > 2) {
+            hash_code SubKey =
+                hash_value(LIt->second.back()->getPointerOperand());
+            DoNotReverseVals.insert(LIt->second.back());
+            return SubKey;
+          }
+        }
+      }
+      LoadKeyUsed.insert(Key);
+      LoadsMap.try_emplace(Ptr).first->second.push_back(LI);
+      return hash_value(LI->getPointerOperand());
+    };
+
     while (!Worklist.empty()) {
       Instruction *TreeN = Worklist.pop_back_val();
       SmallVector<Value *> Args;
@@ -12927,41 +13019,8 @@ class HorizontalReduction {
         // results.
         for (Value *V : PossibleRedVals) {
           size_t Key, Idx;
-          std::tie(Key, Idx) = generateKeySubkey(
-              V, &TLI,
-              [&](size_t Key, LoadInst *LI) {
-                Value *Ptr = getUnderlyingObject(LI->getPointerOperand());
-                if (LoadKeyUsed.contains(Key)) {
-                  auto LIt = LoadsMap.find(Ptr);
-                  if (LIt != LoadsMap.end()) {
-                    for (LoadInst *RLI: LIt->second) {
-                      if (getPointersDiff(
-                              RLI->getType(), RLI->getPointerOperand(),
-                              LI->getType(), LI->getPointerOperand(), DL, SE,
-                              /*StrictCheck=*/true))
-                        return hash_value(RLI->getPointerOperand());
-                    }
-                    for (LoadInst *RLI : LIt->second) {
-                      if (arePointersCompatible(RLI->getPointerOperand(),
-                                                LI->getPointerOperand(), TLI)) {
-                        hash_code SubKey = hash_value(RLI->getPointerOperand());
-                        DoNotReverseVals.insert(RLI);
-                        return SubKey;
-                      }
-                    }
-                    if (LIt->second.size() > 2) {
-                      hash_code SubKey =
-                          hash_value(LIt->second.back()->getPointerOperand());
-                      DoNotReverseVals.insert(LIt->second.back());
-                      return SubKey;
-                    }
-                  }
-                }
-                LoadKeyUsed.insert(Key);
-                LoadsMap.try_emplace(Ptr).first->second.push_back(LI);
-                return hash_value(LI->getPointerOperand());
-              },
-              /*AllowAlternate=*/false);
+          std::tie(Key, Idx) = generateKeySubkey(V, &TLI, GenerateLoadsSubkey,
+                                                 /*AllowAlternate=*/false);
           ++PossibleReducedVals[Key][Idx]
                 .insert(std::make_pair(V, 0))
                 .first->second;
@@ -12970,40 +13029,8 @@ class HorizontalReduction {
                         PossibleReductionOps.rend());
       } else {
         size_t Key, Idx;
-        std::tie(Key, Idx) = generateKeySubkey(
-            TreeN, &TLI,
-            [&](size_t Key, LoadInst *LI) {
-              Value *Ptr = getUnderlyingObject(LI->getPointerOperand());
-              if (LoadKeyUsed.contains(Key)) {
-                auto LIt = LoadsMap.find(Ptr);
-                if (LIt != LoadsMap.end()) {
-                  for (LoadInst *RLI: LIt->second) {
-                    if (getPointersDiff(RLI->getType(),
-                                        RLI->getPointerOperand(), LI->getType(),
-                                        LI->getPointerOperand(), DL, SE,
-                                        /*StrictCheck=*/true))
-                      return hash_value(RLI->getPointerOperand());
-                  }
-                  for (LoadInst *RLI : LIt->second) {
-                    if (arePointersCompatible(RLI->getPointerOperand(),
-                                              LI->getPointerOperand(), TLI)) {
-                      hash_code SubKey = hash_value(RLI->getPointerOperand());
-                      DoNotReverseVals.insert(RLI);
-                      return SubKey;
-                    }
-                  }
-                  if (LIt->second.size() > 2) {
-                    hash_code SubKey = hash_value(LIt->second.back()->getPointerOperand());
-                    DoNotReverseVals.insert(LIt->second.back());
-                    return SubKey;
-                  }
-                }
-              }
-              LoadKeyUsed.insert(Key);
-              LoadsMap.try_emplace(Ptr).first->second.push_back(LI);
-              return hash_value(LI->getPointerOperand());
-            },
-            /*AllowAlternate=*/false);
+        std::tie(Key, Idx) = generateKeySubkey(TreeN, &TLI, GenerateLoadsSubkey,
+                                               /*AllowAlternate=*/false);
         ++PossibleReducedVals[Key][Idx]
               .insert(std::make_pair(TreeN, 0))
               .first->second;
@@ -13206,9 +13233,11 @@ class HorizontalReduction {
           allConstant(Candidates)) {
         Value *Res = Candidates.front();
         ++VectorizedVals.try_emplace(Candidates.front(), 0).first->getSecond();
-        for (Value *V : ArrayRef(Candidates).drop_front()) {
-          Res = createOp(Builder, RdxKind, Res, V, "const.rdx", ReductionOps);
-          ++VectorizedVals.try_emplace(V, 0).first->getSecond();
+        for (Value *VC : ArrayRef(Candidates).drop_front()) {
+          Res = createOp(Builder, RdxKind, Res, VC, "const.rdx", ReductionOps);
+          ++VectorizedVals.try_emplace(VC, 0).first->getSecond();
+          if (auto *ResI = dyn_cast<Instruction>(Res))
+            V.analyzedReductionRoot(ResI);
         }
         VectorizedTree = GetNewVectorizedTree(VectorizedTree, Res);
         continue;
@@ -13834,7 +13863,7 @@ class HorizontalReduction {
       // 7>, if elements 4th and 6th elements have even number of repeats.
       SmallVector<int> Mask(
           cast<FixedVectorType>(VectorizedValue->getType())->getNumElements(),
-          UndefMaskElem);
+          PoisonMaskElem);
       std::iota(Mask.begin(), Mask.end(), 0);
       bool NeedShuffle = false;
       for (unsigned I = 0, VF = VL.size(); I < VF; ++I) {
@@ -13972,15 +14001,15 @@ static bool findBuildAggregate(Instruction *LastInsertInst,
   return false;
 }
 
-/// Try and get a reduction value from a phi node.
+/// Try and get a reduction instruction from a phi node.
 ///
 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions
 /// if they come from either \p ParentBB or a containing loop latch.
 ///
 /// \returns A candidate reduction value if possible, or \code nullptr \endcode
 /// if not possible.
-static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
-                                BasicBlock *ParentBB, LoopInfo *LI) {
+static Instruction *getReductionInstr(const DominatorTree *DT, PHINode *P,
+                                      BasicBlock *ParentBB, LoopInfo *LI) {
   // There are situations where the reduction value is not dominated by the
   // reduction phi. Vectorizing such cases has been reported to cause
   // miscompiles. See PR25787.
@@ -13989,13 +14018,13 @@ static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
            DT->dominates(P->getParent(), cast<Instruction>(R)->getParent());
   };
 
-  Value *Rdx = nullptr;
+  Instruction *Rdx = nullptr;
 
   // Return the incoming value if it comes from the same BB as the phi node.
   if (P->getIncomingBlock(0) == ParentBB) {
-    Rdx = P->getIncomingValue(0);
+    Rdx = dyn_cast<Instruction>(P->getIncomingValue(0));
   } else if (P->getIncomingBlock(1) == ParentBB) {
-    Rdx = P->getIncomingValue(1);
+    Rdx = dyn_cast<Instruction>(P->getIncomingValue(1));
   }
 
   if (Rdx && DominatedReduxValue(Rdx))
@@ -14012,9 +14041,9 @@ static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
   // There is a loop latch, return the incoming value if it comes from
   // that. This reduction pattern occasionally turns up.
   if (P->getIncomingBlock(0) == BBLatch) {
-    Rdx = P->getIncomingValue(0);
+    Rdx = dyn_cast<Instruction>(P->getIncomingValue(0));
   } else if (P->getIncomingBlock(1) == BBLatch) {
-    Rdx = P->getIncomingValue(1);
+    Rdx = dyn_cast<Instruction>(P->getIncomingValue(1));
   }
 
   if (Rdx && DominatedReduxValue(Rdx))
@@ -14041,21 +14070,63 @@ static bool matchRdxBop(Instruction *I, Value *&V0, Value *&V1) {
   return false;
 }
 
+/// We could have an initial reduction that is not an add.
+///  r *= v1 + v2 + v3 + v4
+/// In such a case start looking for a tree rooted in the first '+'.
+/// \Returns the new root if found, which may be nullptr if not an instruction.
+static Instruction *tryGetSecondaryReductionRoot(PHINode *Phi,
+                                                 Instruction *Root) {
+  assert((isa<BinaryOperator>(Root) || isa<SelectInst>(Root) ||
+          isa<IntrinsicInst>(Root)) &&
+         "Expected binop, select, or intrinsic for reduction matching");
+  Value *LHS =
+      Root->getOperand(HorizontalReduction::getFirstOperandIndex(Root));
+  Value *RHS =
+      Root->getOperand(HorizontalReduction::getFirstOperandIndex(Root) + 1);
+  if (LHS == Phi)
+    return dyn_cast<Instruction>(RHS);
+  if (RHS == Phi)
+    return dyn_cast<Instruction>(LHS);
+  return nullptr;
+}
+
+/// \p Returns the first operand of \p I that does not match \p Phi. If
+/// operand is not an instruction it returns nullptr.
+static Instruction *getNonPhiOperand(Instruction *I, PHINode *Phi) {
+  Value *Op0 = nullptr;
+  Value *Op1 = nullptr;
+  if (!matchRdxBop(I, Op0, Op1))
+    return nullptr;
+  return dyn_cast<Instruction>(Op0 == Phi ? Op1 : Op0);
+}
+
+/// \Returns true if \p I is a candidate instruction for reduction vectorization.
+static bool isReductionCandidate(Instruction *I) {
+  bool IsSelect = match(I, m_Select(m_Value(), m_Value(), m_Value()));
+  Value *B0 = nullptr, *B1 = nullptr;
+  bool IsBinop = matchRdxBop(I, B0, B1);
+  return IsBinop || IsSelect;
+}
+
 bool SLPVectorizerPass::vectorizeHorReduction(
-    PHINode *P, Value *V, BasicBlock *BB, BoUpSLP &R, TargetTransformInfo *TTI,
+    PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R, TargetTransformInfo *TTI,
     SmallVectorImpl<WeakTrackingVH> &PostponedInsts) {
   if (!ShouldVectorizeHor)
     return false;
+  bool TryOperandsAsNewSeeds = P && isa<BinaryOperator>(Root);
 
-  auto *Root = dyn_cast_or_null<Instruction>(V);
-  if (!Root)
+  if (Root->getParent() != BB || isa<PHINode>(Root))
     return false;
 
-  if (!isa<BinaryOperator>(Root))
-    P = nullptr;
+  // If we can find a secondary reduction root, use that instead.
+  auto SelectRoot = [&]() {
+    if (TryOperandsAsNewSeeds && isReductionCandidate(Root) &&
+        HorizontalReduction::getRdxKind(Root) != RecurKind::None)
+      if (Instruction *NewRoot = tryGetSecondaryReductionRoot(P, Root))
+        return NewRoot;
+    return Root;
+  };
 
-  if (Root->getParent() != BB || isa<PHINode>(Root))
-    return false;
   // Start analysis starting from Root instruction. If horizontal reduction is
   // found, try to vectorize it. If it is not a horizontal reduction or
   // vectorization is not possible or not effective, and currently analyzed
@@ -14068,22 +14139,32 @@ bool SLPVectorizerPass::vectorizeHorReduction(
   // If a horizintal reduction was not matched or vectorized we collect
   // instructions for possible later attempts for vectorization.
   std::queue<std::pair<Instruction *, unsigned>> Stack;
-  Stack.emplace(Root, 0);
+  Stack.emplace(SelectRoot(), 0);
   SmallPtrSet<Value *, 8> VisitedInstrs;
   bool Res = false;
-  auto &&TryToReduce = [this, TTI, &P, &R](Instruction *Inst, Value *&B0,
-                                           Value *&B1) -> Value * {
+  auto &&TryToReduce = [this, TTI, &R](Instruction *Inst) -> Value * {
     if (R.isAnalyzedReductionRoot(Inst))
       return nullptr;
-    bool IsBinop = matchRdxBop(Inst, B0, B1);
-    bool IsSelect = match(Inst, m_Select(m_Value(), m_Value(), m_Value()));
-    if (IsBinop || IsSelect) {
-      HorizontalReduction HorRdx;
-      if (HorRdx.matchAssociativeReduction(P, Inst, *SE, *DL, *TLI))
-        return HorRdx.tryToReduce(R, TTI, *TLI);
+    if (!isReductionCandidate(Inst))
+      return nullptr;
+    HorizontalReduction HorRdx;
+    if (!HorRdx.matchAssociativeReduction(R, Inst, *SE, *DL, *TLI))
+      return nullptr;
+    return HorRdx.tryToReduce(R, TTI, *TLI);
+  };
+  auto TryAppendToPostponedInsts = [&](Instruction *FutureSeed) {
+    if (TryOperandsAsNewSeeds && FutureSeed == Root) {
+      FutureSeed = getNonPhiOperand(Root, P);
+      if (!FutureSeed)
+        return false;
     }
-    return nullptr;
+    // Do not collect CmpInst or InsertElementInst/InsertValueInst as their
+    // analysis is done separately.
+    if (!isa<CmpInst, InsertElementInst, InsertValueInst>(FutureSeed))
+      PostponedInsts.push_back(FutureSeed);
+    return true;
   };
+
   while (!Stack.empty()) {
     Instruction *Inst;
     unsigned Level;
@@ -14094,37 +14175,19 @@ bool SLPVectorizerPass::vectorizeHorReduction(
     // iteration while stack was populated before that happened.
     if (R.isDeleted(Inst))
       continue;
-    Value *B0 = nullptr, *B1 = nullptr;
-    if (Value *V = TryToReduce(Inst, B0, B1)) {
+    if (Value *VectorizedV = TryToReduce(Inst)) {
       Res = true;
-      // Set P to nullptr to avoid re-analysis of phi node in
-      // matchAssociativeReduction function unless this is the root node.
-      P = nullptr;
-      if (auto *I = dyn_cast<Instruction>(V)) {
+      if (auto *I = dyn_cast<Instruction>(VectorizedV)) {
         // Try to find another reduction.
         Stack.emplace(I, Level);
         continue;
       }
     } else {
-      bool IsBinop = B0 && B1;
-      if (P && IsBinop) {
-        Inst = dyn_cast<Instruction>(B0);
-        if (Inst == P)
-          Inst = dyn_cast<Instruction>(B1);
-        if (!Inst) {
-          // Set P to nullptr to avoid re-analysis of phi node in
-          // matchAssociativeReduction function unless this is the root node.
-          P = nullptr;
-          continue;
-        }
+      // We could not vectorize `Inst` so try to use it as a future seed.
+      if (!TryAppendToPostponedInsts(Inst)) {
+        assert(Stack.empty() && "Expected empty stack");
+        break;
       }
-      // Set P to nullptr to avoid re-analysis of phi node in
-      // matchAssociativeReduction function unless this is the root node.
-      P = nullptr;
-      // Do not collect CmpInst or InsertElementInst/InsertValueInst as their
-      // analysis is done separately.
-      if (!isa<CmpInst, InsertElementInst, InsertValueInst>(Inst))
-        PostponedInsts.push_back(Inst);
     }
 
     // Try to vectorize operands.
@@ -14143,11 +14206,11 @@ bool SLPVectorizerPass::vectorizeHorReduction(
   return Res;
 }
 
-bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
+bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Instruction *Root,
                                                  BasicBlock *BB, BoUpSLP &R,
                                                  TargetTransformInfo *TTI) {
   SmallVector<WeakTrackingVH> PostponedInsts;
-  bool Res = vectorizeHorReduction(P, V, BB, R, TTI, PostponedInsts);
+  bool Res = vectorizeHorReduction(P, Root, BB, R, TTI, PostponedInsts);
   Res |= tryToVectorize(PostponedInsts, R);
   return Res;
 }
@@ -14360,7 +14423,8 @@ bool SLPVectorizerPass::vectorizeSimpleInstructions(InstSetVector &Instructions,
       if (R.isDeleted(I))
         continue;
       for (Value *Op : I->operands())
-        OpsChanged |= vectorizeRootInstruction(nullptr, Op, BB, R, TTI);
+        if (auto *RootOp = dyn_cast<Instruction>(Op))
+          OpsChanged |= vectorizeRootInstruction(nullptr, RootOp, BB, R, TTI);
     }
     // Try to vectorize operands as vector bundles.
     for (Instruction *I : PostponedCmps) {
@@ -14587,8 +14651,8 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
       // Check that the PHI is a reduction PHI.
       if (P->getNumIncomingValues() == 2) {
         // Try to match and vectorize a horizontal reduction.
-        if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
-                                     TTI)) {
+        Instruction *Root = getReductionInstr(DT, P, BB, LI);
+        if (Root && vectorizeRootInstruction(P, Root, BB, R, TTI)) {
           Changed = true;
           it = BB->begin();
           e = BB->end();
@@ -14610,7 +14674,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
         // vectorization.
         if (auto *PI = dyn_cast<Instruction>(P->getIncomingValue(I));
             PI && !PostProcessInstructions.contains(PI))
-          Changed |= vectorizeRootInstruction(nullptr, P->getIncomingValue(I),
+          Changed |= vectorizeRootInstruction(nullptr, PI,
                                               P->getIncomingBlock(I), R, TTI);
       }
       continue;
@@ -14643,7 +14707,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
           if (auto *VI = dyn_cast<Instruction>(V);
               VI && !PostProcessInstructions.contains(VI))
             // Try to match and vectorize a horizontal reduction.
-            OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
+            OpsChanged |= vectorizeRootInstruction(nullptr, VI, BB, R, TTI);
         }
       }
       // Start vectorization of post-process list of instructions from the
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f185c9aea6bc4..23646901b396d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -163,8 +163,9 @@ VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
 }
 
 void VPBlockBase::setPlan(VPlan *ParentPlan) {
-  assert(ParentPlan->getEntry() == this &&
-         "Can only set plan on its entry block.");
+  assert(
+      (ParentPlan->getEntry() == this || ParentPlan->getPreheader() == this) &&
+      "Can only set plan on its entry or preheader block.");
   Plan = ParentPlan;
 }
 
@@ -212,7 +213,7 @@ VPBasicBlock::iterator VPBasicBlock::getFirstNonPhi() {
 }
 
 Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
-  if (!Def->hasDefiningRecipe())
+  if (Def->isLiveIn())
     return Def->getLiveInIRValue();
 
   if (hasScalarValue(Def, Instance)) {
@@ -590,15 +591,25 @@ VPlan::~VPlan() {
       Block->dropAllReferences(&DummyValue);
 
     VPBlockBase::deleteCFG(Entry);
+
+    Preheader->dropAllReferences(&DummyValue);
+    delete Preheader;
   }
   for (VPValue *VPV : VPLiveInsToFree)
     delete VPV;
-  if (TripCount)
-    delete TripCount;
   if (BackedgeTakenCount)
     delete BackedgeTakenCount;
 }
 
+VPlanPtr VPlan::createInitialVPlan(const SCEV *TripCount, ScalarEvolution &SE) {
+  VPBasicBlock *Preheader = new VPBasicBlock("ph");
+  VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph");
+  auto Plan = std::make_unique<VPlan>(Preheader, VecPreheader);
+  Plan->TripCount =
+      vputils::getOrCreateVPValueForSCEVExpr(*Plan, TripCount, SE);
+  return Plan;
+}
+
 VPActiveLaneMaskPHIRecipe *VPlan::getActiveLaneMaskPhi() {
   VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
   for (VPRecipeBase &R : Header->phis()) {
@@ -612,13 +623,6 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
                              Value *CanonicalIVStartValue,
                              VPTransformState &State,
                              bool IsEpilogueVectorization) {
-
-  // Check if the trip count is needed, and if so build it.
-  if (TripCount && TripCount->getNumUsers()) {
-    for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
-      State.set(TripCount, TripCountV, Part);
-  }
-
   // Check if the backedge taken count is needed, and if so build it.
   if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
     IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
@@ -747,30 +751,29 @@ void VPlan::print(raw_ostream &O) const {
 
   O << "VPlan '" << getName() << "' {";
 
-  bool AnyLiveIn = false;
   if (VectorTripCount.getNumUsers() > 0) {
     O << "\nLive-in ";
     VectorTripCount.printAsOperand(O, SlotTracker);
     O << " = vector-trip-count";
-    AnyLiveIn = true;
-  }
-
-  if (TripCount && TripCount->getNumUsers() > 0) {
-    O << "\nLive-in ";
-    TripCount->printAsOperand(O, SlotTracker);
-    O << " = original trip-count";
-    AnyLiveIn = true;
   }
 
   if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
     O << "\nLive-in ";
     BackedgeTakenCount->printAsOperand(O, SlotTracker);
     O << " = backedge-taken count";
-    AnyLiveIn = true;
   }
 
-  if (AnyLiveIn)
+  O << "\n";
+  if (TripCount->isLiveIn())
+    O << "Live-in ";
+  TripCount->printAsOperand(O, SlotTracker);
+  O << " = original trip-count";
+  O << "\n";
+
+  if (!getPreheader()->empty()) {
     O << "\n";
+    getPreheader()->print(O, "", SlotTracker);
+  }
 
   for (const VPBlockBase *Block : vp_depth_first_shallow(getEntry())) {
     O << '\n';
@@ -897,6 +900,8 @@ void VPlanPrinter::dump() {
   OS << "edge [fontname=Courier, fontsize=30]\n";
   OS << "compound=true\n";
 
+  dumpBlock(Plan.getPreheader());
+
   for (const VPBlockBase *Block : vp_depth_first_shallow(Plan.getEntry()))
     dumpBlock(Block);
 
@@ -1109,16 +1114,19 @@ void VPSlotTracker::assignSlots(const VPlan &Plan) {
   assignSlot(&Plan.VectorTripCount);
   if (Plan.BackedgeTakenCount)
     assignSlot(Plan.BackedgeTakenCount);
-  if (Plan.TripCount)
-    assignSlot(Plan.TripCount);
+  assignSlots(Plan.getPreheader());
 
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<const VPBlockBase *>>
       RPOT(VPBlockDeepTraversalWrapper<const VPBlockBase *>(Plan.getEntry()));
   for (const VPBasicBlock *VPBB :
        VPBlockUtils::blocksOnly<const VPBasicBlock>(RPOT))
-    for (const VPRecipeBase &Recipe : *VPBB)
-      for (VPValue *Def : Recipe.definedValues())
-        assignSlot(Def);
+    assignSlots(VPBB);
+}
+
+void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
+  for (const VPRecipeBase &Recipe : *VPBB)
+    for (VPValue *Def : Recipe.definedValues())
+      assignSlot(Def);
 }
 
 bool vputils::onlyFirstLaneUsed(VPValue *Def) {
@@ -1128,13 +1136,17 @@ bool vputils::onlyFirstLaneUsed(VPValue *Def) {
 
 VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
                                                 ScalarEvolution &SE) {
+  if (auto *Expanded = Plan.getSCEVExpansion(Expr))
+    return Expanded;
+  VPValue *Expanded = nullptr;
   if (auto *E = dyn_cast<SCEVConstant>(Expr))
-    return Plan.getVPValueOrAddLiveIn(E->getValue());
-  if (auto *E = dyn_cast<SCEVUnknown>(Expr))
-    return Plan.getVPValueOrAddLiveIn(E->getValue());
-
-  VPBasicBlock *Preheader = Plan.getEntry()->getEntryBasicBlock();
-  VPExpandSCEVRecipe *Step = new VPExpandSCEVRecipe(Expr, SE);
-  Preheader->appendRecipe(Step);
-  return Step;
+    Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
+  else if (auto *E = dyn_cast<SCEVUnknown>(Expr))
+    Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
+  else {
+    Expanded = new VPExpandSCEVRecipe(Expr, SE);
+    Plan.getPreheader()->appendRecipe(Expanded->getDefiningRecipe());
+  }
+  Plan.addSCEVExpansion(Expr, Expanded);
+  return Expanded;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 3b71a5b551ae2..0360066daa238 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -25,7 +25,6 @@
 
 #include "VPlanValue.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -38,7 +37,6 @@
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/FMF.h"
-#include "llvm/Transforms/Utils/LoopVersioning.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
@@ -62,6 +60,7 @@ class VPlan;
 class VPReplicateRecipe;
 class VPlanSlp;
 class Value;
+class LoopVersioning;
 
 namespace Intrinsic {
 typedef unsigned ID;
@@ -410,7 +409,7 @@ struct VPTransformState {
   ///
   /// This is currently only used to add no-alias metadata based on the
   /// memchecks.  The actually versioning is performed manually.
-  std::unique_ptr<LoopVersioning> LVer;
+  LoopVersioning *LVer = nullptr;
 };
 
 /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
@@ -2210,13 +2209,19 @@ class VPRegionBlock : public VPBlockBase {
 /// to produce efficient output IR, including which branches, basic-blocks and
 /// output IR instructions to generate, and their cost. VPlan holds a
 /// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
-/// VPBlock.
+/// VPBasicBlock.
 class VPlan {
   friend class VPlanPrinter;
   friend class VPSlotTracker;
 
-  /// Hold the single entry to the Hierarchical CFG of the VPlan.
-  VPBlockBase *Entry;
+  /// Hold the single entry to the Hierarchical CFG of the VPlan, i.e. the
+  /// preheader of the vector loop.
+  VPBasicBlock *Entry;
+
+  /// VPBasicBlock corresponding to the original preheader. Used to place
+  /// VPExpandSCEV recipes for expressions used during skeleton creation and the
+  /// rest of VPlan execution.
+  VPBasicBlock *Preheader;
 
   /// Holds the VFs applicable to this VPlan.
   SmallSetVector<ElementCount, 2> VFs;
@@ -2254,14 +2259,41 @@ class VPlan {
   /// Values used outside the plan.
   MapVector<PHINode *, VPLiveOut *> LiveOuts;
 
+  /// Mapping from SCEVs to the VPValues representing their expansions.
+  /// NOTE: This mapping is temporary and will be removed once all users have
+  /// been modeled in VPlan directly.
+  DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
+
 public:
-  VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {
-    if (Entry)
-      Entry->setPlan(this);
+  /// Construct a VPlan with original preheader \p Preheader, trip count \p TC
+  /// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to
+  /// be disconnected, as the bypass blocks between them are not yet modeled in
+  /// VPlan.
+  VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry)
+      : VPlan(Preheader, Entry) {
+    TripCount = TC;
+  }
+
+  /// Construct a VPlan with original preheader \p Preheader and \p Entry to
+  /// the plan. At the moment, \p Preheader and \p Entry need to be
+  /// disconnected, as the bypass blocks between them are not yet modeled in
+  /// VPlan.
+  VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry)
+      : Entry(Entry), Preheader(Preheader) {
+    Entry->setPlan(this);
+    Preheader->setPlan(this);
+    assert(Preheader->getNumSuccessors() == 0 &&
+           Preheader->getNumPredecessors() == 0 &&
+           "preheader must be disconnected");
   }
 
   ~VPlan();
 
+  /// Create an initial VPlan with preheader and entry blocks. Creates a
+  /// VPExpandSCEVRecipe for \p TripCount and uses it as plan's trip count.
+  static VPlanPtr createInitialVPlan(const SCEV *TripCount,
+                                     ScalarEvolution &PSE);
+
   /// Prepare the plan for execution, setting up the required live-in values.
   void prepareToExecute(Value *TripCount, Value *VectorTripCount,
                         Value *CanonicalIVStartValue, VPTransformState &State,
@@ -2270,19 +2302,12 @@ class VPlan {
   /// Generate the IR code for this VPlan.
   void execute(VPTransformState *State);
 
-  VPBlockBase *getEntry() { return Entry; }
-  const VPBlockBase *getEntry() const { return Entry; }
-
-  VPBlockBase *setEntry(VPBlockBase *Block) {
-    Entry = Block;
-    Block->setPlan(this);
-    return Entry;
-  }
+  VPBasicBlock *getEntry() { return Entry; }
+  const VPBasicBlock *getEntry() const { return Entry; }
 
   /// The trip count of the original loop.
-  VPValue *getOrCreateTripCount() {
-    if (!TripCount)
-      TripCount = new VPValue();
+  VPValue *getTripCount() const {
+    assert(TripCount && "trip count needs to be set before accessing it");
     return TripCount;
   }
 
@@ -2326,7 +2351,7 @@ class VPlan {
   void setName(const Twine &newName) { Name = newName.str(); }
 
   void addVPValue(Value *V, VPValue *VPV) {
-    assert((Value2VPValueEnabled || !VPV->getDefiningRecipe()) &&
+    assert((Value2VPValueEnabled || VPV->isLiveIn()) &&
            "Value2VPValue mapping may be out of date!");
     assert(V && "Trying to add a null Value to VPlan");
     assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
@@ -2339,7 +2364,7 @@ class VPlan {
     assert(V && "Trying to get the VPValue of a null Value");
     assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
     assert((Value2VPValueEnabled || OverrideAllowed ||
-            !Value2VPValue[V]->getDefiningRecipe()) &&
+            Value2VPValue[V]->isLiveIn()) &&
            "Value2VPValue mapping may be out of date!");
     return Value2VPValue[V];
   }
@@ -2417,6 +2442,23 @@ class VPlan {
     return LiveOuts;
   }
 
+  VPValue *getSCEVExpansion(const SCEV *S) const {
+    auto I = SCEVToExpansion.find(S);
+    if (I == SCEVToExpansion.end())
+      return nullptr;
+    return I->second;
+  }
+
+  void addSCEVExpansion(const SCEV *S, VPValue *V) {
+    assert(SCEVToExpansion.find(S) == SCEVToExpansion.end() &&
+           "SCEV already expanded");
+    SCEVToExpansion[S] = V;
+  }
+
+  /// \return The block corresponding to the original preheader.
+  VPBasicBlock *getPreheader() { return Preheader; }
+  const VPBasicBlock *getPreheader() const { return Preheader; }
+
 private:
   /// Add to the given dominator tree the header block and every new basic block
   /// that was created between it and the latch block, inclusive.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanCFG.h b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
index f790f7e73e11b..89e2e7514dac2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanCFG.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
@@ -13,6 +13,7 @@
 #define LLVM_TRANSFORMS_VECTORIZE_VPLANCFG_H
 
 #include "VPlan.h"
+#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/SmallVector.h"
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 10fefae1cacf1..f6e3a2a16db85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -73,9 +73,8 @@ class PlainCFGBuilder {
   PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
       : TheLoop(Lp), LI(LI), Plan(P) {}
 
-  /// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected
-  /// to a new VPRegionBlock (TopRegion) enclosing the plain CFG.
-  VPBasicBlock *buildPlainCFG();
+  /// Build plain CFG for TheLoop  and connects it to Plan's entry.
+  void buildPlainCFG();
 };
 } // anonymous namespace
 
@@ -254,7 +253,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
 }
 
 // Main interface to build the plain CFG.
-VPBasicBlock *PlainCFGBuilder::buildPlainCFG() {
+void PlainCFGBuilder::buildPlainCFG() {
   // 1. Scan the body of the loop in a topological order to visit each basic
   // block after having visited its predecessor basic blocks. Create a VPBB for
   // each BB and link it to its successor and predecessor VPBBs. Note that
@@ -267,7 +266,8 @@ VPBasicBlock *PlainCFGBuilder::buildPlainCFG() {
   BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
   assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
          "Unexpected loop preheader");
-  VPBasicBlock *ThePreheaderVPBB = getOrCreateVPBB(ThePreheaderBB);
+  VPBasicBlock *ThePreheaderVPBB = Plan.getEntry();
+  BB2VPBB[ThePreheaderBB] = ThePreheaderVPBB;
   ThePreheaderVPBB->setName("vector.ph");
   for (auto &I : *ThePreheaderBB) {
     if (I.getType()->isVoidTy())
@@ -371,20 +371,17 @@ VPBasicBlock *PlainCFGBuilder::buildPlainCFG() {
   // have a VPlan couterpart. Fix VPlan phi nodes by adding their corresponding
   // VPlan operands.
   fixPhiNodes();
-
-  return ThePreheaderVPBB;
 }
 
-VPBasicBlock *VPlanHCFGBuilder::buildPlainCFG() {
+void VPlanHCFGBuilder::buildPlainCFG() {
   PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
-  return PCFGBuilder.buildPlainCFG();
+  PCFGBuilder.buildPlainCFG();
 }
 
 // Public interface to build a H-CFG.
 void VPlanHCFGBuilder::buildHierarchicalCFG() {
-  // Build Top Region enclosing the plain CFG and set it as VPlan entry.
-  VPBasicBlock *EntryVPBB = buildPlainCFG();
-  Plan.setEntry(EntryVPBB);
+  // Build Top Region enclosing the plain CFG.
+  buildPlainCFG();
   LLVM_DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan);
 
   VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
index 2d52990af2688..299ae36155cba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
@@ -57,9 +57,8 @@ class VPlanHCFGBuilder {
   // are introduced.
   VPDominatorTree VPDomTree;
 
-  /// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected
-  /// to a new VPRegionBlock (TopRegion) enclosing the plain CFG.
-  VPBasicBlock *buildPlainCFG();
+  /// Build plain CFG for TheLoop and connects it to Plan's entry.
+  void buildPlainCFG();
 
 public:
   VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 41335fb8de3b8..4f3e527efcf90 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -54,13 +54,13 @@ bool VPRecipeBase::mayWriteToMemory() const {
   case VPScalarIVStepsSC:
   case VPPredInstPHISC:
     return false;
-  case VPWidenIntOrFpInductionSC:
+  case VPBlendSC:
+  case VPReductionSC:
   case VPWidenCanonicalIVSC:
+  case VPWidenGEPSC:
+  case VPWidenIntOrFpInductionSC:
   case VPWidenPHISC:
-  case VPBlendSC:
   case VPWidenSC:
-  case VPWidenGEPSC:
-  case VPReductionSC:
   case VPWidenSelectSC: {
     const Instruction *I =
         dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
@@ -87,13 +87,13 @@ bool VPRecipeBase::mayReadFromMemory() const {
   case VPScalarIVStepsSC:
   case VPPredInstPHISC:
     return false;
-  case VPWidenIntOrFpInductionSC:
+  case VPBlendSC:
+  case VPReductionSC:
   case VPWidenCanonicalIVSC:
+  case VPWidenGEPSC:
+  case VPWidenIntOrFpInductionSC:
   case VPWidenPHISC:
-  case VPBlendSC:
   case VPWidenSC:
-  case VPWidenGEPSC:
-  case VPReductionSC:
   case VPWidenSelectSC: {
     const Instruction *I =
         dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
@@ -123,17 +123,17 @@ bool VPRecipeBase::mayHaveSideEffects() const {
   case VPWidenCallSC:
     return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
         ->mayHaveSideEffects();
-  case VPWidenIntOrFpInductionSC:
+  case VPBlendSC:
   case VPFirstOrderRecurrencePHISC:
-  case VPWidenPointerInductionSC:
+  case VPReductionSC:
+  case VPScalarIVStepsSC:
   case VPWidenCanonicalIVSC:
+  case VPWidenGEPSC:
+  case VPWidenIntOrFpInductionSC:
   case VPWidenPHISC:
-  case VPBlendSC:
+  case VPWidenPointerInductionSC:
   case VPWidenSC:
-  case VPWidenGEPSC:
-  case VPReductionSC:
-  case VPWidenSelectSC:
-  case VPScalarIVStepsSC: {
+  case VPWidenSelectSC: {
     const Instruction *I =
         dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
     (void)I;
@@ -252,7 +252,7 @@ void VPInstruction::generateInstruction(VPTransformState &State,
     // Get first lane of vector induction variable.
     Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
     // Get the original loop tripcount.
-    Value *ScalarTC = State.get(getOperand(1), Part);
+    Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0));
 
     auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
     auto *PredTy = VectorType::get(Int1Ty, State.VF);
@@ -288,7 +288,7 @@ void VPInstruction::generateInstruction(VPTransformState &State,
     break;
   }
   case VPInstruction::CalculateTripCountMinusVF: {
-    Value *ScalarTC = State.get(getOperand(0), Part);
+    Value *ScalarTC = State.get(getOperand(0), {0, 0});
     Value *Step =
         createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF);
     Value *Sub = Builder.CreateSub(ScalarTC, Step);
@@ -701,9 +701,8 @@ void VPWidenRecipe::execute(VPTransformState &State) {
     State.setDebugLocFromInst(CI);
 
     /// Vectorize casts.
-    Type *DestTy = (State.VF.isScalar())
-                       ? CI->getType()
-                       : VectorType::get(CI->getType(), State.VF);
+    assert(State.VF.isVector() && "not widening");
+    Type *DestTy = VectorType::get(CI->getType(), State.VF);
 
     for (unsigned Part = 0; Part < State.UF; ++Part) {
       Value *A = State.get(getOperand(0), Part);
@@ -1152,7 +1151,7 @@ void VPExpandSCEVRecipe::execute(VPTransformState &State) {
                                  &*State.Builder.GetInsertPoint());
 
   for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
-    State.set(this, Res, Part);
+    State.set(this, Res, {Part, 0});
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ec1a1a8307e96..6f0dfccb682cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -27,8 +27,7 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
     VPlanPtr &Plan,
     function_ref<const InductionDescriptor *(PHINode *)>
         GetIntOrFpInductionDescriptor,
-    SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE,
-    const TargetLibraryInfo &TLI) {
+    ScalarEvolution &SE, const TargetLibraryInfo &TLI) {
 
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
       Plan->getEntry());
@@ -41,12 +40,6 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
 
       VPValue *VPV = Ingredient.getVPSingleValue();
       Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
-      if (DeadInstructions.count(Inst)) {
-        VPValue DummyValue;
-        VPV->replaceAllUsesWith(&DummyValue);
-        Ingredient.eraseFromParent();
-        continue;
-      }
 
       VPRecipeBase *NewRecipe = nullptr;
       if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 430628cb068d8..dbcdbe4d58ee0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -35,7 +35,6 @@ struct VPlanTransforms {
   VPInstructionsToVPRecipes(VPlanPtr &Plan,
                             function_ref<const InductionDescriptor *(PHINode *)>
                                 GetIntOrFpInductionDescriptor,
-                            SmallPtrSetImpl<Instruction *> &DeadInstructions,
                             ScalarEvolution &SE, const TargetLibraryInfo &TLI);
 
   /// Wrap predicated VPReplicateRecipes with a mask operand in an if-then
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index fee004cc92d5e..c1a30133a0f8d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -171,16 +171,19 @@ class VPValue {
   /// Returns true if this VPValue is defined by a recipe.
   bool hasDefiningRecipe() const { return getDefiningRecipe(); }
 
+  /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
+  bool isLiveIn() const { return !hasDefiningRecipe(); }
+
   /// Returns the underlying IR value, if this VPValue is defined outside the
   /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef
   /// inside a VPlan.
   Value *getLiveInIRValue() {
-    assert(!hasDefiningRecipe() &&
+    assert(isLiveIn() &&
            "VPValue is not a live-in; it is defined by a VPDef inside a VPlan");
     return getUnderlyingValue();
   }
   const Value *getLiveInIRValue() const {
-    assert(!hasDefiningRecipe() &&
+    assert(isLiveIn() &&
            "VPValue is not a live-in; it is defined by a VPDef inside a VPlan");
     return getUnderlyingValue();
   }
@@ -437,6 +440,7 @@ class VPSlotTracker {
 
   void assignSlot(const VPValue *V);
   void assignSlots(const VPlan &Plan);
+  void assignSlots(const VPBasicBlock *VPBB);
 
 public:
   VPSlotTracker(const VPlan *Plan = nullptr) {
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index f7bd0b1bec2d8..0f9ac2b8e5409 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -244,7 +244,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   // still need a shuffle to change the vector size.
   auto *Ty = cast<FixedVectorType>(I.getType());
   unsigned OutputNumElts = Ty->getNumElements();
-  SmallVector<int, 16> Mask(OutputNumElts, UndefMaskElem);
+  SmallVector<int, 16> Mask(OutputNumElts, PoisonMaskElem);
   assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
   Mask[0] = OffsetEltIndex;
   if (OffsetEltIndex)
@@ -480,7 +480,7 @@ static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
   // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
   // ShufMask = { 2, undef, undef, undef }
   auto *VecTy = cast<FixedVectorType>(Vec->getType());
-  SmallVector<int, 32> ShufMask(VecTy->getNumElements(), UndefMaskElem);
+  SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
   ShufMask[NewIndex] = OldIndex;
   return Builder.CreateShuffleVector(Vec, ShufMask, "shift");
 }
@@ -914,7 +914,7 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
   auto *CmpTy = cast<FixedVectorType>(CmpInst::makeCmpResultType(X->getType()));
   InstructionCost NewCost = TTI.getCmpSelInstrCost(
       CmpOpcode, X->getType(), CmpInst::makeCmpResultType(X->getType()), Pred);
-  SmallVector<int, 32> ShufMask(VecTy->getNumElements(), UndefMaskElem);
+  SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
   ShufMask[CheapIndex] = ExpensiveIndex;
   NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy,
                                 ShufMask);
@@ -1593,12 +1593,12 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
     V2B.push_back(GetBaseMaskValue(SVI1B, V2[I].first));
   }
   while (V1A.size() < NumElts) {
-    V1A.push_back(UndefMaskElem);
-    V1B.push_back(UndefMaskElem);
+    V1A.push_back(PoisonMaskElem);
+    V1B.push_back(PoisonMaskElem);
   }
   while (V2A.size() < NumElts) {
-    V2A.push_back(UndefMaskElem);
-    V2B.push_back(UndefMaskElem);
+    V2A.push_back(PoisonMaskElem);
+    V2B.push_back(PoisonMaskElem);
   }
 
   auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll
index d1ea923b9dbed..e56883337a278 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-extract_subvector.ll
@@ -18,7 +18,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
index 7f1b69e6fb798..2d7f3b3f5cc36 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
@@ -5,9 +5,9 @@
 
 define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
 ; CHECK-LABEL: 'test_vXf64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -35,9 +35,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 
 define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
 ; CHECK-LABEL: 'test_vXi64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -65,12 +65,12 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 
 define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
 ; CHECK-LABEL: 'test_vXf32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -131,12 +131,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 
 define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
 ; CHECK-LABEL: 'test_vXi32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -197,16 +197,16 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 
 define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16x i16> %src256, <32 x i16> %src512) {
 ; CHECK-LABEL: 'test_vXi16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -272,21 +272,21 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 
 define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
 ; CHECK-LABEL: 'test_vXi8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
index 0b189c9c5054f..cb8105e5ca544 100644
--- a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
@@ -206,23 +206,23 @@ define void @constrained_fadd(float %a, <16 x float> %va) {
 
 define void @fmaximum(float %a, float %b, <16 x float> %va, <16 x float> %vb) {
 ; THRU-LABEL: 'fmaximum'
-; THRU-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; THRU-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; LATE-LABEL: 'fmaximum'
-; LATE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; LATE-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; LATE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
+; LATE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'fmaximum'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'fmaximum'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call float @llvm.maximum.f32(float %a, float %b)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v = call <16 x float> @llvm.maximum.v16f32(<16 x float> %va, <16 x float> %vb)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %s = call float @llvm.maximum.f32(float %a, float %b)
diff --git a/llvm/test/Analysis/CostModel/X86/reduction.ll b/llvm/test/Analysis/CostModel/X86/reduction.ll
index 63faab7c25ff0..9fe127a28320a 100644
--- a/llvm/test/Analysis/CostModel/X86/reduction.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduction.ll
@@ -11,41 +11,41 @@
 
 define fastcc float @reduction_cost_float(<4 x float> %rdx) {
 ; SSE2-LABEL: 'reduction_cost_float'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'reduction_cost_float'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'reduction_cost_float'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX-LABEL: 'reduction_cost_float'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SLM-LABEL: 'reduction_cost_float'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
@@ -61,41 +61,41 @@ define fastcc float @reduction_cost_float(<4 x float> %rdx) {
 
 define fastcc i32 @reduction_cost_int(<8 x i32> %rdx) {
 ; SSE-LABEL: 'reduction_cost_int'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx, %rdx.shuf
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.2 = add <8 x i32> %bin.rdx, %rdx.shuf.2
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.3 = add <8 x i32> %bin.rdx.2, %rdx.shuf.3
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx.3, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX1-LABEL: 'reduction_cost_int'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <8 x i32> %rdx, %rdx.shuf
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx.2 = add <8 x i32> %bin.rdx, %rdx.shuf.2
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx.3 = add <8 x i32> %bin.rdx.2, %rdx.shuf.3
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx.3, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX2-LABEL: 'reduction_cost_int'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i32> %rdx, %rdx.shuf
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.2 = add <8 x i32> %bin.rdx, %rdx.shuf.2
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.3 = add <8 x i32> %bin.rdx.2, %rdx.shuf.3
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx.3, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; SLM-LABEL: 'reduction_cost_int'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.2 = add <8 x i32> %bin.rdx, %rdx.shuf.2
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.3 = shufflevector <8 x i32> %bin.rdx.2, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.3 = add <8 x i32> %bin.rdx.2, %rdx.shuf.3
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx.3, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
@@ -119,55 +119,55 @@ define fastcc i32 @reduction_cost_int(<8 x i32> %rdx) {
 
 define fastcc float @pairwise_hadd(<4 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'pairwise_hadd'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SSSE3-LABEL: 'pairwise_hadd'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SSE42-LABEL: 'pairwise_hadd'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; AVX-LABEL: 'pairwise_hadd'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SLM-LABEL: 'pairwise_hadd'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
@@ -191,55 +191,55 @@ define fastcc float @pairwise_hadd(<4 x float> %rdx, float %f1) {
 
 define fastcc float @pairwise_hadd_assoc(<4 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'pairwise_hadd_assoc'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.1, %rdx.shuf.0.0
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SSSE3-LABEL: 'pairwise_hadd_assoc'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.1, %rdx.shuf.0.0
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SSE42-LABEL: 'pairwise_hadd_assoc'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.1, %rdx.shuf.0.0
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; AVX-LABEL: 'pairwise_hadd_assoc'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.1, %rdx.shuf.0.0
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SLM-LABEL: 'pairwise_hadd_assoc'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.1, %rdx.shuf.0.0
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
@@ -263,50 +263,50 @@ define fastcc float @pairwise_hadd_assoc(<4 x float> %rdx, float %f1) {
 
 define fastcc float @pairwise_hadd_skip_first(<4 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'pairwise_hadd_skip_first'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SSSE3-LABEL: 'pairwise_hadd_skip_first'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %r2 = fadd float %r, %f1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SSE42-LABEL: 'pairwise_hadd_skip_first'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; AVX-LABEL: 'pairwise_hadd_skip_first'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r2
 ;
 ; SLM-LABEL: 'pairwise_hadd_skip_first'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx.1 = fadd <4 x float> %bin.rdx.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx.1, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r2 = fadd float %r, %f1
@@ -328,31 +328,31 @@ define fastcc float @pairwise_hadd_skip_first(<4 x float> %rdx, float %f1) {
 
 define fastcc double @no_pairwise_reduction2double(<2 x double> %rdx, double %f1) {
 ; SSE2-LABEL: 'no_pairwise_reduction2double'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction2double'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction2double'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; AVX-LABEL: 'no_pairwise_reduction2double'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction2double'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <2 x double> %rdx, %rdx.shuf
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
@@ -366,41 +366,41 @@ define fastcc double @no_pairwise_reduction2double(<2 x double> %rdx, double %f1
 
 define fastcc float @no_pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'no_pairwise_reduction4float'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction4float'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction4float'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX-LABEL: 'no_pairwise_reduction4float'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction4float'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
@@ -416,49 +416,49 @@ define fastcc float @no_pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 
 define fastcc double @no_pairwise_reduction4double(<4 x double> %rdx, double %f1) {
 ; SSE2-LABEL: 'no_pairwise_reduction4double'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction4double'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction4double'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; AVX1-LABEL: 'no_pairwise_reduction4double'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; AVX2-LABEL: 'no_pairwise_reduction4double'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction4double'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
@@ -474,61 +474,61 @@ define fastcc double @no_pairwise_reduction4double(<4 x double> %rdx, double %f1
 
 define fastcc float @no_pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'no_pairwise_reduction8float'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction8float'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction8float'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX1-LABEL: 'no_pairwise_reduction8float'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX2-LABEL: 'no_pairwise_reduction8float'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction8float'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = fadd <8 x float> %rdx, %rdx.shuf3
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x float> %bin.rdx4, <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %bin.rdx4, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
@@ -546,13 +546,13 @@ define fastcc float @no_pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 
 define fastcc i64 @no_pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 ; CHECK-LABEL: 'no_pairwise_reduction2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction2i64'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <2 x i64> %rdx, %rdx.shuf
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <2 x i64> %bin.rdx, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
@@ -566,17 +566,17 @@ define fastcc i64 @no_pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 
 define fastcc i32 @no_pairwise_reduction4i32(<4 x i32> %rdx, i32 %f1) {
 ; CHECK-LABEL: 'no_pairwise_reduction4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction4i32'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
@@ -592,33 +592,33 @@ define fastcc i32 @no_pairwise_reduction4i32(<4 x i32> %rdx, i32 %f1) {
 
 define fastcc i64 @no_pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 ; SSE-LABEL: 'no_pairwise_reduction4i64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <4 x i64> %rdx, %rdx.shuf
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <4 x i64> %bin.rdx, %rdx.shuf7
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; AVX1-LABEL: 'no_pairwise_reduction4i64'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <4 x i64> %rdx, %rdx.shuf
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = add <4 x i64> %bin.rdx, %rdx.shuf7
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; AVX2-LABEL: 'no_pairwise_reduction4i64'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i64> %rdx, %rdx.shuf
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i64> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction4i64'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx = add <4 x i64> %rdx, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx8 = add <4 x i64> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
@@ -634,51 +634,51 @@ define fastcc i64 @no_pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 
 define fastcc i16 @no_pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSE2-LABEL: 'no_pairwise_reduction8i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSSE3-LABEL: 'no_pairwise_reduction8i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSE42-LABEL: 'no_pairwise_reduction8i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; AVX-LABEL: 'no_pairwise_reduction8i16'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction8i16'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i16> %rdx, %rdx.shuf3
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i16> %bin.rdx4, <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %bin.rdx4, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
@@ -696,41 +696,41 @@ define fastcc i16 @no_pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 
 define fastcc i32 @no_pairwise_reduction8i32(<8 x i32> %rdx, i32 %f1) {
 ; SSE-LABEL: 'no_pairwise_reduction8i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = add <8 x i32> %rdx, %rdx.shuf3
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %bin.rdx4, %rdx.shuf
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %bin.rdx, %rdx.shuf7
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx8, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX1-LABEL: 'no_pairwise_reduction8i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx4 = add <8 x i32> %rdx, %rdx.shuf3
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <8 x i32> %bin.rdx4, %rdx.shuf
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = add <8 x i32> %bin.rdx, %rdx.shuf7
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx8, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX2-LABEL: 'no_pairwise_reduction8i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx4 = add <8 x i32> %rdx, %rdx.shuf3
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i32> %bin.rdx4, %rdx.shuf
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i32> %bin.rdx, %rdx.shuf7
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; SLM-LABEL: 'no_pairwise_reduction8i32'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf3 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx4 = add <8 x i32> %rdx, %rdx.shuf3
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf = shufflevector <8 x i32> %bin.rdx4, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %bin.rdx4, %rdx.shuf
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf7 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %bin.rdx, %rdx.shuf7
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
@@ -748,36 +748,36 @@ define fastcc i32 @no_pairwise_reduction8i32(<8 x i32> %rdx, i32 %f1) {
 
 define fastcc double @pairwise_reduction2double(<2 x double> %rdx, double %f1) {
 ; SSE2-LABEL: 'pairwise_reduction2double'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction2double'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction2double'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; AVX-LABEL: 'pairwise_reduction2double'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SLM-LABEL: 'pairwise_reduction2double'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 0, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x double> %rdx, <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <2 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <2 x double> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
@@ -792,51 +792,51 @@ define fastcc double @pairwise_reduction2double(<2 x double> %rdx, double %f1) {
 
 define fastcc float @pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'pairwise_reduction4float'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction4float'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction4float'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX-LABEL: 'pairwise_reduction4float'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SLM-LABEL: 'pairwise_reduction4float'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x float> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
@@ -854,61 +854,61 @@ define fastcc float @pairwise_reduction4float(<4 x float> %rdx, float %f1) {
 
 define fastcc double @pairwise_reduction4double(<4 x double> %rdx, double %f1) {
 ; SSE2-LABEL: 'pairwise_reduction4double'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction4double'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction4double'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; AVX1-LABEL: 'pairwise_reduction4double'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; AVX2-LABEL: 'pairwise_reduction4double'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
 ;
 ; SLM-LABEL: 'pairwise_reduction4double'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x double> %rdx, <4 x double> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <4 x double> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x double> %bin.rdx, <4 x double> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <4 x double> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <4 x double> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %r
@@ -926,79 +926,79 @@ define fastcc double @pairwise_reduction4double(<4 x double> %rdx, double %f1) {
 
 define fastcc float @pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 ; SSE2-LABEL: 'pairwise_reduction8float'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction8float'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction8float'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX1-LABEL: 'pairwise_reduction8float'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; AVX2-LABEL: 'pairwise_reduction8float'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
 ;
 ; SLM-LABEL: 'pairwise_reduction8float'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x float> %rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = fadd <8 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x float> %bin.rdx, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = fadd <8 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x float> %bin.rdx8, <8 x float> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = fadd <8 x float> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractelement <8 x float> %bin.rdx9, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
@@ -1019,15 +1019,15 @@ define fastcc float @pairwise_reduction8float(<8 x float> %rdx, float %f1) {
 
 define fastcc i64 @pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 ; CHECK-LABEL: 'pairwise_reduction2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; SLM-LABEL: 'pairwise_reduction2i64'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 0, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <2 x i64> %rdx, <2 x i64> undef, <2 x i32> <i32 1, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = add <2 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <2 x i64> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
@@ -1042,21 +1042,21 @@ define fastcc i64 @pairwise_reduction2i64(<2 x i64> %rdx, i64 %f1) {
 
 define fastcc i32 @pairwise_reduction4i32(<4 x i32> %rdx, i32 %f1) {
 ; CHECK-LABEL: 'pairwise_reduction4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; SLM-LABEL: 'pairwise_reduction4i32'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i32> %rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i32> %bin.rdx, <4 x i32> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i32> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
@@ -1074,41 +1074,41 @@ define fastcc i32 @pairwise_reduction4i32(<4 x i32> %rdx, i32 %f1) {
 
 define fastcc i64 @pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 ; SSE-LABEL: 'pairwise_reduction4i64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <4 x i64> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <4 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; AVX1-LABEL: 'pairwise_reduction4i64'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <4 x i64> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = add <4 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; AVX2-LABEL: 'pairwise_reduction4i64'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <4 x i64> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <4 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
 ;
 ; SLM-LABEL: 'pairwise_reduction4i64'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <4 x i64> %rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx = add <4 x i64> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.1.0 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <4 x i64> %bin.rdx, <4 x i64> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %bin.rdx8 = add <4 x i64> %rdx.shuf.1.0, %rdx.shuf.1.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <4 x i64> %bin.rdx8, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %r
@@ -1126,66 +1126,66 @@ define fastcc i64 @pairwise_reduction4i64(<4 x i64> %rdx, i64 %f1) {
 
 define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 ; SSE2-LABEL: 'pairwise_reduction8i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSSE3-LABEL: 'pairwise_reduction8i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SSE42-LABEL: 'pairwise_reduction8i16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; AVX-LABEL: 'pairwise_reduction8i16'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
 ;
 ; SLM-LABEL: 'pairwise_reduction8i16'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i16> %rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i16> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i16> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i16> %bin.rdx8, <8 x i16> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i16> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i16> %bin.rdx9, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i16 %r
@@ -1206,53 +1206,53 @@ define fastcc i16 @pairwise_reduction8i16(<8 x i16> %rdx, i16 %f1) {
 
 define fastcc i32 @pairwise_reduction8i32(<8 x i32> %rdx, i32 %f1) {
 ; SSE-LABEL: 'pairwise_reduction8i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX1-LABEL: 'pairwise_reduction8i32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; AVX2-LABEL: 'pairwise_reduction8i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
 ;
 ; SLM-LABEL: 'pairwise_reduction8i32'
-; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.0 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %rdx.shuf.0.1 = shufflevector <8 x i32> %rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx = add <8 x i32> %rdx.shuf.0.0, %rdx.shuf.0.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.0 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.1.1 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx8 = add <8 x i32> %rdx.shuf.1.0, %rdx.shuf.1.1
-; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %rdx.shuf.2.0 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %rdx.shuf.2.1 = shufflevector <8 x i32> %bin.rdx8, <8 x i32> undef, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %bin.rdx9 = add <8 x i32> %rdx.shuf.2.0, %rdx.shuf.2.1
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %r = extractelement <8 x i32> %bin.rdx9, i32 0
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll
index 2676f7abfd174..7e897f41ee456 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-codesize.ll
@@ -27,7 +27,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -40,7 +40,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -53,7 +53,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll
index 979c3a2612c38..94e26227539bc 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-latency.ll
@@ -27,7 +27,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -40,7 +40,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -53,7 +53,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll
index 8fe74bd43c3eb..787a96f3bfde8 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector-sizelatency.ll
@@ -27,7 +27,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -40,7 +40,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -53,7 +53,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
index e015aea54297b..58e99c9ed6523 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-extract_subvector.ll
@@ -27,7 +27,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
@@ -40,7 +40,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
@@ -53,7 +53,7 @@ define void @test_vXf64(<4 x double> %src256, <8 x double> %src512) {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V512_2345 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of -1 for instruction: %V512_567u = shufflevector <8 x double> %src512, <8 x double> undef, <4 x i32> <i32 5, i32 6, i32 7, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V256_01 = shufflevector <4 x double> %src256, <4 x double> undef, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll
index 1202666d9459f..1af0b9d0fac69 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-codesize.ll
@@ -18,9 +18,9 @@
 
 define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -32,9 +32,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -46,9 +46,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -76,9 +76,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 
 define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -90,9 +90,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -104,9 +104,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -134,12 +134,12 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 
 define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
 ; SSE2-LABEL: 'test_vXf32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -165,12 +165,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -196,12 +196,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -227,12 +227,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -258,12 +258,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -289,12 +289,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -355,12 +355,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 
 define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
 ; SSE2-LABEL: 'test_vXi32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -386,12 +386,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -417,12 +417,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -448,12 +448,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -479,12 +479,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -510,12 +510,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -576,16 +576,16 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 
 define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16x i16> %src256, <32 x i16> %src512) {
 ; SSE2-LABEL: 'test_vXi16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -611,16 +611,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -646,16 +646,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -681,16 +681,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -716,16 +716,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -751,16 +751,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -786,16 +786,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -821,16 +821,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi16'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -896,21 +896,21 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 
 define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
 ; SSE2-LABEL: 'test_vXi8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -936,21 +936,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -976,21 +976,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1016,21 +1016,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi8'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1056,21 +1056,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1096,21 +1096,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1136,21 +1136,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi8'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll
index 0aec2b85ec1a8..7a57e57359e9b 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-latency.ll
@@ -18,9 +18,9 @@
 
 define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -32,9 +32,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -46,9 +46,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -76,9 +76,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 
 define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -90,9 +90,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -104,9 +104,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -134,12 +134,12 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 
 define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
 ; SSE2-LABEL: 'test_vXf32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -165,12 +165,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -196,12 +196,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -227,12 +227,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -258,12 +258,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -289,12 +289,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -355,12 +355,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 
 define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
 ; SSE2-LABEL: 'test_vXi32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -386,12 +386,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -417,12 +417,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -448,12 +448,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -479,12 +479,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -510,12 +510,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -576,16 +576,16 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 
 define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16x i16> %src256, <32 x i16> %src512) {
 ; SSE2-LABEL: 'test_vXi16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -611,16 +611,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -646,16 +646,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -681,16 +681,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -716,16 +716,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -751,16 +751,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -786,16 +786,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -821,16 +821,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi16'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -896,21 +896,21 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 
 define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
 ; SSE2-LABEL: 'test_vXi8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -936,21 +936,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -976,21 +976,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1016,21 +1016,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi8'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1056,21 +1056,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1096,21 +1096,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1136,21 +1136,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi8'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll
index 769a9e3b76df5..20ab11c16bbe0 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector-sizelatency.ll
@@ -18,9 +18,9 @@
 
 define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -32,9 +32,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -46,9 +46,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -76,9 +76,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 
 define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -90,9 +90,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -104,9 +104,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -134,12 +134,12 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 
 define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
 ; SSE2-LABEL: 'test_vXf32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -165,12 +165,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -196,12 +196,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -227,12 +227,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -258,12 +258,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -289,12 +289,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -355,12 +355,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 
 define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
 ; SSE2-LABEL: 'test_vXi32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -386,12 +386,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -417,12 +417,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -448,12 +448,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -479,12 +479,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -510,12 +510,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -576,16 +576,16 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 
 define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16x i16> %src256, <32 x i16> %src512) {
 ; SSE2-LABEL: 'test_vXi16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -611,16 +611,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -646,16 +646,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -681,16 +681,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -716,16 +716,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -751,16 +751,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -786,16 +786,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -821,16 +821,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi16'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -896,21 +896,21 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 
 define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
 ; SSE2-LABEL: 'test_vXi8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -936,21 +936,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -976,21 +976,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1016,21 +1016,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi8'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1056,21 +1056,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1096,21 +1096,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1136,21 +1136,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi8'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
index e1d824713fbe8..6f4568c143468 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
@@ -18,9 +18,9 @@
 
 define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double> %src512) {
 ; SSE-LABEL: 'test_vXf64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -32,9 +32,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -46,9 +46,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -76,9 +76,9 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 
 define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512) {
 ; SSE-LABEL: 'test_vXi64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -90,9 +90,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -104,9 +104,9 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi64'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -134,12 +134,12 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 
 define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512) {
 ; SSE2-LABEL: 'test_vXf32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -165,12 +165,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXf32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -196,12 +196,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXf32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -227,12 +227,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXf32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -258,12 +258,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXf32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -289,12 +289,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXf32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x float> %src64, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x float> %src64, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x float> %src64, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -355,12 +355,12 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 
 define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
 ; SSE2-LABEL: 'test_vXi32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -386,12 +386,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -417,12 +417,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -448,12 +448,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -479,12 +479,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -510,12 +510,12 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512-LABEL: 'test_vXi32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -576,16 +576,16 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 
 define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <16x i16> %src256, <32 x i16> %src512) {
 ; SSE2-LABEL: 'test_vXi16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -611,16 +611,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -646,16 +646,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -681,16 +681,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'test_vXi16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -716,16 +716,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'test_vXi16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -751,16 +751,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -786,16 +786,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -821,16 +821,16 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi16'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <2 x i16> %src32, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <2 x i16> %src32, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <2 x i16> %src32, <2 x i16> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <2 x i16> %src32, <2 x i16> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <4 x i16> %src64, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <4 x i16> %src64, <4 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <4 x i16> %src64, <4 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -896,21 +896,21 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 
 define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i8> %src128, <32 x i8> %src256, <64 x i8> %src512) {
 ; SSE2-LABEL: 'test_vXi8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -936,21 +936,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSSE3-LABEL: 'test_vXi8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -976,21 +976,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'test_vXi8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1016,21 +1016,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX-LABEL: 'test_vXi8'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1056,21 +1056,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512F-LABEL: 'test_vXi8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1096,21 +1096,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512BW-LABEL: 'test_vXi8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -1136,21 +1136,21 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX512VMBI-LABEL: 'test_vXi8'
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_32 = shufflevector <2 x i8> %src16, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_64 = shufflevector <2 x i8> %src16, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_128 = shufflevector <2 x i8> %src16, <2 x i8> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_256 = shufflevector <2 x i8> %src16, <2 x i8> undef, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src16_512 = shufflevector <2 x i8> %src16, <2 x i8> undef, <64 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_64 = shufflevector <4 x i8> %src32, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_128 = shufflevector <4 x i8> %src32, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_256 = shufflevector <4 x i8> %src32, <4 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src32_512 = shufflevector <4 x i8> %src32, <4 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_128 = shufflevector <8 x i8> %src64, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_256 = shufflevector <8 x i8> %src64, <8 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src64_512 = shufflevector <8 x i8> %src64, <8 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; AVX512VMBI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-single-src-codesize.ll b/llvm/test/Analysis/CostModel/X86/shuffle-single-src-codesize.ll
index 65313e61f7fcb..639b20fca8522 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-single-src-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-single-src-codesize.ll
@@ -377,9 +377,9 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 
 define void @identity_vXf32(<4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
 ; CHECK-LABEL: 'identity_vXf32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 undef, i32 2, i32 undef, i32 4, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 poison, i32 2, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 poison, i32 2, i32 poison, i32 4, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-single-src-latency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-single-src-latency.ll
index 0909aba40cd39..3fa54588625f1 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-single-src-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-single-src-latency.ll
@@ -377,9 +377,9 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 
 define void @identity_vXf32(<4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
 ; CHECK-LABEL: 'identity_vXf32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 undef, i32 2, i32 undef, i32 4, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 poison, i32 2, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 poison, i32 2, i32 poison, i32 4, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-single-src-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/shuffle-single-src-sizelatency.ll
index 2dd6bb3a686d0..ee9001f2a9515 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-single-src-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-single-src-sizelatency.ll
@@ -377,9 +377,9 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 
 define void @identity_vXf32(<4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
 ; CHECK-LABEL: 'identity_vXf32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 undef, i32 2, i32 undef, i32 4, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 poison, i32 2, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 poison, i32 2, i32 poison, i32 4, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll b/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll
index 5dc0591c1e646..ed3ddc060c167 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-single-src.ll
@@ -377,9 +377,9 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 
 define void @identity_vXf32(<4 x float> %a128, <4 x float> %b128, <8 x float> %a256, <8 x float> %b256, <16 x float> %a512, <16 x float> %b512) {
 ; CHECK-LABEL: 'identity_vXf32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 undef, i32 2, i32 undef, i32 4, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 undef>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 poison, i32 2, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V256 = shufflevector <8 x float> %a256, <8 x float> %b256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 0, i32 poison, i32 2, i32 poison, i32 4, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V128 = shufflevector <4 x float> %a128, <4 x float> %b128, <4 x i32> <i32 0, i32 undef, i32 2, i32 undef>
diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll
new file mode 100644
index 0000000000000..c0704d7423ae6
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes='print<scalar-evolution>' -disable-output %s 2>&1 | FileCheck %s
+
+define void @loop_guard_improves_exact_backedge_taken_count_1(i32 %conv) {
+; CHECK-LABEL: 'loop_guard_improves_exact_backedge_taken_count_1'
+; CHECK-NEXT:  Classifying expressions for: @loop_guard_improves_exact_backedge_taken_count_1
+; CHECK-NEXT:    %and = and i32 %conv, 1
+; CHECK-NEXT:    --> (zext i1 (trunc i32 %conv to i1) to i32) U: [0,2) S: [0,2)
+; CHECK-NEXT:    %conv8 = zext i32 %and to i64
+; CHECK-NEXT:    --> (zext i1 (trunc i32 %conv to i1) to i64) U: [0,2) S: [0,2)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%loop> U: [0,2) S: [0,2) Exits: (zext i1 (trunc i32 %conv to i1) to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%loop> U: [1,3) S: [1,3) Exits: (1 + (zext i1 (trunc i32 %conv to i1) to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @loop_guard_improves_exact_backedge_taken_count_1
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is 1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64)
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %and = and i32 %conv, 1
+  %conv8 = zext i32 %and to i64
+  %c = icmp ugt i32 %and, 0
+  br i1 %c, label %exit, label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  call void @clobber()
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv, %conv8
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @loop_guard_improves_exact_backedge_taken_count_2(i32 %conv) {
+; CHECK-LABEL: 'loop_guard_improves_exact_backedge_taken_count_2'
+; CHECK-NEXT:  Classifying expressions for: @loop_guard_improves_exact_backedge_taken_count_2
+; CHECK-NEXT:    %and = and i32 %conv, 1
+; CHECK-NEXT:    --> (zext i1 (trunc i32 %conv to i1) to i32) U: [0,2) S: [0,2)
+; CHECK-NEXT:    %conv8 = zext i32 %and to i64
+; CHECK-NEXT:    --> (zext i1 (trunc i32 %conv to i1) to i64) U: [0,2) S: [0,2)
+; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%loop> U: [0,2) S: [0,2) Exits: (zext i1 (trunc i32 %conv to i1) to i64) LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.next = add i64 %iv, 1
+; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%loop> U: [1,3) S: [1,3) Exits: (1 + (zext i1 (trunc i32 %conv to i1) to i64))<nuw><nsw> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @loop_guard_improves_exact_backedge_taken_count_2
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64)
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is 1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64)
+; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is (zext i1 (trunc i32 %conv to i1) to i64)
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %loop: Trip multiple is 1
+;
+entry:
+  %and = and i32 %conv, 1
+  %conv8 = zext i32 %and to i64
+  %c = icmp eq i32 %and, 1
+  br i1 %c, label %loop, label %exit
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  call void @clobber()
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv, %conv8
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare void @clobber()
diff --git a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll
index 98b832dedacea..825b4abb5e8c5 100644
--- a/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll
+++ b/llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll
@@ -21,7 +21,7 @@ define dso_local i32 @f() {
 ; CHECK-NEXT:    %idxprom20 = zext i32 %storemerge1921 to i64
 ; CHECK-NEXT:    --> (zext i32 {3,+,-1}<nsw><%for.cond6> to i64) U: [3,4) S: [3,4) Exits: <<Unknown>> LoopDispositions: { %for.cond6: Computable, %outer.loop: Variant }
 ; CHECK-NEXT:    %arrayidx7 = getelementptr inbounds [1 x [4 x i16]], ptr @__const.f.g, i64 0, i64 0, i64 %idxprom20
-; CHECK-NEXT:    --> ((2 * (zext i32 {3,+,-1}<nsw><%for.cond6> to i64))<nuw><nsw> + @__const.f.g)<nuw> U: [6,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %for.cond6: Computable, %outer.loop: Variant }
+; CHECK-NEXT:    --> ((2 * (zext i32 {3,+,-1}<nsw><%for.cond6> to i64))<nuw><nsw> + @__const.f.g)<nuw> U: [6,-3) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %for.cond6: Computable, %outer.loop: Variant }
 ; CHECK-NEXT:    %i = load i16, ptr %arrayidx7, align 2
 ; CHECK-NEXT:    --> %i U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.cond6: Variant, %outer.loop: Variant }
 ; CHECK-NEXT:    %storemerge1822.lcssa.ph = phi i32 [ 0, %for.cond6 ]
@@ -45,7 +45,7 @@ define dso_local i32 @f() {
 ; CHECK-NEXT:    %idxprom20.3 = zext i32 %storemerge1921.3 to i64
 ; CHECK-NEXT:    --> (zext i32 {3,+,-1}<nsw><%inner.loop> to i64) U: [3,4) S: [3,4) Exits: <<Unknown>> LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant }
 ; CHECK-NEXT:    %arrayidx7.3 = getelementptr inbounds [1 x [4 x i16]], ptr @__const.f.g, i64 0, i64 0, i64 %idxprom20.3
-; CHECK-NEXT:    --> ((2 * (zext i32 {3,+,-1}<nsw><%inner.loop> to i64))<nuw><nsw> + @__const.f.g)<nuw> U: [6,-1) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant }
+; CHECK-NEXT:    --> ((2 * (zext i32 {3,+,-1}<nsw><%inner.loop> to i64))<nuw><nsw> + @__const.f.g)<nuw> U: [6,-3) S: [-9223372036854775808,9223372036854775807) Exits: <<Unknown>> LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant }
 ; CHECK-NEXT:    %i7 = load i16, ptr %arrayidx7.3, align 2
 ; CHECK-NEXT:    --> %i7 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %inner.loop: Variant, %outer.loop: Variant }
 ; CHECK-NEXT:    %i8 = load volatile i32, ptr @b, align 4
diff --git a/llvm/test/Analysis/ScalarEvolution/load.ll b/llvm/test/Analysis/ScalarEvolution/load.ll
index 5ba3dff9d9df1..b95ad89f2def9 100644
--- a/llvm/test/Analysis/ScalarEvolution/load.ll
+++ b/llvm/test/Analysis/ScalarEvolution/load.ll
@@ -16,11 +16,11 @@ define i32 @test1() nounwind readnone {
 ; CHECK-NEXT:    %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
 ; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%for.body> U: [0,50) S: [0,50) Exits: 49 LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds [50 x i32], ptr @arr1, i32 0, i32 %i.03
-; CHECK-NEXT:    --> {@arr1,+,4}<nuw><%for.body> U: [0,-3) S: [-2147483648,2147483645) Exits: (196 + @arr1) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    --> {@arr1,+,4}<nuw><%for.body> U: [0,-7) S: [-2147483648,2147483645) Exits: (196 + @arr1)<nuw> LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %0 = load i32, ptr %arrayidx, align 4
 ; CHECK-NEXT:    --> %0 U: full-set S: full-set Exits: 50 LoopDispositions: { %for.body: Variant }
 ; CHECK-NEXT:    %arrayidx1 = getelementptr inbounds [50 x i32], ptr @arr2, i32 0, i32 %i.03
-; CHECK-NEXT:    --> {@arr2,+,4}<nuw><%for.body> U: [0,-3) S: [-2147483648,2147483645) Exits: (196 + @arr2) LoopDispositions: { %for.body: Computable }
+; CHECK-NEXT:    --> {@arr2,+,4}<nuw><%for.body> U: [0,-7) S: [-2147483648,2147483645) Exits: (196 + @arr2)<nuw> LoopDispositions: { %for.body: Computable }
 ; CHECK-NEXT:    %1 = load i32, ptr %arrayidx1, align 4
 ; CHECK-NEXT:    --> %1 U: full-set S: full-set Exits: 0 LoopDispositions: { %for.body: Variant }
 ; CHECK-NEXT:    %add = add i32 %0, %sum.04
diff --git a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll
index d557f50326d53..4ca2fbef9014b 100644
--- a/llvm/test/Analysis/ScalarEvolution/logical-operations.ll
+++ b/llvm/test/Analysis/ScalarEvolution/logical-operations.ll
@@ -427,7 +427,7 @@ define ptr @tautological_select_like_phi(i32 %tc) {
 ; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
 ; CHECK-NEXT:    --> {0,+,1}<nuw><nsw><%loop> U: [0,101) S: [0,101) Exits: 100 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:    %r = phi ptr [ @constant, %truebb ], [ @another_constant, %falsebb ]
-; CHECK-NEXT:    --> %r U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
+; CHECK-NEXT:    --> @constant U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: @constant LoopDispositions: { %loop: Invariant }
 ; CHECK-NEXT:    %iv.next = add i32 %iv, 1
 ; CHECK-NEXT:    --> {1,+,1}<nuw><nsw><%loop> U: [1,102) S: [1,102) Exits: 101 LoopDispositions: { %loop: Computable }
 ; CHECK-NEXT:  Determining loop execution counts for: @tautological_select_like_phi
diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
index d4d3a9e13e277..e6fbcbe5333f8 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
@@ -150,7 +150,7 @@ define void @test_guard_eq_12(ptr nocapture %a, i64 %N) {
 ; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is %N
 ; CHECK-NEXT:  Loop %loop: Predicated backedge-taken count is %N
 ; CHECK-NEXT:   Predicates:
-; CHECK:       Loop %loop: Trip multiple is 1
+; CHECK:       Loop %loop: Trip multiple is 13
 ;
 entry:
   %c.1 = icmp eq i64 %N, 12
diff --git a/llvm/test/Analysis/ScalarEvolution/pr62380.ll b/llvm/test/Analysis/ScalarEvolution/pr62380.ll
new file mode 100644
index 0000000000000..59e8717a994f0
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/pr62380.ll
@@ -0,0 +1,55 @@
+; RUN: opt -passes='loop(loop-deletion),loop-mssa(loop-predication,licm<allowspeculation>,simple-loop-unswitch<nontrivial>),loop(loop-predication)' -S < %s | FileCheck %s
+
+; REQUIRES: asserts
+; XFAIL: *
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test(i32 %arg) {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb2, %bb
+  %load = load i32, ptr null, align 4
+  br label %bb2
+
+bb2:                                              ; preds = %bb1
+  br i1 false, label %bb3, label %bb1
+
+bb3:                                              ; preds = %bb6, %bb2
+  %phi = phi i32 [ %add, %bb6 ], [ 0, %bb2 ]
+  %add = add i32 %phi, 1
+  %icmp = icmp ult i32 %phi, %load
+  br i1 %icmp, label %bb5, label %bb4
+
+bb4:                                              ; preds = %bb3
+  ret void
+
+bb5:                                              ; preds = %bb3
+  %call = call i1 @llvm.experimental.widenable.condition()
+  br i1 %call, label %bb9, label %bb14
+
+bb6:                                              ; preds = %bb9
+  %add7 = add i32 %phi10, 1
+  %icmp8 = icmp ugt i32 %phi10, 1
+  br i1 %icmp8, label %bb3, label %bb9
+
+bb9:                                              ; preds = %bb6, %bb5
+  %phi10 = phi i32 [ %add7, %bb6 ], [ %phi, %bb5 ]
+  %icmp11 = icmp ult i32 %phi10, %arg
+  %call12 = call i1 @llvm.experimental.widenable.condition()
+  %and = and i1 %icmp11, %call12
+  br i1 %and, label %bb6, label %bb13
+
+bb13:                                             ; preds = %bb9
+  ret void
+
+bb14:                                             ; preds = %bb5
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(inaccessiblemem: readwrite)
+declare noundef i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(inaccessiblemem: readwrite) }
diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint-global.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint-global.ll
new file mode 100644
index 0000000000000..dc73e28db9527
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint-global.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes='print<scalar-evolution>' -disable-output %s 2>&1 | FileCheck %s
+
+@glob.i32.align2 = global i32 zeroinitializer, align 2
+@glob.i32.align8 = global i32 zeroinitializer, align 8
+@glob.i32.align16 = global i32 zeroinitializer, align 16
+@array4xi32 = global [4 x i32] zeroinitializer, align 16
+
+define i64 @ptrtoint_align_2_size_4_add_5() {
+; CHECK-LABEL: 'ptrtoint_align_2_size_4_add_5'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_2_size_4_add_5
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @glob.i32.align2 to i64), 5
+; CHECK-NEXT:    --> (5 + (ptrtoint ptr @glob.i32.align2 to i64))<nuw> U: [5,0) S: [5,0)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_2_size_4_add_5
+;
+entry:
+  %add = add i64 ptrtoint (ptr @glob.i32.align2 to i64), 5
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_2_size_4_add_6() {
+; CHECK-LABEL: 'ptrtoint_align_2_size_4_add_6'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_2_size_4_add_6
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @glob.i32.align2 to i64), 6
+; CHECK-NEXT:    --> (6 + (ptrtoint ptr @glob.i32.align2 to i64)) U: [0,-1) S: [-9223372036854775808,9223372036854775807)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_2_size_4_add_6
+;
+entry:
+  %add = add i64 ptrtoint (ptr @glob.i32.align2 to i64), 6
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_8_size_4_add_7() {
+; CHECK-LABEL: 'ptrtoint_align_8_size_4_add_7'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_8_size_4_add_7
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @glob.i32.align8 to i64), 7
+; CHECK-NEXT:    --> (7 + (ptrtoint ptr @glob.i32.align8 to i64))<nuw><nsw> U: [7,0) S: [-9223372036854775801,-9223372036854775808)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_8_size_4_add_7
+;
+entry:
+  %add = add i64 ptrtoint (ptr @glob.i32.align8 to i64), 7
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_8_size_4_add_8() {
+; CHECK-LABEL: 'ptrtoint_align_8_size_4_add_8'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_8_size_4_add_8
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @glob.i32.align8 to i64), 8
+; CHECK-NEXT:    --> (8 + (ptrtoint ptr @glob.i32.align8 to i64)) U: [0,-7) S: [-9223372036854775808,9223372036854775801)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_8_size_4_add_8
+;
+entry:
+  %add = add i64 ptrtoint (ptr @glob.i32.align8 to i64), 8
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_4_add_15() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_4_add_15'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_4_add_15
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @glob.i32.align16 to i64), 15
+; CHECK-NEXT:    --> (15 + (ptrtoint ptr @glob.i32.align16 to i64))<nuw><nsw> U: [15,0) S: [-9223372036854775793,-9223372036854775808)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_4_add_15
+;
+entry:
+  %add = add i64 ptrtoint (ptr @glob.i32.align16 to i64), 15
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_4_add_16() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_4_add_16'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_4_add_16
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @glob.i32.align16 to i64), 16
+; CHECK-NEXT:    --> (16 + (ptrtoint ptr @glob.i32.align16 to i64)) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_4_add_16
+;
+entry:
+  %add = add i64 ptrtoint (ptr @glob.i32.align16 to i64), 16
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_16_add_16() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_16'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_16
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @array4xi32 to i64), 16
+; CHECK-NEXT:    --> (16 + (ptrtoint ptr @array4xi32 to i64))<nuw> U: [16,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_16
+;
+entry:
+  %add = add i64 ptrtoint (ptr @array4xi32 to i64), 16
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_16_add_31() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_31'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_31
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @array4xi32 to i64), 31
+; CHECK-NEXT:    --> (31 + (ptrtoint ptr @array4xi32 to i64))<nuw> U: [31,0) S: [31,0)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_31
+;
+entry:
+  %add = add i64 ptrtoint (ptr @array4xi32 to i64), 31
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_16_add_32() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_32'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_32
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @array4xi32 to i64), 32
+; CHECK-NEXT:    --> (32 + (ptrtoint ptr @array4xi32 to i64)) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_32
+;
+entry:
+  %add = add i64 ptrtoint (ptr @array4xi32 to i64), 32
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_16_add_33() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_33'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_33
+; CHECK-NEXT:    %add = add i64 ptrtoint (ptr @array4xi32 to i64), 33
+; CHECK-NEXT:    --> (33 + (ptrtoint ptr @array4xi32 to i64)) U: [33,2) S: [-9223372036854775775,-9223372036854775790)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_33
+;
+entry:
+  %add = add i64 ptrtoint (ptr @array4xi32 to i64), 33
+  ret i64 %add
+}
+
+define i64 @ptrtoint_align_16_size_16_add_16_umax_sub() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_16_umax_sub'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_16_umax_sub
+; CHECK-NEXT:    %add.16 = add i64 ptrtoint (ptr @array4xi32 to i64), 16
+; CHECK-NEXT:    --> (16 + (ptrtoint ptr @array4xi32 to i64))<nuw> U: [16,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:    %umax = call i64 @llvm.umax.i64(i64 ptrtoint (ptr @array4xi32 to i64), i64 %add.16)
+; CHECK-NEXT:    --> (16 + (ptrtoint ptr @array4xi32 to i64))<nuw> U: [16,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:    %add = add i64 %umax, 16
+; CHECK-NEXT:    --> (32 + (ptrtoint ptr @array4xi32 to i64)) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:    %sub = sub i64 %add, ptrtoint (ptr @array4xi32 to i64)
+; CHECK-NEXT:    --> 32 U: [32,33) S: [32,33)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_16_umax_sub
+;
+entry:
+  %add.16 = add i64 ptrtoint (ptr @array4xi32 to i64), 16
+  %umax = call i64 @llvm.umax.i64(i64 ptrtoint (ptr @array4xi32 to i64), i64 %add.16)
+  %add = add i64 %umax, 16
+  %sub = sub i64 %add, ptrtoint (ptr @array4xi32 to i64)
+  ret i64 %sub
+}
+
+define i64 @ptrtoint_align_16_size_16_add_31_umax_sub() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_31_umax_sub'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_31_umax_sub
+; CHECK-NEXT:    %add.31 = add i64 ptrtoint (ptr @array4xi32 to i64), 31
+; CHECK-NEXT:    --> (31 + (ptrtoint ptr @array4xi32 to i64))<nuw> U: [31,0) S: [31,0)
+; CHECK-NEXT:    %umax = call i64 @llvm.umax.i64(i64 ptrtoint (ptr @array4xi32 to i64), i64 %add.31)
+; CHECK-NEXT:    --> (31 + (ptrtoint ptr @array4xi32 to i64))<nuw> U: [31,0) S: [31,0)
+; CHECK-NEXT:    %add = add i64 %umax, 16
+; CHECK-NEXT:    --> (47 + (ptrtoint ptr @array4xi32 to i64)) U: [47,16) S: [-9223372036854775761,-9223372036854775776)
+; CHECK-NEXT:    %sub = sub i64 %add, ptrtoint (ptr @array4xi32 to i64)
+; CHECK-NEXT:    --> 47 U: [47,48) S: [47,48)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_31_umax_sub
+;
+entry:
+  %add.31 = add i64 ptrtoint (ptr @array4xi32 to i64), 31
+  %umax = call i64 @llvm.umax.i64(i64 ptrtoint (ptr @array4xi32 to i64), i64 %add.31)
+  %add = add i64 %umax, 16
+  %sub = sub i64 %add, ptrtoint (ptr @array4xi32 to i64)
+  ret i64 %sub
+}
+
+define i64 @ptrtoint_align_16_size_16_add_32_umax_sub() {
+; CHECK-LABEL: 'ptrtoint_align_16_size_16_add_32_umax_sub'
+; CHECK-NEXT:  Classifying expressions for: @ptrtoint_align_16_size_16_add_32_umax_sub
+; CHECK-NEXT:    %add.32 = add i64 ptrtoint (ptr @array4xi32 to i64), 32
+; CHECK-NEXT:    --> (32 + (ptrtoint ptr @array4xi32 to i64)) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:    %umax = call i64 @llvm.umax.i64(i64 ptrtoint (ptr @array4xi32 to i64), i64 %add.32)
+; CHECK-NEXT:    --> ((32 + (ptrtoint ptr @array4xi32 to i64)) umax (ptrtoint ptr @array4xi32 to i64)) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:    %add = add i64 %umax, 16
+; CHECK-NEXT:    --> (16 + ((32 + (ptrtoint ptr @array4xi32 to i64)) umax (ptrtoint ptr @array4xi32 to i64))) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:    %sub = sub i64 %add, ptrtoint (ptr @array4xi32 to i64)
+; CHECK-NEXT:    --> (16 + (-1 * (ptrtoint ptr @array4xi32 to i64)) + ((32 + (ptrtoint ptr @array4xi32 to i64)) umax (ptrtoint ptr @array4xi32 to i64))) U: [0,-15) S: [-9223372036854775808,9223372036854775793)
+; CHECK-NEXT:  Determining loop execution counts for: @ptrtoint_align_16_size_16_add_32_umax_sub
+;
+entry:
+  %add.32 = add i64 ptrtoint (ptr @array4xi32 to i64), 32
+  %umax = call i64 @llvm.umax.i64(i64 ptrtoint (ptr @array4xi32 to i64), i64 %add.32)
+  %add = add i64 %umax, 16
+  %sub = sub i64 %add, ptrtoint (ptr @array4xi32 to i64)
+  ret i64 %sub
+}
+
+declare i64 @llvm.umax.i64(i64, i64)
diff --git a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll
index 6c4e77cffb1c6..e307bd589a93d 100644
--- a/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll
+++ b/llvm/test/Analysis/ScalarEvolution/smax-br-phi-idioms.ll
@@ -1,59 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -disable-output "-passes=print<scalar-evolution>" < %s 2>&1 | FileCheck %s
 
 define i32 @f0(i32 %x, i32 %y) {
-; CHECK-LABEL: Classifying expressions for: @f0
- entry:
+; CHECK-LABEL: 'f0'
+; CHECK-NEXT:  Classifying expressions for: @f0
+; CHECK-NEXT:    %sum = add i32 %x, %y
+; CHECK-NEXT:    --> (%x + %y) U: full-set S: full-set
+; CHECK-NEXT:    %v = phi i32 [ %sum, %add ], [ %x, %entry ]
+; CHECK-NEXT:    --> ((0 smax %y) + %x) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @f0
+;
+entry:
   %c = icmp sgt i32 %y, 0
   br i1 %c, label %add, label %merge
 
- add:
+add:
   %sum = add i32 %x, %y
   br label %merge
 
- merge:
+merge:
   %v = phi i32 [ %sum, %add ], [ %x, %entry ]
-; CHECK:  %v = phi i32 [ %sum, %add ], [ %x, %entry ]
-; CHECK-NEXT:  -->  ((0 smax %y) + %x) U: full-set S: full-set
   ret i32 %v
 }
 
 define i32 @f1(i32 %x, i32 %y) {
-; CHECK-LABEL: Classifying expressions for: @f1
- entry:
+; CHECK-LABEL: 'f1'
+; CHECK-NEXT:  Classifying expressions for: @f1
+; CHECK-NEXT:    %sum = add i32 %x, %y
+; CHECK-NEXT:    --> (%x + %y) U: full-set S: full-set
+; CHECK-NEXT:    %v = phi i32 [ %sum, %add ], [ %x, %entry ]
+; CHECK-NEXT:    --> ((0 smax %y) + %x) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @f1
+;
+entry:
   %c = icmp sge i32 %y, 0
   br i1 %c, label %add, label %merge
 
- add:
+add:
   %sum = add i32 %x, %y
   br label %merge
 
- merge:
+merge:
   %v = phi i32 [ %sum, %add ], [ %x, %entry ]
-; CHECK:  %v = phi i32 [ %sum, %add ], [ %x, %entry ]
-; CHECK-NEXT:  -->  ((0 smax %y) + %x) U: full-set S: full-set
   ret i32 %v
 }
 
 define i32 @f2(i32 %x, i32 %y, ptr %ptr) {
-; CHECK-LABEL: Classifying expressions for: @f2
- entry:
+; CHECK-LABEL: 'f2'
+; CHECK-NEXT:  Classifying expressions for: @f2
+; CHECK-NEXT:    %lv = load i32, ptr %ptr, align 4
+; CHECK-NEXT:    --> %lv U: full-set S: full-set
+; CHECK-NEXT:    %v = phi i32 [ %lv, %add ], [ %x, %entry ]
+; CHECK-NEXT:    --> %v U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @f2
+;
+entry:
   %c = icmp sge i32 %y, 0
   br i1 %c, label %add, label %merge
 
- add:
+add:
   %lv = load i32, ptr %ptr
   br label %merge
 
- merge:
+merge:
   %v = phi i32 [ %lv, %add ], [ %x, %entry ]
-; CHECK:  %v = phi i32 [ %lv, %add ], [ %x, %entry ]
-; CHECK-NEXT:  -->  %v U: full-set S: full-set
   ret i32 %v
 }
 
 define i32 @f3(i32 %x, i32 %init, i32 %lim) {
-; CHECK-LABEL: Classifying expressions for: @f3
- entry:
+; CHECK-LABEL: 'f3'
+; CHECK-NEXT:  Classifying expressions for: @f3
+; CHECK-NEXT:    %iv = phi i32 [ %init, %entry ], [ %iv.inc, %merge ]
+; CHECK-NEXT:    --> {%init,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 1
+; CHECK-NEXT:    --> {(1 + %init),+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %sum = add i32 %x, %iv
+; CHECK-NEXT:    --> {(%x + %init),+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %v = phi i32 [ %sum, %add ], [ %x, %loop ]
+; CHECK-NEXT:    --> ((0 smax {%init,+,1}<%loop>) + %x) U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:  Determining loop execution counts for: @f3
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
   br label %loop
 
 loop:
@@ -62,49 +93,78 @@ loop:
   %c = icmp sge i32 %iv, 0
   br i1 %c, label %add, label %merge
 
- add:
+add:
   %sum = add i32 %x, %iv
   br label %merge
 
- merge:
+merge:
   %v = phi i32 [ %sum, %add ], [ %x, %loop ]
-; CHECK:  %v = phi i32 [ %sum, %add ], [ %x, %loop ]
-; CHECK-NEXT:  -->  ((0 smax {%init,+,1}<%loop>) + %x) U: full-set S: full-set
   %be.cond = icmp eq i32 %iv.inc, %lim
   br i1 %be.cond, label %loop, label %leave
 
- leave:
+leave:
   ret i32 0
 }
 
 define i32 @f4(i32 %x, i32 %init, i32 %lim) {
-; CHECK-LABEL: Classifying expressions for: @f4
- entry:
+; CHECK-LABEL: 'f4'
+; CHECK-NEXT:  Classifying expressions for: @f4
+; CHECK-NEXT:    %iv = phi i32 [ %init, %add ], [ %iv.inc, %loop ]
+; CHECK-NEXT:    --> {%init,+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %iv.inc = add i32 %iv, 1
+; CHECK-NEXT:    --> {(1 + %init),+,1}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
+; CHECK-NEXT:    %sum = add i32 %x, %iv
+; CHECK-NEXT:    --> {(%x + %init),+,1}<%loop> U: full-set S: full-set
+; CHECK-NEXT:    %v = phi i32 [ %sum, %add.cont ], [ %x, %entry ]
+; CHECK-NEXT:    --> %v U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @f4
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable predicated backedge-taken count.
+;
+entry:
   %c = icmp sge i32 %init, 0
   br i1 %c, label %add, label %merge
 
- add:
+add:
   br label %loop
 
- loop:
+loop:
   %iv = phi i32 [ %init, %add ], [ %iv.inc, %loop ]
   %iv.inc = add i32 %iv, 1
   %be.cond = icmp eq i32 %iv.inc, %lim
   br i1 %be.cond, label %loop, label %add.cont
 
- add.cont:
+add.cont:
   %sum = add i32 %x, %iv
   br label %merge
 
- merge:
+merge:
   %v = phi i32 [ %sum, %add.cont ], [ %x, %entry ]
-; CHECK:  %v = phi i32 [ %sum, %add.cont ], [ %x, %entry ]
-; CHECK-NEXT:  -->  %v U: full-set S: full-set
   ret i32 %v
 }
 
+; It's okay to match "through" %init, as SCEV expressions don't preserve LCSSA.
 define i32 @f5(ptr %val) {
-; CHECK-LABEL: Classifying expressions for: @f5
+; CHECK-LABEL: 'f5'
+; CHECK-NEXT:  Classifying expressions for: @f5
+; CHECK-NEXT:    %inc = load i32, ptr %val, align 4
+; CHECK-NEXT:    --> %inc U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %for.end: Variant }
+; CHECK-NEXT:    %init = phi i32 [ 0, %for.condt ], [ %inc, %for.end ]
+; CHECK-NEXT:    --> %inc U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @f5
+; CHECK-NEXT:  Loop %for.end: <multiple exits> backedge-taken count is false
+; CHECK-NEXT:    exit count for for.end: false
+; CHECK-NEXT:    exit count for for.condt: false
+; CHECK-NEXT:  Loop %for.end: constant max backedge-taken count is false
+; CHECK-NEXT:  Loop %for.end: symbolic max backedge-taken count is false
+; CHECK-NEXT:    symbolic max exit count for for.end: false
+; CHECK-NEXT:    symbolic max exit count for for.condt: false
+; CHECK-NEXT:  Loop %for.end: Predicated backedge-taken count is false
+; CHECK-NEXT:   Predicates:
+; CHECK:       Loop %for.end: Trip multiple is 1
+;
 entry:
   br label %for.end
 
@@ -117,26 +177,27 @@ for.end:
 
 for.cond.0:
   %init = phi i32 [ 0, %for.condt ], [ %inc, %for.end ]
-
-; CHECK:  %init = phi i32 [ 0, %for.condt ], [ %inc, %for.end ]
-; CHECK-NEXT:  -->  %init U: full-set S: full-set
-
-; Matching "through" %init will break LCSSA at the SCEV expression
-; level.
-
   ret i32 %init
 }
 
-define i32 @f6(i32 %x, i32 %y) {
 ; Do the right thing for unreachable code:
-
-; CHECK-LABEL: Classifying expressions for: @f6
- entry:
+define i32 @f6(i32 %x, i32 %y) {
+; CHECK-LABEL: 'f6'
+; CHECK-NEXT:  Classifying expressions for: @f6
+; CHECK-NEXT:    %sum = add i32 %x, %y
+; CHECK-NEXT:    --> (%x + %y) U: full-set S: full-set
+; CHECK-NEXT:    %v0 = phi i32 [ %sum, %entry ], [ %v1, %unreachable ]
+; CHECK-NEXT:    --> %v0 U: full-set S: full-set
+; CHECK-NEXT:    %v1 = phi i32 [ %v0, %merge ], [ 0, %leave_0_cond ]
+; CHECK-NEXT:    --> %v1 U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @f6
+;
+entry:
   %c0 = icmp sgt i32 %y, 0
   %sum = add i32 %x, %y
   br i1 %c0, label %merge, label %leave_1
 
- merge:
+merge:
   %v0 = phi i32 [ %sum, %entry ], [ %v1, %unreachable ]
   %c1 = icmp slt i32 %y, 0
   br i1 %c1, label %leave_0, label %leave_0_cond
diff --git a/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll b/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
index ecd5d46e84f01..7916c509a33bc 100644
--- a/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
@@ -24,7 +24,7 @@ do.end:                                           ; preds = %do.body
 
 ; CHECK-LABEL: @test2
 ; CHECK:      %tmp24 = phi i64 [ %tmp14, %bb22 ], [ %tmp14, %bb13 ]
-; CHECK-NEXT: -->  %tmp24 U: full-set S: full-set       Exits: <<Unknown>>      LoopDispositions: { %bb13: Variant, %bb8: Variant, %bb17: Invariant, %bb27: Invariant }
+; CHECK-NEXT: -->  {1,+,1}<nuw><nsw><%bb13> U: [1,9223372036854775807) S: [1,9223372036854775807) Exits: (-2 + %arg)
 
 define void @test2(i64 %arg, ptr noalias %arg1) {
 bb:
diff --git a/llvm/test/Analysis/ValueTracking/known-non-zero.ll b/llvm/test/Analysis/ValueTracking/known-non-zero.ll
index 37e0b11c98af1..6b7ecdcc6f688 100644
--- a/llvm/test/Analysis/ValueTracking/known-non-zero.ll
+++ b/llvm/test/Analysis/ValueTracking/known-non-zero.ll
@@ -8,6 +8,20 @@ declare i16 @llvm.bswap.i16(i16)
 declare i8 @llvm.ctpop.i8(i8)
 declare <2 x i8> @llvm.uadd.sat.2xi8(<2 x i8>, <2 x i8>)
 declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i8 @llvm.fshr.i8(i8, i8, i8)
+declare i8 @llvm.fshl.i8(i8, i8, i8)
+declare i8 @llvm.ctlz.i8(i8, i1)
+declare i8 @llvm.cttz.i8(i8, i1)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare i8 @llvm.smax.i8(i8, i8)
+declare i8 @llvm.smin.i8(i8, i8)
+declare i8 @llvm.sshl.sat.i8(i8, i8)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare i8 @llvm.umax.i8(i8, i8)
+declare i8 @llvm.umin.i8(i8, i8)
+declare i8 @llvm.ushl.sat.i8(i8, i8)
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare float @llvm.maximum.f32(float, float)
 
 ;; Throughout use: X > Y || Y == 0 which folds to X > Y iff X known
 ;; non-zero. Do this because many of the expressions already have
@@ -282,7 +296,6 @@ define <2 x i1> @ashr_nz_bounded_cnt_vec(<2 x i32> %x, <2 x i32> %y) {
   ret <2 x i1> %r
 }
 
-
 define i1 @lshr_nz_bounded_cnt_fail(i32 %cnt, i32 %y) {
 ; CHECK-LABEL: @lshr_nz_bounded_cnt_fail(
 ; CHECK-NEXT:    [[CNT_ULT:%.*]] = icmp ult i32 [[CNT:%.*]], 20
@@ -315,9 +328,6 @@ define <2 x i1> @ashr_nz_bounded_cnt_vec_fail(<2 x i32> %x, <2 x i32> %y) {
   ret <2 x i1> %r
 }
 
-
-
-
 define i1 @lshr_nonzero_and_shift_out_zeros(i32 %cnt, i32 %y) {
 ; CHECK-LABEL: @lshr_nonzero_and_shift_out_zeros(
 ; CHECK-NEXT:    [[CNT_ULT:%.*]] = icmp ult i32 [[CNT:%.*]], 4
@@ -329,18 +339,14 @@ define i1 @lshr_nonzero_and_shift_out_zeros(i32 %cnt, i32 %y) {
 ;
   %cnt_ult = icmp ult i32 %cnt, 4
   call void @llvm.assume(i1 %cnt_ult)
-
   %val = and i32 %y, -131072
-
-  %val_nz =  icmp ne i32 %val, 0
+  %val_nz = icmp ne i32 %val, 0
   call void @llvm.assume(i1 %val_nz)
-
   %shl = lshr i32 %val, %cnt
   %r = icmp eq i32 %shl, 0
   ret i1 %r
 }
 
-
 define i1 @ashr_nonzero_and_shift_out_zeros(i32 %ccnt, i32 %y) {
 ; CHECK-LABEL: @ashr_nonzero_and_shift_out_zeros(
 ; CHECK-NEXT:    [[VAL:%.*]] = and i32 [[Y:%.*]], -131072
@@ -350,14 +356,13 @@ define i1 @ashr_nonzero_and_shift_out_zeros(i32 %ccnt, i32 %y) {
 ;
   %cnt = and i32 %ccnt, 7
   %val = and i32 %y, -131072
-  %val_nz =  icmp ne i32 %val, 0
+  %val_nz = icmp ne i32 %val, 0
   call void @llvm.assume(i1 %val_nz)
   %shl = ashr i32 %val, %cnt
   %r = icmp eq i32 %shl, 0
   ret i1 %r
 }
 
-
 define i1 @shl_nonzero_and_shift_out_zeros(i32 %ccnt, i32 %y) {
 ; CHECK-LABEL: @shl_nonzero_and_shift_out_zeros(
 ; CHECK-NEXT:    [[VAL:%.*]] = and i32 [[Y:%.*]], 131071
@@ -367,15 +372,13 @@ define i1 @shl_nonzero_and_shift_out_zeros(i32 %ccnt, i32 %y) {
 ;
   %cnt = and i32 %ccnt, 6
   %val = and i32 %y, 131071
-  %val_nz =  icmp ne i32 %val, 0
+  %val_nz = icmp ne i32 %val, 0
   call void @llvm.assume(i1 %val_nz)
   %shl = shl i32 %val, %cnt
   %r = icmp eq i32 %shl, 0
   ret i1 %r
 }
 
-
-
 define i1 @lshr_nonzero_and_shift_out_zeros_fail(i32 %cnt, i32 %y) {
 ; CHECK-LABEL: @lshr_nonzero_and_shift_out_zeros_fail(
 ; CHECK-NEXT:    [[CNT_ULT:%.*]] = icmp ult i32 [[CNT:%.*]], 19
@@ -389,18 +392,14 @@ define i1 @lshr_nonzero_and_shift_out_zeros_fail(i32 %cnt, i32 %y) {
 ;
   %cnt_ult = icmp ult i32 %cnt, 19
   call void @llvm.assume(i1 %cnt_ult)
-
   %val = and i32 %y, -131072
-
-  %val_nz =  icmp ne i32 %val, 0
+  %val_nz = icmp ne i32 %val, 0
   call void @llvm.assume(i1 %val_nz)
-
   %shl = lshr i32 %val, %cnt
   %r = icmp eq i32 %shl, 0
   ret i1 %r
 }
 
-
 define i1 @ashr_nonzero_and_shift_out_zeros_fail(i32 %ccnt, i32 %y) {
 ; CHECK-LABEL: @ashr_nonzero_and_shift_out_zeros_fail(
 ; CHECK-NEXT:    [[CNT:%.*]] = and i32 [[CCNT:%.*]], 18
@@ -413,14 +412,13 @@ define i1 @ashr_nonzero_and_shift_out_zeros_fail(i32 %ccnt, i32 %y) {
 ;
   %cnt = and i32 %ccnt, 18
   %val = and i32 %y, -131072
-  %val_nz =  icmp ne i32 %val, 0
+  %val_nz = icmp ne i32 %val, 0
   call void @llvm.assume(i1 %val_nz)
   %shl = ashr i32 %val, %cnt
   %r = icmp eq i32 %shl, 0
   ret i1 %r
 }
 
-
 define i1 @shl_nonzero_and_shift_out_zeros_fail(i32 %ccnt, i32 %y) {
 ; CHECK-LABEL: @shl_nonzero_and_shift_out_zeros_fail(
 ; CHECK-NEXT:    [[CNT:%.*]] = and i32 [[CCNT:%.*]], 6
@@ -433,9 +431,668 @@ define i1 @shl_nonzero_and_shift_out_zeros_fail(i32 %ccnt, i32 %y) {
 ;
   %cnt = and i32 %ccnt, 6
   %val = and i32 %y, 268435455
-  %val_nz =  icmp ne i32 %val, 0
+  %val_nz = icmp ne i32 %val, 0
   call void @llvm.assume(i1 %val_nz)
   %shl = shl i32 %val, %cnt
   %r = icmp eq i32 %shl, 0
   ret i1 %r
 }
+
+define i1 @sub_nonzero_ops_ne(i8 %xx, i8 %yy, i8 %z) {
+; CHECK-LABEL: @sub_nonzero_ops_ne(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = and i8 %xx, 191
+  %y = or i8 %yy, 64
+  %s = sub i8 %x, %y
+  %exp = or i8 %z, %s
+  %r = icmp eq i8 %exp, 0
+  ret i1 %r
+}
+
+define i1 @sub_nonzero_ops_ne_fail(i8 %xx, i8 %yy, i8 %z) {
+; CHECK-LABEL: @sub_nonzero_ops_ne_fail(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[XX:%.*]], -64
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[YY:%.*]], 64
+; CHECK-NEXT:    [[S:%.*]] = sub i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[EXP:%.*]] = or i8 [[Z:%.*]], [[S]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[EXP]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = and i8 %xx, 192
+  %y = or i8 %yy, 64
+  %s = sub i8 %x, %y
+  %exp = or i8 %z, %s
+  %r = icmp eq i8 %exp, 0
+  ret i1 %r
+}
+
+define i1 @add_nonzero_nuw(i8 %x, i8 %y) {
+; CHECK-LABEL: @add_nonzero_nuw(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_nz = icmp ne i8 %x, 0
+  call void @llvm.assume(i1 %x_nz)
+  %a = add nuw i8 %x, %y
+  %r = icmp eq i8 %a, 0
+  ret i1 %r
+}
+
+define i1 @add_nonzero_nsw_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @add_nonzero_nsw_fail(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x_nz = icmp ne i8 %x, 0
+  call void @llvm.assume(i1 %x_nz)
+  %a = add nsw i8 %x, %y
+  %r = icmp eq i8 %a, 0
+  ret i1 %r
+}
+
+define i1 @udiv_y_le_x(i8 %xx, i8 %yy, i8 %z) {
+; CHECK-LABEL: @udiv_y_le_x(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = or i8 %xx, 7
+  %y = and i8 %yy, 7
+  %d = udiv i8 %x, %y
+  %o = or i8 %d, %z
+  %r = icmp eq i8 %o, 0
+  ret i1 %r
+}
+
+define i1 @udiv_y_le_x_fail(i8 %xx, i8 %yy, i8 %z) {
+; CHECK-LABEL: @udiv_y_le_x_fail(
+; CHECK-NEXT:    [[X:%.*]] = or i8 [[XX:%.*]], 6
+; CHECK-NEXT:    [[Y:%.*]] = and i8 [[YY:%.*]], 7
+; CHECK-NEXT:    [[D:%.*]] = udiv i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[O:%.*]] = or i8 [[D]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[O]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = or i8 %xx, 6
+  %y = and i8 %yy, 7
+  %d = udiv i8 %x, %y
+  %o = or i8 %d, %z
+  %r = icmp eq i8 %o, 0
+  ret i1 %r
+}
+
+define i1 @fshr_non_zero(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @fshr_non_zero(
+; CHECK-NEXT:    [[PRED0:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRED0]])
+; CHECK-NEXT:    ret i1 false
+;
+  %pred0 = icmp ne i8 %x, 0
+  call void @llvm.assume(i1 %pred0)
+  %v = tail call i8 @llvm.fshr.i8(i8 %x, i8 %x, i8 %y)
+  %or = or i8 %v, %z
+  %r = icmp eq i8 %or, 0
+  ret i1 %r
+}
+
+define i1 @fshr_non_zero_fail(i8 %x, i8 %y, i8 %z, i8 %w) {
+; CHECK-LABEL: @fshr_non_zero_fail(
+; CHECK-NEXT:    [[PRED0:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRED0]])
+; CHECK-NEXT:    [[PRED1:%.*]] = icmp ne i8 [[W:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRED1]])
+; CHECK-NEXT:    [[V:%.*]] = tail call i8 @llvm.fshr.i8(i8 [[X]], i8 [[W]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[V]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[OR]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %pred0 = icmp ne i8 %x, 0
+  call void @llvm.assume(i1 %pred0)
+  %pred1 = icmp ne i8 %w, 0
+  call void @llvm.assume(i1 %pred1)
+  %v = tail call i8 @llvm.fshr.i8(i8 %x, i8 %w, i8 %y)
+  %or = or i8 %v, %z
+  %r = icmp eq i8 %or, 0
+  ret i1 %r
+}
+
+define i1 @fshl_non_zero(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @fshl_non_zero(
+; CHECK-NEXT:    [[PRED0:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRED0]])
+; CHECK-NEXT:    ret i1 false
+;
+  %pred0 = icmp ne i8 %x, 0
+  call void @llvm.assume(i1 %pred0)
+  %v = tail call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 %y)
+  %or = or i8 %v, %z
+  %r = icmp eq i8 %or, 0
+  ret i1 %r
+}
+
+define i1 @fshl_non_zero_fail(i8 %x, i8 %y, i8 %z, i8 %w) {
+; CHECK-LABEL: @fshl_non_zero_fail(
+; CHECK-NEXT:    [[PRED0:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRED0]])
+; CHECK-NEXT:    [[PRED1:%.*]] = icmp ne i8 [[W:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[PRED1]])
+; CHECK-NEXT:    [[V:%.*]] = tail call i8 @llvm.fshl.i8(i8 [[X]], i8 [[W]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[V]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[OR]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %pred0 = icmp ne i8 %x, 0
+  call void @llvm.assume(i1 %pred0)
+  %pred1 = icmp ne i8 %w, 0
+  call void @llvm.assume(i1 %pred1)
+  %v = tail call i8 @llvm.fshl.i8(i8 %x, i8 %w, i8 %y)
+  %or = or i8 %v, %z
+  %r = icmp eq i8 %or, 0
+  ret i1 %r
+}
+
+define i1 @bitcast_nonzero(<2 x i8> %xx, i16 %ind) {
+; CHECK-LABEL: @bitcast_nonzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %xa = add nuw nsw <2 x i8> %xx, <i8 1, i8 1>
+  %x = bitcast <2 x i8> %xa to i16
+  %z = or i16 %x, %ind
+  %r = icmp eq i16 %z, 0
+  ret i1 %r
+}
+
+define i1 @bitcast_todo_partial_nonzero_vec_to_int(<2 x i8> %xx, i16 %ind) {
+; CHECK-LABEL: @bitcast_todo_partial_nonzero_vec_to_int(
+; CHECK-NEXT:    [[XA:%.*]] = add nuw nsw <2 x i8> [[XX:%.*]], <i8 1, i8 0>
+; CHECK-NEXT:    [[X:%.*]] = bitcast <2 x i8> [[XA]] to i16
+; CHECK-NEXT:    [[Z:%.*]] = or i16 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xa = add nuw nsw <2 x i8> %xx, <i8 1, i8 0>
+  %x = bitcast <2 x i8> %xa to i16
+  %z = or i16 %x, %ind
+  %r = icmp eq i16 %z, 0
+  ret i1 %r
+}
+
+define <2 x i1> @bitcast_fail_nonzero_int_to_vec(i16 %xx, <2 x i8> %ind) {
+; CHECK-LABEL: @bitcast_fail_nonzero_int_to_vec(
+; CHECK-NEXT:    [[XA:%.*]] = add nuw nsw i16 [[XX:%.*]], 1
+; CHECK-NEXT:    [[X:%.*]] = bitcast i16 [[XA]] to <2 x i8>
+; CHECK-NEXT:    [[Z:%.*]] = or <2 x i8> [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[Z]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %xa = add nuw nsw i16 %xx, 1
+  %x = bitcast i16 %xa to <2 x i8>
+  %z = or <2 x i8> %x, %ind
+  %r = icmp eq <2 x i8> %z, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @bitcast_veci8_to_veci16(<4 x i8> %xx, <2 x i16> %ind) {
+; CHECK-LABEL: @bitcast_veci8_to_veci16(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %xa = add nuw nsw <4 x i8> %xx, <i8 1, i8 1, i8 1, i8 1>
+  %x = bitcast <4 x i8> %xa to <2 x i16>
+  %z = or <2 x i16> %x, %ind
+  %r = icmp eq <2 x i16> %z, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @bitcast_veci3_to_veci4_fail_not_multiple(<4 x i3> %xx, <3 x i4> %ind) {
+; CHECK-LABEL: @bitcast_veci3_to_veci4_fail_not_multiple(
+; CHECK-NEXT:    [[XA:%.*]] = add nuw nsw <4 x i3> [[XX:%.*]], <i3 1, i3 1, i3 1, i3 1>
+; CHECK-NEXT:    [[X:%.*]] = bitcast <4 x i3> [[XA]] to <3 x i4>
+; CHECK-NEXT:    [[Z:%.*]] = or <3 x i4> [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i4> [[Z]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %xa = add nuw nsw <4 x i3> %xx, <i3 1, i3 1, i3 1, i3 1>
+  %x = bitcast <4 x i3> %xa to <3 x i4>
+  %z = or <3 x i4> %x, %ind
+  %r = icmp eq <3 x i4> %z, zeroinitializer
+  ret <3 x i1> %r
+}
+
+define <4 x i1> @bitcast_fail_veci16_to_veci8(<2 x i16> %xx, <4 x i8> %ind) {
+; CHECK-LABEL: @bitcast_fail_veci16_to_veci8(
+; CHECK-NEXT:    [[XA:%.*]] = add nuw nsw <2 x i16> [[XX:%.*]], <i16 1, i16 1>
+; CHECK-NEXT:    [[X:%.*]] = bitcast <2 x i16> [[XA]] to <4 x i8>
+; CHECK-NEXT:    [[Z:%.*]] = or <4 x i8> [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <4 x i8> [[Z]], zeroinitializer
+; CHECK-NEXT:    ret <4 x i1> [[R]]
+;
+  %xa = add nuw nsw <2 x i16> %xx, <i16 1, i16 1>
+  %x = bitcast <2 x i16> %xa to <4 x i8>
+  %z = or <4 x i8> %x, %ind
+  %r = icmp eq <4 x i8> %z, zeroinitializer
+  ret <4 x i1> %r
+}
+
+define i1 @bitcast_nonzero_fail_dont_check_float(float %xx, i32 %ind) {
+; CHECK-LABEL: @bitcast_nonzero_fail_dont_check_float(
+; CHECK-NEXT:    [[XA:%.*]] = call float @llvm.maximum.f32(float [[XX:%.*]], float 1.000000e+00)
+; CHECK-NEXT:    [[X:%.*]] = bitcast float [[XA]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xa = call float @llvm.maximum.f32(float %xx, float 1.000000e+00)
+  %x = bitcast float %xa to i32
+  %z = or i32 %x, %ind
+  %r = icmp eq i32 %z, 0
+  ret i1 %r
+}
+
+define i1 @ctlz_true_nonzero(i8 %xx, i8 %ind) {
+; CHECK-LABEL: @ctlz_true_nonzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %xs = lshr i8 %xx, 1
+  %x = call i8 @llvm.ctlz.i8(i8 %xs, i1 true)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @ctlz_false_nonzero(i8 %xx, i8 %ind) {
+; CHECK-LABEL: @ctlz_false_nonzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %xa = and i8 %xx, 127
+  %x = call i8 @llvm.ctlz.i8(i8 %xa, i1 true)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @ctlz_nonzero_fail_maybe_neg(i8 %xx, i8 %ind) {
+; CHECK-LABEL: @ctlz_nonzero_fail_maybe_neg(
+; CHECK-NEXT:    [[XS:%.*]] = ashr i8 [[XX:%.*]], 1
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.ctlz.i8(i8 [[XS]], i1 true)
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xs = ashr i8 %xx, 1
+  %x = call i8 @llvm.ctlz.i8(i8 %xs, i1 true)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @cttz_true_nonzero(i8 %xx, i8 %ind) {
+; CHECK-LABEL: @cttz_true_nonzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %xs = shl i8 %xx, 1
+  %x = call i8 @llvm.cttz.i8(i8 %xs, i1 true)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @cttz_false_nonzero(i8 %xx, i8 %ind) {
+; CHECK-LABEL: @cttz_false_nonzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %xa = and i8 %xx, -2
+  %x = call i8 @llvm.cttz.i8(i8 %xa, i1 true)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @cttz_nonzero_fail_maybe_odd(i8 %xx, i8 %cnt, i8 %ind) {
+; CHECK-LABEL: @cttz_nonzero_fail_maybe_odd(
+; CHECK-NEXT:    [[XS:%.*]] = shl i8 [[XX:%.*]], [[CNT:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.cttz.i8(i8 [[XS]], i1 true)
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xs = shl i8 %xx, %cnt
+  %x = call i8 @llvm.cttz.i8(i8 %xs, i1 true)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @mul_nonzero_odd(i8 %xx, i8 %y, i8 %ind) {
+; CHECK-LABEL: @mul_nonzero_odd(
+; CHECK-NEXT:    [[Y_NZ:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %xo = or i8 %xx, 1
+  %y_nz = icmp ne i8 %y, 0
+  call void @llvm.assume(i1 %y_nz)
+  %x = mul i8 %xo, %y
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @mul_nonzero_odd_fail_y_maybe_zero(i8 %xx, i8 %y, i8 %ind) {
+; CHECK-LABEL: @mul_nonzero_odd_fail_y_maybe_zero(
+; CHECK-NEXT:    [[XO:%.*]] = or i8 [[XX:%.*]], 1
+; CHECK-NEXT:    [[X:%.*]] = mul i8 [[XO]], [[Y:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xo = or i8 %xx, 1
+  %x = mul i8 %xo, %y
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @sshl_nonzero(i8 %xx, i8 %y, i8 %ind) {
+; CHECK-LABEL: @sshl_nonzero(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_nz = icmp ne i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %x = call i8 @llvm.sshl.sat.i8(i8 %xx, i8 %y)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @sshl_nonzero_fail_x_maybe_z(i8 %xx, i8 %y, i8 %ind) {
+; CHECK-LABEL: @sshl_nonzero_fail_x_maybe_z(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.sshl.sat.i8(i8 [[XX:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = call i8 @llvm.sshl.sat.i8(i8 %xx, i8 %y)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @ushl_nonzero(i8 %xx, i8 %y, i8 %ind) {
+; CHECK-LABEL: @ushl_nonzero(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_nz = icmp ne i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %x = call i8 @llvm.ushl.sat.i8(i8 %xx, i8 %y)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @ushl_nonzero_fail_x_maybe_z(i8 %xx, i8 %y, i8 %ind) {
+; CHECK-LABEL: @ushl_nonzero_fail_x_maybe_z(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.ushl.sat.i8(i8 [[XX:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = call i8 @llvm.ushl.sat.i8(i8 %xx, i8 %y)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_nonzero(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @ssub_sat_nonzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %xa = and i8 %xx, 191
+  %yo = or i8 %yy, 64
+  %x = call i8 @llvm.ssub.sat.i8(i8 %xa, i8 %yo)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_nonzero_ne_known_bits_fail_overlap(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @ssub_sat_nonzero_ne_known_bits_fail_overlap(
+; CHECK-NEXT:    [[XA:%.*]] = and i8 [[XX:%.*]], -64
+; CHECK-NEXT:    [[YO:%.*]] = or i8 [[YY:%.*]], 64
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[XA]], i8 [[YO]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xa = and i8 %xx, 192
+  %yo = or i8 %yy, 64
+  %x = call i8 @llvm.ssub.sat.i8(i8 %xa, i8 %yo)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @usub_sat_nonzero(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @usub_sat_nonzero(
+; CHECK-NEXT:    [[Y_ULT_31:%.*]] = icmp ult i8 [[YY:%.*]], 31
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_ULT_31]])
+; CHECK-NEXT:    [[XO:%.*]] = or i8 [[XX:%.*]], 34
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[XO]], i8 [[YY]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y_ult_31 = icmp ult i8 %yy, 31
+  call void @llvm.assume(i1 %y_ult_31)
+  %xo = or i8 %xx, 34
+  %x = call i8 @llvm.usub.sat.i8(i8 %xo, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @usub_sat_nonzero_fail(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @usub_sat_nonzero_fail(
+; CHECK-NEXT:    [[XA:%.*]] = and i8 [[XX:%.*]], 16
+; CHECK-NEXT:    [[YO:%.*]] = or i8 [[YY:%.*]], 7
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[XA]], i8 [[YO]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xa = and i8 %xx, 16
+  %yo = or i8 %yy, 7
+  %x = call i8 @llvm.usub.sat.i8(i8 %xa, i8 %yo)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @sadd_sat_nonzero(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @sadd_sat_nonzero(
+; CHECK-NEXT:    [[X_STRICT_POS:%.*]] = icmp sgt i8 [[XX:%.*]], 0
+; CHECK-NEXT:    [[Y_POS:%.*]] = icmp sge i8 [[YY:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_STRICT_POS]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_POS]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_strict_pos = icmp sgt i8 %xx, 0
+  %y_pos = icmp sge i8 %yy, 0
+  call void @llvm.assume(i1 %x_strict_pos)
+  call void @llvm.assume(i1 %y_pos)
+  %x = call i8 @llvm.sadd.sat.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @sadd_sat_nonzero_fail_maybe_zz(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @sadd_sat_nonzero_fail_maybe_zz(
+; CHECK-NEXT:    [[X_POS:%.*]] = icmp sge i8 [[XX:%.*]], 0
+; CHECK-NEXT:    [[Y_POS:%.*]] = icmp sge i8 [[YY:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_POS]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_POS]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[XX]], i8 [[YY]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x_pos = icmp sge i8 %xx, 0
+  %y_pos = icmp sge i8 %yy, 0
+  call void @llvm.assume(i1 %x_pos)
+  call void @llvm.assume(i1 %y_pos)
+  %x = call i8 @llvm.sadd.sat.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @umax_nonzero(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @umax_nonzero(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_nz = icmp ne i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %x = call i8 @llvm.umax.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @umax_nonzero_fail_x_maybe_z(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @umax_nonzero_fail_x_maybe_z(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp sge i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.umax.i8(i8 [[XX]], i8 [[YY:%.*]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x_nz = icmp sge i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %x = call i8 @llvm.umax.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @umin_nonzero(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @umin_nonzero(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    [[Y_NZ:%.*]] = icmp ne i8 [[YY:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_nz = icmp ne i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %y_nz = icmp ne i8 %yy, 0
+  call void @llvm.assume(i1 %y_nz)
+  %x = call i8 @llvm.umin.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @umin_nonzero_fail_y_maybe_z(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @umin_nonzero_fail_y_maybe_z(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.umin.i8(i8 [[XX]], i8 [[YY:%.*]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x_nz = icmp ne i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %x = call i8 @llvm.umin.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @smin_nonzero(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @smin_nonzero(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp ne i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    [[Y_NZ:%.*]] = icmp ne i8 [[YY:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_NZ]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_nz = icmp ne i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %y_nz = icmp ne i8 %yy, 0
+  call void @llvm.assume(i1 %y_nz)
+  %x = call i8 @llvm.umin.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @smin_nonzero_neg_arg(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @smin_nonzero_neg_arg(
+; CHECK-NEXT:    [[X_NEG:%.*]] = icmp slt i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NEG]])
+; CHECK-NEXT:    ret i1 false
+;
+  %x_neg = icmp slt i8 %xx, 0
+  call void @llvm.assume(i1 %x_neg)
+  %x = call i8 @llvm.smin.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @smin_nonzero_fail_y_maybe_z(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @smin_nonzero_fail_y_maybe_z(
+; CHECK-NEXT:    [[X_NZ:%.*]] = icmp sle i8 [[XX:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[X_NZ]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.smin.i8(i8 [[XX]], i8 [[YY:%.*]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x_nz = icmp sle i8 %xx, 0
+  call void @llvm.assume(i1 %x_nz)
+  %x = call i8 @llvm.smin.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @smax_nonzero_pos_arg(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @smax_nonzero_pos_arg(
+; CHECK-NEXT:    ret i1 false
+;
+  %ya = and i8 %yy, 127
+  %yo = or i8 %ya, 1
+  %x = call i8 @llvm.smax.i8(i8 %xx, i8 %yo)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
+
+define i1 @smax_nonzero_pos_arg_fail_nonstrict_pos(i8 %xx, i8 %yy, i8 %ind) {
+; CHECK-LABEL: @smax_nonzero_pos_arg_fail_nonstrict_pos(
+; CHECK-NEXT:    [[Y_POS:%.*]] = icmp sge i8 [[YY:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Y_POS]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.smax.i8(i8 [[XX:%.*]], i8 [[YY]])
+; CHECK-NEXT:    [[Z:%.*]] = or i8 [[X]], [[IND:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[Z]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %y_pos = icmp sge i8 %yy, 0
+  call void @llvm.assume(i1 %y_pos)
+  %x = call i8 @llvm.smax.i8(i8 %xx, i8 %yy)
+  %z = or i8 %x, %ind
+  %r = icmp eq i8 %z, 0
+  ret i1 %r
+}
diff --git a/llvm/test/Assembler/thinlto-memprof-summary.ll b/llvm/test/Assembler/thinlto-memprof-summary.ll
index 92e085e5473be..20b1306ba1b03 100644
--- a/llvm/test/Assembler/thinlto-memprof-summary.ll
+++ b/llvm/test/Assembler/thinlto-memprof-summary.ll
@@ -5,7 +5,7 @@
 
 ^0 = module: (path: "thinlto-memprof-summary.o", hash: (1369602428, 2747878711, 259090915, 2507395659, 1141468049))
 ;; Function with single alloc, multiple memprof MIBs, no versioning
-^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (8632435727821051414)), (type: cold, stackIds: (15025054523792398438, 12345678)), (type: notcoldandcold, stackIds: (23456789))))))))
+^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (8632435727821051414)), (type: cold, stackIds: (15025054523792398438, 12345678))))))))
 ;; Function with callsite stack ids calling above function, no versioning
 ^2 = gv: (guid: 25, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 22, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1)), callsites: ((callee: ^1, clones: (0), stackIds: (8632435727821051414)), (callee: ^1, clones: (0), stackIds: (15025054523792398438, 12345678)), (callee: ^1, clones: (0), stackIds: (23456789))))))
 ;; Function with multiple allocs, multiple memprof MIBs, multiple versions
@@ -17,7 +17,7 @@
 
 ; Make sure we get back from llvm-dis what we put in via llvm-as.
 ; CHECK: ^0 = module: (path: "thinlto-memprof-summary.o", hash: (1369602428, 2747878711, 259090915, 2507395659, 1141468049))
-; CHECK: ^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (8632435727821051414)), (type: cold, stackIds: (15025054523792398438, 12345678)), (type: notcoldandcold, stackIds: (23456789))))))))
+; CHECK: ^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (8632435727821051414)), (type: cold, stackIds: (15025054523792398438, 12345678))))))))
 ; CHECK: ^2 = gv: (guid: 25, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 22, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1)), callsites: ((callee: ^1, clones: (0), stackIds: (8632435727821051414)), (callee: ^1, clones: (0), stackIds: (15025054523792398438, 12345678)), (callee: ^1, clones: (0), stackIds: (23456789))))))
 ; CHECK: ^3 = gv: (guid: 26, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (cold, notcold), memProf: ((type: notcold, stackIds: (3456789)), (type: cold, stackIds: (456789)))), (versions: (notcold, cold), memProf: ((type: cold, stackIds: (3456789)), (type: notcold, stackIds: (456789))))))))
 ; CHECK: ^4 = gv: (guid: 27, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 22, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^3)), callsites: ((callee: ^3, clones: (0, 1), stackIds: (3456789)), (callee: ^3, clones: (1, 1), stackIds: (456789))))))
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index fdde869224c11..0a2d695acb4e0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -154,6 +154,20 @@ define i32 @gc_intr() gc "statepoint-example" {
    ret i32 %ret
 }
 
+declare void @llvm.assume(i1)
+
+; FALLBACK-WITH-REPORT-ERR: <unknown>:0:0: unable to translate instruction: call: '  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cchi},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)' (in function: inline_asm_with_output_constraint)
+; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for inline_asm_with_output_constraint
+; FALLBACK-WITH-REPORT-OUT-LABEL: inline_asm_with_output_constraint
+define i32 @inline_asm_with_output_constraint(i64 %a) {
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cchi},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
 attributes #1 = { "target-features"="+sve" }
 attributes #2 = { "target-features"="+ls64" }
 
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index da0e428ec33db..e4d733fd7c1c6 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon < %s -o -| FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NEON
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
 
 define <8 x i16> @smull_v8i8_v8i16(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: smull_v8i8_v8i16:
@@ -119,19 +120,32 @@ define <4 x i32> @smull_zext_v4i16_v4i32(ptr %A, ptr %B) nounwind {
 }
 
 define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: smull_zext_v2i32_v2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x1]
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    ldrh w11, [x0, #2]
-; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    mov x10, v0.d[1]
-; CHECK-NEXT:    smull x8, w8, w9
-; CHECK-NEXT:    smull x9, w11, w10
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    mov v0.d[1], x9
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: smull_zext_v2i32_v2i64:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    ldr d0, [x1]
+; CHECK-NEON-NEXT:    ldrh w8, [x0]
+; CHECK-NEON-NEXT:    ldrh w11, [x0, #2]
+; CHECK-NEON-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NEON-NEXT:    fmov x9, d0
+; CHECK-NEON-NEXT:    mov x10, v0.d[1]
+; CHECK-NEON-NEXT:    smull x8, w8, w9
+; CHECK-NEON-NEXT:    smull x9, w11, w10
+; CHECK-NEON-NEXT:    fmov d0, x8
+; CHECK-NEON-NEXT:    mov v0.d[1], x9
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: smull_zext_v2i32_v2i64:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    ldrh w8, [x0]
+; CHECK-SVE-NEXT:    ptrue p0.d, vl2
+; CHECK-SVE-NEXT:    ldr d1, [x1]
+; CHECK-SVE-NEXT:    fmov d0, x8
+; CHECK-SVE-NEXT:    ldrh w8, [x0, #2]
+; CHECK-SVE-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-SVE-NEXT:    mov v0.d[1], x8
+; CHECK-SVE-NEXT:    mul z0.d, p0/m, z0.d, z1.d
+; CHECK-SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT:    ret
   %load.A = load <2 x i16>, ptr %A
   %load.B = load <2 x i32>, ptr %B
   %zext.A = zext <2 x i16> %load.A to <2 x i64>
@@ -611,7 +625,7 @@ define <8 x i16> @smull_noextvec_v8i8_v8i16(<8 x i8> %arg) nounwind {
 ; Do not use SMULL if the BUILD_VECTOR element values are too big.
 ; CHECK-LABEL: smull_noextvec_v8i8_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #64537
+; CHECK-NEXT:    mov w8, #64537 // =0xfc19
 ; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
 ; CHECK-NEXT:    dup v1.8h, w8
 ; CHECK-NEXT:    mul v0.8h, v0.8h, v1.8h
@@ -635,7 +649,7 @@ define <4 x i32> @smull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {
 define <2 x i64> @smull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {
 ; CHECK-LABEL: smull_extvec_v2i32_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1234
+; CHECK-NEXT:    mov w8, #-1234 // =0xfffffb2e
 ; CHECK-NEXT:    dup v1.2s, w8
 ; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
@@ -659,7 +673,7 @@ define <8 x i16> @umull_noextvec_v8i8_v8i16(<8 x i8> %arg) nounwind {
 ; Do not use SMULL if the BUILD_VECTOR element values are too big.
 ; CHECK-LABEL: umull_noextvec_v8i8_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #999
+; CHECK-NEXT:    mov w8, #999 // =0x3e7
 ; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-NEXT:    dup v1.8h, w8
 ; CHECK-NEXT:    mul v0.8h, v0.8h, v1.8h
@@ -672,7 +686,7 @@ define <8 x i16> @umull_noextvec_v8i8_v8i16(<8 x i8> %arg) nounwind {
 define <4 x i32> @umull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {
 ; CHECK-LABEL: umull_extvec_v4i16_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1234
+; CHECK-NEXT:    mov w8, #1234 // =0x4d2
 ; CHECK-NEXT:    dup v1.4h, w8
 ; CHECK-NEXT:    umull v0.4s, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
@@ -684,7 +698,7 @@ define <4 x i32> @umull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {
 define <2 x i64> @umull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {
 ; CHECK-LABEL: umull_extvec_v2i32_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1234
+; CHECK-NEXT:    mov w8, #1234 // =0x4d2
 ; CHECK-NEXT:    dup v1.2s, w8
 ; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
@@ -709,7 +723,7 @@ define <8 x i16> @amull_extvec_v8i8_v8i16(<8 x i8> %arg) nounwind {
 define <4 x i32> @amull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {
 ; CHECK-LABEL: amull_extvec_v4i16_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1234
+; CHECK-NEXT:    mov w8, #1234 // =0x4d2
 ; CHECK-NEXT:    movi v1.2d, #0x00ffff0000ffff
 ; CHECK-NEXT:    dup v2.4h, w8
 ; CHECK-NEXT:    smull v0.4s, v0.4h, v2.4h
@@ -724,7 +738,7 @@ define <4 x i32> @amull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {
 define <2 x i64> @amull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {
 ; CHECK-LABEL: amull_extvec_v2i32_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1234
+; CHECK-NEXT:    mov w8, #1234 // =0x4d2
 ; CHECK-NEXT:    movi v1.2d, #0x000000ffffffff
 ; CHECK-NEXT:    dup v2.2s, w8
 ; CHECK-NEXT:    smull v0.2d, v0.2s, v2.2s
diff --git a/llvm/test/CodeGen/AArch64/aarch64_fnmadd.ll b/llvm/test/CodeGen/AArch64/aarch64_fnmadd.ll
new file mode 100644
index 0000000000000..6b71ebe6d929f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64_fnmadd.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -O3 | FileCheck %s
+; REQUIRES: aarch64-registered-target
+
+define void @fnmaddd(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: fnmaddd:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x1]
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    ldr d2, [x2]
+; CHECK-NEXT:    fnmadd d0, d0, d1, d2
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load double, ptr %a, align 8
+  %1 = load double, ptr %b, align 8
+  %mul = fmul fast double %1, %0
+  %2 = load double, ptr %c, align 8
+  %add = fadd fast double %mul, %2
+  %fneg = fneg fast double %add
+  store double %fneg, ptr %a, align 8
+  ret void
+}
+
+; Don't combine: No flags
+define void @fnmaddd_no_fast(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: fnmaddd_no_fast:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ldr d1, [x1]
+; CHECK-NEXT:    fmul d0, d1, d0
+; CHECK-NEXT:    ldr d1, [x2]
+; CHECK-NEXT:    fadd d0, d0, d1
+; CHECK-NEXT:    fneg d0, d0
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load double, ptr %a, align 8
+  %1 = load double, ptr %b, align 8
+  %mul = fmul double %1, %0
+  %2 = load double, ptr %c, align 8
+  %add = fadd double %mul, %2
+  %fneg = fneg double %add
+  store double %fneg, ptr %a, align 8
+  ret void
+}
+
+define void @fnmadds(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: fnmadds:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x1]
+; CHECK-NEXT:    ldr s1, [x0]
+; CHECK-NEXT:    ldr s2, [x2]
+; CHECK-NEXT:    fnmadd s0, s0, s1, s2
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load float, ptr %a, align 4
+  %1 = load float, ptr %b, align 4
+  %mul = fmul fast float %1, %0
+  %2 = load float, ptr %c, align 4
+  %add = fadd fast float %mul, %2
+  %fneg = fneg fast float %add
+  store float %fneg, ptr %a, align 4
+  ret void
+}
+
+define void @fnmadds_nsz_contract(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: fnmadds_nsz_contract:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x1]
+; CHECK-NEXT:    ldr s1, [x0]
+; CHECK-NEXT:    ldr s2, [x2]
+; CHECK-NEXT:    fnmadd s0, s0, s1, s2
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load float, ptr %a, align 4
+  %1 = load float, ptr %b, align 4
+  %mul = fmul contract nsz float %1, %0
+  %2 = load float, ptr %c, align 4
+  %add = fadd contract nsz float %mul, %2
+  %fneg = fneg contract nsz float %add
+  store float %fneg, ptr %a, align 4
+  ret void
+}
+
+; Don't combine: Missing nsz
+define void @fnmadds_contract(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: fnmadds_contract:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x1]
+; CHECK-NEXT:    ldr s1, [x0]
+; CHECK-NEXT:    ldr s2, [x2]
+; CHECK-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-NEXT:    fneg s0, s0
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load float, ptr %a, align 4
+  %1 = load float, ptr %b, align 4
+  %mul = fmul contract float %1, %0
+  %2 = load float, ptr %c, align 4
+  %add = fadd contract float %mul, %2
+  %fneg = fneg contract float %add
+  store float %fneg, ptr %a, align 4
+  ret void
+}
+
+; Don't combine: Missing contract
+define void @fnmadds_nsz(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: fnmadds_nsz:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    ldr s1, [x1]
+; CHECK-NEXT:    fmul s0, s1, s0
+; CHECK-NEXT:    ldr s1, [x2]
+; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-NEXT:    fneg s0, s0
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %0 = load float, ptr %a, align 4
+  %1 = load float, ptr %b, align 4
+  %mul = fmul nsz float %1, %0
+  %2 = load float, ptr %c, align 4
+  %add = fadd nsz float %mul, %2
+  %fneg = fneg nsz float %add
+  store float %fneg, ptr %a, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/addsub.ll b/llvm/test/CodeGen/AArch64/addsub.ll
index 3848a3304c7dc..430462fadefa3 100644
--- a/llvm/test/CodeGen/AArch64/addsub.ll
+++ b/llvm/test/CodeGen/AArch64/addsub.ll
@@ -651,9 +651,6 @@ declare dso_local i32 @crng_reseed(...) local_unnamed_addr
 define dso_local i32 @_extract_crng_crng() {
 ; CHECK-LABEL: _extract_crng_crng:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x8, _extract_crng_crng
 ; CHECK-NEXT:    add x8, x8, :lo12:_extract_crng_crng
 ; CHECK-NEXT:    tbnz x8, #63, .LBB36_2
@@ -665,6 +662,9 @@ define dso_local i32 @_extract_crng_crng() {
 ; CHECK-NEXT:    cmn x8, #1272
 ; CHECK-NEXT:    b.pl .LBB36_3
 ; CHECK-NEXT:  .LBB36_2: // %if.then
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    adrp x8, primary_crng
 ; CHECK-NEXT:    adrp x9, input_pool
 ; CHECK-NEXT:    add x9, x9, :lo12:input_pool
@@ -672,8 +672,10 @@ define dso_local i32 @_extract_crng_crng() {
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel x0, xzr, x9, eq
 ; CHECK-NEXT:    bl crng_reseed
-; CHECK-NEXT:  .LBB36_3: // %if.end
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:  .LBB36_3: // %if.end
+; CHECK-NEXT:    .cfi_def_cfa wsp, 0
+; CHECK-NEXT:    .cfi_same_value w30
 ; CHECK-NEXT:    ret
 entry:
   br i1 icmp slt (ptr @_extract_crng_crng, ptr null), label %if.then, label %lor.lhs.false
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel-crc32.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel-crc32.ll
new file mode 100644
index 0000000000000..3381c297898b9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel-crc32.ll
@@ -0,0 +1,71 @@
+; RUN: llc -mtriple=arm64-eabi -fast-isel -fast-isel-abort=3 -mattr=+crc -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-eabi -fast-isel -fast-isel-abort=3 -mattr=+v8r -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-eabi -fast-isel -fast-isel-abort=3 -mcpu=cortex-a53 -mattr=+crc -o - %s | FileCheck %s
+
+; Note: tests are a copy of arm64-crc32.ll
+
+define i32 @test_crc32b(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32b:
+; CHECK: crc32b w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32b(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32h(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32h:
+; CHECK: crc32h w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32h(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32w(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32w:
+; CHECK: crc32w w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32w(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32x(i32 %cur, i64 %next) {
+; CHECK-LABEL: test_crc32x:
+; CHECK: crc32x w0, w0, x1
+  %val = call i32 @llvm.aarch64.crc32x(i32 %cur, i64 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32cb(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32cb:
+; CHECK: crc32cb w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32cb(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32ch(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32ch:
+; CHECK: crc32ch w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32ch(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32cw(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32cw:
+; CHECK: crc32cw w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32cw(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32cx(i32 %cur, i64 %next) {
+; CHECK-LABEL: test_crc32cx:
+; CHECK: crc32cx w0, w0, x1
+  %val = call i32 @llvm.aarch64.crc32cx(i32 %cur, i64 %next)
+  ret i32 %val
+}
+
+declare i32 @llvm.aarch64.crc32b(i32, i32)
+declare i32 @llvm.aarch64.crc32h(i32, i32)
+declare i32 @llvm.aarch64.crc32w(i32, i32)
+declare i32 @llvm.aarch64.crc32x(i32, i64)
+
+declare i32 @llvm.aarch64.crc32cb(i32, i32)
+declare i32 @llvm.aarch64.crc32ch(i32, i32)
+declare i32 @llvm.aarch64.crc32cw(i32, i32)
+declare i32 @llvm.aarch64.crc32cx(i32, i64)
diff --git a/llvm/test/CodeGen/AArch64/arm64-preserve-all.ll b/llvm/test/CodeGen/AArch64/arm64-preserve-all.ll
new file mode 100644
index 0000000000000..4ccce266c997d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-preserve-all.ll
@@ -0,0 +1,56 @@
+; RUN: llc -O0 --march=aarch64 -verify-machineinstrs --filetype=asm %s -o - 2>&1 | FileCheck %s
+; RUN: llc -O1 --march=aarch64 -verify-machineinstrs --filetype=asm %s -o - 2>&1 | FileCheck %s
+; RUN: llc -O2 --march=aarch64 -verify-machineinstrs --filetype=asm %s -o - 2>&1 | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+declare void @normal_cc()
+
+; Caller: preserve_allcc; callee: normalcc. Normally callee saved registers
+; x9~x15 need to be spilled. Since most of them will be spilled in pairs in
+; reverse order, we only check the odd number ones due to FileCheck not
+; matching the same line of assembly twice.
+; CHECK-LABEL: preserve_all
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q8(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q10(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q12(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q14(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q16(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q18(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q22(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q24(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q26(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q28(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(q[0-9]+, )?q30(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x9(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x11(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x13(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+; CHECK-DAG: {{st[rp]}} {{(x[0-9]+, )?x15(, x[0-9]+)?}}, [sp, #{{[-0-9]+}}]
+define preserve_allcc void @preserve_all() {
+  call void @normal_cc()
+  ret void
+}
+
+; Caller: normalcc; callee: preserve_allcc. x9/q9 does not need to be spilled.
+; The same holds for other x and q registers, but we only check x9 and q9.
+; CHECK-LABEL: normal_cc_caller
+; CHECK-NOT: stp {{x[0-9]+}}, x9, [sp, #{{[-0-9]+}}]
+; CHECK-NOT: stp x9, {{x[0-9]+}}, [sp, #{{[-0-9]+}}]
+; CHECK-NOT: str x9, [sp, {{#[-0-9]+}}]
+; CHECK-NOT: stp {{q[0-9]+}}, q9, [sp, #{{[-0-9]+}}]
+; CHECK-NOT: stp q9, {{q[0-9]+}}, [sp, #{{[-0-9]+}}]
+; CHECK-NOT: str q9, [sp, {{#[-0-9]+}}]
+define dso_local void @normal_cc_caller() {
+entry:
+  %v = alloca i32, align 4
+  call void asm sideeffect "mov x9, $0", "N,~{x9}"(i32 48879) #2
+  call void asm sideeffect "movi v9.2d, #0","~{v9}" () #2
+
+
+  call preserve_allcc void @preserve_all()
+  %0 = load i32, i32* %v, align 4
+  %1 = call i32 asm sideeffect "mov ${0:w}, w9", "=r,r"(i32 %0) #2
+  %2 = call i32 asm sideeffect "fneg v9.4s, v9.4s", "=r,~{v9}"() #2
+  store i32 %1, i32* %v, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
index 79d3291b2fa97..c47f9926e936b 100644
--- a/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
@@ -1004,25 +1004,28 @@ end:
 define i32 @stack_realign(i32 %a, i32 %b, ptr %ptr1, ptr %ptr2) {
 ; ENABLE-LABEL: stack_realign:
 ; ENABLE:       ; %bb.0:
+; ENABLE-NEXT:    lsl w8, w0, w1
+; ENABLE-NEXT:    lsl w9, w1, w0
+; ENABLE-NEXT:    cmp w0, w1
+; ENABLE-NEXT:    b.ge LBB13_2
+; ENABLE-NEXT:  ; %bb.1: ; %true
 ; ENABLE-NEXT:    stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill
 ; ENABLE-NEXT:    .cfi_def_cfa_offset 16
 ; ENABLE-NEXT:    mov x29, sp
 ; ENABLE-NEXT:    .cfi_def_cfa w29, 16
 ; ENABLE-NEXT:    .cfi_offset w30, -8
 ; ENABLE-NEXT:    .cfi_offset w29, -16
-; ENABLE-NEXT:    sub x9, sp, #16
-; ENABLE-NEXT:    and sp, x9, #0xffffffffffffffe0
-; ENABLE-NEXT:    lsl w8, w0, w1
-; ENABLE-NEXT:    lsl w9, w1, w0
-; ENABLE-NEXT:    cmp w0, w1
-; ENABLE-NEXT:    b.ge LBB13_2
-; ENABLE-NEXT:  ; %bb.1: ; %true
+; ENABLE-NEXT:    sub x1, sp, #16
+; ENABLE-NEXT:    and sp, x1, #0xffffffffffffffe0
 ; ENABLE-NEXT:    str w0, [sp]
+; ENABLE-NEXT:    mov sp, x29
+; ENABLE-NEXT:    ldp x29, x30, [sp], #16 ; 16-byte Folded Reload
 ; ENABLE-NEXT:  LBB13_2: ; %false
+; ENABLE-NEXT:    .cfi_def_cfa wsp, 0
+; ENABLE-NEXT:    .cfi_same_value w30
+; ENABLE-NEXT:    .cfi_same_value w29
 ; ENABLE-NEXT:    str w8, [x2]
 ; ENABLE-NEXT:    str w9, [x3]
-; ENABLE-NEXT:    mov sp, x29
-; ENABLE-NEXT:    ldp x29, x30, [sp], #16 ; 16-byte Folded Reload
 ; ENABLE-NEXT:    ret
 ;
 ; DISABLE-LABEL: stack_realign:
diff --git a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
index 1506c68a447a9..185f847646b49 100644
--- a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
+++ b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
@@ -71,7 +71,7 @@ define i64 @sign_i64(i64 %a) {
 define i64 @not_sign_i64(i64 %a) {
 ; CHECK-LABEL: not_sign_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1
+; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    cmp x0, #0
 ; CHECK-NEXT:    cneg x0, x8, le
 ; CHECK-NEXT:    ret
@@ -104,7 +104,7 @@ define i64 @not_sign_i64_3(i64 %a) {
 define i64 @not_sign_i64_4(i64 %a) {
 ; CHECK-LABEL: not_sign_i64_4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x0, #-1
+; CHECK-NEXT:    mov x0, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ret
   %c = icmp ugt i64 %a, -1
   %res = select i1 %c, i64 1, i64 -1
@@ -229,10 +229,10 @@ define <4 x i32> @not_sign_4xi32_3(<4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0xffffffffffffffff
 ; CHECK-NEXT:    adrp x8, .LCPI18_0
-; CHECK-NEXT:    movi v2.4s, #1
-; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI18_0]
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI18_0]
 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
+; CHECK-NEXT:    bic v1.16b, v2.16b, v0.16b
+; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
   %c = icmp sgt <4 x i32> %a, <i32 -1, i32 -1, i32 -1, i32 -1>
   %res = select <4 x i1> %c, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 1>
diff --git a/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll b/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
index 4a4f0571fb4fc..101e5696c70f9 100644
--- a/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
+++ b/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
@@ -658,28 +658,28 @@ return:                                           ; preds = %if.end, %land.lhs.t
 define i32 @fcmpri(i32 %argc, ptr nocapture readonly %argv) #0 {
 ; CHECK-LABEL: fcmpri:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_offset b8, -32
 ; CHECK-NEXT:    cmp w0, #2
 ; CHECK-NEXT:    b.lt .LBB9_3
 ; CHECK-NEXT:  // %bb.1: // %land.lhs.true
 ; CHECK-NEXT:    ldr x8, [x1, #8]
 ; CHECK-NEXT:    cbz x8, .LBB9_3
 ; CHECK-NEXT:  // %bb.2:
-; CHECK-NEXT:    mov w0, #3
-; CHECK-NEXT:    b .LBB9_4
+; CHECK-NEXT:    mov w0, #3 // =0x3
+; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB9_3: // %if.end
-; CHECK-NEXT:    mov w0, #1
+; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_offset b8, -32
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    bl zoo
 ; CHECK-NEXT:    mov w19, w0
-; CHECK-NEXT:    mov w0, #-1
+; CHECK-NEXT:    mov w0, #-1 // =0xffffffff
 ; CHECK-NEXT:    bl yoo
 ; CHECK-NEXT:    cmp w19, #0
-; CHECK-NEXT:    mov w1, #2
+; CHECK-NEXT:    mov w1, #2 // =0x2
 ; CHECK-NEXT:    cinc w0, w19, gt
 ; CHECK-NEXT:    fmov d8, d0
 ; CHECK-NEXT:    bl xoo
@@ -689,9 +689,8 @@ define i32 @fcmpri(i32 %argc, ptr nocapture readonly %argv) #0 {
 ; CHECK-NEXT:    fadd d0, d8, d0
 ; CHECK-NEXT:    fcsel d0, d8, d0, gt
 ; CHECK-NEXT:    bl woo
-; CHECK-NEXT:    mov w0, #4
-; CHECK-NEXT:  .LBB9_4: // %return
 ; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov w0, #4 // =0x4
 ; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
diff --git a/llvm/test/CodeGen/AArch64/csr-split.ll b/llvm/test/CodeGen/AArch64/csr-split.ll
index 9143ddd545cdf..da73c3a13a2e9 100644
--- a/llvm/test/CodeGen/AArch64/csr-split.ll
+++ b/llvm/test/CodeGen/AArch64/csr-split.ll
@@ -9,23 +9,17 @@
 define dso_local signext i32 @test1(ptr %b) local_unnamed_addr uwtable  {
 ; CHECK-LABEL: test1:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    adrp x8, a
 ; CHECK-NEXT:    ldrsw x8, [x8, :lo12:a]
 ; CHECK-NEXT:    cmp x8, x0
 ; CHECK-NEXT:    b.eq .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %if.end
-; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    .cfi_restore w19
-; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: // %if.then
-; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl callVoid
 ; CHECK-NEXT:    mov x0, x19
@@ -37,14 +31,6 @@ define dso_local signext i32 @test1(ptr %b) local_unnamed_addr uwtable  {
 ;
 ; CHECK-APPLE-LABEL: test1:
 ; CHECK-APPLE:       ; %bb.0: ; %entry
-; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
-; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
-; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
-; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
-; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
-; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
-; CHECK-APPLE-NEXT:    .cfi_remember_state
 ; CHECK-APPLE-NEXT:  Lloh0:
 ; CHECK-APPLE-NEXT:    adrp x8, _a@PAGE
 ; CHECK-APPLE-NEXT:  Lloh1:
@@ -52,16 +38,15 @@ define dso_local signext i32 @test1(ptr %b) local_unnamed_addr uwtable  {
 ; CHECK-APPLE-NEXT:    cmp x8, x0
 ; CHECK-APPLE-NEXT:    b.eq LBB0_2
 ; CHECK-APPLE-NEXT:  ; %bb.1: ; %if.end
-; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
-; CHECK-APPLE-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
-; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-APPLE-NEXT:    .cfi_restore w30
-; CHECK-APPLE-NEXT:    .cfi_restore w29
-; CHECK-APPLE-NEXT:    .cfi_restore w19
-; CHECK-APPLE-NEXT:    .cfi_restore w20
 ; CHECK-APPLE-NEXT:    ret
 ; CHECK-APPLE-NEXT:  LBB0_2: ; %if.then
-; CHECK-APPLE-NEXT:    .cfi_restore_state
+; CHECK-APPLE-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-APPLE-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-APPLE-NEXT:    .cfi_offset w30, -8
+; CHECK-APPLE-NEXT:    .cfi_offset w29, -16
+; CHECK-APPLE-NEXT:    .cfi_offset w19, -24
+; CHECK-APPLE-NEXT:    .cfi_offset w20, -32
 ; CHECK-APPLE-NEXT:    mov x19, x0
 ; CHECK-APPLE-NEXT:    bl _callVoid
 ; CHECK-APPLE-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll b/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
index 0bbe869c5da02..71b4ed3880e66 100644
--- a/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
+++ b/llvm/test/CodeGen/AArch64/dag-combine-setcc.ll
@@ -193,10 +193,10 @@ define i1 @combine_setcc_ne_vecreduce_and_v8i1(<8 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_ne_vecreduce_and_v8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    uminv b0, v0.8b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    bic w0, w8, w9
 ; CHECK-NEXT:    ret
   %cmp1 = icmp ne <8 x i8> %a, zeroinitializer
   %cast = bitcast <8 x i1> %cmp1 to i8
@@ -208,10 +208,10 @@ define i1 @combine_setcc_ne_vecreduce_and_v16i1(<16 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_ne_vecreduce_and_v16i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v0.16b
+; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    uminv b0, v0.16b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    bic w0, w8, w9
 ; CHECK-NEXT:    ret
   %cmp1 = icmp ne <16 x i8> %a, zeroinitializer
   %cast = bitcast <16 x i1> %cmp1 to i16
@@ -223,12 +223,14 @@ define i1 @combine_setcc_ne_vecreduce_and_v32i1(<32 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_ne_vecreduce_and_v32i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v0.16b
+; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    cmeq v1.16b, v1.16b, #0
 ; CHECK-NEXT:    bic v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    uminv b0, v0.16b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    bic w0, w8, w9
 ; CHECK-NEXT:    ret
   %cmp1 = icmp ne <32 x i8> %a, zeroinitializer
   %cast = bitcast <32 x i1> %cmp1 to i32
@@ -240,16 +242,18 @@ define i1 @combine_setcc_ne_vecreduce_and_v64i1(<64 x i8> %a) {
 ; CHECK-LABEL: combine_setcc_ne_vecreduce_and_v64i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v1.16b, v1.16b, v1.16b
+; CHECK-NEXT:    mov w8, #1 // =0x1
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v0.16b
 ; CHECK-NEXT:    cmeq v3.16b, v3.16b, #0
 ; CHECK-NEXT:    cmeq v2.16b, v2.16b, #0
 ; CHECK-NEXT:    bic v1.16b, v1.16b, v3.16b
 ; CHECK-NEXT:    bic v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    uminv b0, v0.16b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tst w8, #0x1
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    bic w0, w8, w9
 ; CHECK-NEXT:    ret
   %cmp1 = icmp ne <64 x i8> %a, zeroinitializer
   %cast = bitcast <64 x i1> %cmp1 to i64
diff --git a/llvm/test/CodeGen/AArch64/double_reduct.ll b/llvm/test/CodeGen/AArch64/double_reduct.ll
index 78408ae10e0ba..b2dfe3f160e3b 100644
--- a/llvm/test/CodeGen/AArch64/double_reduct.ll
+++ b/llvm/test/CodeGen/AArch64/double_reduct.ll
@@ -131,9 +131,9 @@ define i32 @and_i32(<8 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
   %r1 = call i32 @llvm.vector.reduce.and.i32.v8i32(<8 x i32> %a)
   %r2 = call i32 @llvm.vector.reduce.and.i32.v4i32(<4 x i32> %b)
@@ -148,9 +148,9 @@ define i32 @or_i32(<8 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    orr w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
   %r1 = call i32 @llvm.vector.reduce.or.i32.v8i32(<8 x i32> %a)
   %r2 = call i32 @llvm.vector.reduce.or.i32.v4i32(<4 x i32> %b)
@@ -165,9 +165,9 @@ define i32 @xor_i32(<8 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    eor v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
   %r1 = call i32 @llvm.vector.reduce.xor.i32.v8i32(<8 x i32> %a)
   %r2 = call i32 @llvm.vector.reduce.xor.i32.v4i32(<4 x i32> %b)
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-call-struct-return-fallback.ll b/llvm/test/CodeGen/AArch64/fast-isel-call-struct-return-fallback.ll
new file mode 100644
index 0000000000000..52d8eb35da7b0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fast-isel-call-struct-return-fallback.ll
@@ -0,0 +1,14 @@
+; RUN: llc -fast-isel -pass-remarks-missed=isel < %s 2>&1 >/dev/null | FileCheck -check-prefix=STDERR -allow-empty %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+declare { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s10i64()
+
+define i64 @call_ret_s10i64() {
+; STDERR: FastISel missed call:   %ret = call { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s10i64() (in function: call_ret_s10i64)
+  %ret = call { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s10i64()
+  %ext0 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 0
+  %ext1 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 1
+  %sum = add i64 %ext0, %ext1
+  ret i64 %sum
+}
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-call-struct-return.ll b/llvm/test/CodeGen/AArch64/fast-isel-call-struct-return.ll
new file mode 100644
index 0000000000000..6b43bdfa276ec
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fast-isel-call-struct-return.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no-generate-body-for-unused-prefixes
+; RUN: llc -fast-isel -fast-isel-abort=3 < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+declare { i64, i64 } @ret_s2i64()
+
+define i64 @call_ret_s2i64() {
+; CHECK-LABEL: call_ret_s2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl ret_s2i64
+; CHECK-NEXT:    add x0, x0, x1
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ret = call { i64, i64 } @ret_s2i64()
+  %ext0 = extractvalue { i64, i64 } %ret, 0
+  %ext1 = extractvalue { i64, i64 } %ret, 1
+  %sum = add i64 %ext0, %ext1
+  ret i64 %sum
+}
+
+declare { i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s8i64()
+
+define i64 @call_ret_s8i64() {
+; CHECK-LABEL: call_ret_s8i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    bl ret_s8i64
+; CHECK-NEXT:    add x0, x0, x7
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ret = call { i64, i64, i64, i64, i64, i64, i64, i64 } @ret_s8i64()
+  %ext0 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 0
+  %ext7 = extractvalue { i64, i64, i64, i64, i64, i64, i64, i64 } %ret, 7
+  %sum = add i64 %ext0, %ext7
+  ret i64 %sum
+}
diff --git a/llvm/test/CodeGen/AArch64/gep-sink-loop-inv-live.ll b/llvm/test/CodeGen/AArch64/gep-sink-loop-inv-live.ll
deleted file mode 100644
index 8d7a38028985a..0000000000000
--- a/llvm/test/CodeGen/AArch64/gep-sink-loop-inv-live.ll
+++ /dev/null
@@ -1,62 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -codegenprepare < %s | FileCheck %s
-target triple = "aarch64-linux"
-
-declare void @use(...)
-declare i64 @next(i64)
-
-define void @f(ptr %a, i64 %k, i64 %n, ptr %q) {
-; CHECK-LABEL: @f(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[COND:%.*]]
-; CHECK:       cond:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[J:%.*]], [[IF_THEN:%.*]] ], [ [[I]], [[IF_ELSE:%.*]] ]
-; CHECK-NEXT:    [[P:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[I]]
-; CHECK-NEXT:    [[C0:%.*]] = icmp ult i64 [[I]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[C0]], label [[LOOP:%.*]], label [[EXIT:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[J]] = call i64 @next(i64 [[I]])
-; CHECK-NEXT:    [[SUNKADDR:%.*]] = mul i64 [[I]], 4
-; CHECK-NEXT:    [[SUNKADDR1:%.*]] = getelementptr i8, ptr [[A]], i64 [[SUNKADDR]]
-; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR1]], align 4
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[V]], 0
-; CHECK-NEXT:    br i1 [[C1]], label [[IF_THEN]], label [[IF_ELSE]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store ptr [[P]], ptr [[Q:%.*]], align 8
-; CHECK-NEXT:    br label [[COND]]
-; CHECK:       if.else:
-; CHECK-NEXT:    call void @use(ptr [[A]])
-; CHECK-NEXT:    br label [[COND]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
-;
-entry:
-  br label %cond
-
-cond:
-  %i = phi i64 [0, %entry], [%i.next, %next]
-  %p = getelementptr i32, ptr %a, i64 %i
-  %c0 = icmp ult i64 %i, %n
-  br i1 %c0, label %loop, label %exit
-
-loop:
-  %j = call i64 @next(i64 %i)
-  %v = load i32, ptr %p
-  %c1 = icmp slt i32 %v, 0
-  br i1 %c1, label %if.then, label %if.else
-
-if.then:
-  store ptr %p, ptr %q
-  br label %next
-
-if.else:
-  call void @use(ptr %a)
-  br label %next
-
-next:
-  %i.next = phi i64 [%j, %if.then], [%i, %if.else]
-  br label %cond
-
-exit:
-  ret void
-}
diff --git a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
index 018f58c36eb1a..f5f31e796815b 100644
--- a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
+++ b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
@@ -36,6 +36,12 @@ define void @f3(ptr %x0, ptr %x1) {
   ret void
 }
 
+define void @f4(ptr %x0, ptr %x1) {
+  ; 0x1000010 (access-size-index = 0, is-write = 1, match-all = 0x0)
+  call void @llvm.hwasan.check.memaccess.shortgranules(ptr %x0, ptr %x1, i32 16777232)
+  ret void
+}
+
 declare void @llvm.hwasan.check.memaccess(ptr, ptr, i32)
 declare void @llvm.hwasan.check.memaccess.shortgranules(ptr, ptr, i32)
 
@@ -98,11 +104,40 @@ declare void @llvm.hwasan.check.memaccess.shortgranules(ptr, ptr, i32)
 ; CHECK-NEXT: .Ltmp6:
 ; CHECK-NEXT: ret
 ; CHECK-NEXT: .Ltmp5:
-; CHECK-NEXT: lsr x16, x1, #56
-; CHECK-NEXT: cmp x16, #255
+; CHECK-NEXT: lsr x17, x1, #56
+; CHECK-NEXT: cmp x17, #255
 ; CHECK-NEXT: b.eq .Ltmp6
 ; CHECK-NEXT: stp x0, x1, [sp, #-256]!
 ; CHECK-NEXT: stp x29, x30, [sp, #232]
 ; CHECK-NEXT: mov x0, x1
 ; CHECK-NEXT: mov x1, #0
 ; CHECK-NEXT: b __hwasan_tag_mismatch
+
+; CHECK:      __hwasan_check_x1_16777232_short_v2:
+; CHECK-NEXT: sbfx	x16, x1, #4, #52
+; CHECK-NEXT: ldrb	w16, [x20, x16]
+; CHECK-NEXT: cmp	x16, x1, lsr #56
+; CHECK-NEXT: b.ne	.Ltmp7
+; CHECK-NEXT: .Ltmp8:
+; CHECK-NEXT: ret
+; CHECK-NEXT: .Ltmp7:
+; CHECK-NEXT: lsr	x17, x1, #56
+; CHECK-NEXT: cmp	x17, #0
+; CHECK-NEXT: b.eq	.Ltmp8
+; CHECK-NEXT: cmp	w16, #15
+; CHECK-NEXT: b.hi	.Ltmp9
+; CHECK-NEXT: and	x17, x1, #0xf
+; CHECK-NEXT: cmp	w16, w17
+; CHECK-NEXT: b.ls	.Ltmp9
+; CHECK-NEXT: orr	x16, x1, #0xf
+; CHECK-NEXT: ldrb	w16, [x16]
+; CHECK-NEXT: cmp	x16, x1, lsr #56
+; CHECK-NEXT: b.eq	.Ltmp8
+; CHECK-NEXT: .Ltmp9:
+; CHECK-NEXT: stp	x0, x1, [sp, #-256]!
+; CHECK-NEXT: stp	x29, x30, [sp, #232]
+; CHECK-NEXT: mov	x0, x1
+; CHECK-NEXT: mov	x1, #16
+; CHECK-NEXT: adrp	x16, :got:__hwasan_tag_mismatch_v2
+; CHECK-NEXT: ldr	x16, [x16, :got_lo12:__hwasan_tag_mismatch_v2]
+; CHECK-NEXT: br	x16
diff --git a/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll b/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
index 278a36d1a66fb..f098b1e7b6201 100644
--- a/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
+++ b/llvm/test/CodeGen/AArch64/illegal-floating-point-vector-compares.ll
@@ -59,12 +59,14 @@ define i1 @unordered_floating_point_compare_on_v32f32(<32 x float> %a_vec) {
 ; CHECK-NEXT:    fcmgt v4.4s, v4.4s, #0.0
 ; CHECK-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
 ; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    uzp1 v6.8h, v6.8h, v7.8h
-; CHECK-NEXT:    uzp1 v1.8h, v4.8h, v5.8h
+; CHECK-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
+; CHECK-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
 ; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    uzp1 v1.16b, v1.16b, v6.16b
+; CHECK-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    orn v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    shl v0.16b, v0.16b, #7
+; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    umaxv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    bic w0, w9, w8
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-flag-output.ll b/llvm/test/CodeGen/AArch64/inline-asm-flag-output.ll
new file mode 100644
index 0000000000000..c761758ee51d3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-flag-output.ll
@@ -0,0 +1,259 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s
+define i32 @test_cchi(i64 %a) {
+; CHECK-LABEL: test_cchi:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cchi},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_cccs(i64 %a) {
+; CHECK-LABEL: test_cccs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cccs},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_cclo(i64 %a) {
+; CHECK-LABEL: test_cclo:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cclo},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccls(i64 %a) {
+; CHECK-LABEL: test_ccls:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccls},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_cccc(i64 %a) {
+; CHECK-LABEL: test_cccc:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cccc},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_cceq(i64 %a) {
+; CHECK-LABEL: test_cceq:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cceq},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccgt(i64 %a) {
+; CHECK-LABEL: test_ccgt:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, gt
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccgt},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccge(i64 %a) {
+; CHECK-LABEL: test_ccge:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccge},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_cclt(i64 %a) {
+; CHECK-LABEL: test_cclt:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cclt},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccle(i64 %a) {
+; CHECK-LABEL: test_ccle:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccle},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_cchs(i64 %a) {
+; CHECK-LABEL: test_cchs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@cchs},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccne(i64 %a) {
+; CHECK-LABEL: test_ccne:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccne},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccvc(i64 %a) {
+; CHECK-LABEL: test_ccvc:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, vc
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccvc},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccpl(i64 %a) {
+; CHECK-LABEL: test_ccpl:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, pl
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccpl},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccvs(i64 %a) {
+; CHECK-LABEL: test_ccvs:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, vs
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccvs},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+define i32 @test_ccmi(i64 %a) {
+; CHECK-LABEL: test_ccmi:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    subs x0, x0, #3
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    cset w0, mi
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call { i64, i32 } asm "subs $0, $0, #3", "=r,={@ccmi},0,~{dirflag},~{fpsr},~{flags}"(i64 %a)
+  %asmresult1 = extractvalue { i64, i32 } %0, 1
+  %1 = icmp ult i32 %asmresult1, 2
+  tail call void @llvm.assume(i1 %1)
+  ret i32 %asmresult1
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir b/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir
new file mode 100644
index 0000000000000..9a8e5c6341bca
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir
@@ -0,0 +1,261 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -start-before=machine-latecleanup -stop-after=machine-latecleanup \
+# RUN:   -o - %s | FileCheck %s
+--- |
+  ; ModuleID = 'reduced5.ll'
+  source_filename = "reduced5.ll"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-unknown-linux-gnu"
+
+  ; Function Attrs: nounwind
+  define void @zot(ptr %arg, ptr %arg1, ptr %arg2, i64 %arg3, i1 %arg4, ptr %arg5) #0 {
+  bb:
+    %const = bitcast i64 29273397577910035 to i64
+    callbr void asm sideeffect "b $0", "!i"()
+            to label %bb9 [label %bb8]
+
+  bb8:                                              ; preds = %bb
+    %const_mat = add i64 %const, 2048
+    br label %bb9
+
+  bb9:                                              ; preds = %bb8, %bb
+    %phi = phi i64 [ %const_mat, %bb8 ], [ %const, %bb ]
+    store volatile i64 %phi, ptr null, align 8
+    br label %bb10
+
+  bb10:                                             ; preds = %bb22, %bb9
+    %phi11 = phi i64 [ 0, %bb9 ], [ %or24, %bb22 ]
+    %phi12 = phi i64 [ 0, %bb9 ], [ %select, %bb22 ]
+    %load = load volatile i64, ptr %arg2, align 8
+    %select = select i1 %arg4, i64 %arg3, i64 0
+    store i64 %const, ptr %arg5, align 8
+    %call = call i64 %arg()
+    callbr void asm sideeffect "", "!i"()
+            to label %bb22 [label %bb22]
+
+  bb22:                                             ; preds = %bb10, %bb10
+    store i64 0, ptr %arg1, align 8
+    store volatile i64 1, ptr null, align 8
+    %or = or i64 %phi11, 0
+    %or24 = or i64 %phi11, %phi12
+    br label %bb10
+  }
+
+  attributes #0 = { nounwind "frame-pointer"="non-leaf" }
+
+...
+---
+name:            zot
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+callsEHReturn:   false
+callsUnwindInit: false
+hasEHCatchret:   false
+hasEHScopes:     false
+hasEHFunclets:   false
+isOutlined:      false
+debugInstrRef:   false
+failsVerification: false
+tracksDebugUserValues: true
+registers:       []
+liveins:
+  - { reg: '$x0', virtual-reg: '' }
+  - { reg: '$x1', virtual-reg: '' }
+  - { reg: '$x2', virtual-reg: '' }
+  - { reg: '$x3', virtual-reg: '' }
+  - { reg: '$w4', virtual-reg: '' }
+  - { reg: '$x5', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       96
+  offsetAdjustment: 0
+  maxAlignment:    8
+  adjustsStack:    true
+  hasCalls:        true
+  stackProtector:  ''
+  functionContext: ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  hasTailCall:     false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x19', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -16, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x20', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, name: '', type: spill-slot, offset: -24, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x21', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 3, name: '', type: spill-slot, offset: -32, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x22', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 4, name: '', type: spill-slot, offset: -40, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x23', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 5, name: '', type: spill-slot, offset: -48, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x24', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 6, name: '', type: spill-slot, offset: -56, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x25', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 7, name: '', type: spill-slot, offset: -64, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x26', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 8, name: '', type: spill-slot, offset: -72, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x27', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 9, name: '', type: spill-slot, offset: -80, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$x28', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 10, name: '', type: spill-slot, offset: -88, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 11, name: '', type: spill-slot, offset: -96, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '$fp', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  ; CHECK-LABEL: name: zot
+  ; CHECK: bb.0.bb:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000), %bb.1(0x00000000)
+  ; CHECK-NEXT:   liveins: $w4, $x0, $x1, $x2, $x3, $x5, $lr, $x27, $x28, $x25, $x26, $x23, $x24, $x21, $x22, $x19, $x20
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -12 :: (store (s64) into %stack.11), (store (s64) into %stack.10)
+  ; CHECK-NEXT:   frame-setup STPXi killed $x28, killed $x27, $sp, 2 :: (store (s64) into %stack.9), (store (s64) into %stack.8)
+  ; CHECK-NEXT:   frame-setup STPXi killed $x26, killed $x25, $sp, 4 :: (store (s64) into %stack.7), (store (s64) into %stack.6)
+  ; CHECK-NEXT:   frame-setup STPXi killed $x24, killed $x23, $sp, 6 :: (store (s64) into %stack.5), (store (s64) into %stack.4)
+  ; CHECK-NEXT:   frame-setup STPXi killed $x22, killed $x21, $sp, 8 :: (store (s64) into %stack.3), (store (s64) into %stack.2)
+  ; CHECK-NEXT:   frame-setup STPXi killed $x20, killed $x19, $sp, 10 :: (store (s64) into %stack.1), (store (s64) into %stack.0)
+  ; CHECK-NEXT:   $fp = frame-setup ADDXri $sp, 0, 0
+  ; CHECK-NEXT:   renamable $x19 = COPY $x5
+  ; CHECK-NEXT:   renamable $w20 = COPY $w4
+  ; CHECK-NEXT:   renamable $x21 = COPY $x3
+  ; CHECK-NEXT:   renamable $x22 = COPY $x2
+  ; CHECK-NEXT:   renamable $x23 = COPY $x1
+  ; CHECK-NEXT:   renamable $x24 = COPY $x0
+  ; CHECK-NEXT:   INLINEASM_BR &"b $0", 1 /* sideeffect attdialect */, 13 /* imm */, %bb.1
+  ; CHECK-NEXT:   renamable $x8 = MOVi64imm 29273397577910035
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.bb8 (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $w20, $x19, $x21, $x22, $x23, $x24
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x8 = MOVi64imm 29273397577910035
+  ; CHECK-NEXT:   renamable $x8 = ADDXri killed renamable $x8, 2048, 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.bb9:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT:   liveins: $w20, $x8, $x19, $x21, $x22, $x23, $x24
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x25 = COPY $xzr
+  ; CHECK-NEXT:   renamable $w27 = MOVi32imm 1, implicit-def $x27
+  ; CHECK-NEXT:   renamable $x26 = COPY $xzr
+  ; CHECK-NEXT:   STRXui killed renamable $x8, renamable $x25, 0 :: (volatile store (s64) into `ptr null`)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3.bb10:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT:   liveins: $w20, $x19, $x21, $x22, $x23, $x24, $x25, $x26, $x27
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead $wzr = ANDSWri renamable $w20, 0, implicit-def $nzcv
+  ; CHECK-NEXT:   dead $xzr = LDRXui renamable $x22, 0 :: (volatile load (s64) from %ir.arg2)
+  ; CHECK-NEXT:   renamable $x28 = CSELXr renamable $x21, $xzr, 1, implicit killed $nzcv
+  ; CHECK-NEXT:   renamable $x8 = MOVi64imm 29273397577910035
+  ; CHECK-NEXT:   STRXui killed renamable $x8, renamable $x19, 0 :: (store (s64) into %ir.arg5)
+  ; CHECK-NEXT:   BLR renamable $x24, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
+  ; CHECK-NEXT:   INLINEASM_BR &"", 1 /* sideeffect attdialect */, 13 /* imm */, %bb.4
+  ; CHECK-NEXT:   B %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4.bb22 (machine-block-address-taken, inlineasm-br-indirect-target):
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT:   liveins: $w20, $x19, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x8 = COPY $xzr
+  ; CHECK-NEXT:   renamable $x25 = ORRXrr killed renamable $x25, killed renamable $x26
+  ; CHECK-NEXT:   renamable $x26 = COPY killed renamable $x28
+  ; CHECK-NEXT:   STRXui $xzr, renamable $x23, 0 :: (store (s64) into %ir.arg1)
+  ; CHECK-NEXT:   STRXui renamable $x27, killed renamable $x8, 0 :: (volatile store (s64) into `ptr null`)
+  ; CHECK-NEXT:   B %bb.3
+  bb.0.bb:
+    successors: %bb.2(0x80000000), %bb.1(0x00000000)
+    liveins: $w4, $x0, $x1, $x2, $x3, $x5, $lr, $x27, $x28, $x25, $x26, $x23, $x24, $x21, $x22, $x19, $x20
+
+    early-clobber $sp = frame-setup STPXpre killed $fp, killed $lr, $sp, -12 :: (store (s64) into %stack.11), (store (s64) into %stack.10)
+    frame-setup STPXi killed $x28, killed $x27, $sp, 2 :: (store (s64) into %stack.9), (store (s64) into %stack.8)
+    frame-setup STPXi killed $x26, killed $x25, $sp, 4 :: (store (s64) into %stack.7), (store (s64) into %stack.6)
+    frame-setup STPXi killed $x24, killed $x23, $sp, 6 :: (store (s64) into %stack.5), (store (s64) into %stack.4)
+    frame-setup STPXi killed $x22, killed $x21, $sp, 8 :: (store (s64) into %stack.3), (store (s64) into %stack.2)
+    frame-setup STPXi killed $x20, killed $x19, $sp, 10 :: (store (s64) into %stack.1), (store (s64) into %stack.0)
+    $fp = frame-setup ADDXri $sp, 0, 0
+    renamable $x19 = COPY $x5
+    renamable $w20 = COPY $w4
+    renamable $x21 = COPY $x3
+    renamable $x22 = COPY $x2
+    renamable $x23 = COPY $x1
+    renamable $x24 = COPY $x0
+    INLINEASM_BR &"b $0", 1 /* sideeffect attdialect */, 13 /* imm */, %bb.1
+    renamable $x8 = MOVi64imm 29273397577910035
+    B %bb.2
+
+  bb.1.bb8 (machine-block-address-taken, inlineasm-br-indirect-target):
+    successors: %bb.2(0x80000000)
+    liveins: $w20, $x19, $x21, $x22, $x23, $x24
+
+    renamable $x8 = MOVi64imm 29273397577910035
+    renamable $x8 = ADDXri killed renamable $x8, 2048, 0
+
+  bb.2.bb9:
+    successors: %bb.3(0x80000000)
+    liveins: $w20, $x8, $x19, $x21, $x22, $x23, $x24
+
+    renamable $x25 = COPY $xzr
+    renamable $w27 = MOVi32imm 1, implicit-def $x27
+    renamable $x26 = COPY $xzr
+    STRXui killed renamable $x8, renamable $x25, 0 :: (volatile store (s64) into `ptr null`)
+
+  bb.3.bb10:
+    successors: %bb.4(0x80000000)
+    liveins: $w20, $x19, $x21, $x22, $x23, $x24, $x25, $x26, $x27
+
+    dead $wzr = ANDSWri renamable $w20, 0, implicit-def $nzcv
+    dead $xzr = LDRXui renamable $x22, 0 :: (volatile load (s64) from %ir.arg2)
+    renamable $x28 = CSELXr renamable $x21, $xzr, 1, implicit killed $nzcv
+    renamable $x8 = MOVi64imm 29273397577910035
+    STRXui killed renamable $x8, renamable $x19, 0 :: (store (s64) into %ir.arg5)
+    BLR renamable $x24, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
+    INLINEASM_BR &"", 1 /* sideeffect attdialect */, 13 /* imm */, %bb.4
+    B %bb.4
+
+  bb.4.bb22 (machine-block-address-taken, inlineasm-br-indirect-target):
+    successors: %bb.3(0x80000000)
+    liveins: $w20, $x19, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28
+
+    renamable $x8 = COPY $xzr
+    renamable $x25 = ORRXrr killed renamable $x25, killed renamable $x26
+    renamable $x26 = COPY killed renamable $x28
+    STRXui $xzr, renamable $x23, 0 :: (store (s64) into %ir.arg1)
+    STRXui renamable $x27, killed renamable $x8, 0 :: (volatile store (s64) into `ptr null`)
+    B %bb.3
+
+...
diff --git a/llvm/test/CodeGen/AArch64/neon-rshrn.ll b/llvm/test/CodeGen/AArch64/neon-rshrn.ll
index 53880502c9ee1..b29d1a52fb762 100644
--- a/llvm/test/CodeGen/AArch64/neon-rshrn.ll
+++ b/llvm/test/CodeGen/AArch64/neon-rshrn.ll
@@ -769,7 +769,7 @@ entry:
 define <4 x i32> @rshrn_v4i64_33(<4 x i64> %a) {
 ; CHECK-LABEL: rshrn_v4i64_33:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, #4294967296
+; CHECK-NEXT:    mov x8, #4294967296 // =0x100000000
 ; CHECK-NEXT:    dup v2.2d, x8
 ; CHECK-NEXT:    add v0.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    add v1.2d, v1.2d, v2.2d
@@ -783,3 +783,118 @@ entry:
   %m = trunc <4 x i64> %s to <4 x i32>
   ret <4 x i32> %m
 }
+
+define <8 x i8> @rshrn_v8i16_5(<8 x i16> %a) {
+; CHECK-LABEL: rshrn_v8i16_5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rshrn v0.8b, v0.8h, #5
+; CHECK-NEXT:    ret
+entry:
+  %b = add <8 x i16> %a, <i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16>
+  %s = lshr <8 x i16> %b, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  %m = trunc <8 x i16> %s to <8 x i8>
+  ret <8 x i8> %m
+}
+
+define <4 x i16> @rshrn_v4i32_4(<4 x i32> %a) {
+; CHECK-LABEL: rshrn_v4i32_4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rshrn v0.4h, v0.4s, #4
+; CHECK-NEXT:    ret
+entry:
+  %b = add <4 x i32> %a, <i32 8, i32 8, i32 8, i32 8>
+  %s = lshr <4 x i32> %b, <i32 4, i32 4, i32 4, i32 4>
+  %m = trunc <4 x i32> %s to <4 x i16>
+  ret <4 x i16> %m
+}
+
+define <2 x i32> @rshrn_v2i64_5(<2 x i64> %a) {
+; CHECK-LABEL: rshrn_v2i64_5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rshrn v0.2s, v0.2d, #5
+; CHECK-NEXT:    ret
+entry:
+  %b = add <2 x i64> %a, <i64 16, i64 16>
+  %s = lshr <2 x i64> %b, <i64 5, i64 5>
+  %m = trunc <2 x i64> %s to <2 x i32>
+  ret <2 x i32> %m
+}
+
+define void @rshrn_v8i32i8_5(<8 x i32> %a, ptr %p) {
+; CHECK-LABEL: rshrn_v8i32i8_5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    rshrn v0.8b, v0.8h, #5
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %b = add <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %s = lshr <8 x i32> %b, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
+  %m = trunc <8 x i32> %s to <8 x i8>
+  store <8 x i8> %m, ptr %p
+  ret void
+}
+
+define void @rshrn_v4i64i16_4(<4 x i64> %a, ptr %p) {
+; CHECK-LABEL: rshrn_v4i64i16_4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    rshrn v0.4h, v0.4s, #4
+; CHECK-NEXT:    str d0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %b = add <4 x i64> %a, <i64 8, i64 8, i64 8, i64 8>
+  %s = lshr <4 x i64> %b, <i64 4, i64 4, i64 4, i64 4>
+  %m = trunc <4 x i64> %s to <4 x i16>
+  store <4 x i16> %m, ptr %p
+  ret void
+}
+
+define void @rshrn_v4i16_5(<4 x i16> %a, ptr %p) {
+; CHECK-LABEL: rshrn_v4i16_5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    rshrn v0.8b, v0.8h, #5
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %b = add <4 x i16> %a, <i16 16, i16 16, i16 16, i16 16>
+  %s = lshr <4 x i16> %b, <i16 5, i16 5, i16 5, i16 5>
+  %m = trunc <4 x i16> %s to <4 x i8>
+  store <4 x i8> %m, ptr %p
+  ret void
+}
+
+define void @rshrn_v2i32_4(<2 x i32> %a, ptr %p) {
+; CHECK-LABEL: rshrn_v2i32_4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.2s, #8
+; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ushr v0.2s, v0.2s, #4
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    strh w9, [x0]
+; CHECK-NEXT:    strh w8, [x0, #2]
+; CHECK-NEXT:    ret
+entry:
+  %b = add <2 x i32> %a, <i32 8, i32 8>
+  %s = lshr <2 x i32> %b, <i32 4, i32 4>
+  %m = trunc <2 x i32> %s to <2 x i16>
+  store <2 x i16> %m, ptr %p
+  ret void
+}
+
+define void @rshrn_v1i64_5(<1 x i64> %a, ptr %p) {
+; CHECK-LABEL: rshrn_v1i64_5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    rshrn v0.2s, v0.2d, #5
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %b = add <1 x i64> %a, <i64 16>
+  %s = lshr <1 x i64> %b, <i64 5>
+  %m = trunc <1 x i64> %s to <1 x i32>
+  store <1 x i32> %m, ptr %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/preserve.ll b/llvm/test/CodeGen/AArch64/preserve.ll
new file mode 100644
index 0000000000000..924e6f8487ebf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/preserve.ll
@@ -0,0 +1,22 @@
+
+; RUN: llc -enable-ipra -print-regusage -o /dev/null 2>&1 < %s | FileCheck %s
+
+target triple = "aarch64-unknown-unknown"
+declare void @bar1()
+define preserve_mostcc void @baz() #0 {
+; CHECK: baz Clobbered Registers: $ffr $fpcr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $z0_hi $z1_hi $z2_hi $z3_hi $z4_hi $z5_hi $z6_hi $z7_hi $z8_hi $z9_hi $z10_hi $z11_hi $z12_hi $z13_hi $z14_hi $z15_hi $z16_hi $z17_hi $z18_hi $z19_hi $z20_hi $z21_hi $z22_hi $z23_hi $z24_hi $z25_hi $z26_hi $z27_hi $z28_hi $z29_hi $z30_hi $z31_hi $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+  call void @bar1()
+  call void @bar2()
+  ret void
+}
+define preserve_allcc void @foo() #0 {
+; CHECK: foo Clobbered Registers: $ffr $fpcr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $z0_hi $z1_hi $z2_hi $z3_hi $z4_hi $z5_hi $z6_hi $z7_hi $z8_hi $z9_hi $z10_hi $z11_hi $z12_hi $z13_hi $z14_hi $z15_hi $z16_hi $z17_hi $z18_hi $z19_hi $z20_hi $z21_hi $z22_hi $z23_hi $z24_hi $z25_hi $z26_hi $z27_hi $z28_hi $z29_hi $z30_hi $z31_hi $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+  call void @bar1()
+  call void @bar2()
+  ret void
+}
+declare void @bar2()
+
+@llvm.used = appending global [2 x ptr] [ptr @foo, ptr @baz]
+
+attributes #0 = {nounwind}
diff --git a/llvm/test/CodeGen/AArch64/reduce-and.ll b/llvm/test/CodeGen/AArch64/reduce-and.ll
index 66eb3080f14ea..71b3f1d247991 100644
--- a/llvm/test/CodeGen/AArch64/reduce-and.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-and.ll
@@ -256,9 +256,14 @@ define i8 @test_redand_v1i8(<1 x i8> %a) {
 define i8 @test_redand_v3i8(<3 x i8> %a) {
 ; CHECK-LABEL: test_redand_v3i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, w1
-; CHECK-NEXT:    and w8, w8, w2
-; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    movi d0, #0xff00ff00ff00ff
+; CHECK-NEXT:    mov v0.h[0], w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v3i8:
@@ -273,14 +278,10 @@ define i8 @test_redand_v3i8(<3 x i8> %a) {
 define i8 @test_redand_v4i8(<4 x i8> %a) {
 ; CHECK-LABEL: test_redand_v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[1]
-; CHECK-NEXT:    umov w11, v0.h[0]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w10, w11, w10
-; CHECK-NEXT:    and w0, w10, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v4i8:
@@ -304,22 +305,11 @@ define i8 @test_redand_v4i8(<4 x i8> %a) {
 define i8 @test_redand_v8i8(<8 x i8> %a) {
 ; CHECK-LABEL: test_redand_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.b[5]
-; CHECK-NEXT:    umov w9, v0.b[4]
-; CHECK-NEXT:    umov w10, v0.b[1]
-; CHECK-NEXT:    umov w11, v0.b[0]
-; CHECK-NEXT:    umov w12, v0.b[3]
-; CHECK-NEXT:    umov w13, v0.b[2]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    umov w15, v0.b[7]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w10, w11, w10
-; CHECK-NEXT:    and w11, w13, w12
-; CHECK-NEXT:    and w9, w10, w11
-; CHECK-NEXT:    and w8, w8, w14
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w0, w8, w15
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    and x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v8i8:
@@ -357,20 +347,10 @@ define i8 @test_redand_v16i8(<16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[7]
-; CHECK-NEXT:    and w10, w10, w11
-; CHECK-NEXT:    and w11, w12, w13
-; CHECK-NEXT:    and w8, w8, w10
-; CHECK-NEXT:    and w10, w11, w14
-; CHECK-NEXT:    and w8, w8, w10
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    and x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
 ; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -411,20 +391,10 @@ define i8 @test_redand_v32i8(<32 x i8> %a) {
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[7]
-; CHECK-NEXT:    and w10, w10, w11
-; CHECK-NEXT:    and w11, w12, w13
-; CHECK-NEXT:    and w8, w8, w10
-; CHECK-NEXT:    and w10, w11, w14
-; CHECK-NEXT:    and w8, w8, w10
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    and x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
 ; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -463,14 +433,10 @@ define i8 @test_redand_v32i8(<32 x i8> %a) {
 define i16 @test_redand_v4i16(<4 x i16> %a) {
 ; CHECK-LABEL: test_redand_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[1]
-; CHECK-NEXT:    umov w11, v0.h[0]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w10, w11, w10
-; CHECK-NEXT:    and w0, w10, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v4i16:
@@ -496,12 +462,9 @@ define i16 @test_redand_v8i16(<8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w9, w10, w11
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
 ; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -530,12 +493,9 @@ define i16 @test_redand_v16i16(<16 x i16> %a) {
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w9, w10, w11
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
 ; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -562,10 +522,9 @@ define i16 @test_redand_v16i16(<16 x i16> %a) {
 define i32 @test_redand_v2i32(<2 x i32> %a) {
 ; CHECK-LABEL: test_redand_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v2i32:
@@ -585,9 +544,9 @@ define i32 @test_redand_v4i32(<4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v4i32:
@@ -609,9 +568,9 @@ define i32 @test_redand_v8i32(<8 x i32> %a) {
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redand_v8i32:
diff --git a/llvm/test/CodeGen/AArch64/reduce-or.ll b/llvm/test/CodeGen/AArch64/reduce-or.ll
index a44dcee3978ba..591182018164c 100644
--- a/llvm/test/CodeGen/AArch64/reduce-or.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-or.ll
@@ -256,8 +256,16 @@ define i8 @test_redor_v1i8(<1 x i8> %a) {
 define i8 @test_redor_v3i8(<3 x i8> %a) {
 ; CHECK-LABEL: test_redor_v3i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    orr w8, w0, w1
-; CHECK-NEXT:    orr w0, w8, w2
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.h[0], w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    lsr x10, x9, #32
+; CHECK-NEXT:    lsr x9, x9, #16
+; CHECK-NEXT:    orr w8, w8, w10
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v3i8:
@@ -272,14 +280,10 @@ define i8 @test_redor_v3i8(<3 x i8> %a) {
 define i8 @test_redor_v4i8(<4 x i8> %a) {
 ; CHECK-LABEL: test_redor_v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[1]
-; CHECK-NEXT:    umov w11, v0.h[0]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w10, w11, w10
-; CHECK-NEXT:    orr w0, w10, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v4i8:
@@ -303,22 +307,11 @@ define i8 @test_redor_v4i8(<4 x i8> %a) {
 define i8 @test_redor_v8i8(<8 x i8> %a) {
 ; CHECK-LABEL: test_redor_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.b[5]
-; CHECK-NEXT:    umov w9, v0.b[4]
-; CHECK-NEXT:    umov w10, v0.b[1]
-; CHECK-NEXT:    umov w11, v0.b[0]
-; CHECK-NEXT:    umov w12, v0.b[3]
-; CHECK-NEXT:    umov w13, v0.b[2]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    umov w15, v0.b[7]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w10, w11, w10
-; CHECK-NEXT:    orr w11, w13, w12
-; CHECK-NEXT:    orr w9, w10, w11
-; CHECK-NEXT:    orr w8, w8, w14
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w0, w8, w15
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    orr x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v8i8:
@@ -356,20 +349,10 @@ define i8 @test_redor_v16i8(<16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[7]
-; CHECK-NEXT:    orr w10, w10, w11
-; CHECK-NEXT:    orr w11, w12, w13
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    orr w10, w11, w14
-; CHECK-NEXT:    orr w8, w8, w10
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    orr x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
 ; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -410,20 +393,10 @@ define i8 @test_redor_v32i8(<32 x i8> %a) {
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[7]
-; CHECK-NEXT:    orr w10, w10, w11
-; CHECK-NEXT:    orr w11, w12, w13
-; CHECK-NEXT:    orr w8, w8, w10
-; CHECK-NEXT:    orr w10, w11, w14
-; CHECK-NEXT:    orr w8, w8, w10
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    orr x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
 ; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -462,14 +435,10 @@ define i8 @test_redor_v32i8(<32 x i8> %a) {
 define i16 @test_redor_v4i16(<4 x i16> %a) {
 ; CHECK-LABEL: test_redor_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[1]
-; CHECK-NEXT:    umov w11, v0.h[0]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w10, w11, w10
-; CHECK-NEXT:    orr w0, w10, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v4i16:
@@ -495,12 +464,9 @@ define i16 @test_redor_v8i16(<8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w9, w10, w11
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
 ; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -529,12 +495,9 @@ define i16 @test_redor_v16i16(<16 x i16> %a) {
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    orr w8, w9, w8
-; CHECK-NEXT:    orr w9, w10, w11
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    orr x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
 ; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -561,10 +524,9 @@ define i16 @test_redor_v16i16(<16 x i16> %a) {
 define i32 @test_redor_v2i32(<2 x i32> %a) {
 ; CHECK-LABEL: test_redor_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    orr w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v2i32:
@@ -584,9 +546,9 @@ define i32 @test_redor_v4i32(<4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    orr w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v4i32:
@@ -608,9 +570,9 @@ define i32 @test_redor_v8i32(<8 x i32> %a) {
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    orr w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    orr w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redor_v8i32:
diff --git a/llvm/test/CodeGen/AArch64/reduce-xor.ll b/llvm/test/CodeGen/AArch64/reduce-xor.ll
index 7bed71dffb93a..494399f808579 100644
--- a/llvm/test/CodeGen/AArch64/reduce-xor.ll
+++ b/llvm/test/CodeGen/AArch64/reduce-xor.ll
@@ -245,8 +245,16 @@ define i8 @test_redxor_v1i8(<1 x i8> %a) {
 define i8 @test_redxor_v3i8(<3 x i8> %a) {
 ; CHECK-LABEL: test_redxor_v3i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    eor w8, w0, w1
-; CHECK-NEXT:    eor w0, w8, w2
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.h[0], w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    lsr x10, x9, #32
+; CHECK-NEXT:    lsr x9, x9, #16
+; CHECK-NEXT:    eor w8, w8, w10
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v3i8:
@@ -261,14 +269,10 @@ define i8 @test_redxor_v3i8(<3 x i8> %a) {
 define i8 @test_redxor_v4i8(<4 x i8> %a) {
 ; CHECK-LABEL: test_redxor_v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[1]
-; CHECK-NEXT:    umov w11, v0.h[0]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    eor w10, w11, w10
-; CHECK-NEXT:    eor w0, w10, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v4i8:
@@ -292,22 +296,11 @@ define i8 @test_redxor_v4i8(<4 x i8> %a) {
 define i8 @test_redxor_v8i8(<8 x i8> %a) {
 ; CHECK-LABEL: test_redxor_v8i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.b[5]
-; CHECK-NEXT:    umov w9, v0.b[4]
-; CHECK-NEXT:    umov w10, v0.b[1]
-; CHECK-NEXT:    umov w11, v0.b[0]
-; CHECK-NEXT:    umov w12, v0.b[3]
-; CHECK-NEXT:    umov w13, v0.b[2]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    umov w15, v0.b[7]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    eor w10, w11, w10
-; CHECK-NEXT:    eor w11, w13, w12
-; CHECK-NEXT:    eor w9, w10, w11
-; CHECK-NEXT:    eor w8, w8, w14
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    eor w0, w8, w15
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    eor x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v8i8:
@@ -345,20 +338,10 @@ define i8 @test_redxor_v16i8(<16 x i8> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[7]
-; CHECK-NEXT:    eor w10, w10, w11
-; CHECK-NEXT:    eor w11, w12, w13
-; CHECK-NEXT:    eor w8, w8, w10
-; CHECK-NEXT:    eor w10, w11, w14
-; CHECK-NEXT:    eor w8, w8, w10
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    eor x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -399,20 +382,10 @@ define i8 @test_redxor_v32i8(<32 x i8> %a) {
 ; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.b[1]
-; CHECK-NEXT:    umov w9, v0.b[0]
-; CHECK-NEXT:    umov w10, v0.b[2]
-; CHECK-NEXT:    umov w11, v0.b[3]
-; CHECK-NEXT:    umov w12, v0.b[4]
-; CHECK-NEXT:    umov w13, v0.b[5]
-; CHECK-NEXT:    umov w14, v0.b[6]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    umov w9, v0.b[7]
-; CHECK-NEXT:    eor w10, w10, w11
-; CHECK-NEXT:    eor w11, w12, w13
-; CHECK-NEXT:    eor w8, w8, w10
-; CHECK-NEXT:    eor w10, w11, w14
-; CHECK-NEXT:    eor w8, w8, w10
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    eor x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -451,14 +424,10 @@ define i8 @test_redxor_v32i8(<32 x i8> %a) {
 define i16 @test_redxor_v4i16(<4 x i16> %a) {
 ; CHECK-LABEL: test_redxor_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[3]
-; CHECK-NEXT:    umov w9, v0.h[2]
-; CHECK-NEXT:    umov w10, v0.h[1]
-; CHECK-NEXT:    umov w11, v0.h[0]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    eor w10, w11, w10
-; CHECK-NEXT:    eor w0, w10, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v4i16:
@@ -484,12 +453,9 @@ define i16 @test_redxor_v8i16(<8 x i16> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    eor w9, w10, w11
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -518,12 +484,9 @@ define i16 @test_redxor_v16i16(<16 x i16> %a) {
 ; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    umov w9, v0.h[0]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    eor w8, w9, w8
-; CHECK-NEXT:    eor w9, w10, w11
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    eor x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
 ; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
@@ -550,10 +513,9 @@ define i16 @test_redxor_v16i16(<16 x i16> %a) {
 define i32 @test_redxor_v2i32(<2 x i32> %a) {
 ; CHECK-LABEL: test_redxor_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v2i32:
@@ -573,9 +535,9 @@ define i32 @test_redxor_v4i32(<4 x i32> %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v4i32:
@@ -597,9 +559,9 @@ define i32 @test_redxor_v8i32(<8 x i32> %a) {
 ; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    eor v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    eor w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    eor w0, w8, w9
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_redxor_v8i32:
diff --git a/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll b/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll
index cf7b9b173905c..e482833ffe452 100644
--- a/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll
+++ b/llvm/test/CodeGen/AArch64/setcc-type-mismatch.ll
@@ -1,9 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
 
 define void @test_mismatched_setcc(<4 x i22> %l, <4 x i22> %r, ptr %addr) {
 ; CHECK-LABEL: test_mismatched_setcc:
-; CHECK: cmeq [[CMP128:v[0-9]+]].4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
-; CHECK: xtn {{v[0-9]+}}.4h, [[CMP128]].4s
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.4s, #63, msl #16
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    ldr q3, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    ret
 
   %tst = icmp eq <4 x i22> %l, %r
   store <4 x i1> %tst, ptr %addr
diff --git a/llvm/test/CodeGen/AArch64/shrink-wrap-byval-inalloca-preallocated.ll b/llvm/test/CodeGen/AArch64/shrink-wrap-byval-inalloca-preallocated.ll
new file mode 100644
index 0000000000000..23f28b1e9ce1a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shrink-wrap-byval-inalloca-preallocated.ll
@@ -0,0 +1,206 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -o - %s | FileCheck %s
+
+target triple = "arm64-apple-ios"
+
+%struct.s = type {double, double }
+
+declare void @fn(ptr, ptr)
+
+; %l.a and %l.b read memory allocated in the caller and should not block
+; shrink-wrapping.
+define void @test_regular_pointers(ptr %a, ptr %b) {
+; CHECK-LABEL: test_regular_pointers:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    mov x8, #1 ; =0x1
+; CHECK-NEXT:    ldr d1, [x1, #8]
+; CHECK-NEXT:    movk x8, #2047, lsl #16
+; CHECK-NEXT:    fadd d0, d0, d1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcmp d0, d1
+; CHECK-NEXT:    str d0, [x1]
+; CHECK-NEXT:    b.mi LBB0_2
+; CHECK-NEXT:    b.gt LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %then
+; CHECK-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    mov x19, x1
+; CHECK-NEXT:    bl _fn
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT:    str xzr, [x19]
+; CHECK-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-NEXT:  LBB0_2: ; %exit
+; CHECK-NEXT:    .cfi_def_cfa wsp, 0
+; CHECK-NEXT:    .cfi_same_value w30
+; CHECK-NEXT:    .cfi_same_value w29
+; CHECK-NEXT:    .cfi_same_value w19
+; CHECK-NEXT:    .cfi_same_value w20
+; CHECK-NEXT:    ret
+entry:
+  %l.a = load double, ptr %a, align 8
+  %gep.b = getelementptr inbounds %struct.s, ptr %b, i64 0, i32 1
+  %l.b = load double, ptr %gep.b, align 8
+  %add = fadd double %l.a, %l.b
+  store double %add, ptr %b, align 8
+  %c = fcmp ueq double %add, 0x7FF0001
+  br i1 %c, label %then, label %exit
+
+then:
+  tail call void @fn(ptr %a, ptr %b)
+  store double 0.000000e+00, ptr %b, align 8
+  br label %exit
+
+exit:
+  ret void
+}
+
+; %l.b may read memory from the callee's stack due to byval.
+define void @test_byval_pointers(ptr %a, ptr byval(%struct.s) %b) {
+; CHECK-LABEL: test_byval_pointers:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    ldr d0, [sp, #40]
+; CHECK-NEXT:    mov x8, #1 ; =0x1
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    movk x8, #2047, lsl #16
+; CHECK-NEXT:    fadd d0, d1, d0
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcmp d0, d1
+; CHECK-NEXT:    str d0, [sp, #32]
+; CHECK-NEXT:    b.mi LBB1_2
+; CHECK-NEXT:    b.gt LBB1_2
+; CHECK-NEXT:  ; %bb.1: ; %then
+; CHECK-NEXT:    add x1, sp, #32
+; CHECK-NEXT:    add x19, sp, #32
+; CHECK-NEXT:    bl _fn
+; CHECK-NEXT:    str xzr, [x19]
+; CHECK-NEXT:  LBB1_2: ; %exit
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %l.a = load double, ptr %a, align 8
+  %gep.b = getelementptr inbounds %struct.s, ptr %b, i64 0, i32 1
+  %l.b = load double, ptr %gep.b, align 8
+  %add = fadd double %l.a, %l.b
+  store double %add, ptr %b, align 8
+  %c = fcmp ueq double %add, 0x7FF0001
+  br i1 %c, label %then, label %exit
+
+then:
+  tail call void @fn(ptr %a, ptr %b)
+  store double 0.000000e+00, ptr %b, align 8
+  br label %exit
+
+exit:
+  ret void
+}
+
+; %l.b may read memory from the callee's stack due to inalloca.
+define void @test_inalloca_pointers(ptr %a, ptr inalloca(%struct.s) %b) {
+; CHECK-LABEL: test_inalloca_pointers:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    ldr d0, [sp, #40]
+; CHECK-NEXT:    mov x8, #1 ; =0x1
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    movk x8, #2047, lsl #16
+; CHECK-NEXT:    fadd d0, d1, d0
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcmp d0, d1
+; CHECK-NEXT:    str d0, [sp, #32]
+; CHECK-NEXT:    b.mi LBB2_2
+; CHECK-NEXT:    b.gt LBB2_2
+; CHECK-NEXT:  ; %bb.1: ; %then
+; CHECK-NEXT:    add x1, sp, #32
+; CHECK-NEXT:    add x19, sp, #32
+; CHECK-NEXT:    bl _fn
+; CHECK-NEXT:    str xzr, [x19]
+; CHECK-NEXT:  LBB2_2: ; %exit
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %l.a = load double, ptr %a, align 8
+  %gep.b = getelementptr inbounds %struct.s, ptr %b, i64 0, i32 1
+  %l.b = load double, ptr %gep.b, align 8
+  %add = fadd double %l.a, %l.b
+  store double %add, ptr %b, align 8
+  %c = fcmp ueq double %add, 0x7FF0001
+  br i1 %c, label %then, label %exit
+
+then:
+  tail call void @fn(ptr %a, ptr %b)
+  store double 0.000000e+00, ptr %b, align 8
+  br label %exit
+
+exit:
+  ret void
+}
+
+; %l.b may read memory from the callee's stack due to preallocated.
+define void @test_preallocated_pointers(ptr %a, ptr preallocated(%struct.s) %b) {
+; CHECK-LABEL: test_preallocated_pointers:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    stp x20, x19, [sp, #-32]! ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    ldr d0, [sp, #40]
+; CHECK-NEXT:    mov x8, #1 ; =0x1
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    movk x8, #2047, lsl #16
+; CHECK-NEXT:    fadd d0, d1, d0
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fcmp d0, d1
+; CHECK-NEXT:    str d0, [sp, #32]
+; CHECK-NEXT:    b.mi LBB3_2
+; CHECK-NEXT:    b.gt LBB3_2
+; CHECK-NEXT:  ; %bb.1: ; %then
+; CHECK-NEXT:    add x1, sp, #32
+; CHECK-NEXT:    add x19, sp, #32
+; CHECK-NEXT:    bl _fn
+; CHECK-NEXT:    str xzr, [x19]
+; CHECK-NEXT:  LBB3_2: ; %exit
+; CHECK-NEXT:    ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp], #32 ; 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %l.a = load double, ptr %a, align 8
+  %gep.b = getelementptr inbounds %struct.s, ptr %b, i64 0, i32 1
+  %l.b = load double, ptr %gep.b, align 8
+  %add = fadd double %l.a, %l.b
+  store double %add, ptr %b, align 8
+  %c = fcmp ueq double %add, 0x7FF0001
+  br i1 %c, label %then, label %exit
+
+then:
+  tail call void @fn(ptr %a, ptr %b)
+  store double 0.000000e+00, ptr %b, align 8
+  br label %exit
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AArch64/shuffles.ll b/llvm/test/CodeGen/AArch64/shuffles.ll
index e5740fd33b1ed..9b908dc0be0a9 100644
--- a/llvm/test/CodeGen/AArch64/shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/shuffles.ll
@@ -127,3 +127,138 @@ define <4 x i32> @test0067(<4 x i32> %a, <4 x i32> %b)
   %r = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 0, i32 6, i32 7>
   ret <4 x i32> %r
 }
+
+define <4 x i32> @test_shuf6(<4 x i32> %a, <4 x i32> %b)
+; CHECK-LABEL: test_shuf6:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov v0.s[2], v1.s[3]
+; CHECK-NEXT:    trn1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 0, i32 7, i32 7>
+  ret <4 x i32> %r
+}
+
+define <4 x i16> @test_shuf7(<4 x i16> %a, <4 x i16> %b)
+; CHECK-LABEL: test_shuf7:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mov v0.h[2], v1.h[3]
+; CHECK-NEXT:    trn1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <4 x i16> %a, <4 x i16> %b, <4 x i32> <i32 0, i32 0, i32 7, i32 7>
+  ret <4 x i16> %r
+}
+
+define <8 x i8> @test_shuf8(<8 x i8> %a, <8 x i8> %b)
+; CHECK-LABEL: test_shuf8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI12_0
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI12_0]
+; CHECK-NEXT:    tbl v0.8b, { v0.16b }, v1.8b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x i8> %a, <8 x i8> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i8> %r
+}
+
+define <8 x i16> @test_shuf9(<8 x i16> %a, <8 x i16> %b)
+; CHECK-LABEL: test_shuf9:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI13_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI13_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @test_shuf10(<16 x i8> %a, <16 x i8> %b)
+; CHECK-LABEL: test_shuf10:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI14_0
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32     8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8 >
+  ret <16 x i8> %r
+}
+
+define <8 x half> @test_shuf11(<8 x half> %a, <8 x half> %b)
+; CHECK-LABEL: test_shuf11:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI15_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI15_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x half> %r
+}
+
+define <8 x half> @test_shuf12(<8 x half> %a, <8 x half> %b)
+; CHECK-LABEL: test_shuf12:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI16_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI16_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 1, i32 15>
+  ret <8 x half> %r
+}
+
+define <8 x half> @test_shuf13(<8 x half> %a, <8 x half> %b)
+; CHECK-LABEL: test_shuf13:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI17_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI17_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 1, i32 2, i32 0, i32 0, i32 0, i32 8, i32 1, i32 15>
+  ret <8 x half> %r
+}
+
+define <8 x half> @test_shuf14(<8 x half> %a, <8 x half> %b)
+; CHECK-LABEL: test_shuf14:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI18_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI18_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 1, i32 2, i32 1, i32 1, i32 0, i32 8, i32 1, i32 15>
+  ret <8 x half> %r
+}
+
+define <8 x half> @test_shuf15(<8 x half> %a, <8 x half> %b)
+; CHECK-LABEL: test_shuf15:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI19_0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI19_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
+; CHECK-NEXT:    ret
+{
+  %r = shufflevector <8 x half> %a, <8 x half> %b, <8 x i32> <i32 1, i32 2, i32 7, i32 2, i32 0, i32 3, i32 2, i32 15>
+  ret <8 x half> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-log-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-log-reduce.ll
index ed9b73c753148..61dbcbdb475db 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-log-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-log-reduce.ll
@@ -51,7 +51,7 @@ define i8 @andv_v32i8(ptr %a) vscale_range(2,0) #0 {
 define i8 @andv_v64i8(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: andv_v64i8:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov w8, #32
+; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
 ; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
 ; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x0]
@@ -140,7 +140,7 @@ define i16 @andv_v16i16(ptr %a) vscale_range(2,0) #0 {
 define i16 @andv_v32i16(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: andv_v32i16:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
@@ -229,7 +229,7 @@ define i32 @andv_v8i32(ptr %a) vscale_range(2,0) #0 {
 define i32 @andv_v16i32(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: andv_v16i32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
@@ -316,7 +316,7 @@ define i64 @andv_v4i64(ptr %a) vscale_range(2,0) #0 {
 define i64 @andv_v8i64(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: andv_v8i64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
@@ -409,7 +409,7 @@ define i8 @eorv_v32i8(ptr %a) vscale_range(2,0) #0 {
 define i8 @eorv_v64i8(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: eorv_v64i8:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov w8, #32
+; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
 ; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
 ; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x0]
@@ -498,7 +498,7 @@ define i16 @eorv_v16i16(ptr %a) vscale_range(2,0) #0 {
 define i16 @eorv_v32i16(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: eorv_v32i16:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
@@ -587,7 +587,7 @@ define i32 @eorv_v8i32(ptr %a) vscale_range(2,0) #0 {
 define i32 @eorv_v16i32(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: eorv_v16i32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
@@ -674,7 +674,7 @@ define i64 @eorv_v4i64(ptr %a) vscale_range(2,0) #0 {
 define i64 @eorv_v8i64(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: eorv_v8i64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
@@ -767,7 +767,7 @@ define i8 @orv_v32i8(ptr %a) vscale_range(2,0) #0 {
 define i8 @orv_v64i8(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: orv_v64i8:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov w8, #32
+; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
 ; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
 ; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x0]
@@ -856,7 +856,7 @@ define i16 @orv_v16i16(ptr %a) vscale_range(2,0) #0 {
 define i16 @orv_v32i16(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: orv_v32i16:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #16
+; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
@@ -945,7 +945,7 @@ define i32 @orv_v8i32(ptr %a) vscale_range(2,0) #0 {
 define i32 @orv_v16i32(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: orv_v16i32:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #8
+; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
@@ -1032,7 +1032,7 @@ define i64 @orv_v4i64(ptr %a) vscale_range(2,0) #0 {
 define i64 @orv_v8i64(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: orv_v8i64:
 ; VBITS_GE_256:       // %bb.0:
-; VBITS_GE_256-NEXT:    mov x8, #4
+; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
index 559b1982832bf..11b57ad319bea 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll
@@ -580,8 +580,6 @@ define void @masked_gather_v2f16(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    mov v0.h[0], v1.h[0]
 ; CHECK-NEXT:    mov w8, v1.s[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #15
-; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
index ff1c899b3d825..cb58eb5ebb891 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll
@@ -536,20 +536,18 @@ define void @masked_scatter_v2f16(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    ldr s1, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl4
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr q3, [x1]
 ; CHECK-NEXT:    fcmeq v2.4h, v1.4h, #0.0
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    sshll v2.4s, v2.4h, #0
 ; CHECK-NEXT:    mov v0.h[0], v2.h[0]
 ; CHECK-NEXT:    mov w8, v2.s[1]
-; CHECK-NEXT:    ldr q2, [x1]
 ; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #15
-; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
 ; CHECK-NEXT:    uunpklo z0.d, z1.s
-; CHECK-NEXT:    st1h { z0.d }, p0, [z2.d]
+; CHECK-NEXT:    st1h { z0.d }, p0, [z3.d]
 ; CHECK-NEXT:    ret
   %vals = load <2 x half>, ptr %a
   %ptrs = load <2 x ptr>, ptr %b
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll
index 05640e68b2366..60fe507f7882b 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll
@@ -21,8 +21,6 @@ define void @masked_store_v2f16(ptr %ap, ptr %bp) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    mov v0.h[0], v2.h[0]
 ; CHECK-NEXT:    mov w8, v2.s[1]
 ; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    shl v0.4h, v0.4h, #15
-; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
 ; CHECK-NEXT:    st1h { z1.h }, p0, [x1]
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-ptest.ll
index 3c0407a82938d..1304bb8bc6968 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-ptest.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-ptest.ll
@@ -17,8 +17,7 @@ define i1 @ptest_v16i1_256bit_min_sve(ptr %a, ptr %b) vscale_range(2, 0) {
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
 ; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
 ; CHECK-NEXT:    mov v1.d[1], v0.d[0]
-; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    orv b0, p0, z1.b
+; CHECK-NEXT:    umaxv b0, v1.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -35,10 +34,9 @@ define i1 @ptest_v16i1_512bit_min_sve(ptr %a, ptr %b) vscale_range(4, 0) {
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    orv b0, p0, z0.b
+; CHECK-NEXT:    umaxv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -55,10 +53,9 @@ define i1 @ptest_v16i1_512bit_sve(ptr %a, ptr %b) vscale_range(4, 4) {
 ; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    orv b0, p0, z0.b
+; CHECK-NEXT:    umaxv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -78,10 +75,9 @@ define i1 @ptest_or_v16i1_512bit_min_sve(ptr %a, ptr %b) vscale_range(4, 0) {
 ; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov p0.b, p1/m, p1.b
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    orv b0, p0, z0.b
+; CHECK-NEXT:    umaxv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -109,10 +105,9 @@ define i1 @ptest_and_v16i1_512bit_sve(ptr %a, ptr %b) vscale_range(4, 4) {
 ; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    andv b0, p0, z0.b
+; CHECK-NEXT:    uminv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -135,10 +130,9 @@ define i1 @ptest_and_v16i1_512bit_min_sve(ptr %a, ptr %b) vscale_range(4, 0) {
 ; CHECK-NEXT:    fcmne p0.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    and p0.b, p1/z, p1.b, p0.b
 ; CHECK-NEXT:    mov z0.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.b, vl16
 ; CHECK-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; CHECK-NEXT:    uzp1 z0.b, z0.b, z0.b
-; CHECK-NEXT:    andv b0, p0, z0.b
+; CHECK-NEXT:    uminv b0, v0.16b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
index ca5c1e25781d1..0841fbe1763c6 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
@@ -29,7 +29,7 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    ptrue p0.b, vl8
 ; CHECK-NEXT:    splice z1.b, p0, z1.b, z0.b
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    orv b0, p0, z1.b
+; CHECK-NEXT:    umaxv b0, p0, z1.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -86,7 +86,7 @@ define i1 @ptest_or_v16i1(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    splice z3.b, p0, z3.b, z2.b
 ; CHECK-NEXT:    orr z0.d, z1.d, z3.d
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    orv b0, p0, z0.b
+; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
@@ -153,7 +153,7 @@ define i1 @ptest_and_v16i1(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    splice z3.b, p0, z3.b, z2.b
 ; CHECK-NEXT:    and z0.d, z1.d, z3.d
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    andv b0, p0, z0.b
+; CHECK-NEXT:    uminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/tail-duplicate-debug-variance.mir b/llvm/test/CodeGen/AArch64/tail-duplicate-debug-variance.mir
new file mode 100644
index 0000000000000..8787d85aa3e44
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/tail-duplicate-debug-variance.mir
@@ -0,0 +1,95 @@
+# RUN: llc %s -o - -run-pass=early-tailduplication | FileCheck %s --check-prefixes=CHECK,DEBUG
+# RUN: llc %s -o - -run-pass=mir-strip-debug,early-tailduplication -mir-strip-debugify-only=0 | FileCheck %s --check-prefix=CHECK
+
+# We should get the same result with and without debug information.
+# Specifically, the register class of [[COPY]] should not be changed to
+# gpr32common with debug info while staying as gpr32sp without.
+
+--- |
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-none-linux-gnu"
+
+  define void @test() !dbg !8 {
+  entry:
+    ret void
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2, !3, !4, !5}
+  !llvm.ident = !{!6}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "My compiler", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "foo.c", directory: "/bar")
+  !2 = !{i32 7, !"Dwarf Version", i32 4}
+  !3 = !{i32 2, !"Debug Info Version", i32 3}
+  !4 = !{i32 1, !"wchar_size", i32 1}
+  !5 = !{i32 7, !"frame-pointer", i32 2}
+  !6 = !{!"My compiler"}
+  !7 = !DILocalVariable(name: "i", scope: !8, file: !1, line: 6, type: !12)
+  !8 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+  !9 = !DISubroutineType(types: !10)
+  !10 = !{null}
+  !11 = !{!7}
+  !12 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+  !13 = !DILocation(line: 0, scope: !8)
+...
+---
+name:            test
+tracksRegLiveness: true
+liveins:         []
+body:             |
+  ; CHECK-LABEL: name: test
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $w0, $wzr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32sp = COPY $w0
+  ; CHECK-NEXT:   CBNZW $wzr, %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $x0 = COPY undef $x0
+  ; DEBUG-NEXT:   DBG_VALUE [[COPY]], $noreg, !11, !DIExpression(), debug-location !DILocation(line: 0, scope: !7)
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32 = COPY [[COPY]]
+  ; CHECK-NEXT:   B %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; DEBUG-NEXT:   DBG_VALUE [[COPY]], $noreg, !11, !DIExpression(), debug-location !DILocation(line: 0, scope: !7)
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY [[COPY]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   RET undef $lr, implicit undef $w0
+  bb.0:
+    successors: %bb.2, %bb.1
+    liveins: $w0, $wzr
+
+    %1:gpr32sp = COPY $w0
+    CBNZW $wzr, %bb.2
+
+  bb.1:
+    successors: %bb.3
+
+    $x0 = COPY undef $x0
+    B %bb.3
+
+  bb.2:
+    successors: %bb.3
+
+  bb.3:
+    successors: %bb.5
+
+    %2:gpr32 = PHI %1, %bb.1, %1, %bb.2
+    DBG_VALUE %2, $noreg, !7, !DIExpression(), debug-location !13
+    B %bb.5
+
+  bb.4:
+    RET undef $lr, implicit undef $w0
+
+  bb.5:
+    successors: %bb.4
+
+    B %bb.4
+...
diff --git a/llvm/test/CodeGen/AArch64/tailcall-ccmismatch2.ll b/llvm/test/CodeGen/AArch64/tailcall-ccmismatch2.ll
new file mode 100644
index 0000000000000..1b264a09642c0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/tailcall-ccmismatch2.ll
@@ -0,0 +1,25 @@
+; RUN: llc -o - %s | FileCheck %s
+; RUN: llc -global-isel -verify-machineinstrs -o - %s | FileCheck %s
+target triple="aarch64--"
+
+declare void @somefunc()
+define preserve_allcc void @test_ccmismatch_notail() {
+; Ensure that no tail call is used here, as the called function somefunc does
+; not preserve enough registers for preserve_allcc.
+; CHECK-LABEL: test_ccmismatch_notail:
+; CHECK-NOT: b somefunc
+; CHECK: bl somefunc
+  tail call void @somefunc()
+  ret void
+}
+
+declare preserve_allcc void @some_preserve_all_func()
+define void @test_ccmismatch_tail() {
+; We can perform a tail call here, because some_preserve_all_func preserves
+; all registers necessary for test_ccmismatch_tail.
+; CHECK-LABEL: test_ccmismatch_tail:
+; CHECK-NOT: bl some_preserve_all_func
+; CHECK: b some_preserve_all_func
+  tail call preserve_allcc void @some_preserve_all_func()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/taildup-cfi.ll b/llvm/test/CodeGen/AArch64/taildup-cfi.ll
index 23b4891752d2c..221503009cdb6 100644
--- a/llvm/test/CodeGen/AArch64/taildup-cfi.ll
+++ b/llvm/test/CodeGen/AArch64/taildup-cfi.ll
@@ -32,7 +32,7 @@ if.then:                                          ; preds = %entry
   store i32 0, ptr @f, align 4, !tbaa !2
   br label %if.end
 
-; DARWIN:           Merging into block
+; DARWIN-NOT:           Merging into block
 ; LINUX:    	      Merging into block
 
 if.end:                                           ; preds = %entry.if.end_crit_edge, %if.then
diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
index f6f0b1d417883..f26f183630e9a 100644
--- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll
@@ -271,64 +271,66 @@ exit:
 ; CHECK-BE-NEXT:    	.byte	119                             // 0x77
 ; CHECK-BE-NEXT:    	.byte	127                             // 0x7f
 define void @trunc_v16i64_to_v16i8_in_loop(ptr %A, ptr %dst) {
-; CHECK-LABEL:  trunc_v16i64_to_v16i8_in_loop:
-; CHECK:    ; %bb.0:                                ; %entry
-; CHECK-NEXT:   Lloh4:
-; CHECK-NEXT:   	adrp	x9, lCPI3_0@PAGE
-; CHECK-NEXT:   	mov	x8, xzr
-; CHECK-NEXT:   Lloh5:
-; CHECK-NEXT:   	ldr	q0, [x9, lCPI3_0@PAGEOFF]
-; CHECK-NEXT:   LBB3_1:                                 ; %loop
-; CHECK-NEXT:                                           ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   	add	x9, x0, x8, lsl #7
-; CHECK-NEXT:   	ldp	q1, q2, [x9]
-; CHECK-NEXT:   	ldp	q3, q4, [x9, #32]
-; CHECK-NEXT:   	ldp	q16, q17, [x9, #64]
-; CHECK-NEXT:   	tbl.16b	v1, { v1, v2, v3, v4 }, v0
-; CHECK-NEXT:   	ldp	q18, q19, [x9, #96]
-; CHECK-NEXT:   	tbl.16b	v2, { v16, v17, v18, v19 }, v0
-; CHECK-NEXT:   	mov.d	v1[1], v2[0]
-; CHECK-NEXT:   	str	q1, [x1, x8, lsl  #4]
-; CHECK-NEXT:   	add	x8, x8, #1
-; CHECK-NEXT:   	cmp	x8, #1000
-; CHECK-NEXT:   	b.eq	LBB3_1
-; CHECK-NEXT:   ; %bb.2:                                ; %exit
-; CHECK-NEXT:   	ret
-
-; CHECK-BE-LABEL:  trunc_v16i64_to_v16i8_in_loop:
-; CHECK-BE:  // %bb.0:                               // %entry
-; CHECK-BE-NEXT:   	adrp	x8, .LCPI3_0
-; CHECK-BE-NEXT:   	add	x8, x8, :lo12:.LCPI3_0
-; CHECK-BE-NEXT:   	ld1	{ v0.16b }, [x8]
-; CHECK-BE-NEXT:   	mov	x8, xzr
-; CHECK-BE-NEXT:   .LBB3_1:                                // %loop
-; CHECK-BE-NEXT:                                           // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:   	add	x9, x0, x8, lsl #7
-; CHECK-BE-NEXT:   	add	x10, x9, #16
-; CHECK-BE-NEXT:   	add	x11, x9, #32
-; CHECK-BE-NEXT:   	ld1	{ v1.16b }, [x9]
-; CHECK-BE-NEXT:   	ld1	{ v2.16b }, [x10]
-; CHECK-BE-NEXT:   	add	x10, x9, #48
-; CHECK-BE-NEXT:   	ld1	{ v3.16b }, [x11]
-; CHECK-BE-NEXT:   	add	x11, x9, #64
-; CHECK-BE-NEXT:   	ld1	{ v4.16b }, [x10]
-; CHECK-BE-NEXT:   	add	x10, x9, #80
-; CHECK-BE-NEXT:   	ld1	{ v16.16b }, [x11]
-; CHECK-BE-NEXT:   	add	x11, x9, #96
-; CHECK-BE-NEXT:   	add	x9, x9, #112
-; CHECK-BE-NEXT:   	ld1	{ v17.16b }, [x10]
-; CHECK-BE-NEXT:   	tbl	v1.16b, { v1.16b, v2.16b, v3.16b, v4.16b }, v0.16b
-; CHECK-BE-NEXT:   	ld1	{ v18.16b }, [x11]
-; CHECK-BE-NEXT:   	ld1	{ v19.16b }, [x9]
-; CHECK-BE-NEXT:   	add	x9, x1, x8, lsl #4
-; CHECK-BE-NEXT:   	add	x8, x8, #1
-; CHECK-BE-NEXT:   	cmp	x8, #1000
-; CHECK-BE-NEXT:   	tbl	v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v0.16b
-; CHECK-BE-NEXT:   	mov	v1.d[1], v2.d[0]
-; CHECK-BE-NEXT:   	st1	{ v1.16b }, [x9]
-; CHECK-BE-NEXT:   	b.eq	.LBB3_1
-; CHECK-BE-NEXT:   // %bb.2:                               // %exit
-; CHECK-BE-NEXT:   	ret
+; CHECK-LABEL: trunc_v16i64_to_v16i8_in_loop:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:  Lloh4:
+; CHECK-NEXT:    adrp x9, lCPI3_0@PAGE
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  Lloh5:
+; CHECK-NEXT:    ldr q0, [x9, lCPI3_0@PAGEOFF]
+; CHECK-NEXT:  LBB3_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8, lsl #7
+; CHECK-NEXT:    ldp q1, q2, [x9]
+; CHECK-NEXT:    ldp q3, q4, [x9, #32]
+; CHECK-NEXT:    ldp q16, q17, [x9, #64]
+; CHECK-NEXT:    tbl.16b v1, { v1, v2, v3, v4 }, v0
+; CHECK-NEXT:    ldp q18, q19, [x9, #96]
+; CHECK-NEXT:    tbl.16b v2, { v16, v17, v18, v19 }, v0
+; CHECK-NEXT:    mov.d v1[1], v2[0]
+; CHECK-NEXT:    str q1, [x1, x8, lsl #4]
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    b.eq LBB3_1
+; CHECK-NEXT:  ; %bb.2: ; %exit
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .loh AdrpLdr Lloh4, Lloh5
+;
+; CHECK-BE-LABEL: trunc_v16i64_to_v16i8_in_loop:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    adrp x8, .LCPI3_0
+; CHECK-BE-NEXT:    add x8, x8, :lo12:.LCPI3_0
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x8]
+; CHECK-BE-NEXT:    mov x8, xzr
+; CHECK-BE-NEXT:  .LBB3_1: // %loop
+; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    add x9, x0, x8, lsl #7
+; CHECK-BE-NEXT:    add x10, x9, #16
+; CHECK-BE-NEXT:    add x11, x9, #32
+; CHECK-BE-NEXT:    ld1 { v1.16b }, [x9]
+; CHECK-BE-NEXT:    ld1 { v2.16b }, [x10]
+; CHECK-BE-NEXT:    add x10, x9, #48
+; CHECK-BE-NEXT:    ld1 { v3.16b }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #64
+; CHECK-BE-NEXT:    ld1 { v4.16b }, [x10]
+; CHECK-BE-NEXT:    add x10, x9, #80
+; CHECK-BE-NEXT:    ld1 { v16.16b }, [x11]
+; CHECK-BE-NEXT:    add x11, x9, #96
+; CHECK-BE-NEXT:    add x9, x9, #112
+; CHECK-BE-NEXT:    ld1 { v17.16b }, [x10]
+; CHECK-BE-NEXT:    tbl v1.16b, { v1.16b, v2.16b, v3.16b, v4.16b }, v0.16b
+; CHECK-BE-NEXT:    ld1 { v18.16b }, [x11]
+; CHECK-BE-NEXT:    ld1 { v19.16b }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, x8, lsl #4
+; CHECK-BE-NEXT:    add x8, x8, #1
+; CHECK-BE-NEXT:    cmp x8, #1000
+; CHECK-BE-NEXT:    tbl v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v0.16b
+; CHECK-BE-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-BE-NEXT:    st1 { v1.16b }, [x9]
+; CHECK-BE-NEXT:    b.eq .LBB3_1
+; CHECK-BE-NEXT:  // %bb.2: // %exit
+; CHECK-BE-NEXT:    ret
+
 
 entry:
   br label %loop
@@ -384,51 +386,52 @@ exit:
 ; CHECK-BE-NEXT:   	.byte	255                             // 0xff
 ; CHECK-BE-NEXT:   	.byte	255                             // 0xff
 define void @trunc_v8i64_to_v8i8_in_loop(ptr %A, ptr %dst) {
-; CHECK-LABEL:	trunc_v8i64_to_v8i8_in_loop:
-; CHECK:    ; %bb.0:                                ; %entry
-; CHECK-NEXT:   Lloh6:
-; CHECK-NEXT:   	adrp	x9, lCPI4_0@PAGE
-; CHECK-NEXT:   	mov	x8, xzr
-; CHECK-NEXT:   Lloh7:
-; CHECK-NEXT:   	ldr	q0, [x9, lCPI4_0@PAGEOFF]
-; CHECK-NEXT:   LBB4_1:                                 ; %loop
-; CHECK-NEXT:                                           ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   	add	x9, x0, x8, lsl #6
-; CHECK-NEXT:   	ldp	q1, q2, [x9]
-; CHECK-NEXT:   	ldp	q3, q4, [x9, #32]
-; CHECK-NEXT:   	tbl.16b	v1, { v1, v2, v3, v4 }, v0
-; CHECK-NEXT:   	str	d1, [x1, x8, lsl  #3]
-; CHECK-NEXT:   	add	x8, x8, #1
-; CHECK-NEXT:   	cmp	x8, #1000
-; CHECK-NEXT:   	b.eq	LBB4_1
-; CHECK-NEXT:   ; %bb.2:                                ; %exit
-; CHECK-NEXT:   	ret
-; CHECK-NEXT:   	.loh AdrpLdr	Lloh6, Lloh7
-
-; CHECK-BE-LABEL:  trunc_v8i64_to_v8i8_in_loop:
-; CHECK-BE:  // %bb.0:                               // %entry
-; CHECK-BE-NEXT:   	adrp	x8, .LCPI4_0
-; CHECK-BE-NEXT:   	add	x8, x8, :lo12:.LCPI4_0
-; CHECK-BE-NEXT:   	ld1	{ v0.16b }, [x8]
-; CHECK-BE-NEXT:   	mov	x8, xzr
-; CHECK-BE-NEXT:  .LBB4_1:                                // %loop
-; CHECK-BE-NEXT:                                           // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:   	add	x9, x0, x8, lsl #6
-; CHECK-BE-NEXT:   	add	x10, x9, #16
-; CHECK-BE-NEXT:   	add	x11, x9, #32
-; CHECK-BE-NEXT:   	ld1	{ v1.16b }, [x9]
-; CHECK-BE-NEXT:   	add	x9, x9, #48
-; CHECK-BE-NEXT:   	ld1	{ v2.16b }, [x10]
-; CHECK-BE-NEXT:   	ld1	{ v3.16b }, [x11]
-; CHECK-BE-NEXT:   	ld1	{ v4.16b }, [x9]
-; CHECK-BE-NEXT:   	add	x9, x1, x8, lsl #3
-; CHECK-BE-NEXT:   	add	x8, x8, #1
-; CHECK-BE-NEXT:   	cmp	x8, #1000
-; CHECK-BE-NEXT:   	tbl	v1.16b, { v1.16b, v2.16b, v3.16b, v4.16b }, v0.16b
-; CHECK-BE-NEXT:   	st1	{ v1.8b }, [x9]
-; CHECK-BE-NEXT:   	b.eq	.LBB4_1
-; CHECK-BE-NEXT:   // %bb.2:                               // %exit
-; CHECK-BE-NEXT:   	ret
+; CHECK-LABEL: trunc_v8i64_to_v8i8_in_loop:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:  Lloh6:
+; CHECK-NEXT:    adrp x9, lCPI4_0@PAGE
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  Lloh7:
+; CHECK-NEXT:    ldr q0, [x9, lCPI4_0@PAGEOFF]
+; CHECK-NEXT:  LBB4_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8, lsl #6
+; CHECK-NEXT:    ldp q1, q2, [x9]
+; CHECK-NEXT:    ldp q3, q4, [x9, #32]
+; CHECK-NEXT:    tbl.16b v1, { v1, v2, v3, v4 }, v0
+; CHECK-NEXT:    str d1, [x1, x8, lsl #3]
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    b.eq LBB4_1
+; CHECK-NEXT:  ; %bb.2: ; %exit
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    .loh AdrpLdr Lloh6, Lloh7
+;
+; CHECK-BE-LABEL: trunc_v8i64_to_v8i8_in_loop:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    adrp x8, .LCPI4_0
+; CHECK-BE-NEXT:    add x8, x8, :lo12:.LCPI4_0
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x8]
+; CHECK-BE-NEXT:    mov x8, xzr
+; CHECK-BE-NEXT:  .LBB4_1: // %loop
+; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    add x9, x0, x8, lsl #6
+; CHECK-BE-NEXT:    add x10, x9, #16
+; CHECK-BE-NEXT:    add x11, x9, #32
+; CHECK-BE-NEXT:    ld1 { v1.16b }, [x9]
+; CHECK-BE-NEXT:    add x9, x9, #48
+; CHECK-BE-NEXT:    ld1 { v2.16b }, [x10]
+; CHECK-BE-NEXT:    ld1 { v3.16b }, [x11]
+; CHECK-BE-NEXT:    ld1 { v4.16b }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, x8, lsl #3
+; CHECK-BE-NEXT:    add x8, x8, #1
+; CHECK-BE-NEXT:    cmp x8, #1000
+; CHECK-BE-NEXT:    tbl v1.16b, { v1.16b, v2.16b, v3.16b, v4.16b }, v0.16b
+; CHECK-BE-NEXT:    st1 { v1.8b }, [x9]
+; CHECK-BE-NEXT:    b.eq .LBB4_1
+; CHECK-BE-NEXT:  // %bb.2: // %exit
+; CHECK-BE-NEXT:    ret
+
 
 entry:
   br label %loop
@@ -548,7 +551,7 @@ exit:
 define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ; CHECK-LABEL: trunc_v11i64_to_v11i8_in_loop:
 ; CHECK:       ; %bb.0: ; %entry
-; CHECK-NEXT:    mov w8, #1000
+; CHECK-NEXT:    mov w8, #1000 ; =0x3e8
 ; CHECK-NEXT:  LBB6_1: ; %loop
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
@@ -575,7 +578,7 @@ define void @trunc_v11i64_to_v11i8_in_loop(ptr %A, ptr %dst) {
 ;
 ; CHECK-BE-LABEL: trunc_v11i64_to_v11i8_in_loop:
 ; CHECK-BE:       // %bb.0: // %entry
-; CHECK-BE-NEXT:    mov w8, #1000
+; CHECK-BE-NEXT:    mov w8, #1000 // =0x3e8
 ; CHECK-BE-NEXT:  .LBB6_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    add x9, x0, #48
@@ -627,39 +630,40 @@ exit:
 }
 
 define void @trunc_v16i16_to_v16i8_in_loop(ptr %A, ptr %dst) {
-; CHECK-LABEL:  trunc_v16i16_to_v16i8_in_loop:
-; CHECK:    ; %bb.0:                                ; %entry
-; CHECK-NEXT:   	mov	x8, xzr
-; CHECK-NEXT:   LBB7_1:                                 ; %loop
-; CHECK-NEXT:                                           ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   	add	x9, x0, x8, lsl #5
-; CHECK-NEXT:   	ldp	q1, q0, [x9]
-; CHECK-NEXT:   	uzp1.16b	v0, v1, v0
-; CHECK-NEXT:   	str	q0, [x1, x8, lsl  #4]
-; CHECK-NEXT:   	add	x8, x8, #1
-; CHECK-NEXT:   	cmp	x8, #1000
-; CHECK-NEXT:   	b.eq	LBB7_1
-; CHECK-NEXT:   ; %bb.2:                                ; %exit
-; CHECK-NEXT:   	ret
-
-
-; CHECK-BE-LABEL:  trunc_v16i16_to_v16i8_in_loop:
-; CHECK-BE:  // %bb.0:                               // %entry
-; CHECK-BE-NEXT:    	mov	x8, xzr
-; CHECK-BE-NEXT:    .LBB7_1:                                // %loop
-; CHECK-BE-NEXT:                                            // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    	add	x9, x0, x8, lsl #5
-; CHECK-BE-NEXT:    	add	x10, x9, #16
-; CHECK-BE-NEXT:    	ld1	{ v0.8h }, [x9]
-; CHECK-BE-NEXT:    	add	x9, x1, x8, lsl #4
-; CHECK-BE-NEXT:    	add	x8, x8, #1
-; CHECK-BE-NEXT:    	ld1	{ v1.8h }, [x10]
-; CHECK-BE-NEXT:    	cmp	x8, #1000
-; CHECK-BE-NEXT:    	uzp1	v0.16b, v0.16b, v1.16b
-; CHECK-BE-NEXT:    	st1	{ v0.16b }, [x9]
-; CHECK-BE-NEXT:    	b.eq	.LBB7_1
-; CHECK-BE-NEXT:    // %bb.2:                               // %exit
-; CHECK-BE-NEXT:    	ret
+; CHECK-LABEL: trunc_v16i16_to_v16i8_in_loop:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  LBB7_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add x9, x0, x8, lsl #5
+; CHECK-NEXT:    ldp q1, q0, [x9]
+; CHECK-NEXT:    uzp1.16b v0, v1, v0
+; CHECK-NEXT:    str q0, [x1, x8, lsl #4]
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    b.eq LBB7_1
+; CHECK-NEXT:  ; %bb.2: ; %exit
+; CHECK-NEXT:    ret
+;
+; CHECK-BE-LABEL: trunc_v16i16_to_v16i8_in_loop:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    mov x8, xzr
+; CHECK-BE-NEXT:  .LBB7_1: // %loop
+; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    add x9, x0, x8, lsl #5
+; CHECK-BE-NEXT:    add x10, x9, #16
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, x8, lsl #4
+; CHECK-BE-NEXT:    add x8, x8, #1
+; CHECK-BE-NEXT:    ld1 { v1.8h }, [x10]
+; CHECK-BE-NEXT:    cmp x8, #1000
+; CHECK-BE-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x9]
+; CHECK-BE-NEXT:    b.eq .LBB7_1
+; CHECK-BE-NEXT:  // %bb.2: // %exit
+; CHECK-BE-NEXT:    ret
+
+
 
 
 entry:
@@ -681,36 +685,37 @@ exit:
 }
 
 define void @trunc_v8i16_to_v8i8_in_loop(ptr %A, ptr %dst) {
-; CHECK-LABEL:	trunc_v8i16_to_v8i8_in_loop:
-; CHECK:		; %bb.0:                                ; %entry
-; CHECK-NEXT:   	mov	x8, xzr
-; CHECK-NEXT:   LBB8_1:                                 ; %loop
-; CHECK-NEXT:                                           ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   	ldr	q0, [x0, x8, lsl  #4]
-; CHECK-NEXT:   	xtn.8b	v0, v0
-; CHECK-NEXT:   	str	d0, [x1, x8, lsl  #3]
-; CHECK-NEXT:   	add	x8, x8, #1
-; CHECK-NEXT:   	cmp	x8, #1000
-; CHECK-NEXT:   	b.eq	LBB8_1
-; CHECK-NEXT:   ; %bb.2:                                ; %exit
-; CHECK-NEXT:   	ret
-
-
-; CHECK-BE-LABEL:  trunc_v8i16_to_v8i8_in_loop:
-; CHECK-BE:  // %bb.0:                               // %entry
-; CHECK-BE-NEXT:    	mov	x8, xzr
-; CHECK-BE-NEXT:    .LBB8_1:                                // %loop
-; CHECK-BE-NEXT:                                            // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    	add	x9, x0, x8, lsl #4
-; CHECK-BE-NEXT:    	ld1	{ v0.8h }, [x9]
-; CHECK-BE-NEXT:    	add	x9, x1, x8, lsl #3
-; CHECK-BE-NEXT:    	add	x8, x8, #1
-; CHECK-BE-NEXT:    	cmp	x8, #1000
-; CHECK-BE-NEXT:    	xtn	v0.8b, v0.8h
-; CHECK-BE-NEXT:    	st1	{ v0.8b }, [x9]
-; CHECK-BE-NEXT:    	b.eq	.LBB8_1
-; CHECK-BE-NEXT:    // %bb.2:                               // %exit
-; CHECK-BE-NEXT:    	ret
+; CHECK-LABEL: trunc_v8i16_to_v8i8_in_loop:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:  LBB8_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr q0, [x0, x8, lsl #4]
+; CHECK-NEXT:    xtn.8b v0, v0
+; CHECK-NEXT:    str d0, [x1, x8, lsl #3]
+; CHECK-NEXT:    add x8, x8, #1
+; CHECK-NEXT:    cmp x8, #1000
+; CHECK-NEXT:    b.eq LBB8_1
+; CHECK-NEXT:  ; %bb.2: ; %exit
+; CHECK-NEXT:    ret
+;
+; CHECK-BE-LABEL: trunc_v8i16_to_v8i8_in_loop:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    mov x8, xzr
+; CHECK-BE-NEXT:  .LBB8_1: // %loop
+; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-BE-NEXT:    add x9, x0, x8, lsl #4
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x9]
+; CHECK-BE-NEXT:    add x9, x1, x8, lsl #3
+; CHECK-BE-NEXT:    add x8, x8, #1
+; CHECK-BE-NEXT:    cmp x8, #1000
+; CHECK-BE-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x9]
+; CHECK-BE-NEXT:    b.eq .LBB8_1
+; CHECK-BE-NEXT:  // %bb.2: // %exit
+; CHECK-BE-NEXT:    ret
+
+
 
 
 entry:
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
index e1daead54c6ff..49380820beb3e 100644
--- a/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll
@@ -418,18 +418,59 @@ define i4 @convert_to_bitmask_float(<4 x float> %vec) {
   ret i4 %bitmask
 }
 
-; TODO(lawben): Change this in follow-up patch to #D145301, as truncating stores fix this.
-; Larger vector types don't map directly.
-define i8 @no_convert_large_vector(<8 x i32> %vec) {
+; Larger vector types don't map directly, but the can be split/truncated and then converted.
+; After the comparison against 0, this is truncated to <8 x i16>, which is valid again.
+define i8 @convert_large_vector(<8 x i32> %vec) {
+; CHECK-LABEL: lCPI15_0:
+; CHECK-NEXT:  .short	1
+; CHECK-NEXT:  .short	2
+; CHECK-NEXT:  .short	4
+; CHECK-NEXT:  .short	8
+; CHECK-NEXT:  .short	16
+; CHECK-NEXT:  .short	32
+; CHECK-NEXT:  .short	64
+; CHECK-NEXT:  .short	128
+
 ; CHECK-LABEL: convert_large_vector:
-; CHECK:       cmeq.4s	v1, v1, #0
-; CHECK-NOT:   addv
+; CHECK:      Lloh30:
+; CHECK-NEXT:  adrp	x8, lCPI15_0@PAGE
+; CHECK-NEXT:  cmeq.4s	v1, v1, #0
+; CHECK-NEXT:  cmeq.4s	v0, v0, #0
+; CHECK-NEXT:  uzp1.8h	v0, v0, v1
+; CHECK-NEXT: Lloh31:
+; CHECK-NEXT:  ldr	q1, [x8, lCPI15_0@PAGEOFF]
+; CHECK-NEXT:  bic.16b	v0, v1, v0
+; CHECK-NEXT:  addv.8h	h0, v0
+; CHECK-NEXT:  fmov	w8, s0
+; CHECK-NEXT:  and	w0, w8, #0xff
+; CHECK-NEXT:  add	sp, sp, #16
+; CHECK-NEXT:  ret
 
    %cmp_result = icmp ne <8 x i32> %vec, zeroinitializer
    %bitmask = bitcast <8 x i1> %cmp_result to i8
    ret i8 %bitmask
 }
 
+define i4 @convert_legalized_illegal_element_size(<4 x i22> %vec) {
+; CHECK-LABEL: convert_legalized_illegal_element_size
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  movi.4s  v1, #63, msl #16
+; CHECK-NEXT:  Lloh32:
+; CHECK-NEXT:  adrp	    x8, lCPI16_0@PAGE
+; CHECK-NEXT:  cmtst.4s v0, v0, v1
+; CHECK-NEXT:  Lloh33:
+; CHECK-NEXT:  ldr	    d1, [x8, lCPI16_0@PAGEOFF]
+; CHECK-NEXT:  xtn.4h   v0, v0
+; CHECK-NEXT:  and.8b   v0, v0, v1
+; CHECK-NEXT:  addv.4h  h0, v0
+; CHECK-NEXT:  fmov     w0, s0
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <4 x i22> %vec, zeroinitializer
+  %bitmask = bitcast <4 x i1> %cmp_result to i4
+  ret i4 %bitmask
+}
+
 ; This may still be converted as a v8i8 after the vector concat (but not as v4iX).
 define i8 @no_direct_convert_for_bad_concat(<4 x i32> %vec) {
 ; CHECK-LABEL: no_direct_convert_for_bad_concat:
@@ -450,3 +491,12 @@ define <8 x i1> @no_convert_without_direct_bitcast(<8 x i16> %vec) {
    %cmp_result = icmp ne <8 x i16> %vec, zeroinitializer
    ret <8 x i1> %cmp_result
 }
+
+define i6 @no_combine_illegal_num_elements(<6 x i32> %vec) {
+; CHECK-LABEL: no_combine_illegal_num_elements
+; CHECK-NOT: addv
+
+  %cmp_result = icmp ne <6 x i32> %vec, zeroinitializer
+  %bitmask = bitcast <6 x i1> %cmp_result to i6
+  ret i6 %bitmask
+}
diff --git a/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
new file mode 100644
index 0000000000000..dc878090d3214
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll
@@ -0,0 +1,281 @@
+; RUN: llc -mtriple=aarch64-apple-darwin -mattr=+neon -verify-machineinstrs < %s | FileCheck %s
+
+define void @store_16_elements(<16 x i8> %vec, ptr %out) {
+; Bits used in mask
+; CHECK-LABEL: lCPI0_0
+; CHECK-NEXT: .byte	1
+; CHECK-NEXT: .byte	2
+; CHECK-NEXT: .byte	4
+; CHECK-NEXT: .byte	8
+; CHECK-NEXT: .byte	16
+; CHECK-NEXT: .byte	32
+; CHECK-NEXT: .byte	64
+; CHECK-NEXT: .byte	128
+; CHECK-NEXT: .byte	1
+; CHECK-NEXT: .byte	2
+; CHECK-NEXT: .byte	4
+; CHECK-NEXT: .byte	8
+; CHECK-NEXT: .byte	16
+; CHECK-NEXT: .byte	32
+; CHECK-NEXT: .byte	64
+; CHECK-NEXT: .byte	128
+
+; Actual conversion
+; CHECK-LABEL: store_16_elements
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh0:
+; CHECK-NEXT:  adrp	    x8, lCPI0_0@PAGE
+; CHECK-NEXT:  cmeq.16b	v0, v0, #0
+; CHECK-NEXT:  Lloh1:
+; CHECK-NEXT:  ldr	    q1, [x8, lCPI0_0@PAGEOFF]
+; CHECK-NEXT:  bic.16b	v0, v1, v0
+; CHECK-NEXT:  ext.16b	v1, v0, v0, #8
+; CHECK-NEXT:  addv.8b	b0, v0
+; CHECK-NEXT:  addv.8b	b1, v1
+; CHECK-NEXT:  fmov	    w9, s0
+; CHECK-NEXT:  fmov	    w8, s1
+; CHECK-NEXT:  orr	    w8, w9, w8, lsl #8
+; CHECK-NEXT:  strh	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <16 x i8> %vec, zeroinitializer
+  store <16 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @store_8_elements(<8 x i16> %vec, ptr %out) {
+; CHECK-LABEL: lCPI1_0:
+; CHECK-NEXT: .short	1
+; CHECK-NEXT: .short	2
+; CHECK-NEXT: .short	4
+; CHECK-NEXT: .short	8
+; CHECK-NEXT: .short	16
+; CHECK-NEXT: .short	32
+; CHECK-NEXT: .short	64
+; CHECK-NEXT: .short	128
+
+; CHECK-LABEL: store_8_elements
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh2:
+; CHECK-NEXT:  adrp	    x8, lCPI1_0@PAGE
+; CHECK-NEXT:  cmeq.8h	v0, v0, #0
+; CHECK-NEXT:  Lloh3:
+; CHECK-NEXT:  ldr	    q1, [x8, lCPI1_0@PAGEOFF]
+; CHECK-NEXT:  bic.16b	v0, v1, v0
+; CHECK-NEXT:  addv.8h	h0, v0
+; CHECK-NEXT:  fmov	    w8, s0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <8 x i16> %vec, zeroinitializer
+  store <8 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @store_4_elements(<4 x i32> %vec, ptr %out) {
+; CHECK-LABEL: lCPI2_0:
+; CHECK-NEXT:  .long	1
+; CHECK-NEXT:  .long	2
+; CHECK-NEXT:  .long	4
+; CHECK-NEXT:  .long	8
+
+; CHECK-LABEL: store_4_elements
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh4:
+; CHECK-NEXT:  adrp	x8, lCPI2_0@PAGE
+; CHECK-NEXT:  cmeq.4s	v0, v0, #0
+; CHECK-NEXT:  Lloh5:
+; CHECK-NEXT:  ldr	    q1, [x8, lCPI2_0@PAGEOFF]
+; CHECK-NEXT:  bic.16b	v0, v1, v0
+; CHECK-NEXT:  addv.4s	s0, v0
+; CHECK-NEXT:  fmov	    w8, s0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <4 x i32> %vec, zeroinitializer
+  store <4 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @store_2_elements(<2 x i64> %vec, ptr %out) {
+; CHECK-LABEL: lCPI3_0:
+; CHECK-NEXT: .quad	1
+; CHECK-NEXT: .quad	2
+
+; CHECK-LABEL: store_2_elements
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh6:
+; CHECK-NEXT:  adrp	x8, lCPI3_0@PAGE
+; CHECK-NEXT:  cmeq.2d	v0, v0, #0
+; CHECK-NEXT:  Lloh7:
+; CHECK-NEXT:  ldr	    q1, [x8, lCPI3_0@PAGEOFF]
+; CHECK-NEXT:  bic.16b	v0, v1, v0
+; CHECK-NEXT:  addp.2d	d0, v0
+; CHECK-NEXT:  fmov	    x8, d0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <2 x i64> %vec, zeroinitializer
+  store <2 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @add_trunc_compare_before_store(<4 x i32> %vec, ptr %out) {
+; CHECK-LABEL: lCPI4_0:
+; CHECK-NEXT:  .long	1
+; CHECK-NEXT:  .long	2
+; CHECK-NEXT:  .long	4
+; CHECK-NEXT:  .long	8
+
+; CHECK-LABEL: add_trunc_compare_before_store
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh8:
+; CHECK-NEXT:  adrp	    x8, lCPI4_0@PAGE
+; CHECK-NEXT:  shl.4s	v0, v0, #31
+; CHECK-NEXT:  cmlt.4s	v0, v0, #0
+; CHECK-NEXT:  Lloh9:
+; CHECK-NEXT:  ldr	    q1, [x8, lCPI4_0@PAGEOFF]
+; CHECK-NEXT:  and.16b	v0, v0, v1
+; CHECK-NEXT:  addv.4s	s0, v0
+; CHECK-NEXT:  fmov 	w8, s0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %trunc = trunc <4 x i32> %vec to <4 x i1>
+  store <4 x i1> %trunc, ptr %out
+  ret void
+}
+
+define void @add_trunc_mask_unknown_vector_type(<4 x i1> %vec, ptr %out) {
+; CHECK-LABEL: lCPI5_0:
+; CHECK: .short	1
+; CHECK: .short	2
+; CHECK: .short	4
+; CHECK: .short	8
+
+; CHECK-LABEL: add_trunc_mask_unknown_vector_type
+; CHECK:       ; %bb.0:
+; CHECK-NEXT: Lloh10:
+; CHECK-NEXT:  adrp	    x8, lCPI5_0@PAGE
+; CHECK-NEXT:  shl.4h	v0, v0, #15
+; CHECK-NEXT:  cmlt.4h	v0, v0, #0
+; CHECK-NEXT: Lloh11:
+; CHECK-NEXT:  ldr	    d1, [x8, lCPI5_0@PAGEOFF]
+; CHECK-NEXT:  and.8b    v0, v0, v1
+; CHECK-NEXT:  addv.4h	h0, v0
+; CHECK-NEXT:  fmov	    w8, s0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  store <4 x i1> %vec, ptr %out
+  ret void
+}
+
+define void @store_8_elements_64_bit_vector(<8 x i8> %vec, ptr %out) {
+; CHECK-LABEL: lCPI6_0:
+; CHECK-NEXT:  .byte	1
+; CHECK-NEXT:  .byte	2
+; CHECK-NEXT:  .byte	4
+; CHECK-NEXT:  .byte	8
+; CHECK-NEXT:  .byte	16
+; CHECK-NEXT:  .byte	32
+; CHECK-NEXT:  .byte	64
+; CHECK-NEXT:  .byte	128
+
+; CHECK-LABEL: store_8_elements_64_bit_vector
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh12:
+; CHECK-NEXT:  adrp	x8, lCPI6_0@PAGE
+; CHECK-NEXT:  cmeq.8b	v0, v0, #0
+; CHECK-NEXT:  Lloh13:
+; CHECK-NEXT:  ldr	    d1, [x8, lCPI6_0@PAGEOFF]
+; CHECK-NEXT:  bic.8b	v0, v1, v0
+; CHECK-NEXT:  addv.8b	b0, v0
+; CHECK-NEXT:  st1.b	{ v0 }[0], [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <8 x i8> %vec, zeroinitializer
+  store <8 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @store_4_elements_64_bit_vector(<4 x i16> %vec, ptr %out) {
+; CHECK-LABEL: lCPI7_0:
+; CHECK-NEXT:  .short	1
+; CHECK-NEXT:  .short	2
+; CHECK-NEXT:  .short	4
+; CHECK-NEXT:  .short	8
+
+; CHECK-LABEL: store_4_elements_64_bit_vector
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh14:
+; CHECK-NEXT:  adrp	x8, lCPI7_0@PAGE
+; CHECK-NEXT:  cmeq.4h	v0, v0, #0
+; CHECK-NEXT:  Lloh15:
+; CHECK-NEXT:  ldr	    d1, [x8, lCPI7_0@PAGEOFF]
+; CHECK-NEXT:  bic.8b	v0, v1, v0
+; CHECK-NEXT:  addv.4h	h0, v0
+; CHECK-NEXT:  fmov	    w8, s0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <4 x i16> %vec, zeroinitializer
+  store <4 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @store_2_elements_64_bit_vector(<2 x i32> %vec, ptr %out) {
+; CHECK-LABEL: lCPI8_0:
+; CHECK-NEXT:  .long	1
+; CHECK-NEXT:  .long	2
+
+; CHECK-LABEL: store_2_elements_64_bit_vector
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:  Lloh16:
+; CHECK-NEXT:  adrp	x8, lCPI8_0@PAGE
+; CHECK-NEXT:  cmeq.2s	v0, v0, #0
+; CHECK-NEXT:  Lloh17:
+; CHECK-NEXT:  ldr	    d1, [x8, lCPI8_0@PAGEOFF]
+; CHECK-NEXT:  bic.8b	v0, v1, v0
+; CHECK-NEXT:  addp.2s	v0, v0, v0
+; CHECK-NEXT:  fmov	    w8, s0
+; CHECK-NEXT:  strb	    w8, [x0]
+; CHECK-NEXT:  ret
+
+  %cmp_result = icmp ne <2 x i32> %vec, zeroinitializer
+  store <2 x i1> %cmp_result, ptr %out
+  ret void
+}
+
+define void @no_combine_without_truncate(<16 x i8> %vec, ptr %out) {
+; CHECK-LABEL: no_combine_without_truncate
+; CHECK:     cmtst.16b v0, v0, v0
+; CHECK-NOT: addv.8b	b0, v0
+
+  %cmp_result = icmp ne <16 x i8> %vec, zeroinitializer
+  %extended_result = sext <16 x i1> %cmp_result to <16 x i8>
+  store <16 x i8> %extended_result, ptr %out
+  ret void
+}
+
+define void @no_combine_for_non_bool_truncate(<4 x i32> %vec, ptr %out) {
+; CHECK-LABEL: no_combine_for_non_bool_truncate
+; CHECK:     xtn.4h v0, v0
+; CHECK-NOT: addv.4s s0, v0
+
+  %trunc = trunc <4 x i32> %vec to <4 x i8>
+  store <4 x i8> %trunc, ptr %out
+  ret void
+}
+
+define void @no_combine_for_build_vector(i1 %a, i1 %b, i1 %c, i1 %d, ptr %out) {
+; CHECK-LABEL: no_combine_for_build_vector
+; CHECK-NOT: addv
+
+  %1 =   insertelement <4 x i1> undef, i1 %a, i64 0
+  %2 =   insertelement <4 x i1>    %1, i1 %b, i64 1
+  %3 =   insertelement <4 x i1>    %2, i1 %c, i64 2
+  %vec = insertelement <4 x i1>    %3, i1 %d, i64 3
+  store <4 x i1> %vec, ptr %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index 4ccc2c642a0dc..a4e1c801d98ec 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -246,22 +246,20 @@ define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; CHECK-LABEL: uaddo_v4i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v2.4h, #1
+; CHECK-NEXT:    adrp x8, .LCPI10_0
+; CHECK-NEXT:    ldr d3, [x8, :lo12:.LCPI10_0]
 ; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
 ; CHECK-NEXT:    and v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    umov w8, v0.h[0]
-; CHECK-NEXT:    umov w9, v0.h[1]
-; CHECK-NEXT:    umov w10, v0.h[2]
-; CHECK-NEXT:    umov w11, v0.h[3]
-; CHECK-NEXT:    and v1.8b, v0.8b, v2.8b
-; CHECK-NEXT:    cmeq v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    and w8, w8, #0x1
-; CHECK-NEXT:    bfi w8, w9, #1, #1
+; CHECK-NEXT:    shl v1.4h, v0.4h, #15
+; CHECK-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-NEXT:    cmeq v0.4h, v2.4h, v0.4h
+; CHECK-NEXT:    cmlt v1.4h, v1.4h, #0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
-; CHECK-NEXT:    bfi w8, w10, #2, #1
-; CHECK-NEXT:    orr w8, w8, w11, lsl #3
-; CHECK-NEXT:    and w8, w8, #0xf
+; CHECK-NEXT:    and v1.8b, v1.8b, v3.8b
+; CHECK-NEXT:    addv h1, v1.4h
 ; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    strb w8, [x0]
 ; CHECK-NEXT:    ret
   %t = call {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
diff --git a/llvm/test/CodeGen/AArch64/vec_umulo.ll b/llvm/test/CodeGen/AArch64/vec_umulo.ll
index a66cb6a7e5680..e40f477eebcf6 100644
--- a/llvm/test/CodeGen/AArch64/vec_umulo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_umulo.ll
@@ -296,18 +296,15 @@ define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; CHECK-LABEL: umulo_v4i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmov d2, d0
+; CHECK-NEXT:    adrp x8, .LCPI10_0
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    shl v0.4h, v0.4h, #15
+; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
+; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    addv h1, v0.4h
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    and v1.8b, v2.8b, v1.8b
-; CHECK-NEXT:    umov w8, v1.h[0]
-; CHECK-NEXT:    umov w9, v1.h[1]
-; CHECK-NEXT:    umov w10, v1.h[2]
-; CHECK-NEXT:    umov w11, v1.h[3]
-; CHECK-NEXT:    and w8, w8, #0x1
-; CHECK-NEXT:    bfi w8, w9, #1, #1
-; CHECK-NEXT:    bfi w8, w10, #2, #1
-; CHECK-NEXT:    orr w8, w8, w11, lsl #3
-; CHECK-NEXT:    and w8, w8, #0xf
+; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    strb w8, [x0]
 ; CHECK-NEXT:    ret
   %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
index e97ace9c38588..1b07063556795 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-and-legalization.ll
@@ -85,9 +85,14 @@ define i128 @test_v1i128(<1 x i128> %a) nounwind {
 define i8 @test_v3i8(<3 x i8> %a) nounwind {
 ; CHECK-LABEL: test_v3i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    and w8, w0, w1
-; CHECK-NEXT:    and w8, w8, w2
-; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    movi d0, #0xff00ff00ff00ff
+; CHECK-NEXT:    mov v0.h[0], w0
+; CHECK-NEXT:    mov v0.h[1], w1
+; CHECK-NEXT:    mov v0.h[2], w2
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    lsr x9, x8, #16
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
   %b = call i8 @llvm.vector.reduce.and.v3i8(<3 x i8> %a)
   ret i8 %b
@@ -97,28 +102,21 @@ define i8 @test_v9i8(<9 x i8> %a) nounwind {
 ; CHECK-LABEL: test_v9i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    umov w14, v0.b[6]
 ; CHECK-NEXT:    mov v1.16b, v0.16b
 ; CHECK-NEXT:    mov v1.b[9], w8
 ; CHECK-NEXT:    mov v1.b[10], w8
 ; CHECK-NEXT:    mov v1.b[11], w8
+; CHECK-NEXT:    mov v1.b[12], w8
 ; CHECK-NEXT:    mov v1.b[13], w8
-; CHECK-NEXT:    umov w8, v0.b[4]
+; CHECK-NEXT:    mov v1.b[14], w8
+; CHECK-NEXT:    mov v1.b[15], w8
 ; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-NEXT:    and v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    umov w9, v1.b[1]
-; CHECK-NEXT:    umov w10, v1.b[0]
-; CHECK-NEXT:    umov w11, v1.b[2]
-; CHECK-NEXT:    umov w12, v1.b[3]
-; CHECK-NEXT:    umov w13, v1.b[5]
-; CHECK-NEXT:    and w9, w10, w9
-; CHECK-NEXT:    umov w10, v0.b[7]
-; CHECK-NEXT:    and w11, w11, w12
-; CHECK-NEXT:    and w8, w8, w13
-; CHECK-NEXT:    and w9, w9, w11
-; CHECK-NEXT:    and w8, w8, w14
-; CHECK-NEXT:    and w8, w9, w8
-; CHECK-NEXT:    and w0, w8, w10
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    and x8, x8, x8, lsr #32
+; CHECK-NEXT:    and x8, x8, x8, lsr #16
+; CHECK-NEXT:    lsr x9, x8, #8
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
   %b = call i8 @llvm.vector.reduce.and.v9i8(<9 x i8> %a)
   ret i8 %b
@@ -128,9 +126,10 @@ define i32 @test_v3i32(<3 x i32> %a) nounwind {
 ; CHECK-LABEL: test_v3i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x8, x8, #32
+; CHECK-NEXT:    and v1.8b, v0.8b, v1.8b
+; CHECK-NEXT:    fmov x9, d1
 ; CHECK-NEXT:    and w0, w9, w8
 ; CHECK-NEXT:    ret
   %b = call i32 @llvm.vector.reduce.and.v3i32(<3 x i32> %a)
@@ -155,9 +154,9 @@ define i24 @test_v4i24(<4 x i24> %a) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
   %b = call i24 @llvm.vector.reduce.and.v4i24(<4 x i24> %a)
   ret i24 %b
@@ -181,9 +180,9 @@ define i32 @test_v16i32(<16 x i32> %a) nounwind {
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    lsr x9, x8, #32
+; CHECK-NEXT:    and w0, w8, w9
 ; CHECK-NEXT:    ret
   %b = call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %a)
   ret i32 %b
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 65b9cbe30c08a..4016b50d5daad 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -1,8 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=arm64-apple-ios -mattr=+sve -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64_be-unknown-linux -mattr=+sve -o - %s | FileCheck --check-prefix=CHECK-BE %s
-; RUN: llc -mtriple=arm64-apple-ios -mattr=+global-isel -mattr=+sve -o - %s | FileCheck %s
-; RUN: llc -mtriple=aarch64_be-unknown-linux -mattr=+global-isel -mattr=+sve -o - %s | FileCheck --check-prefix=CHECK-BE %s
 
 ; CHECK-LABEL: lCPI0_0:
 ; CHECK-NEXT:    .byte   0                               ; 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-no-rtn.ll
index c81e2612e5503..80d3ef5159ac2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-no-rtn.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offset_no_rtn(float %val, <4 x i32
   ; GFX908_GFX11-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908_GFX11-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_offset_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -29,7 +29,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offset_no_rtn(float %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
@@ -48,7 +48,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offen_no_rtn(float %val, <4 x i32>
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908_GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908_GFX11-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_offen_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -62,7 +62,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offen_no_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -81,7 +81,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_idxen_no_rtn(float %val, <4 x i32>
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908_GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908_GFX11-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_idxen_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -95,7 +95,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_idxen_no_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -116,7 +116,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_bothen_no_rtn(float %val, <4 x i32
   ; GFX908_GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX908_GFX11-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_bothen_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -132,7 +132,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_bothen_no_rtn(float %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-rtn.ll
index 5b5480db58ff2..acacc04241c15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f32-rtn.ll
@@ -15,7 +15,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offset_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_offset_rtn
@@ -29,7 +29,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offset_rtn(float %val, <4 x i32>
   ; GFX11-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
@@ -49,7 +49,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offen_rtn(float %val, <4 x i32> i
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_offen_rtn
@@ -64,7 +64,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offen_rtn(float %val, <4 x i32> i
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -84,7 +84,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_idxen_rtn(float %val, <4 x i32> i
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_idxen_rtn
@@ -99,7 +99,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_idxen_rtn(float %val, <4 x i32> i
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
@@ -121,7 +121,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_bothen_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_bothen_rtn
@@ -138,7 +138,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_bothen_rtn(float %val, <4 x i32>
   ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX11-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f64.ll
index acc5b16a54661..d81818d960474 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.f64.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_offset_no_rtn(double %val, <4 x i3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFSET [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFSET [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
@@ -37,7 +37,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_offen_no_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -58,7 +58,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_idxen_no_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_IDXEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_IDXEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -81,7 +81,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_bothen_no_rtn(double %val, <4 x i3
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
@@ -101,7 +101,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offset_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE1]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
@@ -128,7 +128,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
@@ -155,7 +155,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_idxen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
@@ -184,7 +184,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_bothen_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-no-rtn.ll
index c8df9d752a6c9..08e1b44cd13b1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-no-rtn.ll
@@ -15,7 +15,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offset_no_rtn(<2 x half> %val, <
   ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_offset_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -28,7 +28,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offset_no_rtn(<2 x half> %val, <
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -47,7 +47,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offen_no_rtn(<2 x half> %val, <4
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_offen_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -61,7 +61,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offen_no_rtn(<2 x half> %val, <4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -80,7 +80,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_idxen_no_rtn(<2 x half> %val, <4
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_idxen_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -94,7 +94,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_idxen_no_rtn(<2 x half> %val, <4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -115,7 +115,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_bothen_no_rtn(<2 x half> %val, <
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_bothen_no_rtn
   ; GFX90A_GFX940: bb.1 (%ir-block.0):
@@ -131,7 +131,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_bothen_no_rtn(<2 x half> %val, <
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-rtn.ll
index 8c82b8c2e68bd..910dd72cc992f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/buffer-atomic-fadd.v2f16-rtn.ll
@@ -14,7 +14,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_offset_rtn(<2 x half> %val
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
@@ -34,7 +34,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_offen_rtn(<2 x half> %val,
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -54,7 +54,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_idxen_rtn(<2 x half> %val,
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
@@ -76,7 +76,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_bothen_rtn(<2 x half> %val
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr4
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
index e639fce9d690e..db4e678c74a41 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -2768,8 +2768,13 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v12, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v13, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v14, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v14, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v0
+; GCN-NEXT:    ; kill: def $vgpr15 killed $sgpr14 killed $exec
+; GCN-NEXT:    ; kill: def $vgpr16 killed $sgpr15 killed $exec
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v15, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v16, vcc
 ; GCN-NEXT:    v_readfirstlane_b32 s0, v0
 ; GCN-NEXT:    v_readfirstlane_b32 s1, v1
 ; GCN-NEXT:    ; return to shader part epilog
@@ -2808,8 +2813,11 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s12, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s13, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2847,8 +2855,11 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s12, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s13, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    ; return to shader part epilog
@@ -2879,6 +2890,9 @@ define double @dyn_extract_v7f64_v_v(<7 x double> %vec, i32 %sel) {
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v14
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v14
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: dyn_extract_v7f64_v_v:
@@ -2903,6 +2917,9 @@ define double @dyn_extract_v7f64_v_v(<7 x double> %vec, i32 %sel) {
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v14
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v14
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: dyn_extract_v7f64_v_v:
@@ -2921,6 +2938,8 @@ define double @dyn_extract_v7f64_v_v(<7 x double> %vec, i32 %sel) {
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v10 :: v_dual_cndmask_b32 v1, v1, v11
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v14
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v12 :: v_dual_cndmask_b32 v1, v1, v13
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v14
+; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v14 :: v_dual_cndmask_b32 v1, v1, v15
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %ext = extractelement <7 x double> %vec, i32 %sel
@@ -3422,42 +3441,82 @@ define float @dyn_extract_v15f32_const_s_v(i32 %sel) {
 ; GCN-NEXT:    v_mov_b32_e32 v12, 0x41700000
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 14, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v12, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v12, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 15, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: dyn_extract_v15f32_const_s_v:
-; GFX10PLUS:       ; %bb.0: ; %entry
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, 1.0, 2.0, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40400000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 4.0, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40a00000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40c00000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40e00000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41000000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41100000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41200000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41300000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41400000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41500000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41600000, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v0, v1, 0x41700000, vcc_lo
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: dyn_extract_v15f32_const_s_v:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 1.0, 2.0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40400000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 4.0, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40a00000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40c00000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40e00000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41000000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41100000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41200000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41300000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41400000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41500000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41600000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41700000, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s4, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: dyn_extract_v15f32_const_s_v:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 1.0, 2.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40400000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 4.0, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40a00000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40c00000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x40e00000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41000000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41100000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41200000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41300000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41400000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41500000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41600000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, 0x41700000, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s0, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %ext = extractelement <15 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0>, i32 %sel
   ret float %ext
@@ -3557,7 +3616,9 @@ define amdgpu_ps float @dyn_extract_v15f32_s_v(<15 x float> inreg %vec, i32 %sel
 ; GCN-NEXT:    v_mov_b32_e32 v15, s16
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v14, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 14, v0
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v15, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 15, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10PLUS-LABEL: dyn_extract_v15f32_s_v:
@@ -3590,7 +3651,9 @@ define amdgpu_ps float @dyn_extract_v15f32_s_v(<15 x float> inreg %vec, i32 %sel
 ; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v0
 ; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, s15, vcc_lo
 ; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v0, v1, s16, vcc_lo
+; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v1, v1, s16, vcc_lo
+; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v0
+; GFX10PLUS-NEXT:    v_cndmask_b32_e64 v0, v1, s0, vcc_lo
 ; GFX10PLUS-NEXT:    ; return to shader part epilog
 entry:
   %ext = extractelement <15 x float> %vec, i32 %sel
@@ -3629,41 +3692,81 @@ define float @dyn_extract_v15f32_v_v(<15 x float> %vec, i32 %sel) {
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 14, v15
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 15, v15
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: dyn_extract_v15f32_v_v:
-; GFX10PLUS:       ; %bb.0: ; %entry
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: dyn_extract_v15f32_v_v:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v15
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: dyn_extract_v15f32_v_v:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v15
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %ext = extractelement <15 x float> %vec, i32 %sel
   ret float %ext
@@ -3825,6 +3928,8 @@ define float @dyn_extract_v15f32_v_v_offset3(<15 x float> %vec, i32 %sel) {
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 14, v15
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 15, v15
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
 ; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; MOVREL-LABEL: dyn_extract_v15f32_v_v_offset3:
@@ -3859,42 +3964,83 @@ define float @dyn_extract_v15f32_v_v_offset3(<15 x float> %vec, i32 %sel) {
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 14, v15
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 15, v15
+; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v0, vcc
 ; MOVREL-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10PLUS-LABEL: dyn_extract_v15f32_v_v_offset3:
-; GFX10PLUS:       ; %bb.0: ; %entry
-; GFX10PLUS-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10PLUS-NEXT:    v_add_nc_u32_e32 v15, 3, v15
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc_lo
-; GFX10PLUS-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v15
-; GFX10PLUS-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
-; GFX10PLUS-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: dyn_extract_v15f32_v_v_offset3:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v15
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v15
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s4, vcc_lo
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: dyn_extract_v15f32_v_v_offset3:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_add_nc_u32_e32 v15, 3, v15
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 8, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 9, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 10, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 11, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 12, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v12, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 13, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v13, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 14, v15
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 15, v15
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %add = add i32 %sel, 3
   %ext = extractelement <15 x float> %vec, i32 %add
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index 7db9d3769a743..9e51bfd71ad52 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck %s -check-prefix=GFX90A
+; RUN: llc < %s -march=amdgcn -mcpu=gfx940 -verify-machineinstrs | FileCheck %s -check-prefix=GFX940
 
 declare double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)
 declare double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32 immarg)
@@ -26,6 +27,17 @@ define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> inreg %rsrc
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 offen
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_add_noret_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 offen
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
   ret void
@@ -38,6 +50,13 @@ define amdgpu_ps void @raw_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, doub
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_add_rtn_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
   store double %ret, ptr undef
@@ -59,6 +78,21 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> inre
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 4 offen sc0 nt
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
   store double %ret, ptr addrspace(1) %out, align 8
@@ -76,6 +110,17 @@ define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> inreg %r
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_add_noret_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   ret void
@@ -88,6 +133,13 @@ define amdgpu_ps void @struct_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, d
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_add_rtn_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   store double %ret, ptr undef
@@ -109,6 +161,21 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> i
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[4:7], 0 idxen offset:4 sc0 nt
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
   store double %ret, ptr addrspace(1) %out, align 8
@@ -126,6 +193,17 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> inreg %rsrc
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_min_noret_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 offen
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
   ret void
@@ -138,6 +216,13 @@ define amdgpu_ps void @raw_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, doub
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_min_rtn_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
   store double %ret, ptr undef
@@ -159,6 +244,21 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> inre
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[4:7], 4 offen sc0 nt
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
   store double %ret, ptr addrspace(1) %out, align 8
@@ -176,6 +276,17 @@ define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> inreg %r
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 idxen
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_min_noret_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 idxen
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   ret void
@@ -188,6 +299,13 @@ define amdgpu_ps void @struct_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, d
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_min_rtn_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   store double %ret, ptr undef
@@ -209,6 +327,21 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> i
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[4:7], 0 idxen offset:4 sc0 nt
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
   store double %ret, ptr addrspace(1) %out, align 8
@@ -226,6 +359,17 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> inreg %rsrc
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_max_noret_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 offen
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
   ret void
@@ -238,6 +382,13 @@ define amdgpu_ps void @raw_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, doub
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_max_rtn_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)
   store double %ret, ptr undef
@@ -259,6 +410,21 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> inre
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[4:7], 4 offen sc0 nt
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)
   store double %ret, ptr addrspace(1) %out, align 8
@@ -276,6 +442,17 @@ define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> inreg %r
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 idxen
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_max_noret_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s8, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 idxen
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   ret void
@@ -288,6 +465,13 @@ define amdgpu_ps void @struct_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, d
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_max_rtn_f64:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)
   store double %ret, ptr undef
@@ -309,6 +493,21 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> i
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX940-NEXT:    s_load_dword s10, s[0:1], 0x3c
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x44
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[4:7], 0 idxen offset:4 sc0 nt
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)
   store double %ret, ptr addrspace(1) %out, align 8
@@ -324,6 +523,16 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret(ptr addrspace(1) %ptr, d
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fadd_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s2
+; GFX940-NEXT:    v_mov_b32_e32 v1, s3
+; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.global.atomic.fadd.f64.p1.f64(ptr addrspace(1) %ptr, double %data)
   ret void
@@ -338,6 +547,16 @@ define amdgpu_kernel void @global_atomic_fmin_f64_noret(ptr addrspace(1) %ptr, d
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    global_atomic_min_f64 v2, v[0:1], s[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fmin_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s2
+; GFX940-NEXT:    v_mov_b32_e32 v1, s3
+; GFX940-NEXT:    global_atomic_min_f64 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.global.atomic.fmin.f64.p1.f64(ptr addrspace(1) %ptr, double %data)
   ret void
@@ -352,6 +571,16 @@ define amdgpu_kernel void @global_atomic_fmax_f64_noret(ptr addrspace(1) %ptr, d
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    global_atomic_max_f64 v2, v[0:1], s[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fmax_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s2
+; GFX940-NEXT:    v_mov_b32_e32 v1, s3
+; GFX940-NEXT:    global_atomic_max_f64 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.global.atomic.fmax.f64.p1.f64(ptr addrspace(1) %ptr, double %data)
   ret void
@@ -383,6 +612,18 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB21_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fadd_f64_noret_pat:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst
   ret void
@@ -400,6 +641,18 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
   ret void
@@ -431,6 +684,18 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB23_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst
   ret void
@@ -448,6 +713,18 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
   ret void
@@ -460,6 +737,13 @@ define double @global_atomic_fadd_f64_rtn(ptr addrspace(1) %ptr, double %data) {
 ; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_atomic_fadd_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.global.atomic.fadd.f64.p1.f64(ptr addrspace(1) %ptr, double %data)
   ret double %ret
@@ -491,6 +775,17 @@ define double @global_atomic_fadd_f64_rtn_pat(ptr addrspace(1) %ptr, double %dat
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_atomic_fadd_f64_rtn_pat:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst
   ret double %ret
@@ -507,6 +802,17 @@ define double @global_atomic_fadd_f64_rtn_pat_agent(ptr addrspace(1) %ptr, doubl
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_atomic_fadd_f64_rtn_pat_agent:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
   ret double %ret
@@ -538,6 +844,17 @@ define double @global_atomic_fadd_f64_rtn_pat_system(ptr addrspace(1) %ptr, doub
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_atomic_fadd_f64_rtn_pat_system:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst
   ret double %ret
@@ -550,6 +867,13 @@ define double @global_atomic_fmax_f64_rtn(ptr addrspace(1) %ptr, double %data) {
 ; GFX90A-NEXT:    global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_atomic_fmax_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_max_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.global.atomic.fmax.f64.p1.f64(ptr addrspace(1) %ptr, double %data)
   ret double %ret
@@ -562,6 +886,13 @@ define double @global_atomic_fmin_f64_rtn(ptr addrspace(1) %ptr, double %data) {
 ; GFX90A-NEXT:    global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: global_atomic_fmin_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_min_f64 v[0:1], v[0:1], v[2:3], off sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.global.atomic.fmin.f64.p1.f64(ptr addrspace(1) %ptr, double %data)
   ret double %ret
@@ -591,6 +922,18 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB31_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1]
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst
   ret void
@@ -622,6 +965,19 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB32_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_noret_pat:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3] sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 seq_cst
   ret void
@@ -640,6 +996,19 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_noret_pat_agent:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst
   ret void
@@ -672,6 +1041,19 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB34_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_noret_pat_system:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3] sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("one-as") seq_cst
   ret void
@@ -703,6 +1085,17 @@ define double @flat_atomic_fadd_f64_rtn_pat(ptr %ptr) #1 {
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_rtn_pat:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 seq_cst
   ret double %ret
@@ -719,6 +1112,17 @@ define double @flat_atomic_fadd_f64_rtn_pat_agent(ptr %ptr) #1 {
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    buffer_wbinvl1_vol
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst
   ret double %ret
@@ -751,6 +1155,18 @@ define double @flat_atomic_fadd_f64_rtn_pat_system(ptr %ptr) #1 {
 ; GFX90A-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_rtn_pat_system:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    buffer_wbl2 sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    buffer_inv sc0 sc1
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("one-as") seq_cst
   ret double %ret
@@ -765,6 +1181,17 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret(ptr %ptr, double %data) {
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s2
+; GFX940-NEXT:    v_mov_b32_e32 v3, s3
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr %ptr, double %data)
   ret void
@@ -777,6 +1204,13 @@ define double @flat_atomic_fadd_f64_rtn(ptr %ptr, double %data) {
 ; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr %ptr, double %data)
   ret double %ret
@@ -806,6 +1240,19 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB40_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX940-NEXT:    buffer_wbl2 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    buffer_inv sc1
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst
   ret void
@@ -820,6 +1267,17 @@ define amdgpu_kernel void @flat_atomic_fmin_f64_noret(ptr %ptr, double %data) {
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    flat_atomic_min_f64 v[0:1], v[2:3]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fmin_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s2
+; GFX940-NEXT:    v_mov_b32_e32 v3, s3
+; GFX940-NEXT:    flat_atomic_min_f64 v[0:1], v[2:3]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %ptr, double %data)
   ret void
@@ -832,6 +1290,13 @@ define double @flat_atomic_fmin_f64_rtn(ptr %ptr, double %data) {
 ; GFX90A-NEXT:    flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_atomic_fmin_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %ptr, double %data)
   ret double %ret
@@ -846,6 +1311,17 @@ define amdgpu_kernel void @flat_atomic_fmax_f64_noret(ptr %ptr, double %data) {
 ; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    flat_atomic_max_f64 v[0:1], v[2:3]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: flat_atomic_fmax_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s2
+; GFX940-NEXT:    v_mov_b32_e32 v3, s3
+; GFX940-NEXT:    flat_atomic_max_f64 v[0:1], v[2:3]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %ptr, double %data)
   ret void
@@ -858,6 +1334,13 @@ define double @flat_atomic_fmax_f64_rtn(ptr %ptr, double %data) {
 ; GFX90A-NEXT:    flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: flat_atomic_fmax_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX940-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %ptr, double %data)
   ret double %ret
@@ -873,6 +1356,16 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret(ptr addrspace(3) %ptr, do
 ; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: local_atomic_fadd_f64_noret:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dword s4, s[0:1], 0x24
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, s4
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX940-NEXT:    ds_add_f64 v2, v[0:1]
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
   ret void
@@ -887,6 +1380,15 @@ define double @local_atomic_fadd_f64_rtn(ptr addrspace(3) %ptr, double %data) {
 ; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[4:5]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: local_atomic_fadd_f64_rtn:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_mov_b32_e32 v2, v1
+; GFX940-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
   ret double %ret
@@ -904,6 +1406,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: local_atomic_fadd_f64_noret_pat:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, s0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    ds_add_f64 v2, v[0:1]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
   ret void
@@ -921,6 +1434,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dword s0, s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v2, s0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    ds_add_f64 v2, v[0:1]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
   ret void
@@ -949,6 +1473,29 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ; GFX90A-NEXT:    s_cbranch_execnz .LBB49_1
 ; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX90A-NEXT:    s_endpgm
+;
+; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_load_dword s2, s[0:1], 0x24
+; GFX940-NEXT:    s_mov_b64 s[0:1], 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s2
+; GFX940-NEXT:    ds_read_b64 v[0:1], v0
+; GFX940-NEXT:  .LBB49_1: ; %atomicrmw.start
+; GFX940-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_add_f64 v[2:3], v[0:1], 4.0
+; GFX940-NEXT:    v_mov_b32_e32 v4, s2
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
+; GFX940-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], v[2:3]
+; GFX940-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX940-NEXT:    s_cbranch_execnz .LBB49_1
+; GFX940-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX940-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
   ret void
@@ -964,6 +1511,15 @@ define double @local_atomic_fadd_f64_rtn_pat(ptr addrspace(3) %ptr, double %data
 ; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: local_atomic_fadd_f64_rtn_pat:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
   ret double %ret
@@ -978,6 +1534,15 @@ define double @local_atomic_fadd_f64_rtn_ieee_unsafe(ptr addrspace(3) %ptr, doub
 ; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[4:5]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: local_atomic_fadd_f64_rtn_ieee_unsafe:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_mov_b32_e32 v2, v1
+; GFX940-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
   ret double %ret
@@ -992,6 +1557,15 @@ define double @local_atomic_fadd_f64_rtn_ieee_safe(ptr addrspace(3) %ptr, double
 ; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[4:5]
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX940-LABEL: local_atomic_fadd_f64_rtn_ieee_safe:
+; GFX940:       ; %bb.0: ; %main_body
+; GFX940-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v3, v2
+; GFX940-NEXT:    v_mov_b32_e32 v2, v1
+; GFX940-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_setpc_b64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
   ret double %ret
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index 730e3f5dcd33b..7d5c1fade0b3b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -5670,6 +5670,10 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GPRIDX-NEXT:    s_mov_b32 s12, s14
 ; GPRIDX-NEXT:    s_mov_b32 s13, s15
 ; GPRIDX-NEXT:    v_mov_b32_e32 v18, s15
+; GPRIDX-NEXT:    v_mov_b32_e32 v3, s0
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GPRIDX-NEXT:    v_mov_b32_e32 v4, s1
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc
 ; GPRIDX-NEXT:    v_mov_b32_e32 v17, s14
 ; GPRIDX-NEXT:    v_mov_b32_e32 v16, s13
 ; GPRIDX-NEXT:    v_mov_b32_e32 v15, s12
@@ -5683,43 +5687,39 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GPRIDX-NEXT:    v_mov_b32_e32 v7, s4
 ; GPRIDX-NEXT:    v_mov_b32_e32 v6, s3
 ; GPRIDX-NEXT:    v_mov_b32_e32 v5, s2
-; GPRIDX-NEXT:    v_mov_b32_e32 v4, s1
-; GPRIDX-NEXT:    v_mov_b32_e32 v3, s0
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 2, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 3, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 4, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 5, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 1, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v5, v0, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v7, v0, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v9, v0, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v11, v0, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v13, v0, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v14, v1, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v1, v16, v1, s[8:9]
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s0, v3
-; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v4
-; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v2
-; GPRIDX-NEXT:    v_readfirstlane_b32 s3, v6
-; GPRIDX-NEXT:    v_readfirstlane_b32 s4, v5
-; GPRIDX-NEXT:    v_readfirstlane_b32 s5, v8
-; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v7
-; GPRIDX-NEXT:    v_readfirstlane_b32 s7, v10
-; GPRIDX-NEXT:    v_readfirstlane_b32 s8, v9
-; GPRIDX-NEXT:    v_readfirstlane_b32 s9, v12
-; GPRIDX-NEXT:    v_readfirstlane_b32 s10, v11
-; GPRIDX-NEXT:    v_readfirstlane_b32 s11, v13
-; GPRIDX-NEXT:    v_readfirstlane_b32 s12, v0
-; GPRIDX-NEXT:    v_readfirstlane_b32 s13, v1
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
+; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v3
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v5, v0, vcc
+; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v3
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v7, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v8, v1, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v9, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v10, v1, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v11, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v12, v1, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v10, v13, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v11, v14, v1, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v12, v15, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v13, v16, v1, vcc
+; GPRIDX-NEXT:    v_readfirstlane_b32 s3, v3
+; GPRIDX-NEXT:    v_readfirstlane_b32 s4, v4
+; GPRIDX-NEXT:    v_readfirstlane_b32 s5, v5
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v6
+; GPRIDX-NEXT:    v_readfirstlane_b32 s7, v7
+; GPRIDX-NEXT:    v_readfirstlane_b32 s8, v8
+; GPRIDX-NEXT:    v_readfirstlane_b32 s9, v9
+; GPRIDX-NEXT:    v_readfirstlane_b32 s10, v10
+; GPRIDX-NEXT:    v_readfirstlane_b32 s11, v11
+; GPRIDX-NEXT:    v_readfirstlane_b32 s12, v12
+; GPRIDX-NEXT:    v_readfirstlane_b32 s13, v13
 ; GPRIDX-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: dyn_insertelement_v7f64_s_v_v:
@@ -5739,9 +5739,13 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GFX10-NEXT:    s_mov_b32 s12, s14
 ; GFX10-NEXT:    s_mov_b32 s13, s15
 ; GFX10-NEXT:    v_mov_b32_e32 v18, s15
+; GFX10-NEXT:    v_mov_b32_e32 v3, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX10-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v17, s14
 ; GFX10-NEXT:    v_mov_b32_e32 v16, s13
 ; GFX10-NEXT:    v_mov_b32_e32 v15, s12
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_mov_b32_e32 v14, s11
 ; GFX10-NEXT:    v_mov_b32_e32 v13, s10
 ; GFX10-NEXT:    v_mov_b32_e32 v12, s9
@@ -5752,43 +5756,39 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GFX10-NEXT:    v_mov_b32_e32 v7, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s3
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s2
-; GFX10-NEXT:    v_mov_b32_e32 v4, s1
-; GFX10-NEXT:    v_mov_b32_e32 v3, s0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 6, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v5, v0, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s2, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v12, v1, vcc_lo
-; GFX10-NEXT:    v_readfirstlane_b32 s3, v6
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v13, v0, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v14, v1, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v16, v1, s1
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v3
-; GFX10-NEXT:    v_readfirstlane_b32 s1, v4
-; GFX10-NEXT:    v_readfirstlane_b32 s4, v7
-; GFX10-NEXT:    v_readfirstlane_b32 s5, v8
-; GFX10-NEXT:    v_readfirstlane_b32 s6, v9
-; GFX10-NEXT:    v_readfirstlane_b32 s7, v10
-; GFX10-NEXT:    v_readfirstlane_b32 s8, v11
-; GFX10-NEXT:    v_readfirstlane_b32 s9, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s10, v12
-; GFX10-NEXT:    v_readfirstlane_b32 s11, v13
-; GFX10-NEXT:    v_readfirstlane_b32 s12, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s13, v1
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v12, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v13, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v14, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v15, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v16, v1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: dyn_insertelement_v7f64_s_v_v:
@@ -5808,45 +5808,45 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GFX11-NEXT:    s_mov_b32 s12, s14
 ; GFX11-NEXT:    s_mov_b32 s13, s15
 ; GFX11-NEXT:    v_dual_mov_b32 v18, s15 :: v_dual_mov_b32 v17, s14
+; GFX11-NEXT:    v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX11-NEXT:    v_dual_mov_b32 v16, s13 :: v_dual_mov_b32 v15, s12
 ; GFX11-NEXT:    v_dual_mov_b32 v14, s11 :: v_dual_mov_b32 v13, s10
 ; GFX11-NEXT:    v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v11, s8
 ; GFX11-NEXT:    v_dual_mov_b32 v10, s7 :: v_dual_mov_b32 v9, s6
 ; GFX11-NEXT:    v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v7, s4
 ; GFX11-NEXT:    v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v5, s2
-; GFX11-NEXT:    v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 6, v2
-; GFX11-NEXT:    v_dual_cndmask_b32 v3, v3, v0 :: v_dual_cndmask_b32 v4, v4, v1
+; GFX11-NEXT:    v_dual_cndmask_b32 v18, v3, v0 :: v_dual_cndmask_b32 v17, v4, v1
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v2
-; GFX11-NEXT:    v_dual_cndmask_b32 v7, v7, v0 :: v_dual_cndmask_b32 v8, v8, v1
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s0
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v2
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v5
-; GFX11-NEXT:    v_dual_cndmask_b32 v11, v11, v0 :: v_dual_cndmask_b32 v2, v12, v1
-; GFX11-NEXT:    v_readfirstlane_b32 s3, v6
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v13, v0, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v14, v1, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v16, v1, s1
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v3
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v4
-; GFX11-NEXT:    v_readfirstlane_b32 s4, v7
-; GFX11-NEXT:    v_readfirstlane_b32 s5, v8
-; GFX11-NEXT:    v_readfirstlane_b32 s6, v9
-; GFX11-NEXT:    v_readfirstlane_b32 s7, v10
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 4, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v5, v0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v4, v6, v1, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s9, 6, v2
+; GFX11-NEXT:    v_dual_cndmask_b32 v6, v7, v0 :: v_dual_cndmask_b32 v5, v8, v1
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s6
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v18
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v17
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
+; GFX11-NEXT:    v_dual_cndmask_b32 v7, v9, v0 :: v_dual_cndmask_b32 v8, v10, v1
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v9, v12, v1, s6
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v15, v0, s9
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v4
+; GFX11-NEXT:    v_readfirstlane_b32 s4, v6
+; GFX11-NEXT:    v_cndmask_b32_e32 v10, v13, v0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v14, v14, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v16, v1, s9
+; GFX11-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX11-NEXT:    v_readfirstlane_b32 s6, v7
+; GFX11-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX11-NEXT:    v_readfirstlane_b32 s8, v11
-; GFX11-NEXT:    v_readfirstlane_b32 s9, v2
-; GFX11-NEXT:    v_readfirstlane_b32 s10, v12
-; GFX11-NEXT:    v_readfirstlane_b32 s11, v13
-; GFX11-NEXT:    v_readfirstlane_b32 s12, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s13, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX11-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX11-NEXT:    v_readfirstlane_b32 s11, v14
+; GFX11-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX11-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX11-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <7 x double> %vec, double %val, i32 %idx
@@ -5908,26 +5908,26 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; GPRIDX-LABEL: dyn_insertelement_v7f64_v_v_v:
 ; GPRIDX:       ; %bb.0: ; %entry
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 2, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 3, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 4, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 5, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 6, v16
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v14, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v14, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s[10:11]
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v15, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v15, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s[10:11]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v14, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v15, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v14, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v15, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v8, v14, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v9, v15, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v10, v10, v14, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v11, v11, v15, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v13, v13, v15, vcc
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s0, v0
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v1
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v2
@@ -5947,38 +5947,38 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; GFX10-LABEL: dyn_insertelement_v7f64_v_v_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v16
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 2, v16
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 3, v16
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s3, 4, v16
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 5, v16
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 6, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v14, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v14, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v15, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v15, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s5
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
 ; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v15, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
 ; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -5986,14 +5986,14 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; GFX11-LABEL: dyn_insertelement_v7f64_v_v_v:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v16
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 6, v16
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s9, 5, v16
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s10, 6, v16
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v14 :: v_dual_cndmask_b32 v1, v1, v15
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s9
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s9
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s10
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s10
 ; GFX11-NEXT:    v_dual_cndmask_b32 v2, v2, v14 :: v_dual_cndmask_b32 v3, v3, v15
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v16
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll
deleted file mode 100644
index c182cff9329f4..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-non-integral-address-spaces.ll
+++ /dev/null
@@ -1,37 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -o - -stop-after=irtranslator %s | FileCheck %s
-
-; Check that the CSEMIRBuilder doesn't fold away the getelementptr during IRTranslator
-define ptr addrspace(7) @no_auto_constfold_gep() {
-  ; CHECK-LABEL: name: no_auto_constfold_gep
-  ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(p7) = G_CONSTANT i64 0
-  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 123
-  ; CHECK-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p7) = G_PTR_ADD [[C]], [[C1]](s64)
-  ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PTR_ADD]](p7)
-  ; CHECK-NEXT:   $vgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $vgpr1 = COPY [[UV1]](s32)
-  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
-  %gep = getelementptr i8, ptr addrspace(7) null, i64 123
-  ret ptr addrspace(7) %gep
-}
-
-; Check that the CSEMIRBuilder doesn't fold away the getelementptr during IRTranslator
-define <2 x ptr addrspace(7)> @no_auto_constfold_gep_vector() {
-  ; CHECK-LABEL: name: no_auto_constfold_gep_vector
-  ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK-NEXT:   [[C:%[0-9]+]]:_(p7) = G_CONSTANT i64 0
-  ; CHECK-NEXT:   [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p7>) = G_BUILD_VECTOR [[C]](p7), [[C]](p7)
-  ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 123
-  ; CHECK-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C1]](s64), [[C1]](s64)
-  ; CHECK-NEXT:   [[PTR_ADD:%[0-9]+]]:_(<2 x p7>) = G_PTR_ADD [[BUILD_VECTOR]], [[BUILD_VECTOR1]](<2 x s64>)
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x p7>) = COPY [[PTR_ADD]](<2 x p7>)
-  ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x p7>)
-  ; CHECK-NEXT:   $vgpr0 = COPY [[UV]](s32)
-  ; CHECK-NEXT:   $vgpr1 = COPY [[UV1]](s32)
-  ; CHECK-NEXT:   $vgpr2 = COPY [[UV2]](s32)
-  ; CHECK-NEXT:   $vgpr3 = COPY [[UV3]](s32)
-  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
-  %gep = getelementptr i8, <2 x ptr addrspace(7)> zeroinitializer, <2 x i64> <i64 123, i64 123>
-  ret <2 x ptr addrspace(7)> %gep
-}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
index 1749481d35924..3722dc09aaefb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
@@ -20,7 +20,7 @@ define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_swap_1d
@@ -39,7 +39,7 @@ define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -65,7 +65,7 @@ define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_1d
@@ -84,7 +84,7 @@ define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -110,7 +110,7 @@ define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_sub_1d
@@ -129,7 +129,7 @@ define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -155,7 +155,7 @@ define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_smin_1d
@@ -174,7 +174,7 @@ define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -201,7 +201,7 @@ define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_umin_1d
@@ -220,7 +220,7 @@ define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -246,7 +246,7 @@ define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_smax_1d
@@ -265,7 +265,7 @@ define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -291,7 +291,7 @@ define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_umax_1d
@@ -310,7 +310,7 @@ define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -336,7 +336,7 @@ define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_and_1d
@@ -355,7 +355,7 @@ define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -381,7 +381,7 @@ define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_or_1d
@@ -400,7 +400,7 @@ define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -426,7 +426,7 @@ define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_xor_1d
@@ -445,7 +445,7 @@ define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -471,7 +471,7 @@ define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_inc_1d
@@ -490,7 +490,7 @@ define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -516,7 +516,7 @@ define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_dec_1d
@@ -535,7 +535,7 @@ define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -563,7 +563,7 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_1d
@@ -584,7 +584,7 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX10NSA-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -613,7 +613,7 @@ define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
   ; GFX9-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_2d
@@ -635,7 +635,7 @@ define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
   ; GFX10NSA-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -669,7 +669,7 @@ define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_3d
@@ -696,7 +696,7 @@ define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -730,7 +730,7 @@ define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_cube
@@ -757,7 +757,7 @@ define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -786,7 +786,7 @@ define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX9-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_1darray
@@ -808,7 +808,7 @@ define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX10NSA-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](s32), [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -842,7 +842,7 @@ define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_2darray
@@ -869,7 +869,7 @@ define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -903,7 +903,7 @@ define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_2dmsaa
@@ -930,7 +930,7 @@ define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -965,7 +965,7 @@ define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data,
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_2darraymsaa
@@ -993,7 +993,7 @@ define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data,
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](s32), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1019,7 +1019,7 @@ define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX9-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_add_1d_slc
@@ -1038,7 +1038,7 @@ define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY9]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](s32), [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1069,7 +1069,7 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY11]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_2d
@@ -1093,7 +1093,7 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY11]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1129,7 +1129,7 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_3d
@@ -1158,7 +1158,7 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1195,7 +1195,7 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: atomic_cmpswap_2darraymsaa
@@ -1225,7 +1225,7 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (volatile dereferenceable load store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
index 30610db9f6a10..1b4936af2f95c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
@@ -22,7 +22,7 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -47,7 +47,7 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -84,7 +84,7 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -114,7 +114,7 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -159,7 +159,7 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -196,7 +196,7 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -242,7 +242,7 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -279,7 +279,7 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -318,7 +318,7 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -348,7 +348,7 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -393,7 +393,7 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -430,7 +430,7 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -476,7 +476,7 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2dmsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2dmsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -513,7 +513,7 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2dmsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2dmsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -561,7 +561,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -600,7 +600,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -640,7 +640,7 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -670,7 +670,7 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -715,7 +715,7 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -752,7 +752,7 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -800,7 +800,7 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -839,7 +839,7 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -888,7 +888,7 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -927,7 +927,7 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -974,7 +974,7 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1011,7 +1011,7 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1059,7 +1059,7 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1098,7 +1098,7 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1138,7 +1138,7 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d
   ; GFX10NSA: bb.1.main_body:
@@ -1163,7 +1163,7 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -1200,7 +1200,7 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_2d
   ; GFX10NSA: bb.1.main_body:
@@ -1230,7 +1230,7 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -1275,7 +1275,7 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_3d
   ; GFX10NSA: bb.1.main_body:
@@ -1312,7 +1312,7 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -1358,7 +1358,7 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_cube
   ; GFX10NSA: bb.1.main_body:
@@ -1395,7 +1395,7 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -1434,7 +1434,7 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1darray
   ; GFX10NSA: bb.1.main_body:
@@ -1464,7 +1464,7 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -1509,7 +1509,7 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_2darray
   ; GFX10NSA: bb.1.main_body:
@@ -1546,7 +1546,7 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -1592,7 +1592,7 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2dmsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2dmsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_2dmsaa
   ; GFX10NSA: bb.1.main_body:
@@ -1629,7 +1629,7 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2dmsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2dmsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -1677,7 +1677,7 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_2darraymsaa
   ; GFX10NSA: bb.1.main_body:
@@ -1716,7 +1716,7 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -1756,7 +1756,7 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_mip_1d
   ; GFX10NSA: bb.1.main_body:
@@ -1786,7 +1786,7 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -1831,7 +1831,7 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_mip_2d
   ; GFX10NSA: bb.1.main_body:
@@ -1868,7 +1868,7 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -1916,7 +1916,7 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_mip_3d
   ; GFX10NSA: bb.1.main_body:
@@ -1955,7 +1955,7 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -2004,7 +2004,7 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_mip_cube
   ; GFX10NSA: bb.1.main_body:
@@ -2043,7 +2043,7 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -2090,7 +2090,7 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_mip_1darray
   ; GFX10NSA: bb.1.main_body:
@@ -2127,7 +2127,7 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -2175,7 +2175,7 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_mip_2darray
   ; GFX10NSA: bb.1.main_body:
@@ -2214,7 +2214,7 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x s32>), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords_lo, i32 0
@@ -2700,7 +2700,7 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: load_1d_V1
@@ -2721,7 +2721,7 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -2749,7 +2749,7 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2772,7 +2772,7 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2803,7 +2803,7 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](s32), 2, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (s32), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](s32), 2, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (s32), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_V1
   ; GFX10NSA: bb.1.main_body:
@@ -2824,7 +2824,7 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](s32), 2, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](s32), 2, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (s32), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -2854,7 +2854,7 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x s32>), 12, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<2 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x s32>), 12, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 3 :: (dereferenceable store (<2 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_V2
   ; GFX10NSA: bb.1.main_body:
@@ -2877,7 +2877,7 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x s32>), 12, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<2 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x s32>), 12, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 1 :: (dereferenceable store (<2 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -2904,7 +2904,7 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2929,7 +2929,7 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2961,7 +2961,7 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2986,7 +2986,7 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3018,7 +3018,7 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3043,7 +3043,7 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3080,7 +3080,7 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_glc
   ; GFX10NSA: bb.1.main_body:
@@ -3105,7 +3105,7 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 1, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -3137,7 +3137,7 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_slc
   ; GFX10NSA: bb.1.main_body:
@@ -3162,7 +3162,7 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 2, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -3194,7 +3194,7 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 3 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10NSA-LABEL: name: store_1d_glc_slc
   ; GFX10NSA: bb.1.main_body:
@@ -3219,7 +3219,7 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 1 :: (dereferenceable store (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x s32>), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 0, 3, 1 :: (dereferenceable store (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   S_ENDPGM 0
 main_body:
   %s = extractelement <2 x i16> %coords, i32 0
@@ -3276,7 +3276,7 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX9-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX9-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3303,7 +3303,7 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x s16>)
   ; GFX10NSA-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX10NSA-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3345,7 +3345,7 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX9-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3377,7 +3377,7 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
   ; GFX10NSA-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX10NSA-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3427,7 +3427,7 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX9-NEXT:   [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF1]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX9-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3466,7 +3466,7 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
   ; GFX10NSA-NEXT:   [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF1]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX10NSA-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3519,7 +3519,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX9-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -3560,7 +3560,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10NSA-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10NSA-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX10NSA-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
index 236f21d7788d5..b86c0effc16d3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
@@ -19,7 +19,7 @@ define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s16) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s16) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](s16)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[ANYEXT]](s32)
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -39,7 +39,7 @@ define amdgpu_ps half @image_load_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s16) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s16) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](s16)
   ; PACKED-NEXT:   $vgpr0 = COPY [[ANYEXT]](s32)
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -64,7 +64,7 @@ define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -91,7 +91,7 @@ define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s16>)
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -115,7 +115,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -148,7 +148,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x s16>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[UV1]](<2 x s16>)
   ; PACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
@@ -182,7 +182,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -215,7 +215,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x s16>)
   ; PACKED-NEXT:   $vgpr0 = COPY [[UV]](<2 x s16>)
   ; PACKED-NEXT:   $vgpr1 = COPY [[UV1]](<2 x s16>)
@@ -242,7 +242,7 @@ define amdgpu_ps half @image_load_tfe_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t)
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -264,7 +264,7 @@ define amdgpu_ps half @image_load_tfe_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t)
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; PACKED-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -294,7 +294,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s,
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -323,7 +323,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s,
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -354,7 +354,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s,
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV3]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -389,7 +389,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s,
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV1]](s32)
@@ -430,7 +430,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s,
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<5 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -465,7 +465,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s,
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV1]](s32)
@@ -520,7 +520,7 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
   ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
@@ -546,7 +546,7 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s16>)
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %tex = call <2 x half> @llvm.amdgcn.image.load.2d.v2f16.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -593,7 +593,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -625,7 +625,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; PACKED-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; PACKED-NEXT:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
@@ -655,7 +655,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
   ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
@@ -684,7 +684,7 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; PACKED-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; PACKED-NEXT:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
@@ -757,7 +757,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -790,7 +790,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x s16>)
   ; PACKED-NEXT:   $vgpr0 = COPY [[UV]](<2 x s16>)
   ; PACKED-NEXT:   $vgpr1 = COPY [[UV1]](<2 x s16>)
@@ -816,7 +816,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -848,7 +848,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s16>)
   ; PACKED-NEXT:   $vgpr1 = COPY [[DEF]](<2 x s16>)
@@ -874,7 +874,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
   ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
@@ -903,7 +903,7 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc,
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s16>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s16>)
   ; PACKED-NEXT:   $vgpr1 = COPY [[DEF]](<2 x s16>)
@@ -957,7 +957,7 @@ define amdgpu_ps half @image_load_tfe_f16_dmask_0000(<8 x i32> inreg %rsrc, i32
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -979,7 +979,7 @@ define amdgpu_ps half @image_load_tfe_f16_dmask_0000(<8 x i32> inreg %rsrc, i32
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; PACKED-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -1009,7 +1009,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -1038,7 +1038,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -1069,7 +1069,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
   ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
@@ -1098,7 +1098,7 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -1129,7 +1129,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1163,7 +1163,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -1200,7 +1200,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1232,7 +1232,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -1269,7 +1269,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1301,7 +1301,7 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -1338,7 +1338,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV3]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1373,7 +1373,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s16>), align 8, addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV1]](s32)
@@ -1406,7 +1406,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1440,7 +1440,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s16>), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1473,7 +1473,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1505,7 +1505,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
@@ -1538,7 +1538,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rs
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; UNPACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; UNPACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; UNPACKED-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
@@ -1570,7 +1570,7 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rs
   ; PACKED-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; PACKED-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; PACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 7)
+  ; PACKED-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s16), addrspace 8)
   ; PACKED-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x s32>)
   ; PACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[UV]](s32)
   ; PACKED-NEXT:   [[DEF1:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll
index 137e78fffa9df..4b12bddb61df4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.ll
@@ -18,7 +18,7 @@ define amdgpu_ps float @image_load_f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %tex = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -42,7 +42,7 @@ define amdgpu_ps <2 x float> @image_load_v2f32(<8 x i32> inreg %rsrc, i32 %s, i3
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GCN-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -68,7 +68,7 @@ define amdgpu_ps <3 x float> @image_load_v3f32(<8 x i32> inreg %rsrc, i32 %s, i3
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GCN-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -95,7 +95,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32(<8 x i32> inreg %rsrc, i32 %s, i3
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GCN-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -124,7 +124,7 @@ define amdgpu_ps float @image_load_tfe_f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -154,7 +154,7 @@ define amdgpu_ps <2 x float> @image_load_tfe_v2f32(<8 x i32> inreg %rsrc, i32 %s
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN-NEXT:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -185,7 +185,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32(<8 x i32> inreg %rsrc, i32 %s
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GCN-NEXT:   G_STORE [[UV3]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -217,7 +217,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32(<8 x i32> inreg %rsrc, i32 %s
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GCN-NEXT:   G_STORE [[UV4]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -263,7 +263,7 @@ define amdgpu_ps <2 x float> @image_load_v2f32_dmask_1000(<8 x i32> inreg %rsrc,
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN-NEXT:   $vgpr1 = COPY [[DEF]](s32)
@@ -305,7 +305,7 @@ define amdgpu_ps <3 x float> @image_load_v3f32_dmask_1100(<8 x i32> inreg %rsrc,
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -333,7 +333,7 @@ define amdgpu_ps <3 x float> @image_load_v3f32_dmask_1000(<8 x i32> inreg %rsrc,
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN-NEXT:   $vgpr1 = COPY [[DEF]](s32)
@@ -377,7 +377,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32_dmask_1110(<8 x i32> inreg %rsrc,
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -406,7 +406,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32_dmask_1100(<8 x i32> inreg %rsrc,
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -435,7 +435,7 @@ define amdgpu_ps <4 x float> @image_load_v4f32_dmask_1000(<8 x i32> inreg %rsrc,
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GCN-NEXT:   $vgpr1 = COPY [[DEF]](s32)
@@ -482,7 +482,7 @@ define amdgpu_ps float @image_load_tfe_f32_dmask_0000(<8 x i32> inreg %rsrc, i32
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GCN-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -512,7 +512,7 @@ define amdgpu_ps <2 x float> @image_load_tfe_v2f32_dmask_1000(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -544,7 +544,7 @@ define amdgpu_ps <2 x float> @image_load_tfe_v2f32_dmask_0000(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -576,7 +576,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32_dmask_1100(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -609,7 +609,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32_dmask_1000(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -642,7 +642,7 @@ define amdgpu_ps <3 x float> @image_load_tfe_v3f32_dmask_0000(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -675,7 +675,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_1110(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<3 x s32>), align 16, addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV3]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -709,7 +709,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_1100(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<3 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<3 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV2]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -743,7 +743,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_1000(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
@@ -777,7 +777,7 @@ define amdgpu_ps <4 x float> @image_load_tfe_v4f32_dmask_0000(<8 x i32> inreg %r
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GCN-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GCN-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GCN-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GCN-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2darraymsaa.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2darraymsaa.ll
index 49ca0667b849d..740e1e801cf6a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2darraymsaa.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2darraymsaa.ll
@@ -22,7 +22,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i3
   ; GFX6-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX6-NEXT:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX6-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
-  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX6-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX6-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX6-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -46,7 +46,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i3
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA-NEXT:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10NSA-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -79,7 +79,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr ad
   ; GFX6-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX6-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr3
   ; GFX6-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32)
-  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<4 x s32>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX6-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX6-NEXT:   G_STORE [[UV4]](s32), [[MV]](p1) :: (store (s32) into %ir.out, addrspace 1)
   ; GFX6-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -107,7 +107,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr ad
   ; GFX10NSA-NEXT:   [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr3
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x s32>)
   ; GFX10NSA-NEXT:   G_STORE [[UV4]](s32), [[MV]](p1) :: (store (s32) into %ir.out, addrspace 1)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.3d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.3d.ll
index 8a5765c860c28..c39b1e0f60f91 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.3d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.3d.ll
@@ -21,7 +21,7 @@ define amdgpu_ps float @image_load_3d_f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX6-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX6-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX6-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32)
-  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[BUILD_VECTOR1]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[BUILD_VECTOR1]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GFX6-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX6-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10NSA-LABEL: name: image_load_3d_f32
@@ -40,7 +40,7 @@ define amdgpu_ps float @image_load_3d_f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX10NSA-NEXT:   [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10NSA-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %tex = call float @llvm.amdgcn.image.load.3d.f32.i32(i32 1, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
@@ -66,7 +66,7 @@ define amdgpu_ps float @image_load_3d_tfe_f32(<8 x i32> inreg %rsrc, i32 %s, i32
   ; GFX6-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX6-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; GFX6-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32)
-  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[BUILD_VECTOR1]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX6-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[BUILD_VECTOR1]](<3 x s32>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GFX6-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX6-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX6-NEXT:   $vgpr0 = COPY [[UV]](s32)
@@ -88,7 +88,7 @@ define amdgpu_ps float @image_load_3d_tfe_f32(<8 x i32> inreg %rsrc, i32 %s, i32
   ; GFX10NSA-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10NSA-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10NSA-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX10NSA-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.3d), 1, [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[BUILD_VECTOR]](<8 x s32>), 1, 0, 0 :: (dereferenceable load (s32), addrspace 8)
   ; GFX10NSA-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX10NSA-NEXT:   G_STORE [[UV1]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
   ; GFX10NSA-NEXT:   $vgpr0 = COPY [[UV]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
index 0f4c8bd62c9ac..baf3e6bc98c7c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
@@ -24,7 +24,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GFX9-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -51,7 +51,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX10-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -78,7 +78,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX11-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GFX11-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX11-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -114,7 +114,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX9-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -144,7 +144,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -174,7 +174,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX11-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX11-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -215,7 +215,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -250,7 +250,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -285,7 +285,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -326,7 +326,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -361,7 +361,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -396,7 +396,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -432,7 +432,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -462,7 +462,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -492,7 +492,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX11-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -533,7 +533,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -568,7 +568,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -603,7 +603,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -641,7 +641,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -673,7 +673,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -705,7 +705,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -744,7 +744,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -777,7 +777,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -810,7 +810,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -846,7 +846,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX9-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -876,7 +876,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -906,7 +906,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX11-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -947,7 +947,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -982,7 +982,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1017,7 +1017,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1056,7 +1056,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1089,7 +1089,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1122,7 +1122,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1165,7 +1165,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1201,7 +1201,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1237,7 +1237,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1276,7 +1276,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1309,7 +1309,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1342,7 +1342,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1383,7 +1383,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1418,7 +1418,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1453,7 +1453,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1494,7 +1494,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1528,7 +1528,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1562,7 +1562,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1605,7 +1605,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1641,7 +1641,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1677,7 +1677,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1718,7 +1718,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1753,7 +1753,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1788,7 +1788,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1832,7 +1832,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1869,7 +1869,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1906,7 +1906,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1949,7 +1949,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1985,7 +1985,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2021,7 +2021,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2067,7 +2067,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2106,7 +2106,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2145,7 +2145,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[TRUNC2]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC3]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2187,7 +2187,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2222,7 +2222,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2257,7 +2257,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2304,7 +2304,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2344,7 +2344,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2384,7 +2384,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2441,7 +2441,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[TRUNC7]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC8]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[BUILD_VECTOR6]](<2 x s16>), [[BUILD_VECTOR7]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2492,7 +2492,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[TRUNC7]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC8]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[BUILD_VECTOR6]](<2 x s16>), [[BUILD_VECTOR7]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2543,7 +2543,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[TRUNC7]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC8]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR6]](<2 x s16>), [[BUILD_VECTOR7]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2587,7 +2587,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2624,7 +2624,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2661,7 +2661,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2710,7 +2710,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2752,7 +2752,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2794,7 +2794,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2838,7 +2838,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2875,7 +2875,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2912,7 +2912,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -2963,7 +2963,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3007,7 +3007,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3051,7 +3051,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3097,7 +3097,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3136,7 +3136,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3175,7 +3175,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3228,7 +3228,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<10 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<10 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3274,7 +3274,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3320,7 +3320,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3362,7 +3362,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3397,7 +3397,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3432,7 +3432,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3479,7 +3479,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3519,7 +3519,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3559,7 +3559,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3603,7 +3603,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3640,7 +3640,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3677,7 +3677,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3726,7 +3726,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3768,7 +3768,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3810,7 +3810,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3854,7 +3854,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3891,7 +3891,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3928,7 +3928,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -3979,7 +3979,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4023,7 +4023,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4067,7 +4067,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4113,7 +4113,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[CONCAT_VECTORS]](<8 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4152,7 +4152,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4191,7 +4191,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4244,7 +4244,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<10 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<10 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4290,7 +4290,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4336,7 +4336,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4372,7 +4372,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4402,7 +4402,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4432,7 +4432,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX11-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4473,7 +4473,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4508,7 +4508,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4543,7 +4543,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4582,7 +4582,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4615,7 +4615,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4648,7 +4648,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4691,7 +4691,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4727,7 +4727,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4763,7 +4763,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4796,7 +4796,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GFX9-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4823,7 +4823,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GFX10-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX10-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4850,7 +4850,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GFX11-NEXT:   [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; GFX11-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY12]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[TRUNC]](s16), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4886,7 +4886,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX9-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4916,7 +4916,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4946,7 +4946,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX11-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY13]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -4984,7 +4984,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5016,7 +5016,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5048,7 +5048,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5087,7 +5087,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX9-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5120,7 +5120,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5153,7 +5153,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY12]](s32)
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5208,7 +5208,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (s32), addrspace 8)
   ; GFX9-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX9-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX10-LABEL: name: sample_c_d_o_2darray_V1
@@ -5253,7 +5253,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (s32), addrspace 8)
   ; GFX10-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX11-LABEL: name: sample_c_d_o_2darray_V1
@@ -5298,7 +5298,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (s32), addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -5349,7 +5349,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
   ; GFX9-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX9-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX9-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX9-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX9-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX9-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5396,7 +5396,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
   ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX10-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -5443,7 +5443,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
   ; GFX11-NEXT:   [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
   ; GFX11-NEXT:   [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.d.ll
index 309dc5a737ce0..17ed5cbee74dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.d.ll
@@ -31,7 +31,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr7
   ; GFX10-NEXT:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr8
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<9 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<9 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<9 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -66,7 +66,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr7
   ; GFX11-NEXT:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr8
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<5 x s32>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -108,7 +108,7 @@ define amdgpu_ps <4 x float> @sample_c_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr8
   ; GFX10-NEXT:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr9
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<10 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.3d), 15, [[BUILD_VECTOR2]](<10 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.3d), 15, [[BUILD_VECTOR2]](<10 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -144,7 +144,7 @@ define amdgpu_ps <4 x float> @sample_c_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr8
   ; GFX11-NEXT:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr9
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<6 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<6 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -187,7 +187,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_3d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr9
   ; GFX10-NEXT:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr10
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<11 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.3d), 15, [[BUILD_VECTOR2]](<11 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.3d), 15, [[BUILD_VECTOR2]](<11 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -224,7 +224,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_3d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr9
   ; GFX11-NEXT:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr10
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<7 x s32>) = G_BUILD_VECTOR [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<7 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<7 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -268,7 +268,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_o_3d(<8 x i32> inreg %rsrc, <4 x i32
   ; GFX10-NEXT:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr10
   ; GFX10-NEXT:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr11
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<12 x s32>) = G_BUILD_VECTOR [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.o.3d), 15, [[BUILD_VECTOR2]](<12 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.o.3d), 15, [[BUILD_VECTOR2]](<12 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -306,7 +306,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_o_3d(<8 x i32> inreg %rsrc, <4 x i32
   ; GFX11-NEXT:   [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr10
   ; GFX11-NEXT:   [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr11
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.o.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<8 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.o.3d), 15, [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[BUILD_VECTOR2]](<8 x s32>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
index 3a26e7b62df15..72c7dcfc28cbc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
@@ -31,7 +31,7 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -66,7 +66,7 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -112,7 +112,7 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -152,7 +152,7 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC4]](s16), [[TRUNC5]](s16)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -209,7 +209,7 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
   ; GFX10-NEXT:   [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[TRUNC7]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC8]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[BUILD_VECTOR6]](<2 x s16>), [[BUILD_VECTOR7]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -260,7 +260,7 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
   ; GFX11-NEXT:   [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC6]](s16), [[TRUNC7]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC8]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR6]](<2 x s16>), [[BUILD_VECTOR7]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
index d75a33c5a8bd6..0119a28305400 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
@@ -30,7 +30,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -64,7 +64,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -109,7 +109,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -148,7 +148,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -203,7 +203,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY20]](s32)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<14 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[CONCAT_VECTORS]](<14 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -252,7 +252,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY20]](s32)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BUILD_VECTOR4]](<2 x s16>), [[BUILD_VECTOR5]](<2 x s16>), [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -294,7 +294,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -330,7 +330,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -377,7 +377,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -418,7 +418,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -460,7 +460,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -496,7 +496,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -543,7 +543,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -584,7 +584,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -628,7 +628,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -666,7 +666,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -716,7 +716,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
   ; GFX10-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -760,7 +760,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
   ; GFX11-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -800,7 +800,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -834,7 +834,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -879,7 +879,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -918,7 +918,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -960,7 +960,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -996,7 +996,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1043,7 +1043,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1084,7 +1084,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[TRUNC3]](s16)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1126,7 +1126,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1162,7 +1162,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY14]](s32)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1209,7 +1209,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1250,7 +1250,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
   ; GFX11-NEXT:   [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY17]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1294,7 +1294,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1332,7 +1332,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC1]](s16), [[DEF]](s16)
   ; GFX11-NEXT:   [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY15]](s32)
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY16]](s32)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1382,7 +1382,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
   ; GFX10-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[CONCAT_VECTORS]](<12 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1426,7 +1426,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY18]](s32)
   ; GFX11-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[CONCAT_VECTORS]](<4 x s16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1478,7 +1478,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX10-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX10-NEXT:   [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY20]](s32)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<14 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[CONCAT_VECTORS]](<14 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (s32), addrspace 8)
   ; GFX10-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX10-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; GFX11-LABEL: name: sample_c_d_o_2darray_V1
@@ -1520,7 +1520,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
   ; GFX11-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX11-NEXT:   [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY20]](s32)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (s32), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (s32), addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](s32)
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
 main_body:
@@ -1568,7 +1568,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
   ; GFX10-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX10-NEXT:   [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY20]](s32)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
-  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<14 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX10-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[CONCAT_VECTORS]](<14 x s16>), $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX10-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX10-NEXT:   $vgpr1 = COPY [[UV1]](s32)
@@ -1612,7 +1612,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
   ; GFX11-NEXT:   [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY19]](s32)
   ; GFX11-NEXT:   [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY20]](s32)
   ; GFX11-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
-  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x s32>), addrspace 7)
+  ; GFX11-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>), [[CONCAT_VECTORS]](<6 x s16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x s32>), addrspace 8)
   ; GFX11-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x s32>)
   ; GFX11-NEXT:   $vgpr0 = COPY [[UV]](s32)
   ; GFX11-NEXT:   $vgpr1 = COPY [[UV1]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
index 091f30c7df5b1..6a67651ab785e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
@@ -24,7 +24,7 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
   ; UNPACKED-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 7)
+  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; GFX81-LABEL: name: image_store_f16
   ; GFX81: bb.1 (%ir-block.0):
@@ -44,7 +44,7 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
   ; GFX81-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX81-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX81-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 7)
+  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 8)
   ; GFX81-NEXT:   S_ENDPGM 0
   ; GFX9-LABEL: name: image_store_f16
   ; GFX9: bb.1 (%ir-block.0):
@@ -64,7 +64,7 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
   ; GFX9-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX9-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10-LABEL: name: image_store_f16
   ; GFX10: bb.1 (%ir-block.0):
@@ -84,7 +84,7 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr2
   ; GFX10-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY10]](s32)
   ; GFX10-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 7)
+  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[TRUNC]](s16), 1, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (s16), addrspace 8)
   ; GFX10-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
@@ -112,7 +112,7 @@ define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; UNPACKED-NEXT:   [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
   ; UNPACKED-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32)
-  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<2 x s32>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<2 x s32>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; GFX81-LABEL: name: image_store_v2f16
   ; GFX81: bb.1 (%ir-block.0):
@@ -134,7 +134,7 @@ define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX81-NEXT:   [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x s16>)
   ; GFX81-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX81-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[DEF]](s32)
-  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<2 x s32>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 7)
+  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<2 x s32>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 8)
   ; GFX81-NEXT:   S_ENDPGM 0
   ; GFX9-LABEL: name: image_store_v2f16
   ; GFX9: bb.1 (%ir-block.0):
@@ -153,7 +153,7 @@ define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX9-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX9-NEXT:   [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](<2 x s16>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](<2 x s16>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10-LABEL: name: image_store_v2f16
   ; GFX10: bb.1 (%ir-block.0):
@@ -172,7 +172,7 @@ define amdgpu_ps void @image_store_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX10-NEXT:   [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr1
   ; GFX10-NEXT:   [[COPY10:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
   ; GFX10-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](<2 x s16>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 7)
+  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](<2 x s16>), 3, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<2 x s16>), addrspace 8)
   ; GFX10-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.image.store.2d.v2f16.i32(<2 x half> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
@@ -202,7 +202,7 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; UNPACKED-NEXT:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x s16>)
   ; UNPACKED-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
   ; UNPACKED-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32)
-  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<3 x s32>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 7)
+  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<3 x s32>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; GFX81-LABEL: name: image_store_v3f16
   ; GFX81: bb.1 (%ir-block.0):
@@ -241,7 +241,7 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX81-NEXT:   [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](s32)
   ; GFX81-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
   ; GFX81-NEXT:   [[BITCAST5:%[0-9]+]]:_(<3 x s32>) = G_BITCAST [[CONCAT_VECTORS]](<6 x s16>)
-  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST5]](<3 x s32>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 7)
+  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST5]](<3 x s32>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 8)
   ; GFX81-NEXT:   S_ENDPGM 0
   ; GFX9-LABEL: name: image_store_v3f16
   ; GFX9: bb.1 (%ir-block.0):
@@ -272,7 +272,7 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX9-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX9-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10-LABEL: name: image_store_v3f16
   ; GFX10: bb.1 (%ir-block.0):
@@ -303,7 +303,7 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX10-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
   ; GFX10-NEXT:   [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC2]](s16), [[DEF]](s16)
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x s16>), [[BUILD_VECTOR3]](<2 x s16>)
-  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 7)
+  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 7, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<3 x s16>), align 8, addrspace 8)
   ; GFX10-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half> %in, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
@@ -334,7 +334,7 @@ define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; UNPACKED-NEXT:   [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY11]](<2 x s16>)
   ; UNPACKED-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
   ; UNPACKED-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[BITCAST]](s32), [[LSHR]](s32), [[BITCAST1]](s32), [[LSHR1]](s32)
-  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<4 x s32>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 7)
+  ; UNPACKED-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<4 x s32>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; GFX81-LABEL: name: image_store_v4f16
   ; GFX81: bb.1 (%ir-block.0):
@@ -359,7 +359,7 @@ define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX81-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
   ; GFX81-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; GFX81-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[DEF]](s32), [[DEF]](s32)
-  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<4 x s32>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 7)
+  ; GFX81-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR2]](<4 x s32>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 8)
   ; GFX81-NEXT:   S_ENDPGM 0
   ; GFX9-LABEL: name: image_store_v4f16
   ; GFX9: bb.1 (%ir-block.0):
@@ -380,7 +380,7 @@ define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX9-NEXT:   [[COPY11:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
   ; GFX9-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY10]](<2 x s16>), [[COPY11]](<2 x s16>)
   ; GFX9-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 7)
+  ; GFX9-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 8)
   ; GFX9-NEXT:   S_ENDPGM 0
   ; GFX10-LABEL: name: image_store_v4f16
   ; GFX10: bb.1 (%ir-block.0):
@@ -401,7 +401,7 @@ define amdgpu_ps void @image_store_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
   ; GFX10-NEXT:   [[COPY11:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
   ; GFX10-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY10]](<2 x s16>), [[COPY11]](<2 x s16>)
   ; GFX10-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32)
-  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 7)
+  ; GFX10-NEXT:   G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[CONCAT_VECTORS]](<4 x s16>), 15, [[BUILD_VECTOR1]](<2 x s32>), $noreg, [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable store (<4 x s16>), addrspace 8)
   ; GFX10-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.image.store.2d.v4f16.i32(<4 x half> %in, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir
index 7e5b3fe3be7f3..c2d557446a537 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.mir
@@ -20,7 +20,7 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si]].sub0
     ; GFX6-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX6-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -30,7 +30,7 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi]].sub0
     ; GFX8-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX8-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -40,7 +40,7 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_]].sub0
     ; GFX10-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -50,14 +50,14 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_]].sub0
     ; GFX11-NEXT: $vgpr0 = COPY [[COPY3]]
     ; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
     %2:vgpr(s32) = COPY $vgpr2
-    %3:vgpr(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s32>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s32), addrspace 7)
+    %3:vgpr(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s32>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s32), addrspace 8)
     $vgpr0 = COPY %3(s32)
     SI_RETURN_TO_EPILOG implicit $vgpr0
 ...
@@ -78,7 +78,7 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_si:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_si [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX6-NEXT: S_ENDPGM 0
     ; GFX8-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX8: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -86,7 +86,7 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_vi:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 3, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX8-NEXT: S_ENDPGM 0
     ; GFX10-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX10: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -94,7 +94,7 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX10-NEXT: S_ENDPGM 0
     ; GFX11-LABEL: name: atomic_cmpswap_i32_1d_no_return
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1, $vgpr2
@@ -102,12 +102,12 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 7)
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11_:%[0-9]+]]:vreg_64 = IMAGE_ATOMIC_CMPSWAP_V1_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 3, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), addrspace 8)
     ; GFX11-NEXT: S_ENDPGM 0
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
     %2:vgpr(s32) = COPY $vgpr2
-    %3:vgpr(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s32>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s32), addrspace 7)
+    %3:vgpr(s32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s32>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s32), addrspace 8)
     S_ENDPGM 0
 ...
 
@@ -127,7 +127,7 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX6-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si]].sub0_sub1
     ; GFX6-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX6-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
@@ -137,7 +137,7 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX8-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi]].sub0_sub1
     ; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX8-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
@@ -147,7 +147,7 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX10-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_]].sub0_sub1
     ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX10-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
@@ -157,14 +157,14 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY killed [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_]].sub0_sub1
     ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[COPY3]]
     ; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:vgpr(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %2:vgpr(s32) = COPY $vgpr4
-    %3:vgpr(s64) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s64>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s64), addrspace 7)
+    %3:vgpr(s64) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s64>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s64), addrspace 8)
     $vgpr0_vgpr1 = COPY %3(s64)
     SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
 ...
@@ -185,7 +185,7 @@ body: |
     ; GFX6-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX6-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_si:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_si [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX6-NEXT: S_ENDPGM 0
     ; GFX8-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX8: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -193,7 +193,7 @@ body: |
     ; GFX8-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX8-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX8-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_vi:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_vi [[COPY1]], [[COPY2]], [[COPY]], 15, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX8-NEXT: S_ENDPGM 0
     ; GFX10-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX10: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -201,7 +201,7 @@ body: |
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX10-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx10 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX10-NEXT: S_ENDPGM 0
     ; GFX11-LABEL: name: atomic_cmpswap_i64_1d_no_return
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -209,11 +209,11 @@ body: |
     ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
+    ; GFX11-NEXT: [[IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11_:%[0-9]+]]:vreg_128 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 [[COPY1]], [[COPY2]], [[COPY]], 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 8)
     ; GFX11-NEXT: S_ENDPGM 0
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:vgpr(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %2:vgpr(s32) = COPY $vgpr4
-    %3:vgpr(s64) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s64>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s64), addrspace 7)
+    %3:vgpr(s64) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), %1(<2 x s64>), $noreg, %2(s32), %0(<8 x s32>), 0, 0, 0 :: (volatile dereferenceable load store (s64), addrspace 8)
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll
index 8710c3fc86e0a..dfe45f52bdb37 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll
@@ -15,7 +15,7 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_vof
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_OFFEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -36,7 +36,7 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32_noret__vgpr_val__sgpr_rsrc__vg
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_OFFEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -59,7 +59,7 @@ define amdgpu_ps <2 x float> @raw_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vg
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_X2_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_X2_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_OFFEN_RTN]].sub0
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_OFFEN_RTN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY8]]
@@ -85,7 +85,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgp
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_X2_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_X2_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.add.i64(i64 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -133,7 +133,7 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_vof
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -192,7 +192,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgp
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -220,7 +220,7 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_vof
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_OFFEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
@@ -243,7 +243,7 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_vof
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_OFFEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll
index b60bccafdff0c..5578a533c02be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll
@@ -17,7 +17,7 @@ define amdgpu_ps float @raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN]].sub0
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY8]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -42,7 +42,7 @@ define amdgpu_ps void @raw_buffer_atomic_cmpswap_i32_noret__vgpr_val__vgpr_cmp__
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -93,7 +93,7 @@ define amdgpu_ps float @raw_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN]].sub0
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
@@ -156,7 +156,7 @@ define amdgpu_ps void @raw_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -186,7 +186,7 @@ define amdgpu_ps float @raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN]].sub0
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY8]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll
index cb56d83623dbc..14e76bd39fc28 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll
@@ -4,7 +4,7 @@
 declare float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i32 immarg)
 declare <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
 
-; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, 0 :: (volatile dereferenceable load store (s32), align 1, addrspace 7) (in function: buffer_atomic_add_f32_rtn)
+; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, 0 :: (volatile dereferenceable load store (s32), align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
 
 ; GFX90A-LABEL: {{^}}buffer_atomic_add_f32_rtn:
 ; GFX90A: buffer_atomic_add_f32 v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9:]+}}], s{{[0-9]+}} offen glc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd.ll
index fc43734903a2a..682507dcbc5ae 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -30,7 +30,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -49,7 +49,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_plus4095__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -63,7 +63,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -82,7 +82,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_4095__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -95,7 +95,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX90A-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -114,7 +114,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_v
   ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -127,7 +127,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_v
   ; GFX90A-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
@@ -175,7 +175,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__sgp
   ; GFX908-NEXT: bb.3:
   ; GFX908-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX908-NEXT: {{  $}}
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX908-NEXT: {{  $}}
@@ -226,7 +226,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__sgp
   ; GFX90A-NEXT: bb.3:
   ; GFX90A-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX90A-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX90A-NEXT: {{  $}}
@@ -281,7 +281,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__0_v
   ; GFX908-NEXT: bb.3:
   ; GFX908-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX908-NEXT: {{  $}}
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX908-NEXT: {{  $}}
@@ -330,7 +330,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__0_v
   ; GFX90A-NEXT: bb.3:
   ; GFX90A-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX90A-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX90A-NEXT: {{  $}}
@@ -358,7 +358,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
   ; GFX90A: bb.1 (%ir-block.0):
@@ -372,7 +372,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %voffset = add i32 %voffset.base, 4095
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -393,7 +393,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
   ; GFX90A: bb.1 (%ir-block.0):
@@ -407,7 +407,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
   ret void
@@ -426,7 +426,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__v
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -440,7 +440,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__v
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -458,7 +458,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0
   ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -471,7 +471,7 @@ define amdgpu_ps void @raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0
   ; GFX90A-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
index ee3fe32daa7de..8e19f175a6575 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
@@ -15,7 +15,7 @@ define amdgpu_ps half @raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; UNPACKED-LABEL: name: raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -29,7 +29,7 @@ define amdgpu_ps half @raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -48,7 +48,7 @@ define amdgpu_ps <2 x half> @raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; UNPACKED-LABEL: name: raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -62,7 +62,7 @@ define amdgpu_ps <2 x half> @raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
   ; UNPACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
@@ -98,7 +98,7 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub0
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub1
   ; PACKED-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -115,7 +115,7 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
@@ -183,7 +183,7 @@ define amdgpu_ps half @raw_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__vgpr
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -233,7 +233,7 @@ define amdgpu_ps half @raw_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__vgpr
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -261,7 +261,7 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub0
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub1
   ; PACKED-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -278,7 +278,7 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.ll
index 5ba8a6b834648..8e2662530a486 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.ll
@@ -14,7 +14,7 @@ define amdgpu_ps float @raw_buffer_load_format_f32__sgpr_rsrc__vgpr_voffset__sgp
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -33,7 +33,7 @@ define amdgpu_ps <2 x float> @raw_buffer_load_format_v2f32__sgpr_rsrc__vgpr_voff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XY_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XY_OFFEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -55,7 +55,7 @@ define amdgpu_ps <3 x float> @raw_buffer_load_format_v3f32__sgpr_rsrc__vgpr_voff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub2
@@ -79,7 +79,7 @@ define amdgpu_ps <4 x float> @raw_buffer_load_format_v4f32__sgpr_rsrc__vgpr_voff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
@@ -133,7 +133,7 @@ define amdgpu_ps float @raw_buffer_load_format_f32__vgpr_rsrc__sgpr_voffset__vgp
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -161,7 +161,7 @@ define amdgpu_ps <4 x float> @raw_buffer_load_format_v4f32__sgpr_rsrc__vgpr_voff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
index 7e72873dfe972..093e11481062e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
@@ -15,7 +15,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -36,7 +36,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -79,7 +79,7 @@ define amdgpu_ps float @raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -134,7 +134,7 @@ define amdgpu_ps float @raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffs
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -163,7 +163,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 1)
@@ -183,7 +183,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
@@ -203,7 +203,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 4)
@@ -223,7 +223,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 6)
@@ -243,7 +243,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 5)
@@ -263,7 +263,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 7)
@@ -283,7 +283,7 @@ define amdgpu_ps <2 x float> @raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sg
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -305,7 +305,7 @@ define amdgpu_ps <3 x float> @raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sg
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX3_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX3_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub2
@@ -329,7 +329,7 @@ define amdgpu_ps <4 x float> @raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sg
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
@@ -355,7 +355,7 @@ define amdgpu_ps half @raw_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -374,7 +374,7 @@ define amdgpu_ps <2 x half> @raw_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgp
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -399,7 +399,7 @@ define amdgpu_ps <4 x half> @raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgp
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -421,7 +421,7 @@ define amdgpu_ps float @raw_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -442,7 +442,7 @@ define amdgpu_ps float @raw_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_SBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_SBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -463,7 +463,7 @@ define amdgpu_ps float @raw_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -484,7 +484,7 @@ define amdgpu_ps float @raw_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_SSHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_SSHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -529,7 +529,7 @@ define amdgpu_ps half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -581,7 +581,7 @@ define amdgpu_ps float @raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -610,7 +610,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
@@ -628,7 +628,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[COPY4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], [[COPY4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
@@ -648,7 +648,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY5]], [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY5]], [[REG_SEQUENCE]], [[COPY4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4096, i32 %soffset, i32 0)
@@ -667,7 +667,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 16
@@ -687,7 +687,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
@@ -710,7 +710,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4096
@@ -730,7 +730,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
@@ -749,7 +749,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
@@ -770,7 +770,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
   ; CHECK-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %soffset = add i32 %soffset.base, 16
@@ -792,7 +792,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
   ; CHECK-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %soffset = add i32 %soffset.base, 4095
@@ -814,7 +814,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %soffset = add i32 %soffset.base, 4096
@@ -860,7 +860,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -916,7 +916,7 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE1]], [[COPY5]], 904, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE1]], [[COPY5]], 904, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index 6043ab3d71811..89bfb53abda43 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -15,7 +15,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
   ; PACKED: bb.1 (%ir-block.0):
@@ -29,7 +29,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -47,7 +47,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
   ; PACKED: bb.1 (%ir-block.0):
@@ -60,7 +60,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -83,7 +83,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
   ; PACKED: bb.1 (%ir-block.0):
@@ -97,7 +97,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -123,7 +123,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
   ; PACKED: bb.1 (%ir-block.0):
@@ -139,7 +139,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -189,7 +189,7 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -237,7 +237,7 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -269,7 +269,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
   ; PACKED: bb.1 (%ir-block.0):
@@ -283,7 +283,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
   ret void
@@ -306,7 +306,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
   ; PACKED: bb.1 (%ir-block.0):
@@ -320,7 +320,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
   ret void
@@ -343,7 +343,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
   ; PACKED: bb.1 (%ir-block.0):
@@ -357,7 +357,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 16
   call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -381,7 +381,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
   ; PACKED: bb.1 (%ir-block.0):
@@ -395,7 +395,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -422,7 +422,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
   ; PACKED: bb.1 (%ir-block.0):
@@ -439,7 +439,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; PACKED-NEXT:   [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4096
   call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -494,7 +494,7 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -545,7 +545,7 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
index 35bd42cb7f7e7..63943a3779379 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
@@ -15,7 +15,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -33,7 +33,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -54,7 +54,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -76,7 +76,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -99,7 +99,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -145,7 +145,7 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -175,7 +175,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
   ret void
@@ -196,7 +196,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
   ret void
@@ -217,7 +217,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 16
   call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -239,7 +239,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -264,7 +264,7 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4096
   call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -316,7 +316,7 @@ define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffse
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index 802911f6f429a..0e28ad09e0b7f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -38,7 +38,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr8
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY7]], [[COPY8]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY7]], [[COPY8]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -81,7 +81,7 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -124,7 +124,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -179,7 +179,7 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -207,7 +207,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 1)
   ret void
@@ -226,7 +226,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
   ret void
@@ -245,7 +245,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 3, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 3, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 3)
   ret void
@@ -264,7 +264,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 4, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 4, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 4)
   ret void
@@ -283,7 +283,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 6, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 6, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 6)
   ret void
@@ -302,7 +302,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 5, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 5, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 5)
   ret void
@@ -321,7 +321,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 7, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 7, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 7)
   ret void
@@ -342,7 +342,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -364,7 +364,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -387,7 +387,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -406,7 +406,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_BYTE_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_BYTE_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %val.trunc = trunc i32 %val to i8
   call void @llvm.amdgcn.raw.buffer.store.i8(i8 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -426,7 +426,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %val.trunc = trunc i32 %val to i16
   call void @llvm.amdgcn.raw.buffer.store.i16(i16 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -446,7 +446,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -465,7 +465,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -486,7 +486,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -530,7 +530,7 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -557,7 +557,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_v
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -577,7 +577,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 0)
   ret void
@@ -596,7 +596,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 16
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -616,7 +616,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -639,7 +639,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4096
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -659,7 +659,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
   ret void
@@ -678,7 +678,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
   ret void
@@ -697,7 +697,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 16
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -717,7 +717,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -740,7 +740,7 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, dead [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64 = V_ADD_CO_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4096
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
@@ -787,7 +787,7 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE1]], [[COPY6]], 904, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE1]], [[COPY6]], 904, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -841,7 +841,7 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY5]], 904, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY5]], 904, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
index 37d30ffc41824..cdc068ae33da0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
@@ -14,7 +14,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -28,7 +28,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -47,7 +47,7 @@ define amdgpu_ps <2 x half> @raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sg
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
   ; UNPACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
@@ -72,7 +72,7 @@ define amdgpu_ps <2 x half> @raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sg
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -97,7 +97,7 @@ define amdgpu_ps <4 x half> @raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sg
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
@@ -132,7 +132,7 @@ define amdgpu_ps <4 x half> @raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sg
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub0
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub1
   ; PACKED-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -181,7 +181,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffs
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -231,7 +231,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffs
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -259,7 +259,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
@@ -273,7 +273,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
@@ -292,7 +292,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
@@ -306,7 +306,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
@@ -325,7 +325,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
@@ -339,7 +339,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
@@ -358,7 +358,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_OFFEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
@@ -372,7 +372,7 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll
index a1b8d711c915b..1549828ffd853 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll
@@ -14,7 +14,7 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -33,7 +33,7 @@ define amdgpu_ps <2 x float> @raw_tbuffer_load_v2f32__sgpr_rsrc__vgpr_voffset__s
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_XY_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_OFFEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY6]]
@@ -55,7 +55,7 @@ define amdgpu_ps <3 x float> @raw_tbuffer_load_v3f32__sgpr_rsrc__vgpr_voffset__s
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub2
@@ -79,7 +79,7 @@ define amdgpu_ps <4 x float> @raw_tbuffer_load_v4f32__sgpr_rsrc__vgpr_voffset__s
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
@@ -132,7 +132,7 @@ define amdgpu_ps float @raw_tbuffer_load_f32__vgpr_rsrc__sgpr_voffset__vgpr_soff
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -160,7 +160,7 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
@@ -179,7 +179,7 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
@@ -198,7 +198,7 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
@@ -217,7 +217,7 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 567ce5838a521..9d998e5a5f5e0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -15,7 +15,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -29,7 +29,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -52,7 +52,7 @@ define amdgpu_ps void @raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -66,7 +66,7 @@ define amdgpu_ps void @raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -98,7 +98,7 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY1]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -114,7 +114,7 @@ define amdgpu_ps void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -157,7 +157,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -203,7 +203,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -258,7 +258,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -307,7 +307,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -363,7 +363,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -413,7 +413,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -441,7 +441,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
   ; PACKED: bb.1 (%ir-block.0):
@@ -455,7 +455,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
   ret void
@@ -474,7 +474,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
   ; PACKED: bb.1 (%ir-block.0):
@@ -488,7 +488,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
   ret void
@@ -507,7 +507,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
   ; PACKED: bb.1 (%ir-block.0):
@@ -521,7 +521,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
   ret void
@@ -540,7 +540,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_gfx80_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
   ; PACKED: bb.1 (%ir-block.0):
@@ -554,7 +554,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
index 8cf87683a237d..299bf6a45d763 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
@@ -15,7 +15,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -29,7 +29,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
   ret void
@@ -72,7 +72,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -118,7 +118,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -173,7 +173,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -222,7 +222,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -278,7 +278,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
   ; UNPACKED-NEXT: bb.3:
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
-  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; UNPACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -328,7 +328,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
   ; PACKED-NEXT: bb.3:
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
-  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; PACKED-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
index b7c8eb8458209..0e89af6d3aabb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -38,7 +38,7 @@ define amdgpu_ps void @raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -61,7 +61,7 @@ define amdgpu_ps void @raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE]], [[COPY7]], [[REG_SEQUENCE1]], [[COPY8]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE]], [[COPY7]], [[REG_SEQUENCE1]], [[COPY8]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -85,7 +85,7 @@ define amdgpu_ps void @raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE]], [[COPY8]], [[REG_SEQUENCE1]], [[COPY9]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE]], [[COPY8]], [[REG_SEQUENCE1]], [[COPY9]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
   ret void
@@ -106,7 +106,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr7
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
   ret void
@@ -149,7 +149,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -204,7 +204,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soff
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -260,7 +260,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soff
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -289,7 +289,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
   ret void
@@ -309,7 +309,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
   ret void
@@ -329,7 +329,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
   ret void
@@ -349,7 +349,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
   ret void
@@ -369,7 +369,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 94, i32 0)
   ret void
@@ -387,7 +387,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], [[REG_SEQUENCE]], [[COPY5]], 4095, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 94, i32 0)
   ret void
@@ -407,7 +407,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 94, i32 0)
   ret void
@@ -426,7 +426,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset = add i32 %voffset.base, 16
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
@@ -446,7 +446,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset = add i32 %voffset.base, 4095
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
@@ -469,7 +469,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %voffset = add i32 %voffset.base, 4096
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
@@ -489,7 +489,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 94, i32 0)
   ret void
@@ -508,7 +508,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 94, i32 0)
   ret void
@@ -529,7 +529,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
   ; CHECK-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %soffset = add i32 %soffset.base, 16
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
@@ -551,7 +551,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
   ; CHECK-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %soffset = add i32 %soffset.base, 4095
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
@@ -573,7 +573,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
   ; CHECK-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY6]], [[S_MOV_B32_]], implicit-def $scc
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %soffset = add i32 %soffset.base, 4096
   call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
@@ -619,7 +619,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -675,7 +675,7 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE1]], [[COPY6]], 904, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE1]], [[COPY6]], 904, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll
index 29a35bfcdb764..99dfe9fb95cfe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll
@@ -17,7 +17,7 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -40,7 +40,7 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32_noret__vgpr_val__sgpr_rsrc_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -65,7 +65,7 @@ define amdgpu_ps <2 x float> @struct_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc_
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_X2_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_X2_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_BOTHEN_RTN]].sub0
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_BOTHEN_RTN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY9]]
@@ -93,7 +93,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_X2_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_X2_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -144,7 +144,7 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -206,7 +206,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -237,7 +237,7 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_ADD_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll
index c994dfdeb5a38..dab8d6f8c421c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll
@@ -20,7 +20,7 @@ define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sg
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN]].sub0
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY9]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
@@ -47,7 +47,7 @@ define amdgpu_ps void @struct_buffer_atomic_cmpswap_noret_i32__vgpr_val__vgpr_cm
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -101,7 +101,7 @@ define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vg
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN]].sub0
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
@@ -167,7 +167,7 @@ define amdgpu_ps void @struct_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cm
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -199,7 +199,7 @@ define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sg
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN]].sub0
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY9]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll
index 85cf4781d5cb4..a85e82d74c8ae 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
 ; RUN: not --crash llc -global-isel < %s -march=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
 
-; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (s32), align 1, addrspace 7) (in function: buffer_atomic_add_f32_rtn)
+; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (s32), align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
 
 declare float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i32, i32 immarg)
 declare <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd.ll
index 54a3e45658535..ec50e65845392 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd.ll
@@ -18,7 +18,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -34,7 +34,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX90A-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -55,7 +55,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_plus4095__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -71,7 +71,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX90A-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
@@ -91,7 +91,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__4095_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -105,7 +105,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -125,7 +125,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -139,7 +139,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -190,7 +190,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__
   ; GFX908-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX908-NEXT: {{  $}}
@@ -244,7 +244,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__
   ; GFX90A-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX90A-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX90A-NEXT: {{  $}}
@@ -301,7 +301,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__
   ; GFX908-NEXT: bb.3:
   ; GFX908-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX908-NEXT: {{  $}}
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX908-NEXT: {{  $}}
@@ -352,7 +352,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__sgpr_val__vgpr_rsrc__
   ; GFX90A-NEXT: bb.3:
   ; GFX90A-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GFX90A-NEXT: {{  $}}
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GFX90A-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GFX90A-NEXT: {{  $}}
@@ -383,7 +383,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
   ; GFX90A: bb.1 (%ir-block.0):
@@ -399,7 +399,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX90A-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
@@ -418,7 +418,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset_slc
   ; GFX90A: bb.1 (%ir-block.0):
@@ -432,7 +432,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 2)
   ret void
@@ -453,7 +453,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -469,7 +469,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; GFX90A-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; GFX90A-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -488,7 +488,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: struct_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset
   ; GFX90A: bb.1 (%ir-block.0):
@@ -502,7 +502,7 @@ define amdgpu_ps void @struct_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc
   ; GFX90A-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
   ; GFX90A-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GFX90A-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 7)
+  ; GFX90A-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (<2 x s16>), align 1, addrspace 8)
   ; GFX90A-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
index ce4490281c368..b93f6256f6fca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
@@ -16,7 +16,7 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -32,7 +32,7 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.struct.buffer.load.format.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -53,7 +53,7 @@ define amdgpu_ps <2 x half> @struct_buffer_load_format_v2f16__sgpr_rsrc__vgpr_vi
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub0
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub1
   ; UNPACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
@@ -80,7 +80,7 @@ define amdgpu_ps <2 x half> @struct_buffer_load_format_v2f16__sgpr_rsrc__vgpr_vi
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call <2 x half> @llvm.amdgcn.struct.buffer.load.format.v2f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -107,7 +107,7 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__sgpr_rsrc__vgpr_vi
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
@@ -144,7 +144,7 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__sgpr_rsrc__vgpr_vi
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN]].sub0
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN]].sub1
   ; PACKED-NEXT:   $vgpr0 = COPY [[COPY7]]
@@ -197,7 +197,7 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__vpr_rsrc__sgpr_vin
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
   ; UNPACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -271,7 +271,7 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__vpr_rsrc__sgpr_vin
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
   ; PACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -304,7 +304,7 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffsset_add_4095
@@ -320,7 +320,7 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
@@ -342,7 +342,7 @@ define amdgpu_ps half @struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vg
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; PACKED-LABEL: name: struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -358,7 +358,7 @@ define amdgpu_ps half @struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vg
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.struct.buffer.load.format.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
index 48565a00e2889..cb356f4761ad2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
@@ -15,7 +15,7 @@ define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -36,7 +36,7 @@ define amdgpu_ps <2 x float> @struct_buffer_load_format_v2f32__sgpr_rsrc__vgpr_v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XY_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XY_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XY_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XY_BOTHEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY7]]
@@ -60,7 +60,7 @@ define amdgpu_ps <3 x float> @struct_buffer_load_format_v3f32__sgpr_rsrc__vgpr_v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_BOTHEN]].sub1
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_BOTHEN]].sub2
@@ -86,7 +86,7 @@ define amdgpu_ps <4 x float> @struct_buffer_load_format_v4f32__sgpr_rsrc__vgpr_v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]].sub1
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]].sub2
@@ -143,7 +143,7 @@ define amdgpu_ps <4 x float> @struct_buffer_load_format_v4f32__vpr_rsrc__sgpr_vi
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -180,7 +180,7 @@ define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
@@ -202,7 +202,7 @@ define amdgpu_ps float @struct_buffer_load_format_i32__sgpr_rsrc__vgpr_vindex__v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i32 @llvm.amdgcn.struct.buffer.load.format.i32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -228,7 +228,7 @@ define amdgpu_cs void @struct_buffer_load_format_v4i32_tfe(<4 x i32> inreg %rsrc
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub0
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub1
   ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_TFE_IDXEN]].sub2
@@ -267,7 +267,7 @@ define amdgpu_cs void @struct_buffer_load_format_v3i32_tfe(<4 x i32> inreg %rsrc
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub0
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub1
   ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_TFE_IDXEN]].sub2
@@ -305,7 +305,7 @@ define amdgpu_cs void @struct_buffer_load_format_i32_tfe(<4 x i32> inreg %rsrc,
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_X_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_TFE_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_X_TFE_IDXEN [[COPY8]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_X_TFE_IDXEN]].sub0
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_X_TFE_IDXEN]].sub1
   ; CHECK-NEXT:   FLAT_STORE_DWORD [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %ir.value, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
index 6ed343e06d843..6c4608be544f0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
@@ -16,7 +16,7 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -38,7 +38,7 @@ define amdgpu_ps <2 x float> @struct_buffer_load_v2f32__sgpr_rsrc__vgpr_vindex__
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY7]]
@@ -63,7 +63,7 @@ define amdgpu_ps <3 x float> @struct_buffer_load_v3f32__sgpr_rsrc__vgpr_vindex__
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX3_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX3_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_BOTHEN]].sub1
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_BOTHEN]].sub2
@@ -90,7 +90,7 @@ define amdgpu_ps <4 x float> @struct_buffer_load_v4f32__sgpr_rsrc__vgpr_vindex__
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub1
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub2
@@ -120,7 +120,7 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %voffset, i32 %soffset, i32 0)
@@ -142,7 +142,7 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
@@ -164,7 +164,7 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 64
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 64, i32 0)
@@ -214,7 +214,7 @@ define amdgpu_ps float @struct_buffer_load_f32__vgpr_rsrc__sgpr_vindex__sgpr_vof
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -244,7 +244,7 @@ define amdgpu_ps float @struct_buffer_load_i8_zext__sgpr_rsrc__vgpr_vindex__vgpr
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_UBYTE_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_UBYTE_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -267,7 +267,7 @@ define amdgpu_ps float @struct_buffer_load_i8_sext__sgpr_rsrc__vgpr_vindex__vgpr
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_SBYTE_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_SBYTE_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -290,7 +290,7 @@ define amdgpu_ps float @struct_buffer_load_i16_zext__sgpr_rsrc__vgpr_vindex__vgp
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -313,7 +313,7 @@ define amdgpu_ps float @struct_buffer_load_i16_sext__sgpr_rsrc__vgpr_vindex__vgp
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_SSHORT_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_SSHORT_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -337,7 +337,7 @@ define amdgpu_ps half @struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voff
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_USHORT_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -359,7 +359,7 @@ define amdgpu_ps <2 x half> @struct_buffer_load_v2f16__sgpr_rsrc__vgpr_vindex__v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -387,7 +387,7 @@ define amdgpu_ps <4 x half> @struct_buffer_load_v4f16__sgpr_rsrc__vgpr_vindex__v
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORDX2_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY7]]
@@ -412,7 +412,7 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index 312bdd6ed14e3..cbaa4c5a99ef1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -17,7 +17,7 @@ define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -33,7 +33,7 @@ define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -58,7 +58,7 @@ define amdgpu_ps void @struct_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc__vg
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
   ; UNPACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: struct_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -74,7 +74,7 @@ define amdgpu_ps void @struct_buffer_store_format_v2f16__vgpr_val__sgpr_rsrc__vg
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XY_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -108,7 +108,7 @@ define amdgpu_ps void @struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vg
   ; UNPACKED-NEXT:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY10]], [[COPY1]], implicit $exec
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
   ; UNPACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -126,7 +126,7 @@ define amdgpu_ps void @struct_buffer_store_format_v4f16__vgpr_val__sgpr_rsrc__vg
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -176,7 +176,7 @@ define amdgpu_ps void @struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
   ; UNPACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -230,7 +230,7 @@ define amdgpu_ps void @struct_buffer_store_format_f16__sgpr_val__vgpr_rsrc__sgpr
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
   ; PACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -260,7 +260,7 @@ define amdgpu_ps void @struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_gfx80_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   S_ENDPGM 0
   ; PACKED-LABEL: name: struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
   ; PACKED: bb.1 (%ir-block.0):
@@ -276,7 +276,7 @@ define amdgpu_ps void @struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   BUFFER_STORE_FORMAT_D16_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.i16(i16 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f32.ll
index e4f432f2b0a62..40b14cd3b7942 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f32.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @struct_buffer_store_format_f32__vgpr_val__sgpr_rsrc__vgpr
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -39,7 +39,7 @@ define amdgpu_ps void @struct_buffer_store_format_v2f32__vgpr_val__sgpr_rsrc__vg
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XY_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -63,7 +63,7 @@ define amdgpu_ps void @struct_buffer_store_format_v3f32__vgpr_val__sgpr_rsrc__vg
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZ_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZ_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -88,7 +88,7 @@ define amdgpu_ps void @struct_buffer_store_format_v4f32__vgpr_val__sgpr_rsrc__vg
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -138,7 +138,7 @@ define amdgpu_ps void @struct_buffer_store_format_f32__sgpr_val__vgpr_rsrc__sgpr
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -168,7 +168,7 @@ define amdgpu_ps void @struct_buffer_store_format_i32__vgpr_val__sgpr_rsrc__vgpr
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_FORMAT_X_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.format.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll
index 289cc6c4f75c9..1f6620fb2104e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll
@@ -17,7 +17,7 @@ define amdgpu_ps void @struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -40,7 +40,7 @@ define amdgpu_ps void @struct_buffer_store_v2f32_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -64,7 +64,7 @@ define amdgpu_ps void @struct_buffer_store_v3f32_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX3_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX3_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -89,7 +89,7 @@ define amdgpu_ps void @struct_buffer_store_v4f32_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -143,7 +143,7 @@ define amdgpu_ps void @struct_buffer_store_v4f32_vgpr_rsrc__sgpr_val__sgpr_vinde
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact [[COPY11]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact [[COPY11]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -173,7 +173,7 @@ define amdgpu_ps void @struct_buffer_store_i8_sgpr_rsrc__vgpr_val__vgpr_vindex__
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_BYTE_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_BYTE_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %val.trunc = trunc i32 %val to i8
   call void @llvm.amdgcn.struct.buffer.store.i8(i8 %val.trunc, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -195,7 +195,7 @@ define amdgpu_ps void @struct_buffer_store_i16_sgpr_rsrc__vgpr_val__vgpr_vindex_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_SHORT_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_SHORT_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   %val.trunc = trunc i32 %val to i16
   call void @llvm.amdgcn.struct.buffer.store.i16(i16 %val.trunc, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -217,7 +217,7 @@ define amdgpu_ps void @struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex_
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 1)
   ret void
@@ -238,7 +238,7 @@ define amdgpu_ps void @struct_buffer_store_v2f16_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -267,7 +267,7 @@ define amdgpu_ps void @struct_buffer_store_v4f16_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 7)
+  ; CHECK-NEXT:   BUFFER_STORE_DWORDX2_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
index 182b321a280c9..29759d3c636f8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
@@ -18,7 +18,7 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -34,7 +34,7 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.struct.tbuffer.load.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -55,7 +55,7 @@ define amdgpu_ps <2 x half> @struct_tbuffer_load_v2f16__sgpr_rsrc__vgpr_vindex__
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; UNPACKED-LABEL: name: struct_tbuffer_load_v2f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -71,7 +71,7 @@ define amdgpu_ps <2 x half> @struct_tbuffer_load_v2f16__sgpr_rsrc__vgpr_vindex__
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub0
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub1
   ; UNPACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
@@ -109,7 +109,7 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN]].sub0
   ; PACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN]].sub1
   ; PACKED-NEXT:   $vgpr0 = COPY [[COPY7]]
@@ -128,7 +128,7 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
   ; UNPACKED-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
   ; UNPACKED-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
@@ -171,7 +171,7 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; PACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
@@ -188,7 +188,7 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; UNPACKED-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.struct.tbuffer.load.f16(<4 x i32> %rsrc, i32 0, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -237,7 +237,7 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
   ; PACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; PACKED-NEXT: {{  $}}
   ; PACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; PACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; PACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; PACKED-NEXT: {{  $}}
@@ -293,7 +293,7 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
   ; UNPACKED-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; UNPACKED-NEXT: {{  $}}
   ; UNPACKED-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; UNPACKED-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; UNPACKED-NEXT: {{  $}}
@@ -344,7 +344,7 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; PACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; PACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; PACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; PACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; PACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
   ; PACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
@@ -360,7 +360,7 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; UNPACKED-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; UNPACKED-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 7)
+  ; UNPACKED-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
   ; UNPACKED-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_gfx80_BOTHEN]]
   ; UNPACKED-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.ll
index 6106d335090c7..526f2e9a2e736 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.ll
@@ -17,7 +17,7 @@ define amdgpu_ps float @struct_tbuffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vo
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -38,7 +38,7 @@ define amdgpu_ps <2 x float> @struct_tbuffer_load_v2f32__sgpr_rsrc__vgpr_vindex_
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XY_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XY_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_XY_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_BOTHEN]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[COPY7]]
@@ -62,7 +62,7 @@ define amdgpu_ps <3 x float> @struct_tbuffer_load_v3f32__sgpr_rsrc__vgpr_vindex_
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZ_BOTHEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZ_BOTHEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_BOTHEN]].sub1
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_BOTHEN]].sub2
@@ -88,7 +88,7 @@ define amdgpu_ps <4 x float> @struct_tbuffer_load_v4f32__sgpr_rsrc__vgpr_vindex_
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN]].sub0
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN]].sub1
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN]].sub2
@@ -117,7 +117,7 @@ define amdgpu_ps float @struct_tbuffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vo
   ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %voffset, i32 %soffset, i32 78, i32 0)
@@ -166,7 +166,7 @@ define amdgpu_ps <4 x float> @struct_tbuffer_load_v4f32__vgpr_rsrc__sgpr_vindex_
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -203,7 +203,7 @@ define amdgpu_ps float @struct_tbuffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vo
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[TBUFFER_LOAD_FORMAT_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_BOTHEN]]
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
index c8d3a32d33def..603d7e4d4f83f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
@@ -19,7 +19,7 @@ define amdgpu_ps void @load_1d_vgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32
   ; FAST-NEXT:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; FAST-NEXT:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
   ; FAST-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   [[COPY9:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
   ; FAST-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[COPY9]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; FAST-NEXT:   S_ENDPGM 0
@@ -38,7 +38,7 @@ define amdgpu_ps void @load_1d_vgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32
   ; GREEDY-NEXT:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
   ; GREEDY-NEXT:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
   ; GREEDY-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   [[COPY9:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
   ; GREEDY-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[COPY9]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; GREEDY-NEXT:   S_ENDPGM 0
@@ -65,7 +65,7 @@ define amdgpu_ps void @load_1d_sgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32
   ; FAST-NEXT:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
   ; FAST-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
   ; FAST-NEXT:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY8]](s32)
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   [[COPY10:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
   ; FAST-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[COPY10]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; FAST-NEXT:   S_ENDPGM 0
@@ -85,7 +85,7 @@ define amdgpu_ps void @load_1d_sgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32
   ; GREEDY-NEXT:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
   ; GREEDY-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
   ; GREEDY-NEXT:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY8]](s32)
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   [[COPY10:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
   ; GREEDY-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[COPY10]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; GREEDY-NEXT:   S_ENDPGM 0
@@ -144,7 +144,7 @@ define amdgpu_ps void @load_1d_vgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 %s) {
   ; FAST-NEXT: bb.3:
   ; FAST-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; FAST-NEXT: {{  $}}
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; FAST-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; FAST-NEXT: {{  $}}
@@ -205,7 +205,7 @@ define amdgpu_ps void @load_1d_vgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 %s) {
   ; GREEDY-NEXT: bb.3:
   ; GREEDY-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GREEDY-NEXT: {{  $}}
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GREEDY-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GREEDY-NEXT: {{  $}}
@@ -274,7 +274,7 @@ define amdgpu_ps void @load_1d_sgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 inreg
   ; FAST-NEXT: bb.3:
   ; FAST-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; FAST-NEXT: {{  $}}
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; FAST-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; FAST-NEXT: {{  $}}
@@ -336,7 +336,7 @@ define amdgpu_ps void @load_1d_sgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 inreg
   ; GREEDY-NEXT: bb.3:
   ; GREEDY-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GREEDY-NEXT: {{  $}}
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GREEDY-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GREEDY-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
index 8e1858a602c19..8dab27b929613 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
@@ -24,7 +24,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inre
   ; FAST-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; FAST-NEXT:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
   ; FAST-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[DEF]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; FAST-NEXT:   S_ENDPGM 0
   ; GREEDY-LABEL: name: sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp
@@ -47,7 +47,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inre
   ; GREEDY-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
   ; GREEDY-NEXT:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
   ; GREEDY-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[DEF]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; GREEDY-NEXT:   S_ENDPGM 0
   %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
@@ -78,7 +78,7 @@ define amdgpu_ps void @sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inre
   ; FAST-NEXT:   [[COPY12:%[0-9]+]]:sgpr(s32) = COPY $sgpr14
   ; FAST-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
   ; FAST-NEXT:   [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[COPY12]](s32)
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[DEF]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; FAST-NEXT:   S_ENDPGM 0
   ; GREEDY-LABEL: name: sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp
@@ -102,7 +102,7 @@ define amdgpu_ps void @sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inre
   ; GREEDY-NEXT:   [[COPY12:%[0-9]+]]:sgpr(s32) = COPY $sgpr14
   ; GREEDY-NEXT:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
   ; GREEDY-NEXT:   [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[COPY12]](s32)
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   G_STORE [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x s32>), [[DEF]](p1) :: (store (<4 x s32>) into `ptr addrspace(1) undef`, addrspace 1)
   ; GREEDY-NEXT:   S_ENDPGM 0
   %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
@@ -165,7 +165,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp(<8 x i32> %rsr
   ; FAST-NEXT: bb.3:
   ; FAST-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; FAST-NEXT: {{  $}}
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; FAST-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; FAST-NEXT: {{  $}}
@@ -230,7 +230,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp(<8 x i32> %rsr
   ; GREEDY-NEXT: bb.3:
   ; GREEDY-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GREEDY-NEXT: {{  $}}
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GREEDY-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GREEDY-NEXT: {{  $}}
@@ -294,7 +294,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp(<8 x i32> inre
   ; FAST-NEXT: bb.3:
   ; FAST-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; FAST-NEXT: {{  $}}
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR2]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR2]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; FAST-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; FAST-NEXT: {{  $}}
@@ -351,7 +351,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp(<8 x i32> inre
   ; GREEDY-NEXT: bb.3:
   ; GREEDY-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GREEDY-NEXT: {{  $}}
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR2]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR2]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GREEDY-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GREEDY-NEXT: {{  $}}
@@ -435,7 +435,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsr
   ; FAST-NEXT: bb.3:
   ; FAST-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; FAST-NEXT: {{  $}}
-  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR3]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; FAST-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR3]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; FAST-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; FAST-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; FAST-NEXT: {{  $}}
@@ -512,7 +512,7 @@ define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsr
   ; GREEDY-NEXT: bb.3:
   ; GREEDY-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; GREEDY-NEXT: {{  $}}
-  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR3]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7)
+  ; GREEDY-NEXT:   [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:vgpr(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR3]](<4 x s32>), 0, 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8)
   ; GREEDY-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; GREEDY-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; GREEDY-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
index 82bd44ce41cdc..a0ebe9f63132b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
@@ -17,7 +17,7 @@ define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
   ; CHECK-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[AMDGPU_BUFFER_LOAD]](s32)
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -40,7 +40,7 @@ define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr
   ; CHECK-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY7]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY7]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[AMDGPU_BUFFER_LOAD]](s32)
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -87,7 +87,7 @@ define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -134,7 +134,7 @@ define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[V_READFIRSTLANE_B32_]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[V_READFIRSTLANE_B32_]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -193,7 +193,7 @@ define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY6]](s32), [[COPY4]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
index 886c31e6f8b6d..06f8f3e52a522 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
@@ -16,7 +16,7 @@ define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[AMDGPU_BUFFER_LOAD]](s32)
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -39,7 +39,7 @@ define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__sgpr_val__sgpr_vindex__sg
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY5]](s32)
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY8]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY8]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $vgpr0 = COPY [[AMDGPU_BUFFER_LOAD]](s32)
   ; CHECK-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
@@ -85,7 +85,7 @@ define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -131,7 +131,7 @@ define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex_vgp
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[V_READFIRSTLANE_B32_]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[V_READFIRSTLANE_B32_]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -189,7 +189,7 @@ define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[V_READFIRSTLANE_B32_4]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(s32) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]], [[V_READFIRSTLANE_B32_4]], 0, 0, -1 :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
index f89f96ad60d84..7aeb43a3f99a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
@@ -17,7 +17,7 @@ define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
   ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
   ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
-  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -41,7 +41,7 @@ define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__sgpr_val__sgpr_vindex__sg
   ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY5]](s32)
   ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[COPY6]](s32)
-  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY8]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY9]](s32), [[COPY10]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY8]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY9]](s32), [[COPY10]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -87,7 +87,7 @@ define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[COPY7]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -133,7 +133,7 @@ define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[V_READFIRSTLANE_B32_]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[V_READFIRSTLANE_B32_]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
@@ -191,7 +191,7 @@ define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vg
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[V_READFIRSTLANE_B32_4]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   G_AMDGPU_BUFFER_STORE [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]], [[V_READFIRSTLANE_B32_4]], 0, 0, -1 :: (dereferenceable store (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
   ; CHECK-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
   ; CHECK-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-captured.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-captured.ll
index db5d39e3884b0..4c98a1fba6e80 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast-captured.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-captured.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -passes=amdgpu-promote-alloca < %s | FileCheck %s
 ; Nothing should be done if the addrspacecast is captured.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 declare void @consume_ptr2int(i32) #0
 
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll
index 013f5e5c1ad47..504c7b4c6430f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-break-large-phis-heuristics.ll
@@ -252,7 +252,7 @@ define amdgpu_kernel void @nontrivial_insertelt_chain(<5 x double> %in, ptr %out
 ; CHECK-NEXT:    br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[X_1:%.*]] = insertelement <5 x double> <double 3.140000e+00, double poison, double poison, double poison, double poison>, double [[X:%.*]], i32 [[IDX:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <5 x double> [[X_1]], <5 x double> <double poison, double poison, double poison, double 6.140000e+00, double 9.900000e+00>, <5 x i32> <i32 0, i32 1, i32 undef, i32 8, i32 9>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <5 x double> [[X_1]], <5 x double> <double poison, double poison, double poison, double 6.140000e+00, double 9.900000e+00>, <5 x i32> <i32 0, i32 1, i32 poison, i32 8, i32 9>
 ; CHECK-NEXT:    [[X_4:%.*]] = insertelement <5 x double> [[TMP0]], double [[X]], i64 2
 ; CHECK-NEXT:    br label [[FINALLY:%.*]]
 ; CHECK:       else:
diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
index f2cf2e5c062f9..e4c694731e239 100644
--- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
+++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -amdgpu-annotate-kernel-features < %s | FileCheck -check-prefixes=HSA,AKF_HSA %s
 ; RUN: opt -mtriple=amdgcn-unknown-amdhsa -S -amdgpu-attributor < %s | FileCheck -check-prefixes=HSA,ATTRIBUTOR_HSA %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 declare i32 @llvm.amdgcn.workgroup.id.x() #0
 declare i32 @llvm.amdgcn.workgroup.id.y() #0
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
new file mode 100644
index 0000000000000..c64ed7f3d48b9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -0,0 +1,1316 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -O3 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -stop-after=branch-folder < %s | FileCheck %s -check-prefix=GFX90A
+
+define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64 %arg2, i1 %arg3, i1 %arg4, i1 %arg5, i1 %arg6, ptr addrspace(3) %arg7, ptr addrspace(3) %arg8, ptr addrspace(3) %arg9, ptr addrspace(3) %arg10) {
+  ; GFX90A-LABEL: name: f1
+  ; GFX90A: bb.0.bb:
+  ; GFX90A-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr0, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr10_sgpr11, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr17, $sgpr12_sgpr13
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $sgpr32 = S_MOV_B32 0
+  ; GFX90A-NEXT:   $flat_scr_lo = S_ADD_U32 $sgpr12, $sgpr17, implicit-def $scc
+  ; GFX90A-NEXT:   $flat_scr_hi = S_ADDC_U32 $sgpr13, 0, implicit-def dead $scc, implicit $scc
+  ; GFX90A-NEXT:   $sgpr0 = S_ADD_U32 $sgpr0, $sgpr17, implicit-def $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT:   renamable $vgpr31 = COPY $vgpr0, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX4_IMM renamable $sgpr8_sgpr9, 24, 0 :: (dereferenceable invariant load (s128) from %ir.arg3.kernarg.offset.align.down, align 8, addrspace 4)
+  ; GFX90A-NEXT:   renamable $sgpr24_sgpr25_sgpr26_sgpr27 = S_LOAD_DWORDX4_IMM renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_LOAD_DWORDX2_IMM renamable $sgpr8_sgpr9, 16, 0 :: (dereferenceable invariant load (s64) from %ir.arg.kernarg.offset1 + 16, align 16, addrspace 4)
+  ; GFX90A-NEXT:   renamable $sgpr33 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 24, 0 :: (dereferenceable invariant load (s32) from %ir.arg4.kernarg.offset.align.down, align 8, addrspace 4)
+  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr20, 0, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_CSELECT_B64 -1, 0, implicit $scc
+  ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr28_sgpr29 = S_XOR_B64 renamable $sgpr12_sgpr13, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr33, 8, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit $scc
+  ; GFX90A-NEXT:   renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = DS_READ_B32_gfx9 renamable $vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCZ %bb.2, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.1.bb103:
+  ; GFX90A-NEXT:   successors: %bb.58(0x40000000), %bb.2(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FC, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr30_sgpr31, implicit-def dead $scc
+  ; GFX90A-NEXT:   $vgpr24 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $agpr0 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $vgpr26 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $vgpr20 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $vgpr22 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.58, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.2:
+  ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr24, $sgpr33, $vgpr31, $agpr0, $vgpr26, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr58, $sgpr59, $sgpr21_sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr2, $vgpr3, $vgpr20, $vgpr22
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr20 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr21 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr23 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr25 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr27 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 0
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.3.Flow17:
+  ; GFX90A-NEXT:   successors: %bb.4(0x40000000), %bb.57(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr20, $sgpr33, $vgpr31, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr4 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCZ %bb.57, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.4.bb15:
+  ; GFX90A-NEXT:   successors: %bb.35(0x40000000), %bb.5(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = V_LSHLREV_B64_e64 2, $vgpr2_vgpr3, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr5 = COPY renamable $sgpr25, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr46, renamable $vcc = V_ADD_CO_U32_e64 $sgpr24, $vgpr0, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr47, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr5, killed $vgpr1, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr0 = V_LSHLREV_B32_e32 2, $vgpr4, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr40, renamable $vcc = V_ADD_CO_U32_e64 $vgpr46, killed $vgpr0, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr41, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr47, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr30_sgpr31, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.35, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.5:
+  ; GFX90A-NEXT:   successors: %bb.6(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr60_vgpr61 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr58_vgpr59 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr56_vgpr57 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr44_vgpr45 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr42_vgpr43 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.6.Flow20:
+  ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr21 = COPY renamable $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr20 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr23 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr22 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr25 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr24 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr27 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr26 = COPY $sgpr17, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.7.Flow19:
+  ; GFX90A-NEXT:   successors: %bb.62(0x40000000), %bb.8(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_MOV_B64 0
+  ; GFX90A-NEXT:   $sgpr24_sgpr25 = S_AND_SAVEEXEC_B64 $sgpr36_sgpr37, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.62, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.8.Flow32:
+  ; GFX90A-NEXT:   successors: %bb.9(0x40000000), %bb.10(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr18_sgpr19, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.10, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.9.bb89:
+  ; GFX90A-NEXT:   successors: %bb.10(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr11, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.10.Flow33:
+  ; GFX90A-NEXT:   successors: %bb.11(0x40000000), %bb.12(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr56_sgpr57, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.12, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.11.bb84:
+  ; GFX90A-NEXT:   successors: %bb.12(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.12.Flow34:
+  ; GFX90A-NEXT:   successors: %bb.13(0x40000000), %bb.14(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr54_sgpr55, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.14, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.13.bb79:
+  ; GFX90A-NEXT:   successors: %bb.14(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr7, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.14.Flow35:
+  ; GFX90A-NEXT:   successors: %bb.15(0x40000000), %bb.16(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr52_sgpr53, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.16, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.15.bb72:
+  ; GFX90A-NEXT:   successors: %bb.16(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr58_sgpr59, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr8 = S_ADD_U32 renamable $sgpr8, 48, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr9 = S_ADDC_U32 killed renamable $sgpr9, 0, implicit-def dead $scc, implicit $scc
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @f2 + 4, target-flags(amdgpu-gotprel32-hi) @f2 + 12, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_LOAD_DWORDX2_IMM killed renamable $sgpr12_sgpr13, 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
+  ; GFX90A-NEXT:   $sgpr12 = COPY killed renamable $sgpr14
+  ; GFX90A-NEXT:   $sgpr13 = COPY killed renamable $sgpr15
+  ; GFX90A-NEXT:   $sgpr14 = COPY killed renamable $sgpr16
+  ; GFX90A-NEXT:   dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr18_sgpr19, @f2, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit undef $sgpr15, implicit $vgpr31, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.16.Flow36:
+  ; GFX90A-NEXT:   successors: %bb.17(0x40000000), %bb.18(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr36_sgpr37, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr50_sgpr51, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.18, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.17.bb67:
+  ; GFX90A-NEXT:   successors: %bb.18(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr47, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr46, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.18.Flow37:
+  ; GFX90A-NEXT:   successors: %bb.19(0x40000000), %bb.20(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr48_sgpr49, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.20, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.19.bb62:
+  ; GFX90A-NEXT:   successors: %bb.20(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr63, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr62, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.20.Flow38:
+  ; GFX90A-NEXT:   successors: %bb.21(0x40000000), %bb.22(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr46_sgpr47, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.22, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.21.bb54:
+  ; GFX90A-NEXT:   successors: %bb.22(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr61, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr60, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.22.Flow39:
+  ; GFX90A-NEXT:   successors: %bb.23(0x40000000), %bb.24(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr44_sgpr45, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.24, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.23.bb47:
+  ; GFX90A-NEXT:   successors: %bb.24(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr59, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr58, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.24.Flow40:
+  ; GFX90A-NEXT:   successors: %bb.25(0x40000000), %bb.26(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr42_sgpr43, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.26, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.25.bb40:
+  ; GFX90A-NEXT:   successors: %bb.26(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr57, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr56, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.26.Flow41:
+  ; GFX90A-NEXT:   successors: %bb.27(0x40000000), %bb.28(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr40_sgpr41, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.28, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.27.bb33:
+  ; GFX90A-NEXT:   successors: %bb.28(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr45, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr44, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.28.Flow42:
+  ; GFX90A-NEXT:   successors: %bb.34(0x40000000), %bb.29(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr38_sgpr39, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.34, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.29.Flow43:
+  ; GFX90A-NEXT:   successors: %bb.30(0x40000000), %bb.31(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GFX90A-NEXT:   $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.31, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.30.bb19:
+  ; GFX90A-NEXT:   successors: %bb.31(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr41, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr40, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.31.Flow44:
+  ; GFX90A-NEXT:   successors: %bb.32(0x40000000), %bb.33(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr58_sgpr59, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr58_sgpr59, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECZ %bb.33, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.32.UnifiedUnreachableBlock:
+  ; GFX90A-NEXT:   successors: %bb.33(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   SI_MASKED_UNREACHABLE
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.33.UnifiedReturnBlock:
+  ; GFX90A-NEXT:   liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   S_ENDPGM 0
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.34.bb26:
+  ; GFX90A-NEXT:   successors: %bb.29(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr58_sgpr59, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr43, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr42, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.29
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.35.bb20:
+  ; GFX90A-NEXT:   successors: %bb.37(0x40000000), %bb.36(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_SBYTE renamable $vgpr40_vgpr41, 1024, 0, implicit $exec :: (load (s8) from %ir.i21, addrspace 1)
+  ; GFX90A-NEXT:   renamable $vgpr42 = V_ADD_CO_U32_e32 1024, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr43, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_LT_I16_e64 0, killed $vgpr0, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr60_vgpr61 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr58_vgpr59 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr56_vgpr57 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr44_vgpr45 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr24_sgpr25 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.37, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.36.Flow21:
+  ; GFX90A-NEXT:   successors: %bb.6(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.6
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.37.bb27:
+  ; GFX90A-NEXT:   successors: %bb.39(0x40000000), %bb.38(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19, $sgpr56_sgpr57, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr44_sgpr45, $sgpr42_sgpr43
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE renamable $vgpr40_vgpr41, 2048, 0, implicit $exec :: (load (s8) from %ir.i28, addrspace 1)
+  ; GFX90A-NEXT:   renamable $vgpr44 = V_ADD_CO_U32_e32 2048, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr45, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr60_vgpr61 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr58_vgpr59 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr56_vgpr57 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr38_sgpr39 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.39, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.38.Flow22:
+  ; GFX90A-NEXT:   successors: %bb.36(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr38_sgpr39, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_AND_B64 killed renamable $sgpr40_sgpr41, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr18_sgpr19, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_ANDN2_B64 killed renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_OR_B64 killed renamable $sgpr36_sgpr37, killed renamable $sgpr58_sgpr59, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.36
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.39.bb34:
+  ; GFX90A-NEXT:   successors: %bb.41(0x40000000), %bb.40(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr18_sgpr19, $sgpr56_sgpr57, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr44_sgpr45
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE renamable $vgpr40_vgpr41, 3072, 0, implicit $exec :: (load (s8) from %ir.i35, addrspace 1)
+  ; GFX90A-NEXT:   renamable $vgpr56 = V_ADD_CO_U32_e32 3072, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr57, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr60_vgpr61 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr58_vgpr59 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr40_sgpr41 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.41, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.40.Flow23:
+  ; GFX90A-NEXT:   successors: %bb.38(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr40_sgpr41, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr18_sgpr19, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_OR_B64 killed renamable $sgpr58_sgpr59, killed renamable $sgpr60_sgpr61, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.38
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.41.bb41:
+  ; GFX90A-NEXT:   successors: %bb.46(0x40000000), %bb.42(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr50_sgpr51, $sgpr48_sgpr49, $sgpr46_sgpr47
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr58 = V_ADD_CO_U32_e32 4096, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = COPY $vcc
+  ; GFX90A-NEXT:   renamable $vgpr59, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, $vgpr41, killed $sgpr18_sgpr19, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE renamable $vgpr58_vgpr59, 0, 0, implicit $exec :: (load (s8) from %ir.i42, addrspace 1)
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr20, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr60_vgpr61 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr42_sgpr43 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.46, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.42.Flow24:
+  ; GFX90A-NEXT:   successors: %bb.40(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr20, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr42_sgpr43, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $vgpr59 = COPY killed renamable $vgpr20, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr18_sgpr19, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_OR_B64 killed renamable $sgpr58_sgpr59, killed renamable $sgpr60_sgpr61, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.40
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.43.bb55:
+  ; GFX90A-NEXT:   successors: %bb.48(0x40000000), %bb.44(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr44_sgpr45, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr54_sgpr55, $sgpr46_sgpr47
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   S_BITCMP1_B32 renamable $sgpr33, 16, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_CSELECT_B64 -1, 0, implicit $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_XOR_B64 renamable $sgpr64_sgpr65, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $vgpr62 = V_ADD_CO_U32_e32 6144, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr63, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; GFX90A-NEXT:   $agpr0 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $vgpr14 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.48, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.44:
+  ; GFX90A-NEXT:   successors: %bb.45(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr58, $vgpr57, $vgpr20, $vgpr61, $vgpr31, $vgpr63, $agpr0, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $vgpr40, $vgpr62, $vgpr60, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21, $sgpr21_sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr56, $vgpr47, $vgpr2, $vgpr3, $vgpr4, $vgpr46, $vgpr45, $vgpr44, $vgpr43, $vgpr42, $vgpr41, $vgpr14
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.45.Flow26:
+  ; GFX90A-NEXT:   successors: %bb.47(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr20, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr70_sgpr71 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = S_OR_B64 killed renamable $sgpr44_sgpr45, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.47
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.46.bb48:
+  ; GFX90A-NEXT:   successors: %bb.43(0x40000000), %bb.47(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr56_sgpr57, $sgpr54_sgpr55, $sgpr44_sgpr45, $sgpr52_sgpr53
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr60 = V_ADD_CO_U32_e32 5120, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = COPY $vcc
+  ; GFX90A-NEXT:   renamable $vgpr0 = V_ADD_CO_U32_e32 4096, $vgpr40, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr1, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr0 = GLOBAL_LOAD_UBYTE killed renamable $vgpr0_vgpr1, 1024, 0, implicit $exec :: (load (s8) from %ir.i49, addrspace 1)
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr64_sgpr65 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $sgpr66_sgpr67 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr68_sgpr69 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr61, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $sgpr18_sgpr19, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr70_sgpr71 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr18_sgpr19 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.43, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.47.Flow25:
+  ; GFX90A-NEXT:   successors: %bb.42(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr20, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr18_sgpr19 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr60_sgpr61 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr58_sgpr59, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.42
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.48.bb63:
+  ; GFX90A-NEXT:   successors: %bb.50(0x40000000), %bb.49(0x40000000)
+  ; GFX90A-NEXT:   liveins: $vcc, $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr58_sgpr59:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr54_sgpr55, $sgpr46_sgpr47
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.50, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.49:
+  ; GFX90A-NEXT:   successors: %bb.44(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr54_sgpr55
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   S_BRANCH %bb.44
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.50.bb68:
+  ; GFX90A-NEXT:   successors: %bb.54(0x40000000), %bb.51(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr58_sgpr59:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr54_sgpr55
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = V_LSHLREV_B64_e64 3, $vgpr4_vgpr5, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.54, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.51:
+  ; GFX90A-NEXT:   successors: %bb.45(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr54_sgpr55
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   S_BRANCH %bb.45
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.52.bb80:
+  ; GFX90A-NEXT:   successors: %bb.59(0x40000000), %bb.53(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr58_sgpr59:0x000000000000000F, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr17 = S_BFE_U32 killed renamable $sgpr33, 65560, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CMP_EQ_U32 killed renamable $sgpr17, 0, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $vgpr8 = V_ADD_CO_U32_e32 4096, $vgpr0, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr9, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr1, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_SCC1 %bb.59, implicit $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.53:
+  ; GFX90A-NEXT:   successors: %bb.61(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   S_BRANCH %bb.61
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.54.bb73:
+  ; GFX90A-NEXT:   successors: %bb.52(0x40000000), %bb.55(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr58_sgpr59:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr56_sgpr57
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr5 = GLOBAL_LOAD_UBYTE renamable $vgpr0_vgpr1, 2048, 0, implicit $exec :: (load (s8) from %ir.i74, addrspace 1)
+  ; GFX90A-NEXT:   renamable $vgpr6 = V_ADD_CO_U32_e32 2048, $vgpr0, implicit-def $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr7, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr1, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr5, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $agpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $sgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr60_sgpr61 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.52, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.55.Flow29:
+  ; GFX90A-NEXT:   successors: %bb.45(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr20, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr60_sgpr61, implicit-def $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.45
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.56.bb90:
+  ; GFX90A-NEXT:   successors: %bb.60(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr52_sgpr53, $sgpr58_sgpr59:0x000000000000000F, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr54 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr5, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr5 = COPY renamable $sgpr21, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr18_vgpr19 = DS_READ_B64_gfx9 killed renamable $vgpr5, 0, 0, implicit $exec :: (load (s64) from %ir.3, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr5 = COPY renamable $sgpr22, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr14_vgpr15 = DS_READ_B64_gfx9 killed renamable $vgpr5, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr5 = COPY renamable $sgpr58, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr13 = V_ALIGNBIT_B32_e64 killed $sgpr59, killed $vgpr5, 1, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr30 = V_ALIGNBIT_B32_e64 $vgpr19, $vgpr18, 1, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr19 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr12_sgpr13, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr17 = V_ALIGNBIT_B32_e64 $vgpr17, $vgpr16, 1, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = S_OR_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.60
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.57:
+  ; GFX90A-NEXT:   successors: %bb.7(0x80000000)
+  ; GFX90A-NEXT:   liveins: $exec:0x000000000000000F, $sgpr14, $sgpr15, $sgpr16, $sgpr17:0x0000000000000003, $sgpr20:0x0000000000000003, $vgpr31, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr17 = COPY killed renamable $sgpr20, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr19 = COPY killed renamable $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr48_sgpr49 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr46_sgpr47 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr44_sgpr45 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr42_sgpr43 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr40_sgpr41 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $sgpr38_sgpr39 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr6_vgpr7 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr62_vgpr63 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr60_vgpr61 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr58_vgpr59 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr56_vgpr57 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr44_vgpr45 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr42_vgpr43 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr40_vgpr41 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr46_vgpr47 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = COPY renamable $vgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr30 = COPY renamable $vgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr18 = COPY renamable $vgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr54 = COPY renamable $vgpr19, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr15 = COPY renamable $vgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr14 = COPY renamable $vgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr34_sgpr35 = S_MOV_B64 0
+  ; GFX90A-NEXT:   S_BRANCH %bb.7
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.58.bb105:
+  ; GFX90A-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr58_sgpr59:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FC, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr17 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 40, 0 :: (dereferenceable invariant load (s32) from %ir.arg3.kernarg.offset.align.down + 16, align 8, addrspace 4)
+  ; GFX90A-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr24_vgpr25 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.434, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr21, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.3, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr0 = COPY killed renamable $sgpr17, implicit $exec
+  ; GFX90A-NEXT:   renamable $agpr0_agpr1 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.435, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr0 = COPY renamable $sgpr22, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr26_vgpr27 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $sgpr20 = S_MOV_B32 0
+  ; GFX90A-NEXT:   renamable $sgpr17 = S_MOV_B32 0
+  ; GFX90A-NEXT:   S_BRANCH %bb.3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.59.bb85:
+  ; GFX90A-NEXT:   successors: %bb.56(0x40000000), %bb.60(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr20, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr58_sgpr59:0x000000000000000F, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr10 = V_OR_B32_e32 1, $vgpr8, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr11 = COPY renamable $vgpr9, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr5 = FLAT_LOAD_UBYTE renamable $vgpr10_vgpr11, 0, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i86)
+  ; GFX90A-NEXT:   renamable $sgpr17 = S_MOV_B32 0
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr5, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr62_sgpr63 = COPY renamable $sgpr36_sgpr37
+  ; GFX90A-NEXT:   renamable $vgpr19 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr17 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr16 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr30 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr18 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr54 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr15 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   renamable $vgpr13 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   $sgpr52_sgpr53 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.56, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.60.Flow31:
+  ; GFX90A-NEXT:   successors: %bb.61(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr20, $vgpr30, $vgpr31, $vgpr54, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000C, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr52_sgpr53, implicit-def $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_MOV_B64 0
+  ; GFX90A-NEXT:   renamable $vgpr12 = COPY renamable $vgpr16, implicit $exec
+  ; GFX90A-NEXT:   renamable $agpr0_agpr1 = COPY killed renamable $vgpr12_vgpr13, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.61.Flow30:
+  ; GFX90A-NEXT:   successors: %bb.55(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr17, $vgpr19, $vgpr20, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr54_sgpr55 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+  ; GFX90A-NEXT:   renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr50_sgpr51, killed renamable $sgpr58_sgpr59, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_BRANCH %bb.55
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.62.bb140:
+  ; GFX90A-NEXT:   successors: %bb.68(0x40000000), %bb.63(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr30_sgpr31, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.68, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.63.Flow13:
+  ; GFX90A-NEXT:   successors: %bb.64(0x40000000), %bb.66(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr36_sgpr37, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.66, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.64.bb159:
+  ; GFX90A-NEXT:   successors: %bb.67(0x40000000), %bb.65(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_NE_U32_e64 0, killed $vgpr4, implicit $exec
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_EXECNZ %bb.67, implicit $exec
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.65.Flow10:
+  ; GFX90A-NEXT:   successors: %bb.66(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $sgpr12_sgpr13 = S_ANDN2_SAVEEXEC_B64 $sgpr12_sgpr13, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX90A-NEXT:   $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.66.Flow14:
+  ; GFX90A-NEXT:   successors: %bb.8(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr58_sgpr59 = COPY $exec
+  ; GFX90A-NEXT:   S_BRANCH %bb.8
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.67.bb161:
+  ; GFX90A-NEXT:   successors: %bb.65(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr23, killed $vgpr25, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr27, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = COPY killed renamable $agpr1, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = V_OR_B32_e32 killed $vgpr3, killed $vgpr21, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_sdwa 0, killed $vgpr54, 0, $vgpr3, 0, 0, 6, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr4 = V_OR_B32_e32 killed $vgpr30, killed $vgpr15, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr4, killed $vgpr2, implicit $exec
+  ; GFX90A-NEXT:   renamable $vcc = V_CMP_EQ_U16_sdwa 0, killed $vgpr19, 0, $vgpr3, 0, 0, 6, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr17, implicit $exec
+  ; GFX90A-NEXT:   DS_WRITE2_B32_gfx9 killed renamable $vgpr3, killed renamable $vgpr2, renamable $vgpr3, 0, 1, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, align 4, addrspace 3)
+  ; GFX90A-NEXT:   S_BRANCH %bb.65
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.68.bb174:
+  ; GFX90A-NEXT:   successors: %bb.72(0x40000000), %bb.69(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000F, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000F, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x000000000000000F, $vgpr24_vgpr25:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr28 = V_OR_B32_e32 1, $vgpr26, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr38 = V_OR_B32_e32 $vgpr28, $vgpr24, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr36 = V_OR_B32_e32 $vgpr38, $vgpr22, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr32 = V_CNDMASK_B32_e64 0, $vgpr36, 0, 0, $sgpr12_sgpr13, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr50 = V_OR_B32_e32 $vgpr32, $vgpr20, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr12_vgpr13 = COPY renamable $agpr0_agpr1, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr48 = V_OR_B32_e32 $vgpr50, killed $vgpr12, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr34 = V_OR_B32_e32 $vgpr48, $vgpr14, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr52 = V_CNDMASK_B32_e64 0, 0, 0, $vgpr34, killed $sgpr12_sgpr13, implicit $exec
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_MOV_B64 -1
+  ; GFX90A-NEXT:   renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr28_sgpr29, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.72, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.69.Flow:
+  ; GFX90A-NEXT:   successors: %bb.70(0x40000000), %bb.71(0x40000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
+  ; GFX90A-NEXT:   S_CBRANCH_VCCNZ %bb.71, implicit $vcc
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.70.bb186:
+  ; GFX90A-NEXT:   successors: %bb.71(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr2_vgpr3 = V_LSHLREV_B64_e64 3, killed $vgpr2_vgpr3, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr5 = COPY renamable $sgpr27, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr2, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr2, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr3, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr5, killed $vgpr3, killed $vcc, 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr29 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr39 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr37 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr51 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr49 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr33 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr53 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr35 = COPY renamable $vgpr29, implicit $exec
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 renamable $vgpr29, renamable $vgpr28_vgpr29, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr5 = COPY renamable $sgpr21, implicit $exec
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr38_vgpr39, 0, 0, implicit $exec :: (store (s64) into %ir.3, addrspace 3)
+  ; GFX90A-NEXT:   renamable $vgpr12 = COPY killed renamable $sgpr22, implicit $exec
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr12, killed renamable $vgpr36_vgpr37, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 renamable $vgpr29, killed renamable $vgpr50_vgpr51, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr48_vgpr49, 0, 0, implicit $exec :: (store (s64) into %ir.3, addrspace 3)
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 renamable $vgpr29, killed renamable $vgpr32_vgpr33, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr5, killed renamable $vgpr52_vgpr53, 0, 0, implicit $exec :: (store (s64) into %ir.3, addrspace 3)
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr29, killed renamable $vgpr34_vgpr35, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
+  ; GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.71.Flow9:
+  ; GFX90A-NEXT:   successors: %bb.63(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $vgpr0_vgpr1:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $sgpr36_sgpr37 = S_MOV_B64 0
+  ; GFX90A-NEXT:   S_BRANCH %bb.63
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.72.bb196:
+  ; GFX90A-NEXT:   successors: %bb.69(0x80000000)
+  ; GFX90A-NEXT:   liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr17, $vgpr19, $vgpr30, $vgpr31, $vgpr54, $agpr0_agpr1:0x000000000000000C, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x0000000000000003, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr14_vgpr15:0x000000000000000C, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x000000000000000C, $vgpr24_vgpr25:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x0000000000000003, $vgpr32_vgpr33:0x0000000000000003, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   renamable $vgpr5 = V_OR_B32_e32 $vgpr52, killed $vgpr18, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr12 = V_OR_B32_e32 killed $vgpr5, killed $vgpr16, implicit $exec
+  ; GFX90A-NEXT:   renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX90A-NEXT:   DS_WRITE_B64_gfx9 killed renamable $vgpr13, renamable $vgpr12_vgpr13, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
+  ; GFX90A-NEXT:   renamable $sgpr12_sgpr13 = S_MOV_B64 0
+  ; GFX90A-NEXT:   S_BRANCH %bb.69
+bb:
+  %i = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %i11 = icmp eq i32 %i, 0
+  %i12 = load i32, ptr addrspace(3) null, align 8
+  %i13 = zext i32 %i12 to i64
+  %i14 = getelementptr i32, ptr addrspace(1) %arg, i64 %i13
+  br i1 %arg3, label %bb15, label %bb103
+
+bb15:
+  %i16 = zext i32 %i to i64
+  %i17 = getelementptr i32, ptr addrspace(1) %i14, i64 %i16
+  %i18 = ptrtoint ptr addrspace(1) %i17 to i64
+  br i1 %arg4, label %bb19, label %bb20
+
+bb19:
+  store i64 %i18, ptr addrspace(5) null, align 8
+  unreachable
+
+bb20:
+  %i21 = getelementptr i32, ptr addrspace(1) %i17, i64 256
+  %i22 = ptrtoint ptr addrspace(1) %i21 to i64
+  %i23 = inttoptr i64 %i22 to ptr
+  %i24 = load i8, ptr %i23, align 1
+  %i25 = icmp sge i8 0, %i24
+  br i1 %i25, label %bb26, label %bb27
+
+bb26:
+  store i64 %i22, ptr addrspace(5) null, align 8
+  unreachable
+
+bb27:
+  %i28 = getelementptr i32, ptr addrspace(1) %i17, i64 512
+  %i29 = ptrtoint ptr addrspace(1) %i28 to i64
+  %i30 = inttoptr i64 %i29 to ptr
+  %i31 = load i8, ptr %i30, align 1
+  %i32 = icmp ne i8 %i31, 0
+  br i1 %i32, label %bb33, label %bb34
+
+bb33:
+  store i64 %i29, ptr addrspace(5) null, align 8
+  unreachable
+
+bb34:
+  %i35 = getelementptr i32, ptr addrspace(1) %i17, i64 768
+  %i36 = ptrtoint ptr addrspace(1) %i35 to i64
+  %i37 = inttoptr i64 %i36 to ptr
+  %i38 = load i8, ptr %i37, align 1
+  %i39 = icmp ne i8 %i38, 0
+  br i1 %i39, label %bb40, label %bb41
+
+bb40:
+  store i64 %i36, ptr addrspace(5) null, align 8
+  unreachable
+
+bb41:
+  %i42 = getelementptr i32, ptr addrspace(1) %i17, i64 1024
+  %i43 = ptrtoint ptr addrspace(1) %i42 to i64
+  %i44 = inttoptr i64 %i43 to ptr
+  %i45 = load i8, ptr %i44, align 1
+  %i46 = icmp ne i8 %i45, 0
+  br i1 %i46, label %bb47, label %bb48
+
+bb47:
+  store i64 %i43, ptr addrspace(5) null, align 8
+  unreachable
+
+bb48:
+  %i49 = getelementptr i32, ptr addrspace(1) %i17, i64 1280
+  %i50 = ptrtoint ptr addrspace(1) %i49 to i64
+  %i51 = inttoptr i64 %i50 to ptr
+  %i52 = load i8, ptr %i51, align 1
+  %i53 = icmp ne i8 %i52, 0
+  br i1 %i53, label %bb54, label %bb55
+
+bb54:
+  store i64 %i50, ptr addrspace(5) null, align 8
+  unreachable
+
+bb55:
+  %i56 = getelementptr i32, ptr addrspace(1) %i17, i64 1536
+  %i57 = ptrtoint ptr addrspace(1) %i56 to i64
+  %i58 = or i64 %i57, 1
+  %i59 = inttoptr i64 %i58 to ptr
+  br i1 true, label %bb61, label %bb60
+
+bb60:
+  br label %bb63
+
+bb61:
+  br i1 %arg5, label %bb62, label %bb63
+
+bb62:
+  store i64 %i57, ptr addrspace(5) null, align 8
+  unreachable
+
+bb63:
+  %i64 = ptrtoint ptr addrspace(1) %i14 to i64
+  br i1 true, label %bb66, label %bb65
+
+bb65:
+  br label %bb68
+
+bb66:
+  br i1 %arg5, label %bb67, label %bb68
+
+bb67:
+  store i64 %i64, ptr addrspace(5) null, align 8
+  unreachable
+
+bb68:
+  %i69 = zext i1 %arg5 to i8
+  %i70 = getelementptr [2 x i32], ptr addrspace(1) null, i64 %i16
+  %i71 = ptrtoint ptr addrspace(1) %i70 to i64
+  br i1 %arg5, label %bb72, label %bb73
+
+bb72:
+  call void @f2(i64 %i71)
+  unreachable
+
+bb73:
+  %i74 = getelementptr [2 x i32], ptr addrspace(1) %i70, i64 256
+  %i75 = ptrtoint ptr addrspace(1) %i74 to i64
+  %i76 = inttoptr i64 %i75 to ptr
+  %i77 = load i8, ptr %i76, align 1
+  %i78 = icmp ne i8 %i77, 0
+  br i1 %i78, label %bb79, label %bb80
+
+bb79:
+  store i64 %i75, ptr addrspace(5) null, align 8
+  unreachable
+
+bb80:
+  %i81 = getelementptr [2 x i32], ptr addrspace(1) %i70, i64 512
+  %i82 = ptrtoint ptr addrspace(1) %i81 to i64
+  %i83 = or i64 %i82, 1
+  br i1 %arg6, label %bb84, label %bb85
+
+bb84:
+  store i64 %i82, ptr addrspace(5) null, align 8
+  unreachable
+
+bb85:
+  %i86 = inttoptr i64 %i83 to ptr
+  %i87 = load i8, ptr %i86, align 1
+  %i88 = icmp ne i8 %i87, 0
+  br i1 %i88, label %bb89, label %bb90
+
+bb89:
+  store i64 %i83, ptr addrspace(5) null, align 8
+  unreachable
+
+bb90:
+  %i91 = load i64, ptr addrspace(3) null, align 8
+  %i92 = load i64, ptr addrspace(3) %arg8, align 8
+  %i93 = load i64, ptr addrspace(3) %arg7, align 8
+  %i94 = trunc i64 %i91 to i32
+  %i95 = lshr i64 %arg2, 1
+  %i96 = trunc i64 %i95 to i32
+  %i97 = trunc i64 %i92 to i32
+  %i98 = lshr i64 %i92, 32
+  %i99 = trunc i64 %i98 to i32
+  %i100 = trunc i64 %i93 to i32
+  %i101 = lshr i64 %i93, 1
+  %i102 = trunc i64 %i101 to i32
+  br label %bb127
+
+bb103:
+  br i1 %arg4, label %bb104, label %bb105
+
+bb104:
+  ret void
+
+bb105:
+  %i106 = load i64, ptr addrspace(3) null, align 8
+  %i107 = load i64, ptr addrspace(3) %arg9, align 8
+  %i108 = load i64, ptr addrspace(3) %arg7, align 8
+  %i109 = load i64, ptr addrspace(3) %arg10, align 8
+  %i110 = load i64, ptr addrspace(3) %arg8, align 8
+  %i111 = trunc i64 %i110 to i32
+  %i112 = lshr i64 %i110, 32
+  %i113 = trunc i64 %i112 to i32
+  %i114 = trunc i64 %i106 to i32
+  %i115 = lshr i64 %i106, 32
+  %i116 = trunc i64 %i115 to i32
+  %i117 = trunc i64 %i107 to i32
+  %i118 = lshr i64 %i107, 32
+  %i119 = trunc i64 %i118 to i32
+  %i120 = trunc i64 %i108 to i32
+  %i121 = lshr i64 %i108, 32
+  %i122 = trunc i64 %i121 to i32
+  %i123 = trunc i64 %i109 to i32
+  %i124 = lshr i64 %i109, 32
+  %i125 = trunc i64 %i124 to i32
+  br i1 false, label %bb105.bb127_crit_edge, label %bb140
+
+bb105.bb127_crit_edge:
+  br label %bb127
+
+bb127:
+  %i128 = phi i32 [ %i94, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i129 = phi i32 [ %i96, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i130 = phi i32 [ %i97, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i131 = phi i32 [ %i99, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i132 = phi i8 [ %i69, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i133 = phi i32 [ %i100, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i134 = phi i32 [ %i102, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i135 = phi i64 [ %i91, %bb90 ], [ 0, %bb105.bb127_crit_edge ]
+  %i136 = zext i1 %arg3 to i8
+  %i137 = trunc i64 %i135 to i32
+  %i138 = lshr i64 %i135, 1
+  %i139 = trunc i64 %i138 to i32
+  br label %bb140
+
+bb140:
+  %i141 = phi i32 [ 0, %bb127 ], [ %i111, %bb105 ]
+  %i142 = phi i32 [ 0, %bb127 ], [ %i113, %bb105 ]
+  %i143 = phi i32 [ 0, %bb127 ], [ %i114, %bb105 ]
+  %i144 = phi i32 [ 0, %bb127 ], [ %i116, %bb105 ]
+  %i145 = phi i32 [ 0, %bb127 ], [ %i117, %bb105 ]
+  %i146 = phi i32 [ 0, %bb127 ], [ %i119, %bb105 ]
+  %i147 = phi i32 [ 0, %bb127 ], [ %i120, %bb105 ]
+  %i148 = phi i32 [ 0, %bb127 ], [ %i122, %bb105 ]
+  %i149 = phi i32 [ %i128, %bb127 ], [ %i123, %bb105 ]
+  %i150 = phi i32 [ %i129, %bb127 ], [ %i125, %bb105 ]
+  %i151 = phi i32 [ %i130, %bb127 ], [ 0, %bb105 ]
+  %i152 = phi i32 [ %i131, %bb127 ], [ 0, %bb105 ]
+  %i153 = phi i8 [ %i132, %bb127 ], [ 0, %bb105 ]
+  %i154 = phi i32 [ %i133, %bb127 ], [ 0, %bb105 ]
+  %i155 = phi i32 [ %i134, %bb127 ], [ 0, %bb105 ]
+  %i156 = phi i32 [ %i137, %bb127 ], [ 0, %bb105 ]
+  %i157 = phi i32 [ %i139, %bb127 ], [ 0, %bb105 ]
+  %i158 = phi i8 [ %i136, %bb127 ], [ 0, %bb105 ]
+  br i1 %arg4, label %bb159, label %bb174
+
+bb159:
+  br i1 %i11, label %bb160, label %bb161
+
+bb160:
+  unreachable
+
+bb161:
+  %i162 = or i32 %i146, %i144
+  %i163 = or i32 %i162, %i142
+  %i164 = or i32 %i150, %i148
+  %i165 = or i32 %i164, %i163
+  %i166 = icmp ne i8 %i153, 0
+  %i167 = select i1 %i166, i32 0, i32 %i165
+  %i168 = or i32 %i155, %i152
+  %i169 = or i32 %i168, %i167
+  %i170 = icmp ne i8 %i158, 0
+  %i171 = select i1 %i170, i32 0, i32 %i169
+  %i172 = or i32 %i171, %i157
+  %i173 = zext i32 %i172 to i64
+  store i64 %i173, ptr addrspace(3) null, align 4
+  unreachable
+
+bb174:
+  %i175 = or i32 1, %i141
+  %i176 = or i32 %i175, %i143
+  %i177 = or i32 %i176, %i145
+  %i178 = select i1 %arg3, i32 0, i32 %i177
+  %i179 = or i32 %i178, %i147
+  %i180 = or i32 %i179, %i149
+  %i181 = or i32 %i180, %i151
+  %i182 = select i1 %arg3, i32 %i181, i32 0
+  %i183 = or i32 %i182, %i154
+  %i184 = or i32 %i183, %i156
+  %i185 = getelementptr [2 x i32], ptr addrspace(1) %arg1, i64 %i13
+  br i1 %arg3, label %bb186, label %bb196
+
+bb186:
+  %i187 = zext i32 %i175 to i64
+  %i188 = zext i32 %i176 to i64
+  %i189 = zext i32 %i177 to i64
+  %i190 = zext i32 %i179 to i64
+  %i191 = zext i32 %i180 to i64
+  %i192 = zext i32 %i178 to i64
+  %i193 = zext i32 %i182 to i64
+  %i194 = zext i32 %i181 to i64
+  store i64 %i187, ptr addrspace(3) null, align 8
+  store i64 %i188, ptr addrspace(3) %arg7, align 8
+  store i64 %i189, ptr addrspace(3) %arg8, align 8
+  store i64 %i190, ptr addrspace(3) null, align 8
+  store i64 %i191, ptr addrspace(3) %arg7, align 8
+  store i64 %i192, ptr addrspace(3) null, align 8
+  store i64 %i193, ptr addrspace(3) %arg7, align 8
+  store i64 %i194, ptr addrspace(3) null, align 8
+  %i195 = ptrtoint ptr addrspace(1) %i185 to i64
+  store i64 %i195, ptr addrspace(5) null, align 8
+  unreachable
+
+bb196:
+  %i197 = zext i32 %i184 to i64
+  store i64 %i197, ptr addrspace(3) null, align 8
+  unreachable
+}
+
+declare void @f2(i64)
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-no-rtn.ll
index c998c9c4b92bb..fcc7432dd038b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-no-rtn.ll
@@ -16,7 +16,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offset_no_rtn(float %val, <4 x i32
   ; GFX908_GFX11-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX908_GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_offset_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -29,7 +29,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offset_no_rtn(float %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
@@ -48,7 +48,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offen_no_rtn(float %val, <4 x i32>
   ; GFX908_GFX11-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX908_GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_offen_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -62,7 +62,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_offen_no_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -81,7 +81,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_idxen_no_rtn(float %val, <4 x i32>
   ; GFX908_GFX11-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX908_GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_idxen_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -95,7 +95,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_idxen_no_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -116,7 +116,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_bothen_no_rtn(float %val, <4 x i32
   ; GFX908_GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX908_GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908_GFX11-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908_GFX11-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_f32_bothen_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -132,7 +132,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f32_bothen_no_rtn(float %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F32_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-rtn.ll
index c6a6a9e848ce0..7953e38e93ec3 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f32-rtn.ll
@@ -15,7 +15,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offset_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_offset_rtn
@@ -29,7 +29,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offset_rtn(float %val, <4 x i32>
   ; GFX11-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFSET_RTN [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFSET_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
@@ -49,7 +49,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offen_rtn(float %val, <4 x i32> i
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_offen_rtn
@@ -64,7 +64,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_offen_rtn(float %val, <4 x i32> i
   ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_OFFEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_OFFEN_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -84,7 +84,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_idxen_rtn(float %val, <4 x i32> i
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_idxen_rtn
@@ -99,7 +99,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_idxen_rtn(float %val, <4 x i32> i
   ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_IDXEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_IDXEN_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
@@ -121,7 +121,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_bothen_rtn(float %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   ; GFX11-LABEL: name: buffer_atomic_fadd_f32_bothen_rtn
@@ -138,7 +138,7 @@ define amdgpu_ps float @buffer_atomic_fadd_f32_bothen_rtn(float %val, <4 x i32>
   ; GFX11-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX11-NEXT:   [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX11-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_F32_BOTHEN_RTN]]
   ; GFX11-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call float @llvm.amdgcn.struct.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
index ad89b68df94d2..bf9de34be29a2 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.f64.ll
@@ -17,7 +17,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_offset_no_rtn(double %val, <4 x i3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFSET killed [[COPY7]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFSET killed [[COPY7]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
@@ -39,7 +39,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_offen_no_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFEN killed [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_OFFEN killed [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -61,7 +61,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_idxen_no_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_IDXEN killed [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_IDXEN killed [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -85,7 +85,7 @@ define amdgpu_ps void @buffer_atomic_fadd_f64_bothen_no_rtn(double %val, <4 x i3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_BOTHEN killed [[COPY9]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_ADD_F64_BOTHEN killed [[COPY9]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
@@ -106,7 +106,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offset_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[COPY7]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFSET_RTN [[COPY7]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFSET_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   $sgpr0 = COPY [[COPY8]]
@@ -132,7 +132,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_offen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_OFFEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_OFFEN_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   $sgpr0 = COPY [[COPY9]]
@@ -158,7 +158,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_idxen_rtn(double %val, <4 x i32>
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_IDXEN_RTN [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_IDXEN_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   $sgpr0 = COPY [[COPY9]]
@@ -186,7 +186,7 @@ define amdgpu_ps double @buffer_atomic_fadd_f64_bothen_rtn(double %val, <4 x i32
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
   ; GFX90A_GFX940-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[COPY9]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN [[COPY9]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub0
   ; GFX90A_GFX940-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_F64_BOTHEN_RTN]].sub1
   ; GFX90A_GFX940-NEXT:   $sgpr0 = COPY [[COPY10]]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-no-rtn.ll
index 3c2c90d68bfab..3d95e130b9536 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-no-rtn.ll
@@ -15,7 +15,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offset_no_rtn(<2 x half> %val, <
   ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_offset_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -28,7 +28,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offset_no_rtn(<2 x half> %val, <
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFSET [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 4095, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
   ret void
@@ -47,7 +47,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offen_no_rtn(<2 x half> %val, <4
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_offen_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -61,7 +61,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_offen_no_rtn(<2 x half> %val, <4
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_OFFEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -80,7 +80,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_idxen_no_rtn(<2 x half> %val, <4
   ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_idxen_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -94,7 +94,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_idxen_no_rtn(<2 x half> %val, <4
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_IDXEN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
@@ -115,7 +115,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_bothen_no_rtn(<2 x half> %val, <
   ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX908-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A_GFX940-LABEL: name: buffer_atomic_fadd_v2f16_bothen_no_rtn
   ; GFX90A_GFX940: bb.0 (%ir-block.0):
@@ -131,7 +131,7 @@ define amdgpu_ps void @buffer_atomic_fadd_v2f16_bothen_no_rtn(<2 x half> %val, <
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   BUFFER_ATOMIC_PK_ADD_F16_BOTHEN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 2, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   S_ENDPGM 0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-rtn.ll b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-rtn.ll
index 555d4ca394b14..c36a37608b945 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-atomic-fadd.v2f16-rtn.ll
@@ -14,7 +14,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_offset_rtn(<2 x half> %val
   ; GFX90A_GFX940-NEXT:   [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN [[COPY5]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_OFFSET_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
@@ -34,7 +34,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_offen_rtn(<2 x half> %val,
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_OFFEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
@@ -54,7 +54,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_idxen_rtn(<2 x half> %val,
   ; GFX90A_GFX940-NEXT:   [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
   ; GFX90A_GFX940-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_IDXEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
@@ -76,7 +76,7 @@ define amdgpu_ps <2 x half> @buffer_atomic_fadd_v2f16_bothen_rtn(<2 x half> %val
   ; GFX90A_GFX940-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
   ; GFX90A_GFX940-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GFX90A_GFX940-NEXT:   [[BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN [[COPY7]], killed [[REG_SEQUENCE1]], killed [[REG_SEQUENCE]], [[COPY]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX90A_GFX940-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_PK_ADD_F16_BOTHEN_RTN]]
   ; GFX90A_GFX940-NEXT:   SI_RETURN_TO_EPILOG $vgpr0
   %ret = call <2 x half> @llvm.amdgcn.struct.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
index 9867d1208f257..2f6c942372bcc 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
@@ -11,186 +11,186 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
   ; GCN-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
   ; GCN-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1
   ; GCN-NEXT:   [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM killed [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg0, addrspace 6)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 16, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 16, align 1, addrspace 8)
   ; GCN-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 32, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 32, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 32, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 48, align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 48, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 48, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 64, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 64, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 64, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 80, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 80, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 96, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 96, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 96, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[V_MOV_B32_e32_1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 112, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFSET [[V_MOV_B32_e32_1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 112, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_OFFEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_F32_IDXEN [[V_MOV_B32_e32_1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 112, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 128, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 128, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 64
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 128, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 128, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 128
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 128, align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 128, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY2]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY2]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 144, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 144, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 72
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_3]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 144, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_3]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 144, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 144
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 144, align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 144, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY3]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY3]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 160, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 160, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 80
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 160, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 160, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 160
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 160, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 160, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY4]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY4]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 176, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 176, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sreg_32 = S_MOV_B32 88
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 176, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 176, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sreg_32 = S_MOV_B32 176
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 176, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 176, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[COPY5]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[COPY5]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 192, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 192, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sreg_32 = S_MOV_B32 96
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_9]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 192, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_9]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 192, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sreg_32 = S_MOV_B32 192
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 192, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 192, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 208, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 208, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 104
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_11]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 208, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_11]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 208, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 208
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 208, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 208, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY7]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[COPY7]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY8]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 224, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY8]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 224, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 112
   ; GCN-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY9]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 224, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY9]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 224, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sreg_32 = S_MOV_B32 224
   ; GCN-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY10]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 224, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY10]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 224, align 1, addrspace 8)
   ; GCN-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY12:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY11]], [[S_LOAD_DWORDX4_IMM]], [[COPY12]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY11]], [[S_LOAD_DWORDX4_IMM]], [[COPY12]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY13]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 240, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY13]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 240, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 120
   ; GCN-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY14]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 240, align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY14]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 240, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 240
   ; GCN-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY15]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 240, align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY15]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from unknown-address + 240, align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY17:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY16]], [[S_LOAD_DWORDX4_IMM]], [[COPY17]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY16]], [[S_LOAD_DWORDX4_IMM]], [[COPY17]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY18]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 256, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY18]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 256, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 256, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 256, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 256
   ; GCN-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 256, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 256, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY22:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[COPY22]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[COPY22]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY23:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY23]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 272, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY23]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 272, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sreg_32 = S_MOV_B32 136
   ; GCN-NEXT:   [[COPY24:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY24]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 272, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY24]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 272, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sreg_32 = S_MOV_B32 272
   ; GCN-NEXT:   [[COPY25:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY25]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 272, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY25]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on unknown-address + 272, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY26:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY27:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY26]], [[S_LOAD_DWORDX4_IMM]], [[COPY27]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY26]], [[S_LOAD_DWORDX4_IMM]], [[COPY27]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY28:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 288, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 288, align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY29:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 288, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 288, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sreg_32 = S_MOV_B32 288
   ; GCN-NEXT:   [[COPY30:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 288, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 288, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY31:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY32:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[COPY32]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[COPY32]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */
   ; GCN-NEXT:   [[COPY33:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY33]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 304, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY33]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 304, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 152
   ; GCN-NEXT:   [[COPY34:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY34]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 304, align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY34]], [[S_LOAD_DWORDX4_IMM]], killed [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 304, align 1, addrspace 8)
   ; GCN-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 304
   ; GCN-NEXT:   [[COPY35:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[COPY35]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 304, align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[COPY35]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into unknown-address + 304, align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY36:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
   ; GCN-NEXT:   [[COPY37:%[0-9]+]]:sreg_32 = COPY [[COPY]]
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY36]], [[S_LOAD_DWORDX4_IMM]], [[COPY37]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY36]], [[S_LOAD_DWORDX4_IMM]], [[COPY37]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GCN-NEXT:   BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
   ; GCN-NEXT:   S_ENDPGM 0
 bb.0:
   %tmp0 = load <4 x i32>, ptr addrspace(6) %arg0, align 16, !invariant.load !0
diff --git a/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll b/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll
new file mode 100644
index 0000000000000..a9cbcf073af61
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bug-deadlanes.ll
@@ -0,0 +1,90 @@
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -amdgpu-codegenprepare-break-large-phis=false < %s | FileCheck %s
+
+; CHECK-LABEL: {{^}}_amdgpu_ps_main:
+;
+; This test case used to hit an assertion in SIOptimizeVGPRLiveRange because of a vector with dead lanes
+; leading to an effectively dead INSERT_SUBREG.
+
+
+define dllexport amdgpu_ps void @_amdgpu_ps_main(i32 %descTable2) #0 {
+.entry:
+  %i2 = zext i32 %descTable2 to i64
+  %i4 = inttoptr i64 %i2 to ptr addrspace(4)
+  %i159 = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32 15, float poison, float poison, float poison, float poison, float poison, float poison, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0)
+  %i1540 = shufflevector <4 x float> %i159, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  %i1526 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> poison, i32 2688, i32 0)
+  %i1746 = load <4 x i32>, ptr addrspace(4) %i4, align 16
+  br label %bb1750
+
+bb1750:                                           ; preds = %bb1897, %.entry
+  %__llpc_global_proxy_r3.12.vec.extract2358295 = phi i32 [ 0, %.entry ], [ %__llpc_global_proxy_r3.12.vec.extract2358, %bb1897 ]
+  %__llpc_global_proxy_r13.20293 = phi <4 x i32> [ undef, %.entry ], [ %__llpc_global_proxy_r13.22, %bb1897 ]
+  %__llpc_global_proxy_r10.19291 = phi <4 x i32> [ poison, %.entry ], [ %i1914, %bb1897 ]
+  %i1751 = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %i1746, i32 poison, i32 0, i32 0, i32 0)
+  %i1754 = shufflevector <4 x i32> %__llpc_global_proxy_r10.19291, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  %__llpc_global_proxy_r7.12.vec.extract1953260 = bitcast float %i1751 to i32
+  %i1760 = or i32 %__llpc_global_proxy_r7.12.vec.extract1953260, %__llpc_global_proxy_r3.12.vec.extract2358295
+  %i1786 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> poison, i32 %i1760, i32 0)
+  %__llpc_global_proxy_r11.12.vec.insert1237 = insertelement <4 x i32> %i1754, i32 %i1786, i64 3
+  %.not2783 = icmp eq i32 %i1786, 0
+  br i1 %.not2783, label %bb1789, label %bb1787
+
+bb1787:                                           ; preds = %bb1750
+  %i1788 = shufflevector <4 x i32> <i32 0, i32 0, i32 0, i32 poison>, <4 x i32> %__llpc_global_proxy_r13.20293, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  br label %bb1897
+
+bb1789:                                           ; preds = %bb1750
+  %i1796 = call reassoc nnan nsz arcp contract afn float @llvm.amdgcn.fmed3.f32(float poison, float 0.000000e+00, float 1.000000e+00)
+  %i1797 = fmul reassoc nnan nsz arcp contract afn float %i1796, %i1796
+  %i1800 = fdiv reassoc nnan nsz arcp contract afn float %i1797, 0.000000e+00
+  %i1801 = bitcast float %i1800 to i32
+  %__llpc_global_proxy_r11.12.vec.insert1245 = insertelement <4 x i32> %__llpc_global_proxy_r11.12.vec.insert1237, i32 poison, i64 3
+  %i1818 = call reassoc nnan nsz arcp contract afn float @llvm.amdgcn.fmed3.f32(float poison, float 0.000000e+00, float 1.000000e+00)
+  %i1819 = bitcast float %i1818 to i32
+  %i1878 = shufflevector <4 x i32> %__llpc_global_proxy_r11.12.vec.insert1245, <4 x i32> poison, <3 x i32> <i32 3, i32 3, i32 3>
+  %i1879 = bitcast <3 x i32> %i1878 to <3 x float>
+  %i1881 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1540, %i1879
+  %i1882 = call <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32> poison, i32 poison, i32 0)
+  %i1883 = shufflevector <3 x i32> %i1882, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %i1884 = bitcast <4 x i32> %i1883 to <4 x float>
+  %i1885 = shufflevector <4 x float> %i1884, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  %i1886 = insertelement <3 x i32> undef, i32 %i1819, i64 0
+  %i1887 = bitcast <3 x i32> %i1886 to <3 x float>
+  %i1888 = insertelement <3 x i32> undef, i32 %i1801, i64 0
+  %i1889 = bitcast <3 x i32> %i1888 to <3 x float>
+  %i1890 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1887, %i1889
+  %i1891 = shufflevector <3 x float> %i1890, <3 x float> poison, <3 x i32> zeroinitializer
+  %i1892 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1885, %i1891
+  %i1893 = fmul reassoc nnan nsz arcp contract afn <3 x float> %i1892, %i1881
+  %i1894 = bitcast <3 x float> %i1893 to <3 x i32>
+  %i1895 = shufflevector <3 x i32> %i1894, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %i1896 = insertelement <4 x i32> %i1895, i32 %i1819, i64 3
+  br label %bb1897
+
+bb1897:                                           ; preds = %bb1789, %bb1787
+  %__llpc_global_proxy_r11.19 = phi <4 x i32> [ %__llpc_global_proxy_r11.12.vec.insert1237, %bb1787 ], [ %__llpc_global_proxy_r11.12.vec.insert1245, %bb1789 ]
+  %__llpc_global_proxy_r13.22 = phi <4 x i32> [ %i1788, %bb1787 ], [ %i1896, %bb1789 ]
+  %i1898 = shufflevector <4 x i32> %__llpc_global_proxy_r11.19, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  %i1899 = bitcast <3 x i32> %i1898 to <3 x float>
+  %i1900 = shufflevector <4 x i32> %__llpc_global_proxy_r13.22, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  %i1901 = bitcast <3 x i32> %i1900 to <3 x float>
+  %i1902 = fadd reassoc nnan nsz arcp contract afn <3 x float> %i1901, %i1899
+  %i1903 = bitcast <3 x float> %i1902 to <3 x i32>
+  %i1907 = shufflevector <3 x i32> %i1903, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %i1908 = shufflevector <4 x i32> %i1907, <4 x i32> %__llpc_global_proxy_r11.19, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  %i1914 = shufflevector <4 x i32> %i1908, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  %__llpc_global_proxy_r3.12.vec.extract2358 = extractelement <2 x i32> zeroinitializer, i64 1
+  %.not2780.not = icmp ult i32 %__llpc_global_proxy_r3.12.vec.extract2358, %i1526
+  br i1 %.not2780.not, label %bb1750, label %._crit_edge298
+
+._crit_edge298:                                   ; preds = %bb1897
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32 immarg, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg)
+declare float @llvm.amdgcn.fmed3.f32(float, float, float)
+declare float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32>, i32, i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32 immarg)
+declare <3 x i32> @llvm.amdgcn.s.buffer.load.v3i32(<4 x i32>, i32, i32 immarg)
+
+attributes #0 = { "target-features"=",+wavefrontsize64,+cumode" }
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
index 66cfa21b6dbf5..2237c243e6342 100644
--- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
@@ -7,7 +7,7 @@
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; OPT-LABEL: @test_sink_global_small_offset_i32(
 ; OPT-CI-NOT: getelementptr i32, ptr addrspace(1) %in
diff --git a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
index d04ffa351b797..a83e98ba7773b 100644
--- a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
@@ -213,8 +213,6 @@ define amdgpu_kernel void @v3i16_registers(i1 %cond) #0 {
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:  .LBB4_3: ; %if.end
-; GCN-NEXT:    s_mov_b32 s4, 0xffff
-; GCN-NEXT:    v_bfi_b32 v0, s4, v0, v0
 ; GCN-NEXT:    global_store_short v[0:1], v1, off
 ; GCN-NEXT:    global_store_dword v[0:1], v0, off
 ; GCN-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/cse-convergent.ll b/llvm/test/CodeGen/AMDGPU/cse-convergent.ll
new file mode 100644
index 0000000000000..ed590763971b3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/cse-convergent.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN
+
+define i32 @test(i32 %val, i32 %cond) {
+; GCN-LABEL: test:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    s_xor_saveexec_b32 s4, -1
+; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; GCN-NEXT:    s_waitcnt_depctr 0xffe3
+; GCN-NEXT:    s_mov_b32 exec_lo, s4
+; GCN-NEXT:    s_or_saveexec_b32 s4, -1
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_mov_b32 exec_lo, s4
+; GCN-NEXT:    v_mov_b32_e32 v3, v0
+; GCN-NEXT:    s_not_b32 exec_lo, exec_lo
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_not_b32 exec_lo, exec_lo
+; GCN-NEXT:    s_or_saveexec_b32 s4, -1
+; GCN-NEXT:    v_mov_b32_dpp v2, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GCN-NEXT:    s_mov_b32 exec_lo, s4
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GCN-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GCN-NEXT:  ; %bb.1: ; %if
+; GCN-NEXT:    s_or_saveexec_b32 s5, -1
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_mov_b32 exec_lo, s5
+; GCN-NEXT:    v_mov_b32_e32 v3, v0
+; GCN-NEXT:    s_not_b32 exec_lo, exec_lo
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_not_b32 exec_lo, exec_lo
+; GCN-NEXT:    s_or_saveexec_b32 s5, -1
+; GCN-NEXT:    v_mov_b32_dpp v2, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GCN-NEXT:    s_mov_b32 exec_lo, s5
+; GCN-NEXT:    v_mov_b32_e32 v5, v2
+; GCN-NEXT:  ; %bb.2: ; %end
+; GCN-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GCN-NEXT:    v_add_nc_u32_e32 v0, v4, v5
+; GCN-NEXT:    s_xor_saveexec_b32 s4, -1
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32
+; GCN-NEXT:    buffer_load_dword v3, off, s[0:3], s32 offset:4
+; GCN-NEXT:    s_waitcnt_depctr 0xffe3
+; GCN-NEXT:    s_mov_b32 exec_lo, s4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  ; %x = subgroup operation over all lanes.
+  %x1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %val, i32 0)
+  %x2 = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %x1, i32 353, i32 15, i32 15, i1 false)
+  %x3 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %x2)
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %if, label %end
+
+if:
+  ; %y = subgroup operations over lanes satisfying %cond.
+  %y1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %val, i32 0)
+  %y2 = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %y1, i32 353, i32 15, i32 15, i1 false)
+  %y3 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %y2)
+  br label %end
+
+end:
+  %y = phi i32 [0, %entry], [%y3, %if]
+  %ret = add i32 %x3, %y
+  ret i32 %ret
+}
+
+declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32)
+declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32 immarg, i32 immarg, i32 immarg, i1 immarg)
+declare i32 @llvm.amdgcn.strict.wwm.i32(i32)
diff --git a/llvm/test/CodeGen/AMDGPU/elf-notes.ll b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
index 7c78a92486494..37299366e13c3 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-notes.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-notes.ll
@@ -25,40 +25,38 @@
 ; OSABI-UNK-ELF-NOT: NT_AMD_PAL_METADATA (AMD PAL Metadata)
 ; OSABI-UNK-ELF-NOT: Unknown note type
 
-; OSABI-HSA: .hsa_code_object_version
-; OSABI-HSA: .hsa_code_object_isa
-; OSABI-HSA: .amd_amdgpu_isa "amdgcn-amd-amdhsa--gfx802"
-; OSABI-HSA: .amd_amdgpu_hsa_metadata
+; OSABI-HSA: amdhsa.target:   amdgcn-amd-amdhsa--gfx802
+; OSABI-HSA: amdhsa.version:
+; OSABI-HSA: .end_amdgpu_metadata
 ; OSABI-HSA-NOT: .amd_amdgpu_pal_metadata
 
-; OSABI-HSA-ELF: NT_AMD_HSA_CODE_OBJECT_VERSION (AMD HSA Code Object Version)
-; OSABI-HSA-ELF: NT_AMD_HSA_ISA_VERSION (AMD HSA ISA Version)
-; OSABI-HSA-ELF: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
-; OSABI-HSA-ELF: AMD HSA ISA Name:
-; OSABI-HSA-ELF: amdgcn-amd-amdhsa--gfx802
-; OSABI-HSA-ELF: NT_AMD_HSA_METADATA (AMD HSA Metadata)
-; OSABI-HSA-ELF: HSA Metadata:
+; OSABI-HSA-ELF: NT_AMDGPU_METADATA (AMDGPU Metadata)
 ; OSABI-HSA-ELF: ---
-; OSABI-HSA-ELF: Version: [ 1, 0 ]
-; OSABI-HSA-ELF: Kernels:
-; OSABI-HSA-ELF:   - Name:       elf_notes
-; OSABI-HSA-ELF:     SymbolName: 'elf_notes@kd'
-; OSABI-HSA-ELF:     CodeProps:
-; OSABI-HSA-ELF:       KernargSegmentSize: 0
-; OSABI-HSA-ELF:       GroupSegmentFixedSize: 0
-; OSABI-HSA-ELF:       PrivateSegmentFixedSize: 0
-; OSABI-HSA-ELF:       KernargSegmentAlign: 4
-; OSABI-HSA-ELF:       WavefrontSize:   64
-; OSABI-HSA-ELF:       NumSGPRs:        96
+; OSABI-HSA-ELF: amdhsa.kernels:
+; OSABI-HSA-ELF:   - .args:           []
+; OSABI-HSA-ELF:     .group_segment_fixed_size: 0
+; OSABI-HSA-ELF:     .kernarg_segment_align: 4
+; OSABI-HSA-ELF:     .kernarg_segment_size: 0
+; OSABI-HSA-ELF:     .max_flat_workgroup_size: 1024
+; OSABI-HSA-ELF:     .name:           elf_notes
+; OSABI-HSA-ELF:     .private_segment_fixed_size: 0
+; OSABI-HSA-ELF:     .sgpr_count:     96
+; OSABI-HSA-ELF:     .sgpr_spill_count: 0
+; OSABI-HSA-ELF:     .symbol:         elf_notes.kd
+; OSABI-HSA-ELF:     .vgpr_count:     0
+; OSABI-HSA-ELF:     .vgpr_spill_count: 0
+; OSABI-HSA-ELF:     .wavefront_size: 64
+; OSABI-HSA-ELF: amdhsa.target:   amdgcn-amd-amdhsa--gfx802
+; OSABI-HSA-ELF: amdhsa.version:
+; OSABI-HSA-ELF:   - 1
+; OSABI-HSA-ELF:   - 1
 ; OSABI-HSA-ELF: ...
 ; OSABI-HSA-ELF-NOT: NT_AMD_PAL_METADATA (AMD PAL Metadata)
 
-; OSABI-PAL-NOT: .hsa_code_object_version
-; OSABI-PAL: .hsa_code_object_isa
-; OSABI-PAL: .amd_amdgpu_isa "amdgcn-amd-amdpal--gfx802"
+; OSABI-PAL: .amd_amdgpu_isa  "amdgcn-amd-amdpal--gfx802"
+; OSABI-PAL: .amdgpu_pal_metadata
 ; OSABI-PAL-NOT: .amd_amdgpu_hsa_metadata
 
-; OSABI-PAL-ELF: NT_AMD_HSA_ISA_VERSION (AMD HSA ISA Version)
 ; OSABI-PAL-ELF: NT_AMD_HSA_ISA_NAME (AMD HSA ISA Name)
 ; OSABI-PAL-ELF: AMD HSA ISA Name:
 ; OSABI-PAL-ELF: amdgcn-amd-amdpal--gfx802
@@ -75,16 +73,6 @@
 ; OSABI-PAL-ELF:     .registers:
 ; OSABI-PAL-ELF:       11794:           11469504
 ; OSABI-PAL-ELF:       11795:           128
-; OSABI-PAL: amdpal.pipelines:
-; OSABI-PAL:   - .hardware_stages:
-; OSABI-PAL:       .cs:
-; OSABI-PAL:         .entry_point:    elf_notes
-; OSABI-PAL:         .scratch_memory_size: 0
-; OSABI-PAL:         .sgpr_count:     0x60
-; OSABI-PAL:         .vgpr_count:     0x1
-; OSABI-PAL:     .registers:
-; OSABI-PAL:       0x2e12 (COMPUTE_PGM_RSRC1): 0xaf02c0
-; OSABI-PAL:       0x2e13 (COMPUTE_PGM_RSRC2): 0x80
 
 ; R600-NOT: .hsa_code_object_version
 ; R600-NOT: .hsa_code_object_isa
@@ -97,4 +85,4 @@ define amdgpu_kernel void @elf_notes() {
 }
 
 !llvm.module.flags = !{!0}
-!0 = !{i32 1, !"amdgpu_code_object_version", i32 200}
+!0 = !{i32 1, !"amdgpu_code_object_version", i32 400}
diff --git a/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll b/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll
index 64a4f72a7c0c6..9391b50c04a5f 100644
--- a/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/enqueue-kernel.ll
@@ -8,11 +8,6 @@ define amdgpu_kernel void @non_caller(ptr addrspace(1) %a, i8 %b, ptr addrspace(
   ret void
 }
 
-define amdgpu_kernel void @caller_indirect(ptr addrspace(1) %a, i8 %b, ptr addrspace(1) %c, i64 %d) {
-  call void @caller(ptr addrspace(1) %a, i8 %b, ptr addrspace(1) %c, i64 %d)
-  ret void
-}
-
 define amdgpu_kernel void @caller(ptr addrspace(1) %a, i8 %b, ptr addrspace(1) %c, i64 %d) {
 entry:
   %block = alloca <{ i32, i32, ptr addrspace(1), i8 }>, align 8, addrspace(5)
@@ -120,12 +115,6 @@ attributes #0 = { "enqueued-block" }
 ; CHECK-NEXT:    ret void
 ;
 ;
-; CHECK-LABEL: define {{[^@]+}}@caller_indirect
-; CHECK-SAME: (ptr addrspace(1) [[A:%.*]], i8 [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:    call void @caller(ptr addrspace(1) [[A]], i8 [[B]], ptr addrspace(1) [[C]], i64 [[D]])
-; CHECK-NEXT:    ret void
-;
-;
 ; CHECK-LABEL: define {{[^@]+}}@caller
 ; CHECK-SAME: (ptr addrspace(1) [[A:%.*]], i8 [[B:%.*]], ptr addrspace(1) [[C:%.*]], i64 [[D:%.*]]) {
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index 24df34e94b4e7..6c564b244012a 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -916,3 +916,79 @@ exit:
   %r2 = select <4 x i1> %b2, <4 x half> <half 0xH3900, half 0xH3900, half 0xH3900, half 0xH3900>, <4 x half> <half 0xH3D00, half 0xH3D00, half 0xH3D00, half 0xH3D00>
   ret <4 x half> %r2
 }
+
+define <8 x i16> @large_vector(ptr addrspace(3) %p, i32 %idxp) {
+; SI-LABEL: large_vector:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 5, v1
+; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; SI-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 8, v0
+; SI-NEXT:    v_add_i32_e32 v5, vcc, 12, v0
+; SI-NEXT:    s_mov_b32 m0, -1
+; SI-NEXT:    ds_read_b32 v0, v0
+; SI-NEXT:    ds_read_b32 v2, v1
+; SI-NEXT:    ds_read_b32 v4, v3
+; SI-NEXT:    ds_read_b32 v6, v5
+; SI-NEXT:    s_waitcnt lgkmcnt(3)
+; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(2)
+; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; SI-NEXT:    s_waitcnt lgkmcnt(1)
+; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: large_vector:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_lshl_add_u32 v2, v1, 5, v0
+; GFX9-NEXT:    ds_read2_b32 v[0:1], v2 offset1:1
+; GFX9-NEXT:    ds_read2_b32 v[2:3], v2 offset0:2 offset1:3
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %idx = shl i32 %idxp, 4
+
+  %i.0 = or i32 %idx, 0
+  %p.0 = getelementptr half, ptr addrspace(3) %p, i32 %i.0
+  %x.0 = load i16, ptr addrspace(3) %p.0, align 4
+  %v0p = insertelement <8 x i16> poison, i16 %x.0, i32 0
+  %i.1 = or i32 %idx, 1
+  %p.1 = getelementptr half, ptr addrspace(3) %p, i32 %i.1
+  %x.1 = load i16, ptr addrspace(3) %p.1, align 2
+  %v0 = insertelement <8 x i16> %v0p, i16 %x.1, i32 1
+
+  %i.2 = or i32 %idx, 2
+  %p.2 = getelementptr half, ptr addrspace(3) %p, i32 %i.2
+  %x.2 = load i16, ptr addrspace(3) %p.2, align 4
+  %v1p = insertelement <8 x i16> poison, i16 %x.2, i32 0
+  %i.3 = or i32 %idx, 3
+  %p.3 = getelementptr half, ptr addrspace(3) %p, i32 %i.3
+  %x.3 = load i16, ptr addrspace(3) %p.3, align 2
+  %v1 = insertelement <8 x i16> %v1p, i16 %x.3, i32 1
+
+  %i.4 = or i32 %idx, 4
+  %p.4 = getelementptr half, ptr addrspace(3) %p, i32 %i.4
+  %x.4 = load i16, ptr addrspace(3) %p.4, align 4
+  %v2p = insertelement <8 x i16> poison, i16 %x.4, i32 0
+  %i.5 = or i32 %idx, 5
+  %p.5 = getelementptr half, ptr addrspace(3) %p, i32 %i.5
+  %x.5 = load i16, ptr addrspace(3) %p.5, align 2
+  %v2 = insertelement <8 x i16> %v2p, i16 %x.5, i32 1
+
+  %i.6 = or i32 %idx, 6
+  %p.6 = getelementptr half, ptr addrspace(3) %p, i32 %i.6
+  %x.6 = load i16, ptr addrspace(3) %p.6, align 4
+  %v3p = insertelement <8 x i16> poison, i16 %x.6, i32 0
+  %i.7 = or i32 %idx, 7
+  %p.7 = getelementptr half, ptr addrspace(3) %p, i32 %i.7
+  %x.7 = load i16, ptr addrspace(3) %p.7, align 2
+  %v3 = insertelement <8 x i16> %v3p, i16 %x.7, i32 1
+
+  %z.1 = shufflevector <8 x i16> %v0, <8 x i16> %v1, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+  %z.2 = shufflevector <8 x i16> %z.1, <8 x i16> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
+  %z.3 = shufflevector <8 x i16> %z.2, <8 x i16> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+  ret <8 x i16> %z.3
+}
diff --git a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
index 3a4999ad17f4b..d3d7b0a95b634 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
@@ -12,7 +12,7 @@ define amdgpu_hs void @main(ptr addrspace(6) inreg %arg) {
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
   ; GCN-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
   ; GCN-NEXT:   [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
-  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 7)
+  ; GCN-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
   ; GCN-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
   ; GCN-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
   ; GCN-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
@@ -21,7 +21,7 @@ define amdgpu_hs void @main(ptr addrspace(6) inreg %arg) {
   ; GCN-NEXT:   [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
   ; GCN-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF2]]
   ; GCN-NEXT:   [[DEF3:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
-  ; GCN-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 7)
+  ; GCN-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 8)
   ; GCN-NEXT:   S_ENDPGM 0
 main_body:
   %tmp25 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> undef, i32 undef, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll b/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll
index 788487f2e9bee..4a7b5961827dd 100644
--- a/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll
+++ b/llvm/test/CodeGen/AMDGPU/force-alwaysinline-lds-global-address.ll
@@ -3,7 +3,7 @@
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-stress-function-calls -passes=amdgpu-always-inline -amdgpu-enable-lower-module-lds=false %s | FileCheck --check-prefix=ALL %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-stress-function-calls -passes=amdgpu-always-inline -amdgpu-enable-lower-module-lds=false %s | FileCheck --check-prefix=ALL %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 @lds0 = addrspace(3) global i32 undef, align 4
 @lds1 = addrspace(3) global [512 x i32] undef, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll b/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll
index 67e745a8a8386..6e95091a6b98c 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-asm.i128.ll
@@ -8,15 +8,15 @@
 define amdgpu_kernel void @s_input_output_i128() {
   ; GFX908-LABEL: name: s_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6946826 /* regdef:SGPR_128 */, def %4
+  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 7143434 /* regdef:SGPR_128 */, def %4
   ; GFX908-NEXT:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %4
-  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6946825 /* reguse:SGPR_128 */, [[COPY]]
+  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 7143433 /* reguse:SGPR_128 */, [[COPY]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: s_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6946826 /* regdef:SGPR_128 */, def %4
+  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 7143434 /* regdef:SGPR_128 */, def %4
   ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:sgpr_128 = COPY %4
-  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6946825 /* reguse:SGPR_128 */, [[COPY]]
+  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 7143433 /* reguse:SGPR_128 */, [[COPY]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   %val = tail call i128 asm sideeffect "; def $0", "=s"()
   call void asm sideeffect "; use $0", "s"(i128 %val)
@@ -26,15 +26,15 @@ define amdgpu_kernel void @s_input_output_i128() {
 define amdgpu_kernel void @v_input_output_i128() {
   ; GFX908-LABEL: name: v_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 5898250 /* regdef:VReg_128 */, def %4
+  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %4
   ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vreg_128 = COPY %4
-  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 5898249 /* reguse:VReg_128 */, [[COPY]]
+  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6094857 /* reguse:VReg_128 */, [[COPY]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: v_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6225930 /* regdef:VReg_128_Align2 */, def %4
+  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %4
   ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:vreg_128_align2 = COPY %4
-  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6225929 /* reguse:VReg_128_Align2 */, [[COPY]]
+  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6422537 /* reguse:VReg_128_Align2 */, [[COPY]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   %val = tail call i128 asm sideeffect "; def $0", "=v"()
   call void asm sideeffect "; use $0", "v"(i128 %val)
@@ -44,15 +44,15 @@ define amdgpu_kernel void @v_input_output_i128() {
 define amdgpu_kernel void @a_input_output_i128() {
   ; GFX908-LABEL: name: a_input_output_i128
   ; GFX908: bb.0 (%ir-block.0):
-  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 5832714 /* regdef:AReg_128 */, def %4
+  ; GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6029322 /* regdef:AReg_128 */, def %4
   ; GFX908-NEXT:   [[COPY:%[0-9]+]]:areg_128 = COPY %4
-  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 5832713 /* reguse:AReg_128 */, [[COPY]]
+  ; GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6029321 /* reguse:AReg_128 */, [[COPY]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ; GFX90A-LABEL: name: a_input_output_i128
   ; GFX90A: bb.0 (%ir-block.0):
-  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:AReg_128_Align2 */, def %4
+  ; GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6291466 /* regdef:AReg_128_Align2 */, def %4
   ; GFX90A-NEXT:   [[COPY:%[0-9]+]]:areg_128_align2 = COPY %4
-  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6094857 /* reguse:AReg_128_Align2 */, [[COPY]]
+  ; GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   %val = call i128 asm sideeffect "; def $0", "=a"()
   call void asm sideeffect "; use $0", "a"(i128 %val)
diff --git a/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll
index 93fc998098c48..99ed605e7ed52 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll
@@ -4,12 +4,14 @@
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX11 %s
 
 ; RUN: llc -global-isel -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=G_SI %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s  | FileCheck -enable-var-scope -check-prefix=G_GFX7 %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=G_VI %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=G_GFX9 %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=G_GFX10 %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=G_GFX11 %s
 
 declare float @llvm.amdgcn.ds.fmin.f32(ptr addrspace(3) nocapture, float, i32, i32, i1)
 declare float @llvm.amdgcn.ds.fmax.f32(ptr addrspace(3) nocapture, float, i32, i32, i1)
@@ -154,6 +156,26 @@ define amdgpu_kernel void @lds_ds_fmin(ptr addrspace(5) %out, ptr addrspace(3) %
 ; GFX10-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
 ; GFX10-NEXT:    s_endpgm
 ;
+; GFX11-LABEL: lds_ds_fmin:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b32 s3, s2, 3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0x42280000 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    s_lshl_b32 s2, s2, 4
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT:    ds_min_rtn_f32 v1, v1, v0 offset:32
+; GFX11-NEXT:    ds_min_f32 v2, v0 offset:64
+; GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX11-NEXT:    ds_min_rtn_f32 v0, v3, v1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    scratch_store_b32 off, v0, s0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
 ; G_SI-LABEL: lds_ds_fmin:
 ; G_SI:       ; %bb.0:
 ; G_SI-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
@@ -291,6 +313,28 @@ define amdgpu_kernel void @lds_ds_fmin(ptr addrspace(5) %out, ptr addrspace(3) %
 ; G_GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
 ; G_GFX10-NEXT:    s_endpgm
+;
+; G_GFX11-LABEL: lds_ds_fmin:
+; G_GFX11:       ; %bb.0:
+; G_GFX11-NEXT:    s_clause 0x1
+; G_GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; G_GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; G_GFX11-NEXT:    v_mov_b32_e32 v1, 0x42280000
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    s_add_i32 s2, s2, 4
+; G_GFX11-NEXT:    v_mov_b32_e32 v3, s1
+; G_GFX11-NEXT:    s_lshl_b32 s3, s2, 3
+; G_GFX11-NEXT:    s_lshl_b32 s2, s2, 4
+; G_GFX11-NEXT:    v_mov_b32_e32 v0, s3
+; G_GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; G_GFX11-NEXT:    ds_min_rtn_f32 v0, v0, v1
+; G_GFX11-NEXT:    ds_min_f32 v2, v1
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; G_GFX11-NEXT:    ds_min_rtn_f32 v0, v3, v0
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    scratch_store_b32 off, v0, s0
+; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
   %shl1 = shl i32 %idx.add, 4
@@ -440,6 +484,26 @@ define amdgpu_kernel void @lds_ds_fmax(ptr addrspace(5) %out, ptr addrspace(3) %
 ; GFX10-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen
 ; GFX10-NEXT:    s_endpgm
 ;
+; GFX11-LABEL: lds_ds_fmax:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b32 s3, s2, 3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, 0x42280000 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    s_lshl_b32 s2, s2, 4
+; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT:    ds_max_rtn_f32 v1, v1, v0 offset:32
+; GFX11-NEXT:    ds_max_f32 v2, v0 offset:64
+; GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX11-NEXT:    ds_max_rtn_f32 v0, v3, v1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    scratch_store_b32 off, v0, s0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
 ; G_SI-LABEL: lds_ds_fmax:
 ; G_SI:       ; %bb.0:
 ; G_SI-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
@@ -577,6 +641,28 @@ define amdgpu_kernel void @lds_ds_fmax(ptr addrspace(5) %out, ptr addrspace(3) %
 ; G_GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX10-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen
 ; G_GFX10-NEXT:    s_endpgm
+;
+; G_GFX11-LABEL: lds_ds_fmax:
+; G_GFX11:       ; %bb.0:
+; G_GFX11-NEXT:    s_clause 0x1
+; G_GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; G_GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; G_GFX11-NEXT:    v_mov_b32_e32 v1, 0x42280000
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    s_add_i32 s2, s2, 4
+; G_GFX11-NEXT:    v_mov_b32_e32 v3, s1
+; G_GFX11-NEXT:    s_lshl_b32 s3, s2, 3
+; G_GFX11-NEXT:    s_lshl_b32 s2, s2, 4
+; G_GFX11-NEXT:    v_mov_b32_e32 v0, s3
+; G_GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; G_GFX11-NEXT:    ds_max_rtn_f32 v0, v0, v1
+; G_GFX11-NEXT:    ds_max_f32 v2, v1
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; G_GFX11-NEXT:    ds_max_rtn_f32 v0, v3, v0
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    scratch_store_b32 off, v0, s0
+; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
   %shl1 = shl i32 %idx.add, 4
@@ -753,6 +839,30 @@ define amdgpu_kernel void @lds_ds_fmin_f64(ptr addrspace(5) %out, ptr addrspace(
 ; GFX10-NEXT:    buffer_store_dword v0, v2, s[8:11], 0 offen
 ; GFX10-NEXT:    s_endpgm
 ;
+; GFX11-LABEL: lds_ds_fmin_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_mov_b32 s2, 0
+; GFX11-NEXT:    s_mov_b32 s3, 0x40450000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b32 s5, s4, 3
+; GFX11-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v2, s5
+; GFX11-NEXT:    s_lshl_b32 s2, s4, 4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_mov_b32_e32 v4, s2
+; GFX11-NEXT:    ds_min_rtn_f64 v[2:3], v2, v[0:1] offset:32
+; GFX11-NEXT:    ds_min_f64 v4, v[0:1] offset:64
+; GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX11-NEXT:    ds_min_rtn_f64 v[0:1], v5, v[2:3]
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
 ; G_SI-LABEL: lds_ds_fmin_f64:
 ; G_SI:       ; %bb.0:
 ; G_SI-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
@@ -917,6 +1027,30 @@ define amdgpu_kernel void @lds_ds_fmin_f64(ptr addrspace(5) %out, ptr addrspace(
 ; G_GFX10-NEXT:    buffer_store_dword v0, v2, s[8:11], 0 offen
 ; G_GFX10-NEXT:    buffer_store_dword v1, v2, s[8:11], 0 offen offset:4
 ; G_GFX10-NEXT:    s_endpgm
+;
+; G_GFX11-LABEL: lds_ds_fmin_f64:
+; G_GFX11:       ; %bb.0:
+; G_GFX11-NEXT:    s_clause 0x1
+; G_GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; G_GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    s_add_i32 s4, s2, 4
+; G_GFX11-NEXT:    s_mov_b32 s2, 0
+; G_GFX11-NEXT:    s_lshl_b32 s5, s4, 3
+; G_GFX11-NEXT:    s_mov_b32 s3, 0x40450000
+; G_GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s1
+; G_GFX11-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v1, s3
+; G_GFX11-NEXT:    s_lshl_b32 s2, s4, 4
+; G_GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; G_GFX11-NEXT:    v_mov_b32_e32 v4, s2
+; G_GFX11-NEXT:    ds_min_rtn_f64 v[2:3], v2, v[0:1]
+; G_GFX11-NEXT:    ds_min_f64 v4, v[0:1]
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; G_GFX11-NEXT:    ds_min_rtn_f64 v[0:1], v5, v[2:3]
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
+; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
   %shl1 = shl i32 %idx.add, 4
@@ -1093,6 +1227,30 @@ define amdgpu_kernel void @lds_ds_fmax_f64(ptr addrspace(5) %out, ptr addrspace(
 ; GFX10-NEXT:    buffer_store_dword v0, v2, s[8:11], 0 offen
 ; GFX10-NEXT:    s_endpgm
 ;
+; GFX11-LABEL: lds_ds_fmax_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_mov_b32 s2, 0
+; GFX11-NEXT:    s_mov_b32 s3, 0x40450000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b32 s5, s4, 3
+; GFX11-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v2, s5
+; GFX11-NEXT:    s_lshl_b32 s2, s4, 4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_mov_b32_e32 v4, s2
+; GFX11-NEXT:    ds_max_rtn_f64 v[2:3], v2, v[0:1] offset:32
+; GFX11-NEXT:    ds_max_f64 v4, v[0:1] offset:64
+; GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; GFX11-NEXT:    ds_max_rtn_f64 v[0:1], v5, v[2:3]
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
 ; G_SI-LABEL: lds_ds_fmax_f64:
 ; G_SI:       ; %bb.0:
 ; G_SI-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
@@ -1257,6 +1415,30 @@ define amdgpu_kernel void @lds_ds_fmax_f64(ptr addrspace(5) %out, ptr addrspace(
 ; G_GFX10-NEXT:    buffer_store_dword v0, v2, s[8:11], 0 offen
 ; G_GFX10-NEXT:    buffer_store_dword v1, v2, s[8:11], 0 offen offset:4
 ; G_GFX10-NEXT:    s_endpgm
+;
+; G_GFX11-LABEL: lds_ds_fmax_f64:
+; G_GFX11:       ; %bb.0:
+; G_GFX11-NEXT:    s_clause 0x1
+; G_GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; G_GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    s_add_i32 s4, s2, 4
+; G_GFX11-NEXT:    s_mov_b32 s2, 0
+; G_GFX11-NEXT:    s_lshl_b32 s5, s4, 3
+; G_GFX11-NEXT:    s_mov_b32 s3, 0x40450000
+; G_GFX11-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s1
+; G_GFX11-NEXT:    v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v1, s3
+; G_GFX11-NEXT:    s_lshl_b32 s2, s4, 4
+; G_GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; G_GFX11-NEXT:    v_mov_b32_e32 v4, s2
+; G_GFX11-NEXT:    ds_max_rtn_f64 v[2:3], v2, v[0:1]
+; G_GFX11-NEXT:    ds_max_f64 v4, v[0:1]
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(1)
+; G_GFX11-NEXT:    ds_max_rtn_f64 v[0:1], v5, v[2:3]
+; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; G_GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
+; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
   %shl1 = shl i32 %idx.add, 4
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.format.f16.ll
new file mode 100644
index 0000000000000..9c1444ffdfdd4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.format.f16.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+; Natural mapping
+define half @raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret half %val
+}
+
+define <2 x half> @raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XY_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_XY_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call <2 x half> @llvm.amdgcn.raw.buffer.load.format.v2f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <2 x half> %val
+}
+
+define <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <4 x half> @llvm.amdgcn.raw.buffer.load.format.v4f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x half> %val
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define half @raw_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret half %val
+}
+
+define <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add_4095(<4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add_4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %voffset = add i32 %voffset.base, 4095
+  %val = call <4 x half> @llvm.amdgcn.raw.buffer.load.format.v4f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x half> %val
+}
+
+declare half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32>, i32, i32, i32 immarg) #0
+declare <2 x half> @llvm.amdgcn.raw.buffer.load.format.v2f16(<4 x i32>, i32, i32, i32 immarg) #0
+declare <3 x half> @llvm.amdgcn.raw.buffer.load.format.v3f16(<4 x i32>, i32, i32, i32 immarg) #0
+declare <4 x half> @llvm.amdgcn.raw.buffer.load.format.v4f16(<4 x i32>, i32, i32, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.format.ll
new file mode 100644
index 0000000000000..2bd9288a29347
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.format.ll
@@ -0,0 +1,352 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+; Natural mapping
+define float @raw_buffer_load_format_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define <2 x float> @raw_buffer_load_format_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_XY_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XY_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XY_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <2 x float> %val
+}
+
+define <3 x float> @raw_buffer_load_format_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub2
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <3 x float> %val
+}
+
+define <4 x float> @raw_buffer_load_format_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub3
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   $vgpr3 = COPY [[COPY9]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x float> %val
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define float @raw_buffer_load_format_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define <4 x float> @raw_buffer_load_format_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add_4095(<4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_format_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add_4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub3
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   $vgpr3 = COPY [[COPY9]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %voffset = add i32 %voffset.base, 4095
+  %val = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x float> %val
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32>, i32, i32, i32 immarg) #0
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32 immarg) #0
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.ll
new file mode 100644
index 0000000000000..a317e2093e01d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.load.ll
@@ -0,0 +1,1925 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+; Natural mapping
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Copies for VGPR arguments
+define float @raw_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc
+define float @raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc and soffset
+define float @raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Natural mapping + glc
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 1)
+  ret float %val
+}
+
+; Natural mapping + slc
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 2, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
+  ret float %val
+}
+
+; Natural mapping + dlc
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 4, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 4)
+  ret float %val
+}
+
+; Natural mapping + slc + dlc
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_dlc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 6, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 6)
+  ret float %val
+}
+
+; Natural mapping + glc + dlc
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_dlc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 5, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 5)
+  ret float %val
+}
+
+; Natural mapping + glc + slc + dlc
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_slc_dlc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc_slc_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 7, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 7)
+  ret float %val
+}
+
+; Natural mapping
+define <2 x float> @raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <2 x float> %val
+}
+
+define <3 x float> @raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX3_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub2
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <3 x float> %val
+}
+
+define <4 x float> @raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub3
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   $vgpr3 = COPY [[COPY9]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x float> %val
+}
+
+define half @raw_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_USHORT_OFFEN]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret half %val
+}
+
+define <2 x half> @raw_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <2 x half> %val
+}
+
+define <4 x half> @raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x half> %val
+}
+
+define float @raw_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  %zext = zext i8 %val to i32
+  %cast = bitcast i32 %zext to float
+  ret float %cast
+}
+
+define float @raw_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_SBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  %zext = sext i8 %val to i32
+  %cast = bitcast i32 %zext to float
+  ret float %cast
+}
+
+define float @raw_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_zext
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  %zext = zext i16 %val to i32
+  %cast = bitcast i32 %zext to float
+  ret float %cast
+}
+
+define float @raw_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_i16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_sext
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_SSHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  %sext = sext i16 %val to i32
+  %cast = bitcast i32 %sext to float
+  ret float %cast
+}
+
+; Waterfall for rsrc
+define half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_USHORT_OFFEN]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret half %val
+}
+
+; Waterfall for rsrc
+define float @raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  %zext = zext i8 %val to i32
+  %cast = bitcast i32 %zext to float
+  ret float %cast
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0(<4 x i32> %rsrc, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095(<4 x i32> %rsrc, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096(<4 x i32> %rsrc, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4096, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16(<4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %voffset = add i32 %voffset.base, 16
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095(<4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %voffset = add i32 %voffset.base, 4095
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096(<4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %voffset = add i32 %voffset.base, 4096
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095(<4 x i32> %rsrc, i32 %voffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096(<4 x i32> %rsrc, i32 %voffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %soffset = add i32 %soffset.base, 16
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095(<4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %soffset = add i32 %soffset.base, 4095
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096(<4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %soffset = add i32 %soffset.base, 4096
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000(<4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %soffset = add i32 %soffset.base, 5000
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000(<4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 904, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %voffset = add i32 %voffset.base, 5000
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32 immarg)
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32 immarg)
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32>, i32, i32, i32 immarg)
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32 immarg)
+
+declare half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32>, i32, i32, i32 immarg)
+declare <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32>, i32, i32, i32 immarg)
+declare <3 x half> @llvm.amdgcn.raw.buffer.load.v3f16(<4 x i32>, i32, i32, i32 immarg)
+declare <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32>, i32, i32, i32 immarg)
+
+declare i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32>, i32, i32, i32 immarg)
+declare i16 @llvm.amdgcn.raw.buffer.load.i16(<4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.format.f16.ll
new file mode 100644
index 0000000000000..58ef4f752382c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.format.f16.ll
@@ -0,0 +1,606 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> %rsrc, half %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16(<4 x i32> %rsrc, half %val, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_X_OFFSET_exact [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Make sure unpack code is emitted outside of loop
+define void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY1]], [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY1]], [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY2]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+
+; Check what happens with offset add inside a waterfall loop
+define void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[COPY8]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.format.f16(half, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.format.f32.ll
new file mode 100644
index 0000000000000..3eb5bd5f1197b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.format.f32.ll
@@ -0,0 +1,708 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f32(<4 x i32> %rsrc, float %val, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32(<4 x i32> %rsrc, <3 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[COPY9]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1, [[COPY7]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY10:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[COPY10]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1, [[COPY7]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY10:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[COPY10]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4095(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY7]], [[COPY]], killed [[REG_SEQUENCE4]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4096(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY7]], [[COPY]], killed [[REG_SEQUENCE4]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_16(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 16, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4095(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4096(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY8]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+
+; Check what happens with offset add inside a waterfall loop
+define void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32_add_4096(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32_add_4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1, [[COPY7]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[COPY10:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[COPY10]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.format.f32(float, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.ll
new file mode 100644
index 0000000000000..0d837e8383c24
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.buffer.store.ll
@@ -0,0 +1,1808 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+; Natural mapping
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define void @raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for soffset
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 1)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 2, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 3, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 3)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_dlc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 4, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 4)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc_dlc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 6, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 6)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_dlc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 5, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 5)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc_dlc(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 7, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 7)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32(<4 x i32> %rsrc, <2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32(<4 x i32> %rsrc, <3 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact [[COPY9]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1, [[COPY7]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY10:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact [[COPY10]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i8(<4 x i32> %rsrc, i32 %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i8
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_BYTE_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %val.trunc = trunc i32 %val to i8
+  call void @llvm.amdgcn.raw.buffer.store.i8(i8 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i16(<4 x i32> %rsrc, i32 %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %val.trunc = trunc i32 %val to i16
+  call void @llvm.amdgcn.raw.buffer.store.i16(i16 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> %rsrc, half %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY7]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4095(<4 x i32> %rsrc, float %val, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFSET_exact [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4096(<4 x i32> %rsrc, float %val, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY1]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY1]], [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY1]], [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096(<4 x i32> %rsrc, <2 x half> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 904, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset.add = add i32 %voffset, 5000
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define void @raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset(<4 x i32> %rsrc, float %val, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY1]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 904, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 5000, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.i8(i8, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.i16(i16, <4 x i32>, i32, i32, i32 immarg)
+
+declare void @llvm.amdgcn.raw.buffer.store.f16(half, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg)
+
+declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.load.f16.ll
new file mode 100644
index 0000000000000..3f16c42c35bf6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.load.f16.ll
@@ -0,0 +1,436 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+define half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret half %val
+}
+
+define <2 x half> @raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XY_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_XY_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret <2 x half> %val
+}
+
+define <4 x half> @raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret <4 x half> %val
+}
+
+define half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret half %val
+}
+
+define half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
+  ret half %val
+}
+
+define half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
+  ret half %val
+}
+
+define half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
+  ret half %val
+}
+
+define half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
+  ret half %val
+}
+
+declare half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+declare <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+declare <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+declare <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.load.ll
new file mode 100644
index 0000000000000..05795b94a6450
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.load.ll
@@ -0,0 +1,501 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+define float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret float %val
+}
+
+define <2 x float> @raw_tbuffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_XY_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret <2 x float> %val
+}
+
+define <3 x float> @raw_tbuffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub2
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret <3 x float> %val
+}
+
+define <4 x float> @raw_tbuffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub3
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX908-NEXT:   $vgpr3 = COPY [[COPY9]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret <4 x float> %val
+}
+
+define float @raw_tbuffer_load_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret float %val
+}
+
+define float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
+  ret float %val
+}
+
+define float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
+  ret float %val
+}
+
+define float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
+  ret float %val
+}
+
+define float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
+  ret float %val
+}
+
+declare float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+declare <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+declare <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+declare <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.store.f16.ll
new file mode 100644
index 0000000000000..b57c09c9361cd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.store.f16.ll
@@ -0,0 +1,545 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+define void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
+  ret void
+}
+
+define void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
+  ret void
+}
+
+define void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
+  ret void
+}
+
+define void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.tbuffer.store.f16(half, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v3f16(<3 x half>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.store.ll
new file mode 100644
index 0000000000000..a5036c6c69570
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.tbuffer.store.ll
@@ -0,0 +1,1377 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+; Natural mapping
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping
+define void @raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY8]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping
+define void @raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[COPY9]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping
+define void @raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF6:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF7:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY8]], %subreg.sub1, [[COPY7]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY10:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE1]]
+  ; GFX908-NEXT:   [[DEF8:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[COPY10]], [[COPY1]], killed [[REG_SEQUENCE4]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define void @raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 1)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset, copy for voffset
+define void @raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
+  ret void
+}
+
+; Natural mapping + glc
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
+  ret void
+}
+
+; Natural mapping + slc
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
+  ret void
+}
+
+; Natural mapping + glc + slc
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
+  ret void
+}
+
+; Natural mapping + dlc
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0(float %val, <4 x i32> %rsrc, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY5]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095(float %val, <4 x i32> %rsrc, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY5]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096(float %val, <4 x i32> %rsrc, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY5]], [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16(float %val, <4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 16, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset = add i32 %voffset.base, 16
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095(float %val, <4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 4095, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset = add i32 %voffset.base, 4095
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096(float %val, <4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset = add i32 %voffset.base, 4096
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095(float %val, <4 x i32> %rsrc, i32 %voffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY5]], [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096(float %val, <4 x i32> %rsrc, i32 %voffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY5]], [[COPY]], killed [[REG_SEQUENCE3]], [[S_MOV_B32_]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %soffset = add i32 %soffset.base, 16
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %soffset = add i32 %soffset.base, 4095
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %soffset = add i32 %soffset.base, 4096
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset.base) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[V_ADD_U32_e64_]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[COPY1]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %soffset = add i32 %soffset.base, 5000
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000(float %val, <4 x i32> %rsrc, i32 %voffset.base, i32 %soffset) {
+  ; GFX908-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX908-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY1]], killed [[S_MOV_B32_]], 0, implicit $exec
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub0, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub1, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[REG_SEQUENCE]].sub0_sub1, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub2, implicit $exec
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[REG_SEQUENCE]].sub3, implicit $exec
+  ; GFX908-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; GFX908-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[REG_SEQUENCE]].sub2_sub3, implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def $scc
+  ; GFX908-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def $scc
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY6]], [[V_ADD_U32_e64_]], killed [[REG_SEQUENCE3]], killed [[V_READFIRSTLANE_B32_4]], 904, 94, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  %voffset = add i32 %voffset.base, 5000
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.tbuffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-soffset-mbuf.ll b/llvm/test/CodeGen/AMDGPU/legalize-soffset-mbuf.ll
new file mode 100644
index 0000000000000..b810bfacccb3d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/legalize-soffset-mbuf.ll
@@ -0,0 +1,635 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -march=amdgcn -mcpu=gfx908 -stop-after=si-fix-sgpr-copies -verify-machineinstrs < %s | FileCheck -check-prefix=GFX908 %s
+
+define float @llvm_amdgcn_raw_buffer_load_f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_load_f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+define float @llvm_amdgcn_raw_tbuffer_load_f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_load_f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_X_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_X_OFFEN]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret float %val
+}
+
+define <2 x float> @llvm_amdgcn_raw_buffer_load_v2f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_load_v2f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY2]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY3]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret <2 x float> %val
+}
+
+define <2 x float> @llvm_amdgcn_raw_tbuffer_load_v2f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_load_v2f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_XY_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_XY_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XY_OFFEN]].sub1
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY2]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY3]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %val = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret <2 x float> %val
+}
+
+define <3 x float> @llvm_amdgcn_raw_buffer_load_v3f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_load_v3f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX3_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFEN]].sub2
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY2]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY3]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY4]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret <3 x float> %val
+}
+
+define <3 x float> @llvm_amdgcn_raw_tbuffer_load_v3f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_load_v3f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_OFFEN]].sub2
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY2]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY3]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY4]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %val = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret <3 x float> %val
+}
+
+define <4 x float> @llvm_amdgcn_raw_buffer_load_v4f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_load_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub3
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY2]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY3]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY4]]
+  ; GFX908-NEXT:   $vgpr3 = COPY [[COPY5]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret <4 x float> %val
+}
+
+define <4 x float> @llvm_amdgcn_raw_tbuffer_load_v4f32(i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_load_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub0
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFEN]].sub3
+  ; GFX908-NEXT:   $vgpr0 = COPY [[COPY2]]
+  ; GFX908-NEXT:   $vgpr1 = COPY [[COPY3]]
+  ; GFX908-NEXT:   $vgpr2 = COPY [[COPY4]]
+  ; GFX908-NEXT:   $vgpr3 = COPY [[COPY5]]
+  ; GFX908-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  %val = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret <4 x float> %val
+}
+
+define void @llvm_amdgcn_raw_buffer_store_f32(float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_store_f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY2]], [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_tbuffer_store_f32(float %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_store_f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY2]], [[COPY1]], [[DEF]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.f32(float %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_buffer_store_v2f32(<2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_store_v2f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[COPY4]], [[COPY1]], [[DEF2]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_tbuffer_store_v2f32(<2 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_store_v2f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_XY_OFFEN_exact [[COPY4]], [[COPY1]], [[DEF2]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float> %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_buffer_store_v3f32(<3 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_store_v3f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE]]
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact [[COPY5]], [[COPY1]], [[DEF3]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float> %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_tbuffer_store_v3f32(<3 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_store_v3f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vreg_96 = COPY [[REG_SEQUENCE]]
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[COPY5]], [[COPY1]], [[DEF3]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float> %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_buffer_store_v4f32(<4 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_buffer_store_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact [[COPY6]], [[COPY1]], [[DEF4]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define void @llvm_amdgcn_raw_tbuffer_store_v4f32(<4 x float> %val, i32 %voffset, i32 %soffset) {
+  ; GFX908-LABEL: name: llvm_amdgcn_raw_tbuffer_store_v4f32
+  ; GFX908: bb.0 (%ir-block.0):
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX908-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX908-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX908-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX908-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX908-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+  ; GFX908-NEXT:   [[COPY6:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+  ; GFX908-NEXT:   [[DEF4:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF5:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY]], implicit $exec
+  ; GFX908-NEXT:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.2:
+  ; GFX908-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[COPY6]], [[COPY1]], [[DEF4]], killed [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+  ; GFX908-NEXT:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; GFX908-NEXT:   SI_WATERFALL_LOOP %bb.1, implicit $exec
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.3:
+  ; GFX908-NEXT:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX908-NEXT:   SI_RETURN
+  call void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float> %val, <4 x i32> poison, i32 %voffset, i32 %soffset, i32 0, i32 0)
+  ret void
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32 )
+declare float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32)
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32)
+declare <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32)
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32>, i32, i32, i32)
+declare <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32)
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32)
+declare <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32)
+declare void @llvm.amdgcn.raw.tbuffer.store.f32(float, <4 x i32>, i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32)
+declare void @llvm.amdgcn.raw.tbuffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32)
+declare void @llvm.amdgcn.raw.tbuffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32, i32)
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32)
+declare void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll b/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll
index d8358db7e96b9..5a49314691aaa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.dbg.value.ll
@@ -20,7 +20,7 @@ entry:
 ; SIOptimizeExecMaskingPreRA (somehow related to undef argument).
 
 ; GCN-LABEL: {{^}}only_undef_dbg_value:
-; NOOPT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] undef
+; NOOPT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- undef
 ; NOOPT-NEXT: s_endpgm
 
 ; OPT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/loop-idiom.ll b/llvm/test/CodeGen/AMDGPU/loop-idiom.ll
index c16b43555237b..0582e96d709e7 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-idiom.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-idiom.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -passes=loop-idiom -S < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 ; RUN: opt -passes=loop-idiom -S < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Make sure loop-idiom doesn't create memcpy or memset.  There are no library
 ; implementations of these for R600.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-constexpr-alias.ll b/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-constexpr-alias.ll
index bf3f98223be23..825ebf203a3ba 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-constexpr-alias.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-constexpr-alias.ll
@@ -18,8 +18,6 @@
 @foo.alias = hidden alias void (), ptr @foo
 
 ;.
-; CHECK-NOT: @llvm.global_ctors
-; CHECK-NOT: @llvm.global_dtors
 ; CHECK: @llvm.used = appending global [2 x ptr] [ptr @amdgcn.device.init, ptr @amdgcn.device.fini], section "llvm.metadata"
 ; CHECK: @foo.alias = hidden alias void (), ptr @foo
 ;.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-existing.ll b/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-existing.ll
index 5fa23e4ae8235..f0919b971b104 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-existing.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor-existing.ll
@@ -7,25 +7,17 @@
 @llvm.global_ctors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @foo, ptr null }]
 @llvm.global_dtors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @bar, ptr null }]
 
-; CHECK-NOT: @llvm.global_ctors
-; CHECK-NOT: @llvm.global_dtors
-
-; CHECK-LABEL: amdgpu_kernel void @amdgcn.device.init() #0
-; CHECK-NEXT: store
-; CHECK-NEXT: ret void
-
-; CHECK-LABEL: amdgpu_kernel void @amdgcn.device.fini() #1
-; CHECK-NEXT: store
-; CHECK-NEXT: ret void
-
-
-; CHECK-LABEL: amdgpu_kernel void @amdgcn.device.init.1() #0
-; CHECK-NEXT: call void @foo
-; CHECK-NEXT: ret void
-
-; CHECK-LABEL: amdgpu_kernel void @amdgcn.device.fini.2() #1
-; CHECK-NEXT: call void @bar
-; CHECK-NEXT: ret void
+; CHECK-LABEL: amdgpu_kernel void @amdgcn.device.init() #0 {
+; CHECK-NEXT:   store volatile i32 1, ptr addrspace(1) null
+; CHECK-NEXT:   call void @foo()
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+; CHECK-LABEL: define amdgpu_kernel void @amdgcn.device.fini() #1 {
+; CHECK-NEXT:    store volatile i32 0, ptr addrspace(1) null
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    ret void
+; CHECK-NEXT:  }
 
 ; CHECK-NOT: amdgcn.device.
 
@@ -34,12 +26,6 @@
 ; CHECK-VIS: FUNC   GLOBAL PROTECTED {{.*}} amdgcn.device.fini{{$}}
 ; CHECK-VIS: OBJECT   GLOBAL DEFAULT {{.*}} amdgcn.device.fini.kd{{$}}
 
-; CHECK-VIS: FUNC   GLOBAL PROTECTED {{.*}} amdgcn.device.init.1{{$}}
-; CHECK-VIS: OBJECT GLOBAL DEFAULT {{.*}} amdgcn.device.init.1.kd{{$}}
-; CHECK-VIS: FUNC   GLOBAL PROTECTED {{.*}} amdgcn.device.fini.2{{$}}
-; CHECK-VIS: OBJECT   GLOBAL DEFAULT {{.*}} amdgcn.device.fini.2.kd{{$}}
-
-
 define internal void @foo() {
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor.ll b/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor.ll
index f230d8d5e2dda..a197727eaec37 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-ctor-dtor.ll
@@ -4,12 +4,12 @@
 ; Make sure we get the same result if we run multiple times
 ; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-ctor-dtor,amdgpu-lower-ctor-dtor < %s | FileCheck %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf -s - 2>&1 | FileCheck %s -check-prefix=VISIBILITY
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readelf -S - 2>&1 | FileCheck %s -check-prefix=SECTION
 
 @llvm.global_ctors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @foo, ptr null }]
 @llvm.global_dtors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @bar, ptr null }]
 
-; CHECK-NOT: @llvm.global_ctors
-; CHECK-NOT: @llvm.global_dtors
+; CHECK: @llvm.used = appending global [2 x ptr] [ptr @amdgcn.device.init, ptr @amdgcn.device.fini]
 
 ; CHECK-LABEL: amdgpu_kernel void @amdgcn.device.init() #0
 ; CHECK-NEXT: call void @foo
@@ -25,6 +25,8 @@
 ; VISIBILITY: OBJECT GLOBAL DEFAULT {{.*}} amdgcn.device.init.kd
 ; VISIBILITY: FUNC   GLOBAL PROTECTED {{.*}} amdgcn.device.fini
 ; VISIBILITY: OBJECT   GLOBAL DEFAULT {{.*}} amdgcn.device.fini.kd
+; SECTION: .init_array.1     INIT_ARRAY      {{.*}} {{.*}} 000008 00  WA  0   0  8
+; SECTION: .fini_array.1     FINI_ARRAY      {{.*}} {{.*}} 000008 00  WA  0   0  8
 
 define internal void @foo() {
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
index fc80ff217bdf9..6ee6bbbff69f9 100644
--- a/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/mdt-preserving-crash.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 @_RSENC_gDcd_______________________________ = external protected addrspace(1) externally_initialized global [4096 x i8], align 16
diff --git a/llvm/test/CodeGen/AMDGPU/merge-sbuffer-load.mir b/llvm/test/CodeGen/AMDGPU/merge-sbuffer-load.mir
index 5fc8a0e9be39c..5ff0dbe65b5d1 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-sbuffer-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-sbuffer-load.mir
@@ -132,7 +132,7 @@ body:             |
 ---
 
 # CHECK-LABEL: name: merge_s_buffer_load_sgpr_imm
-# CHECK: S_BUFFER_LOAD_DWORDX4_SGPR %0, %1, 0 :: (dereferenceable invariant load (s128), align 4)
+# CHECK: S_BUFFER_LOAD_DWORDX4_SGPR_IMM %0, %1, 0, 0 :: (dereferenceable invariant load (s128), align 4)
 name: merge_s_buffer_load_sgpr_imm
 tracksRegLiveness: true
 body:             |
@@ -141,7 +141,7 @@ body:             |
 
     %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sreg_32 = COPY $sgpr4
-    %2:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR %0:sgpr_128, %1:sreg_32, 0 :: (dereferenceable invariant load (s32))
+    %2:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM %0:sgpr_128, %1:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
     %3:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM %0:sgpr_128, %1:sreg_32, 4, 0 :: (dereferenceable invariant load (s32))
     %4:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM %0:sgpr_128, %1:sreg_32, 8, 0 :: (dereferenceable invariant load (s32))
     %5:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM %0:sgpr_128, %1:sreg_32, 12, 0 :: (dereferenceable invariant load (s32))
diff --git a/llvm/test/CodeGen/AMDGPU/noop-shader-O0.ll b/llvm/test/CodeGen/AMDGPU/noop-shader-O0.ll
index af47170a4a9c9..dfcfe38855207 100644
--- a/llvm/test/CodeGen/AMDGPU/noop-shader-O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/noop-shader-O0.ll
@@ -6,7 +6,7 @@
 ; Confirm registers reserved in SIMachineFunctionInfo are those expected during
 ; lowering, even when e.g. spilling is required due to being at OptNone.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 target triple = "amdgcn-amd-amdpal"
 
 define amdgpu_vs void @noop_vs() {
diff --git a/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll b/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll
new file mode 100644
index 0000000000000..f8958cfa5f48e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/nullptr-long-address-spaces.ll
@@ -0,0 +1,18 @@
+; XFAIL: *
+; RUN: llc < %s -march=amdgcn -mtriple=amdgcn-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,GCN %s
+; RUN: llc < %s -march=r600 -mtriple=r600-- -verify-machineinstrs | FileCheck -check-prefixes=CHECK,R600 %s
+
+; This is a temporary xfail, as the assembly printer is broken when dealing with
+; lowerConstant() trying to return a value of size greater than 8 bytes.
+
+; CHECK-LABEL: nullptr7:
+; The exact form of the GCN output depends on how the printer gets fixed.
+; GCN-NEXT: .zeroes 5
+; R600-NEXT: .long 0
+@nullptr7 = global ptr addrspace(7) addrspacecast (ptr null to ptr addrspace(7))
+
+; CHECK-LABEL: nullptr8:
+; The exact form of the GCN output depends on how the printer gets fixed.
+; GCN-NEXT: .zeroes 4
+; R600-NEXT: .long 0
+@nullptr8 = global ptr addrspace(8) addrspacecast (ptr null to ptr addrspace(7))
diff --git a/llvm/test/CodeGen/AMDGPU/nullptr.ll b/llvm/test/CodeGen/AMDGPU/nullptr.ll
index ed3d98dff154a..6bbfdc6363be9 100644
--- a/llvm/test/CodeGen/AMDGPU/nullptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/nullptr.ll
@@ -29,13 +29,14 @@
 ; R600-NEXT: .long 0
 @nullptr6 = global ptr addrspace(6) addrspacecast (ptr null to ptr addrspace(6))
 
-; CHECK-LABEL: nullptr7:
-; R600-NEXT: .long 0
-@nullptr7 = global ptr addrspace(7) addrspacecast (ptr null to ptr addrspace(7))
-
-; CHECK-LABEL: nullptr8:
-; R600-NEXT: .long 0
-@nullptr8 = global ptr addrspace(8) addrspacecast (ptr null to ptr addrspace(8))
+; FIXME: AsmPrinter can't handle 128-bit constants
+; FIXME-LABEL: nullptr7:
+; FIXME-R600-NEXT: .long 0
+; FIXME @nullptr7 = global ptr addrspace(7) addrspacecast (ptr null to ptr addrspace(7))
+
+; FIXME-LABEL: nullptr8:
+; FIXME-R600-NEXT: .long 0
+; FIXME @nullptr8 = global ptr addrspace(8) addrspacecast (ptr null to ptr addrspace(8))
 
 ; CHECK-LABEL: nullptr9:
 ; R600-NEXT: .long 0
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 2450a989ecb8f..8ae07a74503a2 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX908-NEXT:   liveins: $sgpr4_sgpr5
   ; REGALLOC-GFX908-NEXT: {{  $}}
   ; REGALLOC-GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 1703945 /* reguse:AGPR_32 */, undef %5:agpr_32
-  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 5898250 /* regdef:VReg_128 */, def %26
+  ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def %26
   ; REGALLOC-GFX908-NEXT:   [[COPY:%[0-9]+]]:av_128 = COPY %26
   ; REGALLOC-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3080202 /* regdef:VReg_64 */, def %23
   ; REGALLOC-GFX908-NEXT:   SI_SPILL_V64_SAVE %23, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
@@ -35,7 +35,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX908-NEXT:   $sgpr8 = S_ADD_U32 $sgpr8, $sgpr7, implicit-def $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
   ; PEI-GFX908-NEXT:   $sgpr9 = S_ADDC_U32 $sgpr9, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
   ; PEI-GFX908-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 1703945 /* reguse:AGPR_32 */, undef renamable $agpr0
-  ; PEI-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 5898250 /* regdef:VReg_128 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
+  ; PEI-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6094858 /* regdef:VReg_128 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
   ; PEI-GFX908-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
   ; PEI-GFX908-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3080202 /* regdef:VReg_64 */, def renamable $vgpr0_vgpr1
   ; PEI-GFX908-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, 0, 4, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5)
@@ -58,7 +58,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; REGALLOC-GFX90A-NEXT:   liveins: $sgpr4_sgpr5
   ; REGALLOC-GFX90A-NEXT: {{  $}}
   ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 1703945 /* reguse:AGPR_32 */, undef %5:agpr_32
-  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6225930 /* regdef:VReg_128_Align2 */, def %25
+  ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def %25
   ; REGALLOC-GFX90A-NEXT:   [[COPY:%[0-9]+]]:av_128_align2 = COPY %25
   ; REGALLOC-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3407882 /* regdef:VReg_64_Align2 */, def %23
   ; REGALLOC-GFX90A-NEXT:   SI_SPILL_V64_SAVE %23, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
@@ -80,7 +80,7 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
   ; PEI-GFX90A-NEXT:   $sgpr8 = S_ADD_U32 $sgpr8, $sgpr7, implicit-def $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
   ; PEI-GFX90A-NEXT:   $sgpr9 = S_ADDC_U32 $sgpr9, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
   ; PEI-GFX90A-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 1703945 /* reguse:AGPR_32 */, undef renamable $agpr0
-  ; PEI-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6225930 /* regdef:VReg_128_Align2 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
+  ; PEI-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 6422538 /* regdef:VReg_128_Align2 */, def renamable $vgpr0_vgpr1_vgpr2_vgpr3
   ; PEI-GFX90A-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, implicit $exec
   ; PEI-GFX90A-NEXT:   INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 3407882 /* regdef:VReg_64_Align2 */, def renamable $vgpr0_vgpr1
   ; PEI-GFX90A-NEXT:   BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, 0, 4, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store (s32) into %stack.0, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-lifetime.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-lifetime.ll
index 359d456203721..2230e12a30f59 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-lifetime.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-lifetime.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -passes=amdgpu-promote-alloca %s | FileCheck -check-prefix=OPT %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 declare void @llvm.lifetime.start.p5(i64, ptr addrspace(5) nocapture) #0
 declare void @llvm.lifetime.end.p5(i64, ptr addrspace(5) nocapture) #0
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
index 8fcbccc3a5573..4a15dad069c63 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -passes=amdgpu-promote-alloca < %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; CHECK-LABEL: @lds_promoted_alloca_select_invalid_pointer_operand(
 ; CHECK: %alloca = alloca i32
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
index 9d028602ad6f3..72d0053693b78 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
@@ -977,7 +977,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_v3f32.body
 ; CHECK-SAME: (ptr [[OUT:%.*]], <3 x float> [[VALUE:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[BITCAST_STRUCT_V3F32_V3F32:%.*]] poison, <4 x float> [[EXTRACTVEC]], 0
 ; CHECK-NEXT:    ret [[BITCAST_STRUCT_V3F32_V3F32]] [[TMP1]]
 ;
@@ -992,7 +992,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_v3i32.body
 ; CHECK-SAME: (ptr [[OUT:%.*]], <3 x i32> [[VALUE:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x i32> [[VALUE]], <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x i32> [[VALUE]], <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[BITCAST_STRUCT_V3F32_V3I32:%.*]] poison, <4 x i32> [[EXTRACTVEC]], 0
 ; CHECK-NEXT:    ret [[BITCAST_STRUCT_V3F32_V3I32]] [[TMP1]]
 ;
@@ -1035,7 +1035,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v4f32_v3f32.body
 ; CHECK-SAME: (ptr [[OUT:%.*]], <3 x float> [[VALUE:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[BITCAST_STRUCT_V4F32_V3F32:%.*]] poison, <4 x float> [[EXTRACTVEC]], 0
 ; CHECK-NEXT:    ret [[BITCAST_STRUCT_V4F32_V3F32]] [[TMP1]]
 ;
@@ -1064,7 +1064,7 @@ attributes #2 = { alwaysinline nounwind }
 ;
 ; CHECK-LABEL: define {{[^@]+}}@bitcast_struct_v3f32_f32_v3f32.body
 ; CHECK-SAME: (ptr [[OUT:%.*]], <3 x float> [[VALUE:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[BITCAST_STRUCT_V3F32_F32_V3F32:%.*]] poison, <4 x float> [[EXTRACTVEC]], 0
 ; CHECK-NEXT:    ret [[BITCAST_STRUCT_V3F32_F32_V3F32]] [[TMP1]]
 ;
@@ -1124,7 +1124,7 @@ attributes #2 = { alwaysinline nounwind }
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 [[COND]], label [[RET0:%.*]], label [[RET1:%.*]]
 ; CHECK:       ret0:
-; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[VALUE]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue [[MULTI_RETURN_BITCAST_STRUCT_V3F32_V3F32:%.*]] poison, <4 x float> [[EXTRACTVEC]], 0
 ; CHECK-NEXT:    ret [[MULTI_RETURN_BITCAST_STRUCT_V3F32_V3F32]] [[TMP0]]
 ; CHECK:       ret1:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll
index 0f65920946ab1..baee88b69d060 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.v8i16.ll
@@ -10,13 +10,12 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX900-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
 ; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX900-NEXT:    s_pack_lh_b32_b16 s4, s0, s0
 ; GFX900-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX900-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
+; GFX900-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX900-NEXT:    v_mov_b32_e32 v1, s1
+; GFX900-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX900-NEXT:    v_mov_b32_e32 v3, s0
-; GFX900-NEXT:    v_mov_b32_e32 v0, s4
-; GFX900-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX900-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX900-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX900-NEXT:    s_endpgm
@@ -26,13 +25,12 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
 ; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    s_pack_lh_b32_b16 s4, s0, s0
 ; GFX906-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX906-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
+; GFX906-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX906-NEXT:    v_mov_b32_e32 v1, s1
+; GFX906-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX906-NEXT:    v_mov_b32_e32 v3, s0
-; GFX906-NEXT:    v_mov_b32_e32 v0, s4
-; GFX906-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX906-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX906-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX906-NEXT:    s_endpgm
@@ -42,13 +40,12 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX908-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
 ; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX908-NEXT:    s_pack_lh_b32_b16 s4, s0, s0
 ; GFX908-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX908-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
+; GFX908-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX908-NEXT:    v_mov_b32_e32 v1, s1
+; GFX908-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX908-NEXT:    v_mov_b32_e32 v3, s0
-; GFX908-NEXT:    v_mov_b32_e32 v0, s4
-; GFX908-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX908-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX908-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX908-NEXT:    s_endpgm
@@ -58,13 +55,12 @@ define amdgpu_kernel void @scalar_to_vector_v8i16(<2 x i32> %in, ptr %out) #0 {
 ; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT:    s_pack_lh_b32_b16 s4, s0, s0
 ; GFX90A-NEXT:    v_mov_b32_e32 v5, s3
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, s2, v4
+; GFX90A-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX90A-NEXT:    v_mov_b32_e32 v1, s1
+; GFX90A-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX90A-NEXT:    v_mov_b32_e32 v3, s0
-; GFX90A-NEXT:    v_mov_b32_e32 v0, s4
-; GFX90A-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX90A-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX90A-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir b/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir
new file mode 100644
index 0000000000000..4b6e204ecf957
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/schedule-ilp-liveness-tracking.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -amdgpu-enable-max-ilp-scheduling-strategy -verify-machineinstrs -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
+
+---
+name:            max-ilp-liveness-tracking
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: max-ilp-liveness-tracking
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %src0:vgpr_32 = V_MOV_B32_e64 0, implicit $exec
+  ; CHECK-NEXT:   %src1:vgpr_32 = V_MOV_B32_e64 1, implicit $exec
+  ; CHECK-NEXT:   %live0:vgpr_32 = V_ADD_U32_e32 %src0, %src1, implicit $exec
+  ; CHECK-NEXT:   %live1:vgpr_32 = V_ADD_U32_e32 %live0, %src1, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   %out0:vgpr_32 = V_ADD_U32_e32 %live0, %live1, implicit $exec
+  ; CHECK-NEXT:   dead %out1:vgpr_32 = V_ADD_U32_e32 %out0, %live1, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1
+    %src0:vgpr_32 = V_MOV_B32_e64 0, implicit $exec
+    %src1:vgpr_32 = V_MOV_B32_e64 1, implicit $exec
+    %live0:vgpr_32 = V_ADD_U32_e32 %src0:vgpr_32, %src1:vgpr_32, implicit $exec
+    %live1:vgpr_32 = V_ADD_U32_e32 %live0:vgpr_32, %src1:vgpr_32, implicit $exec
+
+  bb.1:
+    %out0:vgpr_32 = V_ADD_U32_e32 %live0:vgpr_32, %live1:vgpr_32, implicit $exec
+    %out1:vgpr_32 = V_ADD_U32_e32 %out0:vgpr_32, %live1:vgpr_32, implicit $exec
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll b/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll
index 6635d5f4d6bde..8abd4b4302f54 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-machineinstrs -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 ; CHECK-LABEL: {{^}}t0:
diff --git a/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir b/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir
new file mode 100644
index 0000000000000..71c408ced015b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-opt-vgpr-liverange-bug-deadlanes.mir
@@ -0,0 +1,97 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass=si-opt-vgpr-liverange -o - %s | FileCheck -check-prefixes=CHECK %s
+
+# Tests a case that used to assert in SIOptimizeVGPRLiveRange when trying to optimize %3 which still appears
+# (though in an undef operand) in the REG_SEQUENCE of the "endif block". This undef pattern was caused by
+# DetectDeadLanes.
+
+--- |
+  define dllexport amdgpu_ps void @_amdgpu_ps_main() #0 {
+    unreachable
+  }
+
+  attributes #0 = { "target-cpu"="gfx1100" "target-features"=",+wavefrontsize64,+cumode" "uniform-work-group-size"="false" }
+...
+---
+name:            _amdgpu_ps_main
+tracksRegLiveness: true
+registers:
+  - { id: 1, class: vgpr_32, preferred-register: '' }
+  - { id: 2, class: vgpr_32, preferred-register: '' }
+  - { id: 3, class: vgpr_32, preferred-register: '' }
+  - { id: 4, class: sreg_64, preferred-register: '$vcc' }
+  - { id: 5, class: vreg_128, preferred-register: '' }
+  - { id: 6, class: sgpr_128, preferred-register: '' }
+  - { id: 7, class: vgpr_32, preferred-register: '' }
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%1' }
+body:             |
+  ; CHECK-LABEL: name: _amdgpu_ps_main
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN killed [[COPY]], undef %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32))
+  ; CHECK-NEXT:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 0, [[BUFFER_LOAD_DWORD_OFFEN]], implicit $exec
+  ; CHECK-NEXT:   [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_NE_U32_e64_]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1, [[V_MOV_B32_e32_]], %subreg.sub2, undef %4.sub3, %subreg.sub3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:vreg_128 = PHI undef %10:vreg_128, %bb.0, [[REG_SEQUENCE]], %bb.1
+  ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[BUFFER_LOAD_DWORD_OFFEN]], %bb.0, undef %15:vgpr_32, %bb.1
+  ; CHECK-NEXT:   [[SI_ELSE:%[0-9]+]]:sreg_64 = SI_ELSE killed [[SI_IF]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[PHI1]], %subreg.sub0, [[PHI1]], %subreg.sub1, [[PHI1]], %subreg.sub2, undef %6:vgpr_32, %subreg.sub3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   [[PHI2:%[0-9]+]]:vreg_128 = PHI [[PHI]], %bb.2, [[REG_SEQUENCE1]], %bb.3
+  ; CHECK-NEXT:   SI_END_CF killed [[SI_ELSE]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; CHECK-NEXT:   dead [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[PHI2]].sub2, %subreg.sub0, [[PHI2]].sub2, %subreg.sub1, [[PHI2]].sub2, %subreg.sub2, undef [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub3
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.5(0x40000000), %bb.6(0x40000000)
+    liveins: $vgpr0
+
+    %1:vgpr_32 = COPY killed $vgpr0
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %1, undef %6:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load (s32))
+    %4:sreg_64 = V_CMP_NE_U32_e64 0, %3, implicit $exec
+    %8:sreg_64 = SI_IF killed %4, %bb.6, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.5
+
+  bb.5:
+    successors: %bb.6(0x80000000)
+
+    %9:vreg_128 = REG_SEQUENCE %2, %subreg.sub0, %2, %subreg.sub1, %2, %subreg.sub2, undef %5.sub3:vreg_128, %subreg.sub3
+
+  bb.6:
+    successors: %bb.7(0x40000000), %bb.8(0x40000000)
+
+    %10:vreg_128 = PHI undef %156:vreg_128, %bb.0, %9, %bb.5
+    %11:sreg_64 = SI_ELSE killed %8, %bb.8, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.7
+
+  bb.7:
+    successors: %bb.8(0x80000000)
+
+    %12:vreg_128 = REG_SEQUENCE %3, %subreg.sub0, %3, %subreg.sub1, killed %3, %subreg.sub2, undef %7, %subreg.sub3
+
+  bb.8:
+    %13:vreg_128 = PHI %10, %bb.6, %12, %bb.7
+    SI_END_CF killed %11, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    %5:vreg_128 = REG_SEQUENCE %13.sub2, %subreg.sub0, %13.sub2, %subreg.sub1, killed %13.sub2, %subreg.sub2, undef %3, %subreg.sub3
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
index e0ccbcda7fa02..4d73e438f617a 100644
--- a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
@@ -1,10 +1,25 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
+%struct.lds = type { [64 x ptr], [16 x i8] }
+@stored_lds_struct = addrspace(3) global %struct.lds addrspace(3) *undef, align 16
 @stored_lds_ptr = addrspace(3) global ptr addrspace(3) undef, align 4
 @stored_constant_ptr = addrspace(3) global ptr addrspace(4) undef, align 8
 @stored_global_ptr = addrspace(3) global ptr addrspace(1) undef, align 8
 
+; GCN-LABEL: {{^}}no_reorder_flat_load_local_store_local_load:
+; GCN: flat_load_dwordx4
+; GCN: ds_write_b128 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:512
+; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:129 offset1:130
+define amdgpu_kernel void @no_reorder_flat_load_local_store_local_load(ptr addrspace(3) %out, ptr %fptr) #0 {
+  %ptr1 = getelementptr %struct.lds, ptr addrspace(3) @stored_lds_struct, i32 0, i32 1
+  %ptr2 = getelementptr %struct.lds, ptr addrspace(3) @stored_lds_struct, i32 0, i32 1, i32 4
+  call void @llvm.memcpy.p3.p0(ptr addrspace(3) align 16 %ptr1, ptr align 8 %fptr, i64 16, i1 false)
+  %vector_load = load <2 x i32>, ptr addrspace(3) %ptr2, align 4
+  store <2 x i32> %vector_load, ptr addrspace(3) %out, align 4
+  ret void
+}
+
 ; GCN-LABEL: {{^}}reorder_local_load_global_store_local_load:
 ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3
 ; CI: buffer_store_dword
@@ -312,6 +327,7 @@ define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(ptr addrspace
   ret void
 }
 
+declare void @llvm.memcpy.p3.p0(ptr addrspace(3), ptr, i64, i1)
 declare void @llvm.amdgcn.s.barrier() #1
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 declare void @llvm.amdgcn.struct.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32 immarg, i32 immarg) #3
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index a91aa9def9895..ac485d388a5e7 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -49,8 +49,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_LOAD_DWORDX4_IMM]], 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   undef %302.sub1:sgpr_128 = S_MOV_B32 0
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %89:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], undef %89:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   KILL undef %89:sgpr_128
   ; CHECK-NEXT:   [[S_SUB_I32_:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM]], 29, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_SUB_I32_1:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM]], 30, implicit-def dead $scc
@@ -101,7 +101,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM %302, undef %314:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM %302, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %302, 16, 0 :: (dereferenceable invariant load (s32))
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %118:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET undef %118:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %369:sgpr_128, undef %370:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %380:sgpr_128, 16, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM3:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %156, 0, 0 :: (invariant load (s128) from %ir.92, addrspace 4)
@@ -109,7 +109,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %176, 0, 0 :: (invariant load (s128) from %ir.104, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM6:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %183, 0, 0 :: (invariant load (s128) from %ir.109, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM7:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %190, 0, 0 :: (invariant load (s128) from %ir.114, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM2]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM2]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %364:sgpr_128, [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %375:sgpr_128, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_ADD_I32_2:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM]], -98, implicit-def dead $scc
@@ -126,18 +126,18 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   undef %351.sub0:sreg_64 = S_ADD_U32 [[COPY9]], [[S_LSHL_B32_2]], implicit-def $scc
   ; CHECK-NEXT:   %351.sub1:sreg_64 = S_ADDC_U32 undef %39:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LSHL_B32_3:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY10]], 4, implicit-def dead $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM3]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM3]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_ADD_I32_6:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LSHL_B32_3]], 16, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_SGPR_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %396:sgpr_128, [[S_ADD_I32_6]], 0, 0 :: (dereferenceable invariant load (s32))
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN4:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM4]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN4:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM4]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM9:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %50, 224, 0 :: (invariant load (s128) from %ir.126, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM10:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %210, 0, 0 :: (invariant load (s128) from %ir.127, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN5:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM5]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN5:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM5]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM11:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %217, 0, 0 :: (invariant load (s128) from %ir.132, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM12:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %224, 0, 0 :: (invariant load (s128) from %ir.137, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN6:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM6]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN7:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM7]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN8:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM8]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN6:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM6]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN7:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM7]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN8:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM8]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_ADD_I32_7:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM4]], -217, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_ADD_I32_8:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM3]], -233, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_ADD_I32_9:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM5]], -249, implicit-def dead $scc
@@ -151,7 +151,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   undef %411.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_3]], [[S_LSHL_B32_2]], implicit-def $scc
   ; CHECK-NEXT:   %411.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_3]], [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LSHL_B32_4:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY11]], 4, implicit-def dead $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN9:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM10]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN9:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM10]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_ASHR_I32_4:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_LSHL_B32_4]], 31, implicit-def dead $scc
   ; CHECK-NEXT:   undef %425.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_3]], [[S_LSHL_B32_4]], implicit-def $scc
   ; CHECK-NEXT:   %425.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_3]], [[S_ASHR_I32_4]], implicit-def dead $scc, implicit $scc
@@ -159,16 +159,16 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[S_ADDC_U32_4:%[0-9]+]]:sreg_32 = S_ADDC_U32 undef %57:sreg_32, 0, implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM13:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %241, 0, 0 :: (invariant load (s128) from %ir.147, addrspace 4)
   ; CHECK-NEXT:   [[S_LSHL_B32_5:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], 3, implicit-def dead $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN10:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM11]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN10:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM11]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_ASHR_I32_5:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_LSHL_B32_5]], 31, implicit-def dead $scc
   ; CHECK-NEXT:   undef %441.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_4]], [[S_LSHL_B32_5]], implicit-def $scc
   ; CHECK-NEXT:   %441.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_4]], [[S_ASHR_I32_5]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %441, 0, 0 :: (invariant load (s32) from %ir.269, align 8, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM14:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %253, 0, 0 :: (invariant load (s128) from %ir.154, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN11:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM12]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN11:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM12]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM15:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %261, 0, 0 :: (invariant load (s128) from %ir.159, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN12:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM9]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN13:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM13]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN12:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM9]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN13:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM13]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   %71.sub3:sgpr_128 = S_MOV_B32 553734060
   ; CHECK-NEXT:   %71.sub2:sgpr_128 = S_MOV_B32 -1
   ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:sgpr_128 = COPY %71
@@ -176,20 +176,20 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[COPY13]].sub1:sgpr_128 = COPY %302.sub1
   ; CHECK-NEXT:   [[COPY13]].sub0:sgpr_128 = COPY [[S_LOAD_DWORD_IMM]]
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[COPY13]], 0, 0 :: (dereferenceable invariant load (s32))
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN14:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM14]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN15:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM15]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN14:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM14]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN15:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM15]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM17:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %286, 0, 0 :: (invariant load (s128) from %ir.175, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM18:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %293, 0, 0 :: (invariant load (s128) from %ir.180, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN16:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM16]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN16:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM16]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LSHL_B32_6:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY3]], 3, implicit-def dead $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_ASHR_I32_6:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_LSHL_B32_6]], 31, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_ADD_I32_15:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM4]], -467, implicit-def dead $scc
   ; CHECK-NEXT:   undef %453.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_4]], [[S_LSHL_B32_6]], implicit-def $scc
   ; CHECK-NEXT:   %453.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_4]], [[S_ASHR_I32_6]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM %453, 0, 0 :: (invariant load (s64) from %ir.277, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM17]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM18]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM17]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFSET3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM18]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM19:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %327, 0, 0 :: (invariant load (s128) from %ir.202, addrspace 4)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM20:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %335, 0, 0 :: (invariant load (s128) from %ir.208, addrspace 4)
   ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:sgpr_128 = COPY %71
@@ -199,15 +199,15 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[COPY14]].sub1:sgpr_128 = COPY [[S_AND_B32_]]
   ; CHECK-NEXT:   [[S_BUFFER_LOAD_DWORD_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[COPY14]], 0, 0 :: (dereferenceable invariant load (s32))
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM22:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %351, 0, 0 :: (invariant load (s128) from %ir.218, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN17:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM19]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN18:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM20]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN17:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM19]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN18:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM20]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LSHL_B32_7:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY2]], 3, implicit-def dead $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN19:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM21]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN19:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM21]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_ASHR_I32_7:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_LSHL_B32_7]], 31, implicit-def dead $scc
   ; CHECK-NEXT:   [[S_ADD_I32_16:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM5]], -468, implicit-def dead $scc
   ; CHECK-NEXT:   undef %468.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_4]], [[S_LSHL_B32_7]], implicit-def $scc
   ; CHECK-NEXT:   %468.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_4]], [[S_ASHR_I32_7]], implicit-def dead $scc, implicit $scc
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN20:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM22]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN20:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM22]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM %468, 0, 0 :: (invariant load (s64) from %ir.287, addrspace 4)
   ; CHECK-NEXT:   [[COPY15:%[0-9]+]]:sgpr_128 = COPY %71
   ; CHECK-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_LOAD_DWORDX2_IMM1]].sub1, 65535, implicit-def dead $scc
@@ -226,8 +226,8 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   undef %485.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_4]], [[S_LSHL_B32_8]], implicit-def $scc
   ; CHECK-NEXT:   %485.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_4]], [[S_ASHR_I32_8]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %485, 0, 0 :: (invariant load (s32) from %ir.298, align 8, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN21:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM23]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN22:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN21:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM23]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN22:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   KILL [[S_LOAD_DWORDX4_IMM24]]
   ; CHECK-NEXT:   KILL [[S_LOAD_DWORDX4_IMM23]]
   ; CHECK-NEXT:   [[S_AND_B32_2:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_LOAD_DWORD_IMM1]], 65535, implicit-def dead $scc
@@ -252,9 +252,9 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   undef %530.sub0:sreg_64 = S_ADD_U32 [[S_ADD_U32_5]], [[S_LSHL_B32_2]], implicit-def $scc
   ; CHECK-NEXT:   %530.sub1:sreg_64 = S_ADDC_U32 [[S_ADDC_U32_5]], [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
   ; CHECK-NEXT:   [[S_LOAD_DWORDX4_IMM27:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM %530, 0, 0 :: (invariant load (s128) from %ir.326, addrspace 4)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN23:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM25]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM26]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
-  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM27]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 7)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN23:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM25]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM26]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; CHECK-NEXT:   [[BUFFER_LOAD_FORMAT_X_IDXEN25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM27]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
   ; CHECK-NEXT:   KILL [[S_LOAD_DWORDX4_IMM27]]
   ; CHECK-NEXT:   KILL [[S_LOAD_DWORDX4_IMM25]]
   ; CHECK-NEXT:   KILL [[V_MOV_B32_e32_]]
@@ -376,7 +376,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK-NEXT:   [[V_OR_B32_e64_67:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[S_ADD_I32_24]], [[V_OR_B32_e64_66]], implicit $exec
   ; CHECK-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 0, [[V_OR_B32_e64_67]], implicit $exec
   ; CHECK-NEXT:   undef %624.sub3:vreg_128 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U32_e64_]], implicit $exec
-  ; CHECK-NEXT:   IMAGE_STORE_V4_V2_gfx10 %624, undef %578:vreg_64, [[S_LOAD_DWORDX8_IMM]], 15, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 7)
+  ; CHECK-NEXT:   IMAGE_STORE_V4_V2_gfx10 %624, undef %578:vreg_64, [[S_LOAD_DWORDX8_IMM]], 15, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
   ; CHECK-NEXT:   S_ENDPGM 0
 .expVert:
   %0 = extractelement <31 x i32> %userData, i64 2
diff --git a/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll b/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll
index 4059d2d80f894..756182dc243bb 100644
--- a/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll
+++ b/llvm/test/CodeGen/AMDGPU/tail-call-amdgpu-gfx.ll
@@ -2,8 +2,6 @@
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN -enable-var-scope %s
 ; RUN: llc -global-isel -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN -enable-var-scope %s
 
-; FIXME: @caller uses s[4:5] to store the address of @callee.
-;        These registers are callee-save in the amdgpu_gfx calling convention, so they must not be clobbered, but they are clobbered here.
 
 ; Callee with VGPR arguments
 define hidden amdgpu_gfx float @callee(float %v.arg0, float %v.arg1) {
@@ -20,22 +18,12 @@ define amdgpu_gfx float @caller(float %arg0) {
 ; GCN-LABEL: caller:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; GCN-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[34:35]
-; GCN-NEXT:    v_writelane_b32 v2, s4, 0
-; GCN-NEXT:    v_writelane_b32 v2, s5, 1
-; GCN-NEXT:    s_getpc_b64 s[4:5]
-; GCN-NEXT:    s_add_u32 s4, s4, callee@rel32@lo+4
-; GCN-NEXT:    s_addc_u32 s5, s5, callee@rel32@hi+12
 ; GCN-NEXT:    v_add_f32_e32 v0, 1.0, v0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 2.0
-; GCN-NEXT:    v_readlane_b32 s5, v2, 1
-; GCN-NEXT:    v_readlane_b32 s4, v2, 0
-; GCN-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[34:35]
-; GCN-NEXT:    s_setpc_b64 s[4:5]
+; GCN-NEXT:    s_getpc_b64 s[36:37]
+; GCN-NEXT:    s_add_u32 s36, s36, callee@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s37, s37, callee@rel32@hi+12
+; GCN-NEXT:    s_setpc_b64 s[36:37]
   %add = fadd float %arg0, 1.0
   %call = tail call amdgpu_gfx float @callee(float %add, float 2.0)
   ret float %call
diff --git a/llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-neg-offset.mir b/llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-neg-offset.mir
new file mode 100644
index 0000000000000..80337069a4622
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/triv-disjoint-mem-access-neg-offset.mir
@@ -0,0 +1,37 @@
+# RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-misched --debug-only=machine-scheduler -run-pass=machine-scheduler -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Make sure handling of unsigned immediate values interpreted as negative values
+# still works for SIInstrInfo::areMemAccessesTriviallyDisjoint.
+
+# LABEL: {{^}}no_reorder_flat_load_local_store_local_load:
+# CHECK:      SU(5):   %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+# CHECK:      SU(6):   DS_WRITE_B128_gfx9 %5:vgpr_32, %4:vreg_128, 512, 0, implicit $exec
+# CHECK:      SU(7):   %6:vreg_64 = DS_READ2_B32_gfx9 %5:vgpr_32, -127, -126, 0, implicit $exec
+# CHECK:      Predecessors:
+# CHECK-NEXT: SU(6): Ord
+# CHECK-NEXT: SU(5): Data
+---
+name:            no_reorder_flat_load_local_store_local_load
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    %1:sgpr_64(p4) = COPY $sgpr0_sgpr1
+    %4:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %1(p4), 36, 0
+    %5:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1(p4), 44, 0
+    %7:vreg_64 = COPY %5
+    %6:vreg_128 = FLAT_LOAD_DWORDX4 killed %7, 0, 0, implicit $exec, implicit $flat_scr
+    %8:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    DS_WRITE_B128_gfx9 %8, killed %6, 512, 0, implicit $exec
+    %9:vreg_64 = DS_READ2_B32_gfx9 %8, -127, -126, 0, implicit $exec
+    %10:sreg_32 = COPY %9.sub1
+    %11:sreg_32 = COPY %9.sub0
+    %12:vgpr_32 = COPY %4
+    %13:vgpr_32 = COPY %11
+    %14:vgpr_32 = COPY %10
+    DS_WRITE2_B32_gfx9 killed %12, killed %13, killed %14, 0, 1, 0, implicit $exec
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/unroll.ll b/llvm/test/CodeGen/AMDGPU/unroll.ll
index 41bb4bb095c2a..119f936fcd020 100644
--- a/llvm/test/CodeGen/AMDGPU/unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/unroll.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-- -passes='loop-unroll,simplifycfg,sroa' %s -S -o - | FileCheck %s
 ; RUN: opt -mtriple=r600-- -passes='loop-unroll,simplifycfg,sroa' %s -S -o - | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; This test contains a simple loop that initializes an array declared in
 ; private memory.  We want to make sure these kinds of loops are always
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll b/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll
index f27d8fd88b8bb..6dc7e49051028 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll
@@ -5,7 +5,7 @@
 ; feature, and instead generates a selection error.
 
 ; SDAG-ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.image.load.1d
-; GISEL-ERR: LLVM ERROR: unable to legalize instruction: %{{[0-9]+}}:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(<8 x s32>), 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7) (in function: load_1d)
+; GISEL-ERR: LLVM ERROR: unable to legalize instruction: %{{[0-9]+}}:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(<8 x s32>), 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8) (in function: load_1d)
 
 define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll b/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll
index 058b339989761..917dd781bf4e4 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll
@@ -8,7 +8,7 @@
 ; generates a selection error.
 
 ; SDAG-ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.image.sample.d.1d
-; GISEL-ERR: LLVM ERROR: unable to legalize instruction: %{{[0-9]+}}:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s32), %{{[0-9]+}}:_(<8 x s32>), %{{[0-9]+}}:_(<4 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 7) (in function: sample_d_1d)
+; GISEL-ERR: LLVM ERROR: unable to legalize instruction: %{{[0-9]+}}:_(<4 x s32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s32), %{{[0-9]+}}:_(<8 x s32>), %{{[0-9]+}}:_(<4 x s32>), 0, 0, 0 :: (dereferenceable load (<4 x s32>), addrspace 8) (in function: sample_d_1d)
 
 define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
index f6203859a91be..350d0985fbfed 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/valu-mask-write-hazard.mir
@@ -134,20 +134,6 @@ body:            |
     S_ENDPGM 0
 ...
 
----
-name:            mask_hazard_cndmask_dpp3
-body:            |
-  bb.0:
-    ; GCN-LABEL: name: mask_hazard_cndmask_dpp3
-    ; GCN: $vgpr0 = V_CNDMASK_B16_dpp $vgpr0, 0, $vgpr1, 0, $vgpr2, 1, 15, 15, 1, implicit $vcc, implicit $exec
-    ; GCN-NEXT: $vcc = S_CSELECT_B64 -1, 0, implicit $scc
-    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
-    ; GCN-NEXT: S_ENDPGM 0
-    $vgpr0 = V_CNDMASK_B16_dpp $vgpr0, 0, $vgpr1, 0, $vgpr2, 1, 15, 15, 1, implicit $vcc, implicit $exec
-    $vcc = S_CSELECT_B64 -1, 0, implicit $scc
-    S_ENDPGM 0
-...
-
 ---
 name:            mask_hazard_cndmask_dpp4
 body:            |
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 5ba0c3f371ddc..366c44f5ea3f4 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -635,7 +635,7 @@ define protected amdgpu_kernel void @nested_waterfalls(ptr addrspace(1) %tex.coe
   ; SI-NEXT: bb.5:
   ; SI-NEXT:   successors: %bb.4(0x40000000), %bb.6(0x40000000)
   ; SI-NEXT: {{  $}}
-  ; SI-NEXT:   [[IMAGE_SAMPLE_V1_V2_gfx10_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_gfx10 undef %22:vreg_64, [[REG_SEQUENCE5]], killed [[REG_SEQUENCE8]], 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 7)
+  ; SI-NEXT:   [[IMAGE_SAMPLE_V1_V2_gfx10_:%[0-9]+]]:vgpr_32 = IMAGE_SAMPLE_V1_V2_gfx10 undef %22:vreg_64, [[REG_SEQUENCE5]], killed [[REG_SEQUENCE8]], 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 8)
   ; SI-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, killed [[S_AND_SAVEEXEC_B32_1]], implicit-def dead $scc
   ; SI-NEXT:   SI_WATERFALL_LOOP %bb.4, implicit $exec
   ; SI-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/ARM/ctors_dtors.ll b/llvm/test/CodeGen/ARM/ctors_dtors.ll
index 066c250ca1448..f3d65ee982b58 100644
--- a/llvm/test/CodeGen/ARM/ctors_dtors.ll
+++ b/llvm/test/CodeGen/ARM/ctors_dtors.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s -mtriple=arm-apple-darwin  | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=arm-apple-darwin -disable-atexit-based-global-dtor-lowering  | FileCheck %s -check-prefix=DARWIN-LEGACY
 ; RUN: llc < %s -mtriple=arm-linux-gnu -target-abi=apcs  | FileCheck %s -check-prefix=ELF
 ; RUN: llc < %s -mtriple=arm-linux-gnueabi | FileCheck %s -check-prefix=GNUEABI
 
@@ -7,6 +8,10 @@
 ; DARWIN: .section	__DATA,__mod_init_func,mod_init_funcs
 ; DARWIN-NOT: __mod_term_func
 
+; DARWIN-LEGACY-NOT: atexit
+; DARWIN-LEGACY: .section	__DATA,__mod_init_func,mod_init_funcs
+; DARWIN-LEGACY: .section	__DATA,__mod_term_func,mod_term_funcs
+
 ; ELF: .section .ctors,"aw",%progbits
 ; ELF: .section .dtors,"aw",%progbits
 
diff --git a/llvm/test/CodeGen/ARM/debug-segmented-stacks.ll b/llvm/test/CodeGen/ARM/debug-segmented-stacks.ll
index 06050cbe8f841..e2f32fb45e206 100644
--- a/llvm/test/CodeGen/ARM/debug-segmented-stacks.ll
+++ b/llvm/test/CodeGen/ARM/debug-segmented-stacks.ll
@@ -20,7 +20,7 @@ define void @test_basic() #0 !dbg !4 {
 ; ARM-linux-NEXT: mov     r5, sp
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB0_2
+; ARM-linux-NEXT: bls     .LBB0_2
 
 ; ARM-linux:      mov     r4, #48
 ; ARM-linux-NEXT: mov     r5, #0
diff --git a/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll b/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll
index 9544e395744a4..4b27e804e6df9 100644
--- a/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/ARM/fptosi-sat-scalar.ll
@@ -571,71 +571,61 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #12
 ; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI6_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #27
-; SOFT-NEXT:    lsls r5, r0, #27
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    lsls r1, r0, #27
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB6_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB6_2:
-; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    movs r6, #0
 ; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB6_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r6, r7
+; SOFT-NEXT:    mvns r4, r6
 ; SOFT-NEXT:  .LBB6_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bne .LBB6_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:  .LBB6_6:
-; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB6_6:
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    beq .LBB6_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    b .LBB6_9
 ; SOFT-NEXT:  .LBB6_8:
-; SOFT-NEXT:    ldr r5, .LCPI6_1
+; SOFT-NEXT:    ldr r3, .LCPI6_1
 ; SOFT-NEXT:  .LBB6_9:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI6_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB6_11
 ; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r5, .LCPI6_2
+; SOFT-NEXT:    ldr r3, .LCPI6_2
 ; SOFT-NEXT:  .LBB6_11:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB6_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:  .LBB6_13:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -699,72 +689,62 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #12
 ; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI7_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #223
-; SOFT-NEXT:    lsls r5, r0, #24
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    lsls r1, r0, #24
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB7_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB7_2:
-; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    movs r6, #0
 ; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB7_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r6, r7
+; SOFT-NEXT:    mvns r4, r6
 ; SOFT-NEXT:  .LBB7_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bne .LBB7_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:  .LBB7_6:
-; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB7_6:
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    beq .LBB7_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    b .LBB7_9
 ; SOFT-NEXT:  .LBB7_8:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r5, r0, #31
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    lsls r3, r2, #31
 ; SOFT-NEXT:  .LBB7_9:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI7_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB7_11
 ; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r5, .LCPI7_1
+; SOFT-NEXT:    ldr r3, .LCPI7_1
 ; SOFT-NEXT:  .LBB7_11:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB7_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:  .LBB7_13:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -825,134 +805,110 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #20
 ; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    ldr r1, .LCPI8_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #241
-; SOFT-NEXT:    lsls r5, r0, #24
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    lsls r1, r0, #24
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB8_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB8_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r1, r6
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mvns r1, r5
+; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB8_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:  .LBB8_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bne .LBB8_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:  .LBB8_6:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB8_6:
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB8_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB8_8:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    bne .LBB8_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB8_10:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB8_12
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    beq .LBB8_18
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB8_19
 ; SOFT-NEXT:  .LBB8_12:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB8_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:  .LBB8_13:
+; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB8_14:
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bne .LBB8_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB8_16:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r7, r6
-; SOFT-NEXT:    bne .LBB8_18
+; SOFT-NEXT:    movs r4, #7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB8_20
 ; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:    b .LBB8_21
 ; SOFT-NEXT:  .LBB8_18:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    movs r5, #7
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB8_20
-; SOFT-NEXT:  @ %bb.19:
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB8_12
+; SOFT-NEXT:  .LBB8_19:
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB8_13
+; SOFT-NEXT:    b .LBB8_14
 ; SOFT-NEXT:  .LBB8_20:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB8_22
-; SOFT-NEXT:  @ %bb.21:
-; SOFT-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:  .LBB8_22:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB8_24
-; SOFT-NEXT:  @ %bb.23:
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:  .LBB8_24:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:  .LBB8_21:
+; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB8_23
+; SOFT-NEXT:  @ %bb.22:
+; SOFT-NEXT:    mov r4, r7
+; SOFT-NEXT:  .LBB8_23:
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bne .LBB8_25
+; SOFT-NEXT:  @ %bb.24:
+; SOFT-NEXT:    mov r5, r4
+; SOFT-NEXT:  .LBB8_25:
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    add sp, #20
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.25:
+; SOFT-NEXT:  @ %bb.26:
 ; SOFT-NEXT:  .LCPI8_0:
 ; SOFT-NEXT:    .long 1895825407 @ 0x70ffffff
 ;
@@ -1031,136 +987,111 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #20
 ; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    ldr r1, .LCPI9_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #255
-; SOFT-NEXT:    lsls r7, r0, #24
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    lsls r1, r0, #24
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __fixsfti
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
 ; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB9_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB9_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r1, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r5, r1
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mvns r1, r5
+; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB9_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:  .LBB9_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bne .LBB9_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:  .LBB9_6:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB9_6:
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB9_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB9_8:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    bne .LBB9_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB9_10:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    beq .LBB9_18
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB9_19
 ; SOFT-NEXT:  .LBB9_12:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB9_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:  .LBB9_13:
+; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB9_14:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bne .LBB9_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB9_16:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r5, r6
-; SOFT-NEXT:    bne .LBB9_18
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB9_20
 ; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    bne .LBB9_21
+; SOFT-NEXT:    b .LBB9_22
 ; SOFT-NEXT:  .LBB9_18:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    beq .LBB9_20
-; SOFT-NEXT:  @ %bb.19:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB9_21
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:  .LBB9_19:
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB9_13
+; SOFT-NEXT:    b .LBB9_14
 ; SOFT-NEXT:  .LBB9_20:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r7, r0, #31
+; SOFT-NEXT:    movs r4, #1
+; SOFT-NEXT:    lsls r6, r4, #31
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB9_22
 ; SOFT-NEXT:  .LBB9_21:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    beq .LBB9_23
-; SOFT-NEXT:  @ %bb.22:
-; SOFT-NEXT:    ldr r7, .LCPI9_1
-; SOFT-NEXT:  .LBB9_23:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB9_25
-; SOFT-NEXT:  @ %bb.24:
-; SOFT-NEXT:    mov r6, r7
-; SOFT-NEXT:  .LBB9_25:
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    ldr r6, .LCPI9_1
+; SOFT-NEXT:  .LBB9_22:
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bne .LBB9_24
+; SOFT-NEXT:  @ %bb.23:
+; SOFT-NEXT:    mov r5, r6
+; SOFT-NEXT:  .LBB9_24:
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    add sp, #20
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.26:
+; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI9_0:
 ; SOFT-NEXT:    .long 2130706431 @ 0x7effffff
 ; SOFT-NEXT:  .LCPI9_1:
@@ -1891,94 +1822,77 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    .pad #12
+; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r0, #15
 ; SOFT-NEXT:    mvns r2, r0
 ; SOFT-NEXT:    ldr r3, .LCPI16_0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #195
 ; SOFT-NEXT:    lsls r3, r0, #24
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    mov r2, r0
+; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB16_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:  .LBB16_2:
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
+; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    bne .LBB16_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    b .LBB16_5
 ; SOFT-NEXT:  .LBB16_4:
-; SOFT-NEXT:    mvns r7, r6
+; SOFT-NEXT:    mvns r0, r4
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB16_5:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r7
 ; SOFT-NEXT:    bl __aeabi_dcmpun
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bne .LBB16_7
 ; SOFT-NEXT:  @ %bb.6:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB16_7:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI16_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    ldr r3, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB16_9
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB16_9
 ; SOFT-NEXT:  @ %bb.8:
-; SOFT-NEXT:    ldr r0, .LCPI16_1
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    b .LBB16_10
 ; SOFT-NEXT:  .LBB16_9:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB16_11
-; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB16_12
-; SOFT-NEXT:  .LBB16_11:
-; SOFT-NEXT:    ldr r7, .LCPI16_2
+; SOFT-NEXT:    ldr r3, .LCPI16_1
+; SOFT-NEXT:  .LBB16_10:
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
+; SOFT-NEXT:    beq .LBB16_12
+; SOFT-NEXT:  @ %bb.11:
+; SOFT-NEXT:    ldr r3, .LCPI16_2
 ; SOFT-NEXT:  .LBB16_12:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB16_14
 ; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:    mov r4, r3
 ; SOFT-NEXT:  .LBB16_14:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.15:
@@ -2075,85 +1989,67 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #12
 ; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r6, r1
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    mvns r2, r4
 ; SOFT-NEXT:    ldr r3, .LCPI17_0
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    ldr r3, .LCPI17_1
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpge
 ; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_d2lz
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB17_2
 ; SOFT-NEXT:  @ %bb.1:
 ; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB17_2:
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB17_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB17_4:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mov r2, r5
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpun
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bne .LBB17_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB17_6:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI17_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI17_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB17_8
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB17_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r0, r0, #31
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    ldr r3, [sp] @ 4-byte Reload
+; SOFT-NEXT:    b .LBB17_9
 ; SOFT-NEXT:  .LBB17_8:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB17_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB17_11
-; SOFT-NEXT:  .LBB17_10:
-; SOFT-NEXT:    ldr r7, .LCPI17_2
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    lsls r3, r2, #31
+; SOFT-NEXT:  .LBB17_9:
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
+; SOFT-NEXT:    beq .LBB17_11
+; SOFT-NEXT:  @ %bb.10:
+; SOFT-NEXT:    ldr r3, .LCPI17_2
 ; SOFT-NEXT:  .LBB17_11:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB17_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r4, r7
+; SOFT-NEXT:    mov r4, r3
 ; SOFT-NEXT:  .LBB17_13:
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
@@ -2262,16 +2158,16 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
+; SOFT-NEXT:    .pad #28
+; SOFT-NEXT:    sub sp, #28
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    mvns r2, r4
 ; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; SOFT-NEXT:    ldr r3, .LCPI18_1
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
@@ -2281,145 +2177,99 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB18_2
 ; SOFT-NEXT:  @ %bb.1:
 ; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB18_2:
-; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB18_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:  .LBB18_4:
+; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpun
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bne .LBB18_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB18_6:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI18_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB18_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB18_8:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB18_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB18_10:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB18_12
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    beq .LBB18_18
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB18_19
 ; SOFT-NEXT:  .LBB18_12:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI18_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB18_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:  .LBB18_13:
+; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB18_14:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bne .LBB18_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB18_16:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB18_18
+; SOFT-NEXT:    movs r5, #7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB18_20
 ; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB18_21
+; SOFT-NEXT:    b .LBB18_22
 ; SOFT-NEXT:  .LBB18_18:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI18_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    movs r7, #7
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    beq .LBB18_20
-; SOFT-NEXT:  @ %bb.19:
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB18_21
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB18_12
+; SOFT-NEXT:  .LBB18_19:
+; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB18_13
+; SOFT-NEXT:    b .LBB18_14
 ; SOFT-NEXT:  .LBB18_20:
-; SOFT-NEXT:    mvns r0, r7
+; SOFT-NEXT:    mvns r7, r5
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB18_22
 ; SOFT-NEXT:  .LBB18_21:
-; SOFT-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    bne .LBB18_23
-; SOFT-NEXT:  @ %bb.22:
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:  .LBB18_23:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB18_25
-; SOFT-NEXT:  @ %bb.24:
-; SOFT-NEXT:    mov r4, r7
-; SOFT-NEXT:  .LBB18_25:
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    mov r5, r7
+; SOFT-NEXT:  .LBB18_22:
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bne .LBB18_24
+; SOFT-NEXT:  @ %bb.23:
+; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:  .LBB18_24:
 ; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    add sp, #28
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.26:
+; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI18_0:
 ; SOFT-NEXT:    .long 1176502271 @ 0x461fffff
 ; SOFT-NEXT:  .LCPI18_1:
@@ -2569,16 +2419,16 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
+; SOFT-NEXT:    .pad #28
+; SOFT-NEXT:    sub sp, #28
 ; SOFT-NEXT:    mov r5, r1
 ; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    movs r4, #0
 ; SOFT-NEXT:    mvns r2, r4
 ; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r0, [sp, #20] @ 4-byte Spill
 ; SOFT-NEXT:    ldr r3, .LCPI19_1
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
@@ -2588,145 +2438,99 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB19_2
 ; SOFT-NEXT:  @ %bb.1:
 ; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB19_2:
-; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB19_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:  .LBB19_4:
+; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    bl __aeabi_dcmpun
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bne .LBB19_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB19_6:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI19_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    ldr r6, [sp, #20] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB19_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB19_8:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB19_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB19_10:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB19_12
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    beq .LBB19_18
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB19_19
 ; SOFT-NEXT:  .LBB19_12:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI19_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB19_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:  .LBB19_13:
+; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB19_14:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bne .LBB19_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB19_16:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB19_18
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB19_20
 ; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB19_21
+; SOFT-NEXT:    b .LBB19_22
 ; SOFT-NEXT:  .LBB19_18:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    ldr r2, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI19_1
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB19_20
-; SOFT-NEXT:  @ %bb.19:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r0, r0, #31
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:  .LBB19_20:
+; SOFT-NEXT:    mov r1, r2
 ; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    bne .LBB19_22
-; SOFT-NEXT:  @ %bb.21:
-; SOFT-NEXT:    ldr r7, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB19_23
+; SOFT-NEXT:    bne .LBB19_12
+; SOFT-NEXT:  .LBB19_19:
+; SOFT-NEXT:    str r7, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB19_13
+; SOFT-NEXT:    b .LBB19_14
+; SOFT-NEXT:  .LBB19_20:
+; SOFT-NEXT:    movs r5, #1
+; SOFT-NEXT:    lsls r5, r5, #31
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB19_22
+; SOFT-NEXT:  .LBB19_21:
+; SOFT-NEXT:    ldr r5, .LCPI19_2
 ; SOFT-NEXT:  .LBB19_22:
-; SOFT-NEXT:    ldr r7, .LCPI19_2
-; SOFT-NEXT:  .LBB19_23:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB19_25
-; SOFT-NEXT:  @ %bb.24:
-; SOFT-NEXT:    mov r4, r7
-; SOFT-NEXT:  .LBB19_25:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bne .LBB19_24
+; SOFT-NEXT:  @ %bb.23:
+; SOFT-NEXT:    mov r4, r5
+; SOFT-NEXT:  .LBB19_24:
 ; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    add sp, #28
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.26:
+; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI19_0:
 ; SOFT-NEXT:    .long 1205862399 @ 0x47dfffff
 ; SOFT-NEXT:  .LCPI19_1:
@@ -3485,71 +3289,61 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI26_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #27
-; SOFT-NEXT:    lsls r5, r0, #27
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    lsls r1, r0, #27
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB26_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB26_2:
-; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    movs r6, #0
 ; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB26_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r6, r7
+; SOFT-NEXT:    mvns r4, r6
 ; SOFT-NEXT:  .LBB26_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bne .LBB26_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:  .LBB26_6:
-; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB26_6:
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    beq .LBB26_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    b .LBB26_9
 ; SOFT-NEXT:  .LBB26_8:
-; SOFT-NEXT:    ldr r5, .LCPI26_1
+; SOFT-NEXT:    ldr r3, .LCPI26_1
 ; SOFT-NEXT:  .LBB26_9:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI26_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB26_11
 ; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r5, .LCPI26_2
+; SOFT-NEXT:    ldr r3, .LCPI26_2
 ; SOFT-NEXT:  .LBB26_11:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB26_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:  .LBB26_13:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -3660,72 +3454,62 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #12
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI27_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; SOFT-NEXT:    movs r0, #223
-; SOFT-NEXT:    lsls r5, r0, #24
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    lsls r1, r0, #24
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2lz
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB27_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r6, r7
+; SOFT-NEXT:    mov r4, r5
 ; SOFT-NEXT:  .LBB27_2:
-; SOFT-NEXT:    movs r7, #0
+; SOFT-NEXT:    movs r6, #0
 ; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r0, #0
 ; SOFT-NEXT:    beq .LBB27_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r6, r7
+; SOFT-NEXT:    mvns r4, r6
 ; SOFT-NEXT:  .LBB27_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r1, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bne .LBB27_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:  .LBB27_6:
-; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB27_6:
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    beq .LBB27_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:    b .LBB27_9
 ; SOFT-NEXT:  .LBB27_8:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r5, r0, #31
+; SOFT-NEXT:    movs r2, #1
+; SOFT-NEXT:    lsls r3, r2, #31
 ; SOFT-NEXT:  .LBB27_9:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI27_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    beq .LBB27_11
 ; SOFT-NEXT:  @ %bb.10:
-; SOFT-NEXT:    ldr r5, .LCPI27_1
+; SOFT-NEXT:    ldr r3, .LCPI27_1
 ; SOFT-NEXT:  .LBB27_11:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r1, #0
 ; SOFT-NEXT:    bne .LBB27_13
 ; SOFT-NEXT:  @ %bb.12:
-; SOFT-NEXT:    mov r7, r5
+; SOFT-NEXT:    mov r6, r3
 ; SOFT-NEXT:  .LBB27_13:
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    add sp, #12
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -3834,134 +3618,110 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #20
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    ldr r1, .LCPI28_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #241
-; SOFT-NEXT:    lsls r5, r0, #24
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    lsls r1, r0, #24
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    str r1, [sp] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB28_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB28_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r1, r6
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mvns r1, r5
+; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB28_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:  .LBB28_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bne .LBB28_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:  .LBB28_6:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB28_6:
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB28_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB28_8:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    bne .LBB28_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB28_10:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB28_12
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    beq .LBB28_18
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB28_19
 ; SOFT-NEXT:  .LBB28_12:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB28_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:  .LBB28_13:
+; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB28_14:
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bne .LBB28_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB28_16:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r7, r6
-; SOFT-NEXT:    bne .LBB28_18
+; SOFT-NEXT:    movs r4, #7
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB28_20
 ; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r7, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
+; SOFT-NEXT:    b .LBB28_21
 ; SOFT-NEXT:  .LBB28_18:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    movs r5, #7
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB28_20
-; SOFT-NEXT:  @ %bb.19:
-; SOFT-NEXT:    mvns r0, r5
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB28_12
+; SOFT-NEXT:  .LBB28_19:
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB28_13
+; SOFT-NEXT:    b .LBB28_14
 ; SOFT-NEXT:  .LBB28_20:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB28_22
-; SOFT-NEXT:  @ %bb.21:
-; SOFT-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:  .LBB28_22:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB28_24
-; SOFT-NEXT:  @ %bb.23:
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:  .LBB28_24:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    mvns r7, r4
+; SOFT-NEXT:  .LBB28_21:
+; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB28_23
+; SOFT-NEXT:  @ %bb.22:
+; SOFT-NEXT:    mov r4, r7
+; SOFT-NEXT:  .LBB28_23:
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bne .LBB28_25
+; SOFT-NEXT:  @ %bb.24:
+; SOFT-NEXT:    mov r5, r4
+; SOFT-NEXT:  .LBB28_25:
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    add sp, #20
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.25:
+; SOFT-NEXT:  @ %bb.26:
 ; SOFT-NEXT:  .LCPI28_0:
 ; SOFT-NEXT:    .long 1895825407 @ 0x70ffffff
 ;
@@ -4112,136 +3872,111 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #20
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:    ldr r1, .LCPI29_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #255
-; SOFT-NEXT:    lsls r7, r0, #24
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    lsls r1, r0, #24
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    mov r0, r6
 ; SOFT-NEXT:    bl __fixsfti
 ; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
 ; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    cmp r7, #0
 ; SOFT-NEXT:    bne .LBB29_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:  .LBB29_2:
-; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r1, r6
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r5, r1
+; SOFT-NEXT:    movs r5, #0
+; SOFT-NEXT:    mvns r1, r5
+; SOFT-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:    bne .LBB29_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:  .LBB29_4:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpun
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:    bne .LBB29_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:  .LBB29_6:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB29_6:
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB29_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    str r7, [sp, #4] @ 4-byte Spill
 ; SOFT-NEXT:  .LBB29_8:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    bne .LBB29_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    ldr r5, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB29_10:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    beq .LBB29_18
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB29_19
 ; SOFT-NEXT:  .LBB29_12:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB29_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:  .LBB29_13:
+; SOFT-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB29_14:
-; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    bne .LBB29_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB29_16:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r5, r6
-; SOFT-NEXT:    bne .LBB29_18
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    beq .LBB29_20
 ; SOFT-NEXT:  @ %bb.17:
-; SOFT-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    bne .LBB29_21
+; SOFT-NEXT:    b .LBB29_22
 ; SOFT-NEXT:  .LBB29_18:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    beq .LBB29_20
-; SOFT-NEXT:  @ %bb.19:
-; SOFT-NEXT:    ldr r7, [sp] @ 4-byte Reload
-; SOFT-NEXT:    b .LBB29_21
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:  .LBB29_19:
+; SOFT-NEXT:    str r7, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB29_13
+; SOFT-NEXT:    b .LBB29_14
 ; SOFT-NEXT:  .LBB29_20:
-; SOFT-NEXT:    movs r0, #1
-; SOFT-NEXT:    lsls r7, r0, #31
+; SOFT-NEXT:    movs r4, #1
+; SOFT-NEXT:    lsls r6, r4, #31
+; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB29_22
 ; SOFT-NEXT:  .LBB29_21:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    beq .LBB29_23
-; SOFT-NEXT:  @ %bb.22:
-; SOFT-NEXT:    ldr r7, .LCPI29_1
-; SOFT-NEXT:  .LBB29_23:
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bl __aeabi_fcmpun
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB29_25
-; SOFT-NEXT:  @ %bb.24:
-; SOFT-NEXT:    mov r6, r7
-; SOFT-NEXT:  .LBB29_25:
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    ldr r6, .LCPI29_1
+; SOFT-NEXT:  .LBB29_22:
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    bne .LBB29_24
+; SOFT-NEXT:  @ %bb.23:
+; SOFT-NEXT:    mov r5, r6
+; SOFT-NEXT:  .LBB29_24:
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:    add sp, #20
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
-; SOFT-NEXT:  @ %bb.26:
+; SOFT-NEXT:  @ %bb.25:
 ; SOFT-NEXT:  .LCPI29_0:
 ; SOFT-NEXT:    .long 2130706431 @ 0x7effffff
 ; SOFT-NEXT:  .LCPI29_1:
diff --git a/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll b/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll
index 0fc1eabe9b9f0..3438fb113015c 100644
--- a/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/ARM/fptoui-sat-scalar.ll
@@ -443,48 +443,45 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    .pad #4
 ; SOFT-NEXT:    sub sp, #4
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI6_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2ulz
 ; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB6_2
+; SOFT-NEXT:    beq .LBB6_5
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    bne .LBB6_6
 ; SOFT-NEXT:  .LBB6_2:
-; SOFT-NEXT:    mov r5, r1
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB6_7
+; SOFT-NEXT:  .LBB6_3:
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB6_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r0, r7
+; SOFT-NEXT:    bne .LBB6_8
 ; SOFT-NEXT:  .LBB6_4:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    ldr r1, .LCPI6_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB6_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB6_5:
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB6_2
 ; SOFT-NEXT:  .LBB6_6:
+; SOFT-NEXT:    mvns r0, r6
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB6_3
+; SOFT-NEXT:  .LBB6_7:
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB6_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, .LCPI6_1
+; SOFT-NEXT:    beq .LBB6_4
 ; SOFT-NEXT:  .LBB6_8:
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    ldr r1, .LCPI6_1
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -529,54 +526,50 @@ define i64 @test_signed_i64_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI7_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2ulz
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB7_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:  .LBB7_2:
-; SOFT-NEXT:    mvns r5, r7
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bne .LBB7_4
+; SOFT-NEXT:    mvns r2, r6
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    beq .LBB7_7
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB7_8
 ; SOFT-NEXT:  .LBB7_4:
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI7_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB7_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:  .LBB7_5:
+; SOFT-NEXT:    mov r2, r1
 ; SOFT-NEXT:  .LBB7_6:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB7_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB7_7:
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB7_4
 ; SOFT-NEXT:  .LBB7_8:
 ; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    add sp, #12
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB7_5
+; SOFT-NEXT:    b .LBB7_6
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
 ; SOFT-NEXT:  .LCPI7_0:
@@ -622,94 +615,76 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI8_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB8_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:  .LBB8_2:
-; SOFT-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mvns r7, r6
+; SOFT-NEXT:    mvns r2, r6
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r6, r2
 ; SOFT-NEXT:    bne .LBB8_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:  .LBB8_4:
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB8_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:  .LBB8_6:
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    bne .LBB8_8
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    beq .LBB8_13
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB8_14
 ; SOFT-NEXT:  .LBB8_8:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB8_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB8_15
+; SOFT-NEXT:  .LBB8_9:
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB8_16
 ; SOFT-NEXT:  .LBB8_10:
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB8_12
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB8_12
+; SOFT-NEXT:  .LBB8_11:
+; SOFT-NEXT:    movs r3, #15
 ; SOFT-NEXT:  .LBB8_12:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    ldr r1, .LCPI8_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB8_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB8_13:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB8_8
 ; SOFT-NEXT:  .LBB8_14:
+; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    beq .LBB8_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    bne .LBB8_9
+; SOFT-NEXT:  .LBB8_15:
+; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB8_10
 ; SOFT-NEXT:  .LBB8_16:
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    add sp, #20
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    bne .LBB8_11
+; SOFT-NEXT:    b .LBB8_12
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI8_0:
@@ -771,96 +746,75 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI9_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB9_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:  .LBB9_2:
-; SOFT-NEXT:    mvns r6, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mvns r6, r6
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bne .LBB9_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:  .LBB9_4:
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB9_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:  .LBB9_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bne .LBB9_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:  .LBB9_8:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB9_8:
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB9_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:  .LBB9_10:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:    beq .LBB9_15
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB9_16
 ; SOFT-NEXT:  .LBB9_12:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI9_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB9_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:  .LBB9_13:
+; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB9_14:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB9_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:  .LBB9_16:
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:    mov r2, r7
 ; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB9_15:
+; SOFT-NEXT:    mov r7, r2
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB9_12
+; SOFT-NEXT:  .LBB9_16:
+; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB9_13
+; SOFT-NEXT:    b .LBB9_14
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI9_0:
@@ -1431,64 +1385,54 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
-; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r0, #7
 ; SOFT-NEXT:    mvns r2, r0
 ; SOFT-NEXT:    ldr r3, .LCPI16_0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_d2ulz
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB16_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:  .LBB16_2:
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    cmp r2, #0
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    beq .LBB16_4
+; SOFT-NEXT:    bne .LBB16_6
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r0, r7
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB16_7
 ; SOFT-NEXT:  .LBB16_4:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
-; SOFT-NEXT:    ldr r3, .LCPI16_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    mov r3, r7
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB16_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    cmp r2, #0
+; SOFT-NEXT:    bne .LBB16_8
+; SOFT-NEXT:  .LBB16_5:
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:  .LBB16_6:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    beq .LBB16_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r1, .LCPI16_1
+; SOFT-NEXT:    mvns r0, r6
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB16_4
+; SOFT-NEXT:  .LBB16_7:
+; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    cmp r2, #0
+; SOFT-NEXT:    beq .LBB16_5
 ; SOFT-NEXT:  .LBB16_8:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    add sp, #12
+; SOFT-NEXT:    ldr r1, .LCPI16_1
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
@@ -1554,66 +1498,56 @@ define i64 @test_signed_i64_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r2, r6
+; SOFT-NEXT:    mvns r4, r6
 ; SOFT-NEXT:    ldr r3, .LCPI17_0
-; SOFT-NEXT:    str r2, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __aeabi_d2ulz
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r2, r0
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB17_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB17_2:
-; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    bne .LBB17_4
+; SOFT-NEXT:    ldr r3, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    beq .LBB17_7
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB17_8
 ; SOFT-NEXT:  .LBB17_4:
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI17_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r3, #0
 ; SOFT-NEXT:    bne .LBB17_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:  .LBB17_5:
+; SOFT-NEXT:    mov r4, r1
 ; SOFT-NEXT:  .LBB17_6:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB17_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:  .LBB17_8:
 ; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB17_7:
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB17_4
+; SOFT-NEXT:  .LBB17_8:
+; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    cmp r3, #0
+; SOFT-NEXT:    beq .LBB17_5
+; SOFT-NEXT:    b .LBB17_6
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
 ; SOFT-NEXT:  .LCPI17_0:
@@ -1693,117 +1627,82 @@ define i100 @test_signed_i100_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #28
-; SOFT-NEXT:    sub sp, #28
-; SOFT-NEXT:    mov r4, r1
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    mvns r2, r5
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mvns r2, r6
 ; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    str r2, [sp, #20] @ 4-byte Spill
+; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __fixunsdfti
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB18_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r4, r6
 ; SOFT-NEXT:  .LBB18_2:
-; SOFT-NEXT:    str r3, [sp, #24] @ 4-byte Spill
-; SOFT-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r7, [sp, #20] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
 ; SOFT-NEXT:    bne .LBB18_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:  .LBB18_4:
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB18_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r7, r6
 ; SOFT-NEXT:  .LBB18_6:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    bne .LBB18_8
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    ldr r4, [sp] @ 4-byte Reload
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    beq .LBB18_13
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB18_14
 ; SOFT-NEXT:  .LBB18_8:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB18_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB18_15
+; SOFT-NEXT:  .LBB18_9:
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    beq .LBB18_16
 ; SOFT-NEXT:  .LBB18_10:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    bne .LBB18_12
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB18_12
+; SOFT-NEXT:  .LBB18_11:
+; SOFT-NEXT:    movs r3, #15
 ; SOFT-NEXT:  .LBB18_12:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    ldr r3, .LCPI18_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    mov r3, r5
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    ldr r3, [sp, #24] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB18_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:    mov r2, r4
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB18_13:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB18_8
 ; SOFT-NEXT:  .LBB18_14:
-; SOFT-NEXT:    cmp r7, #0
-; SOFT-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    beq .LBB18_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    mov r2, r6
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB18_9
+; SOFT-NEXT:  .LBB18_15:
+; SOFT-NEXT:    mov r4, r2
+; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    bne .LBB18_10
 ; SOFT-NEXT:  .LBB18_16:
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    add sp, #28
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r3, r6
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB18_11
+; SOFT-NEXT:    b .LBB18_12
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI18_0:
@@ -1915,117 +1814,77 @@ define i128 @test_signed_i128_f64(double %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #28
-; SOFT-NEXT:    sub sp, #28
-; SOFT-NEXT:    mov r5, r1
-; SOFT-NEXT:    mov r7, r0
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    mov r5, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mvns r2, r6
+; SOFT-NEXT:    mvns r4, r6
 ; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    str r2, [sp, #24] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r4
 ; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r1, r7
 ; SOFT-NEXT:    bl __fixunsdfti
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #20] @ 4-byte Spill
-; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB19_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r6
 ; SOFT-NEXT:  .LBB19_2:
-; SOFT-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    cmp r1, #0
-; SOFT-NEXT:    ldr r4, [sp, #24] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    bne .LBB19_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:  .LBB19_4:
-; SOFT-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB19_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:  .LBB19_6:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB19_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:  .LBB19_8:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  @ %bb.5:
+; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:  .LBB19_6:
+; SOFT-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r1, #0
+; SOFT-NEXT:    mov r1, r4
+; SOFT-NEXT:    bne .LBB19_8
+; SOFT-NEXT:  @ %bb.7:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:  .LBB19_8:
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB19_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r6
 ; SOFT-NEXT:  .LBB19_10:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    ldr r5, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    mov r5, r4
 ; SOFT-NEXT:    bne .LBB19_12
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    mov r5, r2
 ; SOFT-NEXT:  .LBB19_12:
-; SOFT-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    ldr r3, .LCPI19_0
-; SOFT-NEXT:    bl __aeabi_dcmpgt
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    mov r2, r6
-; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    bl __aeabi_dcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r6, #0
 ; SOFT-NEXT:    bne .LBB19_14
 ; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:  .LBB19_14:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r2, #0
 ; SOFT-NEXT:    bne .LBB19_16
 ; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
+; SOFT-NEXT:    mov r4, r3
 ; SOFT-NEXT:  .LBB19_16:
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:    mov r3, r4
-; SOFT-NEXT:    add sp, #28
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
@@ -2617,48 +2476,45 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 ; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r6, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI26_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2ulz
 ; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    bne .LBB26_2
+; SOFT-NEXT:    beq .LBB26_5
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    bne .LBB26_6
 ; SOFT-NEXT:  .LBB26_2:
-; SOFT-NEXT:    mov r5, r1
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB26_7
+; SOFT-NEXT:  .LBB26_3:
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_4
-; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mvns r0, r7
+; SOFT-NEXT:    bne .LBB26_8
 ; SOFT-NEXT:  .LBB26_4:
-; SOFT-NEXT:    str r0, [sp] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    ldr r1, .LCPI26_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB26_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB26_5:
+; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB26_2
 ; SOFT-NEXT:  .LBB26_6:
+; SOFT-NEXT:    mvns r0, r6
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB26_3
+; SOFT-NEXT:  .LBB26_7:
+; SOFT-NEXT:    mov r1, r5
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, .LCPI26_1
+; SOFT-NEXT:    beq .LBB26_4
 ; SOFT-NEXT:  .LBB26_8:
-; SOFT-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; SOFT-NEXT:    mov r1, r5
+; SOFT-NEXT:    ldr r1, .LCPI26_1
 ; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
 ; SOFT-NEXT:    .p2align 2
@@ -2733,56 +2589,52 @@ define i64 @test_signed_i64_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #12
-; SOFT-NEXT:    sub sp, #12
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI27_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
 ; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __aeabi_f2ulz
-; SOFT-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB27_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r3, r5
 ; SOFT-NEXT:  .LBB27_2:
-; SOFT-NEXT:    mvns r5, r7
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bne .LBB27_4
+; SOFT-NEXT:    mvns r2, r6
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r0, r2
+; SOFT-NEXT:    beq .LBB27_7
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB27_8
 ; SOFT-NEXT:  .LBB27_4:
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI27_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB27_6
-; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:  .LBB27_5:
+; SOFT-NEXT:    mov r2, r1
 ; SOFT-NEXT:  .LBB27_6:
-; SOFT-NEXT:    cmp r6, #0
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB27_8
-; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB27_7:
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB27_4
 ; SOFT-NEXT:  .LBB27_8:
 ; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    add sp, #12
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB27_5
+; SOFT-NEXT:    b .LBB27_6
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.9:
 ; SOFT-NEXT:  .LCPI27_0:
@@ -2863,96 +2715,78 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI28_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
 ; SOFT-NEXT:    mov r4, r0
 ; SOFT-NEXT:    movs r6, #0
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r7, r0
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    cmp r7, #0
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    str r2, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB28_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:  .LBB28_2:
-; SOFT-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    mvns r7, r6
+; SOFT-NEXT:    mvns r2, r6
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r6, r2
 ; SOFT-NEXT:    bne .LBB28_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r6, r0
 ; SOFT-NEXT:  .LBB28_4:
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB28_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:  .LBB28_6:
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    mov r0, r7
-; SOFT-NEXT:    bne .LBB28_8
+; SOFT-NEXT:    mov r1, r2
+; SOFT-NEXT:    beq .LBB28_13
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB28_14
 ; SOFT-NEXT:  .LBB28_8:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    bne .LBB28_10
-; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB28_15
+; SOFT-NEXT:  .LBB28_9:
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB28_16
 ; SOFT-NEXT:  .LBB28_10:
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB28_12
-; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    beq .LBB28_12
+; SOFT-NEXT:  .LBB28_11:
+; SOFT-NEXT:    movs r3, #15
 ; SOFT-NEXT:  .LBB28_12:
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    ldr r1, .LCPI28_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r4, r0
-; SOFT-NEXT:    mov r0, r5
-; SOFT-NEXT:    mov r1, r6
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
-; SOFT-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB28_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    mov r3, r0
+; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    add sp, #4
+; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB28_13:
+; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB28_8
 ; SOFT-NEXT:  .LBB28_14:
+; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
 ; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    beq .LBB28_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    movs r3, #15
+; SOFT-NEXT:    bne .LBB28_9
+; SOFT-NEXT:  .LBB28_15:
+; SOFT-NEXT:    ldr r2, [sp] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB28_10
 ; SOFT-NEXT:  .LBB28_16:
-; SOFT-NEXT:    mov r2, r7
-; SOFT-NEXT:    add sp, #20
-; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    bne .LBB28_11
+; SOFT-NEXT:    b .LBB28_12
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI28_0:
@@ -3065,98 +2899,77 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; SOFT:       @ %bb.0:
 ; SOFT-NEXT:    .save {r4, r5, r6, r7, lr}
 ; SOFT-NEXT:    push {r4, r5, r6, r7, lr}
-; SOFT-NEXT:    .pad #20
-; SOFT-NEXT:    sub sp, #20
+; SOFT-NEXT:    .pad #4
+; SOFT-NEXT:    sub sp, #4
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
-; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    mov r7, r0
 ; SOFT-NEXT:    ldr r1, .LCPI29_0
 ; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    movs r7, #0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
+; SOFT-NEXT:    mov r4, r0
+; SOFT-NEXT:    movs r6, #0
+; SOFT-NEXT:    mov r0, r7
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    mov r6, r0
-; SOFT-NEXT:    mov r0, r4
+; SOFT-NEXT:    mov r5, r0
+; SOFT-NEXT:    mov r0, r7
 ; SOFT-NEXT:    bl __fixunssfti
-; SOFT-NEXT:    str r1, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; SOFT-NEXT:    cmp r6, #0
+; SOFT-NEXT:    mov r7, r1
+; SOFT-NEXT:    str r3, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB29_2
 ; SOFT-NEXT:  @ %bb.1:
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    mov r0, r5
 ; SOFT-NEXT:  .LBB29_2:
-; SOFT-NEXT:    mvns r6, r7
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r1, r6
+; SOFT-NEXT:    mvns r6, r6
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r3, r6
 ; SOFT-NEXT:    bne .LBB29_4
 ; SOFT-NEXT:  @ %bb.3:
-; SOFT-NEXT:    mov r1, r0
+; SOFT-NEXT:    mov r3, r0
 ; SOFT-NEXT:  .LBB29_4:
-; SOFT-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB29_6
 ; SOFT-NEXT:  @ %bb.5:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; SOFT-NEXT:    mov r7, r5
 ; SOFT-NEXT:  .LBB29_6:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r0, r6
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r1, r6
 ; SOFT-NEXT:    bne .LBB29_8
 ; SOFT-NEXT:  @ %bb.7:
-; SOFT-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:  .LBB29_8:
-; SOFT-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
 ; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:  .LBB29_8:
+; SOFT-NEXT:    cmp r5, #0
 ; SOFT-NEXT:    bne .LBB29_10
 ; SOFT-NEXT:  @ %bb.9:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; SOFT-NEXT:    mov r2, r5
 ; SOFT-NEXT:  .LBB29_10:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    mov r0, r6
-; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    mov r7, r6
+; SOFT-NEXT:    beq .LBB29_15
 ; SOFT-NEXT:  @ %bb.11:
-; SOFT-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    beq .LBB29_16
 ; SOFT-NEXT:  .LBB29_12:
-; SOFT-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    ldr r1, .LCPI29_0
-; SOFT-NEXT:    bl __aeabi_fcmpgt
-; SOFT-NEXT:    mov r5, r0
-; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    mov r1, r7
-; SOFT-NEXT:    bl __aeabi_fcmpge
-; SOFT-NEXT:    cmp r0, #0
+; SOFT-NEXT:    cmp r4, #0
 ; SOFT-NEXT:    bne .LBB29_14
-; SOFT-NEXT:  @ %bb.13:
-; SOFT-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; SOFT-NEXT:  .LBB29_13:
+; SOFT-NEXT:    ldr r6, [sp] @ 4-byte Reload
 ; SOFT-NEXT:  .LBB29_14:
-; SOFT-NEXT:    cmp r5, #0
-; SOFT-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; SOFT-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; SOFT-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
-; SOFT-NEXT:    bne .LBB29_16
-; SOFT-NEXT:  @ %bb.15:
-; SOFT-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
-; SOFT-NEXT:  .LBB29_16:
+; SOFT-NEXT:    mov r0, r3
+; SOFT-NEXT:    mov r2, r7
 ; SOFT-NEXT:    mov r3, r6
-; SOFT-NEXT:    add sp, #20
+; SOFT-NEXT:    add sp, #4
 ; SOFT-NEXT:    pop {r4, r5, r6, r7, pc}
+; SOFT-NEXT:  .LBB29_15:
+; SOFT-NEXT:    mov r7, r2
+; SOFT-NEXT:    cmp r5, #0
+; SOFT-NEXT:    bne .LBB29_12
+; SOFT-NEXT:  .LBB29_16:
+; SOFT-NEXT:    str r5, [sp] @ 4-byte Spill
+; SOFT-NEXT:    cmp r4, #0
+; SOFT-NEXT:    beq .LBB29_13
+; SOFT-NEXT:    b .LBB29_14
 ; SOFT-NEXT:    .p2align 2
 ; SOFT-NEXT:  @ %bb.17:
 ; SOFT-NEXT:  .LCPI29_0:
diff --git a/llvm/test/CodeGen/ARM/segmented-stacks-dynamic.ll b/llvm/test/CodeGen/ARM/segmented-stacks-dynamic.ll
index 6bfa7a3508b20..ca589afbb933a 100644
--- a/llvm/test/CodeGen/ARM/segmented-stacks-dynamic.ll
+++ b/llvm/test/CodeGen/ARM/segmented-stacks-dynamic.ll
@@ -27,7 +27,7 @@ false:
 ; ARM-linux-NEXT: mov     r5, sp
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB0_2
+; ARM-linux-NEXT: bls     .LBB0_2
 
 ; ARM-linux:      mov     r4, #16
 ; ARM-linux-NEXT: mov     r5, #0
@@ -47,7 +47,7 @@ false:
 ; ARM-android-NEXT: mov     r5, sp
 ; ARM-android-NEXT: ldr     r4, [r4, #252]
 ; ARM-android-NEXT: cmp     r4, r5
-; ARM-android-NEXT: blo     .LBB0_2
+; ARM-android-NEXT: bls     .LBB0_2
 
 ; ARM-android:      mov     r4, #16
 ; ARM-android-NEXT: mov     r5, #0
diff --git a/llvm/test/CodeGen/ARM/segmented-stacks.ll b/llvm/test/CodeGen/ARM/segmented-stacks.ll
index 3acae3ccf3236..a9c3be81562d2 100644
--- a/llvm/test/CodeGen/ARM/segmented-stacks.ll
+++ b/llvm/test/CodeGen/ARM/segmented-stacks.ll
@@ -21,7 +21,7 @@ define void @test_basic() #0 {
 ; ARM-linux-NEXT: mov     r5, sp
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB0_2
+; ARM-linux-NEXT: bls     .LBB0_2
 
 ; ARM-linux:      mov     r4, #48
 ; ARM-linux-NEXT: mov     r5, #0
@@ -40,7 +40,7 @@ define void @test_basic() #0 {
 ; ARM-android-NEXT: mov     r5, sp
 ; ARM-android-NEXT: ldr     r4, [r4, #252]
 ; ARM-android-NEXT: cmp     r4, r5
-; ARM-android-NEXT: blo     .LBB0_2
+; ARM-android-NEXT: bls     .LBB0_2
 
 ; ARM-android:      mov     r4, #48
 ; ARM-android-NEXT: mov     r5, #0
@@ -68,7 +68,7 @@ define i32 @test_nested(ptr nest %closure, i32 %other) #0 {
 ; ARM-linux-NEXT: mov     r5, sp
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB1_2
+; ARM-linux-NEXT: bls     .LBB1_2
 
 ; ARM-linux:      mov     r4, #56
 ; ARM-linux-NEXT: mov     r5, #0
@@ -87,7 +87,7 @@ define i32 @test_nested(ptr nest %closure, i32 %other) #0 {
 ; ARM-android-NEXT: mov     r5, sp
 ; ARM-android-NEXT: ldr     r4, [r4, #252]
 ; ARM-android-NEXT: cmp     r4, r5
-; ARM-android-NEXT: blo     .LBB1_2
+; ARM-android-NEXT: bls     .LBB1_2
 
 ; ARM-android:      mov     r4, #56
 ; ARM-android-NEXT: mov     r5, #0
@@ -114,7 +114,7 @@ define void @test_large() #0 {
 ; ARM-linux-NEXT: mrc     p15, #0, r4, c13, c0, #3
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB2_2
+; ARM-linux-NEXT: bls     .LBB2_2
 
 ; ARM-linux:      ldr     r4, .LCPI2_0
 ; ARM-linux-NEXT: mov     r5, #0
@@ -137,7 +137,7 @@ define void @test_large() #0 {
 ; ARM-android-NEXT: mrc     p15, #0, r4, c13, c0, #3
 ; ARM-android-NEXT: ldr     r4, [r4, #252]
 ; ARM-android-NEXT: cmp     r4, r5
-; ARM-android-NEXT: blo     .LBB2_2
+; ARM-android-NEXT: bls     .LBB2_2
 
 ; ARM-android:      ldr     r4, .LCPI2_0
 ; ARM-android-NEXT: mov     r5, #0
@@ -166,7 +166,7 @@ define fastcc void @test_fastcc() #0 {
 ; ARM-linux-NEXT: mov     r5, sp
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB3_2
+; ARM-linux-NEXT: bls     .LBB3_2
 
 ; ARM-linux:      mov     r4, #48
 ; ARM-linux-NEXT: mov     r5, #0
@@ -185,7 +185,7 @@ define fastcc void @test_fastcc() #0 {
 ; ARM-android-NEXT: mov     r5, sp
 ; ARM-android-NEXT: ldr     r4, [r4, #252]
 ; ARM-android-NEXT: cmp     r4, r5
-; ARM-android-NEXT: blo     .LBB3_2
+; ARM-android-NEXT: bls     .LBB3_2
 
 ; ARM-android:      mov     r4, #48
 ; ARM-android-NEXT: mov     r5, #0
@@ -212,7 +212,7 @@ define fastcc void @test_fastcc_large() #0 {
 ; ARM-linux-NEXT: mrc     p15, #0, r4, c13, c0, #3
 ; ARM-linux-NEXT: ldr     r4, [r4, #4]
 ; ARM-linux-NEXT: cmp     r4, r5
-; ARM-linux-NEXT: blo     .LBB4_2
+; ARM-linux-NEXT: bls     .LBB4_2
 
 ; ARM-linux:      ldr     r4, .LCPI4_0
 ; ARM-linux-NEXT: mov     r5, #0
@@ -235,7 +235,7 @@ define fastcc void @test_fastcc_large() #0 {
 ; ARM-android-NEXT: mrc     p15, #0, r4, c13, c0, #3
 ; ARM-android-NEXT: ldr     r4, [r4, #252]
 ; ARM-android-NEXT: cmp     r4, r5
-; ARM-android-NEXT: blo     .LBB4_2
+; ARM-android-NEXT: bls     .LBB4_2
 
 ; ARM-android:      ldr     r4, .LCPI4_0
 ; ARM-android-NEXT: mov     r5, #0
diff --git a/llvm/test/CodeGen/DirectX/min_vec_size.ll b/llvm/test/CodeGen/DirectX/min_vec_size.ll
index 4f73f5ab8301f..ba6e83ea83733 100644
--- a/llvm/test/CodeGen/DirectX/min_vec_size.ll
+++ b/llvm/test/CodeGen/DirectX/min_vec_size.ll
@@ -6,7 +6,7 @@ target triple = "dxil-unknown-shadermodel6.7-library"
 ; Make sure vec combine min vec size is 1 instead of 4 for float.
 ; CHECK:@foo()
 ; CHECK-NEXT:%[[LD:[0-9]+]] = load <1 x float>, ptr @a, align 8
-; CHECK-NEXT:%insert = shufflevector <1 x float> %[[LD]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:%insert = shufflevector <1 x float> %[[LD]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:%shuffle = shufflevector <2 x float> %insert, <2 x float> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:ret <2 x float> %shuffle
 
diff --git a/llvm/test/CodeGen/Generic/denormal-fp-math-cl-opt.ll b/llvm/test/CodeGen/Generic/denormal-fp-math-cl-opt.ll
new file mode 100644
index 0000000000000..7e0922c7d0810
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/denormal-fp-math-cl-opt.ll
@@ -0,0 +1,9 @@
+; RUN: llc -denormal-fp-math=dynamic --denormal-fp-math-f32=preserve-sign -stop-after=finalize-isel < %s | FileCheck %s
+
+; Check that the command line flag annotates the IR with the
+; appropriate attributes.
+
+; CHECK: attributes #0 = { "denormal-fp-math"="dynamic,dynamic" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+define float @foo(float %var) {
+  ret float %var
+}
diff --git a/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll b/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
index 91c905574f65a..b8e473cbaa4c8 100644
--- a/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
+++ b/llvm/test/CodeGen/Generic/expand-experimental-reductions.ll
@@ -23,7 +23,7 @@ declare i8 @llvm.vector.reduce.and.i8.v3i8(<3 x i8>)
 define i64 @add_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @add_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <2 x i64> [[VEC]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[BIN_RDX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -36,7 +36,7 @@ entry:
 define i64 @mul_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @mul_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = mul <2 x i64> [[VEC]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[BIN_RDX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -49,7 +49,7 @@ entry:
 define i64 @and_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @and_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <2 x i64> [[VEC]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[BIN_RDX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -62,7 +62,7 @@ entry:
 define i64 @or_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @or_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <2 x i64> [[VEC]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[BIN_RDX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -75,7 +75,7 @@ entry:
 define i64 @xor_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @xor_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <2 x i64> [[VEC]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[BIN_RDX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -88,9 +88,9 @@ entry:
 define float @fadd_f32(<4 x float> %vec) {
 ; CHECK-LABEL: @fadd_f32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[VEC]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[BIN_RDX3:%.*]] = fadd fast float 0.000000e+00, [[TMP0]]
@@ -104,9 +104,9 @@ entry:
 define float @fadd_f32_accum(float %accum, <4 x float> %vec) {
 ; CHECK-LABEL: @fadd_f32_accum(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[VEC]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[BIN_RDX3:%.*]] = fadd fast float [[ACCUM:%.*]], [[TMP0]]
@@ -156,9 +156,9 @@ entry:
 define float @fmul_f32(<4 x float> %vec) {
 ; CHECK-LABEL: @fmul_f32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fmul fast <4 x float> [[VEC]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fmul fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[BIN_RDX3:%.*]] = fmul fast float 1.000000e+00, [[TMP0]]
@@ -172,9 +172,9 @@ entry:
 define float @fmul_f32_accum(float %accum, <4 x float> %vec) {
 ; CHECK-LABEL: @fmul_f32_accum(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[VEC:%.*]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fmul fast <4 x float> [[VEC]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fmul fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[BIN_RDX3:%.*]] = fmul fast float [[ACCUM:%.*]], [[TMP0]]
@@ -224,7 +224,7 @@ entry:
 define i64 @smax_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @smax_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> [[VEC]], <2 x i64> [[RDX_SHUF]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[RDX_MINMAX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -237,7 +237,7 @@ entry:
 define i64 @smin_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @smin_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> [[VEC]], <2 x i64> [[RDX_SHUF]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[RDX_MINMAX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -250,7 +250,7 @@ entry:
 define i64 @umax_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @umax_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> [[VEC]], <2 x i64> [[RDX_SHUF]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[RDX_MINMAX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
@@ -263,7 +263,7 @@ entry:
 define i64 @umin_i64(<2 x i64> %vec) {
 ; CHECK-LABEL: @umin_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x i64> [[VEC:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> [[VEC]], <2 x i64> [[RDX_SHUF]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i64> [[RDX_MINMAX]], i32 0
 ; CHECK-NEXT:    ret i64 [[TMP0]]
diff --git a/llvm/test/CodeGen/M68k/load-extend.ll b/llvm/test/CodeGen/M68k/load-extend.ll
new file mode 100644
index 0000000000000..51159730ecc0e
--- /dev/null
+++ b/llvm/test/CodeGen/M68k/load-extend.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -mtriple=m68k-linux -verify-machineinstrs | FileCheck %s
+
+@0 = external constant <{ [32 x i8] }>
+
+define i32 @"test_zext_pcd_i8_to_i32"() {
+; CHECK-LABEL: test_zext_pcd_i8_to_i32:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0:
+; CHECK-NEXT:    move.b (__unnamed_1+16,%pc), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    rts
+  %p = getelementptr inbounds i8, ptr @0, i32 16
+  %val = load i8, ptr %p
+  %val2 = zext i8 %val to i32
+  ret i32 %val2
+}
+
+define i16 @"test_zext_pcd_i8_to_i16"() {
+; CHECK-LABEL: test_zext_pcd_i8_to_i16:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0:
+; CHECK-NEXT:    move.b (__unnamed_1+16,%pc), %d0
+; CHECK-NEXT:    and.w #255, %d0
+; CHECK-NEXT:    rts
+  %p = getelementptr inbounds i8, ptr @0, i32 16
+  %val = load i8, ptr %p
+  %val2 = zext i8 %val to i16
+  ret i16 %val2
+}
+
+define i32 @"test_zext_pcd_i16_to_i32"() {
+; CHECK-LABEL: test_zext_pcd_i16_to_i32:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ; %bb.0:
+; CHECK-NEXT:    move.w (__unnamed_1+16,%pc), %d0
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    rts
+  %p = getelementptr inbounds i16, ptr @0, i32 8
+  %val = load i16, ptr %p
+  %val2 = zext i16 %val to i32
+  ret i32 %val2
+}
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/custom-pseudo-source-values.ll b/llvm/test/CodeGen/MIR/AMDGPU/custom-pseudo-source-values.ll
index e40536c08f983..34c39459c9327 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/custom-pseudo-source-values.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/custom-pseudo-source-values.ll
@@ -4,8 +4,8 @@
 ; Test that custom pseudo source values can be round trip serialized through MIR.
 
 ; CHECK-LABEL: {{^}}name: shader
-; CHECK: %[[#]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET killed %17, %18, 4, 0, 0, implicit $exec :: (dereferenceable load (s32) from unknown-address + 4, align 1, addrspace 7)
-; CHECK: IMAGE_STORE_V4_V3_nsa_gfx10 killed %[[#]], %[[#]], %[[#]], %[[#]], killed %[[#]], 15, 2, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 7)
+; CHECK: %[[#]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET killed %17, %18, 4, 0, 0, implicit $exec :: (dereferenceable load (s32) from unknown-address + 4, align 1, addrspace 8)
+; CHECK: IMAGE_STORE_V4_V3_nsa_gfx10 killed %[[#]], %[[#]], %[[#]], %[[#]], killed %[[#]], 15, 2, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128), addrspace 8)
 ; CHECK: DS_GWS_BARRIER %[[#]], 63, implicit $m0, implicit $exec :: (load (s32) from custom "GWSResource")
 define amdgpu_cs void @shader(i32 %arg0, i32 %arg1, <8 x i32> inreg %arg2, <4 x i32> inreg %arg3) {
   %bload0 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %arg3, i32 4, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll b/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll
index a743b146bf057..0a411f4ef4d5a 100644
--- a/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll
+++ b/llvm/test/CodeGen/NVPTX/call_bitcast_byval.ll
@@ -13,8 +13,9 @@ target triple = "nvptx64-nvidia-cuda"
 %"class.sycl::_V1::detail::half_impl::half" = type { half }
 %complex_half = type { half, half }
 
-; CHECK: .param .align 4 .b8 param2[4];
-; CHECK: st.param.v2.b16         [param2+0], {%h2, %h1};
+; CHECK: .param .align 2 .b8 param2[4];
+; CHECK: st.param.b16   [param2+0], %h1;
+; CHECK: st.param.b16   [param2+2], %h2;
 ; CHECK: .param .align 2 .b8 retval0[4];
 ; CHECK: call.uni (retval0),
 ; CHECK-NEXT: _Z20__spirv_GroupCMulKHRjjN5__spv12complex_halfE,
@@ -29,15 +30,16 @@ entry:
 ;;
 declare ptr @usefp(ptr %fp)
 ; CHECK: .func callee(
-; CHECK-NEXT: .param .align 4 .b8 callee_param_0[4]
+; CHECK-NEXT: .param .align 2 .b8 callee_param_0[4]
 define internal void @callee(ptr byval(%"class.complex") %byval_arg) {
   ret void
 }
 define void @boom() {
   %fp = call ptr @usefp(ptr @callee)
-  ; CHECK: .param .align 4 .b8 param0[4];
-  ; CHECK: st.param.v2.b16 [param0+0]
-  ; CHECK: .callprototype ()_ (.param .align 4 .b8 _[4]);
+  ; CHECK: .param .align 2 .b8 param0[4];
+  ; CHECK: st.param.b16 [param0+0], %h1;
+  ; CHECK: st.param.b16 [param0+2], %h2;
+  ; CHECK: .callprototype ()_ (.param .align 2 .b8 _[4]);
   call void %fp(ptr byval(%"class.complex") null)
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll b/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll
new file mode 100644
index 0000000000000..9dd3dfb2cf4cd
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/lower-ctor-dtor.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -mtriple=nvptx64-- -nvptx-lower-ctor-dtor < %s | FileCheck %s
+; RUN: opt -S -mtriple=nvptx64-- -passes=nvptx-lower-ctor-dtor < %s | FileCheck %s
+; RUN: opt -S -mtriple=nvptx64-- -passes=nvptx-lower-ctor-dtor \
+; RUN:     -nvptx-lower-global-ctor-dtor-id=unique_id < %s | FileCheck %s --check-prefix=GLOBAL
+
+; Make sure we get the same result if we run multiple times
+; RUN: opt -S -mtriple=nvptx64-- -passes=nvptx-lower-ctor-dtor,nvptx-lower-ctor-dtor < %s | FileCheck %s
+; RUN: llc -nvptx-lower-global-ctor-dtor -mtriple=nvptx64-amd-amdhsa -mcpu=sm_70 -filetype=asm -o - < %s | FileCheck %s -check-prefix=VISIBILITY
+
+@llvm.global_ctors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @foo, ptr null }]
+@llvm.global_dtors = appending addrspace(1) global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @bar, ptr null }]
+
+; CHECK-NOT: @llvm.global_ctors
+; CHECK-NOT: @llvm.global_dtors
+
+; CHECK: @__init_array_object_foo_[[HASH:[0-9a-f]+]]_1 = protected addrspace(4) constant ptr @foo, section ".init_array.1"
+; CHECK: @__fini_array_object_bar_[[HASH:[0-9a-f]+]]_1 = protected addrspace(4) constant ptr @bar, section ".fini_array.1"
+; CHECK: @llvm.used = appending global [2 x ptr] [ptr addrspacecast (ptr addrspace(4) @__init_array_object_foo_[[HASH]]_1 to ptr), ptr addrspacecast (ptr addrspace(4) @__fini_array_object_bar_[[HASH]]_1 to ptr)], section "llvm.metadata"
+; GLOBAL: @__init_array_object_foo_unique_id_1 = protected addrspace(4) constant ptr @foo, section ".init_array.1"
+; GLOBAL: @__fini_array_object_bar_unique_id_1 = protected addrspace(4) constant ptr @bar, section ".fini_array.1"
+; GLOBAL: @llvm.used = appending global [2 x ptr] [ptr addrspacecast (ptr addrspace(4) @__init_array_object_foo_unique_id_1 to ptr), ptr addrspacecast (ptr addrspace(4) @__fini_array_object_bar_unique_id_1 to ptr)], section "llvm.metadata"
+
+; VISIBILITY: .visible .const .align 8 .u64 __init_array_object_foo_[[HASH:[0-9a-f]+]]_1 = foo;
+; VISIBILITY: .visible .const .align 8 .u64 __fini_array_object_bar_[[HASH:[0-9a-f]+]]_1 = bar;
+
+define internal void @foo() {
+  ret void
+}
+
+define internal void @bar() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/param-align.ll b/llvm/test/CodeGen/NVPTX/param-align.ll
index f3d3003189142..5f5d77b22477b 100644
--- a/llvm/test/CodeGen/NVPTX/param-align.ll
+++ b/llvm/test/CodeGen/NVPTX/param-align.ll
@@ -1,5 +1,7 @@
-; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck %s
+; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck %s --check-prefixes=CHECK,NOALIGN4
+; RUN: llc < %s -march=nvptx -mcpu=sm_20 -nvptx-force-min-byval-param-align | FileCheck %s --check-prefixes=CHECK,ALIGN4
 ; RUN: %if ptxas %{ llc < %s -march=nvptx -mcpu=sm_20 | %ptxas-verify %}
+; RUN: %if ptxas %{ llc < %s -march=nvptx -mcpu=sm_20 -nvptx-force-min-byval-param-align | %ptxas-verify %}
 
 ;;; Need 4-byte alignment on ptr passed byval
 define ptx_device void @t1(ptr byval(float) %x) {
@@ -25,20 +27,21 @@ define ptx_device void @t3(ptr byval(%struct.float2) %x) {
   ret void
 }
 
-;;; Need at least 4-byte alignment in order to avoid miscompilation by
-;;; ptxas for sm_50+
 define ptx_device void @t4(ptr byval(i8) %x) {
 ; CHECK: .func t4
-; CHECK: .param .align 4 .b8 t4_param_0[1]
+; NOALIGN4: .param .align 1 .b8 t4_param_0[1]
+; ALIGN4: .param .align 4 .b8 t4_param_0[1]
   ret void
 }
 
 ;;; Make sure we adjust alignment at the call site as well.
 define ptx_device void @t5(ptr align 2 byval(i8) %x) {
 ; CHECK: .func t5
-; CHECK: .param .align 4 .b8 t5_param_0[1]
+; NOALIGN4: .param .align 2 .b8 t5_param_0[1]
+; ALIGN4: .param .align 4 .b8 t5_param_0[1]
 ; CHECK: {
-; CHECK: .param .align 4 .b8 param0[1];
+; NOALIGN4: .param .align 1 .b8 param0[1];
+; ALIGN4:   .param .align 4 .b8 param0[1];
 ; CHECK: call.uni
   call void @t4(ptr byval(i8) %x)
   ret void
@@ -56,11 +59,13 @@ define ptx_device void @t6() {
   call void %fp(ptr byval(double) null);
 
   %fp2 = call ptr @getfp(i32 1)
-; CHECK: prototype_4 : .callprototype ()_ (.param .align 4 .b8 _[4]);
+; NOALIGN4: prototype_4 : .callprototype ()_ (.param .align 2 .b8 _[4]);
+; ALIGN4: prototype_4 : .callprototype ()_ (.param .align 4 .b8 _[4]);
   call void %fp(ptr byval(%struct.half2) null);
 
   %fp3 = call ptr @getfp(i32 2)
-; CHECK: prototype_6 : .callprototype ()_ (.param .align 4 .b8 _[1]);
+; NOALIGN4: prototype_6 : .callprototype ()_ (.param .align 1 .b8 _[1]);
+; ALIGN4: prototype_6 : .callprototype ()_ (.param .align 4 .b8 _[1]);
   call void %fp(ptr byval(i8) null);
   ret void
 }
diff --git a/llvm/test/CodeGen/PowerPC/arg_promotion.ll b/llvm/test/CodeGen/PowerPC/arg_promotion.ll
index 83d720b80aa96..2aaf7c70b97f1 100644
--- a/llvm/test/CodeGen/PowerPC/arg_promotion.ll
+++ b/llvm/test/CodeGen/PowerPC/arg_promotion.ll
@@ -90,9 +90,9 @@ declare noundef signext i32 @printf(ptr nocapture noundef readonly, ...) nounwin
 define dso_local void @test1(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[B:%.*]], <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[VECINIT22:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    tail call fastcc void @printWideVec(<16 x i32> [[VECINIT22]])
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll
index 1cf70192d6dc7..adbb956ba32ad 100644
--- a/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/PowerPC/atomicrmw-uinc-udec-wrap.ll
@@ -126,17 +126,17 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    li 6, 0
-; CHECK-NEXT:    lwz 7, 0(3)
+; CHECK-NEXT:    lwz 5, 0(3)
 ; CHECK-NEXT:    b .LBB2_2
 ; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    cmplw 5, 7
-; CHECK-NEXT:    mr 7, 5
 ; CHECK-NEXT:    beq 0, .LBB2_7
 ; CHECK-NEXT:  .LBB2_2: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB2_5 Depth 2
-; CHECK-NEXT:    addi 5, 7, 1
+; CHECK-NEXT:    mr 7, 5
+; CHECK-NEXT:    addi 5, 5, 1
 ; CHECK-NEXT:    cmplw 7, 4
 ; CHECK-NEXT:    bc 12, 0, .LBB2_4
 ; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
@@ -169,18 +169,18 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK-LABEL: atomicrmw_uinc_wrap_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    ld 7, 0(3)
+; CHECK-NEXT:    ld 5, 0(3)
 ; CHECK-NEXT:    li 6, 0
 ; CHECK-NEXT:    b .LBB3_2
 ; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    cmpld 5, 7
-; CHECK-NEXT:    mr 7, 5
 ; CHECK-NEXT:    beq 0, .LBB3_7
 ; CHECK-NEXT:  .LBB3_2: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB3_5 Depth 2
-; CHECK-NEXT:    addi 5, 7, 1
+; CHECK-NEXT:    mr 7, 5
+; CHECK-NEXT:    addi 5, 5, 1
 ; CHECK-NEXT:    cmpld 7, 4
 ; CHECK-NEXT:    bc 12, 0, .LBB3_4
 ; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
@@ -334,19 +334,19 @@ define i32 @atomicrmw_udec_wrap_i32(ptr %ptr, i32 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    lwz 6, 0(3)
+; CHECK-NEXT:    lwz 5, 0(3)
 ; CHECK-NEXT:    b .LBB6_2
 ; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    cmplw 5, 6
-; CHECK-NEXT:    mr 6, 5
 ; CHECK-NEXT:    beq 0, .LBB6_7
 ; CHECK-NEXT:  .LBB6_2: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB6_5 Depth 2
+; CHECK-NEXT:    mr 6, 5
 ; CHECK-NEXT:    cmpwi 6, 0
 ; CHECK-NEXT:    cmplw 1, 6, 4
-; CHECK-NEXT:    addi 5, 6, -1
+; CHECK-NEXT:    addi 5, 5, -1
 ; CHECK-NEXT:    cror 20, 2, 5
 ; CHECK-NEXT:    bc 12, 20, .LBB6_4
 ; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
@@ -379,19 +379,18 @@ define i64 @atomicrmw_udec_wrap_i64(ptr %ptr, i64 %val) {
 ; CHECK-LABEL: atomicrmw_udec_wrap_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sync
-; CHECK-NEXT:    ld 6, 0(3)
+; CHECK-NEXT:    ld 5, 0(3)
 ; CHECK-NEXT:    b .LBB7_2
 ; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    cmpld 5, 6
-; CHECK-NEXT:    mr 6, 5
 ; CHECK-NEXT:    beq 0, .LBB7_7
 ; CHECK-NEXT:  .LBB7_2: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB7_5 Depth 2
-; CHECK-NEXT:    cmpdi 6, 0
+; CHECK-NEXT:    mr. 6, 5
 ; CHECK-NEXT:    cmpld 1, 6, 4
-; CHECK-NEXT:    addi 5, 6, -1
+; CHECK-NEXT:    addi 5, 5, -1
 ; CHECK-NEXT:    cror 20, 2, 5
 ; CHECK-NEXT:    bc 12, 20, .LBB7_4
 ; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
diff --git a/llvm/test/CodeGen/PowerPC/branch-on-store-cond.ll b/llvm/test/CodeGen/PowerPC/branch-on-store-cond.ll
index 2d9a9ab477880..811ed1b1d6054 100644
--- a/llvm/test/CodeGen/PowerPC/branch-on-store-cond.ll
+++ b/llvm/test/CodeGen/PowerPC/branch-on-store-cond.ll
@@ -7,38 +7,38 @@
 define dso_local zeroext i8 @test1(ptr noundef %addr, i8 noundef zeroext %newval) local_unnamed_addr #0 {
 ; CHECK-LABEL: test1:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stbcx. 4, 0, 3
+; CHECK-NEXT:    bne 0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    stdu 1, -32(1)
 ; CHECK-NEXT:    std 0, 48(1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    stbcx. 4, 0, 3
-; CHECK-NEXT:    bne 0, .LBB0_2
-; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    bl dummy
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:  .LBB0_2: # %if.end
-; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    addi 1, 1, 32
 ; CHECK-NEXT:    ld 0, 16(1)
 ; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:  .LBB0_2: # %if.end
+; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-AIX-LABEL: test1:
 ; CHECK-AIX:       # %bb.0: # %entry
-; CHECK-AIX-NEXT:    mflr 0
-; CHECK-AIX-NEXT:    stdu 1, -112(1)
-; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    stbcx. 4, 0, 3
 ; CHECK-AIX-NEXT:    bne 0, L..BB0_2
 ; CHECK-AIX-NEXT:  # %bb.1: # %if.then
+; CHECK-AIX-NEXT:    mflr 0
+; CHECK-AIX-NEXT:    stdu 1, -112(1)
+; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    bl .dummy[PR]
 ; CHECK-AIX-NEXT:    nop
-; CHECK-AIX-NEXT:  L..BB0_2: # %if.end
-; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    addi 1, 1, 112
 ; CHECK-AIX-NEXT:    ld 0, 16(1)
 ; CHECK-AIX-NEXT:    mtlr 0
+; CHECK-AIX-NEXT:  L..BB0_2: # %if.end
+; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    blr
 entry:
   %conv = zext i8 %newval to i32
@@ -57,38 +57,38 @@ if.end:                                           ; preds = %if.then, %entry
 define dso_local signext i16 @test2(ptr noundef %addr, i16 noundef signext %newval) local_unnamed_addr #0 {
 ; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sthcx. 4, 0, 3
+; CHECK-NEXT:    bne 0, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    stdu 1, -32(1)
 ; CHECK-NEXT:    std 0, 48(1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    sthcx. 4, 0, 3
-; CHECK-NEXT:    bne 0, .LBB1_2
-; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    bl dummy
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:  .LBB1_2: # %if.end
-; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    addi 1, 1, 32
 ; CHECK-NEXT:    ld 0, 16(1)
 ; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:  .LBB1_2: # %if.end
+; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-AIX-LABEL: test2:
 ; CHECK-AIX:       # %bb.0: # %entry
-; CHECK-AIX-NEXT:    mflr 0
-; CHECK-AIX-NEXT:    stdu 1, -112(1)
-; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    sthcx. 4, 0, 3
 ; CHECK-AIX-NEXT:    bne 0, L..BB1_2
 ; CHECK-AIX-NEXT:  # %bb.1: # %if.then
+; CHECK-AIX-NEXT:    mflr 0
+; CHECK-AIX-NEXT:    stdu 1, -112(1)
+; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    bl .dummy[PR]
 ; CHECK-AIX-NEXT:    nop
-; CHECK-AIX-NEXT:  L..BB1_2: # %if.end
-; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    addi 1, 1, 112
 ; CHECK-AIX-NEXT:    ld 0, 16(1)
 ; CHECK-AIX-NEXT:    mtlr 0
+; CHECK-AIX-NEXT:  L..BB1_2: # %if.end
+; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    blr
 entry:
   %0 = sext i16 %newval to i32
@@ -107,38 +107,38 @@ if.end:                                           ; preds = %if.then, %entry
 define dso_local signext i32 @test3(ptr noundef %addr, i32 noundef signext %newval) local_unnamed_addr #0 {
 ; CHECK-LABEL: test3:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stwcx. 4, 0, 3
+; CHECK-NEXT:    bne 0, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    stdu 1, -32(1)
 ; CHECK-NEXT:    std 0, 48(1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    stwcx. 4, 0, 3
-; CHECK-NEXT:    bne 0, .LBB2_2
-; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    bl dummy
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:  .LBB2_2: # %if.end
-; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    addi 1, 1, 32
 ; CHECK-NEXT:    ld 0, 16(1)
 ; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:  .LBB2_2: # %if.end
+; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-AIX-LABEL: test3:
 ; CHECK-AIX:       # %bb.0: # %entry
-; CHECK-AIX-NEXT:    mflr 0
-; CHECK-AIX-NEXT:    stdu 1, -112(1)
-; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    stwcx. 4, 0, 3
 ; CHECK-AIX-NEXT:    bne 0, L..BB2_2
 ; CHECK-AIX-NEXT:  # %bb.1: # %if.then
+; CHECK-AIX-NEXT:    mflr 0
+; CHECK-AIX-NEXT:    stdu 1, -112(1)
+; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    bl .dummy[PR]
 ; CHECK-AIX-NEXT:    nop
-; CHECK-AIX-NEXT:  L..BB2_2: # %if.end
-; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    addi 1, 1, 112
 ; CHECK-AIX-NEXT:    ld 0, 16(1)
 ; CHECK-AIX-NEXT:    mtlr 0
+; CHECK-AIX-NEXT:  L..BB2_2: # %if.end
+; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    blr
 entry:
   %0 = tail call i32 @llvm.ppc.stwcx(ptr %addr, i32 %newval)
@@ -156,38 +156,38 @@ if.end:                                           ; preds = %if.then, %entry
 define dso_local i64 @test4(ptr noundef %addr, i64 noundef %newval) local_unnamed_addr #0 {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stdcx. 4, 0, 3
+; CHECK-NEXT:    bne 0, .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    mflr 0
 ; CHECK-NEXT:    stdu 1, -32(1)
 ; CHECK-NEXT:    std 0, 48(1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    stdcx. 4, 0, 3
-; CHECK-NEXT:    bne 0, .LBB3_2
-; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    bl dummy
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:  .LBB3_2: # %if.end
-; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    addi 1, 1, 32
 ; CHECK-NEXT:    ld 0, 16(1)
 ; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:  .LBB3_2: # %if.end
+; CHECK-NEXT:    li 3, 55
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-AIX-LABEL: test4:
 ; CHECK-AIX:       # %bb.0: # %entry
-; CHECK-AIX-NEXT:    mflr 0
-; CHECK-AIX-NEXT:    stdu 1, -112(1)
-; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    stdcx. 4, 0, 3
 ; CHECK-AIX-NEXT:    bne 0, L..BB3_2
 ; CHECK-AIX-NEXT:  # %bb.1: # %if.then
+; CHECK-AIX-NEXT:    mflr 0
+; CHECK-AIX-NEXT:    stdu 1, -112(1)
+; CHECK-AIX-NEXT:    std 0, 128(1)
 ; CHECK-AIX-NEXT:    bl .dummy[PR]
 ; CHECK-AIX-NEXT:    nop
-; CHECK-AIX-NEXT:  L..BB3_2: # %if.end
-; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    addi 1, 1, 112
 ; CHECK-AIX-NEXT:    ld 0, 16(1)
 ; CHECK-AIX-NEXT:    mtlr 0
+; CHECK-AIX-NEXT:  L..BB3_2: # %if.end
+; CHECK-AIX-NEXT:    li 3, 55
 ; CHECK-AIX-NEXT:    blr
 entry:
   %0 = tail call i32 @llvm.ppc.stdcx(ptr %addr, i64 %newval)
diff --git a/llvm/test/CodeGen/PowerPC/cmp_elimination.ll b/llvm/test/CodeGen/PowerPC/cmp_elimination.ll
index 56af49f0c267e..871cc5df1f5fb 100644
--- a/llvm/test/CodeGen/PowerPC/cmp_elimination.ll
+++ b/llvm/test/CodeGen/PowerPC/cmp_elimination.ll
@@ -779,6 +779,29 @@ if.end3:
   ret void
 }
 
+;; The result of %cmp may change in a tail call. Don't lift %cmp to the entry block.
+; CHECK-LABEL: func_tailrecurse:
+; CHECK-NOT:     cmp
+; CHECK:       .LBB{{.*}}:
+; CHECK:         cmplw
+; CHECK:         blt
+define fastcc zeroext i32 @func_tailrecurse(i32 zeroext %a, i32 zeroext %b) {
+entry:
+  br label %tailrecurse
+
+tailrecurse:                                      ; preds = %tailrecurse, %entry
+  %a.tr = phi i32 [ %a, %entry ], [ %b.tr, %tailrecurse ]
+  %b.tr = phi i32 [ %b, %entry ], [ %a.tr, %tailrecurse ]
+  %cmp = icmp ult i32 %a.tr, %b.tr
+  %conv = zext i1 %cmp to i32
+  %ignore = call signext i32 (i32) @func(i32 %conv)
+  br i1 %cmp, label %tailrecurse, label %if.end
+
+if.end:                                           ; preds = %tailrecurse
+  %sub = sub nsw i32 %a.tr, %b.tr
+  ret i32 %sub
+}
+
 declare void @dummy1()
 declare void @dummy2()
 declare void @dummy3()
diff --git a/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll b/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll
index 94beed98d2457..5d2232319c1f5 100644
--- a/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll
+++ b/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll
@@ -10,6 +10,14 @@ target triple = "powerpc64le-unknown-linux-gnu"
 define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %.vy02, ptr %.vy03, ptr %.vy04, ptr %.vy05, ptr %.vy06, ptr %.vy07, ptr %.vy08, ptr %.vy09, ptr %.vy0a, ptr %.vy0b, ptr %.vy0c, ptr %.vy21, ptr %.vy22, ptr %.vy23, ptr %.vy24, ptr %.vy25, ptr %.vy26, ptr %.vy27, ptr %.vy28, ptr %.vy29, ptr %.vy2a, ptr %.vy2b, ptr %.vy2c) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz 4, 0(4)
+; CHECK-NEXT:    cmpwi 4, 1
+; CHECK-NEXT:    bltlr 0
+; CHECK-NEXT:  # %bb.1: # %_loop_1_do_.lr.ph
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    cmpwi 3, 1
+; CHECK-NEXT:    bltlr 0
+; CHECK-NEXT:  # %bb.2: # %_loop_1_do_.preheader
 ; CHECK-NEXT:    stdu 1, -592(1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 592
 ; CHECK-NEXT:    .cfi_offset r14, -192
@@ -48,34 +56,92 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %.
 ; CHECK-NEXT:    .cfi_offset v29, -240
 ; CHECK-NEXT:    .cfi_offset v30, -224
 ; CHECK-NEXT:    .cfi_offset v31, -208
-; CHECK-NEXT:    lwz 4, 0(4)
-; CHECK-NEXT:    std 14, 400(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 15, 408(1) # 8-byte Folded Spill
-; CHECK-NEXT:    cmpwi 4, 1
-; CHECK-NEXT:    std 16, 416(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 17, 424(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 22, 464(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 23, 472(1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr 22, 5
+; CHECK-NEXT:    ld 5, 848(1)
+; CHECK-NEXT:    mr 11, 7
+; CHECK-NEXT:    ld 23, 688(1)
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    ld 2, 760(1)
+; CHECK-NEXT:    std 28, 512(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 29, 520(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 28, 824(1)
+; CHECK-NEXT:    ld 7, 728(1)
 ; CHECK-NEXT:    std 18, 432(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 19, 440(1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr 18, 6
+; CHECK-NEXT:    ld 6, 712(1)
+; CHECK-NEXT:    cmpldi 3, 9
+; CHECK-NEXT:    ld 19, 768(1)
+; CHECK-NEXT:    std 10, 64(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 6, 72(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 5, 200(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 5, 840(1)
+; CHECK-NEXT:    lxv 33, 0(6)
+; CHECK-NEXT:    std 14, 400(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 15, 408(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 30, 528(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 31, 536(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 20, 448(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 21, 456(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 22, 464(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 23, 472(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 21, 784(1)
+; CHECK-NEXT:    ld 20, 776(1)
+; CHECK-NEXT:    lxv 10, 0(19)
+; CHECK-NEXT:    lxv 7, 0(21)
+; CHECK-NEXT:    ld 15, 736(1)
+; CHECK-NEXT:    ld 29, 704(1)
+; CHECK-NEXT:    ld 30, 720(1)
+; CHECK-NEXT:    std 2, 112(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 19, 120(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 42, 0(9)
+; CHECK-NEXT:    lxv 37, 0(7)
+; CHECK-NEXT:    std 20, 128(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 21, 136(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 43, 0(8)
+; CHECK-NEXT:    lxv 41, 0(10)
+; CHECK-NEXT:    std 16, 416(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 17, 424(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 17, 752(1)
+; CHECK-NEXT:    ld 16, 744(1)
 ; CHECK-NEXT:    std 24, 480(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 25, 488(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 25, 800(1)
+; CHECK-NEXT:    ld 24, 792(1)
 ; CHECK-NEXT:    std 26, 496(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 27, 504(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 28, 512(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 29, 520(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 30, 528(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 31, 536(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 27, 816(1)
+; CHECK-NEXT:    ld 26, 808(1)
+; CHECK-NEXT:    std 8, 48(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 9, 56(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 40, 0(23)
+; CHECK-NEXT:    lxv 38, 0(29)
+; CHECK-NEXT:    std 7, 80(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 15, 88(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 32, 0(30)
+; CHECK-NEXT:    lxv 36, 0(15)
+; CHECK-NEXT:    mr 8, 29
+; CHECK-NEXT:    mr 10, 30
+; CHECK-NEXT:    std 26, 160(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 27, 168(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 13, 0(16)
+; CHECK-NEXT:    lxv 12, 0(17)
 ; CHECK-NEXT:    stfd 26, 544(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd 27, 552(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 11, 0(2)
+; CHECK-NEXT:    lxv 9, 0(20)
 ; CHECK-NEXT:    stfd 28, 560(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd 29, 568(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 5, 0(24)
+; CHECK-NEXT:    lxv 4, 0(25)
 ; CHECK-NEXT:    stfd 30, 576(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd 31, 584(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 2, 0(26)
+; CHECK-NEXT:    lxv 0, 0(27)
+; CHECK-NEXT:    li 27, 0
 ; CHECK-NEXT:    stxv 52, 208(1) # 16-byte Folded Spill
 ; CHECK-NEXT:    stxv 53, 224(1) # 16-byte Folded Spill
+; CHECK-NEXT:    lxv 1, 0(28)
 ; CHECK-NEXT:    stxv 54, 240(1) # 16-byte Folded Spill
 ; CHECK-NEXT:    stxv 55, 256(1) # 16-byte Folded Spill
 ; CHECK-NEXT:    stxv 56, 272(1) # 16-byte Folded Spill
@@ -86,58 +152,8 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %.
 ; CHECK-NEXT:    stxv 61, 352(1) # 16-byte Folded Spill
 ; CHECK-NEXT:    stxv 62, 368(1) # 16-byte Folded Spill
 ; CHECK-NEXT:    stxv 63, 384(1) # 16-byte Folded Spill
-; CHECK-NEXT:    blt 0, .LBB0_7
-; CHECK-NEXT:  # %bb.1: # %_loop_1_do_.lr.ph
-; CHECK-NEXT:    lwz 3, 0(3)
-; CHECK-NEXT:    cmpwi 3, 1
-; CHECK-NEXT:    blt 0, .LBB0_7
-; CHECK-NEXT:  # %bb.2: # %_loop_1_do_.preheader
-; CHECK-NEXT:    mr 22, 5
-; CHECK-NEXT:    ld 5, 848(1)
-; CHECK-NEXT:    ld 28, 824(1)
-; CHECK-NEXT:    mr 11, 7
-; CHECK-NEXT:    mr 18, 6
-; CHECK-NEXT:    addi 3, 3, 1
-; CHECK-NEXT:    std 8, 48(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 9, 56(1) # 8-byte Folded Spill
-; CHECK-NEXT:    ld 6, 712(1)
-; CHECK-NEXT:    ld 23, 688(1)
-; CHECK-NEXT:    std 10, 64(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 6, 72(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 5, 200(1) # 8-byte Folded Spill
-; CHECK-NEXT:    ld 5, 840(1)
-; CHECK-NEXT:    cmpldi 3, 9
-; CHECK-NEXT:    ld 19, 768(1)
-; CHECK-NEXT:    ld 2, 760(1)
-; CHECK-NEXT:    lxv 33, 0(6)
-; CHECK-NEXT:    ld 21, 784(1)
-; CHECK-NEXT:    ld 20, 776(1)
-; CHECK-NEXT:    std 2, 112(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 19, 120(1) # 8-byte Folded Spill
-; CHECK-NEXT:    lxv 10, 0(19)
-; CHECK-NEXT:    ld 7, 728(1)
-; CHECK-NEXT:    std 20, 128(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 21, 136(1) # 8-byte Folded Spill
-; CHECK-NEXT:    lxv 7, 0(21)
-; CHECK-NEXT:    ld 27, 816(1)
-; CHECK-NEXT:    std 27, 168(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 5, 192(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    ld 5, 832(1)
-; CHECK-NEXT:    ld 26, 808(1)
-; CHECK-NEXT:    ld 25, 800(1)
-; CHECK-NEXT:    ld 24, 792(1)
-; CHECK-NEXT:    ld 17, 752(1)
-; CHECK-NEXT:    ld 16, 744(1)
-; CHECK-NEXT:    ld 15, 736(1)
-; CHECK-NEXT:    ld 29, 704(1)
-; CHECK-NEXT:    ld 30, 720(1)
-; CHECK-NEXT:    lxv 42, 0(9)
-; CHECK-NEXT:    std 7, 80(1) # 8-byte Folded Spill
-; CHECK-NEXT:    std 15, 88(1) # 8-byte Folded Spill
-; CHECK-NEXT:    lxv 37, 0(7)
-; CHECK-NEXT:    lxv 43, 0(8)
-; CHECK-NEXT:    mr 8, 29
-; CHECK-NEXT:    std 26, 160(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 16, 96(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 17, 104(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 24, 144(1) # 8-byte Folded Spill
@@ -145,30 +161,14 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %.
 ; CHECK-NEXT:    std 5, 184(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 28, 176(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    ld 5, 696(1)
-; CHECK-NEXT:    lxv 41, 0(10)
-; CHECK-NEXT:    lxv 40, 0(23)
-; CHECK-NEXT:    lxv 38, 0(29)
-; CHECK-NEXT:    lxv 32, 0(30)
-; CHECK-NEXT:    lxv 36, 0(15)
-; CHECK-NEXT:    lxv 13, 0(16)
-; CHECK-NEXT:    lxv 12, 0(17)
-; CHECK-NEXT:    lxv 11, 0(2)
-; CHECK-NEXT:    lxv 9, 0(20)
-; CHECK-NEXT:    lxv 5, 0(24)
-; CHECK-NEXT:    lxv 4, 0(25)
-; CHECK-NEXT:    mr 10, 30
+; CHECK-NEXT:    li 28, 1
+; CHECK-NEXT:    ld 7, 184(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lxv 3, 0(7)
 ; CHECK-NEXT:    std 5, 32(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std 23, 40(1) # 8-byte Folded Spill
 ; CHECK-NEXT:    lwa 5, 0(11)
 ; CHECK-NEXT:    li 11, 9
-; CHECK-NEXT:    ld 7, 184(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lxv 2, 0(26)
-; CHECK-NEXT:    lxv 0, 0(27)
-; CHECK-NEXT:    lxv 1, 0(28)
-; CHECK-NEXT:    li 28, 1
-; CHECK-NEXT:    li 27, 0
 ; CHECK-NEXT:    ld 9, 32(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lxv 3, 0(7)
 ; CHECK-NEXT:    iselgt 3, 3, 11
 ; CHECK-NEXT:    addi 3, 3, -2
 ; CHECK-NEXT:    mulli 6, 5, 40
@@ -270,16 +270,52 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %.
 ; CHECK-NEXT:    ble 0, .LBB0_3
 ; CHECK-NEXT:  # %bb.6: # %_loop_1_loopHeader_._return_bb_crit_edge.loopexit
 ; CHECK-NEXT:    ld 3, 48(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lxv 63, 384(1) # 16-byte Folded Reload
 ; CHECK-NEXT:    stxv 43, 0(3)
 ; CHECK-NEXT:    ld 3, 56(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lxv 62, 368(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 61, 352(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 60, 336(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 59, 320(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 58, 304(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 57, 288(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 56, 272(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 55, 256(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 54, 240(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 53, 224(1) # 16-byte Folded Reload
+; CHECK-NEXT:    lxv 52, 208(1) # 16-byte Folded Reload
 ; CHECK-NEXT:    stxv 42, 0(3)
 ; CHECK-NEXT:    ld 3, 64(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 31, 584(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 30, 576(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 29, 568(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 28, 560(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 27, 552(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 26, 544(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 31, 536(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 30, 528(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 29, 520(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 28, 512(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 27, 504(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stxv 41, 0(3)
 ; CHECK-NEXT:    ld 3, 40(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 26, 496(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 25, 488(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 24, 480(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 23, 472(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 22, 464(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 21, 456(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 20, 448(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 19, 440(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 18, 432(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 17, 424(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 16, 416(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stxv 40, 0(3)
 ; CHECK-NEXT:    ld 3, 72(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stxv 39, 0(9)
 ; CHECK-NEXT:    stxv 38, 0(8)
+; CHECK-NEXT:    ld 15, 408(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 14, 400(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stxv 33, 0(3)
 ; CHECK-NEXT:    ld 3, 80(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stxv 32, 0(10)
@@ -314,43 +350,6 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %.
 ; CHECK-NEXT:    stxv 8, 0(3)
 ; CHECK-NEXT:    ld 3, 200(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    stxv 6, 0(3)
-; CHECK-NEXT:  .LBB0_7: # %_return_bb
-; CHECK-NEXT:    lxv 63, 384(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 62, 368(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 61, 352(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 60, 336(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 59, 320(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 58, 304(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 57, 288(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 56, 272(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 55, 256(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 54, 240(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 53, 224(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lxv 52, 208(1) # 16-byte Folded Reload
-; CHECK-NEXT:    lfd 31, 584(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lfd 30, 576(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lfd 29, 568(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lfd 28, 560(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lfd 27, 552(1) # 8-byte Folded Reload
-; CHECK-NEXT:    lfd 26, 544(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 31, 536(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 30, 528(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 29, 520(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 28, 512(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 27, 504(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 26, 496(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 25, 488(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 24, 480(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 23, 472(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 22, 464(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 21, 456(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 20, 448(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 19, 440(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 18, 432(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 17, 424(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 16, 416(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 15, 408(1) # 8-byte Folded Reload
-; CHECK-NEXT:    ld 14, 400(1) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi 1, 1, 592
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-sibcall-shrinkwrap.ll b/llvm/test/CodeGen/PowerPC/ppc64-sibcall-shrinkwrap.ll
index d67f6a029a0d7..1a3aaaec037ab 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-sibcall-shrinkwrap.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-sibcall-shrinkwrap.ll
@@ -1,10 +1,9 @@
 ; RUN: llc -relocation-model=static -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -disable-ppc-sco=false --enable-shrink-wrap=false | FileCheck %s -check-prefix=CHECK-SCO-ONLY
-; RUN: llc -relocation-model=static -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -disable-ppc-sco=false --enable-shrink-wrap=true | FileCheck %s -check-prefix=CHECK-SCO-ONLY
+; RUN: llc -relocation-model=static -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -disable-ppc-sco=false --enable-shrink-wrap=true | FileCheck %s -check-prefix=CHECK-SCO-SR
 ; RUN: llc -relocation-model=static -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu -disable-ppc-sco=false --enable-shrink-wrap=false | FileCheck %s -check-prefix=CHECK-SCO-ONLY
-; RUN: llc -relocation-model=static -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu -disable-ppc-sco=false --enable-shrink-wrap=true | FileCheck %s -check-prefix=CHECK-SCO-ONLY
+; RUN: llc -relocation-model=static -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu -disable-ppc-sco=false --enable-shrink-wrap=true | FileCheck %s -check-prefix=CHECK-SCO-SR
 ; RUN: not --crash llc -relocation-model=pic -verify-machineinstrs < %s -mtriple=powerpc64-ibm-aix-xcoff -tailcallopt -disable-ppc-sco=false --enable-shrink-wrap=true 2>&1 | FileCheck %s -check-prefix=CHECK-AIX
 ;; The above RUN command is expected to fail on AIX since tail calling is not implemented ATM
-;; Edit: D63152 prevents stack popping before loads and stores, so shrink-wrap does nothing here
 %"class.clang::NamedDecl" = type { i32 }
 declare void @__assert_fail();
 
@@ -31,6 +30,12 @@ exit:
 ; CHECK-SCO-ONLY: #TC_RETURNd8
 ; CHECK-SCO-ONLY: bl __assert_fail
 ;
+; CHECK-SCO-SR-LABEL: _ZNK5clang9NamedDecl23getLinkageAndVisibilityEv:
+; CHECK-SCO-SR: b LVComputationKind
+; CHECK-SCO-SR: #TC_RETURNd8
+; CHECK-SCO-SR: stdu 1, -{{[0-9]+}}(1)
+; CHECK-SCO-SR: bl __assert_fail
+
 ; CHECK-AIX: LLVM ERROR: Tail call support is unimplemented on AIX.
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/vperm-swap.ll b/llvm/test/CodeGen/PowerPC/vperm-swap.ll
new file mode 100644
index 0000000000000..6a2750faa61a8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vperm-swap.ll
@@ -0,0 +1,39 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-linux-gnu < %s | \
+; RUN: FileCheck %s --check-prefix=CHECK-LE-P8
+
+define <16 x i8> @test_none_v16i8(i8 %arg, ptr nocapture noundef readonly %b) {
+; CHECK-LE-P8: .LCPI0_0:
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   30                              # 0x1e
+; CHECK-LE-P8-NEXT: .byte   7                               # 0x7
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-NEXT: .byte   31                              # 0x1f
+; CHECK-LE-P8-LABEL: test_none_v16i8:
+; CHECK-LE-P8:       # %bb.0: # %entry
+; CHECK-LE-P8-NEXT:    addis r5, r2, .LCPI0_0@toc@ha
+; CHECK-LE-P8-NEXT:    lxvd2x vs0, 0, r4
+; CHECK-LE-P8-NEXT:    mtvsrd v4, r3
+; CHECK-LE-P8-NEXT:    addi r5, r5, .LCPI0_0@toc@l
+; CHECK-LE-P8-NEXT:    lxvd2x vs1, 0, r5
+; CHECK-LE-P8-NEXT:    xxswapd v2, vs0
+; CHECK-LE-P8-NEXT:    xxswapd v3, vs1
+; CHECK-LE-P8-NEXT:    vperm v2, v4, v2, v3
+; CHECK-LE-P8-NEXT:    blr
+entry:
+  %lhs = load <16 x i8>, ptr %b, align 4
+  %rhs = insertelement <16 x i8> undef, i8 %arg, i32 0
+  %shuffle = shufflevector <16 x i8> %lhs, <16 x i8> %rhs, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i8> %shuffle
+}
diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
index 779abf813894c..f18e6a34ecdd1 100644
--- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll
@@ -64,7 +64,6 @@
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Stack Frame Layout Analysis
 ; CHECK-NEXT:       RISC-V pseudo instruction expansion pass
-; CHECK-NEXT:       RISC-V insert NTLH instruction pass
 ; CHECK-NEXT:       RISC-V atomic pseudo instruction expansion pass
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 539e91fce18ae..9179c8c922f16 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -177,7 +177,6 @@
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Stack Frame Layout Analysis
 ; CHECK-NEXT:       RISC-V pseudo instruction expansion pass
-; CHECK-NEXT:       RISC-V insert NTLH instruction pass
 ; CHECK-NEXT:       RISC-V atomic pseudo instruction expansion pass
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
index 687bd304143ee..f23d2590bf95e 100644
--- a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
@@ -32,9 +32,8 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; RV32I-NEXT:    addi a0, a3, 1
 ; RV32I-NEXT:    andi a1, a3, 255
 ; RV32I-NEXT:    sltu a1, a1, s1
-; RV32I-NEXT:    xori a1, a1, 1
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a2, a1, a0
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    and a2, a2, a0
 ; RV32I-NEXT:    sb a3, 3(sp)
 ; RV32I-NEXT:    addi a1, sp, 3
 ; RV32I-NEXT:    li a3, 5
@@ -69,8 +68,7 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; RV32IA-NEXT:    andi a6, a4, 255
 ; RV32IA-NEXT:    addi a4, a4, 1
 ; RV32IA-NEXT:    sltu a6, a6, a1
-; RV32IA-NEXT:    xori a6, a6, 1
-; RV32IA-NEXT:    addi a6, a6, -1
+; RV32IA-NEXT:    neg a6, a6
 ; RV32IA-NEXT:    and a4, a6, a4
 ; RV32IA-NEXT:    andi a4, a4, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
@@ -108,9 +106,8 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; RV64I-NEXT:    addi a0, a3, 1
 ; RV64I-NEXT:    andi a1, a3, 255
 ; RV64I-NEXT:    sltu a1, a1, s1
-; RV64I-NEXT:    xori a1, a1, 1
-; RV64I-NEXT:    addi a1, a1, -1
-; RV64I-NEXT:    and a2, a1, a0
+; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    and a2, a2, a0
 ; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    li a3, 5
@@ -145,8 +142,7 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; RV64IA-NEXT:    andi a7, a5, 255
 ; RV64IA-NEXT:    addiw a5, a5, 1
 ; RV64IA-NEXT:    sltu a7, a7, a1
-; RV64IA-NEXT:    xori a7, a7, 1
-; RV64IA-NEXT:    addiw a7, a7, -1
+; RV64IA-NEXT:    negw a7, a7
 ; RV64IA-NEXT:    and a5, a7, a5
 ; RV64IA-NEXT:    andi a5, a5, 255
 ; RV64IA-NEXT:    sllw a5, a5, a0
@@ -192,9 +188,8 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; RV32I-NEXT:    and a0, a3, s1
 ; RV32I-NEXT:    addi a1, a3, 1
 ; RV32I-NEXT:    sltu a0, a0, s2
-; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a2, a0, a1
+; RV32I-NEXT:    neg a2, a0
+; RV32I-NEXT:    and a2, a2, a1
 ; RV32I-NEXT:    sh a3, 14(sp)
 ; RV32I-NEXT:    addi a1, sp, 14
 ; RV32I-NEXT:    li a3, 5
@@ -231,8 +226,7 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; RV32IA-NEXT:    and a7, a5, a3
 ; RV32IA-NEXT:    addi a5, a5, 1
 ; RV32IA-NEXT:    sltu a7, a7, a1
-; RV32IA-NEXT:    xori a7, a7, 1
-; RV32IA-NEXT:    addi a7, a7, -1
+; RV32IA-NEXT:    neg a7, a7
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    and a5, a7, a5
 ; RV32IA-NEXT:    sll a5, a5, a0
@@ -274,9 +268,8 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; RV64I-NEXT:    and a0, a3, s1
 ; RV64I-NEXT:    addi a1, a3, 1
 ; RV64I-NEXT:    sltu a0, a0, s2
-; RV64I-NEXT:    xori a0, a0, 1
-; RV64I-NEXT:    addi a0, a0, -1
-; RV64I-NEXT:    and a2, a0, a1
+; RV64I-NEXT:    neg a2, a0
+; RV64I-NEXT:    and a2, a2, a1
 ; RV64I-NEXT:    sh a3, 14(sp)
 ; RV64I-NEXT:    addi a1, sp, 14
 ; RV64I-NEXT:    li a3, 5
@@ -313,8 +306,7 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; RV64IA-NEXT:    and t0, a6, a3
 ; RV64IA-NEXT:    addiw a6, a6, 1
 ; RV64IA-NEXT:    sltu t0, t0, a1
-; RV64IA-NEXT:    xori t0, t0, 1
-; RV64IA-NEXT:    addiw t0, t0, -1
+; RV64IA-NEXT:    negw t0, t0
 ; RV64IA-NEXT:    and a6, a6, a3
 ; RV64IA-NEXT:    and a6, t0, a6
 ; RV64IA-NEXT:    sllw a6, a6, a0
@@ -355,9 +347,8 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    addi a0, a3, 1
 ; RV32I-NEXT:    sltu a1, a3, s1
-; RV32I-NEXT:    xori a1, a1, 1
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a2, a1, a0
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    and a2, a2, a0
 ; RV32I-NEXT:    sw a3, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
 ; RV32I-NEXT:    li a3, 5
@@ -383,8 +374,7 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; RV32IA-NEXT:    mv a3, a2
 ; RV32IA-NEXT:    addi a2, a2, 1
 ; RV32IA-NEXT:    sltu a4, a3, a1
-; RV32IA-NEXT:    xori a4, a4, 1
-; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    neg a4, a4
 ; RV32IA-NEXT:    and a4, a4, a2
 ; RV32IA-NEXT:  .LBB2_3: # %atomicrmw.start
 ; RV32IA-NEXT:    # Parent Loop BB2_1 Depth=1
@@ -417,9 +407,8 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64I-NEXT:    addiw a0, a3, 1
 ; RV64I-NEXT:    sltu a1, a3, s1
-; RV64I-NEXT:    xori a1, a1, 1
-; RV64I-NEXT:    addi a1, a1, -1
-; RV64I-NEXT:    and a2, a1, a0
+; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    and a2, a2, a0
 ; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    li a3, 5
@@ -446,8 +435,7 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; RV64IA-NEXT:    addiw a3, a2, 1
 ; RV64IA-NEXT:    sext.w a4, a2
 ; RV64IA-NEXT:    sltu a2, a4, a1
-; RV64IA-NEXT:    xori a2, a2, 1
-; RV64IA-NEXT:    addi a2, a2, -1
+; RV64IA-NEXT:    neg a2, a2
 ; RV64IA-NEXT:    and a3, a2, a3
 ; RV64IA-NEXT:  .LBB2_3: # %atomicrmw.start
 ; RV64IA-NEXT:    # Parent Loop BB2_1 Depth=1
@@ -596,9 +584,8 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64I-NEXT:    addi a0, a3, 1
 ; RV64I-NEXT:    sltu a1, a3, s1
-; RV64I-NEXT:    xori a1, a1, 1
-; RV64I-NEXT:    addi a1, a1, -1
-; RV64I-NEXT:    and a2, a1, a0
+; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    and a2, a2, a0
 ; RV64I-NEXT:    sd a3, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    li a3, 5
@@ -624,8 +611,7 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; RV64IA-NEXT:    mv a3, a2
 ; RV64IA-NEXT:    addi a2, a2, 1
 ; RV64IA-NEXT:    sltu a4, a3, a1
-; RV64IA-NEXT:    xori a4, a4, 1
-; RV64IA-NEXT:    addi a4, a4, -1
+; RV64IA-NEXT:    neg a4, a4
 ; RV64IA-NEXT:    and a4, a4, a2
 ; RV64IA-NEXT:  .LBB3_3: # %atomicrmw.start
 ; RV64IA-NEXT:    # Parent Loop BB3_1 Depth=1
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 5ddfad5eddb9b..ba4a126ef6889 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -52,8 +52,11 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zcb %s -o - | FileCheck --check-prefixes=CHECK,RV32ZCB %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zcd %s -o - | FileCheck --check-prefixes=CHECK,RV32ZCD %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zcf %s -o - | FileCheck --check-prefixes=CHECK,RV32ZCF %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zcmt %s -o - | FileCheck --check-prefixes=CHECK,RV32ZCMT %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicsr %s -o - | FileCheck --check-prefixes=CHECK,RV32ZICSR %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zifencei %s -o - | FileCheck --check-prefixes=CHECK,RV32ZIFENCEI %s
+; RUN: llc -mtriple=riscv32 -mattr=+zicntr %s -o - | FileCheck --check-prefixes=CHECK,RV32ZICNTR %s
+; RUN: llc -mtriple=riscv32 -mattr=+zihpm %s -o - | FileCheck --check-prefixes=CHECK,RV32ZIHPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfa %s -o - | FileCheck --check-prefixes=CHECK,RV32ZFA %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvbb %s -o - | FileCheck --check-prefix=RV32ZVBB %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zve64x -mattr=+experimental-zvbc %s -o - | FileCheck --check-prefix=RV32ZVBC %s
@@ -69,6 +72,8 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvksh %s -o - | FileCheck --check-prefix=RV32ZVKSH %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvkt %s -o - | FileCheck --check-prefix=RV32ZVKT %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicond %s -o - | FileCheck --check-prefix=RV32ZICOND %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-smaia %s -o - | FileCheck --check-prefixes=CHECK,RV32SMAIA %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssaia %s -o - | FileCheck --check-prefixes=CHECK,RV32SSAIA %s
 
 ; RUN: llc -mtriple=riscv64 %s -o - | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+m %s -o - | FileCheck --check-prefixes=CHECK,RV64M %s
@@ -128,8 +133,11 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zca %s -o - | FileCheck --check-prefixes=CHECK,RV64ZCA %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zcb %s -o - | FileCheck --check-prefixes=CHECK,RV64ZCB %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zcd %s -o - | FileCheck --check-prefixes=CHECK,RV64ZCD %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zcmt %s -o - | FileCheck --check-prefixes=CHECK,RV64ZCMT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicsr %s -o - | FileCheck --check-prefixes=CHECK,RV64ZICSR %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zifencei %s -o - | FileCheck --check-prefixes=CHECK,RV64ZIFENCEI %s
+; RUN: llc -mtriple=riscv64 -mattr=+zicntr %s -o - | FileCheck --check-prefixes=CHECK,RV64ZICNTR %s
+; RUN: llc -mtriple=riscv64 -mattr=+zihpm %s -o - | FileCheck --check-prefixes=CHECK,RV64ZIHPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfa %s -o - | FileCheck --check-prefixes=CHECK,RV64ZFA %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvbb %s -o - | FileCheck --check-prefix=RV64ZVBB %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zve64x -mattr=+experimental-zvbc %s -o - | FileCheck --check-prefix=RV64ZVBC %s
@@ -145,6 +153,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvksh %s -o - | FileCheck --check-prefix=RV64ZVKSH %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvkt %s -o - | FileCheck --check-prefix=RV64ZVKT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicond %s -o - | FileCheck --check-prefix=RV64ZICOND %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-smaia %s -o - | FileCheck --check-prefixes=CHECK,RV64SMAIA %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-ssaia %s -o - | FileCheck --check-prefixes=CHECK,RV64SSAIA %s
 
 ; CHECK: .attribute 4, 16
 
@@ -199,8 +209,11 @@
 ; RV32ZCB: .attribute 5, "rv32i2p1_zca1p0_zcb1p0"
 ; RV32ZCD: .attribute 5, "rv32i2p1_zcd1p0"
 ; RV32ZCF: .attribute 5, "rv32i2p1_zcf1p0"
+; RV32ZCMT: .attribute 5, "rv32i2p1_zicsr2p0_zca1p0_zcmt1p0"
 ; RV32ZICSR: .attribute 5, "rv32i2p1_zicsr2p0"
 ; RV32ZIFENCEI: .attribute 5, "rv32i2p1_zifencei2p0"
+; RV32ZICNTR: .attribute 5, "rv32i2p1_zicntr1p0_zicsr2p0"
+; RV32ZIHPM: .attribute 5, "rv32i2p1_zicsr2p0_zihpm1p0"
 ; RV32ZFA: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfa0p2"
 ; RV32ZVBB: .attribute 5, "rv32i2p1_zicsr2p0_zvbb0p5_zve32x1p0_zvl32b1p0"
 ; RV32ZVBC: .attribute 5, "rv32i2p1_zicsr2p0_zvbc0p5_zve32x1p0_zve64x1p0_zvl32b1p0_zvl64b1p0"
@@ -216,6 +229,8 @@
 ; RV32ZVKSH: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvksh0p5_zvl32b1p0"
 ; RV32ZVKT: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvkt0p5_zvl32b1p0"
 ; RV32ZICOND: .attribute 5, "rv32i2p1_zicond1p0"
+; RV32SMAIA: .attribute 5, "rv32i2p1_smaia1p0"
+; RV32SSAIA: .attribute 5, "rv32i2p1_ssaia1p0"
 
 ; RV64M: .attribute 5, "rv64i2p1_m2p0"
 ; RV64ZMMUL: .attribute 5, "rv64i2p1_zmmul1p0"
@@ -274,8 +289,11 @@
 ; RV64ZCA: .attribute 5, "rv64i2p1_zca1p0"
 ; RV64ZCB: .attribute 5, "rv64i2p1_zca1p0_zcb1p0"
 ; RV64ZCD: .attribute 5, "rv64i2p1_zcd1p0"
+; RV64ZCMT: .attribute 5, "rv64i2p1_zicsr2p0_zca1p0_zcmt1p0"
 ; RV64ZICSR: .attribute 5, "rv64i2p1_zicsr2p0"
 ; RV64ZIFENCEI: .attribute 5, "rv64i2p1_zifencei2p0"
+; RV64ZICNTR: .attribute 5, "rv64i2p1_zicntr1p0_zicsr2p0"
+; RV64ZIHPM: .attribute 5, "rv64i2p1_zicsr2p0_zihpm1p0"
 ; RV64ZFA: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfa0p2"
 ; RV64ZVBB: .attribute 5, "rv64i2p1_zicsr2p0_zvbb0p5_zve32x1p0_zvl32b1p0"
 ; RV64ZVBC: .attribute 5, "rv64i2p1_zicsr2p0_zvbc0p5_zve32x1p0_zve64x1p0_zvl32b1p0_zvl64b1p0"
@@ -291,6 +309,8 @@
 ; RV64ZVKSH: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvksh0p5_zvl32b1p0"
 ; RV64ZVKT: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvkt0p5_zvl32b1p0"
 ; RV64ZICOND: .attribute 5, "rv64i2p1_zicond1p0"
+; RV64SMAIA: .attribute 5, "rv64i2p1_smaia1p0"
+; RV64SSAIA: .attribute 5, "rv64i2p1_ssaia1p0"
 
 define i32 @addi(i32 %a) {
   %1 = add i32 %a, 1
diff --git a/llvm/test/CodeGen/RISCV/double-arith-strict.ll b/llvm/test/CodeGen/RISCV/double-arith-strict.ll
index aa278b7513e4b..2e84e24470f69 100644
--- a/llvm/test/CodeGen/RISCV/double-arith-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-arith-strict.ll
@@ -5,6 +5,9 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -16,6 +19,11 @@ define double @fadd_d(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    fadd.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fadd_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fadd_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -44,6 +52,11 @@ define double @fsub_d(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    fsub.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fsub_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsub.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsub_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -72,6 +85,11 @@ define double @fmul_d(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    fmul.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmul_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmul.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmul_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -100,6 +118,11 @@ define double @fdiv_d(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    fdiv.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fdiv_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fdiv.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fdiv_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -128,6 +151,11 @@ define double @fsqrt_d(double %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fsqrt.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fsqrt_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsqrt.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsqrt_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -169,6 +197,15 @@ define double @fmin_d(double %a, double %b) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmin_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call fmin@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmin_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -210,6 +247,15 @@ define double @fmax_d(double %a, double %b) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmax_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call fmax@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmax_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -238,6 +284,11 @@ define double @fmadd_d(double %a, double %b, double %c) nounwind strictfp {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmadd_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmadd_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -275,6 +326,12 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind strictfp {
 ; RV64IFD-NEXT:    fmsub.d fa0, fa0, fa1, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmsub_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fmsub.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmsub_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -353,6 +410,13 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind strictfp {
 ; RV64IFD-NEXT:    fnmadd.d fa0, fa4, fa1, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fnmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -449,6 +513,13 @@ define double @fnmadd_d_2(double %a, double %b, double %c) nounwind strictfp {
 ; RV64IFD-NEXT:    fnmadd.d fa0, fa4, fa0, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_d_2:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fnmadd.d a0, a1, a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_d_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -544,6 +615,12 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind strictfp {
 ; RV64IFD-NEXT:    fnmsub.d fa0, fa5, fa1, fa2
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmsub_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fnmsub.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -616,6 +693,12 @@ define double @fnmsub_d_2(double %a, double %b, double %c) nounwind strictfp {
 ; RV64IFD-NEXT:    fnmsub.d fa0, fa5, fa0, fa2
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmsub_d_2:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, zero
+; RV64IZFINXZDINX-NEXT:    fnmsub.d a0, a1, a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_d_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
diff --git a/llvm/test/CodeGen/RISCV/double-arith.ll b/llvm/test/CodeGen/RISCV/double-arith.ll
index 708271c0f43f6..1ba56c80d8083 100644
--- a/llvm/test/CodeGen/RISCV/double-arith.ll
+++ b/llvm/test/CodeGen/RISCV/double-arith.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -19,6 +21,11 @@ define double @fadd_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fadd.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fadd_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fadd_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -46,6 +53,11 @@ define double @fsub_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fsub.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fsub_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsub.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsub_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -73,6 +85,11 @@ define double @fmul_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fmul.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmul_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmul.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmul_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -100,6 +117,11 @@ define double @fdiv_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fdiv.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fdiv_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fdiv.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fdiv_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -129,6 +151,11 @@ define double @fsqrt_d(double %a) nounwind {
 ; CHECKIFD-NEXT:    fsqrt.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fsqrt_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsqrt.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsqrt_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -158,6 +185,11 @@ define double @fsgnj_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fsgnj.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fsgnj_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsgnj_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
@@ -189,6 +221,13 @@ define i32 @fneg_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    feq.d a0, fa5, fa4
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fneg_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    fneg.d a1, a0
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fneg_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -235,6 +274,14 @@ define double @fsgnjn_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fsgnjn.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fsgnjn_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    li a2, -1
+; RV64IZFINXZDINX-NEXT:    slli a2, a2, 63
+; RV64IZFINXZDINX-NEXT:    xor a1, a1, a2
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsgnjn_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    not a2, a3
@@ -271,6 +318,13 @@ define double @fabs_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fadd.d fa0, fa4, fa5
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fabs_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    fabs.d a1, a0
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fabs_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -311,6 +365,11 @@ define double @fmin_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fmin.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmin_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmin.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmin_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -340,6 +399,11 @@ define double @fmax_d(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fmax.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmax_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmax.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmax_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -369,6 +433,11 @@ define double @fmadd_d(double %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmadd_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmadd_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -405,6 +474,12 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fmsub.d fa0, fa0, fa1, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmsub_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fmsub.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmsub_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -483,6 +558,13 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fnmadd.d fa0, fa4, fa1, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fnmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -579,6 +661,13 @@ define double @fnmadd_d_2(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fnmadd.d fa0, fa4, fa0, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_d_2:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fnmadd.d a0, a1, a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_d_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -666,6 +755,14 @@ define double @fnmadd_d_3(double %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fneg.d fa0, fa5
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_d_3:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    li a1, -1
+; RV64IZFINXZDINX-NEXT:    slli a1, a1, 63
+; RV64IZFINXZDINX-NEXT:    xor a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_d_3:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -700,6 +797,14 @@ define double @fnmadd_nsz(double %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fnmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_nsz:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    li a1, -1
+; RV64IZFINXZDINX-NEXT:    slli a1, a1, 63
+; RV64IZFINXZDINX-NEXT:    xor a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_nsz:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -742,6 +847,12 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fnmsub.d fa0, fa5, fa1, fa2
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmsub_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fnmsub.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -814,6 +925,12 @@ define double @fnmsub_d_2(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fnmsub.d fa0, fa5, fa0, fa2
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmsub_d_2:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, zero
+; RV64IZFINXZDINX-NEXT:    fnmsub.d a0, a1, a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_d_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -881,6 +998,11 @@ define double @fmadd_d_contract(double %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmadd_d_contract:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmadd_d_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -932,6 +1054,12 @@ define double @fmsub_d_contract(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fmsub.d fa0, fa0, fa1, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmsub_d_contract:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fmsub.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmsub_d_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1020,6 +1148,14 @@ define double @fnmadd_d_contract(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fnmadd.d fa0, fa4, fa3, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmadd_d_contract:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a2, a2, zero
+; RV64IZFINXZDINX-NEXT:    fnmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_d_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1133,6 +1269,13 @@ define double @fnmsub_d_contract(double %a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fnmsub.d fa0, fa4, fa5, fa2
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fnmsub_d_contract:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, zero
+; RV64IZFINXZDINX-NEXT:    fnmsub.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_d_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
diff --git a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
index 5bf27f5dded30..b294f88df84f4 100644
--- a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
@@ -7,6 +7,8 @@
 ; RUN:   | FileCheck -check-prefix=RV64I %s
 ; RUN: llc -mtriple=riscv64 -target-abi=lp64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64IFD %s
+; RUN: llc -mtriple=riscv64 -target-abi=lp64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ;
 ; This file tests cases where simple floating point operations can be
 ; profitably handled though bit manipulation if a soft-float ABI is being used
@@ -43,6 +45,13 @@ define double @fneg(double %a) nounwind {
 ; RV64IFD-NEXT:    slli a1, a1, 63
 ; RV64IFD-NEXT:    xor a0, a0, a1
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fneg:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    li a1, -1
+; RV64IZFINXZDINX-NEXT:    slli a1, a1, 63
+; RV64IZFINXZDINX-NEXT:    xor a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = fneg double %a
   ret double %1
 }
@@ -73,6 +82,12 @@ define double @fabs(double %a) nounwind {
 ; RV64IFD-NEXT:    slli a0, a0, 1
 ; RV64IFD-NEXT:    srli a0, a0, 1
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fabs:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    slli a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    srli a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = call double @llvm.fabs.f64(double %a)
   ret double %1
 }
@@ -130,6 +145,14 @@ define double @fcopysign_fneg(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    fsgnj.d fa5, fa4, fa5
 ; RV64IFD-NEXT:    fmv.x.d a0, fa5
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fcopysign_fneg:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    li a2, -1
+; RV64IZFINXZDINX-NEXT:    slli a2, a2, 63
+; RV64IZFINXZDINX-NEXT:    xor a1, a1, a2
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = fneg double %b
   %2 = call double @llvm.copysign.f64(double %a, double %1)
   ret double %2
diff --git a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
index 513221301d652..dd68a94f7836d 100644
--- a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefix=RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefix=RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=RV64IZFINXZDINX %s
 
 declare void @abort()
 declare void @exit(i32)
@@ -29,6 +31,17 @@ define void @br_fcmp_false(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_false:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    li a0, 1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB0_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.then
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB0_2: # %if.else
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp false double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -60,6 +73,17 @@ define void @br_fcmp_oeq(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_oeq:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB1_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB1_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp oeq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -94,6 +118,17 @@ define void @br_fcmp_oeq_alt(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_oeq_alt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB2_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB2_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp oeq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -125,6 +160,17 @@ define void @br_fcmp_ogt(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ogt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB3_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB3_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ogt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -156,6 +202,17 @@ define void @br_fcmp_oge(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_oge:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB4_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB4_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp oge double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -187,6 +244,17 @@ define void @br_fcmp_olt(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_olt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB5_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB5_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp olt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -218,6 +286,17 @@ define void @br_fcmp_ole(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ole:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB6_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB6_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ole double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -253,6 +332,19 @@ define void @br_fcmp_one(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_one:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    or a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB7_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB7_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp one double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -288,6 +380,19 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ord:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a1, a1, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB8_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB8_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ord double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -323,6 +428,19 @@ define void @br_fcmp_ueq(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ueq:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    or a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB9_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB9_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ueq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -354,6 +472,17 @@ define void @br_fcmp_ugt(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ugt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB10_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB10_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ugt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -385,6 +514,17 @@ define void @br_fcmp_uge(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_uge:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB11_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB11_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp uge double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -416,6 +556,17 @@ define void @br_fcmp_ult(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ult:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB12_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB12_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ult double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -447,6 +598,17 @@ define void @br_fcmp_ule(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_ule:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB13_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB13_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp ule double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -478,6 +640,17 @@ define void @br_fcmp_une(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_une:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB14_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB14_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp une double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -513,6 +686,19 @@ define void @br_fcmp_uno(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_uno:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a1, a1, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    beqz a0, .LBB15_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB15_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp uno double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -544,6 +730,17 @@ define void @br_fcmp_true(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call abort@plt
+;
+; RV64IZFINXZDINX-LABEL: br_fcmp_true:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    li a0, 1
+; RV64IZFINXZDINX-NEXT:    bnez a0, .LBB16_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    ret
+; RV64IZFINXZDINX-NEXT:  .LBB16_2: # %if.then
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call abort@plt
   %1 = fcmp true double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
diff --git a/llvm/test/CodeGen/RISCV/double-convert-strict.ll b/llvm/test/CodeGen/RISCV/double-convert-strict.ll
index 1c8291d3ac27e..bd78dc5ecd25e 100644
--- a/llvm/test/CodeGen/RISCV/double-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert-strict.ll
@@ -5,6 +5,9 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -20,6 +23,11 @@ define float @fcvt_s_d(double %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.s.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_s_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.s.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -48,6 +56,11 @@ define double @fcvt_d_s(float %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.s fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_s:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.s a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -76,6 +89,11 @@ define i32 @fcvt_w_d(double %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -106,6 +124,11 @@ define i32 @fcvt_wu_d(double %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -138,6 +161,13 @@ define i32 @fcvt_wu_d_multiple_use(double %x, ptr %y) nounwind {
 ; CHECKIFD-NEXT:    add a0, a0, a1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_d_multiple_use:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    seqz a1, a0
+; RV64IZFINXZDINX-NEXT:    add a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_d_multiple_use:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -171,6 +201,11 @@ define double @fcvt_d_w(i32 %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -201,6 +236,12 @@ define double @fcvt_d_w_load(ptr %p) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_load:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lw a0, 0(a0)
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -231,6 +272,11 @@ define double @fcvt_d_wu(i32 %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -267,6 +313,12 @@ define double @fcvt_d_wu_load(ptr %p) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.d.wu fa0, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_load:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lwu a0, 0(a0)
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -306,6 +358,11 @@ define i64 @fcvt_l_d(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_l_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_l_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -343,6 +400,11 @@ define i64 @fcvt_lu_d(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_lu_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_lu_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -380,6 +442,11 @@ define double @fcvt_d_l(i64 %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.d.l fa0, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_l:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_l:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -417,6 +484,11 @@ define double @fcvt_d_lu(i64 %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.d.lu fa0, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_lu:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.lu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_lu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -445,6 +517,11 @@ define double @fcvt_d_w_i8(i8 signext %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_i8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -473,6 +550,11 @@ define double @fcvt_d_wu_i8(i8 zeroext %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -501,6 +583,11 @@ define double @fcvt_d_w_i16(i16 signext %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_i16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -529,6 +616,11 @@ define double @fcvt_d_wu_i16(i16 zeroext %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -567,6 +659,15 @@ define signext i32 @fcvt_d_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IFD-NEXT:    fsd fa5, 0(a1)
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_demanded_bits:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addiw a2, a0, 1
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    sd a0, 0(a1)
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -625,6 +726,13 @@ define signext i32 @fcvt_d_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IFD-NEXT:    fsd fa5, 0(a1)
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_demanded_bits:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addiw a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a2, a0
+; RV64IZFINXZDINX-NEXT:    sd a2, 0(a1)
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index b4f9e3b9e2a79..249395f1e36e7 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=RV64IZFINXZDINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -14,6 +16,11 @@ define float @fcvt_s_d(double %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.s.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_s_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.s.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -41,6 +48,11 @@ define double @fcvt_d_s(float %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.s fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_s:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.s a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -68,6 +80,11 @@ define i32 @fcvt_w_d(double %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -99,6 +116,15 @@ define i32 @fcvt_w_d_sat(double %a) nounwind {
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_d_sat:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_d_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -209,6 +235,11 @@ define i32 @fcvt_wu_d(double %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -240,6 +271,13 @@ define i32 @fcvt_wu_d_multiple_use(double %x, ptr %y) nounwind {
 ; CHECKIFD-NEXT:    add a0, a0, a1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_d_multiple_use:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    seqz a1, a0
+; RV64IZFINXZDINX-NEXT:    add a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_d_multiple_use:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -288,6 +326,17 @@ define i32 @fcvt_wu_d_sat(double %a) nounwind {
 ; RV64IFD-NEXT:    srli a0, a0, 32
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_d_sat:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addiw a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    slli a0, a0, 32
+; RV64IZFINXZDINX-NEXT:    srli a0, a0, 32
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_d_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -372,6 +421,11 @@ define double @fcvt_d_w(i32 %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -401,6 +455,12 @@ define double @fcvt_d_w_load(ptr %p) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_load:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lw a0, 0(a0)
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -431,6 +491,11 @@ define double @fcvt_d_wu(i32 %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -466,6 +531,12 @@ define double @fcvt_d_wu_load(ptr %p) nounwind {
 ; RV64IFD-NEXT:    fcvt.d.wu fa0, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_load:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lwu a0, 0(a0)
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -505,6 +576,11 @@ define i64 @fcvt_l_d(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_l_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_l_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -538,27 +614,27 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi@plt
-; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bnez s0, .LBB12_2
+; RV32IFD-NEXT:    lui a4, 524288
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    beqz s0, .LBB12_2
 ; RV32IFD-NEXT:  # %bb.1: # %start
-; RV32IFD-NEXT:    lui a1, 524288
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB12_2: # %start
-; RV32IFD-NEXT:    lui a2, %hi(.LCPI12_1)
-; RV32IFD-NEXT:    fld fa5, %lo(.LCPI12_1)(a2)
-; RV32IFD-NEXT:    flt.d a2, fa5, fs0
-; RV32IFD-NEXT:    beqz a2, .LBB12_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI12_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI12_1)(a1)
+; RV32IFD-NEXT:    flt.d a3, fa5, fs0
+; RV32IFD-NEXT:    beqz a3, .LBB12_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a1, a3, -1
+; RV32IFD-NEXT:    addi a2, a4, -1
 ; RV32IFD-NEXT:  .LBB12_4: # %start
-; RV32IFD-NEXT:    feq.d a3, fs0, fs0
-; RV32IFD-NEXT:    seqz a3, a3
-; RV32IFD-NEXT:    addi a3, a3, -1
-; RV32IFD-NEXT:    and a1, a3, a1
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    neg a4, s0
-; RV32IFD-NEXT:    and a0, a4, a0
-; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a4, a1
+; RV32IFD-NEXT:    and a1, a4, a2
+; RV32IFD-NEXT:    neg a2, a3
+; RV32IFD-NEXT:    neg a3, s0
 ; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a4, a0
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -574,6 +650,15 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_l_d_sat:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_l_d_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -584,32 +669,30 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lui s2, 278016
-; RV32I-NEXT:    addi s2, s2, -1
+; RV32I-NEXT:    lui a3, 278016
+; RV32I-NEXT:    addi a3, a3, -1
 ; RV32I-NEXT:    li a2, -1
-; RV32I-NEXT:    mv a3, s2
 ; RV32I-NEXT:    call __gtdf2@plt
-; RV32I-NEXT:    mv s4, a0
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lui a3, 802304
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    call __gedf2@plt
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    call __fixdfdi@plt
-; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    mv s5, a1
 ; RV32I-NEXT:    lui a0, 524288
-; RV32I-NEXT:    bgez s6, .LBB12_2
+; RV32I-NEXT:    bgez s3, .LBB12_2
 ; RV32I-NEXT:  # %bb.1: # %start
 ; RV32I-NEXT:    lui s5, 524288
 ; RV32I-NEXT:  .LBB12_2: # %start
-; RV32I-NEXT:    blez s4, .LBB12_4
+; RV32I-NEXT:    blez s2, .LBB12_4
 ; RV32I-NEXT:  # %bb.3: # %start
 ; RV32I-NEXT:    addi s5, a0, -1
 ; RV32I-NEXT:  .LBB12_4: # %start
@@ -620,32 +703,14 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    call __unorddf2@plt
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and s4, a0, s5
-; RV32I-NEXT:    lui a3, 802304
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    li a2, 0
-; RV32I-NEXT:    call __gedf2@plt
-; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and s3, a0, s3
-; RV32I-NEXT:    li a2, -1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    mv a3, s2
-; RV32I-NEXT:    call __gtdf2@plt
-; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg a0, a0
-; RV32I-NEXT:    or s2, a0, s3
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    mv a2, s1
-; RV32I-NEXT:    mv a3, s0
-; RV32I-NEXT:    call __unorddf2@plt
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a0, a0, s2
-; RV32I-NEXT:    mv a1, s4
+; RV32I-NEXT:    and a1, a0, s5
+; RV32I-NEXT:    slti a2, s3, 0
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    and a2, a2, s4
+; RV32I-NEXT:    sgtz a3, s2
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    and a0, a0, a2
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -653,7 +718,6 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -721,6 +785,11 @@ define i64 @fcvt_lu_d(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_lu_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_lu_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -777,6 +846,15 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind {
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_lu_d_sat:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_lu_d_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -785,54 +863,33 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lui s2, 278272
-; RV32I-NEXT:    addi s2, s2, -1
+; RV32I-NEXT:    lui a3, 278272
+; RV32I-NEXT:    addi a3, a3, -1
 ; RV32I-NEXT:    li a2, -1
-; RV32I-NEXT:    mv a3, s2
 ; RV32I-NEXT:    call __gtdf2@plt
 ; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg s4, a0
+; RV32I-NEXT:    neg s2, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __gedf2@plt
 ; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi s5, a0, -1
+; RV32I-NEXT:    addi s3, a0, -1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    call __fixunsdfdi@plt
-; RV32I-NEXT:    mv s3, a1
-; RV32I-NEXT:    and a0, s5, a0
-; RV32I-NEXT:    or s4, s4, a0
-; RV32I-NEXT:    li a2, -1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    mv a3, s2
-; RV32I-NEXT:    call __gtdf2@plt
-; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg s2, a0
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    li a2, 0
-; RV32I-NEXT:    li a3, 0
-; RV32I-NEXT:    call __gedf2@plt
-; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a0, a0, s3
-; RV32I-NEXT:    or a1, s2, a0
-; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    and a0, s3, a0
+; RV32I-NEXT:    or a0, s2, a0
+; RV32I-NEXT:    and a1, s3, a1
+; RV32I-NEXT:    or a1, s2, a1
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -886,6 +943,11 @@ define i64 @fmv_x_d(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    fmv.x.d a0, fa5
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmv_x_d:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmv_x_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -923,6 +985,11 @@ define double @fcvt_d_l(i64 %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.d.l fa0, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_l:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_l:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -959,6 +1026,11 @@ define double @fcvt_d_lu(i64 %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.d.lu fa0, a0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_lu:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.lu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_lu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1002,6 +1074,11 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind {
 ; RV64IFD-NEXT:    fadd.d fa0, fa5, fa4
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmv_d_x:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmv_d_x:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1031,6 +1108,11 @@ define double @fcvt_d_w_i8(i8 signext %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_i8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1058,6 +1140,11 @@ define double @fcvt_d_wu_i8(i8 zeroext %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1085,6 +1172,11 @@ define double @fcvt_d_w_i16(i16 signext %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.w fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_i16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1112,6 +1204,11 @@ define double @fcvt_d_wu_i16(i16 zeroext %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.d.wu fa0, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1149,6 +1246,15 @@ define signext i32 @fcvt_d_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IFD-NEXT:    fsd fa5, 0(a1)
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_w_demanded_bits:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addiw a2, a0, 1
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    fcvt.d.w a0, a0
+; RV64IZFINXZDINX-NEXT:    sd a0, 0(a1)
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_w_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1207,6 +1313,13 @@ define signext i32 @fcvt_d_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IFD-NEXT:    fsd fa5, 0(a1)
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_d_wu_demanded_bits:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addiw a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    fcvt.d.wu a2, a0
+; RV64IZFINXZDINX-NEXT:    sd a2, 0(a1)
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_d_wu_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1260,6 +1373,11 @@ define signext i16 @fcvt_w_s_i16(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_s_i16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1288,13 +1406,12 @@ define signext i16 @fcvt_w_s_sat_i16(double %a) nounwind {
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI26_0)(a0)
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI26_1)
 ; RV32IFD-NEXT:    fld fa4, %lo(.LCPI26_1)(a0)
+; RV32IFD-NEXT:    feq.d a0, fa0, fa0
+; RV32IFD-NEXT:    neg a0, a0
 ; RV32IFD-NEXT:    fmax.d fa5, fa0, fa5
 ; RV32IFD-NEXT:    fmin.d fa5, fa5, fa4
-; RV32IFD-NEXT:    fcvt.w.d a0, fa5, rtz
-; RV32IFD-NEXT:    feq.d a1, fa0, fa0
-; RV32IFD-NEXT:    seqz a1, a1
-; RV32IFD-NEXT:    addi a1, a1, -1
-; RV32IFD-NEXT:    and a0, a1, a0
+; RV32IFD-NEXT:    fcvt.w.d a1, fa5, rtz
+; RV32IFD-NEXT:    and a0, a0, a1
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fcvt_w_s_sat_i16:
@@ -1303,15 +1420,28 @@ define signext i16 @fcvt_w_s_sat_i16(double %a) nounwind {
 ; RV64IFD-NEXT:    fld fa5, %lo(.LCPI26_0)(a0)
 ; RV64IFD-NEXT:    lui a0, %hi(.LCPI26_1)
 ; RV64IFD-NEXT:    fld fa4, %lo(.LCPI26_1)(a0)
+; RV64IFD-NEXT:    feq.d a0, fa0, fa0
 ; RV64IFD-NEXT:    fmax.d fa5, fa0, fa5
+; RV64IFD-NEXT:    neg a0, a0
 ; RV64IFD-NEXT:    fmin.d fa5, fa5, fa4
-; RV64IFD-NEXT:    fcvt.l.d a0, fa5, rtz
-; RV64IFD-NEXT:    feq.d a1, fa0, fa0
-; RV64IFD-NEXT:    seqz a1, a1
-; RV64IFD-NEXT:    addi a1, a1, -1
-; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    fcvt.l.d a1, fa5, rtz
+; RV64IFD-NEXT:    and a0, a0, a1
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_s_sat_i16:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI26_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI26_0)(a1)
+; RV64IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI26_1)
+; RV64IZFINXZDINX-NEXT:    ld a2, %lo(.LCPI26_1)(a2)
+; RV64IZFINXZDINX-NEXT:    fmax.d a1, a0, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    neg a0, a0
+; RV64IZFINXZDINX-NEXT:    fmin.d a1, a1, a2
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a1, rtz
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_sat_i16:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1427,6 +1557,11 @@ define zeroext i16 @fcvt_wu_s_i16(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_s_i16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1469,6 +1604,15 @@ define zeroext i16 @fcvt_wu_s_sat_i16(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa5, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_s_sat_i16:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI28_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI28_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fmax.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fmin.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_sat_i16:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1567,6 +1711,11 @@ define signext i8 @fcvt_w_s_i8(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_s_i8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1595,13 +1744,12 @@ define signext i8 @fcvt_w_s_sat_i8(double %a) nounwind {
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI30_0)(a0)
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI30_1)
 ; RV32IFD-NEXT:    fld fa4, %lo(.LCPI30_1)(a0)
+; RV32IFD-NEXT:    feq.d a0, fa0, fa0
+; RV32IFD-NEXT:    neg a0, a0
 ; RV32IFD-NEXT:    fmax.d fa5, fa0, fa5
 ; RV32IFD-NEXT:    fmin.d fa5, fa5, fa4
-; RV32IFD-NEXT:    fcvt.w.d a0, fa5, rtz
-; RV32IFD-NEXT:    feq.d a1, fa0, fa0
-; RV32IFD-NEXT:    seqz a1, a1
-; RV32IFD-NEXT:    addi a1, a1, -1
-; RV32IFD-NEXT:    and a0, a1, a0
+; RV32IFD-NEXT:    fcvt.w.d a1, fa5, rtz
+; RV32IFD-NEXT:    and a0, a0, a1
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fcvt_w_s_sat_i8:
@@ -1610,15 +1758,28 @@ define signext i8 @fcvt_w_s_sat_i8(double %a) nounwind {
 ; RV64IFD-NEXT:    fld fa5, %lo(.LCPI30_0)(a0)
 ; RV64IFD-NEXT:    lui a0, %hi(.LCPI30_1)
 ; RV64IFD-NEXT:    fld fa4, %lo(.LCPI30_1)(a0)
+; RV64IFD-NEXT:    feq.d a0, fa0, fa0
 ; RV64IFD-NEXT:    fmax.d fa5, fa0, fa5
+; RV64IFD-NEXT:    neg a0, a0
 ; RV64IFD-NEXT:    fmin.d fa5, fa5, fa4
-; RV64IFD-NEXT:    fcvt.l.d a0, fa5, rtz
-; RV64IFD-NEXT:    feq.d a1, fa0, fa0
-; RV64IFD-NEXT:    seqz a1, a1
-; RV64IFD-NEXT:    addi a1, a1, -1
-; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    fcvt.l.d a1, fa5, rtz
+; RV64IFD-NEXT:    and a0, a0, a1
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_s_sat_i8:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI30_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI30_0)(a1)
+; RV64IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI30_1)
+; RV64IZFINXZDINX-NEXT:    ld a2, %lo(.LCPI30_1)(a2)
+; RV64IZFINXZDINX-NEXT:    fmax.d a1, a0, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    neg a0, a0
+; RV64IZFINXZDINX-NEXT:    fmin.d a1, a1, a2
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a1, rtz
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_sat_i8:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1731,6 +1892,11 @@ define zeroext i8 @fcvt_wu_s_i8(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_s_i8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1775,6 +1941,15 @@ define zeroext i8 @fcvt_wu_s_sat_i8(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa5, rtz
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_s_sat_i8:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI32_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI32_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fmax.d a0, a0, zero
+; RV64IZFINXZDINX-NEXT:    fmin.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_sat_i8:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1877,6 +2052,17 @@ define zeroext i32 @fcvt_wu_d_sat_zext(double %a) nounwind {
 ; RV64IFD-NEXT:    srli a0, a0, 32
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_wu_d_sat_zext:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addiw a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    slli a0, a0, 32
+; RV64IZFINXZDINX-NEXT:    srli a0, a0, 32
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_d_sat_zext:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -1966,6 +2152,15 @@ define signext i32 @fcvt_w_d_sat_sext(double %a) nounwind {
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcvt_w_d_sat_sext:
+; RV64IZFINXZDINX:       # %bb.0: # %start
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_d_sat_sext:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll b/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
index e6cf098eae57e..0cac562005784 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
@@ -5,6 +5,9 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefix=CHECKIFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -16,6 +19,11 @@ define i32 @fcmp_oeq(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d a0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_oeq:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oeq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -50,6 +58,15 @@ define i32 @fcmp_ogt(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ogt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a3, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a2, a1, a0
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a3
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ogt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -83,6 +100,15 @@ define i32 @fcmp_oge(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_oge:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a3, fflags
+; RV64IZFINXZDINX-NEXT:    fle.d a2, a1, a0
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a3
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -118,6 +144,15 @@ define i32 @fcmp_olt(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_olt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a3, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a3
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a0, a1
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_olt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -151,6 +186,15 @@ define i32 @fcmp_ole(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ole:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a3, fflags
+; RV64IZFINXZDINX-NEXT:    fle.d a2, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a3
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a0, a1
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ole:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -191,6 +235,20 @@ define i32 @fcmp_one(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_one:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a3, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a4, a1, a0
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    or a2, a4, a3
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -257,6 +315,13 @@ define i32 @fcmp_ord(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ord:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a1, a1, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ord:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -298,6 +363,21 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ueq:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a3, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a4, a1, a0
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    or a3, a4, a3
+; RV64IZFINXZDINX-NEXT:    xori a2, a3, 1
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ueq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -366,6 +446,16 @@ define i32 @fcmp_ugt(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ugt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    fle.d a3, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    xori a2, a3, 1
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a0, a1
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ugt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -400,6 +490,16 @@ define i32 @fcmp_uge(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_uge:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a3, a0, a1
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    xori a2, a3, 1
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a0, a1
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -436,6 +536,16 @@ define i32 @fcmp_ult(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ult:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    fle.d a3, a1, a0
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    xori a2, a3, 1
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ult:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -470,6 +580,16 @@ define i32 @fcmp_ule(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    feq.d zero, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_ule:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    csrr a2, fflags
+; RV64IZFINXZDINX-NEXT:    flt.d a3, a1, a0
+; RV64IZFINXZDINX-NEXT:    csrw fflags, a2
+; RV64IZFINXZDINX-NEXT:    xori a2, a3, 1
+; RV64IZFINXZDINX-NEXT:    feq.d zero, a1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ule:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -501,6 +621,12 @@ define i32 @fcmp_une(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_une:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_une:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -534,6 +660,14 @@ define i32 @fcmp_uno(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmp_uno:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    feq.d a1, a1, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uno:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -566,6 +700,13 @@ define i32 @fcmps_oeq(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_oeq:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a2, a1, a0
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_oeq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -597,6 +738,11 @@ define i32 @fcmps_ogt(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    flt.d a0, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ogt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ogt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -627,6 +773,11 @@ define i32 @fcmps_oge(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    fle.d a0, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_oge:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_oge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -659,6 +810,11 @@ define i32 @fcmps_olt(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    flt.d a0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_olt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_olt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -689,6 +845,11 @@ define i32 @fcmps_ole(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    fle.d a0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ole:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ole:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -721,6 +882,13 @@ define i32 @fcmps_one(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    or a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_one:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    or a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -787,6 +955,13 @@ define i32 @fcmps_ord(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ord:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a1, a1, a1
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ord:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -820,6 +995,14 @@ define i32 @fcmps_ueq(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ueq:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    or a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ueq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -885,6 +1068,12 @@ define i32 @fcmps_ugt(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ugt:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ugt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -916,6 +1105,12 @@ define i32 @fcmps_uge(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_uge:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_uge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -949,6 +1144,12 @@ define i32 @fcmps_ult(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ult:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ult:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -980,6 +1181,12 @@ define i32 @fcmps_ule(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_ule:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ule:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1013,6 +1220,14 @@ define i32 @fcmps_une(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_une:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a2, a1, a0
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_une:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1046,6 +1261,14 @@ define i32 @fcmps_uno(double %a, double %b) nounwind strictfp {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fcmps_uno:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fle.d a1, a1, a1
+; RV64IZFINXZDINX-NEXT:    fle.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    xori a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_uno:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp.ll b/llvm/test/CodeGen/RISCV/double-fcmp.ll
index 9f59dd21106a0..d22df14c5d7c0 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefix=CHECKIFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefix=CHECKIFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=CHECKIZFINXZDINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -14,6 +16,11 @@ define i32 @fcmp_false(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    li a0, 0
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_false:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    li a0, 0
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_false:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    li a0, 0
@@ -34,6 +41,11 @@ define i32 @fcmp_oeq(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    feq.d a0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_oeq:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oeq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -64,6 +76,11 @@ define i32 @fcmp_ogt(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    flt.d a0, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ogt:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ogt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -94,6 +111,11 @@ define i32 @fcmp_oge(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fle.d a0, fa1, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_oge:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    fle.d a0, a1, a0
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -126,6 +148,11 @@ define i32 @fcmp_olt(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    flt.d a0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_olt:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    flt.d a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_olt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -156,6 +183,11 @@ define i32 @fcmp_ole(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fle.d a0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ole:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ole:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -188,6 +220,13 @@ define i32 @fcmp_one(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    or a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_one:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; CHECKIZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; CHECKIZFINXZDINX-NEXT:    or a0, a0, a2
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -254,6 +293,13 @@ define i32 @fcmp_ord(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ord:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    feq.d a1, a1, a1
+; CHECKIZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; CHECKIZFINXZDINX-NEXT:    and a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ord:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -287,6 +333,14 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ueq:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    flt.d a2, a0, a1
+; CHECKIZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; CHECKIZFINXZDINX-NEXT:    or a0, a0, a2
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ueq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -352,6 +406,12 @@ define i32 @fcmp_ugt(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ugt:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    fle.d a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ugt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -383,6 +443,12 @@ define i32 @fcmp_uge(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_uge:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    flt.d a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -416,6 +482,12 @@ define i32 @fcmp_ult(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ult:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    fle.d a0, a1, a0
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ult:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -447,6 +519,12 @@ define i32 @fcmp_ule(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_ule:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    flt.d a0, a1, a0
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ule:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -478,6 +556,12 @@ define i32 @fcmp_une(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_une:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    feq.d a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_une:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -511,6 +595,14 @@ define i32 @fcmp_uno(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    xori a0, a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_uno:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    feq.d a1, a1, a1
+; CHECKIZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; CHECKIZFINXZDINX-NEXT:    and a0, a0, a1
+; CHECKIZFINXZDINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uno:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -541,6 +633,11 @@ define i32 @fcmp_true(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    li a0, 1
 ; CHECKIFD-NEXT:    ret
 ;
+; CHECKIZFINXZDINX-LABEL: fcmp_true:
+; CHECKIZFINXZDINX:       # %bb.0:
+; CHECKIZFINXZDINX-NEXT:    li a0, 1
+; CHECKIZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_true:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    li a0, 1
diff --git a/llvm/test/CodeGen/RISCV/double-frem.ll b/llvm/test/CodeGen/RISCV/double-frem.ll
index 555793ec291c0..1a7bd79763c88 100644
--- a/llvm/test/CodeGen/RISCV/double-frem.ll
+++ b/llvm/test/CodeGen/RISCV/double-frem.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 
 define double @frem_f64(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: frem_f64:
@@ -12,6 +14,10 @@ define double @frem_f64(double %a, double %b) nounwind {
 ; RV64IFD-LABEL: frem_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    tail fmod@plt
+;
+; RV64IZFINXZDINX-LABEL: frem_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail fmod@plt
   %1 = frem double %a, %b
   ret double %1
 }
diff --git a/llvm/test/CodeGen/RISCV/double-imm.ll b/llvm/test/CodeGen/RISCV/double-imm.ll
index d0b0ddd1ca028..62954586eb59e 100644
--- a/llvm/test/CodeGen/RISCV/double-imm.ll
+++ b/llvm/test/CodeGen/RISCV/double-imm.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck --check-prefix=CHECKRV64ZDINX %s
 
 define double @double_imm() nounwind {
 ; CHECK-LABEL: double_imm:
@@ -10,6 +12,12 @@ define double @double_imm() nounwind {
 ; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
 ; CHECK-NEXT:    fld fa0, %lo(.LCPI0_0)(a0)
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: double_imm:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECKRV64ZDINX-NEXT:    ld a0, %lo(.LCPI0_0)(a0)
+; CHECKRV64ZDINX-NEXT:    ret
   ret double 3.1415926535897931159979634685441851615905761718750
 }
 
@@ -20,6 +28,13 @@ define double @double_imm_op(double %a) nounwind {
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI1_0)(a0)
 ; CHECK-NEXT:    fadd.d fa0, fa0, fa5
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: double_imm_op:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    lui a1, %hi(.LCPI1_0)
+; CHECKRV64ZDINX-NEXT:    ld a1, %lo(.LCPI1_0)(a1)
+; CHECKRV64ZDINX-NEXT:    fadd.d a0, a0, a1
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fadd double %a, 1.0
   ret double %1
 }
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
index 1ce3f57e9c950..33369da90bae7 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll
@@ -5,6 +5,9 @@
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+d \
 ; RUN:   -verify-machineinstrs -disable-strictnode-mutation -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+zdinx \
+; RUN:   -verify-machineinstrs -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 \
 ; RUN:   -verify-machineinstrs -disable-strictnode-mutation \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
@@ -20,6 +23,11 @@ define double @sqrt_f64(double %a) nounwind strictfp {
 ; CHECKIFD-NEXT:    fsqrt.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: sqrt_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsqrt.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: sqrt_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -63,6 +71,16 @@ define double @powi_f64(double %a, i32 %b) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: powi_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sext.w a1, a1
+; RV64IZFINXZDINX-NEXT:    call __powidf2@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: powi_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -106,6 +124,15 @@ define double @sin_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: sin_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call sin@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: sin_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -148,6 +175,15 @@ define double @cos_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: cos_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call cos@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: cos_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -207,6 +243,24 @@ define double @sincos_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: sincos_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -32
+; RV64IZFINXZDINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    mv s0, a0
+; RV64IZFINXZDINX-NEXT:    call sin@plt
+; RV64IZFINXZDINX-NEXT:    mv s1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, s0
+; RV64IZFINXZDINX-NEXT:    call cos@plt
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, s1, a0
+; RV64IZFINXZDINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: sincos_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -282,6 +336,15 @@ define double @pow_f64(double %a, double %b) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: pow_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call pow@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: pow_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -324,6 +387,15 @@ define double @exp_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: exp_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call exp@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: exp_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -366,6 +438,15 @@ define double @exp2_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: exp2_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call exp2@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: exp2_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -408,6 +489,15 @@ define double @log_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: log_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call log@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: log_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -450,6 +540,15 @@ define double @log10_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: log10_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call log10@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: log10_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -492,6 +591,15 @@ define double @log2_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: log2_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call log2@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: log2_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -521,6 +629,11 @@ define double @fma_f64(double %a, double %b, double %c) nounwind strictfp {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fma_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fma_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -550,6 +663,11 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind strictfp {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmuladd_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmuladd_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -606,6 +724,15 @@ define double @minnum_f64(double %a, double %b) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: minnum_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call fmin@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: minnum_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -648,6 +775,15 @@ define double @maxnum_f64(double %a, double %b) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: maxnum_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call fmax@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: maxnum_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -707,6 +843,15 @@ define double @floor_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: floor_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call floor@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: floor_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -749,6 +894,15 @@ define double @ceil_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: ceil_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call ceil@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: ceil_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -791,6 +945,15 @@ define double @trunc_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: trunc_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call trunc@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: trunc_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -833,6 +996,15 @@ define double @rint_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: rint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call rint@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: rint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -875,6 +1047,15 @@ define double @nearbyint_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: nearbyint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call nearbyint@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: nearbyint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -917,6 +1098,15 @@ define double @round_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: round_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call round@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: round_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -959,6 +1149,15 @@ define double @roundeven_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: roundeven_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    call roundeven@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: roundeven_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -993,6 +1192,11 @@ define iXLen @lrint_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: lrint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: lrint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1027,6 +1231,11 @@ define iXLen @lround_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: lround_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: lround_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1065,6 +1274,11 @@ define i64 @llrint_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: llrint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: llrint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1103,6 +1317,11 @@ define i64 @llround_f64(double %a) nounwind strictfp {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: llround_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: llround_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
index 74cd579e2c4f3..a3056ce891f06 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
@@ -5,6 +5,9 @@
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+d \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+zdinx \
+; RUN:   -verify-machineinstrs -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 \
 ; RUN:   -verify-machineinstrs | FileCheck -check-prefix=RV32I %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 \
@@ -18,6 +21,11 @@ define double @sqrt_f64(double %a) nounwind {
 ; CHECKIFD-NEXT:    fsqrt.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: sqrt_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsqrt.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: sqrt_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -56,6 +64,16 @@ define double @powi_f64(double %a, i32 %b) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: powi_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sext.w a1, a1
+; RV64IZFINXZDINX-NEXT:    call __powidf2@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: powi_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -85,6 +103,10 @@ define double @sin_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail sin@plt
 ;
+; RV64IZFINXZDINX-LABEL: sin_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail sin@plt
+;
 ; RV32I-LABEL: sin_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -113,6 +135,10 @@ define double @cos_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail cos@plt
 ;
+; RV64IZFINXZDINX-LABEL: cos_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail cos@plt
+;
 ; RV32I-LABEL: cos_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -172,6 +198,24 @@ define double @sincos_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: sincos_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -32
+; RV64IZFINXZDINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    mv s0, a0
+; RV64IZFINXZDINX-NEXT:    call sin@plt
+; RV64IZFINXZDINX-NEXT:    mv s1, a0
+; RV64IZFINXZDINX-NEXT:    mv a0, s0
+; RV64IZFINXZDINX-NEXT:    call cos@plt
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, s1, a0
+; RV64IZFINXZDINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: sincos_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -233,6 +277,10 @@ define double @pow_f64(double %a, double %b) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail pow@plt
 ;
+; RV64IZFINXZDINX-LABEL: pow_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail pow@plt
+;
 ; RV32I-LABEL: pow_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -261,6 +309,10 @@ define double @exp_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail exp@plt
 ;
+; RV64IZFINXZDINX-LABEL: exp_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail exp@plt
+;
 ; RV32I-LABEL: exp_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -289,6 +341,10 @@ define double @exp2_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail exp2@plt
 ;
+; RV64IZFINXZDINX-LABEL: exp2_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail exp2@plt
+;
 ; RV32I-LABEL: exp2_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -317,6 +373,10 @@ define double @log_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail log@plt
 ;
+; RV64IZFINXZDINX-LABEL: log_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail log@plt
+;
 ; RV32I-LABEL: log_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -345,6 +405,10 @@ define double @log10_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail log10@plt
 ;
+; RV64IZFINXZDINX-LABEL: log10_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail log10@plt
+;
 ; RV32I-LABEL: log10_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -373,6 +437,10 @@ define double @log2_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail log2@plt
 ;
+; RV64IZFINXZDINX-LABEL: log2_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail log2@plt
+;
 ; RV32I-LABEL: log2_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -402,6 +470,11 @@ define double @fma_f64(double %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fma_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fma_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -431,6 +504,11 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fmadd.d fa0, fa0, fa1, fa2
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fmuladd_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmadd.d a0, a0, a1, a2
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmuladd_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -474,6 +552,12 @@ define double @fabs_f64(double %a) nounwind {
 ; CHECKIFD-NEXT:    fabs.d fa0, fa0
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: fabs_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    slli a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    srli a0, a0, 1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: fabs_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    slli a1, a1, 1
@@ -497,6 +581,11 @@ define double @minnum_f64(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fmin.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: minnum_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmin.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: minnum_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -526,6 +615,11 @@ define double @maxnum_f64(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fmax.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: maxnum_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fmax.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: maxnum_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -572,6 +666,11 @@ define double @copysign_f64(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fsgnj.d fa0, fa0, fa1
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: copysign_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: copysign_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
@@ -614,6 +713,20 @@ define double @floor_f64(double %a) nounwind {
 ; RV64IFD-NEXT:  .LBB17_2:
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: floor_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI17_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI17_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB17_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rdn
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rdn
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB17_2:
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: floor_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -656,6 +769,20 @@ define double @ceil_f64(double %a) nounwind {
 ; RV64IFD-NEXT:  .LBB18_2:
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: ceil_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI18_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI18_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB18_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rup
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rup
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB18_2:
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: ceil_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -698,6 +825,20 @@ define double @trunc_f64(double %a) nounwind {
 ; RV64IFD-NEXT:  .LBB19_2:
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: trunc_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI19_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI19_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB19_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rtz
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB19_2:
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: trunc_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -740,6 +881,20 @@ define double @rint_f64(double %a) nounwind {
 ; RV64IFD-NEXT:  .LBB20_2:
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: rint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI20_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI20_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB20_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB20_2:
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: rint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -768,6 +923,10 @@ define double @nearbyint_f64(double %a) nounwind {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    tail nearbyint@plt
 ;
+; RV64IZFINXZDINX-LABEL: nearbyint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    tail nearbyint@plt
+;
 ; RV32I-LABEL: nearbyint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -810,6 +969,20 @@ define double @round_f64(double %a) nounwind {
 ; RV64IFD-NEXT:  .LBB22_2:
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: round_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI22_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI22_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB22_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rmm
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rmm
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB22_2:
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: round_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -852,6 +1025,20 @@ define double @roundeven_f64(double %a) nounwind {
 ; RV64IFD-NEXT:  .LBB23_2:
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: roundeven_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI23_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI23_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB23_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rne
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rne
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB23_2:
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: roundeven_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -886,6 +1073,11 @@ define iXLen @lrint_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: lrint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: lrint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -921,6 +1113,11 @@ define iXLen @lround_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: lround_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: lround_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -948,6 +1145,11 @@ define i32 @lround_i32_f64(double %a) nounwind {
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rmm
 ; CHECKIFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: lround_i32_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: lround_i32_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -986,6 +1188,11 @@ define i64 @llrint_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: llrint_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: llrint_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1024,6 +1231,11 @@ define i64 @llround_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
 ;
+; RV64IZFINXZDINX-LABEL: llround_f64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
+;
 ; RV32I-LABEL: llround_f64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1044,3 +1256,44 @@ define i64 @llround_f64(double %a) nounwind {
   %1 = call i64 @llvm.llround.i64.f64(double %a)
   ret i64 %1
 }
+
+declare i1 @llvm.is.fpclass.f64(double, i32)
+define i1 @isnan_d_fpclass(double %x) {
+; CHECKIFD-LABEL: isnan_d_fpclass:
+; CHECKIFD:       # %bb.0:
+; CHECKIFD-NEXT:    fclass.d a0, fa0
+; CHECKIFD-NEXT:    andi a0, a0, 768
+; CHECKIFD-NEXT:    snez a0, a0
+; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: isnan_d_fpclass:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fclass.d a0, a0
+; RV64IZFINXZDINX-NEXT:    andi a0, a0, 768
+; RV64IZFINXZDINX-NEXT:    snez a0, a0
+; RV64IZFINXZDINX-NEXT:    ret
+;
+; RV32I-LABEL: isnan_d_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    lui a2, 524032
+; RV32I-NEXT:    beq a1, a2, .LBB29_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    slt a0, a2, a1
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB29_2:
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnan_d_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 1
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    li a1, 2047
+; RV64I-NEXT:    slli a1, a1, 52
+; RV64I-NEXT:    slt a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f64(double %x, i32 3)  ; nan
+  ret i1 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/double-isnan.ll b/llvm/test/CodeGen/RISCV/double-isnan.ll
index b0bc377380370..9edf2ce72abec 100644
--- a/llvm/test/CodeGen/RISCV/double-isnan.ll
+++ b/llvm/test/CodeGen/RISCV/double-isnan.ll
@@ -3,6 +3,8 @@
 ; RUN:   < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64d -verify-machineinstrs \
 ; RUN:   < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -target-abi lp64 -verify-machineinstrs \
+; RUN:   < %s | FileCheck --check-prefix=CHECKRV64ZDINX %s
 
 define zeroext i1 @double_is_nan(double %a) nounwind {
 ; CHECK-LABEL: double_is_nan:
@@ -10,6 +12,12 @@ define zeroext i1 @double_is_nan(double %a) nounwind {
 ; CHECK-NEXT:    feq.d a0, fa0, fa0
 ; CHECK-NEXT:    xori a0, a0, 1
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: double_is_nan:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a0, a0, a0
+; CHECKRV64ZDINX-NEXT:    xori a0, a0, 1
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp uno double %a, 0.000000e+00
   ret i1 %1
 }
@@ -19,6 +27,11 @@ define zeroext i1 @double_not_nan(double %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    feq.d a0, fa0, fa0
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: double_not_nan:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a0, a0, a0
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ord double %a, 0.000000e+00
   ret i1 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/double-mem.ll b/llvm/test/CodeGen/RISCV/double-mem.ll
index 7b029de617705..93b03d969da51 100644
--- a/llvm/test/CodeGen/RISCV/double-mem.ll
+++ b/llvm/test/CodeGen/RISCV/double-mem.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=RV64IZFINXZDINX %s
 
 define dso_local double @fld(ptr %a) nounwind {
 ; CHECKIFD-LABEL: fld:
@@ -11,6 +13,13 @@ define dso_local double @fld(ptr %a) nounwind {
 ; CHECKIFD-NEXT:    fld fa4, 24(a0)
 ; CHECKIFD-NEXT:    fadd.d fa0, fa5, fa4
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fld:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    ld a1, 0(a0)
+; RV64IZFINXZDINX-NEXT:    ld a0, 24(a0)
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = load double, ptr %a
   %2 = getelementptr double, ptr %a, i32 3
   %3 = load double, ptr %2
@@ -27,6 +36,13 @@ define dso_local void @fsd(ptr %a, double %b, double %c) nounwind {
 ; CHECKIFD-NEXT:    fsd fa5, 0(a0)
 ; CHECKIFD-NEXT:    fsd fa5, 64(a0)
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fsd:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a1, a1, a2
+; RV64IZFINXZDINX-NEXT:    sd a1, 0(a0)
+; RV64IZFINXZDINX-NEXT:    sd a1, 64(a0)
+; RV64IZFINXZDINX-NEXT:    ret
 ; Use %b and %c in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
   %1 = fadd double %b, %c
@@ -50,6 +66,17 @@ define dso_local double @fld_fsd_global(double %a, double %b) nounwind {
 ; CHECKIFD-NEXT:    fld fa5, 72(a1)
 ; CHECKIFD-NEXT:    fsd fa0, 72(a1)
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fld_fsd_global:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(G)
+; RV64IZFINXZDINX-NEXT:    ld a2, %lo(G)(a1)
+; RV64IZFINXZDINX-NEXT:    addi a2, a1, %lo(G)
+; RV64IZFINXZDINX-NEXT:    sd a0, %lo(G)(a1)
+; RV64IZFINXZDINX-NEXT:    ld a1, 72(a2)
+; RV64IZFINXZDINX-NEXT:    sd a0, 72(a2)
+; RV64IZFINXZDINX-NEXT:    ret
 ; Use %a and %b in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
   %1 = fadd double %a, %b
@@ -79,6 +106,15 @@ define dso_local double @fld_fsd_constant(double %a) nounwind {
 ; RV64IFD-NEXT:    fadd.d fa0, fa0, fa5
 ; RV64IFD-NEXT:    fsd fa0, -273(a0)
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fld_fsd_constant:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, 228023
+; RV64IZFINXZDINX-NEXT:    slli a1, a1, 2
+; RV64IZFINXZDINX-NEXT:    ld a2, -273(a1)
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    sd a0, -273(a1)
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = inttoptr i32 3735928559 to ptr
   %2 = load volatile double, ptr %1
   %3 = fadd double %a, %2
@@ -118,6 +154,21 @@ define dso_local double @fld_stack(double %a) nounwind {
 ; RV64IFD-NEXT:    fld fs0, 16(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fld_stack:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -32
+; RV64IZFINXZDINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    mv s0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, sp, 8
+; RV64IZFINXZDINX-NEXT:    call notdead@plt
+; RV64IZFINXZDINX-NEXT:    ld a0, 8(sp)
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, s0
+; RV64IZFINXZDINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = alloca double, align 8
   call void @notdead(ptr %1)
   %2 = load double, ptr %1
@@ -149,6 +200,18 @@ define dso_local void @fsd_stack(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fsd_stack:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    sd a0, 0(sp)
+; RV64IZFINXZDINX-NEXT:    mv a0, sp
+; RV64IZFINXZDINX-NEXT:    call notdead@plt
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = fadd double %a, %b ; force store from FPR64
   %2 = alloca double, align 8
   store double %1, ptr %2
@@ -163,6 +226,12 @@ define dso_local void @fsd_trunc(ptr %a, double %b) nounwind noinline optnone {
 ; CHECKIFD-NEXT:    fcvt.s.d fa5, fa0
 ; CHECKIFD-NEXT:    fsw fa5, 0(a0)
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: fsd_trunc:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.s.d a1, a1
+; RV64IZFINXZDINX-NEXT:    sw a1, 0(a0)
+; RV64IZFINXZDINX-NEXT:    ret
   %1 = fptrunc double %b to float
   store float %1, ptr %a, align 4
   ret void
diff --git a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
index fa67e2a4616b4..caf857588f6d5 100644
--- a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=RV64IZFINXZDINX %s
 
 define signext i32 @test_floor_si32(double %x) {
 ; CHECKIFD-LABEL: test_floor_si32:
@@ -13,6 +15,15 @@ define signext i32 @test_floor_si32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rdn
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = call i32 @llvm.fptosi.sat.i32.f64(double %a)
   ret i32 %b
@@ -31,27 +42,27 @@ define i64 @test_floor_si64(double %x) nounwind {
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi@plt
-; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bnez s0, .LBB1_2
+; RV32IFD-NEXT:    lui a4, 524288
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    beqz s0, .LBB1_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    lui a1, 524288
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB1_2:
-; RV32IFD-NEXT:    lui a2, %hi(.LCPI1_1)
-; RV32IFD-NEXT:    fld fa5, %lo(.LCPI1_1)(a2)
-; RV32IFD-NEXT:    flt.d a2, fa5, fs0
-; RV32IFD-NEXT:    beqz a2, .LBB1_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI1_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI1_1)(a1)
+; RV32IFD-NEXT:    flt.d a3, fa5, fs0
+; RV32IFD-NEXT:    beqz a3, .LBB1_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a1, a3, -1
+; RV32IFD-NEXT:    addi a2, a4, -1
 ; RV32IFD-NEXT:  .LBB1_4:
-; RV32IFD-NEXT:    feq.d a3, fs0, fs0
-; RV32IFD-NEXT:    seqz a3, a3
-; RV32IFD-NEXT:    addi a3, a3, -1
-; RV32IFD-NEXT:    and a1, a3, a1
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    neg a4, s0
-; RV32IFD-NEXT:    and a0, a4, a0
-; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a4, a1
+; RV32IFD-NEXT:    and a1, a4, a2
+; RV32IFD-NEXT:    neg a2, a3
+; RV32IFD-NEXT:    neg a3, s0
 ; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a4, a0
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -66,6 +77,15 @@ define i64 @test_floor_si64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rdn
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = call i64 @llvm.fptosi.sat.i64.f64(double %a)
   ret i64 %b
@@ -80,6 +100,15 @@ define signext i32 @test_floor_ui32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rdn
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = call i32 @llvm.fptoui.sat.i32.f64(double %a)
   ret i32 %b
@@ -119,6 +148,15 @@ define i64 @test_floor_ui64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a1, a0, rdn
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = call i64 @llvm.fptoui.sat.i64.f64(double %a)
   ret i64 %b
@@ -133,6 +171,15 @@ define signext i32 @test_ceil_si32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rup
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = call i32 @llvm.fptosi.sat.i32.f64(double %a)
   ret i32 %b
@@ -151,27 +198,27 @@ define i64 @test_ceil_si64(double %x) nounwind {
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi@plt
-; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bnez s0, .LBB5_2
+; RV32IFD-NEXT:    lui a4, 524288
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    beqz s0, .LBB5_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    lui a1, 524288
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB5_2:
-; RV32IFD-NEXT:    lui a2, %hi(.LCPI5_1)
-; RV32IFD-NEXT:    fld fa5, %lo(.LCPI5_1)(a2)
-; RV32IFD-NEXT:    flt.d a2, fa5, fs0
-; RV32IFD-NEXT:    beqz a2, .LBB5_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI5_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI5_1)(a1)
+; RV32IFD-NEXT:    flt.d a3, fa5, fs0
+; RV32IFD-NEXT:    beqz a3, .LBB5_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a1, a3, -1
+; RV32IFD-NEXT:    addi a2, a4, -1
 ; RV32IFD-NEXT:  .LBB5_4:
-; RV32IFD-NEXT:    feq.d a3, fs0, fs0
-; RV32IFD-NEXT:    seqz a3, a3
-; RV32IFD-NEXT:    addi a3, a3, -1
-; RV32IFD-NEXT:    and a1, a3, a1
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    neg a4, s0
-; RV32IFD-NEXT:    and a0, a4, a0
-; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a4, a1
+; RV32IFD-NEXT:    and a1, a4, a2
+; RV32IFD-NEXT:    neg a2, a3
+; RV32IFD-NEXT:    neg a3, s0
 ; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a4, a0
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -186,6 +233,15 @@ define i64 @test_ceil_si64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rup
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = call i64 @llvm.fptosi.sat.i64.f64(double %a)
   ret i64 %b
@@ -200,6 +256,15 @@ define signext i32 @test_ceil_ui32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rup
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = call i32 @llvm.fptoui.sat.i32.f64(double %a)
   ret i32 %b
@@ -239,6 +304,15 @@ define i64 @test_ceil_ui64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a1, a0, rup
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = call i64 @llvm.fptoui.sat.i64.f64(double %a)
   ret i64 %b
@@ -253,6 +327,15 @@ define signext i32 @test_trunc_si32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = call i32 @llvm.fptosi.sat.i32.f64(double %a)
   ret i32 %b
@@ -271,27 +354,27 @@ define i64 @test_trunc_si64(double %x) nounwind {
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi@plt
-; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bnez s0, .LBB9_2
+; RV32IFD-NEXT:    lui a4, 524288
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    beqz s0, .LBB9_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    lui a1, 524288
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB9_2:
-; RV32IFD-NEXT:    lui a2, %hi(.LCPI9_1)
-; RV32IFD-NEXT:    fld fa5, %lo(.LCPI9_1)(a2)
-; RV32IFD-NEXT:    flt.d a2, fa5, fs0
-; RV32IFD-NEXT:    beqz a2, .LBB9_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI9_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI9_1)(a1)
+; RV32IFD-NEXT:    flt.d a3, fa5, fs0
+; RV32IFD-NEXT:    beqz a3, .LBB9_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a1, a3, -1
+; RV32IFD-NEXT:    addi a2, a4, -1
 ; RV32IFD-NEXT:  .LBB9_4:
-; RV32IFD-NEXT:    feq.d a3, fs0, fs0
-; RV32IFD-NEXT:    seqz a3, a3
-; RV32IFD-NEXT:    addi a3, a3, -1
-; RV32IFD-NEXT:    and a1, a3, a1
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    neg a4, s0
-; RV32IFD-NEXT:    and a0, a4, a0
-; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a4, a1
+; RV32IFD-NEXT:    and a1, a4, a2
+; RV32IFD-NEXT:    neg a2, a3
+; RV32IFD-NEXT:    neg a3, s0
 ; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a4, a0
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -306,6 +389,15 @@ define i64 @test_trunc_si64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = call i64 @llvm.fptosi.sat.i64.f64(double %a)
   ret i64 %b
@@ -320,6 +412,15 @@ define signext i32 @test_trunc_ui32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = call i32 @llvm.fptoui.sat.i32.f64(double %a)
   ret i32 %b
@@ -359,6 +460,15 @@ define i64 @test_trunc_ui64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = call i64 @llvm.fptoui.sat.i64.f64(double %a)
   ret i64 %b
@@ -373,6 +483,15 @@ define signext i32 @test_round_si32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rmm
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = call i32 @llvm.fptosi.sat.i32.f64(double %a)
   ret i32 %b
@@ -391,27 +510,27 @@ define i64 @test_round_si64(double %x) nounwind {
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi@plt
-; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bnez s0, .LBB13_2
+; RV32IFD-NEXT:    lui a4, 524288
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    beqz s0, .LBB13_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    lui a1, 524288
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB13_2:
-; RV32IFD-NEXT:    lui a2, %hi(.LCPI13_1)
-; RV32IFD-NEXT:    fld fa5, %lo(.LCPI13_1)(a2)
-; RV32IFD-NEXT:    flt.d a2, fa5, fs0
-; RV32IFD-NEXT:    beqz a2, .LBB13_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI13_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI13_1)(a1)
+; RV32IFD-NEXT:    flt.d a3, fa5, fs0
+; RV32IFD-NEXT:    beqz a3, .LBB13_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a1, a3, -1
+; RV32IFD-NEXT:    addi a2, a4, -1
 ; RV32IFD-NEXT:  .LBB13_4:
-; RV32IFD-NEXT:    feq.d a3, fs0, fs0
-; RV32IFD-NEXT:    seqz a3, a3
-; RV32IFD-NEXT:    addi a3, a3, -1
-; RV32IFD-NEXT:    and a1, a3, a1
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    neg a4, s0
-; RV32IFD-NEXT:    and a0, a4, a0
-; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a4, a1
+; RV32IFD-NEXT:    and a1, a4, a2
+; RV32IFD-NEXT:    neg a2, a3
+; RV32IFD-NEXT:    neg a3, s0
 ; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a4, a0
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -426,6 +545,15 @@ define i64 @test_round_si64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rmm
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = call i64 @llvm.fptosi.sat.i64.f64(double %a)
   ret i64 %b
@@ -440,6 +568,15 @@ define signext i32 @test_round_ui32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rmm
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = call i32 @llvm.fptoui.sat.i32.f64(double %a)
   ret i32 %b
@@ -479,6 +616,15 @@ define i64 @test_round_ui64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a1, a0, rmm
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = call i64 @llvm.fptoui.sat.i64.f64(double %a)
   ret i64 %b
@@ -493,6 +639,15 @@ define signext i32 @test_roundeven_si32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a1, a0, rne
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = call i32 @llvm.fptosi.sat.i32.f64(double %a)
   ret i32 %b
@@ -511,27 +666,27 @@ define i64 @test_roundeven_si64(double %x) nounwind {
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi@plt
-; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bnez s0, .LBB17_2
+; RV32IFD-NEXT:    lui a4, 524288
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    beqz s0, .LBB17_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    lui a1, 524288
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB17_2:
-; RV32IFD-NEXT:    lui a2, %hi(.LCPI17_1)
-; RV32IFD-NEXT:    fld fa5, %lo(.LCPI17_1)(a2)
-; RV32IFD-NEXT:    flt.d a2, fa5, fs0
-; RV32IFD-NEXT:    beqz a2, .LBB17_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI17_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI17_1)(a1)
+; RV32IFD-NEXT:    flt.d a3, fa5, fs0
+; RV32IFD-NEXT:    beqz a3, .LBB17_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a1, a3, -1
+; RV32IFD-NEXT:    addi a2, a4, -1
 ; RV32IFD-NEXT:  .LBB17_4:
-; RV32IFD-NEXT:    feq.d a3, fs0, fs0
-; RV32IFD-NEXT:    seqz a3, a3
-; RV32IFD-NEXT:    addi a3, a3, -1
-; RV32IFD-NEXT:    and a1, a3, a1
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    neg a4, s0
-; RV32IFD-NEXT:    and a0, a4, a0
-; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a4, a1
+; RV32IFD-NEXT:    and a1, a4, a2
+; RV32IFD-NEXT:    neg a2, a3
+; RV32IFD-NEXT:    neg a3, s0
 ; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a4, a0
 ; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -546,6 +701,15 @@ define i64 @test_roundeven_si64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rne
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = call i64 @llvm.fptosi.sat.i64.f64(double %a)
   ret i64 %b
@@ -560,6 +724,15 @@ define signext i32 @test_roundeven_ui32(double %x) {
 ; CHECKIFD-NEXT:    addi a1, a1, -1
 ; CHECKIFD-NEXT:    and a0, a1, a0
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a1, a0, rne
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = call i32 @llvm.fptoui.sat.i32.f64(double %a)
   ret i32 %b
@@ -599,6 +772,15 @@ define i64 @test_roundeven_ui64(double %x) nounwind {
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a1, a0, rne
+; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
+; RV64IZFINXZDINX-NEXT:    seqz a0, a0
+; RV64IZFINXZDINX-NEXT:    addi a0, a0, -1
+; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = call i64 @llvm.fptoui.sat.i64.f64(double %a)
   ret i64 %b
diff --git a/llvm/test/CodeGen/RISCV/double-round-conv.ll b/llvm/test/CodeGen/RISCV/double-round-conv.ll
index c67b94a972bd3..97bcd0016674a 100644
--- a/llvm/test/CodeGen/RISCV/double-round-conv.ll
+++ b/llvm/test/CodeGen/RISCV/double-round-conv.ll
@@ -3,6 +3,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck -check-prefixes=CHECKIFD,RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck -check-prefixes=CHECKIFD,RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=RV64IZFINXZDINX %s
 
 define signext i8 @test_floor_si8(double %x) {
 ; RV32IFD-LABEL: test_floor_si8:
@@ -14,6 +16,11 @@ define signext i8 @test_floor_si8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rdn
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_si8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptosi double %a to i8
   ret i8 %b
@@ -29,6 +36,11 @@ define signext i16 @test_floor_si16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rdn
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_si16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptosi double %a to i16
   ret i16 %b
@@ -39,6 +51,11 @@ define signext i32 @test_floor_si32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rdn
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptosi double %a to i32
   ret i32 %b
@@ -61,6 +78,11 @@ define i64 @test_floor_si64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rdn
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptosi double %a to i64
   ret i64 %b
@@ -76,6 +98,11 @@ define zeroext i8 @test_floor_ui8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rdn
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_ui8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptoui double %a to i8
   ret i8 %b
@@ -91,6 +118,11 @@ define zeroext i16 @test_floor_ui16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rdn
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_ui16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptoui double %a to i16
   ret i16 %b
@@ -101,6 +133,11 @@ define signext i32 @test_floor_ui32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rdn
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptoui double %a to i32
   ret i32 %b
@@ -123,6 +160,11 @@ define i64 @test_floor_ui64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rdn
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rdn
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   %b = fptoui double %a to i64
   ret i64 %b
@@ -138,6 +180,11 @@ define signext i8 @test_ceil_si8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rup
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_si8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptosi double %a to i8
   ret i8 %b
@@ -153,6 +200,11 @@ define signext i16 @test_ceil_si16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rup
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_si16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptosi double %a to i16
   ret i16 %b
@@ -163,6 +215,11 @@ define signext i32 @test_ceil_si32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rup
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptosi double %a to i32
   ret i32 %b
@@ -185,6 +242,11 @@ define i64 @test_ceil_si64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rup
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptosi double %a to i64
   ret i64 %b
@@ -200,6 +262,11 @@ define zeroext i8 @test_ceil_ui8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rup
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_ui8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptoui double %a to i8
   ret i8 %b
@@ -215,6 +282,11 @@ define zeroext i16 @test_ceil_ui16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rup
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_ui16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptoui double %a to i16
   ret i16 %b
@@ -225,6 +297,11 @@ define signext i32 @test_ceil_ui32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rup
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptoui double %a to i32
   ret i32 %b
@@ -247,6 +324,11 @@ define i64 @test_ceil_ui64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rup
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rup
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   %b = fptoui double %a to i64
   ret i64 %b
@@ -262,6 +344,11 @@ define signext i8 @test_trunc_si8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_si8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptosi double %a to i8
   ret i8 %b
@@ -277,6 +364,11 @@ define signext i16 @test_trunc_si16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_si16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptosi double %a to i16
   ret i16 %b
@@ -287,6 +379,11 @@ define signext i32 @test_trunc_si32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptosi double %a to i32
   ret i32 %b
@@ -309,6 +406,11 @@ define i64 @test_trunc_si64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptosi double %a to i64
   ret i64 %b
@@ -324,6 +426,11 @@ define zeroext i8 @test_trunc_ui8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_ui8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptoui double %a to i8
   ret i8 %b
@@ -339,6 +446,11 @@ define zeroext i16 @test_trunc_ui16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_ui16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptoui double %a to i16
   ret i16 %b
@@ -349,6 +461,11 @@ define signext i32 @test_trunc_ui32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rtz
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptoui double %a to i32
   ret i32 %b
@@ -371,6 +488,11 @@ define i64 @test_trunc_ui64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rtz
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   %b = fptoui double %a to i64
   ret i64 %b
@@ -386,6 +508,11 @@ define signext i8 @test_round_si8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_si8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptosi double %a to i8
   ret i8 %b
@@ -401,6 +528,11 @@ define signext i16 @test_round_si16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_si16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptosi double %a to i16
   ret i16 %b
@@ -411,6 +543,11 @@ define signext i32 @test_round_si32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rmm
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptosi double %a to i32
   ret i32 %b
@@ -433,6 +570,11 @@ define i64 @test_round_si64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptosi double %a to i64
   ret i64 %b
@@ -448,6 +590,11 @@ define zeroext i8 @test_round_ui8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_ui8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptoui double %a to i8
   ret i8 %b
@@ -463,6 +610,11 @@ define zeroext i16 @test_round_ui16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_ui16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptoui double %a to i16
   ret i16 %b
@@ -473,6 +625,11 @@ define signext i32 @test_round_ui32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rmm
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptoui double %a to i32
   ret i32 %b
@@ -495,6 +652,11 @@ define i64 @test_round_ui64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rmm
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rmm
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   %b = fptoui double %a to i64
   ret i64 %b
@@ -510,6 +672,11 @@ define signext i8 @test_roundeven_si8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rne
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_si8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptosi double %a to i8
   ret i8 %b
@@ -525,6 +692,11 @@ define signext i16 @test_roundeven_si16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rne
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_si16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptosi double %a to i16
   ret i16 %b
@@ -535,6 +707,11 @@ define signext i32 @test_roundeven_si32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.w.d a0, fa0, rne
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_si32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.w.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptosi double %a to i32
   ret i32 %b
@@ -557,6 +734,11 @@ define i64 @test_roundeven_si64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.l.d a0, fa0, rne
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_si64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptosi double %a to i64
   ret i64 %b
@@ -572,6 +754,11 @@ define zeroext i8 @test_roundeven_ui8(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rne
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_ui8:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptoui double %a to i8
   ret i8 %b
@@ -587,6 +774,11 @@ define zeroext i16 @test_roundeven_ui16(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rne
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_ui16:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptoui double %a to i16
   ret i16 %b
@@ -597,6 +789,11 @@ define signext i32 @test_roundeven_ui32(double %x) {
 ; CHECKIFD:       # %bb.0:
 ; CHECKIFD-NEXT:    fcvt.wu.d a0, fa0, rne
 ; CHECKIFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_ui32:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.wu.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptoui double %a to i32
   ret i32 %b
@@ -619,6 +816,11 @@ define i64 @test_roundeven_ui64(double %x) {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.lu.d a0, fa0, rne
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_ui64:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    fcvt.lu.d a0, a0, rne
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   %b = fptoui double %a to i64
   ret i64 %b
@@ -642,6 +844,20 @@ define double @test_floor_double(double %x) {
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
 ; RV64IFD-NEXT:  .LBB40_2:
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_floor_double:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI40_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI40_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB40_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rdn
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rdn
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB40_2:
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.floor.f64(double %x)
   ret double %a
 }
@@ -664,6 +880,20 @@ define double @test_ceil_double(double %x) {
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
 ; RV64IFD-NEXT:  .LBB41_2:
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_ceil_double:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI41_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI41_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB41_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rup
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rup
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB41_2:
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.ceil.f64(double %x)
   ret double %a
 }
@@ -686,6 +916,20 @@ define double @test_trunc_double(double %x) {
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
 ; RV64IFD-NEXT:  .LBB42_2:
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_trunc_double:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI42_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI42_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB42_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rtz
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rtz
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB42_2:
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.trunc.f64(double %x)
   ret double %a
 }
@@ -708,6 +952,20 @@ define double @test_round_double(double %x) {
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
 ; RV64IFD-NEXT:  .LBB43_2:
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_round_double:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI43_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI43_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB43_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rmm
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rmm
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB43_2:
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.round.f64(double %x)
   ret double %a
 }
@@ -730,6 +988,20 @@ define double @test_roundeven_double(double %x) {
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa5, fa0
 ; RV64IFD-NEXT:  .LBB44_2:
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: test_roundeven_double:
+; RV64IZFINXZDINX:       # %bb.0:
+; RV64IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI44_0)
+; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI44_0)(a1)
+; RV64IZFINXZDINX-NEXT:    fabs.d a2, a0
+; RV64IZFINXZDINX-NEXT:    flt.d a1, a2, a1
+; RV64IZFINXZDINX-NEXT:    beqz a1, .LBB44_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1:
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a0, rne
+; RV64IZFINXZDINX-NEXT:    fcvt.d.l a1, a1, rne
+; RV64IZFINXZDINX-NEXT:    fsgnj.d a0, a1, a0
+; RV64IZFINXZDINX-NEXT:  .LBB44_2:
+; RV64IZFINXZDINX-NEXT:    ret
   %a = call double @llvm.roundeven.f64(double %x)
   ret double %a
 }
diff --git a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
index b3cb0e8fb5063..f89b6aa21c1eb 100644
--- a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
@@ -3,12 +3,19 @@
 ; RUN:   -target-abi=ilp32d | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck --check-prefix=CHECKRV64ZDINX %s
 
 define double @select_fcmp_false(double %a, double %b) nounwind {
 ; CHECK-LABEL: select_fcmp_false:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_false:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp false double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -23,6 +30,15 @@ define double @select_fcmp_oeq(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_oeq:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB1_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB1_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -37,6 +53,15 @@ define double @select_fcmp_ogt(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ogt:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a1, a0
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB2_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB2_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ogt double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -51,6 +76,15 @@ define double @select_fcmp_oge(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_oge:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    fle.d a2, a1, a0
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB3_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB3_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp oge double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -65,6 +99,15 @@ define double @select_fcmp_olt(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB4_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_olt:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB4_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB4_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp olt double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -79,6 +122,15 @@ define double @select_fcmp_ole(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB5_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ole:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    fle.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB5_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB5_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ole double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -95,6 +147,17 @@ define double @select_fcmp_one(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB6_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_one:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    flt.d a3, a1, a0
+; CHECKRV64ZDINX-NEXT:    or a2, a3, a2
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB6_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB6_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp one double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -111,6 +174,17 @@ define double @select_fcmp_ord(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ord:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a2, a1, a1
+; CHECKRV64ZDINX-NEXT:    feq.d a3, a0, a0
+; CHECKRV64ZDINX-NEXT:    and a2, a3, a2
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB7_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB7_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ord double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -127,6 +201,17 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ueq:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    flt.d a3, a1, a0
+; CHECKRV64ZDINX-NEXT:    or a2, a3, a2
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB8_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB8_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ueq double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -141,6 +226,15 @@ define double @select_fcmp_ugt(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB9_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ugt:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    fle.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB9_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB9_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ugt double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -155,6 +249,15 @@ define double @select_fcmp_uge(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB10_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_uge:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB10_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB10_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp uge double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -169,6 +272,15 @@ define double @select_fcmp_ult(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB11_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ult:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    fle.d a2, a1, a0
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB11_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB11_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ult double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -183,6 +295,15 @@ define double @select_fcmp_ule(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB12_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_ule:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a1, a0
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB12_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB12_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ule double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -197,6 +318,15 @@ define double @select_fcmp_une(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB13_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_une:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a2, a0, a1
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB13_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB13_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp une double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -213,6 +343,17 @@ define double @select_fcmp_uno(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB14_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_uno:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a2, a1, a1
+; CHECKRV64ZDINX-NEXT:    feq.d a3, a0, a0
+; CHECKRV64ZDINX-NEXT:    and a2, a3, a2
+; CHECKRV64ZDINX-NEXT:    beqz a2, .LBB14_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:  .LBB14_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp uno double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -222,6 +363,10 @@ define double @select_fcmp_true(double %a, double %b) nounwind {
 ; CHECK-LABEL: select_fcmp_true:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_true:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp true double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -237,6 +382,16 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB16_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: i32_select_fcmp_oeq:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a1, a0, a1
+; CHECKRV64ZDINX-NEXT:    mv a0, a2
+; CHECKRV64ZDINX-NEXT:    bnez a1, .LBB16_2
+; CHECKRV64ZDINX-NEXT:  # %bb.1:
+; CHECKRV64ZDINX-NEXT:    mv a0, a3
+; CHECKRV64ZDINX-NEXT:  .LBB16_2:
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
   ret i32 %2
@@ -249,6 +404,13 @@ define i32 @select_fcmp_oeq_1_2(double %a, double %b) {
 ; CHECK-NEXT:    li a1, 2
 ; CHECK-NEXT:    sub a0, a1, a0
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_oeq_1_2:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    feq.d a0, a0, a1
+; CHECKRV64ZDINX-NEXT:    li a1, 2
+; CHECKRV64ZDINX-NEXT:    sub a0, a1, a0
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp fast oeq double %a, %b
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
@@ -260,6 +422,12 @@ define signext i32 @select_fcmp_uge_negone_zero(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fle.d a0, fa0, fa1
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_uge_negone_zero:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    fle.d a0, a0, a1
+; CHECKRV64ZDINX-NEXT:    addi a0, a0, -1
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ugt double %a, %b
   %2 = select i1 %1, i32 -1, i32 0
   ret i32 %2
@@ -271,6 +439,12 @@ define signext i32 @select_fcmp_uge_1_2(double %a, double %b) nounwind {
 ; CHECK-NEXT:    fle.d a0, fa0, fa1
 ; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: select_fcmp_uge_1_2:
+; CHECKRV64ZDINX:       # %bb.0:
+; CHECKRV64ZDINX-NEXT:    fle.d a0, a0, a1
+; CHECKRV64ZDINX-NEXT:    addi a0, a0, 1
+; CHECKRV64ZDINX-NEXT:    ret
   %1 = fcmp ugt double %a, %b
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
diff --git a/llvm/test/CodeGen/RISCV/double-select-icmp.ll b/llvm/test/CodeGen/RISCV/double-select-icmp.ll
index 429b791baf199..f29eaa5985b2e 100644
--- a/llvm/test/CodeGen/RISCV/double-select-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-select-icmp.ll
@@ -4,6 +4,8 @@
 ; RUN:   -target-abi=ilp32d | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64d | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=RV64ZDINX %s
 
 define double @select_icmp_eq(i32 signext %a, i32 signext %b, double %c, double %d) {
 ; CHECK-LABEL: select_icmp_eq:
@@ -13,6 +15,15 @@ define double @select_icmp_eq(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_eq:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    beq a0, a1, .LBB0_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB0_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp eq i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -26,6 +37,15 @@ define double @select_icmp_ne(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_ne:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bne a0, a1, .LBB1_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB1_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp ne i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -39,6 +59,15 @@ define double @select_icmp_ugt(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_ugt:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bltu a1, a0, .LBB2_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB2_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp ugt i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -52,6 +81,15 @@ define double @select_icmp_uge(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_uge:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bgeu a0, a1, .LBB3_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB3_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp uge i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -65,6 +103,15 @@ define double @select_icmp_ult(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB4_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_ult:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bltu a0, a1, .LBB4_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB4_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp ult i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -78,6 +125,15 @@ define double @select_icmp_ule(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB5_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_ule:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bgeu a1, a0, .LBB5_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB5_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp ule i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -91,6 +147,15 @@ define double @select_icmp_sgt(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB6_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_sgt:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    blt a1, a0, .LBB6_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB6_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp sgt i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -104,6 +169,15 @@ define double @select_icmp_sge(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_sge:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bge a0, a1, .LBB7_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB7_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp sge i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -117,6 +191,15 @@ define double @select_icmp_slt(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_slt:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    blt a0, a1, .LBB8_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB8_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp slt i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
@@ -130,6 +213,15 @@ define double @select_icmp_sle(i32 signext %a, i32 signext %b, double %c, double
 ; CHECK-NEXT:    fmv.d fa0, fa1
 ; CHECK-NEXT:  .LBB9_2:
 ; CHECK-NEXT:    ret
+;
+; RV64ZDINX-LABEL: select_icmp_sle:
+; RV64ZDINX:       # %bb.0:
+; RV64ZDINX-NEXT:    bge a1, a0, .LBB9_2
+; RV64ZDINX-NEXT:  # %bb.1:
+; RV64ZDINX-NEXT:    mv a2, a3
+; RV64ZDINX-NEXT:  .LBB9_2:
+; RV64ZDINX-NEXT:    mv a0, a2
+; RV64ZDINX-NEXT:    ret
   %1 = icmp sle i32 %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
diff --git a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
index 95438fda9a707..0ab98a75e9dbf 100644
--- a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
+++ b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi=lp64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+zdinx -target-abi=lp64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFINXZDINX %s
 
 define double @func(double %d, i32 %n) nounwind {
 ; RV32IFD-LABEL: func:
@@ -55,6 +57,28 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV64IFD-NEXT:  .LBB0_2: # %return
 ; RV64IFD-NEXT:    fmv.x.d a0, fa5
 ; RV64IFD-NEXT:    ret
+;
+; RV64IZFINXZDINX-LABEL: func:
+; RV64IZFINXZDINX:       # %bb.0: # %entry
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV64IZFINXZDINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64IZFINXZDINX-NEXT:    sext.w a2, a1
+; RV64IZFINXZDINX-NEXT:    mv s0, a0
+; RV64IZFINXZDINX-NEXT:    beqz a2, .LBB0_2
+; RV64IZFINXZDINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINXZDINX-NEXT:    addiw a1, a1, -1
+; RV64IZFINXZDINX-NEXT:    mv a0, s0
+; RV64IZFINXZDINX-NEXT:    call func@plt
+; RV64IZFINXZDINX-NEXT:    fadd.d a0, a0, s0
+; RV64IZFINXZDINX-NEXT:    j .LBB0_3
+; RV64IZFINXZDINX-NEXT:  .LBB0_2: # %return
+; RV64IZFINXZDINX-NEXT:    mv a0, s0
+; RV64IZFINXZDINX-NEXT:  .LBB0_3: # %return
+; RV64IZFINXZDINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IZFINXZDINX-NEXT:    addi sp, sp, 16
+; RV64IZFINXZDINX-NEXT:    ret
 entry:
   %cmp = icmp eq i32 %n, 0
   br i1 %cmp, label %return, label %if.else
diff --git a/llvm/test/CodeGen/RISCV/float-arith-strict.ll b/llvm/test/CodeGen/RISCV/float-arith-strict.ll
index 3c6aceca529ad..0252c8ca0f727 100644
--- a/llvm/test/CodeGen/RISCV/float-arith-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-arith-strict.ll
@@ -9,6 +9,12 @@
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=ilp32 \
+; RUN:   | FileCheck -check-prefixes=CHECKIZFINX,RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefixes=CHECKIZFINX,RV64IZFINX %s
 
 define float @fadd_s(float %a, float %b) nounwind strictfp {
 ; CHECKIF-LABEL: fadd_s:
@@ -33,6 +39,11 @@ define float @fadd_s(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fadd_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.fadd.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -61,6 +72,11 @@ define float @fsub_s(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fsub_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fsub.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.fsub.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -89,6 +105,11 @@ define float @fmul_s(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fmul_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmul.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.fmul.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -117,6 +138,11 @@ define float @fdiv_s(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fdiv_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fdiv.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.fdiv.f32(float %a, float %b, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -145,6 +171,11 @@ define float @fsqrt_s(float %a) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fsqrt_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fsqrt.s a0, a0
+; CHECKIZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.sqrt.f32(float %a, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -186,6 +217,24 @@ define float @fmin_s(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; RV32IZFINX-LABEL: fmin_s:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call fminf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fmin_s:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call fminf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.minnum.f32(float %a, float %b, metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -227,6 +276,24 @@ define float @fmax_s(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; RV32IZFINX-LABEL: fmax_s:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call fmaxf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fmax_s:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call fmaxf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.maxnum.f32(float %a, float %b, metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -255,6 +322,11 @@ define float @fmadd_s(float %a, float %b, float %c) nounwind strictfp {
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fmadd_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
   %1 = call float @llvm.experimental.constrained.fma.f32(float %a, float %b, float %c, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
   ret float %1
 }
@@ -311,6 +383,12 @@ define float @fmsub_s(float %a, float %b, float %c) nounwind strictfp {
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fmsub_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fmsub.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
   %c_ = fadd float 0.0, %c ; avoid negation using xor
   %negc = fneg float %c_
   %1 = call float @llvm.experimental.constrained.fma.f32(float %a, float %b, float %negc, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
@@ -381,6 +459,13 @@ define float @fnmadd_s(float %a, float %b, float %c) nounwind strictfp {
 ; RV64I-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fnmadd_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, zero
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fnmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
   %a_ = fadd float 0.0, %a
   %c_ = fadd float 0.0, %c
   %nega = fneg float %a_
@@ -453,6 +538,13 @@ define float @fnmadd_s_2(float %a, float %b, float %c) nounwind strictfp {
 ; RV64I-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fnmadd_s_2:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, zero
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fnmadd.s a0, a1, a0, a2
+; CHECKIZFINX-NEXT:    ret
   %b_ = fadd float 0.0, %b
   %c_ = fadd float 0.0, %c
   %negb = fneg float %b_
@@ -510,6 +602,12 @@ define float @fnmsub_s(float %a, float %b, float %c) nounwind strictfp {
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fnmsub_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, zero
+; CHECKIZFINX-NEXT:    fnmsub.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
   %a_ = fadd float 0.0, %a
   %nega = fneg float %a_
   %1 = call float @llvm.experimental.constrained.fma.f32(float %nega, float %b, float %c, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
@@ -567,6 +665,12 @@ define float @fnmsub_s_2(float %a, float %b, float %c) nounwind strictfp {
 ; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fnmsub_s_2:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, zero
+; CHECKIZFINX-NEXT:    fnmsub.s a0, a1, a0, a2
+; CHECKIZFINX-NEXT:    ret
   %b_ = fadd float 0.0, %b
   %negb = fneg float %b_
   %1 = call float @llvm.experimental.constrained.fma.f32(float %a, float %negb, float %c, metadata !"round.dynamic", metadata !"fpexcept.strict") strictfp
diff --git a/llvm/test/CodeGen/RISCV/float-arith.ll b/llvm/test/CodeGen/RISCV/float-arith.ll
index d6bb92736f01c..5497827a3f2fa 100644
--- a/llvm/test/CodeGen/RISCV/float-arith.ll
+++ b/llvm/test/CodeGen/RISCV/float-arith.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefix=CHECKIF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefix=CHECKIF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefix=CHECKIZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=CHECKIZFINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -19,6 +23,11 @@ define float @fadd_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fadd.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fadd_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fadd_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -46,6 +55,11 @@ define float @fsub_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fsub.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fsub_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fsub.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsub_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -73,6 +87,11 @@ define float @fmul_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fmul.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmul_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmul.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmul_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -100,6 +119,11 @@ define float @fdiv_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fdiv.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fdiv_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fdiv.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fdiv_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -129,6 +153,11 @@ define float @fsqrt_s(float %a) nounwind {
 ; CHECKIF-NEXT:    fsqrt.s fa0, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fsqrt_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fsqrt.s a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsqrt_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -158,6 +187,11 @@ define float @fsgnj_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fsgnj.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fsgnj_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fsgnj.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsgnj_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
@@ -187,6 +221,13 @@ define i32 @fneg_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    feq.s a0, fa5, fa4
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fneg_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a0
+; CHECKIZFINX-NEXT:    fneg.s a1, a0
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fneg_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -228,6 +269,12 @@ define float @fsgnjn_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fsgnjn.s fa0, fa0, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fsgnjn_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a1, a0, a1
+; CHECKIZFINX-NEXT:    fsgnjn.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fsgnjn_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -279,6 +326,13 @@ define float @fabs_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fadd.s fa0, fa4, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fabs_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKIZFINX-NEXT:    fabs.s a1, a0
+; CHECKIZFINX-NEXT:    fadd.s a0, a1, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fabs_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -318,6 +372,11 @@ define float @fmin_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fmin.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmin_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmin.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmin_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -347,6 +406,11 @@ define float @fmax_s(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fmax.s fa0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmax_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmax.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmax_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -376,6 +440,11 @@ define float @fmadd_s(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmadd_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmadd_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -405,6 +474,12 @@ define float @fmsub_s(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fmsub.s fa0, fa0, fa1, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmsub_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fmsub.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmsub_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -463,6 +538,13 @@ define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmadd.s fa0, fa4, fa1, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmadd_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, zero
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fnmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -535,6 +617,13 @@ define float @fnmadd_s_2(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmadd.s fa0, fa4, fa0, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmadd_s_2:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, zero
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fnmadd.s a0, a1, a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_s_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -617,6 +706,13 @@ define float @fnmadd_s_3(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fneg.s fa0, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmadd_s_3:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    lui a1, 524288
+; CHECKIZFINX-NEXT:    xor a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_s_3:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -659,6 +755,13 @@ define float @fnmadd_nsz(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmadd.s fa0, fa0, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmadd_nsz:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    lui a1, 524288
+; CHECKIZFINX-NEXT:    xor a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_nsz:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -693,6 +796,12 @@ define float @fnmsub_s(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmsub.s fa0, fa5, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmsub_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, zero
+; CHECKIZFINX-NEXT:    fnmsub.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -748,6 +857,12 @@ define float @fnmsub_s_2(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmsub.s fa0, fa5, fa0, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmsub_s_2:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, zero
+; CHECKIZFINX-NEXT:    fnmsub.s a0, a1, a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_s_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -803,6 +918,11 @@ define float @fmadd_s_contract(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmadd_s_contract:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmadd_s_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -843,6 +963,12 @@ define float @fmsub_s_contract(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fmsub.s fa0, fa0, fa1, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmsub_s_contract:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fmsub.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmsub_s_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -908,6 +1034,14 @@ define float @fnmadd_s_contract(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmadd.s fa0, fa4, fa3, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmadd_s_contract:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, zero
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, zero
+; CHECKIZFINX-NEXT:    fadd.s a2, a2, zero
+; CHECKIZFINX-NEXT:    fnmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmadd_s_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -993,6 +1127,13 @@ define float @fnmsub_s_contract(float %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fnmsub.s fa0, fa4, fa5, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fnmsub_s_contract:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, zero
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, zero
+; CHECKIZFINX-NEXT:    fnmsub.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fnmsub_s_contract:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll b/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll
index f0c34317a5629..5c50381ad1701 100644
--- a/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll
@@ -1,10 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi=ilp32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32F %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -target-abi=ilp32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32ZFINX %s
 ; RUN: llc -mtriple=riscv32 -mattr=+f,+d -target-abi=ilp32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32FD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi=lp64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64F %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -target-abi=lp64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64ZFINX %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f,+d -target-abi=lp64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64FD %s
 
@@ -25,6 +29,13 @@ define float @bitcast_and(float %a1, float %a2) nounwind {
 ; RV32F-NEXT:    fmv.x.w a0, fa5
 ; RV32F-NEXT:    ret
 ;
+; RV32ZFINX-LABEL: bitcast_and:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    fadd.s a1, a0, a1
+; RV32ZFINX-NEXT:    fabs.s a1, a1
+; RV32ZFINX-NEXT:    fadd.s a0, a0, a1
+; RV32ZFINX-NEXT:    ret
+;
 ; RV32FD-LABEL: bitcast_and:
 ; RV32FD:       # %bb.0:
 ; RV32FD-NEXT:    fmv.w.x fa5, a1
@@ -45,6 +56,13 @@ define float @bitcast_and(float %a1, float %a2) nounwind {
 ; RV64F-NEXT:    fmv.x.w a0, fa5
 ; RV64F-NEXT:    ret
 ;
+; RV64ZFINX-LABEL: bitcast_and:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    fadd.s a1, a0, a1
+; RV64ZFINX-NEXT:    fabs.s a1, a1
+; RV64ZFINX-NEXT:    fadd.s a0, a0, a1
+; RV64ZFINX-NEXT:    ret
+;
 ; RV64FD-LABEL: bitcast_and:
 ; RV64FD:       # %bb.0:
 ; RV64FD-NEXT:    fmv.w.x fa5, a1
@@ -84,6 +102,27 @@ define double @bitcast_double_and(double %a1, double %a2) nounwind {
 ; RV32F-NEXT:    addi sp, sp, 16
 ; RV32F-NEXT:    ret
 ;
+; RV32ZFINX-LABEL: bitcast_double_and:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    addi sp, sp, -16
+; RV32ZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    mv s0, a1
+; RV32ZFINX-NEXT:    mv s1, a0
+; RV32ZFINX-NEXT:    call __adddf3@plt
+; RV32ZFINX-NEXT:    mv a2, a0
+; RV32ZFINX-NEXT:    slli a1, a1, 1
+; RV32ZFINX-NEXT:    srli a3, a1, 1
+; RV32ZFINX-NEXT:    mv a0, s1
+; RV32ZFINX-NEXT:    mv a1, s0
+; RV32ZFINX-NEXT:    call __adddf3@plt
+; RV32ZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    addi sp, sp, 16
+; RV32ZFINX-NEXT:    ret
+;
 ; RV32FD-LABEL: bitcast_double_and:
 ; RV32FD:       # %bb.0:
 ; RV32FD-NEXT:    addi sp, sp, -16
@@ -118,6 +157,22 @@ define double @bitcast_double_and(double %a1, double %a2) nounwind {
 ; RV64F-NEXT:    addi sp, sp, 16
 ; RV64F-NEXT:    ret
 ;
+; RV64ZFINX-LABEL: bitcast_double_and:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    addi sp, sp, -16
+; RV64ZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ZFINX-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64ZFINX-NEXT:    mv s0, a0
+; RV64ZFINX-NEXT:    call __adddf3@plt
+; RV64ZFINX-NEXT:    slli a0, a0, 1
+; RV64ZFINX-NEXT:    srli a1, a0, 1
+; RV64ZFINX-NEXT:    mv a0, s0
+; RV64ZFINX-NEXT:    call __adddf3@plt
+; RV64ZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ZFINX-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64ZFINX-NEXT:    addi sp, sp, 16
+; RV64ZFINX-NEXT:    ret
+;
 ; RV64FD-LABEL: bitcast_double_and:
 ; RV64FD:       # %bb.0:
 ; RV64FD-NEXT:    fmv.d.x fa5, a1
@@ -147,6 +202,13 @@ define float @bitcast_xor(float %a1, float %a2) nounwind {
 ; RV32F-NEXT:    fmv.x.w a0, fa5
 ; RV32F-NEXT:    ret
 ;
+; RV32ZFINX-LABEL: bitcast_xor:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    fmul.s a1, a0, a1
+; RV32ZFINX-NEXT:    fneg.s a1, a1
+; RV32ZFINX-NEXT:    fmul.s a0, a0, a1
+; RV32ZFINX-NEXT:    ret
+;
 ; RV32FD-LABEL: bitcast_xor:
 ; RV32FD:       # %bb.0:
 ; RV32FD-NEXT:    fmv.w.x fa5, a1
@@ -167,6 +229,13 @@ define float @bitcast_xor(float %a1, float %a2) nounwind {
 ; RV64F-NEXT:    fmv.x.w a0, fa5
 ; RV64F-NEXT:    ret
 ;
+; RV64ZFINX-LABEL: bitcast_xor:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    fmul.s a1, a0, a1
+; RV64ZFINX-NEXT:    fneg.s a1, a1
+; RV64ZFINX-NEXT:    fmul.s a0, a0, a1
+; RV64ZFINX-NEXT:    ret
+;
 ; RV64FD-LABEL: bitcast_xor:
 ; RV64FD:       # %bb.0:
 ; RV64FD-NEXT:    fmv.w.x fa5, a1
@@ -206,6 +275,27 @@ define double @bitcast_double_xor(double %a1, double %a2) nounwind {
 ; RV32F-NEXT:    addi sp, sp, 16
 ; RV32F-NEXT:    ret
 ;
+; RV32ZFINX-LABEL: bitcast_double_xor:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    addi sp, sp, -16
+; RV32ZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    mv s0, a1
+; RV32ZFINX-NEXT:    mv s1, a0
+; RV32ZFINX-NEXT:    call __muldf3@plt
+; RV32ZFINX-NEXT:    mv a2, a0
+; RV32ZFINX-NEXT:    lui a3, 524288
+; RV32ZFINX-NEXT:    xor a3, a1, a3
+; RV32ZFINX-NEXT:    mv a0, s1
+; RV32ZFINX-NEXT:    mv a1, s0
+; RV32ZFINX-NEXT:    call __muldf3@plt
+; RV32ZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    addi sp, sp, 16
+; RV32ZFINX-NEXT:    ret
+;
 ; RV32FD-LABEL: bitcast_double_xor:
 ; RV32FD:       # %bb.0:
 ; RV32FD-NEXT:    addi sp, sp, -16
@@ -241,6 +331,23 @@ define double @bitcast_double_xor(double %a1, double %a2) nounwind {
 ; RV64F-NEXT:    addi sp, sp, 16
 ; RV64F-NEXT:    ret
 ;
+; RV64ZFINX-LABEL: bitcast_double_xor:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    addi sp, sp, -16
+; RV64ZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ZFINX-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64ZFINX-NEXT:    mv s0, a0
+; RV64ZFINX-NEXT:    call __muldf3@plt
+; RV64ZFINX-NEXT:    li a1, -1
+; RV64ZFINX-NEXT:    slli a1, a1, 63
+; RV64ZFINX-NEXT:    xor a1, a0, a1
+; RV64ZFINX-NEXT:    mv a0, s0
+; RV64ZFINX-NEXT:    call __muldf3@plt
+; RV64ZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ZFINX-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64ZFINX-NEXT:    addi sp, sp, 16
+; RV64ZFINX-NEXT:    ret
+;
 ; RV64FD-LABEL: bitcast_double_xor:
 ; RV64FD:       # %bb.0:
 ; RV64FD-NEXT:    fmv.d.x fa5, a1
@@ -270,6 +377,14 @@ define float @bitcast_or(float %a1, float %a2) nounwind {
 ; RV32F-NEXT:    fmv.x.w a0, fa5
 ; RV32F-NEXT:    ret
 ;
+; RV32ZFINX-LABEL: bitcast_or:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    fmul.s a1, a0, a1
+; RV32ZFINX-NEXT:    fabs.s a1, a1
+; RV32ZFINX-NEXT:    fneg.s a1, a1
+; RV32ZFINX-NEXT:    fmul.s a0, a0, a1
+; RV32ZFINX-NEXT:    ret
+;
 ; RV32FD-LABEL: bitcast_or:
 ; RV32FD:       # %bb.0:
 ; RV32FD-NEXT:    fmv.w.x fa5, a1
@@ -292,6 +407,14 @@ define float @bitcast_or(float %a1, float %a2) nounwind {
 ; RV64F-NEXT:    fmv.x.w a0, fa5
 ; RV64F-NEXT:    ret
 ;
+; RV64ZFINX-LABEL: bitcast_or:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    fmul.s a1, a0, a1
+; RV64ZFINX-NEXT:    fabs.s a1, a1
+; RV64ZFINX-NEXT:    fneg.s a1, a1
+; RV64ZFINX-NEXT:    fmul.s a0, a0, a1
+; RV64ZFINX-NEXT:    ret
+;
 ; RV64FD-LABEL: bitcast_or:
 ; RV64FD:       # %bb.0:
 ; RV64FD-NEXT:    fmv.w.x fa5, a1
@@ -332,6 +455,27 @@ define double @bitcast_double_or(double %a1, double %a2) nounwind {
 ; RV32F-NEXT:    addi sp, sp, 16
 ; RV32F-NEXT:    ret
 ;
+; RV32ZFINX-LABEL: bitcast_double_or:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    addi sp, sp, -16
+; RV32ZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32ZFINX-NEXT:    mv s0, a1
+; RV32ZFINX-NEXT:    mv s1, a0
+; RV32ZFINX-NEXT:    call __muldf3@plt
+; RV32ZFINX-NEXT:    mv a2, a0
+; RV32ZFINX-NEXT:    lui a3, 524288
+; RV32ZFINX-NEXT:    or a3, a1, a3
+; RV32ZFINX-NEXT:    mv a0, s1
+; RV32ZFINX-NEXT:    mv a1, s0
+; RV32ZFINX-NEXT:    call __muldf3@plt
+; RV32ZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32ZFINX-NEXT:    addi sp, sp, 16
+; RV32ZFINX-NEXT:    ret
+;
 ; RV32FD-LABEL: bitcast_double_or:
 ; RV32FD:       # %bb.0:
 ; RV32FD-NEXT:    addi sp, sp, -16
@@ -368,6 +512,23 @@ define double @bitcast_double_or(double %a1, double %a2) nounwind {
 ; RV64F-NEXT:    addi sp, sp, 16
 ; RV64F-NEXT:    ret
 ;
+; RV64ZFINX-LABEL: bitcast_double_or:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    addi sp, sp, -16
+; RV64ZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64ZFINX-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64ZFINX-NEXT:    mv s0, a0
+; RV64ZFINX-NEXT:    call __muldf3@plt
+; RV64ZFINX-NEXT:    li a1, -1
+; RV64ZFINX-NEXT:    slli a1, a1, 63
+; RV64ZFINX-NEXT:    or a1, a0, a1
+; RV64ZFINX-NEXT:    mv a0, s0
+; RV64ZFINX-NEXT:    call __muldf3@plt
+; RV64ZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64ZFINX-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64ZFINX-NEXT:    addi sp, sp, 16
+; RV64ZFINX-NEXT:    ret
+;
 ; RV64FD-LABEL: bitcast_double_or:
 ; RV64FD:       # %bb.0:
 ; RV64FD-NEXT:    fmv.d.x fa5, a1
diff --git a/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
index 3741e86480407..3324d366cf0e4 100644
--- a/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
@@ -3,10 +3,14 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IZFINX %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFINX %s
 
 ; This file tests cases where simple floating point operations can be
 ; profitably handled though bit manipulation if a soft-float ABI is being used
@@ -27,6 +31,12 @@ define float @fneg(float %a) nounwind {
 ; RV32IF-NEXT:    xor a0, a0, a1
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fneg:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 524288
+; RV32IZFINX-NEXT:    xor a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64I-LABEL: fneg:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a1, 524288
@@ -38,6 +48,12 @@ define float @fneg(float %a) nounwind {
 ; RV64IF-NEXT:    lui a1, 524288
 ; RV64IF-NEXT:    xor a0, a0, a1
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fneg:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 524288
+; RV64IZFINX-NEXT:    xor a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %1 = fneg float %a
   ret float %1
 }
@@ -57,6 +73,12 @@ define float @fabs(float %a) nounwind {
 ; RV32IF-NEXT:    srli a0, a0, 1
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fabs:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    slli a0, a0, 1
+; RV32IZFINX-NEXT:    srli a0, a0, 1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64I-LABEL: fabs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 33
@@ -68,6 +90,12 @@ define float @fabs(float %a) nounwind {
 ; RV64IF-NEXT:    slli a0, a0, 33
 ; RV64IF-NEXT:    srli a0, a0, 33
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fabs:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    slli a0, a0, 33
+; RV64IZFINX-NEXT:    srli a0, a0, 33
+; RV64IZFINX-NEXT:    ret
   %1 = call float @llvm.fabs.f32(float %a)
   ret float %1
 }
@@ -99,6 +127,13 @@ define float @fcopysign_fneg(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.x.w a0, fa5
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcopysign_fneg:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a2, 524288
+; RV32IZFINX-NEXT:    xor a1, a1, a2
+; RV32IZFINX-NEXT:    fsgnj.s a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64I-LABEL: fcopysign_fneg:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    not a1, a1
@@ -116,6 +151,11 @@ define float @fcopysign_fneg(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    fsgnjn.s fa5, fa4, fa5
 ; RV64IF-NEXT:    fmv.x.w a0, fa5
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcopysign_fneg:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fsgnjn.s a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %1 = fneg float %b
   %2 = call float @llvm.copysign.f32(float %a, float %1)
   ret float %2
diff --git a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
index f9901e6fc9dc5..71b0f77015b56 100644
--- a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefix=RV32IF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefix=RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefix=RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=RV64IZFINX %s
 
 declare void @abort()
 declare void @exit(i32)
@@ -30,6 +34,28 @@ define void @br_fcmp_false(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_false:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    li a0, 1
+; RV32IZFINX-NEXT:    bnez a0, .LBB0_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.then
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB0_2: # %if.else
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_false:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    li a0, 1
+; RV64IZFINX-NEXT:    bnez a0, .LBB0_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.then
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB0_2: # %if.else
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp false float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -61,6 +87,28 @@ define void @br_fcmp_oeq(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_oeq:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    feq.s a0, a0, a1
+; RV32IZFINX-NEXT:    bnez a0, .LBB1_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB1_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_oeq:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    feq.s a0, a0, a1
+; RV64IZFINX-NEXT:    bnez a0, .LBB1_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB1_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp oeq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -95,6 +143,28 @@ define void @br_fcmp_oeq_alt(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_oeq_alt:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    feq.s a0, a0, a1
+; RV32IZFINX-NEXT:    bnez a0, .LBB2_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB2_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_oeq_alt:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    feq.s a0, a0, a1
+; RV64IZFINX-NEXT:    bnez a0, .LBB2_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB2_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp oeq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -126,6 +196,28 @@ define void @br_fcmp_ogt(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ogt:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    bnez a0, .LBB3_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB3_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ogt:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    flt.s a0, a1, a0
+; RV64IZFINX-NEXT:    bnez a0, .LBB3_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB3_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ogt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -157,6 +249,28 @@ define void @br_fcmp_oge(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_oge:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fle.s a0, a1, a0
+; RV32IZFINX-NEXT:    bnez a0, .LBB4_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB4_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_oge:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fle.s a0, a1, a0
+; RV64IZFINX-NEXT:    bnez a0, .LBB4_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB4_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp oge float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -188,6 +302,28 @@ define void @br_fcmp_olt(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_olt:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    flt.s a0, a0, a1
+; RV32IZFINX-NEXT:    bnez a0, .LBB5_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB5_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_olt:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    flt.s a0, a0, a1
+; RV64IZFINX-NEXT:    bnez a0, .LBB5_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB5_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp olt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -219,6 +355,28 @@ define void @br_fcmp_ole(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ole:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fle.s a0, a0, a1
+; RV32IZFINX-NEXT:    bnez a0, .LBB6_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB6_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ole:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fle.s a0, a0, a1
+; RV64IZFINX-NEXT:    bnez a0, .LBB6_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB6_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ole float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -254,6 +412,32 @@ define void @br_fcmp_one(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_one:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    flt.s a2, a0, a1
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    or a0, a0, a2
+; RV32IZFINX-NEXT:    bnez a0, .LBB7_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB7_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_one:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    flt.s a2, a0, a1
+; RV64IZFINX-NEXT:    flt.s a0, a1, a0
+; RV64IZFINX-NEXT:    or a0, a0, a2
+; RV64IZFINX-NEXT:    bnez a0, .LBB7_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB7_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp one float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -289,6 +473,32 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ord:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    feq.s a1, a1, a1
+; RV32IZFINX-NEXT:    feq.s a0, a0, a0
+; RV32IZFINX-NEXT:    and a0, a0, a1
+; RV32IZFINX-NEXT:    bnez a0, .LBB8_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB8_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ord:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    feq.s a1, a1, a1
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    bnez a0, .LBB8_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB8_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ord float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -324,6 +534,32 @@ define void @br_fcmp_ueq(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ueq:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    flt.s a2, a0, a1
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    or a0, a0, a2
+; RV32IZFINX-NEXT:    beqz a0, .LBB9_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB9_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ueq:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    flt.s a2, a0, a1
+; RV64IZFINX-NEXT:    flt.s a0, a1, a0
+; RV64IZFINX-NEXT:    or a0, a0, a2
+; RV64IZFINX-NEXT:    beqz a0, .LBB9_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB9_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ueq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -355,6 +591,28 @@ define void @br_fcmp_ugt(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ugt:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fle.s a0, a0, a1
+; RV32IZFINX-NEXT:    beqz a0, .LBB10_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB10_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ugt:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fle.s a0, a0, a1
+; RV64IZFINX-NEXT:    beqz a0, .LBB10_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB10_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ugt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -386,6 +644,28 @@ define void @br_fcmp_uge(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_uge:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    flt.s a0, a0, a1
+; RV32IZFINX-NEXT:    beqz a0, .LBB11_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB11_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_uge:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    flt.s a0, a0, a1
+; RV64IZFINX-NEXT:    beqz a0, .LBB11_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB11_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp uge float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -417,6 +697,28 @@ define void @br_fcmp_ult(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ult:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fle.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB12_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB12_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ult:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fle.s a0, a1, a0
+; RV64IZFINX-NEXT:    beqz a0, .LBB12_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB12_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ult float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -448,6 +750,28 @@ define void @br_fcmp_ule(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_ule:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB13_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB13_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_ule:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    flt.s a0, a1, a0
+; RV64IZFINX-NEXT:    beqz a0, .LBB13_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB13_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp ule float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -479,6 +803,28 @@ define void @br_fcmp_une(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_une:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    feq.s a0, a0, a1
+; RV32IZFINX-NEXT:    beqz a0, .LBB14_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB14_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_une:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    feq.s a0, a0, a1
+; RV64IZFINX-NEXT:    beqz a0, .LBB14_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB14_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp une float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -514,6 +860,32 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_uno:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    feq.s a1, a1, a1
+; RV32IZFINX-NEXT:    feq.s a0, a0, a0
+; RV32IZFINX-NEXT:    and a0, a0, a1
+; RV32IZFINX-NEXT:    beqz a0, .LBB15_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB15_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_uno:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    feq.s a1, a1, a1
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    beqz a0, .LBB15_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB15_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp uno float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -545,6 +917,28 @@ define void @br_fcmp_true(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_true:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    li a0, 1
+; RV32IZFINX-NEXT:    bnez a0, .LBB16_2
+; RV32IZFINX-NEXT:  # %bb.1: # %if.else
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB16_2: # %if.then
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_true:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    li a0, 1
+; RV64IZFINX-NEXT:    bnez a0, .LBB16_2
+; RV64IZFINX-NEXT:  # %bb.1: # %if.else
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB16_2: # %if.then
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call abort@plt
   %1 = fcmp true float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -604,6 +998,48 @@ define i32 @br_fcmp_store_load_stack_slot(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB17_3: # %if.then
 ; RV64IF-NEXT:    call abort@plt
+;
+; RV32IZFINX-LABEL: br_fcmp_store_load_stack_slot:
+; RV32IZFINX:       # %bb.0: # %entry
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    li a0, 0
+; RV32IZFINX-NEXT:    call dummy@plt
+; RV32IZFINX-NEXT:    feq.s a0, a0, zero
+; RV32IZFINX-NEXT:    beqz a0, .LBB17_3
+; RV32IZFINX-NEXT:  # %bb.1: # %if.end
+; RV32IZFINX-NEXT:    li a0, 0
+; RV32IZFINX-NEXT:    call dummy@plt
+; RV32IZFINX-NEXT:    feq.s a0, a0, zero
+; RV32IZFINX-NEXT:    beqz a0, .LBB17_3
+; RV32IZFINX-NEXT:  # %bb.2: # %if.end4
+; RV32IZFINX-NEXT:    li a0, 0
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+; RV32IZFINX-NEXT:  .LBB17_3: # %if.then
+; RV32IZFINX-NEXT:    call abort@plt
+;
+; RV64IZFINX-LABEL: br_fcmp_store_load_stack_slot:
+; RV64IZFINX:       # %bb.0: # %entry
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    li a0, 0
+; RV64IZFINX-NEXT:    call dummy@plt
+; RV64IZFINX-NEXT:    feq.s a0, a0, zero
+; RV64IZFINX-NEXT:    beqz a0, .LBB17_3
+; RV64IZFINX-NEXT:  # %bb.1: # %if.end
+; RV64IZFINX-NEXT:    li a0, 0
+; RV64IZFINX-NEXT:    call dummy@plt
+; RV64IZFINX-NEXT:    feq.s a0, a0, zero
+; RV64IZFINX-NEXT:    beqz a0, .LBB17_3
+; RV64IZFINX-NEXT:  # %bb.2: # %if.end4
+; RV64IZFINX-NEXT:    li a0, 0
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+; RV64IZFINX-NEXT:  .LBB17_3: # %if.then
+; RV64IZFINX-NEXT:    call abort@plt
 entry:
   %call = call float @dummy(float 0.000000e+00)
   %cmp = fcmp une float %call, 0.000000e+00
diff --git a/llvm/test/CodeGen/RISCV/float-convert-strict.ll b/llvm/test/CodeGen/RISCV/float-convert-strict.ll
index 47ebb49a8923b..38fe0b911005f 100644
--- a/llvm/test/CodeGen/RISCV/float-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert-strict.ll
@@ -5,6 +5,12 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation -target-abi=lp64f \
 ; RUN:   | FileCheck -check-prefixes=CHECKIF,RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=ilp32 \
+; RUN:   | FileCheck -check-prefixes=CHECKIZFINX,RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefixes=CHECKIZFINX,RV64IZFINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -20,6 +26,11 @@ define i32 @fcvt_w_s(float %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_w_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -48,6 +59,11 @@ define i32 @fcvt_wu_s(float %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_wu_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -80,6 +96,13 @@ define i32 @fcvt_wu_s_multiple_use(float %x, ptr %y) nounwind {
 ; CHECKIF-NEXT:    add a0, a0, a1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_wu_s_multiple_use:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZFINX-NEXT:    seqz a1, a0
+; CHECKIZFINX-NEXT:    add a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_multiple_use:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -113,6 +136,11 @@ define float @fcvt_s_w(i32 %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -143,6 +171,12 @@ define float @fcvt_s_w_load(ptr %p) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w_load:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    lw a0, 0(a0)
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -173,6 +207,11 @@ define float @fcvt_s_wu(i32 %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_wu:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -209,6 +248,18 @@ define float @fcvt_s_wu_load(ptr %p) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.s.wu fa0, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_wu_load:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lw a0, 0(a0)
+; RV32IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_wu_load:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lwu a0, 0(a0)
+; RV64IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -248,6 +299,20 @@ define i64 @fcvt_l_s(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_l_s:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_l_s:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_l_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -285,6 +350,20 @@ define i64 @fcvt_lu_s(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_lu_s:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_lu_s:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_lu_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -322,6 +401,20 @@ define float @fcvt_s_l(i64 %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.s.l fa0, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_l:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __floatdisf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_l:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.l a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_l:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -359,6 +452,20 @@ define float @fcvt_s_lu(i64 %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.s.lu fa0, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_lu:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __floatundisf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_lu:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.lu a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_lu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -387,6 +494,11 @@ define float @fcvt_s_w_i8(i8 signext %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w_i8:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -415,6 +527,11 @@ define float @fcvt_s_wu_i8(i8 zeroext %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_wu_i8:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -443,6 +560,11 @@ define float @fcvt_s_w_i16(i16 signext %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w_i16:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -471,6 +593,11 @@ define float @fcvt_s_wu_i16(i16 zeroext %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_wu_i16:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -509,6 +636,22 @@ define signext i32 @fcvt_s_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IF-NEXT:    fsw fa5, 0(a1)
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_w_demanded_bits:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi a0, a0, 1
+; RV32IZFINX-NEXT:    fcvt.s.w a2, a0
+; RV32IZFINX-NEXT:    sw a2, 0(a1)
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_w_demanded_bits:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addiw a2, a0, 1
+; RV64IZFINX-NEXT:    addi a0, a0, 1
+; RV64IZFINX-NEXT:    fcvt.s.w a0, a0
+; RV64IZFINX-NEXT:    sw a0, 0(a1)
+; RV64IZFINX-NEXT:    mv a0, a2
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -566,6 +709,20 @@ define signext i32 @fcvt_s_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IF-NEXT:    fsw fa5, 0(a1)
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_wu_demanded_bits:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi a0, a0, 1
+; RV32IZFINX-NEXT:    fcvt.s.wu a2, a0
+; RV32IZFINX-NEXT:    sw a2, 0(a1)
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_wu_demanded_bits:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addiw a0, a0, 1
+; RV64IZFINX-NEXT:    fcvt.s.wu a2, a0
+; RV64IZFINX-NEXT:    sw a2, 0(a1)
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index fac48ff0ea826..7224f5b79b7a1 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefixes=CHECKIF,RV32IF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefixes=CHECKIF,RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefixes=CHECKIZFINX,RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=CHECKIZFINX,RV64IZFINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -14,6 +18,11 @@ define i32 @fcvt_w_s(float %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_w_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -45,6 +54,15 @@ define i32 @fcvt_w_s_sat(float %a) nounwind {
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_w_s_sat:
+; CHECKIZFINX:       # %bb.0: # %start
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -140,6 +158,11 @@ define i32 @fcvt_wu_s(float %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_wu_s:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -171,6 +194,13 @@ define i32 @fcvt_wu_s_multiple_use(float %x, ptr %y) nounwind {
 ; CHECKIF-NEXT:    add a0, a0, a1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_wu_s_multiple_use:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZFINX-NEXT:    seqz a1, a0
+; CHECKIZFINX-NEXT:    add a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_multiple_use:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -219,6 +249,26 @@ define i32 @fcvt_wu_s_sat(float %a) nounwind {
 ; RV64IF-NEXT:    srli a0, a0, 32
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_wu_s_sat:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    fcvt.wu.s a1, a0, rtz
+; RV32IZFINX-NEXT:    feq.s a0, a0, a0
+; RV32IZFINX-NEXT:    seqz a0, a0
+; RV32IZFINX-NEXT:    addi a0, a0, -1
+; RV32IZFINX-NEXT:    and a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_wu_s_sat:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    fcvt.wu.s a1, a0, rtz
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addiw a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a1, a0
+; RV64IZFINX-NEXT:    slli a0, a0, 32
+; RV64IZFINX-NEXT:    srli a0, a0, 32
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -293,6 +343,11 @@ define i32 @fmv_x_w(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fmv.x.w a0, fa5
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmv_x_w:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmv_x_w:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -322,6 +377,11 @@ define float @fcvt_s_w(i32 %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -351,6 +411,12 @@ define float @fcvt_s_w_load(ptr %p) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w_load:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    lw a0, 0(a0)
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -381,6 +447,11 @@ define float @fcvt_s_wu(i32 %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_wu:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -416,6 +487,18 @@ define float @fcvt_s_wu_load(ptr %p) nounwind {
 ; RV64IF-NEXT:    fcvt.s.wu fa0, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_wu_load:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lw a0, 0(a0)
+; RV32IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_wu_load:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lwu a0, 0(a0)
+; RV64IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_load:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -448,6 +531,11 @@ define float @fmv_w_x(i32 %a, i32 %b) nounwind {
 ; CHECKIF-NEXT:    fadd.s fa0, fa5, fa4
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmv_w_x:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmv_w_x:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -487,6 +575,20 @@ define i64 @fcvt_l_s(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_l_s:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_l_s:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_l_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -520,27 +622,27 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV32IF-NEXT:    fmv.w.x fa5, a0
 ; RV32IF-NEXT:    fle.s s0, fa5, fa0
 ; RV32IF-NEXT:    call __fixsfdi@plt
-; RV32IF-NEXT:    lui a3, 524288
-; RV32IF-NEXT:    bnez s0, .LBB12_2
+; RV32IF-NEXT:    lui a4, 524288
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    beqz s0, .LBB12_2
 ; RV32IF-NEXT:  # %bb.1: # %start
-; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    mv a2, a1
 ; RV32IF-NEXT:  .LBB12_2: # %start
-; RV32IF-NEXT:    lui a2, %hi(.LCPI12_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI12_0)(a2)
-; RV32IF-NEXT:    flt.s a2, fa5, fs0
-; RV32IF-NEXT:    beqz a2, .LBB12_4
+; RV32IF-NEXT:    lui a1, %hi(.LCPI12_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI12_0)(a1)
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    beqz a3, .LBB12_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    addi a1, a3, -1
+; RV32IF-NEXT:    addi a2, a4, -1
 ; RV32IF-NEXT:  .LBB12_4: # %start
-; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    seqz a3, a3
-; RV32IF-NEXT:    addi a3, a3, -1
-; RV32IF-NEXT:    and a1, a3, a1
-; RV32IF-NEXT:    neg a2, a2
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    and a0, a4, a0
-; RV32IF-NEXT:    or a0, a2, a0
+; RV32IF-NEXT:    feq.s a1, fs0, fs0
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    and a1, a4, a2
+; RV32IF-NEXT:    neg a2, a3
+; RV32IF-NEXT:    neg a3, s0
 ; RV32IF-NEXT:    and a0, a3, a0
+; RV32IF-NEXT:    or a0, a2, a0
+; RV32IF-NEXT:    and a0, a4, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -556,6 +658,53 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_l_s_sat:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 913408
+; RV32IZFINX-NEXT:    fle.s s1, a0, s0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lui a4, 524288
+; RV32IZFINX-NEXT:    lui a2, 524288
+; RV32IZFINX-NEXT:    beqz s1, .LBB12_2
+; RV32IZFINX-NEXT:  # %bb.1: # %start
+; RV32IZFINX-NEXT:    mv a2, a1
+; RV32IZFINX-NEXT:  .LBB12_2: # %start
+; RV32IZFINX-NEXT:    lui a1, %hi(.LCPI12_0)
+; RV32IZFINX-NEXT:    lw a1, %lo(.LCPI12_0)(a1)
+; RV32IZFINX-NEXT:    flt.s a3, a1, s0
+; RV32IZFINX-NEXT:    beqz a3, .LBB12_4
+; RV32IZFINX-NEXT:  # %bb.3:
+; RV32IZFINX-NEXT:    addi a2, a4, -1
+; RV32IZFINX-NEXT:  .LBB12_4: # %start
+; RV32IZFINX-NEXT:    feq.s a1, s0, s0
+; RV32IZFINX-NEXT:    neg a4, a1
+; RV32IZFINX-NEXT:    and a1, a4, a2
+; RV32IZFINX-NEXT:    neg a2, s1
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    neg a2, a3
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a0, a4, a0
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_l_s_sat:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a0, rtz
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_l_s_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -565,59 +714,49 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lui a1, 913408
 ; RV32I-NEXT:    call __gesf2@plt
-; RV32I-NEXT:    mv s2, a0
+; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixsfdi@plt
-; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    mv s3, a1
-; RV32I-NEXT:    lui s4, 524288
-; RV32I-NEXT:    bgez s2, .LBB12_2
+; RV32I-NEXT:    lui s5, 524288
+; RV32I-NEXT:    bgez s1, .LBB12_2
 ; RV32I-NEXT:  # %bb.1: # %start
 ; RV32I-NEXT:    lui s3, 524288
 ; RV32I-NEXT:  .LBB12_2: # %start
-; RV32I-NEXT:    lui s2, 389120
-; RV32I-NEXT:    addi s2, s2, -1
+; RV32I-NEXT:    lui a1, 389120
+; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __gtsf2@plt
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    blez a0, .LBB12_4
 ; RV32I-NEXT:  # %bb.3: # %start
-; RV32I-NEXT:    addi s3, s4, -1
+; RV32I-NEXT:    addi s3, s5, -1
 ; RV32I-NEXT:  .LBB12_4: # %start
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    call __unordsf2@plt
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and s3, a0, s3
-; RV32I-NEXT:    lui a1, 913408
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __gesf2@plt
-; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and s1, a0, s1
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s2
-; RV32I-NEXT:    call __gtsf2@plt
-; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg a0, a0
-; RV32I-NEXT:    or s1, a0, s1
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __unordsf2@plt
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    and a1, a0, s3
+; RV32I-NEXT:    slti a2, s1, 0
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    and a2, a2, s2
+; RV32I-NEXT:    sgtz a3, s4
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    and a0, a0, a2
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -683,6 +822,20 @@ define i64 @fcvt_lu_s(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_lu_s:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_lu_s:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_lu_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -739,6 +892,40 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind {
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_lu_s_sat:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    fle.s a0, zero, a0
+; RV32IZFINX-NEXT:    neg s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI14_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI14_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s1, a0
+; RV32IZFINX-NEXT:    flt.s a2, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a1, s1, a1
+; RV32IZFINX-NEXT:    or a1, a2, a1
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_lu_s_sat:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    fcvt.lu.s a1, a0, rtz
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_lu_s_sat:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
@@ -756,27 +943,15 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind {
 ; RV32I-NEXT:    call __fixunssfdi@plt
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    and s3, s2, a0
-; RV32I-NEXT:    lui s2, 391168
-; RV32I-NEXT:    addi s2, s2, -1
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s2
-; RV32I-NEXT:    call __gtsf2@plt
-; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg a0, a0
-; RV32I-NEXT:    or s3, a0, s3
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    call __gesf2@plt
-; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and s1, a0, s1
+; RV32I-NEXT:    lui a1, 391168
+; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __gtsf2@plt
 ; RV32I-NEXT:    sgtz a0, a0
 ; RV32I-NEXT:    neg a1, a0
-; RV32I-NEXT:    or a1, a1, s1
-; RV32I-NEXT:    mv a0, s3
+; RV32I-NEXT:    or a0, a1, s3
+; RV32I-NEXT:    and a2, s2, s1
+; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
@@ -832,6 +1007,20 @@ define float @fcvt_s_l(i64 %a) nounwind {
 ; RV64IF-NEXT:    fcvt.s.l fa0, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_l:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __floatdisf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_l:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.l a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_l:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -868,6 +1057,20 @@ define float @fcvt_s_lu(i64 %a) nounwind {
 ; RV64IF-NEXT:    fcvt.s.lu fa0, a0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_lu:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __floatundisf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_lu:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.lu a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_lu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -895,6 +1098,11 @@ define float @fcvt_s_w_i8(i8 signext %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w_i8:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -922,6 +1130,11 @@ define float @fcvt_s_wu_i8(i8 zeroext %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_wu_i8:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -949,6 +1162,11 @@ define float @fcvt_s_w_i16(i16 signext %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.w fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_w_i16:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.w a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -976,6 +1194,11 @@ define float @fcvt_s_wu_i16(i16 zeroext %a) nounwind {
 ; CHECKIF-NEXT:    fcvt.s.wu fa0, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_s_wu_i16:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1013,6 +1236,22 @@ define signext i32 @fcvt_s_w_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IF-NEXT:    fsw fa5, 0(a1)
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_w_demanded_bits:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi a0, a0, 1
+; RV32IZFINX-NEXT:    fcvt.s.w a2, a0
+; RV32IZFINX-NEXT:    sw a2, 0(a1)
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_w_demanded_bits:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addiw a2, a0, 1
+; RV64IZFINX-NEXT:    addi a0, a0, 1
+; RV64IZFINX-NEXT:    fcvt.s.w a0, a0
+; RV64IZFINX-NEXT:    sw a0, 0(a1)
+; RV64IZFINX-NEXT:    mv a0, a2
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_w_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1070,6 +1309,20 @@ define signext i32 @fcvt_s_wu_demanded_bits(i32 signext %0, ptr %1) nounwind {
 ; RV64IF-NEXT:    fsw fa5, 0(a1)
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_s_wu_demanded_bits:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi a0, a0, 1
+; RV32IZFINX-NEXT:    fcvt.s.wu a2, a0
+; RV32IZFINX-NEXT:    sw a2, 0(a1)
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_s_wu_demanded_bits:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addiw a0, a0, 1
+; RV64IZFINX-NEXT:    fcvt.s.wu a2, a0
+; RV64IZFINX-NEXT:    sw a2, 0(a1)
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_s_wu_demanded_bits:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1122,6 +1375,16 @@ define signext i16 @fcvt_w_s_i16(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_w_s_i16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_w_s_i16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1146,34 +1409,58 @@ define signext i16 @fcvt_w_s_i16(float %a) nounwind {
 define signext i16 @fcvt_w_s_sat_i16(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_w_s_sat_i16:
 ; RV32IF:       # %bb.0: # %start
-; RV32IF-NEXT:    lui a0, %hi(.LCPI24_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI24_0)(a0)
-; RV32IF-NEXT:    lui a0, 815104
-; RV32IF-NEXT:    fmv.w.x fa4, a0
+; RV32IF-NEXT:    feq.s a0, fa0, fa0
+; RV32IF-NEXT:    neg a0, a0
+; RV32IF-NEXT:    lui a1, %hi(.LCPI24_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI24_0)(a1)
+; RV32IF-NEXT:    lui a1, 815104
+; RV32IF-NEXT:    fmv.w.x fa4, a1
 ; RV32IF-NEXT:    fmax.s fa4, fa0, fa4
 ; RV32IF-NEXT:    fmin.s fa5, fa4, fa5
-; RV32IF-NEXT:    fcvt.w.s a0, fa5, rtz
-; RV32IF-NEXT:    feq.s a1, fa0, fa0
-; RV32IF-NEXT:    seqz a1, a1
-; RV32IF-NEXT:    addi a1, a1, -1
-; RV32IF-NEXT:    and a0, a1, a0
+; RV32IF-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IF-NEXT:    and a0, a0, a1
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcvt_w_s_sat_i16:
 ; RV64IF:       # %bb.0: # %start
-; RV64IF-NEXT:    lui a0, %hi(.LCPI24_0)
-; RV64IF-NEXT:    flw fa5, %lo(.LCPI24_0)(a0)
-; RV64IF-NEXT:    lui a0, 815104
-; RV64IF-NEXT:    fmv.w.x fa4, a0
+; RV64IF-NEXT:    feq.s a0, fa0, fa0
+; RV64IF-NEXT:    lui a1, %hi(.LCPI24_0)
+; RV64IF-NEXT:    flw fa5, %lo(.LCPI24_0)(a1)
+; RV64IF-NEXT:    lui a1, 815104
+; RV64IF-NEXT:    fmv.w.x fa4, a1
 ; RV64IF-NEXT:    fmax.s fa4, fa0, fa4
+; RV64IF-NEXT:    neg a0, a0
 ; RV64IF-NEXT:    fmin.s fa5, fa4, fa5
-; RV64IF-NEXT:    fcvt.l.s a0, fa5, rtz
-; RV64IF-NEXT:    feq.s a1, fa0, fa0
-; RV64IF-NEXT:    seqz a1, a1
-; RV64IF-NEXT:    addi a1, a1, -1
-; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IF-NEXT:    and a0, a0, a1
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_w_s_sat_i16:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    feq.s a1, a0, a0
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI24_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI24_0)(a2)
+; RV32IZFINX-NEXT:    neg a1, a1
+; RV32IZFINX-NEXT:    lui a3, 815104
+; RV32IZFINX-NEXT:    fmax.s a0, a0, a3
+; RV32IZFINX-NEXT:    fmin.s a0, a0, a2
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    and a0, a1, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_w_s_sat_i16:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    lui a1, 815104
+; RV64IZFINX-NEXT:    lui a2, %hi(.LCPI24_0)
+; RV64IZFINX-NEXT:    lw a2, %lo(.LCPI24_0)(a2)
+; RV64IZFINX-NEXT:    fmax.s a1, a0, a1
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    neg a0, a0
+; RV64IZFINX-NEXT:    fmin.s a1, a1, a2
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a1, rtz
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_sat_i16:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1274,6 +1561,16 @@ define zeroext i16 @fcvt_wu_s_i16(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_wu_s_i16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_wu_s_i16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1316,6 +1613,24 @@ define zeroext i16 @fcvt_wu_s_sat_i16(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa5, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_wu_s_sat_i16:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    lui a1, %hi(.LCPI26_0)
+; RV32IZFINX-NEXT:    lw a1, %lo(.LCPI26_0)(a1)
+; RV32IZFINX-NEXT:    fmax.s a0, a0, zero
+; RV32IZFINX-NEXT:    fmin.s a0, a0, a1
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_wu_s_sat_i16:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    lui a1, %hi(.LCPI26_0)
+; RV64IZFINX-NEXT:    lw a1, %lo(.LCPI26_0)(a1)
+; RV64IZFINX-NEXT:    fmax.s a0, a0, zero
+; RV64IZFINX-NEXT:    fmin.s a0, a0, a1
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_sat_i16:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1406,6 +1721,16 @@ define signext i8 @fcvt_w_s_i8(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_w_s_i8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_w_s_i8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1430,34 +1755,56 @@ define signext i8 @fcvt_w_s_i8(float %a) nounwind {
 define signext i8 @fcvt_w_s_sat_i8(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_w_s_sat_i8:
 ; RV32IF:       # %bb.0: # %start
-; RV32IF-NEXT:    lui a0, 798720
-; RV32IF-NEXT:    fmv.w.x fa5, a0
+; RV32IF-NEXT:    feq.s a0, fa0, fa0
+; RV32IF-NEXT:    neg a0, a0
+; RV32IF-NEXT:    lui a1, 798720
+; RV32IF-NEXT:    fmv.w.x fa5, a1
 ; RV32IF-NEXT:    fmax.s fa5, fa0, fa5
-; RV32IF-NEXT:    lui a0, 274400
-; RV32IF-NEXT:    fmv.w.x fa4, a0
+; RV32IF-NEXT:    lui a1, 274400
+; RV32IF-NEXT:    fmv.w.x fa4, a1
 ; RV32IF-NEXT:    fmin.s fa5, fa5, fa4
-; RV32IF-NEXT:    fcvt.w.s a0, fa5, rtz
-; RV32IF-NEXT:    feq.s a1, fa0, fa0
-; RV32IF-NEXT:    seqz a1, a1
-; RV32IF-NEXT:    addi a1, a1, -1
-; RV32IF-NEXT:    and a0, a1, a0
+; RV32IF-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IF-NEXT:    and a0, a0, a1
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcvt_w_s_sat_i8:
 ; RV64IF:       # %bb.0: # %start
-; RV64IF-NEXT:    lui a0, 798720
-; RV64IF-NEXT:    fmv.w.x fa5, a0
+; RV64IF-NEXT:    feq.s a0, fa0, fa0
+; RV64IF-NEXT:    neg a0, a0
+; RV64IF-NEXT:    lui a1, 798720
+; RV64IF-NEXT:    fmv.w.x fa5, a1
 ; RV64IF-NEXT:    fmax.s fa5, fa0, fa5
-; RV64IF-NEXT:    lui a0, 274400
-; RV64IF-NEXT:    fmv.w.x fa4, a0
+; RV64IF-NEXT:    lui a1, 274400
+; RV64IF-NEXT:    fmv.w.x fa4, a1
 ; RV64IF-NEXT:    fmin.s fa5, fa5, fa4
-; RV64IF-NEXT:    fcvt.l.s a0, fa5, rtz
-; RV64IF-NEXT:    feq.s a1, fa0, fa0
-; RV64IF-NEXT:    seqz a1, a1
-; RV64IF-NEXT:    addi a1, a1, -1
-; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IF-NEXT:    and a0, a0, a1
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_w_s_sat_i8:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    feq.s a1, a0, a0
+; RV32IZFINX-NEXT:    neg a1, a1
+; RV32IZFINX-NEXT:    lui a2, 798720
+; RV32IZFINX-NEXT:    fmax.s a0, a0, a2
+; RV32IZFINX-NEXT:    lui a2, 274400
+; RV32IZFINX-NEXT:    fmin.s a0, a0, a2
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    and a0, a1, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_w_s_sat_i8:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    feq.s a1, a0, a0
+; RV64IZFINX-NEXT:    neg a1, a1
+; RV64IZFINX-NEXT:    lui a2, 798720
+; RV64IZFINX-NEXT:    fmax.s a0, a0, a2
+; RV64IZFINX-NEXT:    lui a2, 274400
+; RV64IZFINX-NEXT:    fmin.s a0, a0, a2
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    and a0, a1, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_sat_i8:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1554,6 +1901,16 @@ define zeroext i8 @fcvt_wu_s_i8(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_wu_s_i8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_wu_s_i8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1596,6 +1953,22 @@ define zeroext i8 @fcvt_wu_s_sat_i8(float %a) nounwind {
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa5, rtz
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_wu_s_sat_i8:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    fmax.s a0, a0, zero
+; RV32IZFINX-NEXT:    lui a1, 276464
+; RV32IZFINX-NEXT:    fmin.s a0, a0, a1
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_wu_s_sat_i8:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    fmax.s a0, a0, zero
+; RV64IZFINX-NEXT:    lui a1, 276464
+; RV64IZFINX-NEXT:    fmin.s a0, a0, a1
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_sat_i8:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1690,6 +2063,26 @@ define zeroext i32 @fcvt_wu_s_sat_zext(float %a) nounwind {
 ; RV64IF-NEXT:    srli a0, a0, 32
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fcvt_wu_s_sat_zext:
+; RV32IZFINX:       # %bb.0: # %start
+; RV32IZFINX-NEXT:    fcvt.wu.s a1, a0, rtz
+; RV32IZFINX-NEXT:    feq.s a0, a0, a0
+; RV32IZFINX-NEXT:    seqz a0, a0
+; RV32IZFINX-NEXT:    addi a0, a0, -1
+; RV32IZFINX-NEXT:    and a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fcvt_wu_s_sat_zext:
+; RV64IZFINX:       # %bb.0: # %start
+; RV64IZFINX-NEXT:    fcvt.wu.s a1, a0, rtz
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addiw a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a1, a0
+; RV64IZFINX-NEXT:    slli a0, a0, 32
+; RV64IZFINX-NEXT:    srli a0, a0, 32
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_wu_s_sat_zext:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1768,6 +2161,15 @@ define signext i32 @fcvt_w_s_sat_sext(float %a) nounwind {
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcvt_w_s_sat_sext:
+; CHECKIZFINX:       # %bb.0: # %start
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcvt_w_s_sat_sext:
 ; RV32I:       # %bb.0: # %start
 ; RV32I-NEXT:    addi sp, sp, -32
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll b/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
index 2fda42008d457..36eb58fe74542 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
@@ -5,6 +5,12 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation -target-abi=lp64f \
 ; RUN:   | FileCheck -check-prefix=CHECKIF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=ilp32 \
+; RUN:   | FileCheck -check-prefix=CHECKIZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=CHECKIZFINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   -disable-strictnode-mutation | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -16,6 +22,11 @@ define i32 @fcmp_oeq(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s a0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_oeq:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oeq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -50,6 +61,15 @@ define i32 @fcmp_ogt(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ogt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a3, fflags
+; CHECKIZFINX-NEXT:    flt.s a2, a1, a0
+; CHECKIZFINX-NEXT:    csrw fflags, a3
+; CHECKIZFINX-NEXT:    feq.s zero, a1, a0
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ogt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -83,6 +103,15 @@ define i32 @fcmp_oge(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_oge:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a3, fflags
+; CHECKIZFINX-NEXT:    fle.s a2, a1, a0
+; CHECKIZFINX-NEXT:    csrw fflags, a3
+; CHECKIZFINX-NEXT:    feq.s zero, a1, a0
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -118,6 +147,15 @@ define i32 @fcmp_olt(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_olt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a3, fflags
+; CHECKIZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKIZFINX-NEXT:    csrw fflags, a3
+; CHECKIZFINX-NEXT:    feq.s zero, a0, a1
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_olt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -151,6 +189,15 @@ define i32 @fcmp_ole(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ole:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a3, fflags
+; CHECKIZFINX-NEXT:    fle.s a2, a0, a1
+; CHECKIZFINX-NEXT:    csrw fflags, a3
+; CHECKIZFINX-NEXT:    feq.s zero, a0, a1
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ole:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -191,6 +238,20 @@ define i32 @fcmp_one(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_one:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    flt.s a3, a0, a1
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    feq.s zero, a0, a1
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    flt.s a4, a1, a0
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    or a2, a4, a3
+; CHECKIZFINX-NEXT:    feq.s zero, a1, a0
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -249,6 +310,13 @@ define i32 @fcmp_ord(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ord:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a1, a1, a1
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ord:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -290,6 +358,21 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ueq:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    flt.s a3, a0, a1
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    feq.s zero, a0, a1
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    flt.s a4, a1, a0
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    or a3, a4, a3
+; CHECKIZFINX-NEXT:    xori a2, a3, 1
+; CHECKIZFINX-NEXT:    feq.s zero, a1, a0
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ueq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -350,6 +433,16 @@ define i32 @fcmp_ugt(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ugt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    fle.s a3, a0, a1
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    xori a2, a3, 1
+; CHECKIZFINX-NEXT:    feq.s zero, a0, a1
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ugt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -384,6 +477,16 @@ define i32 @fcmp_uge(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_uge:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    flt.s a3, a0, a1
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    xori a2, a3, 1
+; CHECKIZFINX-NEXT:    feq.s zero, a0, a1
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -420,6 +523,16 @@ define i32 @fcmp_ult(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ult:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    fle.s a3, a1, a0
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    xori a2, a3, 1
+; CHECKIZFINX-NEXT:    feq.s zero, a1, a0
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ult:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -454,6 +567,16 @@ define i32 @fcmp_ule(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    feq.s zero, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ule:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    csrr a2, fflags
+; CHECKIZFINX-NEXT:    flt.s a3, a1, a0
+; CHECKIZFINX-NEXT:    csrw fflags, a2
+; CHECKIZFINX-NEXT:    xori a2, a3, 1
+; CHECKIZFINX-NEXT:    feq.s zero, a1, a0
+; CHECKIZFINX-NEXT:    mv a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ule:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -485,6 +608,12 @@ define i32 @fcmp_une(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_une:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_une:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -518,6 +647,14 @@ define i32 @fcmp_uno(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_uno:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a1, a1, a1
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uno:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -550,6 +687,13 @@ define i32 @fcmps_oeq(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_oeq:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a2, a1, a0
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKIZFINX-NEXT:    and a0, a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_oeq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -581,6 +725,11 @@ define i32 @fcmps_ogt(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    flt.s a0, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ogt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ogt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -611,6 +760,11 @@ define i32 @fcmps_oge(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    fle.s a0, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_oge:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a1, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_oge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -643,6 +797,11 @@ define i32 @fcmps_olt(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    flt.s a0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_olt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_olt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -673,6 +832,11 @@ define i32 @fcmps_ole(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    fle.s a0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ole:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ole:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -705,6 +869,13 @@ define i32 @fcmps_one(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    or a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_one:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    or a0, a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -763,6 +934,13 @@ define i32 @fcmps_ord(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ord:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a1, a1, a1
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a0
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ord:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -796,6 +974,14 @@ define i32 @fcmps_ueq(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ueq:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    or a0, a0, a2
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ueq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -853,6 +1039,12 @@ define i32 @fcmps_ugt(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ugt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ugt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -884,6 +1076,12 @@ define i32 @fcmps_uge(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_uge:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_uge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -917,6 +1115,12 @@ define i32 @fcmps_ult(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ult:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a1, a0
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ult:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -948,6 +1152,12 @@ define i32 @fcmps_ule(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_ule:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_ule:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -981,6 +1191,14 @@ define i32 @fcmps_une(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_une:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a2, a1, a0
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKIZFINX-NEXT:    and a0, a0, a2
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_une:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1014,6 +1232,14 @@ define i32 @fcmps_uno(float %a, float %b) nounwind strictfp {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmps_uno:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a1, a1, a1
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a0
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmps_uno:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp.ll b/llvm/test/CodeGen/RISCV/float-fcmp.ll
index 083262d6372c9..b4fbed1321e27 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefix=CHECKIF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefix=CHECKIF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefix=CHECKIZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=CHECKIZFINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -14,6 +18,11 @@ define i32 @fcmp_false(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    li a0, 0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_false:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    li a0, 0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_false:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    li a0, 0
@@ -34,6 +43,11 @@ define i32 @fcmp_oeq(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    feq.s a0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_oeq:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oeq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -64,6 +78,11 @@ define i32 @fcmp_ogt(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    flt.s a0, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ogt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ogt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -94,6 +113,11 @@ define i32 @fcmp_oge(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fle.s a0, fa1, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_oge:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a1, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_oge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -126,6 +150,11 @@ define i32 @fcmp_olt(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    flt.s a0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_olt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_olt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -156,6 +185,11 @@ define i32 @fcmp_ole(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    fle.s a0, fa0, fa1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ole:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ole:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -188,6 +222,13 @@ define i32 @fcmp_one(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    or a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_one:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    or a0, a0, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -246,6 +287,13 @@ define i32 @fcmp_ord(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ord:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a1, a1, a1
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ord:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -279,6 +327,14 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ueq:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    or a0, a0, a2
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ueq:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -336,6 +392,12 @@ define i32 @fcmp_ugt(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ugt:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ugt:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -367,6 +429,12 @@ define i32 @fcmp_uge(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_uge:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uge:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -400,6 +468,12 @@ define i32 @fcmp_ult(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ult:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fle.s a0, a1, a0
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ult:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -431,6 +505,12 @@ define i32 @fcmp_ule(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_ule:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    flt.s a0, a1, a0
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_ule:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -462,6 +542,12 @@ define i32 @fcmp_une(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_une:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_une:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -495,6 +581,14 @@ define i32 @fcmp_uno(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    xori a0, a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_uno:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    feq.s a1, a1, a1
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    xori a0, a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_uno:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -525,6 +619,11 @@ define i32 @fcmp_true(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    li a0, 1
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fcmp_true:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    li a0, 1
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fcmp_true:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    li a0, 1
diff --git a/llvm/test/CodeGen/RISCV/float-frem.ll b/llvm/test/CodeGen/RISCV/float-frem.ll
index d8e90b2029e6c..cb80c6cfbeaa0 100644
--- a/llvm/test/CodeGen/RISCV/float-frem.ll
+++ b/llvm/test/CodeGen/RISCV/float-frem.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFINX %s
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
@@ -22,6 +26,19 @@ define float @frem_f32(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: frem_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail fmodf@plt
+;
+; RV64IZFINX-LABEL: frem_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call fmodf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: frem_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/float-imm.ll b/llvm/test/CodeGen/RISCV/float-imm.ll
index c5fbe6da87a66..a99ba3faa5768 100644
--- a/llvm/test/CodeGen/RISCV/float-imm.ll
+++ b/llvm/test/CodeGen/RISCV/float-imm.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck --check-prefixes=CHECKZFINX,RV32ZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck --check-prefixes=CHECKZFINX,RV64ZFINX %s
 
 ; TODO: constant pool shouldn't be necessary for RV64IF.
 define float @float_imm() nounwind {
@@ -11,6 +15,18 @@ define float @float_imm() nounwind {
 ; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
 ; CHECK-NEXT:    flw fa0, %lo(.LCPI0_0)(a0)
 ; CHECK-NEXT:    ret
+;
+; RV32ZFINX-LABEL: float_imm:
+; RV32ZFINX:       # %bb.0:
+; RV32ZFINX-NEXT:    lui a0, 263313
+; RV32ZFINX-NEXT:    addi a0, a0, -37
+; RV32ZFINX-NEXT:    ret
+;
+; RV64ZFINX-LABEL: float_imm:
+; RV64ZFINX:       # %bb.0:
+; RV64ZFINX-NEXT:    lui a0, %hi(.LCPI0_0)
+; RV64ZFINX-NEXT:    lw a0, %lo(.LCPI0_0)(a0)
+; RV64ZFINX-NEXT:    ret
   ret float 3.14159274101257324218750
 }
 
@@ -21,6 +37,12 @@ define float @float_imm_op(float %a) nounwind {
 ; CHECK-NEXT:    fmv.w.x fa5, a0
 ; CHECK-NEXT:    fadd.s fa0, fa0, fa5
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: float_imm_op:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    lui a1, 260096
+; CHECKZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKZFINX-NEXT:    ret
   %1 = fadd float %a, 1.0
   ret float %1
 }
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
index 736aa8c109553..d149b35f61c8e 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics-strict.ll
@@ -5,6 +5,12 @@
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+f \
 ; RUN:   -verify-machineinstrs -disable-strictnode-mutation -target-abi=lp64f \
 ; RUN:   | FileCheck -check-prefixes=CHECKIF,RV64IF %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+zfinx \
+; RUN:   -verify-machineinstrs -disable-strictnode-mutation -target-abi=ilp32 \
+; RUN:   | FileCheck -check-prefixes=CHECKIZFINX,RV32IZFINX %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+zfinx \
+; RUN:   -verify-machineinstrs -disable-strictnode-mutation -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefixes=CHECKIZFINX,RV64IZFINX %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 \
 ; RUN:   -verify-machineinstrs -disable-strictnode-mutation \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
@@ -20,6 +26,11 @@ define float @sqrt_f32(float %a) nounwind strictfp {
 ; CHECKIF-NEXT:    fsqrt.s fa0, fa0
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: sqrt_f32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fsqrt.s a0, a0
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: sqrt_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -63,6 +74,25 @@ define float @powi_f32(float %a, i32 %b) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: powi_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call __powisf2@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: powi_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sext.w a1, a1
+; RV64IZFINX-NEXT:    call __powisf2@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: powi_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -106,6 +136,24 @@ define float @sin_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: sin_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call sinf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sin_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call sinf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: sin_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -148,6 +196,24 @@ define float @cos_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: cos_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call cosf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: cos_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call cosf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: cos_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -207,6 +273,42 @@ define float @sincos_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: sincos_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    call sinf@plt
+; RV32IZFINX-NEXT:    mv s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call cosf@plt
+; RV32IZFINX-NEXT:    fadd.s a0, s1, a0
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sincos_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -32
+; RV64IZFINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    mv s0, a0
+; RV64IZFINX-NEXT:    call sinf@plt
+; RV64IZFINX-NEXT:    mv s1, a0
+; RV64IZFINX-NEXT:    mv a0, s0
+; RV64IZFINX-NEXT:    call cosf@plt
+; RV64IZFINX-NEXT:    fadd.s a0, s1, a0
+; RV64IZFINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 32
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: sincos_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -273,6 +375,24 @@ define float @pow_f32(float %a, float %b) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: pow_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call powf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: pow_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call powf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: pow_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -315,6 +435,24 @@ define float @exp_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: exp_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call expf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: exp_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call expf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: exp_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -357,6 +495,24 @@ define float @exp2_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: exp2_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call exp2f@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: exp2_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call exp2f@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: exp2_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -399,6 +555,24 @@ define float @log_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: log_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call logf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: log_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call logf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: log_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -441,6 +615,24 @@ define float @log10_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: log10_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call log10f@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: log10_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call log10f@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: log10_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -483,6 +675,24 @@ define float @log2_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: log2_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call log2f@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: log2_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call log2f@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: log2_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -512,6 +722,11 @@ define float @fma_f32(float %a, float %b, float %c) nounwind strictfp {
 ; CHECKIF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fma_f32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fma_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -541,6 +756,11 @@ define float @fmuladd_f32(float %a, float %b, float %c) nounwind strictfp {
 ; CHECKIF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; CHECKIF-NEXT:    ret
 ;
+; CHECKIZFINX-LABEL: fmuladd_f32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; CHECKIZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmuladd_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -593,6 +813,24 @@ define float @minnum_f32(float %a, float %b) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: minnum_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call fminf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: minnum_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call fminf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: minnum_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -635,6 +873,24 @@ define float @maxnum_f32(float %a, float %b) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: maxnum_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call fmaxf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: maxnum_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call fmaxf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: maxnum_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -694,6 +950,24 @@ define float @floor_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: floor_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call floorf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: floor_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call floorf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: floor_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -736,6 +1010,24 @@ define float @ceil_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: ceil_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call ceilf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: ceil_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call ceilf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: ceil_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -778,6 +1070,24 @@ define float @trunc_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: trunc_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call truncf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: trunc_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call truncf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: trunc_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -820,6 +1130,24 @@ define float @rint_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: rint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call rintf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: rint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call rintf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: rint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -862,6 +1190,24 @@ define float @nearbyint_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: nearbyint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call nearbyintf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: nearbyint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call nearbyintf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: nearbyint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -904,6 +1250,24 @@ define float @round_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: round_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call roundf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: round_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call roundf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: round_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -946,6 +1310,24 @@ define float @roundeven_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: roundeven_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call roundevenf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: roundeven_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call roundevenf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: roundeven_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -980,6 +1362,16 @@ define iXLen @lrint_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: lrint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: lrint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: lrint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1014,6 +1406,16 @@ define iXLen @lround_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: lround_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: lround_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: lround_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1052,6 +1454,20 @@ define i64 @llrint_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: llrint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call llrintf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: llrint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: llrint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1090,6 +1506,20 @@ define i64 @llround_f32(float %a) nounwind strictfp {
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: llround_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call llroundf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: llround_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: llround_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index 8ae7ce4256d29..06adc9dd26834 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -2,12 +2,18 @@
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+f \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32f \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+zfinx \
+; RUN:   -verify-machineinstrs -target-abi=ilp32 \
+; RUN:   | FileCheck -check-prefix=RV32IZFINX %s
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+d \
 ; RUN:   -verify-machineinstrs -target-abi=ilp32f \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+f \
 ; RUN:   -verify-machineinstrs -target-abi=lp64f \
 ; RUN:   | FileCheck -check-prefix=RV64IF %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+zfinx \
+; RUN:   -verify-machineinstrs -target-abi=lp64 \
+; RUN:   | FileCheck -check-prefix=RV64IZFINX %s
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+d \
 ; RUN:   -verify-machineinstrs -target-abi=lp64d \
 ; RUN:   | FileCheck -check-prefix=RV64IF %s
@@ -24,11 +30,21 @@ define float @sqrt_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fsqrt.s fa0, fa0
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: sqrt_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fsqrt.s a0, a0
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: sqrt_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fsqrt.s fa0, fa0
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: sqrt_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fsqrt.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: sqrt_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -57,6 +73,10 @@ define float @powi_f32(float %a, i32 %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail __powisf2@plt
 ;
+; RV32IZFINX-LABEL: powi_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail __powisf2@plt
+;
 ; RV64IF-LABEL: powi_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
@@ -67,6 +87,16 @@ define float @powi_f32(float %a, i32 %b) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: powi_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sext.w a1, a1
+; RV64IZFINX-NEXT:    call __powisf2@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: powi_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -96,10 +126,23 @@ define float @sin_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail sinf@plt
 ;
+; RV32IZFINX-LABEL: sin_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail sinf@plt
+;
 ; RV64IF-LABEL: sin_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail sinf@plt
 ;
+; RV64IZFINX-LABEL: sin_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call sinf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: sin_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -128,10 +171,23 @@ define float @cos_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail cosf@plt
 ;
+; RV32IZFINX-LABEL: cos_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail cosf@plt
+;
 ; RV64IF-LABEL: cos_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail cosf@plt
 ;
+; RV64IZFINX-LABEL: cos_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call cosf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: cos_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -173,6 +229,42 @@ define float @sincos_f32(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: sincos_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    call sinf@plt
+; RV32IZFINX-NEXT:    mv s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call cosf@plt
+; RV32IZFINX-NEXT:    fadd.s a0, s1, a0
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sincos_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -32
+; RV64IZFINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    mv s0, a0
+; RV64IZFINX-NEXT:    call sinf@plt
+; RV64IZFINX-NEXT:    mv s1, a0
+; RV64IZFINX-NEXT:    mv a0, s0
+; RV64IZFINX-NEXT:    call cosf@plt
+; RV64IZFINX-NEXT:    fadd.s a0, s1, a0
+; RV64IZFINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 32
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: sincos_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -225,10 +317,23 @@ define float @pow_f32(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail powf@plt
 ;
+; RV32IZFINX-LABEL: pow_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail powf@plt
+;
 ; RV64IF-LABEL: pow_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail powf@plt
 ;
+; RV64IZFINX-LABEL: pow_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call powf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: pow_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -257,10 +362,23 @@ define float @exp_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail expf@plt
 ;
+; RV32IZFINX-LABEL: exp_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail expf@plt
+;
 ; RV64IF-LABEL: exp_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail expf@plt
 ;
+; RV64IZFINX-LABEL: exp_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call expf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: exp_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -289,10 +407,23 @@ define float @exp2_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail exp2f@plt
 ;
+; RV32IZFINX-LABEL: exp2_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail exp2f@plt
+;
 ; RV64IF-LABEL: exp2_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail exp2f@plt
 ;
+; RV64IZFINX-LABEL: exp2_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call exp2f@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: exp2_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -321,10 +452,23 @@ define float @log_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail logf@plt
 ;
+; RV32IZFINX-LABEL: log_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail logf@plt
+;
 ; RV64IF-LABEL: log_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail logf@plt
 ;
+; RV64IZFINX-LABEL: log_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call logf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: log_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -353,10 +497,23 @@ define float @log10_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail log10f@plt
 ;
+; RV32IZFINX-LABEL: log10_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail log10f@plt
+;
 ; RV64IF-LABEL: log10_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail log10f@plt
 ;
+; RV64IZFINX-LABEL: log10_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call log10f@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: log10_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -385,10 +542,23 @@ define float @log2_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail log2f@plt
 ;
+; RV32IZFINX-LABEL: log2_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail log2f@plt
+;
 ; RV64IF-LABEL: log2_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail log2f@plt
 ;
+; RV64IZFINX-LABEL: log2_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call log2f@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: log2_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -418,11 +588,21 @@ define float @fma_f32(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fma_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: fma_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: fma_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fma_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -452,11 +632,21 @@ define float @fmuladd_f32(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fmuladd_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: fmuladd_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fmadd.s fa0, fa0, fa1, fa2
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: fmuladd_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fmadd.s a0, a0, a1, a2
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fmuladd_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -496,11 +686,23 @@ define float @fabs_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fabs.s fa0, fa0
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: fabs_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    slli a0, a0, 1
+; RV32IZFINX-NEXT:    srli a0, a0, 1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: fabs_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fabs.s fa0, fa0
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: fabs_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    slli a0, a0, 33
+; RV64IZFINX-NEXT:    srli a0, a0, 33
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: fabs_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    slli a0, a0, 1
@@ -524,11 +726,21 @@ define float @minnum_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmin.s fa0, fa0, fa1
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: minnum_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fmin.s a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: minnum_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fmin.s fa0, fa0, fa1
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: minnum_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fmin.s a0, a0, a1
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: minnum_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -558,11 +770,21 @@ define float @maxnum_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmax.s fa0, fa0, fa1
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: maxnum_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fmax.s a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: maxnum_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fmax.s fa0, fa0, fa1
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: maxnum_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fmax.s a0, a0, a1
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: maxnum_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -609,11 +831,21 @@ define float @copysign_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fsgnj.s fa0, fa0, fa1
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: copysign_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fsgnj.s a0, a0, a1
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: copysign_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fsgnj.s fa0, fa0, fa1
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: copysign_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fsgnj.s a0, a0, a1
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: copysign_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a2, 524288
@@ -652,6 +884,19 @@ define float @floor_f32(float %a) nounwind {
 ; RV32IF-NEXT:  .LBB17_2:
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: floor_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB17_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB17_2:
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: floor_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 307200
@@ -666,6 +911,19 @@ define float @floor_f32(float %a) nounwind {
 ; RV64IF-NEXT:  .LBB17_2:
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: floor_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB17_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB17_2:
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: floor_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -704,6 +962,19 @@ define float @ceil_f32(float %a) nounwind {
 ; RV32IF-NEXT:  .LBB18_2:
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: ceil_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB18_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rup
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB18_2:
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: ceil_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 307200
@@ -718,6 +989,19 @@ define float @ceil_f32(float %a) nounwind {
 ; RV64IF-NEXT:  .LBB18_2:
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: ceil_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB18_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rup
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB18_2:
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: ceil_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -756,6 +1040,19 @@ define float @trunc_f32(float %a) nounwind {
 ; RV32IF-NEXT:  .LBB19_2:
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: trunc_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB19_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB19_2:
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: trunc_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 307200
@@ -770,6 +1067,19 @@ define float @trunc_f32(float %a) nounwind {
 ; RV64IF-NEXT:  .LBB19_2:
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: trunc_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB19_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB19_2:
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: trunc_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -808,6 +1118,19 @@ define float @rint_f32(float %a) nounwind {
 ; RV32IF-NEXT:  .LBB20_2:
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: rint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB20_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB20_2:
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: rint_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 307200
@@ -822,6 +1145,19 @@ define float @rint_f32(float %a) nounwind {
 ; RV64IF-NEXT:  .LBB20_2:
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: rint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB20_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB20_2:
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: rint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -850,10 +1186,23 @@ define float @nearbyint_f32(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    tail nearbyintf@plt
 ;
+; RV32IZFINX-LABEL: nearbyint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    tail nearbyintf@plt
+;
 ; RV64IF-LABEL: nearbyint_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    tail nearbyintf@plt
 ;
+; RV64IZFINX-LABEL: nearbyint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    call nearbyintf@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: nearbyint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -892,6 +1241,19 @@ define float @round_f32(float %a) nounwind {
 ; RV32IF-NEXT:  .LBB22_2:
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: round_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB22_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB22_2:
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: round_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 307200
@@ -906,6 +1268,19 @@ define float @round_f32(float %a) nounwind {
 ; RV64IF-NEXT:  .LBB22_2:
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: round_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB22_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB22_2:
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: round_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -944,6 +1319,19 @@ define float @roundeven_f32(float %a) nounwind {
 ; RV32IF-NEXT:  .LBB23_2:
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: roundeven_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB23_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rne
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB23_2:
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: roundeven_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    lui a0, 307200
@@ -958,6 +1346,19 @@ define float @roundeven_f32(float %a) nounwind {
 ; RV64IF-NEXT:  .LBB23_2:
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: roundeven_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB23_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rne
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB23_2:
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: roundeven_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -987,11 +1388,21 @@ define iXLen @lrint_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: lrint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: lrint_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: lrint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: lrint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1022,11 +1433,21 @@ define iXLen @lround_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rmm
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: lround_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: lround_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: lround_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: lround_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1056,11 +1477,21 @@ define i32 @lround_i32_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fcvt.w.s a0, fa0, rmm
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: lround_i32_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: lround_i32_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: lround_i32_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: lround_i32_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1094,11 +1525,25 @@ define i64 @llrint_f32(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: llrint_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call llrintf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: llrint_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: llrint_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: llrint_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1132,11 +1577,25 @@ define i64 @llround_f32(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
+; RV32IZFINX-LABEL: llround_f32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    call llroundf@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
 ; RV64IF-LABEL: llround_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
 ;
+; RV64IZFINX-LABEL: llround_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0
+; RV64IZFINX-NEXT:    ret
+;
 ; RV32I-LABEL: llround_f32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -1157,3 +1616,576 @@ define i64 @llround_f32(float %a) nounwind {
   %1 = call i64 @llvm.llround.i64.f32(float %a)
   ret i64 %1
 }
+
+declare i1 @llvm.is.fpclass.f32(float, i32)
+define i1 @fpclass(float %x) {
+; RV32IF-LABEL: fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 927
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 927
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 927
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 927
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    addi a2, a1, -1
+; RV32I-NEXT:    lui a3, 2048
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    sltu a2, a2, a3
+; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    and a2, a2, a0
+; RV32I-NEXT:    seqz a3, a1
+; RV32I-NEXT:    lui a4, 522240
+; RV32I-NEXT:    xor a5, a1, a4
+; RV32I-NEXT:    seqz a5, a5
+; RV32I-NEXT:    or a3, a3, a5
+; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    slt a3, a4, a1
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    lui a3, 1046528
+; RV32I-NEXT:    add a1, a1, a3
+; RV32I-NEXT:    srli a1, a1, 24
+; RV32I-NEXT:    sltiu a1, a1, 127
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    addi a2, a0, -1
+; RV64I-NEXT:    lui a3, 2048
+; RV64I-NEXT:    addiw a3, a3, -1
+; RV64I-NEXT:    sltu a2, a2, a3
+; RV64I-NEXT:    slti a1, a1, 0
+; RV64I-NEXT:    and a2, a2, a1
+; RV64I-NEXT:    seqz a3, a0
+; RV64I-NEXT:    lui a4, 522240
+; RV64I-NEXT:    xor a5, a0, a4
+; RV64I-NEXT:    seqz a5, a5
+; RV64I-NEXT:    or a3, a3, a5
+; RV64I-NEXT:    or a2, a3, a2
+; RV64I-NEXT:    slt a3, a4, a0
+; RV64I-NEXT:    or a2, a2, a3
+; RV64I-NEXT:    lui a3, 1046528
+; RV64I-NEXT:    add a0, a0, a3
+; RV64I-NEXT:    srliw a0, a0, 24
+; RV64I-NEXT:    sltiu a0, a0, 127
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    ret
+  %cmp = call i1 @llvm.is.fpclass.f32(float %x, i32 639)
+  ret i1 %cmp
+}
+
+define i1 @isnan_fpclass(float %x) {
+; RV32IF-LABEL: isnan_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 768
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isnan_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 768
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isnan_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 768
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isnan_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 768
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isnan_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slt a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnan_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slt a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 3)  ; nan
+  ret i1 %1
+}
+
+define i1 @isqnan_fpclass(float %x) {
+; RV32IF-LABEL: isqnan_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    slli a0, a0, 22
+; RV32IF-NEXT:    srli a0, a0, 31
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isqnan_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    slli a0, a0, 22
+; RV32IZFINX-NEXT:    srli a0, a0, 31
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isqnan_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    slli a0, a0, 54
+; RV64IF-NEXT:    srli a0, a0, 63
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isqnan_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    slli a0, a0, 54
+; RV64IZFINX-NEXT:    srli a0, a0, 63
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isqnan_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 523264
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    slt a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isqnan_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 523264
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    slt a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 2)  ; qnan
+  ret i1 %1
+}
+
+define i1 @issnan_fpclass(float %x) {
+; RV32IF-LABEL: issnan_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    slli a0, a0, 23
+; RV32IF-NEXT:    srli a0, a0, 31
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: issnan_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    slli a0, a0, 23
+; RV32IZFINX-NEXT:    srli a0, a0, 31
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: issnan_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    slli a0, a0, 55
+; RV64IF-NEXT:    srli a0, a0, 63
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: issnan_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    slli a0, a0, 55
+; RV64IZFINX-NEXT:    srli a0, a0, 63
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: issnan_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 523264
+; RV32I-NEXT:    slt a1, a0, a1
+; RV32I-NEXT:    lui a2, 522240
+; RV32I-NEXT:    slt a0, a2, a0
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: issnan_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 523264
+; RV64I-NEXT:    slt a1, a0, a1
+; RV64I-NEXT:    lui a2, 522240
+; RV64I-NEXT:    slt a0, a2, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 1)  ; snan
+  ret i1 %1
+}
+
+define i1 @isinf_fpclass(float %x) {
+; RV32IF-LABEL: isinf_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 129
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isinf_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 129
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isinf_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 129
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isinf_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 129
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isinf_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isinf_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 516)  ; 0x204 = "inf"
+  ret i1 %1
+}
+
+define i1 @isposinf_fpclass(float %x) {
+; RV32IF-LABEL: isposinf_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    slli a0, a0, 24
+; RV32IF-NEXT:    srli a0, a0, 31
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isposinf_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    slli a0, a0, 24
+; RV32IZFINX-NEXT:    srli a0, a0, 31
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isposinf_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    slli a0, a0, 56
+; RV64IF-NEXT:    srli a0, a0, 63
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isposinf_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    slli a0, a0, 56
+; RV64IZFINX-NEXT:    srli a0, a0, 63
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isposinf_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isposinf_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 512)  ; 0x200 = "+inf"
+  ret i1 %1
+}
+
+define i1 @isneginf_fpclass(float %x) {
+; RV32IF-LABEL: isneginf_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 1
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isneginf_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 1
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isneginf_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 1
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isneginf_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 1
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isneginf_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1046528
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isneginf_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    lui a1, 1046528
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 4)  ; "-inf"
+  ret i1 %1
+}
+
+define i1 @isfinite_fpclass(float %x) {
+; RV32IF-LABEL: isfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 126
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isfinite_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 126
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 126
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isfinite_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 126
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    slt a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isfinite_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    slt a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 504)  ; 0x1f8 = "finite"
+  ret i1 %1
+}
+
+define i1 @isposfinite_fpclass(float %x) {
+; RV32IF-LABEL: isposfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 112
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isposfinite_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 112
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isposfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 112
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isposfinite_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 112
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isposfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a0, a0, 23
+; RV32I-NEXT:    sltiu a0, a0, 255
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isposfinite_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a0, a0, 23
+; RV64I-NEXT:    sltiu a0, a0, 255
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 448)  ; 0x1c0 = "+finite"
+  ret i1 %1
+}
+
+define i1 @isnegfinite_fpclass(float %x) {
+; RV32IF-LABEL: isnegfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 14
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isnegfinite_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 14
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isnegfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 14
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isnegfinite_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 14
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isnegfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    lui a2, 522240
+; RV32I-NEXT:    slt a1, a1, a2
+; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnegfinite_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a2, 522240
+; RV64I-NEXT:    slt a0, a0, a2
+; RV64I-NEXT:    slti a1, a1, 0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 56)  ; 0x38 = "-finite"
+  ret i1 %1
+}
+
+define i1 @isnotfinite_fpclass(float %x) {
+; RV32IF-LABEL: isnotfinite_fpclass:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fclass.s a0, fa0
+; RV32IF-NEXT:    andi a0, a0, 897
+; RV32IF-NEXT:    snez a0, a0
+; RV32IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: isnotfinite_fpclass:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fclass.s a0, a0
+; RV32IZFINX-NEXT:    andi a0, a0, 897
+; RV32IZFINX-NEXT:    snez a0, a0
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IF-LABEL: isnotfinite_fpclass:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fclass.s a0, fa0
+; RV64IF-NEXT:    andi a0, a0, 897
+; RV64IF-NEXT:    snez a0, a0
+; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: isnotfinite_fpclass:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fclass.s a0, a0
+; RV64IZFINX-NEXT:    andi a0, a0, 897
+; RV64IZFINX-NEXT:    snez a0, a0
+; RV64IZFINX-NEXT:    ret
+;
+; RV32I-LABEL: isnotfinite_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 1
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    lui a1, 522240
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    slt a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnotfinite_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    lui a1, 522240
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    slt a0, a1, a0
+; RV64I-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 519)  ; ox207 = "inf|nan"
+  ret i1 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/float-isnan.ll b/llvm/test/CodeGen/RISCV/float-isnan.ll
index d3857cdc609ee..1c5355c417557 100644
--- a/llvm/test/CodeGen/RISCV/float-isnan.ll
+++ b/llvm/test/CodeGen/RISCV/float-isnan.ll
@@ -3,6 +3,10 @@
 ; RUN:   < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi lp64f -verify-machineinstrs \
 ; RUN:   < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -target-abi ilp32 -verify-machineinstrs \
+; RUN:   < %s | FileCheck --check-prefix=CHECKZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -target-abi lp64 -verify-machineinstrs \
+; RUN:   < %s | FileCheck --check-prefix=CHECKZFINX %s
 
 define zeroext i1 @float_is_nan(float %a) nounwind {
 ; CHECK-LABEL: float_is_nan:
@@ -10,6 +14,12 @@ define zeroext i1 @float_is_nan(float %a) nounwind {
 ; CHECK-NEXT:    feq.s a0, fa0, fa0
 ; CHECK-NEXT:    xori a0, a0, 1
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: float_is_nan:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKZFINX-NEXT:    xori a0, a0, 1
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp uno float %a, 0.000000e+00
   ret i1 %1
 }
@@ -19,6 +29,11 @@ define zeroext i1 @float_not_nan(float %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    feq.s a0, fa0, fa0
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: float_not_nan:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ord float %a, 0.000000e+00
   ret i1 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/float-mem.ll b/llvm/test/CodeGen/RISCV/float-mem.ll
index 145d2e315cf8a..8ef4f9162c2fe 100644
--- a/llvm/test/CodeGen/RISCV/float-mem.ll
+++ b/llvm/test/CodeGen/RISCV/float-mem.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefixes=CHECKIF,RV32IF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefixes=CHECKIF,RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefixes=CHECKIZFINX,RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=CHECKIZFINX,RV64IZFINX %s
 
 define dso_local float @flw(ptr %a) nounwind {
 ; CHECKIF-LABEL: flw:
@@ -11,6 +15,13 @@ define dso_local float @flw(ptr %a) nounwind {
 ; CHECKIF-NEXT:    flw fa4, 12(a0)
 ; CHECKIF-NEXT:    fadd.s fa0, fa5, fa4
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: flw:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    lw a1, 0(a0)
+; CHECKIZFINX-NEXT:    lw a0, 12(a0)
+; CHECKIZFINX-NEXT:    fadd.s a0, a1, a0
+; CHECKIZFINX-NEXT:    ret
   %1 = load float, ptr %a
   %2 = getelementptr float, ptr %a, i32 3
   %3 = load float, ptr %2
@@ -29,6 +40,13 @@ define dso_local void @fsw(ptr %a, float %b, float %c) nounwind {
 ; CHECKIF-NEXT:    fsw fa5, 0(a0)
 ; CHECKIF-NEXT:    fsw fa5, 32(a0)
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: fsw:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a1, a1, a2
+; CHECKIZFINX-NEXT:    sw a1, 0(a0)
+; CHECKIZFINX-NEXT:    sw a1, 32(a0)
+; CHECKIZFINX-NEXT:    ret
   %1 = fadd float %b, %c
   store float %1, ptr %a
   %2 = getelementptr float, ptr %a, i32 8
@@ -52,6 +70,17 @@ define dso_local float @flw_fsw_global(float %a, float %b) nounwind {
 ; CHECKIF-NEXT:    flw fa5, 36(a1)
 ; CHECKIF-NEXT:    fsw fa0, 36(a1)
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: flw_fsw_global:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fadd.s a0, a0, a1
+; CHECKIZFINX-NEXT:    lui a1, %hi(G)
+; CHECKIZFINX-NEXT:    lw a2, %lo(G)(a1)
+; CHECKIZFINX-NEXT:    addi a2, a1, %lo(G)
+; CHECKIZFINX-NEXT:    sw a0, %lo(G)(a1)
+; CHECKIZFINX-NEXT:    lw a1, 36(a2)
+; CHECKIZFINX-NEXT:    sw a0, 36(a2)
+; CHECKIZFINX-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = load volatile float, ptr @G
   store float %1, ptr @G
@@ -79,6 +108,23 @@ define dso_local float @flw_fsw_constant(float %a) nounwind {
 ; RV64IF-NEXT:    fadd.s fa0, fa0, fa5
 ; RV64IF-NEXT:    fsw fa0, -273(a0)
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: flw_fsw_constant:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 912092
+; RV32IZFINX-NEXT:    lw a2, -273(a1)
+; RV32IZFINX-NEXT:    fadd.s a0, a0, a2
+; RV32IZFINX-NEXT:    sw a0, -273(a1)
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: flw_fsw_constant:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 228023
+; RV64IZFINX-NEXT:    slli a1, a1, 2
+; RV64IZFINX-NEXT:    lw a2, -273(a1)
+; RV64IZFINX-NEXT:    fadd.s a0, a0, a2
+; RV64IZFINX-NEXT:    sw a0, -273(a1)
+; RV64IZFINX-NEXT:    ret
   %1 = inttoptr i32 3735928559 to ptr
   %2 = load volatile float, ptr %1
   %3 = fadd float %a, %2
@@ -118,6 +164,36 @@ define dso_local float @flw_stack(float %a) nounwind {
 ; RV64IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: flw_stack:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    addi a0, sp, 4
+; RV32IZFINX-NEXT:    call notdead@plt
+; RV32IZFINX-NEXT:    lw a0, 4(sp)
+; RV32IZFINX-NEXT:    fadd.s a0, a0, s0
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: flw_stack:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -32
+; RV64IZFINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    mv s0, a0
+; RV64IZFINX-NEXT:    addi a0, sp, 12
+; RV64IZFINX-NEXT:    call notdead@plt
+; RV64IZFINX-NEXT:    lw a0, 12(sp)
+; RV64IZFINX-NEXT:    fadd.s a0, a0, s0
+; RV64IZFINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 32
+; RV64IZFINX-NEXT:    ret
   %1 = alloca float, align 4
   call void @notdead(ptr %1)
   %2 = load float, ptr %1
@@ -149,6 +225,30 @@ define dso_local void @fsw_stack(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: fsw_stack:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    fadd.s a0, a0, a1
+; RV32IZFINX-NEXT:    sw a0, 8(sp)
+; RV32IZFINX-NEXT:    addi a0, sp, 8
+; RV32IZFINX-NEXT:    call notdead@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: fsw_stack:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    addi sp, sp, -16
+; RV64IZFINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFINX-NEXT:    fadd.s a0, a0, a1
+; RV64IZFINX-NEXT:    sw a0, 4(sp)
+; RV64IZFINX-NEXT:    addi a0, sp, 4
+; RV64IZFINX-NEXT:    call notdead@plt
+; RV64IZFINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFINX-NEXT:    addi sp, sp, 16
+; RV64IZFINX-NEXT:    ret
   %1 = fadd float %a, %b ; force store from FPR32
   %2 = alloca float, align 4
   store float %1, ptr %2
diff --git a/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll
index a87fd92d5780e..61337216c7fb5 100644
--- a/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefixes=CHECKIF,RV32IF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefixes=CHECKIF,RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefixes=CHECKIZFINX,RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefixes=CHECKIZFINX,RV64IZFINX %s
 
 define signext i32 @test_floor_si32(float %x) {
 ; CHECKIF-LABEL: test_floor_si32:
@@ -13,6 +17,15 @@ define signext i32 @test_floor_si32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_floor_si32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = call i32 @llvm.fptosi.sat.i32.f32(float %a)
   ret i32 %b
@@ -41,27 +54,27 @@ define i64 @test_floor_si64(float %x) nounwind {
 ; RV32IF-NEXT:    fle.s s0, fa5, fs0
 ; RV32IF-NEXT:    fmv.s fa0, fs0
 ; RV32IF-NEXT:    call __fixsfdi@plt
-; RV32IF-NEXT:    lui a3, 524288
-; RV32IF-NEXT:    bnez s0, .LBB1_4
+; RV32IF-NEXT:    lui a4, 524288
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    beqz s0, .LBB1_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    mv a2, a1
 ; RV32IF-NEXT:  .LBB1_4:
-; RV32IF-NEXT:    lui a2, %hi(.LCPI1_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI1_0)(a2)
-; RV32IF-NEXT:    flt.s a2, fa5, fs0
-; RV32IF-NEXT:    beqz a2, .LBB1_6
+; RV32IF-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI1_0)(a1)
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    beqz a3, .LBB1_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a1, a3, -1
+; RV32IF-NEXT:    addi a2, a4, -1
 ; RV32IF-NEXT:  .LBB1_6:
-; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    seqz a3, a3
-; RV32IF-NEXT:    addi a3, a3, -1
-; RV32IF-NEXT:    and a1, a3, a1
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    and a0, a4, a0
-; RV32IF-NEXT:    neg a2, a2
+; RV32IF-NEXT:    feq.s a1, fs0, fs0
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    and a1, a4, a2
+; RV32IF-NEXT:    neg a2, s0
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
-; RV32IF-NEXT:    and a0, a3, a0
+; RV32IF-NEXT:    and a0, a4, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -76,6 +89,64 @@ define i64 @test_floor_si64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB1_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rdn
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rdn
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB1_2:
+; RV32IZFINX-NEXT:    lui a0, 913408
+; RV32IZFINX-NEXT:    fle.s s1, a0, s0
+; RV32IZFINX-NEXT:    neg s2, s1
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI1_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s2, a0
+; RV32IZFINX-NEXT:    flt.s a4, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    feq.s a2, s0, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    lui a5, 524288
+; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    beqz s1, .LBB1_4
+; RV32IZFINX-NEXT:  # %bb.3:
+; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:  .LBB1_4:
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    beqz a4, .LBB1_6
+; RV32IZFINX-NEXT:  # %bb.5:
+; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:  .LBB1_6:
+; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a0, rdn
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = call i64 @llvm.fptosi.sat.i64.f32(float %a)
   ret i64 %b
@@ -90,6 +161,15 @@ define signext i32 @test_floor_ui32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_floor_ui32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a1, a0, rdn
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = call i32 @llvm.fptoui.sat.i32.f32(float %a)
   ret i32 %b
@@ -140,6 +220,49 @@ define i64 @test_floor_ui64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB3_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rdn
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rdn
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB3_2:
+; RV32IZFINX-NEXT:    fle.s a0, zero, s0
+; RV32IZFINX-NEXT:    neg s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI3_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI3_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s1, a0
+; RV32IZFINX-NEXT:    flt.s a2, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a1, s1, a1
+; RV32IZFINX-NEXT:    or a1, a2, a1
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a1, a0, rdn
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = call i64 @llvm.fptoui.sat.i64.f32(float %a)
   ret i64 %b
@@ -154,6 +277,15 @@ define signext i32 @test_ceil_si32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_ceil_si32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = call i32 @llvm.fptosi.sat.i32.f32(float %a)
   ret i32 %b
@@ -182,27 +314,27 @@ define i64 @test_ceil_si64(float %x) nounwind {
 ; RV32IF-NEXT:    fle.s s0, fa5, fs0
 ; RV32IF-NEXT:    fmv.s fa0, fs0
 ; RV32IF-NEXT:    call __fixsfdi@plt
-; RV32IF-NEXT:    lui a3, 524288
-; RV32IF-NEXT:    bnez s0, .LBB5_4
+; RV32IF-NEXT:    lui a4, 524288
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    beqz s0, .LBB5_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    mv a2, a1
 ; RV32IF-NEXT:  .LBB5_4:
-; RV32IF-NEXT:    lui a2, %hi(.LCPI5_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI5_0)(a2)
-; RV32IF-NEXT:    flt.s a2, fa5, fs0
-; RV32IF-NEXT:    beqz a2, .LBB5_6
+; RV32IF-NEXT:    lui a1, %hi(.LCPI5_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI5_0)(a1)
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    beqz a3, .LBB5_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a1, a3, -1
+; RV32IF-NEXT:    addi a2, a4, -1
 ; RV32IF-NEXT:  .LBB5_6:
-; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    seqz a3, a3
-; RV32IF-NEXT:    addi a3, a3, -1
-; RV32IF-NEXT:    and a1, a3, a1
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    and a0, a4, a0
-; RV32IF-NEXT:    neg a2, a2
+; RV32IF-NEXT:    feq.s a1, fs0, fs0
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    and a1, a4, a2
+; RV32IF-NEXT:    neg a2, s0
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
-; RV32IF-NEXT:    and a0, a3, a0
+; RV32IF-NEXT:    and a0, a4, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -217,6 +349,64 @@ define i64 @test_ceil_si64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB5_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rup
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rup
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB5_2:
+; RV32IZFINX-NEXT:    lui a0, 913408
+; RV32IZFINX-NEXT:    fle.s s1, a0, s0
+; RV32IZFINX-NEXT:    neg s2, s1
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI5_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI5_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s2, a0
+; RV32IZFINX-NEXT:    flt.s a4, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    feq.s a2, s0, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    lui a5, 524288
+; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    beqz s1, .LBB5_4
+; RV32IZFINX-NEXT:  # %bb.3:
+; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:  .LBB5_4:
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    beqz a4, .LBB5_6
+; RV32IZFINX-NEXT:  # %bb.5:
+; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:  .LBB5_6:
+; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a0, rup
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = call i64 @llvm.fptosi.sat.i64.f32(float %a)
   ret i64 %b
@@ -231,6 +421,15 @@ define signext i32 @test_ceil_ui32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_ceil_ui32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a1, a0, rup
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = call i32 @llvm.fptoui.sat.i32.f32(float %a)
   ret i32 %b
@@ -281,6 +480,49 @@ define i64 @test_ceil_ui64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB7_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rup
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rup
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB7_2:
+; RV32IZFINX-NEXT:    fle.s a0, zero, s0
+; RV32IZFINX-NEXT:    neg s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI7_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI7_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s1, a0
+; RV32IZFINX-NEXT:    flt.s a2, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a1, s1, a1
+; RV32IZFINX-NEXT:    or a1, a2, a1
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a1, a0, rup
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = call i64 @llvm.fptoui.sat.i64.f32(float %a)
   ret i64 %b
@@ -295,6 +537,15 @@ define signext i32 @test_trunc_si32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_trunc_si32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = call i32 @llvm.fptosi.sat.i32.f32(float %a)
   ret i32 %b
@@ -323,27 +574,27 @@ define i64 @test_trunc_si64(float %x) nounwind {
 ; RV32IF-NEXT:    fle.s s0, fa5, fs0
 ; RV32IF-NEXT:    fmv.s fa0, fs0
 ; RV32IF-NEXT:    call __fixsfdi@plt
-; RV32IF-NEXT:    lui a3, 524288
-; RV32IF-NEXT:    bnez s0, .LBB9_4
+; RV32IF-NEXT:    lui a4, 524288
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    beqz s0, .LBB9_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    mv a2, a1
 ; RV32IF-NEXT:  .LBB9_4:
-; RV32IF-NEXT:    lui a2, %hi(.LCPI9_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI9_0)(a2)
-; RV32IF-NEXT:    flt.s a2, fa5, fs0
-; RV32IF-NEXT:    beqz a2, .LBB9_6
+; RV32IF-NEXT:    lui a1, %hi(.LCPI9_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI9_0)(a1)
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    beqz a3, .LBB9_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a1, a3, -1
+; RV32IF-NEXT:    addi a2, a4, -1
 ; RV32IF-NEXT:  .LBB9_6:
-; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    seqz a3, a3
-; RV32IF-NEXT:    addi a3, a3, -1
-; RV32IF-NEXT:    and a1, a3, a1
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    and a0, a4, a0
-; RV32IF-NEXT:    neg a2, a2
+; RV32IF-NEXT:    feq.s a1, fs0, fs0
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    and a1, a4, a2
+; RV32IF-NEXT:    neg a2, s0
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
-; RV32IF-NEXT:    and a0, a3, a0
+; RV32IF-NEXT:    and a0, a4, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -358,6 +609,64 @@ define i64 @test_trunc_si64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB9_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rtz
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rtz
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB9_2:
+; RV32IZFINX-NEXT:    lui a0, 913408
+; RV32IZFINX-NEXT:    fle.s s1, a0, s0
+; RV32IZFINX-NEXT:    neg s2, s1
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI9_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI9_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s2, a0
+; RV32IZFINX-NEXT:    flt.s a4, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    feq.s a2, s0, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    lui a5, 524288
+; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    beqz s1, .LBB9_4
+; RV32IZFINX-NEXT:  # %bb.3:
+; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:  .LBB9_4:
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    beqz a4, .LBB9_6
+; RV32IZFINX-NEXT:  # %bb.5:
+; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:  .LBB9_6:
+; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a0, rtz
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = call i64 @llvm.fptosi.sat.i64.f32(float %a)
   ret i64 %b
@@ -372,6 +681,15 @@ define signext i32 @test_trunc_ui32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_trunc_ui32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a1, a0, rtz
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = call i32 @llvm.fptoui.sat.i32.f32(float %a)
   ret i32 %b
@@ -422,6 +740,49 @@ define i64 @test_trunc_ui64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB11_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rtz
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rtz
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB11_2:
+; RV32IZFINX-NEXT:    fle.s a0, zero, s0
+; RV32IZFINX-NEXT:    neg s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI11_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI11_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s1, a0
+; RV32IZFINX-NEXT:    flt.s a2, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a1, s1, a1
+; RV32IZFINX-NEXT:    or a1, a2, a1
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a1, a0, rtz
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = call i64 @llvm.fptoui.sat.i64.f32(float %a)
   ret i64 %b
@@ -436,6 +797,15 @@ define signext i32 @test_round_si32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_round_si32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = call i32 @llvm.fptosi.sat.i32.f32(float %a)
   ret i32 %b
@@ -464,27 +834,27 @@ define i64 @test_round_si64(float %x) nounwind {
 ; RV32IF-NEXT:    fle.s s0, fa5, fs0
 ; RV32IF-NEXT:    fmv.s fa0, fs0
 ; RV32IF-NEXT:    call __fixsfdi@plt
-; RV32IF-NEXT:    lui a3, 524288
-; RV32IF-NEXT:    bnez s0, .LBB13_4
+; RV32IF-NEXT:    lui a4, 524288
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    beqz s0, .LBB13_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    mv a2, a1
 ; RV32IF-NEXT:  .LBB13_4:
-; RV32IF-NEXT:    lui a2, %hi(.LCPI13_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI13_0)(a2)
-; RV32IF-NEXT:    flt.s a2, fa5, fs0
-; RV32IF-NEXT:    beqz a2, .LBB13_6
+; RV32IF-NEXT:    lui a1, %hi(.LCPI13_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI13_0)(a1)
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    beqz a3, .LBB13_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a1, a3, -1
+; RV32IF-NEXT:    addi a2, a4, -1
 ; RV32IF-NEXT:  .LBB13_6:
-; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    seqz a3, a3
-; RV32IF-NEXT:    addi a3, a3, -1
-; RV32IF-NEXT:    and a1, a3, a1
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    and a0, a4, a0
-; RV32IF-NEXT:    neg a2, a2
+; RV32IF-NEXT:    feq.s a1, fs0, fs0
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    and a1, a4, a2
+; RV32IF-NEXT:    neg a2, s0
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
-; RV32IF-NEXT:    and a0, a3, a0
+; RV32IF-NEXT:    and a0, a4, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -499,6 +869,64 @@ define i64 @test_round_si64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB13_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rmm
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rmm
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB13_2:
+; RV32IZFINX-NEXT:    lui a0, 913408
+; RV32IZFINX-NEXT:    fle.s s1, a0, s0
+; RV32IZFINX-NEXT:    neg s2, s1
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI13_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI13_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s2, a0
+; RV32IZFINX-NEXT:    flt.s a4, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    feq.s a2, s0, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    lui a5, 524288
+; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    beqz s1, .LBB13_4
+; RV32IZFINX-NEXT:  # %bb.3:
+; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:  .LBB13_4:
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    beqz a4, .LBB13_6
+; RV32IZFINX-NEXT:  # %bb.5:
+; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:  .LBB13_6:
+; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a0, rmm
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = call i64 @llvm.fptosi.sat.i64.f32(float %a)
   ret i64 %b
@@ -513,6 +941,15 @@ define signext i32 @test_round_ui32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_round_ui32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a1, a0, rmm
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = call i32 @llvm.fptoui.sat.i32.f32(float %a)
   ret i32 %b
@@ -563,6 +1000,49 @@ define i64 @test_round_ui64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB15_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rmm
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rmm
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB15_2:
+; RV32IZFINX-NEXT:    fle.s a0, zero, s0
+; RV32IZFINX-NEXT:    neg s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI15_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI15_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s1, a0
+; RV32IZFINX-NEXT:    flt.s a2, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a1, s1, a1
+; RV32IZFINX-NEXT:    or a1, a2, a1
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a1, a0, rmm
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = call i64 @llvm.fptoui.sat.i64.f32(float %a)
   ret i64 %b
@@ -577,6 +1057,15 @@ define signext i32 @test_roundeven_si32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_roundeven_si32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = call i32 @llvm.fptosi.sat.i32.f32(float %a)
   ret i32 %b
@@ -605,27 +1094,27 @@ define i64 @test_roundeven_si64(float %x) nounwind {
 ; RV32IF-NEXT:    fle.s s0, fa5, fs0
 ; RV32IF-NEXT:    fmv.s fa0, fs0
 ; RV32IF-NEXT:    call __fixsfdi@plt
-; RV32IF-NEXT:    lui a3, 524288
-; RV32IF-NEXT:    bnez s0, .LBB17_4
+; RV32IF-NEXT:    lui a4, 524288
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    beqz s0, .LBB17_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    mv a2, a1
 ; RV32IF-NEXT:  .LBB17_4:
-; RV32IF-NEXT:    lui a2, %hi(.LCPI17_0)
-; RV32IF-NEXT:    flw fa5, %lo(.LCPI17_0)(a2)
-; RV32IF-NEXT:    flt.s a2, fa5, fs0
-; RV32IF-NEXT:    beqz a2, .LBB17_6
+; RV32IF-NEXT:    lui a1, %hi(.LCPI17_0)
+; RV32IF-NEXT:    flw fa5, %lo(.LCPI17_0)(a1)
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    beqz a3, .LBB17_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a1, a3, -1
+; RV32IF-NEXT:    addi a2, a4, -1
 ; RV32IF-NEXT:  .LBB17_6:
-; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    seqz a3, a3
-; RV32IF-NEXT:    addi a3, a3, -1
-; RV32IF-NEXT:    and a1, a3, a1
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    and a0, a4, a0
-; RV32IF-NEXT:    neg a2, a2
+; RV32IF-NEXT:    feq.s a1, fs0, fs0
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    and a1, a4, a2
+; RV32IF-NEXT:    neg a2, s0
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
-; RV32IF-NEXT:    and a0, a3, a0
+; RV32IF-NEXT:    and a0, a4, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -640,6 +1129,64 @@ define i64 @test_roundeven_si64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB17_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rne
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rne
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB17_2:
+; RV32IZFINX-NEXT:    lui a0, 913408
+; RV32IZFINX-NEXT:    fle.s s1, a0, s0
+; RV32IZFINX-NEXT:    neg s2, s1
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI17_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI17_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s2, a0
+; RV32IZFINX-NEXT:    flt.s a4, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    feq.s a2, s0, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    lui a5, 524288
+; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    beqz s1, .LBB17_4
+; RV32IZFINX-NEXT:  # %bb.3:
+; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:  .LBB17_4:
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    beqz a4, .LBB17_6
+; RV32IZFINX-NEXT:  # %bb.5:
+; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:  .LBB17_6:
+; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a1, a0, rne
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = call i64 @llvm.fptosi.sat.i64.f32(float %a)
   ret i64 %b
@@ -654,6 +1201,15 @@ define signext i32 @test_roundeven_ui32(float %x) {
 ; CHECKIF-NEXT:    addi a1, a1, -1
 ; CHECKIF-NEXT:    and a0, a1, a0
 ; CHECKIF-NEXT:    ret
+;
+; CHECKIZFINX-LABEL: test_roundeven_ui32:
+; CHECKIZFINX:       # %bb.0:
+; CHECKIZFINX-NEXT:    fcvt.wu.s a1, a0, rne
+; CHECKIZFINX-NEXT:    feq.s a0, a0, a0
+; CHECKIZFINX-NEXT:    seqz a0, a0
+; CHECKIZFINX-NEXT:    addi a0, a0, -1
+; CHECKIZFINX-NEXT:    and a0, a0, a1
+; CHECKIZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = call i32 @llvm.fptoui.sat.i32.f32(float %a)
   ret i32 %b
@@ -704,6 +1260,49 @@ define i64 @test_roundeven_ui64(float %x) nounwind {
 ; RV64IF-NEXT:    addi a1, a1, -1
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    mv s0, a0
+; RV32IZFINX-NEXT:    lui a0, 307200
+; RV32IZFINX-NEXT:    fabs.s a1, s0
+; RV32IZFINX-NEXT:    flt.s a0, a1, a0
+; RV32IZFINX-NEXT:    beqz a0, .LBB19_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, s0, rne
+; RV32IZFINX-NEXT:    fcvt.s.w a0, a0, rne
+; RV32IZFINX-NEXT:    fsgnj.s s0, a0, s0
+; RV32IZFINX-NEXT:  .LBB19_2:
+; RV32IZFINX-NEXT:    fle.s a0, zero, s0
+; RV32IZFINX-NEXT:    neg s1, a0
+; RV32IZFINX-NEXT:    mv a0, s0
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI19_0)
+; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI19_0)(a2)
+; RV32IZFINX-NEXT:    and a0, s1, a0
+; RV32IZFINX-NEXT:    flt.s a2, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a2
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a1, s1, a1
+; RV32IZFINX-NEXT:    or a1, a2, a1
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a1, a0, rne
+; RV64IZFINX-NEXT:    feq.s a0, a0, a0
+; RV64IZFINX-NEXT:    seqz a0, a0
+; RV64IZFINX-NEXT:    addi a0, a0, -1
+; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = call i64 @llvm.fptoui.sat.i64.f32(float %a)
   ret i64 %b
diff --git a/llvm/test/CodeGen/RISCV/float-round-conv.ll b/llvm/test/CodeGen/RISCV/float-round-conv.ll
index 344950df43df1..ed50f867cdb8b 100644
--- a/llvm/test/CodeGen/RISCV/float-round-conv.ll
+++ b/llvm/test/CodeGen/RISCV/float-round-conv.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck -check-prefix=RV32IF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck -check-prefix=RV64IF %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck -check-prefix=RV32IZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck -check-prefix=RV64IZFINX %s
 
 define signext i8 @test_floor_si8(float %x) {
 ; RV32IF-LABEL: test_floor_si8:
@@ -14,6 +18,16 @@ define signext i8 @test_floor_si8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_si8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rdn
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_si8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptosi float %a to i8
   ret i8 %b
@@ -29,6 +43,16 @@ define signext i16 @test_floor_si16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_si16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rdn
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_si16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptosi float %a to i16
   ret i16 %b
@@ -44,6 +68,16 @@ define signext i32 @test_floor_si32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_si32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rdn
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_si32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptosi float %a to i32
   ret i32 %b
@@ -75,6 +109,31 @@ define i64 @test_floor_si64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB3_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB3_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptosi float %a to i64
   ret i64 %b
@@ -90,6 +149,16 @@ define zeroext i8 @test_floor_ui8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_ui8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rdn
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_ui8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptoui float %a to i8
   ret i8 %b
@@ -105,6 +174,16 @@ define zeroext i16 @test_floor_ui16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_ui16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rdn
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_ui16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptoui float %a to i16
   ret i16 %b
@@ -120,6 +199,16 @@ define signext i32 @test_floor_ui32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_ui32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rdn
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_ui32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptoui float %a to i32
   ret i32 %b
@@ -151,6 +240,31 @@ define i64 @test_floor_ui64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rdn
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB7_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB7_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rdn
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   %b = fptoui float %a to i64
   ret i64 %b
@@ -166,6 +280,16 @@ define signext i8 @test_ceil_si8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_si8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rup
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_si8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptosi float %a to i8
   ret i8 %b
@@ -181,6 +305,16 @@ define signext i16 @test_ceil_si16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_si16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rup
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_si16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptosi float %a to i16
   ret i16 %b
@@ -196,6 +330,16 @@ define signext i32 @test_ceil_si32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_si32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rup
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_si32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptosi float %a to i32
   ret i32 %b
@@ -227,6 +371,31 @@ define i64 @test_ceil_si64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB11_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rup
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB11_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptosi float %a to i64
   ret i64 %b
@@ -242,6 +411,16 @@ define zeroext i8 @test_ceil_ui8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_ui8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rup
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_ui8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptoui float %a to i8
   ret i8 %b
@@ -257,6 +436,16 @@ define zeroext i16 @test_ceil_ui16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_ui16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rup
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_ui16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptoui float %a to i16
   ret i16 %b
@@ -272,6 +461,16 @@ define signext i32 @test_ceil_ui32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_ui32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rup
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_ui32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptoui float %a to i32
   ret i32 %b
@@ -303,6 +502,31 @@ define i64 @test_ceil_ui64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rup
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB15_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rup
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB15_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rup
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   %b = fptoui float %a to i64
   ret i64 %b
@@ -318,6 +542,16 @@ define signext i8 @test_trunc_si8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_si8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_si8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptosi float %a to i8
   ret i8 %b
@@ -333,6 +567,16 @@ define signext i16 @test_trunc_si16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_si16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_si16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptosi float %a to i16
   ret i16 %b
@@ -348,6 +592,16 @@ define signext i32 @test_trunc_si32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_si32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_si32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptosi float %a to i32
   ret i32 %b
@@ -379,6 +633,31 @@ define i64 @test_trunc_si64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB19_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB19_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptosi float %a to i64
   ret i64 %b
@@ -394,6 +673,16 @@ define zeroext i8 @test_trunc_ui8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_ui8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_ui8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptoui float %a to i8
   ret i8 %b
@@ -409,6 +698,16 @@ define zeroext i16 @test_trunc_ui16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_ui16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_ui16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptoui float %a to i16
   ret i16 %b
@@ -424,6 +723,16 @@ define signext i32 @test_trunc_ui32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_ui32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_ui32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptoui float %a to i32
   ret i32 %b
@@ -455,6 +764,31 @@ define i64 @test_trunc_ui64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB23_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB23_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   %b = fptoui float %a to i64
   ret i64 %b
@@ -470,6 +804,16 @@ define signext i8 @test_round_si8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_si8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_si8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptosi float %a to i8
   ret i8 %b
@@ -485,6 +829,16 @@ define signext i16 @test_round_si16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_si16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_si16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptosi float %a to i16
   ret i16 %b
@@ -500,6 +854,16 @@ define signext i32 @test_round_si32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_si32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_si32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptosi float %a to i32
   ret i32 %b
@@ -531,6 +895,31 @@ define i64 @test_round_si64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB27_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB27_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptosi float %a to i64
   ret i64 %b
@@ -546,6 +935,16 @@ define zeroext i8 @test_round_ui8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_ui8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_ui8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptoui float %a to i8
   ret i8 %b
@@ -561,6 +960,16 @@ define zeroext i16 @test_round_ui16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_ui16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_ui16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptoui float %a to i16
   ret i16 %b
@@ -576,6 +985,16 @@ define signext i32 @test_round_ui32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_ui32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rmm
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_ui32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptoui float %a to i32
   ret i32 %b
@@ -607,6 +1026,31 @@ define i64 @test_round_ui64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rmm
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB31_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB31_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rmm
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   %b = fptoui float %a to i64
   ret i64 %b
@@ -622,6 +1066,16 @@ define signext i8 @test_roundeven_si8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_si8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rne
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_si8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptosi float %a to i8
   ret i8 %b
@@ -637,6 +1091,16 @@ define signext i16 @test_roundeven_si16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_si16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rne
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_si16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptosi float %a to i16
   ret i16 %b
@@ -652,6 +1116,16 @@ define signext i32 @test_roundeven_si32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_si32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.w.s a0, a0, rne
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_si32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptosi float %a to i32
   ret i32 %b
@@ -683,6 +1157,31 @@ define i64 @test_roundeven_si64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.l.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_si64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB35_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rne
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB35_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixsfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_si64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptosi float %a to i64
   ret i64 %b
@@ -698,6 +1197,16 @@ define zeroext i8 @test_roundeven_ui8(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_ui8:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rne
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_ui8:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptoui float %a to i8
   ret i8 %b
@@ -713,6 +1222,16 @@ define zeroext i16 @test_roundeven_ui16(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_ui16:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rne
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_ui16:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptoui float %a to i16
   ret i16 %b
@@ -728,6 +1247,16 @@ define signext i32 @test_roundeven_ui32(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_ui32:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    fcvt.wu.s a0, a0, rne
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_ui32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptoui float %a to i32
   ret i32 %b
@@ -759,6 +1288,31 @@ define i64 @test_roundeven_ui64(float %x) {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rne
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_ui64:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB39_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rne
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB39_2:
+; RV32IZFINX-NEXT:    addi sp, sp, -16
+; RV32IZFINX-NEXT:    .cfi_def_cfa_offset 16
+; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINX-NEXT:    .cfi_offset ra, -4
+; RV32IZFINX-NEXT:    call __fixunssfdi@plt
+; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINX-NEXT:    addi sp, sp, 16
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_ui64:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rne
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   %b = fptoui float %a to i64
   ret i64 %b
@@ -813,6 +1367,32 @@ define float @test_floor_float(float %x) {
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
 ; RV64IF-NEXT:  .LBB40_2:
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_floor_float:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB40_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB40_2:
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_floor_float:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB40_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rdn
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rdn
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB40_2:
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.floor.f32(float %x)
   ret float %a
 }
@@ -866,6 +1446,32 @@ define float @test_ceil_float(float %x) {
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
 ; RV64IF-NEXT:  .LBB41_2:
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_ceil_float:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB41_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rup
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB41_2:
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_ceil_float:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB41_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rup
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rup
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB41_2:
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.ceil.f32(float %x)
   ret float %a
 }
@@ -919,6 +1525,32 @@ define float @test_trunc_float(float %x) {
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
 ; RV64IF-NEXT:  .LBB42_2:
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_trunc_float:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB42_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB42_2:
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_trunc_float:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB42_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rtz
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rtz
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB42_2:
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.trunc.f32(float %x)
   ret float %a
 }
@@ -972,6 +1604,32 @@ define float @test_round_float(float %x) {
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
 ; RV64IF-NEXT:  .LBB43_2:
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_round_float:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB43_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB43_2:
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_round_float:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB43_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rmm
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rmm
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB43_2:
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.round.f32(float %x)
   ret float %a
 }
@@ -1025,6 +1683,32 @@ define float @test_roundeven_float(float %x) {
 ; RV64IF-NEXT:    fsgnj.s fa0, fa5, fa0
 ; RV64IF-NEXT:  .LBB44_2:
 ; RV64IF-NEXT:    ret
+;
+; RV32IZFINX-LABEL: test_roundeven_float:
+; RV32IZFINX:       # %bb.0:
+; RV32IZFINX-NEXT:    lui a1, 307200
+; RV32IZFINX-NEXT:    fabs.s a2, a0
+; RV32IZFINX-NEXT:    flt.s a1, a2, a1
+; RV32IZFINX-NEXT:    beqz a1, .LBB44_2
+; RV32IZFINX-NEXT:  # %bb.1:
+; RV32IZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; RV32IZFINX-NEXT:    fcvt.s.w a1, a1, rne
+; RV32IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV32IZFINX-NEXT:  .LBB44_2:
+; RV32IZFINX-NEXT:    ret
+;
+; RV64IZFINX-LABEL: test_roundeven_float:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    lui a1, 307200
+; RV64IZFINX-NEXT:    fabs.s a2, a0
+; RV64IZFINX-NEXT:    flt.s a1, a2, a1
+; RV64IZFINX-NEXT:    beqz a1, .LBB44_2
+; RV64IZFINX-NEXT:  # %bb.1:
+; RV64IZFINX-NEXT:    fcvt.w.s a1, a0, rne
+; RV64IZFINX-NEXT:    fcvt.s.w a1, a1, rne
+; RV64IZFINX-NEXT:    fsgnj.s a0, a1, a0
+; RV64IZFINX-NEXT:  .LBB44_2:
+; RV64IZFINX-NEXT:    ret
   %a = call float @llvm.roundeven.f32(float %x)
   ret float %a
 }
diff --git a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
index c6e883624f5ae..a2ff0d33e2d31 100644
--- a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
@@ -3,12 +3,21 @@
 ; RUN:   -target-abi=ilp32f | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck --check-prefix=CHECKZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck --check-prefix=CHECKZFINX %s
 
 define float @select_fcmp_false(float %a, float %b) nounwind {
 ; CHECK-LABEL: select_fcmp_false:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_false:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp false float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -23,6 +32,15 @@ define float @select_fcmp_oeq(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_oeq:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a2, a0, a1
+; CHECKZFINX-NEXT:    bnez a2, .LBB1_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB1_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -37,6 +55,15 @@ define float @select_fcmp_ogt(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ogt:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    flt.s a2, a1, a0
+; CHECKZFINX-NEXT:    bnez a2, .LBB2_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB2_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ogt float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -51,6 +78,15 @@ define float @select_fcmp_oge(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_oge:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    fle.s a2, a1, a0
+; CHECKZFINX-NEXT:    bnez a2, .LBB3_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB3_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp oge float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -65,6 +101,15 @@ define float @select_fcmp_olt(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB4_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_olt:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKZFINX-NEXT:    bnez a2, .LBB4_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB4_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp olt float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -79,6 +124,15 @@ define float @select_fcmp_ole(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB5_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ole:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    fle.s a2, a0, a1
+; CHECKZFINX-NEXT:    bnez a2, .LBB5_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB5_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ole float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -95,6 +149,17 @@ define float @select_fcmp_one(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB6_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_one:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKZFINX-NEXT:    flt.s a3, a1, a0
+; CHECKZFINX-NEXT:    or a2, a3, a2
+; CHECKZFINX-NEXT:    bnez a2, .LBB6_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB6_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp one float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -111,6 +176,17 @@ define float @select_fcmp_ord(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ord:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a2, a1, a1
+; CHECKZFINX-NEXT:    feq.s a3, a0, a0
+; CHECKZFINX-NEXT:    and a2, a3, a2
+; CHECKZFINX-NEXT:    bnez a2, .LBB7_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB7_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ord float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -127,6 +203,17 @@ define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ueq:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKZFINX-NEXT:    flt.s a3, a1, a0
+; CHECKZFINX-NEXT:    or a2, a3, a2
+; CHECKZFINX-NEXT:    beqz a2, .LBB8_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB8_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ueq float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -141,6 +228,15 @@ define float @select_fcmp_ugt(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB9_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ugt:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    fle.s a2, a0, a1
+; CHECKZFINX-NEXT:    beqz a2, .LBB9_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB9_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ugt float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -155,6 +251,15 @@ define float @select_fcmp_uge(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB10_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_uge:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    flt.s a2, a0, a1
+; CHECKZFINX-NEXT:    beqz a2, .LBB10_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB10_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp uge float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -169,6 +274,15 @@ define float @select_fcmp_ult(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB11_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ult:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    fle.s a2, a1, a0
+; CHECKZFINX-NEXT:    beqz a2, .LBB11_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB11_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ult float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -183,6 +297,15 @@ define float @select_fcmp_ule(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB12_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_ule:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    flt.s a2, a1, a0
+; CHECKZFINX-NEXT:    beqz a2, .LBB12_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB12_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ule float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -197,6 +320,15 @@ define float @select_fcmp_une(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB13_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_une:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a2, a0, a1
+; CHECKZFINX-NEXT:    beqz a2, .LBB13_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB13_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp une float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -213,6 +345,17 @@ define float @select_fcmp_uno(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB14_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_uno:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a2, a1, a1
+; CHECKZFINX-NEXT:    feq.s a3, a0, a0
+; CHECKZFINX-NEXT:    and a2, a3, a2
+; CHECKZFINX-NEXT:    beqz a2, .LBB14_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:  .LBB14_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp uno float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -222,6 +365,10 @@ define float @select_fcmp_true(float %a, float %b) nounwind {
 ; CHECK-LABEL: select_fcmp_true:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_true:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp true float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -237,6 +384,16 @@ define i32 @i32_select_fcmp_oeq(float %a, float %b, i32 %c, i32 %d) nounwind {
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:  .LBB16_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: i32_select_fcmp_oeq:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a1, a0, a1
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    bnez a1, .LBB16_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a0, a3
+; CHECKZFINX-NEXT:  .LBB16_2:
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
   ret i32 %2
@@ -249,6 +406,13 @@ define i32 @select_fcmp_oeq_1_2(float %a, float %b) {
 ; CHECK-NEXT:    li a1, 2
 ; CHECK-NEXT:    sub a0, a1, a0
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_oeq_1_2:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    feq.s a0, a0, a1
+; CHECKZFINX-NEXT:    li a1, 2
+; CHECKZFINX-NEXT:    sub a0, a1, a0
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp fast oeq float %a, %b
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
@@ -260,6 +424,12 @@ define signext i32 @select_fcmp_uge_negone_zero(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fle.s a0, fa0, fa1
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_uge_negone_zero:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKZFINX-NEXT:    addi a0, a0, -1
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ugt float %a, %b
   %2 = select i1 %1, i32 -1, i32 0
   ret i32 %2
@@ -271,6 +441,12 @@ define signext i32 @select_fcmp_uge_1_2(float %a, float %b) nounwind {
 ; CHECK-NEXT:    fle.s a0, fa0, fa1
 ; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_fcmp_uge_1_2:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    fle.s a0, a0, a1
+; CHECKZFINX-NEXT:    addi a0, a0, 1
+; CHECKZFINX-NEXT:    ret
   %1 = fcmp ugt float %a, %b
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
diff --git a/llvm/test/CodeGen/RISCV/float-select-icmp.ll b/llvm/test/CodeGen/RISCV/float-select-icmp.ll
index 52fa76b1b28c8..f46f25f777ca9 100644
--- a/llvm/test/CodeGen/RISCV/float-select-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-select-icmp.ll
@@ -3,6 +3,10 @@
 ; RUN:   -target-abi=ilp32f | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   -target-abi=lp64f | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=ilp32 | FileCheck --check-prefix=CHECKZFINX %s
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -verify-machineinstrs < %s \
+; RUN:   -target-abi=lp64 | FileCheck --check-prefix=CHECKZFINX %s
 
 define float @select_icmp_eq(i32 signext %a, i32 signext %b, float %c, float %d) {
 ; CHECK-LABEL: select_icmp_eq:
@@ -12,6 +16,15 @@ define float @select_icmp_eq(i32 signext %a, i32 signext %b, float %c, float %d)
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_eq:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    beq a0, a1, .LBB0_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB0_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp eq i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -25,6 +38,15 @@ define float @select_icmp_ne(i32 signext %a, i32 signext %b, float %c, float %d)
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_ne:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bne a0, a1, .LBB1_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB1_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp ne i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -38,6 +60,15 @@ define float @select_icmp_ugt(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_ugt:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bltu a1, a0, .LBB2_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB2_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp ugt i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -51,6 +82,15 @@ define float @select_icmp_uge(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_uge:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bgeu a0, a1, .LBB3_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB3_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp uge i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -64,6 +104,15 @@ define float @select_icmp_ult(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB4_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_ult:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bltu a0, a1, .LBB4_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB4_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp ult i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -77,6 +126,15 @@ define float @select_icmp_ule(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB5_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_ule:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bgeu a1, a0, .LBB5_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB5_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp ule i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -90,6 +148,15 @@ define float @select_icmp_sgt(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB6_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_sgt:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    blt a1, a0, .LBB6_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB6_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp sgt i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -103,6 +170,15 @@ define float @select_icmp_sge(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_sge:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bge a0, a1, .LBB7_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB7_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp sge i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -116,6 +192,15 @@ define float @select_icmp_slt(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB8_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_slt:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    blt a0, a1, .LBB8_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB8_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp slt i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
@@ -129,6 +214,15 @@ define float @select_icmp_sle(i32 signext %a, i32 signext %b, float %c, float %d
 ; CHECK-NEXT:    fmv.s fa0, fa1
 ; CHECK-NEXT:  .LBB9_2:
 ; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: select_icmp_sle:
+; CHECKZFINX:       # %bb.0:
+; CHECKZFINX-NEXT:    bge a1, a0, .LBB9_2
+; CHECKZFINX-NEXT:  # %bb.1:
+; CHECKZFINX-NEXT:    mv a2, a3
+; CHECKZFINX-NEXT:  .LBB9_2:
+; CHECKZFINX-NEXT:    mv a0, a2
+; CHECKZFINX-NEXT:    ret
   %1 = icmp sle i32 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
index 5b2ffe503a66a..62beaadef003c 100644
--- a/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -84,65 +84,61 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV32IZFH-LABEL: fcvt_si_h_sat:
 ; RV32IZFH:       # %bb.0: # %start
 ; RV32IZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV32IZFH-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV32IZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a0)
-; RV32IZFH-NEXT:    lui a0, 815104
-; RV32IZFH-NEXT:    fmv.w.x fa3, a0
-; RV32IZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV32IZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV32IZFH-NEXT:    fcvt.w.s a0, fa4, rtz
-; RV32IZFH-NEXT:    feq.s a1, fa5, fa5
-; RV32IZFH-NEXT:    seqz a1, a1
-; RV32IZFH-NEXT:    addi a1, a1, -1
-; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV32IZFH-NEXT:    neg a0, a0
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV32IZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
+; RV32IZFH-NEXT:    lui a1, 815104
+; RV32IZFH-NEXT:    fmv.w.x fa3, a1
+; RV32IZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV32IZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV32IZFH-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IZFH-NEXT:    and a0, a0, a1
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: fcvt_si_h_sat:
 ; RV64IZFH:       # %bb.0: # %start
 ; RV64IZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV64IZFH-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV64IZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a0)
-; RV64IZFH-NEXT:    lui a0, 815104
-; RV64IZFH-NEXT:    fmv.w.x fa3, a0
-; RV64IZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV64IZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV64IZFH-NEXT:    fcvt.l.s a0, fa4, rtz
-; RV64IZFH-NEXT:    feq.s a1, fa5, fa5
-; RV64IZFH-NEXT:    seqz a1, a1
-; RV64IZFH-NEXT:    addi a1, a1, -1
-; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IZFH-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV64IZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
+; RV64IZFH-NEXT:    lui a1, 815104
+; RV64IZFH-NEXT:    fmv.w.x fa3, a1
+; RV64IZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV64IZFH-NEXT:    neg a0, a0
+; RV64IZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV64IZFH-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IZFH-NEXT:    and a0, a0, a1
 ; RV64IZFH-NEXT:    ret
 ;
 ; RV32IDZFH-LABEL: fcvt_si_h_sat:
 ; RV32IDZFH:       # %bb.0: # %start
 ; RV32IDZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV32IDZFH-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV32IDZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a0)
-; RV32IDZFH-NEXT:    lui a0, 815104
-; RV32IDZFH-NEXT:    fmv.w.x fa3, a0
-; RV32IDZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV32IDZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV32IDZFH-NEXT:    fcvt.w.s a0, fa4, rtz
-; RV32IDZFH-NEXT:    feq.s a1, fa5, fa5
-; RV32IDZFH-NEXT:    seqz a1, a1
-; RV32IDZFH-NEXT:    addi a1, a1, -1
-; RV32IDZFH-NEXT:    and a0, a1, a0
+; RV32IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV32IDZFH-NEXT:    neg a0, a0
+; RV32IDZFH-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV32IDZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
+; RV32IDZFH-NEXT:    lui a1, 815104
+; RV32IDZFH-NEXT:    fmv.w.x fa3, a1
+; RV32IDZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV32IDZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV32IDZFH-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IDZFH-NEXT:    and a0, a0, a1
 ; RV32IDZFH-NEXT:    ret
 ;
 ; RV64IDZFH-LABEL: fcvt_si_h_sat:
 ; RV64IDZFH:       # %bb.0: # %start
 ; RV64IDZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV64IDZFH-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV64IDZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a0)
-; RV64IDZFH-NEXT:    lui a0, 815104
-; RV64IDZFH-NEXT:    fmv.w.x fa3, a0
-; RV64IDZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV64IDZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV64IDZFH-NEXT:    fcvt.l.s a0, fa4, rtz
-; RV64IDZFH-NEXT:    feq.s a1, fa5, fa5
-; RV64IDZFH-NEXT:    seqz a1, a1
-; RV64IDZFH-NEXT:    addi a1, a1, -1
-; RV64IDZFH-NEXT:    and a0, a1, a0
+; RV64IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IDZFH-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV64IDZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
+; RV64IDZFH-NEXT:    lui a1, 815104
+; RV64IDZFH-NEXT:    fmv.w.x fa3, a1
+; RV64IDZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV64IDZFH-NEXT:    neg a0, a0
+; RV64IDZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV64IDZFH-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IDZFH-NEXT:    and a0, a0, a1
 ; RV64IDZFH-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_si_h_sat:
@@ -234,33 +230,31 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; CHECK32-IZFHMIN-LABEL: fcvt_si_h_sat:
 ; CHECK32-IZFHMIN:       # %bb.0: # %start
 ; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK32-IZFHMIN-NEXT:    lui a0, %hi(.LCPI1_0)
-; CHECK32-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI1_0)(a0)
-; CHECK32-IZFHMIN-NEXT:    lui a0, 815104
-; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa3, a0
-; CHECK32-IZFHMIN-NEXT:    fmax.s fa3, fa5, fa3
-; CHECK32-IZFHMIN-NEXT:    fmin.s fa4, fa3, fa4
-; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a0, fa4, rtz
-; CHECK32-IZFHMIN-NEXT:    feq.s a1, fa5, fa5
-; CHECK32-IZFHMIN-NEXT:    seqz a1, a1
-; CHECK32-IZFHMIN-NEXT:    addi a1, a1, -1
-; CHECK32-IZFHMIN-NEXT:    and a0, a1, a0
+; CHECK32-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK32-IZFHMIN-NEXT:    neg a0, a0
+; CHECK32-IZFHMIN-NEXT:    lui a1, %hi(.LCPI1_0)
+; CHECK32-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
+; CHECK32-IZFHMIN-NEXT:    lui a1, 815104
+; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa3, a1
+; CHECK32-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa3
+; CHECK32-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
+; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a1, fa5, rtz
+; CHECK32-IZFHMIN-NEXT:    and a0, a0, a1
 ; CHECK32-IZFHMIN-NEXT:    ret
 ;
 ; CHECK64-IZFHMIN-LABEL: fcvt_si_h_sat:
 ; CHECK64-IZFHMIN:       # %bb.0: # %start
 ; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK64-IZFHMIN-NEXT:    lui a0, %hi(.LCPI1_0)
-; CHECK64-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI1_0)(a0)
-; CHECK64-IZFHMIN-NEXT:    lui a0, 815104
-; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a0
-; CHECK64-IZFHMIN-NEXT:    fmax.s fa3, fa5, fa3
-; CHECK64-IZFHMIN-NEXT:    fmin.s fa4, fa3, fa4
-; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a0, fa4, rtz
-; CHECK64-IZFHMIN-NEXT:    feq.s a1, fa5, fa5
-; CHECK64-IZFHMIN-NEXT:    seqz a1, a1
-; CHECK64-IZFHMIN-NEXT:    addi a1, a1, -1
-; CHECK64-IZFHMIN-NEXT:    and a0, a1, a0
+; CHECK64-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK64-IZFHMIN-NEXT:    lui a1, %hi(.LCPI1_0)
+; CHECK64-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
+; CHECK64-IZFHMIN-NEXT:    lui a1, 815104
+; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a1
+; CHECK64-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa3
+; CHECK64-IZFHMIN-NEXT:    neg a0, a0
+; CHECK64-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
+; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a1, fa5, rtz
+; CHECK64-IZFHMIN-NEXT:    and a0, a0, a1
 ; CHECK64-IZFHMIN-NEXT:    ret
 start:
   %0 = tail call i16 @llvm.fptosi.sat.i16.f16(half %a)
@@ -1024,27 +1018,27 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32IZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IZFH-NEXT:    call __fixsfdi@plt
-; RV32IZFH-NEXT:    lui a3, 524288
-; RV32IZFH-NEXT:    bnez s0, .LBB10_2
+; RV32IZFH-NEXT:    lui a4, 524288
+; RV32IZFH-NEXT:    lui a2, 524288
+; RV32IZFH-NEXT:    beqz s0, .LBB10_2
 ; RV32IZFH-NEXT:  # %bb.1: # %start
-; RV32IZFH-NEXT:    lui a1, 524288
+; RV32IZFH-NEXT:    mv a2, a1
 ; RV32IZFH-NEXT:  .LBB10_2: # %start
-; RV32IZFH-NEXT:    lui a2, %hi(.LCPI10_0)
-; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI10_0)(a2)
-; RV32IZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFH-NEXT:    beqz a2, .LBB10_4
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI10_0)
+; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI10_0)(a1)
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    beqz a3, .LBB10_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    addi a1, a3, -1
+; RV32IZFH-NEXT:    addi a2, a4, -1
 ; RV32IZFH-NEXT:  .LBB10_4: # %start
-; RV32IZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFH-NEXT:    seqz a3, a3
-; RV32IZFH-NEXT:    addi a3, a3, -1
-; RV32IZFH-NEXT:    and a1, a3, a1
-; RV32IZFH-NEXT:    neg a2, a2
-; RV32IZFH-NEXT:    neg a4, s0
-; RV32IZFH-NEXT:    and a0, a4, a0
-; RV32IZFH-NEXT:    or a0, a2, a0
+; RV32IZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFH-NEXT:    neg a4, a1
+; RV32IZFH-NEXT:    and a1, a4, a2
+; RV32IZFH-NEXT:    neg a2, a3
+; RV32IZFH-NEXT:    neg a3, s0
 ; RV32IZFH-NEXT:    and a0, a3, a0
+; RV32IZFH-NEXT:    or a0, a2, a0
+; RV32IZFH-NEXT:    and a0, a4, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -1072,27 +1066,27 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32IDZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IDZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IDZFH-NEXT:    call __fixsfdi@plt
-; RV32IDZFH-NEXT:    lui a3, 524288
-; RV32IDZFH-NEXT:    bnez s0, .LBB10_2
+; RV32IDZFH-NEXT:    lui a4, 524288
+; RV32IDZFH-NEXT:    lui a2, 524288
+; RV32IDZFH-NEXT:    beqz s0, .LBB10_2
 ; RV32IDZFH-NEXT:  # %bb.1: # %start
-; RV32IDZFH-NEXT:    lui a1, 524288
+; RV32IDZFH-NEXT:    mv a2, a1
 ; RV32IDZFH-NEXT:  .LBB10_2: # %start
-; RV32IDZFH-NEXT:    lui a2, %hi(.LCPI10_0)
-; RV32IDZFH-NEXT:    flw fa5, %lo(.LCPI10_0)(a2)
-; RV32IDZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IDZFH-NEXT:    beqz a2, .LBB10_4
+; RV32IDZFH-NEXT:    lui a1, %hi(.LCPI10_0)
+; RV32IDZFH-NEXT:    flw fa5, %lo(.LCPI10_0)(a1)
+; RV32IDZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IDZFH-NEXT:    beqz a3, .LBB10_4
 ; RV32IDZFH-NEXT:  # %bb.3:
-; RV32IDZFH-NEXT:    addi a1, a3, -1
+; RV32IDZFH-NEXT:    addi a2, a4, -1
 ; RV32IDZFH-NEXT:  .LBB10_4: # %start
-; RV32IDZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IDZFH-NEXT:    seqz a3, a3
-; RV32IDZFH-NEXT:    addi a3, a3, -1
-; RV32IDZFH-NEXT:    and a1, a3, a1
-; RV32IDZFH-NEXT:    neg a2, a2
-; RV32IDZFH-NEXT:    neg a4, s0
-; RV32IDZFH-NEXT:    and a0, a4, a0
-; RV32IDZFH-NEXT:    or a0, a2, a0
+; RV32IDZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IDZFH-NEXT:    neg a4, a1
+; RV32IDZFH-NEXT:    and a1, a4, a2
+; RV32IDZFH-NEXT:    neg a2, a3
+; RV32IDZFH-NEXT:    neg a3, s0
 ; RV32IDZFH-NEXT:    and a0, a3, a0
+; RV32IDZFH-NEXT:    or a0, a2, a0
+; RV32IDZFH-NEXT:    and a0, a4, a0
 ; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IDZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IDZFH-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -1117,62 +1111,52 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    slli a0, a0, 16
 ; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    call __extendhfsf2@plt
-; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    lui a1, 913408
 ; RV32I-NEXT:    call __gesf2@plt
-; RV32I-NEXT:    mv s3, a0
-; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __fixsfdi@plt
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    lui s4, 524288
-; RV32I-NEXT:    bgez s3, .LBB10_2
+; RV32I-NEXT:    mv s2, a0
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    lui s5, 524288
+; RV32I-NEXT:    bgez s0, .LBB10_2
 ; RV32I-NEXT:  # %bb.1: # %start
-; RV32I-NEXT:    lui s2, 524288
+; RV32I-NEXT:    lui s3, 524288
 ; RV32I-NEXT:  .LBB10_2: # %start
-; RV32I-NEXT:    lui s3, 389120
-; RV32I-NEXT:    addi s3, s3, -1
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    lui a1, 389120
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    call __gtsf2@plt
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    blez a0, .LBB10_4
 ; RV32I-NEXT:  # %bb.3: # %start
-; RV32I-NEXT:    addi s2, s4, -1
+; RV32I-NEXT:    addi s3, s5, -1
 ; RV32I-NEXT:  .LBB10_4: # %start
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __unordsf2@plt
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and s2, a0, s2
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    call __gtsf2@plt
-; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg s3, a0
-; RV32I-NEXT:    lui a1, 913408
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __gesf2@plt
-; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    or s1, s3, a0
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s0
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __unordsf2@plt
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    and a1, a0, s3
+; RV32I-NEXT:    sgtz a2, s4
+; RV32I-NEXT:    neg a2, a2
+; RV32I-NEXT:    slti a3, s0, 0
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    and a3, a3, s2
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    and a0, a0, a2
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -1233,27 +1217,27 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32IFZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IFZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IFZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IFZFHMIN-NEXT:    lui a3, 524288
-; RV32IFZFHMIN-NEXT:    bnez s0, .LBB10_2
+; RV32IFZFHMIN-NEXT:    lui a4, 524288
+; RV32IFZFHMIN-NEXT:    lui a2, 524288
+; RV32IFZFHMIN-NEXT:    beqz s0, .LBB10_2
 ; RV32IFZFHMIN-NEXT:  # %bb.1: # %start
-; RV32IFZFHMIN-NEXT:    lui a1, 524288
+; RV32IFZFHMIN-NEXT:    mv a2, a1
 ; RV32IFZFHMIN-NEXT:  .LBB10_2: # %start
-; RV32IFZFHMIN-NEXT:    lui a2, %hi(.LCPI10_0)
-; RV32IFZFHMIN-NEXT:    flw fa5, %lo(.LCPI10_0)(a2)
-; RV32IFZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IFZFHMIN-NEXT:    beqz a2, .LBB10_4
+; RV32IFZFHMIN-NEXT:    lui a1, %hi(.LCPI10_0)
+; RV32IFZFHMIN-NEXT:    flw fa5, %lo(.LCPI10_0)(a1)
+; RV32IFZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IFZFHMIN-NEXT:    beqz a3, .LBB10_4
 ; RV32IFZFHMIN-NEXT:  # %bb.3:
-; RV32IFZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IFZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IFZFHMIN-NEXT:  .LBB10_4: # %start
-; RV32IFZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IFZFHMIN-NEXT:    seqz a3, a3
-; RV32IFZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IFZFHMIN-NEXT:    and a1, a3, a1
-; RV32IFZFHMIN-NEXT:    neg a2, a2
-; RV32IFZFHMIN-NEXT:    neg a4, s0
-; RV32IFZFHMIN-NEXT:    and a0, a4, a0
-; RV32IFZFHMIN-NEXT:    or a0, a2, a0
+; RV32IFZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IFZFHMIN-NEXT:    neg a4, a1
+; RV32IFZFHMIN-NEXT:    and a1, a4, a2
+; RV32IFZFHMIN-NEXT:    neg a2, a3
+; RV32IFZFHMIN-NEXT:    neg a3, s0
 ; RV32IFZFHMIN-NEXT:    and a0, a3, a0
+; RV32IFZFHMIN-NEXT:    or a0, a2, a0
+; RV32IFZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IFZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFZFHMIN-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -1282,27 +1266,27 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32IDZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IDZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IDZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IDZFHMIN-NEXT:    lui a3, 524288
-; RV32IDZFHMIN-NEXT:    bnez s0, .LBB10_2
+; RV32IDZFHMIN-NEXT:    lui a4, 524288
+; RV32IDZFHMIN-NEXT:    lui a2, 524288
+; RV32IDZFHMIN-NEXT:    beqz s0, .LBB10_2
 ; RV32IDZFHMIN-NEXT:  # %bb.1: # %start
-; RV32IDZFHMIN-NEXT:    lui a1, 524288
+; RV32IDZFHMIN-NEXT:    mv a2, a1
 ; RV32IDZFHMIN-NEXT:  .LBB10_2: # %start
-; RV32IDZFHMIN-NEXT:    lui a2, %hi(.LCPI10_0)
-; RV32IDZFHMIN-NEXT:    flw fa5, %lo(.LCPI10_0)(a2)
-; RV32IDZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IDZFHMIN-NEXT:    beqz a2, .LBB10_4
+; RV32IDZFHMIN-NEXT:    lui a1, %hi(.LCPI10_0)
+; RV32IDZFHMIN-NEXT:    flw fa5, %lo(.LCPI10_0)(a1)
+; RV32IDZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IDZFHMIN-NEXT:    beqz a3, .LBB10_4
 ; RV32IDZFHMIN-NEXT:  # %bb.3:
-; RV32IDZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IDZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IDZFHMIN-NEXT:  .LBB10_4: # %start
-; RV32IDZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IDZFHMIN-NEXT:    seqz a3, a3
-; RV32IDZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IDZFHMIN-NEXT:    and a1, a3, a1
-; RV32IDZFHMIN-NEXT:    neg a2, a2
-; RV32IDZFHMIN-NEXT:    neg a4, s0
-; RV32IDZFHMIN-NEXT:    and a0, a4, a0
-; RV32IDZFHMIN-NEXT:    or a0, a2, a0
+; RV32IDZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IDZFHMIN-NEXT:    neg a4, a1
+; RV32IDZFHMIN-NEXT:    and a1, a4, a2
+; RV32IDZFHMIN-NEXT:    neg a2, a3
+; RV32IDZFHMIN-NEXT:    neg a3, s0
 ; RV32IDZFHMIN-NEXT:    and a0, a3, a0
+; RV32IDZFHMIN-NEXT:    or a0, a2, a0
+; RV32IDZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IDZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IDZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IDZFHMIN-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
@@ -1456,53 +1440,36 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ;
 ; RV32I-LABEL: fcvt_lu_h_sat:
 ; RV32I:       # %bb.0: # %start
-; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    slli a0, a0, 16
 ; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    call __extendhfsf2@plt
 ; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lui s1, 391168
-; RV32I-NEXT:    addi s1, s1, -1
-; RV32I-NEXT:    mv a1, s1
+; RV32I-NEXT:    lui a1, 391168
+; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    call __gtsf2@plt
 ; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg s3, a0
+; RV32I-NEXT:    neg s1, a0
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2@plt
 ; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi s4, a0, -1
+; RV32I-NEXT:    addi s2, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfdi@plt
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    and a0, s4, a0
-; RV32I-NEXT:    or s3, s3, a0
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __gtsf2@plt
-; RV32I-NEXT:    sgtz a0, a0
-; RV32I-NEXT:    neg s1, a0
-; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    li a1, 0
-; RV32I-NEXT:    call __gesf2@plt
-; RV32I-NEXT:    slti a0, a0, 0
-; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a1, a0, s2
+; RV32I-NEXT:    and a0, s2, a0
+; RV32I-NEXT:    or a0, s1, a0
+; RV32I-NEXT:    and a1, s2, a1
 ; RV32I-NEXT:    or a1, s1, a1
-; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: fcvt_lu_h_sat:
@@ -2772,65 +2739,61 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV32IZFH-LABEL: fcvt_w_s_sat_i16:
 ; RV32IZFH:       # %bb.0: # %start
 ; RV32IZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV32IZFH-NEXT:    lui a0, %hi(.LCPI32_0)
-; RV32IZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a0)
-; RV32IZFH-NEXT:    lui a0, 815104
-; RV32IZFH-NEXT:    fmv.w.x fa3, a0
-; RV32IZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV32IZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV32IZFH-NEXT:    fcvt.w.s a0, fa4, rtz
-; RV32IZFH-NEXT:    feq.s a1, fa5, fa5
-; RV32IZFH-NEXT:    seqz a1, a1
-; RV32IZFH-NEXT:    addi a1, a1, -1
-; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV32IZFH-NEXT:    neg a0, a0
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI32_0)
+; RV32IZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
+; RV32IZFH-NEXT:    lui a1, 815104
+; RV32IZFH-NEXT:    fmv.w.x fa3, a1
+; RV32IZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV32IZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV32IZFH-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IZFH-NEXT:    and a0, a0, a1
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: fcvt_w_s_sat_i16:
 ; RV64IZFH:       # %bb.0: # %start
 ; RV64IZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV64IZFH-NEXT:    lui a0, %hi(.LCPI32_0)
-; RV64IZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a0)
-; RV64IZFH-NEXT:    lui a0, 815104
-; RV64IZFH-NEXT:    fmv.w.x fa3, a0
-; RV64IZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV64IZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV64IZFH-NEXT:    fcvt.l.s a0, fa4, rtz
-; RV64IZFH-NEXT:    feq.s a1, fa5, fa5
-; RV64IZFH-NEXT:    seqz a1, a1
-; RV64IZFH-NEXT:    addi a1, a1, -1
-; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IZFH-NEXT:    lui a1, %hi(.LCPI32_0)
+; RV64IZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
+; RV64IZFH-NEXT:    lui a1, 815104
+; RV64IZFH-NEXT:    fmv.w.x fa3, a1
+; RV64IZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV64IZFH-NEXT:    neg a0, a0
+; RV64IZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV64IZFH-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IZFH-NEXT:    and a0, a0, a1
 ; RV64IZFH-NEXT:    ret
 ;
 ; RV32IDZFH-LABEL: fcvt_w_s_sat_i16:
 ; RV32IDZFH:       # %bb.0: # %start
 ; RV32IDZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV32IDZFH-NEXT:    lui a0, %hi(.LCPI32_0)
-; RV32IDZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a0)
-; RV32IDZFH-NEXT:    lui a0, 815104
-; RV32IDZFH-NEXT:    fmv.w.x fa3, a0
-; RV32IDZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV32IDZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV32IDZFH-NEXT:    fcvt.w.s a0, fa4, rtz
-; RV32IDZFH-NEXT:    feq.s a1, fa5, fa5
-; RV32IDZFH-NEXT:    seqz a1, a1
-; RV32IDZFH-NEXT:    addi a1, a1, -1
-; RV32IDZFH-NEXT:    and a0, a1, a0
+; RV32IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV32IDZFH-NEXT:    neg a0, a0
+; RV32IDZFH-NEXT:    lui a1, %hi(.LCPI32_0)
+; RV32IDZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
+; RV32IDZFH-NEXT:    lui a1, 815104
+; RV32IDZFH-NEXT:    fmv.w.x fa3, a1
+; RV32IDZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV32IDZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV32IDZFH-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IDZFH-NEXT:    and a0, a0, a1
 ; RV32IDZFH-NEXT:    ret
 ;
 ; RV64IDZFH-LABEL: fcvt_w_s_sat_i16:
 ; RV64IDZFH:       # %bb.0: # %start
 ; RV64IDZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV64IDZFH-NEXT:    lui a0, %hi(.LCPI32_0)
-; RV64IDZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a0)
-; RV64IDZFH-NEXT:    lui a0, 815104
-; RV64IDZFH-NEXT:    fmv.w.x fa3, a0
-; RV64IDZFH-NEXT:    fmax.s fa3, fa5, fa3
-; RV64IDZFH-NEXT:    fmin.s fa4, fa3, fa4
-; RV64IDZFH-NEXT:    fcvt.l.s a0, fa4, rtz
-; RV64IDZFH-NEXT:    feq.s a1, fa5, fa5
-; RV64IDZFH-NEXT:    seqz a1, a1
-; RV64IDZFH-NEXT:    addi a1, a1, -1
-; RV64IDZFH-NEXT:    and a0, a1, a0
+; RV64IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IDZFH-NEXT:    lui a1, %hi(.LCPI32_0)
+; RV64IDZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
+; RV64IDZFH-NEXT:    lui a1, 815104
+; RV64IDZFH-NEXT:    fmv.w.x fa3, a1
+; RV64IDZFH-NEXT:    fmax.s fa5, fa5, fa3
+; RV64IDZFH-NEXT:    neg a0, a0
+; RV64IDZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV64IDZFH-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IDZFH-NEXT:    and a0, a0, a1
 ; RV64IDZFH-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_w_s_sat_i16:
@@ -2926,33 +2889,31 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; CHECK32-IZFHMIN-LABEL: fcvt_w_s_sat_i16:
 ; CHECK32-IZFHMIN:       # %bb.0: # %start
 ; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK32-IZFHMIN-NEXT:    lui a0, %hi(.LCPI32_0)
-; CHECK32-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI32_0)(a0)
-; CHECK32-IZFHMIN-NEXT:    lui a0, 815104
-; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa3, a0
-; CHECK32-IZFHMIN-NEXT:    fmax.s fa3, fa5, fa3
-; CHECK32-IZFHMIN-NEXT:    fmin.s fa4, fa3, fa4
-; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a0, fa4, rtz
-; CHECK32-IZFHMIN-NEXT:    feq.s a1, fa5, fa5
-; CHECK32-IZFHMIN-NEXT:    seqz a1, a1
-; CHECK32-IZFHMIN-NEXT:    addi a1, a1, -1
-; CHECK32-IZFHMIN-NEXT:    and a0, a1, a0
+; CHECK32-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK32-IZFHMIN-NEXT:    neg a0, a0
+; CHECK32-IZFHMIN-NEXT:    lui a1, %hi(.LCPI32_0)
+; CHECK32-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
+; CHECK32-IZFHMIN-NEXT:    lui a1, 815104
+; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa3, a1
+; CHECK32-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa3
+; CHECK32-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
+; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a1, fa5, rtz
+; CHECK32-IZFHMIN-NEXT:    and a0, a0, a1
 ; CHECK32-IZFHMIN-NEXT:    ret
 ;
 ; CHECK64-IZFHMIN-LABEL: fcvt_w_s_sat_i16:
 ; CHECK64-IZFHMIN:       # %bb.0: # %start
 ; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK64-IZFHMIN-NEXT:    lui a0, %hi(.LCPI32_0)
-; CHECK64-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI32_0)(a0)
-; CHECK64-IZFHMIN-NEXT:    lui a0, 815104
-; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a0
-; CHECK64-IZFHMIN-NEXT:    fmax.s fa3, fa5, fa3
-; CHECK64-IZFHMIN-NEXT:    fmin.s fa4, fa3, fa4
-; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a0, fa4, rtz
-; CHECK64-IZFHMIN-NEXT:    feq.s a1, fa5, fa5
-; CHECK64-IZFHMIN-NEXT:    seqz a1, a1
-; CHECK64-IZFHMIN-NEXT:    addi a1, a1, -1
-; CHECK64-IZFHMIN-NEXT:    and a0, a1, a0
+; CHECK64-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK64-IZFHMIN-NEXT:    lui a1, %hi(.LCPI32_0)
+; CHECK64-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
+; CHECK64-IZFHMIN-NEXT:    lui a1, 815104
+; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a1
+; CHECK64-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa3
+; CHECK64-IZFHMIN-NEXT:    neg a0, a0
+; CHECK64-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
+; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a1, fa5, rtz
+; CHECK64-IZFHMIN-NEXT:    and a0, a0, a1
 ; CHECK64-IZFHMIN-NEXT:    ret
 start:
   %0 = tail call i16 @llvm.fptosi.sat.i16.f16(half %a)
@@ -3236,65 +3197,61 @@ define signext i8 @fcvt_w_s_sat_i8(half %a) nounwind {
 ; RV32IZFH-LABEL: fcvt_w_s_sat_i8:
 ; RV32IZFH:       # %bb.0: # %start
 ; RV32IZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV32IZFH-NEXT:    lui a0, 798720
-; RV32IZFH-NEXT:    fmv.w.x fa4, a0
-; RV32IZFH-NEXT:    fmax.s fa4, fa5, fa4
-; RV32IZFH-NEXT:    lui a0, 274400
-; RV32IZFH-NEXT:    fmv.w.x fa3, a0
-; RV32IZFH-NEXT:    fmin.s fa4, fa4, fa3
-; RV32IZFH-NEXT:    fcvt.w.s a0, fa4, rtz
-; RV32IZFH-NEXT:    feq.s a1, fa5, fa5
-; RV32IZFH-NEXT:    seqz a1, a1
-; RV32IZFH-NEXT:    addi a1, a1, -1
-; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV32IZFH-NEXT:    neg a0, a0
+; RV32IZFH-NEXT:    lui a1, 798720
+; RV32IZFH-NEXT:    fmv.w.x fa4, a1
+; RV32IZFH-NEXT:    fmax.s fa5, fa5, fa4
+; RV32IZFH-NEXT:    lui a1, 274400
+; RV32IZFH-NEXT:    fmv.w.x fa4, a1
+; RV32IZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV32IZFH-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IZFH-NEXT:    and a0, a0, a1
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: fcvt_w_s_sat_i8:
 ; RV64IZFH:       # %bb.0: # %start
 ; RV64IZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV64IZFH-NEXT:    lui a0, 798720
-; RV64IZFH-NEXT:    fmv.w.x fa4, a0
-; RV64IZFH-NEXT:    fmax.s fa4, fa5, fa4
-; RV64IZFH-NEXT:    lui a0, 274400
-; RV64IZFH-NEXT:    fmv.w.x fa3, a0
-; RV64IZFH-NEXT:    fmin.s fa4, fa4, fa3
-; RV64IZFH-NEXT:    fcvt.l.s a0, fa4, rtz
-; RV64IZFH-NEXT:    feq.s a1, fa5, fa5
-; RV64IZFH-NEXT:    seqz a1, a1
-; RV64IZFH-NEXT:    addi a1, a1, -1
-; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IZFH-NEXT:    neg a0, a0
+; RV64IZFH-NEXT:    lui a1, 798720
+; RV64IZFH-NEXT:    fmv.w.x fa4, a1
+; RV64IZFH-NEXT:    fmax.s fa5, fa5, fa4
+; RV64IZFH-NEXT:    lui a1, 274400
+; RV64IZFH-NEXT:    fmv.w.x fa4, a1
+; RV64IZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV64IZFH-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IZFH-NEXT:    and a0, a0, a1
 ; RV64IZFH-NEXT:    ret
 ;
 ; RV32IDZFH-LABEL: fcvt_w_s_sat_i8:
 ; RV32IDZFH:       # %bb.0: # %start
 ; RV32IDZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV32IDZFH-NEXT:    lui a0, 798720
-; RV32IDZFH-NEXT:    fmv.w.x fa4, a0
-; RV32IDZFH-NEXT:    fmax.s fa4, fa5, fa4
-; RV32IDZFH-NEXT:    lui a0, 274400
-; RV32IDZFH-NEXT:    fmv.w.x fa3, a0
-; RV32IDZFH-NEXT:    fmin.s fa4, fa4, fa3
-; RV32IDZFH-NEXT:    fcvt.w.s a0, fa4, rtz
-; RV32IDZFH-NEXT:    feq.s a1, fa5, fa5
-; RV32IDZFH-NEXT:    seqz a1, a1
-; RV32IDZFH-NEXT:    addi a1, a1, -1
-; RV32IDZFH-NEXT:    and a0, a1, a0
+; RV32IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV32IDZFH-NEXT:    neg a0, a0
+; RV32IDZFH-NEXT:    lui a1, 798720
+; RV32IDZFH-NEXT:    fmv.w.x fa4, a1
+; RV32IDZFH-NEXT:    fmax.s fa5, fa5, fa4
+; RV32IDZFH-NEXT:    lui a1, 274400
+; RV32IDZFH-NEXT:    fmv.w.x fa4, a1
+; RV32IDZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV32IDZFH-NEXT:    fcvt.w.s a1, fa5, rtz
+; RV32IDZFH-NEXT:    and a0, a0, a1
 ; RV32IDZFH-NEXT:    ret
 ;
 ; RV64IDZFH-LABEL: fcvt_w_s_sat_i8:
 ; RV64IDZFH:       # %bb.0: # %start
 ; RV64IDZFH-NEXT:    fcvt.s.h fa5, fa0
-; RV64IDZFH-NEXT:    lui a0, 798720
-; RV64IDZFH-NEXT:    fmv.w.x fa4, a0
-; RV64IDZFH-NEXT:    fmax.s fa4, fa5, fa4
-; RV64IDZFH-NEXT:    lui a0, 274400
-; RV64IDZFH-NEXT:    fmv.w.x fa3, a0
-; RV64IDZFH-NEXT:    fmin.s fa4, fa4, fa3
-; RV64IDZFH-NEXT:    fcvt.l.s a0, fa4, rtz
-; RV64IDZFH-NEXT:    feq.s a1, fa5, fa5
-; RV64IDZFH-NEXT:    seqz a1, a1
-; RV64IDZFH-NEXT:    addi a1, a1, -1
-; RV64IDZFH-NEXT:    and a0, a1, a0
+; RV64IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IDZFH-NEXT:    neg a0, a0
+; RV64IDZFH-NEXT:    lui a1, 798720
+; RV64IDZFH-NEXT:    fmv.w.x fa4, a1
+; RV64IDZFH-NEXT:    fmax.s fa5, fa5, fa4
+; RV64IDZFH-NEXT:    lui a1, 274400
+; RV64IDZFH-NEXT:    fmv.w.x fa4, a1
+; RV64IDZFH-NEXT:    fmin.s fa5, fa5, fa4
+; RV64IDZFH-NEXT:    fcvt.l.s a1, fa5, rtz
+; RV64IDZFH-NEXT:    and a0, a0, a1
 ; RV64IDZFH-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_w_s_sat_i8:
@@ -3386,33 +3343,31 @@ define signext i8 @fcvt_w_s_sat_i8(half %a) nounwind {
 ; CHECK32-IZFHMIN-LABEL: fcvt_w_s_sat_i8:
 ; CHECK32-IZFHMIN:       # %bb.0: # %start
 ; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK32-IZFHMIN-NEXT:    lui a0, 798720
-; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa4, a0
-; CHECK32-IZFHMIN-NEXT:    fmax.s fa4, fa5, fa4
-; CHECK32-IZFHMIN-NEXT:    lui a0, 274400
-; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa3, a0
-; CHECK32-IZFHMIN-NEXT:    fmin.s fa4, fa4, fa3
-; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a0, fa4, rtz
-; CHECK32-IZFHMIN-NEXT:    feq.s a1, fa5, fa5
-; CHECK32-IZFHMIN-NEXT:    seqz a1, a1
-; CHECK32-IZFHMIN-NEXT:    addi a1, a1, -1
-; CHECK32-IZFHMIN-NEXT:    and a0, a1, a0
+; CHECK32-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK32-IZFHMIN-NEXT:    neg a0, a0
+; CHECK32-IZFHMIN-NEXT:    lui a1, 798720
+; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa4, a1
+; CHECK32-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa4
+; CHECK32-IZFHMIN-NEXT:    lui a1, 274400
+; CHECK32-IZFHMIN-NEXT:    fmv.w.x fa4, a1
+; CHECK32-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
+; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a1, fa5, rtz
+; CHECK32-IZFHMIN-NEXT:    and a0, a0, a1
 ; CHECK32-IZFHMIN-NEXT:    ret
 ;
 ; CHECK64-IZFHMIN-LABEL: fcvt_w_s_sat_i8:
 ; CHECK64-IZFHMIN:       # %bb.0: # %start
 ; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK64-IZFHMIN-NEXT:    lui a0, 798720
-; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa4, a0
-; CHECK64-IZFHMIN-NEXT:    fmax.s fa4, fa5, fa4
-; CHECK64-IZFHMIN-NEXT:    lui a0, 274400
-; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a0
-; CHECK64-IZFHMIN-NEXT:    fmin.s fa4, fa4, fa3
-; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a0, fa4, rtz
-; CHECK64-IZFHMIN-NEXT:    feq.s a1, fa5, fa5
-; CHECK64-IZFHMIN-NEXT:    seqz a1, a1
-; CHECK64-IZFHMIN-NEXT:    addi a1, a1, -1
-; CHECK64-IZFHMIN-NEXT:    and a0, a1, a0
+; CHECK64-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK64-IZFHMIN-NEXT:    neg a0, a0
+; CHECK64-IZFHMIN-NEXT:    lui a1, 798720
+; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa4, a1
+; CHECK64-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa4
+; CHECK64-IZFHMIN-NEXT:    lui a1, 274400
+; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa4, a1
+; CHECK64-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
+; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a1, fa5, rtz
+; CHECK64-IZFHMIN-NEXT:    and a0, a0, a1
 ; CHECK64-IZFHMIN-NEXT:    ret
 start:
   %0 = tail call i8 @llvm.fptosi.sat.i8.f16(half %a)
diff --git a/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
index 7183e8aa93144..654bfe094c502 100644
--- a/llvm/test/CodeGen/RISCV/half-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -1928,3 +1928,53 @@ define half @roundeven_f16(half %a) nounwind {
   %1 = call half @llvm.roundeven.f16(half %a)
   ret half %1
 }
+
+declare i1 @llvm.is.fpclass.f16(half, i32)
+define i1 @isnan_d_fpclass(half %x) {
+; CHECKIZFH-LABEL: isnan_d_fpclass:
+; CHECKIZFH:       # %bb.0:
+; CHECKIZFH-NEXT:    fclass.h a0, fa0
+; CHECKIZFH-NEXT:    andi a0, a0, 768
+; CHECKIZFH-NEXT:    snez a0, a0
+; CHECKIZFH-NEXT:    ret
+;
+; RV32I-LABEL: isnan_d_fpclass:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 17
+; RV32I-NEXT:    srli a0, a0, 17
+; RV32I-NEXT:    li a1, 31
+; RV32I-NEXT:    slli a1, a1, 10
+; RV32I-NEXT:    slt a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: isnan_d_fpclass:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 49
+; RV64I-NEXT:    srli a0, a0, 49
+; RV64I-NEXT:    li a1, 31
+; RV64I-NEXT:    slli a1, a1, 10
+; RV64I-NEXT:    slt a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV32IZFHMIN-LABEL: isnan_d_fpclass:
+; RV32IZFHMIN:       # %bb.0:
+; RV32IZFHMIN-NEXT:    fmv.x.h a0, fa0
+; RV32IZFHMIN-NEXT:    slli a0, a0, 17
+; RV32IZFHMIN-NEXT:    srli a0, a0, 17
+; RV32IZFHMIN-NEXT:    li a1, 31
+; RV32IZFHMIN-NEXT:    slli a1, a1, 10
+; RV32IZFHMIN-NEXT:    slt a0, a1, a0
+; RV32IZFHMIN-NEXT:    ret
+;
+; RV64IZFHMIN-LABEL: isnan_d_fpclass:
+; RV64IZFHMIN:       # %bb.0:
+; RV64IZFHMIN-NEXT:    fmv.x.h a0, fa0
+; RV64IZFHMIN-NEXT:    slli a0, a0, 49
+; RV64IZFHMIN-NEXT:    srli a0, a0, 49
+; RV64IZFHMIN-NEXT:    li a1, 31
+; RV64IZFHMIN-NEXT:    slli a1, a1, 10
+; RV64IZFHMIN-NEXT:    slt a0, a1, a0
+; RV64IZFHMIN-NEXT:    ret
+  %1 = call i1 @llvm.is.fpclass.f16(half %x, i32 3)  ; nan
+  ret i1 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll
index 63af17b80a41b..de3930a157128 100644
--- a/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll
@@ -67,27 +67,27 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IZFH-NEXT:    call __fixsfdi@plt
-; RV32IZFH-NEXT:    lui a3, 524288
-; RV32IZFH-NEXT:    bnez s0, .LBB1_4
+; RV32IZFH-NEXT:    lui a4, 524288
+; RV32IZFH-NEXT:    lui a2, 524288
+; RV32IZFH-NEXT:    beqz s0, .LBB1_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    lui a1, 524288
+; RV32IZFH-NEXT:    mv a2, a1
 ; RV32IZFH-NEXT:  .LBB1_4:
-; RV32IZFH-NEXT:    lui a2, %hi(.LCPI1_1)
-; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI1_1)(a2)
-; RV32IZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFH-NEXT:    beqz a2, .LBB1_6
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI1_1)
+; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI1_1)(a1)
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    beqz a3, .LBB1_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a1, a3, -1
+; RV32IZFH-NEXT:    addi a2, a4, -1
 ; RV32IZFH-NEXT:  .LBB1_6:
-; RV32IZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFH-NEXT:    seqz a3, a3
-; RV32IZFH-NEXT:    addi a3, a3, -1
-; RV32IZFH-NEXT:    and a1, a3, a1
-; RV32IZFH-NEXT:    neg a4, s0
-; RV32IZFH-NEXT:    and a0, a4, a0
-; RV32IZFH-NEXT:    neg a2, a2
+; RV32IZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFH-NEXT:    neg a4, a1
+; RV32IZFH-NEXT:    and a1, a4, a2
+; RV32IZFH-NEXT:    neg a2, s0
+; RV32IZFH-NEXT:    and a0, a2, a0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
-; RV32IZFH-NEXT:    and a0, a3, a0
+; RV32IZFH-NEXT:    and a0, a4, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -127,27 +127,27 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IZFHMIN-NEXT:    lui a3, 524288
-; RV32IZFHMIN-NEXT:    bnez s0, .LBB1_4
+; RV32IZFHMIN-NEXT:    lui a4, 524288
+; RV32IZFHMIN-NEXT:    lui a2, 524288
+; RV32IZFHMIN-NEXT:    beqz s0, .LBB1_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    lui a1, 524288
+; RV32IZFHMIN-NEXT:    mv a2, a1
 ; RV32IZFHMIN-NEXT:  .LBB1_4:
-; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI1_0)
-; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI1_0)(a2)
-; RV32IZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFHMIN-NEXT:    beqz a2, .LBB1_6
+; RV32IZFHMIN-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI1_0)(a1)
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB1_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IZFHMIN-NEXT:  .LBB1_6:
-; RV32IZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFHMIN-NEXT:    seqz a3, a3
-; RV32IZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IZFHMIN-NEXT:    and a1, a3, a1
-; RV32IZFHMIN-NEXT:    neg a4, s0
-; RV32IZFHMIN-NEXT:    and a0, a4, a0
-; RV32IZFHMIN-NEXT:    neg a2, a2
+; RV32IZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFHMIN-NEXT:    neg a4, a1
+; RV32IZFHMIN-NEXT:    and a1, a4, a2
+; RV32IZFHMIN-NEXT:    neg a2, s0
+; RV32IZFHMIN-NEXT:    and a0, a2, a0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
-; RV32IZFHMIN-NEXT:    and a0, a3, a0
+; RV32IZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -407,27 +407,27 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IZFH-NEXT:    call __fixsfdi@plt
-; RV32IZFH-NEXT:    lui a3, 524288
-; RV32IZFH-NEXT:    bnez s0, .LBB5_4
+; RV32IZFH-NEXT:    lui a4, 524288
+; RV32IZFH-NEXT:    lui a2, 524288
+; RV32IZFH-NEXT:    beqz s0, .LBB5_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    lui a1, 524288
+; RV32IZFH-NEXT:    mv a2, a1
 ; RV32IZFH-NEXT:  .LBB5_4:
-; RV32IZFH-NEXT:    lui a2, %hi(.LCPI5_1)
-; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI5_1)(a2)
-; RV32IZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFH-NEXT:    beqz a2, .LBB5_6
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI5_1)
+; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI5_1)(a1)
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    beqz a3, .LBB5_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a1, a3, -1
+; RV32IZFH-NEXT:    addi a2, a4, -1
 ; RV32IZFH-NEXT:  .LBB5_6:
-; RV32IZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFH-NEXT:    seqz a3, a3
-; RV32IZFH-NEXT:    addi a3, a3, -1
-; RV32IZFH-NEXT:    and a1, a3, a1
-; RV32IZFH-NEXT:    neg a4, s0
-; RV32IZFH-NEXT:    and a0, a4, a0
-; RV32IZFH-NEXT:    neg a2, a2
+; RV32IZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFH-NEXT:    neg a4, a1
+; RV32IZFH-NEXT:    and a1, a4, a2
+; RV32IZFH-NEXT:    neg a2, s0
+; RV32IZFH-NEXT:    and a0, a2, a0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
-; RV32IZFH-NEXT:    and a0, a3, a0
+; RV32IZFH-NEXT:    and a0, a4, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -467,27 +467,27 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IZFHMIN-NEXT:    lui a3, 524288
-; RV32IZFHMIN-NEXT:    bnez s0, .LBB5_4
+; RV32IZFHMIN-NEXT:    lui a4, 524288
+; RV32IZFHMIN-NEXT:    lui a2, 524288
+; RV32IZFHMIN-NEXT:    beqz s0, .LBB5_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    lui a1, 524288
+; RV32IZFHMIN-NEXT:    mv a2, a1
 ; RV32IZFHMIN-NEXT:  .LBB5_4:
-; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI5_0)
-; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI5_0)(a2)
-; RV32IZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFHMIN-NEXT:    beqz a2, .LBB5_6
+; RV32IZFHMIN-NEXT:    lui a1, %hi(.LCPI5_0)
+; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI5_0)(a1)
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB5_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IZFHMIN-NEXT:  .LBB5_6:
-; RV32IZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFHMIN-NEXT:    seqz a3, a3
-; RV32IZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IZFHMIN-NEXT:    and a1, a3, a1
-; RV32IZFHMIN-NEXT:    neg a4, s0
-; RV32IZFHMIN-NEXT:    and a0, a4, a0
-; RV32IZFHMIN-NEXT:    neg a2, a2
+; RV32IZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFHMIN-NEXT:    neg a4, a1
+; RV32IZFHMIN-NEXT:    and a1, a4, a2
+; RV32IZFHMIN-NEXT:    neg a2, s0
+; RV32IZFHMIN-NEXT:    and a0, a2, a0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
-; RV32IZFHMIN-NEXT:    and a0, a3, a0
+; RV32IZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -747,27 +747,27 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IZFH-NEXT:    call __fixsfdi@plt
-; RV32IZFH-NEXT:    lui a3, 524288
-; RV32IZFH-NEXT:    bnez s0, .LBB9_4
+; RV32IZFH-NEXT:    lui a4, 524288
+; RV32IZFH-NEXT:    lui a2, 524288
+; RV32IZFH-NEXT:    beqz s0, .LBB9_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    lui a1, 524288
+; RV32IZFH-NEXT:    mv a2, a1
 ; RV32IZFH-NEXT:  .LBB9_4:
-; RV32IZFH-NEXT:    lui a2, %hi(.LCPI9_1)
-; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI9_1)(a2)
-; RV32IZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFH-NEXT:    beqz a2, .LBB9_6
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI9_1)
+; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI9_1)(a1)
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    beqz a3, .LBB9_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a1, a3, -1
+; RV32IZFH-NEXT:    addi a2, a4, -1
 ; RV32IZFH-NEXT:  .LBB9_6:
-; RV32IZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFH-NEXT:    seqz a3, a3
-; RV32IZFH-NEXT:    addi a3, a3, -1
-; RV32IZFH-NEXT:    and a1, a3, a1
-; RV32IZFH-NEXT:    neg a4, s0
-; RV32IZFH-NEXT:    and a0, a4, a0
-; RV32IZFH-NEXT:    neg a2, a2
+; RV32IZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFH-NEXT:    neg a4, a1
+; RV32IZFH-NEXT:    and a1, a4, a2
+; RV32IZFH-NEXT:    neg a2, s0
+; RV32IZFH-NEXT:    and a0, a2, a0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
-; RV32IZFH-NEXT:    and a0, a3, a0
+; RV32IZFH-NEXT:    and a0, a4, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -807,27 +807,27 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IZFHMIN-NEXT:    lui a3, 524288
-; RV32IZFHMIN-NEXT:    bnez s0, .LBB9_4
+; RV32IZFHMIN-NEXT:    lui a4, 524288
+; RV32IZFHMIN-NEXT:    lui a2, 524288
+; RV32IZFHMIN-NEXT:    beqz s0, .LBB9_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    lui a1, 524288
+; RV32IZFHMIN-NEXT:    mv a2, a1
 ; RV32IZFHMIN-NEXT:  .LBB9_4:
-; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI9_0)
-; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI9_0)(a2)
-; RV32IZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFHMIN-NEXT:    beqz a2, .LBB9_6
+; RV32IZFHMIN-NEXT:    lui a1, %hi(.LCPI9_0)
+; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI9_0)(a1)
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB9_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IZFHMIN-NEXT:  .LBB9_6:
-; RV32IZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFHMIN-NEXT:    seqz a3, a3
-; RV32IZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IZFHMIN-NEXT:    and a1, a3, a1
-; RV32IZFHMIN-NEXT:    neg a4, s0
-; RV32IZFHMIN-NEXT:    and a0, a4, a0
-; RV32IZFHMIN-NEXT:    neg a2, a2
+; RV32IZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFHMIN-NEXT:    neg a4, a1
+; RV32IZFHMIN-NEXT:    and a1, a4, a2
+; RV32IZFHMIN-NEXT:    neg a2, s0
+; RV32IZFHMIN-NEXT:    and a0, a2, a0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
-; RV32IZFHMIN-NEXT:    and a0, a3, a0
+; RV32IZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -1087,27 +1087,27 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IZFH-NEXT:    call __fixsfdi@plt
-; RV32IZFH-NEXT:    lui a3, 524288
-; RV32IZFH-NEXT:    bnez s0, .LBB13_4
+; RV32IZFH-NEXT:    lui a4, 524288
+; RV32IZFH-NEXT:    lui a2, 524288
+; RV32IZFH-NEXT:    beqz s0, .LBB13_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    lui a1, 524288
+; RV32IZFH-NEXT:    mv a2, a1
 ; RV32IZFH-NEXT:  .LBB13_4:
-; RV32IZFH-NEXT:    lui a2, %hi(.LCPI13_1)
-; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI13_1)(a2)
-; RV32IZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFH-NEXT:    beqz a2, .LBB13_6
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI13_1)
+; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI13_1)(a1)
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    beqz a3, .LBB13_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a1, a3, -1
+; RV32IZFH-NEXT:    addi a2, a4, -1
 ; RV32IZFH-NEXT:  .LBB13_6:
-; RV32IZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFH-NEXT:    seqz a3, a3
-; RV32IZFH-NEXT:    addi a3, a3, -1
-; RV32IZFH-NEXT:    and a1, a3, a1
-; RV32IZFH-NEXT:    neg a4, s0
-; RV32IZFH-NEXT:    and a0, a4, a0
-; RV32IZFH-NEXT:    neg a2, a2
+; RV32IZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFH-NEXT:    neg a4, a1
+; RV32IZFH-NEXT:    and a1, a4, a2
+; RV32IZFH-NEXT:    neg a2, s0
+; RV32IZFH-NEXT:    and a0, a2, a0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
-; RV32IZFH-NEXT:    and a0, a3, a0
+; RV32IZFH-NEXT:    and a0, a4, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -1147,27 +1147,27 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IZFHMIN-NEXT:    lui a3, 524288
-; RV32IZFHMIN-NEXT:    bnez s0, .LBB13_4
+; RV32IZFHMIN-NEXT:    lui a4, 524288
+; RV32IZFHMIN-NEXT:    lui a2, 524288
+; RV32IZFHMIN-NEXT:    beqz s0, .LBB13_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    lui a1, 524288
+; RV32IZFHMIN-NEXT:    mv a2, a1
 ; RV32IZFHMIN-NEXT:  .LBB13_4:
-; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI13_0)
-; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI13_0)(a2)
-; RV32IZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFHMIN-NEXT:    beqz a2, .LBB13_6
+; RV32IZFHMIN-NEXT:    lui a1, %hi(.LCPI13_0)
+; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI13_0)(a1)
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB13_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IZFHMIN-NEXT:  .LBB13_6:
-; RV32IZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFHMIN-NEXT:    seqz a3, a3
-; RV32IZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IZFHMIN-NEXT:    and a1, a3, a1
-; RV32IZFHMIN-NEXT:    neg a4, s0
-; RV32IZFHMIN-NEXT:    and a0, a4, a0
-; RV32IZFHMIN-NEXT:    neg a2, a2
+; RV32IZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFHMIN-NEXT:    neg a4, a1
+; RV32IZFHMIN-NEXT:    and a1, a4, a2
+; RV32IZFHMIN-NEXT:    neg a2, s0
+; RV32IZFHMIN-NEXT:    and a0, a2, a0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
-; RV32IZFHMIN-NEXT:    and a0, a3, a0
+; RV32IZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -1427,27 +1427,27 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFH-NEXT:    fmv.s fa0, fs0
 ; RV32IZFH-NEXT:    call __fixsfdi@plt
-; RV32IZFH-NEXT:    lui a3, 524288
-; RV32IZFH-NEXT:    bnez s0, .LBB17_4
+; RV32IZFH-NEXT:    lui a4, 524288
+; RV32IZFH-NEXT:    lui a2, 524288
+; RV32IZFH-NEXT:    beqz s0, .LBB17_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    lui a1, 524288
+; RV32IZFH-NEXT:    mv a2, a1
 ; RV32IZFH-NEXT:  .LBB17_4:
-; RV32IZFH-NEXT:    lui a2, %hi(.LCPI17_1)
-; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI17_1)(a2)
-; RV32IZFH-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFH-NEXT:    beqz a2, .LBB17_6
+; RV32IZFH-NEXT:    lui a1, %hi(.LCPI17_1)
+; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI17_1)(a1)
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    beqz a3, .LBB17_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a1, a3, -1
+; RV32IZFH-NEXT:    addi a2, a4, -1
 ; RV32IZFH-NEXT:  .LBB17_6:
-; RV32IZFH-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFH-NEXT:    seqz a3, a3
-; RV32IZFH-NEXT:    addi a3, a3, -1
-; RV32IZFH-NEXT:    and a1, a3, a1
-; RV32IZFH-NEXT:    neg a4, s0
-; RV32IZFH-NEXT:    and a0, a4, a0
-; RV32IZFH-NEXT:    neg a2, a2
+; RV32IZFH-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFH-NEXT:    neg a4, a1
+; RV32IZFH-NEXT:    and a1, a4, a2
+; RV32IZFH-NEXT:    neg a2, s0
+; RV32IZFH-NEXT:    and a0, a2, a0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
-; RV32IZFH-NEXT:    and a0, a3, a0
+; RV32IZFH-NEXT:    and a0, a4, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
@@ -1487,27 +1487,27 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    fle.s s0, fa5, fs0
 ; RV32IZFHMIN-NEXT:    fmv.s fa0, fs0
 ; RV32IZFHMIN-NEXT:    call __fixsfdi@plt
-; RV32IZFHMIN-NEXT:    lui a3, 524288
-; RV32IZFHMIN-NEXT:    bnez s0, .LBB17_4
+; RV32IZFHMIN-NEXT:    lui a4, 524288
+; RV32IZFHMIN-NEXT:    lui a2, 524288
+; RV32IZFHMIN-NEXT:    beqz s0, .LBB17_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    lui a1, 524288
+; RV32IZFHMIN-NEXT:    mv a2, a1
 ; RV32IZFHMIN-NEXT:  .LBB17_4:
-; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI17_0)
-; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI17_0)(a2)
-; RV32IZFHMIN-NEXT:    flt.s a2, fa5, fs0
-; RV32IZFHMIN-NEXT:    beqz a2, .LBB17_6
+; RV32IZFHMIN-NEXT:    lui a1, %hi(.LCPI17_0)
+; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI17_0)(a1)
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB17_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a1, a3, -1
+; RV32IZFHMIN-NEXT:    addi a2, a4, -1
 ; RV32IZFHMIN-NEXT:  .LBB17_6:
-; RV32IZFHMIN-NEXT:    feq.s a3, fs0, fs0
-; RV32IZFHMIN-NEXT:    seqz a3, a3
-; RV32IZFHMIN-NEXT:    addi a3, a3, -1
-; RV32IZFHMIN-NEXT:    and a1, a3, a1
-; RV32IZFHMIN-NEXT:    neg a4, s0
-; RV32IZFHMIN-NEXT:    and a0, a4, a0
-; RV32IZFHMIN-NEXT:    neg a2, a2
+; RV32IZFHMIN-NEXT:    feq.s a1, fs0, fs0
+; RV32IZFHMIN-NEXT:    neg a4, a1
+; RV32IZFHMIN-NEXT:    and a1, a4, a2
+; RV32IZFHMIN-NEXT:    neg a2, s0
+; RV32IZFHMIN-NEXT:    and a0, a2, a0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
-; RV32IZFHMIN-NEXT:    and a0, a3, a0
+; RV32IZFHMIN-NEXT:    and a0, a4, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
index 90dad26a1833f..0968f8afd7fd8 100644
--- a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
+++ b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
@@ -139,34 +139,30 @@ if.end:                                           ; preds = %if.then, %entry
 define dso_local i32 @load_half() nounwind {
 ; RV32-LABEL: load_half:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    lui a0, %hi(foo+8)
 ; RV32-NEXT:    lhu a0, %lo(foo+8)(a0)
 ; RV32-NEXT:    li a1, 140
 ; RV32-NEXT:    bne a0, a1, .LBB8_2
 ; RV32-NEXT:  # %bb.1: # %if.end
 ; RV32-NEXT:    li a0, 0
-; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ; RV32-NEXT:  .LBB8_2: # %if.then
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    call abort@plt
 ;
 ; RV64-LABEL: load_half:
 ; RV64:       # %bb.0: # %entry
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    lui a0, %hi(foo+8)
 ; RV64-NEXT:    lhu a0, %lo(foo+8)(a0)
 ; RV64-NEXT:    li a1, 140
 ; RV64-NEXT:    bne a0, a1, .LBB8_2
 ; RV64-NEXT:  # %bb.1: # %if.end
 ; RV64-NEXT:    li a0, 0
-; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
 ; RV64-NEXT:  .LBB8_2: # %if.then
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    call abort@plt
 entry:
   %0 = load i16, ptr getelementptr inbounds ([6 x i16], ptr @foo, i32 0, i32 4), align 2
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr.ll b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
index 2eb8dbe74453f..eb683f01704f1 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
@@ -14,23 +14,16 @@
 ; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV64-FD
 
 ;
-; Checking for special return instructions (uret, sret, mret).
+; Checking for special return instructions (sret, mret).
 ;
-define void @foo_user() #0 {
-; CHECK-LABEL: foo_user:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    uret
-  ret void
-}
-
-define void @foo_supervisor() #1 {
+define void @foo_supervisor() #0 {
 ; CHECK-LABEL: foo_supervisor:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    sret
   ret void
 }
 
-define void @foo_machine() #2 {
+define void @foo_machine() #1 {
 ; CHECK-LABEL: foo_machine:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mret
@@ -49,7 +42,7 @@ define void @foo_machine() #2 {
 ;
 
 declare i32 @otherfoo(...)
-define void @foo_with_call() #2 {
+define void @foo_with_call() #1 {
 ;
 ; CHECK-RV32-LABEL: foo_with_call:
 ; CHECK-RV32:       # %bb.0:
@@ -547,7 +540,7 @@ define void @foo_with_call() #2 {
 ;
 ; Additionally check frame pointer and return address are properly saved.
 ;
-define void @foo_fp_with_call() #3 {
+define void @foo_fp_with_call() #2 {
 ;
 ; CHECK-RV32-LABEL: foo_fp_with_call:
 ; CHECK-RV32:       # %bb.0:
@@ -1060,7 +1053,6 @@ define void @foo_fp_with_call() #3 {
   ret void
 }
 
-attributes #0 = { nounwind "interrupt"="user" }
-attributes #1 = { nounwind "interrupt"="supervisor" }
-attributes #2 = { nounwind  "interrupt"="machine" }
-attributes #3 = { nounwind "interrupt"="machine" "frame-pointer"="all" }
+attributes #0 = { nounwind "interrupt"="supervisor" }
+attributes #1 = { nounwind "interrupt"="machine" }
+attributes #2 = { nounwind "interrupt"="machine" "frame-pointer"="all" }
diff --git a/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll b/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll
index 6a4ecf38427ec..66dad442dc087 100644
--- a/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll
+++ b/llvm/test/CodeGen/RISCV/nontemporal-scalable.ll
@@ -130,4 +130,520 @@ define void @test_nontemporal_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
   ret void
 }
 
+define <vscale x 2 x i64> @test_nontemporal_P1_load_nxv2i64(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 4 x i32> @test_nontemporal_P1_load_nxv4i32(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i16> @test_nontemporal_P1_load_nxv8i16(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <vscale x 8 x i16> %1
+}
+
+define <vscale x 16 x i8> @test_nontemporal_P1_load_nxv16i8(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <vscale x 16 x i8> %1
+}
+
+define void @test_nontemporal_P1_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_nxv4i32(ptr %p, <vscale x 4 x i32> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_nxv8i16(ptr %p, <vscale x 8 x i16> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define <vscale x 2 x i64> @test_nontemporal_PALL_load_nxv2i64(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 4 x i32> @test_nontemporal_PALL_load_nxv4i32(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i16> @test_nontemporal_PALL_load_nxv8i16(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <vscale x 8 x i16> %1
+}
+
+define <vscale x 16 x i8> @test_nontemporal_PALL_load_nxv16i8(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <vscale x 16 x i8> %1
+}
+
+define void @test_nontemporal_PALL_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_nxv4i32(ptr %p, <vscale x 4 x i32> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_nxv8i16(ptr %p, <vscale x 8 x i16> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define <vscale x 2 x i64> @test_nontemporal_S1_load_nxv2i64(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 4 x i32> @test_nontemporal_S1_load_nxv4i32(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i16> @test_nontemporal_S1_load_nxv8i16(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <vscale x 8 x i16> %1
+}
+
+define <vscale x 16 x i8> @test_nontemporal_S1_load_nxv16i8(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <vscale x 16 x i8> %1
+}
+
+define void @test_nontemporal_S1_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_nxv4i32(ptr %p, <vscale x 4 x i32> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_nxv8i16(ptr %p, <vscale x 8 x i16> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define <vscale x 2 x i64> @test_nontemporal_ALL_load_nxv2i64(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 4 x i32> @test_nontemporal_ALL_load_nxv4i32(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i16> @test_nontemporal_ALL_load_nxv8i16(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <vscale x 8 x i16> %1
+}
+
+define <vscale x 16 x i8> @test_nontemporal_ALL_load_nxv16i8(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <vscale x 16 x i8> %1
+}
+
+define void @test_nontemporal_ALL_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_nxv4i32(ptr %p, <vscale x 4 x i32> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_nxv8i16(ptr %p, <vscale x 8 x i16> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
 !0 = !{i32 1}
+!1 = !{i32 2}
+!2 = !{i32 3}
+!3 = !{i32 4}
+!4 = !{i32 5}
diff --git a/llvm/test/CodeGen/RISCV/nontemporal.ll b/llvm/test/CodeGen/RISCV/nontemporal.ll
index 190b896f486c6..e1b19a9c33835 100644
--- a/llvm/test/CodeGen/RISCV/nontemporal.ll
+++ b/llvm/test/CodeGen/RISCV/nontemporal.ll
@@ -1438,4 +1438,5665 @@ define void @test_nontemporal_store_v2i64(ptr %p, <2 x i64> %v) {
   ret void
 }
 
+define i64 @test_nontemporal_P1_load_i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a1, 4(a0)
+; CHECK-RV32-NEXT:    mv a0, a2
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a2, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a1, 4(a0)
+; CHECK-RV32C-NEXT:    mv a0, a2
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    ld a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    lw a2, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    lw a1, 4(a0)
+; CHECK-RV32V-NEXT:    mv a0, a2
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i64, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret i64 %1
+}
+
+define i32 @test_nontemporal_P1_load_i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    lw a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    lw a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    lw a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    lw a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i32, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret i32 %1
+}
+
+define i16 @test_nontemporal_P1_load_i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    lh a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lh a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    lh a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lh a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    lh a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    lh a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i16, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret i16 %1
+}
+
+define i8 @test_nontemporal_P1_load_i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i8, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret i8 %1
+}
+
+define half @test_nontemporal_P1_load_half(ptr %p) nounwind {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32V-NEXT:    ret
+  %1 = load half, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  %2 = getelementptr half, ptr %p, i32 3
+  %3 = load half, ptr %2, !nontemporal !0, !riscv-nontemporal-domain !1
+  %4 = fadd half %1, %3
+  ret half %4
+}
+
+define float @test_nontemporal_P1_load_float(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load float, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret float %1
+}
+
+define double @test_nontemporal_P1_load_double(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load double, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret double %1
+}
+
+define <16 x i8> @test_nontemporal_P1_load_v16i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <16 x i8> %1
+}
+
+define <8 x i16> @test_nontemporal_P1_load_v8i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @test_nontemporal_P1_load_v4i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @test_nontemporal_P1_load_v2i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_load_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    ld a1, 8(a0)
+; CHECK-RV64-NEXT:    mv a0, a2
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_load_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_load_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a2, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    ld a1, 8(a0)
+; CHECK-RV64C-NEXT:    mv a0, a2
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_load_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_load_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_load_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret <2 x i64> %1
+}
+
+define void @test_nontemporal_P1_store_i64(ptr %p, i64 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    sd a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_i32(ptr %p, i32 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    sw a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i32 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_i16(ptr %p, i16 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    sh a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    sh a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i16 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_i8(ptr %p, i8 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    sb a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    sb a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i8 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_half(ptr %p, half %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store half %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_float(ptr %p, float %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store float %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_double(ptr %p, double %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store double %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_v16i8(ptr %p, <16 x i8> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    addi sp, sp, -16
+; CHECK-RV64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64-NEXT:    lbu a2, 0(a1)
+; CHECK-RV64-NEXT:    lbu a3, 8(a1)
+; CHECK-RV64-NEXT:    lbu a4, 16(a1)
+; CHECK-RV64-NEXT:    lbu a5, 24(a1)
+; CHECK-RV64-NEXT:    lbu a6, 32(a1)
+; CHECK-RV64-NEXT:    lbu a7, 40(a1)
+; CHECK-RV64-NEXT:    lbu t0, 48(a1)
+; CHECK-RV64-NEXT:    lbu t1, 56(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 120(a1)
+; CHECK-RV64-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t6, 15(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb s0, 14(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb s1, 13(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a1, 12(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a7, 5(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a3, 1(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    addi sp, sp, 16
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -16
+; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32-NEXT:    lbu a2, 0(a1)
+; CHECK-RV32-NEXT:    lbu a3, 4(a1)
+; CHECK-RV32-NEXT:    lbu a4, 8(a1)
+; CHECK-RV32-NEXT:    lbu a5, 12(a1)
+; CHECK-RV32-NEXT:    lbu a6, 16(a1)
+; CHECK-RV32-NEXT:    lbu a7, 20(a1)
+; CHECK-RV32-NEXT:    lbu t0, 24(a1)
+; CHECK-RV32-NEXT:    lbu t1, 28(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 60(a1)
+; CHECK-RV32-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t6, 15(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb s0, 14(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb s1, 13(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a1, 12(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a7, 5(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a3, 1(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 16
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    addi sp, sp, -16
+; CHECK-RV64C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64C-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64C-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV64C-NEXT:    lbu a7, 8(a1)
+; CHECK-RV64C-NEXT:    lbu t0, 16(a1)
+; CHECK-RV64C-NEXT:    lbu t1, 24(a1)
+; CHECK-RV64C-NEXT:    lbu t2, 32(a1)
+; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
+; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
+; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
+; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 120(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a2, 15(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb s0, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb s1, 13(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a1, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t6, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t2, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    addi sp, sp, 16
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    addi sp, sp, -16
+; CHECK-RV32C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32C-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32C-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV32C-NEXT:    lbu a7, 4(a1)
+; CHECK-RV32C-NEXT:    lbu t0, 8(a1)
+; CHECK-RV32C-NEXT:    lbu t1, 12(a1)
+; CHECK-RV32C-NEXT:    lbu t2, 16(a1)
+; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
+; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
+; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
+; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 60(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a2, 15(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb s0, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb s1, 13(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a1, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t6, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    addi sp, sp, 16
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_v8i16(ptr %p, <8 x i16> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 56(a1)
+; CHECK-RV64-NEXT:    lh a7, 48(a1)
+; CHECK-RV64-NEXT:    lh t0, 40(a1)
+; CHECK-RV64-NEXT:    lh a1, 32(a1)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a6, 14(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a7, 12(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh t0, 10(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a1, 8(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 28(a1)
+; CHECK-RV32-NEXT:    lh a7, 24(a1)
+; CHECK-RV32-NEXT:    lh t0, 20(a1)
+; CHECK-RV32-NEXT:    lh a1, 16(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a6, 14(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a7, 12(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh t0, 10(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a1, 8(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 56(a1)
+; CHECK-RV64C-NEXT:    lh a3, 48(a1)
+; CHECK-RV64C-NEXT:    lh a4, 40(a1)
+; CHECK-RV64C-NEXT:    lh a1, 32(a1)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a2, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a3, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a1, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 28(a1)
+; CHECK-RV32C-NEXT:    lh a3, 24(a1)
+; CHECK-RV32C-NEXT:    lh a4, 20(a1)
+; CHECK-RV32C-NEXT:    lh a1, 16(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a2, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a3, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a1, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_v4i32(ptr %p, <4 x i32> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lw a2, 24(a1)
+; CHECK-RV64-NEXT:    lw a3, 16(a1)
+; CHECK-RV64-NEXT:    lw a4, 8(a1)
+; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lw a2, 24(a1)
+; CHECK-RV64C-NEXT:    lw a3, 16(a1)
+; CHECK-RV64C-NEXT:    lw a4, 8(a1)
+; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define void @test_nontemporal_P1_store_v2i64(ptr %p, <2 x i64> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_P1_store_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.p1
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_P1_store_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.p1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_P1_store_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.p1
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_P1_store_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.p1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_P1_store_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.p1
+; CHECK-RV64V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_P1_store_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.p1
+; CHECK-RV32V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !1
+  ret void
+}
+
+define i64 @test_nontemporal_PALL_load_i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a1, 4(a0)
+; CHECK-RV32-NEXT:    mv a0, a2
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a2, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a1, 4(a0)
+; CHECK-RV32C-NEXT:    mv a0, a2
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    ld a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    lw a2, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    lw a1, 4(a0)
+; CHECK-RV32V-NEXT:    mv a0, a2
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i64, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret i64 %1
+}
+
+define i32 @test_nontemporal_PALL_load_i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    lw a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    lw a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    lw a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    lw a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i32, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret i32 %1
+}
+
+define i16 @test_nontemporal_PALL_load_i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    lh a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lh a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    lh a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lh a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    lh a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    lh a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i16, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret i16 %1
+}
+
+define i8 @test_nontemporal_PALL_load_i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i8, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret i8 %1
+}
+
+define half @test_nontemporal_PALL_load_half(ptr %p) nounwind {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32V-NEXT:    ret
+  %1 = load half, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  %2 = getelementptr half, ptr %p, i32 3
+  %3 = load half, ptr %2, !nontemporal !0, !riscv-nontemporal-domain !2
+  %4 = fadd half %1, %3
+  ret half %4
+}
+
+define float @test_nontemporal_PALL_load_float(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load float, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret float %1
+}
+
+define double @test_nontemporal_PALL_load_double(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load double, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret double %1
+}
+
+define <16 x i8> @test_nontemporal_PALL_load_v16i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <16 x i8> %1
+}
+
+define <8 x i16> @test_nontemporal_PALL_load_v8i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @test_nontemporal_PALL_load_v4i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @test_nontemporal_PALL_load_v2i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_load_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    ld a1, 8(a0)
+; CHECK-RV64-NEXT:    mv a0, a2
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_load_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_load_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a2, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    ld a1, 8(a0)
+; CHECK-RV64C-NEXT:    mv a0, a2
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_load_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_load_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_load_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret <2 x i64> %1
+}
+
+define void @test_nontemporal_PALL_store_i64(ptr %p, i64 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    sd a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_i32(ptr %p, i32 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    sw a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i32 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_i16(ptr %p, i16 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    sh a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    sh a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i16 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_i8(ptr %p, i8 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    sb a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    sb a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i8 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_half(ptr %p, half %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store half %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_float(ptr %p, float %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store float %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_double(ptr %p, double %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store double %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_v16i8(ptr %p, <16 x i8> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    addi sp, sp, -16
+; CHECK-RV64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64-NEXT:    lbu a2, 0(a1)
+; CHECK-RV64-NEXT:    lbu a3, 8(a1)
+; CHECK-RV64-NEXT:    lbu a4, 16(a1)
+; CHECK-RV64-NEXT:    lbu a5, 24(a1)
+; CHECK-RV64-NEXT:    lbu a6, 32(a1)
+; CHECK-RV64-NEXT:    lbu a7, 40(a1)
+; CHECK-RV64-NEXT:    lbu t0, 48(a1)
+; CHECK-RV64-NEXT:    lbu t1, 56(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 120(a1)
+; CHECK-RV64-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t6, 15(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb s0, 14(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb s1, 13(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a1, 12(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a7, 5(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a3, 1(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    addi sp, sp, 16
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -16
+; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32-NEXT:    lbu a2, 0(a1)
+; CHECK-RV32-NEXT:    lbu a3, 4(a1)
+; CHECK-RV32-NEXT:    lbu a4, 8(a1)
+; CHECK-RV32-NEXT:    lbu a5, 12(a1)
+; CHECK-RV32-NEXT:    lbu a6, 16(a1)
+; CHECK-RV32-NEXT:    lbu a7, 20(a1)
+; CHECK-RV32-NEXT:    lbu t0, 24(a1)
+; CHECK-RV32-NEXT:    lbu t1, 28(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 60(a1)
+; CHECK-RV32-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t6, 15(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb s0, 14(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb s1, 13(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a1, 12(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a7, 5(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a3, 1(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 16
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    addi sp, sp, -16
+; CHECK-RV64C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64C-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64C-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV64C-NEXT:    lbu a7, 8(a1)
+; CHECK-RV64C-NEXT:    lbu t0, 16(a1)
+; CHECK-RV64C-NEXT:    lbu t1, 24(a1)
+; CHECK-RV64C-NEXT:    lbu t2, 32(a1)
+; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
+; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
+; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
+; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 120(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a2, 15(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb s0, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb s1, 13(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a1, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t6, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t2, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    addi sp, sp, 16
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    addi sp, sp, -16
+; CHECK-RV32C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32C-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32C-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV32C-NEXT:    lbu a7, 4(a1)
+; CHECK-RV32C-NEXT:    lbu t0, 8(a1)
+; CHECK-RV32C-NEXT:    lbu t1, 12(a1)
+; CHECK-RV32C-NEXT:    lbu t2, 16(a1)
+; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
+; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
+; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
+; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 60(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a2, 15(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb s0, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb s1, 13(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a1, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t6, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    addi sp, sp, 16
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_v8i16(ptr %p, <8 x i16> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 56(a1)
+; CHECK-RV64-NEXT:    lh a7, 48(a1)
+; CHECK-RV64-NEXT:    lh t0, 40(a1)
+; CHECK-RV64-NEXT:    lh a1, 32(a1)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a6, 14(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a7, 12(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh t0, 10(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a1, 8(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 28(a1)
+; CHECK-RV32-NEXT:    lh a7, 24(a1)
+; CHECK-RV32-NEXT:    lh t0, 20(a1)
+; CHECK-RV32-NEXT:    lh a1, 16(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a6, 14(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a7, 12(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh t0, 10(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a1, 8(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 56(a1)
+; CHECK-RV64C-NEXT:    lh a3, 48(a1)
+; CHECK-RV64C-NEXT:    lh a4, 40(a1)
+; CHECK-RV64C-NEXT:    lh a1, 32(a1)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a2, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a3, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a1, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 28(a1)
+; CHECK-RV32C-NEXT:    lh a3, 24(a1)
+; CHECK-RV32C-NEXT:    lh a4, 20(a1)
+; CHECK-RV32C-NEXT:    lh a1, 16(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a2, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a3, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a1, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_v4i32(ptr %p, <4 x i32> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lw a2, 24(a1)
+; CHECK-RV64-NEXT:    lw a3, 16(a1)
+; CHECK-RV64-NEXT:    lw a4, 8(a1)
+; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lw a2, 24(a1)
+; CHECK-RV64C-NEXT:    lw a3, 16(a1)
+; CHECK-RV64C-NEXT:    lw a4, 8(a1)
+; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define void @test_nontemporal_PALL_store_v2i64(ptr %p, <2 x i64> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_PALL_store_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.pall
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_PALL_store_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.pall
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_PALL_store_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.pall
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_PALL_store_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.pall
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_PALL_store_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.pall
+; CHECK-RV64V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_PALL_store_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.pall
+; CHECK-RV32V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !2
+  ret void
+}
+
+define i64 @test_nontemporal_S1_load_i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a1, 4(a0)
+; CHECK-RV32-NEXT:    mv a0, a2
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a2, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a1, 4(a0)
+; CHECK-RV32C-NEXT:    mv a0, a2
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    ld a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    lw a2, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    lw a1, 4(a0)
+; CHECK-RV32V-NEXT:    mv a0, a2
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i64, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret i64 %1
+}
+
+define i32 @test_nontemporal_S1_load_i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    lw a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    lw a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    lw a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    lw a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i32, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret i32 %1
+}
+
+define i16 @test_nontemporal_S1_load_i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    lh a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lh a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    lh a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lh a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    lh a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    lh a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i16, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret i16 %1
+}
+
+define i8 @test_nontemporal_S1_load_i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i8, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret i8 %1
+}
+
+define half @test_nontemporal_S1_load_half(ptr %p) nounwind {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32V-NEXT:    ret
+  %1 = load half, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  %2 = getelementptr half, ptr %p, i32 3
+  %3 = load half, ptr %2, !nontemporal !0, !riscv-nontemporal-domain !3
+  %4 = fadd half %1, %3
+  ret half %4
+}
+
+define float @test_nontemporal_S1_load_float(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load float, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret float %1
+}
+
+define double @test_nontemporal_S1_load_double(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load double, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret double %1
+}
+
+define <16 x i8> @test_nontemporal_S1_load_v16i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <16 x i8> %1
+}
+
+define <8 x i16> @test_nontemporal_S1_load_v8i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @test_nontemporal_S1_load_v4i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @test_nontemporal_S1_load_v2i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_load_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    ld a1, 8(a0)
+; CHECK-RV64-NEXT:    mv a0, a2
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_load_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_load_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a2, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    ld a1, 8(a0)
+; CHECK-RV64C-NEXT:    mv a0, a2
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_load_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_load_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_load_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret <2 x i64> %1
+}
+
+define void @test_nontemporal_S1_store_i64(ptr %p, i64 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    sd a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_i32(ptr %p, i32 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    sw a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i32 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_i16(ptr %p, i16 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    sh a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    sh a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i16 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_i8(ptr %p, i8 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    sb a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    sb a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i8 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_half(ptr %p, half %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store half %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_float(ptr %p, float %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store float %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_double(ptr %p, double %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store double %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_v16i8(ptr %p, <16 x i8> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    addi sp, sp, -16
+; CHECK-RV64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64-NEXT:    lbu a2, 0(a1)
+; CHECK-RV64-NEXT:    lbu a3, 8(a1)
+; CHECK-RV64-NEXT:    lbu a4, 16(a1)
+; CHECK-RV64-NEXT:    lbu a5, 24(a1)
+; CHECK-RV64-NEXT:    lbu a6, 32(a1)
+; CHECK-RV64-NEXT:    lbu a7, 40(a1)
+; CHECK-RV64-NEXT:    lbu t0, 48(a1)
+; CHECK-RV64-NEXT:    lbu t1, 56(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 120(a1)
+; CHECK-RV64-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t6, 15(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb s0, 14(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb s1, 13(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a1, 12(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a7, 5(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a3, 1(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    addi sp, sp, 16
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -16
+; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32-NEXT:    lbu a2, 0(a1)
+; CHECK-RV32-NEXT:    lbu a3, 4(a1)
+; CHECK-RV32-NEXT:    lbu a4, 8(a1)
+; CHECK-RV32-NEXT:    lbu a5, 12(a1)
+; CHECK-RV32-NEXT:    lbu a6, 16(a1)
+; CHECK-RV32-NEXT:    lbu a7, 20(a1)
+; CHECK-RV32-NEXT:    lbu t0, 24(a1)
+; CHECK-RV32-NEXT:    lbu t1, 28(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 60(a1)
+; CHECK-RV32-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t6, 15(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb s0, 14(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb s1, 13(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a1, 12(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a7, 5(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a3, 1(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 16
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    addi sp, sp, -16
+; CHECK-RV64C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64C-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64C-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV64C-NEXT:    lbu a7, 8(a1)
+; CHECK-RV64C-NEXT:    lbu t0, 16(a1)
+; CHECK-RV64C-NEXT:    lbu t1, 24(a1)
+; CHECK-RV64C-NEXT:    lbu t2, 32(a1)
+; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
+; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
+; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
+; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 120(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a2, 15(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb s0, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb s1, 13(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a1, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t6, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t2, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    addi sp, sp, 16
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    addi sp, sp, -16
+; CHECK-RV32C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32C-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32C-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV32C-NEXT:    lbu a7, 4(a1)
+; CHECK-RV32C-NEXT:    lbu t0, 8(a1)
+; CHECK-RV32C-NEXT:    lbu t1, 12(a1)
+; CHECK-RV32C-NEXT:    lbu t2, 16(a1)
+; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
+; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
+; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
+; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 60(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a2, 15(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb s0, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb s1, 13(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a1, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t6, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    addi sp, sp, 16
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_v8i16(ptr %p, <8 x i16> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 56(a1)
+; CHECK-RV64-NEXT:    lh a7, 48(a1)
+; CHECK-RV64-NEXT:    lh t0, 40(a1)
+; CHECK-RV64-NEXT:    lh a1, 32(a1)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a6, 14(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a7, 12(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh t0, 10(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a1, 8(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 28(a1)
+; CHECK-RV32-NEXT:    lh a7, 24(a1)
+; CHECK-RV32-NEXT:    lh t0, 20(a1)
+; CHECK-RV32-NEXT:    lh a1, 16(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a6, 14(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a7, 12(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh t0, 10(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a1, 8(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 56(a1)
+; CHECK-RV64C-NEXT:    lh a3, 48(a1)
+; CHECK-RV64C-NEXT:    lh a4, 40(a1)
+; CHECK-RV64C-NEXT:    lh a1, 32(a1)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a2, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a3, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a1, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 28(a1)
+; CHECK-RV32C-NEXT:    lh a3, 24(a1)
+; CHECK-RV32C-NEXT:    lh a4, 20(a1)
+; CHECK-RV32C-NEXT:    lh a1, 16(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a2, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a3, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a1, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_v4i32(ptr %p, <4 x i32> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lw a2, 24(a1)
+; CHECK-RV64-NEXT:    lw a3, 16(a1)
+; CHECK-RV64-NEXT:    lw a4, 8(a1)
+; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lw a2, 24(a1)
+; CHECK-RV64C-NEXT:    lw a3, 16(a1)
+; CHECK-RV64C-NEXT:    lw a4, 8(a1)
+; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define void @test_nontemporal_S1_store_v2i64(ptr %p, <2 x i64> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_S1_store_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.s1
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_S1_store_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.s1
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_S1_store_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.s1
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_S1_store_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.s1
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_S1_store_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.s1
+; CHECK-RV64V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_S1_store_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.s1
+; CHECK-RV32V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !3
+  ret void
+}
+
+define i64 @test_nontemporal_ALL_load_i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a1, 4(a0)
+; CHECK-RV32-NEXT:    mv a0, a2
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a2, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a1, 4(a0)
+; CHECK-RV32C-NEXT:    mv a0, a2
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    ld a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    lw a2, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    lw a1, 4(a0)
+; CHECK-RV32V-NEXT:    mv a0, a2
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i64, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret i64 %1
+}
+
+define i32 @test_nontemporal_ALL_load_i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    lw a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    lw a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    lw a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    lw a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i32, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret i32 %1
+}
+
+define i16 @test_nontemporal_ALL_load_i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    lh a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lh a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    lh a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lh a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    lh a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    lh a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i16, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret i16 %1
+}
+
+define i8 @test_nontemporal_ALL_load_i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    lbu a0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load i8, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret i8 %1
+}
+
+define half @test_nontemporal_ALL_load_half(ptr %p) nounwind {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32C-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV64V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    flh fa5, 0(a0)
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    flh fa4, 6(a0)
+; CHECK-RV32V-NEXT:    fadd.h fa0, fa5, fa4
+; CHECK-RV32V-NEXT:    ret
+  %1 = load half, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  %2 = getelementptr half, ptr %p, i32 3
+  %3 = load half, ptr %2, !nontemporal !0, !riscv-nontemporal-domain !4
+  %4 = fadd half %1, %3
+  ret half %4
+}
+
+define float @test_nontemporal_ALL_load_float(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    flw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load float, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret float %1
+}
+
+define double @test_nontemporal_ALL_load_double(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    fld fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load double, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret double %1
+}
+
+define <16 x i8> @test_nontemporal_ALL_load_v16i8(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vle8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <16 x i8>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <16 x i8> %1
+}
+
+define <8 x i16> @test_nontemporal_ALL_load_v8i16(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vle16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <8 x i16>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @test_nontemporal_ALL_load_v4i32(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a2, 8(a1)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a2, 8(a1)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a1, 0(a1)
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <4 x i32>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @test_nontemporal_ALL_load_v2i64(ptr %p) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_load_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    ld a1, 8(a0)
+; CHECK-RV64-NEXT:    mv a0, a2
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_load_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_load_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a2, 0(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    ld a1, 8(a0)
+; CHECK-RV64C-NEXT:    mv a0, a2
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_load_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_load_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_load_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vle64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <2 x i64>, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret <2 x i64> %1
+}
+
+define void @test_nontemporal_ALL_store_i64(ptr %p, i64 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a2, 4(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    sd a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sw a2, 4(a0)
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i64 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_i32(ptr %p, i32 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    sw a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sw a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i32 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_i16(ptr %p, i16 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    sh a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sh a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i16 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_i8(ptr %p, i8 %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    sb a1, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    sb a1, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store i8 %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_half(ptr %p, half %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_half:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_half:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_half:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_half:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_half:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_half:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    fsh fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store half %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_float(ptr %p, float %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_float:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_float:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_float:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_float:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_float:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_float:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    fsw fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store float %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_double(ptr %p, double %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_double:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_double:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_double:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_double:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_double:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_double:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    fsd fa0, 0(a0)
+; CHECK-RV32V-NEXT:    ret
+  store double %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_v16i8(ptr %p, <16 x i8> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v16i8:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    addi sp, sp, -16
+; CHECK-RV64-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64-NEXT:    lbu a2, 0(a1)
+; CHECK-RV64-NEXT:    lbu a3, 8(a1)
+; CHECK-RV64-NEXT:    lbu a4, 16(a1)
+; CHECK-RV64-NEXT:    lbu a5, 24(a1)
+; CHECK-RV64-NEXT:    lbu a6, 32(a1)
+; CHECK-RV64-NEXT:    lbu a7, 40(a1)
+; CHECK-RV64-NEXT:    lbu t0, 48(a1)
+; CHECK-RV64-NEXT:    lbu t1, 56(a1)
+; CHECK-RV64-NEXT:    lbu t2, 64(a1)
+; CHECK-RV64-NEXT:    lbu t3, 72(a1)
+; CHECK-RV64-NEXT:    lbu t4, 80(a1)
+; CHECK-RV64-NEXT:    lbu t5, 88(a1)
+; CHECK-RV64-NEXT:    lbu t6, 120(a1)
+; CHECK-RV64-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t6, 15(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb s0, 14(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb s1, 13(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a1, 12(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t5, 11(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t4, 10(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t3, 9(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t1, 7(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb t0, 6(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a7, 5(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a6, 4(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a5, 3(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a4, 2(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a3, 1(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sb a2, 0(a0)
+; CHECK-RV64-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    addi sp, sp, 16
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v16i8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -16
+; CHECK-RV32-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32-NEXT:    lbu a2, 0(a1)
+; CHECK-RV32-NEXT:    lbu a3, 4(a1)
+; CHECK-RV32-NEXT:    lbu a4, 8(a1)
+; CHECK-RV32-NEXT:    lbu a5, 12(a1)
+; CHECK-RV32-NEXT:    lbu a6, 16(a1)
+; CHECK-RV32-NEXT:    lbu a7, 20(a1)
+; CHECK-RV32-NEXT:    lbu t0, 24(a1)
+; CHECK-RV32-NEXT:    lbu t1, 28(a1)
+; CHECK-RV32-NEXT:    lbu t2, 32(a1)
+; CHECK-RV32-NEXT:    lbu t3, 36(a1)
+; CHECK-RV32-NEXT:    lbu t4, 40(a1)
+; CHECK-RV32-NEXT:    lbu t5, 44(a1)
+; CHECK-RV32-NEXT:    lbu t6, 60(a1)
+; CHECK-RV32-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t6, 15(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb s0, 14(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb s1, 13(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a1, 12(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t5, 11(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t4, 10(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t3, 9(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t2, 8(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t1, 7(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb t0, 6(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a7, 5(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a6, 4(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a5, 3(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a4, 2(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a3, 1(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sb a2, 0(a0)
+; CHECK-RV32-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 16
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v16i8:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    addi sp, sp, -16
+; CHECK-RV64C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV64C-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64C-NEXT:    .cfi_offset s0, -8
+; CHECK-RV64C-NEXT:    .cfi_offset s1, -16
+; CHECK-RV64C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV64C-NEXT:    lbu a7, 8(a1)
+; CHECK-RV64C-NEXT:    lbu t0, 16(a1)
+; CHECK-RV64C-NEXT:    lbu t1, 24(a1)
+; CHECK-RV64C-NEXT:    lbu t2, 32(a1)
+; CHECK-RV64C-NEXT:    lbu t3, 40(a1)
+; CHECK-RV64C-NEXT:    lbu t4, 48(a1)
+; CHECK-RV64C-NEXT:    lbu t5, 56(a1)
+; CHECK-RV64C-NEXT:    lbu t6, 64(a1)
+; CHECK-RV64C-NEXT:    lbu a3, 72(a1)
+; CHECK-RV64C-NEXT:    lbu a4, 80(a1)
+; CHECK-RV64C-NEXT:    lbu a5, 88(a1)
+; CHECK-RV64C-NEXT:    lbu a2, 120(a1)
+; CHECK-RV64C-NEXT:    lbu s0, 112(a1)
+; CHECK-RV64C-NEXT:    lbu s1, 104(a1)
+; CHECK-RV64C-NEXT:    lbu a1, 96(a1)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a2, 15(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb s0, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb s1, 13(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a1, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a5, 11(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a3, 9(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t6, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t5, 7(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t4, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t3, 5(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t2, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t1, 3(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb t0, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a7, 1(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sb a6, 0(a0)
+; CHECK-RV64C-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64C-NEXT:    addi sp, sp, 16
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v16i8:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    addi sp, sp, -16
+; CHECK-RV32C-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-RV32C-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32C-NEXT:    .cfi_offset s0, -4
+; CHECK-RV32C-NEXT:    .cfi_offset s1, -8
+; CHECK-RV32C-NEXT:    lbu a6, 0(a1)
+; CHECK-RV32C-NEXT:    lbu a7, 4(a1)
+; CHECK-RV32C-NEXT:    lbu t0, 8(a1)
+; CHECK-RV32C-NEXT:    lbu t1, 12(a1)
+; CHECK-RV32C-NEXT:    lbu t2, 16(a1)
+; CHECK-RV32C-NEXT:    lbu t3, 20(a1)
+; CHECK-RV32C-NEXT:    lbu t4, 24(a1)
+; CHECK-RV32C-NEXT:    lbu t5, 28(a1)
+; CHECK-RV32C-NEXT:    lbu t6, 32(a1)
+; CHECK-RV32C-NEXT:    lbu a3, 36(a1)
+; CHECK-RV32C-NEXT:    lbu a4, 40(a1)
+; CHECK-RV32C-NEXT:    lbu a5, 44(a1)
+; CHECK-RV32C-NEXT:    lbu a2, 60(a1)
+; CHECK-RV32C-NEXT:    lbu s0, 56(a1)
+; CHECK-RV32C-NEXT:    lbu s1, 52(a1)
+; CHECK-RV32C-NEXT:    lbu a1, 48(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a2, 15(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb s0, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb s1, 13(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a1, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a5, 11(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a3, 9(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t6, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t5, 7(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t4, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t3, 5(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t2, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t1, 3(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb t0, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a7, 1(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sb a6, 0(a0)
+; CHECK-RV32C-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32C-NEXT:    addi sp, sp, 16
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_v16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vse8.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <16 x i8> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_v8i16(ptr %p, <8 x i16> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v8i16:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lh a2, 0(a1)
+; CHECK-RV64-NEXT:    lh a3, 8(a1)
+; CHECK-RV64-NEXT:    lh a4, 16(a1)
+; CHECK-RV64-NEXT:    lh a5, 24(a1)
+; CHECK-RV64-NEXT:    lh a6, 56(a1)
+; CHECK-RV64-NEXT:    lh a7, 48(a1)
+; CHECK-RV64-NEXT:    lh t0, 40(a1)
+; CHECK-RV64-NEXT:    lh a1, 32(a1)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a6, 14(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a7, 12(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh t0, 10(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a1, 8(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a5, 6(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a3, 2(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sh a2, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v8i16:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lh a2, 0(a1)
+; CHECK-RV32-NEXT:    lh a3, 4(a1)
+; CHECK-RV32-NEXT:    lh a4, 8(a1)
+; CHECK-RV32-NEXT:    lh a5, 12(a1)
+; CHECK-RV32-NEXT:    lh a6, 28(a1)
+; CHECK-RV32-NEXT:    lh a7, 24(a1)
+; CHECK-RV32-NEXT:    lh t0, 20(a1)
+; CHECK-RV32-NEXT:    lh a1, 16(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a6, 14(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a7, 12(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh t0, 10(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a1, 8(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a5, 6(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a3, 2(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sh a2, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v8i16:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lh a6, 0(a1)
+; CHECK-RV64C-NEXT:    lh a7, 8(a1)
+; CHECK-RV64C-NEXT:    lh t0, 16(a1)
+; CHECK-RV64C-NEXT:    lh a5, 24(a1)
+; CHECK-RV64C-NEXT:    lh a2, 56(a1)
+; CHECK-RV64C-NEXT:    lh a3, 48(a1)
+; CHECK-RV64C-NEXT:    lh a4, 40(a1)
+; CHECK-RV64C-NEXT:    lh a1, 32(a1)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a2, 14(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a3, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a4, 10(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a1, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a5, 6(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh t0, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a7, 2(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sh a6, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v8i16:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lh a6, 0(a1)
+; CHECK-RV32C-NEXT:    lh a7, 4(a1)
+; CHECK-RV32C-NEXT:    lh t0, 8(a1)
+; CHECK-RV32C-NEXT:    lh a5, 12(a1)
+; CHECK-RV32C-NEXT:    lh a2, 28(a1)
+; CHECK-RV32C-NEXT:    lh a3, 24(a1)
+; CHECK-RV32C-NEXT:    lh a4, 20(a1)
+; CHECK-RV32C-NEXT:    lh a1, 16(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a2, 14(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a3, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a4, 10(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a1, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a5, 6(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh t0, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a7, 2(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sh a6, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_v8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vse16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <8 x i16> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_v4i32(ptr %p, <4 x i32> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v4i32:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    lw a2, 24(a1)
+; CHECK-RV64-NEXT:    lw a3, 16(a1)
+; CHECK-RV64-NEXT:    lw a4, 8(a1)
+; CHECK-RV64-NEXT:    lw a1, 0(a1)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sw a2, 12(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sw a3, 8(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sw a4, 4(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sw a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v4i32:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v4i32:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    lw a2, 24(a1)
+; CHECK-RV64C-NEXT:    lw a3, 16(a1)
+; CHECK-RV64C-NEXT:    lw a4, 8(a1)
+; CHECK-RV64C-NEXT:    lw a1, 0(a1)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sw a2, 12(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sw a3, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sw a4, 4(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sw a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v4i32:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_v4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <4 x i32> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+define void @test_nontemporal_ALL_store_v2i64(ptr %p, <2 x i64> %v) {
+; CHECK-RV64-LABEL: test_nontemporal_ALL_store_v2i64:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sd a2, 8(a0)
+; CHECK-RV64-NEXT:    ntl.all
+; CHECK-RV64-NEXT:    sd a1, 0(a0)
+; CHECK-RV64-NEXT:    ret
+;
+; CHECK-RV32-LABEL: test_nontemporal_ALL_store_v2i64:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 12(a1)
+; CHECK-RV32-NEXT:    lw a3, 8(a1)
+; CHECK-RV32-NEXT:    lw a4, 4(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a2, 12(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a3, 8(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a4, 4(a0)
+; CHECK-RV32-NEXT:    ntl.all
+; CHECK-RV32-NEXT:    sw a1, 0(a0)
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64C-LABEL: test_nontemporal_ALL_store_v2i64:
+; CHECK-RV64C:       # %bb.0:
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sd a2, 8(a0)
+; CHECK-RV64C-NEXT:    c.ntl.all
+; CHECK-RV64C-NEXT:    sd a1, 0(a0)
+; CHECK-RV64C-NEXT:    ret
+;
+; CHECK-RV32C-LABEL: test_nontemporal_ALL_store_v2i64:
+; CHECK-RV32C:       # %bb.0:
+; CHECK-RV32C-NEXT:    lw a2, 12(a1)
+; CHECK-RV32C-NEXT:    lw a3, 8(a1)
+; CHECK-RV32C-NEXT:    lw a4, 4(a1)
+; CHECK-RV32C-NEXT:    lw a1, 0(a1)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a2, 12(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a3, 8(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a4, 4(a0)
+; CHECK-RV32C-NEXT:    c.ntl.all
+; CHECK-RV32C-NEXT:    sw a1, 0(a0)
+; CHECK-RV32C-NEXT:    ret
+;
+; CHECK-RV64V-LABEL: test_nontemporal_ALL_store_v2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_ALL_store_v2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vse64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <2 x i64> %v, ptr %p, !nontemporal !0, !riscv-nontemporal-domain !4
+  ret void
+}
+
+
 !0 = !{i32 1}
+!1 = !{i32 2}
+!2 = !{i32 3}
+!3 = !{i32 4}
+!4 = !{i32 5}
diff --git a/llvm/test/CodeGen/RISCV/pr51206.ll b/llvm/test/CodeGen/RISCV/pr51206.ll
index 7ae250bbc050c..b83903e7c55cd 100644
--- a/llvm/test/CodeGen/RISCV/pr51206.ll
+++ b/llvm/test/CodeGen/RISCV/pr51206.ll
@@ -11,8 +11,6 @@
 define signext i32 @wobble() nounwind {
 ; CHECK-LABEL: wobble:
 ; CHECK:       # %bb.0: # %bb
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    lui a0, %hi(global)
 ; CHECK-NEXT:    lbu a0, %lo(global)(a0)
 ; CHECK-NEXT:    lui a1, %hi(global.2)
@@ -31,11 +29,13 @@ define signext i32 @wobble() nounwind {
 ; CHECK-NEXT:    sw a1, %lo(global.3)(a2)
 ; CHECK-NEXT:    bltu a0, a3, .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %bb10
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    call quux@plt
-; CHECK-NEXT:  .LBB0_2: # %bb12
-; CHECK-NEXT:    li a0, 0
 ; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:  .LBB0_2: # %bb12
+; CHECK-NEXT:    li a0, 0
 ; CHECK-NEXT:    ret
 bb:
   %tmp = load i8, ptr @global, align 1
diff --git a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
index dbc864c79238d..72d5454a3a694 100644
--- a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi=lp64f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64IF
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx -target-abi=lp64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64IZFINX
 
 ; This file exhaustively checks float<->i32 conversions. In general,
 ; fcvt.l[u].s can be selected instead of fcvt.w[u].s because poison is
@@ -12,6 +14,11 @@ define i32 @aext_fptosi(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: aext_fptosi:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %1 = fptosi float %a to i32
   ret i32 %1
 }
@@ -21,6 +28,11 @@ define signext i32 @sext_fptosi(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.w.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sext_fptosi:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %1 = fptosi float %a to i32
   ret i32 %1
 }
@@ -32,6 +44,13 @@ define zeroext i32 @zext_fptosi(float %a) nounwind {
 ; RV64IF-NEXT:    slli a0, a0, 32
 ; RV64IF-NEXT:    srli a0, a0, 32
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: zext_fptosi:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.w.s a0, a0, rtz
+; RV64IZFINX-NEXT:    slli a0, a0, 32
+; RV64IZFINX-NEXT:    srli a0, a0, 32
+; RV64IZFINX-NEXT:    ret
   %1 = fptosi float %a to i32
   ret i32 %1
 }
@@ -41,6 +60,11 @@ define i32 @aext_fptoui(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: aext_fptoui:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %1 = fptoui float %a to i32
   ret i32 %1
 }
@@ -50,6 +74,11 @@ define signext i32 @sext_fptoui(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.wu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sext_fptoui:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.wu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %1 = fptoui float %a to i32
   ret i32 %1
 }
@@ -59,6 +88,11 @@ define zeroext i32 @zext_fptoui(float %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.lu.s a0, fa0, rtz
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: zext_fptoui:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.lu.s a0, a0, rtz
+; RV64IZFINX-NEXT:    ret
   %1 = fptoui float %a to i32
   ret i32 %1
 }
@@ -69,6 +103,11 @@ define i32 @bcvt_f32_to_aext_i32(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    fadd.s fa5, fa0, fa1
 ; RV64IF-NEXT:    fmv.x.w a0, fa5
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: bcvt_f32_to_aext_i32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fadd.s a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = bitcast float %1 to i32
   ret i32 %2
@@ -80,6 +119,12 @@ define signext i32 @bcvt_f32_to_sext_i32(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    fadd.s fa5, fa0, fa1
 ; RV64IF-NEXT:    fmv.x.w a0, fa5
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: bcvt_f32_to_sext_i32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fadd.s a0, a0, a1
+; RV64IZFINX-NEXT:    sext.w a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = bitcast float %1 to i32
   ret i32 %2
@@ -93,6 +138,13 @@ define zeroext i32 @bcvt_f32_to_zext_i32(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    slli a0, a0, 32
 ; RV64IF-NEXT:    srli a0, a0, 32
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: bcvt_f32_to_zext_i32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fadd.s a0, a0, a1
+; RV64IZFINX-NEXT:    slli a0, a0, 32
+; RV64IZFINX-NEXT:    srli a0, a0, 32
+; RV64IZFINX-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = bitcast float %1 to i32
   ret i32 %2
@@ -105,6 +157,11 @@ define float @bcvt_i64_to_f32_via_i32(i64 %a, i64 %b) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa4, a1
 ; RV64IF-NEXT:    fadd.s fa0, fa5, fa4
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: bcvt_i64_to_f32_via_i32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fadd.s a0, a0, a1
+; RV64IZFINX-NEXT:    ret
   %1 = trunc i64 %a to i32
   %2 = trunc i64 %b to i32
   %3 = bitcast i32 %1 to float
@@ -118,6 +175,11 @@ define float @uitofp_aext_i32_to_f32(i32 %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.wu fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: uitofp_aext_i32_to_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = uitofp i32 %a to float
   ret float %1
 }
@@ -127,6 +189,11 @@ define float @uitofp_sext_i32_to_f32(i32 signext %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.wu fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: uitofp_sext_i32_to_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = uitofp i32 %a to float
   ret float %1
 }
@@ -136,6 +203,11 @@ define float @uitofp_zext_i32_to_f32(i32 zeroext %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.wu fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: uitofp_zext_i32_to_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.wu a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = uitofp i32 %a to float
   ret float %1
 }
@@ -145,6 +217,11 @@ define float @sitofp_aext_i32_to_f32(i32 %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.w fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sitofp_aext_i32_to_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.w a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = sitofp i32 %a to float
   ret float %1
 }
@@ -154,6 +231,11 @@ define float @sitofp_sext_i32_to_f32(i32 signext %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.w fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sitofp_sext_i32_to_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.w a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = sitofp i32 %a to float
   ret float %1
 }
@@ -163,6 +245,11 @@ define float @sitofp_zext_i32_to_f32(i32 zeroext %a) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fcvt.s.w fa0, a0
 ; RV64IF-NEXT:    ret
+;
+; RV64IZFINX-LABEL: sitofp_zext_i32_to_f32:
+; RV64IZFINX:       # %bb.0:
+; RV64IZFINX-NEXT:    fcvt.s.w a0, a0
+; RV64IZFINX-NEXT:    ret
   %1 = sitofp i32 %a to float
   ret float %1
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll b/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll
index 8089d0adedc94..b0efd4c6f8851 100644
--- a/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/constant-folding-crash.ll
@@ -28,11 +28,12 @@ define void @constant_folding_crash(ptr %v54, <4 x ptr> %lanes.a, <4 x ptr> %lan
 ; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vmerge.vvm v8, v9, v8, v0
 ; RV32-NEXT:    vmv.x.s a0, v8
-; RV32-NEXT:    vfirst.m a1, v10
-; RV32-NEXT:    seqz a1, a1
 ; RV32-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a1
-; RV32-NEXT:    vmsne.vi v0, v8, 0
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    vmv1r.v v0, v10
+; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    vrgather.vi v9, v8, 0
+; RV32-NEXT:    vmsne.vi v0, v9, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vmv.v.i v8, 10
 ; RV32-NEXT:    vse32.v v8, (a0), v0.t
@@ -50,11 +51,12 @@ define void @constant_folding_crash(ptr %v54, <4 x ptr> %lanes.a, <4 x ptr> %lan
 ; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
 ; RV64-NEXT:    vmerge.vvm v8, v10, v8, v0
 ; RV64-NEXT:    vmv.x.s a0, v8
-; RV64-NEXT:    vfirst.m a1, v12
-; RV64-NEXT:    seqz a1, a1
 ; RV64-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a1
-; RV64-NEXT:    vmsne.vi v0, v8, 0
+; RV64-NEXT:    vmv.v.i v8, 0
+; RV64-NEXT:    vmv1r.v v0, v12
+; RV64-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV64-NEXT:    vrgather.vi v9, v8, 0
+; RV64-NEXT:    vmsne.vi v0, v9, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV64-NEXT:    vmv.v.i v8, 10
 ; RV64-NEXT:    vse32.v v8, (a0), v0.t
diff --git a/llvm/test/CodeGen/RISCV/rvv/extractelt-i1.ll b/llvm/test/CodeGen/RISCV/rvv/extractelt-i1.ll
index 63fcc6ad9e426..2210ed4720c5a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extractelt-i1.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extractelt-i1.ll
@@ -10,7 +10,6 @@ define i1 @extractelt_nxv1i1(<vscale x 1 x i8>* %x, i64 %idx) nounwind {
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -28,7 +27,6 @@ define i1 @extractelt_nxv2i1(<vscale x 2 x i8>* %x, i64 %idx) nounwind {
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -46,7 +44,6 @@ define i1 @extractelt_nxv4i1(<vscale x 4 x i8>* %x, i64 %idx) nounwind {
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -64,7 +61,6 @@ define i1 @extractelt_nxv8i1(<vscale x 8 x i8>* %x, i64 %idx) nounwind {
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -221,7 +217,7 @@ define i1 @extractelt_nxv1i1_idx0(<vscale x 1 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 1 x i8>, <vscale x 1 x i8>* %x
   %b = icmp eq <vscale x 1 x i8> %a, zeroinitializer
@@ -236,7 +232,7 @@ define i1 @extractelt_nxv2i1_idx0(<vscale x 2 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 2 x i8>, <vscale x 2 x i8>* %x
   %b = icmp eq <vscale x 2 x i8> %a, zeroinitializer
@@ -251,7 +247,7 @@ define i1 @extractelt_nxv4i1_idx0(<vscale x 4 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 4 x i8>, <vscale x 4 x i8>* %x
   %b = icmp eq <vscale x 4 x i8> %a, zeroinitializer
@@ -266,7 +262,7 @@ define i1 @extractelt_nxv8i1_idx0(<vscale x 8 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 8 x i8>, <vscale x 8 x i8>* %x
   %b = icmp eq <vscale x 8 x i8> %a, zeroinitializer
@@ -281,7 +277,7 @@ define i1 @extractelt_nxv16i1_idx0(<vscale x 16 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmseq.vi v10, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v10
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 16 x i8>, <vscale x 16 x i8>* %x
   %b = icmp eq <vscale x 16 x i8> %a, zeroinitializer
@@ -296,7 +292,7 @@ define i1 @extractelt_nxv32i1_idx0(<vscale x 32 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vmseq.vi v12, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v12
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 32 x i8>, <vscale x 32 x i8>* %x
   %b = icmp eq <vscale x 32 x i8> %a, zeroinitializer
@@ -311,7 +307,7 @@ define i1 @extractelt_nxv64i1_idx0(<vscale x 64 x i8>* %x) nounwind {
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
 ; CHECK-NEXT:    vmseq.vi v16, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v16
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <vscale x 64 x i8>, <vscale x 64 x i8>* %x
   %b = icmp eq <vscale x 64 x i8> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll
new file mode 100644
index 0000000000000..b6d2764baeb78
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fceil-constrained-sdnode.ll
@@ -0,0 +1,335 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <vscale x 1 x half> @ceil_nxv1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: ceil_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x half> @llvm.experimental.constrained.ceil.nxv1f16(<vscale x 1 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x half> %a
+}
+declare <vscale x 1 x half> @llvm.experimental.constrained.ceil.nxv1f16(<vscale x 1 x half>, metadata)
+
+define <vscale x 2 x half> @ceil_nxv2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: ceil_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x half> @llvm.experimental.constrained.ceil.nxv2f16(<vscale x 2 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x half> %a
+}
+declare <vscale x 2 x half> @llvm.experimental.constrained.ceil.nxv2f16(<vscale x 2 x half>, metadata)
+
+define <vscale x 4 x half> @ceil_nxv4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: ceil_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x half> @llvm.experimental.constrained.ceil.nxv4f16(<vscale x 4 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x half> %a
+}
+declare <vscale x 4 x half> @llvm.experimental.constrained.ceil.nxv4f16(<vscale x 4 x half>, metadata)
+
+define <vscale x 8 x half> @ceil_nxv8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: ceil_nxv8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x half> @llvm.experimental.constrained.ceil.nxv8f16(<vscale x 8 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x half> %a
+}
+declare <vscale x 8 x half> @llvm.experimental.constrained.ceil.nxv8f16(<vscale x 8 x half>, metadata)
+
+define <vscale x 16 x half> @ceil_nxv16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: ceil_nxv16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x half> @llvm.experimental.constrained.ceil.nxv16f16(<vscale x 16 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x half> %a
+}
+declare <vscale x 16 x half> @llvm.experimental.constrained.ceil.nxv16f16(<vscale x 16 x half>, metadata)
+
+define <vscale x 32 x half> @ceil_nxv32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: ceil_nxv32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x half> @llvm.experimental.constrained.ceil.nxv32f16(<vscale x 32 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 32 x half> %a
+}
+declare <vscale x 32 x half> @llvm.experimental.constrained.ceil.nxv32f16(<vscale x 32 x half>, metadata)
+
+define <vscale x 1 x float> @ceil_nxv1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: ceil_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x float> @llvm.experimental.constrained.ceil.nxv1f32(<vscale x 1 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %a
+}
+declare <vscale x 1 x float> @llvm.experimental.constrained.ceil.nxv1f32(<vscale x 1 x float>, metadata)
+
+define <vscale x 2 x float> @ceil_nxv2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: ceil_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.ceil.nxv2f32(<vscale x 2 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %a
+}
+declare <vscale x 2 x float> @llvm.experimental.constrained.ceil.nxv2f32(<vscale x 2 x float>, metadata)
+
+define <vscale x 4 x float> @ceil_nxv4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: ceil_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x float> @llvm.experimental.constrained.ceil.nxv4f32(<vscale x 4 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %a
+}
+declare <vscale x 4 x float> @llvm.experimental.constrained.ceil.nxv4f32(<vscale x 4 x float>, metadata)
+
+define <vscale x 8 x float> @ceil_nxv8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: ceil_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x float> @llvm.experimental.constrained.ceil.nxv8f32(<vscale x 8 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %a
+}
+declare <vscale x 8 x float> @llvm.experimental.constrained.ceil.nxv8f32(<vscale x 8 x float>, metadata)
+
+define <vscale x 16 x float> @ceil_nxv16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: ceil_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x float> @llvm.experimental.constrained.ceil.nxv16f32(<vscale x 16 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x float> %a
+}
+declare <vscale x 16 x float> @llvm.experimental.constrained.ceil.nxv16f32(<vscale x 16 x float>, metadata)
+
+define <vscale x 1 x double> @ceil_nxv1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: ceil_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x double> @llvm.experimental.constrained.ceil.nxv1f64(<vscale x 1 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %a
+}
+declare <vscale x 1 x double> @llvm.experimental.constrained.ceil.nxv1f64(<vscale x 1 x double>, metadata)
+
+define <vscale x 2 x double> @ceil_nxv2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: ceil_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x double> @llvm.experimental.constrained.ceil.nxv2f64(<vscale x 2 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %a
+}
+declare <vscale x 2 x double> @llvm.experimental.constrained.ceil.nxv2f64(<vscale x 2 x double>, metadata)
+
+define <vscale x 4 x double> @ceil_nxv4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: ceil_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x double> @llvm.experimental.constrained.ceil.nxv4f64(<vscale x 4 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %a
+}
+declare <vscale x 4 x double> @llvm.experimental.constrained.ceil.nxv4f64(<vscale x 4 x double>, metadata)
+
+define <vscale x 8 x double> @ceil_nxv8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: ceil_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x double> @llvm.experimental.constrained.ceil.nxv8f64(<vscale x 8 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %a
+}
+declare <vscale x 8 x double> @llvm.experimental.constrained.ceil.nxv8f64(<vscale x 8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll
new file mode 100644
index 0000000000000..7610d918cfb0f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/ffloor-constrained-sdnode.ll
@@ -0,0 +1,335 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <vscale x 1 x half> @floor_nxv1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: floor_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x half> @llvm.experimental.constrained.floor.nxv1f16(<vscale x 1 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x half> %a
+}
+declare <vscale x 1 x half> @llvm.experimental.constrained.floor.nxv1f16(<vscale x 1 x half>, metadata)
+
+define <vscale x 2 x half> @floor_nxv2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: floor_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x half> @llvm.experimental.constrained.floor.nxv2f16(<vscale x 2 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x half> %a
+}
+declare <vscale x 2 x half> @llvm.experimental.constrained.floor.nxv2f16(<vscale x 2 x half>, metadata)
+
+define <vscale x 4 x half> @floor_nxv4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: floor_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x half> @llvm.experimental.constrained.floor.nxv4f16(<vscale x 4 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x half> %a
+}
+declare <vscale x 4 x half> @llvm.experimental.constrained.floor.nxv4f16(<vscale x 4 x half>, metadata)
+
+define <vscale x 8 x half> @floor_nxv8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: floor_nxv8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x half> @llvm.experimental.constrained.floor.nxv8f16(<vscale x 8 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x half> %a
+}
+declare <vscale x 8 x half> @llvm.experimental.constrained.floor.nxv8f16(<vscale x 8 x half>, metadata)
+
+define <vscale x 16 x half> @floor_nxv16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: floor_nxv16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x half> @llvm.experimental.constrained.floor.nxv16f16(<vscale x 16 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x half> %a
+}
+declare <vscale x 16 x half> @llvm.experimental.constrained.floor.nxv16f16(<vscale x 16 x half>, metadata)
+
+define <vscale x 32 x half> @floor_nxv32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: floor_nxv32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x half> @llvm.experimental.constrained.floor.nxv32f16(<vscale x 32 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 32 x half> %a
+}
+declare <vscale x 32 x half> @llvm.experimental.constrained.floor.nxv32f16(<vscale x 32 x half>, metadata)
+
+define <vscale x 1 x float> @floor_nxv1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: floor_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x float> @llvm.experimental.constrained.floor.nxv1f32(<vscale x 1 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %a
+}
+declare <vscale x 1 x float> @llvm.experimental.constrained.floor.nxv1f32(<vscale x 1 x float>, metadata)
+
+define <vscale x 2 x float> @floor_nxv2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: floor_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.floor.nxv2f32(<vscale x 2 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %a
+}
+declare <vscale x 2 x float> @llvm.experimental.constrained.floor.nxv2f32(<vscale x 2 x float>, metadata)
+
+define <vscale x 4 x float> @floor_nxv4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: floor_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x float> @llvm.experimental.constrained.floor.nxv4f32(<vscale x 4 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %a
+}
+declare <vscale x 4 x float> @llvm.experimental.constrained.floor.nxv4f32(<vscale x 4 x float>, metadata)
+
+define <vscale x 8 x float> @floor_nxv8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: floor_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x float> @llvm.experimental.constrained.floor.nxv8f32(<vscale x 8 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %a
+}
+declare <vscale x 8 x float> @llvm.experimental.constrained.floor.nxv8f32(<vscale x 8 x float>, metadata)
+
+define <vscale x 16 x float> @floor_nxv16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: floor_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x float> @llvm.experimental.constrained.floor.nxv16f32(<vscale x 16 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x float> %a
+}
+declare <vscale x 16 x float> @llvm.experimental.constrained.floor.nxv16f32(<vscale x 16 x float>, metadata)
+
+define <vscale x 1 x double> @floor_nxv1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: floor_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x double> @llvm.experimental.constrained.floor.nxv1f64(<vscale x 1 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %a
+}
+declare <vscale x 1 x double> @llvm.experimental.constrained.floor.nxv1f64(<vscale x 1 x double>, metadata)
+
+define <vscale x 2 x double> @floor_nxv2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: floor_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x double> @llvm.experimental.constrained.floor.nxv2f64(<vscale x 2 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %a
+}
+declare <vscale x 2 x double> @llvm.experimental.constrained.floor.nxv2f64(<vscale x 2 x double>, metadata)
+
+define <vscale x 4 x double> @floor_nxv4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: floor_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x double> @llvm.experimental.constrained.floor.nxv4f64(<vscale x 4 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %a
+}
+declare <vscale x 4 x double> @llvm.experimental.constrained.floor.nxv4f64(<vscale x 4 x double>, metadata)
+
+define <vscale x 8 x double> @floor_nxv8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: floor_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x double> @llvm.experimental.constrained.floor.nxv8f64(<vscale x 8 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %a
+}
+declare <vscale x 8 x double> @llvm.experimental.constrained.floor.nxv8f64(<vscale x 8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vector-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vector-shuffle-reverse.ll
index 2d609adaf0e4e..99d7b4963db76 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vector-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vector-shuffle-reverse.ll
@@ -16,18 +16,9 @@ define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 1
-; CHECK-NEXT:    vmv.x.s a0, v8
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vfirst.m a1, v0
-; CHECK-NEXT:    seqz a1, a1
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, tu, ma
-; CHECK-NEXT:    vmv.s.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vand.vi v8, v8, 1
-; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    vslidedown.vi v9, v8, 1
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
+; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
   %res = call <2 x i1> @llvm.experimental.vector.reverse.v2i1(<2 x i1> %a)
   ret <2 x i1> %res
@@ -36,2671 +27,78 @@ define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 define <4 x i1> @reverse_v4i1(<4 x i1> %a) {
 ; CHECK-LABEL: reverse_v4i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    addi a0, sp, 14
-; CHECK-NEXT:    vse8.v v9, (a0)
-; CHECK-NEXT:    vslidedown.vi v9, v8, 2
-; CHECK-NEXT:    addi a0, sp, 13
-; CHECK-NEXT:    vse8.v v9, (a0)
-; CHECK-NEXT:    vslidedown.vi v8, v8, 3
-; CHECK-NEXT:    addi a0, sp, 12
-; CHECK-NEXT:    vse8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vfirst.m a1, v0
-; CHECK-NEXT:    seqz a1, a1
-; CHECK-NEXT:    sb a1, 15(sp)
-; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vand.vi v8, v8, 1
-; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v9, v9, 3
+; CHECK-NEXT:    vrgather.vv v10, v8, v9
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
 ; CHECK-NEXT:    ret
   %res = call <4 x i1> @llvm.experimental.vector.reverse.v4i1(<4 x i1> %a)
   ret <4 x i1> %res
 }
 
 define <8 x i1> @reverse_v8i1(<8 x i1> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v8i1:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, -16
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 16
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vfirst.m a0, v0
-; RV32-BITS-UNKNOWN-NEXT:    seqz a0, a0
-; RV32-BITS-UNKNOWN-NEXT:    sb a0, 15(sp)
-; RV32-BITS-UNKNOWN-NEXT:    vmv.x.s a0, v0
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 30
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 14(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 29
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 13(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 28
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 12(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 27
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 11(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 26
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 10(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 25
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 9(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a0, a0, 24
-; RV32-BITS-UNKNOWN-NEXT:    srli a0, a0, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a0, 8(sp)
-; RV32-BITS-UNKNOWN-NEXT:    addi a0, sp, 8
-; RV32-BITS-UNKNOWN-NEXT:    vle8.v v8, (a0)
-; RV32-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, 16
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v8i1:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    addi sp, sp, -16
-; RV32-BITS-256-NEXT:    .cfi_def_cfa_offset 16
-; RV32-BITS-256-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-BITS-256-NEXT:    vfirst.m a0, v0
-; RV32-BITS-256-NEXT:    seqz a0, a0
-; RV32-BITS-256-NEXT:    sb a0, 15(sp)
-; RV32-BITS-256-NEXT:    vmv.x.s a0, v0
-; RV32-BITS-256-NEXT:    slli a1, a0, 30
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 14(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 29
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 13(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 28
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 12(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 27
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 11(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 26
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 10(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 25
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 9(sp)
-; RV32-BITS-256-NEXT:    slli a0, a0, 24
-; RV32-BITS-256-NEXT:    srli a0, a0, 31
-; RV32-BITS-256-NEXT:    sb a0, 8(sp)
-; RV32-BITS-256-NEXT:    addi a0, sp, 8
-; RV32-BITS-256-NEXT:    vle8.v v8, (a0)
-; RV32-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-256-NEXT:    addi sp, sp, 16
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v8i1:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    addi sp, sp, -16
-; RV32-BITS-512-NEXT:    .cfi_def_cfa_offset 16
-; RV32-BITS-512-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-BITS-512-NEXT:    vfirst.m a0, v0
-; RV32-BITS-512-NEXT:    seqz a0, a0
-; RV32-BITS-512-NEXT:    sb a0, 15(sp)
-; RV32-BITS-512-NEXT:    vmv.x.s a0, v0
-; RV32-BITS-512-NEXT:    slli a1, a0, 30
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 14(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 29
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 13(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 28
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 12(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 27
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 11(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 26
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 10(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 25
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 9(sp)
-; RV32-BITS-512-NEXT:    slli a0, a0, 24
-; RV32-BITS-512-NEXT:    srli a0, a0, 31
-; RV32-BITS-512-NEXT:    sb a0, 8(sp)
-; RV32-BITS-512-NEXT:    addi a0, sp, 8
-; RV32-BITS-512-NEXT:    vle8.v v8, (a0)
-; RV32-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-512-NEXT:    addi sp, sp, 16
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v8i1:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, -16
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 16
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vfirst.m a0, v0
-; RV64-BITS-UNKNOWN-NEXT:    seqz a0, a0
-; RV64-BITS-UNKNOWN-NEXT:    sb a0, 15(sp)
-; RV64-BITS-UNKNOWN-NEXT:    vmv.x.s a0, v0
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 62
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 14(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 61
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 13(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 60
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 12(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 59
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 11(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 58
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 10(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 57
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 9(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a0, a0, 56
-; RV64-BITS-UNKNOWN-NEXT:    srli a0, a0, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a0, 8(sp)
-; RV64-BITS-UNKNOWN-NEXT:    addi a0, sp, 8
-; RV64-BITS-UNKNOWN-NEXT:    vle8.v v8, (a0)
-; RV64-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, 16
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v8i1:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    addi sp, sp, -16
-; RV64-BITS-256-NEXT:    .cfi_def_cfa_offset 16
-; RV64-BITS-256-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-BITS-256-NEXT:    vfirst.m a0, v0
-; RV64-BITS-256-NEXT:    seqz a0, a0
-; RV64-BITS-256-NEXT:    sb a0, 15(sp)
-; RV64-BITS-256-NEXT:    vmv.x.s a0, v0
-; RV64-BITS-256-NEXT:    slli a1, a0, 62
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 14(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 61
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 13(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 60
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 12(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 59
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 11(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 58
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 10(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 57
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 9(sp)
-; RV64-BITS-256-NEXT:    slli a0, a0, 56
-; RV64-BITS-256-NEXT:    srli a0, a0, 63
-; RV64-BITS-256-NEXT:    sb a0, 8(sp)
-; RV64-BITS-256-NEXT:    addi a0, sp, 8
-; RV64-BITS-256-NEXT:    vle8.v v8, (a0)
-; RV64-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-256-NEXT:    addi sp, sp, 16
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v8i1:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    addi sp, sp, -16
-; RV64-BITS-512-NEXT:    .cfi_def_cfa_offset 16
-; RV64-BITS-512-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-BITS-512-NEXT:    vfirst.m a0, v0
-; RV64-BITS-512-NEXT:    seqz a0, a0
-; RV64-BITS-512-NEXT:    sb a0, 15(sp)
-; RV64-BITS-512-NEXT:    vmv.x.s a0, v0
-; RV64-BITS-512-NEXT:    slli a1, a0, 62
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 14(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 61
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 13(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 60
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 12(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 59
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 11(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 58
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 10(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 57
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 9(sp)
-; RV64-BITS-512-NEXT:    slli a0, a0, 56
-; RV64-BITS-512-NEXT:    srli a0, a0, 63
-; RV64-BITS-512-NEXT:    sb a0, 8(sp)
-; RV64-BITS-512-NEXT:    addi a0, sp, 8
-; RV64-BITS-512-NEXT:    vle8.v v8, (a0)
-; RV64-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-512-NEXT:    addi sp, sp, 16
-; RV64-BITS-512-NEXT:    ret
+; CHECK-LABEL: reverse_v8i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v9, v9, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v9
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    ret
   %res = call <8 x i1> @llvm.experimental.vector.reverse.v8i1(<8 x i1> %a)
   ret <8 x i1> %res
 }
 
 define <16 x i1> @reverse_v16i1(<16 x i1> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v16i1:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, -16
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 16
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vfirst.m a0, v0
-; RV32-BITS-UNKNOWN-NEXT:    seqz a0, a0
-; RV32-BITS-UNKNOWN-NEXT:    sb a0, 15(sp)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vmv.x.s a0, v0
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 30
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 14(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 29
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 13(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 28
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 12(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 27
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 11(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 26
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 10(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 25
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 9(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 24
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 8(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 23
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 7(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 22
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 6(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 21
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 5(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 20
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 4(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 19
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 3(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 18
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 2(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a0, 17
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 1(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a0, a0, 16
-; RV32-BITS-UNKNOWN-NEXT:    srli a0, a0, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a0, 0(sp)
-; RV32-BITS-UNKNOWN-NEXT:    mv a0, sp
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle8.v v8, (a0)
-; RV32-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, 16
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v16i1:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    addi sp, sp, -16
-; RV32-BITS-256-NEXT:    .cfi_def_cfa_offset 16
-; RV32-BITS-256-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-BITS-256-NEXT:    vfirst.m a0, v0
-; RV32-BITS-256-NEXT:    seqz a0, a0
-; RV32-BITS-256-NEXT:    sb a0, 15(sp)
-; RV32-BITS-256-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV32-BITS-256-NEXT:    vmv.x.s a0, v0
-; RV32-BITS-256-NEXT:    slli a1, a0, 30
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 14(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 29
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 13(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 28
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 12(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 27
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 11(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 26
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 10(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 25
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 9(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 24
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 8(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 23
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 7(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 22
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 6(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 21
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 5(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 20
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 4(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 19
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 3(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 18
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 2(sp)
-; RV32-BITS-256-NEXT:    slli a1, a0, 17
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 1(sp)
-; RV32-BITS-256-NEXT:    slli a0, a0, 16
-; RV32-BITS-256-NEXT:    srli a0, a0, 31
-; RV32-BITS-256-NEXT:    sb a0, 0(sp)
-; RV32-BITS-256-NEXT:    mv a0, sp
-; RV32-BITS-256-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-BITS-256-NEXT:    vle8.v v8, (a0)
-; RV32-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-256-NEXT:    addi sp, sp, 16
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v16i1:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    addi sp, sp, -16
-; RV32-BITS-512-NEXT:    .cfi_def_cfa_offset 16
-; RV32-BITS-512-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-BITS-512-NEXT:    vfirst.m a0, v0
-; RV32-BITS-512-NEXT:    seqz a0, a0
-; RV32-BITS-512-NEXT:    sb a0, 15(sp)
-; RV32-BITS-512-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV32-BITS-512-NEXT:    vmv.x.s a0, v0
-; RV32-BITS-512-NEXT:    slli a1, a0, 30
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 14(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 29
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 13(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 28
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 12(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 27
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 11(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 26
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 10(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 25
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 9(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 24
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 8(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 23
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 7(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 22
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 6(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 21
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 5(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 20
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 4(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 19
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 3(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 18
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 2(sp)
-; RV32-BITS-512-NEXT:    slli a1, a0, 17
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 1(sp)
-; RV32-BITS-512-NEXT:    slli a0, a0, 16
-; RV32-BITS-512-NEXT:    srli a0, a0, 31
-; RV32-BITS-512-NEXT:    sb a0, 0(sp)
-; RV32-BITS-512-NEXT:    mv a0, sp
-; RV32-BITS-512-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-BITS-512-NEXT:    vle8.v v8, (a0)
-; RV32-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-512-NEXT:    addi sp, sp, 16
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v16i1:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, -16
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 16
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vfirst.m a0, v0
-; RV64-BITS-UNKNOWN-NEXT:    seqz a0, a0
-; RV64-BITS-UNKNOWN-NEXT:    sb a0, 15(sp)
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vmv.x.s a0, v0
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 62
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 14(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 61
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 13(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 60
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 12(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 59
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 11(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 58
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 10(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 57
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 9(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 56
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 8(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 55
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 7(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 54
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 6(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 53
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 5(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 52
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 4(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 51
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 3(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 50
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 2(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a0, 49
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 1(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a0, a0, 48
-; RV64-BITS-UNKNOWN-NEXT:    srli a0, a0, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a0, 0(sp)
-; RV64-BITS-UNKNOWN-NEXT:    mv a0, sp
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vle8.v v8, (a0)
-; RV64-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, 16
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v16i1:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    addi sp, sp, -16
-; RV64-BITS-256-NEXT:    .cfi_def_cfa_offset 16
-; RV64-BITS-256-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-BITS-256-NEXT:    vfirst.m a0, v0
-; RV64-BITS-256-NEXT:    seqz a0, a0
-; RV64-BITS-256-NEXT:    sb a0, 15(sp)
-; RV64-BITS-256-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV64-BITS-256-NEXT:    vmv.x.s a0, v0
-; RV64-BITS-256-NEXT:    slli a1, a0, 62
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 14(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 61
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 13(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 60
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 12(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 59
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 11(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 58
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 10(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 57
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 9(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 56
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 8(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 55
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 7(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 54
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 6(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 53
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 5(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 52
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 4(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 51
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 3(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 50
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 2(sp)
-; RV64-BITS-256-NEXT:    slli a1, a0, 49
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 1(sp)
-; RV64-BITS-256-NEXT:    slli a0, a0, 48
-; RV64-BITS-256-NEXT:    srli a0, a0, 63
-; RV64-BITS-256-NEXT:    sb a0, 0(sp)
-; RV64-BITS-256-NEXT:    mv a0, sp
-; RV64-BITS-256-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-BITS-256-NEXT:    vle8.v v8, (a0)
-; RV64-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-256-NEXT:    addi sp, sp, 16
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v16i1:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    addi sp, sp, -16
-; RV64-BITS-512-NEXT:    .cfi_def_cfa_offset 16
-; RV64-BITS-512-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-BITS-512-NEXT:    vfirst.m a0, v0
-; RV64-BITS-512-NEXT:    seqz a0, a0
-; RV64-BITS-512-NEXT:    sb a0, 15(sp)
-; RV64-BITS-512-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV64-BITS-512-NEXT:    vmv.x.s a0, v0
-; RV64-BITS-512-NEXT:    slli a1, a0, 62
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 14(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 61
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 13(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 60
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 12(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 59
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 11(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 58
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 10(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 57
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 9(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 56
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 8(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 55
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 7(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 54
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 6(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 53
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 5(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 52
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 4(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 51
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 3(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 50
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 2(sp)
-; RV64-BITS-512-NEXT:    slli a1, a0, 49
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 1(sp)
-; RV64-BITS-512-NEXT:    slli a0, a0, 48
-; RV64-BITS-512-NEXT:    srli a0, a0, 63
-; RV64-BITS-512-NEXT:    sb a0, 0(sp)
-; RV64-BITS-512-NEXT:    mv a0, sp
-; RV64-BITS-512-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-BITS-512-NEXT:    vle8.v v8, (a0)
-; RV64-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-512-NEXT:    addi sp, sp, 16
-; RV64-BITS-512-NEXT:    ret
+; CHECK-LABEL: reverse_v16i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v9, v9, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v9
+; CHECK-NEXT:    vmsne.vi v0, v10, 0
+; CHECK-NEXT:    ret
   %res = call <16 x i1> @llvm.experimental.vector.reverse.v16i1(<16 x i1> %a)
   ret <16 x i1> %res
 }
 
 define <32 x i1> @reverse_v32i1(<32 x i1> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v32i1:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, -64
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 64
-; RV32-BITS-UNKNOWN-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32-BITS-UNKNOWN-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_offset ra, -4
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_offset s0, -8
-; RV32-BITS-UNKNOWN-NEXT:    addi s0, sp, 64
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_def_cfa s0, 0
-; RV32-BITS-UNKNOWN-NEXT:    andi sp, sp, -32
-; RV32-BITS-UNKNOWN-NEXT:    li a0, 32
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vfirst.m a1, v0
-; RV32-BITS-UNKNOWN-NEXT:    seqz a1, a1
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 31(sp)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 0, e32, mf2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vmv.x.s a1, v0
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 0(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 30
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 30(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 29
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 29(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 28
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 28(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 27
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 27(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 26
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 26(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 25
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 25(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 24
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 24(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 23
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 23(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 22
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 22(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 21
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 21(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 20
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 20(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 19
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 19(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 18
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 18(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 17
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 17(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 16
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 16(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 15
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 15(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 14
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 14(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 13
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 13(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 12
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 12(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 11
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 11(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 10
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 10(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 9
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 9(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 8
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 8(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 7
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 7(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 6
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 6(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 5
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 5(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 4
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 4(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 3
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 3(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 2
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 2(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a1, 1
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 1(sp)
-; RV32-BITS-UNKNOWN-NEXT:    mv a1, sp
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle8.v v8, (a1)
-; RV32-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, s0, -64
-; RV32-BITS-UNKNOWN-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32-BITS-UNKNOWN-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, 64
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v32i1:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    addi sp, sp, -64
-; RV32-BITS-256-NEXT:    .cfi_def_cfa_offset 64
-; RV32-BITS-256-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32-BITS-256-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32-BITS-256-NEXT:    .cfi_offset ra, -4
-; RV32-BITS-256-NEXT:    .cfi_offset s0, -8
-; RV32-BITS-256-NEXT:    addi s0, sp, 64
-; RV32-BITS-256-NEXT:    .cfi_def_cfa s0, 0
-; RV32-BITS-256-NEXT:    andi sp, sp, -32
-; RV32-BITS-256-NEXT:    li a0, 32
-; RV32-BITS-256-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-BITS-256-NEXT:    vfirst.m a1, v0
-; RV32-BITS-256-NEXT:    seqz a1, a1
-; RV32-BITS-256-NEXT:    sb a1, 31(sp)
-; RV32-BITS-256-NEXT:    vsetivli zero, 0, e32, mf2, ta, ma
-; RV32-BITS-256-NEXT:    vmv.x.s a1, v0
-; RV32-BITS-256-NEXT:    srli a2, a1, 31
-; RV32-BITS-256-NEXT:    sb a2, 0(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 30
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 30(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 29
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 29(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 28
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 28(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 27
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 27(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 26
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 26(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 25
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 25(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 24
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 24(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 23
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 23(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 22
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 22(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 21
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 21(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 20
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 20(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 19
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 19(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 18
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 18(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 17
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 17(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 16
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 16(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 15
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 15(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 14
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 14(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 13
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 13(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 12
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 12(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 11
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 11(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 10
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 10(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 9
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 9(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 8
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 8(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 7
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 7(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 6
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 6(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 5
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 5(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 4
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 4(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 3
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 3(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 2
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 2(sp)
-; RV32-BITS-256-NEXT:    slli a1, a1, 1
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 1(sp)
-; RV32-BITS-256-NEXT:    mv a1, sp
-; RV32-BITS-256-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-BITS-256-NEXT:    vle8.v v8, (a1)
-; RV32-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-256-NEXT:    addi sp, s0, -64
-; RV32-BITS-256-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32-BITS-256-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32-BITS-256-NEXT:    addi sp, sp, 64
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v32i1:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    addi sp, sp, -64
-; RV32-BITS-512-NEXT:    .cfi_def_cfa_offset 64
-; RV32-BITS-512-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32-BITS-512-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32-BITS-512-NEXT:    .cfi_offset ra, -4
-; RV32-BITS-512-NEXT:    .cfi_offset s0, -8
-; RV32-BITS-512-NEXT:    addi s0, sp, 64
-; RV32-BITS-512-NEXT:    .cfi_def_cfa s0, 0
-; RV32-BITS-512-NEXT:    andi sp, sp, -32
-; RV32-BITS-512-NEXT:    li a0, 32
-; RV32-BITS-512-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-BITS-512-NEXT:    vfirst.m a1, v0
-; RV32-BITS-512-NEXT:    seqz a1, a1
-; RV32-BITS-512-NEXT:    sb a1, 31(sp)
-; RV32-BITS-512-NEXT:    vsetivli zero, 0, e32, mf2, ta, ma
-; RV32-BITS-512-NEXT:    vmv.x.s a1, v0
-; RV32-BITS-512-NEXT:    srli a2, a1, 31
-; RV32-BITS-512-NEXT:    sb a2, 0(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 30
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 30(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 29
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 29(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 28
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 28(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 27
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 27(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 26
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 26(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 25
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 25(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 24
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 24(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 23
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 23(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 22
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 22(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 21
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 21(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 20
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 20(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 19
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 19(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 18
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 18(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 17
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 17(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 16
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 16(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 15
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 15(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 14
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 14(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 13
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 13(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 12
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 12(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 11
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 11(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 10
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 10(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 9
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 9(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 8
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 8(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 7
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 7(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 6
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 6(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 5
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 5(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 4
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 4(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 3
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 3(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 2
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 2(sp)
-; RV32-BITS-512-NEXT:    slli a1, a1, 1
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 1(sp)
-; RV32-BITS-512-NEXT:    mv a1, sp
-; RV32-BITS-512-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-BITS-512-NEXT:    vle8.v v8, (a1)
-; RV32-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-512-NEXT:    addi sp, s0, -64
-; RV32-BITS-512-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32-BITS-512-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32-BITS-512-NEXT:    addi sp, sp, 64
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v32i1:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, -64
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 64
-; RV64-BITS-UNKNOWN-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-BITS-UNKNOWN-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_offset ra, -8
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_offset s0, -16
-; RV64-BITS-UNKNOWN-NEXT:    addi s0, sp, 64
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_def_cfa s0, 0
-; RV64-BITS-UNKNOWN-NEXT:    andi sp, sp, -32
-; RV64-BITS-UNKNOWN-NEXT:    li a0, 32
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vfirst.m a1, v0
-; RV64-BITS-UNKNOWN-NEXT:    seqz a1, a1
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 31(sp)
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 0, e32, mf2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vmv.x.s a1, v0
-; RV64-BITS-UNKNOWN-NEXT:    srliw a2, a1, 31
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 0(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 62
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 30(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 61
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 29(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 60
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 28(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 59
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 27(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 58
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 26(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 57
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 25(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 56
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 24(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 55
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 23(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 54
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 22(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 53
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 21(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 52
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 20(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 51
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 19(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 50
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 18(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 49
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 17(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 48
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 16(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 47
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 15(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 46
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 14(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 45
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 13(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 44
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 12(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 43
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 11(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 42
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 10(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 41
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 9(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 40
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 8(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 39
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 7(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 38
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 6(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 37
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 5(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 36
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 4(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 35
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 3(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 34
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 2(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a1, 33
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 1(sp)
-; RV64-BITS-UNKNOWN-NEXT:    mv a1, sp
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vle8.v v8, (a1)
-; RV64-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, s0, -64
-; RV64-BITS-UNKNOWN-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-BITS-UNKNOWN-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, 64
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v32i1:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    addi sp, sp, -64
-; RV64-BITS-256-NEXT:    .cfi_def_cfa_offset 64
-; RV64-BITS-256-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-BITS-256-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-BITS-256-NEXT:    .cfi_offset ra, -8
-; RV64-BITS-256-NEXT:    .cfi_offset s0, -16
-; RV64-BITS-256-NEXT:    addi s0, sp, 64
-; RV64-BITS-256-NEXT:    .cfi_def_cfa s0, 0
-; RV64-BITS-256-NEXT:    andi sp, sp, -32
-; RV64-BITS-256-NEXT:    li a0, 32
-; RV64-BITS-256-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-BITS-256-NEXT:    vfirst.m a1, v0
-; RV64-BITS-256-NEXT:    seqz a1, a1
-; RV64-BITS-256-NEXT:    sb a1, 31(sp)
-; RV64-BITS-256-NEXT:    vsetivli zero, 0, e32, mf2, ta, ma
-; RV64-BITS-256-NEXT:    vmv.x.s a1, v0
-; RV64-BITS-256-NEXT:    srliw a2, a1, 31
-; RV64-BITS-256-NEXT:    sb a2, 0(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 62
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 30(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 61
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 29(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 60
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 28(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 59
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 27(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 58
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 26(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 57
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 25(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 56
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 24(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 55
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 23(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 54
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 22(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 53
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 21(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 52
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 20(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 51
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 19(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 50
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 18(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 49
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 17(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 48
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 16(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 47
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 15(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 46
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 14(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 45
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 13(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 44
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 12(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 43
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 11(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 42
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 10(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 41
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 9(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 40
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 8(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 39
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 7(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 38
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 6(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 37
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 5(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 36
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 4(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 35
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 3(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 34
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 2(sp)
-; RV64-BITS-256-NEXT:    slli a1, a1, 33
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 1(sp)
-; RV64-BITS-256-NEXT:    mv a1, sp
-; RV64-BITS-256-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-BITS-256-NEXT:    vle8.v v8, (a1)
-; RV64-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-256-NEXT:    addi sp, s0, -64
-; RV64-BITS-256-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-BITS-256-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-BITS-256-NEXT:    addi sp, sp, 64
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v32i1:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    addi sp, sp, -64
-; RV64-BITS-512-NEXT:    .cfi_def_cfa_offset 64
-; RV64-BITS-512-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-BITS-512-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-BITS-512-NEXT:    .cfi_offset ra, -8
-; RV64-BITS-512-NEXT:    .cfi_offset s0, -16
-; RV64-BITS-512-NEXT:    addi s0, sp, 64
-; RV64-BITS-512-NEXT:    .cfi_def_cfa s0, 0
-; RV64-BITS-512-NEXT:    andi sp, sp, -32
-; RV64-BITS-512-NEXT:    li a0, 32
-; RV64-BITS-512-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-BITS-512-NEXT:    vfirst.m a1, v0
-; RV64-BITS-512-NEXT:    seqz a1, a1
-; RV64-BITS-512-NEXT:    sb a1, 31(sp)
-; RV64-BITS-512-NEXT:    vsetivli zero, 0, e32, mf2, ta, ma
-; RV64-BITS-512-NEXT:    vmv.x.s a1, v0
-; RV64-BITS-512-NEXT:    srliw a2, a1, 31
-; RV64-BITS-512-NEXT:    sb a2, 0(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 62
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 30(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 61
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 29(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 60
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 28(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 59
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 27(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 58
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 26(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 57
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 25(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 56
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 24(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 55
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 23(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 54
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 22(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 53
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 21(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 52
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 20(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 51
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 19(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 50
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 18(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 49
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 17(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 48
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 16(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 47
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 15(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 46
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 14(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 45
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 13(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 44
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 12(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 43
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 11(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 42
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 10(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 41
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 9(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 40
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 8(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 39
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 7(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 38
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 6(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 37
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 5(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 36
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 4(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 35
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 3(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 34
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 2(sp)
-; RV64-BITS-512-NEXT:    slli a1, a1, 33
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 1(sp)
-; RV64-BITS-512-NEXT:    mv a1, sp
-; RV64-BITS-512-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-BITS-512-NEXT:    vle8.v v8, (a1)
-; RV64-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-512-NEXT:    addi sp, s0, -64
-; RV64-BITS-512-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-BITS-512-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-BITS-512-NEXT:    addi sp, sp, 64
-; RV64-BITS-512-NEXT:    ret
+; CHECK-LABEL: reverse_v32i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI4_0)
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vmerge.vim v10, v10, 1, v0
+; CHECK-NEXT:    vrgather.vv v12, v10, v8
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    ret
   %res = call <32 x i1> @llvm.experimental.vector.reverse.v32i1(<32 x i1> %a)
   ret <32 x i1> %res
 }
 
 define <64 x i1> @reverse_v64i1(<64 x i1> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v64i1:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, -128
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 128
-; RV32-BITS-UNKNOWN-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
-; RV32-BITS-UNKNOWN-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_offset ra, -4
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_offset s0, -8
-; RV32-BITS-UNKNOWN-NEXT:    addi s0, sp, 128
-; RV32-BITS-UNKNOWN-NEXT:    .cfi_def_cfa s0, 0
-; RV32-BITS-UNKNOWN-NEXT:    andi sp, sp, -64
-; RV32-BITS-UNKNOWN-NEXT:    li a0, 64
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vfirst.m a1, v0
-; RV32-BITS-UNKNOWN-NEXT:    seqz a1, a1
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 63(sp)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vmv.x.s a1, v0
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 32(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 30
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 62(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 29
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 61(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 28
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 60(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 27
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 59(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 26
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 58(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 25
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 57(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 24
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 56(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 23
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 55(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 22
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 54(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 21
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 53(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 20
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 52(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 19
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 51(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 18
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 50(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 17
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 49(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 16
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 48(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 15
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 47(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 14
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 46(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 13
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 45(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 12
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 44(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 11
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 43(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 10
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 42(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 9
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 41(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 8
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 40(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 7
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 39(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 6
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 38(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 5
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 37(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 4
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 36(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 3
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 35(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 2
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 34(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a1, 1
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 33(sp)
-; RV32-BITS-UNKNOWN-NEXT:    vslidedown.vi v8, v0, 1
-; RV32-BITS-UNKNOWN-NEXT:    vmv.x.s a1, v8
-; RV32-BITS-UNKNOWN-NEXT:    andi a2, a1, 1
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 31(sp)
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 0(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 30
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 30(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 29
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 29(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 28
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 28(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 27
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 27(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 26
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 26(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 25
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 25(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 24
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 24(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 23
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 23(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 22
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 22(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 21
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 21(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 20
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 20(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 19
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 19(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 18
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 18(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 17
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 17(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 16
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 16(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 15
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 15(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 14
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 14(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 13
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 13(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 12
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 12(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 11
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 11(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 10
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 10(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 9
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 9(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 8
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 8(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 7
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 7(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 6
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 6(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 5
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 5(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 4
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 4(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 3
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 3(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a2, a1, 2
-; RV32-BITS-UNKNOWN-NEXT:    srli a2, a2, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a2, 2(sp)
-; RV32-BITS-UNKNOWN-NEXT:    slli a1, a1, 1
-; RV32-BITS-UNKNOWN-NEXT:    srli a1, a1, 31
-; RV32-BITS-UNKNOWN-NEXT:    sb a1, 1(sp)
-; RV32-BITS-UNKNOWN-NEXT:    mv a1, sp
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle8.v v8, (a1)
-; RV32-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, s0, -128
-; RV32-BITS-UNKNOWN-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
-; RV32-BITS-UNKNOWN-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
-; RV32-BITS-UNKNOWN-NEXT:    addi sp, sp, 128
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v64i1:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    addi sp, sp, -128
-; RV32-BITS-256-NEXT:    .cfi_def_cfa_offset 128
-; RV32-BITS-256-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
-; RV32-BITS-256-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
-; RV32-BITS-256-NEXT:    .cfi_offset ra, -4
-; RV32-BITS-256-NEXT:    .cfi_offset s0, -8
-; RV32-BITS-256-NEXT:    addi s0, sp, 128
-; RV32-BITS-256-NEXT:    .cfi_def_cfa s0, 0
-; RV32-BITS-256-NEXT:    andi sp, sp, -64
-; RV32-BITS-256-NEXT:    li a0, 64
-; RV32-BITS-256-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV32-BITS-256-NEXT:    vfirst.m a1, v0
-; RV32-BITS-256-NEXT:    seqz a1, a1
-; RV32-BITS-256-NEXT:    sb a1, 63(sp)
-; RV32-BITS-256-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV32-BITS-256-NEXT:    vmv.x.s a1, v0
-; RV32-BITS-256-NEXT:    srli a2, a1, 31
-; RV32-BITS-256-NEXT:    sb a2, 32(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 30
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 62(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 29
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 61(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 28
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 60(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 27
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 59(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 26
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 58(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 25
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 57(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 24
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 56(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 23
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 55(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 22
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 54(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 21
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 53(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 20
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 52(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 19
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 51(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 18
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 50(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 17
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 49(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 16
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 48(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 15
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 47(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 14
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 46(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 13
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 45(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 12
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 44(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 11
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 43(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 10
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 42(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 9
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 41(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 8
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 40(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 7
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 39(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 6
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 38(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 5
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 37(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 4
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 36(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 3
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 35(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 2
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 34(sp)
-; RV32-BITS-256-NEXT:    slli a1, a1, 1
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 33(sp)
-; RV32-BITS-256-NEXT:    vslidedown.vi v8, v0, 1
-; RV32-BITS-256-NEXT:    vmv.x.s a1, v8
-; RV32-BITS-256-NEXT:    andi a2, a1, 1
-; RV32-BITS-256-NEXT:    sb a2, 31(sp)
-; RV32-BITS-256-NEXT:    srli a2, a1, 31
-; RV32-BITS-256-NEXT:    sb a2, 0(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 30
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 30(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 29
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 29(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 28
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 28(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 27
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 27(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 26
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 26(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 25
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 25(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 24
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 24(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 23
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 23(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 22
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 22(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 21
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 21(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 20
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 20(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 19
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 19(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 18
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 18(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 17
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 17(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 16
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 16(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 15
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 15(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 14
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 14(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 13
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 13(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 12
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 12(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 11
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 11(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 10
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 10(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 9
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 9(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 8
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 8(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 7
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 7(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 6
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 6(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 5
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 5(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 4
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 4(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 3
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 3(sp)
-; RV32-BITS-256-NEXT:    slli a2, a1, 2
-; RV32-BITS-256-NEXT:    srli a2, a2, 31
-; RV32-BITS-256-NEXT:    sb a2, 2(sp)
-; RV32-BITS-256-NEXT:    slli a1, a1, 1
-; RV32-BITS-256-NEXT:    srli a1, a1, 31
-; RV32-BITS-256-NEXT:    sb a1, 1(sp)
-; RV32-BITS-256-NEXT:    mv a1, sp
-; RV32-BITS-256-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV32-BITS-256-NEXT:    vle8.v v8, (a1)
-; RV32-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-256-NEXT:    addi sp, s0, -128
-; RV32-BITS-256-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
-; RV32-BITS-256-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
-; RV32-BITS-256-NEXT:    addi sp, sp, 128
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v64i1:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    addi sp, sp, -128
-; RV32-BITS-512-NEXT:    .cfi_def_cfa_offset 128
-; RV32-BITS-512-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
-; RV32-BITS-512-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
-; RV32-BITS-512-NEXT:    .cfi_offset ra, -4
-; RV32-BITS-512-NEXT:    .cfi_offset s0, -8
-; RV32-BITS-512-NEXT:    addi s0, sp, 128
-; RV32-BITS-512-NEXT:    .cfi_def_cfa s0, 0
-; RV32-BITS-512-NEXT:    andi sp, sp, -64
-; RV32-BITS-512-NEXT:    li a0, 64
-; RV32-BITS-512-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV32-BITS-512-NEXT:    vfirst.m a1, v0
-; RV32-BITS-512-NEXT:    seqz a1, a1
-; RV32-BITS-512-NEXT:    sb a1, 63(sp)
-; RV32-BITS-512-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV32-BITS-512-NEXT:    vmv.x.s a1, v0
-; RV32-BITS-512-NEXT:    srli a2, a1, 31
-; RV32-BITS-512-NEXT:    sb a2, 32(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 30
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 62(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 29
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 61(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 28
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 60(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 27
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 59(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 26
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 58(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 25
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 57(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 24
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 56(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 23
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 55(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 22
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 54(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 21
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 53(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 20
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 52(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 19
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 51(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 18
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 50(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 17
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 49(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 16
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 48(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 15
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 47(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 14
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 46(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 13
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 45(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 12
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 44(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 11
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 43(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 10
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 42(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 9
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 41(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 8
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 40(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 7
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 39(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 6
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 38(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 5
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 37(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 4
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 36(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 3
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 35(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 2
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 34(sp)
-; RV32-BITS-512-NEXT:    slli a1, a1, 1
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 33(sp)
-; RV32-BITS-512-NEXT:    vslidedown.vi v8, v0, 1
-; RV32-BITS-512-NEXT:    vmv.x.s a1, v8
-; RV32-BITS-512-NEXT:    andi a2, a1, 1
-; RV32-BITS-512-NEXT:    sb a2, 31(sp)
-; RV32-BITS-512-NEXT:    srli a2, a1, 31
-; RV32-BITS-512-NEXT:    sb a2, 0(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 30
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 30(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 29
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 29(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 28
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 28(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 27
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 27(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 26
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 26(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 25
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 25(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 24
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 24(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 23
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 23(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 22
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 22(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 21
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 21(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 20
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 20(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 19
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 19(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 18
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 18(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 17
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 17(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 16
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 16(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 15
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 15(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 14
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 14(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 13
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 13(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 12
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 12(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 11
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 11(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 10
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 10(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 9
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 9(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 8
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 8(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 7
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 7(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 6
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 6(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 5
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 5(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 4
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 4(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 3
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 3(sp)
-; RV32-BITS-512-NEXT:    slli a2, a1, 2
-; RV32-BITS-512-NEXT:    srli a2, a2, 31
-; RV32-BITS-512-NEXT:    sb a2, 2(sp)
-; RV32-BITS-512-NEXT:    slli a1, a1, 1
-; RV32-BITS-512-NEXT:    srli a1, a1, 31
-; RV32-BITS-512-NEXT:    sb a1, 1(sp)
-; RV32-BITS-512-NEXT:    mv a1, sp
-; RV32-BITS-512-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV32-BITS-512-NEXT:    vle8.v v8, (a1)
-; RV32-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV32-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV32-BITS-512-NEXT:    addi sp, s0, -128
-; RV32-BITS-512-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
-; RV32-BITS-512-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
-; RV32-BITS-512-NEXT:    addi sp, sp, 128
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v64i1:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, -128
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_def_cfa_offset 128
-; RV64-BITS-UNKNOWN-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-BITS-UNKNOWN-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_offset ra, -8
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_offset s0, -16
-; RV64-BITS-UNKNOWN-NEXT:    addi s0, sp, 128
-; RV64-BITS-UNKNOWN-NEXT:    .cfi_def_cfa s0, 0
-; RV64-BITS-UNKNOWN-NEXT:    andi sp, sp, -64
-; RV64-BITS-UNKNOWN-NEXT:    li a0, 64
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vfirst.m a1, v0
-; RV64-BITS-UNKNOWN-NEXT:    seqz a1, a1
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 63(sp)
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 0, e64, m1, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vmv.x.s a1, v0
-; RV64-BITS-UNKNOWN-NEXT:    srliw a2, a1, 31
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 32(sp)
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 0(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 62
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 62(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 61
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 61(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 60
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 60(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 59
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 59(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 58
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 58(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 57
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 57(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 56
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 56(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 55
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 55(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 54
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 54(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 53
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 53(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 52
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 52(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 51
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 51(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 50
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 50(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 49
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 49(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 48
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 48(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 47
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 47(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 46
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 46(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 45
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 45(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 44
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 44(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 43
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 43(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 42
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 42(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 41
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 41(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 40
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 40(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 39
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 39(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 38
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 38(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 37
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 37(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 36
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 36(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 35
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 35(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 34
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 34(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 33
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 33(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 31
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 31(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 30
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 30(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 29
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 29(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 28
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 28(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 27
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 27(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 26
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 26(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 25
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 25(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 24
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 24(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 23
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 23(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 22
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 22(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 21
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 21(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 20
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 20(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 19
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 19(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 18
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 18(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 17
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 17(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 16
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 16(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 15
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 15(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 14
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 14(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 13
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 13(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 12
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 12(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 11
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 11(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 10
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 10(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 9
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 9(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 8
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 8(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 7
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 7(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 6
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 6(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 5
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 5(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 4
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 4(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 3
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 3(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a2, a1, 2
-; RV64-BITS-UNKNOWN-NEXT:    srli a2, a2, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a2, 2(sp)
-; RV64-BITS-UNKNOWN-NEXT:    slli a1, a1, 1
-; RV64-BITS-UNKNOWN-NEXT:    srli a1, a1, 63
-; RV64-BITS-UNKNOWN-NEXT:    sb a1, 1(sp)
-; RV64-BITS-UNKNOWN-NEXT:    mv a1, sp
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vle8.v v8, (a1)
-; RV64-BITS-UNKNOWN-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-UNKNOWN-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, s0, -128
-; RV64-BITS-UNKNOWN-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-BITS-UNKNOWN-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-BITS-UNKNOWN-NEXT:    addi sp, sp, 128
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v64i1:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    addi sp, sp, -128
-; RV64-BITS-256-NEXT:    .cfi_def_cfa_offset 128
-; RV64-BITS-256-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-BITS-256-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-BITS-256-NEXT:    .cfi_offset ra, -8
-; RV64-BITS-256-NEXT:    .cfi_offset s0, -16
-; RV64-BITS-256-NEXT:    addi s0, sp, 128
-; RV64-BITS-256-NEXT:    .cfi_def_cfa s0, 0
-; RV64-BITS-256-NEXT:    andi sp, sp, -64
-; RV64-BITS-256-NEXT:    li a0, 64
-; RV64-BITS-256-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV64-BITS-256-NEXT:    vfirst.m a1, v0
-; RV64-BITS-256-NEXT:    seqz a1, a1
-; RV64-BITS-256-NEXT:    sb a1, 63(sp)
-; RV64-BITS-256-NEXT:    vsetivli zero, 0, e64, m1, ta, ma
-; RV64-BITS-256-NEXT:    vmv.x.s a1, v0
-; RV64-BITS-256-NEXT:    srliw a2, a1, 31
-; RV64-BITS-256-NEXT:    sb a2, 32(sp)
-; RV64-BITS-256-NEXT:    srli a2, a1, 63
-; RV64-BITS-256-NEXT:    sb a2, 0(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 62
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 62(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 61
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 61(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 60
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 60(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 59
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 59(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 58
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 58(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 57
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 57(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 56
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 56(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 55
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 55(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 54
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 54(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 53
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 53(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 52
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 52(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 51
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 51(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 50
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 50(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 49
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 49(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 48
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 48(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 47
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 47(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 46
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 46(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 45
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 45(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 44
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 44(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 43
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 43(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 42
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 42(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 41
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 41(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 40
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 40(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 39
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 39(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 38
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 38(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 37
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 37(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 36
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 36(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 35
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 35(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 34
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 34(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 33
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 33(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 31
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 31(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 30
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 30(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 29
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 29(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 28
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 28(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 27
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 27(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 26
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 26(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 25
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 25(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 24
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 24(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 23
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 23(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 22
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 22(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 21
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 21(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 20
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 20(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 19
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 19(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 18
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 18(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 17
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 17(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 16
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 16(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 15
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 15(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 14
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 14(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 13
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 13(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 12
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 12(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 11
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 11(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 10
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 10(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 9
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 9(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 8
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 8(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 7
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 7(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 6
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 6(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 5
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 5(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 4
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 4(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 3
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 3(sp)
-; RV64-BITS-256-NEXT:    slli a2, a1, 2
-; RV64-BITS-256-NEXT:    srli a2, a2, 63
-; RV64-BITS-256-NEXT:    sb a2, 2(sp)
-; RV64-BITS-256-NEXT:    slli a1, a1, 1
-; RV64-BITS-256-NEXT:    srli a1, a1, 63
-; RV64-BITS-256-NEXT:    sb a1, 1(sp)
-; RV64-BITS-256-NEXT:    mv a1, sp
-; RV64-BITS-256-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV64-BITS-256-NEXT:    vle8.v v8, (a1)
-; RV64-BITS-256-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-256-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-256-NEXT:    addi sp, s0, -128
-; RV64-BITS-256-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-BITS-256-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-BITS-256-NEXT:    addi sp, sp, 128
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v64i1:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    addi sp, sp, -128
-; RV64-BITS-512-NEXT:    .cfi_def_cfa_offset 128
-; RV64-BITS-512-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-BITS-512-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-BITS-512-NEXT:    .cfi_offset ra, -8
-; RV64-BITS-512-NEXT:    .cfi_offset s0, -16
-; RV64-BITS-512-NEXT:    addi s0, sp, 128
-; RV64-BITS-512-NEXT:    .cfi_def_cfa s0, 0
-; RV64-BITS-512-NEXT:    andi sp, sp, -64
-; RV64-BITS-512-NEXT:    li a0, 64
-; RV64-BITS-512-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV64-BITS-512-NEXT:    vfirst.m a1, v0
-; RV64-BITS-512-NEXT:    seqz a1, a1
-; RV64-BITS-512-NEXT:    sb a1, 63(sp)
-; RV64-BITS-512-NEXT:    vsetivli zero, 0, e64, m1, ta, ma
-; RV64-BITS-512-NEXT:    vmv.x.s a1, v0
-; RV64-BITS-512-NEXT:    srliw a2, a1, 31
-; RV64-BITS-512-NEXT:    sb a2, 32(sp)
-; RV64-BITS-512-NEXT:    srli a2, a1, 63
-; RV64-BITS-512-NEXT:    sb a2, 0(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 62
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 62(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 61
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 61(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 60
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 60(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 59
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 59(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 58
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 58(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 57
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 57(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 56
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 56(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 55
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 55(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 54
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 54(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 53
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 53(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 52
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 52(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 51
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 51(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 50
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 50(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 49
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 49(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 48
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 48(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 47
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 47(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 46
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 46(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 45
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 45(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 44
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 44(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 43
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 43(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 42
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 42(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 41
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 41(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 40
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 40(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 39
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 39(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 38
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 38(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 37
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 37(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 36
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 36(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 35
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 35(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 34
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 34(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 33
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 33(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 31
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 31(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 30
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 30(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 29
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 29(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 28
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 28(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 27
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 27(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 26
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 26(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 25
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 25(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 24
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 24(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 23
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 23(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 22
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 22(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 21
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 21(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 20
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 20(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 19
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 19(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 18
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 18(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 17
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 17(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 16
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 16(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 15
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 15(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 14
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 14(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 13
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 13(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 12
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 12(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 11
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 11(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 10
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 10(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 9
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 9(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 8
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 8(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 7
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 7(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 6
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 6(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 5
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 5(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 4
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 4(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 3
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 3(sp)
-; RV64-BITS-512-NEXT:    slli a2, a1, 2
-; RV64-BITS-512-NEXT:    srli a2, a2, 63
-; RV64-BITS-512-NEXT:    sb a2, 2(sp)
-; RV64-BITS-512-NEXT:    slli a1, a1, 1
-; RV64-BITS-512-NEXT:    srli a1, a1, 63
-; RV64-BITS-512-NEXT:    sb a1, 1(sp)
-; RV64-BITS-512-NEXT:    mv a1, sp
-; RV64-BITS-512-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; RV64-BITS-512-NEXT:    vle8.v v8, (a1)
-; RV64-BITS-512-NEXT:    vand.vi v8, v8, 1
-; RV64-BITS-512-NEXT:    vmsne.vi v0, v8, 0
-; RV64-BITS-512-NEXT:    addi sp, s0, -128
-; RV64-BITS-512-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-BITS-512-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-BITS-512-NEXT:    addi sp, sp, 128
-; RV64-BITS-512-NEXT:    ret
+; CHECK-LABEL: reverse_v64i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    addi a0, a0, %lo(.LCPI5_0)
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vmv.v.i v12, 0
+; CHECK-NEXT:    vmerge.vim v12, v12, 1, v0
+; CHECK-NEXT:    vrgather.vv v16, v12, v8
+; CHECK-NEXT:    vmsne.vi v0, v16, 0
+; CHECK-NEXT:    ret
   %res = call <64 x i1> @llvm.experimental.vector.reverse.v64i1(<64 x i1> %a)
   ret <64 x i1> %res
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abs.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abs.ll
index 1b1d869fd1237..29d48fb3f41a6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abs.ll
@@ -36,6 +36,41 @@ define void @abs_v8i16(ptr %x) {
 }
 declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1)
 
+define void @abs_v6i16(ptr %x) {
+; LMULMAX1-RV32-LABEL: abs_v6i16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vrsub.vi v9, v8, 0
+; LMULMAX1-RV32-NEXT:    vmax.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: abs_v6i16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vrsub.vi v9, v8, 0
+; LMULMAX1-RV64-NEXT:    vmax.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = call <6 x i16> @llvm.abs.v6i16(<6 x i16> %a, i1 false)
+  store <6 x i16> %b, ptr %x
+  ret void
+}
+declare <6 x i16> @llvm.abs.v6i16(<6 x i16>, i1)
+
 define void @abs_v4i32(ptr %x) {
 ; CHECK-LABEL: abs_v4i32:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
index 6dcf6ee78e76e..853828253b1d9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll
@@ -521,9 +521,8 @@ define <4 x i16> @bitcast_i64_v4i16(i64 %a) {
 ; RV32ELEN32-LABEL: bitcast_i64_v4i16:
 ; RV32ELEN32:       # %bb.0:
 ; RV32ELEN32-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV32ELEN32-NEXT:    vmv.v.x v8, a1
-; RV32ELEN32-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
-; RV32ELEN32-NEXT:    vmv.s.x v8, a0
+; RV32ELEN32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32ELEN32-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32ELEN32-NEXT:    ret
 ;
 ; RV64ELEN32-LABEL: bitcast_i64_v4i16:
@@ -557,9 +556,8 @@ define <2 x i32> @bitcast_i64_v2i32(i64 %a) {
 ; RV32ELEN32-LABEL: bitcast_i64_v2i32:
 ; RV32ELEN32:       # %bb.0:
 ; RV32ELEN32-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV32ELEN32-NEXT:    vmv.v.x v8, a1
-; RV32ELEN32-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
-; RV32ELEN32-NEXT:    vmv.s.x v8, a0
+; RV32ELEN32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32ELEN32-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32ELEN32-NEXT:    ret
 ;
 ; RV64ELEN32-LABEL: bitcast_i64_v2i32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-i1.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-i1.ll
index 47add40335931..2ee0e44253d63 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-i1.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-i1.ll
@@ -30,7 +30,6 @@ define i1 @extractelt_v2i1(ptr %x, i64 %idx) nounwind {
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -48,7 +47,6 @@ define i1 @extractelt_v4i1(ptr %x, i64 %idx) nounwind {
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -455,7 +453,7 @@ define i1 @extractelt_v1i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <1 x i8>, ptr %x
   %b = icmp eq <1 x i8> %a, zeroinitializer
@@ -470,7 +468,7 @@ define i1 @extractelt_v2i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <2 x i8>, ptr %x
   %b = icmp eq <2 x i8> %a, zeroinitializer
@@ -485,7 +483,7 @@ define i1 @extractelt_v4i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <4 x i8>, ptr %x
   %b = icmp eq <4 x i8> %a, zeroinitializer
@@ -500,7 +498,7 @@ define i1 @extractelt_v8i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <8 x i8>, ptr %x
   %b = icmp eq <8 x i8> %a, zeroinitializer
@@ -515,7 +513,7 @@ define i1 @extractelt_v16i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v8, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v8
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <16 x i8>, ptr %x
   %b = icmp eq <16 x i8> %a, zeroinitializer
@@ -531,7 +529,7 @@ define i1 @extractelt_v32i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v10, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v10
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <32 x i8>, ptr %x
   %b = icmp eq <32 x i8> %a, zeroinitializer
@@ -547,7 +545,7 @@ define i1 @extractelt_v64i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v12, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v12
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <64 x i8>, ptr %x
   %b = icmp eq <64 x i8> %a, zeroinitializer
@@ -563,7 +561,7 @@ define i1 @extractelt_v128i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v16, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v16
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <128 x i8>, ptr %x
   %b = icmp eq <128 x i8> %a, zeroinitializer
@@ -579,7 +577,7 @@ define i1 @extractelt_v256i1_idx0(ptr %x) nounwind {
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vmseq.vi v16, v8, 0
 ; CHECK-NEXT:    vfirst.m a0, v16
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %a = load <256 x i8>, ptr %x
   %b = icmp eq <256 x i8> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
index a8bd8dbb5a08a..1913d144139fb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
@@ -248,7 +248,6 @@ define void @extract_v8i1_v64i1_8(ptr %x, ptr %y) {
 ; LMULMAX1:       # %bb.0:
 ; LMULMAX1-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; LMULMAX1-NEXT:    vlm.v v8, (a0)
-; LMULMAX1-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; LMULMAX1-NEXT:    vslidedown.vi v8, v8, 1
 ; LMULMAX1-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; LMULMAX1-NEXT:    vsm.v v8, (a1)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
index b860916a3adf0..16c7611a84430 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
@@ -9,7 +9,6 @@ define i8 @extractelt_v16i8(ptr %x) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 7
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -23,7 +22,6 @@ define i16 @extractelt_v8i16(ptr %x) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 7
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -37,7 +35,6 @@ define i32 @extractelt_v4i32(ptr %x) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 2
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -74,7 +71,6 @@ define half @extractelt_v8f16(ptr %x) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 7
 ; CHECK-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-NEXT:    ret
@@ -88,7 +84,6 @@ define float @extractelt_v4f32(ptr %x) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 2
 ; CHECK-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-NEXT:    ret
@@ -252,7 +247,6 @@ define i8 @extractelt_v16i8_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -266,7 +260,6 @@ define i16 @extractelt_v8i16_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -281,7 +274,6 @@ define i32 @extractelt_v4i32_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    vadd.vv v8, v8, v8
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
@@ -297,10 +289,10 @@ define i64 @extractelt_v2i64_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
 ; RV32-NEXT:    vadd.vv v8, v8, v8
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vx v8, v8, a1
 ; RV32-NEXT:    vmv.x.s a0, v8
 ; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vsrl.vx v8, v8, a1
 ; RV32-NEXT:    vmv.x.s a1, v8
 ; RV32-NEXT:    ret
@@ -310,7 +302,6 @@ define i64 @extractelt_v2i64_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
 ; RV64-NEXT:    vadd.vv v8, v8, v8
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vx v8, v8, a1
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret
@@ -326,7 +317,6 @@ define half @extractelt_v8f16_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    vfadd.vv v8, v8, v8
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-NEXT:    ret
@@ -342,7 +332,6 @@ define float @extractelt_v4f32_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    vfadd.vv v8, v8, v8
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-NEXT:    ret
@@ -358,7 +347,6 @@ define double @extractelt_v2f64_idx(ptr %x, i32 zeroext %idx) nounwind {
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
 ; CHECK-NEXT:    vfadd.vv v8, v8, v8
-; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a1
 ; CHECK-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-NEXT:    ret
@@ -529,8 +517,8 @@ define void @store_extractelt_v16i8(ptr %x, ptr %p) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 7
+; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
   %a = load <16 x i8>, ptr %x
@@ -544,8 +532,8 @@ define void @store_extractelt_v8i16(ptr %x, ptr %p) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 7
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; CHECK-NEXT:    vse16.v v8, (a1)
 ; CHECK-NEXT:    ret
   %a = load <8 x i16>, ptr %x
@@ -559,8 +547,8 @@ define void @store_extractelt_v4i32(ptr %x, ptr %p) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 2
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; CHECK-NEXT:    vse32.v v8, (a1)
 ; CHECK-NEXT:    ret
   %a = load <4 x i32>, ptr %x
@@ -575,9 +563,9 @@ define void @store_extractelt_v2i64(ptr %x, ptr %p) nounwind {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vsrl.vx v9, v8, a0
 ; RV32-NEXT:    vmv.x.s a0, v9
 ; RV32-NEXT:    vmv.x.s a2, v8
@@ -589,8 +577,8 @@ define void @store_extractelt_v2i64(ptr %x, ptr %p) nounwind {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vse64.v v8, (a1)
 ; RV64-NEXT:    ret
   %a = load <2 x i64>, ptr %x
@@ -604,8 +592,8 @@ define void @store_extractelt_v2f64(ptr %x, ptr %p) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 1
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vse64.v v8, (a1)
 ; CHECK-NEXT:    ret
   %a = load <2 x double>, ptr %x
@@ -627,7 +615,6 @@ define i32 @extractelt_add_v4i32(<4 x i32> %x) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64-NEXT:    vadd.vi v8, v8, 13
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret
@@ -650,7 +637,6 @@ define i32 @extractelt_sub_v4i32(<4 x i32> %x) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64-NEXT:    vrsub.vi v8, v8, 13
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret
@@ -665,7 +651,6 @@ define i32 @extractelt_mul_v4i32(<4 x i32> %x) {
 ; RV32NOM-NEXT:    li a0, 13
 ; RV32NOM-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32NOM-NEXT:    vmul.vx v8, v8, a0
-; RV32NOM-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32NOM-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32NOM-NEXT:    vmv.x.s a0, v8
 ; RV32NOM-NEXT:    ret
@@ -684,7 +669,6 @@ define i32 @extractelt_mul_v4i32(<4 x i32> %x) {
 ; RV64-NEXT:    li a0, 13
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64-NEXT:    vmul.vx v8, v8, a0
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret
@@ -712,7 +696,6 @@ define i32 @extractelt_sdiv_v4i32(<4 x i32> %x) {
 ; RV32NOM-NEXT:    vsra.vv v9, v8, v11
 ; RV32NOM-NEXT:    vsrl.vi v8, v8, 31
 ; RV32NOM-NEXT:    vadd.vv v8, v9, v8
-; RV32NOM-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32NOM-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32NOM-NEXT:    vmv.x.s a0, v8
 ; RV32NOM-NEXT:    ret
@@ -748,7 +731,6 @@ define i32 @extractelt_sdiv_v4i32(<4 x i32> %x) {
 ; RV64-NEXT:    vsra.vv v8, v8, v11
 ; RV64-NEXT:    vsrl.vi v9, v8, 31
 ; RV64-NEXT:    vadd.vv v8, v8, v9
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret
@@ -765,7 +747,6 @@ define i32 @extractelt_udiv_v4i32(<4 x i32> %x) {
 ; RV32NOM-NEXT:    lui a0, 322639
 ; RV32NOM-NEXT:    addi a0, a0, -945
 ; RV32NOM-NEXT:    vmulhu.vx v8, v8, a0
-; RV32NOM-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32NOM-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32NOM-NEXT:    vmv.x.s a0, v8
 ; RV32NOM-NEXT:    srli a0, a0, 2
@@ -790,7 +771,6 @@ define i32 @extractelt_udiv_v4i32(<4 x i32> %x) {
 ; RV64-NEXT:    addiw a0, a0, -945
 ; RV64-NEXT:    vmulhu.vx v8, v8, a0
 ; RV64-NEXT:    vsrl.vi v8, v8, 2
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll
new file mode 100644
index 0000000000000..dcbca9cfe03df
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fceil-constrained-sdnode.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <1 x half> @ceil_v1f16(<1 x half> %x) {
+; CHECK-LABEL: ceil_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x half> @llvm.experimental.constrained.ceil.v1f16(<1 x half> %x, metadata !"fpexcept.strict")
+  ret <1 x half> %a
+}
+declare <1 x half> @llvm.experimental.constrained.ceil.v1f16(<1 x half>, metadata)
+
+define <2 x half> @ceil_v2f16(<2 x half> %x) {
+; CHECK-LABEL: ceil_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x half> @llvm.experimental.constrained.ceil.v2f16(<2 x half> %x, metadata !"fpexcept.strict")
+  ret <2 x half> %a
+}
+declare <2 x half> @llvm.experimental.constrained.ceil.v2f16(<2 x half>, metadata)
+
+define <4 x half> @ceil_v4f16(<4 x half> %x) {
+; CHECK-LABEL: ceil_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x half> @llvm.experimental.constrained.ceil.v4f16(<4 x half> %x, metadata !"fpexcept.strict")
+  ret <4 x half> %a
+}
+declare <4 x half> @llvm.experimental.constrained.ceil.v4f16(<4 x half>, metadata)
+
+define <8 x half> @ceil_v8f16(<8 x half> %x) {
+; CHECK-LABEL: ceil_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x half> @llvm.experimental.constrained.ceil.v8f16(<8 x half> %x, metadata !"fpexcept.strict")
+  ret <8 x half> %a
+}
+declare <8 x half> @llvm.experimental.constrained.ceil.v8f16(<8 x half>, metadata)
+
+define <16 x half> @ceil_v16f16(<16 x half> %x) {
+; CHECK-LABEL: ceil_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x half> @llvm.experimental.constrained.ceil.v16f16(<16 x half> %x, metadata !"fpexcept.strict")
+  ret <16 x half> %a
+}
+declare <16 x half> @llvm.experimental.constrained.ceil.v16f16(<16 x half>, metadata)
+
+define <32 x half> @ceil_v32f16(<32 x half> %x) {
+; CHECK-LABEL: ceil_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <32 x half> @llvm.experimental.constrained.ceil.v32f16(<32 x half> %x, metadata !"fpexcept.strict")
+  ret <32 x half> %a
+}
+declare <32 x half> @llvm.experimental.constrained.ceil.v32f16(<32 x half>, metadata)
+
+define <1 x float> @ceil_v1f32(<1 x float> %x) {
+; CHECK-LABEL: ceil_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float> %x, metadata !"fpexcept.strict")
+  ret <1 x float> %a
+}
+declare <1 x float> @llvm.experimental.constrained.ceil.v1f32(<1 x float>, metadata)
+
+define <2 x float> @ceil_v2f32(<2 x float> %x) {
+; CHECK-LABEL: ceil_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x float> @llvm.experimental.constrained.ceil.v2f32(<2 x float> %x, metadata !"fpexcept.strict")
+  ret <2 x float> %a
+}
+declare <2 x float> @llvm.experimental.constrained.ceil.v2f32(<2 x float>, metadata)
+
+define <4 x float> @ceil_v4f32(<4 x float> %x) {
+; CHECK-LABEL: ceil_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x float> @llvm.experimental.constrained.ceil.v4f32(<4 x float> %x, metadata !"fpexcept.strict")
+  ret <4 x float> %a
+}
+declare <4 x float> @llvm.experimental.constrained.ceil.v4f32(<4 x float>, metadata)
+
+define <8 x float> @ceil_v8f32(<8 x float> %x) {
+; CHECK-LABEL: ceil_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x float> @llvm.experimental.constrained.ceil.v8f32(<8 x float> %x, metadata !"fpexcept.strict")
+  ret <8 x float> %a
+}
+declare <8 x float> @llvm.experimental.constrained.ceil.v8f32(<8 x float>, metadata)
+
+define <16 x float> @ceil_v16f32(<16 x float> %x) {
+; CHECK-LABEL: ceil_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x float> @llvm.experimental.constrained.ceil.v16f32(<16 x float> %x, metadata !"fpexcept.strict")
+  ret <16 x float> %a
+}
+declare <16 x float> @llvm.experimental.constrained.ceil.v16f32(<16 x float>, metadata)
+
+define <1 x double> @ceil_v1f64(<1 x double> %x) {
+; CHECK-LABEL: ceil_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x double> @llvm.experimental.constrained.ceil.v1f64(<1 x double> %x, metadata !"fpexcept.strict")
+  ret <1 x double> %a
+}
+declare <1 x double> @llvm.experimental.constrained.ceil.v1f64(<1 x double>, metadata)
+
+define <2 x double> @ceil_v2f64(<2 x double> %x) {
+; CHECK-LABEL: ceil_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double> %x, metadata !"fpexcept.strict")
+  ret <2 x double> %a
+}
+declare <2 x double> @llvm.experimental.constrained.ceil.v2f64(<2 x double>, metadata)
+
+define <4 x double> @ceil_v4f64(<4 x double> %x) {
+; CHECK-LABEL: ceil_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double> %x, metadata !"fpexcept.strict")
+  ret <4 x double> %a
+}
+declare <4 x double> @llvm.experimental.constrained.ceil.v4f64(<4 x double>, metadata)
+
+define <8 x double> @ceil_v8f64(<8 x double> %x) {
+; CHECK-LABEL: ceil_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 3
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x double> @llvm.experimental.constrained.ceil.v8f64(<8 x double> %x, metadata !"fpexcept.strict")
+  ret <8 x double> %a
+}
+declare <8 x double> @llvm.experimental.constrained.ceil.v8f64(<8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll
new file mode 100644
index 0000000000000..751b5dca8987e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ffloor-constrained-sdnode.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <1 x half> @floor_v1f16(<1 x half> %x) {
+; CHECK-LABEL: floor_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x half> @llvm.experimental.constrained.floor.v1f16(<1 x half> %x, metadata !"fpexcept.strict")
+  ret <1 x half> %a
+}
+declare <1 x half> @llvm.experimental.constrained.floor.v1f16(<1 x half>, metadata)
+
+define <2 x half> @floor_v2f16(<2 x half> %x) {
+; CHECK-LABEL: floor_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x half> @llvm.experimental.constrained.floor.v2f16(<2 x half> %x, metadata !"fpexcept.strict")
+  ret <2 x half> %a
+}
+declare <2 x half> @llvm.experimental.constrained.floor.v2f16(<2 x half>, metadata)
+
+define <4 x half> @floor_v4f16(<4 x half> %x) {
+; CHECK-LABEL: floor_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x half> @llvm.experimental.constrained.floor.v4f16(<4 x half> %x, metadata !"fpexcept.strict")
+  ret <4 x half> %a
+}
+declare <4 x half> @llvm.experimental.constrained.floor.v4f16(<4 x half>, metadata)
+
+define <8 x half> @floor_v8f16(<8 x half> %x) {
+; CHECK-LABEL: floor_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x half> @llvm.experimental.constrained.floor.v8f16(<8 x half> %x, metadata !"fpexcept.strict")
+  ret <8 x half> %a
+}
+declare <8 x half> @llvm.experimental.constrained.floor.v8f16(<8 x half>, metadata)
+
+define <16 x half> @floor_v16f16(<16 x half> %x) {
+; CHECK-LABEL: floor_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x half> @llvm.experimental.constrained.floor.v16f16(<16 x half> %x, metadata !"fpexcept.strict")
+  ret <16 x half> %a
+}
+declare <16 x half> @llvm.experimental.constrained.floor.v16f16(<16 x half>, metadata)
+
+define <32 x half> @floor_v32f16(<32 x half> %x) {
+; CHECK-LABEL: floor_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <32 x half> @llvm.experimental.constrained.floor.v32f16(<32 x half> %x, metadata !"fpexcept.strict")
+  ret <32 x half> %a
+}
+declare <32 x half> @llvm.experimental.constrained.floor.v32f16(<32 x half>, metadata)
+
+define <1 x float> @floor_v1f32(<1 x float> %x) {
+; CHECK-LABEL: floor_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float> %x, metadata !"fpexcept.strict")
+  ret <1 x float> %a
+}
+declare <1 x float> @llvm.experimental.constrained.floor.v1f32(<1 x float>, metadata)
+
+define <2 x float> @floor_v2f32(<2 x float> %x) {
+; CHECK-LABEL: floor_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x float> @llvm.experimental.constrained.floor.v2f32(<2 x float> %x, metadata !"fpexcept.strict")
+  ret <2 x float> %a
+}
+declare <2 x float> @llvm.experimental.constrained.floor.v2f32(<2 x float>, metadata)
+
+define <4 x float> @floor_v4f32(<4 x float> %x) {
+; CHECK-LABEL: floor_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x float> @llvm.experimental.constrained.floor.v4f32(<4 x float> %x, metadata !"fpexcept.strict")
+  ret <4 x float> %a
+}
+declare <4 x float> @llvm.experimental.constrained.floor.v4f32(<4 x float>, metadata)
+
+define <8 x float> @floor_v8f32(<8 x float> %x) {
+; CHECK-LABEL: floor_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x float> @llvm.experimental.constrained.floor.v8f32(<8 x float> %x, metadata !"fpexcept.strict")
+  ret <8 x float> %a
+}
+declare <8 x float> @llvm.experimental.constrained.floor.v8f32(<8 x float>, metadata)
+
+define <16 x float> @floor_v16f32(<16 x float> %x) {
+; CHECK-LABEL: floor_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x float> @llvm.experimental.constrained.floor.v16f32(<16 x float> %x, metadata !"fpexcept.strict")
+  ret <16 x float> %a
+}
+declare <16 x float> @llvm.experimental.constrained.floor.v16f32(<16 x float>, metadata)
+
+define <1 x double> @floor_v1f64(<1 x double> %x) {
+; CHECK-LABEL: floor_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x double> @llvm.experimental.constrained.floor.v1f64(<1 x double> %x, metadata !"fpexcept.strict")
+  ret <1 x double> %a
+}
+declare <1 x double> @llvm.experimental.constrained.floor.v1f64(<1 x double>, metadata)
+
+define <2 x double> @floor_v2f64(<2 x double> %x) {
+; CHECK-LABEL: floor_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double> %x, metadata !"fpexcept.strict")
+  ret <2 x double> %a
+}
+declare <2 x double> @llvm.experimental.constrained.floor.v2f64(<2 x double>, metadata)
+
+define <4 x double> @floor_v4f64(<4 x double> %x) {
+; CHECK-LABEL: floor_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double> %x, metadata !"fpexcept.strict")
+  ret <4 x double> %a
+}
+declare <4 x double> @llvm.experimental.constrained.floor.v4f64(<4 x double>, metadata)
+
+define <8 x double> @floor_v8f64(<8 x double> %x) {
+; CHECK-LABEL: floor_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 2
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x double> @llvm.experimental.constrained.floor.v8f64(<8 x double> %x, metadata !"fpexcept.strict")
+  ret <8 x double> %a
+}
+declare <8 x double> @llvm.experimental.constrained.floor.v8f64(<8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
new file mode 100644
index 0000000000000..28da30f9e31b7
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fnearbyint-constrained-sdnode.ll
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+declare <2 x half> @llvm.experimental.constrained.nearbyint.v2f16(<2 x half>, metadata, metadata)
+
+define <2 x half> @nearbyint_v2f16(<2 x half> %v) {
+; CHECK-LABEL: nearbyint_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <2 x half> @llvm.experimental.constrained.nearbyint.v2f16(<2 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x half> %r
+}
+
+declare <4 x half> @llvm.experimental.constrained.nearbyint.v4f16(<4 x half>, metadata, metadata)
+
+define <4 x half> @nearbyint_v4f16(<4 x half> %v) {
+; CHECK-LABEL: nearbyint_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <4 x half> @llvm.experimental.constrained.nearbyint.v4f16(<4 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x half> %r
+}
+
+declare <8 x half> @llvm.experimental.constrained.nearbyint.v8f16(<8 x half>, metadata, metadata)
+
+define <8 x half> @nearbyint_v8f16(<8 x half> %v) {
+; CHECK-LABEL: nearbyint_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <8 x half> @llvm.experimental.constrained.nearbyint.v8f16(<8 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x half> %r
+}
+
+declare <16 x half> @llvm.experimental.constrained.nearbyint.v16f16(<16 x half>, metadata, metadata)
+
+define <16 x half> @nearbyint_v16f16(<16 x half> %v) {
+; CHECK-LABEL: nearbyint_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <16 x half> @llvm.experimental.constrained.nearbyint.v16f16(<16 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <16 x half> %r
+}
+
+declare <32 x half> @llvm.experimental.constrained.nearbyint.v32f16(<32 x half>, metadata, metadata)
+
+define <32 x half> @nearbyint_v32f16(<32 x half> %v) {
+; CHECK-LABEL: nearbyint_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <32 x half> @llvm.experimental.constrained.nearbyint.v32f16(<32 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <32 x half> %r
+}
+
+declare <2 x float> @llvm.experimental.constrained.nearbyint.v2f32(<2 x float>, metadata, metadata)
+
+define <2 x float> @nearbyint_v2f32(<2 x float> %v) {
+; CHECK-LABEL: nearbyint_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <2 x float> @llvm.experimental.constrained.nearbyint.v2f32(<2 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x float> %r
+}
+
+declare <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(<4 x float>, metadata, metadata)
+
+define <4 x float> @nearbyint_v4f32(<4 x float> %v) {
+; CHECK-LABEL: nearbyint_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <4 x float> @llvm.experimental.constrained.nearbyint.v4f32(<4 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x float> %r
+}
+
+declare <8 x float> @llvm.experimental.constrained.nearbyint.v8f32(<8 x float>, metadata, metadata)
+
+define <8 x float> @nearbyint_v8f32(<8 x float> %v) {
+; CHECK-LABEL: nearbyint_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <8 x float> @llvm.experimental.constrained.nearbyint.v8f32(<8 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x float> %r
+}
+
+declare <16 x float> @llvm.experimental.constrained.nearbyint.v16f32(<16 x float>, metadata, metadata)
+
+define <16 x float> @nearbyint_v16f32(<16 x float> %v) {
+; CHECK-LABEL: nearbyint_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <16 x float> @llvm.experimental.constrained.nearbyint.v16f32(<16 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <16 x float> %r
+}
+
+declare <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double>, metadata, metadata)
+
+define <2 x double> @nearbyint_v2f64(<2 x double> %v) {
+; CHECK-LABEL: nearbyint_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI9_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI9_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <2 x double> @llvm.experimental.constrained.nearbyint.v2f64(<2 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <2 x double> %r
+}
+
+declare <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double>, metadata, metadata)
+
+define <4 x double> @nearbyint_v4f64(<4 x double> %v) {
+; CHECK-LABEL: nearbyint_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI10_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI10_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <4 x double> @llvm.experimental.constrained.nearbyint.v4f64(<4 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <4 x double> %r
+}
+
+declare <8 x double> @llvm.experimental.constrained.nearbyint.v8f64(<8 x double>, metadata, metadata)
+
+define <8 x double> @nearbyint_v8f64(<8 x double> %v) {
+; CHECK-LABEL: nearbyint_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <8 x double> @llvm.experimental.constrained.nearbyint.v8f64(<8 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <8 x double> %r
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
index 3c5eaec3b5b7c..3e11193f9b6c4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp.ll
@@ -24,6 +24,41 @@ define void @fadd_v8f16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @fadd_v6f16(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: fadd_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfadd.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fadd_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfadd.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = fadd <6 x half> %a, %b
+  store <6 x half> %c, ptr %x
+  ret void
+}
+
 define void @fadd_v4f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fadd_v4f32:
 ; CHECK:       # %bb.0:
@@ -72,6 +107,41 @@ define void @fsub_v8f16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @fsub_v6f16(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: fsub_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfsub.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fsub_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfsub.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = fsub <6 x half> %a, %b
+  store <6 x half> %c, ptr %x
+  ret void
+}
+
 define void @fsub_v4f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fsub_v4f32:
 ; CHECK:       # %bb.0:
@@ -120,6 +190,41 @@ define void @fmul_v8f16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @fmul_v6f16(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: fmul_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfmul.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmul_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfmul.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = fmul <6 x half> %a, %b
+  store <6 x half> %c, ptr %x
+  ret void
+}
+
 define void @fmul_v4f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fmul_v4f32:
 ; CHECK:       # %bb.0:
@@ -168,6 +273,41 @@ define void @fdiv_v8f16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @fdiv_v6f16(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: fdiv_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfdiv.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fdiv_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfdiv.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = fdiv <6 x half> %a, %b
+  store <6 x half> %c, ptr %x
+  ret void
+}
+
 define void @fdiv_v4f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fdiv_v4f32:
 ; CHECK:       # %bb.0:
@@ -214,6 +354,38 @@ define void @fneg_v8f16(ptr %x) {
   ret void
 }
 
+define void @fneg_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: fneg_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfneg.v v8, v8
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fneg_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfneg.v v8, v8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = fneg <6 x half> %a
+  store <6 x half> %b, ptr %x
+  ret void
+}
+
 define void @fneg_v4f32(ptr %x) {
 ; CHECK-LABEL: fneg_v4f32:
 ; CHECK:       # %bb.0:
@@ -257,6 +429,39 @@ define void @fabs_v8f16(ptr %x) {
 }
 declare <8 x half> @llvm.fabs.v8f16(<8 x half>)
 
+define void @fabs_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: fabs_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfabs.v v8, v8
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fabs_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfabs.v v8, v8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = call <6 x half> @llvm.fabs.v6f16(<6 x half> %a)
+  store <6 x half> %b, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.fabs.v6f16(<6 x half>)
+
 define void @fabs_v4f32(ptr %x) {
 ; CHECK-LABEL: fabs_v4f32:
 ; CHECK:       # %bb.0:
@@ -304,6 +509,42 @@ define void @copysign_v8f16(ptr %x, ptr %y) {
 }
 declare <8 x half> @llvm.copysign.v8f16(<8 x half>, <8 x half>)
 
+define void @copysign_v6f16(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: copysign_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfsgnj.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: copysign_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfsgnj.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = call <6 x half> @llvm.copysign.v6f16(<6 x half> %a, <6 x half> %b)
+  store <6 x half> %c, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.copysign.v6f16(<6 x half>, <6 x half>)
+
 define void @copysign_v4f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: copysign_v4f32:
 ; CHECK:       # %bb.0:
@@ -354,6 +595,40 @@ define void @copysign_vf_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @copysign_vf_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: copysign_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfsgnj.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: copysign_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfsgnj.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = call <6 x half> @llvm.copysign.v6f16(<6 x half> %a, <6 x half> %c)
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @copysign_vf_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: copysign_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -403,6 +678,42 @@ define void @copysign_neg_v8f16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @copysign_neg_v6f16(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: copysign_neg_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfsgnjn.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: copysign_neg_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfsgnjn.vv v8, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = fneg <6 x half> %b
+  %d = call <6 x half> @llvm.copysign.v6f16(<6 x half> %a, <6 x half> %c)
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @copysign_neg_v4f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: copysign_neg_v4f32:
 ; CHECK:       # %bb.0:
@@ -457,6 +768,47 @@ define void @copysign_neg_trunc_v4f16_v4f32(ptr %x, ptr %y) {
 }
 declare <4 x half> @llvm.copysign.v4f16(<4 x half>, <4 x half>)
 
+define void @copysign_neg_trunc_v3f16_v3f32(ptr %x, ptr %y) {
+; LMULMAX1-RV32-LABEL: copysign_neg_trunc_v3f16_v3f32:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vle32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a0)
+; LMULMAX1-RV32-NEXT:    vfncvt.f.f.w v10, v8
+; LMULMAX1-RV32-NEXT:    vfsgnjn.vv v8, v9, v10
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 4
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: copysign_neg_trunc_v3f16_v3f32:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV64-NEXT:    vle32.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfncvt.f.f.w v10, v9
+; LMULMAX1-RV64-NEXT:    vfsgnjn.vv v8, v8, v10
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a1, a0, 4
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; LMULMAX1-RV64-NEXT:    vse16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <3 x half>, ptr %x
+  %b = load <3 x float>, ptr %y
+  %c = fneg <3 x float> %b
+  %d = fptrunc <3 x float> %c to <3 x half>
+  %e = call <3 x half> @llvm.copysign.v3f16(<3 x half> %a, <3 x half> %d)
+  store <3 x half> %e, ptr %x
+  ret void
+}
+declare <3 x half> @llvm.copysign.v3f16(<3 x half>, <3 x half>)
+
 define void @copysign_neg_ext_v2f64_v2f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: copysign_neg_ext_v2f64_v2f32:
 ; CHECK:       # %bb.0:
@@ -492,6 +844,39 @@ define void @sqrt_v8f16(ptr %x) {
 }
 declare <8 x half> @llvm.sqrt.v8f16(<8 x half>)
 
+define void @sqrt_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: sqrt_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfsqrt.v v8, v8
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: sqrt_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfsqrt.v v8, v8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = call <6 x half> @llvm.sqrt.v6f16(<6 x half> %a)
+  store <6 x half> %b, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.sqrt.v6f16(<6 x half>)
+
 define void @sqrt_v4f32(ptr %x) {
 ; CHECK-LABEL: sqrt_v4f32:
 ; CHECK:       # %bb.0:
@@ -541,6 +926,45 @@ define void @fma_v8f16(ptr %x, ptr %y, ptr %z) {
 }
 declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
 
+define void @fma_v6f16(ptr %x, ptr %y, ptr %z) {
+; LMULMAX1-RV32-LABEL: fma_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV32-NEXT:    vfmacc.vv v10, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v10, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fma_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV64-NEXT:    vfmacc.vv v10, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v10, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = load <6 x half>, ptr %z
+  %d = call <6 x half> @llvm.fma.v6f16(<6 x half> %a, <6 x half> %b, <6 x half> %c)
+  store <6 x half> %d, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.fma.v6f16(<6 x half>, <6 x half>, <6 x half>)
+
 define void @fma_v4f32(ptr %x, ptr %y, ptr %z) {
 ; CHECK-LABEL: fma_v4f32:
 ; CHECK:       # %bb.0:
@@ -598,6 +1022,45 @@ define void @fmsub_v8f16(ptr %x, ptr %y, ptr %z) {
   ret void
 }
 
+define void @fmsub_v6f16(ptr %x, ptr %y, ptr %z) {
+; LMULMAX1-RV32-LABEL: fmsub_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV32-NEXT:    vfmsac.vv v10, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v10, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmsub_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV64-NEXT:    vfmsac.vv v10, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v10, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = load <6 x half>, ptr %z
+  %neg = fneg <6 x half> %c
+  %d = call <6 x half> @llvm.fma.v6f16(<6 x half> %a, <6 x half> %b, <6 x half> %neg)
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fnmsub_v4f32(ptr %x, ptr %y, ptr %z) {
 ; CHECK-LABEL: fnmsub_v4f32:
 ; CHECK:       # %bb.0:
@@ -1394,6 +1857,40 @@ define void @fadd_vf_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fadd_vf_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fadd_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfadd.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fadd_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfadd.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fadd <6 x half> %a, %c
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fadd_vf_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fadd_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -1442,6 +1939,40 @@ define void @fadd_fv_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fadd_fv_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fadd_fv_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfadd.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fadd_fv_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfadd.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fadd <6 x half> %c, %a
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fadd_fv_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fadd_fv_v4f32:
 ; CHECK:       # %bb.0:
@@ -1490,6 +2021,40 @@ define void @fsub_vf_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fsub_vf_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fsub_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfsub.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fsub_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfsub.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fsub <6 x half> %a, %c
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fsub_vf_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fsub_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -1538,6 +2103,40 @@ define void @fsub_fv_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fsub_fv_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fsub_fv_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfrsub.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fsub_fv_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfrsub.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fsub <6 x half> %c, %a
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fsub_fv_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fsub_fv_v4f32:
 ; CHECK:       # %bb.0:
@@ -1586,6 +2185,40 @@ define void @fmul_vf_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fmul_vf_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fmul_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfmul.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmul_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfmul.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fmul <6 x half> %a, %c
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fmul_vf_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fmul_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -1634,6 +2267,40 @@ define void @fmul_fv_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fmul_fv_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fmul_fv_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfmul.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmul_fv_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfmul.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fmul <6 x half> %c, %a
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fmul_fv_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fmul_fv_v4f32:
 ; CHECK:       # %bb.0:
@@ -1682,6 +2349,40 @@ define void @fdiv_vf_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fdiv_vf_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fdiv_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfdiv.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fdiv_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfdiv.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fdiv <6 x half> %a, %c
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fdiv_vf_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fdiv_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -1730,6 +2431,40 @@ define void @fdiv_fv_v8f16(ptr %x, half %y) {
   ret void
 }
 
+define void @fdiv_fv_v6f16(ptr %x, half %y) {
+; LMULMAX1-RV32-LABEL: fdiv_fv_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vfrdiv.vf v8, v8, fa0
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fdiv_fv_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vfrdiv.vf v8, v8, fa0
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = insertelement <6 x half> poison, half %y, i32 0
+  %c = shufflevector <6 x half> %b, <6 x half> poison, <6 x i32> zeroinitializer
+  %d = fdiv <6 x half> %c, %a
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fdiv_fv_v4f32(ptr %x, float %y) {
 ; CHECK-LABEL: fdiv_fv_v4f32:
 ; CHECK:       # %bb.0:
@@ -1780,6 +2515,43 @@ define void @fma_vf_v8f16(ptr %x, ptr %y, half %z) {
   ret void
 }
 
+define void @fma_vf_v6f16(ptr %x, ptr %y, half %z) {
+; LMULMAX1-RV32-LABEL: fma_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfmacc.vf v9, fa0, v8
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v9, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fma_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfmacc.vf v9, fa0, v8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v9, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = insertelement <6 x half> poison, half %z, i32 0
+  %d = shufflevector <6 x half> %c, <6 x half> poison, <6 x i32> zeroinitializer
+  %e = call <6 x half> @llvm.fma.v6f16(<6 x half> %a, <6 x half> %d, <6 x half> %b)
+  store <6 x half> %e, ptr %x
+  ret void
+}
+
 define void @fma_vf_v4f32(ptr %x, ptr %y, float %z) {
 ; CHECK-LABEL: fma_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -1834,6 +2606,43 @@ define void @fma_fv_v8f16(ptr %x, ptr %y, half %z) {
   ret void
 }
 
+define void @fma_fv_v6f16(ptr %x, ptr %y, half %z) {
+; LMULMAX1-RV32-LABEL: fma_fv_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfmacc.vf v9, fa0, v8
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v9, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fma_fv_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfmacc.vf v9, fa0, v8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v9, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = insertelement <6 x half> poison, half %z, i32 0
+  %d = shufflevector <6 x half> %c, <6 x half> poison, <6 x i32> zeroinitializer
+  %e = call <6 x half> @llvm.fma.v6f16(<6 x half> %d, <6 x half> %a, <6 x half> %b)
+  store <6 x half> %e, ptr %x
+  ret void
+}
+
 define void @fma_fv_v4f32(ptr %x, ptr %y, float %z) {
 ; CHECK-LABEL: fma_fv_v4f32:
 ; CHECK:       # %bb.0:
@@ -1889,6 +2698,44 @@ define void @fmsub_vf_v8f16(ptr %x, ptr %y, half %z) {
   ret void
 }
 
+define void @fmsub_vf_v6f16(ptr %x, ptr %y, half %z) {
+; LMULMAX1-RV32-LABEL: fmsub_vf_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vfmsac.vf v9, fa0, v8
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v9, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmsub_vf_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vfmsac.vf v9, fa0, v8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v9, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = insertelement <6 x half> poison, half %z, i32 0
+  %d = shufflevector <6 x half> %c, <6 x half> poison, <6 x i32> zeroinitializer
+  %neg = fneg <6 x half> %b
+  %e = call <6 x half> @llvm.fma.v6f16(<6 x half> %a, <6 x half> %d, <6 x half> %neg)
+  store <6 x half> %e, ptr %x
+  ret void
+}
+
 define void @fnmsub_vf_v4f32(ptr %x, ptr %y, float %z) {
 ; CHECK-LABEL: fnmsub_vf_v4f32:
 ; CHECK:       # %bb.0:
@@ -1972,8 +2819,8 @@ define void @trunc_v8f16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI91_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI91_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI115_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI115_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
@@ -1989,6 +2836,53 @@ define void @trunc_v8f16(ptr %x) {
 }
 declare <8 x half> @llvm.trunc.v8f16(<8 x half>)
 
+define void @trunc_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: trunc_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI116_0)
+; LMULMAX1-RV32-NEXT:    flh fa5, %lo(.LCPI116_0)(a1)
+; LMULMAX1-RV32-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV32-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV32-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV32-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: trunc_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI116_0)
+; LMULMAX1-RV64-NEXT:    flh fa5, %lo(.LCPI116_0)(a1)
+; LMULMAX1-RV64-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV64-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV64-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV64-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = call <6 x half> @llvm.trunc.v6f16(<6 x half> %a)
+  store <6 x half> %b, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.trunc.v6f16(<6 x half>)
+
 define void @trunc_v4f32(ptr %x) {
 ; CHECK-LABEL: trunc_v4f32:
 ; CHECK:       # %bb.0:
@@ -2016,8 +2910,8 @@ define void @trunc_v2f64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI93_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI93_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI118_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI118_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
@@ -2038,8 +2932,8 @@ define void @ceil_v8f16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI94_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI94_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI119_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI119_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    fsrmi a1, 3
@@ -2057,6 +2951,57 @@ define void @ceil_v8f16(ptr %x) {
 }
 declare <8 x half> @llvm.ceil.v8f16(<8 x half>)
 
+define void @ceil_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: ceil_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI120_0)
+; LMULMAX1-RV32-NEXT:    flh fa5, %lo(.LCPI120_0)(a1)
+; LMULMAX1-RV32-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV32-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV32-NEXT:    fsrmi a1, 3
+; LMULMAX1-RV32-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    fsrm a1
+; LMULMAX1-RV32-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV32-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: ceil_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI120_0)
+; LMULMAX1-RV64-NEXT:    flh fa5, %lo(.LCPI120_0)(a1)
+; LMULMAX1-RV64-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV64-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV64-NEXT:    fsrmi a1, 3
+; LMULMAX1-RV64-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    fsrm a1
+; LMULMAX1-RV64-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV64-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = call <6 x half> @llvm.ceil.v6f16(<6 x half> %a)
+  store <6 x half> %b, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.ceil.v6f16(<6 x half>)
+
 define void @ceil_v4f32(ptr %x) {
 ; CHECK-LABEL: ceil_v4f32:
 ; CHECK:       # %bb.0:
@@ -2086,8 +3031,8 @@ define void @ceil_v2f64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI96_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI96_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI122_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI122_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    fsrmi a1, 3
@@ -2110,8 +3055,8 @@ define void @floor_v8f16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI97_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI97_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI123_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI123_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    fsrmi a1, 2
@@ -2129,6 +3074,57 @@ define void @floor_v8f16(ptr %x) {
 }
 declare <8 x half> @llvm.floor.v8f16(<8 x half>)
 
+define void @floor_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: floor_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI124_0)
+; LMULMAX1-RV32-NEXT:    flh fa5, %lo(.LCPI124_0)(a1)
+; LMULMAX1-RV32-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV32-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV32-NEXT:    fsrmi a1, 2
+; LMULMAX1-RV32-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    fsrm a1
+; LMULMAX1-RV32-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV32-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: floor_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI124_0)
+; LMULMAX1-RV64-NEXT:    flh fa5, %lo(.LCPI124_0)(a1)
+; LMULMAX1-RV64-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV64-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV64-NEXT:    fsrmi a1, 2
+; LMULMAX1-RV64-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    fsrm a1
+; LMULMAX1-RV64-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV64-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = call <6 x half> @llvm.floor.v6f16(<6 x half> %a)
+  store <6 x half> %b, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.floor.v6f16(<6 x half>)
+
 define void @floor_v4f32(ptr %x) {
 ; CHECK-LABEL: floor_v4f32:
 ; CHECK:       # %bb.0:
@@ -2158,8 +3154,8 @@ define void @floor_v2f64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI99_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI99_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI126_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI126_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    fsrmi a1, 2
@@ -2182,8 +3178,8 @@ define void @round_v8f16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI100_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI100_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI127_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI127_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    fsrmi a1, 4
@@ -2201,6 +3197,57 @@ define void @round_v8f16(ptr %x) {
 }
 declare <8 x half> @llvm.round.v8f16(<8 x half>)
 
+define void @round_v6f16(ptr %x) {
+; LMULMAX1-RV32-LABEL: round_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    lui a1, %hi(.LCPI128_0)
+; LMULMAX1-RV32-NEXT:    flh fa5, %lo(.LCPI128_0)(a1)
+; LMULMAX1-RV32-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV32-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV32-NEXT:    fsrmi a1, 4
+; LMULMAX1-RV32-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    fsrm a1
+; LMULMAX1-RV32-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV32-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: round_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI128_0)
+; LMULMAX1-RV64-NEXT:    flh fa5, %lo(.LCPI128_0)(a1)
+; LMULMAX1-RV64-NEXT:    vfabs.v v9, v8
+; LMULMAX1-RV64-NEXT:    vmflt.vf v0, v9, fa5
+; LMULMAX1-RV64-NEXT:    fsrmi a1, 4
+; LMULMAX1-RV64-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    fsrm a1
+; LMULMAX1-RV64-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; LMULMAX1-RV64-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v8, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = call <6 x half> @llvm.round.v6f16(<6 x half> %a)
+  store <6 x half> %b, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.round.v6f16(<6 x half>)
+
 define void @round_v4f32(ptr %x) {
 ; CHECK-LABEL: round_v4f32:
 ; CHECK:       # %bb.0:
@@ -2230,8 +3277,8 @@ define void @round_v2f64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI102_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI102_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI130_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI130_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    fsrmi a1, 4
@@ -2254,8 +3301,8 @@ define void @rint_v8f16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI103_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI103_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI131_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI131_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
@@ -2298,8 +3345,8 @@ define void @rint_v2f64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI105_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI105_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI133_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI133_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
@@ -2320,8 +3367,8 @@ define void @nearbyint_v8f16(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI106_0)
-; CHECK-NEXT:    flh fa5, %lo(.LCPI106_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI134_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI134_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    frflags a1
@@ -2368,8 +3415,8 @@ define void @nearbyint_v2f64(ptr %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
-; CHECK-NEXT:    lui a1, %hi(.LCPI108_0)
-; CHECK-NEXT:    fld fa5, %lo(.LCPI108_0)(a1)
+; CHECK-NEXT:    lui a1, %hi(.LCPI136_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI136_0)(a1)
 ; CHECK-NEXT:    vfabs.v v9, v8
 ; CHECK-NEXT:    vmflt.vf v0, v9, fa5
 ; CHECK-NEXT:    frflags a1
@@ -2406,6 +3453,45 @@ define void @fmuladd_v8f16(ptr %x, ptr %y, ptr %z) {
 }
 declare <8 x half> @llvm.fmuladd.v8f16(<8 x half>, <8 x half>, <8 x half>)
 
+define void @fmuladd_v6f16(ptr %x, ptr %y, ptr %z) {
+; LMULMAX1-RV32-LABEL: fmuladd_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV32-NEXT:    vfmacc.vv v10, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v10, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmuladd_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV64-NEXT:    vfmacc.vv v10, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v10, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = load <6 x half>, ptr %z
+  %d = call <6 x half> @llvm.fmuladd.v6f16(<6 x half> %a, <6 x half> %b, <6 x half> %c)
+  store <6 x half> %d, ptr %x
+  ret void
+}
+declare <6 x half> @llvm.fmuladd.v6f16(<6 x half>, <6 x half>, <6 x half>)
+
 define void @fmuladd_v4f32(ptr %x, ptr %y, ptr %z) {
 ; CHECK-LABEL: fmuladd_v4f32:
 ; CHECK:       # %bb.0:
@@ -2463,6 +3549,45 @@ define void @fmsub_fmuladd_v8f16(ptr %x, ptr %y, ptr %z) {
   ret void
 }
 
+define void @fmsub_fmuladd_v6f16(ptr %x, ptr %y, ptr %z) {
+; LMULMAX1-RV32-LABEL: fmsub_fmuladd_v6f16:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV32-NEXT:    vfmsac.vv v10, v8, v9
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV32-NEXT:    addi a1, a0, 8
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vse16.v v10, (a0)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: fmsub_fmuladd_v6f16:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle16.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vle16.v v9, (a1)
+; LMULMAX1-RV64-NEXT:    vle16.v v10, (a2)
+; LMULMAX1-RV64-NEXT:    vfmsac.vv v10, v8, v9
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v10, (a0)
+; LMULMAX1-RV64-NEXT:    vslidedown.vi v8, v10, 2
+; LMULMAX1-RV64-NEXT:    addi a0, a0, 8
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x half>, ptr %x
+  %b = load <6 x half>, ptr %y
+  %c = load <6 x half>, ptr %z
+  %neg = fneg <6 x half> %c
+  %d = call <6 x half> @llvm.fmuladd.v6f16(<6 x half> %a, <6 x half> %b, <6 x half> %neg)
+  store <6 x half> %d, ptr %x
+  ret void
+}
+
 define void @fnmsub_fmuladd_v4f32(ptr %x, ptr %y, ptr %z) {
 ; CHECK-LABEL: fnmsub_fmuladd_v4f32:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
index ab3e01e2ff615..9b46d4441a21f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i-sat.ll
@@ -191,68 +191,58 @@ define void @fp2si_v2f64_v2i8(ptr %x, ptr %y) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v9
 ; RV32-NEXT:    lui a0, %hi(.LCPI10_0)
-; RV32-NEXT:    fld fa5, %lo(.LCPI10_0)(a0)
+; RV32-NEXT:    fld fa4, %lo(.LCPI10_0)(a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI10_1)
-; RV32-NEXT:    fld fa4, %lo(.LCPI10_1)(a0)
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa3, v9
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa3, v8
-; RV32-NEXT:    fmax.d fa5, fa3, fa5
-; RV32-NEXT:    fmin.d fa5, fa5, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa5, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    vsetivli zero, 2, e8, mf8, tu, ma
-; RV32-NEXT:    vmv.s.x v9, a0
-; RV32-NEXT:    vse8.v v9, (a1)
+; RV32-NEXT:    fld fa3, %lo(.LCPI10_1)(a0)
+; RV32-NEXT:    feq.d a0, fa5, fa5
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa5, fa5, fa4
+; RV32-NEXT:    fmin.d fa5, fa5, fa3
+; RV32-NEXT:    fcvt.w.d a2, fa5, rtz
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    feq.d a2, fa5, fa5
+; RV32-NEXT:    neg a2, a2
+; RV32-NEXT:    fmax.d fa5, fa5, fa4
+; RV32-NEXT:    fmin.d fa5, fa5, fa3
+; RV32-NEXT:    fcvt.w.d a3, fa5, rtz
+; RV32-NEXT:    and a2, a2, a3
+; RV32-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vse8.v v8, (a1)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: fp2si_v2f64_v2i8:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v9
 ; RV64-NEXT:    lui a0, %hi(.LCPI10_0)
-; RV64-NEXT:    fld fa5, %lo(.LCPI10_0)(a0)
+; RV64-NEXT:    fld fa4, %lo(.LCPI10_0)(a0)
 ; RV64-NEXT:    lui a0, %hi(.LCPI10_1)
-; RV64-NEXT:    fld fa4, %lo(.LCPI10_1)(a0)
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa3, v9
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addi a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; RV64-NEXT:    vmv.v.x v9, a0
-; RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa3, v8
-; RV64-NEXT:    fmax.d fa5, fa3, fa5
-; RV64-NEXT:    fmin.d fa5, fa5, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa5, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addi a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    vsetivli zero, 2, e8, mf8, tu, ma
-; RV64-NEXT:    vmv.s.x v9, a0
-; RV64-NEXT:    vse8.v v9, (a1)
+; RV64-NEXT:    fld fa3, %lo(.LCPI10_1)(a0)
+; RV64-NEXT:    feq.d a0, fa5, fa5
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa5, fa5, fa4
+; RV64-NEXT:    fmin.d fa5, fa5, fa3
+; RV64-NEXT:    fcvt.l.d a2, fa5, rtz
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    feq.d a2, fa5, fa5
+; RV64-NEXT:    neg a2, a2
+; RV64-NEXT:    fmax.d fa5, fa5, fa4
+; RV64-NEXT:    fmin.d fa5, fa5, fa3
+; RV64-NEXT:    fcvt.l.d a3, fa5, rtz
+; RV64-NEXT:    and a2, a2, a3
+; RV64-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; RV64-NEXT:    vslide1down.vx v8, v8, a2
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vse8.v v8, (a1)
 ; RV64-NEXT:    ret
   %a = load <2 x double>, ptr %x
   %d = call <2 x i8> @llvm.fptosi.sat.v2i8.v2f64(<2 x double> %a)
@@ -273,16 +263,16 @@ define void @fp2ui_v2f64_v2i8(ptr %x, ptr %y) {
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
+; RV32-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; RV32-NEXT:    vslide1down.vx v9, v8, a0
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    vfmv.f.s fa4, v8
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa5, fa4, fa5
-; RV32-NEXT:    fcvt.wu.d a2, fa5, rtz
+; RV32-NEXT:    fcvt.wu.d a0, fa5, rtz
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a2
-; RV32-NEXT:    vsetvli zero, zero, e8, mf8, tu, ma
-; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v9, a0
 ; RV32-NEXT:    vse8.v v8, (a1)
 ; RV32-NEXT:    ret
 ;
@@ -297,16 +287,16 @@ define void @fp2ui_v2f64_v2i8(ptr %x, ptr %y) {
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
+; RV64-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; RV64-NEXT:    vslide1down.vx v9, v8, a0
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64-NEXT:    vfmv.f.s fa4, v8
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa5, fa4, fa5
-; RV64-NEXT:    fcvt.lu.d a2, fa5, rtz
+; RV64-NEXT:    fcvt.lu.d a0, fa5, rtz
 ; RV64-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a2
-; RV64-NEXT:    vsetvli zero, zero, e8, mf8, tu, ma
-; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    vslide1down.vx v8, v9, a0
 ; RV64-NEXT:    vse8.v v8, (a1)
 ; RV64-NEXT:    ret
   %a = load <2 x double>, ptr %x
@@ -320,196 +310,194 @@ define void @fp2si_v8f64_v8i8(ptr %x, ptr %y) {
 ;
 ; RV32-LABEL: fp2si_v8f64_v8i8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v12, v8, 1
+; RV32-NEXT:    vfmv.f.s fa3, v12
 ; RV32-NEXT:    lui a0, %hi(.LCPI12_0)
 ; RV32-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI12_1)
 ; RV32-NEXT:    fld fa4, %lo(.LCPI12_1)(a0)
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    and a0, a0, a2
 ; RV32-NEXT:    vfmv.f.s fa3, v8
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
 ; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 8(sp)
+; RV32-NEXT:    neg a2, a2
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a3, fa3, rtz
+; RV32-NEXT:    and a2, a2, a3
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v8, a2
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
 ; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
-; RV32-NEXT:    vslidedown.vi v12, v8, 7
-; RV32-NEXT:    vfmv.f.s fa3, v12
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 15(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 6
-; RV32-NEXT:    vfmv.f.s fa3, v12
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 14(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 5
-; RV32-NEXT:    vfmv.f.s fa3, v12
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 13(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 4
-; RV32-NEXT:    vfmv.f.s fa3, v12
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 12(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 3
-; RV32-NEXT:    vfmv.f.s fa3, v12
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 11(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 2
-; RV32-NEXT:    vfmv.f.s fa3, v12
-; RV32-NEXT:    fmax.d fa2, fa3, fa5
-; RV32-NEXT:    fmin.d fa2, fa2, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa2, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 10(sp)
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vslidedown.vi v16, v8, 2
+; RV32-NEXT:    vfmv.f.s fa3, v16
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 3
+; RV32-NEXT:    vfmv.f.s fa3, v16
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 4
+; RV32-NEXT:    vfmv.f.s fa3, v16
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 5
+; RV32-NEXT:    vfmv.f.s fa3, v16
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 6
+; RV32-NEXT:    vfmv.f.s fa3, v16
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:    fmax.d fa3, fa3, fa5
+; RV32-NEXT:    fmin.d fa3, fa3, fa4
+; RV32-NEXT:    fcvt.w.d a2, fa3, rtz
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
 ; RV32-NEXT:    vfmv.f.s fa3, v8
+; RV32-NEXT:    feq.d a0, fa3, fa3
+; RV32-NEXT:    neg a0, a0
 ; RV32-NEXT:    fmax.d fa5, fa3, fa5
 ; RV32-NEXT:    fmin.d fa5, fa5, fa4
-; RV32-NEXT:    fcvt.w.d a0, fa5, rtz
-; RV32-NEXT:    feq.d a2, fa3, fa3
-; RV32-NEXT:    seqz a2, a2
-; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    and a0, a2, a0
-; RV32-NEXT:    sb a0, 9(sp)
-; RV32-NEXT:    addi a0, sp, 8
+; RV32-NEXT:    fcvt.w.d a2, fa5, rtz
+; RV32-NEXT:    and a0, a0, a2
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vle8.v v8, (a0)
+; RV32-NEXT:    vslide1down.vx v8, v12, a0
 ; RV32-NEXT:    vse8.v v8, (a1)
-; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: fp2si_v8f64_v8i8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v12, v8, 1
+; RV64-NEXT:    vfmv.f.s fa3, v12
 ; RV64-NEXT:    lui a0, %hi(.LCPI12_0)
 ; RV64-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
 ; RV64-NEXT:    lui a0, %hi(.LCPI12_1)
 ; RV64-NEXT:    fld fa4, %lo(.LCPI12_1)(a0)
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    and a0, a0, a2
 ; RV64-NEXT:    vfmv.f.s fa3, v8
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
 ; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 8(sp)
+; RV64-NEXT:    neg a2, a2
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a3, fa3, rtz
+; RV64-NEXT:    and a2, a2, a3
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v8, a2
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
 ; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
-; RV64-NEXT:    vslidedown.vi v12, v8, 7
-; RV64-NEXT:    vfmv.f.s fa3, v12
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 15(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-NEXT:    vfmv.f.s fa3, v12
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 14(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 5
-; RV64-NEXT:    vfmv.f.s fa3, v12
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 13(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 4
-; RV64-NEXT:    vfmv.f.s fa3, v12
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 12(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 3
-; RV64-NEXT:    vfmv.f.s fa3, v12
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 11(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-NEXT:    vfmv.f.s fa3, v12
-; RV64-NEXT:    fmax.d fa2, fa3, fa5
-; RV64-NEXT:    fmin.d fa2, fa2, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa2, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 10(sp)
-; RV64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-NEXT:    vslidedown.vi v16, v8, 2
+; RV64-NEXT:    vfmv.f.s fa3, v16
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 3
+; RV64-NEXT:    vfmv.f.s fa3, v16
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 4
+; RV64-NEXT:    vfmv.f.s fa3, v16
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 5
+; RV64-NEXT:    vfmv.f.s fa3, v16
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 6
+; RV64-NEXT:    vfmv.f.s fa3, v16
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:    fmax.d fa3, fa3, fa5
+; RV64-NEXT:    fmin.d fa3, fa3, fa4
+; RV64-NEXT:    fcvt.l.d a2, fa3, rtz
+; RV64-NEXT:    and a0, a0, a2
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v8, v8, 7
 ; RV64-NEXT:    vfmv.f.s fa3, v8
+; RV64-NEXT:    feq.d a0, fa3, fa3
+; RV64-NEXT:    neg a0, a0
 ; RV64-NEXT:    fmax.d fa5, fa3, fa5
 ; RV64-NEXT:    fmin.d fa5, fa5, fa4
-; RV64-NEXT:    fcvt.l.d a0, fa5, rtz
-; RV64-NEXT:    feq.d a2, fa3, fa3
-; RV64-NEXT:    seqz a2, a2
-; RV64-NEXT:    addiw a2, a2, -1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    sb a0, 9(sp)
-; RV64-NEXT:    addi a0, sp, 8
+; RV64-NEXT:    fcvt.l.d a2, fa5, rtz
+; RV64-NEXT:    and a0, a0, a2
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vle8.v v8, (a0)
+; RV64-NEXT:    vslide1down.vx v8, v12, a0
 ; RV64-NEXT:    vse8.v v8, (a1)
-; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
   %a = load <8 x double>, ptr %x
   %d = call <8 x i8> @llvm.fptosi.sat.v8i8.v8f64(<8 x double> %a)
@@ -522,8 +510,6 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ;
 ; RV32-LABEL: fp2ui_v8f64_v8i8:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI13_0)
@@ -533,61 +519,69 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 8(sp)
+; RV32-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v8, a0
 ; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
-; RV32-NEXT:    vslidedown.vi v12, v8, 7
-; RV32-NEXT:    vfmv.f.s fa4, v12
+; RV32-NEXT:    vslidedown.vi v16, v8, 1
+; RV32-NEXT:    vfmv.f.s fa4, v16
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 15(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 6
-; RV32-NEXT:    vfmv.f.s fa4, v12
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 2
+; RV32-NEXT:    vfmv.f.s fa4, v16
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 14(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 5
-; RV32-NEXT:    vfmv.f.s fa4, v12
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 3
+; RV32-NEXT:    vfmv.f.s fa4, v16
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 13(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 4
-; RV32-NEXT:    vfmv.f.s fa4, v12
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 4
+; RV32-NEXT:    vfmv.f.s fa4, v16
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 12(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 3
-; RV32-NEXT:    vfmv.f.s fa4, v12
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 5
+; RV32-NEXT:    vfmv.f.s fa4, v16
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 11(sp)
-; RV32-NEXT:    vslidedown.vi v12, v8, 2
-; RV32-NEXT:    vfmv.f.s fa4, v12
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v8, 6
+; RV32-NEXT:    vfmv.f.s fa4, v16
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa4, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa4, rtz
-; RV32-NEXT:    sb a0, 10(sp)
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vslide1down.vx v12, v12, a0
+; RV32-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
 ; RV32-NEXT:    vfmv.f.s fa4, v8
 ; RV32-NEXT:    fmax.d fa4, fa4, fa3
 ; RV32-NEXT:    fmin.d fa5, fa4, fa5
 ; RV32-NEXT:    fcvt.wu.d a0, fa5, rtz
-; RV32-NEXT:    sb a0, 9(sp)
-; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vle8.v v8, (a0)
+; RV32-NEXT:    vslide1down.vx v8, v12, a0
 ; RV32-NEXT:    vse8.v v8, (a1)
-; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: fp2ui_v8f64_v8i8:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
 ; RV64-NEXT:    lui a0, %hi(.LCPI13_0)
@@ -597,55 +591,65 @@ define void @fp2ui_v8f64_v8i8(ptr %x, ptr %y) {
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 8(sp)
+; RV64-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v8, a0
 ; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
-; RV64-NEXT:    vslidedown.vi v12, v8, 7
-; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    vslidedown.vi v16, v8, 1
+; RV64-NEXT:    vfmv.f.s fa4, v16
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 15(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 2
+; RV64-NEXT:    vfmv.f.s fa4, v16
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 14(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 5
-; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 3
+; RV64-NEXT:    vfmv.f.s fa4, v16
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 13(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 4
-; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 4
+; RV64-NEXT:    vfmv.f.s fa4, v16
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 12(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 3
-; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 5
+; RV64-NEXT:    vfmv.f.s fa4, v16
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 11(sp)
-; RV64-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v8, 6
+; RV64-NEXT:    vfmv.f.s fa4, v16
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa4, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa4, rtz
-; RV64-NEXT:    sb a0, 10(sp)
-; RV64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vslide1down.vx v12, v12, a0
+; RV64-NEXT:    vsetivli zero, 1, e64, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v8, v8, 7
 ; RV64-NEXT:    vfmv.f.s fa4, v8
 ; RV64-NEXT:    fmax.d fa4, fa4, fa3
 ; RV64-NEXT:    fmin.d fa5, fa4, fa5
 ; RV64-NEXT:    fcvt.lu.d a0, fa5, rtz
-; RV64-NEXT:    sb a0, 9(sp)
-; RV64-NEXT:    addi a0, sp, 8
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vle8.v v8, (a0)
+; RV64-NEXT:    vslide1down.vx v8, v12, a0
 ; RV64-NEXT:    vse8.v v8, (a1)
-; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
   %a = load <8 x double>, ptr %x
   %d = call <8 x i8> @llvm.fptoui.sat.v8i8.v8f64(<8 x double> %a)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
index 36e1fd93b2e4c..88a5dbb1386a9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp2i.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8,LMULMAX8RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8,LMULMAX8RV64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1RV64
 
 define void @fp2si_v2f32_v2i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: fp2si_v2f32_v2i32:
@@ -78,6 +78,340 @@ define <2 x i1> @fp2ui_v2f32_v2i1(<2 x float> %x) {
   ret <2 x i1> %z
 }
 
+define void @fp2si_v3f32_v3i32(ptr %x, ptr %y) {
+; LMULMAX8RV32-LABEL: fp2si_v3f32_v3i32:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV32-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; LMULMAX8RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV32-NEXT:    addi a0, a1, 8
+; LMULMAX8RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: fp2si_v3f32_v3i32:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV64-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; LMULMAX8RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV64-NEXT:    addi a0, a1, 8
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: fp2si_v3f32_v3i32:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV32-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; LMULMAX1RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV32-NEXT:    addi a0, a1, 8
+; LMULMAX1RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: fp2si_v3f32_v3i32:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV64-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; LMULMAX1RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV64-NEXT:    addi a0, a1, 8
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX1RV64-NEXT:    ret
+  %a = load <3 x float>, ptr %x
+  %d = fptosi <3 x float> %a to <3 x i32>
+  store <3 x i32> %d, ptr %y
+  ret void
+}
+
+define void @fp2ui_v3f32_v3i32(ptr %x, ptr %y) {
+; LMULMAX8RV32-LABEL: fp2ui_v3f32_v3i32:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV32-NEXT:    vfcvt.rtz.xu.f.v v8, v8
+; LMULMAX8RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV32-NEXT:    addi a0, a1, 8
+; LMULMAX8RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: fp2ui_v3f32_v3i32:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV64-NEXT:    vfcvt.rtz.xu.f.v v8, v8
+; LMULMAX8RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV64-NEXT:    addi a0, a1, 8
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: fp2ui_v3f32_v3i32:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV32-NEXT:    vfcvt.rtz.xu.f.v v8, v8
+; LMULMAX1RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV32-NEXT:    addi a0, a1, 8
+; LMULMAX1RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: fp2ui_v3f32_v3i32:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV64-NEXT:    vfcvt.rtz.xu.f.v v8, v8
+; LMULMAX1RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV64-NEXT:    addi a0, a1, 8
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX1RV64-NEXT:    ret
+  %a = load <3 x float>, ptr %x
+  %d = fptoui <3 x float> %a to <3 x i32>
+  store <3 x i32> %d, ptr %y
+  ret void
+}
+
+define <3 x i1> @fp2si_v3f32_v3i1(<3 x float> %x) {
+; CHECK-LABEL: fp2si_v3f32_v3i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; CHECK-NEXT:    vand.vi v8, v9, 1
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    ret
+  %z = fptosi <3 x float> %x to <3 x i1>
+  ret <3 x i1> %z
+}
+
+; FIXME: This is expanded when they could be widened + promoted
+define <3 x i15> @fp2si_v3f32_v3i15(<3 x float> %x) {
+; LMULMAX8RV32-LABEL: fp2si_v3f32_v3i15:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX8RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX8RV32-NEXT:    vmv.x.s a1, v8
+; LMULMAX8RV32-NEXT:    slli a2, a1, 17
+; LMULMAX8RV32-NEXT:    srli a2, a2, 19
+; LMULMAX8RV32-NEXT:    sh a2, 4(a0)
+; LMULMAX8RV32-NEXT:    vmv.x.s a2, v9
+; LMULMAX8RV32-NEXT:    lui a3, 8
+; LMULMAX8RV32-NEXT:    addi a3, a3, -1
+; LMULMAX8RV32-NEXT:    and a2, a2, a3
+; LMULMAX8RV32-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX8RV32-NEXT:    vmv.x.s a4, v8
+; LMULMAX8RV32-NEXT:    and a3, a4, a3
+; LMULMAX8RV32-NEXT:    slli a3, a3, 15
+; LMULMAX8RV32-NEXT:    slli a1, a1, 30
+; LMULMAX8RV32-NEXT:    or a1, a2, a1
+; LMULMAX8RV32-NEXT:    or a1, a1, a3
+; LMULMAX8RV32-NEXT:    sw a1, 0(a0)
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: fp2si_v3f32_v3i15:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX8RV64-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX8RV64-NEXT:    vmv.x.s a1, v9
+; LMULMAX8RV64-NEXT:    lui a2, 8
+; LMULMAX8RV64-NEXT:    addiw a2, a2, -1
+; LMULMAX8RV64-NEXT:    and a1, a1, a2
+; LMULMAX8RV64-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX8RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX8RV64-NEXT:    and a2, a3, a2
+; LMULMAX8RV64-NEXT:    slli a2, a2, 15
+; LMULMAX8RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX8RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX8RV64-NEXT:    slli a3, a3, 30
+; LMULMAX8RV64-NEXT:    or a1, a1, a3
+; LMULMAX8RV64-NEXT:    or a1, a1, a2
+; LMULMAX8RV64-NEXT:    sw a1, 0(a0)
+; LMULMAX8RV64-NEXT:    slli a1, a1, 19
+; LMULMAX8RV64-NEXT:    srli a1, a1, 51
+; LMULMAX8RV64-NEXT:    sh a1, 4(a0)
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: fp2si_v3f32_v3i15:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX1RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1RV32-NEXT:    vmv.x.s a1, v8
+; LMULMAX1RV32-NEXT:    slli a2, a1, 17
+; LMULMAX1RV32-NEXT:    srli a2, a2, 19
+; LMULMAX1RV32-NEXT:    sh a2, 4(a0)
+; LMULMAX1RV32-NEXT:    vmv.x.s a2, v9
+; LMULMAX1RV32-NEXT:    lui a3, 8
+; LMULMAX1RV32-NEXT:    addi a3, a3, -1
+; LMULMAX1RV32-NEXT:    and a2, a2, a3
+; LMULMAX1RV32-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX1RV32-NEXT:    vmv.x.s a4, v8
+; LMULMAX1RV32-NEXT:    and a3, a4, a3
+; LMULMAX1RV32-NEXT:    slli a3, a3, 15
+; LMULMAX1RV32-NEXT:    slli a1, a1, 30
+; LMULMAX1RV32-NEXT:    or a1, a2, a1
+; LMULMAX1RV32-NEXT:    or a1, a1, a3
+; LMULMAX1RV32-NEXT:    sw a1, 0(a0)
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: fp2si_v3f32_v3i15:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1RV64-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX1RV64-NEXT:    vmv.x.s a1, v9
+; LMULMAX1RV64-NEXT:    lui a2, 8
+; LMULMAX1RV64-NEXT:    addiw a2, a2, -1
+; LMULMAX1RV64-NEXT:    and a1, a1, a2
+; LMULMAX1RV64-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX1RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX1RV64-NEXT:    and a2, a3, a2
+; LMULMAX1RV64-NEXT:    slli a2, a2, 15
+; LMULMAX1RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX1RV64-NEXT:    slli a3, a3, 30
+; LMULMAX1RV64-NEXT:    or a1, a1, a3
+; LMULMAX1RV64-NEXT:    or a1, a1, a2
+; LMULMAX1RV64-NEXT:    sw a1, 0(a0)
+; LMULMAX1RV64-NEXT:    slli a1, a1, 19
+; LMULMAX1RV64-NEXT:    srli a1, a1, 51
+; LMULMAX1RV64-NEXT:    sh a1, 4(a0)
+; LMULMAX1RV64-NEXT:    ret
+  %z = fptosi <3 x float> %x to <3 x i15>
+  ret <3 x i15> %z
+}
+
+; FIXME: This is expanded when they could be widened + promoted
+define <3 x i15> @fp2ui_v3f32_v3i15(<3 x float> %x) {
+; LMULMAX8RV32-LABEL: fp2ui_v3f32_v3i15:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX8RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX8RV32-NEXT:    vmv.x.s a1, v8
+; LMULMAX8RV32-NEXT:    slli a2, a1, 17
+; LMULMAX8RV32-NEXT:    srli a2, a2, 19
+; LMULMAX8RV32-NEXT:    sh a2, 4(a0)
+; LMULMAX8RV32-NEXT:    vmv.x.s a2, v9
+; LMULMAX8RV32-NEXT:    lui a3, 16
+; LMULMAX8RV32-NEXT:    addi a3, a3, -1
+; LMULMAX8RV32-NEXT:    and a2, a2, a3
+; LMULMAX8RV32-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX8RV32-NEXT:    vmv.x.s a4, v8
+; LMULMAX8RV32-NEXT:    and a3, a4, a3
+; LMULMAX8RV32-NEXT:    slli a3, a3, 15
+; LMULMAX8RV32-NEXT:    slli a1, a1, 30
+; LMULMAX8RV32-NEXT:    or a1, a2, a1
+; LMULMAX8RV32-NEXT:    or a1, a1, a3
+; LMULMAX8RV32-NEXT:    sw a1, 0(a0)
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: fp2ui_v3f32_v3i15:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX8RV64-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX8RV64-NEXT:    vmv.x.s a1, v9
+; LMULMAX8RV64-NEXT:    lui a2, 16
+; LMULMAX8RV64-NEXT:    addiw a2, a2, -1
+; LMULMAX8RV64-NEXT:    and a1, a1, a2
+; LMULMAX8RV64-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX8RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX8RV64-NEXT:    and a2, a3, a2
+; LMULMAX8RV64-NEXT:    slli a2, a2, 15
+; LMULMAX8RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX8RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX8RV64-NEXT:    slli a3, a3, 30
+; LMULMAX8RV64-NEXT:    or a1, a1, a3
+; LMULMAX8RV64-NEXT:    or a1, a1, a2
+; LMULMAX8RV64-NEXT:    sw a1, 0(a0)
+; LMULMAX8RV64-NEXT:    slli a1, a1, 19
+; LMULMAX8RV64-NEXT:    srli a1, a1, 51
+; LMULMAX8RV64-NEXT:    sh a1, 4(a0)
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: fp2ui_v3f32_v3i15:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX1RV32-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1RV32-NEXT:    vmv.x.s a1, v8
+; LMULMAX1RV32-NEXT:    slli a2, a1, 17
+; LMULMAX1RV32-NEXT:    srli a2, a2, 19
+; LMULMAX1RV32-NEXT:    sh a2, 4(a0)
+; LMULMAX1RV32-NEXT:    vmv.x.s a2, v9
+; LMULMAX1RV32-NEXT:    lui a3, 16
+; LMULMAX1RV32-NEXT:    addi a3, a3, -1
+; LMULMAX1RV32-NEXT:    and a2, a2, a3
+; LMULMAX1RV32-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX1RV32-NEXT:    vmv.x.s a4, v8
+; LMULMAX1RV32-NEXT:    and a3, a4, a3
+; LMULMAX1RV32-NEXT:    slli a3, a3, 15
+; LMULMAX1RV32-NEXT:    slli a1, a1, 30
+; LMULMAX1RV32-NEXT:    or a1, a2, a1
+; LMULMAX1RV32-NEXT:    or a1, a1, a3
+; LMULMAX1RV32-NEXT:    sw a1, 0(a0)
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: fp2ui_v3f32_v3i15:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; LMULMAX1RV64-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; LMULMAX1RV64-NEXT:    vmv.x.s a1, v9
+; LMULMAX1RV64-NEXT:    lui a2, 16
+; LMULMAX1RV64-NEXT:    addiw a2, a2, -1
+; LMULMAX1RV64-NEXT:    and a1, a1, a2
+; LMULMAX1RV64-NEXT:    vslidedown.vi v8, v9, 1
+; LMULMAX1RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX1RV64-NEXT:    and a2, a3, a2
+; LMULMAX1RV64-NEXT:    slli a2, a2, 15
+; LMULMAX1RV64-NEXT:    vslidedown.vi v8, v9, 2
+; LMULMAX1RV64-NEXT:    vmv.x.s a3, v8
+; LMULMAX1RV64-NEXT:    slli a3, a3, 30
+; LMULMAX1RV64-NEXT:    or a1, a1, a3
+; LMULMAX1RV64-NEXT:    or a1, a1, a2
+; LMULMAX1RV64-NEXT:    sw a1, 0(a0)
+; LMULMAX1RV64-NEXT:    slli a1, a1, 19
+; LMULMAX1RV64-NEXT:    srli a1, a1, 51
+; LMULMAX1RV64-NEXT:    sh a1, 4(a0)
+; LMULMAX1RV64-NEXT:    ret
+  %z = fptoui <3 x float> %x to <3 x i15>
+  ret <3 x i15> %z
+}
+
+define <3 x i1> @fp2ui_v3f32_v3i1(<3 x float> %x) {
+; CHECK-LABEL: fp2ui_v3f32_v3i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vfncvt.rtz.xu.f.w v9, v8
+; CHECK-NEXT:    vand.vi v8, v9, 1
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    ret
+  %z = fptoui <3 x float> %x to <3 x i1>
+  ret <3 x i1> %z
+}
+
 define void @fp2si_v8f32_v8i32(ptr %x, ptr %y) {
 ; LMULMAX8-LABEL: fp2si_v8f32_v8i32:
 ; LMULMAX8:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-costrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-costrained-sdnode.ll
new file mode 100644
index 0000000000000..6e8c0b7eee014
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fround-costrained-sdnode.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+; This file tests the code generation for `llvm.experimental.constrained.round.*` on scalable vector type.
+
+define <1 x half> @round_v1f16(<1 x half> %x) {
+; CHECK-LABEL: round_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x half> @llvm.experimental.constrained.round.v1f16(<1 x half> %x, metadata !"fpexcept.strict")
+  ret <1 x half> %a
+}
+declare <1 x half> @llvm.experimental.constrained.round.v1f16(<1 x half>, metadata)
+
+define <2 x half> @round_v2f16(<2 x half> %x) {
+; CHECK-LABEL: round_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x half> @llvm.experimental.constrained.round.v2f16(<2 x half> %x, metadata !"fpexcept.strict")
+  ret <2 x half> %a
+}
+declare <2 x half> @llvm.experimental.constrained.round.v2f16(<2 x half>, metadata)
+
+define <4 x half> @round_v4f16(<4 x half> %x) {
+; CHECK-LABEL: round_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x half> @llvm.experimental.constrained.round.v4f16(<4 x half> %x, metadata !"fpexcept.strict")
+  ret <4 x half> %a
+}
+declare <4 x half> @llvm.experimental.constrained.round.v4f16(<4 x half>, metadata)
+
+define <8 x half> @round_v8f16(<8 x half> %x) {
+; CHECK-LABEL: round_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x half> @llvm.experimental.constrained.round.v8f16(<8 x half> %x, metadata !"fpexcept.strict")
+  ret <8 x half> %a
+}
+declare <8 x half> @llvm.experimental.constrained.round.v8f16(<8 x half>, metadata)
+
+define <16 x half> @round_v16f16(<16 x half> %x) {
+; CHECK-LABEL: round_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x half> @llvm.experimental.constrained.round.v16f16(<16 x half> %x, metadata !"fpexcept.strict")
+  ret <16 x half> %a
+}
+declare <16 x half> @llvm.experimental.constrained.round.v16f16(<16 x half>, metadata)
+
+define <32 x half> @round_v32f16(<32 x half> %x) {
+; CHECK-LABEL: round_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <32 x half> @llvm.experimental.constrained.round.v32f16(<32 x half> %x, metadata !"fpexcept.strict")
+  ret <32 x half> %a
+}
+declare <32 x half> @llvm.experimental.constrained.round.v32f16(<32 x half>, metadata)
+
+define <1 x float> @round_v1f32(<1 x float> %x) {
+; CHECK-LABEL: round_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float> %x, metadata !"fpexcept.strict")
+  ret <1 x float> %a
+}
+declare <1 x float> @llvm.experimental.constrained.round.v1f32(<1 x float>, metadata)
+
+define <2 x float> @round_v2f32(<2 x float> %x) {
+; CHECK-LABEL: round_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x float> @llvm.experimental.constrained.round.v2f32(<2 x float> %x, metadata !"fpexcept.strict")
+  ret <2 x float> %a
+}
+declare <2 x float> @llvm.experimental.constrained.round.v2f32(<2 x float>, metadata)
+
+define <4 x float> @round_v4f32(<4 x float> %x) {
+; CHECK-LABEL: round_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x float> @llvm.experimental.constrained.round.v4f32(<4 x float> %x, metadata !"fpexcept.strict")
+  ret <4 x float> %a
+}
+declare <4 x float> @llvm.experimental.constrained.round.v4f32(<4 x float>, metadata)
+
+define <8 x float> @round_v8f32(<8 x float> %x) {
+; CHECK-LABEL: round_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x float> @llvm.experimental.constrained.round.v8f32(<8 x float> %x, metadata !"fpexcept.strict")
+  ret <8 x float> %a
+}
+declare <8 x float> @llvm.experimental.constrained.round.v8f32(<8 x float>, metadata)
+
+define <16 x float> @round_v16f32(<16 x float> %x) {
+; CHECK-LABEL: round_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x float> @llvm.experimental.constrained.round.v16f32(<16 x float> %x, metadata !"fpexcept.strict")
+  ret <16 x float> %a
+}
+declare <16 x float> @llvm.experimental.constrained.round.v16f32(<16 x float>, metadata)
+
+define <1 x double> @round_v1f64(<1 x double> %x) {
+; CHECK-LABEL: round_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x double> @llvm.experimental.constrained.round.v1f64(<1 x double> %x, metadata !"fpexcept.strict")
+  ret <1 x double> %a
+}
+declare <1 x double> @llvm.experimental.constrained.round.v1f64(<1 x double>, metadata)
+
+define <2 x double> @round_v2f64(<2 x double> %x) {
+; CHECK-LABEL: round_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double> %x, metadata !"fpexcept.strict")
+  ret <2 x double> %a
+}
+declare <2 x double> @llvm.experimental.constrained.round.v2f64(<2 x double>, metadata)
+
+define <4 x double> @round_v4f64(<4 x double> %x) {
+; CHECK-LABEL: round_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double> %x, metadata !"fpexcept.strict")
+  ret <4 x double> %a
+}
+declare <4 x double> @llvm.experimental.constrained.round.v4f64(<4 x double>, metadata)
+
+define <8 x double> @round_v8f64(<8 x double> %x) {
+; CHECK-LABEL: round_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x double> @llvm.experimental.constrained.round.v8f64(<8 x double> %x, metadata !"fpexcept.strict")
+  ret <8 x double> %a
+}
+declare <8 x double> @llvm.experimental.constrained.round.v8f64(<8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll
new file mode 100644
index 0000000000000..a4e6fe69e5f77
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-froundeven-constrained-sdnode.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+; This file tests the code generation for `llvm.experimental.constrained.roundeven.*` on scalable vector type.
+
+define <1 x half> @roundeven_v1f16(<1 x half> %x) {
+; CHECK-LABEL: roundeven_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x half> @llvm.experimental.constrained.roundeven.v1f16(<1 x half> %x, metadata !"fpexcept.strict")
+  ret <1 x half> %a
+}
+declare <1 x half> @llvm.experimental.constrained.roundeven.v1f16(<1 x half>, metadata)
+
+define <2 x half> @roundeven_v2f16(<2 x half> %x) {
+; CHECK-LABEL: roundeven_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x half> @llvm.experimental.constrained.roundeven.v2f16(<2 x half> %x, metadata !"fpexcept.strict")
+  ret <2 x half> %a
+}
+declare <2 x half> @llvm.experimental.constrained.roundeven.v2f16(<2 x half>, metadata)
+
+define <4 x half> @roundeven_v4f16(<4 x half> %x) {
+; CHECK-LABEL: roundeven_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x half> @llvm.experimental.constrained.roundeven.v4f16(<4 x half> %x, metadata !"fpexcept.strict")
+  ret <4 x half> %a
+}
+declare <4 x half> @llvm.experimental.constrained.roundeven.v4f16(<4 x half>, metadata)
+
+define <8 x half> @roundeven_v8f16(<8 x half> %x) {
+; CHECK-LABEL: roundeven_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x half> @llvm.experimental.constrained.roundeven.v8f16(<8 x half> %x, metadata !"fpexcept.strict")
+  ret <8 x half> %a
+}
+declare <8 x half> @llvm.experimental.constrained.roundeven.v8f16(<8 x half>, metadata)
+
+define <16 x half> @roundeven_v16f16(<16 x half> %x) {
+; CHECK-LABEL: roundeven_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x half> @llvm.experimental.constrained.roundeven.v16f16(<16 x half> %x, metadata !"fpexcept.strict")
+  ret <16 x half> %a
+}
+declare <16 x half> @llvm.experimental.constrained.roundeven.v16f16(<16 x half>, metadata)
+
+define <32 x half> @roundeven_v32f16(<32 x half> %x) {
+; CHECK-LABEL: roundeven_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <32 x half> @llvm.experimental.constrained.roundeven.v32f16(<32 x half> %x, metadata !"fpexcept.strict")
+  ret <32 x half> %a
+}
+declare <32 x half> @llvm.experimental.constrained.roundeven.v32f16(<32 x half>, metadata)
+
+define <1 x float> @roundeven_v1f32(<1 x float> %x) {
+; CHECK-LABEL: roundeven_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x float> @llvm.experimental.constrained.roundeven.v1f32(<1 x float> %x, metadata !"fpexcept.strict")
+  ret <1 x float> %a
+}
+declare <1 x float> @llvm.experimental.constrained.roundeven.v1f32(<1 x float>, metadata)
+
+define <2 x float> @roundeven_v2f32(<2 x float> %x) {
+; CHECK-LABEL: roundeven_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x float> @llvm.experimental.constrained.roundeven.v2f32(<2 x float> %x, metadata !"fpexcept.strict")
+  ret <2 x float> %a
+}
+declare <2 x float> @llvm.experimental.constrained.roundeven.v2f32(<2 x float>, metadata)
+
+define <4 x float> @roundeven_v4f32(<4 x float> %x) {
+; CHECK-LABEL: roundeven_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x float> @llvm.experimental.constrained.roundeven.v4f32(<4 x float> %x, metadata !"fpexcept.strict")
+  ret <4 x float> %a
+}
+declare <4 x float> @llvm.experimental.constrained.roundeven.v4f32(<4 x float>, metadata)
+
+define <8 x float> @roundeven_v8f32(<8 x float> %x) {
+; CHECK-LABEL: roundeven_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x float> @llvm.experimental.constrained.roundeven.v8f32(<8 x float> %x, metadata !"fpexcept.strict")
+  ret <8 x float> %a
+}
+declare <8 x float> @llvm.experimental.constrained.roundeven.v8f32(<8 x float>, metadata)
+
+define <16 x float> @roundeven_v16f32(<16 x float> %x) {
+; CHECK-LABEL: roundeven_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x float> @llvm.experimental.constrained.roundeven.v16f32(<16 x float> %x, metadata !"fpexcept.strict")
+  ret <16 x float> %a
+}
+declare <16 x float> @llvm.experimental.constrained.roundeven.v16f32(<16 x float>, metadata)
+
+define <1 x double> @roundeven_v1f64(<1 x double> %x) {
+; CHECK-LABEL: roundeven_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x double> @llvm.experimental.constrained.roundeven.v1f64(<1 x double> %x, metadata !"fpexcept.strict")
+  ret <1 x double> %a
+}
+declare <1 x double> @llvm.experimental.constrained.roundeven.v1f64(<1 x double>, metadata)
+
+define <2 x double> @roundeven_v2f64(<2 x double> %x) {
+; CHECK-LABEL: roundeven_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x double> @llvm.experimental.constrained.roundeven.v2f64(<2 x double> %x, metadata !"fpexcept.strict")
+  ret <2 x double> %a
+}
+declare <2 x double> @llvm.experimental.constrained.roundeven.v2f64(<2 x double>, metadata)
+
+define <4 x double> @roundeven_v4f64(<4 x double> %x) {
+; CHECK-LABEL: roundeven_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x double> @llvm.experimental.constrained.roundeven.v4f64(<4 x double> %x, metadata !"fpexcept.strict")
+  ret <4 x double> %a
+}
+declare <4 x double> @llvm.experimental.constrained.roundeven.v4f64(<4 x double>, metadata)
+
+define <8 x double> @roundeven_v8f64(<8 x double> %x) {
+; CHECK-LABEL: roundeven_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x double> @llvm.experimental.constrained.roundeven.v8f64(<8 x double> %x, metadata !"fpexcept.strict")
+  ret <8 x double> %a
+}
+declare <8 x double> @llvm.experimental.constrained.roundeven.v8f64(<8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll
new file mode 100644
index 0000000000000..7c43e28640f3d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ftrunc-constrained-sdnode.ll
@@ -0,0 +1,306 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <1 x half> @trunc_v1f16(<1 x half> %x) {
+; CHECK-LABEL: trunc_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x half> @llvm.experimental.constrained.trunc.v1f16(<1 x half> %x, metadata !"fpexcept.strict")
+  ret <1 x half> %a
+}
+declare <1 x half> @llvm.experimental.constrained.trunc.v1f16(<1 x half>, metadata)
+
+define <2 x half> @trunc_v2f16(<2 x half> %x) {
+; CHECK-LABEL: trunc_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x half> @llvm.experimental.constrained.trunc.v2f16(<2 x half> %x, metadata !"fpexcept.strict")
+  ret <2 x half> %a
+}
+declare <2 x half> @llvm.experimental.constrained.trunc.v2f16(<2 x half>, metadata)
+
+define <4 x half> @trunc_v4f16(<4 x half> %x) {
+; CHECK-LABEL: trunc_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x half> @llvm.experimental.constrained.trunc.v4f16(<4 x half> %x, metadata !"fpexcept.strict")
+  ret <4 x half> %a
+}
+declare <4 x half> @llvm.experimental.constrained.trunc.v4f16(<4 x half>, metadata)
+
+define <8 x half> @trunc_v8f16(<8 x half> %x) {
+; CHECK-LABEL: trunc_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x half> @llvm.experimental.constrained.trunc.v8f16(<8 x half> %x, metadata !"fpexcept.strict")
+  ret <8 x half> %a
+}
+declare <8 x half> @llvm.experimental.constrained.trunc.v8f16(<8 x half>, metadata)
+
+define <16 x half> @trunc_v16f16(<16 x half> %x) {
+; CHECK-LABEL: trunc_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x half> @llvm.experimental.constrained.trunc.v16f16(<16 x half> %x, metadata !"fpexcept.strict")
+  ret <16 x half> %a
+}
+declare <16 x half> @llvm.experimental.constrained.trunc.v16f16(<16 x half>, metadata)
+
+define <32 x half> @trunc_v32f16(<32 x half> %x) {
+; CHECK-LABEL: trunc_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <32 x half> @llvm.experimental.constrained.trunc.v32f16(<32 x half> %x, metadata !"fpexcept.strict")
+  ret <32 x half> %a
+}
+declare <32 x half> @llvm.experimental.constrained.trunc.v32f16(<32 x half>, metadata)
+
+define <1 x float> @trunc_v1f32(<1 x float> %x) {
+; CHECK-LABEL: trunc_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float> %x, metadata !"fpexcept.strict")
+  ret <1 x float> %a
+}
+declare <1 x float> @llvm.experimental.constrained.trunc.v1f32(<1 x float>, metadata)
+
+define <2 x float> @trunc_v2f32(<2 x float> %x) {
+; CHECK-LABEL: trunc_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x float> @llvm.experimental.constrained.trunc.v2f32(<2 x float> %x, metadata !"fpexcept.strict")
+  ret <2 x float> %a
+}
+declare <2 x float> @llvm.experimental.constrained.trunc.v2f32(<2 x float>, metadata)
+
+define <4 x float> @trunc_v4f32(<4 x float> %x) {
+; CHECK-LABEL: trunc_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x float> @llvm.experimental.constrained.trunc.v4f32(<4 x float> %x, metadata !"fpexcept.strict")
+  ret <4 x float> %a
+}
+declare <4 x float> @llvm.experimental.constrained.trunc.v4f32(<4 x float>, metadata)
+
+define <8 x float> @trunc_v8f32(<8 x float> %x) {
+; CHECK-LABEL: trunc_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x float> @llvm.experimental.constrained.trunc.v8f32(<8 x float> %x, metadata !"fpexcept.strict")
+  ret <8 x float> %a
+}
+declare <8 x float> @llvm.experimental.constrained.trunc.v8f32(<8 x float>, metadata)
+
+define <16 x float> @trunc_v16f32(<16 x float> %x) {
+; CHECK-LABEL: trunc_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <16 x float> @llvm.experimental.constrained.trunc.v16f32(<16 x float> %x, metadata !"fpexcept.strict")
+  ret <16 x float> %a
+}
+declare <16 x float> @llvm.experimental.constrained.trunc.v16f32(<16 x float>, metadata)
+
+define <1 x double> @trunc_v1f64(<1 x double> %x) {
+; CHECK-LABEL: trunc_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <1 x double> @llvm.experimental.constrained.trunc.v1f64(<1 x double> %x, metadata !"fpexcept.strict")
+  ret <1 x double> %a
+}
+declare <1 x double> @llvm.experimental.constrained.trunc.v1f64(<1 x double>, metadata)
+
+define <2 x double> @trunc_v2f64(<2 x double> %x) {
+; CHECK-LABEL: trunc_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double> %x, metadata !"fpexcept.strict")
+  ret <2 x double> %a
+}
+declare <2 x double> @llvm.experimental.constrained.trunc.v2f64(<2 x double>, metadata)
+
+define <4 x double> @trunc_v4f64(<4 x double> %x) {
+; CHECK-LABEL: trunc_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double> %x, metadata !"fpexcept.strict")
+  ret <4 x double> %a
+}
+declare <4 x double> @llvm.experimental.constrained.trunc.v4f64(<4 x double>, metadata)
+
+define <8 x double> @trunc_v8f64(<8 x double> %x) {
+; CHECK-LABEL: trunc_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <8 x double> @llvm.experimental.constrained.trunc.v8f64(<8 x double> %x, metadata !"fpexcept.strict")
+  ret <8 x double> %a
+}
+declare <8 x double> @llvm.experimental.constrained.trunc.v8f64(<8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
index 18fcac49d7340..b60f9405a760f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-i2fp.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8,LMULMAX8RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=8 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX8,LMULMAX8RV64
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -riscv-v-fixed-length-vector-lmul-max=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LMULMAX1,LMULMAX1RV64
 
 define void @si2fp_v2i32_v2f32(ptr %x, ptr %y) {
 ; CHECK-LABEL: si2fp_v2i32_v2f32:
@@ -84,6 +84,292 @@ define <2 x float> @ui2fp_v2i1_v2f32(<2 x i1> %x) {
   ret <2 x float> %z
 }
 
+define void @si2fp_v3i32_v3f32(ptr %x, ptr %y) {
+; LMULMAX8RV32-LABEL: si2fp_v3i32_v3f32:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV32-NEXT:    vfcvt.f.x.v v8, v8
+; LMULMAX8RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV32-NEXT:    addi a0, a1, 8
+; LMULMAX8RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: si2fp_v3i32_v3f32:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV64-NEXT:    vfcvt.f.x.v v8, v8
+; LMULMAX8RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV64-NEXT:    addi a0, a1, 8
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: si2fp_v3i32_v3f32:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV32-NEXT:    vfcvt.f.x.v v8, v8
+; LMULMAX1RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV32-NEXT:    addi a0, a1, 8
+; LMULMAX1RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: si2fp_v3i32_v3f32:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV64-NEXT:    vfcvt.f.x.v v8, v8
+; LMULMAX1RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV64-NEXT:    addi a0, a1, 8
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX1RV64-NEXT:    ret
+  %a = load <3 x i32>, ptr %x
+  %d = sitofp <3 x i32> %a to <3 x float>
+  store <3 x float> %d, ptr %y
+  ret void
+}
+
+define void @ui2fp_v3i32_v3f32(ptr %x, ptr %y) {
+; LMULMAX8RV32-LABEL: ui2fp_v3i32_v3f32:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV32-NEXT:    vfcvt.f.xu.v v8, v8
+; LMULMAX8RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV32-NEXT:    addi a0, a1, 8
+; LMULMAX8RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: ui2fp_v3i32_v3f32:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX8RV64-NEXT:    vfcvt.f.xu.v v8, v8
+; LMULMAX8RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX8RV64-NEXT:    addi a0, a1, 8
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX8RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX8RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: ui2fp_v3i32_v3f32:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV32-NEXT:    vfcvt.f.xu.v v8, v8
+; LMULMAX1RV32-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV32-NEXT:    addi a0, a1, 8
+; LMULMAX1RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vse32.v v8, (a1)
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: ui2fp_v3i32_v3f32:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX1RV64-NEXT:    vfcvt.f.xu.v v8, v8
+; LMULMAX1RV64-NEXT:    vslidedown.vi v9, v8, 2
+; LMULMAX1RV64-NEXT:    addi a0, a1, 8
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse32.v v9, (a0)
+; LMULMAX1RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1RV64-NEXT:    vse64.v v8, (a1)
+; LMULMAX1RV64-NEXT:    ret
+  %a = load <3 x i32>, ptr %x
+  %d = uitofp <3 x i32> %a to <3 x float>
+  store <3 x float> %d, ptr %y
+  ret void
+}
+
+define <3 x float> @si2fp_v3i1_v3f32(<3 x i1> %x) {
+; CHECK-LABEL: si2fp_v3i1_v3f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v9, v8, -1, v0
+; CHECK-NEXT:    vfwcvt.f.x.v v8, v9
+; CHECK-NEXT:    ret
+  %z = sitofp <3 x i1> %x to <3 x float>
+  ret <3 x float> %z
+}
+
+; FIXME: This gets expanded instead of widened + promoted
+define <3 x float> @si2fp_v3i7_v3f32(<3 x i7> %x) {
+; LMULMAX8RV32-LABEL: si2fp_v3i7_v3f32:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    lw a1, 0(a0)
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX8RV32-NEXT:    lw a2, 4(a0)
+; LMULMAX8RV32-NEXT:    lw a0, 8(a0)
+; LMULMAX8RV32-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX8RV32-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX8RV32-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX8RV32-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX8RV32-NEXT:    vadd.vv v8, v8, v8
+; LMULMAX8RV32-NEXT:    vsra.vi v8, v8, 1
+; LMULMAX8RV32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vsext.vf2 v9, v8
+; LMULMAX8RV32-NEXT:    vfwcvt.f.x.v v8, v9
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: si2fp_v3i7_v3f32:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    ld a1, 0(a0)
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX8RV64-NEXT:    ld a2, 8(a0)
+; LMULMAX8RV64-NEXT:    ld a0, 16(a0)
+; LMULMAX8RV64-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX8RV64-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX8RV64-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX8RV64-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX8RV64-NEXT:    vadd.vv v8, v8, v8
+; LMULMAX8RV64-NEXT:    vsra.vi v8, v8, 1
+; LMULMAX8RV64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX8RV64-NEXT:    vsext.vf2 v9, v8
+; LMULMAX8RV64-NEXT:    vfwcvt.f.x.v v8, v9
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: si2fp_v3i7_v3f32:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    lw a1, 0(a0)
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX1RV32-NEXT:    lw a2, 4(a0)
+; LMULMAX1RV32-NEXT:    lw a0, 8(a0)
+; LMULMAX1RV32-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX1RV32-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX1RV32-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX1RV32-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX1RV32-NEXT:    vadd.vv v8, v8, v8
+; LMULMAX1RV32-NEXT:    vsra.vi v8, v8, 1
+; LMULMAX1RV32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vsext.vf2 v9, v8
+; LMULMAX1RV32-NEXT:    vfwcvt.f.x.v v8, v9
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: si2fp_v3i7_v3f32:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    ld a1, 0(a0)
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX1RV64-NEXT:    ld a2, 8(a0)
+; LMULMAX1RV64-NEXT:    ld a0, 16(a0)
+; LMULMAX1RV64-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX1RV64-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX1RV64-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX1RV64-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX1RV64-NEXT:    vadd.vv v8, v8, v8
+; LMULMAX1RV64-NEXT:    vsra.vi v8, v8, 1
+; LMULMAX1RV64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX1RV64-NEXT:    vsext.vf2 v9, v8
+; LMULMAX1RV64-NEXT:    vfwcvt.f.x.v v8, v9
+; LMULMAX1RV64-NEXT:    ret
+  %z = sitofp <3 x i7> %x to <3 x float>
+  ret <3 x float> %z
+}
+
+; FIXME: This gets expanded instead of widened + promoted
+define <3 x float> @ui2fp_v3i7_v3f32(<3 x i7> %x) {
+; LMULMAX8RV32-LABEL: ui2fp_v3i7_v3f32:
+; LMULMAX8RV32:       # %bb.0:
+; LMULMAX8RV32-NEXT:    lw a1, 0(a0)
+; LMULMAX8RV32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX8RV32-NEXT:    lw a2, 4(a0)
+; LMULMAX8RV32-NEXT:    lw a0, 8(a0)
+; LMULMAX8RV32-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX8RV32-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX8RV32-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX8RV32-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX8RV32-NEXT:    li a0, 127
+; LMULMAX8RV32-NEXT:    vand.vx v8, v8, a0
+; LMULMAX8RV32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX8RV32-NEXT:    vzext.vf2 v9, v8
+; LMULMAX8RV32-NEXT:    vfwcvt.f.xu.v v8, v9
+; LMULMAX8RV32-NEXT:    ret
+;
+; LMULMAX8RV64-LABEL: ui2fp_v3i7_v3f32:
+; LMULMAX8RV64:       # %bb.0:
+; LMULMAX8RV64-NEXT:    ld a1, 0(a0)
+; LMULMAX8RV64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX8RV64-NEXT:    ld a2, 8(a0)
+; LMULMAX8RV64-NEXT:    ld a0, 16(a0)
+; LMULMAX8RV64-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX8RV64-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX8RV64-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX8RV64-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX8RV64-NEXT:    li a0, 127
+; LMULMAX8RV64-NEXT:    vand.vx v8, v8, a0
+; LMULMAX8RV64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX8RV64-NEXT:    vzext.vf2 v9, v8
+; LMULMAX8RV64-NEXT:    vfwcvt.f.xu.v v8, v9
+; LMULMAX8RV64-NEXT:    ret
+;
+; LMULMAX1RV32-LABEL: ui2fp_v3i7_v3f32:
+; LMULMAX1RV32:       # %bb.0:
+; LMULMAX1RV32-NEXT:    lw a1, 0(a0)
+; LMULMAX1RV32-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX1RV32-NEXT:    lw a2, 4(a0)
+; LMULMAX1RV32-NEXT:    lw a0, 8(a0)
+; LMULMAX1RV32-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX1RV32-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX1RV32-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX1RV32-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX1RV32-NEXT:    li a0, 127
+; LMULMAX1RV32-NEXT:    vand.vx v8, v8, a0
+; LMULMAX1RV32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX1RV32-NEXT:    vzext.vf2 v9, v8
+; LMULMAX1RV32-NEXT:    vfwcvt.f.xu.v v8, v9
+; LMULMAX1RV32-NEXT:    ret
+;
+; LMULMAX1RV64-LABEL: ui2fp_v3i7_v3f32:
+; LMULMAX1RV64:       # %bb.0:
+; LMULMAX1RV64-NEXT:    ld a1, 0(a0)
+; LMULMAX1RV64-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; LMULMAX1RV64-NEXT:    ld a2, 8(a0)
+; LMULMAX1RV64-NEXT:    ld a0, 16(a0)
+; LMULMAX1RV64-NEXT:    vslide1down.vx v8, v8, a1
+; LMULMAX1RV64-NEXT:    vslide1down.vx v8, v8, a2
+; LMULMAX1RV64-NEXT:    vslide1down.vx v8, v8, a0
+; LMULMAX1RV64-NEXT:    vslidedown.vi v8, v8, 1
+; LMULMAX1RV64-NEXT:    li a0, 127
+; LMULMAX1RV64-NEXT:    vand.vx v8, v8, a0
+; LMULMAX1RV64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; LMULMAX1RV64-NEXT:    vzext.vf2 v9, v8
+; LMULMAX1RV64-NEXT:    vfwcvt.f.xu.v v8, v9
+; LMULMAX1RV64-NEXT:    ret
+  %z = uitofp <3 x i7> %x to <3 x float>
+  ret <3 x float> %z
+}
+
+define <3 x float> @ui2fp_v3i1_v3f32(<3 x i1> %x) {
+; CHECK-LABEL: ui2fp_v3i1_v3f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v9, v8, 1, v0
+; CHECK-NEXT:    vfwcvt.f.xu.v v8, v9
+; CHECK-NEXT:    ret
+  %z = uitofp <3 x i1> %x to <3 x float>
+  ret <3 x float> %z
+}
+
 define void @si2fp_v8i32_v8f32(ptr %x, ptr %y) {
 ; LMULMAX8-LABEL: si2fp_v8i32_v8f32:
 ; LMULMAX8:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll
index e25ffcc0922e3..ab8f45f29e726 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-interleave.ll
@@ -172,6 +172,58 @@ define <8 x i32> @interleave_v4i32(<4 x i32> %x, <4 x i32> %y) {
   ret <8 x i32> %a
 }
 
+; %y should be slid down by 2
+define <4 x i32> @interleave_v4i32_offset_2(<4 x i32> %x, <4 x i32> %y) {
+; V128-LABEL: interleave_v4i32_offset_2:
+; V128:       # %bb.0:
+; V128-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
+; V128-NEXT:    vslidedown.vi v10, v9, 2
+; V128-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; V128-NEXT:    vwaddu.vv v9, v8, v10
+; V128-NEXT:    li a0, -1
+; V128-NEXT:    vwmaccu.vx v9, a0, v10
+; V128-NEXT:    vmv1r.v v8, v9
+; V128-NEXT:    ret
+;
+; V512-LABEL: interleave_v4i32_offset_2:
+; V512:       # %bb.0:
+; V512-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; V512-NEXT:    vslidedown.vi v10, v9, 2
+; V512-NEXT:    vwaddu.vv v9, v8, v10
+; V512-NEXT:    li a0, -1
+; V512-NEXT:    vwmaccu.vx v9, a0, v10
+; V512-NEXT:    vmv1r.v v8, v9
+; V512-NEXT:    ret
+  %a = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 6, i32 1, i32 7>
+  ret <4 x i32> %a
+}
+
+; %y should be slid down by 1
+define <4 x i32> @interleave_v4i32_offset_1(<4 x i32> %x, <4 x i32> %y) {
+; V128-LABEL: interleave_v4i32_offset_1:
+; V128:       # %bb.0:
+; V128-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
+; V128-NEXT:    vslidedown.vi v10, v9, 1
+; V128-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; V128-NEXT:    vwaddu.vv v9, v8, v10
+; V128-NEXT:    li a0, -1
+; V128-NEXT:    vwmaccu.vx v9, a0, v10
+; V128-NEXT:    vmv1r.v v8, v9
+; V128-NEXT:    ret
+;
+; V512-LABEL: interleave_v4i32_offset_1:
+; V512:       # %bb.0:
+; V512-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; V512-NEXT:    vslidedown.vi v10, v9, 1
+; V512-NEXT:    vwaddu.vv v9, v8, v10
+; V512-NEXT:    li a0, -1
+; V512-NEXT:    vwmaccu.vx v9, a0, v10
+; V512-NEXT:    vmv1r.v v8, v9
+; V512-NEXT:    ret
+  %a = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 1, i32 6>
+  ret <4 x i32> %a
+}
+
 define <16 x i8> @interleave_v8i8(<8 x i8> %x, <8 x i8> %y) {
 ; V128-LABEL: interleave_v8i8:
 ; V128:       # %bb.0:
@@ -362,8 +414,8 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; RV32-V128-NEXT:    slli a0, a0, 4
 ; RV32-V128-NEXT:    sub sp, sp, a0
 ; RV32-V128-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; RV32-V128-NEXT:    lui a0, %hi(.LCPI15_0)
-; RV32-V128-NEXT:    addi a0, a0, %lo(.LCPI15_0)
+; RV32-V128-NEXT:    lui a0, %hi(.LCPI17_0)
+; RV32-V128-NEXT:    addi a0, a0, %lo(.LCPI17_0)
 ; RV32-V128-NEXT:    li a1, 32
 ; RV32-V128-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; RV32-V128-NEXT:    vle32.v v0, (a0)
@@ -371,8 +423,8 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; RV32-V128-NEXT:    vrgather.vv v8, v24, v0
 ; RV32-V128-NEXT:    addi a0, sp, 16
 ; RV32-V128-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; RV32-V128-NEXT:    lui a0, %hi(.LCPI15_1)
-; RV32-V128-NEXT:    addi a0, a0, %lo(.LCPI15_1)
+; RV32-V128-NEXT:    lui a0, %hi(.LCPI17_1)
+; RV32-V128-NEXT:    addi a0, a0, %lo(.LCPI17_1)
 ; RV32-V128-NEXT:    vle32.v v24, (a0)
 ; RV32-V128-NEXT:    csrr a0, vlenb
 ; RV32-V128-NEXT:    slli a0, a0, 3
@@ -413,8 +465,8 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; RV64-V128-NEXT:    slli a0, a0, 4
 ; RV64-V128-NEXT:    sub sp, sp, a0
 ; RV64-V128-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; RV64-V128-NEXT:    lui a0, %hi(.LCPI15_0)
-; RV64-V128-NEXT:    addi a0, a0, %lo(.LCPI15_0)
+; RV64-V128-NEXT:    lui a0, %hi(.LCPI17_0)
+; RV64-V128-NEXT:    addi a0, a0, %lo(.LCPI17_0)
 ; RV64-V128-NEXT:    li a1, 32
 ; RV64-V128-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; RV64-V128-NEXT:    vle32.v v0, (a0)
@@ -422,8 +474,8 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; RV64-V128-NEXT:    vrgather.vv v8, v24, v0
 ; RV64-V128-NEXT:    addi a0, sp, 16
 ; RV64-V128-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; RV64-V128-NEXT:    lui a0, %hi(.LCPI15_1)
-; RV64-V128-NEXT:    addi a0, a0, %lo(.LCPI15_1)
+; RV64-V128-NEXT:    lui a0, %hi(.LCPI17_1)
+; RV64-V128-NEXT:    addi a0, a0, %lo(.LCPI17_1)
 ; RV64-V128-NEXT:    vle32.v v24, (a0)
 ; RV64-V128-NEXT:    csrr a0, vlenb
 ; RV64-V128-NEXT:    slli a0, a0, 3
@@ -494,6 +546,31 @@ define <4 x i8> @unary_interleave_v4i8(<4 x i8> %x) {
   ret <4 x i8> %a
 }
 
+; This shouldn't be interleaved
+define <4 x i8> @unary_interleave_v4i8_invalid(<4 x i8> %x) {
+; V128-LABEL: unary_interleave_v4i8_invalid:
+; V128:       # %bb.0:
+; V128-NEXT:    lui a0, %hi(.LCPI19_0)
+; V128-NEXT:    addi a0, a0, %lo(.LCPI19_0)
+; V128-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; V128-NEXT:    vle8.v v10, (a0)
+; V128-NEXT:    vrgather.vv v9, v8, v10
+; V128-NEXT:    vmv1r.v v8, v9
+; V128-NEXT:    ret
+;
+; V512-LABEL: unary_interleave_v4i8_invalid:
+; V512:       # %bb.0:
+; V512-NEXT:    lui a0, %hi(.LCPI19_0)
+; V512-NEXT:    addi a0, a0, %lo(.LCPI19_0)
+; V512-NEXT:    vsetivli zero, 4, e8, mf8, ta, ma
+; V512-NEXT:    vle8.v v10, (a0)
+; V512-NEXT:    vrgather.vv v9, v8, v10
+; V512-NEXT:    vmv1r.v v8, v9
+; V512-NEXT:    ret
+  %a = shufflevector <4 x i8> %x, <4 x i8> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 4>
+  ret <4 x i8> %a
+}
+
 define <4 x i16> @unary_interleave_v4i16(<4 x i16> %x) {
 ; V128-LABEL: unary_interleave_v4i16:
 ; V128:       # %bb.0:
@@ -548,8 +625,8 @@ define <4 x i32> @unary_interleave_v4i32(<4 x i32> %x) {
 define <4 x i64> @unary_interleave_v4i64(<4 x i64> %x) {
 ; RV32-V128-LABEL: unary_interleave_v4i64:
 ; RV32-V128:       # %bb.0:
-; RV32-V128-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV32-V128-NEXT:    addi a0, a0, %lo(.LCPI19_0)
+; RV32-V128-NEXT:    lui a0, %hi(.LCPI22_0)
+; RV32-V128-NEXT:    addi a0, a0, %lo(.LCPI22_0)
 ; RV32-V128-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV32-V128-NEXT:    vle16.v v12, (a0)
 ; RV32-V128-NEXT:    vrgatherei16.vv v10, v8, v12
@@ -558,8 +635,8 @@ define <4 x i64> @unary_interleave_v4i64(<4 x i64> %x) {
 ;
 ; RV64-V128-LABEL: unary_interleave_v4i64:
 ; RV64-V128:       # %bb.0:
-; RV64-V128-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64-V128-NEXT:    addi a0, a0, %lo(.LCPI19_0)
+; RV64-V128-NEXT:    lui a0, %hi(.LCPI22_0)
+; RV64-V128-NEXT:    addi a0, a0, %lo(.LCPI22_0)
 ; RV64-V128-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; RV64-V128-NEXT:    vle64.v v12, (a0)
 ; RV64-V128-NEXT:    vrgather.vv v10, v8, v12
@@ -568,8 +645,8 @@ define <4 x i64> @unary_interleave_v4i64(<4 x i64> %x) {
 ;
 ; RV32-V512-LABEL: unary_interleave_v4i64:
 ; RV32-V512:       # %bb.0:
-; RV32-V512-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV32-V512-NEXT:    addi a0, a0, %lo(.LCPI19_0)
+; RV32-V512-NEXT:    lui a0, %hi(.LCPI22_0)
+; RV32-V512-NEXT:    addi a0, a0, %lo(.LCPI22_0)
 ; RV32-V512-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
 ; RV32-V512-NEXT:    vle16.v v10, (a0)
 ; RV32-V512-NEXT:    vrgatherei16.vv v9, v8, v10
@@ -578,8 +655,8 @@ define <4 x i64> @unary_interleave_v4i64(<4 x i64> %x) {
 ;
 ; RV64-V512-LABEL: unary_interleave_v4i64:
 ; RV64-V512:       # %bb.0:
-; RV64-V512-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64-V512-NEXT:    addi a0, a0, %lo(.LCPI19_0)
+; RV64-V512-NEXT:    lui a0, %hi(.LCPI22_0)
+; RV64-V512-NEXT:    addi a0, a0, %lo(.LCPI22_0)
 ; RV64-V512-NEXT:    vsetivli zero, 4, e64, m1, ta, ma
 ; RV64-V512-NEXT:    vle64.v v10, (a0)
 ; RV64-V512-NEXT:    vrgather.vv v9, v8, v10
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-setcc.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-setcc.ll
index 578204eb8c296..f70ab9d5035ed 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-setcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-setcc.ll
@@ -587,7 +587,7 @@ define void @setlt_vi_v128i8(ptr %x, ptr %z) {
 ; CHECK-NEXT:    li a2, 128
 ; CHECK-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vmslt.vx v16, v8, zero
+; CHECK-NEXT:    vmsle.vi v16, v8, -1
 ; CHECK-NEXT:    vsm.v v16, (a1)
 ; CHECK-NEXT:    ret
   %a = load <128 x i8>, ptr %x
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
index 42be739ce9cd7..1788d957229b3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll
@@ -38,6 +38,41 @@ define void @add_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @add_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: add_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: add_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vadd.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = add <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @add_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: add_v4i32:
 ; CHECK:       # %bb.0:
@@ -102,6 +137,41 @@ define void @sub_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @sub_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: sub_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vsub.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sub_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vsub.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = sub <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @sub_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: sub_v4i32:
 ; CHECK:       # %bb.0:
@@ -166,6 +236,41 @@ define void @mul_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @mul_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: mul_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vmul.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mul_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vmul.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = mul <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @mul_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: mul_v4i32:
 ; CHECK:       # %bb.0:
@@ -230,6 +335,41 @@ define void @and_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @and_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: and_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vand.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: and_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vand.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = and <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @and_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: and_v4i32:
 ; CHECK:       # %bb.0:
@@ -294,6 +434,41 @@ define void @or_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @or_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: or_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vor.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: or_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vor.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = or <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @or_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: or_v4i32:
 ; CHECK:       # %bb.0:
@@ -358,6 +533,41 @@ define void @xor_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @xor_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: xor_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vxor.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: xor_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vxor.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = xor <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @xor_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: xor_v4i32:
 ; CHECK:       # %bb.0:
@@ -422,6 +632,41 @@ define void @lshr_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @lshr_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: lshr_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vsrl.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lshr_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vsrl.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = lshr <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @lshr_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: lshr_v4i32:
 ; CHECK:       # %bb.0:
@@ -486,6 +731,41 @@ define void @ashr_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @ashr_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: ashr_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vsra.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: ashr_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vsra.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = ashr <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @ashr_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: ashr_v4i32:
 ; CHECK:       # %bb.0:
@@ -550,6 +830,41 @@ define void @shl_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @shl_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: shl_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vsll.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: shl_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vsll.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = shl <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @shl_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: shl_v4i32:
 ; CHECK:       # %bb.0:
@@ -614,6 +929,56 @@ define void @sdiv_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @sdiv_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: sdiv_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a1)
+; RV32-NEXT:    vle16.v v9, (a0)
+; RV32-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    vslidedown.vi v11, v9, 4
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vdiv.vv v10, v11, v10
+; RV32-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v11, v10, 4
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vdiv.vv v8, v9, v8
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v11, 2
+; RV32-NEXT:    addi a0, a0, 8
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sdiv_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a1)
+; RV64-NEXT:    vle16.v v9, (a0)
+; RV64-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 4
+; RV64-NEXT:    vslidedown.vi v11, v9, 4
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vdiv.vv v10, v11, v10
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vdiv.vv v8, v9, v8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v10, 4
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = sdiv <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @sdiv_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: sdiv_v4i32:
 ; CHECK:       # %bb.0:
@@ -678,6 +1043,56 @@ define void @srem_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @srem_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: srem_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a1)
+; RV32-NEXT:    vle16.v v9, (a0)
+; RV32-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    vslidedown.vi v11, v9, 4
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vrem.vv v10, v11, v10
+; RV32-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v11, v10, 4
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vrem.vv v8, v9, v8
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v11, 2
+; RV32-NEXT:    addi a0, a0, 8
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: srem_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a1)
+; RV64-NEXT:    vle16.v v9, (a0)
+; RV64-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 4
+; RV64-NEXT:    vslidedown.vi v11, v9, 4
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vrem.vv v10, v11, v10
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vrem.vv v8, v9, v8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v10, 4
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = srem <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @srem_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: srem_v4i32:
 ; CHECK:       # %bb.0:
@@ -742,6 +1157,56 @@ define void @udiv_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @udiv_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: udiv_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a1)
+; RV32-NEXT:    vle16.v v9, (a0)
+; RV32-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    vslidedown.vi v11, v9, 4
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vdivu.vv v10, v11, v10
+; RV32-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v11, v10, 4
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vdivu.vv v8, v9, v8
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v11, 2
+; RV32-NEXT:    addi a0, a0, 8
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: udiv_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a1)
+; RV64-NEXT:    vle16.v v9, (a0)
+; RV64-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 4
+; RV64-NEXT:    vslidedown.vi v11, v9, 4
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vdivu.vv v10, v11, v10
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vdivu.vv v8, v9, v8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v10, 4
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = udiv <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @udiv_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: udiv_v4i32:
 ; CHECK:       # %bb.0:
@@ -806,6 +1271,56 @@ define void @urem_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @urem_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: urem_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a1)
+; RV32-NEXT:    vle16.v v9, (a0)
+; RV32-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    vslidedown.vi v11, v9, 4
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vremu.vv v10, v11, v10
+; RV32-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v11, v10, 4
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vremu.vv v8, v9, v8
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v11, 2
+; RV32-NEXT:    addi a0, a0, 8
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: urem_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a1)
+; RV64-NEXT:    vle16.v v9, (a0)
+; RV64-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 4
+; RV64-NEXT:    vslidedown.vi v11, v9, 4
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vremu.vv v10, v11, v10
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vremu.vv v8, v9, v8
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v10, 4
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %c = urem <6 x i16> %a, %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @urem_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: urem_v4i32:
 ; CHECK:       # %bb.0:
@@ -873,8 +1388,8 @@ define void @mulhu_v16i8(ptr %x) {
 ; RV32-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; RV32-NEXT:    vmv.s.x v0, a1
 ; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    lui a1, %hi(.LCPI52_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI52_0)
+; RV32-NEXT:    lui a1, %hi(.LCPI65_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI65_0)
 ; RV32-NEXT:    vle8.v v12, (a1)
 ; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
 ; RV32-NEXT:    vsrl.vv v10, v8, v10
@@ -920,8 +1435,8 @@ define void @mulhu_v16i8(ptr %x) {
 ; RV64-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; RV64-NEXT:    vmv.s.x v0, a1
 ; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    lui a1, %hi(.LCPI52_0)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI52_0)
+; RV64-NEXT:    lui a1, %hi(.LCPI65_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI65_0)
 ; RV64-NEXT:    vle8.v v12, (a1)
 ; RV64-NEXT:    vmerge.vim v10, v10, 1, v0
 ; RV64-NEXT:    vsrl.vv v10, v8, v10
@@ -953,8 +1468,8 @@ define void @mulhu_v8i16(ptr %x) {
 ; CHECK-NEXT:    vsetivli zero, 7, e16, m1, tu, ma
 ; CHECK-NEXT:    vslideup.vi v9, v11, 6
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    lui a1, %hi(.LCPI53_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI53_0)
+; CHECK-NEXT:    lui a1, %hi(.LCPI66_0)
+; CHECK-NEXT:    addi a1, a1, %lo(.LCPI66_0)
 ; CHECK-NEXT:    vle16.v v12, (a1)
 ; CHECK-NEXT:    vsrl.vv v9, v8, v9
 ; CHECK-NEXT:    vmulhu.vv v9, v9, v12
@@ -977,6 +1492,61 @@ define void @mulhu_v8i16(ptr %x) {
   ret void
 }
 
+define void @mulhu_v6i16(ptr %x) {
+; RV32-LABEL: mulhu_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 4
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vid.v v10
+; RV32-NEXT:    vadd.vi v10, v10, 12
+; RV32-NEXT:    vdivu.vv v9, v9, v10
+; RV32-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v10, v9, 4
+; RV32-NEXT:    lui a1, %hi(.LCPI67_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI67_0)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vdivu.vv v8, v8, v9
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v10, 2
+; RV32-NEXT:    addi a0, a0, 8
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhu_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    lui a1, %hi(.LCPI67_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI67_0)
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vdivu.vv v9, v8, v9
+; RV64-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v8, v8, 4
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vid.v v10
+; RV64-NEXT:    vadd.vi v10, v10, 12
+; RV64-NEXT:    vdivu.vv v8, v8, v10
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v9, v8, 4
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v9, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v9, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = udiv <6 x i16> %a, <i16 7, i16 9, i16 10, i16 11, i16 12, i16 13>
+  store <6 x i16> %b, ptr %x
+  ret void
+}
+
 define void @mulhu_v4i32(ptr %x) {
 ; CHECK-LABEL: mulhu_v4i32:
 ; CHECK:       # %bb.0:
@@ -987,8 +1557,8 @@ define void @mulhu_v4i32(ptr %x) {
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vsetivli zero, 3, e32, m1, tu, ma
 ; CHECK-NEXT:    vslideup.vi v10, v9, 2
-; CHECK-NEXT:    lui a1, %hi(.LCPI54_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI54_0)
+; CHECK-NEXT:    lui a1, %hi(.LCPI68_0)
+; CHECK-NEXT:    addi a1, a1, %lo(.LCPI68_0)
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v9, (a1)
 ; CHECK-NEXT:    vmulhu.vv v9, v8, v9
@@ -1012,14 +1582,14 @@ define void @mulhu_v2i64(ptr %x) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vle64.v v8, (a0)
-; RV32-NEXT:    lui a1, %hi(.LCPI55_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI55_0)
+; RV32-NEXT:    lui a1, %hi(.LCPI69_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI69_0)
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV32-NEXT:    vmulhu.vv v8, v8, v9
-; RV32-NEXT:    lui a1, %hi(.LCPI55_1)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI55_1)
+; RV32-NEXT:    lui a1, %hi(.LCPI69_1)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI69_1)
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV32-NEXT:    vle32.v v9, (a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
@@ -1030,11 +1600,11 @@ define void @mulhu_v2i64(ptr %x) {
 ; RV64-LABEL: mulhu_v2i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    lui a1, %hi(.LCPI55_0)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI55_0)
+; RV64-NEXT:    lui a1, %hi(.LCPI69_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI69_0)
 ; RV64-NEXT:    vlse64.v v8, (a1), zero
-; RV64-NEXT:    lui a1, %hi(.LCPI55_1)
-; RV64-NEXT:    ld a1, %lo(.LCPI55_1)(a1)
+; RV64-NEXT:    lui a1, %hi(.LCPI69_1)
+; RV64-NEXT:    ld a1, %lo(.LCPI69_1)(a1)
 ; RV64-NEXT:    vle64.v v9, (a0)
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; RV64-NEXT:    vmv.s.x v8, a1
@@ -1141,6 +1711,69 @@ define void @mulhs_v8i16(ptr %x) {
   ret void
 }
 
+define void @mulhs_v6i16(ptr %x) {
+; RV32-LABEL: mulhs_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vmv.v.i v9, 7
+; RV32-NEXT:    vid.v v10
+; RV32-NEXT:    li a1, -14
+; RV32-NEXT:    vmadd.vx v10, a1, v9
+; RV32-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 4
+; RV32-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32-NEXT:    vdiv.vv v9, v9, v10
+; RV32-NEXT:    vsetivli zero, 6, e16, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v10, v9, 4
+; RV32-NEXT:    li a1, 6
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vmv.v.i v9, -7
+; RV32-NEXT:    vmerge.vim v9, v9, 7, v0
+; RV32-NEXT:    vdiv.vv v8, v8, v9
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v10, 2
+; RV32-NEXT:    addi a0, a0, 8
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: mulhs_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vmv.v.i v9, 7
+; RV64-NEXT:    vid.v v10
+; RV64-NEXT:    li a1, -14
+; RV64-NEXT:    vmadd.vx v10, a1, v9
+; RV64-NEXT:    vsetivli zero, 2, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 4
+; RV64-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64-NEXT:    vdiv.vv v9, v9, v10
+; RV64-NEXT:    li a1, 6
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vmv.v.i v10, -7
+; RV64-NEXT:    vmerge.vim v10, v10, 7, v0
+; RV64-NEXT:    vdiv.vv v8, v8, v10
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v9, 4
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = sdiv <6 x i16> %a, <i16 -7, i16 7, i16 7, i16 -7, i16 7, i16 -7>
+  store <6 x i16> %b, ptr %x
+  ret void
+}
+
 define void @mulhs_v4i32(ptr %x) {
 ; RV32-LABEL: mulhs_v4i32:
 ; RV32:       # %bb.0:
@@ -1165,8 +1798,8 @@ define void @mulhs_v4i32(ptr %x) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a0)
-; RV64-NEXT:    lui a1, %hi(.LCPI58_0)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI58_0)
+; RV64-NEXT:    lui a1, %hi(.LCPI73_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI73_0)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vlse64.v v9, (a1), zero
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
@@ -1218,11 +1851,11 @@ define void @mulhs_v2i64(ptr %x) {
 ; RV64-LABEL: mulhs_v2i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    lui a1, %hi(.LCPI59_0)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI59_0)
+; RV64-NEXT:    lui a1, %hi(.LCPI74_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI74_0)
 ; RV64-NEXT:    vlse64.v v8, (a1), zero
-; RV64-NEXT:    lui a1, %hi(.LCPI59_1)
-; RV64-NEXT:    ld a1, %lo(.LCPI59_1)(a1)
+; RV64-NEXT:    lui a1, %hi(.LCPI74_1)
+; RV64-NEXT:    ld a1, %lo(.LCPI74_1)(a1)
 ; RV64-NEXT:    vle64.v v9, (a0)
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; RV64-NEXT:    vmv.s.x v8, a1
@@ -1277,6 +1910,42 @@ define void @smin_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @smin_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: smin_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vmin.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: smin_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vmin.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %cc = icmp slt <6 x i16> %a, %b
+  %c = select <6 x i1> %cc, <6 x i16> %a, <6 x i16> %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @smin_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: smin_v4i32:
 ; CHECK:       # %bb.0:
@@ -1345,6 +2014,41 @@ define void @smin_vx_v8i16(ptr %x, i16 %y) {
 }
 declare <8 x i16> @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
 
+define void @smin_vx_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: smin_vx_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vmin.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: smin_vx_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vmin.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.smin.v6i16(<6 x i16> %a, <6 x i16> %c)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+declare <6 x i16> @llvm.smin.v6i16(<6 x i16>, <6 x i16>)
+
 define void @smin_vx_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: smin_vx_v4i32:
 ; CHECK:       # %bb.0:
@@ -1394,6 +2098,40 @@ define void @smin_xv_v8i16(ptr %x, i16 %y) {
   ret void
 }
 
+define void @smin_xv_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: smin_xv_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vmin.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: smin_xv_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vmin.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.smin.v6i16(<6 x i16> %c, <6 x i16> %a)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+
 define void @smin_xv_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: smin_xv_v4i32:
 ; CHECK:       # %bb.0:
@@ -1444,6 +2182,42 @@ define void @smax_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @smax_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: smax_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vmax.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: smax_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vmax.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %cc = icmp sgt <6 x i16> %a, %b
+  %c = select <6 x i1> %cc, <6 x i16> %a, <6 x i16> %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @smax_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: smax_v4i32:
 ; CHECK:       # %bb.0:
@@ -1512,6 +2286,41 @@ define void @smax_vx_v8i16(ptr %x, i16 %y) {
 }
 declare <8 x i16> @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
 
+define void @smax_vx_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: smax_vx_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vmax.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: smax_vx_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vmax.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.smax.v6i16(<6 x i16> %a, <6 x i16> %c)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+declare <6 x i16> @llvm.smax.v6i16(<6 x i16>, <6 x i16>)
+
 define void @smax_vx_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: smax_vx_v4i32:
 ; CHECK:       # %bb.0:
@@ -1561,6 +2370,40 @@ define void @smax_xv_v8i16(ptr %x, i16 %y) {
   ret void
 }
 
+define void @smax_xv_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: smax_xv_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vmax.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: smax_xv_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vmax.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.smax.v6i16(<6 x i16> %c, <6 x i16> %a)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+
 define void @smax_xv_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: smax_xv_v4i32:
 ; CHECK:       # %bb.0:
@@ -1611,6 +2454,42 @@ define void @umin_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @umin_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: umin_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vminu.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umin_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vminu.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %cc = icmp ult <6 x i16> %a, %b
+  %c = select <6 x i1> %cc, <6 x i16> %a, <6 x i16> %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @umin_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: umin_v4i32:
 ; CHECK:       # %bb.0:
@@ -1679,6 +2558,41 @@ define void @umin_vx_v8i16(ptr %x, i16 %y) {
 }
 declare <8 x i16> @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
 
+define void @umin_vx_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: umin_vx_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vminu.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umin_vx_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vminu.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.umin.v6i16(<6 x i16> %a, <6 x i16> %c)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+declare <6 x i16> @llvm.umin.v6i16(<6 x i16>, <6 x i16>)
+
 define void @umin_vx_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: umin_vx_v4i32:
 ; CHECK:       # %bb.0:
@@ -1728,6 +2642,40 @@ define void @umin_xv_v8i16(ptr %x, i16 %y) {
   ret void
 }
 
+define void @umin_xv_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: umin_xv_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vminu.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umin_xv_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vminu.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.umin.v6i16(<6 x i16> %c, <6 x i16> %a)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+
 define void @umin_xv_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: umin_xv_v4i32:
 ; CHECK:       # %bb.0:
@@ -1778,6 +2726,42 @@ define void @umax_v8i16(ptr %x, ptr %y) {
   ret void
 }
 
+define void @umax_v6i16(ptr %x, ptr %y) {
+; RV32-LABEL: umax_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vle16.v v9, (a1)
+; RV32-NEXT:    vmaxu.vv v8, v8, v9
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umax_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vle16.v v9, (a1)
+; RV64-NEXT:    vmaxu.vv v8, v8, v9
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = load <6 x i16>, ptr %y
+  %cc = icmp ugt <6 x i16> %a, %b
+  %c = select <6 x i1> %cc, <6 x i16> %a, <6 x i16> %b
+  store <6 x i16> %c, ptr %x
+  ret void
+}
+
 define void @umax_v4i32(ptr %x, ptr %y) {
 ; CHECK-LABEL: umax_v4i32:
 ; CHECK:       # %bb.0:
@@ -1846,6 +2830,41 @@ define void @umax_vx_v8i16(ptr %x, i16 %y) {
 }
 declare <8 x i16> @llvm.umax.v8i16(<8 x i16>, <8 x i16>)
 
+define void @umax_vx_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: umax_vx_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vmaxu.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umax_vx_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vmaxu.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.umax.v6i16(<6 x i16> %a, <6 x i16> %c)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+declare <6 x i16> @llvm.umax.v6i16(<6 x i16>, <6 x i16>)
+
 define void @umax_vx_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: umax_vx_v4i32:
 ; CHECK:       # %bb.0:
@@ -1895,6 +2914,40 @@ define void @umax_xv_v8i16(ptr %x, i16 %y) {
   ret void
 }
 
+define void @umax_xv_v6i16(ptr %x, i16 %y) {
+; RV32-LABEL: umax_xv_v6i16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV32-NEXT:    vle16.v v8, (a0)
+; RV32-NEXT:    vmaxu.vx v8, v8, a1
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    addi a1, a0, 8
+; RV32-NEXT:    vse32.v v9, (a1)
+; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32-NEXT:    vse16.v v8, (a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: umax_xv_v6i16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vle16.v v8, (a0)
+; RV64-NEXT:    vmaxu.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    ret
+  %a = load <6 x i16>, ptr %x
+  %b = insertelement <6 x i16> poison, i16 %y, i32 0
+  %c = shufflevector <6 x i16> %b, <6 x i16> poison, <6 x i32> zeroinitializer
+  %d = call <6 x i16> @llvm.umax.v6i16(<6 x i16> %c, <6 x i16> %a)
+  store <6 x i16> %d, ptr %x
+  ret void
+}
+
 define void @umax_xv_v4i32(ptr %x, i32 %y) {
 ; CHECK-LABEL: umax_xv_v4i32:
 ; CHECK:       # %bb.0:
@@ -2050,6 +3103,74 @@ define void @add_v8i32(ptr %x, ptr %y) {
   ret void
 }
 
+define void @add_v6i32(ptr %x, ptr %y) {
+; LMULMAX2-RV32-LABEL: add_v6i32:
+; LMULMAX2-RV32:       # %bb.0:
+; LMULMAX2-RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; LMULMAX2-RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX2-RV32-NEXT:    vle32.v v10, (a1)
+; LMULMAX2-RV32-NEXT:    vadd.vv v8, v8, v10
+; LMULMAX2-RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; LMULMAX2-RV32-NEXT:    vslidedown.vi v10, v8, 4
+; LMULMAX2-RV32-NEXT:    addi a1, a0, 16
+; LMULMAX2-RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX2-RV32-NEXT:    vse32.v v10, (a1)
+; LMULMAX2-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX2-RV32-NEXT:    vse32.v v8, (a0)
+; LMULMAX2-RV32-NEXT:    ret
+;
+; LMULMAX2-RV64-LABEL: add_v6i32:
+; LMULMAX2-RV64:       # %bb.0:
+; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; LMULMAX2-RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX2-RV64-NEXT:    vle32.v v10, (a1)
+; LMULMAX2-RV64-NEXT:    vadd.vv v8, v8, v10
+; LMULMAX2-RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; LMULMAX2-RV64-NEXT:    vslidedown.vi v10, v8, 2
+; LMULMAX2-RV64-NEXT:    addi a1, a0, 16
+; LMULMAX2-RV64-NEXT:    vse64.v v10, (a1)
+; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX2-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX2-RV64-NEXT:    ret
+;
+; LMULMAX1-RV32-LABEL: add_v6i32:
+; LMULMAX1-RV32:       # %bb.0:
+; LMULMAX1-RV32-NEXT:    addi a2, a0, 16
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1-RV32-NEXT:    vle32.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vle32.v v9, (a1)
+; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
+; LMULMAX1-RV32-NEXT:    addi a1, a1, 16
+; LMULMAX1-RV32-NEXT:    vle32.v v11, (a1)
+; LMULMAX1-RV32-NEXT:    vadd.vv v8, v8, v9
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; LMULMAX1-RV32-NEXT:    vadd.vv v8, v10, v11
+; LMULMAX1-RV32-NEXT:    vse32.v v8, (a2)
+; LMULMAX1-RV32-NEXT:    ret
+;
+; LMULMAX1-RV64-LABEL: add_v6i32:
+; LMULMAX1-RV64:       # %bb.0:
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    addi a2, a1, 16
+; LMULMAX1-RV64-NEXT:    vle32.v v9, (a2)
+; LMULMAX1-RV64-NEXT:    addi a2, a0, 16
+; LMULMAX1-RV64-NEXT:    vle32.v v10, (a2)
+; LMULMAX1-RV64-NEXT:    vle32.v v11, (a1)
+; LMULMAX1-RV64-NEXT:    vadd.vv v9, v10, v9
+; LMULMAX1-RV64-NEXT:    vadd.vv v8, v8, v11
+; LMULMAX1-RV64-NEXT:    vse32.v v8, (a0)
+; LMULMAX1-RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; LMULMAX1-RV64-NEXT:    vse64.v v9, (a2)
+; LMULMAX1-RV64-NEXT:    ret
+  %a = load <6 x i32>, ptr %x
+  %b = load <6 x i32>, ptr %y
+  %c = add <6 x i32> %a, %b
+  store <6 x i32> %c, ptr %x
+  ret void
+}
+
 define void @add_v4i64(ptr %x, ptr %y) {
 ; LMULMAX2-LABEL: add_v4i64:
 ; LMULMAX2:       # %bb.0:
@@ -4360,8 +5481,8 @@ define void @mulhu_v32i8(ptr %x) {
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a2
 ; LMULMAX2-RV32-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
-; LMULMAX2-RV32-NEXT:    lui a2, %hi(.LCPI153_0)
-; LMULMAX2-RV32-NEXT:    addi a2, a2, %lo(.LCPI153_0)
+; LMULMAX2-RV32-NEXT:    lui a2, %hi(.LCPI181_0)
+; LMULMAX2-RV32-NEXT:    addi a2, a2, %lo(.LCPI181_0)
 ; LMULMAX2-RV32-NEXT:    vle8.v v10, (a2)
 ; LMULMAX2-RV32-NEXT:    vmv.v.i v12, 0
 ; LMULMAX2-RV32-NEXT:    vmerge.vim v14, v12, 1, v0
@@ -4410,8 +5531,8 @@ define void @mulhu_v32i8(ptr %x) {
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a2
 ; LMULMAX2-RV64-NEXT:    vsetvli zero, a1, e8, m2, ta, ma
-; LMULMAX2-RV64-NEXT:    lui a2, %hi(.LCPI153_0)
-; LMULMAX2-RV64-NEXT:    addi a2, a2, %lo(.LCPI153_0)
+; LMULMAX2-RV64-NEXT:    lui a2, %hi(.LCPI181_0)
+; LMULMAX2-RV64-NEXT:    addi a2, a2, %lo(.LCPI181_0)
 ; LMULMAX2-RV64-NEXT:    vle8.v v10, (a2)
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, 0
 ; LMULMAX2-RV64-NEXT:    vmerge.vim v14, v12, 1, v0
@@ -4455,8 +5576,8 @@ define void @mulhu_v32i8(ptr %x) {
 ; LMULMAX1-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; LMULMAX1-NEXT:    addi a1, a0, 16
 ; LMULMAX1-NEXT:    vle8.v v8, (a1)
-; LMULMAX1-NEXT:    lui a2, %hi(.LCPI153_0)
-; LMULMAX1-NEXT:    addi a2, a2, %lo(.LCPI153_0)
+; LMULMAX1-NEXT:    lui a2, %hi(.LCPI181_0)
+; LMULMAX1-NEXT:    addi a2, a2, %lo(.LCPI181_0)
 ; LMULMAX1-NEXT:    vle8.v v9, (a2)
 ; LMULMAX1-NEXT:    vle8.v v10, (a0)
 ; LMULMAX1-NEXT:    vdivu.vv v8, v8, v9
@@ -4488,8 +5609,8 @@ define void @mulhu_v16i16(ptr %x) {
 ; LMULMAX2-RV32-NEXT:    li a1, 257
 ; LMULMAX2-RV32-NEXT:    vmv.s.x v0, a1
 ; LMULMAX2-RV32-NEXT:    vmv.v.i v14, 0
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI154_0)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI154_0)
+; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI182_0)
+; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI182_0)
 ; LMULMAX2-RV32-NEXT:    vle16.v v16, (a1)
 ; LMULMAX2-RV32-NEXT:    lui a1, 1048568
 ; LMULMAX2-RV32-NEXT:    vmerge.vxm v18, v14, a1, v0
@@ -4521,8 +5642,8 @@ define void @mulhu_v16i16(ptr %x) {
 ; LMULMAX2-RV64-NEXT:    li a1, 257
 ; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v14, 0
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI154_0)
-; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI154_0)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI182_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI182_0)
 ; LMULMAX2-RV64-NEXT:    vle16.v v16, (a1)
 ; LMULMAX2-RV64-NEXT:    lui a1, 1048568
 ; LMULMAX2-RV64-NEXT:    vmerge.vxm v18, v14, a1, v0
@@ -4542,8 +5663,8 @@ define void @mulhu_v16i16(ptr %x) {
 ; LMULMAX1-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; LMULMAX1-NEXT:    addi a1, a0, 16
 ; LMULMAX1-NEXT:    vle16.v v8, (a1)
-; LMULMAX1-NEXT:    lui a2, %hi(.LCPI154_0)
-; LMULMAX1-NEXT:    addi a2, a2, %lo(.LCPI154_0)
+; LMULMAX1-NEXT:    lui a2, %hi(.LCPI182_0)
+; LMULMAX1-NEXT:    addi a2, a2, %lo(.LCPI182_0)
 ; LMULMAX1-NEXT:    vle16.v v9, (a2)
 ; LMULMAX1-NEXT:    vle16.v v10, (a0)
 ; LMULMAX1-NEXT:    vdivu.vv v8, v8, v9
@@ -4564,8 +5685,8 @@ define void @mulhu_v8i32(ptr %x) {
 ; LMULMAX2-NEXT:    vle32.v v8, (a0)
 ; LMULMAX2-NEXT:    li a1, 68
 ; LMULMAX2-NEXT:    vmv.s.x v0, a1
-; LMULMAX2-NEXT:    lui a1, %hi(.LCPI155_0)
-; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI155_0)
+; LMULMAX2-NEXT:    lui a1, %hi(.LCPI183_0)
+; LMULMAX2-NEXT:    addi a1, a1, %lo(.LCPI183_0)
 ; LMULMAX2-NEXT:    vle32.v v10, (a1)
 ; LMULMAX2-NEXT:    vmv.v.i v12, 0
 ; LMULMAX2-NEXT:    lui a1, 524288
@@ -4593,8 +5714,8 @@ define void @mulhu_v8i32(ptr %x) {
 ; LMULMAX1-RV32-NEXT:    vmv.v.i v11, 0
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 3, e32, m1, tu, ma
 ; LMULMAX1-RV32-NEXT:    vslideup.vi v11, v10, 2
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI155_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI155_0)
+; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI183_0)
+; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI183_0)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
 ; LMULMAX1-RV32-NEXT:    vmulhu.vv v12, v9, v10
@@ -4619,8 +5740,8 @@ define void @mulhu_v8i32(ptr %x) {
 ; LMULMAX1-RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle32.v v8, (a1)
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI155_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI155_0)
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI183_0)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI183_0)
 ; LMULMAX1-RV64-NEXT:    vle32.v v9, (a2)
 ; LMULMAX1-RV64-NEXT:    vle32.v v10, (a0)
 ; LMULMAX1-RV64-NEXT:    vdivu.vv v8, v8, v9
@@ -4639,8 +5760,8 @@ define void @mulhu_v4i64(ptr %x) {
 ; LMULMAX2-RV32:       # %bb.0:
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; LMULMAX2-RV32-NEXT:    vle64.v v8, (a0)
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI156_0)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI156_0)
+; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI184_0)
+; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI184_0)
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; LMULMAX2-RV32-NEXT:    vle32.v v10, (a1)
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
@@ -4655,8 +5776,8 @@ define void @mulhu_v4i64(ptr %x) {
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; LMULMAX2-RV32-NEXT:    vmulhu.vv v8, v8, v14
 ; LMULMAX2-RV32-NEXT:    vadd.vv v8, v8, v10
-; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI156_1)
-; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI156_1)
+; LMULMAX2-RV32-NEXT:    lui a1, %hi(.LCPI184_1)
+; LMULMAX2-RV32-NEXT:    addi a1, a1, %lo(.LCPI184_1)
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; LMULMAX2-RV32-NEXT:    vle32.v v10, (a1)
 ; LMULMAX2-RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
@@ -4674,13 +5795,13 @@ define void @mulhu_v4i64(ptr %x) {
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, 0
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 3, e64, m2, tu, ma
 ; LMULMAX2-RV64-NEXT:    vslideup.vi v12, v10, 2
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI156_0)
-; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI156_0)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI184_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI184_0)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; LMULMAX2-RV64-NEXT:    vle64.v v10, (a1)
 ; LMULMAX2-RV64-NEXT:    vmulhu.vv v10, v8, v10
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI156_1)
-; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI156_1)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI184_1)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI184_1)
 ; LMULMAX2-RV64-NEXT:    vle64.v v14, (a1)
 ; LMULMAX2-RV64-NEXT:    vsub.vv v8, v8, v10
 ; LMULMAX2-RV64-NEXT:    vmulhu.vv v8, v8, v12
@@ -4695,14 +5816,14 @@ define void @mulhu_v4i64(ptr %x) {
 ; LMULMAX1-RV32-NEXT:    vle64.v v8, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV32-NEXT:    vle64.v v9, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI156_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI156_0)
+; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI184_0)
+; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI184_0)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; LMULMAX1-RV32-NEXT:    vdivu.vv v9, v9, v10
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI156_1)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI156_1)
+; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI184_1)
+; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI184_1)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
@@ -4722,12 +5843,12 @@ define void @mulhu_v4i64(ptr %x) {
 ; LMULMAX1-RV64-NEXT:    slli a2, a2, 63
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v10, a2
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI156_0)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI156_0)
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI184_0)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI184_0)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
 ; LMULMAX1-RV64-NEXT:    vlse64.v v11, (a2), zero
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI156_1)
-; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI156_1)(a2)
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI184_1)
+; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI184_1)(a2)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a2
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
@@ -4736,11 +5857,11 @@ define void @mulhu_v4i64(ptr %x) {
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v11
 ; LMULMAX1-RV64-NEXT:    vid.v v10
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI156_2)
-; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI156_2)
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI184_2)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI184_2)
 ; LMULMAX1-RV64-NEXT:    vlse64.v v11, (a2), zero
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI156_3)
-; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI156_3)(a2)
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI184_3)
+; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI184_3)(a2)
 ; LMULMAX1-RV64-NEXT:    vadd.vi v12, v10, 2
 ; LMULMAX1-RV64-NEXT:    vsrl.vv v9, v9, v12
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
@@ -4930,8 +6051,8 @@ define void @mulhs_v8i32(ptr %x) {
 ; LMULMAX2-RV64:       # %bb.0:
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; LMULMAX2-RV64-NEXT:    vle32.v v8, (a0)
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI159_0)
-; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI159_0)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI187_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI187_0)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; LMULMAX2-RV64-NEXT:    vlse64.v v10, (a1), zero
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
@@ -5031,11 +6152,11 @@ define void @mulhs_v4i64(ptr %x) {
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; LMULMAX2-RV64-NEXT:    li a1, 5
 ; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI160_0)
-; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI160_0)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI188_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI188_0)
 ; LMULMAX2-RV64-NEXT:    vlse64.v v8, (a1), zero
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI160_1)
-; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI160_1)(a1)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI188_1)
+; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI188_1)(a1)
 ; LMULMAX2-RV64-NEXT:    vle64.v v10, (a0)
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, -1
 ; LMULMAX2-RV64-NEXT:    vmerge.vim v12, v12, 0, v0
@@ -5057,8 +6178,8 @@ define void @mulhs_v4i64(ptr %x) {
 ; LMULMAX1-RV32-NEXT:    vle64.v v8, (a0)
 ; LMULMAX1-RV32-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV32-NEXT:    vle64.v v9, (a1)
-; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI160_0)
-; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI160_0)
+; LMULMAX1-RV32-NEXT:    lui a2, %hi(.LCPI188_0)
+; LMULMAX1-RV32-NEXT:    addi a2, a2, %lo(.LCPI188_0)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; LMULMAX1-RV32-NEXT:    vle32.v v10, (a2)
 ; LMULMAX1-RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
@@ -5072,11 +6193,11 @@ define void @mulhs_v4i64(ptr %x) {
 ; LMULMAX1-RV64:       # %bb.0:
 ; LMULMAX1-RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; LMULMAX1-RV64-NEXT:    vle64.v v8, (a0)
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI160_0)
-; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI160_0)
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI188_0)
+; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI188_0)
 ; LMULMAX1-RV64-NEXT:    vlse64.v v9, (a1), zero
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI160_1)
-; LMULMAX1-RV64-NEXT:    ld a1, %lo(.LCPI160_1)(a1)
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI188_1)
+; LMULMAX1-RV64-NEXT:    ld a1, %lo(.LCPI188_1)(a1)
 ; LMULMAX1-RV64-NEXT:    addi a2, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v10, (a2)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
@@ -7710,8 +8831,8 @@ define void @mulhu_vx_v2i64(ptr %x) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    lui a1, %hi(.LCPI289_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI289_0)(a1)
+; RV64-NEXT:    lui a1, %hi(.LCPI317_0)
+; RV64-NEXT:    ld a1, %lo(.LCPI317_0)(a1)
 ; RV64-NEXT:    vmulhu.vx v8, v8, a1
 ; RV64-NEXT:    vsrl.vi v8, v8, 1
 ; RV64-NEXT:    vse64.v v8, (a0)
@@ -7828,8 +8949,8 @@ define void @mulhs_vx_v2i64(ptr %x) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    lui a1, %hi(.LCPI293_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI293_0)(a1)
+; RV64-NEXT:    lui a1, %hi(.LCPI321_0)
+; RV64-NEXT:    ld a1, %lo(.LCPI321_0)(a1)
 ; RV64-NEXT:    vmulh.vx v8, v8, a1
 ; RV64-NEXT:    li a1, 63
 ; RV64-NEXT:    vsrl.vx v9, v8, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
index 747a6a3534fd7..2e5f396694104 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
@@ -21,8 +21,6 @@ define <5 x i8> @load_v5i8(ptr %p) {
 define <5 x i8> @load_v5i8_align1(ptr %p) {
 ; RV32-LABEL: load_v5i8_align1:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    lbu a1, 1(a0)
 ; RV32-NEXT:    lbu a2, 0(a0)
 ; RV32-NEXT:    lbu a3, 2(a0)
@@ -35,29 +33,26 @@ define <5 x i8> @load_v5i8_align1(ptr %p) {
 ; RV32-NEXT:    or a1, a3, a1
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV32-NEXT:    vmv.s.x v8, a1
+; RV32-NEXT:    vslidedown.vi v9, v8, 3
 ; RV32-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
+; RV32-NEXT:    vmv.x.s a1, v9
+; RV32-NEXT:    vslidedown.vi v9, v8, 2
+; RV32-NEXT:    vmv.x.s a2, v9
 ; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v11, v8, 3
+; RV32-NEXT:    vmv.x.s a3, v9
+; RV32-NEXT:    vmv.x.s a4, v8
 ; RV32-NEXT:    lb a0, 4(a0)
-; RV32-NEXT:    addi a1, sp, 8
-; RV32-NEXT:    vse8.v v8, (a1)
-; RV32-NEXT:    addi a2, sp, 11
-; RV32-NEXT:    vse8.v v11, (a2)
-; RV32-NEXT:    addi a2, sp, 10
-; RV32-NEXT:    vse8.v v10, (a2)
-; RV32-NEXT:    addi a2, sp, 9
-; RV32-NEXT:    vse8.v v9, (a2)
-; RV32-NEXT:    sb a0, 12(sp)
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    vslide1down.vx v8, v8, a4
+; RV32-NEXT:    vslide1down.vx v8, v8, a3
+; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: load_v5i8_align1:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    lbu a1, 1(a0)
 ; RV64-NEXT:    lbu a2, 0(a0)
 ; RV64-NEXT:    lbu a3, 2(a0)
@@ -70,23 +65,22 @@ define <5 x i8> @load_v5i8_align1(ptr %p) {
 ; RV64-NEXT:    or a1, a3, a1
 ; RV64-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; RV64-NEXT:    vmv.s.x v8, a1
+; RV64-NEXT:    vslidedown.vi v9, v8, 3
 ; RV64-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
+; RV64-NEXT:    vmv.x.s a1, v9
+; RV64-NEXT:    vslidedown.vi v9, v8, 2
+; RV64-NEXT:    vmv.x.s a2, v9
 ; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-NEXT:    vmv.x.s a3, v9
+; RV64-NEXT:    vmv.x.s a4, v8
 ; RV64-NEXT:    lb a0, 4(a0)
-; RV64-NEXT:    addi a1, sp, 8
-; RV64-NEXT:    vse8.v v8, (a1)
-; RV64-NEXT:    addi a2, sp, 11
-; RV64-NEXT:    vse8.v v11, (a2)
-; RV64-NEXT:    addi a2, sp, 10
-; RV64-NEXT:    vse8.v v10, (a2)
-; RV64-NEXT:    addi a2, sp, 9
-; RV64-NEXT:    vse8.v v9, (a2)
-; RV64-NEXT:    sb a0, 12(sp)
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    vslide1down.vx v8, v8, a4
+; RV64-NEXT:    vslide1down.vx v8, v8, a3
+; RV64-NEXT:    vslide1down.vx v8, v8, a2
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vslidedown.vi v8, v8, 3
 ; RV64-NEXT:    ret
   %x = load <5 x i8>, ptr %p, align 1
   ret <5 x i8> %x
@@ -142,15 +136,14 @@ define <6 x half> @load_v6f16(ptr %p) {
 ; RV64-LABEL: load_v6f16:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    ld a2, 0(a1)
-; RV64-NEXT:    addi a1, a1, 8
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vlse64.v v8, (a1), zero
-; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    ld a1, 8(a1)
+; RV64-NEXT:    vslide1down.vx v8, v8, a2
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
 ; RV64-NEXT:    sd a2, 0(a0)
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
   %x = load <6 x half>, ptr %p
@@ -180,60 +173,52 @@ define <6 x double> @load_v6f64(ptr %p) {
 define <6 x i1> @load_v6i1(ptr %p) {
 ; RV32-LABEL: load_v6i1:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
 ; RV32-NEXT:    lbu a0, 0(a0)
-; RV32-NEXT:    slli a1, a0, 30
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    slli a2, a0, 29
+; RV32-NEXT:    srli a1, a0, 5
+; RV32-NEXT:    slli a2, a0, 27
 ; RV32-NEXT:    srli a2, a2, 31
 ; RV32-NEXT:    slli a3, a0, 28
 ; RV32-NEXT:    srli a3, a3, 31
-; RV32-NEXT:    slli a4, a0, 27
+; RV32-NEXT:    slli a4, a0, 29
 ; RV32-NEXT:    srli a4, a4, 31
-; RV32-NEXT:    andi a5, a0, 1
-; RV32-NEXT:    srli a0, a0, 5
-; RV32-NEXT:    sb a0, 13(sp)
-; RV32-NEXT:    sb a5, 8(sp)
-; RV32-NEXT:    sb a4, 12(sp)
-; RV32-NEXT:    sb a3, 11(sp)
-; RV32-NEXT:    sb a2, 10(sp)
-; RV32-NEXT:    sb a1, 9(sp)
-; RV32-NEXT:    addi a0, sp, 8
+; RV32-NEXT:    slli a5, a0, 30
+; RV32-NEXT:    srli a5, a5, 31
+; RV32-NEXT:    andi a0, a0, 1
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vle8.v v8, (a0)
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a5
+; RV32-NEXT:    vslide1down.vx v8, v8, a4
+; RV32-NEXT:    vslide1down.vx v8, v8, a3
+; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-NEXT:    vand.vi v8, v8, 1
 ; RV32-NEXT:    vmsne.vi v0, v8, 0
-; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: load_v6i1:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    lbu a0, 0(a0)
-; RV64-NEXT:    slli a1, a0, 62
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    slli a2, a0, 61
+; RV64-NEXT:    srli a1, a0, 5
+; RV64-NEXT:    slli a2, a0, 59
 ; RV64-NEXT:    srli a2, a2, 63
 ; RV64-NEXT:    slli a3, a0, 60
 ; RV64-NEXT:    srli a3, a3, 63
-; RV64-NEXT:    slli a4, a0, 59
+; RV64-NEXT:    slli a4, a0, 61
 ; RV64-NEXT:    srli a4, a4, 63
-; RV64-NEXT:    andi a5, a0, 1
-; RV64-NEXT:    srli a0, a0, 5
-; RV64-NEXT:    sb a0, 13(sp)
-; RV64-NEXT:    sb a5, 8(sp)
-; RV64-NEXT:    sb a4, 12(sp)
-; RV64-NEXT:    sb a3, 11(sp)
-; RV64-NEXT:    sb a2, 10(sp)
-; RV64-NEXT:    sb a1, 9(sp)
-; RV64-NEXT:    addi a0, sp, 8
+; RV64-NEXT:    slli a5, a0, 62
+; RV64-NEXT:    srli a5, a5, 63
+; RV64-NEXT:    andi a0, a0, 1
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vle8.v v8, (a0)
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vslide1down.vx v8, v8, a5
+; RV64-NEXT:    vslide1down.vx v8, v8, a4
+; RV64-NEXT:    vslide1down.vx v8, v8, a3
+; RV64-NEXT:    vslide1down.vx v8, v8, a2
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    vand.vi v8, v8, 1
 ; RV64-NEXT:    vmsne.vi v0, v8, 0
-; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
   %x = load <6 x i1>, ptr %p
   ret <6 x i1> %x
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
index 27e9918528935..9ef4bb1dfbf04 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
@@ -54,10 +54,8 @@ define <2 x i1> @buildvec_mask_nonconst_v2i1(i1 %x, i1 %y) {
 ; CHECK-LABEL: buildvec_mask_nonconst_v2i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v8, a1
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, tu, ma
-; CHECK-NEXT:    vmv.s.x v8, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
@@ -65,10 +63,8 @@ define <2 x i1> @buildvec_mask_nonconst_v2i1(i1 %x, i1 %y) {
 ; ZVE32F-LABEL: buildvec_mask_nonconst_v2i1:
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; ZVE32F-NEXT:    vmv.v.x v8, a1
-; ZVE32F-NEXT:    vsetvli zero, zero, e8, mf4, tu, ma
-; ZVE32F-NEXT:    vmv.s.x v8, a0
-; ZVE32F-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
@@ -81,30 +77,20 @@ define <2 x i1> @buildvec_mask_nonconst_v2i1(i1 %x, i1 %y) {
 define <2 x i1> @buildvec_mask_optsize_nonconst_v2i1(i1 %x, i1 %y) optsize {
 ; CHECK-LABEL: buildvec_mask_optsize_nonconst_v2i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    sb a1, 15(sp)
-; CHECK-NEXT:    sb a0, 14(sp)
-; CHECK-NEXT:    addi a0, sp, 14
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ;
 ; ZVE32F-LABEL: buildvec_mask_optsize_nonconst_v2i1:
 ; ZVE32F:       # %bb.0:
-; ZVE32F-NEXT:    addi sp, sp, -16
-; ZVE32F-NEXT:    .cfi_def_cfa_offset 16
-; ZVE32F-NEXT:    sb a1, 15(sp)
-; ZVE32F-NEXT:    sb a0, 14(sp)
-; ZVE32F-NEXT:    addi a0, sp, 14
 ; ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32F-NEXT:    addi sp, sp, 16
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <2 x i1> poison, i1 %x, i32 0
   %2 = insertelement <2 x i1> %1,  i1 %y, i32 1
@@ -195,34 +181,24 @@ define <4 x i1> @buildvec_mask_nonconst_v4i1(i1 %x, i1 %y) {
 define <4 x i1> @buildvec_mask_optsize_nonconst_v4i1(i1 %x, i1 %y) optsize {
 ; CHECK-LABEL: buildvec_mask_optsize_nonconst_v4i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    sb a1, 15(sp)
-; CHECK-NEXT:    sb a1, 14(sp)
-; CHECK-NEXT:    sb a0, 13(sp)
-; CHECK-NEXT:    sb a0, 12(sp)
-; CHECK-NEXT:    addi a0, sp, 12
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ;
 ; ZVE32F-LABEL: buildvec_mask_optsize_nonconst_v4i1:
 ; ZVE32F:       # %bb.0:
-; ZVE32F-NEXT:    addi sp, sp, -16
-; ZVE32F-NEXT:    .cfi_def_cfa_offset 16
-; ZVE32F-NEXT:    sb a1, 15(sp)
-; ZVE32F-NEXT:    sb a1, 14(sp)
-; ZVE32F-NEXT:    sb a0, 13(sp)
-; ZVE32F-NEXT:    sb a0, 12(sp)
-; ZVE32F-NEXT:    addi a0, sp, 12
 ; ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32F-NEXT:    addi sp, sp, 16
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <4 x i1> poison, i1 %x, i32 0
   %2 = insertelement <4 x i1> %1,  i1 %x, i32 1
@@ -234,36 +210,26 @@ define <4 x i1> @buildvec_mask_optsize_nonconst_v4i1(i1 %x, i1 %y) optsize {
 define <4 x i1> @buildvec_mask_nonconst_v4i1_2(i1 %x, i1 %y) {
 ; CHECK-LABEL: buildvec_mask_nonconst_v4i1_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    sb a1, 15(sp)
-; CHECK-NEXT:    li a1, 1
-; CHECK-NEXT:    sb a1, 14(sp)
-; CHECK-NEXT:    sb a0, 13(sp)
-; CHECK-NEXT:    sb zero, 12(sp)
-; CHECK-NEXT:    addi a0, sp, 12
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ;
 ; ZVE32F-LABEL: buildvec_mask_nonconst_v4i1_2:
 ; ZVE32F:       # %bb.0:
-; ZVE32F-NEXT:    addi sp, sp, -16
-; ZVE32F-NEXT:    .cfi_def_cfa_offset 16
-; ZVE32F-NEXT:    sb a1, 15(sp)
-; ZVE32F-NEXT:    li a1, 1
-; ZVE32F-NEXT:    sb a1, 14(sp)
-; ZVE32F-NEXT:    sb a0, 13(sp)
-; ZVE32F-NEXT:    sb zero, 12(sp)
-; ZVE32F-NEXT:    addi a0, sp, 12
 ; ZVE32F-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    li a0, 1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32F-NEXT:    addi sp, sp, 16
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <4 x i1> poison, i1 0, i32 0
   %2 = insertelement <4 x i1> %1,  i1 %x, i32 1
@@ -325,44 +291,34 @@ define <8 x i1> @buildvec_mask_nonconst_v8i1(i1 %x, i1 %y) {
 define <8 x i1> @buildvec_mask_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) {
 ; CHECK-LABEL: buildvec_mask_nonconst_v8i1_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    sb a2, 15(sp)
-; CHECK-NEXT:    sb zero, 14(sp)
-; CHECK-NEXT:    sb a3, 13(sp)
-; CHECK-NEXT:    sb a0, 12(sp)
-; CHECK-NEXT:    sb a1, 11(sp)
-; CHECK-NEXT:    li a1, 1
-; CHECK-NEXT:    sb a1, 10(sp)
-; CHECK-NEXT:    sb a0, 9(sp)
-; CHECK-NEXT:    sb a0, 8(sp)
-; CHECK-NEXT:    addi a0, sp, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    li a4, 1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a4
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a3
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, a2
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ;
 ; ZVE32F-LABEL: buildvec_mask_nonconst_v8i1_2:
 ; ZVE32F:       # %bb.0:
-; ZVE32F-NEXT:    addi sp, sp, -16
-; ZVE32F-NEXT:    .cfi_def_cfa_offset 16
-; ZVE32F-NEXT:    sb a2, 15(sp)
-; ZVE32F-NEXT:    sb zero, 14(sp)
-; ZVE32F-NEXT:    sb a3, 13(sp)
-; ZVE32F-NEXT:    sb a0, 12(sp)
-; ZVE32F-NEXT:    sb a1, 11(sp)
-; ZVE32F-NEXT:    li a1, 1
-; ZVE32F-NEXT:    sb a1, 10(sp)
-; ZVE32F-NEXT:    sb a0, 9(sp)
-; ZVE32F-NEXT:    sb a0, 8(sp)
-; ZVE32F-NEXT:    addi a0, sp, 8
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    li a4, 1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32F-NEXT:    addi sp, sp, 16
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
   %2 = insertelement <8 x i1> %1,  i1 %x, i32 1
@@ -378,44 +334,34 @@ define <8 x i1> @buildvec_mask_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) {
 define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) optsize {
 ; CHECK-LABEL: buildvec_mask_optsize_nonconst_v8i1_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    sb a2, 15(sp)
-; CHECK-NEXT:    sb zero, 14(sp)
-; CHECK-NEXT:    sb a3, 13(sp)
-; CHECK-NEXT:    sb a0, 12(sp)
-; CHECK-NEXT:    sb a1, 11(sp)
-; CHECK-NEXT:    li a1, 1
-; CHECK-NEXT:    sb a1, 10(sp)
-; CHECK-NEXT:    sb a0, 9(sp)
-; CHECK-NEXT:    sb a0, 8(sp)
-; CHECK-NEXT:    addi a0, sp, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    li a4, 1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a4
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a3
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, a2
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ;
 ; ZVE32F-LABEL: buildvec_mask_optsize_nonconst_v8i1_2:
 ; ZVE32F:       # %bb.0:
-; ZVE32F-NEXT:    addi sp, sp, -16
-; ZVE32F-NEXT:    .cfi_def_cfa_offset 16
-; ZVE32F-NEXT:    sb a2, 15(sp)
-; ZVE32F-NEXT:    sb zero, 14(sp)
-; ZVE32F-NEXT:    sb a3, 13(sp)
-; ZVE32F-NEXT:    sb a0, 12(sp)
-; ZVE32F-NEXT:    sb a1, 11(sp)
-; ZVE32F-NEXT:    li a1, 1
-; ZVE32F-NEXT:    sb a1, 10(sp)
-; ZVE32F-NEXT:    sb a0, 9(sp)
-; ZVE32F-NEXT:    sb a0, 8(sp)
-; ZVE32F-NEXT:    addi a0, sp, 8
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    li a4, 1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32F-NEXT:    addi sp, sp, 16
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
   %2 = insertelement <8 x i1> %1,  i1 %x, i32 1
@@ -431,42 +377,32 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %
 define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1(i1 %x, i1 %y) optsize {
 ; CHECK-LABEL: buildvec_mask_optsize_nonconst_v8i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    sb a1, 15(sp)
-; CHECK-NEXT:    sb a1, 14(sp)
-; CHECK-NEXT:    sb a1, 13(sp)
-; CHECK-NEXT:    sb a0, 12(sp)
-; CHECK-NEXT:    sb a1, 11(sp)
-; CHECK-NEXT:    sb a1, 10(sp)
-; CHECK-NEXT:    sb a0, 9(sp)
-; CHECK-NEXT:    sb a0, 8(sp)
-; CHECK-NEXT:    addi a0, sp, 8
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ;
 ; ZVE32F-LABEL: buildvec_mask_optsize_nonconst_v8i1:
 ; ZVE32F:       # %bb.0:
-; ZVE32F-NEXT:    addi sp, sp, -16
-; ZVE32F-NEXT:    .cfi_def_cfa_offset 16
-; ZVE32F-NEXT:    sb a1, 15(sp)
-; ZVE32F-NEXT:    sb a1, 14(sp)
-; ZVE32F-NEXT:    sb a1, 13(sp)
-; ZVE32F-NEXT:    sb a0, 12(sp)
-; ZVE32F-NEXT:    sb a1, 11(sp)
-; ZVE32F-NEXT:    sb a1, 10(sp)
-; ZVE32F-NEXT:    sb a0, 9(sp)
-; ZVE32F-NEXT:    sb a0, 8(sp)
-; ZVE32F-NEXT:    addi a0, sp, 8
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; ZVE32F-NEXT:    vle8.v v8, (a0)
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vand.vi v8, v8, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
-; ZVE32F-NEXT:    addi sp, sp, 16
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
   %2 = insertelement <8 x i1> %1,  i1 %x, i32 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index f99a5a300c8e1..946e7d2aaa3dc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -36,7 +36,7 @@ define <1 x i8> @mgather_v1i8(<1 x ptr> %ptrs, <1 x i1> %m, <1 x i8> %passthru)
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB0_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB0_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vle8.v v8, (a0)
@@ -330,7 +330,6 @@ define <2 x i64> @mgather_v2i8_sextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2 x
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    srai a2, a1, 31
@@ -392,7 +391,6 @@ define <2 x i64> @mgather_v2i8_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2 x
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    andi a1, a1, 255
@@ -798,13 +796,12 @@ define <8 x i8> @mgather_baseidx_v8i8(ptr %base, <8 x i8> %idxs, <8 x i1> %m, <8
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB12_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
 ; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e8, mf2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB12_8
@@ -873,7 +870,7 @@ define <1 x i16> @mgather_v1i16(<1 x ptr> %ptrs, <1 x i1> %m, <1 x i16> %passthr
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB13_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB13_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vle16.v v8, (a0)
@@ -1061,7 +1058,6 @@ define <2 x i64> @mgather_v2i16_sextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    srai a2, a1, 31
@@ -1123,7 +1119,6 @@ define <2 x i64> @mgather_v2i16_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    lui a2, 16
@@ -1542,15 +1537,14 @@ define <8 x i16> @mgather_baseidx_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB23_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB23_8
@@ -1691,15 +1685,14 @@ define <8 x i16> @mgather_baseidx_sext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB24_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB24_8
@@ -1845,16 +1838,15 @@ define <8 x i16> @mgather_baseidx_zext_v8i8_v8i16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB25_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lh a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB25_8
@@ -1997,14 +1989,13 @@ define <8 x i16> @mgather_baseidx_v8i16(ptr %base, <8 x i16> %idxs, <8 x i1> %m,
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB26_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lh a2, 0(a2)
 ; RV64ZVE32F-NEXT:    vmv.s.x v10, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB26_8
@@ -2076,7 +2067,7 @@ define <1 x i32> @mgather_v1i32(<1 x ptr> %ptrs, <1 x i1> %m, <1 x i32> %passthr
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB27_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB27_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vle32.v v8, (a0)
@@ -2158,12 +2149,12 @@ define <2 x i64> @mgather_v2i32_sextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    srai a1, a1, 31
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV32ZVE32F-NEXT:    srai a2, a2, 31
+; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vse32.v v9, (a0)
 ; RV32ZVE32F-NEXT:    addi a3, a0, 8
 ; RV32ZVE32F-NEXT:    vse32.v v8, (a3)
@@ -2221,10 +2212,10 @@ define <2 x i64> @mgather_v2i32_zextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
 ; RV32ZVE32F-NEXT:    sw zero, 12(a0)
 ; RV32ZVE32F-NEXT:    sw zero, 4(a0)
+; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vse32.v v9, (a0)
 ; RV32ZVE32F-NEXT:    addi a0, a0, 8
 ; RV32ZVE32F-NEXT:    vse32.v v8, (a0)
@@ -2633,13 +2624,12 @@ define <8 x i32> @mgather_baseidx_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB35_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -2782,13 +2772,12 @@ define <8 x i32> @mgather_baseidx_sext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB36_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -2936,14 +2925,13 @@ define <8 x i32> @mgather_baseidx_zext_v8i8_v8i32(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB37_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -3091,13 +3079,12 @@ define <8 x i32> @mgather_baseidx_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB38_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -3241,13 +3228,12 @@ define <8 x i32> @mgather_baseidx_sext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB39_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lw a2, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -3398,14 +3384,13 @@ define <8 x i32> @mgather_baseidx_zext_v8i16_v8i32(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB40_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
 ; RV64ZVE32F-NEXT:    slli a3, a3, 2
 ; RV64ZVE32F-NEXT:    add a3, a0, a3
 ; RV64ZVE32F-NEXT:    lw a3, 0(a3)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a3
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -3622,7 +3607,7 @@ define <1 x i64> @mgather_v1i64(<1 x ptr> %ptrs, <1 x i1> %m, <1 x i64> %passthr
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vfirst.m a2, v0
-; RV32ZVE32F-NEXT:    bnez a2, .LBB42_2
+; RV32ZVE32F-NEXT:    beqz a2, .LBB42_2
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
@@ -3635,7 +3620,7 @@ define <1 x i64> @mgather_v1i64(<1 x ptr> %ptrs, <1 x i1> %m, <1 x i64> %passthr
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a2, v0
-; RV64ZVE32F-NEXT:    bnez a2, .LBB42_2
+; RV64ZVE32F-NEXT:    beqz a2, .LBB42_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    ld a1, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB42_2: # %else
@@ -4469,7 +4454,6 @@ define <8 x i64> @mgather_baseidx_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB48_12
 ; RV64ZVE32F-NEXT:  # %bb.9: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v9
 ; RV64ZVE32F-NEXT:    slli a7, a7, 3
@@ -4489,7 +4473,6 @@ define <8 x i64> @mgather_baseidx_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi t0, a6, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB48_10
 ; RV64ZVE32F-NEXT:  .LBB48_13: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v8
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
 ; RV64ZVE32F-NEXT:    add t0, a1, t0
@@ -4497,7 +4480,6 @@ define <8 x i64> @mgather_baseidx_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi t1, a6, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB48_11
 ; RV64ZVE32F-NEXT:  .LBB48_14: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v9
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
@@ -4523,7 +4505,6 @@ define <8 x i64> @mgather_baseidx_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8 x i1>
 ; RV64ZVE32F-NEXT:    andi a6, a6, -128
 ; RV64ZVE32F-NEXT:    beqz a6, .LBB48_17
 ; RV64ZVE32F-NEXT:  .LBB48_19: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -4748,7 +4729,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB49_12
 ; RV64ZVE32F-NEXT:  # %bb.9: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v9
 ; RV64ZVE32F-NEXT:    slli a7, a7, 3
@@ -4768,7 +4748,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi t0, a6, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB49_10
 ; RV64ZVE32F-NEXT:  .LBB49_13: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v8
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
 ; RV64ZVE32F-NEXT:    add t0, a1, t0
@@ -4776,7 +4755,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi t1, a6, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB49_11
 ; RV64ZVE32F-NEXT:  .LBB49_14: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v9
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
@@ -4802,7 +4780,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a6, a6, -128
 ; RV64ZVE32F-NEXT:    beqz a6, .LBB49_17
 ; RV64ZVE32F-NEXT:  .LBB49_19: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -5031,7 +5008,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB50_12
 ; RV64ZVE32F-NEXT:  # %bb.9: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v9
 ; RV64ZVE32F-NEXT:    andi a7, a7, 255
@@ -5052,7 +5028,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB50_10
 ; RV64ZVE32F-NEXT:  .LBB50_13: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v8
 ; RV64ZVE32F-NEXT:    andi t0, t0, 255
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
@@ -5061,7 +5036,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB50_11
 ; RV64ZVE32F-NEXT:  .LBB50_14: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v9
 ; RV64ZVE32F-NEXT:    andi t1, t1, 255
@@ -5089,7 +5063,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i8_v8i64(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a5, a5, -128
 ; RV64ZVE32F-NEXT:    beqz a5, .LBB50_17
 ; RV64ZVE32F-NEXT:  .LBB50_19: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -5273,8 +5246,8 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-LABEL: mgather_baseidx_v8i16_v8i64:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vmv.x.s a6, v0
-; RV64ZVE32F-NEXT:    andi a3, a6, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a5, v0
+; RV64ZVE32F-NEXT:    andi a3, a5, 1
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB51_3
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
@@ -5282,14 +5255,14 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
 ; RV64ZVE32F-NEXT:    ld a3, 0(a3)
-; RV64ZVE32F-NEXT:    andi a4, a6, 2
+; RV64ZVE32F-NEXT:    andi a4, a5, 2
 ; RV64ZVE32F-NEXT:    bnez a4, .LBB51_4
 ; RV64ZVE32F-NEXT:  .LBB51_2:
 ; RV64ZVE32F-NEXT:    ld a4, 8(a2)
 ; RV64ZVE32F-NEXT:    j .LBB51_5
 ; RV64ZVE32F-NEXT:  .LBB51_3:
 ; RV64ZVE32F-NEXT:    ld a3, 0(a2)
-; RV64ZVE32F-NEXT:    andi a4, a6, 2
+; RV64ZVE32F-NEXT:    andi a4, a5, 2
 ; RV64ZVE32F-NEXT:    beqz a4, .LBB51_2
 ; RV64ZVE32F-NEXT:  .LBB51_4: # %cond.load1
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
@@ -5300,52 +5273,49 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    ld a4, 0(a4)
 ; RV64ZVE32F-NEXT:  .LBB51_5: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    andi a5, a6, 4
+; RV64ZVE32F-NEXT:    andi a6, a5, 4
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
-; RV64ZVE32F-NEXT:    beqz a5, .LBB51_7
+; RV64ZVE32F-NEXT:    beqz a6, .LBB51_7
 ; RV64ZVE32F-NEXT:  # %bb.6: # %cond.load4
-; RV64ZVE32F-NEXT:    vmv.x.s a5, v9
-; RV64ZVE32F-NEXT:    slli a5, a5, 3
-; RV64ZVE32F-NEXT:    add a5, a1, a5
-; RV64ZVE32F-NEXT:    ld a5, 0(a5)
+; RV64ZVE32F-NEXT:    vmv.x.s a6, v9
+; RV64ZVE32F-NEXT:    slli a6, a6, 3
+; RV64ZVE32F-NEXT:    add a6, a1, a6
+; RV64ZVE32F-NEXT:    ld a6, 0(a6)
 ; RV64ZVE32F-NEXT:    j .LBB51_8
 ; RV64ZVE32F-NEXT:  .LBB51_7:
-; RV64ZVE32F-NEXT:    ld a5, 16(a2)
+; RV64ZVE32F-NEXT:    ld a6, 16(a2)
 ; RV64ZVE32F-NEXT:  .LBB51_8: # %else5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    andi a7, a6, 8
+; RV64ZVE32F-NEXT:    andi a7, a5, 8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB51_12
 ; RV64ZVE32F-NEXT:  # %bb.9: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v9
 ; RV64ZVE32F-NEXT:    slli a7, a7, 3
 ; RV64ZVE32F-NEXT:    add a7, a1, a7
 ; RV64ZVE32F-NEXT:    ld a7, 0(a7)
-; RV64ZVE32F-NEXT:    andi t0, a6, 16
+; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    bnez t0, .LBB51_13
 ; RV64ZVE32F-NEXT:  .LBB51_10:
 ; RV64ZVE32F-NEXT:    ld t0, 32(a2)
-; RV64ZVE32F-NEXT:    andi t1, a6, 32
+; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    bnez t1, .LBB51_14
 ; RV64ZVE32F-NEXT:  .LBB51_11:
 ; RV64ZVE32F-NEXT:    ld t1, 40(a2)
 ; RV64ZVE32F-NEXT:    j .LBB51_15
 ; RV64ZVE32F-NEXT:  .LBB51_12:
 ; RV64ZVE32F-NEXT:    ld a7, 24(a2)
-; RV64ZVE32F-NEXT:    andi t0, a6, 16
+; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB51_10
 ; RV64ZVE32F-NEXT:  .LBB51_13: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v8
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
 ; RV64ZVE32F-NEXT:    add t0, a1, t0
 ; RV64ZVE32F-NEXT:    ld t0, 0(t0)
-; RV64ZVE32F-NEXT:    andi t1, a6, 32
+; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB51_11
 ; RV64ZVE32F-NEXT:  .LBB51_14: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v9
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
@@ -5353,7 +5323,7 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    ld t1, 0(t1)
 ; RV64ZVE32F-NEXT:  .LBB51_15: # %else14
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    andi t2, a6, 64
+; RV64ZVE32F-NEXT:    andi t2, a5, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB51_18
 ; RV64ZVE32F-NEXT:  # %bb.16: # %cond.load16
@@ -5361,17 +5331,16 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    slli t2, t2, 3
 ; RV64ZVE32F-NEXT:    add t2, a1, t2
 ; RV64ZVE32F-NEXT:    ld t2, 0(t2)
-; RV64ZVE32F-NEXT:    andi a6, a6, -128
-; RV64ZVE32F-NEXT:    bnez a6, .LBB51_19
+; RV64ZVE32F-NEXT:    andi a5, a5, -128
+; RV64ZVE32F-NEXT:    bnez a5, .LBB51_19
 ; RV64ZVE32F-NEXT:  .LBB51_17:
 ; RV64ZVE32F-NEXT:    ld a1, 56(a2)
 ; RV64ZVE32F-NEXT:    j .LBB51_20
 ; RV64ZVE32F-NEXT:  .LBB51_18:
 ; RV64ZVE32F-NEXT:    ld t2, 48(a2)
-; RV64ZVE32F-NEXT:    andi a6, a6, -128
-; RV64ZVE32F-NEXT:    beqz a6, .LBB51_17
+; RV64ZVE32F-NEXT:    andi a5, a5, -128
+; RV64ZVE32F-NEXT:    beqz a5, .LBB51_17
 ; RV64ZVE32F-NEXT:  .LBB51_19: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -5380,7 +5349,7 @@ define <8 x i64> @mgather_baseidx_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB51_20: # %else20
 ; RV64ZVE32F-NEXT:    sd a3, 0(a0)
 ; RV64ZVE32F-NEXT:    sd a4, 8(a0)
-; RV64ZVE32F-NEXT:    sd a5, 16(a0)
+; RV64ZVE32F-NEXT:    sd a6, 16(a0)
 ; RV64ZVE32F-NEXT:    sd a7, 24(a0)
 ; RV64ZVE32F-NEXT:    sd t0, 32(a0)
 ; RV64ZVE32F-NEXT:    sd t1, 40(a0)
@@ -5553,8 +5522,8 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-LABEL: mgather_baseidx_sext_v8i16_v8i64:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
-; RV64ZVE32F-NEXT:    vmv.x.s a6, v0
-; RV64ZVE32F-NEXT:    andi a3, a6, 1
+; RV64ZVE32F-NEXT:    vmv.x.s a5, v0
+; RV64ZVE32F-NEXT:    andi a3, a5, 1
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB52_3
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
@@ -5562,14 +5531,14 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
 ; RV64ZVE32F-NEXT:    ld a3, 0(a3)
-; RV64ZVE32F-NEXT:    andi a4, a6, 2
+; RV64ZVE32F-NEXT:    andi a4, a5, 2
 ; RV64ZVE32F-NEXT:    bnez a4, .LBB52_4
 ; RV64ZVE32F-NEXT:  .LBB52_2:
 ; RV64ZVE32F-NEXT:    ld a4, 8(a2)
 ; RV64ZVE32F-NEXT:    j .LBB52_5
 ; RV64ZVE32F-NEXT:  .LBB52_3:
 ; RV64ZVE32F-NEXT:    ld a3, 0(a2)
-; RV64ZVE32F-NEXT:    andi a4, a6, 2
+; RV64ZVE32F-NEXT:    andi a4, a5, 2
 ; RV64ZVE32F-NEXT:    beqz a4, .LBB52_2
 ; RV64ZVE32F-NEXT:  .LBB52_4: # %cond.load1
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
@@ -5580,52 +5549,49 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    ld a4, 0(a4)
 ; RV64ZVE32F-NEXT:  .LBB52_5: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    andi a5, a6, 4
+; RV64ZVE32F-NEXT:    andi a6, a5, 4
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 2
-; RV64ZVE32F-NEXT:    beqz a5, .LBB52_7
+; RV64ZVE32F-NEXT:    beqz a6, .LBB52_7
 ; RV64ZVE32F-NEXT:  # %bb.6: # %cond.load4
-; RV64ZVE32F-NEXT:    vmv.x.s a5, v9
-; RV64ZVE32F-NEXT:    slli a5, a5, 3
-; RV64ZVE32F-NEXT:    add a5, a1, a5
-; RV64ZVE32F-NEXT:    ld a5, 0(a5)
+; RV64ZVE32F-NEXT:    vmv.x.s a6, v9
+; RV64ZVE32F-NEXT:    slli a6, a6, 3
+; RV64ZVE32F-NEXT:    add a6, a1, a6
+; RV64ZVE32F-NEXT:    ld a6, 0(a6)
 ; RV64ZVE32F-NEXT:    j .LBB52_8
 ; RV64ZVE32F-NEXT:  .LBB52_7:
-; RV64ZVE32F-NEXT:    ld a5, 16(a2)
+; RV64ZVE32F-NEXT:    ld a6, 16(a2)
 ; RV64ZVE32F-NEXT:  .LBB52_8: # %else5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
-; RV64ZVE32F-NEXT:    andi a7, a6, 8
+; RV64ZVE32F-NEXT:    andi a7, a5, 8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB52_12
 ; RV64ZVE32F-NEXT:  # %bb.9: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v9
 ; RV64ZVE32F-NEXT:    slli a7, a7, 3
 ; RV64ZVE32F-NEXT:    add a7, a1, a7
 ; RV64ZVE32F-NEXT:    ld a7, 0(a7)
-; RV64ZVE32F-NEXT:    andi t0, a6, 16
+; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    bnez t0, .LBB52_13
 ; RV64ZVE32F-NEXT:  .LBB52_10:
 ; RV64ZVE32F-NEXT:    ld t0, 32(a2)
-; RV64ZVE32F-NEXT:    andi t1, a6, 32
+; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    bnez t1, .LBB52_14
 ; RV64ZVE32F-NEXT:  .LBB52_11:
 ; RV64ZVE32F-NEXT:    ld t1, 40(a2)
 ; RV64ZVE32F-NEXT:    j .LBB52_15
 ; RV64ZVE32F-NEXT:  .LBB52_12:
 ; RV64ZVE32F-NEXT:    ld a7, 24(a2)
-; RV64ZVE32F-NEXT:    andi t0, a6, 16
+; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB52_10
 ; RV64ZVE32F-NEXT:  .LBB52_13: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v8
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
 ; RV64ZVE32F-NEXT:    add t0, a1, t0
 ; RV64ZVE32F-NEXT:    ld t0, 0(t0)
-; RV64ZVE32F-NEXT:    andi t1, a6, 32
+; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB52_11
 ; RV64ZVE32F-NEXT:  .LBB52_14: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v9
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
@@ -5633,7 +5599,7 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    ld t1, 0(t1)
 ; RV64ZVE32F-NEXT:  .LBB52_15: # %else14
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
-; RV64ZVE32F-NEXT:    andi t2, a6, 64
+; RV64ZVE32F-NEXT:    andi t2, a5, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB52_18
 ; RV64ZVE32F-NEXT:  # %bb.16: # %cond.load16
@@ -5641,17 +5607,16 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    slli t2, t2, 3
 ; RV64ZVE32F-NEXT:    add t2, a1, t2
 ; RV64ZVE32F-NEXT:    ld t2, 0(t2)
-; RV64ZVE32F-NEXT:    andi a6, a6, -128
-; RV64ZVE32F-NEXT:    bnez a6, .LBB52_19
+; RV64ZVE32F-NEXT:    andi a5, a5, -128
+; RV64ZVE32F-NEXT:    bnez a5, .LBB52_19
 ; RV64ZVE32F-NEXT:  .LBB52_17:
 ; RV64ZVE32F-NEXT:    ld a1, 56(a2)
 ; RV64ZVE32F-NEXT:    j .LBB52_20
 ; RV64ZVE32F-NEXT:  .LBB52_18:
 ; RV64ZVE32F-NEXT:    ld t2, 48(a2)
-; RV64ZVE32F-NEXT:    andi a6, a6, -128
-; RV64ZVE32F-NEXT:    beqz a6, .LBB52_17
+; RV64ZVE32F-NEXT:    andi a5, a5, -128
+; RV64ZVE32F-NEXT:    beqz a5, .LBB52_17
 ; RV64ZVE32F-NEXT:  .LBB52_19: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -5660,7 +5625,7 @@ define <8 x i64> @mgather_baseidx_sext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:  .LBB52_20: # %else20
 ; RV64ZVE32F-NEXT:    sd a3, 0(a0)
 ; RV64ZVE32F-NEXT:    sd a4, 8(a0)
-; RV64ZVE32F-NEXT:    sd a5, 16(a0)
+; RV64ZVE32F-NEXT:    sd a6, 16(a0)
 ; RV64ZVE32F-NEXT:    sd a7, 24(a0)
 ; RV64ZVE32F-NEXT:    sd t0, 32(a0)
 ; RV64ZVE32F-NEXT:    sd t1, 40(a0)
@@ -5883,7 +5848,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 4
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB53_12
 ; RV64ZVE32F-NEXT:  # %bb.9: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v9
 ; RV64ZVE32F-NEXT:    and t0, t0, a5
@@ -5904,7 +5868,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi t1, a6, 16
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB53_10
 ; RV64ZVE32F-NEXT:  .LBB53_13: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v8
 ; RV64ZVE32F-NEXT:    and t1, t1, a5
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
@@ -5913,7 +5876,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi t2, a6, 32
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB53_11
 ; RV64ZVE32F-NEXT:  .LBB53_14: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t2, v9
 ; RV64ZVE32F-NEXT:    and t2, t2, a5
@@ -5941,7 +5903,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i16_v8i64(ptr %base, <8 x i16> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a6, a6, -128
 ; RV64ZVE32F-NEXT:    beqz a6, .LBB53_17
 ; RV64ZVE32F-NEXT:  .LBB53_19: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    and a2, a2, a5
@@ -6178,7 +6139,6 @@ define <8 x i64> @mgather_baseidx_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a7, a5, 8
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB54_7
 ; RV64ZVE32F-NEXT:  .LBB54_11: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v8
 ; RV64ZVE32F-NEXT:    slli a7, a7, 3
@@ -6187,7 +6147,6 @@ define <8 x i64> @mgather_baseidx_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB54_8
 ; RV64ZVE32F-NEXT:  .LBB54_12: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v10
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
 ; RV64ZVE32F-NEXT:    add t0, a1, t0
@@ -6195,14 +6154,12 @@ define <8 x i64> @mgather_baseidx_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB54_9
 ; RV64ZVE32F-NEXT:  .LBB54_13: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v8
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
 ; RV64ZVE32F-NEXT:    add t1, a1, t1
 ; RV64ZVE32F-NEXT:    ld t1, 0(t1)
 ; RV64ZVE32F-NEXT:  .LBB54_14: # %else14
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi t2, a5, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB54_17
@@ -6221,7 +6178,6 @@ define <8 x i64> @mgather_baseidx_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    andi a5, a5, -128
 ; RV64ZVE32F-NEXT:    beqz a5, .LBB54_16
 ; RV64ZVE32F-NEXT:  .LBB54_18: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -6456,7 +6412,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a7, a5, 8
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB55_7
 ; RV64ZVE32F-NEXT:  .LBB55_11: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v8
 ; RV64ZVE32F-NEXT:    slli a7, a7, 3
@@ -6465,7 +6420,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB55_8
 ; RV64ZVE32F-NEXT:  .LBB55_12: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v10
 ; RV64ZVE32F-NEXT:    slli t0, t0, 3
 ; RV64ZVE32F-NEXT:    add t0, a1, t0
@@ -6473,14 +6427,12 @@ define <8 x i64> @mgather_baseidx_sext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB55_9
 ; RV64ZVE32F-NEXT:  .LBB55_13: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v8
 ; RV64ZVE32F-NEXT:    slli t1, t1, 3
 ; RV64ZVE32F-NEXT:    add t1, a1, t1
 ; RV64ZVE32F-NEXT:    ld t1, 0(t1)
 ; RV64ZVE32F-NEXT:  .LBB55_14: # %else14
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi t2, a5, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB55_17
@@ -6499,7 +6451,6 @@ define <8 x i64> @mgather_baseidx_sext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a5, a5, -128
 ; RV64ZVE32F-NEXT:    beqz a5, .LBB55_16
 ; RV64ZVE32F-NEXT:  .LBB55_18: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -6738,7 +6689,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a7, a5, 8
 ; RV64ZVE32F-NEXT:    beqz a7, .LBB56_7
 ; RV64ZVE32F-NEXT:  .LBB56_11: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a7, v8
 ; RV64ZVE32F-NEXT:    slli a7, a7, 32
@@ -6748,7 +6698,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi t0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz t0, .LBB56_8
 ; RV64ZVE32F-NEXT:  .LBB56_12: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s t0, v10
 ; RV64ZVE32F-NEXT:    slli t0, t0, 32
 ; RV64ZVE32F-NEXT:    srli t0, t0, 29
@@ -6757,7 +6706,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi t1, a5, 32
 ; RV64ZVE32F-NEXT:    beqz t1, .LBB56_9
 ; RV64ZVE32F-NEXT:  .LBB56_13: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s t1, v8
 ; RV64ZVE32F-NEXT:    slli t1, t1, 32
@@ -6765,7 +6713,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    add t1, a1, t1
 ; RV64ZVE32F-NEXT:    ld t1, 0(t1)
 ; RV64ZVE32F-NEXT:  .LBB56_14: # %else14
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi t2, a5, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    beqz t2, .LBB56_17
@@ -6785,7 +6732,6 @@ define <8 x i64> @mgather_baseidx_zext_v8i32_v8i64(ptr %base, <8 x i32> %idxs, <
 ; RV64ZVE32F-NEXT:    andi a5, a5, -128
 ; RV64ZVE32F-NEXT:    beqz a5, .LBB56_16
 ; RV64ZVE32F-NEXT:  .LBB56_18: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 32
@@ -6829,191 +6775,156 @@ define <8 x i64> @mgather_baseidx_v8i64(ptr %base, <8 x i64> %idxs, <8 x i1> %m,
 ;
 ; RV32ZVE32F-LABEL: mgather_baseidx_v8i64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    addi sp, sp, -96
-; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 96
-; RV32ZVE32F-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s2, 84(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s3, 80(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s4, 76(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s5, 72(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s6, 68(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s7, 64(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s8, 60(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s9, 56(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s10, 52(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s11, 48(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    .cfi_offset ra, -4
-; RV32ZVE32F-NEXT:    .cfi_offset s0, -8
-; RV32ZVE32F-NEXT:    .cfi_offset s2, -12
-; RV32ZVE32F-NEXT:    .cfi_offset s3, -16
-; RV32ZVE32F-NEXT:    .cfi_offset s4, -20
-; RV32ZVE32F-NEXT:    .cfi_offset s5, -24
-; RV32ZVE32F-NEXT:    .cfi_offset s6, -28
-; RV32ZVE32F-NEXT:    .cfi_offset s7, -32
-; RV32ZVE32F-NEXT:    .cfi_offset s8, -36
-; RV32ZVE32F-NEXT:    .cfi_offset s9, -40
-; RV32ZVE32F-NEXT:    .cfi_offset s10, -44
-; RV32ZVE32F-NEXT:    .cfi_offset s11, -48
-; RV32ZVE32F-NEXT:    addi s0, sp, 96
-; RV32ZVE32F-NEXT:    .cfi_def_cfa s0, 0
-; RV32ZVE32F-NEXT:    andi sp, sp, -32
-; RV32ZVE32F-NEXT:    lw a4, 60(a3)
-; RV32ZVE32F-NEXT:    lw a5, 56(a3)
-; RV32ZVE32F-NEXT:    lw a6, 52(a3)
-; RV32ZVE32F-NEXT:    lw a7, 48(a3)
-; RV32ZVE32F-NEXT:    lw t0, 44(a3)
-; RV32ZVE32F-NEXT:    lw t1, 40(a3)
-; RV32ZVE32F-NEXT:    lw t2, 36(a3)
-; RV32ZVE32F-NEXT:    lw t3, 32(a3)
-; RV32ZVE32F-NEXT:    lw t4, 28(a3)
-; RV32ZVE32F-NEXT:    lw t5, 24(a3)
-; RV32ZVE32F-NEXT:    lw t6, 20(a3)
-; RV32ZVE32F-NEXT:    lw s2, 16(a3)
-; RV32ZVE32F-NEXT:    lw s3, 12(a3)
-; RV32ZVE32F-NEXT:    lw s5, 8(a3)
-; RV32ZVE32F-NEXT:    lw s4, 4(a3)
-; RV32ZVE32F-NEXT:    lw a3, 0(a3)
-; RV32ZVE32F-NEXT:    lw s6, 0(a2)
-; RV32ZVE32F-NEXT:    lw s7, 8(a2)
-; RV32ZVE32F-NEXT:    lw s8, 16(a2)
-; RV32ZVE32F-NEXT:    lw s9, 24(a2)
-; RV32ZVE32F-NEXT:    lw s10, 56(a2)
-; RV32ZVE32F-NEXT:    lw s11, 48(a2)
-; RV32ZVE32F-NEXT:    lw ra, 40(a2)
-; RV32ZVE32F-NEXT:    lw a2, 32(a2)
-; RV32ZVE32F-NEXT:    sw s10, 28(sp)
-; RV32ZVE32F-NEXT:    sw s11, 24(sp)
-; RV32ZVE32F-NEXT:    sw ra, 20(sp)
-; RV32ZVE32F-NEXT:    sw a2, 16(sp)
-; RV32ZVE32F-NEXT:    sw s9, 12(sp)
-; RV32ZVE32F-NEXT:    sw s8, 8(sp)
-; RV32ZVE32F-NEXT:    sw s7, 4(sp)
-; RV32ZVE32F-NEXT:    sw s6, 0(sp)
-; RV32ZVE32F-NEXT:    mv a2, sp
+; RV32ZVE32F-NEXT:    addi sp, sp, -16
+; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 16
+; RV32ZVE32F-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    .cfi_offset s0, -4
+; RV32ZVE32F-NEXT:    .cfi_offset s1, -8
+; RV32ZVE32F-NEXT:    lw a4, 56(a2)
+; RV32ZVE32F-NEXT:    lw a5, 48(a2)
+; RV32ZVE32F-NEXT:    lw a6, 40(a2)
+; RV32ZVE32F-NEXT:    lw a7, 32(a2)
+; RV32ZVE32F-NEXT:    lw t0, 24(a2)
+; RV32ZVE32F-NEXT:    lw t1, 0(a2)
+; RV32ZVE32F-NEXT:    lw t2, 8(a2)
+; RV32ZVE32F-NEXT:    lw a2, 16(a2)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vle32.v v8, (a2)
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t1
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t2
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
 ; RV32ZVE32F-NEXT:    vsll.vi v8, v8, 3
 ; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
-; RV32ZVE32F-NEXT:    andi a2, a1, 1
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_10
-; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a2, a1, 2
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_11
-; RV32ZVE32F-NEXT:  .LBB57_2: # %else2
-; RV32ZVE32F-NEXT:    andi a2, a1, 4
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_12
-; RV32ZVE32F-NEXT:  .LBB57_3: # %else5
-; RV32ZVE32F-NEXT:    andi a2, a1, 8
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_13
-; RV32ZVE32F-NEXT:  .LBB57_4: # %else8
-; RV32ZVE32F-NEXT:    andi a2, a1, 16
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_14
-; RV32ZVE32F-NEXT:  .LBB57_5: # %else11
-; RV32ZVE32F-NEXT:    andi a2, a1, 32
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_15
-; RV32ZVE32F-NEXT:  .LBB57_6: # %else14
-; RV32ZVE32F-NEXT:    andi a2, a1, 64
-; RV32ZVE32F-NEXT:    bnez a2, .LBB57_16
-; RV32ZVE32F-NEXT:  .LBB57_7: # %else17
-; RV32ZVE32F-NEXT:    andi a1, a1, -128
+; RV32ZVE32F-NEXT:    vmv.x.s t0, v0
+; RV32ZVE32F-NEXT:    andi a1, t0, 1
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB57_9
-; RV32ZVE32F-NEXT:  .LBB57_8: # %cond.load19
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV32ZVE32F-NEXT:    lw a4, 4(a1)
-; RV32ZVE32F-NEXT:    lw a5, 0(a1)
-; RV32ZVE32F-NEXT:  .LBB57_9: # %else20
-; RV32ZVE32F-NEXT:    sw a3, 0(a0)
-; RV32ZVE32F-NEXT:    sw s4, 4(a0)
-; RV32ZVE32F-NEXT:    sw s5, 8(a0)
-; RV32ZVE32F-NEXT:    sw s3, 12(a0)
-; RV32ZVE32F-NEXT:    sw s2, 16(a0)
-; RV32ZVE32F-NEXT:    sw t6, 20(a0)
-; RV32ZVE32F-NEXT:    sw t5, 24(a0)
-; RV32ZVE32F-NEXT:    sw t4, 28(a0)
-; RV32ZVE32F-NEXT:    sw t3, 32(a0)
-; RV32ZVE32F-NEXT:    sw t2, 36(a0)
-; RV32ZVE32F-NEXT:    sw t1, 40(a0)
-; RV32ZVE32F-NEXT:    sw t0, 44(a0)
-; RV32ZVE32F-NEXT:    sw a7, 48(a0)
-; RV32ZVE32F-NEXT:    sw a6, 52(a0)
-; RV32ZVE32F-NEXT:    sw a5, 56(a0)
-; RV32ZVE32F-NEXT:    sw a4, 60(a0)
-; RV32ZVE32F-NEXT:    addi sp, s0, -96
-; RV32ZVE32F-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s2, 84(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s3, 80(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s4, 76(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s5, 72(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s6, 68(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s7, 64(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s8, 60(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s9, 56(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s10, 52(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s11, 48(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    addi sp, sp, 96
-; RV32ZVE32F-NEXT:    ret
-; RV32ZVE32F-NEXT:  .LBB57_10: # %cond.load
+; RV32ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
-; RV32ZVE32F-NEXT:    lw s4, 4(a2)
-; RV32ZVE32F-NEXT:    lw a3, 0(a2)
-; RV32ZVE32F-NEXT:    andi a2, a1, 2
-; RV32ZVE32F-NEXT:    beqz a2, .LBB57_2
-; RV32ZVE32F-NEXT:  .LBB57_11: # %cond.load1
+; RV32ZVE32F-NEXT:    lw a1, 4(a2)
+; RV32ZVE32F-NEXT:    lw a2, 0(a2)
+; RV32ZVE32F-NEXT:    andi a4, t0, 2
+; RV32ZVE32F-NEXT:    bnez a4, .LBB57_10
+; RV32ZVE32F-NEXT:  .LBB57_2:
+; RV32ZVE32F-NEXT:    lw a4, 12(a3)
+; RV32ZVE32F-NEXT:    lw a5, 8(a3)
+; RV32ZVE32F-NEXT:    andi a6, t0, 4
+; RV32ZVE32F-NEXT:    bnez a6, .LBB57_11
+; RV32ZVE32F-NEXT:  .LBB57_3:
+; RV32ZVE32F-NEXT:    lw a6, 20(a3)
+; RV32ZVE32F-NEXT:    lw a7, 16(a3)
+; RV32ZVE32F-NEXT:    andi t1, t0, 8
+; RV32ZVE32F-NEXT:    bnez t1, .LBB57_12
+; RV32ZVE32F-NEXT:  .LBB57_4:
+; RV32ZVE32F-NEXT:    lw t1, 28(a3)
+; RV32ZVE32F-NEXT:    lw t2, 24(a3)
+; RV32ZVE32F-NEXT:    andi t3, t0, 16
+; RV32ZVE32F-NEXT:    bnez t3, .LBB57_13
+; RV32ZVE32F-NEXT:  .LBB57_5:
+; RV32ZVE32F-NEXT:    lw t3, 36(a3)
+; RV32ZVE32F-NEXT:    lw t4, 32(a3)
+; RV32ZVE32F-NEXT:    andi t5, t0, 32
+; RV32ZVE32F-NEXT:    bnez t5, .LBB57_14
+; RV32ZVE32F-NEXT:  .LBB57_6:
+; RV32ZVE32F-NEXT:    lw t5, 44(a3)
+; RV32ZVE32F-NEXT:    lw t6, 40(a3)
+; RV32ZVE32F-NEXT:    andi s0, t0, 64
+; RV32ZVE32F-NEXT:    bnez s0, .LBB57_15
+; RV32ZVE32F-NEXT:  .LBB57_7:
+; RV32ZVE32F-NEXT:    lw s0, 52(a3)
+; RV32ZVE32F-NEXT:    lw s1, 48(a3)
+; RV32ZVE32F-NEXT:    andi t0, t0, -128
+; RV32ZVE32F-NEXT:    bnez t0, .LBB57_16
+; RV32ZVE32F-NEXT:  .LBB57_8:
+; RV32ZVE32F-NEXT:    lw t0, 60(a3)
+; RV32ZVE32F-NEXT:    lw a3, 56(a3)
+; RV32ZVE32F-NEXT:    j .LBB57_17
+; RV32ZVE32F-NEXT:  .LBB57_9:
+; RV32ZVE32F-NEXT:    lw a1, 4(a3)
+; RV32ZVE32F-NEXT:    lw a2, 0(a3)
+; RV32ZVE32F-NEXT:    andi a4, t0, 2
+; RV32ZVE32F-NEXT:    beqz a4, .LBB57_2
+; RV32ZVE32F-NEXT:  .LBB57_10: # %cond.load1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV32ZVE32F-NEXT:    lw s3, 4(a2)
-; RV32ZVE32F-NEXT:    lw s5, 0(a2)
-; RV32ZVE32F-NEXT:    andi a2, a1, 4
-; RV32ZVE32F-NEXT:    beqz a2, .LBB57_3
-; RV32ZVE32F-NEXT:  .LBB57_12: # %cond.load4
+; RV32ZVE32F-NEXT:    vmv.x.s a5, v10
+; RV32ZVE32F-NEXT:    lw a4, 4(a5)
+; RV32ZVE32F-NEXT:    lw a5, 0(a5)
+; RV32ZVE32F-NEXT:    andi a6, t0, 4
+; RV32ZVE32F-NEXT:    beqz a6, .LBB57_3
+; RV32ZVE32F-NEXT:  .LBB57_11: # %cond.load4
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV32ZVE32F-NEXT:    lw t6, 4(a2)
-; RV32ZVE32F-NEXT:    lw s2, 0(a2)
-; RV32ZVE32F-NEXT:    andi a2, a1, 8
-; RV32ZVE32F-NEXT:    beqz a2, .LBB57_4
-; RV32ZVE32F-NEXT:  .LBB57_13: # %cond.load7
+; RV32ZVE32F-NEXT:    vmv.x.s a7, v10
+; RV32ZVE32F-NEXT:    lw a6, 4(a7)
+; RV32ZVE32F-NEXT:    lw a7, 0(a7)
+; RV32ZVE32F-NEXT:    andi t1, t0, 8
+; RV32ZVE32F-NEXT:    beqz t1, .LBB57_4
+; RV32ZVE32F-NEXT:  .LBB57_12: # %cond.load7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV32ZVE32F-NEXT:    lw t4, 4(a2)
-; RV32ZVE32F-NEXT:    lw t5, 0(a2)
-; RV32ZVE32F-NEXT:    andi a2, a1, 16
-; RV32ZVE32F-NEXT:    beqz a2, .LBB57_5
-; RV32ZVE32F-NEXT:  .LBB57_14: # %cond.load10
+; RV32ZVE32F-NEXT:    vmv.x.s t2, v10
+; RV32ZVE32F-NEXT:    lw t1, 4(t2)
+; RV32ZVE32F-NEXT:    lw t2, 0(t2)
+; RV32ZVE32F-NEXT:    andi t3, t0, 16
+; RV32ZVE32F-NEXT:    beqz t3, .LBB57_5
+; RV32ZVE32F-NEXT:  .LBB57_13: # %cond.load10
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV32ZVE32F-NEXT:    lw t2, 4(a2)
-; RV32ZVE32F-NEXT:    lw t3, 0(a2)
-; RV32ZVE32F-NEXT:    andi a2, a1, 32
-; RV32ZVE32F-NEXT:    beqz a2, .LBB57_6
-; RV32ZVE32F-NEXT:  .LBB57_15: # %cond.load13
+; RV32ZVE32F-NEXT:    vmv.x.s t4, v10
+; RV32ZVE32F-NEXT:    lw t3, 4(t4)
+; RV32ZVE32F-NEXT:    lw t4, 0(t4)
+; RV32ZVE32F-NEXT:    andi t5, t0, 32
+; RV32ZVE32F-NEXT:    beqz t5, .LBB57_6
+; RV32ZVE32F-NEXT:  .LBB57_14: # %cond.load13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV32ZVE32F-NEXT:    lw t0, 4(a2)
-; RV32ZVE32F-NEXT:    lw t1, 0(a2)
-; RV32ZVE32F-NEXT:    andi a2, a1, 64
-; RV32ZVE32F-NEXT:    beqz a2, .LBB57_7
-; RV32ZVE32F-NEXT:  .LBB57_16: # %cond.load16
+; RV32ZVE32F-NEXT:    vmv.x.s t6, v10
+; RV32ZVE32F-NEXT:    lw t5, 4(t6)
+; RV32ZVE32F-NEXT:    lw t6, 0(t6)
+; RV32ZVE32F-NEXT:    andi s0, t0, 64
+; RV32ZVE32F-NEXT:    beqz s0, .LBB57_7
+; RV32ZVE32F-NEXT:  .LBB57_15: # %cond.load16
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v10
-; RV32ZVE32F-NEXT:    lw a6, 4(a2)
-; RV32ZVE32F-NEXT:    lw a7, 0(a2)
-; RV32ZVE32F-NEXT:    andi a1, a1, -128
-; RV32ZVE32F-NEXT:    bnez a1, .LBB57_8
-; RV32ZVE32F-NEXT:    j .LBB57_9
+; RV32ZVE32F-NEXT:    vmv.x.s s1, v10
+; RV32ZVE32F-NEXT:    lw s0, 4(s1)
+; RV32ZVE32F-NEXT:    lw s1, 0(s1)
+; RV32ZVE32F-NEXT:    andi t0, t0, -128
+; RV32ZVE32F-NEXT:    beqz t0, .LBB57_8
+; RV32ZVE32F-NEXT:  .LBB57_16: # %cond.load19
+; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vmv.x.s a3, v8
+; RV32ZVE32F-NEXT:    lw t0, 4(a3)
+; RV32ZVE32F-NEXT:    lw a3, 0(a3)
+; RV32ZVE32F-NEXT:  .LBB57_17: # %else20
+; RV32ZVE32F-NEXT:    sw a2, 0(a0)
+; RV32ZVE32F-NEXT:    sw a1, 4(a0)
+; RV32ZVE32F-NEXT:    sw a5, 8(a0)
+; RV32ZVE32F-NEXT:    sw a4, 12(a0)
+; RV32ZVE32F-NEXT:    sw a7, 16(a0)
+; RV32ZVE32F-NEXT:    sw a6, 20(a0)
+; RV32ZVE32F-NEXT:    sw t2, 24(a0)
+; RV32ZVE32F-NEXT:    sw t1, 28(a0)
+; RV32ZVE32F-NEXT:    sw t4, 32(a0)
+; RV32ZVE32F-NEXT:    sw t3, 36(a0)
+; RV32ZVE32F-NEXT:    sw t6, 40(a0)
+; RV32ZVE32F-NEXT:    sw t5, 44(a0)
+; RV32ZVE32F-NEXT:    sw s1, 48(a0)
+; RV32ZVE32F-NEXT:    sw s0, 52(a0)
+; RV32ZVE32F-NEXT:    sw a3, 56(a0)
+; RV32ZVE32F-NEXT:    sw t0, 60(a0)
+; RV32ZVE32F-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s1, 8(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    addi sp, sp, 16
+; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_baseidx_v8i64:
 ; RV64ZVE32F:       # %bb.0:
@@ -7149,7 +7060,7 @@ define <1 x half> @mgather_v1f16(<1 x ptr> %ptrs, <1 x i1> %m, <1 x half> %passt
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB58_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB58_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vle16.v v8, (a0)
@@ -7582,15 +7493,14 @@ define <8 x half> @mgather_baseidx_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8 x i1
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB64_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB64_8
@@ -7731,15 +7641,14 @@ define <8 x half> @mgather_baseidx_sext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB65_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB65_8
@@ -7885,16 +7794,15 @@ define <8 x half> @mgather_baseidx_zext_v8i8_v8f16(ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB66_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flh fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB66_8
@@ -8037,14 +7945,13 @@ define <8 x half> @mgather_baseidx_v8f16(ptr %base, <8 x i16> %idxs, <8 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB67_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flh fa5, 0(a2)
 ; RV64ZVE32F-NEXT:    vfmv.s.f v10, fa5
-; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, tu, ma
+; RV64ZVE32F-NEXT:    vsetvli zero, zero, e16, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v9, v10, 3
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB67_8
@@ -8116,7 +8023,7 @@ define <1 x float> @mgather_v1f32(<1 x ptr> %ptrs, <1 x i1> %m, <1 x float> %pas
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB68_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB68_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vle32.v v8, (a0)
@@ -8547,13 +8454,12 @@ define <8 x float> @mgather_baseidx_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <8 x i
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB74_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -8696,13 +8602,12 @@ define <8 x float> @mgather_baseidx_sext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB75_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -8850,14 +8755,13 @@ define <8 x float> @mgather_baseidx_zext_v8i8_v8f32(ptr %base, <8 x i8> %idxs, <
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB76_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -9005,13 +8909,12 @@ define <8 x float> @mgather_baseidx_v8i16_v8f32(ptr %base, <8 x i16> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB77_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -9155,13 +9058,12 @@ define <8 x float> @mgather_baseidx_sext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB78_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a2)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -9312,14 +9214,13 @@ define <8 x float> @mgather_baseidx_zext_v8i16_v8f32(ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB79_13: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
 ; RV64ZVE32F-NEXT:    slli a3, a3, 2
 ; RV64ZVE32F-NEXT:    add a3, a0, a3
 ; RV64ZVE32F-NEXT:    flw fa5, 0(a3)
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vfmv.s.f v12, fa5
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -9536,7 +9437,7 @@ define <1 x double> @mgather_v1f64(<1 x ptr> %ptrs, <1 x i1> %m, <1 x double> %p
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vfirst.m a0, v0
-; RV32ZVE32F-NEXT:    bnez a0, .LBB81_2
+; RV32ZVE32F-NEXT:    beqz a0, .LBB81_2
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
@@ -9548,7 +9449,7 @@ define <1 x double> @mgather_v1f64(<1 x ptr> %ptrs, <1 x i1> %m, <1 x double> %p
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB81_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB81_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.load
 ; RV64ZVE32F-NEXT:    fld fa0, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB81_2: # %else
@@ -10215,7 +10116,6 @@ define <8 x double> @mgather_baseidx_v8i8_v8f64(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB87_10
 ; RV64ZVE32F-NEXT:  .LBB87_9: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10235,7 +10135,6 @@ define <8 x double> @mgather_baseidx_v8i8_v8f64(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_14
 ; RV64ZVE32F-NEXT:  # %bb.13: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -10252,7 +10151,6 @@ define <8 x double> @mgather_baseidx_v8i8_v8f64(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB87_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10261,7 +10159,6 @@ define <8 x double> @mgather_baseidx_v8i8_v8f64(ptr %base, <8 x i8> %idxs, <8 x
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB87_8
 ; RV64ZVE32F-NEXT:  .LBB87_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
@@ -10433,7 +10330,6 @@ define <8 x double> @mgather_baseidx_sext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB88_10
 ; RV64ZVE32F-NEXT:  .LBB88_9: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10453,7 +10349,6 @@ define <8 x double> @mgather_baseidx_sext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_14
 ; RV64ZVE32F-NEXT:  # %bb.13: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -10470,7 +10365,6 @@ define <8 x double> @mgather_baseidx_sext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB88_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10479,7 +10373,6 @@ define <8 x double> @mgather_baseidx_sext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB88_8
 ; RV64ZVE32F-NEXT:  .LBB88_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
@@ -10655,7 +10548,6 @@ define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB89_10
 ; RV64ZVE32F-NEXT:  .LBB89_9: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    andi a3, a3, 255
@@ -10677,7 +10569,6 @@ define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB89_14
 ; RV64ZVE32F-NEXT:  # %bb.13: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -10695,7 +10586,6 @@ define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB89_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    andi a3, a3, 255
@@ -10705,7 +10595,6 @@ define <8 x double> @mgather_baseidx_zext_v8i8_v8f64(ptr %base, <8 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB89_8
 ; RV64ZVE32F-NEXT:  .LBB89_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    andi a3, a3, 255
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10880,7 +10769,6 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB90_10
 ; RV64ZVE32F-NEXT:  .LBB90_9: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10900,7 +10788,6 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB90_14
 ; RV64ZVE32F-NEXT:  # %bb.13: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -10917,7 +10804,6 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 ; RV64ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB90_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -10926,7 +10812,6 @@ define <8 x double> @mgather_baseidx_v8i16_v8f64(ptr %base, <8 x i16> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB90_8
 ; RV64ZVE32F-NEXT:  .LBB90_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
@@ -11099,7 +10984,6 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB91_10
 ; RV64ZVE32F-NEXT:  .LBB91_9: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -11119,7 +11003,6 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB91_14
 ; RV64ZVE32F-NEXT:  # %bb.13: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -11136,7 +11019,6 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB91_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -11145,7 +11027,6 @@ define <8 x double> @mgather_baseidx_sext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB91_8
 ; RV64ZVE32F-NEXT:  .LBB91_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
@@ -11324,7 +11205,6 @@ define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    andi a4, a3, 32
 ; RV64ZVE32F-NEXT:    beqz a4, .LBB92_10
 ; RV64ZVE32F-NEXT:  .LBB92_9: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a4, v9
 ; RV64ZVE32F-NEXT:    and a4, a4, a2
@@ -11346,7 +11226,6 @@ define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a3, -128
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB92_14
 ; RV64ZVE32F-NEXT:  # %bb.13: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    and a2, a3, a2
@@ -11364,7 +11243,6 @@ define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    fsd fa7, 56(a0)
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB92_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a4, v9
 ; RV64ZVE32F-NEXT:    and a4, a4, a2
@@ -11374,7 +11252,6 @@ define <8 x double> @mgather_baseidx_zext_v8i16_v8f64(ptr %base, <8 x i16> %idxs
 ; RV64ZVE32F-NEXT:    andi a4, a3, 16
 ; RV64ZVE32F-NEXT:    beqz a4, .LBB92_8
 ; RV64ZVE32F-NEXT:  .LBB92_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a4, v8
 ; RV64ZVE32F-NEXT:    and a4, a4, a2
 ; RV64ZVE32F-NEXT:    slli a4, a4, 3
@@ -11542,14 +11419,12 @@ define <8 x double> @mgather_baseidx_v8i32_v8f64(ptr %base, <8 x i32> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB93_9
 ; RV64ZVE32F-NEXT:  .LBB93_8: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
 ; RV64ZVE32F-NEXT:    fld fa5, 0(a3)
 ; RV64ZVE32F-NEXT:  .LBB93_9: # %else14
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a3, a2, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB93_11
@@ -11562,7 +11437,6 @@ define <8 x double> @mgather_baseidx_v8i32_v8f64(ptr %base, <8 x i32> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB93_13
 ; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -11586,7 +11460,6 @@ define <8 x double> @mgather_baseidx_v8i32_v8f64(ptr %base, <8 x i32> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB93_6
 ; RV64ZVE32F-NEXT:  .LBB93_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -11595,7 +11468,6 @@ define <8 x double> @mgather_baseidx_v8i32_v8f64(ptr %base, <8 x i32> %idxs, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB93_7
 ; RV64ZVE32F-NEXT:  .LBB93_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v10
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
@@ -11761,14 +11633,12 @@ define <8 x double> @mgather_baseidx_sext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB94_9
 ; RV64ZVE32F-NEXT:  .LBB94_8: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
 ; RV64ZVE32F-NEXT:    fld fa5, 0(a3)
 ; RV64ZVE32F-NEXT:  .LBB94_9: # %else14
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a3, a2, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB94_11
@@ -11781,7 +11651,6 @@ define <8 x double> @mgather_baseidx_sext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB94_13
 ; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -11805,7 +11674,6 @@ define <8 x double> @mgather_baseidx_sext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB94_6
 ; RV64ZVE32F-NEXT:  .LBB94_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -11814,7 +11682,6 @@ define <8 x double> @mgather_baseidx_sext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB94_7
 ; RV64ZVE32F-NEXT:  .LBB94_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v10
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
@@ -11983,7 +11850,6 @@ define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB95_9
 ; RV64ZVE32F-NEXT:  .LBB95_8: # %cond.load13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 32
@@ -11991,7 +11857,6 @@ define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    add a3, a1, a3
 ; RV64ZVE32F-NEXT:    fld fa5, 0(a3)
 ; RV64ZVE32F-NEXT:  .LBB95_9: # %else14
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a3, a2, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB95_11
@@ -12005,7 +11870,6 @@ define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB95_13
 ; RV64ZVE32F-NEXT:  # %bb.12: # %cond.load19
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 32
@@ -12031,7 +11895,6 @@ define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 8
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB95_6
 ; RV64ZVE32F-NEXT:  .LBB95_15: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    slli a3, a3, 32
@@ -12041,7 +11904,6 @@ define <8 x double> @mgather_baseidx_zext_v8i32_v8f64(ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB95_7
 ; RV64ZVE32F-NEXT:  .LBB95_16: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v10
 ; RV64ZVE32F-NEXT:    slli a3, a3, 32
 ; RV64ZVE32F-NEXT:    srli a3, a3, 29
@@ -12077,34 +11939,23 @@ define <8 x double> @mgather_baseidx_v8f64(ptr %base, <8 x i64> %idxs, <8 x i1>
 ;
 ; RV32ZVE32F-LABEL: mgather_baseidx_v8f64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    addi sp, sp, -64
-; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 64
-; RV32ZVE32F-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    .cfi_offset ra, -4
-; RV32ZVE32F-NEXT:    .cfi_offset s0, -8
-; RV32ZVE32F-NEXT:    addi s0, sp, 64
-; RV32ZVE32F-NEXT:    .cfi_def_cfa s0, 0
-; RV32ZVE32F-NEXT:    andi sp, sp, -32
-; RV32ZVE32F-NEXT:    lw a3, 0(a2)
-; RV32ZVE32F-NEXT:    lw a4, 8(a2)
-; RV32ZVE32F-NEXT:    lw a5, 16(a2)
-; RV32ZVE32F-NEXT:    lw a6, 24(a2)
-; RV32ZVE32F-NEXT:    lw a7, 56(a2)
-; RV32ZVE32F-NEXT:    lw t0, 48(a2)
-; RV32ZVE32F-NEXT:    lw t1, 40(a2)
-; RV32ZVE32F-NEXT:    lw a2, 32(a2)
-; RV32ZVE32F-NEXT:    sw a7, 28(sp)
-; RV32ZVE32F-NEXT:    sw t0, 24(sp)
-; RV32ZVE32F-NEXT:    sw t1, 20(sp)
-; RV32ZVE32F-NEXT:    sw a2, 16(sp)
-; RV32ZVE32F-NEXT:    sw a6, 12(sp)
-; RV32ZVE32F-NEXT:    sw a5, 8(sp)
-; RV32ZVE32F-NEXT:    sw a4, 4(sp)
-; RV32ZVE32F-NEXT:    sw a3, 0(sp)
-; RV32ZVE32F-NEXT:    mv a2, sp
+; RV32ZVE32F-NEXT:    lw a3, 56(a2)
+; RV32ZVE32F-NEXT:    lw a4, 48(a2)
+; RV32ZVE32F-NEXT:    lw a5, 40(a2)
+; RV32ZVE32F-NEXT:    lw a6, 32(a2)
+; RV32ZVE32F-NEXT:    lw a7, 24(a2)
+; RV32ZVE32F-NEXT:    lw t0, 0(a2)
+; RV32ZVE32F-NEXT:    lw t1, 8(a2)
+; RV32ZVE32F-NEXT:    lw a2, 16(a2)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vle32.v v8, (a2)
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t1
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
 ; RV32ZVE32F-NEXT:    vsll.vi v8, v8, 3
 ; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
@@ -12146,10 +11997,6 @@ define <8 x double> @mgather_baseidx_v8f64(ptr %base, <8 x i64> %idxs, <8 x i1>
 ; RV32ZVE32F-NEXT:    fsd fa5, 40(a0)
 ; RV32ZVE32F-NEXT:    fsd fa6, 48(a0)
 ; RV32ZVE32F-NEXT:    fsd fa7, 56(a0)
-; RV32ZVE32F-NEXT:    addi sp, s0, -64
-; RV32ZVE32F-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    addi sp, sp, 64
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB96_10: # %cond.load
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e32, m2, ta, ma
@@ -12462,7 +12309,6 @@ define <16 x i8> @mgather_baseidx_v16i8(ptr %base, <16 x i8> %idxs, <16 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv1r.v v8, v9
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB97_26: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -12517,7 +12363,6 @@ define <16 x i8> @mgather_baseidx_v16i8(ptr %base, <16 x i8> %idxs, <16 x i1> %m
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB97_14
 ; RV64ZVE32F-NEXT:    j .LBB97_15
 ; RV64ZVE32F-NEXT:  .LBB97_31: # %cond.load31
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -12690,7 +12535,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v12, 4
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB98_19
 ; RV64ZVE32F-NEXT:  # %bb.18: # %cond.load31
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v13, v13, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -12885,7 +12729,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv2r.v v8, v10
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB98_50: # %cond.load7
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v12, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -12985,7 +12828,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB98_27
 ; RV64ZVE32F-NEXT:    j .LBB98_28
 ; RV64ZVE32F-NEXT:  .LBB98_58: # %cond.load55
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v12, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -13048,7 +12890,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB98_38
 ; RV64ZVE32F-NEXT:    j .LBB98_39
 ; RV64ZVE32F-NEXT:  .LBB98_63: # %cond.load79
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
index 38103378a0089..1da6953cf28af 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-scatter.ll
@@ -33,7 +33,7 @@ define void @mscatter_v1i8(<1 x i8> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB0_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB0_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v8, (a0)
@@ -528,8 +528,8 @@ define void @mscatter_baseidx_v8i8(<8 x i8> %val, ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB9_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -539,8 +539,8 @@ define void @mscatter_baseidx_v8i8(<8 x i8> %val, ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:  # %bb.5: # %cond.store3
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v11, (a2)
 ; RV64ZVE32F-NEXT:  .LBB9_6: # %else4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -558,8 +558,8 @@ define void @mscatter_baseidx_v8i8(<8 x i8> %val, ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB9_10: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -572,12 +572,11 @@ define void @mscatter_baseidx_v8i8(<8 x i8> %val, ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:  .LBB9_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB9_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB9_8
@@ -594,8 +593,8 @@ define void @mscatter_baseidx_v8i8(<8 x i8> %val, ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:  .LBB9_15: # %cond.store11
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB9_12
@@ -604,8 +603,8 @@ define void @mscatter_baseidx_v8i8(<8 x i8> %val, ptr %base, <8 x i8> %idxs, <8
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v8, (a0)
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i8, ptr %base, <8 x i8> %idxs
@@ -638,7 +637,7 @@ define void @mscatter_v1i16(<1 x i16> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB10_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB10_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v8, (a0)
@@ -1124,7 +1123,6 @@ define void @mscatter_baseidx_v8i8_v8i16(<8 x i16> %val, ptr %base, <8 x i8> %id
 ; RV64ZVE32F-NEXT:  .LBB18_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB18_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -1255,7 +1253,6 @@ define void @mscatter_baseidx_sext_v8i8_v8i16(<8 x i16> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB19_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB19_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -1391,7 +1388,6 @@ define void @mscatter_baseidx_zext_v8i8_v8i16(<8 x i16> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB20_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB20_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -1482,8 +1478,8 @@ define void @mscatter_baseidx_v8i16(<8 x i16> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB21_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
@@ -1494,8 +1490,8 @@ define void @mscatter_baseidx_v8i16(<8 x i16> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v11, (a2)
 ; RV64ZVE32F-NEXT:  .LBB21_6: # %else4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
@@ -1514,8 +1510,8 @@ define void @mscatter_baseidx_v8i16(<8 x i16> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB21_10: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
@@ -1528,13 +1524,12 @@ define void @mscatter_baseidx_v8i16(<8 x i16> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:  .LBB21_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB21_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB21_8
@@ -1553,8 +1548,8 @@ define void @mscatter_baseidx_v8i16(<8 x i16> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB21_12
@@ -1564,8 +1559,8 @@ define void @mscatter_baseidx_v8i16(<8 x i16> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    slli a1, a1, 1
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v8, (a0)
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i16> %idxs
@@ -1598,7 +1593,7 @@ define void @mscatter_v1i32(<1 x i32> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB22_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB22_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v8, (a0)
@@ -1672,20 +1667,18 @@ define void @mscatter_v2i64_truncstore_v2i32(<2 x i64> %val, <2 x ptr> %ptrs, <2
 ; RV32ZVE32F-LABEL: mscatter_v2i64_truncstore_v2i32:
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    lw a1, 0(a0)
-; RV32ZVE32F-NEXT:    addi a0, a0, 8
+; RV32ZVE32F-NEXT:    lw a0, 8(a0)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vlse32.v v9, (a0), zero
-; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
-; RV32ZVE32F-NEXT:    vmv.s.x v9, a1
+; RV32ZVE32F-NEXT:    vslide1down.vx v9, v8, a1
+; RV32ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
 ; RV32ZVE32F-NEXT:    vsoxei32.v v9, (zero), v8, v0.t
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_v2i64_truncstore_v2i32:
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, tu, ma
-; RV64ZVE32F-NEXT:    vmv.s.x v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; RV64ZVE32F-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v0
 ; RV64ZVE32F-NEXT:    andi a1, a0, 1
@@ -2029,7 +2022,6 @@ define void @mscatter_baseidx_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8> %id
 ; RV64ZVE32F-NEXT:  .LBB29_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB29_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2159,7 +2151,6 @@ define void @mscatter_baseidx_sext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB30_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB30_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2294,7 +2285,6 @@ define void @mscatter_baseidx_zext_v8i8_v8i32(<8 x i32> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB31_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB31_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -2430,7 +2420,6 @@ define void @mscatter_baseidx_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i16> %
 ; RV64ZVE32F-NEXT:  .LBB32_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB32_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2561,7 +2550,6 @@ define void @mscatter_baseidx_sext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB33_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB33_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -2699,7 +2687,6 @@ define void @mscatter_baseidx_zext_v8i16_v8i32(<8 x i32> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB34_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB34_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v11
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
@@ -2788,8 +2775,8 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB35_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
@@ -2813,8 +2800,8 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB35_9: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
@@ -2830,8 +2817,8 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v14, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB35_6
@@ -2841,8 +2828,8 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB35_7
@@ -2861,8 +2848,8 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v12, (a2)
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB35_11
@@ -2872,8 +2859,8 @@ define void @mscatter_baseidx_v8i32(<8 x i32> %val, ptr %base, <8 x i32> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV64ZVE32F-NEXT:    slli a1, a1, 2
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v8, (a0)
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i32, ptr %base, <8 x i32> %idxs
@@ -2900,7 +2887,7 @@ define void @mscatter_v1i64(<1 x i64> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vfirst.m a2, v0
-; RV32ZVE32F-NEXT:    bnez a2, .LBB36_2
+; RV32ZVE32F-NEXT:    beqz a2, .LBB36_2
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
@@ -2913,7 +2900,7 @@ define void @mscatter_v1i64(<1 x i64> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a2, v0
-; RV64ZVE32F-NEXT:    bnez a2, .LBB36_2
+; RV64ZVE32F-NEXT:    beqz a2, .LBB36_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    sd a0, 0(a1)
 ; RV64ZVE32F-NEXT:  .LBB36_2: # %else
@@ -3647,7 +3634,6 @@ define void @mscatter_baseidx_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %id
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB42_10
 ; RV64ZVE32F-NEXT:  .LBB42_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -3664,7 +3650,6 @@ define void @mscatter_baseidx_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %id
 ; RV64ZVE32F-NEXT:  .LBB42_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB42_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -3673,7 +3658,6 @@ define void @mscatter_baseidx_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %id
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB42_8
 ; RV64ZVE32F-NEXT:  .LBB42_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
@@ -3689,7 +3673,6 @@ define void @mscatter_baseidx_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8> %id
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB42_12
 ; RV64ZVE32F-NEXT:  .LBB42_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -3894,7 +3877,6 @@ define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB43_10
 ; RV64ZVE32F-NEXT:  .LBB43_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -3911,7 +3893,6 @@ define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB43_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB43_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -3920,7 +3901,6 @@ define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB43_8
 ; RV64ZVE32F-NEXT:  .LBB43_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
@@ -3936,7 +3916,6 @@ define void @mscatter_baseidx_sext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB43_12
 ; RV64ZVE32F-NEXT:  .LBB43_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4145,7 +4124,6 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_10
 ; RV64ZVE32F-NEXT:  .LBB44_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    andi a0, a0, 255
@@ -4163,7 +4141,6 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:  .LBB44_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB44_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    andi a0, a0, 255
@@ -4173,7 +4150,6 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_8
 ; RV64ZVE32F-NEXT:  .LBB44_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    andi a0, a0, 255
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4191,7 +4167,6 @@ define void @mscatter_baseidx_zext_v8i8_v8i64(<8 x i64> %val, ptr %base, <8 x i8
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB44_12
 ; RV64ZVE32F-NEXT:  .LBB44_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    andi a0, a0, 255
@@ -4399,7 +4374,6 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB45_10
 ; RV64ZVE32F-NEXT:  .LBB45_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4416,7 +4390,6 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV64ZVE32F-NEXT:  .LBB45_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB45_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4425,7 +4398,6 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB45_8
 ; RV64ZVE32F-NEXT:  .LBB45_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
@@ -4441,7 +4413,6 @@ define void @mscatter_baseidx_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i16> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB45_12
 ; RV64ZVE32F-NEXT:  .LBB45_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4647,7 +4618,6 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB46_10
 ; RV64ZVE32F-NEXT:  .LBB46_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4664,7 +4634,6 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB46_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB46_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4673,7 +4642,6 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB46_8
 ; RV64ZVE32F-NEXT:  .LBB46_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
@@ -4689,7 +4657,6 @@ define void @mscatter_baseidx_sext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB46_12
 ; RV64ZVE32F-NEXT:  .LBB46_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4901,7 +4868,6 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a5, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_10
 ; RV64ZVE32F-NEXT:  .LBB47_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    and a0, a0, a4
@@ -4919,7 +4885,6 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB47_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB47_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v9
 ; RV64ZVE32F-NEXT:    and a0, a0, a4
@@ -4929,7 +4894,6 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a5, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_8
 ; RV64ZVE32F-NEXT:  .LBB47_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    and a0, a0, a4
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -4947,7 +4911,6 @@ define void @mscatter_baseidx_zext_v8i16_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a5, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB47_12
 ; RV64ZVE32F-NEXT:  .LBB47_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    and a0, a0, a4
@@ -5148,14 +5111,12 @@ define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB48_9
 ; RV64ZVE32F-NEXT:  .LBB48_8: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a5, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB48_9: # %else10
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a0, a4, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB48_15
@@ -5172,7 +5133,6 @@ define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB48_6
 ; RV64ZVE32F-NEXT:  .LBB48_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -5181,7 +5141,6 @@ define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB48_7
 ; RV64ZVE32F-NEXT:  .LBB48_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v10
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
@@ -5197,7 +5156,6 @@ define void @mscatter_baseidx_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i32> %
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB48_11
 ; RV64ZVE32F-NEXT:  .LBB48_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -5396,14 +5354,12 @@ define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB49_9
 ; RV64ZVE32F-NEXT:  .LBB49_8: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a5, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB49_9: # %else10
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a0, a4, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB49_15
@@ -5420,7 +5376,6 @@ define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB49_6
 ; RV64ZVE32F-NEXT:  .LBB49_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -5429,7 +5384,6 @@ define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB49_7
 ; RV64ZVE32F-NEXT:  .LBB49_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v10
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
@@ -5445,7 +5399,6 @@ define void @mscatter_baseidx_sext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB49_11
 ; RV64ZVE32F-NEXT:  .LBB49_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 3
@@ -5647,7 +5600,6 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 32
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB50_9
 ; RV64ZVE32F-NEXT:  .LBB50_8: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 32
@@ -5655,7 +5607,6 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    add a0, a1, a0
 ; RV64ZVE32F-NEXT:    sd a5, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB50_9: # %else10
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a0, a4, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    bnez a0, .LBB50_15
@@ -5673,7 +5624,6 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 8
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB50_6
 ; RV64ZVE32F-NEXT:  .LBB50_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 32
@@ -5683,7 +5633,6 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, 16
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB50_7
 ; RV64ZVE32F-NEXT:  .LBB50_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v10
 ; RV64ZVE32F-NEXT:    slli a0, a0, 32
 ; RV64ZVE32F-NEXT:    srli a0, a0, 29
@@ -5701,7 +5650,6 @@ define void @mscatter_baseidx_zext_v8i32_v8i64(<8 x i64> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:    andi a0, a4, -128
 ; RV64ZVE32F-NEXT:    beqz a0, .LBB50_11
 ; RV64ZVE32F-NEXT:  .LBB50_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a0, v8
 ; RV64ZVE32F-NEXT:    slli a0, a0, 32
@@ -5734,22 +5682,19 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_v8i64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    addi sp, sp, -96
-; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 96
-; RV32ZVE32F-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s2, 84(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s3, 80(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s4, 76(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s5, 72(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s6, 68(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s7, 64(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s8, 60(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s9, 56(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s10, 52(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s11, 48(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    .cfi_offset ra, -4
-; RV32ZVE32F-NEXT:    .cfi_offset s0, -8
+; RV32ZVE32F-NEXT:    addi sp, sp, -48
+; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 48
+; RV32ZVE32F-NEXT:    sw s0, 44(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s1, 40(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s2, 36(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s3, 32(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s4, 28(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s6, 20(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s7, 16(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
+; RV32ZVE32F-NEXT:    .cfi_offset s0, -4
+; RV32ZVE32F-NEXT:    .cfi_offset s1, -8
 ; RV32ZVE32F-NEXT:    .cfi_offset s2, -12
 ; RV32ZVE32F-NEXT:    .cfi_offset s3, -16
 ; RV32ZVE32F-NEXT:    .cfi_offset s4, -20
@@ -5757,12 +5702,6 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    .cfi_offset s6, -28
 ; RV32ZVE32F-NEXT:    .cfi_offset s7, -32
 ; RV32ZVE32F-NEXT:    .cfi_offset s8, -36
-; RV32ZVE32F-NEXT:    .cfi_offset s9, -40
-; RV32ZVE32F-NEXT:    .cfi_offset s10, -44
-; RV32ZVE32F-NEXT:    .cfi_offset s11, -48
-; RV32ZVE32F-NEXT:    addi s0, sp, 96
-; RV32ZVE32F-NEXT:    .cfi_def_cfa s0, 0
-; RV32ZVE32F-NEXT:    andi sp, sp, -32
 ; RV32ZVE32F-NEXT:    lw a3, 60(a0)
 ; RV32ZVE32F-NEXT:    lw a4, 56(a0)
 ; RV32ZVE32F-NEXT:    lw a5, 52(a0)
@@ -5775,55 +5714,51 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    lw t4, 24(a0)
 ; RV32ZVE32F-NEXT:    lw t5, 20(a0)
 ; RV32ZVE32F-NEXT:    lw t6, 16(a0)
-; RV32ZVE32F-NEXT:    lw s3, 12(a0)
-; RV32ZVE32F-NEXT:    lw s2, 8(a0)
-; RV32ZVE32F-NEXT:    lw s5, 4(a0)
-; RV32ZVE32F-NEXT:    lw s4, 0(a0)
-; RV32ZVE32F-NEXT:    lw a0, 0(a2)
-; RV32ZVE32F-NEXT:    lw s6, 8(a2)
-; RV32ZVE32F-NEXT:    lw s7, 16(a2)
-; RV32ZVE32F-NEXT:    lw s8, 24(a2)
-; RV32ZVE32F-NEXT:    lw s9, 56(a2)
-; RV32ZVE32F-NEXT:    lw s10, 48(a2)
-; RV32ZVE32F-NEXT:    lw s11, 40(a2)
-; RV32ZVE32F-NEXT:    lw a2, 32(a2)
-; RV32ZVE32F-NEXT:    sw s9, 28(sp)
-; RV32ZVE32F-NEXT:    sw s10, 24(sp)
-; RV32ZVE32F-NEXT:    sw s11, 20(sp)
-; RV32ZVE32F-NEXT:    sw a2, 16(sp)
-; RV32ZVE32F-NEXT:    sw s8, 12(sp)
-; RV32ZVE32F-NEXT:    sw s7, 8(sp)
-; RV32ZVE32F-NEXT:    sw s6, 4(sp)
-; RV32ZVE32F-NEXT:    sw a0, 0(sp)
-; RV32ZVE32F-NEXT:    mv a0, sp
+; RV32ZVE32F-NEXT:    lw s1, 12(a0)
+; RV32ZVE32F-NEXT:    lw s0, 8(a0)
+; RV32ZVE32F-NEXT:    lw s2, 56(a2)
+; RV32ZVE32F-NEXT:    lw s3, 48(a2)
+; RV32ZVE32F-NEXT:    lw s4, 40(a2)
+; RV32ZVE32F-NEXT:    lw s5, 32(a2)
+; RV32ZVE32F-NEXT:    lw s6, 24(a2)
+; RV32ZVE32F-NEXT:    lw s7, 0(a2)
+; RV32ZVE32F-NEXT:    lw s8, 8(a2)
+; RV32ZVE32F-NEXT:    lw a2, 16(a2)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vle32.v v8, (a0)
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s8
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s5
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s4
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s3
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, s2
 ; RV32ZVE32F-NEXT:    vsll.vi v8, v8, 3
 ; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v0
-; RV32ZVE32F-NEXT:    andi a1, a0, 1
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_10
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v0
+; RV32ZVE32F-NEXT:    andi a2, a1, 1
+; RV32ZVE32F-NEXT:    bnez a2, .LBB51_10
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
-; RV32ZVE32F-NEXT:    andi a1, a0, 2
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_11
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
+; RV32ZVE32F-NEXT:    bnez a0, .LBB51_11
 ; RV32ZVE32F-NEXT:  .LBB51_2: # %else2
-; RV32ZVE32F-NEXT:    andi a1, a0, 4
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_12
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
+; RV32ZVE32F-NEXT:    bnez a0, .LBB51_12
 ; RV32ZVE32F-NEXT:  .LBB51_3: # %else4
-; RV32ZVE32F-NEXT:    andi a1, a0, 8
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_13
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
+; RV32ZVE32F-NEXT:    bnez a0, .LBB51_13
 ; RV32ZVE32F-NEXT:  .LBB51_4: # %else6
-; RV32ZVE32F-NEXT:    andi a1, a0, 16
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_14
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
+; RV32ZVE32F-NEXT:    bnez a0, .LBB51_14
 ; RV32ZVE32F-NEXT:  .LBB51_5: # %else8
-; RV32ZVE32F-NEXT:    andi a1, a0, 32
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_15
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
+; RV32ZVE32F-NEXT:    bnez a0, .LBB51_15
 ; RV32ZVE32F-NEXT:  .LBB51_6: # %else10
-; RV32ZVE32F-NEXT:    andi a1, a0, 64
-; RV32ZVE32F-NEXT:    bnez a1, .LBB51_16
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
+; RV32ZVE32F-NEXT:    bnez a0, .LBB51_16
 ; RV32ZVE32F-NEXT:  .LBB51_7: # %else12
-; RV32ZVE32F-NEXT:    andi a0, a0, -128
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    beqz a0, .LBB51_9
 ; RV32ZVE32F-NEXT:  .LBB51_8: # %cond.store13
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
@@ -5832,75 +5767,73 @@ define void @mscatter_baseidx_v8i64(<8 x i64> %val, ptr %base, <8 x i64> %idxs,
 ; RV32ZVE32F-NEXT:    sw a4, 0(a0)
 ; RV32ZVE32F-NEXT:    sw a3, 4(a0)
 ; RV32ZVE32F-NEXT:  .LBB51_9: # %else14
-; RV32ZVE32F-NEXT:    addi sp, s0, -96
-; RV32ZVE32F-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s2, 84(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s3, 80(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s4, 76(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s5, 72(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s6, 68(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s7, 64(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s8, 60(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s9, 56(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s10, 52(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s11, 48(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    addi sp, sp, 96
+; RV32ZVE32F-NEXT:    lw s0, 44(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s2, 36(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s3, 32(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s4, 28(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s5, 24(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s6, 20(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s7, 16(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
+; RV32ZVE32F-NEXT:    addi sp, sp, 48
 ; RV32ZVE32F-NEXT:    ret
 ; RV32ZVE32F-NEXT:  .LBB51_10: # %cond.store
+; RV32ZVE32F-NEXT:    lw a2, 4(a0)
+; RV32ZVE32F-NEXT:    lw a0, 0(a0)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
-; RV32ZVE32F-NEXT:    sw s5, 4(a1)
-; RV32ZVE32F-NEXT:    sw s4, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a0, 2
-; RV32ZVE32F-NEXT:    beqz a1, .LBB51_2
+; RV32ZVE32F-NEXT:    vmv.x.s s2, v8
+; RV32ZVE32F-NEXT:    sw a2, 4(s2)
+; RV32ZVE32F-NEXT:    sw a0, 0(s2)
+; RV32ZVE32F-NEXT:    andi a0, a1, 2
+; RV32ZVE32F-NEXT:    beqz a0, .LBB51_2
 ; RV32ZVE32F-NEXT:  .LBB51_11: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    sw s3, 4(a1)
-; RV32ZVE32F-NEXT:    sw s2, 0(a1)
-; RV32ZVE32F-NEXT:    andi a1, a0, 4
-; RV32ZVE32F-NEXT:    beqz a1, .LBB51_3
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    sw s1, 4(a0)
+; RV32ZVE32F-NEXT:    sw s0, 0(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 4
+; RV32ZVE32F-NEXT:    beqz a0, .LBB51_3
 ; RV32ZVE32F-NEXT:  .LBB51_12: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    sw t6, 0(a1)
-; RV32ZVE32F-NEXT:    sw t5, 4(a1)
-; RV32ZVE32F-NEXT:    andi a1, a0, 8
-; RV32ZVE32F-NEXT:    beqz a1, .LBB51_4
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    sw t6, 0(a0)
+; RV32ZVE32F-NEXT:    sw t5, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 8
+; RV32ZVE32F-NEXT:    beqz a0, .LBB51_4
 ; RV32ZVE32F-NEXT:  .LBB51_13: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    sw t4, 0(a1)
-; RV32ZVE32F-NEXT:    sw t3, 4(a1)
-; RV32ZVE32F-NEXT:    andi a1, a0, 16
-; RV32ZVE32F-NEXT:    beqz a1, .LBB51_5
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    sw t4, 0(a0)
+; RV32ZVE32F-NEXT:    sw t3, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 16
+; RV32ZVE32F-NEXT:    beqz a0, .LBB51_5
 ; RV32ZVE32F-NEXT:  .LBB51_14: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    sw t2, 0(a1)
-; RV32ZVE32F-NEXT:    sw t1, 4(a1)
-; RV32ZVE32F-NEXT:    andi a1, a0, 32
-; RV32ZVE32F-NEXT:    beqz a1, .LBB51_6
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    sw t2, 0(a0)
+; RV32ZVE32F-NEXT:    sw t1, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 32
+; RV32ZVE32F-NEXT:    beqz a0, .LBB51_6
 ; RV32ZVE32F-NEXT:  .LBB51_15: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    sw t0, 0(a1)
-; RV32ZVE32F-NEXT:    sw a7, 4(a1)
-; RV32ZVE32F-NEXT:    andi a1, a0, 64
-; RV32ZVE32F-NEXT:    beqz a1, .LBB51_7
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    sw t0, 0(a0)
+; RV32ZVE32F-NEXT:    sw a7, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, 64
+; RV32ZVE32F-NEXT:    beqz a0, .LBB51_7
 ; RV32ZVE32F-NEXT:  .LBB51_16: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
-; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
-; RV32ZVE32F-NEXT:    sw a6, 0(a1)
-; RV32ZVE32F-NEXT:    sw a5, 4(a1)
-; RV32ZVE32F-NEXT:    andi a0, a0, -128
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v10
+; RV32ZVE32F-NEXT:    sw a6, 0(a0)
+; RV32ZVE32F-NEXT:    sw a5, 4(a0)
+; RV32ZVE32F-NEXT:    andi a0, a1, -128
 ; RV32ZVE32F-NEXT:    bnez a0, .LBB51_8
 ; RV32ZVE32F-NEXT:    j .LBB51_9
 ;
@@ -6041,7 +5974,7 @@ define void @mscatter_v1f16(<1 x half> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB52_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB52_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v8, (a0)
@@ -6417,7 +6350,6 @@ define void @mscatter_baseidx_v8i8_v8f16(<8 x half> %val, ptr %base, <8 x i8> %i
 ; RV64ZVE32F-NEXT:  .LBB58_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB58_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -6548,7 +6480,6 @@ define void @mscatter_baseidx_sext_v8i8_v8f16(<8 x half> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB59_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB59_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
@@ -6684,7 +6615,6 @@ define void @mscatter_baseidx_zext_v8i8_v8f16(<8 x half> %val, ptr %base, <8 x i
 ; RV64ZVE32F-NEXT:  .LBB60_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB60_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -6775,8 +6705,8 @@ define void @mscatter_baseidx_v8f16(<8 x half> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB61_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
@@ -6787,8 +6717,8 @@ define void @mscatter_baseidx_v8f16(<8 x half> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v11, (a2)
 ; RV64ZVE32F-NEXT:  .LBB61_6: # %else4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
@@ -6807,8 +6737,8 @@ define void @mscatter_baseidx_v8f16(<8 x half> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB61_10: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e16, mf2, ta, ma
@@ -6821,13 +6751,12 @@ define void @mscatter_baseidx_v8f16(<8 x half> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:  .LBB61_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB61_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB61_8
@@ -6846,8 +6775,8 @@ define void @mscatter_baseidx_v8f16(<8 x half> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 1
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB61_12
@@ -6857,8 +6786,8 @@ define void @mscatter_baseidx_v8f16(<8 x half> %val, ptr %base, <8 x i16> %idxs,
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    slli a1, a1, 1
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse16.v v8, (a0)
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds half, ptr %base, <8 x i16> %idxs
@@ -6891,7 +6820,7 @@ define void @mscatter_v1f32(<1 x float> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB62_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB62_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v8, (a0)
@@ -7266,7 +7195,6 @@ define void @mscatter_baseidx_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x i8> %
 ; RV64ZVE32F-NEXT:  .LBB68_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB68_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -7396,7 +7324,6 @@ define void @mscatter_baseidx_sext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:  .LBB69_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB69_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -7531,7 +7458,6 @@ define void @mscatter_baseidx_zext_v8i8_v8f32(<8 x float> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:  .LBB70_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB70_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -7667,7 +7593,6 @@ define void @mscatter_baseidx_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x i16>
 ; RV64ZVE32F-NEXT:  .LBB71_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB71_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -7798,7 +7723,6 @@ define void @mscatter_baseidx_sext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:  .LBB72_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB72_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
@@ -7936,7 +7860,6 @@ define void @mscatter_baseidx_zext_v8i16_v8f32(<8 x float> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:  .LBB73_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB73_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v11
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
@@ -8025,8 +7948,8 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB74_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e32, m2, ta, ma
@@ -8050,8 +7973,8 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB74_9: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
@@ -8067,8 +7990,8 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v14, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB74_6
@@ -8078,8 +8001,8 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB74_7
@@ -8098,8 +8021,8 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 2
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v12, (a2)
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB74_11
@@ -8109,8 +8032,8 @@ define void @mscatter_baseidx_v8f32(<8 x float> %val, ptr %base, <8 x i32> %idxs
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV64ZVE32F-NEXT:    slli a1, a1, 2
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse32.v v8, (a0)
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds float, ptr %base, <8 x i32> %idxs
@@ -8137,7 +8060,7 @@ define void @mscatter_v1f64(<1 x double> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV32ZVE32F:       # %bb.0:
 ; RV32ZVE32F-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vfirst.m a0, v0
-; RV32ZVE32F-NEXT:    bnez a0, .LBB75_2
+; RV32ZVE32F-NEXT:    beqz a0, .LBB75_2
 ; RV32ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV32ZVE32F-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
@@ -8149,7 +8072,7 @@ define void @mscatter_v1f64(<1 x double> %val, <1 x ptr> %ptrs, <1 x i1> %m) {
 ; RV64ZVE32F:       # %bb.0:
 ; RV64ZVE32F-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vfirst.m a1, v0
-; RV64ZVE32F-NEXT:    bnez a1, .LBB75_2
+; RV64ZVE32F-NEXT:    beqz a1, .LBB75_2
 ; RV64ZVE32F-NEXT:  # %bb.1: # %cond.store
 ; RV64ZVE32F-NEXT:    fsd fa0, 0(a0)
 ; RV64ZVE32F-NEXT:  .LBB75_2: # %else
@@ -8745,7 +8668,6 @@ define void @mscatter_baseidx_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x i8>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB81_10
 ; RV64ZVE32F-NEXT:  .LBB81_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -8762,7 +8684,6 @@ define void @mscatter_baseidx_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x i8>
 ; RV64ZVE32F-NEXT:  .LBB81_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB81_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -8771,7 +8692,6 @@ define void @mscatter_baseidx_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x i8>
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB81_8
 ; RV64ZVE32F-NEXT:  .LBB81_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -8787,7 +8707,6 @@ define void @mscatter_baseidx_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x i8>
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB81_12
 ; RV64ZVE32F-NEXT:  .LBB81_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 3
@@ -8948,7 +8867,6 @@ define void @mscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB82_10
 ; RV64ZVE32F-NEXT:  .LBB82_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -8965,7 +8883,6 @@ define void @mscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:  .LBB82_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB82_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -8974,7 +8891,6 @@ define void @mscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB82_8
 ; RV64ZVE32F-NEXT:  .LBB82_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -8990,7 +8906,6 @@ define void @mscatter_baseidx_sext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB82_12
 ; RV64ZVE32F-NEXT:  .LBB82_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 3
@@ -9155,7 +9070,6 @@ define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB83_10
 ; RV64ZVE32F-NEXT:  .LBB83_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -9173,7 +9087,6 @@ define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:  .LBB83_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB83_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
@@ -9183,7 +9096,6 @@ define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB83_8
 ; RV64ZVE32F-NEXT:  .LBB83_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    andi a2, a2, 255
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -9201,7 +9113,6 @@ define void @mscatter_baseidx_zext_v8i8_v8f64(<8 x double> %val, ptr %base, <8 x
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB83_12
 ; RV64ZVE32F-NEXT:  .LBB83_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    andi a1, a1, 255
@@ -9365,7 +9276,6 @@ define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB84_10
 ; RV64ZVE32F-NEXT:  .LBB84_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -9382,7 +9292,6 @@ define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16
 ; RV64ZVE32F-NEXT:  .LBB84_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB84_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -9391,7 +9300,6 @@ define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB84_8
 ; RV64ZVE32F-NEXT:  .LBB84_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -9407,7 +9315,6 @@ define void @mscatter_baseidx_v8i16_v8f64(<8 x double> %val, ptr %base, <8 x i16
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB84_12
 ; RV64ZVE32F-NEXT:  .LBB84_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 3
@@ -9569,7 +9476,6 @@ define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB85_10
 ; RV64ZVE32F-NEXT:  .LBB85_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -9586,7 +9492,6 @@ define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:  .LBB85_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB85_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -9595,7 +9500,6 @@ define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB85_8
 ; RV64ZVE32F-NEXT:  .LBB85_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -9611,7 +9515,6 @@ define void @mscatter_baseidx_sext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB85_12
 ; RV64ZVE32F-NEXT:  .LBB85_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 3
@@ -9779,7 +9682,6 @@ define void @mscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 32
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB86_10
 ; RV64ZVE32F-NEXT:  .LBB86_9: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
@@ -9797,7 +9699,6 @@ define void @mscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:  .LBB86_12: # %else14
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB86_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v9
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
@@ -9807,7 +9708,6 @@ define void @mscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a3, a2, 16
 ; RV64ZVE32F-NEXT:    beqz a3, .LBB86_8
 ; RV64ZVE32F-NEXT:  .LBB86_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a3, v8
 ; RV64ZVE32F-NEXT:    and a3, a3, a1
 ; RV64ZVE32F-NEXT:    slli a3, a3, 3
@@ -9825,7 +9725,6 @@ define void @mscatter_baseidx_zext_v8i16_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a2, -128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB86_12
 ; RV64ZVE32F-NEXT:  .LBB86_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    and a1, a2, a1
@@ -9982,14 +9881,12 @@ define void @mscatter_baseidx_v8i32_v8f64(<8 x double> %val, ptr %base, <8 x i32
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_9
 ; RV64ZVE32F-NEXT:  .LBB87_8: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    fsd fa5, 0(a2)
 ; RV64ZVE32F-NEXT:  .LBB87_9: # %else10
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB87_15
@@ -10006,7 +9903,6 @@ define void @mscatter_baseidx_v8i32_v8f64(<8 x double> %val, ptr %base, <8 x i32
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_6
 ; RV64ZVE32F-NEXT:  .LBB87_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -10015,7 +9911,6 @@ define void @mscatter_baseidx_v8i32_v8f64(<8 x double> %val, ptr %base, <8 x i32
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB87_7
 ; RV64ZVE32F-NEXT:  .LBB87_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -10031,7 +9926,6 @@ define void @mscatter_baseidx_v8i32_v8f64(<8 x double> %val, ptr %base, <8 x i32
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB87_11
 ; RV64ZVE32F-NEXT:  .LBB87_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 3
@@ -10186,14 +10080,12 @@ define void @mscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_9
 ; RV64ZVE32F-NEXT:  .LBB88_8: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    fsd fa5, 0(a2)
 ; RV64ZVE32F-NEXT:  .LBB88_9: # %else10
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB88_15
@@ -10210,7 +10102,6 @@ define void @mscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_6
 ; RV64ZVE32F-NEXT:  .LBB88_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
@@ -10219,7 +10110,6 @@ define void @mscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB88_7
 ; RV64ZVE32F-NEXT:  .LBB88_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 3
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
@@ -10235,7 +10125,6 @@ define void @mscatter_baseidx_sext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB88_11
 ; RV64ZVE32F-NEXT:  .LBB88_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 3
@@ -10393,7 +10282,6 @@ define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 32
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB89_9
 ; RV64ZVE32F-NEXT:  .LBB89_8: # %cond.store9
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 32
@@ -10401,7 +10289,6 @@ define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    fsd fa5, 0(a2)
 ; RV64ZVE32F-NEXT:  .LBB89_9: # %else10
-; RV64ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    andi a2, a1, 64
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 2
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB89_15
@@ -10419,7 +10306,6 @@ define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 8
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB89_6
 ; RV64ZVE32F-NEXT:  .LBB89_13: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    slli a2, a2, 32
@@ -10429,7 +10315,6 @@ define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB89_7
 ; RV64ZVE32F-NEXT:  .LBB89_14: # %cond.store7
-; RV64ZVE32F-NEXT:    vsetivli zero, 0, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    slli a2, a2, 32
 ; RV64ZVE32F-NEXT:    srli a2, a2, 29
@@ -10447,7 +10332,6 @@ define void @mscatter_baseidx_zext_v8i32_v8f64(<8 x double> %val, ptr %base, <8
 ; RV64ZVE32F-NEXT:    andi a1, a1, -128
 ; RV64ZVE32F-NEXT:    beqz a1, .LBB89_11
 ; RV64ZVE32F-NEXT:  .LBB89_16: # %cond.store13
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV64ZVE32F-NEXT:    slli a1, a1, 32
@@ -10480,121 +10364,106 @@ define void @mscatter_baseidx_v8f64(<8 x double> %val, ptr %base, <8 x i64> %idx
 ;
 ; RV32ZVE32F-LABEL: mscatter_baseidx_v8f64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    addi sp, sp, -64
-; RV32ZVE32F-NEXT:    .cfi_def_cfa_offset 64
-; RV32ZVE32F-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32ZVE32F-NEXT:    .cfi_offset ra, -4
-; RV32ZVE32F-NEXT:    .cfi_offset s0, -8
-; RV32ZVE32F-NEXT:    addi s0, sp, 64
-; RV32ZVE32F-NEXT:    .cfi_def_cfa s0, 0
-; RV32ZVE32F-NEXT:    andi sp, sp, -32
-; RV32ZVE32F-NEXT:    lw a2, 0(a1)
-; RV32ZVE32F-NEXT:    lw a3, 8(a1)
-; RV32ZVE32F-NEXT:    lw a4, 16(a1)
-; RV32ZVE32F-NEXT:    lw a5, 24(a1)
-; RV32ZVE32F-NEXT:    lw a6, 56(a1)
-; RV32ZVE32F-NEXT:    lw a7, 48(a1)
-; RV32ZVE32F-NEXT:    lw t0, 40(a1)
-; RV32ZVE32F-NEXT:    lw a1, 32(a1)
-; RV32ZVE32F-NEXT:    sw a6, 28(sp)
-; RV32ZVE32F-NEXT:    sw a7, 24(sp)
-; RV32ZVE32F-NEXT:    sw t0, 20(sp)
-; RV32ZVE32F-NEXT:    sw a1, 16(sp)
-; RV32ZVE32F-NEXT:    sw a5, 12(sp)
-; RV32ZVE32F-NEXT:    sw a4, 8(sp)
-; RV32ZVE32F-NEXT:    sw a3, 4(sp)
-; RV32ZVE32F-NEXT:    sw a2, 0(sp)
-; RV32ZVE32F-NEXT:    mv a1, sp
+; RV32ZVE32F-NEXT:    lw a2, 56(a1)
+; RV32ZVE32F-NEXT:    lw a3, 48(a1)
+; RV32ZVE32F-NEXT:    lw a4, 40(a1)
+; RV32ZVE32F-NEXT:    lw a5, 32(a1)
+; RV32ZVE32F-NEXT:    lw a6, 24(a1)
+; RV32ZVE32F-NEXT:    lw a7, 0(a1)
+; RV32ZVE32F-NEXT:    lw t0, 8(a1)
+; RV32ZVE32F-NEXT:    lw a1, 16(a1)
 ; RV32ZVE32F-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vle32.v v8, (a1)
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV32ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
 ; RV32ZVE32F-NEXT:    vsll.vi v8, v8, 3
 ; RV32ZVE32F-NEXT:    vadd.vx v8, v8, a0
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e8, mf4, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a0, v0
 ; RV32ZVE32F-NEXT:    andi a1, a0, 1
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_10
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_9
 ; RV32ZVE32F-NEXT:  # %bb.1: # %else
 ; RV32ZVE32F-NEXT:    andi a1, a0, 2
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_11
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_10
 ; RV32ZVE32F-NEXT:  .LBB90_2: # %else2
 ; RV32ZVE32F-NEXT:    andi a1, a0, 4
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_12
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_11
 ; RV32ZVE32F-NEXT:  .LBB90_3: # %else4
 ; RV32ZVE32F-NEXT:    andi a1, a0, 8
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_13
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_12
 ; RV32ZVE32F-NEXT:  .LBB90_4: # %else6
 ; RV32ZVE32F-NEXT:    andi a1, a0, 16
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_14
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_13
 ; RV32ZVE32F-NEXT:  .LBB90_5: # %else8
 ; RV32ZVE32F-NEXT:    andi a1, a0, 32
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_15
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_14
 ; RV32ZVE32F-NEXT:  .LBB90_6: # %else10
 ; RV32ZVE32F-NEXT:    andi a1, a0, 64
-; RV32ZVE32F-NEXT:    bnez a1, .LBB90_16
+; RV32ZVE32F-NEXT:    bnez a1, .LBB90_15
 ; RV32ZVE32F-NEXT:  .LBB90_7: # %else12
 ; RV32ZVE32F-NEXT:    andi a0, a0, -128
-; RV32ZVE32F-NEXT:    beqz a0, .LBB90_9
-; RV32ZVE32F-NEXT:  .LBB90_8: # %cond.store13
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
-; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
-; RV32ZVE32F-NEXT:    fsd fa7, 0(a0)
-; RV32ZVE32F-NEXT:  .LBB90_9: # %else14
-; RV32ZVE32F-NEXT:    addi sp, s0, -64
-; RV32ZVE32F-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32ZVE32F-NEXT:    addi sp, sp, 64
+; RV32ZVE32F-NEXT:    bnez a0, .LBB90_16
+; RV32ZVE32F-NEXT:  .LBB90_8: # %else14
 ; RV32ZVE32F-NEXT:    ret
-; RV32ZVE32F-NEXT:  .LBB90_10: # %cond.store
+; RV32ZVE32F-NEXT:  .LBB90_9: # %cond.store
 ; RV32ZVE32F-NEXT:    vsetivli zero, 0, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v8
 ; RV32ZVE32F-NEXT:    fsd fa0, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a1, a0, 2
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB90_2
-; RV32ZVE32F-NEXT:  .LBB90_11: # %cond.store1
+; RV32ZVE32F-NEXT:  .LBB90_10: # %cond.store1
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV32ZVE32F-NEXT:    fsd fa1, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a1, a0, 4
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB90_3
-; RV32ZVE32F-NEXT:  .LBB90_12: # %cond.store3
+; RV32ZVE32F-NEXT:  .LBB90_11: # %cond.store3
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV32ZVE32F-NEXT:    fsd fa2, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a1, a0, 8
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB90_4
-; RV32ZVE32F-NEXT:  .LBB90_13: # %cond.store5
+; RV32ZVE32F-NEXT:  .LBB90_12: # %cond.store5
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 3
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV32ZVE32F-NEXT:    fsd fa3, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a1, a0, 16
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB90_5
-; RV32ZVE32F-NEXT:  .LBB90_14: # %cond.store7
+; RV32ZVE32F-NEXT:  .LBB90_13: # %cond.store7
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 4
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV32ZVE32F-NEXT:    fsd fa4, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a1, a0, 32
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB90_6
-; RV32ZVE32F-NEXT:  .LBB90_15: # %cond.store9
+; RV32ZVE32F-NEXT:  .LBB90_14: # %cond.store9
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 5
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV32ZVE32F-NEXT:    fsd fa5, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a1, a0, 64
 ; RV32ZVE32F-NEXT:    beqz a1, .LBB90_7
-; RV32ZVE32F-NEXT:  .LBB90_16: # %cond.store11
+; RV32ZVE32F-NEXT:  .LBB90_15: # %cond.store11
 ; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
 ; RV32ZVE32F-NEXT:    vslidedown.vi v10, v8, 6
 ; RV32ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV32ZVE32F-NEXT:    fsd fa6, 0(a1)
 ; RV32ZVE32F-NEXT:    andi a0, a0, -128
-; RV32ZVE32F-NEXT:    bnez a0, .LBB90_8
-; RV32ZVE32F-NEXT:    j .LBB90_9
+; RV32ZVE32F-NEXT:    beqz a0, .LBB90_8
+; RV32ZVE32F-NEXT:  .LBB90_16: # %cond.store13
+; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32ZVE32F-NEXT:    vslidedown.vi v8, v8, 7
+; RV32ZVE32F-NEXT:    vmv.x.s a0, v8
+; RV32ZVE32F-NEXT:    fsd fa7, 0(a0)
+; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mscatter_baseidx_v8f64:
 ; RV64ZVE32F:       # %bb.0:
@@ -10724,8 +10593,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -10735,8 +10604,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.5: # %cond.store3
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_6: # %else4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -10754,8 +10623,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v11, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_10: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
@@ -10778,8 +10647,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 9
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_15: # %else18
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -10789,8 +10658,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.16: # %cond.store19
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 10
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v11, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_17: # %else20
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -10808,8 +10677,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 13
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_21: # %else26
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -10819,8 +10688,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.22: # %cond.store27
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 14
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:  .LBB91_23: # %else28
 ; RV64ZVE32F-NEXT:    lui a2, 1048568
@@ -10831,18 +10700,17 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v9, v9, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v9
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 15
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB91_25: # %else30
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB91_26: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v11, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB91_8
@@ -10859,8 +10727,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  .LBB91_28: # %cond.store11
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v11, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB91_12
@@ -10869,8 +10737,8 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 256
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB91_13
@@ -10885,12 +10753,11 @@ define void @mscatter_baseidx_v16i8(<16 x i8> %val, ptr %base, <16 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB91_14
 ; RV64ZVE32F-NEXT:    j .LBB91_15
 ; RV64ZVE32F-NEXT:  .LBB91_31: # %cond.store21
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v8, 11
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v10, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 51
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB91_19
@@ -10958,8 +10825,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 1
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_4: # %else2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -10969,8 +10836,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.5: # %cond.store3
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 2
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_6: # %else4
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -10988,8 +10855,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v13, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 5
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_10: # %else10
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
@@ -11012,8 +10879,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v13, v12, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 9
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_15: # %else18
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -11023,8 +10890,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.16: # %cond.store19
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 10
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_17: # %else20
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -11032,12 +10899,11 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v12, 4
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_19
 ; RV64ZVE32F-NEXT:  # %bb.18: # %cond.store21
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v13, v13, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 11
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_19: # %else22
 ; RV64ZVE32F-NEXT:    vsetivli zero, 16, e8, m2, ta, ma
@@ -11058,8 +10924,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v12, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 13
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_23: # %else26
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -11080,8 +10946,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 17
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_28: # %else34
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -11091,8 +10957,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.29: # %cond.store35
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 18
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_30: # %else36
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -11110,8 +10976,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 21
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_34: # %else42
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
@@ -11134,8 +11000,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 25
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_39: # %else50
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -11145,8 +11011,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.40: # %cond.store51
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 26
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_41: # %else52
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
@@ -11164,8 +11030,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 29
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_45: # %else58
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
@@ -11175,8 +11041,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  # %bb.46: # %cond.store59
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v10
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 30
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:  .LBB92_47: # %else60
 ; RV64ZVE32F-NEXT:    lui a2, 524288
@@ -11187,18 +11053,17 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v10, v10, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a1, v10
 ; RV64ZVE32F-NEXT:    add a0, a0, a1
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v8, v8, 31
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v8, (a0)
 ; RV64ZVE32F-NEXT:  .LBB92_49: # %else62
 ; RV64ZVE32F-NEXT:    ret
 ; RV64ZVE32F-NEXT:  .LBB92_50: # %cond.store5
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v12, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 3
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB92_8
@@ -11215,8 +11080,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  .LBB92_52: # %cond.store11
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 6
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 128
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB92_12
@@ -11225,8 +11090,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v13, v13, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v14, v8, 7
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v14, (a2)
 ; RV64ZVE32F-NEXT:    andi a2, a1, 256
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB92_13
@@ -11243,8 +11108,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  .LBB92_55: # %cond.store27
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 14
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 48
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_25
@@ -11253,8 +11118,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 15
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 47
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_26
@@ -11269,12 +11134,11 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB92_27
 ; RV64ZVE32F-NEXT:    j .LBB92_28
 ; RV64ZVE32F-NEXT:  .LBB92_58: # %cond.store37
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v12, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 19
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 43
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_32
@@ -11291,8 +11155,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:  .LBB92_60: # %cond.store43
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 22
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 40
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_36
@@ -11301,8 +11165,8 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 23
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 39
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_37
@@ -11317,12 +11181,11 @@ define void @mscatter_baseidx_v32i8(<32 x i8> %val, ptr %base, <32 x i8> %idxs,
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB92_38
 ; RV64ZVE32F-NEXT:    j .LBB92_39
 ; RV64ZVE32F-NEXT:  .LBB92_63: # %cond.store53
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v11, v11, 1
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v11
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vslidedown.vi v12, v8, 27
+; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
 ; RV64ZVE32F-NEXT:    vse8.v v12, (a2)
 ; RV64ZVE32F-NEXT:    slli a2, a1, 35
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB92_43
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
index a479d674a7eff..0067b8df6a0f7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
@@ -43,9 +43,9 @@ define void @store_v6i8(ptr %p, <6 x i8> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vse32.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v8, 2
 ; CHECK-NEXT:    addi a0, a0, 4
+; CHECK-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
   store <6 x i8> %v, ptr %p
@@ -67,9 +67,9 @@ define void @store_v12i8(ptr %p, <12 x i8> %v) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
   store <12 x i8> %v, ptr %p
@@ -91,9 +91,9 @@ define void @store_v6i16(ptr %p, <6 x i16> %v) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
   store <6 x i16> %v, ptr %p
@@ -103,26 +103,23 @@ define void @store_v6i16(ptr %p, <6 x i16> %v) {
 define void @store_v6f16(ptr %p, <6 x half> %v) {
 ; RV32-LABEL: store_v6f16:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    lh a2, 4(a1)
-; RV32-NEXT:    lhu a3, 0(a1)
+; RV32-NEXT:    lh a2, 20(a1)
+; RV32-NEXT:    lhu a3, 16(a1)
 ; RV32-NEXT:    slli a2, a2, 16
 ; RV32-NEXT:    or a2, a3, a2
 ; RV32-NEXT:    lh a3, 12(a1)
 ; RV32-NEXT:    lhu a4, 8(a1)
-; RV32-NEXT:    lh a5, 20(a1)
-; RV32-NEXT:    lhu a1, 16(a1)
+; RV32-NEXT:    lh a5, 4(a1)
+; RV32-NEXT:    lhu a1, 0(a1)
 ; RV32-NEXT:    slli a3, a3, 16
 ; RV32-NEXT:    or a3, a4, a3
 ; RV32-NEXT:    slli a5, a5, 16
 ; RV32-NEXT:    or a1, a1, a5
-; RV32-NEXT:    sw a1, 8(sp)
-; RV32-NEXT:    sw a3, 4(sp)
-; RV32-NEXT:    sw a2, 0(sp)
-; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vle32.v v8, (a1)
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a3
+; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV32-NEXT:    vse32.v v8, (a0)
 ; RV32-NEXT:    vslidedown.vi v9, v8, 2
@@ -131,7 +128,6 @@ define void @store_v6f16(ptr %p, <6 x half> %v) {
 ; RV32-NEXT:    vslidedown.vi v8, v8, 1
 ; RV32-NEXT:    addi a0, a0, 4
 ; RV32-NEXT:    vse32.v v8, (a0)
-; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: store_v6f16:
@@ -155,14 +151,13 @@ define void @store_v6f16(ptr %p, <6 x half> %v) {
 ; RV64-NEXT:    slli a1, a1, 32
 ; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a1
-; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, ma
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    vslide1down.vx v8, v8, a2
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64-NEXT:    addi a0, a0, 8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
   store <6 x half> %v, ptr %p
@@ -214,7 +209,7 @@ define void @store_v6i1(ptr %p, <6 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vfirst.m a1, v0
-; CHECK-NEXT:    seqz a1, a1
+; CHECK-NEXT:    snez a1, a1
 ; CHECK-NEXT:    vmv.x.s a2, v0
 ; CHECK-NEXT:    andi a3, a2, 2
 ; CHECK-NEXT:    or a1, a1, a3
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
index 417469dd72d0e..3aa8ce8f892ca 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll
@@ -1,7 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+experimental-zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+experimental-zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64
-; RUN: llc -mtriple=riscv64 -mattr=+f,+zfh,+zve64f,+zvl128b,+experimental-zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,ZVE64F
+; RUN: llc -mtriple=riscv32 -mattr=+v,+zfh,+experimental-zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+experimental-zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,RV64
+; RUN: llc -mtriple=riscv64 -mattr=+v,+zfh,+experimental-zvfh,+unaligned-vector-mem -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64,RV64-MISALIGN
+
+; RUN: llc -mtriple=riscv64 -mattr=+f,+zfh,+zve64f,+zvl128b,+experimental-zvfh -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,CHECK-NO-MISALIGN,ZVE64F
 
 ; The two loads are contigous and should be folded into one
 define void @widen_2xv4i16(ptr %x, ptr %z) {
@@ -109,6 +111,46 @@ define void @widen_4xv4i16(ptr %x, ptr %z) {
   ret void
 }
 
+define void @widen_4xv4i16_unaligned(ptr %x, ptr %z) {
+; CHECK-NO-MISALIGN-LABEL: widen_4xv4i16_unaligned:
+; CHECK-NO-MISALIGN:       # %bb.0:
+; CHECK-NO-MISALIGN-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NO-MISALIGN-NEXT:    vle8.v v8, (a0)
+; CHECK-NO-MISALIGN-NEXT:    addi a2, a0, 8
+; CHECK-NO-MISALIGN-NEXT:    vle8.v v10, (a2)
+; CHECK-NO-MISALIGN-NEXT:    addi a2, a0, 16
+; CHECK-NO-MISALIGN-NEXT:    vle8.v v12, (a2)
+; CHECK-NO-MISALIGN-NEXT:    addi a0, a0, 24
+; CHECK-NO-MISALIGN-NEXT:    vle8.v v14, (a0)
+; CHECK-NO-MISALIGN-NEXT:    vsetivli zero, 8, e16, m2, tu, ma
+; CHECK-NO-MISALIGN-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-NO-MISALIGN-NEXT:    vsetivli zero, 12, e16, m2, tu, ma
+; CHECK-NO-MISALIGN-NEXT:    vslideup.vi v8, v12, 8
+; CHECK-NO-MISALIGN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NO-MISALIGN-NEXT:    vslideup.vi v8, v14, 12
+; CHECK-NO-MISALIGN-NEXT:    vse16.v v8, (a1)
+; CHECK-NO-MISALIGN-NEXT:    ret
+;
+; RV64-MISALIGN-LABEL: widen_4xv4i16_unaligned:
+; RV64-MISALIGN:       # %bb.0:
+; RV64-MISALIGN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; RV64-MISALIGN-NEXT:    vle16.v v8, (a0)
+; RV64-MISALIGN-NEXT:    vse16.v v8, (a1)
+; RV64-MISALIGN-NEXT:    ret
+  %a = load <4 x i16>, ptr %x, align 1
+  %b.gep = getelementptr i8, ptr %x, i64 8
+  %b = load <4 x i16>, ptr %b.gep, align 1
+  %c.gep = getelementptr i8, ptr %b.gep, i64 8
+  %c = load <4 x i16>, ptr %c.gep, align 1
+  %d.gep = getelementptr i8, ptr %c.gep, i64 8
+  %d = load <4 x i16>, ptr %d.gep, align 1
+  %e.0 = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %e.1 = shufflevector <4 x i16> %c, <4 x i16> %d, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %e.2 = shufflevector <8 x i16> %e.0, <8 x i16> %e.1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <16 x i16> %e.2, ptr %z
+  ret void
+}
+
 ; Should be a strided load - with type coercion to i64
 define void @strided_constant(ptr %x, ptr %z) {
 ; CHECK-LABEL: strided_constant:
@@ -365,17 +407,23 @@ define void @strided_runtime_4xv2f32(ptr %x, ptr %z, i64 %s) {
   ret void
 }
 
-; Shouldn't be combined because the resulting load would not be aligned
 define void @strided_unaligned(ptr %x, ptr %z, i64 %s) {
-; CHECK-LABEL: strided_unaligned:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    add a0, a0, a2
-; CHECK-NEXT:    vle8.v v9, (a0)
-; CHECK-NEXT:    vslideup.vi v8, v9, 4
-; CHECK-NEXT:    vse16.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-NO-MISALIGN-LABEL: strided_unaligned:
+; CHECK-NO-MISALIGN:       # %bb.0:
+; CHECK-NO-MISALIGN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NO-MISALIGN-NEXT:    vle8.v v8, (a0)
+; CHECK-NO-MISALIGN-NEXT:    add a0, a0, a2
+; CHECK-NO-MISALIGN-NEXT:    vle8.v v9, (a0)
+; CHECK-NO-MISALIGN-NEXT:    vslideup.vi v8, v9, 4
+; CHECK-NO-MISALIGN-NEXT:    vse16.v v8, (a1)
+; CHECK-NO-MISALIGN-NEXT:    ret
+;
+; RV64-MISALIGN-LABEL: strided_unaligned:
+; RV64-MISALIGN:       # %bb.0:
+; RV64-MISALIGN-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-MISALIGN-NEXT:    vlse64.v v8, (a0), a2
+; RV64-MISALIGN-NEXT:    vse64.v v8, (a1)
+; RV64-MISALIGN-NEXT:    ret
   %a = load <4 x i16>, ptr %x, align 1
   %b.gep = getelementptr i8, ptr %x, i64 %s
   %b = load <4 x i16>, ptr %b.gep, align 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll
index d8e5468fedf6c..2046c51967037 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfcmps-constrained-sdnode.ll
@@ -226,9 +226,7 @@ define <1 x i1> @fcmps_ord_vv_v1f16(<1 x half> %va, <1 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <1 x i1> @llvm.experimental.constrained.fcmps.v1f16(<1 x half> %va, <1 x half> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -241,9 +239,7 @@ define <1 x i1> @fcmps_ord_vf_v1f16(<1 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x half> poison, half %b, i32 0
@@ -258,9 +254,7 @@ define <1 x i1> @fcmps_ord_fv_v1f16(<1 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x half> poison, half %b, i32 0
@@ -502,7 +496,6 @@ define <1 x i1> @fcmps_uno_vv_v1f16(<1 x half> %va, <1 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -517,7 +510,6 @@ define <1 x i1> @fcmps_uno_vf_v1f16(<1 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -536,7 +528,6 @@ define <1 x i1> @fcmps_uno_fv_v1f16(<1 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x half> poison, half %b, i32 0
@@ -767,9 +758,7 @@ define <2 x i1> @fcmps_ord_vv_v2f16(<2 x half> %va, <2 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f16(<2 x half> %va, <2 x half> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -782,9 +771,7 @@ define <2 x i1> @fcmps_ord_vf_v2f16(<2 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x half> poison, half %b, i32 0
@@ -799,9 +786,7 @@ define <2 x i1> @fcmps_ord_fv_v2f16(<2 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x half> poison, half %b, i32 0
@@ -1043,7 +1028,6 @@ define <2 x i1> @fcmps_uno_vv_v2f16(<2 x half> %va, <2 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -1058,7 +1042,6 @@ define <2 x i1> @fcmps_uno_vf_v2f16(<2 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -1077,7 +1060,6 @@ define <2 x i1> @fcmps_uno_fv_v2f16(<2 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x half> poison, half %b, i32 0
@@ -1308,9 +1290,7 @@ define <4 x i1> @fcmps_ord_vv_v4f16(<4 x half> %va, <4 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f16(<4 x half> %va, <4 x half> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -1323,9 +1303,7 @@ define <4 x i1> @fcmps_ord_vf_v4f16(<4 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x half> poison, half %b, i32 0
@@ -1340,9 +1318,7 @@ define <4 x i1> @fcmps_ord_fv_v4f16(<4 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x half> poison, half %b, i32 0
@@ -1584,7 +1560,6 @@ define <4 x i1> @fcmps_uno_vv_v4f16(<4 x half> %va, <4 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -1599,7 +1574,6 @@ define <4 x i1> @fcmps_uno_vf_v4f16(<4 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -1618,7 +1592,6 @@ define <4 x i1> @fcmps_uno_fv_v4f16(<4 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x half> poison, half %b, i32 0
@@ -1849,9 +1822,7 @@ define <8 x i1> @fcmps_ord_vv_v8f16(<8 x half> %va, <8 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f16(<8 x half> %va, <8 x half> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -1864,9 +1835,7 @@ define <8 x i1> @fcmps_ord_vf_v8f16(<8 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x half> poison, half %b, i32 0
@@ -1881,9 +1850,7 @@ define <8 x i1> @fcmps_ord_fv_v8f16(<8 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x half> poison, half %b, i32 0
@@ -2125,7 +2092,6 @@ define <8 x i1> @fcmps_uno_vv_v8f16(<8 x half> %va, <8 x half> %vb) nounwind str
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -2140,7 +2106,6 @@ define <8 x i1> @fcmps_uno_vf_v8f16(<8 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -2159,7 +2124,6 @@ define <8 x i1> @fcmps_uno_fv_v8f16(<8 x half> %va, half %b) nounwind strictfp {
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x half> poison, half %b, i32 0
@@ -2390,10 +2354,8 @@ define <16 x i1> @fcmps_ord_vv_v16f16(<16 x half> %va, <16 x half> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v12, v10, v10
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v10, v12
 ; CHECK-NEXT:    ret
   %1 = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f16(<16 x half> %va, <16 x half> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
   ret <16 x i1> %1
@@ -2405,10 +2367,8 @@ define <16 x i1> @fcmps_ord_vf_v16f16(<16 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v10, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x half> poison, half %b, i32 0
   %splat = shufflevector <16 x half> %head, <16 x half> poison, <16 x i32> zeroinitializer
@@ -2422,10 +2382,8 @@ define <16 x i1> @fcmps_ord_fv_v16f16(<16 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v10, v8
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x half> poison, half %b, i32 0
   %splat = shufflevector <16 x half> %head, <16 x half> poison, <16 x i32> zeroinitializer
@@ -2666,10 +2624,9 @@ define <16 x i1> @fcmps_uno_vv_v16f16(<16 x half> %va, <16 x half> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v12, v10, v10
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v10
+; CHECK-NEXT:    vmorn.mm v0, v8, v12
 ; CHECK-NEXT:    ret
   %1 = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f16(<16 x half> %va, <16 x half> %vb, metadata !"uno", metadata !"fpexcept.strict") strictfp
   ret <16 x i1> %1
@@ -2681,10 +2638,9 @@ define <16 x i1> @fcmps_uno_vf_v16f16(<16 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v10
+; CHECK-NEXT:    vmorn.mm v0, v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x half> poison, half %b, i32 0
   %splat = shufflevector <16 x half> %head, <16 x half> poison, <16 x i32> zeroinitializer
@@ -2700,8 +2656,7 @@ define <16 x i1> @fcmps_uno_fv_v16f16(<16 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
 ; CHECK-NEXT:    vmnot.m v10, v12
 ; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v10, v8
+; CHECK-NEXT:    vmorn.mm v0, v10, v11
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x half> poison, half %b, i32 0
   %splat = shufflevector <16 x half> %head, <16 x half> poison, <16 x i32> zeroinitializer
@@ -2950,10 +2905,8 @@ define <32 x i1> @fcmps_ord_vv_v32f16(<32 x half> %va, <32 x half> %vb) nounwind
 ; CHECK-NEXT:    li a0, 32
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v16, v12, v12
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v16
 ; CHECK-NEXT:    ret
   %1 = call <32 x i1> @llvm.experimental.constrained.fcmps.v32f16(<32 x half> %va, <32 x half> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
   ret <32 x i1> %1
@@ -2966,10 +2919,8 @@ define <32 x i1> @fcmps_ord_vf_v32f16(<32 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <32 x half> poison, half %b, i32 0
   %splat = shufflevector <32 x half> %head, <32 x half> poison, <32 x i32> zeroinitializer
@@ -2983,11 +2934,9 @@ define <32 x i1> @fcmps_ord_fv_v32f16(<32 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    li a0, 32
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
-; CHECK-NEXT:    vmfle.vv v16, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v16
-; CHECK-NEXT:    vmfle.vf v9, v12, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
-; CHECK-NEXT:    vmand.mm v0, v9, v8
+; CHECK-NEXT:    vmfle.vf v16, v12, fa0
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v16, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <32 x half> poison, half %b, i32 0
   %splat = shufflevector <32 x half> %head, <32 x half> poison, <32 x i32> zeroinitializer
@@ -3247,10 +3196,9 @@ define <32 x i1> @fcmps_uno_vv_v32f16(<32 x half> %va, <32 x half> %vb) nounwind
 ; CHECK-NEXT:    li a0, 32
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v16, v12, v12
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v12
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %1 = call <32 x i1> @llvm.experimental.constrained.fcmps.v32f16(<32 x half> %va, <32 x half> %vb, metadata !"uno", metadata !"fpexcept.strict") strictfp
   ret <32 x i1> %1
@@ -3263,10 +3211,9 @@ define <32 x i1> @fcmps_uno_vf_v32f16(<32 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v12
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <32 x half> poison, half %b, i32 0
   %splat = shufflevector <32 x half> %head, <32 x half> poison, <32 x i32> zeroinitializer
@@ -3281,10 +3228,9 @@ define <32 x i1> @fcmps_uno_fv_v32f16(<32 x half> %va, half %b) nounwind strictf
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vv v16, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v16
-; CHECK-NEXT:    vmfle.vf v9, v12, fa0
-; CHECK-NEXT:    vmnot.m v9, v9
-; CHECK-NEXT:    vmorn.mm v0, v9, v8
+; CHECK-NEXT:    vmfle.vf v8, v12, fa0
+; CHECK-NEXT:    vmnot.m v8, v8
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <32 x half> poison, half %b, i32 0
   %splat = shufflevector <32 x half> %head, <32 x half> poison, <32 x i32> zeroinitializer
@@ -3514,9 +3460,7 @@ define <1 x i1> @fcmps_ord_vv_v1f32(<1 x float> %va, <1 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <1 x i1> @llvm.experimental.constrained.fcmps.v1f32(<1 x float> %va, <1 x float> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -3529,9 +3473,7 @@ define <1 x i1> @fcmps_ord_vf_v1f32(<1 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x float> poison, float %b, i32 0
@@ -3546,9 +3488,7 @@ define <1 x i1> @fcmps_ord_fv_v1f32(<1 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x float> poison, float %b, i32 0
@@ -3790,7 +3730,6 @@ define <1 x i1> @fcmps_uno_vv_v1f32(<1 x float> %va, <1 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -3805,7 +3744,6 @@ define <1 x i1> @fcmps_uno_vf_v1f32(<1 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -3824,7 +3762,6 @@ define <1 x i1> @fcmps_uno_fv_v1f32(<1 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x float> poison, float %b, i32 0
@@ -4055,9 +3992,7 @@ define <2 x i1> @fcmps_ord_vv_v2f32(<2 x float> %va, <2 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f32(<2 x float> %va, <2 x float> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -4070,9 +4005,7 @@ define <2 x i1> @fcmps_ord_vf_v2f32(<2 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x float> poison, float %b, i32 0
@@ -4087,9 +4020,7 @@ define <2 x i1> @fcmps_ord_fv_v2f32(<2 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x float> poison, float %b, i32 0
@@ -4331,7 +4262,6 @@ define <2 x i1> @fcmps_uno_vv_v2f32(<2 x float> %va, <2 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -4346,7 +4276,6 @@ define <2 x i1> @fcmps_uno_vf_v2f32(<2 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -4365,7 +4294,6 @@ define <2 x i1> @fcmps_uno_fv_v2f32(<2 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x float> poison, float %b, i32 0
@@ -4596,9 +4524,7 @@ define <4 x i1> @fcmps_ord_vv_v4f32(<4 x float> %va, <4 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f32(<4 x float> %va, <4 x float> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -4611,9 +4537,7 @@ define <4 x i1> @fcmps_ord_vf_v4f32(<4 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x float> poison, float %b, i32 0
@@ -4628,9 +4552,7 @@ define <4 x i1> @fcmps_ord_fv_v4f32(<4 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x float> poison, float %b, i32 0
@@ -4872,7 +4794,6 @@ define <4 x i1> @fcmps_uno_vv_v4f32(<4 x float> %va, <4 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -4887,7 +4808,6 @@ define <4 x i1> @fcmps_uno_vf_v4f32(<4 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -4906,7 +4826,6 @@ define <4 x i1> @fcmps_uno_fv_v4f32(<4 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x float> poison, float %b, i32 0
@@ -5137,10 +5056,8 @@ define <8 x i1> @fcmps_ord_vv_v8f32(<8 x float> %va, <8 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v12, v10, v10
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v10, v12
 ; CHECK-NEXT:    ret
   %1 = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(<8 x float> %va, <8 x float> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
   ret <8 x i1> %1
@@ -5152,10 +5069,8 @@ define <8 x i1> @fcmps_ord_vf_v8f32(<8 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v10, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x float> poison, float %b, i32 0
   %splat = shufflevector <8 x float> %head, <8 x float> poison, <8 x i32> zeroinitializer
@@ -5169,10 +5084,8 @@ define <8 x i1> @fcmps_ord_fv_v8f32(<8 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v10, v8
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x float> poison, float %b, i32 0
   %splat = shufflevector <8 x float> %head, <8 x float> poison, <8 x i32> zeroinitializer
@@ -5413,10 +5326,9 @@ define <8 x i1> @fcmps_uno_vv_v8f32(<8 x float> %va, <8 x float> %vb) nounwind s
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v12, v10, v10
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v10
+; CHECK-NEXT:    vmorn.mm v0, v8, v12
 ; CHECK-NEXT:    ret
   %1 = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f32(<8 x float> %va, <8 x float> %vb, metadata !"uno", metadata !"fpexcept.strict") strictfp
   ret <8 x i1> %1
@@ -5428,10 +5340,9 @@ define <8 x i1> @fcmps_uno_vf_v8f32(<8 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v10
+; CHECK-NEXT:    vmorn.mm v0, v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x float> poison, float %b, i32 0
   %splat = shufflevector <8 x float> %head, <8 x float> poison, <8 x i32> zeroinitializer
@@ -5447,8 +5358,7 @@ define <8 x i1> @fcmps_uno_fv_v8f32(<8 x float> %va, float %b) nounwind strictfp
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
 ; CHECK-NEXT:    vmnot.m v10, v12
 ; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v10, v8
+; CHECK-NEXT:    vmorn.mm v0, v10, v11
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x float> poison, float %b, i32 0
   %splat = shufflevector <8 x float> %head, <8 x float> poison, <8 x i32> zeroinitializer
@@ -5678,10 +5588,8 @@ define <16 x i1> @fcmps_ord_vv_v16f32(<16 x float> %va, <16 x float> %vb) nounwi
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v16, v12, v12
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v16
 ; CHECK-NEXT:    ret
   %1 = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(<16 x float> %va, <16 x float> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
   ret <16 x i1> %1
@@ -5693,10 +5601,8 @@ define <16 x i1> @fcmps_ord_vf_v16f32(<16 x float> %va, float %b) nounwind stric
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x float> poison, float %b, i32 0
   %splat = shufflevector <16 x float> %head, <16 x float> poison, <16 x i32> zeroinitializer
@@ -5710,10 +5616,8 @@ define <16 x i1> @fcmps_ord_fv_v16f32(<16 x float> %va, float %b) nounwind stric
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v12, v8
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v16, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x float> poison, float %b, i32 0
   %splat = shufflevector <16 x float> %head, <16 x float> poison, <16 x i32> zeroinitializer
@@ -5954,10 +5858,9 @@ define <16 x i1> @fcmps_uno_vv_v16f32(<16 x float> %va, <16 x float> %vb) nounwi
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v16, v12, v12
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v12
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %1 = call <16 x i1> @llvm.experimental.constrained.fcmps.v16f32(<16 x float> %va, <16 x float> %vb, metadata !"uno", metadata !"fpexcept.strict") strictfp
   ret <16 x i1> %1
@@ -5969,10 +5872,9 @@ define <16 x i1> @fcmps_uno_vf_v16f32(<16 x float> %va, float %b) nounwind stric
 ; CHECK-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v12
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x float> poison, float %b, i32 0
   %splat = shufflevector <16 x float> %head, <16 x float> poison, <16 x i32> zeroinitializer
@@ -5988,8 +5890,7 @@ define <16 x i1> @fcmps_uno_fv_v16f32(<16 x float> %va, float %b) nounwind stric
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
 ; CHECK-NEXT:    vmnot.m v12, v16
 ; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v12, v8
+; CHECK-NEXT:    vmorn.mm v0, v12, v13
 ; CHECK-NEXT:    ret
   %head = insertelement <16 x float> poison, float %b, i32 0
   %splat = shufflevector <16 x float> %head, <16 x float> poison, <16 x i32> zeroinitializer
@@ -6219,9 +6120,7 @@ define <1 x i1> @fcmps_ord_vv_v1f64(<1 x double> %va, <1 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %va, <1 x double> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -6234,9 +6133,7 @@ define <1 x i1> @fcmps_ord_vf_v1f64(<1 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x double> poison, double %b, i32 0
@@ -6251,9 +6148,7 @@ define <1 x i1> @fcmps_ord_fv_v1f64(<1 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x double> poison, double %b, i32 0
@@ -6495,7 +6390,6 @@ define <1 x i1> @fcmps_uno_vv_v1f64(<1 x double> %va, <1 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -6510,7 +6404,6 @@ define <1 x i1> @fcmps_uno_vf_v1f64(<1 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -6529,7 +6422,6 @@ define <1 x i1> @fcmps_uno_fv_v1f64(<1 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <1 x double> poison, double %b, i32 0
@@ -6760,9 +6652,7 @@ define <2 x i1> @fcmps_ord_vv_v2f64(<2 x double> %va, <2 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %1 = call <2 x i1> @llvm.experimental.constrained.fcmps.v2f64(<2 x double> %va, <2 x double> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
@@ -6775,9 +6665,7 @@ define <2 x i1> @fcmps_ord_vf_v2f64(<2 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x double> poison, double %b, i32 0
@@ -6792,9 +6680,7 @@ define <2 x i1> @fcmps_ord_fv_v2f64(<2 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x double> poison, double %b, i32 0
@@ -7036,7 +6922,6 @@ define <2 x i1> @fcmps_uno_vv_v2f64(<2 x double> %va, <2 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vmfle.vv v9, v9, v9
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -7051,7 +6936,6 @@ define <2 x i1> @fcmps_uno_vf_v2f64(<2 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
-; CHECK-NEXT:    vmmv.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
 ; CHECK-NEXT:    vmnot.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v8, v9
@@ -7070,7 +6954,6 @@ define <2 x i1> @fcmps_uno_fv_v2f64(<2 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vmfle.vf v9, v9, fa0
 ; CHECK-NEXT:    vmnot.m v9, v9
 ; CHECK-NEXT:    vmfle.vv v8, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v8
 ; CHECK-NEXT:    vmorn.mm v0, v9, v8
 ; CHECK-NEXT:    ret
   %head = insertelement <2 x double> poison, double %b, i32 0
@@ -7301,10 +7184,8 @@ define <4 x i1> @fcmps_ord_vv_v4f64(<4 x double> %va, <4 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v12, v10, v10
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v10, v12
 ; CHECK-NEXT:    ret
   %1 = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(<4 x double> %va, <4 x double> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
   ret <4 x i1> %1
@@ -7316,10 +7197,8 @@ define <4 x i1> @fcmps_ord_vf_v4f64(<4 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v10, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x double> poison, double %b, i32 0
   %splat = shufflevector <4 x double> %head, <4 x double> poison, <4 x i32> zeroinitializer
@@ -7333,10 +7212,8 @@ define <4 x i1> @fcmps_ord_fv_v4f64(<4 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmand.mm v0, v10, v8
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x double> poison, double %b, i32 0
   %splat = shufflevector <4 x double> %head, <4 x double> poison, <4 x i32> zeroinitializer
@@ -7577,10 +7454,9 @@ define <4 x i1> @fcmps_uno_vv_v4f64(<4 x double> %va, <4 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vmfle.vv v12, v10, v10
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v10
+; CHECK-NEXT:    vmorn.mm v0, v8, v12
 ; CHECK-NEXT:    ret
   %1 = call <4 x i1> @llvm.experimental.constrained.fcmps.v4f64(<4 x double> %va, <4 x double> %vb, metadata !"uno", metadata !"fpexcept.strict") strictfp
   ret <4 x i1> %1
@@ -7592,10 +7468,9 @@ define <4 x i1> @fcmps_uno_vf_v4f64(<4 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v10, fa0
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
-; CHECK-NEXT:    vmmv.m v10, v12
-; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v8, v10
+; CHECK-NEXT:    vmfle.vv v10, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v10
+; CHECK-NEXT:    vmorn.mm v0, v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x double> poison, double %b, i32 0
   %splat = shufflevector <4 x double> %head, <4 x double> poison, <4 x i32> zeroinitializer
@@ -7611,8 +7486,7 @@ define <4 x i1> @fcmps_uno_fv_v4f64(<4 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vmfle.vf v12, v10, fa0
 ; CHECK-NEXT:    vmnot.m v10, v12
 ; CHECK-NEXT:    vmfle.vv v11, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v11
-; CHECK-NEXT:    vmorn.mm v0, v10, v8
+; CHECK-NEXT:    vmorn.mm v0, v10, v11
 ; CHECK-NEXT:    ret
   %head = insertelement <4 x double> poison, double %b, i32 0
   %splat = shufflevector <4 x double> %head, <4 x double> poison, <4 x i32> zeroinitializer
@@ -7842,10 +7716,8 @@ define <8 x i1> @fcmps_ord_vv_v8f64(<8 x double> %va, <8 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v16, v12, v12
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v16
 ; CHECK-NEXT:    ret
   %1 = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(<8 x double> %va, <8 x double> %vb, metadata !"ord", metadata !"fpexcept.strict") strictfp
   ret <8 x i1> %1
@@ -7857,10 +7729,8 @@ define <8 x i1> @fcmps_ord_vf_v8f64(<8 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v12, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x double> poison, double %b, i32 0
   %splat = shufflevector <8 x double> %head, <8 x double> poison, <8 x i32> zeroinitializer
@@ -7874,10 +7744,8 @@ define <8 x i1> @fcmps_ord_fv_v8f64(<8 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmand.mm v0, v12, v8
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmand.mm v0, v16, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x double> poison, double %b, i32 0
   %splat = shufflevector <8 x double> %head, <8 x double> poison, <8 x i32> zeroinitializer
@@ -8118,10 +7986,9 @@ define <8 x i1> @fcmps_uno_vv_v8f64(<8 x double> %va, <8 x double> %vb) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vmfle.vv v16, v12, v12
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v12
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %1 = call <8 x i1> @llvm.experimental.constrained.fcmps.v8f64(<8 x double> %va, <8 x double> %vb, metadata !"uno", metadata !"fpexcept.strict") strictfp
   ret <8 x i1> %1
@@ -8133,10 +8000,9 @@ define <8 x i1> @fcmps_uno_vf_v8f64(<8 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v12, fa0
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
-; CHECK-NEXT:    vmmv.m v12, v16
-; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmnot.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v8, v12
+; CHECK-NEXT:    vmfle.vv v12, v8, v8
+; CHECK-NEXT:    vmnot.m v8, v12
+; CHECK-NEXT:    vmorn.mm v0, v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x double> poison, double %b, i32 0
   %splat = shufflevector <8 x double> %head, <8 x double> poison, <8 x i32> zeroinitializer
@@ -8152,8 +8018,7 @@ define <8 x i1> @fcmps_uno_fv_v8f64(<8 x double> %va, double %b) nounwind strict
 ; CHECK-NEXT:    vmfle.vf v16, v12, fa0
 ; CHECK-NEXT:    vmnot.m v12, v16
 ; CHECK-NEXT:    vmfle.vv v13, v8, v8
-; CHECK-NEXT:    vmmv.m v8, v13
-; CHECK-NEXT:    vmorn.mm v0, v12, v8
+; CHECK-NEXT:    vmorn.mm v0, v12, v13
 ; CHECK-NEXT:    ret
   %head = insertelement <8 x double> poison, double %b, i32 0
   %splat = shufflevector <8 x double> %head, <8 x double> poison, <8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll
index 9d7564d4dda52..53d12da198e6a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vreductions-mask.ll
@@ -11,7 +11,7 @@ define zeroext i1 @vreduce_or_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -24,7 +24,7 @@ define zeroext i1 @vreduce_xor_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -37,7 +37,7 @@ define zeroext i1 @vreduce_and_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -50,7 +50,7 @@ define zeroext i1 @vreduce_umax_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.umax.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -63,7 +63,7 @@ define zeroext i1 @vreduce_smax_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.smax.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -76,7 +76,7 @@ define zeroext i1 @vreduce_umin_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.umin.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -89,7 +89,7 @@ define zeroext i1 @vreduce_smin_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.smin.v1i1(<1 x i1> %v)
   ret i1 %red
@@ -806,7 +806,7 @@ define zeroext i1 @vreduce_add_v1i1(<1 x i1> %v) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vfirst.m a0, v0
-; CHECK-NEXT:    seqz a0, a0
+; CHECK-NEXT:    snez a0, a0
 ; CHECK-NEXT:    ret
   %red = call i1 @llvm.vector.reduce.add.v1i1(<1 x i1> %v)
   ret i1 %red
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
index 8068c2056808a..0519305e6f2d2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
@@ -1,6 +1,473 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -target-abi=ilp32d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -target-abi=lp64d -mattr=+v,+zfh,+experimental-zvfh,+f,+d -riscv-v-vector-bits-min=128 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64
+
+define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
+; RV32-LABEL: vselect_vv_v6i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    lbu a2, 0(a2)
+; RV32-NEXT:    vle32.v v8, (a1)
+; RV32-NEXT:    andi a1, a2, 1
+; RV32-NEXT:    vslide1down.vx v10, v8, a1
+; RV32-NEXT:    slli a1, a2, 30
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 29
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 28
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 27
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    srli a2, a2, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
+; RV32-NEXT:    vand.vi v10, v10, 1
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vle32.v v8, (a0), v0.t
+; RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    addi a0, a3, 16
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vse32.v v10, (a0)
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vse32.v v8, (a3)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vselect_vv_v6i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    lbu a2, 0(a2)
+; RV64-NEXT:    vle32.v v8, (a1)
+; RV64-NEXT:    andi a1, a2, 1
+; RV64-NEXT:    vslide1down.vx v10, v8, a1
+; RV64-NEXT:    slli a1, a2, 62
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 61
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 60
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 59
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    srli a2, a2, 5
+; RV64-NEXT:    vslide1down.vx v10, v10, a2
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
+; RV64-NEXT:    vand.vi v10, v10, 1
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV64-NEXT:    vle32.v v8, (a0), v0.t
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    addi a0, a3, 16
+; RV64-NEXT:    vse64.v v10, (a0)
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a3)
+; RV64-NEXT:    ret
+  %va = load <6 x i32>, ptr %a
+  %vb = load <6 x i32>, ptr %b
+  %vcc = load <6 x i1>, ptr %cc
+  %vsel = select <6 x i1> %vcc, <6 x i32> %va, <6 x i32> %vb
+  store <6 x i32> %vsel, ptr %z
+  ret void
+}
+
+define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
+; RV32-LABEL: vselect_vx_v6i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    lbu a2, 0(a2)
+; RV32-NEXT:    vle32.v v8, (a1)
+; RV32-NEXT:    andi a1, a2, 1
+; RV32-NEXT:    vslide1down.vx v10, v8, a1
+; RV32-NEXT:    slli a1, a2, 30
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 29
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 28
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 27
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    srli a2, a2, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
+; RV32-NEXT:    vand.vi v10, v10, 1
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vmerge.vxm v8, v8, a0, v0
+; RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    addi a0, a3, 16
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vse32.v v10, (a0)
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vse32.v v8, (a3)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vselect_vx_v6i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    lbu a2, 0(a2)
+; RV64-NEXT:    vle32.v v8, (a1)
+; RV64-NEXT:    andi a1, a2, 1
+; RV64-NEXT:    vslide1down.vx v10, v8, a1
+; RV64-NEXT:    slli a1, a2, 62
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 61
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 60
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 59
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    srli a2, a2, 5
+; RV64-NEXT:    vslide1down.vx v10, v10, a2
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
+; RV64-NEXT:    vand.vi v10, v10, 1
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-NEXT:    vmerge.vxm v8, v8, a0, v0
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    addi a0, a3, 16
+; RV64-NEXT:    vse64.v v10, (a0)
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a3)
+; RV64-NEXT:    ret
+  %vb = load <6 x i32>, ptr %b
+  %ahead = insertelement <6 x i32> poison, i32 %a, i32 0
+  %va = shufflevector <6 x i32> %ahead, <6 x i32> poison, <6 x i32> zeroinitializer
+  %vcc = load <6 x i1>, ptr %cc
+  %vsel = select <6 x i1> %vcc, <6 x i32> %va, <6 x i32> %vb
+  store <6 x i32> %vsel, ptr %z
+  ret void
+}
+
+define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
+; RV32-LABEL: vselect_vi_v6i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    andi a0, a1, 1
+; RV32-NEXT:    vslide1down.vx v10, v8, a0
+; RV32-NEXT:    slli a0, a1, 30
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 29
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 28
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 27
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    srli a1, a1, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
+; RV32-NEXT:    vand.vi v10, v10, 1
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vmerge.vim v8, v8, -1, v0
+; RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    addi a0, a2, 16
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vse32.v v10, (a0)
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vse32.v v8, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vselect_vi_v6i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    lbu a1, 0(a1)
+; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    andi a0, a1, 1
+; RV64-NEXT:    vslide1down.vx v10, v8, a0
+; RV64-NEXT:    slli a0, a1, 62
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 61
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 60
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 59
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    srli a1, a1, 5
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
+; RV64-NEXT:    vand.vi v10, v10, 1
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-NEXT:    vmerge.vim v8, v8, -1, v0
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    addi a0, a2, 16
+; RV64-NEXT:    vse64.v v10, (a0)
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a2)
+; RV64-NEXT:    ret
+  %vb = load <6 x i32>, ptr %b
+  %a = insertelement <6 x i32> poison, i32 -1, i32 0
+  %va = shufflevector <6 x i32> %a, <6 x i32> poison, <6 x i32> zeroinitializer
+  %vcc = load <6 x i1>, ptr %cc
+  %vsel = select <6 x i1> %vcc, <6 x i32> %va, <6 x i32> %vb
+  store <6 x i32> %vsel, ptr %z
+  ret void
+}
+
+
+define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
+; RV32-LABEL: vselect_vv_v6f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    lbu a2, 0(a2)
+; RV32-NEXT:    vle32.v v8, (a1)
+; RV32-NEXT:    andi a1, a2, 1
+; RV32-NEXT:    vslide1down.vx v10, v8, a1
+; RV32-NEXT:    slli a1, a2, 30
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 29
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 28
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    slli a1, a2, 27
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    srli a2, a2, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a2
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
+; RV32-NEXT:    vand.vi v10, v10, 1
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV32-NEXT:    vle32.v v8, (a0), v0.t
+; RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    addi a0, a3, 16
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vse32.v v10, (a0)
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vse32.v v8, (a3)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vselect_vv_v6f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    lbu a2, 0(a2)
+; RV64-NEXT:    vle32.v v8, (a1)
+; RV64-NEXT:    andi a1, a2, 1
+; RV64-NEXT:    vslide1down.vx v10, v8, a1
+; RV64-NEXT:    slli a1, a2, 62
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 61
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 60
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    slli a1, a2, 59
+; RV64-NEXT:    srli a1, a1, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    srli a2, a2, 5
+; RV64-NEXT:    vslide1down.vx v10, v10, a2
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
+; RV64-NEXT:    vand.vi v10, v10, 1
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; RV64-NEXT:    vle32.v v8, (a0), v0.t
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    addi a0, a3, 16
+; RV64-NEXT:    vse64.v v10, (a0)
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a3)
+; RV64-NEXT:    ret
+  %va = load <6 x float>, ptr %a
+  %vb = load <6 x float>, ptr %b
+  %vcc = load <6 x i1>, ptr %cc
+  %vsel = select <6 x i1> %vcc, <6 x float> %va, <6 x float> %vb
+  store <6 x float> %vsel, ptr %z
+  ret void
+}
+
+define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
+; RV32-LABEL: vselect_vx_v6f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    andi a0, a1, 1
+; RV32-NEXT:    vslide1down.vx v10, v8, a0
+; RV32-NEXT:    slli a0, a1, 30
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 29
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 28
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 27
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    srli a1, a1, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
+; RV32-NEXT:    vand.vi v10, v10, 1
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    addi a0, a2, 16
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vse32.v v10, (a0)
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vse32.v v8, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vselect_vx_v6f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    lbu a1, 0(a1)
+; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    andi a0, a1, 1
+; RV64-NEXT:    vslide1down.vx v10, v8, a0
+; RV64-NEXT:    slli a0, a1, 62
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 61
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 60
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 59
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    srli a1, a1, 5
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
+; RV64-NEXT:    vand.vi v10, v10, 1
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-NEXT:    vfmerge.vfm v8, v8, fa0, v0
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    addi a0, a2, 16
+; RV64-NEXT:    vse64.v v10, (a0)
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a2)
+; RV64-NEXT:    ret
+  %vb = load <6 x float>, ptr %b
+  %ahead = insertelement <6 x float> poison, float %a, i32 0
+  %va = shufflevector <6 x float> %ahead, <6 x float> poison, <6 x i32> zeroinitializer
+  %vcc = load <6 x i1>, ptr %cc
+  %vsel = select <6 x i1> %vcc, <6 x float> %va, <6 x float> %vb
+  store <6 x float> %vsel, ptr %z
+  ret void
+}
+
+define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
+; RV32-LABEL: vselect_vfpzero_v6f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    lbu a1, 0(a1)
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    andi a0, a1, 1
+; RV32-NEXT:    vslide1down.vx v10, v8, a0
+; RV32-NEXT:    slli a0, a1, 30
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 29
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 28
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    slli a0, a1, 27
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    srli a1, a1, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
+; RV32-NEXT:    vand.vi v10, v10, 1
+; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vmerge.vim v8, v8, 0, v0
+; RV32-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 4
+; RV32-NEXT:    addi a0, a2, 16
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vse32.v v10, (a0)
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vse32.v v8, (a2)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vselect_vfpzero_v6f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    lbu a1, 0(a1)
+; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    andi a0, a1, 1
+; RV64-NEXT:    vslide1down.vx v10, v8, a0
+; RV64-NEXT:    slli a0, a1, 62
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 61
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 60
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    slli a0, a1, 59
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-NEXT:    srli a1, a1, 5
+; RV64-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
+; RV64-NEXT:    vand.vi v10, v10, 1
+; RV64-NEXT:    vmsne.vi v0, v10, 0
+; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-NEXT:    vmerge.vim v8, v8, 0, v0
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    addi a0, a2, 16
+; RV64-NEXT:    vse64.v v10, (a0)
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vse32.v v8, (a2)
+; RV64-NEXT:    ret
+  %vb = load <6 x float>, ptr %b
+  %a = insertelement <6 x float> poison, float 0.0, i32 0
+  %va = shufflevector <6 x float> %a, <6 x float> poison, <6 x i32> zeroinitializer
+  %vcc = load <6 x i1>, ptr %cc
+  %vsel = select <6 x i1> %vcc, <6 x float> %va, <6 x float> %vb
+  store <6 x float> %vsel, ptr %z
+  ret void
+}
 
 define void @vselect_vv_v8i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; CHECK-LABEL: vselect_vv_v8i32:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
new file mode 100644
index 0000000000000..32d8f9d4a7585
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-constrained-sdnode.ll
@@ -0,0 +1,350 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+declare <vscale x 1 x half> @llvm.experimental.constrained.nearbyint.nxv1f16(<vscale x 1 x half>, metadata, metadata)
+
+define <vscale x 1 x half> @nearbyint_nxv1f16(<vscale x 1 x half> %v) {
+; CHECK-LABEL: nearbyint_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 1 x half> @llvm.experimental.constrained.nearbyint.nxv1f16(<vscale x 1 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 1 x half> %r
+}
+
+declare <vscale x 2 x half> @llvm.experimental.constrained.nearbyint.nxv2f16(<vscale x 2 x half>, metadata, metadata)
+
+define <vscale x 2 x half> @nearbyint_nxv2f16(<vscale x 2 x half> %v) {
+; CHECK-LABEL: nearbyint_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x half> @llvm.experimental.constrained.nearbyint.nxv2f16(<vscale x 2 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 2 x half> %r
+}
+
+declare <vscale x 4 x half> @llvm.experimental.constrained.nearbyint.nxv4f16(<vscale x 4 x half>, metadata, metadata)
+
+define <vscale x 4 x half> @nearbyint_nxv4f16(<vscale x 4 x half> %v) {
+; CHECK-LABEL: nearbyint_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 4 x half> @llvm.experimental.constrained.nearbyint.nxv4f16(<vscale x 4 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 4 x half> %r
+}
+
+declare <vscale x 8 x half> @llvm.experimental.constrained.nearbyint.nxv8f16(<vscale x 8 x half>, metadata, metadata)
+
+define <vscale x 8 x half> @nearbyint_nxv8f16(<vscale x 8 x half> %v) {
+; CHECK-LABEL: nearbyint_nxv8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 8 x half> @llvm.experimental.constrained.nearbyint.nxv8f16(<vscale x 8 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 8 x half> %r
+}
+
+declare <vscale x 16 x half> @llvm.experimental.constrained.nearbyint.nxv16f16(<vscale x 16 x half>, metadata, metadata)
+
+define <vscale x 16 x half> @nearbyint_nxv16f16(<vscale x 16 x half> %v) {
+; CHECK-LABEL: nearbyint_nxv16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 16 x half> @llvm.experimental.constrained.nearbyint.nxv16f16(<vscale x 16 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 16 x half> %r
+}
+
+declare <vscale x 32 x half> @llvm.experimental.constrained.nearbyint.nxv32f16(<vscale x 32 x half>, metadata, metadata)
+
+define <vscale x 32 x half> @nearbyint_nxv32f16(<vscale x 32 x half> %v) {
+; CHECK-LABEL: nearbyint_nxv32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 32 x half> @llvm.experimental.constrained.nearbyint.nxv32f16(<vscale x 32 x half> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 32 x half> %r
+}
+
+declare <vscale x 1 x float> @llvm.experimental.constrained.nearbyint.nxv1f32(<vscale x 1 x float>, metadata, metadata)
+
+define <vscale x 1 x float> @nearbyint_nxv1f32(<vscale x 1 x float> %v) {
+; CHECK-LABEL: nearbyint_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 1 x float> @llvm.experimental.constrained.nearbyint.nxv1f32(<vscale x 1 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %r
+}
+
+declare <vscale x 2 x float> @llvm.experimental.constrained.nearbyint.nxv2f32(<vscale x 2 x float>, metadata, metadata)
+
+define <vscale x 2 x float> @nearbyint_nxv2f32(<vscale x 2 x float> %v) {
+; CHECK-LABEL: nearbyint_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x float> @llvm.experimental.constrained.nearbyint.nxv2f32(<vscale x 2 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %r
+}
+
+declare <vscale x 4 x float> @llvm.experimental.constrained.nearbyint.nxv4f32(<vscale x 4 x float>, metadata, metadata)
+
+define <vscale x 4 x float> @nearbyint_nxv4f32(<vscale x 4 x float> %v) {
+; CHECK-LABEL: nearbyint_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 4 x float> @llvm.experimental.constrained.nearbyint.nxv4f32(<vscale x 4 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %r
+}
+
+declare <vscale x 8 x float> @llvm.experimental.constrained.nearbyint.nxv8f32(<vscale x 8 x float>, metadata, metadata)
+
+define <vscale x 8 x float> @nearbyint_nxv8f32(<vscale x 8 x float> %v) {
+; CHECK-LABEL: nearbyint_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 8 x float> @llvm.experimental.constrained.nearbyint.nxv8f32(<vscale x 8 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %r
+}
+
+declare <vscale x 16 x float> @llvm.experimental.constrained.nearbyint.nxv16f32(<vscale x 16 x float>, metadata, metadata)
+
+define <vscale x 16 x float> @nearbyint_nxv16f32(<vscale x 16 x float> %v) {
+; CHECK-LABEL: nearbyint_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 16 x float> @llvm.experimental.constrained.nearbyint.nxv16f32(<vscale x 16 x float> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 16 x float> %r
+}
+
+declare <vscale x 1 x double> @llvm.experimental.constrained.nearbyint.nxv1f64(<vscale x 1 x double>, metadata, metadata)
+
+define <vscale x 1 x double> @nearbyint_nxv1f64(<vscale x 1 x double> %v) {
+; CHECK-LABEL: nearbyint_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 1 x double> @llvm.experimental.constrained.nearbyint.nxv1f64(<vscale x 1 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %r
+}
+
+declare <vscale x 2 x double> @llvm.experimental.constrained.nearbyint.nxv2f64(<vscale x 2 x double>, metadata, metadata)
+
+define <vscale x 2 x double> @nearbyint_nxv2f64(<vscale x 2 x double> %v) {
+; CHECK-LABEL: nearbyint_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x double> @llvm.experimental.constrained.nearbyint.nxv2f64(<vscale x 2 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %r
+}
+
+declare <vscale x 4 x double> @llvm.experimental.constrained.nearbyint.nxv4f64(<vscale x 4 x double>, metadata, metadata)
+
+define <vscale x 4 x double> @nearbyint_nxv4f64(<vscale x 4 x double> %v) {
+; CHECK-LABEL: nearbyint_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 4 x double> @llvm.experimental.constrained.nearbyint.nxv4f64(<vscale x 4 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %r
+}
+
+declare <vscale x 8 x double> @llvm.experimental.constrained.nearbyint.nxv8f64(<vscale x 8 x double>, metadata, metadata)
+
+define <vscale x 8 x double> @nearbyint_nxv8f64(<vscale x 8 x double> %v) {
+; CHECK-LABEL: nearbyint_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    frflags a0
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    fsflags a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %r = call <vscale x 8 x double> @llvm.experimental.constrained.nearbyint.nxv8f64(<vscale x 8 x double> %v, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %r
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
index 2326db8a3aa10..5c769bc126ba0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
@@ -260,6 +260,45 @@ entry:
   ret float %res
 }
 
+define float @reduce_fadd3(float %x, <4 x float> %v, ptr %rdxptr) {
+; CHECK-LABEL: reduce_fadd3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, zero
+; CHECK-NEXT:    vfredusum.vs v8, v8, v9
+; CHECK-NEXT:    vfmv.f.s fa5, v8
+; CHECK-NEXT:    fadd.s fa0, fa5, fa0
+; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    ret
+entry:
+  %rdx = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %v)
+  %res = fadd fast float %rdx, %x
+  store float %rdx, ptr %rdxptr
+  ret float %res
+}
+
+define float @reduce_fadd4(float %x, float %y, <4 x float> %v, <4 x float> %w) {
+; CHECK-LABEL: reduce_fadd4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vfmv.s.f v10, fa0
+; CHECK-NEXT:    vfredusum.vs v8, v8, v10
+; CHECK-NEXT:    vfmv.f.s fa5, v8
+; CHECK-NEXT:    vfmv.s.f v8, fa1
+; CHECK-NEXT:    vfredusum.vs v8, v9, v8
+; CHECK-NEXT:    vfmv.f.s fa4, v8
+; CHECK-NEXT:    fdiv.s fa0, fa5, fa4
+; CHECK-NEXT:    ret
+entry:
+  %rdx = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %v)
+  %rdx2 = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %w)
+  %res = fadd fast float %rdx, %x
+  %res2 = fadd fast float %rdx2, %y
+  %div = fdiv fast float %res, %res2
+  ret float %div
+}
+
 define float @reduce_fmax(float %x, <4 x float> %v) {
 ; CHECK-LABEL: reduce_fmax:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-vector-cmp.ll b/llvm/test/CodeGen/RISCV/rvv/fold-vector-cmp.ll
index eaabd23a7a582..db2361dd3e586 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-vector-cmp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-vector-cmp.ll
@@ -12,17 +12,14 @@ define i32 @test(i32 %call.i) {
 ;
 ; CHECK-V-LABEL: test:
 ; CHECK-V:       # %bb.0:
-; CHECK-V-NEXT:    lui a1, 524288
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-V-NEXT:    vmv.v.x v8, a1
-; CHECK-V-NEXT:    vsetvli zero, zero, e32, mf2, tu, ma
-; CHECK-V-NEXT:    vmv.s.x v8, a0
-; CHECK-V-NEXT:    addiw a1, a1, 2
-; CHECK-V-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-V-NEXT:    vmslt.vx v0, v8, a1
+; CHECK-V-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-V-NEXT:    lui a0, 524288
+; CHECK-V-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-V-NEXT:    addiw a0, a0, 2
+; CHECK-V-NEXT:    vmslt.vx v0, v8, a0
 ; CHECK-V-NEXT:    vmv.v.i v8, 0
 ; CHECK-V-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-V-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-V-NEXT:    vmv.x.s a0, v8
 ; CHECK-V-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-vp-fadd-and-vp-fmul.ll b/llvm/test/CodeGen/RISCV/rvv/fold-vp-fadd-and-vp-fmul.ll
index 1d30328e97815..ad8965cb624cc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-vp-fadd-and-vp-fmul.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-vp-fadd-and-vp-fmul.ll
@@ -57,3 +57,18 @@ define <vscale x 1 x double> @fma_nonvp(<vscale x 1 x double> %x, <vscale x 1 x
   %2 = call fast <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> %1, <vscale x 1 x double> %z, <vscale x 1 x i1> %m, i32 %vl)
   ret <vscale x 1 x double> %2
 }
+
+define <vscale x 1 x double> @fma_reassociate(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <vscale x 1 x double> %c, <vscale x 1 x double> %d, <vscale x 1 x double> %e, <vscale x 1 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: fma_reassociate:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
+; CHECK-NEXT:    vfmadd.vv v11, v10, v12, v0.t
+; CHECK-NEXT:    vfmadd.vv v9, v8, v11, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %1 = call fast <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <vscale x 1 x i1> %m, i32 %vl)
+  %2 = call fast <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %c, <vscale x 1 x double> %d, <vscale x 1 x i1> %m, i32 %vl)
+  %3 = call fast <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> %1, <vscale x 1 x double> %2, <vscale x 1 x i1> %m, i32 %vl)
+  %4 = call fast <vscale x 1 x double> @llvm.vp.fadd.nxv1f64(<vscale x 1 x double> %3, <vscale x 1 x double> %e, <vscale x 1 x i1> %m, i32 %vl)
+  ret <vscale x 1 x double> %4
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-vp-fsub-and-vp-fmul.ll b/llvm/test/CodeGen/RISCV/rvv/fold-vp-fsub-and-vp-fmul.ll
new file mode 100644
index 0000000000000..56e4a616a36e8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-vp-fsub-and-vp-fmul.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+v -target-abi=lp64d -verify-machineinstrs < %s | FileCheck %s
+
+declare <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x i1> %m, i32 %vl)
+declare <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x i1> %m, i32 %vl)
+declare <vscale x 1 x double> @llvm.vp.fneg.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x i1> %m, i32 %vl)
+
+; (fsub (fmul x, y), z)) -> (fma x, y, (fneg z))
+define <vscale x 1 x double> @test1(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x double> %z, <vscale x 1 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
+; CHECK-NEXT:    vfmsub.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %1 = call fast <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x i1> %m, i32 %vl)
+  %2 = call fast <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> %1, <vscale x 1 x double> %z, <vscale x 1 x i1> %m, i32 %vl)
+  ret <vscale x 1 x double> %2
+}
+
+; (fsub z, (fmul x, y))) -> (fma (fneg y), x, z)
+define <vscale x 1 x double> @test2(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x double> %z, <vscale x 1 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
+; CHECK-NEXT:    vfnmsub.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %1 = call fast <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x i1> %m, i32 %vl)
+  %2 = call fast <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> %z, <vscale x 1 x double> %1, <vscale x 1 x i1> %m, i32 %vl)
+  ret <vscale x 1 x double> %2
+}
+
+; (fsub (fneg (fmul x, y))), z) -> (fma (fneg x), y, (fneg z))
+define <vscale x 1 x double> @test3(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x double> %z, <vscale x 1 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, mu
+; CHECK-NEXT:    vfmsub.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %1 = call fast <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x i1> %m, i32 %vl)
+  %2 = call fast <vscale x 1 x double> @llvm.vp.fneg.nxv1f64(<vscale x 1 x double> %1, <vscale x 1 x i1> %m, i32 %vl)
+  %3 = call fast <vscale x 1 x double> @llvm.vp.fsub.nxv1f64(<vscale x 1 x double> %1, <vscale x 1 x double> %z, <vscale x 1 x i1> %m, i32 %vl)
+  ret <vscale x 1 x double> %3
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fround-costrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fround-costrained-sdnode.ll
new file mode 100644
index 0000000000000..543f7b3076cc4
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fround-costrained-sdnode.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+; This file tests the code generation for `llvm.experimental.constrained.round.*` on scalable vector type.
+
+define <vscale x 1 x half> @round_nxv1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: round_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x half> @llvm.experimental.constrained.round.nxv1f16(<vscale x 1 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x half> %a
+}
+declare <vscale x 1 x half> @llvm.experimental.constrained.round.nxv1f16(<vscale x 1 x half>, metadata)
+
+define <vscale x 2 x half> @round_nxv2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: round_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x half> @llvm.experimental.constrained.round.nxv2f16(<vscale x 2 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x half> %a
+}
+declare <vscale x 2 x half> @llvm.experimental.constrained.round.nxv2f16(<vscale x 2 x half>, metadata)
+
+define <vscale x 4 x half> @round_nxv4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: round_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x half> @llvm.experimental.constrained.round.nxv4f16(<vscale x 4 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x half> %a
+}
+declare <vscale x 4 x half> @llvm.experimental.constrained.round.nxv4f16(<vscale x 4 x half>, metadata)
+
+define <vscale x 8 x half> @round_nxv8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: round_nxv8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x half> @llvm.experimental.constrained.round.nxv8f16(<vscale x 8 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x half> %a
+}
+declare <vscale x 8 x half> @llvm.experimental.constrained.round.nxv8f16(<vscale x 8 x half>, metadata)
+
+define <vscale x 16 x half> @round_nxv16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: round_nxv16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x half> @llvm.experimental.constrained.round.nxv16f16(<vscale x 16 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x half> %a
+}
+declare <vscale x 16 x half> @llvm.experimental.constrained.round.nxv16f16(<vscale x 16 x half>, metadata)
+
+define <vscale x 32 x half> @round_nxv32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: round_nxv32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x half> @llvm.experimental.constrained.round.nxv32f16(<vscale x 32 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 32 x half> %a
+}
+declare <vscale x 32 x half> @llvm.experimental.constrained.round.nxv32f16(<vscale x 32 x half>, metadata)
+
+define <vscale x 1 x float> @round_nxv1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: round_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x float> @llvm.experimental.constrained.round.nxv1f32(<vscale x 1 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %a
+}
+declare <vscale x 1 x float> @llvm.experimental.constrained.round.nxv1f32(<vscale x 1 x float>, metadata)
+
+define <vscale x 2 x float> @round_nxv2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: round_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.round.nxv2f32(<vscale x 2 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %a
+}
+declare <vscale x 2 x float> @llvm.experimental.constrained.round.nxv2f32(<vscale x 2 x float>, metadata)
+
+define <vscale x 4 x float> @round_nxv4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: round_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x float> @llvm.experimental.constrained.round.nxv4f32(<vscale x 4 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %a
+}
+declare <vscale x 4 x float> @llvm.experimental.constrained.round.nxv4f32(<vscale x 4 x float>, metadata)
+
+define <vscale x 8 x float> @round_nxv8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: round_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x float> @llvm.experimental.constrained.round.nxv8f32(<vscale x 8 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %a
+}
+declare <vscale x 8 x float> @llvm.experimental.constrained.round.nxv8f32(<vscale x 8 x float>, metadata)
+
+define <vscale x 16 x float> @round_nxv16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: round_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x float> @llvm.experimental.constrained.round.nxv16f32(<vscale x 16 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x float> %a
+}
+declare <vscale x 16 x float> @llvm.experimental.constrained.round.nxv16f32(<vscale x 16 x float>, metadata)
+
+define <vscale x 1 x double> @round_nxv1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: round_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x double> @llvm.experimental.constrained.round.nxv1f64(<vscale x 1 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %a
+}
+declare <vscale x 1 x double> @llvm.experimental.constrained.round.nxv1f64(<vscale x 1 x double>, metadata)
+
+define <vscale x 2 x double> @round_nxv2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: round_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x double> @llvm.experimental.constrained.round.nxv2f64(<vscale x 2 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %a
+}
+declare <vscale x 2 x double> @llvm.experimental.constrained.round.nxv2f64(<vscale x 2 x double>, metadata)
+
+define <vscale x 4 x double> @round_nxv4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: round_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x double> @llvm.experimental.constrained.round.nxv4f64(<vscale x 4 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %a
+}
+declare <vscale x 4 x double> @llvm.experimental.constrained.round.nxv4f64(<vscale x 4 x double>, metadata)
+
+define <vscale x 8 x double> @round_nxv8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: round_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 4
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x double> @llvm.experimental.constrained.round.nxv8f64(<vscale x 8 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %a
+}
+declare <vscale x 8 x double> @llvm.experimental.constrained.round.nxv8f64(<vscale x 8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll
new file mode 100644
index 0000000000000..58918d118aeb1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/froundeven-constrained-sdnode.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+; This file tests the code generation for `llvm.experimental.constrained.roundeven.*` on scalable vector type.
+
+define <vscale x 1 x half> @roundeven_nxv1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: roundeven_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x half> @llvm.experimental.constrained.roundeven.nxv1f16(<vscale x 1 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x half> %a
+}
+declare <vscale x 1 x half> @llvm.experimental.constrained.roundeven.nxv1f16(<vscale x 1 x half>, metadata)
+
+define <vscale x 2 x half> @roundeven_nxv2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: roundeven_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x half> @llvm.experimental.constrained.roundeven.nxv2f16(<vscale x 2 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x half> %a
+}
+declare <vscale x 2 x half> @llvm.experimental.constrained.roundeven.nxv2f16(<vscale x 2 x half>, metadata)
+
+define <vscale x 4 x half> @roundeven_nxv4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: roundeven_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x half> @llvm.experimental.constrained.roundeven.nxv4f16(<vscale x 4 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x half> %a
+}
+declare <vscale x 4 x half> @llvm.experimental.constrained.roundeven.nxv4f16(<vscale x 4 x half>, metadata)
+
+define <vscale x 8 x half> @roundeven_nxv8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: roundeven_nxv8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x half> @llvm.experimental.constrained.roundeven.nxv8f16(<vscale x 8 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x half> %a
+}
+declare <vscale x 8 x half> @llvm.experimental.constrained.roundeven.nxv8f16(<vscale x 8 x half>, metadata)
+
+define <vscale x 16 x half> @roundeven_nxv16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: roundeven_nxv16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x half> @llvm.experimental.constrained.roundeven.nxv16f16(<vscale x 16 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x half> %a
+}
+declare <vscale x 16 x half> @llvm.experimental.constrained.roundeven.nxv16f16(<vscale x 16 x half>, metadata)
+
+define <vscale x 32 x half> @roundeven_nxv32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: roundeven_nxv32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x half> @llvm.experimental.constrained.roundeven.nxv32f16(<vscale x 32 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 32 x half> %a
+}
+declare <vscale x 32 x half> @llvm.experimental.constrained.roundeven.nxv32f16(<vscale x 32 x half>, metadata)
+
+define <vscale x 1 x float> @roundeven_nxv1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: roundeven_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x float> @llvm.experimental.constrained.roundeven.nxv1f32(<vscale x 1 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %a
+}
+declare <vscale x 1 x float> @llvm.experimental.constrained.roundeven.nxv1f32(<vscale x 1 x float>, metadata)
+
+define <vscale x 2 x float> @roundeven_nxv2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: roundeven_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.roundeven.nxv2f32(<vscale x 2 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %a
+}
+declare <vscale x 2 x float> @llvm.experimental.constrained.roundeven.nxv2f32(<vscale x 2 x float>, metadata)
+
+define <vscale x 4 x float> @roundeven_nxv4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: roundeven_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x float> @llvm.experimental.constrained.roundeven.nxv4f32(<vscale x 4 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %a
+}
+declare <vscale x 4 x float> @llvm.experimental.constrained.roundeven.nxv4f32(<vscale x 4 x float>, metadata)
+
+define <vscale x 8 x float> @roundeven_nxv8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: roundeven_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x float> @llvm.experimental.constrained.roundeven.nxv8f32(<vscale x 8 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %a
+}
+declare <vscale x 8 x float> @llvm.experimental.constrained.roundeven.nxv8f32(<vscale x 8 x float>, metadata)
+
+define <vscale x 16 x float> @roundeven_nxv16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: roundeven_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x float> @llvm.experimental.constrained.roundeven.nxv16f32(<vscale x 16 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x float> %a
+}
+declare <vscale x 16 x float> @llvm.experimental.constrained.roundeven.nxv16f32(<vscale x 16 x float>, metadata)
+
+define <vscale x 1 x double> @roundeven_nxv1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: roundeven_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x double> @llvm.experimental.constrained.roundeven.nxv1f64(<vscale x 1 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %a
+}
+declare <vscale x 1 x double> @llvm.experimental.constrained.roundeven.nxv1f64(<vscale x 1 x double>, metadata)
+
+define <vscale x 2 x double> @roundeven_nxv2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: roundeven_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x double> @llvm.experimental.constrained.roundeven.nxv2f64(<vscale x 2 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %a
+}
+declare <vscale x 2 x double> @llvm.experimental.constrained.roundeven.nxv2f64(<vscale x 2 x double>, metadata)
+
+define <vscale x 4 x double> @roundeven_nxv4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: roundeven_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x double> @llvm.experimental.constrained.roundeven.nxv4f64(<vscale x 4 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %a
+}
+declare <vscale x 4 x double> @llvm.experimental.constrained.roundeven.nxv4f64(<vscale x 4 x double>, metadata)
+
+define <vscale x 8 x double> @roundeven_nxv8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: roundeven_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    fsrmi a0, 0
+; CHECK-NEXT:    vfcvt.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    fsrm a0
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x double> @llvm.experimental.constrained.roundeven.nxv8f64(<vscale x 8 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %a
+}
+declare <vscale x 8 x double> @llvm.experimental.constrained.roundeven.nxv8f64(<vscale x 8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll
new file mode 100644
index 0000000000000..553c66a8b1a7c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/ftrunc-constrained-sdnode.ll
@@ -0,0 +1,305 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <vscale x 1 x half> @trunc_nxv1f16(<vscale x 1 x half> %x) {
+; CHECK-LABEL: trunc_nxv1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI0_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI0_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x half> @llvm.experimental.constrained.trunc.nxv1f16(<vscale x 1 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x half> %a
+}
+declare <vscale x 1 x half> @llvm.experimental.constrained.trunc.nxv1f16(<vscale x 1 x half>, metadata)
+
+define <vscale x 2 x half> @trunc_nxv2f16(<vscale x 2 x half> %x) {
+; CHECK-LABEL: trunc_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI1_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI1_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x half> @llvm.experimental.constrained.trunc.nxv2f16(<vscale x 2 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x half> %a
+}
+declare <vscale x 2 x half> @llvm.experimental.constrained.trunc.nxv2f16(<vscale x 2 x half>, metadata)
+
+define <vscale x 4 x half> @trunc_nxv4f16(<vscale x 4 x half> %x) {
+; CHECK-LABEL: trunc_nxv4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI2_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI2_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x half> @llvm.experimental.constrained.trunc.nxv4f16(<vscale x 4 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x half> %a
+}
+declare <vscale x 4 x half> @llvm.experimental.constrained.trunc.nxv4f16(<vscale x 4 x half>, metadata)
+
+define <vscale x 8 x half> @trunc_nxv8f16(<vscale x 8 x half> %x) {
+; CHECK-LABEL: trunc_nxv8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI3_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x half> @llvm.experimental.constrained.trunc.nxv8f16(<vscale x 8 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x half> %a
+}
+declare <vscale x 8 x half> @llvm.experimental.constrained.trunc.nxv8f16(<vscale x 8 x half>, metadata)
+
+define <vscale x 16 x half> @trunc_nxv16f16(<vscale x 16 x half> %x) {
+; CHECK-LABEL: trunc_nxv16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI4_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x half> @llvm.experimental.constrained.trunc.nxv16f16(<vscale x 16 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x half> %a
+}
+declare <vscale x 16 x half> @llvm.experimental.constrained.trunc.nxv16f16(<vscale x 16 x half>, metadata)
+
+define <vscale x 32 x half> @trunc_nxv32f16(<vscale x 32 x half> %x) {
+; CHECK-LABEL: trunc_nxv32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI5_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e16, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 32 x half> @llvm.experimental.constrained.trunc.nxv32f16(<vscale x 32 x half> %x, metadata !"fpexcept.strict")
+  ret <vscale x 32 x half> %a
+}
+declare <vscale x 32 x half> @llvm.experimental.constrained.trunc.nxv32f16(<vscale x 32 x half>, metadata)
+
+define <vscale x 1 x float> @trunc_nxv1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: trunc_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x float> @llvm.experimental.constrained.trunc.nxv1f32(<vscale x 1 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x float> %a
+}
+declare <vscale x 1 x float> @llvm.experimental.constrained.trunc.nxv1f32(<vscale x 1 x float>, metadata)
+
+define <vscale x 2 x float> @trunc_nxv2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: trunc_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x float> @llvm.experimental.constrained.trunc.nxv2f32(<vscale x 2 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x float> %a
+}
+declare <vscale x 2 x float> @llvm.experimental.constrained.trunc.nxv2f32(<vscale x 2 x float>, metadata)
+
+define <vscale x 4 x float> @trunc_nxv4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: trunc_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x float> @llvm.experimental.constrained.trunc.nxv4f32(<vscale x 4 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x float> %a
+}
+declare <vscale x 4 x float> @llvm.experimental.constrained.trunc.nxv4f32(<vscale x 4 x float>, metadata)
+
+define <vscale x 8 x float> @trunc_nxv8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: trunc_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x float> @llvm.experimental.constrained.trunc.nxv8f32(<vscale x 8 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x float> %a
+}
+declare <vscale x 8 x float> @llvm.experimental.constrained.trunc.nxv8f32(<vscale x 8 x float>, metadata)
+
+define <vscale x 16 x float> @trunc_nxv16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: trunc_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    lui a0, 307200
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x float> @llvm.experimental.constrained.trunc.nxv16f32(<vscale x 16 x float> %x, metadata !"fpexcept.strict")
+  ret <vscale x 16 x float> %a
+}
+declare <vscale x 16 x float> @llvm.experimental.constrained.trunc.nxv16f32(<vscale x 16 x float>, metadata)
+
+define <vscale x 1 x double> @trunc_nxv1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: trunc_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI11_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v9, v8
+; CHECK-NEXT:    vmflt.vf v0, v9, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v9, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v9, v9, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v9, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x double> @llvm.experimental.constrained.trunc.nxv1f64(<vscale x 1 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 1 x double> %a
+}
+declare <vscale x 1 x double> @llvm.experimental.constrained.trunc.nxv1f64(<vscale x 1 x double>, metadata)
+
+define <vscale x 2 x double> @trunc_nxv2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: trunc_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI12_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v10, v8
+; CHECK-NEXT:    vmflt.vf v0, v10, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v10, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v10, v10, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v10, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x double> @llvm.experimental.constrained.trunc.nxv2f64(<vscale x 2 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 2 x double> %a
+}
+declare <vscale x 2 x double> @llvm.experimental.constrained.trunc.nxv2f64(<vscale x 2 x double>, metadata)
+
+define <vscale x 4 x double> @trunc_nxv4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: trunc_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI13_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v12, v8
+; CHECK-NEXT:    vmflt.vf v0, v12, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v12, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v12, v12, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v12, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x double> @llvm.experimental.constrained.trunc.nxv4f64(<vscale x 4 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 4 x double> %a
+}
+declare <vscale x 4 x double> @llvm.experimental.constrained.trunc.nxv4f64(<vscale x 4 x double>, metadata)
+
+define <vscale x 8 x double> @trunc_nxv8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: trunc_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vmfne.vv v0, v8, v8
+; CHECK-NEXT:    lui a0, %hi(.LCPI14_0)
+; CHECK-NEXT:    fld fa5, %lo(.LCPI14_0)(a0)
+; CHECK-NEXT:    vfadd.vv v8, v8, v8, v0.t
+; CHECK-NEXT:    vfabs.v v16, v8
+; CHECK-NEXT:    vmflt.vf v0, v16, fa5
+; CHECK-NEXT:    vfcvt.rtz.x.f.v v16, v8, v0.t
+; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, mu
+; CHECK-NEXT:    vfsgnj.vv v8, v16, v8, v0.t
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x double> @llvm.experimental.constrained.trunc.nxv8f64(<vscale x 8 x double> %x, metadata !"fpexcept.strict")
+  ret <vscale x 8 x double> %a
+}
+declare <vscale x 8 x double> @llvm.experimental.constrained.trunc.nxv8f64(<vscale x 8 x double>, metadata)
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
index 61de81e8d3128..a2ac684604b94 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
@@ -719,7 +719,7 @@ define <vscale x 8 x i1> @icmp_slt_vi_nxv8i8_2(<vscale x 8 x i8> %va) {
 ; CHECK-LABEL: icmp_slt_vi_nxv8i8_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> poison, i8 0, i32 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
@@ -1439,7 +1439,7 @@ define <vscale x 8 x i1> @icmp_slt_vi_nxv8i16_2(<vscale x 8 x i16> %va) {
 ; CHECK-LABEL: icmp_slt_vi_nxv8i16_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> poison, i16 0, i32 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
@@ -2159,7 +2159,7 @@ define <vscale x 8 x i1> @icmp_slt_vi_nxv8i32_2(<vscale x 8 x i32> %va) {
 ; CHECK-LABEL: icmp_slt_vi_nxv8i32_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i32> poison, i32 0, i32 0
   %splat = shufflevector <vscale x 8 x i32> %head, <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
@@ -3113,7 +3113,7 @@ define <vscale x 8 x i1> @icmp_slt_vi_nxv8i64_2(<vscale x 8 x i64> %va) {
 ; CHECK-LABEL: icmp_slt_vi_nxv8i64_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i64> poison, i64 0, i32 0
   %splat = shufflevector <vscale x 8 x i64> %head, <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
index 4a6f52ac940b9..9e8463a085302 100644
--- a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
@@ -22,9 +22,7 @@ define <4 x i8> @v2i8_2(<2 x i8> %a, <2 x i8> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
@@ -172,9 +170,7 @@ define <4 x i16> @v2i16_2(<2 x i16> %a, <2 x i16> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
@@ -321,9 +317,7 @@ define <4 x i32> @v2i32_2(<2 x i32> %a, < 2 x i32> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
@@ -471,9 +465,7 @@ define <4 x i64> @v2i64_2(<2 x i64> %a, < 2 x i64> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
@@ -560,9 +552,7 @@ define <4 x half> @v2f16_2(<2 x half> %a, <2 x half> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
@@ -683,9 +673,7 @@ define <4 x float> @v2f32_2(<2 x float> %a, <2 x float> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
@@ -776,9 +764,7 @@ define <4 x double> @v2f64_2(<2 x double> %a, < 2 x double> %b) {
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 1
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v8, v10, 2
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 0ee01da021809..27805f6bffc56 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -4369,3 +4369,85 @@ vector.body:                                      ; preds = %vector.body, %entry
 for.cond.cleanup:                                 ; preds = %vector.body
   ret void
 }
+
+declare <4 x i1> @llvm.vp.icmp.v4i32(<4 x i32>, <4 x i32>, metadata, <4 x i1>, i32)
+
+define void @sink_splat_vp_icmp(ptr nocapture %x, i32 signext %y, <4 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: sink_splat_vp_icmp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    li a3, 1024
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v9, 0
+; CHECK-NEXT:  .LBB88_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vle32.v v10, (a0)
+; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmseq.vx v0, v10, a1, v0.t
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v9, (a0), v0.t
+; CHECK-NEXT:    addi a3, a3, -4
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    bnez a3, .LBB88_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %y, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, ptr %x, i64 %index
+  %wide.load = load <4 x i32>, ptr %0, align 4
+  %1 = call <4 x i1> @llvm.vp.icmp.v4i32(<4 x i32> %wide.load, <4 x i32> %broadcast.splat, metadata !"eq", <4 x i1> %m, i32 %vl)
+  call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr %0, i32 4, <4 x i1> %1)
+  %index.next = add nuw i64 %index, 4
+  %2 = icmp eq i64 %index.next, 1024
+  br i1 %2, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
+
+declare <4 x i1> @llvm.vp.fcmp.v4f32(<4 x float>, <4 x float>, metadata, <4 x i1>, i32)
+
+define void @sink_splat_vp_fcmp(ptr nocapture %x, float %y, <4 x i1> %m, i32 zeroext %vl) {
+; CHECK-LABEL: sink_splat_vp_fcmp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmv1r.v v8, v0
+; CHECK-NEXT:    li a2, 1024
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v9, 0
+; CHECK-NEXT:  .LBB89_1: # %vector.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vle32.v v10, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmfeq.vf v0, v10, fa0, v0.t
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vse32.v v9, (a0), v0.t
+; CHECK-NEXT:    addi a2, a2, -4
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    bnez a2, .LBB89_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splatinsert = insertelement <4 x float> poison, float %y, i32 0
+  %broadcast.splat = shufflevector <4 x float> %broadcast.splatinsert, <4 x float> poison, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds float, ptr %x, i64 %index
+  %wide.load = load <4 x float>, ptr %0, align 4
+  %1 = call <4 x i1> @llvm.vp.fcmp.v4f32(<4 x float> %wide.load, <4 x float> %broadcast.splat, metadata !"oeq", <4 x i1> %m, i32 %vl)
+  call void @llvm.masked.store.v4f32.p0(<4 x float> zeroinitializer, ptr %0, i32 4, <4 x i1> %1)
+  %index.next = add nuw i64 %index, 4
+  %2 = icmp eq i64 %index.next, 1024
+  br i1 %2, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll
index a734bfa84f55e..443fe93a618c5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sshl_sat_vec.ll
@@ -10,7 +10,7 @@ define <2 x i64> @vec_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: vec_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    srli a1, a0, 1
 ; CHECK-NEXT:    vsll.vv v10, v8, v9
@@ -30,7 +30,7 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 524288
 ; CHECK-NEXT:    addiw a0, a0, -1
 ; CHECK-NEXT:    vsll.vv v10, v8, v9
@@ -51,7 +51,7 @@ define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: vec_v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 8
 ; CHECK-NEXT:    addiw a1, a0, -1
 ; CHECK-NEXT:    vsll.vv v10, v8, v9
@@ -70,7 +70,7 @@ define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: vec_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, 127
 ; CHECK-NEXT:    vsll.vv v10, v8, v9
 ; CHECK-NEXT:    vsra.vv v9, v10, v9
@@ -94,7 +94,7 @@ define <vscale x 2 x i64> @vec_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64>
 ; CHECK-LABEL: vec_nxv2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    srli a1, a0, 1
 ; CHECK-NEXT:    vsll.vv v12, v8, v10
@@ -114,7 +114,7 @@ define <vscale x 4 x i32> @vec_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32>
 ; CHECK-LABEL: vec_nxv4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 524288
 ; CHECK-NEXT:    addiw a0, a0, -1
 ; CHECK-NEXT:    vsll.vv v12, v8, v10
@@ -135,7 +135,7 @@ define <vscale x 8 x i16> @vec_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16>
 ; CHECK-LABEL: vec_nxv8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    lui a0, 8
 ; CHECK-NEXT:    addiw a1, a0, -1
 ; CHECK-NEXT:    vsll.vv v12, v8, v10
@@ -154,7 +154,7 @@ define <vscale x 16 x i8> @vec_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i8>
 ; CHECK-LABEL: vec_nxv16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmslt.vx v0, v8, zero
+; CHECK-NEXT:    vmsle.vi v0, v8, -1
 ; CHECK-NEXT:    li a0, 127
 ; CHECK-NEXT:    vsll.vv v12, v8, v10
 ; CHECK-NEXT:    vsra.vv v14, v12, v10
diff --git a/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll b/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll
index b0117f4bf3b37..3dadbbf6710ee 100644
--- a/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/unaligned-loads-stores.ll
@@ -3,6 +3,11 @@
 ; RUN:    -verify-machineinstrs | FileCheck %s
 ; RUN: llc -mtriple riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v < %s \
 ; RUN:    -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple riscv32 -mattr=+d,+zfh,+experimental-zvfh,+v,+unaligned-vector-mem < %s \
+; RUN:    -verify-machineinstrs | FileCheck --check-prefix=UNALIGNED %s
+; RUN: llc -mtriple riscv64 -mattr=+d,+zfh,+experimental-zvfh,+v,+unaligned-vector-mem < %s \
+; RUN:    -verify-machineinstrs | FileCheck --check-prefix=UNALIGNED %s
+
 
 define <vscale x 1 x i32> @unaligned_load_nxv1i32_a1(<vscale x 1 x i32>* %ptr) {
 ; CHECK-LABEL: unaligned_load_nxv1i32_a1:
@@ -10,6 +15,12 @@ define <vscale x 1 x i32> @unaligned_load_nxv1i32_a1(<vscale x 1 x i32>* %ptr) {
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv1i32_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; UNALIGNED-NEXT:    vle32.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i32>, <vscale x 1 x i32>* %ptr, align 1
   ret <vscale x 1 x i32> %v
 }
@@ -20,6 +31,12 @@ define <vscale x 1 x i32> @unaligned_load_nxv1i32_a2(<vscale x 1 x i32>* %ptr) {
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv1i32_a2:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; UNALIGNED-NEXT:    vle32.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i32>, <vscale x 1 x i32>* %ptr, align 2
   ret <vscale x 1 x i32> %v
 }
@@ -30,6 +47,12 @@ define <vscale x 1 x i32> @aligned_load_nxv1i32_a4(<vscale x 1 x i32>* %ptr) {
 ; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_load_nxv1i32_a4:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; UNALIGNED-NEXT:    vle32.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i32>, <vscale x 1 x i32>* %ptr, align 4
   ret <vscale x 1 x i32> %v
 }
@@ -39,6 +62,11 @@ define <vscale x 1 x i64> @unaligned_load_nxv1i64_a1(<vscale x 1 x i64>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv1i64_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl1re64.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i64>, <vscale x 1 x i64>* %ptr, align 1
   ret <vscale x 1 x i64> %v
 }
@@ -48,6 +76,11 @@ define <vscale x 1 x i64> @unaligned_load_nxv1i64_a4(<vscale x 1 x i64>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv1i64_a4:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl1re64.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i64>, <vscale x 1 x i64>* %ptr, align 4
   ret <vscale x 1 x i64> %v
 }
@@ -57,6 +90,11 @@ define <vscale x 1 x i64> @aligned_load_nxv1i64_a8(<vscale x 1 x i64>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1re64.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_load_nxv1i64_a8:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl1re64.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i64>, <vscale x 1 x i64>* %ptr, align 8
   ret <vscale x 1 x i64> %v
 }
@@ -66,6 +104,11 @@ define <vscale x 2 x i64> @unaligned_load_nxv2i64_a1(<vscale x 2 x i64>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv2i64_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re64.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 2 x i64>, <vscale x 2 x i64>* %ptr, align 1
   ret <vscale x 2 x i64> %v
 }
@@ -75,6 +118,11 @@ define <vscale x 2 x i64> @unaligned_load_nxv2i64_a4(<vscale x 2 x i64>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv2i64_a4:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re64.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 2 x i64>, <vscale x 2 x i64>* %ptr, align 4
   ret <vscale x 2 x i64> %v
 }
@@ -84,6 +132,11 @@ define <vscale x 2 x i64> @aligned_load_nxv2i64_a8(<vscale x 2 x i64>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re64.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_load_nxv2i64_a8:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re64.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 2 x i64>, <vscale x 2 x i64>* %ptr, align 8
   ret <vscale x 2 x i64> %v
 }
@@ -95,6 +148,12 @@ define <vscale x 1 x i1> @unaligned_load_nxv1i1_a1(<vscale x 1 x i1>* %ptr) {
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vlm.v v0, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv1i1_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
+; UNALIGNED-NEXT:    vlm.v v0, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 1 x i1>, <vscale x 1 x i1>* %ptr, align 1
   ret <vscale x 1 x i1> %v
 }
@@ -104,6 +163,11 @@ define <vscale x 4 x float> @unaligned_load_nxv4f32_a1(<vscale x 4 x float>* %pt
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv4f32_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re32.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 4 x float>, <vscale x 4 x float>* %ptr, align 1
   ret <vscale x 4 x float> %v
 }
@@ -113,6 +177,11 @@ define <vscale x 4 x float> @unaligned_load_nxv4f32_a2(<vscale x 4 x float>* %pt
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv4f32_a2:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re32.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 4 x float>, <vscale x 4 x float>* %ptr, align 2
   ret <vscale x 4 x float> %v
 }
@@ -122,6 +191,11 @@ define <vscale x 4 x float> @aligned_load_nxv4f32_a4(<vscale x 4 x float>* %ptr)
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re32.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_load_nxv4f32_a4:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re32.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 4 x float>, <vscale x 4 x float>* %ptr, align 4
   ret <vscale x 4 x float> %v
 }
@@ -131,6 +205,11 @@ define <vscale x 8 x half> @unaligned_load_nxv8f16_a1(<vscale x 8 x half>* %ptr)
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_load_nxv8f16_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re16.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 8 x half>, <vscale x 8 x half>* %ptr, align 1
   ret <vscale x 8 x half> %v
 }
@@ -140,6 +219,11 @@ define <vscale x 8 x half> @aligned_load_nxv8f16_a2(<vscale x 8 x half>* %ptr) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re16.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_load_nxv8f16_a2:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vl2re16.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   %v = load <vscale x 8 x half>, <vscale x 8 x half>* %ptr, align 2
   ret <vscale x 8 x half> %v
 }
@@ -149,6 +233,11 @@ define void @unaligned_store_nxv4i32_a1(<vscale x 4 x i32> %x, <vscale x 4 x i32
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vs2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_store_nxv4i32_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vs2r.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   store <vscale x 4 x i32> %x, <vscale x 4 x i32>* %ptr, align 1
   ret void
 }
@@ -158,6 +247,11 @@ define void @unaligned_store_nxv4i32_a2(<vscale x 4 x i32> %x, <vscale x 4 x i32
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vs2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_store_nxv4i32_a2:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vs2r.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   store <vscale x 4 x i32> %x, <vscale x 4 x i32>* %ptr, align 2
   ret void
 }
@@ -167,6 +261,11 @@ define void @aligned_store_nxv4i32_a4(<vscale x 4 x i32> %x, <vscale x 4 x i32>*
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vs2r.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_store_nxv4i32_a4:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vs2r.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   store <vscale x 4 x i32> %x, <vscale x 4 x i32>* %ptr, align 4
   ret void
 }
@@ -177,6 +276,12 @@ define void @unaligned_store_nxv1i16_a1(<vscale x 1 x i16> %x, <vscale x 1 x i16
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vse8.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: unaligned_store_nxv1i16_a1:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; UNALIGNED-NEXT:    vse16.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   store <vscale x 1 x i16> %x, <vscale x 1 x i16>* %ptr, align 1
   ret void
 }
@@ -187,6 +292,12 @@ define void @aligned_store_nxv1i16_a2(<vscale x 1 x i16> %x, <vscale x 1 x i16>*
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vse16.v v8, (a0)
 ; CHECK-NEXT:    ret
+;
+; UNALIGNED-LABEL: aligned_store_nxv1i16_a2:
+; UNALIGNED:       # %bb.0:
+; UNALIGNED-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
+; UNALIGNED-NEXT:    vse16.v v8, (a0)
+; UNALIGNED-NEXT:    ret
   store <vscale x 1 x i16> %x, <vscale x 1 x i16>* %ptr, align 2
   ret void
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index 1c8463978cf2f..67744dd4a8dc3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -7,242 +7,58 @@
 define {<16 x i1>, <16 x i1>} @vector_deinterleave_v16i1_v32i1(<32 x i1> %vec) {
 ; RV32-LABEL: vector_deinterleave_v16i1_v32i1:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
 ; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    vfirst.m a0, v0
-; RV32-NEXT:    seqz a0, a0
-; RV32-NEXT:    sb a0, 16(sp)
-; RV32-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV32-NEXT:    vmv.x.s a0, v0
-; RV32-NEXT:    slli a1, a0, 17
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 23(sp)
-; RV32-NEXT:    slli a1, a0, 19
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 22(sp)
-; RV32-NEXT:    slli a1, a0, 21
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 21(sp)
-; RV32-NEXT:    slli a1, a0, 23
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 20(sp)
-; RV32-NEXT:    slli a1, a0, 25
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 19(sp)
-; RV32-NEXT:    slli a1, a0, 27
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 18(sp)
-; RV32-NEXT:    slli a1, a0, 29
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 17(sp)
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    vmerge.vim v10, v8, 1, v0
+; RV32-NEXT:    vid.v v9
+; RV32-NEXT:    vadd.vv v11, v9, v9
+; RV32-NEXT:    vrgather.vv v9, v10, v11
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v0, 2
+; RV32-NEXT:    vslidedown.vi v0, v0, 2
 ; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    vfirst.m a1, v8
-; RV32-NEXT:    seqz a1, a1
-; RV32-NEXT:    sb a1, 24(sp)
-; RV32-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV32-NEXT:    vmv.x.s a1, v8
-; RV32-NEXT:    slli a2, a1, 17
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 31(sp)
-; RV32-NEXT:    slli a2, a1, 19
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 30(sp)
-; RV32-NEXT:    slli a2, a1, 21
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 29(sp)
-; RV32-NEXT:    slli a2, a1, 23
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 28(sp)
-; RV32-NEXT:    slli a2, a1, 25
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 27(sp)
-; RV32-NEXT:    slli a2, a1, 27
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 26(sp)
-; RV32-NEXT:    slli a2, a1, 29
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 25(sp)
-; RV32-NEXT:    slli a2, a0, 16
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 7(sp)
-; RV32-NEXT:    slli a2, a0, 18
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 6(sp)
-; RV32-NEXT:    slli a2, a0, 20
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 5(sp)
-; RV32-NEXT:    slli a2, a0, 22
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 4(sp)
-; RV32-NEXT:    slli a2, a0, 24
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 3(sp)
-; RV32-NEXT:    slli a2, a0, 26
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 2(sp)
-; RV32-NEXT:    slli a2, a0, 28
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 1(sp)
-; RV32-NEXT:    slli a0, a0, 30
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 0(sp)
-; RV32-NEXT:    slli a0, a1, 16
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 15(sp)
-; RV32-NEXT:    slli a0, a1, 18
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 14(sp)
-; RV32-NEXT:    slli a0, a1, 20
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 13(sp)
-; RV32-NEXT:    slli a0, a1, 22
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 12(sp)
-; RV32-NEXT:    slli a0, a1, 24
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 11(sp)
-; RV32-NEXT:    slli a0, a1, 26
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 10(sp)
-; RV32-NEXT:    slli a0, a1, 28
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 9(sp)
-; RV32-NEXT:    slli a1, a1, 30
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 8(sp)
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    vle8.v v8, (a0)
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vle8.v v9, (a0)
-; RV32-NEXT:    vand.vi v8, v8, 1
-; RV32-NEXT:    vmsne.vi v0, v8, 0
-; RV32-NEXT:    vand.vi v8, v9, 1
-; RV32-NEXT:    vmsne.vi v8, v8, 0
-; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    lui a0, 16
+; RV32-NEXT:    addi a0, a0, -256
+; RV32-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV32-NEXT:    vmv.s.x v0, a0
+; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; RV32-NEXT:    vadd.vi v12, v11, -16
+; RV32-NEXT:    vrgather.vv v9, v8, v12, v0.t
+; RV32-NEXT:    vmsne.vi v9, v9, 0
+; RV32-NEXT:    vadd.vi v12, v11, 1
+; RV32-NEXT:    vrgather.vv v13, v10, v12
+; RV32-NEXT:    vadd.vi v10, v11, -15
+; RV32-NEXT:    vrgather.vv v13, v8, v10, v0.t
+; RV32-NEXT:    vmsne.vi v8, v13, 0
+; RV32-NEXT:    vmv.v.v v0, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: vector_deinterleave_v16i1_v32i1:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -32
-; RV64-NEXT:    .cfi_def_cfa_offset 32
 ; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    vfirst.m a0, v0
-; RV64-NEXT:    seqz a0, a0
-; RV64-NEXT:    sb a0, 16(sp)
-; RV64-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV64-NEXT:    vmv.x.s a0, v0
-; RV64-NEXT:    slli a1, a0, 49
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 23(sp)
-; RV64-NEXT:    slli a1, a0, 51
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 22(sp)
-; RV64-NEXT:    slli a1, a0, 53
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 21(sp)
-; RV64-NEXT:    slli a1, a0, 55
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 20(sp)
-; RV64-NEXT:    slli a1, a0, 57
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 19(sp)
-; RV64-NEXT:    slli a1, a0, 59
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 18(sp)
-; RV64-NEXT:    slli a1, a0, 61
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 17(sp)
+; RV64-NEXT:    vmv.v.i v8, 0
+; RV64-NEXT:    vmerge.vim v10, v8, 1, v0
+; RV64-NEXT:    vid.v v9
+; RV64-NEXT:    vadd.vv v11, v9, v9
+; RV64-NEXT:    vrgather.vv v9, v10, v11
 ; RV64-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; RV64-NEXT:    vslidedown.vi v8, v0, 2
-; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    vfirst.m a1, v8
-; RV64-NEXT:    seqz a1, a1
-; RV64-NEXT:    sb a1, 24(sp)
-; RV64-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV64-NEXT:    vmv.x.s a1, v8
-; RV64-NEXT:    slli a2, a1, 49
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 31(sp)
-; RV64-NEXT:    slli a2, a1, 51
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 30(sp)
-; RV64-NEXT:    slli a2, a1, 53
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 29(sp)
-; RV64-NEXT:    slli a2, a1, 55
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 28(sp)
-; RV64-NEXT:    slli a2, a1, 57
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 27(sp)
-; RV64-NEXT:    slli a2, a1, 59
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 26(sp)
-; RV64-NEXT:    slli a2, a1, 61
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 25(sp)
-; RV64-NEXT:    slli a2, a0, 48
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 7(sp)
-; RV64-NEXT:    slli a2, a0, 50
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 6(sp)
-; RV64-NEXT:    slli a2, a0, 52
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 5(sp)
-; RV64-NEXT:    slli a2, a0, 54
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 4(sp)
-; RV64-NEXT:    slli a2, a0, 56
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 3(sp)
-; RV64-NEXT:    slli a2, a0, 58
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 2(sp)
-; RV64-NEXT:    slli a2, a0, 60
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 1(sp)
-; RV64-NEXT:    slli a0, a0, 62
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 0(sp)
-; RV64-NEXT:    slli a0, a1, 48
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 15(sp)
-; RV64-NEXT:    slli a0, a1, 50
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 14(sp)
-; RV64-NEXT:    slli a0, a1, 52
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 13(sp)
-; RV64-NEXT:    slli a0, a1, 54
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 12(sp)
-; RV64-NEXT:    slli a0, a1, 56
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 11(sp)
-; RV64-NEXT:    slli a0, a1, 58
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 10(sp)
-; RV64-NEXT:    slli a0, a1, 60
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 9(sp)
-; RV64-NEXT:    slli a1, a1, 62
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 8(sp)
-; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    vslidedown.vi v0, v0, 2
 ; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    vle8.v v8, (a0)
-; RV64-NEXT:    mv a0, sp
-; RV64-NEXT:    vle8.v v9, (a0)
-; RV64-NEXT:    vand.vi v8, v8, 1
-; RV64-NEXT:    vmsne.vi v0, v8, 0
-; RV64-NEXT:    vand.vi v8, v9, 1
-; RV64-NEXT:    vmsne.vi v8, v8, 0
-; RV64-NEXT:    addi sp, sp, 32
+; RV64-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV64-NEXT:    lui a0, 16
+; RV64-NEXT:    addiw a0, a0, -256
+; RV64-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV64-NEXT:    vmv.s.x v0, a0
+; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
+; RV64-NEXT:    vadd.vi v12, v11, -16
+; RV64-NEXT:    vrgather.vv v9, v8, v12, v0.t
+; RV64-NEXT:    vmsne.vi v9, v9, 0
+; RV64-NEXT:    vadd.vi v12, v11, 1
+; RV64-NEXT:    vrgather.vv v13, v10, v12
+; RV64-NEXT:    vadd.vi v10, v11, -15
+; RV64-NEXT:    vrgather.vv v13, v8, v10, v0.t
+; RV64-NEXT:    vmsne.vi v8, v13, 0
+; RV64-NEXT:    vmv.v.v v0, v9
 ; RV64-NEXT:    ret
 %retval = call {<16 x i1>, <16 x i1>} @llvm.experimental.vector.deinterleave2.v32i1(<32 x i1> %vec)
 ret {<16 x i1>, <16 x i1>} %retval
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index 25d160e39f1a6..de0e9db9c76b9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -18,10 +18,8 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
 ; CHECK-NEXT:    vnsrl.wi v8, v12, 0
-; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    vnsrl.wi v8, v12, 8
-; CHECK-NEXT:    vand.vi v10, v8, 1
+; CHECK-NEXT:    vnsrl.wi v10, v12, 8
 ; CHECK-NEXT:    vmsne.vi v8, v10, 0
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.experimental.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
index aa340952749c3..26e31339aaf81 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-fixed.ll
@@ -5,251 +5,28 @@
 ; Integers
 
 define <32 x i1> @vector_interleave_v32i1_v16i1(<16 x i1> %a, <16 x i1> %b) {
-; RV32-LABEL: vector_interleave_v32i1_v16i1:
-; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -64
-; RV32-NEXT:    .cfi_def_cfa_offset 64
-; RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 64
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    andi sp, sp, -32
-; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV32-NEXT:    vfirst.m a0, v8
-; RV32-NEXT:    seqz a0, a0
-; RV32-NEXT:    sb a0, 1(sp)
-; RV32-NEXT:    vfirst.m a0, v0
-; RV32-NEXT:    seqz a0, a0
-; RV32-NEXT:    sb a0, 0(sp)
-; RV32-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV32-NEXT:    vmv.x.s a0, v8
-; RV32-NEXT:    slli a1, a0, 16
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 31(sp)
-; RV32-NEXT:    vmv.x.s a1, v0
-; RV32-NEXT:    slli a2, a1, 16
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 30(sp)
-; RV32-NEXT:    slli a2, a0, 17
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 29(sp)
-; RV32-NEXT:    slli a2, a1, 17
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 28(sp)
-; RV32-NEXT:    slli a2, a0, 18
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 27(sp)
-; RV32-NEXT:    slli a2, a1, 18
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 26(sp)
-; RV32-NEXT:    slli a2, a0, 19
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 25(sp)
-; RV32-NEXT:    slli a2, a1, 19
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 24(sp)
-; RV32-NEXT:    slli a2, a0, 20
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 23(sp)
-; RV32-NEXT:    slli a2, a1, 20
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 22(sp)
-; RV32-NEXT:    slli a2, a0, 21
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 21(sp)
-; RV32-NEXT:    slli a2, a1, 21
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 20(sp)
-; RV32-NEXT:    slli a2, a0, 22
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 19(sp)
-; RV32-NEXT:    slli a2, a1, 22
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 18(sp)
-; RV32-NEXT:    slli a2, a0, 23
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 17(sp)
-; RV32-NEXT:    slli a2, a1, 23
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 16(sp)
-; RV32-NEXT:    slli a2, a0, 24
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 15(sp)
-; RV32-NEXT:    slli a2, a1, 24
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 14(sp)
-; RV32-NEXT:    slli a2, a0, 25
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 13(sp)
-; RV32-NEXT:    slli a2, a1, 25
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 12(sp)
-; RV32-NEXT:    slli a2, a0, 26
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 11(sp)
-; RV32-NEXT:    slli a2, a1, 26
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 10(sp)
-; RV32-NEXT:    slli a2, a0, 27
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 9(sp)
-; RV32-NEXT:    slli a2, a1, 27
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 8(sp)
-; RV32-NEXT:    slli a2, a0, 28
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 7(sp)
-; RV32-NEXT:    slli a2, a1, 28
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 6(sp)
-; RV32-NEXT:    slli a2, a0, 29
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 5(sp)
-; RV32-NEXT:    slli a2, a1, 29
-; RV32-NEXT:    srli a2, a2, 31
-; RV32-NEXT:    sb a2, 4(sp)
-; RV32-NEXT:    slli a0, a0, 30
-; RV32-NEXT:    srli a0, a0, 31
-; RV32-NEXT:    sb a0, 3(sp)
-; RV32-NEXT:    slli a1, a1, 30
-; RV32-NEXT:    srli a1, a1, 31
-; RV32-NEXT:    sb a1, 2(sp)
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    mv a1, sp
-; RV32-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV32-NEXT:    vle8.v v8, (a1)
-; RV32-NEXT:    vand.vi v8, v8, 1
-; RV32-NEXT:    vmsne.vi v0, v8, 0
-; RV32-NEXT:    addi sp, s0, -64
-; RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32-NEXT:    addi sp, sp, 64
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: vector_interleave_v32i1_v16i1:
-; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -64
-; RV64-NEXT:    .cfi_def_cfa_offset 64
-; RV64-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 64
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -32
-; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; RV64-NEXT:    vfirst.m a0, v8
-; RV64-NEXT:    seqz a0, a0
-; RV64-NEXT:    sb a0, 1(sp)
-; RV64-NEXT:    vfirst.m a0, v0
-; RV64-NEXT:    seqz a0, a0
-; RV64-NEXT:    sb a0, 0(sp)
-; RV64-NEXT:    vsetivli zero, 0, e16, mf4, ta, ma
-; RV64-NEXT:    vmv.x.s a0, v8
-; RV64-NEXT:    slli a1, a0, 48
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 31(sp)
-; RV64-NEXT:    vmv.x.s a1, v0
-; RV64-NEXT:    slli a2, a1, 48
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 30(sp)
-; RV64-NEXT:    slli a2, a0, 49
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 29(sp)
-; RV64-NEXT:    slli a2, a1, 49
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 28(sp)
-; RV64-NEXT:    slli a2, a0, 50
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 27(sp)
-; RV64-NEXT:    slli a2, a1, 50
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 26(sp)
-; RV64-NEXT:    slli a2, a0, 51
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 25(sp)
-; RV64-NEXT:    slli a2, a1, 51
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 24(sp)
-; RV64-NEXT:    slli a2, a0, 52
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 23(sp)
-; RV64-NEXT:    slli a2, a1, 52
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 22(sp)
-; RV64-NEXT:    slli a2, a0, 53
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 21(sp)
-; RV64-NEXT:    slli a2, a1, 53
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 20(sp)
-; RV64-NEXT:    slli a2, a0, 54
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 19(sp)
-; RV64-NEXT:    slli a2, a1, 54
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 18(sp)
-; RV64-NEXT:    slli a2, a0, 55
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 17(sp)
-; RV64-NEXT:    slli a2, a1, 55
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 16(sp)
-; RV64-NEXT:    slli a2, a0, 56
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 15(sp)
-; RV64-NEXT:    slli a2, a1, 56
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 14(sp)
-; RV64-NEXT:    slli a2, a0, 57
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 13(sp)
-; RV64-NEXT:    slli a2, a1, 57
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 12(sp)
-; RV64-NEXT:    slli a2, a0, 58
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 11(sp)
-; RV64-NEXT:    slli a2, a1, 58
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 10(sp)
-; RV64-NEXT:    slli a2, a0, 59
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 9(sp)
-; RV64-NEXT:    slli a2, a1, 59
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 8(sp)
-; RV64-NEXT:    slli a2, a0, 60
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 7(sp)
-; RV64-NEXT:    slli a2, a1, 60
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 6(sp)
-; RV64-NEXT:    slli a2, a0, 61
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 5(sp)
-; RV64-NEXT:    slli a2, a1, 61
-; RV64-NEXT:    srli a2, a2, 63
-; RV64-NEXT:    sb a2, 4(sp)
-; RV64-NEXT:    slli a0, a0, 62
-; RV64-NEXT:    srli a0, a0, 63
-; RV64-NEXT:    sb a0, 3(sp)
-; RV64-NEXT:    slli a1, a1, 62
-; RV64-NEXT:    srli a1, a1, 63
-; RV64-NEXT:    sb a1, 2(sp)
-; RV64-NEXT:    li a0, 32
-; RV64-NEXT:    mv a1, sp
-; RV64-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; RV64-NEXT:    vle8.v v8, (a1)
-; RV64-NEXT:    vand.vi v8, v8, 1
-; RV64-NEXT:    vmsne.vi v0, v8, 0
-; RV64-NEXT:    addi sp, s0, -64
-; RV64-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64-NEXT:    addi sp, sp, 64
-; RV64-NEXT:    ret
+; CHECK-LABEL: vector_interleave_v32i1_v16i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vmclr.m v9
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, tu, ma
+; CHECK-NEXT:    vslideup.vi v9, v0, 0
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vslideup.vi v9, v8, 2
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
+; CHECK-NEXT:    vsetivli zero, 16, e8, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 16
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vwaddu.vv v12, v8, v10
+; CHECK-NEXT:    li a1, -1
+; CHECK-NEXT:    vwmaccu.vx v12, a1, v10
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    ret
 	   %res = call <32 x i1> @llvm.experimental.vector.interleave2.v32i1(<16 x i1> %a, <16 x i1> %b)
 	   ret <32 x i1> %res
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index f6e6de24af91f..d22f2edb366f4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -17,15 +17,13 @@ define <vscale x 32 x i1> @vector_interleave_nxv32i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    vwaddu.vv v16, v8, v12
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vwmaccu.vx v16, a0, v12
-; CHECK-NEXT:    vand.vi v8, v18, 1
-; CHECK-NEXT:    vmsne.vi v10, v8, 0
-; CHECK-NEXT:    vand.vi v8, v16, 1
-; CHECK-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-NEXT:    vmsne.vi v8, v18, 0
+; CHECK-NEXT:    vmsne.vi v0, v16, 0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vx v0, v10, a0
+; CHECK-NEXT:    vslideup.vx v0, v8, a0
 ; CHECK-NEXT:    ret
 	   %res = call <vscale x 32 x i1> @llvm.experimental.vector.interleave2.nxv32i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b)
 	   ret <vscale x 32 x i1> %res
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll b/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll
index 6faa4f6b45da7..67431f72c9cc1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-splice.ll
@@ -15,9 +15,7 @@ define <vscale x 1 x i1> @splice_nxv1i1_offset_negone(<vscale x 1 x i1> %a, <vsc
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 3
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v10, v10, a0
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v8, v9, 1, v0
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
@@ -61,9 +59,7 @@ define <vscale x 2 x i1> @splice_nxv2i1_offset_negone(<vscale x 2 x i1> %a, <vsc
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v10, v10, a0
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v8, v9, 1, v0
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
@@ -107,9 +103,7 @@ define <vscale x 4 x i1> @splice_nxv4i1_offset_negone(<vscale x 4 x i1> %a, <vsc
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v10, v10, a0
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v8, v9, 1, v0
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
@@ -152,9 +146,7 @@ define <vscale x 8 x i1> @splice_nxv8i1_offset_negone(<vscale x 8 x i1> %a, <vsc
 ; CHECK-NEXT:    vmerge.vim v10, v9, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v10, v10, a0
-; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v8, v9, 1, v0
 ; CHECK-NEXT:    vslideup.vi v10, v8, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsplats-i1.ll b/llvm/test/CodeGen/RISCV/rvv/vsplats-i1.ll
index b069d2fceeb7a..7b8c2ae3e29bc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsplats-i1.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsplats-i1.ll
@@ -198,11 +198,9 @@ define <vscale x 4 x i1> @splat_idx_nxv4i32(<vscale x 4 x i1> %v, i64 %idx) {
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    andi a0, a0, 1
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vmv.v.x v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/shadowcallstack.ll b/llvm/test/CodeGen/RISCV/shadowcallstack.ll
index 15c09f01c8d4d..fee067ee3ad14 100644
--- a/llvm/test/CodeGen/RISCV/shadowcallstack.ll
+++ b/llvm/test/CodeGen/RISCV/shadowcallstack.ll
@@ -34,8 +34,8 @@ declare i32 @bar()
 define i32 @f3() shadowcallstack {
 ; RV32-LABEL: f3:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw ra, 0(gp)
 ; RV32-NEXT:    addi gp, gp, 4
+; RV32-NEXT:    sw ra, -4(gp)
 ; RV32-NEXT:    .cfi_escape 0x16, 0x03, 0x02, 0x73, 0x7c #
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
@@ -51,8 +51,8 @@ define i32 @f3() shadowcallstack {
 ;
 ; RV64-LABEL: f3:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sd ra, 0(gp)
 ; RV64-NEXT:    addi gp, gp, 8
+; RV64-NEXT:    sd ra, -8(gp)
 ; RV64-NEXT:    .cfi_escape 0x16, 0x03, 0x02, 0x73, 0x78 #
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
@@ -73,8 +73,8 @@ define i32 @f3() shadowcallstack {
 define i32 @f4() shadowcallstack {
 ; RV32-LABEL: f4:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw ra, 0(gp)
 ; RV32-NEXT:    addi gp, gp, 4
+; RV32-NEXT:    sw ra, -4(gp)
 ; RV32-NEXT:    .cfi_escape 0x16, 0x03, 0x02, 0x73, 0x7c #
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
@@ -108,8 +108,8 @@ define i32 @f4() shadowcallstack {
 ;
 ; RV64-LABEL: f4:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sd ra, 0(gp)
 ; RV64-NEXT:    addi gp, gp, 8
+; RV64-NEXT:    sd ra, -8(gp)
 ; RV64-NEXT:    .cfi_escape 0x16, 0x03, 0x02, 0x73, 0x78 #
 ; RV64-NEXT:    addi sp, sp, -32
 ; RV64-NEXT:    .cfi_def_cfa_offset 32
@@ -153,8 +153,8 @@ define i32 @f4() shadowcallstack {
 define i32 @f5() shadowcallstack nounwind {
 ; RV32-LABEL: f5:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    sw ra, 0(gp)
 ; RV32-NEXT:    addi gp, gp, 4
+; RV32-NEXT:    sw ra, -4(gp)
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    call bar@plt
@@ -166,8 +166,8 @@ define i32 @f5() shadowcallstack nounwind {
 ;
 ; RV64-LABEL: f5:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    sd ra, 0(gp)
 ; RV64-NEXT:    addi gp, gp, 8
+; RV64-NEXT:    sd ra, -8(gp)
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    call bar@plt
diff --git a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
index fc7f1f588369f..415136c30b8f4 100644
--- a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
@@ -597,60 +597,68 @@ define void @test_srem_vec(ptr %X) nounwind {
 ;
 ; RV32MV-LABEL: test_srem_vec:
 ; RV32MV:       # %bb.0:
-; RV32MV-NEXT:    addi sp, sp, -64
-; RV32MV-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    sw s2, 52(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    sw s3, 48(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    sw s4, 44(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    sw s5, 40(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    sw s6, 36(sp) # 4-byte Folded Spill
-; RV32MV-NEXT:    addi s0, sp, 64
-; RV32MV-NEXT:    andi sp, sp, -32
-; RV32MV-NEXT:    mv s2, a0
-; RV32MV-NEXT:    lw a0, 8(a0)
-; RV32MV-NEXT:    lw a1, 4(s2)
-; RV32MV-NEXT:    lbu a2, 12(s2)
-; RV32MV-NEXT:    slli a3, a0, 31
-; RV32MV-NEXT:    srli a4, a1, 1
-; RV32MV-NEXT:    or s3, a4, a3
-; RV32MV-NEXT:    slli a3, a2, 30
-; RV32MV-NEXT:    srli a4, a0, 2
-; RV32MV-NEXT:    or s4, a4, a3
-; RV32MV-NEXT:    srli a0, a0, 1
+; RV32MV-NEXT:    addi sp, sp, -48
+; RV32MV-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32MV-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32MV-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32MV-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32MV-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32MV-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32MV-NEXT:    csrr a1, vlenb
+; RV32MV-NEXT:    slli a1, a1, 1
+; RV32MV-NEXT:    sub sp, sp, a1
+; RV32MV-NEXT:    mv s0, a0
+; RV32MV-NEXT:    lbu a0, 12(a0)
+; RV32MV-NEXT:    lw a1, 8(s0)
+; RV32MV-NEXT:    slli a2, a0, 30
+; RV32MV-NEXT:    lw a3, 4(s0)
+; RV32MV-NEXT:    srli s1, a1, 2
+; RV32MV-NEXT:    or s1, s1, a2
+; RV32MV-NEXT:    slli a2, a1, 31
+; RV32MV-NEXT:    srli a4, a3, 1
+; RV32MV-NEXT:    or s2, a4, a2
+; RV32MV-NEXT:    srli a0, a0, 2
 ; RV32MV-NEXT:    slli a0, a0, 31
-; RV32MV-NEXT:    srai s5, a0, 31
-; RV32MV-NEXT:    srli a2, a2, 2
-; RV32MV-NEXT:    slli a2, a2, 31
-; RV32MV-NEXT:    lw a0, 0(s2)
-; RV32MV-NEXT:    srai s6, a2, 31
+; RV32MV-NEXT:    srai s3, a0, 31
+; RV32MV-NEXT:    srli a1, a1, 1
 ; RV32MV-NEXT:    slli a1, a1, 31
+; RV32MV-NEXT:    lw a0, 0(s0)
+; RV32MV-NEXT:    srai s4, a1, 31
+; RV32MV-NEXT:    slli a1, a3, 31
 ; RV32MV-NEXT:    srai a1, a1, 31
 ; RV32MV-NEXT:    li a2, 6
 ; RV32MV-NEXT:    li a3, 0
 ; RV32MV-NEXT:    call __moddi3@plt
-; RV32MV-NEXT:    sw a1, 4(sp)
-; RV32MV-NEXT:    sw a0, 0(sp)
-; RV32MV-NEXT:    li a2, -5
-; RV32MV-NEXT:    li a3, -1
-; RV32MV-NEXT:    mv a0, s4
-; RV32MV-NEXT:    mv a1, s6
-; RV32MV-NEXT:    call __moddi3@plt
-; RV32MV-NEXT:    sw a1, 20(sp)
-; RV32MV-NEXT:    sw a0, 16(sp)
+; RV32MV-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a0
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV32MV-NEXT:    addi a0, sp, 16
+; RV32MV-NEXT:    vs2r.v v8, (a0) # Unknown-size Folded Spill
 ; RV32MV-NEXT:    li a2, 7
-; RV32MV-NEXT:    mv a0, s3
-; RV32MV-NEXT:    mv a1, s5
+; RV32MV-NEXT:    mv a0, s2
+; RV32MV-NEXT:    mv a1, s4
 ; RV32MV-NEXT:    li a3, 0
 ; RV32MV-NEXT:    call __moddi3@plt
-; RV32MV-NEXT:    sw a1, 12(sp)
-; RV32MV-NEXT:    sw a0, 8(sp)
+; RV32MV-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32MV-NEXT:    addi a2, sp, 16
+; RV32MV-NEXT:    vl2r.v v8, (a2) # Unknown-size Folded Reload
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a0
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV32MV-NEXT:    addi a0, sp, 16
+; RV32MV-NEXT:    vs2r.v v8, (a0) # Unknown-size Folded Spill
+; RV32MV-NEXT:    li a2, -5
+; RV32MV-NEXT:    li a3, -1
+; RV32MV-NEXT:    mv a0, s1
+; RV32MV-NEXT:    mv a1, s3
+; RV32MV-NEXT:    call __moddi3@plt
+; RV32MV-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32MV-NEXT:    addi a2, sp, 16
+; RV32MV-NEXT:    vl2r.v v8, (a2) # Unknown-size Folded Reload
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a0
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV32MV-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32MV-NEXT:    li a0, 85
-; RV32MV-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; RV32MV-NEXT:    vmv.s.x v0, a0
-; RV32MV-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32MV-NEXT:    mv a0, sp
-; RV32MV-NEXT:    vle32.v v8, (a0)
 ; RV32MV-NEXT:    vmv.v.i v10, 1
 ; RV32MV-NEXT:    vmerge.vim v10, v10, -1, v0
 ; RV32MV-NEXT:    vand.vv v8, v8, v10
@@ -668,14 +676,14 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    vmv.v.i v8, 0
 ; RV32MV-NEXT:    vmerge.vim v8, v8, -1, v0
 ; RV32MV-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32MV-NEXT:    vse32.v v8, (s2)
+; RV32MV-NEXT:    vse32.v v8, (s0)
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 1
 ; RV32MV-NEXT:    vmv.x.s a0, v10
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 2
 ; RV32MV-NEXT:    vmv.x.s a1, v10
 ; RV32MV-NEXT:    slli a2, a1, 1
 ; RV32MV-NEXT:    sub a2, a2, a0
-; RV32MV-NEXT:    sw a2, 4(s2)
+; RV32MV-NEXT:    sw a2, 4(s0)
 ; RV32MV-NEXT:    vslidedown.vi v10, v8, 4
 ; RV32MV-NEXT:    vmv.x.s a0, v10
 ; RV32MV-NEXT:    srli a2, a0, 30
@@ -684,7 +692,7 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    slli a3, a3, 2
 ; RV32MV-NEXT:    or a2, a3, a2
 ; RV32MV-NEXT:    andi a2, a2, 7
-; RV32MV-NEXT:    sb a2, 12(s2)
+; RV32MV-NEXT:    sb a2, 12(s0)
 ; RV32MV-NEXT:    srli a1, a1, 31
 ; RV32MV-NEXT:    vslidedown.vi v8, v8, 3
 ; RV32MV-NEXT:    vmv.x.s a2, v8
@@ -693,70 +701,65 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    slli a0, a0, 2
 ; RV32MV-NEXT:    or a0, a1, a0
 ; RV32MV-NEXT:    or a0, a0, a2
-; RV32MV-NEXT:    sw a0, 8(s2)
-; RV32MV-NEXT:    addi sp, s0, -64
-; RV32MV-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    lw s2, 52(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    lw s3, 48(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    lw s4, 44(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    lw s5, 40(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    lw s6, 36(sp) # 4-byte Folded Reload
-; RV32MV-NEXT:    addi sp, sp, 64
+; RV32MV-NEXT:    sw a0, 8(s0)
+; RV32MV-NEXT:    csrr a0, vlenb
+; RV32MV-NEXT:    slli a0, a0, 1
+; RV32MV-NEXT:    add sp, sp, a0
+; RV32MV-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
+; RV32MV-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32MV-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32MV-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32MV-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32MV-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32MV-NEXT:    addi sp, sp, 48
 ; RV32MV-NEXT:    ret
 ;
 ; RV64MV-LABEL: test_srem_vec:
 ; RV64MV:       # %bb.0:
-; RV64MV-NEXT:    addi sp, sp, -64
-; RV64MV-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
-; RV64MV-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
-; RV64MV-NEXT:    addi s0, sp, 64
-; RV64MV-NEXT:    andi sp, sp, -32
-; RV64MV-NEXT:    lbu a1, 12(a0)
+; RV64MV-NEXT:    ld a1, 0(a0)
 ; RV64MV-NEXT:    lwu a2, 8(a0)
-; RV64MV-NEXT:    slli a1, a1, 32
-; RV64MV-NEXT:    ld a3, 0(a0)
-; RV64MV-NEXT:    or a1, a2, a1
-; RV64MV-NEXT:    slli a1, a1, 29
-; RV64MV-NEXT:    srai a1, a1, 31
-; RV64MV-NEXT:    srli a4, a3, 2
-; RV64MV-NEXT:    slli a2, a2, 62
-; RV64MV-NEXT:    lui a5, %hi(.LCPI3_0)
-; RV64MV-NEXT:    ld a5, %lo(.LCPI3_0)(a5)
-; RV64MV-NEXT:    or a2, a2, a4
-; RV64MV-NEXT:    slli a3, a3, 31
+; RV64MV-NEXT:    srli a3, a1, 2
+; RV64MV-NEXT:    lbu a4, 12(a0)
+; RV64MV-NEXT:    slli a5, a2, 62
+; RV64MV-NEXT:    or a3, a5, a3
 ; RV64MV-NEXT:    srai a3, a3, 31
-; RV64MV-NEXT:    mulh a4, a3, a5
+; RV64MV-NEXT:    slli a4, a4, 32
+; RV64MV-NEXT:    or a2, a2, a4
+; RV64MV-NEXT:    slli a2, a2, 29
+; RV64MV-NEXT:    lui a4, %hi(.LCPI3_0)
+; RV64MV-NEXT:    ld a4, %lo(.LCPI3_0)(a4)
+; RV64MV-NEXT:    srai a2, a2, 31
+; RV64MV-NEXT:    slli a1, a1, 31
+; RV64MV-NEXT:    srai a1, a1, 31
+; RV64MV-NEXT:    mulh a4, a2, a4
 ; RV64MV-NEXT:    srli a5, a4, 63
+; RV64MV-NEXT:    srai a4, a4, 1
 ; RV64MV-NEXT:    add a4, a4, a5
-; RV64MV-NEXT:    li a5, 6
-; RV64MV-NEXT:    mul a4, a4, a5
 ; RV64MV-NEXT:    lui a5, %hi(.LCPI3_1)
 ; RV64MV-NEXT:    ld a5, %lo(.LCPI3_1)(a5)
-; RV64MV-NEXT:    srai a2, a2, 31
-; RV64MV-NEXT:    sub a3, a3, a4
-; RV64MV-NEXT:    sd a3, 0(sp)
-; RV64MV-NEXT:    mulh a3, a2, a5
-; RV64MV-NEXT:    srli a4, a3, 63
-; RV64MV-NEXT:    srai a3, a3, 1
-; RV64MV-NEXT:    add a3, a3, a4
-; RV64MV-NEXT:    slli a4, a3, 3
+; RV64MV-NEXT:    add a2, a2, a4
+; RV64MV-NEXT:    slli a4, a4, 2
+; RV64MV-NEXT:    add a2, a2, a4
+; RV64MV-NEXT:    mulh a4, a3, a5
+; RV64MV-NEXT:    srli a5, a4, 63
+; RV64MV-NEXT:    srai a4, a4, 1
+; RV64MV-NEXT:    add a4, a4, a5
 ; RV64MV-NEXT:    lui a5, %hi(.LCPI3_2)
 ; RV64MV-NEXT:    ld a5, %lo(.LCPI3_2)(a5)
-; RV64MV-NEXT:    add a2, a2, a3
-; RV64MV-NEXT:    sub a2, a2, a4
-; RV64MV-NEXT:    sd a2, 8(sp)
-; RV64MV-NEXT:    mulh a2, a1, a5
-; RV64MV-NEXT:    srli a3, a2, 63
-; RV64MV-NEXT:    srai a2, a2, 1
-; RV64MV-NEXT:    add a2, a2, a3
-; RV64MV-NEXT:    slli a3, a2, 2
-; RV64MV-NEXT:    add a1, a1, a2
-; RV64MV-NEXT:    add a1, a1, a3
-; RV64MV-NEXT:    sd a1, 16(sp)
-; RV64MV-NEXT:    mv a1, sp
+; RV64MV-NEXT:    add a3, a3, a4
+; RV64MV-NEXT:    slli a4, a4, 3
+; RV64MV-NEXT:    sub a3, a3, a4
+; RV64MV-NEXT:    mulh a4, a1, a5
+; RV64MV-NEXT:    srli a5, a4, 63
+; RV64MV-NEXT:    add a4, a4, a5
+; RV64MV-NEXT:    li a5, 6
+; RV64MV-NEXT:    mul a4, a4, a5
+; RV64MV-NEXT:    sub a1, a1, a4
 ; RV64MV-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64MV-NEXT:    vle64.v v8, (a1)
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a3
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a2
+; RV64MV-NEXT:    vslidedown.vi v8, v8, 1
 ; RV64MV-NEXT:    lui a1, %hi(.LCPI3_3)
 ; RV64MV-NEXT:    addi a1, a1, %lo(.LCPI3_3)
 ; RV64MV-NEXT:    vle64.v v10, (a1)
@@ -784,10 +787,6 @@ define void @test_srem_vec(ptr %X) nounwind {
 ; RV64MV-NEXT:    srli a3, a3, 62
 ; RV64MV-NEXT:    or a2, a3, a2
 ; RV64MV-NEXT:    sw a2, 8(a0)
-; RV64MV-NEXT:    addi sp, s0, -64
-; RV64MV-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
-; RV64MV-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
-; RV64MV-NEXT:    addi sp, sp, 64
 ; RV64MV-NEXT:    ret
   %ld = load <3 x i33>, ptr %X
   %srem = srem <3 x i33> %ld, <i33 6, i33 7, i33 -5>
diff --git a/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll b/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
index c3cb8abd82019..dceae00d2e517 100644
--- a/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/RISCV/urem-seteq-illegal-types.ll
@@ -521,43 +521,41 @@ define void @test_urem_vec(ptr %X) nounwind {
 ;
 ; RV32MV-LABEL: test_urem_vec:
 ; RV32MV:       # %bb.0:
-; RV32MV-NEXT:    addi sp, sp, -16
 ; RV32MV-NEXT:    lw a1, 0(a0)
 ; RV32MV-NEXT:    andi a2, a1, 2047
-; RV32MV-NEXT:    sh a2, 8(sp)
-; RV32MV-NEXT:    slli a2, a1, 10
-; RV32MV-NEXT:    srli a2, a2, 21
-; RV32MV-NEXT:    sh a2, 10(sp)
+; RV32MV-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a2
 ; RV32MV-NEXT:    lbu a2, 4(a0)
+; RV32MV-NEXT:    slli a3, a1, 10
+; RV32MV-NEXT:    srli a3, a3, 21
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a3
 ; RV32MV-NEXT:    slli a2, a2, 10
 ; RV32MV-NEXT:    srli a1, a1, 22
 ; RV32MV-NEXT:    or a1, a1, a2
 ; RV32MV-NEXT:    andi a1, a1, 2047
-; RV32MV-NEXT:    sh a1, 12(sp)
-; RV32MV-NEXT:    addi a1, sp, 8
-; RV32MV-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV32MV-NEXT:    vle16.v v8, (a1)
-; RV32MV-NEXT:    vmv.v.i v9, 10
+; RV32MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV32MV-NEXT:    lui a1, %hi(.LCPI4_0)
+; RV32MV-NEXT:    addi a1, a1, %lo(.LCPI4_0)
+; RV32MV-NEXT:    vle16.v v9, (a1)
+; RV32MV-NEXT:    vslidedown.vi v8, v8, 1
+; RV32MV-NEXT:    vid.v v10
+; RV32MV-NEXT:    vsub.vv v8, v8, v10
+; RV32MV-NEXT:    vmul.vv v8, v8, v9
+; RV32MV-NEXT:    vadd.vv v9, v8, v8
+; RV32MV-NEXT:    vmv.v.i v10, 10
 ; RV32MV-NEXT:    li a1, 9
 ; RV32MV-NEXT:    vsetvli zero, zero, e16, mf2, tu, ma
-; RV32MV-NEXT:    vmv.s.x v9, a1
+; RV32MV-NEXT:    vmv.s.x v10, a1
 ; RV32MV-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV32MV-NEXT:    lui a1, %hi(.LCPI4_0)
-; RV32MV-NEXT:    addi a1, a1, %lo(.LCPI4_0)
-; RV32MV-NEXT:    vle16.v v10, (a1)
-; RV32MV-NEXT:    vid.v v11
-; RV32MV-NEXT:    vsub.vv v8, v8, v11
-; RV32MV-NEXT:    vmul.vv v8, v8, v10
-; RV32MV-NEXT:    vadd.vv v10, v8, v8
-; RV32MV-NEXT:    vsll.vv v9, v10, v9
+; RV32MV-NEXT:    vsll.vv v9, v9, v10
+; RV32MV-NEXT:    li a1, 2047
+; RV32MV-NEXT:    vand.vx v8, v8, a1
 ; RV32MV-NEXT:    vmv.v.i v10, 0
-; RV32MV-NEXT:    li a1, 1
+; RV32MV-NEXT:    li a2, 1
 ; RV32MV-NEXT:    vsetvli zero, zero, e16, mf2, tu, ma
 ; RV32MV-NEXT:    vmv1r.v v11, v10
-; RV32MV-NEXT:    vmv.s.x v11, a1
-; RV32MV-NEXT:    li a1, 2047
+; RV32MV-NEXT:    vmv.s.x v11, a2
 ; RV32MV-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV32MV-NEXT:    vand.vx v8, v8, a1
 ; RV32MV-NEXT:    lui a2, %hi(.LCPI4_1)
 ; RV32MV-NEXT:    addi a2, a2, %lo(.LCPI4_1)
 ; RV32MV-NEXT:    vle16.v v12, (a2)
@@ -566,7 +564,6 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    vand.vx v8, v8, a1
 ; RV32MV-NEXT:    vmsltu.vv v0, v12, v8
 ; RV32MV-NEXT:    vmerge.vim v8, v10, -1, v0
-; RV32MV-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV32MV-NEXT:    vslidedown.vi v9, v8, 2
 ; RV32MV-NEXT:    vmv.x.s a1, v9
 ; RV32MV-NEXT:    slli a2, a1, 21
@@ -582,47 +579,44 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV32MV-NEXT:    or a1, a2, a1
 ; RV32MV-NEXT:    or a1, a1, a3
 ; RV32MV-NEXT:    sw a1, 0(a0)
-; RV32MV-NEXT:    addi sp, sp, 16
 ; RV32MV-NEXT:    ret
 ;
 ; RV64MV-LABEL: test_urem_vec:
 ; RV64MV:       # %bb.0:
-; RV64MV-NEXT:    addi sp, sp, -16
 ; RV64MV-NEXT:    lbu a1, 4(a0)
 ; RV64MV-NEXT:    lwu a2, 0(a0)
 ; RV64MV-NEXT:    slli a1, a1, 32
 ; RV64MV-NEXT:    or a1, a2, a1
-; RV64MV-NEXT:    srli a2, a1, 22
-; RV64MV-NEXT:    sh a2, 12(sp)
 ; RV64MV-NEXT:    andi a2, a1, 2047
-; RV64MV-NEXT:    sh a2, 8(sp)
-; RV64MV-NEXT:    slli a1, a1, 42
-; RV64MV-NEXT:    srli a1, a1, 53
-; RV64MV-NEXT:    sh a1, 10(sp)
-; RV64MV-NEXT:    addi a1, sp, 8
 ; RV64MV-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64MV-NEXT:    vle16.v v8, (a1)
-; RV64MV-NEXT:    vmv.v.i v9, 10
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a2
+; RV64MV-NEXT:    slli a2, a1, 42
+; RV64MV-NEXT:    srli a2, a2, 53
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a2
+; RV64MV-NEXT:    srli a1, a1, 22
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV64MV-NEXT:    lui a1, %hi(.LCPI4_0)
+; RV64MV-NEXT:    addi a1, a1, %lo(.LCPI4_0)
+; RV64MV-NEXT:    vle16.v v9, (a1)
+; RV64MV-NEXT:    vslidedown.vi v8, v8, 1
+; RV64MV-NEXT:    vid.v v10
+; RV64MV-NEXT:    vsub.vv v8, v8, v10
+; RV64MV-NEXT:    vmul.vv v8, v8, v9
+; RV64MV-NEXT:    vadd.vv v9, v8, v8
+; RV64MV-NEXT:    vmv.v.i v10, 10
 ; RV64MV-NEXT:    li a1, 9
 ; RV64MV-NEXT:    vsetvli zero, zero, e16, mf2, tu, ma
-; RV64MV-NEXT:    vmv.s.x v9, a1
+; RV64MV-NEXT:    vmv.s.x v10, a1
 ; RV64MV-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64MV-NEXT:    lui a1, %hi(.LCPI4_0)
-; RV64MV-NEXT:    addi a1, a1, %lo(.LCPI4_0)
-; RV64MV-NEXT:    vle16.v v10, (a1)
-; RV64MV-NEXT:    vid.v v11
-; RV64MV-NEXT:    vsub.vv v8, v8, v11
-; RV64MV-NEXT:    vmul.vv v8, v8, v10
-; RV64MV-NEXT:    vadd.vv v10, v8, v8
-; RV64MV-NEXT:    vsll.vv v9, v10, v9
+; RV64MV-NEXT:    vsll.vv v9, v9, v10
+; RV64MV-NEXT:    li a1, 2047
+; RV64MV-NEXT:    vand.vx v8, v8, a1
 ; RV64MV-NEXT:    vmv.v.i v10, 0
-; RV64MV-NEXT:    li a1, 1
+; RV64MV-NEXT:    li a2, 1
 ; RV64MV-NEXT:    vsetvli zero, zero, e16, mf2, tu, ma
 ; RV64MV-NEXT:    vmv1r.v v11, v10
-; RV64MV-NEXT:    vmv.s.x v11, a1
-; RV64MV-NEXT:    li a1, 2047
+; RV64MV-NEXT:    vmv.s.x v11, a2
 ; RV64MV-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; RV64MV-NEXT:    vand.vx v8, v8, a1
 ; RV64MV-NEXT:    lui a2, %hi(.LCPI4_1)
 ; RV64MV-NEXT:    addi a2, a2, %lo(.LCPI4_1)
 ; RV64MV-NEXT:    vle16.v v12, (a2)
@@ -633,7 +627,6 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV64MV-NEXT:    vmerge.vim v8, v10, -1, v0
 ; RV64MV-NEXT:    vmv.x.s a1, v8
 ; RV64MV-NEXT:    andi a1, a1, 2047
-; RV64MV-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
 ; RV64MV-NEXT:    vslidedown.vi v9, v8, 1
 ; RV64MV-NEXT:    vmv.x.s a2, v9
 ; RV64MV-NEXT:    andi a2, a2, 2047
@@ -647,7 +640,6 @@ define void @test_urem_vec(ptr %X) nounwind {
 ; RV64MV-NEXT:    slli a1, a1, 31
 ; RV64MV-NEXT:    srli a1, a1, 63
 ; RV64MV-NEXT:    sb a1, 4(a0)
-; RV64MV-NEXT:    addi sp, sp, 16
 ; RV64MV-NEXT:    ret
   %ld = load <3 x i11>, ptr %X
   %urem = urem <3 x i11> %ld, <i11 6, i11 7, i11 -5>
diff --git a/llvm/test/CodeGen/SPARC/branches-relax.ll b/llvm/test/CodeGen/SPARC/branches-relax.ll
index 11324faf1146f..334a7ebfb6739 100644
--- a/llvm/test/CodeGen/SPARC/branches-relax.ll
+++ b/llvm/test/CodeGen/SPARC/branches-relax.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=sparc64 -sparc-bpcc-offset-bits=4 | FileCheck --check-prefix=SPARC64 %s
+; RUN: llc < %s -mtriple=sparc64 -sparc-bpcc-offset-bits=4 -sparc-bpr-offset-bits=4 | FileCheck --check-prefix=SPARC64 %s
 
 define i32 @branch_relax_int(i32 %in) {
 ; SPARC64-LABEL: branch_relax_int:
@@ -53,6 +53,57 @@ false:
   ret i32 0
 }
 
+define i64 @branch_relax_reg(i64 %in) {
+; SPARC64-LABEL: branch_relax_reg:
+; SPARC64:         .cfi_startproc
+; SPARC64-NEXT:  ! %bb.0:
+; SPARC64-NEXT:    save %sp, -128, %sp
+; SPARC64-NEXT:    .cfi_def_cfa_register %fp
+; SPARC64-NEXT:    .cfi_window_save
+; SPARC64-NEXT:    .cfi_register %o7, %i7
+; SPARC64-NEXT:    brnz %i0, .LBB1_1
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    ba .LBB1_2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:  .LBB1_1: ! %false
+; SPARC64-NEXT:    !APP
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    !NO_APP
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore %g0, %g0, %o0
+; SPARC64-NEXT:  .LBB1_2: ! %true
+; SPARC64-NEXT:    mov 4, %i0
+; SPARC64-NEXT:    !APP
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    !NO_APP
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %tst = icmp eq i64 %in, 0
+  br i1 %tst, label %true, label %false
+
+true:
+  call void asm sideeffect "nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop", ""()
+  ret i64 4
+
+false:
+  call void asm sideeffect "nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop\0A\09nop", ""()
+  ret i64 0
+}
+
 define float @branch_relax_float(float %in) {
 ; SPARC64-LABEL: branch_relax_float:
 ; SPARC64:         .cfi_startproc
@@ -61,20 +112,20 @@ define float @branch_relax_float(float %in) {
 ; SPARC64-NEXT:    .cfi_def_cfa_register %fp
 ; SPARC64-NEXT:    .cfi_window_save
 ; SPARC64-NEXT:    .cfi_register %o7, %i7
-; SPARC64-NEXT:    sethi %h44(.LCPI1_0), %i0
-; SPARC64-NEXT:    add %i0, %m44(.LCPI1_0), %i0
+; SPARC64-NEXT:    sethi %h44(.LCPI2_0), %i0
+; SPARC64-NEXT:    add %i0, %m44(.LCPI2_0), %i0
 ; SPARC64-NEXT:    sllx %i0, 12, %i0
-; SPARC64-NEXT:    ld [%i0+%l44(.LCPI1_0)], %f0
+; SPARC64-NEXT:    ld [%i0+%l44(.LCPI2_0)], %f0
 ; SPARC64-NEXT:    fcmps %fcc0, %f1, %f0
-; SPARC64-NEXT:    fbe %fcc0, .LBB1_1
+; SPARC64-NEXT:    fbe %fcc0, .LBB2_1
 ; SPARC64-NEXT:    nop
-; SPARC64-NEXT:    ba .LBB1_2
+; SPARC64-NEXT:    ba .LBB2_2
 ; SPARC64-NEXT:    nop
-; SPARC64-NEXT:  .LBB1_1: ! %true
-; SPARC64-NEXT:    sethi %h44(.LCPI1_1), %i0
-; SPARC64-NEXT:    add %i0, %m44(.LCPI1_1), %i0
+; SPARC64-NEXT:  .LBB2_1: ! %true
+; SPARC64-NEXT:    sethi %h44(.LCPI2_1), %i0
+; SPARC64-NEXT:    add %i0, %m44(.LCPI2_1), %i0
 ; SPARC64-NEXT:    sllx %i0, 12, %i0
-; SPARC64-NEXT:    ld [%i0+%l44(.LCPI1_1)], %f0
+; SPARC64-NEXT:    ld [%i0+%l44(.LCPI2_1)], %f0
 ; SPARC64-NEXT:    !APP
 ; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    nop
@@ -87,7 +138,7 @@ define float @branch_relax_float(float %in) {
 ; SPARC64-NEXT:    !NO_APP
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
-; SPARC64-NEXT:  .LBB1_2: ! %false
+; SPARC64-NEXT:  .LBB2_2: ! %false
 ; SPARC64-NEXT:    !APP
 ; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    nop
diff --git a/llvm/test/CodeGen/SPARC/branches-v9.ll b/llvm/test/CodeGen/SPARC/branches-v9.ll
index c1c765d6025c8..6b76888df065a 100644
--- a/llvm/test/CodeGen/SPARC/branches-v9.ll
+++ b/llvm/test/CodeGen/SPARC/branches-v9.ll
@@ -1,13 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=sparcv9 -disable-sparc-leaf-proc | FileCheck %s
 
-;; 1. When emitting code for v9, branches should always explicitly specify
+;; 1. When emitting code for v9, CCR branches should always explicitly specify
 ;;    %icc or %xcc.
-;; 2. There should never be a `ba` that jumps into two instructions immediately
+;; 2. Branches on the result of a 64-bit compare with constant zero should be
+;;    lowered into an instruction in the BPr class (§A.3 in V9 spec).
+;; 3. There should never be a `ba` that jumps into two instructions immediately
 ;;    following it.
 
-define void @i(i32 signext %sel) {
-; CHECK-LABEL: i:
+define void @bricc(i32 signext %sel) {
+; CHECK-LABEL: bricc:
 ; CHECK:         .cfi_startproc
 ; CHECK-NEXT:  ! %bb.0: ! %entry
 ; CHECK-NEXT:    save %sp, -176, %sp
@@ -47,26 +49,68 @@ end:
   ret void
 }
 
-define void @l(i64 %sel) {
-; CHECK-LABEL: l:
+define void @brxcc(i64 %sel) {
+; CHECK-LABEL: brxcc:
 ; CHECK:         .cfi_startproc
 ; CHECK-NEXT:  ! %bb.0: ! %entry
 ; CHECK-NEXT:    save %sp, -176, %sp
 ; CHECK-NEXT:    .cfi_def_cfa_register %fp
 ; CHECK-NEXT:    .cfi_window_save
 ; CHECK-NEXT:    .cfi_register %o7, %i7
-; CHECK-NEXT:    cmp %i0, 0
-; CHECK-NEXT:    be %xcc, .LBB1_2
+; CHECK-NEXT:    cmp %i0, 1
+; CHECK-NEXT:    bne %xcc, .LBB1_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  ! %bb.1: ! %tbb
+; CHECK-NEXT:    call f1
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ba .LBB1_3
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB1_2: ! %fbb
+; CHECK-NEXT:    call f2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  .LBB1_3: ! %end
+; CHECK-NEXT:    call f3
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ret
+; CHECK-NEXT:    restore
+entry:
+  ;; Using 1 here because compares with zero
+  ;; will be lowered into a `brz`, not `be`.
+  %cond = icmp eq i64 %sel, 1
+  br i1 %cond, label %tbb, label %fbb
+
+fbb:
+  call void @f2()
+  br label %end
+
+tbb:
+  call void @f1()
+  br label %end
+
+end:
+  call void @f3()
+  ret void
+}
+
+define void @brreg(i64 %sel) {
+; CHECK-LABEL: brreg:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  ! %bb.0: ! %entry
+; CHECK-NEXT:    save %sp, -176, %sp
+; CHECK-NEXT:    .cfi_def_cfa_register %fp
+; CHECK-NEXT:    .cfi_window_save
+; CHECK-NEXT:    .cfi_register %o7, %i7
+; CHECK-NEXT:    brz %i0, .LBB2_2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:  ! %bb.1: ! %fbb
 ; CHECK-NEXT:    call f2
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:    ba .LBB1_3
+; CHECK-NEXT:    ba .LBB2_3
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:  .LBB1_2: ! %tbb
+; CHECK-NEXT:  .LBB2_2: ! %tbb
 ; CHECK-NEXT:    call f1
 ; CHECK-NEXT:    nop
-; CHECK-NEXT:  .LBB1_3: ! %end
+; CHECK-NEXT:  .LBB2_3: ! %end
 ; CHECK-NEXT:    call f3
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/SPARC/missinglabel.ll b/llvm/test/CodeGen/SPARC/missinglabel.ll
index 88b09a8592634..e29f5ba344440 100644
--- a/llvm/test/CodeGen/SPARC/missinglabel.ll
+++ b/llvm/test/CodeGen/SPARC/missinglabel.ll
@@ -8,8 +8,7 @@ define void @f(i64 %a0) align 2 {
 ; CHECK-LABEL: f:
 ; CHECK:         .cfi_startproc
 ; CHECK-NEXT:  ! %bb.0: ! %entry
-; CHECK-NEXT:    cmp %o0, 0
-; CHECK-NEXT:    be %xcc, .LBB0_2
+; CHECK-NEXT:    brz %o0, .LBB0_2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:  ! %bb.1: ! %cond.false
 ; CHECK-NEXT:  .LBB0_2: ! %targetblock
diff --git a/llvm/test/CodeGen/Thumb/segmented-stacks-dynamic.ll b/llvm/test/CodeGen/Thumb/segmented-stacks-dynamic.ll
index 4f8c39f6496b5..ed554ea84aff5 100644
--- a/llvm/test/CodeGen/Thumb/segmented-stacks-dynamic.ll
+++ b/llvm/test/CodeGen/Thumb/segmented-stacks-dynamic.ll
@@ -27,7 +27,7 @@ false:
 ; Thumb-linux-NEXT: ldr r4, .LCPI0_0
 ; Thumb-linux-NEXT: ldr r4, [r4]
 ; Thumb-linux-NEXT: cmp	r4, r5
-; Thumb-linux-NEXT: blo	.LBB0_2
+; Thumb-linux-NEXT: bls	.LBB0_2
 
 ; Thumb-linux:      mov r4, #16
 ; Thumb-linux-NEXT: mov r5, #0
@@ -47,7 +47,7 @@ false:
 ; Thumb-android-NEXT: ldr r4, .LCPI0_0
 ; Thumb-android-NEXT: ldr r4, [r4]
 ; Thumb-android-NEXT: cmp	r4, r5
-; Thumb-android-NEXT: blo	.LBB0_2
+; Thumb-android-NEXT: bls	.LBB0_2
 
 ; Thumb-android:      mov r4, #16
 ; Thumb-android-NEXT: mov r5, #0
diff --git a/llvm/test/CodeGen/Thumb/segmented-stacks.ll b/llvm/test/CodeGen/Thumb/segmented-stacks.ll
index c3b66b3b91320..1118f4cc98e5b 100644
--- a/llvm/test/CodeGen/Thumb/segmented-stacks.ll
+++ b/llvm/test/CodeGen/Thumb/segmented-stacks.ll
@@ -19,7 +19,7 @@ define void @test_basic() #0 {
 ; Thumb-android-NEXT: ldr     r4, .LCPI0_0
 ; Thumb-android-NEXT: ldr     r4, [r4]
 ; Thumb-android-NEXT: cmp     r4, r5
-; Thumb-android-NEXT: blo     .LBB0_2
+; Thumb-android-NEXT: bls     .LBB0_2
 
 ; Thumb-android:      mov     r4, #48
 ; Thumb-android-NEXT: mov     r5, #0
@@ -43,7 +43,7 @@ define void @test_basic() #0 {
 ; Thumb-linux-NEXT: ldr     r4, .LCPI0_0
 ; Thumb-linux-NEXT: ldr     r4, [r4]
 ; Thumb-linux-NEXT: cmp     r4, r5
-; Thumb-linux-NEXT: blo     .LBB0_2
+; Thumb-linux-NEXT: bls     .LBB0_2
 
 ; Thumb-linux:      mov     r4, #48
 ; Thumb-linux-NEXT: mov     r5, #0
@@ -72,7 +72,7 @@ define i32 @test_nested(ptr nest %closure, i32 %other) #0 {
 ; Thumb-android-NEXT: ldr     r4, .LCPI1_0
 ; Thumb-android-NEXT: ldr     r4, [r4]
 ; Thumb-android-NEXT: cmp     r4, r5
-; Thumb-android-NEXT: blo     .LBB1_2
+; Thumb-android-NEXT: bls     .LBB1_2
 
 ; Thumb-android:      mov     r4, #56
 ; Thumb-android-NEXT: mov     r5, #0
@@ -92,7 +92,7 @@ define i32 @test_nested(ptr nest %closure, i32 %other) #0 {
 ; Thumb-linux-NEXT: ldr     r4, .LCPI1_0
 ; Thumb-linux-NEXT: ldr     r4, [r4]
 ; Thumb-linux-NEXT: cmp     r4, r5
-; Thumb-linux-NEXT: blo     .LBB1_2
+; Thumb-linux-NEXT: bls     .LBB1_2
 
 ; Thumb-linux:      mov     r4, #56
 ; Thumb-linux-NEXT: mov     r5, #0
@@ -121,7 +121,7 @@ define void @test_large() #0 {
 ; Thumb-android-NEXT: ldr     r4, .LCPI2_3
 ; Thumb-android-NEXT: ldr     r4, [r4]
 ; Thumb-android-NEXT: cmp     r4, r5
-; Thumb-android-NEXT: blo     .LBB2_2
+; Thumb-android-NEXT: bls     .LBB2_2
 
 ; Thumb-android:      ldr     r4, .LCPI2_2
 ; Thumb-android-NEXT: mov     r5, #0
@@ -146,7 +146,7 @@ define void @test_large() #0 {
 ; Thumb-linux-NEXT: ldr     r4, .LCPI2_3
 ; Thumb-linux-NEXT: ldr     r4, [r4]
 ; Thumb-linux-NEXT: cmp     r4, r5
-; Thumb-linux-NEXT: blo     .LBB2_2
+; Thumb-linux-NEXT: bls     .LBB2_2
 
 ; Thumb-linux:      ldr     r4, .LCPI2_2
 ; Thumb-linux-NEXT: mov     r5, #0
@@ -173,7 +173,7 @@ define fastcc void @test_fastcc() #0 {
 ; Thumb-android-NEXT: ldr     r4, .LCPI3_0
 ; Thumb-android-NEXT: ldr     r4, [r4]
 ; Thumb-android-NEXT: cmp     r4, r5
-; Thumb-android-NEXT: blo     .LBB3_2
+; Thumb-android-NEXT: bls     .LBB3_2
 
 ; Thumb-android:      mov     r4, #48
 ; Thumb-android-NEXT: mov     r5, #0
@@ -193,7 +193,7 @@ define fastcc void @test_fastcc() #0 {
 ; Thumb-linux-NEXT: ldr     r4, .LCPI3_0
 ; Thumb-linux-NEXT: ldr     r4, [r4]
 ; Thumb-linux-NEXT: cmp     r4, r5
-; Thumb-linux-NEXT: blo     .LBB3_2
+; Thumb-linux-NEXT: bls     .LBB3_2
 
 ; Thumb-linux:      mov     r4, #48
 ; Thumb-linux-NEXT: mov     r5, #0
@@ -222,7 +222,7 @@ define fastcc void @test_fastcc_large() #0 {
 ; Thumb-android-NEXT: ldr     r4, .LCPI4_3
 ; Thumb-android-NEXT: ldr     r4, [r4]
 ; Thumb-android-NEXT: cmp     r4, r5
-; Thumb-android-NEXT: blo     .LBB4_2
+; Thumb-android-NEXT: bls     .LBB4_2
 
 ; Thumb-android:      ldr     r4, .LCPI4_2
 ; Thumb-android-NEXT: mov     r5, #0
@@ -247,7 +247,7 @@ define fastcc void @test_fastcc_large() #0 {
 ; Thumb-linux-NEXT: ldr     r4, .LCPI4_3
 ; Thumb-linux-NEXT: ldr     r4, [r4]
 ; Thumb-linux-NEXT: cmp     r4, r5
-; Thumb-linux-NEXT: blo     .LBB4_2
+; Thumb-linux-NEXT: bls     .LBB4_2
 
 ; Thumb-linux:      ldr     r4, .LCPI4_2
 ; Thumb-linux-NEXT: mov     r5, #0
diff --git a/llvm/test/CodeGen/Thumb/thumb-shrink-wrapping.ll b/llvm/test/CodeGen/Thumb/thumb-shrink-wrapping.ll
index 07e19dd58fe5a..c52a54569b3dd 100644
--- a/llvm/test/CodeGen/Thumb/thumb-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/Thumb/thumb-shrink-wrapping.ll
@@ -1445,47 +1445,53 @@ if.end:
 define i1 @beq_to_bx(ptr %y, i32 %head) {
 ; ENABLE-V4T-LABEL: beq_to_bx:
 ; ENABLE-V4T:       @ %bb.0: @ %entry
-; ENABLE-V4T-NEXT:    push {r4, lr}
-; ENABLE-V4T-NEXT:    .cfi_def_cfa_offset 8
-; ENABLE-V4T-NEXT:    .cfi_offset lr, -4
-; ENABLE-V4T-NEXT:    .cfi_offset r4, -8
 ; ENABLE-V4T-NEXT:    movs r2, r0
 ; ENABLE-V4T-NEXT:    movs r0, #1
 ; ENABLE-V4T-NEXT:    cmp r2, #0
 ; ENABLE-V4T-NEXT:    beq LBB11_3
 ; ENABLE-V4T-NEXT:  @ %bb.1: @ %if.end
+; ENABLE-V4T-NEXT:    push {r4, lr}
+; ENABLE-V4T-NEXT:    .cfi_def_cfa_offset 8
+; ENABLE-V4T-NEXT:    .cfi_offset lr, -4
+; ENABLE-V4T-NEXT:    .cfi_offset r4, -8
 ; ENABLE-V4T-NEXT:    ldr r3, [r2]
 ; ENABLE-V4T-NEXT:    lsls r4, r3, #30
+; ENABLE-V4T-NEXT:    ldr r4, [sp, #4]
+; ENABLE-V4T-NEXT:    mov lr, r4
+; ENABLE-V4T-NEXT:    pop {r4}
+; ENABLE-V4T-NEXT:    add sp, #4
 ; ENABLE-V4T-NEXT:    bpl LBB11_3
 ; ENABLE-V4T-NEXT:  @ %bb.2: @ %if.end4
 ; ENABLE-V4T-NEXT:    str r1, [r2]
 ; ENABLE-V4T-NEXT:    str r3, [r2]
 ; ENABLE-V4T-NEXT:    movs r0, #0
 ; ENABLE-V4T-NEXT:  LBB11_3: @ %cleanup
-; ENABLE-V4T-NEXT:    pop {r4}
-; ENABLE-V4T-NEXT:    pop {r1}
-; ENABLE-V4T-NEXT:    bx r1
+; ENABLE-V4T-NEXT:    bx lr
 ;
 ; ENABLE-V5T-LABEL: beq_to_bx:
 ; ENABLE-V5T:       @ %bb.0: @ %entry
-; ENABLE-V5T-NEXT:    push {r4, lr}
-; ENABLE-V5T-NEXT:    .cfi_def_cfa_offset 8
-; ENABLE-V5T-NEXT:    .cfi_offset lr, -4
-; ENABLE-V5T-NEXT:    .cfi_offset r4, -8
 ; ENABLE-V5T-NEXT:    movs r2, r0
 ; ENABLE-V5T-NEXT:    movs r0, #1
 ; ENABLE-V5T-NEXT:    cmp r2, #0
 ; ENABLE-V5T-NEXT:    beq LBB11_3
 ; ENABLE-V5T-NEXT:  @ %bb.1: @ %if.end
+; ENABLE-V5T-NEXT:    push {r4, lr}
+; ENABLE-V5T-NEXT:    .cfi_def_cfa_offset 8
+; ENABLE-V5T-NEXT:    .cfi_offset lr, -4
+; ENABLE-V5T-NEXT:    .cfi_offset r4, -8
 ; ENABLE-V5T-NEXT:    ldr r3, [r2]
 ; ENABLE-V5T-NEXT:    lsls r4, r3, #30
+; ENABLE-V5T-NEXT:    ldr r4, [sp, #4]
+; ENABLE-V5T-NEXT:    mov lr, r4
+; ENABLE-V5T-NEXT:    pop {r4}
+; ENABLE-V5T-NEXT:    add sp, #4
 ; ENABLE-V5T-NEXT:    bpl LBB11_3
 ; ENABLE-V5T-NEXT:  @ %bb.2: @ %if.end4
 ; ENABLE-V5T-NEXT:    str r1, [r2]
 ; ENABLE-V5T-NEXT:    str r3, [r2]
 ; ENABLE-V5T-NEXT:    movs r0, #0
 ; ENABLE-V5T-NEXT:  LBB11_3: @ %cleanup
-; ENABLE-V5T-NEXT:    pop {r4, pc}
+; ENABLE-V5T-NEXT:    bx lr
 ;
 ; DISABLE-V4T-LABEL: beq_to_bx:
 ; DISABLE-V4T:       @ %bb.0: @ %entry
diff --git a/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll b/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll
index 172f3aa0e32d6..3ca01cfa3a8f7 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fptosi-sat-vector.ll
@@ -355,148 +355,126 @@ define arm_aapcs_vfpcc <2 x i32> @test_signed_v2f64_v2i32(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI9_0
-; CHECK-NEXT:    vmov r9, r8, d9
-; CHECK-NEXT:    vmov r11, r10, d0
-; CHECK-NEXT:    str.w r11, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    str.w r10, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI9_1
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    vmov r5, r3, d0
-; CHECK-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    str r5, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    vmov r8, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vldr d0, .LCPI9_1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r6, r5, d0
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r0, #-2147483648
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r0, #-2147483648
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne.w r11, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r11, #-2147483648
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r11, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov r10, r7, d8
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #1
+; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r5, #-1
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-2147483648
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-1
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r11, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r10, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI9_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI9_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI9_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <2 x i32> @llvm.fptosi.sat.v2f64.v2i32(<2 x double> %f)
     ret <2 x i32> %x
 }
@@ -2768,148 +2746,126 @@ define arm_aapcs_vfpcc <2 x i8> @test_signed_v2f64_v2i8(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI33_0
-; CHECK-NEXT:    vmov r9, r8, d9
-; CHECK-NEXT:    vmov r11, r10, d0
-; CHECK-NEXT:    str.w r11, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    str.w r10, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI33_1
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    vmov r5, r3, d0
-; CHECK-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    str r5, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    vmov r8, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vldr d0, .LCPI33_1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r6, r5, d0
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    mvneq r0, #127
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #127
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    mvnne r11, #127
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    mvneq r11, #127
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r11, #127
 ; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov r10, r7, d8
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #1
+; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r5, #-1
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r5, #127
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #127
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-1
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r11, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r10, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI33_0:
-; CHECK-NEXT:    .long 0 @ double 127
-; CHECK-NEXT:    .long 1080016896
-; CHECK-NEXT:  .LCPI33_1:
 ; CHECK-NEXT:    .long 0 @ double -128
 ; CHECK-NEXT:    .long 3227516928
+; CHECK-NEXT:  .LCPI33_1:
+; CHECK-NEXT:    .long 0 @ double 127
+; CHECK-NEXT:    .long 1080016896
     %x = call <2 x i8> @llvm.fptosi.sat.v2f64.v2i8(<2 x double> %f)
     ret <2 x i8> %x
 }
@@ -2923,147 +2879,129 @@ define arm_aapcs_vfpcc <2 x i13> @test_signed_v2f64_v2i13(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI34_0
-; CHECK-NEXT:    vmov r10, r11, d9
-; CHECK-NEXT:    vmov r8, r5, d0
-; CHECK-NEXT:    str.w r8, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    str r5, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI34_1
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    vmov r8, r7, d9
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    strd r3, r2, [sp, #16] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vldr d0, .LCPI34_1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r6, r5, d0
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r1, #-1
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    mov r11, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r5, r2, [sp, #16] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r8, #-1
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #0
-; CHECK-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov r10, r7, d8
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    movweq r9, #61440
-; CHECK-NEXT:    movteq r9, #65535
-; CHECK-NEXT:    ldrd r2, r3, [sp, #24] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #1
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    str r4, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    movweq r0, #61440
-; CHECK-NEXT:    movteq r0, #65535
-; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r0, #4095
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #24] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r9, #4095
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r5, #61440
+; CHECK-NEXT:    movtne r5, #65535
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r11, #61440
+; CHECK-NEXT:    movtne r11, #65535
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r11, #4095
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r11, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r5, #4095
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r5
-; CHECK-NEXT:    vmov q0[3], q0[1], r8, r0
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r0
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI34_0:
-; CHECK-NEXT:    .long 0 @ double 4095
-; CHECK-NEXT:    .long 1085275648
-; CHECK-NEXT:  .LCPI34_1:
 ; CHECK-NEXT:    .long 0 @ double -4096
 ; CHECK-NEXT:    .long 3232759808
+; CHECK-NEXT:  .LCPI34_1:
+; CHECK-NEXT:    .long 0 @ double 4095
+; CHECK-NEXT:    .long 1085275648
     %x = call <2 x i13> @llvm.fptosi.sat.v2f64.v2i13(<2 x double> %f)
     ret <2 x i13> %x
 }
@@ -3077,147 +3015,129 @@ define arm_aapcs_vfpcc <2 x i16> @test_signed_v2f64_v2i16(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI35_0
-; CHECK-NEXT:    vmov r10, r11, d9
-; CHECK-NEXT:    vmov r8, r5, d0
-; CHECK-NEXT:    str.w r8, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    str r5, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI35_1
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    vmov r8, r7, d9
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    strd r3, r2, [sp, #16] @ 8-byte Folded Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vldr d0, .LCPI35_1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r6, r5, d0
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r1, #-1
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    mov r11, r0
 ; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldrd r5, r2, [sp, #16] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r8, r1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r8, #-1
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r8, #0
-; CHECK-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov r10, r7, d8
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    movweq r9, #32768
-; CHECK-NEXT:    movteq r9, #65535
-; CHECK-NEXT:    ldrd r2, r3, [sp, #24] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #1
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    str r4, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    movweq r0, #32768
-; CHECK-NEXT:    movteq r0, #65535
-; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r0, #32767
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    mov r0, r10
-; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #24] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r9, #32767
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r5, #32768
+; CHECK-NEXT:    movtne r5, #65535
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r11, #32768
+; CHECK-NEXT:    movtne r11, #65535
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r11, #32767
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r11, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r5, #32767
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r5
-; CHECK-NEXT:    vmov q0[3], q0[1], r8, r0
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r0
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI35_0:
-; CHECK-NEXT:    .long 0 @ double 32767
-; CHECK-NEXT:    .long 1088421824
-; CHECK-NEXT:  .LCPI35_1:
 ; CHECK-NEXT:    .long 0 @ double -32768
 ; CHECK-NEXT:    .long 3235905536
+; CHECK-NEXT:  .LCPI35_1:
+; CHECK-NEXT:    .long 0 @ double 32767
+; CHECK-NEXT:    .long 1088421824
     %x = call <2 x i16> @llvm.fptosi.sat.v2f64.v2i16(<2 x double> %f)
     ret <2 x i16> %x
 }
@@ -3231,163 +3151,129 @@ define arm_aapcs_vfpcc <2 x i19> @test_signed_v2f64_v2i19(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #48
-; CHECK-NEXT:    sub sp, #48
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI36_0
-; CHECK-NEXT:    vmov r6, r5, d9
-; CHECK-NEXT:    vmov r8, r3, d0
-; CHECK-NEXT:    str r3, [sp, #36] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    str.w r8, [sp, #44] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI36_1
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    vmov r7, r6, d9
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    strd r2, r3, [sp, #16] @ 8-byte Folded Spill
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    str r0, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r1, #-1
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #0
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r1
-; CHECK-NEXT:    str r5, [sp, #40] @ 4-byte Spill
-; CHECK-NEXT:    vmov r7, r9, d8
-; CHECK-NEXT:    mov r5, r6
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #32] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    ldr r4, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r6, r9
-; CHECK-NEXT:    str.w r9, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r1, r9
-; CHECK-NEXT:    ldr.w r9, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r8, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r6, r5
-; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vldr d0, .LCPI36_1
+; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    vmov r8, r0, d8
+; CHECK-NEXT:    vmov r11, r10, d0
+; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    clz r0, r4
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    ittt ne
+; CHECK-NEXT:    movwne r9, #0
+; CHECK-NEXT:    movtne r9, #65532
+; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    movweq r11, #0
-; CHECK-NEXT:    movteq r11, #65532
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r11, #65535
-; CHECK-NEXT:    movtne r11, #3
-; CHECK-NEXT:    str r5, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    ldr r5, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    ldr r2, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    movwne r9, #65535
+; CHECK-NEXT:    movtne r9, #3
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r4, [sp] @ 4-byte Reload
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    mov r6, r8
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r4, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    moveq r4, #0
-; CHECK-NEXT:    movteq r4, #65532
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r11, r4
+; CHECK-NEXT:    lsr.w r10, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    movtne r4, #65532
+; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    movwne r4, #65535
 ; CHECK-NEXT:    movtne r4, #3
-; CHECK-NEXT:    ldr r5, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    ldr r2, [sp, #44] @ 4-byte Reload
-; CHECK-NEXT:    ldr r3, [sp, #36] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-1
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    cmp.w r10, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r7, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    ldr r1, [sp, #40] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r0
-; CHECK-NEXT:    mov r3, r1
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r9, #0
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r11, r4
-; CHECK-NEXT:    vmov q0[3], q0[1], r10, r0
-; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r4, r9
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r0
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI36_0:
-; CHECK-NEXT:    .long 0 @ double 262143
-; CHECK-NEXT:    .long 1091567608
-; CHECK-NEXT:  .LCPI36_1:
 ; CHECK-NEXT:    .long 0 @ double -262144
 ; CHECK-NEXT:    .long 3239051264
+; CHECK-NEXT:  .LCPI36_1:
+; CHECK-NEXT:    .long 0 @ double 262143
+; CHECK-NEXT:    .long 1091567608
     %x = call <2 x i19> @llvm.fptosi.sat.v2f64.v2i19(<2 x double> %f)
     ret <2 x i19> %x
 }
@@ -3401,148 +3287,126 @@ define arm_aapcs_vfpcc <2 x i32> @test_signed_v2f64_v2i32_duplicate(<2 x double>
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI37_0
-; CHECK-NEXT:    vmov r9, r8, d9
-; CHECK-NEXT:    vmov r11, r10, d0
-; CHECK-NEXT:    str.w r11, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    str.w r10, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI37_1
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    vmov r5, r3, d0
-; CHECK-NEXT:    str r3, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    str r5, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    vmov r8, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vldr d0, .LCPI37_1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    vmov r6, r5, d0
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
 ; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r0, #-2147483648
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r0, #-2147483648
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne.w r11, #-2147483648
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    ldr r5, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r11, #-2147483648
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r11, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov r10, r7, d8
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #1
+; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r5, #-1
-; CHECK-NEXT:    mov r0, r9
-; CHECK-NEXT:    mov r1, r8
-; CHECK-NEXT:    mov r2, r9
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-2147483648
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r5, #-2147483648
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #20] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    ldr r3, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r10, #-1
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #0
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r11, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r10, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI37_0:
-; CHECK-NEXT:    .long 4290772992 @ double 2147483647
-; CHECK-NEXT:    .long 1105199103
-; CHECK-NEXT:  .LCPI37_1:
 ; CHECK-NEXT:    .long 0 @ double -2147483648
 ; CHECK-NEXT:    .long 3252682752
+; CHECK-NEXT:  .LCPI37_1:
+; CHECK-NEXT:    .long 4290772992 @ double 2147483647
+; CHECK-NEXT:    .long 1105199103
     %x = call <2 x i32> @llvm.fptosi.sat.v2f64.v2i32(<2 x double> %f)
     ret <2 x i32> %x
 }
@@ -3556,160 +3420,130 @@ define arm_aapcs_vfpcc <2 x i50> @test_signed_v2f64_v2i50(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI38_0
-; CHECK-NEXT:    vmov r8, r5, d9
+; CHECK-NEXT:    vmov r7, r6, d9
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    str r2, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    str r3, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI38_1
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    str r1, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    csel r0, r0, r11, ne
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    vldr d0, .LCPI38_1
+; CHECK-NEXT:    mov r9, r0
+; CHECK-NEXT:    vmov r8, r0, d8
+; CHECK-NEXT:    vmov r11, r10, d0
+; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    clz r0, r4
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    movtne r1, #65534
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r11
+; CHECK-NEXT:    mov r3, r10
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    vmov r6, r7, d8
-; CHECK-NEXT:    mov r11, r5
-; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    mov r2, r7
+; CHECK-NEXT:    mov r3, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    ldr r5, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    ldr r4, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    movweq r10, #0
-; CHECK-NEXT:    movteq r10, #65534
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r2, r5
-; CHECK-NEXT:    mov r3, r4
+; CHECK-NEXT:    movne r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r10, #65535
-; CHECK-NEXT:    movtne r10, #1
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r9, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r11
-; CHECK-NEXT:    mov r5, r11
-; CHECK-NEXT:    ldr.w r11, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    movwne r5, #65535
+; CHECK-NEXT:    movtne r5, #1
+; CHECK-NEXT:    str r5, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r9, #0
+; CHECK-NEXT:    ldr r4, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    moveq r0, #0
-; CHECK-NEXT:    movteq r0, #65534
-; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r0, #65535
-; CHECK-NEXT:    movtne r0, #1
-; CHECK-NEXT:    ldr r2, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    str r0, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r1, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r7
-; CHECK-NEXT:    csel r4, r1, r0, ne
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    movne r5, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r11, r4
+; CHECK-NEXT:    lsr.w r10, r0, #5
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    cmp.w r10, #0
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    movtne r7, #65534
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r7, #65535
+; CHECK-NEXT:    movtne r7, #1
+; CHECK-NEXT:    cmp.w r10, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r11
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    cmp r5, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r5, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r6
-; CHECK-NEXT:    mov r1, r7
-; CHECK-NEXT:    mov r2, r6
-; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r4, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r10, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r4, r9
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r7, r1
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI38_0:
-; CHECK-NEXT:    .long 4294967280 @ double 562949953421311
-; CHECK-NEXT:    .long 1124073471
-; CHECK-NEXT:  .LCPI38_1:
 ; CHECK-NEXT:    .long 0 @ double -562949953421312
 ; CHECK-NEXT:    .long 3271557120
+; CHECK-NEXT:  .LCPI38_1:
+; CHECK-NEXT:    .long 4294967280 @ double 562949953421311
+; CHECK-NEXT:    .long 1124073471
     %x = call <2 x i50> @llvm.fptosi.sat.v2f64.v2i50(<2 x double> %f)
     ret <2 x i50> %x
 }
@@ -3723,145 +3557,126 @@ define arm_aapcs_vfpcc <2 x i64> @test_signed_v2f64_v2i64(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI39_0
-; CHECK-NEXT:    vmov r8, r5, d9
-; CHECK-NEXT:    vmov r11, r3, d0
-; CHECK-NEXT:    str r3, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    str.w r11, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI39_1
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    vmov r8, r7, d9
 ; CHECK-NEXT:    vmov r2, r3, d0
-; CHECK-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r10, r0
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vldr d0, .LCPI39_1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsrs r4, r0, #5
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    vmov r6, r5, d0
+; CHECK-NEXT:    str r4, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    cmp.w r10, #0
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    csel r0, r0, r10, ne
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r11, r0
+; CHECK-NEXT:    str r1, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r6
 ; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    bl __aeabi_dcmpun
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #0
-; CHECK-NEXT:    ldr r3, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    str r4, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    movne.w r11, #0
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r4, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    ldr.w r10, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r11, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2lz
-; CHECK-NEXT:    cmp.w r11, #0
-; CHECK-NEXT:    mov r9, r1
-; CHECK-NEXT:    csel r11, r0, r11, ne
-; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r6
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r8
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r11, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    vmov r10, r7, d8
+; CHECK-NEXT:    mov r8, r0
 ; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r2, r6
+; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r8, #1
+; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r11, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #24] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r0, #-2147483648
-; CHECK-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r8
-; CHECK-NEXT:    mov r3, r5
-; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r0, #-2147483648
-; CHECK-NEXT:    mov r10, r0
-; CHECK-NEXT:    mov r0, r8
-; CHECK-NEXT:    mov r1, r5
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    bl __aeabi_d2lz
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r0, r10
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpun
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r10, #0
-; CHECK-NEXT:    ldrd r2, r3, [sp, #24] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r3, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    mov r5, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r4
-; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq.w r9, #-2147483648
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r7
-; CHECK-NEXT:    mov r3, r6
-; CHECK-NEXT:    cmp r5, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    mvnne r9, #-2147483648
-; CHECK-NEXT:    bl __aeabi_dcmpun
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r5, r11
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r2, #-2147483648
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r2, #-2147483648
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r2, #0
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r4, #-2147483648
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r4, #-2147483648
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #0
-; CHECK-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r11, r0
-; CHECK-NEXT:    vmov q0[3], q0[1], r9, r10
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    movne r4, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r4, r2
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI39_0:
-; CHECK-NEXT:    .long 4294967295 @ double 9.2233720368547748E+18
-; CHECK-NEXT:    .long 1138753535
-; CHECK-NEXT:  .LCPI39_1:
 ; CHECK-NEXT:    .long 0 @ double -9.2233720368547758E+18
 ; CHECK-NEXT:    .long 3286237184
+; CHECK-NEXT:  .LCPI39_1:
+; CHECK-NEXT:    .long 4294967295 @ double 9.2233720368547748E+18
+; CHECK-NEXT:    .long 1138753535
     %x = call <2 x i64> @llvm.fptosi.sat.v2f64.v2i64(<2 x double> %f)
     ret <2 x i64> %x
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll b/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll
index d7a5b63261e79..8ea12bd1fc0de 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fptoui-sat-vector.ll
@@ -328,114 +328,95 @@ define arm_aapcs_vfpcc <2 x i32> @test_unsigned_v2f64_v2i32(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI9_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI9_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vldr d0, .LCPI9_1
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r7, #1
 ; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r8, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI9_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI9_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI9_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <2 x i32> @llvm.fptoui.sat.v2f64.v2i32(<2 x double> %f)
     ret <2 x i32> %x
 }
@@ -2224,114 +2205,95 @@ define arm_aapcs_vfpcc <2 x i8> @test_unsigned_v2f64_v2i8(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI33_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI33_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vldr d0, .LCPI33_1
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #255
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #255
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #255
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r7, #1
 ; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r8, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #255
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI33_0:
-; CHECK-NEXT:    .long 0 @ double 255
-; CHECK-NEXT:    .long 1081073664
-; CHECK-NEXT:  .LCPI33_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI33_1:
+; CHECK-NEXT:    .long 0 @ double 255
+; CHECK-NEXT:    .long 1081073664
     %x = call <2 x i8> @llvm.fptoui.sat.v2f64.v2i8(<2 x double> %f)
     ret <2 x i8> %x
 }
@@ -2345,114 +2307,95 @@ define arm_aapcs_vfpcc <2 x i13> @test_unsigned_v2f64_v2i13(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI34_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI34_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vldr d0, .LCPI34_1
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r0, #8191
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r9, #8191
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r5, #8191
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r7, #1
 ; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r8, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r0, #8191
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI34_0:
-; CHECK-NEXT:    .long 0 @ double 8191
-; CHECK-NEXT:    .long 1086324480
-; CHECK-NEXT:  .LCPI34_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI34_1:
+; CHECK-NEXT:    .long 0 @ double 8191
+; CHECK-NEXT:    .long 1086324480
     %x = call <2 x i13> @llvm.fptoui.sat.v2f64.v2i13(<2 x double> %f)
     ret <2 x i13> %x
 }
@@ -2466,114 +2409,95 @@ define arm_aapcs_vfpcc <2 x i16> @test_unsigned_v2f64_v2i16(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI35_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI35_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vldr d0, .LCPI35_1
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r0, #65535
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movwne r9, #65535
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r5, #65535
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    movne r7, #1
 ; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r8, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movwne r0, #65535
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI35_0:
-; CHECK-NEXT:    .long 0 @ double 65535
-; CHECK-NEXT:    .long 1089470432
-; CHECK-NEXT:  .LCPI35_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI35_1:
+; CHECK-NEXT:    .long 0 @ double 65535
+; CHECK-NEXT:    .long 1089470432
     %x = call <2 x i16> @llvm.fptoui.sat.v2f64.v2i16(<2 x double> %f)
     ret <2 x i16> %x
 }
@@ -2587,105 +2511,84 @@ define arm_aapcs_vfpcc <2 x i19> @test_unsigned_v2f64_v2i19(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI36_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    vmov r11, r10, d8
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    vldr d0, .LCPI36_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    vmov r5, r7, d0
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    vmov r8, r6, d9
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #1
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r0, #65535
-; CHECK-NEXT:    movtne r0, #7
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    clz r0, r0
 ; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    clz r0, r0
 ; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_d2ulz
+; CHECK-NEXT:    mov r5, r0
 ; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r9, #65535
-; CHECK-NEXT:    movtne r9, #7
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r7, r1
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r5, #65535
+; CHECK-NEXT:    movtne r5, #7
+; CHECK-NEXT:    bl __aeabi_d2ulz
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r0, #65535
+; CHECK-NEXT:    movtne r0, #7
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2710,114 +2613,95 @@ define arm_aapcs_vfpcc <2 x i32> @test_unsigned_v2f64_v2i32_duplicate(<2 x doubl
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI37_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI37_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vldr d0, .LCPI37_1
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #-1
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r5, #0
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    bl __aeabi_dcmpgt
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r7, #1
+; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r8, r4
+; CHECK-NEXT:    bl __aeabi_dcmpge
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI37_0:
-; CHECK-NEXT:    .long 4292870144 @ double 4294967295
-; CHECK-NEXT:    .long 1106247679
-; CHECK-NEXT:  .LCPI37_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI37_1:
+; CHECK-NEXT:    .long 4292870144 @ double 4294967295
+; CHECK-NEXT:    .long 1106247679
     %x = call <2 x i32> @llvm.fptoui.sat.v2f64.v2i32(<2 x double> %f)
     ret <2 x i32> %x
 }
@@ -2831,105 +2715,84 @@ define arm_aapcs_vfpcc <2 x i50> @test_unsigned_v2f64_v2i50(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI38_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    vmov r11, r10, d8
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    strd r2, r3, [sp, #8] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    vldr d0, .LCPI38_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    vmov r5, r7, d0
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    vmov r8, r6, d9
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r4, #1
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r7
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    csel r0, r1, r8, ne
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r0, #65535
-; CHECK-NEXT:    movtne r0, #3
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    clz r0, r0
 ; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    mov r2, r5
+; CHECK-NEXT:    mov r3, r7
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    clz r0, r0
 ; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    lsr.w r9, r0, #5
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    str r0, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    mov r7, r1
 ; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    csel r9, r1, r9, ne
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    movwne r9, #65535
-; CHECK-NEXT:    movtne r9, #3
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
-; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
+; CHECK-NEXT:    movne r7, #0
+; CHECK-NEXT:    ldrd r2, r3, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    mov r1, r6
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r10
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r7, #65535
+; CHECK-NEXT:    movtne r7, #3
+; CHECK-NEXT:    bl __aeabi_d2ulz
+; CHECK-NEXT:    ldr r2, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    movwne r1, #65535
+; CHECK-NEXT:    movtne r1, #3
+; CHECK-NEXT:    cmp.w r9, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #0
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r0, #-1
 ; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[3], q0[1], r9, r0
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r7
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -2954,114 +2817,95 @@ define arm_aapcs_vfpcc <2 x i64> @test_unsigned_v2f64_v2i64(<2 x double> %f) {
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    vldr d0, .LCPI39_0
-; CHECK-NEXT:    vmov r5, r4, d9
-; CHECK-NEXT:    vmov r10, r9, d0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    vldr d0, .LCPI39_1
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    vmov r2, r11, d0
-; CHECK-NEXT:    str r2, [sp, #28] @ 4-byte Spill
-; CHECK-NEXT:    str.w r11, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r11
+; CHECK-NEXT:    vmov r6, r7, d9
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    strd r3, r2, [sp, #4] @ 8-byte Folded Spill
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
+; CHECK-NEXT:    mov r1, r7
 ; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    vmov r7, r6, d8
-; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    cmp.w r8, #0
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    csel r0, r0, r8, ne
-; CHECK-NEXT:    mov r2, r10
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    vldr d0, .LCPI39_1
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r10, r1
+; CHECK-NEXT:    vmov r9, r8, d0
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    vmov r11, r4, d8
+; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    mov r0, r6
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    mov r3, r9
-; CHECK-NEXT:    str.w r10, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    mov r8, r9
-; CHECK-NEXT:    str.w r9, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    movne r5, #0
+; CHECK-NEXT:    mov r2, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr r2, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r3, r11
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    mov r9, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_d2ulz
-; CHECK-NEXT:    cmp.w r9, #0
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT:    csel r9, r0, r9, ne
-; CHECK-NEXT:    ldr r0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r10
+; CHECK-NEXT:    mov r6, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r2, r9
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r9, #-1
+; CHECK-NEXT:    movne r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r5, #-1
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    ldr.w r11, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    ldr.w r10, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r5
-; CHECK-NEXT:    mov r1, r4
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
-; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    mov r7, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r5, r1, r0, ne
-; CHECK-NEXT:    cmp.w r8, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
+; CHECK-NEXT:    movne r7, #1
 ; CHECK-NEXT:    ldrd r3, r2, [sp, #4] @ 8-byte Folded Reload
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    mov r0, r7
-; CHECK-NEXT:    mov r1, r6
-; CHECK-NEXT:    mov r2, r11
-; CHECK-NEXT:    mov r3, r10
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    mov r1, r4
+; CHECK-NEXT:    mov r8, r4
 ; CHECK-NEXT:    bl __aeabi_dcmpge
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r0, ne
+; CHECK-NEXT:    clz r0, r0
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    lsrs r4, r0, #5
+; CHECK-NEXT:    mov r0, r11
+; CHECK-NEXT:    bl __aeabi_d2ulz
 ; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r0, #-1
-; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    vmov q0[2], q0[0], r9, r1
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r5
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #0
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r10, #-1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r1, #0
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r10
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 ; CHECK-NEXT:    .p2align 3
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI39_0:
-; CHECK-NEXT:    .long 4294967295 @ double 1.844674407370955E+19
-; CHECK-NEXT:    .long 1139802111
-; CHECK-NEXT:  .LCPI39_1:
 ; CHECK-NEXT:    .long 0 @ double 0
 ; CHECK-NEXT:    .long 0
+; CHECK-NEXT:  .LCPI39_1:
+; CHECK-NEXT:    .long 4294967295 @ double 1.844674407370955E+19
+; CHECK-NEXT:    .long 1139802111
     %x = call <2 x i64> @llvm.fptoui.sat.v2f64.v2i64(<2 x double> %f)
     ret <2 x i64> %x
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
index 04d0b46dea67b..a270fee96a5a0 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving.ll
@@ -443,24 +443,23 @@ define arm_aapcs_vfpcc <8 x i16> @ext_ops_trunc_i16(<8 x i16> %a, <8 x i16> %b)
 ; CHECK-NEXT:    vshl.s32 q4, q4, q5
 ; CHECK-NEXT:    vneg.s32 q5, q3
 ; CHECK-NEXT:    vsub.i32 q4, q4, q2
-; CHECK-NEXT:    vcmp.i32 eq, q3, q2
 ; CHECK-NEXT:    vmul.i32 q4, q4, q2
 ; CHECK-NEXT:    vmovlb.u16 q1, q1
 ; CHECK-NEXT:    vshl.u32 q4, q4, q5
 ; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    vshl.u32 q4, q4, q2
+; CHECK-NEXT:    vpt.i32 eq, q3, q2
+; CHECK-NEXT:    vshlt.u32 q2, q4, q2
 ; CHECK-NEXT:    vadd.i32 q3, q0, q1
-; CHECK-NEXT:    vpsel q2, q4, q2
 ; CHECK-NEXT:    vneg.s32 q4, q1
 ; CHECK-NEXT:    vshl.s32 q3, q3, q4
 ; CHECK-NEXT:    vneg.s32 q4, q0
 ; CHECK-NEXT:    vsub.i32 q3, q3, q1
-; CHECK-NEXT:    vcmp.i32 eq, q0, q1
 ; CHECK-NEXT:    vmul.i32 q3, q3, q1
 ; CHECK-NEXT:    vshl.u32 q3, q3, q4
-; CHECK-NEXT:    vshl.u32 q3, q3, q1
-; CHECK-NEXT:    vpsel q0, q3, q1
-; CHECK-NEXT:    vmovnt.i32 q0, q2
+; CHECK-NEXT:    vpt.i32 eq, q0, q1
+; CHECK-NEXT:    vshlt.u32 q1, q3, q1
+; CHECK-NEXT:    vmovnt.i32 q1, q2
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
@@ -490,24 +489,23 @@ define arm_aapcs_vfpcc <16 x i8> @ext_ops_trunc_i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-NEXT:    vshl.s16 q4, q4, q5
 ; CHECK-NEXT:    vneg.s16 q5, q3
 ; CHECK-NEXT:    vsub.i16 q4, q4, q2
-; CHECK-NEXT:    vcmp.i16 eq, q3, q2
 ; CHECK-NEXT:    vmul.i16 q4, q4, q2
 ; CHECK-NEXT:    vmovlb.u8 q1, q1
 ; CHECK-NEXT:    vshl.u16 q4, q4, q5
 ; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    vshl.u16 q4, q4, q2
+; CHECK-NEXT:    vpt.i16 eq, q3, q2
+; CHECK-NEXT:    vshlt.u16 q2, q4, q2
 ; CHECK-NEXT:    vadd.i16 q3, q0, q1
-; CHECK-NEXT:    vpsel q2, q4, q2
 ; CHECK-NEXT:    vneg.s16 q4, q1
 ; CHECK-NEXT:    vshl.s16 q3, q3, q4
 ; CHECK-NEXT:    vneg.s16 q4, q0
 ; CHECK-NEXT:    vsub.i16 q3, q3, q1
-; CHECK-NEXT:    vcmp.i16 eq, q0, q1
 ; CHECK-NEXT:    vmul.i16 q3, q3, q1
 ; CHECK-NEXT:    vshl.u16 q3, q3, q4
-; CHECK-NEXT:    vshl.u16 q3, q3, q1
-; CHECK-NEXT:    vpsel q0, q3, q1
-; CHECK-NEXT:    vmovnt.i16 q0, q2
+; CHECK-NEXT:    vpt.i16 eq, q0, q1
+; CHECK-NEXT:    vshlt.u16 q1, q3, q1
+; CHECK-NEXT:    vmovnt.i16 q1, q2
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll
index 09b81295c3b4a..eeb1d0d1e7dbc 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop.ll
@@ -235,6 +235,129 @@ entry:
   ret <16 x i8> %b
 }
 
+define arm_aapcs_vfpcc <4 x i32> @shl_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: shl_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vshlt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = shl <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shl_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: shl_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vshlt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = shl <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shl_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: shl_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vshlt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = shl <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ashr_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: ashr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q2, q2
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vshlt.s32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = ashr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ashr_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: ashr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q2, q2
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vshlt.s16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = ashr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ashr_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: ashr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q2, q2
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vshlt.s8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = ashr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @lshr_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: lshr_v4i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q2, q2
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
+; CHECK-NEXT:    vshlt.u32 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <4 x i32> %z, zeroinitializer
+  %a = lshr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %z
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @lshr_v8i16(<8 x i16> %z, <8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: lshr_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q2, q2
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
+; CHECK-NEXT:    vshlt.u16 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <8 x i16> %z, zeroinitializer
+  %a = lshr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %z
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @lshr_v16i8(<16 x i8> %z, <16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: lshr_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q2, q2
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
+; CHECK-NEXT:    vshlt.u8 q0, q1, q2
+; CHECK-NEXT:    bx lr
+entry:
+  %c = icmp eq <16 x i8> %z, zeroinitializer
+  %a = lshr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %z
+  ret <16 x i8> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: andnot_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
@@ -397,6 +520,56 @@ entry:
   ret <8 x half> %b
 }
 
+define arm_aapcs_vfpcc <4 x float> @fdiv_v4f32(<4 x float> %z, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: fdiv_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdiv.f32 s7, s7, s11
+; CHECK-NEXT:    vcmp.f32 eq, q0, zr
+; CHECK-NEXT:    vdiv.f32 s6, s6, s10
+; CHECK-NEXT:    vdiv.f32 s5, s5, s9
+; CHECK-NEXT:    vdiv.f32 s4, s4, s8
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <4 x float> %z, zeroinitializer
+  %a = fdiv <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %z
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fdiv_v8f16(<8 x half> %z, <8 x half> %x, <8 x half> %y) {
+; CHECK-LABEL: fdiv_v8f16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s14, s4
+; CHECK-NEXT:    vmovx.f16 s12, s8
+; CHECK-NEXT:    vdiv.f16 s4, s4, s8
+; CHECK-NEXT:    vdiv.f16 s12, s14, s12
+; CHECK-NEXT:    vins.f16 s4, s12
+; CHECK-NEXT:    vmovx.f16 s12, s5
+; CHECK-NEXT:    vmovx.f16 s8, s9
+; CHECK-NEXT:    vdiv.f16 s5, s5, s9
+; CHECK-NEXT:    vdiv.f16 s8, s12, s8
+; CHECK-NEXT:    vmovx.f16 s12, s6
+; CHECK-NEXT:    vins.f16 s5, s8
+; CHECK-NEXT:    vmovx.f16 s8, s10
+; CHECK-NEXT:    vdiv.f16 s6, s6, s10
+; CHECK-NEXT:    vdiv.f16 s8, s12, s8
+; CHECK-NEXT:    vins.f16 s6, s8
+; CHECK-NEXT:    vmovx.f16 s10, s7
+; CHECK-NEXT:    vmovx.f16 s8, s11
+; CHECK-NEXT:    vdiv.f16 s7, s7, s11
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vcmp.f16 eq, q0, zr
+; CHECK-NEXT:    vins.f16 s7, s8
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %c = fcmp oeq <8 x half> %z, zeroinitializer
+  %a = fdiv <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %z
+  ret <8 x half> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32(<4 x i32> %z, <4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: icmp_slt_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll
index d8376adcdcd67..de7af894bd4fb 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop2.ll
@@ -253,6 +253,138 @@ entry:
   ret <16 x i8> %b
 }
 
+define arm_aapcs_vfpcc <4 x i32> @shl_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: shl_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = shl <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shl_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: shl_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = shl <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shl_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: shl_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = shl <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ashr_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ashr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q1, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = ashr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ashr_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ashr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q1, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = ashr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ashr_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ashr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q1, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = ashr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @lshr_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: lshr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q1, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = lshr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @lshr_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: lshr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q1, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = lshr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @lshr_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: lshr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q1, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = lshr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
+  ret <16 x i8> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: andnot_v4i32_x:
 ; CHECK:       @ %bb.0: @ %entry
@@ -427,6 +559,58 @@ entry:
   ret <8 x half> %b
 }
 
+define arm_aapcs_vfpcc <4 x float> @fdiv_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdiv.f32 s7, s3, s7
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdiv.f32 s6, s2, s6
+; CHECK-NEXT:    vdiv.f32 s5, s1, s5
+; CHECK-NEXT:    vdiv.f32 s4, s0, s4
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fdiv <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %x
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fdiv_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vmovx.f16 s10, s0
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s4, s0, s4
+; CHECK-NEXT:    vins.f16 s4, s8
+; CHECK-NEXT:    vmovx.f16 s8, s5
+; CHECK-NEXT:    vmovx.f16 s10, s1
+; CHECK-NEXT:    vdiv.f16 s5, s1, s5
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vmovx.f16 s10, s2
+; CHECK-NEXT:    vins.f16 s5, s8
+; CHECK-NEXT:    vmovx.f16 s8, s6
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s6, s2, s6
+; CHECK-NEXT:    vins.f16 s6, s8
+; CHECK-NEXT:    vmovx.f16 s8, s7
+; CHECK-NEXT:    vmovx.f16 s10, s3
+; CHECK-NEXT:    vdiv.f16 s7, s3, s7
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vins.f16 s7, s8
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fdiv <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %x
+  ret <8 x half> %b
+}
+
 define arm_aapcs_vfpcc <4 x float> @fmai_v4f32_x(<4 x float> %x, <4 x float> %y, <4 x float> %z, i32 %n) {
 ; CHECK-LABEL: fmai_v4f32_x:
 ; CHECK:       @ %bb.0: @ %entry
@@ -1601,6 +1785,147 @@ entry:
   ret <16 x i8> %b
 }
 
+define arm_aapcs_vfpcc <4 x i32> @shl_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: shl_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = shl <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shl_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: shl_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = shl <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shl_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: shl_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = shl <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ashr_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ashr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q2, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = ashr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ashr_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ashr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q2, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = ashr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ashr_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ashr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q2, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = ashr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @lshr_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: lshr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q2, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = lshr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @lshr_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: lshr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q2, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = lshr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @lshr_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: lshr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q2, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = lshr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: andnot_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
@@ -1787,6 +2112,126 @@ entry:
   ret <8 x half> %b
 }
 
+define arm_aapcs_vfpcc <4 x float> @fdiv_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdiv.f32 s3, s3, s7
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdiv.f32 s2, s2, s6
+; CHECK-NEXT:    vdiv.f32 s1, s1, s5
+; CHECK-NEXT:    vdiv.f32 s0, s0, s4
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fdiv <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fdiv_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s10, s0
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s0, s0, s4
+; CHECK-NEXT:    vins.f16 s0, s8
+; CHECK-NEXT:    vmovx.f16 s10, s1
+; CHECK-NEXT:    vmovx.f16 s8, s5
+; CHECK-NEXT:    vdiv.f16 s1, s1, s5
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vmovx.f16 s10, s2
+; CHECK-NEXT:    vins.f16 s1, s8
+; CHECK-NEXT:    vmovx.f16 s8, s6
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s2, s2, s6
+; CHECK-NEXT:    vins.f16 s2, s8
+; CHECK-NEXT:    vmovx.f16 s10, s3
+; CHECK-NEXT:    vmovx.f16 s8, s7
+; CHECK-NEXT:    vdiv.f16 s3, s3, s7
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vins.f16 s3, s8
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fdiv <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmai_v4f32_y(<4 x float> %x, <4 x float> %y, <4 x float> %z, i32 %n) {
+; CHECK-LABEL: fmai_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x float> @llvm.fma.v4f32(<4 x float> %y, <4 x float> %z, <4 x float> %x)
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmai_v8f16_y(<8 x half> %x, <8 x half> %y, <8 x half> %z, i32 %n) {
+; CHECK-LABEL: fmai_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x half> @llvm.fma.v8f16(<8 x half> %y, <8 x half> %z, <8 x half> %x)
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fma_v4f32_y(<4 x float> %x, <4 x float> %y, <4 x float> %z, i32 %n) {
+; CHECK-LABEL: fma_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %m = fmul fast <4 x float> %y, %z
+  %a = fadd fast <4 x float> %m, %x
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fma_v8f16_y(<8 x half> %x, <8 x half> %y, <8 x half> %z, i32 %n) {
+; CHECK-LABEL: fma_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %m = fmul fast <8 x half> %y, %z
+  %a = fadd fast <8 x half> %m, %x
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: icmp_slt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
@@ -2727,6 +3172,7 @@ declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
 declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
 declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+
 declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
 declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
 
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll b/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
index 0773b65b5dfe0..080c6c1a1efdc 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-selectop3.ll
@@ -253,13 +253,144 @@ entry:
   ret <16 x i8> %b
 }
 
+define arm_aapcs_vfpcc <4 x i32> @shl_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: shl_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = shl <4 x i32> %x, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shl_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: shl_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = shl <8 x i16> %x, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shl_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: shl_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = shl <16 x i8> %x, %a
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ashr_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ashr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q1, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = ashr <4 x i32> %x, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ashr_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ashr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q1, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = ashr <8 x i16> %x, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ashr_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ashr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q1, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = ashr <16 x i8> %x, %a
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @lshr_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: lshr_v4i32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q1, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x i32> %y, <4 x i32> zeroinitializer
+  %b = lshr <4 x i32> %x, %a
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @lshr_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: lshr_v8i16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q1, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x i16> %y, <8 x i16> zeroinitializer
+  %b = lshr <8 x i16> %x, %a
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @lshr_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: lshr_v16i8_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q1, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = select <16 x i1> %c, <16 x i8> %y, <16 x i8> zeroinitializer
+  %b = lshr <16 x i8> %x, %a
+  ret <16 x i8> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: andnot_v4i32_x:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q1, q1
 ; CHECK-NEXT:    vctp.32 r0
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    vbict q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -272,10 +403,9 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @andnot_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
 ; CHECK-LABEL: andnot_v8i16_x:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q1, q1
 ; CHECK-NEXT:    vctp.16 r0
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    vbict q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -288,10 +418,9 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @andnot_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
 ; CHECK-LABEL: andnot_v16i8_x:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q1, q1
 ; CHECK-NEXT:    vctp.8 r0
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vandt q0, q0, q1
+; CHECK-NEXT:    vbict q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
@@ -304,10 +433,9 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @ornot_v4i32_x(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: ornot_v4i32_x:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q1, q1
 ; CHECK-NEXT:    vctp.32 r0
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    vornt q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
@@ -320,10 +448,9 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @ornot_v8i16_x(<8 x i16> %x, <8 x i16> %y, i32 %n) {
 ; CHECK-LABEL: ornot_v8i16_x:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q1, q1
 ; CHECK-NEXT:    vctp.16 r0
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    vornt q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
@@ -336,10 +463,9 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @ornot_v16i8_x(<16 x i8> %x, <16 x i8> %y, i32 %n) {
 ; CHECK-LABEL: ornot_v16i8_x:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmvn q1, q1
 ; CHECK-NEXT:    vctp.8 r0
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vorrt q0, q0, q1
+; CHECK-NEXT:    vornt q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
@@ -493,6 +619,58 @@ entry:
   ret <8 x half> %b
 }
 
+define arm_aapcs_vfpcc <4 x float> @fdiv_v4f32_x(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v4f32_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdiv.f32 s7, s3, s7
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdiv.f32 s6, s2, s6
+; CHECK-NEXT:    vdiv.f32 s5, s1, s5
+; CHECK-NEXT:    vdiv.f32 s4, s0, s4
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = select <4 x i1> %c, <4 x float> %y, <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+  %b = fdiv <4 x float> %x, %a
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fdiv_v8f16_x(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v8f16_x:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vmovx.f16 s10, s0
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s4, s0, s4
+; CHECK-NEXT:    vins.f16 s4, s8
+; CHECK-NEXT:    vmovx.f16 s8, s5
+; CHECK-NEXT:    vmovx.f16 s10, s1
+; CHECK-NEXT:    vdiv.f16 s5, s1, s5
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vmovx.f16 s10, s2
+; CHECK-NEXT:    vins.f16 s5, s8
+; CHECK-NEXT:    vmovx.f16 s8, s6
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s6, s2, s6
+; CHECK-NEXT:    vins.f16 s6, s8
+; CHECK-NEXT:    vmovx.f16 s8, s7
+; CHECK-NEXT:    vmovx.f16 s10, s3
+; CHECK-NEXT:    vdiv.f16 s7, s3, s7
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vins.f16 s7, s8
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = select <8 x i1> %c, <8 x half> %y, <8 x half> <half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00>
+  %b = fdiv <8 x half> %x, %a
+  ret <8 x half> %b
+}
+
 define arm_aapcs_vfpcc <4 x float> @fmai_v4f32_x(<4 x float> %x, <4 x float> %y, <4 x float> %z, i32 %n) {
 ; CHECK-LABEL: fmai_v4f32_x:
 ; CHECK:       @ %bb.0: @ %entry
@@ -566,8 +744,7 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp slt <4 x i32> %x, %y
-  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %x, <4 x i32> %y)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
   ret <4 x i32> %b
 }
@@ -581,8 +758,7 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp slt <8 x i16> %x, %y
-  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %x, <8 x i16> %y)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
   ret <8 x i16> %b
 }
@@ -596,8 +772,7 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp slt <16 x i8> %x, %y
-  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %x, <16 x i8> %y)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
   ret <16 x i8> %b
 }
@@ -611,8 +786,7 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp sgt <4 x i32> %x, %y
-  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> %y)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
   ret <4 x i32> %b
 }
@@ -626,8 +800,7 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp sgt <8 x i16> %x, %y
-  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %x, <8 x i16> %y)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
   ret <8 x i16> %b
 }
@@ -641,8 +814,7 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp sgt <16 x i8> %x, %y
-  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %x, <16 x i8> %y)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
   ret <16 x i8> %b
 }
@@ -656,8 +828,7 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_x(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp ult <4 x i32> %x, %y
-  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %x, <4 x i32> %y)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
   ret <4 x i32> %b
 }
@@ -671,8 +842,7 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_x(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp ult <8 x i16> %x, %y
-  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %x, <8 x i16> %y)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
   ret <8 x i16> %b
 }
@@ -686,8 +856,7 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_x(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp ult <16 x i8> %x, %y
-  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %x, <16 x i8> %y)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
   ret <16 x i8> %b
 }
@@ -701,8 +870,7 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_x(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp ugt <4 x i32> %x, %y
-  %a = select <4 x i1> %a1, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> %y)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
   ret <4 x i32> %b
 }
@@ -716,8 +884,7 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_x(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp ugt <8 x i16> %x, %y
-  %a = select <8 x i1> %a1, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %x, <8 x i16> %y)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
   ret <8 x i16> %b
 }
@@ -731,8 +898,7 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_x(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp ugt <16 x i8> %x, %y
-  %a = select <16 x i1> %a1, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %x, <16 x i8> %y)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
   ret <16 x i8> %b
 }
@@ -974,7 +1140,7 @@ define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x i32> %ys, <4 x i32> zeroinitializer
   %b = add <4 x i32> %a, %x
@@ -990,7 +1156,7 @@ define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x i16> %ys, <8 x i16> zeroinitializer
   %b = add <8 x i16> %a, %x
@@ -1006,7 +1172,7 @@ define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = select <16 x i1> %c, <16 x i8> %ys, <16 x i8> zeroinitializer
   %b = add <16 x i8> %a, %x
@@ -1022,7 +1188,7 @@ define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x i32> %ys, <4 x i32> zeroinitializer
   %b = sub <4 x i32> %x, %a
@@ -1038,7 +1204,7 @@ define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x i16> %ys, <8 x i16> zeroinitializer
   %b = sub <8 x i16> %x, %a
@@ -1054,7 +1220,7 @@ define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = select <16 x i1> %c, <16 x i8> %ys, <16 x i8> zeroinitializer
   %b = sub <16 x i8> %x, %a
@@ -1070,7 +1236,7 @@ define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x i32> %ys, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %b = mul <4 x i32> %a, %x
@@ -1086,7 +1252,7 @@ define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x i16> %ys, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %b = mul <8 x i16> %a, %x
@@ -1102,7 +1268,7 @@ define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = select <16 x i1> %c, <16 x i8> %ys, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   %b = mul <16 x i8> %a, %x
@@ -1119,7 +1285,7 @@ define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32_x(<4 x float> %x, float %y, i32
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x float> undef, float %y, i32 0
+  %i = insertelement <4 x float> undef, float %y, i64 0
   %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x float> %ys, <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>
   %b = fadd <4 x float> %a, %x
@@ -1136,7 +1302,7 @@ define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16_x(<8 x half> %x, half %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x half> undef, half %y, i32 0
+  %i = insertelement <8 x half> undef, half %y, i64 0
   %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x half> %ys, <8 x half> <half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000>
   %b = fadd <8 x half> %a, %x
@@ -1153,7 +1319,7 @@ define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32_x(<4 x float> %x, float %y, i32
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x float> undef, float %y, i32 0
+  %i = insertelement <4 x float> undef, float %y, i64 0
   %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x float> %ys, <4 x float> zeroinitializer
   %b = fsub <4 x float> %x, %a
@@ -1170,7 +1336,7 @@ define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16_x(<8 x half> %x, half %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x half> undef, half %y, i32 0
+  %i = insertelement <8 x half> undef, half %y, i64 0
   %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x half> %ys, <8 x half> zeroinitializer
   %b = fsub <8 x half> %x, %a
@@ -1187,7 +1353,7 @@ define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32_x(<4 x float> %x, float %y, i32
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x float> undef, float %y, i32 0
+  %i = insertelement <4 x float> undef, float %y, i64 0
   %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x float> %ys, <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
   %b = fmul <4 x float> %a, %x
@@ -1204,7 +1370,7 @@ define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16_x(<8 x half> %x, half %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x half> undef, half %y, i32 0
+  %i = insertelement <8 x half> undef, half %y, i64 0
   %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x half> %ys, <8 x half> <half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00>
   %b = fmul <8 x half> %a, %x
@@ -1220,7 +1386,7 @@ define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
@@ -1236,7 +1402,7 @@ define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
@@ -1252,7 +1418,7 @@ define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
@@ -1268,7 +1434,7 @@ define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
@@ -1284,7 +1450,7 @@ define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
@@ -1300,7 +1466,7 @@ define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
@@ -1316,7 +1482,7 @@ define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
@@ -1332,7 +1498,7 @@ define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
@@ -1348,7 +1514,7 @@ define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
@@ -1364,7 +1530,7 @@ define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32_x(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %x
@@ -1380,7 +1546,7 @@ define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16_x(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %x
@@ -1396,7 +1562,7 @@ define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8_x(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %x
@@ -1673,6 +1839,147 @@ entry:
   ret <16 x i8> %b
 }
 
+define arm_aapcs_vfpcc <4 x i32> @shl_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: shl_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = shl <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @shl_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: shl_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = shl <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @shl_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: shl_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q1, q0, q1
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = shl <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ashr_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: ashr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q2, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s32 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = ashr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ashr_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: ashr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q2, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s16 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = ashr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ashr_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: ashr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q2, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.s8 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = ashr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
+define arm_aapcs_vfpcc <4 x i32> @lshr_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
+; CHECK-LABEL: lshr_v4i32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s32 q2, q1
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u32 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = lshr <4 x i32> %x, %y
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
+  ret <4 x i32> %b
+}
+
+define arm_aapcs_vfpcc <8 x i16> @lshr_v8i16_y(<8 x i16> %x, <8 x i16> %y, i32 %n) {
+; CHECK-LABEL: lshr_v8i16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s16 q2, q1
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u16 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = lshr <8 x i16> %x, %y
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
+  ret <8 x i16> %b
+}
+
+define arm_aapcs_vfpcc <16 x i8> @lshr_v16i8_y(<16 x i8> %x, <16 x i8> %y, i32 %n) {
+; CHECK-LABEL: lshr_v16i8_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vneg.s8 q2, q1
+; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vshlt.u8 q1, q0, q2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
+  %a = lshr <16 x i8> %x, %y
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
+  ret <16 x i8> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @andnot_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: andnot_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
@@ -1859,20 +2166,138 @@ entry:
   ret <8 x half> %b
 }
 
+define arm_aapcs_vfpcc <4 x float> @fdiv_v4f32_y(<4 x float> %x, <4 x float> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vdiv.f32 s3, s3, s7
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vdiv.f32 s2, s2, s6
+; CHECK-NEXT:    vdiv.f32 s1, s1, s5
+; CHECK-NEXT:    vdiv.f32 s0, s0, s4
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = fdiv <4 x float> %x, %y
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fdiv_v8f16_y(<8 x half> %x, <8 x half> %y, i32 %n) {
+; CHECK-LABEL: fdiv_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovx.f16 s10, s0
+; CHECK-NEXT:    vmovx.f16 s8, s4
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s0, s0, s4
+; CHECK-NEXT:    vins.f16 s0, s8
+; CHECK-NEXT:    vmovx.f16 s10, s1
+; CHECK-NEXT:    vmovx.f16 s8, s5
+; CHECK-NEXT:    vdiv.f16 s1, s1, s5
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vmovx.f16 s10, s2
+; CHECK-NEXT:    vins.f16 s1, s8
+; CHECK-NEXT:    vmovx.f16 s8, s6
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vdiv.f16 s2, s2, s6
+; CHECK-NEXT:    vins.f16 s2, s8
+; CHECK-NEXT:    vmovx.f16 s10, s3
+; CHECK-NEXT:    vmovx.f16 s8, s7
+; CHECK-NEXT:    vdiv.f16 s3, s3, s7
+; CHECK-NEXT:    vdiv.f16 s8, s10, s8
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vins.f16 s3, s8
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = fdiv <8 x half> %x, %y
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fmai_v4f32_y(<4 x float> %x, <4 x float> %y, <4 x float> %z, i32 %n) {
+; CHECK-LABEL: fmai_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %a = call <4 x float> @llvm.fma.v4f32(<4 x float> %y, <4 x float> %z, <4 x float> %x)
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fmai_v8f16_y(<8 x half> %x, <8 x half> %y, <8 x half> %z, i32 %n) {
+; CHECK-LABEL: fmai_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %a = call <8 x half> @llvm.fma.v8f16(<8 x half> %y, <8 x half> %z, <8 x half> %x)
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
+define arm_aapcs_vfpcc <4 x float> @fma_v4f32_y(<4 x float> %x, <4 x float> %y, <4 x float> %z, i32 %n) {
+; CHECK-LABEL: fma_v4f32_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f32 q0, q1, q2
+; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
+  %m = fmul fast <4 x float> %y, %z
+  %a = fadd fast <4 x float> %m, %x
+  %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %y
+  ret <4 x float> %b
+}
+
+define arm_aapcs_vfpcc <8 x half> @fma_v8f16_y(<8 x half> %x, <8 x half> %y, <8 x half> %z, i32 %n) {
+; CHECK-LABEL: fma_v8f16_y:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vfma.f16 q0, q1, q2
+; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
+  %m = fmul fast <8 x half> %y, %z
+  %a = fadd fast <8 x half> %m, %x
+  %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %y
+  ret <8 x half> %b
+}
+
 define arm_aapcs_vfpcc <4 x i32> @icmp_slt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i32 %n) {
 ; CHECK-LABEL: icmp_slt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.s32 gt, q1, q0
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s32 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp slt <4 x i32> %x, %y
-  %0 = and <4 x i1> %c, %a1
-  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
   ret <4 x i32> %b
 }
 
@@ -1880,16 +2305,14 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_slt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_slt_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.s16 gt, q1, q0
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s16 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp slt <8 x i16> %x, %y
-  %0 = and <8 x i1> %c, %a1
-  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
   ret <8 x i16> %b
 }
 
@@ -1897,16 +2320,14 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_slt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_slt_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.s8 gt, q1, q0
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.s8 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp slt <16 x i8> %x, %y
-  %0 = and <16 x i1> %c, %a1
-  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
   ret <16 x i8> %b
 }
 
@@ -1914,16 +2335,14 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_sgt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_sgt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.s32 gt, q0, q1
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s32 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp sgt <4 x i32> %x, %y
-  %0 = and <4 x i1> %c, %a1
-  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
   ret <4 x i32> %b
 }
 
@@ -1931,16 +2350,14 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_sgt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_sgt_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.s16 gt, q0, q1
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s16 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp sgt <8 x i16> %x, %y
-  %0 = and <8 x i1> %c, %a1
-  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
   ret <8 x i16> %b
 }
 
@@ -1948,16 +2365,14 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_sgt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_sgt_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.s8 gt, q0, q1
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.s8 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp sgt <16 x i8> %x, %y
-  %0 = and <16 x i1> %c, %a1
-  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
   ret <16 x i8> %b
 }
 
@@ -1965,16 +2380,14 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_ult_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_ult_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.u32 hi, q1, q0
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u32 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp ult <4 x i32> %x, %y
-  %0 = and <4 x i1> %c, %a1
-  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
   ret <4 x i32> %b
 }
 
@@ -1982,16 +2395,14 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_ult_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_ult_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.u16 hi, q1, q0
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u16 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp ult <8 x i16> %x, %y
-  %0 = and <8 x i1> %c, %a1
-  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
   ret <8 x i16> %b
 }
 
@@ -1999,16 +2410,14 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_ult_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_ult_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.u8 hi, q1, q0
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmint.u8 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp ult <16 x i8> %x, %y
-  %0 = and <16 x i1> %c, %a1
-  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
   ret <16 x i8> %b
 }
 
@@ -2016,16 +2425,14 @@ define arm_aapcs_vfpcc <4 x i32> @icmp_ugt_v4i32_y(<4 x i32> %x, <4 x i32> %y, i
 ; CHECK-LABEL: icmp_ugt_v4i32_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.32 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.u32 hi, q0, q1
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u32 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %a1 = icmp ugt <4 x i32> %x, %y
-  %0 = and <4 x i1> %c, %a1
-  %b = select <4 x i1> %0, <4 x i32> %x, <4 x i32> %y
+  %a = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %x, <4 x i32> %y)
+  %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %y
   ret <4 x i32> %b
 }
 
@@ -2033,16 +2440,14 @@ define arm_aapcs_vfpcc <8 x i16> @icmp_ugt_v8i16_y(<8 x i16> %x, <8 x i16> %y, i
 ; CHECK-LABEL: icmp_ugt_v8i16_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.16 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.u16 hi, q0, q1
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u16 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %a1 = icmp ugt <8 x i16> %x, %y
-  %0 = and <8 x i1> %c, %a1
-  %b = select <8 x i1> %0, <8 x i16> %x, <8 x i16> %y
+  %a = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %x, <8 x i16> %y)
+  %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %y
   ret <8 x i16> %b
 }
 
@@ -2050,16 +2455,14 @@ define arm_aapcs_vfpcc <16 x i8> @icmp_ugt_v16i8_y(<16 x i8> %x, <16 x i8> %y, i
 ; CHECK-LABEL: icmp_ugt_v16i8_y:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vctp.8 r0
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vcmpt.u8 hi, q0, q1
-; CHECK-NEXT:    vmovt q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vmaxt.u8 q1, q0, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %a1 = icmp ugt <16 x i8> %x, %y
-  %0 = and <16 x i1> %c, %a1
-  %b = select <16 x i1> %0, <16 x i8> %x, <16 x i8> %y
+  %a = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %x, <16 x i8> %y)
+  %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %y
   ret <16 x i8> %b
 }
 
@@ -2317,12 +2720,12 @@ define arm_aapcs_vfpcc <4 x i32> @addqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
 ; CHECK-NEXT:    vdup.32 q1, r0
 ; CHECK-NEXT:    vctp.32 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vaddt.i32 q1, q1, q0
+; CHECK-NEXT:    vaddt.i32 q1, q0, r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> zeroinitializer
   %b = add <4 x i32> %ys, %a
@@ -2335,12 +2738,12 @@ define arm_aapcs_vfpcc <8 x i16> @addqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
 ; CHECK-NEXT:    vdup.16 q1, r0
 ; CHECK-NEXT:    vctp.16 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vaddt.i16 q1, q1, q0
+; CHECK-NEXT:    vaddt.i16 q1, q0, r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> zeroinitializer
   %b = add <8 x i16> %ys, %a
@@ -2353,12 +2756,12 @@ define arm_aapcs_vfpcc <16 x i8> @addqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
 ; CHECK-NEXT:    vdup.8 q1, r0
 ; CHECK-NEXT:    vctp.8 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vaddt.i8 q1, q1, q0
+; CHECK-NEXT:    vaddt.i8 q1, q0, r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> zeroinitializer
   %b = add <16 x i8> %ys, %a
@@ -2376,7 +2779,7 @@ define arm_aapcs_vfpcc <4 x i32> @subqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = sub <4 x i32> %x, %ys
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
@@ -2394,7 +2797,7 @@ define arm_aapcs_vfpcc <8 x i16> @subqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = sub <8 x i16> %x, %ys
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
@@ -2412,7 +2815,7 @@ define arm_aapcs_vfpcc <16 x i8> @subqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = sub <16 x i8> %x, %ys
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
@@ -2425,12 +2828,12 @@ define arm_aapcs_vfpcc <4 x i32> @mulqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %n) {
 ; CHECK-NEXT:    vdup.32 q1, r0
 ; CHECK-NEXT:    vctp.32 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vmult.i32 q1, q1, q0
+; CHECK-NEXT:    vmult.i32 q1, q0, r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x i32> %x, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
   %b = mul <4 x i32> %ys, %a
@@ -2443,12 +2846,12 @@ define arm_aapcs_vfpcc <8 x i16> @mulqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %n) {
 ; CHECK-NEXT:    vdup.16 q1, r0
 ; CHECK-NEXT:    vctp.16 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vmult.i16 q1, q1, q0
+; CHECK-NEXT:    vmult.i16 q1, q0, r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x i16> %x, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %b = mul <8 x i16> %ys, %a
@@ -2461,12 +2864,12 @@ define arm_aapcs_vfpcc <16 x i8> @mulqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n) {
 ; CHECK-NEXT:    vdup.8 q1, r0
 ; CHECK-NEXT:    vctp.8 r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vmult.i8 q1, q1, q0
+; CHECK-NEXT:    vmult.i8 q1, q0, r0
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = select <16 x i1> %c, <16 x i8> %x, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   %b = mul <16 x i8> %ys, %a
@@ -2485,7 +2888,7 @@ define arm_aapcs_vfpcc <4 x float> @faddqr_v4f32_y(<4 x float> %x, float %y, i32
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x float> undef, float %y, i32 0
+  %i = insertelement <4 x float> undef, float %y, i64 0
   %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x float> %x, <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>
   %b = fadd <4 x float> %ys, %a
@@ -2504,7 +2907,7 @@ define arm_aapcs_vfpcc <8 x half> @faddqr_v8f16_y(<8 x half> %x, half %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x half> undef, half %y, i32 0
+  %i = insertelement <8 x half> undef, half %y, i64 0
   %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x half> %x, <8 x half> <half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000>
   %b = fadd <8 x half> %ys, %a
@@ -2523,7 +2926,7 @@ define arm_aapcs_vfpcc <4 x float> @fsubqr_v4f32_y(<4 x float> %x, float %y, i32
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x float> undef, float %y, i32 0
+  %i = insertelement <4 x float> undef, float %y, i64 0
   %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %a = fsub <4 x float> %x, %ys
   %b = select <4 x i1> %c, <4 x float> %a, <4 x float> %ys
@@ -2542,7 +2945,7 @@ define arm_aapcs_vfpcc <8 x half> @fsubqr_v8f16_y(<8 x half> %x, half %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x half> undef, half %y, i32 0
+  %i = insertelement <8 x half> undef, half %y, i64 0
   %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %a = fsub <8 x half> %x, %ys
   %b = select <8 x i1> %c, <8 x half> %a, <8 x half> %ys
@@ -2556,12 +2959,12 @@ define arm_aapcs_vfpcc <4 x float> @fmulqr_v4f32_y(<4 x float> %x, float %y, i32
 ; CHECK-NEXT:    vctp.32 r0
 ; CHECK-NEXT:    vdup.32 q1, r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vmult.f32 q1, q1, q0
+; CHECK-NEXT:    vmult.f32 q1, q0, r1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x float> undef, float %y, i32 0
+  %i = insertelement <4 x float> undef, float %y, i64 0
   %ys = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %a = select <4 x i1> %c, <4 x float> %x, <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
   %b = fmul <4 x float> %ys, %a
@@ -2575,12 +2978,12 @@ define arm_aapcs_vfpcc <8 x half> @fmulqr_v8f16_y(<8 x half> %x, half %y, i32 %n
 ; CHECK-NEXT:    vctp.16 r0
 ; CHECK-NEXT:    vdup.16 q1, r1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vmult.f16 q1, q1, q0
+; CHECK-NEXT:    vmult.f16 q1, q0, r1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x half> undef, half %y, i32 0
+  %i = insertelement <8 x half> undef, half %y, i64 0
   %ys = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %a = select <8 x i1> %c, <8 x half> %x, <8 x half> <half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00>
   %b = fmul <8 x half> %ys, %a
@@ -2598,7 +3001,7 @@ define arm_aapcs_vfpcc <4 x i32> @sadd_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
@@ -2616,7 +3019,7 @@ define arm_aapcs_vfpcc <8 x i16> @sadd_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
@@ -2634,7 +3037,7 @@ define arm_aapcs_vfpcc <16 x i8> @sadd_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
@@ -2652,7 +3055,7 @@ define arm_aapcs_vfpcc <4 x i32> @uadd_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
@@ -2670,7 +3073,7 @@ define arm_aapcs_vfpcc <8 x i16> @uadd_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
@@ -2688,7 +3091,7 @@ define arm_aapcs_vfpcc <16 x i8> @uadd_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
@@ -2706,7 +3109,7 @@ define arm_aapcs_vfpcc <4 x i32> @ssub_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
@@ -2724,7 +3127,7 @@ define arm_aapcs_vfpcc <8 x i16> @ssub_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
@@ -2742,7 +3145,7 @@ define arm_aapcs_vfpcc <16 x i8> @ssub_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
@@ -2760,7 +3163,7 @@ define arm_aapcs_vfpcc <4 x i32> @usub_satqr_v4i32_y(<4 x i32> %x, i32 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <4 x i1> @llvm.arm.mve.vctp32(i32 %n)
-  %i = insertelement <4 x i32> undef, i32 %y, i32 0
+  %i = insertelement <4 x i32> undef, i32 %y, i64 0
   %ys = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
   %a = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %ys)
   %b = select <4 x i1> %c, <4 x i32> %a, <4 x i32> %ys
@@ -2778,7 +3181,7 @@ define arm_aapcs_vfpcc <8 x i16> @usub_satqr_v8i16_y(<8 x i16> %x, i16 %y, i32 %
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <8 x i1> @llvm.arm.mve.vctp16(i32 %n)
-  %i = insertelement <8 x i16> undef, i16 %y, i32 0
+  %i = insertelement <8 x i16> undef, i16 %y, i64 0
   %ys = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
   %a = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %x, <8 x i16> %ys)
   %b = select <8 x i1> %c, <8 x i16> %a, <8 x i16> %ys
@@ -2796,7 +3199,7 @@ define arm_aapcs_vfpcc <16 x i8> @usub_satqr_v16i8_y(<16 x i8> %x, i8 %y, i32 %n
 ; CHECK-NEXT:    bx lr
 entry:
   %c = call <16 x i1> @llvm.arm.mve.vctp8(i32 %n)
-  %i = insertelement <16 x i8> undef, i8 %y, i32 0
+  %i = insertelement <16 x i8> undef, i8 %y, i64 0
   %ys = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
   %a = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %x, <16 x i8> %ys)
   %b = select <16 x i1> %c, <16 x i8> %a, <16 x i8> %ys
@@ -2815,6 +3218,20 @@ declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
 declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
 declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+
+declare <16 x i8> @llvm.umin.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.umax.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.umax.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.smin.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
+declare <16 x i8> @llvm.smax.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
+
 declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
 declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
 
diff --git a/llvm/test/CodeGen/Thumb2/segmented-stacks.ll b/llvm/test/CodeGen/Thumb2/segmented-stacks.ll
index 6d3dbf4eb38f6..60f029c13719e 100644
--- a/llvm/test/CodeGen/Thumb2/segmented-stacks.ll
+++ b/llvm/test/CodeGen/Thumb2/segmented-stacks.ll
@@ -14,7 +14,7 @@ define void @test_basic() #0 {
 ; THUMB-NEXT:    mov r5, sp
 ; THUMB-NEXT:    ldr.w r4, [r4, #252]
 ; THUMB-NEXT:    cmp r4, r5
-; THUMB-NEXT:    blo .LBB0_2
+; THUMB-NEXT:    bls .LBB0_2
 ; THUMB-NEXT:  @ %bb.1:
 ; THUMB-NEXT:    mov r4, #48
 ; THUMB-NEXT:    mov r5, #0
@@ -42,7 +42,7 @@ define void @test_basic() #0 {
 ; ARM-NEXT:    mov r5, sp
 ; ARM-NEXT:    ldr r4, [r4, #252]
 ; ARM-NEXT:    cmp r4, r5
-; ARM-NEXT:    blo .LBB0_2
+; ARM-NEXT:    bls .LBB0_2
 ; ARM-NEXT:  @ %bb.1:
 ; ARM-NEXT:    mov r4, #48
 ; ARM-NEXT:    mov r5, #0
@@ -82,7 +82,7 @@ define void @test_large() #0 {
 ; THUMB-NEXT:    mrc     p15, #0, r4, c13, c0, #3
 ; THUMB-NEXT:    ldr.w   r4, [r4, #252]
 ; THUMB-NEXT:    cmp     r4, r5
-; THUMB-NEXT:    blo     .LBB1_2
+; THUMB-NEXT:    bls     .LBB1_2
 
 ; THUMB:         movw    r4, #40192
 ; THUMB-NEXT:    movt    r4, #0
@@ -104,7 +104,7 @@ define void @test_large() #0 {
 ; ARM-NEXT:    mrc     p15, #0, r4, c13, c0, #3
 ; ARM-NEXT:    ldr     r4, [r4, #252]
 ; ARM-NEXT:    cmp     r4, r5
-; ARM-NEXT:    blo     .LBB1_2
+; ARM-NEXT:    bls     .LBB1_2
 
 ; ARM:         ldr     r4, .LCPI1_0
 ; ARM-NEXT:    mov     r5, #0
@@ -136,7 +136,7 @@ define fastcc void @test_fastcc_large() #0 {
 ; THUMB-NEXT:    mrc     p15, #0, r4, c13, c0, #3
 ; THUMB-NEXT:    ldr.w   r4, [r4, #252]
 ; THUMB-NEXT:    cmp     r4, r5
-; THUMB-NEXT:    blo     .LBB2_2
+; THUMB-NEXT:    bls     .LBB2_2
 
 ; THUMB:         movw    r4, #40192
 ; THUMB-NEXT:    movt    r4, #0
@@ -157,7 +157,7 @@ define fastcc void @test_fastcc_large() #0 {
 ; ARM-NEXT:    mrc     p15, #0, r4, c13, c0, #3
 ; ARM-NEXT:    ldr     r4, [r4, #252]
 ; ARM-NEXT:    cmp     r4, r5
-; ARM-NEXT:    blo     .LBB2_2
+; ARM-NEXT:    bls     .LBB2_2
 
 ; ARM:         ldr     r4, .LCPI2_0
 ; ARM-NEXT:    mov     r5, #0
diff --git a/llvm/test/CodeGen/VE/null-mctargetstreamer.ll b/llvm/test/CodeGen/VE/null-mctargetstreamer.ll
new file mode 100644
index 0000000000000..fd0ded414391c
--- /dev/null
+++ b/llvm/test/CodeGen/VE/null-mctargetstreamer.ll
@@ -0,0 +1,5 @@
+; RUN: llc -march=ve -filetype=null %s
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/CodeGen/WebAssembly/pr59626.ll b/llvm/test/CodeGen/WebAssembly/pr59626.ll
index a2324b0b48a78..a0c4ba7d67598 100644
--- a/llvm/test/CodeGen/WebAssembly/pr59626.ll
+++ b/llvm/test/CodeGen/WebAssembly/pr59626.ll
@@ -34,7 +34,6 @@ define i8 @f(ptr %0, ptr %1) {
 ; CHECK-64-NEXT:    local.get 2
 ; CHECK-64-NEXT:    i8x16.splat
 ; CHECK-64-NEXT:    v128.store16_lane 0, 0
-; CHECK-64-NEXT:    drop
 ; CHECK-64-NEXT:    v128.const 0, 0
 ; CHECK-64-NEXT:    i32x4.extract_lane 0
 ; CHECK-64-NEXT:    # fallthrough-return
diff --git a/llvm/test/CodeGen/WebAssembly/simd-load-lane-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-load-lane-offset.ll
index 77191f4eb0c4a..587e313fee776 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-load-lane-offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-load-lane-offset.ll
@@ -1,13 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 -mtriple=wasm32-unknown-unknown | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mattr=+simd128 -mtriple=wasm64-unknown-unknown | FileCheck %s --check-prefix MEM64
 
 ; Test SIMD v128.load{8,16,32,64}_lane instructions.
 
 ; TODO: Use the offset field by supporting more patterns. Right now only the
 ; equivalents of LoadPatNoOffset/StorePatNoOffset are supported.
 
-target triple = "wasm32-unknown-unknown"
-
 ;===----------------------------------------------------------------------------
 ; v128.load8_lane / v128.store8_lane
 ;===----------------------------------------------------------------------------
@@ -20,6 +19,14 @@ define <16 x i8> @load_lane_i8_no_offset(ptr %p, <16 x i8> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_no_offset:
+; MEM64:         .functype load_lane_i8_no_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i8, ptr %p
   %t = insertelement <16 x i8> %v, i8 %x, i32 0
   ret <16 x i8> %t
@@ -35,6 +42,18 @@ define <16 x i8> @load_lane_i8_with_folded_offset(ptr %p, <16 x i8> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_with_folded_offset:
+; MEM64:         .functype load_lane_i8_with_folded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -53,6 +72,16 @@ define <16 x i8> @load_lane_i8_with_folded_gep_offset(ptr %p, <16 x i8> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_with_folded_gep_offset:
+; MEM64:         .functype load_lane_i8_with_folded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 6
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, ptr %p, i32 6
   %x = load i8, ptr %s
   %t = insertelement <16 x i8> %v, i8 %x, i32 0
@@ -69,6 +98,16 @@ define <16 x i8> @load_lane_i8_with_unfolded_gep_negative_offset(ptr %p, <16 x i
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_with_unfolded_gep_negative_offset:
+; MEM64:         .functype load_lane_i8_with_unfolded_gep_negative_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const -6
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, ptr %p, i32 -6
   %x = load i8, ptr %s
   %t = insertelement <16 x i8> %v, i8 %x, i32 0
@@ -85,6 +124,18 @@ define <16 x i8> @load_lane_i8_with_unfolded_offset(ptr %p, <16 x i8> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_with_unfolded_offset:
+; MEM64:         .functype load_lane_i8_with_unfolded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -103,6 +154,16 @@ define <16 x i8> @load_lane_i8_with_unfolded_gep_offset(ptr %p, <16 x i8> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_with_unfolded_gep_offset:
+; MEM64:         .functype load_lane_i8_with_unfolded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 6
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i8, ptr %p, i32 6
   %x = load i8, ptr %s
   %t = insertelement <16 x i8> %v, i8 %x, i32 0
@@ -117,6 +178,14 @@ define <16 x i8> @load_lane_i8_from_numeric_address(<16 x i8> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_from_numeric_address:
+; MEM64:         .functype load_lane_i8_from_numeric_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 42
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = load i8, ptr %s
   %t = insertelement <16 x i8> %v, i8 %x, i32 0
@@ -132,6 +201,14 @@ define <16 x i8> @load_lane_i8_from_global_address(<16 x i8> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i8_from_global_address:
+; MEM64:         .functype load_lane_i8_from_global_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const gv_i8
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i8, ptr @gv_i8
   %t = insertelement <16 x i8> %v, i8 %x, i32 0
   ret <16 x i8> %t
@@ -145,6 +222,14 @@ define void @store_lane_i8_no_offset(<16 x i8> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_no_offset:
+; MEM64:         .functype store_lane_i8_no_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, ptr %p
   ret void
@@ -158,6 +243,18 @@ define void @store_lane_i8_with_folded_offset(<16 x i8> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_with_folded_offset:
+; MEM64:         .functype store_lane_i8_with_folded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -174,6 +271,14 @@ define void @store_lane_i8_with_folded_gep_offset(<16 x i8> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 6, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_with_folded_gep_offset:
+; MEM64:         .functype store_lane_i8_with_folded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 6, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, ptr %p, i32 6
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, ptr %s
@@ -190,6 +295,16 @@ define void @store_lane_i8_with_unfolded_gep_negative_offset(<16 x i8> %v, ptr %
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_with_unfolded_gep_negative_offset:
+; MEM64:         .functype store_lane_i8_with_unfolded_gep_negative_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const -6
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i8, ptr %p, i32 -6
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, ptr %s
@@ -206,6 +321,18 @@ define void @store_lane_i8_with_unfolded_offset(<16 x i8> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_with_unfolded_offset:
+; MEM64:         .functype store_lane_i8_with_unfolded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -224,6 +351,16 @@ define void @store_lane_i8_with_unfolded_gep_offset(<16 x i8> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_with_unfolded_gep_offset:
+; MEM64:         .functype store_lane_i8_with_unfolded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const 6
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i8, ptr %p, i32 6
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, ptr %s
@@ -238,6 +375,14 @@ define void @store_lane_i8_to_numeric_address(<16 x i8> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_to_numeric_address:
+; MEM64:         .functype store_lane_i8_to_numeric_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane 42, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, ptr %s
@@ -252,6 +397,14 @@ define void @store_lane_i8_from_global_address(<16 x i8> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store8_lane gv_i8, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i8_from_global_address:
+; MEM64:         .functype store_lane_i8_from_global_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store8_lane gv_i8, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <16 x i8> %v, i32 0
   store i8 %x, ptr @gv_i8
   ret void
@@ -269,6 +422,14 @@ define <8 x i16> @load_lane_i16_no_offset(ptr %p, <8 x i16> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_no_offset:
+; MEM64:         .functype load_lane_i16_no_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i16, ptr %p
   %t = insertelement <8 x i16> %v, i16 %x, i32 0
   ret <8 x i16> %t
@@ -284,6 +445,18 @@ define <8 x i16> @load_lane_i16_with_folded_offset(ptr %p, <8 x i16> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_with_folded_offset:
+; MEM64:         .functype load_lane_i16_with_folded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -302,6 +475,16 @@ define <8 x i16> @load_lane_i16_with_folded_gep_offset(ptr %p, <8 x i16> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_with_folded_gep_offset:
+; MEM64:         .functype load_lane_i16_with_folded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 12
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, ptr %p, i32 6
   %x = load i16, ptr %s
   %t = insertelement <8 x i16> %v, i16 %x, i32 0
@@ -318,6 +501,16 @@ define <8 x i16> @load_lane_i16_with_unfolded_gep_negative_offset(ptr %p, <8 x i
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_with_unfolded_gep_negative_offset:
+; MEM64:         .functype load_lane_i16_with_unfolded_gep_negative_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const -12
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, ptr %p, i32 -6
   %x = load i16, ptr %s
   %t = insertelement <8 x i16> %v, i16 %x, i32 0
@@ -334,6 +527,18 @@ define <8 x i16> @load_lane_i16_with_unfolded_offset(ptr %p, <8 x i16> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_with_unfolded_offset:
+; MEM64:         .functype load_lane_i16_with_unfolded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -352,6 +557,16 @@ define <8 x i16> @load_lane_i16_with_unfolded_gep_offset(ptr %p, <8 x i16> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_with_unfolded_gep_offset:
+; MEM64:         .functype load_lane_i16_with_unfolded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 12
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i16, ptr %p, i32 6
   %x = load i16, ptr %s
   %t = insertelement <8 x i16> %v, i16 %x, i32 0
@@ -366,6 +581,14 @@ define <8 x i16> @load_lane_i16_from_numeric_address(<8 x i16> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_from_numeric_address:
+; MEM64:         .functype load_lane_i16_from_numeric_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 42
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = load i16, ptr %s
   %t = insertelement <8 x i16> %v, i16 %x, i32 0
@@ -381,6 +604,14 @@ define <8 x i16> @load_lane_i16_from_global_address(<8 x i16> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i16_from_global_address:
+; MEM64:         .functype load_lane_i16_from_global_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const gv_i16
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i16, ptr @gv_i16
   %t = insertelement <8 x i16> %v, i16 %x, i32 0
   ret <8 x i16> %t
@@ -394,6 +625,14 @@ define void @store_lane_i16_no_offset(<8 x i16> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_no_offset:
+; MEM64:         .functype store_lane_i16_no_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, ptr %p
   ret void
@@ -407,6 +646,18 @@ define void @store_lane_i16_with_folded_offset(<8 x i16> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_with_folded_offset:
+; MEM64:         .functype store_lane_i16_with_folded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -423,6 +674,14 @@ define void @store_lane_i16_with_folded_gep_offset(<8 x i16> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 12, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_with_folded_gep_offset:
+; MEM64:         .functype store_lane_i16_with_folded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 12, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, ptr %p, i32 6
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, ptr %s
@@ -439,6 +698,16 @@ define void @store_lane_i16_with_unfolded_gep_negative_offset(<8 x i16> %v, ptr
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_with_unfolded_gep_negative_offset:
+; MEM64:         .functype store_lane_i16_with_unfolded_gep_negative_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const -12
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i16, ptr %p, i32 -6
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, ptr %s
@@ -455,6 +724,18 @@ define void @store_lane_i16_with_unfolded_offset(<8 x i16> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_with_unfolded_offset:
+; MEM64:         .functype store_lane_i16_with_unfolded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -473,6 +754,16 @@ define void @store_lane_i16_with_unfolded_gep_offset(<8 x i16> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_with_unfolded_gep_offset:
+; MEM64:         .functype store_lane_i16_with_unfolded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const 12
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i16, ptr %p, i32 6
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, ptr %s
@@ -487,6 +778,14 @@ define void @store_lane_i16_to_numeric_address(<8 x i16> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_to_numeric_address:
+; MEM64:         .functype store_lane_i16_to_numeric_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane 42, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, ptr %s
@@ -501,6 +800,14 @@ define void @store_lane_i16_from_global_address(<8 x i16> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store16_lane gv_i16, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i16_from_global_address:
+; MEM64:         .functype store_lane_i16_from_global_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store16_lane gv_i16, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <8 x i16> %v, i32 0
   store i16 %x, ptr @gv_i16
   ret void
@@ -518,6 +825,14 @@ define <4 x i32> @load_lane_i32_no_offset(ptr %p, <4 x i32> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_no_offset:
+; MEM64:         .functype load_lane_i32_no_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i32, ptr %p
   %t = insertelement <4 x i32> %v, i32 %x, i32 0
   ret <4 x i32> %t
@@ -533,6 +848,18 @@ define <4 x i32> @load_lane_i32_with_folded_offset(ptr %p, <4 x i32> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_with_folded_offset:
+; MEM64:         .functype load_lane_i32_with_folded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -551,6 +878,16 @@ define <4 x i32> @load_lane_i32_with_folded_gep_offset(ptr %p, <4 x i32> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_with_folded_gep_offset:
+; MEM64:         .functype load_lane_i32_with_folded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 24
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, ptr %p, i32 6
   %x = load i32, ptr %s
   %t = insertelement <4 x i32> %v, i32 %x, i32 0
@@ -567,6 +904,16 @@ define <4 x i32> @load_lane_i32_with_unfolded_gep_negative_offset(ptr %p, <4 x i
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_with_unfolded_gep_negative_offset:
+; MEM64:         .functype load_lane_i32_with_unfolded_gep_negative_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const -24
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, ptr %p, i32 -6
   %x = load i32, ptr %s
   %t = insertelement <4 x i32> %v, i32 %x, i32 0
@@ -583,6 +930,18 @@ define <4 x i32> @load_lane_i32_with_unfolded_offset(ptr %p, <4 x i32> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_with_unfolded_offset:
+; MEM64:         .functype load_lane_i32_with_unfolded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -601,6 +960,16 @@ define <4 x i32> @load_lane_i32_with_unfolded_gep_offset(ptr %p, <4 x i32> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_with_unfolded_gep_offset:
+; MEM64:         .functype load_lane_i32_with_unfolded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 24
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i32, ptr %p, i32 6
   %x = load i32, ptr %s
   %t = insertelement <4 x i32> %v, i32 %x, i32 0
@@ -615,6 +984,14 @@ define <4 x i32> @load_lane_i32_from_numeric_address(<4 x i32> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_from_numeric_address:
+; MEM64:         .functype load_lane_i32_from_numeric_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 42
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = load i32, ptr %s
   %t = insertelement <4 x i32> %v, i32 %x, i32 0
@@ -630,6 +1007,14 @@ define <4 x i32> @load_lane_i32_from_global_address(<4 x i32> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i32_from_global_address:
+; MEM64:         .functype load_lane_i32_from_global_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const gv_i32
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i32, ptr @gv_i32
   %t = insertelement <4 x i32> %v, i32 %x, i32 0
   ret <4 x i32> %t
@@ -643,6 +1028,14 @@ define void @store_lane_i32_no_offset(<4 x i32> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_no_offset:
+; MEM64:         .functype store_lane_i32_no_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, ptr %p
   ret void
@@ -656,6 +1049,18 @@ define void @store_lane_i32_with_folded_offset(<4 x i32> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_with_folded_offset:
+; MEM64:         .functype store_lane_i32_with_folded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -672,6 +1077,14 @@ define void @store_lane_i32_with_folded_gep_offset(<4 x i32> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_with_folded_gep_offset:
+; MEM64:         .functype store_lane_i32_with_folded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 24, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, ptr %p, i32 6
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, ptr %s
@@ -688,6 +1101,16 @@ define void @store_lane_i32_with_unfolded_gep_negative_offset(<4 x i32> %v, ptr
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_with_unfolded_gep_negative_offset:
+; MEM64:         .functype store_lane_i32_with_unfolded_gep_negative_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const -24
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i32, ptr %p, i32 -6
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, ptr %s
@@ -704,6 +1127,18 @@ define void @store_lane_i32_with_unfolded_offset(<4 x i32> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_with_unfolded_offset:
+; MEM64:         .functype store_lane_i32_with_unfolded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -722,6 +1157,16 @@ define void @store_lane_i32_with_unfolded_gep_offset(<4 x i32> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_with_unfolded_gep_offset:
+; MEM64:         .functype store_lane_i32_with_unfolded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const 24
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i32, ptr %p, i32 6
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, ptr %s
@@ -736,6 +1181,14 @@ define void @store_lane_i32_to_numeric_address(<4 x i32> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_to_numeric_address:
+; MEM64:         .functype store_lane_i32_to_numeric_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane 42, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, ptr %s
@@ -750,6 +1203,14 @@ define void @store_lane_i32_from_global_address(<4 x i32> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store32_lane gv_i32, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i32_from_global_address:
+; MEM64:         .functype store_lane_i32_from_global_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store32_lane gv_i32, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <4 x i32> %v, i32 0
   store i32 %x, ptr @gv_i32
   ret void
@@ -767,6 +1228,14 @@ define <2 x i64> @load_lane_i64_no_offset(ptr %p, <2 x i64> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_no_offset:
+; MEM64:         .functype load_lane_i64_no_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i64, ptr %p
   %t = insertelement <2 x i64> %v, i64 %x, i32 0
   ret <2 x i64> %t
@@ -782,6 +1251,18 @@ define <2 x i64> @load_lane_i64_with_folded_offset(ptr %p, <2 x i64> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_with_folded_offset:
+; MEM64:         .functype load_lane_i64_with_folded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -800,6 +1281,16 @@ define <2 x i64> @load_lane_i64_with_folded_gep_offset(ptr %p, <2 x i64> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_with_folded_gep_offset:
+; MEM64:         .functype load_lane_i64_with_folded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 48
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, ptr %p, i32 6
   %x = load i64, ptr %s
   %t = insertelement <2 x i64> %v, i64 %x, i32 0
@@ -816,6 +1307,16 @@ define <2 x i64> @load_lane_i64_with_unfolded_gep_negative_offset(ptr %p, <2 x i
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_with_unfolded_gep_negative_offset:
+; MEM64:         .functype load_lane_i64_with_unfolded_gep_negative_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const -48
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, ptr %p, i32 -6
   %x = load i64, ptr %s
   %t = insertelement <2 x i64> %v, i64 %x, i32 0
@@ -832,6 +1333,18 @@ define <2 x i64> @load_lane_i64_with_unfolded_offset(ptr %p, <2 x i64> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_with_unfolded_offset:
+; MEM64:         .functype load_lane_i64_with_unfolded_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -850,6 +1363,16 @@ define <2 x i64> @load_lane_i64_with_unfolded_gep_offset(ptr %p, <2 x i64> %v) {
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_with_unfolded_gep_offset:
+; MEM64:         .functype load_lane_i64_with_unfolded_gep_offset (i64, v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    i64.const 48
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i64, ptr %p, i32 6
   %x = load i64, ptr %s
   %t = insertelement <2 x i64> %v, i64 %x, i32 0
@@ -864,6 +1387,14 @@ define <2 x i64> @load_lane_i64_from_numeric_address(<2 x i64> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_from_numeric_address:
+; MEM64:         .functype load_lane_i64_from_numeric_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 42
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = load i64, ptr %s
   %t = insertelement <2 x i64> %v, i64 %x, i32 0
@@ -879,6 +1410,14 @@ define <2 x i64> @load_lane_i64_from_global_address(<2 x i64> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.load64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: load_lane_i64_from_global_address:
+; MEM64:         .functype load_lane_i64_from_global_address (v128) -> (v128)
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const gv_i64
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.load64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = load i64, ptr @gv_i64
   %t = insertelement <2 x i64> %v, i64 %x, i32 0
   ret <2 x i64> %t
@@ -892,6 +1431,14 @@ define void @store_lane_i64_no_offset(<2 x i64> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_no_offset:
+; MEM64:         .functype store_lane_i64_no_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, ptr %p
   ret void
@@ -905,6 +1452,18 @@ define void @store_lane_i64_with_folded_offset(<2 x i64> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 24, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_with_folded_offset:
+; MEM64:         .functype store_lane_i64_with_folded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nuw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -921,6 +1480,14 @@ define void @store_lane_i64_with_folded_gep_offset(<2 x i64> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 48, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_with_folded_gep_offset:
+; MEM64:         .functype store_lane_i64_with_folded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 48, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, ptr %p, i32 6
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, ptr %s
@@ -937,6 +1504,16 @@ define void @store_lane_i64_with_unfolded_gep_negative_offset(<2 x i64> %v, ptr
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_with_unfolded_gep_negative_offset:
+; MEM64:         .functype store_lane_i64_with_unfolded_gep_negative_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const -48
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr inbounds i64, ptr %p, i32 -6
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, ptr %s
@@ -953,6 +1530,18 @@ define void @store_lane_i64_with_unfolded_offset(<2 x i64> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_with_unfolded_offset:
+; MEM64:         .functype store_lane_i64_with_unfolded_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i32.wrap_i64
+; MEM64-NEXT:    i32.const 24
+; MEM64-NEXT:    i32.add
+; MEM64-NEXT:    i64.extend_i32_u
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %q = ptrtoint ptr %p to i32
   %r = add nsw i32 %q, 24
   %s = inttoptr i32 %r to ptr
@@ -971,6 +1560,16 @@ define void @store_lane_i64_with_unfolded_gep_offset(<2 x i64> %v, ptr %p) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 0, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_with_unfolded_gep_offset:
+; MEM64:         .functype store_lane_i64_with_unfolded_gep_offset (v128, i64) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    local.get 1
+; MEM64-NEXT:    i64.const 48
+; MEM64-NEXT:    i64.add
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 0, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = getelementptr i64, ptr %p, i32 6
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, ptr %s
@@ -985,6 +1584,14 @@ define void @store_lane_i64_to_numeric_address(<2 x i64> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane 42, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_to_numeric_address:
+; MEM64:         .functype store_lane_i64_to_numeric_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane 42, 0
+; MEM64-NEXT:    # fallthrough-return
   %s = inttoptr i32 42 to ptr
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, ptr %s
@@ -999,6 +1606,14 @@ define void @store_lane_i64_from_global_address(<2 x i64> %v) {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.store64_lane gv_i64, 0
 ; CHECK-NEXT:    # fallthrough-return
+;
+; MEM64-LABEL: store_lane_i64_from_global_address:
+; MEM64:         .functype store_lane_i64_from_global_address (v128) -> ()
+; MEM64-NEXT:  # %bb.0:
+; MEM64-NEXT:    i64.const 0
+; MEM64-NEXT:    local.get 0
+; MEM64-NEXT:    v128.store64_lane gv_i64, 0
+; MEM64-NEXT:    # fallthrough-return
   %x = extractelement <2 x i64> %v, i32 0
   store i64 %x, ptr @gv_i64
   ret void
diff --git a/llvm/test/CodeGen/X86/2010-08-04-MaskedSignedCompare.ll b/llvm/test/CodeGen/X86/2010-08-04-MaskedSignedCompare.ll
index 97e2db2a68cfa..0e57e9d135a4d 100644
--- a/llvm/test/CodeGen/X86/2010-08-04-MaskedSignedCompare.ll
+++ b/llvm/test/CodeGen/X86/2010-08-04-MaskedSignedCompare.ll
@@ -9,7 +9,6 @@
 define dso_local i32 @main() nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpq g_16(%rip), %rax
 ; CHECK-NEXT:    sbbl %eax, %eax
@@ -22,6 +21,7 @@ define dso_local i32 @main() nounwind {
 ; CHECK-NEXT:  .LBB0_1: # %entry.if.end_crit_edge
 ; CHECK-NEXT:    movl g_38(%rip), %esi
 ; CHECK-NEXT:  .LBB0_3: # %if.end
+; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movl $.L.str, %edi
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    callq printf@PLT
diff --git a/llvm/test/CodeGen/X86/abdu-vector-128.ll b/llvm/test/CodeGen/X86/abdu-vector-128.ll
index 5281fc78b025f..f57c6e45dc7f4 100644
--- a/llvm/test/CodeGen/X86/abdu-vector-128.ll
+++ b/llvm/test/CodeGen/X86/abdu-vector-128.ll
@@ -58,56 +58,10 @@ define <16 x i8> @abd_ext_v16i8_undef(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @abd_ext_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: abd_ext_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
-; SSE2-NEXT:    psubq %xmm7, %xmm3
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; SSE2-NEXT:    psubq %xmm6, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; SSE2-NEXT:    psubq %xmm6, %xmm4
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    psubq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    psubq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    psubq %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psubusw %xmm0, %xmm2
+; SSE2-NEXT:    psubusw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: abd_ext_v8i16:
@@ -135,56 +89,10 @@ define <8 x i16> @abd_ext_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 define <8 x i16> @abd_ext_v8i16_undef(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: abd_ext_v8i16_undef:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
-; SSE2-NEXT:    psubq %xmm7, %xmm3
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; SSE2-NEXT:    psubq %xmm6, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; SSE2-NEXT:    psubq %xmm6, %xmm4
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm3
-; SSE2-NEXT:    psubq %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    psubq %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    psubq %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psubusw %xmm0, %xmm2
+; SSE2-NEXT:    psubusw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: abd_ext_v8i16_undef:
@@ -481,13 +389,10 @@ define <16 x i8> @abd_minmax_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @abd_minmax_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: abd_minmax_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psubusw %xmm1, %xmm2
-; SSE2-NEXT:    psubusw %xmm0, %xmm1
-; SSE2-NEXT:    paddw %xmm0, %xmm1
-; SSE2-NEXT:    psubw %xmm0, %xmm2
-; SSE2-NEXT:    paddw %xmm1, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psubusw %xmm0, %xmm2
+; SSE2-NEXT:    psubusw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: abd_minmax_v8i16:
@@ -653,17 +558,10 @@ define <16 x i8> @abd_cmp_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 define <8 x i16> @abd_cmp_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: abd_cmp_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    psubusw %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psubw %xmm1, %xmm3
-; SSE2-NEXT:    psubw %xmm0, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psubusw %xmm0, %xmm2
+; SSE2-NEXT:    psubusw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: abd_cmp_v8i16:
@@ -890,19 +788,13 @@ define <2 x i64> @abd_cmp_v2i64_multiuse_cmp(<2 x i64> %a, <2 x i64> %b) nounwin
 define <8 x i16> @abd_cmp_v8i16_multiuse_sub(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: abd_cmp_v8i16_multiuse_sub:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psubw %xmm1, %xmm2
 ; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    psubusw %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpeqw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psubw %xmm1, %xmm3
-; SSE2-NEXT:    psubw %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    paddw %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    psubusw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddw %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: abd_cmp_v8i16_multiuse_sub:
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
index 591256dfeda25..6cbb5e0d3d6d3 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
@@ -7495,10 +7495,7 @@ declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) nounwind re
 define <8 x double> @test_mm256_castpd128_pd256_freeze(<2 x double> %a0) nounwind {
 ; CHECK-LABEL: test_mm256_castpd128_pd256_freeze:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %a1 = freeze <2 x double> poison
   %res = shufflevector <2 x double> %a0, <2 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
@@ -7520,10 +7517,7 @@ define <8 x double> @test_mm256_castpd256_pd256_freeze(<4 x double> %a0) nounwin
 define <16 x float> @test_mm256_castps128_ps512_freeze(<4 x float> %a0) nounwind {
 ; CHECK-LABEL: test_mm256_castps128_ps512_freeze:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %a1 = freeze <4 x float> poison
   %res = shufflevector <4 x float> %a0, <4 x float> %a1, <16x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
@@ -7545,10 +7539,7 @@ define <16 x float> @test_mm256_castps256_ps512_freeze(<8 x float> %a0) nounwind
 define <8 x i64> @test_mm512_castsi128_si512_freeze(<2 x i64> %a0) nounwind {
 ; CHECK-LABEL: test_mm512_castsi128_si512_freeze:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %a1 = freeze <2 x i64> poison
   %res = shufflevector <2 x i64> %a0, <2 x i64> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
new file mode 100644
index 0000000000000..1782e52528700
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
@@ -0,0 +1,225 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown -mattr=+avx512fp16 | FileCheck %s
+
+declare half @llvm.minimum.f16(half, half)
+declare half @llvm.maximum.f16(half, half)
+declare <8 x half> @llvm.minimum.v8f16(<8 x half>, <8 x half>)
+declare <8 x half> @llvm.maximum.v8f16(<8 x half>, <8 x half>)
+
+define half @test_fminimum(half %x, half %y) {
+; CHECK-LABEL: test_fminimum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovw %xmm0, %eax
+; CHECK-NEXT:    testw %ax, %ax
+; CHECK-NEXT:    sets %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vmovaps %xmm1, %xmm2
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    vminsh %xmm2, %xmm0, %xmm1
+; CHECK-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %z = call half @llvm.minimum.f16(half %x, half %y)
+  ret half %z
+}
+
+define <8 x half> @test_fminimum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" {
+; CHECK-LABEL: test_fminimum_scalarize:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vminsh %xmm2, %xmm3, %xmm2
+; CHECK-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[3,3,3,3]
+; CHECK-NEXT:    vshufps {{.*#+}} xmm4 = xmm0[3,3,3,3]
+; CHECK-NEXT:    vminsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vminsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm4 = xmm1[1,0]
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm5 = xmm0[1,0]
+; CHECK-NEXT:    vminsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT:    vpsrlq $48, %xmm1, %xmm3
+; CHECK-NEXT:    vpsrlq $48, %xmm0, %xmm4
+; CHECK-NEXT:    vminsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; CHECK-NEXT:    vminsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT:    vminsh %xmm1, %xmm0, %xmm4
+; CHECK-NEXT:    vpsrld $16, %xmm1, %xmm1
+; CHECK-NEXT:    vpsrld $16, %xmm0, %xmm0
+; CHECK-NEXT:    vminsh %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    retq
+  %r = call <8 x half> @llvm.minimum.v8f16(<8 x half> %x, <8 x half> %y)
+  ret <8 x half> %r
+}
+
+define half @test_fminimum_nnan(half %x, half %y) "no-nans-fp-math"="true" {
+; CHECK-LABEL: test_fminimum_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfpclasssh $5, %xmm1, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm2
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vminsh %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = tail call half @llvm.minimum.f16(half %x, half %y)
+  ret half %1
+}
+
+define half @test_fminimum_zero(half %x, half %y) {
+; CHECK-LABEL: test_fminimum_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcmpunordsh %xmm1, %xmm1, %k1
+; CHECK-NEXT:    vminsh {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    retq
+  %1 = tail call half @llvm.minimum.f16(half -0.0, half %y)
+  ret half %1
+}
+
+define half @test_fminimum_nsz(half %x, half %y) {
+; CHECK-LABEL: test_fminimum_nsz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vminsh %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = tail call nsz half @llvm.minimum.f16(half %x, half %y)
+  ret half %1
+}
+
+define half @test_fminimum_combine_cmps(half %x, half %y) {
+; CHECK-LABEL: test_fminimum_combine_cmps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdivsh %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vfpclasssh $5, %xmm0, %k1
+; CHECK-NEXT:    vmovaps %xmm1, %xmm2
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    vminsh %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %1 = fdiv nnan half %y, %x
+  %2 = tail call half @llvm.minimum.f16(half %x, half %1)
+  ret half %2
+}
+
+define half @test_fmaximum(half %x, half %y) {
+; CHECK-LABEL: test_fmaximum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovw %xmm0, %eax
+; CHECK-NEXT:    testw %ax, %ax
+; CHECK-NEXT:    sets %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm2
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmaxsh %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vcmpunordsh %xmm1, %xmm1, %k1
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    retq
+  %r = call half @llvm.maximum.f16(half %x, half %y)
+  ret half %r
+}
+
+define <8 x half> @test_fmaximum_scalarize(<8 x half> %x, <8 x half> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" {
+; CHECK-LABEL: test_fmaximum_scalarize:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vmaxsh %xmm2, %xmm3, %xmm2
+; CHECK-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[3,3,3,3]
+; CHECK-NEXT:    vshufps {{.*#+}} xmm4 = xmm0[3,3,3,3]
+; CHECK-NEXT:    vmaxsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    vmaxsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm4 = xmm1[1,0]
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm5 = xmm0[1,0]
+; CHECK-NEXT:    vmaxsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; CHECK-NEXT:    vpsrlq $48, %xmm1, %xmm3
+; CHECK-NEXT:    vpsrlq $48, %xmm0, %xmm4
+; CHECK-NEXT:    vmaxsh %xmm3, %xmm4, %xmm3
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; CHECK-NEXT:    vmaxsh %xmm4, %xmm5, %xmm4
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT:    vmaxsh %xmm1, %xmm0, %xmm4
+; CHECK-NEXT:    vpsrld $16, %xmm1, %xmm1
+; CHECK-NEXT:    vpsrld $16, %xmm0, %xmm0
+; CHECK-NEXT:    vmaxsh %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; CHECK-NEXT:    retq
+  %r = call <8 x half> @llvm.maximum.v8f16(<8 x half> %x, <8 x half> %y)
+  ret <8 x half> %r
+}
+
+define half @test_fmaximum_nnan(half %x, half %y) {
+; CHECK-LABEL: test_fmaximum_nnan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddsh %xmm1, %xmm0, %xmm2
+; CHECK-NEXT:    vsubsh %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vfpclasssh $3, %xmm0, %k1
+; CHECK-NEXT:    vmovaps %xmm2, %xmm1
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmovsh %xmm2, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    vmaxsh %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %1 = fadd nnan half %x, %y
+  %2 = fsub nnan half %x, %y
+  %3 = tail call half @llvm.maximum.f16(half %1, half %2)
+  ret half %3
+}
+
+define half @test_fmaximum_zero(half %x, half %y) {
+; CHECK-LABEL: test_fmaximum_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vmaxsh %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    vcmpunordsh %xmm1, %xmm1, %k1
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    retq
+  %1 = tail call half @llvm.maximum.f16(half 0.0, half %y)
+  ret half %1
+}
+
+define half @test_fmaximum_nsz(half %x, half %y) "no-signed-zeros-fp-math"="true" {
+; CHECK-LABEL: test_fmaximum_nsz:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmaxsh %xmm1, %xmm0, %xmm1
+; CHECK-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1 = tail call half @llvm.maximum.f16(half %x, half %y)
+  ret half %1
+}
+
+define half @test_fmaximum_combine_cmps(half %x, half %y) {
+; CHECK-LABEL: test_fmaximum_combine_cmps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vdivsh %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vfpclasssh $3, %xmm0, %k1
+; CHECK-NEXT:    vmovaps %xmm1, %xmm2
+; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm2 {%k1}
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
+; CHECK-NEXT:    vmaxsh %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %1 = fdiv nnan half %y, %x
+  %2 = tail call half @llvm.maximum.f16(half %x, half %1)
+  ret half %2
+}
diff --git a/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll b/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
index 8ea5023908d26..40578fe746edb 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
@@ -1231,10 +1231,7 @@ define <16 x half> @test_mm256_castph128_ph256_freeze(<8 x half> %a0) nounwind {
 define <32 x half> @test_mm512_castph128_ph512_freeze(<8 x half> %a0) nounwind {
 ; CHECK-LABEL: test_mm512_castph128_ph512_freeze:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
-; CHECK-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %a1 = freeze <8 x half> poison
   %res = shufflevector <8 x half> %a0, <8 x half> %a1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/CodeGen/X86/avx512vnni-combine.ll b/llvm/test/CodeGen/X86/avx512vnni-combine.ll
index 7a8d3af368f01..7a0527be05419 100644
--- a/llvm/test/CodeGen/X86/avx512vnni-combine.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni-combine.ll
@@ -1,13 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sapphirerapids | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sapphirerapids -verify-machineinstrs | FileCheck %s
 
 define <8 x i64> @foo_reg_512(<8 x i64> %0, <8 x i64> %1, <8 x i64> %2, <8 x i64> %3, <8 x i64> %4, <8 x i64> %5) {
 ; CHECK-LABEL: foo_reg_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpdpwssd %zmm2, %zmm1, %zmm0
-; CHECK-NEXT:    vpdpwssd %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    vpdpwssd %zmm4, %zmm1, %zmm0
-; CHECK-NEXT:    vpdpwssd %zmm5, %zmm1, %zmm0
+; CHECK-NEXT:    vpmaddwd %zmm3, %zmm1, %zmm2
+; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
+; CHECK-NEXT:    vpmaddwd %zmm4, %zmm1, %zmm2
+; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
+; CHECK-NEXT:    vpmaddwd %zmm5, %zmm1, %zmm1
+; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    retq
   %7 = bitcast <8 x i64> %0 to <16 x i32>
   %8 = bitcast <8 x i64> %1 to <16 x i32>
@@ -54,9 +57,12 @@ define <8 x i64> @foo_512(i32 %0, <8 x i64> %1, <8 x i64> %2, ptr %3) {
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vpdpwssd -192(%rdi), %zmm1, %zmm0
-; CHECK-NEXT:    vpdpwssd -128(%rdi), %zmm1, %zmm0
-; CHECK-NEXT:    vpdpwssd -64(%rdi), %zmm1, %zmm0
-; CHECK-NEXT:    vpdpwssd (%rdi), %zmm1, %zmm0
+; CHECK-NEXT:    vpmaddwd -128(%rdi), %zmm1, %zmm2
+; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
+; CHECK-NEXT:    vpmaddwd -64(%rdi), %zmm1, %zmm2
+; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
+; CHECK-NEXT:    vpmaddwd (%rdi), %zmm1, %zmm2
+; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
 ; CHECK-NEXT:    addq $4, %rcx
 ; CHECK-NEXT:    addq $256, %rdi # imm = 0x100
 ; CHECK-NEXT:    cmpq %rcx, %rdx
@@ -175,12 +181,13 @@ define void @bar_512(i32 %0, ptr %1, <8 x i64> %2, ptr %3) {
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vmovdqa64 -64(%rsi,%r8), %zmm1
-; CHECK-NEXT:    vmovdqa64 (%rsi,%r8), %zmm2
-; CHECK-NEXT:    vpdpwssd -64(%rdx,%r8), %zmm0, %zmm1
-; CHECK-NEXT:    vmovdqa64 %zmm1, -64(%rsi,%r8)
-; CHECK-NEXT:    vpdpwssd (%rdx,%r8), %zmm0, %zmm2
-; CHECK-NEXT:    vmovdqa64 %zmm2, (%rsi,%r8)
+; CHECK-NEXT:    vmovdqa64 (%rsi,%r8), %zmm1
+; CHECK-NEXT:    vpmaddwd -64(%rdx,%r8), %zmm0, %zmm2
+; CHECK-NEXT:    vpaddd -64(%rsi,%r8), %zmm2, %zmm2
+; CHECK-NEXT:    vmovdqa64 %zmm2, -64(%rsi,%r8)
+; CHECK-NEXT:    vpmaddwd (%rdx,%r8), %zmm0, %zmm2
+; CHECK-NEXT:    vpaddd %zmm2, %zmm1, %zmm1
+; CHECK-NEXT:    vmovdqa64 %zmm1, (%rsi,%r8)
 ; CHECK-NEXT:    addq $2, %rcx
 ; CHECK-NEXT:    subq $-128, %r8
 ; CHECK-NEXT:    cmpq %rcx, %rdi
@@ -190,9 +197,9 @@ define void @bar_512(i32 %0, ptr %1, <8 x i64> %2, ptr %3) {
 ; CHECK-NEXT:    je .LBB2_5
 ; CHECK-NEXT:  # %bb.4:
 ; CHECK-NEXT:    shlq $6, %rcx
-; CHECK-NEXT:    vmovdqa64 (%rsi,%rcx), %zmm1
-; CHECK-NEXT:    vpdpwssd (%rdx,%rcx), %zmm0, %zmm1
-; CHECK-NEXT:    vmovdqa64 %zmm1, (%rsi,%rcx)
+; CHECK-NEXT:    vpmaddwd (%rdx,%rcx), %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd (%rsi,%rcx), %zmm0, %zmm0
+; CHECK-NEXT:    vmovdqa64 %zmm0, (%rsi,%rcx)
 ; CHECK-NEXT:  .LBB2_5:
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avxvnni-combine.ll b/llvm/test/CodeGen/X86/avxvnni-combine.ll
index fc86c80e2441b..edd27bb63c707 100644
--- a/llvm/test/CodeGen/X86/avxvnni-combine.ll
+++ b/llvm/test/CodeGen/X86/avxvnni-combine.ll
@@ -1,23 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=alderlake | FileCheck %s --check-prefixes=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sapphirerapids | FileCheck %s --check-prefixes=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=icelake-server | FileCheck %s --check-prefixes=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=alderlake -verify-machineinstrs| FileCheck %s --check-prefixes=AVX,ADL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sapphirerapids -verify-machineinstrs | FileCheck %s --check-prefixes=AVX,SPR
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=icelake-server -verify-machineinstrs | FileCheck %s --check-prefixes=AVX512
 
 define <2 x i64> @foo_reg_128(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2, <2 x i64> %3, <2 x i64> %4, <2 x i64> %5) {
 ; AVX-LABEL: foo_reg_128:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    {vex} vpdpwssd %xmm2, %xmm1, %xmm0
-; AVX-NEXT:    {vex} vpdpwssd %xmm3, %xmm1, %xmm0
-; AVX-NEXT:    {vex} vpdpwssd %xmm4, %xmm1, %xmm0
-; AVX-NEXT:    {vex} vpdpwssd %xmm5, %xmm1, %xmm0
+; AVX-NEXT:    vpmaddwd %xmm3, %xmm1, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpmaddwd %xmm4, %xmm1, %xmm2
+; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpmaddwd %xmm5, %xmm1, %xmm1
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: foo_reg_128:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
-; AVX512-NEXT:    vpdpwssd %xmm3, %xmm1, %xmm0
-; AVX512-NEXT:    vpdpwssd %xmm4, %xmm1, %xmm0
-; AVX512-NEXT:    vpdpwssd %xmm5, %xmm1, %xmm0
+; AVX512-NEXT:    vpmaddwd %xmm3, %xmm1, %xmm2
+; AVX512-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaddwd %xmm4, %xmm1, %xmm2
+; AVX512-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaddwd %xmm5, %xmm1, %xmm1
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %7 = bitcast <2 x i64> %0 to <4 x i32>
   %8 = bitcast <2 x i64> %1 to <4 x i32>
@@ -36,49 +42,99 @@ define <2 x i64> @foo_reg_128(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2, <2 x i64
 declare <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>) #1
 
 define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
-; AVX-LABEL: foo_128:
-; AVX:       # %bb.0:
-; AVX-NEXT:    testl %edi, %edi
-; AVX-NEXT:    jle .LBB1_6
-; AVX-NEXT:  # %bb.1:
-; AVX-NEXT:    movl %edi, %edx
-; AVX-NEXT:    movl %edx, %eax
-; AVX-NEXT:    andl $3, %eax
-; AVX-NEXT:    cmpl $4, %edi
-; AVX-NEXT:    jae .LBB1_7
-; AVX-NEXT:  # %bb.2:
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    jmp .LBB1_3
-; AVX-NEXT:  .LBB1_7:
-; AVX-NEXT:    andl $-4, %edx
-; AVX-NEXT:    leaq 48(%rsi), %rdi
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
-; AVX-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    {vex} vpdpwssd -48(%rdi), %xmm1, %xmm0
-; AVX-NEXT:    {vex} vpdpwssd -32(%rdi), %xmm1, %xmm0
-; AVX-NEXT:    {vex} vpdpwssd -16(%rdi), %xmm1, %xmm0
-; AVX-NEXT:    {vex} vpdpwssd (%rdi), %xmm1, %xmm0
-; AVX-NEXT:    addq $4, %rcx
-; AVX-NEXT:    addq $64, %rdi
-; AVX-NEXT:    cmpq %rcx, %rdx
-; AVX-NEXT:    jne .LBB1_8
-; AVX-NEXT:  .LBB1_3:
-; AVX-NEXT:    testq %rax, %rax
-; AVX-NEXT:    je .LBB1_6
-; AVX-NEXT:  # %bb.4: # %.preheader
-; AVX-NEXT:    shlq $4, %rcx
-; AVX-NEXT:    addq %rcx, %rsi
-; AVX-NEXT:    shlq $4, %rax
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
-; AVX-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %xmm1, %xmm0
-; AVX-NEXT:    addq $16, %rcx
-; AVX-NEXT:    cmpq %rcx, %rax
-; AVX-NEXT:    jne .LBB1_5
-; AVX-NEXT:  .LBB1_6:
-; AVX-NEXT:    retq
+; ADL-LABEL: foo_128:
+; ADL:       # %bb.0:
+; ADL-NEXT:    testl %edi, %edi
+; ADL-NEXT:    jle .LBB1_6
+; ADL-NEXT:  # %bb.1:
+; ADL-NEXT:    movl %edi, %edx
+; ADL-NEXT:    movl %edx, %eax
+; ADL-NEXT:    andl $3, %eax
+; ADL-NEXT:    cmpl $4, %edi
+; ADL-NEXT:    jae .LBB1_7
+; ADL-NEXT:  # %bb.2:
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    jmp .LBB1_3
+; ADL-NEXT:  .LBB1_7:
+; ADL-NEXT:    andl $-4, %edx
+; ADL-NEXT:    leaq 48(%rsi), %rdi
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    .p2align 4, 0x90
+; ADL-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
+; ADL-NEXT:    {vex} vpdpwssd -48(%rdi), %xmm1, %xmm0
+; ADL-NEXT:    vpmaddwd -32(%rdi), %xmm1, %xmm2
+; ADL-NEXT:    vpmaddwd -16(%rdi), %xmm1, %xmm3
+; ADL-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; ADL-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
+; ADL-NEXT:    vpmaddwd (%rdi), %xmm1, %xmm2
+; ADL-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; ADL-NEXT:    addq $4, %rcx
+; ADL-NEXT:    addq $64, %rdi
+; ADL-NEXT:    cmpq %rcx, %rdx
+; ADL-NEXT:    jne .LBB1_8
+; ADL-NEXT:  .LBB1_3:
+; ADL-NEXT:    testq %rax, %rax
+; ADL-NEXT:    je .LBB1_6
+; ADL-NEXT:  # %bb.4: # %.preheader
+; ADL-NEXT:    shlq $4, %rcx
+; ADL-NEXT:    addq %rcx, %rsi
+; ADL-NEXT:    shlq $4, %rax
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    .p2align 4, 0x90
+; ADL-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
+; ADL-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %xmm1, %xmm0
+; ADL-NEXT:    addq $16, %rcx
+; ADL-NEXT:    cmpq %rcx, %rax
+; ADL-NEXT:    jne .LBB1_5
+; ADL-NEXT:  .LBB1_6:
+; ADL-NEXT:    retq
+;
+; SPR-LABEL: foo_128:
+; SPR:       # %bb.0:
+; SPR-NEXT:    testl %edi, %edi
+; SPR-NEXT:    jle .LBB1_6
+; SPR-NEXT:  # %bb.1:
+; SPR-NEXT:    movl %edi, %edx
+; SPR-NEXT:    movl %edx, %eax
+; SPR-NEXT:    andl $3, %eax
+; SPR-NEXT:    cmpl $4, %edi
+; SPR-NEXT:    jae .LBB1_7
+; SPR-NEXT:  # %bb.2:
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    jmp .LBB1_3
+; SPR-NEXT:  .LBB1_7:
+; SPR-NEXT:    andl $-4, %edx
+; SPR-NEXT:    leaq 48(%rsi), %rdi
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    .p2align 4, 0x90
+; SPR-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
+; SPR-NEXT:    {vex} vpdpwssd -48(%rdi), %xmm1, %xmm0
+; SPR-NEXT:    vpmaddwd -32(%rdi), %xmm1, %xmm2
+; SPR-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; SPR-NEXT:    vpmaddwd -16(%rdi), %xmm1, %xmm2
+; SPR-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; SPR-NEXT:    vpmaddwd (%rdi), %xmm1, %xmm2
+; SPR-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; SPR-NEXT:    addq $4, %rcx
+; SPR-NEXT:    addq $64, %rdi
+; SPR-NEXT:    cmpq %rcx, %rdx
+; SPR-NEXT:    jne .LBB1_8
+; SPR-NEXT:  .LBB1_3:
+; SPR-NEXT:    testq %rax, %rax
+; SPR-NEXT:    je .LBB1_6
+; SPR-NEXT:  # %bb.4: # %.preheader
+; SPR-NEXT:    shlq $4, %rcx
+; SPR-NEXT:    addq %rcx, %rsi
+; SPR-NEXT:    shlq $4, %rax
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    .p2align 4, 0x90
+; SPR-NEXT:  .LBB1_5: # =>This Inner Loop Header: Depth=1
+; SPR-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %xmm1, %xmm0
+; SPR-NEXT:    addq $16, %rcx
+; SPR-NEXT:    cmpq %rcx, %rax
+; SPR-NEXT:    jne .LBB1_5
+; SPR-NEXT:  .LBB1_6:
+; SPR-NEXT:    retq
 ;
 ; AVX512-LABEL: foo_128:
 ; AVX512:       # %bb.0:
@@ -100,9 +156,12 @@ define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    .p2align 4, 0x90
 ; AVX512-NEXT:  .LBB1_8: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpdpwssd -48(%rdi), %xmm1, %xmm0
-; AVX512-NEXT:    vpdpwssd -32(%rdi), %xmm1, %xmm0
-; AVX512-NEXT:    vpdpwssd -16(%rdi), %xmm1, %xmm0
-; AVX512-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX512-NEXT:    vpmaddwd -32(%rdi), %xmm1, %xmm2
+; AVX512-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaddwd -16(%rdi), %xmm1, %xmm2
+; AVX512-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaddwd (%rdi), %xmm1, %xmm2
+; AVX512-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    addq $4, %rcx
 ; AVX512-NEXT:    addq $64, %rdi
 ; AVX512-NEXT:    cmpq %rcx, %rdx
@@ -197,44 +256,85 @@ define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {
 }
 
 define void @bar_128(i32 %0, ptr %1, <2 x i64> %2, ptr %3) {
-; AVX-LABEL: bar_128:
-; AVX:       # %bb.0:
-; AVX-NEXT:    testl %edi, %edi
-; AVX-NEXT:    jle .LBB2_5
-; AVX-NEXT:  # %bb.1:
-; AVX-NEXT:    movl %edi, %eax
-; AVX-NEXT:    cmpl $1, %edi
-; AVX-NEXT:    jne .LBB2_6
-; AVX-NEXT:  # %bb.2:
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    jmp .LBB2_3
-; AVX-NEXT:  .LBB2_6:
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    andl $-2, %edi
-; AVX-NEXT:    movl $16, %r8d
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
-; AVX-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    vmovdqa -16(%rsi,%r8), %xmm1
-; AVX-NEXT:    vmovdqa (%rsi,%r8), %xmm2
-; AVX-NEXT:    {vex} vpdpwssd -16(%rdx,%r8), %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, -16(%rsi,%r8)
-; AVX-NEXT:    {vex} vpdpwssd (%rdx,%r8), %xmm0, %xmm2
-; AVX-NEXT:    vmovdqa %xmm2, (%rsi,%r8)
-; AVX-NEXT:    addq $2, %rcx
-; AVX-NEXT:    addq $32, %r8
-; AVX-NEXT:    cmpq %rcx, %rdi
-; AVX-NEXT:    jne .LBB2_7
-; AVX-NEXT:  .LBB2_3:
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB2_5
-; AVX-NEXT:  # %bb.4:
-; AVX-NEXT:    shlq $4, %rcx
-; AVX-NEXT:    vmovdqa (%rsi,%rcx), %xmm1
-; AVX-NEXT:    {vex} vpdpwssd (%rdx,%rcx), %xmm0, %xmm1
-; AVX-NEXT:    vmovdqa %xmm1, (%rsi,%rcx)
-; AVX-NEXT:  .LBB2_5:
-; AVX-NEXT:    retq
+; ADL-LABEL: bar_128:
+; ADL:       # %bb.0:
+; ADL-NEXT:    testl %edi, %edi
+; ADL-NEXT:    jle .LBB2_5
+; ADL-NEXT:  # %bb.1:
+; ADL-NEXT:    movl %edi, %eax
+; ADL-NEXT:    cmpl $1, %edi
+; ADL-NEXT:    jne .LBB2_6
+; ADL-NEXT:  # %bb.2:
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    jmp .LBB2_3
+; ADL-NEXT:  .LBB2_6:
+; ADL-NEXT:    movl %eax, %edi
+; ADL-NEXT:    andl $-2, %edi
+; ADL-NEXT:    movl $16, %r8d
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    .p2align 4, 0x90
+; ADL-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
+; ADL-NEXT:    vmovdqa (%rsi,%r8), %xmm1
+; ADL-NEXT:    vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2
+; ADL-NEXT:    vpaddd -16(%rsi,%r8), %xmm2, %xmm2
+; ADL-NEXT:    vmovdqa %xmm2, -16(%rsi,%r8)
+; ADL-NEXT:    vpmaddwd (%rdx,%r8), %xmm0, %xmm2
+; ADL-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; ADL-NEXT:    vmovdqa %xmm1, (%rsi,%r8)
+; ADL-NEXT:    addq $2, %rcx
+; ADL-NEXT:    addq $32, %r8
+; ADL-NEXT:    cmpq %rcx, %rdi
+; ADL-NEXT:    jne .LBB2_7
+; ADL-NEXT:  .LBB2_3:
+; ADL-NEXT:    testb $1, %al
+; ADL-NEXT:    je .LBB2_5
+; ADL-NEXT:  # %bb.4:
+; ADL-NEXT:    shlq $4, %rcx
+; ADL-NEXT:    vmovdqa (%rsi,%rcx), %xmm1
+; ADL-NEXT:    {vex} vpdpwssd (%rdx,%rcx), %xmm0, %xmm1
+; ADL-NEXT:    vmovdqa %xmm1, (%rsi,%rcx)
+; ADL-NEXT:  .LBB2_5:
+; ADL-NEXT:    retq
+;
+; SPR-LABEL: bar_128:
+; SPR:       # %bb.0:
+; SPR-NEXT:    testl %edi, %edi
+; SPR-NEXT:    jle .LBB2_5
+; SPR-NEXT:  # %bb.1:
+; SPR-NEXT:    movl %edi, %eax
+; SPR-NEXT:    cmpl $1, %edi
+; SPR-NEXT:    jne .LBB2_6
+; SPR-NEXT:  # %bb.2:
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    jmp .LBB2_3
+; SPR-NEXT:  .LBB2_6:
+; SPR-NEXT:    movl %eax, %edi
+; SPR-NEXT:    andl $-2, %edi
+; SPR-NEXT:    movl $16, %r8d
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    .p2align 4, 0x90
+; SPR-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
+; SPR-NEXT:    vmovdqa (%rsi,%r8), %xmm1
+; SPR-NEXT:    vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2
+; SPR-NEXT:    vpaddd -16(%rsi,%r8), %xmm2, %xmm2
+; SPR-NEXT:    vmovdqa %xmm2, -16(%rsi,%r8)
+; SPR-NEXT:    vpmaddwd (%rdx,%r8), %xmm0, %xmm2
+; SPR-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; SPR-NEXT:    vmovdqa %xmm1, (%rsi,%r8)
+; SPR-NEXT:    addq $2, %rcx
+; SPR-NEXT:    addq $32, %r8
+; SPR-NEXT:    cmpq %rcx, %rdi
+; SPR-NEXT:    jne .LBB2_7
+; SPR-NEXT:  .LBB2_3:
+; SPR-NEXT:    testb $1, %al
+; SPR-NEXT:    je .LBB2_5
+; SPR-NEXT:  # %bb.4:
+; SPR-NEXT:    shlq $4, %rcx
+; SPR-NEXT:    vpmaddwd (%rdx,%rcx), %xmm0, %xmm0
+; SPR-NEXT:    vpaddd (%rsi,%rcx), %xmm0, %xmm0
+; SPR-NEXT:    vmovdqa %xmm0, (%rsi,%rcx)
+; SPR-NEXT:  .LBB2_5:
+; SPR-NEXT:    retq
 ;
 ; AVX512-LABEL: bar_128:
 ; AVX512:       # %bb.0:
@@ -254,12 +354,13 @@ define void @bar_128(i32 %0, ptr %1, <2 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    xorl %ecx, %ecx
 ; AVX512-NEXT:    .p2align 4, 0x90
 ; AVX512-NEXT:  .LBB2_7: # =>This Inner Loop Header: Depth=1
-; AVX512-NEXT:    vmovdqa -16(%rsi,%r8), %xmm1
-; AVX512-NEXT:    vmovdqa (%rsi,%r8), %xmm2
-; AVX512-NEXT:    vpdpwssd -16(%rdx,%r8), %xmm0, %xmm1
-; AVX512-NEXT:    vmovdqa %xmm1, -16(%rsi,%r8)
-; AVX512-NEXT:    vpdpwssd (%rdx,%r8), %xmm0, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm2, (%rsi,%r8)
+; AVX512-NEXT:    vmovdqa (%rsi,%r8), %xmm1
+; AVX512-NEXT:    vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2
+; AVX512-NEXT:    vpaddd -16(%rsi,%r8), %xmm2, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm2, -16(%rsi,%r8)
+; AVX512-NEXT:    vpmaddwd (%rdx,%r8), %xmm0, %xmm2
+; AVX512-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vmovdqa %xmm1, (%rsi,%r8)
 ; AVX512-NEXT:    addq $2, %rcx
 ; AVX512-NEXT:    addq $32, %r8
 ; AVX512-NEXT:    cmpq %rcx, %rdi
@@ -269,9 +370,9 @@ define void @bar_128(i32 %0, ptr %1, <2 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    je .LBB2_5
 ; AVX512-NEXT:  # %bb.4:
 ; AVX512-NEXT:    shlq $4, %rcx
-; AVX512-NEXT:    vmovdqa (%rsi,%rcx), %xmm1
-; AVX512-NEXT:    vpdpwssd (%rdx,%rcx), %xmm0, %xmm1
-; AVX512-NEXT:    vmovdqa %xmm1, (%rsi,%rcx)
+; AVX512-NEXT:    vpmaddwd (%rdx,%rcx), %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd (%rsi,%rcx), %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, (%rsi,%rcx)
 ; AVX512-NEXT:  .LBB2_5:
 ; AVX512-NEXT:    retq
   %5 = icmp sgt i32 %0, 0
@@ -333,17 +434,23 @@ define <4 x i64> @foo_reg_256(<4 x i64> %0, <4 x i64> %1, <4 x i64> %2, <4 x i64
 ; AVX-LABEL: foo_reg_256:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    {vex} vpdpwssd %ymm2, %ymm1, %ymm0
-; AVX-NEXT:    {vex} vpdpwssd %ymm3, %ymm1, %ymm0
-; AVX-NEXT:    {vex} vpdpwssd %ymm4, %ymm1, %ymm0
-; AVX-NEXT:    {vex} vpdpwssd %ymm5, %ymm1, %ymm0
+; AVX-NEXT:    vpmaddwd %ymm3, %ymm1, %ymm2
+; AVX-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX-NEXT:    vpmaddwd %ymm4, %ymm1, %ymm2
+; AVX-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX-NEXT:    vpmaddwd %ymm5, %ymm1, %ymm1
+; AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: foo_reg_256:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
-; AVX512-NEXT:    vpdpwssd %ymm3, %ymm1, %ymm0
-; AVX512-NEXT:    vpdpwssd %ymm4, %ymm1, %ymm0
-; AVX512-NEXT:    vpdpwssd %ymm5, %ymm1, %ymm0
+; AVX512-NEXT:    vpmaddwd %ymm3, %ymm1, %ymm2
+; AVX512-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vpmaddwd %ymm4, %ymm1, %ymm2
+; AVX512-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vpmaddwd %ymm5, %ymm1, %ymm1
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
   %7 = bitcast <4 x i64> %0 to <8 x i32>
   %8 = bitcast <4 x i64> %1 to <8 x i32>
@@ -369,49 +476,99 @@ define <4 x i64> @foo_reg_256(<4 x i64> %0, <4 x i64> %1, <4 x i64> %2, <4 x i64
 ; }
 
 define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {
-; AVX-LABEL: foo_256:
-; AVX:       # %bb.0:
-; AVX-NEXT:    testl %edi, %edi
-; AVX-NEXT:    jle .LBB4_6
-; AVX-NEXT:  # %bb.1:
-; AVX-NEXT:    movl %edi, %edx
-; AVX-NEXT:    movl %edx, %eax
-; AVX-NEXT:    andl $3, %eax
-; AVX-NEXT:    cmpl $4, %edi
-; AVX-NEXT:    jae .LBB4_7
-; AVX-NEXT:  # %bb.2:
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    jmp .LBB4_3
-; AVX-NEXT:  .LBB4_7:
-; AVX-NEXT:    andl $-4, %edx
-; AVX-NEXT:    leaq 96(%rsi), %rdi
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
-; AVX-NEXT:  .LBB4_8: # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    {vex} vpdpwssd -96(%rdi), %ymm1, %ymm0
-; AVX-NEXT:    {vex} vpdpwssd -64(%rdi), %ymm1, %ymm0
-; AVX-NEXT:    {vex} vpdpwssd -32(%rdi), %ymm1, %ymm0
-; AVX-NEXT:    {vex} vpdpwssd (%rdi), %ymm1, %ymm0
-; AVX-NEXT:    addq $4, %rcx
-; AVX-NEXT:    subq $-128, %rdi
-; AVX-NEXT:    cmpq %rcx, %rdx
-; AVX-NEXT:    jne .LBB4_8
-; AVX-NEXT:  .LBB4_3:
-; AVX-NEXT:    testq %rax, %rax
-; AVX-NEXT:    je .LBB4_6
-; AVX-NEXT:  # %bb.4: # %.preheader
-; AVX-NEXT:    shlq $5, %rcx
-; AVX-NEXT:    addq %rcx, %rsi
-; AVX-NEXT:    shlq $5, %rax
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
-; AVX-NEXT:  .LBB4_5: # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %ymm1, %ymm0
-; AVX-NEXT:    addq $32, %rcx
-; AVX-NEXT:    cmpq %rcx, %rax
-; AVX-NEXT:    jne .LBB4_5
-; AVX-NEXT:  .LBB4_6:
-; AVX-NEXT:    retq
+; ADL-LABEL: foo_256:
+; ADL:       # %bb.0:
+; ADL-NEXT:    testl %edi, %edi
+; ADL-NEXT:    jle .LBB4_6
+; ADL-NEXT:  # %bb.1:
+; ADL-NEXT:    movl %edi, %edx
+; ADL-NEXT:    movl %edx, %eax
+; ADL-NEXT:    andl $3, %eax
+; ADL-NEXT:    cmpl $4, %edi
+; ADL-NEXT:    jae .LBB4_7
+; ADL-NEXT:  # %bb.2:
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    jmp .LBB4_3
+; ADL-NEXT:  .LBB4_7:
+; ADL-NEXT:    andl $-4, %edx
+; ADL-NEXT:    leaq 96(%rsi), %rdi
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    .p2align 4, 0x90
+; ADL-NEXT:  .LBB4_8: # =>This Inner Loop Header: Depth=1
+; ADL-NEXT:    {vex} vpdpwssd -96(%rdi), %ymm1, %ymm0
+; ADL-NEXT:    vpmaddwd -64(%rdi), %ymm1, %ymm2
+; ADL-NEXT:    vpmaddwd -32(%rdi), %ymm1, %ymm3
+; ADL-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; ADL-NEXT:    vpaddd %ymm3, %ymm0, %ymm0
+; ADL-NEXT:    vpmaddwd (%rdi), %ymm1, %ymm2
+; ADL-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; ADL-NEXT:    addq $4, %rcx
+; ADL-NEXT:    subq $-128, %rdi
+; ADL-NEXT:    cmpq %rcx, %rdx
+; ADL-NEXT:    jne .LBB4_8
+; ADL-NEXT:  .LBB4_3:
+; ADL-NEXT:    testq %rax, %rax
+; ADL-NEXT:    je .LBB4_6
+; ADL-NEXT:  # %bb.4: # %.preheader
+; ADL-NEXT:    shlq $5, %rcx
+; ADL-NEXT:    addq %rcx, %rsi
+; ADL-NEXT:    shlq $5, %rax
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    .p2align 4, 0x90
+; ADL-NEXT:  .LBB4_5: # =>This Inner Loop Header: Depth=1
+; ADL-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %ymm1, %ymm0
+; ADL-NEXT:    addq $32, %rcx
+; ADL-NEXT:    cmpq %rcx, %rax
+; ADL-NEXT:    jne .LBB4_5
+; ADL-NEXT:  .LBB4_6:
+; ADL-NEXT:    retq
+;
+; SPR-LABEL: foo_256:
+; SPR:       # %bb.0:
+; SPR-NEXT:    testl %edi, %edi
+; SPR-NEXT:    jle .LBB4_6
+; SPR-NEXT:  # %bb.1:
+; SPR-NEXT:    movl %edi, %edx
+; SPR-NEXT:    movl %edx, %eax
+; SPR-NEXT:    andl $3, %eax
+; SPR-NEXT:    cmpl $4, %edi
+; SPR-NEXT:    jae .LBB4_7
+; SPR-NEXT:  # %bb.2:
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    jmp .LBB4_3
+; SPR-NEXT:  .LBB4_7:
+; SPR-NEXT:    andl $-4, %edx
+; SPR-NEXT:    leaq 96(%rsi), %rdi
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    .p2align 4, 0x90
+; SPR-NEXT:  .LBB4_8: # =>This Inner Loop Header: Depth=1
+; SPR-NEXT:    {vex} vpdpwssd -96(%rdi), %ymm1, %ymm0
+; SPR-NEXT:    vpmaddwd -64(%rdi), %ymm1, %ymm2
+; SPR-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; SPR-NEXT:    vpmaddwd -32(%rdi), %ymm1, %ymm2
+; SPR-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; SPR-NEXT:    vpmaddwd (%rdi), %ymm1, %ymm2
+; SPR-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; SPR-NEXT:    addq $4, %rcx
+; SPR-NEXT:    subq $-128, %rdi
+; SPR-NEXT:    cmpq %rcx, %rdx
+; SPR-NEXT:    jne .LBB4_8
+; SPR-NEXT:  .LBB4_3:
+; SPR-NEXT:    testq %rax, %rax
+; SPR-NEXT:    je .LBB4_6
+; SPR-NEXT:  # %bb.4: # %.preheader
+; SPR-NEXT:    shlq $5, %rcx
+; SPR-NEXT:    addq %rcx, %rsi
+; SPR-NEXT:    shlq $5, %rax
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    .p2align 4, 0x90
+; SPR-NEXT:  .LBB4_5: # =>This Inner Loop Header: Depth=1
+; SPR-NEXT:    {vex} vpdpwssd (%rsi,%rcx), %ymm1, %ymm0
+; SPR-NEXT:    addq $32, %rcx
+; SPR-NEXT:    cmpq %rcx, %rax
+; SPR-NEXT:    jne .LBB4_5
+; SPR-NEXT:  .LBB4_6:
+; SPR-NEXT:    retq
 ;
 ; AVX512-LABEL: foo_256:
 ; AVX512:       # %bb.0:
@@ -433,9 +590,12 @@ define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    .p2align 4, 0x90
 ; AVX512-NEXT:  .LBB4_8: # =>This Inner Loop Header: Depth=1
 ; AVX512-NEXT:    vpdpwssd -96(%rdi), %ymm1, %ymm0
-; AVX512-NEXT:    vpdpwssd -64(%rdi), %ymm1, %ymm0
-; AVX512-NEXT:    vpdpwssd -32(%rdi), %ymm1, %ymm0
-; AVX512-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX512-NEXT:    vpmaddwd -64(%rdi), %ymm1, %ymm2
+; AVX512-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vpmaddwd -32(%rdi), %ymm1, %ymm2
+; AVX512-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vpmaddwd (%rdi), %ymm1, %ymm2
+; AVX512-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    addq $4, %rcx
 ; AVX512-NEXT:    subq $-128, %rdi
 ; AVX512-NEXT:    cmpq %rcx, %rdx
@@ -537,45 +697,87 @@ declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
 ;     }
 ; }
 define void @bar_256(i32 %0, ptr %1, <4 x i64> %2, ptr %3) {
-; AVX-LABEL: bar_256:
-; AVX:       # %bb.0:
-; AVX-NEXT:    testl %edi, %edi
-; AVX-NEXT:    jle .LBB5_5
-; AVX-NEXT:  # %bb.1:
-; AVX-NEXT:    movl %edi, %eax
-; AVX-NEXT:    cmpl $1, %edi
-; AVX-NEXT:    jne .LBB5_6
-; AVX-NEXT:  # %bb.2:
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    jmp .LBB5_3
-; AVX-NEXT:  .LBB5_6:
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    andl $-2, %edi
-; AVX-NEXT:    movl $32, %r8d
-; AVX-NEXT:    xorl %ecx, %ecx
-; AVX-NEXT:    .p2align 4, 0x90
-; AVX-NEXT:  .LBB5_7: # =>This Inner Loop Header: Depth=1
-; AVX-NEXT:    vmovdqa -32(%rsi,%r8), %ymm1
-; AVX-NEXT:    vmovdqa (%rsi,%r8), %ymm2
-; AVX-NEXT:    {vex} vpdpwssd -32(%rdx,%r8), %ymm0, %ymm1
-; AVX-NEXT:    vmovdqa %ymm1, -32(%rsi,%r8)
-; AVX-NEXT:    {vex} vpdpwssd (%rdx,%r8), %ymm0, %ymm2
-; AVX-NEXT:    vmovdqa %ymm2, (%rsi,%r8)
-; AVX-NEXT:    addq $2, %rcx
-; AVX-NEXT:    addq $64, %r8
-; AVX-NEXT:    cmpq %rcx, %rdi
-; AVX-NEXT:    jne .LBB5_7
-; AVX-NEXT:  .LBB5_3:
-; AVX-NEXT:    testb $1, %al
-; AVX-NEXT:    je .LBB5_5
-; AVX-NEXT:  # %bb.4:
-; AVX-NEXT:    shlq $5, %rcx
-; AVX-NEXT:    vmovdqa (%rsi,%rcx), %ymm1
-; AVX-NEXT:    {vex} vpdpwssd (%rdx,%rcx), %ymm0, %ymm1
-; AVX-NEXT:    vmovdqa %ymm1, (%rsi,%rcx)
-; AVX-NEXT:  .LBB5_5:
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; ADL-LABEL: bar_256:
+; ADL:       # %bb.0:
+; ADL-NEXT:    testl %edi, %edi
+; ADL-NEXT:    jle .LBB5_5
+; ADL-NEXT:  # %bb.1:
+; ADL-NEXT:    movl %edi, %eax
+; ADL-NEXT:    cmpl $1, %edi
+; ADL-NEXT:    jne .LBB5_6
+; ADL-NEXT:  # %bb.2:
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    jmp .LBB5_3
+; ADL-NEXT:  .LBB5_6:
+; ADL-NEXT:    movl %eax, %edi
+; ADL-NEXT:    andl $-2, %edi
+; ADL-NEXT:    movl $32, %r8d
+; ADL-NEXT:    xorl %ecx, %ecx
+; ADL-NEXT:    .p2align 4, 0x90
+; ADL-NEXT:  .LBB5_7: # =>This Inner Loop Header: Depth=1
+; ADL-NEXT:    vmovdqa (%rsi,%r8), %ymm1
+; ADL-NEXT:    vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2
+; ADL-NEXT:    vpaddd -32(%rsi,%r8), %ymm2, %ymm2
+; ADL-NEXT:    vmovdqa %ymm2, -32(%rsi,%r8)
+; ADL-NEXT:    vpmaddwd (%rdx,%r8), %ymm0, %ymm2
+; ADL-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; ADL-NEXT:    vmovdqa %ymm1, (%rsi,%r8)
+; ADL-NEXT:    addq $2, %rcx
+; ADL-NEXT:    addq $64, %r8
+; ADL-NEXT:    cmpq %rcx, %rdi
+; ADL-NEXT:    jne .LBB5_7
+; ADL-NEXT:  .LBB5_3:
+; ADL-NEXT:    testb $1, %al
+; ADL-NEXT:    je .LBB5_5
+; ADL-NEXT:  # %bb.4:
+; ADL-NEXT:    shlq $5, %rcx
+; ADL-NEXT:    vmovdqa (%rsi,%rcx), %ymm1
+; ADL-NEXT:    {vex} vpdpwssd (%rdx,%rcx), %ymm0, %ymm1
+; ADL-NEXT:    vmovdqa %ymm1, (%rsi,%rcx)
+; ADL-NEXT:  .LBB5_5:
+; ADL-NEXT:    vzeroupper
+; ADL-NEXT:    retq
+;
+; SPR-LABEL: bar_256:
+; SPR:       # %bb.0:
+; SPR-NEXT:    testl %edi, %edi
+; SPR-NEXT:    jle .LBB5_5
+; SPR-NEXT:  # %bb.1:
+; SPR-NEXT:    movl %edi, %eax
+; SPR-NEXT:    cmpl $1, %edi
+; SPR-NEXT:    jne .LBB5_6
+; SPR-NEXT:  # %bb.2:
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    jmp .LBB5_3
+; SPR-NEXT:  .LBB5_6:
+; SPR-NEXT:    movl %eax, %edi
+; SPR-NEXT:    andl $-2, %edi
+; SPR-NEXT:    movl $32, %r8d
+; SPR-NEXT:    xorl %ecx, %ecx
+; SPR-NEXT:    .p2align 4, 0x90
+; SPR-NEXT:  .LBB5_7: # =>This Inner Loop Header: Depth=1
+; SPR-NEXT:    vmovdqa (%rsi,%r8), %ymm1
+; SPR-NEXT:    vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2
+; SPR-NEXT:    vpaddd -32(%rsi,%r8), %ymm2, %ymm2
+; SPR-NEXT:    vmovdqa %ymm2, -32(%rsi,%r8)
+; SPR-NEXT:    vpmaddwd (%rdx,%r8), %ymm0, %ymm2
+; SPR-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; SPR-NEXT:    vmovdqa %ymm1, (%rsi,%r8)
+; SPR-NEXT:    addq $2, %rcx
+; SPR-NEXT:    addq $64, %r8
+; SPR-NEXT:    cmpq %rcx, %rdi
+; SPR-NEXT:    jne .LBB5_7
+; SPR-NEXT:  .LBB5_3:
+; SPR-NEXT:    testb $1, %al
+; SPR-NEXT:    je .LBB5_5
+; SPR-NEXT:  # %bb.4:
+; SPR-NEXT:    shlq $5, %rcx
+; SPR-NEXT:    vpmaddwd (%rdx,%rcx), %ymm0, %ymm0
+; SPR-NEXT:    vpaddd (%rsi,%rcx), %ymm0, %ymm0
+; SPR-NEXT:    vmovdqa %ymm0, (%rsi,%rcx)
+; SPR-NEXT:  .LBB5_5:
+; SPR-NEXT:    vzeroupper
+; SPR-NEXT:    retq
 ;
 ; AVX512-LABEL: bar_256:
 ; AVX512:       # %bb.0:
@@ -595,12 +797,13 @@ define void @bar_256(i32 %0, ptr %1, <4 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    xorl %ecx, %ecx
 ; AVX512-NEXT:    .p2align 4, 0x90
 ; AVX512-NEXT:  .LBB5_7: # =>This Inner Loop Header: Depth=1
-; AVX512-NEXT:    vmovdqa -32(%rsi,%r8), %ymm1
-; AVX512-NEXT:    vmovdqa (%rsi,%r8), %ymm2
-; AVX512-NEXT:    vpdpwssd -32(%rdx,%r8), %ymm0, %ymm1
-; AVX512-NEXT:    vmovdqa %ymm1, -32(%rsi,%r8)
-; AVX512-NEXT:    vpdpwssd (%rdx,%r8), %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa %ymm2, (%rsi,%r8)
+; AVX512-NEXT:    vmovdqa (%rsi,%r8), %ymm1
+; AVX512-NEXT:    vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2
+; AVX512-NEXT:    vpaddd -32(%rsi,%r8), %ymm2, %ymm2
+; AVX512-NEXT:    vmovdqa %ymm2, -32(%rsi,%r8)
+; AVX512-NEXT:    vpmaddwd (%rdx,%r8), %ymm0, %ymm2
+; AVX512-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; AVX512-NEXT:    vmovdqa %ymm1, (%rsi,%r8)
 ; AVX512-NEXT:    addq $2, %rcx
 ; AVX512-NEXT:    addq $64, %r8
 ; AVX512-NEXT:    cmpq %rcx, %rdi
@@ -610,9 +813,9 @@ define void @bar_256(i32 %0, ptr %1, <4 x i64> %2, ptr %3) {
 ; AVX512-NEXT:    je .LBB5_5
 ; AVX512-NEXT:  # %bb.4:
 ; AVX512-NEXT:    shlq $5, %rcx
-; AVX512-NEXT:    vmovdqa (%rsi,%rcx), %ymm1
-; AVX512-NEXT:    vpdpwssd (%rdx,%rcx), %ymm0, %ymm1
-; AVX512-NEXT:    vmovdqa %ymm1, (%rsi,%rcx)
+; AVX512-NEXT:    vpmaddwd (%rdx,%rcx), %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd (%rsi,%rcx), %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa %ymm0, (%rsi,%rcx)
 ; AVX512-NEXT:  .LBB5_5:
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/build-vector-128.ll b/llvm/test/CodeGen/X86/build-vector-128.ll
index df664a92425a0..30f55a1d41922 100644
--- a/llvm/test/CodeGen/X86/build-vector-128.ll
+++ b/llvm/test/CodeGen/X86/build-vector-128.ll
@@ -556,9 +556,8 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
 ; SSE2-32-LABEL: pr60168_buildvector_of_zeros_and_undef:
 ; SSE2-32:       # %bb.0:
 ; SSE2-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; SSE2-32-NEXT:    xorpd %xmm1, %xmm1
-; SSE2-32-NEXT:    movsd {{.*#+}} xmm1 = xmm1[0,1]
 ; SSE2-32-NEXT:    paddd %xmm0, %xmm0
+; SSE2-32-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-32-NEXT:    psubd %xmm0, %xmm1
 ; SSE2-32-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-32-NEXT:    psrad $31, %xmm0
@@ -569,9 +568,8 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
 ;
 ; SSE2-64-LABEL: pr60168_buildvector_of_zeros_and_undef:
 ; SSE2-64:       # %bb.0:
-; SSE2-64-NEXT:    xorpd %xmm1, %xmm1
-; SSE2-64-NEXT:    movsd {{.*#+}} xmm1 = xmm1[0,1]
 ; SSE2-64-NEXT:    paddd %xmm0, %xmm0
+; SSE2-64-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-64-NEXT:    psubd %xmm0, %xmm1
 ; SSE2-64-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-64-NEXT:    psrad $31, %xmm0
diff --git a/llvm/test/CodeGen/X86/cmov.ll b/llvm/test/CodeGen/X86/cmov.ll
index dbe85eced6a59..374e75967d52f 100644
--- a/llvm/test/CodeGen/X86/cmov.ll
+++ b/llvm/test/CodeGen/X86/cmov.ll
@@ -78,7 +78,6 @@ define void @test3(i64 %a, i64 %b, i1 %p) nounwind {
 define i1 @test4() nounwind {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movsbl g_3(%rip), %edx
 ; CHECK-NEXT:    movzbl %dl, %ecx
 ; CHECK-NEXT:    shrl $7, %ecx
@@ -91,6 +90,7 @@ define i1 @test4() nounwind {
 ; CHECK-NEXT:  # %bb.1: # %bb.i.i.i
 ; CHECK-NEXT:    movzbl g_100(%rip), %ecx
 ; CHECK-NEXT:  .LBB3_2: # %func_4.exit.i
+; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    xorl %esi, %esi
 ; CHECK-NEXT:    testb %dl, %dl
 ; CHECK-NEXT:    setne %bl
diff --git a/llvm/test/CodeGen/X86/combine-srl.ll b/llvm/test/CodeGen/X86/combine-srl.ll
index 7bd2683286ab7..d2704e8f7af3a 100644
--- a/llvm/test/CodeGen/X86/combine-srl.ll
+++ b/llvm/test/CodeGen/X86/combine-srl.ll
@@ -346,23 +346,18 @@ define <4 x i32> @combine_vec_lshr_lzcnt_bit1(<4 x i32> %x) {
 ; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; SSE-NEXT:    movdqa %xmm1, %xmm2
 ; SSE-NEXT:    pshufb %xmm0, %xmm2
-; SSE-NEXT:    movdqa %xmm0, %xmm3
-; SSE-NEXT:    psrlw $4, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pshufb %xmm3, %xmm1
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm3
-; SSE-NEXT:    pand %xmm2, %xmm3
-; SSE-NEXT:    paddb %xmm1, %xmm3
-; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    pcmpeqb %xmm4, %xmm1
-; SSE-NEXT:    psrlw $8, %xmm1
-; SSE-NEXT:    pand %xmm3, %xmm1
-; SSE-NEXT:    psrlw $8, %xmm3
-; SSE-NEXT:    paddw %xmm1, %xmm3
-; SSE-NEXT:    pcmpeqw %xmm4, %xmm0
+; SSE-NEXT:    psrlw $4, %xmm0
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    pshufb %xmm0, %xmm1
+; SSE-NEXT:    pcmpeqb %xmm3, %xmm0
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    paddb %xmm1, %xmm0
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE-NEXT:    pand %xmm0, %xmm1
+; SSE-NEXT:    psrlw $8, %xmm0
+; SSE-NEXT:    paddw %xmm1, %xmm0
+; SSE-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
 ; SSE-NEXT:    psrld $16, %xmm0
-; SSE-NEXT:    pand %xmm3, %xmm0
-; SSE-NEXT:    psrld $16, %xmm3
 ; SSE-NEXT:    paddd %xmm3, %xmm0
 ; SSE-NEXT:    psrld $5, %xmm0
 ; SSE-NEXT:    retq
@@ -372,22 +367,18 @@ define <4 x i32> @combine_vec_lshr_lzcnt_bit1(<4 x i32> %x) {
 ; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
 ; AVX-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm3
-; AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpcmpeqb %xmm4, %xmm3, %xmm5
-; AVX-NEXT:    vpand %xmm5, %xmm2, %xmm2
-; AVX-NEXT:    vpshufb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vpaddb %xmm1, %xmm2, %xmm1
-; AVX-NEXT:    vpcmpeqb %xmm4, %xmm0, %xmm2
-; AVX-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm2
-; AVX-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm4
+; AVX-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
 ; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0
-; AVX-NEXT:    vpand %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrld $5, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = and <4 x i32> %x, <i32 4, i32 32, i32 64, i32 128>
diff --git a/llvm/test/CodeGen/X86/copy-eflags.ll b/llvm/test/CodeGen/X86/copy-eflags.ll
index 15833fa3f0c08..6af80860401af 100644
--- a/llvm/test/CodeGen/X86/copy-eflags.ll
+++ b/llvm/test/CodeGen/X86/copy-eflags.ll
@@ -43,7 +43,6 @@ define dso_local i32 @test1() nounwind {
 ;
 ; X64-LABEL: test1:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    movzbl b(%rip), %ecx
 ; X64-NEXT:    leal 1(%rcx), %eax
 ; X64-NEXT:    movb %al, b(%rip)
@@ -57,11 +56,12 @@ define dso_local i32 @test1() nounwind {
 ; X64-NEXT:    testb %dl, %dl
 ; X64-NEXT:    jne .LBB0_2
 ; X64-NEXT:  # %bb.1: # %if.then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    movsbl %al, %edi
 ; X64-NEXT:    callq external
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB0_2: # %if.end
 ; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
 entry:
   %bval = load i8, ptr @b
diff --git a/llvm/test/CodeGen/X86/csr-split.ll b/llvm/test/CodeGen/X86/csr-split.ll
index 81057d80ffdcb..460e705d1847a 100644
--- a/llvm/test/CodeGen/X86/csr-split.ll
+++ b/llvm/test/CodeGen/X86/csr-split.ll
@@ -9,18 +9,15 @@
 define dso_local signext i32 @test1(ptr %b) local_unnamed_addr  {
 ; CHECK-LABEL: test1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset %rbx, -16
 ; CHECK-NEXT:    movslq a(%rip), %rax
 ; CHECK-NEXT:    cmpq %rdi, %rax
 ; CHECK-NEXT:    je .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %if.end
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB0_2: # %if.then
+; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbx, -16
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    callq callVoid@PLT
 ; CHECK-NEXT:    movq %rbx, %rdi
diff --git a/llvm/test/CodeGen/X86/expand-vp-fp-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-fp-intrinsics.ll
index 1b5a8f5221b9c..b2c336576fc8b 100644
--- a/llvm/test/CodeGen/X86/expand-vp-fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-fp-intrinsics.ll
@@ -156,47 +156,251 @@ define void @vp_frem_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp)
 }
 declare <4 x float> @llvm.vp.frem.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32)
 
-; TODO: @llvm.vp.fabs.v4f32
-;define void @vp_fabs_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
-;  %res = call <4 x float> @llvm.vp.fabs.v4f32(<4 x float> %a0, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
-;  store <4 x float> %res, ptr %out
-;  ret void
-;}
-;declare <4 x float> @llvm.vp.fabs.v4f32(<4 x float>, <4 x i1>, i32)
+define void @vp_fabs_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
+; SSE-LABEL: vp_fabs_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: vp_fabs_v4f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: vp_fabs_v4f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
+; AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: vp_fabs_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    retq
+  %res = call <4 x float> @llvm.vp.fabs.v4f32(<4 x float> %a0, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
+  store <4 x float> %res, ptr %out
+  ret void
+}
+declare <4 x float> @llvm.vp.fabs.v4f32(<4 x float>, <4 x i1>, i32)
 
-; TODO: @llvm.vp.sqrt.v4f32
-;define void @vp_sqrt_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
-;  %res = call <4 x float> @llvm.vp.sqrt.v4f32(<4 x float> %a0, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
-;  store <4 x float> %res, ptr %out
-;  ret void
-;}
-;declare <4 x float> @llvm.vp.sqrt.v4f32(<4 x float>, <4 x i1>, i32)
+define void @vp_sqrt_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
+; SSE-LABEL: vp_sqrt_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    sqrtps %xmm0, %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: vp_sqrt_v4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vsqrtps %xmm0, %xmm0
+; AVX-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX-NEXT:    retq
+  %res = call <4 x float> @llvm.vp.sqrt.v4f32(<4 x float> %a0, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
+  store <4 x float> %res, ptr %out
+  ret void
+}
+declare <4 x float> @llvm.vp.sqrt.v4f32(<4 x float>, <4 x i1>, i32)
 
-; TODO: @llvm.vp.fneg.v4f32
-;define void @vp_fneg_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
-;  %res = call <4 x float> @llvm.vp.fneg.v4f32(<4 x float> %a0, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
-;  store <4 x float> %res, ptr %out
-;  ret void
-;}
-;declare <4 x float> @llvm.vp.fneg.v4f32(<4 x float>, <4 x i1>, i32)
+define void @vp_fneg_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind {
+; SSE-LABEL: vp_fneg_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: vp_fneg_v4f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: vp_fneg_v4f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
+; AVX2-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: vp_fneg_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    retq
+  %res = call <4 x float> @llvm.vp.fneg.v4f32(<4 x float> %a0, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
+  store <4 x float> %res, ptr %out
+  ret void
+}
+declare <4 x float> @llvm.vp.fneg.v4f32(<4 x float>, <4 x i1>, i32)
 
-; TODO: @llvm.vp.fma.v4f32
-;define void @vp_fma_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind {
-;  %res = call <4 x float> @llvm.vp.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 4)
-;  store <4 x float> %res, ptr %out
-;  ret void
-;}
-;declare <4 x float> @llvm.vp.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32)
+define void @vp_fma_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind {
+; SSE-LABEL: vp_fma_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pushq %rbx
+; SSE-NEXT:    subq $64, %rsp
+; SSE-NEXT:    movq %rdi, %rbx
+; SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; SSE-NEXT:    movaps %xmm1, %xmm2
+; SSE-NEXT:    callq fmaf@PLT
+; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; SSE-NEXT:    movaps %xmm1, %xmm2
+; SSE-NEXT:    callq fmaf@PLT
+; SSE-NEXT:    unpcklps (%rsp), %xmm0 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-NEXT:    movaps %xmm1, %xmm2
+; SSE-NEXT:    callq fmaf@PLT
+; SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE-NEXT:    movaps %xmm1, %xmm2
+; SSE-NEXT:    callq fmaf@PLT
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT:    unpcklpd (%rsp), %xmm1 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm1 = xmm1[0],mem[0]
+; SSE-NEXT:    movaps %xmm1, (%rbx)
+; SSE-NEXT:    addq $64, %rsp
+; SSE-NEXT:    popq %rbx
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: vp_fma_v4f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    pushq %rbx
+; AVX1-NEXT:    subq $48, %rsp
+; AVX1-NEXT:    movq %rdi, %rbx
+; AVX1-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-NEXT:    callq fmaf@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = mem[1,1,3,3]
+; AVX1-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = mem[1,1,3,3]
+; AVX1-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-NEXT:    callq fmaf@PLT
+; AVX1-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = mem[1,0]
+; AVX1-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = mem[1,0]
+; AVX1-NEXT:    vmovapd %xmm1, %xmm2
+; AVX1-NEXT:    callq fmaf@PLT
+; AVX1-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = mem[3,3,3,3]
+; AVX1-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = mem[3,3,3,3]
+; AVX1-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-NEXT:    callq fmaf@PLT
+; AVX1-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    vmovaps %xmm0, (%rbx)
+; AVX1-NEXT:    addq $48, %rsp
+; AVX1-NEXT:    popq %rbx
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: vp_fma_v4f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    pushq %rbx
+; AVX2-NEXT:    subq $48, %rsp
+; AVX2-NEXT:    movq %rdi, %rbx
+; AVX2-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT:    vmovaps %xmm1, %xmm2
+; AVX2-NEXT:    callq fmaf@PLT
+; AVX2-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT:    # xmm0 = mem[1,1,3,3]
+; AVX2-NEXT:    vmovshdup {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-NEXT:    # xmm1 = mem[1,1,3,3]
+; AVX2-NEXT:    vmovaps %xmm1, %xmm2
+; AVX2-NEXT:    callq fmaf@PLT
+; AVX2-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3]
+; AVX2-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT:    # xmm0 = mem[1,0]
+; AVX2-NEXT:    vpermilpd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-NEXT:    # xmm1 = mem[1,0]
+; AVX2-NEXT:    vmovapd %xmm1, %xmm2
+; AVX2-NEXT:    callq fmaf@PLT
+; AVX2-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0],xmm1[3]
+; AVX2-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-NEXT:    # xmm0 = mem[3,3,3,3]
+; AVX2-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-NEXT:    # xmm1 = mem[3,3,3,3]
+; AVX2-NEXT:    vmovaps %xmm1, %xmm2
+; AVX2-NEXT:    callq fmaf@PLT
+; AVX2-NEXT:    vmovaps (%rsp), %xmm1 # 16-byte Reload
+; AVX2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX2-NEXT:    vmovaps %xmm0, (%rbx)
+; AVX2-NEXT:    addq $48, %rsp
+; AVX2-NEXT:    popq %rbx
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: vp_fma_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm1
+; AVX512-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX512-NEXT:    retq
+  %res = call <4 x float> @llvm.vp.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 4)
+  store <4 x float> %res, ptr %out
+  ret void
+}
+declare <4 x float> @llvm.vp.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32)
 
-; TODO: @llvm.vp.fmuladd.v4f32
-;define void @vp_fmuladd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind {
-;  %res = call <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 4)
-;  store <4 x float> %res, ptr %out
-;  ret void
-;}
-;declare <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32)
+define void @vp_fmuladd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind {
+; SSE-LABEL: vp_fmuladd_v4f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulps %xmm1, %xmm0
+; SSE-NEXT:    addps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm0, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: vp_fmuladd_v4f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: vp_fmuladd_v4f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: vp_fmuladd_v4f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm1
+; AVX512-NEXT:    vmovaps %xmm0, (%rdi)
+; AVX512-NEXT:    retq
+  %res = call <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 4)
+  store <4 x float> %res, ptr %out
+  ret void
+}
+declare <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32)
 
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX1: {{.*}}
-; AVX2: {{.*}}
-; AVX512: {{.*}}
diff --git a/llvm/test/CodeGen/X86/extract-fp.ll b/llvm/test/CodeGen/X86/extract-fp.ll
index 1ef32156a348b..58aaab9c83f95 100644
--- a/llvm/test/CodeGen/X86/extract-fp.ll
+++ b/llvm/test/CodeGen/X86/extract-fp.ll
@@ -105,17 +105,33 @@ define double @ext_minnum_v2f64(<2 x double> %x) nounwind {
   ret double %r
 }
 
-;define double @ext_maximum_v4f64(<2 x double> %x) nounwind {
-;  %v = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double> <double 42.0, double 43.0>)
-;  %r = extractelement <2 x double> %v, i32 1
-;  ret double %r
-;}
+define double @ext_maximum_v4f64(<2 x double> %x) nounwind {
+; CHECK-LABEL: ext_maximum_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    maxsd %xmm0, %xmm1
+; CHECK-NEXT:    movapd %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %v = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double> <double 42.0, double 43.0>)
+  %r = extractelement <2 x double> %v, i32 1
+  ret double %r
+}
 
-;define float @ext_minimum_v4f32(<4 x float> %x) nounwind {
-;  %v = call <4 x float> @llvm.minimum.v4f32(<4 x float> %x, <4 x float> <float 0.0, float 1.0, float 2.0, float 42.0>)
-;  %r = extractelement <4 x float> %v, i32 1
-;  ret float %r
-;}
+define float @ext_minimum_v4f32(<4 x float> %x) nounwind {
+; CHECK-LABEL: ext_minimum_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    minss %xmm0, %xmm1
+; CHECK-NEXT:    movaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %v = call <4 x float> @llvm.minimum.v4f32(<4 x float> %x, <4 x float> <float 0.0, float 1.0, float 2.0, float 42.0>)
+  %r = extractelement <4 x float> %v, i32 1
+  ret float %r
+}
 
 declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>)
 declare <2 x double> @llvm.minnum.v2f64(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.maximum.v2f64(<2 x double>, <2 x double>)
+declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
diff --git a/llvm/test/CodeGen/X86/extractelement-fp.ll b/llvm/test/CodeGen/X86/extractelement-fp.ll
index 816a18bff7e9d..5bc6d00022e23 100644
--- a/llvm/test/CodeGen/X86/extractelement-fp.ll
+++ b/llvm/test/CodeGen/X86/extractelement-fp.ll
@@ -672,29 +672,189 @@ define double @fminnum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
   ret double %r
 }
 
-;define float @fmaximum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
-;  %v = call <4 x float> @llvm.maximum.v4f32(<4 x float> %x, <4 x float> %y)
-;  %r = extractelement <4 x float> %v, i32 0
-;  ret float %r
-;}
+define float @fmaximum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
+; X64-LABEL: fmaximum_v4f32:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    js .LBB30_1
+; X64-NEXT:  # %bb.2:
+; X64-NEXT:    vmovdqa %xmm0, %xmm2
+; X64-NEXT:    jmp .LBB30_3
+; X64-NEXT:  .LBB30_1:
+; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    vmovdqa %xmm0, %xmm1
+; X64-NEXT:  .LBB30_3:
+; X64-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vcmpunordss %xmm1, %xmm1, %xmm2
+; X64-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: fmaximum_v4f32:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovd %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB30_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    jmp .LBB30_3
+; X86-NEXT:  .LBB30_1:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:  .LBB30_3:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vcmpunordss %xmm1, %xmm1, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %v = call <4 x float> @llvm.maximum.v4f32(<4 x float> %x, <4 x float> %y)
+  %r = extractelement <4 x float> %v, i32 0
+  ret float %r
+}
 
-;define double @fmaximum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
-;  %v = call <4 x double> @llvm.maximum.v4f64(<4 x double> %x, <4 x double> %y)
-;  %r = extractelement <4 x double> %v, i32 0
-;  ret double %r
-;}
+define double @fmaximum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
+; X64-LABEL: fmaximum_v4f64:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovq %xmm0, %rax
+; X64-NEXT:    testq %rax, %rax
+; X64-NEXT:    js .LBB31_1
+; X64-NEXT:  # %bb.2:
+; X64-NEXT:    vmovdqa %xmm0, %xmm2
+; X64-NEXT:    jmp .LBB31_3
+; X64-NEXT:  .LBB31_1:
+; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    vmovdqa %xmm0, %xmm1
+; X64-NEXT:  .LBB31_3:
+; X64-NEXT:    vmaxsd %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vcmpunordsd %xmm1, %xmm1, %xmm2
+; X64-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: fmaximum_v4f64:
+; X86:       # %bb.0:
+; X86-NEXT:    vextractps $1, %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB31_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovapd %xmm0, %xmm2
+; X86-NEXT:    jmp .LBB31_3
+; X86-NEXT:  .LBB31_1:
+; X86-NEXT:    vmovapd %xmm1, %xmm2
+; X86-NEXT:    vmovapd %xmm0, %xmm1
+; X86-NEXT:  .LBB31_3:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vmaxsd %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vcmpunordsd %xmm1, %xmm1, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+  %v = call <4 x double> @llvm.maximum.v4f64(<4 x double> %x, <4 x double> %y)
+  %r = extractelement <4 x double> %v, i32 0
+  ret double %r
+}
 
-;define float @fminimum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
-;  %v = call <4 x float> @llvm.minimum.v4f32(<4 x float> %x, <4 x float> %y)
-;  %r = extractelement <4 x float> %v, i32 0
-;  ret float %r
-;}
+define float @fminimum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
+; X64-LABEL: fminimum_v4f32:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    js .LBB32_1
+; X64-NEXT:  # %bb.2:
+; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    jmp .LBB32_3
+; X64-NEXT:  .LBB32_1:
+; X64-NEXT:    vmovdqa %xmm0, %xmm2
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
+; X64-NEXT:  .LBB32_3:
+; X64-NEXT:    vminss %xmm2, %xmm0, %xmm1
+; X64-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: fminimum_v4f32:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovd %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB32_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB32_3
+; X86-NEXT:  .LBB32_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB32_3:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vminss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %v = call <4 x float> @llvm.minimum.v4f32(<4 x float> %x, <4 x float> %y)
+  %r = extractelement <4 x float> %v, i32 0
+  ret float %r
+}
 
-;define double @fminimum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
-;  %v = call <4 x double> @llvm.minimum.v4f64(<4 x double> %x, <4 x double> %y)
-;  %r = extractelement <4 x double> %v, i32 0
-;  ret double %r
-;}
+define double @fminimum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
+; X64-LABEL: fminimum_v4f64:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovq %xmm0, %rax
+; X64-NEXT:    testq %rax, %rax
+; X64-NEXT:    js .LBB33_1
+; X64-NEXT:  # %bb.2:
+; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    jmp .LBB33_3
+; X64-NEXT:  .LBB33_1:
+; X64-NEXT:    vmovdqa %xmm0, %xmm2
+; X64-NEXT:    vmovdqa %xmm1, %xmm0
+; X64-NEXT:  .LBB33_3:
+; X64-NEXT:    vminsd %xmm2, %xmm0, %xmm1
+; X64-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
+; X64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: fminimum_v4f64:
+; X86:       # %bb.0:
+; X86-NEXT:    vextractps $1, %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovapd %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_3
+; X86-NEXT:  .LBB33_1:
+; X86-NEXT:    vmovapd %xmm0, %xmm2
+; X86-NEXT:    vmovapd %xmm1, %xmm0
+; X86-NEXT:  .LBB33_3:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vminsd %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+  %v = call <4 x double> @llvm.minimum.v4f64(<4 x double> %x, <4 x double> %y)
+  %r = extractelement <4 x double> %v, i32 0
+  ret double %r
+}
 
 define float @maxps_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
 ; X64-LABEL: maxps_v4f32:
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
new file mode 100644
index 0000000000000..94e3afdbbec69
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -0,0 +1,1032 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2     | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx      | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512,AVX512DQ
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx        | FileCheck %s --check-prefixes=X86
+
+declare float @llvm.maximum.f32(float, float)
+declare double @llvm.maximum.f64(double, double)
+declare float @llvm.minimum.f32(float, float)
+declare double @llvm.minimum.f64(double, double)
+declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
+declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
+
+;
+; fmaximum
+;
+
+define float @test_fmaximum(float %x, float %y) nounwind {
+; SSE2-LABEL: test_fmaximum:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    js .LBB0_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:  .LBB0_2:
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm3, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm3, %xmm4
+; SSE2-NEXT:    js .LBB0_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:  .LBB0_4:
+; SSE2-NEXT:    maxss %xmm1, %xmm3
+; SSE2-NEXT:    andnps %xmm3, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB0_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:    jmp .LBB0_3
+; AVX1-NEXT:  .LBB0_1:
+; AVX1-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:  .LBB0_3:
+; AVX1-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    testl %eax, %eax
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm2, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB0_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB0_3
+; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB0_3:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.maximum.f32(float %x, float %y)
+  ret float %1
+}
+
+define <4 x float> @test_fmaximum_scalarize(<4 x float> %x, <4 x float> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" {
+; SSE2-LABEL: test_fmaximum_scalarize:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm1, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3],xmm1[3,3]
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm0[3,3]
+; SSE2-NEXT:    maxss %xmm2, %xmm3
+; SSE2-NEXT:    movaps %xmm1, %xmm2
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
+; SSE2-NEXT:    maxss %xmm2, %xmm4
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    maxss %xmm1, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT:    maxss %xmm1, %xmm0
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fmaximum_scalarize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmaxss %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX-NEXT:    vmaxss %xmm3, %xmm4, %xmm3
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
+; AVX-NEXT:    vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; AVX-NEXT:    vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; AVX-NEXT:    vmaxss %xmm3, %xmm4, %xmm3
+; AVX-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_scalarize:
+; X86:       # %bb.0:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm2
+; X86-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; X86-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; X86-NEXT:    vmaxss %xmm3, %xmm4, %xmm3
+; X86-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[2,3]
+; X86-NEXT:    vshufpd {{.*#+}} xmm3 = xmm1[1,0]
+; X86-NEXT:    vshufpd {{.*#+}} xmm4 = xmm0[1,0]
+; X86-NEXT:    vmaxss %xmm3, %xmm4, %xmm3
+; X86-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],xmm3[0],xmm2[3]
+; X86-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X86-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[0]
+; X86-NEXT:    retl
+  %r = call <4 x float> @llvm.maximum.v4f32(<4 x float> %x, <4 x float> %y)
+  ret <4 x float> %r
+}
+
+define float @test_fmaximum_nan0(float %x, float %y) {
+; SSE2-LABEL: test_fmaximum_nan0:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fmaximum_nan0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_nan0:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.maximum.f32(float 0x7fff000000000000, float %y)
+  ret float %1
+}
+
+define float @test_fmaximum_nan1(float %x, float %y) {
+; SSE2-LABEL: test_fmaximum_nan1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fmaximum_nan1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_nan1:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.maximum.f32(float %x, float 0x7fff000000000000)
+  ret float %1
+}
+
+define float @test_fmaximum_nnan(float %x, float %y) nounwind {
+; SSE2-LABEL: test_fmaximum_nnan:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    addss %xmm1, %xmm2
+; SSE2-NEXT:    subss %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    js .LBB4_1
+; SSE2-NEXT:  # %bb.2:
+; SSE2-NEXT:    maxss %xmm2, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB4_1:
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    maxss %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_nnan:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovd %xmm2, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB4_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmaxss %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB4_1:
+; AVX1-NEXT:    vmovaps %xmm0, %xmm1
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: test_fmaximum_nnan:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vaddss %xmm1, %xmm0, %xmm2
+; AVX512F-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovd %xmm2, %eax
+; AVX512F-NEXT:    testl %eax, %eax
+; AVX512F-NEXT:    sets %al
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    vmovaps %xmm2, %xmm1
+; AVX512F-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512F-NEXT:    vmovss %xmm2, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_fmaximum_nnan:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vaddss %xmm1, %xmm0, %xmm2
+; AVX512DQ-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vfpclassss $3, %xmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %k1
+; AVX512DQ-NEXT:    vmovaps %xmm2, %xmm1
+; AVX512DQ-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512DQ-NEXT:    vmovss %xmm2, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_nnan:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X86-NEXT:    vaddss %xmm0, %xmm2, %xmm1
+; X86-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB4_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovaps %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB4_3
+; X86-NEXT:  .LBB4_1:
+; X86-NEXT:    vmovaps %xmm0, %xmm2
+; X86-NEXT:    vmovaps %xmm1, %xmm0
+; X86-NEXT:  .LBB4_3:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = fadd nnan float %x, %y
+  %2 = fsub nnan float %x, %y
+  %3 = tail call float @llvm.maximum.f32(float %1, float %2)
+  ret float %3
+}
+
+define double @test_fmaximum_zero0(double %x, double %y) nounwind {
+; SSE2-LABEL: test_fmaximum_zero0:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    cmpunordsd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm1, %xmm2
+; SSE2-NEXT:    xorpd %xmm3, %xmm3
+; SSE2-NEXT:    maxsd %xmm3, %xmm1
+; SSE2-NEXT:    andnpd %xmm1, %xmm0
+; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_zero0:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vcmpunordsd %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_zero0:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordsd %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_zero0:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.maximum.f64(double 0.0, double %y)
+  ret double %1
+}
+
+define double @test_fmaximum_zero1(double %x, double %y) nounwind {
+; SSE2-LABEL: test_fmaximum_zero1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm2
+; SSE2-NEXT:    andpd %xmm0, %xmm2
+; SSE2-NEXT:    xorpd %xmm3, %xmm3
+; SSE2-NEXT:    maxsd %xmm3, %xmm0
+; SSE2-NEXT:    andnpd %xmm0, %xmm1
+; SSE2-NEXT:    orpd %xmm2, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_zero1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_zero1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_zero1:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.maximum.f64(double %x, double 0.0)
+  ret double %1
+}
+
+define double @test_fmaximum_zero2(double %x, double %y) {
+; SSE2-LABEL: test_fmaximum_zero2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorps %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fmaximum_zero2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_zero2:
+; X86:       # %bb.0:
+; X86-NEXT:    fldz
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.maximum.f64(double 0.0, double -0.0)
+  ret double %1
+}
+
+define float @test_fmaximum_nsz(float %x, float %y) "no-signed-zeros-fp-math"="true" nounwind {
+; SSE2-LABEL: test_fmaximum_nsz:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm0, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm3
+; SSE2-NEXT:    andps %xmm0, %xmm3
+; SSE2-NEXT:    maxss %xmm1, %xmm0
+; SSE2-NEXT:    andnps %xmm0, %xmm2
+; SSE2-NEXT:    orps %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_nsz:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_nsz:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_nsz:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm1
+; X86-NEXT:    vmaxss {{[0-9]+}}(%esp), %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.maximum.f32(float %x, float %y)
+  ret float %1
+}
+
+define float @test_fmaximum_combine_cmps(float %x, float %y) nounwind {
+; SSE2-LABEL: test_fmaximum_combine_cmps:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    divss %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    js .LBB9_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movaps %xmm1, %xmm3
+; SSE2-NEXT:  .LBB9_2:
+; SSE2-NEXT:    movaps %xmm3, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm4
+; SSE2-NEXT:    andps %xmm3, %xmm4
+; SSE2-NEXT:    js .LBB9_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movaps %xmm0, %xmm1
+; SSE2-NEXT:  .LBB9_4:
+; SSE2-NEXT:    maxss %xmm1, %xmm3
+; SSE2-NEXT:    andnps %xmm3, %xmm2
+; SSE2-NEXT:    orps %xmm4, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_combine_cmps:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vdivss %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB9_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovaps %xmm0, %xmm2
+; AVX1-NEXT:    jmp .LBB9_3
+; AVX1-NEXT:  .LBB9_1:
+; AVX1-NEXT:    vmovaps %xmm1, %xmm2
+; AVX1-NEXT:    vmovaps %xmm0, %xmm1
+; AVX1-NEXT:  .LBB9_3:
+; AVX1-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm1, %xmm1, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: test_fmaximum_combine_cmps:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vdivss %xmm0, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovd %xmm0, %eax
+; AVX512F-NEXT:    testl %eax, %eax
+; AVX512F-NEXT:    sets %al
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    vmovaps %xmm0, %xmm2
+; AVX512F-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512F-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512F-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512F-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_fmaximum_combine_cmps:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vdivss %xmm0, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vfpclassss $3, %xmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %k1
+; AVX512DQ-NEXT:    vmovaps %xmm1, %xmm2
+; AVX512DQ-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512DQ-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT:    vmaxss %xmm2, %xmm0, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_combine_cmps:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vdivss %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB9_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovaps %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB9_3
+; X86-NEXT:  .LBB9_1:
+; X86-NEXT:    vmovaps %xmm0, %xmm2
+; X86-NEXT:    vmovaps %xmm1, %xmm0
+; X86-NEXT:  .LBB9_3:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = fdiv nnan float %y, %x
+  %2 = tail call float @llvm.maximum.f32(float %x, float %1)
+  ret float %2
+}
+
+;
+; fminimum
+;
+
+define float @test_fminimum(float %x, float %y) nounwind {
+; SSE2-LABEL: test_fminimum:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    js .LBB10_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:  .LBB10_2:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm4
+; SSE2-NEXT:    andps %xmm3, %xmm4
+; SSE2-NEXT:    js .LBB10_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:  .LBB10_4:
+; SSE2-NEXT:    minss %xmm0, %xmm3
+; SSE2-NEXT:    andnps %xmm3, %xmm2
+; SSE2-NEXT:    orps %xmm4, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fminimum:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB10_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX1-NEXT:    jmp .LBB10_3
+; AVX1-NEXT:  .LBB10_1:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX1-NEXT:  .LBB10_3:
+; AVX1-NEXT:    vminss %xmm2, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fminimum:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    testl %eax, %eax
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    vmovaps %xmm1, %xmm2
+; AVX512-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    vminss %xmm2, %xmm0, %xmm1
+; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fminimum:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB10_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB10_3
+; X86-NEXT:  .LBB10_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB10_3:
+; X86-NEXT:    vminss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.minimum.f32(float %x, float %y)
+  ret float %1
+}
+
+define <2 x double> @test_fminimum_scalarize(<2 x double> %x, <2 x double> %y) "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" {
+; SSE2-LABEL: test_fminimum_scalarize:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    minsd %xmm1, %xmm2
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
+; SSE2-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; SSE2-NEXT:    minsd %xmm1, %xmm0
+; SSE2-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; SSE2-NEXT:    movapd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fminimum_scalarize:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vminsd %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-NEXT:    vminsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_scalarize:
+; X86:       # %bb.0:
+; X86-NEXT:    vminsd %xmm1, %xmm0, %xmm2
+; X86-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1,0]
+; X86-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1,0]
+; X86-NEXT:    vminsd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm2[0],xmm0[0]
+; X86-NEXT:    retl
+  %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> %x, <2 x double> %y)
+  ret <2 x double> %r
+}
+
+define float @test_fminimum_nan0(float %x, float %y) {
+; SSE2-LABEL: test_fminimum_nan0:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fminimum_nan0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_nan0:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.minimum.f32(float 0x7fff000000000000, float %y)
+  ret float %1
+}
+
+define float @test_fminimum_nan1(float %x, float %y) {
+; SSE2-LABEL: test_fminimum_nan1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fminimum_nan1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_nan1:
+; X86:       # %bb.0:
+; X86-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X86-NEXT:    retl
+  %1 = tail call float @llvm.minimum.f32(float %x, float 0x7fff000000000000)
+  ret float %1
+}
+
+define double @test_fminimum_nnan(double %x, double %y) "no-nans-fp-math"="true" nounwind {
+; SSE2-LABEL: test_fminimum_nnan:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq %xmm0, %rax
+; SSE2-NEXT:    testq %rax, %rax
+; SSE2-NEXT:    js .LBB14_1
+; SSE2-NEXT:  # %bb.2:
+; SSE2-NEXT:    minsd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+; SSE2-NEXT:  .LBB14_1:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    minsd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fminimum_nnan:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovq %xmm0, %rax
+; AVX1-NEXT:    testq %rax, %rax
+; AVX1-NEXT:    js .LBB14_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vminsd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+; AVX1-NEXT:  .LBB14_1:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:    vminsd %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: test_fminimum_nnan:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovq %xmm0, %rax
+; AVX512F-NEXT:    testq %rax, %rax
+; AVX512F-NEXT:    sets %al
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    vmovapd %xmm1, %xmm2
+; AVX512F-NEXT:    vmovsd %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT:    vminsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_fminimum_nnan:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vfpclasssd $5, %xmm1, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %k1
+; AVX512DQ-NEXT:    vmovapd %xmm0, %xmm2
+; AVX512DQ-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512DQ-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512DQ-NEXT:    vminsd %xmm2, %xmm1, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_nnan:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; X86-NEXT:    vextractps $1, %xmm2, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB14_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovapd %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB14_3
+; X86-NEXT:  .LBB14_1:
+; X86-NEXT:    vmovapd %xmm0, %xmm2
+; X86-NEXT:    vmovapd %xmm1, %xmm0
+; X86-NEXT:  .LBB14_3:
+; X86-NEXT:    vminsd %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.minimum.f64(double %x, double %y)
+  ret double %1
+}
+
+define double @test_fminimum_zero0(double %x, double %y) nounwind {
+; SSE2-LABEL: test_fminimum_zero0:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    cmpunordsd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm1, %xmm2
+; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    andnpd %xmm1, %xmm0
+; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fminimum_zero0:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vcmpunordsd %xmm1, %xmm1, %xmm0
+; AVX1-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fminimum_zero0:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpunordsd %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_zero0:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm1
+; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.minimum.f64(double -0.0, double %y)
+  ret double %1
+}
+
+define double @test_fminimum_zero1(double %x, double %y) nounwind {
+; SSE2-LABEL: test_fminimum_zero1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    cmpunordsd %xmm0, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm2
+; SSE2-NEXT:    andpd %xmm0, %xmm2
+; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    andnpd %xmm0, %xmm1
+; SSE2-NEXT:    orpd %xmm2, %xmm1
+; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fminimum_zero1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm1
+; AVX1-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fminimum_zero1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovapd %xmm1, %xmm0
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_zero1:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm1
+; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    fldl (%esp)
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.minimum.f64(double %x, double -0.0)
+  ret double %1
+}
+
+define double @test_fminimum_zero2(double %x, double %y) {
+; SSE2-LABEL: test_fminimum_zero2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    retq
+;
+; AVX-LABEL: test_fminimum_zero2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; AVX-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_zero2:
+; X86:       # %bb.0:
+; X86-NEXT:    fldz
+; X86-NEXT:    fchs
+; X86-NEXT:    retl
+  %1 = tail call double @llvm.minimum.f64(double -0.0, double 0.0)
+  ret double %1
+}
+
+define float @test_fminimum_nsz(float %x, float %y) nounwind {
+; SSE2-LABEL: test_fminimum_nsz:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm0, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm3
+; SSE2-NEXT:    andps %xmm0, %xmm3
+; SSE2-NEXT:    minss %xmm1, %xmm0
+; SSE2-NEXT:    andnps %xmm0, %xmm2
+; SSE2-NEXT:    orps %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fminimum_nsz:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vminss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fminimum_nsz:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vminss %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_nsz:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm1
+; X86-NEXT:    vminss {{[0-9]+}}(%esp), %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm1, %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = tail call nsz float @llvm.minimum.f32(float %x, float %y)
+  ret float %1
+}
+
+define float @test_fminimum_combine_cmps(float %x, float %y) nounwind {
+; SSE2-LABEL: test_fminimum_combine_cmps:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    divss %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movaps %xmm1, %xmm3
+; SSE2-NEXT:    js .LBB19_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:  .LBB19_2:
+; SSE2-NEXT:    movaps %xmm3, %xmm2
+; SSE2-NEXT:    cmpunordss %xmm3, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm4
+; SSE2-NEXT:    andps %xmm3, %xmm4
+; SSE2-NEXT:    js .LBB19_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:  .LBB19_4:
+; SSE2-NEXT:    minss %xmm0, %xmm3
+; SSE2-NEXT:    andnps %xmm3, %xmm2
+; SSE2-NEXT:    orps %xmm4, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fminimum_combine_cmps:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vdivss %xmm0, %xmm1, %xmm2
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB19_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovaps %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB19_3
+; AVX1-NEXT:  .LBB19_1:
+; AVX1-NEXT:    vmovaps %xmm0, %xmm1
+; AVX1-NEXT:    vmovaps %xmm2, %xmm0
+; AVX1-NEXT:  .LBB19_3:
+; AVX1-NEXT:    vminss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512F-LABEL: test_fminimum_combine_cmps:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vdivss %xmm0, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovd %xmm0, %eax
+; AVX512F-NEXT:    testl %eax, %eax
+; AVX512F-NEXT:    sets %al
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    vmovaps %xmm1, %xmm2
+; AVX512F-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512F-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512F-NEXT:    vminss %xmm2, %xmm0, %xmm1
+; AVX512F-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
+; AVX512F-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512F-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512DQ-LABEL: test_fminimum_combine_cmps:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    vdivss %xmm0, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vfpclassss $5, %xmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %k1
+; AVX512DQ-NEXT:    vmovaps %xmm1, %xmm2
+; AVX512DQ-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
+; AVX512DQ-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512DQ-NEXT:    vminss %xmm2, %xmm0, %xmm0
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: test_fminimum_combine_cmps:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vdivss %xmm0, %xmm1, %xmm2
+; X86-NEXT:    vmovd %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB19_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovaps %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB19_3
+; X86-NEXT:  .LBB19_1:
+; X86-NEXT:    vmovaps %xmm0, %xmm1
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:  .LBB19_3:
+; X86-NEXT:    vminss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    flds (%esp)
+; X86-NEXT:    popl %eax
+; X86-NEXT:    retl
+  %1 = fdiv nnan float %y, %x
+  %2 = tail call float @llvm.minimum.f32(float %x, float %1)
+  ret float %2
+}
diff --git a/llvm/test/CodeGen/X86/fpclamptosat_vec.ll b/llvm/test/CodeGen/X86/fpclamptosat_vec.ll
index c351c1b82cf19..37986f647804b 100644
--- a/llvm/test/CodeGen/X86/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/X86/fpclamptosat_vec.ll
@@ -76,18 +76,16 @@ define <2 x i32> @utest_f64i32(<2 x double> %x) {
 ; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; CHECK-NEXT:    pxor %xmm1, %xmm0
-; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; CHECK-NEXT:    pcmpgtd %xmm2, %xmm3
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; CHECK-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-NEXT:    pand %xmm3, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-NEXT:    por %xmm0, %xmm2
-; CHECK-NEXT:    pand %xmm2, %xmm1
-; CHECK-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; CHECK-NEXT:    por %xmm1, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; CHECK-NEXT:    pand %xmm0, %xmm1
+; CHECK-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-NEXT:    retq
 entry:
   %conv = fptoui <2 x double> %x to <2 x i64>
@@ -276,31 +274,27 @@ define <4 x i32> @utest_f32i32(<4 x float> %x) {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; CHECK-NEXT:    movdqa %xmm0, %xmm4
 ; CHECK-NEXT:    pxor %xmm3, %xmm4
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm3, %xmm5
-; CHECK-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259455,9223372039002259455]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm6 = [2147483647,2147483647,2147483647,2147483647]
 ; CHECK-NEXT:    movdqa %xmm6, %xmm7
-; CHECK-NEXT:    pcmpgtd %xmm4, %xmm7
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; CHECK-NEXT:    pand %xmm5, %xmm4
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; CHECK-NEXT:    por %xmm4, %xmm5
-; CHECK-NEXT:    pand %xmm5, %xmm0
-; CHECK-NEXT:    pandn %xmm2, %xmm5
-; CHECK-NEXT:    por %xmm0, %xmm5
+; CHECK-NEXT:    pcmpgtd %xmm5, %xmm7
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm3, %xmm4
+; CHECK-NEXT:    pand %xmm7, %xmm4
+; CHECK-NEXT:    pand %xmm4, %xmm0
+; CHECK-NEXT:    pandn %xmm2, %xmm4
+; CHECK-NEXT:    por %xmm0, %xmm4
 ; CHECK-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-NEXT:    pxor %xmm3, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm3, %xmm4
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm6
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; CHECK-NEXT:    pand %xmm4, %xmm3
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; CHECK-NEXT:    por %xmm3, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; CHECK-NEXT:    pcmpgtd %xmm5, %xmm6
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm3, %xmm0
+; CHECK-NEXT:    pand %xmm6, %xmm0
 ; CHECK-NEXT:    pand %xmm0, %xmm1
 ; CHECK-NEXT:    pandn %xmm2, %xmm0
 ; CHECK-NEXT:    por %xmm1, %xmm0
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm5[0,2]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; CHECK-NEXT:    retq
 entry:
   %conv = fptoui <4 x float> %x to <4 x i64>
@@ -560,33 +554,28 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; CHECK-NEXT:    movdqa %xmm0, %xmm3
 ; CHECK-NEXT:    pxor %xmm2, %xmm3
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm2, %xmm4
-; CHECK-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
 ; CHECK-NEXT:    movdqa %xmm5, %xmm6
-; CHECK-NEXT:    pcmpgtd %xmm3, %xmm6
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; CHECK-NEXT:    pand %xmm4, %xmm3
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; CHECK-NEXT:    por %xmm3, %xmm4
-; CHECK-NEXT:    pand %xmm4, %xmm0
-; CHECK-NEXT:    pandn %xmm1, %xmm4
-; CHECK-NEXT:    por %xmm0, %xmm4
+; CHECK-NEXT:    pcmpgtd %xmm4, %xmm6
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm2, %xmm3
+; CHECK-NEXT:    pand %xmm6, %xmm3
+; CHECK-NEXT:    pand %xmm3, %xmm0
+; CHECK-NEXT:    pandn %xmm1, %xmm3
+; CHECK-NEXT:    por %xmm0, %xmm3
 ; CHECK-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; CHECK-NEXT:    movdqa %xmm6, %xmm0
 ; CHECK-NEXT:    pxor %xmm2, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm2, %xmm3
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm5
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
-; CHECK-NEXT:    pand %xmm3, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; CHECK-NEXT:    por %xmm2, %xmm0
-; CHECK-NEXT:    movdqa %xmm6, %xmm2
-; CHECK-NEXT:    pand %xmm0, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; CHECK-NEXT:    pcmpgtd %xmm4, %xmm5
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm2, %xmm0
+; CHECK-NEXT:    pand %xmm5, %xmm0
+; CHECK-NEXT:    pand %xmm0, %xmm6
 ; CHECK-NEXT:    pandn %xmm1, %xmm0
-; CHECK-NEXT:    por %xmm2, %xmm0
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
+; CHECK-NEXT:    por %xmm6, %xmm0
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
 ; CHECK-NEXT:    addq $72, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -1661,18 +1650,16 @@ define <2 x i32> @utest_f64i32_mm(<2 x double> %x) {
 ; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; CHECK-NEXT:    pxor %xmm1, %xmm0
-; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; CHECK-NEXT:    pcmpgtd %xmm2, %xmm3
 ; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; CHECK-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-NEXT:    pand %xmm3, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-NEXT:    por %xmm0, %xmm2
-; CHECK-NEXT:    pand %xmm2, %xmm1
-; CHECK-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; CHECK-NEXT:    por %xmm1, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; CHECK-NEXT:    pand %xmm0, %xmm1
+; CHECK-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    por %xmm1, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-NEXT:    retq
 entry:
   %conv = fptoui <2 x double> %x to <2 x i64>
@@ -1855,32 +1842,28 @@ define <4 x i32> @utest_f32i32_mm(<4 x float> %x) {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; CHECK-NEXT:    movdqa %xmm0, %xmm3
 ; CHECK-NEXT:    pxor %xmm2, %xmm3
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm2, %xmm4
-; CHECK-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
 ; CHECK-NEXT:    movdqa %xmm5, %xmm6
-; CHECK-NEXT:    pcmpgtd %xmm3, %xmm6
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; CHECK-NEXT:    pand %xmm4, %xmm3
-; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; CHECK-NEXT:    por %xmm3, %xmm4
-; CHECK-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
-; CHECK-NEXT:    pand %xmm4, %xmm0
-; CHECK-NEXT:    pandn %xmm3, %xmm4
-; CHECK-NEXT:    por %xmm0, %xmm4
+; CHECK-NEXT:    pcmpgtd %xmm4, %xmm6
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm2, %xmm3
+; CHECK-NEXT:    pand %xmm6, %xmm3
+; CHECK-NEXT:    movdqa {{.*#+}} xmm4 = [4294967295,4294967295]
+; CHECK-NEXT:    pand %xmm3, %xmm0
+; CHECK-NEXT:    pandn %xmm4, %xmm3
+; CHECK-NEXT:    por %xmm0, %xmm3
 ; CHECK-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-NEXT:    pxor %xmm2, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm2, %xmm6
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm5
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
-; CHECK-NEXT:    pand %xmm6, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; CHECK-NEXT:    por %xmm2, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; CHECK-NEXT:    pcmpgtd %xmm6, %xmm5
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm2, %xmm0
+; CHECK-NEXT:    pand %xmm5, %xmm0
 ; CHECK-NEXT:    pand %xmm0, %xmm1
-; CHECK-NEXT:    pandn %xmm3, %xmm0
+; CHECK-NEXT:    pandn %xmm4, %xmm0
 ; CHECK-NEXT:    por %xmm1, %xmm0
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
 ; CHECK-NEXT:    retq
 entry:
   %conv = fptoui <4 x float> %x to <4 x i64>
@@ -2134,34 +2117,29 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) {
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; CHECK-NEXT:    movdqa %xmm0, %xmm2
 ; CHECK-NEXT:    pxor %xmm1, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm1, %xmm3
-; CHECK-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm4 = [2147483647,2147483647,2147483647,2147483647]
 ; CHECK-NEXT:    movdqa %xmm4, %xmm5
-; CHECK-NEXT:    pcmpgtd %xmm2, %xmm5
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
-; CHECK-NEXT:    pand %xmm3, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
-; CHECK-NEXT:    por %xmm2, %xmm3
-; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,4294967295]
-; CHECK-NEXT:    pand %xmm3, %xmm0
-; CHECK-NEXT:    pandn %xmm2, %xmm3
-; CHECK-NEXT:    por %xmm0, %xmm3
+; CHECK-NEXT:    pcmpgtd %xmm3, %xmm5
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm2
+; CHECK-NEXT:    pand %xmm5, %xmm2
+; CHECK-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
+; CHECK-NEXT:    pand %xmm2, %xmm0
+; CHECK-NEXT:    pandn %xmm3, %xmm2
+; CHECK-NEXT:    por %xmm0, %xmm2
 ; CHECK-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; CHECK-NEXT:    movdqa %xmm6, %xmm0
 ; CHECK-NEXT:    pxor %xmm1, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; CHECK-NEXT:    pcmpeqd %xmm1, %xmm5
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm4
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,0,2,2]
-; CHECK-NEXT:    pand %xmm5, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; CHECK-NEXT:    por %xmm1, %xmm0
-; CHECK-NEXT:    movdqa %xmm6, %xmm1
-; CHECK-NEXT:    pand %xmm0, %xmm1
-; CHECK-NEXT:    pandn %xmm2, %xmm0
-; CHECK-NEXT:    por %xmm1, %xmm0
-; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; CHECK-NEXT:    pcmpgtd %xmm5, %xmm4
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-NEXT:    pand %xmm4, %xmm0
+; CHECK-NEXT:    pand %xmm0, %xmm6
+; CHECK-NEXT:    pandn %xmm3, %xmm0
+; CHECK-NEXT:    por %xmm6, %xmm0
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
 ; CHECK-NEXT:    addq $72, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll b/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
index 5a60a9e00aa3f..76e15ca0bf919 100644
--- a/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
@@ -183,7 +183,7 @@ define i13 @test_signed_i13_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-4096, %cx # imm = 0xF000
+; X86-X87-NEXT:    movl $61440, %ecx # imm = 0xF000
 ; X86-X87-NEXT:    jb .LBB2_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -255,7 +255,7 @@ define i16 @test_signed_i16_f32(float %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-32768, %cx # imm = 0x8000
+; X86-X87-NEXT:    movl $32768, %ecx # imm = 0x8000
 ; X86-X87-NEXT:    jb .LBB3_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -1203,7 +1203,7 @@ define i13 @test_signed_i13_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-4096, %cx # imm = 0xF000
+; X86-X87-NEXT:    movl $61440, %ecx # imm = 0xF000
 ; X86-X87-NEXT:    jb .LBB12_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -1275,7 +1275,7 @@ define i16 @test_signed_i16_f64(double %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-32768, %cx # imm = 0x8000
+; X86-X87-NEXT:    movl $32768, %ecx # imm = 0x8000
 ; X86-X87-NEXT:    jb .LBB13_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -2268,7 +2268,7 @@ define i13 @test_signed_i13_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-4096, %cx # imm = 0xF000
+; X86-X87-NEXT:    movl $61440, %ecx # imm = 0xF000
 ; X86-X87-NEXT:    jb .LBB22_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -2364,7 +2364,7 @@ define i16 @test_signed_i16_f16(half %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-32768, %cx # imm = 0x8000
+; X86-X87-NEXT:    movl $32768, %ecx # imm = 0x8000
 ; X86-X87-NEXT:    jb .LBB23_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -3495,7 +3495,7 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-4096, %cx # imm = 0xF000
+; X86-X87-NEXT:    movl $61440, %ecx # imm = 0xF000
 ; X86-X87-NEXT:    jb .LBB32_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -3536,24 +3536,22 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
-; X86-SSE-NEXT:    movw $-4096, %ax # imm = 0xF000
-; X86-SSE-NEXT:    jb .LBB32_2
-; X86-SSE-NEXT:  # %bb.1:
-; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:  .LBB32_2:
+; X86-SSE-NEXT:    movl $61440, %ecx # imm = 0xF000
+; X86-SSE-NEXT:    cmovael %eax, %ecx
 ; X86-SSE-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
-; X86-SSE-NEXT:    movl $4095, %ecx # imm = 0xFFF
-; X86-SSE-NEXT:    cmovbel %eax, %ecx
+; X86-SSE-NEXT:    movl $4095, %edx # imm = 0xFFF
+; X86-SSE-NEXT:    cmovbel %ecx, %edx
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    fucompi %st(0), %st
-; X86-SSE-NEXT:    cmovnpl %ecx, %eax
+; X86-SSE-NEXT:    cmovnpl %edx, %eax
 ; X86-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-SSE-NEXT:    addl $8, %esp
 ; X86-SSE-NEXT:    retl
@@ -3568,24 +3566,22 @@ define i13 @test_signed_i13_f80(x86_fp80 %f) nounwind {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    fists -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}(%rip)
 ; X64-NEXT:    fxch %st(1)
 ; X64-NEXT:    fucomi %st(1), %st
 ; X64-NEXT:    fstp %st(1)
-; X64-NEXT:    movw $-4096, %ax # imm = 0xF000
-; X64-NEXT:    jb .LBB32_2
-; X64-NEXT:  # %bb.1:
-; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:  .LBB32_2:
+; X64-NEXT:    movl $61440, %ecx # imm = 0xF000
+; X64-NEXT:    cmovael %eax, %ecx
 ; X64-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}(%rip)
 ; X64-NEXT:    fxch %st(1)
 ; X64-NEXT:    fucomi %st(1), %st
 ; X64-NEXT:    fstp %st(1)
-; X64-NEXT:    movl $4095, %ecx # imm = 0xFFF
-; X64-NEXT:    cmovbel %eax, %ecx
+; X64-NEXT:    movl $4095, %edx # imm = 0xFFF
+; X64-NEXT:    cmovbel %ecx, %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    fucompi %st(0), %st
-; X64-NEXT:    cmovnpl %ecx, %eax
+; X64-NEXT:    cmovnpl %edx, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
     %x = call i13 @llvm.fptosi.sat.i13.f80(x86_fp80 %f)
@@ -3611,7 +3607,7 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X86-X87-NEXT:    fnstsw %ax
 ; X86-X87-NEXT:    # kill: def $ah killed $ah killed $ax
 ; X86-X87-NEXT:    sahf
-; X86-X87-NEXT:    movw $-32768, %cx # imm = 0x8000
+; X86-X87-NEXT:    movl $32768, %ecx # imm = 0x8000
 ; X86-X87-NEXT:    jb .LBB33_2
 ; X86-X87-NEXT:  # %bb.1:
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
@@ -3652,24 +3648,22 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fists {{[0-9]+}}(%esp)
 ; X86-SSE-NEXT:    fldcw {{[0-9]+}}(%esp)
+; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
-; X86-SSE-NEXT:    movw $-32768, %ax # imm = 0x8000
-; X86-SSE-NEXT:    jb .LBB33_2
-; X86-SSE-NEXT:  # %bb.1:
-; X86-SSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:  .LBB33_2:
+; X86-SSE-NEXT:    movl $32768, %ecx # imm = 0x8000
+; X86-SSE-NEXT:    cmovael %eax, %ecx
 ; X86-SSE-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
 ; X86-SSE-NEXT:    fxch %st(1)
 ; X86-SSE-NEXT:    fucomi %st(1), %st
 ; X86-SSE-NEXT:    fstp %st(1)
-; X86-SSE-NEXT:    movl $32767, %ecx # imm = 0x7FFF
-; X86-SSE-NEXT:    cmovbel %eax, %ecx
+; X86-SSE-NEXT:    movl $32767, %edx # imm = 0x7FFF
+; X86-SSE-NEXT:    cmovbel %ecx, %edx
 ; X86-SSE-NEXT:    xorl %eax, %eax
 ; X86-SSE-NEXT:    fucompi %st(0), %st
-; X86-SSE-NEXT:    cmovnpl %ecx, %eax
+; X86-SSE-NEXT:    cmovnpl %edx, %eax
 ; X86-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-SSE-NEXT:    addl $8, %esp
 ; X86-SSE-NEXT:    retl
@@ -3684,24 +3678,22 @@ define i16 @test_signed_i16_f80(x86_fp80 %f) nounwind {
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    fists -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}(%rip)
 ; X64-NEXT:    fxch %st(1)
 ; X64-NEXT:    fucomi %st(1), %st
 ; X64-NEXT:    fstp %st(1)
-; X64-NEXT:    movw $-32768, %ax # imm = 0x8000
-; X64-NEXT:    jb .LBB33_2
-; X64-NEXT:  # %bb.1:
-; X64-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT:  .LBB33_2:
+; X64-NEXT:    movl $32768, %ecx # imm = 0x8000
+; X64-NEXT:    cmovael %eax, %ecx
 ; X64-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}(%rip)
 ; X64-NEXT:    fxch %st(1)
 ; X64-NEXT:    fucomi %st(1), %st
 ; X64-NEXT:    fstp %st(1)
-; X64-NEXT:    movl $32767, %ecx # imm = 0x7FFF
-; X64-NEXT:    cmovbel %eax, %ecx
+; X64-NEXT:    movl $32767, %edx # imm = 0x7FFF
+; X64-NEXT:    cmovbel %ecx, %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    fucompi %st(0), %st
-; X64-NEXT:    cmovnpl %ecx, %eax
+; X64-NEXT:    cmovnpl %edx, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
     %x = call i16 @llvm.fptosi.sat.i16.f80(x86_fp80 %f)
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 5f060fb0d4d93..45587f1f33ece 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -355,7 +355,8 @@ define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
 ; X86-NEXT:    vmovd %edx, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,7]
+; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
 ; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
 ; X86-NEXT:    retl
@@ -370,7 +371,8 @@ define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst
 ; X64-NEXT:    vmovdqa %xmm0, (%rdx)
 ; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2,3]
+; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
 ; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    retq
@@ -397,13 +399,14 @@ define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1,
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl (%edx), %edx
 ; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vmovd %edx, %xmm0
-; X86-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]
-; X86-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; X86-NEXT:    vmovd %edx, %xmm1
+; X86-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6,7]
 ; X86-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7]
-; X86-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X86-NEXT:    vmovdqa %xmm1, (%ecx)
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vmovdqa %xmm0, (%ecx)
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,1,0,1]
 ; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
 ; X86-NEXT:    retl
@@ -412,13 +415,14 @@ define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1,
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    andl $15, %eax
-; X64-NEXT:    vmovd %eax, %xmm0
-; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0,1,2,3]
+; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; X64-NEXT:    vmovd %eax, %xmm1
+; X64-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
 ; X64-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
-; X64-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X64-NEXT:    vmovdqa %xmm1, (%rdx)
 ; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vmovdqa %xmm0, (%rdx)
+; X64-NEXT:    vpand %xmm2, %xmm1, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i32, ptr %origin0
@@ -443,14 +447,13 @@ define void @freeze_two_buildvectors_one_undef_elt(ptr %origin0, ptr %origin1, p
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl (%edx), %edx
 ; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vmovd %edx, %xmm0
-; X86-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,6,7]
-; X86-NEXT:    vmovddup {{.*#+}} xmm2 = [7,7]
-; X86-NEXT:    # xmm2 = mem[0,0]
-; X86-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X86-NEXT:    vmovdqa %xmm1, (%ecx)
-; X86-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
-; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vmovddup {{.*#+}} xmm0 = [7,7]
+; X86-NEXT:    # xmm0 = mem[0,0]
+; X86-NEXT:    vmovd %edx, %xmm1
+; X86-NEXT:    vpand %xmm0, %xmm1, %xmm2
+; X86-NEXT:    vmovdqa %xmm2, (%ecx)
+; X86-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
+; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index 5f0fa26c3d3d3..954a5c6b9bcf4 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -954,11 +954,10 @@ define half @PR40273(half) #0 {
   ret half %3
 }
 
-define dso_local void @brcond(half %0) {
+define void @brcond(half %0) #0 {
 ; CHECK-LIBCALL-LABEL: brcond:
 ; CHECK-LIBCALL:       # %bb.0: # %entry
 ; CHECK-LIBCALL-NEXT:    pushq %rax
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-LIBCALL-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-LIBCALL-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-LIBCALL-NEXT:    ucomiss %xmm1, %xmm0
@@ -968,7 +967,6 @@ define dso_local void @brcond(half %0) {
 ; CHECK-LIBCALL-NEXT:    jne .LBB18_2
 ; CHECK-LIBCALL-NEXT:  # %bb.1: # %if.then
 ; CHECK-LIBCALL-NEXT:    popq %rax
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-LIBCALL-NEXT:    retq
 ; CHECK-LIBCALL-NEXT:  .LBB18_2: # %if.end
 ;
@@ -991,7 +989,6 @@ define dso_local void @brcond(half %0) {
 ; CHECK-I686-LABEL: brcond:
 ; CHECK-I686:       # %bb.0: # %entry
 ; CHECK-I686-NEXT:    subl $12, %esp
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-I686-NEXT:    pinsrw $0, {{[0-9]+}}(%esp), %xmm0
 ; CHECK-I686-NEXT:    pextrw $0, %xmm0, %eax
 ; CHECK-I686-NEXT:    movw %ax, (%esp)
@@ -1006,7 +1003,6 @@ define dso_local void @brcond(half %0) {
 ; CHECK-I686-NEXT:    jne .LBB18_2
 ; CHECK-I686-NEXT:  # %bb.1: # %if.then
 ; CHECK-I686-NEXT:    addl $12, %esp
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-I686-NEXT:    retl
 ; CHECK-I686-NEXT:  .LBB18_2: # %if.end
 entry:
@@ -1020,16 +1016,14 @@ if.end:                                           ; preds = %entry
   unreachable
 }
 
-define half @test_sqrt(half %0) {
+define half @test_sqrt(half %0) #0 {
 ; CHECK-LIBCALL-LABEL: test_sqrt:
 ; CHECK-LIBCALL:       # %bb.0: # %entry
 ; CHECK-LIBCALL-NEXT:    pushq %rax
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-LIBCALL-NEXT:    callq __extendhfsf2@PLT
 ; CHECK-LIBCALL-NEXT:    sqrtss %xmm0, %xmm0
 ; CHECK-LIBCALL-NEXT:    callq __truncsfhf2@PLT
 ; CHECK-LIBCALL-NEXT:    popq %rax
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-LIBCALL-NEXT:    retq
 ;
 ; BWON-F16C-LABEL: test_sqrt:
@@ -1047,7 +1041,6 @@ define half @test_sqrt(half %0) {
 ; CHECK-I686-LABEL: test_sqrt:
 ; CHECK-I686:       # %bb.0: # %entry
 ; CHECK-I686-NEXT:    subl $12, %esp
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-I686-NEXT:    pinsrw $0, {{[0-9]+}}(%esp), %xmm0
 ; CHECK-I686-NEXT:    pextrw $0, %xmm0, %eax
 ; CHECK-I686-NEXT:    movw %ax, (%esp)
@@ -1058,7 +1051,6 @@ define half @test_sqrt(half %0) {
 ; CHECK-I686-NEXT:    movss %xmm0, (%esp)
 ; CHECK-I686-NEXT:    calll __truncsfhf2
 ; CHECK-I686-NEXT:    addl $12, %esp
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-I686-NEXT:    retl
 entry:
   %1 = call half @llvm.sqrt.f16(half %0)
@@ -1067,7 +1059,7 @@ entry:
 
 declare half @llvm.sqrt.f16(half)
 
-define void @main.158() local_unnamed_addr #0 {
+define void @main.158() #0 {
 ; CHECK-LIBCALL-LABEL: main.158:
 ; CHECK-LIBCALL:       # %bb.0: # %entry
 ; CHECK-LIBCALL-NEXT:    pushq %rax
@@ -1143,23 +1135,14 @@ entry:
   ret void
 }
 
-define void @main.45() local_unnamed_addr {
+define void @main.45() #0 {
 ; CHECK-LIBCALL-LABEL: main.45:
 ; CHECK-LIBCALL:       # %bb.0: # %entry
 ; CHECK-LIBCALL-NEXT:    pushq %rbp
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-LIBCALL-NEXT:    pushq %r15
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 24
 ; CHECK-LIBCALL-NEXT:    pushq %r14
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-LIBCALL-NEXT:    pushq %rbx
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 40
 ; CHECK-LIBCALL-NEXT:    pushq %rax
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-LIBCALL-NEXT:    .cfi_offset %rbx, -40
-; CHECK-LIBCALL-NEXT:    .cfi_offset %r14, -32
-; CHECK-LIBCALL-NEXT:    .cfi_offset %r15, -24
-; CHECK-LIBCALL-NEXT:    .cfi_offset %rbp, -16
 ; CHECK-LIBCALL-NEXT:    pinsrw $0, (%rax), %xmm0
 ; CHECK-LIBCALL-NEXT:    pextrw $0, %xmm0, %eax
 ; CHECK-LIBCALL-NEXT:    movd %eax, %xmm1
@@ -1183,15 +1166,10 @@ define void @main.45() local_unnamed_addr {
 ; CHECK-LIBCALL-NEXT:    movw %r15w, (%rax)
 ; CHECK-LIBCALL-NEXT:    movw %bp, (%rax)
 ; CHECK-LIBCALL-NEXT:    addq $8, %rsp
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 40
 ; CHECK-LIBCALL-NEXT:    popq %rbx
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-LIBCALL-NEXT:    popq %r14
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 24
 ; CHECK-LIBCALL-NEXT:    popq %r15
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-LIBCALL-NEXT:    popq %rbp
-; CHECK-LIBCALL-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-LIBCALL-NEXT:    retq
 ;
 ; BWON-F16C-LABEL: main.45:
@@ -1213,13 +1191,8 @@ define void @main.45() local_unnamed_addr {
 ; CHECK-I686-LABEL: main.45:
 ; CHECK-I686:       # %bb.0: # %entry
 ; CHECK-I686-NEXT:    pushl %edi
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-I686-NEXT:    pushl %esi
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 12
 ; CHECK-I686-NEXT:    subl $20, %esp
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-I686-NEXT:    .cfi_offset %esi, -12
-; CHECK-I686-NEXT:    .cfi_offset %edi, -8
 ; CHECK-I686-NEXT:    pinsrw $0, (%eax), %xmm0
 ; CHECK-I686-NEXT:    pextrw $0, %xmm0, %eax
 ; CHECK-I686-NEXT:    movd %eax, %xmm0
@@ -1238,11 +1211,8 @@ define void @main.45() local_unnamed_addr {
 ; CHECK-I686-NEXT:    movw %di, (%eax)
 ; CHECK-I686-NEXT:    movw %si, (%eax)
 ; CHECK-I686-NEXT:    addl $20, %esp
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 12
 ; CHECK-I686-NEXT:    popl %esi
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-I686-NEXT:    popl %edi
-; CHECK-I686-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-I686-NEXT:    retl
 entry:
   %0 = load half, ptr undef, align 8
@@ -1360,22 +1330,17 @@ declare half @llvm.minnum.f16(half, half)
 define half @pr61271(half %0, half %1) #0 {
 ; CHECK-LIBCALL-LABEL: pr61271:
 ; CHECK-LIBCALL:       # %bb.0:
-; CHECK-LIBCALL-NEXT:    subq $40, %rsp
+; CHECK-LIBCALL-NEXT:    pushq %rax
 ; CHECK-LIBCALL-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-LIBCALL-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-LIBCALL-NEXT:    callq __extendhfsf2@PLT
-; CHECK-LIBCALL-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-LIBCALL-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
 ; CHECK-LIBCALL-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
 ; CHECK-LIBCALL-NEXT:    # xmm0 = mem[0],zero,zero,zero
 ; CHECK-LIBCALL-NEXT:    callq __extendhfsf2@PLT
-; CHECK-LIBCALL-NEXT:    movaps %xmm0, %xmm1
-; CHECK-LIBCALL-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; CHECK-LIBCALL-NEXT:    cmpltss %xmm2, %xmm1
-; CHECK-LIBCALL-NEXT:    andps %xmm1, %xmm0
-; CHECK-LIBCALL-NEXT:    andnps %xmm2, %xmm1
-; CHECK-LIBCALL-NEXT:    orps %xmm1, %xmm0
+; CHECK-LIBCALL-NEXT:    minss (%rsp), %xmm0 # 4-byte Folded Reload
 ; CHECK-LIBCALL-NEXT:    callq __truncsfhf2@PLT
-; CHECK-LIBCALL-NEXT:    addq $40, %rsp
+; CHECK-LIBCALL-NEXT:    popq %rax
 ; CHECK-LIBCALL-NEXT:    retq
 ;
 ; BWON-F16C-LABEL: pr61271:
@@ -1388,8 +1353,7 @@ define half @pr61271(half %0, half %1) #0 {
 ; BWON-F16C-NEXT:    movzwl %ax, %eax
 ; BWON-F16C-NEXT:    vmovd %eax, %xmm1
 ; BWON-F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
-; BWON-F16C-NEXT:    vcmpltss %xmm0, %xmm1, %xmm2
-; BWON-F16C-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; BWON-F16C-NEXT:    vminss %xmm0, %xmm1, %xmm0
 ; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; BWON-F16C-NEXT:    vmovd %xmm0, %eax
 ; BWON-F16C-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
@@ -1411,13 +1375,8 @@ define half @pr61271(half %0, half %1) #0 {
 ; CHECK-I686-NEXT:    calll __extendhfsf2
 ; CHECK-I686-NEXT:    fstps {{[0-9]+}}(%esp)
 ; CHECK-I686-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT:    movaps %xmm1, %xmm2
-; CHECK-I686-NEXT:    cmpltss %xmm0, %xmm2
-; CHECK-I686-NEXT:    andps %xmm2, %xmm1
-; CHECK-I686-NEXT:    andnps %xmm0, %xmm2
-; CHECK-I686-NEXT:    orps %xmm1, %xmm2
-; CHECK-I686-NEXT:    movss %xmm2, (%esp)
+; CHECK-I686-NEXT:    minss {{[0-9]+}}(%esp), %xmm0
+; CHECK-I686-NEXT:    movss %xmm0, (%esp)
 ; CHECK-I686-NEXT:    calll __truncsfhf2
 ; CHECK-I686-NEXT:    addl $44, %esp
 ; CHECK-I686-NEXT:    retl
diff --git a/llvm/test/CodeGen/X86/i686-win-shrink-wrapping.ll b/llvm/test/CodeGen/X86/i686-win-shrink-wrapping.ll
index 65372d6006a9f..5cb1b8f4a89e4 100644
--- a/llvm/test/CodeGen/X86/i686-win-shrink-wrapping.ll
+++ b/llvm/test/CodeGen/X86/i686-win-shrink-wrapping.ll
@@ -1,5 +1,6 @@
-; RUN: llc %s -o - -enable-shrink-wrap=true | FileCheck %s
-; RUN: llc %s -o - -enable-shrink-wrap=false | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc %s -o - -enable-shrink-wrap=true | FileCheck --check-prefix=SHRINK-WRAP %s
+; RUN: llc %s -o - -enable-shrink-wrap=false | FileCheck --check-prefix=NO-SHRINK-WRAP %s
 target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc18.0.0"
 
@@ -7,17 +8,60 @@ target triple = "i686-pc-windows-msvc18.0.0"
 
 ; Check that we do not use a basic block that has EFLAGS as live-in
 ; if we need to realign the stack.
-; PR27531.
-; CHECK-LABEL: stackRealignment:
-; Prologue code.
-; CHECK: pushl
-; Make sure we actually perform some stack realignment.
-; CHECK: andl ${{[-0-9]+}}, %esp
-; This is the end of the entry block.
-; The prologue should have happened before that point because past
-; this point, EFLAGS is live.
-; CHECK: jg
 define x86_thiscallcc void @stackRealignment(ptr %this) {
+; SHRINK-WRAP-LABEL: stackRealignment:
+; SHRINK-WRAP:       # %bb.0: # %entry
+; SHRINK-WRAP-NEXT:    movl (%ecx), %eax
+; SHRINK-WRAP-NEXT:    cmpl $33, %eax
+; SHRINK-WRAP-NEXT:    movl $42, %edx
+; SHRINK-WRAP-NEXT:    jge LBB0_2
+; SHRINK-WRAP-NEXT:  # %bb.1: # %entry
+; SHRINK-WRAP-NEXT:    movl $128, %edx
+; SHRINK-WRAP-NEXT:  LBB0_2: # %entry
+; SHRINK-WRAP-NEXT:    pushl %ebp
+; SHRINK-WRAP-NEXT:    movl %esp, %ebp
+; SHRINK-WRAP-NEXT:    andl $-8, %esp
+; SHRINK-WRAP-NEXT:    subl $16, %esp
+; SHRINK-WRAP-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; SHRINK-WRAP-NEXT:    cmpl $32, %eax
+; SHRINK-WRAP-NEXT:    jl LBB0_4
+; SHRINK-WRAP-NEXT:  # %bb.3: # %if.end
+; SHRINK-WRAP-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; SHRINK-WRAP-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; SHRINK-WRAP-NEXT:    pushl %eax
+; SHRINK-WRAP-NEXT:    pushl %edx
+; SHRINK-WRAP-NEXT:    calll _bar
+; SHRINK-WRAP-NEXT:  LBB0_4: # %cleanup
+; SHRINK-WRAP-NEXT:    movl %ebp, %esp
+; SHRINK-WRAP-NEXT:    popl %ebp
+; SHRINK-WRAP-NEXT:    retl
+;
+; NO-SHRINK-WRAP-LABEL: stackRealignment:
+; NO-SHRINK-WRAP:       # %bb.0: # %entry
+; NO-SHRINK-WRAP-NEXT:    pushl %ebp
+; NO-SHRINK-WRAP-NEXT:    movl %esp, %ebp
+; NO-SHRINK-WRAP-NEXT:    andl $-8, %esp
+; NO-SHRINK-WRAP-NEXT:    subl $16, %esp
+; NO-SHRINK-WRAP-NEXT:    movl (%ecx), %eax
+; NO-SHRINK-WRAP-NEXT:    cmpl $33, %eax
+; NO-SHRINK-WRAP-NEXT:    movl $42, %edx
+; NO-SHRINK-WRAP-NEXT:    jge LBB0_2
+; NO-SHRINK-WRAP-NEXT:  # %bb.1: # %entry
+; NO-SHRINK-WRAP-NEXT:    movl $128, %edx
+; NO-SHRINK-WRAP-NEXT:  LBB0_2: # %entry
+; NO-SHRINK-WRAP-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; NO-SHRINK-WRAP-NEXT:    cmpl $32, %eax
+; NO-SHRINK-WRAP-NEXT:    jl LBB0_4
+; NO-SHRINK-WRAP-NEXT:  # %bb.3: # %if.end
+; NO-SHRINK-WRAP-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; NO-SHRINK-WRAP-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; NO-SHRINK-WRAP-NEXT:    pushl %eax
+; NO-SHRINK-WRAP-NEXT:    pushl %edx
+; NO-SHRINK-WRAP-NEXT:    calll _bar
+; NO-SHRINK-WRAP-NEXT:  LBB0_4: # %cleanup
+; NO-SHRINK-WRAP-NEXT:    movl %ebp, %esp
+; NO-SHRINK-WRAP-NEXT:    popl %ebp
+; NO-SHRINK-WRAP-NEXT:    retl
 entry:
   %data = alloca [1 x i32], align 4
   %d = alloca double, align 8
diff --git a/llvm/test/CodeGen/X86/inline-asm-flag-output.ll b/llvm/test/CodeGen/X86/inline-asm-flag-output.ll
index e2c407ce264c2..0afdb740233d9 100644
--- a/llvm/test/CodeGen/X86/inline-asm-flag-output.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-flag-output.ll
@@ -888,15 +888,15 @@ define void @test_cca_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_cca_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jbe .LBB28_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB28_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@cca},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -928,15 +928,15 @@ define void @test_ccae_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccae_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jb .LBB29_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB29_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccae},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -968,15 +968,15 @@ define void @test_ccb_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccb_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jae .LBB30_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB30_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccb},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1008,15 +1008,15 @@ define void @test_ccbe_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccbe_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    ja .LBB31_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB31_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccbe},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1048,15 +1048,15 @@ define void @test_ccc_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccc_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jae .LBB32_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB32_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccc},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1088,15 +1088,15 @@ define void @test_cce_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_cce_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jne .LBB33_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB33_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@cce},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1128,15 +1128,15 @@ define void @test_ccz_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccz_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jne .LBB34_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB34_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccz},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1168,15 +1168,15 @@ define void @test_ccg_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccg_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jle .LBB35_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB35_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccg},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1208,15 +1208,15 @@ define void @test_ccge_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccge_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jl .LBB36_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB36_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccge},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1248,15 +1248,15 @@ define void @test_ccl_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccl_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jge .LBB37_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB37_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccl},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1288,15 +1288,15 @@ define void @test_ccle_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccle_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jg .LBB38_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB38_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccle},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1328,15 +1328,15 @@ define void @test_ccna_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccna_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    ja .LBB39_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB39_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccna},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1368,15 +1368,15 @@ define void @test_ccnae_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnae_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jae .LBB40_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB40_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnae},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1408,15 +1408,15 @@ define void @test_ccnb_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnb_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jb .LBB41_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB41_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnb},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1448,15 +1448,15 @@ define void @test_ccnbe_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnbe_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jbe .LBB42_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB42_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnbe},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1488,15 +1488,15 @@ define void @test_ccnc_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnc_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jb .LBB43_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB43_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnc},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1528,15 +1528,15 @@ define void @test_ccne_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccne_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    je .LBB44_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB44_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccne},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1568,15 +1568,15 @@ define void @test_ccnz_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnz_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    je .LBB45_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB45_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnz},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1608,15 +1608,15 @@ define void @test_ccng_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccng_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jg .LBB46_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB46_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccng},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1648,15 +1648,15 @@ define void @test_ccnge_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnge_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jge .LBB47_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB47_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnge},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1688,15 +1688,15 @@ define void @test_ccnl_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnl_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jl .LBB48_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB48_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnl},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1728,15 +1728,15 @@ define void @test_ccnle_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnle_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jle .LBB49_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB49_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnle},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1768,15 +1768,15 @@ define void @test_ccno_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccno_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jo .LBB50_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB50_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccno},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1808,15 +1808,15 @@ define void @test_ccnp_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccnp_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jp .LBB51_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB51_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccnp},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1848,15 +1848,15 @@ define void @test_ccns_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccns_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    js .LBB52_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB52_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccns},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1888,15 +1888,15 @@ define void @test_cco_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_cco_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jno .LBB53_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB53_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@cco},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1928,15 +1928,15 @@ define void @test_ccp_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccp_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jnp .LBB54_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB54_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccp},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
@@ -1968,15 +1968,15 @@ define void @test_ccs_branch(i64 %nr, ptr %addr) nounwind {
 ;
 ; X64-LABEL: test_ccs_branch:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    pushq %rax
 ; X64-NEXT:    #APP
 ; X64-NEXT:    cmp %rdi,(%rsi)
 ; X64-NEXT:    #NO_APP
 ; X64-NEXT:    jns .LBB55_2
 ; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    pushq %rax
 ; X64-NEXT:    callq bar@PLT
+; X64-NEXT:    addq $8, %rsp
 ; X64-NEXT:  .LBB55_2: # %exit
-; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
 entry:
   %cc = tail call i8 asm "cmp $2,$1", "={@ccs},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i64) %addr, i64 %nr) nounwind
diff --git a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
index fba710e839b22..c15e73daceb9b 100644
--- a/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
+++ b/llvm/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -11,59 +11,51 @@
 define void @truncstore_v8i64_v8i32(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pxor %xmm8, %xmm8
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm1, %xmm10
-; SSE2-NEXT:    pxor %xmm8, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm11
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    movdqa %xmm9, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm11
-; SSE2-NEXT:    pand %xmm11, %xmm1
-; SSE2-NEXT:    pandn %xmm6, %xmm11
-; SSE2-NEXT:    por %xmm1, %xmm11
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm1, %xmm9
+; SSE2-NEXT:    pxor %xmm10, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm7, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm9
+; SSE2-NEXT:    pand %xmm12, %xmm9
+; SSE2-NEXT:    pand %xmm9, %xmm1
+; SSE2-NEXT:    pandn %xmm6, %xmm9
+; SSE2-NEXT:    por %xmm1, %xmm9
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
-; SSE2-NEXT:    movdqa %xmm9, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm10, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm13, %xmm1
+; SSE2-NEXT:    pxor %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm7, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm1
+; SSE2-NEXT:    pand %xmm12, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pandn %xmm6, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm11[0,2]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm9[0,2]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
-; SSE2-NEXT:    movdqa %xmm9, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pand %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
-; SSE2-NEXT:    movdqa %xmm2, %xmm11
-; SSE2-NEXT:    pxor %xmm8, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm12, %xmm8
+; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm7, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm9
+; SSE2-NEXT:    pxor %xmm10, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm9[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm11
-; SSE2-NEXT:    pxor %xmm11, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE2-NEXT:    pxor %xmm11, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
 ; SSE2-NEXT:    packssdw %xmm5, %xmm4
 ; SSE2-NEXT:    packsswb %xmm4, %xmm4
 ; SSE2-NEXT:    pmovmskb %xmm4, %eax
@@ -72,8 +64,8 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; SSE2-NEXT:  # %bb.1: # %cond.store
 ; SSE2-NEXT:    movss %xmm1, (%rdi)
 ; SSE2-NEXT:  .LBB0_2: # %else
-; SSE2-NEXT:    por %xmm10, %xmm0
-; SSE2-NEXT:    por %xmm9, %xmm8
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pand %xmm7, %xmm9
 ; SSE2-NEXT:    testb $2, %al
 ; SSE2-NEXT:    je .LBB0_4
 ; SSE2-NEXT:  # %bb.3: # %cond.store1
@@ -82,8 +74,8 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; SSE2-NEXT:  .LBB0_4: # %else2
 ; SSE2-NEXT:    pand %xmm0, %xmm3
 ; SSE2-NEXT:    pandn %xmm6, %xmm0
-; SSE2-NEXT:    pand %xmm8, %xmm2
-; SSE2-NEXT:    pandn %xmm6, %xmm8
+; SSE2-NEXT:    pand %xmm9, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm9
 ; SSE2-NEXT:    testb $4, %al
 ; SSE2-NEXT:    je .LBB0_6
 ; SSE2-NEXT:  # %bb.5: # %cond.store3
@@ -91,7 +83,7 @@ define void @truncstore_v8i64_v8i32(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; SSE2-NEXT:    movd %xmm4, 8(%rdi)
 ; SSE2-NEXT:  .LBB0_6: # %else4
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm9, %xmm2
 ; SSE2-NEXT:    testb $8, %al
 ; SSE2-NEXT:    je .LBB0_8
 ; SSE2-NEXT:  # %bb.7: # %cond.store5
@@ -319,68 +311,60 @@ define void @truncstore_v8i64_v8i16(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; SSE2-LABEL: truncstore_v8i64_v8i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    pxor %xmm9, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm11
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pxor %xmm9, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE2-NEXT:    movdqa %xmm10, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm13, %xmm8
-; SSE2-NEXT:    pand %xmm8, %xmm2
-; SSE2-NEXT:    pandn %xmm7, %xmm8
-; SSE2-NEXT:    por %xmm2, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm7
+; SSE2-NEXT:    por %xmm2, %xmm7
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm10, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm13, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pand %xmm12, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm9, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm3[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm10, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm11
-; SSE2-NEXT:    pand %xmm11, %xmm0
-; SSE2-NEXT:    pandn %xmm7, %xmm11
-; SSE2-NEXT:    por %xmm0, %xmm11
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE2-NEXT:    pand %xmm12, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pandn %xmm8, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm7, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
@@ -817,58 +801,50 @@ define void @truncstore_v8i64_v8i8(<8 x i64> %x, ptr %p, <8 x i32> %mask) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm10
 ; SSE2-NEXT:    pxor %xmm8, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm11
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm9, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm11
-; SSE2-NEXT:    pand %xmm11, %xmm1
-; SSE2-NEXT:    pandn %xmm7, %xmm11
-; SSE2-NEXT:    por %xmm1, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    pand %xmm12, %xmm10
+; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    pandn %xmm7, %xmm10
+; SSE2-NEXT:    por %xmm1, %xmm10
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm9, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm10, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm13, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm1
+; SSE2-NEXT:    pand %xmm12, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pandn %xmm7, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm11, %xmm1
+; SSE2-NEXT:    packuswb %xmm10, %xmm1
 ; SSE2-NEXT:    movdqa %xmm3, %xmm0
 ; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm9, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[0,0,2,2]
-; SSE2-NEXT:    pand %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm10
-; SSE2-NEXT:    pand %xmm10, %xmm3
-; SSE2-NEXT:    pandn %xmm7, %xmm10
-; SSE2-NEXT:    por %xmm3, %xmm10
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pandn %xmm7, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm8, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm2
 ; SSE2-NEXT:    pandn %xmm7, %xmm3
 ; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    packuswb %xmm10, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
 ; SSE2-NEXT:    packuswb %xmm3, %xmm1
 ; SSE2-NEXT:    packuswb %xmm1, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
@@ -1300,31 +1276,27 @@ define void @truncstore_v4i64_v4i32(<4 x i64> %x, ptr %p, <4 x i32> %mask) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
 ; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483647,2147483647,2147483647,2147483647]
 ; SSE2-NEXT:    movdqa %xmm8, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm1
-; SSE2-NEXT:    pandn %xmm4, %xmm7
-; SSE2-NEXT:    por %xmm1, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pandn %xmm4, %xmm6
+; SSE2-NEXT:    por %xmm1, %xmm6
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
 ; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pandn %xmm4, %xmm1
 ; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm7[0,2]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    movmskps %xmm3, %eax
 ; SSE2-NEXT:    xorl $15, %eax
@@ -1496,33 +1468,29 @@ define void @truncstore_v4i64_v4i16(<4 x i64> %x, ptr %p, <4 x i32> %mask) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm6
 ; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE2-NEXT:    movdqa %xmm8, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pandn %xmm4, %xmm7
-; SSE2-NEXT:    por %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2]
+; SSE2-NEXT:    pand %xmm9, %xmm6
 ; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm4, %xmm5
-; SSE2-NEXT:    por %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; SSE2-NEXT:    pandn %xmm4, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
@@ -1780,33 +1748,29 @@ define void @truncstore_v4i64_v4i8(<4 x i64> %x, ptr %p, <4 x i32> %mask) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pxor %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm8, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm0
 ; SSE2-NEXT:    pandn %xmm5, %xmm4
 ; SSE2-NEXT:    por %xmm0, %xmm4
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    pxor %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm6
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm5, %xmm6
-; SSE2-NEXT:    por %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm8, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pand %xmm5, %xmm4
-; SSE2-NEXT:    packuswb %xmm6, %xmm4
+; SSE2-NEXT:    packuswb %xmm0, %xmm4
 ; SSE2-NEXT:    packuswb %xmm4, %xmm4
 ; SSE2-NEXT:    packuswb %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
@@ -2068,18 +2032,16 @@ define void @truncstore_v2i64_v2i32(<2 x i64> %x, ptr %p, <2 x i64> %mask) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm1
@@ -2206,18 +2168,16 @@ define void @truncstore_v2i64_v2i16(<2 x i64> %x, ptr %p, <2 x i64> %mask) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,0,3,2]
@@ -2355,28 +2315,26 @@ define void @truncstore_v2i64_v2i8(<2 x i64> %x, ptr %p, <2 x i64> %mask) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
 ; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    packuswb %xmm4, %xmm4
-; SSE2-NEXT:    packuswb %xmm4, %xmm4
-; SSE2-NEXT:    packuswb %xmm4, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm0, %xmm3
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
+; SSE2-NEXT:    packuswb %xmm3, %xmm3
 ; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
 ; SSE2-NEXT:    xorl $3, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    movd %xmm4, %ecx
+; SSE2-NEXT:    movd %xmm3, %ecx
 ; SSE2-NEXT:    jne .LBB8_1
 ; SSE2-NEXT:  # %bb.2: # %else
 ; SSE2-NEXT:    testb $2, %al
diff --git a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
index 6d95be38effa7..f70b7a227172a 100644
--- a/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int-vec-128.ll
@@ -1922,9 +1922,7 @@ define <8 x i16> @vec128_i16_unsigned_reg_reg(<8 x i16> %a1, <8 x i16> %a2) noun
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    psubusw %xmm1, %xmm3
 ; SSE2-NEXT:    psubusw %xmm0, %xmm1
-; SSE2-NEXT:    paddw %xmm0, %xmm1
-; SSE2-NEXT:    psubw %xmm0, %xmm3
-; SSE2-NEXT:    paddw %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
 ; SSE2-NEXT:    psrlw $1, %xmm3
 ; SSE2-NEXT:    pmullw %xmm2, %xmm3
 ; SSE2-NEXT:    paddw %xmm3, %xmm0
diff --git a/llvm/test/CodeGen/X86/peep-test-5.ll b/llvm/test/CodeGen/X86/peep-test-5.ll
index 8284824741d11..52bcbe9f83d7a 100644
--- a/llvm/test/CodeGen/X86/peep-test-5.ll
+++ b/llvm/test/CodeGen/X86/peep-test-5.ll
@@ -12,25 +12,22 @@
 define void @decref(ptr %p) {
 ; CHECK-LABEL: decref:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    movl (%rdi), %eax
 ; CHECK-NEXT:    cmpl $1, %eax
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %bb_free
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq free_object@PLT
-; CHECK-NEXT:  .LBB0_4: # %end
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:  .LBB0_4: # %end
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB0_2: # %bb2
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    jle .LBB0_4
 ; CHECK-NEXT:  # %bb.3: # %bb_dec
 ; CHECK-NEXT:    decl %eax
 ; CHECK-NEXT:    movl %eax, (%rdi)
-; CHECK-NEXT:    popq %rax
-; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
   %count = load i32, ptr %p, align 4
   %cmp0 = icmp eq i32 %count, 1
diff --git a/llvm/test/CodeGen/X86/powi-int32min.ll b/llvm/test/CodeGen/X86/powi-int32min.ll
new file mode 100644
index 0000000000000..3a1304c61c49a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/powi-int32min.ll
@@ -0,0 +1,17 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck %s
+
+define float @test_powi(ptr %p) nounwind {
+; CHECK-LABEL: test_powi:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:        movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-COUNT-31:    mulss %xmm1, %xmm1
+; CHECK-NEXT:        movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:        divss %xmm1, %xmm0
+; CHECK-NEXT:        retq
+bb:
+  %load = load float, ptr %p, align 4
+  %call = call contract float @llvm.powi.f32.i32(float %load, i32 -2147483648)
+  ret float %call
+}
+
+declare float @llvm.powi.f32.i32(float, i32)
diff --git a/llvm/test/CodeGen/X86/pr62242.ll b/llvm/test/CodeGen/X86/pr62242.ll
new file mode 100644
index 0000000000000..2753a18bfc122
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr62242.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
+
+define <4 x i32> @unpck_permute_canonicalized_mask(<4 x i32> %a, <4 x i32> %b) {
+; AVX2-LABEL: unpck_permute_canonicalized_mask:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,3,2,1]
+; AVX2-NEXT:    retq
+  ; This mask will get canonicalized to vector_shuffle<6, 3, -1, 2> %b, %a.
+  ; Make sure the generated permute masks are still correct.
+  %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 2, i32 7, i32 undef, i32 6>
+  ret <4 x i32> %shuffle
+}
diff --git a/llvm/test/CodeGen/X86/psubus.ll b/llvm/test/CodeGen/X86/psubus.ll
index 1afadf814c76b..9b25707fcc8be 100644
--- a/llvm/test/CodeGen/X86/psubus.ll
+++ b/llvm/test/CodeGen/X86/psubus.ll
@@ -1611,15 +1611,13 @@ define <8 x i16> @psubus_8i64_max(<8 x i16> %x, <8 x i64> %y) nounwind {
 ; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE2OR3-NEXT:    movdqa %xmm2, %xmm7
 ; SSE2OR3-NEXT:    pxor %xmm5, %xmm7
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2OR3-NEXT:    movdqa %xmm6, %xmm9
+; SSE2OR3-NEXT:    pcmpgtd %xmm8, %xmm9
 ; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
 ; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
-; SSE2OR3-NEXT:    movdqa %xmm6, %xmm9
-; SSE2OR3-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm8, %xmm7
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm7, %xmm8
+; SSE2OR3-NEXT:    pand %xmm9, %xmm8
 ; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm7 = [65535,65535]
 ; SSE2OR3-NEXT:    pand %xmm8, %xmm2
 ; SSE2OR3-NEXT:    pandn %xmm7, %xmm8
@@ -1628,48 +1626,42 @@ define <8 x i16> @psubus_8i64_max(<8 x i16> %x, <8 x i64> %y) nounwind {
 ; SSE2OR3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
 ; SSE2OR3-NEXT:    movdqa %xmm1, %xmm8
 ; SSE2OR3-NEXT:    pxor %xmm5, %xmm8
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm9
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
 ; SSE2OR3-NEXT:    movdqa %xmm6, %xmm10
-; SSE2OR3-NEXT:    pcmpgtd %xmm8, %xmm10
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm9, %xmm8
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm8, %xmm9
-; SSE2OR3-NEXT:    pand %xmm9, %xmm1
-; SSE2OR3-NEXT:    pandn %xmm7, %xmm9
-; SSE2OR3-NEXT:    por %xmm1, %xmm9
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm1 = xmm9[0,2,2,3]
+; SSE2OR3-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSE2OR3-NEXT:    pand %xmm10, %xmm8
+; SSE2OR3-NEXT:    pand %xmm8, %xmm1
+; SSE2OR3-NEXT:    pandn %xmm7, %xmm8
+; SSE2OR3-NEXT:    por %xmm1, %xmm8
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[0,2,2,3]
 ; SSE2OR3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2OR3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE2OR3-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2OR3-NEXT:    pxor %xmm5, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
 ; SSE2OR3-NEXT:    movdqa %xmm6, %xmm9
-; SSE2OR3-NEXT:    pcmpgtd %xmm2, %xmm9
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm8, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm2, %xmm8
-; SSE2OR3-NEXT:    pand %xmm8, %xmm4
-; SSE2OR3-NEXT:    pandn %xmm7, %xmm8
-; SSE2OR3-NEXT:    por %xmm4, %xmm8
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm8[0,2,2,3]
+; SSE2OR3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2OR3-NEXT:    pand %xmm9, %xmm2
+; SSE2OR3-NEXT:    pand %xmm2, %xmm4
+; SSE2OR3-NEXT:    pandn %xmm7, %xmm2
+; SSE2OR3-NEXT:    por %xmm4, %xmm2
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; SSE2OR3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,2,4,5,6,7]
 ; SSE2OR3-NEXT:    movdqa %xmm3, %xmm4
 ; SSE2OR3-NEXT:    pxor %xmm5, %xmm4
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSE2OR3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm8, %xmm4
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm4, %xmm5
-; SSE2OR3-NEXT:    pand %xmm5, %xmm3
-; SSE2OR3-NEXT:    pandn %xmm7, %xmm5
-; SSE2OR3-NEXT:    por %xmm3, %xmm5
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm8, %xmm6
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2OR3-NEXT:    pand %xmm6, %xmm4
+; SSE2OR3-NEXT:    pand %xmm4, %xmm3
+; SSE2OR3-NEXT:    pandn %xmm7, %xmm4
+; SSE2OR3-NEXT:    por %xmm3, %xmm4
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
 ; SSE2OR3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,0,2,4,5,6,7]
 ; SSE2OR3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; SSE2OR3-NEXT:    movsd {{.*#+}} xmm3 = xmm1[0],xmm3[1]
@@ -1678,56 +1670,53 @@ define <8 x i16> @psubus_8i64_max(<8 x i16> %x, <8 x i64> %y) nounwind {
 ;
 ; SSE41-LABEL: psubus_8i64_max:
 ; SSE41:       # %bb.0: # %vector.ph
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm8, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE41-NEXT:    movdqa %xmm6, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [65535,65535]
-; SSE41-NEXT:    movapd %xmm7, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm9
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm7, %xmm8
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [65535,65535]
+; SSE41-NEXT:    movapd %xmm8, %xmm10
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm10
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm8, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    movdqa %xmm6, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm4
+; SSE41-NEXT:    movapd %xmm8, %xmm4
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
-; SSE41-NEXT:    packusdw %xmm9, %xmm4
+; SSE41-NEXT:    packusdw %xmm10, %xmm4
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm8, %xmm0
-; SSE41-NEXT:    movdqa %xmm6, %xmm3
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm3
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm6, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm7, %xmm3
+; SSE41-NEXT:    movapd %xmm8, %xmm3
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm8
-; SSE41-NEXT:    movdqa %xmm6, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm8, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
-; SSE41-NEXT:    packusdw %xmm3, %xmm7
-; SSE41-NEXT:    packusdw %xmm4, %xmm7
-; SSE41-NEXT:    psubusw %xmm7, %xmm5
+; SSE41-NEXT:    pxor %xmm1, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pand %xmm7, %xmm5
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm8
+; SSE41-NEXT:    packusdw %xmm3, %xmm8
+; SSE41-NEXT:    packusdw %xmm4, %xmm8
+; SSE41-NEXT:    psubusw %xmm8, %xmm6
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: psubus_8i64_max:
@@ -2675,127 +2664,116 @@ define <8 x i32> @test33(<8 x i32> %a0, <8 x i64> %a1) {
 ; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
 ; SSE2OR3-NEXT:    movdqa %xmm3, %xmm8
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm8
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2OR3-NEXT:    movdqa %xmm7, %xmm10
+; SSE2OR3-NEXT:    pcmpgtd %xmm9, %xmm10
 ; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[1,1,3,3]
 ; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm9
-; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259455,9223372039002259455]
-; SSE2OR3-NEXT:    movdqa %xmm7, %xmm10
-; SSE2OR3-NEXT:    pcmpgtd %xmm8, %xmm10
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm9, %xmm8
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm8, %xmm9
+; SSE2OR3-NEXT:    pand %xmm10, %xmm9
 ; SSE2OR3-NEXT:    pcmpeqd %xmm8, %xmm8
 ; SSE2OR3-NEXT:    pand %xmm9, %xmm3
 ; SSE2OR3-NEXT:    pxor %xmm8, %xmm9
 ; SSE2OR3-NEXT:    por %xmm3, %xmm9
 ; SSE2OR3-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm10
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2]
 ; SSE2OR3-NEXT:    movdqa %xmm7, %xmm11
-; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm11
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm10, %xmm3
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm3, %xmm10
-; SSE2OR3-NEXT:    pand %xmm10, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm8, %xmm10
-; SSE2OR3-NEXT:    por %xmm2, %xmm10
-; SSE2OR3-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,2],xmm9[0,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2OR3-NEXT:    pand %xmm11, %xmm3
+; SSE2OR3-NEXT:    pand %xmm3, %xmm2
+; SSE2OR3-NEXT:    pxor %xmm8, %xmm3
+; SSE2OR3-NEXT:    por %xmm2, %xmm3
+; SSE2OR3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm9[0,2]
 ; SSE2OR3-NEXT:    movdqa %xmm0, %xmm2
-; SSE2OR3-NEXT:    psubd %xmm10, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm6, %xmm10
+; SSE2OR3-NEXT:    psubd %xmm3, %xmm2
+; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm0
-; SSE2OR3-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm0
 ; SSE2OR3-NEXT:    pand %xmm2, %xmm0
 ; SSE2OR3-NEXT:    movdqa %xmm5, %xmm2
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
 ; SSE2OR3-NEXT:    movdqa %xmm7, %xmm9
-; SSE2OR3-NEXT:    pcmpgtd %xmm2, %xmm9
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm3, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm9[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm2, %xmm3
-; SSE2OR3-NEXT:    pand %xmm3, %xmm5
-; SSE2OR3-NEXT:    pxor %xmm8, %xmm3
-; SSE2OR3-NEXT:    por %xmm5, %xmm3
-; SSE2OR3-NEXT:    movdqa %xmm4, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm6, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2OR3-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm5, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm2, %xmm5
-; SSE2OR3-NEXT:    pxor %xmm5, %xmm8
-; SSE2OR3-NEXT:    pand %xmm4, %xmm5
-; SSE2OR3-NEXT:    por %xmm8, %xmm5
-; SSE2OR3-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2],xmm3[0,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE2OR3-NEXT:    pand %xmm9, %xmm2
+; SSE2OR3-NEXT:    pand %xmm2, %xmm5
+; SSE2OR3-NEXT:    pxor %xmm8, %xmm2
+; SSE2OR3-NEXT:    por %xmm5, %xmm2
+; SSE2OR3-NEXT:    movdqa %xmm4, %xmm3
+; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2OR3-NEXT:    pand %xmm7, %xmm3
+; SSE2OR3-NEXT:    pxor %xmm3, %xmm8
+; SSE2OR3-NEXT:    pand %xmm4, %xmm3
+; SSE2OR3-NEXT:    por %xmm8, %xmm3
+; SSE2OR3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2]
 ; SSE2OR3-NEXT:    movdqa %xmm1, %xmm2
-; SSE2OR3-NEXT:    psubd %xmm5, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm6, %xmm5
+; SSE2OR3-NEXT:    psubd %xmm3, %xmm2
+; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm1
-; SSE2OR3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm1
 ; SSE2OR3-NEXT:    pand %xmm2, %xmm1
 ; SSE2OR3-NEXT:    retq
 ;
 ; SSE41-LABEL: test33:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm0, %xmm7
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm7, %xmm8
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm7, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSE41-NEXT:    movapd %xmm8, %xmm10
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm10
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm8, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm9 = [4294967295,4294967295]
+; SSE41-NEXT:    movapd %xmm9, %xmm11
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm11
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm3
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm7, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm0
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm11, %xmm0
-; SSE41-NEXT:    movapd %xmm8, %xmm3
+; SSE41-NEXT:    movapd %xmm9, %xmm3
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm10[0,2]
-; SSE41-NEXT:    pmaxud %xmm3, %xmm6
-; SSE41-NEXT:    psubd %xmm3, %xmm6
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm11[0,2]
+; SSE41-NEXT:    pmaxud %xmm3, %xmm7
+; SSE41-NEXT:    psubd %xmm3, %xmm7
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm7, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
 ; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm8, %xmm2
+; SSE41-NEXT:    movapd %xmm9, %xmm2
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm9
-; SSE41-NEXT:    movdqa %xmm7, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
-; SSE41-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,2],xmm2[0,2]
-; SSE41-NEXT:    pmaxud %xmm8, %xmm1
-; SSE41-NEXT:    psubd %xmm8, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm10
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm8
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pand %xmm8, %xmm6
 ; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm9
+; SSE41-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,2],xmm2[0,2]
+; SSE41-NEXT:    pmaxud %xmm9, %xmm1
+; SSE41-NEXT:    psubd %xmm9, %xmm1
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test33:
@@ -2906,70 +2884,62 @@ define <8 x i32> @test34(<8 x i32> %a0, <8 x i64> %a1) {
 ; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
 ; SSE2OR3-NEXT:    movdqa %xmm3, %xmm8
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm8
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2OR3-NEXT:    movdqa %xmm7, %xmm10
+; SSE2OR3-NEXT:    pcmpgtd %xmm9, %xmm10
 ; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[1,1,3,3]
 ; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm9
-; SSE2OR3-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259455,9223372039002259455]
-; SSE2OR3-NEXT:    movdqa %xmm7, %xmm10
-; SSE2OR3-NEXT:    pcmpgtd %xmm8, %xmm10
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm9, %xmm8
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm8, %xmm9
+; SSE2OR3-NEXT:    pand %xmm10, %xmm9
 ; SSE2OR3-NEXT:    pcmpeqd %xmm8, %xmm8
 ; SSE2OR3-NEXT:    pand %xmm9, %xmm3
 ; SSE2OR3-NEXT:    pxor %xmm8, %xmm9
 ; SSE2OR3-NEXT:    por %xmm3, %xmm9
 ; SSE2OR3-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm10
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[0,0,2,2]
 ; SSE2OR3-NEXT:    movdqa %xmm7, %xmm11
-; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm11
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm11[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm10, %xmm3
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm3, %xmm10
-; SSE2OR3-NEXT:    pand %xmm10, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm8, %xmm10
-; SSE2OR3-NEXT:    por %xmm2, %xmm10
-; SSE2OR3-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,2],xmm9[0,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2OR3-NEXT:    pand %xmm11, %xmm3
+; SSE2OR3-NEXT:    pand %xmm3, %xmm2
+; SSE2OR3-NEXT:    pxor %xmm8, %xmm3
+; SSE2OR3-NEXT:    por %xmm2, %xmm3
+; SSE2OR3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm9[0,2]
 ; SSE2OR3-NEXT:    movdqa %xmm0, %xmm2
-; SSE2OR3-NEXT:    psubd %xmm10, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm6, %xmm10
+; SSE2OR3-NEXT:    psubd %xmm3, %xmm2
+; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
 ; SSE2OR3-NEXT:    por %xmm6, %xmm0
-; SSE2OR3-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm0
 ; SSE2OR3-NEXT:    pand %xmm2, %xmm0
 ; SSE2OR3-NEXT:    movdqa %xmm5, %xmm2
 ; SSE2OR3-NEXT:    pxor %xmm6, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
 ; SSE2OR3-NEXT:    movdqa %xmm7, %xmm9
-; SSE2OR3-NEXT:    pcmpgtd %xmm2, %xmm9
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm3, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm9[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm2, %xmm3
-; SSE2OR3-NEXT:    pand %xmm3, %xmm5
-; SSE2OR3-NEXT:    pxor %xmm8, %xmm3
-; SSE2OR3-NEXT:    por %xmm5, %xmm3
-; SSE2OR3-NEXT:    movdqa %xmm4, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm6, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2OR3-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,0,2,2]
-; SSE2OR3-NEXT:    pand %xmm5, %xmm2
-; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2OR3-NEXT:    por %xmm2, %xmm5
-; SSE2OR3-NEXT:    pxor %xmm5, %xmm8
-; SSE2OR3-NEXT:    pand %xmm4, %xmm5
-; SSE2OR3-NEXT:    por %xmm8, %xmm5
-; SSE2OR3-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2],xmm3[0,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE2OR3-NEXT:    pand %xmm9, %xmm2
+; SSE2OR3-NEXT:    pand %xmm2, %xmm5
+; SSE2OR3-NEXT:    pxor %xmm8, %xmm2
+; SSE2OR3-NEXT:    por %xmm5, %xmm2
+; SSE2OR3-NEXT:    movdqa %xmm4, %xmm3
+; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2OR3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2OR3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2OR3-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE2OR3-NEXT:    pand %xmm7, %xmm3
+; SSE2OR3-NEXT:    pxor %xmm3, %xmm8
+; SSE2OR3-NEXT:    pand %xmm4, %xmm3
+; SSE2OR3-NEXT:    por %xmm8, %xmm3
+; SSE2OR3-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2]
 ; SSE2OR3-NEXT:    movdqa %xmm1, %xmm2
-; SSE2OR3-NEXT:    psubd %xmm5, %xmm2
-; SSE2OR3-NEXT:    pxor %xmm6, %xmm5
+; SSE2OR3-NEXT:    psubd %xmm3, %xmm2
+; SSE2OR3-NEXT:    pxor %xmm6, %xmm3
 ; SSE2OR3-NEXT:    por %xmm6, %xmm1
-; SSE2OR3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2OR3-NEXT:    pcmpgtd %xmm3, %xmm1
 ; SSE2OR3-NEXT:    pand %xmm2, %xmm1
 ; SSE2OR3-NEXT:    retq
 ;
@@ -2979,56 +2949,53 @@ define <8 x i32> @test34(<8 x i32> %a0, <8 x i64> %a1) {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [1,1,1,1]
 ; SSE41-NEXT:    pand %xmm0, %xmm1
 ; SSE41-NEXT:    pand %xmm0, %xmm6
-; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm7, %xmm8
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm7, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [4294967295,4294967295]
-; SSE41-NEXT:    movapd %xmm8, %xmm10
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm10
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm8, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm9 = [4294967295,4294967295]
+; SSE41-NEXT:    movapd %xmm9, %xmm11
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm11
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm3
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm7, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm0
 ; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm11, %xmm0
-; SSE41-NEXT:    movapd %xmm8, %xmm3
+; SSE41-NEXT:    movapd %xmm9, %xmm3
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm10[0,2]
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm11[0,2]
 ; SSE41-NEXT:    pmaxud %xmm3, %xmm6
 ; SSE41-NEXT:    psubd %xmm3, %xmm6
 ; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm7, %xmm2
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm2
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    movdqa %xmm7, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
 ; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movapd %xmm8, %xmm2
+; SSE41-NEXT:    movapd %xmm9, %xmm2
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm9
-; SSE41-NEXT:    movdqa %xmm7, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
-; SSE41-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,2],xmm2[0,2]
-; SSE41-NEXT:    pmaxud %xmm8, %xmm1
-; SSE41-NEXT:    psubd %xmm8, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm10
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm8
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pand %xmm8, %xmm7
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm9
+; SSE41-NEXT:    shufps {{.*#+}} xmm9 = xmm9[0,2],xmm2[0,2]
+; SSE41-NEXT:    pmaxud %xmm9, %xmm1
+; SSE41-NEXT:    psubd %xmm9, %xmm1
 ; SSE41-NEXT:    movdqa %xmm6, %xmm0
 ; SSE41-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll
index ef62872f51069..9407c5b6aa25c 100644
--- a/llvm/test/CodeGen/X86/sat-add.ll
+++ b/llvm/test/CodeGen/X86/sat-add.ll
@@ -750,14 +750,12 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [42,42]
 ; SSE2-NEXT:    paddq %xmm0, %xmm1
 ; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
@@ -765,14 +763,12 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [42,42]
 ; SSE41-NEXT:    paddq %xmm0, %xmm1
 ; SSE41-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE41-NEXT:    pand %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
 ; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; SSE42-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
diff --git a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
index ba3233943418d..0793c33d2e8eb 100644
--- a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
+++ b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
@@ -9,10 +9,6 @@
 define void @fn1() nounwind uwtable {
 ; CHECK-LABEL: fn1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movl $4136, %eax # imm = 0x1028
-; CHECK-NEXT:    callq ___chkstk_ms
-; CHECK-NEXT:    subq %rax, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 4144
 ; CHECK-NEXT:    movl a(%rip), %eax
 ; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    jne .LBB0_2
@@ -24,6 +20,12 @@ define void @fn1() nounwind uwtable {
 ; CHECK-NEXT:    shrq $32, %rax
 ; CHECK-NEXT:    addl %ecx, %eax
 ; CHECK-NEXT:  .LBB0_2: # %select.end
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl $4128, %eax # imm = 0x1020
+; CHECK-NEXT:    callq ___chkstk_ms
+; CHECK-NEXT:    subq %rax, %rsp
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 4144
 ; CHECK-NEXT:    movl %eax, b(%rip)
 ; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-NEXT:    # kill: def $ecx killed $ecx killed $rcx
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath.ll b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
index 333aebc503f58..54ea207ac5dc6 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
@@ -183,8 +183,8 @@ define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {
   ret <4 x float> %call
 }
 
-define <4 x float> @sqrt_v4f32_check_denorms_ninf(<4 x float> %x) #3 {
-; SSE-LABEL: sqrt_v4f32_check_denorms_ninf:
+define <4 x float> @sqrt_v4f32_check_denorms_ieee_ninf(<4 x float> %x) #3 {
+; SSE-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtps %xmm0, %xmm1
 ; SSE-NEXT:    movaps %xmm0, %xmm2
@@ -201,7 +201,7 @@ define <4 x float> @sqrt_v4f32_check_denorms_ninf(<4 x float> %x) #3 {
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: sqrt_v4f32_check_denorms_ninf:
+; AVX1-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vrsqrtps %xmm0, %xmm1
 ; AVX1-NEXT:    vmulps %xmm1, %xmm0, %xmm2
@@ -215,7 +215,58 @@ define <4 x float> @sqrt_v4f32_check_denorms_ninf(<4 x float> %x) #3 {
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
-; AVX512-LABEL: sqrt_v4f32_check_denorms_ninf:
+; AVX512-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrsqrtps %xmm0, %xmm1
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]
+; AVX512-NEXT:    vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; AVX512-NEXT:    vmulps %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vmulps %xmm3, %xmm1, %xmm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
+; AVX512-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]
+; AVX512-NEXT:    vcmpleps %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %call = tail call ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
+  ret <4 x float> %call
+}
+
+define <4 x float> @sqrt_v4f32_check_denorms_dynamic_ninf(<4 x float> %x) #6 {
+; SSE-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:
+; SSE:       # %bb.0:
+; SSE-NEXT:    rsqrtps %xmm0, %xmm1
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    mulps %xmm1, %xmm2
+; SSE-NEXT:    movaps {{.*#+}} xmm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]
+; SSE-NEXT:    mulps %xmm2, %xmm3
+; SSE-NEXT:    mulps %xmm1, %xmm2
+; SSE-NEXT:    addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE-NEXT:    mulps %xmm3, %xmm2
+; SSE-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]
+; SSE-NEXT:    cmpleps %xmm0, %xmm1
+; SSE-NEXT:    andps %xmm2, %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vrsqrtps %xmm0, %xmm1
+; AVX1-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
+; AVX1-NEXT:    vmulps %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vmulps %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]
+; AVX1-NEXT:    vcmpleps %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vrsqrtps %xmm0, %xmm1
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm2
@@ -971,6 +1022,7 @@ define double @sqrt_simplify_before_recip_order(double %x, ptr %p) nounwind {
 attributes #0 = { "unsafe-fp-math"="true" "reciprocal-estimates"="!sqrtf,!vec-sqrtf,!divf,!vec-divf" }
 attributes #1 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" }
 attributes #2 = { nounwind readnone }
-attributes #3 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="ieee" }
+attributes #3 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,ieee" }
 attributes #4 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="ieee,preserve-sign" }
 attributes #5 = { "unsafe-fp-math"="true" "reciprocal-estimates"="all:0" }
+attributes #6 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,dynamic" }
diff --git a/llvm/test/CodeGen/X86/statepoint-vector.ll b/llvm/test/CodeGen/X86/statepoint-vector.ll
index 5ce81592b8b0b..c8ddb89ee5698 100644
--- a/llvm/test/CodeGen/X86/statepoint-vector.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vector.ll
@@ -57,10 +57,10 @@ entry:
 define <2 x ptr addrspace(1)> @test3(i1 %cnd, ptr %ptr) gc "statepoint-example" {
 ; CHECK-LABEL: test3:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $40, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    testb $1, %dil
 ; CHECK-NEXT:    movaps (%rsi), %xmm0
+; CHECK-NEXT:    subq $40, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    movaps %xmm0, (%rsp)
 ; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    callq do_safepoint@PLT
diff --git a/llvm/test/CodeGen/X86/vec_setcc-2.ll b/llvm/test/CodeGen/X86/vec_setcc-2.ll
index 04b98d2f58b4f..367f9f21041a5 100644
--- a/llvm/test/CodeGen/X86/vec_setcc-2.ll
+++ b/llvm/test/CodeGen/X86/vec_setcc-2.ll
@@ -303,14 +303,12 @@ define <2 x i1> @uge_v2i64_splat(<2 x i64> %x) {
 ; SSE2-LABEL: uge_v2i64_splat:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259898,9223372039002259898]
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147484090,2147484090,2147484090,2147484090]
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE2-NEXT:    pxor %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -379,14 +377,12 @@ define <2 x i1> @ult_v2i64_splat(<2 x i64> %x) {
 ; SSE2-LABEL: ult_v2i64_splat:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259898,9223372039002259898]
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147484090,2147484090,2147484090,2147484090]
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_v2i64_splat:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
index be54bf12a43af..0a28126d1b3ac 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-5.ll
@@ -165,7 +165,7 @@ define void @store_i32_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm5
 ; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm6
 ; AVX1-ONLY-NEXT:    vunpcklps {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[4],ymm6[4],ymm5[5],ymm6[5]
-; AVX1-ONLY-NEXT:    vpermilps {{.*#+}} ymm5 = ymm5[u,u,1,0,u,u,u,6]
+; AVX1-ONLY-NEXT:    vshufpd {{.*#+}} ymm5 = ymm5[0,0,2,3]
 ; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
 ; AVX1-ONLY-NEXT:    vunpcklps {{.*#+}} ymm7 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[4],ymm1[4],ymm4[5],ymm1[5]
 ; AVX1-ONLY-NEXT:    vshufpd {{.*#+}} ymm7 = ymm7[0,0,3,3]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index 6f0919967a5d8..04bc4e5da890f 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -205,12 +205,12 @@ define void @store_i8_stride5_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 define void @store_i8_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %out.vec) nounwind {
 ; SSE-LABEL: store_i8_stride5_vf8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
 ; SSE-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
 ; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
 ; SSE-NEXT:    movdqa {{.*#+}} xmm8 = [65535,65535,0,65535,0,65535,65535,0]
 ; SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[2,1,2,3]
@@ -225,8 +225,8 @@ define void @store_i8_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    movdqa {{.*#+}} xmm6 = [255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255]
 ; SSE-NEXT:    movdqa %xmm6, %xmm10
 ; SSE-NEXT:    pandn %xmm5, %xmm10
-; SSE-NEXT:    movdqa %xmm1, %xmm7
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3],xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
+; SSE-NEXT:    movdqa %xmm2, %xmm7
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[2,1,2,3]
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[3,1,0,3,4,5,6,7]
 ; SSE-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,6,6]
@@ -249,7 +249,7 @@ define void @store_i8_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    pandn %xmm10, %xmm8
 ; SSE-NEXT:    por %xmm9, %xmm8
 ; SSE-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255]
-; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[0,0,0,0]
+; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,0,0]
 ; SSE-NEXT:    pand %xmm6, %xmm10
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm7 = xmm7[0,1,2,2,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[0,0,1,3]
@@ -271,8 +271,9 @@ define void @store_i8_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    por %xmm4, %xmm7
 ; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,0,0,255,255,255,255,255,255,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm3, %xmm7
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
-; SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,7,7,7]
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]
 ; SSE-NEXT:    pandn %xmm1, %xmm3
 ; SSE-NEXT:    por %xmm7, %xmm3
 ; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,0,255,255,255,255,0,255,255,255,255,255,255,255,255]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index b7c514efff32f..844dc41240166 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -288,35 +288,34 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-LABEL: store_i8_stride7_vf8:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
 ; SSE-NEXT:    movq {{.*#+}} xmm10 = mem[0],zero
-; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
+; SSE-NEXT:    movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
+; SSE-NEXT:    movq {{.*#+}} xmm7 = mem[0],zero
+; SSE-NEXT:    movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE-NEXT:    movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-NEXT:    movq {{.*#+}} xmm6 = mem[0],zero
-; SSE-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
-; SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm6[0]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm1[0,0,0,0,4,5,6,7]
+; SSE-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
+; SSE-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm3[0,0,0,0,4,5,6,7]
+; SSE-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,0,0,0]
 ; SSE-NEXT:    movdqa {{.*#+}} xmm11 = [255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255]
 ; SSE-NEXT:    pand %xmm11, %xmm6
-; SSE-NEXT:    movdqa %xmm5, %xmm7
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm8 = xmm7[0,0,2,1,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[0,1,1,3]
 ; SSE-NEXT:    pandn %xmm8, %xmm11
 ; SSE-NEXT:    por %xmm6, %xmm11
 ; SSE-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm0[0,0,2,1,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[0,0,2,1,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[0,0,2,1]
 ; SSE-NEXT:    movdqa {{.*#+}} xmm8 = [255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0]
 ; SSE-NEXT:    pand %xmm8, %xmm6
-; SSE-NEXT:    movdqa %xmm10, %xmm3
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm12 = xmm3[0,2,1,3,4,5,6,7]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm12 = xmm10[0,2,1,3,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[0,1,1,0]
 ; SSE-NEXT:    movdqa %xmm8, %xmm13
 ; SSE-NEXT:    pandn %xmm12, %xmm13
@@ -325,9 +324,8 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    pandn %xmm11, %xmm9
 ; SSE-NEXT:    por %xmm13, %xmm9
 ; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    movdqa %xmm4, %xmm15
-; SSE-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE-NEXT:    movdqa %xmm4, %xmm12
+; SSE-NEXT:    movdqa %xmm5, %xmm12
+; SSE-NEXT:    movdqa %xmm5, %xmm15
 ; SSE-NEXT:    punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm6[8],xmm15[9],xmm6[9],xmm15[10],xmm6[10],xmm15[11],xmm6[11],xmm15[12],xmm6[12],xmm15[13],xmm6[13],xmm15[14],xmm6[14],xmm15[15],xmm6[15]
 ; SSE-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1],xmm12[2],xmm6[2],xmm12[3],xmm6[3],xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
 ; SSE-NEXT:    movdqa %xmm12, %xmm13
@@ -351,20 +349,18 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm7[2,2,3,3]
 ; SSE-NEXT:    movdqa %xmm1, %xmm11
 ; SSE-NEXT:    pandn %xmm9, %xmm11
-; SSE-NEXT:    pshuflw $246, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
-; SSE-NEXT:    # xmm9 = mem[2,1,3,3,4,5,6,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm9 = xmm3[2,1,3,3,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm9[0,0,2,1]
 ; SSE-NEXT:    pand %xmm1, %xmm14
 ; SSE-NEXT:    por %xmm11, %xmm14
 ; SSE-NEXT:    movdqa {{.*#+}} xmm9 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255]
 ; SSE-NEXT:    movdqa %xmm9, %xmm11
 ; SSE-NEXT:    pandn %xmm14, %xmm11
-; SSE-NEXT:    pshufhw {{.*#+}} xmm14 = xmm3[0,1,2,3,5,6,6,7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm14 = xmm10[0,1,2,3,5,6,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm14[2,2,2,2]
 ; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255]
 ; SSE-NEXT:    movdqa %xmm0, %xmm14
 ; SSE-NEXT:    pandn %xmm2, %xmm14
-; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
 ; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,2,2,3,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,1,1,3]
 ; SSE-NEXT:    pand %xmm0, %xmm2
@@ -399,19 +395,19 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[0,0,0,0]
 ; SSE-NEXT:    pandn %xmm11, %xmm0
 ; SSE-NEXT:    por %xmm2, %xmm0
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,1,2,2]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm4[1,1,2,3,4,5,6,7]
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,2,1]
-; SSE-NEXT:    pand %xmm1, %xmm3
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm10[0,1,2,2]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm10 = xmm4[1,1,2,3,4,5,6,7]
+; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[0,0,2,1]
+; SSE-NEXT:    pand %xmm1, %xmm10
 ; SSE-NEXT:    pandn %xmm2, %xmm1
-; SSE-NEXT:    por %xmm3, %xmm1
-; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm11[1,1,2,2,4,5,6,7]
+; SSE-NEXT:    por %xmm10, %xmm1
+; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm3[1,1,2,2,4,5,6,7]
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,2,1]
 ; SSE-NEXT:    pand %xmm8, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,2,1]
-; SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,4]
-; SSE-NEXT:    pandn %xmm3, %xmm8
+; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,2,1]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,5,6,4]
+; SSE-NEXT:    pandn %xmm7, %xmm8
 ; SSE-NEXT:    por %xmm2, %xmm8
 ; SSE-NEXT:    pand %xmm9, %xmm8
 ; SSE-NEXT:    pandn %xmm1, %xmm9
@@ -420,27 +416,31 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; SSE-NEXT:    pand %xmm1, %xmm9
 ; SSE-NEXT:    pandn %xmm0, %xmm1
 ; SSE-NEXT:    por %xmm1, %xmm9
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3],xmm5[4],xmm11[4],xmm5[5],xmm11[5],xmm5[6],xmm11[6],xmm5[7],xmm11[7]
-; SSE-NEXT:    psrldq {{.*#+}} xmm5 = xmm5[11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; SSE-NEXT:    pandn %xmm5, %xmm0
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE-NEXT:    pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; SSE-NEXT:    # xmm1 = mem[3,1,2,3]
-; SSE-NEXT:    psrldq {{.*#+}} xmm10 = xmm10[13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE-NEXT:    por %xmm0, %xmm10
-; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm13[3,3,3,3,4,5,6,7]
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,0,0,255,255,255,255,255,255,255,255,255]
-; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7]
-; SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
-; SSE-NEXT:    pandn %xmm1, %xmm2
-; SSE-NEXT:    por %xmm0, %xmm2
+; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,0,0,255,255,255,255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7]
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
+; SSE-NEXT:    pand %xmm1, %xmm2
+; SSE-NEXT:    pandn %xmm0, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
+; SSE-NEXT:    por %xmm2, %xmm1
+; SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm13[3,3,3,3,4,5,6,7]
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,255,255,0,0,255,255,255,255,255,255,255,255,255]
+; SSE-NEXT:    pand %xmm3, %xmm2
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7]
+; SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]
+; SSE-NEXT:    pandn %xmm0, %xmm3
+; SSE-NEXT:    por %xmm2, %xmm3
 ; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [0,255,255,255,255,0,0,0,255,255,255,255,255,255,255,255]
-; SSE-NEXT:    pand %xmm0, %xmm10
-; SSE-NEXT:    pandn %xmm2, %xmm0
-; SSE-NEXT:    por %xmm10, %xmm0
+; SSE-NEXT:    pand %xmm0, %xmm1
+; SSE-NEXT:    pandn %xmm3, %xmm0
+; SSE-NEXT:    por %xmm1, %xmm0
 ; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; SSE-NEXT:    movq %xmm0, 48(%rax)
 ; SSE-NEXT:    movdqa %xmm9, 16(%rax)
diff --git a/llvm/test/CodeGen/X86/vector-popcnt-128-ult-ugt.ll b/llvm/test/CodeGen/X86/vector-popcnt-128-ult-ugt.ll
index d182e126bc105..56daf987c829e 100644
--- a/llvm/test/CodeGen/X86/vector-popcnt-128-ult-ugt.ll
+++ b/llvm/test/CodeGen/X86/vector-popcnt-128-ult-ugt.ll
@@ -16970,15 +16970,10 @@ define <2 x i64> @ugt_2_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_2_v2i64:
@@ -16998,15 +16993,10 @@ define <2 x i64> @ugt_2_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_2_v2i64:
@@ -17022,15 +17012,10 @@ define <2 x i64> @ugt_2_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_2_v2i64:
@@ -17046,15 +17031,10 @@ define <2 x i64> @ugt_2_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_2_v2i64:
@@ -17144,16 +17124,11 @@ define <2 x i64> @ult_3_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651,2147483651,2147483651]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_3_v2i64:
@@ -17173,16 +17148,11 @@ define <2 x i64> @ult_3_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651,2147483651,2147483651]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_3_v2i64:
@@ -17198,16 +17168,11 @@ define <2 x i64> @ult_3_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651,2147483651,2147483651]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_3_v2i64:
@@ -17223,16 +17188,11 @@ define <2 x i64> @ult_3_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483651,2147483651,2147483651,2147483651]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_3_v2i64:
@@ -17326,15 +17286,10 @@ define <2 x i64> @ugt_3_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_3_v2i64:
@@ -17354,15 +17309,10 @@ define <2 x i64> @ugt_3_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_3_v2i64:
@@ -17378,15 +17328,10 @@ define <2 x i64> @ugt_3_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_3_v2i64:
@@ -17402,15 +17347,10 @@ define <2 x i64> @ugt_3_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_3_v2i64:
@@ -17500,16 +17440,11 @@ define <2 x i64> @ult_4_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652,2147483652,2147483652]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_4_v2i64:
@@ -17529,16 +17464,11 @@ define <2 x i64> @ult_4_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652,2147483652,2147483652]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_4_v2i64:
@@ -17554,16 +17484,11 @@ define <2 x i64> @ult_4_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652,2147483652,2147483652]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_4_v2i64:
@@ -17579,16 +17504,11 @@ define <2 x i64> @ult_4_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483652,2147483652,2147483652,2147483652]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_4_v2i64:
@@ -17682,15 +17602,10 @@ define <2 x i64> @ugt_4_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_4_v2i64:
@@ -17710,15 +17625,10 @@ define <2 x i64> @ugt_4_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_4_v2i64:
@@ -17734,15 +17644,10 @@ define <2 x i64> @ugt_4_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_4_v2i64:
@@ -17758,15 +17663,10 @@ define <2 x i64> @ugt_4_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_4_v2i64:
@@ -17856,16 +17756,11 @@ define <2 x i64> @ult_5_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653,2147483653,2147483653]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_5_v2i64:
@@ -17885,16 +17780,11 @@ define <2 x i64> @ult_5_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653,2147483653,2147483653]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_5_v2i64:
@@ -17910,16 +17800,11 @@ define <2 x i64> @ult_5_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653,2147483653,2147483653]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_5_v2i64:
@@ -17935,16 +17820,11 @@ define <2 x i64> @ult_5_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483653,2147483653,2147483653,2147483653]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_5_v2i64:
@@ -18038,15 +17918,10 @@ define <2 x i64> @ugt_5_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_5_v2i64:
@@ -18066,15 +17941,10 @@ define <2 x i64> @ugt_5_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_5_v2i64:
@@ -18090,15 +17960,10 @@ define <2 x i64> @ugt_5_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_5_v2i64:
@@ -18114,15 +17979,10 @@ define <2 x i64> @ugt_5_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_5_v2i64:
@@ -18212,16 +18072,11 @@ define <2 x i64> @ult_6_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654,2147483654,2147483654]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_6_v2i64:
@@ -18241,16 +18096,11 @@ define <2 x i64> @ult_6_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654,2147483654,2147483654]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_6_v2i64:
@@ -18266,16 +18116,11 @@ define <2 x i64> @ult_6_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654,2147483654,2147483654]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_6_v2i64:
@@ -18291,16 +18136,11 @@ define <2 x i64> @ult_6_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483654,2147483654,2147483654,2147483654]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_6_v2i64:
@@ -18394,15 +18234,10 @@ define <2 x i64> @ugt_6_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_6_v2i64:
@@ -18422,15 +18257,10 @@ define <2 x i64> @ugt_6_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_6_v2i64:
@@ -18446,15 +18276,10 @@ define <2 x i64> @ugt_6_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_6_v2i64:
@@ -18470,15 +18295,10 @@ define <2 x i64> @ugt_6_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_6_v2i64:
@@ -18568,16 +18388,11 @@ define <2 x i64> @ult_7_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655,2147483655,2147483655]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_7_v2i64:
@@ -18597,16 +18412,11 @@ define <2 x i64> @ult_7_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655,2147483655,2147483655]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_7_v2i64:
@@ -18622,16 +18432,11 @@ define <2 x i64> @ult_7_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655,2147483655,2147483655]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_7_v2i64:
@@ -18647,16 +18452,11 @@ define <2 x i64> @ult_7_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483655,2147483655,2147483655,2147483655]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_7_v2i64:
@@ -18750,15 +18550,10 @@ define <2 x i64> @ugt_7_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_7_v2i64:
@@ -18778,15 +18573,10 @@ define <2 x i64> @ugt_7_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_7_v2i64:
@@ -18802,15 +18592,10 @@ define <2 x i64> @ugt_7_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_7_v2i64:
@@ -18826,15 +18611,10 @@ define <2 x i64> @ugt_7_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_7_v2i64:
@@ -18924,16 +18704,11 @@ define <2 x i64> @ult_8_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656,2147483656,2147483656]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_8_v2i64:
@@ -18953,16 +18728,11 @@ define <2 x i64> @ult_8_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656,2147483656,2147483656]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_8_v2i64:
@@ -18978,16 +18748,11 @@ define <2 x i64> @ult_8_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656,2147483656,2147483656]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_8_v2i64:
@@ -19003,16 +18768,11 @@ define <2 x i64> @ult_8_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483656,2147483656,2147483656,2147483656]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_8_v2i64:
@@ -19106,15 +18866,10 @@ define <2 x i64> @ugt_8_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_8_v2i64:
@@ -19134,15 +18889,10 @@ define <2 x i64> @ugt_8_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_8_v2i64:
@@ -19158,15 +18908,10 @@ define <2 x i64> @ugt_8_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_8_v2i64:
@@ -19182,15 +18927,10 @@ define <2 x i64> @ugt_8_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_8_v2i64:
@@ -19280,16 +19020,11 @@ define <2 x i64> @ult_9_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657,2147483657,2147483657]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_9_v2i64:
@@ -19309,16 +19044,11 @@ define <2 x i64> @ult_9_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657,2147483657,2147483657]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_9_v2i64:
@@ -19334,16 +19064,11 @@ define <2 x i64> @ult_9_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657,2147483657,2147483657]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_9_v2i64:
@@ -19359,16 +19084,11 @@ define <2 x i64> @ult_9_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483657,2147483657,2147483657,2147483657]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_9_v2i64:
@@ -19462,15 +19182,10 @@ define <2 x i64> @ugt_9_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_9_v2i64:
@@ -19490,15 +19205,10 @@ define <2 x i64> @ugt_9_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_9_v2i64:
@@ -19514,15 +19224,10 @@ define <2 x i64> @ugt_9_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_9_v2i64:
@@ -19538,15 +19243,10 @@ define <2 x i64> @ugt_9_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_9_v2i64:
@@ -19636,16 +19336,11 @@ define <2 x i64> @ult_10_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658,2147483658,2147483658]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_10_v2i64:
@@ -19665,16 +19360,11 @@ define <2 x i64> @ult_10_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658,2147483658,2147483658]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_10_v2i64:
@@ -19690,16 +19380,11 @@ define <2 x i64> @ult_10_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658,2147483658,2147483658]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_10_v2i64:
@@ -19715,16 +19400,11 @@ define <2 x i64> @ult_10_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483658,2147483658,2147483658,2147483658]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_10_v2i64:
@@ -19818,15 +19498,10 @@ define <2 x i64> @ugt_10_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_10_v2i64:
@@ -19846,15 +19521,10 @@ define <2 x i64> @ugt_10_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_10_v2i64:
@@ -19870,15 +19540,10 @@ define <2 x i64> @ugt_10_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_10_v2i64:
@@ -19894,15 +19559,10 @@ define <2 x i64> @ugt_10_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_10_v2i64:
@@ -19992,16 +19652,11 @@ define <2 x i64> @ult_11_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659,2147483659,2147483659]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_11_v2i64:
@@ -20021,16 +19676,11 @@ define <2 x i64> @ult_11_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659,2147483659,2147483659]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_11_v2i64:
@@ -20046,16 +19696,11 @@ define <2 x i64> @ult_11_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659,2147483659,2147483659]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_11_v2i64:
@@ -20071,16 +19716,11 @@ define <2 x i64> @ult_11_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483659,2147483659,2147483659,2147483659]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_11_v2i64:
@@ -20174,15 +19814,10 @@ define <2 x i64> @ugt_11_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_11_v2i64:
@@ -20202,15 +19837,10 @@ define <2 x i64> @ugt_11_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_11_v2i64:
@@ -20226,15 +19856,10 @@ define <2 x i64> @ugt_11_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_11_v2i64:
@@ -20250,15 +19875,10 @@ define <2 x i64> @ugt_11_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_11_v2i64:
@@ -20348,16 +19968,11 @@ define <2 x i64> @ult_12_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660,2147483660,2147483660]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_12_v2i64:
@@ -20377,16 +19992,11 @@ define <2 x i64> @ult_12_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660,2147483660,2147483660]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_12_v2i64:
@@ -20402,16 +20012,11 @@ define <2 x i64> @ult_12_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660,2147483660,2147483660]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_12_v2i64:
@@ -20427,16 +20032,11 @@ define <2 x i64> @ult_12_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483660,2147483660,2147483660,2147483660]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_12_v2i64:
@@ -20530,15 +20130,10 @@ define <2 x i64> @ugt_12_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_12_v2i64:
@@ -20558,15 +20153,10 @@ define <2 x i64> @ugt_12_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_12_v2i64:
@@ -20582,15 +20172,10 @@ define <2 x i64> @ugt_12_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_12_v2i64:
@@ -20606,15 +20191,10 @@ define <2 x i64> @ugt_12_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_12_v2i64:
@@ -20704,16 +20284,11 @@ define <2 x i64> @ult_13_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661,2147483661,2147483661]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_13_v2i64:
@@ -20733,16 +20308,11 @@ define <2 x i64> @ult_13_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661,2147483661,2147483661]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_13_v2i64:
@@ -20758,16 +20328,11 @@ define <2 x i64> @ult_13_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661,2147483661,2147483661]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_13_v2i64:
@@ -20783,16 +20348,11 @@ define <2 x i64> @ult_13_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483661,2147483661,2147483661,2147483661]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_13_v2i64:
@@ -20886,15 +20446,10 @@ define <2 x i64> @ugt_13_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_13_v2i64:
@@ -20914,15 +20469,10 @@ define <2 x i64> @ugt_13_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_13_v2i64:
@@ -20938,15 +20488,10 @@ define <2 x i64> @ugt_13_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_13_v2i64:
@@ -20962,15 +20507,10 @@ define <2 x i64> @ugt_13_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_13_v2i64:
@@ -21060,16 +20600,11 @@ define <2 x i64> @ult_14_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662,2147483662,2147483662]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_14_v2i64:
@@ -21089,16 +20624,11 @@ define <2 x i64> @ult_14_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662,2147483662,2147483662]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_14_v2i64:
@@ -21114,16 +20644,11 @@ define <2 x i64> @ult_14_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662,2147483662,2147483662]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_14_v2i64:
@@ -21139,16 +20664,11 @@ define <2 x i64> @ult_14_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483662,2147483662,2147483662,2147483662]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_14_v2i64:
@@ -21242,15 +20762,10 @@ define <2 x i64> @ugt_14_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_14_v2i64:
@@ -21270,15 +20785,10 @@ define <2 x i64> @ugt_14_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_14_v2i64:
@@ -21294,15 +20804,10 @@ define <2 x i64> @ugt_14_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_14_v2i64:
@@ -21318,15 +20823,10 @@ define <2 x i64> @ugt_14_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_14_v2i64:
@@ -21416,16 +20916,11 @@ define <2 x i64> @ult_15_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663,2147483663,2147483663]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_15_v2i64:
@@ -21445,16 +20940,11 @@ define <2 x i64> @ult_15_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663,2147483663,2147483663]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_15_v2i64:
@@ -21470,16 +20960,11 @@ define <2 x i64> @ult_15_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663,2147483663,2147483663]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_15_v2i64:
@@ -21495,16 +20980,11 @@ define <2 x i64> @ult_15_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483663,2147483663,2147483663,2147483663]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_15_v2i64:
@@ -21598,15 +21078,10 @@ define <2 x i64> @ugt_15_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_15_v2i64:
@@ -21626,15 +21101,10 @@ define <2 x i64> @ugt_15_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_15_v2i64:
@@ -21650,15 +21120,10 @@ define <2 x i64> @ugt_15_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_15_v2i64:
@@ -21674,15 +21139,10 @@ define <2 x i64> @ugt_15_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_15_v2i64:
@@ -21772,16 +21232,11 @@ define <2 x i64> @ult_16_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664,2147483664,2147483664]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_16_v2i64:
@@ -21801,16 +21256,11 @@ define <2 x i64> @ult_16_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664,2147483664,2147483664]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_16_v2i64:
@@ -21826,16 +21276,11 @@ define <2 x i64> @ult_16_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664,2147483664,2147483664]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_16_v2i64:
@@ -21851,16 +21296,11 @@ define <2 x i64> @ult_16_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483664,2147483664,2147483664,2147483664]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_16_v2i64:
@@ -21954,15 +21394,10 @@ define <2 x i64> @ugt_16_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_16_v2i64:
@@ -21982,15 +21417,10 @@ define <2 x i64> @ugt_16_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_16_v2i64:
@@ -22006,15 +21436,10 @@ define <2 x i64> @ugt_16_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_16_v2i64:
@@ -22030,15 +21455,10 @@ define <2 x i64> @ugt_16_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_16_v2i64:
@@ -22128,16 +21548,11 @@ define <2 x i64> @ult_17_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665,2147483665,2147483665]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_17_v2i64:
@@ -22157,16 +21572,11 @@ define <2 x i64> @ult_17_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665,2147483665,2147483665]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_17_v2i64:
@@ -22182,16 +21592,11 @@ define <2 x i64> @ult_17_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665,2147483665,2147483665]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_17_v2i64:
@@ -22207,16 +21612,11 @@ define <2 x i64> @ult_17_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483665,2147483665,2147483665,2147483665]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_17_v2i64:
@@ -22310,15 +21710,10 @@ define <2 x i64> @ugt_17_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_17_v2i64:
@@ -22338,15 +21733,10 @@ define <2 x i64> @ugt_17_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_17_v2i64:
@@ -22362,15 +21752,10 @@ define <2 x i64> @ugt_17_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_17_v2i64:
@@ -22386,15 +21771,10 @@ define <2 x i64> @ugt_17_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_17_v2i64:
@@ -22484,16 +21864,11 @@ define <2 x i64> @ult_18_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666,2147483666,2147483666]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_18_v2i64:
@@ -22513,16 +21888,11 @@ define <2 x i64> @ult_18_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666,2147483666,2147483666]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_18_v2i64:
@@ -22538,16 +21908,11 @@ define <2 x i64> @ult_18_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666,2147483666,2147483666]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_18_v2i64:
@@ -22563,16 +21928,11 @@ define <2 x i64> @ult_18_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483666,2147483666,2147483666,2147483666]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_18_v2i64:
@@ -22666,15 +22026,10 @@ define <2 x i64> @ugt_18_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_18_v2i64:
@@ -22694,15 +22049,10 @@ define <2 x i64> @ugt_18_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_18_v2i64:
@@ -22718,15 +22068,10 @@ define <2 x i64> @ugt_18_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_18_v2i64:
@@ -22742,15 +22087,10 @@ define <2 x i64> @ugt_18_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_18_v2i64:
@@ -22840,16 +22180,11 @@ define <2 x i64> @ult_19_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667,2147483667,2147483667]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_19_v2i64:
@@ -22869,16 +22204,11 @@ define <2 x i64> @ult_19_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667,2147483667,2147483667]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_19_v2i64:
@@ -22894,16 +22224,11 @@ define <2 x i64> @ult_19_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667,2147483667,2147483667]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_19_v2i64:
@@ -22919,16 +22244,11 @@ define <2 x i64> @ult_19_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483667,2147483667,2147483667,2147483667]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_19_v2i64:
@@ -23022,15 +22342,10 @@ define <2 x i64> @ugt_19_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_19_v2i64:
@@ -23050,15 +22365,10 @@ define <2 x i64> @ugt_19_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_19_v2i64:
@@ -23074,15 +22384,10 @@ define <2 x i64> @ugt_19_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_19_v2i64:
@@ -23098,15 +22403,10 @@ define <2 x i64> @ugt_19_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_19_v2i64:
@@ -23196,16 +22496,11 @@ define <2 x i64> @ult_20_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668,2147483668,2147483668]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_20_v2i64:
@@ -23225,16 +22520,11 @@ define <2 x i64> @ult_20_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668,2147483668,2147483668]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_20_v2i64:
@@ -23250,16 +22540,11 @@ define <2 x i64> @ult_20_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668,2147483668,2147483668]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_20_v2i64:
@@ -23275,16 +22560,11 @@ define <2 x i64> @ult_20_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483668,2147483668,2147483668,2147483668]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_20_v2i64:
@@ -23378,15 +22658,10 @@ define <2 x i64> @ugt_20_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_20_v2i64:
@@ -23406,15 +22681,10 @@ define <2 x i64> @ugt_20_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_20_v2i64:
@@ -23430,15 +22700,10 @@ define <2 x i64> @ugt_20_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_20_v2i64:
@@ -23454,15 +22719,10 @@ define <2 x i64> @ugt_20_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_20_v2i64:
@@ -23552,16 +22812,11 @@ define <2 x i64> @ult_21_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669,2147483669,2147483669]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_21_v2i64:
@@ -23581,16 +22836,11 @@ define <2 x i64> @ult_21_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669,2147483669,2147483669]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_21_v2i64:
@@ -23606,16 +22856,11 @@ define <2 x i64> @ult_21_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669,2147483669,2147483669]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_21_v2i64:
@@ -23631,16 +22876,11 @@ define <2 x i64> @ult_21_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483669,2147483669,2147483669,2147483669]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_21_v2i64:
@@ -23734,15 +22974,10 @@ define <2 x i64> @ugt_21_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_21_v2i64:
@@ -23762,15 +22997,10 @@ define <2 x i64> @ugt_21_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_21_v2i64:
@@ -23786,15 +23016,10 @@ define <2 x i64> @ugt_21_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_21_v2i64:
@@ -23810,15 +23035,10 @@ define <2 x i64> @ugt_21_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_21_v2i64:
@@ -23908,16 +23128,11 @@ define <2 x i64> @ult_22_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670,2147483670,2147483670]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_22_v2i64:
@@ -23937,16 +23152,11 @@ define <2 x i64> @ult_22_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670,2147483670,2147483670]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_22_v2i64:
@@ -23962,16 +23172,11 @@ define <2 x i64> @ult_22_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670,2147483670,2147483670]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_22_v2i64:
@@ -23987,16 +23192,11 @@ define <2 x i64> @ult_22_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483670,2147483670,2147483670,2147483670]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_22_v2i64:
@@ -24090,15 +23290,10 @@ define <2 x i64> @ugt_22_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_22_v2i64:
@@ -24118,15 +23313,10 @@ define <2 x i64> @ugt_22_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_22_v2i64:
@@ -24142,15 +23332,10 @@ define <2 x i64> @ugt_22_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_22_v2i64:
@@ -24166,15 +23351,10 @@ define <2 x i64> @ugt_22_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_22_v2i64:
@@ -24264,16 +23444,11 @@ define <2 x i64> @ult_23_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671,2147483671,2147483671]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_23_v2i64:
@@ -24293,16 +23468,11 @@ define <2 x i64> @ult_23_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671,2147483671,2147483671]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_23_v2i64:
@@ -24318,16 +23488,11 @@ define <2 x i64> @ult_23_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671,2147483671,2147483671]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_23_v2i64:
@@ -24343,16 +23508,11 @@ define <2 x i64> @ult_23_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483671,2147483671,2147483671,2147483671]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_23_v2i64:
@@ -24446,15 +23606,10 @@ define <2 x i64> @ugt_23_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_23_v2i64:
@@ -24474,15 +23629,10 @@ define <2 x i64> @ugt_23_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_23_v2i64:
@@ -24498,15 +23648,10 @@ define <2 x i64> @ugt_23_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_23_v2i64:
@@ -24522,15 +23667,10 @@ define <2 x i64> @ugt_23_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_23_v2i64:
@@ -24620,16 +23760,11 @@ define <2 x i64> @ult_24_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672,2147483672,2147483672]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_24_v2i64:
@@ -24649,16 +23784,11 @@ define <2 x i64> @ult_24_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672,2147483672,2147483672]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_24_v2i64:
@@ -24674,16 +23804,11 @@ define <2 x i64> @ult_24_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672,2147483672,2147483672]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_24_v2i64:
@@ -24699,16 +23824,11 @@ define <2 x i64> @ult_24_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483672,2147483672,2147483672,2147483672]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_24_v2i64:
@@ -24802,15 +23922,10 @@ define <2 x i64> @ugt_24_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_24_v2i64:
@@ -24830,15 +23945,10 @@ define <2 x i64> @ugt_24_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_24_v2i64:
@@ -24854,15 +23964,10 @@ define <2 x i64> @ugt_24_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_24_v2i64:
@@ -24878,15 +23983,10 @@ define <2 x i64> @ugt_24_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_24_v2i64:
@@ -24976,16 +24076,11 @@ define <2 x i64> @ult_25_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673,2147483673,2147483673]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_25_v2i64:
@@ -25005,16 +24100,11 @@ define <2 x i64> @ult_25_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673,2147483673,2147483673]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_25_v2i64:
@@ -25030,16 +24120,11 @@ define <2 x i64> @ult_25_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673,2147483673,2147483673]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_25_v2i64:
@@ -25055,16 +24140,11 @@ define <2 x i64> @ult_25_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483673,2147483673,2147483673,2147483673]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_25_v2i64:
@@ -25158,15 +24238,10 @@ define <2 x i64> @ugt_25_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_25_v2i64:
@@ -25186,15 +24261,10 @@ define <2 x i64> @ugt_25_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_25_v2i64:
@@ -25210,15 +24280,10 @@ define <2 x i64> @ugt_25_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_25_v2i64:
@@ -25234,15 +24299,10 @@ define <2 x i64> @ugt_25_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_25_v2i64:
@@ -25332,16 +24392,11 @@ define <2 x i64> @ult_26_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674,2147483674,2147483674]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_26_v2i64:
@@ -25361,16 +24416,11 @@ define <2 x i64> @ult_26_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674,2147483674,2147483674]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_26_v2i64:
@@ -25386,16 +24436,11 @@ define <2 x i64> @ult_26_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674,2147483674,2147483674]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_26_v2i64:
@@ -25411,16 +24456,11 @@ define <2 x i64> @ult_26_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483674,2147483674,2147483674,2147483674]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_26_v2i64:
@@ -25514,15 +24554,10 @@ define <2 x i64> @ugt_26_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_26_v2i64:
@@ -25542,15 +24577,10 @@ define <2 x i64> @ugt_26_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_26_v2i64:
@@ -25566,15 +24596,10 @@ define <2 x i64> @ugt_26_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_26_v2i64:
@@ -25590,15 +24615,10 @@ define <2 x i64> @ugt_26_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_26_v2i64:
@@ -25688,16 +24708,11 @@ define <2 x i64> @ult_27_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675,2147483675,2147483675]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_27_v2i64:
@@ -25717,16 +24732,11 @@ define <2 x i64> @ult_27_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675,2147483675,2147483675]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_27_v2i64:
@@ -25742,16 +24752,11 @@ define <2 x i64> @ult_27_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675,2147483675,2147483675]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_27_v2i64:
@@ -25767,16 +24772,11 @@ define <2 x i64> @ult_27_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483675,2147483675,2147483675,2147483675]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_27_v2i64:
@@ -25870,15 +24870,10 @@ define <2 x i64> @ugt_27_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_27_v2i64:
@@ -25898,15 +24893,10 @@ define <2 x i64> @ugt_27_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_27_v2i64:
@@ -25922,15 +24912,10 @@ define <2 x i64> @ugt_27_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_27_v2i64:
@@ -25946,15 +24931,10 @@ define <2 x i64> @ugt_27_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_27_v2i64:
@@ -26044,16 +25024,11 @@ define <2 x i64> @ult_28_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676,2147483676,2147483676]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_28_v2i64:
@@ -26073,16 +25048,11 @@ define <2 x i64> @ult_28_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676,2147483676,2147483676]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_28_v2i64:
@@ -26098,16 +25068,11 @@ define <2 x i64> @ult_28_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676,2147483676,2147483676]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_28_v2i64:
@@ -26123,16 +25088,11 @@ define <2 x i64> @ult_28_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483676,2147483676,2147483676,2147483676]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_28_v2i64:
@@ -26226,15 +25186,10 @@ define <2 x i64> @ugt_28_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_28_v2i64:
@@ -26254,15 +25209,10 @@ define <2 x i64> @ugt_28_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_28_v2i64:
@@ -26278,15 +25228,10 @@ define <2 x i64> @ugt_28_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_28_v2i64:
@@ -26302,15 +25247,10 @@ define <2 x i64> @ugt_28_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_28_v2i64:
@@ -26400,16 +25340,11 @@ define <2 x i64> @ult_29_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677,2147483677,2147483677]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_29_v2i64:
@@ -26429,16 +25364,11 @@ define <2 x i64> @ult_29_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677,2147483677,2147483677]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_29_v2i64:
@@ -26454,16 +25384,11 @@ define <2 x i64> @ult_29_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677,2147483677,2147483677]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_29_v2i64:
@@ -26479,16 +25404,11 @@ define <2 x i64> @ult_29_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483677,2147483677,2147483677,2147483677]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_29_v2i64:
@@ -26582,15 +25502,10 @@ define <2 x i64> @ugt_29_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_29_v2i64:
@@ -26610,15 +25525,10 @@ define <2 x i64> @ugt_29_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_29_v2i64:
@@ -26634,15 +25544,10 @@ define <2 x i64> @ugt_29_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_29_v2i64:
@@ -26658,15 +25563,10 @@ define <2 x i64> @ugt_29_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_29_v2i64:
@@ -26756,16 +25656,11 @@ define <2 x i64> @ult_30_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678,2147483678,2147483678]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_30_v2i64:
@@ -26785,16 +25680,11 @@ define <2 x i64> @ult_30_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678,2147483678,2147483678]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_30_v2i64:
@@ -26810,16 +25700,11 @@ define <2 x i64> @ult_30_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678,2147483678,2147483678]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_30_v2i64:
@@ -26835,16 +25720,11 @@ define <2 x i64> @ult_30_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483678,2147483678,2147483678,2147483678]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_30_v2i64:
@@ -26938,15 +25818,10 @@ define <2 x i64> @ugt_30_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_30_v2i64:
@@ -26966,15 +25841,10 @@ define <2 x i64> @ugt_30_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_30_v2i64:
@@ -26990,15 +25860,10 @@ define <2 x i64> @ugt_30_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_30_v2i64:
@@ -27014,15 +25879,10 @@ define <2 x i64> @ugt_30_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_30_v2i64:
@@ -27112,16 +25972,11 @@ define <2 x i64> @ult_31_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679,2147483679,2147483679]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_31_v2i64:
@@ -27141,16 +25996,11 @@ define <2 x i64> @ult_31_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679,2147483679,2147483679]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_31_v2i64:
@@ -27166,16 +26016,11 @@ define <2 x i64> @ult_31_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679,2147483679,2147483679]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_31_v2i64:
@@ -27191,16 +26036,11 @@ define <2 x i64> @ult_31_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483679,2147483679,2147483679,2147483679]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_31_v2i64:
@@ -27294,15 +26134,10 @@ define <2 x i64> @ugt_31_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_31_v2i64:
@@ -27322,15 +26157,10 @@ define <2 x i64> @ugt_31_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_31_v2i64:
@@ -27346,15 +26176,10 @@ define <2 x i64> @ugt_31_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_31_v2i64:
@@ -27370,15 +26195,10 @@ define <2 x i64> @ugt_31_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_31_v2i64:
@@ -27468,16 +26288,11 @@ define <2 x i64> @ult_32_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680,2147483680,2147483680]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_32_v2i64:
@@ -27497,16 +26312,11 @@ define <2 x i64> @ult_32_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680,2147483680,2147483680]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_32_v2i64:
@@ -27522,16 +26332,11 @@ define <2 x i64> @ult_32_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680,2147483680,2147483680]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_32_v2i64:
@@ -27547,16 +26352,11 @@ define <2 x i64> @ult_32_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483680,2147483680,2147483680,2147483680]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_32_v2i64:
@@ -27650,15 +26450,10 @@ define <2 x i64> @ugt_32_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_32_v2i64:
@@ -27678,15 +26473,10 @@ define <2 x i64> @ugt_32_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_32_v2i64:
@@ -27702,15 +26492,10 @@ define <2 x i64> @ugt_32_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_32_v2i64:
@@ -27726,15 +26511,10 @@ define <2 x i64> @ugt_32_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_32_v2i64:
@@ -27824,16 +26604,11 @@ define <2 x i64> @ult_33_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681,2147483681,2147483681]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_33_v2i64:
@@ -27853,16 +26628,11 @@ define <2 x i64> @ult_33_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681,2147483681,2147483681]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_33_v2i64:
@@ -27878,16 +26648,11 @@ define <2 x i64> @ult_33_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681,2147483681,2147483681]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_33_v2i64:
@@ -27903,16 +26668,11 @@ define <2 x i64> @ult_33_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483681,2147483681,2147483681,2147483681]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_33_v2i64:
@@ -28006,15 +26766,10 @@ define <2 x i64> @ugt_33_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_33_v2i64:
@@ -28034,15 +26789,10 @@ define <2 x i64> @ugt_33_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_33_v2i64:
@@ -28058,15 +26808,10 @@ define <2 x i64> @ugt_33_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_33_v2i64:
@@ -28082,15 +26827,10 @@ define <2 x i64> @ugt_33_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_33_v2i64:
@@ -28180,16 +26920,11 @@ define <2 x i64> @ult_34_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682,2147483682,2147483682]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_34_v2i64:
@@ -28209,16 +26944,11 @@ define <2 x i64> @ult_34_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682,2147483682,2147483682]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_34_v2i64:
@@ -28234,16 +26964,11 @@ define <2 x i64> @ult_34_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682,2147483682,2147483682]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_34_v2i64:
@@ -28259,16 +26984,11 @@ define <2 x i64> @ult_34_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483682,2147483682,2147483682,2147483682]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_34_v2i64:
@@ -28362,15 +27082,10 @@ define <2 x i64> @ugt_34_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_34_v2i64:
@@ -28390,15 +27105,10 @@ define <2 x i64> @ugt_34_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_34_v2i64:
@@ -28414,15 +27124,10 @@ define <2 x i64> @ugt_34_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_34_v2i64:
@@ -28438,15 +27143,10 @@ define <2 x i64> @ugt_34_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_34_v2i64:
@@ -28536,16 +27236,11 @@ define <2 x i64> @ult_35_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683,2147483683,2147483683]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_35_v2i64:
@@ -28565,16 +27260,11 @@ define <2 x i64> @ult_35_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683,2147483683,2147483683]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_35_v2i64:
@@ -28590,16 +27280,11 @@ define <2 x i64> @ult_35_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683,2147483683,2147483683]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_35_v2i64:
@@ -28615,16 +27300,11 @@ define <2 x i64> @ult_35_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483683,2147483683,2147483683,2147483683]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_35_v2i64:
@@ -28718,15 +27398,10 @@ define <2 x i64> @ugt_35_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_35_v2i64:
@@ -28746,15 +27421,10 @@ define <2 x i64> @ugt_35_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_35_v2i64:
@@ -28770,15 +27440,10 @@ define <2 x i64> @ugt_35_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_35_v2i64:
@@ -28794,15 +27459,10 @@ define <2 x i64> @ugt_35_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_35_v2i64:
@@ -28892,16 +27552,11 @@ define <2 x i64> @ult_36_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684,2147483684,2147483684]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_36_v2i64:
@@ -28921,16 +27576,11 @@ define <2 x i64> @ult_36_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684,2147483684,2147483684]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_36_v2i64:
@@ -28946,16 +27596,11 @@ define <2 x i64> @ult_36_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684,2147483684,2147483684]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_36_v2i64:
@@ -28971,16 +27616,11 @@ define <2 x i64> @ult_36_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483684,2147483684,2147483684,2147483684]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_36_v2i64:
@@ -29074,15 +27714,10 @@ define <2 x i64> @ugt_36_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_36_v2i64:
@@ -29102,15 +27737,10 @@ define <2 x i64> @ugt_36_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_36_v2i64:
@@ -29126,15 +27756,10 @@ define <2 x i64> @ugt_36_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_36_v2i64:
@@ -29150,15 +27775,10 @@ define <2 x i64> @ugt_36_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_36_v2i64:
@@ -29248,16 +27868,11 @@ define <2 x i64> @ult_37_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685,2147483685,2147483685]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_37_v2i64:
@@ -29277,16 +27892,11 @@ define <2 x i64> @ult_37_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685,2147483685,2147483685]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_37_v2i64:
@@ -29302,16 +27912,11 @@ define <2 x i64> @ult_37_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685,2147483685,2147483685]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_37_v2i64:
@@ -29327,16 +27932,11 @@ define <2 x i64> @ult_37_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483685,2147483685,2147483685,2147483685]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_37_v2i64:
@@ -29430,15 +28030,10 @@ define <2 x i64> @ugt_37_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_37_v2i64:
@@ -29458,15 +28053,10 @@ define <2 x i64> @ugt_37_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_37_v2i64:
@@ -29482,15 +28072,10 @@ define <2 x i64> @ugt_37_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_37_v2i64:
@@ -29506,15 +28091,10 @@ define <2 x i64> @ugt_37_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_37_v2i64:
@@ -29604,16 +28184,11 @@ define <2 x i64> @ult_38_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686,2147483686,2147483686]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_38_v2i64:
@@ -29633,16 +28208,11 @@ define <2 x i64> @ult_38_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686,2147483686,2147483686]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_38_v2i64:
@@ -29658,16 +28228,11 @@ define <2 x i64> @ult_38_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686,2147483686,2147483686]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_38_v2i64:
@@ -29683,16 +28248,11 @@ define <2 x i64> @ult_38_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483686,2147483686,2147483686,2147483686]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_38_v2i64:
@@ -29786,15 +28346,10 @@ define <2 x i64> @ugt_38_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_38_v2i64:
@@ -29814,15 +28369,10 @@ define <2 x i64> @ugt_38_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_38_v2i64:
@@ -29838,15 +28388,10 @@ define <2 x i64> @ugt_38_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_38_v2i64:
@@ -29862,15 +28407,10 @@ define <2 x i64> @ugt_38_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_38_v2i64:
@@ -29960,16 +28500,11 @@ define <2 x i64> @ult_39_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687,2147483687,2147483687]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_39_v2i64:
@@ -29989,16 +28524,11 @@ define <2 x i64> @ult_39_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687,2147483687,2147483687]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_39_v2i64:
@@ -30014,16 +28544,11 @@ define <2 x i64> @ult_39_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687,2147483687,2147483687]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_39_v2i64:
@@ -30039,16 +28564,11 @@ define <2 x i64> @ult_39_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483687,2147483687,2147483687,2147483687]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_39_v2i64:
@@ -30142,15 +28662,10 @@ define <2 x i64> @ugt_39_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_39_v2i64:
@@ -30170,15 +28685,10 @@ define <2 x i64> @ugt_39_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_39_v2i64:
@@ -30194,15 +28704,10 @@ define <2 x i64> @ugt_39_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_39_v2i64:
@@ -30218,15 +28723,10 @@ define <2 x i64> @ugt_39_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_39_v2i64:
@@ -30316,16 +28816,11 @@ define <2 x i64> @ult_40_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688,2147483688,2147483688]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_40_v2i64:
@@ -30345,16 +28840,11 @@ define <2 x i64> @ult_40_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688,2147483688,2147483688]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_40_v2i64:
@@ -30370,16 +28860,11 @@ define <2 x i64> @ult_40_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688,2147483688,2147483688]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_40_v2i64:
@@ -30395,16 +28880,11 @@ define <2 x i64> @ult_40_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483688,2147483688,2147483688,2147483688]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_40_v2i64:
@@ -30498,15 +28978,10 @@ define <2 x i64> @ugt_40_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_40_v2i64:
@@ -30526,15 +29001,10 @@ define <2 x i64> @ugt_40_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_40_v2i64:
@@ -30550,15 +29020,10 @@ define <2 x i64> @ugt_40_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_40_v2i64:
@@ -30574,15 +29039,10 @@ define <2 x i64> @ugt_40_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_40_v2i64:
@@ -30672,16 +29132,11 @@ define <2 x i64> @ult_41_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689,2147483689,2147483689]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_41_v2i64:
@@ -30701,16 +29156,11 @@ define <2 x i64> @ult_41_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689,2147483689,2147483689]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_41_v2i64:
@@ -30726,16 +29176,11 @@ define <2 x i64> @ult_41_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689,2147483689,2147483689]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_41_v2i64:
@@ -30751,16 +29196,11 @@ define <2 x i64> @ult_41_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483689,2147483689,2147483689,2147483689]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_41_v2i64:
@@ -30854,15 +29294,10 @@ define <2 x i64> @ugt_41_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_41_v2i64:
@@ -30882,15 +29317,10 @@ define <2 x i64> @ugt_41_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_41_v2i64:
@@ -30906,15 +29336,10 @@ define <2 x i64> @ugt_41_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_41_v2i64:
@@ -30930,15 +29355,10 @@ define <2 x i64> @ugt_41_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_41_v2i64:
@@ -31028,16 +29448,11 @@ define <2 x i64> @ult_42_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690,2147483690,2147483690]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_42_v2i64:
@@ -31057,16 +29472,11 @@ define <2 x i64> @ult_42_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690,2147483690,2147483690]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_42_v2i64:
@@ -31082,16 +29492,11 @@ define <2 x i64> @ult_42_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690,2147483690,2147483690]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_42_v2i64:
@@ -31107,16 +29512,11 @@ define <2 x i64> @ult_42_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483690,2147483690,2147483690,2147483690]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_42_v2i64:
@@ -31210,15 +29610,10 @@ define <2 x i64> @ugt_42_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_42_v2i64:
@@ -31238,15 +29633,10 @@ define <2 x i64> @ugt_42_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_42_v2i64:
@@ -31262,15 +29652,10 @@ define <2 x i64> @ugt_42_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_42_v2i64:
@@ -31286,15 +29671,10 @@ define <2 x i64> @ugt_42_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_42_v2i64:
@@ -31384,16 +29764,11 @@ define <2 x i64> @ult_43_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691,2147483691,2147483691]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_43_v2i64:
@@ -31413,16 +29788,11 @@ define <2 x i64> @ult_43_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691,2147483691,2147483691]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_43_v2i64:
@@ -31438,16 +29808,11 @@ define <2 x i64> @ult_43_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691,2147483691,2147483691]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_43_v2i64:
@@ -31463,16 +29828,11 @@ define <2 x i64> @ult_43_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483691,2147483691,2147483691,2147483691]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_43_v2i64:
@@ -31566,15 +29926,10 @@ define <2 x i64> @ugt_43_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_43_v2i64:
@@ -31594,15 +29949,10 @@ define <2 x i64> @ugt_43_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_43_v2i64:
@@ -31618,15 +29968,10 @@ define <2 x i64> @ugt_43_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_43_v2i64:
@@ -31642,15 +29987,10 @@ define <2 x i64> @ugt_43_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_43_v2i64:
@@ -31740,16 +30080,11 @@ define <2 x i64> @ult_44_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692,2147483692,2147483692]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_44_v2i64:
@@ -31769,16 +30104,11 @@ define <2 x i64> @ult_44_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692,2147483692,2147483692]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_44_v2i64:
@@ -31794,16 +30124,11 @@ define <2 x i64> @ult_44_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692,2147483692,2147483692]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_44_v2i64:
@@ -31819,16 +30144,11 @@ define <2 x i64> @ult_44_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483692,2147483692,2147483692,2147483692]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_44_v2i64:
@@ -31921,16 +30241,11 @@ define <2 x i64> @ugt_44_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    psrlw $4, %xmm1
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_44_v2i64:
@@ -31950,15 +30265,10 @@ define <2 x i64> @ugt_44_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_44_v2i64:
@@ -31974,15 +30284,10 @@ define <2 x i64> @ugt_44_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_44_v2i64:
@@ -31998,15 +30303,10 @@ define <2 x i64> @ugt_44_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_44_v2i64:
@@ -32096,16 +30396,11 @@ define <2 x i64> @ult_45_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693,2147483693,2147483693]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_45_v2i64:
@@ -32125,16 +30420,11 @@ define <2 x i64> @ult_45_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693,2147483693,2147483693]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_45_v2i64:
@@ -32150,16 +30440,11 @@ define <2 x i64> @ult_45_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693,2147483693,2147483693]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_45_v2i64:
@@ -32175,16 +30460,11 @@ define <2 x i64> @ult_45_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483693,2147483693,2147483693,2147483693]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_45_v2i64:
@@ -32278,15 +30558,10 @@ define <2 x i64> @ugt_45_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_45_v2i64:
@@ -32306,15 +30581,10 @@ define <2 x i64> @ugt_45_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_45_v2i64:
@@ -32330,15 +30600,10 @@ define <2 x i64> @ugt_45_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_45_v2i64:
@@ -32354,15 +30619,10 @@ define <2 x i64> @ugt_45_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_45_v2i64:
@@ -32452,16 +30712,11 @@ define <2 x i64> @ult_46_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694,2147483694,2147483694]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_46_v2i64:
@@ -32481,16 +30736,11 @@ define <2 x i64> @ult_46_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694,2147483694,2147483694]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_46_v2i64:
@@ -32506,16 +30756,11 @@ define <2 x i64> @ult_46_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694,2147483694,2147483694]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_46_v2i64:
@@ -32531,16 +30776,11 @@ define <2 x i64> @ult_46_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483694,2147483694,2147483694,2147483694]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_46_v2i64:
@@ -32634,15 +30874,10 @@ define <2 x i64> @ugt_46_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_46_v2i64:
@@ -32662,15 +30897,10 @@ define <2 x i64> @ugt_46_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_46_v2i64:
@@ -32686,15 +30916,10 @@ define <2 x i64> @ugt_46_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_46_v2i64:
@@ -32710,15 +30935,10 @@ define <2 x i64> @ugt_46_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_46_v2i64:
@@ -32808,16 +31028,11 @@ define <2 x i64> @ult_47_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695,2147483695,2147483695]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_47_v2i64:
@@ -32837,16 +31052,11 @@ define <2 x i64> @ult_47_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695,2147483695,2147483695]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_47_v2i64:
@@ -32862,16 +31072,11 @@ define <2 x i64> @ult_47_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695,2147483695,2147483695]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_47_v2i64:
@@ -32887,16 +31092,11 @@ define <2 x i64> @ult_47_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483695,2147483695,2147483695,2147483695]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_47_v2i64:
@@ -32990,15 +31190,10 @@ define <2 x i64> @ugt_47_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_47_v2i64:
@@ -33018,15 +31213,10 @@ define <2 x i64> @ugt_47_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_47_v2i64:
@@ -33042,15 +31232,10 @@ define <2 x i64> @ugt_47_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_47_v2i64:
@@ -33066,15 +31251,10 @@ define <2 x i64> @ugt_47_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_47_v2i64:
@@ -33164,16 +31344,11 @@ define <2 x i64> @ult_48_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696,2147483696,2147483696]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_48_v2i64:
@@ -33193,16 +31368,11 @@ define <2 x i64> @ult_48_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696,2147483696,2147483696]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_48_v2i64:
@@ -33218,16 +31388,11 @@ define <2 x i64> @ult_48_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696,2147483696,2147483696]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_48_v2i64:
@@ -33243,16 +31408,11 @@ define <2 x i64> @ult_48_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483696,2147483696,2147483696,2147483696]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_48_v2i64:
@@ -33346,15 +31506,10 @@ define <2 x i64> @ugt_48_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_48_v2i64:
@@ -33374,15 +31529,10 @@ define <2 x i64> @ugt_48_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_48_v2i64:
@@ -33398,15 +31548,10 @@ define <2 x i64> @ugt_48_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_48_v2i64:
@@ -33422,15 +31567,10 @@ define <2 x i64> @ugt_48_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_48_v2i64:
@@ -33520,16 +31660,11 @@ define <2 x i64> @ult_49_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697,2147483697,2147483697]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_49_v2i64:
@@ -33549,16 +31684,11 @@ define <2 x i64> @ult_49_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697,2147483697,2147483697]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_49_v2i64:
@@ -33574,16 +31704,11 @@ define <2 x i64> @ult_49_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697,2147483697,2147483697]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_49_v2i64:
@@ -33599,16 +31724,11 @@ define <2 x i64> @ult_49_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483697,2147483697,2147483697,2147483697]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_49_v2i64:
@@ -33702,15 +31822,10 @@ define <2 x i64> @ugt_49_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_49_v2i64:
@@ -33730,15 +31845,10 @@ define <2 x i64> @ugt_49_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_49_v2i64:
@@ -33754,15 +31864,10 @@ define <2 x i64> @ugt_49_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_49_v2i64:
@@ -33778,15 +31883,10 @@ define <2 x i64> @ugt_49_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_49_v2i64:
@@ -33876,16 +31976,11 @@ define <2 x i64> @ult_50_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698,2147483698,2147483698]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_50_v2i64:
@@ -33905,16 +32000,11 @@ define <2 x i64> @ult_50_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698,2147483698,2147483698]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_50_v2i64:
@@ -33930,16 +32020,11 @@ define <2 x i64> @ult_50_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698,2147483698,2147483698]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_50_v2i64:
@@ -33955,16 +32040,11 @@ define <2 x i64> @ult_50_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483698,2147483698,2147483698,2147483698]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_50_v2i64:
@@ -34058,15 +32138,10 @@ define <2 x i64> @ugt_50_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_50_v2i64:
@@ -34086,15 +32161,10 @@ define <2 x i64> @ugt_50_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_50_v2i64:
@@ -34110,15 +32180,10 @@ define <2 x i64> @ugt_50_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_50_v2i64:
@@ -34134,15 +32199,10 @@ define <2 x i64> @ugt_50_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_50_v2i64:
@@ -34232,16 +32292,11 @@ define <2 x i64> @ult_51_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699,2147483699,2147483699]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_51_v2i64:
@@ -34261,16 +32316,11 @@ define <2 x i64> @ult_51_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699,2147483699,2147483699]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_51_v2i64:
@@ -34286,16 +32336,11 @@ define <2 x i64> @ult_51_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699,2147483699,2147483699]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_51_v2i64:
@@ -34311,16 +32356,11 @@ define <2 x i64> @ult_51_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483699,2147483699,2147483699,2147483699]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_51_v2i64:
@@ -34414,15 +32454,10 @@ define <2 x i64> @ugt_51_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_51_v2i64:
@@ -34442,15 +32477,10 @@ define <2 x i64> @ugt_51_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_51_v2i64:
@@ -34466,15 +32496,10 @@ define <2 x i64> @ugt_51_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_51_v2i64:
@@ -34490,15 +32515,10 @@ define <2 x i64> @ugt_51_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_51_v2i64:
@@ -34588,16 +32608,11 @@ define <2 x i64> @ult_52_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700,2147483700,2147483700]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_52_v2i64:
@@ -34617,16 +32632,11 @@ define <2 x i64> @ult_52_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700,2147483700,2147483700]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_52_v2i64:
@@ -34642,16 +32652,11 @@ define <2 x i64> @ult_52_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700,2147483700,2147483700]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_52_v2i64:
@@ -34667,16 +32672,11 @@ define <2 x i64> @ult_52_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483700,2147483700,2147483700,2147483700]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_52_v2i64:
@@ -34770,15 +32770,10 @@ define <2 x i64> @ugt_52_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_52_v2i64:
@@ -34798,15 +32793,10 @@ define <2 x i64> @ugt_52_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_52_v2i64:
@@ -34822,15 +32812,10 @@ define <2 x i64> @ugt_52_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_52_v2i64:
@@ -34846,15 +32831,10 @@ define <2 x i64> @ugt_52_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_52_v2i64:
@@ -34944,16 +32924,11 @@ define <2 x i64> @ult_53_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701,2147483701,2147483701]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_53_v2i64:
@@ -34973,16 +32948,11 @@ define <2 x i64> @ult_53_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701,2147483701,2147483701]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_53_v2i64:
@@ -34998,16 +32968,11 @@ define <2 x i64> @ult_53_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701,2147483701,2147483701]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_53_v2i64:
@@ -35023,16 +32988,11 @@ define <2 x i64> @ult_53_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483701,2147483701,2147483701,2147483701]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_53_v2i64:
@@ -35126,15 +33086,10 @@ define <2 x i64> @ugt_53_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_53_v2i64:
@@ -35154,15 +33109,10 @@ define <2 x i64> @ugt_53_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_53_v2i64:
@@ -35178,15 +33128,10 @@ define <2 x i64> @ugt_53_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_53_v2i64:
@@ -35202,15 +33147,10 @@ define <2 x i64> @ugt_53_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_53_v2i64:
@@ -35300,16 +33240,11 @@ define <2 x i64> @ult_54_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702,2147483702,2147483702]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_54_v2i64:
@@ -35329,16 +33264,11 @@ define <2 x i64> @ult_54_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702,2147483702,2147483702]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_54_v2i64:
@@ -35354,16 +33284,11 @@ define <2 x i64> @ult_54_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702,2147483702,2147483702]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_54_v2i64:
@@ -35379,16 +33304,11 @@ define <2 x i64> @ult_54_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483702,2147483702,2147483702,2147483702]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_54_v2i64:
@@ -35482,15 +33402,10 @@ define <2 x i64> @ugt_54_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_54_v2i64:
@@ -35510,15 +33425,10 @@ define <2 x i64> @ugt_54_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_54_v2i64:
@@ -35534,15 +33444,10 @@ define <2 x i64> @ugt_54_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_54_v2i64:
@@ -35558,15 +33463,10 @@ define <2 x i64> @ugt_54_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_54_v2i64:
@@ -35656,16 +33556,11 @@ define <2 x i64> @ult_55_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703,2147483703,2147483703]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_55_v2i64:
@@ -35685,16 +33580,11 @@ define <2 x i64> @ult_55_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703,2147483703,2147483703]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_55_v2i64:
@@ -35710,16 +33600,11 @@ define <2 x i64> @ult_55_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703,2147483703,2147483703]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_55_v2i64:
@@ -35735,16 +33620,11 @@ define <2 x i64> @ult_55_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483703,2147483703,2147483703,2147483703]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_55_v2i64:
@@ -35838,15 +33718,10 @@ define <2 x i64> @ugt_55_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_55_v2i64:
@@ -35866,15 +33741,10 @@ define <2 x i64> @ugt_55_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_55_v2i64:
@@ -35890,15 +33760,10 @@ define <2 x i64> @ugt_55_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_55_v2i64:
@@ -35914,15 +33779,10 @@ define <2 x i64> @ugt_55_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_55_v2i64:
@@ -36012,16 +33872,11 @@ define <2 x i64> @ult_56_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704,2147483704,2147483704]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_56_v2i64:
@@ -36041,16 +33896,11 @@ define <2 x i64> @ult_56_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704,2147483704,2147483704]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_56_v2i64:
@@ -36066,16 +33916,11 @@ define <2 x i64> @ult_56_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704,2147483704,2147483704]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_56_v2i64:
@@ -36091,16 +33936,11 @@ define <2 x i64> @ult_56_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483704,2147483704,2147483704,2147483704]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_56_v2i64:
@@ -36194,15 +34034,10 @@ define <2 x i64> @ugt_56_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_56_v2i64:
@@ -36222,15 +34057,10 @@ define <2 x i64> @ugt_56_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_56_v2i64:
@@ -36246,15 +34076,10 @@ define <2 x i64> @ugt_56_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_56_v2i64:
@@ -36270,15 +34095,10 @@ define <2 x i64> @ugt_56_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_56_v2i64:
@@ -36368,16 +34188,11 @@ define <2 x i64> @ult_57_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705,2147483705,2147483705]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_57_v2i64:
@@ -36397,16 +34212,11 @@ define <2 x i64> @ult_57_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705,2147483705,2147483705]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_57_v2i64:
@@ -36422,16 +34232,11 @@ define <2 x i64> @ult_57_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705,2147483705,2147483705]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_57_v2i64:
@@ -36447,16 +34252,11 @@ define <2 x i64> @ult_57_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483705,2147483705,2147483705,2147483705]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_57_v2i64:
@@ -36550,15 +34350,10 @@ define <2 x i64> @ugt_57_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_57_v2i64:
@@ -36578,15 +34373,10 @@ define <2 x i64> @ugt_57_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_57_v2i64:
@@ -36602,15 +34392,10 @@ define <2 x i64> @ugt_57_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_57_v2i64:
@@ -36626,15 +34411,10 @@ define <2 x i64> @ugt_57_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_57_v2i64:
@@ -36724,16 +34504,11 @@ define <2 x i64> @ult_58_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706,2147483706,2147483706]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_58_v2i64:
@@ -36753,16 +34528,11 @@ define <2 x i64> @ult_58_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706,2147483706,2147483706]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_58_v2i64:
@@ -36778,16 +34548,11 @@ define <2 x i64> @ult_58_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706,2147483706,2147483706]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_58_v2i64:
@@ -36803,16 +34568,11 @@ define <2 x i64> @ult_58_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483706,2147483706,2147483706,2147483706]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_58_v2i64:
@@ -36906,15 +34666,10 @@ define <2 x i64> @ugt_58_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_58_v2i64:
@@ -36934,15 +34689,10 @@ define <2 x i64> @ugt_58_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_58_v2i64:
@@ -36958,15 +34708,10 @@ define <2 x i64> @ugt_58_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_58_v2i64:
@@ -36982,15 +34727,10 @@ define <2 x i64> @ugt_58_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_58_v2i64:
@@ -37080,16 +34820,11 @@ define <2 x i64> @ult_59_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707,2147483707,2147483707]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_59_v2i64:
@@ -37109,16 +34844,11 @@ define <2 x i64> @ult_59_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707,2147483707,2147483707]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_59_v2i64:
@@ -37134,16 +34864,11 @@ define <2 x i64> @ult_59_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707,2147483707,2147483707]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_59_v2i64:
@@ -37159,16 +34884,11 @@ define <2 x i64> @ult_59_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483707,2147483707,2147483707,2147483707]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_59_v2i64:
@@ -37262,15 +34982,10 @@ define <2 x i64> @ugt_59_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_59_v2i64:
@@ -37290,15 +35005,10 @@ define <2 x i64> @ugt_59_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_59_v2i64:
@@ -37314,15 +35024,10 @@ define <2 x i64> @ugt_59_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_59_v2i64:
@@ -37338,15 +35043,10 @@ define <2 x i64> @ugt_59_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_59_v2i64:
@@ -37436,16 +35136,11 @@ define <2 x i64> @ult_60_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708,2147483708,2147483708]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_60_v2i64:
@@ -37465,16 +35160,11 @@ define <2 x i64> @ult_60_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708,2147483708,2147483708]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_60_v2i64:
@@ -37490,16 +35180,11 @@ define <2 x i64> @ult_60_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708,2147483708,2147483708]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_60_v2i64:
@@ -37515,16 +35200,11 @@ define <2 x i64> @ult_60_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483708,2147483708,2147483708,2147483708]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_60_v2i64:
@@ -37618,15 +35298,10 @@ define <2 x i64> @ugt_60_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_60_v2i64:
@@ -37646,15 +35321,10 @@ define <2 x i64> @ugt_60_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_60_v2i64:
@@ -37670,15 +35340,10 @@ define <2 x i64> @ugt_60_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_60_v2i64:
@@ -37694,15 +35359,10 @@ define <2 x i64> @ugt_60_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_60_v2i64:
@@ -37792,16 +35452,11 @@ define <2 x i64> @ult_61_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709,2147483709,2147483709]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_61_v2i64:
@@ -37821,16 +35476,11 @@ define <2 x i64> @ult_61_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709,2147483709,2147483709]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_61_v2i64:
@@ -37846,16 +35496,11 @@ define <2 x i64> @ult_61_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709,2147483709,2147483709]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_61_v2i64:
@@ -37871,16 +35516,11 @@ define <2 x i64> @ult_61_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483709,2147483709,2147483709,2147483709]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_61_v2i64:
@@ -37974,15 +35614,10 @@ define <2 x i64> @ugt_61_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_61_v2i64:
@@ -38002,15 +35637,10 @@ define <2 x i64> @ugt_61_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_61_v2i64:
@@ -38026,15 +35656,10 @@ define <2 x i64> @ugt_61_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_61_v2i64:
@@ -38050,15 +35675,10 @@ define <2 x i64> @ugt_61_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_61_v2i64:
@@ -38148,16 +35768,11 @@ define <2 x i64> @ult_62_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710,2147483710,2147483710]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_62_v2i64:
@@ -38177,16 +35792,11 @@ define <2 x i64> @ult_62_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710,2147483710,2147483710]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_62_v2i64:
@@ -38202,16 +35812,11 @@ define <2 x i64> @ult_62_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710,2147483710,2147483710]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_62_v2i64:
@@ -38227,16 +35832,11 @@ define <2 x i64> @ult_62_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483710,2147483710,2147483710,2147483710]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_62_v2i64:
@@ -38330,15 +35930,10 @@ define <2 x i64> @ugt_62_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
-; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ugt_62_v2i64:
@@ -38358,15 +35953,10 @@ define <2 x i64> @ugt_62_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
-; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm3
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
-; SSE3-NEXT:    por %xmm3, %xmm0
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ugt_62_v2i64:
@@ -38382,15 +35972,10 @@ define <2 x i64> @ugt_62_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSSE3-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ugt_62_v2i64:
@@ -38406,15 +35991,10 @@ define <2 x i64> @ugt_62_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ugt_62_v2i64:
@@ -38504,16 +36084,11 @@ define <2 x i64> @ult_63_v2i64(<2 x i64> %0) {
 ; SSE2-NEXT:    paddb %xmm0, %xmm1
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    psadbw %xmm0, %xmm1
+; SSE2-NEXT:    psadbw %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711,2147483711,2147483711]
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE3-LABEL: ult_63_v2i64:
@@ -38533,16 +36108,11 @@ define <2 x i64> @ult_63_v2i64(<2 x i64> %0) {
 ; SSE3-NEXT:    paddb %xmm0, %xmm1
 ; SSE3-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE3-NEXT:    pxor %xmm0, %xmm0
-; SSE3-NEXT:    psadbw %xmm0, %xmm1
+; SSE3-NEXT:    psadbw %xmm1, %xmm0
+; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
 ; SSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE3-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711]
+; SSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711,2147483711,2147483711]
 ; SSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE3-NEXT:    pand %xmm2, %xmm1
-; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE3-NEXT:    por %xmm1, %xmm0
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: ult_63_v2i64:
@@ -38558,16 +36128,11 @@ define <2 x i64> @ult_63_v2i64(<2 x i64> %0) {
 ; SSSE3-NEXT:    pshufb %xmm0, %xmm3
 ; SSSE3-NEXT:    paddb %xmm4, %xmm3
 ; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    psadbw %xmm0, %xmm3
-; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psadbw %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711,2147483711,2147483711]
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ult_63_v2i64:
@@ -38583,16 +36148,11 @@ define <2 x i64> @ult_63_v2i64(<2 x i64> %0) {
 ; SSE41-NEXT:    pshufb %xmm0, %xmm3
 ; SSE41-NEXT:    paddb %xmm4, %xmm3
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psadbw %xmm0, %xmm3
-; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711]
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    psadbw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483711,2147483711,2147483711,2147483711]
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ult_63_v2i64:
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index 04b3de6af98e8..51b013c63b70c 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -24,36 +24,32 @@ define <2 x i32> @trunc_usat_v2i64_v2i32(<2 x i64> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v2i64_v2i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v2i64_v2i32:
@@ -62,13 +58,11 @@ define <2 x i32> @trunc_usat_v2i64_v2i32(<2 x i64> %a0) {
 ; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE41-NEXT:    retq
@@ -124,18 +118,16 @@ define void @trunc_usat_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    movq %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -143,18 +135,16 @@ define void @trunc_usat_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259455,9223372039002259455]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSSE3-NEXT:    movq %xmm0, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
@@ -164,13 +154,11 @@ define void @trunc_usat_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) {
 ; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE41-NEXT:    movq %xmm0, (%rdi)
@@ -230,31 +218,27 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v4i64_v4i32:
@@ -262,54 +246,47 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; SSSE3-NEXT:    por %xmm4, %xmm0
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v4i64_v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pand %xmm5, %xmm3
-; SSE41-NEXT:    por %xmm6, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
 ; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [4294967295,4294967295]
 ; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [4294967295,429496729]
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
@@ -434,184 +411,165 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(ptr %p0) {
 ; SSE2-LABEL: trunc_usat_v8i64_v8i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa (%rdi), %xmm2
-; SSE2-NEXT:    movdqa 16(%rdi), %xmm5
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm0
 ; SSE2-NEXT:    movdqa 32(%rdi), %xmm6
 ; SSE2-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm0, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
-; SSE2-NEXT:    movdqa %xmm4, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pand %xmm8, %xmm1
-; SSE2-NEXT:    pandn %xmm3, %xmm8
-; SSE2-NEXT:    por %xmm1, %xmm8
+; SSE2-NEXT:    pxor %xmm4, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm5, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
+; SSE2-NEXT:    pand %xmm9, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm7
+; SSE2-NEXT:    por %xmm1, %xmm7
 ; SSE2-NEXT:    movdqa %xmm6, %xmm1
-; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm4, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm6
 ; SSE2-NEXT:    pandn %xmm3, %xmm1
 ; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm8[0,2]
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm4, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm5
-; SSE2-NEXT:    pandn %xmm3, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm7[0,2]
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm3, %xmm6
+; SSE2-NEXT:    por %xmm0, %xmm6
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    pandn %xmm3, %xmm0
 ; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm7[0,2]
+; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm6[0,2]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v8i64_v8i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa (%rdi), %xmm2
-; SSSE3-NEXT:    movdqa 16(%rdi), %xmm5
+; SSSE3-NEXT:    movdqa 16(%rdi), %xmm0
 ; SSSE3-NEXT:    movdqa 32(%rdi), %xmm6
 ; SSSE3-NEXT:    movdqa 48(%rdi), %xmm1
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [4294967295,4294967295]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm0, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm8
-; SSSE3-NEXT:    pand %xmm8, %xmm1
-; SSSE3-NEXT:    pandn %xmm3, %xmm8
-; SSSE3-NEXT:    por %xmm1, %xmm8
+; SSSE3-NEXT:    pxor %xmm4, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7
+; SSSE3-NEXT:    pand %xmm9, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm7
+; SSSE3-NEXT:    por %xmm1, %xmm7
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm1
-; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSSE3-NEXT:    movdqa %xmm4, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm10, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm9, %xmm1
 ; SSSE3-NEXT:    pand %xmm1, %xmm6
 ; SSSE3-NEXT:    pandn %xmm3, %xmm1
 ; SSSE3-NEXT:    por %xmm6, %xmm1
-; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm8[0,2]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSSE3-NEXT:    movdqa %xmm4, %xmm8
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm5
-; SSSE3-NEXT:    pandn %xmm3, %xmm7
-; SSSE3-NEXT:    por %xmm5, %xmm7
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm0, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm7[0,2]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pandn %xmm3, %xmm6
+; SSSE3-NEXT:    por %xmm0, %xmm6
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSSE3-NEXT:    pand %xmm5, %xmm0
 ; SSSE3-NEXT:    pand %xmm0, %xmm2
 ; SSSE3-NEXT:    pandn %xmm3, %xmm0
 ; SSSE3-NEXT:    por %xmm2, %xmm0
-; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm7[0,2]
+; SSSE3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm6[0,2]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm3
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm6
-; SSE41-NEXT:    movdqa 32(%rdi), %xmm7
+; SSE41-NEXT:    movdqa (%rdi), %xmm4
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm7
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm8
 ; SSE41-NEXT:    movdqa 48(%rdi), %xmm1
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [4294967295,4294967295]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [4294967295,4294967295]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259455,9223372039002259455]
-; SSE41-NEXT:    movdqa %xmm4, %xmm8
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm4, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm8
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT:    movdqa %xmm5, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm9
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm4, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
 ; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm8[0,2]
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE41-NEXT:    movdqa %xmm4, %xmm8
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm8, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm7[0,2]
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
+; SSE41-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm9[0,2]
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm8
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm8
+; SSE41-NEXT:    pxor %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pand %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm8[0,2]
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i32:
@@ -722,18 +680,16 @@ define <2 x i16> @trunc_usat_v2i64_v2i16(<2 x i64> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    retq
 ;
@@ -741,18 +697,16 @@ define <2 x i16> @trunc_usat_v2i64_v2i16(<2 x i64> %a0) {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSSE3-NEXT:    retq
 ;
@@ -762,13 +716,11 @@ define <2 x i16> @trunc_usat_v2i64_v2i16(<2 x i64> %a0) {
 ; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
@@ -845,18 +797,16 @@ define void @trunc_usat_v2i64_v2i16_store(<2 x i64> %a0, ptr %p1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    movd %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
@@ -865,18 +815,16 @@ define void @trunc_usat_v2i64_v2i16_store(<2 x i64> %a0, ptr %p1) {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002324991,9223372039002324991]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSSE3-NEXT:    movd %xmm0, (%rdi)
 ; SSSE3-NEXT:    retq
@@ -887,13 +835,11 @@ define void @trunc_usat_v2i64_v2i16_store(<2 x i64> %a0, ptr %p1) {
 ; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
@@ -979,33 +925,29 @@ define <4 x i16> @trunc_usat_v4i64_v4i16(<4 x i64> %a0) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pand %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
@@ -1016,65 +958,60 @@ define <4 x i16> @trunc_usat_v4i64_v4i16(<4 x i64> %a0) {
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
 ; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    por %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
 ; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v4i64_v4i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm5, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
-; SSE41-NEXT:    pxor %xmm3, %xmm4
-; SSE41-NEXT:    movdqa %xmm5, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm7
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
+; SSE41-NEXT:    pxor %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    packusdw %xmm6, %xmm2
-; SSE41-NEXT:    packusdw %xmm2, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pand %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    packusdw %xmm7, %xmm3
+; SSE41-NEXT:    packusdw %xmm3, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v4i64_v4i16:
@@ -1152,33 +1089,29 @@ define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pand %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movq %xmm1, (%rdi)
@@ -1190,33 +1123,29 @@ define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) {
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
 ; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    por %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
 ; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSSE3-NEXT:    movq %xmm1, (%rdi)
@@ -1225,31 +1154,30 @@ define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) {
 ; SSE41-LABEL: trunc_usat_v4i64_v4i16_store:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [65535,65535]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm5, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm6
-; SSE41-NEXT:    pxor %xmm2, %xmm4
-; SSE41-NEXT:    movdqa %xmm5, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm7
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm7
+; SSE41-NEXT:    pxor %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    packusdw %xmm6, %xmm3
-; SSE41-NEXT:    packusdw %xmm3, %xmm3
-; SSE41-NEXT:    movq %xmm3, (%rdi)
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    packusdw %xmm7, %xmm4
+; SSE41-NEXT:    packusdw %xmm4, %xmm4
+; SSE41-NEXT:    movq %xmm4, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v4i64_v4i16_store:
@@ -1328,204 +1256,185 @@ define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) {
 define <8 x i16> @trunc_usat_v8i64_v8i16(ptr %p0) {
 ; SSE2-LABEL: trunc_usat_v8i64_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa (%rdi), %xmm4
+; SSE2-NEXT:    movdqa (%rdi), %xmm5
 ; SSE2-NEXT:    movdqa 16(%rdi), %xmm0
 ; SSE2-NEXT:    movdqa 32(%rdi), %xmm6
 ; SSE2-NEXT:    movdqa 48(%rdi), %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
-; SSE2-NEXT:    movdqa %xmm5, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm6
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE2-NEXT:    movdqa %xmm4, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm6, %xmm1
 ; SSE2-NEXT:    movdqa %xmm7, %xmm6
 ; SSE2-NEXT:    pxor %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    movdqa %xmm5, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE2-NEXT:    pand %xmm9, %xmm6
 ; SSE2-NEXT:    pand %xmm6, %xmm7
-; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm6
 ; SSE2-NEXT:    por %xmm7, %xmm6
-; SSE2-NEXT:    movdqa %xmm4, %xmm7
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
 ; SSE2-NEXT:    pxor %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    movdqa %xmm5, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pandn %xmm1, %xmm8
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm4
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSE2-NEXT:    pand %xmm9, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm7
+; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm5
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[0,2,2,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: trunc_usat_v8i64_v8i16:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa (%rdi), %xmm4
+; SSSE3-NEXT:    movdqa (%rdi), %xmm5
 ; SSSE3-NEXT:    movdqa 16(%rdi), %xmm0
 ; SSSE3-NEXT:    movdqa 32(%rdi), %xmm6
 ; SSSE3-NEXT:    movdqa 48(%rdi), %xmm7
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [65535,65535]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [65535,65535]
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm10, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm6
-; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm6, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm9, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm6
+; SSSE3-NEXT:    pandn %xmm2, %xmm1
+; SSSE3-NEXT:    por %xmm6, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm7, %xmm6
 ; SSSE3-NEXT:    pxor %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSSE3-NEXT:    movdqa %xmm5, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm10, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSSE3-NEXT:    pand %xmm9, %xmm6
 ; SSSE3-NEXT:    pand %xmm6, %xmm7
-; SSSE3-NEXT:    pandn %xmm1, %xmm6
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
 ; SSSE3-NEXT:    por %xmm7, %xmm6
-; SSSE3-NEXT:    movdqa %xmm4, %xmm7
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
 ; SSSE3-NEXT:    pxor %xmm3, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSSE3-NEXT:    movdqa %xmm5, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm8
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pandn %xmm1, %xmm8
-; SSSE3-NEXT:    por %xmm4, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pandn %xmm1, %xmm4
-; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
+; SSSE3-NEXT:    pand %xmm9, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm2, %xmm7
+; SSSE3-NEXT:    por %xmm5, %xmm7
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm5
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[0,2,2,3]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,2,2,3]
+; SSSE3-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,1,0,2,4,5,6,7]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; SSSE3-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; SSSE3-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm7
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
-; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
-; SSE41-NEXT:    movdqa 48(%rdi), %xmm6
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [65535,65535]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa (%rdi), %xmm8
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm4
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm7
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [65535,65535]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT:    movdqa %xmm5, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147549183,2147549183,2147549183,2147549183]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002324991,9223372039002324991]
-; SSE41-NEXT:    movdqa %xmm4, %xmm8
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm9
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    packusdw %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm8
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm4, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
 ; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm8
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm4, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    packusdw %xmm8, %xmm1
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE41-NEXT:    movdqa %xmm4, %xmm8
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm8, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    packusdw %xmm7, %xmm2
-; SSE41-NEXT:    packusdw %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm3, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm8
+; SSE41-NEXT:    pxor %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pand %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    packusdw %xmm8, %xmm3
+; SSE41-NEXT:    packusdw %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i16:
@@ -2035,17 +1944,15 @@ define <2 x i8> @trunc_usat_v2i64_v2i8(<2 x i64> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
@@ -2056,17 +1963,15 @@ define <2 x i8> @trunc_usat_v2i64_v2i8(<2 x i64> %a0) {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm1, %xmm0
 ; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSSE3-NEXT:    retq
 ;
@@ -2076,13 +1981,11 @@ define <2 x i8> @trunc_usat_v2i64_v2i8(<2 x i64> %a0) {
 ; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
@@ -2137,22 +2040,20 @@ define void @trunc_usat_v2i64_v2i8_store(<2 x i64> %a0, ptr %p1) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    pxor %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm2
-; SSE2-NEXT:    packuswb %xmm2, %xmm2
-; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    packuswb %xmm1, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
@@ -2160,19 +2061,17 @@ define void @trunc_usat_v2i64_v2i8_store(<2 x i64> %a0, ptr %p1) {
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259711,9223372039002259711]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
 ; SSSE3-NEXT:    pand %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm2
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT:    movd %xmm1, %eax
 ; SSSE3-NEXT:    movw %ax, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
@@ -2182,13 +2081,11 @@ define void @trunc_usat_v2i64_v2i8_store(<2 x i64> %a0, ptr %p1) {
 ; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm2 = xmm2[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    pextrw $0, %xmm2, (%rdi)
@@ -2249,27 +2146,23 @@ define <4 x i8> @trunc_usat_v4i64_v4i8(<4 x i64> %a0) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pand %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    pxor %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pandn %xmm2, %xmm4
 ; SSE2-NEXT:    por %xmm1, %xmm4
@@ -2286,27 +2179,23 @@ define <4 x i8> @trunc_usat_v4i64_v4i8(<4 x i64> %a0) {
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
 ; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm4
 ; SSSE3-NEXT:    pxor %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm6, %xmm4
 ; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    pandn %xmm2, %xmm4
 ; SSSE3-NEXT:    por %xmm1, %xmm4
@@ -2319,32 +2208,31 @@ define <4 x i8> @trunc_usat_v4i64_v4i8(<4 x i64> %a0) {
 ; SSE41-LABEL: trunc_usat_v4i64_v4i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm6, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm6, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm4, %xmm3
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    movdqa %xmm6, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE41-NEXT:    movapd {{.*#+}} xmm5 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm7, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    movapd %xmm5, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm1, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm7, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSE41-NEXT:    pshufb %xmm0, %xmm5
 ; SSE41-NEXT:    pshufb %xmm0, %xmm4
-; SSE41-NEXT:    pshufb %xmm0, %xmm3
-; SSE41-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v4i64_v4i8:
@@ -2426,33 +2314,29 @@ define void @trunc_usat_v4i64_v4i8_store(<4 x i64> %a0, ptr%p1) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
 ; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pand %xmm5, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm7, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm0
 ; SSE2-NEXT:    pandn %xmm2, %xmm3
 ; SSE2-NEXT:    por %xmm0, %xmm3
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pand %xmm5, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    packuswb %xmm4, %xmm3
+; SSE2-NEXT:    packuswb %xmm0, %xmm3
 ; SSE2-NEXT:    packuswb %xmm3, %xmm3
 ; SSE2-NEXT:    packuswb %xmm3, %xmm3
 ; SSE2-NEXT:    movd %xmm3, (%rdi)
@@ -2460,70 +2344,65 @@ define void @trunc_usat_v4i64_v4i8_store(<4 x i64> %a0, ptr%p1) {
 ;
 ; SSSE3-LABEL: trunc_usat_v4i64_v4i8_store:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [255,255]
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm8, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm3, %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSSE3-NEXT:    pshufb %xmm0, %xmm4
-; SSSE3-NEXT:    pshufb %xmm0, %xmm3
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; SSSE3-NEXT:    movd %xmm3, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm1 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; SSSE3-NEXT:    pshufb %xmm1, %xmm0
+; SSSE3-NEXT:    pshufb %xmm1, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    movd %xmm2, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v4i64_v4i8_store:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm5, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm6
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm5, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm6, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm7
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm7
+; SSE41-NEXT:    pxor %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pand %xmm2, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm4
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = <0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
-; SSE41-NEXT:    pshufb %xmm0, %xmm3
-; SSE41-NEXT:    pshufb %xmm0, %xmm6
-; SSE41-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
-; SSE41-NEXT:    movd %xmm6, (%rdi)
+; SSE41-NEXT:    pshufb %xmm0, %xmm4
+; SSE41-NEXT:    pshufb %xmm0, %xmm7
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]
+; SSE41-NEXT:    movd %xmm7, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v4i64_v4i8_store:
@@ -2614,59 +2493,51 @@ define <8 x i8> @trunc_usat_v8i64_v8i8(ptr %p0) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
 ; SSE2-NEXT:    pxor %xmm3, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pand %xmm8, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm0, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE2-NEXT:    pand %xmm9, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm7
+; SSE2-NEXT:    por %xmm0, %xmm7
 ; SSE2-NEXT:    movdqa %xmm6, %xmm0
 ; SSE2-NEXT:    pxor %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm9, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm6
 ; SSE2-NEXT:    pandn %xmm2, %xmm0
 ; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    packuswb %xmm8, %xmm0
+; SSE2-NEXT:    packuswb %xmm7, %xmm0
 ; SSE2-NEXT:    movdqa %xmm5, %xmm6
 ; SSE2-NEXT:    pxor %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm5
-; SSE2-NEXT:    pandn %xmm2, %xmm7
-; SSE2-NEXT:    por %xmm5, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm5, %xmm6
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    pxor %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm2, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    packuswb %xmm7, %xmm4
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm5
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    packuswb %xmm6, %xmm5
+; SSE2-NEXT:    packuswb %xmm5, %xmm0
 ; SSE2-NEXT:    packuswb %xmm0, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -2680,117 +2551,106 @@ define <8 x i8> @trunc_usat_v8i64_v8i8(ptr %p0) {
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm7
 ; SSSE3-NEXT:    pxor %xmm3, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm8
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm8
-; SSSE3-NEXT:    pand %xmm8, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm0, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSSE3-NEXT:    pand %xmm9, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm7
+; SSSE3-NEXT:    por %xmm0, %xmm7
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm0
 ; SSSE3-NEXT:    pxor %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm10, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm9, %xmm0
 ; SSSE3-NEXT:    pand %xmm0, %xmm6
 ; SSSE3-NEXT:    pandn %xmm2, %xmm0
 ; SSSE3-NEXT:    por %xmm6, %xmm0
-; SSSE3-NEXT:    packuswb %xmm8, %xmm0
+; SSSE3-NEXT:    packuswb %xmm7, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm6
 ; SSSE3-NEXT:    pxor %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm8
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm5
-; SSSE3-NEXT:    pandn %xmm2, %xmm7
-; SSSE3-NEXT:    por %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm5
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm5, %xmm6
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm5
 ; SSSE3-NEXT:    pxor %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm2, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    packuswb %xmm7, %xmm4
-; SSSE3-NEXT:    packuswb %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pandn %xmm2, %xmm5
+; SSSE3-NEXT:    por %xmm1, %xmm5
+; SSSE3-NEXT:    packuswb %xmm6, %xmm5
+; SSSE3-NEXT:    packuswb %xmm5, %xmm0
 ; SSSE3-NEXT:    packuswb %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm7
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
-; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
-; SSE41-NEXT:    movdqa 48(%rdi), %xmm6
-; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa (%rdi), %xmm8
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm4
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm7
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm5, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm4, %xmm8
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm9
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT:    packusdw %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm8
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm4, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
 ; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm8
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm4, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    packusdw %xmm8, %xmm1
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm4, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE41-NEXT:    movdqa %xmm4, %xmm8
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm8, %xmm0
-; SSE41-NEXT:    movapd %xmm2, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    packusdw %xmm7, %xmm2
-; SSE41-NEXT:    packusdw %xmm2, %xmm1
-; SSE41-NEXT:    packuswb %xmm1, %xmm1
+; SSE41-NEXT:    movapd %xmm3, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm8
+; SSE41-NEXT:    pxor %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pand %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    packusdw %xmm8, %xmm3
+; SSE41-NEXT:    packusdw %xmm3, %xmm2
+; SSE41-NEXT:    packuswb %xmm2, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i8:
@@ -2872,59 +2732,51 @@ define void @trunc_usat_v8i64_v8i8_store(ptr %p0, ptr%p1) {
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm7
 ; SSE2-NEXT:    pxor %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm8
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm8
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm8
-; SSE2-NEXT:    por %xmm5, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE2-NEXT:    pand %xmm9, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm1, %xmm7
+; SSE2-NEXT:    por %xmm5, %xmm7
 ; SSE2-NEXT:    movdqa %xmm6, %xmm5
 ; SSE2-NEXT:    pxor %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE2-NEXT:    pand %xmm9, %xmm5
 ; SSE2-NEXT:    pand %xmm5, %xmm6
 ; SSE2-NEXT:    pandn %xmm1, %xmm5
 ; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    packuswb %xmm8, %xmm5
+; SSE2-NEXT:    packuswb %xmm7, %xmm5
 ; SSE2-NEXT:    movdqa %xmm4, %xmm6
 ; SSE2-NEXT:    pxor %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm3, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[0,0,2,2]
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pandn %xmm1, %xmm7
-; SSE2-NEXT:    por %xmm4, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pxor %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pandn %xmm1, %xmm3
-; SSE2-NEXT:    por %xmm0, %xmm3
-; SSE2-NEXT:    packuswb %xmm7, %xmm3
-; SSE2-NEXT:    packuswb %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    packuswb %xmm6, %xmm4
+; SSE2-NEXT:    packuswb %xmm4, %xmm5
 ; SSE2-NEXT:    packuswb %xmm5, %xmm5
 ; SSE2-NEXT:    movq %xmm5, (%rsi)
 ; SSE2-NEXT:    retq
@@ -2939,118 +2791,107 @@ define void @trunc_usat_v8i64_v8i8_store(ptr %p0, ptr%p1) {
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm7
 ; SSSE3-NEXT:    pxor %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm8
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm8
-; SSSE3-NEXT:    pand %xmm8, %xmm5
-; SSSE3-NEXT:    pandn %xmm1, %xmm8
-; SSSE3-NEXT:    por %xmm5, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSSE3-NEXT:    pand %xmm9, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm1, %xmm7
+; SSSE3-NEXT:    por %xmm5, %xmm7
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm5
 ; SSSE3-NEXT:    pxor %xmm2, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSSE3-NEXT:    pand %xmm9, %xmm5
 ; SSSE3-NEXT:    pand %xmm5, %xmm6
 ; SSSE3-NEXT:    pandn %xmm1, %xmm5
 ; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    packuswb %xmm8, %xmm5
+; SSSE3-NEXT:    packuswb %xmm7, %xmm5
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm6
 ; SSSE3-NEXT:    pxor %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm8
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm8[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm7
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pandn %xmm1, %xmm7
-; SSSE3-NEXT:    por %xmm4, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm1, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm4
 ; SSSE3-NEXT:    pxor %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pand %xmm3, %xmm0
-; SSSE3-NEXT:    pandn %xmm1, %xmm3
-; SSSE3-NEXT:    por %xmm0, %xmm3
-; SSSE3-NEXT:    packuswb %xmm7, %xmm3
-; SSSE3-NEXT:    packuswb %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    packuswb %xmm6, %xmm4
+; SSSE3-NEXT:    packuswb %xmm4, %xmm5
 ; SSSE3-NEXT:    packuswb %xmm5, %xmm5
 ; SSSE3-NEXT:    movq %xmm5, (%rsi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v8i64_v8i8_store:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm7
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm6
-; SSE41-NEXT:    movdqa 32(%rdi), %xmm2
-; SSE41-NEXT:    movdqa 48(%rdi), %xmm5
-; SSE41-NEXT:    movapd {{.*#+}} xmm1 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa (%rdi), %xmm8
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm7
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm3
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm6
+; SSE41-NEXT:    movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm4, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm9
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm7
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    packusdw %xmm9, %xmm7
 ; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm3, %xmm8
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm8
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm3, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
 ; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm8
+; SSE41-NEXT:    movapd %xmm2, %xmm8
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm8
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    movdqa %xmm3, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT:    movdqa %xmm3, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm6
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm6
-; SSE41-NEXT:    packusdw %xmm8, %xmm6
-; SSE41-NEXT:    movdqa %xmm5, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    movdqa %xmm3, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE41-NEXT:    movdqa %xmm3, %xmm8
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm8, %xmm0
-; SSE41-NEXT:    movapd %xmm1, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm7
-; SSE41-NEXT:    pxor %xmm2, %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    packusdw %xmm7, %xmm1
-; SSE41-NEXT:    packusdw %xmm1, %xmm6
-; SSE41-NEXT:    packuswb %xmm6, %xmm6
-; SSE41-NEXT:    movq %xmm6, (%rsi)
+; SSE41-NEXT:    pxor %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pand %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    packusdw %xmm8, %xmm2
+; SSE41-NEXT:    packusdw %xmm2, %xmm7
+; SSE41-NEXT:    packuswb %xmm7, %xmm7
+; SSE41-NEXT:    movq %xmm7, (%rsi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v8i64_v8i8_store:
@@ -3128,7 +2969,7 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(ptr %p0) {
 ; SSE2-LABEL: trunc_usat_v16i64_v16i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa 96(%rdi), %xmm1
-; SSE2-NEXT:    movdqa 112(%rdi), %xmm3
+; SSE2-NEXT:    movdqa 112(%rdi), %xmm4
 ; SSE2-NEXT:    movdqa 64(%rdi), %xmm6
 ; SSE2-NEXT:    movdqa 80(%rdi), %xmm7
 ; SSE2-NEXT:    movdqa (%rdi), %xmm10
@@ -3136,116 +2977,100 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(ptr %p0) {
 ; SSE2-NEXT:    movdqa 32(%rdi), %xmm8
 ; SSE2-NEXT:    movdqa 48(%rdi), %xmm9
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,255]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm11
-; SSE2-NEXT:    pxor %xmm4, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm12
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSE2-NEXT:    pxor %xmm3, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm13
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm13[0,0,2,2]
-; SSE2-NEXT:    pand %xmm12, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3]
-; SSE2-NEXT:    por %xmm11, %xmm12
-; SSE2-NEXT:    pand %xmm12, %xmm0
-; SSE2-NEXT:    pandn %xmm2, %xmm12
-; SSE2-NEXT:    por %xmm0, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm11
+; SSE2-NEXT:    pand %xmm13, %xmm11
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm11
+; SSE2-NEXT:    por %xmm0, %xmm11
 ; SSE2-NEXT:    movdqa %xmm10, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm11
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm13
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm14
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,3,3]
-; SSE2-NEXT:    por %xmm14, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm13, %xmm0
 ; SSE2-NEXT:    pand %xmm0, %xmm10
 ; SSE2-NEXT:    pandn %xmm2, %xmm0
 ; SSE2-NEXT:    por %xmm10, %xmm0
-; SSE2-NEXT:    packuswb %xmm12, %xmm0
+; SSE2-NEXT:    packuswb %xmm11, %xmm0
 ; SSE2-NEXT:    movdqa %xmm9, %xmm10
-; SSE2-NEXT:    pxor %xmm4, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSE2-NEXT:    movdqa %xmm5, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm10, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm12[0,0,2,2]
-; SSE2-NEXT:    pand %xmm11, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm10, %xmm11
-; SSE2-NEXT:    pand %xmm11, %xmm9
-; SSE2-NEXT:    pandn %xmm2, %xmm11
-; SSE2-NEXT:    por %xmm9, %xmm11
-; SSE2-NEXT:    movdqa %xmm8, %xmm9
-; SSE2-NEXT:    pxor %xmm4, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm10
+; SSE2-NEXT:    pxor %xmm3, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm9, %xmm12
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm12[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm10
+; SSE2-NEXT:    pand %xmm12, %xmm10
 ; SSE2-NEXT:    pand %xmm10, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm12[1,1,3,3]
-; SSE2-NEXT:    por %xmm9, %xmm10
-; SSE2-NEXT:    pand %xmm10, %xmm8
 ; SSE2-NEXT:    pandn %xmm2, %xmm10
-; SSE2-NEXT:    por %xmm8, %xmm10
-; SSE2-NEXT:    packuswb %xmm11, %xmm10
-; SSE2-NEXT:    packuswb %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm7, %xmm8
-; SSE2-NEXT:    pxor %xmm4, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE2-NEXT:    movdqa %xmm5, %xmm10
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[0,0,2,2]
+; SSE2-NEXT:    por %xmm9, %xmm10
+; SSE2-NEXT:    movdqa %xmm8, %xmm9
+; SSE2-NEXT:    pxor %xmm3, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm9
+; SSE2-NEXT:    pand %xmm12, %xmm9
 ; SSE2-NEXT:    pand %xmm9, %xmm8
-; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
-; SSE2-NEXT:    por %xmm8, %xmm9
-; SSE2-NEXT:    pand %xmm9, %xmm7
 ; SSE2-NEXT:    pandn %xmm2, %xmm9
-; SSE2-NEXT:    por %xmm7, %xmm9
+; SSE2-NEXT:    por %xmm8, %xmm9
+; SSE2-NEXT:    packuswb %xmm10, %xmm9
+; SSE2-NEXT:    packuswb %xmm9, %xmm0
+; SSE2-NEXT:    movdqa %xmm7, %xmm8
+; SSE2-NEXT:    pxor %xmm3, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm10
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm8
+; SSE2-NEXT:    pand %xmm10, %xmm8
+; SSE2-NEXT:    pand %xmm8, %xmm7
+; SSE2-NEXT:    pandn %xmm2, %xmm8
+; SSE2-NEXT:    por %xmm7, %xmm8
 ; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT:    pxor %xmm3, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm7[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm10
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm10[1,1,3,3]
-; SSE2-NEXT:    por %xmm11, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSE2-NEXT:    pand %xmm10, %xmm7
 ; SSE2-NEXT:    pand %xmm7, %xmm6
 ; SSE2-NEXT:    pandn %xmm2, %xmm7
 ; SSE2-NEXT:    por %xmm6, %xmm7
-; SSE2-NEXT:    packuswb %xmm9, %xmm7
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    pxor %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT:    packuswb %xmm8, %xmm7
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
 ; SSE2-NEXT:    movdqa %xmm5, %xmm9
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm9
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[0,0,2,2]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    pand %xmm8, %xmm3
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm3, %xmm8
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    pand %xmm4, %xmm1
 ; SSE2-NEXT:    pandn %xmm2, %xmm4
 ; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    packuswb %xmm8, %xmm4
+; SSE2-NEXT:    packuswb %xmm6, %xmm4
 ; SSE2-NEXT:    packuswb %xmm4, %xmm7
 ; SSE2-NEXT:    packuswb %xmm7, %xmm0
 ; SSE2-NEXT:    retq
@@ -3253,7 +3078,7 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(ptr %p0) {
 ; SSSE3-LABEL: trunc_usat_v16i64_v16i8:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa 96(%rdi), %xmm1
-; SSSE3-NEXT:    movdqa 112(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa 112(%rdi), %xmm4
 ; SSSE3-NEXT:    movdqa 64(%rdi), %xmm6
 ; SSSE3-NEXT:    movdqa 80(%rdi), %xmm7
 ; SSSE3-NEXT:    movdqa (%rdi), %xmm10
@@ -3261,226 +3086,203 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(ptr %p0) {
 ; SSSE3-NEXT:    movdqa 32(%rdi), %xmm8
 ; SSSE3-NEXT:    movdqa 48(%rdi), %xmm9
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [255,255]
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm11
-; SSSE3-NEXT:    pxor %xmm4, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm12
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSSE3-NEXT:    pxor %xmm3, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm13
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm13[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm12, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm13[1,1,3,3]
-; SSSE3-NEXT:    por %xmm11, %xmm12
-; SSSE3-NEXT:    pand %xmm12, %xmm0
-; SSSE3-NEXT:    pandn %xmm2, %xmm12
-; SSSE3-NEXT:    por %xmm0, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm13
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm11
+; SSSE3-NEXT:    pand %xmm13, %xmm11
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm11
+; SSSE3-NEXT:    por %xmm0, %xmm11
 ; SSSE3-NEXT:    movdqa %xmm10, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm11
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm13
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm11, %xmm14
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,3,3]
-; SSSE3-NEXT:    por %xmm14, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm13
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSSE3-NEXT:    pand %xmm13, %xmm0
 ; SSSE3-NEXT:    pand %xmm0, %xmm10
 ; SSSE3-NEXT:    pandn %xmm2, %xmm0
 ; SSSE3-NEXT:    por %xmm10, %xmm0
-; SSSE3-NEXT:    packuswb %xmm12, %xmm0
+; SSSE3-NEXT:    packuswb %xmm11, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm9, %xmm10
-; SSSE3-NEXT:    pxor %xmm4, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSSE3-NEXT:    movdqa %xmm5, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm12[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm11, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm10, %xmm11
-; SSSE3-NEXT:    pand %xmm11, %xmm9
-; SSSE3-NEXT:    pandn %xmm2, %xmm11
-; SSSE3-NEXT:    por %xmm9, %xmm11
-; SSSE3-NEXT:    movdqa %xmm8, %xmm9
-; SSSE3-NEXT:    pxor %xmm4, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm10
+; SSSE3-NEXT:    pxor %xmm3, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm12
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm12[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm10
+; SSSE3-NEXT:    pand %xmm12, %xmm10
 ; SSSE3-NEXT:    pand %xmm10, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm12[1,1,3,3]
-; SSSE3-NEXT:    por %xmm9, %xmm10
-; SSSE3-NEXT:    pand %xmm10, %xmm8
 ; SSSE3-NEXT:    pandn %xmm2, %xmm10
-; SSSE3-NEXT:    por %xmm8, %xmm10
-; SSSE3-NEXT:    packuswb %xmm11, %xmm10
-; SSSE3-NEXT:    packuswb %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm7, %xmm8
-; SSSE3-NEXT:    pxor %xmm4, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSSE3-NEXT:    movdqa %xmm5, %xmm10
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm10[0,0,2,2]
+; SSSE3-NEXT:    por %xmm9, %xmm10
+; SSSE3-NEXT:    movdqa %xmm8, %xmm9
+; SSSE3-NEXT:    pxor %xmm3, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm9
+; SSSE3-NEXT:    pand %xmm12, %xmm9
 ; SSSE3-NEXT:    pand %xmm9, %xmm8
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
-; SSSE3-NEXT:    por %xmm8, %xmm9
-; SSSE3-NEXT:    pand %xmm9, %xmm7
 ; SSSE3-NEXT:    pandn %xmm2, %xmm9
-; SSSE3-NEXT:    por %xmm7, %xmm9
+; SSSE3-NEXT:    por %xmm8, %xmm9
+; SSSE3-NEXT:    packuswb %xmm10, %xmm9
+; SSSE3-NEXT:    packuswb %xmm9, %xmm0
+; SSSE3-NEXT:    movdqa %xmm7, %xmm8
+; SSSE3-NEXT:    pxor %xmm3, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm10
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm8
+; SSSE3-NEXT:    pand %xmm10, %xmm8
+; SSSE3-NEXT:    pand %xmm8, %xmm7
+; SSSE3-NEXT:    pandn %xmm2, %xmm8
+; SSSE3-NEXT:    por %xmm7, %xmm8
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm7
-; SSSE3-NEXT:    pxor %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSSE3-NEXT:    pxor %xmm3, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm7[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm10
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm10[1,1,3,3]
-; SSSE3-NEXT:    por %xmm11, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm7
+; SSSE3-NEXT:    pand %xmm10, %xmm7
 ; SSSE3-NEXT:    pand %xmm7, %xmm6
 ; SSSE3-NEXT:    pandn %xmm2, %xmm7
 ; SSSE3-NEXT:    por %xmm6, %xmm7
-; SSSE3-NEXT:    packuswb %xmm9, %xmm7
-; SSSE3-NEXT:    movdqa %xmm3, %xmm6
-; SSSE3-NEXT:    pxor %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSSE3-NEXT:    packuswb %xmm8, %xmm7
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm9
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm9
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm8, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm9[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm8
-; SSSE3-NEXT:    pand %xmm8, %xmm3
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm3, %xmm8
-; SSSE3-NEXT:    movdqa %xmm1, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSSE3-NEXT:    pand %xmm9, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm5, %xmm4
 ; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    pandn %xmm2, %xmm4
 ; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    packuswb %xmm8, %xmm4
+; SSSE3-NEXT:    packuswb %xmm6, %xmm4
 ; SSSE3-NEXT:    packuswb %xmm4, %xmm7
 ; SSSE3-NEXT:    packuswb %xmm7, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: trunc_usat_v16i64_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa 96(%rdi), %xmm2
-; SSE41-NEXT:    movdqa 112(%rdi), %xmm4
-; SSE41-NEXT:    movdqa 64(%rdi), %xmm7
-; SSE41-NEXT:    movdqa 80(%rdi), %xmm8
-; SSE41-NEXT:    movdqa (%rdi), %xmm11
-; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
-; SSE41-NEXT:    movdqa 32(%rdi), %xmm9
-; SSE41-NEXT:    movdqa 48(%rdi), %xmm10
-; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [255,255]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa 96(%rdi), %xmm3
+; SSE41-NEXT:    movdqa 112(%rdi), %xmm5
+; SSE41-NEXT:    movdqa 64(%rdi), %xmm8
+; SSE41-NEXT:    movdqa 80(%rdi), %xmm9
+; SSE41-NEXT:    movdqa (%rdi), %xmm12
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm2
+; SSE41-NEXT:    movdqa 32(%rdi), %xmm10
+; SSE41-NEXT:    movdqa 48(%rdi), %xmm11
+; SSE41-NEXT:    movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT:    movdqa %xmm6, %xmm13
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm13
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
-; SSE41-NEXT:    movdqa %xmm5, %xmm12
+; SSE41-NEXT:    pcmpgtd %xmm14, %xmm0
+; SSE41-NEXT:    pand %xmm13, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm13
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm13
+; SSE41-NEXT:    movdqa %xmm12, %xmm0
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm14, %xmm0
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    movapd %xmm4, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm2
+; SSE41-NEXT:    packusdw %xmm13, %xmm2
+; SSE41-NEXT:    movdqa %xmm11, %xmm0
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm12
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm12
-; SSE41-NEXT:    movdqa %xmm5, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm13
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm13, %xmm0
 ; SSE41-NEXT:    pand %xmm12, %xmm0
-; SSE41-NEXT:    por %xmm13, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm12
-; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm12
-; SSE41-NEXT:    movdqa %xmm11, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm5, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm5, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm13
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[0,0,2,2]
-; SSE41-NEXT:    pand %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm13, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm1
-; SSE41-NEXT:    packusdw %xmm12, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm12
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm12
 ; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm5, %xmm11
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm11
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm11
-; SSE41-NEXT:    movdqa %xmm5, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm13, %xmm0
 ; SSE41-NEXT:    pand %xmm11, %xmm0
-; SSE41-NEXT:    por %xmm12, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm11
+; SSE41-NEXT:    movapd %xmm4, %xmm11
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm11
+; SSE41-NEXT:    packusdw %xmm12, %xmm11
+; SSE41-NEXT:    packusdw %xmm11, %xmm2
 ; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm5, %xmm10
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm10
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
-; SSE41-NEXT:    movdqa %xmm5, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
 ; SSE41-NEXT:    pand %xmm10, %xmm0
-; SSE41-NEXT:    por %xmm12, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm10
+; SSE41-NEXT:    movapd %xmm4, %xmm10
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm10
-; SSE41-NEXT:    packusdw %xmm11, %xmm10
-; SSE41-NEXT:    packusdw %xmm10, %xmm1
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm5, %xmm9
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm9
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
-; SSE41-NEXT:    movdqa %xmm5, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
 ; SSE41-NEXT:    pand %xmm9, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm9
+; SSE41-NEXT:    movapd %xmm4, %xmm9
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm9
-; SSE41-NEXT:    movdqa %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm5, %xmm8
+; SSE41-NEXT:    packusdw %xmm10, %xmm9
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm8
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm8
-; SSE41-NEXT:    movdqa %xmm5, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
 ; SSE41-NEXT:    pand %xmm8, %xmm0
-; SSE41-NEXT:    por %xmm10, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm8
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm8
-; SSE41-NEXT:    packusdw %xmm9, %xmm8
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm7
-; SSE41-NEXT:    movdqa %xmm5, %xmm9
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    por %xmm9, %xmm0
-; SSE41-NEXT:    movapd %xmm3, %xmm7
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm2, %xmm6
-; SSE41-NEXT:    movdqa %xmm5, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT:    packusdw %xmm7, %xmm3
-; SSE41-NEXT:    packusdw %xmm3, %xmm8
-; SSE41-NEXT:    packuswb %xmm8, %xmm1
+; SSE41-NEXT:    movapd %xmm4, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm8
+; SSE41-NEXT:    pxor %xmm3, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pand %xmm6, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm4
+; SSE41-NEXT:    packusdw %xmm8, %xmm4
+; SSE41-NEXT:    packusdw %xmm4, %xmm9
+; SSE41-NEXT:    packuswb %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: trunc_usat_v16i64_v16i8:
diff --git a/llvm/test/DebugInfo/AArch64/debugframeinfo.s b/llvm/test/DebugInfo/AArch64/debugframeinfo.s
new file mode 100644
index 0000000000000..5159c07489e33
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/debugframeinfo.s
@@ -0,0 +1,20 @@
+# RUN: llvm-mc -filetype=obj --triple=arm64-apple-darwin22.1.0 %s -o %t.o
+# RUN: llvm-dwarfdump -debug-frame %t.o | FileCheck %s
+
+# CHECK: .debug_frame contents:
+# CHECK-EMPTY:
+# CHECK-NEXT: {{.+}}
+
+# CHECK: .eh_frame contents:
+# CHECK-EMPTY:
+# CHECK-EMPTY:
+
+        .cfi_sections .debug_frame
+        .cfi_startproc
+        .cfi_personality 0x9b, g
+        .cfi_lsda 0x1b, h
+        .cfi_endproc
+        .global g
+g:
+        .global h
+h:
diff --git a/llvm/test/DebugInfo/AArch64/test-fexceptions-debug-frame.ll b/llvm/test/DebugInfo/AArch64/test-fexceptions-debug-frame.ll
new file mode 100644
index 0000000000000..6d6dcddd65cc0
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/test-fexceptions-debug-frame.ll
@@ -0,0 +1,43 @@
+; This test is added to provide test coverage for the 
+; .cfi_sections .debug_frame intrinsic. It aims to make sure that for a AArch64
+; output compiled with -fexceptions, no .cfi_sections .debug_frame is emitted.
+
+; RUN: llc --filetype=asm %s -o - | FileCheck %s
+; CHECK-NOT: .cfi_sections .debug_frame
+; CHECK: .cfi_startproc
+
+
+; ModuleID = 'test.cpp'
+source_filename = "test.cpp"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx13.0.0"
+
+; Function Attrs: noinline norecurse nounwind optnone ssp uwtable(sync)
+define i32 @main() #0 !dbg !10 {
+  %1 = alloca i32, align 4
+  store i32 0, ptr %1, align 4
+  ret i32 1, !dbg !15
+}
+
+attributes #0 = { noinline norecurse nounwind optnone ssp uwtable(sync)}
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+!llvm.dbg.cu = !{!7}
+!llvm.ident = !{!9}
+
+!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+!1 = !{i32 7, !"Dwarf Version", i32 4}
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 8, !"PIC Level", i32 2}
+!5 = !{i32 7, !"uwtable", i32 1}
+!6 = !{i32 7, !"frame-pointer", i32 1}
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !8, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None, sysroot: "/Applications/Xcode.app", sdk: "MacOSX.sdk")
+!8 = !DIFile(filename: "test.cpp", directory: "/Users/shubham/Development")
+!9 = !{!"clang"}
+!10 = distinct !DISubprogram(name: "main", scope: !8, file: !8, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !7, retainedNodes: !14)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!13}
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{}
+!15 = !DILocation(line: 2, column: 5, scope: !10)
diff --git a/llvm/test/DebugInfo/AArch64/test-fno-exceptions-debug-frame.ll b/llvm/test/DebugInfo/AArch64/test-fno-exceptions-debug-frame.ll
new file mode 100644
index 0000000000000..02e66b44d4fee
--- /dev/null
+++ b/llvm/test/DebugInfo/AArch64/test-fno-exceptions-debug-frame.ll
@@ -0,0 +1,40 @@
+; This test is added to provide test coverage for the 
+; .cfi_sections .debug_frame intrinsic. It aims to make sure that for a AArch64
+; output compiled with -fno-exceptions, a .cfi_sections .debug_frame is emitted.
+
+; RUN: llc --filetype=asm %s -o - | FileCheck %s
+; CHECK: .cfi_sections .debug_frame
+
+; ModuleID = 'test.cpp'
+source_filename = "test.cpp"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx13.0.0"
+
+; Function Attrs: noinline norecurse nounwind optnone ssp
+define i32 @main() #0 !dbg !9 {
+  %1 = alloca i32, align 4
+  store i32 0, ptr %1, align 4
+  ret i32 1, !dbg !14
+}
+
+attributes #0 = { noinline norecurse nounwind optnone ssp }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5}
+!llvm.dbg.cu = !{!6}
+!llvm.ident = !{!8}
+
+!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+!1 = !{i32 7, !"Dwarf Version", i32 4}
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 8, !"PIC Level", i32 2}
+!5 = !{i32 7, !"frame-pointer", i32 1}
+!6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !7, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None, sysroot: "/Applications/Xcode.app", sdk: "MacOSX.sdk")
+!7 = !DIFile(filename: "test.cpp", directory: "/Users/shubham/Development")
+!8 = !{!"clang"}
+!9 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !13)
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12}
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{}
+!14 = !DILocation(line: 2, column: 5, scope: !9)
diff --git a/llvm/test/DebugInfo/ARM/move-dbg-value-after-value-list.mir b/llvm/test/DebugInfo/ARM/move-dbg-value-after-value-list.mir
new file mode 100644
index 0000000000000..23e43ccb046d7
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/move-dbg-value-after-value-list.mir
@@ -0,0 +1,66 @@
+# RUN: llc %s -start-after=arm-mve-vpt-opts -stop-after=arm-prera-ldst-opt -o - | FileCheck %s
+# CHECK: %5:rgpr = t2MOVCCr %3, killed %4, 0 /* CC::eq */, $cpsr, debug-location !14
+# CHECK-NEXT: DBG_VALUE 0, $noreg, !10, !DIExpression(), debug-location !14
+# CHECK-NEXT: DBG_VALUE 1, $noreg, !10, !DIExpression(), debug-location !14
+# CHECK-NEXT: DBG_VALUE_LIST !10, !DIExpression(), $noreg, $noreg, debug-location !14
+# CHECK-NEXT: DBG_VALUE $noreg, $noreg, !10, !DIExpression(), debug-location !14
+# CHECK-NEXT: %0:rgpr = t2LDRi12 %5, 0, 14 /* CC::al */, $noreg, debug-location !14 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+# CHECK-NEXT: %6:rgpr = t2LDRi12 %5, 4, 14 /* CC::al */, $noreg, debug-location !14 :: (load (s32) from %ir.btc_user_copy)
+# CHECK-NEXT: %2:gpr = t2LDRi12 %5, 8, 14 /* CC::al */, $noreg, debug-location !14 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+# CHECK-NEXT: DBG_VALUE %2, $noreg, !10, !DIExpression(), debug-location !14
+# CHECK-NEXT: %7:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+# CHECK-NEXT: %8:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+# CHECK-NEXT: %9:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+# CHECK-NEXT: %10:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+
+# This test specifically checks if a DBG_VALUE_LIST that is succeeded by a DBG_VALUE can be properly undefed and the DBG_VALUE is the one that is moved. Assuming both instructions describe the same local variable. 
+
+--- |
+  target triple = "thumbv7k-apple-watchos8.0.0"
+  %struct.backtrace_control = type {}
+  @backtrace_user.ctl_default = internal unnamed_addr constant %struct.backtrace_control zeroinitializer, !dbg !0
+  define i32 @backtrace_user(ptr nocapture noundef writeonly %bt, i32 noundef %max_frames, ptr noundef readonly %ctl_in, ptr nocapture noundef readnone %info_out) local_unnamed_addr #0 !dbg !2 {entry:
+    %tobool.not = icmp eq ptr %ctl_in, null, !dbg !95
+    %.backtrace_user.ctl_default = select i1 %tobool.not, ptr @backtrace_user.ctl_default, ptr %ctl_in, !dbg !95
+    %btc_user_copy = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    %btc_user_copy_context = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    ret i32 undef, !dbg !95
+  }
+  !llvm.module.flags = !{!88, !93}
+  !llvm.dbg.cu = !{!32}
+  !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+  !1 = distinct !DIGlobalVariable(type: !11, isDefinition: true)
+  !2 = distinct !DISubprogram(unit: !32, retainedNodes: !38)
+  !3 = !DIFile(filename: "backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !12)
+  !12 = distinct !DICompositeType(tag: DW_TAG_member, offset: 32)
+  !32 = distinct !DICompileUnit(language: DW_LANG_C11, file: !33, sdk: "MacOSX13.0.sdk")
+  !33 = !DIFile(filename: "/Users/shubham/Development/Delta/backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !38 = !{!60, !85}
+  !60 = !DILocalVariable(scope: !2, type: !62)
+  !62 = distinct !DICompositeType(tag: DW_TAG_union_type, size: 128)
+  !85 = !DILabel(scope: !2, name: "out", file: !3, line: 102)
+  !88 = !{i32 2, !"Debug Info Version", i32 3}
+  !93 = !{i32 7, !"frame-pointer", i32 1}
+  !95 = !DILocation(line: 5, column: 10, scope: !2)
+name:            backtrace_user
+registers:
+  - {id: 0, class: rgpr, }
+  - {id: 1, class: gpr, }
+  - {id: 2, class: gpr, }
+body:             |
+  bb.0.entry:
+    %10:rgpr = COPY $r2
+    %13:rgpr = t2MOV_ga_pcrel target-flags(arm-nonlazy) @backtrace_user.ctl_default, debug-location !95
+    %15:rgpr = t2MOVCCr %10, killed %13, 0 /* CC::eq */, $cpsr, debug-location !95
+    DBG_VALUE 0, $noreg, !60, !DIExpression(), debug-location !95
+    %0:rgpr = t2LDRi12 %15, 0, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+    DBG_VALUE 1, $noreg, !60, !DIExpression(), debug-location !95
+    %16:rgpr = t2LDRi12 %15, 4, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy)
+    DBG_VALUE_LIST !60, !DIExpression(), %16, %0, debug-location !95
+    %2:gpr = t2LDRi12 %15, 8, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+    DBG_VALUE %2, $noreg, !60, !DIExpression(), debug-location !95
+    %31:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
+    %32:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
+    %33:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
+    %34:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
diff --git a/llvm/test/DebugInfo/ARM/move-dbg-value-lists.mir b/llvm/test/DebugInfo/ARM/move-dbg-value-lists.mir
new file mode 100644
index 0000000000000..681d89b03936d
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/move-dbg-value-lists.mir
@@ -0,0 +1,58 @@
+# RUN: llc %s -start-after=arm-mve-vpt-opts -stop-after=arm-prera-ldst-opt -o - | FileCheck %s
+# CHECK: %5:rgpr = t2MOVCCr %3, killed %4, 0 /* CC::eq */, $cpsr, debug-location !14
+# CHECK-NEXT: DBG_VALUE 0, $noreg, !10, !DIExpression(), debug-location !14
+# CHECK-NEXT: DBG_VALUE 1, $noreg, !10, !DIExpression(), debug-location !14
+# CHECK-NEXT: DBG_VALUE_LIST !10, !DIExpression(), $noreg, $noreg, debug-location !14
+# CHECK-NEXT: %0:rgpr = t2LDRi12 %5, 0, 14 /* CC::al */, $noreg, debug-location !14 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+# CHECK-NEXT: DBG_VALUE_LIST !10, !DIExpression(), $noreg, $noreg, debug-location !14
+# CHECK-NEXT: %6:rgpr = t2LDRi12 %5, 4, 14 /* CC::al */, $noreg, debug-location !14 :: (load (s32) from %ir.btc_user_copy)
+# CHECK-NEXT: DBG_VALUE_LIST !10, !DIExpression(), %6, %0, debug-location !14
+# CHECK-NEXT: %2:gpr = t2LDRi12 %5, 8, 14 /* CC::al */, $noreg, debug-location !14 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+# CHECK-NEXT: %7:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+# CHECK-NEXT: DBG_VALUE %2, $noreg, !10, !DIExpression(), debug-location !14
+--- |
+  target triple = "thumbv7k-apple-watchos8.0.0"
+  %struct.backtrace_control = type {}
+  @backtrace_user.ctl_default = internal unnamed_addr constant %struct.backtrace_control zeroinitializer, !dbg !0
+  define i32 @backtrace_user(ptr nocapture noundef writeonly %bt, i32 noundef %max_frames, ptr noundef readonly %ctl_in, ptr nocapture noundef readnone %info_out) local_unnamed_addr #0 !dbg !2 {entry:
+    %tobool.not = icmp eq ptr %ctl_in, null, !dbg !95
+    %.backtrace_user.ctl_default = select i1 %tobool.not, ptr @backtrace_user.ctl_default, ptr %ctl_in, !dbg !95
+    %btc_user_copy = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    %btc_user_copy_context = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    ret i32 undef, !dbg !95
+  }
+  !llvm.module.flags = !{!88, !93}
+  !llvm.dbg.cu = !{!32}
+  !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+  !1 = distinct !DIGlobalVariable(type: !11, isDefinition: true)
+  !2 = distinct !DISubprogram(unit: !32, retainedNodes: !38)
+  !3 = !DIFile(filename: "backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !12)
+  !12 = distinct !DICompositeType(tag: DW_TAG_member, offset: 32)
+  !32 = distinct !DICompileUnit(language: DW_LANG_C11, file: !33, sdk: "MacOSX13.0.sdk")
+  !33 = !DIFile(filename: "/Users/shubham/Development/Delta/backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !38 = !{!60, !85}
+  !60 = !DILocalVariable(scope: !2, type: !62)
+  !62 = distinct !DICompositeType(tag: DW_TAG_union_type, size: 64)
+  !85 = !DILabel(scope: !2, name: "out", file: !3, line: 102)
+  !88 = !{i32 2, !"Debug Info Version", i32 3}
+  !93 = !{i32 7, !"frame-pointer", i32 1}
+  !95 = !DILocation(line: 5, column: 10, scope: !2)
+name:            backtrace_user
+registers:
+  - {id: 0, class: rgpr, }
+  - {id: 1, class: gpr, }
+  - {id: 2, class: gpr, }
+body:             |
+  bb.0.entry:
+    %10:rgpr = COPY $r2
+    %13:rgpr = t2MOV_ga_pcrel target-flags(arm-nonlazy) @backtrace_user.ctl_default, debug-location !95
+    %15:rgpr = t2MOVCCr %10, killed %13, 0 /* CC::eq */, $cpsr, debug-location !95
+    DBG_VALUE 0, $noreg, !60, !DIExpression(), debug-location !95
+    %0:rgpr = t2LDRi12 %15, 0, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+    DBG_VALUE 1, $noreg, !60, !DIExpression(), debug-location !95
+    %16:rgpr = t2LDRi12 %15, 4, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy)
+    DBG_VALUE_LIST !60, !DIExpression(), %16, %0, debug-location !95
+    %2:gpr = t2LDRi12 %15, 8, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+    %31:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
+    DBG_VALUE %2, $noreg, !60, !DIExpression(), debug-location !95
diff --git a/llvm/test/DebugInfo/ARM/move-dbg-value-same-reg.mir b/llvm/test/DebugInfo/ARM/move-dbg-value-same-reg.mir
new file mode 100644
index 0000000000000..7a5749d8a2495
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/move-dbg-value-same-reg.mir
@@ -0,0 +1,64 @@
+# RUN: llc %s -start-after=arm-mve-vpt-opts -stop-after=arm-prera-ldst-opt -o - | FileCheck %s
+# CHECK: %5:rgpr = t2MOVCCr %3, killed %4, 0 /* CC::eq */, $cpsr, debug-location !{{[0-9]+}}
+# CHECK-NEXT: DBG_VALUE 0, $noreg, ![[DBG:[0-9]+]], !DIExpression(), debug-location !{{[0-9]+}}
+# CHECK-NEXT: DBG_VALUE $noreg, $noreg, ![[DBG]], !DIExpression(), debug-location !{{[0-9]+}}
+# CHECK-NEXT: DBG_VALUE $noreg, $noreg, !{{[0-9]+}}, !DIExpression(), debug-location !{{[0-9]+}}
+# CHECK-NEXT: %0:rgpr = t2LDRi12 %5, 0, 14 /* CC::al */, $noreg, debug-location !{{[0-9]+}} :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+# CHECK-NEXT: DBG_VALUE %0, $noreg, ![[DBG]], !DIExpression(), debug-location !{{[0-9]+}}
+# CHECK-NEXT: DBG_VALUE %0, $noreg, !{{[0-9]+}}, !DIExpression(), debug-location !{{[0-9]+}}
+# CHECK-NEXT: %6:rgpr = t2LDRi12 %5, 4, 14 /* CC::al */, $noreg, debug-location !{{[0-9]+}} :: (load (s32) from %ir.btc_user_copy)
+# CHECK-NEXT: %2:gpr = t2LDRi12 %5, 8, 14 /* CC::al */, $noreg, debug-location !{{[0-9]+}} :: (load (s32) from %ir.btc_user_copy_context, align 8)
+# CHECK-NEXT: %7:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+# CHECK-NEXT: DBG_VALUE %2, $noreg, ![[DBG]], !DIExpression(), debug-location !{{[0-9]+}}
+
+# This test checks the case where two DBG_VALUEs share the same virtual register but describe different local variables.
+
+--- |
+  target triple = "thumbv7k-apple-watchos8.0.0"
+  %struct.backtrace_control = type {}
+  @backtrace_user.ctl_default = internal unnamed_addr constant %struct.backtrace_control zeroinitializer, !dbg !0
+  define i32 @backtrace_user(ptr nocapture noundef writeonly %bt, i32 noundef %max_frames, ptr noundef readonly %ctl_in, ptr nocapture noundef readnone %info_out) local_unnamed_addr #0 !dbg !2 {entry:
+    %tobool.not = icmp eq ptr %ctl_in, null, !dbg !95
+    %.backtrace_user.ctl_default = select i1 %tobool.not, ptr @backtrace_user.ctl_default, ptr %ctl_in, !dbg !95
+    %btc_user_copy = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    %btc_user_copy_context = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    ret i32 undef, !dbg !95
+  }
+  !llvm.module.flags = !{!88, !93}
+  !llvm.dbg.cu = !{!32}
+  !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+  !1 = distinct !DIGlobalVariable(type: !11, isDefinition: true)
+  !2 = distinct !DISubprogram(unit: !32, retainedNodes: !38)
+  !3 = !DIFile(filename: "backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !12)
+  !12 = distinct !DICompositeType(tag: DW_TAG_member, offset: 32)
+  !16 = !DIDerivedType(tag: DW_TAG_typedef, baseType: !21)
+  !21 = !DIBasicType()
+  !32 = distinct !DICompileUnit(language: DW_LANG_C11, file: !33, sdk: "MacOSX13.0.sdk")
+  !33 = !DIFile(filename: "/Users/shubham/Development/Delta/backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !38 = !{!59, !60, !85}
+  !59 = !DILocalVariable(scope: !2, type: !16)
+  !60 = !DILocalVariable(scope: !2, type: !62)
+  !62 = distinct !DICompositeType(tag: DW_TAG_union_type, size: 128)
+  !85 = !DILabel(scope: !2, name: "out", file: !3, line: 102)
+  !88 = !{i32 2, !"Debug Info Version", i32 3}
+  !93 = !{i32 7, !"frame-pointer", i32 1}
+  !95 = !DILocation(line: 5, column: 10, scope: !2)
+name:            backtrace_user
+registers:
+  - {id: 0, class: rgpr, }
+  - {id: 1, class: gpr, }
+  - {id: 2, class: gpr, }
+body:             |
+  bb.0.entry:
+    %10:rgpr = COPY $r2
+    %13:rgpr = t2MOV_ga_pcrel target-flags(arm-nonlazy) @backtrace_user.ctl_default, debug-location !95
+    %15:rgpr = t2MOVCCr %10, killed %13, 0 /* CC::eq */, $cpsr, debug-location !95
+    DBG_VALUE 0, $noreg, !60, !DIExpression(), debug-location !95
+    %0:rgpr = t2LDRi12 %15, 0, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+    DBG_VALUE %0, $noreg, !60, !DIExpression(), debug-location !95
+    DBG_VALUE %0, $noreg, !59, !DIExpression(), debug-location !95
+    %16:rgpr = t2LDRi12 %15, 4, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy)
+    %2:gpr = t2LDRi12 %15, 8, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+    %31:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
+    DBG_VALUE %2, $noreg, !60, !DIExpression(), debug-location !95
diff --git a/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir b/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir
new file mode 100644
index 0000000000000..a6bc006c1fe42
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir
@@ -0,0 +1,125 @@
+# RUN: llc -O2 %s -start-after=arm-mve-vpt-opts -stop-after=arm-prera-ldst-opt -o -  | FileCheck %s
+# CHECK: DBG_VALUE 13, $noreg, !11, !DIExpression(), debug-location !13
+
+
+# This test checks to make sure that a DBG_VALUE that uses an immediate doesn't get zero-ed out by a DBG_VALUE that comes later in the basic block, and describes the same variable.
+
+--- |
+  ; ModuleID = '/Users/shubham/Development/llvm-project/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll'
+  source_filename = "/Users/shubham/Development/llvm-project/llvm/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "armv8-unknown-linux"
+  
+  ; Function Attrs: uwtable
+  define i32 @main() !dbg !6 {
+  entry:
+    %c = alloca i32, align 4
+    tail call void @llvm.dbg.value(metadata i32 13, metadata !11, metadata !DIExpression()), !dbg !13
+    store volatile i32 13, ptr %c, align 4, !dbg !14
+    %call = tail call i32 @_Z4funcv(), !dbg !15
+    tail call void @llvm.dbg.value(metadata i32 %call, metadata !11, metadata !DIExpression()), !dbg !13
+    store volatile i32 %call, ptr %c, align 4, !dbg !15
+    tail call void @llvm.dbg.value(metadata ptr %c, metadata !11, metadata !DIExpression(DW_OP_deref)), !dbg !13
+    %c.0.c.0. = load volatile i32, ptr %c, align 4, !dbg !16
+    ret i32 %c.0.c.0., !dbg !16
+  }
+  
+  declare i32 @_Z4funcv()
+  
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+  
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4}
+  !llvm.ident = !{!5}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.6.0 (trunk 223522)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+  !1 = !DIFile(filename: "test.cpp", directory: "/home/kromanova/ngh/ToT_latest/llvm/test/DebugInfo")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 2}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{!"clang version 3.6.0 (trunk 223522)"}
+  !6 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !7, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !10)
+  !7 = !DISubroutineType(types: !8)
+  !8 = !{!9}
+  !9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !10 = !{!11}
+  !11 = !DILocalVariable(name: "c", scope: !6, file: !1, line: 5, type: !12)
+  !12 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !9)
+  !13 = !DILocation(line: 5, column: 16, scope: !6)
+  !14 = !DILocation(line: 5, column: 3, scope: !6)
+  !15 = !DILocation(line: 6, column: 7, scope: !6)
+  !16 = !DILocation(line: 7, column: 3, scope: !6)
+
+...
+---
+name:            main
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+callsEHReturn:   false
+callsUnwindInit: false
+hasEHCatchret:   false
+hasEHScopes:     false
+hasEHFunclets:   false
+isOutlined:      false
+debugInstrRef:   false
+failsVerification: false
+tracksDebugUserValues: false
+registers:
+  - { id: 0, class: gpr, preferred-register: '' }
+  - { id: 1, class: gpr, preferred-register: '' }
+  - { id: 2, class: gpr, preferred-register: '' }
+liveins:         []
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    true
+  hasCalls:        true
+  stackProtector:  ''
+  functionContext: ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  hasTailCall:     false
+  localFrameSize:  4
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: c, type: default, offset: 0, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true, 
+      local-offset: -4, debug-info-variable: '', debug-info-expression: '', 
+      debug-info-location: '' }
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    DBG_VALUE 13, $noreg, !11, !DIExpression(), debug-location !13
+    %0:gpr = MOVi 13, 14 /* CC::al */, $noreg, $noreg
+    STRi12 killed %0, %stack.0.c, 0, 14 /* CC::al */, $noreg, debug-location !14 :: (volatile store (s32) into %ir.c)
+    ADJCALLSTACKDOWN 0, 0, 14 /* CC::al */, $noreg, implicit-def dead $sp, implicit $sp, debug-location !15
+    BL @_Z4funcv, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $r0, debug-location !15
+    ADJCALLSTACKUP 0, -1, 14 /* CC::al */, $noreg, implicit-def dead $sp, implicit $sp, debug-location !15
+    %1:gpr = COPY $r0, debug-location !15
+    DBG_VALUE %1, $noreg, !11, !DIExpression(), debug-location !13
+    STRi12 %1, %stack.0.c, 0, 14 /* CC::al */, $noreg, debug-location !15 :: (volatile store (s32) into %ir.c)
+    DBG_VALUE %stack.0.c, $noreg, !11, !DIExpression(DW_OP_deref), debug-location !13
+    %2:gpr = LDRi12 %stack.0.c, 0, 14 /* CC::al */, $noreg, debug-location !16 :: (volatile dereferenceable load (s32) from %ir.c)
+    $r0 = COPY %2, debug-location !16
+    BX_RET 14 /* CC::al */, $noreg, implicit $r0, debug-location !16
+
+...
diff --git a/llvm/test/DebugInfo/ARM/move-dbg-values.mir b/llvm/test/DebugInfo/ARM/move-dbg-values.mir
new file mode 100644
index 0000000000000..1d2f07d8b8d6b
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/move-dbg-values.mir
@@ -0,0 +1,60 @@
+# RUN: llc %s -start-after=arm-mve-vpt-opts -stop-after=arm-prera-ldst-opt -o - | FileCheck %s
+# CHECK: %5:rgpr = t2MOVCCr %3, killed %4, 0 /* CC::eq */, $cpsr, debug-location !15
+# CHECK-NEXT: DBG_VALUE 0, $noreg, !10, !DIExpression(), debug-location !15
+# CHECK-NEXT: DBG_VALUE $noreg, $noreg, !10, !DIExpression(), debug-location !15
+# CHECK-NEXT: DBG_VALUE $noreg, $noreg, !10, !DIExpression(), debug-location !15
+# CHECK-NEXT: %0:rgpr = t2LDRi12 %5, 0, 14 /* CC::al */, $noreg, debug-location !15 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+# CHECK-NEXT: %6:rgpr = t2LDRi12 %5, 4, 14 /* CC::al */, $noreg, debug-location !15 :: (load (s32) from %ir.btc_user_copy)
+# CHECK-NEXT: DBG_VALUE %6, $noreg, !10, !DIExpression(), debug-location !15
+# CHECK-NEXT: %2:gpr = t2LDRi12 %5, 8, 14 /* CC::al */, $noreg, debug-location !15 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+# CHECK-NEXT: %7:rgpr = t2ANDrr %6, %0, 14 /* CC::al */, $noreg, def $cpsr
+# CHECK-NEXT: DBG_VALUE %2, $noreg, !10, !DIExpression(), debug-location !15
+--- |
+  target triple = "thumbv7k-apple-watchos8.0.0"
+  %struct.backtrace_control = type {}
+  @backtrace_user.ctl_default = internal unnamed_addr constant %struct.backtrace_control zeroinitializer, !dbg !0
+  define i32 @backtrace_user(ptr nocapture noundef writeonly %bt, i32 noundef %max_frames, ptr noundef readonly %ctl_in, ptr nocapture noundef readnone %info_out) local_unnamed_addr #0 !dbg !2 {entry:
+    %tobool.not = icmp eq ptr %ctl_in, null, !dbg !95
+    %.backtrace_user.ctl_default = select i1 %tobool.not, ptr @backtrace_user.ctl_default, ptr %ctl_in, !dbg !95
+    %btc_user_copy = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    %btc_user_copy_context = getelementptr inbounds %struct.backtrace_control, ptr %.backtrace_user.ctl_default, !dbg !95
+    ret i32 undef, !dbg !95
+  }
+  !llvm.module.flags = !{!88,
+   !93}
+  !llvm.dbg.cu = !{!32}
+  !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+  !1 = distinct !DIGlobalVariable(type: !11, isDefinition: true)
+  !2 = distinct !DISubprogram(unit: !32, retainedNodes: !38)
+  !3 = !DIFile(filename: "backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !8 = !DIDerivedType(tag: DW_TAG_typedef, baseType: !11, size: 32)
+  !11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !12)
+  !12 = distinct !DICompositeType(tag: DW_TAG_structure_type, size: 32)
+  !22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !31, size: 32)
+  !31 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagFwdDecl)
+  !32 = distinct !DICompileUnit(language: DW_LANG_C11, file: !33, sdk: "MacOSX13.0.sdk")
+  !33 = !DIFile(filename: "/Users/shubham/Development/Delta/backtrace.pp.c", directory: "/Users/shubham/Development/Delta")
+  !38 = !{!60, !85}
+  !60 = !DILocalVariable(scope: !2, type: !22)
+  !85 = !DILabel(scope: !2, name: "out", file: !3, line: 102)
+  !88 = !{i32 2, !"Debug Info Version", i32 3}
+  !93 = !{i32 7, !"frame-pointer", i32 1}
+  !95 = !DILocation(line: 5, column: 10, scope: !2)
+name:            backtrace_user
+registers:
+  - {id: 0, class: rgpr, }
+  - {id: 1, class: gpr, }
+  - {id: 2, class: gpr, }
+body:             |
+  bb.0.entry:
+    %10:rgpr = COPY $r2
+    %13:rgpr = t2MOV_ga_pcrel target-flags(arm-nonlazy) @backtrace_user.ctl_default, debug-location !95
+    %15:rgpr = t2MOVCCr %10, killed %13, 0 /* CC::eq */, $cpsr, debug-location !95
+    DBG_VALUE 0, $noreg, !60, !DIExpression(), debug-location !95
+    %0:rgpr = t2LDRi12 %15, 0, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir..backtrace_user.ctl_default, align 8)
+    DBG_VALUE %0, $noreg, !60, !DIExpression(), debug-location !95
+    %16:rgpr = t2LDRi12 %15, 4, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy)
+    DBG_VALUE %16, $noreg, !60, !DIExpression(), debug-location !95
+    %2:gpr = t2LDRi12 %15, 8, 14 /* CC::al */, $noreg, debug-location !95 :: (load (s32) from %ir.btc_user_copy_context, align 8)
+    %31:rgpr = t2ANDrr %16, %0, 14 /* CC::al */, $noreg, def $cpsr
+    DBG_VALUE %2, $noreg, !60, !DIExpression(), debug-location !95
diff --git a/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll b/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll
index d8b3d35f87cb7..5bd19a03e745c 100644
--- a/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-realign-alloca.ll
@@ -1,4 +1,5 @@
 ; RUN: llc < %s | FileCheck %s
+; RUN: llc -mtriple=i686-windows-msvc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; C source:
 ; void usethings(ptr, ptr p);
@@ -33,6 +34,15 @@
 ; CHECK:         retl
 ; CHECK:         .cv_fpo_endproc
 
+; OBJ-LABEL:     FunctionType: realign_and_alloca (0x1002)
+; OBJ-NEXT:      CodeOffset: _realign_and_alloca+0x0
+; OBJ-NEXT:      Segment: 0x0
+; OBJ-NEXT:      Flags [ (0x81)
+; OBJ-NEXT:        HasFP (0x1)
+; OBJ-NEXT:        HasOptimizedDebugInfo (0x80)
+; OBJ-NEXT:      ]
+; OBJ-NEXT:      DisplayName: realign_and_alloca
+; OBJ-NEXT:      LinkageName: _realign_and_alloca
 
 ; ModuleID = 't.c'
 source_filename = "t.c"
diff --git a/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll b/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
index 0a032a8a5bfa5..5f1d6a5c28def 100644
--- a/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
+++ b/llvm/test/DebugInfo/COFF/fpo-realign-vframe.ll
@@ -95,6 +95,10 @@
 ; OBJ:   SubSectionType: Symbols (0xF1)
 ; OBJ:   GlobalProcIdSym {
 ; OBJ:     Kind: S_GPROC32_ID (0x1147)
+; OBJ:     Flags [ (0x81)
+; OBJ:       HasFP (0x1)
+; OBJ:       HasOptimizedDebugInfo (0x80)
+; OBJ:     ]
 ; OBJ:     DisplayName: realign_with_csrs
 ; OBJ:     LinkageName: _realign_with_csrs
 ; OBJ:   }
diff --git a/llvm/test/DebugInfo/COFF/frameproc-flags.ll b/llvm/test/DebugInfo/COFF/frameproc-flags.ll
index 9f571b8e70e33..905418969b6a7 100644
--- a/llvm/test/DebugInfo/COFF/frameproc-flags.ll
+++ b/llvm/test/DebugInfo/COFF/frameproc-flags.ll
@@ -63,22 +63,27 @@
 ; }
 
 ; CHECK-LABEL: S_GPROC32_ID [size = 52] `use_alloca`
+; CHECK:   type = `0x1002 (use_alloca)`, debug start = 0, debug end = 0, flags = has fp | opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = VFRAME, param fp reg = EBP
 ; CHECK:   flags = has alloca | secure checks | strict secure checks | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 52] `call_setjmp`
+; CHECK:   type = `0x1003 (call_setjmp)`, debug start = 0, debug end = 0, flags = opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = NONE, param fp reg = NONE
 ; CHECK:   flags = has setjmp | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 56] `use_inlineasm`
+; CHECK:   type = `0x1006 (use_inlineasm)`, debug start = 0, debug end = 0, flags = opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = NONE, param fp reg = NONE
 ; CHECK:   flags = has inline asm | safe buffers | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 48] `cpp_eh`
+; CHECK:   type = `0x1007 (cpp_eh)`, debug start = 0, debug end = 0, flags = has fp | opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = EBP, param fp reg = EBP
 ; CHECK:   flags = has eh | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 52] `use_inline`
+; CHECK:   type = `0x100C (use_inline)`, debug start = 0, debug end = 0, flags = opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = NONE, param fp reg = NONE
 ; CHECK:   flags = safe buffers | opt speed
@@ -87,6 +92,7 @@
 ; CHECK:   local fp reg = NONE, param fp reg = NONE
 ; CHECK:   flags = marked inline | safe buffers | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 44] `seh`
+; CHECK:   type = `0x100E (seh)`, debug start = 0, debug end = 0, flags = has fp | opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = EBP, param fp reg = EBP
 ; CHECK:   flags = has seh | opt speed
@@ -95,10 +101,12 @@
 ; CHECK:   local fp reg = EBP, param fp reg = EBP
 ; CHECK:   flags = safe buffers | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 52] `use_naked`
+; CHECK:   type = `0x1010 (use_naked)`, debug start = 0, debug end = 0, flags = noinline | opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = NONE, param fp reg = NONE
 ; CHECK:   flags = has inline asm | naked | safe buffers | opt speed
 ; CHECK-LABEL: S_GPROC32_ID [size = 52] `stack_guard`
+; CHECK:   type = `0x1011 (stack_guard)`, debug start = 0, debug end = 0, flags = opt debuginfo
 ; CHECK: S_FRAMEPROC [size = 32]
 ; CHECK:   local fp reg = VFRAME, param fp reg = VFRAME
 ; CHECK:   flags = secure checks | strict secure checks | opt speed
diff --git a/llvm/test/DebugInfo/COFF/function-options.ll b/llvm/test/DebugInfo/COFF/function-options.ll
index 835f3fbe3c8ae..863994be1d5ef 100644
--- a/llvm/test/DebugInfo/COFF/function-options.ll
+++ b/llvm/test/DebugInfo/COFF/function-options.ll
@@ -376,6 +376,97 @@
 ; CHECK:   }
 ; CHECK: ]
 
+; CHECK-LABEL: FunctionType: Func_AClass (0x1008)
+; CHECK-NEXT:  CodeOffset: ?Func_AClass@@YA?AVAClass@@AEAV1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_AClass
+; CHECK-NEXT:  LinkageName: ?Func_AClass@@YA?AVAClass@@AEAV1@@Z
+; CHECK-LABEL: FunctionType: Func_BClass (0x1013)
+; CHECK-NEXT:  CodeOffset: ?Func_BClass@@YA?AVBClass@@AEAV1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_BClass
+; CHECK-NEXT:  LinkageName: ?Func_BClass@@YA?AVBClass@@AEAV1@@Z
+; CHECK-LABEL: FunctionType: Func_C1Class (0x101D)
+; CHECK-NEXT:  CodeOffset: ?Func_C1Class@@YA?AVC1Class@@AEAV1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_C1Class
+; CHECK-NEXT:  LinkageName: ?Func_C1Class@@YA?AVC1Class@@AEAV1@@Z
+; CHECK-LABEL: FunctionType: Func_C2Class (0x1027)
+; CHECK-NEXT:  CodeOffset: ?Func_C2Class@@YA?AVC2Class@@AEAV1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_C2Class
+; CHECK-NEXT:  LinkageName: ?Func_C2Class@@YA?AVC2Class@@AEAV1@@Z
+; CHECK-LABEL: FunctionType: Func_DClass (0x102F)
+; CHECK-NEXT:  CodeOffset: ?Func_DClass@@YA?AVDClass@@AEAV1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_DClass
+; CHECK-NEXT:  LinkageName: ?Func_DClass@@YA?AVDClass@@AEAV1@@Z
+; CHECK-LABEL: FunctionType: Func_FClass (0x103A)
+; CHECK-NEXT:  CodeOffset: ?Func_FClass@@YA?AVFClass@@AEAV1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_FClass
+; CHECK-NEXT:  LinkageName: ?Func_FClass@@YA?AVFClass@@AEAV1@@Z
+; CHECK-LABEL: FunctionType: Func_AStruct (0x1041)
+; CHECK-NEXT:  CodeOffset: ?Func_AStruct@@YA?AUAStruct@@AEAU1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_AStruct
+; CHECK-NEXT:  LinkageName: ?Func_AStruct@@YA?AUAStruct@@AEAU1@@Z
+; CHECK-LABEL: FunctionType: Func_BStruct (0x104B)
+; CHECK-NEXT:  CodeOffset: ?Func_BStruct@@YA?AUBStruct@@AEAU1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_BStruct
+; CHECK-NEXT:  LinkageName: ?Func_BStruct@@YA?AUBStruct@@AEAU1@@Z
+; CHECK-LABEL: FunctionType: Func_AUnion (0x1052)
+; CHECK-NEXT:  CodeOffset: ?Func_AUnion@@YA?ATAUnion@@AEAT1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_AUnion
+; CHECK-NEXT:  LinkageName: ?Func_AUnion@@YA?ATAUnion@@AEAT1@@Z
+; CHECK-LABEL: FunctionType: Func_BUnion (0x105C)
+; CHECK-NEXT:  CodeOffset: ?Func_BUnion@@YA?ATBUnion@@AEAT1@@Z+0x0
+; CHECK-NEXT:  Segment: 0x0
+; CHECK-NEXT:  Flags [ (0xC0)
+; CHECK-NEXT:    HasOptimizedDebugInfo (0x80)
+; CHECK-NEXT:    IsNoInline (0x40)
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  DisplayName: Func_BUnion
+; CHECK-NEXT:  LinkageName: ?Func_BUnion@@YA?ATBUnion@@AEAT1@@Z
+
 ; ModuleID = 't.cpp'
 source_filename = "t.cpp"
 target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/DebugInfo/COFF/inlining-header.ll b/llvm/test/DebugInfo/COFF/inlining-header.ll
index bead300d6988d..0e990cbb0fcca 100644
--- a/llvm/test/DebugInfo/COFF/inlining-header.ll
+++ b/llvm/test/DebugInfo/COFF/inlining-header.ll
@@ -68,7 +68,8 @@
 ; OBJ:     FunctionType: main (0x1005)
 ; OBJ:     CodeOffset: _main+0x0
 ; OBJ:     Segment: 0x0
-; OBJ:     Flags [ (0x0)
+; OBJ:     Flags [ (0x80)
+; OBJ:       HasOptimizedDebugInfo (0x80)
 ; OBJ:     ]
 ; OBJ:     DisplayName: main
 ; OBJ:     LinkageName: _main
diff --git a/llvm/test/DebugInfo/COFF/inlining.ll b/llvm/test/DebugInfo/COFF/inlining.ll
index de0d78936a0dc..d0b03cc5b21e3 100644
--- a/llvm/test/DebugInfo/COFF/inlining.ll
+++ b/llvm/test/DebugInfo/COFF/inlining.ll
@@ -160,7 +160,8 @@
 ; OBJ:     FunctionType: baz (0x1004)
 ; OBJ:     CodeOffset: ?baz@@YAXXZ+0x0
 ; OBJ:     Segment: 0x0
-; OBJ:     Flags [ (0x0)
+; OBJ:     Flags [ (0x80)
+; OBJ:       HasOptimizedDebugInfo (0x80)
 ; OBJ:     ]
 ; OBJ:     DisplayName: baz
 ; OBJ:     LinkageName: ?baz@@YAXXZ
diff --git a/llvm/test/DebugInfo/COFF/long-name.ll b/llvm/test/DebugInfo/COFF/long-name.ll
index 073bd7909cb5e..4ea5b4328d246 100644
--- a/llvm/test/DebugInfo/COFF/long-name.ll
+++ b/llvm/test/DebugInfo/COFF/long-name.ll
@@ -5,7 +5,8 @@
 ; CHECK:   FunctionType: {{A+}} (0x1002)
 ; CHECK:   CodeOffset: f+0x0
 ; CHECK:   Segment: 0x0
-; CHECK:   Flags [ (0x0)
+; CHECK:   Flags [ (0x80)
+; CHECK:     HasOptimizedDebugInfo (0x80)
 ; CHECK:   ]
 ; CHECK:   DisplayName: {{A+$}}
 ; CHECK:   LinkageName: f
diff --git a/llvm/test/DebugInfo/COFF/multifunction.ll b/llvm/test/DebugInfo/COFF/multifunction.ll
index 98b0e4b2d8a05..04c643602f09e 100644
--- a/llvm/test/DebugInfo/COFF/multifunction.ll
+++ b/llvm/test/DebugInfo/COFF/multifunction.ll
@@ -78,7 +78,7 @@
 ; X86-NEXT: .long   4098
 ; X86-NEXT: .secrel32 _x
 ; X86-NEXT: .secidx _x
-; X86-NEXT: .byte   0
+; X86-NEXT: .byte   128
 ; X86-NEXT: .asciz "x"
 ; X86-NEXT: .p2align 2
 ; X86-NEXT: [[PROC_SEGMENT_END]]:
@@ -117,7 +117,7 @@
 ; X86-NEXT: .long   4099
 ; X86-NEXT: .secrel32 _y
 ; X86-NEXT: .secidx _y
-; X86-NEXT: .byte   0
+; X86-NEXT: .byte   128
 ; X86-NEXT: .asciz "y"
 ; X86-NEXT: .p2align 2
 ; X86-NEXT: [[PROC_SEGMENT_END]]:
@@ -156,7 +156,7 @@
 ; X86-NEXT: .long   4100
 ; X86-NEXT: .secrel32 _f
 ; X86-NEXT: .secidx _f
-; X86-NEXT: .byte   0
+; X86-NEXT: .byte   128
 ; X86-NEXT: .asciz "f"
 ; X86-NEXT: .p2align 2
 ; X86-NEXT: [[PROC_SEGMENT_END]]:
@@ -390,7 +390,7 @@
 ; X64-NEXT: .long   4098
 ; X64-NEXT: .secrel32 x
 ; X64-NEXT: .secidx x
-; X64-NEXT: .byte   0
+; X64-NEXT: .byte   128
 ; X64-NEXT: .asciz "x"
 ; X64-NEXT: .p2align 2
 ; X64-NEXT: [[PROC_SEGMENT_END]]:
@@ -428,7 +428,7 @@
 ; X64-NEXT: .long   4099
 ; X64-NEXT: .secrel32 y
 ; X64-NEXT: .secidx y
-; X64-NEXT: .byte   0
+; X64-NEXT: .byte   128
 ; X64-NEXT: .asciz "y"
 ; X64-NEXT: .p2align 2
 ; X64-NEXT: [[PROC_SEGMENT_END]]:
@@ -466,7 +466,7 @@
 ; X64-NEXT: .long   4100
 ; X64-NEXT: .secrel32 f
 ; X64-NEXT: .secidx f
-; X64-NEXT: .byte   0
+; X64-NEXT: .byte   128
 ; X64-NEXT: .asciz "f"
 ; X64-NEXT: .p2align 2
 ; X64-NEXT: [[PROC_SEGMENT_END]]:
diff --git a/llvm/test/DebugInfo/COFF/simple.ll b/llvm/test/DebugInfo/COFF/simple.ll
index 85819b0d990ee..f81de98b50036 100644
--- a/llvm/test/DebugInfo/COFF/simple.ll
+++ b/llvm/test/DebugInfo/COFF/simple.ll
@@ -58,7 +58,7 @@
 ; X86-NEXT: .long   4098
 ; X86-NEXT: .secrel32 _f
 ; X86-NEXT: .secidx _f
-; X86-NEXT: .byte   0
+; X86-NEXT: .byte   128
 ; X86-NEXT: .asciz "f"
 ; X86-NEXT: .p2align 2
 ; X86-NEXT: [[PROC_SEGMENT_END]]:
@@ -188,7 +188,7 @@
 ; X64-NEXT: .long   4098
 ; X64-NEXT: .secrel32 f
 ; X64-NEXT: .secidx f
-; X64-NEXT: .byte   0
+; X64-NEXT: .byte   128
 ; X64-NEXT: .asciz "f"
 ; X64-NEXT: .p2align 2
 ; X64-NEXT: [[PROC_SEGMENT_END]]:
diff --git a/llvm/test/DebugInfo/COFF/types-array.ll b/llvm/test/DebugInfo/COFF/types-array.ll
index 07828c3406fec..5dec93d1fdbba 100644
--- a/llvm/test/DebugInfo/COFF/types-array.ll
+++ b/llvm/test/DebugInfo/COFF/types-array.ll
@@ -58,7 +58,9 @@
 ; CHECK:       FunctionType: f (0x1002)
 ; CHECK:       CodeOffset: ?f@@YAXXZ+0x0
 ; CHECK:       Segment: 0x0
-; CHECK:       Flags [ (0x0)
+; CHECK:       Flags [ (0x81)
+; CHECK:         HasFP (0x1)
+; CHECK:         HasOptimizedDebugInfo (0x80)
 ; CHECK:       ]
 ; CHECK:       DisplayName: f
 ; CHECK:       LinkageName: ?f@@YAXXZ
diff --git a/llvm/test/DebugInfo/COFF/types-basic.ll b/llvm/test/DebugInfo/COFF/types-basic.ll
index b33e8d338e574..e87f6f6864221 100644
--- a/llvm/test/DebugInfo/COFF/types-basic.ll
+++ b/llvm/test/DebugInfo/COFF/types-basic.ll
@@ -221,7 +221,8 @@
 ; CHECK:       FunctionType: f (0x1002)
 ; CHECK:       CodeOffset: ?f@@YAXMN_J@Z+0x0
 ; CHECK:       Segment: 0x0
-; CHECK:       Flags [ (0x0)
+; CHECK:       Flags [ (0x80)
+; CHECK:         HasOptimizedDebugInfo (0x80)
 ; CHECK:       ]
 ; CHECK:       DisplayName: f
 ; CHECK:       LinkageName: ?f@@YAXMN_J@Z
diff --git a/llvm/test/DebugInfo/Generic/empty-metadata.ll b/llvm/test/DebugInfo/Generic/empty-metadata.ll
new file mode 100644
index 0000000000000..24af10c8e779b
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/empty-metadata.ll
@@ -0,0 +1,37 @@
+; RUN: opt -passes=dce -S %s -o - | FileCheck %s
+
+;; Check that DCE doesn't remove a dbg intrinsic that has an empty metadata
+;; operand.
+; CHECK: call void @llvm.dbg.declare(metadata ![[EMPTY:[0-9]+]],
+; CHECK: ![[EMPTY]] = !{}
+
+define dso_local void @fun() local_unnamed_addr #0 !dbg !9 {
+entry:
+  call void @llvm.dbg.declare(metadata !{}, metadata !13, metadata !DIExpression()), !dbg !15
+  ret void, !dbg !16
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 16.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"PIE Level", i32 2}
+!7 = !{i32 7, !"uwtable", i32 2}
+!8 = !{!"clang version 16.0.0"}
+!9 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!10 = !DISubroutineType(types: !11)
+!11 = !{null}
+!12 = !{!13}
+!13 = !DILocalVariable(name: "a", scope: !9, file: !1, line: 1, type: !14)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DILocation(line: 1, column: 18, scope: !9)
+!16 = !DILocation(line: 1, column: 21, scope: !9)
+
diff --git a/llvm/test/DebugInfo/Generic/empty.ll b/llvm/test/DebugInfo/Generic/empty.ll
index 8de50ff23bb5b..d9de71039702f 100644
--- a/llvm/test/DebugInfo/Generic/empty.ll
+++ b/llvm/test/DebugInfo/Generic/empty.ll
@@ -1,9 +1,5 @@
 ; UNSUPPORTED: target={{.*}}-aix{{.*}}
 ; RUN: %llc_dwarf < %s -filetype=obj | llvm-dwarfdump -v - | FileCheck %s
-; RUN: %llc_dwarf -split-dwarf-file=foo.dwo < %s -filetype=obj | llvm-dwarfdump -v - | FileCheck --check-prefix=FISSION %s
-
-; darwin has a workaround for a linker bug so it always emits one line table entry
-; XFAIL: target={{.*}}-darwin{{.*}}
 
 ; Expect no line table entry since there are no functions and file references in this compile unit
 ; CHECK: .debug_line contents:
@@ -14,11 +10,6 @@
 ; CHECK-NOT: .debug_pubnames contents:
 ; CHECK: contents:
 
-; Don't emit DW_AT_addr_base when there are no addresses.
-; Also don't emit a split line table when there are no type units.
-; FISSION-NOT: DW_AT_GNU_addr_base [DW_FORM_sec_offset]
-; FISSION-NOT: .debug_line.dwo contents:
-
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!5}
 
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe-subreg.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe-subreg.mir
new file mode 100644
index 0000000000000..a8060c0d07184
--- /dev/null
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe-subreg.mir
@@ -0,0 +1,66 @@
+# RUN: llc -emit-call-site-info -start-before=livedebugvalues -filetype=obj -o - %s \
+# RUN:     | llvm-dwarfdump - | FileCheck %s --implicit-check-not=DW_TAG_GNU_call_site_parameter
+#
+# Test that we don't crash on complicated copies with avx registers. For
+# complex intrinsic code, LLVM will produce register copies to 128-bit
+# registers that (through writing to them) implicitly zero out the upper
+# bit of the 256 ymm register.
+#
+# However, this is now what call-site-info tracking expects to see, as
+# demonstrated in the MIR below -- a ymm argument to a function might be
+# implicit-def'd by such a copy, but without being the 'Destination'
+# register. For this non-trivial copy, test that we presently produce
+# no parameter location. We might be able to describe it in the future.
+#
+# CHECK:      DW_TAG_GNU_call_site
+# CHECK-NEXT:    DW_AT_abstract_origin
+
+--- |
+  target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+  target triple = "x86_64-unknown-linux-gnu"
+
+  ; Function Attrs: nounwind uwtable
+  define void @foo() !dbg !12 {
+  entry:
+    ret void, !dbg !16
+  }
+
+  declare !dbg !4 void @call()
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!8, !9, !10}
+  !llvm.ident = !{!11}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "clobber.c", directory: "/")
+  !2 = !{}
+  !3 = !{!4}
+  !4 = !DISubprogram(name: "call", scope: !1, file: !1, line: 1, type: !5, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+  !5 = !DISubroutineType(types: !6)
+  !6 = !{null, !7, !7}
+  !7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !8 = !{i32 7, !"Dwarf Version", i32 4}
+  !9 = !{i32 2, !"Debug Info Version", i32 3}
+  !10 = !{i32 1, !"wchar_size", i32 4}
+  !11 = !{!"clang version 11.0.0"}
+  !12 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !13, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+  !13 = !DISubroutineType(types: !14)
+  !14 = !{null}
+  !15 = !DILocation(line: 5, scope: !12)
+  !16 = !DILocation(line: 6, scope: !12)
+
+...
+---
+name:            foo
+callSites:
+  - { bb: 0, offset: 1, fwdArgRegs:
+      - { arg: 0, reg: '$ymm0' } }
+body:             |
+  bb.0.entry:
+    liveins: $xmm1
+
+    renamable $xmm0 = VMOVAPSrr renamable $xmm1, implicit-def $ymm0
+    CALL64pcrel32 @call, csr_64, implicit $rsp, implicit $ssp, implicit killed $ymm0, implicit-def $rsp, implicit-def $ssp, debug-location !15
+    RET64 debug-location !16
+
+...
diff --git a/llvm/test/DebugInfo/MSP430/cu-ranges.ll b/llvm/test/DebugInfo/MSP430/cu-ranges.ll
index c558c24c665ff..5c15c31e2d116 100644
--- a/llvm/test/DebugInfo/MSP430/cu-ranges.ll
+++ b/llvm/test/DebugInfo/MSP430/cu-ranges.ll
@@ -2,6 +2,8 @@
 ; RUN: llvm-dwarfdump -v %t | FileCheck %s
 
 ; Ported from generic test to cover 2-byte address size case
+; Check that 16-bit addresses of MSP430 are properly stored
+; in 32-bit DWARF fields
 
 ; Check that we emit ranges for this which has a non-traditional section and a normal section.
 
@@ -15,8 +17,8 @@
 ; CHECK: DW_AT_high_pc
 
 ; CHECK: .debug_ranges contents:
-; CHECK-NEXT: 00000000 0000 0030
-; CHECK-NEXT: 00000000 0000 0030
+; CHECK-NEXT: 00000000 00000000 00000030
+; CHECK-NEXT: 00000000 00000000 00000030
 ; CHECK-NEXT: 00000000 <End of list>
 
 ; Function Attrs: nounwind uwtable
diff --git a/llvm/test/DebugInfo/MSP430/dwarf-basics-v5.ll b/llvm/test/DebugInfo/MSP430/dwarf-basics-v5.ll
index 2e61566c7993b..dc6977dec5182 100644
--- a/llvm/test/DebugInfo/MSP430/dwarf-basics-v5.ll
+++ b/llvm/test/DebugInfo/MSP430/dwarf-basics-v5.ll
@@ -4,7 +4,7 @@
 
 ; This file was based on output of
 ;
-;   clang -target msp430 -S -emit-llvm -gdwarf-5 -Os dwarf-basics-v5.c
+;   clang -target msp430-elf -S -emit-llvm -gdwarf-5 -Os dwarf-basics-v5.c
 ;
 ; for the following dwarf-basics-v5.c
 ;
@@ -21,11 +21,11 @@
 ; CHECK: file format elf32-msp430
 
 ; CHECK: .debug_info contents:
-; CHECK: Compile Unit: length = 0x{{.*}}, format = DWARF32, version = 0x0005, unit_type = DW_UT_compile, abbr_offset = 0x0000, addr_size = 0x02 (next unit at 0x{{.*}})
+; CHECK: Compile Unit: length = 0x{{.*}}, format = DWARF32, version = 0x0005, unit_type = DW_UT_compile, abbr_offset = 0x0000, addr_size = 0x04 (next unit at 0x{{.*}})
 
 ; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_AT_producer    ("clang version 14.0.0 (git@...)")
-; CHECK:   DW_AT_language    (DW_LANG_C99)
+; CHECK:   DW_AT_producer    ("clang version 17.0.0 (git@...)")
+; CHECK:   DW_AT_language    (DW_LANG_C11)
 ; CHECK:   DW_AT_name        ("dwarf-basics-v5.c")
 ; CHECK:   DW_AT_str_offsets_base    (0x00000008)
 ; CHECK:   DW_AT_stmt_list   (0x{{.*}})
@@ -49,7 +49,7 @@
 
 ; CHECK:       DW_TAG_formal_parameter
 ; CHECK:         DW_AT_location        (indexed (0x0) loclist = 0x{{.*}}:
-; CHECK:            [0x0000, 0x0004): DW_OP_reg12 R12)
+; CHECK:            [0x00000000, 0x00000004): DW_OP_reg12 R12)
 ; CHECK:         DW_AT_name    ("y")
 ; CHECK:         DW_AT_decl_file       ("/tmp{{[/\\]}}dwarf-basics-v5.c")
 ; CHECK:         DW_AT_decl_line       (5)
@@ -97,46 +97,43 @@
 ; CHECK:     NULL
 
 ; CHECK:      .debug_aranges contents:
-; CHECK-NEXT: Address Range Header: length = 0x{{.*}}, format = DWARF32, version = 0x0002, cu_offset = 0x00000000, addr_size = 0x02, seg_size = 0x00
-; CHECK-NEXT: [0x0000, 0x0006)
+; CHECK-NEXT: Address Range Header: length = 0x{{.*}}, format = DWARF32, version = 0x0002, cu_offset = 0x00000000, addr_size = 0x04, seg_size = 0x00
+; CHECK-NEXT: [0x00000000, 0x00000006)
 
 ; CHECK:      .debug_addr contents:
-; CHECK-NEXT: Address table header: length = 0x{{.*}}, format = DWARF32, version = 0x0005, addr_size = 0x02, seg_size = 0x00
+; CHECK-NEXT: Address table header: length = 0x{{.*}}, format = DWARF32, version = 0x0005, addr_size = 0x04, seg_size = 0x00
 ; CHECK-NEXT: Addrs: [
-; CHECK-NEXT: 0x0000
+; CHECK-NEXT: 0x00000000
 ; CHECK-NEXT: ]
 
 ; ModuleID = 'dwarf-basics-v5.c'
 source_filename = "dwarf-basics-v5.c"
 target datalayout = "e-m:e-p:16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
-target triple = "msp430"
+target triple = "msp430-unknown-unknown-elf"
 
-%struct.X = type { i8* }
-
-; Function Attrs: mustprogress nofree norecurse nosync nounwind optsize readnone willreturn
-define dso_local i16 @f(i32 noundef %y, %struct.X* nocapture noundef readnone %p) local_unnamed_addr #0 !dbg !6 {
-entry:
-  call void @llvm.dbg.value(metadata i32 %y, metadata !17, metadata !DIExpression()), !dbg !19
-  call void @llvm.dbg.value(metadata %struct.X* %p, metadata !18, metadata !DIExpression()), !dbg !19
+; Function Attrs: mustprogress nofree norecurse nosync nounwind optsize willreturn memory(none)
+define dso_local i16 @f(i32 noundef %0, ptr nocapture noundef readnone %1) local_unnamed_addr #0 !dbg !6 {
+  call void @llvm.dbg.value(metadata i32 %0, metadata !17, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata ptr %1, metadata !18, metadata !DIExpression()), !dbg !19
   ret i16 42, !dbg !20
 }
 
-; Function Attrs: nofree nosync nounwind readnone speculatable willreturn
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
 declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
-attributes #0 = { mustprogress nofree norecurse nosync nounwind optsize readnone willreturn "frame-pointer"="none" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
+attributes #0 = { mustprogress nofree norecurse nosync nounwind optsize willreturn memory(none) "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0 (git@...)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 17.0.0 (git@...)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "dwarf-basics-v5.c", directory: "/tmp", checksumkind: CSK_MD5, checksum: "ead340d457001e2ce340630cfa3a9cb8")
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 2}
-!5 = !{!"clang version 14.0.0 (git@...)"}
+!5 = !{!"clang version 17.0.0 (git@...)"}
 !6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 5, type: !7, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
 !7 = !DISubroutineType(types: !8)
 !8 = !{!9, !10, !11}
@@ -152,3 +149,4 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !18 = !DILocalVariable(name: "p", arg: 2, scope: !6, file: !1, line: 5, type: !11)
 !19 = !DILocation(line: 0, scope: !6)
 !20 = !DILocation(line: 7, column: 3, scope: !6)
+
diff --git a/llvm/test/DebugInfo/MSP430/dwarf-basics.ll b/llvm/test/DebugInfo/MSP430/dwarf-basics.ll
index 7d81d9c28d6f2..0dc23b5199838 100644
--- a/llvm/test/DebugInfo/MSP430/dwarf-basics.ll
+++ b/llvm/test/DebugInfo/MSP430/dwarf-basics.ll
@@ -21,7 +21,7 @@
 ; CHECK: file format elf32-msp430
 
 ; CHECK: .debug_info contents:
-; CHECK: Compile Unit: length = 0x{{.*}}, format = DWARF32, version = 0x0003, abbr_offset = 0x0000, addr_size = 0x02 (next unit at 0x{{.*}})
+; CHECK: Compile Unit: length = 0x{{.*}}, format = DWARF32, version = 0x0003, abbr_offset = 0x0000, addr_size = 0x04 (next unit at 0x{{.*}})
 
 ; CHECK: DW_TAG_compile_unit
 ; CHECK:   DW_AT_producer    ("clang version 11.0.0 (git@...)")
@@ -45,7 +45,7 @@
 
 ; CHECK:       DW_TAG_formal_parameter
 ; CHECK:         DW_AT_location        (0x{{.*}}:
-; CHECK:            [0x0000, 0x0004): DW_OP_reg12 R12)
+; CHECK:            [0x00000000, 0x00000004): DW_OP_reg12 R12)
 ; CHECK:         DW_AT_name    ("y")
 ; CHECK:         DW_AT_decl_file       ("/tmp{{[/\\]}}dwarf-basics.c")
 ; CHECK:         DW_AT_decl_line       (5)
diff --git a/llvm/test/DebugInfo/MSP430/ranges_always.ll b/llvm/test/DebugInfo/MSP430/ranges_always.ll
index f8fe03d210877..753c82b9cbd55 100644
--- a/llvm/test/DebugInfo/MSP430/ranges_always.ll
+++ b/llvm/test/DebugInfo/MSP430/ranges_always.ll
@@ -14,6 +14,8 @@
 ; RUN:     --implicit-check-not=DW_TAG --implicit-check-not=NULL --implicit-check-not=_pc %s
 
 ; Ported from X86 test to cover 2-byte address size case
+; Check that 16-bit addresses of MSP430 are properly stored
+; in 32-bit DWARF fields
 
 ; Generated from the following source. f4 is used to put a hole in the CU
 ; ranges while keeping f2 and f4 in the same section (as opposed to
@@ -51,14 +53,14 @@
 ; CHECK-LABEL: .debug_info contents:
 ; CHECK: DW_TAG_compile_unit
 ; CHECK:   DW_AT_low_pc
-; CHECK-SAME: (0x0000)
+; CHECK-SAME: (0x00000000)
 ; RNG:     DW_AT_ranges
 ; RNG-SAME:  (indexed (0x3) rangelist = [[CU_RANGE:.*]]
 ; EXPRORFORM: DW_AT_ranges
 ; EXPRORFORM-SAME: (indexed (0x0) rangelist = [[CU_RANGE:.*]]
 ; CHECK:   DW_TAG_subprogram
 ; CHECK:     DW_AT_low_pc
-; CHECK-SAME:  (indexed (00000000) address = 0x0000 ".text")
+; CHECK-SAME:  (indexed (00000000) address = 0x00000000 ".text")
 ; CHECK:     DW_AT_high_pc
 ; CHECK-SAME:  (0x00000002)
 ; CHECK:     DW_AT_name
@@ -70,7 +72,7 @@
 ; EXPR:      DW_AT_low_pc
 ; EXPR-SAME:   (DW_OP_addrx 0x0, DW_OP_const4u 0x2, DW_OP_plus)
 ; FORM:      DW_AT_low_pc
-; FORM-SAME:   [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0x2 address = 0x0002 ".text")
+; FORM-SAME:   [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0x2 address = 0x00000002 ".text")
 ; EXPRORFORM: DW_AT_high_pc
 ; EXPRORFORM-SAME: (0x0000000a)
 ; RNG:       DW_AT_ranges
@@ -81,31 +83,31 @@
 ; EXPR:        DW_AT_low_pc
 ; EXPR-SAME:     [DW_FORM_exprloc] (DW_OP_addrx 0x0, DW_OP_const4u 0x6, DW_OP_plus)
 ; FORM:        DW_AT_low_pc
-; FORM-SAME:     [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0x6 address = 0x0006 ".text")
+; FORM-SAME:     [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0x6 address = 0x00000006 ".text")
 ; EXPRORFORM:  DW_AT_high_pc
 ; EXPRORFORM-SAME: (0x00000004)
 ; RNG:         DW_AT_ranges
 ; RNG-SAME:      (indexed (0x1) rangelist = [[INL_RANGE:.*]]
 ; CHECK:     DW_TAG_call_site
 ; RNG:         DW_AT_call_return_pc
-; RNG-SAME:      (indexed (00000001) address = 0x0006 ".text")
+; RNG-SAME:      (indexed (00000001) address = 0x00000006 ".text")
 ; EXPR:        DW_AT_call_return_pc
 ; EXPR-SAME:     [DW_FORM_exprloc] (DW_OP_addrx 0x0, DW_OP_const4u 0x6, DW_OP_plus)
 ; FORM:        DW_AT_call_return_pc
-; FORM-SAME:     [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0x6 address = 0x0006 ".text")
+; FORM-SAME:     [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0x6 address = 0x00000006 ".text")
 ; CHECK:     DW_TAG_call_site
 ; RNG:         DW_AT_call_return_pc
-; RNG-SAME:      (indexed (00000002) address = 0x000a ".text")
+; RNG-SAME:      (indexed (00000002) address = 0x0000000a ".text")
 ; EXPR:        DW_AT_call_return_pc
 ; EXPR-SAME:     [DW_FORM_exprloc] (DW_OP_addrx 0x0, DW_OP_const4u 0xa, DW_OP_plus)
 ; FORM:        DW_AT_call_return_pc
-; FORM-SAME:     [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0xa address = 0x000a ".text")
+; FORM-SAME:     [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0xa address = 0x0000000a ".text")
 ; CHECK:     NULL
 ; CHECK:   DW_TAG_subprogram
 ; EXPR:      DW_AT_low_pc
 ; EXPR-SAME:   [DW_FORM_exprloc] (DW_OP_addrx 0x0, DW_OP_const4u 0xe, DW_OP_plus)
 ; FORM:      DW_AT_low_pc
-; FORM-SAME:   [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0xe address = 0x000e ".text")
+; FORM-SAME:   [DW_FORM_LLVM_addrx_offset] (indexed (00000000) + 0xe address = 0x0000000e ".text")
 ; EXPRORFORM: DW_AT_high_pc
 ; EXPRORFORM-SAME: (0x00000002)
 ; RNG:       DW_AT_ranges
@@ -116,16 +118,16 @@
 ; CHECK:     DW_AT_low_pc [DW_FORM_addrx]    (indexed (
 ; RNG-SAME: 00000003
 ; EXPRORFORM-SAME: 00000001
-; CHECK: ) address = 0x0000 ".other")
+; CHECK: ) address = 0x00000000 ".other")
 ; CHECK:     DW_AT_high_pc
 ; CHECK-SAME:  (0x00000006)
 ; CHECK:     DW_AT_name
 ; CHECK-SAME: "f6"
 ; CHECK:       DW_TAG_inlined_subroutine
 ; RNG:           DW_AT_low_pc
-; RNG-SAME:        (indexed (00000003) address = 0x0000 ".other")
+; RNG-SAME:        (indexed (00000003) address = 0x00000000 ".other")
 ; EXPRORFORM:    DW_AT_low_pc
-; EXPRORFORM-SAME: (indexed (00000001) address = 0x0000 ".other")
+; EXPRORFORM-SAME: (indexed (00000001) address = 0x00000000 ".other")
 ; CHECK:         DW_AT_high_pc
 ; CHECK-SAME:      (0x00000004)
 ; CHECK:       DW_TAG_call_site
@@ -136,11 +138,11 @@
 ; CHECK-LABEL: .debug_addr contents:
 ; CHECK: 0x00000000: Address table
 ; CHECK-NEXT: Addrs: [
-; CHECK-NEXT: 0x0000
-; RNG-NEXT:   0x0006
-; RNG-NEXT:   0x000a
-; CHECK-NEXT: 0x0000
-; RNG-NEXT:   0x0004
+; CHECK-NEXT: 0x00000000
+; RNG-NEXT:   0x00000006
+; RNG-NEXT:   0x0000000a
+; CHECK-NEXT: 0x00000000
+; RNG-NEXT:   0x00000004
 ; CHECK-NEXT: ]
 
 ; CHECK-LABEL: .debug_rnglists contents:
@@ -148,28 +150,28 @@
 ; EXPRORFORM: 0x00000000: range list header: {{.*}}, offset_entry_count = 0x00000001
 ; CHECK: ranges:
 ; RNG-NEXT:   [[F3_RANGE]]: [DW_RLE_base_addressx]:
-; RNG-SAME:                   0x0000
+; RNG-SAME:                   0x00000000
 ; RNG-NEXT:                 [DW_RLE_offset_pair  ]
 ; RNG-NEXT:                 [DW_RLE_end_of_list  ]
 
 ; RNG-NEXT:   [[INL_RANGE]]: [DW_RLE_base_addressx]:
-; RNG-SAME:                    0x0000
+; RNG-SAME:                    0x00000000
 ; RNG-NEXT:                  [DW_RLE_offset_pair  ]
 ; RNG-NEXT:                  [DW_RLE_end_of_list  ]
 
 ; RNG-NEXT:   [[F5_RANGE]]: [DW_RLE_base_addressx]:
-; RNG-SAME:                   0x0000
+; RNG-SAME:                   0x00000000
 ; RNG-NEXT:                 [DW_RLE_offset_pair  ]
 ; RNG-NEXT:                 [DW_RLE_end_of_list  ]
 
 ; CHECK-NEXT: [[CU_RANGE]]: [DW_RLE_base_addressx]:
-; CHECK-SAME:                 0x0000
+; CHECK-SAME:                 0x00000000
 ; CHECK-NEXT:               [DW_RLE_offset_pair  ]
 ; CHECK-NEXT:               [DW_RLE_offset_pair  ]
 ; RNG-NEXT:                 [DW_RLE_startx_length]:
-; RNG-SAME:                   0x0003
+; RNG-SAME:                   0x00000003
 ; EXPRORFORM-NEXT:          [DW_RLE_startx_length]:
-; EXPRORFORM-SAME:            0x0001
+; EXPRORFORM-SAME:            0x00000001
 ; CHECK-NEXT:               [DW_RLE_end_of_list  ]
 
 ; Function Attrs: mustprogress noinline nounwind optnone uwtable
diff --git a/llvm/test/DebugInfo/X86/dbg-empty-metadata-lowering.ll b/llvm/test/DebugInfo/X86/dbg-empty-metadata-lowering.ll
new file mode 100644
index 0000000000000..3e2b957d7f29e
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dbg-empty-metadata-lowering.ll
@@ -0,0 +1,49 @@
+; RUN: llc %s -stop-after=finalize-isel -o - | FileCheck %s --implicit-check-not=DBG
+
+;; Check that dbg.values with empty metadata are treated as kills (i.e. become
+;; DBG_VALUE $noreg, ...). dbg.declares with empty metadata location operands
+;; should be ignored.
+
+; CHECK: stack: []
+; CHECK: DBG_VALUE float 5.000000e+00
+; CHECK: @ext
+; CHECK: DBG_VALUE $noreg
+; CHECK: @ext
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @fun() local_unnamed_addr #0 !dbg !9 {
+entry:
+  call void @llvm.dbg.declare(metadata !{}, metadata !20, metadata !DIExpression()), !dbg !15
+  call void @llvm.dbg.value(metadata float 5.000000e+00, metadata !13, metadata !DIExpression()), !dbg !15
+  tail call void @ext(), !dbg !16
+  call void @llvm.dbg.value(metadata !{}, metadata !13, metadata !DIExpression()), !dbg !15
+  tail call void @ext(), !dbg !16
+  ret void, !dbg !17
+}
+
+declare !dbg !18 void @ext() local_unnamed_addr
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 16.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{!"clang version 16.0.0"}
+!9 = distinct !DISubprogram(name: "fun", linkageName: "fun", scope: !1, file: !1, line: 2, type: !10, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!10 = !DISubroutineType(types: !11)
+!11 = !{null}
+!12 = !{!13}
+!13 = !DILocalVariable(name: "f", scope: !9, file: !1, line: 3, type: !14)
+!14 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+!15 = !DILocation(line: 0, scope: !9)
+!16 = !DILocation(line: 4, column: 3, scope: !9)
+!17 = !DILocation(line: 5, column: 1, scope: !9)
+!18 = !DISubprogram(name: "ext", linkageName: "ext", scope: !1, file: !1, line: 1, type: !10, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !19)
+!19 = !{}
+!20 = !DILocalVariable(name: "g", scope: !9, file: !1, line: 3, type: !14)
diff --git a/llvm/test/DebugInfo/X86/empty-split-dwarf.ll b/llvm/test/DebugInfo/X86/empty-split-dwarf.ll
new file mode 100644
index 0000000000000..b6dfef9d20f7c
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/empty-split-dwarf.ll
@@ -0,0 +1,18 @@
+; UNSUPPORTED: target={{.*}}-aix{{.*}}
+; RUN: %llc_dwarf -split-dwarf-file=foo.dwo < %s -filetype=obj --mtriple=x86_64-unknown-linux-elf | llvm-dwarfdump -v - | FileCheck --check-prefix=FISSION %s
+
+; Don't emit DW_AT_addr_base when there are no addresses.
+; Also don't emit a split line table when there are no type units.
+; FISSION-NOT: DW_AT_GNU_addr_base [DW_FORM_sec_offset]
+; FISSION-NOT: .debug_line.dwo contents:
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.1 (trunk 143523)", isOptimized: true, emissionKind: FullDebug, file: !4, enums: !2, retainedTypes: !6, globals: !2)
+!2 = !{}
+!3 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
+!4 = !DIFile(filename: "empty.c", directory: "/home/nlewycky")
+!5 = !{i32 1, !"Debug Info Version", i32 3}
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
diff --git a/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll b/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll
new file mode 100644
index 0000000000000..b8259dcfab85d
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/instcombine-fold-cast-into-phi.ll
@@ -0,0 +1,48 @@
+; RUN: opt -passes=instcombine -S %s | FileCheck %s
+
+;; Instcombine folds the trunc %x into the phi. Check it updates the phi's dbg
+;; user (otherwise the dbg use becomes poison after the original phi is
+;; deleted). Check the new phi inherits the DebugLoc.
+
+; CHECK: %[[phi:.*]] = phi i8 [ 1, %{{.*}} ], [ 0, %{{.*}} ], !dbg ![[dbg:[0-9]+]]
+; CHECK: call void @llvm.dbg.value(metadata i8 %[[phi]], metadata ![[#]], metadata !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed, DW_OP_stack_value)
+; CHECK: ![[dbg]] = !DILocation(line: 123,
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define linkonce_odr float @f(i1 %cond) {
+entry:
+  br i1 %cond, label %if.then, label %if.end
+
+if.then:                                          ; preds = entry
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %p.0 = phi i32 [ 1, %if.then ], [ 0, %entry ], !dbg !13
+  call void @llvm.dbg.value(metadata i32 %p.0, metadata !4, metadata !DIExpression()), !dbg !13
+  %x = trunc i32 %p.0 to i8
+  %callff = call float @ff(i8  %x)
+  ret float %callff
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+declare float @ff(float)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_11, file: !1, producer: "clang version 17.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "btConvexTriangleMeshShape.cpp", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !DILocalVariable(name: "p", scope: !5, file: !6, line: 2057, type: !12)
+!5 = distinct !DILexicalBlock(scope: !7, file: !6, line: 2056, column: 4)
+!6 = !DIFile(filename: "reduce.cpp", directory: "/")
+!7 = distinct !DISubprogram(name: "diagonalize", linkageName: "_ZN11btMatrix3x311diagonalizeERS_fi", scope: !8, file: !6, line: 2054, type: !9, scopeLine: 2055, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, declaration: !11, retainedNodes: !2)
+!8 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "btMatrix3x3", file: !6, line: 2050, size: 24, flags: DIFlagTypePassByValue, elements: !2, identifier: "_ZTS11btMatrix3x3")
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DISubprogram(name: "diagonalize", linkageName: "_ZN11btMatrix3x311diagonalizeERS_fi", scope: !8, file: !6, line: 2054, type: !9, scopeLine: 2054, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !DILocation(line: 123, scope: !5)
diff --git a/llvm/test/DebugInfo/X86/merge_inlined_loc.ll b/llvm/test/DebugInfo/X86/merge_inlined_loc.ll
index 07a72d6d30d70..41dcbe4785304 100644
--- a/llvm/test/DebugInfo/X86/merge_inlined_loc.ll
+++ b/llvm/test/DebugInfo/X86/merge_inlined_loc.ll
@@ -26,7 +26,9 @@
 
 ; CHECK: .loc 1 2 25 epilogue_begin
 ; CHECK-NEXT: popq %rax
+; CHECK-NEXT: .Ltmp{{.*}}:
 ; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: .loc 1 2 25 is_stmt 0
 ; CHECK-NEXT: jmp _Z6commonv
 ; CHECK-NEXT: [[LABEL:.*]]:
 
diff --git a/llvm/test/DebugInfo/X86/test-fexceptions-debug-frame.ll b/llvm/test/DebugInfo/X86/test-fexceptions-debug-frame.ll
new file mode 100644
index 0000000000000..689a692106fa8
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/test-fexceptions-debug-frame.ll
@@ -0,0 +1,42 @@
+; This test is added to provide test coverage for the 
+; .cfi_sections .debug_frame intrinsic. It aims to make sure that for an X86_64
+; output compiled with -fexceptions, no .cfi_sections .debug_frame is emitted.
+
+; RUN: llc --filetype=asm %s -o - | FileCheck %s
+; CHECK-NOT: .cfi_sections .debug_frame
+; CHECK: .cfi_startproc
+
+; ModuleID = 'test.cpp'
+source_filename = "test.cpp"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx13.0.0"
+
+; Function Attrs: noinline norecurse nounwind optnone ssp uwtable
+define i32 @main() #0 !dbg !10 {
+  %1 = alloca i32, align 4
+  store i32 0, ptr %1, align 4
+  ret i32 1, !dbg !15
+}
+
+attributes #0 = { noinline norecurse nounwind optnone ssp uwtable  }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+!llvm.dbg.cu = !{!7}
+!llvm.ident = !{!9}
+
+!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+!1 = !{i32 7, !"Dwarf Version", i32 4}
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 8, !"PIC Level", i32 2}
+!5 = !{i32 7, !"uwtable", i32 2}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !8, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None, sysroot: "/Applications/Xcode.app", sdk: "MacOSX.sdk")
+!8 = !DIFile(filename: "test.cpp", directory: "/Users/shubham/Development")
+!9 = !{!"clang"}
+!10 = distinct !DISubprogram(name: "main", scope: !8, file: !8, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !7, retainedNodes: !14)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!13}
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{}
+!15 = !DILocation(line: 2, column: 5, scope: !10)
diff --git a/llvm/test/DebugInfo/X86/test-fno-exceptions-debug-frame.ll b/llvm/test/DebugInfo/X86/test-fno-exceptions-debug-frame.ll
new file mode 100644
index 0000000000000..d5e8e9329bb8c
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/test-fno-exceptions-debug-frame.ll
@@ -0,0 +1,43 @@
+; This test is added to provide test coverage for the 
+; .cfi_sections .debug_frame intrinsic. It aims to make sure that for an X86_64
+; output compiled with -fno-exceptions, no .cfi_sections .debug_frame is 
+; emitted.
+
+; RUN: llc --filetype=asm %s -o - | FileCheck %s
+; CHECK-NOT: .cfi_sections .debug_frame
+; CHECK: .cfi_startproc
+
+; ModuleID = 'test.cpp'
+source_filename = "test.cpp"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx13.0.0"
+
+; Function Attrs: noinline norecurse nounwind optnone ssp uwtable
+define i32 @main() #0 !dbg !10 {
+  %1 = alloca i32, align 4
+  store i32 0, ptr %1, align 4
+  ret i32 1, !dbg !15
+}
+
+attributes #0 = { noinline norecurse nounwind optnone ssp uwtable }
+
+!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6}
+!llvm.dbg.cu = !{!7}
+!llvm.ident = !{!9}
+
+!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 14, i32 0]}
+!1 = !{i32 7, !"Dwarf Version", i32 4}
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 8, !"PIC Level", i32 2}
+!5 = !{i32 7, !"uwtable", i32 2}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !8, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None, sysroot: "/Applications/Xcode.app", sdk: "MacOSX.sdk")
+!8 = !DIFile(filename: "test.cpp", directory: "/Users/shubham/Development")
+!9 = !{!"clang"}
+!10 = distinct !DISubprogram(name: "main", scope: !8, file: !8, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !7, retainedNodes: !14)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!13}
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{}
+!15 = !DILocation(line: 2, column: 5, scope: !10)
diff --git a/llvm/test/DebugInfo/assignment-tracking/AArch64/lit.local.cfg b/llvm/test/DebugInfo/assignment-tracking/AArch64/lit.local.cfg
new file mode 100644
index 0000000000000..7184443994b69
--- /dev/null
+++ b/llvm/test/DebugInfo/assignment-tracking/AArch64/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/DebugInfo/assignment-tracking/AArch64/scalable-vectors.ll b/llvm/test/DebugInfo/assignment-tracking/AArch64/scalable-vectors.ll
new file mode 100644
index 0000000000000..4f3db535f6eb8
--- /dev/null
+++ b/llvm/test/DebugInfo/assignment-tracking/AArch64/scalable-vectors.ll
@@ -0,0 +1,51 @@
+; RUN: llc %s -stop-after=finalize-isel -o - | FileCheck %s
+
+;; Hand written. Check AssignmentTrackingAnalysis doesn't try to get the size
+;; of scalable vectors (which causes an assertion failure).
+
+; CHECK: DBG_VALUE %stack.0, $noreg, ![[#]], !DIExpression(DW_OP_deref)
+; CHECK: DBG_VALUE i256 0, $noreg, ![[#]], !DIExpression()
+;; FIXME: We should reinstate (or poison) the variable at the store. See
+;; llvm.org/PR62346.
+
+target triple = "aarch64"
+
+define dso_local void @square(ptr %0) local_unnamed_addr #0 !dbg !9 {
+entry:
+   %1 = alloca <32 x i8>, !DIAssignID !18
+   call void @llvm.dbg.assign(metadata i1 poison, metadata !19, metadata !DIExpression(), metadata !18, metadata ptr %1, metadata !DIExpression()), !dbg !21
+   %2 = load <vscale x 8 x i8>, ptr %0, align 1
+   call void @llvm.dbg.assign(metadata i256 0, metadata !19, metadata !DIExpression(), metadata !22, metadata ptr %1, metadata !DIExpression()), !dbg !21
+   store <vscale x 8 x i8> %2, ptr %1, align 1
+   ret void
+}
+
+declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
+
+attributes #0 = { vscale_range(1,16) "target-cpu"="generic" "target-features"="+sve" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 17.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "/app/example.c", directory: "/app")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!8 = !{!"clang version 17.0.0"}
+!9 = distinct !DISubprogram(name: "square", scope: !10, file: !10, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
+!10 = !DIFile(filename: "example.c", directory: "/app")
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !13, !14, !16}
+!13 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !14)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64)
+!15 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !{}
+!18 =  distinct !DIAssignID()
+!19 = !DILocalVariable(name: "i", scope: !9, file: !10, line: 2, type: !20)
+!20 = !DIBasicType(name: "cats", size: 256, encoding: DW_ATE_signed)
+!21 = !DILocation(line: 0, scope: !9)
+!22 =  distinct !DIAssignID()
+
diff --git a/llvm/test/DebugInfo/assignment-tracking/X86/global-storage.ll b/llvm/test/DebugInfo/assignment-tracking/X86/global-storage.ll
new file mode 100644
index 0000000000000..7d265acdb5bb4
--- /dev/null
+++ b/llvm/test/DebugInfo/assignment-tracking/X86/global-storage.ll
@@ -0,0 +1,45 @@
+; RUN: llc %s -stop-after=finalize-isel -o - | FileCheck %s
+
+;; Local variable has global storage. Check AssignmentTrackingAnalysis doesn't
+;; crash/assert.
+
+;; FIXME: We ideally want a DBG_VALUE deref here. It's not possible with the
+;; current setup, but will be possible when assignment tracking is extended to
+;; understand non-alloca storage.
+
+; CHECK: stack: []
+; CHECK-NOT: DBG_
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = dso_local global i32 0, align 4
+
+define dso_local void @_Z3funi(i32 noundef %x) !dbg !15 {
+entry:
+  store i32 %x, ptr @a, align 4, !DIAssignID !27
+  call void @llvm.dbg.assign(metadata i32 %x, metadata !23, metadata !DIExpression(), metadata !27, metadata ptr @a, metadata !DIExpression()), !dbg !21
+  ret void
+}
+
+declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!6, !7, !13}
+!llvm.ident = !{!14}
+
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "test.cpp", directory: "/")
+!4 = !{}
+!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!6 = !{i32 7, !"Dwarf Version", i32 5}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!14 = !{!"clang version 17.0.0"}
+!15 = distinct !DISubprogram(name: "fun", linkageName: "_Z3funi", scope: !3, file: !3, line: 2, type: !16, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !18)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !5}
+!18 = !{}
+!20 = !DILocalVariable(name: "x", arg: 1, scope: !15, file: !3, line: 2, type: !5)
+!21 = !DILocation(line: 0, scope: !15)
+!23 = !DILocalVariable(name: "a", scope: !15, file: !3, line: 3, type: !5)
+!27 = distinct !DIAssignID()
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s
index 3eb72e5b2456a..a74f37e22d331 100644
--- a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s
@@ -241,6 +241,30 @@ test_ld64_gotlo12_external:
         ldr   x0, [x0, :got_lo12:external_data]
         .size test_ld64_gotlo12_external, .-test_ld64_gotlo12_external
 
+# Check R_AARCH64_CONDBR19 for compare and branch instructions
+#
+# jitlink-check: decode_operand(test_condbr19_cbz, 1) = \
+# jitlink-check:     (test_condbr19_cbz_target - test_condbr19_cbz)[21:2]
+        .globl test_condbr19_cbz, test_condbr19_cbz_target
+        .p2align 2
+test_condbr19_cbz:
+        cbz x0, test_condbr19_cbz_target
+        .skip (1 << 19)
+test_condbr19_cbz_target:
+        .size test_condbr19_cbz, .-test_condbr19_cbz
+
+# Check R_AARCH64_CONDBR19 for conditional branch instructions
+#
+# jitlink-check: decode_operand(test_condbr19_bc, 1) = \
+# jitlink-check:     (test_condbr19_bc_target - test_condbr19_bc)[21:2]
+        .globl test_condbr19_bc, test_condbr19_bc_target
+        .p2align 2
+test_condbr19_bc:
+        b.eq test_condbr19_bc_target
+        .skip (1 << 19)
+test_condbr19_bc_target:
+        .size test_condbr19_bc, .-test_condbr19_bc
+
         .globl  named_data
         .p2align  4
         .type   named_data,@object
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_constant_global_redzones.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_constant_global_redzones.ll
index e7de64579f03b..c4d21b059558b 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_constant_global_redzones.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_constant_global_redzones.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 ; Here we check that the global redzone sizes grow with the object size
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_global_redzones.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_global_redzones.ll
index 9aa8b431eb83a..5b7821c2c146b 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_global_redzones.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/adaptive_global_redzones.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 ; Here we check that the global redzone sizes grow with the object size
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_lds.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_lds.ll
index 06f653ba6f406..8b0fa816721bb 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_lds.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_lds.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 ; Memory access to lds are not instrumented
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_scratch.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_scratch.ll
index 323654a686688..c614c9621a761 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_scratch.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_scratch.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 ; Memory access to scratch are not instrumented
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
index 872a6374580e3..bf2038994a935 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 @x = addrspace(4) global [2 x i32] zeroinitializer, align 4
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
index 7c5259b096932..465d7c9c4eb66 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 define protected amdgpu_kernel void @generic_store(ptr addrspace(1) %p, i32 %i) sanitize_address {
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
index b25d9f8ae995d..a051b4197098d 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 define protected amdgpu_kernel void @global_store(ptr addrspace(1) %p, i32 %i) sanitize_address {
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/global_metadata_addrspacecasts.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/global_metadata_addrspacecasts.ll
index 1a5282f49e001..53331a6bbe49b 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/global_metadata_addrspacecasts.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/global_metadata_addrspacecasts.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 @g = addrspace(1) global [1 x i32] zeroinitializer, align 4
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_lds_globals.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_lds_globals.ll
index a4f674d108c37..fca6c24ba31ad 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_lds_globals.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_lds_globals.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 @G10 = addrspace(3) global [10 x i8] zeroinitializer, align 1
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_scratch_globals.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_scratch_globals.ll
index 2e7f59c47cc26..daf9aa48b96f0 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_scratch_globals.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_scratch_globals.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 @G10 = addrspace(5) global [10 x i8] zeroinitializer, align 1
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
index e8f1ca44f4ced..bbfb35549b5b6 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation when tags are generated by HWASan function.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s
@@ -8,12 +9,54 @@ target triple = "riscv64-unknown-linux"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
-; CHECK: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
-; CHECK: %[[B:[^ ]*]] = ptrtoint ptr %x to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[A]], 56
-; CHECK: or i64 %[[B]], %[[C]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @test_alloca to i64), [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = call i8 @__hwasan_generate_tag()
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i8 [[TMP15]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], 72057594037927935
+; CHECK-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP16]], 56
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP20]] to ptr
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP16]] to i8
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], 72057594037927935
+; CHECK-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP23]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i32 0
+; CHECK-NEXT:    store i8 4, ptr [[TMP26]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; CHECK-NEXT:    store i8 [[TMP21]], ptr [[TMP27]], align 1
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP29:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 72057594037927935
+; CHECK-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP31]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 [[TMP28]], i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %x = alloca i32, align 4
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
index 488b8bda359f8..96ff7cb7efb29 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
@@ -1,8 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ZERO-BASED-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,UAR-TAGS
+; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ZERO-BASED-SHADOW
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
@@ -10,44 +10,76 @@ target triple = "riscv64-unknown-linux"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress !dbg !15 {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
-
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
-; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-
-; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
-; CHECK: %[[E:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to ptr
-; CHECK: %[[X_SHADOW_GEP:[^ ]*]] = getelementptr i8, ptr %[[X_SHADOW]], i32 0
-; CHECK: store i8 4, ptr %[[X_SHADOW_GEP]]
-; CHECK: %[[X_I8_GEP:[^ ]*]] = getelementptr i8, ptr %[[X]], i32 15
-; CHECK: store i8 %[[X_TAG2]], ptr %[[X_I8_GEP]]
-; CHECK: call void @llvm.dbg.value(
-; CHECK-SAME: metadata !DIArgList(ptr %[[X]], ptr %[[X]])
-; CHECK-SAME: metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0,
-; CHECK: call void @use32(ptr nonnull %[[X_HWASAN]])
-
-; UAR-TAGS: %[[BASE_TAG_COMPL:[^ ]*]] = xor i64 %[[BASE_TAG]], 255
-; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
-; CHECK: %[[E2:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F2]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to ptr
-; NO-UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 0, i64 1, i1 false)
-; UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 %[[X_TAG_UAR]], i64 1, i1 false)
-; CHECK: ret void
-
-
+; DYNAMIC-SHADOW-LABEL: define void @test_alloca
+; DYNAMIC-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; DYNAMIC-SHADOW-NEXT:  entry:
+; DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; DYNAMIC-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; DYNAMIC-SHADOW-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; DYNAMIC-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927935, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 72057594037927935, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 4, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP11]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 4, ptr [[TMP13]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 [[TMP8]], ptr [[TMP14]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8, !dbg [[DBG14:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72057594037927935, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP17]], 4, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP18]], !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP19]], i8 [[TMP15]], i64 1, i1 false), !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
+; ZERO-BASED-SHADOW-LABEL: define void @test_alloca
+; ZERO-BASED-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; ZERO-BASED-SHADOW-NEXT:  entry:
+; ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; ZERO-BASED-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; ZERO-BASED-SHADOW-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; ZERO-BASED-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927935, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]], !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 72057594037927935, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 4, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 4, ptr [[TMP13]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 [[TMP8]], ptr [[TMP14]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8, !dbg [[DBG14:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72057594037927935, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP17]], 4, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP19]], i8 [[TMP15]], i64 1, i1 false), !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
 entry:
   %x = alloca i32, align 4
   call void @llvm.dbg.value(metadata !DIArgList(ptr %x, ptr %x), metadata !22, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_deref)), !dbg !21
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
index 47ad90648711b..0e83b42001538 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,10 +7,18 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
-; CHECK-LABEL: @atomicrmw(
-; CHECK: call void @llvm.hwasan.check.memaccess.shortgranules({{.*}}, ptr %ptr, i32 19)
-; CHECK: atomicrmw add ptr %ptr, i64 1 seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @atomicrmw
+; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw add ptr [[PTR]], i64 1 seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = atomicrmw add ptr %ptr, i64 1 seq_cst
@@ -17,10 +26,18 @@ entry:
 }
 
 define void @cmpxchg(ptr %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
-; CHECK-LABEL: @cmpxchg(
-; CHECK: call void @llvm.hwasan.check.memaccess.shortgranules({{.*}}, ptr %ptr, i32 19)
-; CHECK: cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @cmpxchg
+; CHECK-SAME: (ptr [[PTR:%.*]], i64 [[COMPARE_TO:%.*]], i64 [[NEW_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[COMPARE_TO]], i64 [[NEW_VALUE]] seq_cst seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll
index cb3667bf63b03..12822d86a3910 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-DYNAMIC-SHADOW
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.used = appending global [1 x ptr] [ptr @hwasan.module_ctor]
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @hwasan.module_ctor, ptr @hwasan.module_ctor }]
@@ -12,49 +13,104 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 96", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 0
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 0)
-
-; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4
-; CHECK: ret i8 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 96", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 96", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i8, ptr %a, align 4
@@ -62,49 +118,104 @@ entry:
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 97", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 1
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 1)
-
-; CHECK: %[[G:[^ ]*]] = load i16, ptr %a, align 4
-; CHECK: ret i16 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 97", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 97", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i16, ptr %a, align 4
@@ -112,25 +223,99 @@ entry:
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 98", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 2)
-
-; CHECK: %[[G:[^ ]*]] = load i32, ptr %a, align 4
-; CHECK: ret i32 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 98", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 98", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+
+
+
 
 entry:
   %b = load i32, ptr %a, align 4
@@ -138,25 +323,99 @@ entry:
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 99", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 3)
-
-; CHECK: %[[G:[^ ]*]] = load i64, ptr %a, align 8
-; CHECK: ret i64 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 99", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 99", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+
+
+
 
 entry:
   %b = load i64, ptr %a, align 8
@@ -164,25 +423,99 @@ entry:
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 100", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 4)
-
-; CHECK: %[[G:[^ ]*]] = load i128, ptr %a, align 16
-; CHECK: ret i128 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 100", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 100", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+
+
+
 
 entry:
   %b = load i128, ptr %a, align 16
@@ -190,12 +523,42 @@ entry:
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
-; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
-; CHECK: ret i40 %[[B]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
 
 entry:
   %b = load i40, ptr %a, align 4
@@ -203,25 +566,99 @@ entry:
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 112", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 16)
-
-; CHECK: store i8 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 112", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 112", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i8 %b, ptr %a, align 4
@@ -229,25 +666,99 @@ entry:
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 113", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 17)
-
-; CHECK: store i16 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 113", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 113", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i16 %b, ptr %a, align 4
@@ -255,25 +766,99 @@ entry:
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 114", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 18)
-
-; CHECK: store i32 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 114", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 114", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i32 %b, ptr %a, align 4
@@ -281,25 +866,99 @@ entry:
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 115", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 19)
-
-; CHECK: store i64 %b, ptr %a, align 8
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 115", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 115", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i64 %b, ptr %a, align 8
@@ -307,25 +966,99 @@ entry:
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 116", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 20)
-
-; CHECK: store i128 %b, ptr %a, align 16
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 116", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 116", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i128 %b, ptr %a, align 16
@@ -333,12 +1066,42 @@ entry:
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
-; CHECK: store i40 %b, ptr %a
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i40 %b, ptr %a, align 4
@@ -346,12 +1109,42 @@ entry:
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store_unaligned(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
-; CHECK: store i64 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i64 %b, ptr %a, align 4
@@ -359,33 +1152,108 @@ entry:
 }
 
 define i8 @test_load_noattr(ptr %a) {
-; CHECK-LABEL: @test_load_noattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_noattr
+; CHECK-SAME: (ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_notmyattr(ptr %a) sanitize_address {
-; CHECK-LABEL: @test_load_notmyattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_notmyattr
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load_addrspace(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr addrspace(256) %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_addrspace
+; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr addrspace(256) %a, align 4
   ret i8 %b
@@ -393,9 +1261,7 @@ entry:
 
 ; CHECK: declare void @__hwasan_init()
 
-; CHECK:      define internal void @hwasan.module_ctor() #[[#ATTR:]] comdat {
 ; CHECK-NEXT:   call void @__hwasan_init()
 ; CHECK-NEXT:   ret void
 ; CHECK-NEXT: }
 
-; CHECK:      attributes #[[#ATTR]] = { nounwind }
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
index 7b51acd09c97b..a4fef0d49dc23 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test allocas with multiple lifetime ends, as frequently seen for exception
 ; handling.
 ;
-; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
@@ -14,6 +15,85 @@ declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 declare i32 @__gxx_personality_v0(...)
 
 define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define void @test
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk.__gxx_personality_v0 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @test to i64), [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72057594037927935
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP15]], 56
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP15]] to i8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP24]], i32 0
+; CHECK-NEXT:    store i8 4, ptr [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; CHECK-NEXT:    store i8 [[TMP20]], ptr [[TMP26]], align 1
+; CHECK-NEXT:    invoke void @mayFail(ptr [[X_HWASAN]])
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 72057594037927935
+; CHECK-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP30]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP32:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    [[TMP33:%.*]] = extractvalue { ptr, i32 } [[TMP32]], 0
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EXN_SLOT]], i32 19)
+; CHECK-NEXT:    store ptr [[TMP33]], ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    [[TMP34:%.*]] = extractvalue { ptr, i32 } [[TMP32]], 1
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EHSELECTOR_SLOT]], i32 18)
+; CHECK-NEXT:    store i32 [[TMP34]], ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    call void @onExcept(ptr [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP35:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 72057594037927935
+; CHECK-NEXT:    [[TMP38:%.*]] = lshr i64 [[TMP37]], 4
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP38]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP39]], i8 [[TMP35]], i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    br label [[EH_RESUME:%.*]]
+; CHECK:       eh.resume:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EXN_SLOT]], i32 3)
+; CHECK-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EHSELECTOR_SLOT]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } undef, ptr [[EXN]], 0
+; CHECK-NEXT:    [[LPAD_VAL1:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
+; CHECK-NEXT:    resume { ptr, i32 } [[LPAD_VAL1]]
+;
 entry:
   %x = alloca i32, align 8
   %exn.slot = alloca ptr, align 8
@@ -22,19 +102,11 @@ entry:
   invoke void @mayFail(ptr %x) to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
-; CHECK: invoke.cont:
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  ret void
 
   call void @llvm.lifetime.end.p0(i64 8, ptr %x)
   ret void
 
 lpad:                                             ; preds = %entry
-; CHECK: lpad
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  br label %eh.resume
 
   %0 = landingpad { ptr, i32 }
   cleanup
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
index 0f50ee753a818..eec644757b0a8 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-use-after-scope -S < %s | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
@@ -8,12 +9,69 @@ target triple = "riscv64-unknown-linux"
 declare void @may_jump()
 
 define dso_local noundef i1 @_Z6targetv() sanitize_hwaddress {
+; CHECK-LABEL: define dso_local noundef i1 @_Z6targetv
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @_Z6targetv to i64), [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP4]], 56
+; CHECK-NEXT:    [[BUF:%.*]] = alloca [4096 x i8], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72057594037927935
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP15]], 56
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[BUF_HWASAN:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP15]] to i8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP23]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP24]], i8 [[TMP20]], i64 256, i1 false)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @setjmp(ptr noundef @jbuf)
+; CHECK-NEXT:    switch i32 [[CALL]], label [[WHILE_BODY:%.*]] [
+; CHECK-NEXT:    i32 1, label [[RETURN:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       while.body:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr @stackbuf, i32 19)
+; CHECK-NEXT:    store ptr [[BUF_HWASAN]], ptr @stackbuf, align 8
+; CHECK-NEXT:    call void @may_jump()
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i1 [ true, [[WHILE_BODY]] ], [ true, [[SW_BB1]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; CHECK-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP28]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 256, i1 false)
+; CHECK-NEXT:    ret i1 [[RETVAL_0]]
+;
 entry:
   %buf = alloca [4096 x i8], align 1
   %call = call i32 @setjmp(ptr noundef @jbuf)
   switch i32 %call, label %while.body [
-    i32 1, label %return
-    i32 2, label %sw.bb1
+  i32 1, label %return
+  i32 2, label %sw.bb1
   ]
 
 sw.bb1:                                           ; preds = %entry
@@ -29,8 +87,6 @@ while.body:                                       ; preds = %entry
   call void @llvm.lifetime.end.p0(i64 4096, ptr nonnull %buf) #10
   br label %return
 
-; CHECK-LABEL: return:
-; CHECK: void @llvm.memset.p0.i64({{.*}}, i8 0, i64 256, i1 false)
 return:                                           ; preds = %entry, %while.body, %sw.bb1
   %retval.0 = phi i1 [ true, %while.body ], [ true, %sw.bb1 ], [ false, %entry ]
   ret i1 %retval.0
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
index d7155ede54ebd..72a0f6bc8df5b 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -6,9 +7,42 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use(ptr, ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-  ; CHECK: alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 20
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = and i64 [[TMP4]], 63
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP2]], 57
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP5]], 63
+; CHECK-NEXT:    [[X:%.*]] = alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP11]], i64 16)
+; CHECK-NEXT:    [[Y:%.*]] = alloca i8, i64 16, align 16
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], -9079256848778919937
+; CHECK-NEXT:    [[TMP15:%.*]] = shl i64 [[TMP12]], 57
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[Y_HWASAN:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP12]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[Y]], i8 [[TMP17]], i64 16)
+; CHECK-NEXT:    call void @use(ptr [[X_HWASAN]], ptr [[Y_HWASAN]])
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP18]], i64 16)
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[Y]], i8 [[TMP19]], i64 16)
+; CHECK-NEXT:    ret void
+;
   %x = alloca i8, i64 4
-  ; CHECK: alloca i8, i64 16, align 16
   %y = alloca i8, i64 16
   call void @use(ptr %x, ptr %y)
   ret void
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
index aaa70c51c6f2b..2a82d708a0ef1 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation when tags are generated by HWASan function.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s
@@ -8,12 +9,29 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
-; CHECK: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
-; CHECK: %[[B:[^ ]*]] = ptrtoint ptr %x to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[A]], 57
-; CHECK: or i64 %[[B]], %[[C]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 57
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP2]], 63
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @__hwasan_generate_tag()
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -9079256848778919937
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP4]], 57
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP4]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP9]], i64 16)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP10]], i64 16)
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %x = alloca i32, align 4
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
index 6e2d535154905..4bba5365b07cc 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -S | FileCheck %s --check-prefixes=CHECK,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,UAR-TAGS
+; RUN: opt < %s -passes=hwasan -hwasan-use-stack-safety=0 -S | FileCheck %s
+; RUN: opt < %s -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-instrument-with-calls=0 -S | FileCheck %s  --check-prefixes=INLINE
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -9,34 +10,202 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[A_XOR_B:[^ ]*]] = xor i64 %[[A]], %[[B]]
-; CHECK: %[[BASE_TAG:[^ ]*]] = and i64 %[[A_XOR_B]], 63
-
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 57
-; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-
-; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
-; CHECK: call void @__hwasan_tag_memory(ptr %[[X]], i8 %[[X_TAG2]], i64 16)
-
-; CHECK: call void @use32(ptr nonnull %[[X_HWASAN]])
-
-; UAR-TAGS: %[[BASE_TAG_COMPL:[^ ]*]] = xor i64 %[[BASE_TAG]], 63
-; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
-; NO-UAR-TAGS: call void @__hwasan_tag_memory(ptr %[[X]], i8 0, i64 16)
-; UAR-TAGS: call void @__hwasan_tag_memory(ptr %[[X]], i8 %[[X_TAG_UAR]], i64 16)
-; CHECK: ret void
-
-
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = and i64 [[TMP3]], 63
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP1]], 57
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP4]], 63
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], -9079256848778919937
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP5]], 57
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP10]], i64 16)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP11]], i64 16)
+; CHECK-NEXT:    ret void
+;
+; INLINE-LABEL: define void @test_alloca
+; INLINE-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; INLINE-NEXT:  entry:
+; INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; INLINE-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; INLINE-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; INLINE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; INLINE-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @test_alloca to i64), [[TMP5]]
+; INLINE-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; INLINE-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; INLINE-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; INLINE-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; INLINE-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; INLINE-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; INLINE-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; INLINE-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; INLINE-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; INLINE-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; INLINE-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP4]], 57
+; INLINE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP15]], 63
+; INLINE-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; INLINE-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP2]], 0
+; INLINE-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[X]] to i64
+; INLINE-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], -9079256848778919937
+; INLINE-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP16]], 57
+; INLINE-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; INLINE-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP20]] to ptr
+; INLINE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP16]] to i8
+; INLINE-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[X]] to i64
+; INLINE-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], -9079256848778919937
+; INLINE-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP23]], 4
+; INLINE-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP24]]
+; INLINE-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i32 0
+; INLINE-NEXT:    store i8 4, ptr [[TMP26]], align 1
+; INLINE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; INLINE-NEXT:    store i8 [[TMP21]], ptr [[TMP27]], align 1
+; INLINE-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; INLINE-NEXT:    [[TMP28:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; INLINE-NEXT:    [[TMP29:%.*]] = ptrtoint ptr [[X]] to i64
+; INLINE-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], -9079256848778919937
+; INLINE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
+; INLINE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP31]]
+; INLINE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 [[TMP28]], i64 1, i1 false)
+; INLINE-NEXT:    ret void
+;
 entry:
   %x = alloca i32, align 4
   call void @use32(ptr nonnull %x)
   ret void
 }
+
+
+define i32 @test_simple(ptr %a) sanitize_hwaddress {
+; CHECK-LABEL: define i32 @test_simple
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = and i64 [[TMP3]], 63
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP1]], 57
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP4]], 63
+; CHECK-NEXT:    [[BUF_SROA_0:%.*]] = alloca { i8, [15 x i8] }, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[BUF_SROA_0]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], -9079256848778919937
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP5]], 57
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[BUF_SROA_0_HWASAN:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[BUF_SROA_0]], i8 [[TMP10]], i64 16)
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[BUF_SROA_0_HWASAN]] to i64
+; CHECK-NEXT:    call void @__hwasan_store1(i64 [[TMP11]])
+; CHECK-NEXT:    store volatile i8 0, ptr [[BUF_SROA_0_HWASAN]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[BUF_SROA_0]], i8 [[TMP12]], i64 16)
+; CHECK-NEXT:    ret i32 0
+;
+; INLINE-LABEL: define i32 @test_simple
+; INLINE-SAME: (ptr [[A:%.*]]) #[[ATTR0]] personality ptr @__hwasan_personality_thunk {
+; INLINE-NEXT:  entry:
+; INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; INLINE-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; INLINE-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; INLINE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; INLINE-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @test_simple to i64), [[TMP5]]
+; INLINE-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; INLINE-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; INLINE-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; INLINE-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; INLINE-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; INLINE-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; INLINE-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; INLINE-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; INLINE-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; INLINE-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; INLINE-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP4]], 57
+; INLINE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP15]], 63
+; INLINE-NEXT:    [[BUF_SROA_0:%.*]] = alloca { i8, [15 x i8] }, align 16
+; INLINE-NEXT:    [[TMP16:%.*]] = xor i64 [[TMP2]], 0
+; INLINE-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[BUF_SROA_0]] to i64
+; INLINE-NEXT:    [[TMP18:%.*]] = and i64 [[TMP17]], -9079256848778919937
+; INLINE-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP16]], 57
+; INLINE-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; INLINE-NEXT:    [[BUF_SROA_0_HWASAN:%.*]] = inttoptr i64 [[TMP20]] to ptr
+; INLINE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP16]] to i8
+; INLINE-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[BUF_SROA_0]] to i64
+; INLINE-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], -9079256848778919937
+; INLINE-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP23]], 4
+; INLINE-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP24]]
+; INLINE-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i32 0
+; INLINE-NEXT:    store i8 1, ptr [[TMP26]], align 1
+; INLINE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[BUF_SROA_0]], i32 15
+; INLINE-NEXT:    store i8 [[TMP21]], ptr [[TMP27]], align 1
+; INLINE-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[BUF_SROA_0_HWASAN]] to i64
+; INLINE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 57
+; INLINE-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i8
+; INLINE-NEXT:    [[TMP31:%.*]] = and i64 [[TMP28]], -9079256848778919937
+; INLINE-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 4
+; INLINE-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP32]]
+; INLINE-NEXT:    [[TMP34:%.*]] = load i8, ptr [[TMP33]], align 1
+; INLINE-NEXT:    [[TMP35:%.*]] = icmp ne i8 [[TMP30]], [[TMP34]]
+; INLINE-NEXT:    br i1 [[TMP35]], label [[TMP36:%.*]], label [[TMP50:%.*]], !prof [[PROF1:![0-9]+]]
+; INLINE:       36:
+; INLINE-NEXT:    [[TMP37:%.*]] = icmp ugt i8 [[TMP34]], 15
+; INLINE-NEXT:    br i1 [[TMP37]], label [[TMP38:%.*]], label [[TMP39:%.*]], !prof [[PROF1]]
+; INLINE:       38:
+; INLINE-NEXT:    call void asm sideeffect "int3\0Anopl 80([[RAX:%.*]])", "{rdi}"(i64 [[TMP28]])
+; INLINE-NEXT:    unreachable
+; INLINE:       39:
+; INLINE-NEXT:    [[TMP40:%.*]] = and i64 [[TMP28]], 15
+; INLINE-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP40]] to i8
+; INLINE-NEXT:    [[TMP42:%.*]] = add i8 [[TMP41]], 0
+; INLINE-NEXT:    [[TMP43:%.*]] = icmp uge i8 [[TMP42]], [[TMP34]]
+; INLINE-NEXT:    br i1 [[TMP43]], label [[TMP38]], label [[TMP44:%.*]], !prof [[PROF1]]
+; INLINE:       44:
+; INLINE-NEXT:    [[TMP45:%.*]] = or i64 [[TMP31]], 15
+; INLINE-NEXT:    [[TMP46:%.*]] = inttoptr i64 [[TMP45]] to ptr
+; INLINE-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
+; INLINE-NEXT:    [[TMP48:%.*]] = icmp ne i8 [[TMP30]], [[TMP47]]
+; INLINE-NEXT:    br i1 [[TMP48]], label [[TMP38]], label [[TMP49:%.*]], !prof [[PROF1]]
+; INLINE:       49:
+; INLINE-NEXT:    br label [[TMP50]]
+; INLINE:       50:
+; INLINE-NEXT:    store volatile i8 0, ptr [[BUF_SROA_0_HWASAN]], align 4
+; INLINE-NEXT:    [[TMP51:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; INLINE-NEXT:    [[TMP52:%.*]] = ptrtoint ptr [[BUF_SROA_0]] to i64
+; INLINE-NEXT:    [[TMP53:%.*]] = and i64 [[TMP52]], -9079256848778919937
+; INLINE-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP53]], 4
+; INLINE-NEXT:    [[TMP55:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP54]]
+; INLINE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP55]], i8 [[TMP51]], i64 1, i1 false)
+; INLINE-NEXT:    ret i32 0
+;
+entry:
+  %buf.sroa.0 = alloca i8, align 4
+  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  store volatile i8 0, ptr %buf.sroa.0, align 4
+  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  ret i32 0
+}
+
+; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+
+; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
index 8949f62fbeb4d..49b55139fb980 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,13 +7,17 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
-; CHECK-LABEL: @atomicrmw(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %ptr to i64
+; CHECK-LABEL: define void @atomicrmw
+; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    call void @__hwasan_store8(i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[PTR]], i64 1 seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
 
-; CHECK: atomicrmw add ptr %ptr, i64 1 seq_cst
-; CHECK: ret void
 
 entry:
   %0 = atomicrmw add ptr %ptr, i64 1 seq_cst
@@ -20,13 +25,17 @@ entry:
 }
 
 define void @cmpxchg(ptr %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
-; CHECK-LABEL: @cmpxchg(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %ptr to i64
+; CHECK-LABEL: define void @cmpxchg
+; CHECK-SAME: (ptr [[PTR:%.*]], i64 [[COMPARE_TO:%.*]], i64 [[NEW_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    call void @__hwasan_store8(i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[PTR]], i64 [[COMPARE_TO]], i64 [[NEW_VALUE]] seq_cst seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
 
-; CHECK: cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
-; CHECK: ret void
 
 entry:
   %0 = cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
index 0c4b1a8f2482a..db56303379026 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
@@ -1,22 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ; Generic code is covered by ../basic.ll, only the x86_64 specific code is
 ; tested here.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,RECOVER
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=ABORT
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=RECOVER
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-instrument-with-calls=0 -S | FileCheck %s  --check-prefixes=ABORT-INLINE
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-instrument-with-calls=0 -S | FileCheck %s  --check-prefixes=RECOVER-INLINE
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define i8 @test_load8
+; ABORT-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_load1(i64 [[TMP0]])
+; ABORT-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-NEXT:    ret i8 [[B]]
+;
+; RECOVER-LABEL: define i8 @test_load8
+; RECOVER-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_load1_noabort(i64 [[TMP0]])
+; RECOVER-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-NEXT:    ret i8 [[B]]
+;
+; ABORT-INLINE-LABEL: define i8 @test_load8
+; ABORT-INLINE-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-INLINE-NEXT:  entry:
+; ABORT-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; ABORT-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; ABORT-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; ABORT-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-INLINE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 57
+; ABORT-INLINE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; ABORT-INLINE-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; ABORT-INLINE-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; ABORT-INLINE-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; ABORT-INLINE-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; ABORT-INLINE-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP26:%.*]], !prof [[PROF1:![0-9]+]]
+; ABORT-INLINE:       12:
+; ABORT-INLINE-NEXT:    [[TMP13:%.*]] = icmp ugt i8 [[TMP10]], 15
+; ABORT-INLINE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       14:
+; ABORT-INLINE-NEXT:    call void asm sideeffect "int3\0Anopl 64([[RAX:%.*]])", "{rdi}"(i64 [[TMP4]])
+; ABORT-INLINE-NEXT:    unreachable
+; ABORT-INLINE:       15:
+; ABORT-INLINE-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], 15
+; ABORT-INLINE-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i8
+; ABORT-INLINE-NEXT:    [[TMP18:%.*]] = add i8 [[TMP17]], 0
+; ABORT-INLINE-NEXT:    [[TMP19:%.*]] = icmp uge i8 [[TMP18]], [[TMP10]]
+; ABORT-INLINE-NEXT:    br i1 [[TMP19]], label [[TMP14]], label [[TMP20:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       20:
+; ABORT-INLINE-NEXT:    [[TMP21:%.*]] = or i64 [[TMP7]], 15
+; ABORT-INLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
+; ABORT-INLINE-NEXT:    [[TMP24:%.*]] = icmp ne i8 [[TMP6]], [[TMP23]]
+; ABORT-INLINE-NEXT:    br i1 [[TMP24]], label [[TMP14]], label [[TMP25:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       25:
+; ABORT-INLINE-NEXT:    br label [[TMP26]]
+; ABORT-INLINE:       26:
+; ABORT-INLINE-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-INLINE-NEXT:    ret i8 [[B]]
+;
+; RECOVER-INLINE-LABEL: define i8 @test_load8
+; RECOVER-INLINE-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-INLINE-NEXT:  entry:
+; RECOVER-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; RECOVER-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; RECOVER-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; RECOVER-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-INLINE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 57
+; RECOVER-INLINE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; RECOVER-INLINE-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; RECOVER-INLINE-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; RECOVER-INLINE-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; RECOVER-INLINE-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; RECOVER-INLINE-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP26:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-INLINE:       12:
+; RECOVER-INLINE-NEXT:    [[TMP13:%.*]] = icmp ugt i8 [[TMP10]], 15
+; RECOVER-INLINE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; RECOVER-INLINE:       14:
+; RECOVER-INLINE-NEXT:    call void asm sideeffect "int3\0Anopl 96([[RAX:%.*]])", "{rdi}"(i64 [[TMP4]])
+; RECOVER-INLINE-NEXT:    br label [[TMP25:%.*]]
+; RECOVER-INLINE:       15:
+; RECOVER-INLINE-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], 15
+; RECOVER-INLINE-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i8
+; RECOVER-INLINE-NEXT:    [[TMP18:%.*]] = add i8 [[TMP17]], 0
+; RECOVER-INLINE-NEXT:    [[TMP19:%.*]] = icmp uge i8 [[TMP18]], [[TMP10]]
+; RECOVER-INLINE-NEXT:    br i1 [[TMP19]], label [[TMP14]], label [[TMP20:%.*]], !prof [[PROF1]]
+; RECOVER-INLINE:       20:
+; RECOVER-INLINE-NEXT:    [[TMP21:%.*]] = or i64 [[TMP7]], 15
+; RECOVER-INLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
+; RECOVER-INLINE-NEXT:    [[TMP24:%.*]] = icmp ne i8 [[TMP6]], [[TMP23]]
+; RECOVER-INLINE-NEXT:    br i1 [[TMP24]], label [[TMP14]], label [[TMP25]], !prof [[PROF1]]
+; RECOVER-INLINE:       25:
+; RECOVER-INLINE-NEXT:    br label [[TMP26]]
+; RECOVER-INLINE:       26:
+; RECOVER-INLINE-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-INLINE-NEXT:    ret i8 [[B]]
+;
 
-; ABORT: call void @__hwasan_load1(i64 %[[A]])
-; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 
-; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4
-; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, ptr %a, align 4
@@ -24,14 +121,52 @@ entry:
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define i40 @test_load40
+; ABORT-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-NEXT:    ret i40 [[B]]
+;
+; RECOVER-LABEL: define i40 @test_load40
+; RECOVER-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-NEXT:    ret i40 [[B]]
+;
+; ABORT-INLINE-LABEL: define i40 @test_load40
+; ABORT-INLINE-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-INLINE-NEXT:  entry:
+; ABORT-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; ABORT-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; ABORT-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; ABORT-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-INLINE-NEXT:    call void @__hwasan_loadN(i64 [[TMP4]], i64 5)
+; ABORT-INLINE-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-INLINE-NEXT:    ret i40 [[B]]
+;
+; RECOVER-INLINE-LABEL: define i40 @test_load40
+; RECOVER-INLINE-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-INLINE-NEXT:  entry:
+; RECOVER-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; RECOVER-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; RECOVER-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; RECOVER-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-INLINE-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP4]], i64 5)
+; RECOVER-INLINE-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-INLINE-NEXT:    ret i40 [[B]]
+;
 
-; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
-; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, ptr %a, align 4
@@ -39,14 +174,108 @@ entry:
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store8(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define void @test_store8
+; ABORT-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_store1(i64 [[TMP0]])
+; ABORT-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-NEXT:    ret void
+;
+; RECOVER-LABEL: define void @test_store8
+; RECOVER-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_store1_noabort(i64 [[TMP0]])
+; RECOVER-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-NEXT:    ret void
+;
+; ABORT-INLINE-LABEL: define void @test_store8
+; ABORT-INLINE-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-INLINE-NEXT:  entry:
+; ABORT-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; ABORT-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; ABORT-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; ABORT-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-INLINE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 57
+; ABORT-INLINE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; ABORT-INLINE-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; ABORT-INLINE-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; ABORT-INLINE-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; ABORT-INLINE-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; ABORT-INLINE-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       12:
+; ABORT-INLINE-NEXT:    [[TMP13:%.*]] = icmp ugt i8 [[TMP10]], 15
+; ABORT-INLINE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       14:
+; ABORT-INLINE-NEXT:    call void asm sideeffect "int3\0Anopl 80([[RAX:%.*]])", "{rdi}"(i64 [[TMP4]])
+; ABORT-INLINE-NEXT:    unreachable
+; ABORT-INLINE:       15:
+; ABORT-INLINE-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], 15
+; ABORT-INLINE-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i8
+; ABORT-INLINE-NEXT:    [[TMP18:%.*]] = add i8 [[TMP17]], 0
+; ABORT-INLINE-NEXT:    [[TMP19:%.*]] = icmp uge i8 [[TMP18]], [[TMP10]]
+; ABORT-INLINE-NEXT:    br i1 [[TMP19]], label [[TMP14]], label [[TMP20:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       20:
+; ABORT-INLINE-NEXT:    [[TMP21:%.*]] = or i64 [[TMP7]], 15
+; ABORT-INLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
+; ABORT-INLINE-NEXT:    [[TMP24:%.*]] = icmp ne i8 [[TMP6]], [[TMP23]]
+; ABORT-INLINE-NEXT:    br i1 [[TMP24]], label [[TMP14]], label [[TMP25:%.*]], !prof [[PROF1]]
+; ABORT-INLINE:       25:
+; ABORT-INLINE-NEXT:    br label [[TMP26]]
+; ABORT-INLINE:       26:
+; ABORT-INLINE-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-INLINE-NEXT:    ret void
+;
+; RECOVER-INLINE-LABEL: define void @test_store8
+; RECOVER-INLINE-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-INLINE-NEXT:  entry:
+; RECOVER-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; RECOVER-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; RECOVER-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; RECOVER-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-INLINE-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 57
+; RECOVER-INLINE-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
+; RECOVER-INLINE-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP7]], 4
+; RECOVER-INLINE-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
+; RECOVER-INLINE-NEXT:    [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; RECOVER-INLINE-NEXT:    [[TMP11:%.*]] = icmp ne i8 [[TMP6]], [[TMP10]]
+; RECOVER-INLINE-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP26:%.*]], !prof [[PROF1]]
+; RECOVER-INLINE:       12:
+; RECOVER-INLINE-NEXT:    [[TMP13:%.*]] = icmp ugt i8 [[TMP10]], 15
+; RECOVER-INLINE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP15:%.*]], !prof [[PROF1]]
+; RECOVER-INLINE:       14:
+; RECOVER-INLINE-NEXT:    call void asm sideeffect "int3\0Anopl 112([[RAX:%.*]])", "{rdi}"(i64 [[TMP4]])
+; RECOVER-INLINE-NEXT:    br label [[TMP25:%.*]]
+; RECOVER-INLINE:       15:
+; RECOVER-INLINE-NEXT:    [[TMP16:%.*]] = and i64 [[TMP4]], 15
+; RECOVER-INLINE-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i8
+; RECOVER-INLINE-NEXT:    [[TMP18:%.*]] = add i8 [[TMP17]], 0
+; RECOVER-INLINE-NEXT:    [[TMP19:%.*]] = icmp uge i8 [[TMP18]], [[TMP10]]
+; RECOVER-INLINE-NEXT:    br i1 [[TMP19]], label [[TMP14]], label [[TMP20:%.*]], !prof [[PROF1]]
+; RECOVER-INLINE:       20:
+; RECOVER-INLINE-NEXT:    [[TMP21:%.*]] = or i64 [[TMP7]], 15
+; RECOVER-INLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP23:%.*]] = load i8, ptr [[TMP22]], align 1
+; RECOVER-INLINE-NEXT:    [[TMP24:%.*]] = icmp ne i8 [[TMP6]], [[TMP23]]
+; RECOVER-INLINE-NEXT:    br i1 [[TMP24]], label [[TMP14]], label [[TMP25]], !prof [[PROF1]]
+; RECOVER-INLINE:       25:
+; RECOVER-INLINE-NEXT:    br label [[TMP26]]
+; RECOVER-INLINE:       26:
+; RECOVER-INLINE-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-INLINE-NEXT:    ret void
+;
 
-; ABORT: call void @__hwasan_store1(i64 %[[A]])
-; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
 
-; CHECK: store i8 %b, ptr %a, align 4
-; CHECK: ret void
 
 entry:
   store i8 %b, ptr %a, align 4
@@ -54,14 +283,52 @@ entry:
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define void @test_store40
+; ABORT-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-NEXT:    ret void
+;
+; RECOVER-LABEL: define void @test_store40
+; RECOVER-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-NEXT:    ret void
+;
+; ABORT-INLINE-LABEL: define void @test_store40
+; ABORT-INLINE-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-INLINE-NEXT:  entry:
+; ABORT-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; ABORT-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; ABORT-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; ABORT-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-INLINE-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 5)
+; ABORT-INLINE-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-INLINE-NEXT:    ret void
+;
+; RECOVER-INLINE-LABEL: define void @test_store40
+; RECOVER-INLINE-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-INLINE-NEXT:  entry:
+; RECOVER-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; RECOVER-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; RECOVER-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; RECOVER-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-INLINE-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP4]], i64 5)
+; RECOVER-INLINE-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-INLINE-NEXT:    ret void
+;
 
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: store i40 %b, ptr %a
-; CHECK: ret void
 
 entry:
   store i40 %b, ptr %a, align 4
@@ -69,14 +336,52 @@ entry:
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store_unaligned(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define void @test_store_unaligned
+; ABORT-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-NEXT:    ret void
+;
+; RECOVER-LABEL: define void @test_store_unaligned
+; RECOVER-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-NEXT:    ret void
+;
+; ABORT-INLINE-LABEL: define void @test_store_unaligned
+; ABORT-INLINE-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-INLINE-NEXT:  entry:
+; ABORT-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; ABORT-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; ABORT-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; ABORT-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; ABORT-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; ABORT-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-INLINE-NEXT:    call void @__hwasan_storeN(i64 [[TMP4]], i64 8)
+; ABORT-INLINE-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-INLINE-NEXT:    ret void
+;
+; RECOVER-INLINE-LABEL: define void @test_store_unaligned
+; RECOVER-INLINE-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-INLINE-NEXT:  entry:
+; RECOVER-INLINE-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; RECOVER-INLINE-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -9079256848778919937
+; RECOVER-INLINE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; RECOVER-INLINE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; RECOVER-INLINE-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; RECOVER-INLINE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-INLINE-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP4]], i64 8)
+; RECOVER-INLINE-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-INLINE-NEXT:    ret void
+;
 
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
 
-; CHECK: store i64 %b, ptr %a, align 4
-; CHECK: ret void
 
 entry:
   store i64 %b, ptr %a, align 4
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll
index fe2f9e1d9ab74..51d34ce5b3882 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -6,9 +7,71 @@ target triple = "aarch64--linux-android"
 declare void @use(ptr, ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-  ; CHECK: alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 48
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 44
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr i64 [[TMP3]], 56
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], -1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 72057594037927935
+; CHECK-NEXT:    [[TMP21:%.*]] = shl i64 [[TMP18]], 56
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP24:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 72057594037927935
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP23]], i64 1, i1 false)
+; CHECK-NEXT:    [[Y:%.*]] = alloca i8, i64 16, align 16
+; CHECK-NEXT:    [[TMP28:%.*]] = xor i64 [[TMP4]], 128
+; CHECK-NEXT:    [[TMP29:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 72057594037927935
+; CHECK-NEXT:    [[TMP31:%.*]] = shl i64 [[TMP28]], 56
+; CHECK-NEXT:    [[TMP32:%.*]] = or i64 [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[Y_HWASAN:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT:    [[TMP33:%.*]] = trunc i64 [[TMP28]] to i8
+; CHECK-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 72057594037927935
+; CHECK-NEXT:    [[TMP36:%.*]] = lshr i64 [[TMP35]], 4
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP36]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP37]], i8 [[TMP33]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use(ptr [[X_HWASAN]], ptr [[Y_HWASAN]])
+; CHECK-NEXT:    [[TMP38:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP39:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP40:%.*]] = and i64 [[TMP39]], 72057594037927935
+; CHECK-NEXT:    [[TMP41:%.*]] = lshr i64 [[TMP40]], 4
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP41]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP42]], i8 [[TMP38]], i64 1, i1 false)
+; CHECK-NEXT:    [[TMP43:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP44:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP45:%.*]] = and i64 [[TMP44]], 72057594037927935
+; CHECK-NEXT:    [[TMP46:%.*]] = lshr i64 [[TMP45]], 4
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP46]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP47]], i8 [[TMP43]], i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
   %x = alloca i8, i64 4
-  ; CHECK: alloca i8, i64 16, align 16
   %y = alloca i8, i64 16
   call void @use(ptr %x, ptr %y)
   ret void
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
index c6d5d8e8be598..9e9ed50d35daf 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test that alloca instrumentation with old API levels does not use short granules.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -8,9 +9,51 @@ target triple = "aarch64--linux-android"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[X_TAG:[^ ]*]] = trunc i64 {{.*}} to i8
-; CHECK: call void @llvm.memset.p0.i64(ptr align 1 {{.*}}, i8 %[[X_TAG]], i64 1, i1 false)
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 48
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 44
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr i64 [[TMP3]], 56
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], -1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 72057594037927935
+; CHECK-NEXT:    [[TMP21:%.*]] = shl i64 [[TMP18]], 56
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP24:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 72057594037927935
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP23]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP29:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 72057594037927935
+; CHECK-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 [[TMP28]], i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
   %x = alloca i32, align 4
   call void @use32(ptr nonnull %x)
   ret void
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
index 8c386f2e6a0fb..0ef09321e41ad 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation when tags are generated by HWASan function.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s
@@ -8,12 +9,53 @@ target triple = "aarch64--linux-android"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
-; CHECK: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
-; CHECK: %[[B:[^ ]*]] = ptrtoint ptr %x to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[A]], 56
-; CHECK: or i64 %[[B]], %[[C]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP6]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = call i8 @__hwasan_generate_tag()
+; CHECK-NEXT:    [[TMP18:%.*]] = zext i8 [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 72057594037927935
+; CHECK-NEXT:    [[TMP21:%.*]] = shl i64 [[TMP18]], 56
+; CHECK-NEXT:    [[TMP22:%.*]] = or i64 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP22]] to ptr
+; CHECK-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP24:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = and i64 [[TMP24]], 72057594037927935
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP26]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP23]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP29:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP30:%.*]] = and i64 [[TMP29]], 72057594037927935
+; CHECK-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP31]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 [[TMP28]], i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %x = alloca i32, align 4
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
index 7a26ae8af425a..4248c3e3a8ac9 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
@@ -1,8 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ZERO-BASED-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,UAR-TAGS
+; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ZERO-BASED-SHADOW
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android10000"
@@ -10,44 +10,76 @@ target triple = "aarch64--linux-android10000"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress !dbg !15 {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
-
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
-; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-
-; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
-; CHECK: %[[E:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to ptr
-; CHECK: %[[X_SHADOW_GEP:[^ ]*]] = getelementptr i8, ptr %[[X_SHADOW]], i32 0
-; CHECK: store i8 4, ptr %[[X_SHADOW_GEP]]
-; CHECK: %[[X_I8_GEP:[^ ]*]] = getelementptr i8, ptr %[[X]], i32 15
-; CHECK: store i8 %[[X_TAG2]], ptr %[[X_I8_GEP]]
-; CHECK: call void @llvm.dbg.value(
-; CHECK-SAME: metadata !DIArgList(ptr %[[X]], ptr %[[X]])
-; CHECK-SAME: metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0,
-; CHECK: call void @use32(ptr nonnull %[[X_HWASAN]])
-
-; UAR-TAGS: %[[BASE_TAG_COMPL:[^ ]*]] = xor i64 %[[BASE_TAG]], 255
-; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
-; CHECK: %[[E2:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F2]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to ptr
-; NO-UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 0, i64 1, i1 false)
-; UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 %[[X_TAG_UAR]], i64 1, i1 false)
-; CHECK: ret void
-
-
+; DYNAMIC-SHADOW-LABEL: define void @test_alloca
+; DYNAMIC-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; DYNAMIC-SHADOW-NEXT:  entry:
+; DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; DYNAMIC-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; DYNAMIC-SHADOW-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; DYNAMIC-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927935, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 72057594037927935, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 4, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP11]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 4, ptr [[TMP13]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 [[TMP8]], ptr [[TMP14]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8, !dbg [[DBG14:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72057594037927935, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP17]], 4, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP18]], !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP19]], i8 [[TMP15]], i64 1, i1 false), !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
+; ZERO-BASED-SHADOW-LABEL: define void @test_alloca
+; ZERO-BASED-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; ZERO-BASED-SHADOW-NEXT:  entry:
+; ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; ZERO-BASED-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; ZERO-BASED-SHADOW-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; ZERO-BASED-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927935, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]], !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 72057594037927935, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 4, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 4, ptr [[TMP13]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 [[TMP8]], ptr [[TMP14]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8, !dbg [[DBG14:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 72057594037927935, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP17]], 4, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP19]], i8 [[TMP15]], i64 1, i1 false), !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
 entry:
   %x = alloca i32, align 4
   call void @llvm.dbg.value(metadata !DIArgList(ptr %x, ptr %x), metadata !22, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_deref)), !dbg !21
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
index 37d4cb77286ca..93b9678f1d5d3 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,10 +7,14 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
-; CHECK-LABEL: @atomicrmw(
-; CHECK: call void @llvm.hwasan.check.memaccess({{.*}}, ptr %ptr, i32 19)
-; CHECK: atomicrmw add ptr %ptr, i64 1 seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @atomicrmw
+; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i64 1 seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = atomicrmw add ptr %ptr, i64 1 seq_cst
@@ -17,10 +22,14 @@ entry:
 }
 
 define void @cmpxchg(ptr %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
-; CHECK-LABEL: @cmpxchg(
-; CHECK: call void @llvm.hwasan.check.memaccess({{.*}}, ptr %ptr, i32 19)
-; CHECK: cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @cmpxchg
+; CHECK-SAME: (ptr [[PTR:%.*]], i64 [[COMPARE_TO:%.*]], i64 [[NEW_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg ptr [[PTR]], i64 [[COMPARE_TO]], i64 [[NEW_VALUE]] seq_cst seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
index 7b0097e866c79..4831c0c47bb4c 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test that the old outlined check is used with old API levels.
 
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,8 +7,13 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; CHECK: call void @llvm.hwasan.check.memaccess(ptr {{.*}}, ptr {{.*}}, i32 0)
+; CHECK-LABEL: define i8 @test_load8
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
index 30bbe642904aa..3b0ec5880920e 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-DYNAMIC-SHADOW
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.used = appending global [1 x ptr] [ptr @hwasan.module_ctor]
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @hwasan.module_ctor, ptr @hwasan.module_ctor }]
@@ -12,49 +13,104 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android10000"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 0
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 0)
-
-; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4
-; CHECK: ret i8 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2336", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2336", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i8, ptr %a, align 4
@@ -62,49 +118,104 @@ entry:
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "brk #2337", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 1
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 1)
-
-; CHECK: %[[G:[^ ]*]] = load i16, ptr %a, align 4
-; CHECK: ret i16 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2337", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2337", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i16, ptr %a, align 4
@@ -112,25 +223,99 @@ entry:
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 2)
-
-; CHECK: %[[G:[^ ]*]] = load i32, ptr %a, align 4
-; CHECK: ret i32 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2338", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2338", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+
+
+
 
 entry:
   %b = load i32, ptr %a, align 4
@@ -138,25 +323,99 @@ entry:
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 3)
-
-; CHECK: %[[G:[^ ]*]] = load i64, ptr %a, align 8
-; CHECK: ret i64 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2339", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2339", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+
+
+
 
 entry:
   %b = load i64, ptr %a, align 8
@@ -164,25 +423,99 @@ entry:
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 4)
-
-; CHECK: %[[G:[^ ]*]] = load i128, ptr %a, align 16
-; CHECK: ret i128 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2340", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2340", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+
+
+
 
 entry:
   %b = load i128, ptr %a, align 16
@@ -190,12 +523,42 @@ entry:
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
-; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
-; CHECK: ret i40 %[[B]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
 
 entry:
   %b = load i40, ptr %a, align 4
@@ -203,25 +566,99 @@ entry:
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 16)
-
-; CHECK: store i8 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2352", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2352", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i8 %b, ptr %a, align 4
@@ -229,25 +666,99 @@ entry:
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 17)
-
-; CHECK: store i16 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2353", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2353", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i16 %b, ptr %a, align 4
@@ -255,25 +766,99 @@ entry:
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 18)
-
-; CHECK: store i32 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2354", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2354", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i32 %b, ptr %a, align 4
@@ -281,25 +866,99 @@ entry:
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 19)
-
-; CHECK: store i64 %b, ptr %a, align 8
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2355", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2355", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i64 %b, ptr %a, align 8
@@ -307,25 +966,99 @@ entry:
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 20)
-
-; CHECK: store i128 %b, ptr %a, align 16
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2356", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2356", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i128 %b, ptr %a, align 16
@@ -333,12 +1066,42 @@ entry:
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
-; CHECK: store i40 %b, ptr %a
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i40 %b, ptr %a, align 4
@@ -346,12 +1109,42 @@ entry:
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store_unaligned(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
-; CHECK: store i64 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i64 %b, ptr %a, align 4
@@ -359,33 +1152,108 @@ entry:
 }
 
 define i8 @test_load_noattr(ptr %a) {
-; CHECK-LABEL: @test_load_noattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_noattr
+; CHECK-SAME: (ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_notmyattr(ptr %a) sanitize_address {
-; CHECK-LABEL: @test_load_notmyattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_notmyattr
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load_addrspace(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr addrspace(256) %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_addrspace
+; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr addrspace(256) %a, align 4
   ret i8 %b
@@ -393,9 +1261,7 @@ entry:
 
 ; CHECK: declare void @__hwasan_init()
 
-; CHECK:      define internal void @hwasan.module_ctor() #[[#ATTR:]] comdat {
 ; CHECK-NEXT:   call void @__hwasan_init()
 ; CHECK-NEXT:   ret void
 ; CHECK-NEXT: }
 
-; CHECK:      attributes #[[#ATTR]] = { nounwind }
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll b/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
index ab52691a320a9..9e9fceb5eb472 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test allocas with multiple lifetime ends, as frequently seen for exception
 ; handling.
 ;
-; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
@@ -14,6 +15,86 @@ declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 declare i32 @__gxx_personality_v0(...)
 
 define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define void @test
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP6]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 72057594037927935
+; CHECK-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP17]], 56
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 72057594037927935
+; CHECK-NEXT:    [[TMP25:%.*]] = lshr i64 [[TMP24]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP25]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP26]], i8 [[TMP22]], i64 1, i1 false)
+; CHECK-NEXT:    invoke void @mayFail(ptr [[X_HWASAN]])
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 72057594037927935
+; CHECK-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP30]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP32:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    [[TMP33:%.*]] = call i64 @llvm.read_register.i64(metadata [[META2:![0-9]+]])
+; CHECK-NEXT:    call void @__hwasan_handle_vfork(i64 [[TMP33]])
+; CHECK-NEXT:    [[TMP34:%.*]] = extractvalue { ptr, i32 } [[TMP32]], 0
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EXN_SLOT]], i32 19)
+; CHECK-NEXT:    store ptr [[TMP34]], ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    [[TMP35:%.*]] = extractvalue { ptr, i32 } [[TMP32]], 1
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EHSELECTOR_SLOT]], i32 18)
+; CHECK-NEXT:    store i32 [[TMP35]], ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    call void @onExcept(ptr [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP37:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP38:%.*]] = and i64 [[TMP37]], 72057594037927935
+; CHECK-NEXT:    [[TMP39:%.*]] = lshr i64 [[TMP38]], 4
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP39]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP40]], i8 [[TMP36]], i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    br label [[EH_RESUME:%.*]]
+; CHECK:       eh.resume:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EXN_SLOT]], i32 3)
+; CHECK-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EHSELECTOR_SLOT]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } undef, ptr [[EXN]], 0
+; CHECK-NEXT:    [[LPAD_VAL1:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
+; CHECK-NEXT:    resume { ptr, i32 } [[LPAD_VAL1]]
+;
 entry:
   %x = alloca i32, align 8
   %exn.slot = alloca ptr, align 8
@@ -22,19 +103,11 @@ entry:
   invoke void @mayFail(ptr %x) to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
-; CHECK: invoke.cont:
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  ret void
 
   call void @llvm.lifetime.end.p0(i64 8, ptr %x)
   ret void
 
 lpad:                                             ; preds = %entry
-; CHECK: lpad
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  br label %eh.resume
 
   %0 = landingpad { ptr, i32 }
   cleanup
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
index 92ff51ac7c184..7652587ce4ec0 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test kernel hwasan instrumentation for alloca.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-kernel=1 -S | FileCheck %s
@@ -8,19 +9,39 @@ target triple = "aarch64--linux-android"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; CHECK-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP4]], -72057594037927936
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], 72057594037927935
+; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP10]], -72057594037927936
+; CHECK-NEXT:    [[TMP12:%.*]] = lshr i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP13]], i8 [[TMP9]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP15]], -72057594037927936
+; CHECK-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP16]], 4
+; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP18]], i8 [[TMP14]], i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
 
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
-; CHECK: %[[D:[^ ]*]] = or i64 %[[C]], 72057594037927935
-; CHECK: %[[E:[^ ]*]] = and i64 %[[X1]], %[[D]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[E]] to ptr
 
 entry:
   %x = alloca i32, align 4
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll b/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll
index 5a405a32a2a75..f8e0364ba19f4 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll
@@ -1,45 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --global-value-regex "![0-9]+" --version 2
 ; Test -hwasan-with-ifunc flag.
 ;
 ; RUN: opt -passes=hwasan -S < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS-SLOT,CHECK-HISTORY,CHECK-HISTORY-TLS-SLOT,CHECK-HISTORY-TLS
+; RUN:     FileCheck %s
 ; RUN: opt -passes=hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=instr < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS-SLOT,CHECK-HISTORY,CHECK-HISTORY-TLS-SLOT,CHECK-HISTORY-TLS
+; RUN:     FileCheck %s --check-prefixes=NOIFUNC-TLS-HISTORY
 ; RUN: opt -passes=hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=none < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN:     FileCheck %s --check-prefixes=NOIFUNC-TLS-NOHISTORY
 ; RUN: opt -passes=hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL,CHECK-NOHISTORY
+; RUN:     FileCheck %s --check-prefixes=NOIFUNC-NOTLS
 ; RUN: opt -passes=hwasan -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN:     FileCheck %s --check-prefixes=IFUNC-NOTLS
 ; RUN: opt -passes=hwasan -S -mtriple=aarch64-fuchsia < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-ZERO-OFFSET,CHECK-SHORT-GRANULES,CHECK-HISTORY,CHECK-HWASAN-TLS,CHECK-HISTORY-HWASAN-TLS,CHECK-HISTORY-TLS
+; RUN:     FileCheck %s --check-prefixes=FUCHSIA
 ; RUN: opt -passes=hwasan -S -mtriple=aarch64-fuchsia -hwasan-record-stack-history=libcall < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-HISTORY,CHECK-HISTORY-LIBCALL
+; RUN:     FileCheck %s --check-prefixes=FUCHSIA-LIBCALL
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android22"
 
-; CHECK-IFUNC: @__hwasan_shadow = external global [0 x i8]
-; CHECK-NOIFUNC: @__hwasan_shadow_memory_dynamic_address = external global i64
 
 define i32 @test_load(ptr %a) sanitize_hwaddress {
 ; First instrumentation in the function must be to load the dynamic shadow
 ; address into a local variable.
-; CHECK-LABEL: @test_load
-; CHECK: entry:
-
-; CHECK-NOGLOBAL:   %[[A:[^ ]*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
-; CHECK-NOGLOBAL:   @llvm.hwasan.check.memaccess(ptr %[[A]]
-; CHECK-ZERO-OFFSET:  %[[A:[^ ]*]] = call ptr asm "", "=r,0"(ptr null)
-; CHECK-SHORT-GRANULES:  @llvm.hwasan.check.memaccess.shortgranules(ptr %[[A]]
-
-; CHECK-GLOBAL: load ptr, ptr @__hwasan_shadow_memory_dynamic_address
-
 ; "store i64" is only used to update stack history (this input IR intentionally does not use any i64)
 ; W/o any allocas, the history is not updated, even if it is enabled explicitly with -hwasan-record-stack-history=1
-; CHECK-NOT: store i64
-
-; CHECK: ret i32
-
+; CHECK-LABEL: define i32 @test_load
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[X]]
+;
+; NOIFUNC-TLS-HISTORY-LABEL: define i32 @test_load
+; NOIFUNC-TLS-HISTORY-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOIFUNC-TLS-HISTORY-NEXT:  entry:
+; NOIFUNC-TLS-HISTORY-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOIFUNC-TLS-HISTORY-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; NOIFUNC-TLS-HISTORY-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; NOIFUNC-TLS-HISTORY-NEXT:    ret i32 [[X]]
+;
+; NOIFUNC-TLS-NOHISTORY-LABEL: define i32 @test_load
+; NOIFUNC-TLS-NOHISTORY-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOIFUNC-TLS-NOHISTORY-NEXT:  entry:
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOIFUNC-TLS-NOHISTORY-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; NOIFUNC-TLS-NOHISTORY-NEXT:    ret i32 [[X]]
+;
+; NOIFUNC-NOTLS-LABEL: define i32 @test_load
+; NOIFUNC-NOTLS-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOIFUNC-NOTLS-NEXT:  entry:
+; NOIFUNC-NOTLS-NEXT:    [[TMP0:%.*]] = load ptr, ptr @__hwasan_shadow_memory_dynamic_address, align 8
+; NOIFUNC-NOTLS-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP0]], ptr [[A]], i32 2)
+; NOIFUNC-NOTLS-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; NOIFUNC-NOTLS-NEXT:    ret i32 [[X]]
+;
+; IFUNC-NOTLS-LABEL: define i32 @test_load
+; IFUNC-NOTLS-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; IFUNC-NOTLS-NEXT:  entry:
+; IFUNC-NOTLS-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; IFUNC-NOTLS-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; IFUNC-NOTLS-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; IFUNC-NOTLS-NEXT:    ret i32 [[X]]
+;
+; FUCHSIA-LABEL: define i32 @test_load
+; FUCHSIA-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; FUCHSIA-NEXT:  entry:
+; FUCHSIA-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FUCHSIA-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; FUCHSIA-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; FUCHSIA-NEXT:    ret i32 [[X]]
+;
+; FUCHSIA-LIBCALL-LABEL: define i32 @test_load
+; FUCHSIA-LIBCALL-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; FUCHSIA-LIBCALL-NEXT:  entry:
+; FUCHSIA-LIBCALL-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FUCHSIA-LIBCALL-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; FUCHSIA-LIBCALL-NEXT:    [[X:%.*]] = load i32, ptr [[A]], align 4
+; FUCHSIA-LIBCALL-NEXT:    ret i32 [[X]]
+;
 entry:
   %x = load i32, ptr %a, align 4
   ret i32 %x
@@ -50,54 +91,278 @@ declare void @use(ptr %p)
 define void @test_alloca() sanitize_hwaddress {
 ; First instrumentation in the function must be to load the dynamic shadow
 ; address into a local variable.
-; CHECK-LABEL: @test_alloca
-; CHECK: entry:
-
-; CHECK-IFUNC:   %[[A:[^ ]*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
-; CHECK-IFUNC:   getelementptr i8, ptr %[[A]]
-
-; CHECK-GLOBAL: load ptr, ptr @__hwasan_shadow_memory_dynamic_address
-
-; CHECK-TLS-SLOT:   %[[A:[^ ]*]] = call ptr @llvm.thread.pointer()
-; CHECK-TLS-SLOT:   %[[B:[^ ]*]] = getelementptr i8, ptr %[[A]], i32 48
-; CHECK-TLS-SLOT:   %[[D:[^ ]*]] = load i64, ptr %[[B]]
-; CHECK-TLS-SLOT:   %[[E:[^ ]*]] = ashr i64 %[[D]], 3
-; CHECK-HWASAN-TLS: %[[D:[^ ]*]] = load i64, ptr @__hwasan_tls, align 8
-; CHECK-HWASAN-TLS: %[[E:[^ ]*]] = ashr i64 %[[D]], 3
-
-; CHECK-NOHISTORY-NOT: store i64
-
 ; When watching stack history, all code paths attempt to get PC and SP and mix them together.
-; CHECK-HISTORY: %[[PC:[^ ]*]] = call i64 @llvm.read_register.i64(metadata [[MD:![0-9]*]])
-; CHECK-HISTORY: %[[SP0:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK-HISTORY: %[[SP1:[^ ]*]] = ptrtoint ptr %[[SP0]] to i64
-; CHECK-HISTORY: %[[SP2:[^ ]*]] = shl i64 %[[SP1]], 44
-
-; CHECK-HISTORY: %[[MIX:[^ ]*]] = or i64 %[[PC]], %[[SP2]]
-; CHECK-HISTORY-TLS: %[[PTR:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; CHECK-HISTORY-TLS: store i64 %[[MIX]], ptr %[[PTR]]
-; CHECK-HISTORY-TLS: %[[D1:[^ ]*]] = ashr i64 %[[D]], 56
-; CHECK-HISTORY-TLS: %[[D2:[^ ]*]] = shl nuw nsw i64 %[[D1]], 12
-; CHECK-HISTORY-TLS: %[[D3:[^ ]*]] = xor i64 %[[D2]], -1
-; CHECK-HISTORY-TLS: %[[D4:[^ ]*]] = add i64 %[[D]], 8
-; CHECK-HISTORY-TLS: %[[D5:[^ ]*]] = and i64 %[[D4]], %[[D3]]
-; CHECK-HISTORY-TLS-SLOT: store i64 %[[D5]], ptr %[[B]]
-; CHECK-HISTORY-HWASAN-TLS: store i64 %[[D5]], ptr @__hwasan_tls
-; CHECK-HISTORY-LIBCALL: call void @__hwasan_add_frame_record(i64 %[[MIX]])
-
-; CHECK-TLS:   %[[F:[^ ]*]] = or i64 %[[D]], 4294967295
-; CHECK-TLS:   = add i64 %[[F]], 1
-
-; CHECK-HISTORY-LIBCALL: %[[E:hwasan.stack.base.tag]] = xor
-; CHECK-HISTORY: = xor i64 %[[E]], 0
-
-; CHECK-NOHISTORY-NOT: store i64
-
-
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP6]], 56
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 72057594037927935
+; CHECK-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP17]], 56
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 72057594037927935
+; CHECK-NEXT:    [[TMP25:%.*]] = lshr i64 [[TMP24]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP25]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP26]], i8 [[TMP22]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use(ptr [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 72057594037927935
+; CHECK-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP30]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
+; NOIFUNC-TLS-HISTORY-LABEL: define void @test_alloca
+; NOIFUNC-TLS-HISTORY-SAME: () #[[ATTR0]] {
+; NOIFUNC-TLS-HISTORY-NEXT:  entry:
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; NOIFUNC-TLS-HISTORY-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; NOIFUNC-TLS-HISTORY-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; NOIFUNC-TLS-HISTORY-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; NOIFUNC-TLS-HISTORY-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP6]], 56
+; NOIFUNC-TLS-HISTORY-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP3]], 0
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 72057594037927935
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP17]], 56
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; NOIFUNC-TLS-HISTORY-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP17]] to i8
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 72057594037927935
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP25:%.*]] = lshr i64 [[TMP24]], 4
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP25]]
+; NOIFUNC-TLS-HISTORY-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP26]], i8 [[TMP22]], i64 1, i1 false)
+; NOIFUNC-TLS-HISTORY-NEXT:    call void @use(ptr [[X_HWASAN]])
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP27:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 72057594037927935
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
+; NOIFUNC-TLS-HISTORY-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP30]]
+; NOIFUNC-TLS-HISTORY-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 1, i1 false)
+; NOIFUNC-TLS-HISTORY-NEXT:    ret void
+;
+; NOIFUNC-TLS-NOHISTORY-LABEL: define void @test_alloca
+; NOIFUNC-TLS-NOHISTORY-SAME: () #[[ATTR0]] {
+; NOIFUNC-TLS-NOHISTORY-NEXT:  entry:
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927935
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]]
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 72057594037927935
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 4
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP11]]
+; NOIFUNC-TLS-NOHISTORY-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP12]], i8 [[TMP8]], i64 1, i1 false)
+; NOIFUNC-TLS-NOHISTORY-NEXT:    call void @use(ptr [[X_HWASAN]])
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP13:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 72057594037927935
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 4
+; NOIFUNC-TLS-NOHISTORY-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP16]]
+; NOIFUNC-TLS-NOHISTORY-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP17]], i8 [[TMP13]], i64 1, i1 false)
+; NOIFUNC-TLS-NOHISTORY-NEXT:    ret void
+;
+; NOIFUNC-NOTLS-LABEL: define void @test_alloca
+; NOIFUNC-NOTLS-SAME: () #[[ATTR0]] {
+; NOIFUNC-NOTLS-NEXT:  entry:
+; NOIFUNC-NOTLS-NEXT:    [[TMP0:%.*]] = load ptr, ptr @__hwasan_shadow_memory_dynamic_address, align 8
+; NOIFUNC-NOTLS-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; NOIFUNC-NOTLS-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; NOIFUNC-NOTLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 20
+; NOIFUNC-NOTLS-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP2]], [[TMP3]]
+; NOIFUNC-NOTLS-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP2]], 56
+; NOIFUNC-NOTLS-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; NOIFUNC-NOTLS-NEXT:    [[TMP4:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; NOIFUNC-NOTLS-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-NOTLS-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], 72057594037927935
+; NOIFUNC-NOTLS-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP4]], 56
+; NOIFUNC-NOTLS-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; NOIFUNC-NOTLS-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; NOIFUNC-NOTLS-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP4]] to i8
+; NOIFUNC-NOTLS-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-NOTLS-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], 72057594037927935
+; NOIFUNC-NOTLS-NEXT:    [[TMP12:%.*]] = lshr i64 [[TMP11]], 4
+; NOIFUNC-NOTLS-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[TMP12]]
+; NOIFUNC-NOTLS-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP13]], i8 [[TMP9]], i64 1, i1 false)
+; NOIFUNC-NOTLS-NEXT:    call void @use(ptr [[X_HWASAN]])
+; NOIFUNC-NOTLS-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; NOIFUNC-NOTLS-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[X]] to i64
+; NOIFUNC-NOTLS-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], 72057594037927935
+; NOIFUNC-NOTLS-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP16]], 4
+; NOIFUNC-NOTLS-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[TMP17]]
+; NOIFUNC-NOTLS-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP18]], i8 [[TMP14]], i64 1, i1 false)
+; NOIFUNC-NOTLS-NEXT:    ret void
+;
+; IFUNC-NOTLS-LABEL: define void @test_alloca
+; IFUNC-NOTLS-SAME: () #[[ATTR0]] {
+; IFUNC-NOTLS-NEXT:  entry:
+; IFUNC-NOTLS-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; IFUNC-NOTLS-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; IFUNC-NOTLS-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; IFUNC-NOTLS-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; IFUNC-NOTLS-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; IFUNC-NOTLS-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP1]], 56
+; IFUNC-NOTLS-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; IFUNC-NOTLS-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; IFUNC-NOTLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64
+; IFUNC-NOTLS-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927935
+; IFUNC-NOTLS-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP3]], 56
+; IFUNC-NOTLS-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]]
+; IFUNC-NOTLS-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; IFUNC-NOTLS-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8
+; IFUNC-NOTLS-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64
+; IFUNC-NOTLS-NEXT:    [[TMP10:%.*]] = and i64 [[TMP9]], 72057594037927935
+; IFUNC-NOTLS-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP10]], 4
+; IFUNC-NOTLS-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP11]]
+; IFUNC-NOTLS-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP12]], i8 [[TMP8]], i64 1, i1 false)
+; IFUNC-NOTLS-NEXT:    call void @use(ptr [[X_HWASAN]])
+; IFUNC-NOTLS-NEXT:    [[TMP13:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; IFUNC-NOTLS-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[X]] to i64
+; IFUNC-NOTLS-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 72057594037927935
+; IFUNC-NOTLS-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 4
+; IFUNC-NOTLS-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP16]]
+; IFUNC-NOTLS-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP17]], i8 [[TMP13]], i64 1, i1 false)
+; IFUNC-NOTLS-NEXT:    ret void
+;
+; FUCHSIA-LABEL: define void @test_alloca
+; FUCHSIA-SAME: () #[[ATTR0]] personality ptr @__hwasan_personality_thunk {
+; FUCHSIA-NEXT:  entry:
+; FUCHSIA-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FUCHSIA-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; FUCHSIA-NEXT:    [[TMP1:%.*]] = ashr i64 [[TMP0]], 3
+; FUCHSIA-NEXT:    [[TMP2:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; FUCHSIA-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; FUCHSIA-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; FUCHSIA-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; FUCHSIA-NEXT:    [[TMP6:%.*]] = or i64 [[TMP2]], [[TMP5]]
+; FUCHSIA-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; FUCHSIA-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; FUCHSIA-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; FUCHSIA-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; FUCHSIA-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; FUCHSIA-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; FUCHSIA-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; FUCHSIA-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; FUCHSIA-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP4]], 56
+; FUCHSIA-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; FUCHSIA-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP1]], 0
+; FUCHSIA-NEXT:    [[TMP14:%.*]] = ptrtoint ptr [[X]] to i64
+; FUCHSIA-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 72057594037927935
+; FUCHSIA-NEXT:    [[TMP16:%.*]] = shl i64 [[TMP13]], 56
+; FUCHSIA-NEXT:    [[TMP17:%.*]] = or i64 [[TMP15]], [[TMP16]]
+; FUCHSIA-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; FUCHSIA-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP13]] to i8
+; FUCHSIA-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; FUCHSIA-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 72057594037927935
+; FUCHSIA-NEXT:    [[TMP21:%.*]] = lshr i64 [[TMP20]], 4
+; FUCHSIA-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; FUCHSIA-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[TMP22]], i32 0
+; FUCHSIA-NEXT:    store i8 4, ptr [[TMP23]], align 1
+; FUCHSIA-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; FUCHSIA-NEXT:    store i8 [[TMP18]], ptr [[TMP24]], align 1
+; FUCHSIA-NEXT:    call void @use(ptr [[X_HWASAN]])
+; FUCHSIA-NEXT:    [[TMP25:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; FUCHSIA-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[X]] to i64
+; FUCHSIA-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; FUCHSIA-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; FUCHSIA-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; FUCHSIA-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; FUCHSIA-NEXT:    ret void
+;
+; FUCHSIA-LIBCALL-LABEL: define void @test_alloca
+; FUCHSIA-LIBCALL-SAME: () #[[ATTR0]] personality ptr @__hwasan_personality_thunk {
+; FUCHSIA-LIBCALL-NEXT:  entry:
+; FUCHSIA-LIBCALL-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; FUCHSIA-LIBCALL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; FUCHSIA-LIBCALL-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; FUCHSIA-LIBCALL-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; FUCHSIA-LIBCALL-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 44
+; FUCHSIA-LIBCALL-NEXT:    [[TMP4:%.*]] = or i64 [[TMP0]], [[TMP3]]
+; FUCHSIA-LIBCALL-NEXT:    call void @__hwasan_add_frame_record(i64 [[TMP4]])
+; FUCHSIA-LIBCALL-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP2]], 20
+; FUCHSIA-LIBCALL-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP2]], [[TMP5]]
+; FUCHSIA-LIBCALL-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP2]], 56
+; FUCHSIA-LIBCALL-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; FUCHSIA-LIBCALL-NEXT:    [[TMP6:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; FUCHSIA-LIBCALL-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X]] to i64
+; FUCHSIA-LIBCALL-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], 72057594037927935
+; FUCHSIA-LIBCALL-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 56
+; FUCHSIA-LIBCALL-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; FUCHSIA-LIBCALL-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; FUCHSIA-LIBCALL-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; FUCHSIA-LIBCALL-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[X]] to i64
+; FUCHSIA-LIBCALL-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 72057594037927935
+; FUCHSIA-LIBCALL-NEXT:    [[TMP14:%.*]] = lshr i64 [[TMP13]], 4
+; FUCHSIA-LIBCALL-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; FUCHSIA-LIBCALL-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i32 0
+; FUCHSIA-LIBCALL-NEXT:    store i8 4, ptr [[TMP16]], align 1
+; FUCHSIA-LIBCALL-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; FUCHSIA-LIBCALL-NEXT:    store i8 [[TMP11]], ptr [[TMP17]], align 1
+; FUCHSIA-LIBCALL-NEXT:    call void @use(ptr [[X_HWASAN]])
+; FUCHSIA-LIBCALL-NEXT:    [[TMP18:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; FUCHSIA-LIBCALL-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; FUCHSIA-LIBCALL-NEXT:    [[TMP20:%.*]] = and i64 [[TMP19]], 72057594037927935
+; FUCHSIA-LIBCALL-NEXT:    [[TMP21:%.*]] = lshr i64 [[TMP20]], 4
+; FUCHSIA-LIBCALL-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; FUCHSIA-LIBCALL-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP22]], i8 [[TMP18]], i64 1, i1 false)
+; FUCHSIA-LIBCALL-NEXT:    ret void
+;
 entry:
   %x = alloca i32, align 4
   call void @use(ptr %x)
   ret void
 }
 
-; CHECK-HISTORY: [[MD]] = !{!"pc"}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll b/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll
index 74cd9720d5f1d..253e976a7d608 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll
@@ -14,7 +14,7 @@ target triple = "aarch64-unknown-linux-android29"
 ; REQUIRES: aarch64-registered-target
 
 ; COLOR: sub	sp, sp, #240
-; NOCOLOR: sub	sp, sp, #368
+; NOCOLOR: sub	sp, sp, #384
 
 define i32 @myCall_w2(i32 %in) sanitize_hwaddress {
 entry:
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
index d0f152d55d569..4bb846bff2745 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-use-after-scope -S < %s | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-android29"
@@ -8,12 +9,71 @@ target triple = "aarch64-unknown-linux-android29"
 declare void @may_jump()
 
 define dso_local noundef i1 @_Z6targetv() sanitize_hwaddress {
+; CHECK-LABEL: define dso_local noundef i1 @_Z6targetv
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP6]], 56
+; CHECK-NEXT:    [[BUF:%.*]] = alloca [4096 x i8], align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP18]], 72057594037927935
+; CHECK-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP17]], 56
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[BUF_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 72057594037927935
+; CHECK-NEXT:    [[TMP25:%.*]] = lshr i64 [[TMP24]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP25]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP26]], i8 [[TMP22]], i64 256, i1 false)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @setjmp(ptr noundef @jbuf)
+; CHECK-NEXT:    switch i32 [[CALL]], label [[WHILE_BODY:%.*]] [
+; CHECK-NEXT:    i32 1, label [[RETURN:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       while.body:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr @stackbuf, i32 19)
+; CHECK-NEXT:    store ptr [[BUF_HWASAN]], ptr @stackbuf, align 8
+; CHECK-NEXT:    call void @may_jump()
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i1 [ true, [[WHILE_BODY]] ], [ true, [[SW_BB1]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; CHECK-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP29:%.*]] = and i64 [[TMP28]], 72057594037927935
+; CHECK-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP30]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 256, i1 false)
+; CHECK-NEXT:    ret i1 [[RETVAL_0]]
+;
 entry:
   %buf = alloca [4096 x i8], align 1
   %call = call i32 @setjmp(ptr noundef @jbuf)
   switch i32 %call, label %while.body [
-    i32 1, label %return
-    i32 2, label %sw.bb1
+  i32 1, label %return
+  i32 2, label %sw.bb1
   ]
 
 sw.bb1:                                           ; preds = %entry
@@ -29,8 +89,6 @@ while.body:                                       ; preds = %entry
   call void @llvm.lifetime.end.p0(i64 4096, ptr nonnull %buf) #10
   br label %return
 
-; CHECK-LABEL: return:
-; CHECK: void @llvm.memset.p0.i64({{.*}}, i8 0, i64 256, i1 false)
 return:                                           ; preds = %entry, %while.body, %sw.bb1
   %retval.0 = phi i1 [ true, %while.body ], [ true, %sw.bb1 ], [ false, %entry ]
   ret i1 %retval.0
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll
index e7681138cf722..105ce5197fa6e 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll
@@ -25,44 +25,56 @@
 define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-LABEL: @standard_lifetime(
 ; X86-SCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-SCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-SCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-SCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-SCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-SCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-SCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-SCOPE-NEXT:    br label [[TMP7:%.*]]
-; X86-SCOPE:       7:
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP8]], i64 16)
-; X86-SCOPE-NEXT:    [[TMP9:%.*]] = tail call i1 (...) @cond()
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP7]]
-; X86-SCOPE:       10:
+; X86-SCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-SCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-SCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-SCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-SCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-SCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-SCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-SCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-SCOPE-NEXT:    br label [[TMP11:%.*]]
+; X86-SCOPE:       11:
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP12]], i64 16)
+; X86-SCOPE-NEXT:    [[TMP13:%.*]] = tail call i1 (...) @cond()
+; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    br i1 [[TMP13]], label [[TMP15:%.*]], label [[TMP11]]
+; X86-SCOPE:       15:
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; X86-SCOPE-NEXT:    ret i32 0
 ;
 ; X86-NOSCOPE-LABEL: @standard_lifetime(
 ; X86-NOSCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
-; X86-NOSCOPE-NEXT:    br label [[TMP8:%.*]]
-; X86-NOSCOPE:       8:
-; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = tail call i1 (...) @cond()
-; X86-NOSCOPE-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP8]]
-; X86-NOSCOPE:       10:
+; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-NOSCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-NOSCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
+; X86-NOSCOPE-NEXT:    br label [[TMP12:%.*]]
+; X86-NOSCOPE:       12:
+; X86-NOSCOPE-NEXT:    [[TMP13:%.*]] = tail call i1 (...) @cond()
+; X86-NOSCOPE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP12]]
+; X86-NOSCOPE:       14:
 ; X86-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP15:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP15]], i64 16)
 ; X86-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SCOPE-LABEL: @standard_lifetime(
@@ -86,29 +98,34 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SCOPE-NEXT:    br label [[TMP24:%.*]]
-; AARCH64-SCOPE:       24:
+; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SCOPE-NEXT:    br label [[TMP25:%.*]]
+; AARCH64-SCOPE:       25:
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP26]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP27]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP28]], i8 [[TMP25]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP29]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP30]], i8 [[TMP26]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    br i1 [[TMP29]], label [[TMP33:%.*]], label [[TMP24]]
-; AARCH64-SCOPE:       33:
+; AARCH64-SCOPE-NEXT:    br i1 [[TMP31]], label [[TMP37:%.*]], label [[TMP25]]
+; AARCH64-SCOPE:       37:
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SCOPE-NEXT:    ret i32 0
 ;
@@ -133,28 +150,33 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
-; AARCH64-NOSCOPE-NEXT:    br label [[TMP28:%.*]]
-; AARCH64-NOSCOPE:       28:
-; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = tail call i1 (...) @cond()
-; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP29]], label [[TMP30:%.*]], label [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    br label [[TMP30:%.*]]
 ; AARCH64-NOSCOPE:       30:
+; AARCH64-NOSCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
+; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP31]], label [[TMP32:%.*]], label [[TMP30]]
+; AARCH64-NOSCOPE:       32:
 ; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-NOSCOPE-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP32]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP33]], i8 0, i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP36:%.*]] = lshr i64 [[TMP35]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP37:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP36]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP37]], i8 [[TMP33]], i64 1, i1 false)
 ; AARCH64-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-SCOPE-LABEL: @standard_lifetime(
@@ -178,32 +200,37 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP24:%.*]]
-; AARCH64-SHORT-SCOPE:       24:
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP25:%.*]]
+; AARCH64-SHORT-SCOPE:       25:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP26]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP27]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP28]], i32 0
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP29]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP30]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP29]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP30]], i32 0
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP31]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP26]], ptr [[TMP32]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP31]], label [[TMP35:%.*]], label [[TMP24]]
-; AARCH64-SHORT-SCOPE:       35:
+; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP33]], label [[TMP39:%.*]], label [[TMP25]]
+; AARCH64-SHORT-SCOPE:       39:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
 ;
@@ -228,31 +255,36 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP30:%.*]]
-; AARCH64-SHORT-NOSCOPE:       30:
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP31]], label [[TMP32:%.*]], label [[TMP30]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP32:%.*]]
 ; AARCH64-SHORT-NOSCOPE:       32:
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP33:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP33]], label [[TMP34:%.*]], label [[TMP32]]
+; AARCH64-SHORT-NOSCOPE:       34:
 ; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP34:%.*]] = lshr i64 [[TMP33]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP34]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP35]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP38:%.*]] = lshr i64 [[TMP37]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP39:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP38]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP39]], i8 [[TMP35]], i64 1, i1 false)
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
@@ -274,44 +306,56 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noinline sanitize_hwaddress {
 ; X86-SCOPE-LABEL: @standard_lifetime_optnone(
 ; X86-SCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-SCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-SCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-SCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-SCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-SCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-SCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-SCOPE-NEXT:    br label [[TMP7:%.*]]
-; X86-SCOPE:       7:
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP8]], i64 16)
-; X86-SCOPE-NEXT:    [[TMP9:%.*]] = tail call i1 (...) @cond()
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP7]]
-; X86-SCOPE:       10:
+; X86-SCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-SCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-SCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-SCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-SCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-SCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-SCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-SCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-SCOPE-NEXT:    br label [[TMP11:%.*]]
+; X86-SCOPE:       11:
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP12]], i64 16)
+; X86-SCOPE-NEXT:    [[TMP13:%.*]] = tail call i1 (...) @cond()
+; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    br i1 [[TMP13]], label [[TMP15:%.*]], label [[TMP11]]
+; X86-SCOPE:       15:
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; X86-SCOPE-NEXT:    ret i32 0
 ;
 ; X86-NOSCOPE-LABEL: @standard_lifetime_optnone(
 ; X86-NOSCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
-; X86-NOSCOPE-NEXT:    br label [[TMP8:%.*]]
-; X86-NOSCOPE:       8:
-; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = tail call i1 (...) @cond()
-; X86-NOSCOPE-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP8]]
-; X86-NOSCOPE:       10:
+; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-NOSCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-NOSCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
+; X86-NOSCOPE-NEXT:    br label [[TMP12:%.*]]
+; X86-NOSCOPE:       12:
+; X86-NOSCOPE-NEXT:    [[TMP13:%.*]] = tail call i1 (...) @cond()
+; X86-NOSCOPE-NEXT:    br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP12]]
+; X86-NOSCOPE:       14:
 ; X86-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP15:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP15]], i64 16)
 ; X86-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SCOPE-LABEL: @standard_lifetime_optnone(
@@ -335,29 +379,34 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SCOPE-NEXT:    br label [[TMP24:%.*]]
-; AARCH64-SCOPE:       24:
+; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SCOPE-NEXT:    br label [[TMP25:%.*]]
+; AARCH64-SCOPE:       25:
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP26]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP27]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP28]], i8 [[TMP25]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP29]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP30]], i8 [[TMP26]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    br i1 [[TMP29]], label [[TMP33:%.*]], label [[TMP24]]
-; AARCH64-SCOPE:       33:
+; AARCH64-SCOPE-NEXT:    br i1 [[TMP31]], label [[TMP37:%.*]], label [[TMP25]]
+; AARCH64-SCOPE:       37:
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SCOPE-NEXT:    ret i32 0
 ;
@@ -382,28 +431,33 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
-; AARCH64-NOSCOPE-NEXT:    br label [[TMP28:%.*]]
-; AARCH64-NOSCOPE:       28:
-; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = tail call i1 (...) @cond()
-; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP29]], label [[TMP30:%.*]], label [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    br label [[TMP30:%.*]]
 ; AARCH64-NOSCOPE:       30:
+; AARCH64-NOSCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
+; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP31]], label [[TMP32:%.*]], label [[TMP30]]
+; AARCH64-NOSCOPE:       32:
 ; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-NOSCOPE-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP32]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP33]], i8 0, i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP35:%.*]] = and i64 [[TMP34]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP36:%.*]] = lshr i64 [[TMP35]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP37:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP36]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP37]], i8 [[TMP33]], i64 1, i1 false)
 ; AARCH64-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-SCOPE-LABEL: @standard_lifetime_optnone(
@@ -427,32 +481,37 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP24:%.*]]
-; AARCH64-SHORT-SCOPE:       24:
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP25:%.*]]
+; AARCH64-SHORT-SCOPE:       25:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP26]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP27]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP28]], i32 0
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP29]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP30]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP29]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP30]], i32 0
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP31]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP26]], ptr [[TMP32]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP31]], label [[TMP35:%.*]], label [[TMP24]]
-; AARCH64-SHORT-SCOPE:       35:
+; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP33]], label [[TMP39:%.*]], label [[TMP25]]
+; AARCH64-SHORT-SCOPE:       39:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
 ;
@@ -477,31 +536,36 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP30:%.*]]
-; AARCH64-SHORT-NOSCOPE:       30:
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP31]], label [[TMP32:%.*]], label [[TMP30]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP32:%.*]]
 ; AARCH64-SHORT-NOSCOPE:       32:
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP33:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP33]], label [[TMP34:%.*]], label [[TMP32]]
+; AARCH64-SHORT-NOSCOPE:       34:
 ; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP34:%.*]] = lshr i64 [[TMP33]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP34]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP35]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP37:%.*]] = and i64 [[TMP36]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP38:%.*]] = lshr i64 [[TMP37]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP39:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP38]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP39]], i8 [[TMP35]], i64 1, i1 false)
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
@@ -523,34 +587,46 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-LABEL: @multiple_lifetimes(
 ; X86-SCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-SCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-SCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-SCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-SCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-SCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-SCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-SCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
+; X86-SCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-SCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-SCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-SCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-SCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-SCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-SCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-SCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-SCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-SCOPE-NEXT:    [[TMP12:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP12]], i64 16)
 ; X86-SCOPE-NEXT:    ret i32 0
 ;
 ; X86-NOSCOPE-LABEL: @multiple_lifetimes(
 ; X86-NOSCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-NOSCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-NOSCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
 ; X86-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; X86-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP12:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP12]], i64 16)
 ; X86-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SCOPE-LABEL: @multiple_lifetimes(
@@ -574,24 +650,29 @@ define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP29]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP30]], i8 0, i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = and i64 [[TMP31]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 [[TMP30]], i64 1, i1 false)
 ; AARCH64-SCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-NOSCOPE-LABEL: @multiple_lifetimes(
@@ -615,24 +696,29 @@ define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
 ; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP28]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP29]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP30]], i8 0, i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    [[TMP30:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP32:%.*]] = and i64 [[TMP31]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 [[TMP30]], i64 1, i1 false)
 ; AARCH64-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-SCOPE-LABEL: @multiple_lifetimes(
@@ -656,27 +742,32 @@ define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
 ; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-NOSCOPE-LABEL: @multiple_lifetimes(
@@ -700,27 +791,32 @@ define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
 ; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
@@ -738,45 +834,59 @@ define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress
 define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-LABEL: @unreachable_exit(
 ; X86-SCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-SCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-SCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-SCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-SCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-SCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-SCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
-; X86-SCOPE-NEXT:    [[TMP8:%.*]] = tail call i1 (...) @cond()
-; X86-SCOPE-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
-; X86-SCOPE:       9:
+; X86-SCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-SCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-SCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-SCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-SCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-SCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-SCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-SCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
+; X86-SCOPE-NEXT:    [[TMP12:%.*]] = tail call i1 (...) @cond()
+; X86-SCOPE-NEXT:    br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]]
+; X86-SCOPE:       13:
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
 ; X86-SCOPE-NEXT:    ret i32 0
-; X86-SCOPE:       10:
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-SCOPE:       15:
+; X86-SCOPE-NEXT:    [[TMP16:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP16]], i64 16)
 ; X86-SCOPE-NEXT:    ret i32 0
 ;
 ; X86-NOSCOPE-LABEL: @unreachable_exit(
 ; X86-NOSCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
-; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = tail call i1 (...) @cond()
-; X86-NOSCOPE-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
-; X86-NOSCOPE:       9:
+; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-NOSCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-NOSCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP12:%.*]] = tail call i1 (...) @cond()
+; X86-NOSCOPE-NEXT:    br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]]
+; X86-NOSCOPE:       13:
 ; X86-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
 ; X86-NOSCOPE-NEXT:    ret i32 0
-; X86-NOSCOPE:       10:
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-NOSCOPE:       15:
+; X86-NOSCOPE-NEXT:    [[TMP16:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP16]], i64 16)
 ; X86-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SCOPE-LABEL: @unreachable_exit(
@@ -800,33 +910,40 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SCOPE-NEXT:    br i1 [[TMP28]], label [[TMP29:%.*]], label [[TMP33:%.*]]
-; AARCH64-SCOPE:       29:
+; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP37:%.*]]
+; AARCH64-SCOPE:       31:
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    ret i32 0
-; AARCH64-SCOPE:       33:
-; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
 ; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
 ; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 0, i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    ret i32 0
+; AARCH64-SCOPE:       37:
+; AARCH64-SCOPE-NEXT:    [[TMP38:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP39:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP40:%.*]] = and i64 [[TMP39]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP41:%.*]] = lshr i64 [[TMP40]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP42:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP41]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP42]], i8 [[TMP38]], i64 1, i1 false)
 ; AARCH64-SCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-NOSCOPE-LABEL: @unreachable_exit(
@@ -850,32 +967,39 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
-; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = tail call i1 (...) @cond()
-; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP28]], label [[TMP29:%.*]], label [[TMP33:%.*]]
-; AARCH64-NOSCOPE:       29:
+; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
+; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP37:%.*]]
+; AARCH64-NOSCOPE:       31:
 ; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-NOSCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
-; AARCH64-NOSCOPE-NEXT:    ret i32 0
-; AARCH64-NOSCOPE:       33:
-; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
 ; AARCH64-NOSCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
 ; AARCH64-NOSCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 0, i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    ret i32 0
+; AARCH64-NOSCOPE:       37:
+; AARCH64-NOSCOPE-NEXT:    [[TMP38:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP39:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP40:%.*]] = and i64 [[TMP39]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP41:%.*]] = lshr i64 [[TMP40]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP42:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP41]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP42]], i8 [[TMP38]], i64 1, i1 false)
 ; AARCH64-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-SCOPE-LABEL: @unreachable_exit(
@@ -899,36 +1023,43 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP35:%.*]]
-; AARCH64-SHORT-SCOPE:       31:
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP32]], label [[TMP33:%.*]], label [[TMP39:%.*]]
+; AARCH64-SHORT-SCOPE:       33:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
-; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
-; AARCH64-SHORT-SCOPE:       35:
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 72057594037927935
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
+; AARCH64-SHORT-SCOPE:       39:
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP40:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP41:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP42:%.*]] = and i64 [[TMP41]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP43:%.*]] = lshr i64 [[TMP42]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP44:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP43]]
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP44]], i8 [[TMP40]], i64 1, i1 false)
 ; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-NOSCOPE-LABEL: @unreachable_exit(
@@ -952,35 +1083,42 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP35:%.*]]
-; AARCH64-SHORT-NOSCOPE:       31:
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP32:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP32]], label [[TMP33:%.*]], label [[TMP39:%.*]]
+; AARCH64-SHORT-NOSCOPE:       33:
 ; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
-; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
-; AARCH64-SHORT-NOSCOPE:       35:
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP34:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 72057594037927935
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
+; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
+; AARCH64-SHORT-NOSCOPE:       39:
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP40:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP41:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP42:%.*]] = and i64 [[TMP41]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP43:%.*]] = lshr i64 [[TMP42]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP44:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP43]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP44]], i8 [[TMP40]], i64 1, i1 false)
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
@@ -1000,50 +1138,63 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-LABEL: @diamond_lifetime(
 ; X86-SCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-SCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-SCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-SCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-SCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-SCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-SCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
-; X86-SCOPE-NEXT:    [[TMP8:%.*]] = tail call i1 (...) @cond()
-; X86-SCOPE-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
-; X86-SCOPE:       9:
+; X86-SCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-SCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-SCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-SCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-SCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-SCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-SCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-SCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
+; X86-SCOPE-NEXT:    [[TMP12:%.*]] = tail call i1 (...) @cond()
+; X86-SCOPE-NEXT:    br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]]
+; X86-SCOPE:       13:
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    br label [[TMP11:%.*]]
-; X86-SCOPE:       10:
-; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP1]])
-; X86-SCOPE-NEXT:    br label [[TMP11]]
-; X86-SCOPE:       11:
+; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    br label [[TMP17:%.*]]
+; X86-SCOPE:       15:
+; X86-SCOPE-NEXT:    [[TMP16:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP16]], i64 16)
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    br label [[TMP17]]
+; X86-SCOPE:       17:
 ; X86-SCOPE-NEXT:    ret i32 0
 ;
 ; X86-NOSCOPE-LABEL: @diamond_lifetime(
 ; X86-NOSCOPE-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
-; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = alloca { i8, [15 x i8] }, align 16
-; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = call i8 @__hwasan_generate_tag()
-; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP2]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP1]] to i64
-; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 57
-; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]]
-; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 [[TMP7]], i64 16)
-; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = tail call i1 (...) @cond()
-; X86-NOSCOPE-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
-; X86-NOSCOPE:       9:
+; X86-NOSCOPE-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; X86-NOSCOPE-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 57
+; X86-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = and i64 [[TMP3]], 63
+; X86-NOSCOPE-NEXT:    [[TMP4:%.*]] = alloca { i8, [15 x i8] }, align 16
+; X86-NOSCOPE-NEXT:    [[TMP5:%.*]] = call i8 @__hwasan_generate_tag()
+; X86-NOSCOPE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; X86-NOSCOPE-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -9079256848778919937
+; X86-NOSCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
+; X86-NOSCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
+; X86-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; X86-NOSCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
+; X86-NOSCOPE-NEXT:    [[TMP12:%.*]] = tail call i1 (...) @cond()
+; X86-NOSCOPE-NEXT:    br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP14:%.*]]
+; X86-NOSCOPE:       13:
 ; X86-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; X86-NOSCOPE-NEXT:    br label [[TMP11:%.*]]
-; X86-NOSCOPE:       10:
-; X86-NOSCOPE-NEXT:    br label [[TMP11]]
-; X86-NOSCOPE:       11:
-; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP1]], i8 0, i64 16)
+; X86-NOSCOPE-NEXT:    br label [[TMP15:%.*]]
+; X86-NOSCOPE:       14:
+; X86-NOSCOPE-NEXT:    br label [[TMP15]]
+; X86-NOSCOPE:       15:
+; X86-NOSCOPE-NEXT:    [[TMP16:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; X86-NOSCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP16]], i64 16)
 ; X86-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SCOPE-LABEL: @diamond_lifetime(
@@ -1067,37 +1218,44 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; AARCH64-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SCOPE-NEXT:    br i1 [[TMP28]], label [[TMP29:%.*]], label [[TMP33:%.*]]
-; AARCH64-SCOPE:       29:
+; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP37:%.*]]
+; AARCH64-SCOPE:       31:
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SCOPE-NEXT:    [[TMP30:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SCOPE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 4
-; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP31]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP32]], i8 0, i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    br label [[TMP37:%.*]]
-; AARCH64-SCOPE:       33:
-; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP32:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP33:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP33]], 72057594037927935
 ; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
 ; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
-; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 0, i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
 ; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SCOPE-NEXT:    br label [[TMP37]]
+; AARCH64-SCOPE-NEXT:    br label [[TMP43:%.*]]
 ; AARCH64-SCOPE:       37:
+; AARCH64-SCOPE-NEXT:    [[TMP38:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SCOPE-NEXT:    [[TMP39:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SCOPE-NEXT:    [[TMP40:%.*]] = and i64 [[TMP39]], 72057594037927935
+; AARCH64-SCOPE-NEXT:    [[TMP41:%.*]] = lshr i64 [[TMP40]], 4
+; AARCH64-SCOPE-NEXT:    [[TMP42:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP41]]
+; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP42]], i8 [[TMP38]], i64 1, i1 false)
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    br label [[TMP43]]
+; AARCH64-SCOPE:       43:
 ; AARCH64-SCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-NOSCOPE-LABEL: @diamond_lifetime(
@@ -1121,30 +1279,35 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 [[TMP24]], i64 1, i1 false)
-; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = tail call i1 (...) @cond()
-; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP28]], label [[TMP29:%.*]], label [[TMP30:%.*]]
-; AARCH64-NOSCOPE:       29:
-; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-NOSCOPE-NEXT:    br label [[TMP31:%.*]]
-; AARCH64-NOSCOPE:       30:
-; AARCH64-NOSCOPE-NEXT:    br label [[TMP31]]
+; AARCH64-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP29]], i8 [[TMP25]], i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
+; AARCH64-NOSCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP32:%.*]]
 ; AARCH64-NOSCOPE:       31:
-; AARCH64-NOSCOPE-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-NOSCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
-; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
-; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
+; AARCH64-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
+; AARCH64-NOSCOPE-NEXT:    br label [[TMP33:%.*]]
+; AARCH64-NOSCOPE:       32:
+; AARCH64-NOSCOPE-NEXT:    br label [[TMP33]]
+; AARCH64-NOSCOPE:       33:
+; AARCH64-NOSCOPE-NEXT:    [[TMP34:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-NOSCOPE-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-NOSCOPE-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 72057594037927935
+; AARCH64-NOSCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
+; AARCH64-NOSCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
+; AARCH64-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
 ; AARCH64-NOSCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-SCOPE-LABEL: @diamond_lifetime(
@@ -1168,40 +1331,47 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP35:%.*]]
-; AARCH64-SHORT-SCOPE:       31:
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-SCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP32]], label [[TMP33:%.*]], label [[TMP39:%.*]]
+; AARCH64-SHORT-SCOPE:       33:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP33]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP39:%.*]]
-; AARCH64-SHORT-SCOPE:       35:
-; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP34:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP35:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP36:%.*]] = and i64 [[TMP35]], 72057594037927935
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
-; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP39]]
+; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP45:%.*]]
 ; AARCH64-SHORT-SCOPE:       39:
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP40:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP41:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP42:%.*]] = and i64 [[TMP41]], 72057594037927935
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP43:%.*]] = lshr i64 [[TMP42]], 4
+; AARCH64-SHORT-SCOPE-NEXT:    [[TMP44:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP43]]
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP44]], i8 [[TMP40]], i64 1, i1 false)
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP45]]
+; AARCH64-SHORT-SCOPE:       45:
 ; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
 ;
 ; AARCH64-SHORT-NOSCOPE-LABEL: @diamond_lifetime(
@@ -1225,33 +1395,38 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[HWASAN_UAR_TAG:%.*]] = lshr i64 [[TMP7]], 56
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP18:%.*]] = alloca { i8, [15 x i8] }, align 16
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP19:%.*]] = call i8 @__hwasan_generate_tag()
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP20:%.*]] = zext i8 [[TMP19]] to i64
 ; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = shl i64 [[TMP20]], 56
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = or i64 [[TMP21]], [[TMP22]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP23]] to ptr
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP20]] to i8
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP26]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP27]], i32 0
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP28]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
-; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP24]], ptr [[TMP29]], align 1
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = tail call i1 (...) @cond()
-; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP30]], label [[TMP31:%.*]], label [[TMP32:%.*]]
-; AARCH64-SHORT-NOSCOPE:       31:
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
-; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP33:%.*]]
-; AARCH64-SHORT-NOSCOPE:       32:
-; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP33]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP27]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP28]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i32 0
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 1, ptr [[TMP30]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP18]], i32 15
+; AARCH64-SHORT-NOSCOPE-NEXT:    store i8 [[TMP25]], ptr [[TMP31]], align 1
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP32:%.*]] = tail call i1 (...) @cond()
+; AARCH64-SHORT-NOSCOPE-NEXT:    br i1 [[TMP32]], label [[TMP33:%.*]], label [[TMP34:%.*]]
 ; AARCH64-SHORT-NOSCOPE:       33:
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[TMP18]] to i64
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
-; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
-; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 0, i64 1, i1 false)
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
+; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP35:%.*]]
+; AARCH64-SHORT-NOSCOPE:       34:
+; AARCH64-SHORT-NOSCOPE-NEXT:    br label [[TMP35]]
+; AARCH64-SHORT-NOSCOPE:       35:
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP36:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP37:%.*]] = ptrtoint ptr [[TMP18]] to i64
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP38:%.*]] = and i64 [[TMP37]], 72057594037927935
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP39:%.*]] = lshr i64 [[TMP38]], 4
+; AARCH64-SHORT-NOSCOPE-NEXT:    [[TMP40:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP39]]
+; AARCH64-SHORT-NOSCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP40]], i8 [[TMP36]], i64 1, i1 false)
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/basic-kernel.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/basic-kernel.ll
new file mode 100644
index 0000000000000..5a8db5608adf7
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/basic-kernel.ll
@@ -0,0 +1,169 @@
+; RUN: opt < %s -S -mcpu=z13 -msan-kernel=1 -float-abi=soft -passes=msan 2>&1 | FileCheck %s
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
+target triple = "s390x-unknown-linux-gnu"
+
+define void @Store1(ptr %p, i8 %x) sanitize_memory {
+entry:
+  store i8 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store1(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_store_1(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i8 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store2(ptr %p, i16 %x) sanitize_memory {
+entry:
+  store i16 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store2(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_store_2(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i16 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store4(ptr %p, i32 %x) sanitize_memory {
+entry:
+  store i32 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store4(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_store_4(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i32 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store8(ptr %p, i64 %x) sanitize_memory {
+entry:
+  store i64 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store8(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_store_8(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i64 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define void @Store16(ptr %p, i128 %x) sanitize_memory {
+entry:
+  store i128 %x, ptr %p
+  ret void
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Store16(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_store_n(ptr [[META_PTR]], ptr %p, i64 16)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: store i128 {{.+}}, ptr [[SHADOW]]
+; CHECK: ret void
+
+define i8 @Load1(ptr %p) sanitize_memory {
+entry:
+  %0 = load i8, ptr %p
+  ret i8 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load1(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_load_1(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i8, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i8 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i8 {{.+}}
+
+define i16 @Load2(ptr %p) sanitize_memory {
+entry:
+  %0 = load i16, ptr %p
+  ret i16 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load2(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_load_2(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i16, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i16 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i16 {{.+}}
+
+define i32 @Load4(ptr %p) sanitize_memory {
+entry:
+  %0 = load i32, ptr %p
+  ret i32 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load4(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_load_4(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i32, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i32 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i32 {{.+}}
+
+define i64 @Load8(ptr %p) sanitize_memory {
+entry:
+  %0 = load i64, ptr %p
+  ret i64 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load8(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_load_8(ptr [[META_PTR]], ptr %p)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i64, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i64 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i64 {{.+}}
+
+define i128 @Load16(ptr %p) sanitize_memory {
+entry:
+  %0 = load i128, ptr %p
+  ret i128 %0
+}
+
+; CHECK-LABEL: define {{[^@]+}}@Load16(
+; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }
+; CHECK: call void @__msan_metadata_ptr_for_load_n(ptr [[META_PTR]], ptr %p, i64 16)
+; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]]
+; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0
+; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1
+; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i128, ptr [[SHADOW]]
+; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]]
+; CHECK: store i128 [[SHADOW_VAL]], ptr %retval_shadow
+; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin
+; CHECK: ret i128 {{.+}}
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
index de8c9d095e15d..1535fccfc2110 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
@@ -3,9 +3,60 @@
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
 target triple = "s390x-unknown-linux-gnu"
 
-declare i64 @foo(i64 %guard, ...) #0
+%struct.__va_list = type { i64, i64, ptr, ptr }
+declare void @llvm.lifetime.start.p0(i64, ptr)
+declare void @llvm.va_start(ptr)
+declare void @llvm.va_end(ptr)
+declare void @llvm.lifetime.end.p0(i64, ptr)
+
+define i64 @foo(i64 %guard, ...) #1 {
+  %vl = alloca %struct.__va_list
+  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.va_start(ptr %vl)
+  call void @llvm.va_end(ptr %vl)
+  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  ret i64 0
+}
 
-attributes #0 = { "target-features"="+soft-float" "use-soft-float"="true" }
+; CHECK-LABEL: define {{[^@]+}}@foo(
+
+; Callers store variadic arguments' shadow and origins into va_arg_shadow and
+; va_arg_origin. Their layout is: the register save area (160 bytes) followed
+; by the overflow arg area. It does not depend on "packed-stack".
+; Check that callees correctly backup shadow into a local variable.
+
+; CHECK: [[TMP:%.*]] = alloca { ptr, ptr }
+; CHECK: [[OverflowSize:%.*]] = load i64, ptr %va_arg_overflow_size
+; CHECK: [[MetaSize:%.*]] = add i64 160, [[OverflowSize]]
+; CHECK: [[ShadowBackup:%.*]] = alloca {{.*}} [[MetaSize]]
+; CHECK: [[MetaCopySize:%.*]] = call i64 @llvm.umin.i64(i64 [[MetaSize]], i64 800)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[ShadowBackup]], ptr align 8 %va_arg_shadow, i64 [[MetaCopySize]], i1 false)
+; CHECK: [[OverflowBackup:%.*]] = alloca {{.*}} [[MetaSize]]
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[OverflowBackup]], ptr align 8 %va_arg_origin, i64 [[MetaCopySize]], i1 false)
+
+; Check that va_start() correctly copies the shadow backup into the shadow of
+; the va_list. Register save area and overflow arg area are copied separately.
+; Only 56 bytes of the register save area is copied, because of
+; "use-soft-float".
+
+; CHECK: call void @llvm.va_start(ptr %vl)
+; CHECK: [[VlAddr:%.*]] = ptrtoint ptr %vl to i64
+; CHECK: [[RegSaveAreaAddrAddr:%.*]] = add i64 [[VlAddr]], 24
+; CHECK: [[RegSaveAreaAddr:%.*]] = inttoptr i64 [[RegSaveAreaAddrAddr]] to ptr
+; CHECK: [[RegSaveArea:%.*]] = load ptr, ptr [[RegSaveAreaAddr]]
+; CHECK: call void @__msan_metadata_ptr_for_store_1(ptr [[TMP]], ptr [[RegSaveArea]])
+; CHECK: [[RegSaveAreaMeta:%.*]] = load { ptr, ptr }, ptr [[TMP]]
+; CHECK: [[RegSaveAreaShadow:%.*]] = extractvalue { ptr, ptr } [[RegSaveAreaMeta]], 0
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[RegSaveAreaShadow]], ptr align 8 [[ShadowBackup]], i64 56, i1 false)
+; CHECK: [[VlAddr:%.*]] = ptrtoint ptr %vl to i64
+; CHECK: [[OverflowAddrAddr:%.*]] = add i64 [[VlAddr]], 16
+; CHECK: [[OverflowAddr:%.*]] = inttoptr i64 [[OverflowAddrAddr]] to ptr
+; CHECK: [[Overflow:%.*]] = load ptr, ptr [[OverflowAddr]]
+; CHECK: call void @__msan_metadata_ptr_for_store_1(ptr [[TMP]], ptr [[Overflow]])
+; CHECK: [[OverflowMeta:%.*]] = load { ptr, ptr }, ptr [[TMP]]
+; CHECK: [[OverflowShadow:%.*]] = extractvalue { ptr, ptr } [[OverflowMeta]], 0
+; CHECK: [[OverflowShadowBackup:%.*]] = getelementptr i8, ptr [[ShadowBackup]], i32 160
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[OverflowShadow]], ptr align 8 [[OverflowShadowBackup]], i64 [[OverflowSize]], i1 false)
 
 declare i32 @random_i32()
 declare i64 @random_i64()
diff --git a/llvm/test/Instrumentation/MemorySanitizer/avx-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/avx-intrinsics-x86.ll
index 9d3300207aafa..a93f841649282 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/avx-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/avx-intrinsics-x86.ll
@@ -1343,8 +1343,8 @@ define void @movnt_dq(ptr %p, <2 x i64> %a1) nounwind #0 {
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSPROP:%.*]] = or <2 x i64> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    [[A2:%.*]] = add <2 x i64> [[A1:%.*]], <i64 1, i64 1>
-; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[_MSPROP]], <2 x i64> <i64 -1, i64 -1>, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[A3:%.*]] = shufflevector <2 x i64> [[A2]], <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = shufflevector <2 x i64> [[_MSPROP]], <2 x i64> <i64 -1, i64 -1>, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[A3:%.*]] = shufflevector <2 x i64> [[A2]], <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP2]], 0
 ; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP3:%.*]], label [[TMP4:%.*]], !prof [[PROF0]]
 ; CHECK:       3:
diff --git a/llvm/test/Instrumentation/MemorySanitizer/is-fpclass.ll b/llvm/test/Instrumentation/MemorySanitizer/is-fpclass.ll
new file mode 100644
index 0000000000000..e3467d8da3011
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/is-fpclass.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=msan < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: sanitize_memory
+define i1 @not_isfinite_or_zero_f16(half %x) sanitize_memory {
+; CHECK-LABEL: @not_isfinite_or_zero_f16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i16 [[TMP1]], 0
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 615)
+; CHECK-NEXT:    store i1 [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f16(half %x, i32 615)
+  ret i1 %class
+}
+
+; Function Attrs: sanitize_memory
+define <2 x i1> @not_isfinite_or_zero_v2f16_pos0_neg0_vec(<2 x half> %x) sanitize_memory {
+; CHECK-LABEL: @not_isfinite_or_zero_v2f16_pos0_neg0_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i16>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[CLASS:%.*]] = call <2 x i1> @llvm.is.fpclass.v2f16(<2 x half> [[X:%.*]], i32 615)
+; CHECK-NEXT:    store <2 x i1> [[TMP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <2 x i1> [[CLASS]]
+;
+  %class = call <2 x i1> @llvm.is.fpclass.v2f16(<2 x half> %x, i32 615)
+  ret <2 x i1> %class
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i1 @llvm.is.fpclass.f16(half, i32 immarg) #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <2 x i1> @llvm.is.fpclass.v2f16(<2 x half>, i32 immarg) #1
+
diff --git a/llvm/test/LTO/Resolution/X86/protected-def-dllimport.ll b/llvm/test/LTO/Resolution/X86/protected-def-dllimport.ll
new file mode 100644
index 0000000000000..af5958a57d019
--- /dev/null
+++ b/llvm/test/LTO/Resolution/X86/protected-def-dllimport.ll
@@ -0,0 +1,31 @@
+; RUN: llvm-as --disable-verify -o %t0.bc %s
+
+; RUN: llvm-lto2 run -r %t0.bc,__imp_f,l \
+; RUN:               -r %t0.bc,g,p \
+; RUN:               -r %t0.bc,main,x \
+; RUN:               -save-temps -o %t1 %t0.bc
+; RUN: llvm-dis %t1.0.0.preopt.bc -o - | FileCheck %s
+source_filename = "test.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@llvm.used = appending global [2 x ptr] [ ptr @main, ptr @f ], section "llvm.metadata"
+
+$g = comdat any
+@g = global i8 42, comdat, !type !0
+
+; CHECK: declare protected ptr @f()
+define protected dllimport i8* @f() {
+  ret i8* @g
+}
+
+define available_externally i32 @main() {
+  %1 = call i8* @f()
+  %2 = ptrtoint i8* %1 to i32
+  ret i32 %2
+}
+
+!llvm.module.flags = !{!1}
+
+!0 = !{i32 0, !"typeid"}
+!1 = !{i32 1, !"ThinLTO", i32 0}
diff --git a/llvm/test/MC/AArch64/coff-debug.ll b/llvm/test/MC/AArch64/coff-debug.ll
index cbea03592d578..ea1cc3286b042 100644
--- a/llvm/test/MC/AArch64/coff-debug.ll
+++ b/llvm/test/MC/AArch64/coff-debug.ll
@@ -95,7 +95,9 @@ attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-ma
 ; CHECK:       FunctionType: main (0x1002)
 ; CHECK:       CodeOffset: main+0x0
 ; CHECK:       Segment: 0x0
-; CHECK:       Flags [ (0x0)
+; CHECK:       Flags [ (0xC0)
+; CHECK:         HasOptimizedDebugInfo (0x80)
+; CHECK:         IsNoInline (0x40)
 ; CHECK:       ]
 ; CHECK:       DisplayName: main
 ; CHECK:       LinkageName: main
diff --git a/llvm/test/MC/ARM/directive_parsing.s b/llvm/test/MC/ARM/directive_parsing.s
index 90d15ef92d9d5..a8039840ec595 100644
--- a/llvm/test/MC/ARM/directive_parsing.s
+++ b/llvm/test/MC/ARM/directive_parsing.s
@@ -63,7 +63,7 @@
 // CHECK-NOT: [[@LINE+1]]:{{[0-9]+}}: error:
         .personality __gxx_personality_v0 @ EOL COMMENET
 
-// CHECK: [[@LINE+1]]:28: error: unexpected token
+// CHECK: [[#@LINE+1]]:28: error: expected newline
         .setfp  fp, sp, #0 $
 // CHECK-NOT: [[@LINE+1]]:{{[0-9]+}}: error:
         .setfp  fp, sp, #0 @ EOL COMMENT
diff --git a/llvm/test/MC/COFF/addrsig.s b/llvm/test/MC/COFF/addrsig.s
index a181bb06508be..7c3b25d52edae 100644
--- a/llvm/test/MC/COFF/addrsig.s
+++ b/llvm/test/MC/COFF/addrsig.s
@@ -3,7 +3,7 @@
 // CHECK:      Name: .llvm_addrsig
 // CHECK-NEXT: VirtualSize: 0x0
 // CHECK-NEXT: VirtualAddress: 0x0
-// CHECK-NEXT: RawDataSize: 6
+// CHECK-NEXT: RawDataSize: 4
 // CHECK-NEXT: PointerToRawData:
 // CHECK-NEXT: PointerToRelocations: 0x0
 // CHECK-NEXT: PointerToLineNumbers: 0x0
@@ -46,8 +46,6 @@
 // CHECK-NEXT:   Sym: g3 (11)
 // CHECK-NEXT:   Sym: local (10)
 // CHECK-NEXT:   Sym: .data (2)
-// CHECK-NEXT:   Sym: weak_sym (12)
-// CHECK-NEXT:   Sym: .data (2)
 // CHECK-NEXT: ]
 
 .globl g1
@@ -66,10 +64,3 @@ local:
 
 .data
 .Llocal:
-
-.weak weak_sym
-weak_sym:
-.addrsig_sym weak_sym
-
-.set .Lalias_weak_sym, weak_sym
-.addrsig_sym .Lalias_weak_sym
diff --git a/llvm/test/MC/COFF/alias.s b/llvm/test/MC/COFF/alias.s
index cfb44ca1503fe..b6828c658132b 100644
--- a/llvm/test/MC/COFF/alias.s
+++ b/llvm/test/MC/COFF/alias.s
@@ -24,7 +24,7 @@ weak_aliased_to_external = external2
 // CHECK:        0x0 IMAGE_REL_I386_DIR32 external_aliased_to_local
 // CHECK:        0x4 IMAGE_REL_I386_DIR32 external1
 // CHECK:        0x8 IMAGE_REL_I386_DIR32 global_aliased_to_local
-// CHECK:        0xC IMAGE_REL_I386_DIR32 weak_aliased_to_external
+// CHECK:        0xC IMAGE_REL_I386_DIR32 external2
 // CHECK:      ]
 // CHECK:      Symbols [
 // CHECK-NEXT:   Symbol {
diff --git a/llvm/test/MC/COFF/weak-anti-dep.s b/llvm/test/MC/COFF/weak-anti-dep.s
deleted file mode 100644
index dbf5ffc0a35bf..0000000000000
--- a/llvm/test/MC/COFF/weak-anti-dep.s
+++ /dev/null
@@ -1,68 +0,0 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-win32 %s | llvm-readobj --symbols - | FileCheck %s
-
-// CHECK:       Symbol {
-// CHECK-NEXT:    Name: .text
-// CHECK:       Symbol {
-// CHECK-NEXT:    Name: .data
-// CHECK:       Symbol {
-// CHECK-NEXT:    Name: .bss
-
-.weak_anti_dep a
-a = b
-
-// CHECK:       Symbol {
-// CHECK-NEXT:    Name: a
-// CHECK-NEXT:    Value: 0
-// CHECK-NEXT:    Section: IMAGE_SYM_UNDEFINED (0)
-// CHECK-NEXT:    BaseType: Null (0x0)
-// CHECK-NEXT:    ComplexType: Null (0x0)
-// CHECK-NEXT:    StorageClass: WeakExternal (0x69)
-// CHECK-NEXT:    AuxSymbolCount: 1
-// CHECK-NEXT:    AuxWeakExternal {
-// CHECK-NEXT:      Linked: b (8)
-// CHECK-NEXT:      Search: AntiDependency (0x4)
-// CHECK-NEXT:    }
-// CHECK-NEXT:  }
-// CHECK-NEXT:  Symbol {
-// CHECK-NEXT:    Name: b
-// CHECK-NEXT:    Value: 0
-// CHECK-NEXT:    Section: IMAGE_SYM_UNDEFINED (0)
-// CHECK-NEXT:    BaseType: Null (0x0)
-// CHECK-NEXT:    ComplexType: Null (0x0)
-// CHECK-NEXT:    StorageClass: External (0x2)
-// CHECK-NEXT:    AuxSymbolCount: 0
-// CHECK-NEXT:  }
-
-
-.weak_anti_dep r1
-.weak_anti_dep r2
-r1 = r2
-r2 = r1
-
-
-// CHECK:       Symbol {
-// CHECK-NEXT:    Name: r1
-// CHECK-NEXT:    Value: 0
-// CHECK-NEXT:    Section: IMAGE_SYM_UNDEFINED (0)
-// CHECK-NEXT:    BaseType: Null (0x0)
-// CHECK-NEXT:    ComplexType: Null (0x0)
-// CHECK-NEXT:    StorageClass: WeakExternal (0x69)
-// CHECK-NEXT:    AuxSymbolCount: 1
-// CHECK-NEXT:    AuxWeakExternal {
-// CHECK-NEXT:      Linked: r2 (11)
-// CHECK-NEXT:      Search: AntiDependency (0x4)
-// CHECK-NEXT:    }
-// CHECK-NEXT:  }
-// CHECK-NEXT:  Symbol {
-// CHECK-NEXT:    Name: r2
-// CHECK-NEXT:    Value: 0
-// CHECK-NEXT:    Section: IMAGE_SYM_UNDEFINED (0)
-// CHECK-NEXT:    BaseType: Null (0x0)
-// CHECK-NEXT:    ComplexType: Null (0x0)
-// CHECK-NEXT:    StorageClass: WeakExternal (0x69)
-// CHECK-NEXT:    AuxSymbolCount: 1
-// CHECK-NEXT:    AuxWeakExternal {
-// CHECK-NEXT:      Linked: r1 (9)
-// CHECK-NEXT:      Search: AntiDependency (0x4)
-// CHECK-NEXT:    }
-// CHECK-NEXT:  }
diff --git a/llvm/test/MC/Disassembler/AMDGPU/comments.txt b/llvm/test/MC/Disassembler/AMDGPU/comments.txt
new file mode 100644
index 0000000000000..c47c8da79066f
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/comments.txt
@@ -0,0 +1,8 @@
+# RUN: llvm-mc -assemble -triple=amdgcn--amdhsa -mcpu=gfx1100 -filetype=obj %s -o - | \
+# RUN:   llvm-objdump -d - | FileCheck %s
+
+; Make sure disassembling of this instruction does not cause any errors
+; generated in the comment. For this test to do its job, the instruction
+; has to be the last or the only one in the object file.
+; CHECK: v_perm_b32 v14, v52, v51, 0x5040100 // 000000000000: D644000E 03FE6734 05040100{{$}}
+  v_perm_b32 v14, v52, v51, 0x5040100
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-dfp.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-dfp.txt
new file mode 100644
index 0000000000000..3b7fa5f4e3bd5
--- /dev/null
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-encoding-dfp.txt
@@ -0,0 +1,25 @@
+# RUN: llvm-mc --disassemble %s -triple powerpc64-unknown-unknown -mcpu=pwr7 | FileCheck %s
+
+# CHECK: dadd 2, 3, 4
+0xec 0x43 0x20 0x04
+
+# CHECK: dadd. 2, 3, 4
+0xec 0x43 0x20 0x05
+
+# CHECK: daddq 2, 6, 4
+0xfc 0x46 0x20 0x04
+
+# CHECK: daddq. 2, 6, 4
+0xfc 0x46 0x20 0x05
+
+# CHECK: dsub 2, 3, 4
+0xec 0x43 0x24 0x04
+
+# CHECK: dsub. 2, 3, 4
+0xec 0x43 0x24 0x05
+
+# CHECK: dsubq 2, 6, 4
+0xfc 0x46 0x24 0x04
+
+# CHECK: dsubq. 2, 6, 4
+0xfc 0x46 0x24 0x05
diff --git a/llvm/test/MC/ELF/debug-prefix-map.s b/llvm/test/MC/ELF/debug-prefix-map.s
index 28a0bb03a45c7..2f47022d9867e 100644
--- a/llvm/test/MC/ELF/debug-prefix-map.s
+++ b/llvm/test/MC/ELF/debug-prefix-map.s
@@ -14,9 +14,13 @@
 # RUN: cd %t.foo/bar
 # RUN: llvm-mc -triple=x86_64 -g -dwarf-version=4 %t.foo%{fs-sep}bar%{fs-sep}src.s -filetype=obj -o mapabs.4.o -fdebug-prefix-map=%t.foo=/broken_root -fdebug-prefix-map=%t.foo%{fs-sep}bar=/src_root
 # RUN: llvm-dwarfdump -v -debug-info -debug-line mapabs.4.o | FileCheck --check-prefix=MAPABS_V4 %s
-# RUN: llvm-mc -triple=x86_64 -g -dwarf-version=5 %t.foo%{fs-sep}bar%{fs-sep}src.s -filetype=obj -o mapabs.5.o -fdebug-prefix-map=%t.foo%{fs-sep}bar=/src_root -fdebug-prefix-map=%t.foo=/broken_root
+# RUN: llvm-mc -triple=x86_64 -g -dwarf-version=5 %t.foo%{fs-sep}bar%{fs-sep}src.s -filetype=obj -o mapabs.5.o -fdebug-prefix-map=%t.foo=/broken_root -fdebug-prefix-map=%t.foo%{fs-sep}bar=/src_root
 # RUN: llvm-dwarfdump -v -debug-info -debug-line mapabs.5.o | FileCheck --check-prefix=MAPABS_V5 %s
 
+## The last -fdebug-prefix-map= wins even if the prefix is shorter.
+# RUN: llvm-mc -triple=x86_64 -g -dwarf-version=5 %t.foo%{fs-sep}bar%{fs-sep}src.s -filetype=obj -o mapabs.5.o -fdebug-prefix-map=%t.foo%{fs-sep}bar=/broken_root -fdebug-prefix-map=%t.foo=/src_root
+# RUN: llvm-dwarfdump -v -debug-info -debug-line mapabs.5.o | FileCheck --check-prefix=MAPABS_V5_A %s
+
 f:
   nop
 
@@ -46,3 +50,6 @@ f:
 # MAPABS_V5:      DW_AT_comp_dir [DW_FORM_string] ("{{(/|\\)+}}src_root")
 # MAPABS_V5:      DW_AT_decl_file [DW_FORM_data4] ("/src_root{{(/|\\)+}}src.s")
 # MAPABS_V5:      include_directories[ 0] = .debug_line_str[0x00000000] = "/src_root"
+
+# MAPABS_V5_A:    DW_AT_comp_dir [DW_FORM_string] ("{{(/|\\)+}}src_root{{(/|\\)+}}bar")
+# MAPABS_V5_A:    DW_AT_decl_file [DW_FORM_data4] ("/src_root{{(/|\\)+}}bar{{(/|\\)+}}src.s")
diff --git a/llvm/test/MC/MachO/ARM/build-version-diagnostics.s b/llvm/test/MC/MachO/ARM/build-version-diagnostics.s
index 31aa87add3acd..15687c2ad1fb0 100644
--- a/llvm/test/MC/MachO/ARM/build-version-diagnostics.s
+++ b/llvm/test/MC/MachO/ARM/build-version-diagnostics.s
@@ -53,4 +53,4 @@
 // CHECK: build-version-diagnostics.s:[[@LINE-1]]:25: error: invalid OS update version number
 
 .build_version ios,10,0,0,
-// CHECK: build-version-diagnostics.s:[[@LINE-1]]:26: error: unexpected token in '.build_version' directive
+// CHECK: build-version-diagnostics.s:[[#@LINE-1]]:26: error: expected newline in '.build_version' directive
diff --git a/llvm/test/MC/PowerPC/invalid-instructions-spellcheck.s b/llvm/test/MC/PowerPC/invalid-instructions-spellcheck.s
index b63e861fdc574..49798637eb511 100644
--- a/llvm/test/MC/PowerPC/invalid-instructions-spellcheck.s
+++ b/llvm/test/MC/PowerPC/invalid-instructions-spellcheck.s
@@ -39,6 +39,6 @@ xsnmsubad %r1, %r2
 
 adXd %r1, %r2, %r3
 
-# CHECK:      error: invalid instruction, did you mean: add, addc, adde, addi, addo, fadd?
+# CHECK:      error: invalid instruction, did you mean: add, addc, adde, addi, addo, dadd, fadd?
 # CHECK-NEXT: adXd %r1, %r2, %r3
 # CHECK-NEXT: ^
diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-dfp.s b/llvm/test/MC/PowerPC/ppc64-encoding-dfp.s
new file mode 100644
index 0000000000000..4b76bd21c7911
--- /dev/null
+++ b/llvm/test/MC/PowerPC/ppc64-encoding-dfp.s
@@ -0,0 +1,28 @@
+# RUN: llvm-mc -triple powerpc64-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-BE %s
+# RUN: llvm-mc -triple powerpc64le-unknown-unknown --show-encoding %s | FileCheck -check-prefix=CHECK-LE %s
+
+
+# CHECK-BE: dadd 2, 3, 4                   # encoding: [0xec,0x43,0x20,0x04]
+# CHECK-LE: dadd 2, 3, 4                   # encoding: [0x04,0x20,0x43,0xec]
+            dadd 2, 3, 4
+# CHECK-BE: dadd. 2, 3, 4                  # encoding: [0xec,0x43,0x20,0x05]
+# CHECK-LE: dadd. 2, 3, 4                  # encoding: [0x05,0x20,0x43,0xec]
+            dadd. 2, 3, 4
+# CHECK-BE: daddq 2, 6, 4                  # encoding: [0xfc,0x46,0x20,0x04]
+# CHECK-LE: daddq 2, 6, 4                  # encoding: [0x04,0x20,0x46,0xfc]
+            daddq 2, 6, 4
+# CHECK-BE: daddq. 2, 6, 4                 # encoding: [0xfc,0x46,0x20,0x05]
+# CHECK-LE: daddq. 2, 6, 4                 # encoding: [0x05,0x20,0x46,0xfc]
+            daddq. 2, 6, 4
+# CHECK-BE: dsub 2, 3, 4                   # encoding: [0xec,0x43,0x24,0x04]
+# CHECK-LE: dsub 2, 3, 4                   # encoding: [0x04,0x24,0x43,0xec]
+            dsub 2, 3, 4
+# CHECK-BE: dsub. 2, 3, 4                  # encoding: [0xec,0x43,0x24,0x05]
+# CHECK-LE: dsub. 2, 3, 4                  # encoding: [0x05,0x24,0x43,0xec]
+            dsub. 2, 3, 4
+# CHECK-BE: dsubq 2, 6, 4                  # encoding: [0xfc,0x46,0x24,0x04]
+# CHECK-LE: dsubq 2, 6, 4                  # encoding: [0x04,0x24,0x46,0xfc]
+            dsubq 2, 6, 4
+# CHECK-BE: dsubq. 2, 6, 4                 # encoding: [0xfc,0x46,0x24,0x05]
+# CHECK-LE: dsubq. 2, 6, 4                 # encoding: [0x05,0x24,0x46,0xfc]
+            dsubq. 2, 6, 4
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index ef5a95c715fb8..b22b202af437f 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -231,6 +231,9 @@
 .attribute arch, "rv32izcb1p0"
 # CHECK: attribute      5, "rv32i2p1_zca1p0_zcb1p0"
 
+.attribute arch, "rv32izcmt1p0"
+# CHECK: attribute      5, "rv32i2p1_zca1p0_zcmt1p0"
+
 .attribute arch, "rv64i_xsfvcp"
 # CHECK: attribute      5, "rv64i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xsfvcp1p0"
 
@@ -251,3 +254,9 @@
 
 .attribute arch, "rv32izicond1p0"
 # CHECK: attribute      5, "rv32i2p1_zicond1p0"
+
+.attribute arch, "rv32i_smaia1p0"
+# CHECK: attribute      5, "rv32i2p1_smaia1p0"
+
+.attribute arch, "rv32i_ssaia1p0"
+# CHECK: attribute      5, "rv32i2p1_ssaia1p0"
diff --git a/llvm/test/MC/RISCV/deprecated-csr-names.s b/llvm/test/MC/RISCV/deprecated-csr-names.s
index f3c475d54699f..e895732ae23f6 100644
--- a/llvm/test/MC/RISCV/deprecated-csr-names.s
+++ b/llvm/test/MC/RISCV/deprecated-csr-names.s
@@ -44,22 +44,6 @@ csrrw zero, 0x343, zero
 
 # CHECK-WARN: warning: 'mbadaddr' is a deprecated alias for 'mtval'
 
-# ubadaddr
-# name
-# CHECK-INST: csrrw zero, utval, zero
-# CHECK-ENC: encoding: [0x73,0x10,0x30,0x04]
-# CHECK-INST-ALIAS: csrw utval, zero
-# uimm12
-# CHECK-INST: csrrw zero, utval, zero
-# CHECK-ENC: encoding: [0x73,0x10,0x30,0x04]
-# CHECK-INST-ALIAS: csrw utval, zero
-# name
-csrw ubadaddr, zero
-# uimm12
-csrrw zero, 0x043, zero
-
-# CHECK-WARN: warning: 'ubadaddr' is a deprecated alias for 'utval'
-
 # sptbr
 # name
 # CHECK-INST: csrrw zero, satp, zero
diff --git a/llvm/test/MC/RISCV/hypervisor-csr-names.s b/llvm/test/MC/RISCV/hypervisor-csr-names.s
index 00006201de8e9..fed803c39e00d 100644
--- a/llvm/test/MC/RISCV/hypervisor-csr-names.s
+++ b/llvm/test/MC/RISCV/hypervisor-csr-names.s
@@ -447,3 +447,119 @@ csrrs t2, 0x60E, zero
 csrrs t1, hstateen3, zero
 # uimm12
 csrrs t2, 0x60F, zero
+
+#########################################
+# Advanced Interrupt Architecture (Smaia and Ssaia)
+#########################################
+
+# hvien
+# name
+# CHECK-INST: csrrs t1, hvien, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x80,0x60]
+# CHECK-INST-ALIAS: csrr t1, hvien
+# uimm12
+# CHECK-INST: csrrs t2, hvien, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x80,0x60]
+# CHECK-INST-ALIAS: csrr t2, hvien
+# name
+csrrs t1, hvien, zero
+# uimm12
+csrrs t2, 0x608, zero
+
+# hvictl
+# name
+# CHECK-INST: csrrs t1, hvictl, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x90,0x60]
+# CHECK-INST-ALIAS: csrr t1, hvictl
+# uimm12
+# CHECK-INST: csrrs t2, hvictl, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x90,0x60]
+# CHECK-INST-ALIAS: csrr t2, hvictl
+# name
+csrrs t1, hvictl, zero
+# uimm12
+csrrs t2, 0x609, zero
+
+# hviprio1
+# name
+# CHECK-INST: csrrs t1, hviprio1, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x60,0x64]
+# CHECK-INST-ALIAS: csrr t1, hviprio1
+# uimm12
+# CHECK-INST: csrrs t2, hviprio1, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x60,0x64]
+# CHECK-INST-ALIAS: csrr t2, hviprio1
+# name
+csrrs t1, hviprio1, zero
+# uimm12
+csrrs t2, 0x646, zero
+
+# hviprio2
+# name
+# CHECK-INST: csrrs t1, hviprio2, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x70,0x64]
+# CHECK-INST-ALIAS: csrr t1, hviprio2
+# uimm12
+# CHECK-INST: csrrs t2, hviprio2, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x70,0x64]
+# CHECK-INST-ALIAS: csrr t2, hviprio2
+# name
+csrrs t1, hviprio2, zero
+# uimm12
+csrrs t2, 0x647, zero
+
+# vsiselect
+# name
+# CHECK-INST: csrrs t1, vsiselect, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0x25]
+# CHECK-INST-ALIAS: csrr t1, vsiselect
+# uimm12
+# CHECK-INST: csrrs t2, vsiselect, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0x25]
+# CHECK-INST-ALIAS: csrr t2, vsiselect
+# name
+csrrs t1, vsiselect, zero
+# uimm12
+csrrs t2, 0x250, zero
+
+# vsireg
+# name
+# CHECK-INST: csrrs t1, vsireg, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x10,0x25]
+# CHECK-INST-ALIAS: csrr t1, vsireg
+# uimm12
+# CHECK-INST: csrrs t2, vsireg, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x10,0x25]
+# CHECK-INST-ALIAS: csrr t2, vsireg
+# name
+csrrs t1, vsireg, zero
+# uimm12
+csrrs t2, 0x251, zero
+
+# vstopei
+# name
+# CHECK-INST: csrrs t1, vstopei, zero
+# CHECK-ENC: encoding: [0x73,0x23,0xc0,0x25]
+# CHECK-INST-ALIAS: csrr t1, vstopei
+# uimm12
+# CHECK-INST: csrrs t2, vstopei, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0xc0,0x25]
+# CHECK-INST-ALIAS: csrr t2, vstopei
+# name
+csrrs t1, vstopei, zero
+# uimm12
+csrrs t2, 0x25C, zero
+
+# vstopi
+# name
+# CHECK-INST: csrrs t1, vstopi, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0xeb]
+# CHECK-INST-ALIAS: csrr t1, vstopi
+# uimm12
+# CHECK-INST: csrrs t2, vstopi, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0xeb]
+# CHECK-INST-ALIAS: csrr t2, vstopi
+# name
+csrrs t1, vstopi, zero
+# uimm12
+csrrs t2, 0xEB0, zero
diff --git a/llvm/test/MC/RISCV/machine-csr-names.s b/llvm/test/MC/RISCV/machine-csr-names.s
index 7e3efce5c5a22..3b6d73b5118be 100644
--- a/llvm/test/MC/RISCV/machine-csr-names.s
+++ b/llvm/test/MC/RISCV/machine-csr-names.s
@@ -2410,3 +2410,91 @@ csrrs t2, 0x30E, zero
 csrrs t1, mstateen3, zero
 # uimm12
 csrrs t2, 0x30F, zero
+
+#########################################
+# Advanced Interrupt Architecture (Smaia and Ssaia)
+#########################################
+
+# miselect
+# name
+# CHECK-INST: csrrs t1, miselect, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0x35]
+# CHECK-INST-ALIAS: csrr t1, miselect
+# uimm12
+# CHECK-INST: csrrs t2, miselect, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0x35]
+# CHECK-INST-ALIAS: csrr t2, miselect
+# name
+csrrs t1, miselect, zero
+# uimm12
+csrrs t2, 0x350, zero
+
+# mireg
+# name
+# CHECK-INST: csrrs t1, mireg, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x10,0x35]
+# CHECK-INST-ALIAS: csrr t1, mireg
+# uimm12
+# CHECK-INST: csrrs t2, mireg, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x10,0x35]
+# CHECK-INST-ALIAS: csrr t2, mireg
+# name
+csrrs t1, mireg, zero
+# uimm12
+csrrs t2, 0x351, zero
+
+# mtopei
+# name
+# CHECK-INST: csrrs t1, mtopei, zero
+# CHECK-ENC: encoding: [0x73,0x23,0xc0,0x35]
+# CHECK-INST-ALIAS: csrr t1, mtopei
+# uimm12
+# CHECK-INST: csrrs t2, mtopei, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0xc0,0x35]
+# CHECK-INST-ALIAS: csrr t2, mtopei
+# name
+csrrs t1, mtopei, zero
+# uimm12
+csrrs t2, 0x35C, zero
+
+# mtopi
+# name
+# CHECK-INST: csrrs t1, mtopi, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0xfb]
+# CHECK-INST-ALIAS: csrr t1, mtopi
+# uimm12
+# CHECK-INST: csrrs t2, mtopi, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0xfb]
+# CHECK-INST-ALIAS: csrr t2, mtopi
+# name
+csrrs t1, mtopi, zero
+# uimm12
+csrrs t2, 0xFB0, zero
+
+# mvien
+# name
+# CHECK-INST: csrrs t1, mvien, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x80,0x30]
+# CHECK-INST-ALIAS: csrr t1, mvien
+# uimm12
+# CHECK-INST: csrrs t2, mvien, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x80,0x30]
+# CHECK-INST-ALIAS: csrr t2, mvien
+# name
+csrrs t1, mvien, zero
+# uimm12
+csrrs t2, 0x308, zero
+
+# mvip
+# name
+# CHECK-INST: csrrs t1, mvip, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x90,0x30]
+# CHECK-INST-ALIAS: csrr t1, mvip
+# uimm12
+# CHECK-INST: csrrs t2, mvip, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x90,0x30]
+# CHECK-INST-ALIAS: csrr t2, mvip
+# name
+csrrs t1, mvip, zero
+# uimm12
+csrrs t2, 0x309, zero
diff --git a/llvm/test/MC/RISCV/priv-valid.s b/llvm/test/MC/RISCV/priv-valid.s
index 323acc500d383..561c76bf4fa29 100644
--- a/llvm/test/MC/RISCV/priv-valid.s
+++ b/llvm/test/MC/RISCV/priv-valid.s
@@ -9,10 +9,6 @@
 # RUN:     | llvm-objdump --mattr=+svinval -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 
-# CHECK-INST: uret
-# CHECK: encoding: [0x73,0x00,0x20,0x00]
-uret
-
 # CHECK-INST: sret
 # CHECK: encoding: [0x73,0x00,0x20,0x10]
 sret
diff --git a/llvm/test/MC/RISCV/rv32-hypervisor-csr-names.s b/llvm/test/MC/RISCV/rv32-hypervisor-csr-names.s
index f7d2341070ac4..83e6d21d3067c 100644
--- a/llvm/test/MC/RISCV/rv32-hypervisor-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-hypervisor-csr-names.s
@@ -117,3 +117,105 @@ csrrs t2, 0x61E, zero
 csrrs t1, hstateen3h, zero
 # uimm12
 csrrs t2, 0x61F, zero
+
+#########################################
+# Advanced Interrupt Architecture (Smaia and Ssaia)
+#########################################
+
+# hidelegh
+# name
+# CHECK-INST: csrrs t1, hidelegh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x30,0x61]
+# CHECK-INST-ALIAS: csrr t1, hidelegh
+# uimm12
+# CHECK-INST: csrrs t2, hidelegh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x30,0x61]
+# CHECK-INST-ALIAS: csrr t2, hidelegh
+# name
+csrrs t1, hidelegh, zero
+# uimm12
+csrrs t2, 0x613, zero
+
+# hvienh
+# name
+# CHECK-INST: csrrs t1, hvienh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x80,0x61]
+# CHECK-INST-ALIAS: csrr t1, hvienh
+# uimm12
+# CHECK-INST: csrrs t2, hvienh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x80,0x61]
+# CHECK-INST-ALIAS: csrr t2, hvienh
+# name
+csrrs t1, hvienh, zero
+# uimm12
+csrrs t2, 0x618, zero
+
+# hviph
+# name
+# CHECK-INST: csrrs t1, hviph, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x50,0x65]
+# CHECK-INST-ALIAS: csrr t1, hviph
+# uimm12
+# CHECK-INST: csrrs t2, hviph, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x50,0x65]
+# CHECK-INST-ALIAS: csrr t2, hviph
+# name
+csrrs t1, hviph, zero
+# uimm12
+csrrs t2, 0x655, zero
+
+# hviprio1h
+# name
+# CHECK-INST: csrrs t1, hviprio1h, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x60,0x65]
+# CHECK-INST-ALIAS: csrr t1, hviprio1h
+# uimm12
+# CHECK-INST: csrrs t2, hviprio1h, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x60,0x65]
+# CHECK-INST-ALIAS: csrr t2, hviprio1h
+# name
+csrrs t1, hviprio1h, zero
+# uimm12
+csrrs t2, 0x656, zero
+
+# hviprio2h
+# name
+# CHECK-INST: csrrs t1, hviprio2h, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x70,0x65]
+# CHECK-INST-ALIAS: csrr t1, hviprio2h
+# uimm12
+# CHECK-INST: csrrs t2, hviprio2h, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x70,0x65]
+# CHECK-INST-ALIAS: csrr t2, hviprio2h
+# name
+csrrs t1, hviprio2h, zero
+# uimm12
+csrrs t2, 0x657, zero
+
+# vsieh
+# name
+# CHECK-INST: csrrs t1, vsieh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x21]
+# CHECK-INST-ALIAS: csrr t1, vsieh
+# uimm12
+# CHECK-INST: csrrs t2, vsieh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x21]
+# CHECK-INST-ALIAS: csrr t2, vsieh
+# name
+csrrs t1, vsieh, zero
+# uimm12
+csrrs t2, 0x214, zero
+
+# vsiph
+# name
+# CHECK-INST: csrrs t1, vsiph, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x25]
+# CHECK-INST-ALIAS: csrr t1, vsiph
+# uimm12
+# CHECK-INST: csrrs t2, vsiph, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x25]
+# CHECK-INST-ALIAS: csrr t2, vsiph
+# name
+csrrs t1, vsiph, zero
+# uimm12
+csrrs t2, 0x254, zero
diff --git a/llvm/test/MC/RISCV/rv32-machine-csr-names.s b/llvm/test/MC/RISCV/rv32-machine-csr-names.s
index 7c57ee389af81..e7a6d9ce718f2 100644
--- a/llvm/test/MC/RISCV/rv32-machine-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-machine-csr-names.s
@@ -1075,3 +1075,77 @@ csrrs t2, 0x31E, zero
 csrrs t1, mstateen3h, zero
 # uimm12
 csrrs t2, 0x31F, zero
+
+#########################################
+# Advanced Interrupt Architecture (Smaia and Ssaia)
+#########################################
+
+# midelegh
+# name
+# CHECK-INST: csrrs t1, midelegh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x30,0x31]
+# CHECK-INST-ALIAS: csrr t1, midelegh
+# uimm12
+# CHECK-INST: csrrs t2, midelegh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x30,0x31]
+# CHECK-INST-ALIAS: csrr t2, midelegh
+# name
+csrrs t1, midelegh, zero
+# uimm12
+csrrs t2, 0x313, zero
+
+# mieh
+# name
+# CHECK-INST: csrrs t1, mieh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x31]
+# CHECK-INST-ALIAS: csrr t1, mieh
+# uimm12
+# CHECK-INST: csrrs t2, mieh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x31]
+# CHECK-INST-ALIAS: csrr t2, mieh
+# name
+csrrs t1, mieh, zero
+# uimm12
+csrrs t2, 0x314, zero
+
+# mvienh
+# name
+# CHECK-INST: csrrs t1, mvienh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x80,0x31]
+# CHECK-INST-ALIAS: csrr t1, mvienh
+# uimm12
+# CHECK-INST: csrrs t2, mvienh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x80,0x31]
+# CHECK-INST-ALIAS: csrr t2, mvienh
+# name
+csrrs t1, mvienh, zero
+# uimm12
+csrrs t2, 0x318, zero
+
+# mviph
+# name
+# CHECK-INST: csrrs t1, mviph, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x90,0x31]
+# CHECK-INST-ALIAS: csrr t1, mviph
+# uimm12
+# CHECK-INST: csrrs t2, mviph, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x90,0x31]
+# CHECK-INST-ALIAS: csrr t2, mviph
+# name
+csrrs t1, mviph, zero
+# uimm12
+csrrs t2, 0x319, zero
+
+# miph
+# name
+# CHECK-INST: csrrs t1, miph, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x35]
+# CHECK-INST-ALIAS: csrr t1, miph
+# uimm12
+# CHECK-INST: csrrs t2, miph, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x35]
+# CHECK-INST-ALIAS: csrr t2, miph
+# name
+csrrs t1, miph, zero
+# uimm12
+csrrs t2, 0x354, zero
diff --git a/llvm/test/MC/RISCV/rv32-only-csr-names.s b/llvm/test/MC/RISCV/rv32-only-csr-names.s
index 10c4b9cc6750e..5a91aea4c791b 100644
--- a/llvm/test/MC/RISCV/rv32-only-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-only-csr-names.s
@@ -131,3 +131,18 @@ csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system regis
 
 csrrs t1, stimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
 csrrs t1, vstimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+
+csrrs t1, midelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, miph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, sieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, siph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, hidelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, hvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, hviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, hviprio1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, hviprio2h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, vsieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, vsiph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
diff --git a/llvm/test/MC/RISCV/rv32-supervisor-csr-names.s b/llvm/test/MC/RISCV/rv32-supervisor-csr-names.s
index 28dcd19bd38b5..4c1fef446a3d8 100644
--- a/llvm/test/MC/RISCV/rv32-supervisor-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-supervisor-csr-names.s
@@ -21,3 +21,35 @@
 csrrs t1, stimecmph, zero
 # uimm12
 csrrs t2, 0x15D, zero
+
+#########################################
+# Advanced Interrupt Architecture (Smaia and Ssaia)
+#########################################
+
+# sieh
+# name
+# CHECK-INST: csrrs t1, sieh, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x11]
+# CHECK-INST-ALIAS: csrr t1, sieh
+# uimm12
+# CHECK-INST: csrrs t2, sieh, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x11]
+# CHECK-INST-ALIAS: csrr t2, sieh
+# name
+csrrs t1, sieh, zero
+# uimm12
+csrrs t2, 0x114, zero
+
+# siph
+# name
+# CHECK-INST: csrrs t1, siph, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x40,0x15]
+# CHECK-INST-ALIAS: csrr t1, siph
+# uimm12
+# CHECK-INST: csrrs t2, siph, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x40,0x15]
+# CHECK-INST-ALIAS: csrr t2, siph
+# name
+csrrs t1, siph, zero
+# uimm12
+csrrs t2, 0x154, zero
diff --git a/llvm/test/MC/RISCV/rv32e-valid.s b/llvm/test/MC/RISCV/rv32e-valid.s
index c2b77736d92b5..77ce86f018302 100644
--- a/llvm/test/MC/RISCV/rv32e-valid.s
+++ b/llvm/test/MC/RISCV/rv32e-valid.s
@@ -116,9 +116,9 @@ csrrw t0, 0xfff, t1
 csrrs s0, 0xc00, x0
 # CHECK-ASM-AND-OBJ: csrrs s0, fflags, a5
 csrrs s0, 0x001, a5
-# CHECK-ASM-AND-OBJ: csrrc sp, ustatus, ra
+# CHECK-ASM-AND-OBJ: csrrc sp, 0, ra
 csrrc sp, 0x000, ra
-# CHECK-ASM-AND-OBJ: csrrwi a5, ustatus, 0
+# CHECK-ASM-AND-OBJ: csrrwi a5, 0, 0
 csrrwi a5, 0x000, 0
 # CHECK-ASM-AND-OBJ: csrrsi t2, 4095, 31
 csrrsi t2, 0xfff, 31
diff --git a/llvm/test/MC/RISCV/rv32i-valid.s b/llvm/test/MC/RISCV/rv32i-valid.s
index 64883e8b5ba6f..3b6e93c739e5a 100644
--- a/llvm/test/MC/RISCV/rv32i-valid.s
+++ b/llvm/test/MC/RISCV/rv32i-valid.s
@@ -361,10 +361,10 @@ csrrs s0, 0xc00, x0
 # CHECK-ASM-AND-OBJ: csrrs s3, fflags, s5
 # CHECK-ASM: encoding: [0xf3,0xa9,0x1a,0x00]
 csrrs s3, 0x001, s5
-# CHECK-ASM-AND-OBJ: csrrc sp, ustatus, ra
+# CHECK-ASM-AND-OBJ: csrrc sp, 0, ra
 # CHECK-ASM: encoding: [0x73,0xb1,0x00,0x00]
 csrrc sp, 0x000, ra
-# CHECK-ASM-AND-OBJ: csrrwi a5, ustatus, 0
+# CHECK-ASM-AND-OBJ: csrrwi a5, 0, 0
 # CHECK-ASM: encoding: [0xf3,0x57,0x00,0x00]
 csrrwi a5, 0x000, 0
 # CHECK-ASM-AND-OBJ: csrrsi t2, 4095, 31
diff --git a/llvm/test/MC/RISCV/rv32zcmt-invalid.s b/llvm/test/MC/RISCV/rv32zcmt-invalid.s
new file mode 100644
index 0000000000000..d784dc4bfb8f9
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv32zcmt-invalid.s
@@ -0,0 +1,13 @@
+# RUN: not llvm-mc -triple=riscv32 -mattr=+experimental-zcmt -riscv-no-aliases -show-encoding < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK-ERROR %s
+# RUN: not llvm-mc -triple=riscv64 -mattr=+experimental-zcmt -riscv-no-aliases -show-encoding < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK-ERROR %s
+
+# CHECK-ERROR: error: immediate must be an integer in the range [0, 31]
+cm.jt 64
+
+# CHECK-ERROR: error: immediate must be an integer in the range [32, 255]
+cm.jalt 256
+
+# CHECK-ERROR: error: immediate must be an integer in the range [32, 255]
+cm.jalt 31
diff --git a/llvm/test/MC/RISCV/rv32zcmt-valid.s b/llvm/test/MC/RISCV/rv32zcmt-valid.s
new file mode 100644
index 0000000000000..75f6442ebd22f
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv32zcmt-valid.s
@@ -0,0 +1,33 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zcmt\
+# RUN:  -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-zcmt\
+# RUN:  -mattr=m < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zcmt\
+# RUN:  -M no-aliases -d -r - \
+# RUN:     | FileCheck --check-prefixes=CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zcmt\
+# RUN:  -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zcmt\
+# RUN:  -mattr=m < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zcmt\
+# RUN:  -M no-aliases -d -r - \
+# RUN:     | FileCheck --check-prefixes=CHECK-ASM-AND-OBJ %s
+#
+# RUN: not llvm-mc -triple riscv32 \
+# RUN:     -riscv-no-aliases -show-encoding < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK-NO-EXT %s
+# RUN: not llvm-mc -triple riscv64 \
+# RUN:     -riscv-no-aliases -show-encoding < %s 2>&1 \
+# RUN:     | FileCheck -check-prefixes=CHECK-NO-EXT %s
+
+# CHECK-ASM-AND-OBJ: cm.jt 1
+# CHECK-ASM: encoding: [0x06,0xa0]
+# CHECK-NO-EXT: error: instruction requires the following: 'Zcmt' (table jump instuctions for code-size reduction){{$}}
+cm.jt 1
+
+# CHECK-ASM-AND-OBJ: cm.jalt 32
+# CHECK-ASM: encoding: [0x82,0xa0]
+# CHECK-NO-EXT: error: instruction requires the following: 'Zcmt' (table jump instuctions for code-size reduction){{$}}
+cm.jalt 32
diff --git a/llvm/test/MC/RISCV/rvi-aliases-valid.s b/llvm/test/MC/RISCV/rvi-aliases-valid.s
index 8b166d48992ee..b5d3cb06c8e09 100644
--- a/llvm/test/MC/RISCV/rvi-aliases-valid.s
+++ b/llvm/test/MC/RISCV/rvi-aliases-valid.s
@@ -207,8 +207,8 @@ rdcycle x24
 # CHECK-S-OBJ: rdtime s9
 rdtime x25
 
-# CHECK-S-OBJ-NOALIAS: csrrs  s0, 336, zero
-# CHECK-S-OBJ: csrr s0, 336
+# CHECK-S-OBJ-NOALIAS: csrrs s0, siselect, zero
+# CHECK-S-OBJ: csrr s0, siselect
 csrr x8, 0x150
 # CHECK-S-OBJ-NOALIAS: csrrw zero, sscratch, s1
 # CHECK-S-OBJ: csrw sscratch, s1
@@ -220,8 +220,8 @@ csrs 0xfff, x22
 # CHECK-S-OBJ: csrc 4095, s7
 csrc 0xfff, x23
 
-# CHECK-S-OBJ-NOALIAS: csrrwi zero, 336, 15
-# CHECK-S-OBJ: csrwi 336, 15
+# CHECK-S-OBJ-NOALIAS: csrrwi zero, siselect, 15
+# CHECK-S-OBJ: csrwi siselect, 15
 csrwi 0x150, 0xf
 # CHECK-S-OBJ-NOALIAS: csrrsi zero, 4095, 16
 # CHECK-S-OBJ: csrsi 4095, 16
@@ -230,18 +230,18 @@ csrsi 0xfff, 0x10
 # CHECK-S-OBJ: csrci sscratch, 17
 csrci 0x140, 0x11
 
-# CHECK-S-OBJ-NOALIAS: csrrwi zero, 336, 7
-# CHECK-S-OBJ: csrwi 336, 7
+# CHECK-S-OBJ-NOALIAS: csrrwi zero, siselect, 7
+# CHECK-S-OBJ: csrwi siselect, 7
 csrw 0x150, 7
-# CHECK-S-OBJ-NOALIAS: csrrsi zero, 336, 7
-# CHECK-S-OBJ: csrsi 336, 7
+# CHECK-S-OBJ-NOALIAS: csrrsi zero, siselect, 7
+# CHECK-S-OBJ: csrsi siselect, 7
 csrs 0x150, 7
-# CHECK-S-OBJ-NOALIAS: csrrci zero, 336, 7
-# CHECK-S-OBJ: csrci 336, 7
+# CHECK-S-OBJ-NOALIAS: csrrci zero, siselect, 7
+# CHECK-S-OBJ: csrci siselect, 7
 csrc 0x150, 7
 
-# CHECK-S-OBJ-NOALIAS: csrrwi t0, 336, 15
-# CHECK-S-OBJ: csrrwi t0, 336, 15
+# CHECK-S-OBJ-NOALIAS: csrrwi t0, siselect, 15
+# CHECK-S-OBJ: csrrwi t0, siselect, 15
 csrrw t0, 0x150, 0xf
 # CHECK-S-OBJ-NOALIAS: csrrsi t0, 4095, 16
 # CHECK-S-OBJ: csrrsi t0, 4095, 16
diff --git a/llvm/test/MC/RISCV/rvv/zvbb-invalid.s b/llvm/test/MC/RISCV/rvv/zvbb-invalid.s
new file mode 100644
index 0000000000000..46bde3a8ce702
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvbb-invalid.s
@@ -0,0 +1,14 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+zve64x --mattr=+experimental-zvbb %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+vwsll.vv v2, v2, v4
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsll.vv v2, v2, v4
+
+vwsll.vx v2, v2, x10
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsll.vx v2, v2, x10
+
+vwsll.vi v2, v2, 1
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vwsll.vi v2, v2, 1
diff --git a/llvm/test/MC/RISCV/rvzcmt-user-csr-name.s b/llvm/test/MC/RISCV/rvzcmt-user-csr-name.s
new file mode 100644
index 0000000000000..0036981bd7a60
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvzcmt-user-csr-name.s
@@ -0,0 +1,29 @@
+# RUN: llvm-mc %s -triple=riscv32 -riscv-no-aliases -mattr=+experimental-zcmt -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-INST,CHECK-ENC %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zcmt < %s \
+# RUN:     | llvm-objdump -d --mattr=+experimental-zcmt - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST-ALIAS %s
+#
+# RUN: llvm-mc %s -triple=riscv64 -riscv-no-aliases -mattr=+experimental-zcmt -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-INST,CHECK-ENC %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zcmt < %s \
+# RUN:     | llvm-objdump -d --mattr=+experimental-zcmt - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST-ALIAS %s
+
+##################################
+# Jump Vector Table CSR
+##################################
+
+# jvt
+# name
+# CHECK-INST: csrrs t1, jvt, zero
+# CHECK-ENC:  encoding: [0x73,0x23,0x70,0x01]
+# CHECK-INST-ALIAS: csrr t1, jvt
+# uimm12
+# CHECK-INST: csrrs t2, jvt, zero
+# CHECK-ENC:  encoding: [0xf3,0x23,0x70,0x01]
+# CHECK-INST-ALIAS: csrr t2, jvt
+# name
+csrrs t1, jvt, zero
+# uimm12
+csrrs t2, 0x017, zero
diff --git a/llvm/test/MC/RISCV/supervisor-csr-names.s b/llvm/test/MC/RISCV/supervisor-csr-names.s
index e2bf942beac07..84f9edd595d09 100644
--- a/llvm/test/MC/RISCV/supervisor-csr-names.s
+++ b/llvm/test/MC/RISCV/supervisor-csr-names.s
@@ -28,34 +28,6 @@ csrrs t1, sstatus, zero
 # uimm12
 csrrs t2, 0x100, zero
 
-# sedeleg
-# name
-# CHECK-INST: csrrs t1, sedeleg, zero
-# CHECK-ENC: encoding: [0x73,0x23,0x20,0x10]
-# CHECK-INST-ALIAS: csrr t1, sedeleg
-# uimm12
-# CHECK-INST: csrrs t2, sedeleg, zero
-# CHECK-ENC: encoding: [0xf3,0x23,0x20,0x10]
-# CHECK-INST-ALIAS: csrr t2, sedeleg
-# name
-csrrs t1, sedeleg, zero
-# uimm12
-csrrs t2, 0x102, zero
-
-# sideleg
-# name
-# CHECK-INST: csrrs t1, sideleg, zero
-# CHECK-ENC: encoding: [0x73,0x23,0x30,0x10]
-# CHECK-INST-ALIAS: csrr t1, sideleg
-# uimm12
-# CHECK-INST: csrrs t2, sideleg, zero
-# CHECK-ENC: encoding: [0xf3,0x23,0x30,0x10]
-# CHECK-INST-ALIAS: csrr t2, sideleg
-# name
-csrrs t1, sideleg, zero
-# uimm12
-csrrs t2, 0x103, zero
-
 # sie
 # name
 # CHECK-INST: csrrs t1, sie, zero
@@ -319,3 +291,63 @@ csrrs t2, 0x10E, zero
 csrrs t1, sstateen3, zero
 # uimm12
 csrrs t2, 0x10F, zero
+
+#########################################
+# Advanced Interrupt Architecture (Smaia and Ssaia)
+#########################################
+
+# siselect
+# name
+# CHECK-INST: csrrs t1, siselect, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0x15]
+# CHECK-INST-ALIAS: csrr t1, siselect
+# uimm12
+# CHECK-INST: csrrs t2, siselect, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0x15]
+# CHECK-INST-ALIAS: csrr t2, siselect
+# name
+csrrs t1, siselect, zero
+# uimm12
+csrrs t2, 0x150, zero
+
+# sireg
+# name
+# CHECK-INST: csrrs t1, sireg, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x10,0x15]
+# CHECK-INST-ALIAS: csrr t1, sireg
+# uimm12
+# CHECK-INST: csrrs t2, sireg, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x10,0x15]
+# CHECK-INST-ALIAS: csrr t2, sireg
+# name
+csrrs t1, sireg, zero
+# uimm12
+csrrs t2, 0x151, zero
+
+# stopei
+# name
+# CHECK-INST: csrrs t1, stopei, zero
+# CHECK-ENC: encoding: [0x73,0x23,0xc0,0x15]
+# CHECK-INST-ALIAS: csrr t1, stopei
+# uimm12
+# CHECK-INST: csrrs t2, stopei, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0xc0,0x15]
+# CHECK-INST-ALIAS: csrr t2, stopei
+# name
+csrrs t1, stopei, zero
+# uimm12
+csrrs t2, 0x15C, zero
+
+# stopi
+# name
+# CHECK-INST: csrrs t1, stopi, zero
+# CHECK-ENC: encoding: [0x73,0x23,0x00,0xdb]
+# CHECK-INST-ALIAS: csrr t1, stopi
+# uimm12
+# CHECK-INST: csrrs t2, stopi, zero
+# CHECK-ENC: encoding: [0xf3,0x23,0x00,0xdb]
+# CHECK-INST-ALIAS: csrr t2, stopi
+# name
+csrrs t1, stopi, zero
+# uimm12
+csrrs t2, 0xDB0, zero
diff --git a/llvm/test/MC/RISCV/user-csr-names.s b/llvm/test/MC/RISCV/user-csr-names.s
index f2759d277ce70..9f8f029e56427 100644
--- a/llvm/test/MC/RISCV/user-csr-names.s
+++ b/llvm/test/MC/RISCV/user-csr-names.s
@@ -11,126 +11,7 @@
 # RUN:     | FileCheck -check-prefix=CHECK-INST-ALIAS %s
 
 ##################################
-# User Trap Setup
-##################################
-
-# ustatus
-# name
-# CHECK-INST: csrrs t1, ustatus, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x00,0x00]
-# CHECK-INST-ALIAS: csrr t1, ustatus
-# uimm12
-# CHECK-INST: csrrs t2, ustatus, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x00,0x00]
-# CHECK-INST-ALIAS: csrr t2, ustatus
-csrrs t1, ustatus, zero
-# uimm12
-csrrs t2, 0x000, zero
-
-# uie
-# name
-# CHECK-INST: csrrs t1, uie, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x40,0x00]
-# CHECK-INST-ALIAS: csrr t1, uie
-# uimm12
-# CHECK-INST: csrrs t2, uie, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x40,0x00]
-# CHECK-INST-ALIAS: csrr t2, uie
-# name
-csrrs t1, uie, zero
-# uimm12
-csrrs t2, 0x004, zero
-
-# utvec
-# name
-# CHECK-INST: csrrs t1, utvec, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x50,0x00]
-# CHECK-INST-ALIAS: csrr t1, utvec
-# uimm12
-# CHECK-INST: csrrs t2, utvec, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x50,0x00]
-# CHECK-INST-ALIAS: csrr t2, utvec
-# name
-csrrs t1, utvec, zero
-# uimm12
-csrrs t2, 0x005, zero
-
-##################################
-# User Trap Handling
-##################################
-
-# uscratch
-# name
-# CHECK-INST: csrrs t1, uscratch, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x00,0x04]
-# CHECK-INST-ALIAS: csrr t1, uscratch
-# uimm12
-# CHECK-INST: csrrs t2, uscratch, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x00,0x04]
-# CHECK-INST-ALIAS: csrr t2, uscratch
-# name
-csrrs t1, uscratch, zero
-# uimm12
-csrrs t2, 0x040, zero
-
-# uepc
-# name
-# CHECK-INST: csrrs t1, uepc, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x10,0x04]
-# CHECK-INST-ALIAS: csrr t1, uepc
-# uimm12
-# CHECK-INST: csrrs t2, uepc, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x10,0x04]
-# CHECK-INST-ALIAS: csrr t2, uepc
-# name
-csrrs t1, uepc, zero
-# uimm12
-csrrs t2, 0x041, zero
-
-# ucause
-# name
-# CHECK-INST: csrrs t1, ucause, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x20,0x04]
-# CHECK-INST-ALIAS: csrr t1, ucause
-# uimm12
-# CHECK-INST: csrrs t2, ucause, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x20,0x04]
-# CHECK-INST-ALIAS: csrr t2, ucause
-# name
-csrrs t1, ucause, zero
-# uimm12
-csrrs t2, 0x042, zero
-
-# utval
-# name
-# CHECK-INST: csrrs t1, utval, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x30,0x04]
-# CHECK-INST-ALIAS: csrr t1, utval
-# uimm12
-# CHECK-INST: csrrs t2, utval, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x30,0x04]
-# CHECK-INST-ALIAS: csrr t2, utval
-# name
-csrrs t1, utval, zero
-# uimm12
-csrrs t2, 0x043, zero
-
-# uip
-# name
-# CHECK-INST: csrrs t1, uip, zero
-# CHECK-ENC:  encoding: [0x73,0x23,0x40,0x04]
-# CHECK-INST-ALIAS: csrr t1, uip
-# uimm12
-# CHECK-INST: csrrs t2, uip, zero
-# CHECK-ENC:  encoding: [0xf3,0x23,0x40,0x04]
-# CHECK-INST-ALIAS: csrr t2, uip
-#name
-csrrs t1, uip, zero
-# uimm12
-csrrs t2, 0x044, zero
-
-##################################
-# User Floating Pont CSRs
+# User Floating Point CSRs
 ##################################
 # Tests in rvf-user-mode-csr.s
 
diff --git a/llvm/test/MC/Sparc/sparc64-bpr-offset.s b/llvm/test/MC/Sparc/sparc64-bpr-offset.s
new file mode 100644
index 0000000000000..6c853c339c285
--- /dev/null
+++ b/llvm/test/MC/Sparc/sparc64-bpr-offset.s
@@ -0,0 +1,31 @@
+! RUN: llvm-mc -arch=sparcv9 -filetype=obj %s | llvm-objdump -d - | FileCheck %s --check-prefix=BIN
+
+        !! SPARCv9/SPARC64 BPr branches have different offset encoding from the others,
+        !! make sure that our offset bits don't trample on other fields.
+        !! This is particularly important with backwards branches.
+
+        ! BIN:  0: 02 c8 40 01  	brz %g1, 1
+        ! BIN:  4: 04 c8 40 01  	brlez %g1, 1
+        ! BIN:  8: 06 c8 40 01  	brlz %g1, 1
+        ! BIN:  c: 0a c8 40 01  	brnz %g1, 1
+        ! BIN: 10: 0c c8 40 01  	brgz %g1, 1
+        ! BIN: 14: 0e c8 40 01  	brgez %g1, 1
+        brz   %g1, .+4
+        brlez %g1, .+4
+        brlz  %g1, .+4
+        brnz  %g1, .+4
+        brgz  %g1, .+4
+        brgez %g1, .+4
+
+        ! BIN: 18: 02 f8 7f ff  	brz %g1, 65535
+        ! BIN: 1c: 04 f8 7f ff  	brlez %g1, 65535
+        ! BIN: 20: 06 f8 7f ff  	brlz %g1, 65535
+        ! BIN: 24: 0a f8 7f ff  	brnz %g1, 65535
+        ! BIN: 28: 0c f8 7f ff  	brgz %g1, 65535
+        ! BIN: 2c: 0e f8 7f ff  	brgez %g1, 65535
+        brz   %g1, .-4
+        brlez %g1, .-4
+        brlz  %g1, .-4
+        brnz  %g1, .-4
+        brgz  %g1, .-4
+        brgez %g1, .-4
diff --git a/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s b/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
index 737b95338c998..a21b17552eb15 100644
--- a/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
+++ b/llvm/test/MC/Sparc/sparc64-ctrl-instructions.s
@@ -1150,24 +1150,18 @@
         fbne,a,pn %fcc3, .BB0
 
 
-        ! CHECK:                brz %g1, .BB0                   ! encoding: [0x02,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                                           !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
-        ! CHECK:                brlez %g1, .BB0                 ! encoding: [0x04,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                                           !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
-        ! CHECK:                brlz %g1, .BB0                  ! encoding: [0x06,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                                           !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
-        ! CHECK:                brnz %g1, .BB0                  ! encoding: [0x0a,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                                           !  fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                                           !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
-        ! CHECK:                brgz %g1, .BB0                  ! encoding: [0x0c,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                                           !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
-        ! CHECK:                brgez %g1, .BB0                 ! encoding: [0x0e,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                                           !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
+        ! CHECK:                brz %g1, .BB0                   ! encoding: [0x02'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
+        ! CHECK:                brlez %g1, .BB0                 ! encoding: [0x04'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
+        ! CHECK:                brlz %g1, .BB0                  ! encoding: [0x06'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
+        ! CHECK:                brnz %g1, .BB0                  ! encoding: [0x0a'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
+        ! CHECK:                brgz %g1, .BB0                  ! encoding: [0x0c'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
+        ! CHECK:                brgez %g1, .BB0                 ! encoding: [0x0e'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                                           !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
 
         brz   %g1, .BB0
         brlez %g1, .BB0
@@ -1176,29 +1170,24 @@
         brgz  %g1, .BB0
         brgez %g1, .BB0
 
-        ! CHECK: brz %g1, .BB0                   ! encoding: [0x02,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                            !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
+        ! CHECK: brz %g1, .BB0                   ! encoding: [0x02'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,pt   %g1, .BB0
 
-        ! CHECK: brz,a %g1, .BB0                 ! encoding: [0x22,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                            !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
+        ! CHECK: brz,a %g1, .BB0                 ! encoding: [0x22'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a   %g1, .BB0
 
-        ! CHECK: brz,a %g1, .BB0                 ! encoding: [0x22,0b11AA1000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                            !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
+        ! CHECK: brz,a %g1, .BB0                 ! encoding: [0x22'A',0xc8'A',0x40'A',A]
+        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a,pt   %g1, .BB0
 
-        ! CHECK:  brz,pn %g1, .BB0               ! encoding: [0x02,0b11AA0000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                            !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
+        ! CHECK:  brz,pn %g1, .BB0               ! encoding: [0x02'A',0xc0'A',0x40'A',A]
+        ! CHECK-NEXT:                            !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,pn   %g1, .BB0
 
-        ! CHECK:  brz,a,pn %g1, .BB0              ! encoding: [0x22,0b11AA0000,0b01BBBBBB,B]
-        ! CHECK-NEXT:                             !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16_2
-        ! CHECK-NEXT:                             !   fixup B - offset: 0, value: .BB0, kind: fixup_sparc_br16_14
+        ! CHECK:  brz,a,pn %g1, .BB0              ! encoding: [0x22'A',0xc0'A',0x40'A',A]
+        ! CHECK-NEXT:                             !   fixup A - offset: 0, value: .BB0, kind: fixup_sparc_br16
         brz,a,pn   %g1, .BB0
 
         ! CHECK: movrz   %g1, %g2, %g3 ! encoding: [0x87,0x78,0x44,0x02]
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index c4e32e6878d26..ddde17fb7e0c7 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -205,6 +205,7 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index eb4ad0610b326..7f0b335b867d0 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -106,6 +106,7 @@
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo
 ; CHECK-O23SZ-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass on foo
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
index e99250d768aa5..d8ce164628eed 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
@@ -146,6 +146,7 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass on loop
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index e460b541ac5b1..781911b7be977 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -133,6 +133,7 @@
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index a6dbe5d0a0ef9..46fd7f1a12a19 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -140,6 +140,7 @@
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll
index 8baca2bd0c484..2ad8e0dde2d9f 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll
@@ -146,6 +146,7 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass on loop
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index 648402c911a57..fe68369b7238e 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -171,6 +171,7 @@
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
index 8ad725c2d15d4..81f326fe70f7c 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
@@ -136,6 +136,7 @@
 ; CHECK-O-NEXT: Running pass: ADCEPass
 ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass
 ; CHECK-O23SZ-NEXT: Running pass: DSEPass
+; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo
 ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass
 ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass
 ; CHECK-O23SZ-NEXT: Running pass: LICMPass
diff --git a/llvm/test/TableGen/ListSlices-fail.td b/llvm/test/TableGen/ListSlices-fail.td
new file mode 100644
index 0000000000000..9e623f5a06b1f
--- /dev/null
+++ b/llvm/test/TableGen/ListSlices-fail.td
@@ -0,0 +1,72 @@
+// Each RUN line is scattered.
+
+defvar list_int = [0, 1, 2, 3, 4, 5];
+defvar list_str = ["foo", "bar"];
+defvar str = "hoge";
+defvar und = ?;
+
+#ifdef ERR0
+// RUN: not llvm-tblgen %s -DERR0 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR0
+// ERR0: [[FILE]]:[[@LINE+1]]:26: error: Unknown or reserved token when parsing a value
+defvar errs = list_str [ ] ;
+#endif
+
+#ifdef ERR1
+// RUN: not llvm-tblgen %s -DERR1 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR1
+// ERR1: [[FILE]]:[[@LINE+1]]:26: error: Unknown or reserved token when parsing a value
+defvar errs = list_str [ , ] ;
+#endif
+
+#ifdef ERR2
+// RUN: not llvm-tblgen %s -DERR2 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR2
+// ERR2: [[FILE]]:[[@LINE+1]]:26: error: expected list<int>, got list<string>
+defvar errs = list_str [ list_str ] ;
+#endif
+
+#ifdef ERR3
+// RUN: not llvm-tblgen %s -DERR3 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR3
+// ERR3: [[FILE]]:[[@LINE+1]]:26: error: expected int...int, got list<string>
+defvar errs = list_str [ list_str ... 42 ] ;
+#endif
+
+#ifdef ERR4
+// RUN: not llvm-tblgen %s -DERR4 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR4
+// ERR4: [[FILE]]:[[@LINE+1]]:32: error: expected int...int, got list<string>
+defvar errs = list_str [ 0 ... list_str ] ;
+#endif
+
+#ifdef ERR5
+// RUN: not llvm-tblgen %s -DERR5 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR5
+// ERR5: [[FILE]]:[[@LINE+1]]:26: error: unhandled type string in range
+defvar errs = list_str [ str ] ;
+#endif
+
+#ifdef ERR6
+// RUN: not llvm-tblgen %s -DERR6 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR6
+// ERR6: [[FILE]]:[[@LINE+1]]:28: error: invalid range, cannot be negative
+defvar errs = list_str [ 5 1 ] ;
+#endif
+
+#ifdef ERR7
+// RUN: not llvm-tblgen %s -DERR7 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR7
+// ERR7: [[FILE]]:[[@LINE+1]]:15: error: Type 'string' is invalid, list expected
+defvar errs = str [ 0 ] ;
+#endif
+
+#ifdef ERR8
+// RUN: not llvm-tblgen %s -DERR8 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR8
+// ERR8: [[FILE]]:[[@LINE+1]]:26: error: Unknown or reserved token when parsing a value
+defvar errs = list_int [ ... ] ;
+#endif
+
+#ifdef ERR9
+// RUN: not llvm-tblgen %s -DERR9 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR9
+// ERR9: [[FILE]]:[[@LINE+1]]:32: error: Unknown or reserved token when parsing a value
+defvar errs = list_int [ 0 ... ] ;
+#endif
+
+#ifdef ERRA
+// RUN: not llvm-tblgen %s -DERRA 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERRA
+// ERRA: [[FILE]]:[[@LINE+1]]:15: error: Invalid value, list expected
+defvar errs = und [ 0 ] ;
+#endif
diff --git a/llvm/test/TableGen/ListSlices.td b/llvm/test/TableGen/ListSlices.td
index 9e80fb215f885..0b69a3567c525 100644
--- a/llvm/test/TableGen/ListSlices.td
+++ b/llvm/test/TableGen/ListSlices.td
@@ -123,3 +123,42 @@ def Rec10 {
   int Zero = Class1<[?, ?, 2, 3, ?, 5, ?]>.Zero;
   list<int> TwoFive = Class1<[?, ?, 2, 3, ?, 5, ?]>.TwoFive;
 }
+
+// Test list[list] and list[int]
+// CHECK: def Rec11
+def Rec11 {
+  list<int> s5 = Var1[0...4];
+
+  // list[expr]
+  // CHECK:   list<int> rev = [4, 3, 2, 1, 0];
+  list<int> rev = !foreach(i, s5, Var1[!sub(4, i)]);
+
+  // Slice by list[foreach]
+  // CHECK:   list<int> revf = [4, 3, 2, 1, 0];
+  list<int> revf = Var1[!foreach(i, s5, !sub(4, i))];
+
+  // Simple slice
+  // CHECK:   list<int> rr = [0, 1, 2, 3, 4];
+  list<int> rr  = rev[rev];
+
+  // Trailing comma is acceptable
+  // CHECK:   list<int> rr_ = [0, 1, 2, 3, 4];
+  list<int> rr_ = rev[rev,];
+
+  // Concatenation in slice
+  // CHECK:   list<int> rrr = [1, 2, 4, 3, 2, 1, 0, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 8];
+  list<int> empty = [];
+  list<int> rrr = Var1[1, 2, rev, 3...6, 7, empty, rr, 8];
+
+  // Recognized as slice by the trailing comma
+  // CHECK:   list<list<int>> rl1 = {{\[}}[0], [1], [2], [3], [4]];
+  list<list<int>> rl1 = !foreach(i, rev, rev[i,]);
+
+  // Slice by pair<int,int>
+  // CHECK:   list<list<int>> rll = {{\[}}[0, 4], [1, 3], [2, 2], [3, 1], [4, 0]];
+  list<list<int>> rll = !foreach(i, rev, rev[i, !sub(4, i)]);
+
+  // Slice by dynamic range
+  // CHECK:   list<list<int>> rlr = {{\[}}[4, 3, 2, 1, 0], [3, 2, 1], [2], [1, 2, 3], [0, 1, 2, 3, 4]];
+  list<list<int>> rlr = !foreach(i, s5, rev[i...!sub(4, i)]);
+}
diff --git a/llvm/test/TableGen/range-op-fail.td b/llvm/test/TableGen/range-op-fail.td
new file mode 100644
index 0000000000000..3238d60abe7f8
--- /dev/null
+++ b/llvm/test/TableGen/range-op-fail.td
@@ -0,0 +1,45 @@
+// Each RUN line is scattered.
+
+defvar list_int = !range(4);
+
+#ifdef ERR_LIST_INT
+// RUN: not llvm-tblgen %s -DERR_LIST_INT 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_LIST_INT
+// ERR_LIST_INT: [[FILE]]:[[@LINE+1]]:32: error: expected one list, got extra value of type 'int'
+defvar errs = !range(list_int, 42);
+#endif
+
+#ifdef ERR_INT_LIST
+// RUN: not llvm-tblgen %s -DERR_INT_LIST 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_INT_LIST
+// ERR_INT_LIST: [[FILE]]:[[@LINE+1]]:25: error: expected value of type 'int', got 'list<int>'
+defvar errs = !range(0, list_int);
+#endif
+
+#ifdef ERR_TOO_MANY_ARGS
+// RUN: not llvm-tblgen %s -DERR_TOO_MANY_ARGS 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_TOO_MANY_ARGS
+// ERR_TOO_MANY_ARGS: [[FILE]]:[[@LINE+1]]:15: error: expected at most two values of integer
+defvar errs = !range(0, 42, 255);
+#endif
+
+#ifdef ERR_UNEXPECTED_TYPE_0
+// RUN: not llvm-tblgen %s -DERR_UNEXPECTED_TYPE_0 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_UNEXPECTED_TYPE_0
+// ERR_UNEXPECTED_TYPE_0: [[FILE]]:[[@LINE+1]]:22: error: expected list or int, got value of type 'string'
+defvar errs = !range("hoge");
+#endif
+
+#ifdef ERR_UNEXPECTED_TYPE_1
+// RUN: not llvm-tblgen %s -DERR_UNEXPECTED_TYPE_1 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_UNEXPECTED_TYPE_1
+// ERR_UNEXPECTED_TYPE_1: [[FILE]]:[[@LINE+1]]:22: error: expected list or int, got value of type 'string'
+defvar errs = !range("hoge", 42);
+#endif
+
+#ifdef ERR_UNEXPECTED_TYPE_2
+// RUN: not llvm-tblgen %s -DERR_UNEXPECTED_TYPE_2 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_UNEXPECTED_TYPE_2
+// ERR_UNEXPECTED_TYPE_2: [[FILE]]:[[@LINE+1]]:25: error: expected value of type 'int', got 'string'
+defvar errs = !range(6, "fuga");
+#endif
+
+#ifdef ERR_EMPTY_ARG
+// RUN: not llvm-tblgen %s -DERR_EMPTY_ARG 2>&1 | FileCheck -DFILE=%s %s --check-prefix=ERR_EMPTY_ARG
+// ERR_EMPTY_ARG: [[FILE]]:[[@LINE+1]]:22: error: Unknown or reserved token when parsing a value
+defvar errs = !range();
+#endif
diff --git a/llvm/test/TableGen/range-op.td b/llvm/test/TableGen/range-op.td
new file mode 100644
index 0000000000000..b7e587d47b8cb
--- /dev/null
+++ b/llvm/test/TableGen/range-op.td
@@ -0,0 +1,37 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+// CHECK: def range_op_ranges {
+def range_op_ranges {
+  // !range(n) generates half-open interval "[0,n)"
+  // CHECK: list<int> idxs8 = [0, 1, 2, 3, 4, 5, 6, 7];
+  list<int> idxs8 = !range(8);
+
+  // !range(m, n) generates half-open interval "[m,n)"
+  // CHECK: list<int> idxs4 = [4, 5, 6, 7];
+  list<int> idxs4 = !range(4, 8);
+
+  // !range(m, n) generates empty set if m >= n
+  // CHECK: list<int> idxs84 = [];
+  list<int> idxs84 = !range(8, 4);
+
+  // !range(list) generates index values for the list. (Not a copy of the list)
+  // CHECK: list<int> idxsl = [0, 1, 2, 3];
+  list<int> idxsl = !range(idxs4);
+
+  // !range(emptylist) generates empty set
+  // CHECK: list<int> idxs0 = [];
+  list<int> idxs0 = !range(idxs84);
+
+  // Example: Dynamic !range(n)
+  // CHECK: list<list<int>> rn = {{\[}}[0, 1, 2, 3], [0, 1, 2, 3, 4], [0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5, 6]];
+  list<list<int>> rn = !foreach(i, idxs4, !range(i));
+
+  // Example: Dynamic !range(m, n)
+  // CHECK: list<list<int>> rm = {{\[}}[0, 1, 2, 3, 4, 5, 6], [1, 2, 3, 4, 5], [2, 3, 4], [3], [], [], [], []];
+  list<list<int>> rm = !foreach(i, idxs8, !range(i, !sub(7, i)));
+
+  // Example: Dynamic !range(list)
+  // CHECK: list<list<int>> rl = {{\[}}[0, 1, 2, 3, 4, 5, 6], [0, 1, 2, 3, 4], [0, 1, 2], [0], [], [], [], []];
+  list<list<int>> rl = !foreach(r, rm, !range(r));
+}
diff --git a/llvm/test/ThinLTO/X86/check_var_import_odr.ll b/llvm/test/ThinLTO/X86/check_var_import_odr.ll
new file mode 100644
index 0000000000000..184bddc2f5cda
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/check_var_import_odr.ll
@@ -0,0 +1,41 @@
+;; Test to ensure that we don't assert when checking variable importing
+;; correctness for read only variables where the importing module contains
+;; a linkonce_odr copy of the variable. In that case we do not need to
+;; import the read only variable even when it has been exported from its
+;; original module (in this case when we decided to import @bar).
+
+; RUN: split-file %s %t
+; RUN: opt -module-summary %t/foo.ll -o %t/foo.o
+; RUN: opt -module-summary %t/bar.ll -o %t/bar.o
+; RUN: llvm-lto2 run %t/foo.o %t/bar.o -r=%t/foo.o,foo,pl -r=%t/foo.o,bar,l -r=%t/foo.o,qux,pl -r=%t/bar.o,bar,pl -r=%t/bar.o,qux, -o %t.out -save-temps
+; RUN: llvm-dis %t.out.1.3.import.bc -o - | FileCheck %s
+
+;; Check that we have internalized @qux (since it is read only), and imported @bar.
+; CHECK: @qux = internal global i32 0
+; CHECK: define available_externally hidden void @bar()
+
+;--- foo.ll
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@qux = linkonce_odr hidden global i32 0
+
+define linkonce_odr hidden void @foo() {
+  call void @bar()
+  ret void
+}
+
+declare hidden void @bar()
+
+;--- bar.ll
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@qux = linkonce_odr hidden global i32 0
+
+define hidden void @bar() {
+  %1 = load i32, i32* @qux, align 8
+  ret void
+}
diff --git a/llvm/test/ThinLTO/X86/memprof-basic.ll b/llvm/test/ThinLTO/X86/memprof-basic.ll
index 4d11244b2b42b..4df89cdb12afd 100644
--- a/llvm/test/ThinLTO/X86/memprof-basic.ll
+++ b/llvm/test/ThinLTO/X86/memprof-basic.ll
@@ -26,11 +26,14 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -thinlto-bc %s >%t.o
 ; RUN: llvm-lto2 run %t.o -enable-memprof-context-disambiguation \
 ; RUN:	-r=%t.o,main,plx \
diff --git a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll
index 3b297dd96cede..12e2fc39b5f5e 100644
--- a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll
+++ b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids.ll
@@ -38,7 +38,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The code below was created by forcing inlining of C into both B and E.
@@ -52,6 +52,9 @@
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -thinlto-bc %s >%t.o
 ; RUN: llvm-lto2 run %t.o -enable-memprof-context-disambiguation \
 ; RUN:  -r=%t.o,main,plx \
diff --git a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids2.ll b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids2.ll
index af7dece9421a9..b77df883d35b7 100644
--- a/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids2.ll
+++ b/llvm/test/ThinLTO/X86/memprof-duplicate-context-ids2.ll
@@ -76,7 +76,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The code below was created by forcing inlining of A into its callers,
diff --git a/llvm/test/ThinLTO/X86/memprof-indirectcall.ll b/llvm/test/ThinLTO/X86/memprof-indirectcall.ll
index 9cf209271ce4d..bd9f5e9250592 100644
--- a/llvm/test/ThinLTO/X86/memprof-indirectcall.ll
+++ b/llvm/test/ThinLTO/X86/memprof-indirectcall.ll
@@ -44,13 +44,16 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; Compiled without optimization to prevent inlining and devirtualization.
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -thinlto-bc %s >%t.o
 ; RUN: llvm-lto2 run %t.o -enable-memprof-context-disambiguation \
 ; RUN:  -r=%t.o,main,plx \
diff --git a/llvm/test/ThinLTO/X86/memprof-inlined.ll b/llvm/test/ThinLTO/X86/memprof-inlined.ll
index 7a2304f7202b4..e87168b4e3f92 100644
--- a/llvm/test/ThinLTO/X86/memprof-inlined.ll
+++ b/llvm/test/ThinLTO/X86/memprof-inlined.ll
@@ -27,7 +27,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The code below was created by forcing inlining of baz into foo, and
@@ -40,6 +40,9 @@
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -thinlto-bc %s >%t.o
 ; RUN: llvm-lto2 run %t.o -enable-memprof-context-disambiguation \
 ; RUN:	-r=%t.o,main,plx \
diff --git a/llvm/test/ThinLTO/X86/memprof-inlined2.ll b/llvm/test/ThinLTO/X86/memprof-inlined2.ll
index 1ffae8cd59cef..02baf9fb4cd8e 100644
--- a/llvm/test/ThinLTO/X86/memprof-inlined2.ll
+++ b/llvm/test/ThinLTO/X86/memprof-inlined2.ll
@@ -29,7 +29,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; Both foo and baz are inlined into main, at both foo callsites.
diff --git a/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll b/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
index de614173ec6da..842b1f781eac7 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
@@ -2253,3 +2253,53 @@ define i32 @loadCombine_4consecutive_badinsert6(ptr %p) {
   %o3 = or i32 %o2, %e1
   ret i32 %o3
 }
+
+define i64 @loadCombine_nonConstShift1(ptr %arg, i8 %b) {
+; ALL-LABEL: @loadCombine_nonConstShift1(
+; ALL-NEXT:    [[G1:%.*]] = getelementptr i8, ptr [[ARG:%.*]], i64 1
+; ALL-NEXT:    [[LD0:%.*]] = load i8, ptr [[ARG]], align 1
+; ALL-NEXT:    [[LD1:%.*]] = load i8, ptr [[G1]], align 1
+; ALL-NEXT:    [[Z0:%.*]] = zext i8 [[LD0]] to i64
+; ALL-NEXT:    [[Z1:%.*]] = zext i8 [[LD1]] to i64
+; ALL-NEXT:    [[Z6:%.*]] = zext i8 [[B:%.*]] to i64
+; ALL-NEXT:    [[S0:%.*]] = shl i64 [[Z0]], [[Z6]]
+; ALL-NEXT:    [[S1:%.*]] = shl i64 [[Z1]], 8
+; ALL-NEXT:    [[O7:%.*]] = or i64 [[S0]], [[S1]]
+; ALL-NEXT:    ret i64 [[O7]]
+;
+  %g1 = getelementptr i8, ptr %arg, i64 1
+  %ld0 = load i8, ptr %arg, align 1
+  %ld1 = load i8, ptr %g1, align 1
+  %z0 = zext i8 %ld0 to i64
+  %z1 = zext i8 %ld1 to i64
+  %z6 = zext i8 %b to i64
+  %s0 = shl i64 %z0, %z6
+  %s1 = shl i64 %z1, 8
+  %o7 = or i64 %s0, %s1
+  ret i64 %o7
+}
+
+define i64 @loadCombine_nonConstShift2(ptr %arg, i8 %b) {
+; ALL-LABEL: @loadCombine_nonConstShift2(
+; ALL-NEXT:    [[G1:%.*]] = getelementptr i8, ptr [[ARG:%.*]], i64 1
+; ALL-NEXT:    [[LD0:%.*]] = load i8, ptr [[ARG]], align 1
+; ALL-NEXT:    [[LD1:%.*]] = load i8, ptr [[G1]], align 1
+; ALL-NEXT:    [[Z0:%.*]] = zext i8 [[LD0]] to i64
+; ALL-NEXT:    [[Z1:%.*]] = zext i8 [[LD1]] to i64
+; ALL-NEXT:    [[Z6:%.*]] = zext i8 [[B:%.*]] to i64
+; ALL-NEXT:    [[S0:%.*]] = shl i64 [[Z0]], [[Z6]]
+; ALL-NEXT:    [[S1:%.*]] = shl i64 [[Z1]], 8
+; ALL-NEXT:    [[O7:%.*]] = or i64 [[S1]], [[S0]]
+; ALL-NEXT:    ret i64 [[O7]]
+;
+  %g1 = getelementptr i8, ptr %arg, i64 1
+  %ld0 = load i8, ptr %arg, align 1
+  %ld1 = load i8, ptr %g1, align 1
+  %z0 = zext i8 %ld0 to i64
+  %z1 = zext i8 %ld1 to i64
+  %z6 = zext i8 %b to i64
+  %s0 = shl i64 %z0, %z6
+  %s1 = shl i64 %z1, 8
+  %o7 = or i64 %s1, %s0
+  ret i64 %o7
+}
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll b/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll
index 6d15883ca379d..4ef07d78a9774 100644
--- a/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/amdgpu-crash.ll
@@ -1,7 +1,7 @@
 ; Test that we don't crash.
 ; RUN: opt < %s -passes=alignment-from-assumptions -S
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 
 %"core::str::CharIndices.29.66.90.114.138.149.165.173.181.197.205.213.229.387.398" = type { [0 x i64], i64, [0 x i64], { ptr, ptr }, [0 x i64] }
 %"unwind::libunwind::_Unwind_Exception.9.51.75.99.123.147.163.171.179.195.203.211.227.385.396" = type { [0 x i64], i64, [0 x i64], ptr, [0 x i64], [6 x i64], [0 x i64] }
diff --git a/llvm/test/Transforms/ArgumentPromotion/minsize.ll b/llvm/test/Transforms/ArgumentPromotion/minsize.ll
new file mode 100644
index 0000000000000..7657c557d9e4a
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/minsize.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=argpromotion -S < %s | FileCheck %s
+
+define internal i32 @f1(ptr %p) {
+; CHECK-LABEL: define internal i32 @f1
+; CHECK-SAME: (i32 [[P_0_VAL:%.*]]) {
+; CHECK-NEXT:    ret i32 [[P_0_VAL]]
+;
+  %i = load i32, ptr %p
+  ret i32 %i
+}
+
+define i32 @g1(ptr %p) {
+; CHECK-LABEL: define i32 @g1
+; CHECK-SAME: (ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[P_VAL:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[I:%.*]] = call i32 @f1(i32 [[P_VAL]])
+; CHECK-NEXT:    ret i32 [[I]]
+;
+  %i = call i32 @f1(ptr %p)
+  ret i32 %i
+}
+
+define i32 @g2(ptr %p) {
+; CHECK-LABEL: define i32 @g2
+; CHECK-SAME: (ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[P_VAL:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[I:%.*]] = call i32 @f1(i32 [[P_VAL]])
+; CHECK-NEXT:    ret i32 [[I]]
+;
+  %i = call i32 @f1(ptr %p)
+  ret i32 %i
+}
+
+define internal i32 @f2(ptr %p) {
+; CHECK-LABEL: define internal i32 @f2
+; CHECK-SAME: (ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    ret i32 [[I]]
+;
+  %i = load i32, ptr %p
+  ret i32 %i
+}
+
+define i32 @h1(ptr %p) minsize {
+; CHECK-LABEL: define i32 @h1
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[I:%.*]] = call i32 @f2(ptr [[P]])
+; CHECK-NEXT:    ret i32 [[I]]
+;
+  %i = call i32 @f2(ptr %p)
+  ret i32 %i
+}
+
+define i32 @h2(ptr %p) {
+; CHECK-LABEL: define i32 @h2
+; CHECK-SAME: (ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = call i32 @f2(ptr [[P]])
+; CHECK-NEXT:    ret i32 [[I]]
+;
+  %i = call i32 @f2(ptr %p)
+  ret i32 %i
+}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
index 15f86a33c1db8..3172098392bb0 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -atomic-expand %s | FileCheck %s
 ; RUN: opt -mtriple=r600-mesa-mesa3d -S -atomic-expand %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define i16 @test_atomicrmw_xchg_i16_global(ptr addrspace(1) %ptr, i16 %value) {
 ; CHECK-LABEL: @test_atomicrmw_xchg_i16_global(
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index bac02149eb090..05200a8149239 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -14,7 +14,7 @@ declare void @deref_phi_user(ptr %a);
 define ptr @test1(ptr dereferenceable(4) %0, ptr dereferenceable(8) %1, i1 zeroext %2) local_unnamed_addr {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (ptr nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0:%.*]], ptr nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0:%.*]], ptr nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], ptr [[TMP0]], ptr [[TMP1]]
 ; CHECK-NEXT:    ret ptr [[TMP4]]
 ;
@@ -26,7 +26,7 @@ define ptr @test1(ptr dereferenceable(4) %0, ptr dereferenceable(8) %1, i1 zeroe
 define ptr @test2(ptr dereferenceable_or_null(4) %0, ptr dereferenceable(8) %1, i1 zeroext %2) local_unnamed_addr {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (ptr nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[TMP0:%.*]], ptr nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[TMP0:%.*]], ptr nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP1:%.*]], i1 zeroext [[TMP2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], ptr [[TMP0]], ptr [[TMP1]]
 ; CHECK-NEXT:    ret ptr [[TMP4]]
 ;
@@ -39,7 +39,7 @@ define ptr @test2(ptr dereferenceable_or_null(4) %0, ptr dereferenceable(8) %1,
 define ptr @test3_1(ptr dereferenceable(8) %0) local_unnamed_addr {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test3_1
-; CHECK-SAME: (ptr nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
@@ -50,7 +50,7 @@ define ptr @test3_1(ptr dereferenceable(8) %0) local_unnamed_addr {
 define ptr @test3_2(ptr dereferenceable_or_null(32) %0) local_unnamed_addr {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test3_2
-; CHECK-SAME: (ptr nofree readnone dereferenceable_or_null(32) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef readnone dereferenceable_or_null(32) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:    [[RET:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
@@ -61,7 +61,7 @@ define ptr @test3_2(ptr dereferenceable_or_null(32) %0) local_unnamed_addr {
 define ptr @test3_3(ptr dereferenceable(8) %0, ptr dereferenceable(16) %1, i1 %2) local_unnamed_addr {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test3_3
-; CHECK-SAME: (ptr nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]], ptr nofree nonnull readnone dereferenceable(16) "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]], ptr nofree noundef nonnull readnone dereferenceable(16) "no-capture-maybe-returned" [[TMP1:%.*]], i1 [[TMP2:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:    [[RET1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1
 ; CHECK-NEXT:    [[RET2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 2
 ; CHECK-NEXT:    [[RET:%.*]] = select i1 [[TMP2]], ptr [[RET1]], ptr [[RET2]]
@@ -79,7 +79,7 @@ define ptr @test3_3(ptr dereferenceable(8) %0, ptr dereferenceable(16) %1, i1 %2
 define dereferenceable(4) ptr @test4(ptr dereferenceable(8) %0) local_unnamed_addr {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@test4
-; CHECK-SAME: (ptr nofree nonnull readnone returned dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-SAME: (ptr nofree noundef nonnull readnone returned dereferenceable(8) "no-capture-maybe-returned" [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; CHECK-NEXT:    ret ptr [[TMP0]]
 ;
   ret ptr %0
@@ -89,7 +89,7 @@ define dereferenceable(4) ptr @test4(ptr dereferenceable(8) %0) local_unnamed_ad
 ; loop in which dereferenceabily "grows"
 define void @deref_phi_growing(ptr dereferenceable(4000) %a) {
 ; CHECK-LABEL: define {{[^@]+}}@deref_phi_growing
-; CHECK-SAME: (ptr nonnull dereferenceable(4000) [[A:%.*]]) {
+; CHECK-SAME: (ptr noundef nonnull dereferenceable(4000) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
@@ -140,7 +140,7 @@ for.end:                                          ; preds = %for.cond.cleanup
 ; loop in which dereferenceabily "shrinks"
 define void @deref_phi_shrinking(ptr dereferenceable(4000) %a) {
 ; CHECK-LABEL: define {{[^@]+}}@deref_phi_shrinking
-; CHECK-SAME: (ptr nonnull dereferenceable(4000) [[A:%.*]]) {
+; CHECK-SAME: (ptr noundef nonnull dereferenceable(4000) [[A:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll
index a71110e23b2eb..e9b2346873958 100644
--- a/llvm/test/Transforms/Attributor/nocapture-1.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-1.ll
@@ -730,7 +730,7 @@ define i1 @nocaptureInboundsGEPICmpRev(ptr %x) {
 define i1 @nocaptureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@nocaptureDereferenceableOrNullICmp
-; CHECK-SAME: (ptr nocapture nofree readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (ptr nocapture nofree noundef readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
@@ -741,13 +741,13 @@ define i1 @nocaptureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x)
 define i1 @captureDereferenceableOrNullICmp(ptr dereferenceable_or_null(4) %x) null_pointer_is_valid {
 ; TUNIT: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid willreturn memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@captureDereferenceableOrNullICmp
-; TUNIT-SAME: (ptr nofree readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR8:[0-9]+]] {
+; TUNIT-SAME: (ptr nofree noundef readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR8:[0-9]+]] {
 ; TUNIT-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; TUNIT-NEXT:    ret i1 [[TMP1]]
 ;
 ; CGSCC: Function Attrs: nofree norecurse nosync nounwind null_pointer_is_valid willreturn memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@captureDereferenceableOrNullICmp
-; CGSCC-SAME: (ptr nofree readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR11:[0-9]+]] {
+; CGSCC-SAME: (ptr nofree noundef readnone dereferenceable_or_null(4) [[X:%.*]]) #[[ATTR11:[0-9]+]] {
 ; CGSCC-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[X]], null
 ; CGSCC-NEXT:    ret i1 [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
index 332ecfb754ad0..186de18cff106 100644
--- a/llvm/test/Transforms/Attributor/nocapture-2.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -166,13 +166,13 @@ entry:
 define float* @scc_A(i32* dereferenceable_or_null(4) %a) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@scc_A
-; TUNIT-SAME: (i32* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; TUNIT-SAME: (i32* nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[TOBOOL:%.*]] = icmp ne i32* [[A]], null
 ; TUNIT-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; TUNIT:       cond.true:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to i16*
-; TUNIT-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR9:[0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree noundef nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR9:[0-9]+]]
 ; TUNIT-NEXT:    [[TMP1:%.*]] = bitcast i8* [[CALL]] to double*
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP1]]) #[[ATTR9]]
 ; TUNIT-NEXT:    [[TMP2:%.*]] = bitcast i64* [[CALL1]] to i32*
@@ -188,13 +188,13 @@ define float* @scc_A(i32* dereferenceable_or_null(4) %a) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@scc_A
-; CGSCC-SAME: (i32* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (i32* nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TOBOOL:%.*]] = icmp ne i32* [[A]], null
 ; CGSCC-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CGSCC:       cond.true:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to i16*
-; CGSCC-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR10:[0-9]+]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree noundef nonnull readnone dereferenceable(4) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR10:[0-9]+]]
 ; CGSCC-NEXT:    [[TMP1:%.*]] = bitcast i8* [[CALL]] to double*
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP1]]) #[[ATTR10]]
 ; CGSCC-NEXT:    [[TMP2:%.*]] = bitcast i64* [[CALL1]] to i32*
@@ -235,13 +235,13 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define i64* @scc_B(double* dereferenceable_or_null(8) %a) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@scc_B
-; TUNIT-SAME: (double* nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
+; TUNIT-SAME: (double* nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[TOBOOL:%.*]] = icmp ne double* [[A]], null
 ; TUNIT-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; TUNIT:       cond.true:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to i32*
-; TUNIT-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR9]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR9]]
 ; TUNIT-NEXT:    [[TMP1:%.*]] = bitcast float* [[CALL]] to double*
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP1]]) #[[ATTR9]]
 ; TUNIT-NEXT:    [[TMP2:%.*]] = bitcast i64* [[CALL1]] to i16*
@@ -257,13 +257,13 @@ define i64* @scc_B(double* dereferenceable_or_null(8) %a) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@scc_B
-; CGSCC-SAME: (double* nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (double* nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[TOBOOL:%.*]] = icmp ne double* [[A]], null
 ; CGSCC-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CGSCC:       cond.true:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to i32*
-; CGSCC-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree noundef nonnull readnone dereferenceable(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR10]]
 ; CGSCC-NEXT:    [[TMP1:%.*]] = bitcast float* [[CALL]] to double*
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP1]]) #[[ATTR10]]
 ; CGSCC-NEXT:    [[TMP2:%.*]] = bitcast i64* [[CALL1]] to i16*
@@ -303,20 +303,20 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define i8* @scc_C(i16* dereferenceable_or_null(2) %a) {
 ; TUNIT: Function Attrs: nofree nosync nounwind memory(none)
 ; TUNIT-LABEL: define {{[^@]+}}@scc_C
-; TUNIT-SAME: (i16* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
+; TUNIT-SAME: (i16* nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[BC:%.*]] = bitcast i16* [[A]] to i32*
-; TUNIT-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[BC]]) #[[ATTR9]]
+; TUNIT-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[BC]]) #[[ATTR9]]
 ; TUNIT-NEXT:    [[BC2:%.*]] = bitcast float* [[CALL]] to i8*
 ; TUNIT-NEXT:    [[TOBOOL:%.*]] = icmp ne i8* [[BC2]], null
 ; TUNIT-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; TUNIT:       cond.true:
 ; TUNIT-NEXT:    [[TMP0:%.*]] = bitcast i16* [[A]] to double*
-; TUNIT-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR9]]
+; TUNIT-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR9]]
 ; TUNIT-NEXT:    [[TMP1:%.*]] = bitcast i64* [[CALL1]] to i8*
 ; TUNIT-NEXT:    br label [[COND_END:%.*]]
 ; TUNIT:       cond.false:
-; TUNIT-NEXT:    [[CALL2:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR9]]
+; TUNIT-NEXT:    [[CALL2:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR9]]
 ; TUNIT-NEXT:    br label [[COND_END]]
 ; TUNIT:       cond.end:
 ; TUNIT-NEXT:    [[COND:%.*]] = phi i8* [ [[TMP1]], [[COND_TRUE]] ], [ [[CALL2]], [[COND_FALSE]] ]
@@ -327,20 +327,20 @@ define i8* @scc_C(i16* dereferenceable_or_null(2) %a) {
 ;
 ; CGSCC: Function Attrs: nofree nosync nounwind memory(none)
 ; CGSCC-LABEL: define {{[^@]+}}@scc_C
-; CGSCC-SAME: (i16* nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
+; CGSCC-SAME: (i16* nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A:%.*]]) #[[ATTR2]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[BC:%.*]] = bitcast i16* [[A]] to i32*
-; CGSCC-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[BC]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[CALL:%.*]] = call dereferenceable_or_null(4) float* @scc_A(i32* noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[BC]]) #[[ATTR10]]
 ; CGSCC-NEXT:    [[BC2:%.*]] = bitcast float* [[CALL]] to i8*
 ; CGSCC-NEXT:    [[TOBOOL:%.*]] = icmp ne i8* [[BC2]], null
 ; CGSCC-NEXT:    br i1 [[TOBOOL]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CGSCC:       cond.true:
 ; CGSCC-NEXT:    [[TMP0:%.*]] = bitcast i16* [[A]] to double*
-; CGSCC-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[CALL1:%.*]] = call dereferenceable_or_null(4) i64* @scc_B(double* noalias nofree noundef readnone dereferenceable_or_null(8) "no-capture-maybe-returned" [[TMP0]]) #[[ATTR10]]
 ; CGSCC-NEXT:    [[TMP1:%.*]] = bitcast i64* [[CALL1]] to i8*
 ; CGSCC-NEXT:    br label [[COND_END:%.*]]
 ; CGSCC:       cond.false:
-; CGSCC-NEXT:    [[CALL2:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR10]]
+; CGSCC-NEXT:    [[CALL2:%.*]] = call dereferenceable_or_null(4) i8* @scc_C(i16* noalias nofree noundef readnone dereferenceable_or_null(4) "no-capture-maybe-returned" [[A]]) #[[ATTR10]]
 ; CGSCC-NEXT:    br label [[COND_END]]
 ; CGSCC:       cond.end:
 ; CGSCC-NEXT:    [[COND:%.*]] = phi i8* [ [[TMP1]], [[COND_TRUE]] ], [ [[CALL2]], [[COND_FALSE]] ]
diff --git a/llvm/test/Transforms/Attributor/nofpclass-ceil.ll b/llvm/test/Transforms/Attributor/nofpclass-ceil.ll
new file mode 100644
index 0000000000000..04a2997a5d9cc
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-ceil.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.ceil.f32(float)
+declare ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128)
+
+define float @ret_ceil(float %arg0) {
+; CHECK-LABEL: define float @ret_ceil
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noqnan(float nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nosnan(float nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nozero(float nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopzero(float nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonzero(float nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonorm(float nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonorm
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonnorm(float nofpclass(nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonnorm
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopnorm(float nofpclass(pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopnorm
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonsub(float nofpclass(nsub) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonsub
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopsub(float nofpclass(psub) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopsub
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonorm_nosub(float nofpclass(norm sub) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonorm_nosub
+; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopnorm_nopsub
+; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nonnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define ppc_fp128 @ret_ceil_ppcf128(ppc_fp128 %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_ceil_ppcf128
+; CHECK-SAME: (ppc_fp128 [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_ceil_noinf_ppcf128(ppc_fp128 nofpclass(inf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_ceil_noinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_ceil_nopinf_ppcf128(ppc_fp128 nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_ceil_nopinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_ceil_noninf_ppcf128(ppc_fp128 nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_ceil_noninf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_ceil_nonan_ppcf128(ppc_fp128 nofpclass(nan) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_ceil_nonan_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.ceil.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define float @ret_ceil_noneg(float nofpclass(ninf nsub nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noneg_nonegzero(float nofpclass(ninf nsub nnorm nzero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noneg_nonegzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noneg_nonegzero_nonan(float nofpclass(ninf nsub nnorm nzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noneg_nonegzero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noneg_nozero(float nofpclass(ninf nsub nnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_noneg_nozero_nonan(float nofpclass(ninf nsub nnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_noneg_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopos(float nofpclass(pinf psub pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopos
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopos_nopzero(float nofpclass(pinf psub pnorm pzero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopos_nopzero
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopos_nopzero_nonan(float nofpclass(pinf psub pnorm pzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopos_nopzero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopos_nozero(float nofpclass(pinf psub pnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopos_nozero
+; CHECK-SAME: (float nofpclass(pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_ceil_nopos_nozero_nonan(float nofpclass(pinf psub pnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_ceil_nopos_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.ceil.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.ceil.f32(float %arg0)
+  ret float %call
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-copysign.ll b/llvm/test/Transforms/Attributor/nofpclass-copysign.ll
new file mode 100644
index 0000000000000..231f0a42d0565
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-copysign.ll
@@ -0,0 +1,295 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.copysign.f32(float, float)
+declare float @llvm.fabs.f32(float)
+
+define float @ret_copysign(float %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_fabs_rhs(float %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_copysign_fabs_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FABS_ARG1:%.*]] = call float @llvm.fabs.f32(float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.copysign.f32(float [[ARG0]], float [[FABS_ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %fabs.arg1 = call float @llvm.fabs.f32(float %arg1)
+  %call = call float @llvm.copysign.f32(float %arg0, float %fabs.arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nnan_lhs_fabs_rhs(float nofpclass(nan) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_copysign_nnan_lhs_fabs_rhs
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FABS_ARG1:%.*]] = call float @llvm.fabs.f32(float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.copysign.f32(float [[ARG0]], float [[FABS_ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %fabs.arg1 = call float @llvm.fabs.f32(float %arg1)
+  %call = call float @llvm.copysign.f32(float %arg0, float %fabs.arg1)
+  ret float %call
+}
+
+
+define float @ret_copysign_lhs_fabs_nnan_rhs(float %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_copysign_lhs_fabs_nnan_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FABS_ARG1:%.*]] = call nnan float @llvm.fabs.f32(float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.copysign.f32(float [[ARG0]], float [[FABS_ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %fabs.arg1 = call nnan float @llvm.fabs.f32(float %arg1)
+  %call = call float @llvm.copysign.f32(float %arg0, float %fabs.arg1)
+  ret float %call
+}
+
+
+define float @ret_copysign_noneg_lhs(float nofpclass(ninf nnorm nsub nzero) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_noneg_lhs
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_noneg_rhs(float %arg0, float nofpclass(ninf nnorm nsub nzero) %arg1) {
+; CHECK-LABEL: define float @ret_copysign_noneg_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nzero nsub nnorm) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_noneg_nonan_rhs(float %arg0, float nofpclass(ninf nnorm nsub nzero nan) %arg1) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_copysign_noneg_nonan_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan ninf nzero nsub nnorm) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopos_lhs(float nofpclass(pinf pnorm psub pzero) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nopos_lhs
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopos_rhs(float %arg0, float nofpclass(pinf pnorm psub pzero) %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nopos_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(pinf pzero psub pnorm) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nonan_lhs(float nofpclass(nan) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(nan) float @ret_copysign_nonan_lhs
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nonan_rhs(float %arg0, float nofpclass(nan) %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nonan_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_noneg_nonan_lhs(float nofpclass(ninf nnorm nsub nzero nan) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(nan) float @ret_copysign_noneg_nonan_lhs
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopos_nonan_lhs(float nofpclass(pinf pnorm psub pzero nan) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(nan) float @ret_copysign_nopos_nonan_lhs
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_mixed_lhs0(float nofpclass(ninf pnorm nsub nzero) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_mixed_lhs0
+; CHECK-SAME: (float nofpclass(ninf nzero nsub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_mixed_lhs1(float nofpclass(pinf nnorm psub pzero) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_mixed_lhs1
+; CHECK-SAME: (float nofpclass(pinf pzero psub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_mixed_lhs2(float nofpclass(ninf pnorm psub nzero qnan) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(qnan) float @ret_copysign_mixed_lhs2
+; CHECK-SAME: (float nofpclass(qnan ninf nzero psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(qnan) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_noninf_lhs(float nofpclass(ninf) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_noninf_lhs
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopinf_lhs(float nofpclass(pinf) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nopinf_lhs
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_noinf_lhs(float nofpclass(inf) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(inf) float @ret_copysign_noinf_lhs
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nonzero_lhs(float nofpclass(nzero) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nonzero_lhs
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopzero_lhs(float nofpclass(pzero) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nopzero_lhs
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nozero_lhs(float nofpclass(zero) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(zero) float @ret_copysign_nozero_lhs
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nonsub_lhs(float nofpclass(nsub) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nonsub_lhs
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopsub_lhs(float nofpclass(psub) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nopsub_lhs
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nosub_lhs(float nofpclass(sub) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(sub) float @ret_copysign_nosub_lhs
+; CHECK-SAME: (float nofpclass(sub) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nonnorm_lhs(float nofpclass(nnorm) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nonnorm_lhs
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nopnorm_lhs(float nofpclass(pnorm) %arg0, float %arg1) {
+; CHECK-LABEL: define float @ret_copysign_nopnorm_lhs
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_copysign_nonorm_lhs(float nofpclass(norm) %arg0, float %arg1) {
+; CHECK-LABEL: define nofpclass(norm) float @ret_copysign_nonorm_lhs
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(norm) float @llvm.copysign.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.copysign.f32(float %arg0, float %arg1)
+  ret float %call
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-exp.ll b/llvm/test/Transforms/Attributor/nofpclass-exp.ll
new file mode 100644
index 0000000000000..9c82c60f14ed9
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-exp.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.exp.f32(float)
+declare float @llvm.exp2.f32(float)
+
+define float @ret_exp(float %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_nonan_noinf(float nofpclass(nan inf) %arg0) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp_nonan_noinf
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_nonan_noinf_nozero(float nofpclass(nan inf zero) %arg0) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp_nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_noinf_nozero(float nofpclass(inf zero) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp_noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_noinf_nonegzero(float nofpclass(inf nzero) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp_positive_source(i32 %arg) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp_positive_source
+; CHECK-SAME: (i32 [[ARG:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[UITOFP:%.*]] = uitofp i32 [[ARG]] to float
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp.f32(float [[UITOFP]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %uitofp = uitofp i32 %arg to float
+  %call = call float @llvm.exp.f32(float %uitofp)
+  ret float %call
+}
+
+; Could produce a nan because we don't know if the multiply is negative.
+define float @ret_exp_unknown_sign(float nofpclass(nan) %arg0, float nofpclass(nan) %arg1) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp_unknown_sign
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[UNKNOWN_SIGN_NOT_NAN:%.*]] = fmul nnan float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp.f32(float [[UNKNOWN_SIGN_NOT_NAN]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %unknown.sign.not.nan = fmul nnan float %arg0, %arg1
+  %call = call float @llvm.exp.f32(float %unknown.sign.not.nan)
+  ret float %call
+}
+
+define float @ret_exp2(float %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp2
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp2_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp2_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp2_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp2_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_nonan_noinf(float nofpclass(nan inf) %arg0) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp2_nonan_noinf
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_nonan_noinf_nozero(float nofpclass(nan inf zero) %arg0) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp2_nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_noinf_nozero(float nofpclass(inf zero) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp2_noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_noinf_nonegzero(float nofpclass(inf nzero) %arg0) {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_exp2_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.exp2.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_exp2_positive_source(i32 %arg) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp2_positive_source
+; CHECK-SAME: (i32 [[ARG:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[UITOFP:%.*]] = uitofp i32 [[ARG]] to float
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[UITOFP]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %uitofp = uitofp i32 %arg to float
+  %call = call float @llvm.exp2.f32(float %uitofp)
+  ret float %call
+}
+
+; Could produce a nan because we don't know if the multiply is negative.
+define float @ret_exp2_unknown_sign(float nofpclass(nan) %arg0, float nofpclass(nan) %arg1) {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_exp2_unknown_sign
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[UNKNOWN_SIGN_NOT_NAN:%.*]] = fmul nnan float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[UNKNOWN_SIGN_NOT_NAN]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %unknown.sign.not.nan = fmul nnan float %arg0, %arg1
+  %call = call float @llvm.exp2.f32(float %unknown.sign.not.nan)
+  ret float %call
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-fdiv.ll b/llvm/test/Transforms/Attributor/nofpclass-fdiv.ll
index 999a236a881f4..82a10f60e838e 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-fdiv.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-fdiv.ll
@@ -356,7 +356,7 @@ define float @ret_fdiv_daz_noinf__nozero_nosub(float nofpclass(inf) %arg0, float
 }
 
 define float @ret_fdiv_same_operands(float %arg) #0 {
-; CHECK-LABEL: define float @ret_fdiv_same_operands
+; CHECK-LABEL: define nofpclass(inf zero sub nnorm) float @ret_fdiv_same_operands
 ; CHECK-SAME: (float [[ARG:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[FDIV:%.*]] = fdiv float [[ARG]], [[ARG]]
 ; CHECK-NEXT:    ret float [[FDIV]]
@@ -366,7 +366,7 @@ define float @ret_fdiv_same_operands(float %arg) #0 {
 }
 
 define float @ret_fdiv_same_operands_nosnan(float nofpclass(snan) %arg) #0 {
-; CHECK-LABEL: define float @ret_fdiv_same_operands_nosnan
+; CHECK-LABEL: define nofpclass(inf zero sub nnorm) float @ret_fdiv_same_operands_nosnan
 ; CHECK-SAME: (float nofpclass(snan) [[ARG:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[FDIV:%.*]] = fdiv float [[ARG]], [[ARG]]
 ; CHECK-NEXT:    ret float [[FDIV]]
@@ -376,7 +376,7 @@ define float @ret_fdiv_same_operands_nosnan(float nofpclass(snan) %arg) #0 {
 }
 
 define float @ret_fdiv_same_operands_noqnan(float nofpclass(qnan) %arg) #0 {
-; CHECK-LABEL: define float @ret_fdiv_same_operands_noqnan
+; CHECK-LABEL: define nofpclass(inf zero sub nnorm) float @ret_fdiv_same_operands_noqnan
 ; CHECK-SAME: (float nofpclass(qnan) [[ARG:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[FDIV:%.*]] = fdiv float [[ARG]], [[ARG]]
 ; CHECK-NEXT:    ret float [[FDIV]]
@@ -386,7 +386,7 @@ define float @ret_fdiv_same_operands_noqnan(float nofpclass(qnan) %arg) #0 {
 }
 
 define float @ret_fdiv_same_operands_nonan(float nofpclass(nan) %arg) #0 {
-; CHECK-LABEL: define float @ret_fdiv_same_operands_nonan
+; CHECK-LABEL: define nofpclass(inf zero sub nnorm) float @ret_fdiv_same_operands_nonan
 ; CHECK-SAME: (float nofpclass(nan) [[ARG:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[FDIV:%.*]] = fdiv float [[ARG]], [[ARG]]
 ; CHECK-NEXT:    ret float [[FDIV]]
@@ -436,7 +436,7 @@ define float @ret_fdiv_no_neg(float nofpclass(ninf nsub nnorm) %arg0, float nofp
 }
 
 define float @ret_fdiv_no_neg_nzero(float nofpclass(ninf nsub nnorm nzero) %arg0, float nofpclass(ninf nsub nnorm nzero) %arg1) #0 {
-; CHECK-LABEL: define float @ret_fdiv_no_neg_nzero
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_fdiv_no_neg_nzero
 ; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], float nofpclass(ninf nzero nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[FDIV:%.*]] = fdiv float [[ARG0]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[FDIV]]
@@ -456,7 +456,7 @@ define float @ret_fdiv_no_neg_rhs_no_nzero(float nofpclass(ninf nsub nnorm) %arg
 }
 
 define float @ret_fdiv_no_neg_no_zero_rhs(float nofpclass(ninf nsub nnorm nzero) %arg0, float nofpclass(ninf nsub nnorm zero) %arg1) #0 {
-; CHECK-LABEL: define float @ret_fdiv_no_neg_no_zero_rhs
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_fdiv_no_neg_no_zero_rhs
 ; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], float nofpclass(ninf zero nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    [[FDIV:%.*]] = fdiv float [[ARG0]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[FDIV]]
diff --git a/llvm/test/Transforms/Attributor/nofpclass-floor.ll b/llvm/test/Transforms/Attributor/nofpclass-floor.ll
new file mode 100644
index 0000000000000..b4f28f4212df8
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-floor.ll
@@ -0,0 +1,351 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.floor.f32(float)
+declare ppc_fp128 @llvm.floor.ppcf128(ppc_fp128)
+
+define float @ret_floor(float %arg0) {
+; CHECK-LABEL: define float @ret_floor
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noqnan(float nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nosnan(float nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nozero(float nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopzero(float nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonzero(float nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonorm(float nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonorm
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonnorm(float nofpclass(nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonnorm
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopnorm(float nofpclass(pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopnorm
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonsub(float nofpclass(nsub) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonsub
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopsub(float nofpclass(psub) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopsub
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonorm_nosub(float nofpclass(norm sub) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonorm_nosub
+; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopnorm_nopsub
+; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nonnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define ppc_fp128 @ret_floor_ppcf128(ppc_fp128 %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_floor_ppcf128
+; CHECK-SAME: (ppc_fp128 [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_floor_noinf_ppcf128(ppc_fp128 nofpclass(inf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_floor_noinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_floor_nopinf_ppcf128(ppc_fp128 nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_floor_nopinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_floor_noninf_ppcf128(ppc_fp128 nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_floor_noninf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_floor_nonan_ppcf128(ppc_fp128 nofpclass(nan) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_floor_nonan_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.floor.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define float @ret_floor_noneg(float nofpclass(ninf nsub nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noneg_nonegzero(float nofpclass(ninf nsub nnorm nzero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noneg_nonegzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noneg_nonegzero_nonan(float nofpclass(ninf nsub nnorm nzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noneg_nonegzero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noneg_nozero(float nofpclass(ninf nsub nnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_noneg_nozero_nonan(float nofpclass(ninf nsub nnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_noneg_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopos(float nofpclass(pinf psub pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopos
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopos_nopzero(float nofpclass(pinf psub pnorm pzero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopos_nopzero
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopos_nopzero_nonan(float nofpclass(pinf psub pnorm pzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopos_nopzero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopos_nozero(float nofpclass(pinf psub pnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopos_nozero
+; CHECK-SAME: (float nofpclass(pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_floor_nopos_nozero_nonan(float nofpclass(pinf psub pnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_floor_nopos_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.floor.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.floor.f32(float %arg0)
+  ret float %call
+}
+
+
+
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-fma.ll b/llvm/test/Transforms/Attributor/nofpclass-fma.ll
index 54cad82109119..07a01ae81f3b6 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-fma.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-fma.ll
@@ -5,9 +5,9 @@ declare float @llvm.fma.f32(float, float, float)
 declare float @llvm.fmuladd.f32(float, float, float)
 
 define float @ret_fma_same_mul_arg(float %arg0, float %arg1) {
-; CHECK-LABEL: define float @ret_fma_same_mul_arg
+; CHECK-LABEL: define nofpclass(nzero) float @ret_fma_same_mul_arg
 ; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.fma.f32(float [[ARG0]], float [[ARG0]], float [[ARG1]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nzero) float @llvm.fma.f32(float [[ARG0]], float [[ARG0]], float [[ARG1]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.fma.f32(float %arg0, float %arg0, float %arg1)
@@ -15,9 +15,9 @@ define float @ret_fma_same_mul_arg(float %arg0, float %arg1) {
 }
 
 define float @ret_fma_same_mul_arg_positive_addend(float %arg0, float nofpclass(ninf nsub nnorm) %arg1) {
-; CHECK-LABEL: define float @ret_fma_same_mul_arg_positive_addend
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_fma_same_mul_arg_positive_addend
 ; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.fma.f32(float [[ARG0]], float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.fma.f32(float [[ARG0]], float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.fma.f32(float %arg0, float %arg0, float %arg1)
@@ -35,9 +35,9 @@ define float @ret_fma_different_mul_arg_positive_addend(float %arg0, float %arg1
 }
 
 define float @ret_fmuladd_different_same_arg_positive_addend(float %arg0, float nofpclass(ninf nsub nnorm) %arg1) {
-; CHECK-LABEL: define float @ret_fmuladd_different_same_arg_positive_addend
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_fmuladd_different_same_arg_positive_addend
 ; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.fmuladd.f32(float [[ARG0]], float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.fmuladd.f32(float [[ARG0]], float [[ARG0]], float [[ARG1]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.fmuladd.f32(float %arg0, float %arg0, float %arg1)
diff --git a/llvm/test/Transforms/Attributor/nofpclass-fptrunc.ll b/llvm/test/Transforms/Attributor/nofpclass-fptrunc.ll
index a5a4b004737d3..9274114611a84 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-fptrunc.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-fptrunc.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
 
+declare float @llvm.fptrunc.round.f32.f64(double, metadata)
+
 define float @ret_fptrunc(double %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc
-; CHECK-SAME: (double [[ARG0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (double [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -13,7 +15,7 @@ define float @ret_fptrunc(double %arg0) {
 
 define float @ret_fptrunc_nonan(double nofpclass(nan) %arg0) {
 ; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_nonan
-; CHECK-SAME: (double nofpclass(nan) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -23,7 +25,7 @@ define float @ret_fptrunc_nonan(double nofpclass(nan) %arg0) {
 
 define float @ret_fptrunc_noqnan(double nofpclass(qnan) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_noqnan
-; CHECK-SAME: (double nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -33,7 +35,7 @@ define float @ret_fptrunc_noqnan(double nofpclass(qnan) %arg0) {
 
 define float @ret_fptrunc_nosnan(double nofpclass(snan) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nosnan
-; CHECK-SAME: (double nofpclass(snan) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -43,7 +45,7 @@ define float @ret_fptrunc_nosnan(double nofpclass(snan) %arg0) {
 
 define float @ret_fptrunc_noinf(double nofpclass(inf) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_noinf
-; CHECK-SAME: (double nofpclass(inf) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -53,7 +55,7 @@ define float @ret_fptrunc_noinf(double nofpclass(inf) %arg0) {
 
 define float @ret_fptrunc_nopinf(double nofpclass(pinf) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nopinf
-; CHECK-SAME: (double nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -63,7 +65,7 @@ define float @ret_fptrunc_nopinf(double nofpclass(pinf) %arg0) {
 
 define float @ret_fptrunc_noninf(double nofpclass(ninf) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_noninf
-; CHECK-SAME: (double nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -73,7 +75,7 @@ define float @ret_fptrunc_noninf(double nofpclass(ninf) %arg0) {
 
 define float @ret_fptrunc_nozero(double nofpclass(zero) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nozero
-; CHECK-SAME: (double nofpclass(zero) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -83,7 +85,7 @@ define float @ret_fptrunc_nozero(double nofpclass(zero) %arg0) {
 
 define float @ret_fptrunc_nopzero(double nofpclass(pzero) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nopzero
-; CHECK-SAME: (double nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -93,7 +95,7 @@ define float @ret_fptrunc_nopzero(double nofpclass(pzero) %arg0) {
 
 define float @ret_fptrunc_nonzero(double nofpclass(nzero) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nonzero
-; CHECK-SAME: (double nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -103,7 +105,7 @@ define float @ret_fptrunc_nonzero(double nofpclass(nzero) %arg0) {
 
 define float @ret_fptrunc_nonan_noinf(double nofpclass(nan inf) %arg0) {
 ; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_nonan_noinf
-; CHECK-SAME: (double nofpclass(nan inf) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(nan inf) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -113,7 +115,7 @@ define float @ret_fptrunc_nonan_noinf(double nofpclass(nan inf) %arg0) {
 
 define float @ret_fptrunc_nosub(double nofpclass(sub) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nosub
-; CHECK-SAME: (double nofpclass(sub) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(sub) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -123,7 +125,7 @@ define float @ret_fptrunc_nosub(double nofpclass(sub) %arg0) {
 
 define float @ret_fptrunc_nonorm(double nofpclass(norm) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_nonorm
-; CHECK-SAME: (double nofpclass(norm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -133,7 +135,7 @@ define float @ret_fptrunc_nonorm(double nofpclass(norm) %arg0) {
 
 define float @ret_fptrunc_posonly(double nofpclass(ninf nnorm nsub nzero) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_posonly
-; CHECK-SAME: (double nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -143,7 +145,7 @@ define float @ret_fptrunc_posonly(double nofpclass(ninf nnorm nsub nzero) %arg0)
 
 define float @ret_fptrunc_posonly_zero(double nofpclass(ninf nnorm nsub) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_posonly_zero
-; CHECK-SAME: (double nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -153,7 +155,7 @@ define float @ret_fptrunc_posonly_zero(double nofpclass(ninf nnorm nsub) %arg0)
 
 define float @ret_fptrunc_posonly_zero_nan(double nofpclass(ninf nnorm nsub nan) %arg0) {
 ; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_posonly_zero_nan
-; CHECK-SAME: (double nofpclass(nan ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(nan ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -163,7 +165,7 @@ define float @ret_fptrunc_posonly_zero_nan(double nofpclass(ninf nnorm nsub nan)
 
 define float @ret_fptrunc_posonly_nan(double nofpclass(ninf nnorm nsub nzero nan) %arg0) {
 ; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_posonly_nan
-; CHECK-SAME: (double nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -173,7 +175,7 @@ define float @ret_fptrunc_posonly_nan(double nofpclass(ninf nnorm nsub nzero nan
 
 define float @ret_fptrunc_negonly(double nofpclass(pinf pnorm psub pzero) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_negonly
-; CHECK-SAME: (double nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -183,7 +185,7 @@ define float @ret_fptrunc_negonly(double nofpclass(pinf pnorm psub pzero) %arg0)
 
 define float @ret_fptrunc_negonly_zero(double nofpclass(pinf pnorm psub) %arg0) {
 ; CHECK-LABEL: define float @ret_fptrunc_negonly_zero
-; CHECK-SAME: (double nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -193,7 +195,7 @@ define float @ret_fptrunc_negonly_zero(double nofpclass(pinf pnorm psub) %arg0)
 
 define float @ret_fptrunc_negonly_zero_nan(double nofpclass(pinf pnorm psub nan) %arg0) {
 ; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_negonly_zero_nan
-; CHECK-SAME: (double nofpclass(nan pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: (double nofpclass(nan pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    [[EXT:%.*]] = fptrunc double [[ARG0]] to float
 ; CHECK-NEXT:    ret float [[EXT]]
 ;
@@ -201,5 +203,135 @@ define float @ret_fptrunc_negonly_zero_nan(double nofpclass(pinf pnorm psub nan)
   ret float %ext
 }
 
+define float @ret_fptrunc_round(double %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round
+; CHECK-SAME: (double [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nonan(double nofpclass(nan) %arg0) {
+; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_round_nonan
+; CHECK-SAME: (double nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call nofpclass(nan) float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_noqnan(double nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_noqnan
+; CHECK-SAME: (double nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nosnan(double nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nosnan
+; CHECK-SAME: (double nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_noinf(double nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_noinf
+; CHECK-SAME: (double nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nopinf(double nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nopinf
+; CHECK-SAME: (double nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_noninf(double nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_noninf
+; CHECK-SAME: (double nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nozero(double nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nozero
+; CHECK-SAME: (double nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nopzero(double nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nopzero
+; CHECK-SAME: (double nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nonzero(double nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nonzero
+; CHECK-SAME: (double nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nonan_noinf(double nofpclass(nan inf) %arg0) {
+; CHECK-LABEL: define nofpclass(nan) float @ret_fptrunc_round_nonan_noinf
+; CHECK-SAME: (double nofpclass(nan inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call nofpclass(nan) float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nosub(double nofpclass(sub) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nosub
+; CHECK-SAME: (double nofpclass(sub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
+define float @ret_fptrunc_round_nonorm(double nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_fptrunc_round_nonorm
+; CHECK-SAME: (double nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[EXT:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0]], metadata !"round.downward") #[[ATTR2]]
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %ext = call float @llvm.fptrunc.round.f32.f64(double %arg0, metadata !"round.downward")
+  ret float %ext
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-frem.ll b/llvm/test/Transforms/Attributor/nofpclass-frem.ll
new file mode 100644
index 0000000000000..930b1a77da57c
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-frem.ll
@@ -0,0 +1,513 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+define float @ret_frem_ieee(float %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz(float %arg0, float %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_noinf_nozero__all(float nofpclass(inf zero) %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_noinf_nozero__all
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_nozero__all(float nofpclass(nan inf zero) %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf_nozero__all
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_all__nonan_noinf_nozero(float %arg0, float nofpclass(nan inf zero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_all__nonan_noinf_nozero
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_nozero__nonan(float nofpclass(nan inf zero) %arg0, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf_nozero__nonan
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan__nonan_noinf_nozero(float nofpclass(nan) %arg0, float nofpclass(nan inf zero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_nozero__nonan_noinf(float nofpclass(nan zero) %arg0, float nofpclass(nan inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_nozero__nonan_noinf
+; CHECK-SAME: (float nofpclass(nan zero) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf__nonan_nozero(float nofpclass(nan inf) %arg0, float nofpclass(nan zero) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_ieee_nonan_noinf__nonan_nozero
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_nonan_nozero__nonan_noinf(float nofpclass(nan zero) %arg0, float nofpclass(nan inf) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_nonan_nozero__nonan_noinf
+; CHECK-SAME: (float nofpclass(nan zero) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_nonan_noinf__nonan_nozero(float nofpclass(nan inf) %arg0, float nofpclass(nan zero) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_nonan_noinf__nonan_nozero
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan zero) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_nonan_nozero_nosub__nonan_noinf(float nofpclass(nan zero sub) %arg0, float nofpclass(nan inf) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_nonan_nozero_nosub__nonan_noinf
+; CHECK-SAME: (float nofpclass(nan zero sub) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_nonan_noinf__nonan_nozero_nosub(float nofpclass(nan inf) %arg0, float nofpclass(nan zero sub) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_daz_nonan_noinf__nonan_nozero_nosub
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan zero sub) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf__nonan_noinf(float nofpclass(nan) %arg0, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf__nonan_noinf
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_nozero__nonan_noinf_nozero(float nofpclass(nan inf zero) %arg0, float nofpclass(nan inf zero) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_ieee_nonan_noinf_nozero__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_nonan_noinf_nozero__nonan_noinf_nozero(float nofpclass(nan inf zero) %arg0, float nofpclass(nan inf zero) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_nonan_noinf_nozero__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_dapz_nonan_noinf_nozero__nonan_noinf_nozero(float nofpclass(nan inf zero) %arg0, float nofpclass(nan inf zero) %arg1) #2 {
+; CHECK-LABEL: define float @ret_frem_dapz_nonan_noinf_nozero__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_dynamic_nonan_noinf_nozero__nonan_noinf_nozero(float nofpclass(nan inf zero) %arg0, float nofpclass(nan inf zero) %arg1) #3 {
+; CHECK-LABEL: define float @ret_frem_dynamic_nonan_noinf_nozero__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_noinf__nonan_noinf_noinf(float nofpclass(nan inf) %arg0, float nofpclass(nan inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf_noinf__nonan_noinf_noinf
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_nozero__nonan_noinf_noinf(float nofpclass(nan inf zero) %arg0, float nofpclass(nan inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf_nozero__nonan_noinf_noinf
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_noinf__nonan_noinf_nozero(float nofpclass(nan inf) %arg0, float nofpclass(nan inf zero) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_ieee_nonan_noinf_noinf__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_noinf__nonan_noinf_noposzero(float nofpclass(nan inf) %arg0, float nofpclass(nan pzero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf_noinf__nonan_noinf_noposzero
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan pzero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_noposzero__nonan_noinf_noinf(float nofpclass(nan pzero) %arg0, float nofpclass(nan inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nonan_noinf_noposzero__nonan_noinf_noinf
+; CHECK-SAME: (float nofpclass(nan pzero) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub(float nofpclass(nan inf zero sub) %arg0, float nofpclass(nan inf zero sub) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_ieee_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub
+; CHECK-SAME: (float nofpclass(nan inf zero sub) [[ARG0:%.*]], float nofpclass(nan inf zero sub) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+; Denormal mode doesn't matter because sources are nofpclass(sub)
+define float @ret_frem_daz_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub(float nofpclass(nan inf zero sub) %arg0, float nofpclass(nan inf zero sub) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_daz_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub
+; CHECK-SAME: (float nofpclass(nan inf zero sub) [[ARG0:%.*]], float nofpclass(nan inf zero sub) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_dapz_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub(float nofpclass(nan inf zero sub) %arg0, float nofpclass(nan inf zero sub) %arg1) #2 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_dapz_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub
+; CHECK-SAME: (float nofpclass(nan inf zero sub) [[ARG0:%.*]], float nofpclass(nan inf zero sub) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_dynamic_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub(float nofpclass(nan inf zero sub) %arg0, float nofpclass(nan inf zero sub) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_dynamic_nonan_noinf_nozero_nosub__nonan_noinf_nozero_nosub
+; CHECK-SAME: (float nofpclass(nan inf zero sub) [[ARG0:%.*]], float nofpclass(nan inf zero sub) [[ARG1:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+; Missing no-subnormal on lhs
+define float @ret_frem_daz_nonan_noinf_nozero__nonan_noinf_nozero_nosub(float nofpclass(nan inf zero) %arg0, float nofpclass(nan inf zero sub) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_frem_daz_nonan_noinf_nozero__nonan_noinf_nozero_nosub
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG0:%.*]], float nofpclass(nan inf zero sub) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+; Missing no-subnormal on lhs
+define float @ret_frem_daz_nonan_noinf_nozero_nosub__nonan_noinf_nozero(float nofpclass(nan inf zero sub) %arg0, float nofpclass(nan inf zero) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_nonan_noinf_nozero_nosub__nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero sub) [[ARG0:%.*]], float nofpclass(nan inf zero) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+; should be able to infer noinf
+define float @ret_frem_ieee_noinf__nozero(float nofpclass(inf) %arg0, float nofpclass(zero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_noinf__nozero
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_all__nozero(float %arg0, float nofpclass(zero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_all__nozero
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_noinf__all(float nofpclass(inf) %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_noinf__all
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_ieee_nozero__noinf(float nofpclass(zero) %arg0, float nofpclass(inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_ieee_nozero__noinf
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_noinf__nozero(float nofpclass(inf) %arg0, float nofpclass(zero) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_noinf__nozero
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_daz_noinf__nozero_nosub(float nofpclass(inf) %arg0, float nofpclass(zero sub) %arg1) #1 {
+; CHECK-LABEL: define float @ret_frem_daz_noinf__nozero_nosub
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(zero sub) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_same_operands(float %arg) #0 {
+; CHECK-LABEL: define nofpclass(inf sub norm) float @ret_frem_same_operands
+; CHECK-SAME: (float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg, %arg
+  ret float %frem
+}
+
+define float @ret_frem_same_operands_nosnan(float nofpclass(snan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(inf sub norm) float @ret_frem_same_operands_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg, %arg
+  ret float %frem
+}
+
+define float @ret_frem_same_operands_noqnan(float nofpclass(qnan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(inf sub norm) float @ret_frem_same_operands_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg, %arg
+  ret float %frem
+}
+
+define float @ret_frem_same_operands_nonan(float nofpclass(nan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(inf sub norm) float @ret_frem_same_operands_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg, %arg
+  ret float %frem
+}
+
+define float @ret_frem_no_neg_lhs(float nofpclass(ninf nsub nnorm) %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_neg_lhs
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_neg_rhs(float %arg0, float nofpclass(ninf nsub nnorm) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_neg_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_neg_nzero_rhs(float %arg0, float nofpclass(ninf nsub nnorm nzero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_neg_nzero_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nzero nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_neg(float nofpclass(ninf nsub nnorm) %arg0, float nofpclass(ninf nsub nnorm) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_neg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], float nofpclass(ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_neg_nzero(float nofpclass(ninf nsub nnorm nzero) %arg0, float nofpclass(ninf nsub nnorm nzero) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_frem_no_neg_nzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], float nofpclass(ninf nzero nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_neg_rhs_no_nzero(float nofpclass(ninf nsub nnorm) %arg0, float nofpclass(ninf nsub nnorm nzero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_neg_rhs_no_nzero
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], float nofpclass(ninf nzero nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_neg_no_zero_rhs(float nofpclass(ninf nsub nnorm nzero) %arg0, float nofpclass(ninf nsub nnorm zero) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_frem_no_neg_no_zero_rhs
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], float nofpclass(ninf zero nsub nnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_pos_lhs(float nofpclass(pinf psub pnorm pzero) %arg0, float %arg1) #0 {
+; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @ret_frem_no_pos_lhs
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_pos_rhs(float %arg0, float nofpclass(pinf psub pnorm pzero) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_pos_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(pinf pzero psub pnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_pos_zero_lhs(float nofpclass(pinf psub pnorm) %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_pos_zero_lhs
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+define float @ret_frem_no_pos_zero_rhs(float %arg0, float nofpclass(pinf psub pnorm) %arg1) #0 {
+; CHECK-LABEL: define float @ret_frem_no_pos_zero_rhs
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FREM:%.*]] = frem float [[ARG0]], [[ARG1]]
+; CHECK-NEXT:    ret float [[FREM]]
+;
+  %frem = frem float %arg0, %arg1
+  ret float %frem
+}
+
+attributes #0 = { "denormal-fp-math"="ieee,ieee" }
+attributes #1 = { "denormal-fp-math"="ieee,preserve-sign" }
+attributes #2 = { "denormal-fp-math"="ieee,positive-zero" }
+attributes #3 = { "denormal-fp-math"="ieee,dynamic" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-log.ll b/llvm/test/Transforms/Attributor/nofpclass-log.ll
new file mode 100644
index 0000000000000..f96f4080c3975
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-log.ll
@@ -0,0 +1,397 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.log.f32(float)
+declare float @llvm.log2.f32(float)
+declare float @llvm.log10.f32(float)
+
+declare <2 x float> @llvm.log.v2f32(<2 x float>)
+declare float @llvm.experimental.constrained.log.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log2.f32(float, metadata, metadata)
+declare float @llvm.experimental.constrained.log10.f32(float, metadata, metadata)
+
+define float @ret_log(float %arg) #0 {
+; CHECK-LABEL: define float @ret_log
+; CHECK-SAME: (float [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.log.f32(float [[ARG]]) #[[ATTR10:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noinf(float nofpclass(inf) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noneg(float nofpclass(ninf nsub nnorm) %arg) #0 {
+; CHECK-LABEL: define float @ret_log_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noneg_nonan(float nofpclass(ninf nsub nnorm nan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_log_noneg_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noinf_noneg(float nofpclass(inf nsub nnorm) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_noinf_noneg
+; CHECK-SAME: (float nofpclass(inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noinf_noneg_nonan(float nofpclass(inf nsub nnorm nan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(nan pinf) float @ret_log_noinf_noneg_nonan
+; CHECK-SAME: (float nofpclass(nan inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_nopinf(float nofpclass(pinf) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noninf(float nofpclass(ninf) %arg) #0 {
+; CHECK-LABEL: define float @ret_log_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_nonan(float nofpclass(nan) %arg) #0 {
+; CHECK-LABEL: define float @ret_log_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_nonan_noinf(float nofpclass(nan inf) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_nonan_noinf
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_nonan_noinf_nozero(float nofpclass(nan inf zero) %arg) #0 {
+; CHECK-LABEL: define nofpclass(inf) float @ret_log_nonan_noinf_nozero
+; CHECK-SAME: (float nofpclass(nan inf zero) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noinf_nozero(float nofpclass(inf zero) %arg) #0 {
+; CHECK-LABEL: define nofpclass(inf) float @ret_log_noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_noinf_nonegzero(float nofpclass(inf nzero) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_positive_source(i32 %arg) #0 {
+; CHECK-LABEL: define nofpclass(nan pinf) float @ret_log_positive_source
+; CHECK-SAME: (i32 [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[UITOFP:%.*]] = uitofp i32 [[ARG]] to float
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan pinf) float @llvm.log.f32(float [[UITOFP]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %uitofp = uitofp i32 %arg to float
+  %call = call float @llvm.log.f32(float %uitofp)
+  ret float %call
+}
+
+; Could produce a nan because we don't know if the multiply is negative.
+define float @ret_log_unknown_sign(float nofpclass(nan) %arg, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define float @ret_log_unknown_sign
+; CHECK-SAME: (float nofpclass(nan) [[ARG:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[UNKNOWN_SIGN_NOT_NAN:%.*]] = fmul nnan float [[ARG]], [[ARG1]]
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.log.f32(float [[UNKNOWN_SIGN_NOT_NAN]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %unknown.sign.not.nan = fmul nnan float %arg, %arg1
+  %call = call float @llvm.log.f32(float %unknown.sign.not.nan)
+  ret float %call
+}
+
+define float @ret_log_daz_noinf_nozero(float nofpclass(inf zero) %arg) #1 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_daz_noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define <2 x float> @ret_log_daz_noinf_nozero_v2f32(<2 x float> nofpclass(inf zero) %arg) #1 {
+; CHECK-LABEL: define nofpclass(pinf) <2 x float> @ret_log_daz_noinf_nozero_v2f32
+; CHECK-SAME: (<2 x float> nofpclass(inf zero) [[ARG:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) <2 x float> @llvm.log.v2f32(<2 x float> [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret <2 x float> [[CALL]]
+;
+  %call = call <2 x float> @llvm.log.v2f32(<2 x float> %arg)
+  ret <2 x float> %call
+}
+
+define float @ret_log_daz_noinf_nonegzero(float nofpclass(inf nzero) %arg) #1 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_daz_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_dapz_noinf_nozero(float nofpclass(inf zero) %arg) #2 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_dapz_noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_dapz_noinf_nonegzero(float nofpclass(inf nzero) %arg) #2 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_dapz_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_dynamic_noinf_nozero(float nofpclass(inf zero) %arg) #3 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_dynamic_noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_dynamic_noinf_nonegzero(float nofpclass(inf nzero) %arg) #3 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_dynamic_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR5]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_ftz_noinf_nonegzero(float nofpclass(inf nzero) %arg) #4 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_ftz_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_ftpz_noinf_nonegzero(float nofpclass(inf nzero) %arg) #5 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_ftpz_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log_ftz_dynamic_noinf_nonegzero(float nofpclass(inf nzero) %arg) #6 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log_ftz_dynamic_noinf_nonegzero
+; CHECK-SAME: (float nofpclass(inf nzero) [[ARG:%.*]]) #[[ATTR8:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log.f32(float %arg)
+  ret float %call
+}
+
+define float @constrained_log(float %arg) strictfp {
+; CHECK-LABEL: define float @constrained_log
+; CHECK-SAME: (float [[ARG:%.*]]) #[[ATTR9:[0-9]+]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.experimental.constrained.log.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.experimental.constrained.log.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @constrained_log_nonan(float nofpclass(nan) %arg) strictfp {
+; CHECK-LABEL: define float @constrained_log_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG:%.*]]) #[[ATTR9]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.experimental.constrained.log.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.experimental.constrained.log.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @constrained_log_nopinf(float nofpclass(pinf) %arg) strictfp {
+; CHECK-LABEL: define nofpclass(pinf) float @constrained_log_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG:%.*]]) #[[ATTR9]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call nofpclass(pinf) float @llvm.experimental.constrained.log.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.experimental.constrained.log.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @constrained_log_nonegzero(float nofpclass(nzero) %arg) strictfp {
+; CHECK-LABEL: define float @constrained_log_nonegzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG:%.*]]) #[[ATTR9]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.experimental.constrained.log.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.experimental.constrained.log.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @constrained_log_nozero(float nofpclass(zero) %arg) strictfp {
+; CHECK-LABEL: define nofpclass(ninf) float @constrained_log_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG:%.*]]) #[[ATTR9]] {
+; CHECK-NEXT:    [[VAL:%.*]] = call nofpclass(ninf) float @llvm.experimental.constrained.log.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.experimental.constrained.log.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %val
+}
+
+define float @ret_log2_noinf_noneg(float nofpclass(inf nsub nnorm) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log2_noinf_noneg
+; CHECK-SAME: (float nofpclass(inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log2.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log2.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log2_noinf_noneg_nonan(float nofpclass(inf nsub nnorm nan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(nan pinf) float @ret_log2_noinf_noneg_nonan
+; CHECK-SAME: (float nofpclass(nan inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan pinf) float @llvm.log2.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log2.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log2_noinf_noneg_noqnan(float nofpclass(inf nsub nnorm qnan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log2_noinf_noneg_noqnan
+; CHECK-SAME: (float nofpclass(qnan inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log2.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log2.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log2_noinf_noneg_nosnan(float nofpclass(inf nsub nnorm snan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log2_noinf_noneg_nosnan
+; CHECK-SAME: (float nofpclass(snan inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log2.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log2.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_log10_noinf_noneg(float nofpclass(inf nsub nnorm) %arg) #0 {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_log10_noinf_noneg
+; CHECK-SAME: (float nofpclass(inf nsub nnorm) [[ARG:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.log10.f32(float [[ARG]]) #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.log10.f32(float %arg)
+  ret float %call
+}
+
+define float @ret_constrained_log2_noinf_noneg(float nofpclass(inf nsub nnorm) %arg) strictfp {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_constrained_log2_noinf_noneg
+; CHECK-SAME: (float nofpclass(inf nsub nnorm) [[ARG:%.*]]) #[[ATTR9]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.experimental.constrained.log2.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.experimental.constrained.log2.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %call
+}
+
+define float @ret_constrained_log10_noinf_noneg(float nofpclass(inf nsub nnorm) %arg) strictfp {
+; CHECK-LABEL: define nofpclass(pinf) float @ret_constrained_log10_noinf_noneg
+; CHECK-SAME: (float nofpclass(inf nsub nnorm) [[ARG:%.*]]) #[[ATTR9]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf) float @llvm.experimental.constrained.log10.f32(float [[ARG]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR10]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.experimental.constrained.log10.f32(float %arg, metadata !"round.dynamic", metadata !"fpexcept.strict")
+  ret float %call
+}
+
+attributes #0 = { "denormal-fp-math"="ieee,ieee" }
+attributes #1 = { "denormal-fp-math"="ieee,preserve-sign" }
+attributes #2 = { "denormal-fp-math"="ieee,positive-zero" }
+attributes #3 = { "denormal-fp-math"="ieee,dynamic" }
+attributes #4 = { "denormal-fp-math"="preserve-sign,ieee" }
+attributes #5 = { "denormal-fp-math"="positive-zero,ieee" }
+attributes #6 = { "denormal-fp-math"="dynamic,ieee" }
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll b/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll
new file mode 100644
index 0000000000000..09c7c4b3fb858
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-minimum-maximum.ll
@@ -0,0 +1,516 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.minimum.f32(float, float)
+declare float @llvm.maximum.f32(float, float)
+declare <2 x float> @llvm.minimum.v2f32(<2 x float>, <2 x float>)
+
+define float @ret_minimum(float %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_minimum
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_noinf__noinf(float nofpclass(inf) %arg0, float nofpclass(inf) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(inf) float @ret_minimum_noinf__noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_noinf__nonan(float nofpclass(inf) %arg0, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define float @ret_minimum_noinf__nonan
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_nonan__noinf(float nofpclass(nan) %arg0, float nofpclass(inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_minimum_nonan__noinf
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_noinf_nonan__nonan(float nofpclass(inf nan) %arg0, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_minimum_noinf_nonan__nonan
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_nonan__noinf_nonan(float nofpclass(nan) %arg0, float nofpclass(inf nan) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_minimum_nonan__noinf_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_norm_zero__norm_sub(float nofpclass(norm zero) %arg0, float nofpclass(norm sub) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(norm) float @ret_minimum_norm_zero__norm_sub
+; CHECK-SAME: (float nofpclass(zero norm) [[ARG0:%.*]], float nofpclass(sub norm) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(norm) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum(float %arg0, float %arg1) #0 {
+; CHECK-LABEL: define float @ret_maximum
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_noinf__noinf(float nofpclass(inf) %arg0, float nofpclass(inf) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(inf) float @ret_maximum_noinf__noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_noinf__nonan(float nofpclass(inf) %arg0, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define float @ret_maximum_noinf__nonan
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_nonan__noinf(float nofpclass(nan) %arg0, float nofpclass(inf) %arg1) #0 {
+; CHECK-LABEL: define float @ret_maximum_nonan__noinf
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_noinf_nonan__nonan(float nofpclass(inf nan) %arg0, float nofpclass(nan) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_maximum_noinf_nonan__nonan
+; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_nonan__noinf_nonan(float nofpclass(nan) %arg0, float nofpclass(inf nan) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(nan) float @ret_maximum_nonan__noinf_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_norm_zero__norm_sub(float nofpclass(norm zero) %arg0, float nofpclass(norm sub) %arg1) #0 {
+; CHECK-LABEL: define nofpclass(norm) float @ret_maximum_norm_zero__norm_sub
+; CHECK-SAME: (float nofpclass(zero norm) [[ARG0:%.*]], float nofpclass(sub norm) [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(norm) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_daz_daz(float %arg0, float %arg1) #1 {
+; CHECK-LABEL: define float @ret_minimum_daz_daz
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dapz_dappz(float %arg0, float %arg1) #2 {
+; CHECK-LABEL: define float @ret_minimum_dapz_dappz
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+define float @ret_minimum_dynamic_dynamic(float %arg0, float %arg1) #3 {
+; CHECK-LABEL: define float @ret_minimum_dynamic_dynamic
+; CHECK-SAME: (float [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_noinf_nozero__noinf_nozero(float nofpclass(inf zero) %arg0, float nofpclass(inf zero) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(inf zero) float @ret_minimum_noinf_nozero__noinf_nozero
+; CHECK-SAME: (float nofpclass(inf zero) [[ARG0:%.*]], float nofpclass(inf zero) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define <2 x float> @ret_minimum_noinf_nozero__noinf_nozero_v2f32(<2 x float> nofpclass(inf zero) %arg0, <2 x float> nofpclass(inf zero) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(inf zero) <2 x float> @ret_minimum_noinf_nozero__noinf_nozero_v2f32
+; CHECK-SAME: (<2 x float> nofpclass(inf zero) [[ARG0:%.*]], <2 x float> nofpclass(inf zero) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf zero) <2 x float> @llvm.minimum.v2f32(<2 x float> [[ARG0]], <2 x float> [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret <2 x float> [[CALL]]
+;
+  %call = call <2 x float> @llvm.minimum.v2f32(<2 x float> %arg0, <2 x float> %arg1)
+  ret <2 x float> %call
+}
+
+define float @ret_minimum_daz_daz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_daz_daz_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dapz_dapz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #2 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_dapz_dapz_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dynamic_dynamic_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_dynamic_dynamic_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_daz_daz_nozero_nosub__nozero_nosub(float nofpclass(zero sub) %arg0, float nofpclass(zero sub) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(zero sub) float @ret_minimum_daz_daz_nozero_nosub__nozero_nosub
+; CHECK-SAME: (float nofpclass(zero sub) [[ARG0:%.*]], float nofpclass(zero sub) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero sub) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dynamic_dynamic_nozero_nosub__nozero_nosub(float nofpclass(zero sub) %arg0, float nofpclass(zero sub) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(zero sub) float @ret_minimum_dynamic_dynamic_nozero_nosub__nozero_nosub
+; CHECK-SAME: (float nofpclass(zero sub) [[ARG0:%.*]], float nofpclass(zero sub) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero sub) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_daz_daz_nopzero__nopzero(float nofpclass(pzero) %arg0, float nofpclass(pzero) %arg1) #1 {
+; CHECK-LABEL: define float @ret_minimum_daz_daz_nopzero__nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]], float nofpclass(pzero) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_daz_daz_nonzero__nonzero(float nofpclass(nzero) %arg0, float nofpclass(nzero) %arg1) #1 {
+; CHECK-LABEL: define float @ret_minimum_daz_daz_nonzero__nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]], float nofpclass(nzero) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_daz_daz_nonzero_nonsub__nonzero_nonsub(float nofpclass(nzero nsub) %arg0, float nofpclass(nzero nsub) %arg1) #1 {
+; CHECK-LABEL: define nofpclass(nsub) float @ret_minimum_daz_daz_nonzero_nonsub__nonzero_nonsub
+; CHECK-SAME: (float nofpclass(nzero nsub) [[ARG0:%.*]], float nofpclass(nzero nsub) [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nsub) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dapz_dapz_nopzero__nopzero(float nofpclass(pzero) %arg0, float nofpclass(pzero) %arg1) #2 {
+; CHECK-LABEL: define float @ret_minimum_dapz_dapz_nopzero__nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]], float nofpclass(pzero) [[ARG1:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dapz_dapz_nopzero_nopsub__nopzero_nopsub(float nofpclass(pzero psub) %arg0, float nofpclass(pzero psub) %arg1) #2 {
+; CHECK-LABEL: define nofpclass(psub) float @ret_minimum_dapz_dapz_nopzero_nopsub__nopzero_nopsub
+; CHECK-SAME: (float nofpclass(pzero psub) [[ARG0:%.*]], float nofpclass(pzero psub) [[ARG1:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(psub) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dapz_dapz_nonzero__nonzero(float nofpclass(nzero) %arg0, float nofpclass(nzero) %arg1) #2 {
+; CHECK-LABEL: define float @ret_minimum_dapz_dapz_nonzero__nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]], float nofpclass(nzero) [[ARG1:%.*]]) #[[ATTR3]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_ieee_daz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #4 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_ieee_daz_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR5:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_daz_ieee_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #5 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_daz_ieee_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR6:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_ieee_dapz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #6 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_ieee_dapz_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_dapz_ieee_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #7 {
+; CHECK-LABEL: define nofpclass(zero) float @ret_minimum_dapz_ieee_nozero__nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR8:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_noneg_nan__any(float nofpclass(ninf nsub nnorm nan) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define float @ret_minimum_noneg_nan__any
+; CHECK-SAME: (float nofpclass(nan ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_any__noneg_nan(float %arg0, float nofpclass(ninf nsub nnorm nan) %arg1) #3 {
+; CHECK-LABEL: define float @ret_minimum_any__noneg_nan
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_nopos_nan__any(float nofpclass(pinf psub pnorm nan) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define nofpclass(pinf psub pnorm) float @ret_minimum_nopos_nan__any
+; CHECK-SAME: (float nofpclass(nan pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf psub pnorm) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_any__nopos_nan(float %arg0, float nofpclass(pinf psub pnorm nan) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(pinf psub pnorm) float @ret_minimum_any__nopos_nan
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf psub pnorm) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_noneg__any(float nofpclass(ninf nsub nnorm) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define float @ret_minimum_noneg__any
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_any__noneg(float %arg0, float nofpclass(ninf nsub nnorm) %arg1) #3 {
+; CHECK-LABEL: define float @ret_minimum_any__noneg
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_nopos__any(float nofpclass(pinf psub pnorm) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define nofpclass(pinf psub pnorm) float @ret_minimum_nopos__any
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf psub pnorm) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_minimum_any__nopos(float %arg0, float nofpclass(pinf psub pnorm) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(pinf psub pnorm) float @ret_minimum_any__nopos
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(pinf psub pnorm) float @llvm.minimum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.minimum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_noneg_nan__any(float nofpclass(ninf nsub nnorm nan) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define nofpclass(ninf nsub nnorm) float @ret_maximum_noneg_nan__any
+; CHECK-SAME: (float nofpclass(nan ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nsub nnorm) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_any__noneg_nan(float %arg0, float nofpclass(ninf nsub nnorm nan) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(ninf nsub nnorm) float @ret_maximum_any__noneg_nan
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nsub nnorm) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_nopos_nan__any(float nofpclass(pinf psub pnorm nan) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define float @ret_maximum_nopos_nan__any
+; CHECK-SAME: (float nofpclass(nan pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_any__nopos_nan(float %arg0, float nofpclass(pinf psub pnorm nan) %arg1) #3 {
+; CHECK-LABEL: define float @ret_maximum_any__nopos_nan
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_noneg__any(float nofpclass(ninf nsub nnorm) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define nofpclass(ninf nsub nnorm) float @ret_maximum_noneg__any
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nsub nnorm) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_any__noneg(float %arg0, float nofpclass(ninf nsub nnorm) %arg1) #3 {
+; CHECK-LABEL: define nofpclass(ninf nsub nnorm) float @ret_maximum_any__noneg
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nsub nnorm) float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_nopos__any(float nofpclass(pinf psub pnorm) %arg0, float %arg1) #3 {
+; CHECK-LABEL: define float @ret_maximum_nopos__any
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+define float @ret_maximum_any__nopos(float %arg0, float nofpclass(pinf psub pnorm) %arg1) #3 {
+; CHECK-LABEL: define float @ret_maximum_any__nopos
+; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maximum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.maximum.f32(float %arg0, float %arg1)
+  ret float %call
+}
+
+attributes #0 = { "denormal-fp-math"="ieee,ieee" }
+attributes #1 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #2 = { "denormal-fp-math"="positive-zero,positive-zero" }
+attributes #3 = { "denormal-fp-math"="dynamic,dynamic" }
+attributes #4 = { "denormal-fp-math"="ieee,preserve-sign" }
+attributes #5 = { "denormal-fp-math"="preserve-sign,ieee" }
+attributes #6 = { "denormal-fp-math"="ieee,positive-zero" }
+attributes #7 = { "denormal-fp-math"="positive-zero-sign,ieee" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll b/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll
index 2a41d0fd099dd..8be3d59a9b3b4 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-minnum-maxnum.ll
@@ -16,9 +16,9 @@ define float @ret_minnum(float %arg0, float %arg1) #0 {
 }
 
 define float @ret_minnum_noinf__noinf(float nofpclass(inf) %arg0, float nofpclass(inf) %arg1) #0 {
-; CHECK-LABEL: define float @ret_minnum_noinf__noinf
+; CHECK-LABEL: define nofpclass(inf) float @ret_minnum_noinf__noinf
 ; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -26,9 +26,9 @@ define float @ret_minnum_noinf__noinf(float nofpclass(inf) %arg0, float nofpclas
 }
 
 define float @ret_minnum_noinf__nonan(float nofpclass(inf) %arg0, float nofpclass(nan) %arg1) #0 {
-; CHECK-LABEL: define float @ret_minnum_noinf__nonan
+; CHECK-LABEL: define nofpclass(nan) float @ret_minnum_noinf__nonan
 ; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -36,9 +36,9 @@ define float @ret_minnum_noinf__nonan(float nofpclass(inf) %arg0, float nofpclas
 }
 
 define float @ret_minnum_nonan__noinf(float nofpclass(nan) %arg0, float nofpclass(inf) %arg1) #0 {
-; CHECK-LABEL: define float @ret_minnum_nonan__noinf
+; CHECK-LABEL: define nofpclass(nan) float @ret_minnum_nonan__noinf
 ; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -46,9 +46,9 @@ define float @ret_minnum_nonan__noinf(float nofpclass(nan) %arg0, float nofpclas
 }
 
 define float @ret_minnum_noinf_nonan__nonan(float nofpclass(inf nan) %arg0, float nofpclass(nan) %arg1) #0 {
-; CHECK-LABEL: define float @ret_minnum_noinf_nonan__nonan
+; CHECK-LABEL: define nofpclass(nan) float @ret_minnum_noinf_nonan__nonan
 ; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -56,9 +56,9 @@ define float @ret_minnum_noinf_nonan__nonan(float nofpclass(inf nan) %arg0, floa
 }
 
 define float @ret_minnum_nonan__noinf_nonan(float nofpclass(nan) %arg0, float nofpclass(inf nan) %arg1) #0 {
-; CHECK-LABEL: define float @ret_minnum_nonan__noinf_nonan
+; CHECK-LABEL: define nofpclass(nan) float @ret_minnum_nonan__noinf_nonan
 ; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -66,9 +66,9 @@ define float @ret_minnum_nonan__noinf_nonan(float nofpclass(nan) %arg0, float no
 }
 
 define float @ret_minnum_norm_zero__norm_sub(float nofpclass(norm zero) %arg0, float nofpclass(norm sub) %arg1) #0 {
-; CHECK-LABEL: define float @ret_minnum_norm_zero__norm_sub
+; CHECK-LABEL: define nofpclass(norm) float @ret_minnum_norm_zero__norm_sub
 ; CHECK-SAME: (float nofpclass(zero norm) [[ARG0:%.*]], float nofpclass(sub norm) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(norm) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -86,9 +86,9 @@ define float @ret_maxnum(float %arg0, float %arg1) #0 {
 }
 
 define float @ret_maxnum_noinf__noinf(float nofpclass(inf) %arg0, float nofpclass(inf) %arg1) #0 {
-; CHECK-LABEL: define float @ret_maxnum_noinf__noinf
+; CHECK-LABEL: define nofpclass(inf) float @ret_maxnum_noinf__noinf
 ; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -96,9 +96,9 @@ define float @ret_maxnum_noinf__noinf(float nofpclass(inf) %arg0, float nofpclas
 }
 
 define float @ret_maxnum_noinf__nonan(float nofpclass(inf) %arg0, float nofpclass(nan) %arg1) #0 {
-; CHECK-LABEL: define float @ret_maxnum_noinf__nonan
+; CHECK-LABEL: define nofpclass(nan) float @ret_maxnum_noinf__nonan
 ; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -106,9 +106,9 @@ define float @ret_maxnum_noinf__nonan(float nofpclass(inf) %arg0, float nofpclas
 }
 
 define float @ret_maxnum_nonan__noinf(float nofpclass(nan) %arg0, float nofpclass(inf) %arg1) #0 {
-; CHECK-LABEL: define float @ret_maxnum_nonan__noinf
+; CHECK-LABEL: define nofpclass(nan) float @ret_maxnum_nonan__noinf
 ; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(inf) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -116,9 +116,9 @@ define float @ret_maxnum_nonan__noinf(float nofpclass(nan) %arg0, float nofpclas
 }
 
 define float @ret_maxnum_noinf_nonan__nonan(float nofpclass(inf nan) %arg0, float nofpclass(nan) %arg1) #0 {
-; CHECK-LABEL: define float @ret_maxnum_noinf_nonan__nonan
+; CHECK-LABEL: define nofpclass(nan) float @ret_maxnum_noinf_nonan__nonan
 ; CHECK-SAME: (float nofpclass(nan inf) [[ARG0:%.*]], float nofpclass(nan) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -126,9 +126,9 @@ define float @ret_maxnum_noinf_nonan__nonan(float nofpclass(inf nan) %arg0, floa
 }
 
 define float @ret_maxnum_nonan__noinf_nonan(float nofpclass(nan) %arg0, float nofpclass(inf nan) %arg1) #0 {
-; CHECK-LABEL: define float @ret_maxnum_nonan__noinf_nonan
+; CHECK-LABEL: define nofpclass(nan) float @ret_maxnum_nonan__noinf_nonan
 ; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]], float nofpclass(nan inf) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -136,9 +136,9 @@ define float @ret_maxnum_nonan__noinf_nonan(float nofpclass(nan) %arg0, float no
 }
 
 define float @ret_maxnum_norm_zero__norm_sub(float nofpclass(norm zero) %arg0, float nofpclass(norm sub) %arg1) #0 {
-; CHECK-LABEL: define float @ret_maxnum_norm_zero__norm_sub
+; CHECK-LABEL: define nofpclass(norm) float @ret_maxnum_norm_zero__norm_sub
 ; CHECK-SAME: (float nofpclass(zero norm) [[ARG0:%.*]], float nofpclass(sub norm) [[ARG1:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(norm) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -175,9 +175,9 @@ define float @ret_minnum_dynamic_dynamic(float %arg0, float %arg1) #3 {
 }
 
 define float @ret_minnum_noinf_nozero__noinf_nozero(float nofpclass(inf zero) %arg0, float nofpclass(inf zero) %arg1) #1 {
-; CHECK-LABEL: define float @ret_minnum_noinf_nozero__noinf_nozero
+; CHECK-LABEL: define nofpclass(inf zero) float @ret_minnum_noinf_nozero__noinf_nozero
 ; CHECK-SAME: (float nofpclass(inf zero) [[ARG0:%.*]], float nofpclass(inf zero) [[ARG1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -185,9 +185,9 @@ define float @ret_minnum_noinf_nozero__noinf_nozero(float nofpclass(inf zero) %a
 }
 
 define <2 x float> @ret_minnum_noinf_nozero__noinf_nozero_v2f32(<2 x float> nofpclass(inf zero) %arg0, <2 x float> nofpclass(inf zero) %arg1) #1 {
-; CHECK-LABEL: define <2 x float> @ret_minnum_noinf_nozero__noinf_nozero_v2f32
+; CHECK-LABEL: define nofpclass(inf zero) <2 x float> @ret_minnum_noinf_nozero__noinf_nozero_v2f32
 ; CHECK-SAME: (<2 x float> nofpclass(inf zero) [[ARG0:%.*]], <2 x float> nofpclass(inf zero) [[ARG1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[ARG0]], <2 x float> [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(inf zero) <2 x float> @llvm.minnum.v2f32(<2 x float> [[ARG0]], <2 x float> [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret <2 x float> [[CALL]]
 ;
   %call = call <2 x float> @llvm.minnum.v2f32(<2 x float> %arg0, <2 x float> %arg1)
@@ -195,9 +195,9 @@ define <2 x float> @ret_minnum_noinf_nozero__noinf_nozero_v2f32(<2 x float> nofp
 }
 
 define float @ret_minnum_daz_daz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #1 {
-; CHECK-LABEL: define float @ret_minnum_daz_daz_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_daz_daz_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -205,9 +205,9 @@ define float @ret_minnum_daz_daz_nozero__nozero(float nofpclass(zero) %arg0, flo
 }
 
 define float @ret_minnum_dapz_dapz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #2 {
-; CHECK-LABEL: define float @ret_minnum_dapz_dapz_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_dapz_dapz_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR3]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -215,9 +215,9 @@ define float @ret_minnum_dapz_dapz_nozero__nozero(float nofpclass(zero) %arg0, f
 }
 
 define float @ret_minnum_dynamic_dynamic_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #3 {
-; CHECK-LABEL: define float @ret_minnum_dynamic_dynamic_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_dynamic_dynamic_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -225,9 +225,9 @@ define float @ret_minnum_dynamic_dynamic_nozero__nozero(float nofpclass(zero) %a
 }
 
 define float @ret_minnum_daz_daz_nozero_nosub__nozero_nosub(float nofpclass(zero sub) %arg0, float nofpclass(zero sub) %arg1) #1 {
-; CHECK-LABEL: define float @ret_minnum_daz_daz_nozero_nosub__nozero_nosub
+; CHECK-LABEL: define nofpclass(zero sub) float @ret_minnum_daz_daz_nozero_nosub__nozero_nosub
 ; CHECK-SAME: (float nofpclass(zero sub) [[ARG0:%.*]], float nofpclass(zero sub) [[ARG1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero sub) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -235,9 +235,9 @@ define float @ret_minnum_daz_daz_nozero_nosub__nozero_nosub(float nofpclass(zero
 }
 
 define float @ret_minnum_dynamic_dynamic_nozero_nosub__nozero_nosub(float nofpclass(zero sub) %arg0, float nofpclass(zero sub) %arg1) #3 {
-; CHECK-LABEL: define float @ret_minnum_dynamic_dynamic_nozero_nosub__nozero_nosub
+; CHECK-LABEL: define nofpclass(zero sub) float @ret_minnum_dynamic_dynamic_nozero_nosub__nozero_nosub
 ; CHECK-SAME: (float nofpclass(zero sub) [[ARG0:%.*]], float nofpclass(zero sub) [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero sub) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -265,9 +265,9 @@ define float @ret_minnum_daz_daz_nonzero__nonzero(float nofpclass(nzero) %arg0,
 }
 
 define float @ret_minnum_daz_daz_nonzero_nonsub__nonzero_nonsub(float nofpclass(nzero nsub) %arg0, float nofpclass(nzero nsub) %arg1) #1 {
-; CHECK-LABEL: define float @ret_minnum_daz_daz_nonzero_nonsub__nonzero_nonsub
+; CHECK-LABEL: define nofpclass(nsub) float @ret_minnum_daz_daz_nonzero_nonsub__nonzero_nonsub
 ; CHECK-SAME: (float nofpclass(nzero nsub) [[ARG0:%.*]], float nofpclass(nzero nsub) [[ARG1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nsub) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -285,9 +285,9 @@ define float @ret_minnum_dapz_dapz_nopzero__nopzero(float nofpclass(pzero) %arg0
 }
 
 define float @ret_minnum_dapz_dapz_nopzero_nopsub__nopzero_nopsub(float nofpclass(pzero psub) %arg0, float nofpclass(pzero psub) %arg1) #2 {
-; CHECK-LABEL: define float @ret_minnum_dapz_dapz_nopzero_nopsub__nopzero_nopsub
+; CHECK-LABEL: define nofpclass(psub) float @ret_minnum_dapz_dapz_nopzero_nopsub__nopzero_nopsub
 ; CHECK-SAME: (float nofpclass(pzero psub) [[ARG0:%.*]], float nofpclass(pzero psub) [[ARG1:%.*]]) #[[ATTR3]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(psub) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -305,9 +305,9 @@ define float @ret_minnum_dapz_dapz_nonzero__nonzero(float nofpclass(nzero) %arg0
 }
 
 define float @ret_minnum_ieee_daz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #4 {
-; CHECK-LABEL: define float @ret_minnum_ieee_daz_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_ieee_daz_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR5:[0-9]+]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -315,9 +315,9 @@ define float @ret_minnum_ieee_daz_nozero__nozero(float nofpclass(zero) %arg0, fl
 }
 
 define float @ret_minnum_daz_ieee_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #5 {
-; CHECK-LABEL: define float @ret_minnum_daz_ieee_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_daz_ieee_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR6:[0-9]+]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -325,9 +325,9 @@ define float @ret_minnum_daz_ieee_nozero__nozero(float nofpclass(zero) %arg0, fl
 }
 
 define float @ret_minnum_ieee_dapz_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #6 {
-; CHECK-LABEL: define float @ret_minnum_ieee_dapz_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_ieee_dapz_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR7:[0-9]+]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -335,9 +335,9 @@ define float @ret_minnum_ieee_dapz_nozero__nozero(float nofpclass(zero) %arg0, f
 }
 
 define float @ret_minnum_dapz_ieee_nozero__nozero(float nofpclass(zero) %arg0, float nofpclass(zero) %arg1) #7 {
-; CHECK-LABEL: define float @ret_minnum_dapz_ieee_nozero__nozero
+; CHECK-LABEL: define nofpclass(zero) float @ret_minnum_dapz_ieee_nozero__nozero
 ; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]], float nofpclass(zero) [[ARG1:%.*]]) #[[ATTR8:[0-9]+]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(zero) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -345,9 +345,9 @@ define float @ret_minnum_dapz_ieee_nozero__nozero(float nofpclass(zero) %arg0, f
 }
 
 define float @ret_minnum_noneg_nan__any(float nofpclass(ninf nsub nnorm nan) %arg0, float %arg1) #3 {
-; CHECK-LABEL: define float @ret_minnum_noneg_nan__any
+; CHECK-LABEL: define nofpclass(nan) float @ret_minnum_noneg_nan__any
 ; CHECK-SAME: (float nofpclass(nan ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -355,9 +355,9 @@ define float @ret_minnum_noneg_nan__any(float nofpclass(ninf nsub nnorm nan) %ar
 }
 
 define float @ret_minnum_any__noneg_nan(float %arg0, float nofpclass(ninf nsub nnorm nan) %arg1) #3 {
-; CHECK-LABEL: define float @ret_minnum_any__noneg_nan
+; CHECK-LABEL: define nofpclass(nan) float @ret_minnum_any__noneg_nan
 ; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -365,9 +365,9 @@ define float @ret_minnum_any__noneg_nan(float %arg0, float nofpclass(ninf nsub n
 }
 
 define float @ret_minnum_nopos_nan__any(float nofpclass(pinf psub pnorm nan) %arg0, float %arg1) #3 {
-; CHECK-LABEL: define float @ret_minnum_nopos_nan__any
+; CHECK-LABEL: define nofpclass(nan pinf psub pnorm) float @ret_minnum_nopos_nan__any
 ; CHECK-SAME: (float nofpclass(nan pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan pinf psub pnorm) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -375,9 +375,9 @@ define float @ret_minnum_nopos_nan__any(float nofpclass(pinf psub pnorm nan) %ar
 }
 
 define float @ret_minnum_any__nopos_nan(float %arg0, float nofpclass(pinf psub pnorm nan) %arg1) #3 {
-; CHECK-LABEL: define float @ret_minnum_any__nopos_nan
+; CHECK-LABEL: define nofpclass(nan pinf psub pnorm) float @ret_minnum_any__nopos_nan
 ; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan pinf psub pnorm) float @llvm.minnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.minnum.f32(float %arg0, float %arg1)
@@ -425,9 +425,9 @@ define float @ret_minnum_any__nopos(float %arg0, float nofpclass(pinf psub pnorm
 }
 
 define float @ret_maxnum_noneg_nan__any(float nofpclass(ninf nsub nnorm nan) %arg0, float %arg1) #3 {
-; CHECK-LABEL: define float @ret_maxnum_noneg_nan__any
+; CHECK-LABEL: define nofpclass(nan ninf nsub nnorm) float @ret_maxnum_noneg_nan__any
 ; CHECK-SAME: (float nofpclass(nan ninf nsub nnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nsub nnorm) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -435,9 +435,9 @@ define float @ret_maxnum_noneg_nan__any(float nofpclass(ninf nsub nnorm nan) %ar
 }
 
 define float @ret_maxnum_any__noneg_nan(float %arg0, float nofpclass(ninf nsub nnorm nan) %arg1) #3 {
-; CHECK-LABEL: define float @ret_maxnum_any__noneg_nan
+; CHECK-LABEL: define nofpclass(nan ninf nsub nnorm) float @ret_maxnum_any__noneg_nan
 ; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan ninf nsub nnorm) [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan ninf nsub nnorm) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -445,9 +445,9 @@ define float @ret_maxnum_any__noneg_nan(float %arg0, float nofpclass(ninf nsub n
 }
 
 define float @ret_maxnum_nopos_nan__any(float nofpclass(pinf psub pnorm nan) %arg0, float %arg1) #3 {
-; CHECK-LABEL: define float @ret_maxnum_nopos_nan__any
+; CHECK-LABEL: define nofpclass(nan) float @ret_maxnum_nopos_nan__any
 ; CHECK-SAME: (float nofpclass(nan pinf psub pnorm) [[ARG0:%.*]], float [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
@@ -455,9 +455,9 @@ define float @ret_maxnum_nopos_nan__any(float nofpclass(pinf psub pnorm nan) %ar
 }
 
 define float @ret_maxnum_any__nopos_nan(float %arg0, float nofpclass(pinf psub pnorm nan) %arg1) #3 {
-; CHECK-LABEL: define float @ret_maxnum_any__nopos_nan
+; CHECK-LABEL: define nofpclass(nan) float @ret_maxnum_any__nopos_nan
 ; CHECK-SAME: (float [[ARG0:%.*]], float nofpclass(nan pinf psub pnorm) [[ARG1:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(nan) float @llvm.maxnum.f32(float [[ARG0]], float [[ARG1]]) #[[ATTR9]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.maxnum.f32(float %arg0, float %arg1)
diff --git a/llvm/test/Transforms/Attributor/nofpclass-nan-fmul.ll b/llvm/test/Transforms/Attributor/nofpclass-nan-fmul.ll
index d4ea8156f5104..fb7d6e8d6d188 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-nan-fmul.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-nan-fmul.ll
@@ -164,6 +164,36 @@ define float @ret_fmul_daz_nonan_nozero_nosub__nonan_nozero(float nofpclass(nan
   ret float %fmul
 }
 
+define float @ret_fmul_square(float %arg) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_fmul_square
+; CHECK-SAME: (float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FMUL:%.*]] = fmul float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FMUL]]
+;
+  %fmul = fmul float %arg, %arg
+  ret float %fmul
+}
+
+define float @ret_fmul_square_nnan(float nofpclass(nan) %arg) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_fmul_square_nnan
+; CHECK-SAME: (float nofpclass(nan) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FMUL:%.*]] = fmul float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FMUL]]
+;
+  %fmul = fmul float %arg, %arg
+  ret float %fmul
+}
+
+define float @ret_fmul_square_nnan_nzero(float nofpclass(nan zero) %arg) #0 {
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_fmul_square_nnan_nzero
+; CHECK-SAME: (float nofpclass(nan zero) [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[FMUL:%.*]] = fmul float [[ARG]], [[ARG]]
+; CHECK-NEXT:    ret float [[FMUL]]
+;
+  %fmul = fmul float %arg, %arg
+  ret float %fmul
+}
+
 attributes #0 = { "denormal-fp-math"="ieee,ieee" }
 attributes #1 = { "denormal-fp-math"="ieee,preserve-sign" }
 attributes #2 = { "denormal-fp-math"="ieee,positive-zero" }
diff --git a/llvm/test/Transforms/Attributor/nofpclass-nearbyint.ll b/llvm/test/Transforms/Attributor/nofpclass-nearbyint.ll
new file mode 100644
index 0000000000000..30a6cc28eed5e
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-nearbyint.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.nearbyint.f32(float)
+declare ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128)
+
+define float @ret_nearbyint(float %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noqnan(float nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nosnan(float nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nozero(float nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopzero(float nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonzero(float nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonorm(float nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonorm
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonnorm(float nofpclass(nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonnorm
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopnorm(float nofpclass(pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopnorm
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonsub(float nofpclass(nsub) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonsub
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopsub(float nofpclass(psub) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopsub
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonorm_nosub(float nofpclass(norm sub) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonorm_nosub
+; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopnorm_nopsub
+; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nonnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define ppc_fp128 @ret_nearbyint_ppcf128(ppc_fp128 %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_nearbyint_ppcf128
+; CHECK-SAME: (ppc_fp128 [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_nearbyint_noinf_ppcf128(ppc_fp128 nofpclass(inf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_nearbyint_noinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_nearbyint_nopinf_ppcf128(ppc_fp128 nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_nearbyint_nopinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_nearbyint_noninf_ppcf128(ppc_fp128 nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_nearbyint_noninf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_nearbyint_nonan_ppcf128(ppc_fp128 nofpclass(nan) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_nearbyint_nonan_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.nearbyint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define float @ret_nearbyint_noneg(float nofpclass(ninf nsub nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noneg_nonegzero(float nofpclass(ninf nsub nnorm nzero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noneg_nonegzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noneg_nonegzero_nonan(float nofpclass(ninf nsub nnorm nzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noneg_nonegzero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noneg_nozero(float nofpclass(ninf nsub nnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_noneg_nozero_nonan(float nofpclass(ninf nsub nnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_noneg_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopos(float nofpclass(pinf psub pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopos
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopos_nopzero(float nofpclass(pinf psub pnorm pzero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopos_nopzero
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopos_nopzero_nonan(float nofpclass(pinf psub pnorm pzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopos_nopzero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopos_nozero(float nofpclass(pinf psub pnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopos_nozero
+; CHECK-SAME: (float nofpclass(pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_nearbyint_nopos_nozero_nonan(float nofpclass(pinf psub pnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_nearbyint_nopos_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.nearbyint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.nearbyint.f32(float %arg0)
+  ret float %call
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-powi.ll b/llvm/test/Transforms/Attributor/nofpclass-powi.ll
new file mode 100644
index 0000000000000..29e6d990bf3ef
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-powi.ll
@@ -0,0 +1,241 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.powi.f32.i32(float, i32)
+declare float @llvm.powi.f32.i64(float, i64)
+
+declare <2 x float> @llvm.powi.v2f32.v2i32(<2 x float>, <2 x i32>)
+
+define float @ret_powi_f32(float %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32
+; CHECK-SAME: (float [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_i64(float %arg0, i64 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_i64
+; CHECK-SAME: (float [[ARG0:%.*]], i64 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i64(float [[ARG0]], i64 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i64(float %arg0, i64 %arg1)
+  ret float %call
+}
+
+define <2 x float> @ret_powi_v2f32(<2 x float> %arg0, <2 x i32> %arg1) #0 {
+; CHECK-LABEL: define <2 x float> @ret_powi_v2f32
+; CHECK-SAME: (<2 x float> [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> [[ARG0]], <2 x i32> [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret <2 x float> [[CALL]]
+;
+  %call = call <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> %arg0, <2 x i32> %arg1)
+  ret <2 x float> %call
+}
+
+define float @ret_powi_f32_odd_constant(float %arg0) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_odd_constant
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 noundef 3) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 3)
+  ret float %call
+}
+
+define float @ret_powi_f32_even_constant(float %arg0) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_even_constant
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 noundef 4) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 4)
+  ret float %call
+}
+
+define <2 x float> @ret_powi_v2f32_even_nonsplat(<2 x float> %arg0) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) <2 x float> @ret_powi_v2f32_even_nonsplat
+; CHECK-SAME: (<2 x float> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> [[ARG0]], <2 x i32> noundef <i32 2, i32 4>) #[[ATTR5]]
+; CHECK-NEXT:    ret <2 x float> [[CALL]]
+;
+  %call = call <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> %arg0, <2 x i32> <i32 2, i32 4>)
+  ret <2 x float> %call
+}
+
+define <2 x float> @ret_powi_v2f32_odd_nonsplat(<2 x float> %arg0) #0 {
+; CHECK-LABEL: define <2 x float> @ret_powi_v2f32_odd_nonsplat
+; CHECK-SAME: (<2 x float> [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> [[ARG0]], <2 x i32> noundef <i32 3, i32 4>) #[[ATTR5]]
+; CHECK-NEXT:    ret <2 x float> [[CALL]]
+;
+  %call = call <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> %arg0, <2 x i32> <i32 3, i32 4>)
+  ret <2 x float> %call
+}
+
+define float @ret_powi_f32_masked_to_even(float %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_masked_to_even
+; CHECK-SAME: (float [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[KNOWN_EVEN:%.*]] = and i32 [[ARG1]], -2
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 [[KNOWN_EVEN]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %known.even = and i32 %arg1, -2
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %known.even)
+  ret float %call
+}
+
+define float @ret_powi_f32_masked_to_even_extrabits(float %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_masked_to_even_extrabits
+; CHECK-SAME: (float [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[KNOWN_EVEN:%.*]] = and i32 [[ARG1]], -4
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 [[KNOWN_EVEN]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %known.even = and i32 %arg1, -4
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %known.even)
+  ret float %call
+}
+
+define <2 x float> @ret_powi_v2f32_masked_to_even(<2 x float> %arg0, <2 x i32> %arg1) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) <2 x float> @ret_powi_v2f32_masked_to_even
+; CHECK-SAME: (<2 x float> [[ARG0:%.*]], <2 x i32> [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[KNOWN_EVEN:%.*]] = and <2 x i32> [[ARG1]], <i32 -2, i32 -2>
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> [[ARG0]], <2 x i32> [[KNOWN_EVEN]]) #[[ATTR5]]
+; CHECK-NEXT:    ret <2 x float> [[CALL]]
+;
+  %known.even = and <2 x i32> %arg1, <i32 -2, i32 -2>
+  %call = call <2 x float> @llvm.powi.v2f32.v2i32(<2 x float> %arg0, <2 x i32> %known.even)
+  ret <2 x float> %call
+}
+
+define float @ret_powi_f32_noneg(float nofpclass(ninf nsub nnorm) %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_nonzero(float nofpclass(ninf nsub nnorm nzero) %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_noneg_nonzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_nozero(float nofpclass(ninf nsub nnorm) %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_nonzero(float nofpclass(nzero) %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_nopzero(float nofpclass(pzero) %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_ftz_daz(float nofpclass(ninf nsub nnorm) %arg0, i32 %arg1) #1 {
+; CHECK-LABEL: define float @ret_powi_f32_noneg_ftz_daz
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_nonzero_ftz_daz(float nofpclass(ninf nsub nnorm nzero) %arg0, i32 %arg1) #1 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_noneg_nonzero_ftz_daz
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_nonzero_ftpz_dapz(float nofpclass(ninf nsub nnorm nzero) %arg0, i32 %arg1) #2 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_noneg_nonzero_ftpz_dapz
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noninf_nonnorm(float nofpclass(ninf nnorm) %arg0, i32 %arg1) #0 {
+; CHECK-LABEL: define float @ret_powi_f32_noninf_nonnorm
+; CHECK-SAME: (float nofpclass(ninf nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noninf_nonnorm_ftz_daz(float nofpclass(ninf nnorm) %arg0, i32 %arg1) #1 {
+; CHECK-LABEL: define float @ret_powi_f32_noninf_nonnorm_ftz_daz
+; CHECK-SAME: (float nofpclass(ninf nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_ftz_ieee(float nofpclass(ninf nsub nnorm) %arg0, i32 %arg1) #3 {
+; CHECK-LABEL: define float @ret_powi_f32_noneg_ftz_ieee
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR4:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+define float @ret_powi_f32_noneg_nonzero_ftz_ieee(float nofpclass(ninf nsub nnorm nzero) %arg0, i32 %arg1) #3 {
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_powi_f32_noneg_nonzero_ftz_ieee
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]], i32 [[ARG1:%.*]]) #[[ATTR4]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.powi.f32.i32(float [[ARG0]], i32 [[ARG1]]) #[[ATTR5]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
+  ret float %call
+}
+
+attributes #0 = { "denormal-fp-math"="ieee,ieee" }
+attributes #1 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #2 = { "denormal-fp-math"="positive-zero,positive-zero" }
+attributes #3 = { "denormal-fp-math"="preserve-sign,ieee" }
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-rint.ll b/llvm/test/Transforms/Attributor/nofpclass-rint.ll
new file mode 100644
index 0000000000000..94964193bb8be
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-rint.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.rint.f32(float)
+declare ppc_fp128 @llvm.rint.ppcf128(ppc_fp128)
+
+define float @ret_rint(float %arg0) {
+; CHECK-LABEL: define float @ret_rint
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noqnan(float nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nosnan(float nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nozero(float nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopzero(float nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonzero(float nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonorm(float nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonorm
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonnorm(float nofpclass(nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonnorm
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopnorm(float nofpclass(pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopnorm
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonsub(float nofpclass(nsub) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonsub
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopsub(float nofpclass(psub) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopsub
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonorm_nosub(float nofpclass(norm sub) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonorm_nosub
+; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopnorm_nopsub
+; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nonnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define ppc_fp128 @ret_rint_ppcf128(ppc_fp128 %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_rint_ppcf128
+; CHECK-SAME: (ppc_fp128 [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_rint_noinf_ppcf128(ppc_fp128 nofpclass(inf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_rint_noinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_rint_nopinf_ppcf128(ppc_fp128 nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_rint_nopinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_rint_noninf_ppcf128(ppc_fp128 nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_rint_noninf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_rint_nonan_ppcf128(ppc_fp128 nofpclass(nan) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_rint_nonan_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.rint.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define float @ret_rint_noneg(float nofpclass(ninf nsub nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noneg_nonegzero(float nofpclass(ninf nsub nnorm nzero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noneg_nonegzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noneg_nonegzero_nonan(float nofpclass(ninf nsub nnorm nzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noneg_nonegzero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noneg_nozero(float nofpclass(ninf nsub nnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_noneg_nozero_nonan(float nofpclass(ninf nsub nnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_noneg_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopos(float nofpclass(pinf psub pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopos
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopos_nopzero(float nofpclass(pinf psub pnorm pzero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopos_nopzero
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopos_nopzero_nonan(float nofpclass(pinf psub pnorm pzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopos_nopzero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopos_nozero(float nofpclass(pinf psub pnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopos_nozero
+; CHECK-SAME: (float nofpclass(pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_rint_nopos_nozero_nonan(float nofpclass(pinf psub pnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_rint_nopos_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.rint.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.rint.f32(float %arg0)
+  ret float %call
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-round.ll b/llvm/test/Transforms/Attributor/nofpclass-round.ll
new file mode 100644
index 0000000000000..dd524e0f62455
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-round.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.round.f32(float)
+declare ppc_fp128 @llvm.round.ppcf128(ppc_fp128)
+
+define float @ret_round(float %arg0) {
+; CHECK-LABEL: define float @ret_round
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_round_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_round_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noqnan(float nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_round_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nosnan(float nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_round_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nozero(float nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_round_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopzero(float nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonzero(float nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonorm(float nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonorm
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonnorm(float nofpclass(nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonnorm
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopnorm(float nofpclass(pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopnorm
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonsub(float nofpclass(nsub) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonsub
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopsub(float nofpclass(psub) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopsub
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonorm_nosub(float nofpclass(norm sub) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonorm_nosub
+; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopnorm_nopsub
+; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_round_nonnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define ppc_fp128 @ret_round_ppcf128(ppc_fp128 %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_round_ppcf128
+; CHECK-SAME: (ppc_fp128 [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_round_noinf_ppcf128(ppc_fp128 nofpclass(inf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_round_noinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_round_nopinf_ppcf128(ppc_fp128 nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_round_nopinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_round_noninf_ppcf128(ppc_fp128 nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_round_noninf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_round_nonan_ppcf128(ppc_fp128 nofpclass(nan) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_round_nonan_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.round.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define float @ret_round_noneg(float nofpclass(ninf nsub nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_round_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noneg_nonegzero(float nofpclass(ninf nsub nnorm nzero) %arg0) {
+; CHECK-LABEL: define float @ret_round_noneg_nonegzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noneg_nonegzero_nonan(float nofpclass(ninf nsub nnorm nzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_round_noneg_nonegzero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noneg_nozero(float nofpclass(ninf nsub nnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_round_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_noneg_nozero_nonan(float nofpclass(ninf nsub nnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_round_noneg_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopos(float nofpclass(pinf psub pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopos
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopos_nopzero(float nofpclass(pinf psub pnorm pzero) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopos_nopzero
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopos_nopzero_nonan(float nofpclass(pinf psub pnorm pzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopos_nopzero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopos_nozero(float nofpclass(pinf psub pnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopos_nozero
+; CHECK-SAME: (float nofpclass(pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_round_nopos_nozero_nonan(float nofpclass(pinf psub pnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_round_nopos_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.round.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.round.f32(float %arg0)
+  ret float %call
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-roundeven.ll b/llvm/test/Transforms/Attributor/nofpclass-roundeven.ll
new file mode 100644
index 0000000000000..7f07841723784
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/nofpclass-roundeven.ll
@@ -0,0 +1,348 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+
+declare float @llvm.roundeven.f32(float)
+declare ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128)
+
+define float @ret_roundeven(float %arg0) {
+; CHECK-LABEL: define float @ret_roundeven
+; CHECK-SAME: (float [[ARG0:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noinf(float nofpclass(inf) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noinf
+; CHECK-SAME: (float nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopinf(float nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopinf
+; CHECK-SAME: (float nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noninf(float nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noninf
+; CHECK-SAME: (float nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonan(float nofpclass(nan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonan
+; CHECK-SAME: (float nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noqnan(float nofpclass(qnan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noqnan
+; CHECK-SAME: (float nofpclass(qnan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nosnan(float nofpclass(snan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nosnan
+; CHECK-SAME: (float nofpclass(snan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nozero(float nofpclass(zero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nozero
+; CHECK-SAME: (float nofpclass(zero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopzero(float nofpclass(pzero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopzero
+; CHECK-SAME: (float nofpclass(pzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonzero(float nofpclass(nzero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonzero
+; CHECK-SAME: (float nofpclass(nzero) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonorm(float nofpclass(norm) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonorm
+; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonnorm(float nofpclass(nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonnorm
+; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopnorm(float nofpclass(pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopnorm
+; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonsub(float nofpclass(nsub) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonsub
+; CHECK-SAME: (float nofpclass(nsub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopsub(float nofpclass(psub) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopsub
+; CHECK-SAME: (float nofpclass(psub) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonorm_nosub(float nofpclass(norm sub) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonorm_nosub
+; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopnorm_nopsub
+; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nonnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopnorm_nonsub
+; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define ppc_fp128 @ret_roundeven_ppcf128(ppc_fp128 %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_roundeven_ppcf128
+; CHECK-SAME: (ppc_fp128 [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_roundeven_noinf_ppcf128(ppc_fp128 nofpclass(inf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_roundeven_noinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(inf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_roundeven_nopinf_ppcf128(ppc_fp128 nofpclass(pinf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_roundeven_nopinf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(pinf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_roundeven_noninf_ppcf128(ppc_fp128 nofpclass(ninf) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_roundeven_noninf_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(ninf) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define ppc_fp128 @ret_roundeven_nonan_ppcf128(ppc_fp128 nofpclass(nan) %arg0) {
+; CHECK-LABEL: define ppc_fp128 @ret_roundeven_nonan_ppcf128
+; CHECK-SAME: (ppc_fp128 nofpclass(nan) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret ppc_fp128 [[CALL]]
+;
+  %call = call ppc_fp128 @llvm.roundeven.ppcf128(ppc_fp128 %arg0)
+  ret ppc_fp128 %call
+}
+
+define float @ret_roundeven_noneg(float nofpclass(ninf nsub nnorm) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noneg
+; CHECK-SAME: (float nofpclass(ninf nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noneg_nonegzero(float nofpclass(ninf nsub nnorm nzero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noneg_nonegzero
+; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noneg_nonegzero_nonan(float nofpclass(ninf nsub nnorm nzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noneg_nonegzero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf nzero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noneg_nozero(float nofpclass(ninf nsub nnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noneg_nozero
+; CHECK-SAME: (float nofpclass(ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_noneg_nozero_nonan(float nofpclass(ninf nsub nnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_noneg_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan ninf zero nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopos(float nofpclass(pinf psub pnorm) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopos
+; CHECK-SAME: (float nofpclass(pinf psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopos_nopzero(float nofpclass(pinf psub pnorm pzero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopos_nopzero
+; CHECK-SAME: (float nofpclass(pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopos_nopzero_nonan(float nofpclass(pinf psub pnorm pzero nan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopos_nopzero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf pzero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopos_nozero(float nofpclass(pinf psub pnorm zero) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopos_nozero
+; CHECK-SAME: (float nofpclass(pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+define float @ret_roundeven_nopos_nozero_nonan(float nofpclass(pinf psub pnorm zero nan) %arg0) {
+; CHECK-LABEL: define float @ret_roundeven_nopos_nozero_nonan
+; CHECK-SAME: (float nofpclass(nan pinf zero psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.roundeven.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %call = call float @llvm.roundeven.f32(float %arg0)
+  ret float %call
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; TUNIT: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/nofpclass-trunc.ll b/llvm/test/Transforms/Attributor/nofpclass-trunc.ll
index eda3bdf379af3..63c9d2f066c7b 100644
--- a/llvm/test/Transforms/Attributor/nofpclass-trunc.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass-trunc.ll
@@ -105,9 +105,9 @@ define float @ret_trunc_nonzero(float nofpclass(nzero) %arg0) {
 }
 
 define float @ret_trunc_nonorm(float nofpclass(norm) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nonorm
+; CHECK-LABEL: define nofpclass(sub norm) float @ret_trunc_nonorm
 ; CHECK-SAME: (float nofpclass(norm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub norm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
@@ -115,9 +115,9 @@ define float @ret_trunc_nonorm(float nofpclass(norm) %arg0) {
 }
 
 define float @ret_trunc_nonnorm(float nofpclass(nnorm) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nonnorm
+; CHECK-LABEL: define nofpclass(sub nnorm) float @ret_trunc_nonnorm
 ; CHECK-SAME: (float nofpclass(nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub nnorm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
@@ -125,9 +125,9 @@ define float @ret_trunc_nonnorm(float nofpclass(nnorm) %arg0) {
 }
 
 define float @ret_trunc_nopnorm(float nofpclass(pnorm) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nopnorm
+; CHECK-LABEL: define nofpclass(sub pnorm) float @ret_trunc_nopnorm
 ; CHECK-SAME: (float nofpclass(pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub pnorm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
@@ -155,9 +155,9 @@ define float @ret_trunc_nopsub(float nofpclass(psub) %arg0) {
 }
 
 define float @ret_trunc_nonorm_nosub(float nofpclass(norm sub) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nonorm_nosub
+; CHECK-LABEL: define nofpclass(sub norm) float @ret_trunc_nonorm_nosub
 ; CHECK-SAME: (float nofpclass(sub norm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub norm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
@@ -165,9 +165,9 @@ define float @ret_trunc_nonorm_nosub(float nofpclass(norm sub) %arg0) {
 }
 
 define float @ret_trunc_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nopnorm_nopsub
+; CHECK-LABEL: define nofpclass(sub pnorm) float @ret_trunc_nopnorm_nopsub
 ; CHECK-SAME: (float nofpclass(psub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub pnorm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
@@ -175,9 +175,9 @@ define float @ret_trunc_nopnorm_nopsub(float nofpclass(pnorm psub) %arg0) {
 }
 
 define float @ret_trunc_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nonnorm_nonsub
+; CHECK-LABEL: define nofpclass(sub nnorm) float @ret_trunc_nonnorm_nonsub
 ; CHECK-SAME: (float nofpclass(nsub nnorm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub nnorm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
@@ -185,9 +185,9 @@ define float @ret_trunc_nonnorm_nonsub(float nofpclass(nnorm nsub) %arg0) {
 }
 
 define float @ret_trunc_nopnorm_nonsub(float nofpclass(pnorm nsub) %arg0) {
-; CHECK-LABEL: define nofpclass(sub) float @ret_trunc_nopnorm_nonsub
+; CHECK-LABEL: define nofpclass(sub pnorm) float @ret_trunc_nopnorm_nonsub
 ; CHECK-SAME: (float nofpclass(nsub pnorm) [[ARG0:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
+; CHECK-NEXT:    [[CALL:%.*]] = call nofpclass(sub pnorm) float @llvm.trunc.f32(float [[ARG0]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
   %call = call float @llvm.trunc.f32(float %arg0)
diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll
index d6381c01798e5..6680dd3675904 100644
--- a/llvm/test/Transforms/Attributor/nofpclass.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass.ll
@@ -489,7 +489,7 @@ define half @fcmp_assume_issubnormal_callsite_arg_return(half %arg) {
 ; CHECK-LABEL: define nofpclass(nan inf norm) half @fcmp_assume_issubnormal_callsite_arg_return
 ; CHECK-SAME: (half returned nofpclass(nan inf norm) [[ARG:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FABS:%.*]] = call half @llvm.fabs.f16(half nofpclass(nan inf norm) [[ARG]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(ninf nzero nsub nnorm) half @llvm.fabs.f16(half nofpclass(nan inf norm) [[ARG]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[IS_SUBNORMAL:%.*]] = fcmp olt half [[FABS]], 0xH0400
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef [[IS_SUBNORMAL]]) #[[ATTR7]]
 ; CHECK-NEXT:    call void @extern.use.f16(half nofpclass(nan inf norm) [[ARG]])
@@ -525,7 +525,7 @@ define half @fcmp_assume2_callsite_arg_return(half %arg) {
 ; CHECK-LABEL: define nofpclass(nan ninf zero sub norm) half @fcmp_assume2_callsite_arg_return
 ; CHECK-SAME: (half returned nofpclass(nan ninf zero sub norm) [[ARG:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan zero sub norm) half @llvm.fabs.f16(half nofpclass(nan ninf zero sub norm) [[ARG]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan ninf zero sub norm) half @llvm.fabs.f16(half nofpclass(nan ninf zero sub norm) [[ARG]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[NOT_SUBNORMAL_OR_ZERO:%.*]] = fcmp oge half [[FABS]], 0xH0400
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef [[NOT_SUBNORMAL_OR_ZERO]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[NOT_INF:%.*]] = fcmp oeq half [[ARG]], 0xH7C00
@@ -567,13 +567,13 @@ define half @assume_fcmp_fabs_with_other_fabs_assume(half %arg) {
 ; CHECK-LABEL: define nofpclass(nan inf norm) half @assume_fcmp_fabs_with_other_fabs_assume
 ; CHECK-SAME: (half returned nofpclass(nan inf norm) [[ARG:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan inf zero norm) half @llvm.fabs.f16(half nofpclass(nan inf norm) [[ARG]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan inf zero nsub norm) half @llvm.fabs.f16(half nofpclass(nan inf norm) [[ARG]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[UNRELATED_FABS:%.*]] = fcmp one half [[FABS]], 0xH0000
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef [[UNRELATED_FABS]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[IS_SUBNORMAL:%.*]] = fcmp olt half [[FABS]], 0xH0400
 ; CHECK-NEXT:    call void @llvm.assume(i1 noundef [[IS_SUBNORMAL]]) #[[ATTR7]]
 ; CHECK-NEXT:    call void @extern.use.f16(half nofpclass(nan inf norm) [[ARG]])
-; CHECK-NEXT:    call void @extern.use.f16(half nofpclass(nan inf zero norm) [[FABS]])
+; CHECK-NEXT:    call void @extern.use.f16(half nofpclass(nan inf zero nsub norm) [[FABS]])
 ; CHECK-NEXT:    ret half [[ARG]]
 ;
 entry:
@@ -686,9 +686,9 @@ define float @pass_nofpclass_inf_through_memory(float nofpclass(inf) %arg) {
 
 define float @returned_fabs(float %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define float @returned_fabs
+; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @returned_fabs
 ; CHECK-SAME: (float [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %fabs = call float @llvm.fabs.f32(float %x)
@@ -697,9 +697,9 @@ define float @returned_fabs(float %x) {
 
 define float @returned_fabs_nosnan(float nofpclass(snan) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(snan) float @returned_fabs_nosnan
+; CHECK-LABEL: define nofpclass(snan ninf nzero nsub nnorm) float @returned_fabs_nosnan
 ; CHECK-SAME: (float nofpclass(snan) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(snan) float @llvm.fabs.f32(float nofpclass(snan) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(snan ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(snan) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %fabs = call float @llvm.fabs.f32(float %x)
@@ -708,9 +708,9 @@ define float @returned_fabs_nosnan(float nofpclass(snan) %x) {
 
 define float @returned_fabs_noqnan(float nofpclass(qnan) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(qnan) float @returned_fabs_noqnan
+; CHECK-LABEL: define nofpclass(qnan ninf nzero nsub nnorm) float @returned_fabs_noqnan
 ; CHECK-SAME: (float nofpclass(qnan) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(qnan) float @llvm.fabs.f32(float nofpclass(qnan) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(qnan ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(qnan) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %fabs = call float @llvm.fabs.f32(float %x)
@@ -719,9 +719,9 @@ define float @returned_fabs_noqnan(float nofpclass(qnan) %x) {
 
 define float @returned_fabs_nonan(float nofpclass(nan) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(nan) float @returned_fabs_nonan
+; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @returned_fabs_nonan
 ; CHECK-SAME: (float nofpclass(nan) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan) float @llvm.fabs.f32(float nofpclass(nan) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(nan) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %fabs = call float @llvm.fabs.f32(float %x)
@@ -730,9 +730,9 @@ define float @returned_fabs_nonan(float nofpclass(nan) %x) {
 
 define float @returned_fabs_noinf(float nofpclass(inf) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(inf) float @returned_fabs_noinf
+; CHECK-LABEL: define nofpclass(inf nzero nsub nnorm) float @returned_fabs_noinf
 ; CHECK-SAME: (float nofpclass(inf) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(inf) float @llvm.fabs.f32(float nofpclass(inf) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(inf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(inf) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %fabs = call float @llvm.fabs.f32(float %x)
@@ -873,9 +873,9 @@ define float @returned_fneg_mixed(float nofpclass(psub nnorm nzero qnan ninf) %x
 
 define float @returned_fneg_fabs(float %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define float @returned_fneg_fabs
+; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @returned_fneg_fabs
 ; CHECK-SAME: (float [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -886,9 +886,9 @@ define float @returned_fneg_fabs(float %x) {
 
 define float @returned_fneg_fabs_nosnan(float nofpclass(snan) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(snan) float @returned_fneg_fabs_nosnan
+; CHECK-LABEL: define nofpclass(snan pinf pzero psub pnorm) float @returned_fneg_fabs_nosnan
 ; CHECK-SAME: (float nofpclass(snan) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(snan) float @llvm.fabs.f32(float nofpclass(snan) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(snan ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(snan) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -899,9 +899,9 @@ define float @returned_fneg_fabs_nosnan(float nofpclass(snan) %x) {
 
 define float @returned_fneg_fabs_noqnan(float nofpclass(qnan) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(qnan) float @returned_fneg_fabs_noqnan
+; CHECK-LABEL: define nofpclass(qnan pinf pzero psub pnorm) float @returned_fneg_fabs_noqnan
 ; CHECK-SAME: (float nofpclass(qnan) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(qnan) float @llvm.fabs.f32(float nofpclass(qnan) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(qnan ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(qnan) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -912,9 +912,9 @@ define float @returned_fneg_fabs_noqnan(float nofpclass(qnan) %x) {
 
 define float @returned_fneg_fabs_nonan(float nofpclass(nan) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(nan) float @returned_fneg_fabs_nonan
+; CHECK-LABEL: define nofpclass(nan pinf pzero psub pnorm) float @returned_fneg_fabs_nonan
 ; CHECK-SAME: (float nofpclass(nan) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan) float @llvm.fabs.f32(float nofpclass(nan) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(nan ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(nan) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -925,9 +925,9 @@ define float @returned_fneg_fabs_nonan(float nofpclass(nan) %x) {
 
 define float @returned_fneg_fabs_noneg(float nofpclass(ninf nsub nnorm nzero) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define float @returned_fneg_fabs_noneg
+; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @returned_fneg_fabs_noneg
 ; CHECK-SAME: (float nofpclass(ninf nzero nsub nnorm) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float nofpclass(ninf nzero nsub nnorm) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.fabs.f32(float nofpclass(ninf nzero nsub nnorm) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -951,9 +951,9 @@ define float @returned_fneg_fabs_nopos(float nofpclass(pinf psub pnorm pzero) %x
 
 define float @returned_fneg_fabs_mixed(float nofpclass(psub nnorm nzero qnan ninf) %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(qnan sub) float @returned_fneg_fabs_mixed
+; CHECK-LABEL: define nofpclass(qnan pinf pzero sub pnorm) float @returned_fneg_fabs_mixed
 ; CHECK-SAME: (float nofpclass(qnan ninf nzero psub nnorm) [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(qnan sub) float @llvm.fabs.f32(float nofpclass(qnan ninf nzero psub nnorm) [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(qnan ninf nzero sub nnorm) float @llvm.fabs.f32(float nofpclass(qnan ninf nzero psub nnorm) [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -964,9 +964,9 @@ define float @returned_fneg_fabs_mixed(float nofpclass(psub nnorm nzero qnan nin
 
 define float @returned_fneg_fabs_ninf_flag_fabs(float %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(inf) float @returned_fneg_fabs_ninf_flag_fabs
+; CHECK-LABEL: define nofpclass(inf pzero psub pnorm) float @returned_fneg_fabs_ninf_flag_fabs
 ; CHECK-SAME: (float [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call ninf nofpclass(inf) float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call ninf nofpclass(inf nzero nsub nnorm) float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -977,9 +977,9 @@ define float @returned_fneg_fabs_ninf_flag_fabs(float %x) {
 
 define float @returned_fneg_fabs_ninf_flag_fneg(float %x) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define nofpclass(inf) float @returned_fneg_fabs_ninf_flag_fneg
+; CHECK-LABEL: define nofpclass(inf pzero psub pnorm) float @returned_fneg_fabs_ninf_flag_fneg
 ; CHECK-SAME: (float [[X:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
+; CHECK-NEXT:    [[FABS:%.*]] = call nofpclass(ninf nzero nsub nnorm) float @llvm.fabs.f32(float [[X]]) #[[ATTR7]]
 ; CHECK-NEXT:    [[FNEG_FABS:%.*]] = fneg ninf float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG_FABS]]
 ;
@@ -1421,7 +1421,7 @@ define <4 x float> @shufflevector_undef_demanded(<2 x float> %arg0, <2 x float>
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define <4 x float> @shufflevector_undef_demanded
 ; CHECK-SAME: (<2 x float> [[ARG0:%.*]], <2 x float> nofpclass(inf) [[ARG1:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> [[ARG1]], <4 x i32> undef
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> [[ARG1]], <4 x i32> poison
 ; CHECK-NEXT:    ret <4 x float> [[SHUFFLE]]
 ;
   %shuffle = shufflevector <2 x float> %arg0, <2 x float> %arg1, <4 x i32> undef
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
index 8353506b7e3d0..c7084d737f6f2 100644
--- a/llvm/test/Transforms/Attributor/nonnull.ll
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -957,7 +957,7 @@ define ptr addrspace(3) @gep2(ptr addrspace(3) %p) {
 define ptr addrspace(3) @as(ptr addrspace(3) dereferenceable(4) %p) {
 ; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
 ; CHECK-LABEL: define {{[^@]+}}@as
-; CHECK-SAME: (ptr addrspace(3) nofree nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: (ptr addrspace(3) nofree noundef nonnull readnone returned dereferenceable(4) "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:    ret ptr addrspace(3) [[P]]
 ;
   ret ptr addrspace(3) %p
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll
index 4dece2f836b62..72d1672eb4f7d 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll
@@ -121,17 +121,17 @@ if_false:
 
 define <4 x i32> @test_32bit(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
 ; CHECK-SSE2-LABEL: @test_32bit(
-; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-SSE2-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK-SSE2:       if_true:
 ; CHECK-SSE2-NEXT:    ret <4 x i32> [[MASK]]
 ; CHECK-SSE2:       if_false:
-; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-SSE2-NEXT:    [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[TMP1]]
 ; CHECK-SSE2-NEXT:    ret <4 x i32> [[RES]]
 ;
 ; CHECK-XOP-LABEL: @test_32bit(
-; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-XOP-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK-XOP:       if_true:
 ; CHECK-XOP-NEXT:    ret <4 x i32> [[MASK]]
@@ -140,7 +140,7 @@ define <4 x i32> @test_32bit(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
 ; CHECK-XOP-NEXT:    ret <4 x i32> [[RES]]
 ;
 ; CHECK-AVX-LABEL: @test_32bit(
-; CHECK-AVX-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-AVX-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-AVX-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK-AVX:       if_true:
 ; CHECK-AVX-NEXT:    ret <4 x i32> [[MASK]]
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll
index 9d9acaf800277..c14918a6956f1 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll
@@ -121,17 +121,17 @@ if_false:
 
 define <4 x i32> @test_32bit(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
 ; CHECK-SSE2-LABEL: @test_32bit(
-; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-SSE2-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK-SSE2:       if_true:
 ; CHECK-SSE2-NEXT:    ret <4 x i32> [[MASK]]
 ; CHECK-SSE2:       if_false:
-; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-SSE2-NEXT:    [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[TMP1]]
 ; CHECK-SSE2-NEXT:    ret <4 x i32> [[RES]]
 ;
 ; CHECK-XOP-LABEL: @test_32bit(
-; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-XOP-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK-XOP:       if_true:
 ; CHECK-XOP-NEXT:    ret <4 x i32> [[MASK]]
@@ -140,7 +140,7 @@ define <4 x i32> @test_32bit(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
 ; CHECK-XOP-NEXT:    ret <4 x i32> [[RES]]
 ;
 ; CHECK-AVX-LABEL: @test_32bit(
-; CHECK-AVX-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-AVX-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-AVX-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK-AVX:       if_true:
 ; CHECK-AVX-NEXT:    ret <4 x i32> [[MASK]]
diff --git a/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll b/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll
new file mode 100644
index 0000000000000..241798dac925a
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/mul-nsw.ll
@@ -0,0 +1,1031 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=constraint-elimination -S %s | FileCheck %s
+
+declare void @use(i1)
+declare void @llvm.assume(i1)
+
+define void @slt_mul_nsw_3_known_positive_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_known_positive_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[T_0:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[T_1:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[T_2:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[T_3:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %t.0 = icmp slt i8 %start, %high
+  call void @use(i1 %t.0)
+  %start.1 = mul nsw i8 %start, 1
+  %t.1 = icmp slt i8 %start.1, %high
+  call void @use(i1 %t.1)
+  %start.2 = mul nsw i8 %start, 2
+  %t.2 = icmp slt i8 %start.2, %high
+  call void @use(i1 %t.2)
+  %start.3 = mul nsw i8 %start, 3
+  %t.3 = icmp slt i8 %start.3, %high
+  call void @use(i1 %t.3)
+  %start.4 = mul nsw i8 %start, 4
+  %c.3 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.3)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_nsw_3_known_positive_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_known_positive_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.4 = icmp slt i8 %start, %high
+  call void @use(i1 %c.4)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.5 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.5)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.6 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.6)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.7 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.7)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.8 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.8)
+  ret void
+}
+
+define void @slt_mul_no_nsw_3_known_positive_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_no_nsw_3_known_positive_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %c.3 = icmp slt i8 %start, %high
+  call void @use(i1 %c.3)
+  %start.1 = mul nsw i8 %start, 1
+  %c.4 = icmp slt i8 %start.1, %high
+  call void @use(i1 %c.4)
+  %start.2 = mul nsw i8 %start, 2
+  %c.5 = icmp slt i8 %start.2, %high
+  call void @use(i1 %c.5)
+  %start.3 = mul nsw i8 %start, 3
+  %c.6 = icmp slt i8 %start.3, %high
+  call void @use(i1 %c.6)
+  %start.4 = mul nsw i8 %start, 4
+  %c.7 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.7)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_no_nsw_3_known_positive_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_no_nsw_3_known_positive_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_9:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_9]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_10:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_10]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_11:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_11]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_12:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_12]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.8 = icmp slt i8 %start, %high
+  call void @use(i1 %c.8)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.9 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.9)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.10 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.10)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.11 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.11)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.12 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.12)
+  ret void
+}
+
+define void @slt_mul_nsw_3_not_known_positive_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_not_known_positive_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  br i1 %c.1, label %then, label %else
+
+then:
+  %c.3 = icmp slt i8 %start, %high
+  call void @use(i1 %c.3)
+  %start.1 = mul nsw i8 %start, 1
+  %c.4 = icmp slt i8 %start.1, %high
+  call void @use(i1 %c.4)
+  %start.2 = mul nsw i8 %start, 2
+  %c.5 = icmp slt i8 %start.2, %high
+  call void @use(i1 %c.5)
+  %start.3 = mul nsw i8 %start, 3
+  %c.6 = icmp slt i8 %start.3, %high
+  call void @use(i1 %c.6)
+  %start.4 = mul nsw i8 %start, 4
+  %c.7 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.7)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_nsw_3_not_known_positive_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_not_known_positive_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_9:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_9]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_10:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_10]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_11:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_11]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_12:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_12]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  br i1 %c.1, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.8 = icmp slt i8 %start, %high
+  call void @use(i1 %c.8)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.9 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.9)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.10 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.10)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.11 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.11)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.12 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.12)
+  ret void
+}
+
+define void @slt_mul_nsw_neg_3_known_negative_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_neg_3_known_negative_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], -3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[T_0:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], -1
+; CHECK-NEXT:    [[T_1:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], -2
+; CHECK-NEXT:    [[T_2:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], -3
+; CHECK-NEXT:    [[T_3:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], -4
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, -3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp slt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %t.0 = icmp slt i8 %start, %high
+  call void @use(i1 %t.0)
+  %start.1 = mul nsw i8 %start, -1
+  %t.1 = icmp slt i8 %start.1, %high
+  call void @use(i1 %t.1)
+  %start.2 = mul nsw i8 %start, -2
+  %t.2 = icmp slt i8 %start.2, %high
+  call void @use(i1 %t.2)
+  %start.3 = mul nsw i8 %start, -3
+  %t.3 = icmp slt i8 %start.3, %high
+  call void @use(i1 %t.3)
+  %start.4 = mul nsw i8 %start, -4
+  %c.3 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.3)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_nsw_neg_3_known_negative_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_neg_3_known_negative_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], -3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], -1
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], -2
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], -3
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], -4
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, -3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp slt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.4 = icmp slt i8 %start, %high
+  call void @use(i1 %c.4)
+  %else.start.1 = mul nsw i8 %start, -1
+  %c.5 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.5)
+  %else.start.2 = mul nsw i8 %start, -2
+  %c.6 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.6)
+  %else.start.3 = mul nsw i8 %start, -3
+  %c.7 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.7)
+  %else.start.4 = mul nsw i8 %start, -4
+  %c.8 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.8)
+  ret void
+}
+
+define void @slt_mul_nsw_3_known_negative_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_known_negative_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[T_0:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[T_1:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp slt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %c.3 = icmp slt i8 %start, %high
+  call void @use(i1 %c.3)
+  %start.1 = mul nsw i8 %start, 1
+  %c.4 = icmp slt i8 %start.1, %high
+  call void @use(i1 %c.4)
+  %start.2 = mul nsw i8 %start, 2
+  %c.5 = icmp slt i8 %start.2, %high
+  call void @use(i1 %c.5)
+  %start.3 = mul nsw i8 %start, 3
+  %t.0 = icmp slt i8 %start.3, %high
+  call void @use(i1 %t.0)
+  %start.4 = mul nsw i8 %start, 4
+  %t.1 = icmp slt i8 %start.4, %high
+  call void @use(i1 %t.1)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_nsw_3_known_negative_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_known_negative_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_9:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_9]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_10:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_10]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp slt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.6 = icmp slt i8 %start, %high
+  call void @use(i1 %c.6)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.7 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.7)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.8 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.8)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.9 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.9)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.10 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.10)
+  ret void
+}
+
+define void @slt_mul_nsw_neg_3_known_positive_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_neg_3_known_positive_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], -3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], -1
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], -2
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], -3
+; CHECK-NEXT:    [[T_0:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], -4
+; CHECK-NEXT:    [[T_1:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, -3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %c.3 = icmp slt i8 %start, %high
+  call void @use(i1 %c.3)
+  %start.1 = mul nsw i8 %start, -1
+  %c.4 = icmp slt i8 %start.1, %high
+  call void @use(i1 %c.4)
+  %start.2 = mul nsw i8 %start, -2
+  %c.5 = icmp slt i8 %start.2, %high
+  call void @use(i1 %c.5)
+  %start.3 = mul nsw i8 %start, -3
+  %t.0 = icmp slt i8 %start.3, %high
+  call void @use(i1 %t.0)
+  %start.4 = mul nsw i8 %start, -4
+  %t.1 = icmp slt i8 %start.4, %high
+  call void @use(i1 %t.1)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_nsw_neg_3_known_positive_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_neg_3_known_positive_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], -3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], -1
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], -2
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], -3
+; CHECK-NEXT:    [[C_9:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_9]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], -4
+; CHECK-NEXT:    [[C_10:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_10]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, -3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.6 = icmp slt i8 %start, %high
+  call void @use(i1 %c.6)
+  %else.start.1 = mul nsw i8 %start, -1
+  %c.7 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.7)
+  %else.start.2 = mul nsw i8 %start, -2
+  %c.8 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.8)
+  %else.start.3 = mul nsw i8 %start, -3
+  %c.9 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.9)
+  %else.start.4 = mul nsw i8 %start, -4
+  %c.10 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.10)
+  ret void
+}
+
+define void @slt_mul_nsw_3_known_nonnegative_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_known_nonnegative_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sge i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[T_0:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[T_1:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[T_2:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[T_3:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sge i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %t.0 = icmp slt i8 %start, %high
+  call void @use(i1 %t.0)
+  %start.1 = mul nsw i8 %start, 1
+  %t.1 = icmp slt i8 %start.1, %high
+  call void @use(i1 %t.1)
+  %start.2 = mul nsw i8 %start, 2
+  %t.2 = icmp slt i8 %start.2, %high
+  call void @use(i1 %t.2)
+  %start.3 = mul nsw i8 %start, 3
+  %t.3 = icmp slt i8 %start.3, %high
+  call void @use(i1 %t.3)
+  %start.4 = mul nsw i8 %start, 4
+  %c.3 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.3)
+  ret void
+
+else:
+  ret void
+}
+
+define void @slt_mul_nsw_3_known_nonnegative_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_3_known_nonnegative_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sge i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sge i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.4 = icmp slt i8 %start, %high
+  call void @use(i1 %c.4)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.5 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.5)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.6 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.6)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.7 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.7)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.8 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.8)
+  ret void
+}
+
+define void @slt_mul_nsw_both_var_non_negative_1(i8 %start, i8 %scale, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_both_var_non_negative_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], [[SCALE:%.*]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i8 [[SCALE]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C3]])
+; CHECK-NEXT:    [[C4:%.*]] = icmp sle i8 [[SCALE]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C3]])
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[T_0:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_0]])
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[T_1:%.*]] = icmp slt i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_1]])
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[T_2:%.*]] = icmp slt i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_2]])
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[T_3:%.*]] = icmp slt i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_3]])
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, %scale
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %c3 = icmp sgt i8 %scale, 0
+  call void @llvm.assume(i1 %c3)
+  %c4 = icmp sle i8 %scale, 3
+  call void @llvm.assume(i1 %c3)
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %t.0 = icmp slt i8 %start, %high
+  call void @use(i1 %t.0)
+  %start.1 = mul nsw i8 %start, 1
+  %t.1 = icmp slt i8 %start.1, %high
+  call void @use(i1 %t.1)
+  %start.2 = mul nsw i8 %start, 2
+  %t.2 = icmp slt i8 %start.2, %high
+  call void @use(i1 %t.2)
+  %start.3 = mul nsw i8 %start, 3
+  %t.3 = icmp slt i8 %start.3, %high
+  call void @use(i1 %t.3)
+  %start.4 = mul nsw i8 %start, 4
+  %c.3 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.3)
+  ret void
+
+else:
+  %c.4 = icmp slt i8 %start, %high
+  call void @use(i1 %c.4)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.5 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.5)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.6 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.6)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.7 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.7)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.8 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.8)
+  ret void
+}
+
+define void @slt_mul_nsw_both_var_non_negative_2(i8 %start, i8 %scale, i8 %high) {
+; CHECK-LABEL: @slt_mul_nsw_both_var_non_negative_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], [[SCALE:%.*]]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp sgt i8 [[START]], 0
+; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i8 [[SCALE]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C3]])
+; CHECK-NEXT:    [[C4:%.*]] = icmp sle i8 [[SCALE]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[C3]])
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, %scale
+  %c.1 = icmp slt i8 %mul.3, %high
+  %c.2 = icmp sgt i8 %start, 0
+  %c3 = icmp sgt i8 %scale, 0
+  call void @llvm.assume(i1 %c3)
+  %c4 = icmp sle i8 %scale, 3
+  call void @llvm.assume(i1 %c3)
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.4 = icmp slt i8 %start, %high
+  call void @use(i1 %c.4)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.5 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.5)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.6 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.6)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.7 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.7)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.8 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.8)
+  ret void
+}
+
+define void @ult_mul_nsw_3_known_positive_1(i8 %start, i8 %high) {
+; CHECK-LABEL: @ult_mul_nsw_3_known_positive_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp ugt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[T_0:%.*]] = icmp ult i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_0]])
+; CHECK-NEXT:    [[START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[T_1:%.*]] = icmp ult i8 [[START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_1]])
+; CHECK-NEXT:    [[START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[T_2:%.*]] = icmp ult i8 [[START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_2]])
+; CHECK-NEXT:    [[START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[T_3:%.*]] = icmp ult i8 [[START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[T_3]])
+; CHECK-NEXT:    [[START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_3:%.*]] = icmp slt i8 [[START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp ult i8 %mul.3, %high
+  %c.2 = icmp ugt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  %t.0 = icmp ult i8 %start, %high
+  call void @use(i1 %t.0)
+  %start.1 = mul nsw i8 %start, 1
+  %t.1 = icmp ult i8 %start.1, %high
+  call void @use(i1 %t.1)
+  %start.2 = mul nsw i8 %start, 2
+  %t.2 = icmp ult i8 %start.2, %high
+  call void @use(i1 %t.2)
+  %start.3 = mul nsw i8 %start, 3
+  %t.3 = icmp ult i8 %start.3, %high
+  call void @use(i1 %t.3)
+  %start.4 = mul nsw i8 %start, 4
+  %c.3 = icmp slt i8 %start.4, %high
+  call void @use(i1 %c.3)
+  ret void
+
+else:
+  ret void
+}
+
+define void @ult_mul_nsw_3_known_positive_2(i8 %start, i8 %high) {
+; CHECK-LABEL: @ult_mul_nsw_3_known_positive_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL_3:%.*]] = mul nsw i8 [[START:%.*]], 3
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i8 [[MUL_3]], [[HIGH:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp ugt i8 [[START]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    [[C_4:%.*]] = icmp slt i8 [[START]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_4]])
+; CHECK-NEXT:    [[ELSE_START_1:%.*]] = mul nsw i8 [[START]], 1
+; CHECK-NEXT:    [[C_5:%.*]] = icmp slt i8 [[ELSE_START_1]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_5]])
+; CHECK-NEXT:    [[ELSE_START_2:%.*]] = mul nsw i8 [[START]], 2
+; CHECK-NEXT:    [[C_6:%.*]] = icmp slt i8 [[ELSE_START_2]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_6]])
+; CHECK-NEXT:    [[ELSE_START_3:%.*]] = mul nsw i8 [[START]], 3
+; CHECK-NEXT:    [[C_7:%.*]] = icmp slt i8 [[ELSE_START_3]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_7]])
+; CHECK-NEXT:    [[ELSE_START_4:%.*]] = mul nsw i8 [[START]], 4
+; CHECK-NEXT:    [[C_8:%.*]] = icmp slt i8 [[ELSE_START_4]], [[HIGH]]
+; CHECK-NEXT:    call void @use(i1 [[C_8]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul.3  = mul nsw i8 %start, 3
+  %c.1 = icmp ult i8 %mul.3, %high
+  %c.2 = icmp ugt i8 %start, 0
+  %and = and i1 %c.1, %c.2
+  br i1 %and, label %then, label %else
+
+then:
+  ret void
+
+else:
+  %c.4 = icmp slt i8 %start, %high
+  call void @use(i1 %c.4)
+  %else.start.1 = mul nsw i8 %start, 1
+  %c.5 = icmp slt i8 %else.start.1, %high
+  call void @use(i1 %c.5)
+  %else.start.2 = mul nsw i8 %start, 2
+  %c.6 = icmp slt i8 %else.start.2, %high
+  call void @use(i1 %c.6)
+  %else.start.3 = mul nsw i8 %start, 3
+  %c.7 = icmp slt i8 %else.start.3, %high
+  call void @use(i1 %c.7)
+  %else.start.4 = mul nsw i8 %start, 4
+  %c.8 = icmp slt i8 %else.start.4, %high
+  call void @use(i1 %c.8)
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/masked-dead-store-inseltpoison.ll b/llvm/test/Transforms/DeadStoreElimination/masked-dead-store-inseltpoison.ll
index 2ed6350ec6d0b..0b4d657f976e2 100644
--- a/llvm/test/Transforms/DeadStoreElimination/masked-dead-store-inseltpoison.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/masked-dead-store-inseltpoison.ll
@@ -13,7 +13,7 @@ define dllexport i32 @f0(ptr %a0, ptr %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i
 ; CHECK-NEXT:    [[V8:%.*]] = getelementptr i8, ptr [[V7]], i32 [[A5:%.*]]
 ; CHECK-NEXT:    [[V10:%.*]] = tail call <128 x i8> @llvm.masked.load.v128i8.p0(ptr [[V8]], i32 32, <128 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <128 x i8> undef), [[TBAA5:!tbaa !.*]]
 ; CHECK-NEXT:    [[V11:%.*]] = shufflevector <128 x i8> [[V10]], <128 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[V14:%.*]] = shufflevector <32 x i8> [[V11]], <32 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V14:%.*]] = shufflevector <32 x i8> [[V11]], <32 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[V16:%.*]] = shufflevector <128 x i8> [[V14]], <128 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[V17:%.*]] = getelementptr inbounds ptr, ptr [[A1]], i32 [[A6:%.*]]
 ; CHECK-NEXT:    [[V18:%.*]] = load ptr, ptr [[V17]], align 4, [[TBAA3]]
@@ -22,7 +22,7 @@ define dllexport i32 @f0(ptr %a0, ptr %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i
 ; CHECK-NEXT:    [[V22:%.*]] = shufflevector <128 x i8> [[V21]], <128 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[V23:%.*]] = icmp ugt <32 x i8> [[V16]], [[V22]]
 ; CHECK-NEXT:    [[V24:%.*]] = select <32 x i1> [[V23]], <32 x i8> [[V16]], <32 x i8> [[V22]]
-; CHECK-NEXT:    [[V25:%.*]] = shufflevector <32 x i8> [[V24]], <32 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V25:%.*]] = shufflevector <32 x i8> [[V24]], <32 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    tail call void @llvm.masked.store.v128i8.p0(<128 x i8> [[V25]], ptr [[V2]], i32 32, <128 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>), [[TBAA8:!tbaa !.*]]
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/DeadStoreElimination/masked-dead-store.ll b/llvm/test/Transforms/DeadStoreElimination/masked-dead-store.ll
index 7e778327c16b5..f0f52d56a9ff0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/masked-dead-store.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/masked-dead-store.ll
@@ -13,7 +13,7 @@ define dllexport i32 @f0(ptr %a0, ptr %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i
 ; CHECK-NEXT:    [[V8:%.*]] = getelementptr i8, ptr [[V7]], i32 [[A5:%.*]]
 ; CHECK-NEXT:    [[V10:%.*]] = tail call <128 x i8> @llvm.masked.load.v128i8.p0(ptr [[V8]], i32 32, <128 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <128 x i8> undef), !tbaa [[TBAA5:![0-9]+]]
 ; CHECK-NEXT:    [[V11:%.*]] = shufflevector <128 x i8> [[V10]], <128 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[V14:%.*]] = shufflevector <32 x i8> [[V11]], <32 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V14:%.*]] = shufflevector <32 x i8> [[V11]], <32 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[V16:%.*]] = shufflevector <128 x i8> [[V14]], <128 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[V17:%.*]] = getelementptr inbounds ptr, ptr [[A1]], i32 [[A6:%.*]]
 ; CHECK-NEXT:    [[V18:%.*]] = load ptr, ptr [[V17]], align 4, !tbaa [[TBAA3]]
@@ -22,7 +22,7 @@ define dllexport i32 @f0(ptr %a0, ptr %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i
 ; CHECK-NEXT:    [[V22:%.*]] = shufflevector <128 x i8> [[V21]], <128 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[V23:%.*]] = icmp ugt <32 x i8> [[V16]], [[V22]]
 ; CHECK-NEXT:    [[V24:%.*]] = select <32 x i1> [[V23]], <32 x i8> [[V16]], <32 x i8> [[V22]]
-; CHECK-NEXT:    [[V25:%.*]] = shufflevector <32 x i8> [[V24]], <32 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V25:%.*]] = shufflevector <32 x i8> [[V24]], <32 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    tail call void @llvm.masked.store.v128i8.p0(<128 x i8> [[V25]], ptr [[V2]], i32 32, <128 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>), !tbaa [[TBAA8:![0-9]+]]
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll b/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll
index 69995ae921cc4..6d313a6cb417d 100644
--- a/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll
+++ b/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes='early-cse<memssa>' -earlycse-debug-hash < %s | FileCheck %s
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; CHECK-LABEL: @memrealtime(
 ; CHECK: call i64 @llvm.amdgcn.s.memrealtime()
diff --git a/llvm/test/Transforms/EarlyCSE/basic.ll b/llvm/test/Transforms/EarlyCSE/basic.ll
index 0a743749cf3a5..c6b746026c94d 100644
--- a/llvm/test/Transforms/EarlyCSE/basic.ll
+++ b/llvm/test/Transforms/EarlyCSE/basic.ll
@@ -137,15 +137,18 @@ declare i32 @func(ptr%P) readonly
 ;; Simple call CSE'ing.
 define i32 @test5(ptr%P) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[V1:%.*]] = call i32 @func(ptr [[P:%.*]])
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @func(ptr [[P:%.*]]), !prof !0
 ; CHECK-NEXT:    ret i32 0
 ;
-  %V1 = call i32 @func(ptr %P)
-  %V2 = call i32 @func(ptr %P)
+  %V1 = call i32 @func(ptr %P), !prof !0
+  %V2 = call i32 @func(ptr %P), !prof !1
   %Diff = sub i32 %V1, %V2
   ret i32 %Diff
 }
 
+!0 = !{!"branch_weights", i32 95}
+!1 = !{!"branch_weights", i32 95}
+
 ;; Trivial Store->load forwarding
 define i32 @test6(ptr%P) {
 ; CHECK-LABEL: @test6(
diff --git a/llvm/test/Transforms/EarlyCSE/flags.ll b/llvm/test/Transforms/EarlyCSE/flags.ll
index 57ea0866d07ac..78282b081c298 100644
--- a/llvm/test/Transforms/EarlyCSE/flags.ll
+++ b/llvm/test/Transforms/EarlyCSE/flags.ll
@@ -90,3 +90,15 @@ define void @load_first_nonnull_noundef(ptr %p) {
   call void @use.ptr(ptr %v2)
   ret void
 }
+
+define ptr @store_to_load_forward(ptr %p, ptr %p2) {
+; CHECK-LABEL: @store_to_load_forward(
+; CHECK-NEXT:    [[P3:%.*]] = load ptr, ptr [[P:%.*]], align 8, !nonnull !0
+; CHECK-NEXT:    store ptr [[P3]], ptr [[P2:%.*]], align 8
+; CHECK-NEXT:    ret ptr [[P3]]
+;
+  %p3 = load ptr, ptr %p, !nonnull !{}
+  store ptr %p3, ptr %p2
+  %res = load ptr, ptr %p2
+  ret ptr %res
+}
diff --git a/llvm/test/Transforms/EarlyCSE/metadata.ll b/llvm/test/Transforms/EarlyCSE/metadata.ll
new file mode 100644
index 0000000000000..e54f5ff49a054
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/metadata.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -S -passes=early-cse | FileCheck %s
+
+; Test that when the load from %x is CSE'ed to %call1 by EarlyCSE, the !prof metadata
+; is not dropped.
+define i32 @indirect_call_value_profile_preserved(ptr %x, i32 %a, i32 %b, i32 %c)  {
+; CHECK-LABEL: define i32 @indirect_call_value_profile_preserved
+; CHECK-SAME: (ptr [[X:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call ptr @_Z10createTypei(i32 [[C]])
+; CHECK-NEXT:    [[VTABLE:%.*]] = load ptr, ptr [[CALL]], align 8
+; CHECK-NEXT:    [[VFN:%.*]] = getelementptr inbounds ptr, ptr [[VTABLE]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VFN]], align 8
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 [[TMP0]](ptr [[CALL]], i32 [[A]], i32 [[B]]), !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    store i32 [[CALL1]], ptr [[X]], align 4
+; CHECK-NEXT:    ret i32 [[CALL1]]
+;
+entry:
+  %call = tail call ptr @_Z10createTypei(i32 %c)
+  %vtable = load ptr, ptr %call, align 8
+  %vfn = getelementptr inbounds ptr, ptr %vtable, i64 1
+  %0 = load ptr, ptr %vfn, align 8
+  %call1 = tail call i32 %0(ptr %call, i32 %a, i32 %b), !prof !0
+  store i32 %call1, ptr %x
+  %foo = load i32, ptr %x
+  ret i32 %foo
+}
+
+declare ptr @_Z10createTypei(i32)
+
+!0 =!{!"VP", i32 0, i64 1600, i64 12345, i64 1030, i64 678, i64 410}
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
index 9be81d96a8a37..c444b060d6130 100644
--- a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
+++ b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
@@ -99,6 +99,13 @@ define ptr @tailcaller2() #8 {
 ; CHECK: ret
 }
 
+;; naked functions are not instrumented, otherwise the argument registers
+;; and the return address register (if present) would be clobbered.
+define void @naked() naked { entry: ret void }
+; CHECK-LABEL:      define void @naked(
+; CHECK-LABEL-NEXT: entry:
+; CHECK-LABEL-NEXT:   ret void
+
 ; The attributes are "consumed" when the instrumentation is inserted.
 ; CHECK: attributes
 ; CHECK-NOT: instrument-function
diff --git a/llvm/test/Transforms/FunctionSpecialization/compiler-crash-promote-alloca.ll b/llvm/test/Transforms/FunctionSpecialization/compiler-crash-promote-alloca.ll
index 4d00d7a284ab7..fff454db303c9 100644
--- a/llvm/test/Transforms/FunctionSpecialization/compiler-crash-promote-alloca.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/compiler-crash-promote-alloca.ll
@@ -4,7 +4,7 @@
 ; while attempting to promote the alloca in `entry`.
 ; CHECK: bar.1
 
-@block = internal global [8 x i8] zeroinitializer, align 1
+@block = internal constant [8 x i8] zeroinitializer, align 1
 
 define dso_local void @entry() {
   %1 = alloca i32, align 4
diff --git a/llvm/test/Transforms/FunctionSpecialization/global-var-constants.ll b/llvm/test/Transforms/FunctionSpecialization/global-var-constants.ll
new file mode 100644
index 0000000000000..14c4855b58a24
--- /dev/null
+++ b/llvm/test/Transforms/FunctionSpecialization/global-var-constants.ll
@@ -0,0 +1,78 @@
+; RUN:  opt -S --passes=ipsccp,deadargelim --force-specialization                       < %s | FileCheck %s --check-prefix=NO-GLOBALS
+; RUN:  opt -S --passes=ipsccp,deadargelim --force-specialization --funcspec-on-address < %s | FileCheck %s --check-prefix=GLOBALS
+@G = global [10 x i32] zeroinitializer, align 4
+
+define internal i32 @f(ptr %p) noinline {
+entry:
+  %0 = load i32, ptr %p, align 4
+  store i32 0, ptr %p, align 4
+  ret i32 %0
+}
+
+define internal i32 @g(i32 %x, i32 %y, ptr %p) noinline {
+entry:
+  %cmp = icmp sgt i32 %x, %y
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  br label %if.end
+
+if.else:
+  br label %if.end
+
+if.end:
+  %x.addr.0 = phi i32 [ %x, %if.then ], [ 11, %if.else ]
+  %p.addr.0 = phi ptr [ @G, %if.then ], [ %p, %if.else ]
+  %call = call i32 @f(ptr %p.addr.0)
+  %add = add nsw i32 %call, %x.addr.0
+  ret i32 %add
+}
+
+define i32 @h0(ptr %p) {
+entry:
+  %call = call i32 @g(i32 2, i32 1, ptr %p)
+  ret i32 %call
+}
+
+define i32 @h1() {
+entry:
+  %call = call i32 @f(ptr @G)
+  ret i32 %call
+}
+
+define i32 @h2() {
+entry:
+  %call = call i32 @f(ptr getelementptr inbounds (i32, ptr @G, i64 1))
+  ret i32 %call
+}
+
+; Check if specialisation on the address of a non-const global variable
+; is not allowed, then it is not performed.
+
+; NO-GLOBALS-LABEL: define internal i32 @g()
+; NO-GLOBALS: call i32 @f(ptr @G)
+
+; NO-GLOBALS-LABEL: define i32 @h0(ptr %p)
+; NO-GLOBALS:call i32 @g()
+
+; NO-GLOBALS-LABEL: define i32 @h1()
+; NO-GLOBALS: call i32 @f(ptr @G)
+
+; NO-GLOBALS-LABEL: define i32 @h2()
+; NO-GLOBALS: call i32 @f(ptr getelementptr inbounds (i32, ptr @G, i64 1))
+
+; Check if specialisation on the address of a non-const global variable
+; is allowed, then it is performed where possible.
+
+; GLOBALS-LABEL: define internal i32 @g()
+; GLOBALS: call i32 @f.2()
+
+; GLOBALS-LABEL: define i32 @h0(ptr %p)
+; GLOBALS: call i32 @g()
+
+; GLOBALS-LABEL: define i32 @h1()
+; GLOBALS: call i32 @f.2()
+
+; GLOBALS-LABEL: define i32 @h2()
+; GLOBALS: call i32 @f.1()
+
diff --git a/llvm/test/Transforms/GVN/calls-readonly.ll b/llvm/test/Transforms/GVN/calls-readonly.ll
index 5c24740c881b4..b4855e41a64f5 100644
--- a/llvm/test/Transforms/GVN/calls-readonly.ll
+++ b/llvm/test/Transforms/GVN/calls-readonly.ll
@@ -6,7 +6,7 @@ target triple = "i386-apple-darwin7"
 
 define ptr @test(ptr %P, ptr %Q, i32 %x, i32 %y) nounwind readonly {
 entry:
-  %0 = tail call i32 @strlen(ptr %P)              ; <i32> [#uses=2]
+  %0 = tail call i32 @strlen(ptr %P), !prof !0    ; <i32> [#uses=2]
   %1 = icmp eq i32 %0, 0                          ; <i1> [#uses=1]
   br i1 %1, label %bb, label %bb1
 
@@ -17,7 +17,7 @@ bb:                                               ; preds = %entry
 bb1:                                              ; preds = %bb, %entry
   %x_addr.0 = phi i32 [ %2, %bb ], [ %x, %entry ] ; <i32> [#uses=1]
   %3 = tail call ptr @strchr(ptr %Q, i32 97)      ; <ptr> [#uses=1]
-  %4 = tail call i32 @strlen(ptr %P)              ; <i32> [#uses=1]
+  %4 = tail call i32 @strlen(ptr %P) , !prof !1   ; <i32> [#uses=1]
   %5 = add i32 %x_addr.0, %0                      ; <i32> [#uses=1]
   %.sum = sub i32 %5, %4                          ; <i32> [#uses=1]
   %6 = getelementptr i8, ptr %3, i32 %.sum            ; <ptr> [#uses=1]
@@ -26,7 +26,7 @@ bb1:                                              ; preds = %bb, %entry
 
 ; CHECK: define ptr @test(ptr %P, ptr %Q, i32 %x, i32 %y) #0 {
 ; CHECK: entry:
-; CHECK-NEXT:   %0 = tail call i32 @strlen(ptr %P)
+; CHECK-NEXT:   %0 = tail call i32 @strlen(ptr %P), !prof !0
 ; CHECK-NEXT:   %1 = icmp eq i32 %0, 0
 ; CHECK-NEXT:   br i1 %1, label %bb, label %bb1
 ; CHECK: bb:
@@ -43,3 +43,6 @@ bb1:                                              ; preds = %bb, %entry
 declare i32 @strlen(ptr) nounwind readonly
 
 declare ptr @strchr(ptr, i32) nounwind readonly
+
+!0 = !{!"branch_weights", i32 95}
+!1 = !{!"branch_weights", i32 95}
diff --git a/llvm/test/Transforms/GlobalOpt/X86/alias-used-with-asm.ll b/llvm/test/Transforms/GlobalOpt/X86/alias-used-with-asm.ll
new file mode 100644
index 0000000000000..3161b72ed1ef8
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/X86/alias-used-with-asm.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -passes=globalopt -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm ".global foo1"
+module asm "foo1: jmp bar1"
+module asm ".global foo2"
+module asm "foo2: jmp bar2"
+
+; The `llvm.compiler.used` indicates that `foo1` and `foo2` have associated symbol references in asm.
+; Checking globalopt does not remove these two symbols.
+@llvm.compiler.used = appending global [2 x ptr] [ptr @bar1, ptr @bar2], section "llvm.metadata"
+; CHECK: @llvm.compiler.used = appending global [2 x ptr] [ptr @bar1, ptr @bar2], section "llvm.metadata"
+
+@bar2 = internal alias void (), ptr @bar1
+; CHECK: @bar2 = internal alias void (), ptr @bar1
+
+define internal void @bar1() {
+; CHECK: define internal void @bar1()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/addrspacecast.ll b/llvm/test/Transforms/GlobalOpt/addrspacecast.ll
new file mode 100644
index 0000000000000..7920629664d54
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/addrspacecast.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -passes=globalopt < %s | FileCheck %s
+
+; Test behavior of trying to fold access of global variables
+; initialized one time through addrspacecast.
+
+@as0_ptr = internal global ptr null
+@as1_ptr = internal global ptr addrspace(1) null
+@as2_ptr = internal global ptr addrspace(2) null
+@as1_null_valid_ptr = internal global ptr addrspace(1) null
+@as0_null_valid_ptr = internal global ptr null
+
+@Actual.as0 = internal global i32 1234
+@Actual.as1 = internal addrspace(1) global i32 5678
+@Actual.as2 = internal addrspace(2) global i32 91011
+@Actual.as1.null.valid = internal addrspace(1) global i32 91011
+@Actual.as0.null.valid = internal global i32 9999
+
+; Initialize each global once.
+define void @init() {
+; CHECK-LABEL: define void @init() local_unnamed_addr {
+; CHECK-NEXT:    store ptr addrspace(1) @Actual.as1, ptr @as1_ptr, align 8
+; CHECK-NEXT:    store ptr addrspace(2) @Actual.as2, ptr @as2_ptr, align 8
+; CHECK-NEXT:    store ptr addrspace(1) @Actual.as1.null.valid, ptr @as1_null_valid_ptr, align 8
+; CHECK-NEXT:    store ptr @Actual.as0.null.valid, ptr @as0_null_valid_ptr, align 8
+; CHECK-NEXT:    ret void
+;
+  store ptr @Actual.as0, ptr @as0_ptr
+  store ptr addrspace(1) @Actual.as1, ptr @as1_ptr
+  store ptr addrspace(2) @Actual.as2, ptr @as2_ptr
+  store ptr addrspace(1) @Actual.as1.null.valid, ptr @as1_null_valid_ptr
+  store ptr @Actual.as0.null.valid, ptr @as0_null_valid_ptr
+  ret void
+}
+
+; Access stored value is null in the default address space, which we
+; can assume cannot alias a valid object in another address space.
+define i32 @load_from_as0_ptr_in_as1() {
+; CHECK-LABEL: define i32 @load_from_as0_ptr_in_as1() local_unnamed_addr {
+; CHECK-NEXT:    ret i32 1234
+;
+  %ptr = load ptr, ptr @as0_ptr
+  %cast = addrspacecast ptr %ptr to ptr addrspace(1)
+  %load = load i32, ptr addrspace(1) %cast
+  ret i32 %load
+}
+
+; Cast is to default address space and access is done through default
+; address space in a function where null is valid.
+define i32 @load_from_as0_ptr_in_as1_null_is_valid() null_pointer_is_valid {
+; CHECK-LABEL: define i32 @load_from_as0_ptr_in_as1_null_is_valid
+; CHECK-SAME: () local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[PTR:%.*]] = load ptr addrspace(1), ptr @as1_null_valid_ptr, align 8
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[CAST]], align 4
+; CHECK-NEXT:    ret i32 [[LOAD]]
+;
+  %ptr = load ptr addrspace(1), ptr @as1_null_valid_ptr
+  %cast = addrspacecast ptr addrspace(1) %ptr to ptr
+  %load = load i32, ptr %cast
+  ret i32 %load
+}
+
+; Original addrspace(0) null was valid, and we also have to assume the
+; access is valid in addrspace(1)
+define i32 @load_from_as1_ptr_in_as0_null_is_valid() null_pointer_is_valid {
+; CHECK-LABEL: define i32 @load_from_as1_ptr_in_as0_null_is_valid
+; CHECK-SAME: () local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT:    [[PTR:%.*]] = load ptr, ptr @as0_null_valid_ptr, align 8
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[CAST]], align 4
+; CHECK-NEXT:    ret i32 [[LOAD]]
+;
+  %ptr = load ptr, ptr @as0_null_valid_ptr
+  %cast = addrspacecast ptr %ptr to ptr addrspace(1)
+  %load = load i32, ptr addrspace(1) %cast
+  ret i32 %load
+}
+
+; Underlying pointer could be valid, can't fold this out.
+define i32 @load_from_as1_ptr_in_as0() {
+; CHECK-LABEL: define i32 @load_from_as1_ptr_in_as0() local_unnamed_addr {
+; CHECK-NEXT:    [[PTR:%.*]] = load ptr addrspace(1), ptr @as1_ptr, align 8
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(1) [[PTR]] to ptr
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[CAST]], align 4
+; CHECK-NEXT:    ret i32 [[LOAD]]
+;
+  %ptr = load ptr addrspace(1), ptr @as1_ptr
+  %cast = addrspacecast ptr addrspace(1) %ptr to ptr
+  %load = load i32, ptr %cast
+  ret i32 %load
+}
+
+; Two non-default address spaces, can't fold.
+define i32 @load_from_as2_ptr_in_as1() {
+; CHECK-LABEL: define i32 @load_from_as2_ptr_in_as1() local_unnamed_addr {
+; CHECK-NEXT:    [[PTR:%.*]] = load ptr addrspace(2), ptr @as2_ptr, align 8
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr addrspace(1)
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[CAST]], align 4
+; CHECK-NEXT:    ret i32 [[LOAD]]
+;
+  %ptr = load ptr addrspace(2), ptr @as2_ptr
+  %cast = addrspacecast ptr addrspace(2) %ptr to ptr addrspace(1)
+  %load = load i32, ptr addrspace(1) %cast
+  ret i32 %load
+}
diff --git a/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll b/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll
index e60d35c599743..6bac291ad9c0e 100644
--- a/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll
+++ b/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll
@@ -5,19 +5,20 @@ target datalayout = "p:32:32:32-p1:16:16:16"
 @c = hidden addrspace(1) global i8 42
 
 @i = internal addrspace(1) global i8 42
-
-; CHECK: @ia = internal addrspace(1) global i8 42
-@ia = internal alias i8, ptr addrspace(1) @i
+; CHECK: @i = internal addrspace(1) global i8 42
 
 @llvm.used = appending global [1 x ptr] [ptr addrspacecast (ptr addrspace(1) @ca to ptr)], section "llvm.metadata"
 ; CHECK-DAG: @llvm.used = appending global [1 x ptr] [ptr addrspacecast (ptr addrspace(1) @ca to ptr)], section "llvm.metadata"
 
 @llvm.compiler.used = appending global [2 x ptr] [ptr addrspacecast(ptr addrspace(1) @ia to ptr), ptr addrspacecast (ptr addrspace(1) @i to ptr)], section "llvm.metadata"
-; CHECK-DAG: @llvm.compiler.used = appending global [1 x ptr] [ptr addrspacecast (ptr addrspace(1) @ia to ptr)], section "llvm.metadata"
+; CHECK-DAG: @llvm.compiler.used = appending global [2 x ptr] [ptr addrspacecast (ptr addrspace(1) @i to ptr), ptr addrspacecast (ptr addrspace(1) @ia to ptr)], section "llvm.metadata"
 
 @sameAsUsed = global [1 x ptr] [ptr addrspacecast(ptr addrspace(1) @ca to ptr)]
 ; CHECK-DAG: @sameAsUsed = local_unnamed_addr global [1 x ptr] [ptr addrspacecast (ptr addrspace(1) @c to ptr)]
 
+@ia = internal alias i8, ptr addrspace(1) @i
+; CHECK: @ia = internal alias i8, ptr addrspace(1) @i
+
 @ca = internal alias i8, ptr addrspace(1) @c
 ; CHECK: @ca = internal alias i8, ptr addrspace(1) @c
 
diff --git a/llvm/test/Transforms/GlobalOpt/alias-used.ll b/llvm/test/Transforms/GlobalOpt/alias-used.ll
index 35889c1e59754..7137bb13af58c 100644
--- a/llvm/test/Transforms/GlobalOpt/alias-used.ll
+++ b/llvm/test/Transforms/GlobalOpt/alias-used.ll
@@ -2,15 +2,16 @@
 
 @c = dso_local global i8 42
 
+; CHECK: @i = internal global i8 42
 @i = internal global i8 42
-; CHECK: @ia = internal global i8 42
+; CHECK-DAG: @ia = internal alias i8, ptr @i
 @ia = internal alias i8, ptr @i
 
 @llvm.used = appending global [3 x ptr] [ptr @fa, ptr @f, ptr @ca], section "llvm.metadata"
 ; CHECK-DAG: @llvm.used = appending global [3 x ptr] [ptr @ca, ptr @f, ptr @fa], section "llvm.metadata"
 
 @llvm.compiler.used = appending global [4 x ptr] [ptr @fa3, ptr @fa, ptr @ia, ptr @i], section "llvm.metadata"
-; CHECK-DAG: @llvm.compiler.used = appending global [2 x ptr] [ptr @fa3, ptr @ia], section "llvm.metadata"
+; CHECK-DAG: @llvm.compiler.used = appending global [3 x ptr] [ptr @fa3, ptr @i, ptr @ia], section "llvm.metadata"
 
 @sameAsUsed = global [3 x ptr] [ptr @fa, ptr @f, ptr @ca]
 ; CHECK-DAG: @sameAsUsed = local_unnamed_addr global [3 x ptr] [ptr @f, ptr @f, ptr @c]
diff --git a/llvm/test/Transforms/GlobalOpt/deadglobal-diarglist-use.ll b/llvm/test/Transforms/GlobalOpt/deadglobal-diarglist-use.ll
index 4b396928b48e4..f077e0013278d 100644
--- a/llvm/test/Transforms/GlobalOpt/deadglobal-diarglist-use.ll
+++ b/llvm/test/Transforms/GlobalOpt/deadglobal-diarglist-use.ll
@@ -6,7 +6,7 @@
 
 ; CHECK-NOT: @s
 
-; CHECK: call void @llvm.dbg.value(metadata !DIArgList(ptr undef
+; CHECK: call void @llvm.dbg.value(metadata !DIArgList(ptr poison
 
 %struct.S = type { i32 }
 
diff --git a/llvm/test/Transforms/GlobalOpt/global-opt-addrspace.ll b/llvm/test/Transforms/GlobalOpt/global-opt-addrspace.ll
new file mode 100644
index 0000000000000..9df8a4bfbfae1
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/global-opt-addrspace.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -passes=globalopt < %s | FileCheck %s
+;; Check that Global Opt preserves address space of llvm.used and
+;; llvm.compiler.used variables.
+
+%struct.FakeDeviceGlobal = type { ptr addrspace(4) }
+%class.anon = type { i8 }
+
+@_ZM2C = internal addrspace(1) global %struct.FakeDeviceGlobal zeroinitializer, align 8
+@_ZL1C = internal addrspace(1) global %struct.FakeDeviceGlobal zeroinitializer, align 8
+
+@llvm.compiler.used = appending global [2 x ptr addrspace(4)] [ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZM2C to ptr addrspace(4)), ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZL1C to ptr addrspace(4))]
+
+; CHECK: @llvm.compiler.used = appending global [2 x ptr addrspace(4)] [ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZL1C to ptr addrspace(4)), ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZM2C to ptr addrspace(4))]
+
+define weak_odr dso_local void @foo() {
+entry:
+  %A = alloca %class.anon, align 1
+  %A.addrspacecast = addrspacecast ptr %A to ptr addrspace(4)
+  call void @bar(ptr addrspace(4) noundef align 1 dereferenceable_or_null(1) %A.addrspacecast)
+  ret void
+}
+
+define internal void @bar(ptr addrspace(4) noundef align 1 dereferenceable_or_null(1) %this) align 2 {
+entry:
+  %this.addr = alloca ptr addrspace(4), align 8
+  %this.addr.ascast = addrspacecast ptr %this.addr to ptr addrspace(4)
+  store ptr addrspace(4) %this, ptr addrspace(4) %this.addr.ascast, align 8
+  %v1 = load ptr addrspace(4), ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZM2C to ptr addrspace(4)), align 8
+  store i32 42, ptr addrspace(4) %v1, align 4
+  %v2 = load ptr addrspace(4), ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZL1C to ptr addrspace(4)), align 8
+  store i32 42, ptr addrspace(4) %v2, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/issue62384.ll b/llvm/test/Transforms/GlobalOpt/issue62384.ll
new file mode 100644
index 0000000000000..cc2bc8940b891
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/issue62384.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -p=globalopt %s | FileCheck %s
+
+
+@llvm.used = appending global [1 x ptr] [ptr @ctor]
+@gv0 = internal addrspace(1) global ptr null
+@gv1 = internal addrspace(1) global ptr addrspace(3) null
+@gv2 = internal addrspace(1) global ptr addrspace(1) null
+@gv3 = internal addrspace(1) global ptr addrspace(1) null
+
+define internal void @ctor() {
+; CHECK-LABEL: define internal void @ctor() {
+; CHECK-NEXT:    tail call fastcc void @init0(ptr addrspacecast (ptr addrspace(1) null to ptr))
+; CHECK-NEXT:    tail call fastcc void @init1(ptr addrspacecast (ptr addrspace(3) null to ptr))
+; CHECK-NEXT:    tail call fastcc void @init2(ptr addrspace(1) addrspacecast (ptr null to ptr addrspace(1)))
+; CHECK-NEXT:    tail call fastcc void @init3(ptr addrspace(1) addrspacecast (ptr addrspace(2) null to ptr addrspace(1)))
+; CHECK-NEXT:    ret void
+;
+  tail call void @init0(ptr addrspacecast (ptr addrspace(1) null to ptr))
+  tail call void @init1(ptr addrspacecast (ptr addrspace(3) null to ptr))
+  tail call void @init2(ptr addrspace(1) addrspacecast (ptr null to ptr addrspace(1)))
+  tail call void @init3(ptr addrspace(1) addrspacecast (ptr addrspace(2) null to ptr addrspace(1)))
+  ret void
+}
+
+define internal void @init0(ptr %T) {
+; CHECK-LABEL: define internal fastcc void @init0
+; CHECK-SAME: (ptr [[T:%.*]]) unnamed_addr {
+; CHECK-NEXT:    [[LD:%.*]] = load ptr, ptr addrspace(1) @gv0, align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LD]], null
+; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(1) null to ptr), ptr addrspace(1) @gv0, align 8
+; CHECK-NEXT:    ret void
+;
+  %ld = load ptr, ptr addrspace(1) @gv0, align 8
+  %cmp = icmp eq ptr %ld, null
+  store ptr addrspacecast (ptr addrspace(1) null to ptr), ptr addrspace(1) @gv0, align 8
+  ret void
+}
+
+define internal void @init1(ptr %T) {
+; CHECK-LABEL: define internal fastcc void @init1
+; CHECK-SAME: (ptr [[T:%.*]]) unnamed_addr {
+; CHECK-NEXT:    [[LD:%.*]] = load ptr addrspace(3), ptr addrspace(1) @gv1, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(3) [[LD]], null
+; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) null to ptr), ptr addrspace(1) @gv1, align 8
+; CHECK-NEXT:    ret void
+;
+  %ld = load ptr addrspace(3), ptr addrspace(1) @gv1, align 4
+  %cmp = icmp eq ptr addrspace(3) %ld, null
+  store ptr addrspacecast (ptr addrspace(3) null to ptr), ptr addrspace(1) @gv1, align 8
+  ret void
+}
+
+define internal void @init2(ptr addrspace(1) %T) {
+; CHECK-LABEL: define internal fastcc void @init2
+; CHECK-SAME: (ptr addrspace(1) [[T:%.*]]) unnamed_addr {
+; CHECK-NEXT:    [[LD:%.*]] = load ptr addrspace(1), ptr addrspace(1) @gv2, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(1) [[LD]], null
+; CHECK-NEXT:    store ptr addrspace(1) addrspacecast (ptr null to ptr addrspace(1)), ptr addrspace(1) @gv2, align 8
+; CHECK-NEXT:    ret void
+;
+  %ld = load ptr addrspace(1), ptr addrspace(1) @gv2, align 4
+  %cmp = icmp eq ptr addrspace(1) %ld, null
+  store ptr addrspace(1) addrspacecast (ptr null to ptr addrspace(1)), ptr addrspace(1) @gv2, align 8
+  ret void
+}
+
+define internal void @init3(ptr addrspace(1) %T) {
+; CHECK-LABEL: define internal fastcc void @init3
+; CHECK-SAME: (ptr addrspace(1) [[T:%.*]]) unnamed_addr {
+; CHECK-NEXT:    [[LD:%.*]] = load ptr addrspace(1), ptr addrspace(1) @gv3, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(1) [[LD]], null
+; CHECK-NEXT:    store ptr addrspace(1) addrspacecast (ptr addrspace(2) null to ptr addrspace(1)), ptr addrspace(1) @gv3, align 8
+; CHECK-NEXT:    ret void
+;
+  %ld = load ptr addrspace(1), ptr addrspace(1) @gv3, align 4
+  %cmp = icmp eq ptr addrspace(1) %ld, null
+  store ptr addrspace(1) addrspacecast (ptr addrspace(2) null to ptr addrspace(1)), ptr addrspace(1) @gv3, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll b/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll
index 5118d95ab633c..a589a3dc763c5 100644
--- a/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll
+++ b/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll
@@ -9,7 +9,7 @@
 ; twice as expensive as that on a 32-bit integer, or split into 2
 ; 32-bit components.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; CHECK-LABEL: @indvar_32_bit(
 ; CHECK-NOT: sext i32
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll b/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll
index 02de829421f4c..3e24bcd98e42d 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-pr20680.ll
@@ -21,9 +21,6 @@ define void @f() {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC13:%.*]] ], [ -14, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br i1 [[TOBOOL2]], label [[FOR_INC13]], label [[FOR_BODY3_LR_PH:%.*]]
 ; CHECK:       for.body3.lr.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i32 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 3
-; CHECK-NEXT:    [[DIV:%.*]] = select i1 [[TMP3]], i32 [[INDVARS_IV]], i32 0
 ; CHECK-NEXT:    br i1 false, label [[FOR_BODY3_LR_PH_SPLIT_US:%.*]], label [[FOR_BODY3_LR_PH_FOR_BODY3_LR_PH_SPLIT_CRIT_EDGE:%.*]]
 ; CHECK:       for.body3.lr.ph.for.body3.lr.ph.split_crit_edge:
 ; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH_SPLIT:%.*]]
@@ -38,8 +35,8 @@ define void @f() {
 ; CHECK:       cond.false.us.us:
 ; CHECK-NEXT:    br label [[COND_END_US_US]]
 ; CHECK:       cond.end.us.us:
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @b, align 4
-; CHECK-NEXT:    [[CMP91_US_US:%.*]] = icmp slt i32 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[CMP91_US_US:%.*]] = icmp slt i32 [[TMP2]], 1
 ; CHECK-NEXT:    br i1 [[CMP91_US_US]], label [[FOR_INC_LR_PH_US_US:%.*]], label [[FOR_COND2_LOOPEXIT_US_US:%.*]]
 ; CHECK:       for.cond2.loopexit.us.us:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA_US:%.*]], label [[FOR_BODY3_US_US]]
@@ -49,8 +46,8 @@ define void @f() {
 ; CHECK-NEXT:    store i32 1, ptr @b, align 4
 ; CHECK-NEXT:    br label [[FOR_COND2_LOOPEXIT_US_US]]
 ; CHECK:       for.inc.us.us:
-; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP4]], [[FOR_INC_LR_PH_US_US]] ], [ [[INC_US_US:%.*]], [[FOR_INC_US_US]] ]
-; CHECK-NEXT:    [[INC_US_US]] = add nsw i32 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[TMP2]], [[FOR_INC_LR_PH_US_US]] ], [ [[INC_US_US:%.*]], [[FOR_INC_US_US]] ]
+; CHECK-NEXT:    [[INC_US_US]] = add nsw i32 [[TMP3]], 1
 ; CHECK-NEXT:    [[EXITCOND3:%.*]] = icmp ne i32 [[INC_US_US]], 1
 ; CHECK-NEXT:    br i1 [[EXITCOND3]], label [[FOR_INC_US_US]], label [[FOR_COND8_FOR_COND2_LOOPEXIT_CRIT_EDGE_US_US:%.*]]
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa.us:
@@ -62,12 +59,12 @@ define void @f() {
 ; CHECK:       cond.false.us:
 ; CHECK-NEXT:    br label [[COND_END_US]]
 ; CHECK:       cond.end.us:
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr @b, align 4
-; CHECK-NEXT:    [[CMP91_US:%.*]] = icmp slt i32 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[CMP91_US:%.*]] = icmp slt i32 [[TMP4]], 1
 ; CHECK-NEXT:    br i1 [[CMP91_US]], label [[FOR_INC_LR_PH_US:%.*]], label [[FOR_COND2_LOOPEXIT_US:%.*]]
 ; CHECK:       for.inc.us:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP6]], [[FOR_INC_LR_PH_US]] ], [ [[INC_US:%.*]], [[FOR_INC_US:%.*]] ]
-; CHECK-NEXT:    [[INC_US]] = add nsw i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ [[TMP4]], [[FOR_INC_LR_PH_US]] ], [ [[INC_US:%.*]], [[FOR_INC_US:%.*]] ]
+; CHECK-NEXT:    [[INC_US]] = add nsw i32 [[TMP5]], 1
 ; CHECK-NEXT:    [[EXITCOND2:%.*]] = icmp ne i32 [[INC_US]], 1
 ; CHECK-NEXT:    br i1 [[EXITCOND2]], label [[FOR_INC_US]], label [[FOR_COND8_FOR_COND2_LOOPEXIT_CRIT_EDGE_US:%.*]]
 ; CHECK:       for.cond2.loopexit.us:
@@ -80,7 +77,6 @@ define void @f() {
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa:
 ; CHECK-NEXT:    br label [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US]]
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa.us:
-; CHECK-NEXT:    [[COND_LCSSA_PH_US:%.*]] = phi i32 [ [[DIV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA]] ], [ [[DIV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_US_LCSSA_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND2_FOR_INC13_CRIT_EDGE:%.*]]
 ; CHECK:       for.body3.lr.ph.split:
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[FOR_BODY3_LR_PH_SPLIT_SPLIT_US:%.*]], label [[FOR_BODY3_LR_PH_SPLIT_FOR_BODY3_LR_PH_SPLIT_SPLIT_CRIT_EDGE:%.*]]
@@ -93,12 +89,12 @@ define void @f() {
 ; CHECK:       cond.false.us4:
 ; CHECK-NEXT:    br label [[COND_END_US5]]
 ; CHECK:       cond.end.us5:
-; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr @b, align 4
-; CHECK-NEXT:    [[CMP91_US7:%.*]] = icmp slt i32 [[TMP8]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[CMP91_US7:%.*]] = icmp slt i32 [[TMP6]], 1
 ; CHECK-NEXT:    br i1 [[CMP91_US7]], label [[FOR_INC_LR_PH_US12:%.*]], label [[FOR_COND2_LOOPEXIT_US11:%.*]]
 ; CHECK:       for.inc.us8:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP8]], [[FOR_INC_LR_PH_US12]] ], [ [[INC_US9:%.*]], [[FOR_INC_US8:%.*]] ]
-; CHECK-NEXT:    [[INC_US9]] = add nsw i32 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP6]], [[FOR_INC_LR_PH_US12]] ], [ [[INC_US9:%.*]], [[FOR_INC_US8:%.*]] ]
+; CHECK-NEXT:    [[INC_US9]] = add nsw i32 [[TMP7]], 1
 ; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp ne i32 [[INC_US9]], 1
 ; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_INC_US8]], label [[FOR_COND8_FOR_COND2_LOOPEXIT_CRIT_EDGE_US13:%.*]]
 ; CHECK:       for.cond2.loopexit.us11:
@@ -122,24 +118,22 @@ define void @f() {
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    br label [[COND_END]]
 ; CHECK:       cond.end:
-; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr @b, align 4
-; CHECK-NEXT:    [[CMP91:%.*]] = icmp slt i32 [[TMP10]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr @b, align 4
+; CHECK-NEXT:    [[CMP91:%.*]] = icmp slt i32 [[TMP8]], 1
 ; CHECK-NEXT:    br i1 [[CMP91]], label [[FOR_INC_LR_PH:%.*]], label [[FOR_COND2_LOOPEXIT]]
 ; CHECK:       for.inc.lr.ph:
 ; CHECK-NEXT:    br label [[FOR_INC:%.*]]
 ; CHECK:       for.inc:
-; CHECK-NEXT:    [[TMP11:%.*]] = phi i32 [ [[TMP10]], [[FOR_INC_LR_PH]] ], [ [[INC:%.*]], [[FOR_INC]] ]
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP8]], [[FOR_INC_LR_PH]] ], [ [[INC:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[TMP9]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INC]], 1
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_INC]], label [[FOR_COND8_FOR_COND2_LOOPEXIT_CRIT_EDGE:%.*]]
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa:
 ; CHECK-NEXT:    br label [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA]]
 ; CHECK:       for.cond2.for.inc13_crit_edge.us-lcssa:
-; CHECK-NEXT:    [[COND_LCSSA_PH:%.*]] = phi i32 [ [[INDVARS_IV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_LCSSA]] ], [ [[INDVARS_IV]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US_LCSSA_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND2_FOR_INC13_CRIT_EDGE]]
 ; CHECK:       for.cond2.for.inc13_crit_edge:
-; CHECK-NEXT:    [[COND_LCSSA:%.*]] = phi i32 [ [[COND_LCSSA_PH]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA]] ], [ [[COND_LCSSA_PH_US]], [[FOR_COND2_FOR_INC13_CRIT_EDGE_US_LCSSA_US]] ]
-; CHECK-NEXT:    store i32 [[COND_LCSSA]], ptr @c, align 4
+; CHECK-NEXT:    store i32 [[INDVARS_IV]], ptr @c, align 4
 ; CHECK-NEXT:    br label [[FOR_INC13]]
 ; CHECK:       for.inc13:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i32 [[INDVARS_IV]], 1
diff --git a/llvm/test/Transforms/IndVarSimplify/predicated_ranges.ll b/llvm/test/Transforms/IndVarSimplify/predicated_ranges.ll
index 208a587171a96..d005804ff633e 100644
--- a/llvm/test/Transforms/IndVarSimplify/predicated_ranges.ll
+++ b/llvm/test/Transforms/IndVarSimplify/predicated_ranges.ll
@@ -19,7 +19,7 @@ define void @test_predicated_simple_unsigned(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -68,7 +68,7 @@ define void @test_predicated_simple_signed(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -477,7 +477,7 @@ define void @test_can_predicate_simple_unsigned(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK_FIRST_ITER]], label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -528,7 +528,7 @@ define void @test_can_predicate_simple_unsigned_inverted(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK_FIRST_ITER]], label [[FAIL:%.*]], label [[BACKEDGE]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -578,7 +578,7 @@ define void @test_can_predicate_simple_signed(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i32 [[IV]], 1
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp slt i32 [[IV_NEXT]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
@@ -631,7 +631,7 @@ define void @test_can_predicate_trunc_unsigned(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i64 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK_FIRST_ITER]], label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[ARR:%.*]], i64 [[IV]]
@@ -685,7 +685,7 @@ define void @test_can_predicate_trunc_unsigned_inverted(ptr %p, ptr %arr) {
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i64 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK_FIRST_ITER]], label [[FAIL:%.*]], label [[BACKEDGE]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[ARR:%.*]], i64 [[IV]]
@@ -736,7 +736,7 @@ define void @test_can_predicate_trunc_unsigned_different_start_and_len(ptr %p, i
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i64 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i64 [[IV]], 1
 ; CHECK-NEXT:    [[NARROW:%.*]] = trunc i64 [[IV_NEXT]] to i32
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp ult i32 [[NARROW]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[FAIL:%.*]]
@@ -788,7 +788,7 @@ define void @test_can_predicate_trunc_unsigned_different_start_and_len_inverted(
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i64 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i64 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i64 [[IV]], 1
 ; CHECK-NEXT:    [[NARROW:%.*]] = trunc i64 [[IV_NEXT]] to i32
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp uge i32 [[NARROW]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[FAIL:%.*]], label [[BACKEDGE]]
@@ -842,7 +842,7 @@ define void @test_can_predicate_simple_unsigned_different_start(i32 %start, ptr
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK_FIRST_ITER]], label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -893,7 +893,7 @@ define void @test_can_predicate_simple_unsigned_inverted_different_start(i32 %st
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK_FIRST_ITER]], label [[FAIL:%.*]], label [[BACKEDGE]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -943,7 +943,7 @@ define void @test_can_predicate_simple_signed_different_start(i32 %start, ptr %p
 ; CHECK-NEXT:    [[ZERO_COND:%.*]] = icmp eq i32 [[IV]], 0
 ; CHECK-NEXT:    br i1 [[ZERO_COND]], label [[EXIT:%.*]], label [[RANGE_CHECK_BLOCK:%.*]]
 ; CHECK:       range_check_block:
-; CHECK-NEXT:    [[IV_NEXT]] = sub i32 [[IV]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = sub nuw i32 [[IV]], 1
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp slt i32 [[IV_NEXT]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[FAIL:%.*]]
 ; CHECK:       backedge:
diff --git a/llvm/test/Transforms/IndVarSimplify/ptrtoint-global-cmp.ll b/llvm/test/Transforms/IndVarSimplify/ptrtoint-global-cmp.ll
new file mode 100644
index 0000000000000..e06d83753fe29
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/ptrtoint-global-cmp.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=indvars -S %s | FileCheck %s
+
+@a = global [4 x i32] zeroinitializer, align 16
+
+define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[RED_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i8 [[IV]] to i64
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr @a, i64 [[IDXPROM]]
+; CHECK-NEXT:    br i1 false, label [[LOOP_LATCH]], label [[EXIT_1:%.*]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[RED_NEXT]] = add nsw i32 [[L]], [[RED]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; CHECK-NEXT:    br i1 true, label [[LOOP_HEADER]], label [[EXIT_2:%.*]]
+; CHECK:       exit.1:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       exit.2:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %red = phi i32 [ 0, %entry ], [ %red.next, %loop.latch ]
+  %idxprom = zext i8 %iv to i64
+  %gep = getelementptr i32, ptr @a, i64 %idxprom
+  %f = icmp ult ptr %gep, getelementptr inbounds ([4 x i32], ptr @a, i64 1, i64 0)
+  br i1 %f, label %loop.latch, label %exit.1
+
+loop.latch:
+  %l = load i32, ptr %gep, align 4
+  %red.next = add nsw i32 %l, %red
+  %iv.next = add nuw nsw i8 %iv, 1
+  %cmp = icmp ult i8 %iv, 4
+  br i1 %cmp, label %loop.header, label %exit.2
+
+exit.1:                                             ; preds = %for.body
+  ret i32 0
+
+exit.2:
+  %add.lcssa = phi i32 [ %red.next, %loop.latch ]
+  ret i32 %add.lcssa
+}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll
index a772f446e5d6d..a73cd517fd49d 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/noop-ptrint-pair.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -o - -passes=infer-address-spaces %s | FileCheck -check-prefixes=COMMON,AMDGCN %s
 ; RUN: opt -S -o - -passes=infer-address-spaces -assume-default-is-flat-addrspace %s | FileCheck -check-prefixes=COMMON,NOTTI %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 
 ; COMMON-LABEL: @noop_ptrint_pair(
 ; AMDGCN-NEXT: store i32 0, ptr addrspace(1) %{{.*}}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/ptrmask.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/ptrmask.ll
index 1925dd2a04398..c9db0656c6b7d 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/ptrmask.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/ptrmask.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=infer-address-spaces,instsimplify %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define i8 @ptrmask_cast_local_to_flat(ptr addrspace(3) %src.ptr, i64 %mask) {
 ; CHECK-LABEL: @ptrmask_cast_local_to_flat(
diff --git a/llvm/test/Transforms/InferAddressSpaces/X86/noop-ptrint-pair.ll b/llvm/test/Transforms/InferAddressSpaces/X86/noop-ptrint-pair.ll
index ff050744f8df3..0eaf6e32e5a94 100644
--- a/llvm/test/Transforms/InferAddressSpaces/X86/noop-ptrint-pair.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/X86/noop-ptrint-pair.ll
@@ -2,7 +2,7 @@
 
 ; Check that assert in X86TargetMachine::isNoopAddrSpaceCast is not triggered.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 
 ; CHECK-LABEL: @noop_ptrint_pair(
 ; CHECK: addrspacecast ptr addrspace(1) %x to ptr addrspace(4)
diff --git a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-alloca-argument.ll b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-alloca-argument.ll
index e2dc0ab3ee0fc..6f4b3274cdfd1 100644
--- a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-alloca-argument.ll
+++ b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-alloca-argument.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn--amdhsa -S -passes=inline -inline-threshold=0 < %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define void @use_flat_ptr_arg(ptr nocapture %p) {
 entry:
diff --git a/llvm/test/Transforms/Inline/AMDGPU/inline-denormal-fp-math.ll b/llvm/test/Transforms/Inline/AMDGPU/inline-denormal-fp-math.ll
index ed63486edb01f..4c802ef57267e 100644
--- a/llvm/test/Transforms/Inline/AMDGPU/inline-denormal-fp-math.ll
+++ b/llvm/test/Transforms/Inline/AMDGPU/inline-denormal-fp-math.ll
@@ -37,9 +37,45 @@ define i32 @func_ieee_psz() #3 {
   ret i32 4
 }
 
+define i32 @func_dynamic_dynamic() #4 {
+; CHECK-LABEL: @func_dynamic_dynamic(
+; CHECK-NEXT:    ret i32 5
+;
+  ret i32 5
+}
+
+define i32 @func_dynamic_ieee() #5 {
+; CHECK-LABEL: @func_dynamic_ieee(
+; CHECK-NEXT:    ret i32 6
+;
+  ret i32 6
+}
+
+define i32 @func_ieee_dynamic() #6 {
+; CHECK-LABEL: @func_ieee_dynamic(
+; CHECK-NEXT:    ret i32 7
+;
+  ret i32 7
+}
+
+define i32 @func_psz_dynamic() #7 {
+; CHECK-LABEL: @func_psz_dynamic(
+; CHECK-NEXT:    ret i32 8
+;
+  ret i32 8
+}
+
+define i32 @func_dynamic_psz() #8 {
+; CHECK-LABEL: @func_dynamic_psz(
+; CHECK-NEXT:    ret i32 9
+;
+  ret i32 9
+}
+
 define i32 @call_default_from_psz_psz() #1 {
 ; CHECK-LABEL: @call_default_from_psz_psz(
-; CHECK-NEXT:    ret i32 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_default()
+; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @func_default()
   ret i32 %call
@@ -55,7 +91,8 @@ define i32 @call_ieee_ieee_from_ieee_ieee() #0 {
 
 define i32 @call_ieee_ieee_from_psz_psz() #1 {
 ; CHECK-LABEL: @call_ieee_ieee_from_psz_psz(
-; CHECK-NEXT:    ret i32 1
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @func_ieee_ieee()
   ret i32 %call
@@ -80,7 +117,8 @@ define i32 @call_psz_psz_from_psz_psz() #1 {
 
 define i32 @call_psz_ieee_from_psz_psz() #1 {
 ; CHECK-LABEL: @call_psz_ieee_from_psz_psz(
-; CHECK-NEXT:    ret i32 3
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @func_psz_ieee()
   ret i32 %call
@@ -88,7 +126,8 @@ define i32 @call_psz_ieee_from_psz_psz() #1 {
 
 define i32 @call_ieee_psz_from_psz_psz() #1 {
 ; CHECK-LABEL: @call_ieee_psz_from_psz_psz(
-; CHECK-NEXT:    ret i32 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @func_ieee_psz()
   ret i32 %call
@@ -114,7 +153,8 @@ define i32 @call_ieee_psz_from_ieee_ieee() #0 {
 
 define i32 @call_ieee_ieee_from_psz_ieee() #2 {
 ; CHECK-LABEL: @call_ieee_ieee_from_psz_ieee(
-; CHECK-NEXT:    ret i32 1
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @func_ieee_ieee()
   ret i32 %call
@@ -148,7 +188,8 @@ define i32 @call_psz_psz_from_psz_ieee() #2 {
 
 define i32 @call_ieee_ieee_from_ieee_psz() #3 {
 ; CHECK-LABEL: @call_ieee_ieee_from_ieee_psz(
-; CHECK-NEXT:    ret i32 1
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @func_ieee_ieee()
   ret i32 %call
@@ -180,7 +221,614 @@ define i32 @call_psz_psz_from_ieee_psz() #3 {
   ret i32 %call
 }
 
+define i32 @call_dynamic_dynamic_from_ieee_ieee() #0 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_ieee_ieee(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_ieee_ieee() #0 {
+; CHECK-LABEL: @call_dynamic_ieee_from_ieee_ieee(
+; CHECK-NEXT:    ret i32 6
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_ieee_ieee() #0 {
+; CHECK-LABEL: @call_ieee_dynamic_from_ieee_ieee(
+; CHECK-NEXT:    ret i32 7
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_ieee_ieee() #0 {
+; CHECK-LABEL: @call_dynamic_psz_from_ieee_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_ieee_ieee() #0 {
+; CHECK-LABEL: @call_psz_dynamic_from_ieee_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_psz_psz() #1 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_psz_psz(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_psz_psz() #1 {
+; CHECK-LABEL: @call_ieee_dynamic_from_psz_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_psz_psz() #1 {
+; CHECK-LABEL: @call_dynamic_ieee_from_psz_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_psz_psz() #1 {
+; CHECK-LABEL: @call_psz_dynamic_from_psz_psz(
+; CHECK-NEXT:    ret i32 8
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_psz_psz() #1 {
+; CHECK-LABEL: @call_dynamic_psz_from_psz_psz(
+; CHECK-NEXT:    ret i32 9
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_psz_ieee() #2 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_psz_ieee(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_psz_ieee() #2 {
+; CHECK-LABEL: @call_dynamic_ieee_from_psz_ieee(
+; CHECK-NEXT:    ret i32 6
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_psz_ieee() #2 {
+; CHECK-LABEL: @call_ieee_dynamic_from_psz_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_psz_ieee() #2 {
+; CHECK-LABEL: @call_dynamic_psz_from_psz_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_psz_ieee() #2 {
+; CHECK-LABEL: @call_psz_dynamic_from_psz_ieee(
+; CHECK-NEXT:    ret i32 8
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_ieee_psz() #3 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_ieee_psz(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_ieee_psz() #3 {
+; CHECK-LABEL: @call_ieee_dynamic_from_ieee_psz(
+; CHECK-NEXT:    ret i32 7
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_ieee_psz() #3 {
+; CHECK-LABEL: @call_dynamic_ieee_from_ieee_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_ieee_psz() #3 {
+; CHECK-LABEL: @call_dynamic_psz_from_ieee_psz(
+; CHECK-NEXT:    ret i32 9
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_ieee_psz() #3 {
+; CHECK-LABEL: @call_psz_dynamic_from_ieee_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_dynamic_dynamic(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_ieee_ieee_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_ieee_ieee_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_ieee_dynamic_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_dynamic_ieee_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_psz_dynamic_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_dynamic_psz_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_psz_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_psz_psz_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_ieee_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_psz_ieee_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_psz_from_dynamic_dynamic() #4 {
+; CHECK-LABEL: @call_ieee_psz_from_dynamic_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_psz()
+  ret i32 %call
+}
+
+define i32 @call_ieee_ieee_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_ieee_ieee_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_ieee_dynamic_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_dynamic_ieee_from_dynamic_ieee(
+; CHECK-NEXT:    ret i32 6
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_psz_dynamic_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_dynamic_psz_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_psz_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_psz_psz_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_ieee_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_psz_ieee_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_psz_from_dynamic_ieee() #5 {
+; CHECK-LABEL: @call_ieee_psz_from_dynamic_ieee(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_psz()
+  ret i32 %call
+}
+
+define i32 @call_ieee_ieee_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_ieee_ieee_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_ieee_dynamic_from_ieee_dynamic(
+; CHECK-NEXT:    ret i32 7
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_dynamic_ieee_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_ieee_dynamic(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_dynamic_psz_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_psz_dynamic_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_psz_psz_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_psz_psz_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_ieee_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_psz_ieee_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_psz_from_ieee_dynamic() #6 {
+; CHECK-LABEL: @call_ieee_psz_from_ieee_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_psz()
+  ret i32 %call
+}
+
+define i32 @call_ieee_ieee_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_ieee_ieee_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_ieee_dynamic_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_dynamic_ieee_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_dynamic_psz_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_psz_dynamic_from_psz_dynamic(
+; CHECK-NEXT:    ret i32 8
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_psz_dynamic(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_psz_ieee_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_psz_ieee_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_psz_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_ieee_psz_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_psz_from_psz_dynamic() #7 {
+; CHECK-LABEL: @call_psz_psz_from_psz_dynamic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_psz()
+  ret i32 %call
+}
+
+define i32 @call_ieee_ieee_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_ieee_ieee_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_psz_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_ieee_psz_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_ieee_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_psz_ieee_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_ieee()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_dynamic_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_dynamic_psz(
+; CHECK-NEXT:    ret i32 5
+;
+  %call = call i32 @func_dynamic_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_psz_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_dynamic_psz_from_dynamic_psz(
+; CHECK-NEXT:    ret i32 9
+;
+  %call = call i32 @func_dynamic_psz()
+  ret i32 %call
+}
+
+define i32 @call_psz_dynamic_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_psz_dynamic_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_dynamic_ieee_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_dynamic_ieee_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_dynamic_ieee()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_dynamic_ieee()
+  ret i32 %call
+}
+
+define i32 @call_ieee_dynamic_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_ieee_dynamic_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_ieee_dynamic()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_ieee_dynamic()
+  ret i32 %call
+}
+
+define i32 @call_psz_psz_from_dynamic_psz() #8 {
+; CHECK-LABEL: @call_psz_psz_from_dynamic_psz(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @func_psz_psz()
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @func_psz_psz()
+  ret i32 %call
+}
+
+; --------------------------------------------------------------------
+; denormal-fp-math-f32
+; --------------------------------------------------------------------
+
+define i32 @func_dynamic_dynamic_f32() #9 {
+; CHECK-LABEL: @func_dynamic_dynamic_f32(
+; CHECK-NEXT:    ret i32 10
+;
+  ret i32 10
+}
+
+define i32 @func_psz_psz_f32() #10 {
+; CHECK-LABEL: @func_psz_psz_f32(
+; CHECK-NEXT:    ret i32 11
+;
+  ret i32 11
+}
+
+define i32 @call_dynamic_dynamic_from_psz_psz_f32() #10 {
+; CHECK-LABEL: @call_dynamic_dynamic_from_psz_psz_f32(
+; CHECK-NEXT:    ret i32 10
+;
+  %result = call i32 @func_dynamic_dynamic_f32()
+  ret i32 %result
+}
+
+define i32 @call_psz_psz_from_psz_psz_f32() #10 {
+; CHECK-LABEL: @call_psz_psz_from_psz_psz_f32(
+; CHECK-NEXT:    ret i32 10
+;
+  %result = call i32 @func_dynamic_dynamic_f32()
+  ret i32 %result
+}
+
+define i32 @call_psz_psz_from_ieee_ieee_f32() #11 {
+; CHECK-LABEL: @call_psz_psz_from_ieee_ieee_f32(
+; CHECK-NEXT:    [[RESULT:%.*]] = call i32 @func_psz_psz_f32()
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %result = call i32 @func_psz_psz_f32()
+  ret i32 %result
+}
+
 attributes #0 = { "denormal-fp-math"="ieee,ieee" }
 attributes #1 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
 attributes #2 = { "denormal-fp-math"="preserve-sign,ieee" }
 attributes #3 = { "denormal-fp-math"="ieee,preserve-sign" }
+attributes #4 = { "denormal-fp-math"="dynamic,dynamic" }
+attributes #5 = { "denormal-fp-math"="dynamic,ieee" }
+attributes #6 = { "denormal-fp-math"="ieee,dynamic" }
+attributes #7 = { "denormal-fp-math"="preserve-sign,dynamic" }
+attributes #8 = { "denormal-fp-math"="dynamic,preserve-sign" }
+attributes #9 = { "denormal-fp-math-f32"="dynamic,dynamic" }
+attributes #10 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+attributes #11 = { "denormal-fp-math-f32"="ieee,ieee" "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll b/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll
index 3edde213c0c08..394f48ae6b60b 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/demandelts.ll
@@ -4,7 +4,7 @@
 define <2 x float> @fcvtxn(<2 x double> %d1) {
 ; CHECK-LABEL: @fcvtxn(
 ; CHECK-NEXT:    [[I:%.*]] = call <2 x float> @llvm.aarch64.neon.fcvtxn.v2f32.v2f64(<2 x double> [[D1:%.*]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x float> [[I]], <2 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x float> [[I]], <2 x float> undef, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    ret <2 x float> [[S]]
 ;
   %a = shufflevector <2 x double> %d1, <2 x double> undef, <2 x i32> <i32 0, i32 0>
@@ -15,9 +15,9 @@ define <2 x float> @fcvtxn(<2 x double> %d1) {
 
 define <4 x i16> @rshrn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @rshrn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.rshrn.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -28,9 +28,9 @@ define <4 x i16> @rshrn(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @sqrshrn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @sqrshrn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqrshrn.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -41,9 +41,9 @@ define <4 x i16> @sqrshrn(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @sqrshrun(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @sqrshrun(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqrshrun.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -54,9 +54,9 @@ define <4 x i16> @sqrshrun(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @sqshrn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @sqshrn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqshrn.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -67,9 +67,9 @@ define <4 x i16> @sqshrn(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @sqshrun(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @sqshrun(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqshrun.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -80,9 +80,9 @@ define <4 x i16> @sqshrun(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @sqxtn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @sqxtn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> [[A]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -93,9 +93,9 @@ define <4 x i16> @sqxtn(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @sqxtun(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @sqxtun(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> [[A]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -106,9 +106,9 @@ define <4 x i16> @sqxtun(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @uqrshrn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @uqrshrn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqrshrn.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -119,9 +119,9 @@ define <4 x i16> @uqrshrn(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @uqshrn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @uqshrn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqshrn.v4i16(<4 x i32> [[A]], i32 9)
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -132,9 +132,9 @@ define <4 x i16> @uqshrn(<2 x i32> %d1, <2 x i32> %d2) {
 
 define <4 x i16> @uqxtn(<2 x i32> %d1, <2 x i32> %d2) {
 ; CHECK-LABEL: @uqxtn(
-; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A:%.*]] = shufflevector <2 x i32> [[D1:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I:%.*]] = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> [[A]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i16> [[I]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[S]]
 ;
   %a = shufflevector <2 x i32> %d1, <2 x i32> %d2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
index 3bec481054b96..609dac52038ee 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
@@ -211,7 +211,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i3
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
 ; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 undef, i32 1>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 poison, i32 1>
 ; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
 ; CHECK-NEXT:    ret <2 x float> [[RET]]
 ;
@@ -229,7 +229,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i3
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
 ; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
 ; CHECK-NEXT:    ret <2 x float> [[RET]]
 ;
@@ -376,7 +376,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v4f32(<4 x i3
 define double @extract01_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
 ; CHECK-LABEL: @extract01_bitcast_buffer_load_format_v4f32(
 ; CHECK-NEXT:    [[VAR:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[VAR]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[VAR]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VAR1:%.*]] = bitcast <4 x float> [[TMP1]] to <2 x double>
 ; CHECK-NEXT:    [[VAR2:%.*]] = extractelement <2 x double> [[VAR1]], i64 0
 ; CHECK-NEXT:    ret double [[VAR2]]
@@ -3081,7 +3081,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
   %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
@@ -3092,7 +3092,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
   %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
@@ -3291,7 +3291,7 @@ define amdgpu_ps half @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(float %d
 define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32(i32 7, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <3 x half> [[DATA]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <3 x half> [[DATA]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
   %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
@@ -3302,7 +3302,7 @@ define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32
 define amdgpu_ps <4 x half> @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32(i32 3, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x half> [[DATA]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x half> [[DATA]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
   %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index ddf3bc9423e39..d892819fcfd3c 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -211,7 +211,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i3
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
 ; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 undef, i32 1>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 poison, i32 1>
 ; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
 ; CHECK-NEXT:    ret <2 x float> [[RET]]
 ;
@@ -229,7 +229,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i3
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> [[RSRC:%.*]], i32 [[IDX:%.*]], i32 [[OFS:%.*]], i1 false, i1 false)
 ; CHECK-NEXT:    [[INS1:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> [[DATA]], <3 x float> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[INS1]], [[SHUF]]
 ; CHECK-NEXT:    ret <2 x float> [[RET]]
 ;
@@ -376,7 +376,7 @@ define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v4f32(<4 x i3
 define double @extract01_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
 ; CHECK-LABEL: @extract01_bitcast_buffer_load_format_v4f32(
 ; CHECK-NEXT:    [[VAR:%.*]] = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> undef, i32 [[ARG:%.*]], i32 16, i1 false, i1 false)
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[VAR]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[VAR]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VAR1:%.*]] = bitcast <4 x float> [[TMP1]] to <2 x double>
 ; CHECK-NEXT:    [[VAR2:%.*]] = extractelement <2 x double> [[VAR1]], i64 0
 ; CHECK-NEXT:    ret double [[VAR2]]
@@ -3081,7 +3081,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
   %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
@@ -3092,7 +3092,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
   %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
@@ -3291,7 +3291,7 @@ define amdgpu_ps half @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(float %d
 define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32(i32 7, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <3 x half> [[DATA]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <3 x half> [[DATA]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
   %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
@@ -3302,7 +3302,7 @@ define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32
 define amdgpu_ps <4 x half> @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(
 ; CHECK-NEXT:    [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32(i32 3, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
-; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x half> [[DATA]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x half> [[DATA]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
   %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
index 89979bcedf814..72bffe62fbb14 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
@@ -83,4 +83,28 @@ define float @test_finite(i32 %x, i32 %y, float %z) {
   ret float %call
 }
 
+; Combine to fma because neither argument can be infinity or NaN based on assumptions
+define float @test_finite_assumed(float %x, float %y, float %z) {
+; CHECK-LABEL: @test_finite_assumed(
+; CHECK-NEXT:    [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[IS_FINITE_X:%.*]] = fcmp one float [[FABS_X]], 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS_Y:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], 0x7FF0000000000000
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_FINITE_X]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_FINITE_Y]])
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %is.finite.x = fcmp one float %fabs.x, 0x7FF0000000000000
+  %fabs.y = call float @llvm.fabs.f32(float %y)
+  %is.finite.y = fcmp one float %fabs.y, 0x7FF0000000000000
+  call void @llvm.assume(i1 %is.finite.x)
+  call void @llvm.assume(i1 %is.finite.y)
+  %call = call float @llvm.amdgcn.fma.legacy(float %x, float %y, float %z)
+  ret float %call
+}
+
 declare float @llvm.amdgcn.fma.legacy(float, float, float)
+declare float @llvm.fabs.f32(float)
+declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
index 439bedbe60dd8..d58470d6b12ad 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
@@ -53,4 +53,28 @@ define float @test_finite(i32 %x, i32 %y) {
   ret float %call
 }
 
+; Combine to fmul because neither argument can be infinity or NaN based on assumptions
+define float @test_finite_assumed(float %x, float %y) {
+; CHECK-LABEL: @test_finite_assumed(
+; CHECK-NEXT:    [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[IS_FINITE_X:%.*]] = fcmp one float [[FABS_X]], 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS_Y:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], 0x7FF0000000000000
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_FINITE_X]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_FINITE_Y]])
+; CHECK-NEXT:    [[CALL:%.*]] = fmul float [[X]], [[Y]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+  %fabs.x = call float @llvm.fabs.f32(float %x)
+  %is.finite.x = fcmp one float %fabs.x, 0x7FF0000000000000
+  %fabs.y = call float @llvm.fabs.f32(float %y)
+  %is.finite.y = fcmp one float %fabs.y, 0x7FF0000000000000
+  call void @llvm.assume(i1 %is.finite.x)
+  call void @llvm.assume(i1 %is.finite.y)
+  %call = call float @llvm.amdgcn.fmul.legacy(float %x, float %y)
+  ret float %call
+}
+
 declare float @llvm.amdgcn.fmul.legacy(float, float)
+declare float @llvm.fabs.f32(float)
+declare void @llvm.assume(i1 noundef)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll b/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
index 2d2a93b28a5ba..3ea58c90f4934 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/memcpy-from-constant.ll
@@ -4,7 +4,7 @@
 ; Make sure the optimization from memcpy-from-global.ll happens, but
 ; the constant source is not a global variable.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Simple memcpy to alloca from constant address space argument.
 define i8 @memcpy_constant_arg_ptr_to_alloca(ptr addrspace(4) noalias readonly align 4 dereferenceable(32) %arg, i32 %idx) {
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
index 07465a041e2b2..ad4088c63e761 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
@@ -13,7 +13,7 @@ declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8 immarg
 
 define double @elts_addsub_v2f64(<2 x double> %0, <2 x double> %1) {
 ; CHECK-LABEL: @elts_addsub_v2f64(
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
 ; CHECK-NEXT:    ret double [[TMP5]]
@@ -181,7 +181,7 @@ define double @PR48476_fsub(<2 x double> %x) {
 define double @PR48476_fadd_fsub(<2 x double> %x) {
 ; CHECK-LABEL: @PR48476_fadd_fsub(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], <double poison, double 0.000000e+00>
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> [[S]], [[X]]
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
 ; CHECK-NEXT:    ret double [[VECEXT]]
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
index c54f3f27d4a4e..3ade1ba3cf78b 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
@@ -13,7 +13,7 @@ declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8 immarg
 
 define double @elts_addsub_v2f64(<2 x double> %0, <2 x double> %1) {
 ; CHECK-LABEL: @elts_addsub_v2f64(
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
 ; CHECK-NEXT:    ret double [[TMP5]]
@@ -181,7 +181,7 @@ define double @PR48476_fsub(<2 x double> %x) {
 define double @PR48476_fadd_fsub(<2 x double> %x) {
 ; CHECK-LABEL: @PR48476_fadd_fsub(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], <double poison, double 0.000000e+00>
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> [[S]], [[X]]
 ; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
 ; CHECK-NEXT:    ret double [[VECEXT]]
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2-inseltpoison.ll
index aa94bb8d3f548..5be43f8d164ac 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx2-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2-inseltpoison.ll
@@ -65,7 +65,7 @@ define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: @undef_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -74,7 +74,7 @@ define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
 
 define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
 ; CHECK-LABEL: @undef_test_vpermps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -85,7 +85,7 @@ define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) {
 ; CHECK-LABEL: @elts_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = insertelement <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i32 %a1, i32 0
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
index 0d1d3150b509f..23bcb4816af8b 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
@@ -65,7 +65,7 @@ define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
 ; CHECK-LABEL: @undef_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -74,7 +74,7 @@ define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
 
 define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
 ; CHECK-LABEL: @undef_test_vpermps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -85,7 +85,7 @@ define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
 
 define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) {
 ; CHECK-LABEL: @elts_test_vpermd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = insertelement <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i32 %a1, i32 0
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
index dddde60d88125..b27c94667d56d 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
@@ -1917,7 +1917,7 @@ define <8 x i32> @shuffle_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %pas
 define <8 x i32> @undef_test_permvar_si_256(<8 x i32> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -1927,7 +1927,7 @@ define <8 x i32> @undef_test_permvar_si_256(<8 x i32> %a0) {
 define <8 x i32> @undef_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
@@ -2013,7 +2013,7 @@ define <8 x float> @shuffle_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float
 define <8 x float> @undef_test_permvar_sf_256(<8 x float> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -2023,7 +2023,7 @@ define <8 x float> @undef_test_permvar_sf_256(<8 x float> %a0) {
 define <8 x float> @undef_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x float> [[TMP3]]
@@ -2115,7 +2115,7 @@ define <4 x i64> @shuffle_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %pas
 define <4 x i64> @undef_test_permvar_di_256(<4 x i64> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
@@ -2125,7 +2125,7 @@ define <4 x i64> @undef_test_permvar_di_256(<4 x i64> %a0) {
 define <4 x i64> @undef_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
@@ -2219,7 +2219,7 @@ define <4 x double> @shuffle_test_permvar_df_256_mask(<4 x double> %a0, <4 x dou
 define <4 x double> @undef_test_permvar_df_256(<4 x double> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
@@ -2229,7 +2229,7 @@ define <4 x double> @undef_test_permvar_df_256(<4 x double> %a0) {
 define <4 x double> @undef_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
@@ -2317,7 +2317,7 @@ define <16 x i32> @shuffle_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %
 define <16 x i32> @undef_test_permvar_si_512(<16 x i32> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -2327,7 +2327,7 @@ define <16 x i32> @undef_test_permvar_si_512(<16 x i32> %a0) {
 define <16 x i32> @undef_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
@@ -2413,7 +2413,7 @@ define <16 x float> @shuffle_test_permvar_sf_512_mask(<16 x float> %a0, <16 x fl
 define <16 x float> @undef_test_permvar_sf_512(<16 x float> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -2423,7 +2423,7 @@ define <16 x float> @undef_test_permvar_sf_512(<16 x float> %a0) {
 define <16 x float> @undef_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x float> [[TMP3]]
@@ -2509,7 +2509,7 @@ define <8 x i64> @shuffle_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %pas
 define <8 x i64> @undef_test_permvar_di_512(<8 x i64> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
@@ -2519,7 +2519,7 @@ define <8 x i64> @undef_test_permvar_di_512(<8 x i64> %a0) {
 define <8 x i64> @undef_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
@@ -2605,7 +2605,7 @@ define <8 x double> @shuffle_test_permvar_df_512_mask(<8 x double> %a0, <8 x dou
 define <8 x double> @undef_test_permvar_df_512(<8 x double> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
@@ -2615,7 +2615,7 @@ define <8 x double> @undef_test_permvar_df_512(<8 x double> %a0) {
 define <8 x double> @undef_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x double> [[TMP3]]
@@ -2701,7 +2701,7 @@ define <8 x i16> @shuffle_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %pas
 define <8 x i16> @undef_test_permvar_hi_128(<8 x i16> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 undef, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2711,7 +2711,7 @@ define <8 x i16> @undef_test_permvar_hi_128(<8 x i16> %a0) {
 define <8 x i16> @undef_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_128_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
@@ -2797,7 +2797,7 @@ define <16 x i16> @shuffle_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %
 define <16 x i16> @undef_test_permvar_hi_256(<16 x i16> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 undef, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2807,7 +2807,7 @@ define <16 x i16> @undef_test_permvar_hi_256(<16 x i16> %a0) {
 define <16 x i16> @undef_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
@@ -2893,7 +2893,7 @@ define <32 x i16> @shuffle_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %
 define <32 x i16> @undef_test_permvar_hi_512(<32 x i16> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 undef, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2903,7 +2903,7 @@ define <32 x i16> @undef_test_permvar_hi_512(<32 x i16> %a0) {
 define <32 x i16> @undef_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
@@ -2989,7 +2989,7 @@ define <16 x i8> @shuffle_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %pas
 define <16 x i8> @undef_test_permvar_qi_128(<16 x i8> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
@@ -2999,7 +2999,7 @@ define <16 x i8> @undef_test_permvar_qi_128(<16 x i8> %a0) {
 define <16 x i8> @undef_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_128_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x i8> [[TMP3]]
@@ -3085,7 +3085,7 @@ define <32 x i8> @shuffle_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %pas
 define <32 x i8> @undef_test_permvar_qi_256(<32 x i8> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 undef, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
@@ -3095,7 +3095,7 @@ define <32 x i8> @undef_test_permvar_qi_256(<32 x i8> %a0) {
 define <32 x i8> @undef_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <32 x i8> [[TMP3]]
@@ -3181,7 +3181,7 @@ define <64 x i8> @shuffle_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %pas
 define <64 x i8> @undef_test_permvar_qi_512(<64 x i8> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 poison, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 undef, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
@@ -3191,7 +3191,7 @@ define <64 x i8> @undef_test_permvar_qi_512(<64 x i8> %a0) {
 define <64 x i8> @undef_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 poison, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
index 845a25aba4434..ea806a89679c5 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
@@ -1917,7 +1917,7 @@ define <8 x i32> @shuffle_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %pas
 define <8 x i32> @undef_test_permvar_si_256(<8 x i32> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
   %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -1927,7 +1927,7 @@ define <8 x i32> @undef_test_permvar_si_256(<8 x i32> %a0) {
 define <8 x i32> @undef_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
@@ -2013,7 +2013,7 @@ define <8 x float> @shuffle_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float
 define <8 x float> @undef_test_permvar_sf_256(<8 x float> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -2023,7 +2023,7 @@ define <8 x float> @undef_test_permvar_sf_256(<8 x float> %a0) {
 define <8 x float> @undef_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x float> [[TMP3]]
@@ -2115,7 +2115,7 @@ define <4 x i64> @shuffle_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %pas
 define <4 x i64> @undef_test_permvar_di_256(<4 x i64> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
@@ -2125,7 +2125,7 @@ define <4 x i64> @undef_test_permvar_di_256(<4 x i64> %a0) {
 define <4 x i64> @undef_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
@@ -2219,7 +2219,7 @@ define <4 x double> @shuffle_test_permvar_df_256_mask(<4 x double> %a0, <4 x dou
 define <4 x double> @undef_test_permvar_df_256(<4 x double> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
@@ -2229,7 +2229,7 @@ define <4 x double> @undef_test_permvar_df_256(<4 x double> %a0) {
 define <4 x double> @undef_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
@@ -2317,7 +2317,7 @@ define <16 x i32> @shuffle_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %
 define <16 x i32> @undef_test_permvar_si_512(<16 x i32> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
 ;
   %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -2327,7 +2327,7 @@ define <16 x i32> @undef_test_permvar_si_512(<16 x i32> %a0) {
 define <16 x i32> @undef_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_si_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
@@ -2413,7 +2413,7 @@ define <16 x float> @shuffle_test_permvar_sf_512_mask(<16 x float> %a0, <16 x fl
 define <16 x float> @undef_test_permvar_sf_512(<16 x float> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
@@ -2423,7 +2423,7 @@ define <16 x float> @undef_test_permvar_sf_512(<16 x float> %a0) {
 define <16 x float> @undef_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_sf_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x float> [[TMP3]]
@@ -2509,7 +2509,7 @@ define <8 x i64> @shuffle_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %pas
 define <8 x i64> @undef_test_permvar_di_512(<8 x i64> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
 ;
   %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
@@ -2519,7 +2519,7 @@ define <8 x i64> @undef_test_permvar_di_512(<8 x i64> %a0) {
 define <8 x i64> @undef_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_di_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
@@ -2605,7 +2605,7 @@ define <8 x double> @shuffle_test_permvar_df_512_mask(<8 x double> %a0, <8 x dou
 define <8 x double> @undef_test_permvar_df_512(<8 x double> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
@@ -2615,7 +2615,7 @@ define <8 x double> @undef_test_permvar_df_512(<8 x double> %a0) {
 define <8 x double> @undef_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_df_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x double> [[TMP3]]
@@ -2701,7 +2701,7 @@ define <8 x i16> @shuffle_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %pas
 define <8 x i16> @undef_test_permvar_hi_128(<8 x i16> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 ;
   %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 undef, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2711,7 +2711,7 @@ define <8 x i16> @undef_test_permvar_hi_128(<8 x i16> %a0) {
 define <8 x i16> @undef_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_128_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> poison, <8 x i32> <i32 poison, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
@@ -2797,7 +2797,7 @@ define <16 x i16> @shuffle_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %
 define <16 x i16> @undef_test_permvar_hi_256(<16 x i16> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
 ;
   %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 undef, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2807,7 +2807,7 @@ define <16 x i16> @undef_test_permvar_hi_256(<16 x i16> %a0) {
 define <16 x i16> @undef_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
@@ -2893,7 +2893,7 @@ define <32 x i16> @shuffle_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %
 define <32 x i16> @undef_test_permvar_hi_512(<32 x i16> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
 ;
   %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 undef, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
@@ -2903,7 +2903,7 @@ define <32 x i16> @undef_test_permvar_hi_512(<32 x i16> %a0) {
 define <32 x i16> @undef_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_hi_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
@@ -2989,7 +2989,7 @@ define <16 x i8> @shuffle_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %pas
 define <16 x i8> @undef_test_permvar_qi_128(<16 x i8> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
@@ -2999,7 +2999,7 @@ define <16 x i8> @undef_test_permvar_qi_128(<16 x i8> %a0) {
 define <16 x i8> @undef_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_128_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <16 x i8> [[TMP3]]
@@ -3085,7 +3085,7 @@ define <32 x i8> @shuffle_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %pas
 define <32 x i8> @undef_test_permvar_qi_256(<32 x i8> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 undef, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
@@ -3095,7 +3095,7 @@ define <32 x i8> @undef_test_permvar_qi_256(<32 x i8> %a0) {
 define <32 x i8> @undef_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_256_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> poison, <32 x i32> <i32 poison, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <32 x i8> [[TMP3]]
@@ -3181,7 +3181,7 @@ define <64 x i8> @shuffle_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %pas
 define <64 x i8> @undef_test_permvar_qi_512(<64 x i8> %a0) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 poison, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 undef, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
@@ -3191,7 +3191,7 @@ define <64 x i8> @undef_test_permvar_qi_512(<64 x i8> %a0) {
 define <64 x i8> @undef_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
 ;
 ; CHECK-LABEL: @undef_test_permvar_qi_512_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> poison, <64 x i32> <i32 poison, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
 ; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-muldq-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-muldq-inseltpoison.ll
index bc998669c9604..80d3c42341f82 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-muldq-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-muldq-inseltpoison.ll
@@ -159,8 +159,8 @@ define <8 x i64> @fold_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) {
 
 define <2 x i64> @test_demanded_elts_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @test_demanded_elts_pmuludq_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i64> [[TMP3]], <i64 4294967295, i64 poison>
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll
index 9a2676645076f..d32bce2193c2a 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll
@@ -159,8 +159,8 @@ define <8 x i64> @fold_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) {
 
 define <2 x i64> @test_demanded_elts_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @test_demanded_elts_pmuludq_128(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i64> [[TMP3]], <i64 4294967295, i64 poison>
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll
index 9c17b9a159cb6..c84839b7ee9b8 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll
@@ -208,7 +208,7 @@ define <64 x i8> @fold_packuswb_512() {
 define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_128(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> undef)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
   %1 = shufflevector <4 x i32> %a0, <4 x i32> poison, <4 x i32> <i32 3, i32 1, i32 poison, i32 poison>
@@ -221,7 +221,7 @@ define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 define <8 x i16> @elts_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_128(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
   %1 = insertelement <4 x i32> %a0, i32 0, i32 0
@@ -256,7 +256,7 @@ define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP2]]
 ;
   %1 = shufflevector <8 x i32> %a0, <8 x i32> poison, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -270,7 +270,7 @@ define <16 x i16> @elts_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> poison, <8 x i32> [[TMP1]])
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
 ;
   %1 = shufflevector <8 x i32> %a0, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -305,7 +305,7 @@ define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) {
 define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> poison, <32 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 undef, i32 undef, i32 27, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> poison, <32 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 poison, i32 poison, i32 27, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
 ;
   %1 = shufflevector <16 x i32> %a0, <16 x i32> poison, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -319,7 +319,7 @@ define <32 x i16> @elts_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> poison, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> poison, <16 x i32> [[TMP1]])
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
 ;
   %1 = shufflevector <16 x i32> %a0, <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
index 537b1639ed562..cea7974e29f6e 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
@@ -208,7 +208,7 @@ define <64 x i8> @fold_packuswb_512() {
 define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_128(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> undef)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
   %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 1, i32 undef, i32 undef>
@@ -221,7 +221,7 @@ define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 define <8 x i16> @elts_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_128(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 ;
   %1 = insertelement <4 x i32> %a0, i32 0, i32 0
@@ -256,7 +256,7 @@ define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) {
 define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> undef, <16 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> undef, <16 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP2]]
 ;
   %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -270,7 +270,7 @@ define <16 x i16> @elts_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_256(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> poison, <8 x i32> [[TMP1]])
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
 ;
   %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -305,7 +305,7 @@ define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) {
 define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: @elts_packssdw_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef)
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> undef, <32 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 undef, i32 undef, i32 27, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> undef, <32 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 24, i32 poison, i32 poison, i32 27, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
 ;
   %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -319,7 +319,7 @@ define <32 x i16> @elts_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: @elts_packusdw_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> poison, <16 x i32> [[TMP1]])
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
 ;
   %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb-inseltpoison.ll
index c05e5cb6182af..c98f1c7b5c839 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb-inseltpoison.ll
@@ -419,7 +419,7 @@ define <64 x i8> @permute3_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @fold_with_poison_elts(<16 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_poison_elts(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 0, i32 16, i32 undef, i32 16, i32 1, i32 16, i32 undef, i32 16, i32 2, i32 16, i32 undef, i32 16, i32 3, i32 16, i32 undef, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 0, i32 16, i32 poison, i32 16, i32 1, i32 16, i32 poison, i32 16, i32 2, i32 16, i32 poison, i32 16, i32 3, i32 16, i32 poison, i32 16>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128>)
@@ -428,7 +428,7 @@ define <16 x i8> @fold_with_poison_elts(<16 x i8> %InVec) {
 
 define <32 x i8> @fold_with_poison_elts_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_poison_elts_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <32 x i32> <i32 0, i32 32, i32 undef, i32 32, i32 1, i32 32, i32 undef, i32 32, i32 2, i32 32, i32 undef, i32 32, i32 3, i32 32, i32 undef, i32 32, i32 16, i32 48, i32 undef, i32 48, i32 17, i32 48, i32 undef, i32 48, i32 18, i32 48, i32 undef, i32 48, i32 19, i32 48, i32 undef, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <32 x i32> <i32 0, i32 32, i32 poison, i32 32, i32 1, i32 32, i32 poison, i32 32, i32 2, i32 32, i32 poison, i32 32, i32 3, i32 32, i32 poison, i32 32, i32 16, i32 48, i32 poison, i32 48, i32 17, i32 48, i32 poison, i32 48, i32 18, i32 48, i32 poison, i32 48, i32 19, i32 48, i32 poison, i32 48>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128, i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128>)
@@ -437,7 +437,7 @@ define <32 x i8> @fold_with_poison_elts_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @fold_with_poison_elts_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_poison_elts_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <64 x i32> <i32 0, i32 64, i32 undef, i32 64, i32 1, i32 64, i32 undef, i32 64, i32 2, i32 64, i32 undef, i32 64, i32 3, i32 64, i32 undef, i32 64, i32 16, i32 80, i32 undef, i32 80, i32 17, i32 80, i32 undef, i32 80, i32 18, i32 80, i32 undef, i32 80, i32 19, i32 80, i32 undef, i32 80, i32 32, i32 96, i32 undef, i32 96, i32 33, i32 96, i32 undef, i32 96, i32 34, i32 96, i32 undef, i32 96, i32 35, i32 96, i32 undef, i32 96, i32 48, i32 112, i32 undef, i32 112, i32 49, i32 112, i32 undef, i32 112, i32 50, i32 112, i32 undef, i32 112, i32 51, i32 112, i32 undef, i32 112>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <64 x i32> <i32 0, i32 64, i32 poison, i32 64, i32 1, i32 64, i32 poison, i32 64, i32 2, i32 64, i32 poison, i32 64, i32 3, i32 64, i32 poison, i32 64, i32 16, i32 80, i32 poison, i32 80, i32 17, i32 80, i32 poison, i32 80, i32 18, i32 80, i32 poison, i32 80, i32 19, i32 80, i32 poison, i32 80, i32 32, i32 96, i32 poison, i32 96, i32 33, i32 96, i32 poison, i32 96, i32 34, i32 96, i32 poison, i32 96, i32 35, i32 96, i32 poison, i32 96, i32 48, i32 112, i32 poison, i32 112, i32 49, i32 112, i32 poison, i32 112, i32 50, i32 112, i32 poison, i32 112, i32 51, i32 112, i32 poison, i32 112>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128, i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128, i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128, i8 0, i8 -128, i8 poison, i8 -128, i8 1, i8 -128, i8 poison, i8 -128, i8 2, i8 -128, i8 poison, i8 -128, i8 3, i8 -128, i8 poison, i8 -128>)
@@ -473,7 +473,7 @@ define <64 x i8> @fold_with_allpoison_elts_avx512(<64 x i8> %InVec) {
 define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask, i8 %M0, i8 %M15) {
 ; CHECK-LABEL: @demanded_elts_insertion(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> [[INVEC:%.*]], <16 x i8> [[BASEMASK:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 ;
   %1 = insertelement <16 x i8> %BaseMask, i8 %M0, i32 0
@@ -486,7 +486,7 @@ define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask,
 define <32 x i8> @demanded_elts_insertion_avx2(<32 x i8> %InVec, <32 x i8> %BaseMask, i8 %M0, i8 %M22) {
 ; CHECK-LABEL: @demanded_elts_insertion_avx2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[INVEC:%.*]], <32 x i8> [[BASEMASK:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <32 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 undef, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <32 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 poison, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP2]]
 ;
   %1 = insertelement <32 x i8> %BaseMask, i8 %M0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
index 59324b157cb47..9e49b09424da4 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
@@ -419,7 +419,7 @@ define <64 x i8> @permute3_avx512(<64 x i8> %InVec) {
 
 define <16 x i8> @fold_with_undef_elts(<16 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_undef_elts(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 0, i32 16, i32 undef, i32 16, i32 1, i32 16, i32 undef, i32 16, i32 2, i32 16, i32 undef, i32 16, i32 3, i32 16, i32 undef, i32 16>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 0, i32 16, i32 poison, i32 16, i32 1, i32 16, i32 poison, i32 16, i32 2, i32 16, i32 poison, i32 16, i32 3, i32 16, i32 poison, i32 16>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
@@ -428,7 +428,7 @@ define <16 x i8> @fold_with_undef_elts(<16 x i8> %InVec) {
 
 define <32 x i8> @fold_with_undef_elts_avx2(<32 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_undef_elts_avx2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <32 x i32> <i32 0, i32 32, i32 undef, i32 32, i32 1, i32 32, i32 undef, i32 32, i32 2, i32 32, i32 undef, i32 32, i32 3, i32 32, i32 undef, i32 32, i32 16, i32 48, i32 undef, i32 48, i32 17, i32 48, i32 undef, i32 48, i32 18, i32 48, i32 undef, i32 48, i32 19, i32 48, i32 undef, i32 48>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[INVEC:%.*]], <32 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <32 x i32> <i32 0, i32 32, i32 poison, i32 32, i32 1, i32 32, i32 poison, i32 32, i32 2, i32 32, i32 poison, i32 32, i32 3, i32 32, i32 poison, i32 32, i32 16, i32 48, i32 poison, i32 48, i32 17, i32 48, i32 poison, i32 48, i32 18, i32 48, i32 poison, i32 48, i32 19, i32 48, i32 poison, i32 48>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
 ;
   %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
@@ -437,7 +437,7 @@ define <32 x i8> @fold_with_undef_elts_avx2(<32 x i8> %InVec) {
 
 define <64 x i8> @fold_with_undef_elts_avx512(<64 x i8> %InVec) {
 ; CHECK-LABEL: @fold_with_undef_elts_avx512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <64 x i32> <i32 0, i32 64, i32 undef, i32 64, i32 1, i32 64, i32 undef, i32 64, i32 2, i32 64, i32 undef, i32 64, i32 3, i32 64, i32 undef, i32 64, i32 16, i32 80, i32 undef, i32 80, i32 17, i32 80, i32 undef, i32 80, i32 18, i32 80, i32 undef, i32 80, i32 19, i32 80, i32 undef, i32 80, i32 32, i32 96, i32 undef, i32 96, i32 33, i32 96, i32 undef, i32 96, i32 34, i32 96, i32 undef, i32 96, i32 35, i32 96, i32 undef, i32 96, i32 48, i32 112, i32 undef, i32 112, i32 49, i32 112, i32 undef, i32 112, i32 50, i32 112, i32 undef, i32 112, i32 51, i32 112, i32 undef, i32 112>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[INVEC:%.*]], <64 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <64 x i32> <i32 0, i32 64, i32 poison, i32 64, i32 1, i32 64, i32 poison, i32 64, i32 2, i32 64, i32 poison, i32 64, i32 3, i32 64, i32 poison, i32 64, i32 16, i32 80, i32 poison, i32 80, i32 17, i32 80, i32 poison, i32 80, i32 18, i32 80, i32 poison, i32 80, i32 19, i32 80, i32 poison, i32 80, i32 32, i32 96, i32 poison, i32 96, i32 33, i32 96, i32 poison, i32 96, i32 34, i32 96, i32 poison, i32 96, i32 35, i32 96, i32 poison, i32 96, i32 48, i32 112, i32 poison, i32 112, i32 49, i32 112, i32 poison, i32 112, i32 50, i32 112, i32 poison, i32 112, i32 51, i32 112, i32 poison, i32 112>
 ; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
 ;
   %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
@@ -473,7 +473,7 @@ define <64 x i8> @fold_with_allundef_elts_avx512(<64 x i8> %InVec) {
 define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask, i8 %M0, i8 %M15) {
 ; CHECK-LABEL: @demanded_elts_insertion(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> [[INVEC:%.*]], <16 x i8> [[BASEMASK:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 poison, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 ;
   %1 = insertelement <16 x i8> %BaseMask, i8 %M0, i32 0
@@ -486,7 +486,7 @@ define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask,
 define <32 x i8> @demanded_elts_insertion_avx2(<32 x i8> %InVec, <32 x i8> %BaseMask, i8 %M0, i8 %M22) {
 ; CHECK-LABEL: @demanded_elts_insertion_avx2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[INVEC:%.*]], <32 x i8> [[BASEMASK:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> undef, <32 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 undef, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> undef, <32 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 poison, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <32 x i8> [[TMP2]]
 ;
   %1 = insertelement <32 x i8> %BaseMask, i8 %M0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse4a-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse4a-inseltpoison.ll
index 68924c839eb88..c8cc053fc97dc 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse4a-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse4a-inseltpoison.ll
@@ -25,7 +25,7 @@ define <2 x i64> @test_extrq_zero_arg0(<2 x i64> %x, <16 x i8> %y) {
 define <2 x i64> @test_extrq_zero_arg1(<2 x i64> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @test_extrq_zero_arg1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -61,7 +61,7 @@ define <2 x i64> @test_extrq_constant_poison(<2 x i64> %x, <16 x i8> %y) {
 define <2 x i64> @test_extrq_call_constexpr(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrq_call_constexpr(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -85,7 +85,7 @@ define <2 x i64> @test_extrqi_call(<2 x i64> %x) {
 define <2 x i64> @test_extrqi_shuffle_1zuu(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrqi_shuffle_1zuu(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -96,7 +96,7 @@ define <2 x i64> @test_extrqi_shuffle_1zuu(<2 x i64> %x) {
 define <2 x i64> @test_extrqi_shuffle_2zzzzzzzuuuuuuuu(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrqi_shuffle_2zzzzzzzuuuuuuuu(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 2, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 2, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -197,7 +197,7 @@ define <2 x i64> @test_insertq_call_constexpr(<2 x i64> %x) {
 
 define <16 x i8> @test_insertqi_shuffle_04uu(<16 x i8> %v, <16 x i8> %i) {
 ; CHECK-LABEL: @test_insertqi_shuffle_04uu(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[V:%.*]], <16 x i8> [[I:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[V:%.*]], <16 x i8> [[I:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = bitcast <16 x i8> %v to <2 x i64>
@@ -209,7 +209,7 @@ define <16 x i8> @test_insertqi_shuffle_04uu(<16 x i8> %v, <16 x i8> %i) {
 
 define <16 x i8> @test_insertqi_shuffle_8123uuuu(<16 x i8> %v, <16 x i8> %i) {
 ; CHECK-LABEL: @test_insertqi_shuffle_8123uuuu(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[I:%.*]], <16 x i8> [[V:%.*]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[I:%.*]], <16 x i8> [[V:%.*]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = bitcast <16 x i8> %v to <2 x i64>
@@ -242,7 +242,7 @@ define <2 x i64> @test_insertqi_call_constexpr(<2 x i64> %x) {
 define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
 ; CHECK-LABEL: @testInsert64Bits(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[I:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -253,7 +253,7 @@ define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
 define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i) {
 ; CHECK-LABEL: @testZeroLength(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[I:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
index c37cdf77a76f8..36140be21c90d 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
@@ -25,7 +25,7 @@ define <2 x i64> @test_extrq_zero_arg0(<2 x i64> %x, <16 x i8> %y) {
 define <2 x i64> @test_extrq_zero_arg1(<2 x i64> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @test_extrq_zero_arg1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -61,7 +61,7 @@ define <2 x i64> @test_extrq_constant_undef(<2 x i64> %x, <16 x i8> %y) {
 define <2 x i64> @test_extrq_call_constexpr(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrq_call_constexpr(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -85,7 +85,7 @@ define <2 x i64> @test_extrqi_call(<2 x i64> %x) {
 define <2 x i64> @test_extrqi_shuffle_1zuu(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrqi_shuffle_1zuu(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -96,7 +96,7 @@ define <2 x i64> @test_extrqi_shuffle_1zuu(<2 x i64> %x) {
 define <2 x i64> @test_extrqi_shuffle_2zzzzzzzuuuuuuuu(<2 x i64> %x) {
 ; CHECK-LABEL: @test_extrqi_shuffle_2zzzzzzzuuuuuuuu(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 2, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, <16 x i32> <i32 2, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -197,7 +197,7 @@ define <2 x i64> @test_insertq_call_constexpr(<2 x i64> %x) {
 
 define <16 x i8> @test_insertqi_shuffle_04uu(<16 x i8> %v, <16 x i8> %i) {
 ; CHECK-LABEL: @test_insertqi_shuffle_04uu(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[V:%.*]], <16 x i8> [[I:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[V:%.*]], <16 x i8> [[I:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = bitcast <16 x i8> %v to <2 x i64>
@@ -209,7 +209,7 @@ define <16 x i8> @test_insertqi_shuffle_04uu(<16 x i8> %v, <16 x i8> %i) {
 
 define <16 x i8> @test_insertqi_shuffle_8123uuuu(<16 x i8> %v, <16 x i8> %i) {
 ; CHECK-LABEL: @test_insertqi_shuffle_8123uuuu(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[I:%.*]], <16 x i8> [[V:%.*]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[I:%.*]], <16 x i8> [[V:%.*]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 ;
   %1 = bitcast <16 x i8> %v to <2 x i64>
@@ -242,7 +242,7 @@ define <2 x i64> @test_insertqi_call_constexpr(<2 x i64> %x) {
 define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
 ; CHECK-LABEL: @testInsert64Bits(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[I:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
@@ -253,7 +253,7 @@ define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
 define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i) {
 ; CHECK-LABEL: @testZeroLength(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[I:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil-inseltpoison.ll
index b6c7f264ce233..d8ae48b5a9803 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil-inseltpoison.ll
@@ -169,7 +169,7 @@ define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @poison_test_vpermilvar_ps(<4 x float> %v) {
 ; CHECK-LABEL: @poison_test_vpermilvar_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 poison, i32 2, i32 1, i32 poison>)
@@ -178,7 +178,7 @@ define <4 x float> @poison_test_vpermilvar_ps(<4 x float> %v) {
 
 define <8 x float> @poison_test_vpermilvar_ps_256(<8 x float> %v) {
 ; CHECK-LABEL: @poison_test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 2, i32 1, i32 poison, i32 7, i32 6, i32 5, i32 4>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 poison, i32 6, i32 5, i32 poison, i32 3, i32 2, i32 1, i32 0>)
@@ -187,7 +187,7 @@ define <8 x float> @poison_test_vpermilvar_ps_256(<8 x float> %v) {
 
 define <16 x float> @poison_test_vpermilvar_ps_512(<16 x float> %v) {
 ; CHECK-LABEL: @poison_test_vpermilvar_ps_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4, i32 undef, i32 10, i32 9, i32 undef, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> <i32 poison, i32 2, i32 1, i32 poison, i32 7, i32 6, i32 5, i32 4, i32 poison, i32 10, i32 9, i32 poison, i32 15, i32 14, i32 13, i32 12>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 poison, i32 6, i32 5, i32 poison, i32 3, i32 2, i32 1, i32 0, i32 poison, i32 6, i32 5, i32 poison, i32 3, i32 2, i32 1, i32 0>)
@@ -196,7 +196,7 @@ define <16 x float> @poison_test_vpermilvar_ps_512(<16 x float> %v) {
 
 define <2 x double> @poison_test_vpermilvar_pd(<2 x double> %v) {
 ; CHECK-LABEL: @poison_test_vpermilvar_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 poison, i64 0>)
@@ -205,7 +205,7 @@ define <2 x double> @poison_test_vpermilvar_pd(<2 x double> %v) {
 
 define <4 x double> @poison_test_vpermilvar_pd_256(<4 x double> %v) {
 ; CHECK-LABEL: @poison_test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> <i32 undef, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> <i32 poison, i32 0, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 poison, i64 1, i64 2, i64 poison>)
@@ -214,7 +214,7 @@ define <4 x double> @poison_test_vpermilvar_pd_256(<4 x double> %v) {
 
 define <8 x double> @poison_test_vpermilvar_pd_512(<8 x double> %v) {
 ; CHECK-LABEL: @poison_test_vpermilvar_pd_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> <i32 undef, i32 0, i32 3, i32 undef, i32 undef, i32 4, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> <i32 poison, i32 0, i32 3, i32 poison, i32 poison, i32 4, i32 7, i32 poison>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 poison, i64 1, i64 2, i64 poison, i64 poison, i64 1, i64 2, i64 poison>)
@@ -225,7 +225,7 @@ define <8 x double> @poison_test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %a1, i32 3
@@ -236,7 +236,7 @@ define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
 
 define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 6, i32 undef, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 0, i32 poison, i32 1, i32 poison, i32 6, i32 poison, i32 7>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %1 = shufflevector <8 x i32> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 2, i32 1, i32 0>, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
@@ -248,7 +248,7 @@ define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1)
 define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a1, i32 %a2) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[A0:%.*]], <16 x i32> [[A1:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> poison, <16 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> poison, <16 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x float> [[TMP2]]
 ;
   %1 = insertelement <16 x i32> %a1, i32 %a2, i32 0
@@ -259,7 +259,7 @@ define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a
 
 define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x i64> <i64 0, i64 2>, i64 %a1, i32 1
@@ -270,7 +270,7 @@ define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
 
 define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %1 = shufflevector <4 x i64> <i64 0, i64 2, i64 0, i64 2>, <4 x i64> %a1, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
index b7dac07387a67..1060dc27b0910 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
@@ -169,7 +169,7 @@ define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>)
@@ -178,7 +178,7 @@ define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) {
 
 define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[V:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 2, i32 1, i32 poison, i32 7, i32 6, i32 5, i32 4>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>)
@@ -187,7 +187,7 @@ define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) {
 
 define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_ps_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4, i32 undef, i32 10, i32 9, i32 undef, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[V:%.*]], <16 x float> poison, <16 x i32> <i32 poison, i32 2, i32 1, i32 poison, i32 7, i32 6, i32 5, i32 4, i32 poison, i32 10, i32 9, i32 poison, i32 15, i32 14, i32 13, i32 12>
 ; CHECK-NEXT:    ret <16 x float> [[TMP1]]
 ;
   %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0, i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>)
@@ -196,7 +196,7 @@ define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) {
 
 define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[V:%.*]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 undef, i64 0>)
@@ -205,7 +205,7 @@ define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) {
 
 define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> <i32 undef, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V:%.*]], <4 x double> poison, <4 x i32> <i32 poison, i32 0, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 undef, i64 1, i64 2, i64 undef>)
@@ -214,7 +214,7 @@ define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) {
 
 define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) {
 ; CHECK-LABEL: @undef_test_vpermilvar_pd_512(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> <i32 undef, i32 0, i32 3, i32 undef, i32 undef, i32 4, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[V:%.*]], <8 x double> poison, <8 x i32> <i32 poison, i32 0, i32 3, i32 poison, i32 poison, i32 4, i32 7, i32 poison>
 ; CHECK-NEXT:    ret <8 x double> [[TMP1]]
 ;
   %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 undef, i64 1, i64 2, i64 undef, i64 undef, i64 1, i64 2, i64 undef>)
@@ -225,7 +225,7 @@ define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) {
 
 define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %1 = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %a1, i32 3
@@ -236,7 +236,7 @@ define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
 
 define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 6, i32 undef, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 0, i32 poison, i32 1, i32 poison, i32 6, i32 poison, i32 7>
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
   %1 = shufflevector <8 x i32> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 2, i32 1, i32 0>, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
@@ -248,7 +248,7 @@ define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1)
 define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a1, i32 %a2) {
 ; CHECK-LABEL: @elts_test_vpermilvar_ps_512(
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> [[A0:%.*]], <16 x i32> [[A1:%.*]])
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> undef, <16 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> undef, <16 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x float> [[TMP2]]
 ;
   %1 = insertelement <16 x i32> %a1, i32 %a2, i32 0
@@ -259,7 +259,7 @@ define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a
 
 define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A0:%.*]], <2 x double> undef, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %1 = insertelement <2 x i64> <i64 0, i64 2>, i64 %a1, i32 1
@@ -270,7 +270,7 @@ define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
 
 define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: @elts_test_vpermilvar_pd_256(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
   %1 = shufflevector <4 x i64> <i64 0, i64 2, i64 0, i64 2>, <4 x i64> %a1, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
diff --git a/llvm/test/Transforms/InstCombine/alloca-in-non-alloca-as.ll b/llvm/test/Transforms/InstCombine/alloca-in-non-alloca-as.ll
index 3a89106443271..079b728d38245 100644
--- a/llvm/test/Transforms/InstCombine/alloca-in-non-alloca-as.ll
+++ b/llvm/test/Transforms/InstCombine/alloca-in-non-alloca-as.ll
@@ -3,7 +3,7 @@
 
 ; Gracefully handle the alloca that is not in the alloca AS (=5)
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 declare void @use(ptr, ptr)
diff --git a/llvm/test/Transforms/InstCombine/binop-select.ll b/llvm/test/Transforms/InstCombine/binop-select.ll
index da6c17984eb91..a59e19897f061 100644
--- a/llvm/test/Transforms/InstCombine/binop-select.ll
+++ b/llvm/test/Transforms/InstCombine/binop-select.ll
@@ -6,6 +6,7 @@ declare void @use_f32(float)
 declare void @use_v2f16(<2 x half>)
 declare void @use_v2i8(<2 x i8>)
 declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8>, <2 x i8>)
 
 define i32 @test1(i1 %c, i32 %x, i32 %y) {
 ; CHECK-LABEL: @test1(
@@ -109,6 +110,47 @@ define i32 @test_sub_deduce_false(i32 %x, i32 %y) {
   ret i32 %sub
 }
 
+define <2 x i8> @test_sub_dont_deduce_with_undef_cond_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @test_sub_dont_deduce_with_undef_cond_vec(
+; CHECK-NEXT:    [[C_NOT:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 9, i8 undef>
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[C_NOT]], <2 x i8> <i8 7, i8 7>, <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[X]], <2 x i8> [[COND]])
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %c = icmp ne <2 x i8> %x, <i8 9, i8 undef>
+  %cond = select <2 x i1> %c, <2 x i8> %y, <2 x i8> <i8 7, i8 7>
+  %sub = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %x, <2 x i8> %cond)
+  ret <2 x i8> %sub
+}
+
+define <2 x i8> @test_sub_dont_deduce_with_poison_cond_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @test_sub_dont_deduce_with_poison_cond_vec(
+; CHECK-NEXT:    [[C_NOT:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 poison, i8 9>
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[C_NOT]], <2 x i8> <i8 7, i8 7>, <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[X]], <2 x i8> [[COND]])
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %c = icmp ne <2 x i8> %x, <i8 poison, i8 9>
+  %cond = select <2 x i1> %c, <2 x i8> %y, <2 x i8> <i8 7, i8 7>
+  %sub = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %x, <2 x i8> %cond)
+  ret <2 x i8> %sub
+}
+
+
+define <2 x i8> @test_sub_deduce_with_undef_val_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @test_sub_deduce_with_undef_val_vec(
+; CHECK-NEXT:    [[C_NOT:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[X]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[SUB:%.*]] = select <2 x i1> [[C_NOT]], <2 x i8> <i8 4, i8 -1>, <2 x i8> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %c = icmp ne <2 x i8> %x, <i8 1, i8 2>
+  %cond = select <2 x i1> %c, <2 x i8> %y, <2 x i8> <i8 3, i8 undef>
+  %sub = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %x, <2 x i8> %cond)
+  ret <2 x i8> %sub
+}
+
+
 define i32 @test6(i1 %c, i32 %x, i32 %y) {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[C:%.*]], i32 7, i32 [[X:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/bit_floor.ll b/llvm/test/Transforms/InstCombine/bit_floor.ll
index f9452b27e6c30..9daa8eee8969c 100644
--- a/llvm/test/Transforms/InstCombine/bit_floor.ll
+++ b/llvm/test/Transforms/InstCombine/bit_floor.ll
@@ -4,9 +4,11 @@
 define i32 @bit_floor_32(i32 %x) {
 ; CHECK-LABEL: @bit_floor_32(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X]], i1 false), !range [[RNG0:![0-9]+]]
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 -2147483648, [[TMP1]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[X]], 1
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG0:![0-9]+]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 32, [[CTLZ]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i32 0, i32 [[SHL]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %eq0 = icmp eq i32 %x, 0
@@ -21,9 +23,11 @@ define i32 @bit_floor_32(i32 %x) {
 define i64 @bit_floor_64(i64 %x) {
 ; CHECK-LABEL: @bit_floor_64(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq i64 [[X:%.*]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[X]], i1 false), !range [[RNG1:![0-9]+]]
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i64 -9223372036854775808, [[TMP1]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i64 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[X]], 1
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[LSHR]], i1 false), !range [[RNG1:![0-9]+]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i64 64, [[CTLZ]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i64 1, [[SUB]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i64 0, i64 [[SHL]]
 ; CHECK-NEXT:    ret i64 [[SEL]]
 ;
   %eq0 = icmp eq i64 %x, 0
@@ -39,9 +43,11 @@ define i64 @bit_floor_64(i64 %x) {
 define i32 @bit_floor_commuted_operands(i32 %x) {
 ; CHECK-LABEL: @bit_floor_commuted_operands(
 ; CHECK-NEXT:    [[NE0_NOT:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X]], i1 false), !range [[RNG0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 -2147483648, [[TMP1]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[NE0_NOT]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[X]], 1
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG0]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 32, [[CTLZ]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[NE0_NOT]], i32 0, i32 [[SHL]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %ne0 = icmp ne i32 %x, 0
@@ -58,7 +64,7 @@ define i32 @bit_floor_lshr_used_twice(i32 %x, ptr %p) {
 ; CHECK-LABEL: @bit_floor_lshr_used_twice(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[X]], 1
-; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG2:![0-9]+]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG0]]
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 32, [[CTLZ]]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i32 0, i32 [[SHL]]
@@ -80,7 +86,7 @@ define i32 @bit_floor_ctlz_used_twice(i32 %x, ptr %p) {
 ; CHECK-LABEL: @bit_floor_ctlz_used_twice(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[X]], 1
-; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG2]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG0]]
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 32, [[CTLZ]]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i32 0, i32 [[SHL]]
@@ -102,7 +108,7 @@ define i32 @bit_floor_sub_used_twice(i32 %x, ptr %p) {
 ; CHECK-LABEL: @bit_floor_sub_used_twice(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[X]], 1
-; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG2]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG0]]
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 32, [[CTLZ]]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i32 0, i32 [[SHL]]
@@ -124,7 +130,7 @@ define i32 @bit_floor_shl_used_twice(i32 %x, ptr %p) {
 ; CHECK-LABEL: @bit_floor_shl_used_twice(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[X]], 1
-; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG2]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG0]]
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 32, [[CTLZ]]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[SUB]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[EQ0]], i32 0, i32 [[SHL]]
@@ -145,9 +151,11 @@ define i32 @bit_floor_shl_used_twice(i32 %x, ptr %p) {
 define <4 x i32> @bit_floor_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: @bit_floor_v4i32(
 ; CHECK-NEXT:    [[EQ0:%.*]] = icmp eq <4 x i32> [[X:%.*]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[X]], i1 false), !range [[RNG0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, [[TMP1]]
-; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[EQ0]], <4 x i32> zeroinitializer, <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <4 x i32> [[X]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[LSHR]], i1 false), !range [[RNG0]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw <4 x i32> <i32 32, i32 32, i32 32, i32 32>, [[CTLZ]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw <4 x i32> <i32 1, i32 1, i32 1, i32 1>, [[SUB]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[EQ0]], <4 x i32> zeroinitializer, <4 x i32> [[SHL]]
 ; CHECK-NEXT:    ret <4 x i32> [[SEL]]
 ;
   %eq0 = icmp eq <4 x i32> %x, <i32 0, i32 0, i32 0, i32 0>
diff --git a/llvm/test/Transforms/InstCombine/bitreverse.ll b/llvm/test/Transforms/InstCombine/bitreverse.ll
index ab41debc65c77..d4874e9e2b15a 100644
--- a/llvm/test/Transforms/InstCombine/bitreverse.ll
+++ b/llvm/test/Transforms/InstCombine/bitreverse.ll
@@ -286,7 +286,7 @@ define i4 @shuf_bitcast_twice_4bits(i4 %x) {
 ; CHECK-NEXT:    ret i4 [[CAST2]]
 ;
   %cast1 = bitcast i4 %x to <4 x i1>
-  %bitreverse = shufflevector <4 x i1> %cast1, <4 x i1> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+  %bitreverse = shufflevector <4 x i1> %cast1, <4 x i1> undef, <4 x i32> <i32 poison, i32 2, i32 1, i32 0>
   %cast2 = bitcast <4 x i1> %bitreverse to i4
   ret i4 %cast2
 }
diff --git a/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll b/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll
index ad868b4e036f3..f5a288ad297ba 100644
--- a/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll
@@ -77,7 +77,7 @@ define <4 x float> @good5(float %v) {
 define <4 x float> @splat_undef1(float %arg) {
 ; CHECK-LABEL: @splat_undef1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0
-; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[T6]]
 ;
   %t = insertelement <4 x float> poison, float %arg, i32 1
@@ -92,7 +92,7 @@ define <4 x float> @splat_undef1(float %arg) {
 define <4 x float> @splat_undef2(float %arg) {
 ; CHECK-LABEL: @splat_undef2(
 ; CHECK-NEXT:    [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0
-; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[T6]]
 ;
   %t = insertelement <4 x float> poison, float %arg, i32 0
@@ -131,7 +131,7 @@ define <1 x float> @bad4(float %arg) {
 define <4 x float> @splat_undef3(float %arg) {
 ; CHECK-LABEL: @splat_undef3(
 ; CHECK-NEXT:    [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0
-; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[T7:%.*]] = fadd <4 x float> [[T6]], [[T4]]
 ; CHECK-NEXT:    ret <4 x float> [[T7]]
diff --git a/llvm/test/Transforms/InstCombine/broadcast.ll b/llvm/test/Transforms/InstCombine/broadcast.ll
index 7ee70605f2b0e..6f820430c466d 100644
--- a/llvm/test/Transforms/InstCombine/broadcast.ll
+++ b/llvm/test/Transforms/InstCombine/broadcast.ll
@@ -77,7 +77,7 @@ define <4 x float> @good5(float %v) {
 define <4 x float> @splat_undef1(float %arg) {
 ; CHECK-LABEL: @splat_undef1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i64 0
-; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[T6]]
 ;
   %t = insertelement <4 x float> undef, float %arg, i32 1
@@ -92,7 +92,7 @@ define <4 x float> @splat_undef1(float %arg) {
 define <4 x float> @splat_undef2(float %arg) {
 ; CHECK-LABEL: @splat_undef2(
 ; CHECK-NEXT:    [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0
-; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[T6]]
 ;
   %t = insertelement <4 x float> undef, float %arg, i32 0
@@ -131,7 +131,7 @@ define <1 x float> @bad4(float %arg) {
 define <4 x float> @splat_undef3(float %arg) {
 ; CHECK-LABEL: @splat_undef3(
 ; CHECK-NEXT:    [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i64 0
-; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[T7:%.*]] = fadd <4 x float> [[T6]], [[T4]]
 ; CHECK-NEXT:    ret <4 x float> [[T7]]
diff --git a/llvm/test/Transforms/InstCombine/bswap-inseltpoison.ll b/llvm/test/Transforms/InstCombine/bswap-inseltpoison.ll
index fd114d4c06533..783f14672256b 100644
--- a/llvm/test/Transforms/InstCombine/bswap-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/bswap-inseltpoison.ll
@@ -73,7 +73,7 @@ define i128 @shuf_16bytes(<16 x i8> %x) {
 
 define i32 @shuf_2bytes_widening(<2 x i8> %x) {
 ; CHECK-LABEL: @shuf_2bytes_widening(
-; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[CAST:%.*]] = bitcast <4 x i8> [[BSWAP]] to i32
 ; CHECK-NEXT:    ret i32 [[CAST]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/bswap.ll b/llvm/test/Transforms/InstCombine/bswap.ll
index bb70b4e0c1be2..631d02ad8d806 100644
--- a/llvm/test/Transforms/InstCombine/bswap.ll
+++ b/llvm/test/Transforms/InstCombine/bswap.ll
@@ -718,7 +718,7 @@ define i128 @shuf_16bytes(<16 x i8> %x) {
 
 define i32 @shuf_2bytes_widening(<2 x i8> %x) {
 ; CHECK-LABEL: @shuf_2bytes_widening(
-; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BSWAP:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[CAST:%.*]] = bitcast <4 x i8> [[BSWAP]] to i32
 ; CHECK-NEXT:    ret i32 [[CAST]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll b/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll
index 6e1f8bc1d7ab6..ab7a50e55db0f 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll
@@ -30,7 +30,7 @@ define <8 x i32> @trivial_nop(<8 x i32> %vec, <8 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_a(<8 x i32> %vec, <2 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_a(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[VEC:%.*]], <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -40,7 +40,7 @@ define <8 x i32> @valid_insertion_a(<8 x i32> %vec, <2 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_b(<8 x i32> %vec, <2 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_b(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -50,7 +50,7 @@ define <8 x i32> @valid_insertion_b(<8 x i32> %vec, <2 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_c(<8 x i32> %vec, <2 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_c(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -60,7 +60,7 @@ define <8 x i32> @valid_insertion_c(<8 x i32> %vec, <2 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_d(<8 x i32> %vec, <2 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_d(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -70,7 +70,7 @@ define <8 x i32> @valid_insertion_d(<8 x i32> %vec, <2 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_e(<8 x i32> %vec, <4 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_e(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SUBVEC:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SUBVEC:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[VEC:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -80,7 +80,7 @@ define <8 x i32> @valid_insertion_e(<8 x i32> %vec, <4 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_f(<8 x i32> %vec, <4 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_f(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SUBVEC:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SUBVEC:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -90,7 +90,7 @@ define <8 x i32> @valid_insertion_f(<8 x i32> %vec, <4 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_g(<8 x i32> %vec, <3 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_g(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[SUBVEC:%.*]], <3 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[SUBVEC:%.*]], <3 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[VEC:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
@@ -100,7 +100,7 @@ define <8 x i32> @valid_insertion_g(<8 x i32> %vec, <3 x i32> %subvec) {
 
 define <8 x i32> @valid_insertion_h(<8 x i32> %vec, <3 x i32> %subvec) {
 ; CHECK-LABEL: @valid_insertion_h(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[SUBVEC:%.*]], <3 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[SUBVEC:%.*]], <3 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 10, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll b/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll
index 65a89976cf713..5955650167c21 100644
--- a/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll
+++ b/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll
@@ -7,6 +7,9 @@ declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
 declare i32 @llvm.cttz.i32(i32, i1)
 declare i33 @llvm.cttz.i33(i33, i1)
 declare i32 @llvm.ctlz.i32(i32, i1)
+declare i8 @llvm.umax.i8(i8, i8)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
 declare i33 @llvm.ctlz.i33(i33, i1)
 declare i8 @llvm.ctpop.i8(i8)
 declare i11 @llvm.ctpop.i11(i11)
@@ -16,6 +19,7 @@ declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
 declare i8 @llvm.bitreverse.i8(i8)
 declare <2 x i8> @llvm.bitreverse.v2i8(<2 x i8>)
 declare void @use6(i6)
+declare void @use8(i8)
 
 define i1 @bswap_eq_i16(i16 %x) {
 ; CHECK-LABEL: @bswap_eq_i16(
@@ -786,7 +790,6 @@ define i1 @bitreverse_ult_22_fail_not_equality_pred(i8 %x) {
   ret i1 %z
 }
 
-
 define <2 x i1> @bitreverse_vec_eq_2_2(<2 x i8> %x) {
 ; CHECK-LABEL: @bitreverse_vec_eq_2_2(
 ; CHECK-NEXT:    [[Z:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 64, i8 64>
@@ -807,3 +810,168 @@ define <2 x i1> @bitreverse_vec_eq_1_2_todo_no_splat(<2 x i8> %x) {
   %z = icmp eq <2 x i8> %y, <i8 1, i8 2>
   ret <2 x i1> %z
 }
+
+define i1 @umax_eq_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_eq_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %r = icmp eq i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @umax_eq_1_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_eq_1_fail(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[M]], 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %r = icmp eq i8 %m, 1
+  ret i1 %r
+}
+
+define i1 @umax_sle_zero_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_sle_zero_fail(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[M]], 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %r = icmp sle i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @umax_ne_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_ne_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  %r = icmp ne i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @umax_ne_zero_fail_multiuse(i8 %x, i8 %y) {
+; CHECK-LABEL: @umax_ne_zero_fail_multiuse(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    call void @use8(i8 [[M]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[M]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.umax.i8(i8 %x, i8 %y)
+  call void @use8(i8 %m)
+  %r = icmp ne i8 %m, 0
+  ret i1 %r
+}
+
+
+define i1 @uadd_sat_ne_zero_fail_multiuse(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_sat_ne_zero_fail_multiuse(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    call void @use8(i8 [[M]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[M]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  call void @use8(i8 %m)
+  %r = icmp ne i8 %m, 0
+  ret i1 %r
+}
+
+
+define i1 @ssub_sat_ne_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_ne_zero(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp ne i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_ne_fail_nonzero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_ne_fail_nonzero(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[M]], 4
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp ne i8 %m, 4
+  ret i1 %r
+}
+
+define i1 @ssub_sat_eq_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_eq_zero(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp eq i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_sle_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_sle_zero(
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp sle i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_sge_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_sge_zero(
+; CHECK-NEXT:    [[R:%.*]] = icmp sge i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp sge i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_slt_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_slt_zero(
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp slt i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_slt_neg1_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_slt_neg1_fail(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[M]], -1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp slt i8 %m, -1
+  ret i1 %r
+}
+
+define i1 @ssub_sat_sgt_zero(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_sgt_zero(
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp sgt i8 %m, 0
+  ret i1 %r
+}
+
+define i1 @ssub_sat_sgt_one_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @ssub_sat_sgt_one_fail(
+; CHECK-NEXT:    [[M:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[M]], 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %m = call i8 @llvm.ssub.sat.i8(i8 %x, i8 %y)
+  %r = icmp sgt i8 %m, 1
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll
index 3f379b405ad83..580a977634e1c 100644
--- a/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll
@@ -315,8 +315,8 @@ define float @bitcasted_inselt_to_and_from_FP_uses2(double %x) {
 
 define <4 x double> @invalid_extractelement(<2 x double> %a, <4 x double> %b, ptr %p) {
 ; ANY-LABEL: @invalid_extractelement(
-; ANY-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; ANY-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 undef, i32 1, i32 4, i32 3>
+; ANY-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; ANY-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 poison, i32 1, i32 4, i32 3>
 ; ANY-NEXT:    [[E:%.*]] = extractelement <4 x double> [[B]], i64 1
 ; ANY-NEXT:    store double [[E]], ptr [[P:%.*]], align 8
 ; ANY-NEXT:    ret <4 x double> [[T4]]
diff --git a/llvm/test/Transforms/InstCombine/extractelement.ll b/llvm/test/Transforms/InstCombine/extractelement.ll
index 9d4bef20a0fdc..9f8795b83b688 100644
--- a/llvm/test/Transforms/InstCombine/extractelement.ll
+++ b/llvm/test/Transforms/InstCombine/extractelement.ll
@@ -317,8 +317,8 @@ define float @bitcasted_inselt_to_and_from_FP_uses2(double %x) {
 
 define <4 x double> @invalid_extractelement(<2 x double> %a, <4 x double> %b, ptr %p) {
 ; ANY-LABEL: @invalid_extractelement(
-; ANY-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; ANY-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 undef, i32 1, i32 4, i32 3>
+; ANY-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; ANY-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 poison, i32 1, i32 4, i32 3>
 ; ANY-NEXT:    [[E:%.*]] = extractelement <4 x double> [[B]], i64 1
 ; ANY-NEXT:    store double [[E]], ptr [[P:%.*]], align 8
 ; ANY-NEXT:    ret <4 x double> [[T4]]
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index ddaa5762bc020..15716b68defd6 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -1389,4 +1389,17 @@ define ptr @const_gep_0xi8_global() {
   ret ptr getelementptr ([0 x i8], ptr @g_0xi8_e, i64 0, i64 10)
 }
 
+define ptr @const_gep_chain(ptr %p, i64 %a) {
+; CHECK-LABEL: @const_gep_chain(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 [[A:%.*]]
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 6
+; CHECK-NEXT:    ret ptr [[P4]]
+;
+  %p1 = getelementptr inbounds i8, ptr %p, i64 %a
+  %p2 = getelementptr inbounds i8, ptr %p1, i64 1
+  %p3 = getelementptr inbounds i8, ptr %p2, i64 2
+  %p4 = getelementptr inbounds i8, ptr %p3, i64 3
+  ret ptr %p4
+}
+
 !0 = !{!"branch_weights", i32 2, i32 10}
diff --git a/llvm/test/Transforms/InstCombine/insert-const-shuf-inseltpoison.ll b/llvm/test/Transforms/InstCombine/insert-const-shuf-inseltpoison.ll
index fa5463e2110c7..cb31545cf8923 100644
--- a/llvm/test/Transforms/InstCombine/insert-const-shuf-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/insert-const-shuf-inseltpoison.ll
@@ -28,7 +28,7 @@ define <4 x float> @twoInserts(<4 x float> %x) {
 
 define <4 x i32> @shuffleRetain(<4 x i32> %base) {
 ; CHECK-LABEL: @shuffleRetain(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[BASE:%.*]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 1>, <4 x i32> <i32 1, i32 2, i32 undef, i32 7>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[BASE:%.*]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 1>, <4 x i32> <i32 1, i32 2, i32 poison, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
 ;
   %shuf = shufflevector <4 x i32> %base, <4 x i32> <i32 4, i32 3, i32 2, i32 1>, <4 x i32> <i32 1, i32 2, i32 poison, i32 7>
@@ -39,7 +39,7 @@ define <4 x i32> @shuffleRetain(<4 x i32> %base) {
 
 define <4 x float> @disguisedSelect(<4 x float> %x) {
 ; CHECK-LABEL: @disguisedSelect(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 undef, i32 6, i32 5, i32 3>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 poison, i32 6, i32 5, i32 3>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i64 0
 ; CHECK-NEXT:    ret <4 x float> [[INS]]
 ;
@@ -52,7 +52,7 @@ define <4 x float> @disguisedSelect(<4 x float> %x) {
 
 define <4 x float> @notSelectButNoMaskDifference(<4 x float> %x) {
 ; CHECK-LABEL: @notSelectButNoMaskDifference(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 1, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 1, i32 5, i32 6, i32 poison>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i64 3
 ; CHECK-NEXT:    ret <4 x float> [[INS]]
 ;
@@ -65,7 +65,7 @@ define <4 x float> @notSelectButNoMaskDifference(<4 x float> %x) {
 
 define <4 x float> @tooRisky(<4 x float> %x) {
 ; CHECK-LABEL: @tooRisky(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 1, i32 4, i32 4, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 1, i32 4, i32 4, i32 poison>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i64 3
 ; CHECK-NEXT:    ret <4 x float> [[INS]]
 ;
@@ -94,7 +94,7 @@ define <3 x float> @twoShufUses(<3 x float> %x) {
 
 define <5 x i8> @longerMask(<3 x i8> %x) {
 ; CHECK-LABEL: @longerMask(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> <i8 poison, i8 1, i8 poison>, <5 x i32> <i32 2, i32 1, i32 4, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> <i8 poison, i8 1, i8 poison>, <5 x i32> <i32 2, i32 1, i32 4, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <5 x i8> [[SHUF]], i8 42, i64 4
 ; CHECK-NEXT:    ret <5 x i8> [[INS]]
 ;
@@ -107,7 +107,7 @@ define <5 x i8> @longerMask(<3 x i8> %x) {
 
 define <3 x i8> @shorterMask(<5 x i8> %x) {
 ; CHECK-LABEL: @shorterMask(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <5 x i8> [[X:%.*]], <5 x i8> poison, <3 x i32> <i32 undef, i32 1, i32 4>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <5 x i8> [[X:%.*]], <5 x i8> poison, <3 x i32> <i32 poison, i32 1, i32 4>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <3 x i8> [[SHUF]], i8 42, i64 0
 ; CHECK-NEXT:    ret <3 x i8> [[INS]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll
index fdcd08a22ced3..68dcc45e4b6c3 100644
--- a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll
+++ b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll
@@ -28,7 +28,7 @@ define <4 x float> @twoInserts(<4 x float> %x) {
 
 define <4 x i32> @shuffleRetain(<4 x i32> %base) {
 ; CHECK-LABEL: @shuffleRetain(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[BASE:%.*]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 1>, <4 x i32> <i32 1, i32 2, i32 undef, i32 7>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[BASE:%.*]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 1>, <4 x i32> <i32 1, i32 2, i32 poison, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
 ;
   %shuf = shufflevector <4 x i32> %base, <4 x i32> <i32 4, i32 3, i32 2, i32 1>, <4 x i32> <i32 1, i32 2, i32 undef, i32 7>
@@ -39,7 +39,7 @@ define <4 x i32> @shuffleRetain(<4 x i32> %base) {
 
 define <4 x float> @disguisedSelect(<4 x float> %x) {
 ; CHECK-LABEL: @disguisedSelect(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 undef, i32 6, i32 5, i32 3>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 poison, i32 6, i32 5, i32 3>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i64 0
 ; CHECK-NEXT:    ret <4 x float> [[INS]]
 ;
@@ -52,7 +52,7 @@ define <4 x float> @disguisedSelect(<4 x float> %x) {
 
 define <4 x float> @notSelectButNoMaskDifference(<4 x float> %x) {
 ; CHECK-LABEL: @notSelectButNoMaskDifference(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 1, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float poison, float 1.000000e+00, float 2.000000e+00, float poison>, <4 x i32> <i32 1, i32 5, i32 6, i32 poison>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i64 3
 ; CHECK-NEXT:    ret <4 x float> [[INS]]
 ;
@@ -65,7 +65,7 @@ define <4 x float> @notSelectButNoMaskDifference(<4 x float> %x) {
 
 define <4 x float> @tooRisky(<4 x float> %x) {
 ; CHECK-LABEL: @tooRisky(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 1, i32 4, i32 4, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float 1.000000e+00, float poison, float poison, float poison>, <4 x i32> <i32 1, i32 4, i32 4, i32 poison>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i64 3
 ; CHECK-NEXT:    ret <4 x float> [[INS]]
 ;
@@ -94,7 +94,7 @@ define <3 x float> @twoShufUses(<3 x float> %x) {
 
 define <5 x i8> @longerMask(<3 x i8> %x) {
 ; CHECK-LABEL: @longerMask(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> <i8 poison, i8 1, i8 poison>, <5 x i32> <i32 2, i32 1, i32 4, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> <i8 poison, i8 1, i8 poison>, <5 x i32> <i32 2, i32 1, i32 4, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <5 x i8> [[SHUF]], i8 42, i64 4
 ; CHECK-NEXT:    ret <5 x i8> [[INS]]
 ;
@@ -107,7 +107,7 @@ define <5 x i8> @longerMask(<3 x i8> %x) {
 
 define <3 x i8> @shorterMask(<5 x i8> %x) {
 ; CHECK-LABEL: @shorterMask(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <5 x i8> [[X:%.*]], <5 x i8> poison, <3 x i32> <i32 undef, i32 1, i32 4>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <5 x i8> [[X:%.*]], <5 x i8> poison, <3 x i32> <i32 poison, i32 1, i32 4>
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <3 x i8> [[SHUF]], i8 42, i64 0
 ; CHECK-NEXT:    ret <3 x i8> [[INS]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll
index 5407254001561..b67487e417f95 100644
--- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll
@@ -31,7 +31,7 @@ define <4 x i16> @test2(<8 x i16> %in, <8 x i16> %in2) {
 
 define <2 x i64> @test_vcopyq_lane_p64(<2 x i64> %a, <1 x i64> %b) {
 ; CHECK-LABEL: @test_vcopyq_lane_p64(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i64> [[B:%.*]], <1 x i64> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i64> [[B:%.*]], <1 x i64> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[TMP1]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
 ;
@@ -44,7 +44,7 @@ define <2 x i64> @test_vcopyq_lane_p64(<2 x i64> %a, <1 x i64> %b) {
 
 define <4 x float> @widen_extract2(<4 x float> %ins, <2 x float> %ext) {
 ; CHECK-LABEL: @widen_extract2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I2:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 4, i32 2, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[I2]]
 ;
@@ -57,7 +57,7 @@ define <4 x float> @widen_extract2(<4 x float> %ins, <2 x float> %ext) {
 
 define <4 x float> @widen_extract3(<4 x float> %ins, <3 x float> %ext) {
 ; CHECK-LABEL: @widen_extract3(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[EXT:%.*]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[EXT:%.*]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[I3:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 6, i32 5, i32 4, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[I3]]
 ;
@@ -72,7 +72,7 @@ define <4 x float> @widen_extract3(<4 x float> %ins, <3 x float> %ext) {
 
 define <8 x float> @widen_extract4(<8 x float> %ins, <2 x float> %ext) {
 ; CHECK-LABEL: @widen_extract4(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I1:%.*]] = shufflevector <8 x float> [[INS:%.*]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x float> [[I1]]
 ;
@@ -86,7 +86,7 @@ define <8 x float> @widen_extract4(<8 x float> %ins, <2 x float> %ext) {
 
 define <8 x i16> @pr26015(<4 x i16> %t0) {
 ; CHECK-LABEL: @pr26015(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0:%.*]], <4 x i16> poison, <8 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0:%.*]], <4 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T5:%.*]] = shufflevector <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 0, i16 0, i16 0, i16 poison>, <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 10, i32 4, i32 5, i32 6, i32 11>
 ; CHECK-NEXT:    ret <8 x i16> [[T5]]
 ;
@@ -106,7 +106,7 @@ define <8 x i16> @pr25999(<4 x i16> %t0, i1 %b) {
 ; CHECK-NEXT:    [[T1:%.*]] = extractelement <4 x i16> [[T0:%.*]], i64 2
 ; CHECK-NEXT:    br i1 [[B:%.*]], label [[IF:%.*]], label [[END:%.*]]
 ; CHECK:       if:
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0]], <4 x i16> poison, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0]], <4 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 0, i16 0, i16 0, i16 poison>, i16 [[T1]], i64 3
 ; CHECK-NEXT:    [[T5:%.*]] = shufflevector <8 x i16> [[T3]], <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
 ; CHECK-NEXT:    ret <8 x i16> [[T5]]
@@ -144,7 +144,7 @@ define <4 x double> @pr25999_phis1(i1 %c, <2 x double> %a, <4 x double> %b) {
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[T1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
 ; CHECK-NEXT:    [[T2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[T1]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[T1]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[T2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[T4]]
 ;
@@ -176,7 +176,7 @@ define <4 x double> @pr25999_phis2(i1 %c, <2 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[T1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
 ; CHECK-NEXT:    [[T2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
 ; CHECK-NEXT:    [[D:%.*]] = fadd <2 x double> [[T1]], [[T1]]
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[D]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[D]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[T2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[T4]]
 ;
@@ -266,7 +266,7 @@ bb2:
 define <4 x i32> @extractelt_insertion(<2 x i32> %x, i32 %y) {
 ; CHECK-LABEL: @extractelt_insertion(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[B:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 0, i32 poison>, <4 x i32> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; CHECK-NEXT:    [[C:%.*]] = add i32 [[Y:%.*]], 3
 ; CHECK-NEXT:    [[D:%.*]] = extractelement <4 x i32> [[TMP0]], i32 [[C]]
@@ -417,7 +417,7 @@ define <4 x float> @insert_not_undef_shuffle_translate_commute_uses(float %x, <4
 define <5 x float> @insert_not_undef_shuffle_translate_commute_lengthen(float %x, <4 x float> %y, <4 x float> %q) {
 ; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute_lengthen(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 2
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[XV]], <5 x i32> <i32 0, i32 6, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[XV]], <5 x i32> <i32 0, i32 6, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <5 x float> [[R]]
 ;
   %xv = insertelement <4 x float> %q, float %x, i32 2
@@ -428,7 +428,7 @@ define <5 x float> @insert_not_undef_shuffle_translate_commute_lengthen(float %x
 define <4 x float> @insert_nonzero_index_splat(float %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> poison, float %x, i32 2
@@ -439,7 +439,7 @@ define <4 x float> @insert_nonzero_index_splat(float %x) {
 define <3 x double> @insert_nonzero_index_splat_narrow(double %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_narrow(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> undef, double [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <3 x double> [[TMP1]], <3 x double> poison, <3 x i32> <i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <3 x double> [[TMP1]], <3 x double> poison, <3 x i32> <i32 0, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x double> [[SPLAT]]
 ;
   %xv = insertelement <4 x double> poison, double %x, i32 3
@@ -450,7 +450,7 @@ define <3 x double> @insert_nonzero_index_splat_narrow(double %x) {
 define <5 x i7> @insert_nonzero_index_splat_widen(i7 %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_widen(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <5 x i7> undef, i7 [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <5 x i7> [[TMP1]], <5 x i7> poison, <5 x i32> <i32 undef, i32 0, i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <5 x i7> [[TMP1]], <5 x i7> poison, <5 x i32> <i32 poison, i32 0, i32 0, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <5 x i7> [[SPLAT]]
 ;
   %xv = insertelement <4 x i7> poison, i7 %x, i32 1
@@ -464,7 +464,7 @@ define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_extra_use(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 2
 ; CHECK-NEXT:    call void @use(<4 x float> [[XV]])
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 2, i32 2, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 2, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> poison, float %x, i32 2
@@ -478,7 +478,7 @@ define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) {
 define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> %y) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_base(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i64 2
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> %y, float %x, i32 2
@@ -491,7 +491,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float>
 define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_index(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 [[INDEX:%.*]]
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> poison, float %x, i32 %index
@@ -502,7 +502,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index)
 define <4 x float> @insert_in_splat(float %x) {
 ; CHECK-LABEL: @insert_in_splat(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %xv = insertelement <4 x float> poison, float %x, i32 0
@@ -515,9 +515,9 @@ define <4 x float> @insert_in_splat_extra_uses(float %x) {
 ; CHECK-LABEL: @insert_in_splat_extra_uses(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
 ; CHECK-NEXT:    call void @use(<4 x float> [[XV]])
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    call void @use(<4 x float> [[SPLAT]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %xv = insertelement <4 x float> poison, float %x, i32 0
@@ -533,7 +533,7 @@ define <4 x float> @insert_in_splat_extra_uses(float %x) {
 define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) {
 ; CHECK-LABEL: @insert_in_splat_variable_index(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i32 [[Y:%.*]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -548,7 +548,7 @@ define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) {
 define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) {
 ; CHECK-LABEL: @insert_in_nonsplat(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> [[Y:%.*]], <4 x i32> <i32 undef, i32 0, i32 4, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> [[Y:%.*]], <4 x i32> <i32 poison, i32 0, i32 4, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i64 3
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -563,7 +563,7 @@ define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) {
 define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) {
 ; CHECK-LABEL: @insert_in_nonsplat2(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i64 3
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -575,7 +575,7 @@ define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) {
 
 define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) {
 ; CHECK-LABEL: @shuf_identity_padding(
-; CHECK-NEXT:    [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2
 ; CHECK-NEXT:    ret <4 x i8> [[V2]]
 ;
@@ -588,7 +588,7 @@ define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) {
 
 define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) {
 ; CHECK-LABEL: @shuf_identity_extract(
-; CHECK-NEXT:    [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <3 x i8> [[V1]], i8 [[Y:%.*]], i64 2
 ; CHECK-NEXT:    ret <3 x i8> [[V2]]
 ;
@@ -601,9 +601,9 @@ define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) {
 
 define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) {
 ; CHECK-LABEL: @shuf_identity_extract_extra_use(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 3>
 ; CHECK-NEXT:    call void @use(<4 x float> [[V0]])
-; CHECK-NEXT:    [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> poison, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> poison, <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[Y:%.*]], i64 1
 ; CHECK-NEXT:    ret <4 x float> [[V2]]
 ;
@@ -619,7 +619,7 @@ define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) {
 
 define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %index) {
 ; CHECK-LABEL: @shuf_identity_padding_variable_index(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 [[INDEX:%.*]]
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2
@@ -636,7 +636,7 @@ define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %i
 
 define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x i8> %other) {
 ; CHECK-LABEL: @shuf_identity_padding_wrong_source_vec(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[OTHER:%.*]], i64 1
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i64 1
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2
@@ -653,7 +653,7 @@ define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x
 
 define <4 x i8> @shuf_identity_padding_wrong_index(<2 x i8> %x, i8 %y) {
 ; CHECK-LABEL: @shuf_identity_padding_wrong_index(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[X]], i64 1
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i64 2
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 3
diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
index f94935513f531..6afd737ba4357 100644
--- a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
@@ -31,7 +31,7 @@ define <4 x i16> @test2(<8 x i16> %in, <8 x i16> %in2) {
 
 define <2 x i64> @test_vcopyq_lane_p64(<2 x i64> %a, <1 x i64> %b) {
 ; CHECK-LABEL: @test_vcopyq_lane_p64(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i64> [[B:%.*]], <1 x i64> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i64> [[B:%.*]], <1 x i64> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[TMP1]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
 ;
@@ -44,7 +44,7 @@ define <2 x i64> @test_vcopyq_lane_p64(<2 x i64> %a, <1 x i64> %b) {
 
 define <4 x float> @widen_extract2(<4 x float> %ins, <2 x float> %ext) {
 ; CHECK-LABEL: @widen_extract2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I2:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 4, i32 2, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[I2]]
 ;
@@ -57,7 +57,7 @@ define <4 x float> @widen_extract2(<4 x float> %ins, <2 x float> %ext) {
 
 define <4 x float> @widen_extract3(<4 x float> %ins, <3 x float> %ext) {
 ; CHECK-LABEL: @widen_extract3(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[EXT:%.*]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[EXT:%.*]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[I3:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 6, i32 5, i32 4, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[I3]]
 ;
@@ -72,7 +72,7 @@ define <4 x float> @widen_extract3(<4 x float> %ins, <3 x float> %ext) {
 
 define <8 x float> @widen_extract4(<8 x float> %ins, <2 x float> %ext) {
 ; CHECK-LABEL: @widen_extract4(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[I1:%.*]] = shufflevector <8 x float> [[INS:%.*]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x float> [[I1]]
 ;
@@ -86,7 +86,7 @@ define <8 x float> @widen_extract4(<8 x float> %ins, <2 x float> %ext) {
 
 define <8 x i16> @pr26015(<4 x i16> %t0) {
 ; CHECK-LABEL: @pr26015(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0:%.*]], <4 x i16> poison, <8 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0:%.*]], <4 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T5:%.*]] = shufflevector <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 0, i16 0, i16 0, i16 poison>, <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 10, i32 4, i32 5, i32 6, i32 11>
 ; CHECK-NEXT:    ret <8 x i16> [[T5]]
 ;
@@ -106,7 +106,7 @@ define <8 x i16> @pr25999(<4 x i16> %t0, i1 %b) {
 ; CHECK-NEXT:    [[T1:%.*]] = extractelement <4 x i16> [[T0:%.*]], i64 2
 ; CHECK-NEXT:    br i1 [[B:%.*]], label [[IF:%.*]], label [[END:%.*]]
 ; CHECK:       if:
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0]], <4 x i16> poison, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0]], <4 x i16> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 poison, i16 0, i16 0, i16 0, i16 poison>, i16 [[T1]], i64 3
 ; CHECK-NEXT:    [[T5:%.*]] = shufflevector <8 x i16> [[T3]], <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
 ; CHECK-NEXT:    ret <8 x i16> [[T5]]
@@ -144,7 +144,7 @@ define <4 x double> @pr25999_phis1(i1 %c, <2 x double> %a, <4 x double> %b) {
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[T1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
 ; CHECK-NEXT:    [[T2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[T1]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[T1]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[T2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[T4]]
 ;
@@ -176,7 +176,7 @@ define <4 x double> @pr25999_phis2(i1 %c, <2 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[T1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
 ; CHECK-NEXT:    [[T2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
 ; CHECK-NEXT:    [[D:%.*]] = fadd <2 x double> [[T1]], [[T1]]
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[D]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[D]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x double> [[T2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[T4]]
 ;
@@ -266,7 +266,7 @@ bb2:
 define <4 x i32> @extractelt_insertion(<2 x i32> %x, i32 %y) {
 ; CHECK-LABEL: @extractelt_insertion(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[B:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 0, i32 poison>, <4 x i32> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; CHECK-NEXT:    [[C:%.*]] = add i32 [[Y:%.*]], 3
 ; CHECK-NEXT:    [[D:%.*]] = extractelement <4 x i32> [[TMP0]], i32 [[C]]
@@ -417,7 +417,7 @@ define <4 x float> @insert_not_undef_shuffle_translate_commute_uses(float %x, <4
 define <5 x float> @insert_not_undef_shuffle_translate_commute_lengthen(float %x, <4 x float> %y, <4 x float> %q) {
 ; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute_lengthen(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 2
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[XV]], <5 x i32> <i32 0, i32 6, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[XV]], <5 x i32> <i32 0, i32 6, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <5 x float> [[R]]
 ;
   %xv = insertelement <4 x float> %q, float %x, i32 2
@@ -428,7 +428,7 @@ define <5 x float> @insert_not_undef_shuffle_translate_commute_lengthen(float %x
 define <4 x float> @insert_nonzero_index_splat(float %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> undef, float %x, i32 2
@@ -439,7 +439,7 @@ define <4 x float> @insert_nonzero_index_splat(float %x) {
 define <3 x double> @insert_nonzero_index_splat_narrow(double %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_narrow(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> undef, double [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <3 x double> [[TMP1]], <3 x double> poison, <3 x i32> <i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <3 x double> [[TMP1]], <3 x double> poison, <3 x i32> <i32 0, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x double> [[SPLAT]]
 ;
   %xv = insertelement <4 x double> undef, double %x, i32 3
@@ -450,7 +450,7 @@ define <3 x double> @insert_nonzero_index_splat_narrow(double %x) {
 define <5 x i7> @insert_nonzero_index_splat_widen(i7 %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_widen(
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <5 x i7> undef, i7 [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <5 x i7> [[TMP1]], <5 x i7> poison, <5 x i32> <i32 undef, i32 0, i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <5 x i7> [[TMP1]], <5 x i7> poison, <5 x i32> <i32 poison, i32 0, i32 0, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <5 x i7> [[SPLAT]]
 ;
   %xv = insertelement <4 x i7> undef, i7 %x, i32 1
@@ -464,7 +464,7 @@ define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_extra_use(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 2
 ; CHECK-NEXT:    call void @use(<4 x float> [[XV]])
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 2, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 poison, i32 2, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> undef, float %x, i32 2
@@ -478,7 +478,7 @@ define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) {
 define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> %y) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_base(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i64 2
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 poison, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> %y, float %x, i32 2
@@ -491,7 +491,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float>
 define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) {
 ; CHECK-LABEL: @insert_nonzero_index_splat_wrong_index(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 [[INDEX:%.*]]
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SPLAT]]
 ;
   %xv = insertelement <4 x float> undef, float %x, i32 %index
@@ -502,7 +502,7 @@ define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index)
 define <4 x float> @insert_in_splat(float %x) {
 ; CHECK-LABEL: @insert_in_splat(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %xv = insertelement <4 x float> undef, float %x, i32 0
@@ -515,9 +515,9 @@ define <4 x float> @insert_in_splat_extra_uses(float %x) {
 ; CHECK-LABEL: @insert_in_splat_extra_uses(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0
 ; CHECK-NEXT:    call void @use(<4 x float> [[XV]])
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 poison, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    call void @use(<4 x float> [[SPLAT]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %xv = insertelement <4 x float> undef, float %x, i32 0
@@ -533,7 +533,7 @@ define <4 x float> @insert_in_splat_extra_uses(float %x) {
 define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) {
 ; CHECK-LABEL: @insert_in_splat_variable_index(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 poison, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i32 [[Y:%.*]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -548,7 +548,7 @@ define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) {
 define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) {
 ; CHECK-LABEL: @insert_in_nonsplat(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> [[Y:%.*]], <4 x i32> <i32 undef, i32 0, i32 4, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> [[Y:%.*]], <4 x i32> <i32 poison, i32 0, i32 4, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i64 3
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -563,7 +563,7 @@ define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) {
 define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) {
 ; CHECK-LABEL: @insert_in_nonsplat2(
 ; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i64 3
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -575,7 +575,7 @@ define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) {
 
 define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) {
 ; CHECK-LABEL: @shuf_identity_padding(
-; CHECK-NEXT:    [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2
 ; CHECK-NEXT:    ret <4 x i8> [[V2]]
 ;
@@ -588,7 +588,7 @@ define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) {
 
 define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) {
 ; CHECK-LABEL: @shuf_identity_extract(
-; CHECK-NEXT:    [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <3 x i8> [[V1]], i8 [[Y:%.*]], i64 2
 ; CHECK-NEXT:    ret <3 x i8> [[V2]]
 ;
@@ -601,9 +601,9 @@ define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) {
 
 define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) {
 ; CHECK-LABEL: @shuf_identity_extract_extra_use(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 3>
 ; CHECK-NEXT:    call void @use(<4 x float> [[V0]])
-; CHECK-NEXT:    [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> undef, <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[Y:%.*]], i64 1
 ; CHECK-NEXT:    ret <4 x float> [[V2]]
 ;
@@ -619,7 +619,7 @@ define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) {
 
 define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %index) {
 ; CHECK-LABEL: @shuf_identity_padding_variable_index(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 [[INDEX:%.*]]
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2
@@ -636,7 +636,7 @@ define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %i
 
 define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x i8> %other) {
 ; CHECK-LABEL: @shuf_identity_padding_wrong_source_vec(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[OTHER:%.*]], i64 1
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i64 1
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 2
@@ -653,7 +653,7 @@ define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x
 
 define <4 x i8> @shuf_identity_padding_wrong_index(<2 x i8> %x, i8 %y) {
 ; CHECK-LABEL: @shuf_identity_padding_wrong_index(
-; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i8> [[X]], i64 1
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i64 2
 ; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i64 3
@@ -696,7 +696,7 @@ define <5 x float> @insert_undemanded_element_unequal_length_op0(<4 x float> %x,
 ; CHECK-LABEL: @insert_undemanded_element_unequal_length_op0(
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i64 3
 ; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Y:%.*]], <5 x i32> <i32 undef, i32 0, i32 7, i32 1, i32 4>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Y:%.*]], <5 x i32> <i32 poison, i32 0, i32 7, i32 1, i32 4>
 ; CHECK-NEXT:    ret <5 x float> [[S]]
 ;
   %ins = insertelement <4 x float> %x, float 42.0, i32 3
@@ -709,7 +709,7 @@ define <5 x float> @insert_undemanded_element_unequal_length_op1(<4 x float> %x,
 ; CHECK-LABEL: @insert_undemanded_element_unequal_length_op1(
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i64 3
 ; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[X]], <5 x i32> <i32 undef, i32 3, i32 2, i32 1, i32 4>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[X]], <5 x i32> <i32 poison, i32 3, i32 2, i32 1, i32 4>
 ; CHECK-NEXT:    ret <5 x float> [[S]]
 ;
   %ins = insertelement <4 x float> %x, float 42.0, i32 3
@@ -754,7 +754,7 @@ define <5 x float> @insert_demanded_element_unequal_length_op0(<4 x float> %x, <
 ; CHECK-LABEL: @insert_demanded_element_unequal_length_op0(
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i64 3
 ; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[INS]], <4 x float> [[Y:%.*]], <5 x i32> <i32 undef, i32 3, i32 2, i32 1, i32 4>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[INS]], <4 x float> [[Y:%.*]], <5 x i32> <i32 poison, i32 3, i32 2, i32 1, i32 4>
 ; CHECK-NEXT:    ret <5 x float> [[S]]
 ;
   %ins = insertelement <4 x float> %x, float 42.0, i32 3
@@ -769,7 +769,7 @@ define <5 x float> @insert_demanded_element_unequal_length_op1(<4 x float> %x, <
 ; CHECK-LABEL: @insert_demanded_element_unequal_length_op1(
 ; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.300000e+01, i64 3
 ; CHECK-NEXT:    call void @use(<4 x float> [[INS]])
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[INS]], <5 x i32> <i32 undef, i32 0, i32 7, i32 1, i32 4>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[INS]], <5 x i32> <i32 poison, i32 0, i32 7, i32 1, i32 4>
 ; CHECK-NEXT:    ret <5 x float> [[S]]
 ;
   %ins = insertelement <4 x float> %x, float 43.0, i32 3
diff --git a/llvm/test/Transforms/InstCombine/is_fpclass.ll b/llvm/test/Transforms/InstCombine/is_fpclass.ll
index 12d6373d20f4a..95c4178e1718c 100644
--- a/llvm/test/Transforms/InstCombine/is_fpclass.ll
+++ b/llvm/test/Transforms/InstCombine/is_fpclass.ll
@@ -2465,8 +2465,7 @@ define i1 @test_class_is_zero_nonegzero_src(float nofpclass(nzero) %arg) {
 
 define i1 @test_class_is_pzero_nozero_src(float nofpclass(zero) %arg) {
 ; CHECK-LABEL: @test_class_is_pzero_nozero_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 64)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 64)
   ret i1 %class
@@ -2474,8 +2473,7 @@ define i1 @test_class_is_pzero_nozero_src(float nofpclass(zero) %arg) {
 
 define i1 @test_class_is_pzero_nopzero_src(float nofpclass(pzero) %arg) {
 ; CHECK-LABEL: @test_class_is_pzero_nopzero_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 64)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 64)
   ret i1 %class
@@ -2492,8 +2490,7 @@ define i1 @test_class_is_pzero_nonzero_src(float nofpclass(nzero) %arg) {
 
 define i1 @test_class_is_nzero_nozero_src(float nofpclass(zero) %arg) {
 ; CHECK-LABEL: @test_class_is_nzero_nozero_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 32)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 32)
   ret i1 %class
@@ -2510,8 +2507,7 @@ define i1 @test_class_is_nzero_nopzero_src(float nofpclass(pzero) %arg) {
 
 define i1 @test_class_is_nzero_nonzero_src(float nofpclass(nzero) %arg) {
 ; CHECK-LABEL: @test_class_is_nzero_nonzero_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 32)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 32)
   ret i1 %class
@@ -2519,7 +2515,7 @@ define i1 @test_class_is_nzero_nonzero_src(float nofpclass(nzero) %arg) {
 
 define i1 @test_class_is_normal_or_zero_nozero_src(float nofpclass(zero) %arg) {
 ; CHECK-LABEL: @test_class_is_normal_or_zero_nozero_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 360)
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 264)
 ; CHECK-NEXT:    ret i1 [[CLASS]]
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 360)
@@ -2616,8 +2612,7 @@ define i1 @test_class_is_posinf_noinf_src(float nofpclass(inf) %arg) {
 
 define i1 @test_class_is_subnormal_nosub_src(float nofpclass(sub) %arg) {
 ; CHECK-LABEL: @test_class_is_subnormal_nosub_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 144)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 144)
   ret i1 %class
@@ -2625,7 +2620,7 @@ define i1 @test_class_is_subnormal_nosub_src(float nofpclass(sub) %arg) {
 
 define i1 @test_class_is_subnormal_nonsub_src(float nofpclass(nsub) %arg) {
 ; CHECK-LABEL: @test_class_is_subnormal_nonsub_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 144)
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 128)
 ; CHECK-NEXT:    ret i1 [[CLASS]]
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 144)
@@ -2634,8 +2629,7 @@ define i1 @test_class_is_subnormal_nonsub_src(float nofpclass(nsub) %arg) {
 
 define i1 @test_class_is_not_subnormal_nosub_src(float nofpclass(sub) %arg) {
 ; CHECK-LABEL: @test_class_is_not_subnormal_nosub_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 879)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 879)
   ret i1 %class
@@ -2643,8 +2637,7 @@ define i1 @test_class_is_not_subnormal_nosub_src(float nofpclass(sub) %arg) {
 
 define i1 @test_class_is_not_negsubnormal_nosub_src(float nofpclass(sub) %arg) {
 ; CHECK-LABEL: @test_class_is_not_negsubnormal_nosub_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1007)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1007)
   ret i1 %class
@@ -2652,8 +2645,7 @@ define i1 @test_class_is_not_negsubnormal_nosub_src(float nofpclass(sub) %arg) {
 
 define i1 @test_class_is_not_negsubnormal_nonegsub_src(float nofpclass(nsub) %arg) {
 ; CHECK-LABEL: @test_class_is_not_negsubnormal_nonegsub_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1007)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1007)
   ret i1 %class
@@ -2661,8 +2653,7 @@ define i1 @test_class_is_not_negsubnormal_nonegsub_src(float nofpclass(nsub) %ar
 
 define i1 @test_class_is_nnormal_nonorm_src(float nofpclass(norm) %arg) {
 ; CHECK-LABEL: @test_class_is_nnormal_nonorm_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 264)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 264)
   ret i1 %class
@@ -2670,8 +2661,7 @@ define i1 @test_class_is_nnormal_nonorm_src(float nofpclass(norm) %arg) {
 
 define i1 @test_class_is_not_nnormal_nonorm_src(float nofpclass(norm) %arg) {
 ; CHECK-LABEL: @test_class_is_not_nnormal_nonorm_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 759)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 759)
   ret i1 %class
@@ -2679,8 +2669,7 @@ define i1 @test_class_is_not_nnormal_nonorm_src(float nofpclass(norm) %arg) {
 
 define i1 @test_class_is_not_nnormal_onlynorm_src(float nofpclass(nan inf zero sub) %arg) {
 ; CHECK-LABEL: @test_class_is_not_nnormal_onlynorm_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 240)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 759)
   ret i1 %class
@@ -2688,8 +2677,7 @@ define i1 @test_class_is_not_nnormal_onlynorm_src(float nofpclass(nan inf zero s
 
 define i1 @test_class_is_nnormal_onlynorm_src(float nofpclass(nan inf zero sub) %arg) {
 ; CHECK-LABEL: @test_class_is_nnormal_onlynorm_src(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 264)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 264)
   ret i1 %class
@@ -2700,8 +2688,7 @@ define i1 @test_class_is_normal_assume_normal(float %x) {
 ; CHECK-LABEL: @test_class_is_normal_assume_normal(
 ; CHECK-NEXT:    [[ASSUMED_IS_NORMAL:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X:%.*]], i32 264)
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUMED_IS_NORMAL]])
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 264)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %assumed.is.normal = call i1 @llvm.is.fpclass.f32(float %x, i32 264)
   call void @llvm.assume(i1 %assumed.is.normal)
@@ -2713,8 +2700,7 @@ define i1 @test_class_is_normal_assume_not_normal(float %x) {
 ; CHECK-LABEL: @test_class_is_normal_assume_not_normal(
 ; CHECK-NEXT:    [[ASSUMED_IS_NORMAL:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X:%.*]], i32 264)
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUMED_IS_NORMAL]])
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X]], i32 759)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 false
 ;
   %assumed.is.normal = call i1 @llvm.is.fpclass.f32(float %x, i32 264)
   call void @llvm.assume(i1 %assumed.is.normal)
@@ -2761,9 +2747,1186 @@ define i1 @test_class_is_nan_assume_not_eq_pinf(float %x) {
   ret i1 %class
 }
 
-declare i1 @llvm.is.fpclass.f32(float, i32 immarg)
-declare i1 @llvm.is.fpclass.f64(double, i32 immarg)
-declare <2 x i1> @llvm.is.fpclass.v2f32(<2 x float>, i32 immarg)
-declare float @llvm.fabs.f32(float)
-declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
-declare void @llvm.assume(i1 noundef)
+
+; --------------------------------------------------------------------
+; fcmp ogt/oge/olt/ole 0 ieee
+; --------------------------------------------------------------------
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 960)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 960)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 961)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 961)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 962)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 962)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 963)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 963)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 896)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 896)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 897)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 897)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 898)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 898)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 899)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 899)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 768)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 768)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pzero_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 704)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 704)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_pzero_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 707)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 707)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 992)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 992)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 993)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 993)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 994)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 994)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 995)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 995)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1017)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1017)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1009)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1009)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1010)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1010)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1011)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1011)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 928)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 928)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_nsub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_nzero_nsub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 816)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 816)
+  ret i1 %class
+}
+
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 63)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 63)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 62)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 62)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 61)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 61)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 960)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 960)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 127)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 127)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 126)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 126)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 125)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 125)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 124)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 124)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 255)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 255)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pzero_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 319)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 319)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_pzero_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 316)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 316)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 22)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 22)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 30)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 30)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 29)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 29)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 28)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 28)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 6)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 6)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_snan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_snan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 14)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 14)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_qnan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_qnan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 13)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 13)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_nan__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_nan__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 12)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 12)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_psub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nzero_psub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 95)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 95)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_nsub_pnorm_pinf__ieee(float %arg) #0 {
+; CHECK-LABEL: @test_class_is_not_nzero_nsub_pnorm_pinf__ieee(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 207)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 207)
+  ret i1 %class
+}
+
+; --------------------------------------------------------------------
+; fcmp ogt/oge/olt/ole 0 daz
+; --------------------------------------------------------------------
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 960)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 960)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 961)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 961)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 962)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 962)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 963)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 963)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 896)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 896)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 897)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 897)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 898)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 898)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 899)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 899)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 768)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 768)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pzero_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 704)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 704)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_pzero_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 707)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 707)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 992)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 992)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 993)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 993)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 994)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 994)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 995)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 995)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1017)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1017)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1009)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1009)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1010)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1010)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1011)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1011)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 928)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 928)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_nsub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_nzero_nsub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 816)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 816)
+  ret i1 %class
+}
+
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 63)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 63)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 62)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 62)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 61)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 61)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 960)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 960)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 127)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 127)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 126)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 126)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 125)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 125)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 124)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 124)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 255)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 255)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pzero_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 319)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 319)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_pzero_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 316)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 316)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 22)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 22)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 30)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 30)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 29)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 29)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 28)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 28)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 6)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 6)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_snan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_snan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 14)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 14)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_qnan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_qnan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 13)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 13)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_nan__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_nan__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 12)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 12)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_psub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nzero_psub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 95)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 95)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_nsub_pnorm_pinf__daz(float %arg) #1 {
+; CHECK-LABEL: @test_class_is_not_nzero_nsub_pnorm_pinf__daz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 207)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 207)
+  ret i1 %class
+}
+
+; --------------------------------------------------------------------
+; fcmp ogt/oge/olt/ole 0 dapz
+; --------------------------------------------------------------------
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 960)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 960)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 961)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 961)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 962)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 962)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pzero_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 963)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 963)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 896)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 896)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 897)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 897)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 898)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 898)
+  ret i1 %class
+}
+
+define i1 @test_class_is_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 899)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 899)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 768)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 768)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pzero_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 704)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 704)
+  ret i1 %class
+}
+
+define i1 @test_class_is_pzero_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_pzero_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 707)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 707)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 992)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 992)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 993)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 993)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 994)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 994)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_pzero_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nzero_pzero_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 995)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 995)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1017)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1017)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1009)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1009)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1010)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1010)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nsub_nzero_pzero_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 1011)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 1011)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 928)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 928)
+  ret i1 %class
+}
+
+define i1 @test_class_is_nzero_nsub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_nzero_nsub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 816)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 816)
+  ret i1 %class
+}
+
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 63)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 63)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 62)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 62)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 61)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 61)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pzero_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 960)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 960)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 127)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 127)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 126)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 126)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 125)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 125)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 124)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 124)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 255)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 255)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pzero_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 319)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 319)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_pzero_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_pzero_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 316)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 316)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 22)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 22)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 30)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 30)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 29)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 29)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_pzero_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nzero_pzero_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 28)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 28)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 6)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 6)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_snan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_snan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 14)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 14)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_qnan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_qnan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 13)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 13)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_nan__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nsub_nzero_pzero_psub_pnorm_pinf_nan__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 12)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 12)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_psub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nzero_psub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 95)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 95)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_nzero_nsub_pnorm_pinf__dapz(float %arg) #2 {
+; CHECK-LABEL: @test_class_is_not_nzero_nsub_pnorm_pinf__dapz(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 207)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 207)
+  ret i1 %class
+}
+
+; --------------------------------------------------------------------
+; fcmp ogt/oge/olt/ole 0 dynamic
+; --------------------------------------------------------------------
+
+define i1 @test_class_is_psub_pnorm_pinf__dynamic(float %arg) #3 {
+; CHECK-LABEL: @test_class_is_psub_pnorm_pinf__dynamic(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 896)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 896)
+  ret i1 %class
+}
+
+define i1 @test_class_is_not_psub_pnorm_pinf__dynamic(float %arg) #3 {
+; CHECK-LABEL: @test_class_is_not_psub_pnorm_pinf__dynamic(
+; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG:%.*]], i32 127)
+; CHECK-NEXT:    ret i1 [[CLASS]]
+;
+  %class = call i1 @llvm.is.fpclass.f32(float %arg, i32 127)
+  ret i1 %class
+}
+
+declare i1 @llvm.is.fpclass.f32(float, i32 immarg)
+declare i1 @llvm.is.fpclass.f64(double, i32 immarg)
+declare <2 x i1> @llvm.is.fpclass.v2f32(<2 x float>, i32 immarg)
+declare float @llvm.fabs.f32(float)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+declare void @llvm.assume(i1 noundef)
+
+attributes #0 = { "denormal-fp-math"="dynamic,ieee" }
+attributes #1 = { "denormal-fp-math"="dynamic,preserve-sign" }
+attributes #2 = { "denormal-fp-math"="dynamic,positive-zero" }
+attributes #3 = { "denormal-fp-math"="dynamic,dynamic" }
diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
index 58d7d08594b3f..af9406ce382ca 100644
--- a/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
@@ -202,7 +202,7 @@ define <4 x double> @gather_lane2(ptr %base, double %pt)  {
 ; CHECK-LABEL: @gather_lane2(
 ; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> <i64 poison, i64 poison, i64 2, i64 poison>
 ; CHECK-NEXT:    [[PT_V1:%.*]] = insertelement <4 x double> poison, double [[PT:%.*]], i64 0
-; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 0>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x double> [[PT_V2]])
 ; CHECK-NEXT:    ret <4 x double> [[RES]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
index 12bd8660d574f..6155e0afa1001 100644
--- a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
@@ -202,7 +202,7 @@ define <4 x double> @gather_lane2(ptr %base, double %pt)  {
 ; CHECK-LABEL: @gather_lane2(
 ; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> <i64 poison, i64 poison, i64 2, i64 poison>
 ; CHECK-NEXT:    [[PT_V1:%.*]] = insertelement <4 x double> undef, double [[PT:%.*]], i64 0
-; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 poison, i32 0>
 ; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> [[PTRS]], i32 4, <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x double> [[PT_V2]])
 ; CHECK-NEXT:    ret <4 x double> [[RES]]
 ;
@@ -346,7 +346,7 @@ entry:
 define void @negative_scatter_v4i16_no_uniform_vals_uniform_ptrs_all_inactive_mask(ptr %dst, ptr %src) {
 ; CHECK-LABEL: @negative_scatter_v4i16_no_uniform_vals_uniform_ptrs_all_inactive_mask(
 ; CHECK-NEXT:    [[INSERT_ELT:%.*]] = insertelement <4 x ptr> poison, ptr [[DST:%.*]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[INSERT_ELT]], <4 x ptr> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 0>
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[INSERT_ELT]], <4 x ptr> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[SRC:%.*]], align 2
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v4i16.v4p0(<4 x i16> [[WIDE_LOAD]], <4 x ptr> [[BROADCAST_SPLAT]], i32 2, <4 x i1> <i1 false, i1 false, i1 true, i1 true>)
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
index b2a559ea63f64..7a4da66ae2151 100644
--- a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
@@ -2377,10 +2377,10 @@ define <3 x i8> @smax_unary_shuffle_ops(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i8> @smin_unary_shuffle_ops_use_poison_mask_elt(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @smin_unary_shuffle_ops_use_poison_mask_elt(
-; CHECK-NEXT:    [[SX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    [[SX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> <i32 poison, i32 0, i32 2>
 ; CHECK-NEXT:    call void @use_vec(<3 x i8> [[SX]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <3 x i8> @llvm.smin.v3i8(<3 x i8> [[X]], <3 x i8> [[Y:%.*]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[TMP1]], <3 x i8> poison, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[TMP1]], <3 x i8> poison, <3 x i32> <i32 poison, i32 0, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %sx = shufflevector <3 x i8> %x, <3 x i8> poison, <3 x i32> <i32 poison, i32 0, i32 2>
diff --git a/llvm/test/Transforms/InstCombine/nsw-inseltpoison.ll b/llvm/test/Transforms/InstCombine/nsw-inseltpoison.ll
index cb1fb109a3095..8a508af632962 100644
--- a/llvm/test/Transforms/InstCombine/nsw-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/nsw-inseltpoison.ll
@@ -118,7 +118,7 @@ define <3 x i32> @shl_nuw_nsw_shuffle_splat_vec(<2 x i8> %x) {
 define <3 x i32> @shl_nuw_nsw_shuffle_undef_elt_splat_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @shl_nuw_nsw_shuffle_undef_elt_splat_vec(
 ; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> poison, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> poison, <3 x i32> <i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[SHUF]], <i32 17, i32 17, i32 17>
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/nsw.ll b/llvm/test/Transforms/InstCombine/nsw.ll
index 651579d67e630..c3c7b127a464c 100644
--- a/llvm/test/Transforms/InstCombine/nsw.ll
+++ b/llvm/test/Transforms/InstCombine/nsw.ll
@@ -118,7 +118,7 @@ define <3 x i32> @shl_nuw_nsw_shuffle_splat_vec(<2 x i8> %x) {
 define <3 x i32> @shl_nuw_nsw_shuffle_undef_elt_splat_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @shl_nuw_nsw_shuffle_undef_elt_splat_vec(
 ; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> undef, <3 x i32> <i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[SHUF]], <i32 17, i32 17, i32 17>
 ; CHECK-NEXT:    ret <3 x i32> [[T3]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/reduction-shufflevector.ll b/llvm/test/Transforms/InstCombine/reduction-shufflevector.ll
index 00dd6de038d10..4f887d1f28c64 100644
--- a/llvm/test/Transforms/InstCombine/reduction-shufflevector.ll
+++ b/llvm/test/Transforms/InstCombine/reduction-shufflevector.ll
@@ -159,7 +159,7 @@ define i32 @reduce_and_failed(<4 x i32> %x) {
 
 define i32 @reduce_xor_failed(<4 x i32> %x) {
 ; CHECK-LABEL: @reduce_xor_failed(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[SHUF]])
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -181,7 +181,7 @@ define i32 @reduce_umax_failed(<2 x i32> %x, <2 x i32> %y) {
 
 define i32 @reduce_umin_failed(<2 x i32> %x) {
 ; CHECK-LABEL: @reduce_umin_failed(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> [[SHUF]])
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
@@ -225,7 +225,7 @@ define float @reduce_fmax_failed(<4 x float> %x) {
 
 define float @reduce_fmin_failed(<4 x float> %x) {
 ; CHECK-LABEL: @reduce_fmin_failed(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[SHUF]])
 ; CHECK-NEXT:    ret float [[RES]]
 ;
@@ -247,7 +247,7 @@ define float @reduce_fadd_failed(float %a, <4 x float> %x) {
 
 define float @reduce_fmul_failed(float %a, <2 x float> %x) {
 ; CHECK-LABEL: @reduce_fmul_failed(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
 ; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.vector.reduce.fmul.v4f32(float [[A:%.*]], <4 x float> [[SHUF]])
 ; CHECK-NEXT:    ret float [[RES]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/rem.ll b/llvm/test/Transforms/InstCombine/rem.ll
index 5ab4b7bee78eb..b5a71681522a3 100644
--- a/llvm/test/Transforms/InstCombine/rem.ll
+++ b/llvm/test/Transforms/InstCombine/rem.ll
@@ -66,8 +66,9 @@ define i5 @biggest_divisor(i5 %x) {
 
 define i8 @urem_with_sext_bool_divisor(i1 %x, i8 %y) {
 ; CHECK-LABEL: @urem_with_sext_bool_divisor(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[Y:%.*]], -1
-; CHECK-NEXT:    [[REM:%.*]] = select i1 [[TMP1]], i8 0, i8 [[Y]]
+; CHECK-NEXT:    [[Y_FROZEN:%.*]] = freeze i8 [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[Y_FROZEN]], -1
+; CHECK-NEXT:    [[REM:%.*]] = select i1 [[TMP1]], i8 0, i8 [[Y_FROZEN]]
 ; CHECK-NEXT:    ret i8 [[REM]]
 ;
   %s = sext i1 %x to i8
@@ -77,8 +78,9 @@ define i8 @urem_with_sext_bool_divisor(i1 %x, i8 %y) {
 
 define <2 x i8> @urem_with_sext_bool_divisor_vec(<2 x i1> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @urem_with_sext_bool_divisor_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
-; CHECK-NEXT:    [[REM:%.*]] = select <2 x i1> [[TMP1]], <2 x i8> zeroinitializer, <2 x i8> [[Y]]
+; CHECK-NEXT:    [[Y_FROZEN:%.*]] = freeze <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[Y_FROZEN]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[REM:%.*]] = select <2 x i1> [[TMP1]], <2 x i8> zeroinitializer, <2 x i8> [[Y_FROZEN]]
 ; CHECK-NEXT:    ret <2 x i8> [[REM]]
 ;
   %s = sext <2 x i1> %x to <2 x i8>
@@ -1020,3 +1022,16 @@ define i32 @urem_select_of_constants_divisor(i1 %b, i32 %x) {
   %r = urem i32 %x, %s
   ret i32 %r
 }
+
+; https://alive2.llvm.org/ce/z/bh2KHm
+define <2 x i32> @PR62401(<2 x i1> %x, <2 x i32> %y) {
+; CHECK-LABEL: @PR62401(
+; CHECK-NEXT:    [[Y_FROZEN:%.*]] = freeze <2 x i32> [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[Y_FROZEN]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> zeroinitializer, <2 x i32> [[Y_FROZEN]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %sext.i1 = sext <2 x i1> %x to <2 x i32>
+  %r = urem <2 x i32> %y, %sext.i1
+  ret <2 x i32> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
index 2a210d659e66f..7b7e7c7960c1c 100644
--- a/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
+++ b/llvm/test/Transforms/InstCombine/select-ctlz-to-cttz.ll
@@ -141,6 +141,7 @@ define i32 @select_clz_to_ctz_wrong_sub(i32 %a) {
   ret i32 %cond
 }
 
+; TODO: https://alive2.llvm.org/ce/z/X6QjcB
 define i64 @select_clz_to_ctz_i64_wrong_xor(i64 %a) {
 ; CHECK-LABEL: @select_clz_to_ctz_i64_wrong_xor(
 ; CHECK-NEXT:    [[SUB:%.*]] = sub i64 0, [[A:%.*]]
diff --git a/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll
index ec822f8952fea..3b4977f1927ab 100644
--- a/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll
@@ -109,11 +109,11 @@ define <4 x float> @simple_vector_select(<4 x float> %a, <4 x float> %b, <4 x i3
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[C]], i64 1
 ; CHECK-NEXT:    [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[A_SINK1:%.*]] = select i1 [[TOBOOL1_NOT]], <4 x float> [[B]], <4 x float> [[A]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[C]], i64 2
 ; CHECK-NEXT:    [[TOBOOL6_NOT:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    [[A_SINK2:%.*]] = select i1 [[TOBOOL6_NOT]], <4 x float> [[B]], <4 x float> [[A]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[C]], i64 3
 ; CHECK-NEXT:    [[TOBOOL11_NOT:%.*]] = icmp eq i32 [[TMP5]], 0
 ; CHECK-NEXT:    [[A_SINK3:%.*]] = select i1 [[TOBOOL11_NOT]], <4 x float> [[B]], <4 x float> [[A]]
diff --git a/llvm/test/Transforms/InstCombine/select-extractelement.ll b/llvm/test/Transforms/InstCombine/select-extractelement.ll
index bc88142b854ea..51e0451cb6dcd 100644
--- a/llvm/test/Transforms/InstCombine/select-extractelement.ll
+++ b/llvm/test/Transforms/InstCombine/select-extractelement.ll
@@ -109,11 +109,11 @@ define <4 x float> @simple_vector_select(<4 x float> %a, <4 x float> %b, <4 x i3
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[C]], i64 1
 ; CHECK-NEXT:    [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[A_SINK1:%.*]] = select i1 [[TOBOOL1_NOT]], <4 x float> [[B]], <4 x float> [[A]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[C]], i64 2
 ; CHECK-NEXT:    [[TOBOOL6_NOT:%.*]] = icmp eq i32 [[TMP3]], 0
 ; CHECK-NEXT:    [[A_SINK2:%.*]] = select i1 [[TOBOOL6_NOT]], <4 x float> [[B]], <4 x float> [[A]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[C]], i64 3
 ; CHECK-NEXT:    [[TOBOOL11_NOT:%.*]] = icmp eq i32 [[TMP5]], 0
 ; CHECK-NEXT:    [[A_SINK3:%.*]] = select i1 [[TOBOOL11_NOT]], <4 x float> [[B]], <4 x float> [[A]]
diff --git a/llvm/test/Transforms/InstCombine/select-select.ll b/llvm/test/Transforms/InstCombine/select-select.ll
index 7d7b67ad89011..785766136fb71 100644
--- a/llvm/test/Transforms/InstCombine/select-select.ll
+++ b/llvm/test/Transforms/InstCombine/select-select.ll
@@ -115,7 +115,7 @@ define <4 x i8> @sel_shuf_use(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
 
 define <4 x i8> @sel_shuf_undef(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @sel_shuf_undef(
-; CHECK-NEXT:    [[BLEND:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[BLEND:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i8> [[BLEND]], <4 x i8> [[Y]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
index 91036b60c270c..b45ad7fa29ffb 100644
--- a/llvm/test/Transforms/InstCombine/shift.ll
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -1713,12 +1713,13 @@ define void @ashr_out_of_range(ptr %A) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 -1, i64 -2
 ; CHECK-NEXT:    [[G11:%.*]] = getelementptr i177, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[L7:%.*]] = load i177, ptr [[G11]], align 4
-; CHECK-NEXT:    [[C171:%.*]] = icmp slt i177 [[L7]], 0
-; CHECK-NEXT:    [[C17:%.*]] = select i1 [[TMP1]], i1 [[C171]], i1 false
+; CHECK-NEXT:    [[L7_FROZEN:%.*]] = freeze i177 [[L7]]
+; CHECK-NEXT:    [[C171:%.*]] = icmp slt i177 [[L7_FROZEN]], 0
+; CHECK-NEXT:    [[C17:%.*]] = and i1 [[TMP1]], [[C171]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = sext i1 [[C17]] to i64
 ; CHECK-NEXT:    [[G62:%.*]] = getelementptr i177, ptr [[G11]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i177 [[L7]], -1
-; CHECK-NEXT:    [[B28:%.*]] = select i1 [[TMP4]], i177 0, i177 [[L7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i177 [[L7_FROZEN]], -1
+; CHECK-NEXT:    [[B28:%.*]] = select i1 [[TMP4]], i177 0, i177 [[L7_FROZEN]]
 ; CHECK-NEXT:    store i177 [[B28]], ptr [[G62]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/shuffle-binop.ll b/llvm/test/Transforms/InstCombine/shuffle-binop.ll
index b298471a9428d..8460f8b2c6cd3 100644
--- a/llvm/test/Transforms/InstCombine/shuffle-binop.ll
+++ b/llvm/test/Transforms/InstCombine/shuffle-binop.ll
@@ -3,7 +3,7 @@
 
 define <4 x i8> @splat_binop_non_splat_x(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @splat_binop_non_splat_x(
-; CHECK-NEXT:    [[XSPLAT:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[XSPLAT:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use(<4 x i8> [[XSPLAT]])
 ; CHECK-NEXT:    [[B:%.*]] = add <4 x i8> [[XSPLAT]], [[Y:%.*]]
 ; CHECK-NEXT:    [[BSPLAT:%.*]] = shufflevector <4 x i8> [[B]], <4 x i8> poison, <4 x i32> zeroinitializer
@@ -21,7 +21,7 @@ define <4 x i8> @non_splat_binop_splat_x(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-NEXT:    [[XSPLAT:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    call void @use(<4 x i8> [[XSPLAT]])
 ; CHECK-NEXT:    [[B:%.*]] = sub <4 x i8> [[XSPLAT]], [[Y:%.*]]
-; CHECK-NEXT:    [[BSPLAT:%.*]] = shufflevector <4 x i8> [[B]], <4 x i8> poison, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BSPLAT:%.*]] = shufflevector <4 x i8> [[B]], <4 x i8> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[BSPLAT]]
 ;
   %xsplat = shufflevector <4 x i8> %x, <4 x i8> poison, <4 x i32> zeroinitializer
@@ -66,7 +66,7 @@ define <4 x i8> @splat_binop_splat_y(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-NEXT:    [[YSPLAT:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    call void @use(<4 x i8> [[YSPLAT]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub <4 x i8> [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[BSPLAT:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[BSPLAT:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x i8> [[BSPLAT]]
 ;
   %ysplat = shufflevector <4 x i8> %y, <4 x i8> poison, <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstCombine/shuffle-select-narrow-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shuffle-select-narrow-inseltpoison.ll
index acf7f28e1c632..3f050887c7fc9 100644
--- a/llvm/test/Transforms/InstCombine/shuffle-select-narrow-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/shuffle-select-narrow-inseltpoison.ll
@@ -35,8 +35,8 @@ define <2 x i8> @narrow_shuffle_of_select_overspecified_extend(<2 x i1> %cmp, <4
 
 define <3 x float> @narrow_shuffle_of_select_undefs(<3 x i1> %cmp, <4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_undefs(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = select <3 x i1> [[CMP:%.*]], <3 x float> [[TMP1]], <3 x float> [[TMP2]]
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
@@ -53,7 +53,7 @@ declare void @use_cmp(<4 x i1>)
 
 define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_use1(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    call void @use(<4 x i8> [[WIDESEL]])
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
@@ -70,7 +70,7 @@ define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x
 
 define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_use2(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use_cmp(<4 x i1> [[WIDECMP]])
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
@@ -87,7 +87,7 @@ define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x
 
 define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types1(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> poison, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
@@ -102,7 +102,7 @@ define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8
 
 define <3 x i8> @narrow_shuffle_of_select_mismatch_types2(<4 x i1> %cmp, <6 x i8> %x, <6 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types2(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <6 x i1> [[WIDECMP]], <6 x i8> [[X:%.*]], <6 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <6 x i8> [[WIDESEL]], <6 x i8> poison, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
diff --git a/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll b/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll
index 9b265c3d8f1ef..98d81f1030bd4 100644
--- a/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll
+++ b/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll
@@ -35,8 +35,8 @@ define <2 x i8> @narrow_shuffle_of_select_overspecified_extend(<2 x i1> %cmp, <4
 
 define <3 x float> @narrow_shuffle_of_select_undefs(<3 x i1> %cmp, <4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_undefs(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = select <3 x i1> [[CMP:%.*]], <3 x float> [[TMP1]], <3 x float> [[TMP2]]
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
@@ -53,7 +53,7 @@ declare void @use_cmp(<4 x i1>)
 
 define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_use1(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    call void @use(<4 x i8> [[WIDESEL]])
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
@@ -70,7 +70,7 @@ define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x
 
 define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_use2(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use_cmp(<4 x i1> [[WIDECMP]])
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
@@ -87,7 +87,7 @@ define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x
 
 define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types1(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
@@ -102,7 +102,7 @@ define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8
 
 define <3 x i8> @narrow_shuffle_of_select_mismatch_types2(<4 x i1> %cmp, <6 x i8> %x, <6 x i8> %y) {
 ; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types2(
-; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[WIDESEL:%.*]] = select <6 x i1> [[WIDECMP]], <6 x i8> [[X:%.*]], <6 x i8> [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <6 x i8> [[WIDESEL]], <6 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
diff --git a/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll
index e6420e35a2cf0..44ec77e471bb4 100644
--- a/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll
@@ -240,7 +240,7 @@ define <4 x i32> @udiv_exact(<4 x i32> %v) {
 define <4 x i32> @udiv_undef_mask_elt(<4 x i32> %v) {
 ; CHECK-LABEL: @udiv_undef_mask_elt(
 ; CHECK-NEXT:    [[B:%.*]] = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
@@ -251,7 +251,7 @@ define <4 x i32> @udiv_undef_mask_elt(<4 x i32> %v) {
 define <4 x i32> @udiv_exact_undef_mask_elt(<4 x i32> %v) {
 ; CHECK-LABEL: @udiv_exact_undef_mask_elt(
 ; CHECK-NEXT:    [[B:%.*]] = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
@@ -315,7 +315,7 @@ define <4 x i32> @urem(<4 x i32> %v) {
 define <4 x i32> @urem_undef_mask_elt(<4 x i32> %v) {
 ; CHECK-LABEL: @urem_undef_mask_elt(
 ; CHECK-NEXT:    [[B:%.*]] = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
@@ -349,7 +349,7 @@ define <4 x float> @fadd(<4 x float> %v) {
 define <4 x double> @fsub(<4 x double> %v) {
 ; CHECK-LABEL: @fsub(
 ; CHECK-NEXT:    [[B:%.*]] = fsub <4 x double> <double poison, double poison, double 4.300000e+01, double 4.400000e+01>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x double> [[S]]
 ;
   %b = fsub <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
@@ -372,7 +372,7 @@ define <4 x float> @fmul(<4 x float> %v) {
 define <4 x double> @fdiv_constant_op0(<4 x double> %v) {
 ; CHECK-LABEL: @fdiv_constant_op0(
 ; CHECK-NEXT:    [[B:%.*]] = fdiv fast <4 x double> <double poison, double poison, double 4.300000e+01, double 4.400000e+01>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x double> [[S]]
 ;
   %b = fdiv fast <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
@@ -842,7 +842,7 @@ define <4 x i32> @sub_2_vars_nsw(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sub_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sub_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -856,7 +856,7 @@ define <4 x i32> @sub_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sub_2_vars_nsw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sub_2_vars_nsw_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -895,7 +895,7 @@ define <4 x i32> @mul_2_vars_nuw(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @mul_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @mul_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 undef, i32 3, i32 8>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -909,7 +909,7 @@ define <4 x i32> @mul_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @mul_2_vars_nuw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @mul_2_vars_nuw_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 undef, i32 3, i32 8>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -949,7 +949,7 @@ define <4 x i32> @shl_2_vars_nsw(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @shl_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @shl_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 5, i32 2, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[TMP1]], <i32 0, i32 6, i32 3, i32 0>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -963,7 +963,7 @@ define <4 x i32> @shl_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @shl_2_vars_nsw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @shl_2_vars_nsw_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 5, i32 2, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = shl nsw <4 x i32> [[TMP1]], <i32 0, i32 6, i32 3, i32 0>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1005,7 +1005,7 @@ define <4 x i32> @lshr_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @lshr_2_vars_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = lshr <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1020,7 +1020,7 @@ define <4 x i32> @lshr_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1)
 ; CHECK-LABEL: @lshr_2_vars_exact_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = lshr exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1045,7 +1045,7 @@ define <3 x i32> @ashr_2_vars(<3 x i32> %v0, <3 x i32> %v1) {
 
 define <3 x i42> @and_2_vars(<3 x i42> %v0, <3 x i42> %v1) {
 ; CHECK-LABEL: @and_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i42> [[V0:%.*]], <3 x i42> [[V1:%.*]], <3 x i32> <i32 0, i32 4, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i42> [[V0:%.*]], <3 x i42> [[V1:%.*]], <3 x i32> <i32 0, i32 4, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = and <3 x i42> [[TMP1]], <i42 1, i42 5, i42 undef>
 ; CHECK-NEXT:    ret <3 x i42> [[T3]]
 ;
@@ -1123,7 +1123,7 @@ define <4 x i32> @udiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @udiv_2_vars_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1138,7 +1138,7 @@ define <4 x i32> @udiv_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1)
 ; CHECK-LABEL: @udiv_2_vars_exact_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = udiv exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1177,7 +1177,7 @@ define <4 x i32> @sdiv_2_vars_exact(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sdiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sdiv_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1191,7 +1191,7 @@ define <4 x i32> @sdiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sdiv_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sdiv_2_vars_exact_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = sdiv exact <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1219,7 +1219,7 @@ define <4 x i32> @srem_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @srem_2_vars(
 ; CHECK-NEXT:    [[T1:%.*]] = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = srem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 0, i32 undef, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 0, i32 poison, i32 6, i32 3>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1244,7 +1244,7 @@ define <4 x float> @fadd_2_vars(<4 x float> %v0, <4 x float> %v1) {
 
 define <4 x double> @fsub_2_vars(<4 x double> %v0, <4 x double> %v1) {
 ; CHECK-LABEL: @fsub_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = fsub <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x double> [[T3]]
 ;
@@ -1270,7 +1270,7 @@ define <4 x float> @fmul_2_vars(<4 x float> %v0, <4 x float> %v1) {
 
 define <4 x double> @frem_2_vars(<4 x double> %v0, <4 x double> %v1) {
 ; CHECK-LABEL: @frem_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = frem <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x double> [[T3]]
 ;
@@ -1365,7 +1365,7 @@ define <4 x i32> @mul_shl_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @shl_mul_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @shl_mul_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 undef, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 poison, i32 6, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 5, i32 undef, i32 8, i32 16>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1458,7 +1458,7 @@ define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) {
 
 define <4 x i32> @PR41419(<4 x i32> %v) {
 ; CHECK-LABEL: @PR41419(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> <i32 undef, i32 undef, i32 2, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %s = shufflevector <4 x i32> %v, <4 x i32> poison, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll
index 49283cca3d817..233d9b008bc79 100644
--- a/llvm/test/Transforms/InstCombine/shuffle_select.ll
+++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll
@@ -240,7 +240,7 @@ define <4 x i32> @udiv_exact(<4 x i32> %v) {
 define <4 x i32> @udiv_undef_mask_elt(<4 x i32> %v) {
 ; CHECK-LABEL: @udiv_undef_mask_elt(
 ; CHECK-NEXT:    [[B:%.*]] = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
@@ -251,7 +251,7 @@ define <4 x i32> @udiv_undef_mask_elt(<4 x i32> %v) {
 define <4 x i32> @udiv_exact_undef_mask_elt(<4 x i32> %v) {
 ; CHECK-LABEL: @udiv_exact_undef_mask_elt(
 ; CHECK-NEXT:    [[B:%.*]] = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
@@ -315,7 +315,7 @@ define <4 x i32> @urem(<4 x i32> %v) {
 define <4 x i32> @urem_undef_mask_elt(<4 x i32> %v) {
 ; CHECK-LABEL: @urem_undef_mask_elt(
 ; CHECK-NEXT:    [[B:%.*]] = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %b = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
@@ -349,7 +349,7 @@ define <4 x float> @fadd(<4 x float> %v) {
 define <4 x double> @fsub(<4 x double> %v) {
 ; CHECK-LABEL: @fsub(
 ; CHECK-NEXT:    [[B:%.*]] = fsub <4 x double> <double poison, double poison, double 4.300000e+01, double 4.400000e+01>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x double> [[S]]
 ;
   %b = fsub <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
@@ -372,7 +372,7 @@ define <4 x float> @fmul(<4 x float> %v) {
 define <4 x double> @fdiv_constant_op0(<4 x double> %v) {
 ; CHECK-LABEL: @fdiv_constant_op0(
 ; CHECK-NEXT:    [[B:%.*]] = fdiv fast <4 x double> <double poison, double poison, double 4.300000e+01, double 4.400000e+01>, [[V:%.*]]
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x double> [[S]]
 ;
   %b = fdiv fast <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
@@ -842,7 +842,7 @@ define <4 x i32> @sub_2_vars_nsw(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sub_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sub_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -856,7 +856,7 @@ define <4 x i32> @sub_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sub_2_vars_nsw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sub_2_vars_nsw_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -895,7 +895,7 @@ define <4 x i32> @mul_2_vars_nuw(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @mul_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @mul_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 undef, i32 3, i32 8>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -909,7 +909,7 @@ define <4 x i32> @mul_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @mul_2_vars_nuw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @mul_2_vars_nuw_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 undef, i32 3, i32 8>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -949,7 +949,7 @@ define <4 x i32> @shl_2_vars_nsw(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @shl_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @shl_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 5, i32 2, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[TMP1]], <i32 0, i32 6, i32 3, i32 0>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -963,7 +963,7 @@ define <4 x i32> @shl_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @shl_2_vars_nsw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @shl_2_vars_nsw_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 poison, i32 5, i32 2, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = shl nsw <4 x i32> [[TMP1]], <i32 0, i32 6, i32 3, i32 0>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1005,7 +1005,7 @@ define <4 x i32> @lshr_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @lshr_2_vars_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = lshr <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1020,7 +1020,7 @@ define <4 x i32> @lshr_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1)
 ; CHECK-LABEL: @lshr_2_vars_exact_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = lshr exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1045,7 +1045,7 @@ define <3 x i32> @ashr_2_vars(<3 x i32> %v0, <3 x i32> %v1) {
 
 define <3 x i42> @and_2_vars(<3 x i42> %v0, <3 x i42> %v1) {
 ; CHECK-LABEL: @and_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i42> [[V0:%.*]], <3 x i42> [[V1:%.*]], <3 x i32> <i32 0, i32 4, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i42> [[V0:%.*]], <3 x i42> [[V1:%.*]], <3 x i32> <i32 0, i32 4, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = and <3 x i42> [[TMP1]], <i42 1, i42 5, i42 undef>
 ; CHECK-NEXT:    ret <3 x i42> [[T3]]
 ;
@@ -1076,7 +1076,7 @@ define <4 x i32> @or_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @or_2_vars_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = or <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 3, i32 4>
 ; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T1]])
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = or <4 x i32> [[TMP1]], <i32 5, i32 6, i32 3, i32 undef>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1138,7 +1138,7 @@ define <4 x i32> @udiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @udiv_2_vars_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1153,7 +1153,7 @@ define <4 x i32> @udiv_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1)
 ; CHECK-LABEL: @udiv_2_vars_exact_undef_mask_elt(
 ; CHECK-NEXT:    [[T1:%.*]] = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = udiv exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1192,7 +1192,7 @@ define <4 x i32> @sdiv_2_vars_exact(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sdiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sdiv_2_vars_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1206,7 +1206,7 @@ define <4 x i32> @sdiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @sdiv_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @sdiv_2_vars_exact_undef_mask_elt(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 poison>
 ; CHECK-NEXT:    [[T3:%.*]] = sdiv exact <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1234,7 +1234,7 @@ define <4 x i32> @srem_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @srem_2_vars(
 ; CHECK-NEXT:    [[T1:%.*]] = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
 ; CHECK-NEXT:    [[T2:%.*]] = srem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 0, i32 undef, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 0, i32 poison, i32 6, i32 3>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
   %t1 = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
@@ -1259,7 +1259,7 @@ define <4 x float> @fadd_2_vars(<4 x float> %v0, <4 x float> %v1) {
 
 define <4 x double> @fsub_2_vars(<4 x double> %v0, <4 x double> %v1) {
 ; CHECK-LABEL: @fsub_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = fsub <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x double> [[T3]]
 ;
@@ -1285,7 +1285,7 @@ define <4 x float> @fmul_2_vars(<4 x float> %v0, <4 x float> %v1) {
 
 define <4 x double> @frem_2_vars(<4 x double> %v0, <4 x double> %v1) {
 ; CHECK-LABEL: @frem_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 poison, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = frem <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    ret <4 x double> [[T3]]
 ;
@@ -1380,7 +1380,7 @@ define <4 x i32> @mul_shl_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
 
 define <4 x i32> @shl_mul_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-LABEL: @shl_mul_2_vars(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 undef, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 poison, i32 6, i32 7>
 ; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 5, i32 undef, i32 8, i32 16>
 ; CHECK-NEXT:    ret <4 x i32> [[T3]]
 ;
@@ -1521,7 +1521,7 @@ define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) {
 
 define <4 x i32> @PR41419(<4 x i32> %v) {
 ; CHECK-LABEL: @PR41419(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 undef, i32 2, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> undef, <4 x i32> <i32 poison, i32 poison, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %s = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
@@ -1573,7 +1573,7 @@ define <5 x i4> @sel_common_op_commute3(<5 x i4> %x, <5 x i4> %y) {
 
 define <5 x i4> @sel_common_op_commute3_poison_mask_elts(<5 x i4> %x, <5 x i4> %y) {
 ; CHECK-LABEL: @sel_common_op_commute3_poison_mask_elts(
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <5 x i4> [[Y:%.*]], <5 x i4> [[X:%.*]], <5 x i32> <i32 0, i32 6, i32 undef, i32 undef, i32 9>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <5 x i4> [[Y:%.*]], <5 x i4> [[X:%.*]], <5 x i32> <i32 0, i32 6, i32 poison, i32 poison, i32 9>
 ; CHECK-NEXT:    ret <5 x i4> [[S2]]
 ;
   %s1 = shufflevector <5 x i4> %y, <5 x i4> %x, <5 x i32> <i32 0, i32 6, i32 2, i32 poison, i32 9>
@@ -1585,7 +1585,7 @@ define <5 x i4> @sel_common_op_commute3_poison_mask_elts(<5 x i4> %x, <5 x i4> %
 
 define <5 x i4> @sel_not_common_op_commute3(<5 x i4> %x, <5 x i4> %y, <5 x i4> %z) {
 ; CHECK-LABEL: @sel_not_common_op_commute3(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <5 x i4> [[Y:%.*]], <5 x i4> [[Z:%.*]], <5 x i32> <i32 0, i32 undef, i32 undef, i32 3, i32 9>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <5 x i4> [[Y:%.*]], <5 x i4> [[Z:%.*]], <5 x i32> <i32 0, i32 poison, i32 poison, i32 3, i32 9>
 ; CHECK-NEXT:    [[S2:%.*]] = shufflevector <5 x i4> [[S1]], <5 x i4> [[X:%.*]], <5 x i32> <i32 0, i32 6, i32 7, i32 3, i32 4>
 ; CHECK-NEXT:    ret <5 x i4> [[S2]]
 ;
@@ -1598,7 +1598,7 @@ define <5 x i4> @sel_not_common_op_commute3(<5 x i4> %x, <5 x i4> %y, <5 x i4> %
 
 define <5 x i4> @not_sel_common_op(<5 x i4> %x, <5 x i4> %y) {
 ; CHECK-LABEL: @not_sel_common_op(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <5 x i4> [[Y:%.*]], <5 x i4> [[X:%.*]], <5 x i32> <i32 undef, i32 6, i32 undef, i32 3, i32 9>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <5 x i4> [[Y:%.*]], <5 x i4> [[X:%.*]], <5 x i32> <i32 poison, i32 6, i32 poison, i32 3, i32 9>
 ; CHECK-NEXT:    [[S2:%.*]] = shufflevector <5 x i4> [[S1]], <5 x i4> [[X]], <5 x i32> <i32 1, i32 6, i32 7, i32 3, i32 4>
 ; CHECK-NEXT:    ret <5 x i4> [[S2]]
 ;
@@ -1624,7 +1624,7 @@ define <4 x i32> @sel_common_op_extra_use(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x float> @identity_mask(<4 x float>%x, <4 x float> %y) {
 ; CHECK-LABEL: @identity_mask(
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[S2]]
 ;
   %s1 = shufflevector <4 x float> %x, <4 x float> %y, <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
diff --git a/llvm/test/Transforms/InstCombine/shufflevec-bitcast-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shufflevec-bitcast-inseltpoison.ll
index 3ff269ef9d279..8d607414277c7 100644
--- a/llvm/test/Transforms/InstCombine/shufflevec-bitcast-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/shufflevec-bitcast-inseltpoison.ll
@@ -159,7 +159,7 @@ define <16 x i8> @shuf_bitcast_operand_cannot_widen_undef(<4 x i32> %x) {
 ; CHECK-LABEL: @shuf_bitcast_operand_cannot_widen_undef(
 ; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x i32> [[S1]] to <16 x i8>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> poison, <16 x i32> <i32 12, i32 undef, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> poison, <16 x i32> <i32 12, i32 poison, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <16 x i8> [[S2]]
 ;
   %s1 = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
diff --git a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
index 65120bc505efd..cedb882d0d228 100644
--- a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
+++ b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
@@ -159,7 +159,7 @@ define <16 x i8> @shuf_bitcast_operand_cannot_widen_undef(<4 x i32> %x) {
 ; CHECK-LABEL: @shuf_bitcast_operand_cannot_widen_undef(
 ; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x i32> [[S1]] to <16 x i8>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> undef, <16 x i32> <i32 12, i32 undef, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[BC]], <16 x i8> undef, <16 x i32> <i32 12, i32 poison, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <16 x i8> [[S2]]
 ;
   %s1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll
index 442175d9c9691..253d42e9029df 100644
--- a/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll
@@ -28,7 +28,7 @@ define <2 x i16> @test_srem(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_srem(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
@@ -43,7 +43,7 @@ define <2 x i16> @test_urem(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_urem(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
@@ -58,7 +58,7 @@ define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_sdiv(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
@@ -73,7 +73,7 @@ define <2 x i16> @test_udiv(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_udiv(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll
index 0aadcbc5cdefd..12d81e5d1944b 100644
--- a/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll
+++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem.ll
@@ -28,7 +28,7 @@ define <2 x i16> @test_srem(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_srem(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
@@ -43,7 +43,7 @@ define <2 x i16> @test_urem(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_urem(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
@@ -58,7 +58,7 @@ define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_sdiv(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
@@ -73,7 +73,7 @@ define <2 x i16> @test_udiv(i16 %a, i1 %cmp) {
 ; CHECK-LABEL: @test_udiv(
 ; CHECK-NEXT:    [[SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
-; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
 ; CHECK-NEXT:    ret <2 x i16> [[T2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
index 47917fd5434e7..1601a086c8748 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
@@ -2,60 +2,60 @@
 ; RUN: opt < %s -passes=instcombine -optimize-hot-cold-new -S | FileCheck %s --check-prefix=HOTCOLD
 
 ;; Check that operator new(unsigned long) converted to
-;; operator new(unsigned long, hot_cold_t) with a hot or cold attribute.
+;; operator new(unsigned long, __hot_cold_t) with a hot or cold attribute.
 ; HOTCOLD-LABEL: @new()
 define void @new() {
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_Znwm10hot_cold_t(i64 10, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_Znwm12__hot_cold_t(i64 10, i8 0)
   %call = call ptr @_Znwm(i64 10) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_Znwm(i64 10)
   %call1 = call ptr @_Znwm(i64 10) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_Znwm10hot_cold_t(i64 10, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_Znwm12__hot_cold_t(i64 10, i8 -1)
   %call2 = call ptr @_Znwm(i64 10) #2
   call void @dummy(ptr %call2)
   ret void
 }
 
 ;; Check that operator new(unsigned long, std::align_val_t) converted to
-;; operator new(unsigned long, std::align_val_t, hot_cold_t) with a hot or
+;; operator new(unsigned long, std::align_val_t, __hot_cold_t) with a hot or
 ;; cold attribute.
 ; HOTCOLD-LABEL: @new_align()
 define void @new_align() {
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_ZnwmSt11align_val_t10hot_cold_t(i64 10, i64 8, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 0)
   %call = call ptr @_ZnwmSt11align_val_t(i64 10, i64 8) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_ZnwmSt11align_val_t(i64 10, i64 8)
   %call1 = call ptr @_ZnwmSt11align_val_t(i64 10, i64 8) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_ZnwmSt11align_val_t10hot_cold_t(i64 10, i64 8, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_ZnwmSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 -1)
   %call2 = call ptr @_ZnwmSt11align_val_t(i64 10, i64 8) #2
   call void @dummy(ptr %call2)
   ret void
 }
 
 ;; Check that operator new(unsigned long, const std::nothrow_t&) converted to
-;; operator new(unsigned long, const std::nothrow_t&, hot_cold_t) with a hot or
+;; operator new(unsigned long, const std::nothrow_t&, __hot_cold_t) with a hot or
 ;; cold attribute.
 ; HOTCOLD-LABEL: @new_nothrow()
 define void @new_nothrow() {
   %nt = alloca i8
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_ZnwmRKSt9nothrow_t10hot_cold_t(i64 10, ptr nonnull %nt, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 0)
   %call = call ptr @_ZnwmRKSt9nothrow_t(i64 10, ptr %nt) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_ZnwmRKSt9nothrow_t(i64 10, ptr nonnull %nt)
   %call1 = call ptr @_ZnwmRKSt9nothrow_t(i64 10, ptr %nt) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_ZnwmRKSt9nothrow_t10hot_cold_t(i64 10, ptr nonnull %nt, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 -1)
   %call2 = call ptr @_ZnwmRKSt9nothrow_t(i64 10, ptr %nt) #2
   call void @dummy(ptr %call2)
   ret void
@@ -63,81 +63,81 @@ define void @new_nothrow() {
 
 ;; Check that operator new(unsigned long, std::align_val_t, const std::nothrow_t&)
 ;; converted to
-;; operator new(unsigned long, std::align_val_t, const std::nothrow_t&, hot_cold_t)
+;; operator new(unsigned long, std::align_val_t, const std::nothrow_t&, __hot_cold_t)
 ;; with a hot or cold attribute.
 ; HOTCOLD-LABEL: @new_align_nothrow()
 define void @new_align_nothrow() {
   %nt = alloca i8
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 0)
   %call = call ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr %nt) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr nonnull %nt)
   %call1 = call ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr %nt) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 -1)
   %call2 = call ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr %nt) #2
   call void @dummy(ptr %call2)
   ret void
 }
 
 ;; Check that operator new[](unsigned long) converted to
-;; operator new[](unsigned long, hot_cold_t) with a hot or cold attribute.
+;; operator new[](unsigned long, __hot_cold_t) with a hot or cold attribute.
 ; HOTCOLD-LABEL: @array_new()
 define void @array_new() {
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_Znam10hot_cold_t(i64 10, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_Znam12__hot_cold_t(i64 10, i8 0)
   %call = call ptr @_Znam(i64 10) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_Znam(i64 10)
   %call1 = call ptr @_Znam(i64 10) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_Znam10hot_cold_t(i64 10, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_Znam12__hot_cold_t(i64 10, i8 -1)
   %call2 = call ptr @_Znam(i64 10) #2
   call void @dummy(ptr %call2)
   ret void
 }
 
 ;; Check that operator new[](unsigned long, std::align_val_t) converted to
-;; operator new[](unsigned long, std::align_val_t, hot_cold_t) with a hot or
+;; operator new[](unsigned long, std::align_val_t, __hot_cold_t) with a hot or
 ;; cold attribute.
 ; HOTCOLD-LABEL: @array_new_align()
 define void @array_new_align() {
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_ZnamSt11align_val_t10hot_cold_t(i64 10, i64 8, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 0)
   %call = call ptr @_ZnamSt11align_val_t(i64 10, i64 8) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_ZnamSt11align_val_t(i64 10, i64 8)
   %call1 = call ptr @_ZnamSt11align_val_t(i64 10, i64 8) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_ZnamSt11align_val_t10hot_cold_t(i64 10, i64 8, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_ZnamSt11align_val_t12__hot_cold_t(i64 10, i64 8, i8 -1)
   %call2 = call ptr @_ZnamSt11align_val_t(i64 10, i64 8) #2
   call void @dummy(ptr %call2)
   ret void
 }
 
 ;; Check that operator new[](unsigned long, const std::nothrow_t&) converted to
-;; operator new[](unsigned long, const std::nothrow_t&, hot_cold_t) with a hot or
+;; operator new[](unsigned long, const std::nothrow_t&, __hot_cold_t) with a hot or
 ;; cold attribute.
 ; HOTCOLD-LABEL: @array_new_nothrow()
 define void @array_new_nothrow() {
   %nt = alloca i8
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_ZnamRKSt9nothrow_t10hot_cold_t(i64 10, ptr nonnull %nt, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 0)
   %call = call ptr @_ZnamRKSt9nothrow_t(i64 10, ptr %nt) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_ZnamRKSt9nothrow_t(i64 10, ptr nonnull %nt)
   %call1 = call ptr @_ZnamRKSt9nothrow_t(i64 10, ptr %nt) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_ZnamRKSt9nothrow_t10hot_cold_t(i64 10, ptr nonnull %nt, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_ZnamRKSt9nothrow_t12__hot_cold_t(i64 10, ptr nonnull %nt, i8 -1)
   %call2 = call ptr @_ZnamRKSt9nothrow_t(i64 10, ptr %nt) #2
   call void @dummy(ptr %call2)
   ret void
@@ -145,21 +145,21 @@ define void @array_new_nothrow() {
 
 ;; Check that operator new[](unsigned long, std::align_val_t, const std::nothrow_t&)
 ;; converted to
-;; operator new[](unsigned long, std::align_val_t, const std::nothrow_t&, hot_cold_t)
+;; operator new[](unsigned long, std::align_val_t, const std::nothrow_t&, __hot_cold_t)
 ;; with a hot or cold attribute.
 ; HOTCOLD-LABEL: @array_new_align_nothrow()
 define void @array_new_align_nothrow() {
   %nt = alloca i8
-  ;; Attribute cold converted to hot_cold_t value 0 (coldest).
-  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 0)
+  ;; Attribute cold converted to __hot_cold_t value 0 (coldest).
+  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 0)
   %call = call ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr %nt) #0
   call void @dummy(ptr %call)
   ;; Attribute notcold has no effect.
   ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr nonnull %nt)
   %call1 = call ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr %nt) #1
   call void @dummy(ptr %call1)
-  ;; Attribute hot converted to hot_cold_t value 255 (-1) (hottest).
-  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 -1)
+  ;; Attribute hot converted to __hot_cold_t value 255 (-1) (hottest).
+  ; HOTCOLD: @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64 10, i64 8, ptr nonnull %nt, i8 -1)
   %call2 = call ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 10, i64 8, ptr %nt) #2
   call void @dummy(ptr %call2)
   ret void
diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll
index d56c19f52773d..e924e17baad00 100644
--- a/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll
@@ -819,7 +819,7 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y)
 define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) {
 ; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef(
 ; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
index 727dea404d84b..2aed639da6dd7 100644
--- a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
+++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
@@ -843,7 +843,7 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y)
 define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) {
 ; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef(
 ; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> undef, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll b/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll
index 121e58664e9f6..ac0115a0f5715 100644
--- a/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll
@@ -950,7 +950,7 @@ define <3 x i31> @wide_splat2(<3 x i33> %x) {
 
 define <3 x i31> @wide_splat3(<3 x i33> %x) {
 ; CHECK-LABEL: @wide_splat3(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i33> [[X:%.*]], <3 x i33> poison, <3 x i32> <i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i33> [[X:%.*]], <3 x i33> poison, <3 x i32> <i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <3 x i33> [[SHUF]] to <3 x i31>
 ; CHECK-NEXT:    ret <3 x i31> [[TRUNC]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
index a45bbe4648001..e04bcaf073b64 100644
--- a/llvm/test/Transforms/InstCombine/trunc.ll
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -950,7 +950,7 @@ define <3 x i31> @wide_splat2(<3 x i33> %x) {
 
 define <3 x i31> @wide_splat3(<3 x i33> %x) {
 ; CHECK-LABEL: @wide_splat3(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i33> [[X:%.*]], <3 x i33> undef, <3 x i32> <i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i33> [[X:%.*]], <3 x i33> undef, <3 x i32> <i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <3 x i33> [[SHUF]] to <3 x i31>
 ; CHECK-NEXT:    ret <3 x i31> [[TRUNC]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/type_pun-inseltpoison.ll b/llvm/test/Transforms/InstCombine/type_pun-inseltpoison.ll
index 43fdeb4749982..85524c8330281 100644
--- a/llvm/test/Transforms/InstCombine/type_pun-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/type_pun-inseltpoison.ll
@@ -42,7 +42,7 @@ define i32 @type_pun_first(<16 x i8> %in) {
 ; Extracting an i32 that isn't aligned to any natural boundary.
 define i32 @type_pun_misaligned(<16 x i8> %in) {
 ; CHECK-LABEL: @type_pun_misaligned(
-; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> poison, <16 x i32> <i32 6, i32 7, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> poison, <16 x i32> <i32 6, i32 7, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[SROA_EXTRACT]] to <4 x i32>
 ; CHECK-NEXT:    [[SROA_EXTRACT1:%.*]] = extractelement <4 x i32> [[SROA_BC]], i64 0
 ; CHECK-NEXT:    ret i32 [[SROA_EXTRACT1]]
diff --git a/llvm/test/Transforms/InstCombine/type_pun.ll b/llvm/test/Transforms/InstCombine/type_pun.ll
index 15315f4677cef..a439f77902365 100644
--- a/llvm/test/Transforms/InstCombine/type_pun.ll
+++ b/llvm/test/Transforms/InstCombine/type_pun.ll
@@ -42,7 +42,7 @@ define i32 @type_pun_first(<16 x i8> %in) {
 ; Extracting an i32 that isn't aligned to any natural boundary.
 define i32 @type_pun_misaligned(<16 x i8> %in) {
 ; CHECK-LABEL: @type_pun_misaligned(
-; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> poison, <16 x i32> <i32 6, i32 7, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> poison, <16 x i32> <i32 6, i32 7, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[SROA_EXTRACT]] to <4 x i32>
 ; CHECK-NEXT:    [[SROA_EXTRACT1:%.*]] = extractelement <4 x i32> [[SROA_BC]], i64 0
 ; CHECK-NEXT:    ret i32 [[SROA_EXTRACT1]]
diff --git a/llvm/test/Transforms/InstCombine/vec-binop-select-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec-binop-select-inseltpoison.ll
index c8b92635d406c..ac1d0d8eb7a97 100644
--- a/llvm/test/Transforms/InstCombine/vec-binop-select-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec-binop-select-inseltpoison.ll
@@ -97,8 +97,8 @@ define <4 x i32> @add_non_select_mask(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @add_masks_with_undefs(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @add_masks_with_undefs(
-; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
-; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = add nsw <4 x i32> [[SEL1]], [[SEL2]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vec-binop-select.ll b/llvm/test/Transforms/InstCombine/vec-binop-select.ll
index f8b81afc3902d..a9958e732194f 100644
--- a/llvm/test/Transforms/InstCombine/vec-binop-select.ll
+++ b/llvm/test/Transforms/InstCombine/vec-binop-select.ll
@@ -97,8 +97,8 @@ define <4 x i32> @add_non_select_mask(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @add_masks_with_undefs(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @add_masks_with_undefs(
-; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
-; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = add nsw <4 x i32> [[SEL1]], [[SEL2]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
index 3a1ca6c8dd5ea..1a1d5a892daae 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
@@ -64,7 +64,7 @@ declare i32 @fgetc(ptr)
 
 define <4 x float> @dead_shuffle_elt(<4 x float> %x, <2 x float> %y) nounwind {
 ; CHECK-LABEL: @dead_shuffle_elt(
-; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SHUFFLE9_I:%.*]] = shufflevector <4 x float> [[SHUFFLE_I]], <4 x float> [[X:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x float> [[SHUFFLE9_I]]
 ;
@@ -175,7 +175,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b)
 ; CHECK-NEXT:    [[OUT0:%.*]] = insertelement <4 x i32> poison, i32 [[A0:%.*]], i64 0
 ; CHECK-NEXT:    [[OUT01:%.*]] = insertelement <4 x i32> [[OUT0]], i32 [[A1:%.*]], i64 1
 ; CHECK-NEXT:    [[FOO:%.*]] = add <4 x i32> [[OUT01]], [[B:%.*]]
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE]]
 ;
   %out0 = insertelement <4 x i32> poison, i32 %a0, i32 0
@@ -190,7 +190,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b)
 define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) {
 ; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain(
 ; CHECK-NEXT:    [[OUT12:%.*]] = insertelement <4 x float> poison, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]]
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SHUFFLE]]
 ;
   %out1 = insertelement <4 x float> poison, float %a1, i32 1
@@ -205,7 +205,7 @@ define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1,
 define <3 x i8> @shuf_add(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_add(
 ; CHECK-NEXT:    [[BO:%.*]] = add nsw <3 x i8> [[X:%.*]], <i8 poison, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = add nsw <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -216,7 +216,7 @@ define <3 x i8> @shuf_add(<3 x i8> %x) {
 define <3 x i8> @shuf_sub(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sub(
 ; CHECK-NEXT:    [[BO:%.*]] = sub nuw <3 x i8> <i8 1, i8 poison, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sub nuw <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -249,7 +249,7 @@ define <3 x i8> @shuf_and(<3 x i8> %x) {
 define <3 x i8> @shuf_or(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_or(
 ; CHECK-NEXT:    [[BO:%.*]] = or <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = or <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -260,7 +260,7 @@ define <3 x i8> @shuf_or(<3 x i8> %x) {
 define <3 x i8> @shuf_xor(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_xor(
 ; CHECK-NEXT:    [[BO:%.*]] = xor <3 x i8> [[X:%.*]], <i8 1, i8 poison, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = xor <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -271,7 +271,7 @@ define <3 x i8> @shuf_xor(<3 x i8> %x) {
 define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_lshr_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = lshr <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -282,7 +282,7 @@ define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_lshr_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = lshr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = lshr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -293,7 +293,7 @@ define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_ashr_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = ashr <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -304,7 +304,7 @@ define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_ashr_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = ashr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = ashr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -315,7 +315,7 @@ define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_shl_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -326,7 +326,7 @@ define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_shl_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = shl nuw <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = shl nuw <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -337,7 +337,7 @@ define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sdiv_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -348,7 +348,7 @@ define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sdiv_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = sdiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sdiv <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -359,7 +359,7 @@ define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_srem_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = srem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 1, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = srem <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -370,7 +370,7 @@ define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_srem_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = srem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = srem <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -381,7 +381,7 @@ define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_udiv_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -392,7 +392,7 @@ define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_udiv_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = udiv <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -403,7 +403,7 @@ define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_urem_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = urem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = urem <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -414,7 +414,7 @@ define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_urem_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = urem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> poison, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = urem <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -425,7 +425,7 @@ define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) {
 define <3 x float> @shuf_fadd(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fadd(
 ; CHECK-NEXT:    [[BO:%.*]] = fadd <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fadd <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -436,7 +436,7 @@ define <3 x float> @shuf_fadd(<3 x float> %x) {
 define <3 x float> @shuf_fsub(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fsub(
 ; CHECK-NEXT:    [[BO:%.*]] = fsub fast <3 x float> <float 1.000000e+00, float poison, float 3.000000e+00>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 poison, i32 0, i32 2>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fsub fast <3 x float> <float 1.0, float 2.0, float 3.0>, %x
@@ -447,7 +447,7 @@ define <3 x float> @shuf_fsub(<3 x float> %x) {
 define <3 x float> @shuf_fmul(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fmul(
 ; CHECK-NEXT:    [[BO:%.*]] = fmul reassoc <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fmul reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -458,7 +458,7 @@ define <3 x float> @shuf_fmul(<3 x float> %x) {
 define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fdiv_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = fdiv reassoc ninf <3 x float> <float 1.000000e+00, float poison, float 3.000000e+00>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 poison, i32 0, i32 2>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fdiv ninf reassoc <3 x float> <float 1.0, float 2.0, float 3.0>, %x
@@ -469,7 +469,7 @@ define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) {
 define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fdiv_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = fdiv nnan ninf <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fdiv ninf nnan <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -480,7 +480,7 @@ define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) {
 define <3 x float> @shuf_frem_const_op0(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_frem_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = frem nnan <3 x float> <float 1.000000e+00, float poison, float 3.000000e+00>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 undef, i32 2, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 poison, i32 2, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = frem nnan <3 x float> <float 1.0, float 2.0, float 3.0>, %x
@@ -491,7 +491,7 @@ define <3 x float> @shuf_frem_const_op0(<3 x float> %x) {
 define <3 x float> @shuf_frem_const_op1(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_frem_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = frem reassoc ninf <3 x float> [[X:%.*]], <float poison, float 2.000000e+00, float 3.000000e+00>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> poison, <3 x i32> <i32 1, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = frem ninf reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -685,7 +685,7 @@ define <4 x i8> @select_cond_with_eq_true_false_elts(<4 x i8> %x, <4 x i8> %y, <
 
 define <4 x i8> @select_cond_with_eq_true_false_elts2(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @select_cond_with_eq_true_false_elts2(
-; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> <i32 undef, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> <i32 poison, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[Y:%.*]], <4 x i8> [[X:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[SEL]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
@@ -701,9 +701,9 @@ define <4 x i8> @select_cond_with_eq_true_false_elts2(<4 x i8> %x, <4 x i8> %y,
 
 define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x float> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @select_cond_with_eq_true_false_elts3(
-; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 3, i32 5, i32 undef>
-; CHECK-NEXT:    [[FVAL:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[X]], <4 x i32> <i32 0, i32 7, i32 6, i32 undef>
-; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 3, i32 5, i32 poison>
+; CHECK-NEXT:    [[FVAL:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[X]], <4 x i32> <i32 0, i32 7, i32 6, i32 poison>
+; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[COND]], <4 x float> [[TVAL]], <4 x float> [[FVAL]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -716,7 +716,7 @@ define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x fl
 
 define <4 x i8> @select_cond_with_undef_true_false_elts(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @select_cond_with_undef_true_false_elts(
-; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[TVAL]], <4 x i8> [[X:%.*]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
@@ -829,7 +829,7 @@ define <4 x i4> @ins_of_ext_undef_elts_propagation(<4 x i4> %v, <4 x i4> %v2, i4
 define <8 x i4> @ins_of_ext_undef_elts_propagation2(<8 x i4> %v, <8 x i4> %v2, i4 %x) {
 ; CHECK-LABEL: @ins_of_ext_undef_elts_propagation2(
 ; CHECK-NEXT:    [[I19:%.*]] = insertelement <8 x i4> [[V:%.*]], i4 [[X:%.*]], i64 2
-; CHECK-NEXT:    [[I20:%.*]] = shufflevector <8 x i4> [[I19]], <8 x i4> [[V2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 10, i32 9, i32 8, i32 undef>
+; CHECK-NEXT:    [[I20:%.*]] = shufflevector <8 x i4> [[I19]], <8 x i4> [[V2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 10, i32 9, i32 8, i32 poison>
 ; CHECK-NEXT:    [[I21:%.*]] = shufflevector <8 x i4> [[I20]], <8 x i4> [[V]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
 ; CHECK-NEXT:    ret <8 x i4> [[I21]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
index ee789612f8b5e..4aa783881683a 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -67,7 +67,7 @@ declare i32 @fgetc(ptr)
 
 define <4 x float> @dead_shuffle_elt(<4 x float> %x, <2 x float> %y) nounwind {
 ; CHECK-LABEL: @dead_shuffle_elt(
-; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SHUFFLE9_I:%.*]] = shufflevector <4 x float> [[SHUFFLE_I]], <4 x float> [[X:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x float> [[SHUFFLE9_I]]
 ;
@@ -178,7 +178,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b)
 ; CHECK-NEXT:    [[OUT0:%.*]] = insertelement <4 x i32> undef, i32 [[A0:%.*]], i64 0
 ; CHECK-NEXT:    [[OUT01:%.*]] = insertelement <4 x i32> [[OUT0]], i32 [[A1:%.*]], i64 1
 ; CHECK-NEXT:    [[FOO:%.*]] = add <4 x i32> [[OUT01]], [[B:%.*]]
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUFFLE]]
 ;
   %out0 = insertelement <4 x i32> undef, i32 %a0, i32 0
@@ -193,7 +193,7 @@ define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b)
 define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) {
 ; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain(
 ; CHECK-NEXT:    [[OUT12:%.*]] = insertelement <4 x float> undef, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]]
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[SHUFFLE]]
 ;
   %out1 = insertelement <4 x float> undef, float %a1, i32 1
@@ -208,7 +208,7 @@ define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1,
 define <3 x i8> @shuf_add(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_add(
 ; CHECK-NEXT:    [[BO:%.*]] = add nsw <3 x i8> [[X:%.*]], <i8 poison, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = add nsw <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -219,11 +219,11 @@ define <3 x i8> @shuf_add(<3 x i8> %x) {
 define <3 x i8> @shuf_sub(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sub(
 ; CHECK-NEXT:    [[BO:%.*]] = sub nuw <3 x i8> <i8 1, i8 poison, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sub nuw <3 x i8> <i8 1, i8 2, i8 3>, %x
-  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 poison, i32 2>
   ret <3 x i8> %r
 }
 
@@ -252,7 +252,7 @@ define <3 x i8> @shuf_and(<3 x i8> %x) {
 define <3 x i8> @shuf_or(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_or(
 ; CHECK-NEXT:    [[BO:%.*]] = or <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = or <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -263,7 +263,7 @@ define <3 x i8> @shuf_or(<3 x i8> %x) {
 define <3 x i8> @shuf_xor(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_xor(
 ; CHECK-NEXT:    [[BO:%.*]] = xor <3 x i8> [[X:%.*]], <i8 1, i8 poison, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = xor <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -274,7 +274,7 @@ define <3 x i8> @shuf_xor(<3 x i8> %x) {
 define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_lshr_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = lshr <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -285,7 +285,7 @@ define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_lshr_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = lshr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = lshr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -296,7 +296,7 @@ define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_ashr_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = ashr <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -307,7 +307,7 @@ define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_ashr_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = ashr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = ashr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -318,7 +318,7 @@ define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_shl_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -329,7 +329,7 @@ define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_shl_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = shl nuw <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = shl nuw <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -340,7 +340,7 @@ define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sdiv_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -351,7 +351,7 @@ define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_sdiv_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = sdiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = sdiv <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -362,7 +362,7 @@ define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_srem_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = srem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = srem <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -373,7 +373,7 @@ define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_srem_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = srem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 poison, i32 1>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = srem <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -384,7 +384,7 @@ define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_udiv_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -395,7 +395,7 @@ define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_udiv_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = udiv <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -406,7 +406,7 @@ define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) {
 define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_urem_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = urem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = urem <3 x i8> <i8 1, i8 2, i8 3>, %x
@@ -417,7 +417,7 @@ define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) {
 define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) {
 ; CHECK-LABEL: @shuf_urem_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = urem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
 ;
   %bo = urem <3 x i8> %x, <i8 1, i8 2, i8 3>
@@ -428,7 +428,7 @@ define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) {
 define <3 x float> @shuf_fadd(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fadd(
 ; CHECK-NEXT:    [[BO:%.*]] = fadd <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fadd <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -439,7 +439,7 @@ define <3 x float> @shuf_fadd(<3 x float> %x) {
 define <3 x float> @shuf_fsub(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fsub(
 ; CHECK-NEXT:    [[BO:%.*]] = fsub fast <3 x float> <float 1.000000e+00, float poison, float 3.000000e+00>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 poison, i32 0, i32 2>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fsub fast <3 x float> <float 1.0, float 2.0, float 3.0>, %x
@@ -450,7 +450,7 @@ define <3 x float> @shuf_fsub(<3 x float> %x) {
 define <3 x float> @shuf_fmul(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fmul(
 ; CHECK-NEXT:    [[BO:%.*]] = fmul reassoc <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fmul reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -461,7 +461,7 @@ define <3 x float> @shuf_fmul(<3 x float> %x) {
 define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fdiv_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = fdiv reassoc ninf <3 x float> <float 1.000000e+00, float poison, float 3.000000e+00>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 poison, i32 0, i32 2>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fdiv ninf reassoc <3 x float> <float 1.0, float 2.0, float 3.0>, %x
@@ -472,7 +472,7 @@ define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) {
 define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_fdiv_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = fdiv nnan ninf <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = fdiv ninf nnan <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -483,7 +483,7 @@ define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) {
 define <3 x float> @shuf_frem_const_op0(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_frem_const_op0(
 ; CHECK-NEXT:    [[BO:%.*]] = frem nnan <3 x float> <float 1.000000e+00, float poison, float 3.000000e+00>, [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 2, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 poison, i32 2, i32 0>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = frem nnan <3 x float> <float 1.0, float 2.0, float 3.0>, %x
@@ -494,7 +494,7 @@ define <3 x float> @shuf_frem_const_op0(<3 x float> %x) {
 define <3 x float> @shuf_frem_const_op1(<3 x float> %x) {
 ; CHECK-LABEL: @shuf_frem_const_op1(
 ; CHECK-NEXT:    [[BO:%.*]] = frem reassoc ninf <3 x float> [[X:%.*]], <float poison, float 2.000000e+00, float 3.000000e+00>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 1, i32 poison, i32 2>
 ; CHECK-NEXT:    ret <3 x float> [[R]]
 ;
   %bo = frem ninf reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
@@ -688,7 +688,7 @@ define <4 x i8> @select_cond_with_eq_true_false_elts(<4 x i8> %x, <4 x i8> %y, <
 
 define <4 x i8> @select_cond_with_eq_true_false_elts2(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @select_cond_with_eq_true_false_elts2(
-; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 undef, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 poison, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[Y:%.*]], <4 x i8> [[X:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[SEL]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
@@ -704,9 +704,9 @@ define <4 x i8> @select_cond_with_eq_true_false_elts2(<4 x i8> %x, <4 x i8> %y,
 
 define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x float> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @select_cond_with_eq_true_false_elts3(
-; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 3, i32 5, i32 undef>
-; CHECK-NEXT:    [[FVAL:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[X]], <4 x i32> <i32 0, i32 7, i32 6, i32 undef>
-; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 3, i32 5, i32 poison>
+; CHECK-NEXT:    [[FVAL:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[X]], <4 x i32> <i32 0, i32 7, i32 6, i32 poison>
+; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[COND]], <4 x float> [[TVAL]], <4 x float> [[FVAL]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -719,7 +719,7 @@ define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x fl
 
 define <4 x i8> @select_cond_with_undef_true_false_elts(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
 ; CHECK-LABEL: @select_cond_with_undef_true_false_elts(
-; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TVAL:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> poison, <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[TVAL]], <4 x i8> [[X:%.*]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
@@ -832,7 +832,7 @@ define <4 x i4> @ins_of_ext_undef_elts_propagation(<4 x i4> %v, <4 x i4> %v2, i4
 define <8 x i4> @ins_of_ext_undef_elts_propagation2(<8 x i4> %v, <8 x i4> %v2, i4 %x) {
 ; CHECK-LABEL: @ins_of_ext_undef_elts_propagation2(
 ; CHECK-NEXT:    [[I19:%.*]] = insertelement <8 x i4> [[V:%.*]], i4 [[X:%.*]], i64 2
-; CHECK-NEXT:    [[I20:%.*]] = shufflevector <8 x i4> [[I19]], <8 x i4> [[V2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 10, i32 9, i32 8, i32 undef>
+; CHECK-NEXT:    [[I20:%.*]] = shufflevector <8 x i4> [[I19]], <8 x i4> [[V2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 10, i32 9, i32 8, i32 poison>
 ; CHECK-NEXT:    [[I21:%.*]] = shufflevector <8 x i4> [[I20]], <8 x i4> [[V]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
 ; CHECK-NEXT:    ret <8 x i4> [[I21]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
index 0b2419e1158a8..65f1ab290dfa9 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
@@ -75,7 +75,7 @@ define float @testvscale6(<vscale x 4 x float> %X) {
 
 define <4 x float> @test7(<4 x float> %x) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %r = shufflevector <4 x float> %x, <4 x float> poison, <4 x i32> < i32 0, i32 1, i32 6, i32 7 >
@@ -85,7 +85,7 @@ define <4 x float> @test7(<4 x float> %x) {
 ; This should turn into a single shuffle.
 define <4 x float> @test8(<4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 undef, i32 3, i32 4>
+; CHECK-NEXT:    [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 poison, i32 3, i32 4>
 ; CHECK-NEXT:    ret <4 x float> [[T134]]
 ;
   %t4 = extractelement <4 x float> %x, i32 1
@@ -116,8 +116,8 @@ define <4 x i8> @test9(<16 x i8> %t6) {
 
 define <4 x i8> @test9a(<16 x i8> %t6) {
 ; CHECK-LABEL: @test9a(
-; CHECK-NEXT:    [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> poison, <4 x i32> <i32 undef, i32 9, i32 4, i32 8>
-; CHECK-NEXT:    [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> poison, <4 x i32> <i32 3, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> poison, <4 x i32> <i32 poison, i32 9, i32 4, i32 8>
+; CHECK-NEXT:    [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> poison, <4 x i32> <i32 3, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[T9]]
 ;
   %t7 = shufflevector <16 x i8> %t6, <16 x i8> poison, <4 x i32> < i32 undef, i32 9, i32 4, i32 8 >
@@ -202,7 +202,7 @@ define <2 x i8> @extract_subvector_of_shuffle(<2 x i8> %x, <2 x i8> %y) {
 
 define <4 x i8> @extract_subvector_of_shuffle_undefs_types(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @extract_subvector_of_shuffle_undefs_types(
-; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 2, i32 0, i32 undef>
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 poison, i32 2, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
 ;
   %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
@@ -216,9 +216,9 @@ declare void @use_v5i8(<5 x i8>)
 
 define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @extract_subvector_of_shuffle_extra_use(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 poison, i32 2, i32 0, i32 1, i32 0>
 ; CHECK-NEXT:    call void @use_v5i8(<5 x i8> [[SHUF]])
-; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
 ;
   %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
@@ -248,7 +248,7 @@ define <3 x i32> @add_wider(i32 %y, i32 %z) {
 ; CHECK-NEXT:    [[I0:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i64 1
 ; CHECK-NEXT:    [[A:%.*]] = add <2 x i32> [[I1]], <i32 255, i32 255>
-; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i32> [[EXT]]
 ;
   %i0 = insertelement <2 x i32> poison, i32 %y, i32 0
@@ -265,7 +265,7 @@ define <3 x i32> @div_wider(i32 %y, i32 %z) {
 ; CHECK-NEXT:    [[I0:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i64 1
 ; CHECK-NEXT:    [[A:%.*]] = sdiv <2 x i32> [[I1]], <i32 255, i32 255>
-; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i32> [[EXT]]
 ;
   %i0 = insertelement <2 x i32> poison, i32 %y, i32 0
@@ -584,7 +584,7 @@ define <2 x float> @fmul_const_invalid_constant(<2 x float> %v) {
 define <4 x i8> @widening_shuffle_add_1(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 42, i8 43>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %widex = shufflevector <2 x i8> %x, <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -597,7 +597,7 @@ define <4 x i8> @widening_shuffle_add_1(<2 x i8> %x) {
 define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 43, i8 42>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %widex = shufflevector <2 x i8> %x, <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
@@ -609,7 +609,7 @@ define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) {
 
 define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_invalid_constant(
-; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 1, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -622,7 +622,7 @@ define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) {
 
 define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_invalid_mask(
-; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -637,7 +637,7 @@ define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) {
 define <4 x i16> @widening_shuffle_shl_constant_op0(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_shl_constant_op0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i16> <i16 42, i16 -42>, [[V:%.*]]
-; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
   %shuf = shufflevector <2 x i16> %v, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -651,7 +651,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op0(<2 x i16> %v) {
 define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_shl_constant_op1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i16> [[V:%.*]], <i16 2, i16 4>
-; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
   %shuf = shufflevector <2 x i16> %v, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -664,7 +664,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) {
 
 define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BO:%.*]] = shl <4 x i16> [[SHUF]], <i16 2, i16 4, i16 1, i16 2>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
@@ -678,7 +678,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) {
 
 define <4 x i16> @widening_shuffle_or(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_or(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BO:%.*]] = or <4 x i16> [[SHUF]], <i16 42, i16 -42, i16 -1, i16 -1>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
@@ -1024,7 +1024,7 @@ define <2 x i32> @and_splat_constant(<2 x i32> %x) {
 define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) {
 ; CHECK-LABEL: @and_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 0, i16 0, i16 -1, i16 -1>
 ; CHECK-NEXT:    ret <4 x i16> [[AND]]
 ;
@@ -1039,7 +1039,7 @@ entry:
 define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) {
 ; CHECK-LABEL: @and_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 -1, i16 0>
 ; CHECK-NEXT:    ret <4 x i16> [[AND]]
 ;
@@ -1066,7 +1066,7 @@ define <4 x i16> @and_constant_mask_undef_4(<4 x i16> %add) {
 ; CHECK-LABEL: @and_constant_mask_undef_4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], <i16 9, i16 20, i16 poison, i16 poison>
-; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[AND]]
 ;
 entry:
@@ -1093,7 +1093,7 @@ entry:
 define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @or_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 0, i16 0>
 ; CHECK-NEXT:    ret <4 x i16> [[OR]]
 ;
@@ -1108,7 +1108,7 @@ entry:
 define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) {
 ; CHECK-LABEL: @or_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 0, i16 0, i16 -1>
 ; CHECK-NEXT:    ret <4 x i16> [[OR]]
 ;
@@ -1135,7 +1135,7 @@ define <4 x i16> @or_constant_mask_undef_4(<4 x i16> %in) {
 ; CHECK-LABEL: @or_constant_mask_undef_4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], <i16 poison, i16 99, i16 poison, i16 poison>
-; CHECK-NEXT:    [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[OR]]
 ;
 entry:
@@ -1160,7 +1160,7 @@ entry:
 define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @shl_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 0, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], <i16 10, i16 3, i16 0, i16 0>
 ; CHECK-NEXT:    ret <4 x i16> [[SHL]]
 ;
@@ -1173,7 +1173,7 @@ entry:
 define <4 x i16> @add_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @add_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i16> [[ADD]]
 ;
 entry:
@@ -1186,7 +1186,7 @@ define <4 x i16> @add_constant_mask_undef_2(<4 x i16> %in) {
 ; CHECK-LABEL: @add_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 poison, i16 0, i16 3, i16 poison>
-; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 1>
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i16> [[ADD]]
 ;
 entry:
@@ -1198,7 +1198,7 @@ entry:
 define <4 x i16> @sub_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @sub_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i16> [[SUB]]
 ;
 entry:
@@ -1211,7 +1211,7 @@ define <4 x i16> @sub_constant_mask_undef_2(<4 x i16> %in) {
 ; CHECK-LABEL: @sub_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 poison, i16 0, i16 -10, i16 poison>
-; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[SUB]]
 ;
 entry:
@@ -1360,9 +1360,9 @@ define <2 x i1> @PR40734(<1 x i1> %x, <4 x i1> %y) {
 
 define <7 x i8> @insert_subvector_shuffles(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @insert_subvector_shuffles(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> poison, <7 x i32> <i32 undef, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <7 x i32> <i32 0, i32 8, i32 1, i32 undef, i32 8, i32 1, i32 9>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <7 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> poison, <7 x i32> <i32 poison, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <7 x i32> <i32 0, i32 8, i32 1, i32 poison, i32 8, i32 1, i32 9>
 ; CHECK-NEXT:    ret <7 x i8> [[S3]]
 ;
   %s1 = shufflevector <3 x i8> %x, <3 x i8> poison, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1373,7 +1373,7 @@ define <7 x i8> @insert_subvector_shuffles(<3 x i8> %x, <3 x i8> %y) {
 
 define <8 x i8> @insert_subvector_shuffles_pow2elts(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @insert_subvector_shuffles_pow2elts(
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <8 x i32> <i32 0, i32 2, i32 1, i32 undef, i32 2, i32 1, i32 3, i32 0>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <8 x i32> <i32 0, i32 2, i32 1, i32 poison, i32 2, i32 1, i32 3, i32 0>
 ; CHECK-NEXT:    ret <8 x i8> [[S3]]
 ;
   %s1 = shufflevector <2 x i8> %x, <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1387,8 +1387,8 @@ define <8 x i8> @insert_subvector_shuffles_pow2elts(<2 x i8> %x, <2 x i8> %y) {
 
 define <2 x i8> @insert_subvector_shuffles_narrowing(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @insert_subvector_shuffles_narrowing(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <7 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> poison, <7 x i32> <i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <7 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> poison, <7 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <2 x i32> <i32 0, i32 8>
 ; CHECK-NEXT:    ret <2 x i8> [[S3]]
 ;
@@ -1413,7 +1413,7 @@ define <2 x i8> @insert_subvector_shuffles_narrowing_pow2elts(<4 x i8> %x, <4 x
 
 define <4 x double> @insert_subvector_shuffles_identity(<2 x double> %x) {
 ; CHECK-LABEL: @insert_subvector_shuffles_identity(
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[S3]]
 ;
   %s1 = shufflevector <2 x double> %x, <2 x double> poison, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
@@ -1426,9 +1426,9 @@ define <4 x double> @insert_subvector_shuffles_identity(<2 x double> %x) {
 
 define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) {
 ; CHECK-LABEL: @not_insert_subvector_shuffle(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 undef, i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 7, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 7, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[S3]]
 ;
   %s1 = shufflevector <2 x double> %x, <2 x double> poison, <4 x i32> <i32 undef, i32 1, i32 undef, i32 1>
@@ -1441,9 +1441,9 @@ define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) {
 
 define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> %x, <3 x double> %y) {
 ; CHECK-LABEL: @not_insert_subvector_shuffles_with_same_size(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[S3]]
 ;
   %s1 = shufflevector <2 x double> %x, <2 x double> poison, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
@@ -1457,8 +1457,8 @@ define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> %
 
 define <4 x float> @insert_subvector_crash_invalid_mask_elt(<2 x float> %x, ptr %p) {
 ; CHECK-LABEL: @insert_subvector_crash_invalid_mask_elt(
-; CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[I:%.*]] = shufflevector <2 x float> [[X]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[I:%.*]] = shufflevector <2 x float> [[X]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    store <4 x float> [[I]], ptr [[P:%.*]], align 16
 ; CHECK-NEXT:    ret <4 x float> [[WIDEN]]
 ;
@@ -1554,7 +1554,7 @@ define <4 x i32> @splat_assoc_add_undef_constant_elt_at_splat_index(<4 x i32> %x
 
 define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_assoc_add_undef_mask_elts_undef_constant_elts(
-; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 0, i32 poison>
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -1567,7 +1567,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32>
 
 define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(
-; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -1580,7 +1580,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_e
 
 define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(
-; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 undef, i32 42, i32 undef, i32 42>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -1639,7 +1639,7 @@ define <3 x i8> @splat_assoc_mul_undef_elt1(<3 x i8> %x, <3 x i8> %y, <3 x i8> %
 define <3 x i8> @splat_assoc_mul_undef_elt2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
 ; CHECK-LABEL: @splat_assoc_mul_undef_elt2(
 ; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> <i32 2, i32 2, i32 2>
-; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> <i32 undef, i32 2, i32 2>
+; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> <i32 poison, i32 2, i32 2>
 ; CHECK-NEXT:    [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]]
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
@@ -1668,7 +1668,7 @@ define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index1(<3 x i8> %x, <3 x i8>
 define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
 ; CHECK-LABEL: @splat_assoc_mul_undef_elt_at_splat_index2(
 ; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> poison, <3 x i32> <i32 2, i32 2, i32 2>
-; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> <i32 2, i32 2, i32 undef>
+; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> poison, <3 x i32> <i32 2, i32 2, i32 poison>
 ; CHECK-NEXT:    [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]]
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
@@ -1779,7 +1779,7 @@ define <4 x i32> @splat_assoc_add_mul(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @PR46872(<4 x i32> %x) {
 ; CHECK-LABEL: @PR46872(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 1>
 ; CHECK-NEXT:    [[A:%.*]] = and <4 x i32> [[S]], bitcast (<2 x i64> <i64 ptrtoint (ptr @PR46872 to i64), i64 ptrtoint (ptr @PR46872 to i64)> to <4 x i32>)
 ; CHECK-NEXT:    ret <4 x i32> [[A]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index 516c99c428818..b67c0de6ccb01 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -81,7 +81,7 @@ define float @testvscale6(<vscale x 4 x float> %X) {
 
 define <4 x float> @test7(<4 x float> %x) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 >
@@ -91,7 +91,7 @@ define <4 x float> @test7(<4 x float> %x) {
 ; This should turn into a single shuffle.
 define <4 x float> @test8(<4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 undef, i32 3, i32 4>
+; CHECK-NEXT:    [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 poison, i32 3, i32 4>
 ; CHECK-NEXT:    ret <4 x float> [[T134]]
 ;
   %t4 = extractelement <4 x float> %x, i32 1
@@ -122,8 +122,8 @@ define <4 x i8> @test9(<16 x i8> %t6) {
 
 define <4 x i8> @test9a(<16 x i8> %t6) {
 ; CHECK-LABEL: @test9a(
-; CHECK-NEXT:    [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> <i32 undef, i32 9, i32 4, i32 8>
-; CHECK-NEXT:    [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> <i32 3, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> <i32 poison, i32 9, i32 4, i32 8>
+; CHECK-NEXT:    [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> <i32 3, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[T9]]
 ;
   %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 undef, i32 9, i32 4, i32 8 >
@@ -208,7 +208,7 @@ define <2 x i8> @extract_subvector_of_shuffle(<2 x i8> %x, <2 x i8> %y) {
 
 define <4 x i8> @extract_subvector_of_shuffle_undefs_types(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @extract_subvector_of_shuffle_undefs_types(
-; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 2, i32 0, i32 undef>
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 poison, i32 2, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
 ;
   %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
@@ -222,9 +222,9 @@ declare void @use_v5i8(<5 x i8>)
 
 define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @extract_subvector_of_shuffle_extra_use(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 poison, i32 2, i32 0, i32 1, i32 0>
 ; CHECK-NEXT:    call void @use_v5i8(<5 x i8> [[SHUF]])
-; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
 ;
   %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
@@ -254,7 +254,7 @@ define <3 x i32> @add_wider(i32 %y, i32 %z) {
 ; CHECK-NEXT:    [[I0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i64 1
 ; CHECK-NEXT:    [[A:%.*]] = add <2 x i32> [[I1]], <i32 255, i32 255>
-; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i32> [[EXT]]
 ;
   %i0 = insertelement <2 x i32> undef, i32 %y, i32 0
@@ -271,7 +271,7 @@ define <3 x i32> @div_wider(i32 %y, i32 %z) {
 ; CHECK-NEXT:    [[I0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i64 1
 ; CHECK-NEXT:    [[A:%.*]] = sdiv <2 x i32> [[I1]], <i32 255, i32 255>
-; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x i32> [[EXT]]
 ;
   %i0 = insertelement <2 x i32> undef, i32 %y, i32 0
@@ -591,7 +591,7 @@ define <2 x float> @fmul_const_invalid_constant(<2 x float> %v) {
 define <4 x i8> @widening_shuffle_add_1(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 42, i8 43>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -604,7 +604,7 @@ define <4 x i8> @widening_shuffle_add_1(<2 x i8> %x) {
 define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 43, i8 42>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
@@ -616,7 +616,7 @@ define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) {
 
 define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_invalid_constant(
-; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -629,7 +629,7 @@ define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) {
 
 define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) {
 ; CHECK-LABEL: @widening_shuffle_add_invalid_mask(
-; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 0>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -644,7 +644,7 @@ define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) {
 define <4 x i16> @widening_shuffle_shl_constant_op0(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_shl_constant_op0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i16> <i16 42, i16 -42>, [[V:%.*]]
-; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
   %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -658,7 +658,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op0(<2 x i16> %v) {
 define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_shl_constant_op1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i16> [[V:%.*]], <i16 2, i16 4>
-; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
   %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
@@ -671,7 +671,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) {
 
 define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BO:%.*]] = shl <4 x i16> [[SHUF]], <i16 2, i16 4, i16 1, i16 2>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
@@ -685,7 +685,7 @@ define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) {
 
 define <4 x i16> @widening_shuffle_or(<2 x i16> %v) {
 ; CHECK-LABEL: @widening_shuffle_or(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BO:%.*]] = or <4 x i16> [[SHUF]], <i16 42, i16 -42, i16 -1, i16 -1>
 ; CHECK-NEXT:    ret <4 x i16> [[BO]]
 ;
@@ -1031,7 +1031,7 @@ define <2 x i32> @and_splat_constant(<2 x i32> %x) {
 define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) {
 ; CHECK-LABEL: @and_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 0, i16 0, i16 -1, i16 -1>
 ; CHECK-NEXT:    ret <4 x i16> [[AND]]
 ;
@@ -1046,7 +1046,7 @@ entry:
 define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) {
 ; CHECK-LABEL: @and_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 -1, i16 0>
 ; CHECK-NEXT:    ret <4 x i16> [[AND]]
 ;
@@ -1073,7 +1073,7 @@ define <4 x i16> @and_constant_mask_undef_4(<4 x i16> %add) {
 ; CHECK-LABEL: @and_constant_mask_undef_4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], <i16 9, i16 20, i16 poison, i16 poison>
-; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[AND]]
 ;
 entry:
@@ -1100,7 +1100,7 @@ entry:
 define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @or_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 0, i16 0>
 ; CHECK-NEXT:    ret <4 x i16> [[OR]]
 ;
@@ -1115,7 +1115,7 @@ entry:
 define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) {
 ; CHECK-LABEL: @or_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 0, i16 0, i16 -1>
 ; CHECK-NEXT:    ret <4 x i16> [[OR]]
 ;
@@ -1142,7 +1142,7 @@ define <4 x i16> @or_constant_mask_undef_4(<4 x i16> %in) {
 ; CHECK-LABEL: @or_constant_mask_undef_4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], <i16 poison, i16 99, i16 poison, i16 poison>
-; CHECK-NEXT:    [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[OR]]
 ;
 entry:
@@ -1167,7 +1167,7 @@ entry:
 define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @shl_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 0, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], <i16 10, i16 3, i16 0, i16 0>
 ; CHECK-NEXT:    ret <4 x i16> [[SHL]]
 ;
@@ -1180,7 +1180,7 @@ entry:
 define <4 x i16> @add_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @add_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i16> [[ADD]]
 ;
 entry:
@@ -1193,7 +1193,7 @@ define <4 x i16> @add_constant_mask_undef_2(<4 x i16> %in) {
 ; CHECK-LABEL: @add_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 poison, i16 0, i16 3, i16 poison>
-; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 undef, i32 2, i32 1, i32 1>
+; CHECK-NEXT:    [[ADD:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 poison, i32 2, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i16> [[ADD]]
 ;
 entry:
@@ -1205,7 +1205,7 @@ entry:
 define <4 x i16> @sub_constant_mask_undef(<4 x i16> %in) {
 ; CHECK-LABEL: @sub_constant_mask_undef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i16> [[SUB]]
 ;
 entry:
@@ -1218,7 +1218,7 @@ define <4 x i16> @sub_constant_mask_undef_2(<4 x i16> %in) {
 ; CHECK-LABEL: @sub_constant_mask_undef_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 poison, i16 0, i16 -10, i16 poison>
-; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[SUB:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[SUB]]
 ;
 entry:
@@ -1367,9 +1367,9 @@ define <2 x i1> @PR40734(<1 x i1> %x, <4 x i1> %y) {
 
 define <7 x i8> @insert_subvector_shuffles(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @insert_subvector_shuffles(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> <i32 undef, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <7 x i32> <i32 0, i32 8, i32 1, i32 undef, i32 8, i32 1, i32 9>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> <i32 poison, i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <7 x i32> <i32 0, i32 8, i32 1, i32 poison, i32 8, i32 1, i32 9>
 ; CHECK-NEXT:    ret <7 x i8> [[S3]]
 ;
   %s1 = shufflevector <3 x i8> %x, <3 x i8> undef, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1380,7 +1380,7 @@ define <7 x i8> @insert_subvector_shuffles(<3 x i8> %x, <3 x i8> %y) {
 
 define <8 x i8> @insert_subvector_shuffles_pow2elts(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @insert_subvector_shuffles_pow2elts(
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <8 x i32> <i32 0, i32 2, i32 1, i32 undef, i32 2, i32 1, i32 3, i32 0>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <8 x i32> <i32 0, i32 2, i32 1, i32 poison, i32 2, i32 1, i32 3, i32 0>
 ; CHECK-NEXT:    ret <8 x i8> [[S3]]
 ;
   %s1 = shufflevector <2 x i8> %x, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1394,8 +1394,8 @@ define <8 x i8> @insert_subvector_shuffles_pow2elts(<2 x i8> %x, <2 x i8> %y) {
 
 define <2 x i8> @insert_subvector_shuffles_narrowing(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @insert_subvector_shuffles_narrowing(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> <i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <2 x i32> <i32 0, i32 8>
 ; CHECK-NEXT:    ret <2 x i8> [[S3]]
 ;
@@ -1420,7 +1420,7 @@ define <2 x i8> @insert_subvector_shuffles_narrowing_pow2elts(<4 x i8> %x, <4 x
 
 define <4 x double> @insert_subvector_shuffles_identity(<2 x double> %x) {
 ; CHECK-LABEL: @insert_subvector_shuffles_identity(
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[S3]]
 ;
   %s1 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
@@ -1433,9 +1433,9 @@ define <4 x double> @insert_subvector_shuffles_identity(<2 x double> %x) {
 
 define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) {
 ; CHECK-LABEL: @not_insert_subvector_shuffle(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 7, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 7, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[S3]]
 ;
   %s1 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 1>
@@ -1448,9 +1448,9 @@ define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) {
 
 define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> %x, <3 x double> %y) {
 ; CHECK-LABEL: @not_insert_subvector_shuffles_with_same_size(
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> undef, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[S3]]
 ;
   %s1 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
@@ -1464,8 +1464,8 @@ define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> %
 
 define <4 x float> @insert_subvector_crash_invalid_mask_elt(<2 x float> %x, ptr %p) {
 ; CHECK-LABEL: @insert_subvector_crash_invalid_mask_elt(
-; CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[I:%.*]] = shufflevector <2 x float> [[X]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[I:%.*]] = shufflevector <2 x float> [[X]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    store <4 x float> [[I]], ptr [[P:%.*]], align 16
 ; CHECK-NEXT:    ret <4 x float> [[WIDEN]]
 ;
@@ -1561,7 +1561,7 @@ define <4 x i32> @splat_assoc_add_undef_constant_elt_at_splat_index(<4 x i32> %x
 
 define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_assoc_add_undef_mask_elts_undef_constant_elts(
-; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 undef>
+; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 poison, i32 0, i32 poison>
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -1574,7 +1574,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32>
 
 define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(
-; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -1587,7 +1587,7 @@ define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_e
 
 define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(
-; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 poison, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 undef, i32 42, i32 undef, i32 42>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -1646,7 +1646,7 @@ define <3 x i8> @splat_assoc_mul_undef_elt1(<3 x i8> %x, <3 x i8> %y, <3 x i8> %
 define <3 x i8> @splat_assoc_mul_undef_elt2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
 ; CHECK-LABEL: @splat_assoc_mul_undef_elt2(
 ; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
-; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 undef, i32 2, i32 2>
+; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 poison, i32 2, i32 2>
 ; CHECK-NEXT:    [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]]
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
@@ -1675,7 +1675,7 @@ define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index1(<3 x i8> %x, <3 x i8>
 define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
 ; CHECK-LABEL: @splat_assoc_mul_undef_elt_at_splat_index2(
 ; CHECK-NEXT:    [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
-; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 undef>
+; CHECK-NEXT:    [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 poison>
 ; CHECK-NEXT:    [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]]
 ; CHECK-NEXT:    ret <3 x i8> [[R]]
@@ -1786,7 +1786,7 @@ define <4 x i32> @splat_assoc_add_mul(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @PR46872(<4 x i32> %x) {
 ; CHECK-LABEL: @PR46872(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 poison, i32 0, i32 1, i32 1>
 ; CHECK-NEXT:    [[A:%.*]] = and <4 x i32> [[S]], bitcast (<2 x i64> <i64 ptrtoint (ptr @PR46872 to i64), i64 ptrtoint (ptr @PR46872 to i64)> to <4 x i32>)
 ; CHECK-NEXT:    ret <4 x i32> [[A]]
 ;
@@ -1808,7 +1808,7 @@ define <2 x float> @fabs_unary_shuf(<2 x float> %x) {
 
 define <4 x half> @fabs_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf_widen(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = call ninf <4 x half> @llvm.fabs.v4f16(<4 x half> [[TMP1]])
 ; CHECK-NEXT:    ret <4 x half> [[R]]
 ;
@@ -1834,7 +1834,7 @@ define <2 x float> @fabs_unary_shuf_use(<2 x float> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf_use(
 ; CHECK-NEXT:    [[NX:%.*]] = call nsz <2 x float> @llvm.fabs.v2f32(<2 x float> [[X:%.*]])
 ; CHECK-NEXT:    call void @use(<2 x float> [[NX]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[NX]], <2 x float> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[NX]], <2 x float> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
   %nx = call nsz <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
@@ -1863,7 +1863,7 @@ define <4 x float> @fabs_shuf_widen_use1(<2 x float> %x, <2 x float> %y) {
 ; CHECK-LABEL: @fabs_shuf_widen_use1(
 ; CHECK-NEXT:    [[NX:%.*]] = call nnan <2 x float> @llvm.fabs.v2f32(<2 x float> [[X:%.*]])
 ; CHECK-NEXT:    call void @use(<2 x float> [[NX]])
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X]], <2 x float> [[Y:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X]], <2 x float> [[Y:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = call nnan <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -1923,7 +1923,7 @@ define <2 x float> @fneg_unary_shuf(<2 x float> %x) {
 
 define <4 x half> @fneg_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf_widen(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = fneg ninf <4 x half> [[TMP1]]
 ; CHECK-NEXT:    ret <4 x half> [[R]]
 ;
@@ -1949,7 +1949,7 @@ define <2 x float> @fneg_unary_shuf_use(<2 x float> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf_use(
 ; CHECK-NEXT:    [[NX:%.*]] = fneg nsz <2 x float> [[X:%.*]]
 ; CHECK-NEXT:    call void @use(<2 x float> [[NX]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[NX]], <2 x float> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[NX]], <2 x float> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
   %nx = fneg nsz <2 x float> %x
@@ -1978,7 +1978,7 @@ define <4 x float> @fneg_shuf_widen_use1(<2 x float> %x, <2 x float> %y) {
 ; CHECK-LABEL: @fneg_shuf_widen_use1(
 ; CHECK-NEXT:    [[NX:%.*]] = fneg nnan <2 x float> [[X:%.*]]
 ; CHECK-NEXT:    call void @use(<2 x float> [[NX]])
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X]], <2 x float> [[Y:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X]], <2 x float> [[Y:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = fneg nnan <4 x float> [[TMP1]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -2087,7 +2087,7 @@ define <2 x float> @uitofp_unary_shuf(<2 x i32> %x) {
 define <4 x i16> @fptosi_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fptosi_unary_shuf_widen(
 ; CHECK-NEXT:    [[NX:%.*]] = fptosi <2 x half> [[X:%.*]] to <2 x i16>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i16> [[NX]], <2 x i16> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i16> [[NX]], <2 x i16> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x i16> [[R]]
 ;
   %nx = fptosi <2 x half> %x to <2 x i16>
@@ -2098,7 +2098,7 @@ define <4 x i16> @fptosi_unary_shuf_widen(<2 x half> %x) {
 define <4 x i32> @fptoui_unary_shuf_widen_widen_elts(<2 x half> %x) {
 ; CHECK-LABEL: @fptoui_unary_shuf_widen_widen_elts(
 ; CHECK-NEXT:    [[NX:%.*]] = fptoui <2 x half> [[X:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[NX]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[NX]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %nx = fptoui <2 x half> %x to <2 x i32>
@@ -2109,7 +2109,7 @@ define <4 x i32> @fptoui_unary_shuf_widen_widen_elts(<2 x half> %x) {
 define <4 x half> @sitofp_unary_shuf_widen_narrow_elts(<2 x i32> %x) {
 ; CHECK-LABEL: @sitofp_unary_shuf_widen_narrow_elts(
 ; CHECK-NEXT:    [[NX:%.*]] = sitofp <2 x i32> [[X:%.*]] to <2 x half>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x half> [[NX]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x half> [[NX]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[R]]
 ;
   %nx = sitofp <2 x i32> %x to <2 x half>
@@ -2120,7 +2120,7 @@ define <4 x half> @sitofp_unary_shuf_widen_narrow_elts(<2 x i32> %x) {
 define <4 x half> @uitofp_unary_shuf_widen(<2 x i16> %x) {
 ; CHECK-LABEL: @uitofp_unary_shuf_widen(
 ; CHECK-NEXT:    [[NX:%.*]] = uitofp <2 x i16> [[X:%.*]] to <2 x half>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x half> [[NX]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x half> [[NX]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[R]]
 ;
   %nx = uitofp <2 x i16> %x to <2 x half>
@@ -2186,7 +2186,7 @@ define <4 x i32> @fptosi_shuf(<4 x float> %x, <4 x float> %y) {
 
 define <3 x i16> @fptoui_shuf(<3 x half> %x, <3 x half> %y) {
 ; CHECK-LABEL: @fptoui_shuf(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x half> [[X:%.*]], <3 x half> [[Y:%.*]], <3 x i32> <i32 0, i32 undef, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x half> [[X:%.*]], <3 x half> [[Y:%.*]], <3 x i32> <i32 0, i32 poison, i32 4>
 ; CHECK-NEXT:    [[R:%.*]] = fptoui <3 x half> [[TMP1]] to <3 x i16>
 ; CHECK-NEXT:    ret <3 x i16> [[R]]
 ;
@@ -2312,7 +2312,7 @@ define <4 x float> @sitofp_shuf_widen(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @sitofp_shuf_widen(
 ; CHECK-NEXT:    [[NX:%.*]] = sitofp <2 x i32> [[X:%.*]] to <2 x float>
 ; CHECK-NEXT:    [[NY:%.*]] = sitofp <2 x i32> [[Y:%.*]] to <2 x float>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[NX]], <2 x float> [[NY]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[NX]], <2 x float> [[NY]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %nx = sitofp <2 x i32> %x to <2 x float>
diff --git a/llvm/test/Transforms/InstCombine/vector-casts.ll b/llvm/test/Transforms/InstCombine/vector-casts.ll
index 877e20d3c2981..8ad81831277b6 100644
--- a/llvm/test/Transforms/InstCombine/vector-casts.ll
+++ b/llvm/test/Transforms/InstCombine/vector-casts.ll
@@ -414,7 +414,7 @@ define <2 x i64> @zext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
 define <4 x float> @sitofp_shuf(<4 x i32> %x) {
 ; CHECK-LABEL: @sitofp_shuf(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[X:%.*]] to <4 x float>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 undef>
@@ -425,7 +425,7 @@ define <4 x float> @sitofp_shuf(<4 x i32> %x) {
 define <3 x half> @uitofp_shuf(<3 x i16> %x) {
 ; CHECK-LABEL: @uitofp_shuf(
 ; CHECK-NEXT:    [[TMP1:%.*]] = uitofp <3 x i16> [[X:%.*]] to <3 x half>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <3 x i32> <i32 2, i32 poison, i32 0>
 ; CHECK-NEXT:    ret <3 x half> [[R]]
 ;
   %s = shufflevector <3 x i16> %x, <3 x i16> poison, <3 x i32> <i32 2, i32 undef, i32 0>
@@ -436,7 +436,7 @@ define <3 x half> @uitofp_shuf(<3 x i16> %x) {
 define <4 x i64> @fptosi_shuf(<4 x double> %x) {
 ; CHECK-LABEL: @fptosi_shuf(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fptosi <4 x double> [[X:%.*]] to <4 x i64>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> <i32 undef, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> <i32 poison, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    ret <4 x i64> [[R]]
 ;
   %s = shufflevector <4 x double> %x, <4 x double> poison, <4 x i32> <i32 undef, i32 2, i32 3, i32 undef>
@@ -460,7 +460,7 @@ define <2 x i32> @fptoui_shuf(<2 x float> %x) {
 
 define <4 x half> @narrowing_sitofp_shuf(<4 x i32> %x) {
 ; CHECK-LABEL: @narrowing_sitofp_shuf(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = sitofp <4 x i32> [[S]] to <4 x half>
 ; CHECK-NEXT:    ret <4 x half> [[R]]
 ;
@@ -473,7 +473,7 @@ define <4 x half> @narrowing_sitofp_shuf(<4 x i32> %x) {
 
 define <4 x double> @widening_uitofp_shuf(<4 x i32> %x) {
 ; CHECK-LABEL: @widening_uitofp_shuf(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = uitofp <4 x i32> [[S]] to <4 x double>
 ; CHECK-NEXT:    ret <4 x double> [[R]]
 ;
@@ -486,7 +486,7 @@ define <4 x double> @widening_uitofp_shuf(<4 x i32> %x) {
 
 define <3 x i64> @fptosi_narrowing_shuf(<4 x double> %x) {
 ; CHECK-LABEL: @fptosi_narrowing_shuf(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[X:%.*]], <4 x double> poison, <3 x i32> <i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[X:%.*]], <4 x double> poison, <3 x i32> <i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = fptosi <3 x double> [[S]] to <3 x i64>
 ; CHECK-NEXT:    ret <3 x i64> [[R]]
 ;
@@ -514,7 +514,7 @@ define <3 x i32> @fptoui_widening_shuf(<2 x float> %x) {
 
 define <4 x half> @narrowing_sitofp_widening_shuf(<2 x i32> %x) {
 ; CHECK-LABEL: @narrowing_sitofp_widening_shuf(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = sitofp <4 x i32> [[S]] to <4 x half>
 ; CHECK-NEXT:    ret <4 x half> [[R]]
 ;
@@ -529,7 +529,7 @@ declare void @use(<4 x i32>)
 
 define <4 x float> @sitofp_shuf_extra_use(<4 x i32> %x) {
 ; CHECK-LABEL: @sitofp_shuf_extra_use(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 poison>
 ; CHECK-NEXT:    call void @use(<4 x i32> [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = sitofp <4 x i32> [[S]] to <4 x float>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
diff --git a/llvm/test/Transforms/InstCombine/vector-concat-binop-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector-concat-binop-inseltpoison.ll
index 350b921c7192c..a6e93e024d3d5 100644
--- a/llvm/test/Transforms/InstCombine/vector-concat-binop-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vector-concat-binop-inseltpoison.ll
@@ -48,7 +48,7 @@ define <4 x i8> @mul(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 
 define <4 x i8> @and(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @and(
-; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = and <4 x i8> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
@@ -64,7 +64,7 @@ define <4 x i8> @and(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 define <4 x i8> @or(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @or(
 ; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = or <4 x i8> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -78,8 +78,8 @@ define <4 x i8> @or(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 
 define <4 x i8> @xor(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @xor(
-; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
-; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = xor <4 x i8> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -93,7 +93,7 @@ define <4 x i8> @shl(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @shl(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
@@ -106,7 +106,7 @@ define <4 x i8> @lshr(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @lshr(
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 poison, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
@@ -139,7 +139,7 @@ define <4 x i8> @sdiv(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @sdiv(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sdiv exact <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sdiv exact <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
@@ -180,7 +180,7 @@ define <4 x i8> @urem(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @urem(
 ; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = urem <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
@@ -208,7 +208,7 @@ define <4 x float> @fsub(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x fl
 ; CHECK-LABEL: @fsub(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x float> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fsub fast <2 x float> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
@@ -222,8 +222,8 @@ declare void @use2(<4 x float>)
 
 define <4 x float> @fmul(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
 ; CHECK-LABEL: @fmul(
-; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
-; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x float> [[C:%.*]], <2 x float> [[D:%.*]], <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x float> [[C:%.*]], <2 x float> [[D:%.*]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    call void @use2(<4 x float> [[CONCAT2]])
 ; CHECK-NEXT:    [[R:%.*]] = fmul nnan <4 x float> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
@@ -254,7 +254,7 @@ define <4 x float> @frem(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x fl
 ; CHECK-LABEL: @frem(
 ; CHECK-NEXT:    [[TMP1:%.*]] = frem <2 x float> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = frem <2 x float> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
diff --git a/llvm/test/Transforms/InstCombine/vector-concat-binop.ll b/llvm/test/Transforms/InstCombine/vector-concat-binop.ll
index 2d097e783c4c8..e149726db82ee 100644
--- a/llvm/test/Transforms/InstCombine/vector-concat-binop.ll
+++ b/llvm/test/Transforms/InstCombine/vector-concat-binop.ll
@@ -48,7 +48,7 @@ define <4 x i8> @mul(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 
 define <4 x i8> @and(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @and(
-; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = and <4 x i8> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
@@ -64,7 +64,7 @@ define <4 x i8> @and(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 define <4 x i8> @or(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @or(
 ; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = or <4 x i8> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -78,8 +78,8 @@ define <4 x i8> @or(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 
 define <4 x i8> @xor(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @xor(
-; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
-; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = xor <4 x i8> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
@@ -93,7 +93,7 @@ define <4 x i8> @shl(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @shl(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
@@ -106,7 +106,7 @@ define <4 x i8> @lshr(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @lshr(
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 poison, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
@@ -139,7 +139,7 @@ define <4 x i8> @sdiv(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @sdiv(
 ; CHECK-NEXT:    [[TMP1:%.*]] = sdiv exact <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sdiv exact <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
@@ -180,7 +180,7 @@ define <4 x i8> @urem(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
 ; CHECK-LABEL: @urem(
 ; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i8> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = urem <2 x i8> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 poison, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x i8> [[R]]
 ;
   %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
@@ -208,7 +208,7 @@ define <4 x float> @fsub(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x fl
 ; CHECK-LABEL: @fsub(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x float> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = fsub fast <2 x float> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
@@ -222,8 +222,8 @@ declare void @use2(<4 x float>)
 
 define <4 x float> @fmul(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
 ; CHECK-LABEL: @fmul(
-; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
-; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x float> [[C:%.*]], <2 x float> [[D:%.*]], <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x float> [[C:%.*]], <2 x float> [[D:%.*]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    call void @use2(<4 x float> [[CONCAT2]])
 ; CHECK-NEXT:    [[R:%.*]] = fmul nnan <4 x float> [[CONCAT1]], [[CONCAT2]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
@@ -254,7 +254,7 @@ define <4 x float> @frem(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x fl
 ; CHECK-LABEL: @frem(
 ; CHECK-NEXT:    [[TMP1:%.*]] = frem <2 x float> [[A:%.*]], [[C:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = frem <2 x float> [[B:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
diff --git a/llvm/test/Transforms/InstSimplify/canonicalize.ll b/llvm/test/Transforms/InstSimplify/canonicalize.ll
index f3d3ce9c8ad0c..96b23acdf1c73 100644
--- a/llvm/test/Transforms/InstSimplify/canonicalize.ll
+++ b/llvm/test/Transforms/InstSimplify/canonicalize.ll
@@ -146,6 +146,100 @@ define float @canonicalize_neg_denorm_positive_zero_input() "denormal-fp-math"="
   ret float %ret
 }
 
+define float @canonicalize_pos_denorm_dynamic_dynamic() "denormal-fp-math"="dynamic,dynamic" {
+; CHECK-LABEL: @canonicalize_pos_denorm_dynamic_dynamic(
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.canonicalize.f32(float 0x380FFFFFC0000000)
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 8388607 to float))
+  ret float %ret
+}
+
+define float @canonicalize_neg_denorm_dynamic_dynamic() "denormal-fp-math"="dynamic,dynamic" {
+; CHECK-LABEL: @canonicalize_neg_denorm_dynamic_dynamic(
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.canonicalize.f32(float 0xB80FFFFFC0000000)
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 -2139095041 to float))
+  ret float %ret
+}
+
+; Dynamic output - cannot flush
+define float @canonicalize_pos_denorm_dynamic_output() "denormal-fp-math"="dynamic,ieee" {
+; CHECK-LABEL: @canonicalize_pos_denorm_dynamic_output(
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.canonicalize.f32(float 0x380FFFFFC0000000)
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 8388607 to float))
+  ret float %ret
+}
+
+; Dynamic output - cannot flush
+define float @canonicalize_neg_denorm_dynamic_output() "denormal-fp-math"="dynamic,ieee" {
+; CHECK-LABEL: @canonicalize_neg_denorm_dynamic_output(
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.canonicalize.f32(float 0xB80FFFFFC0000000)
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 -2139095041 to float))
+  ret float %ret
+}
+
+; Dynamic input - cannot flush
+define float @canonicalize_pos_denorm_dynamic_input() "denormal-fp-math"="ieee,dynamic" {
+; CHECK-LABEL: @canonicalize_pos_denorm_dynamic_input(
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.canonicalize.f32(float 0x380FFFFFC0000000)
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 8388607 to float))
+  ret float %ret
+}
+
+; Dynamic input - cannot flush
+define float @canonicalize_neg_denorm_dynamic_input() "denormal-fp-math"="ieee,dynamic" {
+; CHECK-LABEL: @canonicalize_neg_denorm_dynamic_input(
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.canonicalize.f32(float 0xB80FFFFFC0000000)
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 -2139095041 to float))
+  ret float %ret
+}
+
+; Input is flushed, can fold
+define float @canonicalize_pos_denorm_dynamic_output_preserve_sign_input() "denormal-fp-math"="dynamic,preserve-sign" {
+; CHECK-LABEL: @canonicalize_pos_denorm_dynamic_output_preserve_sign_input(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 8388607 to float))
+  ret float %ret
+}
+
+; Input is flushed, can fold
+define float @canonicalize_neg_denorm_dynamic_output_preserve_sign_input() "denormal-fp-math"="dynamic,preserve-sign" {
+; CHECK-LABEL: @canonicalize_neg_denorm_dynamic_output_preserve_sign_input(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 -2139095041 to float))
+  ret float %ret
+}
+
+; Output is known flushed, can fold
+define float @canonicalize_pos_preserve_sign_output_denorm_dynamic_input() "denormal-fp-math"="preserve-sign,dynamic" {
+; CHECK-LABEL: @canonicalize_pos_preserve_sign_output_denorm_dynamic_input(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 8388607 to float))
+  ret float %ret
+}
+
+; Output is known flushed, can fold
+define float @canonicalize_neg_denorm_preserve_sign_output_dynamic_input() "denormal-fp-math"="preserve-sign,dynamic" {
+; CHECK-LABEL: @canonicalize_neg_denorm_preserve_sign_output_dynamic_input(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %ret = call float @llvm.canonicalize.f32(float bitcast (i32 -2139095041 to float))
+  ret float %ret
+}
+
 define float @canonicalize_inf() {
 ; CHECK-LABEL: @canonicalize_inf(
 ; CHECK-NEXT:    ret float 0x7FF0000000000000
diff --git a/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal.ll b/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal.ll
index 09159a14c6f25..e3ff9416abdd1 100644
--- a/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal.ll
+++ b/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal.ll
@@ -1100,6 +1100,109 @@ entry:
   ret i1 %cmp
 }
 
+; ============================================================================ ;
+; dynamic mode tests
+; ============================================================================ ;
+
+define float @test_float_fadd_dynamic_ieee() #9 {
+; CHECK-LABEL: @test_float_fadd_dynamic_ieee(
+; CHECK-NEXT:    [[RESULT:%.*]] = fadd float 0xB810000000000000, 0x3800000000000000
+; CHECK-NEXT:    ret float [[RESULT]]
+;
+  %result = fadd float 0xB810000000000000, 0x3800000000000000
+  ret float %result
+}
+
+define float @test_float_fadd_ieee_dynamic() #10 {
+; CHECK-LABEL: @test_float_fadd_ieee_dynamic(
+; CHECK-NEXT:    [[RESULT:%.*]] = fadd float 0xB810000000000000, 0x3800000000000000
+; CHECK-NEXT:    ret float [[RESULT]]
+;
+  %result = fadd float 0xB810000000000000, 0x3800000000000000
+  ret float %result
+}
+
+define float @test_float_fadd_dynamic_dynamic() #11 {
+; CHECK-LABEL: @test_float_fadd_dynamic_dynamic(
+; CHECK-NEXT:    [[RESULT:%.*]] = fadd float 0xB810000000000000, 0x3800000000000000
+; CHECK-NEXT:    ret float [[RESULT]]
+;
+  %result = fadd float 0xB810000000000000, 0x3800000000000000
+  ret float %result
+}
+
+; Check for failed to fold on each operand
+define float @test_float_fadd_dynamic_dynamic_commute() #11 {
+; CHECK-LABEL: @test_float_fadd_dynamic_dynamic_commute(
+; CHECK-NEXT:    [[RESULT:%.*]] = fadd float 0x3800000000000000, 0xB810000000000000
+; CHECK-NEXT:    ret float [[RESULT]]
+;
+  %result = fadd float 0x3800000000000000, 0xB810000000000000
+  ret float %result
+}
+
+define i1 @fcmp_double_dynamic_ieee() #9 {
+; CHECK-LABEL: @fcmp_double_dynamic_ieee(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double 0x0008000000000000, 0x0
+  ret i1 %cmp
+}
+
+define i1 @fcmp_double_ieee_dynamic() #10 {
+; CHECK-LABEL: @fcmp_double_ieee_dynamic(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double 0x8000000000000, 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double 0x0008000000000000, 0x0
+  ret i1 %cmp
+}
+
+define i1 @fcmp_double_dynamic_dynamic() #11 {
+; CHECK-LABEL: @fcmp_double_dynamic_dynamic(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double 0x8000000000000, 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double 0x0008000000000000, 0x0
+  ret i1 %cmp
+}
+
+define i1 @fcmp_double_dynamic_dynamic_commute() #11 {
+; CHECK-LABEL: @fcmp_double_dynamic_dynamic_commute(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double 0.000000e+00, 0x8000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double 0x0, 0x0008000000000000
+  ret i1 %cmp
+}
+
+; Output doesn't matter.
+define i1 @fcmp_double_dynamic_psz() #12 {
+; CHECK-LABEL: @fcmp_double_dynamic_psz(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp une double 0x0008000000000000, 0x0
+  ret i1 %cmp
+}
+
+; Non-denormal values should fold
+define float @test_float_fadd_dynamic_dynamic_normals() #11 {
+; CHECK-LABEL: @test_float_fadd_dynamic_dynamic_normals(
+; CHECK-NEXT:    ret float 3.000000e+00
+;
+  %result = fadd float 1.0, 2.0
+  ret float %result
+}
+
+; Non-denormal values should fold
+define i1 @fcmp_double_dynamic_dynamic_normals() #11 {
+; CHECK-LABEL: @fcmp_double_dynamic_dynamic_normals(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double 1.0, 2.0
+  ret i1 %cmp
+}
+
 attributes #0 = { nounwind "denormal-fp-math"="ieee,ieee" }
 attributes #1 = { nounwind "denormal-fp-math"="positive-zero,ieee" }
 attributes #2 = { nounwind "denormal-fp-math"="preserve-sign,ieee" }
@@ -1109,3 +1212,7 @@ attributes #5 = { nounwind "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"
 attributes #6 = { nounwind "denormal-fp-math"="positive-zero,positive-zero" }
 attributes #7 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
 attributes #8 = { nounwind "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="positive-zero,positive-zero" }
+attributes #9 = { nounwind "denormal-fp-math"="dynamic,ieee" }
+attributes #10 = { nounwind "denormal-fp-math"="ieee,dynamic" }
+attributes #11 = { nounwind "denormal-fp-math"="dynamic,dynamic" }
+attributes #12 = { nounwind "denormal-fp-math"="dynamic,preserve-sign" }
diff --git a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
index a5785275afff4..a8a9e96a652fa 100644
--- a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
+++ b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
@@ -1206,9 +1206,7 @@ define float @maximum_inf_commute(float %x) {
 define float @maximum_maximum_minimum(float %x, float %y) {
 ; CHECK-LABEL: @maximum_maximum_minimum(
 ; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.minimum.f32(float [[X]], float [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.maximum.f32(float [[MAX]], float [[MIN]])
-; CHECK-NEXT:    ret float [[VAL]]
+; CHECK-NEXT:    ret float [[MAX]]
 ;
   %max = call float @llvm.maximum.f32(float %x, float %y)
   %min = call float @llvm.minimum.f32(float %x, float %y)
@@ -1219,9 +1217,7 @@ define float @maximum_maximum_minimum(float %x, float %y) {
 define double @maximum_minimum_maximum(double %x, double %y) {
 ; CHECK-LABEL: @maximum_minimum_maximum(
 ; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.maximum.f64(double [[X:%.*]], double [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.minimum.f64(double [[X]], double [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.maximum.f64(double [[MIN]], double [[MAX]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double [[MAX]]
 ;
   %max = call double @llvm.maximum.f64(double %x, double %y)
   %min = call double @llvm.minimum.f64(double %x, double %y)
@@ -1245,9 +1241,7 @@ define float @maximum_minimum_minimum(float %x, float %y) {
 define half @maximum_maximum_maximum(half %x, half %y) {
 ; CHECK-LABEL: @maximum_maximum_maximum(
 ; CHECK-NEXT:    [[MAX1:%.*]] = call half @llvm.maximum.f16(half [[X:%.*]], half [[Y:%.*]])
-; CHECK-NEXT:    [[MAX2:%.*]] = call half @llvm.maximum.f16(half [[X]], half [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call half @llvm.maximum.f16(half [[MAX1]], half [[MAX2]])
-; CHECK-NEXT:    ret half [[VAL]]
+; CHECK-NEXT:    ret half [[MAX1]]
 ;
   %max1 = call half @llvm.maximum.f16(half %x, half %y)
   %max2 = call half @llvm.maximum.f16(half %x, half %y)
@@ -1257,10 +1251,8 @@ define half @maximum_maximum_maximum(half %x, half %y) {
 
 define <2 x float> @minimum_maximum_minimum(<2 x float> %x, <2 x float> %y) {
 ; CHECK-LABEL: @minimum_maximum_minimum(
-; CHECK-NEXT:    [[MAX:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[X]], <2 x float> [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[MAX]], <2 x float> [[MIN]])
-; CHECK-NEXT:    ret <2 x float> [[VAL]]
+; CHECK-NEXT:    [[MIN:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[MIN]]
 ;
   %max = call <2 x float> @llvm.maximum.v2f32(<2 x float> %x, <2 x float> %y)
   %min = call <2 x float> @llvm.minimum.v2f32(<2 x float> %x, <2 x float> %y)
@@ -1270,10 +1262,8 @@ define <2 x float> @minimum_maximum_minimum(<2 x float> %x, <2 x float> %y) {
 
 define <2 x double> @minimum_minimum_maximum(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: @minimum_minimum_maximum(
-; CHECK-NEXT:    [[MAX:%.*]] = call <2 x double> @llvm.maximum.v2f64(<2 x double> [[X:%.*]], <2 x double> [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call <2 x double> @llvm.minimum.v2f64(<2 x double> [[X]], <2 x double> [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call <2 x double> @llvm.minimum.v2f64(<2 x double> [[MIN]], <2 x double> [[MAX]])
-; CHECK-NEXT:    ret <2 x double> [[VAL]]
+; CHECK-NEXT:    [[MIN:%.*]] = call <2 x double> @llvm.minimum.v2f64(<2 x double> [[X:%.*]], <2 x double> [[Y:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[MIN]]
 ;
   %max = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double> %y)
   %min = call <2 x double> @llvm.minimum.v2f64(<2 x double> %x, <2 x double> %y)
@@ -1297,9 +1287,7 @@ define float @minimum_maximum_maximum(float %x, float %y) {
 define float @minimum_minimum_minimum(float %x, float %y) {
 ; CHECK-LABEL: @minimum_minimum_minimum(
 ; CHECK-NEXT:    [[MIN1:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
-; CHECK-NEXT:    [[MIN2:%.*]] = call float @llvm.minimum.f32(float [[X]], float [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.minimum.f32(float [[MIN1]], float [[MIN2]])
-; CHECK-NEXT:    ret float [[VAL]]
+; CHECK-NEXT:    ret float [[MIN1]]
 ;
   %min1 = call float @llvm.minimum.f32(float %x, float %y)
   %min2 = call float @llvm.minimum.f32(float %x, float %y)
@@ -1310,9 +1298,7 @@ define float @minimum_minimum_minimum(float %x, float %y) {
 define double @maxnum_maxnum_minnum(double %x, double %y) {
 ; CHECK-LABEL: @maxnum_maxnum_minnum(
 ; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.maxnum.f64(double [[X:%.*]], double [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.minnum.f64(double [[X]], double [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.maxnum.f64(double [[MAX]], double [[MIN]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    ret double [[MAX]]
 ;
   %max = call double @llvm.maxnum.f64(double %x, double %y)
   %min = call double @llvm.minnum.f64(double %x, double %y)
@@ -1323,9 +1309,7 @@ define double @maxnum_maxnum_minnum(double %x, double %y) {
 define <2 x float> @maxnum_minnum_maxnum(<2 x float> %x, <2 x float> %y) {
 ; CHECK-LABEL: @maxnum_minnum_maxnum(
 ; CHECK-NEXT:    [[MAX:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[X]], <2 x float> [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[MIN]], <2 x float> [[MAX]])
-; CHECK-NEXT:    ret <2 x float> [[VAL]]
+; CHECK-NEXT:    ret <2 x float> [[MAX]]
 ;
   %max = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %x, <2 x float> %y)
   %min = call <2 x float> @llvm.minnum.v2f32(<2 x float> %x, <2 x float> %y)
@@ -1349,9 +1333,7 @@ define <2 x double> @maxnum_minnum_minmum(<2 x double> %x, <2 x double> %y) {
 define float @maxnum_maxnum_maxnum(float %x, float %y) {
 ; CHECK-LABEL: @maxnum_maxnum_maxnum(
 ; CHECK-NEXT:    [[MAX1:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
-; CHECK-NEXT:    [[MAX2:%.*]] = call float @llvm.maxnum.f32(float [[X]], float [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.maxnum.f32(float [[MAX1]], float [[MAX2]])
-; CHECK-NEXT:    ret float [[VAL]]
+; CHECK-NEXT:    ret float [[MAX1]]
 ;
   %max1 = call float @llvm.maxnum.f32(float %x, float %y)
   %max2 = call float @llvm.maxnum.f32(float %x, float %y)
@@ -1361,10 +1343,8 @@ define float @maxnum_maxnum_maxnum(float %x, float %y) {
 
 define double @minnum_maxnum_minnum(double %x, double %y) {
 ; CHECK-LABEL: @minnum_maxnum_minnum(
-; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.maxnum.f64(double [[X:%.*]], double [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.minnum.f64(double [[X]], double [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call double @llvm.minnum.f64(double [[MAX]], double [[MIN]])
-; CHECK-NEXT:    ret double [[VAL]]
+; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.minnum.f64(double [[X:%.*]], double [[Y:%.*]])
+; CHECK-NEXT:    ret double [[MIN]]
 ;
   %max = call double @llvm.maxnum.f64(double %x, double %y)
   %min = call double @llvm.minnum.f64(double %x, double %y)
@@ -1374,10 +1354,8 @@ define double @minnum_maxnum_minnum(double %x, double %y) {
 
 define float @minnum_minnum_maxnum(float %x, float %y) {
 ; CHECK-LABEL: @minnum_minnum_maxnum(
-; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
-; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.minnum.f32(float [[X]], float [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.minnum.f32(float [[MIN]], float [[MAX]])
-; CHECK-NEXT:    ret float [[VAL]]
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[MIN]]
 ;
   %max = call float @llvm.maxnum.f32(float %x, float %y)
   %min = call float @llvm.minnum.f32(float %x, float %y)
@@ -1401,9 +1379,7 @@ define <2 x float> @minnum_maxnum_maxnum(<2 x float> %x, <2 x float> %y) {
 define <2 x double> @minnum_minnum_minmum(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: @minnum_minnum_minmum(
 ; CHECK-NEXT:    [[MIN1:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[X:%.*]], <2 x double> [[Y:%.*]])
-; CHECK-NEXT:    [[MIN2:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[X]], <2 x double> [[Y]])
-; CHECK-NEXT:    [[VAL:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[MIN1]], <2 x double> [[MIN2]])
-; CHECK-NEXT:    ret <2 x double> [[VAL]]
+; CHECK-NEXT:    ret <2 x double> [[MIN1]]
 ;
   %min1 = call <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> %y)
   %min2 = call <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> %y)
diff --git a/llvm/test/Transforms/InstSimplify/implies.ll b/llvm/test/Transforms/InstSimplify/implies.ll
index f98ca3d6b1230..e94ad62e5e32c 100644
--- a/llvm/test/Transforms/InstSimplify/implies.ll
+++ b/llvm/test/Transforms/InstSimplify/implies.ll
@@ -1,10 +1,10 @@
-; NOTE: Assertions have been autogenerated by update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S %s -passes=instsimplify | FileCheck %s
 
 ; A ==> A -> true
 define i1 @test(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %var29 = icmp slt i32 %i, %length.i
   %res = icmp uge i1 %var29, %var29
@@ -14,7 +14,7 @@ define i1 @test(i32 %length.i, i32 %i) {
 ; i +_{nsw} C_{>0} <s L ==> i <s L -> true
 define i1 @test2(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test2(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nsw i32 %i, 1
   %var29 = icmp slt i32 %i, %length.i
@@ -26,9 +26,9 @@ define i1 @test2(i32 %length.i, i32 %i) {
 ; i + C_{>0} <s L ==> i <s L -> unknown without the nsw
 define i1 @test2_neg(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test2_neg(
-; CHECK:         [[IPLUS1:%.*]] = add i32 %i, 1
-; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 %i, %length.i
-; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], %length.i
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 [[I]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], [[LENGTH_I]]
 ; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
@@ -42,9 +42,9 @@ define i1 @test2_neg(i32 %length.i, i32 %i) {
 ; sle is not implication
 define i1 @test2_neg2(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test2_neg2(
-; CHECK:         [[IPLUS1:%.*]] = add i32 %i, 1
-; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 %i, %length.i
-; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], %length.i
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 [[I]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], [[LENGTH_I]]
 ; CHECK-NEXT:    [[RES:%.*]] = icmp sle i1 [[VAR30]], [[VAR29]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
@@ -58,9 +58,9 @@ define i1 @test2_neg2(i32 %length.i, i32 %i) {
 ; The binary operator has to be an add
 define i1 @test2_neg3(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test2_neg3(
-; CHECK:         [[IPLUS1:%.*]] = sub nsw i32 %i, 1
-; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 %i, %length.i
-; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], %length.i
+; CHECK-NEXT:    [[IPLUS1:%.*]] = sub nsw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 [[I]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], [[LENGTH_I]]
 ; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
@@ -75,7 +75,7 @@ define i1 @test2_neg3(i32 %length.i, i32 %i) {
 ; With an inverted conditional (ule B A rather than canonical ugt A B
 define i1 @test3(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test3(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nsw i32 %i, 1
   %var29 = icmp slt i32 %i, %length.i
@@ -87,7 +87,7 @@ define i1 @test3(i32 %length.i, i32 %i) {
 ; i +_{nuw} C <u L ==> i <u L
 define i1 @test4(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test4(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nuw i32 %i, 1
   %var29 = icmp ult i32 %i, %length.i
@@ -99,7 +99,7 @@ define i1 @test4(i32 %length.i, i32 %i) {
 ; A ==> A for vectors
 define <4 x i1> @test5(<4 x i1> %vec) {
 ; CHECK-LABEL: @test5(
-; CHECK:         ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
 ;
   %res = icmp ule <4 x i1> %vec, %vec
   ret <4 x i1> %res
@@ -108,7 +108,7 @@ define <4 x i1> @test5(<4 x i1> %vec) {
 ; Don't crash on vector inputs - pr25040
 define <4 x i1> @test6(<4 x i1> %a, <4 x i1> %b) {
 ; CHECK-LABEL: @test6(
-; CHECK:         [[RES:%.*]] = icmp ule <4 x i1> %a, %b
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule <4 x i1> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret <4 x i1> [[RES]]
 ;
   %res = icmp ule <4 x i1> %a, %b
@@ -118,7 +118,7 @@ define <4 x i1> @test6(<4 x i1> %a, <4 x i1> %b) {
 ; i +_{nsw} 1 <s L  ==> i < L +_{nsw} 1
 define i1 @test7(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test7(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nsw i32 %i, 1
   %len.plus.one = add nsw i32 %length.i, 1
@@ -131,7 +131,7 @@ define i1 @test7(i32 %length.i, i32 %i) {
 ; i +_{nuw} 1 <u L  ==> i < L +_{nuw} 1
 define i1 @test8(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test8(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nuw i32 %i, 1
   %len.plus.one = add nuw i32 %length.i, 1
@@ -144,7 +144,7 @@ define i1 @test8(i32 %length.i, i32 %i) {
 ; i +_{nuw} C <u L ==> i < L, even if C is negative
 define i1 @test9(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test9(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nuw i32 %i, -100
   %var29 = icmp ult i32 %i, %length.i
@@ -155,7 +155,7 @@ define i1 @test9(i32 %length.i, i32 %i) {
 
 define i1 @test10(i32 %length.i, i32 %x.full) {
 ; CHECK-LABEL: @test10(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %x = and i32 %x.full, 4294901760  ;; 4294901760 == 0xffff0000
   %large = or i32 %x, 100
@@ -168,10 +168,10 @@ define i1 @test10(i32 %length.i, i32 %x.full) {
 
 define i1 @test11(i32 %length.i, i32 %x) {
 ; CHECK-LABEL: @test11(
-; CHECK:         [[LARGE:%.*]] = or i32 %x, 100
-; CHECK-NEXT:    [[SMALL:%.*]] = or i32 %x, 90
-; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], %length.i
-; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], %length.i
+; CHECK-NEXT:    [[LARGE:%.*]] = or i32 [[X:%.*]], 100
+; CHECK-NEXT:    [[SMALL:%.*]] = or i32 [[X]], 90
+; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], [[LENGTH_I]]
 ; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[KNOWN]], [[TO_PROVE]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
@@ -185,11 +185,11 @@ define i1 @test11(i32 %length.i, i32 %x) {
 
 define i1 @test12(i32 %length.i, i32 %x.full) {
 ; CHECK-LABEL: @test12(
-; CHECK:         [[X:%.*]] = and i32 [[X:%.*]].full, -65536
+; CHECK-NEXT:    [[X:%.*]] = and i32 [[X_FULL:%.*]], -65536
 ; CHECK-NEXT:    [[LARGE:%.*]] = or i32 [[X]], 65536
 ; CHECK-NEXT:    [[SMALL:%.*]] = or i32 [[X]], 90
-; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], %length.i
-; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], %length.i
+; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], [[LENGTH_I]]
 ; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[KNOWN]], [[TO_PROVE]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
@@ -204,7 +204,7 @@ define i1 @test12(i32 %length.i, i32 %x.full) {
 
 define i1 @test13(i32 %length.i, i32 %x) {
 ; CHECK-LABEL: @test13(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %large = add nuw i32 %x, 100
   %small = add nuw i32 %x, 90
@@ -216,7 +216,7 @@ define i1 @test13(i32 %length.i, i32 %x) {
 
 define i1 @test14(i32 %length.i, i32 %x.full) {
 ; CHECK-LABEL: @test14(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %x = and i32 %x.full, 4294905615  ;; 4294905615 == 0xffff0f0f
   %large = or i32 %x, 8224 ;; == 0x2020
@@ -229,10 +229,10 @@ define i1 @test14(i32 %length.i, i32 %x.full) {
 
 define i1 @test15(i32 %length.i, i32 %x) {
 ; CHECK-LABEL: @test15(
-; CHECK:         [[LARGE:%.*]] = add nuw i32 %x, 100
-; CHECK-NEXT:    [[SMALL:%.*]] = add nuw i32 %x, 110
-; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], %length.i
-; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], %length.i
+; CHECK-NEXT:    [[LARGE:%.*]] = add nuw i32 [[X:%.*]], 100
+; CHECK-NEXT:    [[SMALL:%.*]] = add nuw i32 [[X]], 110
+; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], [[LENGTH_I]]
 ; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[KNOWN]], [[TO_PROVE]]
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
@@ -247,7 +247,7 @@ define i1 @test15(i32 %length.i, i32 %x) {
 ; X >=(s) Y == X ==> Y (i1 1 becomes -1 for reasoning)
 define i1 @test_sge(i32 %length.i, i32 %i) {
 ; CHECK-LABEL: @test_sge(
-; CHECK:         ret i1 true
+; CHECK-NEXT:    ret i1 true
 ;
   %iplus1 = add nsw nuw i32 %i, 1
   %var29 = icmp ult i32 %i, %length.i
@@ -267,3 +267,213 @@ define i1 @test_sle(i32 %length.i, i32 %i) {
   %res = icmp sle i1 %var29, %var30
   ret i1 %res
 }
+
+; X +_{nsw} 1 <(s) Y ==> X <(s) Y
+define i1 @test_sgt_icmp(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sgt_icmp(
+; CHECK-NEXT:    ret i1 true
+;
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp sgt i32 %length.i, %i
+  %var30 = icmp sgt i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; X +_{nsw} 1 <=(s) Y ==> X <=(s) Y
+define i1 @test_sge_icmp(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sge_icmp(
+; CHECK-NEXT:    ret i1 true
+;
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp sge i32 %length.i, %i
+  %var30 = icmp sge i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; X +_{nuw} 1 <(u) Y ==> X <(u) Y
+define i1 @test_ugt_icmp(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_ugt_icmp(
+; CHECK-NEXT:    ret i1 true
+;
+  %iplus1 = add nuw i32 %i, 1
+  %var29 = icmp ugt i32 %length.i, %i
+  %var30 = icmp ugt i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; X +_{nuw} 1 <=(u) Y ==> X <=(u) Y
+define i1 @test_uge_icmp(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_uge_icmp(
+; CHECK-NEXT:    ret i1 true
+;
+  %iplus1 = add nuw i32 %i, 1
+  %var29 = icmp uge i32 %length.i, %i
+  %var30 = icmp uge i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X + 1 <(s) Y !==> X <(s) Y (X = 0x7fffffff, Y = 0x7fbfffff)
+define i1 @test_sgt_icmp_no_nsw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sgt_icmp_no_nsw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp sgt i32 [[LENGTH_I:%.*]], [[I]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp sgt i32 [[LENGTH_I]], [[IPLUS1]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp sgt i32 %length.i, %i
+  %var30 = icmp sgt i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X +_{nuw} 1 <(s) Y !==> X <(s) Y
+define i1 @test_sgt_icmp_nuw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sgt_icmp_nuw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add nuw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp sgt i32 [[I]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp sgt i32 [[IPLUS1]], [[LENGTH_I]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add nuw i32 %i, 1
+  %var29 = icmp sgt i32 %i, %length.i
+  %var30 = icmp sgt i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X + 1 <=(s) Y !==> X <=(s) Y (X = 0x7fffffff, Y = 0x80000000)
+define i1 @test_sge_icmp_no_nsw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sge_icmp_no_nsw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp sge i32 [[LENGTH_I:%.*]], [[I]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp sge i32 [[LENGTH_I]], [[IPLUS1]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp sge i32 %length.i, %i
+  %var30 = icmp sge i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X +_{nuw} 1 <=(s) Y !==> X <=(s) Y
+define i1 @test_sge_icmp_nuw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sge_icmp_nuw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add nuw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp sge i32 [[I]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp sge i32 [[IPLUS1]], [[LENGTH_I]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add nuw i32 %i, 1
+  %var29 = icmp sge i32 %i, %length.i
+  %var30 = icmp sge i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+;negative case, X + 1 <(u) Y !==> X <(u) Y (X = Y = 0xffffffff)
+define i1 @test_ugt_icmp_no_nuw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_ugt_icmp_no_nuw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp ugt i32 [[LENGTH_I:%.*]], [[I]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp ugt i32 [[LENGTH_I]], [[IPLUS1]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp ugt i32 %length.i, %i
+  %var30 = icmp ugt i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X +_{nsw} 1 <(u) Y !==> X <(u) Y (X = 0xffffffff, Y = 0xfffffffe)
+define i1 @test_ugt_icmp_nsw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_ugt_icmp_nsw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add nsw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp ugt i32 [[LENGTH_I:%.*]], [[I]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp ugt i32 [[LENGTH_I]], [[IPLUS1]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp ugt i32 %length.i, %i
+  %var30 = icmp ugt i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X + 1 <=(u) Y !==> X <=(u) Y (X = 0xffffffff, Y = 0x0)
+define i1 @test_uge_icmp_no_nuw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_uge_icmp_no_nuw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp uge i32 [[LENGTH_I:%.*]], [[I]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp uge i32 [[LENGTH_I]], [[IPLUS1]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp uge i32 %length.i, %i
+  %var30 = icmp uge i32 %length.i, %iplus1
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; negative case, X +_{nsw} 1 <=(u) Y !==> X <=(u) Y
+define i1 @test_uge_icmp_nsw(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_uge_icmp_nsw(
+; CHECK-NEXT:    [[IPLUS1:%.*]] = add nsw i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp uge i32 [[I]], [[LENGTH_I:%.*]]
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp uge i32 [[IPLUS1]], [[LENGTH_I]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp uge i32 %i, %length.i
+  %var30 = icmp uge i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; (X >> Z) <=(u) Y ==> X <=(u) Y
+define i1 @test_shift(i64 %x, i64 %y, i64 %shift) {
+; CHECK-LABEL: @test_shift(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[X:%.*]], [[SHIFT:%.*]]
+; CHECK-NEXT:    [[ICMP1:%.*]] = icmp ugt i64 [[LSHR]], [[Y:%.*]]
+; CHECK-NEXT:    [[ICMP2:%.*]] = icmp ugt i64 [[X]], [[Y]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[ICMP1]])
+; CHECK-NEXT:    ret i1 [[ICMP2]]
+;
+  %lshr = lshr i64 %x, %shift
+  %icmp1 = icmp ugt i64 %lshr, %y
+  %icmp2 = icmp ugt i64 %x, %y
+  call void @llvm.assume(i1 %icmp1)
+  ret i1 %icmp2
+}
+
+; (X > Y +_{nuw} 1) ==> X != Y
+define i1 @assume_x_ugt_y_plus_1(i64  %x, i64  %y)  {
+; CHECK-LABEL: @assume_x_ugt_y_plus_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i64 [[Y:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %1 = add nuw i64 %y, 1
+  %2 = icmp ugt i64 %x, %1
+  tail call void @llvm.assume(i1 %2)
+  %3 = icmp eq i64 %x, %y
+  ret i1 %3
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index 0b5812a79315a..8fb136f3e394d 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -1404,3 +1404,37 @@ define <2 x i32> @select_icmp_and_shl2_vect(<2 x i32> %x) {
   %cond = select <2 x i1> %tobool.not, <2 x i32> %mul, <2 x i32> zeroinitializer
   ret <2 x i32> %cond
 }
+
+define ptr @select_op_replacement_in_phi(ptr %head) {
+; CHECK-LABEL: @select_op_replacement_in_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[CURRENT:%.*]] = phi ptr [ [[HEAD:%.*]], [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[PREV:%.*]] = phi ptr [ null, [[ENTRY]] ], [ [[CURRENT]], [[LATCH]] ]
+; CHECK-NEXT:    [[CURRENT_NULL:%.*]] = icmp eq ptr [[CURRENT]], null
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CURRENT_NULL]], ptr [[PREV]], ptr null
+; CHECK-NEXT:    br i1 [[CURRENT_NULL]], label [[EXIT:%.*]], label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[NEXT]] = load ptr, ptr [[CURRENT]], align 8
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret ptr [[SEL]]
+;
+entry:
+  br label %loop
+
+loop:
+  %current = phi ptr [ %head, %entry ], [ %next, %latch ]
+  %prev = phi ptr [ null, %entry ], [ %current, %latch ]
+  %current.null = icmp eq ptr %current, null
+  %sel = select i1 %current.null, ptr %prev, ptr null
+  br i1 %current.null, label %exit, label %latch
+
+latch:
+  %next = load ptr, ptr %current
+  br label %loop
+
+exit:
+  ret ptr %sel
+}
diff --git a/llvm/test/Transforms/InstSimplify/select_or_and.ll b/llvm/test/Transforms/InstSimplify/select_or_and.ll
index 2188d3fcb28d5..6bb7fbf5dc41d 100644
--- a/llvm/test/Transforms/InstSimplify/select_or_and.ll
+++ b/llvm/test/Transforms/InstSimplify/select_or_and.ll
@@ -64,10 +64,7 @@ define <4 x i32> @select_or_vec(<4 x i32> %x, <4 x i32> %y) {
 ; select(Y | X == 0, Y | X, Y)
 define i32 @select_or_not_1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_or_not_1(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], 0
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 [[Y]]
-; CHECK-NEXT:    ret i32 [[RET]]
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
 ;
   %or = or i32 %y, %x
   %cmp = icmp eq i32 %or, 0
@@ -77,10 +74,7 @@ define i32 @select_or_not_1(i32 %x, i32 %y) {
 ; select(Y | X != 0, Y, Y | X)
 define i32 @select_or_not_2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_or_not_2(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[OR]], 0
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
-; CHECK-NEXT:    ret i32 [[RET]]
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
 ;
   %or = or i32 %y, %x
   %cmp = icmp ne i32 %or, 0
@@ -153,10 +147,7 @@ define i32 @select_and_4(i32 %x, i32 %y) {
 ; select(Y & X != -1, Y, Y & X)
 define i32 @select_and_not_1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_and_not_1(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -1
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[AND]], i32 [[Y]]
-; CHECK-NEXT:    ret i32 [[RET]]
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
 ;
   %and = and i32 %y, %x
   %cmp = icmp eq i32 %and, -1
@@ -167,10 +158,7 @@ define i32 @select_and_not_1(i32 %x, i32 %y) {
 ; select(Y & X != -1, Y, Y & X)
 define i32 @select_and_not_2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @select_and_not_2(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], -1
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND]]
-; CHECK-NEXT:    ret i32 [[RET]]
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
 ;
   %and = and i32 %y, %x
   %cmp = icmp ne i32 %and, -1
@@ -191,3 +179,117 @@ define i32 @select_and_not_3(i32 %x, i32 %y) {
   %ret = select i1 %cmp, i32 %y, i32 %and
   ret i32 %ret
 }
+
+; https://alive2.llvm.org/ce/z/1ILbih
+define i32 @select_icmp_and_eq(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_and_eq(
+; CHECK-NEXT:    ret i32 -1
+;
+  %and = and i32 %a, %b
+  %tobool = icmp eq i32 %and, -1
+  %cond = select i1 %tobool, i32 %a, i32 -1
+  ret i32 %cond
+}
+
+define i32 @select_icmp_and_eq_commuted(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_and_eq_commuted(
+; CHECK-NEXT:    ret i32 -1
+;
+  %and = and i32 %a, %b
+  %tobool = icmp eq i32 %and, -1
+  %cond = select i1 %tobool, i32 %b, i32 -1
+  ret i32 %cond
+}
+
+; https://alive2.llvm.org/ce/z/HfYXvx
+define <2 x i16> @select_icmp_and_eq_vec(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: @select_icmp_and_eq_vec(
+; CHECK-NEXT:    ret <2 x i16> <i16 -1, i16 -1>
+;
+  %and = and <2 x i16> %a, %b
+  %tobool = icmp eq <2 x i16> %and, <i16 -1, i16 -1>
+  %cond = select <2 x i1> %tobool, <2 x i16> %a, <2 x i16> <i16 -1, i16 -1>
+  ret <2 x i16> %cond
+}
+
+; The ne should also be macthed
+define i32 @select_icmp_and_ne(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_and_ne(
+; CHECK-NEXT:    ret i32 -1
+;
+  %and = and i32 %a, %b
+  %tobool = icmp ne i32 %and, -1
+  %cond = select i1 %tobool, i32 -1, i32 %a
+  ret i32 %cond
+}
+
+; Negative test: Incorrect const value for icmp
+define i32 @select_icmp_and_eq_incorrect_const(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_and_eq_incorrect_const(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[A]], i32 -1
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %and = and i32 %a, %b
+  %tobool = icmp eq i32 %and, 0  ; Incorrect const value
+  %cond = select i1 %tobool, i32 %a, i32 -1
+  ret i32 %cond
+}
+
+; https://alive2.llvm.org/ce/z/hSyCuR
+define i32 @select_icmp_or_eq(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_or_eq(
+; CHECK-NEXT:    ret i32 0
+;
+  %or = or i32 %a, %b
+  %tobool = icmp eq i32 %or, 0
+  %cond = select i1 %tobool, i32 %a, i32 0
+  ret i32 %cond
+}
+
+define i32 @select_icmp_or_eq_commuted(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_or_eq_commuted(
+; CHECK-NEXT:    ret i32 0
+;
+  %or = or i32 %a, %b
+  %tobool = icmp eq i32 %or, 0
+  %cond = select i1 %tobool, i32 %b, i32 0
+  ret i32 %cond
+}
+
+; https://alive2.llvm.org/ce/z/S_pQek
+define <2 x i16> @select_icmp_or_eq_vec(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: @select_icmp_or_eq_vec(
+; CHECK-NEXT:    ret <2 x i16> zeroinitializer
+;
+  %or = or <2 x i16> %a, %b
+  %tobool = icmp eq <2 x i16> %or, <i16 0, i16 0>
+  %cond = select <2 x i1> %tobool, <2 x i16> %a, <2 x i16> zeroinitializer
+  ret <2 x i16> %cond
+}
+
+; The ne will also be matched
+define i32 @select_icmp_or_ne(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_or_ne(
+; CHECK-NEXT:    ret i32 0
+;
+  %or = or i32 %a, %b
+  %tobool = icmp ne i32 %or, 0
+  %cond = select i1 %tobool, i32 0, i32 %a
+  ret i32 %cond
+}
+
+; Negative test: Incorrect const value for icmp
+define i32 @select_icmp_or_eq_incorrect_const(i32 %a, i32 %b) {
+; CHECK-LABEL: @select_icmp_or_eq_incorrect_const(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[OR]], -1
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[A]], i32 0
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %or = or i32 %a, %b
+  %tobool = icmp eq i32 %or, -1
+  %cond = select i1 %tobool, i32 %a, i32 0
+  ret i32 %cond
+}
diff --git a/llvm/test/Transforms/InstSimplify/shufflevector-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/shufflevector-inseltpoison.ll
index 3aa3cc6902cc2..49cacdc1e8d0f 100644
--- a/llvm/test/Transforms/InstSimplify/shufflevector-inseltpoison.ll
+++ b/llvm/test/Transforms/InstSimplify/shufflevector-inseltpoison.ll
@@ -79,7 +79,7 @@ define <4 x i32> @splat_operand3(<4 x i32> %x) {
 define <8 x i32> @splat_operand_negative(<4 x i32> %x) {
 ; CHECK-LABEL: @splat_operand_negative(
 ; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> poison, <8 x i32> <i32 0, i32 3, i32 2, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> poison, <8 x i32> <i32 0, i32 3, i32 2, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[SHUF]]
 ;
   %splat = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> zeroinitializer
@@ -111,8 +111,8 @@ define <4 x i32> @splat_operand_negative3(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @splat_operand_negative4(<4 x i32> %x) {
 ; CHECK-LABEL: @splat_operand_negative4(
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 2, i32 undef>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 2, i32 poison>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
 ;
   %splat = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 2, i32 undef>
@@ -122,7 +122,7 @@ define <4 x i32> @splat_operand_negative4(<4 x i32> %x) {
 
 define <4 x i32> @undef_mask(<4 x i32> %x) {
 ; CHECK-LABEL: @undef_mask(
-; CHECK-NEXT:    ret <4 x i32> undef
+; CHECK-NEXT:    ret <4 x i32> poison
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> poison
   ret <4 x i32> %shuf
@@ -130,7 +130,7 @@ define <4 x i32> @undef_mask(<4 x i32> %x) {
 
 define <4 x i32> @undef_mask_1(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @undef_mask_1(
-; CHECK-NEXT:    ret <4 x i32> undef
+; CHECK-NEXT:    ret <4 x i32> poison
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> poison
   ret <4 x i32> %shuf
@@ -173,7 +173,7 @@ define <4 x i32> @not_identity_mask(<4 x i32> %x) {
 
 define <4 x i32> @possible_identity_mask(<4 x i32> %x) {
 ; CHECK-LABEL: @possible_identity_mask(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
@@ -269,7 +269,7 @@ define <5 x i8> @splat_inserted_constant(<4 x i8> %x) {
 
 define <4 x float> @splat_inserted_constant_undef_elt(<4 x float> %x) {
 ; CHECK-LABEL: @splat_inserted_constant_undef_elt(
-; CHECK-NEXT:    ret <4 x float> <float 1.200000e+01, float 1.200000e+01, float undef, float 1.200000e+01>
+; CHECK-NEXT:    ret <4 x float> <float 1.200000e+01, float 1.200000e+01, float poison, float 1.200000e+01>
 ;
   %ins1 = insertelement <4 x float> %x, float 12.0, i32 1
   %splat1 = shufflevector <4 x float> %ins1, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 undef, i32 1>
@@ -278,7 +278,7 @@ define <4 x float> @splat_inserted_constant_undef_elt(<4 x float> %x) {
 
 define <2 x i8> @splat_inserted_constant_not_canonical(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @splat_inserted_constant_not_canonical(
-; CHECK-NEXT:    ret <2 x i8> <i8 undef, i8 23>
+; CHECK-NEXT:    ret <2 x i8> <i8 poison, i8 23>
 ;
   %ins2 = insertelement <3 x i8> %x, i8 23, i7 2
   %splat2 = shufflevector <3 x i8> %y, <3 x i8> %ins2, <2 x i32> <i32 undef, i32 5>
diff --git a/llvm/test/Transforms/InstSimplify/shufflevector.ll b/llvm/test/Transforms/InstSimplify/shufflevector.ll
index cdd7df33a94bf..460e90aa31d91 100644
--- a/llvm/test/Transforms/InstSimplify/shufflevector.ll
+++ b/llvm/test/Transforms/InstSimplify/shufflevector.ll
@@ -79,7 +79,7 @@ define <4 x i32> @splat_operand3(<4 x i32> %x) {
 define <8 x i32> @splat_operand_negative(<4 x i32> %x) {
 ; CHECK-LABEL: @splat_operand_negative(
 ; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[SHUF]]
 ;
   %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
@@ -111,8 +111,8 @@ define <4 x i32> @splat_operand_negative3(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @splat_operand_negative4(<4 x i32> %x) {
 ; CHECK-LABEL: @splat_operand_negative4(
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 2, i32 undef>
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 poison, i32 2, i32 poison>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
 ;
   %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 2, i32 undef>
@@ -122,7 +122,7 @@ define <4 x i32> @splat_operand_negative4(<4 x i32> %x) {
 
 define <4 x i32> @undef_mask(<4 x i32> %x) {
 ; CHECK-LABEL: @undef_mask(
-; CHECK-NEXT:    ret <4 x i32> undef
+; CHECK-NEXT:    ret <4 x i32> poison
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> undef
   ret <4 x i32> %shuf
@@ -130,7 +130,7 @@ define <4 x i32> @undef_mask(<4 x i32> %x) {
 
 define <4 x i32> @undef_mask_1(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @undef_mask_1(
-; CHECK-NEXT:    ret <4 x i32> undef
+; CHECK-NEXT:    ret <4 x i32> poison
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> undef
   ret <4 x i32> %shuf
@@ -173,7 +173,7 @@ define <4 x i32> @not_identity_mask(<4 x i32> %x) {
 
 define <4 x i32> @possible_identity_mask(<4 x i32> %x) {
 ; CHECK-LABEL: @possible_identity_mask(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
@@ -269,7 +269,7 @@ define <5 x i8> @splat_inserted_constant(<4 x i8> %x) {
 
 define <4 x float> @splat_inserted_constant_undef_elt(<4 x float> %x) {
 ; CHECK-LABEL: @splat_inserted_constant_undef_elt(
-; CHECK-NEXT:    ret <4 x float> <float 1.200000e+01, float 1.200000e+01, float undef, float 1.200000e+01>
+; CHECK-NEXT:    ret <4 x float> <float 1.200000e+01, float 1.200000e+01, float poison, float 1.200000e+01>
 ;
   %ins1 = insertelement <4 x float> %x, float 12.0, i32 1
   %splat1 = shufflevector <4 x float> %ins1, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 1>
@@ -278,7 +278,7 @@ define <4 x float> @splat_inserted_constant_undef_elt(<4 x float> %x) {
 
 define <2 x i8> @splat_inserted_constant_not_canonical(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @splat_inserted_constant_not_canonical(
-; CHECK-NEXT:    ret <2 x i8> <i8 undef, i8 23>
+; CHECK-NEXT:    ret <2 x i8> <i8 poison, i8 23>
 ;
   %ins2 = insertelement <3 x i8> %x, i8 23, i7 2
   %splat2 = shufflevector <3 x i8> %y, <3 x i8> %ins2, <2 x i32> <i32 undef, i32 5>
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll
index d122ca6ede3f2..77fbb8e7f2cad 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll
@@ -576,10 +576,10 @@ define void @load_factor2_wide3(ptr %ptr) {
 ; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 1
 ; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 0
 ; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; NEON-NEXT:       [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; NEON-NEXT:       ret void
 ; NO_NEON-LABEL: @load_factor2_wide3(
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
index d122ca6ede3f2..77fbb8e7f2cad 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
@@ -576,10 +576,10 @@ define void @load_factor2_wide3(ptr %ptr) {
 ; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 1
 ; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 0
 ; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; NEON-NEXT:       [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; NEON-NEXT:       ret void
 ; NO_NEON-LABEL: @load_factor2_wide3(
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
index a31bf6b4e7636..94f63e5921f8b 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
@@ -436,7 +436,7 @@ define void @store_double_factor4(ptr %ptr, <4 x double> %v0, <4 x double> %v1,
 define void @store_float_factor3(ptr %ptr, <8 x float> %v0, <8 x float> %v1, <8 x float> %v2) #0 {
 ; CHECK-LABEL: @store_float_factor3(
 ; CHECK-NEXT:    [[S0:%.*]] = shufflevector <8 x float> [[V0:%.*]], <8 x float> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x float> [[V2:%.*]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <8 x float> [[V2:%.*]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x float> [[S0]], <16 x float> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v8f32(<vscale x 4 x float> undef, <8 x float> [[TMP2]], i64 0)
diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll
index 13989465bac22..9ea1c5e94a983 100644
--- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll
+++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll
@@ -118,7 +118,7 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
 define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
 ; CHECK-NEON-LABEL: @store_factor3(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -127,14 +127,14 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
 ;
 ; CHECK-MVE-LABEL: @store_factor3(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_factor3(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
@@ -296,7 +296,7 @@ define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
 define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2) {
 ; CHECK-NEON-LABEL: @store_ptrvec_factor3(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <2 x ptr> [[V0:%.*]], <2 x ptr> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP1:%.*]] = ptrtoint <4 x ptr> [[S0]] to <4 x i32>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = ptrtoint <4 x ptr> [[S1]] to <4 x i32>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
@@ -307,14 +307,14 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
 ;
 ; CHECK-MVE-LABEL: @store_ptrvec_factor3(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <2 x ptr> [[V0:%.*]], <2 x ptr> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x ptr> [[S0]], <4 x ptr> [[S1]], <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
 ; CHECK-MVE-NEXT:    store <6 x ptr> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_ptrvec_factor3(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <2 x ptr> [[V0:%.*]], <2 x ptr> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x ptr> [[S0]], <4 x ptr> [[S1]], <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
 ; CHECK-NONE-NEXT:    store <6 x ptr> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
@@ -375,8 +375,8 @@ define void @load_undef_mask_factor2(ptr %ptr) {
 ;
 ; CHECK-NONE-LABEL: @load_undef_mask_factor2(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> poison, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
-; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> poison, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> poison, <4 x i32> <i32 poison, i32 2, i32 poison, i32 6>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 7>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <8 x i32>, ptr %ptr, align 4
@@ -397,14 +397,14 @@ define void @load_undef_mask_factor3(ptr %ptr) {
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <12 x i32>, ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_undef_mask_factor3(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <12 x i32>, ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <12 x i32>, ptr %ptr, align 4
@@ -425,18 +425,18 @@ define void @load_undef_mask_factor4(ptr %ptr) {
 ;
 ; CHECK-MVE-LABEL: @load_undef_mask_factor4(
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_undef_mask_factor4(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i32>, ptr %ptr, align 4
@@ -462,7 +462,7 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_undef_mask_factor2(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -474,7 +474,7 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
 define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
 ; CHECK-NEON-LABEL: @store_undef_mask_factor3(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -483,15 +483,15 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
 ;
 ; CHECK-MVE-LABEL: @store_undef_mask_factor3(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 poison, i32 1, i32 poison, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_undef_mask_factor3(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 poison, i32 1, i32 poison, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -516,14 +516,14 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
 ; CHECK-MVE-LABEL: @store_undef_mask_factor4(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 poison, i32 poison, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
 ; CHECK-MVE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_undef_mask_factor4(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 poison, i32 poison, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
 ; CHECK-NONE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -634,17 +634,17 @@ define void @store_f16_factor2(ptr %ptr, <4 x half> %v0, <4 x half> %v1) {
 define void @load_illegal_factor2(ptr %ptr) nounwind {
 ; CHECK-NEON-LABEL: @load_illegal_factor2(
 ; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, ptr [[PTR:%.*]], align 16
-; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @load_illegal_factor2(
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, ptr [[PTR:%.*]], align 16
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_illegal_factor2(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, ptr [[PTR:%.*]], align 16
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <3 x float>, ptr %ptr, align 16
@@ -654,17 +654,17 @@ define void @load_illegal_factor2(ptr %ptr) nounwind {
 
 define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
 ; CHECK-NEON-LABEL: @store_illegal_factor2(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NEON-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 16
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_illegal_factor2(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-MVE-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 16
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_illegal_factor2(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> poison, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NONE-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 16
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -707,12 +707,12 @@ define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefbeg(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 poison, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefbeg(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 poison, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -731,12 +731,12 @@ define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefend(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 poison>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefend(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 poison>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -755,12 +755,12 @@ define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefmid(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 32, i32 8, i32 5, i32 17, i32 poison, i32 9>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefmid(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 32, i32 8, i32 5, i32 17, i32 poison, i32 9>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -779,12 +779,12 @@ define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefmulti(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 poison, i32 9>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefmulti(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 poison, i32 9>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -825,12 +825,12 @@ define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0,
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_undefmultimid(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_undefmultimid(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -841,17 +841,17 @@ define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0,
 
 define void @store_general_mask_factor3_undef_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
 ; CHECK-NEON-LABEL: @store_general_mask_factor3_undef_fail(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
 ; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_undef_fail(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_undef_fail(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -869,12 +869,12 @@ define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_undeflane(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_undeflane(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -885,17 +885,17 @@ define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32
 
 define void @store_general_mask_factor3_endstart_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
 ; CHECK-NEON-LABEL: @store_general_mask_factor3_endstart_fail(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
 ; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_endstart_fail(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_endstart_fail(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -913,12 +913,12 @@ define void @store_general_mask_factor3_endstart_pass(ptr %ptr, <32 x i32> %v0,
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_endstart_pass(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_endstart_pass(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -929,17 +929,17 @@ define void @store_general_mask_factor3_endstart_pass(ptr %ptr, <32 x i32> %v0,
 
 define void @store_general_mask_factor3_midstart_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
 ; CHECK-NEON-LABEL: @store_general_mask_factor3_midstart_fail(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 0, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_midstart_fail(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 0, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_midstart_fail(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 0, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -957,12 +957,12 @@ define void @store_general_mask_factor3_midstart_pass(ptr %ptr, <32 x i32> %v0,
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_midstart_pass(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 1, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_midstart_pass(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 1, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -976,17 +976,17 @@ define void @store_general_mask_factor3_midstart_pass(ptr %ptr, <32 x i32> %v0,
 ; The following does not give a valid interleaved store
 define void @no_interleave(<4 x float> %a0) {
 ; CHECK-NEON-LABEL: @no_interleave(
-; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 poison>
 ; CHECK-NEON-NEXT:    store <4 x float> [[V0]], ptr @g, align 16
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @no_interleave(
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 poison>
 ; CHECK-MVE-NEXT:    store <4 x float> [[V0]], ptr @g, align 16
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @no_interleave(
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 poison>
 ; CHECK-NONE-NEXT:    store <4 x float> [[V0]], ptr @g, align 16
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -1046,10 +1046,10 @@ define void @load_factor2_wide3(ptr %ptr) {
 ; CHECK-NEON-NEXT:    [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
 ; CHECK-NEON-NEXT:    [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
 ; CHECK-NEON-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEON-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEON-NEXT:    ret void
 ;
@@ -1066,10 +1066,10 @@ define void @load_factor2_wide3(ptr %ptr) {
 ; CHECK-MVE-NEXT:    [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
 ; CHECK-MVE-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
 ; CHECK-MVE-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP11]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-MVE-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-MVE-NEXT:    ret void
 ;
@@ -1201,7 +1201,7 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
 define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
 ; CHECK-NEON-LABEL: @store_factor3_wide(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
@@ -1215,14 +1215,14 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
 ;
 ; CHECK-MVE-LABEL: @store_factor3_wide(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
 ; CHECK-MVE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_factor3_wide(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
 ; CHECK-NONE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
@@ -1348,20 +1348,20 @@ define void @load_factor2_wide_pointer(ptr %ptr) {
 define void @load_out_of_range(ptr %ptr) {
 ; CHECK-NEON-LABEL: @load_out_of_range(
 ; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @load_out_of_range(
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_out_of_range(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <4 x i32>, ptr %ptr, align 4
diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
index 23c5cf762512b..c84d759ae0fb3 100644
--- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
@@ -118,7 +118,7 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
 define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
 ; CHECK-NEON-LABEL: @store_factor3(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -127,14 +127,14 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
 ;
 ; CHECK-MVE-LABEL: @store_factor3(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_factor3(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
@@ -296,7 +296,7 @@ define void @store_ptrvec_factor2(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1) {
 define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x ptr> %v2) {
 ; CHECK-NEON-LABEL: @store_ptrvec_factor3(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <2 x ptr> [[V0:%.*]], <2 x ptr> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP1:%.*]] = ptrtoint <4 x ptr> [[S0]] to <4 x i32>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = ptrtoint <4 x ptr> [[S1]] to <4 x i32>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
@@ -307,14 +307,14 @@ define void @store_ptrvec_factor3(ptr %ptr, <2 x ptr> %v0, <2 x ptr> %v1, <2 x p
 ;
 ; CHECK-MVE-LABEL: @store_ptrvec_factor3(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <2 x ptr> [[V0:%.*]], <2 x ptr> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x ptr> [[S0]], <4 x ptr> [[S1]], <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
 ; CHECK-MVE-NEXT:    store <6 x ptr> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_ptrvec_factor3(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <2 x ptr> [[V0:%.*]], <2 x ptr> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <2 x ptr> [[V2:%.*]], <2 x ptr> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x ptr> [[S0]], <4 x ptr> [[S1]], <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
 ; CHECK-NONE-NEXT:    store <6 x ptr> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
@@ -375,8 +375,8 @@ define void @load_undef_mask_factor2(ptr %ptr) {
 ;
 ; CHECK-NONE-LABEL: @load_undef_mask_factor2(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <8 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
-; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <4 x i32> <i32 poison, i32 2, i32 poison, i32 6>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <8 x i32> [[INTERLEAVED_VEC]], <8 x i32> undef, <4 x i32> <i32 poison, i32 3, i32 poison, i32 7>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <8 x i32>, ptr %ptr, align 4
@@ -397,14 +397,14 @@ define void @load_undef_mask_factor3(ptr %ptr) {
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <12 x i32>, ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_undef_mask_factor3(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <12 x i32>, ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <12 x i32> [[INTERLEAVED_VEC]], <12 x i32> undef, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <12 x i32>, ptr %ptr, align 4
@@ -425,18 +425,18 @@ define void @load_undef_mask_factor4(ptr %ptr) {
 ;
 ; CHECK-MVE-LABEL: @load_undef_mask_factor4(
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_undef_mask_factor4(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <16 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V2:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V3:%.*]] = shufflevector <16 x i32> [[INTERLEAVED_VEC]], <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <16 x i32>, ptr %ptr, align 4
@@ -462,7 +462,7 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_undef_mask_factor2(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -474,7 +474,7 @@ define void @store_undef_mask_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
 define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
 ; CHECK-NEON-LABEL: @store_undef_mask_factor3(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -483,15 +483,15 @@ define void @store_undef_mask_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
 ;
 ; CHECK-MVE-LABEL: @store_undef_mask_factor3(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 poison, i32 1, i32 poison, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_undef_mask_factor3(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <12 x i32> <i32 0, i32 4, i32 poison, i32 1, i32 poison, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -516,14 +516,14 @@ define void @store_undef_mask_factor4(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
 ; CHECK-MVE-LABEL: @store_undef_mask_factor4(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 poison, i32 poison, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
 ; CHECK-MVE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_undef_mask_factor4(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <16 x i32> <i32 0, i32 4, i32 8, i32 poison, i32 poison, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
 ; CHECK-NONE-NEXT:    store <16 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -634,17 +634,17 @@ define void @store_f16_factor2(ptr %ptr, <4 x half> %v0, <4 x half> %v1) {
 define void @load_illegal_factor2(ptr %ptr) nounwind {
 ; CHECK-NEON-LABEL: @load_illegal_factor2(
 ; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, ptr [[PTR:%.*]], align 16
-; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @load_illegal_factor2(
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, ptr [[PTR:%.*]], align 16
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_illegal_factor2(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <3 x float>, ptr [[PTR:%.*]], align 16
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <3 x float> [[INTERLEAVED_VEC]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <3 x float>, ptr %ptr, align 16
@@ -654,17 +654,17 @@ define void @load_illegal_factor2(ptr %ptr) nounwind {
 
 define void @store_illegal_factor2(ptr %ptr, <3 x float> %v0) nounwind {
 ; CHECK-NEON-LABEL: @store_illegal_factor2(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NEON-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 16
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_illegal_factor2(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-MVE-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 16
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_illegal_factor2(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <3 x float> [[V0:%.*]], <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 poison>
 ; CHECK-NONE-NEXT:    store <3 x float> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 16
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -707,12 +707,12 @@ define void @store_general_mask_factor4_undefbeg(ptr %ptr, <32 x i32> %v0, <32 x
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefbeg(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 poison, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefbeg(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 poison, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -731,12 +731,12 @@ define void @store_general_mask_factor4_undefend(ptr %ptr, <32 x i32> %v0, <32 x
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefend(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 poison>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefend(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 poison>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -755,12 +755,12 @@ define void @store_general_mask_factor4_undefmid(ptr %ptr, <32 x i32> %v0, <32 x
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefmid(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 32, i32 8, i32 5, i32 17, i32 poison, i32 9>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefmid(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 32, i32 8, i32 5, i32 17, i32 poison, i32 9>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -779,12 +779,12 @@ define void @store_general_mask_factor4_undefmulti(ptr %ptr, <32 x i32> %v0, <32
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor4_undefmulti(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 poison, i32 9>
 ; CHECK-MVE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor4_undefmulti(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <8 x i32> <i32 4, i32 poison, i32 poison, i32 8, i32 poison, i32 poison, i32 poison, i32 9>
 ; CHECK-NONE-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -825,12 +825,12 @@ define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0,
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_undefmultimid(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_undefmultimid(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -841,17 +841,17 @@ define void @store_general_mask_factor3_undefmultimid(ptr %ptr, <32 x i32> %v0,
 
 define void @store_general_mask_factor3_undef_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
 ; CHECK-NEON-LABEL: @store_general_mask_factor3_undef_fail(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
 ; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_undef_fail(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_undef_fail(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 4, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 8, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -869,12 +869,12 @@ define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_undeflane(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_undeflane(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -885,17 +885,17 @@ define void @store_general_mask_factor3_undeflane(ptr %ptr, <32 x i32> %v0, <32
 
 define void @store_general_mask_factor3_endstart_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
 ; CHECK-NEON-LABEL: @store_general_mask_factor3_endstart_fail(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
 ; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_endstart_fail(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_endstart_fail(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 2, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -913,12 +913,12 @@ define void @store_general_mask_factor3_endstart_pass(ptr %ptr, <32 x i32> %v0,
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_endstart_pass(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_endstart_pass(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 poison, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 7, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -929,17 +929,17 @@ define void @store_general_mask_factor3_endstart_pass(ptr %ptr, <32 x i32> %v0,
 
 define void @store_general_mask_factor3_midstart_fail(ptr %ptr, <32 x i32> %v0, <32 x i32> %v1) {
 ; CHECK-NEON-LABEL: @store_general_mask_factor3_midstart_fail(
-; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 0, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NEON-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_midstart_fail(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 0, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_midstart_fail(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 0, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -957,12 +957,12 @@ define void @store_general_mask_factor3_midstart_pass(ptr %ptr, <32 x i32> %v0,
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @store_general_mask_factor3_midstart_pass(
-; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 1, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-MVE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_general_mask_factor3_midstart_pass(
-; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[V0:%.*]], <32 x i32> [[V1:%.*]], <12 x i32> <i32 poison, i32 32, i32 16, i32 1, i32 33, i32 17, i32 poison, i32 34, i32 18, i32 poison, i32 35, i32 19>
 ; CHECK-NONE-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -976,17 +976,17 @@ define void @store_general_mask_factor3_midstart_pass(ptr %ptr, <32 x i32> %v0,
 ; The following does not give a valid interleaved store
 define void @no_interleave(<4 x float> %a0) {
 ; CHECK-NEON-LABEL: @no_interleave(
-; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 poison>
 ; CHECK-NEON-NEXT:    store <4 x float> [[V0]], ptr @g, align 16
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @no_interleave(
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 poison>
 ; CHECK-MVE-NEXT:    store <4 x float> [[V0]], ptr @g, align 16
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @no_interleave(
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A0]], <4 x i32> <i32 0, i32 7, i32 1, i32 poison>
 ; CHECK-NONE-NEXT:    store <4 x float> [[V0]], ptr @g, align 16
 ; CHECK-NONE-NEXT:    ret void
 ;
@@ -1046,10 +1046,10 @@ define void @load_factor2_wide3(ptr %ptr) {
 ; CHECK-NEON-NEXT:    [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
 ; CHECK-NEON-NEXT:    [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
 ; CHECK-NEON-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEON-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEON-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEON-NEXT:    ret void
 ;
@@ -1066,10 +1066,10 @@ define void @load_factor2_wide3(ptr %ptr) {
 ; CHECK-MVE-NEXT:    [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
 ; CHECK-MVE-NEXT:    [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
 ; CHECK-MVE-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP11]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-MVE-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-MVE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-MVE-NEXT:    ret void
 ;
@@ -1201,7 +1201,7 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
 define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
 ; CHECK-NEON-LABEL: @store_factor3_wide(
 ; CHECK-NEON-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEON-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEON-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
@@ -1215,14 +1215,14 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
 ;
 ; CHECK-MVE-LABEL: @store_factor3_wide(
 ; CHECK-MVE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
 ; CHECK-MVE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @store_factor3_wide(
 ; CHECK-NONE-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
 ; CHECK-NONE-NEXT:    store <24 x i32> [[INTERLEAVED_VEC]], ptr [[PTR:%.*]], align 4
 ; CHECK-NONE-NEXT:    ret void
@@ -1348,20 +1348,20 @@ define void @load_factor2_wide_pointer(ptr %ptr) {
 define void @load_out_of_range(ptr %ptr) {
 ; CHECK-NEON-LABEL: @load_out_of_range(
 ; CHECK-NEON-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NEON-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NEON-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-NEON-NEXT:    ret void
 ;
 ; CHECK-MVE-LABEL: @load_out_of_range(
 ; CHECK-MVE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-MVE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-MVE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-MVE-NEXT:    ret void
 ;
 ; CHECK-NONE-LABEL: @load_out_of_range(
 ; CHECK-NONE-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
-; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+; CHECK-NONE-NEXT:    [[V0:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NONE-NEXT:    [[V1:%.*]] = shufflevector <4 x i32> [[INTERLEAVED_VEC]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 poison, i32 poison>
 ; CHECK-NONE-NEXT:    ret void
 ;
   %interleaved.vec = load <4 x i32>, ptr %ptr, align 4
diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
index 838467e3b1366..736a7ce4ecf0c 100644
--- a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
@@ -220,7 +220,7 @@ define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
 define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
 ; RV32-LABEL: @store_factor3(
 ; RV32-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; RV32-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; RV32-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; RV32-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; RV32-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; RV32-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -229,7 +229,7 @@ define void @store_factor3(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2
 ;
 ; RV64-LABEL: @store_factor3(
 ; RV64-NEXT:    [[S0:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; RV64-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; RV64-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[V2:%.*]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; RV64-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; RV64-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; RV64-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[S0]], <8 x i32> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -293,7 +293,7 @@ define void @store_factor2_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1) {
 define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
 ; RV32-LABEL: @store_factor3_wide(
 ; RV32-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; RV32-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; RV32-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; RV32-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; RV32-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; RV32-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
@@ -302,7 +302,7 @@ define void @store_factor3_wide(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32
 ;
 ; RV64-LABEL: @store_factor3_wide(
 ; RV64-NEXT:    [[S0:%.*]] = shufflevector <8 x i32> [[V0:%.*]], <8 x i32> [[V1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; RV64-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; RV64-NEXT:    [[S1:%.*]] = shufflevector <8 x i32> [[V2:%.*]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; RV64-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; RV64-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; RV64-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[S0]], <16 x i32> [[S1]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll
index 3e45350bc6876..b4128dda7cee7 100644
--- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll
@@ -228,7 +228,7 @@ define <4 x double> @test_unhandled(<4 x double> %b) {
 ; CHECK-LABEL: @test_unhandled(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr @a, align 32
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x double> [[SHUFFLE]]
 ;
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll
index 37b78b5a68f39..ec905a999dca1 100644
--- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll
@@ -228,7 +228,7 @@ define <4 x double> @test_unhandled(<4 x double> %b) {
 ; CHECK-LABEL: @test_unhandled(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr @a, align 32
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> undef, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 0, i32 0>
 ; CHECK-NEXT:    ret <4 x double> [[SHUFFLE]]
 ;
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll
index 7e7194b31a097..301abda5708cf 100644
--- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll
@@ -90,7 +90,7 @@ ret void
 define void @interleaved_store_vf8_i8_stride3(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf8_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP2]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
 ; CHECK-NEXT:    store <24 x i8> [[INTERLEAVED_VEC]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -105,7 +105,7 @@ ret void
 define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf16_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[C:%.*]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[C:%.*]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
@@ -121,7 +121,7 @@ define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> poison, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> poison, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> [[TMP15]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> [[TMP18]], <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
 ; CHECK-NEXT:    store <48 x i8> [[TMP19]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -136,7 +136,7 @@ ret void
 define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf32_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A:%.*]], <32 x i8> [[B:%.*]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[C:%.*]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[C:%.*]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
@@ -152,7 +152,7 @@ define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> [[TMP11]], <32 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5, i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP13]], <32 x i32> <i32 16, i32 27, i32 22, i32 17, i32 28, i32 23, i32 18, i32 29, i32 24, i32 19, i32 30, i32 25, i32 20, i32 31, i32 26, i32 21, i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> [[TMP15]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <64 x i8> [[TMP17]], <64 x i8> [[TMP18]], <96 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
 ; CHECK-NEXT:    store <96 x i8> [[TMP19]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -167,7 +167,7 @@ ret void
 define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf64_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[C:%.*]], <64 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[C:%.*]], <64 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
@@ -189,7 +189,7 @@ define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> [[TMP17]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <32 x i8> [[TMP18]], <32 x i8> [[TMP19]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <64 x i8> [[TMP20]], <64 x i8> [[TMP21]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <64 x i8> [[TMP22]], <64 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <64 x i8> [[TMP22]], <64 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <128 x i8> [[TMP23]], <128 x i8> [[TMP24]], <192 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
 ; CHECK-NEXT:    store <192 x i8> [[TMP25]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll
index 4aa8d639cbab1..75595463c7404 100644
--- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll
@@ -90,7 +90,7 @@ ret void
 define void @interleaved_store_vf8_i8_stride3(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf8_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP2]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
 ; CHECK-NEXT:    store <24 x i8> [[INTERLEAVED_VEC]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -105,7 +105,7 @@ ret void
 define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf16_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[C:%.*]], <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[C:%.*]], <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
@@ -121,7 +121,7 @@ define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> poison, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> poison, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> [[TMP15]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> [[TMP18]], <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
 ; CHECK-NEXT:    store <48 x i8> [[TMP19]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -136,7 +136,7 @@ ret void
 define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf32_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A:%.*]], <32 x i8> [[B:%.*]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[C:%.*]], <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[C:%.*]], <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
@@ -152,7 +152,7 @@ define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> [[TMP11]], <32 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5, i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP13]], <32 x i32> <i32 16, i32 27, i32 22, i32 17, i32 28, i32 23, i32 18, i32 29, i32 24, i32 19, i32 30, i32 25, i32 20, i32 31, i32 26, i32 21, i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> [[TMP15]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <64 x i8> [[TMP17]], <64 x i8> [[TMP18]], <96 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
 ; CHECK-NEXT:    store <96 x i8> [[TMP19]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
@@ -167,7 +167,7 @@ ret void
 define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, ptr %p) {
 ; CHECK-LABEL: @interleaved_store_vf64_i8_stride3(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[C:%.*]], <64 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[C:%.*]], <64 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
@@ -189,7 +189,7 @@ define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> [[TMP17]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <32 x i8> [[TMP18]], <32 x i8> [[TMP19]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <64 x i8> [[TMP20]], <64 x i8> [[TMP21]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <64 x i8> [[TMP22]], <64 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <64 x i8> [[TMP22]], <64 x i8> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <128 x i8> [[TMP23]], <128 x i8> [[TMP24]], <192 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
 ; CHECK-NEXT:    store <192 x i8> [[TMP25]], ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/LICM/explicit_guards.ll b/llvm/test/Transforms/LICM/explicit_guards.ll
index 469acf1bfffe2..84e55178f5342 100644
--- a/llvm/test/Transforms/LICM/explicit_guards.ll
+++ b/llvm/test/Transforms/LICM/explicit_guards.ll
@@ -3,6 +3,8 @@
 
 declare void @llvm.experimental.guard(i1,...)
 declare void @maythrow()
+declare void @llvm.experimental.deoptimize.isVoid(...)
+declare i1 @llvm.experimental.widenable.condition()
 
 ; Make sure that we do not hoist widenable_cond out of loop.
 ; Widenable conditions don't actually alias anything or throw, however
@@ -125,3 +127,109 @@ loop:
 exit:
   ret void
 }
+
+; TODO: Hoist widenable condition out of loop since it's the only
+; non-invariant operand of its user. If we hoist wc here, we can hoist
+; `and` as well, otherwise it prevents us from doing that.
+define void @hoist_widenable_cond_1(ptr %p, i32 %x) {
+; CHECK-LABEL: @hoist_widenable_cond_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load atomic i32, ptr [[P:%.*]] unordered, align 8
+; CHECK-NEXT:    [[ICMP7:%.*]] = icmp ult i32 0, [[LOAD]]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i32 [[IV]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP_BODY:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.body:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[ICMP7]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid(i32 13) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %load = load atomic i32, ptr %p unordered, align 8
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ %iv.next, %guarded ], [ 0, %entry ]
+  %icmp = icmp ult i32 %iv, %x
+  br i1 %icmp, label %loop.body, label %exit
+
+loop.body:
+  %icmp7 = icmp ult i32 0, %load
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %icmp7, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt
+
+deopt:
+  call void (...) @llvm.experimental.deoptimize.isVoid(i32 13) [ "deopt"() ]
+  ret void
+
+guarded:
+  %iv.next = add nuw nsw i32 %iv, 1
+  br label %loop.header
+
+exit:
+  ret void
+}
+
+; Same as hoist_widenable_cond_1 test, but widenable condition goes before
+; icmp - the second operand of wc's user.
+; TODO: Hoist widenable condition out of loop.
+define void @hoist_widenable_cond_2(ptr %p, i32 %x) {
+; CHECK-LABEL: @hoist_widenable_cond_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load atomic i32, ptr [[P:%.*]] unordered, align 8
+; CHECK-NEXT:    [[ICMP7:%.*]] = icmp ult i32 0, [[LOAD]]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ult i32 [[IV]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP_BODY:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.body:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[ICMP7]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid(i32 13) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %load = load atomic i32, ptr %p unordered, align 8
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ %iv.next, %guarded ], [ 0, %entry ]
+  %icmp = icmp ult i32 %iv, %x
+  br i1 %icmp, label %loop.body, label %exit
+
+loop.body:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %icmp7 = icmp ult i32 0, %load
+  %exiplicit_guard_cond = and i1 %icmp7, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt
+
+deopt:
+  call void (...) @llvm.experimental.deoptimize.isVoid(i32 13) [ "deopt"() ]
+  ret void
+
+guarded:
+  %iv.next = add nuw nsw i32 %iv, 1
+  br label %loop.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/hoisting.ll b/llvm/test/Transforms/LICM/hoisting.ll
index 6001c52d885e1..c5971044d5a7e 100644
--- a/llvm/test/Transforms/LICM/hoisting.ll
+++ b/llvm/test/Transforms/LICM/hoisting.ll
@@ -83,16 +83,17 @@ Out:
 }
 
 
-; This loop invariant instruction should be constant folded, not hoisted.
+; Don't bother constant folding the add, just hoist it.
 define i32 @test3(i1 %c) {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr @X, align 4
+; CHECK-NEXT:    [[B:%.*]] = add i32 4, 2
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       Loop:
-; CHECK-NEXT:    call void @foo2(i32 6)
+; CHECK-NEXT:    call void @foo2(i32 [[B]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[OUT:%.*]]
 ; CHECK:       Out:
-; CHECK-NEXT:    [[B_LCSSA:%.*]] = phi i32 [ 6, [[LOOP]] ]
+; CHECK-NEXT:    [[B_LCSSA:%.*]] = phi i32 [ [[B]], [[LOOP]] ]
 ; CHECK-NEXT:    [[C:%.*]] = sub i32 [[A]], [[B_LCSSA]]
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
diff --git a/llvm/test/Transforms/LICM/pr23608.ll b/llvm/test/Transforms/LICM/pr23608.ll
index 2477aa845ce93..4390e02ea05d8 100644
--- a/llvm/test/Transforms/LICM/pr23608.ll
+++ b/llvm/test/Transforms/LICM/pr23608.ll
@@ -25,9 +25,8 @@ define void @fn1() {
 ; NO_ASSUME-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[TMP4]], 0
 ; NO_ASSUME-NEXT:    br i1 [[TOBOOL]], label [[BB13:%.*]], label [[BB15:%.*]]
 ; NO_ASSUME:       bb13:
-; NO_ASSUME-NEXT:    [[F_IBLOCK_LCSSA:%.*]] = phi ptr [ [[TMP]], [[BB2]] ]
-; NO_ASSUME-NEXT:    [[TMP4_LE:%.*]] = ptrtoint ptr [[F_IBLOCK_LCSSA]] to i64
-; NO_ASSUME-NEXT:    [[TMP8_LE:%.*]] = inttoptr i64 [[TMP4_LE]] to ptr
+; NO_ASSUME-NEXT:    [[TMP4_LCSSA:%.*]] = phi i64 [ [[TMP4]], [[BB2]] ]
+; NO_ASSUME-NEXT:    [[TMP8_LE:%.*]] = inttoptr i64 [[TMP4_LCSSA]] to ptr
 ; NO_ASSUME-NEXT:    call void @__msan_warning_noreturn()
 ; NO_ASSUME-NEXT:    unreachable
 ; NO_ASSUME:       bb15:
@@ -54,9 +53,8 @@ define void @fn1() {
 ; USE_ASSUME-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[TMP4]], 0
 ; USE_ASSUME-NEXT:    br i1 [[TOBOOL]], label [[BB13:%.*]], label [[BB15:%.*]]
 ; USE_ASSUME:       bb13:
-; USE_ASSUME-NEXT:    [[F_IBLOCK_LCSSA:%.*]] = phi ptr [ [[TMP]], [[BB2]] ]
-; USE_ASSUME-NEXT:    [[TMP4_LE:%.*]] = ptrtoint ptr [[F_IBLOCK_LCSSA]] to i64
-; USE_ASSUME-NEXT:    [[TMP8_LE:%.*]] = inttoptr i64 [[TMP4_LE]] to ptr
+; USE_ASSUME-NEXT:    [[TMP4_LCSSA:%.*]] = phi i64 [ [[TMP4]], [[BB2]] ]
+; USE_ASSUME-NEXT:    [[TMP8_LE:%.*]] = inttoptr i64 [[TMP4_LCSSA]] to ptr
 ; USE_ASSUME-NEXT:    call void @__msan_warning_noreturn()
 ; USE_ASSUME-NEXT:    unreachable
 ; USE_ASSUME:       bb15:
diff --git a/llvm/test/Transforms/LICM/pr32129.ll b/llvm/test/Transforms/LICM/pr32129.ll
index 02163a6989251..79025556eb51f 100644
--- a/llvm/test/Transforms/LICM/pr32129.ll
+++ b/llvm/test/Transforms/LICM/pr32129.ll
@@ -1,10 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -S -passes=licm,simple-loop-unswitch,licm < %s | FileCheck %s
 
 declare void @llvm.experimental.guard(i1, ...)
 
 define void @test() {
-; CHECK-LABEL: @test(
-; CHECK-NOT: guard
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 0, 400
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP0]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header.loopexit:
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       header:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 true, label [[HEADER_LOOPEXIT:%.*]], label [[LOOP]]
+;
 entry:
   br label %header
 
diff --git a/llvm/test/Transforms/LICM/scalar-promote-opaque-ptrs.ll b/llvm/test/Transforms/LICM/scalar-promote-opaque-ptrs.ll
deleted file mode 100644
index 64bbd4fa2800b..0000000000000
--- a/llvm/test/Transforms/LICM/scalar-promote-opaque-ptrs.ll
+++ /dev/null
@@ -1,605 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=licm -opaque-pointers -S %s | FileCheck %s
-target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
-
-@X = global i32 7   ; <ptr> [#uses=4]
-
-define void @test1(i32 %i) {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:  Entry:
-; CHECK-NEXT:    [[X_PROMOTED:%.*]] = load i32, ptr @X, align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[X_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[X2]] = add i32 [[X21]], 1
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[OUT:%.*]], label [[LOOP]]
-; CHECK:       Out:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[X2_LCSSA]], ptr @X, align 4
-; CHECK-NEXT:    ret void
-;
-Entry:
-  br label %Loop
-
-Loop:   ; preds = %Loop, %0
-  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]    ; <i32> [#uses=1]
-  %x = load i32, ptr @X   ; <i32> [#uses=1]
-  %x2 = add i32 %x, 1   ; <i32> [#uses=1]
-  store i32 %x2, ptr @X
-  %Next = add i32 %j, 1   ; <i32> [#uses=2]
-  %cond = icmp eq i32 %Next, 0    ; <i1> [#uses=1]
-  br i1 %cond, label %Out, label %Loop
-
-Out:
-  ret void
-}
-
-define void @test2(i32 %i) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:  Entry:
-; CHECK-NEXT:    [[DOTPROMOTED:%.*]] = load i32, ptr getelementptr inbounds (i32, ptr @X, i64 1), align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[V1:%.*]] = phi i32 [ [[V:%.*]], [[LOOP]] ], [ [[DOTPROMOTED]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[V]] = add i32 [[V1]], 1
-; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       Exit:
-; CHECK-NEXT:    [[V_LCSSA:%.*]] = phi i32 [ [[V]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[V_LCSSA]], ptr getelementptr inbounds (i32, ptr @X, i64 1), align 4
-; CHECK-NEXT:    ret void
-;
-Entry:
-  br label %Loop
-
-Loop:   ; preds = %Loop, %0
-  %X1 = getelementptr i32, ptr @X, i64 1    ; <ptr> [#uses=1]
-  %A = load i32, ptr %X1    ; <i32> [#uses=1]
-  %V = add i32 %A, 1    ; <i32> [#uses=1]
-  %X2 = getelementptr i32, ptr @X, i64 1    ; <ptr> [#uses=1]
-  store i32 %V, ptr %X2
-  br i1 false, label %Loop, label %Exit
-
-Exit:   ; preds = %Loop
-  ret void
-}
-
-define void @test3(i32 %i) {
-; CHECK-LABEL: @test3(
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[X:%.*]] = load volatile i32, ptr @X, align 4
-; CHECK-NEXT:    [[X2:%.*]] = add i32 [[X]], 1
-; CHECK-NEXT:    store i32 [[X2]], ptr @X, align 4
-; CHECK-NEXT:    br i1 true, label [[OUT:%.*]], label [[LOOP]]
-; CHECK:       Out:
-; CHECK-NEXT:    ret void
-;
-  br label %Loop
-Loop:
-  ; Should not promote this to a register
-  %x = load volatile i32, ptr @X
-  %x2 = add i32 %x, 1
-  store i32 %x2, ptr @X
-  br i1 true, label %Out, label %Loop
-
-Out:    ; preds = %Loop
-  ret void
-}
-
-; Should not promote this to a register
-define void @test3b(i32 %i) {
-; CHECK-LABEL: @test3b(
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr @X, align 4
-; CHECK-NEXT:    [[X2:%.*]] = add i32 [[X]], 1
-; CHECK-NEXT:    store volatile i32 [[X2]], ptr @X, align 4
-; CHECK-NEXT:    br i1 true, label [[OUT:%.*]], label [[LOOP]]
-; CHECK:       Out:
-; CHECK-NEXT:    ret void
-;
-  br label %Loop
-Loop:
-  %x = load i32, ptr @X
-  %x2 = add i32 %x, 1
-  store volatile i32 %x2, ptr @X
-  br i1 true, label %Out, label %Loop
-
-Out:    ; preds = %Loop
-  ret void
-}
-
-; PR8041
-; Should have promoted 'handle2' accesses.
-; Should not have promoted offsetx1 loads.
-define void @test4(ptr %x, i8 %n) {
-; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[HANDLE1:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    [[HANDLE2:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    store ptr [[X:%.*]], ptr [[HANDLE1]], align 8
-; CHECK-NEXT:    [[TMP:%.*]] = getelementptr i8, ptr [[X]], i64 8
-; CHECK-NEXT:    [[OFFSETX1:%.*]] = load ptr, ptr [[HANDLE1]], align 8
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    br label [[SUBLOOP:%.*]]
-; CHECK:       subloop:
-; CHECK-NEXT:    [[NEWOFFSETX21:%.*]] = phi ptr [ [[TMP]], [[LOOP]] ], [ [[NEWOFFSETX2:%.*]], [[SUBLOOP]] ]
-; CHECK-NEXT:    [[COUNT:%.*]] = phi i8 [ 0, [[LOOP]] ], [ [[NEXTCOUNT:%.*]], [[SUBLOOP]] ]
-; CHECK-NEXT:    store i8 [[N:%.*]], ptr [[NEWOFFSETX21]], align 1
-; CHECK-NEXT:    [[NEWOFFSETX2]] = getelementptr i8, ptr [[NEWOFFSETX21]], i64 -1
-; CHECK-NEXT:    [[NEXTCOUNT]] = add i8 [[COUNT]], 1
-; CHECK-NEXT:    [[INNEREXITCOND:%.*]] = icmp sge i8 [[NEXTCOUNT]], 8
-; CHECK-NEXT:    br i1 [[INNEREXITCOND]], label [[INNEREXIT:%.*]], label [[SUBLOOP]]
-; CHECK:       innerexit:
-; CHECK-NEXT:    [[NEWOFFSETX2_LCSSA:%.*]] = phi ptr [ [[NEWOFFSETX2]], [[SUBLOOP]] ]
-; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[OFFSETX1]], align 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[VAL]], [[N]]
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[NEWOFFSETX2_LCSSA_LCSSA:%.*]] = phi ptr [ [[NEWOFFSETX2_LCSSA]], [[INNEREXIT]] ]
-; CHECK-NEXT:    store ptr [[NEWOFFSETX2_LCSSA_LCSSA]], ptr [[HANDLE2]], align 8
-; CHECK-NEXT:    ret void
-;
-  %handle1 = alloca ptr
-  %handle2 = alloca ptr
-  store ptr %x, ptr %handle1
-  br label %loop
-
-loop:
-  %tmp = getelementptr i8, ptr %x, i64 8
-  store ptr %tmp, ptr %handle2
-  br label %subloop
-
-subloop:
-  %count = phi i8 [ 0, %loop ], [ %nextcount, %subloop ]
-  %offsetx2 = load ptr, ptr %handle2
-  store i8 %n, ptr %offsetx2
-  %newoffsetx2 = getelementptr i8, ptr %offsetx2, i64 -1
-  store ptr %newoffsetx2, ptr %handle2
-  %nextcount = add i8 %count, 1
-  %innerexitcond = icmp sge i8 %nextcount, 8
-  br i1 %innerexitcond, label %innerexit, label %subloop
-
-innerexit:
-  %offsetx1 = load ptr, ptr %handle1
-  %val = load i8, ptr %offsetx1
-  %cond = icmp eq i8 %val, %n
-  br i1 %cond, label %exit, label %loop
-
-exit:
-  ret void
-}
-
-define void @test5(i32 %i, ptr noalias %P2) {
-; CHECK-LABEL: @test5(
-; CHECK-NEXT:  Entry:
-; CHECK-NEXT:    [[X_PROMOTED:%.*]] = load i32, ptr @X, align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[X_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[X2]] = add i32 [[X21]], 1
-; CHECK-NEXT:    store atomic ptr @X, ptr [[P2:%.*]] monotonic, align 8
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[OUT:%.*]], label [[LOOP]]
-; CHECK:       Out:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[X2_LCSSA]], ptr @X, align 4
-; CHECK-NEXT:    ret void
-;
-Entry:
-  br label %Loop
-
-Loop:   ; preds = %Loop, %0
-  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]    ; <i32> [#uses=1]
-  %x = load i32, ptr @X   ; <i32> [#uses=1]
-  %x2 = add i32 %x, 1   ; <i32> [#uses=1]
-  store i32 %x2, ptr @X
-
-  store atomic ptr @X, ptr %P2 monotonic, align 8
-
-  %Next = add i32 %j, 1   ; <i32> [#uses=2]
-  %cond = icmp eq i32 %Next, 0    ; <i1> [#uses=1]
-  br i1 %cond, label %Out, label %Loop
-
-Out:
-  ret void
-
-}
-
-
-; PR14753 - Preserve TBAA tags when promoting values in a loop.
-define void @test6(i32 %n, ptr nocapture %a, ptr %gi) {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    store i32 0, ptr [[GI:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, [[N:%.*]]
-; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
-; CHECK:       for.body.lr.ph:
-; CHECK-NEXT:    [[GI_PROMOTED:%.*]] = load i32, ptr [[GI]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[INC1:%.*]] = phi i32 [ [[GI_PROMOTED]], [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[STOREMERGE2:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STOREMERGE2]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IDXPROM]]
-; CHECK-NEXT:    store float 0.000000e+00, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA4:![0-9]+]]
-; CHECK-NEXT:    [[INC]] = add nsw i32 [[INC1]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
-; CHECK:       for.cond.for.end_crit_edge:
-; CHECK-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[FOR_BODY]] ]
-; CHECK-NEXT:    store i32 [[INC_LCSSA]], ptr [[GI]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT:    br label [[FOR_END]]
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
-;
-entry:
-  store i32 0, ptr %gi, align 4, !tbaa !0
-  %cmp1 = icmp slt i32 0, %n
-  br i1 %cmp1, label %for.body.lr.ph, label %for.end
-
-for.body.lr.ph:                                   ; preds = %entry
-  br label %for.body
-
-for.body:                                         ; preds = %for.body.lr.ph, %for.body
-  %storemerge2 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
-  %idxprom = sext i32 %storemerge2 to i64
-  %arrayidx = getelementptr inbounds float, ptr %a, i64 %idxprom
-  store float 0.000000e+00, ptr %arrayidx, align 4, !tbaa !3
-  %0 = load i32, ptr %gi, align 4, !tbaa !0
-  %inc = add nsw i32 %0, 1
-  store i32 %inc, ptr %gi, align 4, !tbaa !0
-  %cmp = icmp slt i32 %inc, %n
-  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
-
-for.cond.for.end_crit_edge:                       ; preds = %for.body
-  br label %for.end
-
-for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
-  ret void
-
-}
-
-declare i32 @opaque(i32) argmemonly
-declare void @capture(ptr)
-
-; We can promote even if opaque may throw.
-define i32 @test7() {
-; CHECK-LABEL: @test7(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LOCAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @capture(ptr [[LOCAL]])
-; CHECK-NEXT:    [[LOCAL_PROMOTED:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[LOCAL_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[X2]] = call i32 @opaque(i32 [[X21]])
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[X2_LCSSA]], ptr [[LOCAL]], align 4
-; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-entry:
-  %local = alloca i32
-  call void @capture(ptr %local)
-  br label %loop
-
-loop:
-  %j = phi i32 [ 0, %entry ], [ %next, %loop ]
-  %x = load i32, ptr %local
-  %x2 = call i32 @opaque(i32 %x) ; Note this does not capture %local
-  store i32 %x2, ptr %local
-  %next = add i32 %j, 1
-  %cond = icmp eq i32 %next, 0
-  br i1 %cond, label %exit, label %loop
-
-exit:
-  %ret = load i32, ptr %local
-  ret i32 %ret
-}
-
-; Make sure we don't promote if the store is really control-flow dependent.
-define i32 @test7bad() {
-; CHECK-LABEL: @test7bad(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LOCAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @capture(ptr [[LOCAL]])
-; CHECK-NEXT:    [[LOCAL_PROMOTED:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[X22:%.*]] = phi i32 [ [[LOCAL_PROMOTED]], [[ENTRY:%.*]] ], [ [[X21:%.*]], [[ELSE:%.*]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[ELSE]] ]
-; CHECK-NEXT:    [[X2:%.*]] = call i32 @opaque(i32 [[X22]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X2]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[ELSE]]
-; CHECK:       if:
-; CHECK-NEXT:    store i32 [[X2]], ptr [[LOCAL]], align 4
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[X21]] = phi i32 [ [[X2]], [[IF]] ], [ [[X22]], [[LOOP]] ]
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-entry:
-  %local = alloca i32
-  call void @capture(ptr %local)
-  br label %loop
-loop:
-  %j = phi i32 [ 0, %entry ], [ %next, %else ]
-  %x = load i32, ptr %local
-  %x2 = call i32 @opaque(i32 %x) ; Note this does not capture %local
-  %cmp = icmp eq i32 %x2, 0
-  br i1 %cmp, label %if, label %else
-
-if:
-  store i32 %x2, ptr %local
-  br label %else
-
-else:
-  %next = add i32 %j, 1
-  %cond = icmp eq i32 %next, 0
-  br i1 %cond, label %exit, label %loop
-
-exit:
-  %ret = load i32, ptr %local
-  ret i32 %ret
-}
-
-; Even if neither the load nor the store or guaranteed to execute because
-; opaque() may throw, we can still promote - the load not being guaranteed
-; doesn't block us, because %local is always dereferenceable.
-define i32 @test8() {
-; CHECK-LABEL: @test8(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LOCAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @capture(ptr [[LOCAL]])
-; CHECK-NEXT:    [[LOCAL_PROMOTED:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[LOCAL_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[THROWAWAY:%.*]] = call i32 @opaque(i32 [[J]])
-; CHECK-NEXT:    [[X2]] = call i32 @opaque(i32 [[X21]])
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[X2_LCSSA]], ptr [[LOCAL]], align 4
-; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-entry:
-  %local = alloca i32
-  call void @capture(ptr %local)
-  br label %loop
-
-loop:
-  %j = phi i32 [ 0, %entry ], [ %next, %loop ]
-  %throwaway = call i32 @opaque(i32 %j)
-  %x = load i32, ptr %local
-  %x2 = call i32 @opaque(i32 %x)
-  store i32 %x2, ptr %local
-  %next = add i32 %j, 1
-  %cond = icmp eq i32 %next, 0
-  br i1 %cond, label %exit, label %loop
-
-exit:
-  %ret = load i32, ptr %local
-  ret i32 %ret
-}
-
-
-; If the store is "guaranteed modulo exceptions", and the load depends on
-; control flow, we can only promote if the pointer is otherwise known to be
-; dereferenceable
-define i32 @test9() {
-; CHECK-LABEL: @test9(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LOCAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @capture(ptr [[LOCAL]])
-; CHECK-NEXT:    [[LOCAL_PROMOTED:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[LOCAL_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[ELSE:%.*]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[ELSE]] ]
-; CHECK-NEXT:    [[J2:%.*]] = call i32 @opaque(i32 [[J]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[J2]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[ELSE]]
-; CHECK:       if:
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[X2]] = phi i32 [ 0, [[LOOP]] ], [ [[X21]], [[IF]] ]
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[ELSE]] ]
-; CHECK-NEXT:    store i32 [[X2_LCSSA]], ptr [[LOCAL]], align 4
-; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[LOCAL]], align 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-entry:
-  %local = alloca i32
-  call void @capture(ptr %local)
-  br label %loop
-
-loop:
-  %j = phi i32 [ 0, %entry ], [ %next, %else ]
-  %j2 = call i32 @opaque(i32 %j)
-  %cmp = icmp eq i32 %j2, 0
-  br i1 %cmp, label %if, label %else
-
-if:
-  %x = load i32, ptr %local
-  br label %else
-
-else:
-  %x2 = phi i32 [ 0, %loop ], [ %x, %if]
-  store i32 %x2, ptr %local
-  %next = add i32 %j, 1
-  %cond = icmp eq i32 %next, 0
-  br i1 %cond, label %exit, label %loop
-
-exit:
-  %ret = load i32, ptr %local
-  ret i32 %ret
-}
-
-define i32 @test9bad(i32 %i) {
-; CHECK-LABEL: @test9bad(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LOCAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @capture(ptr [[LOCAL]])
-; CHECK-NEXT:    [[NOTDEREF:%.*]] = getelementptr i32, ptr [[LOCAL]], i32 [[I:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[NEXT:%.*]], [[ELSE:%.*]] ]
-; CHECK-NEXT:    [[J2:%.*]] = call i32 @opaque(i32 [[J]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[J2]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[ELSE]]
-; CHECK:       if:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[NOTDEREF]], align 4
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[X2:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[X]], [[IF]] ]
-; CHECK-NEXT:    store i32 [[X2]], ptr [[NOTDEREF]], align 4
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[NOTDEREF]], align 4
-; CHECK-NEXT:    ret i32 [[RET]]
-;
-entry:
-  %local = alloca i32
-  call void @capture(ptr %local)
-  %notderef = getelementptr i32, ptr %local, i32 %i
-  br label %loop
-
-loop:
-  %j = phi i32 [ 0, %entry ], [ %next, %else ]
-  %j2 = call i32 @opaque(i32 %j)
-  %cmp = icmp eq i32 %j2, 0
-  br i1 %cmp, label %if, label %else
-
-if:
-  %x = load i32, ptr %notderef
-  br label %else
-
-else:
-  %x2 = phi i32 [ 0, %loop ], [ %x, %if]
-  store i32 %x2, ptr %notderef
-  %next = add i32 %j, 1
-  %cond = icmp eq i32 %next, 0
-  br i1 %cond, label %exit, label %loop
-
-exit:
-  %ret = load i32, ptr %notderef
-  ret i32 %ret
-}
-
-define void @test10(i32 %i) {
-; CHECK-LABEL: @test10(
-; CHECK-NEXT:  Entry:
-; CHECK-NEXT:    [[X_PROMOTED:%.*]] = load atomic i32, ptr @X unordered, align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[X_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[X2]] = add i32 [[X21]], 1
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[OUT:%.*]], label [[LOOP]]
-; CHECK:       Out:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[LOOP]] ]
-; CHECK-NEXT:    store atomic i32 [[X2_LCSSA]], ptr @X unordered, align 4
-; CHECK-NEXT:    ret void
-;
-Entry:
-  br label %Loop
-
-
-Loop:   ; preds = %Loop, %0
-  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]    ; <i32> [#uses=1]
-  %x = load atomic i32, ptr @X unordered, align 4
-  %x2 = add i32 %x, 1
-  store atomic i32 %x2, ptr @X unordered, align 4
-  %Next = add i32 %j, 1
-  %cond = icmp eq i32 %Next, 0
-  br i1 %cond, label %Out, label %Loop
-
-Out:
-  ret void
-
-}
-
-; Early exit is known not to be taken on first iteration and thus doesn't
-; effect whether load is known to execute.
-define void @test11(i32 %i) {
-; CHECK-LABEL: @test11(
-; CHECK-NEXT:  Entry:
-; CHECK-NEXT:    [[X_PROMOTED:%.*]] = load i32, ptr @X, align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       Loop:
-; CHECK-NEXT:    [[X21:%.*]] = phi i32 [ [[X_PROMOTED]], [[ENTRY:%.*]] ], [ [[X2:%.*]], [[BODY:%.*]] ]
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[NEXT:%.*]], [[BODY]] ]
-; CHECK-NEXT:    [[EARLY_TEST:%.*]] = icmp ult i32 [[J]], 32
-; CHECK-NEXT:    br i1 [[EARLY_TEST]], label [[BODY]], label [[EARLY:%.*]]
-; CHECK:       body:
-; CHECK-NEXT:    [[X2]] = add i32 [[X21]], 1
-; CHECK-NEXT:    [[NEXT]] = add i32 [[J]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[NEXT]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[OUT:%.*]], label [[LOOP]]
-; CHECK:       Early:
-; CHECK-NEXT:    [[X21_LCSSA:%.*]] = phi i32 [ [[X21]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[X21_LCSSA]], ptr @X, align 4
-; CHECK-NEXT:    ret void
-; CHECK:       Out:
-; CHECK-NEXT:    [[X2_LCSSA:%.*]] = phi i32 [ [[X2]], [[BODY]] ]
-; CHECK-NEXT:    store i32 [[X2_LCSSA]], ptr @X, align 4
-; CHECK-NEXT:    ret void
-;
-Entry:
-  br label %Loop
-
-
-Loop:   ; preds = %Loop, %0
-  %j = phi i32 [ 0, %Entry ], [ %Next, %body ]    ; <i32> [#uses=1]
-  %early.test = icmp ult i32 %j, 32
-  br i1 %early.test, label %body, label %Early
-body:
-  %x = load i32, ptr @X   ; <i32> [#uses=1]
-  %x2 = add i32 %x, 1   ; <i32> [#uses=1]
-  store i32 %x2, ptr @X
-  %Next = add i32 %j, 1   ; <i32> [#uses=2]
-  %cond = icmp eq i32 %Next, 0    ; <i1> [#uses=1]
-  br i1 %cond, label %Out, label %Loop
-
-Early:
-  ret void
-Out:
-  ret void
-
-}
-
-!0 = !{!4, !4, i64 0}
-!1 = !{!"omnipotent char", !2}
-!2 = !{!"Simple C/C++ TBAA"}
-!3 = !{!5, !5, i64 0}
-!4 = !{!"int", !1}
-!5 = !{!"float", !1}
diff --git a/llvm/test/Transforms/LICM/scalar-promote.ll b/llvm/test/Transforms/LICM/scalar-promote.ll
index 1597a911a413b..edcadf53fa477 100644
--- a/llvm/test/Transforms/LICM/scalar-promote.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote.ll
@@ -41,15 +41,17 @@ Out:
 define void @test2(i32 %i) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  Entry:
-; CHECK-NEXT:    [[DOTPROMOTED:%.*]] = load i32, ptr getelementptr inbounds (i32, ptr @X, i64 1), align 4
+; CHECK-NEXT:    [[X1:%.*]] = getelementptr i32, ptr @X, i64 1
+; CHECK-NEXT:    [[X2:%.*]] = getelementptr i32, ptr @X, i64 1
+; CHECK-NEXT:    [[X1_PROMOTED:%.*]] = load i32, ptr [[X1]], align 4
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       Loop:
-; CHECK-NEXT:    [[V1:%.*]] = phi i32 [ [[V:%.*]], [[LOOP]] ], [ [[DOTPROMOTED]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[V]] = add i32 [[V1]], 1
+; CHECK-NEXT:    [[A1:%.*]] = phi i32 [ [[V:%.*]], [[LOOP]] ], [ [[X1_PROMOTED]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[V]] = add i32 [[A1]], 1
 ; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       Exit:
 ; CHECK-NEXT:    [[V_LCSSA:%.*]] = phi i32 [ [[V]], [[LOOP]] ]
-; CHECK-NEXT:    store i32 [[V_LCSSA]], ptr getelementptr inbounds (i32, ptr @X, i64 1), align 4
+; CHECK-NEXT:    store i32 [[V_LCSSA]], ptr [[X1]], align 4
 ; CHECK-NEXT:    ret void
 ;
 Entry:
diff --git a/llvm/test/Transforms/LICM/sink-foldable.ll b/llvm/test/Transforms/LICM/sink-foldable.ll
index 5cc15e4f6dc7a..bf2cc77e6f47b 100644
--- a/llvm/test/Transforms/LICM/sink-foldable.ll
+++ b/llvm/test/Transforms/LICM/sink-foldable.ll
@@ -188,11 +188,11 @@ define ptr @test3(i64 %j, ptr readonly %P, ptr readnone %Q) {
 ; CHECK-NEXT:    [[P1:%.*]] = phi ptr [ [[ARRAYIDX0]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       loopexit1:
-; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i64 [ [[ADD]], [[IF_END]] ]
+; CHECK-NEXT:    [[TRUNC_LCSSA1:%.*]] = phi i32 [ [[TRUNC]], [[IF_END]] ]
 ; CHECK-NEXT:    [[P_ADDR_LCSSA:%.*]] = phi ptr [ [[P_ADDR]], [[IF_END]] ]
-; CHECK-NEXT:    [[TRUNC_LE:%.*]] = trunc i64 [[ADD_LCSSA]] to i32
-; CHECK-NEXT:    [[ARRAYIDX1_LE:%.*]] = getelementptr inbounds ptr, ptr [[P_ADDR_LCSSA]], i32 [[TRUNC_LE]]
-; CHECK-NEXT:    call void @dummy(i32 [[TRUNC_LE]])
+; CHECK-NEXT:    [[TRUNC_LCSSA:%.*]] = phi i32 [ [[TRUNC]], [[IF_END]] ]
+; CHECK-NEXT:    [[ARRAYIDX1_LE:%.*]] = getelementptr inbounds ptr, ptr [[P_ADDR_LCSSA]], i32 [[TRUNC_LCSSA1]]
+; CHECK-NEXT:    call void @dummy(i32 [[TRUNC_LCSSA]])
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi ptr [ [[P1]], [[LOOPEXIT0]] ], [ [[ARRAYIDX1_LE]], [[LOOPEXIT1]] ], [ null, [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/LICM/sinking.ll b/llvm/test/Transforms/LICM/sinking.ll
index 153601ac642d6..5888f2dcf65b0 100644
--- a/llvm/test/Transforms/LICM/sinking.ll
+++ b/llvm/test/Transforms/LICM/sinking.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=licm -S -verify-memoryssa | FileCheck %s
 
+target datalayout = "n64"
 
 declare i32 @strlen(ptr) readonly nounwind willreturn
 
@@ -997,6 +998,59 @@ out:
   ret i32 %x
 }
 
+declare void @use.i32(i32)
+declare void @use.i64(i64)
+
+; Don't duplicate freeze just because it's free.
+define i32 @duplicate_freeze(i1 %c, i32 %x) {
+; CHECK-LABEL: @duplicate_freeze(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @use.i32(i32 [[FR]])
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[FR_LCSSA:%.*]] = phi i32 [ [[FR]], [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[FR_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %fr = freeze i32 %x
+  call void @use.i32(i32 %fr)
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret i32 %fr
+}
+
+; Don't duplicate ptrtoint just because it's free.
+define i64 @duplicate_ptrtoint(i1 %c, ptr %p) {
+; CHECK-LABEL: @duplicate_ptrtoint(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PI:%.*]] = ptrtoint ptr [[P:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @use.i64(i64 [[PI]])
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[PI_LCSSA:%.*]] = phi i64 [ [[PI]], [[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[PI_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %pi = ptrtoint ptr %p to i64
+  call void @use.i64(i64 %pi)
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret i64 %pi
+}
+
 declare void @may_throw()
 declare void @may_throw2()
 declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/LICM/speculate-div.ll b/llvm/test/Transforms/LICM/speculate-div.ll
new file mode 100644
index 0000000000000..fde63e43d0164
--- /dev/null
+++ b/llvm/test/Transforms/LICM/speculate-div.ll
@@ -0,0 +1,202 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=licm -S | FileCheck %s
+
+declare void @maythrow()
+declare void @use(i16)
+
+define void @sdiv_not_ok(i16 %n, i16 noundef %xx) {
+; CHECK-LABEL: @sdiv_not_ok(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 [[N:%.*]], [[X]]
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %x = or i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = sdiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @srem_not_ok2(i16 %nn, i16 noundef %x) {
+; CHECK-LABEL: @srem_not_ok2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N:%.*]] = and i16 [[NN:%.*]], 1323
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[DIV:%.*]] = srem i16 [[N]], [[X:%.*]]
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %n = and i16 %nn, 1323
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = srem i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @sdiv_ok(i16 %n, i16 noundef %xx) {
+; CHECK-LABEL: @sdiv_ok(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[XO:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    [[X:%.*]] = and i16 [[XO]], 123
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 [[N:%.*]], [[X]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %xo = or i16 %xx, 1
+  %x = and i16 %xo, 123
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = sdiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @srem_ok2(i16 noundef %nn, i16 noundef %xx) {
+; CHECK-LABEL: @srem_ok2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N:%.*]] = and i16 [[NN:%.*]], 123
+; CHECK-NEXT:    [[X:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    [[DIV:%.*]] = srem i16 [[N]], [[X]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %n = and i16 %nn, 123
+  %x = or i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = srem i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @sdiv_not_ok3_maybe_poison_denum(i16 noundef %nn, i16 %xx) {
+; CHECK-LABEL: @sdiv_not_ok3_maybe_poison_denum(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N:%.*]] = and i16 [[NN:%.*]], 123
+; CHECK-NEXT:    [[X:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 [[N]], [[X]]
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %n = and i16 %nn, 123
+  %x = or i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = sdiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @sdiv_not_ok3_maybe_poison_num(i16 %nn, i16 noundef  %xx) {
+; CHECK-LABEL: @sdiv_not_ok3_maybe_poison_num(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N:%.*]] = and i16 [[NN:%.*]], 123
+; CHECK-NEXT:    [[X:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 [[N]], [[X]]
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %n = and i16 %nn, 123
+  %x = or i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = sdiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @udiv_not_ok(i16 %n, i16 noundef %xx) {
+; CHECK-LABEL: @udiv_not_ok(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = xor i16 [[XX:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i16 [[N:%.*]], [[X]]
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %x = xor i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = udiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @udiv_ok(i16 %n, i16 noundef %xx) {
+; CHECK-LABEL: @udiv_ok(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i16 [[N:%.*]], [[X]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %x = or i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = udiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
+
+define void @urem_not_ok_maybe_poison(i16 %n, i16 %xx) {
+; CHECK-LABEL: @urem_not_ok_maybe_poison(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = or i16 [[XX:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i16 [[N:%.*]], [[X]]
+; CHECK-NEXT:    call void @use(i16 [[DIV]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  %x = or i16 %xx, 1
+  br label %loop
+loop:
+  call void @maythrow()
+  %div = udiv i16 %n, %x
+  call void @use(i16 %div)
+  br label %loop
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
index 3c2eea39689df..27dd7b42f54b3 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa,scoped-noalias-aa -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefix=SCOPE %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefix=NOSCOPE %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; This fails to vectorize if the !alias.scope is not used
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
index 26f9bcfdf550d..17fc7f6c1c951 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
@@ -5,7 +5,7 @@
 ; RUN: opt -S -passes='function(load-store-vectorizer)' --mcpu=hawaii -mattr=+unaligned-access-mode,+unaligned-scratch-access,+max-private-element-size-16 < %s | FileCheck -check-prefixes=CHECK,UNALIGNED %s
 
 target triple = "amdgcn--"
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define amdgpu_kernel void @load_unknown_offset_align1_i8(ptr addrspace(1) noalias %out, i32 %offset) #0 {
 ; ALIGNED-LABEL: @load_unknown_offset_align1_i8(
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
index 36f6b09a114d0..067bbdf43922c 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 declare i64 @_Z12get_local_idj(i32)
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
index 3a065b3db9f42..6df55ffba8633 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
index 06eae3bf38398..d6eac209ac682 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -S -mtriple=amdgcn--amdhsa -passes=load-store-vectorizer < %s | FileCheck %s
 ; RUN: opt -S -mtriple=amdgcn--amdhsa -passes='function(load-store-vectorizer)' < %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Check that vectorizer can find a GEP through bitcast
 ; CHECK-LABEL: @vect_zext_bitcast_f32_to_i32_idx
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
index 418b03e653042..8629869fc2be6 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Check position of the inserted vector load/store.  Vectorized loads should be
 ; inserted at the position of the first load in the chain, and stores should be
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
index f6f98d6fbafad..09b65121894d5 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; This is NOT OK to vectorize, as either load may alias either store.
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
index 9b0eb5bcea66f..a6c3e0db098a7 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; CHECK-LABEL: @interleave
 ; CHECK: load <2 x double>, ptr addrspace(1) %{{.}}, align 8{{$}}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
index 4c631e178cbc6..cd4717f5152e0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
@@ -5,7 +5,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-UNALIGNED,UNALIGNED,ALL %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-UNALIGNED,UNALIGNED,ALL %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32
 ; ELT4-ALIGNED: store i32
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
index 3e93555f29d0a..55f311d9a2fca 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa --mcpu=hawaii -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 ; Copy of test/CodeGen/AMDGPU/merge-stores.ll with some additions
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; TODO: Vector element tests
 ; TODO: Non-zero base offset for load and store combinations
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
index 7bfd6d59b2705..e9cd3547e5353 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -mtriple=amdgcn-- -mcpu=tonga -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 @lds = internal addrspace(3) global [512 x float] undef, align 4
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
index 7dadbee6b74b7..dc7ae189626c8 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
@@ -1,7 +1,7 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=GCN,GFX7 %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Checks that there is no crash when there are multiple tails
 ; for a the same head starting a chain.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
index 7161d7c03b6e5..290607e90d8ec 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; CHECK-LABEL: @no_implicit_float(
 ; CHECK: store i32
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
index a132cc49a95fc..458606eadae79 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; CHECK-LABEL: @optnone(
 ; CHECK: store i32
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
index e0985c2f57ec8..8e5a20ec8ae1d 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
index 78eddcb335632..e848af9beabde 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer,dce -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define void @base_case(i1 %cnd, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @base_case
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
index 7fa7ae27730d7..d49a13ca259e4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer,dce -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define void @base_case(i1 %cnd, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @base_case
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
index 8d6b286077647..c62ae57e45806 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Check that, in the presence of an aliasing load, the stores preceding the
 ; aliasing load are safe to vectorize.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
index 8ab1a22259b75..5ad427db3d863 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Checks that we don't merge loads/stores of types smaller than one
 ; byte, or vectors with elements smaller than one byte.
diff --git a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
index 3a57dfd5148bd..d1cff14c3f4b7 100644
--- a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
+++ b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
@@ -1,11 +1,11 @@
-; RUN: opt -opaque-pointers=0 -aa-pipeline=basic-aa -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN: opt -aa-pipeline=basic-aa -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
 ; RUN:   < %s | FileCheck %s
 
-; RUN: opt -opaque-pointers=0 -aa-pipeline=basic-aa -passes='loop-distribute,loop-vectorize' -enable-loop-distribute -force-vector-width=4 \
+; RUN: opt -aa-pipeline=basic-aa -passes='loop-distribute,loop-vectorize' -enable-loop-distribute -force-vector-width=4 \
 ; RUN:   -verify-loop-info -verify-dom-info -S < %s | \
 ; RUN:   FileCheck --check-prefix=VECTORIZE %s
 
-; RUN: opt -opaque-pointers=0 -aa-pipeline=basic-aa -passes='loop-distribute,print<access-info>' -enable-loop-distribute \
+; RUN: opt -aa-pipeline=basic-aa -passes='loop-distribute,print<access-info>' -enable-loop-distribute \
 ; RUN:   -verify-loop-info -verify-dom-info -disable-output < %s 2>&1 | FileCheck %s --check-prefix=ANALYSIS
 
 ; The memcheck version of basic.ll.  We should distribute and vectorize the
@@ -20,20 +20,20 @@
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
 
-@B = common global i32* null, align 8
-@A = common global i32* null, align 8
-@C = common global i32* null, align 8
-@D = common global i32* null, align 8
-@E = common global i32* null, align 8
+@B = common global ptr null, align 8
+@A = common global ptr null, align 8
+@C = common global ptr null, align 8
+@D = common global ptr null, align 8
+@E = common global ptr null, align 8
 
 ; CHECK-LABEL: @f(
 define void @f() {
 entry:
-  %a = load i32*, i32** @A, align 8
-  %b = load i32*, i32** @B, align 8
-  %c = load i32*, i32** @C, align 8
-  %d = load i32*, i32** @D, align 8
-  %e = load i32*, i32** @E, align 8
+  %a = load ptr, ptr @A, align 8
+  %b = load ptr, ptr @B, align 8
+  %c = load ptr, ptr @C, align 8
+  %d = load ptr, ptr @D, align 8
+  %e = load ptr, ptr @E, align 8
   br label %for.body
 
 ; We have two compares for each array overlap check.
@@ -54,7 +54,7 @@ entry:
 ; CHECK:     = icmp
 
 ; CHECK-NOT: = icmp
-; CHECK:     br i1 %conflict.rdx25, label %for.body.ph.lver.orig, label %for.body.ph.ldist1
+; CHECK:     br i1 %conflict.rdx15, label %for.body.ph.lver.orig, label %for.body.ph.ldist1
 
 ; The non-distributed loop that the memchecks fall back on.
 
@@ -83,28 +83,28 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
 
-  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
-  %loadA = load i32, i32* %arrayidxA, align 4
+  %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
+  %loadA = load i32, ptr %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
-  %loadB = load i32, i32* %arrayidxB, align 4
+  %arrayidxB = getelementptr inbounds i32, ptr %b, i64 %ind
+  %loadB = load i32, ptr %arrayidxB, align 4
 
   %mulA = mul i32 %loadB, %loadA
 
   %add = add nuw nsw i64 %ind, 1
-  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
-  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+  %arrayidxA_plus_4 = getelementptr inbounds i32, ptr %a, i64 %add
+  store i32 %mulA, ptr %arrayidxA_plus_4, align 4
 
-  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
-  %loadD = load i32, i32* %arrayidxD, align 4
+  %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
+  %loadD = load i32, ptr %arrayidxD, align 4
 
-  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
-  %loadE = load i32, i32* %arrayidxE, align 4
+  %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
+  %loadE = load i32, ptr %arrayidxE, align 4
 
   %mulC = mul i32 %loadD, %loadE
 
-  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
-  store i32 %mulC, i32* %arrayidxC, align 4
+  %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
+  store i32 %mulC, ptr %arrayidxC, align 4
 
   %exitcond = icmp eq i64 %add, 20
   br i1 %exitcond, label %for.end, label %for.body
@@ -137,38 +137,38 @@ for.end:                                          ; preds = %for.body
 ; VECTORIZE-NOT: mul <4 x i32>
 define void @f_volatile_load() {
 entry:
-  %a = load i32*, i32** @A, align 8
-  %b = load i32*, i32** @B, align 8
-  %c = load i32*, i32** @C, align 8
-  %d = load i32*, i32** @D, align 8
-  %e = load i32*, i32** @E, align 8
+  %a = load ptr, ptr @A, align 8
+  %b = load ptr, ptr @B, align 8
+  %c = load ptr, ptr @C, align 8
+  %d = load ptr, ptr @D, align 8
+  %e = load ptr, ptr @E, align 8
   br label %for.body
 
 for.body:
   %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
 
-  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
-  %loadA = load i32, i32* %arrayidxA, align 4
+  %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
+  %loadA = load i32, ptr %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
-  %loadB = load i32, i32* %arrayidxB, align 4
+  %arrayidxB = getelementptr inbounds i32, ptr %b, i64 %ind
+  %loadB = load i32, ptr %arrayidxB, align 4
 
   %mulA = mul i32 %loadB, %loadA
 
   %add = add nuw nsw i64 %ind, 1
-  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
-  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+  %arrayidxA_plus_4 = getelementptr inbounds i32, ptr %a, i64 %add
+  store i32 %mulA, ptr %arrayidxA_plus_4, align 4
 
-  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
-  %loadD = load volatile i32, i32* %arrayidxD, align 4
+  %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
+  %loadD = load volatile i32, ptr %arrayidxD, align 4
 
-  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
-  %loadE = load i32, i32* %arrayidxE, align 4
+  %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
+  %loadE = load i32, ptr %arrayidxE, align 4
 
   %mulC = mul i32 %loadD, %loadE
 
-  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
-  store i32 %mulC, i32* %arrayidxC, align 4
+  %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
+  store i32 %mulC, ptr %arrayidxC, align 4
 
   %exitcond = icmp eq i64 %add, 20
   br i1 %exitcond, label %for.end, label %for.body
@@ -191,39 +191,39 @@ declare i32 @llvm.convergent(i32) #0
 ; ANALYSIS: Report: cannot add control dependency to convergent operation
 define void @f_with_convergent() #1 {
 entry:
-  %a = load i32*, i32** @A, align 8
-  %b = load i32*, i32** @B, align 8
-  %c = load i32*, i32** @C, align 8
-  %d = load i32*, i32** @D, align 8
-  %e = load i32*, i32** @E, align 8
+  %a = load ptr, ptr @A, align 8
+  %b = load ptr, ptr @B, align 8
+  %c = load ptr, ptr @C, align 8
+  %d = load ptr, ptr @D, align 8
+  %e = load ptr, ptr @E, align 8
   br label %for.body
 
 for.body:                                         ; preds = %for.body, %entry
   %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
 
-  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
-  %loadA = load i32, i32* %arrayidxA, align 4
+  %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
+  %loadA = load i32, ptr %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
-  %loadB = load i32, i32* %arrayidxB, align 4
+  %arrayidxB = getelementptr inbounds i32, ptr %b, i64 %ind
+  %loadB = load i32, ptr %arrayidxB, align 4
 
   %mulA = mul i32 %loadB, %loadA
 
   %add = add nuw nsw i64 %ind, 1
-  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
-  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+  %arrayidxA_plus_4 = getelementptr inbounds i32, ptr %a, i64 %add
+  store i32 %mulA, ptr %arrayidxA_plus_4, align 4
 
-  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
-  %loadD = load i32, i32* %arrayidxD, align 4
+  %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
+  %loadD = load i32, ptr %arrayidxD, align 4
 
-  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
-  %loadE = load i32, i32* %arrayidxE, align 4
+  %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
+  %loadE = load i32, ptr %arrayidxE, align 4
 
   %convergentD = call i32 @llvm.convergent(i32 %loadD)
   %mulC = mul i32 %convergentD, %loadE
 
-  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
-  store i32 %mulC, i32* %arrayidxC, align 4
+  %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
+  store i32 %mulC, ptr %arrayidxC, align 4
 
   %exitcond = icmp eq i64 %add, 20
   br i1 %exitcond, label %for.end, label %for.body
@@ -240,39 +240,39 @@ for.end:                                          ; preds = %for.body
 ; CHECK-NOT: call i32 @llvm.convergent
 define void @f_with_convergent_forced_distribute() #1 {
 entry:
-  %a = load i32*, i32** @A, align 8
-  %b = load i32*, i32** @B, align 8
-  %c = load i32*, i32** @C, align 8
-  %d = load i32*, i32** @D, align 8
-  %e = load i32*, i32** @E, align 8
+  %a = load ptr, ptr @A, align 8
+  %b = load ptr, ptr @B, align 8
+  %c = load ptr, ptr @C, align 8
+  %d = load ptr, ptr @D, align 8
+  %e = load ptr, ptr @E, align 8
   br label %for.body
 
 for.body:                                         ; preds = %for.body, %entry
   %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
 
-  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
-  %loadA = load i32, i32* %arrayidxA, align 4
+  %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
+  %loadA = load i32, ptr %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
-  %loadB = load i32, i32* %arrayidxB, align 4
+  %arrayidxB = getelementptr inbounds i32, ptr %b, i64 %ind
+  %loadB = load i32, ptr %arrayidxB, align 4
 
   %mulA = mul i32 %loadB, %loadA
 
   %add = add nuw nsw i64 %ind, 1
-  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
-  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+  %arrayidxA_plus_4 = getelementptr inbounds i32, ptr %a, i64 %add
+  store i32 %mulA, ptr %arrayidxA_plus_4, align 4
 
-  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
-  %loadD = load i32, i32* %arrayidxD, align 4
+  %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
+  %loadD = load i32, ptr %arrayidxD, align 4
 
-  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
-  %loadE = load i32, i32* %arrayidxE, align 4
+  %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
+  %loadE = load i32, ptr %arrayidxE, align 4
 
   %convergentD = call i32 @llvm.convergent(i32 %loadD)
   %mulC = mul i32 %convergentD, %loadE
 
-  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
-  store i32 %mulC, i32* %arrayidxC, align 4
+  %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
+  store i32 %mulC, ptr %arrayidxC, align 4
 
   %exitcond = icmp eq i64 %add, 20
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
diff --git a/llvm/test/Transforms/LoopDistribute/followup.ll b/llvm/test/Transforms/LoopDistribute/followup.ll
index 99cd3fdd49bd9..86cfb1855ae34 100644
--- a/llvm/test/Transforms/LoopDistribute/followup.ll
+++ b/llvm/test/Transforms/LoopDistribute/followup.ll
@@ -1,39 +1,39 @@
-; RUN: opt -opaque-pointers=0 -passes=loop-distribute -S < %s | FileCheck %s
+; RUN: opt -passes=loop-distribute -S < %s | FileCheck %s
 ;
 ; Check that followup loop-attributes are applied to the loops after
 ; loop distribution.
 ;
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
-define void @f(i32* %a, i32* %b, i32* %c, i32* %d, i32* %e) {
+define void @f(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) {
 entry:
   br label %for.body
 
 for.body:
   %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
 
-  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
-  %loadA = load i32, i32* %arrayidxA, align 4
+  %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
+  %loadA = load i32, ptr %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
-  %loadB = load i32, i32* %arrayidxB, align 4
+  %arrayidxB = getelementptr inbounds i32, ptr %b, i64 %ind
+  %loadB = load i32, ptr %arrayidxB, align 4
 
   %mulA = mul i32 %loadB, %loadA
 
   %add = add nuw nsw i64 %ind, 1
-  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
-  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+  %arrayidxA_plus_4 = getelementptr inbounds i32, ptr %a, i64 %add
+  store i32 %mulA, ptr %arrayidxA_plus_4, align 4
 
-  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
-  %loadD = load i32, i32* %arrayidxD, align 4
+  %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
+  %loadD = load i32, ptr %arrayidxD, align 4
 
-  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
-  %loadE = load i32, i32* %arrayidxE, align 4
+  %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
+  %loadE = load i32, ptr %arrayidxE, align 4
 
   %mulC = mul i32 %loadD, %loadE
 
-  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
-  store i32 %mulC, i32* %arrayidxC, align 4
+  %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
+  store i32 %mulC, ptr %arrayidxC, align 4
 
   %exitcond = icmp eq i64 %add, 20
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
@@ -55,10 +55,10 @@ for.end:
 ; CHECK-LABEL: for.body.ldist1:
 ; CHECK: br i1 %exitcond.ldist1, label %for.body.ph, label %for.body.ldist1, !llvm.loop ![[LOOP_SEQUENTIAL:[0-9]+]]
 ; CHECK-LABEL: for.body:
-; CHECK: br i1 %exitcond, label %for.end.loopexit26, label %for.body, !llvm.loop ![[LOOP_COINCIDENT:[0-9]+]]
+; CHECK: br i1 %exitcond, label %for.end.loopexit16, label %for.body, !llvm.loop ![[LOOP_COINCIDENT:[0-9]+]]
 ; CHECK-LABEL: for.end.loopexit:
 ; CHECK: br label %for.end
-; CHECK-LABEL: for.end.loopexit26:
+; CHECK-LABEL: for.end.loopexit16:
 ; CHECK: br label %for.end
 
 ; CHECK: ![[LOOP_ORIG]] = distinct !{![[LOOP_ORIG]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOUP_FALLBACK:[0-9]+]]}
diff --git a/llvm/test/Transforms/LoopDistribute/outside-use.ll b/llvm/test/Transforms/LoopDistribute/outside-use.ll
index 2d87dfc421b4a..e564c445fc6b9 100644
--- a/llvm/test/Transforms/LoopDistribute/outside-use.ll
+++ b/llvm/test/Transforms/LoopDistribute/outside-use.ll
@@ -1,4 +1,4 @@
-; RUN: opt -opaque-pointers=0 -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S < %s \
+; RUN: opt -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S < %s \
 ; RUN:   | FileCheck %s
 
 ; Check that definitions used outside the loop are handled correctly: (1) they
@@ -14,20 +14,20 @@
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
 
-@B = common global i32* null, align 8
-@A = common global i32* null, align 8
-@C = common global i32* null, align 8
-@D = common global i32* null, align 8
-@E = common global i32* null, align 8
+@B = common global ptr null, align 8
+@A = common global ptr null, align 8
+@C = common global ptr null, align 8
+@D = common global ptr null, align 8
+@E = common global ptr null, align 8
 @SUM = common global i32 0, align 8
 
 define void @f() {
 entry:
-  %a = load i32*, i32** @A, align 8
-  %b = load i32*, i32** @B, align 8
-  %c = load i32*, i32** @C, align 8
-  %d = load i32*, i32** @D, align 8
-  %e = load i32*, i32** @E, align 8
+  %a = load ptr, ptr @A, align 8
+  %b = load ptr, ptr @B, align 8
+  %c = load ptr, ptr @C, align 8
+  %d = load ptr, ptr @D, align 8
+  %e = load ptr, ptr @E, align 8
 
   br label %for.body
 
@@ -38,29 +38,29 @@ entry:
 ; CHECK:   %sum_add = add nuw nsw i32 %sum, %loadC
 ; CHECK: for.end.loopexit:
 ; CHECK:   %sum_add.lver.ph = phi i32 [ %sum_add.lver.orig, %for.body.lver.orig ]
-; CHECK: for.end.loopexit6:
-; CHECK:   %sum_add.lver.ph7 = phi i32 [ %sum_add, %for.body ]
+; CHECK: for.end.loopexit2:
+; CHECK:   %sum_add.lver.ph3 = phi i32 [ %sum_add, %for.body ]
 ; CHECK: for.end:
-; CHECK:   %sum_add.lver = phi i32 [ %sum_add.lver.ph, %for.end.loopexit ], [ %sum_add.lver.ph7, %for.end.loopexit6 ]
+; CHECK:   %sum_add.lver = phi i32 [ %sum_add.lver.ph, %for.end.loopexit ], [ %sum_add.lver.ph3, %for.end.loopexit2 ]
 
 for.body:                                         ; preds = %for.body, %entry
   %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
   %sum = phi i32 [ 0, %entry ], [ %sum_add, %for.body ]
 
-  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
-  %loadA = load i32, i32* %arrayidxA, align 4
+  %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
+  %loadA = load i32, ptr %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
-  %loadB = load i32, i32* %arrayidxB, align 4
+  %arrayidxB = getelementptr inbounds i32, ptr %b, i64 %ind
+  %loadB = load i32, ptr %arrayidxB, align 4
 
   %mulA = mul i32 %loadB, %loadA
 
   %add = add nuw nsw i64 %ind, 1
-  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
-  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+  %arrayidxA_plus_4 = getelementptr inbounds i32, ptr %a, i64 %add
+  store i32 %mulA, ptr %arrayidxA_plus_4, align 4
 
-  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
-  %loadC = load i32, i32* %arrayidxC, align 4
+  %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
+  %loadC = load i32, ptr %arrayidxC, align 4
 
   %sum_add = add nuw nsw i32 %sum, %loadC
 
@@ -68,6 +68,6 @@ for.body:                                         ; preds = %for.body, %entry
   br i1 %exitcond, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body
-  store i32 %sum_add, i32* @SUM, align 4
+  store i32 %sum_add, ptr @SUM, align 4
   ret void
 }
diff --git a/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll b/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll
index dc049c9b5e68e..83379f6a54fc1 100644
--- a/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll
+++ b/llvm/test/Transforms/LoopLoadElim/pr46854-adress-spaces.ll
@@ -3,7 +3,7 @@
 
 ; RUN: opt -passes='require<globals-aa>,loop-simplify,loop-load-elim' -S %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 %struct.foo = type { %struct.pluto, i8, ptr, i32 }
diff --git a/llvm/test/Transforms/LoopReroll/basic.ll b/llvm/test/Transforms/LoopReroll/basic.ll
index 9143604a50d0c..92d3456505759 100644
--- a/llvm/test/Transforms/LoopReroll/basic.ll
+++ b/llvm/test/Transforms/LoopReroll/basic.ll
@@ -1,4 +1,5 @@
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-reroll -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=loop-reroll -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -11,8 +12,20 @@ target triple = "x86_64-unknown-linux-gnu"
 ;   }
 ; }
 
-; Function Attrs: nounwind uwtable
-define void @bar(i32* nocapture readnone %x) #0 {
+define void @bar(ptr nocapture readnone %x) #0 {
+; CHECK-LABEL: define void @bar
+; CHECK-SAME: (ptr nocapture readnone [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 [[INDVAR]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i32 [[INDVAR]], 500
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -27,17 +40,6 @@ for.body:                                         ; preds = %for.body, %entry
   %exitcond = icmp sge i32 %add3, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @bar
-
-; CHECK: for.body:
-; CHECK: %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, %entry ]
-; CHECK: %call = tail call i32 @foo(i32 %indvar) #1
-; CHECK: %indvar.next = add i32 %indvar, 1
-; CHECK: %exitcond1 = icmp eq i32 %indvar, 500
-; CHECK: br i1 %exitcond1, label %for.end, label %for.body
-
-; CHECK: ret
-
 for.end:                                          ; preds = %for.body
   ret void
 }
@@ -53,42 +55,44 @@ declare i32 @foo(i32)
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @hi1(i32* nocapture %x) #0 {
+define void @hi1(ptr nocapture %x) #0 {
+; CHECK-LABEL: define void @hi1
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVAR]] to i32
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[INDVAR]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP0]], 1499
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %call = tail call i32 @foo(i32 0) #1
-  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
-  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %indvars.iv
+  store i32 %call, ptr %arrayidx, align 4
   %call1 = tail call i32 @foo(i32 0) #1
   %0 = add nsw i64 %indvars.iv, 1
-  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %0
-  store i32 %call1, i32* %arrayidx3, align 4
+  %arrayidx3 = getelementptr inbounds i32, ptr %x, i64 %0
+  store i32 %call1, ptr %arrayidx3, align 4
   %call4 = tail call i32 @foo(i32 0) #1
   %1 = add nsw i64 %indvars.iv, 2
-  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %1
-  store i32 %call4, i32* %arrayidx7, align 4
+  %arrayidx7 = getelementptr inbounds i32, ptr %x, i64 %1
+  store i32 %call4, ptr %arrayidx7, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 3
   %2 = trunc i64 %indvars.iv.next to i32
   %cmp = icmp slt i32 %2, 1500
   br i1 %cmp, label %for.body, label %for.end
 
-; CHECK-LABEL: @hi1
-
-; CHECK: for.body:
-; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
-; CHECK: %0 = trunc i64 %indvar to i32
-; CHECK: %call = tail call i32 @foo(i32 0) #1
-; CHECK: %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvar
-; CHECK: store i32 %call, i32* %arrayidx, align 4
-; CHECK: %indvar.next = add i64 %indvar, 1
-; CHECK: %exitcond = icmp eq i32 %0, 1499
-; CHECK: br i1 %exitcond, label %for.end, label %for.body
-
-; CHECK: ret
-
 for.end:                                          ; preds = %for.body
   ret void
 }
@@ -102,7 +106,22 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @hi2(i32* nocapture %x) #0 {
+define void @hi2(ptr nocapture %x) #0 {
+; CHECK-LABEL: define void @hi2
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i64 [[INDVARS_IV]], 1499
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -110,33 +129,20 @@ for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %call = tail call i32 @foo(i32 0) #1
   %0 = mul nsw i64 %indvars.iv, 3
-  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
-  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %0
+  store i32 %call, ptr %arrayidx, align 4
   %call1 = tail call i32 @foo(i32 0) #1
   %1 = add nsw i64 %0, 1
-  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
-  store i32 %call1, i32* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds i32, ptr %x, i64 %1
+  store i32 %call1, ptr %arrayidx4, align 4
   %call5 = tail call i32 @foo(i32 0) #1
   %2 = add nsw i64 %0, 2
-  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
-  store i32 %call5, i32* %arrayidx9, align 4
+  %arrayidx9 = getelementptr inbounds i32, ptr %x, i64 %2
+  store i32 %call5, ptr %arrayidx9, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @hi2
-
-; CHECK: for.body:
-; CHECK: %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-; CHECK: %call = tail call i32 @foo(i32 0) #1
-; CHECK: %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
-; CHECK: store i32 %call, i32* %arrayidx, align 4
-; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-; CHECK: %exitcond1 = icmp eq i64 %indvars.iv, 1499
-; CHECK: br i1 %exitcond1, label %for.end, label %for.body
-
-; CHECK: ret
-
 for.end:                                          ; preds = %for.body
   ret void
 }
@@ -152,74 +158,76 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @goo(float %alpha, float* nocapture %a, float* nocapture readonly %b) #0 {
+define void @goo(float %alpha, ptr nocapture %a, ptr nocapture readonly %b) #0 {
+; CHECK-LABEL: define void @goo
+; CHECK-SAME: (float [[ALPHA:%.*]], ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVAR]] to i32
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP1]], [[ALPHA]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP2]], [[MUL]]
+; CHECK-NEXT:    store float [[ADD]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP0]], 3199
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds float, ptr %b, i64 %indvars.iv
+  %0 = load float, ptr %arrayidx, align 4
   %mul = fmul float %0, %alpha
-  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
-  %1 = load float, float* %arrayidx2, align 4
+  %arrayidx2 = getelementptr inbounds float, ptr %a, i64 %indvars.iv
+  %1 = load float, ptr %arrayidx2, align 4
   %add = fadd float %1, %mul
-  store float %add, float* %arrayidx2, align 4
+  store float %add, ptr %arrayidx2, align 4
   %2 = add nsw i64 %indvars.iv, 1
-  %arrayidx5 = getelementptr inbounds float, float* %b, i64 %2
-  %3 = load float, float* %arrayidx5, align 4
+  %arrayidx5 = getelementptr inbounds float, ptr %b, i64 %2
+  %3 = load float, ptr %arrayidx5, align 4
   %mul6 = fmul float %3, %alpha
-  %arrayidx9 = getelementptr inbounds float, float* %a, i64 %2
-  %4 = load float, float* %arrayidx9, align 4
+  %arrayidx9 = getelementptr inbounds float, ptr %a, i64 %2
+  %4 = load float, ptr %arrayidx9, align 4
   %add10 = fadd float %4, %mul6
-  store float %add10, float* %arrayidx9, align 4
+  store float %add10, ptr %arrayidx9, align 4
   %5 = add nsw i64 %indvars.iv, 2
-  %arrayidx13 = getelementptr inbounds float, float* %b, i64 %5
-  %6 = load float, float* %arrayidx13, align 4
+  %arrayidx13 = getelementptr inbounds float, ptr %b, i64 %5
+  %6 = load float, ptr %arrayidx13, align 4
   %mul14 = fmul float %6, %alpha
-  %arrayidx17 = getelementptr inbounds float, float* %a, i64 %5
-  %7 = load float, float* %arrayidx17, align 4
+  %arrayidx17 = getelementptr inbounds float, ptr %a, i64 %5
+  %7 = load float, ptr %arrayidx17, align 4
   %add18 = fadd float %7, %mul14
-  store float %add18, float* %arrayidx17, align 4
+  store float %add18, ptr %arrayidx17, align 4
   %8 = add nsw i64 %indvars.iv, 3
-  %arrayidx21 = getelementptr inbounds float, float* %b, i64 %8
-  %9 = load float, float* %arrayidx21, align 4
+  %arrayidx21 = getelementptr inbounds float, ptr %b, i64 %8
+  %9 = load float, ptr %arrayidx21, align 4
   %mul22 = fmul float %9, %alpha
-  %arrayidx25 = getelementptr inbounds float, float* %a, i64 %8
-  %10 = load float, float* %arrayidx25, align 4
+  %arrayidx25 = getelementptr inbounds float, ptr %a, i64 %8
+  %10 = load float, ptr %arrayidx25, align 4
   %add26 = fadd float %10, %mul22
-  store float %add26, float* %arrayidx25, align 4
+  store float %add26, ptr %arrayidx25, align 4
   %11 = add nsw i64 %indvars.iv, 4
-  %arrayidx29 = getelementptr inbounds float, float* %b, i64 %11
-  %12 = load float, float* %arrayidx29, align 4
+  %arrayidx29 = getelementptr inbounds float, ptr %b, i64 %11
+  %12 = load float, ptr %arrayidx29, align 4
   %mul30 = fmul float %12, %alpha
-  %arrayidx33 = getelementptr inbounds float, float* %a, i64 %11
-  %13 = load float, float* %arrayidx33, align 4
+  %arrayidx33 = getelementptr inbounds float, ptr %a, i64 %11
+  %13 = load float, ptr %arrayidx33, align 4
   %add34 = fadd float %13, %mul30
-  store float %add34, float* %arrayidx33, align 4
+  store float %add34, ptr %arrayidx33, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 5
   %14 = trunc i64 %indvars.iv.next to i32
   %cmp = icmp slt i32 %14, 3200
   br i1 %cmp, label %for.body, label %for.end
 
-; CHECK-LABEL: @goo
-
-; CHECK: for.body:
-; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
-; CHECK: %0 = trunc i64 %indvar to i32
-; CHECK: %arrayidx = getelementptr inbounds float, float* %b, i64 %indvar
-; CHECK: %1 = load float, float* %arrayidx, align 4
-; CHECK: %mul = fmul float %1, %alpha
-; CHECK: %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvar
-; CHECK: %2 = load float, float* %arrayidx2, align 4
-; CHECK: %add = fadd float %2, %mul
-; CHECK: store float %add, float* %arrayidx2, align 4
-; CHECK: %indvar.next = add i64 %indvar, 1
-; CHECK: %exitcond = icmp eq i32 %0, 3199
-; CHECK: br i1 %exitcond, label %for.end, label %for.body
-
-; CHECK: ret
-
 for.end:                                          ; preds = %for.body
   ret void
 }
@@ -235,91 +243,96 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @hoo(float %alpha, float* nocapture %a, float* nocapture readonly %b, i32* nocapture readonly %ip) #0 {
+define void @hoo(float %alpha, ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %ip) #0 {
+; CHECK-LABEL: define void @hoo
+; CHECK-SAME: (float [[ALPHA:%.*]], ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]], ptr nocapture readonly [[IP:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVAR]] to i32
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[IP]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP2]], [[ALPHA]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP3]], [[MUL]]
+; CHECK-NEXT:    store float [[ADD]], ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP0]], 3199
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds i32, i32* %ip, i64 %indvars.iv
-  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %ip, i64 %indvars.iv
+  %0 = load i32, ptr %arrayidx, align 4
   %idxprom1 = sext i32 %0 to i64
-  %arrayidx2 = getelementptr inbounds float, float* %b, i64 %idxprom1
-  %1 = load float, float* %arrayidx2, align 4
+  %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %idxprom1
+  %1 = load float, ptr %arrayidx2, align 4
   %mul = fmul float %1, %alpha
-  %arrayidx4 = getelementptr inbounds float, float* %a, i64 %indvars.iv
-  %2 = load float, float* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds float, ptr %a, i64 %indvars.iv
+  %2 = load float, ptr %arrayidx4, align 4
   %add = fadd float %2, %mul
-  store float %add, float* %arrayidx4, align 4
+  store float %add, ptr %arrayidx4, align 4
   %3 = add nsw i64 %indvars.iv, 1
-  %arrayidx7 = getelementptr inbounds i32, i32* %ip, i64 %3
-  %4 = load i32, i32* %arrayidx7, align 4
+  %arrayidx7 = getelementptr inbounds i32, ptr %ip, i64 %3
+  %4 = load i32, ptr %arrayidx7, align 4
   %idxprom8 = sext i32 %4 to i64
-  %arrayidx9 = getelementptr inbounds float, float* %b, i64 %idxprom8
-  %5 = load float, float* %arrayidx9, align 4
+  %arrayidx9 = getelementptr inbounds float, ptr %b, i64 %idxprom8
+  %5 = load float, ptr %arrayidx9, align 4
   %mul10 = fmul float %5, %alpha
-  %arrayidx13 = getelementptr inbounds float, float* %a, i64 %3
-  %6 = load float, float* %arrayidx13, align 4
+  %arrayidx13 = getelementptr inbounds float, ptr %a, i64 %3
+  %6 = load float, ptr %arrayidx13, align 4
   %add14 = fadd float %6, %mul10
-  store float %add14, float* %arrayidx13, align 4
+  store float %add14, ptr %arrayidx13, align 4
   %7 = add nsw i64 %indvars.iv, 2
-  %arrayidx17 = getelementptr inbounds i32, i32* %ip, i64 %7
-  %8 = load i32, i32* %arrayidx17, align 4
+  %arrayidx17 = getelementptr inbounds i32, ptr %ip, i64 %7
+  %8 = load i32, ptr %arrayidx17, align 4
   %idxprom18 = sext i32 %8 to i64
-  %arrayidx19 = getelementptr inbounds float, float* %b, i64 %idxprom18
-  %9 = load float, float* %arrayidx19, align 4
+  %arrayidx19 = getelementptr inbounds float, ptr %b, i64 %idxprom18
+  %9 = load float, ptr %arrayidx19, align 4
   %mul20 = fmul float %9, %alpha
-  %arrayidx23 = getelementptr inbounds float, float* %a, i64 %7
-  %10 = load float, float* %arrayidx23, align 4
+  %arrayidx23 = getelementptr inbounds float, ptr %a, i64 %7
+  %10 = load float, ptr %arrayidx23, align 4
   %add24 = fadd float %10, %mul20
-  store float %add24, float* %arrayidx23, align 4
+  store float %add24, ptr %arrayidx23, align 4
   %11 = add nsw i64 %indvars.iv, 3
-  %arrayidx27 = getelementptr inbounds i32, i32* %ip, i64 %11
-  %12 = load i32, i32* %arrayidx27, align 4
+  %arrayidx27 = getelementptr inbounds i32, ptr %ip, i64 %11
+  %12 = load i32, ptr %arrayidx27, align 4
   %idxprom28 = sext i32 %12 to i64
-  %arrayidx29 = getelementptr inbounds float, float* %b, i64 %idxprom28
-  %13 = load float, float* %arrayidx29, align 4
+  %arrayidx29 = getelementptr inbounds float, ptr %b, i64 %idxprom28
+  %13 = load float, ptr %arrayidx29, align 4
   %mul30 = fmul float %13, %alpha
-  %arrayidx33 = getelementptr inbounds float, float* %a, i64 %11
-  %14 = load float, float* %arrayidx33, align 4
+  %arrayidx33 = getelementptr inbounds float, ptr %a, i64 %11
+  %14 = load float, ptr %arrayidx33, align 4
   %add34 = fadd float %14, %mul30
-  store float %add34, float* %arrayidx33, align 4
+  store float %add34, ptr %arrayidx33, align 4
   %15 = add nsw i64 %indvars.iv, 4
-  %arrayidx37 = getelementptr inbounds i32, i32* %ip, i64 %15
-  %16 = load i32, i32* %arrayidx37, align 4
+  %arrayidx37 = getelementptr inbounds i32, ptr %ip, i64 %15
+  %16 = load i32, ptr %arrayidx37, align 4
   %idxprom38 = sext i32 %16 to i64
-  %arrayidx39 = getelementptr inbounds float, float* %b, i64 %idxprom38
-  %17 = load float, float* %arrayidx39, align 4
+  %arrayidx39 = getelementptr inbounds float, ptr %b, i64 %idxprom38
+  %17 = load float, ptr %arrayidx39, align 4
   %mul40 = fmul float %17, %alpha
-  %arrayidx43 = getelementptr inbounds float, float* %a, i64 %15
-  %18 = load float, float* %arrayidx43, align 4
+  %arrayidx43 = getelementptr inbounds float, ptr %a, i64 %15
+  %18 = load float, ptr %arrayidx43, align 4
   %add44 = fadd float %18, %mul40
-  store float %add44, float* %arrayidx43, align 4
+  store float %add44, ptr %arrayidx43, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 5
   %19 = trunc i64 %indvars.iv.next to i32
   %cmp = icmp slt i32 %19, 3200
   br i1 %cmp, label %for.body, label %for.end
 
-; CHECK-LABEL: @hoo
-
-; CHECK: for.body:
-; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
-; CHECK: %0 = trunc i64 %indvar to i32
-; CHECK: %arrayidx = getelementptr inbounds i32, i32* %ip, i64 %indvar
-; CHECK: %1 = load i32, i32* %arrayidx, align 4
-; CHECK: %idxprom1 = sext i32 %1 to i64
-; CHECK: %arrayidx2 = getelementptr inbounds float, float* %b, i64 %idxprom1
-; CHECK: %2 = load float, float* %arrayidx2, align 4
-; CHECK: %mul = fmul float %2, %alpha
-; CHECK: %arrayidx4 = getelementptr inbounds float, float* %a, i64 %indvar
-; CHECK: %3 = load float, float* %arrayidx4, align 4
-; CHECK: %add = fadd float %3, %mul
-; CHECK: store float %add, float* %arrayidx4, align 4
-; CHECK: %indvar.next = add i64 %indvar, 1
-; CHECK: %exitcond = icmp eq i32 %0, 3199
-; CHECK: br i1 %exitcond, label %for.end, label %for.body
-
-; CHECK: ret
+
+
 
 for.end:                                          ; preds = %for.body
   ret void
@@ -338,7 +351,25 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @multi1(i32* nocapture %x) #0 {
+define void @multi1(ptr nocapture %x) #0 {
+; CHECK-LABEL: define void @multi1
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDVARS_IV]], 6
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[TMP0]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i64 [[INDVARS_IV]], 1499
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = tail call i32 @foo(i32 0) #1
   br label %for.body
@@ -346,39 +377,28 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %0 = mul nsw i64 %indvars.iv, 3
-  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
-  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %0
+  store i32 %call, ptr %arrayidx, align 4
   %1 = add nsw i64 %0, 1
-  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
-  store i32 %call, i32* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds i32, ptr %x, i64 %1
+  store i32 %call, ptr %arrayidx4, align 4
   %2 = add nsw i64 %0, 2
-  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
-  store i32 %call, i32* %arrayidx9, align 4
+  %arrayidx9 = getelementptr inbounds i32, ptr %x, i64 %2
+  store i32 %call, ptr %arrayidx9, align 4
   %3 = add nsw i64 %0, 6
-  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %3
-  store i32 %call, i32* %arrayidx6, align 4
+  %arrayidx6 = getelementptr inbounds i32, ptr %x, i64 %3
+  store i32 %call, ptr %arrayidx6, align 4
   %4 = add nsw i64 %0, 7
-  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %4
-  store i32 %call, i32* %arrayidx7, align 4
+  %arrayidx7 = getelementptr inbounds i32, ptr %x, i64 %4
+  store i32 %call, ptr %arrayidx7, align 4
   %5 = add nsw i64 %0, 8
-  %arrayidx8 = getelementptr inbounds i32, i32* %x, i64 %5
-  store i32 %call, i32* %arrayidx8, align 4
+  %arrayidx8 = getelementptr inbounds i32, ptr %x, i64 %5
+  store i32 %call, ptr %arrayidx8, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @multi1
 
-; CHECK:for.body:
-; CHECK:  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-; CHECK:  %0 = add i64 %indvars.iv, 6
-; CHECK:  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
-; CHECK:  store i32 %call, i32* %arrayidx, align 4
-; CHECK:  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %0
-; CHECK:  store i32 %call, i32* %arrayidx6, align 4
-; CHECK:  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-; CHECK:  %exitcond1 = icmp eq i64 %indvars.iv, 1499
-; CHECK:  br i1 %exitcond1, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body
   ret void
@@ -397,7 +417,25 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @multi2(i32* nocapture %x) #0 {
+define void @multi2(ptr nocapture %x) #0 {
+; CHECK-LABEL: define void @multi2
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[TMP0]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i64 [[INDVARS_IV]], 1499
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = tail call i32 @foo(i32 0) #1
   br label %for.body
@@ -407,38 +445,27 @@ for.body:                                         ; preds = %for.body, %entry
   %0 = mul nsw i64 %indvars.iv, 3
   %add = add nsw i64 %indvars.iv, 1
   %newmul = mul nsw i64 %add, 3
-  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
-  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %0
+  store i32 %call, ptr %arrayidx, align 4
   %1 = add nsw i64 %0, 1
-  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
-  store i32 %call, i32* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds i32, ptr %x, i64 %1
+  store i32 %call, ptr %arrayidx4, align 4
   %2 = add nsw i64 %0, 2
-  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
-  store i32 %call, i32* %arrayidx9, align 4
-  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %newmul
-  store i32 %call, i32* %arrayidx6, align 4
+  %arrayidx9 = getelementptr inbounds i32, ptr %x, i64 %2
+  store i32 %call, ptr %arrayidx9, align 4
+  %arrayidx6 = getelementptr inbounds i32, ptr %x, i64 %newmul
+  store i32 %call, ptr %arrayidx6, align 4
   %3 = add nsw i64 %newmul, 1
-  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %3
-  store i32 %call, i32* %arrayidx7, align 4
+  %arrayidx7 = getelementptr inbounds i32, ptr %x, i64 %3
+  store i32 %call, ptr %arrayidx7, align 4
   %4 = add nsw i64 %newmul, 2
-  %arrayidx8 = getelementptr inbounds i32, i32* %x, i64 %4
-  store i32 %call, i32* %arrayidx8, align 4
+  %arrayidx8 = getelementptr inbounds i32, ptr %x, i64 %4
+  store i32 %call, ptr %arrayidx8, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @multi2
 
-; CHECK:for.body:
-; CHECK:  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-; CHECK:  %0 = add i64 %indvars.iv, 3
-; CHECK:  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
-; CHECK:  store i32 %call, i32* %arrayidx, align 4
-; CHECK:  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %0
-; CHECK:  store i32 %call, i32* %arrayidx6, align 4
-; CHECK:  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-; CHECK:  %exitcond1 = icmp eq i64 %indvars.iv, 1499
-; CHECK:  br i1 %exitcond1, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body
   ret void
@@ -455,7 +482,23 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @multi3(i32* nocapture %x) #0 {
+define void @multi3(ptr nocapture %x) #0 {
+; CHECK-LABEL: define void @multi3
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[TMP0]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i64 [[INDVARS_IV]], 1499
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = tail call i32 @foo(i32 0) #1
   br label %for.body
@@ -465,27 +508,18 @@ for.body:                                         ; preds = %for.body, %entry
   %0 = mul nsw i64 %indvars.iv, 3
   %x0 = add nsw i64 %0, 3
   %add = add nsw i64 %indvars.iv, 1
-  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %x0
-  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %x0
+  store i32 %call, ptr %arrayidx, align 4
   %1 = add nsw i64 %0, 4
-  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
-  store i32 %call, i32* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds i32, ptr %x, i64 %1
+  store i32 %call, ptr %arrayidx4, align 4
   %2 = add nsw i64 %0, 5
-  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
-  store i32 %call, i32* %arrayidx9, align 4
+  %arrayidx9 = getelementptr inbounds i32, ptr %x, i64 %2
+  store i32 %call, ptr %arrayidx9, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @multi3
-; CHECK: for.body:
-; CHECK:   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-; CHECK:   %0 = add i64 %indvars.iv, 3
-; CHECK:   %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
-; CHECK:   store i32 %call, i32* %arrayidx, align 4
-; CHECK:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-; CHECK:   %exitcond1 = icmp eq i64 %indvars.iv, 1499
-; CHECK:   br i1 %exitcond1, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body
   ret void
@@ -501,7 +535,23 @@ for.end:                                          ; preds = %for.body
 ; }
 
 ; Function Attrs: nounwind uwtable
-define void @bar2(i32* nocapture readnone %x, i32 %y, i32 %z) #0 {
+define void @bar2(ptr nocapture readnone %x, i32 %y, i32 %z) #0 {
+; CHECK-LABEL: define void @bar2
+; CHECK-SAME: (ptr nocapture readnone [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDVAR]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[INDVAR]], [[Z]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 [[TMP3]]) #[[ATTR1]]
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i32 [[INDVAR]], 500
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -518,7 +568,7 @@ for.body:                                         ; preds = %for.body, %entry
   %tmp1a = add i32 %add, %y
   %tmp3a = add i32 %tmp2a, %tmp1a
   %calla = tail call i32 @foo(i32 %tmp3a) #1
-  
+
   %add2 = add nsw i32 %i.08, 2
   %tmp2b = mul i32 %add2, %z
   %tmp1b = add i32 %add2, %y
@@ -530,20 +580,6 @@ for.body:                                         ; preds = %for.body, %entry
   %exitcond = icmp sge i32 %add3, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @bar2
-
-; CHECK: for.body:
-; CHECK: %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, %entry ]
-; CHECK: %tmp1 = add i32 %indvar, %y
-; CHECK: %tmp2 = mul i32 %indvar, %z
-; CHECK: %tmp3 = add i32 %tmp2, %tmp1
-; CHECK: %call = tail call i32 @foo(i32 %tmp3) #1
-; CHECK: %indvar.next = add i32 %indvar, 1
-; CHECK: %exitcond1 = icmp eq i32 %indvar, 500
-; CHECK: br i1 %exitcond1, label %for.end, label %for.body
-
-; CHECK: ret
-
 for.end:                                          ; preds = %for.body
   ret void
 }
@@ -551,7 +587,29 @@ for.end:                                          ; preds = %for.body
 %struct.s = type { i32, i32 }
 
 ; Function Attrs: nounwind uwtable
-define void @gep1(%struct.s* nocapture %x) #0 {
+define void @gep1(ptr nocapture %x) #0 {
+; CHECK-LABEL: define void @gep1
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[X]], i64 [[TMP0]], i32 0
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[X]], i64 [[TMP1]], i32 0
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[X]], i64 [[TMP2]], i32 0
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 500
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = tail call i32 @foo(i32 0) #1
   br label %for.body
@@ -559,26 +617,40 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %0 = mul nsw i64 %indvars.iv, 3
-  %arrayidx = getelementptr inbounds %struct.s, %struct.s* %x, i64 %0, i32 0
-  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds %struct.s, ptr %x, i64 %0, i32 0
+  store i32 %call, ptr %arrayidx, align 4
   %1 = add nsw i64 %0, 1
-  %arrayidx4 = getelementptr inbounds %struct.s, %struct.s* %x, i64 %1, i32 0
-  store i32 %call, i32* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds %struct.s, ptr %x, i64 %1, i32 0
+  store i32 %call, ptr %arrayidx4, align 4
   %2 = add nsw i64 %0, 2
-  %arrayidx9 = getelementptr inbounds %struct.s, %struct.s* %x, i64 %2, i32 0
-  store i32 %call, i32* %arrayidx9, align 4
+  %arrayidx9 = getelementptr inbounds %struct.s, ptr %x, i64 %2, i32 0
+  store i32 %call, ptr %arrayidx9, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @gep1
 ; This test is a crash test only.
-; CHECK: ret
 for.end:                                          ; preds = %for.body
   ret void
 }
 
-define void @gep-indexing(i32* nocapture %x) {
+define void @gep-indexing(ptr nocapture %x) {
+; CHECK-LABEL: define void @gep-indexing
+; CHECK-SAME: (ptr nocapture [[X:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 0) #[[ATTR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[X]], i64 [[TMP0]]
+; CHECK-NEXT:    store i32 [[CALL]], ptr [[SCEVGEP]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND1:%.*]] = icmp eq i64 [[INDVARS_IV]], 1499
+; CHECK-NEXT:    br i1 [[EXITCOND1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = tail call i32 @foo(i32 0) #1
   br label %for.body
@@ -586,43 +658,38 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %0 = mul nsw i64 %indvars.iv, 3
-  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
-  store i32 %call, i32* %arrayidx, align 4
-  %arrayidx4 = getelementptr inbounds i32, i32* %arrayidx, i64 1
-  store i32 %call, i32* %arrayidx4, align 4
-  %arrayidx9 = getelementptr inbounds i32, i32* %arrayidx, i64 2
-  store i32 %call, i32* %arrayidx9, align 4
+  %arrayidx = getelementptr inbounds i32, ptr %x, i64 %0
+  store i32 %call, ptr %arrayidx, align 4
+  %arrayidx4 = getelementptr inbounds i32, ptr %arrayidx, i64 1
+  store i32 %call, ptr %arrayidx4, align 4
+  %arrayidx9 = getelementptr inbounds i32, ptr %arrayidx, i64 2
+  store i32 %call, ptr %arrayidx9, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @gep-indexing
-; CHECK:      for.body:
-; CHECK-NEXT:   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-; CHECK-NEXT:   %scevgep = getelementptr i32, i32* %x, i64 %indvars.iv
-; CHECK-NEXT:   store i32 %call, i32* %scevgep, align 4
-; CHECK-NEXT:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-; CHECK-NEXT:   %exitcond1 = icmp eq i64 %indvars.iv, 1499
-; CHECK-NEXT:   br i1 %exitcond1, label %for.end, label %for.body
-
 for.end:                                          ; preds = %for.body
   ret void
 }
 
 
-define void @unordered_atomic_ops(i32* noalias %buf_0, i32* noalias %buf_1) {
-; CHECK-LABEL: @unordered_atomic_ops(
-
-; CHECK: for.body:
-; CHECK-NEXT:   %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, %entry ]
-; CHECK-NEXT:   %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvar
-; CHECK-NEXT:   %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvar
-; CHECK-NEXT:   %va = load atomic i32, i32* %buf0_a unordered, align 4
-; CHECK-NEXT:   store atomic i32 %va, i32* %buf1_a unordered, align 4
-; CHECK-NEXT:   %indvar.next = add i32 %indvar, 1
-; CHECK-NEXT:   %exitcond = icmp eq i32 %indvar, 3199
-; CHECK-NEXT:   br i1 %exitcond, label %for.end, label %for.body
-
+define void @unordered_atomic_ops(ptr noalias %buf_0, ptr noalias %buf_1) {
+; CHECK-LABEL: define void @unordered_atomic_ops
+; CHECK-SAME: (ptr noalias [[BUF_0:%.*]], ptr noalias [[BUF_1:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BUF0_A:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVAR]]
+; CHECK-NEXT:    [[BUF1_A:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVAR]]
+; CHECK-NEXT:    [[VA:%.*]] = load atomic i32, ptr [[BUF0_A]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[VA]], ptr [[BUF1_A]] unordered, align 4
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDVAR]], 3199
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -630,14 +697,14 @@ for.body:
   %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %indvars.iv.next = add i32 %indvars.iv, 2
   %indvars.mid = add i32 %indvars.iv, 1
-  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
-  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
-  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
-  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
-  %va = load atomic i32, i32* %buf0_a unordered, align 4
-  %vb = load atomic i32, i32* %buf0_b unordered, align 4
-  store atomic i32 %va, i32* %buf1_a unordered, align 4
-  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+  %buf0_a = getelementptr i32, ptr %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, ptr %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, ptr %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, ptr %buf_1, i32 %indvars.mid
+  %va = load atomic i32, ptr %buf0_a unordered, align 4
+  %vb = load atomic i32, ptr %buf0_b unordered, align 4
+  store atomic i32 %va, ptr %buf1_a unordered, align 4
+  store atomic i32 %vb, ptr %buf1_b unordered, align 4
   %cmp = icmp slt i32 %indvars.iv.next, 3200
   br i1 %cmp, label %for.body, label %for.end
 
@@ -645,31 +712,45 @@ for.end:
   ret void
 }
 
-define void @unordered_atomic_ops_nomatch(i32* noalias %buf_0, i32* noalias %buf_1) {
+define void @unordered_atomic_ops_nomatch(ptr noalias %buf_0, ptr noalias %buf_1) {
 ; Negative test
-
-; CHECK-LABEL: @unordered_atomic_ops_nomatch(
+; CHECK-LABEL: define void @unordered_atomic_ops_nomatch
+; CHECK-SAME: (ptr noalias [[BUF_0:%.*]], ptr noalias [[BUF_1:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[INDVARS_MID:%.*]] = add i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[BUF0_A:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[BUF0_B:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVARS_MID]]
+; CHECK-NEXT:    [[BUF1_A:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[BUF1_B:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVARS_MID]]
+; CHECK-NEXT:    [[VA:%.*]] = load atomic i32, ptr [[BUF0_A]] unordered, align 4
+; CHECK-NEXT:    [[VB:%.*]] = load atomic i32, ptr [[BUF0_B]] unordered, align 4
+; CHECK-NEXT:    store i32 [[VA]], ptr [[BUF1_A]], align 4
+; CHECK-NEXT:    store atomic i32 [[VB]], ptr [[BUF1_B]] unordered, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INDVARS_IV_NEXT]], 3200
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:
-; CHECK: for.body:
-; CHECK:   %indvars.iv.next = add i32 %indvars.iv, 2
-; CHECK:   %indvars.mid = add i32 %indvars.iv, 1
-; CHECK:   %cmp = icmp slt i32 %indvars.iv.next, 3200
-; CHECK:   br i1 %cmp, label %for.body, label %for.end
 
   %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %indvars.iv.next = add i32 %indvars.iv, 2
   %indvars.mid = add i32 %indvars.iv, 1
-  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
-  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
-  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
-  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
-  %va = load atomic i32, i32* %buf0_a unordered, align 4
-  %vb = load atomic i32, i32* %buf0_b unordered, align 4
-  store i32 %va, i32* %buf1_a, align 4  ;; Not atomic
-  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+  %buf0_a = getelementptr i32, ptr %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, ptr %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, ptr %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, ptr %buf_1, i32 %indvars.mid
+  %va = load atomic i32, ptr %buf0_a unordered, align 4
+  %vb = load atomic i32, ptr %buf0_b unordered, align 4
+  store i32 %va, ptr %buf1_a, align 4  ;; Not atomic
+  store atomic i32 %vb, ptr %buf1_b unordered, align 4
   %cmp = icmp slt i32 %indvars.iv.next, 3200
   br i1 %cmp, label %for.body, label %for.end
 
@@ -677,31 +758,45 @@ for.end:
   ret void
 }
 
-define void @ordered_atomic_ops(i32* noalias %buf_0, i32* noalias %buf_1) {
+define void @ordered_atomic_ops(ptr noalias %buf_0, ptr noalias %buf_1) {
 ; Negative test
-
-; CHECK-LABEL: @ordered_atomic_ops(
+; CHECK-LABEL: define void @ordered_atomic_ops
+; CHECK-SAME: (ptr noalias [[BUF_0:%.*]], ptr noalias [[BUF_1:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[INDVARS_MID:%.*]] = add i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[BUF0_A:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[BUF0_B:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVARS_MID]]
+; CHECK-NEXT:    [[BUF1_A:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[BUF1_B:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVARS_MID]]
+; CHECK-NEXT:    [[VA:%.*]] = load atomic i32, ptr [[BUF0_A]] acquire, align 4
+; CHECK-NEXT:    [[VB:%.*]] = load atomic i32, ptr [[BUF0_B]] acquire, align 4
+; CHECK-NEXT:    store atomic i32 [[VA]], ptr [[BUF1_A]] release, align 4
+; CHECK-NEXT:    store atomic i32 [[VB]], ptr [[BUF1_B]] release, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INDVARS_IV_NEXT]], 3200
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:
-; CHECK: for.body:
-; CHECK:   %indvars.iv.next = add i32 %indvars.iv, 2
-; CHECK:   %indvars.mid = add i32 %indvars.iv, 1
-; CHECK:   %cmp = icmp slt i32 %indvars.iv.next, 3200
-; CHECK:   br i1 %cmp, label %for.body, label %for.end
 
   %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %indvars.iv.next = add i32 %indvars.iv, 2
   %indvars.mid = add i32 %indvars.iv, 1
-  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
-  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
-  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
-  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
-  %va = load atomic i32, i32* %buf0_a acquire, align 4
-  %vb = load atomic i32, i32* %buf0_b acquire, align 4
-  store atomic i32 %va, i32* %buf1_a release, align 4
-  store atomic i32 %vb, i32* %buf1_b release, align 4
+  %buf0_a = getelementptr i32, ptr %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, ptr %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, ptr %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, ptr %buf_1, i32 %indvars.mid
+  %va = load atomic i32, ptr %buf0_a acquire, align 4
+  %vb = load atomic i32, ptr %buf0_b acquire, align 4
+  store atomic i32 %va, ptr %buf1_a release, align 4
+  store atomic i32 %vb, ptr %buf1_b release, align 4
   %cmp = icmp slt i32 %indvars.iv.next, 3200
   br i1 %cmp, label %for.body, label %for.end
 
@@ -709,31 +804,46 @@ for.end:
   ret void
 }
 
-define void @unordered_atomic_ops_with_fence(i32* noalias %buf_0, i32* noalias %buf_1) {
-; CHECK-LABEL: @unordered_atomic_ops_with_fence(
+define void @unordered_atomic_ops_with_fence(ptr noalias %buf_0, ptr noalias %buf_1) {
+; CHECK-LABEL: define void @unordered_atomic_ops_with_fence
+; CHECK-SAME: (ptr noalias [[BUF_0:%.*]], ptr noalias [[BUF_1:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[INDVARS_MID:%.*]] = add i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[BUF0_A:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[BUF0_B:%.*]] = getelementptr i32, ptr [[BUF_0]], i32 [[INDVARS_MID]]
+; CHECK-NEXT:    [[BUF1_A:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVARS_IV]]
+; CHECK-NEXT:    [[BUF1_B:%.*]] = getelementptr i32, ptr [[BUF_1]], i32 [[INDVARS_MID]]
+; CHECK-NEXT:    [[VA:%.*]] = load atomic i32, ptr [[BUF0_A]] unordered, align 4
+; CHECK-NEXT:    [[VB:%.*]] = load atomic i32, ptr [[BUF0_B]] unordered, align 4
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    store atomic i32 [[VA]], ptr [[BUF1_A]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[VB]], ptr [[BUF1_B]] unordered, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INDVARS_IV_NEXT]], 3200
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
 for.body:
-; CHECK: for.body:
-; CHECK:  %va = load atomic i32, i32* %buf0_a unordered, align 4
-; CHECK-NEXT:  %vb = load atomic i32, i32* %buf0_b unordered, align 4
-; CHECK-NEXT:  fence seq_cst
-; CHECK-NEXT:  store atomic i32 %va, i32* %buf1_a unordered, align 4
-; CHECK-NEXT:  store atomic i32 %vb, i32* %buf1_b unordered, align 4
 
   %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %indvars.iv.next = add i32 %indvars.iv, 2
   %indvars.mid = add i32 %indvars.iv, 1
-  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
-  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
-  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
-  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
-  %va = load atomic i32, i32* %buf0_a unordered, align 4
-  %vb = load atomic i32, i32* %buf0_b unordered, align 4
+  %buf0_a = getelementptr i32, ptr %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, ptr %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, ptr %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, ptr %buf_1, i32 %indvars.mid
+  %va = load atomic i32, ptr %buf0_a unordered, align 4
+  %vb = load atomic i32, ptr %buf0_b unordered, align 4
   fence seq_cst
-  store atomic i32 %va, i32* %buf1_a unordered, align 4
-  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+  store atomic i32 %va, ptr %buf1_a unordered, align 4
+  store atomic i32 %vb, ptr %buf1_b unordered, align 4
   %cmp = icmp slt i32 %indvars.iv.next, 3200
   br i1 %cmp, label %for.body, label %for.end
 
@@ -741,51 +851,72 @@ for.end:
   ret void
 }
 
-define void @pointer_bitcast_baseinst(i16* %arg, i8* %arg1, i64 %arg2) {
-; CHECK-LABEL: @pointer_bitcast_baseinst(
+define void @pointer_bitcast_baseinst(ptr %arg, ptr %arg1, i64 %arg2) {
+; CHECK-LABEL: define void @pointer_bitcast_baseinst
+; CHECK-SAME: (ptr [[ARG:%.*]], ptr [[ARG1:%.*]], i64 [[ARG2:%.*]]) {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[ARG2]], -17
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %bb3 ], [ 0, %bb ]
-; CHECK-NEXT:    %4 = shl nuw i64 %indvar, 3
-; CHECK-NEXT:    %5 = add i64 %4, 1
-; CHECK-NEXT:    %tmp5 = shl nuw i64 %5, 1
-; CHECK-NEXT:    %tmp6 = getelementptr i8, i8* %arg1, i64 %tmp5
-; CHECK-NEXT:    %tmp7 = bitcast i8* %tmp6 to <8 x i16>*
-; CHECK-NEXT:    %tmp8 = load <8 x i16>, <8 x i16>* %tmp7, align 2
-; CHECK-NEXT:    %tmp13 = getelementptr i16, i16* %arg, i64 %5
-; CHECK-NEXT:    %tmp14 = bitcast i16* %tmp13 to <8 x i16>*
-; CHECK-NEXT:    store <8 x i16> %tmp8, <8 x i16>* %tmp14, align 2
-; CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
-; CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %3
-; CHECK-NEXT:    br i1 %exitcond, label %bb19, label %bb3
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[BB3]] ], [ 0, [[BB:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[INDVAR]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 1
+; CHECK-NEXT:    [[INST5:%.*]] = shl nuw i64 [[TMP5]], 1
+; CHECK-NEXT:    [[INST6:%.*]] = getelementptr i8, ptr [[ARG1]], i64 [[INST5]]
+; CHECK-NEXT:    [[INST8:%.*]] = load <8 x i16>, ptr [[INST6]], align 2
+; CHECK-NEXT:    [[INST13:%.*]] = getelementptr i16, ptr [[ARG]], i64 [[TMP5]]
+; CHECK-NEXT:    store <8 x i16> [[INST8]], ptr [[INST13]], align 2
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVAR]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[BB19:%.*]], label [[BB3]]
+; CHECK:       bb19:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb3
 
 bb3:                                              ; preds = %bb3, %bb
-  %tmp = phi i64 [ 1, %bb ], [ %tmp17, %bb3 ]
-  %tmp4 = add nuw i64 %tmp, 8
-  %tmp5 = shl nuw i64 %tmp, 1
-  %tmp6 = getelementptr i8, i8* %arg1, i64 %tmp5
-  %tmp7 = bitcast i8* %tmp6 to <8 x i16>*
-  %tmp8 = load <8 x i16>, <8 x i16>* %tmp7, align 2
-  %tmp9 = shl i64 %tmp4, 1
-  %tmp10 = getelementptr i8, i8* %arg1, i64 %tmp9
-  %tmp11 = bitcast i8* %tmp10 to <8 x i16>*
-  %tmp12 = load <8 x i16>, <8 x i16>* %tmp11, align 2
-  %tmp13 = getelementptr i16, i16* %arg, i64 %tmp
-  %tmp14 = bitcast i16* %tmp13 to <8 x i16>*
-  store <8 x i16> %tmp8, <8 x i16>* %tmp14, align 2
-  %tmp15 = getelementptr i16, i16* %arg, i64 %tmp4
-  %tmp16 = bitcast i16* %tmp15 to <8 x i16>*
-  store <8 x i16> %tmp12, <8 x i16>* %tmp16, align 2
-  %tmp17 = add nuw nsw i64 %tmp, 16
-  %tmp18 = icmp eq i64 %tmp17, %arg2
-  br i1 %tmp18, label %bb19, label %bb3
+  %inst = phi i64 [ 1, %bb ], [ %inst17, %bb3 ]
+  %inst4 = add nuw i64 %inst, 8
+  %inst5 = shl nuw i64 %inst, 1
+  %inst6 = getelementptr i8, ptr %arg1, i64 %inst5
+  %inst8 = load <8 x i16>, ptr %inst6, align 2
+  %inst9 = shl i64 %inst4, 1
+  %inst10 = getelementptr i8, ptr %arg1, i64 %inst9
+  %inst12 = load <8 x i16>, ptr %inst10, align 2
+  %inst13 = getelementptr i16, ptr %arg, i64 %inst
+  store <8 x i16> %inst8, ptr %inst13, align 2
+  %inst15 = getelementptr i16, ptr %arg, i64 %inst4
+  store <8 x i16> %inst12, ptr %inst15, align 2
+  %inst17 = add nuw nsw i64 %inst, 16
+  %inst18 = icmp eq i64 %inst17, %arg2
+  br i1 %inst18, label %bb19, label %bb3
 
 bb19:                                             ; preds = %bb3
   ret void
 }
 
-define void @bad_step(i32* nocapture readnone %x) #0 {
+define void @bad_step(ptr nocapture readnone %x) #0 {
+; CHECK-LABEL: define void @bad_step
+; CHECK-SAME: (ptr nocapture readnone [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD3:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @foo(i32 [[I_08]]) #[[ATTR1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[I_08]], 2
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @foo(i32 [[ADD]]) #[[ATTR1]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[I_08]], 3
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call i32 @foo(i32 [[ADD2]]) #[[ATTR1]]
+; CHECK-NEXT:    [[ADD3]] = add nsw i32 [[I_08]], 6
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp sge i32 [[ADD3]], 500
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -800,10 +931,6 @@ for.body:                                         ; preds = %for.body, %entry
   %exitcond = icmp sge i32 %add3, 500
   br i1 %exitcond, label %for.end, label %for.body
 
-; CHECK-LABEL: @bad_step
-; CHECK: %add = add nsw i32 %i.08, 2
-; CHECK: %add2 = add nsw i32 %i.08, 3
-; CHECK: %add3 = add nsw i32 %i.08, 6
 
 for.end:                                          ; preds = %for.body
   ret void
@@ -813,7 +940,21 @@ for.end:                                          ; preds = %for.body
 @b = external global [512 x [4 x i64]], align 16
 
 define void @ptr_step_crash() {
-; CHECK-LABEL: @ptr_step_crash(
+; CHECK-LABEL: define void @ptr_step_crash() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY42_3:%.*]]
+; CHECK:       for.body42.3:
+; CHECK-NEXT:    [[K_2207_3:%.*]] = phi i32 [ -512, [[ENTRY:%.*]] ], [ [[INC63_3:%.*]], [[FOR_BODY42_3]] ]
+; CHECK-NEXT:    [[SUB46_3:%.*]] = add nsw i32 [[K_2207_3]], 512
+; CHECK-NEXT:    [[IDXPROM47_3:%.*]] = zext i32 [[SUB46_3]] to i64
+; CHECK-NEXT:    [[ARRAYIDX48_3:%.*]] = getelementptr inbounds [2 x [512 x i64]], ptr @a, i64 0, i64 0, i64 [[IDXPROM47_3]]
+; CHECK-NEXT:    [[ARRAYIDX55_3:%.*]] = getelementptr inbounds [512 x [4 x i64]], ptr @b, i64 0, i64 [[IDXPROM47_3]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX55_3]], align 8
+; CHECK-NEXT:    [[INC63_3]] = add nsw i32 [[K_2207_3]], 1
+; CHECK-NEXT:    br i1 true, label [[FOR_INC65_3:%.*]], label [[FOR_BODY42_3]]
+; CHECK:       for.inc65.3:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body42.3
 
@@ -821,9 +962,9 @@ for.body42.3:                                     ; preds = %for.body42.3, %entr
   %k.2207.3 = phi i32 [ -512, %entry ], [ %inc63.3, %for.body42.3 ]
   %sub46.3 = add nsw i32 %k.2207.3, 512
   %idxprom47.3 = zext i32 %sub46.3 to i64
-  %arrayidx48.3 = getelementptr inbounds [2 x [512 x i64]], [2 x [512 x i64]]* @a, i64 0, i64 0, i64 %idxprom47.3
-  %arrayidx55.3 = getelementptr inbounds [512 x [4 x i64]], [512 x [4 x i64]]* @b, i64 0, i64 %idxprom47.3, i64 3
-  %0 = load i64, i64* %arrayidx55.3, align 8
+  %arrayidx48.3 = getelementptr inbounds [2 x [512 x i64]], ptr @a, i64 0, i64 0, i64 %idxprom47.3
+  %arrayidx55.3 = getelementptr inbounds [512 x [4 x i64]], ptr @b, i64 0, i64 %idxprom47.3, i64 3
+  %0 = load i64, ptr %arrayidx55.3, align 8
   %inc63.3 = add nsw i32 %k.2207.3, 1
   br i1 undef, label %for.inc65.3, label %for.body42.3
 
diff --git a/llvm/test/Transforms/LoopReroll/extra_instr.ll b/llvm/test/Transforms/LoopReroll/extra_instr.ll
index 84ebec10048aa..3114463a3e041 100644
--- a/llvm/test/Transforms/LoopReroll/extra_instr.ll
+++ b/llvm/test/Transforms/LoopReroll/extra_instr.ll
@@ -1,18 +1,33 @@
-; RUN: opt -opaque-pointers=0 -S -passes=loop-reroll   %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -passes=loop-reroll   %s | FileCheck %s
 target triple = "aarch64--linux-gnu"
 
-define void @rerollable1([2 x i32]* nocapture %a) {
+define void @rerollable1(ptr nocapture %a) {
+; CHECK-LABEL: define void @rerollable1
+; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 160
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 80
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[VALUE0:%.*]] = load i32, ptr [[SCEVGEP1]], align 4
+; CHECK-NEXT:    store i32 [[VALUE0]], ptr [[SCEVGEP]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND2:%.*]] = icmp eq i64 [[IV]], 9
+; CHECK-NEXT:    br i1 [[EXITCOND2]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
 loop:
 
-; CHECK-LABEL: loop:
-; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr [2 x i32], [2 x i32]* %a, i64 20, i64 %iv
-; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr [2 x i32], [2 x i32]* %a, i64 10, i64 %iv
-; CHECK-NEXT:    [[VALUE:%.*]] = load i32, i32* [[SCEVGEP1]], align 4
-; CHECK-NEXT:    store i32 [[VALUE]], i32* [[SCEVGEP2]], align 4
 
   ; base instruction
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
@@ -24,16 +39,16 @@ loop:
   %plus10 = add nuw nsw i64 %iv, 10
 
   ; root instruction 0
-  %ldptr0 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus20, i64 0
-  %value0 = load i32, i32* %ldptr0, align 4
-  %stptr0 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus10, i64 0
-  store i32 %value0, i32* %stptr0, align 4
+  %ldptr0 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus20, i64 0
+  %value0 = load i32, ptr %ldptr0, align 4
+  %stptr0 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus10, i64 0
+  store i32 %value0, ptr %stptr0, align 4
 
   ; root instruction 1
-  %ldptr1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus20, i64 1
-  %value1 = load i32, i32* %ldptr1, align 4
-  %stptr1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus10, i64 1
-  store i32 %value1, i32* %stptr1, align 4
+  %ldptr1 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus20, i64 1
+  %value1 = load i32, ptr %ldptr1, align 4
+  %stptr1 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus10, i64 1
+  store i32 %value1, ptr %stptr1, align 4
 
   ; loop-increment
   %iv.next = add nuw nsw i64 %iv, 1
@@ -46,39 +61,59 @@ exit:
   ret void
 }
 
-define void @unrerollable1([2 x i32]* nocapture %a) {
+define void @unrerollable1(ptr nocapture %a) {
+; CHECK-LABEL: define void @unrerollable1
+; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STPTRX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[IV]], i64 0
+; CHECK-NEXT:    store i32 999, ptr [[STPTRX]], align 4
+; CHECK-NEXT:    [[PLUS20:%.*]] = add nuw nsw i64 [[IV]], 20
+; CHECK-NEXT:    [[PLUS10:%.*]] = add nuw nsw i64 [[IV]], 10
+; CHECK-NEXT:    [[LDPTR0:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS20]], i64 0
+; CHECK-NEXT:    [[VALUE0:%.*]] = load i32, ptr [[LDPTR0]], align 4
+; CHECK-NEXT:    [[STPTR0:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS10]], i64 0
+; CHECK-NEXT:    store i32 [[VALUE0]], ptr [[STPTR0]], align 4
+; CHECK-NEXT:    [[LDPTR1:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS20]], i64 1
+; CHECK-NEXT:    [[VALUE1:%.*]] = load i32, ptr [[LDPTR1]], align 4
+; CHECK-NEXT:    [[STPTR1:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS10]], i64 1
+; CHECK-NEXT:    store i32 [[VALUE1]], ptr [[STPTR1]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
 loop:
 
-; CHECK-LABEL: loop:
-; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT:    %stptrx = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %iv, i64 0
-; CHECK-NEXT:    store i32 999, i32* %stptrx, align 4
 
   ; base instruction
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
 
   ; unrerollable instructions using %iv
-  %stptrx = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %iv, i64 0
-  store i32 999, i32* %stptrx, align 4
+  %stptrx = getelementptr inbounds [2 x i32], ptr %a, i64 %iv, i64 0
+  store i32 999, ptr %stptrx, align 4
 
   ; extra simple arithmetic operations, used by root instructions
   %plus20 = add nuw nsw i64 %iv, 20
   %plus10 = add nuw nsw i64 %iv, 10
 
   ; root instruction 0
-  %ldptr0 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus20, i64 0
-  %value0 = load i32, i32* %ldptr0, align 4
-  %stptr0 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus10, i64 0
-  store i32 %value0, i32* %stptr0, align 4
+  %ldptr0 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus20, i64 0
+  %value0 = load i32, ptr %ldptr0, align 4
+  %stptr0 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus10, i64 0
+  store i32 %value0, ptr %stptr0, align 4
 
   ; root instruction 1
-  %ldptr1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus20, i64 1
-  %value1 = load i32, i32* %ldptr1, align 4
-  %stptr1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus10, i64 1
-  store i32 %value1, i32* %stptr1, align 4
+  %ldptr1 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus20, i64 1
+  %value1 = load i32, ptr %ldptr1, align 4
+  %stptr1 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus10, i64 1
+  store i32 %value1, ptr %stptr1, align 4
 
   ; loop-increment
   %iv.next = add nuw nsw i64 %iv, 1
@@ -91,17 +126,36 @@ exit:
   ret void
 }
 
-define void @unrerollable2([2 x i32]* nocapture %a) {
+define void @unrerollable2(ptr nocapture %a) {
+; CHECK-LABEL: define void @unrerollable2
+; CHECK-SAME: (ptr nocapture [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[STPTRX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[IV_NEXT]], i64 0
+; CHECK-NEXT:    store i32 999, ptr [[STPTRX]], align 4
+; CHECK-NEXT:    [[PLUS20:%.*]] = add nuw nsw i64 [[IV]], 20
+; CHECK-NEXT:    [[PLUS10:%.*]] = add nuw nsw i64 [[IV]], 10
+; CHECK-NEXT:    [[LDPTR0:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS20]], i64 0
+; CHECK-NEXT:    [[VALUE0:%.*]] = load i32, ptr [[LDPTR0]], align 4
+; CHECK-NEXT:    [[STPTR0:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS10]], i64 0
+; CHECK-NEXT:    store i32 [[VALUE0]], ptr [[STPTR0]], align 4
+; CHECK-NEXT:    [[LDPTR1:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS20]], i64 1
+; CHECK-NEXT:    [[VALUE1:%.*]] = load i32, ptr [[LDPTR1]], align 4
+; CHECK-NEXT:    [[STPTR1:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 [[PLUS10]], i64 1
+; CHECK-NEXT:    store i32 [[VALUE1]], ptr [[STPTR1]], align 4
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
 loop:
 
-; CHECK-LABEL: loop:
-; CHECK-NEXT:    %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT:    %iv.next = add nuw nsw i64 %iv, 1
-; CHECK-NEXT:    %stptrx = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %iv.next, i64 0
-; CHECK-NEXT:    store i32 999, i32* %stptrx, align 4
 
   ; base instruction
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
@@ -110,24 +164,24 @@ loop:
   %iv.next = add nuw nsw i64 %iv, 1
 
   ; unrerollable instructions using %iv.next
-  %stptrx = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %iv.next, i64 0
-  store i32 999, i32* %stptrx, align 4
+  %stptrx = getelementptr inbounds [2 x i32], ptr %a, i64 %iv.next, i64 0
+  store i32 999, ptr %stptrx, align 4
 
   ; extra simple arithmetic operations, used by root instructions
   %plus20 = add nuw nsw i64 %iv, 20
   %plus10 = add nuw nsw i64 %iv, 10
 
   ; root instruction 0
-  %ldptr0 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus20, i64 0
-  %value0 = load i32, i32* %ldptr0, align 4
-  %stptr0 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus10, i64 0
-  store i32 %value0, i32* %stptr0, align 4
+  %ldptr0 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus20, i64 0
+  %value0 = load i32, ptr %ldptr0, align 4
+  %stptr0 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus10, i64 0
+  store i32 %value0, ptr %stptr0, align 4
 
   ; root instruction 1
-  %ldptr1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus20, i64 1
-  %value1 = load i32, i32* %ldptr1, align 4
-  %stptr1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 %plus10, i64 1
-  store i32 %value1, i32* %stptr1, align 4
+  %ldptr1 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus20, i64 1
+  %value1 = load i32, ptr %ldptr1, align 4
+  %stptr1 = getelementptr inbounds [2 x i32], ptr %a, i64 %plus10, i64 1
+  store i32 %value1, ptr %stptr1, align 4
 
   ; latch
   %exitcond = icmp eq i64 %iv.next, 5
@@ -138,15 +192,28 @@ exit:
 }
 
 define dso_local void @rerollable2() {
+; CHECK-LABEL: define dso_local void @rerollable2() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[IV]], 24
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[IV]], 20
+; CHECK-NEXT:    [[IV_SCALED_DIV5:%.*]] = udiv i32 [[TMP1]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_DIV5]])
+; CHECK-NEXT:    [[IV_SCALED_ADD4_DIV5:%.*]] = udiv i32 [[TMP0]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_ADD4_DIV5]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
 loop:
 
-; CHECK-LABEL: loop:
-; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT:    {{%.*}} = add i32 %iv, {{20|24}}
-; CHECK-NEXT:    {{%.*}} = add i32 %iv, {{20|24}}
 
   ; induction variable
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
@@ -200,17 +267,43 @@ exit:
 }
 
 define dso_local void @unrerollable3() {
+; CHECK-LABEL: define dso_local void @unrerollable3() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_MUL3:%.*]] = mul nuw nsw i32 [[IV]], 3
+; CHECK-NEXT:    [[IV_SCALED:%.*]] = add nuw nsw i32 [[IV_MUL3]], 20
+; CHECK-NEXT:    [[IV_MUL7:%.*]] = mul nuw nsw i32 [[IV]], 7
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_MUL7]])
+; CHECK-NEXT:    [[IV_SCALED_DIV5:%.*]] = udiv i32 [[IV_SCALED]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_DIV5]])
+; CHECK-NEXT:    [[IV_SCALED_ADD1:%.*]] = add nuw nsw i32 [[IV_SCALED]], 1
+; CHECK-NEXT:    [[IV_SCALED_ADD1_DIV5:%.*]] = udiv i32 [[IV_SCALED_ADD1]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_ADD1_DIV5]])
+; CHECK-NEXT:    [[IV_SCALED_ADD2:%.*]] = add nuw nsw i32 [[IV_SCALED]], 2
+; CHECK-NEXT:    [[IV_SCALED_ADD2_DIV5:%.*]] = udiv i32 [[IV_SCALED_ADD2]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_ADD2_DIV5]])
+; CHECK-NEXT:    [[IV_SCALED_ADD4:%.*]] = add nuw nsw i32 [[IV_SCALED]], 4
+; CHECK-NEXT:    [[IV_SCALED_ADD4_DIV5:%.*]] = udiv i32 [[IV_SCALED_ADD4]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_ADD4_DIV5]])
+; CHECK-NEXT:    [[IV_SCALED_ADD5:%.*]] = add nuw nsw i32 [[IV_SCALED]], 5
+; CHECK-NEXT:    [[IV_SCALED_ADD5_DIV5:%.*]] = udiv i32 [[IV_SCALED_ADD5]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_ADD5_DIV5]])
+; CHECK-NEXT:    [[IV_SCALED_ADD6:%.*]] = add nuw nsw i32 [[IV_SCALED]], 6
+; CHECK-NEXT:    [[IV_SCALED_ADD6_DIV5:%.*]] = udiv i32 [[IV_SCALED_ADD6]], 5
+; CHECK-NEXT:    tail call void @bar(i32 [[IV_SCALED_ADD6_DIV5]])
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], 3
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %loop
 
 loop:
 
-; CHECK-LABEL: loop:
-; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEXT:    %iv.mul3 = mul nuw nsw i32 %iv, 3
-; CHECK-NEXT:    %iv.scaled = add nuw nsw i32 %iv.mul3, 20
-; CHECK-NEXT:    %iv.mul7 = mul nuw nsw i32 %iv, 7
-; CHECK-NEXT:    tail call void @bar(i32 %iv.mul7)
 
   ; induction variable
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
diff --git a/llvm/test/Transforms/LoopReroll/ptrindvar.ll b/llvm/test/Transforms/LoopReroll/ptrindvar.ll
index a61724a07c617..90f6353197b24 100644
--- a/llvm/test/Transforms/LoopReroll/ptrindvar.ll
+++ b/llvm/test/Transforms/LoopReroll/ptrindvar.ll
@@ -1,34 +1,57 @@
-; RUN: opt -opaque-pointers=0 -S -passes=loop-reroll   %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -passes=loop-reroll   %s | FileCheck %s
 target triple = "aarch64--linux-gnu"
 
-define i32 @test(i32* readonly %buf, i32* readnone %end) #0 {
+define i32 @test(ptr readonly %buf, ptr readnone %end) #0 {
+; CHECK-LABEL: define i32 @test
+; CHECK-SAME: (ptr readonly [[BUF:%.*]], ptr readnone [[END:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BUF2:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT:    [[CMP_9:%.*]] = icmp eq ptr [[BUF]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP_9]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CHECK:       while.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[BUF2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[S_011:%.*]] = phi i32 [ [[ADD:%.*]], [[WHILE_BODY]] ], [ undef, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[INDVAR]], 2
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[BUF]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[SCEVGEP]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP6]], [[S_011]]
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVAR]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CHECK:       while.end.loopexit:
+; CHECK-NEXT:    [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD]], [[WHILE_BODY]] ]
+; CHECK-NEXT:    br label [[WHILE_END]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[S_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[ADD2_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_0_LCSSA]]
+;
 entry:
-  %cmp.9 = icmp eq i32* %buf, %end
+  %cmp.9 = icmp eq ptr %buf, %end
   br i1 %cmp.9, label %while.end, label %while.body.preheader
 
 while.body.preheader:
   br label %while.body
 
 while.body:
-;CHECK-LABEL: while.body:
-;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %while.body.preheader ]
-;CHECK-NEXT:    %S.011 = phi i32 [ %add, %while.body ], [ undef, %while.body.preheader ]
-;CHECK-NEXT:    %scevgep = getelementptr i32, i32* %buf, i64 %indvar
-;CHECK-NEXT:    %5 = load i32, i32* %scevgep, align 4
-;CHECK-NEXT:    %add = add nsw i32 %5, %S.011
-;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
-;CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %4
-;CHECK-NEXT:    br i1 %exitcond, label %while.end.loopexit, label %while.body
 
   %S.011 = phi i32 [ %add2, %while.body ], [ undef, %while.body.preheader ]
-  %buf.addr.010 = phi i32* [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
-  %0 = load i32, i32* %buf.addr.010, align 4
+  %buf.addr.010 = phi ptr [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
+  %0 = load i32, ptr %buf.addr.010, align 4
   %add = add nsw i32 %0, %S.011
-  %arrayidx1 = getelementptr inbounds i32, i32* %buf.addr.010, i64 1
-  %1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx1 = getelementptr inbounds i32, ptr %buf.addr.010, i64 1
+  %1 = load i32, ptr %arrayidx1, align 4
   %add2 = add nsw i32 %add, %1
-  %add.ptr = getelementptr inbounds i32, i32* %buf.addr.010, i64 2
-  %cmp = icmp eq i32* %add.ptr, %end
+  %add.ptr = getelementptr inbounds i32, ptr %buf.addr.010, i64 2
+  %cmp = icmp eq ptr %add.ptr, %end
   br i1 %cmp, label %while.end.loopexit, label %while.body
 
 while.end.loopexit:
@@ -40,35 +63,56 @@ while.end:
   ret i32 %S.0.lcssa
 }
 
-define i32 @test2(i32* readonly %buf, i32* readnone %end) #0 {
+define i32 @test2(ptr readonly %buf, ptr readnone %end) #0 {
+; CHECK-LABEL: define i32 @test2
+; CHECK-SAME: (ptr readonly [[BUF:%.*]], ptr readnone [[END:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[END2:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT:    [[BUF1:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[CMP_9:%.*]] = icmp eq ptr [[BUF]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP_9]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CHECK:       while.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[BUF1]], -8
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[END2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[S_011:%.*]] = phi i32 [ [[ADD:%.*]], [[WHILE_BODY]] ], [ undef, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw i64 [[INDVAR]], -4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[BUF]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[SCEVGEP]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP6]], [[S_011]]
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVAR]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CHECK:       while.end.loopexit:
+; CHECK-NEXT:    [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD]], [[WHILE_BODY]] ]
+; CHECK-NEXT:    br label [[WHILE_END]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[S_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[ADD2_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_0_LCSSA]]
+;
 entry:
-  %cmp.9 = icmp eq i32* %buf, %end
+  %cmp.9 = icmp eq ptr %buf, %end
   br i1 %cmp.9, label %while.end, label %while.body.preheader
 
 while.body.preheader:
   br label %while.body
 
 while.body:
-;CHECK-LABEL: while.body:
-;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %while.body.preheader ]
-;CHECK-NEXT:    %S.011 = phi i32 [ %add, %while.body ], [ undef, %while.body.preheader ]
-;CHECK-NEXT:    %5 = mul nsw i64 %indvar, -1
-;CHECK-NEXT:    %scevgep = getelementptr i32, i32* %buf, i64 %5
-;CHECK-NEXT:    %6 = load i32, i32* %scevgep, align 4
-;CHECK-NEXT:    %add = add nsw i32 %6, %S.011
-;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
-;CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %4
-;CHECK-NEXT:    br i1 %exitcond, label %while.end.loopexit, label %while.body
 
   %S.011 = phi i32 [ %add2, %while.body ], [ undef, %while.body.preheader ]
-  %buf.addr.010 = phi i32* [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
-  %0 = load i32, i32* %buf.addr.010, align 4
+  %buf.addr.010 = phi ptr [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
+  %0 = load i32, ptr %buf.addr.010, align 4
   %add = add nsw i32 %0, %S.011
-  %arrayidx1 = getelementptr inbounds i32, i32* %buf.addr.010, i64 -1
-  %1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx1 = getelementptr inbounds i32, ptr %buf.addr.010, i64 -1
+  %1 = load i32, ptr %arrayidx1, align 4
   %add2 = add nsw i32 %add, %1
-  %add.ptr = getelementptr inbounds i32, i32* %buf.addr.010, i64 -2
-  %cmp = icmp eq i32* %add.ptr, %end
+  %add.ptr = getelementptr inbounds i32, ptr %buf.addr.010, i64 -2
+  %cmp = icmp eq ptr %add.ptr, %end
   br i1 %cmp, label %while.end.loopexit, label %while.body
 
 while.end.loopexit:
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll
index b23ef8f4c2185..f036a0fc2c76a 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -mtriple=amdgcn-- -mcpu=bonaire -loop-reduce < %s | FileCheck -check-prefix=OPT %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; Make sure the pointer / address space of AtomicRMW is considered
 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
index 40306e7998927..f51595f27b501 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
@@ -3,7 +3,7 @@
 ; Test that loops with different maximum offsets for different address
 ; spaces are correctly handled.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 ; OPT-LABEL: @test_global_addressing_loop_uniform_index_max_offset_i32(
 ; OPT: .lr.ph.preheader:
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-invalid-ptr-extend.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-invalid-ptr-extend.ll
index 5e596f08380a6..2d365ef527e93 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-invalid-ptr-extend.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-invalid-ptr-extend.ll
@@ -5,7 +5,7 @@
 ; Test that LSR does not attempt to extend a pointer type to an integer type,
 ; which causes a SCEV analysis assertion.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 
 target triple = "amdgcn-amd-amdhsa"
 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll
index 2c38c12e307bf..b776a6f0193d4 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll
@@ -4,7 +4,7 @@
 ; Test various conditions where OptimizeLoopTermCond doesn't look at a
 ; memory instruction use and fails to find the address space.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define amdgpu_kernel void @local_cmp_user(i32 %arg0) nounwind {
 ; CHECK-LABEL: @local_cmp_user(
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll
index 99432cc86691e..cd161fea9c148 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll
@@ -3,7 +3,7 @@
 ; Test for assert resulting from inconsistent isLegalAddressingMode
 ; answers when the address space was dropped from the query.
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 %0 = type { i32, double, i32, float }
 
diff --git a/llvm/test/Transforms/LoopUnroll/X86/pr46430-inseltpoison.ll b/llvm/test/Transforms/LoopUnroll/X86/pr46430-inseltpoison.ll
index 93ee40420254e..e290a3c23a8aa 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/pr46430-inseltpoison.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/pr46430-inseltpoison.ll
@@ -6,7 +6,7 @@ define void @g() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[F_EXIT:%.*]]
 ; CHECK:       f.exit:
-; CHECK-NEXT:    [[RDX_SHUF9_I:%.*]] = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF9_I:%.*]] = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX10_I:%.*]] = xor <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, [[RDX_SHUF9_I]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[BIN_RDX10_I]], i32 0
 ; CHECK-NEXT:    br label [[F_EXIT]]
diff --git a/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll b/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll
index 6e835ed1f64f8..f22243170e909 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/pr46430.ll
@@ -6,7 +6,7 @@ define void @g() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[F_EXIT:%.*]]
 ; CHECK:       f.exit:
-; CHECK-NEXT:    [[RDX_SHUF9_I:%.*]] = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF9_I:%.*]] = shufflevector <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, <4 x i16> undef, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX10_I:%.*]] = xor <4 x i16> <i16 1, i16 0, i16 undef, i16 undef>, [[RDX_SHUF9_I]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[BIN_RDX10_I]], i32 0
 ; CHECK-NEXT:    br label [[F_EXIT]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll
index c77d0bd4e8cf1..87674f611251c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-store-of-first-order-recurrence.ll
@@ -17,7 +17,7 @@ define void @interleaved_store_first_order_recurrence(ptr noalias %src, ptr %dst
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 -2
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i32> zeroinitializer, <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> [[TMP10]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <12 x i32> [[TMP11]], <12 x i32> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
new file mode 100644
index 0000000000000..cff60b10d0187
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -S -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; REQUIRES: asserts
+; XFAIL: *
+
+; Test cases for PR60831.
+
+define void @test_invar_gep(ptr %dst) #0 {
+; CHECK-LABEL: @test_invar_gep(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <vscale x 4 x i64> [[TMP5]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP7:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 0
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP13]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = sub i32 [[TMP14]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i64> [[TMP7]], i32 [[TMP15]]
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP12]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP_INVAR:%.*]] = getelementptr i8, ptr [[DST]], i64 0
+; CHECK-NEXT:    store i64 [[IV]], ptr [[GEP_INVAR]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.invar = getelementptr i8, ptr %dst, i64 0
+  store i64 %iv, ptr %gep.invar, align 1
+  %iv.next = add nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+define void @test_loop2(i64 %n, ptr %dst) {
+; CHECK-LABEL: @test_loop2(
+; CHECK-NEXT:  iter.check:
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = sub nsw i64 [[N:%.*]], [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sub nsw i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw i64 [[N]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = sub nsw i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = sub nsw i64 [[N]], [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = sub nsw i64 [[N]], [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sub nsw i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub nsw i64 [[N]], [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sub nsw i64 [[N]], [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = sub nsw i64 [[N]], [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = sub nsw i64 [[N]], [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = sub nsw i64 [[N]], [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = sub nsw i64 [[N]], [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = sub nsw i64 [[N]], [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = sub nsw i64 [[N]], [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = sub nsw i64 [[N]], [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <16 x i64> poison, i64 [[TMP16]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <16 x i64> [[TMP32]], i64 [[TMP17]], i32 1
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <16 x i64> [[TMP33]], i64 [[TMP18]], i32 2
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <16 x i64> [[TMP34]], i64 [[TMP19]], i32 3
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <16 x i64> [[TMP35]], i64 [[TMP20]], i32 4
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <16 x i64> [[TMP36]], i64 [[TMP21]], i32 5
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <16 x i64> [[TMP37]], i64 [[TMP22]], i32 6
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <16 x i64> [[TMP38]], i64 [[TMP23]], i32 7
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <16 x i64> [[TMP39]], i64 [[TMP24]], i32 8
+; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <16 x i64> [[TMP40]], i64 [[TMP25]], i32 9
+; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <16 x i64> [[TMP41]], i64 [[TMP26]], i32 10
+; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <16 x i64> [[TMP42]], i64 [[TMP27]], i32 11
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <16 x i64> [[TMP43]], i64 [[TMP28]], i32 12
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <16 x i64> [[TMP44]], i64 [[TMP29]], i32 13
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <16 x i64> [[TMP45]], i64 [[TMP30]], i32 14
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <16 x i64> [[TMP46]], i64 [[TMP31]], i32 15
+; CHECK-NEXT:    [[TMP48:%.*]] = trunc <16 x i64> [[TMP47]] to <16 x i8>
+; CHECK-NEXT:    [[TMP49:%.*]] = add i64 [[TMP0]], [[TMP16]]
+; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP49]]
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <16 x i8> [[TMP48]], i32 15
+; CHECK-NEXT:    store i8 [[TMP51]], ptr [[TMP50]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 992
+; CHECK-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1001, 992
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX2]], 0
+; CHECK-NEXT:    [[TMP54:%.*]] = add i64 [[INDEX2]], 1
+; CHECK-NEXT:    [[TMP55:%.*]] = add i64 [[INDEX2]], 2
+; CHECK-NEXT:    [[TMP56:%.*]] = add i64 [[INDEX2]], 3
+; CHECK-NEXT:    [[TMP57:%.*]] = add i64 [[INDEX2]], 4
+; CHECK-NEXT:    [[TMP58:%.*]] = add i64 [[INDEX2]], 5
+; CHECK-NEXT:    [[TMP59:%.*]] = add i64 [[INDEX2]], 6
+; CHECK-NEXT:    [[TMP60:%.*]] = add i64 [[INDEX2]], 7
+; CHECK-NEXT:    [[TMP61:%.*]] = sub nsw i64 [[N]], [[TMP53]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sub nsw i64 [[N]], [[TMP54]]
+; CHECK-NEXT:    [[TMP63:%.*]] = sub nsw i64 [[N]], [[TMP55]]
+; CHECK-NEXT:    [[TMP64:%.*]] = sub nsw i64 [[N]], [[TMP56]]
+; CHECK-NEXT:    [[TMP65:%.*]] = sub nsw i64 [[N]], [[TMP57]]
+; CHECK-NEXT:    [[TMP66:%.*]] = sub nsw i64 [[N]], [[TMP58]]
+; CHECK-NEXT:    [[TMP67:%.*]] = sub nsw i64 [[N]], [[TMP59]]
+; CHECK-NEXT:    [[TMP68:%.*]] = sub nsw i64 [[N]], [[TMP60]]
+; CHECK-NEXT:    [[TMP69:%.*]] = insertelement <8 x i64> poison, i64 [[TMP61]], i32 0
+; CHECK-NEXT:    [[TMP70:%.*]] = insertelement <8 x i64> [[TMP69]], i64 [[TMP62]], i32 1
+; CHECK-NEXT:    [[TMP71:%.*]] = insertelement <8 x i64> [[TMP70]], i64 [[TMP63]], i32 2
+; CHECK-NEXT:    [[TMP72:%.*]] = insertelement <8 x i64> [[TMP71]], i64 [[TMP64]], i32 3
+; CHECK-NEXT:    [[TMP73:%.*]] = insertelement <8 x i64> [[TMP72]], i64 [[TMP65]], i32 4
+; CHECK-NEXT:    [[TMP74:%.*]] = insertelement <8 x i64> [[TMP73]], i64 [[TMP66]], i32 5
+; CHECK-NEXT:    [[TMP75:%.*]] = insertelement <8 x i64> [[TMP74]], i64 [[TMP67]], i32 6
+; CHECK-NEXT:    [[TMP76:%.*]] = insertelement <8 x i64> [[TMP75]], i64 [[TMP68]], i32 7
+; CHECK-NEXT:    [[TMP77:%.*]] = trunc <8 x i64> [[TMP76]] to <8 x i8>
+; CHECK-NEXT:    [[TMP78:%.*]] = add i64 [[TMP53]], [[TMP61]]
+; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <8 x i8> [[TMP77]], i32 7
+; CHECK-NEXT:    store i8 [[TMP80]], ptr [[TMP79]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX2]], 8
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1000
+; CHECK-NEXT:    br i1 [[TMP81]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N1:%.*]] = icmp eq i64 1001, 1000
+; CHECK-NEXT:    br i1 [[CMP_N1]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SUB_N:%.*]] = sub nsw i64 [[N]], [[IV]]
+; CHECK-NEXT:    [[SUB_N_TRUNC:%.*]] = trunc i64 [[SUB_N]] to i8
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[IV]], [[SUB_N]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[ADD]]
+; CHECK-NEXT:    store i8 [[SUB_N_TRUNC]], ptr [[GEP]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[C:%.*]] = icmp sle i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %sub.n = sub nsw i64 %n, %iv
+  %sub.n.trunc = trunc i64 %sub.n to i8
+  %add = add i64 %iv, %sub.n
+  %gep = getelementptr i8, ptr %dst, i64 %add
+  store i8 %sub.n.trunc, ptr %gep, align 1
+  %iv.next = add nsw i64 %iv, 1
+  %c = icmp sle i64 %iv.next, 1000
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-features"="+neon,+sve" vscale_range(1, 16) }
+
+!0 = distinct !{!0, !1, !2, !3, !4, !5}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.vectorize.enable", i1 true}
+!5 = !{!"llvm.loop.interleave.count", i32 1}
+
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
index 87347ef1162a9..269562fa70549 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
@@ -95,7 +95,7 @@ for.body:
   %conv = sitofp i8 %i.08 to float
   %add = fadd float %s.09, %conv
   %inc = add nuw nsw i8 %i.08, 1
-  %exitcond.not = icmp eq i8 %inc, 12345
+  %exitcond.not = icmp eq i8 %inc, 241
   br i1 %exitcond.not, label %for.end, label %for.body
 
 for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index 5ad0994d02e0b..d3297d8c29d3a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -10,7 +10,11 @@ target triple = "aarch64-unknown-linux-gnu"
 
 ; VPLANS-LABEL: Checking a loop in 'simple_memset'
 ; VPLANS:      VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {
-; VPLANS-NEXT: Live-in vp<[[TC:%[0-9]+]]> = original trip-count
+; VPLANS-NEXT: vp<[[TC:%[0-9]+]]> = original trip-count
+; VPLANS-EMPTY:
+; VPLANS-NEXT: ph:
+; VPLANS-NEXT:  EMIT vp<[[TC]]> = EXPAND SCEV (1 umax %n)
+; VPLANS-NEXT: No successors
 ; VPLANS-EMPTY:
 ; VPLANS-NEXT: vector.ph:
 ; VPLANS-NEXT:   EMIT vp<[[VF:%[0-9]+]]> = VF * Part +  ir<0>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index bcc53f05a47bb..e02b6bf84947f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -1,22 +1,22 @@
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=disabled+simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV
-; RUN: opt -opaque-pointers=0 < %s -passes=loop-vectorize -sve-tail-folding=reductions -S | FileCheck %s -check-prefix=CHECK-TF-ONLYRED
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=disabled -S | FileCheck %s -check-prefix=CHECK-NOTF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=default -S | FileCheck %s -check-prefix=CHECK-NOTF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all -S | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=disabled+simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all+noreductions -S | FileCheck %s -check-prefix=CHECK-TF-NORED
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all+norecurrences -S | FileCheck %s -check-prefix=CHECK-TF-NOREC
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=all+noreverse -S | FileCheck %s -check-prefix=CHECK-TF-NOREV
+; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding=reductions -S | FileCheck %s -check-prefix=CHECK-TF-ONLYRED
 
 target triple = "aarch64-unknown-linux-gnu"
 
-define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 {
+define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NOTF-LABEL: @simple_memset(
 ; CHECK-NOTF:       vector.ph:
 ; CHECK-NOTF:         %[[INSERT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %val, i64 0
 ; CHECK-NOTF:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NOTF:       vector.body:
 ; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF:         store <vscale x 4 x i32> %[[SPLAT]], <vscale x 4 x i32>*
+; CHECK-NOTF:         store <vscale x 4 x i32> %[[SPLAT]], ptr
 
 ; CHECK-TF-NORED-LABEL: @simple_memset(
 ; CHECK-TF-NORED:       vector.ph:
@@ -24,7 +24,7 @@ define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 {
 ; CHECK-TF-NORED:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NORED:       vector.body:
 ; CHECK-TF-NORED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NORED:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NORED:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
 
 ; CHECK-TF-NOREC-LABEL: @simple_memset(
 ; CHECK-TF-NOREC:       vector.ph:
@@ -32,7 +32,7 @@ define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 {
 ; CHECK-TF-NOREC:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NOREC:       vector.body:
 ; CHECK-TF-NOREC:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREC:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NOREC:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
 
 ; CHECK-TF-NOREV-LABEL: @simple_memset(
 ; CHECK-TF-NOREV:       vector.ph:
@@ -40,7 +40,7 @@ define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 {
 ; CHECK-TF-NOREV:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-NOREV:       vector.body:
 ; CHECK-TF-NOREV:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NOREV:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
 
 ; CHECK-TF-LABEL: @simple_memset(
 ; CHECK-TF:       vector.ph:
@@ -48,7 +48,7 @@ define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 {
 ; CHECK-TF:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF:       vector.body:
 ; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
-; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[SPLAT]], {{.*}} %[[ACTIVE_LANE_MASK]]
 
 ; CHECK-TF-ONLYRED-LABEL: @simple_memset(
 ; CHECK-TF-ONLYRED:       vector.ph:
@@ -56,15 +56,15 @@ define void @simple_memset(i32 %val, i32* %ptr, i64 %n) #0 {
 ; CHECK-TF-ONLYRED:         %[[SPLAT:.*]] = shufflevector <vscale x 4 x i32> %[[INSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-TF-ONLYRED:       vector.body:
 ; CHECK-TF-ONLYRED-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-ONLYRED:         store <vscale x 4 x i32> %[[SPLAT]], <vscale x 4 x i32>*
+; CHECK-TF-ONLYRED:         store <vscale x 4 x i32> %[[SPLAT]], ptr
 
 entry:
   br label %while.body
 
 while.body:                                       ; preds = %while.body, %entry
   %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]
-  %gep = getelementptr i32, i32* %ptr, i64 %index
-  store i32 %val, i32* %gep
+  %gep = getelementptr i32, ptr %ptr, i64 %index
+  store i32 %val, ptr %gep
   %index.next = add nsw i64 %index, 1
   %cmp10 = icmp ult i64 %index.next, %n
   br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0
@@ -73,7 +73,7 @@ while.end.loopexit:                               ; preds = %while.body
   ret void
 }
 
-define float @fadd_red_fast(float* noalias nocapture readonly %a, i64 %n) #0 {
+define float @fadd_red_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-NOTF-LABEL: @fadd_red_fast
 ; CHECK-NOTF:       vector.body:
 ; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
@@ -94,7 +94,7 @@ define float @fadd_red_fast(float* noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-TF-NOREC:       vector.body:
 ; CHECK-TF-NOREC:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF-NOREC:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREC:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NOREC:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF-NOREC:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
 ; CHECK-TF-NOREC:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
 ; CHECK-TF-NOREC:       middle.block:
@@ -104,7 +104,7 @@ define float @fadd_red_fast(float* noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-TF-NOREV:       vector.body:
 ; CHECK-TF-NOREV:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF-NOREV:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-NOREV:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NOREV:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF-NOREV:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
 ; CHECK-TF-NOREV:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
 ; CHECK-TF-NOREV:       middle.block:
@@ -114,7 +114,7 @@ define float @fadd_red_fast(float* noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-TF:       vector.body:
 ; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
 ; CHECK-TF:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
 ; CHECK-TF:       middle.block:
@@ -124,7 +124,7 @@ define float @fadd_red_fast(float* noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-TF-ONLYRED:       vector.body:
 ; CHECK-TF-ONLYRED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF-ONLYRED:         %[[VEC_PHI:.*]] = phi <vscale x 4 x float>
-; CHECK-TF-ONLYRED:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0nxv4f32(<vscale x 4 x float>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-ONLYRED:         %[[LOAD:.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF-ONLYRED:         %[[ADD:.*]] = fadd fast <vscale x 4 x float> %[[LOAD]]
 ; CHECK-TF-ONLYRED:         %[[SEL:.*]] = select fast <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]], <vscale x 4 x float> %[[ADD]], <vscale x 4 x float> %[[VEC_PHI]]
 ; CHECK-TF-ONLYRED:       middle.block:
@@ -135,8 +135,8 @@ entry:
 for.body:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
   %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
-  %arrayidx = getelementptr inbounds float, float* %a, i64 %iv
-  %0 = load float, float* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+  %0 = load float, ptr %arrayidx, align 4
   %add = fadd fast float %0, %sum.07
   %iv.next = add nuw nsw i64 %iv, 1
   %exitcond.not = icmp eq i64 %iv.next, %n
@@ -146,10 +146,10 @@ for.end:
   ret float %add
 }
 
-define void @add_recur(i32* noalias %dst, i32* noalias %src, i64 %n) #0 {
+define void @add_recur(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NOTF-LABEL: @add_recur
 ; CHECK-NOTF:       entry:
-; CHECK-NOTF:         %[[PRE:.*]] = load i32, i32* %src, align 4
+; CHECK-NOTF:         %[[PRE:.*]] = load i32, ptr %src, align 4
 ; CHECK-NOTF:       vector.ph:
 ; CHECK-NOTF:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
 ; CHECK-NOTF:       vector.body:
@@ -162,20 +162,20 @@ define void @add_recur(i32* noalias %dst, i32* noalias %src, i64 %n) #0 {
 
 ; CHECK-TF-NORED-LABEL: @add_recur
 ; CHECK-TF-NORED:       entry:
-; CHECK-TF-NORED:         %[[PRE:.*]] = load i32, i32* %src, align 4
+; CHECK-TF-NORED:         %[[PRE:.*]] = load i32, ptr %src, align 4
 ; CHECK-TF-NORED:       vector.ph:
 ; CHECK-TF-NORED:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
 ; CHECK-TF-NORED:       vector.body:
 ; CHECK-TF-NORED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF-NORED:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NORED:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NORED:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF-NORED:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.splice.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 -1)
 ; CHECK-TF-NORED:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NORED:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
+; CHECK-TF-NORED:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
 
 ; CHECK-TF-NOREC-LABEL: @add_recur
 ; CHECK-TF-NOREC:       entry:
-; CHECK-TF-NOREC:         %[[PRE:.*]] = load i32, i32* %src, align 4
+; CHECK-TF-NOREC:         %[[PRE:.*]] = load i32, ptr %src, align 4
 ; CHECK-TF-NOREC:       vector.ph:
 ; CHECK-TF-NOREC:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
 ; CHECK-TF-NOREC:       vector.body:
@@ -188,33 +188,33 @@ define void @add_recur(i32* noalias %dst, i32* noalias %src, i64 %n) #0 {
 
 ; CHECK-TF-NOREV-LABEL: @add_recur
 ; CHECK-TF-NOREV:       entry:
-; CHECK-TF-NOREV:         %[[PRE:.*]] = load i32, i32* %src, align 4
+; CHECK-TF-NOREV:         %[[PRE:.*]] = load i32, ptr %src, align 4
 ; CHECK-TF-NOREV:       vector.ph:
 ; CHECK-TF-NOREV:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
 ; CHECK-TF-NOREV:       vector.body:
 ; CHECK-TF-NOREV:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF-NOREV:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF-NOREV:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF-NOREV:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF-NOREV:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.splice.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 -1)
 ; CHECK-TF-NOREV:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF-NOREV:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
+; CHECK-TF-NOREV:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
 
 ; CHECK-TF-LABEL: @add_recur
 ; CHECK-TF:       entry:
-; CHECK-TF:         %[[PRE:.*]] = load i32, i32* %src, align 4
+; CHECK-TF:         %[[PRE:.*]] = load i32, ptr %src, align 4
 ; CHECK-TF:       vector.ph:
 ; CHECK-TF:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
 ; CHECK-TF:       vector.body:
 ; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 4 x i1>
 ; CHECK-TF:         %[[VECTOR_RECUR:.*]] = phi <vscale x 4 x i32> [ %[[RECUR_INIT]], %vector.ph ], [ %[[LOAD:.*]], %vector.body ]
-; CHECK-TF:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0nxv4i32(<vscale x 4 x i32>{{.*}} %[[ACTIVE_LANE_MASK]]
+; CHECK-TF:         %[[LOAD]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0({{.*}} %[[ACTIVE_LANE_MASK]]
 ; CHECK-TF:         %[[SPLICE:.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.splice.nxv4i32(<vscale x 4 x i32> %[[VECTOR_RECUR]], <vscale x 4 x i32> %[[LOAD]], i32 -1)
 ; CHECK-TF:         %[[ADD:.*]] = add nsw <vscale x 4 x i32> %[[LOAD]], %[[SPLICE]]
-; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0nxv4i32(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
+; CHECK-TF:         call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> %[[ADD]], {{.*}} <vscale x 4 x i1> %[[ACTIVE_LANE_MASK]])
 
 ; CHECK-TF-ONLYRED-LABEL: @add_recur
 ; CHECK-TF-ONLYRED:       entry:
-; CHECK-TF-ONLYRED:         %[[PRE:.*]] = load i32, i32* %src, align 4
+; CHECK-TF-ONLYRED:         %[[PRE:.*]] = load i32, ptr %src, align 4
 ; CHECK-TF-ONLYRED:       vector.ph:
 ; CHECK-TF-ONLYRED:         %[[RECUR_INIT:.*]] = insertelement <vscale x 4 x i32> poison, i32 %[[PRE]]
 ; CHECK-TF-ONLYRED:       vector.body:
@@ -226,18 +226,18 @@ define void @add_recur(i32* noalias %dst, i32* noalias %src, i64 %n) #0 {
 ; CHECK-TF-ONLYRED:         store <vscale x 4 x i32> %[[ADD]]
 
 entry:
-  %.pre = load i32, i32* %src, align 4
+  %.pre = load i32, ptr %src, align 4
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %0 = phi i32 [ %1, %for.body ], [ %.pre, %entry ]
   %i.010 = phi i64 [ %add, %for.body ], [ 0, %entry ]
   %add = add nuw nsw i64 %i.010, 1
-  %arrayidx1 = getelementptr inbounds i32, i32* %src, i64 %add
-  %1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx1 = getelementptr inbounds i32, ptr %src, i64 %add
+  %1 = load i32, ptr %arrayidx1, align 4
   %add2 = add nsw i32 %1, %0
-  %arrayidx3 = getelementptr inbounds i32, i32* %dst, i64 %i.010
-  store i32 %add2, i32* %arrayidx3, align 4
+  %arrayidx3 = getelementptr inbounds i32, ptr %dst, i64 %i.010
+  store i32 %add2, ptr %arrayidx3, align 4
   %exitcond.not = icmp eq i64 %add, %n
   br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
 
@@ -245,34 +245,34 @@ for.end:                                          ; preds = %for.body
   ret void
 }
 
-define void @interleave(float* noalias %dst, float* noalias %src, i64 %n) #0 {
+define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NOTF-LABEL: @interleave(
 ; CHECK-NOTF:       vector.body:
-; CHECK-NOTF:         %[[LOAD:.*]] = load <8 x float>, <8 x float>
+; CHECK-NOTF:         %[[LOAD:.*]] = load <8 x float>, ptr
 ; CHECK-NOTF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NOTF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 
 ; CHECK-TF-LABEL: @interleave(
 ; CHECK-TF:       vector.body:
-; CHECK-TF:         %[[LOAD:.*]] = load <8 x float>, <8 x float>
+; CHECK-TF:         %[[LOAD:.*]] = load <8 x float>, ptr
 ; CHECK-TF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-TF:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 
 ; CHECK-TF-NORED-LABEL: @interleave(
 ; CHECK-TF-NORED:       vector.body:
-; CHECK-TF-NORED:         %[[LOAD:.*]] = load <8 x float>, <8 x float>
+; CHECK-TF-NORED:         %[[LOAD:.*]] = load <8 x float>, ptr
 ; CHECK-TF-NORED:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-TF-NORED:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 
 ; CHECK-TF-NOREC-LABEL: @interleave(
 ; CHECK-TF-NOREC:       vector.body:
-; CHECK-TF-NOREC:         %[[LOAD:.*]] = load <8 x float>, <8 x float>
+; CHECK-TF-NOREC:         %[[LOAD:.*]] = load <8 x float>, ptr
 ; CHECK-TF-NOREC:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-TF-NOREC:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 
 ; CHECK-TF-NOREV-LABEL: @interleave(
 ; CHECK-TF-NOREV:       vector.body:
-; CHECK-TF-NOREV:         %[[LOAD:.*]] = load <8 x float>, <8 x float>
+; CHECK-TF-NOREV:         %[[LOAD:.*]] = load <8 x float>, ptr
 ; CHECK-TF-NOREV:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-TF-NOREV:         %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 
@@ -282,20 +282,20 @@ entry:
 for.body:                                         ; preds = %entry, %for.body
   %i.021 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
   %mul = shl nuw nsw i64 %i.021, 1
-  %arrayidx = getelementptr inbounds float, float* %src, i64 %mul
-  %0 = load float, float* %arrayidx, align 4
+  %arrayidx = getelementptr inbounds float, ptr %src, i64 %mul
+  %0 = load float, ptr %arrayidx, align 4
   %mul1 = mul nuw nsw i64 %i.021, 3
-  %arrayidx2 = getelementptr inbounds float, float* %dst, i64 %mul1
-  store float %0, float* %arrayidx2, align 4
+  %arrayidx2 = getelementptr inbounds float, ptr %dst, i64 %mul1
+  store float %0, ptr %arrayidx2, align 4
   %add = or i64 %mul, 1
-  %arrayidx4 = getelementptr inbounds float, float* %src, i64 %add
-  %1 = load float, float* %arrayidx4, align 4
+  %arrayidx4 = getelementptr inbounds float, ptr %src, i64 %add
+  %1 = load float, ptr %arrayidx4, align 4
   %add6 = add nuw nsw i64 %mul1, 1
-  %arrayidx7 = getelementptr inbounds float, float* %dst, i64 %add6
-  store float %1, float* %arrayidx7, align 4
+  %arrayidx7 = getelementptr inbounds float, ptr %dst, i64 %add6
+  store float %1, ptr %arrayidx7, align 4
   %add9 = add nuw nsw i64 %mul1, 2
-  %arrayidx10 = getelementptr inbounds float, float* %dst, i64 %add9
-  store float 3.000000e+00, float* %arrayidx10, align 4
+  %arrayidx10 = getelementptr inbounds float, ptr %dst, i64 %add9
+  store float 3.000000e+00, ptr %arrayidx10, align 4
   %inc = add nuw nsw i64 %i.021, 1
   %exitcond.not = icmp eq i64 %inc, %n
   br i1 %exitcond.not, label %for.end, label %for.body
@@ -304,47 +304,47 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
-define void @reverse(double* noalias %dst, double* noalias %src) #0 {
+define void @reverse(ptr noalias %dst, ptr noalias %src) #0 {
 ; CHECK-NOTF-LABEL: @reverse(
 ; CHECK-NOTF:       vector.body:
 ; CHECK-NOTF-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-NOTF:         %[[LOAD:.*]] = load <vscale x 2 x double>, <vscale x 2 x double>* %18, align 8
+; CHECK-NOTF:         %[[LOAD:.*]] = load <vscale x 2 x double>, ptr %17, align 8
 ; CHECK-NOTF:         %{{.*}} = call <vscale x 2 x double> @llvm.experimental.vector.reverse.nxv2f64(<vscale x 2 x double> %[[LOAD]])
 
 ; CHECK-TF-NOREV-LABEL: @reverse(
 ; CHECK-TF-NOREV:       vector.body:
 ; CHECK-TF-NOREV-NOT:     %{{.*}} = phi <vscale x 4 x i1>
-; CHECK-TF-NOREV:         %[[LOAD:.*]] = load <vscale x 2 x double>, <vscale x 2 x double>* %18, align 8
+; CHECK-TF-NOREV:         %[[LOAD:.*]] = load <vscale x 2 x double>, ptr %17, align 8
 ; CHECK-TF-NOREV:         %{{.*}} = call <vscale x 2 x double> @llvm.experimental.vector.reverse.nxv2f64(<vscale x 2 x double> %[[LOAD]])
 
 ; CHECK-TF-LABEL: @reverse(
 ; CHECK-TF:       vector.body:
 ; CHECK-TF:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
 ; CHECK-TF:         %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.experimental.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0nxv2f64(<vscale x 2 x double>{{.*}} <vscale x 2 x i1> %reverse
+; CHECK-TF:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
 
 ; CHECK-TF-NORED-LABEL: @reverse(
 ; CHECK-TF-NORED:       vector.body:
 ; CHECK-TF-NORED:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
 ; CHECK-TF-NORED:         %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.experimental.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NORED:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0nxv2f64(<vscale x 2 x double>{{.*}} <vscale x 2 x i1> %reverse
+; CHECK-TF-NORED:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
 
 ; CHECK-TF-NOREC-LABEL: @reverse(
 ; CHECK-TF-NOREC:       vector.body:
 ; CHECK-TF-NOREC:         %[[ACTIVE_LANE_MASK:.*]] = phi <vscale x 2 x i1>
 ; CHECK-TF-NOREC:         %[[REVERSE_MASK:.*]] = call <vscale x 2 x i1> @llvm.experimental.vector.reverse.nxv2i1(<vscale x 2 x i1> %[[ACTIVE_LANE_MASK]])
-; CHECK-TF-NOREC:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0nxv2f64(<vscale x 2 x double>{{.*}} <vscale x 2 x i1> %reverse
+; CHECK-TF-NOREC:         %[[MASKED_LOAD:.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0({{.*}} <vscale x 2 x i1> %reverse
 
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds double, double* %src, i64 %indvars.iv
-  %0 = load double, double* %arrayidx, align 8
+  %arrayidx = getelementptr inbounds double, ptr %src, i64 %indvars.iv
+  %0 = load double, ptr %arrayidx, align 8
   %add = fadd double %0, 1.000000e+00
-  %arrayidx2 = getelementptr inbounds double, double* %dst, i64 %indvars.iv
-  store double %add, double* %arrayidx2, align 8
+  %arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %indvars.iv
+  store double %add, ptr %arrayidx2, align 8
   %indvars.iv.next = add nsw i64 %indvars.iv, -1
   %cmp.not = icmp eq i64 %indvars.iv, 0
   br i1 %cmp.not, label %for.end, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index fd0bf10c07eda..c7698bdfcee61 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -10,6 +10,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ;
 ; CHECK:        VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' {
 ; CHECK-NEXT:   Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT:   Live-in ir<%N> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index dab4d0d60922d..9a6202d65e9dc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -11,6 +11,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-LABEL: LV: Checking a loop in 'test_v4_v4m'
 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -36,6 +37,7 @@ target triple = "aarch64-unknown-linux-gnu"
 
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -66,6 +68,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4m'
 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -91,6 +94,7 @@ target triple = "aarch64-unknown-linux-gnu"
 
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -120,6 +124,7 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4'
 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -145,6 +150,7 @@ target triple = "aarch64-unknown-linux-gnu"
 
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
index ab50b3281c020..e8e2008912c83 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll
@@ -37,27 +37,25 @@ define void @vector_reverse_mask_v4i1(ptr noalias %a, ptr noalias %cond, i64 %N)
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i64 -3
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x double> [[WIDE_LOAD]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i64 -4
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, ptr [[TMP5]], i64 -3
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i64 -7
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
 ; CHECK-NEXT:    [[REVERSE2:%.*]] = shufflevector <4 x double> [[WIDE_LOAD1]], <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP8:%.*]] = fcmp une <4 x double> [[REVERSE]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp une <4 x double> [[REVERSE2]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr double, ptr [[TMP10]], i64 -3
-; CHECK-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x i1> [[TMP8]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP11]], i32 8, <4 x i1> [[REVERSE3]], <4 x double> poison)
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr double, ptr [[TMP10]], i64 -4
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP13]], i64 -3
-; CHECK-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i1> [[TMP9]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP14]], i32 8, <4 x i1> [[REVERSE5]], <4 x double> poison)
-; CHECK-NEXT:    [[TMP16:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-; CHECK-NEXT:    [[TMP17:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD6]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
-; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP16]], ptr [[TMP11]], i32 8, <4 x i1> [[REVERSE3]])
-; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP17]], ptr [[TMP14]], i32 8, <4 x i1> [[REVERSE5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp une <4 x double> [[REVERSE]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = fcmp une <4 x double> [[REVERSE2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr double, ptr [[TMP7]], i64 -3
+; CHECK-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x i1> [[TMP5]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP8]], i32 8, <4 x i1> [[REVERSE3]], <4 x double> poison)
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr double, ptr [[TMP7]], i64 -7
+; CHECK-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr [[TMP9]], i32 8, <4 x i1> [[REVERSE5]], <4 x double> poison)
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD6]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP10]], ptr [[TMP8]], i32 8, <4 x i1> [[REVERSE3]])
+; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP11]], ptr [[TMP9]], i32 8, <4 x i1> [[REVERSE5]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -72,18 +70,18 @@ define void @vector_reverse_mask_v4i1(ptr noalias %a, ptr noalias %cond, i64 %N)
 ; CHECK-NEXT:    [[I_08_IN:%.*]] = phi i64 [ [[I_08:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[I_08]] = add nsw i64 [[I_08_IN]], -1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[COND]], i64 [[I_08]]
-; CHECK-NEXT:    [[TMP21:%.*]] = load double, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[TMP21]], 0.000000e+00
+; CHECK-NEXT:    [[TMP13:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[TMP13]], 0.000000e+00
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[I_08]]
-; CHECK-NEXT:    [[TMP22:%.*]] = load double, ptr [[ARRAYIDX1]], align 8
-; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP22]], 1.000000e+00
+; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP14]], 1.000000e+00
 ; CHECK-NEXT:    store double [[ADD]], ptr [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I_08_IN]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP4:![0-9]+]]
 ;
 
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
index bf8c175f88a85..df2e52349eea9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll
@@ -8,6 +8,7 @@ target triple = "arm64-apple-ios"
 ; CHECK-LABEL: LV: Checking a loop in 'test'
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -34,6 +35,7 @@ target triple = "arm64-apple-ios"
 
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index 9fb2510344402..4662c9ec6cc92 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -173,7 +173,7 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[STRIDED_VEC2]], <i32 3, i32 3>
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[TMP8]], i32 -2
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <6 x i32> [[TMP13]], <6 x i32> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
 ; CHECK-NEXT:    store <6 x i32> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index d44e9461e5537..4025c4e51c41d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -S | FileCheck --check-prefixes=CHECK,NOSTRIDED %s
-; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -lv-strided-pointer-ivs=true -S | FileCheck --check-prefixes=CHECK,STRIDED %s
+; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -lv-strided-pointer-ivs=true -laa-speculate-unit-stride=false -S | FileCheck --check-prefixes=CHECK,STRIDED %s
 
 
 define void @single_constant_stride_int_scaled(ptr %p) {
@@ -214,54 +214,70 @@ exit:
 
 
 define void @single_stride_int_scaled(ptr %p, i64 %stride) {
-; CHECK-LABEL: @single_stride_int_scaled(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
-; CHECK:       vector.scevcheck:
-; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
-; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP5]], [[STRIDE]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]]
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; NOSTRIDED-LABEL: @single_stride_int_scaled(
+; NOSTRIDED-NEXT:  entry:
+; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
+; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
+; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED:       vector.scevcheck:
+; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
+; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; NOSTRIDED:       vector.ph:
+; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NOSTRIDED:       vector.body:
+; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw nsw i64 [[TMP5]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; NOSTRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; NOSTRIDED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; NOSTRIDED:       middle.block:
+; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED:       scalar.ph:
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
+; NOSTRIDED:       loop:
+; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; NOSTRIDED-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; NOSTRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; NOSTRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; NOSTRIDED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; NOSTRIDED:       exit:
+; NOSTRIDED-NEXT:    ret void
+;
+; STRIDED-LABEL: @single_stride_int_scaled(
+; STRIDED-NEXT:  entry:
+; STRIDED-NEXT:    br label [[LOOP:%.*]]
+; STRIDED:       loop:
+; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE:%.*]]
+; STRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET]]
+; STRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; STRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; STRIDED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
+; STRIDED:       exit:
+; STRIDED-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -282,58 +298,75 @@ exit:
 }
 
 define void @single_stride_int_iv(ptr %p, i64 %stride) {
-; CHECK-LABEL: @single_stride_int_iv(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
-; CHECK:       vector.scevcheck:
-; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
-; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 0, [[STRIDE]]
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
-; CHECK-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE]]
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; NOSTRIDED-LABEL: @single_stride_int_iv(
+; NOSTRIDED-NEXT:  entry:
+; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
+; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
+; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED:       vector.scevcheck:
+; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
+; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; NOSTRIDED:       vector.ph:
+; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; NOSTRIDED-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
+; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NOSTRIDED:       vector.body:
+; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[TMP5:%.*]] = mul i64 0, [[STRIDE]]
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; NOSTRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; NOSTRIDED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; NOSTRIDED:       middle.block:
+; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED:       scalar.ph:
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
+; NOSTRIDED:       loop:
+; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; NOSTRIDED-NEXT:    [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
+; NOSTRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; NOSTRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; NOSTRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; NOSTRIDED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; NOSTRIDED-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; NOSTRIDED:       exit:
+; NOSTRIDED-NEXT:    ret void
+;
+; STRIDED-LABEL: @single_stride_int_iv(
+; STRIDED-NEXT:  entry:
+; STRIDED-NEXT:    br label [[LOOP:%.*]]
+; STRIDED:       loop:
+; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[OFFSET:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET]]
+; STRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; STRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; STRIDED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
+; STRIDED-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE:%.*]]
+; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
+; STRIDED:       exit:
+; STRIDED-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -391,66 +424,143 @@ exit:
 }
 
 define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
-; CHECK-LABEL: @double_stride_int_scaled(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[P3:%.*]] = ptrtoint ptr [[P:%.*]] to i64
-; CHECK-NEXT:    [[P21:%.*]] = ptrtoint ptr [[P2:%.*]] to i64
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
-; CHECK:       vector.scevcheck:
-; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
-; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK:       vector.memcheck:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
-; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[P21]], [[P3]]
-; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
-; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw nsw i64 [[TMP9]], [[STRIDE]]
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[P2]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i32 0
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP13]], ptr [[TMP15]], align 4
-; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]]
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P2]], i64 [[OFFSET]]
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; NOSTRIDED-LABEL: @double_stride_int_scaled(
+; NOSTRIDED-NEXT:  entry:
+; NOSTRIDED-NEXT:    [[P3:%.*]] = ptrtoint ptr [[P:%.*]] to i64
+; NOSTRIDED-NEXT:    [[P21:%.*]] = ptrtoint ptr [[P2:%.*]] to i64
+; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
+; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
+; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED:       vector.scevcheck:
+; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
+; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
+; NOSTRIDED:       vector.memcheck:
+; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = sub i64 [[P21]], [[P3]]
+; NOSTRIDED-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
+; NOSTRIDED-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; NOSTRIDED:       vector.ph:
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]]
+; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NOSTRIDED:       vector.body:
+; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; NOSTRIDED-NEXT:    [[TMP10:%.*]] = mul nuw nsw i64 [[TMP9]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP10]]
+; NOSTRIDED-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; NOSTRIDED-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; NOSTRIDED-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[P2]], i64 [[TMP10]]
+; NOSTRIDED-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i32 0
+; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP13]], ptr [[TMP15]], align 4
+; NOSTRIDED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
+; NOSTRIDED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; NOSTRIDED:       middle.block:
+; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED:       scalar.ph:
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
+; NOSTRIDED:       loop:
+; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; NOSTRIDED-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; NOSTRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; NOSTRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; NOSTRIDED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P2]], i64 [[OFFSET]]
+; NOSTRIDED-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
+; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; NOSTRIDED:       exit:
+; NOSTRIDED-NEXT:    ret void
+;
+; STRIDED-LABEL: @double_stride_int_scaled(
+; STRIDED-NEXT:  entry:
+; STRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 24, i64 [[TMP1]])
+; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
+; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; STRIDED:       vector.memcheck:
+; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 [[STRIDE:%.*]], 4092
+; STRIDED-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2:%.*]], i64 [[TMP3]]
+; STRIDED-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; STRIDED-NEXT:    [[UMIN:%.*]] = select i1 [[TMP4]], ptr [[P2]], ptr [[SCEVGEP]]
+; STRIDED-NEXT:    [[TMP5:%.*]] = icmp ugt ptr [[P2]], [[SCEVGEP]]
+; STRIDED-NEXT:    [[UMAX:%.*]] = select i1 [[TMP5]], ptr [[P2]], ptr [[SCEVGEP]]
+; STRIDED-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[UMAX]], i64 4
+; STRIDED-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP3]]
+; STRIDED-NEXT:    [[TMP6:%.*]] = icmp ult ptr [[P]], [[SCEVGEP2]]
+; STRIDED-NEXT:    [[UMIN3:%.*]] = select i1 [[TMP6]], ptr [[P]], ptr [[SCEVGEP2]]
+; STRIDED-NEXT:    [[TMP7:%.*]] = icmp ugt ptr [[P]], [[SCEVGEP2]]
+; STRIDED-NEXT:    [[UMAX4:%.*]] = select i1 [[TMP7]], ptr [[P]], ptr [[SCEVGEP2]]
+; STRIDED-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[UMAX4]], i64 4
+; STRIDED-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[UMIN]], [[SCEVGEP5]]
+; STRIDED-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[UMIN3]], [[SCEVGEP1]]
+; STRIDED-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; STRIDED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; STRIDED:       vector.ph:
+; STRIDED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]]
+; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; STRIDED-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; STRIDED-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP10]], zeroinitializer
+; STRIDED-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i64> [[TMP11]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; STRIDED-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP12]]
+; STRIDED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 4
+; STRIDED-NEXT:    [[TMP15:%.*]] = mul i64 1, [[TMP14]]
+; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP15]], i64 0
+; STRIDED-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
+; STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; STRIDED:       vector.body:
+; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; STRIDED-NEXT:    [[TMP16:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP16]]
+; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP17]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison), !alias.scope !8
+; STRIDED-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP16]]
+; STRIDED-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> [[TMP19]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)), !alias.scope !11, !noalias !8
+; STRIDED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]]
+; STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; STRIDED-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; STRIDED-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; STRIDED:       middle.block:
+; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; STRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; STRIDED:       scalar.ph:
+; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    br label [[LOOP:%.*]]
+; STRIDED:       loop:
+; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], [[STRIDE]]
+; STRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; STRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; STRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; STRIDED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P2]], i64 [[OFFSET]]
+; STRIDED-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
+; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; STRIDED:       exit:
+; STRIDED-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -472,59 +582,77 @@ exit:
 }
 
 define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
-; CHECK-LABEL: @double_stride_int_iv(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
-; CHECK:       vector.scevcheck:
-; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
-; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 0, [[STRIDE]]
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
-; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
-; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
-; CHECK-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
-; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
-; CHECK-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE]]
-; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; NOSTRIDED-LABEL: @double_stride_int_iv(
+; NOSTRIDED-NEXT:  entry:
+; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
+; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
+; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED:       vector.scevcheck:
+; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
+; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; NOSTRIDED:       vector.ph:
+; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; NOSTRIDED-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
+; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NOSTRIDED:       vector.body:
+; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[TMP5:%.*]] = mul i64 0, [[STRIDE]]
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
+; NOSTRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i32 0
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; NOSTRIDED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; NOSTRIDED:       middle.block:
+; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED:       scalar.ph:
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
+; NOSTRIDED:       loop:
+; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; NOSTRIDED-NEXT:    [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
+; NOSTRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; NOSTRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; NOSTRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; NOSTRIDED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; NOSTRIDED-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
+; NOSTRIDED-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE]]
+; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; NOSTRIDED:       exit:
+; NOSTRIDED-NEXT:    ret void
+;
+; STRIDED-LABEL: @double_stride_int_iv(
+; STRIDED-NEXT:  entry:
+; STRIDED-NEXT:    br label [[LOOP:%.*]]
+; STRIDED:       loop:
+; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[OFFSET:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[OFFSET]]
+; STRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
+; STRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
+; STRIDED-NEXT:    [[Q1:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
+; STRIDED-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
+; STRIDED-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE:%.*]]
+; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
+; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
+; STRIDED:       exit:
+; STRIDED-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -570,79 +698,87 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:  entry:
 ; STRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; STRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
-; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
+; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
-; STRIDED:       vector.scevcheck:
-; STRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
-; STRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
+; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; STRIDED:       vector.memcheck:
-; STRIDED-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2:%.*]], i64 1027
-; STRIDED-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 1027
-; STRIDED-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP1]]
-; STRIDED-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[P]], [[SCEVGEP]]
+; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 [[STRIDE:%.*]], 1023
+; STRIDED-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2:%.*]], i64 [[TMP3]]
+; STRIDED-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[P2]], [[SCEVGEP]]
+; STRIDED-NEXT:    [[UMIN:%.*]] = select i1 [[TMP4]], ptr [[P2]], ptr [[SCEVGEP]]
+; STRIDED-NEXT:    [[TMP5:%.*]] = icmp ugt ptr [[P2]], [[SCEVGEP]]
+; STRIDED-NEXT:    [[UMAX:%.*]] = select i1 [[TMP5]], ptr [[P2]], ptr [[SCEVGEP]]
+; STRIDED-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[UMAX]], i64 4
+; STRIDED-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP3]]
+; STRIDED-NEXT:    [[TMP6:%.*]] = icmp ult ptr [[P]], [[SCEVGEP2]]
+; STRIDED-NEXT:    [[UMIN3:%.*]] = select i1 [[TMP6]], ptr [[P]], ptr [[SCEVGEP2]]
+; STRIDED-NEXT:    [[TMP7:%.*]] = icmp ugt ptr [[P]], [[SCEVGEP2]]
+; STRIDED-NEXT:    [[UMAX4:%.*]] = select i1 [[TMP7]], ptr [[P]], ptr [[SCEVGEP2]]
+; STRIDED-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[UMAX4]], i64 4
+; STRIDED-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[UMIN]], [[SCEVGEP5]]
+; STRIDED-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[UMIN3]], [[SCEVGEP1]]
 ; STRIDED-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; STRIDED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
-; STRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
+; STRIDED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]]
 ; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; STRIDED-NEXT:    [[TMP5:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
-; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; STRIDED-NEXT:    [[TMP6:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
-; STRIDED-NEXT:    [[IND_END3:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP6]]
+; STRIDED-NEXT:    [[TMP10:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
+; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]]
+; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
+; STRIDED-NEXT:    [[IND_END7:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP11]]
 ; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT:    [[POINTER_PHI7:%.*]] = phi ptr [ [[P2]], [[VECTOR_PH]] ], [ [[PTR_IND8:%.*]], [[VECTOR_BODY]] ]
+; STRIDED-NEXT:    [[POINTER_PHI11:%.*]] = phi ptr [ [[P2]], [[VECTOR_PH]] ], [ [[PTR_IND12:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 1
-; STRIDED-NEXT:    [[TMP10:%.*]] = mul i64 [[STRIDE]], [[TMP9]]
-; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP8]], 0
-; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
+; STRIDED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; STRIDED-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 1
+; STRIDED-NEXT:    [[TMP15:%.*]] = mul i64 [[STRIDE]], [[TMP14]]
+; STRIDED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP13]], 0
+; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP16]], i64 0
 ; STRIDED-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; STRIDED-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
-; STRIDED-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT]], [[TMP12]]
-; STRIDED-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
-; STRIDED-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; STRIDED-NEXT:    [[VECTOR_GEP:%.*]] = mul <vscale x 4 x i64> [[TMP13]], [[DOTSPLAT6]]
-; STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[VECTOR_GEP]]
-; STRIDED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
-; STRIDED-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 1
-; STRIDED-NEXT:    [[TMP18:%.*]] = mul i64 [[STRIDE]], [[TMP17]]
-; STRIDED-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP16]], 0
-; STRIDED-NEXT:    [[DOTSPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP19]], i64 0
+; STRIDED-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; STRIDED-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT]], [[TMP17]]
+; STRIDED-NEXT:    [[DOTSPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
 ; STRIDED-NEXT:    [[DOTSPLAT10:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; STRIDED-NEXT:    [[TMP20:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
-; STRIDED-NEXT:    [[TMP21:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT10]], [[TMP20]]
-; STRIDED-NEXT:    [[VECTOR_GEP13:%.*]] = mul <vscale x 4 x i64> [[TMP21]], [[DOTSPLAT6]]
-; STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[POINTER_PHI7]], <vscale x 4 x i64> [[VECTOR_GEP13]]
-; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP14]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison), !alias.scope !16
-; STRIDED-NEXT:    [[TMP23:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; STRIDED-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP23]], <vscale x 4 x ptr> [[TMP22]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)), !alias.scope !19, !noalias !16
-; STRIDED-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 4
-; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP25]]
-; STRIDED-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP10]]
-; STRIDED-NEXT:    [[PTR_IND8]] = getelementptr i8, ptr [[POINTER_PHI7]], i64 [[TMP18]]
-; STRIDED-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; STRIDED-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; STRIDED-NEXT:    [[VECTOR_GEP:%.*]] = mul <vscale x 4 x i64> [[TMP18]], [[DOTSPLAT10]]
+; STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[VECTOR_GEP]]
+; STRIDED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; STRIDED-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 1
+; STRIDED-NEXT:    [[TMP23:%.*]] = mul i64 [[STRIDE]], [[TMP22]]
+; STRIDED-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP21]], 0
+; STRIDED-NEXT:    [[DOTSPLATINSERT13:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP24]], i64 0
+; STRIDED-NEXT:    [[DOTSPLAT14:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT13]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; STRIDED-NEXT:    [[TMP25:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; STRIDED-NEXT:    [[TMP26:%.*]] = add <vscale x 4 x i64> [[DOTSPLAT14]], [[TMP25]]
+; STRIDED-NEXT:    [[VECTOR_GEP17:%.*]] = mul <vscale x 4 x i64> [[TMP26]], [[DOTSPLAT10]]
+; STRIDED-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[POINTER_PHI11]], <vscale x 4 x i64> [[VECTOR_GEP17]]
+; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP19]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x i32> poison), !alias.scope !15
+; STRIDED-NEXT:    [[TMP28:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; STRIDED-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP28]], <vscale x 4 x ptr> [[TMP27]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)), !alias.scope !18, !noalias !15
+; STRIDED-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
+; STRIDED-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 4
+; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP30]]
+; STRIDED-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP15]]
+; STRIDED-NEXT:    [[PTR_IND12]] = getelementptr i8, ptr [[POINTER_PHI11]], i64 [[TMP23]]
+; STRIDED-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; STRIDED-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; STRIDED:       middle.block:
 ; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; STRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; STRIDED:       scalar.ph:
-; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; STRIDED-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P]], [[ENTRY]] ], [ [[P]], [[VECTOR_SCEVCHECK]] ], [ [[P]], [[VECTOR_MEMCHECK]] ]
-; STRIDED-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[P2]], [[ENTRY]] ], [ [[P2]], [[VECTOR_SCEVCHECK]] ], [ [[P2]], [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P]], [[ENTRY]] ], [ [[P]], [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END7]], [[MIDDLE_BLOCK]] ], [ [[P2]], [[ENTRY]] ], [ [[P2]], [[VECTOR_MEMCHECK]] ]
 ; STRIDED-NEXT:    br label [[LOOP:%.*]]
 ; STRIDED:       loop:
 ; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; STRIDED-NEXT:    [[PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[PTR_NEXT:%.*]], [[LOOP]] ]
-; STRIDED-NEXT:    [[PTR2:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ], [ [[PTR2_NEXT:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL6]], [[SCALAR_PH]] ], [ [[PTR_NEXT:%.*]], [[LOOP]] ]
+; STRIDED-NEXT:    [[PTR2:%.*]] = phi ptr [ [[BC_RESUME_VAL8]], [[SCALAR_PH]] ], [ [[PTR2_NEXT:%.*]], [[LOOP]] ]
 ; STRIDED-NEXT:    [[X0:%.*]] = load i32, ptr [[PTR]], align 4
 ; STRIDED-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
 ; STRIDED-NEXT:    store i32 [[Y0]], ptr [[PTR2]], align 4
@@ -650,7 +786,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[PTR2_NEXT]] = getelementptr inbounds i8, ptr [[PTR2]], i64 [[STRIDE]]
 ; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
 ; STRIDED:       exit:
 ; STRIDED-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
index 021ac57bdfe66..2c78b7208c19f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
@@ -263,10 +263,9 @@ define double @external_use_without_fast_math(ptr %a, i64 %n) {
 ; AUTO_VEC-LABEL: @external_use_without_fast_math(
 ; AUTO_VEC-NEXT:  entry:
 ; AUTO_VEC-NEXT:    [[SMAX:%.*]] = tail call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 1)
-; AUTO_VEC-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; AUTO_VEC-NEXT:    [[XTRAITER:%.*]] = and i64 [[SMAX]], 7
-; AUTO_VEC-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
-; AUTO_VEC-NEXT:    br i1 [[TMP1]], label [[FOR_END_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]]
+; AUTO_VEC-NEXT:    [[TMP0:%.*]] = icmp ult i64 [[SMAX]], 8
+; AUTO_VEC-NEXT:    br i1 [[TMP0]], label [[FOR_END_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]]
 ; AUTO_VEC:       entry.new:
 ; AUTO_VEC-NEXT:    [[UNROLL_ITER:%.*]] = and i64 [[SMAX]], 9223372036854775800
 ; AUTO_VEC-NEXT:    br label [[FOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
index 6f30105645f1d..4a3fc8fbdceb6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr54634.ll
@@ -29,13 +29,13 @@ define ptr addrspace(10) @japi1_vect_42283(ptr nocapture readonly %0, i32 %1) lo
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr addrspace(13) [[TMP7]], i64 [[MUL_RESULT]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult ptr addrspace(13) [[TMP10]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr addrspace(13) [[TMP7]], i64 8
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr addrspace(13) [[TMP7]], i64 8
 ; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[TMP2]])
 ; CHECK-NEXT:    [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
 ; CHECK-NEXT:    [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub i64 0, [[MUL_RESULT2]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr addrspace(13) [[UGLYGEP]], i64 [[MUL_RESULT2]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult ptr addrspace(13) [[TMP14]], [[UGLYGEP]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr addrspace(13) [[SCEVGEP]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult ptr addrspace(13) [[TMP14]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW3]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP12]], [[TMP16]]
 ; CHECK-NEXT:    br i1 [[TMP17]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
@@ -99,7 +99,7 @@ define ptr addrspace(10) @japi1_vect_42283(ptr nocapture readonly %0, i32 %1) lo
 ; CHECK-NEXT:    store i64 [[DOTUNPACK2]], ptr addrspace(13) [[DOTREPACK4]], align 8, !tbaa [[TBAA10]]
 ; CHECK-NEXT:    [[TMP27]] = add i64 [[VALUE_PHI5]], 1
 ; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i64 [[VALUE_PHI5]], [[TMP2]]
-; CHECK-NEXT:    br i1 [[DOTNOT]], label [[L44]], label [[L26]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DOTNOT]], label [[L44]], label [[L26]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       L44:
 ; CHECK-NEXT:    ret ptr addrspace(10) null
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
index 5f8f5c0f23bea..c95870b6ae342 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -14,7 +14,9 @@ target triple = "x86_64-unknown-linux"
 ;
 
 ;
-; This loop will be vectorized, although the trip count is below the threshold, but vectorization is explicitly forced in metadata.
+; This loop will be vectorized, although the trip count is below the threshold, but
+; vectorization is explicitly forced in metadata. The trip count of 4 is chosen as
+; it more nicely divides the loop count of 20, produce a lower total cost.
 ;
 define void @vectorized(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {
 ; CHECK-LABEL: @vectorized(
@@ -27,20 +29,20 @@ define void @vectorized(ptr noalias nocapture %A, ptr noalias nocapture readonly
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP2]], align 4, !llvm.access.group [[ACC_GRP0:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4, !llvm.access.group [[ACC_GRP0:![0-9]+]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x float>, ptr [[TMP4]], align 4, !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT:    store <8 x float> [[TMP5]], ptr [[TMP4]], align 4, !llvm.access.group [[ACC_GRP0]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP4]], align 4, !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    store <4 x float> [[TMP5]], ptr [[TMP4]], align 4, !llvm.access.group [[ACC_GRP0]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 20, 16
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 20, 20
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -52,7 +54,7 @@ define void @vectorized(ptr noalias nocapture %A, ptr noalias nocapture readonly
 ; CHECK-NEXT:    store float [[ADD]], ptr [[ARRAYIDX2]], align 4, !llvm.access.group [[ACC_GRP0]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 20
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index 16f85df40f1cd..d28f1f6767a56 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -83,7 +83,7 @@ define dso_local void @test1(ptr noalias nocapture %points, ptr noalias nocaptur
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = or i64 [[TMP3]], 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[TMP0]], i64 [[TMP6]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD1]], <16 x i32> <i32 0, i32 4, i32 undef, i32 undef, i32 1, i32 5, i32 undef, i32 undef, i32 2, i32 6, i32 undef, i32 undef, i32 3, i32 7, i32 undef, i32 undef>
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD1]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[TMP7]], i32 2, <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
@@ -254,7 +254,7 @@ define dso_local void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP5]], i32 2, <4 x i1> [[TMP1]], <4 x i16> poison)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = or i64 [[TMP4]], 1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP0]], i64 [[TMP7]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_MASKED_LOAD]], <4 x i16> [[WIDE_MASKED_LOAD3]], <16 x i32> <i32 0, i32 4, i32 undef, i32 undef, i32 1, i32 5, i32 undef, i32 undef, i32 2, i32 6, i32 undef, i32 undef, i32 3, i32 7, i32 undef, i32 undef>
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_MASKED_LOAD]], <4 x i16> [[WIDE_MASKED_LOAD3]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
 ; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[TMP8]], i32 2, <16 x i1> [[TMP10]])
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index 0d9a7789d1223..8f79f54e8ce8e 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -450,14 +450,12 @@ for.end:
 ; INTER:       vector.body:
 ; INTER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
 ; INTER-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr %A, i64 [[INDEX]]
-; INTER-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
-; INTER-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 -3
-; INTER-NEXT:    [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[TMP6]], align 1
+; INTER-NEXT:    [[WIDE_VEC:%.*]] = load <32 x i8>, ptr [[TMP4]], align 1
 ; INTER-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
-; INTER-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
-; INTER-NEXT:    [[TMP6:%.*]] = xor <4 x i8> [[STRIDED_VEC5]], [[STRIDED_VEC]]
-; INTER-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr %B, i64 [[INDEX]]
-; INTER-NEXT:    store <4 x i8> [[TMP6]], ptr [[TMP7]], align 1
+; INTER-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> poison, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
+; INTER-NEXT:    [[TMP5:%.*]] = xor <4 x i8> [[STRIDED_VEC3]], [[STRIDED_VEC]]
+; INTER-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr %B, i64 [[INDEX]]
+; INTER-NEXT:    store <4 x i8> [[TMP5]], ptr [[TMP6]], align 1
 ; INTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; INTER:         br i1 {{.*}}, label %middle.block, label %vector.body
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll b/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll
index 6aec21ead0fd4..04cc0efb29077 100644
--- a/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll
+++ b/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll
@@ -29,7 +29,7 @@ define void @test(i32 %arg, i32 %L1.limit, i32 %L2.switch, i1 %c) {
 ; CHECK:       L1.early.exit:
 ; CHECK-NEXT:    ret void
 ; CHECK:       L1.exit:
-; CHECK-NEXT:    [[INDUCTION_IV_LCSSA3:%.*]] = phi i32 [ [[INDUCTION_IV]], [[L1_BACKEDGE]] ]
+; CHECK-NEXT:    [[INDUCTION_IV_LCSSA2:%.*]] = phi i32 [ [[INDUCTION_IV]], [[L1_BACKEDGE]] ]
 ; CHECK-NEXT:    [[INDUCTION_IV_LCSSA1:%.*]] = phi i32 [ [[INDUCTION_IV]], [[L1_BACKEDGE]] ]
 ; CHECK-NEXT:    [[L1_EXIT_VAL:%.*]] = phi i32 [ [[L1_SUM_NEXT]], [[L1_BACKEDGE]] ]
 ; CHECK-NEXT:    br label [[L2_HEADER:%.*]]
@@ -45,7 +45,7 @@ define void @test(i32 %arg, i32 %L1.limit, i32 %L2.switch, i1 %c) {
 ; CHECK:       L2.Inner.header.preheader:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 12, [[INDUCTION_IV_LCSSA1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 12, [[INDUCTION_IV_LCSSA2]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i32 1, [[TMP3]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -58,16 +58,16 @@ define void @test(i32 %arg, i32 %L1.limit, i32 %L2.switch, i1 %c) {
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[L2_HEADER_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[L2_INNER_HEADER_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 13, [[MIDDLE_BLOCK]] ], [ 1, [[L2_INNER_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 13, [[MIDDLE_BLOCK]] ], [ 1, [[L2_INNER_HEADER_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[L2_INNER_HEADER:%.*]]
 ; CHECK:       L2.Inner.header:
 ; CHECK-NEXT:    [[L2_ACCUM:%.*]] = phi i32 [ [[L2_ACCUM_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[L2_IV:%.*]] = phi i64 [ [[L2_IV_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[L2_IV:%.*]] = phi i64 [ [[L2_IV_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[L2_ACCUM_NEXT]] = sub i32 [[L2_ACCUM]], [[L1_EXIT_VAL]]
 ; CHECK-NEXT:    [[L2_DUMMY_BUT_NEED_IT:%.*]] = sext i32 [[L2_ACCUM_NEXT]] to i64
 ; CHECK-NEXT:    [[L2_IV_NEXT]] = add nuw nsw i64 [[L2_IV]], 1
 ; CHECK-NEXT:    [[L2_EXIT_COND:%.*]] = icmp ugt i64 [[L2_IV]], 11
-; CHECK-NEXT:    br i1 [[L2_EXIT_COND]], label [[L2_HEADER_LOOPEXIT]], label [[L2_INNER_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[L2_EXIT_COND]], label [[L2_HEADER_LOOPEXIT]], label [[L2_INNER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       L2.exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
index b60241b89619e..9253ae6f3f8c3 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll
@@ -6,6 +6,7 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
 ; CHECK-LABEL: 'test_chained_first_order_recurrences_1'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1000> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -55,6 +56,7 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
 ; CHECK-LABEL: 'test_chained_first_order_recurrences_3'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1000> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index f68a31410450f..47733956e1526 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -11,6 +11,7 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<20001> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -100,6 +101,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<20001> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -168,6 +170,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<20001> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -241,6 +244,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<20001> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -338,6 +342,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:  EMIT vp<[[TC]]> = EXPAND SCEV (1 smax (1 + (sext i8 %y to i32))<nsw>)
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -411,6 +420,7 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<3> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index fd3fb3f8b496f..c624314508004 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -895,15 +895,13 @@ define i32 @PR27246() {
 ; UNROLL-NO-IC-NEXT:  entry:
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
 ; UNROLL-NO-IC:       for.cond1.preheader:
-; UNROLL-NO-IC-NEXT:    [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[FOR_COND_CLEANUP3:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[I_016:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3]] ]
+; UNROLL-NO-IC-NEXT:    [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
 ; UNROLL-NO-IC-NEXT:    [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add i32 [[INDVAR]], 1
-; UNROLL-NO-IC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 8
+; UNROLL-NO-IC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I_016]], 8
 ; UNROLL-NO-IC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; UNROLL-NO-IC:       vector.ph:
-; UNROLL-NO-IC-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 8
-; UNROLL-NO-IC-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
+; UNROLL-NO-IC-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[I_016]], 8
+; UNROLL-NO-IC-NEXT:    [[N_VEC:%.*]] = sub i32 [[I_016]], [[N_MOD_VF]]
 ; UNROLL-NO-IC-NEXT:    [[IND_END:%.*]] = sub i32 [[I_016]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[E_015]], i32 3
 ; UNROLL-NO-IC-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I_016]], i64 0
@@ -915,14 +913,14 @@ define i32 @PR27246() {
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4>
-; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; UNROLL-NO-IC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], <i32 -4, i32 -4, i32 -4, i32 -4>
-; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; UNROLL-NO-IC:       middle.block:
-; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]]
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 3
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[STEP_ADD]], i32 2
 ; UNROLL-NO-IC-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
@@ -943,38 +941,35 @@ define i32 @PR27246() {
 ; UNROLL-NO-IC-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_COND1]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    [[INC]] = add nuw nsw i32 [[I_016]], 1
 ; UNROLL-NO-IC-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49
-; UNROLL-NO-IC-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
 ; UNROLL-NO-IC-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
 ;
 ; UNROLL-NO-VF-LABEL: @PR27246(
 ; UNROLL-NO-VF-NEXT:  entry:
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
 ; UNROLL-NO-VF:       for.cond1.preheader:
-; UNROLL-NO-VF-NEXT:    [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[FOR_COND_CLEANUP3:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[I_016:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3]] ]
+; UNROLL-NO-VF-NEXT:    [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
 ; UNROLL-NO-VF-NEXT:    [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ]
-; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = add i32 [[INDVAR]], 1
-; UNROLL-NO-VF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 2
+; UNROLL-NO-VF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I_016]], 2
 ; UNROLL-NO-VF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; UNROLL-NO-VF:       vector.ph:
-; UNROLL-NO-VF-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2
-; UNROLL-NO-VF-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
+; UNROLL-NO-VF-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[I_016]], 2
+; UNROLL-NO-VF-NEXT:    [[N_VEC:%.*]] = sub i32 [[I_016]], [[N_MOD_VF]]
 ; UNROLL-NO-VF-NEXT:    [[IND_END:%.*]] = sub i32 [[I_016]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
 ; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ [[E_015]], [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi i32 [ [[E_015]], [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[I_016]], [[INDEX]]
-; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 0
-; UNROLL-NO-VF-NEXT:    [[TMP2]] = add i32 [[OFFSET_IDX]], -1
+; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = add i32 [[OFFSET_IDX]], 0
+; UNROLL-NO-VF-NEXT:    [[TMP1]] = add i32 [[OFFSET_IDX]], -1
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
-; UNROLL-NO-VF-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]]
 ; UNROLL-NO-VF-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[E_015]], [[FOR_COND1_PREHEADER]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[E_015]], [[FOR_COND1_PREHEADER]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[I_016]], [[FOR_COND1_PREHEADER]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_COND1:%.*]]
 ; UNROLL-NO-VF:       for.cond.cleanup:
@@ -987,25 +982,22 @@ define i32 @PR27246() {
 ; UNROLL-NO-VF-NEXT:    [[DEC]] = add nsw i32 [[K_0]], -1
 ; UNROLL-NO-VF-NEXT:    br i1 [[CMP2]], label [[FOR_COND1]], label [[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP9:![0-9]+]]
 ; UNROLL-NO-VF:       for.cond.cleanup3:
-; UNROLL-NO-VF-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_COND1]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-VF-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_COND1]] ], [ [[TMP0]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    [[INC]] = add nuw nsw i32 [[I_016]], 1
 ; UNROLL-NO-VF-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49
-; UNROLL-NO-VF-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
 ; UNROLL-NO-VF-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
 ;
 ; SINK-AFTER-LABEL: @PR27246(
 ; SINK-AFTER-NEXT:  entry:
 ; SINK-AFTER-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
 ; SINK-AFTER:       for.cond1.preheader:
-; SINK-AFTER-NEXT:    [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], [[FOR_COND_CLEANUP3:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; SINK-AFTER-NEXT:    [[I_016:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3]] ]
+; SINK-AFTER-NEXT:    [[I_016:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
 ; SINK-AFTER-NEXT:    [[E_015:%.*]] = phi i32 [ poison, [[ENTRY]] ], [ [[E_1_LCSSA:%.*]], [[FOR_COND_CLEANUP3]] ]
-; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add i32 [[INDVAR]], 1
-; SINK-AFTER-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 4
+; SINK-AFTER-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I_016]], 4
 ; SINK-AFTER-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SINK-AFTER:       vector.ph:
-; SINK-AFTER-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4
-; SINK-AFTER-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
+; SINK-AFTER-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[I_016]], 4
+; SINK-AFTER-NEXT:    [[N_VEC:%.*]] = sub i32 [[I_016]], [[N_MOD_VF]]
 ; SINK-AFTER-NEXT:    [[IND_END:%.*]] = sub i32 [[I_016]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[E_015]], i32 3
 ; SINK-AFTER-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[I_016]], i64 0
@@ -1016,13 +1008,13 @@ define i32 @PR27246() {
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VEC_IND]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; SINK-AFTER-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 -4, i32 -4, i32 -4, i32 -4>
-; SINK-AFTER-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SINK-AFTER-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SINK-AFTER-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SINK-AFTER-NEXT:    br i1 [[TMP1]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SINK-AFTER:       middle.block:
-; SINK-AFTER-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; SINK-AFTER-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[I_016]], [[N_VEC]]
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 3
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 2
 ; SINK-AFTER-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[SCALAR_PH]]
@@ -1043,7 +1035,6 @@ define i32 @PR27246() {
 ; SINK-AFTER-NEXT:    [[E_1_LCSSA]] = phi i32 [ [[SCALAR_RECUR]], [[FOR_COND1]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    [[INC]] = add nuw nsw i32 [[I_016]], 1
 ; SINK-AFTER-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 49
-; SINK-AFTER-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
 ; SINK-AFTER-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index 217810c61eb17..ef61ac9b19196 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -39,6 +39,7 @@ for.end:
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<14> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 97d912ece06f8..922070e844380 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -943,7 +943,7 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
 ; IND-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; IND-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
 ; IND-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; IND-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; IND-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 9
 ; IND-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IND:       vector.ph:
 ; IND-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], -2
@@ -1003,7 +1003,7 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; UNROLL-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
 ; UNROLL-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; UNROLL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 24
+; UNROLL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 25
 ; UNROLL-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; UNROLL:       vector.ph:
 ; UNROLL-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], -4
@@ -1160,10 +1160,10 @@ define float @scalarize_induction_variable_02(ptr %a, ptr %b, i64 %n) {
 ; INTERLEAVE-LABEL: @scalarize_induction_variable_02(
 ; INTERLEAVE-NEXT:  entry:
 ; INTERLEAVE-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N:%.*]], i64 8)
-; INTERLEAVE-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
-; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 64
+; INTERLEAVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 65
 ; INTERLEAVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; INTERLEAVE:       vector.ph:
+; INTERLEAVE-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; INTERLEAVE-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
 ; INTERLEAVE-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; INTERLEAVE-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 7
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index 3bd26252b4916..71ecdcd947549 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -6,6 +6,11 @@
 ; DBG-LABEL: 'test_scalarize_call'
 ; DBG:      VPlan 'Initial VPlan for VF={1},UF>=1' {
 ; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; DBG-NEXT: vp<[[TC:%.+]]> = original trip-count
+; DBG-EMPTY:
+; DBG-NEXT: ph:
+; DBG-NEXT:  EMIT vp<[[TC]]> = EXPAND SCEV (1000 + (-1 * %start))
+; DBG-NEXT: No successors
 ; DBG-EMPTY:
 ; DBG-NEXT: vector.ph:
 ; DBG-NEXT: Successor(s): vector loop
@@ -63,6 +68,7 @@ declare i32 @llvm.smin.i32(i32, i32)
 ; DBG-LABEL: 'test_scalarize_with_branch_cond'
 
 ; DBG:       Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; DBG-NEXT:  Live-in ir<1000> = original trip-count
 ; DBG-EMPTY:
 ; DBG-NEXT: vector.ph:
 ; DBG-NEXT: Successor(s): vector loop
@@ -168,6 +174,11 @@ exit:
 ; DBG-LABEL: 'first_order_recurrence_using_induction'
 ; DBG:      VPlan 'Initial VPlan for VF={1},UF>=1' {
 ; DBG-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; DBG-NEXT: vp<[[TC:%.+]]> = original trip-count
+; DBG-EMPTY:
+; DBG-NEXT: ph:
+; DBG-NEXT:  EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 (1 smax %n) to i64)
+; DBG-NEXT: No successors
 ; DBG-EMPTY:
 ; DBG-NEXT: vector.ph:
 ; DBG-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index d86c90ed26f95..bccf4bcba8253 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -119,7 +119,7 @@ define void @test_struct_array_load3_store3() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], <i32 3, i32 3, i32 3, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [1024 x %struct.ST3], ptr @S, i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
 ; CHECK-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP4]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -483,10 +483,10 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-LABEL: @even_load_dynamic_tc(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 2)
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX]], -1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 9
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[UMAX]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 3
@@ -745,18 +745,16 @@ define void @mixed_load3_store3(ptr nocapture %A) {
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
 ; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[VEC_IND]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_IND]]
-; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[STRIDED_VEC3]], [[VEC_IND]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 -2
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> [[TMP7]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
-; CHECK-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[STRIDED_VEC2]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[STRIDED_VEC3]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP5]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT:    store <12 x i32> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -1278,7 +1276,7 @@ define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 6
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 7
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], -4
diff --git a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
index 7aa8efa566b99..2459658f76360 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
@@ -22,8 +22,8 @@ define void @vector_gep(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    store <2 x ptr> [[TMP0]], ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -37,7 +37,7 @@ define void @vector_gep(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    store ptr [[VAR0]], ptr [[VAR1]], align 8
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -65,7 +65,7 @@ define void @scalar_store(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], -2
@@ -125,7 +125,7 @@ define void @expansion(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 3
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], -2
@@ -142,8 +142,8 @@ define void @expansion(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    store ptr [[TMP4]], ptr [[TMP6]], align 8
 ; CHECK-NEXT:    store ptr [[TMP5]], ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -191,13 +191,13 @@ define void @no_gep_or_bitcast(ptr noalias %a, i64 %n) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds ptr, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP0]], align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT:    store i32 0, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i64 1
 ; CHECK-NEXT:    store i32 0, ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x ptr> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT:    store i32 0, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll b/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll
index 24bcfe079ae80..c78e005311ef2 100644
--- a/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/novect-lcssa-cfg-invalidation.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 define i32 @novect(ptr %p) {
 
 ; CHECK:           Running pass: LoopVectorizePass on novect
-; CHECK:           Invalidating analysis: ScalarEvolutionAnalysis on novect
+; CHECK-NOT:       Invalidating analysis: ScalarEvolutionAnalysis on novect
 ; CHECK-NOT:       Invalidating analysis: BranchProbabilityAnalysis on novect
 ; CHECK-NOT:       Invalidating analysis: BlockFrequencyAnalysis on novect
 ; CHECK:           Invalidating analysis: DemandedBitsAnalysis on novect
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
index 8768d30f4f838..098b5a6c448c2 100644
--- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
@@ -71,7 +71,7 @@ define dso_local void @f1(ptr noalias %aa, ptr noalias %bb, ptr noalias %cc, i32
 ; CHECK-NEXT:    store <4 x float> [[TMP14]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC4]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    [[CMP_N5:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC4]]
 ; CHECK-NEXT:    br i1 [[CMP_N5]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
@@ -417,7 +417,7 @@ define void @f3(ptr noalias %A, i64 %n) {
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    store <2 x i8> <i8 1, i8 1>, ptr [[TMP6]], align 1
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], 2
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
-; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[TMP7]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[TMP7]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.middle.block:
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
 ; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[CMP_N4]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll
index bdb005b0c265c..2652009501a83 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll
@@ -24,26 +24,26 @@ define void @non_constant_scalar_expansion(i32 %0, ptr %call) {
 ; STRIDED-LABEL: @non_constant_scalar_expansion(
 ; STRIDED-NEXT:  entry:
 ; STRIDED-NEXT:    [[MUL:%.*]] = shl i32 [[TMP0:%.*]], 1
+; STRIDED-NEXT:    [[TMP1:%.*]] = sext i32 [[MUL]] to i64
 ; STRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
-; STRIDED-NEXT:    [[TMP1:%.*]] = sext i32 [[MUL]] to i64
-; STRIDED-NEXT:    [[TMP2:%.*]] = mul i64 4294967264, [[TMP1]]
-; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr null, i64 [[TMP2]]
-; STRIDED-NEXT:    [[TMP3:%.*]] = sext i32 [[MUL]] to i64
+; STRIDED-NEXT:    [[TMP2:%.*]] = sext i32 [[MUL]] to i64
+; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 4294967264, [[TMP2]]
+; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr null, i64 [[TMP3]]
 ; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
-; STRIDED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP3]]
+; STRIDED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
 ; STRIDED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP5]]
 ; STRIDED-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 1
-; STRIDED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], [[TMP3]]
+; STRIDED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], [[TMP1]]
 ; STRIDED-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr null, i64 [[TMP7]]
 ; STRIDED-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 2
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], [[TMP3]]
+; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], [[TMP1]]
 ; STRIDED-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP9]]
 ; STRIDED-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 3
-; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], [[TMP3]]
+; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], [[TMP1]]
 ; STRIDED-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
 ; STRIDED-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[INDEX]] to i32
 ; STRIDED-NEXT:    [[OFFSET_IDX:%.*]] = add i32 30, [[DOTCAST]]
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index 91f2156301560..55b553b7a87ea 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -228,20 +228,20 @@ define void @non_constant_vector_expansion(i32 %0, ptr %call) {
 ; STRIDED-LABEL: @non_constant_vector_expansion(
 ; STRIDED-NEXT:  entry:
 ; STRIDED-NEXT:    [[MUL:%.*]] = shl i32 [[TMP0:%.*]], 1
+; STRIDED-NEXT:    [[TMP1:%.*]] = sext i32 [[MUL]] to i64
 ; STRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; STRIDED:       vector.scevcheck:
 ; STRIDED-NEXT:    br i1 true, label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
-; STRIDED-NEXT:    [[TMP1:%.*]] = sext i32 [[MUL]] to i64
-; STRIDED-NEXT:    [[TMP2:%.*]] = mul i64 4294967264, [[TMP1]]
-; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr null, i64 [[TMP2]]
-; STRIDED-NEXT:    [[TMP3:%.*]] = sext i32 [[MUL]] to i64
+; STRIDED-NEXT:    [[TMP2:%.*]] = sext i32 [[MUL]] to i64
+; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 4294967264, [[TMP2]]
+; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr null, i64 [[TMP3]]
 ; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ null, [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
-; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; STRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP1]], 4
+; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
 ; STRIDED-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; STRIDED-NEXT:    [[VECTOR_GEP:%.*]] = mul <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[DOTSPLAT]]
 ; STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> [[VECTOR_GEP]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll b/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
index 208f3f5929eb5..0cda697e0337a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
@@ -105,7 +105,7 @@ define void @doit1(i32 %n, i32 %step) local_unnamed_addr {
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[CONV]], [[STEP]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
@@ -210,7 +210,7 @@ define void @doit2(i32 %n, i32 %step) local_unnamed_addr  {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], [[DOTSPLAT3]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -227,7 +227,7 @@ define void @doit2(i32 %n, i32 %step) local_unnamed_addr  {
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[CONV]], [[STEP]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
@@ -405,7 +405,7 @@ define void @doit4(i32 %n, i8 signext %cstep) local_unnamed_addr {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], [[DOTSPLAT3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -423,7 +423,7 @@ define void @doit4(i32 %n, i8 signext %cstep) local_unnamed_addr {
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[CONV2]], [[CONV]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       for.end.loopexit:
 ; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
index 49d3da8a2aefd..707de7b8f02f8 100644
--- a/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
@@ -1,9 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
 
-; REQUIRES: asserts
-; XFAIL: *
-
 define void @test1_pr58811() {
 ; CHECK-LABEL: @test1_pr58811(
 ; CHECK-NEXT:  entry:
@@ -22,38 +19,38 @@ define void @test1_pr58811() {
 ; CHECK-NEXT:    [[INDUCTION_IV_NEXT]] = add i32 [[INDUCTION_IV]], [[TMP1]]
 ; CHECK-NEXT:    br i1 false, label [[LOOP_1]], label [[LOOP_2_PREHEADER:%.*]]
 ; CHECK:       loop.2.preheader:
-; CHECK-NEXT:    [[INDUCTION_IV_LCSSA2:%.*]] = phi i32 [ [[INDUCTION_IV]], [[LOOP_1]] ]
+; CHECK-NEXT:    [[INDUCTION_IV_LCSSA1:%.*]] = phi i32 [ [[INDUCTION_IV]], [[LOOP_1]] ]
 ; CHECK-NEXT:    [[INDUCTION_IV_LCSSA:%.*]] = phi i32 [ [[INDUCTION_IV]], [[LOOP_1]] ]
 ; CHECK-NEXT:    [[IV_1_LCSSA:%.*]] = phi i32 [ [[IV_1]], [[LOOP_1]] ]
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i32 196, [[INDUCTION_IV_LCSSA]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i32 196, [[INDUCTION_IV_LCSSA1]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i32 [[INDEX]], [[INDUCTION_IV_LCSSA2]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 0, [[INDUCTION_IV_LCSSA2]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i32 [[INDEX]], [[INDUCTION_IV_LCSSA]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 0, [[INDUCTION_IV_LCSSA]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[INDUCTION_IV_LCSSA2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 1, [[INDUCTION_IV_LCSSA]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 2, [[INDUCTION_IV_LCSSA2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 2, [[INDUCTION_IV_LCSSA]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 3, [[INDUCTION_IV_LCSSA2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 3, [[INDUCTION_IV_LCSSA]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], [[TMP8]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 196
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 199, 196
-; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = mul i32 195, [[INDUCTION_IV_LCSSA2]]
+; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = mul i32 195, [[INDUCTION_IV_LCSSA]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_3_PREHEADER:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 196, [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_2_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_2_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_2_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop.2:
 ; CHECK-NEXT:    [[IV_3:%.*]] = phi i16 [ [[IV_3_NEXT:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[IV_4:%.*]] = phi i32 [ [[IV_4_NEXT:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV_4:%.*]] = phi i32 [ [[IV_4_NEXT:%.*]], [[LOOP_2]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[IV_4_NEXT]] = sub i32 [[IV_4]], [[IV_1_LCSSA]]
 ; CHECK-NEXT:    [[IV_3_NEXT]] = add i16 [[IV_3]], 1
 ; CHECK-NEXT:    [[CMP88_1:%.*]] = icmp ult i16 [[IV_3]], 198
diff --git a/llvm/test/Transforms/LoopVectorize/smax-idx.ll b/llvm/test/Transforms/LoopVectorize/smax-idx.ll
new file mode 100644
index 0000000000000..dab2fd7781625
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/smax-idx.ll
@@ -0,0 +1,234 @@
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK
+
+define i64 @smax_idx(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1 = icmp slt i64 %max.09, %0
+  %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %1, ptr %res_max
+  ret i64 %spec.select7
+}
+
+;
+; Check the different order of reduction phis.
+;
+define i64 @smax_idx_inverted_phi(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx_inverted_phi(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1 = icmp slt i64 %max.09, %0
+  %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %1, ptr %res_max
+  ret i64 %spec.select7
+}
+
+; Check if it is a MMI when smax is not used outside the loop.
+;
+; Currently at the end, it will check if smax has exitInstruction.
+; But in fact MMI should be possible to use the exitInstruction of
+; SelectICmp be the exitInstruction.
+;
+define i64 @smax_idx_max_no_exit_user(ptr nocapture readonly %a, i64 %mm, i64 %ii, i64 %n) {
+; CHECK-LABEL: @smax_idx_max_no_exit_user(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1 = icmp slt i64 %max.09, %0
+  %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  ; %1 has no external users
+  ret i64 %spec.select7
+}
+
+; Check smax implemented in terms of select(cmp()).
+;
+; Currently SelectICmp does not support icmp with multiple users.
+; It may be possible to reuse some of the methods in Combination pass to check
+; whether icmp can be copied.
+;
+define i64 @smax_idx_select_cmp(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx_select_cmp(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %spec.select, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %cmp1 = icmp slt i64 %max.09, %0
+  %spec.select = select i1 %cmp1, i64 %0, i64 %max.09
+  %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %spec.select, ptr %res_max
+  ret i64 %spec.select7
+}
+
+;
+; Check sge case.
+;
+define i64 @smax_idx_inverted_pred(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx_inverted_pred(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1 = icmp sge i64 %0, %max.09
+  %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %1, ptr %res_max
+  ret i64 %spec.select7
+}
+
+;
+; In such cases, the last index should be extracted.
+;
+define i64 @smax_idx_extract_last(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx_extract_last(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1.not = icmp sgt i64 %max.09, %0
+  %spec.select7 = select i1 %cmp1.not, i64 %idx.011, i64 %indvars.iv
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %1, ptr %res_max
+  ret i64 %spec.select7
+}
+
+;
+; The operands of smax intrinsic and icmp are not the same to be recognized as MMI.
+;
+define i64 @smax_idx_not_vec_1(ptr nocapture readonly %a, ptr nocapture readonly %b, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx_not_vec_1(
+; CHECK-NOT:   vector.body:
+;
+  entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %2, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %arrayidx.01 = getelementptr inbounds i64, ptr %b, i64 %indvars.iv
+  %1 = load i64, ptr %arrayidx
+  %2 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1 = icmp slt i64 %max.09, %1
+  %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %2, ptr %res_max
+  ret i64 %spec.select7
+}
+
+;
+; It cannot be recognized as MMI when the operand of index select is not an induction variable.
+;
+define i64 @smax_idx_not_vec_2(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {
+; CHECK-LABEL: @smax_idx_not_vec_2(
+; CHECK-NOT:   vector.body:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]
+  %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  %0 = load i64, ptr %arrayidx
+  %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)
+  %cmp1 = icmp slt i64 %max.09, %0
+  %spec.select7 = select i1 %cmp1, i64 123, i64 %idx.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  store i64 %1, ptr %res_max
+  ret i64 %spec.select7
+}
+
+declare i64 @llvm.smax.i64(i64, i64)
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll
new file mode 100644
index 0000000000000..4dbfe2e433681
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll
@@ -0,0 +1,1219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck %s
+
+; Tests for checking uniformity within a VF.
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv/1] + 42;
+define void @ld_div1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv/2] + 42;
+define void @ld_div2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv/3] + 42;
+define void @ld_div3_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv/1] + 42;
+define void @ld_div1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv/2] + 42;
+define void @ld_div2_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv/3] + 42;
+define void @ld_div3_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[iv/1] + 42;
+define void @ld_div1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[iv/2] + 42;
+define void @ld_div2_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[iv/3] + 42;
+define void @ld_div3_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 1) B[iv] = A[iv/1] + 42;
+define void @ld_div1_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step1_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 1) B[iv] = A[iv/2] + 42;
+define void @ld_div2_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step1_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 1) B[iv] = A[iv/3] + 42;
+define void @ld_div3_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step1_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 2) B[iv] = A[iv/1] + 42;
+define void @ld_div1_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step2_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 2) B[iv] = A[iv/2] + 42;
+define void @ld_div2_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step2_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; CHECK-NEXT:    store i64 [[TMP10]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 2) B[iv] = A[iv/3] + 42;
+define void @ld_div3_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step2_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv/1] + 42;
+define void @ld_div1_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv/2] + 42;
+define void @ld_div2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv/3] + 42;
+define void @ld_div3_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; CHECK-NEXT:    store i64 [[TMP10]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %div
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll
new file mode 100644
index 0000000000000..be9a87be94610
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll
@@ -0,0 +1,756 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck %s
+
+; Tests for checking uniformity within a VF.
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[i&-1] + 42;
+define void @ld_and_neg1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg1_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv&-2] + 42;
+define void @ld_and_neg2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg2_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -2, i64 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv&-3] + 42;
+define void @ld_and_neg3_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg3_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -3, i64 -3>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv&-1] + 42;
+define void @ld_and_neg1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg1_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -1, i64 -1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv&-2] + 42;
+define void @ld_and_neg2_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg2_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[BROADCAST_SPLAT]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[iv&-1] + 42;
+define void @ld_and_neg1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg1_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -1, i64 -1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[i&-2] + 42;
+define void @ld_and_neg2_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg2_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -2, i64 -2>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 1) B[iv] = A[iv&-2] + 42;
+define void @ld_and_neg2_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg2_step1_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -2, i64 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 2) B[iv] = A[iv&-2] + 42;
+define void @ld_and_neg2_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg2_step2_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -2, i64 -2>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv&-2] + 42;
+define void @ld_and_neg2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg2_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -2, i64 -2>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv&-3] + 42;
+define void @ld_and_neg3_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_and_neg3_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[VEC_IND]], <i64 -3, i64 -3>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -3
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %and = and i64 %iv, -3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %and
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll
new file mode 100644
index 0000000000000..737336377adb8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll
@@ -0,0 +1,748 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck %s
+
+; Tests for checking uniformity within a VF.
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv>>0] + 42;
+define void @ld_lshr0_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr0_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 0
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv>>1] + 42;
+define void @ld_lshr1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr1_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[iv>>2] + 42;
+define void @ld_lshr2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr2_step1_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv>>0] + 42;
+define void @ld_lshr0_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr0_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 0
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 2) B[iv] = A[iv>>1] + 42;
+define void @ld_lshr1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr1_step2_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[iv>>0] + 42;
+define void @ld_lshr0_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr0_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[VEC_IND]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 0
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 3) B[iv] = A[iv>>1] + 42;
+define void @ld_lshr1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr1_step3_start0_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 1) B[iv] = A[iv>>1] + 42;
+define void @ld_lshr1_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr1_step1_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 2) B[iv] = A[iv>>1] + 42;
+define void @ld_lshr1_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr1_step2_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; CHECK-NEXT:    store i64 [[TMP10]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv>>1] + 42;
+define void @ld_lshr1_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr1_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 1
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1 ; ; iv += 3) B[iv] = A[iv>>2] + 42;
+define void @ld_lshr2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_lshr2_step3_start1_ind1
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP12]], i32 0
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP12]], i32 1
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 2
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %lshr = lshr i64 %iv, 2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %lshr
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll
new file mode 100644
index 0000000000000..2569e1785053d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll
@@ -0,0 +1,1493 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck %s
+
+; for (iv = 0, iv2 = 0 ; ; iv += 1, iv2 += 1) B[iv] = A[iv/1 + iv2/1] + 42;
+define void @ld_div1_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step1_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 1
+  %div2 = udiv i64 %iv2, 1
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 1, iv2 += 1) B[iv] = A[iv/2 + iv2/2] + 42;
+define void @ld_div2_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step1_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 2
+  %div2 = udiv i64 %iv2, 2
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 1, iv2 += 1) B[iv] = A[iv/3 + iv2/3] + 42;
+define void @ld_div3_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step1_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1000, 1000
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 3
+  %div2 = udiv i64 %iv2, 3
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 2, iv2 += 1) B[iv] = A[iv/1 + iv2/1] + 42;
+define void @ld_div1_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step2_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <2 x i64> [[TMP12]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 1
+  %div2 = udiv i64 %iv2, 1
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 2, iv2 += 1) B[iv] = A[iv/2 + iv2/2] + 42;
+define void @ld_div2_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step2_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <2 x i64> [[TMP12]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 2
+  %div2 = udiv i64 %iv2, 2
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 2, iv2 += 1) B[iv] = A[iv/3 + iv2/3] + 42;
+define void @ld_div3_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step2_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <2 x i64> [[TMP12]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 500, 500
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 3
+  %div2 = udiv i64 %iv2, 3
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 3, iv2 += 1) B[iv] = A[iv/1 + iv2/1] + 42;
+define void @ld_div1_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step3_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <2 x i64> [[TMP12]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 1
+  %div2 = udiv i64 %iv2, 1
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 3, iv2 += 1) B[iv] = A[iv/2 + iv2/2] + 42;
+define void @ld_div2_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step3_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <2 x i64> [[TMP12]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 2
+  %div2 = udiv i64 %iv2, 2
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0, iv2 = 0 ; ; iv += 3, iv2 += 1) B[iv] = A[iv/3 + iv2/3] + 42;
+define void @ld_div3_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step3_start0_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <2 x i64> [[TMP12]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i64> [[TMP13]], i32 0
+; CHECK-NEXT:    store i64 [[TMP16]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP13]], i32 1
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 3
+  %div2 = udiv i64 %iv2, 3
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 1, iv2 += 1) B[iv] = A[iv/1 + iv2/1] + 42;
+define void @ld_div1_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step1_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 998, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 1
+  %div2 = udiv i64 %iv2, 1
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 1, iv2 += 1) B[iv] = A[iv/2 + iv2/2] + 42;
+define void @ld_div2_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step1_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 998, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 2
+  %div2 = udiv i64 %iv2, 2
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 1, iv2 += 1) B[iv] = A[iv/3 + iv2/3] + 42;
+define void @ld_div3_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step1_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 999, 998
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 998, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 3
+  %div2 = udiv i64 %iv2, 3
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 2, iv2 += 1) B[iv] = A[iv/1 + iv2/1] + 42;
+define void @ld_div1_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step2_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i64> [[TMP13]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1
+; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 498, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 1
+  %div2 = udiv i64 %iv2, 1
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 2, iv2 += 1) B[iv] = A[iv/2 + iv2/2] + 42;
+define void @ld_div2_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step2_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i64> [[TMP13]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1
+; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 498, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 2
+  %div2 = udiv i64 %iv2, 2
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 2, iv2 += 1) B[iv] = A[iv/3 + iv2/3] + 42;
+define void @ld_div3_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step2_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i64> [[TMP13]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1
+; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 4, i64 4>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 499, 498
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 498, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 3
+  %div2 = udiv i64 %iv2, 3
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 2
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 3, iv2 += 1) B[iv] = A[iv/1 + iv2/1] + 42;
+define void @ld_div1_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div1_step3_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i64> [[TMP13]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1
+; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 1
+  %div2 = udiv i64 %iv2, 1
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 3, iv2 += 1) B[iv] = A[iv/2 + iv2/2] + 42;
+define void @ld_div2_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_step3_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i64> [[TMP13]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1
+; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 2
+  %div2 = udiv i64 %iv2, 2
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 1, iv2 = 0 ; ; iv += 3, iv2 += 1) B[iv] = A[iv/3 + iv2/3] + 42;
+define void @ld_div3_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div3_step3_start1_ind2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], <i64 3, i64 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw <2 x i64> [[TMP13]], <i64 42, i64 42>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i64> [[TMP14]], i32 1
+; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 6, i64 6>
+; CHECK-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 333, 332
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
+; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
+; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
+; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
+; CHECK-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv_next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2_next, %loop ]
+  %div1 = udiv i64 %iv, 3
+  %div2 = udiv i64 %iv2, 3
+  %add = add i64 %div1, %div2
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %add
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv2_next = add nsw i64 %iv2, 1
+  %iv_next = add nsw i64 %iv, 3
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vector-no-scevcheck.ll b/llvm/test/Transforms/LoopVectorize/vector-no-scevcheck.ll
new file mode 100644
index 0000000000000..60d7828159159
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vector-no-scevcheck.ll
@@ -0,0 +1,80 @@
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+; This test is to ensure that SCEV checks (which are costly performancewise) are
+; not generated when appropriate aliasing checks are sufficient.
+
+define void @foo(ptr %pout, ptr %pin, i64 %val0, i64 %val1, i64 %val2) {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.scevcheck
+; CHECK: vector.body
+entry:
+  %0 = getelementptr double, ptr %pin, i64 %val0
+  br label %loop1.header
+
+loop1.header:                                     ; preds = %loop1.latch, %entry
+  %i = phi i64 [ %i.next, %loop1.latch ], [ 0, %entry ]
+  %mul0 = mul nsw i64 %i, %val2
+  %arrayidx0 = getelementptr inbounds double, ptr %0, i64 %mul0
+  %mul1 = mul nsw i64 %i, %val1
+  br label %loop2.header
+
+loop2.header:                                     ; preds = %loop1.header, %loop2.header
+  %j = phi i64 [ 0, %loop1.header ], [ %j.next, %loop2.header ]
+  %1 = load double, ptr %arrayidx0, align 8
+  %arrayidx1 = getelementptr inbounds double, ptr %0, i64 %j
+  %2 = load double, ptr %arrayidx1, align 8
+  %sum = fadd contract double %1, %2
+  %3 = getelementptr double, ptr %pout, i64 %mul1
+  %arrayidx2 = getelementptr inbounds double, ptr %3, i64 %j
+  store double %sum, ptr %arrayidx2, align 8
+  %j.next = add nuw nsw i64 %j, 1
+  %cmp = icmp slt i64 %j.next, %val1
+  br i1 %cmp, label %loop2.header, label %loop1.latch
+
+loop1.latch:                                      ; preds = %loop2.header
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %val1
+  br i1 %exitcond, label %exit, label %loop1.header
+
+exit:                                             ; preds = %loop1.latch
+  ret void
+}
+
+; Similar test to the above but with the %arrayidx0 moved to the loop2.header
+
+define void @bar(ptr %pout, ptr %pin, i64 %val0, i64 %val1, i64 %val2) {
+; CHECK-LABEL: @bar(
+; CHECK-NOT: vector.scevcheck
+; CHECK: vector.body
+entry:
+  %0 = getelementptr double, ptr %pin, i64 %val0
+  br label %loop1.header
+
+loop1.header:                                     ; preds = %loop1.latch, %entry
+  %i = phi i64 [ %i.next, %loop1.latch ], [ 0, %entry ]
+  %mul0 = mul nsw i64 %i, %val2
+  %mul1 = mul nsw i64 %i, %val1
+  br label %loop2.header
+
+loop2.header:                                     ; preds = %loop1.header, %loop2.header
+  %j = phi i64 [ 0, %loop1.header ], [ %j.next, %loop2.header ]
+  %arrayidx0 = getelementptr inbounds double, ptr %0, i64 %mul0
+  %1 = load double, ptr %arrayidx0, align 8
+  %arrayidx1 = getelementptr inbounds double, ptr %0, i64 %j
+  %2 = load double, ptr %arrayidx1, align 8
+  %sum = fadd contract double %1, %2
+  %3 = getelementptr double, ptr %pout, i64 %mul1
+  %arrayidx2 = getelementptr inbounds double, ptr %3, i64 %j
+  store double %sum, ptr %arrayidx2, align 8
+  %j.next = add nuw nsw i64 %j, 1
+  %cmp = icmp slt i64 %j.next, %val1
+  br i1 %cmp, label %loop2.header, label %loop1.latch
+
+loop1.latch:                                      ; preds = %loop2.header
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %val1
+  br i1 %exitcond, label %exit, label %loop1.header
+
+exit:                                             ; preds = %loop1.latch
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index a6a9cfa31a6c2..31169339b8820 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -13,14 +13,18 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-NEXT:  edge [fontname=Courier, fontsize=30]
 ; CHECK-NEXT:  compound=true
 ; CHECK-NEXT:  N0 [label =
+; CHECK-NEXT:    "ph:\l" +
+; CHECK-NEXT:    "No successors\l"
+; CHECK-NEXT:  ]
+; CHECK-NEXT:  N1 [label =
 ; CHECK-NEXT:    "vector.ph:\l" +
 ; CHECK-NEXT:    "Successor(s): vector loop\l"
 ; CHECK-NEXT:  ]
-; CHECK-NEXT:  N0 -> N1 [ label="" lhead=cluster_N2]
-; CHECK-NEXT:  subgraph cluster_N2 {
+; CHECK-NEXT:  N1 -> N2 [ label="" lhead=cluster_N3]
+; CHECK-NEXT:  subgraph cluster_N3 {
 ; CHECK-NEXT:    fontname=Courier
 ; CHECK-NEXT:    label="\<x1\> vector loop"
-; CHECK-NEXT:    N1 [label =
+; CHECK-NEXT:    N2 [label =
 ; CHECK-NEXT:    "vector.body:\l" +
 ; CHECK-NEXT:    "  EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION\l" +
 ; CHECK-NEXT:    "  vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" +
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 431d25d29a7f3..2b59a3012674a 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -5,6 +5,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
 ; CHECK-LABEL: LV: Checking a loop in 'iv_no_binary_op_in_descriptor'
 ; CHECK:      VPlan 'Initial VPlan for VF={8},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1000> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index a497522498790..94fc421f6b48e 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -8,6 +8,8 @@
 define void @foo(i64 %n) {
 ; CHECK:      VPlan 'HCFGBuilder: Plain CFG
 ; CHECK-NEXT: {
+; CHECK-NEXT: ir<8> = original trip-count
+; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): outer.header
 ; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index c6197f2377e46..a4c7c83e972aa 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -10,6 +10,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-LABEL: Checking a loop in 'print_call_and_memory'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -56,6 +57,7 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
 ; CHECK-LABEL: Checking a loop in 'print_widen_gep_and_select'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -107,6 +109,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-LABEL: Checking a loop in 'print_reduction'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -152,6 +155,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-LABEL: Checking a loop in 'print_reduction_with_invariant_store'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -196,6 +200,11 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-LABEL: Checking a loop in 'print_replicate_predicated_phi'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:  EMIT vp<[[TC]]> = EXPAND SCEV (1 smax %n)
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -269,6 +278,7 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
 ; CHECK-LABEL: Checking a loop in 'print_interleave_groups'
 ; CHECK:       VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<256> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -336,6 +346,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-LABEL: Checking a loop in 'print_fmuladd_strict'
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -386,6 +397,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-LABEL: Checking a loop in 'debug_loc_vpinstruction'
 ; CHECK:    VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<128> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -471,9 +483,14 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
 ; CHECK-LABEL: Checking a loop in 'print_expand_scev'
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
 ; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060))<nuw><nsw> /u (1 + (%y /u 492802768830814060))<nuw><nsw>))<nuw><nsw>
 ; CHECK-NEXT:   EMIT vp<[[EXP_SCEV:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060))<nuw><nsw>
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
@@ -521,6 +538,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
 ; CHECK-LABEL: Checking a loop in 'print_exit_value'
 ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1000> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index 5b6e1f4bbb645..b394fcce48704 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -9,6 +9,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
 ; CHECK-LABEL: sink_with_sideeffects
 ; CHECK:      VPlan 'Initial VPlan for VF={1},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: ir<0> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 405fa0f628593..0ca2d337df4c4 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -13,6 +13,11 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -76,6 +81,11 @@ exit:
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -154,6 +164,11 @@ exit:
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -234,6 +249,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<11> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -303,6 +319,11 @@ define void @pred_cfg1(i32 %k, i32 %j) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -397,6 +418,11 @@ define void @pred_cfg2(i32 %k, i32 %j) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -500,6 +526,11 @@ define void @pred_cfg3(i32 %k, i32 %j) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -603,6 +634,11 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -702,6 +738,11 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -763,6 +804,11 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:   EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -842,6 +888,7 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {
 ; CHECK-LABEL: LV: Checking a loop in 'update_multiple_users'
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<999> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -905,6 +952,7 @@ define void @sinking_requires_duplication(ptr %addr) {
 ; CHECK-LABEL: LV: Checking a loop in 'sinking_requires_duplication'
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<201> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -973,6 +1021,7 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n
 ; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
@@ -1037,6 +1086,11 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-LABEL: LV: Checking a loop in 'ptr_induction_remove_dead_recipe'
 ; CHECK:       VPlan 'Initial VPlan for VF={2},UF>=1' {
 ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ph:
+; CHECK-NEXT:  EMIT vp<[[TC]]> = EXPAND SCEV ((-1 * (ptrtoint ptr %end to i64)) + (ptrtoint ptr %start to i64))
+; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
 ; CHECK-NEXT: Successor(s): vector loop
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
index 54fc716c5b7d9..0e7ce5ea51478 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
@@ -49,7 +49,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
@@ -68,7 +68,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
@@ -87,7 +87,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
@@ -106,7 +106,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP54:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
-; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
@@ -125,7 +125,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP64:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
 ; CHECK-NEXT:    [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
-; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
@@ -144,7 +144,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP74:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
 ; CHECK-NEXT:    [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
@@ -163,7 +163,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP84:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
 ; CHECK-NEXT:    [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
-; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
@@ -182,7 +182,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP94:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
 ; CHECK-NEXT:    [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
-; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
@@ -201,7 +201,7 @@ define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP104:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
 ; CHECK-NEXT:    [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
-; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    store <3 x double> [[TMP47]], ptr [[C_PTR:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP87:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
@@ -283,7 +283,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
@@ -302,7 +302,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
@@ -321,7 +321,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
@@ -340,7 +340,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP54:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
-; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
@@ -359,7 +359,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP64:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
 ; CHECK-NEXT:    [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
-; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
@@ -378,7 +378,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP74:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
 ; CHECK-NEXT:    [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
@@ -397,7 +397,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP84:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
 ; CHECK-NEXT:    [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
-; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
@@ -416,7 +416,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP94:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
 ; CHECK-NEXT:    [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
-; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
@@ -435,7 +435,7 @@ define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP104:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
 ; CHECK-NEXT:    [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
-; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[COL_LOAD87:%.*]] = load <3 x double>, ptr [[C_PTR:%.*]], align 8
 ; CHECK-NEXT:    [[VEC_GEP88:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
index dc97410bdd22a..680fe6b560362 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
@@ -26,7 +26,7 @@ define void @test(i32 %r, i32 %c) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT4:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT3]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK2]], [[SPLAT_SPLAT4]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[COL_LOAD]], i64 0
@@ -39,7 +39,7 @@ define void @test(i32 %r, i32 %c) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT9]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK8]], [[SPLAT_SPLAT10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK11:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD1]], i64 0
@@ -52,7 +52,7 @@ define void @test(i32 %r, i32 %c) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[COL_LOAD1]], i64 0
@@ -65,7 +65,7 @@ define void @test(i32 %r, i32 %c) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    store <2 x double> [[COL_LOAD]], ptr getelementptr inbounds ([5 x <4 x double>], ptr @foo, i64 0, i64 2), align 8
 ; CHECK-NEXT:    store <2 x double> [[COL_LOAD1]], ptr getelementptr (double, ptr getelementptr inbounds ([5 x <4 x double>], ptr @foo, i64 0, i64 2), i64 2), align 8
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll
index f1509ec869160..9f1578e8dc9d9 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-transpose-int.ll
@@ -178,7 +178,7 @@ define <1 x i32> @test_builtin_column_major_variable_stride_multiuse(ptr %src, <
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i32> [[COL_LOAD]], <1 x i32> [[COL_LOAD3]], <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i32> [[COL_LOAD6]], <1 x i32> [[COL_LOAD9]], <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x i32> [[COL_LOAD12]], <1 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x i32> [[COL_LOAD12]], <1 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <1 x i32> [[COL_LOAD]], i64 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <5 x i32> poison, i32 [[TMP6]], i64 0
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
index f027cf397c6b1..d22a20d9982f9 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
@@ -96,7 +96,7 @@ define <6 x float> @strided_load_2x3_align16_stride2(ptr %in) {
 ; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr float, ptr %in, i64 4
 ; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[COL_LOAD5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[COL_LOAD5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    ret <6 x float> [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
index 3eed0d357bcb2..92cce63643235 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
@@ -46,7 +46,7 @@ define void @multiply_sub_add_2x3_3x2(ptr %a.ptr, ptr %b.ptr, ptr %c.ptr) {
 ; RM-NEXT:    [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP11:%.*]] = fmul <1 x double> [[SPLAT_SPLAT15]], [[BLOCK13]]
 ; RM-NEXT:    [[TMP12:%.*]] = fadd <1 x double> [[TMP9]], [[TMP11]]
-; RM-NEXT:    [[TMP13:%.*]] = shufflevector <1 x double> [[TMP12]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP13:%.*]] = shufflevector <1 x double> [[TMP12]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP13]], <2 x i32> <i32 2, i32 1>
 ; RM-NEXT:    [[BLOCK16:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP15:%.*]] = extractelement <3 x double> [[TMP0]], i64 0
@@ -65,7 +65,7 @@ define void @multiply_sub_add_2x3_3x2(ptr %a.ptr, ptr %b.ptr, ptr %c.ptr) {
 ; RM-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[SPLAT_SPLAT24]], [[BLOCK22]]
 ; RM-NEXT:    [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
-; RM-NEXT:    [[TMP23:%.*]] = shufflevector <1 x double> [[TMP22]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP23:%.*]] = shufflevector <1 x double> [[TMP22]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP24:%.*]] = shufflevector <2 x double> [[TMP14]], <2 x double> [[TMP23]], <2 x i32> <i32 0, i32 2>
 ; RM-NEXT:    [[BLOCK25:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP25:%.*]] = extractelement <3 x double> [[TMP1]], i64 0
@@ -84,7 +84,7 @@ define void @multiply_sub_add_2x3_3x2(ptr %a.ptr, ptr %b.ptr, ptr %c.ptr) {
 ; RM-NEXT:    [[SPLAT_SPLAT33:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[SPLAT_SPLAT33]], [[BLOCK31]]
 ; RM-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
-; RM-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP34:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP33]], <2 x i32> <i32 2, i32 1>
 ; RM-NEXT:    [[BLOCK34:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP35:%.*]] = extractelement <3 x double> [[TMP1]], i64 0
@@ -103,7 +103,7 @@ define void @multiply_sub_add_2x3_3x2(ptr %a.ptr, ptr %b.ptr, ptr %c.ptr) {
 ; RM-NEXT:    [[SPLAT_SPLAT42:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT41]], <1 x double> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[SPLAT_SPLAT42]], [[BLOCK40]]
 ; RM-NEXT:    [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
-; RM-NEXT:    [[TMP43:%.*]] = shufflevector <1 x double> [[TMP42]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP43:%.*]] = shufflevector <1 x double> [[TMP42]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP44:%.*]] = shufflevector <2 x double> [[TMP34]], <2 x double> [[TMP43]], <2 x i32> <i32 0, i32 2>
 ; RM-NEXT:    [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[C_PTR:%.*]], align 8
 ; RM-NEXT:    [[VEC_GEP44:%.*]] = getelementptr double, ptr [[C_PTR]], i64 2
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll
index b3b1f9352bf4f..b6b02d8ae46f8 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction-fmf.ll
@@ -19,7 +19,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> poison, double [[TMP2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK4]], <1 x double> [[SPLAT_SPLAT6]], <1 x double> [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x double> [[TMP3]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x double> [[TMP3]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP4]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -31,7 +31,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> poison, double [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK10]], <1 x double> [[SPLAT_SPLAT12]], <1 x double> [[TMP7]])
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP10]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -43,7 +43,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> poison, double [[TMP14]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP15:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK16]], <1 x double> [[SPLAT_SPLAT18]], <1 x double> [[TMP13]])
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP15]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP15]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -55,7 +55,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK22]], <1 x double> [[SPLAT_SPLAT24]], <1 x double> [[TMP19]])
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x double> [[TMP21]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x double> [[TMP21]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP22]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP24]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll
index 5879e9d0abc9a..c39da56b7ae18 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-contraction.ll
@@ -19,7 +19,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x double> poison, double [[TMP2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK4]], <1 x double> [[SPLAT_SPLAT6]], <1 x double> [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x double> [[TMP3]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x double> [[TMP3]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP4]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -31,7 +31,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> poison, double [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK10]], <1 x double> [[SPLAT_SPLAT12]], <1 x double> [[TMP7]])
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP10]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -43,7 +43,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> poison, double [[TMP14]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP15:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK16]], <1 x double> [[SPLAT_SPLAT18]], <1 x double> [[TMP13]])
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP15]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x double> [[TMP15]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -55,7 +55,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[BLOCK22]], <1 x double> [[SPLAT_SPLAT24]], <1 x double> [[TMP19]])
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x double> [[TMP21]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x double> [[TMP21]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP22]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP24]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-row-major.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-row-major.ll
index af7cdc8af4720..70f8993e10346 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-row-major.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double-row-major.ll
@@ -20,7 +20,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[SPLAT_SPLAT6]], [[BLOCK4]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
@@ -33,7 +33,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[SPLAT_SPLAT12]], [[BLOCK10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
@@ -46,7 +46,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[SPLAT_SPLAT18]], [[BLOCK16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
@@ -59,7 +59,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[SPLAT_SPLAT24]], [[BLOCK22]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP28]]
@@ -82,28 +82,28 @@ define <4 x double> @multiply_1x2(<2 x double> %a, <2 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP0]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[SPLAT_SPLAT]], [[BLOCK]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP2]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x double> [[SPLIT]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x double> poison, double [[TMP4]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT4]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <1 x double> [[SPLAT_SPLAT5]], [[BLOCK3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> poison, double [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul <1 x double> [[SPLAT_SPLAT8]], [[BLOCK6]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP12]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x double> [[SPLAT_SPLAT11]], [[BLOCK9]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x double> [[TMP13]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x double> [[TMP13]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP14]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP16]]
@@ -134,7 +134,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT6]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[SPLAT_SPLAT7]], [[BLOCK5]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
@@ -147,7 +147,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT12]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[SPLAT_SPLAT13]], [[BLOCK11]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
@@ -160,7 +160,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[SPLAT_SPLAT19]], [[BLOCK17]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
@@ -173,7 +173,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[SPLAT_SPLAT25]], [[BLOCK23]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
@@ -186,7 +186,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[SPLAT_SPLAT31]], [[BLOCK29]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
@@ -199,7 +199,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x double> [[SPLAT_SPLAT37]], [[BLOCK35]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x double> [[TMP34]], <3 x double> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -212,7 +212,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x double> [[SPLAT_SPLAT43]], [[BLOCK41]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
-; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -225,7 +225,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul <1 x double> [[SPLAT_SPLAT49]], [[BLOCK47]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = fadd <1 x double> [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x double> [[TMP48]], <3 x double> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -238,10 +238,10 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[SPLAT_SPLAT55]], [[BLOCK53]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = fadd <1 x double> [[TMP57]], [[TMP59]]
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x double> [[TMP60]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x double> [[TMP60]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x double> [[TMP55]], <3 x double> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x double> [[TMP20]], <3 x double> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x double> [[TMP62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x double> [[TMP62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x double> [[TMP63]], <6 x double> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x double> [[TMP65]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
index af82d1ee0ba73..94205d7b5a50e 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
@@ -20,7 +20,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK4]], [[SPLAT_SPLAT6]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -33,7 +33,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK10]], [[SPLAT_SPLAT12]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -46,7 +46,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK16]], [[SPLAT_SPLAT18]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -59,7 +59,7 @@ define <4 x double> @multiply_2x2(<4 x double> %a, <4 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK22]], [[SPLAT_SPLAT24]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP28]]
@@ -82,28 +82,28 @@ define <4 x double> @multiply_1x2(<2 x double> %a, <2 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP0]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP2]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x double> poison, double [[TMP4]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT4]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <1 x double> [[BLOCK3]], [[SPLAT_SPLAT5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x double> [[SPLIT2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> poison, double [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x double> [[SPLIT2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP12]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x double> [[TMP13]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x double> [[TMP13]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP14]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP16]]
@@ -134,7 +134,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT6]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK5]], [[SPLAT_SPLAT7]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -147,7 +147,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT12]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK11]], [[SPLAT_SPLAT13]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -160,7 +160,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -173,7 +173,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -186,7 +186,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -199,7 +199,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x double> [[TMP34]], <3 x double> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
@@ -212,7 +212,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x double> [[BLOCK41]], [[SPLAT_SPLAT43]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
-; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
@@ -225,7 +225,7 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul <1 x double> [[BLOCK47]], [[SPLAT_SPLAT49]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = fadd <1 x double> [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x double> [[TMP53]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x double> [[TMP48]], <3 x double> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
@@ -238,10 +238,10 @@ define <9 x double> @multiply_2x3(<6 x double> %a, <6 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = fadd <1 x double> [[TMP57]], [[TMP59]]
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x double> [[TMP60]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x double> [[TMP60]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x double> [[TMP55]], <3 x double> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x double> [[TMP20]], <3 x double> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x double> [[TMP62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x double> [[TMP62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x double> [[TMP63]], <6 x double> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x double> [[TMP65]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll
index 9c69c6471cdf2..8c724eecf316b 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction-fmf.ll
@@ -19,7 +19,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> poison, float [[TMP2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK4]], <1 x float> [[SPLAT_SPLAT6]], <1 x float> [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x float> [[TMP3]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x float> [[TMP3]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP4]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -31,7 +31,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> poison, float [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK10]], <1 x float> [[SPLAT_SPLAT12]], <1 x float> [[TMP7]])
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -43,7 +43,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> poison, float [[TMP14]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP15:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK16]], <1 x float> [[SPLAT_SPLAT18]], <1 x float> [[TMP13]])
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP15]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP15]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP16]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -55,7 +55,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> poison, float [[TMP20]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK22]], <1 x float> [[SPLAT_SPLAT24]], <1 x float> [[TMP19]])
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x float> [[TMP21]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x float> [[TMP21]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x float> [[TMP17]], <2 x float> [[TMP22]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[TMP24]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll
index ab23352e807e1..5472b4c420b11 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float-contraction.ll
@@ -19,7 +19,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> poison, float [[TMP2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK4]], <1 x float> [[SPLAT_SPLAT6]], <1 x float> [[TMP1]])
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x float> [[TMP3]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <1 x float> [[TMP3]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP4]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -31,7 +31,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> poison, float [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK10]], <1 x float> [[SPLAT_SPLAT12]], <1 x float> [[TMP7]])
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -43,7 +43,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> poison, float [[TMP14]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP15:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK16]], <1 x float> [[SPLAT_SPLAT18]], <1 x float> [[TMP13]])
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP15]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <1 x float> [[TMP15]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP16]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -55,7 +55,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> poison, float [[TMP20]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = call contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK22]], <1 x float> [[SPLAT_SPLAT24]], <1 x float> [[TMP19]])
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x float> [[TMP21]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <1 x float> [[TMP21]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x float> [[TMP17]], <2 x float> [[TMP22]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[TMP24]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
index b9068d48bcf67..d32a20f85ac17 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
@@ -20,7 +20,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x float> [[BLOCK4]], [[SPLAT_SPLAT6]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -33,7 +33,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x float> [[BLOCK10]], [[SPLAT_SPLAT12]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x float> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x float> [[TMP11]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x float> [[TMP11]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -46,7 +46,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x float> [[BLOCK16]], [[SPLAT_SPLAT18]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x float> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP18]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP18]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -59,7 +59,7 @@ define <4 x float> @multiply_2x2(<4 x float> %a, <4 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x float> [[BLOCK22]], [[SPLAT_SPLAT24]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x float> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x float> [[TMP25]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x float> [[TMP25]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x float> [[TMP20]], <2 x float> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[TMP28]]
@@ -82,28 +82,28 @@ define <4 x float> @multiply_1x2(<2 x float> %a, <2 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP2]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x float> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x float> poison, float [[TMP4]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT4]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <1 x float> [[BLOCK3]], [[SPLAT_SPLAT5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP6]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x float> [[SPLIT2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x float> poison, float [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT7]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul <1 x float> [[BLOCK6]], [[SPLAT_SPLAT8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP10]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x float> [[SPLIT2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x float> poison, float [[TMP12]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT10]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = fmul <1 x float> [[BLOCK9]], [[SPLAT_SPLAT11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x float> [[TMP13]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x float> [[TMP13]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP14]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[TMP16]]
@@ -134,7 +134,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT6]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x float> [[BLOCK5]], [[SPLAT_SPLAT7]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x float> undef, <3 x float> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -147,7 +147,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT12]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x float> [[BLOCK11]], [[SPLAT_SPLAT13]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x float> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x float> [[TMP11]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x float> [[TMP11]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x float> [[TMP6]], <3 x float> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -160,7 +160,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT18]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x float> [[BLOCK17]], [[SPLAT_SPLAT19]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x float> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP18]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x float> [[TMP18]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x float> [[TMP13]], <3 x float> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -173,7 +173,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT24]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x float> [[BLOCK23]], [[SPLAT_SPLAT25]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x float> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x float> [[TMP25]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x float> [[TMP25]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x float> undef, <3 x float> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -186,7 +186,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT30]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x float> [[BLOCK29]], [[SPLAT_SPLAT31]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x float> [[TMP29]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x float> [[TMP32]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x float> [[TMP32]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x float> [[TMP27]], <3 x float> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -199,7 +199,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT36]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x float> [[BLOCK35]], [[SPLAT_SPLAT37]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x float> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x float> [[TMP39]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x float> [[TMP39]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x float> [[TMP34]], <3 x float> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 0
@@ -212,7 +212,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT42]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x float> [[BLOCK41]], [[SPLAT_SPLAT43]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x float> [[TMP43]], [[TMP45]]
-; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x float> [[TMP46]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x float> [[TMP46]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x float> undef, <3 x float> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 0
@@ -225,7 +225,7 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT48]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul <1 x float> [[BLOCK47]], [[SPLAT_SPLAT49]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = fadd <1 x float> [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x float> [[TMP53]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x float> [[TMP53]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x float> [[TMP48]], <3 x float> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x float> [[SPLIT]], <3 x float> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x float> [[SPLIT4]], i64 0
@@ -238,10 +238,10 @@ define <9 x float> @multiply_2x3(<6 x float> %a, <6 x float> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT54]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x float> [[BLOCK53]], [[SPLAT_SPLAT55]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = fadd <1 x float> [[TMP57]], [[TMP59]]
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x float> [[TMP60]], <1 x float> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x float> [[TMP60]], <1 x float> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x float> [[TMP55]], <3 x float> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x float> [[TMP20]], <3 x float> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x float> [[TMP62]], <3 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x float> [[TMP62]], <3 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x float> [[TMP63]], <6 x float> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x float> [[TMP65]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32-row-major.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32-row-major.ll
index c7a5f73b9879a..67fde6d6ccf49 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32-row-major.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32-row-major.ll
@@ -22,7 +22,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT5]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP3:%.*]] = mul <1 x i32> [[SPLAT_SPLAT6]], [[BLOCK4]]
 ; RM-NEXT:    [[TMP4:%.*]] = add <1 x i32> [[TMP1]], [[TMP3]]
-; RM-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; RM-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x i32> [[SPLIT2]], <2 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
@@ -35,7 +35,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT11]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP10:%.*]] = mul <1 x i32> [[SPLAT_SPLAT12]], [[BLOCK10]]
 ; RM-NEXT:    [[TMP11:%.*]] = add <1 x i32> [[TMP8]], [[TMP10]]
-; RM-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; RM-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x i32> [[SPLIT2]], <2 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
@@ -48,7 +48,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT17]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP17:%.*]] = mul <1 x i32> [[SPLAT_SPLAT18]], [[BLOCK16]]
 ; RM-NEXT:    [[TMP18:%.*]] = add <1 x i32> [[TMP15]], [[TMP17]]
-; RM-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP20:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; RM-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x i32> [[SPLIT2]], <2 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
@@ -61,7 +61,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT23]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP24:%.*]] = mul <1 x i32> [[SPLAT_SPLAT24]], [[BLOCK22]]
 ; RM-NEXT:    [[TMP25:%.*]] = add <1 x i32> [[TMP22]], [[TMP24]]
-; RM-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP27:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; RM-NEXT:    [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; RM-NEXT:    ret <4 x i32> [[TMP28]]
@@ -84,28 +84,28 @@ define <4 x i32> @multiply_1x2(<2 x i32> %a, <2 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> poison, i32 [[TMP0]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP1:%.*]] = mul <1 x i32> [[SPLAT_SPLAT]], [[BLOCK]]
-; RM-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 1>
 ; RM-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x i32> [[SPLIT2]], <2 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP4:%.*]] = extractelement <1 x i32> [[SPLIT]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x i32> poison, i32 [[TMP4]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT4]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP5:%.*]] = mul <1 x i32> [[SPLAT_SPLAT5]], [[BLOCK3]]
-; RM-NEXT:    [[TMP6:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP6:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP6]], <2 x i32> <i32 0, i32 2>
 ; RM-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x i32> [[SPLIT2]], <2 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP8:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x i32> poison, i32 [[TMP8]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT7]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP9:%.*]] = mul <1 x i32> [[SPLAT_SPLAT8]], [[BLOCK6]]
-; RM-NEXT:    [[TMP10:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP10:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP10]], <2 x i32> <i32 2, i32 1>
 ; RM-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x i32> [[SPLIT2]], <2 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP12:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x i32> poison, i32 [[TMP12]], i64 0
 ; RM-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT10]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP13:%.*]] = mul <1 x i32> [[SPLAT_SPLAT11]], [[BLOCK9]]
-; RM-NEXT:    [[TMP14:%.*]] = shufflevector <1 x i32> [[TMP13]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT:    [[TMP14:%.*]] = shufflevector <1 x i32> [[TMP13]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; RM-NEXT:    [[TMP15:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> [[TMP14]], <2 x i32> <i32 0, i32 2>
 ; RM-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; RM-NEXT:    ret <4 x i32> [[TMP16]]
@@ -136,7 +136,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT6]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP3:%.*]] = mul <1 x i32> [[SPLAT_SPLAT7]], [[BLOCK5]]
 ; RM-NEXT:    [[TMP4:%.*]] = add <1 x i32> [[TMP1]], [[TMP3]]
-; RM-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP6:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
 ; RM-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
@@ -149,7 +149,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT12]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP10:%.*]] = mul <1 x i32> [[SPLAT_SPLAT13]], [[BLOCK11]]
 ; RM-NEXT:    [[TMP11:%.*]] = add <1 x i32> [[TMP8]], [[TMP10]]
-; RM-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP13:%.*]] = shufflevector <3 x i32> [[TMP6]], <3 x i32> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
 ; RM-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> <i32 2>
 ; RM-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT]], i64 0
@@ -162,7 +162,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT18]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP17:%.*]] = mul <1 x i32> [[SPLAT_SPLAT19]], [[BLOCK17]]
 ; RM-NEXT:    [[TMP18:%.*]] = add <1 x i32> [[TMP15]], [[TMP17]]
-; RM-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP20:%.*]] = shufflevector <3 x i32> [[TMP13]], <3 x i32> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
 ; RM-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
@@ -175,7 +175,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT24]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP24:%.*]] = mul <1 x i32> [[SPLAT_SPLAT25]], [[BLOCK23]]
 ; RM-NEXT:    [[TMP25:%.*]] = add <1 x i32> [[TMP22]], [[TMP24]]
-; RM-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP27:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; RM-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP28:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
@@ -188,7 +188,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT30]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP31:%.*]] = mul <1 x i32> [[SPLAT_SPLAT31]], [[BLOCK29]]
 ; RM-NEXT:    [[TMP32:%.*]] = add <1 x i32> [[TMP29]], [[TMP31]]
-; RM-NEXT:    [[TMP33:%.*]] = shufflevector <1 x i32> [[TMP32]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP33:%.*]] = shufflevector <1 x i32> [[TMP32]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP34:%.*]] = shufflevector <3 x i32> [[TMP27]], <3 x i32> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
 ; RM-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> <i32 2>
 ; RM-NEXT:    [[TMP35:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
@@ -201,7 +201,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT36]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP38:%.*]] = mul <1 x i32> [[SPLAT_SPLAT37]], [[BLOCK35]]
 ; RM-NEXT:    [[TMP39:%.*]] = add <1 x i32> [[TMP36]], [[TMP38]]
-; RM-NEXT:    [[TMP40:%.*]] = shufflevector <1 x i32> [[TMP39]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP40:%.*]] = shufflevector <1 x i32> [[TMP39]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP41:%.*]] = shufflevector <3 x i32> [[TMP34]], <3 x i32> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
 ; RM-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
@@ -214,7 +214,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT42]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP45:%.*]] = mul <1 x i32> [[SPLAT_SPLAT43]], [[BLOCK41]]
 ; RM-NEXT:    [[TMP46:%.*]] = add <1 x i32> [[TMP43]], [[TMP45]]
-; RM-NEXT:    [[TMP47:%.*]] = shufflevector <1 x i32> [[TMP46]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP47:%.*]] = shufflevector <1 x i32> [[TMP46]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP48:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
 ; RM-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> <i32 1>
 ; RM-NEXT:    [[TMP49:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
@@ -227,7 +227,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT48]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP52:%.*]] = mul <1 x i32> [[SPLAT_SPLAT49]], [[BLOCK47]]
 ; RM-NEXT:    [[TMP53:%.*]] = add <1 x i32> [[TMP50]], [[TMP52]]
-; RM-NEXT:    [[TMP54:%.*]] = shufflevector <1 x i32> [[TMP53]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP54:%.*]] = shufflevector <1 x i32> [[TMP53]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP55:%.*]] = shufflevector <3 x i32> [[TMP48]], <3 x i32> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
 ; RM-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x i32> [[SPLIT3]], <3 x i32> poison, <1 x i32> <i32 2>
 ; RM-NEXT:    [[TMP56:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
@@ -240,10 +240,10 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; RM-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT54]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; RM-NEXT:    [[TMP59:%.*]] = mul <1 x i32> [[SPLAT_SPLAT55]], [[BLOCK53]]
 ; RM-NEXT:    [[TMP60:%.*]] = add <1 x i32> [[TMP57]], [[TMP59]]
-; RM-NEXT:    [[TMP61:%.*]] = shufflevector <1 x i32> [[TMP60]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP61:%.*]] = shufflevector <1 x i32> [[TMP60]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP62:%.*]] = shufflevector <3 x i32> [[TMP55]], <3 x i32> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
 ; RM-NEXT:    [[TMP63:%.*]] = shufflevector <3 x i32> [[TMP20]], <3 x i32> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; RM-NEXT:    [[TMP64:%.*]] = shufflevector <3 x i32> [[TMP62]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; RM-NEXT:    [[TMP64:%.*]] = shufflevector <3 x i32> [[TMP62]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; RM-NEXT:    [[TMP65:%.*]] = shufflevector <6 x i32> [[TMP63]], <6 x i32> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; RM-NEXT:    ret <9 x i32> [[TMP65]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
index 68b71cad799d3..a0ff13fa7a954 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
@@ -20,7 +20,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT5]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul <1 x i32> [[BLOCK4]], [[SPLAT_SPLAT6]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <1 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
@@ -33,7 +33,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT11]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <1 x i32> [[BLOCK10]], [[SPLAT_SPLAT12]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = add <1 x i32> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
@@ -46,7 +46,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT17]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul <1 x i32> [[BLOCK16]], [[SPLAT_SPLAT18]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = add <1 x i32> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
@@ -59,7 +59,7 @@ define <4 x i32> @multiply_2x2(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT23]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = mul <1 x i32> [[BLOCK22]], [[SPLAT_SPLAT24]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = add <1 x i32> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP28]]
@@ -82,28 +82,28 @@ define <4 x i32> @multiply_1x2(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> poison, i32 [[TMP0]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul <1 x i32> [[BLOCK]], [[SPLAT_SPLAT]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK3:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x i32> poison, i32 [[TMP4]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT4]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul <1 x i32> [[BLOCK3]], [[SPLAT_SPLAT5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP6]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x i32> poison, i32 [[TMP8]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT7]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul <1 x i32> [[BLOCK6]], [[SPLAT_SPLAT8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> undef, <2 x i32> [[TMP10]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x i32> poison, i32 [[TMP12]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT10]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <1 x i32> [[BLOCK9]], [[SPLAT_SPLAT11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x i32> [[TMP13]], <1 x i32> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <1 x i32> [[TMP13]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> [[TMP14]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP16]]
@@ -134,7 +134,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT6]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul <1 x i32> [[BLOCK5]], [[SPLAT_SPLAT7]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <1 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x i32> [[TMP4]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP5]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
@@ -147,7 +147,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT12]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <1 x i32> [[BLOCK11]], [[SPLAT_SPLAT13]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = add <1 x i32> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x i32> [[TMP11]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <3 x i32> [[TMP6]], <3 x i32> [[TMP12]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[SPLIT2]], i64 0
@@ -160,7 +160,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT18]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul <1 x i32> [[BLOCK17]], [[SPLAT_SPLAT19]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = add <1 x i32> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x i32> [[TMP18]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x i32> [[TMP13]], <3 x i32> [[TMP19]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
@@ -173,7 +173,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT24]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = mul <1 x i32> [[BLOCK23]], [[SPLAT_SPLAT25]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = add <1 x i32> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x i32> [[TMP25]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
@@ -186,7 +186,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT30]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP31:%.*]] = mul <1 x i32> [[BLOCK29]], [[SPLAT_SPLAT31]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = add <1 x i32> [[TMP29]], [[TMP31]]
-; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x i32> [[TMP32]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <1 x i32> [[TMP32]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <3 x i32> [[TMP27]], <3 x i32> [[TMP33]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <2 x i32> [[SPLIT3]], i64 0
@@ -199,7 +199,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT36]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = mul <1 x i32> [[BLOCK35]], [[SPLAT_SPLAT37]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = add <1 x i32> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x i32> [[TMP39]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x i32> [[TMP39]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x i32> [[TMP34]], <3 x i32> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 0
@@ -212,7 +212,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT42]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP45:%.*]] = mul <1 x i32> [[BLOCK41]], [[SPLAT_SPLAT43]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = add <1 x i32> [[TMP43]], [[TMP45]]
-; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x i32> [[TMP46]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <1 x i32> [[TMP46]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <3 x i32> undef, <3 x i32> [[TMP47]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 0
@@ -225,7 +225,7 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT48]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP52:%.*]] = mul <1 x i32> [[BLOCK47]], [[SPLAT_SPLAT49]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = add <1 x i32> [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x i32> [[TMP53]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <1 x i32> [[TMP53]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x i32> [[TMP48]], <3 x i32> [[TMP54]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x i32> [[SPLIT]], <3 x i32> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <2 x i32> [[SPLIT4]], i64 0
@@ -238,10 +238,10 @@ define <9 x i32> @multiply_2x3(<6 x i32> %a, <6 x i32> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT54]], <1 x i32> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP59:%.*]] = mul <1 x i32> [[BLOCK53]], [[SPLAT_SPLAT55]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = add <1 x i32> [[TMP57]], [[TMP59]]
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x i32> [[TMP60]], <1 x i32> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <1 x i32> [[TMP60]], <1 x i32> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <3 x i32> [[TMP55]], <3 x i32> [[TMP61]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <3 x i32> [[TMP20]], <3 x i32> [[TMP41]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x i32> [[TMP62]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <3 x i32> [[TMP62]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <6 x i32> [[TMP63]], <6 x i32> [[TMP64]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x i32> [[TMP65]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-left-transpose-row-major.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-left-transpose-row-major.ll
index ff6bc25d4c45d..021c987ab488c 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-left-transpose-row-major.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-left-transpose-row-major.ll
@@ -20,7 +20,7 @@ define <4 x double> @multiply_left_transpose_2x2(<4 x double> %a, <4 x double> %
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[SPLAT_SPLAT6]], [[BLOCK4]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT]], i64 0
@@ -33,7 +33,7 @@ define <4 x double> @multiply_left_transpose_2x2(<4 x double> %a, <4 x double> %
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[SPLAT_SPLAT12]], [[BLOCK10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT]], i64 1
@@ -46,7 +46,7 @@ define <4 x double> @multiply_left_transpose_2x2(<4 x double> %a, <4 x double> %
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[SPLAT_SPLAT18]], [[BLOCK16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT]], i64 1
@@ -59,7 +59,7 @@ define <4 x double> @multiply_left_transpose_2x2(<4 x double> %a, <4 x double> %
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[SPLAT_SPLAT24]], [[BLOCK22]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP28]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
index 0b178335d76c1..f8c7bacdd7990 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
@@ -19,7 +19,7 @@ define <4 x double> @multiply_right_transpose_2x2(<4 x double> %a, <4 x double>
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT5]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK4]], [[SPLAT_SPLAT6]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -32,7 +32,7 @@ define <4 x double> @multiply_right_transpose_2x2(<4 x double> %a, <4 x double>
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK10]], [[SPLAT_SPLAT12]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
@@ -45,7 +45,7 @@ define <4 x double> @multiply_right_transpose_2x2(<4 x double> %a, <4 x double>
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <1 x double> [[BLOCK16]], [[SPLAT_SPLAT18]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fadd <1 x double> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <1 x double> [[TMP18]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP19]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 1
@@ -58,7 +58,7 @@ define <4 x double> @multiply_right_transpose_2x2(<4 x double> %a, <4 x double>
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK22]], [[SPLAT_SPLAT24]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP28]]
@@ -114,7 +114,7 @@ define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fadd <1 x double> [[TMP16]], [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <1 x double> [[TMP19]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <1 x double> [[TMP19]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -133,7 +133,7 @@ define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double
 ; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = fadd <1 x double> [[TMP26]], [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <1 x double> [[TMP29]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <1 x double> [[TMP29]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> [[TMP30]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
@@ -152,7 +152,7 @@ define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double
 ; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP40]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
@@ -171,7 +171,7 @@ define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double
 ; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <2 x double> [[TMP41]], <2 x double> [[TMP50]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> [[TMP51]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[TMP52]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/preserve-existing-fast-math-flags.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/preserve-existing-fast-math-flags.ll
index ed6791009b680..eb77efd544167 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/preserve-existing-fast-math-flags.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/preserve-existing-fast-math-flags.ll
@@ -17,7 +17,7 @@ define <4 x float> @preserve_fmf_fast(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> poison, float [[TMP3]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK4]], <1 x float> [[SPLAT_SPLAT6]], <1 x float> [[TMP2]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -29,7 +29,7 @@ define <4 x float> @preserve_fmf_fast(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> poison, float [[TMP9]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK10]], <1 x float> [[SPLAT_SPLAT12]], <1 x float> [[TMP8]])
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <1 x float> [[TMP10]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <1 x float> [[TMP10]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP11]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -41,7 +41,7 @@ define <4 x float> @preserve_fmf_fast(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> poison, float [[TMP15]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = call fast <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK16]], <1 x float> [[SPLAT_SPLAT18]], <1 x float> [[TMP14]])
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x float> [[TMP16]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x float> [[TMP16]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP17]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -53,7 +53,7 @@ define <4 x float> @preserve_fmf_fast(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> poison, float [[TMP21]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = call fast <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK22]], <1 x float> [[SPLAT_SPLAT24]], <1 x float> [[TMP20]])
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <1 x float> [[TMP22]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <1 x float> [[TMP22]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x float> [[TMP18]], <2 x float> [[TMP23]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[SPLIT25:%.*]] = shufflevector <4 x float> [[N]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT26:%.*]] = shufflevector <4 x float> [[N]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
@@ -84,7 +84,7 @@ define <4 x float> @preserve_fmf_reassoc(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul reassoc <1 x float> [[BLOCK4]], [[SPLAT_SPLAT6]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd reassoc <1 x float> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP6]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -97,7 +97,7 @@ define <4 x float> @preserve_fmf_reassoc(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul reassoc <1 x float> [[BLOCK10]], [[SPLAT_SPLAT12]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = fadd reassoc <1 x float> [[TMP9]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <1 x float> [[TMP12]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <1 x float> [[TMP12]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> [[TMP13]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -110,7 +110,7 @@ define <4 x float> @preserve_fmf_reassoc(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul reassoc <1 x float> [[BLOCK16]], [[SPLAT_SPLAT18]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fadd reassoc <1 x float> [[TMP16]], [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <1 x float> [[TMP19]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <1 x float> [[TMP19]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP20]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -123,7 +123,7 @@ define <4 x float> @preserve_fmf_reassoc(<4 x float> %m, <4 x float> %n) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = fmul reassoc <1 x float> [[BLOCK22]], [[SPLAT_SPLAT24]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = fadd reassoc <1 x float> [[TMP23]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <1 x float> [[TMP26]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <1 x float> [[TMP26]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x float> [[TMP21]], <2 x float> [[TMP27]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[SPLIT25:%.*]] = shufflevector <4 x float> [[N]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT26:%.*]] = shufflevector <4 x float> [[N]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
@@ -153,7 +153,7 @@ define <4 x float> @preserve_fmf_contract_reassoc(<4 x float> %m, <4 x float> %n
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <1 x float> poison, float [[TMP3]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT5]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = call reassoc contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK4]], <1 x float> [[SPLAT_SPLAT6]], <1 x float> [[TMP2]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x float> [[TMP4]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK7:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[SPLIT2]], i64 0
@@ -165,7 +165,7 @@ define <4 x float> @preserve_fmf_contract_reassoc(<4 x float> %m, <4 x float> %n
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x float> poison, float [[TMP9]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT11]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = call reassoc contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK10]], <1 x float> [[SPLAT_SPLAT12]], <1 x float> [[TMP8]])
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <1 x float> [[TMP10]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <1 x float> [[TMP10]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP11]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -177,7 +177,7 @@ define <4 x float> @preserve_fmf_contract_reassoc(<4 x float> %m, <4 x float> %n
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x float> poison, float [[TMP15]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT17]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = call reassoc contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK16]], <1 x float> [[SPLAT_SPLAT18]], <1 x float> [[TMP14]])
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x float> [[TMP16]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <1 x float> [[TMP16]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> undef, <2 x float> [[TMP17]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK19:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x float> [[SPLIT3]], i64 0
@@ -189,7 +189,7 @@ define <4 x float> @preserve_fmf_contract_reassoc(<4 x float> %m, <4 x float> %n
 ; CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x float> poison, float [[TMP21]], i64 0
 ; CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT23]], <1 x float> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = call reassoc contract <1 x float> @llvm.fmuladd.v1f32(<1 x float> [[BLOCK22]], <1 x float> [[SPLAT_SPLAT24]], <1 x float> [[TMP20]])
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <1 x float> [[TMP22]], <1 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <1 x float> [[TMP22]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x float> [[TMP18]], <2 x float> [[TMP23]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[SPLIT25:%.*]] = shufflevector <4 x float> [[N]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT26:%.*]] = shufflevector <4 x float> [[N]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
index bd1a10bd18f43..be7eeb91f8c76 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
@@ -29,7 +29,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> [[TMP11]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <6 x double> [[TMP18]], <6 x double> [[TMP19]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       if.else:
@@ -55,7 +55,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
 ; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <3 x double> [[TMP36]], double [[TMP37]], i64 2
 ; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <3 x double> [[TMP26]], <3 x double> [[TMP32]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <3 x double> [[TMP38]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <3 x double> [[TMP38]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <6 x double> [[TMP39]], <6 x double> [[TMP40]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
@@ -83,7 +83,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP50]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0
@@ -102,7 +102,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP58:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
-; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP60]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0
@@ -121,7 +121,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP68:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = fadd <1 x double> [[TMP66]], [[TMP68]]
-; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <3 x double> [[TMP61]], <3 x double> [[TMP70]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0
@@ -140,7 +140,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP78:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
 ; CHECK-NEXT:    [[TMP79:%.*]] = fadd <1 x double> [[TMP76]], [[TMP78]]
-; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP81:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP80]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0
@@ -159,7 +159,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP88:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
 ; CHECK-NEXT:    [[TMP89:%.*]] = fadd <1 x double> [[TMP86]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP91:%.*]] = shufflevector <3 x double> [[TMP81]], <3 x double> [[TMP90]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0
@@ -178,7 +178,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP98:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
 ; CHECK-NEXT:    [[TMP99:%.*]] = fadd <1 x double> [[TMP96]], [[TMP98]]
-; CHECK-NEXT:    [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP101:%.*]] = shufflevector <3 x double> [[TMP91]], <3 x double> [[TMP100]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0
@@ -197,7 +197,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP108:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
 ; CHECK-NEXT:    [[TMP109:%.*]] = fadd <1 x double> [[TMP106]], [[TMP108]]
-; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP111:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP110]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP112:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0
@@ -216,7 +216,7 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP118:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
 ; CHECK-NEXT:    [[TMP119:%.*]] = fadd <1 x double> [[TMP116]], [[TMP118]]
-; CHECK-NEXT:    [[TMP120:%.*]] = shufflevector <1 x double> [[TMP119]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP120:%.*]] = shufflevector <1 x double> [[TMP119]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP121:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> [[TMP120]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0
@@ -235,10 +235,10 @@ define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B,
 ; CHECK-NEXT:    [[SPLAT_SPLAT89:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT88]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP128:%.*]] = fmul <1 x double> [[BLOCK87]], [[SPLAT_SPLAT89]]
 ; CHECK-NEXT:    [[TMP129:%.*]] = fadd <1 x double> [[TMP126]], [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = shufflevector <1 x double> [[TMP129]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP130:%.*]] = shufflevector <1 x double> [[TMP129]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP131:%.*]] = shufflevector <3 x double> [[TMP121]], <3 x double> [[TMP130]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP132:%.*]] = shufflevector <3 x double> [[TMP71]], <3 x double> [[TMP101]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP133:%.*]] = shufflevector <3 x double> [[TMP131]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP133:%.*]] = shufflevector <3 x double> [[TMP131]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP134:%.*]] = shufflevector <6 x double> [[TMP132]], <6 x double> [[TMP133]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x double> [[TMP134]]
 ;
@@ -287,7 +287,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <3 x double> [[TMP16]], double [[TMP17]], i64 2
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x double> [[TMP18]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <3 x double> [[TMP18]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <6 x double> [[TMP19]], <6 x double> [[TMP20]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[A_FOO:%.*]] = call <9 x double> @foo(<9 x double> [[TMP21]])
 ; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
@@ -313,7 +313,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = fadd <1 x double> [[TMP26]], [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <1 x double> [[TMP29]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <1 x double> [[TMP29]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP30]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 0
@@ -332,7 +332,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x double> [[TMP31]], <3 x double> [[TMP40]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <3 x double> [[SPLIT6]], i64 0
@@ -351,7 +351,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <3 x double> [[TMP41]], <3 x double> [[TMP50]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 0
@@ -370,7 +370,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP58:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
-; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP60]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 0
@@ -389,7 +389,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP68:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = fadd <1 x double> [[TMP66]], [[TMP68]]
-; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <3 x double> [[TMP61]], <3 x double> [[TMP70]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <3 x double> [[SPLIT7]], i64 0
@@ -408,7 +408,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP78:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
 ; CHECK-NEXT:    [[TMP79:%.*]] = fadd <1 x double> [[TMP76]], [[TMP78]]
-; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP81:%.*]] = shufflevector <3 x double> [[TMP71]], <3 x double> [[TMP80]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 0
@@ -427,7 +427,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP88:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
 ; CHECK-NEXT:    [[TMP89:%.*]] = fadd <1 x double> [[TMP86]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP91:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP90]], <3 x i32> <i32 3, i32 1, i32 2>
 ; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 0
@@ -446,7 +446,7 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP98:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
 ; CHECK-NEXT:    [[TMP99:%.*]] = fadd <1 x double> [[TMP96]], [[TMP98]]
-; CHECK-NEXT:    [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP101:%.*]] = shufflevector <3 x double> [[TMP91]], <3 x double> [[TMP100]], <3 x i32> <i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[SPLIT3]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <3 x double> [[SPLIT8]], i64 0
@@ -465,10 +465,10 @@ define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B
 ; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP108:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
 ; CHECK-NEXT:    [[TMP109:%.*]] = fadd <1 x double> [[TMP106]], [[TMP108]]
-; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP111:%.*]] = shufflevector <3 x double> [[TMP101]], <3 x double> [[TMP110]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP112:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP81]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP113:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP113:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP114:%.*]] = shufflevector <6 x double> [[TMP112]], <6 x double> [[TMP113]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x double> [[TMP114]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
index ab600462aead9..ae7da19e1641e 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
@@ -14,7 +14,7 @@ define <9 x double> @strided_load_3x3(ptr %in, i64 %stride) {
 ; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr double, ptr [[IN]], i64 [[VEC_START5]]
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP6]], align 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x double> [[COL_LOAD]], <3 x double> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <6 x double> [[TMP0]], <6 x double> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x double> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
index 6e54684fbc2c9..c52dbdc15be95 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
@@ -14,7 +14,7 @@ define <9 x float> @strided_load_3x3(ptr %in, i64 %stride) {
 ; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr float, ptr [[IN]], i64 [[VEC_START5]]
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x float>, ptr [[VEC_GEP6]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[COL_LOAD]], <3 x float> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[COL_LOAD8]], <3 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[COL_LOAD8]], <3 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x float> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x float> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
index baf6cb7853fa2..469e0cbb2612a 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
@@ -14,7 +14,7 @@ define <9 x i32> @strided_load_3x3(ptr %in, i64 %stride) {
 ; CHECK-NEXT:    [[VEC_GEP6:%.*]] = getelementptr i32, ptr [[IN]], i64 [[VEC_START5]]
 ; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x i32>, ptr [[VEC_GEP6]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i32> [[COL_LOAD]], <3 x i32> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[COL_LOAD8]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[COL_LOAD8]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <6 x i32> [[TMP0]], <6 x i32> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    ret <9 x i32> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll
index fd17aa253b585..afb3f540da6fc 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-double.ll
@@ -103,7 +103,7 @@ define <12 x double> @transpose_double_3x4(<12 x double> %a) {
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x double> [[TMP21]], double [[TMP22]], i64 3
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x double> [[TMP24]], <8 x double> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <12 x double> [[TMP26]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll
index d2fa4378cbb2f..26d15b4c3eb75 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-float.ll
@@ -103,7 +103,7 @@ define <12 x float> @transpose_float_3x4(<12 x float> %a) {
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x float> [[SPLIT3]], i64 2
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP22]], i64 3
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x float> [[TMP24]], <8 x float> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <12 x float> [[TMP26]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll
index 5e2dec6ce312d..40ea6a9cb0be8 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-i32.ll
@@ -103,7 +103,7 @@ define <12 x i32> @transpose_i32_3x4(<12 x i32> %a) {
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x i32> [[SPLIT3]], i64 2
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP22]], i64 3
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP23]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP25]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <12 x i32> [[TMP26]]
 ;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-iterator-invalidation.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-iterator-invalidation.ll
index c27d965fc217d..90d3b54bb4e5a 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-iterator-invalidation.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-iterator-invalidation.ll
@@ -20,7 +20,7 @@ define <2 x double> @test(<4 x double> %a, <2 x double> %b, i1 %c) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT4]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <1 x double> [[BLOCK3]], [[SPLAT_SPLAT5]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <1 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP5]], <2 x i32> <i32 2, i32 1>
 ; CHECK-NEXT:    [[BLOCK6:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -33,7 +33,7 @@ define <2 x double> @test(<4 x double> %a, <2 x double> %b, i1 %c) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd <1 x double> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <1 x double> [[TMP11]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[TMP13]], i64 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <1 x double> poison, double [[TMP14]], i64 0
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
index 5a0fbf1f5dca9..036e68b124277 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
@@ -79,7 +79,7 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[VEC_GEP41:%.*]] = getelementptr double, ptr [[A]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[VEC_GEP41]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> [[COL_LOAD40]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD43]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD43]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[COL_LOAD45:%.*]] = load <2 x double>, ptr [[B:%.*]], align 16
 ; CHECK-NEXT:    [[VEC_GEP46:%.*]] = getelementptr double, ptr [[B]], i64 2
@@ -100,7 +100,7 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT61]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul <2 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = fadd <2 x double> [[TMP7]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP10]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x double> [[TMP10]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP11]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <4 x double> [[COL_LOAD53]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x double> [[COL_LOAD45]], i64 0
@@ -113,7 +113,7 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT67]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = fmul <2 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = fadd <2 x double> [[TMP14]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <4 x double> [[COL_LOAD53]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[COL_LOAD48]], i64 0
@@ -126,7 +126,7 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT73]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP23:%.*]] = fmul <2 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = fadd <2 x double> [[TMP21]], [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <2 x double> [[TMP24]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <2 x double> [[TMP24]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP25]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <4 x double> [[COL_LOAD53]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <2 x double> [[COL_LOAD48]], i64 0
@@ -139,7 +139,7 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT79]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul <2 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = fadd <2 x double> [[TMP28]], [[TMP30]]
-; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <4 x double> [[TMP26]], <4 x double> [[TMP32]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <4 x double> [[COL_LOAD53]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <2 x double> [[COL_LOAD51]], i64 0
@@ -152,7 +152,7 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT85]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul <2 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = fadd <2 x double> [[TMP35]], [[TMP37]]
-; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP39]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK87:%.*]] = shufflevector <4 x double> [[COL_LOAD53]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x double> [[COL_LOAD51]], i64 0
@@ -165,10 +165,10 @@ define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT91]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP44:%.*]] = fmul <2 x double> [[BLOCK90]], [[SPLAT_SPLAT92]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = fadd <2 x double> [[TMP42]], [[TMP44]]
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <2 x double> [[TMP45]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <2 x double> [[TMP45]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <4 x double> [[TMP40]], <4 x double> [[TMP46]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP49:%.*]] = shufflevector <4 x double> [[TMP47]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP49:%.*]] = shufflevector <4 x double> [[TMP47]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <8 x double> [[TMP48]], <8 x double> [[TMP49]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> poison, <2 x i32> <i32 2, i32 3>
@@ -322,7 +322,7 @@ define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP6]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP10]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x double> [[COL_LOAD7]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
@@ -341,7 +341,7 @@ define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT27]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <1 x double> [[BLOCK26]], [[SPLAT_SPLAT28]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fadd <1 x double> [[TMP16]], [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <1 x double> [[TMP19]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <1 x double> [[TMP19]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> [[TMP20]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK29:%.*]] = shufflevector <3 x double> [[COL_LOAD7]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
@@ -360,7 +360,7 @@ define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT36]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul <2 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = fadd <2 x double> [[TMP26]], [[TMP28]]
-; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP30]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x double> [[COL_LOAD7]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
@@ -379,7 +379,7 @@ define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = fmul <1 x double> [[BLOCK44]], [[SPLAT_SPLAT46]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <3 x double> [[TMP31]], <3 x double> [[TMP40]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK47:%.*]] = shufflevector <3 x double> [[COL_LOAD7]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
@@ -398,7 +398,7 @@ define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT54]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul <2 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = fadd <2 x double> [[TMP46]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <2 x double> [[TMP49]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <2 x double> [[TMP49]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP50]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK56:%.*]] = shufflevector <3 x double> [[COL_LOAD7]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
@@ -417,7 +417,7 @@ define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT64:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT63]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP58:%.*]] = fmul <1 x double> [[BLOCK62]], [[SPLAT_SPLAT64]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
-; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP60]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    store <3 x double> [[TMP21]], ptr [[C:%.*]], align 16
 ; CHECK-NEXT:    [[VEC_GEP66:%.*]] = getelementptr double, ptr [[C]], i64 3
@@ -465,7 +465,7 @@ define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[VEC_GEP41:%.*]] = getelementptr double, ptr [[A]], i64 4
 ; CHECK-NEXT:    [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[VEC_GEP41]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> [[COL_LOAD40]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD43]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD43]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
 ; CHECK-NEXT:    [[COL_LOAD45:%.*]] = load <4 x double>, ptr [[B:%.*]], align 16
 ; CHECK-NEXT:    [[VEC_GEP46:%.*]] = getelementptr double, ptr [[B]], i64 4
@@ -473,7 +473,7 @@ define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[VEC_GEP49:%.*]] = getelementptr double, ptr [[B]], i64 8
 ; CHECK-NEXT:    [[COL_LOAD51:%.*]] = load <4 x double>, ptr [[VEC_GEP49]], align 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[COL_LOAD45]], <4 x double> [[COL_LOAD48]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[COL_LOAD51]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[COL_LOAD51]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP6]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <12 x double> [[TMP7]], <12 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <12 x double> [[TMP7]], <12 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -498,7 +498,7 @@ define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT10]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = fmul <2 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = fadd <2 x double> [[TMP12]], [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP16]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -517,7 +517,7 @@ define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT19]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = fmul <2 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fadd <2 x double> [[TMP22]], [[TMP24]]
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <2 x double> [[TMP25]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <2 x double> [[TMP25]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> [[TMP26]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
@@ -536,7 +536,7 @@ define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT28]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP34:%.*]] = fmul <2 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = fadd <2 x double> [[TMP32]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <2 x double> [[TMP35]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <2 x double> [[TMP35]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP36]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
@@ -555,7 +555,7 @@ define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT37]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP44:%.*]] = fmul <2 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = fadd <2 x double> [[TMP42]], [[TMP44]]
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <2 x double> [[TMP45]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <2 x double> [[TMP45]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> [[TMP46]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    store <4 x double> [[TMP27]], ptr [[C:%.*]], align 16
 ; CHECK-NEXT:    [[VEC_GEP53:%.*]] = getelementptr double, ptr [[C]], i64 4
@@ -602,7 +602,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[VEC_GEP60:%.*]] = getelementptr double, ptr [[A]], i64 6
 ; CHECK-NEXT:    [[COL_LOAD62:%.*]] = load <3 x double>, ptr [[VEC_GEP60]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD]], <3 x double> [[COL_LOAD59]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <3 x double> [[COL_LOAD62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <3 x double> [[COL_LOAD62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <6 x double> [[TMP1]], <6 x double> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[COL_LOAD64:%.*]] = load <3 x double>, ptr [[B:%.*]], align 16
 ; CHECK-NEXT:    [[VEC_GEP65:%.*]] = getelementptr double, ptr [[B]], i64 3
@@ -631,7 +631,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT87:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT86]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = fmul <2 x double> [[BLOCK85]], [[SPLAT_SPLAT87]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = fadd <2 x double> [[TMP10]], [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP14]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK88:%.*]] = shufflevector <3 x double> [[COL_LOAD72]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD64]], i64 0
@@ -650,7 +650,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT96:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT95]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = fmul <1 x double> [[BLOCK94]], [[SPLAT_SPLAT96]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = fadd <1 x double> [[TMP20]], [[TMP22]]
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <1 x double> [[TMP23]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <1 x double> [[TMP23]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <3 x double> [[TMP15]], <3 x double> [[TMP24]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK97:%.*]] = shufflevector <3 x double> [[COL_LOAD72]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <3 x double> [[COL_LOAD67]], i64 0
@@ -669,7 +669,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT105:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT104]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP32:%.*]] = fmul <2 x double> [[BLOCK103]], [[SPLAT_SPLAT105]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = fadd <2 x double> [[TMP30]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP34]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK106:%.*]] = shufflevector <3 x double> [[COL_LOAD72]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <3 x double> [[COL_LOAD67]], i64 0
@@ -688,7 +688,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT114:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT113]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP42:%.*]] = fmul <1 x double> [[BLOCK112]], [[SPLAT_SPLAT114]]
 ; CHECK-NEXT:    [[TMP43:%.*]] = fadd <1 x double> [[TMP40]], [[TMP42]]
-; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <1 x double> [[TMP43]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <1 x double> [[TMP43]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP45:%.*]] = shufflevector <3 x double> [[TMP35]], <3 x double> [[TMP44]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK115:%.*]] = shufflevector <3 x double> [[COL_LOAD72]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <3 x double> [[COL_LOAD70]], i64 0
@@ -707,7 +707,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT123:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT122]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul <2 x double> [[BLOCK121]], [[SPLAT_SPLAT123]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = fadd <2 x double> [[TMP50]], [[TMP52]]
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <2 x double> [[TMP53]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <2 x double> [[TMP53]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP54]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK124:%.*]] = shufflevector <3 x double> [[COL_LOAD72]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP56:%.*]] = extractelement <3 x double> [[COL_LOAD70]], i64 0
@@ -726,10 +726,10 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT132:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT131]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP62:%.*]] = fmul <1 x double> [[BLOCK130]], [[SPLAT_SPLAT132]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = fadd <1 x double> [[TMP60]], [[TMP62]]
-; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <1 x double> [[TMP63]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <1 x double> [[TMP63]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <3 x double> [[TMP55]], <3 x double> [[TMP64]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <3 x double> [[TMP25]], <3 x double> [[TMP45]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP65]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP65]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <6 x double> [[TMP66]], <6 x double> [[TMP67]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
 ; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[TMP68]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <9 x double> [[TMP68]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
@@ -754,7 +754,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP75:%.*]] = fmul <2 x double> [[BLOCK10]], [[SPLAT_SPLAT12]]
 ; CHECK-NEXT:    [[TMP76:%.*]] = fadd <2 x double> [[TMP73]], [[TMP75]]
-; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <2 x double> [[TMP76]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <2 x double> [[TMP76]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP78:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP77]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK13:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP79:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0
@@ -773,7 +773,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP85:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]]
 ; CHECK-NEXT:    [[TMP86:%.*]] = fadd <1 x double> [[TMP83]], [[TMP85]]
-; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <1 x double> [[TMP86]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <1 x double> [[TMP86]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP88:%.*]] = shufflevector <3 x double> [[TMP78]], <3 x double> [[TMP87]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK22:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP89:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1
@@ -792,7 +792,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT29]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP95:%.*]] = fmul <2 x double> [[BLOCK28]], [[SPLAT_SPLAT30]]
 ; CHECK-NEXT:    [[TMP96:%.*]] = fadd <2 x double> [[TMP93]], [[TMP95]]
-; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <2 x double> [[TMP96]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <2 x double> [[TMP96]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP98:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP97]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK31:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1
@@ -811,7 +811,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT39:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP105:%.*]] = fmul <1 x double> [[BLOCK37]], [[SPLAT_SPLAT39]]
 ; CHECK-NEXT:    [[TMP106:%.*]] = fadd <1 x double> [[TMP103]], [[TMP105]]
-; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <1 x double> [[TMP106]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <1 x double> [[TMP106]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP108:%.*]] = shufflevector <3 x double> [[TMP98]], <3 x double> [[TMP107]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    [[BLOCK40:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP109:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2
@@ -830,7 +830,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT48:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT47]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP115:%.*]] = fmul <2 x double> [[BLOCK46]], [[SPLAT_SPLAT48]]
 ; CHECK-NEXT:    [[TMP116:%.*]] = fadd <2 x double> [[TMP113]], [[TMP115]]
-; CHECK-NEXT:    [[TMP117:%.*]] = shufflevector <2 x double> [[TMP116]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP117:%.*]] = shufflevector <2 x double> [[TMP116]], <2 x double> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP118:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP117]], <3 x i32> <i32 3, i32 4, i32 2>
 ; CHECK-NEXT:    [[BLOCK49:%.*]] = shufflevector <3 x double> [[SPLIT]], <3 x double> poison, <1 x i32> <i32 2>
 ; CHECK-NEXT:    [[TMP119:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2
@@ -849,7 +849,7 @@ define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT57:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT56]], <1 x double> poison, <1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP125:%.*]] = fmul <1 x double> [[BLOCK55]], [[SPLAT_SPLAT57]]
 ; CHECK-NEXT:    [[TMP126:%.*]] = fadd <1 x double> [[TMP123]], [[TMP125]]
-; CHECK-NEXT:    [[TMP127:%.*]] = shufflevector <1 x double> [[TMP126]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP127:%.*]] = shufflevector <1 x double> [[TMP126]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP128:%.*]] = shufflevector <3 x double> [[TMP118]], <3 x double> [[TMP127]], <3 x i32> <i32 0, i32 1, i32 3>
 ; CHECK-NEXT:    store <3 x double> [[TMP88]], ptr [[R:%.*]], align 16
 ; CHECK-NEXT:    [[VEC_GEP134:%.*]] = getelementptr double, ptr [[R]], i64 3
@@ -961,7 +961,7 @@ define <12 x double> @factor_transpose(<6 x double> %a, <8 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT6]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[BLOCK5]], [[SPLAT_SPLAT7]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP6]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK8:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT2]], i64 0
@@ -974,7 +974,7 @@ define <12 x double> @factor_transpose(<6 x double> %a, <8 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT12]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul <2 x double> [[BLOCK11]], [[SPLAT_SPLAT13]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP9]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -987,7 +987,7 @@ define <12 x double> @factor_transpose(<6 x double> %a, <8 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <2 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fadd <2 x double> [[TMP16]], [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP20]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
@@ -1000,7 +1000,7 @@ define <12 x double> @factor_transpose(<6 x double> %a, <8 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT24]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP25:%.*]] = fmul <2 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = fadd <2 x double> [[TMP23]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
@@ -1013,7 +1013,7 @@ define <12 x double> @factor_transpose(<6 x double> %a, <8 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT30]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP32:%.*]] = fmul <2 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = fadd <2 x double> [[TMP30]], [[TMP32]]
-; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x double> undef, <4 x double> [[TMP34]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <4 x double> [[SPLIT]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
@@ -1026,7 +1026,7 @@ define <12 x double> @factor_transpose(<6 x double> %a, <8 x double> %b) {
 ; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT36]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP39:%.*]] = fmul <2 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = fadd <2 x double> [[TMP37]], [[TMP39]]
-; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <2 x double> [[TMP40]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <2 x double> [[TMP40]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP42:%.*]] = shufflevector <4 x double> [[TMP35]], <4 x double> [[TMP41]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <4 x double> [[TMP14]], i64 0
 ; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <3 x double> poison, double [[TMP43]], i64 0
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/basic.ll b/llvm/test/Transforms/MemProfContextDisambiguation/basic.ll
index 7a48d66af47e0..99a8d68a5b1d2 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/basic.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/basic.ll
@@ -26,11 +26,14 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -passes=memprof-context-disambiguation \
 ; RUN:	-memprof-verify-ccg -memprof-verify-nodes -memprof-dump-ccg \
 ; RUN:	-memprof-export-to-dot -memprof-dot-file-path-prefix=%t. \
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll b/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll
index aa5f539a61832..143f892c18950 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids.ll
@@ -38,7 +38,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The code below was created by forcing inlining of C into both B and E.
@@ -52,6 +52,9 @@
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -passes=memprof-context-disambiguation \
 ; RUN:  -memprof-verify-ccg -memprof-verify-nodes -memprof-dump-ccg \
 ; RUN:  -memprof-export-to-dot -memprof-dot-file-path-prefix=%t. \
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids2.ll b/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids2.ll
index da0fd3f44b45e..d1659b524860a 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids2.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/duplicate-context-ids2.ll
@@ -76,7 +76,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The code below was created by forcing inlining of A into its callers,
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/indirectcall.ll b/llvm/test/Transforms/MemProfContextDisambiguation/indirectcall.ll
index e66ec0d76fe8b..49ca9407d9250 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/indirectcall.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/indirectcall.ll
@@ -44,13 +44,16 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; Compiled without optimization to prevent inlining and devirtualization.
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -passes=memprof-context-disambiguation \
 ; RUN:  -memprof-verify-ccg -memprof-verify-nodes -memprof-dump-ccg \
 ; RUN:  -memprof-export-to-dot -memprof-dot-file-path-prefix=%t. \
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/inlined.ll b/llvm/test/Transforms/MemProfContextDisambiguation/inlined.ll
index a2fa703e04e7b..70a6f39980ede 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/inlined.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/inlined.ll
@@ -27,7 +27,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; The code below was created by forcing inlining of baz into foo, and
@@ -40,6 +40,9 @@
 ;;
 ;; The IR was then reduced using llvm-reduce with the expected FileCheck input.
 
+;; -stats requires asserts
+; REQUIRES: asserts
+
 ; RUN: opt -passes=memprof-context-disambiguation \
 ; RUN:	-memprof-verify-ccg -memprof-verify-nodes -memprof-dump-ccg \
 ; RUN:	-memprof-export-to-dot -memprof-dot-file-path-prefix=%t. \
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll b/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll
index a3a056ade8c49..5a247f8607996 100644
--- a/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/inlined2.ll
@@ -29,7 +29,7 @@
 ;;   return 0;
 ;; }
 ;;
-;; Code compiled with -mllvm -memprof-min-lifetime-cold-threshold=5 so that the
+;; Code compiled with -mllvm -memprof-ave-lifetime-cold-threshold=5 so that the
 ;; memory freed after sleep(10) results in cold lifetimes.
 ;;
 ;; Both foo and baz are inlined into main, at both foo callsites.
diff --git a/llvm/test/Transforms/MoveAutoInit/branch.ll b/llvm/test/Transforms/MoveAutoInit/branch.ll
new file mode 100644
index 0000000000000..6c51e33c8c8e9
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/branch.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+@__const.foo.buffer = private unnamed_addr constant [8 x i32] [i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766], align 16
+
+define void @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BUFFER:%.*]] = alloca [8 x i32], align 16
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[BUFFER]], ptr align 16 @__const.foo.buffer, i64 32, i1 false), !annotation !0
+; CHECK-NEXT:    call void @dump(ptr [[BUFFER]])
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %buffer = alloca [8 x i32], align 16
+  call void @llvm.memcpy.p0.p0.i64(ptr align 16 %buffer, ptr align 16 @__const.foo.buffer, i64 32, i1 false), !annotation !0
+  %tobool = icmp ne i32 %x, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @dump(ptr %buffer)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+
+declare void @dump(ptr)
+
+!0 = !{!"auto-init"}
+
diff --git a/llvm/test/Transforms/MoveAutoInit/clobber.ll b/llvm/test/Transforms/MoveAutoInit/clobber.ll
new file mode 100644
index 0000000000000..0d70d85119f50
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/clobber.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Checks that move-auto-init can move instruction passed unclobbering memory
+; instructions.
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @foo(i32 noundef %0, i32 noundef %1, i32 noundef %2) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP4:%.*]] = alloca [100 x i8], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = alloca [2 x i8], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [100 x i8], ptr [[TMP4]], i64 0, i64 0
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 100, ptr nonnull [[TMP6]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 0
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr nonnull [[TMP7]]) #[[ATTR3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP1:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP15:%.*]], label [[TMP10:%.*]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 dereferenceable(100) [[TMP6]], i8 -86, i64 100, i1 false), !annotation !0
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP0:%.*]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [100 x i8], ptr [[TMP4]], i64 0, i64 [[TMP11]]
+; CHECK-NEXT:    store i8 12, ptr [[TMP12]], align 1
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i8 [[TMP13]] to i32
+; CHECK-NEXT:    br label [[TMP22:%.*]]
+; CHECK:       15:
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP2:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP16]], label [[TMP22]], label [[TMP17:%.*]]
+; CHECK:       17:
+; CHECK-NEXT:    store i8 -86, ptr [[TMP7]], align 1, !annotation !0
+; CHECK-NEXT:    store i8 -86, ptr [[TMP8]], align 1, !annotation !0
+; CHECK-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 [[TMP18]]
+; CHECK-NEXT:    store i8 12, ptr [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = sext i8 [[TMP20]] to i32
+; CHECK-NEXT:    br label [[TMP22]]
+; CHECK:       22:
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i32 [ [[TMP14]], [[TMP10]] ], [ [[TMP21]], [[TMP17]] ], [ 0, [[TMP15]] ]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr nonnull [[TMP7]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 100, ptr nonnull [[TMP6]]) #[[ATTR3]]
+; CHECK-NEXT:    ret i32 [[TMP23]]
+;
+
+  %4 = alloca [100 x i8], align 16
+  %5 = alloca [2 x i8], align 1
+  %6 = getelementptr inbounds [100 x i8], [100 x i8]* %4, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 100, i8* nonnull %6) #3
+  ; This memset must move.
+  call void @llvm.memset.p0i8.i64(i8* noundef nonnull align 16 dereferenceable(100) %6, i8 -86, i64 100, i1 false), !annotation !0
+  %7 = getelementptr inbounds [2 x i8], [2 x i8]* %5, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* nonnull %7) #3
+  ; This store must move.
+  store i8 -86, i8* %7, align 1, !annotation !0
+  %8 = getelementptr inbounds [2 x i8], [2 x i8]* %5, i64 0, i64 1
+  ; This store must move.
+  store i8 -86, i8* %8, align 1, !annotation !0
+  %9 = icmp eq i32 %1, 0
+  br i1 %9, label %15, label %10
+
+10:
+  %11 = sext i32 %0 to i64
+  %12 = getelementptr inbounds [100 x i8], [100 x i8]* %4, i64 0, i64 %11
+  store i8 12, i8* %12, align 1
+  %13 = load i8, i8* %6, align 16
+  %14 = sext i8 %13 to i32
+  br label %22
+
+15:
+  %16 = icmp eq i32 %2, 0
+  br i1 %16, label %22, label %17
+
+17:
+  %18 = sext i32 %0 to i64
+  %19 = getelementptr inbounds [2 x i8], [2 x i8]* %5, i64 0, i64 %18
+  store i8 12, i8* %19, align 1
+  %20 = load i8, i8* %7, align 1
+  %21 = sext i8 %20 to i32
+  br label %22
+
+22:
+  %23 = phi i32 [ %14, %10 ], [ %21, %17 ], [ 0, %15 ]
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* nonnull %7) #3
+  call void @llvm.lifetime.end.p0i8(i64 100, i8* nonnull %6) #3
+  ret i32 %23
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #2
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { mustprogress nofree nosync nounwind readnone uwtable willreturn }
+attributes #1 = { argmemonly mustprogress nofree nosync nounwind willreturn }
+attributes #2 = { argmemonly mustprogress nofree nounwind willreturn writeonly }
+attributes #3 = { nounwind }
+
+!0 = !{!"auto-init"}
diff --git a/llvm/test/Transforms/MoveAutoInit/fence.ll b/llvm/test/Transforms/MoveAutoInit/fence.ll
new file mode 100644
index 0000000000000..bce9c136942e9
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/fence.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; In that case, the store to %val happens before the fence and cannot go past
+; it.
+define void @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 -1431655766, ptr [[VAL]], align 4, !annotation !0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    fence acquire
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @dump(ptr [[VAL]])
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %val = alloca i32, align 4
+  store i32 -1431655766, ptr %val, align 4, !annotation !0
+  %tobool = icmp ne i32 %x, 0
+  fence acquire
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @dump(ptr %val)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+; In that case, the store to %val happens after the fence and it is moved within
+; the true branch as expected.
+define void @bar(i32 %x) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    fence acquire
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 -1431655766, ptr [[VAL]], align 4, !annotation !0
+; CHECK-NEXT:    call void @dump(ptr [[VAL]])
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %val = alloca i32, align 4
+  %tobool = icmp ne i32 %x, 0
+  fence acquire
+  store i32 -1431655766, ptr %val, align 4, !annotation !0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @dump(ptr %val)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+declare void @dump(ptr)
+
+!0 = !{!"auto-init"}
diff --git a/llvm/test/Transforms/MoveAutoInit/loop.ll b/llvm/test/Transforms/MoveAutoInit/loop.ll
new file mode 100644
index 0000000000000..71153e58f4e35
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/loop.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BUFFER:%.*]] = alloca [80 x i32], align 16
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[BUFFER]], i8 -86, i64 320, i1 false), !annotation !0
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[X_ADDR_0:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[DEC:%.*]], [[DO_COND:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [80 x i32], ptr [[BUFFER]], i64 0, i64 0
+; CHECK-NEXT:    call void @dump(ptr [[ARRAYIDX]])
+; CHECK-NEXT:    br label [[DO_COND]]
+; CHECK:       do.cond:
+; CHECK-NEXT:    [[DEC]] = add nsw i32 [[X_ADDR_0]], -1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[X_ADDR_0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[DO_BODY]], label [[DO_END:%.*]]
+; CHECK:       do.end:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %buffer = alloca [80 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %buffer, i8 -86, i64 320, i1 false), !annotation !0
+  br label %do.body
+
+do.body:                                          ; preds = %do.cond, %entry
+  %x.addr.0 = phi i32 [ %x, %entry ], [ %dec, %do.cond ]
+  %arrayidx = getelementptr inbounds [80 x i32], ptr %buffer, i64 0, i64 0
+  call void @dump(ptr %arrayidx)
+  br label %do.cond
+
+do.cond:                                          ; preds = %do.body
+  %dec = add nsw i32 %x.addr.0, -1
+  %tobool = icmp ne i32 %x.addr.0, 0
+  br i1 %tobool, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.cond
+  ret void
+}
+
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
+
+declare void @dump(ptr )
+
+define void @bar(i32 %x, i32 %y) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BUFFER:%.*]] = alloca [80 x i32], align 16
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[BUFFER]], i8 -86, i64 320, i1 false), !annotation !0
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[X_ADDR_0:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[DEC:%.*]], [[DO_COND:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [80 x i32], ptr [[BUFFER]], i64 0, i64 0
+; CHECK-NEXT:    call void @dump(ptr [[ARRAYIDX]])
+; CHECK-NEXT:    br label [[DO_COND]]
+; CHECK:       do.cond:
+; CHECK-NEXT:    [[DEC]] = add nsw i32 [[X_ADDR_0]], -1
+; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp ne i32 [[X_ADDR_0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[DO_BODY]], label [[DO_END:%.*]]
+; CHECK:       do.end:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %buffer = alloca [80 x i32], align 16
+  call void @llvm.memset.p0.i64(ptr align 16 %buffer, i8 -86, i64 320, i1 false), !annotation !0
+  %tobool = icmp ne i32 %y, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %add = add nsw i32 %x, %y
+  br label %do.body
+
+do.body:                                          ; preds = %do.cond, %if.then
+  %x.addr.0 = phi i32 [ %add, %if.then ], [ %dec, %do.cond ]
+  %arrayidx = getelementptr inbounds [80 x i32], ptr %buffer, i64 0, i64 0
+  call void @dump(ptr %arrayidx)
+  br label %do.cond
+
+do.cond:                                          ; preds = %do.body
+  %dec = add nsw i32 %x.addr.0, -1
+  %tobool1 = icmp ne i32 %x.addr.0, 0
+  br i1 %tobool1, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.cond
+  br label %if.end
+
+if.end:                                           ; preds = %do.end, %entry
+  ret void
+}
+
+!0 = !{!"auto-init"}
diff --git a/llvm/test/Transforms/MoveAutoInit/scalar.ll b/llvm/test/Transforms/MoveAutoInit/scalar.ll
new file mode 100644
index 0000000000000..6929aebc58ebb
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/scalar.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 -1431655766, ptr [[VAL]], align 4, !annotation !0
+; CHECK-NEXT:    call void @dump(ptr [[VAL]])
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %val = alloca i32, align 4
+  store i32 -1431655766, ptr %val, align 4, !annotation !0
+  %tobool = icmp ne i32 %x, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @dump(ptr %val)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+declare void @dump(ptr)
+
+!0 = !{!"auto-init"}
diff --git a/llvm/test/Transforms/MoveAutoInit/sret.ll b/llvm/test/Transforms/MoveAutoInit/sret.ll
new file mode 100644
index 0000000000000..204259e6fa121
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/sret.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+; Checks that auto-init memory isntruction are mot moved when writing to an sret argument.
+
+target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+
+%struct.S = type { i64 }
+
+@pattern = private unnamed_addr constant %struct.S { i64 -1 }, align 4
+
+define void @f(ptr noalias sret(%struct.S) align 4 %0, i32 noundef %1) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[TMP3:%.*]] = alloca ptr, align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = alloca [[STRUCT_S:%.*]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast ptr [[TMP0:%.*]] to ptr
+; CHECK-NEXT:    store ptr [[TMP6]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    store i32 [[TMP1:%.*]], ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast ptr [[TMP0]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[TMP7]], ptr align 4 @pattern, i32 8, i1 false), !annotation !0
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 42
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP13:%.*]]
+; CHECK:       10:
+; CHECK-NEXT:    call void @g(ptr sret([[STRUCT_S]]) align 4 [[TMP5]])
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast ptr [[TMP0]] to ptr
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast ptr [[TMP5]] to ptr
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[TMP11]], ptr align 4 [[TMP12]], i32 8, i1 false)
+; CHECK-NEXT:    br label [[TMP13]]
+; CHECK:       13:
+; CHECK-NEXT:    ret void
+;
+  %3 = alloca ptr, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca %struct.S, align 4
+  %6 = bitcast ptr %0 to ptr
+  store ptr %6, ptr %3, align 4
+  store i32 %1, ptr %4, align 4
+  %7 = bitcast ptr %0 to ptr
+  call void @llvm.memcpy.p0.p0.i32(ptr align 4 %7, ptr align 4 @pattern, i32 8, i1 false), !annotation !0
+  %8 = load i32, ptr %4, align 4
+  %9 = icmp eq i32 %8, 42
+  br i1 %9, label %10, label %13
+
+10:                                               ; preds = %2
+  call void @g(ptr sret(%struct.S) align 4 %5)
+  %11 = bitcast ptr %0 to ptr
+  %12 = bitcast ptr %5 to ptr
+  call void @llvm.memcpy.p0.p0.i32(ptr align 4 %11, ptr align 4 %12, i32 8, i1 false)
+  br label %13
+
+13:                                               ; preds = %10, %2
+  ret void
+}
+
+declare void @g(ptr sret(%struct.S) align 4, ...)
+
+declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) #0
+
+!0 = !{!"auto-init"}
+
diff --git a/llvm/test/Transforms/MoveAutoInit/volatile.ll b/llvm/test/Transforms/MoveAutoInit/volatile.ll
new file mode 100644
index 0000000000000..da3cd067ff4fe
--- /dev/null
+++ b/llvm/test/Transforms/MoveAutoInit/volatile.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s
+
+; Make sure that volatile operationsa re not moved. This is overly conservative.
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store volatile i32 -1431655766, ptr [[VAL]], align 4, !annotation !0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @dump(ptr [[VAL]])
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %val = alloca i32, align 4
+  store volatile i32 -1431655766, ptr %val, align 4, !annotation !0
+  %tobool = icmp ne i32 %x, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  call void @dump(ptr %val)
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+declare void @dump(ptr)
+
+!0 = !{!"auto-init"}
diff --git a/llvm/test/Transforms/OpenMP/attributor_pointer_offset_crash.ll b/llvm/test/Transforms/OpenMP/attributor_pointer_offset_crash.ll
index e684f65cf2e0c..d2ac2e6e30a40 100644
--- a/llvm/test/Transforms/OpenMP/attributor_pointer_offset_crash.ll
+++ b/llvm/test/Transforms/OpenMP/attributor_pointer_offset_crash.ll
@@ -2,7 +2,7 @@
 
 ; Verify the address space cast doesn't cause a crash
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 
 %"struct.(anonymous namespace)::TeamStateTy" = type { %"struct.(anonymous namespace)::ICVStateTy", i32, ptr }
 %"struct.(anonymous namespace)::ICVStateTy" = type { i32, i32, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll b/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
index c7c34d342964d..dce799104e70b 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
@@ -8,7 +8,7 @@
 ; CHECK: store i32 1, ptr addrspace(3) @IsSPMDMode
 ; CHECK-NOT: store i32 0, ptr addrspace(3) @IsSPMDMode
 ;
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 %struct.ident_t = type { i32, i32, i32, i32, ptr }
diff --git a/llvm/test/Transforms/OpenMP/values_in_offload_arrays.alloca.ll b/llvm/test/Transforms/OpenMP/values_in_offload_arrays.alloca.ll
index a16138be5481d..a933b7d393eef 100644
--- a/llvm/test/Transforms/OpenMP/values_in_offload_arrays.alloca.ll
+++ b/llvm/test/Transforms/OpenMP/values_in_offload_arrays.alloca.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=openmp-opt-cgscc -aa-pipeline=basic-aa -openmp-hide-memory-transfer-latency < %s | FileCheck %s
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 
 @.__omp_offloading_heavyComputation.region_id = weak constant i8 0
 @.offload_maptypes. = private unnamed_addr constant [2 x i64] [i64 35, i64 35]
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/mve-floatreduce.ll b/llvm/test/Transforms/PhaseOrdering/ARM/mve-floatreduce.ll
index dc3f3a92ad891..6cbba5ca2a6ad 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/mve-floatreduce.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/mve-floatreduce.ll
@@ -11,7 +11,7 @@ define arm_aapcs_vfpcc half @vecAddAcrossF16Mve(<8 x half> %in) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <8 x half> [[IN:%.*]], <8 x half> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast <8 x half> [[TMP0]], [[IN]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x half> [[TMP1]] to <4 x i32>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 3, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x half>
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast <8 x half> [[TMP1]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x half> [[TMP5]], i64 0
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
index 8948225cf1d70..dd5ff12fda613 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
@@ -12,9 +12,9 @@ define i32 @add_v4i32(ptr %p) #0 {
 ; CHECK-LABEL: @add_v4i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    ret i32 [[TMP2]]
@@ -50,11 +50,11 @@ define signext i16 @mul_v8i16(ptr %p) #0 {
 ; CHECK-LABEL: @mul_v8i16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2, !tbaa [[TBAA4:![0-9]+]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = mul <8 x i16> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = mul <8 x i16> [[BIN_RDX]], [[RDX_SHUF3]]
-; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <8 x i16> [[BIN_RDX4]], <8 x i16> poison, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <8 x i16> [[BIN_RDX4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX6:%.*]] = mul <8 x i16> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i16> [[BIN_RDX6]], i32 0
 ; CHECK-NEXT:    ret i16 [[TMP2]]
@@ -93,13 +93,13 @@ define signext i8 @or_v16i8(ptr %p) #0 {
 ; CHECK-LABEL: @or_v16i8(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[P:%.*]], align 1, !tbaa [[TBAA6:![0-9]+]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <16 x i8> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x i8> [[BIN_RDX]], <16 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x i8> [[BIN_RDX]], <16 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX5:%.*]] = or <16 x i8> [[BIN_RDX]], [[RDX_SHUF4]]
-; CHECK-NEXT:    [[RDX_SHUF6:%.*]] = shufflevector <16 x i8> [[BIN_RDX5]], <16 x i8> poison, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF6:%.*]] = shufflevector <16 x i8> [[BIN_RDX5]], <16 x i8> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX7:%.*]] = or <16 x i8> [[BIN_RDX5]], [[RDX_SHUF6]]
-; CHECK-NEXT:    [[RDX_SHUF8:%.*]] = shufflevector <16 x i8> [[BIN_RDX7]], <16 x i8> poison, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF8:%.*]] = shufflevector <16 x i8> [[BIN_RDX7]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX9:%.*]] = or <16 x i8> [[BIN_RDX7]], [[RDX_SHUF8]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i8> [[BIN_RDX9]], i32 0
 ; CHECK-NEXT:    ret i8 [[TMP2]]
@@ -138,9 +138,9 @@ define i32 @smin_v4i32(ptr %p) #0 {
 ; CHECK-LABEL: @smin_v4i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[RDX_SHUF]])
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX2:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[RDX_MINMAX]], <4 x i32> [[RDX_SHUF3]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[RDX_MINMAX2]], i32 0
 ; CHECK-NEXT:    ret i32 [[TMP2]]
@@ -189,9 +189,9 @@ define i32 @umax_v4i32(ptr %p) #0 {
 ; CHECK-LABEL: @umax_v4i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[RDX_SHUF]])
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX2:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[RDX_MINMAX]], <4 x i32> [[RDX_SHUF3]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[RDX_MINMAX2]], i32 0
 ; CHECK-NEXT:    ret i32 [[TMP2]]
@@ -240,9 +240,9 @@ define float @fadd_v4i32(ptr %p) #0 {
 ; CHECK-LABEL: @fadd_v4i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4, !tbaa [[TBAA7:![0-9]+]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[BIN_RDX5:%.*]] = fadd fast float 4.200000e+01, [[TMP2]]
@@ -279,9 +279,9 @@ define float @fmul_v4i32(ptr %p) #0 {
 ; CHECK-LABEL: @fmul_v4i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4, !tbaa [[TBAA7]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fmul fast <4 x float> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fmul fast <4 x float> [[BIN_RDX]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[BIN_RDX5:%.*]] = fmul fast float 1.000000e+00, [[TMP2]]
@@ -319,10 +319,10 @@ define float @fmin_v4f32(ptr %p) #0 {
 ; CHECK-LABEL: @fmin_v4f32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4, !tbaa [[TBAA7]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt <4 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select fast <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP1]], <4 x float> [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP4:%.*]] = fcmp fast olt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT5:%.*]] = select fast <4 x i1> [[RDX_MINMAX_CMP4]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT5]], i32 0
@@ -398,13 +398,13 @@ define float @findMax(ptr byval(<8 x float>) align 16 %0) {
 ; CHECK-LABEL: @findMax(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[V:%.*]] = load <8 x float>, ptr [[TMP0:%.*]], align 16, !tbaa [[TBAA0]]
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[V]], <8 x float> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[V]], <8 x float> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz ogt <8 x float> [[V]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[V]], <8 x float> [[RDX_SHUF]]
-; CHECK-NEXT:    [[RDX_SHUF8:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> poison, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF8:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP9:%.*]] = fcmp nnan ninf nsz ogt <8 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF8]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT10:%.*]] = select nnan ninf nsz <8 x i1> [[RDX_MINMAX_CMP9]], <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> [[RDX_SHUF8]]
-; CHECK-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT10]], <8 x float> poison, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT10]], <8 x float> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP12:%.*]] = fcmp nnan ninf nsz ogt <8 x float> [[RDX_MINMAX_SELECT10]], [[RDX_SHUF11]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT13:%.*]] = select nnan ninf nsz <8 x i1> [[RDX_MINMAX_CMP12]], <8 x float> [[RDX_MINMAX_SELECT10]], <8 x float> [[RDX_SHUF11]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT13]], i32 0
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
index 05ef21d5a1123..b1324a522fd92 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
@@ -15,7 +15,7 @@ define float @test_merge_allof_v4sf(<4 x float> %t) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP6]], i64 0
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[OR_COND]], float 0.000000e+00, float [[ADD]]
@@ -93,7 +93,7 @@ define float @test_merge_anyof_v4sf(<4 x float> %t) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
 ; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i8 [[TMP4]], 0
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[SHIFT]], [[T]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP5]], i64 0
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[DOTNOT]], float [[ADD]], float 0.000000e+00
@@ -172,7 +172,7 @@ define float @test_separate_allof_v4sf(<4 x float> %t) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP6]], i64 0
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[OR_COND]], float 0.000000e+00, float [[ADD]]
@@ -254,7 +254,7 @@ define float @test_separate_anyof_v4sf(<4 x float> %t) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
 ; CHECK-NEXT:    [[DOTNOT6:%.*]] = icmp eq i4 [[TMP3]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[DOTNOT]], [[DOTNOT6]]
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T_FR]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x float> [[TMP4]], i64 0
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[OR_COND]], float [[ADD]], float 0.000000e+00
@@ -336,7 +336,7 @@ define float @test_merge_allof_v4si(<4 x i32> %t) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
 ; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[ADD]] to float
@@ -408,7 +408,7 @@ define float @test_merge_anyof_v4si(<4 x i32> %t) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
 ; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i8 [[TMP4]], 0
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[SHIFT]], [[T]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP5]], i64 0
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[ADD]] to float
@@ -482,7 +482,7 @@ define i32 @test_separate_allof_v4si(<4 x i32> %t) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[T_FR]], <i32 256, i32 256, i32 256, i32 256>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i4 [[TMP4]], 0
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP6]], i64 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP5]], i32 0, i32 [[ADD]]
@@ -560,7 +560,7 @@ define i32 @test_separate_anyof_v4si(<4 x i32> %t) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt <4 x i32> [[T_FR]], <i32 255, i32 255, i32 255, i32 255>
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
 ; CHECK-NEXT:    [[DOTNOT6:%.*]] = icmp eq i4 [[TMP3]], 0
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T_FR]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw <4 x i32> [[T_FR]], [[SHIFT]]
 ; CHECK-NEXT:    [[ADD:%.*]] = extractelement <4 x i32> [[TMP4]], i64 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[DOTNOT6]], i32 [[ADD]], i32 0
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
index 6f7e6d8119b64..315b9eb0fe3da 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
@@ -24,9 +24,9 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 
 define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[SHIFT1]]
 ; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
 ; CHECK-NEXT:    ret i32 [[X210]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
index ee24944e5cec6..6db27e597a63f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
@@ -26,7 +26,7 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -219,7 +219,7 @@ define <4 x float> @exp_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.exp.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -299,7 +299,7 @@ define <4 x float> @log_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -472,7 +472,7 @@ define <4 x float> @sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -511,7 +511,7 @@ define <4 x float> @cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -999,7 +999,7 @@ define <4 x float> @int_cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll
index 2bac49a58cd47..24e16deacb3af 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions.ll
@@ -26,7 +26,7 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -219,7 +219,7 @@ define <4 x float> @exp_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.exp.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -299,7 +299,7 @@ define <4 x float> @log_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -472,7 +472,7 @@ define <4 x float> @sin_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -511,7 +511,7 @@ define <4 x float> @cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -999,7 +999,7 @@ define <4 x float> @int_cos_4x(ptr %a) {
 ; NOACCELERATE-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; NOACCELERATE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; NOACCELERATE-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.cos.v2f32(<2 x float> [[TMP3]])
-; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; NOACCELERATE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; NOACCELERATE-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; NOACCELERATE-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
index 94d12f23456f9..109c6a9eacbd3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
@@ -426,18 +426,18 @@ define i32 @reduce_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i8>, ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i8>, ptr [[ADD_PTR64]], align 1
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP17:%.*]] = zext <16 x i8> [[TMP16]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP19:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP19]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <16 x i8> [[TMP22]], <16 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i8> [[TMP21]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP19]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <16 x i8> [[TMP22]], <16 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i8> [[TMP21]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <16 x i8> [[TMP24]], <16 x i8> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP27:%.*]] = zext <16 x i8> [[TMP26]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP28:%.*]] = mul nuw nsw <16 x i32> [[TMP17]], [[TMP27]]
@@ -1225,32 +1225,32 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
 ; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
 ; CHECK-NEXT:    [[TMP25:%.*]] = load <4 x i8>, ptr [[ADD_PTR_2]], align 1
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP25]], <4 x i8> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP26]], <16 x i8> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP25]], <4 x i8> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP26]], <16 x i8> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <16 x i8> [[TMP28]], <16 x i8> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP31:%.*]] = zext <16 x i8> [[TMP30]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP33:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_2]], align 1
-; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <4 x i8> [[TMP33]], <4 x i8> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <16 x i8> [[TMP34]], <16 x i8> [[TMP35]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <4 x i8> [[TMP33]], <4 x i8> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <16 x i8> [[TMP34]], <16 x i8> [[TMP35]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <16 x i8> [[TMP36]], <16 x i8> [[TMP37]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP39:%.*]] = zext <16 x i8> [[TMP38]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP40:%.*]] = sub nsw <16 x i32> [[TMP31]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP42:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_3]], align 1
-; CHECK-NEXT:    [[TMP43:%.*]] = shufflevector <4 x i8> [[TMP42]], <4 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP45:%.*]] = shufflevector <16 x i8> [[TMP43]], <16 x i8> [[TMP44]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP43:%.*]] = shufflevector <4 x i8> [[TMP42]], <4 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP45:%.*]] = shufflevector <16 x i8> [[TMP43]], <16 x i8> [[TMP44]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i8> [[TMP45]], <16 x i8> [[TMP46]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP48:%.*]] = zext <16 x i8> [[TMP47]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP50:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
-; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <4 x i8> [[TMP50]], <4 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <4 x i8> [[TMP15]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP53:%.*]] = shufflevector <16 x i8> [[TMP51]], <16 x i8> [[TMP52]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <4 x i8> [[TMP50]], <4 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <4 x i8> [[TMP15]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP53:%.*]] = shufflevector <16 x i8> [[TMP51]], <16 x i8> [[TMP52]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <16 x i8> [[TMP53]], <16 x i8> [[TMP54]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP56:%.*]] = zext <16 x i8> [[TMP55]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP57:%.*]] = sub nsw <16 x i32> [[TMP48]], [[TMP56]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
index 8b28762cce87a..f073251c139b5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
@@ -135,8 +135,8 @@ define <4 x i32> @build_vec_v4i32_reuse_1(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i32> [[V0]], [[V1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub <2 x i32> [[TMP8]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2_31:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
index 125b209de1e3e..5fb82a5b3568d 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
@@ -135,8 +135,8 @@ define <4 x i32> @build_vec_v4i32_reuse_1(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i32> [[V0]], [[V1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub <2 x i32> [[TMP8]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2_31:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP2_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
index 31ddc782cee14..28af0de171231 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
@@ -23,11 +23,11 @@ define void @s116_modified(ptr %a) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[GEP1]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[GEP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> poison, float [[LD0]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 undef, i32 2, i32 4>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 poison, i32 2, i32 4>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast <4 x float> [[TMP9]], [[TMP11]]
 ; CHECK-NEXT:    store <4 x float> [[TMP12]], ptr [[A]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll
index 48ebff8ebabb6..a1c61e945c7fe 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll
@@ -95,7 +95,7 @@ define void @noop_extract_second_2_lanes(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x double> [[V_1]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 2>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_3]])
 ; CHECK-NEXT:    store <4 x double> [[TMP3]], ptr [[PTR_1]], align 8
@@ -170,7 +170,7 @@ define void @extract_lanes_1_and_2(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x double> [[V_1]], <4 x double> poison, <2 x i32> <i32 1, i32 2>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 2>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
 ; CHECK-NEXT:    store <4 x double> [[TMP3]], ptr [[PTR_1]], align 8
@@ -213,7 +213,7 @@ define void @noop_extracts_existing_vector_4_lanes(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <4 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
@@ -261,7 +261,7 @@ define void @extracts_jumbled_4_lanes(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <4 x i32> <i32 2, i32 1, i32 2, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <4 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_0]])
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_1]])
 ; CHECK-NEXT:    call void @use(double [[V1_LANE_2]])
@@ -311,13 +311,13 @@ define void @noop_extracts_9_lanes(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 0, i32 2, i32 1, i32 0, i32 2, i32 0, i32 2, i32 1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[TMP3]], double [[A_LANE_8]], i32 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 6, i32 7, i32 8, i32 0, i32 1, i32 2, i32 3, i32 4>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 2, i32 1, i32 0, i32 2, i32 1, i32 0, i32 2, i32 1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul <8 x double> [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[TMP7]], double [[B_LANE_8]], i32 8
 ; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
 ; CHECK-NEXT:    store <9 x double> [[RES]], ptr [[PTR_1]], align 8
@@ -400,12 +400,12 @@ define void @first_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 2, i32 1, i32 0, i32 2>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[TMP3]], double [[A_LANE_8]], i32 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 6, i32 7, i32 8, i32 0, i32 1, i32 2, i32 3, i32 4>
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <8 x double> [[TMP4]], [[TMP1]]
 ; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[TMP6]], double [[B_LANE_8]], i32 8
 ; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
 ; CHECK-NEXT:    store <9 x double> [[RES]], ptr [[PTR_1]], align 8
@@ -488,13 +488,13 @@ define void @first_and_second_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 0, i32 2, i32 1, i32 2, i32 1, i32 0, i32 2, i32 1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:    [[A_INS_8:%.*]] = insertelement <9 x double> [[TMP3]], double [[A_LANE_8]], i32 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 7, i32 6, i32 8, i32 1, i32 0, i32 3, i32 2, i32 5>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 2, i32 1, i32 0, i32 2, i32 0, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul <8 x double> [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[B_LANE_8:%.*]] = fmul double [[V1_LANE_4]], [[V2_LANE_2]]
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
 ; CHECK-NEXT:    [[B_INS_8:%.*]] = insertelement <9 x double> [[TMP7]], double [[B_LANE_8]], i32 8
 ; CHECK-NEXT:    [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
 ; CHECK-NEXT:    store <9 x double> [[RES]], ptr [[PTR_1]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
index e6d80675635a2..9fd20da8e4bbf 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
@@ -248,7 +248,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
 ; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
index e99be89ef9bba..34bc2b338df5c 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
@@ -248,7 +248,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
 ; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 undef>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
index 63eaa8bdc2428..4ccef0f15c8b9 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
@@ -4,7 +4,7 @@
 ; Make sure there's no SCEV assert when the indexes are for different
 ; sized address spaces
 
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
 
 define void @slp_scev_assert(i32 %idx, i64 %tmp3) #0 {
 ; CHECK-LABEL: @slp_scev_assert(
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/crash_extract_subvector_cost.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/crash_extract_subvector_cost.ll
index 68295bb0d78e3..e474bab2ad965 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/crash_extract_subvector_cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/crash_extract_subvector_cost.ll
@@ -4,7 +4,7 @@
 define <2 x i16> @uadd_sat_v9i16_combine_vi16(<9 x i16> %arg0, <9 x i16> %arg1) {
 ; CHECK-LABEL: @uadd_sat_v9i16_combine_vi16(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <9 x i16> [[ARG0:%.*]], <9 x i16> poison, <2 x i32> <i32 undef, i32 8>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <9 x i16> [[ARG0:%.*]], <9 x i16> poison, <2 x i32> <i32 poison, i32 8>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <9 x i16> [[ARG1:%.*]], <9 x i16> poison, <2 x i32> <i32 7, i32 8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x i16> [[TMP2]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll
index eb4117bbf7209..25c04e5183221 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll
@@ -14,8 +14,8 @@ define <4 x half> @phis(i1 %cmp1, <4 x half> %in1, <4 x half> %in2)  {
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x half> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP2]], [[BB0]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = phi <2 x half> [ [[TMP1]], [[ENTRY]] ], [ [[TMP3]], [[BB0]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x half> [[TMP4]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x half> [[TMP5]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x half> [[TMP4]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x half> [[TMP5]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[O31:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x half> [[O31]]
 ;
@@ -59,8 +59,8 @@ define <4 x half> @phis_reverse(i1 %cmp1, <4 x half> %in1, <4 x half> %in2)  {
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x half> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP2]], [[BB0]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = phi <2 x half> [ [[TMP1]], [[ENTRY]] ], [ [[TMP3]], [[BB0]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x half> [[TMP4]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x half> [[TMP5]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x half> [[TMP4]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x half> [[TMP5]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[O31:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x half> [[O31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll
index 64c43458bd3a9..c646574fbdcae 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/math-function.ll
@@ -201,7 +201,7 @@ define <4 x float> @exp_4x(ptr %a) {
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.exp.v2f32(<2 x float> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -256,7 +256,7 @@ define <4 x float> @int_exp_4x(ptr %a) {
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.exp.v2f32(<2 x float> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -311,7 +311,7 @@ define <4 x float> @log_4x(ptr %a) {
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -366,7 +366,7 @@ define <4 x float> @int_log_4x(ptr %a) {
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -421,7 +421,7 @@ define <4 x float> @sin_4x(ptr %a) {
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
@@ -476,7 +476,7 @@ define <4 x float> @int_sin_4x(ptr %a) {
 ; CHECK-NEXT:    [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast <2 x float> @llvm.sin.v2f32(<2 x float> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_31:%.*]] = shufflevector <4 x float> [[VECINS_1]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[VECINS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
index 336ee9f530e93..0f41c9ce5d0d8 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
@@ -834,11 +834,11 @@ define i32 @stride_sum_abs_diff(ptr %p, ptr %q, i64 %stride) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[Q]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr [[P_2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i32>, ptr [[Q_2]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub <4 x i32> [[TMP7]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP11]], i1 true)
@@ -883,8 +883,8 @@ define i32 @reduce_sum_2arrays_a(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P:%.*]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[Q:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])
@@ -932,8 +932,8 @@ define i32 @reduce_sum_2arrays_b(ptr noalias noundef %x, ptr noalias %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[X:%.*]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[Y:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP5]])
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
index 750dac29a57e1..ff06bdc0e8446 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
@@ -14,7 +14,7 @@ define void @foo() local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr getelementptr inbounds ([4 x [4 x i32]], ptr @bar, i64 0, i64 3, i64 2), align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[ADD277]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> undef, [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = ashr <4 x i32> [[TMP6]], <i32 6, i32 6, i32 6, i32 6>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll
index cc1fa07d4c4f3..a4c239019bc9f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll
@@ -7,8 +7,8 @@ define void @test(<16 x half> %v) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x half> [[V:%.*]], <16 x half> poison, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x half> [[TMP0]] to <2 x float>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[TMP1]] to <2 x i32>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[VECINS_I_5_I1:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[VECINS_I_5_I1:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll
index a9e3aa33ae92d..d9f6a0d2bf7e0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll
@@ -7,7 +7,7 @@ define void @test(<16 x half> %v) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x half> [[V:%.*]], <16 x half> poison, <2 x i32> <i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x half> [[TMP0]] to <2 x float>
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x float> [[TMP1]] to <2 x i32>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINS_I_5_I1:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 14, i32 15>
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
index 99f0b4e9f6244..94534274cab2f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
@@ -340,11 +340,11 @@ define void @vec_shuff_reorder() #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr getelementptr inbounds ([4 x float], ptr @fa, i32 0, i64 2), align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP3]], i32 1
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x float> [[TMP11]], float [[TMP4]], i32 1
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x float> [[TMP12]], <4 x float> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    [[TMP15:%.*]] = fadd <4 x float> [[TMP10]], [[TMP14]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = fsub <4 x float> [[TMP10]], [[TMP14]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
index ee217c07895ee..da6f4e70c7c6f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -19,12 +19,12 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; SSE-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
 ; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
@@ -40,12 +40,12 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
 ; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SLM-NEXT:    ret <8 x float> [[R71]]
 ;
@@ -61,12 +61,12 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; AVX-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
 ; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:    ret <8 x float> [[R71]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index 0f7d3b5647ff9..3b84e70d847ff 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -19,12 +19,12 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; SSE-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
 ; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
@@ -40,12 +40,12 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
 ; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SLM-NEXT:    ret <8 x float> [[R71]]
 ;
@@ -61,12 +61,12 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
 ; AVX-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
 ; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i64 0
-; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:    ret <8 x float> [[R71]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
index 11eddced91a12..f4296c7775cd9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
@@ -209,10 +209,10 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp <2 x i8> [[TMP8]] to <2 x float>
 ; CHECK-NEXT:    [[TMP10:%.*]] = uitofp <2 x i8> [[TMP8]] to <2 x float>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
index 59a77598b6640..d798601722658 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
@@ -209,10 +209,10 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp <2 x i8> [[TMP8]] to <2 x float>
 ; CHECK-NEXT:    [[TMP10:%.*]] = uitofp <2 x i8> [[TMP8]] to <2 x float>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
index 0f59f595f911e..5a1de4f3e3d7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
@@ -101,7 +101,7 @@ define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
 ; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
-; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[A2]], i64 2
 ; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i64 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
index ef13f7956e16d..046ed781f4c8d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
@@ -101,7 +101,7 @@ define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
 ; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
-; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[A2]], i64 2
 ; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i64 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
index 9cf0e1184c4fe..8e878f3f8b80f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
@@ -105,7 +105,7 @@ define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
 ; SSE-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:    ret <8 x i32> [[R71]]
 ;
@@ -245,9 +245,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i64 6
 ; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
@@ -449,11 +449,11 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
 ; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 undef, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
-; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 5, i32 8, i32 9>
+; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @sdiv_v8i32_undefs(
@@ -466,11 +466,11 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
 ; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 undef, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
-; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 5, i32 8, i32 9>
+; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index 976be7d7f1c7e..3334d494621af 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -105,7 +105,7 @@ define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
 ; SSE-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE-NEXT:    ret <8 x i32> [[R71]]
 ;
@@ -245,9 +245,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i64 6
 ; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
@@ -449,11 +449,11 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
 ; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, i32 [[AB1]], i64 1
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 undef, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
-; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 5, i32 8, i32 9>
+; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @sdiv_v8i32_undefs(
@@ -466,11 +466,11 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
 ; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, i32 [[AB1]], i64 1
-; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 undef, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
-; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 undef, i32 5, i32 8, i32 9>
+; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
index c556ea7491d48..7f7e77eadc987 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
@@ -619,12 +619,12 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
 ; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[TMP12:%.*]] = fdiv <2 x double> [[TMP10]], [[TMP11]]
-; SLM-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R31:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x double> [[R31]], <8 x double> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SLM-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R73:%.*]] = shufflevector <8 x double> [[R52]], <8 x double> [[TMP16]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SLM-NEXT:    ret <8 x double> [[R73]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
index f331735558b0a..8b8bc71c2ceda 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
@@ -619,12 +619,12 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
 ; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
 ; SLM-NEXT:    [[TMP12:%.*]] = fdiv <2 x double> [[TMP10]], [[TMP11]]
-; SLM-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SLM-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R31:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x double> [[R31]], <8 x double> [[TMP15]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SLM-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R73:%.*]] = shufflevector <8 x double> [[R52]], <8 x double> [[TMP16]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SLM-NEXT:    ret <8 x double> [[R73]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
index 90f79eedbd3ee..7671d15cc3984 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
@@ -39,7 +39,7 @@ define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
 
 define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @h_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 3, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 poison, i32 3, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
index 6d8280ae3d60e..6b8f20c57d7d9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
@@ -39,7 +39,7 @@ define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
 
 define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @h_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 3, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 poison, i32 3, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
 ; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/broadcast_long.ll b/llvm/test/Transforms/SLPVectorizer/X86/broadcast_long.ll
index a5c953d48a555..5f14cea1d6164 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/broadcast_long.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/broadcast_long.ll
@@ -17,7 +17,7 @@ define void @bcast_long(ptr %A, ptr %S) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A0:%.*]] = load i32, ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i32> poison, i32 [[A0]], i32 0
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> poison, <8 x i32> <i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> poison, <8 x i32> <i32 0, i32 0, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0>
 ; CHECK-NEXT:    [[TMP1:%.*]] = freeze <8 x i32> [[SHUFFLE]]
 ; CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr [[S:%.*]], align 8
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-insert-mask-size.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-insert-mask-size.ll
index f3a2ea07ddfe1..b8331c9e1faa3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-insert-mask-size.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-insert-mask-size.ll
@@ -5,7 +5,7 @@ define void @test() {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr undef, i32 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <3 x i32> <i32 0, i32 poison, i32 1>
 ; CHECK-NEXT:    store <3 x float> [[TMP3]], ptr null, align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-same-lane-insert.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-same-lane-insert.ll
index 752c35fc5f351..6077c376251ef 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-same-lane-insert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-same-lane-insert.ll
@@ -12,7 +12,7 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp olt float [[TMP6]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> zeroinitializer, float 0.000000e+00, i64 0
 ; CHECK-NEXT:    store <2 x float> zeroinitializer, ptr null, align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    store <2 x float> zeroinitializer, ptr null, align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-shuffle.ll
index 79a4054b63a2b..f8522bc546e6b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-shuffle.ll
@@ -5,7 +5,7 @@ define void @b() {
 ; CHECK-LABEL: @b(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float 0x7FF8000000000000, i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000, float undef, float undef>, <4 x i32> <i32 0, i32 4, i32 5, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000, float undef, float undef>, <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float 0x7FF8000000000000, i32 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP2]], <4 x float> zeroinitializer, <4 x float> zeroinitializer)
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul <4 x float> [[TMP3]], <float undef, float undef, float undef, float 2.000000e+00>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
index b7f161b056fe2..4af23a6c96510 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
@@ -67,7 +67,7 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
 ; CHECK-NEXT:    [[MUL88:%.*]] = fmul double [[TMP4]], 2.000000e+00
 ; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x double> poison, double [[FNEG87]], i32 0
 ; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[CALL]], i32 1
-; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP27]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP27]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <2 x double> [[TMP28]], double [[TMP12]], i32 1
 ; CHECK-NEXT:    [[TMP30:%.*]] = fsub <2 x double> [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <2 x double> poison, double [[MUL88]], i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll b/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll
index 4ebb78017db1c..b21ba424a63c7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll
@@ -95,7 +95,7 @@ define void @same_opcode_on_one_side(i32 %a, i32 %b, i32 %c) {
 ; AVX-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i32 0
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; AVX-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 4, i32 0>
+; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 poison, i32 4, i32 0>
 ; AVX-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[B:%.*]], i32 1
 ; AVX-NEXT:    [[TMP8:%.*]] = xor <4 x i32> [[TMP5]], [[TMP7]]
 ; AVX-NEXT:    store <4 x i32> [[TMP8]], ptr @cle32, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
index 88b705f14c530..849c1d7c192dc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
@@ -32,7 +32,7 @@ define void @testfunc(ptr nocapture %dest, ptr nocapture readonly %src) {
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x float> [[TMP10]], i32 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[TMP10]], i32 1
 ; CHECK-NEXT:    [[ADD13]] = fadd float [[TMP11]], [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x float> [[TMP13]], float [[ADD13]], i32 1
 ; CHECK-NEXT:    [[TMP15:%.*]] = fcmp olt <2 x float> [[TMP14]], <float 1.000000e+00, float 1.000000e+00>
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[TMP15]], <2 x float> [[TMP14]], <2 x float> <float 1.000000e+00, float 1.000000e+00>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll
index 512b0ab45a2be..6ac588524f845 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod-inseltpoison.ll
@@ -128,7 +128,7 @@ define fastcc void @dct36(ptr %inbuf) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds double, ptr [[INBUF:%.*]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[INBUF]], align 8
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[ARRAYIDX44]], align 8
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll
index 620d589406387..e63840ebf8f7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll
@@ -24,7 +24,7 @@ define void @_foo(double %p1, double %p2, double %p3) #0 {
 ; CHECK-NEXT:    [[X13:%.*]] = tail call i32 @_xfn(<2 x double> [[TMP4]])
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [256 x i32], ptr [[TAB1]], i64 0, i64 [[INDVARS_IV266]]
 ; CHECK-NEXT:    store i32 [[X13]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[X14:%.*]] = tail call i32 @_xfn(<2 x double> [[TMP5]])
 ; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [256 x i32], ptr [[TAB2]], i64 0, i64 [[INDVARS_IV266]]
 ; CHECK-NEXT:    store i32 [[X14]], ptr [[ARRAYIDX26]], align 4, !tbaa [[TBAA0]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
index 13eaeda3e65dc..6f3b1f8fa9cd4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
@@ -24,7 +24,7 @@ define void @_foo(double %p1, double %p2, double %p3) #0 {
 ; CHECK-NEXT:    [[X13:%.*]] = tail call i32 @_xfn(<2 x double> [[TMP4]])
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [256 x i32], ptr [[TAB1]], i64 0, i64 [[INDVARS_IV266]]
 ; CHECK-NEXT:    store i32 [[X13]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[X14:%.*]] = tail call i32 @_xfn(<2 x double> [[TMP5]])
 ; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [256 x i32], ptr [[TAB2]], i64 0, i64 [[INDVARS_IV266]]
 ; CHECK-NEXT:    store i32 [[X14]], ptr [[ARRAYIDX26]], align 4, !tbaa [[TBAA0]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll b/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll
index d9942887d6c7d..610cc5bdeb310 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll
@@ -9,7 +9,7 @@ define i1 @test() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 0, i64 poison, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i64> [[TMP1]], i64 0, i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i64> [[TMP2]], i64 0, i32 3
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <4 x i32> <i32 3, i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <4 x i32> <i32 3, i32 poison, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> <i64 undef, i64 0, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef, i64 undef>, <8 x i64> [[TMP3]], <8 x i32> <i32 11, i32 11, i32 11, i32 1, i32 9, i32 9, i32 1, i32 8>
 ; CHECK-NEXT:    [[TMP6:%.*]] = or <8 x i64> [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub <8 x i64> [[TMP3]], [[TMP5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/external-used-across-reductions.ll b/llvm/test/Transforms/SLPVectorizer/X86/external-used-across-reductions.ll
index 6443de6af6c65..01aced4ab3acf 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/external-used-across-reductions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/external-used-across-reductions.ll
@@ -9,7 +9,7 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i64>, ptr [[IDX2]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i64> [[TMP1]], i32 7
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr null, align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <8 x i32> <i32 undef, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <8 x i32> <i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x i64> [[TMP4]], i64 [[TMP3]], i32 0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-many-users-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-many-users-buildvector.ll
index 432a0eda30fe7..cac0491d0b643 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract-many-users-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-many-users-buildvector.ll
@@ -9,18 +9,18 @@ define i1 @test(float %0, double %1) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP1]], i32 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> zeroinitializer, [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP6]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> <double poison, double 0.000000e+00, double poison, double 0.000000e+00>, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> <double poison, double 0.000000e+00, double poison, double 0.000000e+00>, <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
 ; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> poison, <4 x i32> <i32 1, i32 2, i32 0, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> poison, <4 x i32> <i32 1, i32 2, i32 0, i32 poison>
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP7]], i32 3
 ; CHECK-NEXT:    [[TMP13:%.*]] = fmul <4 x double> [[TMP10]], [[TMP12]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = fmul <4 x double> zeroinitializer, [[TMP4]]
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <8 x double> <double poison, double poison, double poison, double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, <8 x double> [[TMP15]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <8 x double> <double poison, double poison, double poison, double poison, double poison, double poison, double 0.000000e+00, double 0.000000e+00>, <8 x double> [[TMP17]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <8 x double> <double poison, double poison, double poison, double poison, double poison, double poison, double 0.000000e+00, double 0.000000e+00>, <8 x double> [[TMP17]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <8 x double> [[TMP18]], <8 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP21:%.*]] = fsub <8 x double> [[TMP16]], [[TMP20]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = fmul <8 x double> [[TMP16]], [[TMP20]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
index 5d9c6b6a62650..a184e2ec40b3c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-scalar-from-undef.ll
@@ -9,7 +9,7 @@ define i64 @foo(i32 %tmp7) {
 ; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 undef, 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 undef, i32 0>, i32 [[TMP24]], i32 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 0, i32 5
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[TMP24]], i32 6
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <8 x i32> [[TMP3]], [[TMP5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll
index 0d60de7935032..8277ca193540c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract_with_non_const_index.ll
@@ -10,7 +10,7 @@ define void @test(ptr nocapture %o, ptr nocapture nonnull readonly dereferenceab
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[A:%.*]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[B:%.*]], align 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 [[COMPONENT:%.*]] to i8
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP5]], i8 [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> undef, float [[TMP6]], i64 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
index af78274cd994b..78fc3a60f0514 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
@@ -8,7 +8,7 @@ define void @test(i16 %0) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP1]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, <4 x i32> [[TMP5]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
 ; CHECK-NEXT:    br label [[FOR_BODY92:%.*]]
 ; CHECK:       for.body92:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll
new file mode 100644
index 0000000000000..9e43cefef2801
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skx -S < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skx -S < %s -slp-threshold=-100 | FileCheck %s --check-prefix=CHECK-SLP-THRESHOLD
+
+define void @test() {
+; CHECK-LABEL: define void @test
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_IN_V:%.*]] = select i1 false, ptr null, ptr null
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[COND_IN_V]], align 8
+; CHECK-NEXT:    [[BV:%.*]] = icmp eq i64 [[V]], 0
+; CHECK-NEXT:    [[IN_1:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 4
+; CHECK-NEXT:    [[V_1:%.*]] = load i64, ptr [[IN_1]], align 8
+; CHECK-NEXT:    [[BV_1:%.*]] = icmp eq i64 [[V_1]], 0
+; CHECK-NEXT:    [[IN_2:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 8
+; CHECK-NEXT:    [[V_2:%.*]] = load i64, ptr [[IN_2]], align 8
+; CHECK-NEXT:    [[BV_2:%.*]] = icmp eq i64 [[V_2]], 0
+; CHECK-NEXT:    [[IN_3:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 12
+; CHECK-NEXT:    [[V_3:%.*]] = load i64, ptr [[IN_3]], align 8
+; CHECK-NEXT:    [[BV_3:%.*]] = icmp eq i64 [[V_3]], 0
+; CHECK-NEXT:    ret void
+;
+; CHECK-SLP-THRESHOLD-LABEL: define void @test
+; CHECK-SLP-THRESHOLD-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-SLP-THRESHOLD-NEXT:  entry:
+; CHECK-SLP-THRESHOLD-NEXT:    [[COND_IN_V:%.*]] = select i1 false, ptr null, ptr null
+; CHECK-SLP-THRESHOLD-NEXT:    br label [[BB:%.*]]
+; CHECK-SLP-THRESHOLD:       bb:
+; CHECK-SLP-THRESHOLD-NEXT:    [[TMP0:%.*]] = insertelement <4 x ptr> poison, ptr [[COND_IN_V]], i32 0
+; CHECK-SLP-THRESHOLD-NEXT:    [[TMP1:%.*]] = shufflevector <4 x ptr> [[TMP0]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; CHECK-SLP-THRESHOLD-NEXT:    [[TMP2:%.*]] = getelementptr i64, <4 x ptr> [[TMP1]], <4 x i64> <i64 12, i64 8, i64 4, i64 0>
+; CHECK-SLP-THRESHOLD-NEXT:    [[TMP3:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP2]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
+; CHECK-SLP-THRESHOLD-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP3]], zeroinitializer
+; CHECK-SLP-THRESHOLD-NEXT:    ret void
+;
+entry:
+  %cond.in.v = select i1 false, ptr null, ptr null
+  br label %bb
+
+bb:                            ; preds = %entry
+  %v = load i64, ptr %cond.in.v, align 8
+  %bv = icmp eq i64 %v, 0
+  %in.1 = getelementptr i64, ptr %cond.in.v, i64 4
+  %v.1 = load i64, ptr %in.1, align 8
+  %bv.1 = icmp eq i64 %v.1, 0
+  %in.2 = getelementptr i64, ptr %cond.in.v, i64 8
+  %v.2 = load i64, ptr %in.2, align 8
+  %bv.2 = icmp eq i64 %v.2, 0
+  %in.3 = getelementptr i64, ptr %cond.in.v, i64 12
+  %v.3 = load i64, ptr %in.3, align 8
+  %bv.3 = icmp eq i64 %v.3, 0
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
index f8fbcc58295a1..5fcd339f333e8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
@@ -220,7 +220,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
 ; CHECK-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3
 ; CHECK-NEXT:    ret <4 x double> [[R03]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
index 5507cc9404861..fe8bff870a1ab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
@@ -220,7 +220,7 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
 ; CHECK-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3
 ; CHECK-NEXT:    ret <4 x double> [[R03]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-const-undef.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-const-undef.ll
index d1350bc029019..6a62663f292f3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-const-undef.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-const-undef.ll
@@ -8,7 +8,7 @@ define <4 x float> @simple_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c)
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP2]], <2 x float> [[TMP3]], <2 x float> [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[TMP6]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll
index 654121553b4cf..10369e3aa270e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll
@@ -283,9 +283,9 @@ define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[TMP7]], <2 x float> [[TMP8]], <2 x float> [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[RD1:%.*]] = shufflevector <4 x float> [[TMP12]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[RD1:%.*]] = shufflevector <4 x float> [[TMP12]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    ret <4 x float> [[RD1]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
index d855dd6292c00..f008075d27ec7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
@@ -43,7 +43,7 @@ define <8 x float> @simple_select2(<4 x float> %a, <4 x float> %b, <4 x i32> %c)
 ; CHECK-LABEL: @simple_select2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 undef, i32 1, i32 undef, i32 2, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 poison, i32 3>
 ; CHECK-NEXT:    ret <8 x float> [[TMP3]]
 ;
   %c0 = extractelement <4 x i32> %c, i32 0
@@ -317,8 +317,8 @@ define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP10:%.*]] = select <2 x i1> [[TMP7]], <2 x float> [[TMP8]], <2 x float> [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RD1:%.*]] = shufflevector <4 x float> [[TMP12]], <4 x float> undef, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
 ; CHECK-NEXT:    ret <4 x float> [[RD1]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll b/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll
index 6e67c412d9a43..bb1aac8fb932c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/jumbled_store_crash.ll
@@ -37,7 +37,7 @@ define dso_local void @j() local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x float> [[TMP12]], i32 1
 ; CHECK-NEXT:    store float [[TMP16]], ptr @f, align 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x float> <float poison, float -1.000000e+00, float poison, float -1.000000e+00>, float [[CONV19]], i32 0
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 5, i32 3>
 ; CHECK-NEXT:    [[TMP20:%.*]] = fsub <4 x float> [[TMP12]], [[TMP19]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = fadd <4 x float> [[TMP12]], [[TMP19]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll b/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll
index 9ed021448e020..f795fef9d92f9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll
@@ -33,8 +33,8 @@ define void @foo() personality ptr @bar {
 ; CHECK:       bb9:
 ; CHECK-NEXT:    [[INDVARS_IV528799:%.*]] = phi i64 [ poison, [[BB10]] ], [ poison, [[BB12]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi <2 x i32> [ [[TMP9:%.*]], [[BB10]] ], [ [[TMP10:%.*]], [[BB12]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP8]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb10:
 ; CHECK-NEXT:    [[TMP9]] = phi <2 x i32> [ [[TMP1]], [[BB3]] ]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
index 5b0ecdb779d23..9a41c1dc5de22 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
@@ -55,7 +55,7 @@ define <4 x float> @PR16739_byref(ptr nocapture readonly dereferenceable(16) %x)
 ; AVX2-NEXT:    [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[X:%.*]], i64 0, i64 2
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[X]], align 4
 ; AVX2-NEXT:    [[X2:%.*]] = load float, ptr [[GEP2]], align 4
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[I2:%.*]] = insertelement <4 x float> [[TMP2]], float [[X2]], i32 2
 ; AVX2-NEXT:    [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[X2]], i32 3
 ; AVX2-NEXT:    ret <4 x float> [[I3]]
@@ -65,8 +65,8 @@ define <4 x float> @PR16739_byref(ptr nocapture readonly dereferenceable(16) %x)
 ; AVX512-NEXT:    [[X0:%.*]] = load float, ptr [[X]], align 4
 ; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[GEP1]], align 4
 ; AVX512-NEXT:    [[I0:%.*]] = insertelement <4 x float> poison, float [[X0]], i32 0
-; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; AVX512-NEXT:    [[I21:%.*]] = shufflevector <4 x float> [[I0]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 4, i32 5, i32 undef>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; AVX512-NEXT:    [[I21:%.*]] = shufflevector <4 x float> [[I0]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[I21]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; AVX512-NEXT:    ret <4 x float> [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
index 49b88f85968a7..bc8e6626e5508 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
@@ -55,7 +55,7 @@ define <4 x float> @PR16739_byref(ptr nocapture readonly dereferenceable(16) %x)
 ; AVX2-NEXT:    [[GEP2:%.*]] = getelementptr inbounds <4 x float>, ptr [[X:%.*]], i64 0, i64 2
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[X]], align 4
 ; AVX2-NEXT:    [[X2:%.*]] = load float, ptr [[GEP2]], align 4
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[I2:%.*]] = insertelement <4 x float> [[TMP2]], float [[X2]], i32 2
 ; AVX2-NEXT:    [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[X2]], i32 3
 ; AVX2-NEXT:    ret <4 x float> [[I3]]
@@ -65,7 +65,7 @@ define <4 x float> @PR16739_byref(ptr nocapture readonly dereferenceable(16) %x)
 ; AVX512-NEXT:    [[X0:%.*]] = load float, ptr [[X]], align 4
 ; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[GEP1]], align 4
 ; AVX512-NEXT:    [[I0:%.*]] = insertelement <4 x float> undef, float [[X0]], i32 0
-; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; AVX512-NEXT:    [[I21:%.*]] = shufflevector <4 x float> [[I0]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 4, i32 5, i32 3>
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[I21]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; AVX512-NEXT:    ret <4 x float> [[TMP3]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll
index 1e24cab158d8c..8d37665224e7e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-partial-vector-shuffle.ll
@@ -37,10 +37,10 @@ define <2 x i64> @load_00123456(ptr nocapture noundef readonly %data) {
 ; AVX-NEXT:    [[TMP1:%.*]] = load <2 x i16>, ptr [[DATA]], align 2
 ; AVX-NEXT:    [[T2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 2
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[ARRAYIDX3]], align 2
-; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[VECINIT2_I_I2:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP3]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX-NEXT:    [[VECINIT3_I_I:%.*]] = insertelement <8 x i16> [[VECINIT2_I_I2]], i16 [[T2]], i64 3
-; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[VECINIT7_I_I1:%.*]] = shufflevector <8 x i16> [[VECINIT3_I_I]], <8 x i16> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX-NEXT:    [[T7:%.*]] = bitcast <8 x i16> [[VECINIT7_I_I1]] to <2 x i64>
 ; AVX-NEXT:    ret <2 x i64> [[T7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
index 0365d046cbf36..384eac30d1297 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
@@ -529,7 +529,7 @@ define i1 @foo(float %a, float %b, float %c, <4 x float> %vec, i64 %idx2) {
 ; CHECK-NEXT:    [[SUB14_I167:%.*]] = fsub float undef, [[VECEXT_I291_I166]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[C:%.*]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[VEC]], <4 x float> poison, <2 x i32> <i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[VEC]], <4 x float> poison, <2 x i32> <i32 poison, i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[SUB14_I167]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x float> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> <float poison, float 3.000000e+01>, float [[B:%.*]], i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll b/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll
index 584eeb3710907..3d12da79ebf95 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/matched-shuffled-entries.ll
@@ -16,7 +16,7 @@ define i32 @bar() local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[SUB86_1]], i32 7
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[ADD78_2]], i32 9
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 9, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 6, i32 5, i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 6, i32 5, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[SUB102_3]], i32 12
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP7]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 12>
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <16 x i32> [[TMP5]], [[TMP8]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll b/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll
index e5947dcdd0825..f614796916baa 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/memory-runtime-checks.ll
@@ -187,8 +187,8 @@ define void @gather_sequence_crash(<2 x float> %arg, ptr %arg1, float %arg2, ptr
 ; CHECK-NEXT:    br i1 [[C_1:%.*]], label [[BB16:%.*]], label [[BB6:%.*]]
 ; CHECK:       bb6:
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[ARG1:%.*]], i32 3
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x float> [[ARG:%.*]], <2 x float> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> <float poison, float poison, float poison, float 0.000000e+00>, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x float> [[ARG:%.*]], <2 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> <float poison, float poison, float poison, float 0.000000e+00>, <4 x i32> <i32 poison, i32 1, i32 2, i32 7>
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[ARG2:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], zeroinitializer
 ; CHECK-NEXT:    store <4 x float> [[TMP3]], ptr [[TMP8]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
new file mode 100644
index 0000000000000..2ec4c04c2c8dc
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-107 | FileCheck %s
+
+define void @test(i64 %p0, i64 %p1, i64 %p2, i64 %p3) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[P0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> [[TMP0]], i64 [[P1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[A2:%.*]] = add i64 [[P2:%.*]], [[P2]]
+; CHECK-NEXT:    [[A3:%.*]] = add i64 [[P3:%.*]], [[P3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <2 x i64> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[M2:%.*]] = mul i64 [[P2]], [[P2]]
+; CHECK-NEXT:    [[M3:%.*]] = mul i64 [[P3]], [[P3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sdiv <2 x i64> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[D2:%.*]] = sdiv i64 [[P2]], [[P2]]
+; CHECK-NEXT:    [[D3:%.*]] = sdiv i64 [[P3]], [[P3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; CHECK-NEXT:    [[S0:%.*]] = sub i64 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; CHECK-NEXT:    [[S1:%.*]] = sub i64 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[S2:%.*]] = sub i64 [[M2]], [[D2]]
+; CHECK-NEXT:    [[S3:%.*]] = sub i64 [[M3]], [[D3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; CHECK-NEXT:    [[SHL1:%.*]] = shl i64 [[TMP9]], [[S0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; CHECK-NEXT:    [[SHL2:%.*]] = shl i64 [[TMP10]], [[S1]]
+; CHECK-NEXT:    [[SHL3:%.*]] = shl i64 [[A2]], [[S2]]
+; CHECK-NEXT:    [[SHL4:%.*]] = shl i64 [[A3]], [[S3]]
+; CHECK-NEXT:    [[O0:%.*]] = or i64 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TT0:%.*]] = trunc i64 [[O0]] to i32
+; CHECK-NEXT:    [[O1:%.*]] = or i64 [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TT1:%.*]] = trunc i64 [[O1]] to i32
+; CHECK-NEXT:    [[O2:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TT2:%.*]] = trunc i64 [[O2]] to i32
+; CHECK-NEXT:    [[O3:%.*]] = or i64 [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TT3:%.*]] = trunc i64 [[O3]] to i32
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[PHI0:%.*]] = phi i32 [ [[T1:%.*]], [[BB]] ], [ [[TT0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[PHI1:%.*]] = phi i32 [ [[T2:%.*]], [[BB]] ], [ [[TT1]], [[ENTRY]] ]
+; CHECK-NEXT:    [[PHI2:%.*]] = phi i32 [ [[T3:%.*]], [[BB]] ], [ [[TT2]], [[ENTRY]] ]
+; CHECK-NEXT:    [[PHI3:%.*]] = phi i32 [ [[T4:%.*]], [[BB]] ], [ [[TT3]], [[ENTRY]] ]
+; CHECK-NEXT:    [[T1]] = trunc i64 [[SHL1]] to i32
+; CHECK-NEXT:    [[T2]] = trunc i64 [[SHL2]] to i32
+; CHECK-NEXT:    [[T3]] = trunc i64 [[SHL3]] to i32
+; CHECK-NEXT:    [[T4]] = trunc i64 [[SHL4]] to i32
+; CHECK-NEXT:    br label [[BB]]
+;
+entry:
+  %a0 = add i64 %p0, %p0
+  %a1 = add i64 %p1, %p1
+  %a2 = add i64 %p2, %p2
+  %a3 = add i64 %p3, %p3
+  %m0 = mul i64 %p0, %p0
+  %m1 = mul i64 %p1, %p1
+  %m2 = mul i64 %p2, %p2
+  %m3 = mul i64 %p3, %p3
+  %d0 = sdiv i64 %p0, %p0
+  %d1 = sdiv i64 %p1, %p1
+  %d2 = sdiv i64 %p2, %p2
+  %d3 = sdiv i64 %p3, %p3
+  %s0 = sub i64 %m0, %d0
+  %s1 = sub i64 %m1, %d1
+  %s2 = sub i64 %m2, %d2
+  %s3 = sub i64 %m3, %d3
+  %shl1 = shl i64 %a0, %s0
+  %shl2 = shl i64 %a1, %s1
+  %shl3 = shl i64 %a2, %s2
+  %shl4 = shl i64 %a3, %s3
+  %o0 = or i64 %a0, %a1
+  %tt0 = trunc i64 %o0 to i32
+  %o1 = or i64 %m0, %m1
+  %tt1 = trunc i64 %o1 to i32
+  %o2 = or i64 %d0, %d1
+  %tt2 = trunc i64 %o2 to i32
+  %o3 = or i64 %m0, %m1
+  %tt3 = trunc i64 %o3 to i32
+  br label %bb
+
+bb:
+  %phi0 = phi i32 [ %t1, %bb ], [ %tt0, %entry ]
+  %phi1 = phi i32 [ %t2, %bb ], [ %tt1, %entry ]
+  %phi2 = phi i32 [ %t3, %bb ], [ %tt2, %entry ]
+  %phi3 = phi i32 [ %t4, %bb ], [ %tt3, %entry ]
+  %t1 = trunc i64 %shl1 to i32
+  %t2 = trunc i64 %shl2 to i32
+  %t3 = trunc i64 %shl3 to i32
+  %t4 = trunc i64 %shl4 to i32
+  br label %bb
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
index 359c074d6a211..7d07dca6da983 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -42,7 +42,7 @@ define void @vecload_vs_broadcast(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
@@ -56,7 +56,7 @@ define void @vecload_vs_broadcast(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2:       lp:
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP2]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
@@ -90,7 +90,7 @@ define void @vecload_vs_broadcast2(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
@@ -104,7 +104,7 @@ define void @vecload_vs_broadcast2(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2:       lp:
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
@@ -138,7 +138,7 @@ define void @vecload_vs_broadcast3(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
@@ -152,7 +152,7 @@ define void @vecload_vs_broadcast3(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2:       lp:
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 undef, i32 0>
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
 ; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
@@ -356,7 +356,7 @@ define void @good_load_order() {
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
 ; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[TMP1]], i64 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <4 x float> [[TMP7]], [[TMP9]]
 ; CHECK-NEXT:    store <4 x float> [[TMP10]], ptr [[ARRAYIDX5]], align 4
@@ -391,7 +391,7 @@ define void @good_load_order() {
 ; SSE2-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
 ; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
 ; SSE2-NEXT:    [[TMP7:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
-; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 2>
+; SSE2-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
 ; SSE2-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[TMP1]], i64 0
 ; SSE2-NEXT:    [[TMP10:%.*]] = fmul <4 x float> [[TMP7]], [[TMP9]]
 ; SSE2-NEXT:    store <4 x float> [[TMP10]], ptr [[ARRAYIDX5]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/peek-through-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/peek-through-shuffle.ll
index 047a0d43d7866..c157f6117df95 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/peek-through-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/peek-through-shuffle.ll
@@ -6,9 +6,9 @@ define void @foo(ptr %0, <4 x float> %1) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, ptr null, i64 22
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> zeroinitializer, <4 x float> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP1:%.*]], <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP1:%.*]], <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 4>
 ; CHECK-NEXT:    [[TMP8:%.*]] = fpext <4 x float> [[TMP7]] to <4 x double>
 ; CHECK-NEXT:    store <4 x double> [[TMP8]], ptr [[TMP0:%.*]], align 32
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
index 57a92385fb125..495a503311ab9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
@@ -139,7 +139,7 @@ define float @foo3(ptr nocapture readonly %A) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 0
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
@@ -156,7 +156,7 @@ define float @foo3(ptr nocapture readonly %A) #0 {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 3
 ; CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP9]] = load <2 x float>, ptr [[ARRAYIDX19]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP9]], <4 x i32> <i32 1, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP9]], <4 x i32> <i32 1, i32 poison, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> [[TMP10]], float [[TMP8]], i32 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = fmul <4 x float> [[TMP11]], <float 8.000000e+00, float 9.000000e+00, float 1.000000e+01, float 1.100000e+01>
 ; CHECK-NEXT:    [[TMP13]] = fadd <4 x float> [[TMP4]], [[TMP12]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/postponed_gathers.ll b/llvm/test/Transforms/SLPVectorizer/X86/postponed_gathers.ll
new file mode 100644
index 0000000000000..681d131c50727
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/postponed_gathers.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=slp-vectorizer -slp-threshold=-10 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck %s
+
+define void @foo() {
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT:  bci_0:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(1) null, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[BCI_252:%.*]]
+; CHECK:       bci_252:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x i32> [ zeroinitializer, [[BCI_0:%.*]] ], [ [[TMP16:%.*]], [[BCI_252_1:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or <2 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = or <2 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or <2 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <2 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <2 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    br i1 false, label [[NOT_ZERO70:%.*]], label [[BCI_252_1]]
+; CHECK:       bci_252.1:
+; CHECK-NEXT:    [[TMP10:%.*]] = or <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = or <2 x i32> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = mul <2 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = or <2 x i32> [[TMP2]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or <2 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <2 x i32> [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16]] = or <2 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    br label [[BCI_252]]
+; CHECK:       not_zero70:
+; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP9]], [[BCI_252]] ]
+; CHECK-NEXT:    ret void
+;
+bci_0:
+  %0 = load i32, ptr addrspace(1) null, align 8
+  br label %bci_252
+
+bci_252:
+  %1 = phi i32 [ 0, %bci_0 ], [ %20, %bci_252.1 ]
+  %2 = phi i32 [ 0, %bci_0 ], [ %15, %bci_252.1 ]
+  %3 = mul i32 %0, 0
+  %4 = or i32 %0, %3
+  %5 = or i32 %4, 0
+  %.neg91.neg = or i32 %2, 0
+  %.neg446 = or i32 %.neg91.neg, %5
+  %6 = or i32 %.neg446, 0
+  %7 = mul i32 0, 0
+  %8 = or i32 %0, %7
+  %9 = or i32 %8, 0
+  %.neg91.1.neg = or i32 %1, 0
+  %.neg448 = or i32 %.neg91.1.neg, %9
+  %10 = or i32 %.neg448, 0
+  br i1 false, label %not_zero70, label %bci_252.1
+
+bci_252.1:
+  %11 = or i32 %0, 0
+  %12 = mul i32 %11, 0
+  %13 = or i32 %0, %12
+  %14 = or i32 %13, 0
+  %.neg91.neg.1 = or i32 %6, 0
+  %.neg446.1 = or i32 %.neg91.neg.1, %14
+  %15 = or i32 %.neg446.1, 0
+  %16 = or i32 %0, 0
+  %17 = mul i32 %16, 0
+  %18 = or i32 %0, %17
+  %19 = or i32 %18, 0
+  %.neg91.1.neg.1 = or i32 %10, 0
+  %.neg448.1 = or i32 %.neg91.1.neg.1, %19
+  %20 = or i32 %.neg448.1, 0
+  br label %bci_252
+
+not_zero70:
+  %.lcssa546 = phi i32 [ %6, %bci_252 ]
+  %.lcssa545 = phi i32 [ %10, %bci_252 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
index bebf761a6b270..c62fb97da5c89 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
@@ -12,7 +12,7 @@ define <4 x i32> @foo(<4 x i32> %x, i32 %f) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 2, i32 3>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[VECINIT51:%.*]] = shufflevector <4 x i32> [[VECINIT1]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[VECINIT51]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
index d91470c50992e..9be8fb76ecb5e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
@@ -8,10 +8,10 @@ define dso_local <4 x float> @foo(<4 x i32> %0) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0:%.*]], i64 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp <2 x i32> [[TMP6]] to <2 x float>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x float> [[TMP9]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll
index 7faff0c39a739..0c1d96ff20b65 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-gathered-vectorized.ll
@@ -15,10 +15,10 @@ define i16 @test() {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[A1]], align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, ptr [[A2]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, ptr [[A]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i64> [[TMP5]], i64 [[TMP1]], i32 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i64> [[TMP6]], i64 [[TMP2]], i32 3
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i64> [[TMP7]], <8 x i64> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> zeroinitializer, <8 x i32> <i32 8, i32 8, i32 2, i32 3, i32 4, i32 5, i32 8, i32 8>
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.xor.v8i64(<8 x i64> [[TMP10]])
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
index 458f4f98d815f..b244715caa793 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
@@ -89,7 +89,7 @@ define i1 @fcmp_lt_gt(double %a, double %b, double %c) {
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[C:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[FNEG]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[B]], i32 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> poison, double [[MUL]], i32 0
@@ -138,7 +138,7 @@ define i1 @fcmp_lt(double %a, double %b, double %c) {
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> poison, double [[C:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[FNEG]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[B]], i32 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> poison, double [[MUL]], i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder-clustered-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder-clustered-node.ll
index 16578504de084..b5533463c3930 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder-clustered-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder-clustered-node.ll
@@ -14,7 +14,7 @@ define i1 @test(ptr %arg, ptr %i233, i64 %i241, ptr %i235, ptr %i237, ptr %i227)
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x ptr> <ptr poison, ptr null, ptr poison, ptr null, ptr null, ptr null, ptr null, ptr null>, ptr [[I242]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x ptr> [[TMP2]], ptr [[I250]], i32 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult <8 x ptr> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x ptr> [[TMP3]], <8 x ptr> poison, <4 x i32> <i32 2, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x ptr> [[TMP3]], <8 x ptr> poison, <4 x i32> <i32 2, i32 0, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x ptr> [[TMP5]], ptr [[I245]], i32 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x ptr> [[TMP6]], ptr [[I248]], i32 3
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x ptr> [[TMP7]], <4 x ptr> poison, <8 x i32> <i32 2, i32 0, i32 1, i32 3, i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll
index 035bc3ecc09d4..f65f61975a61f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather.ll
@@ -9,7 +9,7 @@ define void @test(ptr noalias %0, ptr %p) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP0:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> [[TMP4]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> poison)
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> poison, <16 x i32> <i32 4, i32 3, i32 0, i32 1, i32 2, i32 0, i32 1, i32 2, i32 0, i32 2, i32 5, i32 6, i32 7, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x float> [[TMP8]], <16 x float> <float poison, float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[TMP10:%.*]] = fadd reassoc nsz arcp contract afn <16 x float> [[TMP7]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x float> [[TMP10]], <16 x float> poison, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 9, i32 0, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
index 3968f58877bc0..78c6d9516a3de 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -21,12 +21,12 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6
 ; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:    [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8>
 ; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i8> [[TMP13]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll
index 358eb9269dd49..35cb8c729e106 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-extractelements.ll
@@ -14,7 +14,7 @@
 
 define <2 x i32> @g(<2 x i32> %x, i32 %a, i32 %b) {
 ; CHECK-LABEL: @g(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[B:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i32> [[TMP2]], [[TMP3]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-scalars-in-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-scalars-in-buildvector.ll
index 5cf0dc6a40a81..14033f28b9316 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-scalars-in-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-scalars-in-buildvector.ll
@@ -12,7 +12,7 @@ define void @test(ptr %p, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[C:%.*]], i32 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul <4 x i32> [[TMP4]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP12:%.*]] = add <4 x i32> [[TMP11]], [[TMP9]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP12]], ptr [[P:%.*]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/root-trunc-extract-reuse.ll b/llvm/test/Transforms/SLPVectorizer/X86/root-trunc-extract-reuse.ll
index ccde383afa3ed..87dd2cfd20044 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/root-trunc-extract-reuse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/root-trunc-extract-reuse.ll
@@ -17,7 +17,7 @@ define i1 @test() {
 ; CHECK-NEXT:    [[T13:%.*]] = and <2 x i32> [[TMP4]], zeroinitializer
 ; CHECK-NEXT:    br label [[ELSE1:%.*]]
 ; CHECK:       else1:
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[T13]], <2 x i32> poison, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[T13]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[BF_CAST162]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt <2 x i32> [[TMP6]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll b/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll
index 8dbb79cd81617..9e16f5ce5deae 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll
@@ -16,8 +16,8 @@ define void @test(i1 %c, ptr %arg) {
 ; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <2 x ptr> [[TMP4]], <2 x ptr> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, <2 x ptr> [[SHUFFLE1]], <2 x i64> <i64 32, i64 24>
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x ptr> poison, ptr [[ARG]], i32 3
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x ptr> [[TMP5]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x ptr> [[TMP6]], <4 x ptr> [[TMP7]], <4 x i32> <i32 4, i32 5, i32 undef, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x ptr> [[TMP5]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x ptr> [[TMP6]], <4 x ptr> [[TMP7]], <4 x i32> <i32 4, i32 5, i32 poison, i32 3>
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x ptr> [[TMP8]], ptr [[ARG_1]], i32 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> [[TMP9]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> poison)
 ; CHECK-NEXT:    br label [[JOIN]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-multiple-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-multiple-nodes.ll
index 370dc46a01f99..b30facfe6af77 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-multiple-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-multiple-nodes.ll
@@ -5,9 +5,9 @@ define i32 @main(<16 x i32> %bc47.i, <16 x i32> %bc) {
 ; CHECK-LABEL: define i32 @main
 ; CHECK-SAME: (<16 x i32> [[BC47_I:%.*]], <16 x i32> [[BC:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i32> [[BC]], <16 x i32> [[BC47_I]], <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 16, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i32> [[BC]], <16 x i32> [[BC47_I]], <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 0, i32 0>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 5, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 -1, i32 0, i32 0>, <8 x i32> <i32 undef, i32 0, i32 undef, i32 5, i32 13, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 -1, i32 0, i32 0>, <8 x i32> <i32 poison, i32 0, i32 poison, i32 5, i32 13, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> <i32 1, i32 undef, i32 0, i32 undef, i32 undef, i32 0, i32 0, i32 0>, <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 12, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub <8 x i32> [[TMP4]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll
index c4b2a9c33a014..d65e5b37c6f0e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll
@@ -1092,8 +1092,8 @@ define <4 x double> @sitofp_4xi32_4f64(i32 %a0, i32 %a1, i32 %a2, i32 %a3) #0 {
 ; SSE-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[A2:%.*]], i32 0
 ; SSE-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[A3:%.*]], i32 1
 ; SSE-NEXT:    [[TMP6:%.*]] = sitofp <2 x i32> [[TMP5]] to <2 x double>
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[RES31:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:    ret <4 x double> [[RES31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
index 83205cc772994..791f3008cea79 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
@@ -1092,8 +1092,8 @@ define <4 x double> @sitofp_4xi32_4f64(i32 %a0, i32 %a1, i32 %a2, i32 %a3) #0 {
 ; SSE-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[A2:%.*]], i32 0
 ; SSE-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[A3:%.*]], i32 1
 ; SSE-NEXT:    [[TMP6:%.*]] = sitofp <2 x i32> [[TMP5]] to <2 x double>
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
-; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[RES31:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SSE-NEXT:    ret <4 x double> [[RES31]]
 ;
@@ -1122,7 +1122,7 @@ define <4 x double> @sitofp_with_const_4xi32_4f64(i32 %a2, i32 %a3) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[A3:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <2 x i32> [[TMP2]] to <2 x double>
 ; CHECK-NEXT:    [[RES0:%.*]] = insertelement <4 x double> undef, double 1.000000e+00, i32 3
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[RES31:%.*]] = shufflevector <4 x double> [[RES0]], <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x double> [[RES31]]
 ;
@@ -1139,7 +1139,7 @@ define <4 x double> @sitofp_with_undef_4xi32_4f64(i32 %a2, i32 %a3) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[A2:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[A3:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <2 x i32> [[TMP2]] to <2 x double>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x double> [[TMP4]]
 ;
   %cvt2 = sitofp i32 %a2 to double
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-load8_2-unord.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-load8_2-unord.ll
index 3b0710f72c8d4..63d13452bc96d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-load8_2-unord.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-load8_2-unord.ll
@@ -103,8 +103,8 @@ define dso_local void @test_unordered_splits(ptr nocapture %p) local_unnamed_add
 ; CHECK-NEXT:    [[G20:%.*]] = getelementptr inbounds [16 x i32], ptr [[P2]], i32 0, i64 12
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[G10]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr [[G20]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 7, i32 5, i32 6, i32 4>
 ; CHECK-NEXT:    store <8 x i32> [[SHUFFLE]], ptr [[P:%.*]], align 4
@@ -162,12 +162,12 @@ define dso_local void @test_cost_splits(ptr nocapture %p) local_unnamed_addr {
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr [[G12]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i32>, ptr [[G20]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i32>, ptr [[G22]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP12]], <8 x i32> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    store <8 x i32> [[TMP14]], ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
index 8559021b49950..0d6d0c2dc1533 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -160,7 +160,7 @@ define void @tiny_tree_not_fully_vectorizable2(ptr noalias nocapture %dst, ptr n
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    store <4 x float> [[TMP7]], ptr [[DST_ADDR_022]], align 4
 ; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds float, ptr [[SRC_ADDR_021]], i64 [[I_023]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
index 0ba168c1bb1ae..7bcb2ece77921 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
@@ -49,7 +49,7 @@ define void @test(ptr nocapture %t2) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP4]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP4]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    [[T71:%.*]] = insertelement <8 x i32> [[TMP11]], i32 [[T34]], i32 6
 ; CHECK-NEXT:    [[T76:%.*]] = shl <8 x i32> [[T71]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
 ; CHECK-NEXT:    store <8 x i32> [[T76]], ptr [[T2]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll
index 5347ae31ed6cb..82f8aa5f9be1b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias.ll
@@ -49,7 +49,7 @@ define void @test(ptr nocapture %t2) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP4]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP4]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 poison, i32 3>
 ; CHECK-NEXT:    [[T71:%.*]] = insertelement <8 x i32> [[TMP11]], i32 [[T34]], i32 6
 ; CHECK-NEXT:    [[T76:%.*]] = shl <8 x i32> [[T71]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
 ; CHECK-NEXT:    store <8 x i32> [[T76]], ptr [[T2]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll
index bea9ac26ba0d2..69ecf1852aedd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias_external_insert_shuffled.ll
@@ -40,11 +40,11 @@ define void @test(ptr nocapture %t2) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[T9]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[T48]], i32 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> <i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T67:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T32]], i32 2
 ; CHECK-NEXT:    [[T68:%.*]] = insertelement <8 x i32> [[T67]], i32 [[T49]], i32 3
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[T701:%.*]] = shufflevector <8 x i32> [[T68]], <8 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[T701:%.*]] = shufflevector <8 x i32> [[T68]], <8 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[T71:%.*]] = insertelement <8 x i32> [[T701]], i32 [[T34]], i32 6
 ; CHECK-NEXT:    [[T72:%.*]] = insertelement <8 x i32> [[T71]], i32 [[T49]], i32 7
 ; CHECK-NEXT:    [[T76:%.*]] = shl <8 x i32> [[T72]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
index e378909b49b92..6ac6884ca5377 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
@@ -8,7 +8,7 @@ define void @test(ptr %a, ptr %b) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr null, align 4
 ; CHECK-NEXT:    [[ARRAYIDX120:%.*]] = getelementptr [4 x float], ptr [[B:%.*]], i64 0, i64 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[ARRAYIDX120]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr null, align 4
diff --git a/llvm/test/Transforms/SROA/slice-width.ll b/llvm/test/Transforms/SROA/slice-width.ll
index 43d8a0a19763b..eabb6978c9125 100644
--- a/llvm/test/Transforms/SROA/slice-width.ll
+++ b/llvm/test/Transforms/SROA/slice-width.ll
@@ -96,7 +96,7 @@ define i32 @memcpy_vec3float_widening(ptr %x) {
 ; CHECK-LABEL: @memcpy_vec3float_widening(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1_SROA_0_0_COPYLOAD:%.*]] = load <3 x float>, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[TMP1_SROA_0_0_VEC_EXPAND:%.*]] = shufflevector <3 x float> [[TMP1_SROA_0_0_COPYLOAD]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[TMP1_SROA_0_0_VEC_EXPAND:%.*]] = shufflevector <3 x float> [[TMP1_SROA_0_0_COPYLOAD]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    [[TMP1_SROA_0_0_VECBLEND:%.*]] = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x float> [[TMP1_SROA_0_0_VEC_EXPAND]], <4 x float> undef
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca [[S_VEC3FLOAT:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1_SROA_0_0_VEC_EXTRACT:%.*]] = shufflevector <4 x float> [[TMP1_SROA_0_0_VECBLEND]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
diff --git a/llvm/test/Transforms/SROA/vector-conversion.ll b/llvm/test/Transforms/SROA/vector-conversion.ll
index 3a798ab31956b..339810cb03ead 100644
--- a/llvm/test/Transforms/SROA/vector-conversion.ll
+++ b/llvm/test/Transforms/SROA/vector-conversion.ll
@@ -7,11 +7,11 @@ define <4 x i64> @vector_ptrtoint({<2 x ptr>, <2 x ptr>} %x) {
 ; CHECK-LABEL: @vector_ptrtoint(
 ; CHECK-NEXT:    [[X_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[X:%.*]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint <2 x ptr> [[X_FCA_0_EXTRACT]] to <2 x i64>
-; CHECK-NEXT:    [[A_SROA_0_0_VEC_EXPAND:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A_SROA_0_0_VEC_EXPAND:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[A_SROA_0_0_VECBLEND:%.*]] = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i64> [[A_SROA_0_0_VEC_EXPAND]], <4 x i64> undef
 ; CHECK-NEXT:    [[X_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x ptr>, <2 x ptr> } [[X]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint <2 x ptr> [[X_FCA_1_EXTRACT]] to <2 x i64>
-; CHECK-NEXT:    [[A_SROA_0_16_VEC_EXPAND:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT:    [[A_SROA_0_16_VEC_EXPAND:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    [[A_SROA_0_16_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i64> [[A_SROA_0_16_VEC_EXPAND]], <4 x i64> [[A_SROA_0_0_VECBLEND]]
 ; CHECK-NEXT:    ret <4 x i64> [[A_SROA_0_16_VECBLEND]]
 ;
@@ -28,11 +28,11 @@ define <4 x ptr> @vector_inttoptr({<2 x i64>, <2 x i64>} %x) {
 ; CHECK-LABEL: @vector_inttoptr(
 ; CHECK-NEXT:    [[X_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[X:%.*]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = inttoptr <2 x i64> [[X_FCA_0_EXTRACT]] to <2 x ptr>
-; CHECK-NEXT:    [[A_SROA_0_0_VEC_EXPAND:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A_SROA_0_0_VEC_EXPAND:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[A_SROA_0_0_VECBLEND:%.*]] = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x ptr> [[A_SROA_0_0_VEC_EXPAND]], <4 x ptr> undef
 ; CHECK-NEXT:    [[X_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[X]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr <2 x i64> [[X_FCA_1_EXTRACT]] to <2 x ptr>
-; CHECK-NEXT:    [[A_SROA_0_16_VEC_EXPAND:%.*]] = shufflevector <2 x ptr> [[TMP2]], <2 x ptr> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT:    [[A_SROA_0_16_VEC_EXPAND:%.*]] = shufflevector <2 x ptr> [[TMP2]], <2 x ptr> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    [[A_SROA_0_16_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x ptr> [[A_SROA_0_16_VEC_EXPAND]], <4 x ptr> [[A_SROA_0_0_VECBLEND]]
 ; CHECK-NEXT:    ret <4 x ptr> [[A_SROA_0_16_VECBLEND]]
 ;
diff --git a/llvm/test/Transforms/SROA/vector-promotion.ll b/llvm/test/Transforms/SROA/vector-promotion.ll
index c748e36a74714..b1a09cb33b766 100644
--- a/llvm/test/Transforms/SROA/vector-promotion.ll
+++ b/llvm/test/Transforms/SROA/vector-promotion.ll
@@ -518,13 +518,13 @@ define <4 x float> @test_subvec_memcpy(ptr %x, ptr %y, ptr %z, ptr %f, ptr %out)
 ; CHECK-LABEL: @test_subvec_memcpy(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_0_COPYLOAD:%.*]] = load <2 x float>, ptr [[X:%.*]], align 1
-; CHECK-NEXT:    [[A_0_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_0_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[A_0_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_0_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[A_0_VECBLEND:%.*]] = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x float> [[A_0_VEC_EXPAND]], <4 x float> undef
 ; CHECK-NEXT:    [[A_4_COPYLOAD:%.*]] = load <2 x float>, ptr [[Y:%.*]], align 1
-; CHECK-NEXT:    [[A_4_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_4_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[A_4_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_4_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    [[A_4_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 false>, <4 x float> [[A_4_VEC_EXPAND]], <4 x float> [[A_0_VECBLEND]]
 ; CHECK-NEXT:    [[A_8_COPYLOAD:%.*]] = load <2 x float>, ptr [[Z:%.*]], align 1
-; CHECK-NEXT:    [[A_8_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_8_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT:    [[A_8_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_8_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>
 ; CHECK-NEXT:    [[A_8_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x float> [[A_8_VEC_EXPAND]], <4 x float> [[A_4_VECBLEND]]
 ; CHECK-NEXT:    [[A_12_COPYLOAD:%.*]] = load float, ptr [[F:%.*]], align 1
 ; CHECK-NEXT:    [[A_12_VEC_INSERT:%.*]] = insertelement <4 x float> [[A_8_VECBLEND]], float [[A_12_COPYLOAD]], i32 3
@@ -536,15 +536,15 @@ define <4 x float> @test_subvec_memcpy(ptr %x, ptr %y, ptr %z, ptr %f, ptr %out)
 ; DEBUG-NEXT:  entry:
 ; DEBUG-NEXT:    call void @llvm.dbg.value(metadata ptr undef, metadata [[META232:![0-9]+]], metadata !DIExpression()), !dbg [[DBG237:![0-9]+]]
 ; DEBUG-NEXT:    [[A_0_COPYLOAD:%.*]] = load <2 x float>, ptr [[X:%.*]], align 1, !dbg [[DBG238:![0-9]+]]
-; DEBUG-NEXT:    [[A_0_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_0_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>, !dbg [[DBG238]]
+; DEBUG-NEXT:    [[A_0_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_0_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>, !dbg [[DBG238]]
 ; DEBUG-NEXT:    [[A_0_VECBLEND:%.*]] = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x float> [[A_0_VEC_EXPAND]], <4 x float> undef, !dbg [[DBG238]]
 ; DEBUG-NEXT:    call void @llvm.dbg.value(metadata ptr undef, metadata [[META233:![0-9]+]], metadata !DIExpression()), !dbg [[DBG239:![0-9]+]]
 ; DEBUG-NEXT:    [[A_4_COPYLOAD:%.*]] = load <2 x float>, ptr [[Y:%.*]], align 1, !dbg [[DBG240:![0-9]+]]
-; DEBUG-NEXT:    [[A_4_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_4_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>, !dbg [[DBG240]]
+; DEBUG-NEXT:    [[A_4_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_4_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>, !dbg [[DBG240]]
 ; DEBUG-NEXT:    [[A_4_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 false>, <4 x float> [[A_4_VEC_EXPAND]], <4 x float> [[A_0_VECBLEND]], !dbg [[DBG240]]
 ; DEBUG-NEXT:    call void @llvm.dbg.value(metadata ptr undef, metadata [[META234:![0-9]+]], metadata !DIExpression()), !dbg [[DBG241:![0-9]+]]
 ; DEBUG-NEXT:    [[A_8_COPYLOAD:%.*]] = load <2 x float>, ptr [[Z:%.*]], align 1, !dbg [[DBG242:![0-9]+]]
-; DEBUG-NEXT:    [[A_8_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_8_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>, !dbg [[DBG242]]
+; DEBUG-NEXT:    [[A_8_VEC_EXPAND:%.*]] = shufflevector <2 x float> [[A_8_COPYLOAD]], <2 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 0, i32 1>, !dbg [[DBG242]]
 ; DEBUG-NEXT:    [[A_8_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x float> [[A_8_VEC_EXPAND]], <4 x float> [[A_4_VECBLEND]], !dbg [[DBG242]]
 ; DEBUG-NEXT:    call void @llvm.dbg.value(metadata ptr undef, metadata [[META235:![0-9]+]], metadata !DIExpression()), !dbg [[DBG243:![0-9]+]]
 ; DEBUG-NEXT:    [[A_12_COPYLOAD:%.*]] = load float, ptr [[F:%.*]], align 1, !dbg [[DBG244:![0-9]+]]
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo
deleted file mode 100644
index b08efbecf55fe..0000000000000
Binary files a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo and /dev/null differ
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.extbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.extbinary.afdo
new file mode 100644
index 0000000000000..51cac9231fcf6
Binary files /dev/null and b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.extbinary.afdo differ
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline.compactbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/inline.compactbinary.afdo
deleted file mode 100644
index c9fde76998281..0000000000000
Binary files a/llvm/test/Transforms/SampleProfile/Inputs/inline.compactbinary.afdo and /dev/null differ
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching.prof b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching.prof
new file mode 100644
index 0000000000000..4d6241bb8568d
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/pseudo-probe-stale-profile-matching.prof
@@ -0,0 +1,28 @@
+main:1497:0
+ 1: 0
+ 2: 112
+ 3: 112 bar:60 dummy_calltarget:50
+ 4: 116
+ 5: 0
+ 7: 124 bar:124
+ 9: 126 bar:126
+ 6: foo:452
+  1: 112
+  2: 101
+  3: 13
+  4: 112
+  5: 101 bar:109
+  6: 13 bar:14
+  !CFGChecksum: 563022570642068
+ 8: foo:472
+  1: 117
+  2: 104
+  3: 13
+  4: 121
+  5: 104 bar:104
+  6: 14 bar:14
+  !CFGChecksum: 563022570642068
+ !CFGChecksum: 1125988587804525
+bar:491:491
+ 1: 491
+ !CFGChecksum: 4294967295
diff --git a/llvm/test/Transforms/SampleProfile/indirect-call.ll b/llvm/test/Transforms/SampleProfile/indirect-call.ll
index 53aaa977c1807..089474c7c149a 100644
--- a/llvm/test/Transforms/SampleProfile/indirect-call.ll
+++ b/llvm/test/Transforms/SampleProfile/indirect-call.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/indirect-call.prof | FileCheck %s
-; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/indirect-call.compact.afdo | FileCheck %s
+; RUN: opt -S %s -passes=sample-profile -sample-profile-file=%S/Inputs/indirect-call.extbinary.afdo | FileCheck %s
 
 ; CHECK-LABEL: @test
 define void @test(ptr) #0 !dbg !3 {
diff --git a/llvm/test/Transforms/SampleProfile/profile-format.ll b/llvm/test/Transforms/SampleProfile/profile-format.ll
index be36267d8e3c9..d9e26404b7ae0 100644
--- a/llvm/test/Transforms/SampleProfile/profile-format.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-format.ll
@@ -1,5 +1,4 @@
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
-; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.compactbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.extbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.md5extbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.fixlenmd5.extbinary.afdo -S | FileCheck %s
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
new file mode 100644
index 0000000000000..54dbc1d50d419
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
@@ -0,0 +1,342 @@
+; REQUIRES: x86_64-linux
+; REQUIRES: asserts
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-stale-profile-matching.prof --salvage-stale-profile -S --debug-only=sample-profile 2>&1 | FileCheck %s
+
+; The profiled source code:
+
+;  volatile int x = 1;
+;  __attribute__((noinline)) int bar(int p) {
+;    return p;
+;  }
+
+;  __attribute__((always_inline)) int foo(int i, int p) {
+;    if (i % 10) return  bar(p);
+;    else return bar(p + 1);
+;  }
+
+;  int main() {
+;    for (int i = 0; i < 1000 * 1000; i++) {
+;       x += foo(i, x);
+;       x += bar(x);
+;       x += foo(i, x);
+;       x += bar(x);
+;    }
+;  }
+
+; The source code for the current build:
+
+;  volatile int x = 1;
+;  __attribute__((noinline)) int bar(int p) {
+;    return p;
+;  }
+
+;  __attribute__((always_inline)) int foo(int i, int p) {
+;    if (i % 10) return  bar(p);
+;    else return bar(p + 1);
+;  }
+
+;  int main() {
+;    if (x == 0)          // code change
+;      return 0;          // code change
+;    for (int i = 0; i < 1000 * 1000; i++) {
+;       x += foo(i, x);
+;       x += bar(x);
+;       if (i < 0)        // code change
+;         return 0;       // code change
+;       x += foo(i, x);
+;       x += bar(x);
+;    }
+;  }
+
+
+; CHECK: Run stale profile matching for main
+
+; CHECK: Location is matched from 1 to 1
+; CHECK: Location is matched from 2 to 2
+; CHECK: Location is matched from 3 to 3
+; CHECK: Location is matched from 4 to 4
+; CHECK: Location is matched from 5 to 5
+; CHECK: Location is matched from 6 to 6
+; CHECK: Location is matched from 7 to 7
+; CHECK: Location is matched from 8 to 8
+; CHECK: Location is matched from 9 to 9
+; CHECK: Location is matched from 10 to 10
+; CHECK: Location is matched from 11 to 11
+
+; CHECK: Callsite with callee:foo is matched from 13 to 6
+; CHECK: Location is rematched backwards from 7 to 0
+; CHECK: Location is rematched backwards from 8 to 1
+; CHECK: Location is rematched backwards from 9 to 2
+; CHECK: Location is rematched backwards from 10 to 3
+; CHECK: Location is rematched backwards from 11 to 4
+; CHECK: Callsite with callee:bar is matched from 14 to 7
+; CHECK: Callsite with callee:foo is matched from 15 to 8
+; CHECK: Callsite with callee:bar is matched from 16 to 9
+
+
+; CHECK:    2:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 2, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
+; CHECK:    3:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 3, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
+; CHECK:    4:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 4, i32 0, i64 -1), !dbg ![[#]] - weight: 116 - factor: 1.00)
+; CHECK:    5:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 5, i32 0, i64 -1), !dbg ![[#]] - weight: 0 - factor: 1.00)
+; CHECK:    1:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
+; CHECK:    2:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 2, i32 0, i64 -1), !dbg ![[#]] - weight: 101 - factor: 1.00)
+; CHECK:    5:  %call.i3 = call i32 @bar(i32 noundef %1), !dbg ![[#]] - weight: 101 - factor: 1.00)
+; CHECK:    3:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 3, i32 0, i64 -1), !dbg ![[#]] - weight: 13 - factor: 1.00)
+; CHECK:    6:  %call1.i6 = call i32 @bar(i32 noundef %add.i5), !dbg ![[#]] - weight: 13 - factor: 1.00)
+; CHECK:    4:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 4, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
+; CHECK:    14:  %call2 = call i32 @bar(i32 noundef %3), !dbg ![[#]] - weight: 124 - factor: 1.00)
+; CHECK:    8:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 8, i32 0, i64 -1), !dbg ![[#]] - weight: 0 - factor: 1.00)
+; CHECK:    1:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1), !dbg ![[#]] - weight: 117 - factor: 1.00)
+; CHECK:    2:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 2, i32 0, i64 -1), !dbg ![[#]] - weight: 104 - factor: 1.00)
+; CHECK:    5:  %call.i = call i32 @bar(i32 noundef %5), !dbg ![[#]] - weight: 104 - factor: 1.00)
+; CHECK:    3:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 3, i32 0, i64 -1), !dbg ![[#]] - weight: 13 - factor: 1.00)
+; CHECK:    6:  %call1.i = call i32 @bar(i32 noundef %add.i), !dbg ![[#]] - weight: 14 - factor: 1.00)
+; CHECK:    4:  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 4, i32 0, i64 -1), !dbg ![[#]] - weight: 121 - factor: 1.00)
+; CHECK:    16:  %call9 = call i32 @bar(i32 noundef %7), !dbg ![[#]] - weight: 126 - factor: 1.00)
+; CHECK:    9:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 9, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
+; CHECK:    10:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 10, i32 0, i64 -1), !dbg ![[#]] - weight: 112 - factor: 1.00)
+; CHECK:    11:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 11, i32 0, i64 -1), !dbg ![[#]] - weight: 116 - factor: 1.00)
+; CHECK:    1:  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 1, i32 0, i64 -1), !dbg ![[#]] - weight: 0 - factor: 1.00)
+
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = dso_local global i32 1, align 4, !dbg !0
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @bar(i32 noundef %p) #0 !dbg !16 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %p, metadata !20, metadata !DIExpression()), !dbg !21
+  call void @llvm.pseudoprobe(i64 -2012135647395072713, i64 1, i32 0, i64 -1), !dbg !22
+  ret i32 %p, !dbg !23
+}
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: alwaysinline nounwind uwtable
+define dso_local i32 @foo(i32 noundef %i, i32 noundef %p) #2 !dbg !24 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %i, metadata !28, metadata !DIExpression()), !dbg !30
+  call void @llvm.dbg.value(metadata i32 %p, metadata !29, metadata !DIExpression()), !dbg !30
+  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1), !dbg !31
+  %rem = srem i32 %i, 10, !dbg !33
+  %tobool = icmp ne i32 %rem, 0, !dbg !33
+  br i1 %tobool, label %if.then, label %if.else, !dbg !34
+
+if.then:                                          ; preds = %entry
+  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 2, i32 0, i64 -1), !dbg !35
+  %call = call i32 @bar(i32 noundef %p), !dbg !36
+  br label %return, !dbg !38
+
+if.else:                                          ; preds = %entry
+  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 3, i32 0, i64 -1), !dbg !39
+  %add = add nsw i32 %p, 1, !dbg !40
+  %call1 = call i32 @bar(i32 noundef %add), !dbg !41
+  br label %return, !dbg !43
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ %call, %if.then ], [ %call1, %if.else ], !dbg !44
+  call void @llvm.pseudoprobe(i64 6699318081062747564, i64 4, i32 0, i64 -1), !dbg !45
+  ret i32 %retval.0, !dbg !45
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main() #3 !dbg !46 {
+entry:
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 1, i32 0, i64 -1), !dbg !52
+  %0 = load volatile i32, ptr @x, align 4, !dbg !52, !tbaa !54
+  %cmp = icmp eq i32 %0, 0, !dbg !58
+  br i1 %cmp, label %if.then, label %if.end, !dbg !59
+
+if.then:                                          ; preds = %entry
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 2, i32 0, i64 -1), !dbg !60
+  br label %for.end, !dbg !60
+
+if.end:                                           ; preds = %entry
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 3, i32 0, i64 -1), !dbg !61
+  call void @llvm.dbg.value(metadata i32 0, metadata !50, metadata !DIExpression()), !dbg !62
+  br label %for.cond, !dbg !63
+
+for.cond:                                         ; preds = %if.end6, %if.end
+  %i.0 = phi i32 [ 0, %if.end ], [ %inc, %if.end6 ], !dbg !64
+  call void @llvm.dbg.value(metadata i32 %i.0, metadata !50, metadata !DIExpression()), !dbg !62
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 4, i32 0, i64 -1), !dbg !65
+  %cmp1 = icmp slt i32 %i.0, 1000000, !dbg !67
+  br i1 %cmp1, label %for.body, label %for.cond.cleanup, !dbg !68
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 5, i32 0, i64 -1), !dbg !68
+  br label %cleanup, !dbg !68
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 6, i32 0, i64 -1), !dbg !69
+  %1 = load volatile i32, ptr @x, align 4, !dbg !71, !tbaa !54
+  %call = call i32 @foo(i32 noundef %i.0, i32 noundef %1), !dbg !72
+  %2 = load volatile i32, ptr @x, align 4, !dbg !74, !tbaa !54
+  %add = add nsw i32 %2, %call, !dbg !74
+  store volatile i32 %add, ptr @x, align 4, !dbg !74, !tbaa !54
+  %3 = load volatile i32, ptr @x, align 4, !dbg !75, !tbaa !54
+  %call2 = call i32 @bar(i32 noundef %3), !dbg !76
+  %4 = load volatile i32, ptr @x, align 4, !dbg !78, !tbaa !54
+  %add3 = add nsw i32 %4, %call2, !dbg !78
+  store volatile i32 %add3, ptr @x, align 4, !dbg !78, !tbaa !54
+  br i1 false, label %if.then5, label %if.end6, !dbg !79
+
+if.then5:                                         ; preds = %for.body
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 7, i32 0, i64 -1), !dbg !80
+  br label %cleanup, !dbg !80
+
+if.end6:                                          ; preds = %for.body
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 8, i32 0, i64 -1), !dbg !82
+  %5 = load volatile i32, ptr @x, align 4, !dbg !83, !tbaa !54
+  %call7 = call i32 @foo(i32 noundef %i.0, i32 noundef %5), !dbg !84
+  %6 = load volatile i32, ptr @x, align 4, !dbg !86, !tbaa !54
+  %add8 = add nsw i32 %6, %call7, !dbg !86
+  store volatile i32 %add8, ptr @x, align 4, !dbg !86, !tbaa !54
+  %7 = load volatile i32, ptr @x, align 4, !dbg !87, !tbaa !54
+  %call9 = call i32 @bar(i32 noundef %7), !dbg !88
+  %8 = load volatile i32, ptr @x, align 4, !dbg !90, !tbaa !54
+  %add10 = add nsw i32 %8, %call9, !dbg !90
+  store volatile i32 %add10, ptr @x, align 4, !dbg !90, !tbaa !54
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 9, i32 0, i64 -1), !dbg !91
+  %inc = add nsw i32 %i.0, 1, !dbg !91
+  call void @llvm.dbg.value(metadata i32 %inc, metadata !50, metadata !DIExpression()), !dbg !62
+  br label %for.cond, !dbg !92, !llvm.loop !93
+
+cleanup:                                          ; preds = %if.then5, %for.cond.cleanup
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 10, i32 0, i64 -1), !dbg !96
+  br label %for.end
+
+for.end:                                          ; preds = %cleanup, %if.then
+  call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 11, i32 0, i64 -1), !dbg !97
+  ret i32 0, !dbg !97
+}
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #4
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #4
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) #1
+
+; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
+declare void @llvm.pseudoprobe(i64, i64, i32, i64) #5
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare void @llvm.dbg.value(metadata, metadata, metadata) #6
+
+attributes #0 = { noinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { alwaysinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" }
+attributes #3 = { nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" }
+attributes #4 = { mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #5 = { mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
+attributes #6 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9, !10, !11}
+!llvm.ident = !{!12}
+!llvm.pseudo_probe_desc = !{!13, !14, !15}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "x", scope: !2, file: !3, line: 1, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C11, file: !3, producer: "clang version 17.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "test.c", directory: "path")
+!4 = !{!0}
+!5 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !6)
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 7, !"Dwarf Version", i32 5}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{i32 7, !"uwtable", i32 2}
+!11 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
+!12 = !{!"clang version 17.0.0"}
+!13 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
+!14 = !{i64 6699318081062747564, i64 563022570642068, !"foo"}
+!15 = !{i64 -2624081020897602054, i64 1126158552146340, !"main"}
+!16 = distinct !DISubprogram(name: "bar", scope: !3, file: !3, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !19)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!6, !6}
+!19 = !{!20}
+!20 = !DILocalVariable(name: "p", arg: 1, scope: !16, file: !3, line: 2, type: !6)
+!21 = !DILocation(line: 0, scope: !16)
+!22 = !DILocation(line: 3, column: 10, scope: !16)
+!23 = !DILocation(line: 3, column: 3, scope: !16)
+!24 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 6, type: !25, scopeLine: 6, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !27)
+!25 = !DISubroutineType(types: !26)
+!26 = !{!6, !6, !6}
+!27 = !{!28, !29}
+!28 = !DILocalVariable(name: "i", arg: 1, scope: !24, file: !3, line: 6, type: !6)
+!29 = !DILocalVariable(name: "p", arg: 2, scope: !24, file: !3, line: 6, type: !6)
+!30 = !DILocation(line: 0, scope: !24)
+!31 = !DILocation(line: 7, column: 6, scope: !32)
+!32 = distinct !DILexicalBlock(scope: !24, file: !3, line: 7, column: 6)
+!33 = !DILocation(line: 7, column: 8, scope: !32)
+!34 = !DILocation(line: 7, column: 6, scope: !24)
+!35 = !DILocation(line: 7, column: 26, scope: !32)
+!36 = !DILocation(line: 7, column: 22, scope: !37)
+!37 = !DILexicalBlockFile(scope: !32, file: !3, discriminator: 186646575)
+!38 = !DILocation(line: 7, column: 14, scope: !32)
+!39 = !DILocation(line: 8, column: 19, scope: !32)
+!40 = !DILocation(line: 8, column: 21, scope: !32)
+!41 = !DILocation(line: 8, column: 15, scope: !42)
+!42 = !DILexicalBlockFile(scope: !32, file: !3, discriminator: 186646583)
+!43 = !DILocation(line: 8, column: 8, scope: !32)
+!44 = !DILocation(line: 0, scope: !32)
+!45 = !DILocation(line: 9, column: 1, scope: !24)
+!46 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 11, type: !47, scopeLine: 11, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !49)
+!47 = !DISubroutineType(types: !48)
+!48 = !{!6}
+!49 = !{!50}
+!50 = !DILocalVariable(name: "i", scope: !51, file: !3, line: 14, type: !6)
+!51 = distinct !DILexicalBlock(scope: !46, file: !3, line: 14, column: 3)
+!52 = !DILocation(line: 12, column: 6, scope: !53)
+!53 = distinct !DILexicalBlock(scope: !46, file: !3, line: 12, column: 6)
+!54 = !{!55, !55, i64 0}
+!55 = !{!"int", !56, i64 0}
+!56 = !{!"omnipotent char", !57, i64 0}
+!57 = !{!"Simple C/C++ TBAA"}
+!58 = !DILocation(line: 12, column: 8, scope: !53)
+!59 = !DILocation(line: 12, column: 6, scope: !46)
+!60 = !DILocation(line: 13, column: 5, scope: !53)
+!61 = !DILocation(line: 14, column: 11, scope: !51)
+!62 = !DILocation(line: 0, scope: !51)
+!63 = !DILocation(line: 14, column: 7, scope: !51)
+!64 = !DILocation(line: 14, scope: !51)
+!65 = !DILocation(line: 14, column: 18, scope: !66)
+!66 = distinct !DILexicalBlock(scope: !51, file: !3, line: 14, column: 3)
+!67 = !DILocation(line: 14, column: 20, scope: !66)
+!68 = !DILocation(line: 14, column: 3, scope: !51)
+!69 = !DILocation(line: 15, column: 15, scope: !70)
+!70 = distinct !DILexicalBlock(scope: !66, file: !3, line: 14, column: 40)
+!71 = !DILocation(line: 15, column: 18, scope: !70)
+!72 = !DILocation(line: 15, column: 11, scope: !73)
+!73 = !DILexicalBlockFile(scope: !70, file: !3, discriminator: 186646639)
+!74 = !DILocation(line: 15, column: 8, scope: !70)
+!75 = !DILocation(line: 16, column: 15, scope: !70)
+!76 = !DILocation(line: 16, column: 11, scope: !77)
+!77 = !DILexicalBlockFile(scope: !70, file: !3, discriminator: 186646647)
+!78 = !DILocation(line: 16, column: 8, scope: !70)
+!79 = !DILocation(line: 17, column: 9, scope: !70)
+!80 = !DILocation(line: 18, column: 8, scope: !81)
+!81 = distinct !DILexicalBlock(scope: !70, file: !3, line: 17, column: 9)
+!82 = !DILocation(line: 19, column: 15, scope: !70)
+!83 = !DILocation(line: 19, column: 18, scope: !70)
+!84 = !DILocation(line: 19, column: 11, scope: !85)
+!85 = !DILexicalBlockFile(scope: !70, file: !3, discriminator: 186646655)
+!86 = !DILocation(line: 19, column: 8, scope: !70)
+!87 = !DILocation(line: 20, column: 15, scope: !70)
+!88 = !DILocation(line: 20, column: 11, scope: !89)
+!89 = !DILexicalBlockFile(scope: !70, file: !3, discriminator: 186646663)
+!90 = !DILocation(line: 20, column: 8, scope: !70)
+!91 = !DILocation(line: 14, column: 36, scope: !66)
+!92 = !DILocation(line: 14, column: 3, scope: !66)
+!93 = distinct !{!93, !68, !94, !95}
+!94 = !DILocation(line: 21, column: 3, scope: !51)
+!95 = !{!"llvm.loop.mustprogress"}
+!96 = !DILocation(line: 0, scope: !46)
+!97 = !DILocation(line: 22, column: 1, scope: !46)
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/pr62379-zeroext-negative.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/pr62379-zeroext-negative.ll
new file mode 100644
index 0000000000000..16c1b52070e3f
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/pr62379-zeroext-negative.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S '-passes=separate-const-offset-from-gep<lower-gep>' < %s | FileCheck %s
+
+; These tests check that this pass does not incorrectly handle negative offsets.
+
+@c = internal constant [4 x i32] [i32 0, i32 1, i32 2, i32 3]
+
+; FIXME: We could optimize this case, but we don't currently.  This test just
+; checks that we don't miscompile it.
+define i32 @sub_positive(i32 %a, i32 %b, ptr %ptr) {
+; CHECK-LABEL: define i32 @sub_positive
+; CHECK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw nsw i32 15, [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw nsw i32 [[B]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [4 x i32], ptr [[PTR]], i64 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+entry:
+  %0 = sub nuw nsw i32 15, %a
+  %1 = sub nuw nsw i32 %b, %0
+  %2 = zext i32 %1 to i64
+  %3 = getelementptr inbounds [4 x i32], ptr %ptr, i64 0, i64 %2
+  %4 = load i32, ptr %3
+  ret i32 %4
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
new file mode 100644
index 0000000000000..b44f09ca81c4d
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
@@ -0,0 +1,432 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -S < %s | FileCheck %s
+
+; Non-trivial loop unswitching of select instruction.
+
+declare i1 @foo()
+declare i1 @bar(i32)
+
+define i32 @basic(i32 %N, i1 %cond, i32 %select_input) {
+; CHECK-LABEL: define i32 @basic
+; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND:%.*]], i32 [[SELECT_INPUT:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[COND]], i32 [[SELECT_INPUT]], i32 42
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[COND1]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %cond1 = select i1 %cond, i32 %select_input, i32 42
+  %add = add nuw nsw i32 %cond1, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define i32 @select_phi_input(i32 %N, i1 %cond) {
+; CHECK-LABEL: define i32 @select_phi_input
+; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[COND]], i32 [[I]], i32 42
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[COND1]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %cond1 = select i1 %cond, i32 %i, i32 42
+  %add = add nuw nsw i32 %cond1, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define i32 @basic_cond_noundef(i32 %N, i1 noundef %cond) {
+; CHECK-LABEL: define i32 @basic_cond_noundef
+; CHECK-SAME: (i32 [[N:%.*]], i1 noundef [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[COND]], i32 [[I]], i32 42
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[COND1]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %cond1 = select i1 %cond, i32 %i, i32 42
+  %add = add nuw nsw i32 %cond1, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define i32 @cond_invariant(i32 %N) {
+; CHECK-LABEL: define i32 @cond_invariant
+; CHECK-SAME: (i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[COND:%.*]] = call i1 @foo()
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[COND]], i32 [[I]], i32 42
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[COND1]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %cond = call i1 @foo()
+  %cond1 = select i1 %cond, i32 %i, i32 42
+  %add = add nuw nsw i32 %cond1, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define i32 @chained_select(i32 %N, i1 %cond, i1 %cond2) {
+; CHECK-LABEL: define i32 @chained_select
+; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[SELECT1:%.*]] = select i1 [[COND]], i32 [[I]], i32 42
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[COND2]], i32 [[SELECT1]], i32 24
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[SELECT2]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %select1 = select i1 %cond, i32 %i, i32 42
+  %select2 = select i1 %cond2, i32 %select1, i32 24
+  %add = add nuw nsw i32 %select2, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define i32 @select_in_if(i32 %N, i1 %cond) {
+; CHECK-LABEL: define i32 @select_in_if
+; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY_END:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY_END]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[UREM:%.*]] = urem i32 [[I]], 2
+; CHECK-NEXT:    [[IF_COND:%.*]] = icmp eq i32 [[UREM]], 0
+; CHECK-NEXT:    br i1 [[IF_COND]], label [[FOR_BODY_IF:%.*]], label [[FOR_BODY_END]]
+; CHECK:       for.body.if:
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[COND]], i32 [[I]], i32 42
+; CHECK-NEXT:    br label [[FOR_BODY_END]]
+; CHECK:       for.body.end:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[COND1]], [[FOR_BODY_IF]] ], [ 24, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[P]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body.end, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body.end ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body.end ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %urem = urem i32 %i, 2
+  %if.cond = icmp eq i32 %urem, 0
+  br i1 %if.cond, label %for.body.if, label %for.body.end
+
+for.body.if:                                      ; preds = %for.body
+  %cond1 = select i1 %cond, i32 %i, i32 42
+  br label %for.body.end
+
+for.body.end:                                     ; preds = %for.body, %for.body.if
+  %p = phi i32 [ %cond1, %for.body.if ], [ 24, %for.body ]
+  %add = add nuw nsw i32 %p, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define i32 @select_in_if_else(i32 %N, i1 %cond) {
+; CHECK-LABEL: define i32 @select_in_if_else
+; CHECK-SAME: (i32 [[N:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_BODY_END:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY_END]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[UREM:%.*]] = urem i32 [[I]], 2
+; CHECK-NEXT:    [[IF_COND:%.*]] = icmp eq i32 [[UREM]], 0
+; CHECK-NEXT:    br i1 [[IF_COND]], label [[FOR_BODY_IF:%.*]], label [[FOR_BODY_ELSE:%.*]]
+; CHECK:       for.body.if:
+; CHECK-NEXT:    [[COND1A:%.*]] = select i1 [[COND]], i32 [[I]], i32 42
+; CHECK-NEXT:    br label [[FOR_BODY_END]]
+; CHECK:       for.body.else:
+; CHECK-NEXT:    [[COND1B:%.*]] = select i1 [[COND]], i32 24, i32 [[I]]
+; CHECK-NEXT:    br label [[FOR_BODY_END]]
+; CHECK:       for.body.end:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[COND1A]], [[FOR_BODY_IF]] ], [ [[COND1B]], [[FOR_BODY_ELSE]] ]
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[P]], [[RES]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_COND]] ]
+; CHECK-NEXT:    ret i32 [[RES_LCSSA]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body.end, %entry
+  %res = phi i32 [ 0, %entry ], [ %add, %for.body.end ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body.end ]
+  %cmp = icmp slt i32 %i, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.cond
+  %urem = urem i32 %i, 2
+  %if.cond = icmp eq i32 %urem, 0
+  br i1 %if.cond, label %for.body.if, label %for.body.else
+
+for.body.if:                                      ; preds = %for.body
+  %cond1a = select i1 %cond, i32 %i, i32 42
+  br label %for.body.end
+
+for.body.else:                                    ; preds = %for.body
+  %cond1b = select i1 %cond, i32 24, i32 %i
+  br label %for.body.end
+
+for.body.end:                                     ; preds = %for.body.if, %for.body.else
+  %p = phi i32 [ %cond1a, %for.body.if ], [ %cond1b, %for.body.else ]
+  %add = add nuw nsw i32 %p, %res
+  %inc = add nuw nsw i32 %i, 1
+  br label %for.cond
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret i32 %res
+}
+
+define dso_local void @select_nested_loop(i1 noundef zeroext %cond, i32 noundef %n, i32 noundef %m) {
+; CHECK-LABEL: define dso_local void @select_nested_loop
+; CHECK-SAME: (i1 noundef zeroext [[COND:%.*]], i32 noundef [[N:%.*]], i32 noundef [[M:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP17_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    [[CMP215_NOT:%.*]] = icmp eq i32 [[M]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP17_NOT]], [[CMP215_NOT]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader.us.preheader:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK:       for.cond1.preheader.us:
+; CHECK-NEXT:    [[I_018_US:%.*]] = phi i32 [ [[INC7_US:%.*]], [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY4_US:%.*]]
+; CHECK:       for.body4.us:
+; CHECK-NEXT:    [[J_016_US:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_US]] ], [ [[INC_US:%.*]], [[FOR_BODY4_US]] ]
+; CHECK-NEXT:    [[COND5_US:%.*]] = select i1 [[COND]], i32 [[I_018_US]], i32 [[J_016_US]]
+; CHECK-NEXT:    tail call void @bar(i32 noundef [[COND5_US]])
+; CHECK-NEXT:    [[INC_US]] = add nuw i32 [[J_016_US]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC_US]], [[M]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US]], label [[FOR_BODY4_US]]
+; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.us:
+; CHECK-NEXT:    [[INC7_US]] = add nuw i32 [[I_018_US]], 1
+; CHECK-NEXT:    [[EXITCOND21_NOT:%.*]] = icmp eq i32 [[INC7_US]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND21_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_COND1_PREHEADER_US]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp17.not = icmp eq i32 %n, 0
+  %cmp215.not = icmp eq i32 %m, 0
+  %or.cond = or i1 %cmp17.not, %cmp215.not
+  br i1 %or.cond, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %entry, %for.cond1.for.cond.cleanup3_crit_edge.us
+  %i.018.us = phi i32 [ %inc7.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %entry ]
+  br label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.cond1.preheader.us, %for.body4.us
+  %j.016.us = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us, %for.body4.us ]
+  %cond5.us = select i1 %cond, i32 %i.018.us, i32 %j.016.us
+  tail call void @bar(i32 noundef %cond5.us) #2
+  %inc.us = add nuw i32 %j.016.us, 1
+  %exitcond.not = icmp eq i32 %inc.us, %m
+  br i1 %exitcond.not, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us
+  %inc7.us = add nuw i32 %i.018.us, 1
+  %exitcond21.not = icmp eq i32 %inc7.us, %n
+  br i1 %exitcond21.not, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
+
+define dso_local void @select_invariant_outer_loop(i1 noundef zeroext %cond, i32 noundef %n, i32 noundef %m) {
+; CHECK-LABEL: define dso_local void @select_invariant_outer_loop
+; CHECK-SAME: (i1 noundef zeroext [[COND:%.*]], i32 noundef [[N:%.*]], i32 noundef [[M:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP20_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    [[CMP218_NOT:%.*]] = icmp eq i32 [[M]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[CMP20_NOT]], [[CMP218_NOT]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader.us.preheader:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_US:%.*]]
+; CHECK:       for.cond1.preheader.us:
+; CHECK-NEXT:    [[I_021_US:%.*]] = phi i32 [ [[INC9_US:%.*]], [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US:%.*]] ], [ 0, [[FOR_COND1_PREHEADER_US_PREHEADER]] ]
+; CHECK-NEXT:    [[REM_US:%.*]] = and i32 [[I_021_US]], 1
+; CHECK-NEXT:    [[CMP5_US:%.*]] = icmp eq i32 [[REM_US]], 0
+; CHECK-NEXT:    br label [[FOR_BODY4_US:%.*]]
+; CHECK:       for.body4.us:
+; CHECK-NEXT:    [[J_019_US:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_US]] ], [ [[INC_US:%.*]], [[FOR_BODY4_US]] ]
+; CHECK-NEXT:    [[COND7_US:%.*]] = select i1 [[CMP5_US]], i32 [[I_021_US]], i32 [[J_019_US]]
+; CHECK-NEXT:    tail call void @bar(i32 noundef [[COND7_US]])
+; CHECK-NEXT:    [[INC_US]] = add nuw i32 [[J_019_US]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC_US]], [[M]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US]], label [[FOR_BODY4_US]]
+; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.us:
+; CHECK-NEXT:    [[INC9_US]] = add nuw i32 [[I_021_US]], 1
+; CHECK-NEXT:    [[EXITCOND24_NOT:%.*]] = icmp eq i32 [[INC9_US]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND24_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_COND1_PREHEADER_US]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp20.not = icmp eq i32 %n, 0
+  %cmp218.not = icmp eq i32 %m, 0
+  %or.cond = or i1 %cmp20.not, %cmp218.not
+  br i1 %or.cond, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %entry, %for.cond1.for.cond.cleanup3_crit_edge.us
+  %i.021.us = phi i32 [ %inc9.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %entry ]
+  %rem.us = and i32 %i.021.us, 1
+  %cmp5.us = icmp eq i32 %rem.us, 0
+  br label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.cond1.preheader.us, %for.body4.us
+  %j.019.us = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us, %for.body4.us ]
+  %cond7.us = select i1 %cmp5.us, i32 %i.021.us, i32 %j.019.us
+  tail call void @bar(i32 noundef %cond7.us) #2
+  %inc.us = add nuw i32 %j.019.us, 1
+  %exitcond.not = icmp eq i32 %inc.us, %m
+  br i1 %exitcond.not, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us
+  %inc9.us = add nuw i32 %i.021.us, 1
+  %exitcond24.not = icmp eq i32 %inc9.us, %n
+  br i1 %exitcond24.not, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/convergent.ll b/llvm/test/Transforms/SimplifyCFG/convergent.ll
index bbdba53524806..6ba51e06460c2 100644
--- a/llvm/test/Transforms/SimplifyCFG/convergent.ll
+++ b/llvm/test/Transforms/SimplifyCFG/convergent.ll
@@ -82,6 +82,8 @@ define void @test_02(ptr %y.coerce) convergent {
 ; SINK-NEXT:    [[TMP0:%.*]] = tail call i32 @tid()
 ; SINK-NEXT:    [[REM:%.*]] = and i32 [[TMP0]], 1
 ; SINK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[REM]], 0
+; SINK-NEXT:    [[IDXPROM4:%.*]] = zext i32 [[TMP0]] to i64
+; SINK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[Y_COERCE:%.*]], i64 [[IDXPROM4]]
 ; SINK-NEXT:    br i1 [[CMP_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
 ; SINK:       if.then:
 ; SINK-NEXT:    [[TMP1:%.*]] = tail call i32 @mbcnt(i32 -1, i32 0)
@@ -101,8 +103,6 @@ define void @test_02(ptr %y.coerce) convergent {
 ; SINK-NEXT:    br label [[IF_END]]
 ; SINK:       if.end:
 ; SINK-NEXT:    [[DOTSINK:%.*]] = phi i32 [ [[TMP6]], [[IF_ELSE]] ], [ [[TMP3]], [[IF_THEN]] ]
-; SINK-NEXT:    [[IDXPROM4:%.*]] = zext i32 [[TMP0]] to i64
-; SINK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[Y_COERCE:%.*]], i64 [[IDXPROM4]]
 ; SINK-NEXT:    store i32 [[DOTSINK]], ptr [[ARRAYIDX5]], align 4
 ; SINK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-in-hoist.ll b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-in-hoist.ll
new file mode 100644
index 0000000000000..e57033b345384
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-in-hoist.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; RUN: opt < %s -passes='simplifycfg<no-sink-common-insts;hoist-common-insts>' -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s --check-prefix=HOIST
+
+; Test case based on C++ code with manualy annotated !prof metadata.
+; This is to test that when calls to 'func1' from 'if.then' block
+; and 'if.else' block are hoisted, the branch_weights are merged and
+; attached to merged call rather than dropped.
+;
+; int func1(int a, int b) ;
+; int func2(int a, int b) ;
+
+; int func(int a, int b, bool c) {
+;    int sum= 0;
+;    if(c) {
+;        sum += func1(a, b);
+;    } else {
+;        sum += func1(a, b);
+;        sum -= func2(a, b);
+;    }
+;    return sum;
+; }
+define i32 @_Z4funciib(i32 %a, i32 %b, i1 %c) {
+; HOIST-LABEL: define i32 @_Z4funciib
+; HOIST-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i1 [[C:%.*]]) {
+; HOIST-NEXT:  entry:
+; HOIST-NEXT:    [[CALL:%.*]] = tail call i32 @_Z5func1ii(i32 [[A]], i32 [[B]]), !prof [[PROF0:![0-9]+]]
+; HOIST-NEXT:    br i1 [[C]], label [[IF_END:%.*]], label [[IF_ELSE:%.*]]
+; HOIST:       if.else:
+; HOIST-NEXT:    [[CALL3:%.*]] = tail call i32 @_Z5func2ii(i32 [[A]], i32 [[B]])
+; HOIST-NEXT:    [[SUB:%.*]] = sub i32 [[CALL]], [[CALL3]]
+; HOIST-NEXT:    br label [[IF_END]]
+; HOIST:       if.end:
+; HOIST-NEXT:    [[SUM_0:%.*]] = phi i32 [ [[SUB]], [[IF_ELSE]] ], [ [[CALL]], [[ENTRY:%.*]] ]
+; HOIST-NEXT:    ret i32 [[SUM_0]]
+;
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 @_Z5func1ii(i32 %a, i32 %b), !prof !0
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call1 = tail call i32 @_Z5func1ii(i32 %a, i32 %b), !prof !1
+  %call3 = tail call i32 @_Z5func2ii(i32 %a, i32 %b)
+  %sub = sub i32 %call1, %call3
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %call, %if.then ], [ %sub, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @_Z5func1ii(i32, i32)
+
+declare i32 @_Z5func2ii(i32, i32)
+
+!0 = !{!"branch_weights", i32 10}
+!1 = !{!"branch_weights", i32 90}
+;.
+; HOIST: [[PROF0]] = !{!"branch_weights", i64 100}
+;.
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-in-sink.ll b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-in-sink.ll
new file mode 100644
index 0000000000000..3206746b13a33
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-direct-call-branch-weights-in-sink.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; RUN: opt < %s -passes='simplifycfg<sink-common-insts;no-hoist-common-insts>' -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s --check-prefix=SINK
+
+
+; Test case based on the following C++ code with manualy annotated !prof metadata.
+; This is to test that when calls to 'func1' from 'if.then' and 'if.else' are
+; sinked, the branch weights are merged and attached to sinked call.
+;
+; int func1(int a, int b) ;
+; int func2(int a, int b) ;
+
+; int func(int a, int b, bool c) {
+;    int sum = 0;
+;    if (c) {
+;        sum += func1(a,b);
+;    } else {
+;        b -= func2(a,b);
+;        sum += func1(a,b);
+;    }
+;    return sum;
+; }
+
+define i32 @_Z4funciib(i32 %a, i32 %b, i1 %c) {
+; SINK-LABEL: define i32 @_Z4funciib
+; SINK-SAME: (i32 [[A:%.*]], i32 [[B:%.*]], i1 [[C:%.*]]) {
+; SINK-NEXT:  entry:
+; SINK-NEXT:    br i1 [[C]], label [[IF_END:%.*]], label [[IF_ELSE:%.*]]
+; SINK:       if.else:
+; SINK-NEXT:    [[CALL1:%.*]] = tail call i32 @_Z5func2ii(i32 [[A]], i32 [[B]])
+; SINK-NEXT:    [[SUB:%.*]] = sub i32 [[B]], [[CALL1]]
+; SINK-NEXT:    br label [[IF_END]]
+; SINK:       if.end:
+; SINK-NEXT:    [[SUB_SINK:%.*]] = phi i32 [ [[SUB]], [[IF_ELSE]] ], [ [[B]], [[ENTRY:%.*]] ]
+; SINK-NEXT:    [[CALL2:%.*]] = tail call i32 @_Z5func1ii(i32 [[A]], i32 [[SUB_SINK]]), !prof [[PROF0:![0-9]+]]
+; SINK-NEXT:    ret i32 [[CALL2]]
+;
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call = tail call i32 @_Z5func1ii(i32 %a, i32 %b), !prof !0
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %call1 = tail call i32 @_Z5func2ii(i32 %a, i32 %b)
+  %sub = sub i32 %b, %call1
+  %call2 = tail call i32 @_Z5func1ii(i32 %a, i32 %sub), !prof !1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %sum.0 = phi i32 [ %call, %if.then ], [ %call2, %if.else ]
+  ret i32 %sum.0
+}
+
+declare i32 @_Z5func1ii(i32, i32)
+
+declare i32 @_Z5func2ii(i32, i32)
+
+!0 = !{!"branch_weights", i32 10}
+!1 = !{!"branch_weights", i32 90}
+;.
+; SINK: [[PROF0]] = !{!"branch_weights", i64 100}
+;.
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
index 142a7bd0eb09f..8ea96918335c7 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
@@ -505,7 +505,7 @@ define i31 @load_with_non_power_of_2_element_type(ptr %x) {
 define i32 @load_multiple_extracts_with_constant_idx(ptr %x) {
 ; CHECK-LABEL: @load_multiple_extracts_with_constant_idx(
 ; CHECK-NEXT:    [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[LV]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[LV]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[LV]], [[SHIFT]]
 ; CHECK-NEXT:    [[RES:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret i32 [[RES]]
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll
index 939e11e268660..25a002106a9c0 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll
@@ -22,10 +22,10 @@ define <16 x i32> @test1(<16 x i32> %x, <16 x i32> %y) {
 
 define i32 @test1_reduce(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @test1_reduce(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 6, i32 9, i32 12, i32 15, i32 21, i32 24, i32 27, i32 30, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 23, i32 26, i32 29, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 19, i32 18, i32 17, i32 16, i32 4, i32 7, i32 10, i32 13, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 31, i32 28, i32 25, i32 22, i32 5, i32 8, i32 11, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 6, i32 9, i32 12, i32 15, i32 21, i32 24, i32 27, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 23, i32 26, i32 29, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 19, i32 18, i32 17, i32 16, i32 4, i32 7, i32 10, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 31, i32 28, i32 25, i32 22, i32 5, i32 8, i32 11, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
@@ -130,10 +130,10 @@ define <16 x i32> @test2_1_ins(<16 x i32> %x1, <16 x i32> %x2) {
 
 define <16 x i32> @test2_2(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @test2_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 0, i32 1, i32 12, i32 15, i32 18, i32 19, i32 22, i32 25, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 10, i32 11, i32 13, i32 14, i32 20, i32 21, i32 23, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 29, i32 26, i32 27, i32 30, i32 7, i32 4, i32 5, i32 8, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 17, i32 28, i32 16, i32 31, i32 3, i32 6, i32 2, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 0, i32 1, i32 12, i32 15, i32 18, i32 19, i32 22, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 10, i32 11, i32 13, i32 14, i32 20, i32 21, i32 23, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 29, i32 26, i32 27, i32 30, i32 7, i32 4, i32 5, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 17, i32 28, i32 16, i32 31, i32 3, i32 6, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 19, i32 18, i32 23, i32 22, i32 5, i32 4, i32 21, i32 7, i32 6, i32 20, i32 1, i32 0, i32 17, i32 3, i32 2, i32 16>
@@ -187,10 +187,10 @@ define <16 x i32> @test3_1(<16 x i32> %x, <16 x i32> %y) {
 
 define <16 x i32> @test3_2(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @test3_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 17, i32 21, i32 25, i32 29, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 19, i32 23, i32 27, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 18, i32 22, i32 26, i32 30, i32 3, i32 7, i32 11, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 16, i32 20, i32 24, i32 28, i32 1, i32 5, i32 9, i32 13, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 17, i32 21, i32 25, i32 29, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 19, i32 23, i32 27, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 18, i32 22, i32 26, i32 30, i32 3, i32 7, i32 11, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[Y]], <16 x i32> [[X]], <16 x i32> <i32 16, i32 20, i32 24, i32 28, i32 1, i32 5, i32 9, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 17, i32 21, i32 1, i32 5, i32 4, i32 7, i32 6, i32 0, i32 3, i32 2, i32 20, i32 23, i32 22, i32 16, i32 19, i32 18>
@@ -257,10 +257,10 @@ define <16 x i32> @test23(<16 x i32> %x, <16 x i32> %y) {
 
 define <16 x i32> @testgood(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @testgood(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -276,10 +276,10 @@ define <16 x i32> @testgood(<16 x i32> %x, <16 x i32> %y) {
 
 define <16 x i32> @test_shufshufin(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @test_shufshufin(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -295,10 +295,10 @@ define <16 x i32> @test_shufshufin(<16 x i32> %x, <16 x i32> %y) {
 
 define <16 x i32> @testshufshufout(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @testshufshufout(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -318,10 +318,10 @@ define <16 x i32> @testshufshufout(<16 x i32> %x, <16 x i32> %y) {
 
 define <16 x i32> @testtwoshufout(<16 x i32> %x, <16 x i32> %y) {
 ; CHECK-LABEL: @testtwoshufout(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[X:%.*]], <16 x i32> [[Y:%.*]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[X]], <16 x i32> [[Y]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <16 x i32> [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
@@ -434,11 +434,11 @@ define void @test_31(ptr %src, ptr %dst) {
 
 define <16 x i32> @test_1651256324(<16 x i32> %l0, <16 x i32> %l1, <16 x i32> %l6, <16 x i32> %l7) {
 ; CHECK-LABEL: @test_1651256324(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[L0:%.*]], <16 x i32> [[L6:%.*]], <16 x i32> <i32 0, i32 1, i32 3, i32 10, i32 10, i32 15, i32 20, i32 20, i32 30, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[L1:%.*]], <16 x i32> [[L1]], <16 x i32> <i32 0, i32 1, i32 4, i32 5, i32 7, i32 10, i32 11, i32 11, i32 12, i32 22, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[L1]], <16 x i32> [[L1]], <16 x i32> <i32 0, i32 1, i32 4, i32 5, i32 7, i32 10, i32 11, i32 11, i32 12, i32 22, i32 24, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[L0:%.*]], <16 x i32> [[L6:%.*]], <16 x i32> <i32 0, i32 1, i32 3, i32 10, i32 10, i32 15, i32 20, i32 20, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[L1:%.*]], <16 x i32> [[L1]], <16 x i32> <i32 0, i32 1, i32 4, i32 5, i32 7, i32 10, i32 11, i32 11, i32 12, i32 22, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[L1]], <16 x i32> [[L1]], <16 x i32> <i32 0, i32 1, i32 4, i32 5, i32 7, i32 10, i32 11, i32 11, i32 12, i32 22, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i32> [[L7:%.*]], <16 x i32> [[L7]], <16 x i32> <i32 23, i32 20, i32 29, i32 25, i32 14, i32 21, i32 11, i32 9, i32 2, i32 7, i32 5, i32 15, i32 24, i32 30, i32 26, i32 5>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[L6]], <16 x i32> [[L7]], <16 x i32> <i32 29, i32 13, i32 30, i32 24, i32 0, i32 15, i32 15, i32 8, i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[L6]], <16 x i32> [[L7]], <16 x i32> <i32 29, i32 13, i32 30, i32 24, i32 0, i32 15, i32 15, i32 8, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <16 x i32> [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub <16 x i32> [[TMP1]], [[TMP4]]
 ; CHECK-NEXT:    [[T0:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> <i32 17, i32 7, i32 23, i32 1, i32 2, i32 1, i32 8, i32 10, i32 6, i32 6, i32 18, i32 24, i32 17, i32 9, i32 21, i32 16>
@@ -532,62 +532,62 @@ define dso_local i32 @full(i8* nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i8>, ptr [[TMP22]], align 1
 ; CHECK-NEXT:    [[TMP24:%.*]] = bitcast ptr [[ADD_PTR_2]] to ptr
 ; CHECK-NEXT:    [[TMP25:%.*]] = load <4 x i8>, ptr [[TMP24]], align 1
-; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP25]], <4 x i8> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP26]], <16 x i8> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i8> [[TMP25]], <4 x i8> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP26]], <16 x i8> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <16 x i8> [[TMP28]], <16 x i8> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP31:%.*]] = zext <16 x i8> [[TMP30]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP32:%.*]] = bitcast ptr [[ADD_PTR64_2]] to ptr
 ; CHECK-NEXT:    [[TMP33:%.*]] = load <4 x i8>, ptr [[TMP32]], align 1
-; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <4 x i8> [[TMP33]], <4 x i8> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <16 x i8> [[TMP34]], <16 x i8> [[TMP35]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <4 x i8> [[TMP33]], <4 x i8> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <16 x i8> [[TMP34]], <16 x i8> [[TMP35]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <16 x i8> [[TMP36]], <16 x i8> [[TMP37]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP39:%.*]] = zext <16 x i8> [[TMP38]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP40:%.*]] = sub nsw <16 x i32> [[TMP31]], [[TMP39]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = bitcast ptr [[ARRAYIDX3_3]] to ptr
 ; CHECK-NEXT:    [[TMP42:%.*]] = load <4 x i8>, ptr [[TMP41]], align 1
-; CHECK-NEXT:    [[TMP43:%.*]] = shufflevector <4 x i8> [[TMP42]], <4 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP45:%.*]] = shufflevector <16 x i8> [[TMP43]], <16 x i8> [[TMP44]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP43:%.*]] = shufflevector <4 x i8> [[TMP42]], <4 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP45:%.*]] = shufflevector <16 x i8> [[TMP43]], <16 x i8> [[TMP44]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i8> [[TMP45]], <16 x i8> [[TMP46]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP48:%.*]] = zext <16 x i8> [[TMP47]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP49:%.*]] = bitcast ptr [[ARRAYIDX5_3]] to ptr
 ; CHECK-NEXT:    [[TMP50:%.*]] = load <4 x i8>, ptr [[TMP49]], align 1
-; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <4 x i8> [[TMP50]], <4 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <4 x i8> [[TMP15]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP53:%.*]] = shufflevector <16 x i8> [[TMP51]], <16 x i8> [[TMP52]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <4 x i8> [[TMP50]], <4 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <4 x i8> [[TMP15]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP53:%.*]] = shufflevector <16 x i8> [[TMP51]], <16 x i8> [[TMP52]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <16 x i8> [[TMP53]], <16 x i8> [[TMP54]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP56:%.*]] = zext <16 x i8> [[TMP55]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP57:%.*]] = sub nsw <16 x i32> [[TMP48]], [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = shl nsw <16 x i32> [[TMP57]], <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP59:%.*]] = add nsw <16 x i32> [[TMP58]], [[TMP40]]
-; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> [[TMP59]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> [[TMP59]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> [[TMP59]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> [[TMP59]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP59]], <16 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP64:%.*]] = add nsw <16 x i32> [[TMP61]], [[TMP63]]
 ; CHECK-NEXT:    [[TMP65:%.*]] = sub nsw <16 x i32> [[TMP60]], [[TMP62]]
-; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP69:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP69:%.*]] = shufflevector <16 x i32> [[TMP64]], <16 x i32> [[TMP65]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP70:%.*]] = add nsw <16 x i32> [[TMP67]], [[TMP69]]
 ; CHECK-NEXT:    [[TMP71:%.*]] = sub nsw <16 x i32> [[TMP66]], [[TMP68]]
-; CHECK-NEXT:    [[TMP72:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP73:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP74:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP75:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP72:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP73:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP74:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP75:%.*]] = shufflevector <16 x i32> [[TMP70]], <16 x i32> [[TMP71]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP76:%.*]] = add nsw <16 x i32> [[TMP73]], [[TMP75]]
 ; CHECK-NEXT:    [[TMP77:%.*]] = sub nsw <16 x i32> [[TMP72]], [[TMP74]]
-; CHECK-NEXT:    [[TMP78:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP79:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP81:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP78:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP79:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP81:%.*]] = shufflevector <16 x i32> [[TMP76]], <16 x i32> [[TMP77]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP82:%.*]] = add nsw <16 x i32> [[TMP79]], [[TMP81]]
 ; CHECK-NEXT:    [[TMP83:%.*]] = sub nsw <16 x i32> [[TMP78]], [[TMP80]]
 ; CHECK-NEXT:    [[TMP84:%.*]] = shufflevector <16 x i32> [[TMP82]], <16 x i32> [[TMP83]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
@@ -741,67 +741,67 @@ define i32 @full_reorder(ptr nocapture noundef readonly %pix1, i32 noundef %i_pi
 ; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ADD_PTR_2]], align 1
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP15]], <16 x i8> [[TMP16]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <16 x i8> [[TMP17]], <16 x i8> [[TMP18]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP20:%.*]] = zext <16 x i8> [[TMP19]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_2]], align 1
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP21]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <4 x i8> [[TMP21]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <16 x i8> [[TMP22]], <16 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <16 x i8> [[TMP24]], <16 x i8> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP26]], <16 x i8> [[TMP27]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP29:%.*]] = zext <16 x i8> [[TMP28]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP30:%.*]] = sub nsw <16 x i32> [[TMP20]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_3]], align 1
-; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <4 x i8> [[TMP31]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <4 x i8> [[TMP10]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <4 x i8> [[TMP31]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <4 x i8> [[TMP10]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP34:%.*]] = shufflevector <16 x i8> [[TMP32]], <16 x i8> [[TMP33]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP6]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP6]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <16 x i8> [[TMP34]], <16 x i8> [[TMP35]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <16 x i8> [[TMP36]], <16 x i8> [[TMP37]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP39:%.*]] = zext <16 x i8> [[TMP38]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP40:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
-; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <4 x i8> [[TMP40]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP42:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <4 x i8> [[TMP40]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP42:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP43:%.*]] = shufflevector <16 x i8> [[TMP41]], <16 x i8> [[TMP42]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i8> [[TMP7]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP45:%.*]] = shufflevector <16 x i8> [[TMP43]], <16 x i8> [[TMP44]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i8> [[TMP45]], <16 x i8> [[TMP46]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:    [[TMP48:%.*]] = zext <16 x i8> [[TMP47]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP49:%.*]] = sub nsw <16 x i32> [[TMP39]], [[TMP48]]
 ; CHECK-NEXT:    [[TMP50:%.*]] = shl nsw <16 x i32> [[TMP49]], <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
 ; CHECK-NEXT:    [[TMP51:%.*]] = add nsw <16 x i32> [[TMP50]], [[TMP30]]
-; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> [[TMP51]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP53:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> [[TMP51]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> [[TMP51]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP53:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> [[TMP51]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP54:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP55:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP56:%.*]] = add nsw <16 x i32> [[TMP53]], [[TMP55]]
 ; CHECK-NEXT:    [[TMP57:%.*]] = sub nsw <16 x i32> [[TMP52]], [[TMP54]]
-; CHECK-NEXT:    [[TMP58:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP58:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <16 x i32> [[TMP56]], <16 x i32> [[TMP57]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP62:%.*]] = add nsw <16 x i32> [[TMP59]], [[TMP61]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = sub nsw <16 x i32> [[TMP58]], [[TMP60]]
-; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP64:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP65:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <16 x i32> [[TMP62]], <16 x i32> [[TMP63]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP68:%.*]] = add nsw <16 x i32> [[TMP65]], [[TMP67]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = sub nsw <16 x i32> [[TMP64]], [[TMP66]]
-; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP72:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP73:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP72:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP73:%.*]] = shufflevector <16 x i32> [[TMP68]], <16 x i32> [[TMP69]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP74:%.*]] = add nsw <16 x i32> [[TMP71]], [[TMP73]]
 ; CHECK-NEXT:    [[TMP75:%.*]] = sub nsw <16 x i32> [[TMP70]], [[TMP72]]
 ; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <16 x i32> [[TMP74]], <16 x i32> [[TMP75]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
@@ -984,7 +984,7 @@ define <64 x i32> @testlargerextrashuffle2(i32 %call.i, <16 x i32> %0) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = sub <16 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[TMP3]], <16 x i32> [[TMP4]], <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 20, i32 5, i32 6, i32 23, i32 8, i32 9, i32 26, i32 11, i32 12, i32 29, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <64 x i32> [[TMP6]]
 ;
 entry:
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
index 1079217f5c21e..bdd9ef6b3e68f 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
@@ -132,7 +132,7 @@ define i32 @reduceshuffle_twoin_uneven_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define i32 @reduceshuffle_twoin_undef_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @reduceshuffle_twoin_undef_v4i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 undef, i32 1, i32 5>
+; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 poison, i32 1, i32 5>
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -143,7 +143,7 @@ define i32 @reduceshuffle_twoin_undef_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define i32 @reduceshuffle_twoin_undef2_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @reduceshuffle_twoin_undef2_v4i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -154,7 +154,7 @@ define i32 @reduceshuffle_twoin_undef2_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define i32 @reduceshuffle_twoin_multiundef_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: @reduceshuffle_twoin_multiundef_v4i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
diff --git a/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll
index 9fe30ae6e3b71..36ea20f01871c 100644
--- a/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s --check-prefixes=CHECK
 
 ; ModuleID = 'load-as-transition.ll'
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 %struct.hoge = type { float }
@@ -13,7 +13,7 @@ define protected amdgpu_kernel void @load_from_other_as(ptr nocapture nonnull %r
 ; CHECK-NEXT:    [[A:%.*]] = alloca [[STRUCT_HOGE:%.*]], align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[E:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[E:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    store <4 x float> [[E]], ptr [[RESULTPTR:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition.ll b/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition.ll
index ecf85d1258b0e..94b8c98a80df5 100644
--- a/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition.ll
+++ b/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s --check-prefixes=CHECK
 
 ; ModuleID = 'load-as-transition.ll'
-target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 %struct.hoge = type { float }
@@ -13,7 +13,7 @@ define protected amdgpu_kernel void @load_from_other_as(ptr nocapture nonnull %r
 ; CHECK-NEXT:    [[A:%.*]] = alloca [[STRUCT_HOGE:%.*]], align 4, addrspace(5)
 ; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[E:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[E:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    store <4 x float> [[E]], ptr [[RESULTPTR:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
index a78f93bc5f1c8..97be7304bb0c8 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
@@ -264,7 +264,7 @@ define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext0_ext1_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
 ; AVX-NEXT:    ret i8 [[R]]
@@ -283,7 +283,7 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext5_ext0_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
 ; AVX-NEXT:    ret i8 [[R]]
@@ -302,7 +302,7 @@ define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext1_ext6_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
 ; AVX-NEXT:    ret i8 [[R]]
@@ -315,7 +315,7 @@ define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
 
 define float @ext1_ext0_fmul(<4 x float> %x) {
 ; CHECK-LABEL: @ext1_ext0_fmul(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[SHIFT]], [[X]]
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; CHECK-NEXT:    ret float [[R]]
@@ -330,7 +330,7 @@ define float @ext0_ext3_fmul_extra_use1(<4 x float> %x) {
 ; CHECK-LABEL: @ext0_ext3_fmul_extra_use1(
 ; CHECK-NEXT:    [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
 ; CHECK-NEXT:    call void @use_f32(float [[E0]])
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul nnan <4 x float> [[X]], [[SHIFT]]
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret float [[R]]
@@ -365,7 +365,7 @@ define float @ext0_ext4_fmul_v8f32(<8 x float> %x) {
 ; SSE-NEXT:    ret float [[R]]
 ;
 ; AVX-LABEL: @ext0_ext4_fmul_v8f32(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[X]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
 ; AVX-NEXT:    ret float [[R]]
@@ -384,7 +384,7 @@ define float @ext7_ext4_fmul_v8f32(<8 x float> %x) {
 ; SSE-NEXT:    ret float [[R]]
 ;
 ; AVX-LABEL: @ext7_ext4_fmul_v8f32(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[SHIFT]], [[X]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i64 4
 ; AVX-NEXT:    ret float [[R]]
@@ -423,7 +423,7 @@ define float @ext14_ext15_fmul_v16f32(<16 x float> %x) {
 
 define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 poison, i32 2>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[SHIFT]], [[A]]
 ; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A23]], i32 3
@@ -441,7 +441,7 @@ define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @ins_bo_ext_ext_uses(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext_uses(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 3, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
 ; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
 ; CHECK-NEXT:    call void @use_f32(float [[A23]])
@@ -460,13 +460,13 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @PR34724(
 ; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 3, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
 ; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[B]], [[SHIFT1]]
 ; CHECK-NEXT:    [[B01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 poison, i32 2>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[SHIFT2]], [[B]]
 ; CHECK-NEXT:    [[B23:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> poison, float [[A23]], i32 1
@@ -497,11 +497,11 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_or_reduction_v4i32(
 ; CHECK-NEXT:    [[Z:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = or <4 x i32> [[Z]], [[SHIFT]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
-; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
 ; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
 ; CHECK-NEXT:    ret i32 [[Z0123]]
@@ -519,9 +519,9 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 
 define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
 ; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
 ; CHECK-NEXT:    ret i32 [[X210]]
@@ -536,11 +536,11 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 
 define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
 ; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
 ; CHECK-NEXT:    ret i32 [[X2Y210]]
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index 65584b2e1f37d..c32ce70c2653f 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -264,7 +264,7 @@ define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext0_ext1_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
 ; AVX-NEXT:    ret i8 [[R]]
@@ -283,7 +283,7 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext5_ext0_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
 ; AVX-NEXT:    ret i8 [[R]]
@@ -302,7 +302,7 @@ define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
 ; SSE-NEXT:    ret i8 [[R]]
 ;
 ; AVX-LABEL: @ext1_ext6_add(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> poison, <16 x i32> <i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
 ; AVX-NEXT:    ret i8 [[R]]
@@ -315,7 +315,7 @@ define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
 
 define float @ext1_ext0_fmul(<4 x float> %x) {
 ; CHECK-LABEL: @ext1_ext0_fmul(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[SHIFT]], [[X]]
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; CHECK-NEXT:    ret float [[R]]
@@ -330,7 +330,7 @@ define float @ext0_ext3_fmul_extra_use1(<4 x float> %x) {
 ; CHECK-LABEL: @ext0_ext3_fmul_extra_use1(
 ; CHECK-NEXT:    [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
 ; CHECK-NEXT:    call void @use_f32(float [[E0]])
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul nnan <4 x float> [[X]], [[SHIFT]]
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret float [[R]]
@@ -365,7 +365,7 @@ define float @ext0_ext4_fmul_v8f32(<8 x float> %x) {
 ; SSE-NEXT:    ret float [[R]]
 ;
 ; AVX-LABEL: @ext0_ext4_fmul_v8f32(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[X]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
 ; AVX-NEXT:    ret float [[R]]
@@ -384,7 +384,7 @@ define float @ext7_ext4_fmul_v8f32(<8 x float> %x) {
 ; SSE-NEXT:    ret float [[R]]
 ;
 ; AVX-LABEL: @ext7_ext4_fmul_v8f32(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[SHIFT]], [[X]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i64 4
 ; AVX-NEXT:    ret float [[R]]
@@ -423,7 +423,7 @@ define float @ext14_ext15_fmul_v16f32(<16 x float> %x) {
 
 define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 poison, i32 2>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[SHIFT]], [[A]]
 ; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A23]], i32 3
@@ -441,7 +441,7 @@ define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
 
 define <4 x float> @ins_bo_ext_ext_uses(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @ins_bo_ext_ext_uses(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 3, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
 ; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
 ; CHECK-NEXT:    call void @use_f32(float [[A23]])
@@ -460,13 +460,13 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @PR34724(
 ; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 3, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
 ; CHECK-NEXT:    [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[B]], [[SHIFT1]]
 ; CHECK-NEXT:    [[B01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 poison, i32 2>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[SHIFT2]], [[B]]
 ; CHECK-NEXT:    [[B23:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
 ; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[A23]], i32 1
@@ -497,11 +497,11 @@ define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
 define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_or_reduction_v4i32(
 ; CHECK-NEXT:    [[Z:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = or <4 x i32> [[Z]], [[SHIFT]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
-; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
 ; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
 ; CHECK-NEXT:    ret i32 [[Z0123]]
@@ -519,9 +519,9 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 
 define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
 ; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
 ; CHECK-NEXT:    ret i32 [[X210]]
@@ -536,11 +536,11 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 
 define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y]]
-; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
 ; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
 ; CHECK-NEXT:    ret i32 [[X2Y210]]
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
index 733d88e1f9adb..99f1fba8ccf42 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp-binop.ll
@@ -13,7 +13,7 @@ define i1 @fcmp_and_v2f64(<2 x double> %a) {
 ;
 ; AVX-LABEL: @fcmp_and_v2f64(
 ; AVX-NEXT:    [[TMP1:%.*]] = fcmp olt <2 x double> [[A:%.*]], <double 4.200000e+01, double -8.000000e+00>
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <2 x i32> <i32 1, i32 poison>
 ; AVX-NEXT:    [[TMP2:%.*]] = and <2 x i1> [[TMP1]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
 ; AVX-NEXT:    ret i1 [[R]]
@@ -37,7 +37,7 @@ define i1 @fcmp_or_v4f64(<4 x double> %a) {
 ;
 ; AVX-LABEL: @fcmp_or_v4f64(
 ; AVX-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[A:%.*]], <double 4.200000e+01, double undef, double -8.000000e+00, double undef>
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP2:%.*]] = or <4 x i1> [[TMP1]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
 ; AVX-NEXT:    ret i1 [[R]]
@@ -53,7 +53,7 @@ define i1 @fcmp_or_v4f64(<4 x double> %a) {
 define i1 @icmp_xor_v4i32(<4 x i32> %a) {
 ; CHECK-LABEL: @icmp_xor_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A:%.*]], <i32 undef, i32 -8, i32 undef, i32 42>
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 undef, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], [[SHIFT]]
 ; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -79,7 +79,7 @@ define i1 @icmp_add_v8i32(<8 x i32> %a) {
 ;
 ; AVX-LABEL: @icmp_add_v8i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i32> [[A:%.*]], <i32 undef, i32 undef, i32 -8, i32 undef, i32 undef, i32 undef, i32 undef, i32 42>
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <8 x i32> <i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> poison, <8 x i32> <i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP2:%.*]] = add <8 x i1> [[TMP1]], [[SHIFT]]
 ; AVX-NEXT:    [[R:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
 ; AVX-NEXT:    ret i1 [[R]]
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
index 273567acafb5f..64c86a741b177 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
@@ -109,7 +109,7 @@ define i1 @cmp01_v2f64(<2 x double> %x, <2 x double> %y) {
 ; SSE-NEXT:    ret i1 [[CMP]]
 ;
 ; AVX-LABEL: @cmp01_v2f64(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[Y:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[Y:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fcmp oge <2 x double> [[X:%.*]], [[SHIFT]]
 ; AVX-NEXT:    [[CMP:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
 ; AVX-NEXT:    ret i1 [[CMP]]
@@ -128,7 +128,7 @@ define i1 @cmp10_v2f64(<2 x double> %x, <2 x double> %y) {
 ; SSE-NEXT:    ret i1 [[CMP]]
 ;
 ; AVX-LABEL: @cmp10_v2f64(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fcmp ule <2 x double> [[SHIFT]], [[Y:%.*]]
 ; AVX-NEXT:    [[CMP:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
 ; AVX-NEXT:    ret i1 [[CMP]]
@@ -141,7 +141,7 @@ define i1 @cmp10_v2f64(<2 x double> %x, <2 x double> %y) {
 
 define i1 @cmp12_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @cmp12_v4i32(
-; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 undef, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 2, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[SHIFT]]
 ; CHECK-NEXT:    [[CMP:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -161,7 +161,7 @@ define <4 x i1> @ins_fcmp_ext_ext(<4 x float> %a, <4 x i1> %b) {
 ; SSE-NEXT:    ret <4 x i1> [[R]]
 ;
 ; AVX-LABEL: @ins_fcmp_ext_ext(
-; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
+; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 1, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fcmp ugt <4 x float> [[A]], [[SHIFT]]
 ; AVX-NEXT:    [[A21:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
 ; AVX-NEXT:    [[R:%.*]] = insertelement <4 x i1> [[B:%.*]], i1 [[A21]], i32 2
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
index ffb5bf7244dbb..f883282f40c6b 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
@@ -159,7 +159,7 @@ define double @larger_fp_scalar_256bit_vec(ptr align 32 dereferenceable(32) %p)
 define <4 x float> @load_f32_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -170,7 +170,7 @@ define <4 x float> @load_f32_insert_v4f32(ptr align 16 dereferenceable(16) %p) n
 define <4 x float> @casted_load_f32_insert_v4f32(ptr align 4 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @casted_load_f32_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -183,7 +183,7 @@ define <4 x float> @casted_load_f32_insert_v4f32(ptr align 4 dereferenceable(16)
 define <4 x i32> @load_i32_insert_v4i32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_i32_insert_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -196,7 +196,7 @@ define <4 x i32> @load_i32_insert_v4i32(ptr align 16 dereferenceable(16) %p) nof
 define <4 x i32> @casted_load_i32_insert_v4i32(ptr align 4 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @casted_load_i32_insert_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -209,7 +209,7 @@ define <4 x i32> @casted_load_i32_insert_v4i32(ptr align 4 dereferenceable(16) %
 define <4 x float> @gep00_load_f32_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @gep00_load_f32_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 16
@@ -222,7 +222,7 @@ define <4 x float> @gep00_load_f32_insert_v4f32(ptr align 16 dereferenceable(16)
 define <4 x float> @gep00_load_f32_insert_v4f32_addrspace(ptr addrspace(44) align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @gep00_load_f32_insert_v4f32_addrspace(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr addrspace(44) [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr addrspace(44) %p, align 16
@@ -236,7 +236,7 @@ define <4 x i32> @unsafe_load_i32_insert_v4i32_addrspace(ptr align 16 dereferenc
 ; CHECK-LABEL: @unsafe_load_i32_insert_v4i32_addrspace(
 ; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[V3:%.*]] to ptr addrspace(42)
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(42) [[TMP1]], align 16
-; CHECK-NEXT:    [[INSELT:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[INSELT:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[INSELT]]
 ;
   %t0 = getelementptr inbounds i32, ptr %v3, i32 1
@@ -253,7 +253,7 @@ define <8 x i16> @gep01_load_i16_insert_v8i16(ptr align 16 dereferenceable(18) %
 ; CHECK-LABEL: @gep01_load_i16_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[GEP]], align 2
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
@@ -273,7 +273,7 @@ define <8 x i16> @gep01_load_i16_insert_v8i16_deref(ptr align 16 dereferenceable
 ;
 ; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref(
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
@@ -293,7 +293,7 @@ define <8 x i16> @gep01_load_i16_insert_v8i16_deref_minalign(ptr align 2 derefer
 ;
 ; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
@@ -322,7 +322,7 @@ define <4 x i32> @gep01_bitcast_load_i32_insert_v4i32(ptr align 1 dereferenceabl
 define <4 x i32> @gep012_bitcast_load_i32_insert_v4i32(ptr align 1 dereferenceable(20) %p) nofree nosync {
 ; CHECK-LABEL: @gep012_bitcast_load_i32_insert_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 1
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %gep = getelementptr inbounds <16 x i8>, ptr %p, i64 0, i64 12
@@ -354,7 +354,7 @@ define <8 x i16> @gep10_load_i16_insert_v8i16(ptr align 16 dereferenceable(32) %
 ; CHECK-LABEL: @gep10_load_i16_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 1, i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[GEP]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 1, i64 0
@@ -457,7 +457,7 @@ define <4 x float> @load_f32_insert_v4f32_volatile(ptr align 16 dereferenceable(
 define <4 x float> @load_f32_insert_v4f32_align(ptr align 1 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v4f32_align(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -481,7 +481,7 @@ define <4 x float> @load_f32_insert_v4f32_deref(ptr align 4 dereferenceable(15)
 define <8 x i32> @load_i32_insert_v8i32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_i32_insert_v8i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -492,7 +492,7 @@ define <8 x i32> @load_i32_insert_v8i32(ptr align 16 dereferenceable(16) %p) nof
 define <8 x i32> @casted_load_i32_insert_v8i32(ptr align 4 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @casted_load_i32_insert_v8i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -503,7 +503,7 @@ define <8 x i32> @casted_load_i32_insert_v8i32(ptr align 4 dereferenceable(16) %
 define <16 x float> @load_f32_insert_v16f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v16f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -514,7 +514,7 @@ define <16 x float> @load_f32_insert_v16f32(ptr align 16 dereferenceable(16) %p)
 define <2 x float> @load_f32_insert_v2f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v2f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -568,7 +568,7 @@ define void @PR47558_multiple_use_load(ptr nocapture nonnull %resultptr, ptr noc
 define <4 x float> @load_v2f32_extract_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_v2f32_extract_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %l = load <2 x float>, ptr %p, align 4
@@ -580,7 +580,7 @@ define <4 x float> @load_v2f32_extract_insert_v4f32(ptr align 16 dereferenceable
 define <4 x float> @load_v8f32_extract_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_v8f32_extract_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %l = load <8 x float>, ptr %p, align 4
@@ -616,7 +616,7 @@ define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceab
 ;
 ; AVX2-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 4
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <2 x i16>, ptr %p, i64 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
index 11a3dc04442d6..c1030c7d54462 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
@@ -294,7 +294,7 @@ define <4 x float> @load_v3f32_v4f32(ptr dereferenceable(16) %p) {
 define <4 x float> @load_v3f32_v4f32_wrong_mask(ptr dereferenceable(16) %p) {
 ; CHECK-LABEL: @load_v3f32_v4f32_wrong_mask(
 ; CHECK-NEXT:    [[L:%.*]] = load <3 x float>, ptr [[P:%.*]], align 1
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <3 x float> [[L]], <3 x float> poison, <4 x i32> <i32 1, i32 0, i32 2, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <3 x float> [[L]], <3 x float> poison, <4 x i32> <i32 1, i32 0, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[S]]
 ;
   %l = load <3 x float>, ptr %p, align 1
@@ -307,7 +307,7 @@ define <4 x float> @load_v3f32_v4f32_wrong_mask(ptr dereferenceable(16) %p) {
 define <4 x float> @load_v3f32_v4f32_not_deref(ptr dereferenceable(15) %p) {
 ; CHECK-LABEL: @load_v3f32_v4f32_not_deref(
 ; CHECK-NEXT:    [[L:%.*]] = load <3 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <3 x float> [[L]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <3 x float> [[L]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[S]]
 ;
   %l = load <3 x float>, ptr %p, align 16
@@ -320,7 +320,7 @@ define <4 x float> @load_v3f32_v4f32_not_deref(ptr dereferenceable(15) %p) {
 define <8 x float> @load_v2f32_v8f32(ptr dereferenceable(32) %p) {
 ; SSE-LABEL: @load_v2f32_v8f32(
 ; SSE-NEXT:    [[L:%.*]] = load <2 x float>, ptr [[P:%.*]], align 1
-; SSE-NEXT:    [[S:%.*]] = shufflevector <2 x float> [[L]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SSE-NEXT:    [[S:%.*]] = shufflevector <2 x float> [[L]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    ret <8 x float> [[S]]
 ;
 ; AVX-LABEL: @load_v2f32_v8f32(
@@ -349,7 +349,7 @@ define <4 x i32> @load_v2i32_v4i32(ptr dereferenceable(16) %p) {
 define <4 x i32> @load_v2i32_v4i32_non_canonical_mask(ptr dereferenceable(16) %p) {
 ; CHECK-LABEL: @load_v2i32_v4i32_non_canonical_mask(
 ; CHECK-NEXT:    [[L:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x i32> [[L]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x i32> [[L]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %l = load <2 x i32>, ptr %p, align 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
index db40c84483bbd..d5c19b35838d7 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -159,7 +159,7 @@ define double @larger_fp_scalar_256bit_vec(ptr align 32 dereferenceable(32) %p)
 define <4 x float> @load_f32_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -170,7 +170,7 @@ define <4 x float> @load_f32_insert_v4f32(ptr align 16 dereferenceable(16) %p) n
 define <4 x float> @casted_load_f32_insert_v4f32(ptr align 4 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @casted_load_f32_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -183,7 +183,7 @@ define <4 x float> @casted_load_f32_insert_v4f32(ptr align 4 dereferenceable(16)
 define <4 x i32> @load_i32_insert_v4i32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_i32_insert_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -196,7 +196,7 @@ define <4 x i32> @load_i32_insert_v4i32(ptr align 16 dereferenceable(16) %p) nof
 define <4 x i32> @casted_load_i32_insert_v4i32(ptr align 4 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @casted_load_i32_insert_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -209,7 +209,7 @@ define <4 x i32> @casted_load_i32_insert_v4i32(ptr align 4 dereferenceable(16) %
 define <4 x float> @gep00_load_f32_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @gep00_load_f32_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 16
@@ -222,7 +222,7 @@ define <4 x float> @gep00_load_f32_insert_v4f32(ptr align 16 dereferenceable(16)
 define <4 x float> @gep00_load_f32_insert_v4f32_addrspace(ptr addrspace(44) align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @gep00_load_f32_insert_v4f32_addrspace(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr addrspace(44) [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr addrspace(44) %p, align 16
@@ -236,7 +236,7 @@ define <8 x i16> @gep01_load_i16_insert_v8i16(ptr align 16 dereferenceable(18) %
 ; CHECK-LABEL: @gep01_load_i16_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[GEP]], align 2
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
@@ -256,7 +256,7 @@ define <8 x i16> @gep01_load_i16_insert_v8i16_deref(ptr align 16 dereferenceable
 ;
 ; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref(
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 16
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
@@ -276,7 +276,7 @@ define <8 x i16> @gep01_load_i16_insert_v8i16_deref_minalign(ptr align 2 derefer
 ;
 ; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
@@ -305,7 +305,7 @@ define <4 x i32> @gep01_bitcast_load_i32_insert_v4i32(ptr align 1 dereferenceabl
 define <4 x i32> @gep012_bitcast_load_i32_insert_v4i32(ptr align 1 dereferenceable(20) %p) nofree nosync {
 ; CHECK-LABEL: @gep012_bitcast_load_i32_insert_v4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 1
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
   %gep = getelementptr inbounds <16 x i8>, ptr %p, i64 0, i64 12
@@ -337,7 +337,7 @@ define <8 x i16> @gep10_load_i16_insert_v8i16(ptr align 16 dereferenceable(32) %
 ; CHECK-LABEL: @gep10_load_i16_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 1, i64 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[GEP]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 1, i64 0
@@ -440,7 +440,7 @@ define <4 x float> @load_f32_insert_v4f32_volatile(ptr align 16 dereferenceable(
 define <4 x float> @load_f32_insert_v4f32_align(ptr align 1 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v4f32_align(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -464,7 +464,7 @@ define <4 x float> @load_f32_insert_v4f32_deref(ptr align 4 dereferenceable(15)
 define <8 x i32> @load_i32_insert_v8i32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_i32_insert_v8i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -475,7 +475,7 @@ define <8 x i32> @load_i32_insert_v8i32(ptr align 16 dereferenceable(16) %p) nof
 define <8 x i32> @casted_load_i32_insert_v8i32(ptr align 4 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @casted_load_i32_insert_v8i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %s = load i32, ptr %p, align 4
@@ -486,7 +486,7 @@ define <8 x i32> @casted_load_i32_insert_v8i32(ptr align 4 dereferenceable(16) %
 define <16 x float> @load_f32_insert_v16f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v16f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <16 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -497,7 +497,7 @@ define <16 x float> @load_f32_insert_v16f32(ptr align 16 dereferenceable(16) %p)
 define <2 x float> @load_f32_insert_v2f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_f32_insert_v2f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <2 x i32> <i32 0, i32 poison>
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
   %s = load float, ptr %p, align 4
@@ -551,7 +551,7 @@ define void @PR47558_multiple_use_load(ptr nocapture nonnull %resultptr, ptr noc
 define <4 x float> @load_v2f32_extract_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_v2f32_extract_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %l = load <2 x float>, ptr %p, align 4
@@ -563,7 +563,7 @@ define <4 x float> @load_v2f32_extract_insert_v4f32(ptr align 16 dereferenceable
 define <4 x float> @load_v8f32_extract_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @load_v8f32_extract_insert_v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %l = load <8 x float>, ptr %p, align 4
@@ -599,7 +599,7 @@ define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceab
 ;
 ; AVX2-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
 ; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 4
-; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    ret <8 x i16> [[R]]
 ;
   %gep = getelementptr inbounds <2 x i16>, ptr %p, i64 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll
index d51ac6a33911d..685d661ea6bcd 100644
--- a/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/select-shuffle.ll
@@ -14,8 +14,8 @@ define <4 x double> @PR60649() {
 ; CHECK:       end:
 ; CHECK-NEXT:    [[T0:%.*]] = phi <4 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ zeroinitializer, [[UNREACHABLE:%.*]] ]
 ; CHECK-NEXT:    [[T1:%.*]] = phi <4 x double> [ zeroinitializer, [[ENTRY]] ], [ zeroinitializer, [[UNREACHABLE]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x double> [[T0]], <4 x double> [[T0]], <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[T0]], <4 x double> [[T0]], <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x double> [[T0]], <4 x double> [[T0]], <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[T0]], <4 x double> [[T0]], <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = fdiv <4 x double> [[TMP1]], <double 0.000000e+00, double 0.000000e+00, double undef, double undef>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x double> [[TMP0]], <double 0.000000e+00, double 0.000000e+00, double undef, double undef>
 ; CHECK-NEXT:    [[T5:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
index 5d4dc6ae5d013..61ac2b1eca32a 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
@@ -170,8 +170,8 @@ define <4 x float> @shuf_fdiv_v4f32_yy(<4 x float> %x, <4 x float> %y, <4 x floa
 
 define <4 x i32> @shuf_add_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
 ; CHECK-LABEL: @shuf_add_v4i32_xx(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 undef, i32 undef, i32 2, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]], <4 x i32> <i32 undef, i32 undef, i32 6, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]], <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
 ; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
@@ -326,7 +326,7 @@ define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(<16 x i16> %x, <16 x i16> %
 ; CHECK-LABEL: @shuf_and_v16i16_yy_expensive_shuf(
 ; CHECK-NEXT:    [[B0:%.*]] = and <16 x i16> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[B1:%.*]] = and <16 x i16> [[Y]], [[Z:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <16 x i16> [[B0]], <16 x i16> [[B1]], <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 undef, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <16 x i16> [[B0]], <16 x i16> [[B1]], <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 poison, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
 ;
   %b0 = and <16 x i16> %x, %y
diff --git a/llvm/test/tools/llvm-ar/Inputs/aix-foo.ll b/llvm/test/tools/llvm-ar/Inputs/aix-foo.ll
new file mode 100644
index 0000000000000..a22652ddd108e
--- /dev/null
+++ b/llvm/test/tools/llvm-ar/Inputs/aix-foo.ll
@@ -0,0 +1,8 @@
+target datalayout = "E-m:a-Fi64-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+target triple = "powerpc64-ibm-aix7.2.0.0"
+
+define signext i32 @foo() {
+entry:
+  ret i32 42
+}
+
diff --git a/llvm/test/tools/llvm-ar/aix-bitcode.test b/llvm/test/tools/llvm-ar/aix-bitcode.test
new file mode 100644
index 0000000000000..d3ce6f1b7343a
--- /dev/null
+++ b/llvm/test/tools/llvm-ar/aix-bitcode.test
@@ -0,0 +1,7 @@
+# Check if llvm-ar correctly archives bitcode files for AIX.
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-as %S/Inputs/aix-foo.ll -o %t/aix-foo.bc
+# RUN: llvm-ar qc %t/aix-foo.a %t/aix-foo.bc
+# RUN: FileCheck %s --input-file=%t/aix-foo.a
+
+# CHECK:  bigaf
diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/dummy-counters.test b/llvm/test/tools/llvm-exegesis/X86/latency/dummy-counters.test
index 471b9f89578a4..c74c12c94793f 100644
--- a/llvm/test/tools/llvm-exegesis/X86/latency/dummy-counters.test
+++ b/llvm/test/tools/llvm-exegesis/X86/latency/dummy-counters.test
@@ -1,6 +1,5 @@
 # RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mode=latency -opcode-name=LEA64r --use-dummy-perf-counters | FileCheck %s
-# TODO Support non-Linux systems
-# REQUIRES: exegesis-can-execute-x86_64, system-linux
+# REQUIRES: exegesis-can-execute-x86_64
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
diff --git a/llvm/test/tools/llvm-gsymutil/cmdline.test b/llvm/test/tools/llvm-gsymutil/cmdline.test
index 9191309f5ca56..ea5f76fac1635 100644
--- a/llvm/test/tools/llvm-gsymutil/cmdline.test
+++ b/llvm/test/tools/llvm-gsymutil/cmdline.test
@@ -3,21 +3,17 @@ RUN: llvm-gsymutil --help 2>&1 | FileCheck --check-prefix=HELP %s
 HELP: OVERVIEW: A tool for dumping, searching and creating GSYM files.
 HELP: USAGE: llvm-gsymutil{{[^ ]*}} [options] <input GSYM files>
 HELP: OPTIONS:
-HELP: Conversion Options:
-HELP: --arch=<arch>
-HELP: --convert=<path>
-HELP: --num-threads=<n>
-HELP: --out-file=<path>
+HELP: --address=<value>
+HELP: --addresses-from-stdin
+HELP: --arch=<value>
+HELP: --convert=<value>
+HELP: --help
+HELP: --num-threads=<value>
+HELP: --out-file=<value>
 HELP: --quiet
+HELP: --verbose
 HELP: --verify
-HELP: Generic Options:
-HELP: --help
 HELP: --version
-HELP: Lookup Options:
-HELP: --address=<addr>
-HELP: --addresses-from-stdin
-HELP: Options:
-HELP: --verbose
 
 RUN: llvm-gsymutil --version 2>&1 | FileCheck --check-prefix=VERSION %s
 VERSION: {{ version }}
diff --git a/llvm/test/tools/llvm-lib/ecsymbols.test b/llvm/test/tools/llvm-lib/ecsymbols.test
index cea065d78aa25..81f8988c89adf 100644
--- a/llvm/test/tools/llvm-lib/ecsymbols.test
+++ b/llvm/test/tools/llvm-lib/ecsymbols.test
@@ -1,5 +1,6 @@
 # Check that llvm-lib doesn't list the members of an archive which contains the
 # special member /<ECSYMBOLS>/.
+# UNSUPPORTED: system-zos
 # REQUIRES: aarch64-registered-target
 
 # RUN: yaml2obj %s -o %t.lib
diff --git a/llvm/test/tools/llvm-mca/X86/Generic/no-duplicate-symbols.s b/llvm/test/tools/llvm-mca/X86/Generic/no-duplicate-symbols.s
new file mode 100644
index 0000000000000..8b8144c08b857
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/Generic/no-duplicate-symbols.s
@@ -0,0 +1,8 @@
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 < %s 2>&1 | FileCheck %s
+
+# This test checks that https://github.com/llvm/llvm-project/issues/62528 is resolved.
+foo:
+  pushq   %rbp
+
+# CHECK-NOT:      <stdin>:4:1: error: symbol 'foo' is already defined
+
diff --git a/llvm/test/tools/llvm-profdata/Inputs/compat-sample.profdata b/llvm/test/tools/llvm-profdata/Inputs/compat-sample.profdata
deleted file mode 100644
index 9c954d4feaf38..0000000000000
Binary files a/llvm/test/tools/llvm-profdata/Inputs/compat-sample.profdata and /dev/null differ
diff --git a/llvm/test/tools/llvm-profdata/compact-sample.proftext b/llvm/test/tools/llvm-profdata/compact-sample.proftext
deleted file mode 100644
index 6ebb83949481f..0000000000000
--- a/llvm/test/tools/llvm-profdata/compact-sample.proftext
+++ /dev/null
@@ -1,20 +0,0 @@
-# Make sure "llvm-profdata show" works for sample profile in binary compact format
-
-# RUN: llvm-profdata show -sample %S/Inputs/compat-sample.profdata | FileCheck %s
-
-# CHECK: Function: 15822663052811949562: 17, 0, 6 sampled lines
-# CHECK-NEXT: Samples collected in the function's body {
-# CHECK: Samples collected in inlined callsites {
-# CHECK-NEXT: 1: inlined callee: 6309742469962978389: 17, 0, 1
-
-# RUN: llvm-profdata show -hot-func-list -sample %S/Inputs/compat-sample.profdata | FileCheck %s --check-prefix=HOTFUNC
-# HOTFUNC:      24753993 out of 24754010 profile counts (100.00%) are from hot functions.
-# HOTFUNC:       Total sample (%)       Max sample        Entry sample    Function name
-# HOTFUNC-NEXT:  24753993 (100.00%)     1284671           1284671         16429767378996342100
-
-# Make sure "llvm-profdata overlap" works for sample profile in binary compact format
-# RUN: llvm-profdata overlap -sample %S/Inputs/compat-sample.profdata %S/Inputs/compat-sample.profdata | FileCheck %s --check-prefix=OVERLAP
-
-# OVERLAP:      Program level:
-# OVERLAP-NEXT:   Whole program profile similarity: 100.000%
-# OVERLAP-NEXT:   Whole program sample overlap: 100.000%
diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
new file mode 100644
index 0000000000000..24f3f563e9689
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test
@@ -0,0 +1,19 @@
+// Magic
+RUN: printf '\377lprofr\201' > %t
+// Version
+RUN: printf '\0\01\0\0\0\0\0\10' >> %t
+// The rest of the header needs to be there to prevent a broken header error.
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\2' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\3' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\20' >> %t
+RUN: printf '\0\0\0\1\0\4\0\0' >> %t
+RUN: printf '\0\0\0\2\0\4\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+
+RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s
+
+CHECK: raw profile version mismatch: Profile uses raw profile format version = 281474976710664; expected version = {{[0-9]+}}
+CHECK-NEXT: PLEASE update this tool to version in the raw profile, or regenerate raw profile with expected version.
diff --git a/llvm/test/tools/llvm-profdata/roundtrip.test b/llvm/test/tools/llvm-profdata/roundtrip.test
index a458b21ac0839..7af76e0a58224 100644
--- a/llvm/test/tools/llvm-profdata/roundtrip.test
+++ b/llvm/test/tools/llvm-profdata/roundtrip.test
@@ -16,11 +16,3 @@ RUN: llvm-profdata merge --sample --binary -output=%t.4.profdata %S/Inputs/sampl
 RUN: llvm-profdata merge --sample --extbinary -output=%t.5.profdata %t.4.profdata
 RUN: llvm-profdata merge --sample --text -output=%t.4.proftext %t.5.profdata
 RUN: diff -b %t.4.proftext %S/Inputs/sample-profile.proftext
-# Trip from text --> extbinary --> md5text
-# Trip from text --> compbinary --> md5text
-# Compare the two md5 texts
-RUN: llvm-profdata merge --sample --compbinary -output=%t.6.profdata %S/Inputs/sample-profile.proftext
-RUN: llvm-profdata merge --sample --text -output=%t.6.proftext %t.6.profdata
-RUN: llvm-profdata merge --sample --extbinary -use-md5 -output=%t.7.profdata %S/Inputs/sample-profile.proftext
-RUN: llvm-profdata merge --sample --text -output=%t.7.proftext %t.7.profdata
-RUN: diff -b %t.6.proftext %t.7.proftext
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index b1d4c1303dc7d..23d3482fb89a7 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -38,7 +38,7 @@
 #
 # WRONG_DETAIL: << TLI yes SDK no : '_ZdaPv' aka operator delete[](void*)
 # WRONG_DETAIL: >> TLI no  SDK yes: '_ZdaPvj' aka operator delete[](void*, unsigned int)
-# WRONG_DETAIL-COUNT-8: << TLI yes SDK no : {{.*}}hot_cold_t
+# WRONG_DETAIL-COUNT-8: << TLI yes SDK no : {{.*}}__hot_cold_t
 # WRONG_SUMMARY: << Total TLI yes SDK no:  9{{$}}
 # WRONG_SUMMARY: >> Total TLI no  SDK yes: 1{{$}}
 # WRONG_SUMMARY: == Total TLI yes SDK yes: 234
diff --git a/llvm/tools/dsymutil/CMakeLists.txt b/llvm/tools/dsymutil/CMakeLists.txt
index b8d34d80817e5..d2f964efc581a 100644
--- a/llvm/tools/dsymutil/CMakeLists.txt
+++ b/llvm/tools/dsymutil/CMakeLists.txt
@@ -8,6 +8,7 @@ set(LLVM_LINK_COMPONENTS
   AllTargetsInfos
   AsmPrinter
   CodeGen
+  CodeGenTypes
   DWARFLinker
   DebugInfoDWARF
   MC
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
index 6c756c0d26138..62251162e7b33 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
@@ -939,28 +939,28 @@ void DwarfLinkerForBinary::AddressManager::printReloc(const ValidReloc &Reloc) {
                    uint64_t(Mapping.BinaryAddress));
 }
 
-void DwarfLinkerForBinary::AddressManager::fillDieInfo(
-    const ValidReloc &Reloc, CompileUnit::DIEInfo &Info) {
-  Info.AddrAdjust = relocate(Reloc);
+int64_t
+DwarfLinkerForBinary::AddressManager::getRelocValue(const ValidReloc &Reloc) {
+  int64_t AddrAdjust = relocate(Reloc);
   if (Reloc.Mapping->getValue().ObjectAddress)
-    Info.AddrAdjust -= uint64_t(*Reloc.Mapping->getValue().ObjectAddress);
-  Info.InDebugMap = true;
+    AddrAdjust -= uint64_t(*Reloc.Mapping->getValue().ObjectAddress);
+  return AddrAdjust;
 }
 
-bool DwarfLinkerForBinary::AddressManager::hasValidRelocationAt(
+std::optional<int64_t>
+DwarfLinkerForBinary::AddressManager::hasValidRelocationAt(
     const std::vector<ValidReloc> &AllRelocs, uint64_t StartOffset,
-    uint64_t EndOffset, CompileUnit::DIEInfo &Info) {
+    uint64_t EndOffset) {
   std::vector<ValidReloc> Relocs =
       getRelocations(AllRelocs, StartOffset, EndOffset);
 
   if (Relocs.size() == 0)
-    return false;
+    return std::nullopt;
 
   if (Linker.Options.Verbose)
     printReloc(Relocs[0]);
-  fillDieInfo(Relocs[0], Info);
 
-  return true;
+  return getRelocValue(Relocs[0]);
 }
 
 /// Get the starting and ending (exclusive) offset for the
@@ -984,14 +984,15 @@ getAttributeOffsets(const DWARFAbbreviationDeclaration *Abbrev, unsigned Idx,
   return std::make_pair(Offset, End);
 }
 
-bool DwarfLinkerForBinary::AddressManager::isLiveVariable(
-    const DWARFDie &DIE, CompileUnit::DIEInfo &MyInfo) {
+std::optional<int64_t>
+DwarfLinkerForBinary::AddressManager::getVariableRelocAdjustment(
+    const DWARFDie &DIE) {
   const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
 
   std::optional<uint32_t> LocationIdx =
       Abbrev->findAttributeIndex(dwarf::DW_AT_location);
   if (!LocationIdx)
-    return false;
+    return std::nullopt;
 
   uint64_t Offset = DIE.getOffset() + getULEB128Size(Abbrev->getCode());
   uint64_t LocationOffset, LocationEndOffset;
@@ -1000,17 +1001,18 @@ bool DwarfLinkerForBinary::AddressManager::isLiveVariable(
 
   // FIXME: Support relocations debug_addr.
   return hasValidRelocationAt(ValidDebugInfoRelocs, LocationOffset,
-                              LocationEndOffset, MyInfo);
+                              LocationEndOffset);
 }
 
-bool DwarfLinkerForBinary::AddressManager::isLiveSubprogram(
-    const DWARFDie &DIE, CompileUnit::DIEInfo &MyInfo) {
+std::optional<int64_t>
+DwarfLinkerForBinary::AddressManager::getSubprogramRelocAdjustment(
+    const DWARFDie &DIE) {
   const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
 
   std::optional<uint32_t> LowPcIdx =
       Abbrev->findAttributeIndex(dwarf::DW_AT_low_pc);
   if (!LowPcIdx)
-    return false;
+    return std::nullopt;
 
   dwarf::Form Form = Abbrev->getFormByIndex(*LowPcIdx);
 
@@ -1021,7 +1023,7 @@ bool DwarfLinkerForBinary::AddressManager::isLiveSubprogram(
     std::tie(LowPcOffset, LowPcEndOffset) =
         getAttributeOffsets(Abbrev, *LowPcIdx, Offset, *DIE.getDwarfUnit());
     return hasValidRelocationAt(ValidDebugInfoRelocs, LowPcOffset,
-                                LowPcEndOffset, MyInfo);
+                                LowPcEndOffset);
   }
   case dwarf::DW_FORM_addrx:
   case dwarf::DW_FORM_addrx1:
@@ -1034,15 +1036,14 @@ bool DwarfLinkerForBinary::AddressManager::isLiveSubprogram(
       uint64_t StartOffset = *AddrOffsetSectionBase + AddrValue->getRawUValue();
       uint64_t EndOffset =
           StartOffset + DIE.getDwarfUnit()->getAddressByteSize();
-      return hasValidRelocationAt(ValidDebugAddrRelocs, StartOffset, EndOffset,
-                                  MyInfo);
+      return hasValidRelocationAt(ValidDebugAddrRelocs, StartOffset, EndOffset);
     }
 
     Linker.reportWarning("no base offset for address table", SrcFileName);
-    return false;
+    return std::nullopt;
   }
   default:
-    return false;
+    return std::nullopt;
   }
 }
 
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.h b/llvm/tools/dsymutil/DwarfLinkerForBinary.h
index b38ff620f19f6..ec54265bdd77b 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.h
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.h
@@ -108,8 +108,8 @@ class DwarfLinkerForBinary {
     /// \returns resolved value.
     uint64_t relocate(const ValidReloc &Reloc) const;
 
-    /// Fill \p Info with address information for the specified \p Reloc.
-    void fillDieInfo(const ValidReloc &Reloc, CompileUnit::DIEInfo &Info);
+    /// \returns value for the specified \p Reloc.
+    int64_t getRelocValue(const ValidReloc &Reloc);
 
     /// Print contents of debug map entry for the specified \p Reloc.
     void printReloc(const ValidReloc &Reloc);
@@ -172,15 +172,15 @@ class DwarfLinkerForBinary {
     /// Checks that there is a relocation in the \p Relocs array against a
     /// debug map entry between \p StartOffset and \p NextOffset.
     ///
-    /// \returns true and sets Info.InDebugMap if it is the case.
-    bool hasValidRelocationAt(const std::vector<ValidReloc> &Relocs,
-                              uint64_t StartOffset, uint64_t EndOffset,
-                              CompileUnit::DIEInfo &Info);
-
-    bool isLiveVariable(const DWARFDie &DIE,
-                        CompileUnit::DIEInfo &Info) override;
-    bool isLiveSubprogram(const DWARFDie &DIE,
-                          CompileUnit::DIEInfo &Info) override;
+    /// \returns relocation value if relocation exist, otherwise std::nullopt.
+    std::optional<int64_t>
+    hasValidRelocationAt(const std::vector<ValidReloc> &Relocs,
+                         uint64_t StartOffset, uint64_t EndOffset);
+
+    std::optional<int64_t>
+    getVariableRelocAdjustment(const DWARFDie &DIE) override;
+    std::optional<int64_t>
+    getSubprogramRelocAdjustment(const DWARFDie &DIE) override;
 
     bool applyValidRelocs(MutableArrayRef<char> Data, uint64_t BaseOffset,
                           bool IsLittleEndian) override;
diff --git a/llvm/tools/llc/CMakeLists.txt b/llvm/tools/llc/CMakeLists.txt
index 17e6e80d1223d..257d5b519f040 100644
--- a/llvm/tools/llc/CMakeLists.txt
+++ b/llvm/tools/llc/CMakeLists.txt
@@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS
   Analysis
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   IRReader
   MC
diff --git a/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp b/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
index 0eb66b0492b72..109c1b27e5872 100644
--- a/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
+++ b/llvm/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
@@ -84,11 +84,11 @@ static std::string demangle(const std::string &Mangled) {
   char *Undecorated = nullptr;
 
   if (Types)
-    Undecorated = itaniumDemangle(DecoratedStr, nullptr, nullptr, nullptr);
+    Undecorated = itaniumDemangle(DecoratedStr);
 
   if (!Undecorated && strncmp(DecoratedStr, "__imp_", 6) == 0) {
     Prefix = "import thunk for ";
-    Undecorated = itaniumDemangle(DecoratedStr + 6, nullptr, nullptr, nullptr);
+    Undecorated = itaniumDemangle(DecoratedStr + 6);
   }
 
   Result = Undecorated ? Prefix + Undecorated : Mangled;
diff --git a/llvm/tools/llvm-dwarfutil/CMakeLists.txt b/llvm/tools/llvm-dwarfutil/CMakeLists.txt
index 5be7056ce0d70..40e09aff18031 100644
--- a/llvm/tools/llvm-dwarfutil/CMakeLists.txt
+++ b/llvm/tools/llvm-dwarfutil/CMakeLists.txt
@@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS
   AllTargetsCodeGens
   AllTargetsDescs
   AllTargetsInfos
+  CodeGenTypes
   DWARFLinker
   DebugInfoDWARF
   MC
diff --git a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
index 5d2352e23b634..534a56a4f7d04 100644
--- a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
+++ b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
@@ -70,8 +70,8 @@ class ObjFileAddressMap : public AddressesMap {
   // should be renamed into has valid address ranges
   bool hasValidRelocs() override { return !DWARFAddressRanges.empty(); }
 
-  bool isLiveSubprogram(const DWARFDie &DIE,
-                        CompileUnit::DIEInfo &Info) override {
+  std::optional<int64_t>
+  getSubprogramRelocAdjustment(const DWARFDie &DIE) override {
     assert((DIE.getTag() == dwarf::DW_TAG_subprogram ||
             DIE.getTag() == dwarf::DW_TAG_label) &&
            "Wrong type of input die");
@@ -80,18 +80,16 @@ class ObjFileAddressMap : public AddressesMap {
             dwarf::toAddress(DIE.find(dwarf::DW_AT_low_pc))) {
       if (!isDeadAddress(*LowPC, DIE.getDwarfUnit()->getVersion(),
                          Opts.Tombstone,
-                         DIE.getDwarfUnit()->getAddressByteSize())) {
-        Info.AddrAdjust = 0;
-        Info.InDebugMap = true;
-        return true;
-      }
+                         DIE.getDwarfUnit()->getAddressByteSize()))
+        // Relocation value for the linked binary is 0.
+        return 0;
     }
 
-    return false;
+    return std::nullopt;
   }
 
-  bool isLiveVariable(const DWARFDie &DIE,
-                      CompileUnit::DIEInfo &Info) override {
+  std::optional<int64_t>
+  getVariableRelocAdjustment(const DWARFDie &DIE) override {
     assert((DIE.getTag() == dwarf::DW_TAG_variable ||
             DIE.getTag() == dwarf::DW_TAG_constant) &&
            "Wrong type of input die");
@@ -114,11 +112,9 @@ class ObjFileAddressMap : public AddressesMap {
                                      DIE.getDwarfUnit()->getAddressByteSize()));
             });
 
-        if (HasLiveAddresses) {
-          Info.AddrAdjust = 0;
-          Info.InDebugMap = true;
-          return true;
-        }
+        if (HasLiveAddresses)
+          // Relocation value for the linked binary is 0.
+          return 0;
       }
     } else {
       // FIXME: missing DW_AT_location is OK here, but other errors should be
@@ -126,7 +122,7 @@ class ObjFileAddressMap : public AddressesMap {
       consumeError(Loc.takeError());
     }
 
-    return false;
+    return std::nullopt;
   }
 
   bool applyValidRelocs(MutableArrayRef<char>, uint64_t, bool) override {
diff --git a/llvm/tools/llvm-exegesis/CMakeLists.txt b/llvm/tools/llvm-exegesis/CMakeLists.txt
index 4ff1db9b430e7..ec418a7d51ecc 100644
--- a/llvm/tools/llvm-exegesis/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   AllTargetsDescs
   AllTargetsDisassemblers
   AllTargetsInfos
+  CodeGenTypes
   MC
   MCParser
   Support
diff --git a/llvm/tools/llvm-exegesis/lib/AArch64/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/AArch64/CMakeLists.txt
index 2dc948d507253..42f8fd30ac2bb 100644
--- a/llvm/tools/llvm-exegesis/lib/AArch64/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/AArch64/CMakeLists.txt
@@ -5,6 +5,7 @@ include_directories(
 
 set(LLVM_LINK_COMPONENTS
   AArch64
+  CodeGenTypes
   Core
   Exegesis
   MC
diff --git a/llvm/tools/llvm-exegesis/lib/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/CMakeLists.txt
index 39ad0a1425c22..92894601d39f6 100644
--- a/llvm/tools/llvm-exegesis/lib/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/CMakeLists.txt
@@ -22,6 +22,7 @@ endforeach()
 set(LLVM_LINK_COMPONENTS
   Analysis
   CodeGen
+  CodeGenTypes
   Core
   ExecutionEngine
   GlobalISel
diff --git a/llvm/tools/llvm-exegesis/lib/Mips/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/Mips/CMakeLists.txt
index 4661c21a6c861..263406c29f518 100644
--- a/llvm/tools/llvm-exegesis/lib/Mips/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/Mips/CMakeLists.txt
@@ -4,6 +4,7 @@ include_directories(
   )
 
 set(LLVM_LINK_COMPONENTS
+  CodeGenTypes
   Core
   Exegesis
   MC
diff --git a/llvm/tools/llvm-exegesis/lib/PowerPC/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/PowerPC/CMakeLists.txt
index 2766f76719654..3da105567f896 100644
--- a/llvm/tools/llvm-exegesis/lib/PowerPC/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/PowerPC/CMakeLists.txt
@@ -4,6 +4,7 @@ include_directories(
   )
 
 set(LLVM_LINK_COMPONENTS
+  CodeGenTypes
   Core
   Exegesis
   MC
diff --git a/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt
index 1d59a67722714..5f1e440516c89 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/X86/CMakeLists.txt
@@ -5,6 +5,7 @@ include_directories(
 
 set(LLVM_LINK_COMPONENTS
   CodeGen
+  CodeGenTypes
   Core
   Exegesis
   MC
diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
index 1fad3e1f47540..833ba01e55467 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
@@ -21,6 +21,7 @@
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/TargetParser/Host.h"
 
@@ -31,7 +32,7 @@
 #include <immintrin.h>
 #include <intrin.h>
 #endif
-#if defined(__x86_64__) && defined(_MSC_VER)
+#if defined(_MSC_VER) && defined(_M_X64)
 #include <float.h> // For _clearfp in ~X86SavedState().
 #endif
 
@@ -625,39 +626,35 @@ namespace {
 class X86SavedState : public ExegesisTarget::SavedState {
 public:
   X86SavedState() {
-#ifdef __x86_64__
-# if defined(_MSC_VER)
+#if defined(_MSC_VER) && defined(_M_X64)
     _fxsave64(FPState);
     Eflags = __readeflags();
-# elif defined(__GNUC__)
+#elif defined(__GNUC__) && defined(__x86_64__)
     __builtin_ia32_fxsave64(FPState);
     Eflags = __builtin_ia32_readeflags_u64();
-# endif
 #else
-    llvm_unreachable("X86 exegesis running on non-X86 target");
+    report_fatal_error("X86 exegesis running on unsupported target");
 #endif
   }
 
   ~X86SavedState() {
     // Restoring the X87 state does not flush pending exceptions, make sure
     // these exceptions are flushed now.
-#ifdef __x86_64__
-# if defined(_MSC_VER)
+#if defined(_MSC_VER) && defined(_M_X64)
     _clearfp();
     _fxrstor64(FPState);
     __writeeflags(Eflags);
-# elif defined(__GNUC__)
+#elif defined(__GNUC__) && defined(__x86_64__)
     asm volatile("fwait");
     __builtin_ia32_fxrstor64(FPState);
     __builtin_ia32_writeeflags_u64(Eflags);
-# endif
 #else
-    llvm_unreachable("X86 exegesis running on non-X86 target");
+    report_fatal_error("X86 exegesis running on unsupported target");
 #endif
   }
 
 private:
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(_M_X64)
   alignas(16) char FPState[512];
   uint64_t Eflags;
 #endif
@@ -767,7 +764,7 @@ class ExegesisX86Target : public ExegesisTarget {
       // If the kernel supports it, the hardware still may not have it.
       return X86LbrCounter::checkLbrSupport();
 #else
-    llvm_unreachable("Running X86 exegesis on non-X86 target");
+    report_fatal_error("Running X86 exegesis on unsupported target");
 #endif
 #endif
     return llvm::make_error<llvm::StringError>(
diff --git a/llvm/tools/llvm-gsymutil/CMakeLists.txt b/llvm/tools/llvm-gsymutil/CMakeLists.txt
index e499d965b704e..7f8a21a39d324 100644
--- a/llvm/tools/llvm-gsymutil/CMakeLists.txt
+++ b/llvm/tools/llvm-gsymutil/CMakeLists.txt
@@ -4,10 +4,20 @@ set(LLVM_LINK_COMPONENTS
   DebugInfoGSYM
   MC
   Object
+  Option
   Support
   TargetParser
   )
 
+set(LLVM_TARGET_DEFINITIONS Opts.td)
+tablegen(LLVM Opts.inc -gen-opt-parser-defs)
+add_public_tablegen_target(GSYMUtilOptsTableGen)
+
 add_llvm_tool(llvm-gsymutil
   llvm-gsymutil.cpp
+
+  DEPENDS
+  GSYMUtilOptsTableGen
+
+  GENERATE_DRIVER
   )
diff --git a/llvm/tools/llvm-gsymutil/Opts.td b/llvm/tools/llvm-gsymutil/Opts.td
new file mode 100644
index 0000000000000..15e4688108175
--- /dev/null
+++ b/llvm/tools/llvm-gsymutil/Opts.td
@@ -0,0 +1,37 @@
+include "llvm/Option/OptParser.td"
+
+class F<string letter, string help> : Flag<["-"], letter>, HelpText<help>;
+class FF<string name, string help> : Flag<["--"], name>, HelpText<help>;
+
+multiclass Eq<string name, string help> {
+  def NAME #_EQ : Joined<["--"], name #"=">, HelpText<help>;
+  def : Separate<["--"], name>, Alias<!cast<Joined>(NAME #_EQ)>;
+}
+
+def help : FF<"help", "Display this help">;
+def : F<"h", "Alias for --help">, Alias<help>;
+def version : FF<"version", "Display the version">;
+def : F<"v", "Alias for --version">, Alias<version>;
+def verbose : FF<"verbose", "Enable verbose logging and encoding details">;
+defm convert :
+  Eq<"convert",
+     "Convert the specified file to the GSYM format.\nSupported files include ELF and mach-o files that will have their debug info (DWARF) and symbol table converted">;
+defm arch :
+  Eq<"arch",
+     "Process debug information for the specified CPU architecture only.\nArchitectures may be specified by name or by number.\nThis option can be specified multiple times, once for each desired architecture">;
+defm out_file :
+  Eq<"out-file",
+     "Specify the path where the converted GSYM file will be saved.\nWhen not specified, a '.gsym' extension will be appended to the file name specified in the --convert option">;
+def : Separate<["-"], "o">, HelpText<"Alias for --out-file">, Alias<out_file_EQ>;
+def verify : FF<"verify", "Verify the generated GSYM file against the information in the file that was converted">;
+defm num_threads :
+  Eq<"num-threads",
+     "Specify the maximum number (n) of simultaneous threads to use when converting files to GSYM.\nDefaults to the number of cores on the current machine">;
+defm segment_size :
+  Eq<"segment-size",
+     "Specify the size in bytes of the size the final GSYM file should be segmented into. This allows GSYM files to be split across multiple files">;
+def quiet : FF<"quiet", "Do not output warnings about the debug information">;
+defm address : Eq<"address", "Lookup an address in a GSYM file">;
+def addresses_from_stdin :
+  FF<"addresses-from-stdin",
+     "Emit a section containing remark diagnostics metadata. By default, this is enabled for the following formats: yaml-strtab, bitstream">;
diff --git a/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp b/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
index faefc7db1bd1b..68d36f282dec0 100644
--- a/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
+++ b/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
@@ -14,9 +14,12 @@
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/MachOUniversal.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Option/Option.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/LLVMDriver.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrettyStackTrace.h"
@@ -52,85 +55,132 @@ using namespace object;
 /// Command line options.
 /// @{
 
-namespace {
-using namespace cl;
-
-OptionCategory GeneralOptions("Options");
-OptionCategory ConversionOptions("Conversion Options");
-OptionCategory LookupOptions("Lookup Options");
-
-static opt<bool> Help("h", desc("Alias for -help"), Hidden,
-                      cat(GeneralOptions));
-
-static opt<bool> Verbose("verbose",
-                         desc("Enable verbose logging and encoding details."),
-                         cat(GeneralOptions));
-
-static list<std::string> InputFilenames(Positional, desc("<input GSYM files>"),
-                                        cat(GeneralOptions));
-
-static opt<std::string>
-    ConvertFilename("convert", cl::init(""),
-                    cl::desc("Convert the specified file to the GSYM format.\n"
-                             "Supported files include ELF and mach-o files "
-                             "that will have their debug info (DWARF) and "
-                             "symbol table converted."),
-                    cl::value_desc("path"), cat(ConversionOptions));
-
-static list<std::string>
-    ArchFilters("arch",
-                desc("Process debug information for the specified CPU "
-                     "architecture only.\nArchitectures may be specified by "
-                     "name or by number.\nThis option can be specified "
-                     "multiple times, once for each desired architecture."),
-                cl::value_desc("arch"), cat(ConversionOptions));
-
-static opt<std::string>
-    OutputFilename("out-file", cl::init(""),
-                   cl::desc("Specify the path where the converted GSYM file "
-                            "will be saved.\nWhen not specified, a '.gsym' "
-                            "extension will be appended to the file name "
-                            "specified in the --convert option."),
-                   cl::value_desc("path"), cat(ConversionOptions));
-static alias OutputFilenameAlias("o", desc("Alias for -out-file."),
-                                 aliasopt(OutputFilename),
-                                 cat(ConversionOptions));
-
-static opt<bool> Verify("verify",
-                        desc("Verify the generated GSYM file against the "
-                             "information in the file that was converted."),
-                        cat(ConversionOptions));
-
-static opt<unsigned>
-    NumThreads("num-threads",
-               desc("Specify the maximum number (n) of simultaneous threads "
-                    "to use when converting files to GSYM.\nDefaults to the "
-                    "number of cores on the current machine."),
-               cl::value_desc("n"), cat(ConversionOptions));
-
-static opt<uint64_t>
-    SegmentSize("segment-size",
-               desc("Specify the size in bytes of the size the final GSYM file "
-                    "should be segmented into. This allows GSYM files to be "
-                    "split across multiple files."),
-               cl::value_desc("s"), cat(ConversionOptions));
-
-static opt<bool>
-    Quiet("quiet", desc("Do not output warnings about the debug information"),
-          cat(ConversionOptions));
-
-static list<uint64_t> LookupAddresses("address",
-                                      desc("Lookup an address in a GSYM file"),
-                                      cl::value_desc("addr"),
-                                      cat(LookupOptions));
-
-static opt<bool> LookupAddressesFromStdin(
-    "addresses-from-stdin",
-    desc("Lookup addresses in a GSYM file that are read from stdin\nEach input "
-         "line is expected to be of the following format: <addr> <gsym-path>"),
-    cat(LookupOptions));
-
-} // namespace
+using namespace llvm::opt;
+enum ID {
+  OPT_INVALID = 0, // This is not an option ID.
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \
+               HELPTEXT, METAVAR, VALUES)                                      \
+  OPT_##ID,
+#include "Opts.inc"
+#undef OPTION
+};
+
+#define PREFIX(NAME, VALUE)                                                    \
+  constexpr llvm::StringLiteral NAME##_init[] = VALUE;                         \
+  constexpr llvm::ArrayRef<llvm::StringLiteral> NAME(                          \
+      NAME##_init, std::size(NAME##_init) - 1);
+#include "Opts.inc"
+#undef PREFIX
+
+const opt::OptTable::Info InfoTable[] = {
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \
+               HELPTEXT, METAVAR, VALUES)                                      \
+  {                                                                            \
+      PREFIX,      NAME,      HELPTEXT,                                        \
+      METAVAR,     OPT_##ID,  opt::Option::KIND##Class,                        \
+      PARAM,       FLAGS,     OPT_##GROUP,                                     \
+      OPT_##ALIAS, ALIASARGS, VALUES},
+#include "Opts.inc"
+#undef OPTION
+};
+
+class GSYMUtilOptTable : public llvm::opt::GenericOptTable {
+public:
+  GSYMUtilOptTable() : GenericOptTable(InfoTable) {
+    setGroupedShortOptions(true);
+  }
+};
+
+static bool Verbose;
+static std::vector<std::string> InputFilenames;
+static std::string ConvertFilename;
+static std::vector<std::string> ArchFilters;
+static std::string OutputFilename;
+static bool Verify;
+static unsigned NumThreads;
+static uint64_t SegmentSize;
+static bool Quiet;
+static std::vector<uint64_t> LookupAddresses;
+static bool LookupAddressesFromStdin;
+
+static void parseArgs(int argc, char **argv) {
+  GSYMUtilOptTable Tbl;
+  llvm::StringRef ToolName = argv[0];
+  llvm::BumpPtrAllocator A;
+  llvm::StringSaver Saver{A};
+  llvm::opt::InputArgList Args =
+      Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
+        llvm::errs() << Msg << '\n';
+        std::exit(1);
+      });
+  if (Args.hasArg(OPT_help)) {
+    const char *Overview =
+        "A tool for dumping, searching and creating GSYM files.\n\n"
+        "Specify one or more GSYM paths as arguments to dump all of the "
+        "information in each GSYM file.\n"
+        "Specify a single GSYM file along with one or more --lookup options to "
+        "lookup addresses within that GSYM file.\n"
+        "Use the --convert option to specify a file with option --out-file "
+        "option to convert to GSYM format.\n";
+
+    Tbl.printHelp(llvm::outs(), "llvm-gsymutil [options] <input GSYM files>",
+                  Overview);
+    std::exit(0);
+  }
+  if (Args.hasArg(OPT_version)) {
+    llvm::outs() << ToolName << '\n';
+    cl::PrintVersionMessage();
+    std::exit(0);
+  }
+
+  Verbose = Args.hasArg(OPT_verbose);
+
+  for (const llvm::opt::Arg *A : Args.filtered(OPT_INPUT))
+    InputFilenames.emplace_back(A->getValue());
+
+  if (const llvm::opt::Arg *A = Args.getLastArg(OPT_convert_EQ))
+    ConvertFilename = A->getValue();
+
+  for (const llvm::opt::Arg *A : Args.filtered(OPT_arch_EQ))
+    ArchFilters.emplace_back(A->getValue());
+
+  if (const llvm::opt::Arg *A = Args.getLastArg(OPT_out_file_EQ))
+    OutputFilename = A->getValue();
+
+  Verify = Args.hasArg(OPT_verify);
+
+  if (const llvm::opt::Arg *A = Args.getLastArg(OPT_num_threads_EQ)) {
+    StringRef S{A->getValue()};
+    if (!llvm::to_integer(S, NumThreads, 0)) {
+      llvm::errs() << ToolName << ": for the --num-threads option: '" << S
+                   << "' value invalid for uint argument!\n";
+      std::exit(1);
+    }
+  }
+
+  if (const llvm::opt::Arg *A = Args.getLastArg(OPT_segment_size_EQ)) {
+    StringRef S{A->getValue()};
+    if (!llvm::to_integer(S, SegmentSize, 0)) {
+      llvm::errs() << ToolName << ": for the --segment-size option: '" << S
+                   << "' value invalid for uint argument!\n";
+      std::exit(1);
+    }
+  }
+
+  Quiet = Args.hasArg(OPT_quiet);
+
+  for (const llvm::opt::Arg *A : Args.filtered(OPT_address_EQ)) {
+    StringRef S{A->getValue()};
+    if (!llvm::to_integer(S, LookupAddresses.emplace_back(), 0)) {
+      llvm::errs() << ToolName << ": for the --segment-size option: '" << S
+                   << "' value invalid for uint argument!\n";
+      std::exit(1);
+    }
+  }
+
+  LookupAddressesFromStdin = Args.hasArg(OPT_addresses_from_stdin);
+}
+
 /// @}
 //===----------------------------------------------------------------------===//
 
@@ -443,7 +493,7 @@ static void doLookup(GsymReader &Gsym, uint64_t Addr, raw_ostream &OS) {
     OS << "\n";
 }
 
-int main(int argc, char const *argv[]) {
+int llvm_gsymutil_main(int argc, char **argv, const llvm::ToolContext &) {
   // Print a stack trace if we signal out.
   sys::PrintStackTraceOnErrorSignal(argv[0]);
   PrettyStackTraceProgram X(argc, argv);
@@ -451,21 +501,7 @@ int main(int argc, char const *argv[]) {
 
   llvm::InitializeAllTargets();
 
-  const char *Overview =
-      "A tool for dumping, searching and creating GSYM files.\n\n"
-      "Specify one or more GSYM paths as arguments to dump all of the "
-      "information in each GSYM file.\n"
-      "Specify a single GSYM file along with one or more --lookup options to "
-      "lookup addresses within that GSYM file.\n"
-      "Use the --convert option to specify a file with option --out-file "
-      "option to convert to GSYM format.\n";
-  HideUnrelatedOptions({&GeneralOptions, &ConversionOptions, &LookupOptions});
-  cl::ParseCommandLineOptions(argc, argv, Overview);
-
-  if (Help) {
-    PrintHelpMessage(/*Hidden =*/false, /*Categorized =*/true);
-    return 0;
-  }
+  parseArgs(argc, argv);
 
   raw_ostream &OS = outs();
 
diff --git a/llvm/tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp b/llvm/tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp
index dcc2bd7ef97b8..6d2dc7d5774c9 100644
--- a/llvm/tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp
+++ b/llvm/tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp
@@ -14,9 +14,7 @@
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   std::string NullTerminatedString((const char *)Data, Size);
-  int status = 0;
-  if (char *demangle = llvm::itaniumDemangle(NullTerminatedString.c_str(), nullptr,
-                                         nullptr, &status))
+  if (char *demangle = llvm::itaniumDemangle(NullTerminatedString.c_str()))
     free(demangle);
 
   return 0;
diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp
index 79e9d93061a21..51921d44d748d 100644
--- a/llvm/tools/llvm-lto/llvm-lto.cpp
+++ b/llvm/tools/llvm-lto/llvm-lto.cpp
@@ -516,11 +516,10 @@ static void getThinLTOOldAndNewPrefix(std::string &OldPrefix,
 /// Given the original \p Path to an output file, replace any path
 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
 /// resulting directory if it does not yet exist.
-static std::string getThinLTOOutputFile(const std::string &Path,
-                                        const std::string &OldPrefix,
-                                        const std::string &NewPrefix) {
+static std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
+                                        StringRef NewPrefix) {
   if (OldPrefix.empty() && NewPrefix.empty())
-    return Path;
+    return std::string(Path);
   SmallString<128> NewPath(Path);
   llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
   StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
diff --git a/llvm/tools/llvm-mca/llvm-mca.cpp b/llvm/tools/llvm-mca/llvm-mca.cpp
index babe73ff640f6..ff29e39bcdbf0 100644
--- a/llvm/tools/llvm-mca/llvm-mca.cpp
+++ b/llvm/tools/llvm-mca/llvm-mca.cpp
@@ -401,11 +401,6 @@ int main(int argc, char **argv) {
   // Tell SrcMgr about this buffer, which is what the parser will pick up.
   SrcMgr.AddNewSourceBuffer(std::move(*BufferPtr), SMLoc());
 
-  MCContext Ctx(TheTriple, MAI.get(), MRI.get(), STI.get(), &SrcMgr);
-  std::unique_ptr<MCObjectFileInfo> MOFI(
-      TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));
-  Ctx.setObjectFileInfo(MOFI.get());
-
   std::unique_ptr<buffer_ostream> BOS;
 
   std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
@@ -433,7 +428,11 @@ int main(int argc, char **argv) {
   }
 
   // Parse the input and create CodeRegions that llvm-mca can analyze.
-  mca::AsmAnalysisRegionGenerator CRG(*TheTarget, SrcMgr, Ctx, *MAI, *STI,
+  MCContext ACtx(TheTriple, MAI.get(), MRI.get(), STI.get(), &SrcMgr);
+  std::unique_ptr<MCObjectFileInfo> AMOFI(
+      TheTarget->createMCObjectFileInfo(ACtx, /*PIC=*/false));
+  ACtx.setObjectFileInfo(AMOFI.get());
+  mca::AsmAnalysisRegionGenerator CRG(*TheTarget, SrcMgr, ACtx, *MAI, *STI,
                                       *MCII);
   Expected<const mca::AnalysisRegions &> RegionsOrErr =
       CRG.parseAnalysisRegions(std::move(IPtemp));
@@ -471,7 +470,11 @@ int main(int argc, char **argv) {
 
   // Parse the input and create InstrumentRegion that llvm-mca
   // can use to improve analysis.
-  mca::AsmInstrumentRegionGenerator IRG(*TheTarget, SrcMgr, Ctx, *MAI, *STI,
+  MCContext ICtx(TheTriple, MAI.get(), MRI.get(), STI.get(), &SrcMgr);
+  std::unique_ptr<MCObjectFileInfo> IMOFI(
+      TheTarget->createMCObjectFileInfo(ICtx, /*PIC=*/false));
+  ICtx.setObjectFileInfo(IMOFI.get());
+  mca::AsmInstrumentRegionGenerator IRG(*TheTarget, SrcMgr, ICtx, *MAI, *STI,
                                         *MCII, *IM);
   Expected<const mca::InstrumentRegions &> InstrumentRegionsOrErr =
       IRG.parseInstrumentRegions(std::move(IPtemp));
@@ -547,7 +550,7 @@ int main(int argc, char **argv) {
   unsigned RegionIdx = 0;
 
   std::unique_ptr<MCCodeEmitter> MCE(
-      TheTarget->createMCCodeEmitter(*MCII, Ctx));
+      TheTarget->createMCCodeEmitter(*MCII, ACtx));
   assert(MCE && "Unable to create code emitter!");
 
   std::unique_ptr<MCAsmBackend> MAB(TheTarget->createMCAsmBackend(
diff --git a/llvm/tools/llvm-objdump/COFFDump.cpp b/llvm/tools/llvm-objdump/COFFDump.cpp
index c608460e39cf3..a7052cb3fb3cd 100644
--- a/llvm/tools/llvm-objdump/COFFDump.cpp
+++ b/llvm/tools/llvm-objdump/COFFDump.cpp
@@ -545,12 +545,18 @@ static void printExportTable(const COFFObjectFile *Obj) {
   outs() << " Ordinal base: " << OrdinalBase << "\n";
   outs() << " Ordinal      RVA  Name\n";
   for (; I != E; I = ++I) {
-    uint32_t Ordinal;
-    if (I->getOrdinal(Ordinal))
-      return;
     uint32_t RVA;
     if (I->getExportRVA(RVA))
       return;
+    StringRef Name;
+    if (I->getSymbolName(Name))
+      continue;
+    if (!RVA && Name.empty())
+      continue;
+
+    uint32_t Ordinal;
+    if (I->getOrdinal(Ordinal))
+      return;
     bool IsForwarder;
     if (I->isForwarder(IsForwarder))
       return;
@@ -559,14 +565,11 @@ static void printExportTable(const COFFObjectFile *Obj) {
       // Export table entries can be used to re-export symbols that
       // this COFF file is imported from some DLLs. This is rare.
       // In most cases IsForwarder is false.
-      outs() << format("    % 4d         ", Ordinal);
+      outs() << format("   %5d         ", Ordinal);
     } else {
-      outs() << format("    % 4d %# 8x", Ordinal, RVA);
+      outs() << format("   %5d %# 8x", Ordinal, RVA);
     }
 
-    StringRef Name;
-    if (I->getSymbolName(Name))
-      continue;
     if (!Name.empty())
       outs() << "  " << Name;
     if (IsForwarder) {
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index 60694c768a9d9..5253943e333bf 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -7280,9 +7280,7 @@ static const char *SymbolizerSymbolLookUp(void *DisInfo,
     } else if (SymbolName != nullptr && strncmp(SymbolName, "__Z", 3) == 0) {
       if (info->demangled_name != nullptr)
         free(info->demangled_name);
-      int status;
-      info->demangled_name =
-          itaniumDemangle(SymbolName + 1, nullptr, nullptr, &status);
+      info->demangled_name = itaniumDemangle(SymbolName + 1);
       if (info->demangled_name != nullptr) {
         *ReferenceName = info->demangled_name;
         *ReferenceType = LLVMDisassembler_ReferenceType_DeMangled_Name;
@@ -7380,9 +7378,7 @@ static const char *SymbolizerSymbolLookUp(void *DisInfo,
   } else if (SymbolName != nullptr && strncmp(SymbolName, "__Z", 3) == 0) {
     if (info->demangled_name != nullptr)
       free(info->demangled_name);
-    int status;
-    info->demangled_name =
-        itaniumDemangle(SymbolName + 1, nullptr, nullptr, &status);
+    info->demangled_name = itaniumDemangle(SymbolName + 1);
     if (info->demangled_name != nullptr) {
       *ReferenceName = info->demangled_name;
       *ReferenceType = LLVMDisassembler_ReferenceType_DeMangled_Name;
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 5abcdcc788441..8e372cb98b164 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -914,6 +914,9 @@ addMissingWasmCodeSymbols(const WasmObjectFile &Obj,
 static void addPltEntries(const ObjectFile &Obj,
                           std::map<SectionRef, SectionSymbolsTy> &AllSymbols,
                           StringSaver &Saver) {
+  auto *ElfObj = dyn_cast<ELFObjectFileBase>(&Obj);
+  if (!ElfObj)
+    return;
   std::optional<SectionRef> Plt;
   for (const SectionRef &Section : Obj.sections()) {
     Expected<StringRef> SecNameOrErr = Section.getName();
@@ -926,26 +929,24 @@ static void addPltEntries(const ObjectFile &Obj,
   }
   if (!Plt)
     return;
-  if (auto *ElfObj = dyn_cast<ELFObjectFileBase>(&Obj)) {
-    for (auto PltEntry : ElfObj->getPltAddresses()) {
-      if (PltEntry.first) {
-        SymbolRef Symbol(*PltEntry.first, ElfObj);
-        uint8_t SymbolType = getElfSymbolType(Obj, Symbol);
-        if (Expected<StringRef> NameOrErr = Symbol.getName()) {
-          if (!NameOrErr->empty())
-            AllSymbols[*Plt].emplace_back(
-                PltEntry.second, Saver.save((*NameOrErr + "@plt").str()),
-                SymbolType);
-          continue;
-        } else {
-          // The warning has been reported in disassembleObject().
-          consumeError(NameOrErr.takeError());
-        }
+  for (auto PltEntry : ElfObj->getPltAddresses()) {
+    if (PltEntry.first) {
+      SymbolRef Symbol(*PltEntry.first, ElfObj);
+      uint8_t SymbolType = getElfSymbolType(Obj, Symbol);
+      if (Expected<StringRef> NameOrErr = Symbol.getName()) {
+        if (!NameOrErr->empty())
+          AllSymbols[*Plt].emplace_back(
+              PltEntry.second, Saver.save((*NameOrErr + "@plt").str()),
+              SymbolType);
+        continue;
+      } else {
+        // The warning has been reported in disassembleObject().
+        consumeError(NameOrErr.takeError());
       }
-      reportWarning("PLT entry at 0x" + Twine::utohexstr(PltEntry.second) +
-                        " references an invalid symbol",
-                    Obj.getFileName());
     }
+    reportWarning("PLT entry at 0x" + Twine::utohexstr(PltEntry.second) +
+                      " references an invalid symbol",
+                  Obj.getFileName());
   }
 }
 
diff --git a/llvm/tools/llvm-opt-report/OptReport.cpp b/llvm/tools/llvm-opt-report/OptReport.cpp
index cc7e60ba13783..aa7a33d0af839 100644
--- a/llvm/tools/llvm-opt-report/OptReport.cpp
+++ b/llvm/tools/llvm-opt-report/OptReport.cpp
@@ -338,16 +338,11 @@ static bool writeReport(LocationInfoTy &LocationInfo) {
 
             bool Printed = false;
             if (!NoDemangle) {
-              int Status = 0;
-              char *Demangled =
-                itaniumDemangle(FuncName.c_str(), nullptr, nullptr, &Status);
-              if (Demangled && Status == 0) {
+              if (char *Demangled = itaniumDemangle(FuncName.c_str())) {
                 OS << Demangled;
                 Printed = true;
-              }
-
-              if (Demangled)
                 std::free(Demangled);
+              }
             }
 
             if (!Printed)
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 3db5479b8e592..a6f7b5256a2d7 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -53,7 +53,7 @@ const std::string DuplicateNameStr = "----";
 enum ProfileFormat {
   PF_None = 0,
   PF_Text,
-  PF_Compact_Binary,
+  PF_Compact_Binary, // Deprecated
   PF_Ext_Binary,
   PF_GCC,
   PF_Binary
@@ -233,9 +233,10 @@ static void overlapInput(const std::string &BaseFilename,
   auto ReaderOrErr = InstrProfReader::create(TestFilename, *FS);
   if (Error E = ReaderOrErr.takeError()) {
     // Skip the empty profiles by returning sliently.
-    instrprof_error IPE = InstrProfError::take(std::move(E));
-    if (IPE != instrprof_error::empty_raw_profile)
-      WC->Errors.emplace_back(make_error<InstrProfError>(IPE), TestFilename);
+    auto [ErrorCode, Msg] = InstrProfError::take(std::move(E));
+    if (ErrorCode != instrprof_error::empty_raw_profile)
+      WC->Errors.emplace_back(make_error<InstrProfError>(ErrorCode, Msg),
+                              TestFilename);
     return;
   }
 
@@ -280,8 +281,9 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
     }
 
     auto MemProfError = [&](Error E) {
-      instrprof_error IPE = InstrProfError::take(std::move(E));
-      WC->Errors.emplace_back(make_error<InstrProfError>(IPE), Filename);
+      auto [ErrorCode, Msg] = InstrProfError::take(std::move(E));
+      WC->Errors.emplace_back(make_error<InstrProfError>(ErrorCode, Msg),
+                              Filename);
     };
 
     // Add the frame mappings into the writer context.
@@ -306,9 +308,10 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
   auto ReaderOrErr = InstrProfReader::create(Input.Filename, *FS, Correlator);
   if (Error E = ReaderOrErr.takeError()) {
     // Skip the empty profiles by returning silently.
-    instrprof_error IPE = InstrProfError::take(std::move(E));
-    if (IPE != instrprof_error::empty_raw_profile)
-      WC->Errors.emplace_back(make_error<InstrProfError>(IPE), Filename);
+    auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
+    if (ErrCode != instrprof_error::empty_raw_profile)
+      WC->Errors.emplace_back(make_error<InstrProfError>(ErrCode, Msg),
+                              Filename);
     return;
   }
 
@@ -335,11 +338,11 @@ static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper,
       }
       Reported = true;
       // Only show hint the first time an error occurs.
-      instrprof_error IPE = InstrProfError::take(std::move(E));
+      auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
       std::unique_lock<std::mutex> ErrGuard{WC->ErrLock};
-      bool firstTime = WC->WriterErrorCodes.insert(IPE).second;
-      handleMergeWriterError(make_error<InstrProfError>(IPE), Input.Filename,
-                             FuncName, firstTime);
+      bool firstTime = WC->WriterErrorCodes.insert(ErrCode).second;
+      handleMergeWriterError(make_error<InstrProfError>(ErrCode, Msg),
+                             Input.Filename, FuncName, firstTime);
     });
   }
 
@@ -370,11 +373,11 @@ static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) {
     exitWithError(std::move(E));
 
   Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) {
-    instrprof_error IPE = InstrProfError::take(std::move(E));
+    auto [ErrorCode, Msg] = InstrProfError::take(std::move(E));
     std::unique_lock<std::mutex> ErrGuard{Dst->ErrLock};
-    bool firstTime = Dst->WriterErrorCodes.insert(IPE).second;
+    bool firstTime = Dst->WriterErrorCodes.insert(ErrorCode).second;
     if (firstTime)
-      warn(toString(make_error<InstrProfError>(IPE)));
+      warn(toString(make_error<InstrProfError>(ErrorCode, Msg)));
   });
 }
 
@@ -406,8 +409,10 @@ mergeInstrProfile(const WeightedFileVector &Inputs, StringRef DebugInfoFilename,
                   uint64_t MaxTraceLength, bool OutputSparse,
                   unsigned NumThreads, FailureMode FailMode,
                   const StringRef ProfiledBinary) {
-  if (OutputFormat != PF_Binary && OutputFormat != PF_Compact_Binary &&
-      OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text)
+  if (OutputFormat == PF_Compact_Binary)
+    exitWithError("Compact Binary is deprecated");
+  if (OutputFormat != PF_Binary && OutputFormat != PF_Ext_Binary &&
+      OutputFormat != PF_Text)
     exitWithError("unknown format is specified");
 
   std::unique_ptr<InstrProfCorrelator> Correlator;
@@ -909,7 +914,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
 static sampleprof::SampleProfileFormat FormatMap[] = {
     sampleprof::SPF_None,
     sampleprof::SPF_Text,
-    sampleprof::SPF_Compact_Binary,
+    sampleprof::SPF_None,
     sampleprof::SPF_Ext_Binary,
     sampleprof::SPF_GCC,
     sampleprof::SPF_Binary};
@@ -1177,12 +1182,11 @@ static int merge_main(int argc, const char *argv[]) {
       cl::values(clEnumVal(instr, "Instrumentation profile (default)"),
                  clEnumVal(sample, "Sample profile")));
   cl::opt<ProfileFormat> OutputFormat(
-      cl::desc("Format of output profile"), cl::init(PF_Binary),
+      cl::desc("Format of output profile"), cl::init(PF_Ext_Binary),
       cl::values(
-          clEnumValN(PF_Binary, "binary", "Binary encoding (default)"),
-          clEnumValN(PF_Compact_Binary, "compbinary",
-                     "Compact binary encoding"),
-          clEnumValN(PF_Ext_Binary, "extbinary", "Extensible binary encoding"),
+          clEnumValN(PF_Binary, "binary", "Binary encoding"),
+          clEnumValN(PF_Ext_Binary, "extbinary", "Extensible binary encoding "
+                     "(default)"),
           clEnumValN(PF_Text, "text", "Text encoding"),
           clEnumValN(PF_GCC, "gcc",
                      "GCC encoding (only meaningful for -sample)")));
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index 2728f80da64bc..8e4b912e1b8b1 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -27,7 +27,6 @@ static cl::opt<SampleProfileFormat> OutputFormat(
     "format", cl::desc("Format of output profile"), cl::init(SPF_Ext_Binary),
     cl::values(
         clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
-        clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"),
         clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"),
         clEnumValN(SPF_Text, "text", "Text encoding"),
         clEnumValN(SPF_GCC, "gcc",
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 43d9a0f576aca..0a32085dfa649 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -5514,10 +5514,16 @@ static AMDGPUNote getAMDGPUNote(uint32_t NoteType, ArrayRef<uint8_t> Desc) {
     if (!MsgPackDoc.readFromBlob(MsgPackString, /*Multi=*/false))
       return {"", ""};
 
-    AMDGPU::HSAMD::V3::MetadataVerifier Verifier(true);
     std::string MetadataString;
-    if (!Verifier.verify(MsgPackDoc.getRoot()))
-      MetadataString = "Invalid AMDGPU Metadata\n";
+
+    // FIXME: Metadata Verifier only works with AMDHSA.
+    //  This is an ugly workaround to avoid the verifier for other MD
+    //  formats (e.g. amdpal)
+    if (MsgPackString.find("amdhsa.") != StringRef::npos) {
+      AMDGPU::HSAMD::V3::MetadataVerifier Verifier(true);
+      if (!Verifier.verify(MsgPackDoc.getRoot()))
+        MetadataString = "Invalid AMDGPU Metadata\n";
+    }
 
     raw_string_ostream StrOS(MetadataString);
     if (MsgPackDoc.getRoot().isScalar()) {
diff --git a/llvm/tools/llvm-reduce/CMakeLists.txt b/llvm/tools/llvm-reduce/CMakeLists.txt
index 810fa55df9c5d..43753768d89cb 100644
--- a/llvm/tools/llvm-reduce/CMakeLists.txt
+++ b/llvm/tools/llvm-reduce/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
   BitReader
   BitWriter
   CodeGen
+  CodeGenTypes
   Core
   IPO
   IRReader
diff --git a/llvm/unittests/ADT/FloatingPointMode.cpp b/llvm/unittests/ADT/FloatingPointMode.cpp
index 5b819f3495afc..5eff9de92ee84 100644
--- a/llvm/unittests/ADT/FloatingPointMode.cpp
+++ b/llvm/unittests/ADT/FloatingPointMode.cpp
@@ -20,6 +20,8 @@ TEST(FloatingPointModeTest, ParseDenormalFPAttributeComponent) {
             parseDenormalFPAttributeComponent("preserve-sign"));
   EXPECT_EQ(DenormalMode::PositiveZero,
             parseDenormalFPAttributeComponent("positive-zero"));
+  EXPECT_EQ(DenormalMode::Dynamic,
+            parseDenormalFPAttributeComponent("dynamic"));
   EXPECT_EQ(DenormalMode::Invalid, parseDenormalFPAttributeComponent("foo"));
 }
 
@@ -27,6 +29,7 @@ TEST(FloatingPointModeTest, DenormalAttributeName) {
   EXPECT_EQ("ieee", denormalModeKindName(DenormalMode::IEEE));
   EXPECT_EQ("preserve-sign", denormalModeKindName(DenormalMode::PreserveSign));
   EXPECT_EQ("positive-zero", denormalModeKindName(DenormalMode::PositiveZero));
+  EXPECT_EQ("dynamic", denormalModeKindName(DenormalMode::Dynamic));
   EXPECT_EQ("", denormalModeKindName(DenormalMode::Invalid));
 }
 
@@ -54,6 +57,10 @@ TEST(FloatingPointModeTest, ParseDenormalFPAttribute) {
   EXPECT_EQ(DenormalMode(DenormalMode::PositiveZero, DenormalMode::PositiveZero),
             parseDenormalFPAttribute("positive-zero,positive-zero"));
 
+  EXPECT_EQ(DenormalMode(DenormalMode::Dynamic, DenormalMode::Dynamic),
+            parseDenormalFPAttribute("dynamic"));
+  EXPECT_EQ(DenormalMode(DenormalMode::Dynamic, DenormalMode::Dynamic),
+            parseDenormalFPAttribute("dynamic,dynamic"));
 
   EXPECT_EQ(DenormalMode(DenormalMode::IEEE, DenormalMode::PositiveZero),
             parseDenormalFPAttribute("ieee,positive-zero"));
@@ -65,6 +72,10 @@ TEST(FloatingPointModeTest, ParseDenormalFPAttribute) {
   EXPECT_EQ(DenormalMode(DenormalMode::IEEE, DenormalMode::PreserveSign),
             parseDenormalFPAttribute("ieee,preserve-sign"));
 
+  EXPECT_EQ(DenormalMode(DenormalMode::Dynamic, DenormalMode::PreserveSign),
+            parseDenormalFPAttribute("dynamic,preserve-sign"));
+  EXPECT_EQ(DenormalMode(DenormalMode::PreserveSign, DenormalMode::Dynamic),
+            parseDenormalFPAttribute("preserve-sign,dynamic"));
 
   EXPECT_EQ(DenormalMode(DenormalMode::Invalid, DenormalMode::Invalid),
             parseDenormalFPAttribute("foo"));
@@ -102,6 +113,13 @@ TEST(FloatingPointModeTest, RenderDenormalFPAttribute) {
   EXPECT_EQ(
     "preserve-sign,positive-zero",
     DenormalMode(DenormalMode::PreserveSign, DenormalMode::PositiveZero).str());
+
+  EXPECT_EQ("dynamic,dynamic",
+            DenormalMode(DenormalMode::Dynamic, DenormalMode::Dynamic).str());
+  EXPECT_EQ("ieee,dynamic",
+            DenormalMode(DenormalMode::IEEE, DenormalMode::Dynamic).str());
+  EXPECT_EQ("dynamic,ieee",
+            DenormalMode(DenormalMode::Dynamic, DenormalMode::IEEE).str());
 }
 
 TEST(FloatingPointModeTest, DenormalModeIsSimple) {
@@ -110,6 +128,10 @@ TEST(FloatingPointModeTest, DenormalModeIsSimple) {
                             DenormalMode::Invalid).isSimple());
   EXPECT_FALSE(DenormalMode(DenormalMode::PreserveSign,
                             DenormalMode::PositiveZero).isSimple());
+  EXPECT_FALSE(DenormalMode(DenormalMode::PreserveSign, DenormalMode::Dynamic)
+                   .isSimple());
+  EXPECT_FALSE(DenormalMode(DenormalMode::Dynamic, DenormalMode::PreserveSign)
+                   .isSimple());
 }
 
 TEST(FloatingPointModeTest, DenormalModeIsValid) {
@@ -125,10 +147,76 @@ TEST(FloatingPointModeTest, DenormalModeConstructor) {
             DenormalMode::getInvalid());
   EXPECT_EQ(DenormalMode(DenormalMode::IEEE, DenormalMode::IEEE),
             DenormalMode::getIEEE());
+  EXPECT_EQ(DenormalMode::getIEEE(), DenormalMode::getDefault());
+  EXPECT_EQ(DenormalMode(DenormalMode::Dynamic, DenormalMode::Dynamic),
+            DenormalMode::getDynamic());
   EXPECT_EQ(DenormalMode(DenormalMode::PreserveSign, DenormalMode::PreserveSign),
             DenormalMode::getPreserveSign());
   EXPECT_EQ(DenormalMode(DenormalMode::PositiveZero, DenormalMode::PositiveZero),
             DenormalMode::getPositiveZero());
 }
 
+TEST(FloatingPointModeTest, DenormalModeMerge) {
+  EXPECT_EQ(
+      DenormalMode::getInvalid(),
+      DenormalMode::getInvalid().mergeCalleeMode(DenormalMode::getInvalid()));
+  EXPECT_EQ(DenormalMode::getIEEE(), DenormalMode::getInvalid().mergeCalleeMode(
+                                         DenormalMode::getIEEE()));
+  EXPECT_EQ(DenormalMode::getInvalid(), DenormalMode::getIEEE().mergeCalleeMode(
+                                            DenormalMode::getInvalid()));
+
+  EXPECT_EQ(DenormalMode::getIEEE(), DenormalMode::getIEEE().mergeCalleeMode(
+                                         DenormalMode::getDynamic()));
+  EXPECT_EQ(DenormalMode::getPreserveSign(),
+            DenormalMode::getPreserveSign().mergeCalleeMode(
+                DenormalMode::getDynamic()));
+  EXPECT_EQ(DenormalMode::getPositiveZero(),
+            DenormalMode::getPositiveZero().mergeCalleeMode(
+                DenormalMode::getDynamic()));
+  EXPECT_EQ(
+      DenormalMode::getDynamic(),
+      DenormalMode::getDynamic().mergeCalleeMode(DenormalMode::getDynamic()));
+
+  EXPECT_EQ(DenormalMode(DenormalMode::IEEE, DenormalMode::PreserveSign),
+            DenormalMode(DenormalMode::IEEE, DenormalMode::PreserveSign)
+                .mergeCalleeMode(
+                    DenormalMode(DenormalMode::IEEE, DenormalMode::Dynamic)));
+
+  EXPECT_EQ(DenormalMode(DenormalMode::PreserveSign, DenormalMode::IEEE),
+            DenormalMode(DenormalMode::PreserveSign, DenormalMode::IEEE)
+                .mergeCalleeMode(
+                    DenormalMode(DenormalMode::Dynamic, DenormalMode::IEEE)));
+
+  EXPECT_EQ(
+      DenormalMode(DenormalMode::PositiveZero, DenormalMode::PreserveSign),
+      DenormalMode(DenormalMode::PositiveZero, DenormalMode::PreserveSign)
+          .mergeCalleeMode(
+              DenormalMode(DenormalMode::Dynamic, DenormalMode::Dynamic)));
+
+  EXPECT_EQ(
+      DenormalMode(DenormalMode::PositiveZero, DenormalMode::PreserveSign),
+      DenormalMode(DenormalMode::PositiveZero, DenormalMode::PreserveSign)
+          .mergeCalleeMode(
+              DenormalMode(DenormalMode::PositiveZero, DenormalMode::Dynamic)));
+
+  EXPECT_EQ(
+      DenormalMode(DenormalMode::PositiveZero, DenormalMode::PreserveSign),
+      DenormalMode(DenormalMode::PositiveZero, DenormalMode::PreserveSign)
+          .mergeCalleeMode(
+              DenormalMode(DenormalMode::Dynamic, DenormalMode::PreserveSign)));
+
+  // Test some invalid / undefined behavior cases
+  EXPECT_EQ(
+      DenormalMode::getPreserveSign(),
+      DenormalMode::getIEEE().mergeCalleeMode(DenormalMode::getPreserveSign()));
+  EXPECT_EQ(
+      DenormalMode::getPreserveSign(),
+      DenormalMode::getIEEE().mergeCalleeMode(DenormalMode::getPreserveSign()));
+  EXPECT_EQ(
+      DenormalMode::getIEEE(),
+      DenormalMode::getPreserveSign().mergeCalleeMode(DenormalMode::getIEEE()));
+  EXPECT_EQ(
+      DenormalMode::getIEEE(),
+      DenormalMode::getPreserveSign().mergeCalleeMode(DenormalMode::getIEEE()));
+}
 }
diff --git a/llvm/unittests/ADT/IListTest.cpp b/llvm/unittests/ADT/IListTest.cpp
index 18f6c41a64807..2fdc8e12d0fa8 100644
--- a/llvm/unittests/ADT/IListTest.cpp
+++ b/llvm/unittests/ADT/IListTest.cpp
@@ -158,42 +158,6 @@ TEST(IListTest, UnsafeClear) {
   EXPECT_EQ(6, List.back().Value);
 }
 
-struct Empty {};
-TEST(IListTest, HasObsoleteCustomizationTrait) {
-  // Negative test for HasObsoleteCustomization.
-  static_assert(!ilist_detail::HasObsoleteCustomization<Empty, Node>::value,
-                "Empty has no customizations");
-}
-
-struct GetNext {
-  Node *getNext(Node *);
-};
-TEST(IListTest, HasGetNextTrait) {
-  static_assert(ilist_detail::HasGetNext<GetNext, Node>::value,
-                "GetNext has a getNext(Node*)");
-  static_assert(ilist_detail::HasObsoleteCustomization<GetNext, Node>::value,
-                "Empty should be obsolete because of getNext()");
-
-  // Negative test for HasGetNext.
-  static_assert(!ilist_detail::HasGetNext<Empty, Node>::value,
-                "Empty does not have a getNext(Node*)");
-}
-
-struct CreateSentinel {
-  Node *createSentinel();
-};
-TEST(IListTest, HasCreateSentinelTrait) {
-  static_assert(ilist_detail::HasCreateSentinel<CreateSentinel>::value,
-                "CreateSentinel has a getNext(Node*)");
-  static_assert(
-      ilist_detail::HasObsoleteCustomization<CreateSentinel, Node>::value,
-      "Empty should be obsolete because of createSentinel()");
-
-  // Negative test for HasCreateSentinel.
-  static_assert(!ilist_detail::HasCreateSentinel<Empty>::value,
-                "Empty does not have a createSentinel()");
-}
-
 struct NodeWithCallback : ilist_node<NodeWithCallback> {
   int Value = 0;
   bool IsInList = false;
diff --git a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
index ff07666ef8325..8d22cca3869de 100644
--- a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
+++ b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
@@ -20,8 +20,8 @@
 using namespace llvm;
 using namespace llvm::memprof;
 
-extern cl::opt<float> MemProfAccessesPerByteColdThreshold;
-extern cl::opt<unsigned> MemProfMinLifetimeColdThreshold;
+extern cl::opt<float> MemProfLifetimeAccessDensityColdThreshold;
+extern cl::opt<unsigned> MemProfAveLifetimeColdThreshold;
 
 namespace {
 
@@ -60,30 +60,36 @@ class MemoryProfileInfoTest : public testing::Test {
 // Basic checks on the allocation type for values just above and below
 // the thresholds.
 TEST_F(MemoryProfileInfoTest, GetAllocType) {
-  // Long lived with more accesses per byte than threshold is not cold.
-  EXPECT_EQ(
-      getAllocType(/*MaxAccessCount=*/MemProfAccessesPerByteColdThreshold + 1,
-                   /*MinSize=*/1,
-                   /*MinLifetime=*/MemProfMinLifetimeColdThreshold * 1000 + 1),
-      AllocationType::NotCold);
-  // Long lived with less accesses per byte than threshold is cold.
-  EXPECT_EQ(
-      getAllocType(/*MaxAccessCount=*/MemProfAccessesPerByteColdThreshold - 1,
-                   /*MinSize=*/1,
-                   /*MinLifetime=*/MemProfMinLifetimeColdThreshold * 1000 + 1),
-      AllocationType::Cold);
-  // Short lived with more accesses per byte than threshold is not cold.
-  EXPECT_EQ(
-      getAllocType(/*MaxAccessCount=*/MemProfAccessesPerByteColdThreshold + 1,
-                   /*MinSize=*/1,
-                   /*MinLifetime=*/MemProfMinLifetimeColdThreshold * 1000 - 1),
-      AllocationType::NotCold);
-  // Short lived with less accesses per byte than threshold is not cold.
-  EXPECT_EQ(
-      getAllocType(/*MaxAccessCount=*/MemProfAccessesPerByteColdThreshold - 1,
-                   /*MinSize=*/1,
-                   /*MinLifetime=*/MemProfMinLifetimeColdThreshold * 1000 - 1),
-      AllocationType::NotCold);
+  const uint64_t AllocCount = 2;
+  // To be cold we require that
+  // ((float)TotalLifetimeAccessDensity) / AllocCount / 100 <
+  //    MemProfLifetimeAccessDensityColdThreshold
+  // so compute the TotalLifetimeAccessDensity right at the threshold.
+  const uint64_t TotalLifetimeAccessDensityThreshold =
+      (uint64_t)(MemProfLifetimeAccessDensityColdThreshold * AllocCount * 100);
+  // To be cold we require that
+  // ((float)TotalLifetime) / AllocCount >=
+  //    MemProfAveLifetimeColdThreshold * 1000
+  // so compute the TotalLifetime right at the threshold.
+  const uint64_t TotalLifetimeThreshold =
+      MemProfAveLifetimeColdThreshold * AllocCount * 1000;
+
+  // Long lived with more accesses per byte per sec than threshold is not cold.
+  EXPECT_EQ(getAllocType(TotalLifetimeAccessDensityThreshold + 1, AllocCount,
+                         TotalLifetimeThreshold + 1),
+            AllocationType::NotCold);
+  // Long lived with less accesses per byte per sec than threshold is cold.
+  EXPECT_EQ(getAllocType(TotalLifetimeAccessDensityThreshold - 1, AllocCount,
+                         TotalLifetimeThreshold + 1),
+            AllocationType::Cold);
+  // Short lived with more accesses per byte per sec than threshold is not cold.
+  EXPECT_EQ(getAllocType(TotalLifetimeAccessDensityThreshold + 1, AllocCount,
+                         TotalLifetimeThreshold - 1),
+            AllocationType::NotCold);
+  // Short lived with less accesses per byte per sec than threshold is not cold.
+  EXPECT_EQ(getAllocType(TotalLifetimeAccessDensityThreshold - 1, AllocCount,
+                         TotalLifetimeThreshold - 1),
+            AllocationType::NotCold);
 }
 
 // Test the hasSingleAllocType helper.
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 5c47dca605364..b91c2617c5077 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -433,26 +433,26 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare i8* @_ZnajSt11align_val_t(i32, i32)\n"
       "declare i8* @_ZnajSt11align_val_tRKSt9nothrow_t(i32, i32, %struct*)\n"
       "declare i8* @_Znam(i64)\n"
-      "declare i8* @_Znam10hot_cold_t(i64, i8)\n"
+      "declare i8* @_Znam12__hot_cold_t(i64, i8)\n"
       "declare i8* @_ZnamRKSt9nothrow_t(i64, %struct*)\n"
-      "declare i8* @_ZnamRKSt9nothrow_t10hot_cold_t(i64, %struct*, i8)\n"
+      "declare i8* @_ZnamRKSt9nothrow_t12__hot_cold_t(i64, %struct*, i8)\n"
       "declare i8* @_ZnamSt11align_val_t(i64, i64)\n"
-      "declare i8* @_ZnamSt11align_val_t10hot_cold_t(i64, i64, i8)\n"
+      "declare i8* @_ZnamSt11align_val_t12__hot_cold_t(i64, i64, i8)\n"
       "declare i8* @_ZnamSt11align_val_tRKSt9nothrow_t(i64, i64, %struct*)\n"
-      "declare i8* @_ZnamSt11align_val_tRKSt9nothrow_t10hot_cold_t(i64, i64, "
+      "declare i8* @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, "
       "%struct*, i8)\n"
       "declare i8* @_Znwj(i32)\n"
       "declare i8* @_ZnwjRKSt9nothrow_t(i32, %struct*)\n"
       "declare i8* @_ZnwjSt11align_val_t(i32, i32)\n"
       "declare i8* @_ZnwjSt11align_val_tRKSt9nothrow_t(i32, i32, %struct*)\n"
       "declare i8* @_Znwm(i64)\n"
-      "declare i8* @_Znwm10hot_cold_t(i64, i8)\n"
+      "declare i8* @_Znwm12__hot_cold_t(i64, i8)\n"
       "declare i8* @_ZnwmRKSt9nothrow_t(i64, %struct*)\n"
-      "declare i8* @_ZnwmRKSt9nothrow_t10hot_cold_t(i64, %struct*, i8)\n"
+      "declare i8* @_ZnwmRKSt9nothrow_t12__hot_cold_t(i64, %struct*, i8)\n"
       "declare i8* @_ZnwmSt11align_val_t(i64, i64)\n"
-      "declare i8* @_ZnwmSt11align_val_t10hot_cold_t(i64, i64, i8)\n"
+      "declare i8* @_ZnwmSt11align_val_t12__hot_cold_t(i64, i64, i8)\n"
       "declare i8* @_ZnwmSt11align_val_tRKSt9nothrow_t(i64, i64, %struct*)\n"
-      "declare i8* @_ZnwmSt11align_val_tRKSt9nothrow_t10hot_cold_t(i64, i64, "
+      "declare i8* @_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, "
       "%struct*, i8)\n"
 
       "declare void @\"??3@YAXPEAX@Z\"(i8*)\n"
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index c086879fc4171..6a8f468a9c330 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -982,6 +982,20 @@ TEST_F(ValueTrackingTest, programUndefinedIfPoison) {
   EXPECT_EQ(programUndefinedIfPoison(A), true);
 }
 
+TEST_F(ValueTrackingTest, programUndefinedIfPoisonSelect) {
+  parseAssembly("declare i32 @any_num()"
+                "define void @test(i1 %Cond) {\n"
+                "  %A = call i32 @any_num()\n"
+                "  %B = add i32 %A, 1\n"
+                "  %C = select i1 %Cond, i32 %A, i32 %B\n"
+                "  udiv i32 1, %C"
+                "  ret void\n"
+                "}\n");
+  // If A is poison, B is also poison, and therefore C is poison regardless of
+  // the value of %Cond.
+  EXPECT_EQ(programUndefinedIfPoison(A), true);
+}
+
 TEST_F(ValueTrackingTest, programUndefinedIfUndefOrPoison) {
   parseAssembly("declare i32 @any_num()"
                 "define void @test(i32 %mask) {\n"
@@ -1448,7 +1462,7 @@ TEST_F(ComputeKnownFPClassTest, FabsUnknown) {
       "  %A = call float @llvm.fabs.f32(float %arg)"
       "  ret float %A\n"
       "}\n");
-  expectKnownFPClass(fcAllFlags, false);
+  expectKnownFPClass(fcPositive | fcNan, false);
 }
 
 TEST_F(ComputeKnownFPClassTest, FNegFabsUnknown) {
@@ -1459,7 +1473,7 @@ TEST_F(ComputeKnownFPClassTest, FNegFabsUnknown) {
       "  %A = fneg float %fabs"
       "  ret float %A\n"
       "}\n");
-  expectKnownFPClass(fcAllFlags, true);
+  expectKnownFPClass(fcNegative | fcNan, true);
 }
 
 TEST_F(ComputeKnownFPClassTest, NegFabsNInf) {
@@ -1470,7 +1484,7 @@ TEST_F(ComputeKnownFPClassTest, NegFabsNInf) {
       "  %A = fneg float %fabs"
       "  ret float %A\n"
       "}\n");
-  expectKnownFPClass(~fcInf, true);
+  expectKnownFPClass((fcNegative & ~fcNegInf) | fcNan, true);
 }
 
 TEST_F(ComputeKnownFPClassTest, FNegFabsNNaN) {
@@ -1481,7 +1495,7 @@ TEST_F(ComputeKnownFPClassTest, FNegFabsNNaN) {
       "  %A = fneg float %fabs"
       "  ret float %A\n"
       "}\n");
-  expectKnownFPClass(~fcNan, true);
+  expectKnownFPClass(fcNegative, true);
 }
 
 TEST_F(ComputeKnownFPClassTest, CopySignNNanSrc0) {
@@ -1505,7 +1519,7 @@ TEST_F(ComputeKnownFPClassTest, CopySignNInfSrc0_NegSign) {
       "  %A = call float @llvm.copysign.f32(float %ninf, float -1.0)"
       "  ret float %A\n"
       "}\n");
-  expectKnownFPClass(fcNegFinite, true);
+  expectKnownFPClass(fcNegFinite | fcNan, true);
 }
 
 TEST_F(ComputeKnownFPClassTest, CopySignNInfSrc0_PosSign) {
@@ -1517,7 +1531,7 @@ TEST_F(ComputeKnownFPClassTest, CopySignNInfSrc0_PosSign) {
       "  %A = call float @llvm.copysign.f32(float %ninf, float 1.0)"
       "  ret float %A\n"
       "}\n");
-  expectKnownFPClass(fcPosFinite, false);
+  expectKnownFPClass(fcPosFinite | fcNan, false);
 }
 
 TEST_F(ComputeKnownFPClassTest, UIToFP) {
@@ -1580,11 +1594,11 @@ TEST_F(ComputeKnownFPClassTest, FSub) {
 
 TEST_F(ComputeKnownFPClassTest, FMul) {
   parseAssembly(
-      "define float @test(float nofpclass(nan inf) %nnan.ninf, float nofpclass(nan) %nnan, float nofpclass(qnan) %no.qnan, float %unknown) {\n"
-      "  %A = fmul float %nnan.ninf, %nnan.ninf"
-      "  %A2 = fmul float %nnan.ninf, %nnan"
-      "  %A3 = fmul float %nnan, %nnan.ninf"
-      "  %A4 = fmul float %nnan.ninf, %no.qnan"
+      "define float @test(float nofpclass(nan inf) %nnan.ninf0, float nofpclass(nan inf) %nnan.ninf1, float nofpclass(nan) %nnan, float nofpclass(qnan) %no.qnan, float %unknown) {\n"
+      "  %A = fmul float %nnan.ninf0, %nnan.ninf1"
+      "  %A2 = fmul float %nnan.ninf0, %nnan"
+      "  %A3 = fmul float %nnan, %nnan.ninf0"
+      "  %A4 = fmul float %nnan.ninf0, %no.qnan"
       "  %A5 = fmul float %nnan, %nnan"
       "  ret float %A\n"
       "}\n");
@@ -1592,23 +1606,23 @@ TEST_F(ComputeKnownFPClassTest, FMul) {
   expectKnownFPClass(fcAllFlags, std::nullopt, A2);
   expectKnownFPClass(fcAllFlags, std::nullopt, A3);
   expectKnownFPClass(fcAllFlags, std::nullopt, A4);
-  expectKnownFPClass(fcAllFlags, std::nullopt, A5);
+  expectKnownFPClass(fcPositive | fcNan, std::nullopt, A5);
 }
 
 TEST_F(ComputeKnownFPClassTest, FMulNoZero) {
   parseAssembly(
-      "define float @test(float nofpclass(zero) %no.zero, float nofpclass(zero nan) %no.zero.nan, float nofpclass(nzero nan) %no.negzero.nan, float nofpclass(pzero nan) %no.poszero.nan, float nofpclass(inf nan) %no.inf.nan, float nofpclass(inf) %no.inf, float nofpclass(nan) %no.nan) {\n"
-      "  %A = fmul float %no.zero.nan, %no.zero.nan"
+      "define float @test(float nofpclass(zero) %no.zero, float nofpclass(zero nan) %no.zero.nan0, float nofpclass(zero nan) %no.zero.nan1, float nofpclass(nzero nan) %no.negzero.nan, float nofpclass(pzero nan) %no.poszero.nan, float nofpclass(inf nan) %no.inf.nan, float nofpclass(inf) %no.inf, float nofpclass(nan) %no.nan) {\n"
+      "  %A = fmul float %no.zero.nan0, %no.zero.nan1"
       "  %A2 = fmul float %no.zero, %no.zero"
-      "  %A3 = fmul float %no.poszero.nan, %no.zero.nan"
+      "  %A3 = fmul float %no.poszero.nan, %no.zero.nan0"
       "  %A4 = fmul float %no.nan, %no.zero"
       "  %A5 = fmul float %no.zero, %no.inf"
-      "  %A6 = fmul float %no.zero.nan, %no.nan"
-      "  %A7 = fmul float %no.nan, %no.zero.nan"
+      "  %A6 = fmul float %no.zero.nan0, %no.nan"
+      "  %A7 = fmul float %no.nan, %no.zero.nan0"
       "  ret float %A\n"
       "}\n");
   expectKnownFPClass(fcFinite | fcInf, std::nullopt, A);
-  expectKnownFPClass(fcAllFlags, std::nullopt, A2);
+  expectKnownFPClass(fcPositive | fcNan, std::nullopt, A2);
   expectKnownFPClass(fcAllFlags, std::nullopt, A3);
   expectKnownFPClass(fcAllFlags, std::nullopt, A4);
   expectKnownFPClass(fcAllFlags, std::nullopt, A5);
diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
index 1987e102fda94..63878995ef811 100644
--- a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
+++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
@@ -30,7 +30,25 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
 
   // Check that AMDGPU targets add -G1 if it's not present.
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "r600"), "e-p:32:32-G1");
-  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64", "amdgcn"), "e-p:64:64-G1");
+  // and that ANDGCN adds p7 and p8 as well.
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64", "amdgcn"),
+            "e-p:64:64-G1-ni:7:8-p7:160:256:256:32-p8:128:128");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-G1", "amdgcn"),
+            "e-p:64:64-G1-ni:7:8-p7:160:256:256:32-p8:128:128");
+  // but that r600 does not.
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G1", "r600"), "e-p:32:32-G1");
+
+  // Ensure that the non-integral direction for address space 8 doesn't get
+  // added in to pointer declarations.
+  EXPECT_EQ(
+      UpgradeDataLayoutString(
+          "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:"
+          "64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-"
+          "v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7",
+          "amdgcn"),
+      "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-"
+      "v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:"
+      "1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8-p7:160:256:256:32-p8:128:128");
 
   // Check that RISCV64 upgrades -n64 to -n32:64.
   EXPECT_EQ(UpgradeDataLayoutString("e-m:e-p:64:64-i64:64-i128:128-n64-S128",
@@ -59,9 +77,20 @@ TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
   // Check that AMDGPU targets don't add -G1 if there is already a -G flag.
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G2", "r600"), "e-p:32:32-G2");
   EXPECT_EQ(UpgradeDataLayoutString("G2", "r600"), "G2");
-  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-G2", "amdgcn"), "e-p:64:64-G2");
-  EXPECT_EQ(UpgradeDataLayoutString("G2-e-p:64:64", "amdgcn"), "G2-e-p:64:64");
-  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-G0", "amdgcn"), "e-p:64:64-G0");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-G2", "amdgcn"),
+            "e-p:64:64-G2-ni:7:8-p7:160:256:256:32-p8:128:128");
+  EXPECT_EQ(UpgradeDataLayoutString("G2-e-p:64:64", "amdgcn"),
+            "G2-e-p:64:64-ni:7:8-p7:160:256:256:32-p8:128:128");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-G0", "amdgcn"),
+            "e-p:64:64-G0-ni:7:8-p7:160:256:256:32-p8:128:128");
+
+  // Check that AMDGCN targets don't add already declared address space 7.
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-p7:64:64", "amdgcn"),
+            "e-p:64:64-p7:64:64-G1-ni:7:8-p8:128:128");
+  EXPECT_EQ(UpgradeDataLayoutString("p7:64:64-G2-e-p:64:64", "amdgcn"),
+            "p7:64:64-G2-e-p:64:64-ni:7:8-p8:128:128");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-p7:64:64-G1", "amdgcn"),
+            "e-p:64:64-p7:64:64-G1-ni:7:8-p8:128:128");
 }
 
 TEST(DataLayoutUpgradeTest, EmptyDataLayout) {
@@ -73,7 +102,8 @@ TEST(DataLayoutUpgradeTest, EmptyDataLayout) {
 
   // Check that AMDGPU targets add G1 if it's not present.
   EXPECT_EQ(UpgradeDataLayoutString("", "r600"), "G1");
-  EXPECT_EQ(UpgradeDataLayoutString("", "amdgcn"), "G1");
+  EXPECT_EQ(UpgradeDataLayoutString("", "amdgcn"),
+            "G1-ni:7:8-p7:160:256:256:32-p8:128:128");
 }
 
 } // end namespace
diff --git a/llvm/unittests/CodeGen/CMakeLists.txt b/llvm/unittests/CodeGen/CMakeLists.txt
index 3dbd570218352..a559e35f12198 100644
--- a/llvm/unittests/CodeGen/CMakeLists.txt
+++ b/llvm/unittests/CodeGen/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
   AsmParser
   AsmPrinter
   CodeGen
+  CodeGenTypes
   Core
   FileCheck
   MC
diff --git a/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt b/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt
index 0d4f4cff2e739..6ed2409f2ad75 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt
+++ b/llvm/unittests/CodeGen/GlobalISel/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   CodeGen
+  CodeGenTypes
   Core
   FileCheck
   GlobalISel
diff --git a/llvm/unittests/CodeGen/LowLevelTypeTest.cpp b/llvm/unittests/CodeGen/LowLevelTypeTest.cpp
index fc7f7bed0235d..ccf39636bf275 100644
--- a/llvm/unittests/CodeGen/LowLevelTypeTest.cpp
+++ b/llvm/unittests/CodeGen/LowLevelTypeTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/LowLevelTypeUtils.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/LLVMContext.h"
diff --git a/llvm/unittests/CodeGen/MachineOperandTest.cpp b/llvm/unittests/CodeGen/MachineOperandTest.cpp
index 165e16829bca4..31ab003cc62aa 100644
--- a/llvm/unittests/CodeGen/MachineOperandTest.cpp
+++ b/llvm/unittests/CodeGen/MachineOperandTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
@@ -21,7 +22,6 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/CodeGen/ScalableVectorMVTsTest.cpp b/llvm/unittests/CodeGen/ScalableVectorMVTsTest.cpp
index 05bf91a4b88c9..424abb03dc129 100644
--- a/llvm/unittests/CodeGen/ScalableVectorMVTsTest.cpp
+++ b/llvm/unittests/CodeGen/ScalableVectorMVTsTest.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/TypeSize.h"
 #include "gtest/gtest.h"
 
diff --git a/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt b/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt
index 21faae03c5c2b..a6c37c6a1519c 100644
--- a/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt
+++ b/llvm/unittests/DebugInfo/DWARF/CMakeLists.txt
@@ -2,6 +2,7 @@ set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   AsmPrinter
   BinaryFormat
+  CodeGenTypes
   DebugInfoDWARF
   MC
   Object
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 78861bda31210..11637f44d97b5 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -5119,6 +5119,62 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
   EXPECT_FALSE(verifyModule(*M, &errs()));
 }
 
+TEST_F(OpenMPIRBuilderTest, TargetRegion) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  OpenMPIRBuilderConfig Config(false, false, false, false);
+  OMPBuilder.setConfig(Config);
+  F->setName("func");
+  IRBuilder<> Builder(BB);
+  auto Int32Ty = Builder.getInt32Ty();
+
+  AllocaInst *APtr = Builder.CreateAlloca(Int32Ty, nullptr, "a_ptr");
+  AllocaInst *BPtr = Builder.CreateAlloca(Int32Ty, nullptr, "b_ptr");
+  AllocaInst *CPtr = Builder.CreateAlloca(Int32Ty, nullptr, "c_ptr");
+
+  Builder.CreateStore(Builder.getInt32(10), APtr);
+  Builder.CreateStore(Builder.getInt32(20), BPtr);
+  auto BodyGenCB = [&](InsertPointTy AllocaIP,
+                       InsertPointTy CodeGenIP) -> InsertPointTy {
+    Builder.restoreIP(CodeGenIP);
+    LoadInst *AVal = Builder.CreateLoad(Int32Ty, APtr);
+    LoadInst *BVal = Builder.CreateLoad(Int32Ty, BPtr);
+    Value *Sum = Builder.CreateAdd(AVal, BVal);
+    Builder.CreateStore(Sum, CPtr);
+    return Builder.saveIP();
+  };
+
+  llvm::SmallVector<llvm::Value *> Inputs;
+  Inputs.push_back(APtr);
+  Inputs.push_back(BPtr);
+  Inputs.push_back(CPtr);
+
+  TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17);
+  OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL});
+  Builder.restoreIP(OMPBuilder.createTarget(OmpLoc, Builder.saveIP(), EntryInfo,
+                                            -1, -1, Inputs, BodyGenCB));
+  OMPBuilder.finalize();
+  Builder.CreateRetVoid();
+
+  // Check the outlined call
+  auto Iter = F->getEntryBlock().rbegin();
+  CallInst *Call = dyn_cast<CallInst>(&*(++Iter));
+  EXPECT_NE(Call, nullptr);
+
+  // Check that the correct aguments are passed in
+  for (auto ArgInput : zip(Call->args(), Inputs)) {
+    EXPECT_EQ(std::get<0>(ArgInput), std::get<1>(ArgInput));
+  }
+
+  // Check that the outlined function exists with the expected prefix
+  Function *OutlinedFunc = Call->getCalledFunction();
+  EXPECT_NE(OutlinedFunc, nullptr);
+  StringRef FunctionName = OutlinedFunc->getName();
+  EXPECT_TRUE(FunctionName.startswith("__omp_offloading"));
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
 TEST_F(OpenMPIRBuilderTest, CreateTask) {
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
   OpenMPIRBuilder OMPBuilder(*M);
diff --git a/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp b/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp
index 7afedf756101c..aed95890075e0 100644
--- a/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp
+++ b/llvm/unittests/FuzzMutate/RandomIRBuilderTest.cpp
@@ -533,4 +533,34 @@ TEST(RandomIRBuilderTest, sinkToInstrinsic) {
   }
   ASSERT_FALSE(Modified);
 }
+
+TEST(RandomIRBuilderTest, DoNotCallPointerWhenSink) {
+  const char *Source = "\n\
+        declare void @g()  \n\
+        define void @f(ptr %ptr) {  \n\
+        Entry:   \n\
+            call void @g()  \n\
+            ret void \n\
+        }";
+  LLVMContext Ctx;
+  std::mt19937 mt(Seed);
+  std::uniform_int_distribution<int> RandInt(INT_MIN, INT_MAX);
+
+  RandomIRBuilder IB(RandInt(mt), {});
+  std::unique_ptr<Module> M = parseAssembly(Source, Ctx);
+  Function &F = *M->getFunction("f");
+  BasicBlock &BB = F.getEntryBlock();
+  bool Modified = false;
+
+  Instruction *I = &*BB.begin();
+  for (int i = 0; i < 20; i++) {
+    Value *OldOperand = I->getOperand(0);
+    Value *Src = F.getArg(0);
+    IB.connectToSink(BB, {I}, Src);
+    Value *NewOperand = I->getOperand(0);
+    Modified |= (OldOperand != NewOperand);
+    ASSERT_FALSE(verifyModule(*M, &errs()));
+  }
+  ASSERT_FALSE(Modified);
+}
 } // namespace
diff --git a/llvm/unittests/FuzzMutate/StrategiesTest.cpp b/llvm/unittests/FuzzMutate/StrategiesTest.cpp
index 1de027c43e49d..850ae16dea442 100644
--- a/llvm/unittests/FuzzMutate/StrategiesTest.cpp
+++ b/llvm/unittests/FuzzMutate/StrategiesTest.cpp
@@ -665,4 +665,30 @@ TEST(ShuffleBlockStrategy, ShuffleLoop) {
     }";
   VerifyBlockShuffle(Source);
 }
+
+TEST(AllStrategies, SkipEHPad) {
+  StringRef Source = "\n\
+    define void @f(i32 %x) personality ptr @__CxxFrameHandler3 { \n\
+    entry: \n\
+      invoke void @g() to label %try.cont unwind label %catch.dispatch \n\
+    catch.dispatch: \n\
+      %0 = catchswitch within none [label %catch] unwind to caller \n\
+    catch: \n\
+      %1 = catchpad within %0 [ptr null, i32 64, ptr null] \n\
+      catchret from %1 to label %try.cont \n\
+    try.cont: \n\
+      ret void \n\
+    } \n\
+    declare void @g() \n\
+    declare i32 @__CxxFrameHandler3(...) \n\
+    ";
+
+  mutateAndVerifyModule<ShuffleBlockStrategy>(Source);
+  mutateAndVerifyModule<InsertPHIStrategy>(Source);
+  mutateAndVerifyModule<InsertFunctionStrategy>(Source);
+  mutateAndVerifyModule<InsertCFGStrategy>(Source);
+  mutateAndVerifyModule<SinkInstructionStrategy>(Source);
+  mutateAndVerifyModule<InjectorIRStrategy>(Source);
+  mutateAndVerifyModule<InstModificationIRStrategy>(Source);
+}
 } // namespace
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index f038ff2fc7f31..a22c7be73f49f 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -190,6 +190,47 @@ TEST(MetadataTest, DeleteInstUsedByDbgValue) {
   EXPECT_TRUE(isa<UndefValue>(DVIs[0]->getValue(0)));
 }
 
+TEST(DbgVariableIntrinsic, EmptyMDIsKillLocation) {
+  LLVMContext Ctx;
+  std::unique_ptr<Module> M = parseIR(Ctx, R"(
+    define dso_local void @fun() local_unnamed_addr #0 !dbg !9 {
+    entry:
+      call void @llvm.dbg.declare(metadata !{}, metadata !13, metadata !DIExpression()), !dbg !16
+      ret void, !dbg !16
+    }
+
+    declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+    !llvm.dbg.cu = !{!0}
+    !llvm.module.flags = !{!2, !3}
+    !llvm.ident = !{!8}
+
+    !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 16.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+    !1 = !DIFile(filename: "test.c", directory: "/")
+    !2 = !{i32 7, !"Dwarf Version", i32 5}
+    !3 = !{i32 2, !"Debug Info Version", i32 3}
+    !8 = !{!"clang version 16.0.0"}
+    !9 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+    !10 = !DISubroutineType(types: !11)
+    !11 = !{null}
+    !12 = !{!13}
+    !13 = !DILocalVariable(name: "a", scope: !9, file: !1, line: 1, type: !14)
+    !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+    !16 = !DILocation(line: 1, column: 21, scope: !9)
+    )");
+
+  bool BrokenDebugInfo = true;
+  verifyModule(*M, &errs(), &BrokenDebugInfo);
+  ASSERT_FALSE(BrokenDebugInfo);
+
+  // Get the dbg.declare.
+  Function &F = *cast<Function>(M->getNamedValue("fun"));
+  DbgVariableIntrinsic *DbgDeclare =
+      cast<DbgVariableIntrinsic>(&F.front().front());
+  // Check that this form counts as a "no location" marker.
+  EXPECT_TRUE(DbgDeclare->isKillLocation());
+}
+
 TEST(DIBuiler, CreateFile) {
   LLVMContext Ctx;
   std::unique_ptr<Module> M(new Module("MyModule", Ctx));
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index eb506f6c9e4f3..b43a8c0bc5e60 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -102,6 +102,11 @@ TEST_F(ModuleWithFunctionTest, CallInst) {
   Call->addRetAttr(Attribute::get(Call->getContext(), "test-str-attr"));
   EXPECT_TRUE(Call->hasRetAttr("test-str-attr"));
   EXPECT_FALSE(Call->hasRetAttr("not-on-call"));
+
+  Call->addFnAttr(Attribute::get(Call->getContext(), "test-str-fn-attr"));
+  ASSERT_TRUE(Call->hasFnAttr("test-str-fn-attr"));
+  Call->removeFnAttr("test-str-fn-attr");
+  EXPECT_FALSE(Call->hasFnAttr("test-str-fn-attr"));
 }
 
 TEST_F(ModuleWithFunctionTest, InvokeInst) {
diff --git a/llvm/unittests/MI/CMakeLists.txt b/llvm/unittests/MI/CMakeLists.txt
index 005373ad21731..a2922ae9b34fd 100644
--- a/llvm/unittests/MI/CMakeLists.txt
+++ b/llvm/unittests/MI/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   CodeGen
+  CodeGenTypes
   Core
   MC
   MIRParser
diff --git a/llvm/unittests/MIR/CMakeLists.txt b/llvm/unittests/MIR/CMakeLists.txt
index 9b70af6578a0b..3c0e9e43f9afb 100644
--- a/llvm/unittests/MIR/CMakeLists.txt
+++ b/llvm/unittests/MIR/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   CodeGen
+  CodeGenTypes
   Core
   FileCheck
   MC
diff --git a/llvm/unittests/Object/CMakeLists.txt b/llvm/unittests/Object/CMakeLists.txt
index 2de3e046cff03..75b4f20922ce6 100644
--- a/llvm/unittests/Object/CMakeLists.txt
+++ b/llvm/unittests/Object/CMakeLists.txt
@@ -11,6 +11,7 @@ add_llvm_unittest(ObjectTests
   ELFObjectFileTest.cpp
   ELFTypesTest.cpp
   ELFTest.cpp
+  GOFFObjectFileTest.cpp
   MinidumpTest.cpp
   ObjectFileTest.cpp
   OffloadingTest.cpp
@@ -20,3 +21,8 @@ add_llvm_unittest(ObjectTests
   )
 
 target_link_libraries(ObjectTests PRIVATE LLVMTestingSupport)
+
+if (MSVC)
+  # Disable warning C4309: '=': truncation of constant value
+  set_source_files_properties(GOFFObjectFileTest.cpp PROPERTIES COMPILE_FLAGS -wd4309)
+endif()
diff --git a/llvm/unittests/Object/GOFFObjectFileTest.cpp b/llvm/unittests/Object/GOFFObjectFileTest.cpp
new file mode 100644
index 0000000000000..734dac6b8507a
--- /dev/null
+++ b/llvm/unittests/Object/GOFFObjectFileTest.cpp
@@ -0,0 +1,504 @@
+//===- GOFFObjectFileTest.cpp - Tests for GOFFObjectFile ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/GOFFObjectFile.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::object;
+using namespace llvm::GOFF;
+
+namespace {
+char GOFFData[GOFF::RecordLength * 3] = {0x00};
+
+void constructValidGOFF(size_t Size) {
+  StringRef ValidSize(GOFFData, Size);
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(ValidSize, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+}
+
+void constructInvalidGOFF(size_t Size) {
+  // Construct GOFFObject with record of length != multiple of 80.
+  StringRef InvalidData(GOFFData, Size);
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(InvalidData, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(
+      GOFFObjOrErr,
+      FailedWithMessage("object file is not the right size. Must be a multiple "
+                        "of 80 bytes, but is " +
+                        std::to_string(Size) + " bytes"));
+}
+} // namespace
+
+TEST(GOFFObjectFileTest, ConstructGOFFObjectValidSize) {
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+  GOFFData[80] = (char)0x03;
+  GOFFData[81] = (char)0x40;
+  constructValidGOFF(160);
+  constructValidGOFF(0);
+}
+
+TEST(GOFFObjectFileTest, ConstructGOFFObjectInvalidSize) {
+  constructInvalidGOFF(70);
+  constructInvalidGOFF(79);
+  constructInvalidGOFF(81);
+}
+
+TEST(GOFFObjectFileTest, MissingHDR) {
+  char GOFFData[GOFF::RecordLength * 2] = {0x00};
+
+  // ESD record.
+  GOFFData[0] = (char)0x03;
+
+  // END record.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+  GOFFData[GOFF::RecordLength + 1] = (char)0x40;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 2);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(
+      GOFFObjOrErr,
+      FailedWithMessage("object file must start with HDR record"));
+}
+
+TEST(GOFFObjectFileTest, MissingEND) {
+  char GOFFData[GOFF::RecordLength * 2] = {0x00};
+
+  // HDR record.
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 2);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(
+      GOFFObjOrErr, FailedWithMessage("object file must end with END record"));
+}
+
+TEST(GOFFObjectFileTest, GetSymbolName) {
+  char GOFFData[GOFF::RecordLength * 3] = {0x00};
+
+  // HDR record.
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+  GOFFData[GOFF::RecordLength + 3] = (char)0x02;
+  GOFFData[GOFF::RecordLength + 7] = (char)0x01;
+  GOFFData[GOFF::RecordLength + 11] = (char)0x01;
+  GOFFData[GOFF::RecordLength + 71] = (char)0x05; // Size of symbol name.
+  GOFFData[GOFF::RecordLength + 72] = (char)0xC8; // Symbol name is Hello.
+  GOFFData[GOFF::RecordLength + 73] = (char)0x85;
+  GOFFData[GOFF::RecordLength + 74] = (char)0x93;
+  GOFFData[GOFF::RecordLength + 75] = (char)0x93;
+  GOFFData[GOFF::RecordLength + 76] = (char)0x96;
+
+  // END record.
+  GOFFData[GOFF::RecordLength * 2] = 0x03;
+  GOFFData[GOFF::RecordLength * 2 + 1] = 0x40;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 3);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+
+  GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get());
+
+  for (SymbolRef Symbol : GOFFObj->symbols()) {
+    Expected<StringRef> SymbolNameOrErr = GOFFObj->getSymbolName(Symbol);
+    ASSERT_THAT_EXPECTED(SymbolNameOrErr, Succeeded());
+    StringRef SymbolName = SymbolNameOrErr.get();
+
+    EXPECT_EQ(SymbolName, "Hello");
+  }
+}
+
+TEST(GOFFObjectFileTest, ConcatenatedGOFFFile) {
+  char GOFFData[GOFF::RecordLength * 6] = {0x00};
+
+  // HDR record.
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+  // ESD record.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+  // END record.
+  GOFFData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 2 + 1] = (char)0x40;
+  // HDR record.
+  GOFFData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 3 + 1] = (char)0xF0;
+  // ESD record.
+  GOFFData[GOFF::RecordLength * 4] = (char)0x03;
+  // END record.
+  GOFFData[GOFF::RecordLength * 5] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 5 + 1] = (char)0x40;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 6);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+}
+
+TEST(GOFFObjectFileTest, ContinuationGetSymbolName) {
+  char GOFFContData[GOFF::RecordLength * 4] = {0x00};
+
+  // HDR record.
+  GOFFContData[0] = (char)0x03;
+  GOFFContData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFContData[GOFF::RecordLength] = (char)0x03;
+  GOFFContData[GOFF::RecordLength + 1] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 3] = (char)0x02;
+  GOFFContData[GOFF::RecordLength + 7] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 11] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 71] = (char)0x0A; // Size of symbol name.
+  GOFFContData[GOFF::RecordLength + 72] = (char)0xC8; // Symbol name is HelloWorld.
+  GOFFContData[GOFF::RecordLength + 73] = (char)0x85;
+  GOFFContData[GOFF::RecordLength + 74] = (char)0x93;
+  GOFFContData[GOFF::RecordLength + 75] = (char)0x93;
+  GOFFContData[GOFF::RecordLength + 76] = (char)0x96;
+  GOFFContData[GOFF::RecordLength + 77] = (char)0xA6;
+  GOFFContData[GOFF::RecordLength + 78] = (char)0x96;
+  GOFFContData[GOFF::RecordLength + 79] = (char)0x99;
+
+  // ESD continuation record.
+  GOFFContData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 2 + 1] = (char)0x02; // No further continuations.
+  GOFFContData[GOFF::RecordLength * 2 + 3] = (char)0x93;
+  GOFFContData[GOFF::RecordLength * 2 + 4] = (char)0x84;
+
+  // END record.
+  GOFFContData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 3 + 1] = (char)0x40;
+
+  StringRef Data(GOFFContData, GOFF::RecordLength * 4);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+
+  GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get());
+
+  for (SymbolRef Symbol : GOFFObj->symbols()) {
+    Expected<StringRef> SymbolNameOrErr = GOFFObj->getSymbolName(Symbol);
+    ASSERT_THAT_EXPECTED(SymbolNameOrErr, Succeeded());
+    StringRef SymbolName = SymbolNameOrErr.get();
+    EXPECT_EQ(SymbolName, "Helloworld");
+  }
+}
+
+TEST(GOFFObjectFileTest, ContinuationBitNotSet) {
+  char GOFFContData[GOFF::RecordLength * 4] = {0x00};
+
+  // HDR record.
+  GOFFContData[0] = (char)0x03;
+  GOFFContData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFContData[GOFF::RecordLength] = (char)0x03;
+  GOFFContData[GOFF::RecordLength + 1] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 3] = (char)0x02;
+  GOFFContData[GOFF::RecordLength + 7] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 11] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 71] = (char)0x0A; // Size of symbol name.
+  GOFFContData[GOFF::RecordLength + 72] = (char)0xC8; // Symbol name is HelloWorld.
+  GOFFContData[GOFF::RecordLength + 73] = (char)0x85;
+  GOFFContData[GOFF::RecordLength + 74] = (char)0x93;
+  GOFFContData[GOFF::RecordLength + 75] = (char)0x93;
+  GOFFContData[GOFF::RecordLength + 76] = (char)0x96;
+  GOFFContData[GOFF::RecordLength + 77] = (char)0xA6;
+  GOFFContData[GOFF::RecordLength + 78] = (char)0x96;
+  GOFFContData[GOFF::RecordLength + 79] = (char)0x99;
+
+  // ESD continuation record.
+  GOFFContData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 2 + 1] = (char)0x00;
+  GOFFContData[GOFF::RecordLength * 2 + 3] = (char)0x93;
+  GOFFContData[GOFF::RecordLength * 2 + 4] = (char)0x84;
+
+  // END record.
+  GOFFContData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 3 + 1] = (char)0x40;
+
+  StringRef Data(GOFFContData, GOFF::RecordLength * 4);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+  EXPECT_THAT_EXPECTED(
+      GOFFObjOrErr,
+      FailedWithMessage("record 2 is not a continuation record but the "
+                        "preceding record is continued"));
+}
+
+TEST(GOFFObjectFileTest, ContinuationRecordNotTerminated) {
+  char GOFFContData[GOFF::RecordLength * 4] = {0x00};
+
+  // HDR record.
+  GOFFContData[0] = (char)0x03;
+  GOFFContData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFContData[GOFF::RecordLength] = (char)0x03;
+  GOFFContData[GOFF::RecordLength + 1] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 3] = (char)0x02;
+  GOFFContData[GOFF::RecordLength + 7] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 11] = (char)0x01;
+  GOFFContData[GOFF::RecordLength + 71] = (char)0x0A; // Size of symbol name.
+  GOFFContData[GOFF::RecordLength + 72] = (char)0xC8; // Symbol name is HelloWorld.
+  GOFFContData[GOFF::RecordLength + 73] = (char)0x85;
+  GOFFContData[GOFF::RecordLength + 74] = (char)0x93;
+  GOFFContData[GOFF::RecordLength + 75] = (char)0x93;
+  GOFFContData[GOFF::RecordLength + 76] = (char)0x96;
+  GOFFContData[GOFF::RecordLength + 77] = (char)0xA6;
+  GOFFContData[GOFF::RecordLength + 78] = (char)0x96;
+  GOFFContData[GOFF::RecordLength + 79] = (char)0x99;
+
+  // ESD continuation record.
+  GOFFContData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 2 + 1] = (char)0x03; // Continued bit set.
+  GOFFContData[GOFF::RecordLength * 2 + 3] = (char)0x93;
+  GOFFContData[GOFF::RecordLength * 2 + 4] = (char)0x84;
+
+  // END record.
+  GOFFContData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 3 + 1] = (char)0x40;
+
+  StringRef Data(GOFFContData, GOFF::RecordLength * 4);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+
+  GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get());
+
+  for (SymbolRef Symbol : GOFFObj->symbols()) {
+    Expected<StringRef> SymbolNameOrErr = GOFFObj->getSymbolName(Symbol);
+    EXPECT_THAT_EXPECTED(SymbolNameOrErr,
+                         FailedWithMessage("continued bit should not be set"));
+  }
+}
+
+TEST(GOFFObjectFileTest, PrevNotContinued) {
+  char GOFFContData[GOFF::RecordLength * 4] = {0x00};
+
+  // HDR record.
+  GOFFContData[0] = (char)0x03;
+  GOFFContData[1] = (char)0xF0;
+
+  // ESD record, with continued bit not set.
+  GOFFContData[GOFF::RecordLength] = (char)0x03;
+
+  // ESD continuation record.
+  GOFFContData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 2 + 1] = (char)0x02;
+
+  // END record.
+  GOFFContData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 3 + 1] = (char)0x40;
+
+  StringRef Data(GOFFContData, GOFF::RecordLength * 4);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(
+      GOFFObjOrErr,
+      FailedWithMessage("record 2 is a continuation record that is not "
+                        "preceded by a continued record"));
+}
+
+TEST(GOFFObjectFileTest, ContinuationTypeMismatch) {
+  char GOFFContData[GOFF::RecordLength * 4] = {0x00};
+
+  // HDR record.
+  GOFFContData[0] = (char)0x03;
+  GOFFContData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFContData[GOFF::RecordLength] = (char)0x03;
+  GOFFContData[GOFF::RecordLength + 1] = (char)0x01; // Continued to next record.
+
+  // END continuation record.
+  GOFFContData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 2 + 1] = (char)0x42;
+
+  // END record.
+  GOFFContData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFContData[GOFF::RecordLength * 3 + 1] = (char)0x40;
+
+  StringRef Data(GOFFContData, GOFF::RecordLength * 4);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(
+      GOFFObjOrErr,
+      FailedWithMessage("record 2 is a continuation record that does not match "
+                        "the type of the previous record"));
+}
+
+TEST(GOFFObjectFileTest, TwoSymbols) {
+  char GOFFData[GOFF::RecordLength * 4] = {0x00};
+
+  // HDR record.
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+
+  // ESD record 1.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+  GOFFData[GOFF::RecordLength + 3] = (char)0x00;
+  GOFFData[GOFF::RecordLength + 7] = (char)0x01;  // ESDID.
+  GOFFData[GOFF::RecordLength + 71] = (char)0x01; // Size of symbol name.
+  GOFFData[GOFF::RecordLength + 72] = (char)0xa7; // Symbol name is x.
+
+  // ESD record 2.
+  GOFFData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 2 + 3] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 2 + 7] = (char)0x02;  // ESDID.
+  GOFFData[GOFF::RecordLength * 2 + 11] = (char)0x01; // Parent ESDID.
+  GOFFData[GOFF::RecordLength * 2 + 71] = (char)0x05; // Size of symbol name.
+  GOFFData[GOFF::RecordLength * 2 + 72] = (char)0xC8; // Symbol name is Hello.
+  GOFFData[GOFF::RecordLength * 2 + 73] = (char)0x85;
+  GOFFData[GOFF::RecordLength * 2 + 74] = (char)0x93;
+  GOFFData[GOFF::RecordLength * 2 + 75] = (char)0x93;
+  GOFFData[GOFF::RecordLength * 2 + 76] = (char)0x96;
+
+  // END record.
+  GOFFData[GOFF::RecordLength * 3] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 3 + 1] = (char)0x40;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 4);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+
+  GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get());
+
+  for (SymbolRef Symbol : GOFFObj->symbols()) {
+    Expected<StringRef> SymbolNameOrErr = GOFFObj->getSymbolName(Symbol);
+    ASSERT_THAT_EXPECTED(SymbolNameOrErr, Succeeded());
+    StringRef SymbolName = SymbolNameOrErr.get();
+    EXPECT_EQ(SymbolName, "Hello");
+  }
+}
+
+TEST(GOFFObjectFileTest, InvalidSymbolType) {
+  char GOFFData[GOFF::RecordLength * 3] = {0x00};
+
+  // HDR record.
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+  GOFFData[GOFF::RecordLength + 3] = (char)0x05;
+  GOFFData[GOFF::RecordLength + 7] = (char)0x01;
+  GOFFData[GOFF::RecordLength + 11] = (char)0x01;
+  GOFFData[GOFF::RecordLength + 71] = (char)0x01; // Size of symbol name.
+  GOFFData[GOFF::RecordLength + 72] = (char)0xC8; // Symbol name.
+
+  // END record.
+  GOFFData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 2 + 1] = (char)0x40;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 3);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+
+  GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get());
+
+  for (SymbolRef Symbol : GOFFObj->symbols()) {
+    Expected<SymbolRef::Type> SymbolType = Symbol.getType();
+    EXPECT_THAT_EXPECTED(
+        SymbolType,
+        FailedWithMessage("ESD record 1 has invalid symbol type 0x05"));
+
+    Expected<section_iterator> SymSI = Symbol.getSection();
+    ASSERT_THAT_EXPECTED(
+        SymSI,
+        FailedWithMessage(
+            "symbol with ESD id 1 refers to invalid section with ESD id 1"));
+  }
+}
+
+TEST(GOFFObjectFileTest, InvalidERSymbolType) {
+  char GOFFData[GOFF::RecordLength * 3] = {0x00};
+
+  // HDR record.
+  GOFFData[0] = (char)0x03;
+  GOFFData[1] = (char)0xF0;
+
+  // ESD record.
+  GOFFData[GOFF::RecordLength] = (char)0x03;
+  GOFFData[GOFF::RecordLength + 3] = (char)0x04;
+  GOFFData[GOFF::RecordLength + 7] = (char)0x01;
+  GOFFData[GOFF::RecordLength + 11] = (char)0x01;
+  GOFFData[GOFF::RecordLength + 63] = (char)0x03; // Unknown executable type.
+  GOFFData[GOFF::RecordLength + 71] = (char)0x01; // Size of symbol name.
+  GOFFData[GOFF::RecordLength + 72] = (char)0xC8; // Symbol name.
+
+  // END record.
+  GOFFData[GOFF::RecordLength * 2] = (char)0x03;
+  GOFFData[GOFF::RecordLength * 2 + 1] = (char)0x40;
+
+  StringRef Data(GOFFData, GOFF::RecordLength * 3);
+
+  Expected<std::unique_ptr<ObjectFile>> GOFFObjOrErr =
+      object::ObjectFile::createGOFFObjectFile(
+          MemoryBufferRef(Data, "dummyGOFF"));
+
+  ASSERT_THAT_EXPECTED(GOFFObjOrErr, Succeeded());
+
+  GOFFObjectFile *GOFFObj = dyn_cast<GOFFObjectFile>((*GOFFObjOrErr).get());
+
+  for (SymbolRef Symbol : GOFFObj->symbols()) {
+    Expected<SymbolRef::Type> SymbolType = Symbol.getType();
+    EXPECT_THAT_EXPECTED(
+        SymbolType,
+        FailedWithMessage("ESD record 1 has unknown Executable type 0x03"));
+  }
+}
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index 3ed21767035d2..30c4dac0bd2f2 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -869,7 +869,9 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1_saturation) {
   const uint64_t MaxEdgeCount = getInstrMaxCountValue();
 
   instrprof_error Result;
-  auto Err = [&](Error E) { Result = InstrProfError::take(std::move(E)); };
+  auto Err = [&](Error E) {
+    Result = std::get<0>(InstrProfError::take(std::move(E)));
+  };
   Result = instrprof_error::success;
   Writer.addRecord({"foo", 0x1234, {1}}, Err);
   ASSERT_EQ(Result, instrprof_error::success);
@@ -1354,16 +1356,18 @@ TEST(ProfileReaderTest, ReadsLargeFiles) {
   if (!RawProfile)
     GTEST_SKIP();
   auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));
-  ASSERT_TRUE(InstrProfError::take(RawProfileReaderOrErr.takeError()) ==
-              instrprof_error::unrecognized_format);
+  ASSERT_TRUE(
+      std::get<0>(InstrProfError::take(RawProfileReaderOrErr.takeError())) ==
+      instrprof_error::unrecognized_format);
 
   auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
   if (!IndexedProfile)
     GTEST_SKIP();
   auto IndexedReaderOrErr =
       IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);
-  ASSERT_TRUE(InstrProfError::take(IndexedReaderOrErr.takeError()) ==
-              instrprof_error::bad_magic);
+  ASSERT_TRUE(
+      std::get<0>(InstrProfError::take(IndexedReaderOrErr.takeError())) ==
+      instrprof_error::bad_magic);
 }
 #endif
 
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index bd34b05336e19..e01025440e4a5 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -289,11 +289,10 @@ struct SampleProfTest : ::testing::Test {
     ASSERT_FALSE(CTMap.getError());
 
     // Because _Z3bazi is not defined in module M, expect _Z3bazi's profile
-    // is not loaded when the profile is ExtBinary or Compact format because
-    // these formats support loading function profiles on demand.
+    // is not loaded when the profile is ExtBinary format because this format
+    // supports loading function profiles on demand.
     FunctionSamples *ReadBazSamples = Reader->getSamplesFor(BazName);
-    if (Format == SampleProfileFormat::SPF_Ext_Binary ||
-        Format == SampleProfileFormat::SPF_Compact_Binary) {
+    if (Format == SampleProfileFormat::SPF_Ext_Binary) {
       ASSERT_TRUE(ReadBazSamples == nullptr);
       ASSERT_EQ(3u, Reader->getProfiles().size());
     } else {
@@ -408,10 +407,6 @@ TEST_F(SampleProfTest, roundtrip_raw_binary_profile) {
   testRoundTrip(SampleProfileFormat::SPF_Binary, false, false);
 }
 
-TEST_F(SampleProfTest, roundtrip_compact_binary_profile) {
-  testRoundTrip(SampleProfileFormat::SPF_Compact_Binary, false, true);
-}
-
 TEST_F(SampleProfTest, roundtrip_ext_binary_profile) {
   testRoundTrip(SampleProfileFormat::SPF_Ext_Binary, false, false);
 }
@@ -473,19 +468,6 @@ TEST_F(SampleProfTest, default_suffix_elision_text) {
   testSuffixElisionPolicy(SampleProfileFormat::SPF_Text, "", Expected);
 }
 
-TEST_F(SampleProfTest, default_suffix_elision_compact_binary) {
-  // Default suffix elision policy: strip everything after first dot.
-  // This implies that all suffix variants will map to "foo", so
-  // we don't expect to see any entries for them in the sample
-  // profile.
-  StringMap<uint64_t> Expected;
-  Expected["foo"] = uint64_t(20301);
-  Expected["foo.bar"] = uint64_t(-1);
-  Expected["foo.llvm.2465"] = uint64_t(-1);
-  testSuffixElisionPolicy(SampleProfileFormat::SPF_Compact_Binary, "",
-                          Expected);
-}
-
 TEST_F(SampleProfTest, selected_suffix_elision_text) {
   // Profile is created and searched using the "selected"
   // suffix elision policy: we only strip a .XXX suffix if
@@ -498,19 +480,6 @@ TEST_F(SampleProfTest, selected_suffix_elision_text) {
   testSuffixElisionPolicy(SampleProfileFormat::SPF_Text, "selected", Expected);
 }
 
-TEST_F(SampleProfTest, selected_suffix_elision_compact_binary) {
-  // Profile is created and searched using the "selected"
-  // suffix elision policy: we only strip a .XXX suffix if
-  // it matches a pattern known to be generated by the compiler
-  // (e.g. ".llvm.<digits>").
-  StringMap<uint64_t> Expected;
-  Expected["foo"] = uint64_t(20301);
-  Expected["foo.bar"] = uint64_t(20303);
-  Expected["foo.llvm.2465"] = uint64_t(-1);
-  testSuffixElisionPolicy(SampleProfileFormat::SPF_Compact_Binary, "selected",
-                          Expected);
-}
-
 TEST_F(SampleProfTest, none_suffix_elision_text) {
   // Profile is created and searched using the "none"
   // suffix elision policy: no stripping of suffixes at all.
@@ -522,16 +491,4 @@ TEST_F(SampleProfTest, none_suffix_elision_text) {
   testSuffixElisionPolicy(SampleProfileFormat::SPF_Text, "none", Expected);
 }
 
-TEST_F(SampleProfTest, none_suffix_elision_compact_binary) {
-  // Profile is created and searched using the "none"
-  // suffix elision policy: no stripping of suffixes at all.
-  // Here we expect to see all variants in the profile.
-  StringMap<uint64_t> Expected;
-  Expected["foo"] = uint64_t(20301);
-  Expected["foo.bar"] = uint64_t(20303);
-  Expected["foo.llvm.2465"] = uint64_t(20305);
-  testSuffixElisionPolicy(SampleProfileFormat::SPF_Compact_Binary, "none",
-                          Expected);
-}
-
 } // end anonymous namespace
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 0caa6ca7dfbd9..d6e4fa5869379 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -1,6 +1,3 @@
-set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/include/llvm/CodeGen/ValueTypes.td)
-tablegen(LLVM GenVT.inc -gen-vt)
-
 set(LLVM_LINK_COMPONENTS
   Support
   TargetParser
@@ -25,6 +22,7 @@ add_llvm_unittest(SupportTests
   Chrono.cpp
   CommandLineTest.cpp
   CompressionTest.cpp
+  ConvertEBCDICTest.cpp
   ConvertUTFTest.cpp
   CRCTest.cpp
   CSKYAttributeParserTest.cpp
@@ -56,7 +54,6 @@ add_llvm_unittest(SupportTests
   LockFileManagerTest.cpp
   MatchersTest.cpp
   MD5Test.cpp
-  MVTTest.cpp
   ManagedStatic.cpp
   MathExtrasTest.cpp
   MemoryBufferRefTest.cpp
@@ -105,7 +102,6 @@ add_llvm_unittest(SupportTests
   raw_pwrite_stream_test.cpp
   raw_sha1_ostream_test.cpp
   xxhashTest.cpp
-  GenVT.inc
   )
 
 target_link_libraries(SupportTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/Support/ConvertEBCDICTest.cpp b/llvm/unittests/Support/ConvertEBCDICTest.cpp
new file mode 100644
index 0000000000000..eec76879ac92c
--- /dev/null
+++ b/llvm/unittests/Support/ConvertEBCDICTest.cpp
@@ -0,0 +1,97 @@
+//===- unittests/Support/ConvertEBCDICTest.cpp - EBCDIC/UTF8 conversion tests
+//-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--------------------------------------------------------------------------===//
+
+#include "llvm/Support/ConvertEBCDIC.h"
+#include "llvm/ADT/SmallString.h"
+#include "gtest/gtest.h"
+using namespace llvm;
+
+namespace {
+
+// String "Hello World!"
+static const char HelloA[] =
+    "\x48\x65\x6C\x6C\x6F\x20\x57\x6F\x72\x6C\x64\x21\x0a";
+static const char HelloE[] =
+    "\xC8\x85\x93\x93\x96\x40\xE6\x96\x99\x93\x84\x5A\x15";
+
+// String "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+static const char ABCStrA[] =
+    "\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F\x50\x51\x52"
+    "\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A"
+    "\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7A";
+static const char ABCStrE[] =
+    "\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9"
+    "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\x81\x82\x83\x84\x85\x86\x87\x88\x89\x91"
+    "\x92\x93\x94\x95\x96\x97\x98\x99\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9";
+
+// String "¡¢£AÄÅÆEÈÉÊaàáâãäeèéêë"
+static const char AccentUTF[] =
+    "\xc2\xa1\xc2\xa2\xc2\xa3\x41\xc3\x84\xc3\x85\xc3\x86\x45\xc3\x88\xc3\x89"
+    "\xc3\x8a\x61\xc3\xa0\xc3\xa1\xc3\xa2\xc3\xa3\xc3\xa4\x65\xc3\xa8\xc3\xa9"
+    "\xc3\xaa\xc3\xab";
+static const char AccentE[] = "\xaa\x4a\xb1\xc1\x63\x67\x9e\xc5\x74\x71\x72"
+                              "\x81\x44\x45\x42\x46\x43\x85\x54\x51\x52\x53";
+
+// String with Cyrillic character ya.
+static const char CyrillicUTF[] = "\xd0\xaf";
+
+TEST(CharSet, FromUTF8) {
+  // Hello string.
+  StringRef Src(HelloA);
+  SmallString<64> Dst;
+
+  std::error_code EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(HelloE, static_cast<std::string>(Dst).c_str());
+  Dst.clear();
+
+  // ABC string.
+  Src = ABCStrA;
+  EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(ABCStrE, static_cast<std::string>(Dst).c_str());
+  Dst.clear();
+
+  // Accent string.
+  Src = AccentUTF;
+  EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_TRUE(!EC);
+  EXPECT_STREQ(AccentE, static_cast<std::string>(Dst).c_str());
+  Dst.clear();
+
+  // Cyrillic string. Results in error because not representable in 1047.
+  Src = CyrillicUTF;
+  EC = ConverterEBCDIC::convertToEBCDIC(Src, Dst);
+  EXPECT_EQ(EC, std::errc::illegal_byte_sequence);
+  Dst.clear();
+}
+
+TEST(CharSet, ToUTF8) {
+  // Hello string.
+  StringRef Src(HelloE);
+  SmallString<64> Dst;
+
+  ConverterEBCDIC::convertToUTF8(Src, Dst);
+  EXPECT_STREQ(HelloA, static_cast<std::string>(Dst).c_str());
+  Dst.clear();
+
+  // ABC string.
+  Src = ABCStrE;
+  ConverterEBCDIC::convertToUTF8(Src, Dst);
+  EXPECT_STREQ(ABCStrA, static_cast<std::string>(Dst).c_str());
+  Dst.clear();
+
+  // Accent string.
+  Src = AccentE;
+  ConverterEBCDIC::convertToUTF8(Src, Dst);
+  EXPECT_STREQ(AccentUTF, static_cast<std::string>(Dst).c_str());
+  Dst.clear();
+}
+
+} // namespace
diff --git a/llvm/unittests/Support/MVTTest.cpp b/llvm/unittests/Support/MVTTest.cpp
deleted file mode 100644
index a646d52aa9778..0000000000000
--- a/llvm/unittests/Support/MVTTest.cpp
+++ /dev/null
@@ -1,106 +0,0 @@
-//===- llvm/unittest/Support/MVTTest.cpp - Test compatibility -------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Make sure the generated version of MachineValueType.h to be equivalent to
-// the constant version of llvm/Support/MachineValueType.h.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MachineValueType.h"
-#include "llvm/Support/MachineValueType.h"
-#include "gtest/gtest.h"
-#include <limits>
-#include <string>
-
-using namespace llvm;
-
-namespace {
-
-TEST(MVTTest, Properties) {
-  for (int i = 0; i <= std::numeric_limits<uint8_t>::max(); ++i) {
-    SCOPED_TRACE("i=" + std::to_string(i));
-    auto Org = MVT(MVT::SimpleValueType(i));
-    auto New = tmp::MVT(tmp::MVT::SimpleValueType(i));
-
-#define MVTTEST_EXPECT_EQ_M(LHS, RHS, M) EXPECT_EQ((LHS).M(), (RHS).M())
-#define MVTTEST_EXPECT_EQ_SVT(LHS, RHS)                                        \
-  EXPECT_EQ(int((LHS).SimpleTy), int((RHS).SimpleTy))
-#define MVTTEST_EXPECT_EQ_SVT_M(LHS, RHS, M)                                   \
-  MVTTEST_EXPECT_EQ_SVT((LHS).M(), (RHS).M())
-#define MVTTEST_EXPECT_EQ_SVT_F(F, ...)                                        \
-  MVTTEST_EXPECT_EQ_SVT(MVT::F(__VA_ARGS__), tmp::MVT::F(__VA_ARGS__))
-
-    MVTTEST_EXPECT_EQ_M(New, Org, isValid);
-    MVTTEST_EXPECT_EQ_M(New, Org, isFloatingPoint);
-    MVTTEST_EXPECT_EQ_M(New, Org, isInteger);
-    MVTTEST_EXPECT_EQ_M(New, Org, isScalarInteger);
-    MVTTEST_EXPECT_EQ_M(New, Org, isVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, isScalableVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, isScalableTargetExtVT);
-    MVTTEST_EXPECT_EQ_M(New, Org, isScalableVT);
-    MVTTEST_EXPECT_EQ_M(New, Org, isFixedLengthVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is16BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is32BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is64BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is128BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is256BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is512BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is1024BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, is2048BitVector);
-    MVTTEST_EXPECT_EQ_M(New, Org, isOverloaded);
-    if (New.isVector()) {
-      MVTTEST_EXPECT_EQ_SVT_M(New, Org, changeVectorElementTypeToInteger);
-      MVTTEST_EXPECT_EQ_SVT_M(New, Org, changeTypeToInteger);
-      if (New.getVectorElementCount().isKnownEven()) {
-        MVTTEST_EXPECT_EQ_SVT_M(New, Org, getHalfNumVectorElementsVT);
-      }
-      MVTTEST_EXPECT_EQ_M(New, Org, isPow2VectorType);
-      MVTTEST_EXPECT_EQ_SVT_M(New, Org, getPow2VectorType);
-      MVTTEST_EXPECT_EQ_SVT_M(New, Org, getVectorElementType);
-      MVTTEST_EXPECT_EQ_M(New, Org, getVectorMinNumElements);
-      MVTTEST_EXPECT_EQ_M(New, Org, getVectorElementCount);
-
-      auto n = New.getVectorMinNumElements();
-      auto sc = New.isScalableVector();
-      auto LHS = tmp::MVT::getVectorVT(New.getVectorElementType(), n, sc);
-      auto RHS = MVT::getVectorVT(Org.getVectorElementType(), n, sc);
-      MVTTEST_EXPECT_EQ_SVT(LHS, RHS);
-    } else if (New.isInteger()) {
-      auto bw = New.getSizeInBits();
-      MVTTEST_EXPECT_EQ_SVT_F(getIntegerVT, bw);
-    } else if (New.isFloatingPoint()) {
-      auto bw = New.getSizeInBits();
-      MVTTEST_EXPECT_EQ_SVT_F(getFloatingPointVT, bw);
-    }
-    MVTTEST_EXPECT_EQ_SVT_M(New, Org, getScalarType);
-    if (New.isValid()) {
-      switch (New.SimpleTy) {
-      case tmp::MVT::Other:
-      case tmp::MVT::Glue:
-      case tmp::MVT::isVoid:
-      case tmp::MVT::Untyped:
-      case tmp::MVT::spirvbuiltin:
-        break;
-      case tmp::MVT::aarch64svcount:
-        break;
-      default:
-        MVTTEST_EXPECT_EQ_M(New, Org, getSizeInBits);
-        MVTTEST_EXPECT_EQ_M(New, Org, getScalarSizeInBits);
-        MVTTEST_EXPECT_EQ_M(New, Org, getStoreSize);
-        MVTTEST_EXPECT_EQ_M(New, Org, getScalarStoreSize);
-        MVTTEST_EXPECT_EQ_M(New, Org, getStoreSizeInBits);
-        MVTTEST_EXPECT_EQ_M(New, Org, isByteSized);
-        if (!New.isScalableVector()) {
-          MVTTEST_EXPECT_EQ_M(New, Org, getFixedSizeInBits);
-        }
-        break;
-      }
-    }
-  }
-}
-} // namespace
diff --git a/llvm/unittests/Support/MachineValueType.h b/llvm/unittests/Support/MachineValueType.h
deleted file mode 100644
index 9e6f2b6cda5b9..0000000000000
--- a/llvm/unittests/Support/MachineValueType.h
+++ /dev/null
@@ -1,1130 +0,0 @@
-//===- MachineValueType.h - Machine-Level types -----------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the set of machine-level target independent types which
-// legal values in the code generator use.
-//
-// Constants and properties are defined in ValueTypes.td.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TMP_MACHINEVALUETYPE_H
-#define LLVM_TMP_MACHINEVALUETYPE_H
-
-#include "llvm/ADT/Sequence.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/TypeSize.h"
-#include <cassert>
-#include <cstdint>
-
-namespace llvm::tmp {
-
-  class Type;
-  class raw_ostream;
-
-  /// Machine Value Type. Every type that is supported natively by some
-  /// processor targeted by LLVM occurs here. This means that any legal value
-  /// type can be represented by an MVT.
-  class MVT {
-  public:
-    enum SimpleValueType : uint8_t {
-      // Simple value types that aren't explicitly part of this enumeration
-      // are considered extended value types.
-      INVALID_SIMPLE_VALUE_TYPE = 0,
-
-#define GET_VT_ATTR(Ty, n, sz) Ty = n,
-#define GET_VT_RANGES
-#include "GenVT.inc"
-#undef GET_VT_ATTR
-#undef GET_VT_RANGES
-
-      FIRST_INTEGER_VALUETYPE = i1,
-      LAST_INTEGER_VALUETYPE  = i128,
-
-      FIRST_FP_VALUETYPE = bf16,
-      LAST_FP_VALUETYPE  = ppcf128,
-
-      FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
-      LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i128,
-
-      FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE = v1f16,
-      LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v256f64,
-
-      FIRST_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
-      LAST_FIXEDLEN_VECTOR_VALUETYPE = v256f64,
-
-      FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
-      LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv32i64,
-
-      FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv1f16,
-      LAST_FP_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
-
-      FIRST_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
-      LAST_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
-
-      FIRST_VECTOR_VALUETYPE = v1i1,
-      LAST_VECTOR_VALUETYPE  = nxv8f64,
-
-      VALUETYPE_SIZE = LAST_VALUETYPE + 1,
-
-      // This is the current maximum for LAST_VALUETYPE.
-      // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
-      // This value must be a multiple of 32.
-      MAX_ALLOWED_VALUETYPE = 224,
-    };
-
-    static_assert(FIRST_VALUETYPE > 0);
-    static_assert(LAST_VALUETYPE < MAX_ALLOWED_VALUETYPE);
-
-    SimpleValueType SimpleTy = INVALID_SIMPLE_VALUE_TYPE;
-
-    constexpr MVT() = default;
-    constexpr MVT(SimpleValueType SVT) : SimpleTy(SVT) {}
-
-    bool operator>(const MVT& S)  const { return SimpleTy >  S.SimpleTy; }
-    bool operator<(const MVT& S)  const { return SimpleTy <  S.SimpleTy; }
-    bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; }
-    bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; }
-    bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; }
-    bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; }
-
-    /// Support for debugging, callable in GDB: VT.dump()
-    void dump() const;
-
-    /// Implement operator<<.
-    void print(raw_ostream &OS) const;
-
-    /// Return true if this is a valid simple valuetype.
-    bool isValid() const {
-      return (SimpleTy >= MVT::FIRST_VALUETYPE &&
-              SimpleTy <= MVT::LAST_VALUETYPE);
-    }
-
-    /// Return true if this is a FP or a vector FP type.
-    bool isFloatingPoint() const {
-      return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE));
-    }
-
-    /// Return true if this is an integer or a vector integer type.
-    bool isInteger() const {
-      return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE));
-    }
-
-    /// Return true if this is an integer, not including vectors.
-    bool isScalarInteger() const {
-      return (SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
-              SimpleTy <= MVT::LAST_INTEGER_VALUETYPE);
-    }
-
-    /// Return true if this is a vector value type.
-    bool isVector() const {
-      return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_VECTOR_VALUETYPE);
-    }
-
-    /// Return true if this is a vector value type where the
-    /// runtime length is machine dependent
-    bool isScalableVector() const {
-      return (SimpleTy >= MVT::FIRST_SCALABLE_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_SCALABLE_VECTOR_VALUETYPE);
-    }
-
-    /// Return true if this is a custom target type that has a scalable size.
-    bool isScalableTargetExtVT() const {
-      return SimpleTy == MVT::aarch64svcount;
-    }
-
-    /// Return true if the type is a scalable type.
-    bool isScalableVT() const {
-      return isScalableVector() || isScalableTargetExtVT();
-    }
-
-    bool isFixedLengthVector() const {
-      return (SimpleTy >= MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE);
-    }
-
-    /// Return true if this is a 16-bit vector type.
-    bool is16BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 16);
-    }
-
-    /// Return true if this is a 32-bit vector type.
-    bool is32BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 32);
-    }
-
-    /// Return true if this is a 64-bit vector type.
-    bool is64BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 64);
-    }
-
-    /// Return true if this is a 128-bit vector type.
-    bool is128BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 128);
-    }
-
-    /// Return true if this is a 256-bit vector type.
-    bool is256BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 256);
-    }
-
-    /// Return true if this is a 512-bit vector type.
-    bool is512BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 512);
-    }
-
-    /// Return true if this is a 1024-bit vector type.
-    bool is1024BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 1024);
-    }
-
-    /// Return true if this is a 2048-bit vector type.
-    bool is2048BitVector() const {
-      return (isFixedLengthVector() && getFixedSizeInBits() == 2048);
-    }
-
-    /// Return true if this is an overloaded type for TableGen.
-    bool isOverloaded() const {
-      return (SimpleTy == MVT::Any || SimpleTy == MVT::iAny ||
-              SimpleTy == MVT::fAny || SimpleTy == MVT::vAny ||
-              SimpleTy == MVT::iPTRAny);
-    }
-
-    /// Return a vector with the same number of elements as this vector, but
-    /// with the element type converted to an integer type with the same
-    /// bitwidth.
-    MVT changeVectorElementTypeToInteger() const {
-      MVT EltTy = getVectorElementType();
-      MVT IntTy = MVT::getIntegerVT(EltTy.getSizeInBits());
-      MVT VecTy = MVT::getVectorVT(IntTy, getVectorElementCount());
-      assert(VecTy.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
-             "Simple vector VT not representable by simple integer vector VT!");
-      return VecTy;
-    }
-
-    /// Return a VT for a vector type whose attributes match ourselves
-    /// with the exception of the element type that is chosen by the caller.
-    MVT changeVectorElementType(MVT EltVT) const {
-      MVT VecTy = MVT::getVectorVT(EltVT, getVectorElementCount());
-      assert(VecTy.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE &&
-             "Simple vector VT not representable by simple integer vector VT!");
-      return VecTy;
-    }
-
-    /// Return the type converted to an equivalently sized integer or vector
-    /// with integer element type. Similar to changeVectorElementTypeToInteger,
-    /// but also handles scalars.
-    MVT changeTypeToInteger() {
-      if (isVector())
-        return changeVectorElementTypeToInteger();
-      return MVT::getIntegerVT(getSizeInBits());
-    }
-
-    /// Return a VT for a vector type with the same element type but
-    /// half the number of elements.
-    MVT getHalfNumVectorElementsVT() const {
-      MVT EltVT = getVectorElementType();
-      auto EltCnt = getVectorElementCount();
-      assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!");
-      return getVectorVT(EltVT, EltCnt.divideCoefficientBy(2));
-    }
-
-    /// Returns true if the given vector is a power of 2.
-    bool isPow2VectorType() const {
-      unsigned NElts = getVectorMinNumElements();
-      return !(NElts & (NElts - 1));
-    }
-
-    /// Widens the length of the given vector MVT up to the nearest power of 2
-    /// and returns that type.
-    MVT getPow2VectorType() const {
-      if (isPow2VectorType())
-        return *this;
-
-      ElementCount NElts = getVectorElementCount();
-      unsigned NewMinCount = 1 << Log2_32_Ceil(NElts.getKnownMinValue());
-      NElts = ElementCount::get(NewMinCount, NElts.isScalable());
-      return MVT::getVectorVT(getVectorElementType(), NElts);
-    }
-
-    /// If this is a vector, return the element type, otherwise return this.
-    MVT getScalarType() const {
-      return isVector() ? getVectorElementType() : *this;
-    }
-
-    MVT getVectorElementType() const {
-      // clang-format off
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("Not a vector MVT!");
-      case v1i1:
-      case v2i1:
-      case v4i1:
-      case v8i1:
-      case v16i1:
-      case v32i1:
-      case v64i1:
-      case v128i1:
-      case v256i1:
-      case v512i1:
-      case v1024i1:
-      case v2048i1:
-      case nxv1i1:
-      case nxv2i1:
-      case nxv4i1:
-      case nxv8i1:
-      case nxv16i1:
-      case nxv32i1:
-      case nxv64i1: return i1;
-      case v128i2:
-      case v256i2: return i2;
-      case v64i4:
-      case v128i4: return i4;
-      case v1i8:
-      case v2i8:
-      case v4i8:
-      case v8i8:
-      case v16i8:
-      case v32i8:
-      case v64i8:
-      case v128i8:
-      case v256i8:
-      case v512i8:
-      case v1024i8:
-      case nxv1i8:
-      case nxv2i8:
-      case nxv4i8:
-      case nxv8i8:
-      case nxv16i8:
-      case nxv32i8:
-      case nxv64i8: return i8;
-      case v1i16:
-      case v2i16:
-      case v3i16:
-      case v4i16:
-      case v8i16:
-      case v16i16:
-      case v32i16:
-      case v64i16:
-      case v128i16:
-      case v256i16:
-      case v512i16:
-      case nxv1i16:
-      case nxv2i16:
-      case nxv4i16:
-      case nxv8i16:
-      case nxv16i16:
-      case nxv32i16: return i16;
-      case v1i32:
-      case v2i32:
-      case v3i32:
-      case v4i32:
-      case v5i32:
-      case v6i32:
-      case v7i32:
-      case v8i32:
-      case v9i32:
-      case v10i32:
-      case v11i32:
-      case v12i32:
-      case v16i32:
-      case v32i32:
-      case v64i32:
-      case v128i32:
-      case v256i32:
-      case v512i32:
-      case v1024i32:
-      case v2048i32:
-      case nxv1i32:
-      case nxv2i32:
-      case nxv4i32:
-      case nxv8i32:
-      case nxv16i32:
-      case nxv32i32: return i32;
-      case v1i64:
-      case v2i64:
-      case v3i64:
-      case v4i64:
-      case v8i64:
-      case v16i64:
-      case v32i64:
-      case v64i64:
-      case v128i64:
-      case v256i64:
-      case nxv1i64:
-      case nxv2i64:
-      case nxv4i64:
-      case nxv8i64:
-      case nxv16i64:
-      case nxv32i64: return i64;
-      case v1i128: return i128;
-      case v1f16:
-      case v2f16:
-      case v3f16:
-      case v4f16:
-      case v8f16:
-      case v16f16:
-      case v32f16:
-      case v64f16:
-      case v128f16:
-      case v256f16:
-      case v512f16:
-      case nxv1f16:
-      case nxv2f16:
-      case nxv4f16:
-      case nxv8f16:
-      case nxv16f16:
-      case nxv32f16: return f16;
-      case v2bf16:
-      case v3bf16:
-      case v4bf16:
-      case v8bf16:
-      case v16bf16:
-      case v32bf16:
-      case v64bf16:
-      case v128bf16:
-      case nxv1bf16:
-      case nxv2bf16:
-      case nxv4bf16:
-      case nxv8bf16:
-      case nxv16bf16:
-      case nxv32bf16: return bf16;
-      case v1f32:
-      case v2f32:
-      case v3f32:
-      case v4f32:
-      case v5f32:
-      case v6f32:
-      case v7f32:
-      case v8f32:
-      case v9f32:
-      case v10f32:
-      case v11f32:
-      case v12f32:
-      case v16f32:
-      case v32f32:
-      case v64f32:
-      case v128f32:
-      case v256f32:
-      case v512f32:
-      case v1024f32:
-      case v2048f32:
-      case nxv1f32:
-      case nxv2f32:
-      case nxv4f32:
-      case nxv8f32:
-      case nxv16f32: return f32;
-      case v1f64:
-      case v2f64:
-      case v3f64:
-      case v4f64:
-      case v8f64:
-      case v16f64:
-      case v32f64:
-      case v64f64:
-      case v128f64:
-      case v256f64:
-      case nxv1f64:
-      case nxv2f64:
-      case nxv4f64:
-      case nxv8f64: return f64;
-      }
-      // clang-format on
-    }
-
-    /// Given a vector type, return the minimum number of elements it contains.
-    unsigned getVectorMinNumElements() const {
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("Not a vector MVT!");
-      case v2048i1:
-      case v2048i32:
-      case v2048f32: return 2048;
-      case v1024i1:
-      case v1024i8:
-      case v1024i32:
-      case v1024f32: return 1024;
-      case v512i1:
-      case v512i8:
-      case v512i16:
-      case v512i32:
-      case v512f16:
-      case v512f32: return 512;
-      case v256i1:
-      case v256i2:
-      case v256i8:
-      case v256i16:
-      case v256f16:
-      case v256i32:
-      case v256i64:
-      case v256f32:
-      case v256f64: return 256;
-      case v128i1:
-      case v128i2:
-      case v128i4:
-      case v128i8:
-      case v128i16:
-      case v128i32:
-      case v128i64:
-      case v128f16:
-      case v128bf16:
-      case v128f32:
-      case v128f64: return 128;
-      case v64i1:
-      case v64i4:
-      case v64i8:
-      case v64i16:
-      case v64i32:
-      case v64i64:
-      case v64f16:
-      case v64bf16:
-      case v64f32:
-      case v64f64:
-      case nxv64i1:
-      case nxv64i8: return 64;
-      case v32i1:
-      case v32i8:
-      case v32i16:
-      case v32i32:
-      case v32i64:
-      case v32f16:
-      case v32bf16:
-      case v32f32:
-      case v32f64:
-      case nxv32i1:
-      case nxv32i8:
-      case nxv32i16:
-      case nxv32i32:
-      case nxv32i64:
-      case nxv32f16:
-      case nxv32bf16: return 32;
-      case v16i1:
-      case v16i8:
-      case v16i16:
-      case v16i32:
-      case v16i64:
-      case v16f16:
-      case v16bf16:
-      case v16f32:
-      case v16f64:
-      case nxv16i1:
-      case nxv16i8:
-      case nxv16i16:
-      case nxv16i32:
-      case nxv16i64:
-      case nxv16f16:
-      case nxv16bf16:
-      case nxv16f32: return 16;
-      case v12i32:
-      case v12f32: return 12;
-      case v11i32:
-      case v11f32: return 11;
-      case v10i32:
-      case v10f32: return 10;
-      case v9i32:
-      case v9f32: return 9;
-      case v8i1:
-      case v8i8:
-      case v8i16:
-      case v8i32:
-      case v8i64:
-      case v8f16:
-      case v8bf16:
-      case v8f32:
-      case v8f64:
-      case nxv8i1:
-      case nxv8i8:
-      case nxv8i16:
-      case nxv8i32:
-      case nxv8i64:
-      case nxv8f16:
-      case nxv8bf16:
-      case nxv8f32:
-      case nxv8f64: return 8;
-      case v7i32:
-      case v7f32: return 7;
-      case v6i32:
-      case v6f32: return 6;
-      case v5i32:
-      case v5f32: return 5;
-      case v4i1:
-      case v4i8:
-      case v4i16:
-      case v4i32:
-      case v4i64:
-      case v4f16:
-      case v4bf16:
-      case v4f32:
-      case v4f64:
-      case nxv4i1:
-      case nxv4i8:
-      case nxv4i16:
-      case nxv4i32:
-      case nxv4i64:
-      case nxv4f16:
-      case nxv4bf16:
-      case nxv4f32:
-      case nxv4f64: return 4;
-      case v3i16:
-      case v3i32:
-      case v3i64:
-      case v3f16:
-      case v3bf16:
-      case v3f32:
-      case v3f64: return 3;
-      case v2i1:
-      case v2i8:
-      case v2i16:
-      case v2i32:
-      case v2i64:
-      case v2f16:
-      case v2bf16:
-      case v2f32:
-      case v2f64:
-      case nxv2i1:
-      case nxv2i8:
-      case nxv2i16:
-      case nxv2i32:
-      case nxv2i64:
-      case nxv2f16:
-      case nxv2bf16:
-      case nxv2f32:
-      case nxv2f64: return 2;
-      case v1i1:
-      case v1i8:
-      case v1i16:
-      case v1i32:
-      case v1i64:
-      case v1i128:
-      case v1f16:
-      case v1f32:
-      case v1f64:
-      case nxv1i1:
-      case nxv1i8:
-      case nxv1i16:
-      case nxv1i32:
-      case nxv1i64:
-      case nxv1f16:
-      case nxv1bf16:
-      case nxv1f32:
-      case nxv1f64: return 1;
-      }
-    }
-
-    ElementCount getVectorElementCount() const {
-      return ElementCount::get(getVectorMinNumElements(), isScalableVector());
-    }
-
-    unsigned getVectorNumElements() const {
-      if (isScalableVector())
-        llvm::reportInvalidSizeRequest(
-            "Possible incorrect use of MVT::getVectorNumElements() for "
-            "scalable vector. Scalable flag may be dropped, use "
-            "MVT::getVectorElementCount() instead");
-      return getVectorMinNumElements();
-    }
-
-    /// Returns the size of the specified MVT in bits.
-    ///
-    /// If the value type is a scalable vector type, the scalable property will
-    /// be set and the runtime size will be a positive integer multiple of the
-    /// base size.
-    TypeSize getSizeInBits() const {
-      switch (SimpleTy) {
-      default:
-        switch (SimpleTy) {
-        default:
-          llvm_unreachable("getSizeInBits called on extended MVT.");
-
-#define GET_VT_ATTR(Ty, N, Sz)                                                 \
-  case Ty:                                                                     \
-    return (MVT(Ty).isScalableVector() ? TypeSize::Scalable(Sz)                \
-                                       : TypeSize::Fixed(Sz));
-#include "GenVT.inc"
-#undef GET_VT_ATTR
-        }
-      case Other:
-        llvm_unreachable("Value type is non-standard value, Other.");
-      case iPTR:
-        llvm_unreachable("Value type size is target-dependent. Ask TLI.");
-      case iPTRAny:
-      case iAny:
-      case fAny:
-      case vAny:
-      case Any:
-        llvm_unreachable("Value type is overloaded.");
-      case token:
-        llvm_unreachable("Token type is a sentinel that cannot be used "
-                         "in codegen and has no size");
-      case Metadata:
-        llvm_unreachable("Value type is metadata.");
-      }
-    }
-
-    /// Return the size of the specified fixed width value type in bits. The
-    /// function will assert if the type is scalable.
-    uint64_t getFixedSizeInBits() const {
-      return getSizeInBits().getFixedValue();
-    }
-
-    uint64_t getScalarSizeInBits() const {
-      return getScalarType().getSizeInBits().getFixedValue();
-    }
-
-    /// Return the number of bytes overwritten by a store of the specified value
-    /// type.
-    ///
-    /// If the value type is a scalable vector type, the scalable property will
-    /// be set and the runtime size will be a positive integer multiple of the
-    /// base size.
-    TypeSize getStoreSize() const {
-      TypeSize BaseSize = getSizeInBits();
-      return {(BaseSize.getKnownMinValue() + 7) / 8, BaseSize.isScalable()};
-    }
-
-    // Return the number of bytes overwritten by a store of this value type or
-    // this value type's element type in the case of a vector.
-    uint64_t getScalarStoreSize() const {
-      return getScalarType().getStoreSize().getFixedValue();
-    }
-
-    /// Return the number of bits overwritten by a store of the specified value
-    /// type.
-    ///
-    /// If the value type is a scalable vector type, the scalable property will
-    /// be set and the runtime size will be a positive integer multiple of the
-    /// base size.
-    TypeSize getStoreSizeInBits() const {
-      return getStoreSize() * 8;
-    }
-
-    /// Returns true if the number of bits for the type is a multiple of an
-    /// 8-bit byte.
-    bool isByteSized() const { return getSizeInBits().isKnownMultipleOf(8); }
-
-    /// Return true if we know at compile time this has more bits than VT.
-    bool knownBitsGT(MVT VT) const {
-      return TypeSize::isKnownGT(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if we know at compile time this has more than or the same
-    /// bits as VT.
-    bool knownBitsGE(MVT VT) const {
-      return TypeSize::isKnownGE(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if we know at compile time this has fewer bits than VT.
-    bool knownBitsLT(MVT VT) const {
-      return TypeSize::isKnownLT(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if we know at compile time this has fewer than or the same
-    /// bits as VT.
-    bool knownBitsLE(MVT VT) const {
-      return TypeSize::isKnownLE(getSizeInBits(), VT.getSizeInBits());
-    }
-
-    /// Return true if this has more bits than VT.
-    bool bitsGT(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsGT(VT);
-    }
-
-    /// Return true if this has no less bits than VT.
-    bool bitsGE(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsGE(VT);
-    }
-
-    /// Return true if this has less bits than VT.
-    bool bitsLT(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsLT(VT);
-    }
-
-    /// Return true if this has no more bits than VT.
-    bool bitsLE(MVT VT) const {
-      assert(isScalableVector() == VT.isScalableVector() &&
-             "Comparison between scalable and fixed types");
-      return knownBitsLE(VT);
-    }
-
-    static MVT getFloatingPointVT(unsigned BitWidth) {
-      switch (BitWidth) {
-      default:
-        llvm_unreachable("Bad bit width!");
-      case 16:
-        return MVT::f16;
-      case 32:
-        return MVT::f32;
-      case 64:
-        return MVT::f64;
-      case 80:
-        return MVT::f80;
-      case 128:
-        return MVT::f128;
-      }
-    }
-
-    static MVT getIntegerVT(unsigned BitWidth) {
-      switch (BitWidth) {
-      default:
-        return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-      case 1:
-        return MVT::i1;
-      case 2:
-        return MVT::i2;
-      case 4:
-        return MVT::i4;
-      case 8:
-        return MVT::i8;
-      case 16:
-        return MVT::i16;
-      case 32:
-        return MVT::i32;
-      case 64:
-        return MVT::i64;
-      case 128:
-        return MVT::i128;
-      }
-    }
-
-    static MVT getVectorVT(MVT VT, unsigned NumElements) {
-      // clang-format off
-      switch (VT.SimpleTy) {
-      default:
-        break;
-      case MVT::i1:
-        if (NumElements == 1)    return MVT::v1i1;
-        if (NumElements == 2)    return MVT::v2i1;
-        if (NumElements == 4)    return MVT::v4i1;
-        if (NumElements == 8)    return MVT::v8i1;
-        if (NumElements == 16)   return MVT::v16i1;
-        if (NumElements == 32)   return MVT::v32i1;
-        if (NumElements == 64)   return MVT::v64i1;
-        if (NumElements == 128)  return MVT::v128i1;
-        if (NumElements == 256)  return MVT::v256i1;
-        if (NumElements == 512)  return MVT::v512i1;
-        if (NumElements == 1024) return MVT::v1024i1;
-        if (NumElements == 2048) return MVT::v2048i1;
-        break;
-      case MVT::i2:
-        if (NumElements == 128) return MVT::v128i2;
-        if (NumElements == 256) return MVT::v256i2;
-        break;
-      case MVT::i4:
-        if (NumElements == 64)  return MVT::v64i4;
-        if (NumElements == 128) return MVT::v128i4;
-        break;
-      case MVT::i8:
-        if (NumElements == 1)   return MVT::v1i8;
-        if (NumElements == 2)   return MVT::v2i8;
-        if (NumElements == 4)   return MVT::v4i8;
-        if (NumElements == 8)   return MVT::v8i8;
-        if (NumElements == 16)  return MVT::v16i8;
-        if (NumElements == 32)  return MVT::v32i8;
-        if (NumElements == 64)  return MVT::v64i8;
-        if (NumElements == 128) return MVT::v128i8;
-        if (NumElements == 256) return MVT::v256i8;
-        if (NumElements == 512) return MVT::v512i8;
-        if (NumElements == 1024) return MVT::v1024i8;
-        break;
-      case MVT::i16:
-        if (NumElements == 1)   return MVT::v1i16;
-        if (NumElements == 2)   return MVT::v2i16;
-        if (NumElements == 3)   return MVT::v3i16;
-        if (NumElements == 4)   return MVT::v4i16;
-        if (NumElements == 8)   return MVT::v8i16;
-        if (NumElements == 16)  return MVT::v16i16;
-        if (NumElements == 32)  return MVT::v32i16;
-        if (NumElements == 64)  return MVT::v64i16;
-        if (NumElements == 128) return MVT::v128i16;
-        if (NumElements == 256) return MVT::v256i16;
-        if (NumElements == 512) return MVT::v512i16;
-        break;
-      case MVT::i32:
-        if (NumElements == 1)    return MVT::v1i32;
-        if (NumElements == 2)    return MVT::v2i32;
-        if (NumElements == 3)    return MVT::v3i32;
-        if (NumElements == 4)    return MVT::v4i32;
-        if (NumElements == 5)    return MVT::v5i32;
-        if (NumElements == 6)    return MVT::v6i32;
-        if (NumElements == 7)    return MVT::v7i32;
-        if (NumElements == 8)    return MVT::v8i32;
-        if (NumElements == 9)    return MVT::v9i32;
-        if (NumElements == 10)   return MVT::v10i32;
-        if (NumElements == 11)   return MVT::v11i32;
-        if (NumElements == 12)   return MVT::v12i32;
-        if (NumElements == 16)   return MVT::v16i32;
-        if (NumElements == 32)   return MVT::v32i32;
-        if (NumElements == 64)   return MVT::v64i32;
-        if (NumElements == 128)  return MVT::v128i32;
-        if (NumElements == 256)  return MVT::v256i32;
-        if (NumElements == 512)  return MVT::v512i32;
-        if (NumElements == 1024) return MVT::v1024i32;
-        if (NumElements == 2048) return MVT::v2048i32;
-        break;
-      case MVT::i64:
-        if (NumElements == 1)  return MVT::v1i64;
-        if (NumElements == 2)  return MVT::v2i64;
-        if (NumElements == 3)  return MVT::v3i64;
-        if (NumElements == 4)  return MVT::v4i64;
-        if (NumElements == 8)  return MVT::v8i64;
-        if (NumElements == 16) return MVT::v16i64;
-        if (NumElements == 32) return MVT::v32i64;
-        if (NumElements == 64) return MVT::v64i64;
-        if (NumElements == 128) return MVT::v128i64;
-        if (NumElements == 256) return MVT::v256i64;
-        break;
-      case MVT::i128:
-        if (NumElements == 1)  return MVT::v1i128;
-        break;
-      case MVT::f16:
-        if (NumElements == 1)   return MVT::v1f16;
-        if (NumElements == 2)   return MVT::v2f16;
-        if (NumElements == 3)   return MVT::v3f16;
-        if (NumElements == 4)   return MVT::v4f16;
-        if (NumElements == 8)   return MVT::v8f16;
-        if (NumElements == 16)  return MVT::v16f16;
-        if (NumElements == 32)  return MVT::v32f16;
-        if (NumElements == 64)  return MVT::v64f16;
-        if (NumElements == 128) return MVT::v128f16;
-        if (NumElements == 256) return MVT::v256f16;
-        if (NumElements == 512) return MVT::v512f16;
-        break;
-      case MVT::bf16:
-        if (NumElements == 2)   return MVT::v2bf16;
-        if (NumElements == 3)   return MVT::v3bf16;
-        if (NumElements == 4)   return MVT::v4bf16;
-        if (NumElements == 8)   return MVT::v8bf16;
-        if (NumElements == 16)  return MVT::v16bf16;
-        if (NumElements == 32)  return MVT::v32bf16;
-        if (NumElements == 64)  return MVT::v64bf16;
-        if (NumElements == 128) return MVT::v128bf16;
-        break;
-      case MVT::f32:
-        if (NumElements == 1)    return MVT::v1f32;
-        if (NumElements == 2)    return MVT::v2f32;
-        if (NumElements == 3)    return MVT::v3f32;
-        if (NumElements == 4)    return MVT::v4f32;
-        if (NumElements == 5)    return MVT::v5f32;
-        if (NumElements == 6)    return MVT::v6f32;
-        if (NumElements == 7)    return MVT::v7f32;
-        if (NumElements == 8)    return MVT::v8f32;
-        if (NumElements == 9)    return MVT::v9f32;
-        if (NumElements == 10)   return MVT::v10f32;
-        if (NumElements == 11)   return MVT::v11f32;
-        if (NumElements == 12)   return MVT::v12f32;
-        if (NumElements == 16)   return MVT::v16f32;
-        if (NumElements == 32)   return MVT::v32f32;
-        if (NumElements == 64)   return MVT::v64f32;
-        if (NumElements == 128)  return MVT::v128f32;
-        if (NumElements == 256)  return MVT::v256f32;
-        if (NumElements == 512)  return MVT::v512f32;
-        if (NumElements == 1024) return MVT::v1024f32;
-        if (NumElements == 2048) return MVT::v2048f32;
-        break;
-      case MVT::f64:
-        if (NumElements == 1)  return MVT::v1f64;
-        if (NumElements == 2)  return MVT::v2f64;
-        if (NumElements == 3)  return MVT::v3f64;
-        if (NumElements == 4)  return MVT::v4f64;
-        if (NumElements == 8)  return MVT::v8f64;
-        if (NumElements == 16) return MVT::v16f64;
-        if (NumElements == 32) return MVT::v32f64;
-        if (NumElements == 64) return MVT::v64f64;
-        if (NumElements == 128) return MVT::v128f64;
-        if (NumElements == 256) return MVT::v256f64;
-        break;
-      }
-      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-      // clang-format on
-    }
-
-    static MVT getScalableVectorVT(MVT VT, unsigned NumElements) {
-      switch(VT.SimpleTy) {
-        default:
-          break;
-        case MVT::i1:
-          if (NumElements == 1)  return MVT::nxv1i1;
-          if (NumElements == 2)  return MVT::nxv2i1;
-          if (NumElements == 4)  return MVT::nxv4i1;
-          if (NumElements == 8)  return MVT::nxv8i1;
-          if (NumElements == 16) return MVT::nxv16i1;
-          if (NumElements == 32) return MVT::nxv32i1;
-          if (NumElements == 64) return MVT::nxv64i1;
-          break;
-        case MVT::i8:
-          if (NumElements == 1)  return MVT::nxv1i8;
-          if (NumElements == 2)  return MVT::nxv2i8;
-          if (NumElements == 4)  return MVT::nxv4i8;
-          if (NumElements == 8)  return MVT::nxv8i8;
-          if (NumElements == 16) return MVT::nxv16i8;
-          if (NumElements == 32) return MVT::nxv32i8;
-          if (NumElements == 64) return MVT::nxv64i8;
-          break;
-        case MVT::i16:
-          if (NumElements == 1)  return MVT::nxv1i16;
-          if (NumElements == 2)  return MVT::nxv2i16;
-          if (NumElements == 4)  return MVT::nxv4i16;
-          if (NumElements == 8)  return MVT::nxv8i16;
-          if (NumElements == 16) return MVT::nxv16i16;
-          if (NumElements == 32) return MVT::nxv32i16;
-          break;
-        case MVT::i32:
-          if (NumElements == 1)  return MVT::nxv1i32;
-          if (NumElements == 2)  return MVT::nxv2i32;
-          if (NumElements == 4)  return MVT::nxv4i32;
-          if (NumElements == 8)  return MVT::nxv8i32;
-          if (NumElements == 16) return MVT::nxv16i32;
-          if (NumElements == 32) return MVT::nxv32i32;
-          break;
-        case MVT::i64:
-          if (NumElements == 1)  return MVT::nxv1i64;
-          if (NumElements == 2)  return MVT::nxv2i64;
-          if (NumElements == 4)  return MVT::nxv4i64;
-          if (NumElements == 8)  return MVT::nxv8i64;
-          if (NumElements == 16) return MVT::nxv16i64;
-          if (NumElements == 32) return MVT::nxv32i64;
-          break;
-        case MVT::f16:
-          if (NumElements == 1)  return MVT::nxv1f16;
-          if (NumElements == 2)  return MVT::nxv2f16;
-          if (NumElements == 4)  return MVT::nxv4f16;
-          if (NumElements == 8)  return MVT::nxv8f16;
-          if (NumElements == 16)  return MVT::nxv16f16;
-          if (NumElements == 32)  return MVT::nxv32f16;
-          break;
-        case MVT::bf16:
-          if (NumElements == 1)  return MVT::nxv1bf16;
-          if (NumElements == 2)  return MVT::nxv2bf16;
-          if (NumElements == 4)  return MVT::nxv4bf16;
-          if (NumElements == 8)  return MVT::nxv8bf16;
-          if (NumElements == 16)  return MVT::nxv16bf16;
-          if (NumElements == 32)  return MVT::nxv32bf16;
-          break;
-        case MVT::f32:
-          if (NumElements == 1)  return MVT::nxv1f32;
-          if (NumElements == 2)  return MVT::nxv2f32;
-          if (NumElements == 4)  return MVT::nxv4f32;
-          if (NumElements == 8)  return MVT::nxv8f32;
-          if (NumElements == 16) return MVT::nxv16f32;
-          break;
-        case MVT::f64:
-          if (NumElements == 1)  return MVT::nxv1f64;
-          if (NumElements == 2)  return MVT::nxv2f64;
-          if (NumElements == 4)  return MVT::nxv4f64;
-          if (NumElements == 8)  return MVT::nxv8f64;
-          break;
-      }
-      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-    }
-
-    static MVT getVectorVT(MVT VT, unsigned NumElements, bool IsScalable) {
-      if (IsScalable)
-        return getScalableVectorVT(VT, NumElements);
-      return getVectorVT(VT, NumElements);
-    }
-
-    static MVT getVectorVT(MVT VT, ElementCount EC) {
-      if (EC.isScalable())
-        return getScalableVectorVT(VT, EC.getKnownMinValue());
-      return getVectorVT(VT, EC.getKnownMinValue());
-    }
-
-    /// Return the value type corresponding to the specified type.  This returns
-    /// all pointers as iPTR.  If HandleUnknown is true, unknown types are
-    /// returned as Other, otherwise they are invalid.
-    static MVT getVT(Type *Ty, bool HandleUnknown = false);
-
-  public:
-    /// SimpleValueType Iteration
-    /// @{
-    static auto all_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_VALUETYPE, MVT::LAST_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto integer_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_INTEGER_VALUETYPE,
-                                MVT::LAST_INTEGER_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fp_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FP_VALUETYPE, MVT::LAST_FP_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_VECTOR_VALUETYPE,
-                                MVT::LAST_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fixedlen_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE,
-                                MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto scalable_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_SCALABLE_VECTOR_VALUETYPE,
-                                MVT::LAST_SCALABLE_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto integer_fixedlen_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
-                                MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fp_fixedlen_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE,
-                                MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto integer_scalable_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
-                                MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-
-    static auto fp_scalable_vector_valuetypes() {
-      return enum_seq_inclusive(MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE,
-                                MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE,
-                                force_iteration_on_noniterable_enum);
-    }
-    /// @}
-  };
-
-  inline raw_ostream &operator<<(raw_ostream &OS, const MVT &VT) {
-    VT.print(OS);
-    return OS;
-  }
-
-} // namespace llvm::tmp
-
-#endif // LLVM_TMP_MACHINEVALUETYPE_H
diff --git a/llvm/unittests/Support/ParallelTest.cpp b/llvm/unittests/Support/ParallelTest.cpp
index bbf27dfab437e..53ef9fa25e826 100644
--- a/llvm/unittests/Support/ParallelTest.cpp
+++ b/llvm/unittests/Support/ParallelTest.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/Parallel.h"
+#include "llvm/Support/ThreadPool.h"
 #include "gtest/gtest.h"
 #include <array>
 #include <random>
@@ -92,4 +93,86 @@ TEST(Parallel, ForEachError) {
   EXPECT_EQ(errText, std::string("asdf\nasdf\nasdf"));
 }
 
+TEST(Parallel, TaskGroupSequentialFor) {
+  size_t Count = 0;
+  {
+    parallel::TaskGroup tg;
+    for (size_t Idx = 0; Idx < 500; Idx++)
+      tg.spawn([&Count, Idx]() { EXPECT_EQ(Count++, Idx); }, true);
+  }
+  EXPECT_EQ(Count, 500ul);
+}
+
+#if LLVM_ENABLE_THREADS
+TEST(Parallel, NestedTaskGroup) {
+  // This test checks:
+  // 1. Root TaskGroup is in Parallel mode.
+  // 2. Nested TaskGroup is not in Parallel mode.
+  parallel::TaskGroup tg;
+
+  tg.spawn([&]() {
+    EXPECT_TRUE(tg.isParallel() || (parallel::strategy.ThreadsRequested == 1));
+  });
+
+  tg.spawn([&]() {
+    parallel::TaskGroup nestedTG;
+    EXPECT_FALSE(nestedTG.isParallel());
+
+    nestedTG.spawn([&]() {
+      // Check that root TaskGroup is in Parallel mode.
+      EXPECT_TRUE(tg.isParallel() ||
+                  (parallel::strategy.ThreadsRequested == 1));
+
+      // Check that nested TaskGroup is not in Parallel mode.
+      EXPECT_FALSE(nestedTG.isParallel());
+    });
+  });
+}
+
+TEST(Parallel, ParallelNestedTaskGroup) {
+  // This test checks that it is possible to have several TaskGroups
+  // run from different threads in Parallel mode.
+  std::atomic<size_t> Count{0};
+
+  {
+    std::function<void()> Fn = [&]() {
+      parallel::TaskGroup tg;
+
+      tg.spawn([&]() {
+        // Check that root TaskGroup is in Parallel mode.
+        EXPECT_TRUE(tg.isParallel() ||
+                    (parallel::strategy.ThreadsRequested == 1));
+
+        // Check that nested TaskGroup is not in Parallel mode.
+        parallel::TaskGroup nestedTG;
+        EXPECT_FALSE(nestedTG.isParallel());
+        ++Count;
+
+        nestedTG.spawn([&]() {
+          // Check that root TaskGroup is in Parallel mode.
+          EXPECT_TRUE(tg.isParallel() ||
+                      (parallel::strategy.ThreadsRequested == 1));
+
+          // Check that nested TaskGroup is not in Parallel mode.
+          EXPECT_FALSE(nestedTG.isParallel());
+          ++Count;
+        });
+      });
+    };
+
+    ThreadPool Pool;
+
+    Pool.async(Fn);
+    Pool.async(Fn);
+    Pool.async(Fn);
+    Pool.async(Fn);
+    Pool.async(Fn);
+    Pool.async(Fn);
+
+    Pool.wait();
+  }
+  EXPECT_EQ(Count, 12ul);
+}
+#endif
+
 #endif
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index d31674a747347..e1b3ab693729c 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -526,6 +527,33 @@ TEST(VirtualFileSystemTest, MultipleWorkingDirs) {
   ASSERT_EQ(CIt, vfs::directory_iterator());
 }
 
+TEST(VirtualFileSystemTest, PhysicalFileSystemWorkingDirFailure) {
+  TempDir D2("d2", /*Unique*/ true);
+  SmallString<128> WD, PrevWD;
+  ASSERT_EQ(sys::fs::current_path(PrevWD), std::error_code());
+  ASSERT_EQ(sys::fs::createUniqueDirectory("d1", WD), std::error_code());
+  ASSERT_EQ(sys::fs::set_current_path(WD), std::error_code());
+  auto Restore =
+      llvm::make_scope_exit([&] { sys::fs::set_current_path(PrevWD); });
+
+  // Delete the working directory to create an error.
+  if (sys::fs::remove_directories(WD, /*IgnoreErrors=*/false))
+    // Some platforms (e.g. Solaris) disallow removal of the working directory.
+    GTEST_SKIP() << "test requires deletion of working directory";
+
+  // Verify that we still get two separate working directories.
+  auto FS1 = vfs::createPhysicalFileSystem();
+  auto FS2 = vfs::createPhysicalFileSystem();
+  ASSERT_EQ(FS1->getCurrentWorkingDirectory().getError(),
+            errc::no_such_file_or_directory);
+  ASSERT_EQ(FS1->setCurrentWorkingDirectory(D2.path()), std::error_code());
+  ASSERT_EQ(FS1->getCurrentWorkingDirectory().get(), D2.path());
+  EXPECT_EQ(FS2->getCurrentWorkingDirectory().getError(),
+            errc::no_such_file_or_directory);
+  SmallString<128> WD2;
+  EXPECT_EQ(sys::fs::current_path(WD2), errc::no_such_file_or_directory);
+}
+
 TEST(VirtualFileSystemTest, BrokenSymlinkRealFSIteration) {
   TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true);
   IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getRealFileSystem();
diff --git a/llvm/unittests/Target/AArch64/CMakeLists.txt b/llvm/unittests/Target/AArch64/CMakeLists.txt
index ca5e9b3251d33..988d13191f4a0 100644
--- a/llvm/unittests/Target/AArch64/CMakeLists.txt
+++ b/llvm/unittests/Target/AArch64/CMakeLists.txt
@@ -10,6 +10,7 @@ set(LLVM_LINK_COMPONENTS
   AArch64Utils
   AsmParser
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   MC
diff --git a/llvm/unittests/Target/AMDGPU/CMakeLists.txt b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
index 3b571e454da01..2d7a47943df69 100644
--- a/llvm/unittests/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
@@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS
   AMDGPUInfo
   AMDGPUUtils
   CodeGen
+  CodeGenTypes
   Core
   MC
   Support
diff --git a/llvm/unittests/Target/ARM/CMakeLists.txt b/llvm/unittests/Target/ARM/CMakeLists.txt
index 7167b70536094..3da6cd95caedc 100644
--- a/llvm/unittests/Target/ARM/CMakeLists.txt
+++ b/llvm/unittests/Target/ARM/CMakeLists.txt
@@ -8,6 +8,7 @@ set(LLVM_LINK_COMPONENTS
   ARMDesc
   ARMInfo
   CodeGen
+  CodeGenTypes
   Core
   GlobalISel
   MC
diff --git a/llvm/unittests/Target/LoongArch/CMakeLists.txt b/llvm/unittests/Target/LoongArch/CMakeLists.txt
index 44d695f9caba8..fef4f8e154618 100644
--- a/llvm/unittests/Target/LoongArch/CMakeLists.txt
+++ b/llvm/unittests/Target/LoongArch/CMakeLists.txt
@@ -5,6 +5,7 @@ include_directories(
 
 set(LLVM_LINK_COMPONENTS
   CodeGen
+  CodeGenTypes
   Core
   LoongArchCodeGen
   LoongArchDesc
diff --git a/llvm/unittests/Target/WebAssembly/CMakeLists.txt b/llvm/unittests/Target/WebAssembly/CMakeLists.txt
index 1a8013b03c15a..6175e6bfe6e1c 100644
--- a/llvm/unittests/Target/WebAssembly/CMakeLists.txt
+++ b/llvm/unittests/Target/WebAssembly/CMakeLists.txt
@@ -5,6 +5,7 @@ include_directories(
 
 set(LLVM_LINK_COMPONENTS
   CodeGen
+  CodeGenTypes
   Core
   MC
   MIRParser
diff --git a/llvm/unittests/Target/X86/CMakeLists.txt b/llvm/unittests/Target/X86/CMakeLists.txt
index ff39b0739a6e5..2b371966d1bcd 100644
--- a/llvm/unittests/Target/X86/CMakeLists.txt
+++ b/llvm/unittests/Target/X86/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
   Analysis
   AsmParser
   CodeGen
+  CodeGenTypes
   Core
   InstCombine
   MC
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index 4f2bc6f3b36c8..537abd91b79e2 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -603,6 +603,47 @@ TEST(Local, SimplifyVScaleWithRange) {
   delete CI;
 }
 
+TEST(Local, wouldInstructionBeTriviallyDead) {
+  LLVMContext Ctx;
+  std::unique_ptr<Module> M = parseIR(Ctx,
+                                      R"(
+    define dso_local void @fun() local_unnamed_addr #0 !dbg !9 {
+    entry:
+      call void @llvm.dbg.declare(metadata !{}, metadata !13, metadata !DIExpression()), !dbg !16
+      ret void, !dbg !16
+    }
+
+    declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+    !llvm.dbg.cu = !{!0}
+    !llvm.module.flags = !{!2, !3}
+    !llvm.ident = !{!8}
+
+    !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 16.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+    !1 = !DIFile(filename: "test.c", directory: "/")
+    !2 = !{i32 7, !"Dwarf Version", i32 5}
+    !3 = !{i32 2, !"Debug Info Version", i32 3}
+    !8 = !{!"clang version 16.0.0"}
+    !9 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+    !10 = !DISubroutineType(types: !11)
+    !11 = !{null}
+    !12 = !{!13}
+    !13 = !DILocalVariable(name: "a", scope: !9, file: !1, line: 1, type: !14)
+    !14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+    !16 = !DILocation(line: 1, column: 21, scope: !9)
+    )");
+  bool BrokenDebugInfo = true;
+  verifyModule(*M, &errs(), &BrokenDebugInfo);
+  ASSERT_FALSE(BrokenDebugInfo);
+
+  // Get the dbg.declare.
+  Function &F = *cast<Function>(M->getNamedValue("fun"));
+  Instruction *DbgDeclare = &F.front().front();
+  ASSERT_TRUE(isa<DbgDeclareInst>(DbgDeclare));
+  // Debug intrinsics with empty metadata arguments are not dead.
+  EXPECT_FALSE(wouldInstructionBeTriviallyDead(DbgDeclare));
+}
+
 TEST(Local, ChangeToUnreachable) {
   LLVMContext Ctx;
 
diff --git a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp
index a4cff86f533cf..af20f5af36bb4 100644
--- a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp
+++ b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp
@@ -421,4 +421,19 @@ TEST(ValueMapperTest, mapValuePoisonWithTypeRemap) {
   EXPECT_EQ(NewPoison, Mapper.mapValue(*OldPoison));
 }
 
+TEST(ValueMapperTest, mapValueConstantTargetNoneToLayoutTypeNullValue) {
+  LLVMContext C;
+  auto *OldTy = TargetExtType::get(C, "spirv.Image");
+  Type *NewTy = OldTy->getLayoutType();
+
+  TestTypeRemapper TM(NewTy);
+  ValueToValueMapTy VM;
+  ValueMapper Mapper(VM, RF_None, &TM);
+
+  // Check that ConstantTargetNone is mapped to '0' constant of its layout type.
+  auto *OldConstant = ConstantTargetNone::get(OldTy);
+  auto *NewConstant = Constant::getNullValue(NewTy);
+  EXPECT_EQ(NewConstant, Mapper.mapValue(*OldConstant));
+}
+
 } // end namespace
diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
index d336e17898761..bb064b5f63b72 100644
--- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
@@ -15,6 +15,8 @@ namespace llvm {
 namespace {
 
 TEST(VPDominatorTreeTest, DominanceNoRegionsTest) {
+  //   VPBB0
+  //    |
   //   R1 {
   //     VPBB1
   //     /   \
@@ -22,6 +24,8 @@ TEST(VPDominatorTreeTest, DominanceNoRegionsTest) {
   //    \    /
   //    VPBB4
   //  }
+  VPBasicBlock *VPPH = new VPBasicBlock("ph");
+  VPBasicBlock *VPBB0 = new VPBasicBlock("VPBB0");
   VPBasicBlock *VPBB1 = new VPBasicBlock("VPBB1");
   VPBasicBlock *VPBB2 = new VPBasicBlock("VPBB2");
   VPBasicBlock *VPBB3 = new VPBasicBlock("VPBB3");
@@ -30,13 +34,14 @@ TEST(VPDominatorTreeTest, DominanceNoRegionsTest) {
   VPBB2->setParent(R1);
   VPBB3->setParent(R1);
 
+  VPBlockUtils::connectBlocks(VPBB0, R1);
   VPBlockUtils::connectBlocks(VPBB1, VPBB2);
   VPBlockUtils::connectBlocks(VPBB1, VPBB3);
   VPBlockUtils::connectBlocks(VPBB2, VPBB4);
   VPBlockUtils::connectBlocks(VPBB3, VPBB4);
 
-  VPlan Plan;
-  Plan.setEntry(R1);
+  auto TC = std::make_unique<VPValue>();
+  VPlan Plan(VPPH, &*TC, VPBB0);
   VPDominatorTree VPDT;
   VPDT.recalculate(Plan);
 
@@ -68,6 +73,8 @@ checkDomChildren(VPDominatorTree &VPDT, VPBlockBase *Src,
 TEST(VPDominatorTreeTest, DominanceRegionsTest) {
   {
     // 2 consecutive regions.
+    // VPBB0
+    //  |
     // R1 {
     //     \
     //     R1BB1     _
@@ -84,6 +91,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
     //    R2BB2
     // }
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
+    VPBasicBlock *VPBB0 = new VPBasicBlock("VPBB0");
     VPBasicBlock *R1BB1 = new VPBasicBlock();
     VPBasicBlock *R1BB2 = new VPBasicBlock();
     VPBasicBlock *R1BB3 = new VPBasicBlock();
@@ -91,6 +100,7 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
     VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB4, "R1");
     R1BB2->setParent(R1);
     R1BB3->setParent(R1);
+    VPBlockUtils::connectBlocks(VPBB0, R1);
     VPBlockUtils::connectBlocks(R1BB1, R1BB2);
     VPBlockUtils::connectBlocks(R1BB1, R1BB3);
     VPBlockUtils::connectBlocks(R1BB2, R1BB4);
@@ -104,8 +114,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
     VPBlockUtils::connectBlocks(R2BB1, R2BB2);
     VPBlockUtils::connectBlocks(R1, R2);
 
-    VPlan Plan;
-    Plan.setEntry(R1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB0);
     VPDominatorTree VPDT;
     VPDT.recalculate(Plan);
 
@@ -157,6 +167,7 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
     //   |
     //  VPBB2
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     VPBasicBlock *R1BB1 = new VPBasicBlock("R1BB1");
     VPBasicBlock *R1BB2 = new VPBasicBlock("R1BB2");
     VPBasicBlock *R1BB3 = new VPBasicBlock("R1BB3");
@@ -183,8 +194,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) {
     VPBasicBlock *VPBB2 = new VPBasicBlock("VPBB2");
     VPBlockUtils::connectBlocks(R1, VPBB2);
 
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
     VPDominatorTree VPDT;
     VPDT.recalculate(Plan);
 
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index f6f283a44a1de..e4664896970e8 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -106,14 +106,19 @@ node [shape=rect, fontname=Courier, fontsize=30]
 edge [fontname=Courier, fontsize=30]
 compound=true
   N0 [label =
+    "ph:\l" +
+    "  EMIT vp\<%1\> = EXPAND SCEV (-1 + %N)\l" +
+    "No successors\l"
+  ]
+  N1 [label =
     "vector.ph:\l" +
     "Successor(s): for.body\l"
   ]
-  N0 -> N1 [ label="" lhead=cluster_N2]
-  subgraph cluster_N2 {
+  N1 -> N2 [ label="" lhead=cluster_N3]
+  subgraph cluster_N3 {
     fontname=Courier
     label="\<x1\> for.body"
-    N1 [label =
+    N2 [label =
       "vector.body:\l" +
       "  WIDEN-PHI ir\<%indvars.iv\> = phi ir\<0\>, ir\<%indvars.iv.next\>\l" +
       "  EMIT ir\<%arr.idx\> = getelementptr ir\<%A\> ir\<%indvars.iv\>\l" +
@@ -126,8 +131,8 @@ compound=true
       "No successors\l"
     ]
   }
-  N1 -> N3 [ label="" ltail=cluster_N2]
-  N3 [label =
+  N2 -> N4 [ label="" ltail=cluster_N3]
+  N4 [label =
     "for.end:\l" +
     "No successors\l"
   ]
@@ -137,9 +142,8 @@ compound=true
 #endif
   TargetLibraryInfoImpl TLII(Triple(M.getTargetTriple()));
   TargetLibraryInfo TLI(TLII);
-  SmallPtrSet<Instruction *, 1> DeadInstructions;
   VPlanTransforms::VPInstructionsToVPRecipes(
-      Plan, [](PHINode *P) { return nullptr; }, DeadInstructions, *SE, TLI);
+      Plan, [](PHINode *P) { return nullptr; }, *SE, TLI);
 }
 
 TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
@@ -166,11 +170,10 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
   auto Plan = buildHCFG(LoopHeader);
 
-  SmallPtrSet<Instruction *, 1> DeadInstructions;
   TargetLibraryInfoImpl TLII(Triple(M.getTargetTriple()));
   TargetLibraryInfo TLI(TLII);
   VPlanTransforms::VPInstructionsToVPRecipes(
-      Plan, [](PHINode *P) { return nullptr; }, DeadInstructions, *SE, TLI);
+      Plan, [](PHINode *P) { return nullptr; }, *SE, TLI);
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 606d8d03fc63f..e0c4ad6b42c7f 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -239,6 +239,7 @@ TEST(VPInstructionTest, releaseOperandsAtDeletion) {
 }
 TEST(VPBasicBlockTest, getPlan) {
   {
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     VPBasicBlock *VPBB1 = new VPBasicBlock();
     VPBasicBlock *VPBB2 = new VPBasicBlock();
     VPBasicBlock *VPBB3 = new VPBasicBlock();
@@ -254,8 +255,8 @@ TEST(VPBasicBlockTest, getPlan) {
     VPBlockUtils::connectBlocks(VPBB2, VPBB4);
     VPBlockUtils::connectBlocks(VPBB3, VPBB4);
 
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
 
     EXPECT_EQ(&Plan, VPBB1->getPlan());
     EXPECT_EQ(&Plan, VPBB2->getPlan());
@@ -264,20 +265,7 @@ TEST(VPBasicBlockTest, getPlan) {
   }
 
   {
-    // Region block is entry into VPlan.
-    VPBasicBlock *R1BB1 = new VPBasicBlock();
-    VPBasicBlock *R1BB2 = new VPBasicBlock();
-    VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB2, "R1");
-    VPBlockUtils::connectBlocks(R1BB1, R1BB2);
-
-    VPlan Plan;
-    Plan.setEntry(R1);
-    EXPECT_EQ(&Plan, R1->getPlan());
-    EXPECT_EQ(&Plan, R1BB1->getPlan());
-    EXPECT_EQ(&Plan, R1BB2->getPlan());
-  }
-
-  {
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     // VPBasicBlock is the entry into the VPlan, followed by a region.
     VPBasicBlock *R1BB1 = new VPBasicBlock();
     VPBasicBlock *R1BB2 = new VPBasicBlock();
@@ -287,8 +275,9 @@ TEST(VPBasicBlockTest, getPlan) {
     VPBasicBlock *VPBB1 = new VPBasicBlock();
     VPBlockUtils::connectBlocks(VPBB1, R1);
 
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
+
     EXPECT_EQ(&Plan, VPBB1->getPlan());
     EXPECT_EQ(&Plan, R1->getPlan());
     EXPECT_EQ(&Plan, R1BB1->getPlan());
@@ -296,6 +285,8 @@ TEST(VPBasicBlockTest, getPlan) {
   }
 
   {
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
+
     VPBasicBlock *R1BB1 = new VPBasicBlock();
     VPBasicBlock *R1BB2 = new VPBasicBlock();
     VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB2, "R1");
@@ -314,8 +305,9 @@ TEST(VPBasicBlockTest, getPlan) {
     VPBlockUtils::connectBlocks(R1, VPBB2);
     VPBlockUtils::connectBlocks(R2, VPBB2);
 
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
+
     EXPECT_EQ(&Plan, VPBB1->getPlan());
     EXPECT_EQ(&Plan, R1->getPlan());
     EXPECT_EQ(&Plan, R1BB1->getPlan());
@@ -336,6 +328,7 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     //    \    /
     //    VPBB4
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     VPBasicBlock *VPBB1 = new VPBasicBlock();
     VPBasicBlock *VPBB2 = new VPBasicBlock();
     VPBasicBlock *VPBB3 = new VPBasicBlock();
@@ -353,12 +346,14 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     EXPECT_EQ(VPBB2, FromIterator[1]);
 
     // Use Plan to properly clean up created blocks.
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
   }
 
   {
     // 2 consecutive regions.
+    // VPBB0
+    //  |
     // R1 {
     //     \
     //     R1BB1
@@ -374,6 +369,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     //      |
     //    R2BB2
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
+    VPBasicBlock *VPBB0 = new VPBasicBlock("VPBB0");
     VPBasicBlock *R1BB1 = new VPBasicBlock();
     VPBasicBlock *R1BB2 = new VPBasicBlock();
     VPBasicBlock *R1BB3 = new VPBasicBlock();
@@ -381,6 +378,7 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB4, "R1");
     R1BB2->setParent(R1);
     R1BB3->setParent(R1);
+    VPBlockUtils::connectBlocks(VPBB0, R1);
     VPBlockUtils::connectBlocks(R1BB1, R1BB2);
     VPBlockUtils::connectBlocks(R1BB1, R1BB3);
     VPBlockUtils::connectBlocks(R1BB2, R1BB4);
@@ -448,8 +446,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     EXPECT_EQ(R1, FromIterator[7]);
 
     // Use Plan to properly clean up created blocks.
-    VPlan Plan;
-    Plan.setEntry(R1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB0);
   }
 
   {
@@ -473,6 +471,7 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     //   |
     //  VPBB2
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     VPBasicBlock *R1BB1 = new VPBasicBlock("R1BB1");
     VPBasicBlock *R1BB2 = new VPBasicBlock("R1BB2");
     VPBasicBlock *R1BB3 = new VPBasicBlock("R1BB3");
@@ -530,8 +529,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     EXPECT_EQ(VPBB1, FromIterator[9]);
 
     // Use Plan to properly clean up created blocks.
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
   }
 
   {
@@ -545,6 +544,7 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     //      R2BB2
     //   }
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     VPBasicBlock *R2BB1 = new VPBasicBlock("R2BB1");
     VPBasicBlock *R2BB2 = new VPBasicBlock("R2BB2");
     VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB2, "R2");
@@ -577,8 +577,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     EXPECT_EQ(VPBB1, FromIterator[4]);
 
     // Use Plan to properly clean up created blocks.
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
   }
 
   {
@@ -600,6 +600,7 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     //   |
     //  VPBB2
     //
+    VPBasicBlock *VPPH = new VPBasicBlock("ph");
     VPBasicBlock *R3BB1 = new VPBasicBlock("R3BB1");
     VPRegionBlock *R3 = new VPRegionBlock(R3BB1, R3BB1, "R3");
 
@@ -668,13 +669,17 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) {
     EXPECT_EQ(VPBB1, FromIterator[3]);
 
     // Use Plan to properly clean up created blocks.
-    VPlan Plan;
-    Plan.setEntry(VPBB1);
+    auto TC = std::make_unique<VPValue>();
+    VPlan Plan(VPPH, &*TC, VPBB1);
   }
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 TEST(VPBasicBlockTest, print) {
+  VPInstruction *TC = new VPInstruction(Instruction::Add, {});
+  VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
+  VPBB0->appendRecipe(TC);
+
   VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
   VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1});
   VPInstruction *I3 = new VPInstruction(Instruction::Br, {I1, I2});
@@ -704,8 +709,7 @@ TEST(VPBasicBlockTest, print) {
     EXPECT_EQ("EMIT br <badref> <badref>", I3Dump);
   }
 
-  VPlan Plan;
-  Plan.setEntry(VPBB1);
+  VPlan Plan(VPBB0, TC, VPBB1);
   std::string FullDump;
   raw_string_ostream OS(FullDump);
   Plan.printDOT(OS);
@@ -716,17 +720,22 @@ node [shape=rect, fontname=Courier, fontsize=30]
 edge [fontname=Courier, fontsize=30]
 compound=true
   N0 [label =
-    "bb1:\l" +
+    "preheader:\l" +
     "  EMIT vp\<%1\> = add\l" +
-    "  EMIT vp\<%2\> = sub vp\<%1\>\l" +
-    "  EMIT br vp\<%1\> vp\<%2\>\l" +
-    "Successor(s): bb2\l"
+    "No successors\l"
   ]
-  N0 -> N1 [ label=""]
   N1 [label =
+    "bb1:\l" +
+    "  EMIT vp\<%2\> = add\l" +
+    "  EMIT vp\<%3\> = sub vp\<%2\>\l" +
+    "  EMIT br vp\<%2\> vp\<%3\>\l" +
+    "Successor(s): bb2\l"
+  ]
+  N1 -> N2 [ label=""]
+  N2 [label =
     "bb2:\l" +
-    "  EMIT vp\<%4\> = mul vp\<%2\> vp\<%1\>\l" +
-    "  EMIT ret vp\<%4\>\l" +
+    "  EMIT vp\<%5\> = mul vp\<%3\> vp\<%2\>\l" +
+    "  EMIT ret vp\<%5\>\l" +
     "No successors\l"
   ]
 }
@@ -734,9 +743,9 @@ compound=true
   EXPECT_EQ(ExpectedStr, FullDump);
 
   const char *ExpectedBlock1Str = R"(bb1:
-  EMIT vp<%1> = add
-  EMIT vp<%2> = sub vp<%1>
-  EMIT br vp<%1> vp<%2>
+  EMIT vp<%2> = add
+  EMIT vp<%3> = sub vp<%2>
+  EMIT br vp<%2> vp<%3>
 Successor(s): bb2
 )";
   std::string Block1Dump;
@@ -746,8 +755,8 @@ Successor(s): bb2
 
   // Ensure that numbering is good when dumping the second block in isolation.
   const char *ExpectedBlock2Str = R"(bb2:
-  EMIT vp<%4> = mul vp<%2> vp<%1>
-  EMIT ret vp<%4>
+  EMIT vp<%5> = mul vp<%3> vp<%2>
+  EMIT ret vp<%5>
 No successors
 )";
   std::string Block2Dump;
@@ -761,7 +770,7 @@ No successors
     VPSlotTracker SlotTracker(&Plan);
     I3->print(OS, "", SlotTracker);
     OS.flush();
-    EXPECT_EQ("EMIT br vp<%1> vp<%2>", I3Dump);
+    EXPECT_EQ("EMIT br vp<%2> vp<%3>", I3Dump);
   }
 
   {
@@ -769,21 +778,24 @@ No successors
     raw_string_ostream OS(I4Dump);
     OS << *I4;
     OS.flush();
-    EXPECT_EQ("EMIT vp<%4> = mul vp<%2> vp<%1>", I4Dump);
+    EXPECT_EQ("EMIT vp<%5> = mul vp<%3> vp<%2>", I4Dump);
   }
 }
 
 TEST(VPBasicBlockTest, printPlanWithVFsAndUFs) {
-  VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
 
+  VPInstruction *TC = new VPInstruction(Instruction::Sub, {});
+  VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
+  VPBB0->appendRecipe(TC);
+
+  VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
   VPBasicBlock *VPBB1 = new VPBasicBlock();
   VPBB1->appendRecipe(I1);
   VPBB1->setName("bb1");
 
-  VPlan Plan;
+  VPlan Plan(VPBB0, TC, VPBB1);
   Plan.setName("TestPlan");
   Plan.addVF(ElementCount::getFixed(4));
-  Plan.setEntry(VPBB1);
 
   {
     std::string FullDump;
@@ -791,8 +803,14 @@ TEST(VPBasicBlockTest, printPlanWithVFsAndUFs) {
     Plan.print(OS);
 
     const char *ExpectedStr = R"(VPlan 'TestPlan for VF={4},UF>=1' {
+vp<%1> = original trip-count
+
+preheader:
+  EMIT vp<%1> = sub
+No successors
+
 bb1:
-  EMIT vp<%1> = add
+  EMIT vp<%2> = add
 No successors
 }
 )";
@@ -806,8 +824,14 @@ No successors
     Plan.print(OS);
 
     const char *ExpectedStr = R"(VPlan 'TestPlan for VF={4,vscale x 8},UF>=1' {
+vp<%1> = original trip-count
+
+preheader:
+  EMIT vp<%1> = sub
+No successors
+
 bb1:
-  EMIT vp<%1> = add
+  EMIT vp<%2> = add
 No successors
 }
 )";
@@ -821,8 +845,14 @@ No successors
     Plan.print(OS);
 
     const char *ExpectedStr = R"(VPlan 'TestPlan for VF={4,vscale x 8},UF={4}' {
+vp<%1> = original trip-count
+
+preheader:
+  EMIT vp<%1> = sub
+No successors
+
 bb1:
-  EMIT vp<%1> = add
+  EMIT vp<%2> = add
 No successors
 }
 )";
@@ -1192,9 +1222,9 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 TEST(VPRecipeTest, dump) {
-  VPlan Plan;
+  VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
   VPBasicBlock *VPBB1 = new VPBasicBlock();
-  Plan.setEntry(VPBB1);
+  VPlan Plan(VPBB0, VPBB1);
 
   LLVMContext C;
 
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index b1139bd5bbdad..d9372174fdfa4 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -64,7 +64,8 @@ class VPlanTestBase : public testing::Test {
   VPlanPtr buildHCFG(BasicBlock *LoopHeader) {
     doAnalysis(*LoopHeader->getParent());
 
-    auto Plan = std::make_unique<VPlan>();
+    auto Plan = VPlan::createInitialVPlan(
+        SE->getBackedgeTakenCount(LI->getLoopFor(LoopHeader)), *SE);
     VPlanHCFGBuilder HCFGBuilder(LI->getLoopFor(LoopHeader), LI.get(), *Plan);
     HCFGBuilder.buildHierarchicalCFG();
     return Plan;
@@ -74,10 +75,10 @@ class VPlanTestBase : public testing::Test {
   VPlanPtr buildPlainCFG(BasicBlock *LoopHeader) {
     doAnalysis(*LoopHeader->getParent());
 
-    auto Plan = std::make_unique<VPlan>();
+    auto Plan = VPlan::createInitialVPlan(
+        SE->getBackedgeTakenCount(LI->getLoopFor(LoopHeader)), *SE);
     VPlanHCFGBuilder HCFGBuilder(LI->getLoopFor(LoopHeader), LI.get(), *Plan);
-    VPBasicBlock *EntryVPBB = HCFGBuilder.buildPlainCFG();
-    Plan->setEntry(EntryVPBB);
+    HCFGBuilder.buildPlainCFG();
     return Plan;
   }
 };
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 516d1142daf87..b080f969fb9bc 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -19,12 +19,13 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
   VPInstruction *DefI = new VPInstruction(Instruction::Add, {});
   VPInstruction *UseI = new VPInstruction(Instruction::Sub, {DefI});
 
+  VPBasicBlock *VPPH = new VPBasicBlock("ph");
   VPBasicBlock *VPBB1 = new VPBasicBlock();
   VPBB1->appendRecipe(UseI);
   VPBB1->appendRecipe(DefI);
 
-  VPlan Plan;
-  Plan.setEntry(VPBB1);
+  auto TC = std::make_unique<VPValue>();
+  VPlan Plan(VPPH, &*TC, VPBB1);
 
 #if GTEST_HAS_STREAM_REDIRECTION
   ::testing::internal::CaptureStderr();
@@ -43,6 +44,7 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
   VPInstruction *BranchOnCond =
       new VPInstruction(VPInstruction::BranchOnCond, {CanIV});
 
+  VPBasicBlock *VPPH = new VPBasicBlock("ph");
   VPBasicBlock *VPBB1 = new VPBasicBlock();
   VPBasicBlock *VPBB2 = new VPBasicBlock();
 
@@ -54,8 +56,8 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
   VPRegionBlock *R1 = new VPRegionBlock(VPBB2, VPBB2, "R1");
   VPBlockUtils::connectBlocks(VPBB1, R1);
 
-  VPlan Plan;
-  Plan.setEntry(VPBB1);
+  auto TC = std::make_unique<VPValue>();
+  VPlan Plan(VPPH, &*TC, VPBB1);
 
 #if GTEST_HAS_STREAM_REDIRECTION
   ::testing::internal::CaptureStderr();
@@ -79,6 +81,7 @@ TEST(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
       new VPInstruction(VPInstruction::BranchOnCond, {CanIV});
   auto *Blend = new VPBlendRecipe(Phi, {DefI});
 
+  VPBasicBlock *VPPH = new VPBasicBlock("ph");
   VPBasicBlock *VPBB1 = new VPBasicBlock();
   VPBasicBlock *VPBB2 = new VPBasicBlock();
   VPBasicBlock *VPBB3 = new VPBasicBlock();
@@ -95,8 +98,8 @@ TEST(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
   VPRegionBlock *R1 = new VPRegionBlock(VPBB2, VPBB4, "R1");
   VPBlockUtils::connectBlocks(VPBB1, R1);
 
-  VPlan Plan;
-  Plan.setEntry(VPBB1);
+  auto TC = std::make_unique<VPValue>();
+  VPlan Plan(VPPH, &*TC, VPBB1);
 
 #if GTEST_HAS_STREAM_REDIRECTION
   ::testing::internal::CaptureStderr();
diff --git a/llvm/unittests/tools/llvm-exegesis/CMakeLists.txt b/llvm/unittests/tools/llvm-exegesis/CMakeLists.txt
index 58e514a11ad33..f3fb01768ea1a 100644
--- a/llvm/unittests/tools/llvm-exegesis/CMakeLists.txt
+++ b/llvm/unittests/tools/llvm-exegesis/CMakeLists.txt
@@ -3,6 +3,7 @@ set(exegesis_includes
   )
 
 set(LLVM_LINK_COMPONENTS
+  CodeGenTypes
   MC
   MCParser
   Object
diff --git a/llvm/unittests/tools/llvm-profdata/OutputSizeLimitTest.cpp b/llvm/unittests/tools/llvm-profdata/OutputSizeLimitTest.cpp
index 25da748865ec7..522a2c2dd3367 100644
--- a/llvm/unittests/tools/llvm-profdata/OutputSizeLimitTest.cpp
+++ b/llvm/unittests/tools/llvm-profdata/OutputSizeLimitTest.cpp
@@ -123,24 +123,6 @@ static ExpectedErrorOr<void *> RunTest(StringRef Input, size_t SizeLimit,
   RETURN_IF_ERROR(sys::fs::file_size(Temp.path(), FileSize));
   EXPECT_LE(FileSize, SizeLimit);
 
-  // For compact binary format, function names are stored as MD5, so we cannot
-  // directly match the samples of the new profile with the old profile. A
-  // simple way is to convert the old profile to compact binary format and read
-  // it back
-  if (Format == llvm::sampleprof::SPF_Compact_Binary) {
-    TempFile CompBinary("compbinary", "afdo", "", true);
-    {
-      DEF_VAR_RETURN_IF_ERROR(
-          Writer, SampleProfileWriter::create(
-                      CompBinary.path(), llvm::sampleprof::SPF_Compact_Binary));
-      RETURN_IF_ERROR(Writer->write(OldProfiles));
-    }
-    VAR_RETURN_IF_ERROR(Reader, SampleProfileReader::create(
-                                    CompBinary.path().str(), Context, *FS));
-    RETURN_IF_ERROR(Reader->read());
-    OldProfiles = Reader->getProfiles();
-  }
-
   // For every sample in the new profile, confirm it is in the old profile and
   // unchanged.
   for (auto Sample : NewProfiles) {
@@ -169,13 +151,6 @@ TEST(TestOutputSizeLimit, TestOutputSizeLimitBinary) {
         Succeeded());
 }
 
-TEST(TestOutputSizeLimit, TestOutputSizeLimitCompBinary) {
-  for (size_t OutputSizeLimit : {277, 276, 275, 264, 263, 250, 200})
-    ASSERT_THAT_EXPECTED(
-        RunTest(Input1, OutputSizeLimit, llvm::sampleprof::SPF_Compact_Binary),
-        Succeeded());
-}
-
 TEST(TestOutputSizeLimit, TestOutputSizeLimitText) {
   for (size_t OutputSizeLimit :
        {229, 228, 227, 213, 212, 211, 189, 188, 187, 186, 150})
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index 5f8dd1594a1f2..474042a3e9a33 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -55,6 +55,7 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   // Emit attribute enums in the same order llvm::Attribute::operator< expects.
   Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM");
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
+  Emit({"ComplexStrAttr"}, "ATTRIBUTE_COMPLEXSTR");
 
   OS << "#undef ATTRIBUTE_ALL\n";
   OS << "#endif\n\n";
diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt
index 3d6f96bcb7fc4..7bdc33d115cc0 100644
--- a/llvm/utils/TableGen/CMakeLists.txt
+++ b/llvm/utils/TableGen/CMakeLists.txt
@@ -1,21 +1,48 @@
 add_subdirectory(GlobalISel)
 
+add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
+  Attributes.cpp
+  CodeGenIntrinsics.cpp
+  DirectiveEmitter.cpp
+  IntrinsicEmitter.cpp
+  RISCVTargetDefEmitter.cpp
+  SDNodeProperties.cpp
+  VTEmitter.cpp
+  PARTIAL_SOURCES_INTENDED
+
+  LINK_COMPONENTS
+  Support
+  TableGen
+  )
+set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning")
+
 set(LLVM_LINK_COMPONENTS Support)
 
+add_tablegen(llvm-min-tblgen LLVM_HEADERS
+  TableGen.cpp
+  $<TARGET_OBJECTS:obj.LLVMTableGenCommon>
+  PARTIAL_SOURCES_INTENDED
+  )
+set_target_properties(llvm-min-tblgen PROPERTIES FOLDER "Tablegenning")
+
+set(LLVM_LINK_COMPONENTS
+  CodeGenTypes
+  Support
+  )
+
 add_tablegen(llvm-tblgen LLVM
   DESTINATION "${LLVM_TOOLS_INSTALL_DIR}"
   EXPORT LLVM
   AsmMatcherEmitter.cpp
   AsmWriterEmitter.cpp
   AsmWriterInst.cpp
-  Attributes.cpp
+  CTagsEmitter.cpp
   CallingConvEmitter.cpp
   CodeEmitterGen.cpp
   CodeGenDAGPatterns.cpp
   CodeGenHwModes.cpp
   CodeGenInstAlias.cpp
   CodeGenInstruction.cpp
-  CodeGenIntrinsics.cpp
   CodeGenMapTable.cpp
   CodeGenRegisters.cpp
   CodeGenSchedule.cpp
@@ -28,7 +55,6 @@ add_tablegen(llvm-tblgen LLVM
   DecoderEmitter.cpp
   DFAEmitter.cpp
   DFAPacketizerEmitter.cpp
-  DirectiveEmitter.cpp
   DisassemblerEmitter.cpp
   DXILEmitter.cpp
   ExegesisEmitter.cpp
@@ -38,7 +64,6 @@ add_tablegen(llvm-tblgen LLVM
   InfoByHwMode.cpp
   InstrInfoEmitter.cpp
   InstrDocsEmitter.cpp
-  IntrinsicEmitter.cpp
   OptEmitter.cpp
   OptParserEmitter.cpp
   OptRSTEmitter.cpp
@@ -47,14 +72,11 @@ add_tablegen(llvm-tblgen LLVM
   CompressInstEmitter.cpp
   RegisterBankEmitter.cpp
   RegisterInfoEmitter.cpp
-  RISCVTargetDefEmitter.cpp
-  SDNodeProperties.cpp
   SearchableTableEmitter.cpp
   SubtargetEmitter.cpp
   SubtargetFeatureInfo.cpp
   TableGen.cpp
   Types.cpp
-  VTEmitter.cpp
   VarLenCodeEmitterGen.cpp
   X86DisassemblerTables.cpp
   X86EVEX2VEXTablesEmitter.cpp
@@ -63,7 +85,7 @@ add_tablegen(llvm-tblgen LLVM
   X86ModRMFilters.cpp
   X86RecognizableInstr.cpp
   WebAssemblyDisassemblerEmitter.cpp
-  CTagsEmitter.cpp
+  $<TARGET_OBJECTS:obj.LLVMTableGenCommon>
   )
 target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel)
 set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning")
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index 2fd877f4d4aa4..8e6b40e6cd20d 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -15,6 +15,8 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
+#include <deque>
+
 using namespace llvm;
 
 namespace {
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index baaa4ee24c104..512358ba915bd 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -1858,7 +1858,7 @@ static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) {
     return 0;  // All return nothing.
 
   if (Operator->isSubClassOf("Intrinsic"))
-    return CDP.getIntrinsic(Operator).IS.RetVTs.size();
+    return CDP.getIntrinsic(Operator).IS.RetTys.size();
 
   if (Operator->isSubClassOf("SDNode"))
     return CDP.getSDNodeInfo(Operator).getNumResults();
@@ -2511,11 +2511,12 @@ bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) {
     bool MadeChange = false;
 
     // Apply the result type to the node.
-    unsigned NumRetVTs = Int->IS.RetVTs.size();
-    unsigned NumParamVTs = Int->IS.ParamVTs.size();
+    unsigned NumRetVTs = Int->IS.RetTys.size();
+    unsigned NumParamVTs = Int->IS.ParamTys.size();
 
     for (unsigned i = 0, e = NumRetVTs; i != e; ++i)
-      MadeChange |= UpdateNodeType(i, Int->IS.RetVTs[i], TP);
+      MadeChange |= UpdateNodeType(
+          i, getValueType(Int->IS.RetTys[i]->getValueAsDef("VT")), TP);
 
     if (getNumChildren() != NumParamVTs + 1) {
       TP.error("Intrinsic '" + Int->Name + "' expects " + Twine(NumParamVTs) +
@@ -2529,9 +2530,10 @@ bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) {
     for (unsigned i = 0, e = getNumChildren()-1; i != e; ++i) {
       MadeChange |= getChild(i+1)->ApplyTypeConstraints(TP, NotRegisters);
 
-      MVT::SimpleValueType OpVT = Int->IS.ParamVTs[i];
-      assert(getChild(i+1)->getNumTypes() == 1 && "Unhandled case");
-      MadeChange |= getChild(i+1)->UpdateNodeType(0, OpVT, TP);
+      MVT::SimpleValueType OpVT =
+          getValueType(Int->IS.ParamTys[i]->getValueAsDef("VT"));
+      assert(getChild(i + 1)->getNumTypes() == 1 && "Unhandled case");
+      MadeChange |= getChild(i + 1)->UpdateNodeType(0, OpVT, TP);
     }
     return MadeChange;
   }
@@ -2983,7 +2985,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
 
     // If this intrinsic returns void, it must have side-effects and thus a
     // chain.
-    if (Int.IS.RetVTs.empty())
+    if (Int.IS.RetTys.empty())
       Operator = getDAGPatterns().get_intrinsic_void_sdnode();
     else if (!Int.ME.doesNotAccessMemory() || Int.hasSideEffects)
       // Has side-effects, requires chain.
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/CodeGenDAGPatterns.h
index 8a02e411f8b8d..c7a5f5c4f81c0 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.h
@@ -23,8 +23,10 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/TableGen/Record.h"
 #include <algorithm>
 #include <array>
 #include <functional>
@@ -34,7 +36,6 @@
 
 namespace llvm {
 
-class Record;
 class Init;
 class ListInit;
 class DagInit;
diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/CodeGenInstruction.h
index 2a1f17c37834a..ee7a1696bab9c 100644
--- a/llvm/utils/TableGen/CodeGenInstruction.h
+++ b/llvm/utils/TableGen/CodeGenInstruction.h
@@ -16,7 +16,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/MachineValueType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include <cassert>
 #include <string>
 #include <utility>
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/CodeGenIntrinsics.cpp
index 73e40c7dd6b80..3be73d5a5e2ba 100644
--- a/llvm/utils/TableGen/CodeGenIntrinsics.cpp
+++ b/llvm/utils/TableGen/CodeGenIntrinsics.cpp
@@ -20,13 +20,6 @@
 #include <algorithm>
 #include <cassert>
 using namespace llvm;
-using namespace llvm::tmp;
-
-/// getValueType - Return the MVT::SimpleValueType that the specified TableGen
-/// record corresponds to.
-MVT::SimpleValueType llvm::tmp::getValueType(Record *Rec) {
-  return (MVT::SimpleValueType)Rec->getValueAsInt("Value");
-}
 
 //===----------------------------------------------------------------------===//
 // CodeGenIntrinsic Implementation
@@ -120,91 +113,16 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
                                   TargetPrefix + ".'!");
   }
 
-  ListInit *RetTypes = R->getValueAsListInit("RetTypes");
-  ListInit *ParamTypes = R->getValueAsListInit("ParamTypes");
-
-  // First collate a list of overloaded types.
-  std::vector<MVT::SimpleValueType> OverloadedVTs;
-  for (ListInit *TypeList : {RetTypes, ParamTypes}) {
-    for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
-      Record *TyEl = TypeList->getElementAsRecord(i);
-      assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
-
-      if (TyEl->isSubClassOf("LLVMMatchType"))
-        continue;
+  if (auto *Types = R->getValue("Types")) {
+    auto *TypeList = cast<ListInit>(Types->getValue());
+    isOverloaded = R->getValueAsBit("isOverloaded");
 
-      MVT::SimpleValueType VT = getValueType(TyEl->getValueAsDef("VT"));
-      if (MVT(VT).isOverloaded()) {
-        OverloadedVTs.push_back(VT);
-        isOverloaded = true;
-      }
-    }
-  }
-
-  // Parse the list of return types.
-  ListInit *TypeList = RetTypes;
-  for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
-    Record *TyEl = TypeList->getElementAsRecord(i);
-    assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
-    MVT::SimpleValueType VT;
-    if (TyEl->isSubClassOf("LLVMMatchType")) {
-      unsigned MatchTy = TyEl->getValueAsInt("Number");
-      assert(MatchTy < OverloadedVTs.size() && "Invalid matching number!");
-      VT = OverloadedVTs[MatchTy];
-      // It only makes sense to use the extended and truncated vector element
-      // variants with iAny types; otherwise, if the intrinsic is not
-      // overloaded, all the types can be specified directly.
-      assert(((!TyEl->isSubClassOf("LLVMExtendedType") &&
-               !TyEl->isSubClassOf("LLVMTruncatedType")) ||
-              VT == MVT::iAny || VT == MVT::vAny) &&
-             "Expected iAny or vAny type");
-    } else {
-      VT = getValueType(TyEl->getValueAsDef("VT"));
-    }
-
-    // Reject invalid types.
-    if (VT == MVT::isVoid)
-      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
-                                  " has void in result type list!");
-
-    IS.RetVTs.push_back(VT);
-    IS.RetTypeDefs.push_back(TyEl);
-  }
-
-  // Parse the list of parameter types.
-  TypeList = ParamTypes;
-  for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
-    Record *TyEl = TypeList->getElementAsRecord(i);
-    assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
-    MVT::SimpleValueType VT;
-    if (TyEl->isSubClassOf("LLVMMatchType")) {
-      unsigned MatchTy = TyEl->getValueAsInt("Number");
-      if (MatchTy >= OverloadedVTs.size()) {
-        PrintError(R->getLoc(), "Parameter #" + Twine(i) +
-                                    " has out of bounds matching "
-                                    "number " +
-                                    Twine(MatchTy));
-        PrintFatalError(DefLoc,
-                        Twine("ParamTypes is ") + TypeList->getAsString());
-      }
-      VT = OverloadedVTs[MatchTy];
-      // It only makes sense to use the extended and truncated vector element
-      // variants with iAny types; otherwise, if the intrinsic is not
-      // overloaded, all the types can be specified directly.
-      assert(((!TyEl->isSubClassOf("LLVMExtendedType") &&
-               !TyEl->isSubClassOf("LLVMTruncatedType")) ||
-              VT == MVT::iAny || VT == MVT::vAny) &&
-             "Expected iAny or vAny type");
-    } else
-      VT = getValueType(TyEl->getValueAsDef("VT"));
-
-    // Reject invalid types.
-    if (VT == MVT::isVoid && i != e - 1 /*void at end means varargs*/)
-      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
-                                  " has void in result type list!");
+    unsigned I = 0;
+    for (unsigned E = R->getValueAsListInit("RetTypes")->size(); I < E; ++I)
+      IS.RetTys.push_back(TypeList->getElementAsRecord(I));
 
-    IS.ParamVTs.push_back(VT);
-    IS.ParamTypeDefs.push_back(TyEl);
+    for (unsigned E = TypeList->size(); I < E; ++I)
+      IS.ParamTys.push_back(TypeList->getElementAsRecord(I));
   }
 
   // Parse the intrinsic properties.
@@ -321,10 +239,10 @@ void CodeGenIntrinsic::setProperty(Record *R) {
 }
 
 bool CodeGenIntrinsic::isParamAPointer(unsigned ParamIdx) const {
-  if (ParamIdx >= IS.ParamVTs.size())
+  if (ParamIdx >= IS.ParamTys.size())
     return false;
-  MVT ParamType = MVT(IS.ParamVTs[ParamIdx]);
-  return ParamType == MVT::iPTR || ParamType == MVT::iPTRAny;
+  return (IS.ParamTys[ParamIdx]->isSubClassOf("LLVMQualPointerType") ||
+          IS.ParamTys[ParamIdx]->isSubClassOf("LLVMAnyPointerType"));
 }
 
 bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const {
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/CodeGenIntrinsics.h
index d704b0e531d3c..f2bab0127e090 100644
--- a/llvm/utils/TableGen/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/CodeGenIntrinsics.h
@@ -15,7 +15,6 @@
 
 #include "SDNodeProperties.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/ModRef.h"
 #include <string>
 #include <tuple>
@@ -25,13 +24,6 @@ namespace llvm {
 class Record;
 class RecordKeeper;
 
-// FIXME: Sweep this in the near future.
-namespace tmp {
-/// getValueType - Return the MVT::SimpleValueType that the specified TableGen
-/// record corresponds to.
-MVT::SimpleValueType getValueType(Record *Rec);
-} // namespace tmp
-
 struct CodeGenIntrinsic {
   Record *TheDef;             // The actual record defining this intrinsic.
   std::string Name;           // The name of the LLVM function "llvm.bswap.i32"
@@ -50,19 +42,13 @@ struct CodeGenIntrinsic {
     /// only populated when in the context of a target .td file. When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<MVT::SimpleValueType> RetVTs;
-
-    /// The records for each return type.
-    std::vector<Record *> RetTypeDefs;
+    std::vector<Record *> RetTys;
 
     /// The MVT::SimpleValueType for each parameter type. Note that this list is
     /// only populated when in the context of a target .td file.  When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<MVT::SimpleValueType> ParamVTs;
-
-    /// The records for each parameter type.
-    std::vector<Record *> ParamTypeDefs;
+    std::vector<Record *> ParamTys;
   };
 
   IntrinsicSignature IS;
diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/CodeGenRegisters.h
index e0d401e51ebb4..15f08d1431f9c 100644
--- a/llvm/utils/TableGen/CodeGenRegisters.h
+++ b/llvm/utils/TableGen/CodeGenRegisters.h
@@ -150,7 +150,8 @@ namespace llvm {
   };
 
   /// CodeGenRegister - Represents a register definition.
-  struct CodeGenRegister {
+  class CodeGenRegister {
+  public:
     Record *TheDef;
     unsigned EnumValue;
     std::vector<int64_t> CostPerUse;
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index ca8ce48c31034..fbdc0499a8cfb 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -15,6 +15,7 @@
 
 #include "CodeGenTarget.h"
 #include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
 #include "CodeGenSchedule.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Twine.h"
@@ -431,6 +432,10 @@ const CodeGenRegister *CodeGenTarget::getRegisterByName(StringRef Name) const {
   return getRegBank().getRegistersByName().lookup(Name);
 }
 
+const CodeGenRegisterClass &CodeGenTarget::getRegisterClass(Record *R) const {
+  return *getRegBank().getRegClass(R);
+}
+
 std::vector<ValueTypeByHwMode> CodeGenTarget::getRegisterVTs(Record *R)
       const {
   const CodeGenRegister *Reg = getRegBank().getReg(R);
diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/CodeGenTarget.h
index fa1b7e161698b..2ba3af724d36d 100644
--- a/llvm/utils/TableGen/CodeGenTarget.h
+++ b/llvm/utils/TableGen/CodeGenTarget.h
@@ -17,14 +17,13 @@
 #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H
 
 #include "CodeGenHwModes.h"
-#include "CodeGenRegisters.h"
 #include "InfoByHwMode.h"
 #include "SDNodeProperties.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/MachineValueType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include <cassert>
 #include <memory>
 #include <optional>
@@ -36,7 +35,11 @@ namespace llvm {
 class RecordKeeper;
 class Record;
 class CodeGenInstruction;
+class CodeGenRegBank;
+class CodeGenRegister;
+class CodeGenRegisterClass;
 class CodeGenSchedModels;
+class CodeGenSubRegIndex;
 
 /// getValueType - Return the MVT::SimpleValueType that the specified TableGen
 /// record corresponds to.
@@ -130,9 +133,7 @@ class CodeGenTarget {
     return RegAltNameIndices;
   }
 
-  const CodeGenRegisterClass &getRegisterClass(Record *R) const {
-    return *getRegBank().getRegClass(R);
-  }
+  const CodeGenRegisterClass &getRegisterClass(Record *R) const;
 
   /// getRegisterVTs - Find the union of all possible SimpleValueTypes for the
   /// specified physical register.
diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/DAGISelMatcher.h
index 826080f2b3ed0..e3cf847edd127 100644
--- a/llvm/utils/TableGen/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/DAGISelMatcher.h
@@ -12,8 +12,8 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cassert>
 #include <cstddef>
 #include <memory>
@@ -21,7 +21,7 @@
 #include <utility>
 
 namespace llvm {
-  struct CodeGenRegister;
+  class CodeGenRegister;
   class CodeGenDAGPatterns;
   class CodeGenInstruction;
   class Matcher;
diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
index da8538fc801a9..64c7884616a57 100644
--- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -24,6 +24,7 @@
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 #include <cstdint>
+#include <deque>
 #include <map>
 #include <set>
 #include <string>
diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
index bce0ffc314860..6d637f45c8890 100644
--- a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
+++ b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  CodeGenTypes
   Support
   TableGen
   )
@@ -12,6 +13,9 @@ add_llvm_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB
   GIMatchDagPredicate.cpp
   GIMatchDagPredicateDependencyEdge.cpp
   GIMatchTree.cpp
+
+  DEPENDS
+  vt_gen
   )
 
 # Users may include its headers as "GlobalISel/*.h"
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index fc32ebc2c0e40..02e389f85206d 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -37,11 +37,11 @@
 #include "InfoByHwMode.h"
 #include "SubtargetFeatureInfo.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/CodeGenCoverage.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/SaveAndRestore.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/TableGen/Error.h"
diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h
index 2576664bc77be..b8a6645baca58 100644
--- a/llvm/utils/TableGen/InfoByHwMode.h
+++ b/llvm/utils/TableGen/InfoByHwMode.h
@@ -18,8 +18,8 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/MachineValueType.h"
 #include <cassert>
 #include <limits>
 #include <map>
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 0e3c9694097c8..bd3445513715b 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -20,7 +20,6 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/ModRef.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
@@ -28,6 +27,7 @@
 #include "llvm/TableGen/StringToOffsetTable.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
+#include <array>
 #include <cassert>
 #include <map>
 #include <optional>
@@ -35,7 +35,6 @@
 #include <utility>
 #include <vector>
 using namespace llvm;
-using namespace llvm::tmp;
 
 cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
 cl::opt<std::string>
@@ -54,6 +53,7 @@ class IntrinsicEmitter {
 
   void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
   void EmitArgKind(raw_ostream &OS);
+  void EmitIITInfo(raw_ostream &OS);
   void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
   void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,
                                 raw_ostream &OS);
@@ -82,6 +82,9 @@ void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
     // Emit ArgKind for Intrinsics.h.
     EmitArgKind(OS);
   } else {
+    // Emit IIT_Info constants.
+    EmitIITInfo(OS);
+
     // Emit the target metadata.
     EmitTargetInfo(Ints, OS);
 
@@ -180,6 +183,27 @@ void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
   OS << "#endif\n\n";
 }
 
+void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
+  OS << "#ifdef GET_INTRINSIC_IITINFO\n";
+  std::array<StringRef, 256> RecsByNumber;
+  auto IIT_Base = Records.getAllDerivedDefinitionsIfDefined("IIT_Base");
+  for (auto Rec : IIT_Base) {
+    auto Number = Rec->getValueAsInt("Number");
+    assert(0 <= Number && Number < (int)RecsByNumber.size() &&
+           "IIT_Info.Number should be uint8_t");
+    assert(RecsByNumber[Number].empty() && "Duplicate IIT_Info.Number");
+    RecsByNumber[Number] = Rec->getName();
+  }
+  if (IIT_Base.size() > 0) {
+    for (unsigned I = 0, E = RecsByNumber.size(); I < E; ++I)
+      if (!RecsByNumber[I].empty())
+        OS << "  " << RecsByNumber[I] << " = " << I << ",\n";
+  } else {
+    OS << "#error \"class IIT_Base is not defined\"\n";
+  }
+  OS << "#endif\n\n";
+}
+
 void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
                                     raw_ostream &OS) {
   OS << "// Target mapping\n";
@@ -226,327 +250,16 @@ void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
   OS << "#endif\n\n";
 }
 
-
-// NOTE: This must be kept in synch with the copy in lib/IR/Function.cpp!
-enum IIT_Info {
-  // Common values should be encoded with 0-15.
-  IIT_Done = 0,
-  IIT_I1 = 1,
-  IIT_I8 = 2,
-  IIT_I16 = 3,
-  IIT_I32 = 4,
-  IIT_I64 = 5,
-  IIT_F16 = 6,
-  IIT_F32 = 7,
-  IIT_F64 = 8,
-  IIT_V2 = 9,
-  IIT_V4 = 10,
-  IIT_V8 = 11,
-  IIT_V16 = 12,
-  IIT_V32 = 13,
-  IIT_PTR = 14,
-  IIT_ARG = 15,
-
-  // Values from 16+ are only encodable with the inefficient encoding.
-  IIT_V64 = 16,
-  IIT_MMX = 17,
-  IIT_TOKEN = 18,
-  IIT_METADATA = 19,
-  IIT_EMPTYSTRUCT = 20,
-  IIT_STRUCT2 = 21,
-  IIT_STRUCT3 = 22,
-  IIT_STRUCT4 = 23,
-  IIT_STRUCT5 = 24,
-  IIT_EXTEND_ARG = 25,
-  IIT_TRUNC_ARG = 26,
-  IIT_ANYPTR = 27,
-  IIT_V1 = 28,
-  IIT_VARARG = 29,
-  IIT_HALF_VEC_ARG = 30,
-  IIT_SAME_VEC_WIDTH_ARG = 31,
-  IIT_PTR_TO_ARG = 32,
-  IIT_PTR_TO_ELT = 33,
-  IIT_VEC_OF_ANYPTRS_TO_ELT = 34,
-  IIT_I128 = 35,
-  IIT_V512 = 36,
-  IIT_V1024 = 37,
-  IIT_STRUCT6 = 38,
-  IIT_STRUCT7 = 39,
-  IIT_STRUCT8 = 40,
-  IIT_F128 = 41,
-  IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43,
-  IIT_SUBDIVIDE2_ARG = 44,
-  IIT_SUBDIVIDE4_ARG = 45,
-  IIT_VEC_OF_BITCASTS_TO_INT = 46,
-  IIT_V128 = 47,
-  IIT_BF16 = 48,
-  IIT_STRUCT9 = 49,
-  IIT_V256 = 50,
-  IIT_AMX = 51,
-  IIT_PPCF128 = 52,
-  IIT_V3 = 53,
-  IIT_EXTERNREF = 54,
-  IIT_FUNCREF = 55,
-  IIT_ANYPTR_TO_ELT = 56,
-  IIT_I2 = 57,
-  IIT_I4 = 58,
-};
-
-static void EncodeFixedValueType(MVT::SimpleValueType VT,
-                                 std::vector<unsigned char> &Sig) {
-  // clang-format off
-  if (MVT(VT).isInteger()) {
-    unsigned BitWidth = MVT(VT).getFixedSizeInBits();
-    switch (BitWidth) {
-    default: PrintFatalError("unhandled integer type width in intrinsic!");
-    case 1: return Sig.push_back(IIT_I1);
-    case 2: return Sig.push_back(IIT_I2);
-    case 4: return Sig.push_back(IIT_I4);
-    case 8: return Sig.push_back(IIT_I8);
-    case 16: return Sig.push_back(IIT_I16);
-    case 32: return Sig.push_back(IIT_I32);
-    case 64: return Sig.push_back(IIT_I64);
-    case 128: return Sig.push_back(IIT_I128);
-    }
-  }
-
-  switch (VT) {
-  default: PrintFatalError("unhandled MVT in intrinsic!");
-  case MVT::f16: return Sig.push_back(IIT_F16);
-  case MVT::bf16: return Sig.push_back(IIT_BF16);
-  case MVT::f32: return Sig.push_back(IIT_F32);
-  case MVT::f64: return Sig.push_back(IIT_F64);
-  case MVT::f128: return Sig.push_back(IIT_F128);
-  case MVT::ppcf128: return Sig.push_back(IIT_PPCF128);
-  case MVT::token: return Sig.push_back(IIT_TOKEN);
-  case MVT::Metadata: return Sig.push_back(IIT_METADATA);
-  case MVT::x86mmx: return Sig.push_back(IIT_MMX);
-  case MVT::x86amx: return Sig.push_back(IIT_AMX);
-  // MVT::OtherVT is used to mean the empty struct type here.
-  case MVT::Other: return Sig.push_back(IIT_EMPTYSTRUCT);
-  // MVT::isVoid is used to represent varargs here.
-  case MVT::isVoid: return Sig.push_back(IIT_VARARG);
-  case MVT::externref:
-    return Sig.push_back(IIT_EXTERNREF);
-  case MVT::funcref:
-    return Sig.push_back(IIT_FUNCREF);
-  }
-  // clang-format on
-}
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#pragma optimize("",off) // MSVC 2015 optimizer can't deal with this function.
-#endif
-
-static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
-                            unsigned &NextArgCode,
-                            std::vector<unsigned char> &Sig,
-                            ArrayRef<unsigned char> Mapping) {
-
-  if (R->isSubClassOf("LLVMMatchType")) {
-    unsigned Number = Mapping[R->getValueAsInt("Number")];
-    assert(Number < ArgCodes.size() && "Invalid matching number!");
-    if (R->isSubClassOf("LLVMExtendedType"))
-      Sig.push_back(IIT_EXTEND_ARG);
-    else if (R->isSubClassOf("LLVMTruncatedType"))
-      Sig.push_back(IIT_TRUNC_ARG);
-    else if (R->isSubClassOf("LLVMHalfElementsVectorType"))
-      Sig.push_back(IIT_HALF_VEC_ARG);
-    else if (R->isSubClassOf("LLVMScalarOrSameVectorWidth")) {
-      Sig.push_back(IIT_SAME_VEC_WIDTH_ARG);
-      Sig.push_back((Number << 3) | ArgCodes[Number]);
-      MVT::SimpleValueType VT = getValueType(R->getValueAsDef("ElTy"));
-      EncodeFixedValueType(VT, Sig);
-      return;
-    }
-    else if (R->isSubClassOf("LLVMPointerTo"))
-      Sig.push_back(IIT_PTR_TO_ARG);
-    else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
-      Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT);
-      // Encode overloaded ArgNo
-      Sig.push_back(NextArgCode++);
-      // Encode LLVMMatchType<Number> ArgNo
-      Sig.push_back(Number);
-      return;
-    } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
-      Sig.push_back(IIT_ANYPTR_TO_ELT);
-      // Encode overloaded ArgNo
-      Sig.push_back(NextArgCode++);
-      // Encode LLVMMatchType<Number> ArgNo
-      Sig.push_back(Number);
-      return;
-    } else if (R->isSubClassOf("LLVMPointerToElt"))
-      Sig.push_back(IIT_PTR_TO_ELT);
-    else if (R->isSubClassOf("LLVMVectorElementType"))
-      Sig.push_back(IIT_VEC_ELEMENT);
-    else if (R->isSubClassOf("LLVMSubdivide2VectorType"))
-      Sig.push_back(IIT_SUBDIVIDE2_ARG);
-    else if (R->isSubClassOf("LLVMSubdivide4VectorType"))
-      Sig.push_back(IIT_SUBDIVIDE4_ARG);
-    else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt"))
-      Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT);
-    else
-      Sig.push_back(IIT_ARG);
-    return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/);
-  }
-
-  MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT"));
-
-  unsigned Tmp = 0;
-  switch (VT) {
-  default: break;
-  case MVT::iPTRAny: ++Tmp; [[fallthrough]];
-  case MVT::vAny: ++Tmp;    [[fallthrough]];
-  case MVT::fAny: ++Tmp;    [[fallthrough]];
-  case MVT::iAny: ++Tmp;    [[fallthrough]];
-  case MVT::Any: {
-    // If this is an "any" valuetype, then the type is the type of the next
-    // type in the list specified to getIntrinsic().
-    Sig.push_back(IIT_ARG);
-
-    // Figure out what arg # this is consuming, and remember what kind it was.
-    assert(NextArgCode < ArgCodes.size() && ArgCodes[NextArgCode] == Tmp &&
-           "Invalid or no ArgCode associated with overloaded VT!");
-    unsigned ArgNo = NextArgCode++;
-
-    // Encode what sort of argument it must be in the low 3 bits of the ArgNo.
-    return Sig.push_back((ArgNo << 3) | Tmp);
-  }
-
-  case MVT::iPTR: {
-    unsigned AddrSpace = 0;
-    if (R->isSubClassOf("LLVMQualPointerType")) {
-      AddrSpace = R->getValueAsInt("AddrSpace");
-      assert(AddrSpace < 256 && "Address space exceeds 255");
-    }
-    if (AddrSpace) {
-      Sig.push_back(IIT_ANYPTR);
-      Sig.push_back(AddrSpace);
-    } else {
-      Sig.push_back(IIT_PTR);
-    }
-    return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig,
-                           Mapping);
-  }
-  }
-
-  if (MVT(VT).isVector()) {
-    MVT VVT = VT;
-    if (VVT.isScalableVector())
-      Sig.push_back(IIT_SCALABLE_VEC);
-    switch (VVT.getVectorMinNumElements()) {
-    default: PrintFatalError("unhandled vector type width in intrinsic!");
-    case 1: Sig.push_back(IIT_V1); break;
-    case 2: Sig.push_back(IIT_V2); break;
-    case 3: Sig.push_back(IIT_V3); break;
-    case 4: Sig.push_back(IIT_V4); break;
-    case 8: Sig.push_back(IIT_V8); break;
-    case 16: Sig.push_back(IIT_V16); break;
-    case 32: Sig.push_back(IIT_V32); break;
-    case 64: Sig.push_back(IIT_V64); break;
-    case 128: Sig.push_back(IIT_V128); break;
-    case 256: Sig.push_back(IIT_V256); break;
-    case 512: Sig.push_back(IIT_V512); break;
-    case 1024: Sig.push_back(IIT_V1024); break;
-    }
-
-    return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig);
-  }
-
-  EncodeFixedValueType(VT, Sig);
-}
-
-static void UpdateArgCodes(Record *R, std::vector<unsigned char> &ArgCodes,
-                           unsigned int &NumInserted,
-                           SmallVectorImpl<unsigned char> &Mapping) {
-  if (R->isSubClassOf("LLVMMatchType")) {
-    if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
-      ArgCodes.push_back(3 /*vAny*/);
-      ++NumInserted;
-    } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
-      ArgCodes.push_back(4 /*iPTRAny*/);
-      ++NumInserted;
-    }
-    return;
-  }
-
-  unsigned Tmp = 0;
-  switch (getValueType(R->getValueAsDef("VT"))) {
-  default: break;
-  case MVT::iPTR:
-    UpdateArgCodes(R->getValueAsDef("ElTy"), ArgCodes, NumInserted, Mapping);
-    break;
-  case MVT::iPTRAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::vAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::fAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::iAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::Any:
-    unsigned OriginalIdx = ArgCodes.size() - NumInserted;
-    assert(OriginalIdx >= Mapping.size());
-    Mapping.resize(OriginalIdx+1);
-    Mapping[OriginalIdx] = ArgCodes.size();
-    ArgCodes.push_back(Tmp);
-    break;
-  }
-}
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#pragma optimize("",on)
-#endif
-
 /// ComputeFixedEncoding - If we can encode the type signature for this
 /// intrinsic into 32 bits, return it.  If not, return ~0U.
 static void ComputeFixedEncoding(const CodeGenIntrinsic &Int,
                                  std::vector<unsigned char> &TypeSig) {
-  std::vector<unsigned char> ArgCodes;
-
-  // Add codes for any overloaded result VTs.
-  unsigned int NumInserted = 0;
-  SmallVector<unsigned char, 8> ArgMapping;
-  for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i)
-    UpdateArgCodes(Int.IS.RetTypeDefs[i], ArgCodes, NumInserted, ArgMapping);
-
-  // Add codes for any overloaded operand VTs.
-  for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i)
-    UpdateArgCodes(Int.IS.ParamTypeDefs[i], ArgCodes, NumInserted, ArgMapping);
-
-  unsigned NextArgCode = 0;
-  if (Int.IS.RetVTs.empty())
-    TypeSig.push_back(IIT_Done);
-  else if (Int.IS.RetVTs.size() == 1 &&
-           Int.IS.RetVTs[0] == MVT::isVoid)
-    TypeSig.push_back(IIT_Done);
-  else {
-    switch (Int.IS.RetVTs.size()) {
-      case 1: break;
-      case 2: TypeSig.push_back(IIT_STRUCT2); break;
-      case 3: TypeSig.push_back(IIT_STRUCT3); break;
-      case 4: TypeSig.push_back(IIT_STRUCT4); break;
-      case 5: TypeSig.push_back(IIT_STRUCT5); break;
-      case 6: TypeSig.push_back(IIT_STRUCT6); break;
-      case 7: TypeSig.push_back(IIT_STRUCT7); break;
-      case 8: TypeSig.push_back(IIT_STRUCT8); break;
-      case 9: TypeSig.push_back(IIT_STRUCT9); break;
-      default: llvm_unreachable("Unhandled case in struct");
+  if (auto *R = Int.TheDef->getValue("TypeSig")) {
+    for (auto &a : cast<ListInit>(R->getValue())->getValues()) {
+      for (auto &b : cast<ListInit>(a)->getValues())
+        TypeSig.push_back(cast<IntInit>(b)->getValue());
     }
-
-    for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i)
-      EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, NextArgCode, TypeSig,
-                      ArgMapping);
   }
-
-  for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i)
-    EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, NextArgCode, TypeSig,
-                    ArgMapping);
 }
 
 static void printIITEntry(raw_ostream &OS, unsigned char X) {
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index 18e98ff9b9f9c..12174fd83f566 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -52,7 +52,6 @@ static void EmitRISCVTargetDef(RecordKeeper &RK, raw_ostream &OS) {
      << "#define PROC(ENUM, NAME, DEFAULT_MARCH)\n"
      << "#endif\n\n";
 
-  OS << "PROC(INVALID, {\"invalid\"}, {\"\"})\n";
   // Iterate on all definition records.
   for (const Record *Rec : RK.getAllDerivedDefinitions("RISCVProcessorModel")) {
     std::string MArch = Rec->getValueAsString("DefaultMarch").str();
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 0c32a6aa51cc6..f35dd368cf667 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -25,10 +25,10 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SparseBitVector.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Format.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
diff --git a/llvm/utils/TableGen/VTEmitter.cpp b/llvm/utils/TableGen/VTEmitter.cpp
index f7ae0f5bc38e2..d398a7e7b58f4 100644
--- a/llvm/utils/TableGen/VTEmitter.cpp
+++ b/llvm/utils/TableGen/VTEmitter.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
@@ -63,26 +62,41 @@ void VTEmitter::run(raw_ostream &OS) {
     }
   };
 
-  OS << "#ifdef GET_VT_ATTR // (Ty, n, sz)\n";
+  OS << "#ifdef GET_VT_ATTR // (Ty, n, sz, Any, Int, FP, Vec, Sc)\n";
   for (const auto *VT : VTsByNumber) {
     if (!VT)
       continue;
-    auto Name = VT->getName();
-    Name = StringSwitch<StringRef>(Name)
-               .Case("OtherVT", "Other")
-               .Case("FlagVT", "Glue")
-               .Case("untyped", "Untyped")
-               .Case("MetadataVT", "Metadata")
-               .Default(Name);
+    auto Name = VT->getValueAsString("LLVMName");
     auto Value = VT->getValueAsInt("Value");
-
+    bool IsInteger = VT->getValueAsInt("isInteger");
+    bool IsFP = VT->getValueAsInt("isFP");
+    bool IsVector = VT->getValueAsInt("isVector");
+    bool IsScalable = VT->getValueAsInt("isScalable");
+
+    UpdateVTRange("INTEGER_FIXEDLEN_VECTOR_VALUETYPE", Name,
+                  IsInteger && IsVector && !IsScalable);
+    UpdateVTRange("INTEGER_SCALABLE_VECTOR_VALUETYPE", Name,
+                  IsInteger && IsScalable);
+    UpdateVTRange("FP_FIXEDLEN_VECTOR_VALUETYPE", Name,
+                  IsFP && IsVector && !IsScalable);
+    UpdateVTRange("FP_SCALABLE_VECTOR_VALUETYPE", Name, IsFP && IsScalable);
+    UpdateVTRange("FIXEDLEN_VECTOR_VALUETYPE", Name, IsVector && !IsScalable);
+    UpdateVTRange("SCALABLE_VECTOR_VALUETYPE", Name, IsScalable);
+    UpdateVTRange("VECTOR_VALUETYPE", Name, IsVector);
+    UpdateVTRange("INTEGER_VALUETYPE", Name, IsInteger && !IsVector);
+    UpdateVTRange("FP_VALUETYPE", Name, IsFP && !IsVector);
     UpdateVTRange("VALUETYPE", Name, Value < 224);
 
     // clang-format off
     OS << "  GET_VT_ATTR("
        << Name << ", "
        << Value << ", "
-       << VT->getValueAsInt("Size") << ")\n";
+       << VT->getValueAsInt("Size") << ", "
+       << VT->getValueAsInt("isOverloaded") << ", "
+       << (IsInteger ? Name[0] == 'i' ? 3 : 1 : 0) << ", "
+       << (IsFP ? Name[0] == 'f' ? 3 : 1 : 0) << ", "
+       << IsVector << ", "
+       << IsScalable << ")\n";
     // clang-format on
   }
   OS << "#endif\n\n";
@@ -94,6 +108,23 @@ void VTEmitter::run(raw_ostream &OS) {
        << "  LAST_" << KV.first << " = " << KV.second.Last << ",\n";
   }
   OS << "#endif\n\n";
+
+  OS << "#ifdef GET_VT_VECATTR // (Ty, Sc, nElem, ElTy, ElSz)\n";
+  for (const auto *VT : VTsByNumber) {
+    if (!VT || !VT->getValueAsInt("isVector"))
+      continue;
+    const auto *ElTy = VT->getValueAsDef("ElementType");
+    assert(ElTy);
+    // clang-format off
+    OS << "  GET_VT_VECATTR("
+       << VT->getValueAsString("LLVMName") << ", "
+       << VT->getValueAsInt("isScalable") << ", "
+       << VT->getValueAsInt("nElem") << ", "
+       << ElTy->getName() << ", "
+       << ElTy->getValueAsInt("Size") << ")\n";
+    // clang-format on
+  }
+  OS << "#endif\n\n";
 }
 
 static TableGen::Emitter::OptClass<VTEmitter> X("gen-vt", "Generate ValueType");
diff --git a/llvm/utils/emacs/llvm-mode.el b/llvm/utils/emacs/llvm-mode.el
index c6d594d118d0b..bc8dd2f12d2f8 100644
--- a/llvm/utils/emacs/llvm-mode.el
+++ b/llvm/utils/emacs/llvm-mode.el
@@ -36,7 +36,7 @@
    '("%[-]?[0-9]+" . font-lock-variable-name-face)
    ;; Types
    `(,(regexp-opt
-       '("void" "i1" "i8" "i16" "i32" "i64" "i128" "half" "bfloat" "float" "double" "fp128" "x86_fp80" "ppc_fp128" "x86_mmx" "x86_amx"
+       '("void" "i1" "i8" "i16" "i32" "i64" "i128" "half" "bfloat" "float" "double" "fp128" "x86_fp80" "ppc_fp128" "x86_mmx" "x86_amx" "ptr"
          "type" "label" "opaque" "token") 'symbols) . font-lock-type-face)
    ;; Integer literals
    '("\\b[-]?[0-9]+\\b" . font-lock-preprocessor-face)
diff --git a/llvm/utils/gn/build/sync_source_lists_from_cmake.py b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
index 7ecff9ddd1398..fe7e8ed10bcf5 100755
--- a/llvm/utils/gn/build/sync_source_lists_from_cmake.py
+++ b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
@@ -73,7 +73,13 @@ def find_gitrev(touched_line, in_file):
         # undefined behavior according to the POSIX extended regex spec.
         posix_re_escape = lambda s: re.sub(r'([.[{()\\*+?|^$])', r'\\\1', s)
         cmd = ['log', '--format=%h', '-1', '--pickaxe-regex',
-               r'-S\b%s\b' % posix_re_escape(touched_line), in_file]
+               # `\<` / `\>` cause issues on Windows (and is a GNU extension).
+               # `\b` is a GNU extension and stopped working in Apple Git-143
+               # (Xcode 13.3).
+               # `[:space:]` is over 10x faster than `^[:alnum:]` and hopefully
+               # good enough.
+               r'-S[[:space:]]%s[[:space:]]' % posix_re_escape(touched_line),
+               in_file]
         return git_out(cmd).rstrip()
 
     # Collect changes to gn files, grouped by revision.
diff --git a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
index 9c1249abfb9ff..3d683e7b0f9e2 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
@@ -24,6 +24,7 @@ static_library("Core") {
     "DynoStats.cpp",
     "Exceptions.cpp",
     "FunctionLayout.cpp",
+    "HashUtilities.cpp",
     "JumpTable.cpp",
     "MCPlusBuilder.cpp",
     "ParallelUtilities.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
index 30ee5f7415969..1687dd3f8debc 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
@@ -43,6 +43,7 @@ static_library("bugprone") {
     "MisplacedPointerArithmeticInAllocCheck.cpp",
     "MisplacedWideningCastCheck.cpp",
     "MoveForwardingReferenceCheck.cpp",
+    "MultipleNewInOneExpressionCheck.cpp",
     "MultipleStatementMacroCheck.cpp",
     "NoEscapeCheck.cpp",
     "NonZeroEnumToBoolConversionCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
index f94cdd0264a66..40b7ba3780c97 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
@@ -163,3 +163,18 @@ clang_tablegen("riscv_vector_builtin_sema") {
   args = [ "-gen-riscv-vector-builtin-sema" ]
   td_file = "riscv_vector.td"
 }
+
+clang_tablegen("riscv_sifive_vector_builtins") {
+  args = [ "-gen-riscv-sifive-vector-builtins" ]
+  td_file = "riscv_sifive_vector.td"
+}
+
+clang_tablegen("riscv_sifive_vector_builtin_cg") {
+  args = [ "-gen-riscv-sifive-vector-builtin-codegen" ]
+  td_file = "riscv_sifive_vector.td"
+}
+
+clang_tablegen("riscv_sifive_vector_builtin_sema") {
+  args = [ "-gen-riscv-sifive-vector-builtin-sema" ]
+  td_file = "riscv_sifive_vector.td"
+}
diff --git a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
index 639276f48c84b..1f7957e4cb490 100644
--- a/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Basic/BUILD.gn
@@ -29,6 +29,7 @@ static_library("Basic") {
     "//clang/include/clang/Basic:arm_sve_builtins",
     "//clang/include/clang/Basic:arm_sve_typeflags",
     "//clang/include/clang/Basic:diags_tablegen",
+    "//clang/include/clang/Basic:riscv_sifive_vector_builtins",
     "//clang/include/clang/Basic:riscv_vector_builtins",
     "//clang/include/clang/Basic:version",
 
diff --git a/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
index 35041fed8e5e3..cfb9c993d57c8 100644
--- a/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
@@ -5,6 +5,7 @@ static_library("CodeGen") {
     "//clang/include/clang/Basic:arm_cde_builtin_cg",
     "//clang/include/clang/Basic:arm_mve_builtin_cg",
     "//clang/include/clang/Basic:arm_sve_builtin_cg",
+    "//clang/include/clang/Basic:riscv_sifive_vector_builtin_cg",
     "//clang/include/clang/Basic:riscv_vector_builtin_cg",
     "//clang/lib/AST",
     "//clang/lib/Analysis",
@@ -13,6 +14,7 @@ static_library("CodeGen") {
     "//clang/lib/Lex",
     "//llvm/lib/Analysis",
     "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/Demangle",
     "//llvm/lib/Frontend/HLSL",
     "//llvm/lib/IR",
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index f39ba39a273c6..1c77969c2631f 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -148,6 +148,7 @@ copy("Headers") {
     "cpuid.h",
     "crc32intrin.h",
     "cuda_wrappers/algorithm",
+    "cuda_wrappers/bits/shared_ptr_base.h",
     "cuda_wrappers/complex",
     "cuda_wrappers/new",
     "emmintrin.h",
@@ -216,11 +217,13 @@ copy("Headers") {
     "raointintrin.h",
     "rdpruintrin.h",
     "rdseedintrin.h",
+    "riscv_ntlh.h",
     "rtmintrin.h",
     "s390intrin.h",
     "serializeintrin.h",
     "sgxintrin.h",
     "shaintrin.h",
+    "sifive_vector.h",
     "smmintrin.h",
     "stdalign.h",
     "stdarg.h",
diff --git a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
index 6bfac4537518d..24e25cfd32fcc 100644
--- a/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -14,6 +14,7 @@ static_library("Sema") {
     "//clang/include/clang/Basic:arm_mve_builtin_aliases",
     "//clang/include/clang/Basic:arm_mve_builtin_sema",
     "//clang/include/clang/Basic:arm_sve_sema_rangechecks",
+    "//clang/include/clang/Basic:riscv_sifive_vector_builtin_sema",
     "//clang/include/clang/Basic:riscv_vector_builtin_sema",
     "//clang/include/clang/Sema:AttrParsedAttrImpl",
     "//clang/include/clang/Sema:AttrParsedAttrKinds",
diff --git a/llvm/utils/gn/secondary/clang/tools/clang-scan-deps/BUILD.gn b/llvm/utils/gn/secondary/clang/tools/clang-scan-deps/BUILD.gn
index 48a9ddc7746c7..f29c8ab1c2ba9 100644
--- a/llvm/utils/gn/secondary/clang/tools/clang-scan-deps/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/tools/clang-scan-deps/BUILD.gn
@@ -1,5 +1,5 @@
-import("//llvm/utils/gn/build/driver_executable.gni")
 import("//llvm/utils/TableGen/tablegen.gni")
+import("//llvm/utils/gn/build/driver_executable.gni")
 
 tablegen("Opts") {
   visibility = [ ":clang-scan-deps" ]
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn
index 19ddd4ea89c00..2fe08699bca3f 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/BUILD.gn
@@ -20,29 +20,3 @@ group("scudo") {
   }
   testonly = true
 }
-
-# This target is unused, it only exists to satisfy
-# sync_source_lists_from_cmake.py.
-source_set("sources") {
-  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
-  configs += [ "//llvm/utils/gn/build:crt_code" ]
-  sources = [
-    "scudo_allocator.cpp",
-    "scudo_allocator.h",
-    "scudo_allocator_combined.h",
-    "scudo_allocator_secondary.h",
-    "scudo_crc32.cpp",
-    "scudo_crc32.h",
-    "scudo_errors.cpp",
-    "scudo_errors.h",
-    "scudo_flags.cpp",
-    "scudo_flags.h",
-    "scudo_interface_internal.h",
-    "scudo_malloc.cpp",
-    "scudo_platform.h",
-    "scudo_termination.cpp",
-    "scudo_tsd.h",
-    "scudo_tsd_exclusive.cpp",
-    "scudo_tsd_shared.cpp",
-  ]
-}
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 29f2b656a8e29..c44f64ba041c2 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -144,6 +144,10 @@ if (current_toolchain == default_toolchain) {
       "__algorithm/partition_point.h",
       "__algorithm/pop_heap.h",
       "__algorithm/prev_permutation.h",
+      "__algorithm/pstl_any_all_none_of.h",
+      "__algorithm/pstl_fill.h",
+      "__algorithm/pstl_find.h",
+      "__algorithm/pstl_for_each.h",
       "__algorithm/push_heap.h",
       "__algorithm/ranges_adjacent_find.h",
       "__algorithm/ranges_all_of.h",
@@ -460,11 +464,17 @@ if (current_toolchain == default_toolchain) {
       "__functional/unwrap_ref.h",
       "__functional/weak_result_type.h",
       "__fwd/array.h",
+      "__fwd/fstream.h",
       "__fwd/get.h",
       "__fwd/hash.h",
+      "__fwd/ios.h",
+      "__fwd/istream.h",
       "__fwd/memory_resource.h",
+      "__fwd/ostream.h",
       "__fwd/pair.h",
       "__fwd/span.h",
+      "__fwd/sstream.h",
+      "__fwd/streambuf.h",
       "__fwd/string.h",
       "__fwd/string_view.h",
       "__fwd/subrange.h",
@@ -569,6 +579,38 @@ if (current_toolchain == default_toolchain) {
       "__numeric/transform_exclusive_scan.h",
       "__numeric/transform_inclusive_scan.h",
       "__numeric/transform_reduce.h",
+      "__pstl/internal/algorithm_fwd.h",
+      "__pstl/internal/algorithm_impl.h",
+      "__pstl/internal/execution_defs.h",
+      "__pstl/internal/execution_impl.h",
+      "__pstl/internal/glue_algorithm_defs.h",
+      "__pstl/internal/glue_algorithm_impl.h",
+      "__pstl/internal/glue_memory_defs.h",
+      "__pstl/internal/glue_memory_impl.h",
+      "__pstl/internal/glue_numeric_defs.h",
+      "__pstl/internal/glue_numeric_impl.h",
+      "__pstl/internal/memory_impl.h",
+      "__pstl/internal/numeric_fwd.h",
+      "__pstl/internal/numeric_impl.h",
+      "__pstl/internal/omp/parallel_for.h",
+      "__pstl/internal/omp/parallel_for_each.h",
+      "__pstl/internal/omp/parallel_invoke.h",
+      "__pstl/internal/omp/parallel_merge.h",
+      "__pstl/internal/omp/parallel_scan.h",
+      "__pstl/internal/omp/parallel_stable_partial_sort.h",
+      "__pstl/internal/omp/parallel_stable_sort.h",
+      "__pstl/internal/omp/parallel_transform_reduce.h",
+      "__pstl/internal/omp/parallel_transform_scan.h",
+      "__pstl/internal/omp/util.h",
+      "__pstl/internal/parallel_backend.h",
+      "__pstl/internal/parallel_backend_omp.h",
+      "__pstl/internal/parallel_backend_serial.h",
+      "__pstl/internal/parallel_backend_tbb.h",
+      "__pstl/internal/parallel_backend_utils.h",
+      "__pstl/internal/parallel_impl.h",
+      "__pstl/internal/pstl_config.h",
+      "__pstl/internal/unseq_backend_simd.h",
+      "__pstl/internal/utils.h",
       "__random/bernoulli_distribution.h",
       "__random/binomial_distribution.h",
       "__random/cauchy_distribution.h",
@@ -645,6 +687,7 @@ if (current_toolchain == default_toolchain) {
       "__ranges/views.h",
       "__ranges/zip_view.h",
       "__split_buffer",
+      "__std_mbstate_t.h",
       "__string/char_traits.h",
       "__string/constexpr_c_functions.h",
       "__string/extern_template_lists.h",
@@ -732,6 +775,7 @@ if (current_toolchain == default_toolchain) {
       "__type_traits/is_empty.h",
       "__type_traits/is_enum.h",
       "__type_traits/is_equality_comparable.h",
+      "__type_traits/is_execution_policy.h",
       "__type_traits/is_final.h",
       "__type_traits/is_floating_point.h",
       "__type_traits/is_function.h",
@@ -832,6 +876,7 @@ if (current_toolchain == default_toolchain) {
       "__utility/priority_tag.h",
       "__utility/rel_ops.h",
       "__utility/swap.h",
+      "__utility/terminate_on_exception.h",
       "__utility/to_underlying.h",
       "__utility/unreachable.h",
       "__variant/monostate.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
index 02af6caf17054..0a60a51ab6253 100644
--- a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
@@ -58,7 +58,6 @@ static_library("Core") {
     "DynamicLoader.cpp",
     "EmulateInstruction.cpp",
     "FileLineResolver.cpp",
-    "FileSpecList.cpp",
     "FormatEntity.cpp",
     "Highlighter.cpp",
     "IOHandler.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
index d030f7eda2aa2..18f48f87064df 100644
--- a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
@@ -22,6 +22,7 @@ static_library("Utility") {
     "Environment.cpp",
     "Event.cpp",
     "FileSpec.cpp",
+    "FileSpecList.cpp",
     "GDBRemote.cpp",
     "IOObject.cpp",
     "Instrumentation.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn b/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn
index 37082746e098d..bb1f3ad20b9d3 100644
--- a/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/tools/lldb-server/BUILD.gn
@@ -39,6 +39,7 @@ executable("lldb-server") {
       #deps += [ "//lldb/source/Plugins/Platform/Android" ]
     }
   }
+
   #} else if (current_os == "freebsd") {
   #deps += [ "//lldb/source/Platform/Process/FreeBSD" ]
   #deps += [ "//lldb/source/Plugins/Process/FreeBSD" ]
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/CodeGen/BUILD.gn
new file mode 100644
index 0000000000000..fdbe4f92d564e
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/CodeGen/BUILD.gn
@@ -0,0 +1,7 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("GenVT") {
+  args = [ "-gen-vt" ]
+  td_file = "//llvm/include/llvm/CodeGen/ValueTypes.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
+}
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 393744f2d6c53..07fdcfa908677 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -49,6 +49,9 @@ declare_args() {
   # Iterate unordered llvm containers in reverse.
   llvm_enable_reverse_iteration = false
 
+  # Force enable statistics.
+  llvm_force_enable_stats = false
+
   # glibc is at least 2.33 which has mallinfo2.
   # TODO: remove this once nobody using the gn build is building against an old glibc.
   llvm_have_mallinfo2 = current_os == "linux"
@@ -321,7 +324,6 @@ write_cmake_config("llvm-config") {
     "LLVM_ENABLE_DUMP=",
     "LLVM_ENABLE_HTTPLIB=",
     "LLVM_ENABLE_ZSTD=",
-    "LLVM_FORCE_ENABLE_STATS=",
     "LLVM_FORCE_USE_OLD_TOOLCHAIN=",
     "LLVM_HAS_ATOMICS=1",
     "LLVM_HAVE_TFLITE=",
@@ -403,6 +405,12 @@ write_cmake_config("llvm-config") {
   } else {
     values += [ "LLVM_ENABLE_CURL=" ]
   }
+
+  if (llvm_force_enable_stats) {
+    values += [ "LLVM_FORCE_ENABLE_STATS=1" ]
+  } else {
+    values += [ "LLVM_FORCE_ENABLE_STATS=" ]
+  }
 }
 
 ##############################################################################
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
index c6d80b2be3f6b..6ce92c9149f0f 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
@@ -4,6 +4,7 @@ tablegen("ACC") {
   visibility = [ ":acc_gen" ]
   args = [ "-gen-directive-decl" ]
   output_name = "ACC.h.inc"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 tablegen("ACCcpp") {
@@ -11,6 +12,7 @@ tablegen("ACCcpp") {
   args = [ "-gen-directive-impl" ]
   output_name = "ACC.inc"
   td_file = "ACC.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 group("acc_gen") {
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
index d99b8b5a1e27d..739f3383c1740 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
@@ -5,11 +5,13 @@ tablegen("OMPh") {
   args = [ "-gen-directive-decl" ]
   output_name = "OMP.h.inc"
   td_file = "OMP.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 tablegen("OMP") {
   visibility = [ ":public_tablegen" ]
   args = [ "-gen-directive-impl" ]
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 # Groups all tablegen() calls that create .inc files that are included in
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
index 148fe951b432e..a594d2afbf7d6 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
@@ -3,18 +3,21 @@ import("//llvm/utils/TableGen/tablegen.gni")
 tablegen("Attributes") {
   visibility = [ ":public_tablegen" ]
   args = [ "-gen-attrs" ]
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 tablegen("IntrinsicImpl") {
   visibility = [ "//llvm/lib/IR" ]
   args = [ "-gen-intrinsic-impl" ]
   td_file = "Intrinsics.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 tablegen("IntrinsicEnums") {
   visibility = [ ":public_tablegen" ]
   args = [ "-gen-intrinsic-enums" ]
   td_file = "Intrinsics.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 template("gen_arch_intrinsics") {
@@ -26,6 +29,7 @@ template("gen_arch_intrinsics") {
       "-intrinsic-prefix=${invoker.intrinsic_prefix}",
     ]
     td_file = "Intrinsics.td"
+    tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
   }
 }
 
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn
index 0644daaa9daa7..aecb65ab6c728 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn
@@ -4,6 +4,7 @@ tablegen("RISCVTargetParserDef") {
   visibility = [ ":gen" ]
   args = [ "-gen-riscv-target-def" ]
   td_file = "//llvm/lib/Target/RISCV/RISCV.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
 group("gen") {
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
index bc37796e1a2d8..e13650d160f45 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
@@ -1,3 +1,18 @@
+static_library("CodeGenTypes") {
+  output_name = "LLVMCodeGenType"
+  public_deps = [
+    # Must be a public_dep because CodeGen's headers include llvm-config.h.
+    "//llvm/include/llvm/Config:llvm-config",
+  ]
+  deps = [
+    "//llvm/include/llvm/CodeGen:GenVT",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "LowLevelType.cpp"
+  ]
+}
+
 static_library("CodeGen") {
   output_name = "LLVMCodeGen"
   public_deps = [
@@ -5,6 +20,7 @@ static_library("CodeGen") {
     "//llvm/include/llvm/Config:llvm-config",
   ]
   deps = [
+    "//llvm/include/llvm/CodeGen:GenVT",
     "//llvm/lib/Analysis",
     "//llvm/lib/Bitcode/Reader",
     "//llvm/lib/Bitcode/Writer",
@@ -99,6 +115,7 @@ static_library("CodeGen") {
     "LocalStackSlotAllocation.cpp",
     "LoopTraversal.cpp",
     "LowLevelType.cpp",
+    "LowLevelTypeUtils.cpp",
     "LowerEmuTLS.cpp",
     "MBFIWrapper.cpp",
     "MIRCanonicalizerPass.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn
index 05e16e71a722e..3e381d975c1f0 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Object/BUILD.gn
@@ -28,6 +28,7 @@ static_library("Object") {
     "ELFObjectFile.cpp",
     "Error.cpp",
     "FaultMapParser.cpp",
+    "GOFFObjectFile.cpp",
     "IRObjectFile.cpp",
     "IRSymtab.cpp",
     "MachOObjectFile.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 0f19232914f66..487cb49a0dfcf 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -61,6 +61,7 @@ static_library("Support") {
     "CodeGenCoverage.cpp",
     "CommandLine.cpp",
     "Compression.cpp",
+    "ConvertEBCDIC.cpp",
     "ConvertUTF.cpp",
     "ConvertUTFWrapper.cpp",
     "CrashRecoveryContext.cpp",
@@ -96,7 +97,6 @@ static_library("Support") {
     "LineIterator.cpp",
     "Locale.cpp",
     "LockFileManager.cpp",
-    "LowLevelType.cpp",
     "MD5.cpp",
     "MSP430AttributeParser.cpp",
     "MSP430Attributes.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn
index 96c7dd41c6143..421eef1ec2c5f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn
@@ -13,6 +13,7 @@ static_library("AsmParser") {
   output_name = "LLVMAArch64AsmParser"
   deps = [
     ":AArch64GenAsmMatcher",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn
index 0f2cccac53290..e0a4362c2e168 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn
@@ -61,6 +61,7 @@ static_library("MCTargetDesc") {
     ":AArch64GenAsmWriter",
     ":AArch64GenAsmWriter1",
     ":AArch64GenMCCodeEmitter",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/Target/AArch64/TargetInfo",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/AsmParser/BUILD.gn
index 9c3d0fc0cd11b..b88679d69f6bc 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/AsmParser/BUILD.gn
@@ -10,6 +10,7 @@ static_library("AsmParser") {
   output_name = "LLVMAMDGPUAsmParser"
   deps = [
     ":AMDGPUGenAsmMatcher",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Disassembler/BUILD.gn
index 07985dd380e75..11bc537936508 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Disassembler/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Disassembler/BUILD.gn
@@ -10,6 +10,7 @@ static_library("Disassembler") {
   output_name = "LLVMAMDGPUDisassembler"
   deps = [
     ":AMDGPUGenDisassemblerTables",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCA/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCA/BUILD.gn
index a1cab620c4f1b..317d7e8f8dd0d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCA/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCA/BUILD.gn
@@ -1,6 +1,7 @@
 static_library("MCA") {
   output_name = "LLVMAMDGPUTargetMCA"
   deps = [
+    "//llvm/lib/CodeGen",
     "//llvm/lib/Target/AMDGPU/MCTargetDesc",
     "//llvm/lib/Target/AMDGPU/Utils",
     "//llvm/lib/TargetParser",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
index acd59e46db10c..7041899d99cd5 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
@@ -87,6 +87,7 @@ static_library("MCTargetDesc") {
     ":R600GenAsmWriter",
     ":R600GenMCCodeEmitter",
     "//llvm/lib/BinaryFormat",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/IR",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn
index 9ff4776c95a9f..631d1ef5c7b09 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/Utils/BUILD.gn
@@ -14,6 +14,7 @@ static_library("Utils") {
     # includes the generated AMDGPUGenRegisterBank.inc file :/
     "../:AMDGPUGenRegisterBank",
     "//llvm/lib/BinaryFormat",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/IR",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn
index 584109aa8c2b2..9f165939c70b3 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn
@@ -10,6 +10,7 @@ static_library("AsmParser") {
   output_name = "LLVMARMAsmParser"
   deps = [
     ":ARMGenAsmMatcher",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn
index a8e800531dc20..12add5188a9d4 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn
@@ -10,6 +10,7 @@ static_library("Disassembler") {
   output_name = "LLVMARMDisassembler"
   deps = [
     ":ARMGenDisassemblerTables",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
index 5acbb391d0f32..981639faf71d3 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
@@ -51,6 +51,7 @@ static_library("MCTargetDesc") {
   deps = [
     ":ARMGenAsmWriter",
     ":ARMGenMCCodeEmitter",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn
index 8f37f20be4b10..5014326dcee23 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn
@@ -10,6 +10,7 @@ static_library("AsmParser") {
   output_name = "LLVMAVRAsmParser"
   deps = [
     ":AVRGenAsmMatcher",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn
index 4f7c3acbc8612..dded556b786fb 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn
@@ -10,6 +10,7 @@ static_library("Disassembler") {
   output_name = "LLVMAVRDisassembler"
   deps = [
     ":AVRGenDisassemblerTables",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Support",
     "//llvm/lib/Target/AVR:LLVMAVRCodeGen",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/AsmParser/BUILD.gn
index 72c98631be07b..5b8674b6b0f5e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/AsmParser/BUILD.gn
@@ -10,6 +10,7 @@ static_library("AsmParser") {
   output_name = "LLVMLanaiAsmParser"
   deps = [
     ":LanaiGenAsmMatcher",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/Disassembler/BUILD.gn
index 3a5ef16b54c76..e3a7db5fa30c2 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/Disassembler/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/Lanai/Disassembler/BUILD.gn
@@ -10,6 +10,7 @@ static_library("Disassembler") {
   output_name = "LLVMLanaiDisassembler"
   deps = [
     ":LanaiGenDisassemblerTables",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Support",
     "//llvm/lib/Target/Lanai/MCTargetDesc",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/Mips/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/Mips/MCTargetDesc/BUILD.gn
index acea4e41a940f..059321503f5b6 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/Mips/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/Mips/MCTargetDesc/BUILD.gn
@@ -51,6 +51,7 @@ static_library("MCTargetDesc") {
   deps = [
     ":MipsGenAsmWriter",
     ":MipsGenMCCodeEmitter",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/Target/Mips/TargetInfo",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/NVPTX/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/NVPTX/BUILD.gn
index 2231f6f6a1d9b..a6d1eeaaad70d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/NVPTX/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/NVPTX/BUILD.gn
@@ -33,6 +33,7 @@ static_library("LLVMNVPTXCodeGen") {
     "NVPTXAsmPrinter.cpp",
     "NVPTXAssignValidGlobalNames.cpp",
     "NVPTXAtomicLower.cpp",
+    "NVPTXCtorDtorLowering.cpp",
     "NVPTXFrameLowering.cpp",
     "NVPTXGenericToNVVM.cpp",
     "NVPTXISelDAGToDAG.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
index 37ca14bfe3224..97b21360b02bd 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
@@ -44,6 +44,7 @@ static_library("MCTargetDesc") {
   deps = [
     ":PPCGenAsmWriter",
     ":PPCGenMCCodeEmitter",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/Target/PowerPC/TargetInfo",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index 390f7a1e284d2..87c23aeb9262c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -75,7 +75,6 @@ static_library("LLVMRISCVCodeGen") {
     "RISCVGatherScatterLowering.cpp",
     "RISCVISelDAGToDAG.cpp",
     "RISCVISelLowering.cpp",
-    "RISCVInsertNTLHInsts.cpp",
     "RISCVInsertVSETVLI.cpp",
     "RISCVInstrInfo.cpp",
     "RISCVMCInstLower.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
index 25504b8614f16..580dfe10f4060 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/MCTargetDesc/BUILD.gn
@@ -44,6 +44,7 @@ static_library("MCTargetDesc") {
   deps = [
     ":SystemZGenAsmWriter",
     ":SystemZGenMCCodeEmitter",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/Target/SystemZ/TargetInfo",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn
index 4413d045c37ff..fabf7579c2910 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn
@@ -10,6 +10,7 @@ static_library("AsmParser") {
   output_name = "LLVMWebAssemblyAsmParser"
   deps = [
     ":WebAssemblyGenAsmMatcher",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn
index e78002eb27bec..48b7e9fcc402f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn
@@ -10,6 +10,7 @@ static_library("Disassembler") {
   output_name = "LLVMWebAssemblyDisassembler"
   deps = [
     ":WebAssemblyGenDisassemblerTables",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn
index a3e71d8f78595..325c932b1cea0 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn
@@ -48,6 +48,7 @@ static_library("MCTargetDesc") {
   deps = [
     ":WebAssemblyGenAsmWriter",
     ":WebAssemblyGenMCCodeEmitter",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/Target/WebAssembly/TargetInfo",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCA/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCA/BUILD.gn
index d8dba5df37cdf..cf863a6b8d687 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCA/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCA/BUILD.gn
@@ -1,6 +1,7 @@
 static_library("MCA") {
   output_name = "LLVMTargetX86MCA"
   deps = [
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
     "//llvm/lib/MCA",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCTargetDesc/BUILD.gn
index 2379eefeaf4f8..27df3149b9cfb 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/MCTargetDesc/BUILD.gn
@@ -57,6 +57,7 @@ static_library("MCTargetDesc") {
   deps = [
     ":X86GenAsmWriter",
     ":X86GenAsmWriter1",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index 035ed661dce39..c07ac5b9eac21 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -66,6 +66,7 @@ static_library("Utils") {
     "MetaRenamer.cpp",
     "MisExpect.cpp",
     "ModuleUtils.cpp",
+    "MoveAutoInit.cpp",
     "NameAnonGlobals.cpp",
     "PredicateInfo.cpp",
     "PromoteMemoryToRegister.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
index 685a0f73b6553..4f75212b40912 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
@@ -8,6 +8,7 @@ tablegen("Options") {
 executable("llvm-dwarfutil") {
   deps = [
     ":Options",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/DWARFLinker",
     "//llvm/lib/DWARFLinkerParallel",
     "//llvm/lib/DebugInfo/DWARF",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn
index f97baf0c5a042..41dd853e40efd 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn
@@ -4,6 +4,7 @@ executable("llvm-exegesis") {
   deps = [
     "lib",
     "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/ExecutionEngine",
     "//llvm/lib/ExecutionEngine/MCJIT",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn
index 58eea54e5ca0e..3657820cca024 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn
@@ -9,6 +9,7 @@ static_library("AArch64") {
   output_name = "LLVMExegesisAArch64"
   deps = [
     ":AArch64GenExegesis",
+    "//llvm/lib/CodeGen",
 
     # Exegesis reaches inside the Target/AArch64 tablegen internals and must
     # depend on these Target/AArch64-internal build targets.
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/Mips/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/Mips/BUILD.gn
index f02c9394e573d..fdb6c17607d3c 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/Mips/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/Mips/BUILD.gn
@@ -9,6 +9,7 @@ static_library("Mips") {
   output_name = "LLVMExegesisMips"
   deps = [
     ":MipsGenExegesis",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/TargetParser",
 
     # Exegesis reaches inside the Target/Mips tablegen internals and must
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn
index 2fd2c073833ff..fcd6b423c9a68 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn
@@ -9,6 +9,7 @@ static_library("PowerPC") {
   output_name = "LLVMExegesisPowerPC"
   deps = [
     ":PPCGenExegesis",
+    "//llvm/lib/CodeGen",
     "//llvm/lib/TargetParser",
 
     # Exegesis reaches inside the Target/PowerPC tablegen internals and must
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-gsymutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-gsymutil/BUILD.gn
index 3a5382bb16d46..733e554c1758c 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-gsymutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-gsymutil/BUILD.gn
@@ -1,8 +1,18 @@
-executable("llvm-gsymutil") {
+import("//llvm/utils/TableGen/tablegen.gni")
+import("//llvm/utils/gn/build/driver_executable.gni")
+
+tablegen("Opts") {
+  visibility = [ ":llvm-gsymutil" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
+driver_executable("llvm-gsymutil") {
   deps = [
+    ":Opts",
     "//llvm/lib/DebugInfo/DWARF",
     "//llvm/lib/DebugInfo/GSYM",
     "//llvm/lib/Object",
+    "//llvm/lib/Option",
     "//llvm/lib/Support",
     "//llvm/lib/Target",
     "//llvm/lib/Target:TargetsToBuild",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
index b77a5c11b0461..91e39b70826f5 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
@@ -2,6 +2,7 @@ import("//third-party/unittest/unittest.gni")
 
 unittest("DebugInfoDWARFTests") {
   deps = [
+    "//llvm/lib/CodeGen",
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DebugInfo/DWARF",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Object/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Object/BUILD.gn
index 53f4b87f118ac..73ffc9bd7f12a 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Object/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Object/BUILD.gn
@@ -14,6 +14,7 @@ unittest("ObjectTests") {
     "ELFObjectFileTest.cpp",
     "ELFTest.cpp",
     "ELFTypesTest.cpp",
+    "GOFFObjectFileTest.cpp",
     "MinidumpTest.cpp",
     "ObjectFileTest.cpp",
     "OffloadingTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index d872973c177f7..a33302865a84a 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -1,15 +1,7 @@
-import("//llvm/utils/TableGen/tablegen.gni")
 import("//third-party/unittest/unittest.gni")
 
-tablegen("GenVT") {
-  visibility = [ ":SupportTests" ]
-  args = [ "-gen-vt" ]
-  td_file = "//llvm/include/llvm/CodeGen/ValueTypes.td"
-}
-
 unittest("SupportTests") {
   deps = [
-    ":GenVT",
     "//llvm/lib/Support",
     "//llvm/lib/TargetParser",
     "//llvm/lib/Testing/Support",
@@ -34,6 +26,7 @@ unittest("SupportTests") {
     "Chrono.cpp",
     "CommandLineTest.cpp",
     "CompressionTest.cpp",
+    "ConvertEBCDICTest.cpp",
     "ConvertUTFTest.cpp",
     "CrashRecoveryTest.cpp",
     "DJBTest.cpp",
@@ -63,7 +56,6 @@ unittest("SupportTests") {
     "LineIteratorTest.cpp",
     "LockFileManagerTest.cpp",
     "MD5Test.cpp",
-    "MVTTest.cpp",
     "ManagedStatic.cpp",
     "MatchersTest.cpp",
     "MathExtrasTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
index 1f201bd783b73..e0ed14f7c1ca0 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
@@ -1,6 +1,30 @@
+source_set("llvm-min-tblgen-sources") {
+  sources = [
+    "Attributes.cpp",
+    "CodeGenIntrinsics.cpp",
+    "DirectiveEmitter.cpp",
+    "IntrinsicEmitter.cpp",
+    "RISCVTargetDefEmitter.cpp",
+    "SDNodeProperties.cpp",
+    "TableGen.cpp",
+    "VTEmitter.cpp",
+  ]
+}
+
+executable("llvm-min-tblgen") {
+  deps = [
+    ":llvm-min-tblgen-sources",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Support",
+    "//llvm/lib/TableGen",
+  ]
+}
+
 executable("llvm-tblgen") {
   deps = [
+    ":llvm-min-tblgen-sources",
     "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/CodeGen:CodeGenTypes",
     "//llvm/lib/Support",
     "//llvm/lib/TableGen",
     "//llvm/utils/TableGen/GlobalISel",
@@ -9,7 +33,6 @@ executable("llvm-tblgen") {
     "AsmMatcherEmitter.cpp",
     "AsmWriterEmitter.cpp",
     "AsmWriterInst.cpp",
-    "Attributes.cpp",
     "CTagsEmitter.cpp",
     "CallingConvEmitter.cpp",
     "CodeEmitterGen.cpp",
@@ -17,7 +40,6 @@ executable("llvm-tblgen") {
     "CodeGenHwModes.cpp",
     "CodeGenInstAlias.cpp",
     "CodeGenInstruction.cpp",
-    "CodeGenIntrinsics.cpp",
     "CodeGenMapTable.cpp",
     "CodeGenRegisters.cpp",
     "CodeGenSchedule.cpp",
@@ -32,7 +54,6 @@ executable("llvm-tblgen") {
     "DFAPacketizerEmitter.cpp",
     "DXILEmitter.cpp",
     "DecoderEmitter.cpp",
-    "DirectiveEmitter.cpp",
     "DisassemblerEmitter.cpp",
     "ExegesisEmitter.cpp",
     "FastISelEmitter.cpp",
@@ -41,22 +62,17 @@ executable("llvm-tblgen") {
     "InfoByHwMode.cpp",
     "InstrDocsEmitter.cpp",
     "InstrInfoEmitter.cpp",
-    "IntrinsicEmitter.cpp",
     "OptEmitter.cpp",
     "OptParserEmitter.cpp",
     "OptRSTEmitter.cpp",
     "PredicateExpander.cpp",
     "PseudoLoweringEmitter.cpp",
-    "RISCVTargetDefEmitter.cpp",
     "RegisterBankEmitter.cpp",
     "RegisterInfoEmitter.cpp",
-    "SDNodeProperties.cpp",
     "SearchableTableEmitter.cpp",
     "SubtargetEmitter.cpp",
     "SubtargetFeatureInfo.cpp",
-    "TableGen.cpp",
     "Types.cpp",
-    "VTEmitter.cpp",
     "VarLenCodeEmitterGen.cpp",
     "WebAssemblyDisassemblerEmitter.cpp",
     "X86DisassemblerTables.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
index b2d95a05956c7..27a425b86d78a 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
@@ -1,5 +1,9 @@
 static_library("GlobalISel") {
-  deps = [ "//llvm/lib/Support" ]
+  deps = [
+    "//llvm/include/llvm/CodeGen:GenVT",
+    "//llvm/lib/CodeGen:CodeGenTypes",
+    "//llvm/lib/Support",
+  ]
   sources = [
     "CodeExpander.cpp",
     "GIMatchDag.cpp",
diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py
index 6995a9a1f61c4..01f5ca4946264 100644
--- a/llvm/utils/lit/lit/TestingConfig.py
+++ b/llvm/utils/lit/lit/TestingConfig.py
@@ -75,6 +75,14 @@ def fromdefaults(litConfig):
                 'USERPROFILE',
             ]
             environment['PYTHONBUFFERED'] = '1'
+            # Avoid Windows heuristics which try to detect potential installer
+            # programs (which may need to run with elevated privileges) and ask
+            # if the user wants to run them in that way. This heuristic may
+            # match for executables containing the substrings "patch" (which is
+            # a substring of "dispatch"), "update", "setup", etc. Set this
+            # environment variable indicating that we want to execute them with
+            # the current user.
+            environment['__COMPAT_LAYER'] = 'RunAsInvoker'
 
         for var in pass_vars:
             val = os.environ.get(var, '')
diff --git a/llvm/utils/phabricator/0002-Fix-PHP-8.1.5-empty-strlen-with-arc-diff-verbatim.patch b/llvm/utils/phabricator/0002-Fix-PHP-8.1.5-empty-strlen-with-arc-diff-verbatim.patch
deleted file mode 100644
index 75fd75a7a0735..0000000000000
--- a/llvm/utils/phabricator/0002-Fix-PHP-8.1.5-empty-strlen-with-arc-diff-verbatim.patch
+++ /dev/null
@@ -1,36 +0,0 @@
-From 3a6e789fe893fc8c855826994d044d17a281fd85 Mon Sep 17 00:00:00 2001
-From: Justin Stitt <justinstitt@google.com>
-Date: Thu, 11 Aug 2022 10:26:21 -0700
-Subject: [PATCH] Fix PHP 8.1.5 empty strlen with <$ arc diff --verbatim>
-
-Fixes the following observed error with PHP 8.1.5:
-| EXCEPTION: (RuntimeException) strlen(): Passing null to parameter #1 ($string) of type string is deprecated at [<arcanist>/src/error/PhutilErrorHandler.php:261]
-| arcanist(head=master, ref.master=680fe6d06e5a)
-|   #0 PhutilErrorHandler::handleError(integer, string, string, integer) called at [<arcanist>/src/workflow/ArcanistDiffWorkflow.php:2364]
-|   #1 ArcanistDiffWorkflow::getDiffOntoTargets() called at [<arcanist>/src/workflow/ArcanistDiffWorkflow.php:2342]
-|   #2 ArcanistDiffWorkflow::updateOntoDiffProperty() called at [<arcanist>/src/workflow/ArcanistDiffWorkflow.php:409]
-|   #3 ArcanistDiffWorkflow::run() called at [<arcanist>/scripts/arcanist.php:427]
-
-Link: https://secure.phabricator.com/book/phabcontrib/article/contributing_code/
-Link: https://reviews.llvm.org/D129232#3634072
-Signed-off-by: Justin Stitt <justinstitt@google.com>
----
- src/workflow/ArcanistDiffWorkflow.php | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/workflow/ArcanistDiffWorkflow.php b/src/workflow/ArcanistDiffWorkflow.php
-index 5b8ff40c..1b717b23 100644
---- a/src/workflow/ArcanistDiffWorkflow.php
-+++ b/src/workflow/ArcanistDiffWorkflow.php
-@@ -2361,7 +2361,7 @@ EOTEXT
- 
-     // If we track an upstream branch either directly or indirectly, use that.
-     $branch = $api->getBranchName();
--    if (strlen($branch)) {
-+    if (phutil_nonempty_string($branch) && strlen($branch)) {
-       $upstream_path = $api->getPathToUpstream($branch);
-       $remote_branch = $upstream_path->getRemoteBranchName();
-       if ($remote_branch !== null) {
--- 
-2.37.1.559.g78731f0fdb-goog
-
diff --git a/mlir/docs/ActionTracing.md b/mlir/docs/ActionTracing.md
new file mode 100644
index 0000000000000..978fdbbe54d81
--- /dev/null
+++ b/mlir/docs/ActionTracing.md
@@ -0,0 +1,281 @@
+# Action: Tracing and Debugging MLIR-based Compilers
+
+[TOC]
+
+See also the [slides](https://mlir.llvm.org/OpenMeetings/2023-02-23-Actions.pdf)
+and the [recording](https://youtu.be/ayQSyekVa3c) from the MLIR Open Meeting
+where this feature was demoed.
+
+## Overview
+
+`Action` are means to encapsulate any transformation of any granularity in a way
+that can be intercepted by the framework for debugging or tracing purposes,
+including skipping a transformation programmatically (think about "compiler
+fuel" or "debug counters" in LLVM). As such, "executing a pass" is an Action, so
+is "try to apply one canonicalization pattern", or "tile this loop".
+
+In MLIR, passes and patterns are the main abstractions to encapsulate general IR
+transformations. The primary way of observing transformations along the way is
+to enable “debug printing” of the IR (e.g. -mlir-print-ir-after-all to print
+after each pass execution). On top of this, finer grain tracing may be available
+with -debug which enables more detailed logs from the transformations
+themselves. However, this method has some scaling issues: it is limited to a
+single stream of text that can be gigantic and requires tedious crawling through
+this log a posteriori. Iterating through multiple runs of collecting such logs
+and analyzing it can be very time consuming and often not very practical beyond
+small input programs.
+
+The `Action` framework doesn't make any assumptions about how the higher level
+driver is controlling the execution, it merely provides a framework for
+connecting the two together. A high level overview of the workflow surrounding
+`Action` execution is shown below:
+
+- Compiler developer defines an `Action` class, that is representing the
+  transformation or utility that they are developing.
+- Depending on the needs, the developer identifies single unit of
+  transformations, and dispatch them to the `MLIRContext` for execution.
+- An external entity registers an "action handler" with the action manager, and
+  provides the logic surrounding the transformation execution.
+
+The exact definition of an `external entity` is left opaque, to allow for more
+interesting handlers.
+
+## Wrapping a Transformation in an Action
+
+There are two parts for getting started with enabling tracing through Action in
+existing or new code: 1) defining an actual `Action` class, and 2) encapsulating
+the transformation in a lambda function.
+
+There are no constraints on the granularity of an “action”, it can be as simple
+as “perform this fold” and as complex as “run this pass pipeline”. An action is
+comprised of the following:
+
+```c++
+/// A custom Action can be defined minimally by deriving from
+/// `tracing::ActionImpl`.
+class MyCustomAction : public tracing::ActionImpl<MyCustomAction> {
+public:
+  using Base = tracing::ActionImpl<MyCustomAction>;
+  /// Actions are initialized with an array of IRUnit (that is either Operation,
+  /// Block, or Region) that provide context for the IR affected by a transformation.
+  MyCustomAction(ArrayRef<IRUnit> irUnits)
+      : Base(irUnits) {}
+  /// This tag should uniquely identify this action, it can be matched for filtering
+  /// during processing.
+  static constexpr StringLiteral tag = "unique-tag-for-my-action";
+  static constexpr StringLiteral desc =
+      "This action will encapsulate a some very specific transformation";
+};
+```
+
+Any transformation can then be dispatched with this `Action` through the
+`MLIRContext`:
+
+```c++
+context->executeAction<ApplyPatternAction>(
+    [&]() {
+      rewriter.setInsertionPoint(op);
+
+      ...
+    },
+    /*IRUnits=*/{op, region});
+```
+
+An action can also carry arbitrary payload, for example we can extend the
+`MyCustomAction` class above with the following member:
+
+```c++
+/// A custom Action can be defined minimally by deriving from
+/// `tracing::ActionImpl`. It can has any members!
+class MyCustomAction : public tracing::ActionImpl<MyCustomAction> {
+public:
+  using Base = tracing::ActionImpl<MyCustomAction>;
+  /// Actions are initialized with an array of IRUnit (that is either Operation,
+  /// Block, or Region) that provide context for the IR affected by a transformation.
+  /// Other constructor arguments can also be required here.
+  MyCustomAction(ArrayRef<IRUnit> irUnits, int count, PaddingStyle padding)
+      : Base(irUnits), count(count), padding(padding) {}
+  /// This tag should uniquely identify this action, it can be matched for filtering
+  /// during processing.
+  static constexpr StringLiteral tag = "unique-tag-for-my-action";
+  static constexpr StringLiteral desc =
+      "This action will encapsulate a some very specific transformation";
+  /// Extra members can be carried by the Action
+  int count;
+  PaddingStyle padding;
+};
+```
+
+These new members must then be passed as arguments when dispatching an `Action`:
+
+```c++
+context->executeAction<ApplyPatternAction>(
+    [&]() {
+      rewriter.setInsertionPoint(op);
+
+      ...
+    },
+    /*IRUnits=*/{op, region},
+    /*count=*/count,
+    /*padding=*/padding);
+```
+
+## Intercepting Actions
+
+When a transformation is executed through an `Action`, it can be directly
+intercepted via a handler that can be set on the `MLIRContext`:
+
+```c++
+  /// Signatures for the action handler that can be registered with the context.
+  using HandlerTy =
+      std::function<void(function_ref<void()>, const tracing::Action &)>;
+
+  /// Register a handler for handling actions that are dispatched through this
+  /// context. A nullptr handler can be set to disable a previously set handler.
+  void registerActionHandler(HandlerTy handler);
+```
+
+This handler takes two arguments: the first on is the transformation wrapped in
+a callback, and the second is a reference to the associated action object. The
+handler has full control of the execution, as such it can also decide to return
+without executing the callback, skipping the transformation entirely!
+
+## MLIR-provided Handlers
+
+MLIR provides some predefined action handlers for immediate use that are
+believed to be useful for most projects built with MLIR.
+
+### Debug Counters
+
+When debugging a compiler issue,
+["bisection"](<https://en.wikipedia.org/wiki/Bisection_(software_engineering)>)
+is a useful technique for locating the root cause of the issue. `Debug Counters`
+enable using this technique for debug actions by attaching a counter value to a
+specific action and enabling/disabling execution of this action based on the
+value of the counter. The counter controls the execution of the action with a
+"skip" and "count" value. The "skip" value is used to skip a certain number of
+initial executions of a debug action. The "count" value is used to prevent a
+debug action from executing after it has executed for a set number of times (not
+including any executions that have been skipped). If the "skip" value is
+negative, the action will always execute. If the "count" value is negative, the
+action will always execute after the "skip" value has been reached. For example,
+a counter for a debug action with `skip=47` and `count=2`, would skip the first
+47 executions, then execute twice, and finally prevent any further executions.
+With a bit of tooling, the values to use for the counter can be automatically
+selected; allowing for finding the exact execution of a debug action that
+potentially causes the bug being investigated.
+
+Note: The DebugCounter action handler does not support multi-threaded execution,
+and should only be used in MLIRContexts where multi-threading is disabled (e.g.
+via `-mlir-disable-threading`).
+
+#### CommandLine Configuration
+
+The `DebugCounter` handler provides several that allow for configuring counters.
+The main option is `mlir-debug-counter`, which accepts a comma separated list of
+`<count-name>=<counter-value>`. A `<counter-name>` is the debug action tag to
+attach the counter, suffixed with either `-skip` or `-count`. A `-skip` suffix
+will set the "skip" value of the counter. A `-count` suffix will set the "count"
+value of the counter. The `<counter-value>` component is a numeric value to use
+for the counter. An example is shown below using `MyCustomAction` defined above:
+
+```shell
+$ mlir-opt foo.mlir -mlir-debug-counter=unique-tag-for-my-action-skip=47,unique-tag-for-my-action-count=2
+```
+
+The above configuration would skip the first 47 executions of
+`ApplyPatternAction`, then execute twice, and finally prevent any further
+executions.
+
+Note: Each counter currently only has one `skip` and one `count` value, meaning
+that sequences of `skip`/`count` will not be chained.
+
+The `mlir-print-debug-counter` option may be used to print out debug counter
+information after all counters have been accumulated. The information is printed
+in the following format:
+
+```shell
+DebugCounter counters:
+<action-tag>                   : {<current-count>,<skip>,<count>}
+```
+
+For example, using the options above we can see how many times an action is
+executed:
+
+```shell
+$ mlir-opt foo.mlir -mlir-debug-counter=unique-tag-for-my-action-skip=-1 -mlir-print-debug-counter --pass-pipeline="builtin.module(func.func(my-pass))" --mlir-disable-threading
+
+DebugCounter counters:
+unique-tag-for-my-action         : {370,-1,-1}
+```
+
+### ExecutionContext
+
+The `ExecutionContext` is a component that provides facility to unify the kind
+of functionalities that most compiler debuggers tool would need, exposed in a
+composable way.
+
+![IMG](/actions/ActionTracing_ExecutionContext.png)
+
+The `ExecutionContext` is itself registered as a handler with the MLIRContext
+and tracks all executed actions, keeping a per-thread stack of action execution.
+It acts as a middleware that handles the flow of action execution while allowing
+injection and control from a debugger.
+
+- Multiple `Observers` can be registered with the `ExecutionContext`. When an
+  action is dispatched for execution, it is passed to each of the `Observers`
+  before and after executing the transformation.
+- Multiple `BreakpointManager` can be registered with the `ExecutionContext`.
+  When an action is dispatched for execution, it is passed to each of the
+  registered `BreakpointManager` until one matches the action and return a valid
+  `Breakpoint` object. In this case, the "callback" set by the client on the
+  `ExecutionContext` is invoked, otherwise the transformation is directly
+  executed.
+- A single callback:
+  `using CallbackTy = function_ref<Control(const ActionActiveStack *)>;` can be
+  registered with the `ExecutionContext`, it is invoked when a `BreakPoint` is
+  hit by an `Action`. The returned value of type `Control` is an enum
+  instructing the `ExecutionContext` of how to proceed next:
+  ```c++
+  /// Enum that allows the client of the context to control the execution of the
+  /// action.
+  /// - Apply: The action is executed.
+  /// - Skip: The action is skipped.
+  /// - Step: The action is executed and the execution is paused before the next
+  ///         action, including for nested actions encountered before the
+  ///         current action finishes.
+  /// - Next: The action is executed and the execution is paused after the
+  ///         current action finishes before the next action.
+  /// - Finish: The action is executed and the execution is paused only when we
+  ///           reach the parent/enclosing operation. If there are no enclosing
+  ///           operation, the execution continues without stopping.
+  enum Control { Apply = 1, Skip = 2, Step = 3, Next = 4, Finish = 5 };
+  ```
+  Since the callback actually controls the execution, there can be only one
+  registered at any given time.
+
+#### Debugger ExecutionContext Hook
+
+MLIR provides a callback for the `ExecutionContext` that implements a small
+runtime suitable for debuggers like `gdb` or `lldb` to interactively control the
+execution. It can be setup with
+`mlir::setupDebuggerExecutionContextHook(executionContext);` or using `mlir-opt`
+with the `--mlir-enable-debugger-hook` flag. This runtime exposes a set of C API
+function that can be called from a debugger to:
+
+- set breakpoints matching either action tags, or the `FileLineCol` locations of
+  the IR associated with the action.
+- set the `Control` flag to be returned to the `ExecutionContext`.
+- control a "cursor" allowing to navigate through the IR and inspect it from the
+  IR context associated with the action.
+
+The implementation of this runtime can serve as an example for other
+implementation of programmatic control of the execution.
+
+#### Logging Observer
+
+One observer is provided that allows to log action execution on a provided
+stream. It can be exercised with `mlir-opt` using `--log-actions-to=<filename>`,
+and optionally filtering the output with
+`--log-mlir-actions-filter=<FileLineCol>`. This observer is not thread-safe at
+the moment.
diff --git a/mlir/docs/DebugActions.md b/mlir/docs/DebugActions.md
deleted file mode 100644
index a40459c6161d6..0000000000000
--- a/mlir/docs/DebugActions.md
+++ /dev/null
@@ -1,241 +0,0 @@
-# Debug Actions
-
-This file documents the infrastructure for `Debug Actions`. This is a DEBUG only
-API that allows for external entities to control various aspects of compiler
-execution. This is conceptually similar to something like `DebugCounters` in
-LLVM, but at a lower level. This framework doesn't make any assumptions about
-how the higher level driver is controlling the execution, it merely provides a
-framework for connecting the two together. A high level overview of the workflow
-surrounding debug actions is shown below:
-
-*   Compiler developer defines an [`action`](#debug-action) that is taken by the
-    a pass, transformation, utility that they are developing.
-*   Depending on the needs, the developer dispatches various queries, pertaining
-    to this action, to an [`action manager`](#debug-action-manager) that will
-    provide an answer as to what behavior the action should take.
-*   An external entity registers an [`action handler`](#debug-action-handler)
-    with the action manager, and provides the logic to resolve queries on
-    actions.
-
-The exact definition of an `external entity` is left opaque, to allow for more
-interesting handlers. The set of possible action queries is detailed in the
-[`action manager`](#debug-action-manager) section below.
-
-[TOC]
-
-## Debug Action
-
-A `debug action` is essentially a marker for a type of action that may be
-performed within the compiler. There are no constraints on the granularity of an
-“action”, it can be as simple as “perform this fold” and as complex as “run this
-pass pipeline”. An action is comprised of the following:
-
-*   Tag:
-
-    -   A unique string identifier, similar to a command line flag or
-        DEBUG_TYPE.
-
-*   Description:
-
-    -   A short description of what the action represents.
-
-*   Parameter Types:
-
-    -   The types of values that are passed to queries related to this action,
-        to help guide decisions.
-
-Below is an example action that may be provided by the
-[pattern rewriter](PatternRewriter.md) framework to control the application of
-rewrite patterns.
-
-```c++
-/// A debug action that allows for controlling the application of patterns.
-/// A new action type can be defined by inheriting from `DebugAction`.
-/// * The Tag is specified via a static `StringRef getTag()` method.
-/// * The Description is specified via a static `StringRef getDescription()`
-///   method.
-/// * `DebugAction` is a CRTP class, so the first template parameter is the
-///   action type class itself.
-/// * The parameters for the action are provided via additional template
-///   parameters when inheriting from `DebugAction`.
-struct ApplyPatternAction
-    : public DebugAction<ApplyPatternAction, Operation *, const Pattern &> {
-  static StringRef getTag() { return "apply-pattern"; }
-  static StringRef getDescription() {
-    return "Control the application of rewrite patterns";
-  }
-};
-```
-
-## Debug Action Manager
-
-The `DebugActionManager` orchestrates the various different queries relating to
-debug actions, and is accessible via the `MLIRContext`. These queries allow for
-external entities to control various aspects of the compiler via
-[action handlers](#debug-action-handler). When resolving a query for an action,
-the result from the most recently registered handler is used.
-
-TODO: It may be interesting to support merging results from multiple action
-handlers, but this is left for future work when driven by a real use case.
-
-The set of available queries are shown below:
-
-```c++
-class DebugActionManager {
-public:
-  /// Returns true if the given action type should be executed, false otherwise.
-  /// `Params` correspond to any action specific parameters that may be used to
-  /// guide the decision.
-  template <typename ActionType, typename... Params>
-  bool shouldExecute(Params &&... params);
-};
-```
-
-Building on the example from the [previous section](#debug-action), an example
-usage of the `shouldExecute` query is shown below:
-
-```c++
-/// A debug action that allows for controlling the application of patterns.
-struct ApplyPatternAction
-    : public DebugAction<ApplyPatternAction, Operation *, const Pattern &> {
-  static StringRef getTag() { return "apply-pattern"; }
-  static StringRef getDescription() {
-    return "Control the application of rewrite patterns";
-  }
-};
-
-// ...
-
-bool shouldApplyPattern(Operation *currentOp, const Pattern &currentPattern) {
-  MLIRContext *context = currentOp->getContext();
-  DebugActionManager &manager = context->getDebugActionManager();
-
-  // Query the action manager to see if `currentPattern` should be applied to
-  // `currentOp`.
-  return manager.shouldExecute<ApplyPatternAction>(currentOp, currentPattern);
-}
-```
-
-## Debug Action Handler
-
-A debug action handler provides the internal implementation for the various
-action related queries within the [`DebugActionManager`](#debug-action-manager).
-Action handlers allow for external entities to control and inject external
-information into the compiler. Handlers can be registered with the
-`DebugActionManager` using `registerActionHandler`. There are two types of
-handlers; action-specific handlers and generic handlers.
-
-### Action Specific Handlers
-
-Action specific handlers handle a specific debug action type, with the
-parameters to its query methods mapping 1-1 to the parameter types of the action
-type. An action specific handler can be defined by inheriting from the handler
-base class defined at `ActionType::Handler` where `ActionType` is the specific
-action that should be handled. An example using our running pattern example is
-shown below:
-
-```c++
-struct MyPatternHandler : public ApplyPatternAction::Handler {
-  /// A variant of `shouldExecute` shown in the `DebugActionManager` class
-  /// above.
-  /// This method returns a FailureOr<bool>, where failure signifies that the
-  /// action was not handled (allowing for other handlers to process it), or the
-  /// boolean true/false signifying if the action should execute or not.
-  FailureOr<bool> shouldExecute(Operation *op,
-                                const RewritePattern &pattern) final;
-};
-```
-
-### Generic Handlers
-
-A generic handler allows for handling queries on any action type. These types of
-handlers are useful for implementing general functionality that doesn’t
-necessarily need to interpret the exact action parameters, or can rely on an
-external interpreter (such as the user). As these handlers are generic, they
-take a set of opaque parameters that try to map the context of the action type
-in a generic way. A generic handler can be defined by inheriting from
-`DebugActionManager::GenericHandler`. An example is shown below:
-
-```c++
-struct MyPatternHandler : public DebugActionManager::GenericHandler {
-  /// The return type of this method is the same as the action-specific handler.
-  /// The arguments to this method map the concepts of an action type in an
-  /// opaque way. The arguments are provided in such a way so that the context
-  /// of the action is still somewhat user readable, or at least loggable as
-  /// such.
-  /// - actionTag: The tag specified by the action type.
-  /// - actionDesc: The description specified by the action type.
-  virtual FailureOr<bool> shouldExecute(StringRef actionTag,
-                                        StringRef actionDesc);
-};
-```
-
-### Common Action Handlers
-
-MLIR provides several common debug action handlers for immediate use that have
-proven useful in general.
-
-#### DebugCounter
-
-When debugging a compiler issue,
-["bisection"](https://en.wikipedia.org/wiki/Bisection_\(software_engineering\))
-is a useful technique for locating the root cause of the issue. `Debug Counters`
-enable using this technique for debug actions by attaching a counter value to a
-specific debug action and enabling/disabling execution of this action based on
-the value of the counter. The counter controls the execution of the action with
-a "skip" and "count" value. The "skip" value is used to skip a certain number of
-initial executions of a debug action. The "count" value is used to prevent a
-debug action from executing after it has executed for a set number of times (not
-including any executions that have been skipped). If the "skip" value is
-negative, the action will always execute. If the "count" value is negative, the
-action will always execute after the "skip" value has been reached. For example,
-a counter for a debug action with `skip=47` and `count=2`, would skip the first
-47 executions, then execute twice, and finally prevent any further executions.
-With a bit of tooling, the values to use for the counter can be automatically
-selected; allowing for finding the exact execution of a debug action that
-potentially causes the bug being investigated.
-
-Note: The DebugCounter action handler does not support multi-threaded execution,
-and should only be used in MLIRContexts where multi-threading is disabled (e.g.
-via `-mlir-disable-threading`).
-
-##### CommandLine Configuration
-
-The `DebugCounter` handler provides several that allow for configuring counters.
-The main option is `mlir-debug-counter`, which accepts a comma separated list of
-`<count-name>=<counter-value>`. A `<counter-name>` is the debug action tag to
-attach the counter, suffixed with either `-skip` or `-count`. A `-skip` suffix
-will set the "skip" value of the counter. A `-count` suffix will set the "count"
-value of the counter. The `<counter-value>` component is a numeric value to use
-for the counter. An example is shown below using `ApplyPatternAction` defined
-above:
-
-```shell
-$ mlir-opt foo.mlir -mlir-debug-counter=apply-pattern-skip=47,apply-pattern-count=2
-```
-
-The above configuration would skip the first 47 executions of
-`ApplyPatternAction`, then execute twice, and finally prevent any further
-executions.
-
-Note: Each counter currently only has one `skip` and one `count` value, meaning
-that sequences of `skip`/`count` will not be chained.
-
-The `mlir-print-debug-counter` option may be used to print out debug counter
-information after all counters have been accumulated. The information is printed
-in the following format:
-
-```shell
-DebugCounter counters:
-<action-tag>                   : {<current-count>,<skip>,<count>}
-```
-
-For example, using the options above we can see how many times an action is
-executed:
-
-```shell
-$ mlir-opt foo.mlir -mlir-debug-counter=apply-pattern-skip=-1 -mlir-print-debug-counter
-
-DebugCounter counters:
-apply-pattern                   : {370,-1,-1}
-```
diff --git a/mlir/docs/LangRef.md b/mlir/docs/LangRef.md
index 5ce86eb88dc61..48e4903216531 100644
--- a/mlir/docs/LangRef.md
+++ b/mlir/docs/LangRef.md
@@ -290,12 +290,14 @@ Syntax:
 operation            ::= op-result-list? (generic-operation | custom-operation)
                          trailing-location?
 generic-operation    ::= string-literal `(` value-use-list? `)`  successor-list?
-                         region-list? dictionary-attribute? `:` function-type
+                         dictionary-properties? region-list? dictionary-attribute?
+                         `:` function-type
 custom-operation     ::= bare-id custom-operation-format
 op-result-list       ::= op-result (`,` op-result)* `=`
 op-result            ::= value-id (`:` integer-literal)
 successor-list       ::= `[` successor (`,` successor)* `]`
 successor            ::= caret-id (`:` block-arg-list)?
+dictionary-propertes ::= `<` dictionary-attribute `>`
 region-list          ::= `(` region (`,` region)* `)`
 dictionary-attribute ::= `{` (attribute-entry (`,` attribute-entry)*)? `}`
 trailing-location    ::= (`loc` `(` location `)`)?
@@ -312,9 +314,10 @@ semantics. For example, MLIR supports
 The internal representation of an operation is simple: an operation is
 identified by a unique string (e.g. `dim`, `tf.Conv2d`, `x86.repmovsb`,
 `ppc.eieio`, etc), can return zero or more results, take zero or more operands,
-has a dictionary of [attributes](#attributes), has zero or more successors, and
-zero or more enclosed [regions](#regions). The generic printing form includes
-all these elements literally, with a function type to indicate the types of the
+has storage for [properties](#properties), has a dictionary of
+[attributes](#attributes), has zero or more successors, and zero or more
+enclosed [regions](#regions). The generic printing form includes all these
+elements literally, with a function type to indicate the types of the
 results and operands.
 
 Example:
@@ -328,8 +331,11 @@ Example:
 %foo, %bar = "foo_div"() : () -> (f32, i32)
 
 // Invoke a TensorFlow function called tf.scramble with two inputs
-// and an attribute "fruit".
-%2 = "tf.scramble"(%result#0, %bar) {fruit = "banana"} : (f32, i32) -> f32
+// and an attribute "fruit" stored in properties.
+%2 = "tf.scramble"(%result#0, %bar) <{fruit = "banana"}> : (f32, i32) -> f32
+
+// Invoke an operation with some discardable attributes
+%foo, %bar = "foo_div"() {some_attr = "value", other_attr = 42 : i64} : () -> (f32, i32)
 ```
 
 In addition to the basic syntax above, dialects may register known operations.
@@ -733,6 +739,15 @@ The [builtin dialect](Dialects/Builtin.md) defines a set of types that are
 directly usable by any other dialect in MLIR. These types cover a range from
 primitive integer and floating-point types, function types, and more.
 
+## Properties
+
+Properties are extra data members stored directly on an Operation class. They
+provide a way to store [inherent attributes](#attributes) and other arbitrary
+data. The semantics of the data is specific to a given operation, and may be
+exposed through [Interfaces](Interfaces.md) accessors and other methods.
+Properties can always be serialized to Attribute in order to be printed
+generically.
+
 ## Attributes
 
 Syntax:
@@ -751,9 +766,10 @@ values. MLIR's builtin dialect provides a rich set of
 arrays, dictionaries, strings, etc.). Additionally, dialects can define their
 own [dialect attribute values](#dialect-attribute-values).
 
-The top-level attribute dictionary attached to an operation has special
-semantics. The attribute entries are considered to be of two different kinds
-based on whether their dictionary key has a dialect prefix:
+For dialects which haven't adopted properties yet, the top-level attribute
+dictionary attached to an operation has special semantics. The attribute
+entries are considered to be of two different kinds based on whether their
+dictionary key has a dialect prefix:
 
 -   *inherent attributes* are inherent to the definition of an operation's
     semantics. The operation itself is expected to verify the consistency of
@@ -771,6 +787,10 @@ Note that attribute values are allowed to themselves be dictionary attributes,
 but only the top-level dictionary attribute attached to the operation is subject
 to the classification above.
 
+When properties are adopted, only discardable attributes are stored in the
+top-level dictionary, while inherent attributes are stored in the properties
+storage.
+
 ### Attribute Value Aliases
 
 ```
diff --git a/mlir/docs/PassManagement.md b/mlir/docs/PassManagement.md
index 661592b13782c..9a7cfd1f9bebc 100644
--- a/mlir/docs/PassManagement.md
+++ b/mlir/docs/PassManagement.md
@@ -1333,9 +1333,9 @@ module {
 #-}
 ```
 
-The configuration dumped can be passed to `mlir-opt`. This will result in
-parsing the configuration of the reproducer and adjusting the necessary opt
-state, e.g. configuring the pass manager, context, etc.
+The configuration dumped can be passed to `mlir-opt` by specifying
+`-run-reproducer` flag. This will result in parsing the configuration of the reproducer
+and adjusting the necessary opt state, e.g. configuring the pass manager, context, etc.
 
 Beyond specifying a filename, one can also register a `ReproducerStreamFactory`
 function that would be invoked in the case of a crash and the reproducer written
diff --git a/mlir/docs/Rationale/SideEffectsAndSpeculation.md b/mlir/docs/Rationale/SideEffectsAndSpeculation.md
index 713cb86e054f7..95320feaebd2d 100644
--- a/mlir/docs/Rationale/SideEffectsAndSpeculation.md
+++ b/mlir/docs/Rationale/SideEffectsAndSpeculation.md
@@ -7,7 +7,7 @@ This rationale only applies to operations used in
 [CFG regions](../LangRef.md#control-flow-and-ssacfg-regions). Side effect
 modeling in [graph regions](../LangRef.md#graph-regions) is TBD.
 
-\[TOC\]
+[TOC]
 
 ## Overview
 
diff --git a/mlir/docs/doxygen.cfg.in b/mlir/docs/doxygen.cfg.in
index b91b242cffaa2..6c437ea138c21 100644
--- a/mlir/docs/doxygen.cfg.in
+++ b/mlir/docs/doxygen.cfg.in
@@ -2091,7 +2091,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2136,7 +2136,7 @@ TEMPLATE_RELATIONS     = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2145,7 +2145,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h
index 84d226b40b71a..90af14461e29e 100644
--- a/mlir/include/mlir-c/IR.h
+++ b/mlir/include/mlir-c/IR.h
@@ -48,6 +48,7 @@ extern "C" {
   };                                                                           \
   typedef struct name name
 
+DEFINE_C_API_STRUCT(MlirBytecodeWriterConfig, void);
 DEFINE_C_API_STRUCT(MlirContext, void);
 DEFINE_C_API_STRUCT(MlirDialect, void);
 DEFINE_C_API_STRUCT(MlirDialectRegistry, void);
@@ -408,6 +409,24 @@ mlirOpPrintingFlagsUseLocalScope(MlirOpPrintingFlags flags);
 MLIR_CAPI_EXPORTED void
 mlirOpPrintingFlagsAssumeVerified(MlirOpPrintingFlags flags);
 
+//===----------------------------------------------------------------------===//
+// Bytecode printing flags API.
+//===----------------------------------------------------------------------===//
+
+/// Creates new printing flags with defaults, intended for customization.
+/// Must be freed with a call to mlirBytecodeWriterConfigDestroy().
+MLIR_CAPI_EXPORTED MlirBytecodeWriterConfig
+mlirBytecodeWriterConfigCreate(void);
+
+/// Destroys printing flags created with mlirBytecodeWriterConfigCreate.
+MLIR_CAPI_EXPORTED void
+mlirBytecodeWriterConfigDestroy(MlirBytecodeWriterConfig config);
+
+/// Sets the version to emit in the writer config.
+MLIR_CAPI_EXPORTED void
+mlirBytecodeWriterConfigDesiredEmitVersion(MlirBytecodeWriterConfig flags,
+                                           int64_t version);
+
 //===----------------------------------------------------------------------===//
 // Operation API.
 //===----------------------------------------------------------------------===//
@@ -546,11 +565,17 @@ MLIR_CAPI_EXPORTED void mlirOperationPrintWithFlags(MlirOperation op,
                                                     MlirStringCallback callback,
                                                     void *userData);
 
-/// Same as mlirOperationPrint but writing the bytecode format out.
+/// Same as mlirOperationPrint but writing the bytecode format.
 MLIR_CAPI_EXPORTED void mlirOperationWriteBytecode(MlirOperation op,
                                                    MlirStringCallback callback,
                                                    void *userData);
 
+/// Same as mlirOperationWriteBytecode but with writer config and returns
+/// failure only if desired bytecode could not be honored.
+MLIR_CAPI_EXPORTED MlirLogicalResult mlirOperationWriteBytecodeWithConfig(
+    MlirOperation op, MlirBytecodeWriterConfig config,
+    MlirStringCallback callback, void *userData);
+
 /// Prints an operation to stderr.
 MLIR_CAPI_EXPORTED void mlirOperationDump(MlirOperation op);
 
@@ -755,6 +780,12 @@ mlirValuePrint(MlirValue value, MlirStringCallback callback, void *userData);
 /// operand if there are no uses.
 MLIR_CAPI_EXPORTED MlirOpOperand mlirValueGetFirstUse(MlirValue value);
 
+/// Replace all uses of 'of' value with the 'with' value, updating anything in
+/// the IR that uses 'of' to use the other value instead.  When this returns
+/// there are zero uses of 'of'.
+MLIR_CAPI_EXPORTED void mlirValueReplaceAllUsesOfWith(MlirValue of,
+                                                      MlirValue with);
+
 //===----------------------------------------------------------------------===//
 // OpOperand API.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir-c/Interfaces.h b/mlir/include/mlir-c/Interfaces.h
index 233f828b924c7..405e2bb7173e0 100644
--- a/mlir/include/mlir-c/Interfaces.h
+++ b/mlir/include/mlir-c/Interfaces.h
@@ -57,8 +57,8 @@ typedef void (*MlirTypesCallback)(intptr_t, MlirType *, void *);
 MLIR_CAPI_EXPORTED MlirLogicalResult mlirInferTypeOpInterfaceInferReturnTypes(
     MlirStringRef opName, MlirContext context, MlirLocation location,
     intptr_t nOperands, MlirValue *operands, MlirAttribute attributes,
-    intptr_t nRegions, MlirRegion *regions, MlirTypesCallback callback,
-    void *userData);
+    void *properties, intptr_t nRegions, MlirRegion *regions,
+    MlirTypesCallback callback, void *userData);
 
 #ifdef __cplusplus
 }
diff --git a/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h b/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h
index abebd7328f823..d27c74b79693f 100644
--- a/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h
+++ b/mlir/include/mlir/Analysis/FlatLinearValueConstraints.h
@@ -492,7 +492,7 @@ class FlatLinearValueConstraints : public FlatLinearConstraints {
   /// Values corresponding to the (column) non-local variables of this
   /// constraint system appearing in the order the variables correspond to
   /// columns. Variables that aren't associated with any Value are set to
-  /// None.
+  /// std::nullopt.
   SmallVector<std::optional<Value>, 8> values;
 };
 
diff --git a/mlir/include/mlir/Analysis/Presburger/Utils.h b/mlir/include/mlir/Analysis/Presburger/Utils.h
index 592782d564497..a3000a26c3f3d 100644
--- a/mlir/include/mlir/Analysis/Presburger/Utils.h
+++ b/mlir/include/mlir/Analysis/Presburger/Utils.h
@@ -156,6 +156,11 @@ class DivisionRepr {
     denoms[i] = divisor;
   }
 
+  // Find the greatest common divisor (GCD) of the dividends and divisor for
+  // each valid division. Divide the dividends and divisor by the GCD to
+  // simplify the expression.
+  void normalizeDivs();
+
   void insertDiv(unsigned pos, ArrayRef<MPInt> dividend, const MPInt &divisor);
   void insertDiv(unsigned pos, unsigned num = 1);
 
diff --git a/mlir/include/mlir/Bytecode/BytecodeImplementation.h b/mlir/include/mlir/Bytecode/BytecodeImplementation.h
index ea9bcad735b36..60f5475609ac7 100644
--- a/mlir/include/mlir/Bytecode/BytecodeImplementation.h
+++ b/mlir/include/mlir/Bytecode/BytecodeImplementation.h
@@ -233,6 +233,9 @@ class DialectBytecodeWriter {
   /// guaranteed to not die before the end of the bytecode process. The blob is
   /// written as-is, with no additional compression or compaction.
   virtual void writeOwnedBlob(ArrayRef<char> blob) = 0;
+
+  /// Return the bytecode version being emitted for.
+  virtual int64_t getBytecodeVersion() const = 0;
 };
 
 //===--------------------------------------------------------------------===//
@@ -345,6 +348,37 @@ class BytecodeDialectInterface
   }
 };
 
+/// Helper for resource handle reading that returns LogicalResult.
+template <typename T, typename... Ts>
+static LogicalResult readResourceHandle(DialectBytecodeReader &reader,
+                                        FailureOr<T> &value, Ts &&...params) {
+  FailureOr<T> handle = reader.readResourceHandle<T>();
+  if (failed(handle))
+    return failure();
+  if (auto *result = dyn_cast<T>(&*handle)) {
+    value = std::move(*result);
+    return success();
+  }
+  return failure();
+}
+
+/// Helper method that injects context only if needed, this helps unify some of
+/// the attribute construction methods.
+template <typename T, typename... Ts>
+auto get(MLIRContext *context, Ts &&...params) {
+  // Prefer a direct `get` method if one exists.
+  if constexpr (llvm::is_detected<detail::has_get_method, T, Ts...>::value) {
+    (void)context;
+    return T::get(std::forward<Ts>(params)...);
+  } else if constexpr (llvm::is_detected<detail::has_get_method, T,
+                                         MLIRContext *, Ts...>::value) {
+    return T::get(context, std::forward<Ts>(params)...);
+  } else {
+    // Otherwise, pass to the base get.
+    return T::Base::get(context, std::forward<Ts>(params)...);
+  }
+}
+
 } // namespace mlir
 
 #endif // MLIR_BYTECODE_BYTECODEIMPLEMENTATION_H
diff --git a/mlir/include/mlir/Bytecode/BytecodeWriter.h b/mlir/include/mlir/Bytecode/BytecodeWriter.h
index fb4329e4d66f0..bd4087f6c498d 100644
--- a/mlir/include/mlir/Bytecode/BytecodeWriter.h
+++ b/mlir/include/mlir/Bytecode/BytecodeWriter.h
@@ -40,6 +40,12 @@ class BytecodeWriterConfig {
   /// Return an instance of the internal implementation.
   const Impl &getImpl() const { return *impl; }
 
+  /// Set the desired bytecode version to emit. This function clamps the version
+  /// to the existing version if larger than existing. The desired version may
+  /// not be used depending on the features used and the actual version required
+  /// is returned by bytecode writer entry point.
+  void setDesiredBytecodeVersion(int64_t bytecodeVersion);
+
   //===--------------------------------------------------------------------===//
   // Resources
   //===--------------------------------------------------------------------===//
@@ -75,8 +81,9 @@ class BytecodeWriterConfig {
 
 /// Write the bytecode for the given operation to the provided output stream.
 /// For streams where it matters, the given stream should be in "binary" mode.
-void writeBytecodeToFile(Operation *op, raw_ostream &os,
-                         const BytecodeWriterConfig &config = {});
+/// It only ever fails if setDesiredByteCodeVersion can't be honored.
+LogicalResult writeBytecodeToFile(Operation *op, raw_ostream &os,
+                                  const BytecodeWriterConfig &config = {});
 
 } // namespace mlir
 
diff --git a/mlir/include/mlir/CAPI/IR.h b/mlir/include/mlir/CAPI/IR.h
index 2f32c76e12f06..b8ccec896c27b 100644
--- a/mlir/include/mlir/CAPI/IR.h
+++ b/mlir/include/mlir/CAPI/IR.h
@@ -15,11 +15,13 @@
 #ifndef MLIR_CAPI_IR_H
 #define MLIR_CAPI_IR_H
 
+#include "mlir/Bytecode/BytecodeWriter.h"
 #include "mlir/CAPI/Wrap.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 
+DEFINE_C_API_PTR_METHODS(MlirBytecodeWriterConfig, mlir::BytecodeWriterConfig)
 DEFINE_C_API_PTR_METHODS(MlirContext, mlir::MLIRContext)
 DEFINE_C_API_PTR_METHODS(MlirDialect, mlir::Dialect)
 DEFINE_C_API_PTR_METHODS(MlirDialectRegistry, mlir::DialectRegistry)
diff --git a/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h b/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h
index feec7332fbfcc..12a1a5e227d67 100644
--- a/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h
+++ b/mlir/include/mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h
@@ -8,7 +8,7 @@
 #ifndef MLIR_CONVERSION_AMDGPUTOROCDL_AMDGPUTOROCDL_H_
 #define MLIR_CONVERSION_AMDGPUTOROCDL_AMDGPUTOROCDL_H_
 
-#include "mlir/Conversion/AMDGPUToROCDL/Chipset.h"
+#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
 #include <memory>
 #include <string>
 
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
index 76ed89563fd7b..28d37a91edb80 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
@@ -89,6 +89,14 @@ class MemRefDescriptor : public StructBuilder {
   /// Returns the (LLVM) pointer type this descriptor contains.
   LLVM::LLVMPointerType getElementPtrType();
 
+  /// Builds IR for getting the start address of the buffer represented
+  /// by this memref:
+  /// `memref.alignedPtr + memref.offset * sizeof(type.getElementType())`.
+  /// \note there is no setter for this one since it is derived from alignedPtr
+  /// and offset.
+  Value bufferPtr(OpBuilder &builder, Location loc,
+                  LLVMTypeConverter &converter, MemRefType type);
+
   /// Builds IR populating a MemRef descriptor structure from a list of
   /// individual values composing that descriptor, in the following order:
   /// - allocated pointer;
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index 90e62aa11787c..f17a2e54a7313 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -145,6 +145,11 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   /// Wrappers around the RewritePattern methods that pass the derived op type.
   void rewrite(Operation *op, ArrayRef<Value> operands,
                ConversionPatternRewriter &rewriter) const final {
+    if constexpr (SourceOp::hasProperties())
+      rewrite(cast<SourceOp>(op),
+              OpAdaptor(operands, op->getAttrDictionary(),
+                        cast<SourceOp>(op).getProperties()),
+              rewriter);
     rewrite(cast<SourceOp>(op), OpAdaptor(operands, op->getAttrDictionary()),
             rewriter);
   }
@@ -154,6 +159,11 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
+    if constexpr (SourceOp::hasProperties())
+      return matchAndRewrite(cast<SourceOp>(op),
+                             OpAdaptor(operands, op->getAttrDictionary(),
+                                       cast<SourceOp>(op).getProperties()),
+                             rewriter);
     return matchAndRewrite(cast<SourceOp>(op),
                            OpAdaptor(operands, op->getAttrDictionary()),
                            rewriter);
diff --git a/mlir/include/mlir/Debug/CLOptionsSetup.h b/mlir/include/mlir/Debug/CLOptionsSetup.h
new file mode 100644
index 0000000000000..ab8e7b436a1e9
--- /dev/null
+++ b/mlir/include/mlir/Debug/CLOptionsSetup.h
@@ -0,0 +1,93 @@
+//===- CLOptionsSetup.h - Helpers to setup debug CL options -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DEBUG_CLOPTIONSSETUP_H
+#define MLIR_DEBUG_CLOPTIONSSETUP_H
+
+#include "mlir/Debug/BreakpointManagers/FileLineColLocBreakpointManager.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <memory>
+
+namespace mlir {
+class MLIRContext;
+namespace tracing {
+class BreakpointManager;
+
+class DebugConfig {
+public:
+  /// Register the options as global LLVM command line options.
+  static void registerCLOptions();
+
+  /// Create a new config with the default set from the CL options.
+  static DebugConfig createFromCLOptions();
+
+  ///
+  /// Options.
+  ///
+
+  /// Enable the Debugger action hook: it makes a debugger (like gdb or lldb)
+  /// able to intercept MLIR Actions.
+  void enableDebuggerActionHook(bool enabled = true) {
+    enableDebuggerActionHookFlag = enabled;
+  }
+
+  /// Return true if the debugger action hook is enabled.
+  bool isDebuggerActionHookEnabled() const {
+    return enableDebuggerActionHookFlag;
+  }
+
+  /// Set the filename to use for logging actions, use "-" for stdout.
+  DebugConfig &logActionsTo(StringRef filename) {
+    logActionsToFlag = filename;
+    return *this;
+  }
+  /// Get the filename to use for logging actions.
+  StringRef getLogActionsTo() const { return logActionsToFlag; }
+
+  /// Set a location breakpoint manager to filter out action logging based on
+  /// the attached IR location in the Action context. Ownership stays with the
+  /// caller.
+  void addLogActionLocFilter(tracing::BreakpointManager *breakpointManager) {
+    logActionLocationFilter.push_back(breakpointManager);
+  }
+
+  /// Get the location breakpoint managers to use to filter out action logging.
+  ArrayRef<tracing::BreakpointManager *> getLogActionsLocFilters() const {
+    return logActionLocationFilter;
+  }
+
+protected:
+  /// Enable the Debugger action hook: a debugger (like gdb or lldb) can
+  /// intercept MLIR Actions.
+  bool enableDebuggerActionHookFlag = false;
+
+  /// Log action execution to the given file (or "-" for stdout)
+  std::string logActionsToFlag;
+
+  /// Location Breakpoints to filter the action logging.
+  std::vector<tracing::BreakpointManager *> logActionLocationFilter;
+};
+
+/// This is a RAII class that installs the debug handlers on the context
+/// based on the provided configuration.
+class InstallDebugHandler {
+public:
+  InstallDebugHandler(MLIRContext &context, const DebugConfig &config);
+  ~InstallDebugHandler();
+
+private:
+  class Impl;
+  std::unique_ptr<Impl> impl;
+};
+
+} // namespace tracing
+} // namespace mlir
+
+#endif // MLIR_DEBUG_CLOPTIONSSETUP_H
diff --git a/mlir/include/mlir/Debug/DebuggerExecutionContextHook.h b/mlir/include/mlir/Debug/DebuggerExecutionContextHook.h
new file mode 100644
index 0000000000000..ad2abcd852058
--- /dev/null
+++ b/mlir/include/mlir/Debug/DebuggerExecutionContextHook.h
@@ -0,0 +1,96 @@
+//===- DebuggerExecutionContextHook.h - Debugger Support --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a set of C API functions that are used by the debugger to
+// interact with the ExecutionContext.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_SUPPORT_DEBUGGEREXECUTIONCONTEXTHOOK_H
+#define MLIR_SUPPORT_DEBUGGEREXECUTIONCONTEXTHOOK_H
+
+#include "mlir-c/IR.h"
+#include "mlir/Debug/ExecutionContext.h"
+#include "llvm/Support/Compiler.h"
+
+extern "C" {
+struct MLIRBreakpoint;
+struct MLIRIRunit;
+typedef struct MLIRBreakpoint *BreakpointHandle;
+typedef struct MLIRIRunit *irunitHandle;
+
+/// This is used by the debugger to control what to do after a breakpoint is
+/// hit. See tracing::ExecutionContext::Control for more information.
+void mlirDebuggerSetControl(int controlOption);
+
+/// Print the available context for the current Action.
+void mlirDebuggerPrintContext();
+
+/// Print the current action backtrace.
+void mlirDebuggerPrintActionBacktrace(bool withContext);
+
+//===----------------------------------------------------------------------===//
+// Cursor Management: The cursor is used to select an IRUnit from the context
+// and to navigate through the IRUnit hierarchy.
+//===----------------------------------------------------------------------===//
+
+/// Print the current IR unit cursor.
+void mlirDebuggerCursorPrint(bool withRegion);
+
+/// Select the IR unit from the current context by ID.
+void mlirDebuggerCursorSelectIRUnitFromContext(int index);
+
+/// Select the parent IR unit of the provided IR unit, or print an error if the
+/// IR unit has no parent.
+void mlirDebuggerCursorSelectParentIRUnit();
+
+/// Select the child IR unit at the provided index, print an error if the index
+/// is out of bound. For example if the irunit is an operation, the children IR
+/// units will be the operation's regions.
+void mlirDebuggerCursorSelectChildIRUnit(int index);
+
+/// Return the next IR unit logically in the IR. For example if the irunit is a
+/// Region the next IR unit will be the next region in the parent operation or
+/// nullptr if there is no next region.
+void mlirDebuggerCursorSelectPreviousIRUnit();
+
+/// Return the previous IR unit logically in the IR. For example if the irunit
+/// is a Region, the previous IR unit will be the previous region in the parent
+/// operation or nullptr if there is no previous region.
+void mlirDebuggerCursorSelectNextIRUnit();
+
+//===----------------------------------------------------------------------===//
+// Breakpoint Management
+//===----------------------------------------------------------------------===//
+
+/// Enable the provided breakpoint.
+void mlirDebuggerEnableBreakpoint(BreakpointHandle breakpoint);
+
+/// Disable the provided breakpoint.
+void mlirDebuggerDisableBreakpoint(BreakpointHandle breakpoint);
+
+/// Add a breakpoint matching exactly the provided tag.
+BreakpointHandle mlirDebuggerAddTagBreakpoint(const char *tag);
+
+/// Add a breakpoint matching a pattern by name.
+void mlirDebuggerAddRewritePatternBreakpoint(const char *patternNameInfo);
+
+/// Add a breakpoint matching a file, line and column.
+void mlirDebuggerAddFileLineColLocBreakpoint(const char *file, int line,
+                                             int col);
+
+} // extern "C"
+
+namespace mlir {
+// Setup the debugger hooks as a callback on the provided ExecutionContext.
+void setupDebuggerExecutionContextHook(
+    tracing::ExecutionContext &executionContext);
+
+} // namespace mlir
+
+#endif // MLIR_SUPPORT_DEBUGGEREXECUTIONCONTEXTHOOK_H
diff --git a/mlir/include/mlir/Debug/ExecutionContext.h b/mlir/include/mlir/Debug/ExecutionContext.h
index 097ae372ca35b..fbad04b9a2f1d 100644
--- a/mlir/include/mlir/Debug/ExecutionContext.h
+++ b/mlir/include/mlir/Debug/ExecutionContext.h
@@ -28,8 +28,16 @@ struct ActionActiveStack {
   const ActionActiveStack *getParent() const { return parent; }
   const Action &getAction() const { return action; }
   int getDepth() const { return depth; }
+  void print(raw_ostream &os, bool withContext) const;
+  void dump() const {
+    print(llvm::errs(), /*withContext=*/true);
+    llvm::errs() << "\n";
+  }
+  Breakpoint *getBreakpoint() const { return breakpoint; }
+  void setBreakpoint(Breakpoint *breakpoint) { this->breakpoint = breakpoint; }
 
 private:
+  Breakpoint *breakpoint = nullptr;
   const ActionActiveStack *parent;
   const Action &action;
   int depth;
@@ -69,7 +77,9 @@ class ExecutionContext {
   ExecutionContext() = default;
 
   /// Set the callback that is used to control the execution.
-  void setCallback(CallbackTy callback);
+  void setCallback(CallbackTy callback) {
+    onBreakpointControlExecutionCallback = callback;
+  }
 
   /// This abstract class defines the interface used to observe an Action
   /// execution. It allows to be notified before and after the callback is
diff --git a/mlir/include/mlir/Dialect/AMDGPU/CMakeLists.txt b/mlir/include/mlir/Dialect/AMDGPU/CMakeLists.txt
index ed074c205a551..9f57627c321fb 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/AMDGPU/CMakeLists.txt
@@ -1,12 +1,2 @@
-add_mlir_dialect(AMDGPU amdgpu)
-add_mlir_doc(AMDGPU AMDGPU Dialects/ -gen-dialect-doc)
-
-set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
-mlir_tablegen(AMDGPUEnums.h.inc -gen-enum-decls)
-mlir_tablegen(AMDGPUEnums.cpp.inc -gen-enum-defs)
-add_public_tablegen_target(MLIRAMDGPUEnumsGen)
-
-set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
-mlir_tablegen(AMDGPUAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=amdgpu)
-mlir_tablegen(AMDGPUAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=amdgpu)
-add_public_tablegen_target(MLIRAMDGPUAttributesIncGen)
+add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
similarity index 90%
rename from mlir/include/mlir/Dialect/AMDGPU/AMDGPU.td
rename to mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 3589fa607b72d..bec2371a6a705 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -136,6 +136,48 @@ def AMDGPU_RawBufferStoreOp :
   let hasVerifier = 1;
 }
 
+// Raw buffer atomic compare-and-swap
+def AMDGPU_RawBufferAtomicCmpswapOp :
+    AMDGPU_Op<"raw_buffer_atomic_cmpswap", [
+      AttrSizedOperandSegments,
+      AllTypesMatch<["src", "cmp", "value"]>,
+      AllElementTypesMatch<["value", "memref"]>]>,
+    Arguments<(ins AnyTypeOf<[I32, I64, F32, F64]>:$src,
+                   AnyType:$cmp,
+                   Arg<AnyMemRef, "buffer to operate on", [MemRead, MemWrite]>:$memref,
+                   Variadic<I32>:$indices,
+                   DefaultValuedAttr<BoolAttr, "true">:$boundsCheck,
+                   OptionalAttr<I32Attr>:$indexOffset,
+                   Optional<I32>:$sgprOffset)>,
+    Results<(outs AnyType:$value)> {
+
+  let summary = "Raw Buffer Atomic compare-and-swap";
+  let description = [{
+    The `amdgpu.raw_buffer_atomic_cmpswap` op is a wrapper around the
+    buffer-based atomic compare-and-swap min available on AMD GPUs.
+
+    The index into the buffer is computed as for `memref.store` with the addition
+    of `indexOffset` (which is used to aid in emitting vectorized code) and,
+    if present `sgprOffset` (which is added after bounds checks and includes
+    any non-zero offset on the memref type).
+
+    All indexing components are given in terms of the memref's element size, not
+    the byte lengths required by the intrinsic.
+
+    Out of bounds atomic operations are ignored in hardware.
+
+    See `amdgpu.raw_buffer_load` for a description of how the underlying
+    instruction is constructed.
+  }];
+  let assemblyFormat = [{
+    attr-dict $src `,` $cmp `->` $memref `[` $indices `]`
+      (`sgprOffset` $sgprOffset^)? `:`
+      type($value) `->` type($memref) `,` type($indices)
+  }];
+  let hasCanonicalizer = 1;
+  let hasVerifier = 1;
+}
+
 // Raw buffer atomic floating point add
 def AMDGPU_RawBufferAtomicFaddOp :
     AMDGPU_Op<"raw_buffer_atomic_fadd", [AllElementTypesMatch<["value", "memref"]>,
@@ -179,7 +221,7 @@ def AMDGPU_RawBufferAtomicFaddOp :
 def AMDGPU_RawBufferAtomicFmaxOp :
     AMDGPU_Op<"raw_buffer_atomic_fmax", [AllElementTypesMatch<["value", "memref"]>,
       AttrSizedOperandSegments]>,
-    Arguments<(ins F32:$value,
+    Arguments<(ins AnyTypeOf<[F32, F64]>:$value,
                    Arg<AnyMemRef, "buffer to operate on", [MemRead, MemWrite]>:$memref,
                    Variadic<I32>:$indices,
                    DefaultValuedAttr<BoolAttr, "true">:$boundsCheck,
diff --git a/mlir/include/mlir/Dialect/AMDGPU/AMDGPUDialect.h b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
similarity index 68%
rename from mlir/include/mlir/Dialect/AMDGPU/AMDGPUDialect.h
rename to mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
index 7a7b86695ccf2..90b7936def019 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/AMDGPUDialect.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h
@@ -11,22 +11,22 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_AMDGPU_AMDGPUDIALECT_H_
-#define MLIR_DIALECT_AMDGPU_AMDGPUDIALECT_H_
+#ifndef MLIR_DIALECT_AMDGPU_IR_AMDGPUDIALECT_H_
+#define MLIR_DIALECT_AMDGPU_IR_AMDGPUDIALECT_H_
 
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
-#include "mlir/Dialect/AMDGPU/AMDGPUDialect.h.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h.inc"
 
-#include "mlir/Dialect/AMDGPU/AMDGPUEnums.h.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.h.inc"
 
 #define GET_ATTRDEF_CLASSES
-#include "mlir/Dialect/AMDGPU/AMDGPUAttributes.h.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUAttributes.h.inc"
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/AMDGPU/AMDGPU.h.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPU.h.inc"
 
-#endif // MLIR_DIALECT_AMDGPU_AMDGPUDIALECT_H_
+#endif // MLIR_DIALECT_AMDGPU_IR_AMDGPUDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/AMDGPU/IR/CMakeLists.txt
new file mode 100644
index 0000000000000..ed074c205a551
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_mlir_dialect(AMDGPU amdgpu)
+add_mlir_doc(AMDGPU AMDGPU Dialects/ -gen-dialect-doc)
+
+set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
+mlir_tablegen(AMDGPUEnums.h.inc -gen-enum-decls)
+mlir_tablegen(AMDGPUEnums.cpp.inc -gen-enum-defs)
+add_public_tablegen_target(MLIRAMDGPUEnumsGen)
+
+set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
+mlir_tablegen(AMDGPUAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=amdgpu)
+mlir_tablegen(AMDGPUAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=amdgpu)
+add_public_tablegen_target(MLIRAMDGPUAttributesIncGen)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/AMDGPU/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..8880989f4fed4
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/CMakeLists.txt
@@ -0,0 +1,6 @@
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name AMDGPU)
+add_public_tablegen_target(MLIRAMDGPUTransformsIncGen)
+add_dependencies(mlir-headers MLIRAMDGPUTransformsIncGen)
+
+add_mlir_doc(Passes AMDGPUPasses ./ -gen-pass-doc)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
new file mode 100644
index 0000000000000..8dd5ff1a4b198
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
@@ -0,0 +1,33 @@
+//===-- Passes.h - AMDGPU transformation pass declarations --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the transformation passes for the TOSA Dialect in MLIR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_H_
+#define MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_H_
+
+#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+class ConversionTarget;
+namespace amdgpu {
+
+#define GEN_PASS_DECL_AMDGPUEMULATEATOMICSPASS
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
+
+void populateAmdgpuEmulateAtomicsPatterns(ConversionTarget &target,
+                                          RewritePatternSet &patterns,
+                                          Chipset chipset);
+} // namespace amdgpu
+} // namespace mlir
+
+#endif // MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
new file mode 100644
index 0000000000000..e6b27aa842dfc
--- /dev/null
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
@@ -0,0 +1,33 @@
+//===-- Passes.td - AMDGPU pass declarations ----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the passes for the AMDGPU Dialect in MLIR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_TD_
+#define MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_TD_
+
+include "mlir/Pass/PassBase.td"
+
+def AmdgpuEmulateAtomicsPass : Pass<"amdgpu-emulate-atomics"> {
+  let summary = "Emulate atomic operations on chipsets that do not support them";
+  let description = [{
+    This pass rewrites any AMDGPU-specific atomic operation that is not supported
+    on the given `chipset` into a compare-and-swap loop.
+  }];
+  let dependentDialects = [
+    "cf::ControlFlowDialect",
+    "arith::ArithDialect",
+  ];
+  let options = [Option<"chipset", "chipset", "std::string",
+                        /*default=*/"\"gfx000\"",
+                        "Chipset that these operations will run on">];
+}
+
+#endif // MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_TD_
diff --git a/mlir/include/mlir/Conversion/AMDGPUToROCDL/Chipset.h b/mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h
similarity index 88%
rename from mlir/include/mlir/Conversion/AMDGPUToROCDL/Chipset.h
rename to mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h
index 82808821b6e95..5031dc4856173 100644
--- a/mlir/include/mlir/Conversion/AMDGPUToROCDL/Chipset.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/Utils/Chipset.h
@@ -5,8 +5,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#ifndef MLIR_CONVERSION_AMDGPUTOROCDL_CHIPSET_H_
-#define MLIR_CONVERSION_AMDGPUTOROCDL_CHIPSET_H_
+#ifndef MLIR_DIALECT_AMDGPU_UTILS_CHIPSET_H_
+#define MLIR_DIALECT_AMDGPU_UTILS_CHIPSET_H_
 
 #include "mlir/Support/LogicalResult.h"
 
diff --git a/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
index 02938d1f7e6c4..8e840e744064d 100644
--- a/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
@@ -15,9 +15,11 @@
 #define MLIR_DIALECT_AFFINE_TRANSFORMS_TRANSFORMS_H
 
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
+#include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
 
 namespace mlir {
+class AffineMap;
 class Location;
 class OpBuilder;
 class OpFoldResult;
@@ -85,6 +87,18 @@ FailureOr<OpFoldResult> reifyShapedValueDimBound(
     ValueBoundsConstraintSet::StopConditionFn stopCondition = nullptr,
     bool closedUB = false);
 
+/// Materialize an already computed bound with Affine dialect ops.
+///
+/// * `ValueBoundsOpInterface::computeBound` computes bounds but does not
+///   create IR. It is dialect independent.
+/// * `materializeComputedBound` materializes computed bounds with Affine
+///   dialect ops.
+/// * `reifyIndexValueBound`/`reifyShapedValueDimBound` are a combination of
+///   the two functions mentioned above.
+OpFoldResult materializeComputedBound(
+    OpBuilder &b, Location loc, AffineMap boundMap,
+    ArrayRef<std::pair<Value, std::optional<int64_t>>> mapOperands);
+
 } // namespace affine
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
index 78fd7bdf012f8..1e272c2c3faaf 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
@@ -24,6 +24,7 @@ def Arith_Dialect : Dialect {
 
   let hasConstantMaterializer = 1;
   let useDefaultAttributePrinterParser = 1;
+  let usePropertiesForAttributes = 1;
 }
 
 // The predicate indicates the type of the comparison to perform:
diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
index 1aa2e55dfdc98..c4010b7c0b57a 100644
--- a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
@@ -62,6 +62,10 @@ void populateIntRangeOptimizationsPatterns(RewritePatternSet &patterns,
 /// Create a pass which do optimizations based on integer range analysis.
 std::unique_ptr<Pass> createIntRangeOptimizationsPass();
 
+/// Add patterns for integer bitwidth narrowing.
+void populateArithIntNarrowingPatterns(RewritePatternSet &patterns,
+                                       const ArithIntNarrowingOptions &options);
+
 //===----------------------------------------------------------------------===//
 // Registration
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td
index 29b5e6f1dee86..50b748435afa9 100644
--- a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td
@@ -83,4 +83,18 @@ def ArithEmulateWideInt : Pass<"arith-emulate-wide-int"> {
   let dependentDialects = ["vector::VectorDialect"];
 }
 
+def ArithIntNarrowing : Pass<"arith-int-narrowing"> {
+  let summary = "Reduce integer operation bitwidth";
+  let description = [{
+    Reduce bitwidths of integer types used in arith operations. This pass
+    prefers the narrowest available integer bitwidths that are guaranteed to
+    produce the same results.
+  }];
+  let dependentDialects = ["vector::VectorDialect"];
+  let options = [
+    ListOption<"bitwidthsSupported", "int-bitwidths-supported", "unsigned",
+               "Integer bitwidths supported">,
+  ];
+ }
+
 #endif // MLIR_DIALECT_ARITH_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index f6a815087673d..a53dea49b8adc 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -37,15 +37,14 @@ def EmitC_ApplyOp : EmitC_Op<"apply", []> {
 
     ```mlir
     // Custom form of applying the & operator.
-    %0 = emitc.apply "&"(%arg0) : (i32) -> !emitc.opaque<"int32_t*">
+    %0 = emitc.apply "&"(%arg0) : (i32) -> !emitc.ptr<i32>
 
     // Generic form of the same operation.
     %0 = "emitc.apply"(%arg0) {applicableOperator = "&"}
-        : (i32) -> !emitc.opaque<"int32_t*">
+        : (i32) -> !emitc.ptr<i32>
 
     ```
   }];
-  // TODO: Disallow to apply the operator & (address of) to emitc.constant operations.
   let arguments = (ins
     Arg<StrAttr, "the operator to apply">:$applicableOperator,
     AnyType:$operand
@@ -143,7 +142,6 @@ def EmitC_ConstantOp : EmitC_Op<"constant", [ConstantLike]> {
   let results = (outs AnyType);
 
   let hasFolder = 1;
-  // TODO: Disallow empty constants.
   let hasVerifier = 1;
 }
 
diff --git a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
index 1a06d6533b2d0..4204bc576970f 100644
--- a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
+++ b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
@@ -23,6 +23,7 @@ def Func_Dialect : Dialect {
   let cppNamespace = "::mlir::func";
   let dependentDialects = ["cf::ControlFlowDialect"];
   let hasConstantMaterializer = 1;
+  let usePropertiesForAttributes = 1;
 }
 
 // Base class for Func dialect ops.
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 860e20720afd9..12e09654191da 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -252,7 +252,9 @@ def GPU_GPUFuncOp : GPU_Op<"func", [
 
   let arguments = (ins TypeAttrOf<FunctionType>:$function_type,
                        OptionalAttr<DictArrayAttr>:$arg_attrs,
-                       OptionalAttr<DictArrayAttr>:$res_attrs);
+                       OptionalAttr<DictArrayAttr>:$res_attrs,
+                       OptionalAttr<DictArrayAttr>:$workgroup_attrib_attrs,
+                       OptionalAttr<DictArrayAttr>:$private_attrib_attrs);
   let regions = (region AnyRegion:$body);
 
   let skipDefaultBuilders = 1;
@@ -279,11 +281,17 @@ def GPU_GPUFuncOp : GPU_Op<"func", [
       return attr ? attr.getInt() : 0;
     }
 
+    /// Return the index of the first workgroup attribution in the block argument
+    /// list.
+    unsigned getFirstWorkgroupAttributionIndex() {
+      return getFunctionType().getNumInputs();
+    }
+
     /// Returns a list of block arguments that correspond to buffers located in
     /// the workgroup memory
     ArrayRef<BlockArgument> getWorkgroupAttributions() {
       auto begin =
-          std::next(getBody().args_begin(), getFunctionType().getNumInputs());
+          std::next(getBody().args_begin(), getFirstWorkgroupAttributionIndex());
       auto end = std::next(begin, getNumWorkgroupAttributions());
       return {begin, end};
     }
@@ -292,20 +300,47 @@ def GPU_GPUFuncOp : GPU_Op<"func", [
     /// workgroup memory.
     BlockArgument addWorkgroupAttribution(Type type, Location loc);
 
+    /// Get the workgroup attribution attribute dictionary for the attribution
+    /// at index `index`, counted from the start of the workgroup attributions.
+    DictionaryAttr getworkgroupAttributionAttrs(unsigned index);
+
+    /// Set the workgroup attribution attribute dictionary for the attribution
+    /// at index `index`, counted from the start of the workgroup attributions.
+    void setworkgroupAttributionAttrs(unsigned index, DictionaryAttr value);
+
+    /// Get an attribute for a workgroup attribution. `index` is counted
+    /// from the start of the workgroup attributions, not the start of the block.
+    Attribute getWorkgroupAttributionAttr(unsigned index, StringAttr name);
+    Attribute getWorkgroupAttributionAttr(unsigned index, StringRef name) {
+      return getWorkgroupAttributionAttr(index, StringAttr::get((*this)->getContext(), name));
+    }
+
+    /// Set an attribute for a workgroup attribution. `index` is counted
+    /// from the start of the workgroup attributions, not the start of the block.
+    /// A null `value` removes an attributino attribute.
+    void setWorkgroupAttributionAttr(unsigned index, StringAttr name, Attribute value);
+    void setWorkgroupAttributionAttr(unsigned index, StringRef name, Attribute value) {
+      return setWorkgroupAttributionAttr(index, StringAttr::get((*this)->getContext(), name), value);
+    }
+
     /// Returns the number of buffers located in the private memory.
     unsigned getNumPrivateAttributions() {
       return getBody().getNumArguments() - getFunctionType().getNumInputs() -
           getNumWorkgroupAttributions();
     }
 
+    /// Returns the index of the first private buffer in the block argument list.
+    unsigned getFirstPrivateAttributionIndex() {
+      // Buffers on the private memory always come after buffers on the workgroup
+      // memory.
+      return getFunctionType().getNumInputs() + getNumWorkgroupAttributions();
+    }
+
     /// Returns a list of block arguments that correspond to buffers located in
     /// the private memory.
     ArrayRef<BlockArgument> getPrivateAttributions() {
-      // Buffers on the private memory always come after buffers on the workgroup
-      // memory.
       auto begin =
-          std::next(getBody().args_begin(),
-                    getFunctionType().getNumInputs() + getNumWorkgroupAttributions());
+          std::next(getBody().args_begin(), getFirstPrivateAttributionIndex());
       return {begin, getBody().args_end()};
     }
 
@@ -313,6 +348,29 @@ def GPU_GPUFuncOp : GPU_Op<"func", [
     /// private memory.
     BlockArgument addPrivateAttribution(Type type, Location loc);
 
+    /// Get the private attribution attribute dictionary for the attribution
+    /// at index `index`, counted from the start of the private attributions.
+    DictionaryAttr getPrivateAttributionAttrs(unsigned index);
+
+    /// Set the private attribution attribute dictionary for the attribution
+    /// at index `index`, counted from the start of the private attributions.
+    void setPrivateAttributionAttrs(unsigned index, DictionaryAttr value);
+
+    /// Get an attribute for a private attribution. `index` is counted
+    /// from the start of the private attributions, not the start of the block.
+    Attribute getPrivateAttributionAttr(unsigned index, StringAttr name);
+    Attribute getPrivateAttributionAttr(unsigned index, StringRef name) {
+      return getPrivateAttributionAttr(index, StringAttr::get((*this)->getContext(), name));
+    }
+
+    /// Set an attribute for a private attribution. `index` is counted
+    /// from the start of the private attributions, not the start of the block.
+    /// A null `value` removes an attribute.
+    void setPrivateAttributionAttr(unsigned index, StringAttr name, Attribute value);
+    void setPrivateAttributionAttr(unsigned index, StringRef name, Attribute value) {
+      return setPrivateAttributionAttr(index, StringAttr::get((*this)->getContext(), name), value);
+    }
+
     /// Returns the name of the attribute containing the number of buffers
     /// located in the workgroup memory.
     static StringRef getNumWorkgroupAttributionsAttrName() {
@@ -536,12 +594,14 @@ def GPU_LaunchOp : GPU_Op<"launch", [
     to the amount of dynamic shared memory a kernel's workgroup should be
     allocated; when this operand is not present, a zero size is assumed.
 
-    The body region has _twelve_ arguments, grouped as follows:
+    The body region has at least _twelve_ arguments, grouped as follows:
 
     -   three arguments that contain block identifiers along x,y,z dimensions;
     -   three arguments that contain thread identifiers along x,y,z dimensions;
     -   operands of the `gpu.launch` operation as is (i.e. the operands for
         grid and block sizes).
+    -   a variadic number of Workgroup memory attributions.
+    -   a variadic number of Private memory attributions.
 
     Syntax:
 
@@ -550,8 +610,11 @@ def GPU_LaunchOp : GPU_Op<"launch", [
                              `block` `(` ssa-id-list `)` `in` ssa-reassignment
                              `threads` `(` ssa-id-list `)` `in` ssa-reassignment
                              (dynamic_shared_memory_size ssa-use)?
+                             memory-attribution
                              region attr-dict?
     ssa-reassignment ::= `(` ssa-id `=` ssa-use (`,` ssa-id `=` ssa-use)* `)`
+    memory-attribution ::= (`workgroup` `(` ssa-id-and-type-list `)`)?
+                           (`private` `(` ssa-id-and-type-list `)`)?
     ```
 
     Example:
@@ -582,6 +645,18 @@ def GPU_LaunchOp : GPU_Op<"launch", [
       "some_op"(%bx, %tx) : (index, index) -> ()
       %3 = "memref.load"(%val1, %bx) : (memref<?xf32, 1>, index) -> f32
     }
+
+    // Launch with memory attributions.
+    gpu.launch blocks(%bx, %by, %bz) in (%sz_bx = %0, %sz_by = %1, %sz_bz = %2)
+               threads(%tx, %ty, %tz) in (%sz_tx = %3, %sz_ty = %4, %sz_tz = %5)
+               workgroup(%workgroup: memref<32xf32, 3>)
+               private(%private: memref<1xf32, 5>) {
+      // Block and thread identifiers, as well as block/grid sizes are
+      // immediately usable inside body region.
+      "some_op"(%bx, %tx) : (index, index) -> ()
+      // Assuming %val1 is defined outside the gpu.launch region.
+      %42 = load %workgroup[%bx] : memref<32xf32, 3>
+    }
     ```
 
     Rationale: using operation/block arguments gives analyses a clear way of
@@ -601,7 +676,9 @@ def GPU_LaunchOp : GPU_Op<"launch", [
       "Value":$blockSizeZ,
       CArg<"Value", "nullptr">:$dynamicSharedMemorySize,
       CArg<"Type", "nullptr">:$asyncTokenType,
-      CArg<"ValueRange", "{}">:$asyncDependencies)>
+      CArg<"ValueRange", "{}">:$asyncDependencies,
+      CArg<"TypeRange", "{}">:$workgroupAttributions,
+      CArg<"TypeRange", "{}">:$privateAttributions)>
   ];
 
   let extraClassDeclaration = [{
@@ -632,6 +709,57 @@ def GPU_LaunchOp : GPU_Op<"launch", [
     /// The number of region attributes containing the launch configuration,
     /// placed in the leading positions of the argument list.
     static constexpr unsigned kNumConfigRegionAttributes = 12;
+
+    /// Returns the keywords used in the custom syntax for this Op.
+    static StringRef getWorkgroupKeyword() { return "workgroup"; }
+    static StringRef getPrivateKeyword() { return "private"; }
+
+    /// Returns the number of buffers located in the workgroup memory.
+    unsigned getNumWorkgroupAttributions() {
+      auto attr = (*this)->getAttrOfType<IntegerAttr>(
+          getNumWorkgroupAttributionsAttrName());
+      return attr ? attr.getInt() : 0;
+    }
+
+    /// Returns a list of block arguments that correspond to buffers located in
+    /// the workgroup memory
+    ArrayRef<BlockArgument> getWorkgroupAttributions() {
+      auto begin =
+          std::next(getBody().args_begin(), kNumConfigRegionAttributes);
+      auto end = std::next(begin, getNumWorkgroupAttributions());
+      return {begin, end};
+    }
+
+    /// Adds a new block argument that corresponds to buffers located in
+    /// workgroup memory.
+    BlockArgument addWorkgroupAttribution(Type type, Location loc);
+
+    /// Returns the number of buffers located in the private memory.
+    unsigned getNumPrivateAttributions() {
+      return getBody().getNumArguments() - kNumConfigRegionAttributes -
+          getNumWorkgroupAttributions();
+    }
+
+    /// Returns a list of block arguments that correspond to buffers located in
+    /// the private memory.
+    ArrayRef<BlockArgument> getPrivateAttributions() {
+      // Buffers on the private memory always come after buffers on the workgroup
+      // memory.
+      auto begin =
+          std::next(getBody().args_begin(),
+                    kNumConfigRegionAttributes + getNumWorkgroupAttributions());
+      return {begin, getBody().args_end()};
+    }
+
+    /// Adds a new block argument that corresponds to buffers located in
+    /// private memory.
+    BlockArgument addPrivateAttribution(Type type, Location loc);
+
+    /// Returns the name of the attribute containing the number of buffers
+    /// located in the workgroup memory.
+    static StringRef getNumWorkgroupAttributionsAttrName() {
+      return "workgroup_attributions";
+    }
   }];
 
   let hasCanonicalizer = 1;
diff --git a/mlir/include/mlir/Dialect/IRDL/AttributeWrapper.h b/mlir/include/mlir/Dialect/IRDL/AttributeWrapper.h
new file mode 100644
index 0000000000000..91d45002162fb
--- /dev/null
+++ b/mlir/include/mlir/Dialect/IRDL/AttributeWrapper.h
@@ -0,0 +1,144 @@
+//===- AttributeWrapper.h - IRDL type wrapper definition --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares wrappers around attribute and type definitions to
+// manipulate them in a unified way, using their names and a list of parameters
+// encoded as a list of attributes. These wrappers are necessary for IRDL, since
+// attributes and types don't have names, nor a way to interact with them in a
+// generic way.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_IRDL_ATTRIBUTEWRAPPER_H_
+#define MLIR_DIALECT_IRDL_ATTRIBUTEWRAPPER_H_
+
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "llvm/ADT/SmallString.h"
+
+namespace mlir {
+namespace irdl {
+
+/// A wrapper around an attribute definition to manipulate its attribute
+/// instances in a unified way, using a name and a list of parameters encoded as
+/// attributes.
+///
+/// If the attribute is defined in C++, CppAttributeWrapper should be preferred.
+class AttributeWrapper {
+public:
+  virtual ~AttributeWrapper(){};
+
+  /// Return the unique identifier of the attribute definition.
+  virtual TypeID getTypeID() = 0;
+
+  /// Check if the given attribute is an instance of the one wrapped.
+  virtual bool isCorrectAttribute(Attribute attr) = 0;
+
+  /// Get the parameters of an attribute. The attribute is expected to be an
+  /// instance of the wrapped attribute, which is checked with
+  /// `isCorrectAttribute`.
+  virtual SmallVector<Attribute> getAttributeParameters(Attribute attr) = 0;
+
+  /// Return the attribute definition name, including the dialect prefix.
+  virtual StringRef getName() = 0;
+
+  /// Instantiate the attribute from parameters.
+  /// It is expected that the amount of parameters is correct, which is checked
+  /// with `getParameterAmount`.
+  virtual Attribute
+  instantiateAttribute(function_ref<InFlightDiagnostic()> emitError,
+                       ArrayRef<Attribute> parameters) = 0;
+
+  /// Return the amount of parameters the attribute expects.
+  virtual size_t getParameterAmount() = 0;
+};
+
+/// A wrapper around a type definition to manipulate its type instances
+/// in an unified way, using a name and a list of parameters encoded as
+/// attributes. The wrappers also acts as an attribute wrapper, and expects
+/// the type to be nested in a `TypeAttr`.
+///
+/// If the type is defined as a C++ class, CppTypeWrapper should be preferred.
+class TypeWrapper : public AttributeWrapper {
+public:
+  virtual ~TypeWrapper(){};
+
+  /// Check if the given type is an instance of the one wrapped.
+  virtual bool isCorrectType(Type t) = 0;
+
+  /// Get the parameters of a type. The type is expected to be an instance of
+  /// the wrapped type, which is checked with `isCorrectType`.
+  virtual SmallVector<Attribute> getTypeParameters(Type t) = 0;
+
+  /// Instantiate the type from parameters.
+  /// It is expected that the amount of parameters is correct, which is checked
+  /// with `getParameterAmount`.
+  virtual Type instantiateType(function_ref<InFlightDiagnostic()> emitError,
+                               ArrayRef<Attribute> parameters) = 0;
+
+  /// Check if the given attribute is a `TypeAttr`, and that it contains an
+  /// instance of the type wrapped.
+  bool isCorrectAttribute(Attribute attr) override;
+
+  /// Get the parameters of a type. The type is expected to be nested in a
+  /// `TypeAttr`, and to be an instance of the wrapped type, which is checked
+  /// with `isCorrectType`.
+  SmallVector<Attribute> getAttributeParameters(Attribute attr) override;
+
+  /// Instantiate the type from parameters, and wrap it in a `TypeAttr`.
+  /// It is expected that the amount of parameters is correct, which is checked
+  /// with `getParameterAmount`.
+  Attribute instantiateAttribute(function_ref<InFlightDiagnostic()> emitError,
+                                 ArrayRef<Attribute> parameters) override;
+};
+
+using AttributeWrapperPtr = AttributeWrapper *;
+using TypeWrapperPtr = TypeWrapper *;
+
+/// A wrapper around an attribute definition defined with a C++ class to
+/// manipulate its attribute instances in a unified way, using a name and a list
+/// of parameters encoded as attributes.
+template <typename A>
+class CppAttributeWrapper : public AttributeWrapper {
+public:
+  TypeID getTypeID() override { return TypeID::get<A>(); }
+
+  /// Get the parameters of an attribute.
+  virtual SmallVector<Attribute> getAttributeParameters(A attr) = 0;
+
+  SmallVector<Attribute> getAttributeParameters(Attribute attr) override {
+    return getAttributeParameters(cast<A>(attr));
+  };
+
+  bool isCorrectAttribute(Attribute attr) override { return isa<A>(attr); }
+};
+
+/// A wrapper around a type definition defined with a C++ class to
+/// manipulate its type instances in a unified way, using a name and a list
+/// of parameters encoded as attributes.
+template <typename T>
+class CppTypeWrapper : public TypeWrapper {
+public:
+  TypeID getTypeID() override { return TypeID::get<T>(); }
+
+  /// Get the parameters of a type.
+  virtual SmallVector<Attribute> getTypeParameters(T t) = 0;
+
+  SmallVector<Attribute> getTypeParameters(Type type) override {
+    return getTypeParameters(cast<T>(type));
+  };
+
+  bool isCorrectType(Type type) override { return isa<T>(type); }
+};
+
+} // namespace irdl
+} // namespace mlir
+
+#endif // MLIR_DIALECT_IRDL_ATTRIBUTEWRAPPER_H_
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
index e165bd72d3b9d..3ad667741f603 100644
--- a/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/IRDL/IR/CMakeLists.txt
@@ -1,5 +1,12 @@
 add_mlir_dialect(IRDL irdl)
 
+# Add IRDL interfaces
+set(LLVM_TARGET_DEFINITIONS IRDLInterfaces.td)
+mlir_tablegen(IRDLInterfaces.h.inc -gen-op-interface-decls)
+mlir_tablegen(IRDLInterfaces.cpp.inc -gen-op-interface-defs)
+add_public_tablegen_target(MLIRIRDLInterfacesIncGen)
+add_dependencies(mlir-generic-headers MLIRIRDLInterfacesIncGen)
+
 # Add IRDL operations
 set(LLVM_TARGET_DEFINITIONS IRDLOps.td)
 mlir_tablegen(IRDLOps.h.inc -gen-op-decls)
@@ -13,3 +20,10 @@ mlir_tablegen(IRDLTypesGen.h.inc -gen-typedef-decls)
 mlir_tablegen(IRDLTypesGen.cpp.inc -gen-typedef-defs)
 add_public_tablegen_target(MLIRIRDLTypesIncGen)
 add_dependencies(mlir-generic-headers MLIRIRDLTypesIncGen)
+
+# Add IRDL attributes
+set(LLVM_TARGET_DEFINITIONS IRDLAttributes.td)
+mlir_tablegen(IRDLAttributes.h.inc -gen-attrdef-decls)
+mlir_tablegen(IRDLAttributes.cpp.inc -gen-attrdef-defs)
+add_public_tablegen_target(MLIRIRDLAttributesIncGen)
+add_dependencies(mlir-generic-headers MLIRIRDLAttributesIncGen)
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/IRDL.h b/mlir/include/mlir/Dialect/IRDL/IR/IRDL.h
index c22f5e28d91c8..91bd6a12e133e 100644
--- a/mlir/include/mlir/Dialect/IRDL/IR/IRDL.h
+++ b/mlir/include/mlir/Dialect/IRDL/IR/IRDL.h
@@ -13,7 +13,10 @@
 #ifndef MLIR_DIALECT_IRDL_IR_IRDL_H_
 #define MLIR_DIALECT_IRDL_IR_IRDL_H_
 
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/Dialect/IRDL/IR/IRDLInterfaces.h"
 #include "mlir/Dialect/IRDL/IR/IRDLTraits.h"
+#include "mlir/Dialect/IRDL/IRDLContext.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
@@ -36,6 +39,9 @@ class OpDefAttr;
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/IRDL/IR/IRDLTypesGen.h.inc"
 
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/IRDL/IR/IRDLAttributes.h.inc"
+
 #define GET_OP_CLASSES
 #include "mlir/Dialect/IRDL/IR/IRDLOps.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/IRDL.td b/mlir/include/mlir/Dialect/IRDL/IR/IRDL.td
index c8303a2cc0ecd..2f69f20292acc 100644
--- a/mlir/include/mlir/Dialect/IRDL/IR/IRDL.td
+++ b/mlir/include/mlir/Dialect/IRDL/IR/IRDL.td
@@ -70,6 +70,13 @@ def IRDL_Dialect : Dialect {
   }];
 
   let useDefaultTypePrinterParser = 1;
+  let useDefaultAttributePrinterParser = 1;
+
+  let extraClassDeclaration = [{
+  public:
+    /// Contains registered attribute and type wrappers.
+    ::mlir::irdl::IRDLContext irdlContext;
+  }];
 
   let name = "irdl";
   let cppNamespace = "::mlir::irdl";
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/IRDLAttributes.td b/mlir/include/mlir/Dialect/IRDL/IR/IRDLAttributes.td
new file mode 100644
index 0000000000000..e1a826a5a1863
--- /dev/null
+++ b/mlir/include/mlir/Dialect/IRDL/IR/IRDLAttributes.td
@@ -0,0 +1,54 @@
+//===- IRDLAttributes.td - IR Definition Language Dialect --*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the attributes used in IRDL.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_IRDL_IR_IRDLATTRIBUTES
+#define MLIR_DIALECT_IRDL_IR_IRDLATTRIBUTES
+
+include "IRDL.td"
+include "mlir/IR/AttrTypeBase.td"
+
+class IRDL_AttrDef<string name, string attrMnemonic, list<Trait> traits = []>
+  : AttrDef<IRDL_Dialect, name, traits, "::mlir::Attribute"> {
+  let mnemonic = attrMnemonic;
+}
+
+def TypeOrAttrDefRefAttr : IRDL_AttrDef<"TypeOrAttrDefRef",
+                                        "type_or_attr_def_ref"> {
+  let summary = "reference to a type or attribute definition";
+  let description = [{
+    An `irdl.type_or_attr_def_ref` references a type or attribute definition.
+    A type or attribute definition can either be a `TypeWrapper *` or
+    an `AttrWrapper *`, which refers to a C++-defined type or attribute,
+    or a `SymbolRefAttr` which refers to a type or attribute defined with IRDL.
+  }];
+  let parameters =
+    (ins OptionalParameter<"mlir::irdl::TypeWrapperPtr">:$typeWrapper,
+         OptionalParameter<"mlir::irdl::AttributeWrapperPtr">:$attrWrapper,
+         OptionalParameter<"mlir::SymbolRefAttr">:$symRef);
+
+  let builders = [
+    AttrBuilder<(ins "mlir::irdl::TypeWrapper*":$typeWrapper), [{
+      return $_get($_ctxt, typeWrapper, nullptr, mlir::SymbolRefAttr());
+    }]>,
+    AttrBuilder<(ins "mlir::irdl::AttributeWrapper*":$attrWrapper), [{
+      return $_get($_ctxt, nullptr, attrWrapper, mlir::SymbolRefAttr());
+    }]>,
+    AttrBuilderWithInferredContext<(ins "mlir::SymbolRefAttr":$symRef), [{
+      return $_get(symRef.getContext(), nullptr, nullptr, symRef);
+    }]>
+  ];
+
+  let genVerifyDecl = 1;
+  let hasCustomAssemblyFormat = 1;
+}
+
+#endif // MLIR_DIALECT_IRDL_IR_IRDLATTRIBUTES
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/IRDLInterfaces.h b/mlir/include/mlir/Dialect/IRDL/IR/IRDLInterfaces.h
new file mode 100644
index 0000000000000..a26f9d04d7c2b
--- /dev/null
+++ b/mlir/include/mlir/Dialect/IRDL/IR/IRDLInterfaces.h
@@ -0,0 +1,37 @@
+//===- IRDLInterfaces.h - IRDL interfaces definition ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the interfaces used by the IRDL dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_IRDL_IR_IRDLINTERFACES_H_
+#define MLIR_DIALECT_IRDL_IR_IRDLINTERFACES_H_
+
+#include "mlir/Dialect/IRDL/IRDLVerifiers.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/Types.h"
+#include "mlir/Support/LogicalResult.h"
+#include <optional>
+
+namespace mlir {
+namespace irdl {
+class TypeOp;
+class AttributeOp;
+} // namespace irdl
+} // namespace mlir
+
+//===----------------------------------------------------------------------===//
+// IRDL Dialect Interfaces
+//===----------------------------------------------------------------------===//
+#include "mlir/Dialect/IRDL/IR/IRDLInterfaces.h.inc"
+
+#endif //  MLIR_DIALECT_IRDL_IR_IRDLINTERFACES_H_
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/IRDLInterfaces.td b/mlir/include/mlir/Dialect/IRDL/IR/IRDLInterfaces.td
new file mode 100644
index 0000000000000..6539ad2b3fd5a
--- /dev/null
+++ b/mlir/include/mlir/Dialect/IRDL/IR/IRDLInterfaces.td
@@ -0,0 +1,40 @@
+//===- IRDLInterfaces.td - IRDL Attributes -----------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the interfaces used by IRDL.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_IRDL_IR_IRDLINTERFACES
+#define MLIR_DIALECT_IRDL_IR_IRDLINTERFACES
+
+include "mlir/IR/OpBase.td"
+
+def VerifyConstraintInterface : OpInterface<"VerifyConstraintInterface"> {
+  let cppNamespace = "::mlir::irdl";
+
+  let description = [{
+    Interface to get an IRDL constraint verifier from an operation. 
+  }];
+
+  let methods = [
+    InterfaceMethod<
+      [{
+        Get an instance of a constraint verifier for the associated operation."
+        Returns `nullptr` upon failure.
+      }],
+      "std::unique_ptr<::mlir::irdl::Constraint>",
+      "getVerifier",
+      (ins "mlir::SmallVector<Value> const&":$valueRes,
+      "llvm::DenseMap<mlir::irdl::TypeOp, std::unique_ptr<mlir::DynamicTypeDefinition>> &":$types,
+      "llvm::DenseMap<mlir::irdl::AttributeOp, std::unique_ptr<mlir::DynamicAttrDefinition>> &":$attrs)
+    >
+  ];
+}
+
+#endif // MLIR_DIALECT_IRDL_IR_IRDLINTERFACES
diff --git a/mlir/include/mlir/Dialect/IRDL/IR/IRDLOps.td b/mlir/include/mlir/Dialect/IRDL/IR/IRDLOps.td
index 59d152440b26e..7b793f7e996c5 100644
--- a/mlir/include/mlir/Dialect/IRDL/IR/IRDLOps.td
+++ b/mlir/include/mlir/Dialect/IRDL/IR/IRDLOps.td
@@ -14,7 +14,9 @@
 #define MLIR_DIALECT_IRDL_IR_IRDLOPS
 
 include "IRDL.td"
+include "IRDLAttributes.td"
 include "IRDLTypes.td"
+include "IRDLInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/IR/SymbolInterfaces.td"
@@ -264,7 +266,8 @@ def IRDL_ResultsOp : IRDL_Op<"results", [HasParent<"OperationOp">]> {
 //===----------------------------------------------------------------------===//
 
 class IRDL_ConstraintOp<string mnemonic, list<Trait> traits = []>
-    : IRDL_Op<mnemonic, traits> {
+    : IRDL_Op<mnemonic, [VerifyConstraintInterface,
+        DeclareOpInterfaceMethods<VerifyConstraintInterface>] # traits> {
 }
 
 def IRDL_Is : IRDL_ConstraintOp<"is",
@@ -300,8 +303,8 @@ def IRDL_Parametric : IRDL_ConstraintOp<"parametric",
   let description = [{
     `irdl.parametric` defines a constraint that accepts only a single type
     or attribute base. The attribute base is defined by a symbolic reference
-    to the corresponding definition. It will additionally constraint the
-    parameters of the type/attribute.
+    to the corresponding definition, or a type/attribute wrapper. It will
+    additionally constraint the parameters of the type/attribute.
 
     Example:
 
@@ -323,7 +326,7 @@ def IRDL_Parametric : IRDL_ConstraintOp<"parametric",
     for any `T` takes a `cmath.complex` with parameter `T` and returns a `T`.
   }];
 
-  let arguments = (ins SymbolRefAttr:$base_type,
+  let arguments = (ins TypeOrAttrDefRefAttr:$base_type,
                        Variadic<IRDL_AttributeType>:$args);
   let results = (outs IRDL_AttributeType:$output);
   let assemblyFormat = " $base_type `<` $args `>` ` ` attr-dict ";
@@ -354,5 +357,74 @@ def IRDL_Any : IRDL_ConstraintOp<"any",
   let assemblyFormat = " attr-dict ";
 }
 
+def IRDL_AnyOf : IRDL_ConstraintOp<"any_of",
+                  [ParentOneOf<["TypeOp", "AttributeOp", "OperationOp"]>,
+                   SameOperandsAndResultType]> {
+  let summary = "Constraints to the union of the provided constraints";
+  let description = [{
+    `irdl.any_of` defines a constraint that accepts any type or attribute that
+    satisfies at least one of its provided type constraints.
+
+    Example:
+
+    ```mlir
+    irdl.dialect cmath {
+      irdl.type complex {
+        %0 = irdl.is i32
+        %1 = irdl.is i64
+        %2 = irdl.is f32
+        %3 = irdl.is f64
+        %4 = irdl.any_of(%0, %1, %2, %3)
+        irdl.parameters(%4)
+      }
+    }
+    ```
+
+    The above program defines a type `complex` inside the dialect `cmath` that
+    can have a single type parameter that can be either `i32`, `i64`, `f32` or
+    `f32`.
+  }];
+
+  let arguments = (ins Variadic<IRDL_AttributeType>:$args);
+  let results = (outs IRDL_AttributeType:$output);
+  let assemblyFormat = [{ `(` $args `)` ` ` attr-dict }];
+}
+
+def IRDL_AllOf : IRDL_ConstraintOp<"all_of",
+                 [ParentOneOf<["TypeOp", "AttributeOp", "OperationOp"]>,
+                  SameOperandsAndResultType]> {
+  let summary = "Constraints to the intersection of the provided constraints";
+  let description = [{
+    `irdl.all_of` defines a constraint that accepts any type or attribute that
+    satisfies all of its provided constraints.
+
+    Example:
+
+    ```mlir
+    irdl.dialect cmath {
+      irdl.type complex_f32 {
+        %0 = irdl.is i32
+        %1 = irdl.is f32
+        %2 = irdl.any_of(%0, %1) // is 32-bit
+
+        %3 = irdl.is f32
+        %4 = irdl.is f64
+        %5 = irdl.any_of(%3, %4) // is a float
+
+        %6 = irdl.all_of(%2, %5) // is a 32-bit float
+        irdl.parameters(%6)
+      }
+    }
+    ```
+
+    The above program defines a type `complex` inside the dialect `cmath` that
+    can has one parameter that must be 32-bit long and a float (in other
+    words, that must be `f32`).
+  }];
+
+  let arguments = (ins Variadic<IRDL_AttributeType>:$args);
+  let results = (outs IRDL_AttributeType:$output);
+  let assemblyFormat = [{ `(` $args `)` ` ` attr-dict }];
+}
 
 #endif // MLIR_DIALECT_IRDL_IR_IRDLOPS
diff --git a/mlir/include/mlir/Dialect/IRDL/IRDLContext.h b/mlir/include/mlir/Dialect/IRDL/IRDLContext.h
new file mode 100644
index 0000000000000..61b6cdc3fed86
--- /dev/null
+++ b/mlir/include/mlir/Dialect/IRDL/IRDLContext.h
@@ -0,0 +1,75 @@
+//===- IRDLContext.h - IRDL context -----------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Manages the registration context of IRDL dialects.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_IRDL_IRDLCONTEXT_H_
+#define MLIR_DIALECT_IRDL_IRDLCONTEXT_H_
+
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/StringMap.h"
+
+namespace mlir {
+namespace irdl {
+
+/// Context for the runtime registration of IRDL dialect definitions.
+/// This class keeps track of all the attribute and types defined in C++
+/// that can be used in IRDL.
+class IRDLContext {
+  llvm::StringMap<std::unique_ptr<AttributeWrapper>> attributes;
+  llvm::StringMap<std::unique_ptr<TypeWrapper>> types;
+  DenseMap<TypeID, AttributeWrapper *> typeIDToAttributeWrapper;
+  DenseMap<TypeID, TypeWrapper *> typeIDToTypeWrapper;
+
+public:
+  IRDLContext();
+
+  /// Add a concrete attribute wrapper to IRDL.
+  /// The attribute definition wrapped can then be used in IRDL with its name.
+  template <typename A>
+  void addAttributeWrapper() {
+    addAttributeWrapper(std::make_unique<A>());
+  }
+
+  /// Add an attribute wrapper to IRDL.
+  /// The attribute definition wrapped can then be used in IRDL with its name.
+  void addAttributeWrapper(std::unique_ptr<AttributeWrapper> wrapper);
+
+  AttributeWrapper *getAttributeWrapper(StringRef typeName);
+  AttributeWrapper *getAttributeWrapper(TypeID typeID);
+
+  /// Add a concrete type wrapper to IRDL.
+  /// The type definition wrapped can then be used in IRDL with its name.
+  template <typename T>
+  void addTypeWrapper() {
+    addTypeWrapper(std::make_unique<T>());
+  }
+
+  /// Add a type wrapper to IRDL.
+  /// The type definition wrapped can then be used in IRDL with its name.
+  void addTypeWrapper(std::unique_ptr<TypeWrapper> wrapper);
+
+  TypeWrapper *getTypeWrapper(StringRef typeName);
+  TypeWrapper *getTypeWrapper(TypeID typeID);
+
+  llvm::StringMap<std::unique_ptr<AttributeWrapper>> const &getAllAttributes() {
+    return attributes;
+  }
+
+  llvm::StringMap<std::unique_ptr<TypeWrapper>> const &getAllTypes() {
+    return types;
+  }
+};
+
+} // namespace irdl
+} // namespace mlir
+
+#endif // MLIR_DIALECT_IRDL_IRDLCONTEXT_H_
diff --git a/mlir/include/mlir/Dialect/IRDL/IRDLVerifiers.h b/mlir/include/mlir/Dialect/IRDL/IRDLVerifiers.h
new file mode 100644
index 0000000000000..f5d0462fe7ff7
--- /dev/null
+++ b/mlir/include/mlir/Dialect/IRDL/IRDLVerifiers.h
@@ -0,0 +1,207 @@
+//===- IRDLVerifiers.h - IRDL verifiers --------------------------- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Verifiers for objects declared by IRDL.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_IRDL_IRDLVERIFIERS_H
+#define MLIR_DIALECT_IRDL_IRDLVERIFIERS_H
+
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include <optional>
+
+namespace mlir {
+struct LogicalResult;
+class InFlightDiagnostic;
+class DynamicAttrDefinition;
+class DynamicTypeDefinition;
+} // namespace mlir
+
+namespace mlir {
+namespace irdl {
+
+class Constraint;
+
+/// Provides context to the verification of constraints.
+/// It contains the assignment of variables to attributes, and the assignment
+/// of variables to constraints.
+class ConstraintVerifier {
+public:
+  ConstraintVerifier(ArrayRef<std::unique_ptr<Constraint>> constraints);
+
+  /// Check that a constraint is satisfied by an attribute.
+  ///
+  /// Constraints may call other constraint verifiers. If that is the case,
+  /// the constraint verifier will check if the variable is already assigned,
+  /// and if so, check that the attribute is the same as the one assigned.
+  /// If the variable is not assigned, the constraint verifier will
+  /// assign the attribute to the variable, and check that the constraint
+  /// is satisfied.
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr, unsigned variable);
+
+private:
+  /// The constraints that can be used for verification.
+  ArrayRef<std::unique_ptr<Constraint>> constraints;
+
+  /// The assignment of variables to attributes. Variables that are not assigned
+  /// are represented by nullopt. Null attributes needs to be supported here as
+  /// some attributes or types might use the null attribute to represent
+  /// optional parameters.
+  SmallVector<std::optional<Attribute>> assigned;
+};
+
+/// Once turned into IRDL verifiers, all constraints are
+/// attribute constraints. Type constraints are represented
+/// as `TypeAttr` attribute constraints to simplify verification.
+/// Verification that a type constraint must yield a
+/// `TypeAttr` attribute happens before conversion, at the MLIR level.
+class Constraint {
+public:
+  virtual ~Constraint() = default;
+
+  /// Check that an attribute is satisfying the constraint.
+  ///
+  /// Constraints may call other constraint verifiers. If that is the case,
+  /// the constraint verifier will check if the variable is already assigned,
+  /// and if so, check that the attribute is the same as the one assigned.
+  /// If the variable is not assigned, the constraint verifier will
+  /// assign the attribute to the variable, and check that the constraint
+  /// is satisfied.
+  virtual LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                               Attribute attr,
+                               ConstraintVerifier &context) const = 0;
+};
+
+/// A constraint that checks that an attribute is equal to a given attribute.
+class IsConstraint : public Constraint {
+public:
+  IsConstraint(Attribute expectedAttribute)
+      : expectedAttribute(expectedAttribute) {}
+
+  virtual ~IsConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+
+private:
+  Attribute expectedAttribute;
+};
+
+/// A constraint that checks that an attribute that has an attribute wrapper is
+/// of a specific attribute definition, and that all of its parameters satisfy
+/// the given constraints.
+/// This class also takes care of types, by expecting a `TypeAttr` attribute.
+class ParametricConstraint : public Constraint {
+public:
+  ParametricConstraint(::mlir::irdl::AttributeWrapper *expectedAttribute,
+                       SmallVector<unsigned> constraints)
+      : expectedAttribute(expectedAttribute),
+        constraints(std::move(constraints)) {}
+
+  virtual ~ParametricConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+
+private:
+  ::mlir::irdl::AttributeWrapper *expectedAttribute;
+  SmallVector<unsigned> constraints;
+};
+
+/// A constraint that checks that an attribute is of a
+/// specific dynamic attribute definition, and that all of its parameters
+/// satisfy the given constraints.
+class DynParametricAttrConstraint : public Constraint {
+public:
+  DynParametricAttrConstraint(DynamicAttrDefinition *attrDef,
+                              SmallVector<unsigned> constraints)
+      : attrDef(attrDef), constraints(std::move(constraints)) {}
+
+  virtual ~DynParametricAttrConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+
+private:
+  DynamicAttrDefinition *attrDef;
+  SmallVector<unsigned> constraints;
+};
+
+/// A constraint that checks that a type is of a specific dynamic type
+/// definition, and that all of its parameters satisfy the given constraints.
+class DynParametricTypeConstraint : public Constraint {
+public:
+  DynParametricTypeConstraint(DynamicTypeDefinition *typeDef,
+                              SmallVector<unsigned> constraints)
+      : typeDef(typeDef), constraints(std::move(constraints)) {}
+
+  virtual ~DynParametricTypeConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+
+private:
+  DynamicTypeDefinition *typeDef;
+  SmallVector<unsigned> constraints;
+};
+
+/// A constraint checking that one of the given constraints is satisfied.
+class AnyOfConstraint : public Constraint {
+public:
+  AnyOfConstraint(SmallVector<unsigned> constraints)
+      : constraints(std::move(constraints)) {}
+
+  virtual ~AnyOfConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+
+private:
+  SmallVector<unsigned> constraints;
+};
+
+/// A constraint checking that all of the given constraints are satisfied.
+class AllOfConstraint : public Constraint {
+public:
+  AllOfConstraint(SmallVector<unsigned> constraints)
+      : constraints(std::move(constraints)) {}
+
+  virtual ~AllOfConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+
+private:
+  SmallVector<unsigned> constraints;
+};
+
+/// A constraint that is always satisfied.
+class AnyAttributeConstraint : public Constraint {
+public:
+  virtual ~AnyAttributeConstraint() = default;
+
+  LogicalResult verify(function_ref<InFlightDiagnostic()> emitError,
+                       Attribute attr,
+                       ConstraintVerifier &context) const override;
+};
+
+} // namespace irdl
+} // namespace mlir
+
+#endif // MLIR_DIALECT_IRDL_IRDLVERIFIERS_H
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index b6b56685df863..05ee0832360dc 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -206,7 +206,9 @@ def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> {
     OptionalParameter<"LoopUnswitchAttr">:$unswitch,
     OptionalParameter<"BoolAttr">:$mustProgress,
     OptionalParameter<"BoolAttr">:$isVectorized,
-    OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses
+    OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses,
+    OptionalParameter<"FusedLoc">:$startLoc,
+    OptionalParameter<"FusedLoc">:$endLoc
   );
 
   let assemblyFormat = "`<` struct(params) `>`";
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
index 9822c092ea473..728f95291a699 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
@@ -29,6 +29,7 @@
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/ThreadLocalCache.h"
+#include "mlir/Transforms/Mem2Reg.h"
 #include "llvm/ADT/PointerEmbeddedInt.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/LLVMContext.h"
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 6e35a3a9d2627..e8e1712cc4f3a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -6,6 +6,7 @@ include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
 include "mlir/Dialect/LLVMIR/LLVMEnums.td"
 include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/Mem2RegInterfaces.td"
 
 // Operations that correspond to LLVM intrinsics. With MLIR operation set being
 // extendable, there is no reason to introduce a hard boundary between "core"
@@ -214,7 +215,8 @@ def LLVM_NoAliasScopeDeclOp
 
 /// Base operation for lifetime markers. The LLVM intrinsics require the size
 /// operand to be an immediate. In MLIR it is encoded as an attribute.
-class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName> {
+class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [],
+    [DeclareOpInterfaceMethods<PromotableOpInterface>]> {
   let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
 
   // Custom builder to convert the size attribute to an integer.
@@ -322,7 +324,8 @@ def LLVM_CoroResumeOp : LLVM_IntrOp<"coro.resume", [], [], [], 0> {
 // Debug function intrinsics.
 //
 
-class LLVM_DbgIntrOp<string name, string argName> : LLVM_IntrOp<name, [], [], [], 0> {
+class LLVM_DbgIntrOp<string name, string argName, list<Trait> traits = []>
+    : LLVM_IntrOp<name, [], [], traits, 0> {
   let llvmBuilder = [{
     llvm::Module *module = builder.GetInsertBlock()->getModule();
     llvm::LLVMContext &ctx = module->getContext();
@@ -363,7 +366,8 @@ class LLVM_DbgIntrOp<string name, string argName> : LLVM_IntrOp<name, [], [], []
   }];
 }
 
-def LLVM_DbgDeclareOp : LLVM_DbgIntrOp<"dbg.declare", "addr"> {
+def LLVM_DbgDeclareOp : LLVM_DbgIntrOp< "dbg.declare", "addr",
+    [DeclareOpInterfaceMethods<PromotableOpInterface>]> {
   let summary = "Declare the address of a local debug info variable.";
   let arguments = (ins LLVM_AnyPointer:$addr, LLVM_DILocalVariableAttr:$varInfo);
 }
@@ -712,6 +716,26 @@ def LLVM_masked_compressstore
   let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
 }
 
+//
+// Trap intrinsics.
+//
+
+def LLVM_Trap : LLVM_ZeroResultIntrOp<"trap">;
+
+def LLVM_DebugTrap : LLVM_ZeroResultIntrOp<"debugtrap">;
+
+def LLVM_UBSanTrap : LLVM_ZeroResultIntrOp<"ubsantrap"> {
+  let arguments = (ins I8Attr:$failureKind);
+  string llvmBuilder = [{
+    builder.CreateIntrinsic(
+      llvm::Intrinsic::ubsantrap, {}, {builder.getInt8($failureKind)});
+  }];
+  string mlirBuilder = [{
+    $_op =
+      $_builder.create<LLVM::UBSanTrap>($_location, $_int_attr($failureKind));
+  }];
+}
+
 /// Create a call to vscale intrinsic.
 def LLVM_vscale : LLVM_IntrOp<"vscale", [0], [], [], 1>;
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 29b41ff122d97..f9025e01c1f69 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -22,6 +22,7 @@ include "mlir/IR/SymbolInterfaces.td"
 include "mlir/Interfaces/CallInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/Mem2RegInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
 class LLVM_Builder<string builder> {
@@ -171,7 +172,9 @@ def LLVM_FNegOp : LLVM_UnaryFloatArithmeticOp<
   LLVM_ScalarOrVectorOf<LLVM_AnyFloat>, "fneg", "FNeg">;
 
 // Memory-related operations.
-def LLVM_AllocaOp : LLVM_Op<"alloca">, LLVM_MemOpPatterns {
+def LLVM_AllocaOp : LLVM_Op<"alloca", 
+    [DeclareOpInterfaceMethods<PromotableAllocationOpInterface>]>,
+  LLVM_MemOpPatterns {
   let arguments = (ins AnyInteger:$arraySize,
                    OptionalAttr<I64Attr>:$alignment,
                    OptionalAttr<TypeAttr>:$elem_type,
@@ -228,7 +231,8 @@ def LLVM_AllocaOp : LLVM_Op<"alloca">, LLVM_MemOpPatterns {
   let hasVerifier = 1;
 }
 
-def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure]> {
+def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure,
+    DeclareOpInterfaceMethods<PromotableOpInterface>]> {
   let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyPointer>:$base,
                    Variadic<LLVM_ScalarOrVectorOf<AnyInteger>>:$dynamicIndices,
                    DenseI32ArrayAttr:$rawConstantIndices,
@@ -311,7 +315,8 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure]> {
   let hasVerifier = 1;
 }
 
-def LLVM_LoadOp : LLVM_MemAccessOpBase<"load"> {
+def LLVM_LoadOp : LLVM_MemAccessOpBase<"load",
+    [DeclareOpInterfaceMethods<PromotableMemOpInterface>]> {
   dag args = (ins Arg<LLVM_PointerTo<LLVM_LoadableType>, "", [MemRead]>:$addr,
               OptionalAttr<I64Attr>:$alignment,
               UnitAttr:$volatile_,
@@ -382,7 +387,8 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load"> {
   let hasVerifier = 1;
 }
 
-def LLVM_StoreOp : LLVM_MemAccessOpBase<"store"> {
+def LLVM_StoreOp : LLVM_MemAccessOpBase<"store",
+    [DeclareOpInterfaceMethods<PromotableMemOpInterface>]> {
   dag args = (ins LLVM_LoadableType:$value,
               Arg<LLVM_PointerTo<LLVM_LoadableType>,"",[MemWrite]>:$addr,
               OptionalAttr<I64Attr>:$alignment,
@@ -465,14 +471,15 @@ class LLVM_CastOp<string mnemonic, string instName, Type type,
       $_location, $_resultType, $arg);
   }];
 }
-def LLVM_BitcastOp : LLVM_CastOp<"bitcast", "BitCast",
-                                 LLVM_AnyNonAggregate, LLVM_AnyNonAggregate> {
+def LLVM_BitcastOp : LLVM_CastOp<"bitcast", "BitCast", LLVM_AnyNonAggregate,
+    LLVM_AnyNonAggregate, [DeclareOpInterfaceMethods<PromotableOpInterface>]> {
   let hasFolder = 1;
   let hasVerifier = 1;
 }
 def LLVM_AddrSpaceCastOp : LLVM_CastOp<"addrspacecast", "AddrSpaceCast",
-                                       LLVM_ScalarOrVectorOf<LLVM_AnyPointer>,
-                                       LLVM_ScalarOrVectorOf<LLVM_AnyPointer>> {
+    LLVM_ScalarOrVectorOf<LLVM_AnyPointer>,
+    LLVM_ScalarOrVectorOf<LLVM_AnyPointer>,
+    [DeclareOpInterfaceMethods<PromotableOpInterface>]> {
   let hasFolder = 1;
 }
 def LLVM_IntToPtrOp : LLVM_CastOp<"inttoptr", "IntToPtr",
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index 470fdfef4f1ee..3187ce1615ff9 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -252,6 +252,25 @@ def ROCDL_RawBufferStoreOp :
   let hasCustomAssemblyFormat = 1;
 }
 
+def ROCDL_RawBufferAtomicCmpSwap :
+  ROCDL_Op<"raw.buffer.atomic.cmpswap", [AllTypesMatch<["res", "src", "cmp"]>]>,
+  Results<(outs LLVM_Type:$res)>,
+  Arguments<(ins LLVM_Type:$src,
+                 LLVM_Type:$cmp,
+                 LLVM_Type:$rsrc,
+                 I32:$offset,
+                 I32:$soffset,
+                 I32:$aux)>{
+  string llvmBuilder = [{
+      $res = createIntrinsicCall(builder,
+          llvm::Intrinsic::amdgcn_raw_buffer_atomic_cmpswap, {$src, $cmp, $rsrc,
+            $offset, $soffset, $aux}, {$_resultType});
+  }];
+  let assemblyFormat = [{
+    attr-dict `(` operands `)` `:` type($res) `,` type($rsrc)
+  }];
+}
+
 //===---------------------------------------------------------------------===//
 // MI-100 and MI-200 buffer atomic floating point add intrinsic
 
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/DialectExtension.h b/mlir/include/mlir/Dialect/Linalg/TransformOps/DialectExtension.h
new file mode 100644
index 0000000000000..b89fbc527d564
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/DialectExtension.h
@@ -0,0 +1,15 @@
+//===- DialectExtension.h - Linalg transform dialect extension --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+namespace mlir {
+class DialectRegistry;
+
+namespace linalg {
+void registerTransformDialectExtension(DialectRegistry &registry);
+} // namespace linalg
+} // namespace mlir
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h
new file mode 100644
index 0000000000000..d6bbcf88b79f4
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h
@@ -0,0 +1,48 @@
+//===- LinalgMatchOps.h - Linalg transform matcher ops ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LINALG_TRANSFORMOPS_LINALGMATCHOPS_H
+#define MLIR_DIALECT_LINALG_TRANSFORMOPS_LINALGMATCHOPS_H
+
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Transform/IR/MatchInterfaces.h"
+#include "mlir/Dialect/Transform/IR/TransformAttrs.h"
+
+namespace mlir {
+namespace transform {
+
+namespace detail {
+LogicalResult verifyStructuredOpPredicateOpTrait(Operation *op,
+                                                 Value structuredOpHandle);
+} // namespace detail
+
+template <typename OpTy>
+class StructuredOpPredicateOpTrait
+    : public OpTrait::TraitBase<OpTy, StructuredOpPredicateOpTrait> {
+public:
+  static LogicalResult verifyTrait(Operation *op) {
+    static_assert(
+        OpTy::template hasTrait<SingleOpMatcherOpTrait>(),
+        "StructuredOpPredicateOpTrait requires SingleOpMatcherOpTrait");
+
+    return detail::verifyStructuredOpPredicateOpTrait(
+        op, cast<OpTy>(op).getOperandHandle());
+  }
+};
+
+} // namespace transform
+} // namespace mlir
+
+//===----------------------------------------------------------------------===//
+// Linalg Matcher Operations
+//===----------------------------------------------------------------------===//
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h.inc"
+
+#endif // MLIR_DIALECT_LINALG_TRANSFORMOPS_LINALGMATCHOPS_H
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h
index 6276c5687808c..775377e5e5bce 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h
@@ -11,7 +11,6 @@
 
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
-#include "mlir/Dialect/Transform/IR/MatchInterfaces.h"
 #include "mlir/Dialect/Transform/IR/TransformAttrs.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformInterfaces.h"
@@ -56,30 +55,7 @@ DiagnosedSilenceableFailure tileToForallOpImpl(
     ArrayRef<OpFoldResult> mixedTileSizes, std::optional<ArrayAttr> mapping,
     SmallVector<Operation *> &tileOps, SmallVector<Operation *> &tiledOps);
 
-namespace detail {
-LogicalResult verifyStructuredOpPredicateOpTrait(Operation *op,
-                                                 Value structuredOpHandle);
-} // namespace detail
-
-template <typename OpTy>
-class StructuredOpPredicateOpTrait
-    : public OpTrait::TraitBase<OpTy, StructuredOpPredicateOpTrait> {
-public:
-  static LogicalResult verifyTrait(Operation *op) {
-    static_assert(
-        OpTy::template hasTrait<SingleOpMatcherOpTrait>(),
-        "StructuredOpPredicateOpTrait requires SingleOpMatcherOpTrait");
-
-    return detail::verifyStructuredOpPredicateOpTrait(
-        op, cast<OpTy>(op).getOperandHandle());
-  }
-};
-
 } // namespace transform
-
-namespace linalg {
-void registerTransformDialectExtension(DialectRegistry &registry);
-} // namespace linalg
 } // namespace mlir
 
 //===----------------------------------------------------------------------===//
@@ -91,7 +67,4 @@ void registerTransformDialectExtension(DialectRegistry &registry);
 #define GET_OP_CLASSES
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h.inc"
 
-#define GET_OP_CLASSES
-#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h.inc"
-
 #endif // MLIR_DIALECT_LINALG_TRANSFORMOPS_LINALGTRANSFORMOPS_H
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 73e830dfc86cb..4a3a86ff6a326 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -204,8 +204,8 @@ struct LinalgPromotionOptions {
   /// If ith element of `useFullTiles` is true the full view should be used
   /// for the promoted buffer of the ith operand in `operandsToPromote`.
   /// Otherwise the partial view will be used. The decision is defaulted to
-  /// `useFullTileBuffersDefault` when `useFullTileBuffers` is None and for
-  /// operands missing from `useFullTileBuffers`.
+  /// `useFullTileBuffersDefault` when `useFullTileBuffers` is std::nullopt and
+  /// for operands missing from `useFullTileBuffers`.
   std::optional<llvm::SmallBitVector> useFullTileBuffers;
   LinalgPromotionOptions &setUseFullTileBuffers(ArrayRef<bool> useFullTiles) {
     unsigned size = useFullTiles.size();
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h b/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
index b599c3c56ca78..41f130e074de1 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRef.h
@@ -16,6 +16,7 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/CopyOpInterface.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "mlir/Interfaces/Mem2RegInterfaces.h"
 #include "mlir/Interfaces/ShapedOpInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index 265e400045d8b..1ea0ef0057b7d 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -15,6 +15,7 @@ include "mlir/Interfaces/CastInterfaces.td"
 include "mlir/Interfaces/ControlFlowInterfaces.td"
 include "mlir/Interfaces/CopyOpInterface.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/Mem2RegInterfaces.td"
 include "mlir/Interfaces/ShapedOpInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/Interfaces/ViewLikeInterface.td"
@@ -94,7 +95,18 @@ class AllocLikeOp<string mnemonic,
               static_cast<int32_t>(symbolOperands.size())}));
       if (alignment)
         $_state.addAttribute(getAlignmentAttrStrName(), alignment);
-    }]>];
+    }]>,
+    OpBuilder<(ins "ArrayRef<OpFoldResult>":$sizes, "Type":$elementType,
+                   CArg<"Attribute", "{}">:$memorySpace), [{
+      SmallVector<int64_t> staticShape;
+      SmallVector<Value> dynamicSizes;
+      dispatchIndexOpFoldResults(sizes, dynamicSizes, staticShape);
+      MemRefLayoutAttrInterface layout;
+      MemRefType memrefType = MemRefType::get(staticShape, elementType, layout,
+                                              memorySpace);
+      return build($_builder, $_state, memrefType, dynamicSizes);
+    }]>
+  ];
 
   let extraClassDeclaration = [{
     static StringRef getAlignmentAttrStrName() { return "alignment"; }
@@ -298,7 +310,8 @@ def MemRef_ReallocOp : MemRef_Op<"realloc"> {
 //===----------------------------------------------------------------------===//
 
 def MemRef_AllocaOp : AllocLikeOp<"alloca", AutomaticAllocationScopeResource,[
-    DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>]> {
+    DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>,
+    DeclareOpInterfaceMethods<PromotableAllocationOpInterface>]> {
   let summary = "stack memory allocation operation";
   let description = [{
     The `alloca` operation allocates memory on the stack, to be automatically
@@ -1148,7 +1161,8 @@ def LoadOp : MemRef_Op<"load",
      [TypesMatchWith<"result type matches element type of 'memref'",
                      "memref", "result",
                      "$_self.cast<MemRefType>().getElementType()">,
-                     MemRefsNormalizable]> {
+      MemRefsNormalizable,
+      DeclareOpInterfaceMethods<PromotableMemOpInterface>]> {
   let summary = "load operation";
   let description = [{
     The `load` op reads an element from a memref specified by an index list. The
@@ -1737,7 +1751,8 @@ def MemRef_StoreOp : MemRef_Op<"store",
      [TypesMatchWith<"type of 'value' matches element type of 'memref'",
                      "memref", "value",
                      "$_self.cast<MemRefType>().getElementType()">,
-                     MemRefsNormalizable]> {
+      MemRefsNormalizable,
+      DeclareOpInterfaceMethods<PromotableMemOpInterface>]> {
   let summary = "store operation";
   let description = [{
     Store a value to a memref location given by indices. The value stored should
diff --git a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td
index a0b5a6858f04e..1cefe5640833b 100644
--- a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td
@@ -93,4 +93,45 @@ def MemRefExtractAddressComputationsOp :
         ::mlir::transform::TransformState &state);
   }];
 }
+
+def MemRefMakeLoopIndependentOp
+    : Op<Transform_Dialect, "memref.make_loop_independent",
+         [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
+          TransformOpInterface, TransformEachOpTrait]> {
+  let description = [{
+    Rewrite the targeted ops such that their index-typed operands no longer
+    depend on any loop induction variable of the `num_loop` enclosing `scf.for`
+    loops. I.e., compute an upper bound that is independent of any such loop IV
+    for every tensor dimension. The transformed op could then be hoisted from
+    the `num_loop` enclosing loops. To preserve the original semantics, place a
+    `memref.subview` inside the loop.
+
+    Currently supported operations are:
+    - memref.alloca: Replaced with a new memref.alloca with upper bound sizes,
+      followed by a memref.subview.
+
+    #### Return modes
+
+    This operation fails if at least one induction variable could not be
+    eliminated. In case the targeted op is already independent of induction
+    variables, this transform succeeds and returns the unmodified target op.
+
+    Otherwise, the returned handle points to a subset of the produced ops:
+    - memref.alloca: The returned handle points to the memref.subview op.
+
+    This transform op consumes the target handle and produces a result handle.
+  }];
+
+  let arguments = (ins PDL_Operation:$target, I64Attr:$num_loops);
+  let results = (outs PDL_Operation:$transformed);
+  let assemblyFormat = "$target attr-dict";
+
+  let extraClassDeclaration = [{
+    ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::Operation *target,
+        ::mlir::transform::ApplyToEachResultList &results,
+        ::mlir::transform::TransformState &state);
+  }];
+}
+
 #endif // MEMREF_TRANSFORM_OPS
diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h b/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
index 8d679ccd1704b..1fe13423fa5ce 100644
--- a/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
@@ -17,8 +17,11 @@
 #include "mlir/Support/LogicalResult.h"
 
 namespace mlir {
+class OpBuilder;
 class RewritePatternSet;
 class RewriterBase;
+class Value;
+class ValueRange;
 
 namespace arith {
 class WideIntEmulationConverter;
@@ -26,6 +29,8 @@ class WideIntEmulationConverter;
 
 namespace memref {
 class AllocOp;
+class AllocaOp;
+
 //===----------------------------------------------------------------------===//
 // Patterns
 //===----------------------------------------------------------------------===//
@@ -121,6 +126,60 @@ FailureOr<memref::AllocOp> multiBuffer(memref::AllocOp allocOp,
 /// ```
 void populateExtractAddressComputationsPatterns(RewritePatternSet &patterns);
 
+/// Build a new memref::AllocaOp whose dynamic sizes are independent of all
+/// given independencies. If the op is already independent of all
+/// independencies, the same AllocaOp result is returned.
+///
+/// Failure indicates the no suitable upper bound for the dynamic sizes could be
+/// found.
+FailureOr<Value> buildIndependentOp(OpBuilder &b, AllocaOp allocaOp,
+                                    ValueRange independencies);
+
+/// Build a new memref::AllocaOp whose dynamic sizes are independent of all
+/// given independencies. If the op is already independent of all
+/// independencies, the same AllocaOp result is returned.
+///
+/// The original AllocaOp is replaced with the new one, wrapped in a SubviewOp.
+/// The result type of the replacement is different from the original allocation
+/// type: it has the same shape, but a different layout map. This function
+/// updates all users that do not have a memref result or memref region block
+/// argument, and some frequently used memref dialect ops (such as
+/// memref.subview). It does not update other uses such as the init_arg of an
+/// scf.for op. Such uses are wrapped in unrealized_conversion_cast.
+///
+/// Failure indicates the no suitable upper bound for the dynamic sizes could be
+/// found.
+///
+/// Example (make independent of %iv):
+/// ```
+/// scf.for %iv = %c0 to %sz step %c1 {
+///   %0 = memref.alloca(%iv) : memref<?xf32>
+///   %1 = memref.subview %0[0][5][1] : ...
+///   linalg.generic outs(%1 : ...) ...
+///   %2 = scf.for ... iter_arg(%arg0 = %0) ...
+///   ...
+/// }
+/// ```
+///
+/// The above IR is rewritten to:
+///
+/// ```
+/// scf.for %iv = %c0 to %sz step %c1 {
+///   %0 = memref.alloca(%sz - 1) : memref<?xf32>
+///   %0_subview = memref.subview %0[0][%iv][1]
+///       : memref<?xf32> to memref<?xf32, #map>
+///   %1 = memref.subview %0_subview[0][5][1] : ...
+///   linalg.generic outs(%1 : ...) ...
+///   %cast = unrealized_conversion_cast %0_subview
+///       : memref<?xf32, #map> to memref<?xf32>
+///   %2 = scf.for ... iter_arg(%arg0 = %cast) ...
+///  ...
+/// }
+/// ```
+FailureOr<Value> replaceWithIndependentOp(RewriterBase &rewriter,
+                                          memref::AllocaOp allocaOp,
+                                          ValueRange independencies);
+
 } // namespace memref
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index c027d0ce199a5..3c1c36778c6f6 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -293,8 +293,8 @@ def OpenACC_GetDevicePtrOp : OpenACC_DataEntryOp<"getdeviceptr",
 class OpenACC_DataExitOp<string mnemonic, string clause, list<Trait> traits = []> :
     OpenACC_Op<mnemonic, !listconcat(traits,
         [AttrSizedOperandSegments])> {
-  let arguments = (ins Optional<OpenACC_PointerLikeTypeInterface>:$varPtr,
-                       OpenACC_PointerLikeTypeInterface:$accPtr,
+  let arguments = (ins OpenACC_PointerLikeTypeInterface:$accPtr,
+                       Optional<OpenACC_PointerLikeTypeInterface>:$varPtr,
                        Variadic<OpenACC_DataBoundsType>:$bounds,
                        DefaultValuedAttr<OpenACC_DataClauseEnum,clause>:$dataClause,
                        DefaultValuedAttr<BoolAttr, "true">:$structured,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 0cfdc3707de49..b6a292e212963 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -71,7 +71,7 @@ def OffloadModuleInterface : OpInterface<"OffloadModuleInterface"> {
       InterfaceMethod<
       /*description=*/[{
         Get the IsDeviceAttr attribute on the current module if it exists and return
-        its value, if it doesn't exit it returns false by default.
+        its value, if it doesn't exist it returns false by default.
       }],
       /*retTy=*/"bool",
       /*methodName=*/"getIsDevice",
@@ -138,6 +138,34 @@ def OffloadModuleInterface : OpInterface<"OffloadModuleInterface"> {
                                              targetCPU.str(),
                                              targetFeatures.str()));
       }]>,
+      InterfaceMethod<
+      /*description=*/[{
+        Set a StringAttr on the current module containing the host IR file path. This 
+        file path is used in two-phase compilation during the device phase to generate
+        device side LLVM IR when lowering MLIR. 
+      }],
+      /*retTy=*/"void",
+      /*methodName=*/"setHostIRFilePath", 
+      (ins "std::string":$hostIRFilePath), [{}], [{
+        $_op->setAttr(
+          mlir::StringAttr::get($_op->getContext(), llvm::Twine{"omp.host_ir_filepath"}),
+            mlir::StringAttr::get($_op->getContext(), hostIRFilePath));
+       }]>,
+      InterfaceMethod<
+      /*description=*/[{
+        Find the host-ir file path StringAttr from the current module if it exists and 
+        return its contained value, if it doesn't exist it returns an empty string. This 
+        file path is used in two-phase compilation during the device phase to generate
+        device side LLVM IR when lowering MLIR. 
+      }],
+      /*retTy=*/"llvm::StringRef",
+      /*methodName=*/"getHostIRFilePath", 
+      (ins), [{}], [{
+        if (Attribute filepath = $_op->getAttr("omp.host_ir_filepath"))
+          if (filepath.isa<mlir::StringAttr>())
+            return filepath.dyn_cast<mlir::StringAttr>().getValue();
+        return {};
+      }]>
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
index 6aa4f341bba96..481c2e629198c 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
@@ -85,16 +85,25 @@ using StaticSize = int64_t;
 namespace mlir {
 namespace sparse_tensor {
 
+// NOTE: `Value::getType` doesn't check for null before trying to
+// dereference things.  Therefore we check, because an assertion-failure
+// is easier to debug than a segfault.  Presumably other `T::getType`
+// methods are similarly susceptible.
+
 /// Convenience method to abbreviate casting `getType()`.
 template <typename T>
-inline RankedTensorType getRankedTensorType(T t) {
-  return t.getType().template cast<RankedTensorType>();
+inline RankedTensorType getRankedTensorType(T &&t) {
+  assert(static_cast<bool>(std::forward<T>(t)) &&
+         "getRankedTensorType got null argument");
+  return std::forward<T>(t).getType().template cast<RankedTensorType>();
 }
 
 /// Convenience method to abbreviate casting `getType()`.
 template <typename T>
-inline MemRefType getMemRefType(T t) {
-  return t.getType().template cast<MemRefType>();
+inline MemRefType getMemRefType(T &&t) {
+  assert(static_cast<bool>(std::forward<T>(t)) &&
+         "getMemRefType got null argument");
+  return std::forward<T>(t).getType().template cast<MemRefType>();
 }
 
 /// Convenience method to get a sparse encoding attribute from a type.
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
index eea58f91b583c..f29ea600e3347 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
@@ -124,9 +124,10 @@ def SparseTensor_PackOp : SparseTensor_Op<"pack", [Pure]>,
 }
 
 def SparseTensor_UnpackOp : SparseTensor_Op<"unpack">,
-    Arguments<(ins AnySparseTensor:$tensor)>,
-    Results<(outs 1DTensorOf<[AnyType]>:$values,
-                  2DTensorOf<[AnySignlessIntegerOrIndex]>:$coordinates,
+    Arguments<(ins AnySparseTensor:$tensor,
+                   OptionalAttr<IndexAttr>:$batched_lvls)>,
+    Results<(outs TensorOf<[AnyType]>:$values,
+                  TensorOf<[AnySignlessIntegerOrIndex]>:$coordinates,
                   AnySignlessIntegerOrIndex:$nse)> {
   let summary = "Returns the (values, coordinates) pair unpacked from the input tensor";
 
@@ -159,11 +160,44 @@ def SparseTensor_UnpackOp : SparseTensor_Op<"unpack">,
     // %coordinates = arith.constant dense<[[0,0], [1,2], [1,3]]> : tensor<3x2xindex>
     // %nse = 3
     ```
+
+    If `batched_lvls` is provided, the operation unpacks each batch of the tensors
+    separately. The returned `nse` is the maximum nse of all batches. For a batch with
+    a smaller nse, trailing zeros are appended in the result.
+    Example:
+
+    ```mlir
+    // input BCOO format |1.1, 2.2, 3.3, 0.0|
+    //      of 2x4 matrix |0.0, 1.2, 2.3, 0.0|
+    %values, %coordinates, %nse = sparse_tensor.unpack %st batched_lvls=1
+        : tensor<2x3xf64>, tensor<2x3x1xindex> to tensor<2x4xf64, #BCOO>
+    // %values      = arith.constant dense<[[ 1.1,   2.2,   3.3 ],
+    //                                      [ 1.2,   2.3,   0.0 ]]> : tensor<2x3xf64>
+    // %coordinates = arith.constant dense<[[ [0],   [1],   [2] ],
+    //                                      [ [1],   [2],   [0] ]> : tensor<2x3x1xindex>
+    ```
+  }];
+
+  let extraClassDeclaration = [{
+    /// Returns the number of leading levels that are batched.
+    unsigned getNumBatchedLvls();
   }];
 
+  let builders = [
+    OpBuilder<(ins "Type":$values, "Type":$coordinates, "Type":$nse, "Value": $tensor),
+    [{
+      build($_builder, $_state, values, coordinates, nse, tensor, nullptr);
+    }]>,
+    OpBuilder<(ins "TypeRange":$resultTypes, "Value": $tensor),
+    [{
+      build($_builder, $_state, resultTypes, tensor, nullptr);
+    }]>
+  ];
+
+
   let assemblyFormat =
-    "$tensor attr-dict `:` type($tensor)"
-    "`to` type($values) `,` type($coordinates) `,` type($nse)";
+    "$tensor (`batched_lvls` `=` $batched_lvls^)? attr-dict `:`"
+    "type($tensor) `to` type($values) `,` type($coordinates) `,` type($nse)";
 
   let hasVerifier = 1;
 }
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
index bdcd3632b5d38..4c4f1f25edfd5 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
@@ -51,6 +51,7 @@ class SparseTensorType {
       : rtp(rtp), enc(getSparseTensorEncoding(rtp)),
         lvlRank(enc ? enc.getLvlRank() : getDimRank()),
         dim2lvl(enc.hasIdDimOrdering() ? AffineMap() : enc.getDimOrdering()) {
+    assert(rtp && "got null RankedTensorType");
     assert((!isIdentity() || getDimRank() == lvlRank) && "Rank mismatch");
   }
 
diff --git a/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt b/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt
index 9f57627c321fb..cb1e9d01821a2 100644
--- a/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(IR)
 add_subdirectory(Transforms)
+add_subdirectory(TransformOps)
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h b/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
index b6e6024719728..cbc93849c0a85 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
+++ b/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
@@ -142,6 +142,9 @@ FailureOr<Value> getOrCreateDestination(OpBuilder &b, Location loc,
 LogicalResult getOrCreateDestinations(OpBuilder &b, Location loc, Operation *op,
                                       SmallVector<Value> &result);
 
+/// Tests if types are the same when ignoring encoding on ranked tensors.
+bool isSameTypeWithoutEncoding(Type tp1, Type tp2);
+
 /// Function to control the folding of constant and extract slice.
 using ControlConstantExtractSliceFusionFn = std::function<bool(ExtractSliceOp)>;
 
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index 0c589ce920fe0..e0e97e7efdcc5 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -53,6 +53,35 @@ class Tensor_OpWithOffsetSizesAndStrides<string mnemonic,
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// BitcastOp
+//===----------------------------------------------------------------------===//
+
+def Tensor_BitcastOp : Tensor_Op<"bitcast", [
+    DeclareOpInterfaceMethods<CastOpInterface>,
+    Pure
+  ]> {
+  let summary = "tensor bitcast operation";
+  let description = [{
+    Bitcast a tensor from one type to another type of equivalent element width.
+    If both are ranked, then the rank should be the same and static dimensions
+    should match. 
+
+    Example:
+
+    ```mlir
+    // Bitcast from unsigned to signed or signless integer.
+    %2 = tensor.bitcast %1 : tensor<4xui32> to tensor<4xi32>
+    ```
+  }];
+
+  let arguments = (ins AnyTensor:$source);
+  let results = (outs AnyTensor:$dest);
+  let assemblyFormat = "$source attr-dict `:` type($source) `to` type($dest)";
+
+  let hasCanonicalizer = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // CastOp
 //===----------------------------------------------------------------------===//
@@ -484,9 +513,7 @@ def Tensor_FromElementsOp : Tensor_Op<"from_elements", [
 
   let assemblyFormat = "$elements attr-dict `:` type($result)";
 
-  let skipDefaultBuilders = 1;
   let builders = [
-    OpBuilder<(ins "Type":$resultType, "ValueRange":$elements)>,
     // Special case builder for when `elements` has size >=1.
     OpBuilder<(ins "ValueRange":$elements)>
   ];
diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/CMakeLists.txt b/mlir/include/mlir/Dialect/Tensor/TransformOps/CMakeLists.txt
new file mode 100644
index 0000000000000..bb9f703602dcf
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/CMakeLists.txt
@@ -0,0 +1,6 @@
+set(LLVM_TARGET_DEFINITIONS TensorTransformOps.td)
+mlir_tablegen(TensorTransformOps.h.inc -gen-op-decls)
+mlir_tablegen(TensorTransformOps.cpp.inc -gen-op-defs)
+add_public_tablegen_target(MLIRTensorTransformOpsIncGen)
+
+add_mlir_doc(TensorTransformOps TensorTransformOps Dialects/ -gen-op-doc)
diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h
index d1b14d206cd77..c735700b77382 100644
--- a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h
+++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h
@@ -11,10 +11,13 @@
 
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
+#include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 
 namespace mlir {
+class DialectRegistry;
+
 namespace tensor {
 
 /// A specialized TrackingListener for transform ops that operate on tensor IR.
@@ -29,7 +32,12 @@ class TrackingListener : public transform::TrackingListener {
                                ValueRange newValues) const override;
 };
 
+void registerTransformDialectExtension(DialectRegistry &registry);
+
 } // namespace tensor
 } // namespace mlir
 
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h.inc"
+
 #endif // MLIR_DIALECT_TENSOR_TRANSFORMOPS_TENSORTRANSFORMOPS_H
diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
new file mode 100644
index 0000000000000..42be8821addc3
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
@@ -0,0 +1,64 @@
+//===- TensorTransformOps.td - Tensor transformation ops ---*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TENSOR_TRANSFORM_OPS
+#define TENSOR_TRANSFORM_OPS
+
+include "mlir/Dialect/PDL/IR/PDLTypes.td"
+include "mlir/Dialect/Transform/IR/TransformDialect.td"
+include "mlir/Dialect/Transform/IR/TransformInterfaces.td"
+include "mlir/Dialect/Transform/IR/TransformTypes.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/IR/OpBase.td"
+
+def Transform_TensorPadOp : Transform_ConcreteOpType<"tensor.pad">;
+
+def MakeLoopIndependentOp
+    : Op<Transform_Dialect, "tensor.make_loop_independent",
+         [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
+          TransformOpInterface, TransformEachOpTrait]> {
+  let description = [{
+    Rewrite the targeted ops such that their index-typed operands no longer
+    depend on any loop induction variable of the `num_loop` enclosing `scf.for`
+    loops. I.e., compute an upper bound that is independent of any such loop IV
+    for every tensor dimension. The transformed op could then be hoisted from
+    the `num_loop` enclosing loops. To preserve the original semantics, place a
+    `tensor.extract_slice` inside the loop.
+
+    Currently supported operations are:
+    - tensor.empty: Replaced with a new tensor.empty with upper bound sizes,
+      followed by a tensor.extract_slice.
+    - tensor.pad: Replaced by an upper bound padding, followed by a
+      tensor.extract_slice.
+
+    #### Return modes
+
+    This operation fails if at least one induction variable could not be
+    eliminated. In case the targeted op is already independent of induction
+    variables, this transform succeeds and returns the unmodified target op.
+
+    Otherwise, the returned handle points to a subset of the produced ops:
+    - tensor.empty: The returned handle points to the tensor.extract_slice op.
+    - tensor.pad: The returned handle points to the tensor.extract_slice op.
+
+    This transform op consumes the target handle and produces a result handle.
+  }];
+
+  let arguments = (ins PDL_Operation:$target, I64Attr:$num_loops);
+  let results = (outs PDL_Operation:$transformed);
+  let assemblyFormat = "$target attr-dict";
+
+  let extraClassDeclaration = [{
+    ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::Operation *target,
+        ::mlir::transform::ApplyToEachResultList &results,
+        ::mlir::transform::TransformState &state);
+  }];
+}
+
+#endif // TENSOR_TRANSFORM_OPS
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
index a3b5abf08fd7a..9922dc8358acb 100644
--- a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
@@ -61,6 +61,45 @@ void populateFoldTensorEmptyPatterns(RewritePatternSet &patterns);
 /// respectively.
 void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns);
 
+//===----------------------------------------------------------------------===//
+// Transform helpers
+//===----------------------------------------------------------------------===//
+
+/// Build a new tensor::PadOp with low/high padding that is independent of all
+/// given independencies. If the op is already independent of all
+/// independencies, the same PadOp result is returned.
+///
+/// Failure indicates the no suitable upper bound for low/high padding could be
+/// found.
+///
+/// Example:
+/// scf.for %iv = %lb to %ub step %step {
+///   %high = affine.apply affine_map<(d0)[s0] -> (s0 - d0)> (%i)[%ub]
+///   %p = tensor.pad %t low[5] high[%high] ...
+///   ...
+/// }
+///
+/// The function builds IR such as:
+/// %high_new = affine.apply affine_map<()[s0, s1] -> (-s0 + s1)> ()[%lb, %ub]
+/// %p_hoistable = tensor.pad %t low[5] high[%high_new]
+/// %dim = tensor.dim %t, %c0
+/// %size = affine.apply affine_map<(d0)[s0, s1] -> (-d0 + s0 + s1 + 5)>
+///     (%iv)[%ub, %dim]
+/// %slice = tensor.extract_slice %p_hoistable [0] [%size] [1]
+///
+/// The slice is returned.
+FailureOr<Value> buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
+                                    ValueRange independencies);
+
+/// Build a new tensor::EmptyOp who's dynamic sizes are independent of all
+/// given independencies. If the op is already independent of all
+/// independencies, the same EmptyOp result is returned.
+///
+/// Failure indicates the no suitable upper bound for the dynamic sizes could be
+/// found.
+FailureOr<Value> buildIndependentOp(OpBuilder &b, tensor::EmptyOp emptyOp,
+                                    ValueRange independencies);
+
 } // namespace tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
index 110a334011021..7e4d67575fdf4 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
@@ -45,6 +45,7 @@ def Tosa_Dialect : Dialect {
   let cppNamespace = "mlir::tosa";
   let hasConstantMaterializer = 1;
   let useDefaultAttributePrinterParser = 1;
+  let usePropertiesForAttributes = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index e36ab18777d14..015be1f8f4885 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1043,6 +1043,8 @@ def Tosa_LogOp : Tosa_Op<"log", [
   let results = (outs
     Tosa_Tensor:$output
   );
+
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
index 3d1ae9c7121b6..b7776110d444f 100644
--- a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
@@ -475,7 +475,7 @@ struct CollapseShapeRankReducingSliceSimplificationInfo {
   /// The shape of the output of the rank-reducing slice.
   RankedTensorType sliceResultType;
   /// The reassociation indices for the new collapse shape op, if required. If
-  /// `None`, the slice should replace the collapse shape op.
+  /// `std::nullopt`, the slice should replace the collapse shape op.
   std::optional<SmallVector<ReassociationIndices>> newReassociationIndices;
 };
 
@@ -522,6 +522,7 @@ getSimplifyCollapseShapeWithRankReducingSliceInfo(
 
 struct PackingMetadata {
   SmallVector<int64_t> insertPositions;
+  SmallVector<int64_t> outerPositions;
   SmallVector<ReassociationIndices> reassociations;
 };
 
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index f4508b9b3ddfd..dec6addf72cc4 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1838,7 +1838,10 @@ def Vector_MaskedStoreOp :
 }
 
 def Vector_GatherOp :
-  Vector_Op<"gather", [DeclareOpInterfaceMethods<MaskableOpInterface>]>,
+  Vector_Op<"gather", [
+    DeclareOpInterfaceMethods<MaskableOpInterface>,
+    DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
+  ]>,
     Arguments<(ins Arg<AnyShaped, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
                VectorOf<[AnyInteger, Index]>:$index_vec,
@@ -2584,7 +2587,7 @@ def Vector_FlatTransposeOp : Vector_Op<"flat_transpose", [Pure,
     into 'vector.flat_transpose' operations to bring the operations closer
     to the hardware ISA.
 
-    The ‘vector.flat_transpose’ op treats the 1-D input `matrix` as
+    The `vector.flat_transpose` op treats the 1-D input `matrix` as
     a 2-D matrix with <rows> rows and <columns> columns, and returns the
     transposed matrix in flattened form in 'res'.
 
@@ -2595,8 +2598,8 @@ def Vector_FlatTransposeOp : Vector_Op<"flat_transpose", [Pure,
     Example:
 
     ```mlir
-    %1 = vector.flat_transpose %0 { rows = 4: i32, columns = 4: i32 }
-       : (vector<16xf32>) -> vector<16xf32>
+    %1 = vector.flat_transpose %0 {columns = 4 : i32, rows = 4 : i32}
+       : vector<16xf32> -> vector<16xf32>
     ```
   }];
   let assemblyFormat = "$matrix attr-dict `:` type($matrix) `->` type($res)";
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index a79bbd0be0975..325860079b3db 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -181,6 +181,22 @@ void populateVectorExtractStridedSliceToExtractInsertChainPatterns(
     std::function<bool(ExtractStridedSliceOp)> controlFn = nullptr,
     PatternBenefit benefit = 1);
 
+/// Populate `patterns` with a pattern to break down 1-D vector.bitcast ops
+/// based on the destination vector shape. Bitcasts from a lower bitwidth
+/// element type to a higher bitwidth one are extracted from the lower bitwidth
+/// based on the native destination vector shape and inserted based on the ratio
+/// of the bitwidths.
+///
+/// This acts as a last resort way to break down vector.bitcast ops to smaller
+/// vector sizes. Because this pattern composes until it is bitcasting to a
+/// single element of the higher bitwidth, the is an optional control function.
+/// If `controlFn` is not nullptr, the pattern will only apply to ops where
+/// `controlFn` returns true, otherwise applies to all bitcast ops.
+void populateBreakDownVectorBitCastOpPatterns(
+    RewritePatternSet &patterns,
+    std::function<bool(BitCastOp)> controlFn = nullptr,
+    PatternBenefit benefit = 1);
+
 /// Populate `patterns` with the following patterns.
 ///
 /// Patterns in populateVectorInsertExtractStridedSliceDecompositionPatterns();
diff --git a/mlir/include/mlir/IR/BuiltinDialectBytecode.td b/mlir/include/mlir/IR/BuiltinDialectBytecode.td
new file mode 100644
index 0000000000000..b59f96c9fa9f7
--- /dev/null
+++ b/mlir/include/mlir/IR/BuiltinDialectBytecode.td
@@ -0,0 +1,566 @@
+//===-- BuiltinBytecode.td - Builtin bytecode defs ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the Builtin bytecode reader/writer definition file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BUILTIN_BYTECODE
+#define BUILTIN_BYTECODE
+
+include "mlir/IR/BytecodeBase.td"
+
+def LocationAttr : AttributeKind;
+def ShapedType: WithType<"ShapedType", Type>;
+
+def Location : CompositeBytecode {
+  dag members = (attr
+    WithGetter<"(LocationAttr)$_attrType", WithType<"LocationAttr", LocationAttr>>:$value
+  );
+  let cBuilder = "Location($_args)";
+}
+
+def String :
+  WithParser <"succeeded($_reader.readString($_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeOwnedString($_getter)",
+  WithGetter <"$_attrType",
+  WithType   <"StringRef">>>>>;
+
+// enum AttributeCode {
+//  ///   ArrayAttr {
+//  ///     elements: Attribute[]
+//  ///   }
+//  ///
+//  kArrayAttr = 0,
+//
+def ArrayAttr : DialectAttribute<(attr
+  Array<Attribute>:$value
+)>;
+
+let cType = "StringAttr" in {
+//  ///   StringAttr {
+//  ///     value: string
+//  ///   }
+//  kStringAttr = 2,
+def StringAttr : DialectAttribute<(attr
+  String:$value
+)> {
+  let printerPredicate = "$_val.getType().isa<NoneType>()";
+}
+
+//  ///   StringAttrWithType {
+//  ///     value: string,
+//  ///     type: Type
+//  ///   }
+//  /// A variant of StringAttr with a type.
+//  kStringAttrWithType = 3,
+def StringAttrWithType : DialectAttribute<(attr
+  String:$value,
+  Type:$type
+)> { let printerPredicate = "!$_val.getType().isa<NoneType>()"; }
+}
+
+//  ///   DictionaryAttr {
+//  ///     attrs: <StringAttr, Attribute>[]
+//  ///   }
+//  kDictionaryAttr = 1,
+def NamedAttribute : CompositeBytecode {
+  dag members = (attr
+    StringAttr:$name,
+    Attribute:$value
+  );
+  let cBuilder = "NamedAttribute($_args)";
+}
+def DictionaryAttr : DialectAttribute<(attr
+  Array<NamedAttribute>:$value
+)>;
+
+//  ///   FlatSymbolRefAttr {
+//  ///     rootReference: StringAttr
+//  ///   }
+//  /// A variant of SymbolRefAttr with no leaf references.
+//  kFlatSymbolRefAttr = 4,
+def FlatSymbolRefAttr: DialectAttribute<(attr
+  StringAttr:$rootReference
+)>;
+
+//  ///   SymbolRefAttr {
+//  ///     rootReference: StringAttr,
+//  ///     leafReferences: FlatSymbolRefAttr[]
+//  ///   }
+//  kSymbolRefAttr = 5,
+def SymbolRefAttr: DialectAttribute<(attr
+  StringAttr:$rootReference,
+  Array<FlatSymbolRefAttr>:$nestedReferences
+)>;
+
+//  ///   TypeAttr {
+//  ///     value: Type
+//  ///   }
+//  kTypeAttr = 6,
+def TypeAttr: DialectAttribute<(attr
+  Type:$value
+)>;
+
+//  ///   UnitAttr {
+//  ///   }
+//  kUnitAttr = 7,
+def UnitAttr: DialectAttribute<(attr)>;
+
+//  ///   IntegerAttr {
+//  ///     type: Type
+//  ///     value: APInt,
+//  ///   }
+//  kIntegerAttr = 8,
+def IntegerAttr: DialectAttribute<(attr
+  Type:$type,
+  KnownWidthAPInt<"type">:$value
+)> {
+  let cBuilder = "get<$_resultType>(context, type, *value)";
+}
+
+//
+//  ///   FloatAttr {
+//  ///     type: FloatType
+//  ///     value: APFloat
+//  ///   }
+//  kFloatAttr = 9,
+defvar FloatType = Type;
+def FloatAttr : DialectAttribute<(attr
+  FloatType:$type,
+  KnownSemanticsAPFloat<"type">:$value
+)> {
+  let cBuilder = "get<$_resultType>(context, type, *value)";
+}
+
+//  ///   CallSiteLoc {
+//  ///    callee: LocationAttr,
+//  ///    caller: LocationAttr
+//  ///   }
+//  kCallSiteLoc = 10,
+def CallSiteLoc : DialectAttribute<(attr
+  LocationAttr:$callee,
+  LocationAttr:$caller
+)>;
+
+//  ///   FileLineColLoc {
+//  ///     filename: StringAttr,
+//  ///     line: varint,
+//  ///     column: varint
+//  ///   }
+//  kFileLineColLoc = 11,
+def FileLineColLoc : DialectAttribute<(attr
+  StringAttr:$filename,
+  VarInt:$line,
+  VarInt:$column
+)>;
+
+let cType = "FusedLoc",
+    cBuilder = "cast<FusedLoc>(get<FusedLoc>(context, $_args))" in {
+//  ///   FusedLoc {
+//  ///     locations: Location[]
+//  ///   }
+//  kFusedLoc = 12,
+def FusedLoc : DialectAttribute<(attr
+  Array<Location>:$locations
+)> {
+  let printerPredicate = "!$_val.getMetadata()";
+}
+
+//  ///   FusedLocWithMetadata {
+//  ///     locations: LocationAttr[],
+//  ///     metadata: Attribute
+//  ///   }
+//  /// A variant of FusedLoc with metadata.
+//  kFusedLocWithMetadata = 13,
+def FusedLocWithMetadata : DialectAttribute<(attr
+  Array<Location>:$locations,
+  Attribute:$metadata
+)> {
+  let printerPredicate = "$_val.getMetadata()";
+}
+}
+
+//  ///   NameLoc {
+//  ///     name: StringAttr,
+//  ///     childLoc: LocationAttr
+//  ///   }
+//  kNameLoc = 14,
+def NameLoc : DialectAttribute<(attr
+  StringAttr:$name,
+  LocationAttr:$childLoc
+)>;
+
+//  ///   UnknownLoc {
+//  ///   }
+//  kUnknownLoc = 15,
+def UnknownLoc : DialectAttribute<(attr)>;
+
+//  ///   DenseResourceElementsAttr {
+//  ///     type: ShapedType,
+//  ///     handle: ResourceHandle
+//  ///   }
+//  kDenseResourceElementsAttr = 16,
+def DenseResourceElementsAttr : DialectAttribute<(attr
+  ShapedType:$type,
+  ResourceHandle<"DenseResourceElementsHandle">:$rawHandle
+)> {
+  // Note: order of serialization does not match order of builder.
+  let cBuilder = "get<$_resultType>(context, type, *rawHandle)";
+}
+
+let cType = "RankedTensorType" in {
+//   ///   RankedTensorType {
+//   ///     shape: svarint[],
+//   ///     elementType: Type,
+//   ///   }
+//   ///
+//   kRankedTensorType = 13,
+def RankedTensorType : DialectType<(type
+  Array<SignedVarInt>:$shape,
+  Type:$elementType
+)> {
+  let printerPredicate = "!$_val.getEncoding()";
+}
+
+//   ///   RankedTensorTypeWithEncoding {
+//   ///     encoding: Attribute,
+//   ///     shape: svarint[],
+//   ///     elementType: Type
+//   ///   }
+//   /// Variant of RankedTensorType with an encoding.
+//   kRankedTensorTypeWithEncoding = 14,
+def RankedTensorTypeWithEncoding : DialectType<(type
+  Attribute:$encoding,
+  Array<SignedVarInt>:$shape,
+  Type:$elementType
+)> {
+  let printerPredicate = "$_val.getEncoding()";
+  // Note: order of serialization does not match order of builder.
+  let cBuilder = "get<$_resultType>(context, shape, elementType, encoding)";
+}
+}
+
+//  ///   DenseArrayAttr {
+//  ///     elementType: Type,
+//  ///     size: varint
+//  ///     data: blob
+//  ///   }
+//  kDenseArrayAttr = 17,
+def DenseArrayAttr : DialectAttribute<(attr
+  Type:$elementType,
+  VarInt:$size,
+  Blob:$rawData
+)>;
+
+//  ///   DenseIntOrFPElementsAttr {
+//  ///     type: ShapedType,
+//  ///     data: blob
+//  ///   }
+//  kDenseIntOrFPElementsAttr = 18,
+def DenseElementsAttr : WithType<"DenseIntElementsAttr", Attribute>;
+def DenseIntOrFPElementsAttr : DialectAttribute<(attr
+  ShapedType:$type,
+  Blob:$rawData
+)> {
+  let cBuilder = "cast<$_resultType>($_resultType::getFromRawBuffer($_args))";
+}
+
+//  ///   DenseStringElementsAttr {
+//  ///     type: ShapedType,
+//  ///     isSplat: varint,
+//  ///     data: string[]
+//  ///   }
+//  kDenseStringElementsAttr = 19,
+def DenseStringElementsAttr : DialectAttribute<(attr
+  ShapedType:$type,
+  WithGetter<"$_attrType.isSplat()", VarInt>:$_isSplat,
+  WithBuilder<"$_args",
+    WithType<"SmallVector<StringRef>",
+    WithParser <"succeeded(readPotentiallySplatString($_reader, type, _isSplat, $_var))",
+    WithPrinter<"writePotentiallySplatString($_writer, $_name)">>>>:$rawStringData
+)>;
+
+//  ///   SparseElementsAttr {
+//  ///     type: ShapedType,
+//  ///     indices: DenseIntElementsAttr,
+//  ///     values: DenseElementsAttr
+//  ///   }
+//  kSparseElementsAttr = 20,
+def DenseIntElementsAttr : WithType<"DenseIntElementsAttr", Attribute>;
+def SparseElementsAttr : DialectAttribute<(attr
+  ShapedType:$type,
+  DenseIntElementsAttr:$indices,
+  DenseElementsAttr:$values
+)>;
+
+// Types
+// -----
+
+// enum TypeCode {
+//   ///   IntegerType {
+//   ///     widthAndSignedness: varint // (width << 2) | (signedness)
+//   ///   }
+//   ///
+//   kIntegerType = 0,
+def IntegerType : DialectType<(type
+  // Yes not pretty,
+  WithParser<"succeeded($_reader.readVarInt($_var))",
+    WithBuilder<"$_args",
+    WithPrinter<"$_writer.writeVarInt($_name.getWidth() << 2 | $_name.getSignedness())",
+    WithType   <"uint64_t">>>>:$_widthAndSignedness,
+  // Split up parsed varint for create method.
+  LocalVar<"uint64_t", "_widthAndSignedness >> 2">:$width,
+  LocalVar<"IntegerType::SignednessSemantics",
+    "static_cast<IntegerType::SignednessSemantics>(_widthAndSignedness & 0x3)">:$signedness
+)>;
+
+// 
+//   ///   IndexType {
+//   ///   }
+//   ///
+//   kIndexType = 1,
+def IndexType : DialectType<(type)>;
+
+//   ///   FunctionType {
+//   ///     inputs: Type[],
+//   ///     results: Type[]
+//   ///   }
+//   ///
+//   kFunctionType = 2,
+def FunctionType : DialectType<(type
+  Array<Type>:$inputs,
+  Array<Type>:$results
+)>;
+
+//   ///   BFloat16Type {
+//   ///   }
+//   ///
+//   kBFloat16Type = 3,
+def BFloat16Type : DialectType<(type)>;
+
+//   ///   Float16Type {
+//   ///   }
+//   ///
+//   kFloat16Type = 4,
+def Float16Type : DialectType<(type)>;
+
+//   ///   Float32Type {
+//   ///   }
+//   ///
+//   kFloat32Type = 5,
+def Float32Type : DialectType<(type)>;
+
+//   ///   Float64Type {
+//   ///   }
+//   ///
+//   kFloat64Type = 6,
+def Float64Type : DialectType<(type)>;
+
+//   ///   Float80Type {
+//   ///   }
+//   ///
+//   kFloat80Type = 7,
+def Float80Type : DialectType<(type)>;
+
+//   ///   Float128Type {
+//   ///   }
+//   ///
+//   kFloat128Type = 8,
+def Float128Type : DialectType<(type)>;
+
+//   ///   ComplexType {
+//   ///     elementType: Type
+//   ///   }
+//   ///
+//   kComplexType = 9,
+def ComplexType : DialectType<(type
+  Type:$elementType
+)>;
+
+def MemRefLayout: WithType<"MemRefLayoutAttrInterface", Attribute>;
+
+let cType = "MemRefType" in {
+//   ///   MemRefType {
+//   ///     shape: svarint[],
+//   ///     elementType: Type,
+//   ///     layout: Attribute
+//   ///   }
+//   ///
+//   kMemRefType = 10,
+def MemRefType : DialectType<(type
+  Array<SignedVarInt>:$shape,
+  Type:$elementType,
+  MemRefLayout:$layout
+)> {
+  let printerPredicate = "!$_val.getMemorySpace()";
+}
+
+//   ///   MemRefTypeWithMemSpace {
+//   ///     memorySpace: Attribute,
+//   ///     shape: svarint[],
+//   ///     elementType: Type,
+//   ///     layout: Attribute
+//   ///   }
+//   /// Variant of MemRefType with non-default memory space.
+//   kMemRefTypeWithMemSpace = 11,
+def MemRefTypeWithMemSpace : DialectType<(type
+  Attribute:$memorySpace,
+  Array<SignedVarInt>:$shape,
+  Type:$elementType,
+  MemRefLayout:$layout
+)> {
+  let printerPredicate = "!!$_val.getMemorySpace()";
+  // Note: order of serialization does not match order of builder.
+  let cBuilder = "get<$_resultType>(context, shape, elementType, layout, memorySpace)";
+}
+}
+
+//   ///   NoneType {
+//   ///   }
+//   ///
+//   kNoneType = 12,
+def NoneType : DialectType<(type)>;
+
+//   ///   TupleType {
+//   ///     elementTypes: Type[]
+//   ///   }
+//   kTupleType = 15,
+def TupleType : DialectType<(type
+  Array<Type>:$types
+)>;
+
+let cType = "UnrankedMemRefType" in {
+//   ///   UnrankedMemRefType {
+//   ///     elementType: Type
+//   ///   }
+//   ///
+//   kUnrankedMemRefType = 16,
+def UnrankedMemRefType : DialectType<(type
+  Type:$elementType
+)> {
+  let printerPredicate = "!$_val.getMemorySpace()";
+  let cBuilder = "get<$_resultType>(context, elementType, Attribute())";
+}
+
+//   ///   UnrankedMemRefTypeWithMemSpace {
+//   ///     memorySpace: Attribute,
+//   ///     elementType: Type
+//   ///   }
+//   /// Variant of UnrankedMemRefType with non-default memory space.
+//   kUnrankedMemRefTypeWithMemSpace = 17,
+def UnrankedMemRefTypeWithMemSpace : DialectType<(type
+  Attribute:$memorySpace,
+  Type:$elementType
+)> {
+  let printerPredicate = "$_val.getMemorySpace()";
+  // Note: order of serialization does not match order of builder.
+  let cBuilder = "get<$_resultType>(context, elementType, memorySpace)";
+}
+}
+
+//   ///   UnrankedTensorType {
+//   ///     elementType: Type
+//   ///   }
+//   ///
+//   kUnrankedTensorType = 18,
+def UnrankedTensorType : DialectType<(type
+  Type:$elementType
+)>;
+
+let cType = "VectorType" in {
+//   ///   VectorType {
+//   ///     shape: svarint[],
+//   ///     elementType: Type
+//   ///   }
+//   ///
+//   kVectorType = 19,
+def VectorType : DialectType<(type
+  Array<SignedVarInt>:$shape,
+  Type:$elementType
+)> {
+  let printerPredicate = "!$_val.getNumScalableDims()";
+}
+
+//   ///   VectorTypeWithScalableDims {
+//   ///     numScalableDims: varint,
+//   ///     shape: svarint[],
+//   ///     elementType: Type
+//   ///   }
+//   /// Variant of VectorType with scalable dimensions.
+//   kVectorTypeWithScalableDims = 20,
+def VectorTypeWithScalableDims : DialectType<(type
+  VarInt:$numScalableDims,
+  Array<SignedVarInt>:$shape,
+  Type:$elementType
+)> {
+  let printerPredicate = "$_val.getNumScalableDims()";
+  // Note: order of serialization does not match order of builder.
+  let cBuilder = "get<$_resultType>(context, shape, elementType, numScalableDims)";
+}
+}
+
+/// This enum contains marker codes used to indicate which attribute is
+/// currently being decoded, and how it should be decoded. The order of these
+/// codes should generally be unchanged, as any changes will inevitably break
+/// compatibility with older bytecode.
+
+def BuiltinDialectAttributes : DialectAttributes<"Builtin"> {
+  let elems = [
+    ArrayAttr,
+    DictionaryAttr,
+    StringAttr,
+    StringAttrWithType,
+    FlatSymbolRefAttr,
+    SymbolRefAttr,
+    TypeAttr,
+    UnitAttr,
+    IntegerAttr,
+    FloatAttr,
+    CallSiteLoc,
+    FileLineColLoc,
+    FusedLoc,
+    FusedLocWithMetadata,
+    NameLoc,
+    UnknownLoc,
+    DenseResourceElementsAttr,
+    DenseArrayAttr,
+    DenseIntOrFPElementsAttr,
+    DenseStringElementsAttr,
+    SparseElementsAttr
+  ];
+}
+
+def BuiltinDialectTypes : DialectTypes<"Builtin"> {
+  let elems = [
+    IntegerType,
+    IndexType,
+    FunctionType,
+    BFloat16Type,
+    Float16Type,
+    Float32Type,
+    Float64Type,
+    Float80Type,
+    Float128Type,
+    ComplexType,
+    MemRefType,
+    MemRefTypeWithMemSpace,
+    NoneType,
+    RankedTensorType,
+    RankedTensorTypeWithEncoding,
+    TupleType,
+    UnrankedMemRefType,
+    UnrankedMemRefTypeWithMemSpace,
+    UnrankedTensorType,
+    VectorType,
+    VectorTypeWithScalableDims
+  ];
+}
+
+#endif // BUILTIN_BYTECODE
diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index baee29c554c3d..f26465ef1d667 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -91,7 +91,7 @@ class TensorType : public Type, public ShapedType::Trait<TensorType> {
   ArrayRef<int64_t> getShape() const;
 
   /// Clone this type with the given shape and element type. If the
-  /// provided shape is `None`, the current shape of the type is used.
+  /// provided shape is `std::nullopt`, the current shape of the type is used.
   TensorType cloneWith(std::optional<ArrayRef<int64_t>> shape,
                        Type elementType) const;
 
@@ -127,7 +127,7 @@ class BaseMemRefType : public Type, public ShapedType::Trait<BaseMemRefType> {
   ArrayRef<int64_t> getShape() const;
 
   /// Clone this type with the given shape and element type. If the
-  /// provided shape is `None`, the current shape of the type is used.
+  /// provided shape is `std::nullopt`, the current shape of the type is used.
   BaseMemRefType cloneWith(std::optional<ArrayRef<int64_t>> shape,
                            Type elementType) const;
 
diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td
index b6c7b2193ffd3..c703d6a79c561 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.td
+++ b/mlir/include/mlir/IR/BuiltinTypes.td
@@ -1089,7 +1089,7 @@ def Builtin_Vector : Builtin_Type<"Vector", [ShapedTypeInterface], "Type"> {
     bool hasRank() const { return true; }
 
     /// Clone this vector type with the given shape and element type. If the
-    /// provided shape is `None`, the current shape of the type is used.
+    /// provided shape is `std::nullopt`, the current shape of the type is used.
     VectorType cloneWith(std::optional<ArrayRef<int64_t>> shape,
                          Type elementType) const;
   }];
diff --git a/mlir/include/mlir/IR/BytecodeBase.td b/mlir/include/mlir/IR/BytecodeBase.td
new file mode 100644
index 0000000000000..8cadf978b347a
--- /dev/null
+++ b/mlir/include/mlir/IR/BytecodeBase.td
@@ -0,0 +1,159 @@
+//===-- BytecodeBase.td - Base bytecode R/W defs -----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the base bytecode reader/writer definition file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BYTECODE_BASE
+#define BYTECODE_BASE
+
+class Bytecode<string parse="", string build="", string print="", string get="", string t=""> {
+  // Template for parsing.
+  //   $_reader == dialect bytecode reader
+  //   $_resultType == result type of parsed instance
+  //   $_var == variable being parsed
+  // If parser is not specified, then the parse of members is used.
+  string cParser = parse;
+
+  // Template for building from parsed.
+  //   $_resultType == result type of parsed instance
+  //   $_args == args/members comma separated
+  string cBuilder = build;
+
+  // Template for printing.
+  //   $_writer == dialect bytecode writer
+  //   $_name == parent attribute/type name
+  //   $_getter == getter
+  string cPrinter = print;
+
+  // Template for getter from in memory form.
+  //   $_attrType == attribute/type
+  //   $_member == member instance
+  //   $_getMember == get + UpperCamelFromSnake($_member)
+  string cGetter = get;
+
+  // Type built.
+  // Note: if cType is empty, then name of def is used.
+  string cType = t;
+
+  // Predicate guarding parse method as an Attribute/Type could have multiple
+  // parse methods, specify predicates to be orthogonal and cover entire
+  // "print space" to avoid order dependence.
+  // If empty then method is unconditional.
+  //   $_val == predicate function to apply on value dyn_casted to cType.
+  string printerPredicate = "";
+}
+
+class WithParser<string p="", Bytecode t=Bytecode<>> :
+  Bytecode<p, t.cBuilder, t.cPrinter, t.cGetter, t.cType>;
+class WithBuilder<string b="", Bytecode t=Bytecode<>> :
+  Bytecode<t.cParser, b, t.cPrinter, t.cGetter, t.cType>;
+class WithPrinter<string p="", Bytecode t=Bytecode<>> :
+  Bytecode<t.cParser, t.cBuilder, p, t.cGetter, t.cType>;
+class WithType<string ty="", Bytecode t=Bytecode<>> :
+  Bytecode<t.cParser, t.cBuilder, t.cPrinter, t.cGetter, ty>;
+class WithGetter<string g="", Bytecode t=Bytecode<>> :
+  Bytecode<t.cParser, t.cBuilder, t.cPrinter, g, t.cType>;
+
+class CompositeBytecode<string t = ""> : WithType<t>;
+
+class AttributeKind :
+  WithParser <"succeeded($_reader.readAttribute($_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeAttribute($_getter)">>>;
+def Attribute : AttributeKind;
+class TypeKind :
+  WithParser <"succeeded($_reader.readType($_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeType($_getter)">>>;
+def Type : TypeKind;
+def VarInt :
+  WithParser <"succeeded($_reader.readVarInt($_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeVarInt($_getter)",
+  WithType   <"uint64_t">>>>;
+def SignedVarInt :
+  WithParser <"succeeded($_reader.readSignedVarInt($_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeSignedVarInt($_getter)",
+  WithGetter<"$_attrType",
+  WithType   <"int64_t">>>>>;
+def Blob :
+  WithParser <"succeeded($_reader.readBlob($_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeOwnedBlob($_getter)",
+  WithType   <"ArrayRef<char>">>>>;
+
+class KnownWidthAPInt<string s> :
+  WithParser <"succeeded(readAPIntWithKnownWidth($_reader, " # s # ", $_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeAPIntWithKnownWidth($_getter)",
+  WithType   <"FailureOr<APInt>">>>>;
+class KnownSemanticsAPFloat<string s> :
+  WithParser <"succeeded(readAPFloatWithKnownSemantics($_reader, " # s # ", $_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeAPFloatWithKnownSemantics($_getter)",
+  WithType   <"FailureOr<APFloat>">>>>;
+class ResourceHandle<string s> :
+  WithParser <"succeeded(readResourceHandle<" # s # ">($_reader, $_var))",
+  WithBuilder<"$_args",
+  WithPrinter<"$_writer.writeResourceHandle($_getter)",
+  WithType   <"FailureOr<" # s # ">">>>>;
+
+// Helper to define variable that is defined later but not parsed nor printed.
+class LocalVar<string t, string d> :
+  WithParser <"(($_var = " # d # "), true)",
+  WithBuilder<"$_args",
+  WithPrinter<"",
+  WithType   <t>>>>;
+
+// Array instances.
+class Array<Bytecode t> {
+  Bytecode elemT = t;
+
+  string cBuilder = "$_args";
+}
+
+// Define dialect attribute or type.
+class DialectAttrOrType<dag d> {
+  // Any members starting with underscore is not fed to create function but
+  // treated as purely local variable.
+  dag members = d;
+
+  // When needing to specify a custom return type.
+  string cType = "";
+
+  // Any post-processing that needs to be done.
+  code postProcess = "";
+}
+
+class DialectAttribute<dag d> : DialectAttrOrType<d>, AttributeKind {
+  let cParser = "succeeded($_reader.readAttribute<$_resultType>($_var))";
+  let cBuilder = "get<$_resultType>(context, $_args)";
+}
+class DialectType<dag d> : DialectAttrOrType<d>, TypeKind {
+  let cParser = "succeeded($_reader.readType<$_resultType>($_var))";
+  let cBuilder = "get<$_resultType>(context, $_args)";
+}
+
+class DialectAttributes<string d> {
+  string dialect = d;
+  list<DialectAttrOrType> elems;
+}
+
+class DialectTypes<string d> {
+  string dialect = d;
+  list<DialectAttrOrType> elems;
+}
+
+def attr;
+def type;
+
+#endif // BYTECODE_BASE
+
diff --git a/mlir/include/mlir/IR/CMakeLists.txt b/mlir/include/mlir/IR/CMakeLists.txt
index 78d41d6dc4ab2..404f130022f1f 100644
--- a/mlir/include/mlir/IR/CMakeLists.txt
+++ b/mlir/include/mlir/IR/CMakeLists.txt
@@ -17,6 +17,10 @@ mlir_tablegen(BuiltinDialect.h.inc -gen-dialect-decls)
 mlir_tablegen(BuiltinDialect.cpp.inc -gen-dialect-defs)
 add_public_tablegen_target(MLIRBuiltinDialectIncGen)
 
+set(LLVM_TARGET_DEFINITIONS BuiltinDialectBytecode.td)
+mlir_tablegen(BuiltinDialectBytecode.cpp.inc -gen-bytecode -bytecode-dialect="Builtin")
+add_public_tablegen_target(MLIRBuiltinDialectBytecodeIncGen)
+
 set(LLVM_TARGET_DEFINITIONS BuiltinLocationAttributes.td)
 mlir_tablegen(BuiltinLocationAttributes.h.inc -gen-attrdef-decls)
 mlir_tablegen(BuiltinLocationAttributes.cpp.inc -gen-attrdef-defs)
diff --git a/mlir/include/mlir/IR/DialectBase.td b/mlir/include/mlir/IR/DialectBase.td
index 3d665da7f04f9..ff8c2beb65757 100644
--- a/mlir/include/mlir/IR/DialectBase.td
+++ b/mlir/include/mlir/IR/DialectBase.td
@@ -103,6 +103,9 @@ class Dialect {
 
   // If this dialect can be extended at runtime with new operations or types.
   bit isExtensible = 0;
+
+  // Whether inherent Attributes defined in ODS will be stored as Properties.
+  bit usePropertiesForAttributes = 0;
 }
 
 #endif // DIALECTBASE_TD
diff --git a/mlir/include/mlir/IR/ExtensibleDialect.h b/mlir/include/mlir/IR/ExtensibleDialect.h
index cb8f4fcfa7224..4ca27ea509868 100644
--- a/mlir/include/mlir/IR/ExtensibleDialect.h
+++ b/mlir/include/mlir/IR/ExtensibleDialect.h
@@ -26,6 +26,8 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Support/TypeID.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <optional>
 
 namespace mlir {
 class AsmParser;
@@ -462,6 +464,35 @@ class DynamicOpDefinition : public OperationName::Impl {
     return verifyRegionFn(op);
   }
 
+  /// Implementation for properties (unsupported right now here).
+  std::optional<Attribute> getInherentAttr(Operation *op,
+                                           StringRef name) final {
+    llvm::report_fatal_error("Unsupported getInherentAttr on Dynamic dialects");
+  }
+  void setInherentAttr(Operation *op, StringAttr name, Attribute value) final {
+    llvm::report_fatal_error("Unsupported setInherentAttr on Dynamic dialects");
+  }
+  void populateInherentAttrs(Operation *op, NamedAttrList &attrs) final {}
+  LogicalResult
+  verifyInherentAttrs(OperationName opName, NamedAttrList &attributes,
+                      function_ref<InFlightDiagnostic()> getDiag) final {
+    return success();
+  }
+  int getOpPropertyByteSize() final { return 0; }
+  void initProperties(OperationName opName, OpaqueProperties storage,
+                      OpaqueProperties init) final {}
+  void deleteProperties(OpaqueProperties prop) final {}
+  void populateDefaultProperties(OperationName opName,
+                                 OpaqueProperties properties) final {}
+
+  LogicalResult setPropertiesFromAttr(Operation *op, Attribute attr,
+                                      InFlightDiagnostic *diag) final {
+    return failure();
+  }
+  Attribute getPropertiesAsAttr(Operation *op) final { return {}; }
+  void copyProperties(OpaqueProperties lhs, OpaqueProperties rhs) final {}
+  llvm::hash_code hashProperties(OpaqueProperties prop) final { return {}; }
+
 private:
   DynamicOpDefinition(
       StringRef name, ExtensibleDialect *dialect,
diff --git a/mlir/include/mlir/IR/ODSSupport.h b/mlir/include/mlir/IR/ODSSupport.h
new file mode 100644
index 0000000000000..1d3cbbd690034
--- /dev/null
+++ b/mlir/include/mlir/IR/ODSSupport.h
@@ -0,0 +1,45 @@
+//===- ODSSupport.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a number of support method for ODS generated code.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_IR_ODSSUPPORT_H
+#define MLIR_IR_ODSSUPPORT_H
+
+#include "mlir/IR/Attributes.h"
+
+namespace mlir {
+
+//===----------------------------------------------------------------------===//
+// Support for properties
+//===----------------------------------------------------------------------===//
+
+/// Convert an IntegerAttr attribute to an int64_t, or return an error if the
+/// attribute isn't an IntegerAttr. If the optional diagnostic is provided an
+/// error message is also emitted.
+LogicalResult convertFromAttribute(int64_t &storage, Attribute attr,
+                                   InFlightDiagnostic *diag);
+
+/// Convert the provided int64_t to an IntegerAttr attribute.
+Attribute convertToAttribute(MLIRContext *ctx, int64_t storage);
+
+/// Convert a DenseI64ArrayAttr to the provided storage. It is expected that the
+/// storage has the same size as the array. An error is returned if the
+/// attribute isn't a DenseI64ArrayAttr or it does not have the same size. If
+/// the optional diagnostic is provided an error message is also emitted.
+LogicalResult convertFromAttribute(MutableArrayRef<int64_t> storage,
+                                   Attribute attr, InFlightDiagnostic *diag);
+
+/// Convert the provided ArrayRef<int64_t> to a DenseI64ArrayAttr attribute.
+Attribute convertToAttribute(MLIRContext *ctx, ArrayRef<int64_t> storage);
+
+} // namespace mlir
+
+#endif // MLIR_IR_ODSSUPPORT_H
\ No newline at end of file
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 5d0843cdf7529..b02c0eafa0c24 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -179,6 +179,69 @@ class TypeConstraint<Pred predicate, string summary = "",
   string cppClassName = cppClassNameParam;
 }
 
+// Base class for defining properties.
+class Property<string storageTypeParam = "", string desc = ""> {
+  // User-readable one line summary used in error reporting messages. If empty,
+  // a generic message will be used.
+  string summary = desc;
+  // The full description of this property.
+  string description = "";
+  code storageType = storageTypeParam;
+  code interfaceType = storageTypeParam;
+
+  // The expression to convert from the storage type to the Interface
+  // type. For example, an enum can be stored as an int but returned as an
+  // enum class.
+  //
+  // Format:
+  // - `$_storage` will contain the property in the storage type.
+  // - `$_ctxt` will contain an `MLIRContext *`.
+  code convertFromStorage = "$_storage";
+
+  // The call expression to build a property storage from the interface type.
+  //
+  // Format:
+  // - `$_storage` will contain the property in the storage type.
+  // - `$_value` will contain the property in the user interface type.
+  code assignToStorage = "$_storage = $_value";
+
+  // The call expression to convert from the storage type to an attribute.
+  //
+  // Format:
+  // - `$_storage` is the storage type value.
+  // - `$_ctxt` is a `MLIRContext *`.
+  //
+  // The expression must result in an Attribute.
+  code convertToAttribute = [{
+    convertToAttribute($_ctxt, $_storage)
+  }];
+
+  // The call expression to convert from an Attribute to the storage type.
+  //
+  // Format:
+  // - `$_storage` is the storage type value.
+  // - `$_attr` is the attribute.
+  // - `$_diag` is an optional Diagnostic pointer to emit error.
+  //
+  // The expression must return a LogicalResult
+  code convertFromAttribute = [{
+    return convertFromAttribute($_storage, $_attr, $_diag);
+  }];
+
+  // The call expression to hash the property.
+  //
+  // Format:
+  // - `$_storage` is the variable to hash.
+  //
+  // The expression should define a llvm::hash_code.
+  code hashProperty = [{
+    llvm::hash_value($_storage);
+  }];
+
+  // Default value for the property.
+  string defaultValue = ?;
+}
+
 // Subclass for constraints on an attribute.
 class AttrConstraint<Pred predicate, string summary = ""> :
     Constraint<predicate, summary>;
@@ -1090,6 +1153,16 @@ class DefaultValuedStrAttr<Attr attr, string val>
 class DefaultValuedOptionalStrAttr<Attr attr, string val>
     : DefaultValuedOptionalAttr<attr, "\"" # val # "\"">;
 
+//===----------------------------------------------------------------------===//
+// Primitive property kinds
+
+class ArrayProperty<string storageTypeParam = "", int n, string desc = ""> :
+  Property<storageTypeParam # "[" # n # "]", desc> {
+  let interfaceType = "::llvm::ArrayRef<" # storageTypeParam # ">";
+  let convertFromStorage = "$_storage";
+  let assignToStorage = "::llvm::copy($_value, $_storage)";
+}
+
 //===----------------------------------------------------------------------===//
 // Primitive attribute kinds
 
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 5a73d776996f9..d08d3de350f3e 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -71,6 +71,21 @@ void ensureRegionTerminator(
 
 } // namespace impl
 
+/// Structure used by default as a "marker" when no "Properties" are set on an
+/// Operation.
+struct EmptyProperties {};
+
+/// Traits to detect whether an Operation defined a `Properties` type, otherwise
+/// it'll default to `EmptyProperties`.
+template <class Op, class = void>
+struct PropertiesSelector {
+  using type = EmptyProperties;
+};
+template <class Op>
+struct PropertiesSelector<Op, std::void_t<typename Op::Properties>> {
+  using type = typename Op::Properties;
+};
+
 /// This is the concrete base class that holds the operation pointer and has
 /// non-generic methods that only depend on State (to avoid having them
 /// instantiated on template types that don't affect them.
@@ -206,6 +221,13 @@ class OpState {
   /// in generic form.
   static void print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect);
 
+  /// Parse properties as a Attribute.
+  static ParseResult genericParseProperties(OpAsmParser &parser,
+                                            Attribute &result);
+
+  /// Print the properties as a Attribute.
+  static void genericPrintProperties(OpAsmPrinter &p, Attribute properties);
+
   /// Print an operation name, eliding the dialect prefix if necessary.
   static void printOpName(Operation *op, OpAsmPrinter &p,
                           StringRef defaultDialect);
@@ -214,6 +236,14 @@ class OpState {
   /// so we can cast it away here.
   explicit OpState(Operation *state) : state(state) {}
 
+  /// For all op which don't have properties, we keep a single instance of
+  /// `EmptyProperties` to be used where a reference to a properties is needed:
+  /// this allow to bind a pointer to the reference without triggering UB.
+  static EmptyProperties &getEmptyProperties() {
+    static EmptyProperties emptyProperties;
+    return emptyProperties;
+  }
+
 private:
   Operation *state;
 
@@ -1471,13 +1501,17 @@ namespace op_definition_impl {
 /// Returns true if this given Trait ID matches the IDs of any of the provided
 /// trait types `Traits`.
 template <template <typename T> class... Traits>
-static bool hasTrait(TypeID traitID) {
+inline bool hasTrait(TypeID traitID) {
   TypeID traitIDs[] = {TypeID::get<Traits>()...};
   for (unsigned i = 0, e = sizeof...(Traits); i != e; ++i)
     if (traitIDs[i] == traitID)
       return true;
   return false;
 }
+template <>
+inline bool hasTrait<>(TypeID traitID) {
+  return false;
+}
 
 //===----------------------------------------------------------------------===//
 // Trait Folding
@@ -1693,6 +1727,33 @@ class Op : public OpState, public Traits<ConcreteType>... {
     (checkInterfaceTarget<Models>(), ...);
     info->attachInterface<Models...>();
   }
+  /// Convert the provided attribute to a property and assigned it to the
+  /// provided properties. This default implementation forwards to a free
+  /// function `setPropertiesFromAttribute` that can be looked up with ADL in
+  /// the namespace where the properties are defined. It can also be overridden
+  /// in the derived ConcreteOp.
+  template <typename PropertiesTy>
+  static LogicalResult setPropertiesFromAttr(PropertiesTy &prop, Attribute attr,
+                                             InFlightDiagnostic *diag) {
+    return setPropertiesFromAttribute(prop, attr, diag);
+  }
+  /// Convert the provided properties to an attribute. This default
+  /// implementation forwards to a free function `getPropertiesAsAttribute` that
+  /// can be looked up with ADL in the namespace where the properties are
+  /// defined. It can also be overridden in the derived ConcreteOp.
+  template <typename PropertiesTy>
+  static Attribute getPropertiesAsAttr(MLIRContext *ctx,
+                                       const PropertiesTy &prop) {
+    return getPropertiesAsAttribute(ctx, prop);
+  }
+  /// Hash the provided properties. This default implementation forwards to a
+  /// free function `computeHash` that can be looked up with ADL in the
+  /// namespace where the properties are defined. It can also be overridden in
+  /// the derived ConcreteOp.
+  template <typename PropertiesTy>
+  static llvm::hash_code computePropertiesHash(const PropertiesTy &prop) {
+    return computeHash(prop);
+  }
 
 private:
   /// Trait to check if T provides a 'fold' method for a single result op.
@@ -1733,10 +1794,35 @@ class Op : public OpState, public Traits<ConcreteType>... {
   template <typename T>
   using detect_has_print = llvm::is_detected<has_print, T>;
 
+  /// Trait to check if printProperties(OpAsmPrinter, T) exist
+  template <typename T, typename... Args>
+  using has_print_properties = decltype(printProperties(
+      std::declval<OpAsmPrinter &>(), std::declval<T>()));
+  template <typename T>
+  using detect_has_print_properties =
+      llvm::is_detected<has_print_properties, T>;
+
+  /// Trait to check if parseProperties(OpAsmParser, T) exist
+  template <typename T, typename... Args>
+  using has_parse_properties = decltype(parseProperties(
+      std::declval<OpAsmParser &>(), std::declval<T &>()));
+  template <typename T>
+  using detect_has_parse_properties =
+      llvm::is_detected<has_parse_properties, T>;
+
   /// Trait to check if T provides a 'ConcreteEntity' type alias.
   template <typename T>
   using has_concrete_entity_t = typename T::ConcreteEntity;
 
+public:
+  /// Returns true if this operation defines a `Properties` inner type.
+  static constexpr bool hasProperties() {
+    return !std::is_same_v<
+        typename ConcreteType::template InferredProperties<ConcreteType>,
+        EmptyProperties>;
+  }
+
+private:
   /// A struct-wrapped type alias to T::ConcreteEntity if provided and to
   /// ConcreteType otherwise. This is akin to std::conditional but doesn't fail
   /// on the missing typedef. Useful for checking if the interface is targeting
@@ -1801,11 +1887,18 @@ class Op : public OpState, public Traits<ConcreteType>... {
   foldSingleResultHook(Operation *op, ArrayRef<Attribute> operands,
                        SmallVectorImpl<OpFoldResult> &results) {
     OpFoldResult result;
-    if constexpr (has_fold_adaptor_single_result_v<ConcreteOpT>)
-      result = cast<ConcreteOpT>(op).fold(typename ConcreteOpT::FoldAdaptor(
-          operands, op->getAttrDictionary(), op->getRegions()));
-    else
+    if constexpr (has_fold_adaptor_single_result_v<ConcreteOpT>) {
+      if constexpr (hasProperties()) {
+        result = cast<ConcreteOpT>(op).fold(typename ConcreteOpT::FoldAdaptor(
+            operands, op->getAttrDictionary(),
+            cast<ConcreteOpT>(op).getProperties(), op->getRegions()));
+      } else {
+        result = cast<ConcreteOpT>(op).fold(typename ConcreteOpT::FoldAdaptor(
+            operands, op->getAttrDictionary(), {}, op->getRegions()));
+      }
+    } else {
       result = cast<ConcreteOpT>(op).fold(operands);
+    }
 
     // If the fold failed or was in-place, try to fold the traits of the
     // operation.
@@ -1824,10 +1917,18 @@ class Op : public OpState, public Traits<ConcreteType>... {
                                 SmallVectorImpl<OpFoldResult> &results) {
     auto result = LogicalResult::failure();
     if constexpr (has_fold_adaptor_v<ConcreteOpT>) {
-      result = cast<ConcreteOpT>(op).fold(
-          typename ConcreteOpT::FoldAdaptor(operands, op->getAttrDictionary(),
-                                            op->getRegions()),
-          results);
+      if constexpr (hasProperties()) {
+        result = cast<ConcreteOpT>(op).fold(
+            typename ConcreteOpT::FoldAdaptor(
+                operands, op->getAttrDictionary(),
+                cast<ConcreteOpT>(op).getProperties(), op->getRegions()),
+            results);
+      } else {
+        result = cast<ConcreteOpT>(op).fold(
+            typename ConcreteOpT::FoldAdaptor(operands, op->getAttrDictionary(),
+                                              {}, op->getRegions()),
+            results);
+      }
     } else {
       result = cast<ConcreteOpT>(op).fold(operands, results);
     }
@@ -1859,6 +1960,48 @@ class Op : public OpState, public Traits<ConcreteType>... {
     };
   }
 
+public:
+  template <typename T>
+  using InferredProperties = typename PropertiesSelector<T>::type;
+  template <typename T = ConcreteType>
+  InferredProperties<T> &getProperties() {
+    if constexpr (!hasProperties())
+      return getEmptyProperties();
+    return *getOperation()
+                ->getPropertiesStorage()
+                .template as<InferredProperties<T> *>();
+  }
+
+  /// This hook populates any unset default attrs when mapped to properties.
+  template <typename T = ConcreteType>
+  static void populateDefaultProperties(OperationName opName,
+                                        InferredProperties<T> &properties) {}
+
+  /// Print the operation properties. Unless overridden, this method will try to
+  /// dispatch to a `printProperties` free-function if it exists, and otherwise
+  /// by converting the properties to an Attribute.
+  template <typename T>
+  static void printProperties(MLIRContext *ctx, OpAsmPrinter &p,
+                              const T &properties) {
+    if constexpr (detect_has_print_properties<T>::value)
+      return printProperties(p, properties);
+    genericPrintProperties(p,
+                           ConcreteType::getPropertiesAsAttr(ctx, properties));
+  }
+
+  /// Parser the properties. Unless overridden, this method will print by
+  /// converting the properties to an Attribute.
+  template <typename T = ConcreteType>
+  static ParseResult parseProperties(OpAsmParser &parser,
+                                     OperationState &result) {
+    if constexpr (detect_has_parse_properties<InferredProperties<T>>::value) {
+      return parseProperties(
+          parser, result.getOrAddProperties<InferredProperties<T>>());
+    }
+    return genericParseProperties(parser, result.propertiesAttr);
+  }
+
+private:
   /// Implementation of `PopulateDefaultAttrsFn` OperationName hook.
   static OperationName::PopulateDefaultAttrsFn getPopulateDefaultAttrsFn() {
     return ConcreteType::populateDefaultAttrs;
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 02abca33f9460..e770c7453129b 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -957,13 +957,13 @@ class AsmParser {
   /// populated in `result`.
   template <typename AttrType>
   std::enable_if_t<detect_has_parse_method<AttrType>::value, ParseResult>
-  parseCustomAttributeWithFallback(AttrType &result) {
+  parseCustomAttributeWithFallback(AttrType &result, Type type = {}) {
     SMLoc loc = getCurrentLocation();
 
     // Parse any kind of attribute.
     Attribute attr;
     if (parseCustomAttributeWithFallback(
-            attr, {}, [&](Attribute &result, Type type) -> ParseResult {
+            attr, type, [&](Attribute &result, Type type) -> ParseResult {
               result = AttrType::parse(*this, type);
               return success(!!result);
             }))
@@ -979,8 +979,8 @@ class AsmParser {
   /// SFINAE parsing method for Attribute that don't implement a parse method.
   template <typename AttrType>
   std::enable_if_t<!detect_has_parse_method<AttrType>::value, ParseResult>
-  parseCustomAttributeWithFallback(AttrType &result) {
-    return parseAttribute(result);
+  parseCustomAttributeWithFallback(AttrType &result, Type type = {}) {
+    return parseAttribute(result, type);
   }
 
   /// Parse an arbitrary optional attribute of a given type and return it in
@@ -1368,6 +1368,7 @@ class OpAsmParser : public AsmParser {
       std::optional<MutableArrayRef<std::unique_ptr<Region>>> parsedRegions =
           std::nullopt,
       std::optional<ArrayRef<NamedAttribute>> parsedAttributes = std::nullopt,
+      std::optional<Attribute> parsedPropertiesAttribute = std::nullopt,
       std::optional<FunctionType> parsedFnType = std::nullopt) = 0;
 
   /// Parse a single SSA value operand name along with a result number if
diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index 632e0677483e5..8bd23a2a1fc57 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -22,6 +22,12 @@
 #include <optional>
 
 namespace mlir {
+namespace detail {
+/// This is a "tag" used for mapping the properties storage in
+/// llvm::TrailingObjects.
+enum class OpProperties : char {};
+} // namespace detail
+
 /// Operation is the basic unit of execution within MLIR.
 ///
 /// The following documentation are recommended to understand this class:
@@ -67,26 +73,35 @@ namespace mlir {
 /// Some operations like branches also refer to other Block, in which case they
 /// would have an array of `BlockOperand`.
 ///
+/// An Operation may contain optionally a "Properties" object: this is a
+/// pre-defined C++ object with a fixed size. This object is owned by the
+/// operation and deleted with the operation. It can be converted to an
+/// Attribute on demand, or loaded from an Attribute.
+///
+///
 /// Finally an Operation also contain an optional `DictionaryAttr`, a Location,
 /// and a pointer to its parent Block (if any).
 class alignas(8) Operation final
     : public llvm::ilist_node_with_parent<Operation, Block>,
       private llvm::TrailingObjects<Operation, detail::OperandStorage,
-                                    BlockOperand, Region, OpOperand> {
+                                    detail::OpProperties, BlockOperand, Region,
+                                    OpOperand> {
 public:
   /// Create a new Operation with the specific fields. This constructor
   /// populates the provided attribute list with default attributes if
   /// necessary.
   static Operation *create(Location location, OperationName name,
                            TypeRange resultTypes, ValueRange operands,
-                           NamedAttrList &&attributes, BlockRange successors,
+                           NamedAttrList &&attributes,
+                           OpaqueProperties properties, BlockRange successors,
                            unsigned numRegions);
 
   /// Create a new Operation with the specific fields. This constructor uses an
   /// existing attribute dictionary to avoid uniquing a list of attributes.
   static Operation *create(Location location, OperationName name,
                            TypeRange resultTypes, ValueRange operands,
-                           DictionaryAttr attributes, BlockRange successors,
+                           DictionaryAttr attributes,
+                           OpaqueProperties properties, BlockRange successors,
                            unsigned numRegions);
 
   /// Create a new Operation from the fields stored in `state`.
@@ -96,6 +111,7 @@ class alignas(8) Operation final
   static Operation *create(Location location, OperationName name,
                            TypeRange resultTypes, ValueRange operands,
                            NamedAttrList &&attributes,
+                           OpaqueProperties properties,
                            BlockRange successors = {},
                            RegionRange regions = {});
 
@@ -414,24 +430,82 @@ class alignas(8) Operation final
   // constants to names.  Attributes may be dynamically added and removed over
   // the lifetime of an operation.
 
+  /// Access an inherent attribute by name: returns an empty optional if there
+  /// is no inherent attribute with this name.
+  ///
+  /// This method is available as a transient facility in the migration process
+  /// to use Properties instead.
+  std::optional<Attribute> getInherentAttr(StringRef name);
+
+  /// Set an inherent attribute by name.
+  ///
+  /// This method is available as a transient facility in the migration process
+  /// to use Properties instead.
+  void setInherentAttr(StringAttr name, Attribute value);
+
+  /// Access a discardable attribute by name, returns an null Attribute if the
+  /// discardable attribute does not exist.
+  Attribute getDiscardableAttr(StringRef name) { return attrs.get(name); }
+
+  /// Access a discardable attribute by name, returns an null Attribute if the
+  /// discardable attribute does not exist.
+  Attribute getDiscardableAttr(StringAttr name) { return attrs.get(name); }
+
+  /// Set a discardable attribute by name.
+  void setDiscardableAttr(StringAttr name, Attribute value) {
+    NamedAttrList attributes(attrs);
+    if (attributes.set(name, value) != value)
+      attrs = attributes.getDictionary(getContext());
+  }
+
+  /// Return all of the discardable attributes on this operation.
+  ArrayRef<NamedAttribute> getDiscardableAttrs() { return attrs.getValue(); }
+
+  /// Return all of the discardable attributes on this operation as a
+  /// DictionaryAttr.
+  DictionaryAttr getDiscardableAttrDictionary() { return attrs; }
+
   /// Return all of the attributes on this operation.
-  ArrayRef<NamedAttribute> getAttrs() { return attrs.getValue(); }
+  ArrayRef<NamedAttribute> getAttrs() {
+    if (!getPropertiesStorage())
+      return getDiscardableAttrs();
+    return getAttrDictionary().getValue();
+  }
 
   /// Return all of the attributes on this operation as a DictionaryAttr.
-  DictionaryAttr getAttrDictionary() { return attrs; }
-
-  /// Set the attribute dictionary on this operation.
-  void setAttrs(DictionaryAttr newAttrs) {
+  DictionaryAttr getAttrDictionary();
+
+  /// Set the attributes from a dictionary on this operation.
+  /// These methods are expensive: if the dictionnary only contains discardable
+  /// attributes, `setDiscardableAttrs` is more efficient.
+  void setAttrs(DictionaryAttr newAttrs);
+  void setAttrs(ArrayRef<NamedAttribute> newAttrs);
+  /// Set the discardable attribute dictionary on this operation.
+  void setDiscardableAttrs(DictionaryAttr newAttrs) {
     assert(newAttrs && "expected valid attribute dictionary");
     attrs = newAttrs;
   }
-  void setAttrs(ArrayRef<NamedAttribute> newAttrs) {
-    setAttrs(DictionaryAttr::get(getContext(), newAttrs));
+  void setDiscardableAttrs(ArrayRef<NamedAttribute> newAttrs) {
+    setDiscardableAttrs(DictionaryAttr::get(getContext(), newAttrs));
   }
 
   /// Return the specified attribute if present, null otherwise.
-  Attribute getAttr(StringAttr name) { return attrs.get(name); }
-  Attribute getAttr(StringRef name) { return attrs.get(name); }
+  /// These methods are expensive: if the dictionnary only contains discardable
+  /// attributes, `getDiscardableAttr` is more efficient.
+  Attribute getAttr(StringAttr name) {
+    if (getPropertiesStorageSize()) {
+      if (std::optional<Attribute> inherentAttr = getInherentAttr(name))
+        return *inherentAttr;
+    }
+    return attrs.get(name);
+  }
+  Attribute getAttr(StringRef name) {
+    if (getPropertiesStorageSize()) {
+      if (std::optional<Attribute> inherentAttr = getInherentAttr(name))
+        return *inherentAttr;
+    }
+    return attrs.get(name);
+  }
 
   template <typename AttrClass>
   AttrClass getAttrOfType(StringAttr name) {
@@ -444,8 +518,20 @@ class alignas(8) Operation final
 
   /// Return true if the operation has an attribute with the provided name,
   /// false otherwise.
-  bool hasAttr(StringAttr name) { return attrs.contains(name); }
-  bool hasAttr(StringRef name) { return attrs.contains(name); }
+  bool hasAttr(StringAttr name) {
+    if (getPropertiesStorageSize()) {
+      if (std::optional<Attribute> inherentAttr = getInherentAttr(name))
+        return (bool)*inherentAttr;
+    }
+    return attrs.contains(name);
+  }
+  bool hasAttr(StringRef name) {
+    if (getPropertiesStorageSize()) {
+      if (std::optional<Attribute> inherentAttr = getInherentAttr(name))
+        return (bool)*inherentAttr;
+    }
+    return attrs.contains(name);
+  }
   template <typename AttrClass, typename NameT>
   bool hasAttrOfType(NameT &&name) {
     return static_cast<bool>(
@@ -455,6 +541,12 @@ class alignas(8) Operation final
   /// If the an attribute exists with the specified name, change it to the new
   /// value. Otherwise, add a new attribute with the specified name/value.
   void setAttr(StringAttr name, Attribute value) {
+    if (getPropertiesStorageSize()) {
+      if (std::optional<Attribute> inherentAttr = getInherentAttr(name)) {
+        setInherentAttr(name, value);
+        return;
+      }
+    }
     NamedAttrList attributes(attrs);
     if (attributes.set(name, value) != value)
       attrs = attributes.getDictionary(getContext());
@@ -467,6 +559,12 @@ class alignas(8) Operation final
   /// attribute that was erased, or nullptr if there was no attribute with such
   /// name.
   Attribute removeAttr(StringAttr name) {
+    if (getPropertiesStorageSize()) {
+      if (std::optional<Attribute> inherentAttr = getInherentAttr(name)) {
+        setInherentAttr(name, {});
+        return *inherentAttr;
+      }
+    }
     NamedAttrList attributes(attrs);
     Attribute removedAttr = attributes.erase(name);
     if (removedAttr)
@@ -511,7 +609,7 @@ class alignas(8) Operation final
     return dialect_attr_iterator(attrs.end(), attrs.end());
   }
 
-  /// Set the dialect attributes for this operation, and preserve all dependent.
+  /// Set the dialect attributes for this operation, and preserve all inherent.
   template <typename DialectAttrT>
   void setDialectAttrs(DialectAttrT &&dialectAttrs) {
     NamedAttrList attrs;
@@ -735,6 +833,44 @@ class alignas(8) Operation final
   /// handlers that may be listening.
   InFlightDiagnostic emitRemark(const Twine &message = {});
 
+  /// Returns the properties storage size.
+  int getPropertiesStorageSize() const {
+    return ((int)propertiesStorageSize) * 8;
+  }
+  /// Returns the properties storage.
+  OpaqueProperties getPropertiesStorage() {
+    if (propertiesStorageSize)
+      return {
+          reinterpret_cast<void *>(getTrailingObjects<detail::OpProperties>())};
+    return {nullptr};
+  }
+  OpaqueProperties getPropertiesStorage() const {
+    if (propertiesStorageSize)
+      return {reinterpret_cast<void *>(const_cast<detail::OpProperties *>(
+          getTrailingObjects<detail::OpProperties>()))};
+    return {nullptr};
+  }
+
+  /// Return the properties converted to an attribute.
+  /// This is expensive, and mostly useful when dealing with unregistered
+  /// operation. Returns an empty attribute if no properties are present.
+  Attribute getPropertiesAsAttribute();
+
+  /// Set the properties from the provided  attribute.
+  /// This is an expensive operation that can fail if the attribute is not
+  /// matching the expectations of the properties for this operation. This is
+  /// mostly useful for unregistered operations or used when parsing the
+  /// generic format. An optional diagnostic can be passed in for richer errors.
+  LogicalResult setPropertiesFromAttribute(Attribute attr,
+                                           InFlightDiagnostic *diagnostic);
+
+  /// Copy properties from an existing other properties object. The two objects
+  /// must be the same type.
+  void copyProperties(OpaqueProperties rhs);
+
+  /// Compute a hash for the op properties (if any).
+  llvm::hash_code hashProperties();
+
 private:
   //===--------------------------------------------------------------------===//
   // Ordering
@@ -758,7 +894,8 @@ class alignas(8) Operation final
 private:
   Operation(Location location, OperationName name, unsigned numResults,
             unsigned numSuccessors, unsigned numRegions,
-            DictionaryAttr attributes, bool hasOperandStorage);
+            int propertiesStorageSize, DictionaryAttr attributes,
+            OpaqueProperties properties, bool hasOperandStorage);
 
   // Operations are deleted through the destroy() member because they are
   // allocated with malloc.
@@ -845,13 +982,21 @@ class alignas(8) Operation final
 
   const unsigned numResults;
   const unsigned numSuccs;
-  const unsigned numRegions : 31;
+  const unsigned numRegions : 23;
 
   /// This bit signals whether this operation has an operand storage or not. The
   /// operand storage may be elided for operations that are known to never have
   /// operands.
   bool hasOperandStorage : 1;
 
+  /// The size of the storage for properties (if any), divided by 8: since the
+  /// Properties storage will always be rounded up to the next multiple of 8 we
+  /// save some bits here.
+  unsigned char propertiesStorageSize : 8;
+  /// This is the maximum size we support to allocate properties inline with an
+  /// operation: this must match the bitwidth above.
+  static constexpr int64_t propertiesCapacity = 8 * 256;
+
   /// This holds the name of the operation.
   OperationName name;
 
@@ -871,8 +1016,9 @@ class alignas(8) Operation final
   friend class llvm::ilist_node_with_parent<Operation, Block>;
 
   // This stuff is used by the TrailingObjects template.
-  friend llvm::TrailingObjects<Operation, detail::OperandStorage, BlockOperand,
-                               Region, OpOperand>;
+  friend llvm::TrailingObjects<Operation, detail::OperandStorage,
+                               detail::OpProperties, BlockOperand, Region,
+                               OpOperand>;
   size_t numTrailingObjects(OverloadToken<detail::OperandStorage>) const {
     return hasOperandStorage ? 1 : 0;
   }
@@ -880,6 +1026,9 @@ class alignas(8) Operation final
     return numSuccs;
   }
   size_t numTrailingObjects(OverloadToken<Region>) const { return numRegions; }
+  size_t numTrailingObjects(OverloadToken<detail::OpProperties>) const {
+    return getPropertiesStorageSize();
+  }
 };
 
 inline raw_ostream &operator<<(raw_ostream &os, const Operation &op) {
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index ba8f0a8bd79de..3631e41c1234d 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -14,8 +14,10 @@
 #ifndef MLIR_IR_OPERATIONSUPPORT_H
 #define MLIR_IR_OPERATIONSUPPORT_H
 
+#include "mlir/IR/Attributes.h"
 #include "mlir/IR/BlockSupport.h"
 #include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/TypeRange.h"
 #include "mlir/IR/Types.h"
@@ -24,6 +26,7 @@
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include "llvm/Support/TrailingObjects.h"
 #include <memory>
@@ -37,6 +40,7 @@ namespace mlir {
 class Dialect;
 class DictionaryAttr;
 class ElementsAttr;
+struct EmptyProperties;
 class MutableOperandRangeRange;
 class NamedAttrList;
 class Operation;
@@ -59,6 +63,25 @@ class ValueRange;
 template <typename ValueRangeT>
 class ValueTypeRange;
 
+//===----------------------------------------------------------------------===//
+// OpaqueProperties
+//===----------------------------------------------------------------------===//
+
+/// Simple wrapper around a void* in order to express generically how to pass
+/// in op properties through APIs.
+class OpaqueProperties {
+public:
+  OpaqueProperties(void *prop) : properties(prop) {}
+  operator bool() const { return properties != nullptr; }
+  template <typename Dest>
+  Dest as() const {
+    return static_cast<Dest>(const_cast<void *>(properties));
+  }
+
+private:
+  void *properties;
+};
+
 //===----------------------------------------------------------------------===//
 // OperationName
 //===----------------------------------------------------------------------===//
@@ -98,6 +121,26 @@ class OperationName {
     virtual void printAssembly(Operation *, OpAsmPrinter &, StringRef) = 0;
     virtual LogicalResult verifyInvariants(Operation *) = 0;
     virtual LogicalResult verifyRegionInvariants(Operation *) = 0;
+    /// Implementation for properties
+    virtual std::optional<Attribute> getInherentAttr(Operation *,
+                                                     StringRef name) = 0;
+    virtual void setInherentAttr(Operation *op, StringAttr name,
+                                 Attribute value) = 0;
+    virtual void populateInherentAttrs(Operation *op, NamedAttrList &attrs) = 0;
+    virtual LogicalResult
+    verifyInherentAttrs(OperationName opName, NamedAttrList &attributes,
+                        function_ref<InFlightDiagnostic()> getDiag) = 0;
+    virtual int getOpPropertyByteSize() = 0;
+    virtual void initProperties(OperationName opName, OpaqueProperties storage,
+                                OpaqueProperties init) = 0;
+    virtual void deleteProperties(OpaqueProperties) = 0;
+    virtual void populateDefaultProperties(OperationName opName,
+                                           OpaqueProperties properties) = 0;
+    virtual LogicalResult setPropertiesFromAttr(Operation *, Attribute,
+                                                InFlightDiagnostic *) = 0;
+    virtual Attribute getPropertiesAsAttr(Operation *) = 0;
+    virtual void copyProperties(OpaqueProperties, OpaqueProperties) = 0;
+    virtual llvm::hash_code hashProperties(OpaqueProperties) = 0;
   };
 
 public:
@@ -158,6 +201,25 @@ class OperationName {
     void printAssembly(Operation *, OpAsmPrinter &, StringRef) final;
     LogicalResult verifyInvariants(Operation *) final;
     LogicalResult verifyRegionInvariants(Operation *) final;
+    /// Implementation for properties
+    std::optional<Attribute> getInherentAttr(Operation *op,
+                                             StringRef name) final;
+    void setInherentAttr(Operation *op, StringAttr name, Attribute value) final;
+    void populateInherentAttrs(Operation *op, NamedAttrList &attrs) final;
+    LogicalResult
+    verifyInherentAttrs(OperationName opName, NamedAttrList &attributes,
+                        function_ref<InFlightDiagnostic()> getDiag) final;
+    int getOpPropertyByteSize() final;
+    void initProperties(OperationName opName, OpaqueProperties storage,
+                        OpaqueProperties init) final;
+    void deleteProperties(OpaqueProperties) final;
+    void populateDefaultProperties(OperationName opName,
+                                   OpaqueProperties properties) final;
+    LogicalResult setPropertiesFromAttr(Operation *, Attribute,
+                                        InFlightDiagnostic *) final;
+    Attribute getPropertiesAsAttr(Operation *) final;
+    void copyProperties(OpaqueProperties, OpaqueProperties) final;
+    llvm::hash_code hashProperties(OpaqueProperties) final;
   };
 
 public:
@@ -309,6 +371,68 @@ class OperationName {
     return !isRegistered() || hasInterface(interfaceID);
   }
 
+  /// Lookup an inherent attribute by name, this method isn't recommended
+  /// and may be removed in the future.
+  std::optional<Attribute> getInherentAttr(Operation *op,
+                                           StringRef name) const {
+    return getImpl()->getInherentAttr(op, name);
+  }
+
+  void setInherentAttr(Operation *op, StringAttr name, Attribute value) const {
+    return getImpl()->setInherentAttr(op, name, value);
+  }
+
+  void populateInherentAttrs(Operation *op, NamedAttrList &attrs) const {
+    return getImpl()->populateInherentAttrs(op, attrs);
+  }
+  /// This method exists for backward compatibility purpose when using
+  /// properties to store inherent attributes, it enables validating the
+  /// attributes when parsed from the older generic syntax pre-Properties.
+  LogicalResult
+  verifyInherentAttrs(NamedAttrList &attributes,
+                      function_ref<InFlightDiagnostic()> getDiag) const {
+    return getImpl()->verifyInherentAttrs(*this, attributes, getDiag);
+  }
+  /// This hooks return the number of bytes to allocate for the op properties.
+  int getOpPropertyByteSize() const {
+    return getImpl()->getOpPropertyByteSize();
+  }
+
+  /// This hooks destroy the op properties.
+  void destroyOpProperties(OpaqueProperties properties) const {
+    getImpl()->deleteProperties(properties);
+  }
+
+  /// Initialize the op properties.
+  void initOpProperties(OpaqueProperties storage, OpaqueProperties init) const {
+    getImpl()->initProperties(*this, storage, init);
+  }
+
+  /// Set the default values on the ODS attribute in the properties.
+  void populateDefaultProperties(OpaqueProperties properties) const {
+    getImpl()->populateDefaultProperties(*this, properties);
+  }
+
+  /// Return the op properties converted to an Attribute.
+  Attribute getOpPropertiesAsAttribute(Operation *op) const {
+    return getImpl()->getPropertiesAsAttr(op);
+  }
+
+  /// Define the op properties from the provided Attribute.
+  LogicalResult
+  setOpPropertiesFromAttribute(Operation *op, Attribute properties,
+                               InFlightDiagnostic *diagnostic) const {
+    return getImpl()->setPropertiesFromAttr(op, properties, diagnostic);
+  }
+
+  void copyOpProperties(OpaqueProperties lhs, OpaqueProperties rhs) const {
+    return getImpl()->copyProperties(lhs, rhs);
+  }
+
+  llvm::hash_code hashOpProperties(OpaqueProperties properties) const {
+    return getImpl()->hashProperties(properties);
+  }
+
   /// Return the dialect this operation is registered to if the dialect is
   /// loaded in the context, or nullptr if the dialect isn't loaded.
   Dialect *getDialect() const {
@@ -413,6 +537,104 @@ class RegisteredOperationName : public OperationName {
     LogicalResult verifyRegionInvariants(Operation *op) final {
       return ConcreteOp::getVerifyRegionInvariantsFn()(op);
     }
+
+    /// Implementation for "Properties"
+
+    using Properties = std::remove_reference_t<
+        decltype(std::declval<ConcreteOp>().getProperties())>;
+
+    std::optional<Attribute> getInherentAttr(Operation *op,
+                                             StringRef name) final {
+      if constexpr (hasProperties) {
+        auto concreteOp = cast<ConcreteOp>(op);
+        return ConcreteOp::getInherentAttr(concreteOp.getProperties(), name);
+      }
+      // If the op does not have support for properties, we dispatch back to the
+      // dictionnary of discardable attributes for now.
+      return cast<ConcreteOp>(op)->getDiscardableAttr(name);
+    }
+    void setInherentAttr(Operation *op, StringAttr name,
+                         Attribute value) final {
+      if constexpr (hasProperties) {
+        auto concreteOp = cast<ConcreteOp>(op);
+        return ConcreteOp::setInherentAttr(concreteOp.getProperties(), name,
+                                           value);
+      }
+      // If the op does not have support for properties, we dispatch back to the
+      // dictionnary of discardable attributes for now.
+      return cast<ConcreteOp>(op)->setDiscardableAttr(name, value);
+    }
+    void populateInherentAttrs(Operation *op, NamedAttrList &attrs) final {
+      if constexpr (hasProperties) {
+        auto concreteOp = cast<ConcreteOp>(op);
+        ConcreteOp::populateInherentAttrs(concreteOp.getProperties(), attrs);
+      }
+    }
+    LogicalResult
+    verifyInherentAttrs(OperationName opName, NamedAttrList &attributes,
+                        function_ref<InFlightDiagnostic()> getDiag) final {
+      if constexpr (hasProperties)
+        return ConcreteOp::verifyInherentAttrs(opName, attributes, getDiag);
+      return success();
+    }
+    // Detect if the concrete operation defined properties.
+    static constexpr bool hasProperties = !std::is_same_v<
+        typename ConcreteOp::template InferredProperties<ConcreteOp>,
+        EmptyProperties>;
+
+    int getOpPropertyByteSize() final {
+      if constexpr (hasProperties)
+        return sizeof(Properties);
+      return 0;
+    }
+    void initProperties(OperationName opName, OpaqueProperties storage,
+                        OpaqueProperties init) final {
+      using Properties =
+          typename ConcreteOp::template InferredProperties<ConcreteOp>;
+      if (init)
+        new (storage.as<Properties *>()) Properties(*init.as<Properties *>());
+      else
+        new (storage.as<Properties *>()) Properties();
+      if constexpr (hasProperties)
+        ConcreteOp::populateDefaultProperties(opName,
+                                              *storage.as<Properties *>());
+    }
+    void deleteProperties(OpaqueProperties prop) final {
+      prop.as<Properties *>()->~Properties();
+    }
+    void populateDefaultProperties(OperationName opName,
+                                   OpaqueProperties properties) final {
+      if constexpr (hasProperties)
+        ConcreteOp::populateDefaultProperties(opName,
+                                              *properties.as<Properties *>());
+    }
+
+    LogicalResult setPropertiesFromAttr(Operation *op, Attribute attr,
+                                        InFlightDiagnostic *diag) final {
+      if constexpr (hasProperties)
+        return ConcreteOp::setPropertiesFromAttr(
+            cast<ConcreteOp>(op).getProperties(), attr, diag);
+      if (diag)
+        *diag << "This operation does not support properties";
+      return failure();
+    }
+    Attribute getPropertiesAsAttr(Operation *op) final {
+      if constexpr (hasProperties) {
+        auto concreteOp = cast<ConcreteOp>(op);
+        return ConcreteOp::getPropertiesAsAttr(concreteOp->getContext(),
+                                               concreteOp.getProperties());
+      }
+      return {};
+    }
+    void copyProperties(OpaqueProperties lhs, OpaqueProperties rhs) final {
+      *lhs.as<Properties *>() = *rhs.as<Properties *>();
+    }
+    llvm::hash_code hashProperties(OpaqueProperties prop) final {
+      if constexpr (hasProperties)
+        return ConcreteOp::computePropertiesHash(*prop.as<Properties *>());
+
+      return {};
+    }
   };
 
   /// Lookup the registered operation information for the given operation.
@@ -600,6 +822,11 @@ class NamedAttrList {
     assign(range.begin(), range.end());
   }
 
+  void clear() {
+    attrs.clear();
+    dictionarySorted.setPointerAndInt(nullptr, false);
+  }
+
   bool empty() const { return attrs.empty(); }
 
   void reserve(size_type N) { attrs.reserve(N); }
@@ -694,6 +921,19 @@ struct OperationState {
   /// Regions that the op will hold.
   SmallVector<std::unique_ptr<Region>, 1> regions;
 
+  // If we're creating an unregistered operation, this Attribute is used to
+  // build the properties. Otherwise it is ignored. For registered operations
+  // see the `getOrAddProperties` method.
+  Attribute propertiesAttr;
+
+private:
+  OpaqueProperties properties = nullptr;
+  TypeID propertiesId;
+  llvm::function_ref<void(OpaqueProperties)> propertiesDeleter;
+  llvm::function_ref<void(OpaqueProperties, const OpaqueProperties)>
+      propertiesSetter;
+  friend class Operation;
+
 public:
   OperationState(Location location, StringRef name);
   OperationState(Location location, OperationName name);
@@ -706,6 +946,37 @@ struct OperationState {
                  TypeRange types, ArrayRef<NamedAttribute> attributes = {},
                  BlockRange successors = {},
                  MutableArrayRef<std::unique_ptr<Region>> regions = {});
+  OperationState(OperationState &&other) = default;
+  OperationState(const OperationState &other) = default;
+  OperationState &operator=(OperationState &&other) = default;
+  OperationState &operator=(const OperationState &other) = default;
+  ~OperationState();
+
+  /// Get (or create) a properties of the provided type to be set on the
+  /// operation on creation.
+  template <typename T>
+  T &getOrAddProperties() {
+    if (!properties) {
+      T *p = new T{};
+      properties = p;
+      propertiesDeleter = [](OpaqueProperties prop) {
+        delete prop.as<const T *>();
+      };
+      propertiesSetter = [](OpaqueProperties new_prop,
+                            const OpaqueProperties prop) {
+        *new_prop.as<T *>() = *prop.as<const T *>();
+      };
+      propertiesId = TypeID::get<T>();
+    }
+    assert(propertiesId == TypeID::get<T>() && "Inconsistent properties");
+    return *properties.as<T *>();
+  }
+  OpaqueProperties getRawProperties() { return properties; }
+
+  // Set the properties defined on this OpState on the given operation,
+  // optionally emit diagnostics on error through the provided diagnostic.
+  LogicalResult setProperties(Operation *op,
+                              InFlightDiagnostic *diagnostic) const;
 
   void addOperands(ValueRange newOperands);
 
@@ -835,7 +1106,7 @@ class OpPrintingFlags {
   OpPrintingFlags &printGenericOpForm();
 
   /// Skip printing regions.
-  OpPrintingFlags &skipRegions();
+  OpPrintingFlags &skipRegions(bool skip = true);
 
   /// Do not verify the operation when using custom operation printers.
   OpPrintingFlags &assumeVerified();
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index 578594eee9dff..e0f154824d8eb 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -14,7 +14,7 @@
 #ifndef MLIR_INITALLDIALECTS_H_
 #define MLIR_INITALLDIALECTS_H_
 
-#include "mlir/Dialect/AMDGPU/AMDGPUDialect.h"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
 #include "mlir/Dialect/AMX/AMXDialect.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
@@ -42,7 +42,7 @@
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.h"
-#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
+#include "mlir/Dialect/Linalg/TransformOps/DialectExtension.h"
 #include "mlir/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Linalg/Transforms/TilingInterfaceImpl.h"
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
@@ -71,6 +71,7 @@
 #include "mlir/Dialect/Tensor/IR/TensorInferTypeOpInterfaceImpl.h"
 #include "mlir/Dialect/Tensor/IR/TensorTilingInterfaceImpl.h"
 #include "mlir/Dialect/Tensor/IR/ValueBoundsOpInterfaceImpl.h"
+#include "mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h"
 #include "mlir/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
@@ -132,6 +133,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
   linalg::registerTransformDialectExtension(registry);
   memref::registerTransformDialectExtension(registry);
   scf::registerTransformDialectExtension(registry);
+  tensor::registerTransformDialectExtension(registry);
   vector::registerTransformDialectExtension(registry);
 
   // Register all external models.
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
index 216fc12323718..a42983d7fe036 100644
--- a/mlir/include/mlir/InitAllPasses.h
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -15,6 +15,7 @@
 #define MLIR_INITALLPASSES_H_
 
 #include "mlir/Conversion/Passes.h"
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
 #include "mlir/Dialect/Affine/Passes.h"
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
 #include "mlir/Dialect/Async/Passes.h"
@@ -56,6 +57,7 @@ inline void registerAllPasses() {
 
   // Dialect passes
   affine::registerAffinePasses();
+  amdgpu::registerAMDGPUPasses();
   registerAsyncPasses();
   arith::registerArithPasses();
   bufferization::registerBufferizationPasses();
diff --git a/mlir/include/mlir/Interfaces/CMakeLists.txt b/mlir/include/mlir/Interfaces/CMakeLists.txt
index 2cb1d489c4bfd..4e0f7ac5a0400 100644
--- a/mlir/include/mlir/Interfaces/CMakeLists.txt
+++ b/mlir/include/mlir/Interfaces/CMakeLists.txt
@@ -7,6 +7,7 @@ add_mlir_interface(DestinationStyleOpInterface)
 add_mlir_interface(InferIntRangeInterface)
 add_mlir_interface(InferTypeOpInterface)
 add_mlir_interface(LoopLikeInterface)
+add_mlir_interface(Mem2RegInterfaces)
 add_mlir_interface(ParallelCombiningOpInterface)
 add_mlir_interface(RuntimeVerifiableOpInterface)
 add_mlir_interface(ShapedOpInterfaces)
diff --git a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
index 1a331f8a706c6..67c668421238a 100644
--- a/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
+++ b/mlir/include/mlir/Interfaces/ControlFlowInterfaces.td
@@ -67,7 +67,7 @@ def BranchOpInterface : OpInterface<"BranchOpInterface"> {
     >,
     InterfaceMethod<[{
         Returns the `BlockArgument` corresponding to operand `operandIndex` in
-        some successor, or None if `operandIndex` isn't a successor operand
+        some successor, or std::nullopt if `operandIndex` isn't a successor operand
         index.
       }],
       "::std::optional<::mlir::BlockArgument>", "getSuccessorBlockArgument",
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.h b/mlir/include/mlir/Interfaces/InferTypeOpInterface.h
index b63d8b6be6739..acfe40484cf63 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.h
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.h
@@ -244,11 +244,11 @@ LogicalResult inferReturnTensorTypes(
     function_ref<
         LogicalResult(MLIRContext *, std::optional<Location> location,
                       ValueShapeRange operands, DictionaryAttr attributes,
-                      RegionRange regions,
+                      OpaqueProperties properties, RegionRange regions,
                       SmallVectorImpl<ShapedTypeComponents> &retComponents)>
         componentTypeFn,
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes);
 
 /// Verifies that the inferred result types match the actual result types for
@@ -281,7 +281,7 @@ class InferTensorType : public TraitBase<ConcreteType, InferTensorType> {
   static LogicalResult
   inferReturnTypes(MLIRContext *context, std::optional<Location> location,
                    ValueRange operands, DictionaryAttr attributes,
-                   RegionRange regions,
+                   OpaqueProperties properties, RegionRange regions,
                    SmallVectorImpl<Type> &inferredReturnTypes) {
     static_assert(
         ConcreteType::template hasTrait<InferShapedTypeOpInterface::Trait>(),
@@ -291,7 +291,7 @@ class InferTensorType : public TraitBase<ConcreteType, InferTensorType> {
         "requires InferTypeOpInterface to ensure succesful invocation");
     return ::mlir::detail::inferReturnTensorTypes(
         ConcreteType::inferReturnTypeComponents, context, location, operands,
-        attributes, regions, inferredReturnTypes);
+        attributes, properties, regions, inferredReturnTypes);
   }
 };
 
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
index 9f7118172389f..6925e39b0a9c6 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
@@ -44,6 +44,7 @@ def InferTypeOpInterface : OpInterface<"InferTypeOpInterface"> {
                     "::std::optional<::mlir::Location>":$location,
                     "::mlir::ValueRange":$operands,
                     "::mlir::DictionaryAttr":$attributes,
+                    "::mlir::OpaqueProperties":$properties,
                     "::mlir::RegionRange":$regions,
                     "::llvm::SmallVectorImpl<::mlir::Type>&":$inferredReturnTypes)
     >,
@@ -75,13 +76,14 @@ def InferTypeOpInterface : OpInterface<"InferTypeOpInterface"> {
                     "::std::optional<::mlir::Location>":$location,
                     "::mlir::ValueRange":$operands,
                     "::mlir::DictionaryAttr":$attributes,
+                    "::mlir::OpaqueProperties":$properties,
                     "::mlir::RegionRange":$regions,
                     "::llvm::SmallVectorImpl<::mlir::Type>&":$returnTypes),
       /*methodBody=*/[{}],
       /*defaultImplementation=*/[{
           llvm::SmallVector<Type, 4> inferredReturnTypes;
           if (failed(ConcreteOp::inferReturnTypes(context, location, operands,
-                                                  attributes, regions,
+                                                  attributes, properties, regions,
                                                   inferredReturnTypes)))
             return failure();
           if (!ConcreteOp::isCompatibleReturnTypes(inferredReturnTypes,
@@ -147,6 +149,7 @@ def InferShapedTypeOpInterface : OpInterface<"InferShapedTypeOpInterface"> {
                     "::std::optional<::mlir::Location>":$location,
                     "::mlir::ValueShapeRange":$operands,
                     "::mlir::DictionaryAttr":$attributes,
+                    "::mlir::OpaqueProperties":$properties,
                     "::mlir::RegionRange":$regions,
                     "::llvm::SmallVectorImpl<::mlir::ShapedTypeComponents>&":
                       $inferredReturnShapes),
diff --git a/mlir/include/mlir/Interfaces/Mem2RegInterfaces.h b/mlir/include/mlir/Interfaces/Mem2RegInterfaces.h
new file mode 100644
index 0000000000000..c962d98624b42
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/Mem2RegInterfaces.h
@@ -0,0 +1,39 @@
+//===-- Mem2RegInterfaces.h - Mem2Reg interfaces ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_MEM2REGINTERFACES_H
+#define MLIR_INTERFACES_MEM2REGINTERFACES_H
+
+#include "mlir/IR/Dominance.h"
+#include "mlir/IR/OpDefinition.h"
+
+namespace mlir {
+
+/// Represents a slot in memory. This is generated by an allocating operation
+/// (for example alloca).
+struct MemorySlot {
+  /// Pointer to the memory slot, used by operations to refer to it.
+  Value ptr;
+  /// Type of the value contained in the slot.
+  Type elemType;
+};
+
+/// Returned by operation promotion logic requesting the deletion of an
+/// operation.
+enum class DeletionKind {
+  /// Keep the operation after promotion.
+  Keep,
+  /// Delete the operation after promotion.
+  Delete,
+};
+
+} // namespace mlir
+
+#include "mlir/Interfaces/Mem2RegInterfaces.h.inc"
+
+#endif // MLIR_INTERFACES_MEM2REGINTERFACES_H
diff --git a/mlir/include/mlir/Interfaces/Mem2RegInterfaces.td b/mlir/include/mlir/Interfaces/Mem2RegInterfaces.td
new file mode 100644
index 0000000000000..b0d0a8cb54bdd
--- /dev/null
+++ b/mlir/include/mlir/Interfaces/Mem2RegInterfaces.td
@@ -0,0 +1,196 @@
+//===-- Mem2RegInterfaces.td - Mem2Reg interfaces ----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INTERFACES_MEM2REGINTERFACES
+#define MLIR_INTERFACES_MEM2REGINTERFACES
+
+include "mlir/IR/OpBase.td"
+
+def PromotableAllocationOpInterface
+    : OpInterface<"PromotableAllocationOpInterface"> {
+  let description = [{
+    Describes an operation allocating a memory slot that can be promoted into
+    SSA values.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<[{
+        Returns a list of memory slots for which promotion should be attempted.
+        This only considers the local semantics of the allocator, ignoring
+        whether the slot pointer is properly used or not. This allocator is the
+        "owner" of the returned slots, meaning no two allocators should return
+        the same slot. The content of the memory slot must only be reachable
+        using loads and stores to the provided slot pointer, no aliasing is
+        allowed.
+
+        Promotion of the slot will lead to the slot pointer no longer being
+        used, leaving the content of the memory slot unreachable.
+      }], "::llvm::SmallVector<::mlir::MemorySlot>", "getPromotableSlots",
+      (ins)
+    >,
+    InterfaceMethod<[{
+        Provides the default Value of this memory slot. The provided Value
+        will be used as the reaching definition of loads done before any store.
+        This Value must outlive the promotion and dominate all the uses of this
+        slot's pointer. The provided builder can be used to create the default
+        value on the fly.
+
+        The builder is located at the beginning of the block where the slot
+        pointer is defined.
+      }], "::mlir::Value", "getDefaultValue",
+      (ins "const ::mlir::MemorySlot &":$slot, "::mlir::OpBuilder &":$builder)
+    >,
+    InterfaceMethod<[{
+        Hook triggered for every new block argument added to a block.
+        This will only be called for slots declared by this operation.
+
+        The builder is located at the beginning of the block on call.
+      }],
+      "void", "handleBlockArgument",
+      (ins
+        "const ::mlir::MemorySlot &":$slot,
+        "::mlir::BlockArgument":$argument,
+        "::mlir::OpBuilder &":$builder
+      )
+    >,
+    InterfaceMethod<[{
+        Hook triggered once the promotion of a slot is complete. This can
+        also clean up the created default value if necessary.
+        This will only be called for slots declared by this operation.
+      }],
+      "void", "handlePromotionComplete",
+      (ins "const ::mlir::MemorySlot &":$slot, "::mlir::Value":$defaultValue)
+    >,
+  ];
+}
+
+def PromotableMemOpInterface : OpInterface<"PromotableMemOpInterface"> {
+  let description = [{
+    Describes an operation that can load from memory slots and/or store
+    to memory slots. Loads and stores must be of whole values of the same
+    type as the slot itself.
+
+    If the same operation does both loads and stores on the same slot, the
+    load must semantically happen first.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<[{
+        Gets whether this operation loads from the specified slot.
+      }],
+      "bool", "loadsFrom",
+      (ins "const ::mlir::MemorySlot &":$slot)
+    >,
+    InterfaceMethod<[{
+        Gets the value stored to the provided memory slot, or returns a null
+        value if this operation does not store to this slot. An operation
+        storing a value to a slot must always be able to provide the value it
+        stores. This method is only called on operations that use the slot.
+      }],
+      "::mlir::Value", "getStored",
+      (ins "const ::mlir::MemorySlot &":$slot)
+    >,
+    InterfaceMethod<[{
+        Checks that this operation can be promoted to no longer use the provided
+        blocking uses, in the context of promoting `slot`.
+
+        If the removal procedure of the use will require that other uses get
+        removed, that dependency should be added to the `newBlockingUses`
+        argument. Dependent uses must only be uses of results of this operation.
+      }], "bool", "canUsesBeRemoved",
+      (ins "const ::mlir::MemorySlot &":$slot,
+           "const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> &":$blockingUses,
+           "::llvm::SmallVectorImpl<::mlir::OpOperand *> &":$newBlockingUses)
+    >,
+    InterfaceMethod<[{
+        Transforms IR to ensure that the current operation does not use the
+        provided memory slot anymore. `reachingDefinition` contains the value
+        currently stored in the provided memory slot, immediately before the
+        current operation.
+
+        During the transformation, *no operation should be deleted*.
+        The operation can only schedule its own deletion by returning the
+        appropriate `DeletionKind`. The deletion must be legal assuming the
+        blocking uses passed through the `newBlockingUses` list in
+        `canUseBeRemoved` have been removed.
+
+        After calling this method, the blocking uses should have disappeared
+        or this operation should have scheduled its own deletion.
+
+        This method will only be called after ensuring promotion is allowed via
+        `canUseBeRemoved`. The requested blocking use removal may or may not
+        have been done at the point of calling this method, but it will be done
+        eventually.
+
+        The builder is located after the promotable operation on call.
+      }],
+      "::mlir::DeletionKind",
+      "removeBlockingUses",
+      (ins "const ::mlir::MemorySlot &":$slot,
+           "const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &":$blockingUses,
+           "::mlir::OpBuilder &":$builder,
+           "::mlir::Value":$reachingDefinition)
+    >,
+  ];
+}
+
+def PromotableOpInterface : OpInterface<"PromotableOpInterface"> {
+  let description = [{
+    Describes an operation that can be transformed or deleted so it no longer
+    uses a provided value (blocking use), in case this would allow the promotion
+    of a memory slot.
+  }];
+  let cppNamespace = "::mlir";
+
+  let methods = [
+    InterfaceMethod<[{
+        Checks that this operation can be promoted to no longer use the provided
+        blocking uses, in the context of promoting `slot`.
+
+        If the removal procedure of the use will require that other uses get
+        removed, that dependency should be added to the `newBlockingUses`
+        argument. Dependent uses must only be uses of results of this operation.
+      }], "bool", "canUsesBeRemoved",
+      (ins "const ::mlir::MemorySlot &":$slot,
+           "const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> &":$blockingUses,
+           "::llvm::SmallVectorImpl<::mlir::OpOperand *> &":$newBlockingUses)
+    >,
+    InterfaceMethod<[{
+        Transforms IR to ensure that the current operation does not use the
+        provided memory slot anymore. In contrast to `PromotableMemOpInterface`,
+        operations implementing this interface must not need access to the
+        reaching definition of the content of the slot.
+
+        During the transformation, *no operation should be deleted*.
+        The operation can only schedule its own deletion by returning the
+        appropriate `DeletionKind`. The deletion must be legal assuming the
+        blocking uses passed through the `newBlockingUses` list in
+        `canUseBeRemoved` have been removed.
+
+        After calling this method, the blocking uses should have disappeared
+        or this operation should have scheduled its own deletion.
+
+        This method will only be called after ensuring promotion is allowed via
+        `canUseBeRemoved`. The requested blocking use removal may or may not
+        have been done at the point of calling this method, but it will be done
+        eventually.
+
+        The builder is located after the promotable operation on call.
+      }],
+      "::mlir::DeletionKind",
+      "removeBlockingUses",
+      (ins "const ::mlir::MemorySlot &":$slot,
+           "const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &":$blockingUses,
+           "::mlir::OpBuilder &":$builder)
+    >,
+  ];
+}
+
+#endif // MLIR_INTERFACES_MEM2REGINTERFACES
diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
index b4d070a42d98f..ac71b73ae1e34 100644
--- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
+++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
@@ -114,12 +114,25 @@ class ValueBoundsConstraintSet {
   /// Compute a bound in terms of the values/dimensions in `dependencies`. The
   /// computed bound consists of only constant terms and dependent values (or
   /// dimension sizes thereof).
-  static LogicalResult computeBound(AffineMap &resultMap,
-                                    ValueDimList &mapOperands,
-                                    presburger::BoundType type, Value value,
-                                    std::optional<int64_t> dim,
-                                    ValueDimList dependencies,
-                                    bool closedUB = false);
+  static LogicalResult
+  computeDependentBound(AffineMap &resultMap, ValueDimList &mapOperands,
+                        presburger::BoundType type, Value value,
+                        std::optional<int64_t> dim, ValueDimList dependencies,
+                        bool closedUB = false);
+
+  /// Compute a bound in that is independent of all values in `independencies`.
+  ///
+  /// Independencies are the opposite of dependencies. The computed bound does
+  /// not contain any SSA values that are part of `independencies`. E.g., this
+  /// function can be used to make ops hoistable from loops. To that end, ops
+  /// must be made independent of loop induction variables (in the case of "for"
+  /// loops). Loop induction variables are the independencies; they may not
+  /// appear in the computed bound.
+  static LogicalResult
+  computeIndependentBound(AffineMap &resultMap, ValueDimList &mapOperands,
+                          presburger::BoundType type, Value value,
+                          std::optional<int64_t> dim, ValueRange independencies,
+                          bool closedUB = false);
 
   /// Compute a constant bound for the given index-typed value or shape
   /// dimension size.
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
index 27352bd971c65..ce2e09612fb1e 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.td
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -25,7 +25,7 @@ def VectorUnrollOpInterface : OpInterface<"VectorUnrollOpInterface"> {
     InterfaceMethod<
       /*desc=*/[{
         Return the shape ratio of unrolling to the target vector shape
-        `targetShape`. Return `None` if the op cannot be unrolled to the target
+        `targetShape`. Return `std::nullopt` if the op cannot be unrolled to the target
         vector shape.
       }],
       /*retTy=*/"::std::optional<::llvm::SmallVector<int64_t, 4>>",
diff --git a/mlir/include/mlir/Rewrite/PatternApplicator.h b/mlir/include/mlir/Rewrite/PatternApplicator.h
index 0bdc00c363d20..f7871f819a273 100644
--- a/mlir/include/mlir/Rewrite/PatternApplicator.h
+++ b/mlir/include/mlir/Rewrite/PatternApplicator.h
@@ -37,8 +37,7 @@ class ApplyPatternAction : public tracing::ActionImpl<ApplyPatternAction> {
       "Encapsulate the application of rewrite patterns";
 
   void print(raw_ostream &os) const override {
-    os << "`" << tag << "`\n"
-       << " pattern: " << pattern.getDebugName() << '\n';
+    os << "`" << tag << " pattern: " << pattern.getDebugName();
   }
 
 private:
diff --git a/mlir/include/mlir/TableGen/Argument.h b/mlir/include/mlir/TableGen/Argument.h
index 5a8f7882c00d6..a2c129b4f28d8 100644
--- a/mlir/include/mlir/TableGen/Argument.h
+++ b/mlir/include/mlir/TableGen/Argument.h
@@ -22,6 +22,7 @@
 #define MLIR_TABLEGEN_ARGUMENT_H_
 
 #include "mlir/TableGen/Attribute.h"
+#include "mlir/TableGen/Property.h"
 #include "mlir/TableGen/Type.h"
 #include "llvm/ADT/PointerUnion.h"
 #include <string>
@@ -58,8 +59,9 @@ struct NamedTypeConstraint {
   TypeConstraint constraint;
 };
 
-// Operation argument: either attribute or operand
-using Argument = llvm::PointerUnion<NamedAttribute *, NamedTypeConstraint *>;
+// Operation argument: either attribute, property, or operand
+using Argument = llvm::PointerUnion<NamedAttribute *, NamedProperty *,
+                                    NamedTypeConstraint *>;
 
 } // namespace tblgen
 } // namespace mlir
diff --git a/mlir/include/mlir/TableGen/Class.h b/mlir/include/mlir/TableGen/Class.h
index 10dfc01961a0f..27b5d5c57e913 100644
--- a/mlir/include/mlir/TableGen/Class.h
+++ b/mlir/include/mlir/TableGen/Class.h
@@ -576,7 +576,12 @@ class ExtraClassDeclaration
 public:
   /// Create an extra class declaration.
   ExtraClassDeclaration(StringRef extraClassDeclaration,
-                        StringRef extraClassDefinition = "")
+                        std::string extraClassDefinition = "")
+      : ExtraClassDeclaration(extraClassDeclaration.str(),
+                              std::move(extraClassDefinition)) {}
+
+  ExtraClassDeclaration(std::string extraClassDeclaration,
+                        std::string extraClassDefinition = "")
       : extraClassDeclaration(extraClassDeclaration),
         extraClassDefinition(extraClassDefinition) {}
 
@@ -590,7 +595,7 @@ class ExtraClassDeclaration
 private:
   /// The string of the extra class declarations. It is re-indented before
   /// printed.
-  StringRef extraClassDeclaration;
+  std::string extraClassDeclaration;
   /// The string of the extra class definitions. It is re-indented before
   /// printed.
   std::string extraClassDefinition;
diff --git a/mlir/include/mlir/TableGen/Dialect.h b/mlir/include/mlir/TableGen/Dialect.h
index d85342c742e2e..5337bd3beb5f9 100644
--- a/mlir/include/mlir/TableGen/Dialect.h
+++ b/mlir/include/mlir/TableGen/Dialect.h
@@ -86,6 +86,10 @@ class Dialect {
   /// operations or types.
   bool isExtensible() const;
 
+  /// Default to use properties for storing Attributes for operations in this
+  /// dialect.
+  bool usePropertiesForAttributes() const;
+
   // Returns whether two dialects are equal by checking the equality of the
   // underlying record.
   bool operator==(const Dialect &other) const;
diff --git a/mlir/include/mlir/TableGen/Operator.h b/mlir/include/mlir/TableGen/Operator.h
index 88dff220d6378..c0d483addeb48 100644
--- a/mlir/include/mlir/TableGen/Operator.h
+++ b/mlir/include/mlir/TableGen/Operator.h
@@ -18,6 +18,7 @@
 #include "mlir/TableGen/Attribute.h"
 #include "mlir/TableGen/Builder.h"
 #include "mlir/TableGen/Dialect.h"
+#include "mlir/TableGen/Property.h"
 #include "mlir/TableGen/Region.h"
 #include "mlir/TableGen/Successor.h"
 #include "mlir/TableGen/Trait.h"
@@ -166,10 +167,14 @@ class Operator {
   unsigned getNumVariableLengthResults() const;
 
   /// Op attribute iterators.
-  using attribute_iterator = const NamedAttribute *;
-  attribute_iterator attribute_begin() const;
-  attribute_iterator attribute_end() const;
-  llvm::iterator_range<attribute_iterator> getAttributes() const;
+  using const_attribute_iterator = const NamedAttribute *;
+  const_attribute_iterator attribute_begin() const;
+  const_attribute_iterator attribute_end() const;
+  llvm::iterator_range<const_attribute_iterator> getAttributes() const;
+  using attribute_iterator = NamedAttribute *;
+  attribute_iterator attribute_begin();
+  attribute_iterator attribute_end();
+  llvm::iterator_range<attribute_iterator> getAttributes();
 
   int getNumAttributes() const { return attributes.size(); }
   int getNumNativeAttributes() const { return numNativeAttributes; }
@@ -185,6 +190,27 @@ class Operator {
   const_value_iterator operand_end() const;
   const_value_range getOperands() const;
 
+  // Op properties iterators.
+  using const_property_iterator = const NamedProperty *;
+  const_property_iterator properties_begin() const {
+    return properties.begin();
+  }
+  const_property_iterator properties_end() const { return properties.end(); }
+  llvm::iterator_range<const_property_iterator> getProperties() const {
+    return properties;
+  }
+  using property_iterator = NamedProperty *;
+  property_iterator properties_begin() { return properties.begin(); }
+  property_iterator properties_end() { return properties.end(); }
+  llvm::iterator_range<property_iterator> getProperties() { return properties; }
+  int getNumCoreAttributes() const { return properties.size(); }
+
+  // Op properties accessors.
+  NamedProperty &getProperty(int index) { return properties[index]; }
+  const NamedProperty &getProperty(int index) const {
+    return properties[index];
+  }
+
   int getNumOperands() const { return operands.size(); }
   NamedTypeConstraint &getOperand(int index) { return operands[index]; }
   const NamedTypeConstraint &getOperand(int index) const {
@@ -353,6 +379,9 @@ class Operator {
   /// computed upon request).
   SmallVector<NamedAttribute, 4> attributes;
 
+  /// The properties of the op.
+  SmallVector<NamedProperty> properties;
+
   /// The arguments of the op (operands and native attributes).
   SmallVector<Argument, 4> arguments;
 
diff --git a/mlir/include/mlir/TableGen/Property.h b/mlir/include/mlir/TableGen/Property.h
new file mode 100644
index 0000000000000..7118ca496de07
--- /dev/null
+++ b/mlir/include/mlir/TableGen/Property.h
@@ -0,0 +1,86 @@
+//===- Property.h - Property wrapper class --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Property wrapper to simplify using TableGen Record defining a MLIR
+// Property.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TABLEGEN_PROPERTY_H_
+#define MLIR_TABLEGEN_PROPERTY_H_
+
+#include "mlir/Support/LLVM.h"
+#include "mlir/TableGen/Constraint.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class DefInit;
+class Record;
+} // namespace llvm
+
+namespace mlir {
+namespace tblgen {
+class Dialect;
+class Type;
+
+// Wrapper class providing helper methods for accessing MLIR Property defined
+// in TableGen. This class should closely reflect what is defined as class
+// `Property` in TableGen.
+class Property {
+public:
+  explicit Property(const llvm::Record *record);
+  explicit Property(const llvm::DefInit *init);
+
+  // Returns the storage type.
+  StringRef getStorageType() const;
+
+  // Returns the interface type for this property.
+  StringRef getInterfaceType() const;
+
+  // Returns the template getter method call which reads this property's
+  // storage and returns the value as of the desired return type.
+  StringRef getConvertFromStorageCall() const;
+
+  // Returns the template setter method call which reads this property's
+  // in the provided interface type and assign it to the storage.
+  StringRef getAssignToStorageCall() const;
+
+  // Returns the conversion method call which reads this property's
+  // in the storage type and builds an attribute.
+  StringRef getConvertToAttributeCall() const;
+
+  // Returns the setter method call which reads this property's
+  // in the provided interface type and assign it to the storage.
+  StringRef getConvertFromAttributeCall() const;
+
+  // Returns the code to compute the hash for this property.
+  StringRef getHashPropertyCall() const;
+
+  // Returns whether this Property has a default value.
+  bool hasDefaultValue() const;
+  // Returns the default value for this Property.
+  StringRef getDefaultValue() const;
+
+  // Returns the TableGen definition this Property was constructed from.
+  const llvm::Record &getDef() const;
+
+private:
+  // The TableGen definition of this constraint.
+  const llvm::Record *def;
+};
+
+// A struct wrapping an op property and its name together
+struct NamedProperty {
+  llvm::StringRef name;
+  Property prop;
+};
+
+} // namespace tblgen
+} // namespace mlir
+
+#endif // MLIR_TABLEGEN_PROPERTY_H_
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index a04e285af580e..9e3e05eec25ee 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -162,13 +162,10 @@ class ModuleTranslation {
 
   /// Returns the OpenMP IR builder associated with the LLVM IR module being
   /// constructed.
-  llvm::OpenMPIRBuilder *getOpenMPBuilder() {
-    if (!ompBuilder) {
-      ompBuilder = std::make_unique<llvm::OpenMPIRBuilder>(*llvmModule);
-      ompBuilder->initialize();
-    }
-    return ompBuilder.get();
-  }
+  llvm::OpenMPIRBuilder *getOpenMPBuilder();
+
+  /// Returns the LLVM module in which the IR is being constructed.
+  llvm::Module *getLLVMModule() { return llvmModule.get(); }
 
   /// Translates the given location.
   const llvm::DILocation *translateLoc(Location loc, llvm::DILocalScope *scope);
diff --git a/mlir/include/mlir/Tools/PDLL/AST/Nodes.h b/mlir/include/mlir/Tools/PDLL/AST/Nodes.h
index 0a34f8085321b..5e86cda27a5b7 100644
--- a/mlir/include/mlir/Tools/PDLL/AST/Nodes.h
+++ b/mlir/include/mlir/Tools/PDLL/AST/Nodes.h
@@ -1052,7 +1052,8 @@ class PatternDecl : public Node::NodeBase<PatternDecl, Decl> {
       : Base(loc, name), benefit(benefit),
         hasBoundedRecursion(hasBoundedRecursion), patternBody(body) {}
 
-  /// The benefit of the pattern if it was explicitly specified, None otherwise.
+  /// The benefit of the pattern if it was explicitly specified, std::nullopt
+  /// otherwise.
   std::optional<uint16_t> benefit;
 
   /// If the pattern has properly bounded rewrite recursion or not.
diff --git a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h
index c6e5906565e01..8f1969cd3ad36 100644
--- a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h
+++ b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h
@@ -13,7 +13,7 @@
 #ifndef MLIR_TOOLS_MLIROPT_MLIROPTMAIN_H
 #define MLIR_TOOLS_MLIROPT_MLIROPTMAIN_H
 
-#include "mlir/Debug/BreakpointManagers/FileLineColLocBreakpointManager.h"
+#include "mlir/Debug/CLOptionsSetup.h"
 #include "mlir/Support/LogicalResult.h"
 #include "llvm/ADT/StringRef.h"
 
@@ -30,9 +30,6 @@ namespace mlir {
 class DialectRegistry;
 class PassPipelineCLParser;
 class PassManager;
-namespace tracing {
-class FileLineColLocBreakpointManager;
-}
 
 /// Configuration options for the mlir-opt tool.
 /// This is intended to help building tools like mlir-opt by collecting the
@@ -64,6 +61,14 @@ class MlirOptMainConfig {
     return allowUnregisteredDialectsFlag;
   }
 
+  /// Set the debug configuration to use.
+  MlirOptMainConfig &setDebugConfig(tracing::DebugConfig config) {
+    debugConfig = std::move(config);
+    return *this;
+  }
+  tracing::DebugConfig &getDebugConfig() { return debugConfig; }
+  const tracing::DebugConfig &getDebugConfig() const { return debugConfig; }
+
   /// Print the pass-pipeline as text before executing.
   MlirOptMainConfig &dumpPassPipeline(bool dump) {
     dumpPassPipelineFlag = dump;
@@ -85,24 +90,13 @@ class MlirOptMainConfig {
   }
   StringRef getIrdlFile() const { return irdlFileFlag; }
 
-  /// Set the filename to use for logging actions, use "-" for stdout.
-  MlirOptMainConfig &logActionsTo(StringRef filename) {
-    logActionsToFlag = filename;
+  /// Set the bytecode version to emit.
+  MlirOptMainConfig &setEmitBytecodeVersion(int64_t version) {
+    emitBytecodeVersion = version;
     return *this;
   }
-  /// Get the filename to use for logging actions.
-  StringRef getLogActionsTo() const { return logActionsToFlag; }
-
-  /// Set a location breakpoint manager to filter out action logging based on
-  /// the attached IR location in the Action context. Ownership stays with the
-  /// caller.
-  void addLogActionLocFilter(tracing::BreakpointManager *breakpointManager) {
-    logActionLocationFilter.push_back(breakpointManager);
-  }
-
-  /// Get the location breakpoint managers to use to filter out action logging.
-  ArrayRef<tracing::BreakpointManager *> getLogActionsLocFilters() const {
-    return logActionLocationFilter;
+  std::optional<int64_t> bytecodeVersionToEmit() const {
+    return emitBytecodeVersion;
   }
 
   /// Set the callback to populate the pass manager.
@@ -122,14 +116,15 @@ class MlirOptMainConfig {
     return success();
   }
 
-  // Deprecated.
-  MlirOptMainConfig &preloadDialectsInContext(bool preload) {
-    preloadDialectsInContextFlag = preload;
+  /// Enable running the reproducer information stored in resources (if
+  /// present).
+  MlirOptMainConfig &runReproducer(bool enableReproducer) {
+    runReproducerFlag = enableReproducer;
     return *this;
-  }
-  bool shouldPreloadDialectsInContext() const {
-    return preloadDialectsInContextFlag;
-  }
+  };
+
+  /// Return true if the reproducer should be run.
+  bool shouldRunReproducer() const { return runReproducerFlag; }
 
   /// Show the registered dialects before trying to load the input file.
   MlirOptMainConfig &showDialects(bool show) {
@@ -174,26 +169,32 @@ class MlirOptMainConfig {
   /// general.
   bool allowUnregisteredDialectsFlag = false;
 
+  /// Configuration for the debugging hooks.
+  tracing::DebugConfig debugConfig;
+
   /// Print the pipeline that will be run.
   bool dumpPassPipelineFlag = false;
 
   /// Emit bytecode instead of textual assembly when generating output.
   bool emitBytecodeFlag = false;
 
+  /// Enable the Debugger action hook: Debugger can intercept MLIR Actions.
+  bool enableDebuggerActionHookFlag = false;
+
   /// IRDL file to register before processing the input.
   std::string irdlFileFlag = "";
 
-  /// Log action execution to the given file (or "-" for stdout)
-  std::string logActionsToFlag;
-
   /// Location Breakpoints to filter the action logging.
   std::vector<tracing::BreakpointManager *> logActionLocationFilter;
 
+  /// Emit bytecode at given version.
+  std::optional<int64_t> emitBytecodeVersion = std::nullopt;
+
   /// The callback to populate the pass manager.
   std::function<LogicalResult(PassManager &)> passPipelineCallback;
 
-  /// Deprecated.
-  bool preloadDialectsInContextFlag = false;
+  /// Enable running the reproducer.
+  bool runReproducerFlag = false;
 
   /// Show the registered dialects before trying to load the input file.
   bool showDialectsFlag = false;
@@ -228,36 +229,11 @@ LogicalResult MlirOptMain(llvm::raw_ostream &outputStream,
                           DialectRegistry &registry,
                           const MlirOptMainConfig &config);
 
-/// Perform the core processing behind `mlir-opt`.
-/// This API is deprecated, use the MlirOptMainConfig version above instead.
-LogicalResult MlirOptMain(llvm::raw_ostream &outputStream,
-                          std::unique_ptr<llvm::MemoryBuffer> buffer,
-                          const PassPipelineCLParser &passPipeline,
-                          DialectRegistry &registry, bool splitInputFile,
-                          bool verifyDiagnostics, bool verifyPasses,
-                          bool allowUnregisteredDialects,
-                          bool preloadDialectsInContext = false,
-                          bool emitBytecode = false, bool explicitModule = true,
-                          bool dumpPassPipeline = false);
-
-/// Perform the core processing behind `mlir-opt`.
-/// This API is deprecated, use the MlirOptMainConfig version above instead.
-LogicalResult MlirOptMain(
-    llvm::raw_ostream &outputStream, std::unique_ptr<llvm::MemoryBuffer> buffer,
-    PassPipelineFn passManagerSetupFn, DialectRegistry &registry,
-    bool splitInputFile, bool verifyDiagnostics, bool verifyPasses,
-    bool allowUnregisteredDialects, bool preloadDialectsInContext = false,
-    bool emitBytecode = false, bool explicitModule = true);
-
 /// Implementation for tools like `mlir-opt`.
 /// - toolName is used for the header displayed by `--help`.
 /// - registry should contain all the dialects that can be parsed in the source.
-/// - preloadDialectsInContext will trigger the upfront loading of all
-///   dialects from the global registry in the MLIRContext. This option is
-///   deprecated and will be removed soon.
 LogicalResult MlirOptMain(int argc, char **argv, llvm::StringRef toolName,
-                          DialectRegistry &registry,
-                          bool preloadDialectsInContext = false);
+                          DialectRegistry &registry);
 
 /// Helper wrapper to return the result of MlirOptMain directly from main.
 ///
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 2821686655c2f..c31ac29a4cd84 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -526,9 +526,14 @@ class OpConversionPattern : public ConversionPattern {
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
-    return matchAndRewrite(cast<SourceOp>(op),
-                           OpAdaptor(operands, op->getAttrDictionary()),
-                           rewriter);
+    auto sourceOp = cast<SourceOp>(op);
+    if constexpr (SourceOp::hasProperties())
+      return matchAndRewrite(sourceOp,
+                             OpAdaptor(operands, op->getAttrDictionary(),
+                                       sourceOp.getProperties()),
+                             rewriter);
+    return matchAndRewrite(
+        sourceOp, OpAdaptor(operands, op->getAttrDictionary()), rewriter);
   }
 
   /// Rewrite and Match methods that operate on the SourceOp type. These must be
diff --git a/mlir/include/mlir/Transforms/Mem2Reg.h b/mlir/include/mlir/Transforms/Mem2Reg.h
new file mode 100644
index 0000000000000..e2da88f7e00c7
--- /dev/null
+++ b/mlir/include/mlir/Transforms/Mem2Reg.h
@@ -0,0 +1,26 @@
+//===-- Mem2Reg.h - Mem2Reg definitions -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TRANSFORMS_MEM2REG_H
+#define MLIR_TRANSFORMS_MEM2REG_H
+
+#include "mlir/IR/Dominance.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Interfaces/Mem2RegInterfaces.h"
+
+namespace mlir {
+
+/// Attempts to promote the memory slots of the provided allocators. Succeeds if
+/// at least one memory slot was promoted.
+LogicalResult
+tryToPromoteMemorySlots(ArrayRef<PromotableAllocationOpInterface> allocators,
+                        OpBuilder &builder, DominanceInfo &dominance);
+
+} // namespace mlir
+
+#endif // MLIR_TRANSFORMS_MEM2REG_H
diff --git a/mlir/include/mlir/Transforms/OneToNTypeConversion.h b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
index 80e61e7a817ca..47bd276eed61f 100644
--- a/mlir/include/mlir/Transforms/OneToNTypeConversion.h
+++ b/mlir/include/mlir/Transforms/OneToNTypeConversion.h
@@ -225,13 +225,16 @@ class OneToNOpConversionPattern : public OneToNConversionPattern {
   public:
     using RangeT = ArrayRef<ValueRange>;
     using BaseT = typename SourceOp::template GenericAdaptor<RangeT>;
+    using Properties = typename SourceOp::template InferredProperties<SourceOp>;
 
     OpAdaptor(const OneToNTypeMapping *operandMapping,
               const OneToNTypeMapping *resultMapping,
               const ValueRange *convertedOperands, RangeT values,
-              DictionaryAttr attrs = nullptr, RegionRange regions = {})
-        : BaseT(values, attrs, regions), operandMapping(operandMapping),
-          resultMapping(resultMapping), convertedOperands(convertedOperands) {}
+              DictionaryAttr attrs = nullptr, Properties &properties = {},
+              RegionRange regions = {})
+        : BaseT(values, attrs, properties, regions),
+          operandMapping(operandMapping), resultMapping(resultMapping),
+          convertedOperands(convertedOperands) {}
 
     /// Get the type mapping of the original operands to the converted operands.
     const OneToNTypeMapping &getOperandMapping() const {
@@ -271,7 +274,8 @@ class OneToNOpConversionPattern : public OneToNConversionPattern {
       valueRanges.push_back(values);
     }
     OpAdaptor adaptor(&operandMapping, &resultMapping, &convertedOperands,
-                      valueRanges, op->getAttrDictionary(), op->getRegions());
+                      valueRanges, op->getAttrDictionary(),
+                      cast<SourceOp>(op).getProperties(), op->getRegions());
 
     // Call overload implemented by the derived class.
     return matchAndRewrite(cast<SourceOp>(op), adaptor, rewriter);
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 1d95e256b9161..f5f76076c8e07 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -33,6 +33,7 @@ class GreedyRewriteConfig;
 #define GEN_PASS_DECL_CSEPASS
 #define GEN_PASS_DECL_INLINER
 #define GEN_PASS_DECL_LOOPINVARIANTCODEMOTION
+#define GEN_PASS_DECL_MEM2REG
 #define GEN_PASS_DECL_PRINTIRPASS
 #define GEN_PASS_DECL_PRINTOPSTATS
 #define GEN_PASS_DECL_STRIPDEBUGINFO
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index cecefb35466bc..1cc357ca1f9f4 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -172,6 +172,25 @@ def LoopInvariantCodeMotion : Pass<"loop-invariant-code-motion"> {
   let constructor = "mlir::createLoopInvariantCodeMotionPass()";
 }
 
+def Mem2Reg : Pass<"mem2reg"> {
+  let summary = "Promotes memory slots into values.";
+  let description = [{
+    This pass removes loads out of and stores into a memory slot, and turns
+    them into direct uses of SSA values. This is done generically using the
+    `PromoteAllocationOpInterface`, `PromoteOpInterface` and
+    `PromoteMemOpInterface` interfaces.
+
+    This pass will attempt to compute which definitions of the content of
+    the memory slot reach operations that use the memory slot pointer. It
+    will rewire or remove operations that use the slot pointer so they no
+    longer use it. If any of this is not possible, the IR will be left
+    without mutation.
+
+    This pass only supports unstructured control-flow. Promotion of operations
+    within subregions will not happen.
+  }];
+}
+
 def PrintOpStats : Pass<"print-op-stats"> {
   let summary = "Print statistics of operations";
   let constructor = "mlir::createPrintOpStatsPass()";
diff --git a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
index b2d8bd9d113fb..348ffbf4d2c71 100644
--- a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
+++ b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
@@ -917,7 +917,7 @@ unsigned FlatLinearValueConstraints::insertVar(VarKind kind, unsigned pos,
   unsigned num = vals.size();
   unsigned absolutePos = IntegerPolyhedron::insertVar(kind, pos, num);
 
-  // If a Value is provided, insert it; otherwise use None.
+  // If a Value is provided, insert it; otherwise use std::nullopt.
   for (unsigned i = 0; i < num; ++i)
     values.insert(values.begin() + absolutePos + i,
                   vals[i] ? std::optional<Value>(vals[i]) : std::nullopt);
diff --git a/mlir/lib/Analysis/Presburger/Utils.cpp b/mlir/lib/Analysis/Presburger/Utils.cpp
index 89ccc4f263290..1d2404eeb59cf 100644
--- a/mlir/lib/Analysis/Presburger/Utils.cpp
+++ b/mlir/lib/Analysis/Presburger/Utils.cpp
@@ -437,6 +437,7 @@ void DivisionRepr::removeDuplicateDivs(
   // variable at position `i` only depends on local variables at position <
   // `i`. This would make sure that all divisions depending on other local
   // variables that can be merged, are merged.
+  normalizeDivs();
   for (unsigned i = 0; i < getNumDivs(); ++i) {
     // Check if a division representation exists for the `i^th` local var.
     if (denoms[i] == 0)
@@ -472,6 +473,14 @@ void DivisionRepr::removeDuplicateDivs(
   }
 }
 
+void DivisionRepr::normalizeDivs() {
+  for (unsigned i = 0, e = getNumDivs(); i < e; ++i) {
+    if (getDenom(i) == 0 || getDividend(i).empty())
+      continue;
+    normalizeDiv(getDividend(i), getDenom(i));
+  }
+}
+
 void DivisionRepr::insertDiv(unsigned pos, ArrayRef<MPInt> dividend,
                              const MPInt &divisor) {
   assert(pos <= getNumDivs() && "Invalid insertion position");
diff --git a/mlir/lib/AsmParser/Parser.cpp b/mlir/lib/AsmParser/Parser.cpp
index 46c1ae976d9d4..ade2465d7fd0a 100644
--- a/mlir/lib/AsmParser/Parser.cpp
+++ b/mlir/lib/AsmParser/Parser.cpp
@@ -540,6 +540,7 @@ class OperationParser : public Parser {
       std::optional<MutableArrayRef<std::unique_ptr<Region>>> parsedRegions =
           std::nullopt,
       std::optional<ArrayRef<NamedAttribute>> parsedAttributes = std::nullopt,
+      std::optional<Attribute> propertiesAttribute = std::nullopt,
       std::optional<FunctionType> parsedFnType = std::nullopt);
 
   /// Parse an operation instance that is in the generic form and insert it at
@@ -1075,7 +1076,8 @@ Value OperationParser::createForwardRefPlaceholder(SMLoc loc, Type type) {
   auto name = OperationName("builtin.unrealized_conversion_cast", getContext());
   auto *op = Operation::create(
       getEncodedSourceLocation(loc), name, type, /*operands=*/{},
-      /*attributes=*/std::nullopt, /*successors=*/{}, /*numRegions=*/0);
+      /*attributes=*/std::nullopt, /*properties=*/nullptr, /*successors=*/{},
+      /*numRegions=*/0);
   forwardRefPlaceholders[op->getResult(0)] = loc;
   return op->getResult(0);
 }
@@ -1255,6 +1257,7 @@ ParseResult OperationParser::parseGenericOperationAfterOpName(
     std::optional<ArrayRef<Block *>> parsedSuccessors,
     std::optional<MutableArrayRef<std::unique_ptr<Region>>> parsedRegions,
     std::optional<ArrayRef<NamedAttribute>> parsedAttributes,
+    std::optional<Attribute> propertiesAttribute,
     std::optional<FunctionType> parsedFnType) {
 
   // Parse the operand list, if not explicitly provided.
@@ -1284,6 +1287,16 @@ ParseResult OperationParser::parseGenericOperationAfterOpName(
     result.addSuccessors(*parsedSuccessors);
   }
 
+  // Parse the properties, if not explicitly provided.
+  if (propertiesAttribute) {
+    result.propertiesAttr = *propertiesAttribute;
+  } else if (consumeIf(Token::less)) {
+    result.propertiesAttr = parseAttribute();
+    if (!result.propertiesAttr)
+      return failure();
+    if (parseToken(Token::greater, "expected '>' to close properties"))
+      return failure();
+  }
   // Parse the region list, if not explicitly provided.
   if (!parsedRegions) {
     if (consumeIf(Token::l_paren)) {
@@ -1390,10 +1403,53 @@ Operation *OperationParser::parseGenericOperation() {
   if (parseGenericOperationAfterOpName(result))
     return nullptr;
 
+  // Operation::create() is not allowed to fail, however setting the properties
+  // from an attribute is a failable operation. So we save the attribute here
+  // and set it on the operation post-parsing.
+  Attribute properties;
+  std::swap(properties, result.propertiesAttr);
+
+  // If we don't have properties in the textual IR, but the operation now has
+  // support for properties, we support some backward-compatible generic syntax
+  // for the operation and as such we accept inherent attributes mixed in the
+  // dictionary of discardable attributes. We pre-validate these here because
+  // invalid attributes can't be casted to the properties storage and will be
+  // silently dropped. For example an attribute { foo = 0 : i32 } that is
+  // declared as F32Attr in ODS would have a C++ type of FloatAttr in the
+  // properties array. When setting it we would do something like:
+  //
+  //   properties.foo = dyn_cast<FloatAttr>(fooAttr);
+  //
+  // which would end up with a null Attribute. The diagnostic from the verifier
+  // would be "missing foo attribute" instead of something like "expects a 32
+  // bits float attribute but got a 32 bits integer attribute".
+  if (!properties && !result.getRawProperties()) {
+    std::optional<RegisteredOperationName> info =
+        result.name.getRegisteredInfo();
+    if (info) {
+      if (failed(info->verifyInherentAttrs(result.attributes, [&]() {
+            return mlir::emitError(srcLocation) << "'" << name << "' op ";
+          })))
+        return nullptr;
+    }
+  }
+
   // Create the operation and try to parse a location for it.
   Operation *op = opBuilder.create(result);
   if (parseTrailingLocationSpecifier(op))
     return nullptr;
+
+  // Try setting the properties for the operation, using a diagnostic to print
+  // errors.
+  if (properties) {
+    InFlightDiagnostic diagnostic =
+        mlir::emitError(srcLocation, "invalid properties ")
+        << properties << " for op " << name << ": ";
+    if (failed(op->setPropertiesFromAttribute(properties, &diagnostic)))
+      return nullptr;
+    diagnostic.abandon();
+  }
+
   return op;
 }
 
@@ -1461,10 +1517,11 @@ class CustomOpAsmParser : public AsmParserImpl<OpAsmParser> {
       std::optional<ArrayRef<Block *>> parsedSuccessors,
       std::optional<MutableArrayRef<std::unique_ptr<Region>>> parsedRegions,
       std::optional<ArrayRef<NamedAttribute>> parsedAttributes,
+      std::optional<Attribute> parsedPropertiesAttribute,
       std::optional<FunctionType> parsedFnType) final {
     return parser.parseGenericOperationAfterOpName(
         result, parsedUnresolvedOperands, parsedSuccessors, parsedRegions,
-        parsedAttributes, parsedFnType);
+        parsedAttributes, parsedPropertiesAttribute, parsedFnType);
   }
   //===--------------------------------------------------------------------===//
   // Utilities
@@ -1933,10 +1990,23 @@ OperationParser::parseCustomOperation(ArrayRef<ResultRecord> resultIDs) {
   if (opAsmParser.didEmitError())
     return nullptr;
 
+  Attribute properties = opState.propertiesAttr;
+  opState.propertiesAttr = Attribute{};
+
   // Otherwise, create the operation and try to parse a location for it.
   Operation *op = opBuilder.create(opState);
   if (parseTrailingLocationSpecifier(op))
     return nullptr;
+
+  // Try setting the properties for the operation.
+  if (properties) {
+    InFlightDiagnostic diagnostic =
+        mlir::emitError(srcLocation, "invalid properties ")
+        << properties << " for op " << op->getName().getStringRef() << ": ";
+    if (failed(op->setPropertiesFromAttribute(properties, &diagnostic)))
+      return nullptr;
+    diagnostic.abandon();
+  }
   return op;
 }
 
diff --git a/mlir/lib/Bindings/Python/IRAttributes.cpp b/mlir/lib/Bindings/Python/IRAttributes.cpp
index 5e7138b21c752..22001957ffa29 100644
--- a/mlir/lib/Bindings/Python/IRAttributes.cpp
+++ b/mlir/lib/Bindings/Python/IRAttributes.cpp
@@ -710,6 +710,10 @@ class PyDenseElementsAttribute
       // f16
       return bufferInfo<uint16_t>(shapedType, "e");
     }
+    if (mlirTypeIsAIndex(elementType)) {
+      // Same as IndexType::kInternalStorageBitWidth
+      return bufferInfo<int64_t>(shapedType);
+    }
     if (mlirTypeIsAInteger(elementType) &&
         mlirIntegerTypeGetWidth(elementType) == 32) {
       if (mlirIntegerTypeIsSignless(elementType) ||
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index f3fd386779373..f2e188e78c4a7 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -13,11 +13,10 @@
 
 #include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir-c/BuiltinAttributes.h"
-#include "mlir-c/BuiltinTypes.h"
 #include "mlir-c/Debug.h"
 #include "mlir-c/Diagnostics.h"
 #include "mlir-c/IR.h"
-//#include "mlir-c/Registration.h"
+#include "mlir-c/Support.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 
@@ -126,6 +125,9 @@ static const char kOperationPrintBytecodeDocstring[] =
 
 Args:
   file: The file like object to write to.
+  desired_version: The version of bytecode to emit.
+Returns:
+  The bytecode writer status.
 )";
 
 static const char kOperationStrDunderDocstring[] =
@@ -154,6 +156,11 @@ position in the argument list. If the value is an operation result, this is
 equivalent to printing the operation that produced it.
 )";
 
+static const char kValueReplaceAllUsesWithDocstring[] =
+    R"(Replace all uses of value with the new value, updating anything in
+the IR that uses 'self' to use the other value instead.
+)";
+
 //------------------------------------------------------------------------------
 // Utilities.
 //------------------------------------------------------------------------------
@@ -1128,12 +1135,24 @@ void PyOperationBase::print(py::object fileObject, bool binary,
   mlirOpPrintingFlagsDestroy(flags);
 }
 
-void PyOperationBase::writeBytecode(const py::object &fileObject) {
+void PyOperationBase::writeBytecode(const py::object &fileObject,
+                                    std::optional<int64_t> bytecodeVersion) {
   PyOperation &operation = getOperation();
   operation.checkValid();
   PyFileAccumulator accum(fileObject, /*binary=*/true);
-  mlirOperationWriteBytecode(operation, accum.getCallback(),
-                             accum.getUserData());
+
+  if (!bytecodeVersion.has_value())
+    return mlirOperationWriteBytecode(operation, accum.getCallback(),
+                                      accum.getUserData());
+
+  MlirBytecodeWriterConfig config = mlirBytecodeWriterConfigCreate();
+  mlirBytecodeWriterConfigDesiredEmitVersion(config, *bytecodeVersion);
+  MlirLogicalResult res = mlirOperationWriteBytecodeWithConfig(
+      operation, config, accum.getCallback(), accum.getUserData());
+  if (mlirLogicalResultIsFailure(res))
+    throw py::value_error((Twine("Unable to honor desired bytecode version ") +
+                           Twine(*bytecodeVersion))
+                              .str());
 }
 
 py::object PyOperationBase::getAsm(bool binary,
@@ -2754,6 +2773,7 @@ void mlir::python::populateIRCore(py::module &m) {
            py::arg("use_local_scope") = false,
            py::arg("assume_verified") = false, kOperationPrintDocstring)
       .def("write_bytecode", &PyOperationBase::writeBytecode, py::arg("file"),
+           py::arg("desired_version") = py::none(),
            kOperationPrintBytecodeDocstring)
       .def("get_asm", &PyOperationBase::getAsm,
            // Careful: Lots of arguments must match up with get_asm method.
@@ -3316,10 +3336,18 @@ void mlir::python::populateIRCore(py::module &m) {
             return printAccum.join();
           },
           kValueDunderStrDocstring)
-      .def_property_readonly("type", [](PyValue &self) {
-        return PyType(self.getParentOperation()->getContext(),
-                      mlirValueGetType(self.get()));
-      });
+      .def_property_readonly("type",
+                             [](PyValue &self) {
+                               return PyType(
+                                   self.getParentOperation()->getContext(),
+                                   mlirValueGetType(self.get()));
+                             })
+      .def(
+          "replace_all_uses_with",
+          [](PyValue &self, PyValue &with) {
+            mlirValueReplaceAllUsesOfWith(self.get(), with.get());
+          },
+          kValueReplaceAllUsesWithDocstring);
   PyBlockArgument::bind(m);
   PyOpResult::bind(m);
   PyOpOperand::bind(m);
diff --git a/mlir/lib/Bindings/Python/IRInterfaces.cpp b/mlir/lib/Bindings/Python/IRInterfaces.cpp
index c8371dcc7229e..766d6f3e4793e 100644
--- a/mlir/lib/Bindings/Python/IRInterfaces.cpp
+++ b/mlir/lib/Bindings/Python/IRInterfaces.cpp
@@ -187,7 +187,7 @@ class PyInferTypeOpInterface
   /// return types. Throws value_error on failure.
   std::vector<PyType>
   inferReturnTypes(std::optional<py::list> operandList,
-                   std::optional<PyAttribute> attributes,
+                   std::optional<PyAttribute> attributes, void *properties,
                    std::optional<std::vector<PyRegion>> regions,
                    DefaultingPyMlirContext context,
                    DefaultingPyLocation location) {
@@ -255,7 +255,7 @@ class PyInferTypeOpInterface
 
     MlirLogicalResult result = mlirInferTypeOpInterfaceInferReturnTypes(
         opNameRef, pyContext.get(), location.resolve(), mlirOperands.size(),
-        mlirOperands.data(), attributeDict, mlirRegions.size(),
+        mlirOperands.data(), attributeDict, properties, mlirRegions.size(),
         mlirRegions.data(), &appendResultsCallback, &data);
 
     if (mlirLogicalResultIsFailure(result)) {
@@ -268,7 +268,8 @@ class PyInferTypeOpInterface
   static void bindDerived(ClassTy &cls) {
     cls.def("inferReturnTypes", &PyInferTypeOpInterface::inferReturnTypes,
             py::arg("operands") = py::none(),
-            py::arg("attributes") = py::none(), py::arg("regions") = py::none(),
+            py::arg("attributes") = py::none(),
+            py::arg("properties") = py::none(), py::arg("regions") = py::none(),
             py::arg("context") = py::none(), py::arg("loc") = py::none(),
             inferReturnTypesDoc);
   }
diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h
index 920e6f467c5bc..ade790ba0ed13 100644
--- a/mlir/lib/Bindings/Python/IRModule.h
+++ b/mlir/lib/Bindings/Python/IRModule.h
@@ -554,7 +554,8 @@ class PyOperationBase {
                           bool assumeVerified);
 
   // Implement the bound 'writeBytecode' method.
-  void writeBytecode(const pybind11::object &fileObject);
+  void writeBytecode(const pybind11::object &fileObject,
+                     std::optional<int64_t> bytecodeVersion);
 
   /// Moves the operation before or after the other operation.
   void moveAfter(PyOperationBase &other);
diff --git a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
index 8433f3d84b852..801f3022d0e47 100644
--- a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
+++ b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
@@ -27,6 +27,10 @@ using namespace mlir::bytecode::detail;
 struct BytecodeWriterConfig::Impl {
   Impl(StringRef producer) : producer(producer) {}
 
+  /// Version to use when writing.
+  /// Note: This only differs from kVersion if a specific version is set.
+  int64_t bytecodeVersion = bytecode::kVersion;
+
   /// The producer of the bytecode.
   StringRef producer;
 
@@ -48,6 +52,12 @@ void BytecodeWriterConfig::attachResourcePrinter(
   impl->externalResourcePrinters.emplace_back(std::move(printer));
 }
 
+void BytecodeWriterConfig::setDesiredBytecodeVersion(int64_t bytecodeVersion) {
+  // Clamp to current version.
+  impl->bytecodeVersion =
+      std::min<int64_t>(bytecodeVersion, bytecode::kVersion);
+}
+
 //===----------------------------------------------------------------------===//
 // EncodingEmitter
 //===----------------------------------------------------------------------===//
@@ -295,7 +305,8 @@ class StringSectionBuilder {
 
 class DialectWriter : public DialectBytecodeWriter {
 public:
-  DialectWriter(EncodingEmitter &emitter, IRNumberingState &numberingState,
+  DialectWriter(int64_t bytecodeVersion, EncodingEmitter &emitter,
+                IRNumberingState &numberingState,
                 StringSectionBuilder &stringSection)
       : emitter(emitter), numberingState(numberingState),
         stringSection(stringSection) {}
@@ -362,7 +373,10 @@ class DialectWriter : public DialectBytecodeWriter {
         reinterpret_cast<const uint8_t *>(blob.data()), blob.size()));
   }
 
+  int64_t getBytecodeVersion() const override { return bytecodeVersion; }
+
 private:
+  int64_t bytecodeVersion;
   EncodingEmitter &emitter;
   IRNumberingState &numberingState;
   StringSectionBuilder &stringSection;
@@ -421,11 +435,11 @@ void EncodingEmitter::emitMultiByteVarInt(uint64_t value) {
 namespace {
 class BytecodeWriter {
 public:
-  BytecodeWriter(Operation *op) : numberingState(op) {}
+  BytecodeWriter(Operation *op, const BytecodeWriterConfig::Impl &config)
+      : numberingState(op), config(config) {}
 
   /// Write the bytecode for the given root operation.
-  void write(Operation *rootOp, raw_ostream &os,
-             const BytecodeWriterConfig::Impl &config);
+  void write(Operation *rootOp, raw_ostream &os);
 
 private:
   //===--------------------------------------------------------------------===//
@@ -449,8 +463,7 @@ class BytecodeWriter {
   //===--------------------------------------------------------------------===//
   // Resources
 
-  void writeResourceSection(Operation *op, EncodingEmitter &emitter,
-                            const BytecodeWriterConfig::Impl &config);
+  void writeResourceSection(Operation *op, EncodingEmitter &emitter);
 
   //===--------------------------------------------------------------------===//
   // Strings
@@ -465,11 +478,13 @@ class BytecodeWriter {
 
   /// The IR numbering state generated for the root operation.
   IRNumberingState numberingState;
+
+  /// Configuration dictating bytecode emission.
+  const BytecodeWriterConfig::Impl &config;
 };
 } // namespace
 
-void BytecodeWriter::write(Operation *rootOp, raw_ostream &os,
-                           const BytecodeWriterConfig::Impl &config) {
+void BytecodeWriter::write(Operation *rootOp, raw_ostream &os) {
   EncodingEmitter emitter;
 
   // Emit the bytecode file header. This is how we identify the output as a
@@ -477,7 +492,7 @@ void BytecodeWriter::write(Operation *rootOp, raw_ostream &os,
   emitter.emitString("ML\xefR");
 
   // Emit the bytecode version.
-  emitter.emitVarInt(bytecode::kVersion);
+  emitter.emitVarInt(config.bytecodeVersion);
 
   // Emit the producer.
   emitter.emitNulTerminatedString(config.producer);
@@ -492,7 +507,7 @@ void BytecodeWriter::write(Operation *rootOp, raw_ostream &os,
   writeIRSection(emitter, rootOp);
 
   // Emit the resources section.
-  writeResourceSection(rootOp, emitter, config);
+  writeResourceSection(rootOp, emitter);
 
   // Emit the string section.
   writeStringSection(emitter);
@@ -540,12 +555,17 @@ void BytecodeWriter::writeDialectSection(EncodingEmitter &emitter) {
     // Write the string section and get the ID.
     size_t nameID = stringSection.insert(dialect.name);
 
+    if (config.bytecodeVersion == 0) {
+      dialectEmitter.emitVarInt(nameID);
+      continue;
+    }
+
     // Try writing the version to the versionEmitter.
     EncodingEmitter versionEmitter;
     if (dialect.interface) {
       // The writer used when emitting using a custom bytecode encoding.
-      DialectWriter versionWriter(versionEmitter, numberingState,
-                                  stringSection);
+      DialectWriter versionWriter(config.bytecodeVersion, versionEmitter,
+                                  numberingState, stringSection);
       dialect.interface->writeVersion(versionWriter);
     }
 
@@ -586,8 +606,8 @@ void BytecodeWriter::writeAttrTypeSection(EncodingEmitter &emitter) {
     bool hasCustomEncoding = false;
     if (const BytecodeDialectInterface *interface = entry.dialect->interface) {
       // The writer used when emitting using a custom bytecode encoding.
-      DialectWriter dialectWriter(attrTypeEmitter, numberingState,
-                                  stringSection);
+      DialectWriter dialectWriter(config.bytecodeVersion, attrTypeEmitter,
+                                  numberingState, stringSection);
 
       if constexpr (std::is_same_v<std::decay_t<decltype(entryValue)>, Type>) {
         // TODO: We don't currently support custom encoded mutable types.
@@ -787,9 +807,8 @@ class ResourceBuilder : public AsmResourceBuilder {
 };
 } // namespace
 
-void BytecodeWriter::writeResourceSection(
-    Operation *op, EncodingEmitter &emitter,
-    const BytecodeWriterConfig::Impl &config) {
+void BytecodeWriter::writeResourceSection(Operation *op,
+                                          EncodingEmitter &emitter) {
   EncodingEmitter resourceEmitter;
   EncodingEmitter resourceOffsetEmitter;
   uint64_t prevOffset = 0;
@@ -868,8 +887,10 @@ void BytecodeWriter::writeStringSection(EncodingEmitter &emitter) {
 // Entry Points
 //===----------------------------------------------------------------------===//
 
-void mlir::writeBytecodeToFile(Operation *op, raw_ostream &os,
-                               const BytecodeWriterConfig &config) {
-  BytecodeWriter writer(op);
-  writer.write(op, os, config.getImpl());
+LogicalResult mlir::writeBytecodeToFile(Operation *op, raw_ostream &os,
+                                        const BytecodeWriterConfig &config) {
+  BytecodeWriter writer(op, config.getImpl());
+  writer.write(op, os);
+  // Currently there is no failure case.
+  return success();
 }
diff --git a/mlir/lib/Bytecode/Writer/IRNumbering.cpp b/mlir/lib/Bytecode/Writer/IRNumbering.cpp
index 06778b9c7c75e..7f56e9a94d299 100644
--- a/mlir/lib/Bytecode/Writer/IRNumbering.cpp
+++ b/mlir/lib/Bytecode/Writer/IRNumbering.cpp
@@ -7,11 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "IRNumbering.h"
+#include "../Encoding.h"
 #include "mlir/Bytecode/BytecodeImplementation.h"
 #include "mlir/Bytecode/BytecodeWriter.h"
 #include "mlir/IR/AsmState.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
+#include "llvm/Support/ErrorHandling.h"
 
 using namespace mlir;
 using namespace mlir::bytecode::detail;
@@ -41,6 +43,10 @@ struct IRNumberingState::NumberingDialectWriter : public DialectBytecodeWriter {
   }
   void writeOwnedBlob(ArrayRef<char> blob) override {}
 
+  int64_t getBytecodeVersion() const override {
+    llvm_unreachable("unexpected querying of version in IRNumbering");
+  }
+
   /// The parent numbering state that is populated by this writer.
   IRNumberingState &state;
 };
diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp
index 051559acd440c..6ed32e1ce8656 100644
--- a/mlir/lib/CAPI/IR/IR.cpp
+++ b/mlir/lib/CAPI/IR/IR.cpp
@@ -145,6 +145,23 @@ void mlirOpPrintingFlagsAssumeVerified(MlirOpPrintingFlags flags) {
   unwrap(flags)->assumeVerified();
 }
 
+//===----------------------------------------------------------------------===//
+// Bytecode printing flags API.
+//===----------------------------------------------------------------------===//
+
+MlirBytecodeWriterConfig mlirBytecodeWriterConfigCreate() {
+  return wrap(new BytecodeWriterConfig());
+}
+
+void mlirBytecodeWriterConfigDestroy(MlirBytecodeWriterConfig config) {
+  delete unwrap(config);
+}
+
+void mlirBytecodeWriterConfigDesiredEmitVersion(MlirBytecodeWriterConfig flags,
+                                                int64_t version) {
+  unwrap(flags)->setDesiredBytecodeVersion(version);
+}
+
 //===----------------------------------------------------------------------===//
 // Location API.
 //===----------------------------------------------------------------------===//
@@ -323,7 +340,8 @@ static LogicalResult inferOperationTypes(OperationState &state) {
 
   if (succeeded(inferInterface->inferReturnTypes(
           context, state.location, state.operands,
-          state.attributes.getDictionary(context), state.regions, state.types)))
+          state.attributes.getDictionary(context), state.getRawProperties(),
+          state.regions, state.types)))
     return success();
 
   // Diagnostic emitted by interface.
@@ -510,7 +528,15 @@ void mlirOperationPrintWithFlags(MlirOperation op, MlirOpPrintingFlags flags,
 void mlirOperationWriteBytecode(MlirOperation op, MlirStringCallback callback,
                                 void *userData) {
   detail::CallbackOstream stream(callback, userData);
-  writeBytecodeToFile(unwrap(op), stream);
+  // As no desired version is set, no failure can occur.
+  (void)writeBytecodeToFile(unwrap(op), stream);
+}
+
+MlirLogicalResult mlirOperationWriteBytecodeWithConfig(
+    MlirOperation op, MlirBytecodeWriterConfig config,
+    MlirStringCallback callback, void *userData) {
+  detail::CallbackOstream stream(callback, userData);
+  return wrap(writeBytecodeToFile(unwrap(op), stream, *unwrap(config)));
 }
 
 void mlirOperationDump(MlirOperation op) { return unwrap(op)->dump(); }
@@ -751,6 +777,10 @@ MlirOpOperand mlirValueGetFirstUse(MlirValue value) {
   return wrap(opOperand);
 }
 
+void mlirValueReplaceAllUsesOfWith(MlirValue oldValue, MlirValue newValue) {
+  unwrap(oldValue).replaceAllUsesWith(unwrap(newValue));
+}
+
 //===----------------------------------------------------------------------===//
 // OpOperand API.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/CAPI/Interfaces/Interfaces.cpp b/mlir/lib/CAPI/Interfaces/Interfaces.cpp
index 5adccbdaf06b5..029feed3a3593 100644
--- a/mlir/lib/CAPI/Interfaces/Interfaces.cpp
+++ b/mlir/lib/CAPI/Interfaces/Interfaces.cpp
@@ -1,3 +1,5 @@
+
+
 //===- Interfaces.cpp - C Interface for MLIR Interfaces -------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
@@ -39,8 +41,8 @@ MlirTypeID mlirInferTypeOpInterfaceTypeID() {
 MlirLogicalResult mlirInferTypeOpInterfaceInferReturnTypes(
     MlirStringRef opName, MlirContext context, MlirLocation location,
     intptr_t nOperands, MlirValue *operands, MlirAttribute attributes,
-    intptr_t nRegions, MlirRegion *regions, MlirTypesCallback callback,
-    void *userData) {
+    void *properties, intptr_t nRegions, MlirRegion *regions,
+    MlirTypesCallback callback, void *userData) {
   StringRef name(opName.data, opName.length);
   std::optional<RegisteredOperationName> info =
       RegisteredOperationName::lookup(name, unwrap(context));
@@ -72,7 +74,7 @@ MlirLogicalResult mlirInferTypeOpInterfaceInferReturnTypes(
   SmallVector<Type> inferredTypes;
   if (failed(info->getInterface<InferTypeOpInterface>()->inferReturnTypes(
           unwrap(context), maybeLocation, unwrappedOperands, attributeDict,
-          unwrappedRegions, inferredTypes)))
+          properties, unwrappedRegions, inferredTypes)))
     return mlirLogicalResultFailure();
 
   SmallVector<MlirType> wrappedInferredTypes;
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index d4ee6e7e5d347..eeac499f49196 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -10,7 +10,7 @@
 
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
-#include "mlir/Dialect/AMDGPU/AMDGPUDialect.h"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Pass/Pass.h"
@@ -62,6 +62,14 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     else
       wantedDataType = gpuOp.getODSResults(0)[0].getType();
 
+    Value atomicCmpData = Value();
+    // Operand index 1 of a load is the indices, trying to read them can crash.
+    if (storeData) {
+      Value maybeCmpData = adaptor.getODSOperands(1)[0];
+      if (maybeCmpData != memref)
+        atomicCmpData = maybeCmpData;
+    }
+
     Type llvmWantedDataType = this->typeConverter->convertType(wantedDataType);
 
     Type i32 = rewriter.getI32Type();
@@ -73,8 +81,16 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
     // If we want to load a vector<NxT> with total size <= 32
     // bits, use a scalar load and bitcast it. Similarly, if bitsize(T) < 32
     // and the total load size is >= 32, use a vector load of N / (bitsize(T) /
-    // 32) x i32 and bitcast.
+    // 32) x i32 and bitcast. Also, the CAS intrinsic requires integer operands,
+    // so bitcast any floats to integers.
     Type llvmBufferValType = llvmWantedDataType;
+    if (atomicCmpData) {
+      if (wantedDataType.isa<VectorType>())
+        return gpuOp.emitOpError("vector compare-and-swap does not exist");
+      if (auto floatType = wantedDataType.dyn_cast<FloatType>())
+        llvmBufferValType = this->getTypeConverter()->convertType(
+            rewriter.getIntegerType(floatType.getWidth()));
+    }
     if (auto dataVector = wantedDataType.dyn_cast<VectorType>()) {
       uint32_t elemBits = dataVector.getElementTypeBitWidth();
       uint32_t totalBits = elemBits * dataVector.getNumElements();
@@ -109,6 +125,16 @@ struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
       }
     }
 
+    if (atomicCmpData) {
+      if (llvmBufferValType != llvmWantedDataType) {
+        Value castForCmp = rewriter.create<LLVM::BitcastOp>(
+            loc, llvmBufferValType, atomicCmpData);
+        args.push_back(castForCmp);
+      } else {
+        args.push_back(atomicCmpData);
+      }
+    }
+
     // Construct buffer descriptor from memref, attributes
     int64_t offset = 0;
     SmallVector<int64_t, 5> strides;
@@ -529,6 +555,8 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
       RawBufferOpLowering<RawBufferAtomicFmaxOp, ROCDL::RawBufferAtomicFMaxOp>,
       RawBufferOpLowering<RawBufferAtomicSmaxOp, ROCDL::RawBufferAtomicSMaxOp>,
       RawBufferOpLowering<RawBufferAtomicUminOp, ROCDL::RawBufferAtomicUMinOp>,
+      RawBufferOpLowering<RawBufferAtomicCmpswapOp,
+                          ROCDL::RawBufferAtomicCmpSwap>,
       MFMAOpLowering>(converter, chipset);
 }
 
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/CMakeLists.txt b/mlir/lib/Conversion/AMDGPUToROCDL/CMakeLists.txt
index f574b4c55dd83..43cbc0732516d 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/CMakeLists.txt
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_mlir_conversion_library(MLIRAMDGPUToROCDL
   AMDGPUToROCDL.cpp
-  Chipset.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/AMDGPUToROCDL
@@ -16,6 +15,7 @@ add_mlir_conversion_library(MLIRAMDGPUToROCDL
   MLIRLLVMDialect
   MLIRROCDLDialect
   MLIRAMDGPUDialect
+  MLIRAMDGPUUtils
   MLIRPass
   MLIRTransforms
   )
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 82c73b5f4dd2e..a13acc691ae35 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -11,7 +11,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 
 using namespace mlir;
@@ -24,7 +24,7 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   SmallVector<LLVM::GlobalOp, 3> workgroupBuffers;
   workgroupBuffers.reserve(gpuFuncOp.getNumWorkgroupAttributions());
   for (const auto &en : llvm::enumerate(gpuFuncOp.getWorkgroupAttributions())) {
-    Value attribution = en.value();
+    BlockArgument attribution = en.value();
 
     auto type = attribution.getType().dyn_cast<MemRefType>();
     assert(type && type.hasStaticShape() && "unexpected type in attribution");
@@ -36,10 +36,17 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
     auto arrayType = LLVM::LLVMArrayType::get(elementType, numElements);
     std::string name = std::string(
         llvm::formatv("__wg_{0}_{1}", gpuFuncOp.getName(), en.index()));
+    uint64_t alignment = 0;
+    if (auto alignAttr =
+            gpuFuncOp
+                .getWorkgroupAttributionAttr(
+                    en.index(), LLVM::LLVMDialect::getAlignAttrName())
+                .dyn_cast_or_null<IntegerAttr>())
+      alignment = alignAttr.getInt();
     auto globalOp = rewriter.create<LLVM::GlobalOp>(
         gpuFuncOp.getLoc(), arrayType, /*isConstant=*/false,
-        LLVM::Linkage::Internal, name, /*value=*/Attribute(),
-        /*alignment=*/0, workgroupAddrSpace);
+        LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
+        workgroupAddrSpace);
     workgroupBuffers.push_back(globalOp);
   }
 
@@ -56,7 +63,10 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
   for (const auto &attr : gpuFuncOp->getAttrs()) {
     if (attr.getName() == SymbolTable::getSymbolAttrName() ||
         attr.getName() == gpuFuncOp.getFunctionTypeAttrName() ||
-        attr.getName() == gpu::GPUFuncOp::getNumWorkgroupAttributionsAttrName())
+        attr.getName() ==
+            gpu::GPUFuncOp::getNumWorkgroupAttributionsAttrName() ||
+        attr.getName() == gpuFuncOp.getWorkgroupAttribAttrsAttrName() ||
+        attr.getName() == gpuFuncOp.getPrivateAttribAttrsAttrName())
       continue;
     attributes.push_back(attr);
   }
@@ -124,9 +134,15 @@ GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
           getTypeConverter()->getPointerType(elementType, allocaAddrSpace);
       Value numElements = rewriter.create<LLVM::ConstantOp>(
           gpuFuncOp.getLoc(), int64Ty, type.getNumElements());
+      uint64_t alignment = 0;
+      if (auto alignAttr =
+              gpuFuncOp
+                  .getPrivateAttributionAttr(
+                      en.index(), LLVM::LLVMDialect::getAlignAttrName())
+                  .dyn_cast_or_null<IntegerAttr>())
+        alignment = alignAttr.getInt();
       Value allocated = rewriter.create<LLVM::AllocaOp>(
-          gpuFuncOp.getLoc(), ptrType, elementType, numElements,
-          /*alignment=*/0);
+          gpuFuncOp.getLoc(), ptrType, elementType, numElements, alignment);
       auto descr = MemRefDescriptor::fromStaticShape(
           rewriter, loc, *getTypeConverter(), type, allocated);
       signatureConversion.remapInput(
@@ -470,8 +486,7 @@ LogicalResult impl::scalarizeVectorOp(Operation *op, ValueRange operands,
         return operand;
       return rewriter.create<LLVM::ExtractElementOp>(loc, operand, index);
     };
-    auto scalarOperands =
-        llvm::to_vector(llvm::map_range(operands, extractElement));
+    auto scalarOperands = llvm::map_to_vector(operands, extractElement);
     Operation *scalarOp =
         rewriter.create(loc, name, scalarOperands, elementType, op->getAttrs());
     rewriter.create<LLVM::InsertElementOp>(loc, result, scalarOp->getResult(0),
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index 248aab3d6838b..2373765dae007 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -199,6 +199,28 @@ LLVM::LLVMPointerType MemRefDescriptor::getElementPtrType() {
       .cast<LLVM::LLVMPointerType>();
 }
 
+Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
+                                  LLVMTypeConverter &converter,
+                                  MemRefType type) {
+  // When we convert to LLVM, the input memref must have been normalized
+  // beforehand. Hence, this call is guaranteed to work.
+  auto [strides, offsetCst] = getStridesAndOffset(type);
+
+  Value ptr = alignedPtr(builder, loc);
+  // Skip if offset is zero.
+  if (offsetCst != 0) {
+    Type indexType = converter.getIndexType();
+    Value offsetVal =
+        ShapedType::isDynamic(offsetCst)
+            ? offset(builder, loc)
+            : createIndexAttrConstant(builder, loc, indexType, offsetCst);
+    Type elementType = converter.convertType(type.getElementType());
+    ptr = builder.create<LLVM::GEPOp>(loc, ptr.getType(), elementType, ptr,
+                                      offsetVal);
+  }
+  return ptr;
+}
+
 /// Creates a MemRef descriptor structure from a list of individual values
 /// composing that descriptor, in the following order:
 /// - allocated pointer;
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index e2dae4044504b..67a2898c02050 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -72,14 +72,14 @@ Value ConvertToLLVMPattern::getStridedElementPtr(
   auto [strides, offset] = getStridesAndOffset(type);
 
   MemRefDescriptor memRefDescriptor(memRefDesc);
-  Value base = memRefDescriptor.alignedPtr(rewriter, loc);
+  // Use a canonical representation of the start address so that later
+  // optimizations have a longer sequence of instructions to CSE.
+  // If we don't do that we would sprinkle the memref.offset in various
+  // position of the different address computations.
+  Value base =
+      memRefDescriptor.bufferPtr(rewriter, loc, *getTypeConverter(), type);
 
   Value index;
-  if (offset != 0) // Skip if offset is zero.
-    index = ShapedType::isDynamic(offset)
-                ? memRefDescriptor.offset(rewriter, loc)
-                : createIndexConstant(rewriter, loc, offset);
-
   for (int i = 0, e = indices.size(); i < e; ++i) {
     Value increment = indices[i];
     if (strides[i] != 1) { // Skip if stride is 1.
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 07d406bbb1c1a..e9fbad30783c3 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -332,6 +332,8 @@ struct AssumeAlignmentOpLowering
     : public ConvertOpToLLVMPattern<memref::AssumeAlignmentOp> {
   using ConvertOpToLLVMPattern<
       memref::AssumeAlignmentOp>::ConvertOpToLLVMPattern;
+  explicit AssumeAlignmentOpLowering(LLVMTypeConverter &converter)
+      : ConvertOpToLLVMPattern<memref::AssumeAlignmentOp>(converter) {}
 
   LogicalResult
   matchAndRewrite(memref::AssumeAlignmentOp op, OpAdaptor adaptor,
@@ -340,17 +342,16 @@ struct AssumeAlignmentOpLowering
     unsigned alignment = op.getAlignment();
     auto loc = op.getLoc();
 
-    MemRefDescriptor memRefDescriptor(memref);
-    Value ptr = memRefDescriptor.alignedPtr(rewriter, memref.getLoc());
+    auto srcMemRefType = op.getMemref().getType().cast<MemRefType>();
+    Value ptr = getStridedElementPtr(loc, srcMemRefType, memref, /*indices=*/{},
+                                     rewriter);
 
-    // Emit llvm.assume(memref.alignedPtr & (alignment - 1) == 0). Notice that
-    // the asserted memref.alignedPtr isn't used anywhere else, as the real
-    // users like load/store/views always re-extract memref.alignedPtr as they
-    // get lowered.
+    // Emit llvm.assume(memref & (alignment - 1) == 0).
     //
     // This relies on LLVM's CSE optimization (potentially after SROA), since
-    // after CSE all memref.alignedPtr instances get de-duplicated into the same
+    // after CSE all memref instances should get de-duplicated into the same
     // pointer SSA value.
+    MemRefDescriptor memRefDescriptor(memref);
     auto intPtrType =
         getIntPtrType(memRefDescriptor.getElementPtrType().getAddressSpace());
     Value zero = createIndexAttrConstant(rewriter, loc, intPtrType, 0);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
index 820cadbf62b56..c1b6d1c60c73b 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
@@ -81,6 +81,7 @@ void mlir::tosa::addTosaToLinalgPasses(OpPassManager &pm,
     pm.addNestedPass<func::FuncOp>(tosa::createTosaOptionalDecompositions());
   pm.addNestedPass<func::FuncOp>(createCanonicalizerPass());
 
+  pm.addNestedPass<func::FuncOp>(tosa::createTosaInferShapesPass());
   pm.addNestedPass<func::FuncOp>(tosa::createTosaMakeBroadcastablePass());
   pm.addNestedPass<func::FuncOp>(tosa::createTosaToLinalgNamed());
   pm.addNestedPass<func::FuncOp>(createCanonicalizerPass());
diff --git a/mlir/lib/Debug/CLOptionsSetup.cpp b/mlir/lib/Debug/CLOptionsSetup.cpp
new file mode 100644
index 0000000000000..24d0bd8808261
--- /dev/null
+++ b/mlir/lib/Debug/CLOptionsSetup.cpp
@@ -0,0 +1,120 @@
+//===- CLOptionsSetup.cpp - Helpers to setup debug CL options ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Debug/CLOptionsSetup.h"
+
+#include "mlir/Debug/Counter.h"
+#include "mlir/Debug/DebuggerExecutionContextHook.h"
+#include "mlir/Debug/ExecutionContext.h"
+#include "mlir/Debug/Observers/ActionLogging.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/Support/FileUtilities.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+using namespace mlir;
+using namespace mlir::tracing;
+using namespace llvm;
+
+namespace {
+struct DebugConfigCLOptions : public DebugConfig {
+  DebugConfigCLOptions() {
+    static cl::opt<std::string, /*ExternalStorage=*/true> logActionsTo{
+        "log-actions-to",
+        cl::desc("Log action execution to a file, or stderr if "
+                 " '-' is passed"),
+        cl::location(logActionsToFlag)};
+
+    static cl::list<std::string> logActionLocationFilter(
+        "log-mlir-actions-filter",
+        cl::desc(
+            "Comma separated list of locations to filter actions from logging"),
+        cl::CommaSeparated,
+        cl::cb<void, std::string>([&](const std::string &location) {
+          static bool register_once = [&] {
+            addLogActionLocFilter(&locBreakpointManager);
+            return true;
+          }();
+          (void)register_once;
+          static std::vector<std::string> locations;
+          locations.push_back(location);
+          StringRef locStr = locations.back();
+
+          // Parse the individual location filters and set the breakpoints.
+          auto diag = [](Twine msg) { llvm::errs() << msg << "\n"; };
+          auto locBreakpoint =
+              tracing::FileLineColLocBreakpoint::parseFromString(locStr, diag);
+          if (failed(locBreakpoint)) {
+            llvm::errs() << "Invalid location filter: " << locStr << "\n";
+            exit(1);
+          }
+          auto [file, line, col] = *locBreakpoint;
+          locBreakpointManager.addBreakpoint(file, line, col);
+        }));
+  }
+  tracing::FileLineColLocBreakpointManager locBreakpointManager;
+};
+
+} // namespace
+
+static ManagedStatic<DebugConfigCLOptions> clOptionsConfig;
+void DebugConfig::registerCLOptions() { *clOptionsConfig; }
+
+DebugConfig DebugConfig::createFromCLOptions() { return *clOptionsConfig; }
+
+class InstallDebugHandler::Impl {
+public:
+  Impl(MLIRContext &context, const DebugConfig &config) {
+    if (config.getLogActionsTo().empty() &&
+        !config.isDebuggerActionHookEnabled()) {
+      if (tracing::DebugCounter::isActivated())
+        context.registerActionHandler(tracing::DebugCounter());
+      return;
+    }
+    errs() << "ExecutionContext registered on the context";
+    if (tracing::DebugCounter::isActivated())
+      emitError(UnknownLoc::get(&context),
+                "Debug counters are incompatible with --log-actions-to and "
+                "--mlir-enable-debugger-hook options and are disabled");
+    if (!config.getLogActionsTo().empty()) {
+      std::string errorMessage;
+      logActionsFile = openOutputFile(config.getLogActionsTo(), &errorMessage);
+      if (!logActionsFile) {
+        emitError(UnknownLoc::get(&context),
+                  "Opening file for --log-actions-to failed: ")
+            << errorMessage << "\n";
+        return;
+      }
+      logActionsFile->keep();
+      raw_fd_ostream &logActionsStream = logActionsFile->os();
+      actionLogger = std::make_unique<tracing::ActionLogger>(logActionsStream);
+      for (const auto *locationBreakpoint : config.getLogActionsLocFilters())
+        actionLogger->addBreakpointManager(locationBreakpoint);
+      executionContext.registerObserver(actionLogger.get());
+    }
+    if (config.isDebuggerActionHookEnabled()) {
+      errs() << " (with Debugger hook)";
+      setupDebuggerExecutionContextHook(executionContext);
+    }
+    errs() << "\n";
+    context.registerActionHandler(executionContext);
+  }
+
+private:
+  std::unique_ptr<ToolOutputFile> logActionsFile;
+  tracing::ExecutionContext executionContext;
+  std::unique_ptr<tracing::ActionLogger> actionLogger;
+  std::vector<std::unique_ptr<tracing::FileLineColLocBreakpoint>>
+      locationBreakpoints;
+};
+
+InstallDebugHandler::InstallDebugHandler(MLIRContext &context,
+                                         const DebugConfig &config)
+    : impl(std::make_unique<Impl>(context, config)) {}
+
+InstallDebugHandler::~InstallDebugHandler() = default;
diff --git a/mlir/lib/Debug/CMakeLists.txt b/mlir/lib/Debug/CMakeLists.txt
index e4b844a9aad4d..d270eb2e7534f 100644
--- a/mlir/lib/Debug/CMakeLists.txt
+++ b/mlir/lib/Debug/CMakeLists.txt
@@ -1,9 +1,11 @@
 add_subdirectory(Observers)
 
 add_mlir_library(MLIRDebug
+  CLOptionsSetup.cpp
   DebugCounter.cpp
   ExecutionContext.cpp
   BreakpointManagers/FileLineColLocBreakpointManager.cpp
+  DebuggerExecutionContextHook.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Debug
@@ -12,5 +14,6 @@ add_mlir_library(MLIRDebug
   ${LLVM_PTHREAD_LIB}
   MLIRIR
   MLIRSupport
+  MLIRObservers
 )
 
diff --git a/mlir/lib/Debug/DebugCounter.cpp b/mlir/lib/Debug/DebugCounter.cpp
index 263fc8c80ebb7..e026a982859a9 100644
--- a/mlir/lib/Debug/DebugCounter.cpp
+++ b/mlir/lib/Debug/DebugCounter.cpp
@@ -110,10 +110,8 @@ void DebugCounter::print(raw_ostream &os) const {
 /// various flags within the DebugCounter. These flags are used when
 /// constructing a DebugCounter for initialization.
 void DebugCounter::registerCLOptions() {
-#ifndef NDEBUG
   // Make sure that the options struct has been initialized.
   *clOptions;
-#endif
 }
 
 bool DebugCounter::isActivated() {
diff --git a/mlir/lib/Debug/DebuggerExecutionContextHook.cpp b/mlir/lib/Debug/DebuggerExecutionContextHook.cpp
new file mode 100644
index 0000000000000..6cbd7f380c07f
--- /dev/null
+++ b/mlir/lib/Debug/DebuggerExecutionContextHook.cpp
@@ -0,0 +1,369 @@
+//===- DebuggerExecutionContextHook.cpp - Debugger Support ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Debug/DebuggerExecutionContextHook.h"
+
+#include "mlir/Debug/BreakpointManagers/FileLineColLocBreakpointManager.h"
+#include "mlir/Debug/BreakpointManagers/TagBreakpointManager.h"
+
+using namespace mlir;
+using namespace mlir::tracing;
+
+namespace {
+/// This structure tracks the state of the interactive debugger.
+struct DebuggerState {
+  /// This variable keeps track of the current control option. This is set by
+  /// the debugger when control is handed over to it.
+  ExecutionContext::Control debuggerControl = ExecutionContext::Apply;
+
+  /// The breakpoint manager that allows the debugger to set breakpoints on
+  /// action tags.
+  TagBreakpointManager tagBreakpointManager;
+
+  /// The breakpoint manager that allows the debugger to set breakpoints on
+  /// FileLineColLoc locations.
+  FileLineColLocBreakpointManager fileLineColLocBreakpointManager;
+
+  /// Map of breakpoint IDs to breakpoint objects.
+  DenseMap<unsigned, Breakpoint *> breakpointIdsMap;
+
+  /// The current stack of actiive actions.
+  const tracing::ActionActiveStack *actionActiveStack;
+
+  /// This is a "cursor" in the IR, it is used for the debugger to navigate the
+  /// IR associated to the actions.
+  IRUnit cursor;
+};
+} // namespace
+
+static DebuggerState &getGlobalDebuggerState() {
+  static LLVM_THREAD_LOCAL DebuggerState debuggerState;
+  return debuggerState;
+}
+
+extern "C" {
+void mlirDebuggerSetControl(int controlOption) {
+  getGlobalDebuggerState().debuggerControl =
+      static_cast<ExecutionContext::Control>(controlOption);
+}
+
+void mlirDebuggerPrintContext() {
+  DebuggerState &state = getGlobalDebuggerState();
+  if (!state.actionActiveStack) {
+    llvm::outs() << "No active action.\n";
+    return;
+  }
+  const ArrayRef<IRUnit> &units =
+      state.actionActiveStack->getAction().getContextIRUnits();
+  llvm::outs() << units.size() << " available IRUnits:\n";
+  for (const IRUnit &unit : units) {
+    llvm::outs() << "  - ";
+    unit.print(
+        llvm::outs(),
+        OpPrintingFlags().useLocalScope().skipRegions().enableDebugInfo());
+    llvm::outs() << "\n";
+  }
+}
+
+void mlirDebuggerPrintActionBacktrace(bool withContext) {
+  DebuggerState &state = getGlobalDebuggerState();
+  if (!state.actionActiveStack) {
+    llvm::outs() << "No active action.\n";
+    return;
+  }
+  state.actionActiveStack->print(llvm::outs(), withContext);
+}
+
+//===----------------------------------------------------------------------===//
+// Cursor Management
+//===----------------------------------------------------------------------===//
+
+void mlirDebuggerCursorPrint(bool withRegion) {
+  auto &state = getGlobalDebuggerState();
+  if (!state.cursor) {
+    llvm::outs() << "No active MLIR cursor, select from the context first\n";
+    return;
+  }
+  state.cursor.print(llvm::outs(), OpPrintingFlags()
+                                       .skipRegions(!withRegion)
+                                       .useLocalScope()
+                                       .enableDebugInfo());
+  llvm::outs() << "\n";
+}
+
+void mlirDebuggerCursorSelectIRUnitFromContext(int index) {
+  auto &state = getGlobalDebuggerState();
+  if (!state.actionActiveStack) {
+    llvm::outs() << "No active MLIR Action stack\n";
+    return;
+  }
+  ArrayRef<IRUnit> units =
+      state.actionActiveStack->getAction().getContextIRUnits();
+  if (index < 0 || index >= static_cast<int>(units.size())) {
+    llvm::outs() << "Index invalid, bounds: [0, " << units.size()
+                 << "] but got " << index << "\n";
+    return;
+  }
+  state.cursor = units[index];
+  state.cursor.print(llvm::outs());
+  llvm::outs() << "\n";
+}
+
+void mlirDebuggerCursorSelectParentIRUnit() {
+  auto &state = getGlobalDebuggerState();
+  if (!state.cursor) {
+    llvm::outs() << "No active MLIR cursor, select from the context first\n";
+    return;
+  }
+  IRUnit *unit = &state.cursor;
+  if (auto *op = unit->dyn_cast<Operation *>()) {
+    state.cursor = op->getBlock();
+  } else if (auto *region = unit->dyn_cast<Region *>()) {
+    state.cursor = region->getParentOp();
+  } else if (auto *block = unit->dyn_cast<Block *>()) {
+    state.cursor = block->getParent();
+  } else {
+    llvm::outs() << "Current cursor is not a valid IRUnit";
+    return;
+  }
+  state.cursor.print(llvm::outs());
+  llvm::outs() << "\n";
+}
+
+void mlirDebuggerCursorSelectChildIRUnit(int index) {
+  auto &state = getGlobalDebuggerState();
+  if (!state.cursor) {
+    llvm::outs() << "No active MLIR cursor, select from the context first\n";
+    return;
+  }
+  IRUnit *unit = &state.cursor;
+  if (auto *op = unit->dyn_cast<Operation *>()) {
+    if (index < 0 || index >= static_cast<int>(op->getNumRegions())) {
+      llvm::outs() << "Index invalid, op has " << op->getNumRegions()
+                   << " but got " << index << "\n";
+      return;
+    }
+    state.cursor = &op->getRegion(index);
+  } else if (auto *region = unit->dyn_cast<Region *>()) {
+    auto block = region->begin();
+    int count = 0;
+    while (block != region->end() && count != index) {
+      ++block;
+      ++count;
+    }
+
+    if (block == region->end()) {
+      llvm::outs() << "Index invalid, region has " << count << " block but got "
+                   << index << "\n";
+      return;
+    }
+    state.cursor = &*block;
+  } else if (auto *block = unit->dyn_cast<Block *>()) {
+    auto op = block->begin();
+    int count = 0;
+    while (op != block->end() && count != index) {
+      ++op;
+      ++count;
+    }
+
+    if (op == block->end()) {
+      llvm::outs() << "Index invalid, block has " << count
+                   << "operations but got " << index << "\n";
+      return;
+    }
+    state.cursor = &*op;
+  } else {
+    llvm::outs() << "Current cursor is not a valid IRUnit";
+    return;
+  }
+  state.cursor.print(llvm::outs());
+  llvm::outs() << "\n";
+}
+
+void mlirDebuggerCursorSelectPreviousIRUnit() {
+  auto &state = getGlobalDebuggerState();
+  if (!state.cursor) {
+    llvm::outs() << "No active MLIR cursor, select from the context first\n";
+    return;
+  }
+  IRUnit *unit = &state.cursor;
+  if (auto *op = unit->dyn_cast<Operation *>()) {
+    Operation *previous = op->getPrevNode();
+    if (!previous) {
+      llvm::outs() << "No previous operation in the current block\n";
+      return;
+    }
+    state.cursor = previous;
+  } else if (auto *region = unit->dyn_cast<Region *>()) {
+    llvm::outs() << "Has region\n";
+    Operation *parent = region->getParentOp();
+    if (!parent) {
+      llvm::outs() << "No parent operation for the current region\n";
+      return;
+    }
+    if (region->getRegionNumber() == 0) {
+      llvm::outs() << "No previous region in the current operation\n";
+      return;
+    }
+    state.cursor =
+        &region->getParentOp()->getRegion(region->getRegionNumber() - 1);
+  } else if (auto *block = unit->dyn_cast<Block *>()) {
+    Block *previous = block->getPrevNode();
+    if (!previous) {
+      llvm::outs() << "No previous block in the current region\n";
+      return;
+    }
+    state.cursor = previous;
+  } else {
+    llvm::outs() << "Current cursor is not a valid IRUnit";
+    return;
+  }
+  state.cursor.print(llvm::outs());
+  llvm::outs() << "\n";
+}
+
+void mlirDebuggerCursorSelectNextIRUnit() {
+  auto &state = getGlobalDebuggerState();
+  if (!state.cursor) {
+    llvm::outs() << "No active MLIR cursor, select from the context first\n";
+    return;
+  }
+  IRUnit *unit = &state.cursor;
+  if (auto *op = unit->dyn_cast<Operation *>()) {
+    Operation *next = op->getNextNode();
+    if (!next) {
+      llvm::outs() << "No next operation in the current block\n";
+      return;
+    }
+    state.cursor = next;
+  } else if (auto *region = unit->dyn_cast<Region *>()) {
+    Operation *parent = region->getParentOp();
+    if (!parent) {
+      llvm::outs() << "No parent operation for the current region\n";
+      return;
+    }
+    if (region->getRegionNumber() == parent->getNumRegions() - 1) {
+      llvm::outs() << "No next region in the current operation\n";
+      return;
+    }
+    state.cursor =
+        &region->getParentOp()->getRegion(region->getRegionNumber() + 1);
+  } else if (auto *block = unit->dyn_cast<Block *>()) {
+    Block *next = block->getNextNode();
+    if (!next) {
+      llvm::outs() << "No next block in the current region\n";
+      return;
+    }
+    state.cursor = next;
+  } else {
+    llvm::outs() << "Current cursor is not a valid IRUnit";
+    return;
+  }
+  state.cursor.print(llvm::outs());
+  llvm::outs() << "\n";
+}
+
+//===----------------------------------------------------------------------===//
+// Breakpoint Management
+//===----------------------------------------------------------------------===//
+
+void mlirDebuggerEnableBreakpoint(BreakpointHandle breakpoint) {
+  reinterpret_cast<Breakpoint *>(breakpoint)->enable();
+}
+
+void mlirDebuggerDisableBreakpoint(BreakpointHandle breakpoint) {
+  reinterpret_cast<Breakpoint *>(breakpoint)->disable();
+}
+
+BreakpointHandle mlirDebuggerAddTagBreakpoint(const char *tag) {
+  DebuggerState &state = getGlobalDebuggerState();
+  Breakpoint *breakpoint =
+      state.tagBreakpointManager.addBreakpoint(StringRef(tag, strlen(tag)));
+  int breakpointId = state.breakpointIdsMap.size() + 1;
+  state.breakpointIdsMap[breakpointId] = breakpoint;
+  return reinterpret_cast<BreakpointHandle>(breakpoint);
+}
+
+void mlirDebuggerAddRewritePatternBreakpoint(const char *patternNameInfo) {}
+
+void mlirDebuggerAddFileLineColLocBreakpoint(const char *file, int line,
+                                             int col) {
+  getGlobalDebuggerState().fileLineColLocBreakpointManager.addBreakpoint(
+      StringRef(file, strlen(file)), line, col);
+}
+
+} // extern "C"
+
+LLVM_ATTRIBUTE_NOINLINE void mlirDebuggerBreakpointHook() {
+  static LLVM_THREAD_LOCAL void *volatile sink;
+  sink = (void *)&sink;
+}
+
+static void preventLinkerDeadCodeElim() {
+  static void *volatile sink;
+  static bool initialized = [&]() {
+    sink = (void *)mlirDebuggerSetControl;
+    sink = (void *)mlirDebuggerEnableBreakpoint;
+    sink = (void *)mlirDebuggerDisableBreakpoint;
+    sink = (void *)mlirDebuggerPrintContext;
+    sink = (void *)mlirDebuggerPrintActionBacktrace;
+    sink = (void *)mlirDebuggerCursorPrint;
+    sink = (void *)mlirDebuggerCursorSelectIRUnitFromContext;
+    sink = (void *)mlirDebuggerCursorSelectParentIRUnit;
+    sink = (void *)mlirDebuggerCursorSelectChildIRUnit;
+    sink = (void *)mlirDebuggerCursorSelectPreviousIRUnit;
+    sink = (void *)mlirDebuggerCursorSelectNextIRUnit;
+    sink = (void *)mlirDebuggerAddTagBreakpoint;
+    sink = (void *)mlirDebuggerAddRewritePatternBreakpoint;
+    sink = (void *)mlirDebuggerAddFileLineColLocBreakpoint;
+    sink = (void *)&sink;
+    return true;
+  }();
+  (void)initialized;
+}
+
+static tracing::ExecutionContext::Control
+debuggerCallBackFunction(const tracing::ActionActiveStack *actionStack) {
+  preventLinkerDeadCodeElim();
+  // Invoke the breakpoint hook, the debugger is supposed to trap this.
+  // The debugger controls the execution from there by invoking
+  // `mlirDebuggerSetControl()`.
+  auto &state = getGlobalDebuggerState();
+  state.actionActiveStack = actionStack;
+  getGlobalDebuggerState().debuggerControl = ExecutionContext::Apply;
+  actionStack->getAction().print(llvm::outs());
+  llvm::outs() << "\n";
+  mlirDebuggerBreakpointHook();
+  return getGlobalDebuggerState().debuggerControl;
+}
+
+namespace {
+/// Manage the stack of actions that are currently active.
+class DebuggerObserver : public ExecutionContext::Observer {
+  void beforeExecute(const ActionActiveStack *action, Breakpoint *breakpoint,
+                     bool willExecute) override {
+    auto &state = getGlobalDebuggerState();
+    state.actionActiveStack = action;
+  }
+  void afterExecute(const ActionActiveStack *action) override {
+    auto &state = getGlobalDebuggerState();
+    state.actionActiveStack = action->getParent();
+    state.cursor = nullptr;
+  }
+};
+} // namespace
+
+void mlir::setupDebuggerExecutionContextHook(
+    tracing::ExecutionContext &executionContext) {
+  executionContext.setCallback(debuggerCallBackFunction);
+  DebuggerState &state = getGlobalDebuggerState();
+  static DebuggerObserver observer;
+  executionContext.registerObserver(&observer);
+  executionContext.addBreakpointManager(&state.fileLineColLocBreakpointManager);
+  executionContext.addBreakpointManager(&state.tagBreakpointManager);
+}
diff --git a/mlir/lib/Debug/ExecutionContext.cpp b/mlir/lib/Debug/ExecutionContext.cpp
index ee7c33a6b3f14..f7505b6608c81 100644
--- a/mlir/lib/Debug/ExecutionContext.cpp
+++ b/mlir/lib/Debug/ExecutionContext.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Debug/ExecutionContext.h"
 
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/FormatVariadic.h"
 
 #include <cstddef>
 
@@ -16,15 +17,39 @@ using namespace mlir;
 using namespace mlir::tracing;
 
 //===----------------------------------------------------------------------===//
-// ExecutionContext
+// ActionActiveStack
 //===----------------------------------------------------------------------===//
 
-static const thread_local ActionActiveStack *actionStack = nullptr;
-
-void ExecutionContext::setCallback(CallbackTy callback) {
-  onBreakpointControlExecutionCallback = callback;
+void ActionActiveStack::print(raw_ostream &os, bool withContext) const {
+  os << "ActionActiveStack depth " << getDepth() << "\n";
+  const ActionActiveStack *current = this;
+  int count = 0;
+  while (current) {
+    llvm::errs() << llvm::formatv("#{0,3}: ", count++);
+    current->action.print(llvm::errs());
+    llvm::errs() << "\n";
+    ArrayRef<IRUnit> context = current->action.getContextIRUnits();
+    if (withContext && !context.empty()) {
+      llvm::errs() << "Context:\n";
+      llvm::interleave(
+          current->action.getContextIRUnits(),
+          [&](const IRUnit &unit) {
+            llvm::errs() << "  - ";
+            unit.print(llvm::errs());
+          },
+          [&]() { llvm::errs() << "\n"; });
+      llvm::errs() << "\n";
+    }
+    current = current->parent;
+  }
 }
 
+//===----------------------------------------------------------------------===//
+// ExecutionContext
+//===----------------------------------------------------------------------===//
+
+static const LLVM_THREAD_LOCAL ActionActiveStack *actionStack = nullptr;
+
 void ExecutionContext::registerObserver(Observer *observer) {
   observers.push_back(observer);
 }
@@ -72,6 +97,7 @@ void ExecutionContext::operator()(llvm::function_ref<void()> transform,
     if (breakpoint)
       break;
   }
+  info.setBreakpoint(breakpoint);
 
   bool shouldExecuteAction = true;
   // If we have a breakpoint, or if `depthToBreak` was previously set and the
diff --git a/mlir/lib/Dialect/AMDGPU/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/CMakeLists.txt
index f33061b2d87cf..31167e6af908b 100644
--- a/mlir/lib/Dialect/AMDGPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/AMDGPU/CMakeLists.txt
@@ -1 +1,3 @@
 add_subdirectory(IR)
+add_subdirectory(Transforms)
+add_subdirectory(Utils)
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 9150e7368849d..cafa6a8fa2dd0 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/AMDGPU/AMDGPUDialect.h"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
@@ -30,16 +30,16 @@
 using namespace mlir;
 using namespace mlir::amdgpu;
 
-#include "mlir/Dialect/AMDGPU/AMDGPUDialect.cpp.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.cpp.inc"
 
 void AMDGPUDialect::initialize() {
   addOperations<
 #define GET_OP_LIST
-#include "mlir/Dialect/AMDGPU/AMDGPU.cpp.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPU.cpp.inc"
       >();
   addAttributes<
 #define GET_ATTRDEF_LIST
-#include "mlir/Dialect/AMDGPU/AMDGPUAttributes.cpp.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUAttributes.cpp.inc"
       >();
 }
 
@@ -90,6 +90,10 @@ LogicalResult RawBufferAtomicUminOp::verify() {
   return verifyRawBufferOp(*this);
 }
 
+LogicalResult RawBufferAtomicCmpswapOp::verify() {
+  return verifyRawBufferOp(*this);
+}
+
 static std::optional<uint32_t> getConstantUint32(Value v) {
   APInt cst;
   if (!v.getType().isInteger(32))
@@ -136,12 +140,11 @@ static bool staticallyOutOfBounds(OpType op) {
 }
 
 namespace {
-struct RemoveStaticallyOobBufferLoads final
-    : public OpRewritePattern<RawBufferLoadOp> {
-  using OpRewritePattern<RawBufferLoadOp>::OpRewritePattern;
+template <typename OpType>
+struct RemoveStaticallyOobBufferLoads final : public OpRewritePattern<OpType> {
+  using OpRewritePattern<OpType>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(RawBufferLoadOp op,
-                                PatternRewriter &rw) const override {
+  LogicalResult matchAndRewrite(OpType op, PatternRewriter &rw) const override {
     if (!staticallyOutOfBounds(op))
       return failure();
     Type loadType = op.getResult().getType();
@@ -167,7 +170,7 @@ struct RemoveStaticallyOobBufferWrites final : public OpRewritePattern<OpType> {
 
 void RawBufferLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                                   MLIRContext *context) {
-  results.add<RemoveStaticallyOobBufferLoads>(context);
+  results.add<RemoveStaticallyOobBufferLoads<RawBufferLoadOp>>(context);
 }
 
 void RawBufferStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
@@ -195,6 +198,12 @@ void RawBufferAtomicUminOp::getCanonicalizationPatterns(
   results.add<RemoveStaticallyOobBufferWrites<RawBufferAtomicUminOp>>(context);
 }
 
+void RawBufferAtomicCmpswapOp::getCanonicalizationPatterns(
+    RewritePatternSet &results, MLIRContext *context) {
+  results.add<RemoveStaticallyOobBufferLoads<RawBufferAtomicCmpswapOp>>(
+      context);
+}
+
 //===----------------------------------------------------------------------===//
 // MFMAOp
 //===----------------------------------------------------------------------===//
@@ -273,10 +282,10 @@ LogicalResult MFMAOp::verify() {
   return success();
 }
 
-#include "mlir/Dialect/AMDGPU/AMDGPUEnums.cpp.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.cpp.inc"
 
 #define GET_ATTRDEF_CLASSES
-#include "mlir/Dialect/AMDGPU/AMDGPUAttributes.cpp.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUAttributes.cpp.inc"
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/AMDGPU/AMDGPU.cpp.inc"
+#include "mlir/Dialect/AMDGPU/IR/AMDGPU.cpp.inc"
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..e11b6cc88bf22
--- /dev/null
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
@@ -0,0 +1,19 @@
+add_mlir_dialect_library(MLIRAMDGPUTransforms
+  EmulateAtomics.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  {$MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AMDGPU/Transforms
+
+  DEPENDS
+  MLIRAMDGPUTransformsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRAMDGPUDialect
+  MLIRAMDGPUUtils
+  MLIRArithDialect
+  MLIRControlFlowDialect
+  MLIRIR
+  MLIRPass
+  MLIRTransforms
+  MLIRTransformUtils
+  )
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
new file mode 100644
index 0000000000000..0c69cdc027912
--- /dev/null
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -0,0 +1,189 @@
+//===- EmulateAtomics.cpp - Emulate unsupported AMDGPU atomics ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
+
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace mlir::amdgpu {
+#define GEN_PASS_DEF_AMDGPUEMULATEATOMICSPASS
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
+} // namespace mlir::amdgpu
+
+using namespace mlir;
+using namespace mlir::amdgpu;
+
+namespace {
+struct AmdgpuEmulateAtomicsPass
+    : public amdgpu::impl::AmdgpuEmulateAtomicsPassBase<
+          AmdgpuEmulateAtomicsPass> {
+  using AmdgpuEmulateAtomicsPassBase<
+      AmdgpuEmulateAtomicsPass>::AmdgpuEmulateAtomicsPassBase;
+  void runOnOperation() override;
+};
+
+template <typename AtomicOp, typename ArithOp>
+struct RawBufferAtomicByCasPattern : public OpConversionPattern<AtomicOp> {
+  using OpConversionPattern<AtomicOp>::OpConversionPattern;
+  using Adaptor = typename AtomicOp::Adaptor;
+
+  LogicalResult
+  matchAndRewrite(AtomicOp atomicOp, Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+} // namespace
+
+namespace {
+enum class DataArgAction : unsigned char {
+  Duplicate,
+  Drop,
+};
+} // namespace
+
+// Fix up the fact that, when we're migrating from a general bugffer atomic
+// to a load or to a CAS, the number of openrands, and thus the number of
+// entries needed in operand_segment_sizes, needs to change. We use this method
+// because we'd like to preserve unknown attributes on the atomic instead of
+// discarding them.
+static void patchOperandSegmentSizes(ArrayRef<NamedAttribute> attrs,
+                                     SmallVectorImpl<NamedAttribute> &newAttrs,
+                                     DataArgAction action) {
+  newAttrs.reserve(attrs.size());
+  for (NamedAttribute attr : attrs) {
+    if (attr.getName().getValue() != "operand_segment_sizes") {
+      newAttrs.push_back(attr);
+      continue;
+    }
+    auto segmentAttr = attr.getValue().cast<DenseI32ArrayAttr>();
+    MLIRContext *context = segmentAttr.getContext();
+    DenseI32ArrayAttr newSegments;
+    switch (action) {
+    case DataArgAction::Drop:
+      newSegments = DenseI32ArrayAttr::get(
+          context, segmentAttr.asArrayRef().drop_front());
+      break;
+    case DataArgAction::Duplicate: {
+      SmallVector<int32_t> newVals;
+      ArrayRef<int32_t> oldVals = segmentAttr.asArrayRef();
+      newVals.push_back(oldVals[0]);
+      newVals.append(oldVals.begin(), oldVals.end());
+      newSegments = DenseI32ArrayAttr::get(context, newVals);
+      break;
+    }
+    }
+    newAttrs.push_back(NamedAttribute(attr.getName(), newSegments));
+  }
+}
+
+template <typename AtomicOp, typename ArithOp>
+LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
+    AtomicOp atomicOp, Adaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  Location loc = atomicOp.getLoc();
+
+  ArrayRef<NamedAttribute> origAttrs = atomicOp->getAttrs();
+  ValueRange operands = adaptor.getOperands();
+  Value data = operands.take_front()[0];
+  ValueRange invariantArgs = operands.drop_front();
+  Type dataType = data.getType();
+
+  SmallVector<NamedAttribute> loadAttrs;
+  patchOperandSegmentSizes(origAttrs, loadAttrs, DataArgAction::Drop);
+  Value initialLoad =
+      rewriter.create<RawBufferLoadOp>(loc, dataType, invariantArgs, loadAttrs);
+  Block *currentBlock = rewriter.getInsertionBlock();
+  Block *afterAtomic =
+      rewriter.splitBlock(currentBlock, rewriter.getInsertionPoint());
+  Block *loopBlock = rewriter.createBlock(afterAtomic, {dataType}, {loc});
+
+  rewriter.setInsertionPointToEnd(currentBlock);
+  rewriter.create<cf::BranchOp>(loc, loopBlock, initialLoad);
+
+  rewriter.setInsertionPointToEnd(loopBlock);
+  Value prevLoad = loopBlock->getArgument(0);
+  Value operated = rewriter.create<ArithOp>(loc, data, prevLoad);
+
+  SmallVector<NamedAttribute> cmpswapAttrs;
+  patchOperandSegmentSizes(origAttrs, cmpswapAttrs, DataArgAction::Duplicate);
+  SmallVector<Value> cmpswapArgs = {operated, prevLoad};
+  cmpswapArgs.append(invariantArgs.begin(), invariantArgs.end());
+  Value atomicRes = rewriter.create<RawBufferAtomicCmpswapOp>(
+      loc, dataType, cmpswapArgs, cmpswapAttrs);
+
+  // We care about exact bitwise equality here, so do some bitcasts.
+  // These will fold away during lowering to the ROCDL dialect, where
+  // an int->float bitcast is introduced to account for the fact that cmpswap
+  // only takes integer arguments.
+
+  Value prevLoadForCompare = prevLoad;
+  Value atomicResForCompare = atomicRes;
+  if (auto floatDataTy = dataType.dyn_cast<FloatType>()) {
+    Type equivInt = rewriter.getIntegerType(floatDataTy.getWidth());
+    prevLoadForCompare =
+        rewriter.create<arith::BitcastOp>(loc, equivInt, prevLoad);
+    atomicResForCompare =
+        rewriter.create<arith::BitcastOp>(loc, equivInt, atomicRes);
+  }
+  Value canLeave = rewriter.create<arith::CmpIOp>(
+      loc, arith::CmpIPredicate::eq, atomicResForCompare, prevLoadForCompare);
+  rewriter.create<cf::CondBranchOp>(loc, canLeave, afterAtomic, ValueRange{},
+                                    loopBlock, atomicRes);
+  rewriter.replaceOp(atomicOp, {});
+  return success();
+}
+
+void mlir::amdgpu::populateAmdgpuEmulateAtomicsPatterns(
+    ConversionTarget &target, RewritePatternSet &patterns, Chipset chipset) {
+  // gfx10 has no atomic adds.
+  if (chipset.majorVersion == 10 || chipset.majorVersion < 9 ||
+      (chipset.majorVersion == 9 && chipset.minorVersion < 0x08)) {
+    target.addIllegalOp<RawBufferAtomicFaddOp>();
+  }
+  // gfx9 has no to a very limited support for floating-point min and max.
+  if (chipset.majorVersion == 9) {
+    if (chipset.minorVersion >= 0x0a) {
+      // gfx90a supports f64 max (and min, but we don't have a min wrapper right
+      // now) but all other types need to be emulated.
+      target.addDynamicallyLegalOp<RawBufferAtomicFmaxOp>(
+          [](RawBufferAtomicFmaxOp op) -> bool {
+            return op.getValue().getType().isF64();
+          });
+    } else {
+      target.addIllegalOp<RawBufferAtomicFmaxOp>();
+    }
+  }
+  patterns
+      .add<RawBufferAtomicByCasPattern<RawBufferAtomicFaddOp, arith::AddFOp>,
+           RawBufferAtomicByCasPattern<RawBufferAtomicFmaxOp, arith::MaxFOp>>(
+          patterns.getContext());
+}
+
+void AmdgpuEmulateAtomicsPass::runOnOperation() {
+  Operation *op = getOperation();
+  FailureOr<Chipset> maybeChipset = Chipset::parse(chipset);
+  if (failed(maybeChipset)) {
+    emitError(op->getLoc(), "Invalid chipset name: " + chipset);
+    return signalPassFailure();
+  }
+
+  MLIRContext &ctx = getContext();
+  ConversionTarget target(ctx);
+  RewritePatternSet patterns(&ctx);
+  target.markUnknownOpDynamicallyLegal(
+      [](Operation *op) -> bool { return true; });
+
+  populateAmdgpuEmulateAtomicsPatterns(target, patterns, *maybeChipset);
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
+    return signalPassFailure();
+}
diff --git a/mlir/lib/Dialect/AMDGPU/Utils/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/Utils/CMakeLists.txt
new file mode 100644
index 0000000000000..cb67399b7386d
--- /dev/null
+++ b/mlir/lib/Dialect/AMDGPU/Utils/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_mlir_dialect_library(MLIRAMDGPUUtils
+  Chipset.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AMDGPU/Utils
+
+  LINK_LIBS PUBLIC
+  MLIRAMDGPUDialect
+  MLIRSupport
+  )
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/Chipset.cpp b/mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp
similarity index 94%
rename from mlir/lib/Conversion/AMDGPUToROCDL/Chipset.cpp
rename to mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp
index 99087e1c24209..4f075c71c947d 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/Chipset.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/AMDGPUToROCDL/Chipset.h"
+#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/StringRef.h"
 
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index bd9811095356b..7a0f1e0983c7b 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include <numeric>
@@ -701,12 +702,12 @@ static std::optional<int64_t> getUpperBound(Value iv) {
 
 /// Get a lower or upper (depending on `isUpper`) bound for `expr` while using
 /// the constant lower and upper bounds for its inputs provided in
-/// `constLowerBounds` and `constUpperBounds`. Return None if such a bound can't
-/// be computed. This method only handles simple sum of product expressions
-/// (w.r.t constant coefficients) so as to not depend on anything heavyweight in
-/// `Analysis`. Expressions of the form: c0*d0 + c1*d1 + c2*s0 + ... + c_n are
-/// handled. Expressions involving floordiv, ceildiv, mod or semi-affine ones
-/// will lead a none being returned.
+/// `constLowerBounds` and `constUpperBounds`. Return std::nullopt if such a
+/// bound can't be computed. This method only handles simple sum of product
+/// expressions (w.r.t constant coefficients) so as to not depend on anything
+/// heavyweight in `Analysis`. Expressions of the form: c0*d0 + c1*d1 + c2*s0 +
+/// ... + c_n are handled. Expressions involving floordiv, ceildiv, mod or
+/// semi-affine ones will lead a none being returned.
 static std::optional<int64_t>
 getBoundForExpr(AffineExpr expr, unsigned numDims, unsigned numSymbols,
                 ArrayRef<std::optional<int64_t>> constLowerBounds,
@@ -934,6 +935,118 @@ static void simplifyExprAndOperands(AffineExpr &expr, unsigned numDims,
   }
 }
 
+/// Simplify the expressions in `map` while making use of lower or upper bounds
+/// of its operands. If `isMax` is true, the map is to be treated as a max of
+/// its result expressions, and min otherwise. Eg: min (d0, d1) -> (8, 4 * d0 +
+/// d1) can be simplified to (8) if the operands are respectively lower bounded
+/// by 2 and 0 (the second expression can't be lower than 8).
+static void simplifyMinOrMaxExprWithOperands(AffineMap &map,
+                                             ArrayRef<Value> operands,
+                                             bool isMax) {
+  // Can't simplify.
+  if (operands.empty())
+    return;
+
+  // Get the upper or lower bound on an affine.for op IV using its range.
+  // Get the constant lower or upper bounds on the operands.
+  SmallVector<std::optional<int64_t>> constLowerBounds, constUpperBounds;
+  constLowerBounds.reserve(operands.size());
+  constUpperBounds.reserve(operands.size());
+  for (Value operand : operands) {
+    constLowerBounds.push_back(getLowerBound(operand));
+    constUpperBounds.push_back(getUpperBound(operand));
+  }
+
+  // We will compute the lower and upper bounds on each of the expressions
+  // Then, we will check (depending on max or min) as to whether a specific
+  // bound is redundant by checking if its highest (in case of max) and its
+  // lowest (in the case of min) value is already lower than (or higher than)
+  // the lower bound (or upper bound in the case of min) of another bound.
+  SmallVector<std::optional<int64_t>, 4> lowerBounds, upperBounds;
+  lowerBounds.reserve(map.getNumResults());
+  upperBounds.reserve(map.getNumResults());
+  for (AffineExpr e : map.getResults()) {
+    if (auto constExpr = e.dyn_cast<AffineConstantExpr>()) {
+      lowerBounds.push_back(constExpr.getValue());
+      upperBounds.push_back(constExpr.getValue());
+    } else {
+      lowerBounds.push_back(getBoundForExpr(e, map.getNumDims(),
+                                            map.getNumSymbols(),
+                                            constLowerBounds, constUpperBounds,
+                                            /*isUpper=*/false));
+      upperBounds.push_back(getBoundForExpr(e, map.getNumDims(),
+                                            map.getNumSymbols(),
+                                            constLowerBounds, constUpperBounds,
+                                            /*isUpper=*/true));
+    }
+  }
+
+  // Collect expressions that are not redundant.
+  SmallVector<AffineExpr, 4> irredundantExprs;
+  for (auto exprEn : llvm::enumerate(map.getResults())) {
+    AffineExpr e = exprEn.value();
+    unsigned i = exprEn.index();
+    // Some expressions can be turned into constants.
+    if (lowerBounds[i] && upperBounds[i] && *lowerBounds[i] == *upperBounds[i])
+      e = getAffineConstantExpr(*lowerBounds[i], e.getContext());
+
+    // Check if the expression is redundant.
+    if (isMax) {
+      if (!upperBounds[i]) {
+        irredundantExprs.push_back(e);
+        continue;
+      }
+      // If there exists another expression such that its lower bound is greater
+      // than this expression's upper bound, it's redundant.
+      if (!llvm::any_of(llvm::enumerate(lowerBounds), [&](const auto &en) {
+            auto otherLowerBound = en.value();
+            unsigned pos = en.index();
+            if (pos == i || !otherLowerBound)
+              return false;
+            if (*otherLowerBound > *upperBounds[i])
+              return true;
+            if (*otherLowerBound < *upperBounds[i])
+              return false;
+            // Equality case. When both expressions are considered redundant, we
+            // don't want to get both of them. We keep the one that appears
+            // first.
+            if (upperBounds[pos] && lowerBounds[i] &&
+                lowerBounds[i] == upperBounds[i] &&
+                otherLowerBound == *upperBounds[pos] && i < pos)
+              return false;
+            return true;
+          }))
+        irredundantExprs.push_back(e);
+    } else {
+      if (!lowerBounds[i]) {
+        irredundantExprs.push_back(e);
+        continue;
+      }
+      // Likewise for the `min` case. Use the complement of the condition above.
+      if (!llvm::any_of(llvm::enumerate(upperBounds), [&](const auto &en) {
+            auto otherUpperBound = en.value();
+            unsigned pos = en.index();
+            if (pos == i || !otherUpperBound)
+              return false;
+            if (*otherUpperBound < *lowerBounds[i])
+              return true;
+            if (*otherUpperBound > *lowerBounds[i])
+              return false;
+            if (lowerBounds[pos] && upperBounds[i] &&
+                lowerBounds[i] == upperBounds[i] &&
+                otherUpperBound == lowerBounds[pos] && i < pos)
+              return false;
+            return true;
+          }))
+        irredundantExprs.push_back(e);
+    }
+  }
+
+  // Create the map without the redundant expressions.
+  map = AffineMap::get(map.getNumDims(), map.getNumSymbols(), irredundantExprs,
+                       map.getContext());
+}
+
 /// Simplify the map while exploiting information on the values in `operands`.
 //  Use "unused attribute" marker to silence warning stemming from the inability
 //  to see through the template expansion.
@@ -1248,11 +1361,11 @@ SmallVector<OpFoldResult>
 mlir::affine::makeComposedFoldedMultiResultAffineApply(
     OpBuilder &b, Location loc, AffineMap map,
     ArrayRef<OpFoldResult> operands) {
-  return llvm::to_vector(llvm::map_range(
-      llvm::seq<unsigned>(0, map.getNumResults()), [&](unsigned i) {
-        return makeComposedFoldedAffineApply(b, loc, map.getSubMap({i}),
-                                             operands);
-      }));
+  return llvm::map_to_vector(llvm::seq<unsigned>(0, map.getNumResults()),
+                             [&](unsigned i) {
+                               return makeComposedFoldedAffineApply(
+                                   b, loc, map.getSubMap({i}), operands);
+                             });
 }
 
 Value mlir::affine::makeComposedAffineMin(OpBuilder &b, Location loc,
@@ -2217,6 +2330,8 @@ static LogicalResult canonicalizeLoopBounds(AffineForOp forOp) {
 
   composeAffineMapAndOperands(&lbMap, &lbOperands);
   canonicalizeMapAndOperands(&lbMap, &lbOperands);
+  simplifyMinOrMaxExprWithOperands(lbMap, lbOperands, /*isMax=*/true);
+  simplifyMinOrMaxExprWithOperands(ubMap, ubOperands, /*isMax=*/false);
   lbMap = removeDuplicateExprs(lbMap);
 
   composeAffineMapAndOperands(&ubMap, &ubOperands);
@@ -4478,13 +4593,13 @@ void AffineDelinearizeIndexOp::build(OpBuilder &builder, OperationState &result,
   result.addTypes(SmallVector<Type>(basis.size(), builder.getIndexType()));
   result.addOperands(linearIndex);
   SmallVector<Value> basisValues =
-      llvm::to_vector(llvm::map_range(basis, [&](OpFoldResult ofr) -> Value {
+      llvm::map_to_vector(basis, [&](OpFoldResult ofr) -> Value {
         std::optional<int64_t> staticDim = getConstantIntValue(ofr);
         if (staticDim.has_value())
           return builder.create<arith::ConstantIndexOp>(result.location,
                                                         *staticDim);
         return ofr.dyn_cast<Value>();
-      }));
+      });
   result.addOperands(basisValues);
 }
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
index 0efe31c6cb289..4990229dfd3c8 100644
--- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
@@ -19,7 +19,7 @@ using namespace mlir::affine;
 static FailureOr<OpFoldResult>
 reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
                 Value value, std::optional<int64_t> dim,
-                function_ref<bool(Value, std::optional<int64_t>)> stopCondition,
+                ValueBoundsConstraintSet::StopConditionFn stopCondition,
                 bool closedUB) {
   // Compute bound.
   AffineMap boundMap;
@@ -28,6 +28,13 @@ reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
           boundMap, mapOperands, type, value, dim, stopCondition, closedUB)))
     return failure();
 
+  // Reify bound.
+  return affine::materializeComputedBound(b, loc, boundMap, mapOperands);
+}
+
+OpFoldResult affine::materializeComputedBound(
+    OpBuilder &b, Location loc, AffineMap boundMap,
+    ArrayRef<std::pair<Value, std::optional<int64_t>>> mapOperands) {
   // Materialize tensor.dim/memref.dim ops.
   SmallVector<Value> operands;
   for (auto valueDim : mapOperands) {
diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index d4c6b8184751f..ba1f3f8bd1d86 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -319,6 +319,20 @@ def TruncationMatchesShiftAmount :
       CPred<"(getScalarOrElementWidth($0) - getScalarOrElementWidth($1)) == "
               "*getIntOrSplatIntValue($2)">]>>;
 
+// trunci(extsi(x)) -> extsi(x), when only the sign-extension bits are truncated
+def TruncIExtSIToExtSI :
+    Pat<(Arith_TruncIOp:$tr (Arith_ExtSIOp:$ext $x)),
+        (Arith_ExtSIOp $x),
+        [(ValueWiderThan $ext, $tr),
+         (ValueWiderThan $tr, $x)]>;
+
+// trunci(extui(x)) -> extui(x), when only the zero-extension bits are truncated
+def TruncIExtUIToExtUI :
+    Pat<(Arith_TruncIOp:$tr (Arith_ExtUIOp:$ext $x)),
+        (Arith_ExtUIOp $x),
+        [(ValueWiderThan $ext, $tr),
+         (ValueWiderThan $tr, $x)]>;
+
 // trunci(shrsi(x, c)) -> trunci(shrui(x, c))
 def TruncIShrSIToTrunciShrUI :
     Pat<(Arith_TruncIOp:$tr
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 446bb6461077d..b4b0572fdee75 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -1290,8 +1290,9 @@ bool arith::TruncIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
 
 void arith::TruncIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                                   MLIRContext *context) {
-  patterns.add<TruncIShrSIToTrunciShrUI, TruncIShrUIMulIToMulSIExtended,
-               TruncIShrUIMulIToMulUIExtended>(context);
+  patterns.add<TruncIExtSIToExtSI, TruncIExtUIToExtUI, TruncIShrSIToTrunciShrUI,
+               TruncIShrUIMulIToMulSIExtended, TruncIShrUIMulIToMulUIExtended>(
+      context);
 }
 
 LogicalResult arith::TruncIOp::verify() {
diff --git a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt
index 80a612883e396..87d9bebfd2a7c 100644
--- a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt
@@ -3,6 +3,7 @@ add_mlir_dialect_library(MLIRArithTransforms
   Bufferize.cpp
   EmulateWideInt.cpp
   ExpandOps.cpp
+  IntNarrowing.cpp
   IntRangeOptimizations.cpp
   ReifyValueBounds.cpp
   UnsignedWhenEquivalent.cpp
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp b/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp
new file mode 100644
index 0000000000000..4f294e6e4c91e
--- /dev/null
+++ b/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp
@@ -0,0 +1,783 @@
+//===- IntNarrowing.cpp - Integer bitwidth reduction optimizations --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/Transforms/Passes.h"
+
+#include "mlir/Analysis/Presburger/IntegerRelation.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Arith/Transforms/Transforms.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypeInterfaces.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Matchers.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/Interfaces/ValueBoundsOpInterface.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include <cassert>
+#include <cstdint>
+
+namespace mlir::arith {
+#define GEN_PASS_DEF_ARITHINTNARROWING
+#include "mlir/Dialect/Arith/Transforms/Passes.h.inc"
+} // namespace mlir::arith
+
+namespace mlir::arith {
+namespace {
+//===----------------------------------------------------------------------===//
+// Common Helpers
+//===----------------------------------------------------------------------===//
+
+/// The base for integer bitwidth narrowing patterns.
+template <typename SourceOp>
+struct NarrowingPattern : OpRewritePattern<SourceOp> {
+  NarrowingPattern(MLIRContext *ctx, const ArithIntNarrowingOptions &options,
+                   PatternBenefit benefit = 1)
+      : OpRewritePattern<SourceOp>(ctx, benefit),
+        supportedBitwidths(options.bitwidthsSupported.begin(),
+                           options.bitwidthsSupported.end()) {
+    assert(!supportedBitwidths.empty() && "Invalid options");
+    assert(!llvm::is_contained(supportedBitwidths, 0) && "Invalid bitwidth");
+    llvm::sort(supportedBitwidths);
+  }
+
+  FailureOr<unsigned>
+  getNarrowestCompatibleBitwidth(unsigned bitsRequired) const {
+    for (unsigned candidate : supportedBitwidths)
+      if (candidate >= bitsRequired)
+        return candidate;
+
+    return failure();
+  }
+
+  /// Returns the narrowest supported type that fits `bitsRequired`.
+  FailureOr<Type> getNarrowType(unsigned bitsRequired, Type origTy) const {
+    assert(origTy);
+    FailureOr<unsigned> bestBitwidth =
+        getNarrowestCompatibleBitwidth(bitsRequired);
+    if (failed(bestBitwidth))
+      return failure();
+
+    Type elemTy = getElementTypeOrSelf(origTy);
+    if (!isa<IntegerType>(elemTy))
+      return failure();
+
+    auto newElemTy = IntegerType::get(origTy.getContext(), *bestBitwidth);
+    if (newElemTy == elemTy)
+      return failure();
+
+    if (origTy == elemTy)
+      return newElemTy;
+
+    if (auto shapedTy = dyn_cast<ShapedType>(origTy))
+      if (auto elemTy = dyn_cast<IntegerType>(shapedTy.getElementType()))
+        return shapedTy.clone(shapedTy.getShape(), newElemTy);
+
+    return failure();
+  }
+
+private:
+  // Supported integer bitwidths in the ascending order.
+  llvm::SmallVector<unsigned, 6> supportedBitwidths;
+};
+
+/// Returns the integer bitwidth required to represent `type`.
+FailureOr<unsigned> calculateBitsRequired(Type type) {
+  assert(type);
+  if (auto intTy = dyn_cast<IntegerType>(getElementTypeOrSelf(type)))
+    return intTy.getWidth();
+
+  return failure();
+}
+
+enum class ExtensionKind { Sign, Zero };
+
+/// Wrapper around `arith::ExtSIOp` and `arith::ExtUIOp` ops that abstracts away
+/// the exact op type. Exposes helper functions to query the types, operands,
+/// and the result. This is so that we can handle both extension kinds without
+/// needing to use templates or branching.
+class ExtensionOp {
+public:
+  /// Attemps to create a new extension op from `op`. Returns an extension op
+  /// wrapper when `op` is either `arith.extsi` or `arith.extui`, and failure
+  /// otherwise.
+  static FailureOr<ExtensionOp> from(Operation *op) {
+    if (auto sext = dyn_cast_or_null<arith::ExtSIOp>(op))
+      return ExtensionOp{op, ExtensionKind::Sign};
+    if (auto zext = dyn_cast_or_null<arith::ExtUIOp>(op))
+      return ExtensionOp{op, ExtensionKind::Zero};
+
+    return failure();
+  }
+
+  ExtensionOp(const ExtensionOp &) = default;
+  ExtensionOp &operator=(const ExtensionOp &) = default;
+
+  /// Creates a new extension op of the same kind.
+  Operation *recreate(PatternRewriter &rewriter, Location loc, Type newType,
+                      Value in) {
+    if (kind == ExtensionKind::Sign)
+      return rewriter.create<arith::ExtSIOp>(loc, newType, in);
+
+    return rewriter.create<arith::ExtUIOp>(loc, newType, in);
+  }
+
+  /// Replaces `toReplace` with a new extension op of the same kind.
+  void recreateAndReplace(PatternRewriter &rewriter, Operation *toReplace,
+                          Value in) {
+    assert(toReplace->getNumResults() == 1);
+    Type newType = toReplace->getResult(0).getType();
+    Operation *newOp = recreate(rewriter, toReplace->getLoc(), newType, in);
+    rewriter.replaceOp(toReplace, newOp->getResult(0));
+  }
+
+  ExtensionKind getKind() { return kind; }
+
+  Value getResult() { return op->getResult(0); }
+  Value getIn() { return op->getOperand(0); }
+
+  Type getType() { return getResult().getType(); }
+  Type getElementType() { return getElementTypeOrSelf(getType()); }
+  Type getInType() { return getIn().getType(); }
+  Type getInElementType() { return getElementTypeOrSelf(getInType()); }
+
+private:
+  ExtensionOp(Operation *op, ExtensionKind kind) : op(op), kind(kind) {
+    assert(op);
+    assert((isa<arith::ExtSIOp, arith::ExtUIOp>(op)) && "Not an extension op");
+  }
+  Operation *op = nullptr;
+  ExtensionKind kind = {};
+};
+
+/// Returns the integer bitwidth required to represent `value`.
+unsigned calculateBitsRequired(const APInt &value,
+                               ExtensionKind lookThroughExtension) {
+  // For unsigned values, we only need the active bits. As a special case, zero
+  // requires one bit.
+  if (lookThroughExtension == ExtensionKind::Zero)
+    return std::max(value.getActiveBits(), 1u);
+
+  // If a signed value is nonnegative, we need one extra bit for the sign.
+  if (value.isNonNegative())
+    return value.getActiveBits() + 1;
+
+  // For the signed min, we need all the bits.
+  if (value.isMinSignedValue())
+    return value.getBitWidth();
+
+  // For negative values, we need all the non-sign bits and one extra bit for
+  // the sign.
+  return value.getBitWidth() - value.getNumSignBits() + 1;
+}
+
+/// Returns the integer bitwidth required to represent `value`.
+/// Looks through either sign- or zero-extension as specified by
+/// `lookThroughExtension`.
+FailureOr<unsigned> calculateBitsRequired(Value value,
+                                          ExtensionKind lookThroughExtension) {
+  // Handle constants.
+  if (TypedAttr attr; matchPattern(value, m_Constant(&attr))) {
+    if (auto intAttr = dyn_cast<IntegerAttr>(attr))
+      return calculateBitsRequired(intAttr.getValue(), lookThroughExtension);
+
+    if (auto elemsAttr = dyn_cast<DenseElementsAttr>(attr)) {
+      if (elemsAttr.getElementType().isIntOrIndex()) {
+        if (elemsAttr.isSplat())
+          return calculateBitsRequired(elemsAttr.getSplatValue<APInt>(),
+                                       lookThroughExtension);
+
+        unsigned maxBits = 1;
+        for (const APInt &elemValue : elemsAttr.getValues<APInt>())
+          maxBits = std::max(
+              maxBits, calculateBitsRequired(elemValue, lookThroughExtension));
+        return maxBits;
+      }
+    }
+  }
+
+  if (lookThroughExtension == ExtensionKind::Sign) {
+    if (auto sext = value.getDefiningOp<arith::ExtSIOp>())
+      return calculateBitsRequired(sext.getIn().getType());
+  } else if (lookThroughExtension == ExtensionKind::Zero) {
+    if (auto zext = value.getDefiningOp<arith::ExtUIOp>())
+      return calculateBitsRequired(zext.getIn().getType());
+  }
+
+  // If nothing else worked, return the type requirements for this element type.
+  return calculateBitsRequired(value.getType());
+}
+
+/// Base pattern for arith binary ops.
+/// Example:
+/// ```
+///   %lhs = arith.extsi %a : i8 to i32
+///   %rhs = arith.extsi %b : i8 to i32
+///   %r = arith.addi %lhs, %rhs : i32
+/// ==>
+///   %lhs = arith.extsi %a : i8 to i16
+///   %rhs = arith.extsi %b : i8 to i16
+///   %add = arith.addi %lhs, %rhs : i16
+///   %r = arith.extsi %add : i16 to i32
+/// ```
+template <typename BinaryOp>
+struct BinaryOpNarrowingPattern : NarrowingPattern<BinaryOp> {
+  using NarrowingPattern<BinaryOp>::NarrowingPattern;
+
+  /// Returns the number of bits required to represent the full result, assuming
+  /// that both operands are `operandBits`-wide. Derived classes must implement
+  /// this, taking into account `BinaryOp` semantics.
+  virtual unsigned getResultBitsProduced(unsigned operandBits) const = 0;
+
+  /// Customization point for patterns that should only apply with
+  /// zero/sign-extension ops as arguments.
+  virtual bool isSupported(ExtensionOp) const { return true; }
+
+  LogicalResult matchAndRewrite(BinaryOp op,
+                                PatternRewriter &rewriter) const final {
+    Type origTy = op.getType();
+    FailureOr<unsigned> resultBits = calculateBitsRequired(origTy);
+    if (failed(resultBits))
+      return failure();
+
+    // For the optimization to apply, we expect the lhs to be an extension op,
+    // and for the rhs to either be the same extension op or a constant.
+    FailureOr<ExtensionOp> ext = ExtensionOp::from(op.getLhs().getDefiningOp());
+    if (failed(ext) || !isSupported(*ext))
+      return failure();
+
+    FailureOr<unsigned> lhsBitsRequired =
+        calculateBitsRequired(ext->getIn(), ext->getKind());
+    if (failed(lhsBitsRequired) || *lhsBitsRequired >= *resultBits)
+      return failure();
+
+    FailureOr<unsigned> rhsBitsRequired =
+        calculateBitsRequired(op.getRhs(), ext->getKind());
+    if (failed(rhsBitsRequired) || *rhsBitsRequired >= *resultBits)
+      return failure();
+
+    // Negotiate a common bit requirements for both lhs and rhs, accounting for
+    // the result requiring more bits than the operands.
+    unsigned commonBitsRequired =
+        getResultBitsProduced(std::max(*lhsBitsRequired, *rhsBitsRequired));
+    FailureOr<Type> narrowTy = this->getNarrowType(commonBitsRequired, origTy);
+    if (failed(narrowTy) || calculateBitsRequired(*narrowTy) >= *resultBits)
+      return failure();
+
+    Location loc = op.getLoc();
+    Value newLhs =
+        rewriter.createOrFold<arith::TruncIOp>(loc, *narrowTy, op.getLhs());
+    Value newRhs =
+        rewriter.createOrFold<arith::TruncIOp>(loc, *narrowTy, op.getRhs());
+    Value newAdd = rewriter.create<BinaryOp>(loc, newLhs, newRhs);
+    ext->recreateAndReplace(rewriter, op, newAdd);
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// AddIOp Pattern
+//===----------------------------------------------------------------------===//
+
+struct AddIPattern final : BinaryOpNarrowingPattern<arith::AddIOp> {
+  using BinaryOpNarrowingPattern::BinaryOpNarrowingPattern;
+
+  // Addition may require one extra bit for the result.
+  // Example: `UINT8_MAX + 1 == 255 + 1 == 256`.
+  unsigned getResultBitsProduced(unsigned operandBits) const override {
+    return operandBits + 1;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// SubIOp Pattern
+//===----------------------------------------------------------------------===//
+
+struct SubIPattern final : BinaryOpNarrowingPattern<arith::SubIOp> {
+  using BinaryOpNarrowingPattern::BinaryOpNarrowingPattern;
+
+  // This optimization only applies to signed arguments.
+  bool isSupported(ExtensionOp ext) const override {
+    return ext.getKind() == ExtensionKind::Sign;
+  }
+
+  // Subtraction may require one extra bit for the result.
+  // Example: `INT8_MAX - (-1) == 127 - (-1) == 128`.
+  unsigned getResultBitsProduced(unsigned operandBits) const override {
+    return operandBits + 1;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// MulIOp Pattern
+//===----------------------------------------------------------------------===//
+
+struct MulIPattern final : BinaryOpNarrowingPattern<arith::MulIOp> {
+  using BinaryOpNarrowingPattern::BinaryOpNarrowingPattern;
+
+  // Multiplication may require up double the operand bits.
+  // Example: `UNT8_MAX * UINT8_MAX == 255 * 255 == 65025`.
+  unsigned getResultBitsProduced(unsigned operandBits) const override {
+    return 2 * operandBits;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// DivSIOp Pattern
+//===----------------------------------------------------------------------===//
+
+struct DivSIPattern final : BinaryOpNarrowingPattern<arith::DivSIOp> {
+  using BinaryOpNarrowingPattern::BinaryOpNarrowingPattern;
+
+  // This optimization only applies to signed arguments.
+  bool isSupported(ExtensionOp ext) const override {
+    return ext.getKind() == ExtensionKind::Sign;
+  }
+
+  // Unlike multiplication, signed division requires only one more result bit.
+  // Example: `INT8_MIN / (-1) == -128 / (-1) == 128`.
+  unsigned getResultBitsProduced(unsigned operandBits) const override {
+    return operandBits + 1;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// DivUIOp Pattern
+//===----------------------------------------------------------------------===//
+
+struct DivUIPattern final : BinaryOpNarrowingPattern<arith::DivUIOp> {
+  using BinaryOpNarrowingPattern::BinaryOpNarrowingPattern;
+
+  // This optimization only applies to unsigned arguments.
+  bool isSupported(ExtensionOp ext) const override {
+    return ext.getKind() == ExtensionKind::Zero;
+  }
+
+  // Unsigned division does not require any extra result bits.
+  unsigned getResultBitsProduced(unsigned operandBits) const override {
+    return operandBits;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Min/Max Patterns
+//===----------------------------------------------------------------------===//
+
+template <typename MinMaxOp, ExtensionKind Kind>
+struct MinMaxPattern final : BinaryOpNarrowingPattern<MinMaxOp> {
+  using BinaryOpNarrowingPattern<MinMaxOp>::BinaryOpNarrowingPattern;
+
+  bool isSupported(ExtensionOp ext) const override {
+    return ext.getKind() == Kind;
+  }
+
+  // Min/max returns one of the arguments and does not require any extra result
+  // bits.
+  unsigned getResultBitsProduced(unsigned operandBits) const override {
+    return operandBits;
+  }
+};
+using MaxSIPattern = MinMaxPattern<arith::MaxSIOp, ExtensionKind::Sign>;
+using MaxUIPattern = MinMaxPattern<arith::MaxUIOp, ExtensionKind::Zero>;
+using MinSIPattern = MinMaxPattern<arith::MinSIOp, ExtensionKind::Sign>;
+using MinUIPattern = MinMaxPattern<arith::MinUIOp, ExtensionKind::Zero>;
+
+//===----------------------------------------------------------------------===//
+// *IToFPOp Patterns
+//===----------------------------------------------------------------------===//
+
+template <typename IToFPOp, ExtensionKind Extension>
+struct IToFPPattern final : NarrowingPattern<IToFPOp> {
+  using NarrowingPattern<IToFPOp>::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(IToFPOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<unsigned> narrowestWidth =
+        calculateBitsRequired(op.getIn(), Extension);
+    if (failed(narrowestWidth))
+      return failure();
+
+    FailureOr<Type> narrowTy =
+        this->getNarrowType(*narrowestWidth, op.getIn().getType());
+    if (failed(narrowTy))
+      return failure();
+
+    Value newIn = rewriter.createOrFold<arith::TruncIOp>(op.getLoc(), *narrowTy,
+                                                         op.getIn());
+    rewriter.replaceOpWithNewOp<IToFPOp>(op, op.getType(), newIn);
+    return success();
+  }
+};
+using SIToFPPattern = IToFPPattern<arith::SIToFPOp, ExtensionKind::Sign>;
+using UIToFPPattern = IToFPPattern<arith::UIToFPOp, ExtensionKind::Zero>;
+
+//===----------------------------------------------------------------------===//
+// Index Cast Patterns
+//===----------------------------------------------------------------------===//
+
+// These rely on the `ValueBounds` interface for index values. For example, we
+// can often statically tell index value bounds of loop induction variables.
+
+template <typename CastOp, ExtensionKind Kind>
+struct IndexCastPattern final : NarrowingPattern<CastOp> {
+  using NarrowingPattern<CastOp>::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(CastOp op,
+                                PatternRewriter &rewriter) const override {
+    Value in = op.getIn();
+    // We only support scalar index -> integer casts.
+    if (!isa<IndexType>(in.getType()))
+      return failure();
+
+    // Check the lower bound in both the signed and unsigned cast case. We
+    // conservatively assume that even unsigned casts may be performed on
+    // negative indices.
+    FailureOr<int64_t> lb = ValueBoundsConstraintSet::computeConstantBound(
+        presburger::BoundType::LB, in);
+    if (failed(lb))
+      return failure();
+
+    FailureOr<int64_t> ub = ValueBoundsConstraintSet::computeConstantBound(
+        presburger::BoundType::UB, in, /*dim=*/std::nullopt,
+        /*stopCondition=*/nullptr, /*closedUB=*/true);
+    if (failed(ub))
+      return failure();
+
+    assert(*lb <= *ub && "Invalid bounds");
+    unsigned lbBitsRequired = calculateBitsRequired(APInt(64, *lb), Kind);
+    unsigned ubBitsRequired = calculateBitsRequired(APInt(64, *ub), Kind);
+    unsigned bitsRequired = std::max(lbBitsRequired, ubBitsRequired);
+
+    IntegerType resultTy = cast<IntegerType>(op.getType());
+    if (resultTy.getWidth() <= bitsRequired)
+      return failure();
+
+    FailureOr<Type> narrowTy = this->getNarrowType(bitsRequired, resultTy);
+    if (failed(narrowTy))
+      return failure();
+
+    Value newCast = rewriter.create<CastOp>(op.getLoc(), *narrowTy, op.getIn());
+
+    if (Kind == ExtensionKind::Sign)
+      rewriter.replaceOpWithNewOp<arith::ExtSIOp>(op, resultTy, newCast);
+    else
+      rewriter.replaceOpWithNewOp<arith::ExtUIOp>(op, resultTy, newCast);
+    return success();
+  }
+};
+using IndexCastSIPattern =
+    IndexCastPattern<arith::IndexCastOp, ExtensionKind::Sign>;
+using IndexCastUIPattern =
+    IndexCastPattern<arith::IndexCastUIOp, ExtensionKind::Zero>;
+
+//===----------------------------------------------------------------------===//
+// Patterns to Commute Extension Ops
+//===----------------------------------------------------------------------===//
+
+struct ExtensionOverBroadcast final : NarrowingPattern<vector::BroadcastOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::BroadcastOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getSource().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    VectorType origTy = op.getResultVectorType();
+    VectorType newTy =
+        origTy.cloneWith(origTy.getShape(), ext->getInElementType());
+    Value newBroadcast =
+        rewriter.create<vector::BroadcastOp>(op.getLoc(), newTy, ext->getIn());
+    ext->recreateAndReplace(rewriter, op, newBroadcast);
+    return success();
+  }
+};
+
+struct ExtensionOverExtract final : NarrowingPattern<vector::ExtractOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::ExtractOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getVector().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    Value newExtract = rewriter.create<vector::ExtractOp>(
+        op.getLoc(), ext->getIn(), op.getPosition());
+    ext->recreateAndReplace(rewriter, op, newExtract);
+    return success();
+  }
+};
+
+struct ExtensionOverExtractElement final
+    : NarrowingPattern<vector::ExtractElementOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::ExtractElementOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getVector().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    Value newExtract = rewriter.create<vector::ExtractElementOp>(
+        op.getLoc(), ext->getIn(), op.getPosition());
+    ext->recreateAndReplace(rewriter, op, newExtract);
+    return success();
+  }
+};
+
+struct ExtensionOverExtractStridedSlice final
+    : NarrowingPattern<vector::ExtractStridedSliceOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getVector().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    VectorType origTy = op.getType();
+    VectorType extractTy =
+        origTy.cloneWith(origTy.getShape(), ext->getInElementType());
+    Value newExtract = rewriter.create<vector::ExtractStridedSliceOp>(
+        op.getLoc(), extractTy, ext->getIn(), op.getOffsets(), op.getSizes(),
+        op.getStrides());
+    ext->recreateAndReplace(rewriter, op, newExtract);
+    return success();
+  }
+};
+
+/// Base pattern for `vector.insert` narrowing patterns.
+template <typename InsertionOp>
+struct ExtensionOverInsertionPattern : NarrowingPattern<InsertionOp> {
+  using NarrowingPattern<InsertionOp>::NarrowingPattern;
+
+  /// Derived classes must provide a function to create the matching insertion
+  /// op based on the original op and new arguments.
+  virtual InsertionOp createInsertionOp(PatternRewriter &rewriter,
+                                        InsertionOp origInsert,
+                                        Value narrowValue,
+                                        Value narrowDest) const = 0;
+
+  LogicalResult matchAndRewrite(InsertionOp op,
+                                PatternRewriter &rewriter) const final {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getSource().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    FailureOr<InsertionOp> newInsert = createNarrowInsert(op, rewriter, *ext);
+    if (failed(newInsert))
+      return failure();
+    ext->recreateAndReplace(rewriter, op, *newInsert);
+    return success();
+  }
+
+  FailureOr<InsertionOp> createNarrowInsert(InsertionOp op,
+                                            PatternRewriter &rewriter,
+                                            ExtensionOp insValue) const {
+    // Calculate the operand and result bitwidths. We can only apply narrowing
+    // when the inserted source value and destination vector require fewer bits
+    // than the result. Because the source and destination may have different
+    // bitwidths requirements, we have to find the common narrow bitwidth that
+    // is greater equal to the operand bitwidth requirements and still narrower
+    // than the result.
+    FailureOr<unsigned> origBitsRequired = calculateBitsRequired(op.getType());
+    if (failed(origBitsRequired))
+      return failure();
+
+    // TODO: We could relax this check by disregarding bitwidth requirements of
+    // elements that we know will be replaced by the insertion.
+    FailureOr<unsigned> destBitsRequired =
+        calculateBitsRequired(op.getDest(), insValue.getKind());
+    if (failed(destBitsRequired) || *destBitsRequired >= *origBitsRequired)
+      return failure();
+
+    FailureOr<unsigned> insertedBitsRequired =
+        calculateBitsRequired(insValue.getIn(), insValue.getKind());
+    if (failed(insertedBitsRequired) ||
+        *insertedBitsRequired >= *origBitsRequired)
+      return failure();
+
+    // Find a narrower element type that satisfies the bitwidth requirements of
+    // both the source and the destination values.
+    unsigned newInsertionBits =
+        std::max(*destBitsRequired, *insertedBitsRequired);
+    FailureOr<Type> newVecTy =
+        this->getNarrowType(newInsertionBits, op.getType());
+    if (failed(newVecTy) || *newVecTy == op.getType())
+      return failure();
+
+    FailureOr<Type> newInsertedValueTy =
+        this->getNarrowType(newInsertionBits, insValue.getType());
+    if (failed(newInsertedValueTy))
+      return failure();
+
+    Location loc = op.getLoc();
+    Value narrowValue = rewriter.createOrFold<arith::TruncIOp>(
+        loc, *newInsertedValueTy, insValue.getResult());
+    Value narrowDest =
+        rewriter.createOrFold<arith::TruncIOp>(loc, *newVecTy, op.getDest());
+    return createInsertionOp(rewriter, op, narrowValue, narrowDest);
+  }
+};
+
+struct ExtensionOverInsert final
+    : ExtensionOverInsertionPattern<vector::InsertOp> {
+  using ExtensionOverInsertionPattern::ExtensionOverInsertionPattern;
+
+  vector::InsertOp createInsertionOp(PatternRewriter &rewriter,
+                                     vector::InsertOp origInsert,
+                                     Value narrowValue,
+                                     Value narrowDest) const override {
+    return rewriter.create<vector::InsertOp>(
+        origInsert.getLoc(), narrowValue, narrowDest, origInsert.getPosition());
+  }
+};
+
+struct ExtensionOverInsertElement final
+    : ExtensionOverInsertionPattern<vector::InsertElementOp> {
+  using ExtensionOverInsertionPattern::ExtensionOverInsertionPattern;
+
+  vector::InsertElementOp createInsertionOp(PatternRewriter &rewriter,
+                                            vector::InsertElementOp origInsert,
+                                            Value narrowValue,
+                                            Value narrowDest) const override {
+    return rewriter.create<vector::InsertElementOp>(
+        origInsert.getLoc(), narrowValue, narrowDest, origInsert.getPosition());
+  }
+};
+
+struct ExtensionOverInsertStridedSlice final
+    : ExtensionOverInsertionPattern<vector::InsertStridedSliceOp> {
+  using ExtensionOverInsertionPattern::ExtensionOverInsertionPattern;
+
+  vector::InsertStridedSliceOp
+  createInsertionOp(PatternRewriter &rewriter,
+                    vector::InsertStridedSliceOp origInsert, Value narrowValue,
+                    Value narrowDest) const override {
+    return rewriter.create<vector::InsertStridedSliceOp>(
+        origInsert.getLoc(), narrowValue, narrowDest, origInsert.getOffsets(),
+        origInsert.getStrides());
+  }
+};
+
+struct ExtensionOverShapeCast final : NarrowingPattern<vector::ShapeCastOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::ShapeCastOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getSource().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    VectorType origTy = op.getResultVectorType();
+    VectorType newTy =
+        origTy.cloneWith(origTy.getShape(), ext->getInElementType());
+    Value newCast =
+        rewriter.create<vector::ShapeCastOp>(op.getLoc(), newTy, ext->getIn());
+    ext->recreateAndReplace(rewriter, op, newCast);
+    return success();
+  }
+};
+
+struct ExtensionOverTranspose final : NarrowingPattern<vector::TransposeOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::TransposeOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getVector().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    VectorType origTy = op.getResultVectorType();
+    VectorType newTy =
+        origTy.cloneWith(origTy.getShape(), ext->getInElementType());
+    Value newTranspose = rewriter.create<vector::TransposeOp>(
+        op.getLoc(), newTy, ext->getIn(), op.getTransp());
+    ext->recreateAndReplace(rewriter, op, newTranspose);
+    return success();
+  }
+};
+
+struct ExtensionOverFlatTranspose final
+    : NarrowingPattern<vector::FlatTransposeOp> {
+  using NarrowingPattern::NarrowingPattern;
+
+  LogicalResult matchAndRewrite(vector::FlatTransposeOp op,
+                                PatternRewriter &rewriter) const override {
+    FailureOr<ExtensionOp> ext =
+        ExtensionOp::from(op.getMatrix().getDefiningOp());
+    if (failed(ext))
+      return failure();
+
+    VectorType origTy = op.getType();
+    VectorType newTy =
+        origTy.cloneWith(origTy.getShape(), ext->getInElementType());
+    Value newTranspose = rewriter.create<vector::FlatTransposeOp>(
+        op.getLoc(), newTy, ext->getIn(), op.getRowsAttr(),
+        op.getColumnsAttr());
+    ext->recreateAndReplace(rewriter, op, newTranspose);
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Pass Definitions
+//===----------------------------------------------------------------------===//
+
+struct ArithIntNarrowingPass final
+    : impl::ArithIntNarrowingBase<ArithIntNarrowingPass> {
+  using ArithIntNarrowingBase::ArithIntNarrowingBase;
+
+  void runOnOperation() override {
+    Operation *op = getOperation();
+    MLIRContext *ctx = op->getContext();
+    RewritePatternSet patterns(ctx);
+    populateArithIntNarrowingPatterns(
+        patterns, ArithIntNarrowingOptions{bitwidthsSupported});
+    if (failed(applyPatternsAndFoldGreedily(op, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+} // namespace
+
+//===----------------------------------------------------------------------===//
+// Public API
+//===----------------------------------------------------------------------===//
+
+void populateArithIntNarrowingPatterns(
+    RewritePatternSet &patterns, const ArithIntNarrowingOptions &options) {
+  // Add commute patterns with a higher benefit. This is to expose more
+  // optimization opportunities to narrowing patterns.
+  patterns.add<ExtensionOverBroadcast, ExtensionOverExtract,
+               ExtensionOverExtractElement, ExtensionOverExtractStridedSlice,
+               ExtensionOverInsert, ExtensionOverInsertElement,
+               ExtensionOverInsertStridedSlice, ExtensionOverShapeCast,
+               ExtensionOverTranspose, ExtensionOverFlatTranspose>(
+      patterns.getContext(), options, PatternBenefit(2));
+
+  patterns.add<AddIPattern, SubIPattern, MulIPattern, DivSIPattern,
+               DivUIPattern, MaxSIPattern, MaxUIPattern, MinSIPattern,
+               MinUIPattern, SIToFPPattern, UIToFPPattern, IndexCastSIPattern,
+               IndexCastUIPattern>(patterns.getContext(), options);
+}
+
+} // namespace mlir::arith
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index 1a54614cae33b..71904762d0b13 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -59,6 +59,10 @@ LogicalResult ApplyOp::verify() {
   if (applicableOperatorStr != "&" && applicableOperatorStr != "*")
     return emitOpError("applicable operator is illegal");
 
+  Operation *op = getOperand().getDefiningOp();
+  if (op && dyn_cast<ConstantOp>(op))
+    return emitOpError("cannot apply to constant");
+
   return success();
 }
 
@@ -121,6 +125,11 @@ LogicalResult emitc::ConstantOp::verify() {
   if (getValueAttr().isa<emitc::OpaqueAttr>())
     return success();
 
+  // Value must not be empty
+  StringAttr strAttr = getValueAttr().dyn_cast<StringAttr>();
+  if (strAttr && strAttr.getValue().empty())
+    return emitOpError() << "value must not be empty";
+
   auto value = cast<TypedAttr>(getValueAttr());
   Type type = getType();
   if (!value.getType().isa<NoneType>() && type != value.getType())
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index f9d929d163445..4c188d3a29263 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -332,6 +332,60 @@ static void printAsyncDependencies(OpAsmPrinter &printer, Operation *op,
   printer << ']';
 }
 
+// GPU Memory attributions functions shared by LaunchOp and GPUFuncOp.
+/// Parses a GPU function memory attribution.
+///
+/// memory-attribution ::= (`workgroup` `(` ssa-id-and-type-list `)`)?
+///                        (`private` `(` ssa-id-and-type-list `)`)?
+///
+/// Note that this function parses only one of the two similar parts, with the
+/// keyword provided as argument.
+static ParseResult
+parseAttributions(OpAsmParser &parser, StringRef keyword,
+                  SmallVectorImpl<OpAsmParser::Argument> &args) {
+  // If we could not parse the keyword, just assume empty list and succeed.
+  if (failed(parser.parseOptionalKeyword(keyword)))
+    return success();
+
+  return parser.parseArgumentList(args, OpAsmParser::Delimiter::Paren,
+                                  /*allowType=*/true);
+}
+
+/// Prints a GPU function memory attribution.
+static void printAttributions(OpAsmPrinter &p, StringRef keyword,
+                              ArrayRef<BlockArgument> values) {
+  if (values.empty())
+    return;
+
+  p << ' ' << keyword << '(';
+  llvm::interleaveComma(
+      values, p, [&p](BlockArgument v) { p << v << " : " << v.getType(); });
+  p << ')';
+}
+
+/// Verifies a GPU function memory attribution.
+static LogicalResult verifyAttributions(Operation *op,
+                                        ArrayRef<BlockArgument> attributions,
+                                        gpu::AddressSpace memorySpace) {
+  for (Value v : attributions) {
+    auto type = v.getType().dyn_cast<MemRefType>();
+    if (!type)
+      return op->emitOpError() << "expected memref type in attribution";
+
+    // We can only verify the address space if it hasn't already been lowered
+    // from the AddressSpaceAttr to a target-specific numeric value.
+    auto addressSpace =
+        type.getMemorySpace().dyn_cast_or_null<gpu::AddressSpaceAttr>();
+    if (!addressSpace)
+      continue;
+    if (addressSpace.getValue() != memorySpace)
+      return op->emitOpError()
+             << "expected memory space " << stringifyAddressSpace(memorySpace)
+             << " in attribution";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // AllReduceOp
 //===----------------------------------------------------------------------===//
@@ -439,7 +493,15 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result,
                      Value gridSizeX, Value gridSizeY, Value gridSizeZ,
                      Value getBlockSizeX, Value getBlockSizeY,
                      Value getBlockSizeZ, Value dynamicSharedMemorySize,
-                     Type asyncTokenType, ValueRange asyncDependencies) {
+                     Type asyncTokenType, ValueRange asyncDependencies,
+                     TypeRange workgroupAttributions,
+                     TypeRange privateAttributions) {
+  // Add a WorkGroup attribution attribute. This attribute is required to
+  // identify private attributions in the list of block argguments.
+  result.addAttribute(getNumWorkgroupAttributionsAttrName(),
+                      builder.getI64IntegerAttr(workgroupAttributions.size()));
+
+  // Add Op operands.
   result.addOperands(asyncDependencies);
   if (asyncTokenType)
     result.types.push_back(builder.getType<AsyncTokenType>());
@@ -450,14 +512,21 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result,
   if (dynamicSharedMemorySize)
     result.addOperands(dynamicSharedMemorySize);
 
-  // Create a kernel body region with kNumConfigRegionAttributes + N arguments,
-  // where the first kNumConfigRegionAttributes arguments have `index` type and
-  // the rest have the same types as the data operands.
+  // Create a kernel body region with kNumConfigRegionAttributes + N memory
+  // attributions, where the first kNumConfigRegionAttributes arguments have
+  // `index` type and the rest have the same types as the data operands.
   Region *kernelRegion = result.addRegion();
   Block *body = new Block();
+  // TODO: Allow passing in proper locations here.
   for (unsigned i = 0; i < kNumConfigRegionAttributes; ++i)
     body->addArgument(builder.getIndexType(), result.location);
+  // Add WorkGroup & Private attributions to the region arguments.
+  for (Type argTy : workgroupAttributions)
+    body->addArgument(argTy, result.location);
+  for (Type argTy : privateAttributions)
+    body->addArgument(argTy, result.location);
   kernelRegion->push_back(body);
+  // Fill OperandSegmentSize Attribute.
   SmallVector<int32_t, 8> segmentSizes(8, 1);
   segmentSizes.front() = asyncDependencies.size();
   segmentSizes.back() = dynamicSharedMemorySize ? 1 : 0;
@@ -504,13 +573,18 @@ LogicalResult LaunchOp::verifyRegions() {
   // sizes and transforms them into kNumConfigRegionAttributes region arguments
   // for block/thread identifiers and grid/block sizes.
   if (!getBody().empty()) {
-    if (getBody().getNumArguments() !=
-        LaunchOp::kNumConfigOperands + getNumOperands() -
-            (getDynamicSharedMemorySize() ? 1 : 0) -
-            getAsyncDependencies().size())
+    if (getBody().getNumArguments() <
+        kNumConfigRegionAttributes + getNumWorkgroupAttributions())
       return emitOpError("unexpected number of region arguments");
   }
 
+  // Verify Attributions Address Spaces.
+  if (failed(verifyAttributions(getOperation(), getWorkgroupAttributions(),
+                                GPUDialect::getWorkgroupAddressSpace())) ||
+      failed(verifyAttributions(getOperation(), getPrivateAttributions(),
+                                GPUDialect::getPrivateAddressSpace())))
+    return failure();
+
   // Block terminators without successors are expected to exit the kernel region
   // and must be `gpu.terminator`.
   for (Block &block : getBody()) {
@@ -563,10 +637,15 @@ void LaunchOp::print(OpAsmPrinter &p) {
     p << ' ' << getDynamicSharedMemorySizeKeyword() << ' '
       << getDynamicSharedMemorySize();
 
+  printAttributions(p, getWorkgroupKeyword(), getWorkgroupAttributions());
+  printAttributions(p, getPrivateKeyword(), getPrivateAttributions());
+
   p << ' ';
+
   p.printRegion(getBody(), /*printEntryBlockArgs=*/false);
   p.printOptionalAttrDict((*this)->getAttrs(), /*elidedAttrs=*/{
-                              LaunchOp::getOperandSegmentSizeAttr()});
+                              LaunchOp::getOperandSegmentSizeAttr(),
+                              getNumWorkgroupAttributionsAttrName()});
 }
 
 // Parse the size assignment blocks for blocks and threads.  These have the form
@@ -601,8 +680,9 @@ parseSizeAssignment(OpAsmParser &parser,
 
 /// Parses a Launch operation.
 /// operation ::= `gpu.launch` (`async` `[` ssa-id-list `]`)?
-//        `blocks` `(` ssa-id-list `)` `in` ssa-reassignment
+///       `blocks` `(` ssa-id-list `)` `in` ssa-reassignment
 ///       `threads` `(` ssa-id-list `)` `in` ssa-reassignment
+///       memory-attribution
 ///       region attr-dict?
 /// ssa-reassignment ::= `(` ssa-id `=` ssa-use (`,` ssa-id `=` ssa-use)* `)`
 ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) {
@@ -659,9 +739,12 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) {
       return failure();
   }
 
-  // Introduce the body region and parse it. The region has
-  // kNumConfigRegionAttributes arguments that correspond to
-  // block/thread identifiers and grid/block sizes, all of the `index` type.
+  // Create the region arguments, it has kNumConfigRegionAttributes arguments
+  // that correspond to block/thread identifiers and grid/block sizes, all
+  // having `index` type, a variadic number of WorkGroup Attributions and
+  // a variadic number of Private Attributions. The number of WorkGroup
+  // Attributions is stored in the attr with name:
+  // LaunchOp::getNumWorkgroupAttributionsAttrName().
   Type index = parser.getBuilder().getIndexType();
   SmallVector<Type, LaunchOp::kNumConfigRegionAttributes> dataTypes(
       LaunchOp::kNumConfigRegionAttributes, index);
@@ -674,6 +757,27 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) {
     regionArguments.push_back(arg);
   }
 
+  Builder &builder = parser.getBuilder();
+  // Parse workgroup memory attributions.
+  if (failed(parseAttributions(parser, LaunchOp::getWorkgroupKeyword(),
+                               regionArguments)))
+    return failure();
+
+  // Store the number of operands we just parsed as the number of workgroup
+  // memory attributions.
+  unsigned numWorkgroupAttrs =
+      regionArguments.size() - LaunchOp::kNumConfigRegionAttributes;
+  result.addAttribute(LaunchOp::getNumWorkgroupAttributionsAttrName(),
+                      builder.getI64IntegerAttr(numWorkgroupAttrs));
+
+  // Parse private memory attributions.
+  if (failed(parseAttributions(parser, LaunchOp::getPrivateKeyword(),
+                               regionArguments)))
+    return failure();
+
+  // Introduce the body region and parse it. The region has
+  // kNumConfigRegionAttributes arguments that correspond to
+  // block/thread identifiers and grid/block sizes, all having `index` type.
   Region *body = result.addRegion();
   if (parser.parseRegion(*body, regionArguments) ||
       parser.parseOptionalAttrDict(result.attributes))
@@ -729,6 +833,25 @@ void LaunchOp::getCanonicalizationPatterns(RewritePatternSet &rewrites,
   rewrites.add<FoldLaunchArguments>(context);
 }
 
+/// Adds a new block argument that corresponds to buffers located in
+/// workgroup memory.
+BlockArgument LaunchOp::addWorkgroupAttribution(Type type, Location loc) {
+  auto attrName = getNumWorkgroupAttributionsAttrName();
+  auto attr = (*this)->getAttrOfType<IntegerAttr>(attrName);
+  (*this)->setAttr(attrName,
+                   IntegerAttr::get(attr.getType(), attr.getValue() + 1));
+  return getBody().insertArgument(
+      LaunchOp::kNumConfigRegionAttributes + attr.getInt(), type, loc);
+}
+
+/// Adds a new block argument that corresponds to buffers located in
+/// private memory.
+BlockArgument LaunchOp::addPrivateAttribution(Type type, Location loc) {
+  // Buffers on the private memory always come after buffers on the workgroup
+  // memory.
+  return getBody().addArgument(type, loc);
+}
+
 //===----------------------------------------------------------------------===//
 // LaunchFuncOp
 //===----------------------------------------------------------------------===//
@@ -903,13 +1026,38 @@ void GPUFuncOp::build(OpBuilder &builder, OperationState &result,
 /// keyword provided as argument.
 static ParseResult
 parseAttributions(OpAsmParser &parser, StringRef keyword,
-                  SmallVectorImpl<OpAsmParser::Argument> &args) {
+                  SmallVectorImpl<OpAsmParser::Argument> &args,
+                  Attribute &attributionAttrs) {
   // If we could not parse the keyword, just assume empty list and succeed.
   if (failed(parser.parseOptionalKeyword(keyword)))
     return success();
 
-  return parser.parseArgumentList(args, OpAsmParser::Delimiter::Paren,
-                                  /*allowType=*/true);
+  size_t existingArgs = args.size();
+  ParseResult result =
+      parser.parseArgumentList(args, OpAsmParser::Delimiter::Paren,
+                               /*allowType=*/true, /*allowAttrs=*/true);
+  if (failed(result))
+    return result;
+
+  bool hadAttrs = llvm::any_of(ArrayRef(args).drop_front(existingArgs),
+                               [](const OpAsmParser::Argument &arg) -> bool {
+                                 return arg.attrs && !arg.attrs.empty();
+                               });
+  if (!hadAttrs) {
+    attributionAttrs = nullptr;
+    return result;
+  }
+
+  Builder &builder = parser.getBuilder();
+  SmallVector<Attribute> attributionAttrsVec;
+  for (const auto &argument : ArrayRef(args).drop_front(existingArgs)) {
+    if (!argument.attrs)
+      attributionAttrsVec.push_back(builder.getDictionaryAttr({}));
+    else
+      attributionAttrsVec.push_back(argument.attrs);
+  }
+  attributionAttrs = builder.getArrayAttr(attributionAttrsVec);
+  return result;
 }
 
 /// Parses a GPU function.
@@ -954,9 +1102,10 @@ ParseResult GPUFuncOp::parse(OpAsmParser &parser, OperationState &result) {
       builder, result, entryArgs, resultAttrs, getArgAttrsAttrName(result.name),
       getResAttrsAttrName(result.name));
 
+  Attribute workgroupAttributionAttrs;
   // Parse workgroup memory attributions.
   if (failed(parseAttributions(parser, GPUFuncOp::getWorkgroupKeyword(),
-                               entryArgs)))
+                               entryArgs, workgroupAttributionAttrs)))
     return failure();
 
   // Store the number of operands we just parsed as the number of workgroup
@@ -964,11 +1113,18 @@ ParseResult GPUFuncOp::parse(OpAsmParser &parser, OperationState &result) {
   unsigned numWorkgroupAttrs = entryArgs.size() - type.getNumInputs();
   result.addAttribute(GPUFuncOp::getNumWorkgroupAttributionsAttrName(),
                       builder.getI64IntegerAttr(numWorkgroupAttrs));
+  if (workgroupAttributionAttrs)
+    result.addAttribute(GPUFuncOp::getWorkgroupAttribAttrsAttrName(result.name),
+                        workgroupAttributionAttrs);
 
+  Attribute privateAttributionAttrs;
   // Parse private memory attributions.
-  if (failed(
-          parseAttributions(parser, GPUFuncOp::getPrivateKeyword(), entryArgs)))
+  if (failed(parseAttributions(parser, GPUFuncOp::getPrivateKeyword(),
+                               entryArgs, privateAttributionAttrs)))
     return failure();
+  if (privateAttributionAttrs)
+    result.addAttribute(GPUFuncOp::getPrivateAttribAttrsAttrName(result.name),
+                        privateAttributionAttrs);
 
   // Parse the kernel attribute if present.
   if (succeeded(parser.parseOptionalKeyword(GPUFuncOp::getKernelKeyword())))
@@ -986,13 +1142,24 @@ ParseResult GPUFuncOp::parse(OpAsmParser &parser, OperationState &result) {
 }
 
 static void printAttributions(OpAsmPrinter &p, StringRef keyword,
-                              ArrayRef<BlockArgument> values) {
+                              ArrayRef<BlockArgument> values,
+                              ArrayAttr attributes) {
   if (values.empty())
     return;
 
   p << ' ' << keyword << '(';
   llvm::interleaveComma(
-      values, p, [&p](BlockArgument v) { p << v << " : " << v.getType(); });
+      llvm::enumerate(values), p, [&p, attributes](auto pair) {
+        BlockArgument v = pair.value();
+        p << v << " : " << v.getType();
+
+        size_t attributionIndex = pair.index();
+        DictionaryAttr attrs;
+        if (attributes && attributionIndex < attributes.size())
+          attrs = attributes[attributionIndex].cast<DictionaryAttr>();
+        if (attrs)
+          p.printOptionalAttrDict(attrs.getValue());
+      });
   p << ')';
 }
 
@@ -1005,8 +1172,10 @@ void GPUFuncOp::print(OpAsmPrinter &p) {
                                                   /*isVariadic=*/false,
                                                   type.getResults());
 
-  printAttributions(p, getWorkgroupKeyword(), getWorkgroupAttributions());
-  printAttributions(p, getPrivateKeyword(), getPrivateAttributions());
+  printAttributions(p, getWorkgroupKeyword(), getWorkgroupAttributions(),
+                    getWorkgroupAttribAttrs().value_or(nullptr));
+  printAttributions(p, getPrivateKeyword(), getPrivateAttributions(),
+                    getPrivateAttribAttrs().value_or(nullptr));
   if (isKernel())
     p << ' ' << getKernelKeyword();
 
@@ -1014,37 +1183,134 @@ void GPUFuncOp::print(OpAsmPrinter &p) {
       p, *this,
       {getNumWorkgroupAttributionsAttrName(),
        GPUDialect::getKernelFuncAttrName(), getFunctionTypeAttrName(),
-       getArgAttrsAttrName(), getResAttrsAttrName()});
+       getArgAttrsAttrName(), getResAttrsAttrName(),
+       getWorkgroupAttribAttrsAttrName(), getPrivateAttribAttrsAttrName()});
   p << ' ';
   p.printRegion(getBody(), /*printEntryBlockArgs=*/false);
 }
 
-LogicalResult GPUFuncOp::verifyType() {
-  if (isKernel() && getFunctionType().getNumResults() != 0)
-    return emitOpError() << "expected void return type for kernel function";
+static DictionaryAttr getAttributionAttrs(GPUFuncOp op, unsigned index,
+                                          StringAttr attrName) {
+  auto allAttrs = op->getAttr(attrName).dyn_cast_or_null<ArrayAttr>();
+  if (!allAttrs || index >= allAttrs.size())
+    return DictionaryAttr();
+  return allAttrs[index].cast<DictionaryAttr>();
+}
+
+DictionaryAttr GPUFuncOp::getworkgroupAttributionAttrs(unsigned index) {
+  return getAttributionAttrs(*this, index, getWorkgroupAttribAttrsAttrName());
+}
+
+DictionaryAttr GPUFuncOp::getPrivateAttributionAttrs(unsigned index) {
+  return getAttributionAttrs(*this, index, getPrivateAttribAttrsAttrName());
+}
+
+static void setAttributionAttrs(GPUFuncOp op, unsigned index,
+                                DictionaryAttr value, StringAttr attrName) {
+  MLIRContext *ctx = op.getContext();
+  auto allAttrs = op->getAttr(attrName).dyn_cast_or_null<ArrayAttr>();
+  SmallVector<Attribute> elements;
+  if (allAttrs)
+    elements.append(allAttrs.begin(), allAttrs.end());
+  while (elements.size() <= index)
+    elements.push_back(DictionaryAttr::get(ctx));
+  if (!value)
+    elements[index] = DictionaryAttr::get(ctx);
+  else
+    elements[index] = value;
+  ArrayAttr newValue = ArrayAttr::get(ctx, elements);
+  op->setAttr(attrName, newValue);
+}
+
+void GPUFuncOp::setworkgroupAttributionAttrs(unsigned index,
+                                             DictionaryAttr value) {
+  setAttributionAttrs(*this, index, value, getWorkgroupAttribAttrsAttrName());
+}
+
+void GPUFuncOp::setPrivateAttributionAttrs(unsigned int index,
+                                           DictionaryAttr value) {
+  setAttributionAttrs(*this, index, value, getPrivateAttribAttrsAttrName());
+}
+
+static Attribute getAttributionAttr(GPUFuncOp op, unsigned index,
+                                    StringAttr name, StringAttr attrsName) {
+  DictionaryAttr dict = getAttributionAttrs(op, index, attrsName);
+  if (!dict)
+    return Attribute();
+  return dict.get(name);
+}
+
+Attribute GPUFuncOp::getWorkgroupAttributionAttr(unsigned index,
+                                                 StringAttr name) {
+  assert(index < getNumWorkgroupAttributions() &&
+         "index must map to a workgroup attribution");
+  return getAttributionAttr(*this, index, name,
+                            getWorkgroupAttribAttrsAttrName());
+}
+
+Attribute GPUFuncOp::getPrivateAttributionAttr(unsigned index,
+                                               StringAttr name) {
+  assert(index < getNumPrivateAttributions() &&
+         "index must map to a private attribution");
+  return getAttributionAttr(*this, index, name,
+                            getPrivateAttribAttrsAttrName());
+}
+
+static void setAttributionAttr(GPUFuncOp op, unsigned index, StringAttr name,
+                               Attribute value, StringAttr attrsName) {
+  MLIRContext *ctx = op.getContext();
+  SmallVector<NamedAttribute> elems;
+  DictionaryAttr oldDict = getAttributionAttrs(op, index, attrsName);
+  if (oldDict)
+    elems.append(oldDict.getValue().begin(), oldDict.getValue().end());
+
+  bool found = false;
+  bool mustSort = true;
+  for (unsigned i = 0, e = elems.size(); i < e; ++i) {
+    if (elems[i].getName() == name) {
+      found = true;
+      if (!value) {
+        std::swap(elems[i], elems[elems.size() - 1]);
+        elems.pop_back();
+      } else {
+        mustSort = false;
+        elems[i] = NamedAttribute(elems[i].getName(), value);
+      }
+      break;
+    }
+  }
+  if (!found) {
+    if (!value)
+      return;
+    elems.emplace_back(name, value);
+  }
+  if (mustSort) {
+    DictionaryAttr::sortInPlace(elems);
+  }
+  auto newDict = DictionaryAttr::getWithSorted(ctx, elems);
+  setAttributionAttrs(op, index, newDict, attrsName);
+}
 
-  return success();
+void GPUFuncOp::setWorkgroupAttributionAttr(unsigned index, StringAttr name,
+                                            Attribute value) {
+  assert(index < getNumWorkgroupAttributions() &&
+         "index must map to a workgroup attribution");
+  setAttributionAttr(*this, index, name, value,
+                     getWorkgroupAttribAttrsAttrName());
 }
 
-static LogicalResult verifyAttributions(Operation *op,
-                                        ArrayRef<BlockArgument> attributions,
-                                        gpu::AddressSpace memorySpace) {
-  for (Value v : attributions) {
-    auto type = v.getType().dyn_cast<MemRefType>();
-    if (!type)
-      return op->emitOpError() << "expected memref type in attribution";
+void GPUFuncOp::setPrivateAttributionAttr(unsigned index, StringAttr name,
+                                          Attribute value) {
+  assert(index < getNumPrivateAttributions() &&
+         "index must map to a private attribution");
+  setAttributionAttr(*this, index, name, value,
+                     getPrivateAttribAttrsAttrName());
+}
+
+LogicalResult GPUFuncOp::verifyType() {
+  if (isKernel() && getFunctionType().getNumResults() != 0)
+    return emitOpError() << "expected void return type for kernel function";
 
-    // We can only verify the address space if it hasn't already been lowered
-    // from the AddressSpaceAttr to a target-specific numeric value.
-    auto addressSpace =
-        type.getMemorySpace().dyn_cast_or_null<gpu::AddressSpaceAttr>();
-    if (!addressSpace)
-      continue;
-    if (addressSpace.getValue() != memorySpace)
-      return op->emitOpError()
-             << "expected memory space " << stringifyAddressSpace(memorySpace)
-             << " in attribution";
-  }
   return success();
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index cb3cebef79730..0890bf2677626 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -113,6 +113,7 @@ struct GpuAsyncRegionPass::ThreadTokenCallback {
     resultTypes.push_back(tokenType);
     auto *newOp = Operation::create(op->getLoc(), op->getName(), resultTypes,
                                     op->getOperands(), op->getAttrDictionary(),
+                                    op->getPropertiesStorage(),
                                     op->getSuccessors(), op->getNumRegions());
 
     // Clone regions into new op.
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 8abf75993a63d..91c1c763f070d 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -190,7 +190,10 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   }
   FunctionType type =
       FunctionType::get(launchOp.getContext(), kernelOperandTypes, {});
-  auto outlinedFunc = builder.create<gpu::GPUFuncOp>(loc, kernelFnName, type);
+  auto outlinedFunc = builder.create<gpu::GPUFuncOp>(
+      loc, kernelFnName, type,
+      TypeRange(ValueRange(launchOp.getWorkgroupAttributions())),
+      TypeRange(ValueRange(launchOp.getPrivateAttributions())));
   outlinedFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
                         builder.getUnitAttr());
 
@@ -213,6 +216,16 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   Region &outlinedFuncBody = outlinedFunc.getBody();
   injectGpuIndexOperations(loc, outlinedFuncBody, launchOpBody, map);
 
+  // Map memory attributions from the LaunOp op to the GPUFuncOp attributions.
+  for (const auto &[launchArg, funcArg] :
+       llvm::zip(launchOp.getWorkgroupAttributions(),
+                 outlinedFunc.getWorkgroupAttributions()))
+    map.map(launchArg, funcArg);
+  for (const auto &[launchArg, funcArg] :
+       llvm::zip(launchOp.getPrivateAttributions(),
+                 outlinedFunc.getPrivateAttributions()))
+    map.map(launchArg, funcArg);
+
   // Map arguments from gpu.launch region to the arguments of the gpu.func
   // operation.
   Block &entryBlock = outlinedFuncBody.front();
diff --git a/mlir/lib/Dialect/IRDL/AttributeWrapper.cpp b/mlir/lib/Dialect/IRDL/AttributeWrapper.cpp
new file mode 100644
index 0000000000000..798bc45d5699b
--- /dev/null
+++ b/mlir/lib/Dialect/IRDL/AttributeWrapper.cpp
@@ -0,0 +1,33 @@
+//===- AttributeWrapper.cpp - IRDL type wrapper definition ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/Dialect/IRDL/IR/IRDL.h"
+
+namespace mlir {
+namespace irdl {
+
+bool TypeWrapper::isCorrectAttribute(Attribute attr) {
+  return isa<TypeAttr>(attr) && isCorrectType(cast<TypeAttr>(attr).getValue());
+}
+
+SmallVector<Attribute> TypeWrapper::getAttributeParameters(Attribute attr) {
+  return getTypeParameters(cast<TypeAttr>(attr).getValue());
+}
+
+Attribute
+TypeWrapper::instantiateAttribute(function_ref<InFlightDiagnostic()> emitError,
+                                  ArrayRef<Attribute> parameters) {
+  Type type = instantiateType(emitError, parameters);
+  if (!type)
+    return {};
+  return TypeAttr::get(type);
+}
+
+} // namespace irdl
+} // namespace mlir
diff --git a/mlir/lib/Dialect/IRDL/CMakeLists.txt b/mlir/lib/Dialect/IRDL/CMakeLists.txt
index 31efd5e37a665..45aee73ab9b7f 100644
--- a/mlir/lib/Dialect/IRDL/CMakeLists.txt
+++ b/mlir/lib/Dialect/IRDL/CMakeLists.txt
@@ -1,6 +1,11 @@
 add_mlir_dialect_library(MLIRIRDL
   IR/IRDL.cpp
-  IRDLLoading.cpp
+  IR/IRDLAttributes.cpp
+  IR/IRDLOps.cpp
+  AttributeWrapper.cpp
+  IRDLLoading.cpp  
+  IRDLVerifiers.cpp
+  IRDLContext.cpp
 
   DEPENDS
   MLIRIRDLIncGen
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
index e2649f22094c7..ac130448ae87b 100644
--- a/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
+++ b/mlir/lib/Dialect/IRDL/IR/IRDL.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/Dialect/IRDL/IRDLContext.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
@@ -21,6 +23,7 @@
 
 using namespace mlir;
 using namespace mlir::irdl;
+using mlir::irdl::TypeWrapper;
 
 //===----------------------------------------------------------------------===//
 // IRDL dialect.
@@ -39,6 +42,10 @@ void IRDLDialect::initialize() {
 #define GET_TYPEDEF_LIST
 #include "mlir/Dialect/IRDL/IR/IRDLTypesGen.cpp.inc"
       >();
+  addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/IRDL/IR/IRDLAttributes.cpp.inc"
+      >();
 }
 
 //===----------------------------------------------------------------------===//
@@ -71,8 +78,13 @@ LogicalResult DialectOp::verify() {
   return success();
 }
 
+#include "mlir/Dialect/IRDL/IR/IRDLInterfaces.cpp.inc"
+
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/IRDL/IR/IRDLTypesGen.cpp.inc"
 
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/IRDL/IR/IRDLAttributes.cpp.inc"
+
 #define GET_OP_CLASSES
 #include "mlir/Dialect/IRDL/IR/IRDLOps.cpp.inc"
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDLAttributes.cpp b/mlir/lib/Dialect/IRDL/IR/IRDLAttributes.cpp
new file mode 100644
index 0000000000000..a07d96d310c1d
--- /dev/null
+++ b/mlir/lib/Dialect/IRDL/IR/IRDLAttributes.cpp
@@ -0,0 +1,93 @@
+//===- IRDLAttributes.cpp - IRDL dialect ------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/Dialect/IRDL/IR/IRDL.h"
+
+using namespace mlir;
+using namespace mlir::irdl;
+
+LogicalResult TypeOrAttrDefRefAttr::verify(
+    function_ref<InFlightDiagnostic()> emitError, TypeWrapper *typeWrapper,
+    AttributeWrapper *attrWrapper, SymbolRefAttr symRef) {
+  if (!typeWrapper && !attrWrapper && !symRef) {
+    emitError()
+        << "expected a type wrapper, attribute wrapper, or symbol reference";
+    return failure();
+  }
+  if ((symRef && (attrWrapper || typeWrapper)) ||
+      (attrWrapper && typeWrapper)) {
+    emitError() << "expected either a type wrapper, an attribute wrapper, or a "
+                   "symbol reference, but not more than one";
+    return failure();
+  }
+  return success();
+}
+
+Attribute TypeOrAttrDefRefAttr::parse(AsmParser &parser, Type type) {
+  auto loc = parser.getCurrentLocation();
+
+  std::string stringLit;
+  auto litStringRes = parser.parseOptionalString(&stringLit);
+
+  // Wrapper cases.
+  if (litStringRes.succeeded()) {
+    auto ctx = parser.getBuilder().getContext();
+    auto irdl = ctx->getOrLoadDialect<IRDLDialect>();
+
+    if (stringLit.empty() || (stringLit[0] != '!' && stringLit[0] != '#')) {
+      parser.emitError(loc) << "attribute or type wrapper name should "
+                               "be prefixed with either '!' or '#'";
+      return {};
+    }
+
+    StringRef wrapperName = StringRef(stringLit).substr(1);
+
+    // Type wrapper case
+    if (stringLit[0] == '!') {
+      auto *typeWrapper = irdl->irdlContext.getTypeWrapper(wrapperName);
+      if (typeWrapper)
+        return TypeOrAttrDefRefAttr::get(ctx, typeWrapper);
+      parser.emitError(loc)
+          << "type wrapper " << wrapperName << " is not registered";
+      return {};
+    }
+
+    // Attribute wrapper case
+    auto *attrWrapper = irdl->irdlContext.getAttributeWrapper(wrapperName);
+    if (attrWrapper)
+      return TypeOrAttrDefRefAttr::get(ctx, attrWrapper);
+
+    parser.emitError(loc) << "attribute wrapper " << wrapperName
+                          << " is not registered";
+    return {};
+  }
+
+  // Symref cases
+  Attribute attr;
+  auto res = parser.parseAttribute(attr);
+
+  if (res.failed())
+    return {};
+  if (auto symRef = llvm::dyn_cast<SymbolRefAttr>(attr))
+    return TypeOrAttrDefRefAttr::get(symRef);
+  parser.emitError(loc) << "expected a string literal, or a symbol reference";
+  return {};
+}
+
+void TypeOrAttrDefRefAttr::print(AsmPrinter &odsPrinter) const {
+  if (auto *typeWrapper = getTypeWrapper())
+    odsPrinter << "\"!" << typeWrapper->getName() << '"';
+  else if (auto *attrWrapper = getAttrWrapper())
+    odsPrinter << "\"#" << attrWrapper->getName() << '"';
+  else if (auto symRef = getSymRef())
+    odsPrinter << symRef;
+  else
+    llvm_unreachable("expected a type wrapper, attribute wrapper, or symbol "
+                     "reference");
+}
diff --git a/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
new file mode 100644
index 0000000000000..17c4a0c5cb48e
--- /dev/null
+++ b/mlir/lib/Dialect/IRDL/IR/IRDLOps.cpp
@@ -0,0 +1,103 @@
+//===- IRDLOps.cpp - IRDL dialect -------------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/IRDL/AttributeWrapper.h"
+#include "mlir/Dialect/IRDL/IR/IRDL.h"
+
+using namespace mlir;
+using namespace mlir::irdl;
+
+std::unique_ptr<Constraint> Is::getVerifier(
+    SmallVector<Value> const &valueToConstr,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  return std::make_unique<IsConstraint>(getExpectedAttr());
+}
+
+std::unique_ptr<Constraint> Parametric::getVerifier(
+    SmallVector<Value> const &valueToConstr,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  SmallVector<unsigned> constraints;
+  for (Value arg : getArgs()) {
+    for (auto [i, value] : enumerate(valueToConstr)) {
+      if (value == arg) {
+        constraints.push_back(i);
+        break;
+      }
+    }
+  }
+
+  // Type or attribute wrapper case for the base
+  if (TypeWrapper *typeWrapper = getBaseType().getTypeWrapper())
+    return std::make_unique<ParametricConstraint>(typeWrapper, constraints);
+
+  if (AttributeWrapper *attrWrapper = getBaseType().getAttrWrapper())
+    return std::make_unique<ParametricConstraint>(attrWrapper, constraints);
+
+  // Symbol reference case for the base
+  SymbolRefAttr symRef = getBaseType().getSymRef();
+  Operation *defOp =
+      SymbolTable::lookupNearestSymbolFrom(getOperation(), symRef);
+  if (!defOp) {
+    emitError() << symRef << " does not refer to any existing symbol";
+    return nullptr;
+  }
+
+  if (auto typeOp = dyn_cast<TypeOp>(defOp))
+    return std::make_unique<DynParametricTypeConstraint>(types[typeOp].get(),
+                                                         constraints);
+
+  if (auto attrOp = dyn_cast<AttributeOp>(defOp))
+    return std::make_unique<DynParametricAttrConstraint>(attrs[attrOp].get(),
+                                                         constraints);
+
+  llvm_unreachable("verifier should ensure that the referenced operation is "
+                   "either a type or an attribute definition");
+}
+
+std::unique_ptr<Constraint> AnyOf::getVerifier(
+    SmallVector<Value> const &valueToConstr,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  SmallVector<unsigned> constraints;
+  for (Value arg : getArgs()) {
+    for (auto [i, value] : enumerate(valueToConstr)) {
+      if (value == arg) {
+        constraints.push_back(i);
+        break;
+      }
+    }
+  }
+
+  return std::make_unique<AnyOfConstraint>(constraints);
+}
+
+std::unique_ptr<Constraint> AllOf::getVerifier(
+    SmallVector<Value> const &valueToConstr,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  SmallVector<unsigned> constraints;
+  for (Value arg : getArgs()) {
+    for (auto [i, value] : enumerate(valueToConstr)) {
+      if (value == arg) {
+        constraints.push_back(i);
+        break;
+      }
+    }
+  }
+
+  return std::make_unique<AllOfConstraint>(constraints);
+}
+
+std::unique_ptr<Constraint> Any::getVerifier(
+    SmallVector<Value> const &valueToConstr,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  return std::make_unique<AnyAttributeConstraint>();
+}
diff --git a/mlir/lib/Dialect/IRDL/IRDLContext.cpp b/mlir/lib/Dialect/IRDL/IRDLContext.cpp
new file mode 100644
index 0000000000000..d8f3ed31386d9
--- /dev/null
+++ b/mlir/lib/Dialect/IRDL/IRDLContext.cpp
@@ -0,0 +1,69 @@
+//===- IRDLContext.cpp - IRDL context ---------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/IRDL/IRDLContext.h"
+#include "mlir/IR/BuiltinTypes.h"
+
+using namespace mlir;
+using namespace irdl;
+
+/// A wrapper around the 'buitin.complex' type.
+class ComplexTypeWrapper : public CppTypeWrapper<ComplexType> {
+  StringRef getName() override { return "builtin.complex"; }
+
+  SmallVector<Attribute> getTypeParameters(ComplexType type) override {
+    return {TypeAttr::get(type.getElementType())};
+  }
+
+  size_t getParameterAmount() override { return 1; }
+
+  Type instantiateType(function_ref<InFlightDiagnostic()> emitError,
+                       ArrayRef<Attribute> parameters) override {
+    if (parameters.size() != this->getParameterAmount()) {
+      emitError().append("invalid number of type parameters ",
+                         parameters.size(), " (expected ",
+                         this->getParameterAmount(), ")");
+      return Type();
+    }
+
+    return ComplexType::getChecked(emitError,
+                                   cast<TypeAttr>(parameters[0]).getValue());
+  }
+};
+
+IRDLContext::IRDLContext() { addTypeWrapper<ComplexTypeWrapper>(); }
+
+void IRDLContext::addAttributeWrapper(
+    std::unique_ptr<AttributeWrapper> wrapper) {
+  auto emplaced =
+      typeIDToAttributeWrapper.insert({wrapper->getTypeID(), wrapper.get()});
+  assert(emplaced.second &&
+         "an attribute wrapper with the same name already exists");
+  attributes.try_emplace(wrapper->getName(), std::move(wrapper));
+}
+
+AttributeWrapper *IRDLContext::getAttributeWrapper(StringRef attrName) {
+  auto it = attributes.find(attrName);
+  if (it == attributes.end())
+    return nullptr;
+  return it->second.get();
+}
+
+void IRDLContext::addTypeWrapper(std::unique_ptr<TypeWrapper> wrapper) {
+  auto emplaced =
+      typeIDToTypeWrapper.insert({wrapper->getTypeID(), wrapper.get()});
+  assert(emplaced.second && "a type wrapper with the same name already exists");
+  types.try_emplace(wrapper->getName(), std::move(wrapper));
+}
+
+TypeWrapper *IRDLContext::getTypeWrapper(StringRef typeName) {
+  auto it = types.find(typeName);
+  if (it == types.end())
+    return nullptr;
+  return it->second.get();
+}
diff --git a/mlir/lib/Dialect/IRDL/IRDLLoading.cpp b/mlir/lib/Dialect/IRDL/IRDLLoading.cpp
index fb00085a7ee07..90c847e82d638 100644
--- a/mlir/lib/Dialect/IRDL/IRDLLoading.cpp
+++ b/mlir/lib/Dialect/IRDL/IRDLLoading.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/IRDL/IRDLLoading.h"
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
+#include "mlir/Dialect/IRDL/IR/IRDLInterfaces.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/ExtensibleDialect.h"
 #include "mlir/Support/LogicalResult.h"
@@ -22,9 +23,139 @@
 using namespace mlir;
 using namespace mlir::irdl;
 
+/// Verify that the given list of parameters satisfy the given constraints.
+/// This encodes the logic of the verification method for attributes and types
+/// defined with IRDL.
+static LogicalResult
+irdlAttrOrTypeVerifier(function_ref<InFlightDiagnostic()> emitError,
+                       ArrayRef<Attribute> params,
+                       ArrayRef<std::unique_ptr<Constraint>> constraints,
+                       ArrayRef<size_t> paramConstraints) {
+  if (params.size() != paramConstraints.size()) {
+    emitError() << "expected " << paramConstraints.size()
+                << " type arguments, but had " << params.size();
+    return failure();
+  }
+
+  ConstraintVerifier verifier(constraints);
+
+  // Check that each parameter satisfies its constraint.
+  for (auto [i, param] : enumerate(params))
+    if (failed(verifier.verify(emitError, param, paramConstraints[i])))
+      return failure();
+
+  return success();
+}
+
+/// Verify that the given operation satisfies the given constraints.
+/// This encodes the logic of the verification method for operations defined
+/// with IRDL.
+static LogicalResult
+irdlOpVerifier(Operation *op, ArrayRef<std::unique_ptr<Constraint>> constraints,
+               ArrayRef<size_t> operandConstrs,
+               ArrayRef<size_t> resultConstrs) {
+  /// Check that we have the right number of operands.
+  unsigned numOperands = op->getNumOperands();
+  size_t numExpectedOperands = operandConstrs.size();
+  if (numOperands != numExpectedOperands)
+    return op->emitOpError(std::to_string(numExpectedOperands) +
+                           " operands expected, but got " +
+                           std::to_string(numOperands));
+
+  /// Check that we have the right number of results.
+  unsigned numResults = op->getNumResults();
+  size_t numExpectedResults = resultConstrs.size();
+  if (numResults != numExpectedResults)
+    return op->emitOpError(std::to_string(numExpectedResults) +
+                           " results expected, but got " +
+                           std::to_string(numResults));
+
+  auto emitError = [op]() { return op->emitError(); };
+
+  ConstraintVerifier verifier(constraints);
+
+  /// Check that all operands satisfy the constraints.
+  for (auto [i, operandType] : enumerate(op->getOperandTypes())) {
+    if (failed(verifier.verify({emitError}, TypeAttr::get(operandType),
+                               operandConstrs[i]))) {
+      return failure();
+    }
+  }
+
+  /// Check that all results satisfy the constraints.
+  for (auto [i, resultType] : enumerate(op->getResultTypes())) {
+    if (failed(verifier.verify({emitError}, TypeAttr::get(resultType),
+                               resultConstrs[i]))) {
+      return failure();
+    }
+  }
+
+  return success();
+}
+
 /// Define and load an operation represented by a `irdl.operation`
 /// operation.
-static WalkResult loadOperation(OperationOp op, ExtensibleDialect *dialect) {
+static WalkResult loadOperation(
+    OperationOp op, ExtensibleDialect *dialect,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  // Resolve SSA values to verifier constraint slots
+  SmallVector<Value> constrToValue;
+  for (Operation &op : op->getRegion(0).getOps()) {
+    if (isa<VerifyConstraintInterface>(op)) {
+      if (op.getNumResults() != 1) {
+        op.emitError()
+            << "IRDL constraint operations must have exactly one result";
+        return WalkResult::interrupt();
+      }
+      constrToValue.push_back(op.getResult(0));
+    }
+  }
+
+  // Build the verifiers for each constraint slot
+  SmallVector<std::unique_ptr<Constraint>> constraints;
+  for (Value v : constrToValue) {
+    VerifyConstraintInterface op =
+        cast<VerifyConstraintInterface>(v.getDefiningOp());
+    std::unique_ptr<Constraint> verifier =
+        op.getVerifier(constrToValue, types, attrs);
+    if (!verifier) {
+      return WalkResult::interrupt();
+    }
+    constraints.push_back(std::move(verifier));
+  }
+
+  SmallVector<size_t> operandConstraints;
+  SmallVector<size_t> resultConstraints;
+
+  // Gather which constraint slots correspond to operand constraints
+  auto operandsOp = op.getOp<OperandsOp>();
+  if (operandsOp.has_value()) {
+    operandConstraints.reserve(operandsOp->getArgs().size());
+    for (Value operand : operandsOp->getArgs()) {
+      for (auto [i, constr] : enumerate(constrToValue)) {
+        if (constr == operand) {
+          operandConstraints.push_back(i);
+          break;
+        }
+      }
+    }
+  }
+
+  // Gather which constraint slots correspond to result constraints
+  auto resultsOp = op.getOp<ResultsOp>();
+  if (resultsOp.has_value()) {
+    resultConstraints.reserve(resultsOp->getArgs().size());
+    for (Value result : resultsOp->getArgs()) {
+      for (auto [i, constr] : enumerate(constrToValue)) {
+        if (constr == result) {
+          resultConstraints.push_back(i);
+          break;
+        }
+      }
+    }
+  }
+
   // IRDL does not support defining custom parsers or printers.
   auto parser = [](OpAsmParser &parser, OperationState &result) {
     return failure();
@@ -33,7 +164,13 @@ static WalkResult loadOperation(OperationOp op, ExtensibleDialect *dialect) {
     printer.printGenericOp(op);
   };
 
-  auto verifier = [](Operation *op) { return success(); };
+  auto verifier =
+      [constraints{std::move(constraints)},
+       operandConstraints{std::move(operandConstraints)},
+       resultConstraints{std::move(resultConstraints)}](Operation *op) {
+        return irdlOpVerifier(op, constraints, operandConstraints,
+                              resultConstraints);
+      };
 
   // IRDL does not support defining regions.
   auto regionVerifier = [](Operation *op) { return success(); };
@@ -46,6 +183,190 @@ static WalkResult loadOperation(OperationOp op, ExtensibleDialect *dialect) {
   return WalkResult::advance();
 }
 
+/// Get the verifier of a type or attribute definition.
+/// Return nullptr if the definition is invalid.
+static DynamicAttrDefinition::VerifierFn getAttrOrTypeVerifier(
+    Operation *attrOrTypeDef, ExtensibleDialect *dialect,
+    DenseMap<TypeOp, std::unique_ptr<DynamicTypeDefinition>> &types,
+    DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> &attrs) {
+  assert((isa<AttributeOp>(attrOrTypeDef) || isa<TypeOp>(attrOrTypeDef)) &&
+         "Expected an attribute or type definition");
+
+  // Resolve SSA values to verifier constraint slots
+  SmallVector<Value> constrToValue;
+  for (Operation &op : attrOrTypeDef->getRegion(0).getOps()) {
+    if (isa<VerifyConstraintInterface>(op)) {
+      assert(op.getNumResults() == 1 &&
+             "IRDL constraint operations must have exactly one result");
+      constrToValue.push_back(op.getResult(0));
+    }
+  }
+
+  // Build the verifiers for each constraint slot
+  SmallVector<std::unique_ptr<Constraint>> constraints;
+  for (Value v : constrToValue) {
+    VerifyConstraintInterface op =
+        cast<VerifyConstraintInterface>(v.getDefiningOp());
+    std::unique_ptr<Constraint> verifier =
+        op.getVerifier(constrToValue, types, attrs);
+    if (!verifier) {
+      return {};
+    }
+    constraints.push_back(std::move(verifier));
+  }
+
+  // Get the parameter definitions.
+  std::optional<ParametersOp> params;
+  if (auto attr = dyn_cast<AttributeOp>(attrOrTypeDef))
+    params = attr.getOp<ParametersOp>();
+  else if (auto type = dyn_cast<TypeOp>(attrOrTypeDef))
+    params = type.getOp<ParametersOp>();
+
+  // Gather which constraint slots correspond to parameter constraints
+  SmallVector<size_t> paramConstraints;
+  if (params.has_value()) {
+    paramConstraints.reserve(params->getArgs().size());
+    for (Value param : params->getArgs()) {
+      for (auto [i, constr] : enumerate(constrToValue)) {
+        if (constr == param) {
+          paramConstraints.push_back(i);
+          break;
+        }
+      }
+    }
+  }
+
+  auto verifier = [paramConstraints{std::move(paramConstraints)},
+                   constraints{std::move(constraints)}](
+                      function_ref<InFlightDiagnostic()> emitError,
+                      ArrayRef<Attribute> params) {
+    return irdlAttrOrTypeVerifier(emitError, params, constraints,
+                                  paramConstraints);
+  };
+  return verifier;
+}
+
+/// Get the possible bases of a constraint. Return `true` if all bases can
+/// potentially be matched.
+/// A base is a type or an attribute definition. For instance, the base of
+/// `irdl.parametric "!builtin.complex"(...)` is `builtin.complex`.
+/// This function returns the following information through arguments:
+/// - `paramIds`: the set of type or attribute IDs that are used as bases.
+/// - `paramIrdlOps`: the set of IRDL operations that are used as bases.
+/// - `isIds`: the set of type or attribute IDs that are used in `irdl.is`
+///   constraints.
+static bool getBases(Operation *op, SmallPtrSet<TypeID, 4> &paramIds,
+                     SmallPtrSet<Operation *, 4> &paramIrdlOps,
+                     SmallPtrSet<TypeID, 4> &isIds) {
+  // For `irdl.any_of`, we get the bases from all its arguments.
+  if (auto anyOf = dyn_cast<AnyOf>(op)) {
+    bool has_any = false;
+    for (Value arg : anyOf.getArgs())
+      has_any &= getBases(arg.getDefiningOp(), paramIds, paramIrdlOps, isIds);
+    return has_any;
+  }
+
+  // For `irdl.all_of`, we get the bases from the first argument.
+  // This is restrictive, but we can relax it later if needed.
+  if (auto allOf = dyn_cast<AllOf>(op))
+    return getBases(allOf.getArgs()[0].getDefiningOp(), paramIds, paramIrdlOps,
+                    isIds);
+
+  // For `irdl.parametric`, we get directly the base from the operation.
+  if (auto params = dyn_cast<Parametric>(op)) {
+    TypeOrAttrDefRefAttr baseType = params.getBaseType();
+    if (SymbolRefAttr symRef = baseType.getSymRef()) {
+      Operation *defOp = SymbolTable::lookupNearestSymbolFrom(op, symRef);
+      assert(defOp && "symbol reference should refer to an existing operation");
+      paramIrdlOps.insert(defOp);
+      return false;
+    }
+    if (TypeWrapper *typeWrapper = baseType.getTypeWrapper()) {
+      paramIds.insert(typeWrapper->getTypeID());
+      return false;
+    }
+    if (AttributeWrapper *attrWrapper = baseType.getAttrWrapper()) {
+      paramIds.insert(attrWrapper->getTypeID());
+      return false;
+    }
+    llvm_unreachable("TypeOrAttrDefAttr has invalid contents");
+  }
+
+  // For `irdl.is`, we get the base TypeID directly.
+  if (auto is = dyn_cast<Is>(op)) {
+    Attribute expected = is.getExpected();
+    isIds.insert(expected.getTypeID());
+    return false;
+  }
+
+  // For `irdl.any`, we return `false` since we can match any type or attribute
+  // base.
+  if (auto isA = dyn_cast<Any>(op))
+    return true;
+
+  llvm_unreachable("unknown IRDL constraint");
+}
+
+/// Check that an any_of is in the subset IRDL can handle.
+/// IRDL uses a greedy algorithm to match constraints. This means that if we
+/// encounter an `any_of` with multiple constraints, we will match the first
+/// constraint that is satisfied. Thus, the order of constraints matter in
+/// `any_of` with our current algorithm.
+/// In order to make the order of constraints irrelevant, we require that
+/// all `any_of` constraint parameters are disjoint. For this, we check that
+/// the base parameters are all disjoints between `parametric` operations, and
+/// that they are disjoint between `parametric` and `is` operations.
+/// This restriction will be relaxed in the future, when we will change our
+/// algorithm to be non-greedy.
+static LogicalResult checkCorrectAnyOf(AnyOf anyOf) {
+  SmallPtrSet<TypeID, 4> paramIds;
+  SmallPtrSet<Operation *, 4> paramIrdlOps;
+  SmallPtrSet<TypeID, 4> isIds;
+
+  for (Value arg : anyOf.getArgs()) {
+    Operation *argOp = arg.getDefiningOp();
+    SmallPtrSet<TypeID, 4> argParamIds;
+    SmallPtrSet<Operation *, 4> argParamIrdlOps;
+    SmallPtrSet<TypeID, 4> argIsIds;
+
+    // Get the bases of this argument. If it can match any type or attribute,
+    // then our `any_of` should not be allowed.
+    if (getBases(argOp, argParamIds, argParamIrdlOps, argIsIds))
+      return failure();
+
+    // We check that the base parameters are all disjoints between `parametric`
+    // operations, and that they are disjoint between `parametric` and `is`
+    // operations.
+    for (TypeID id : argParamIds) {
+      if (isIds.count(id))
+        return failure();
+      bool inserted = paramIds.insert(id).second;
+      if (!inserted)
+        return failure();
+    }
+
+    // We check that the base parameters are all disjoints with `irdl.is`
+    // operations.
+    for (TypeID id : isIds) {
+      if (paramIds.count(id))
+        return failure();
+      isIds.insert(id);
+    }
+
+    // We check that all `parametric` operations are disjoint. We do not
+    // need to check that they are disjoint with `is` operations, since
+    // `is` operations cannot refer to attributes defined with `irdl.parametric`
+    // operations.
+    for (Operation *op : argParamIrdlOps) {
+      bool inserted = paramIrdlOps.insert(op).second;
+      if (!inserted)
+        return failure();
+    }
+  }
+
+  return success();
+}
+
 /// Load all dialects in the given module, without loading any operation, type
 /// or attribute definitions.
 static DenseMap<DialectOp, ExtensibleDialect *> loadEmptyDialects(ModuleOp op) {
@@ -99,6 +420,15 @@ preallocateAttrDefs(ModuleOp op,
 }
 
 LogicalResult mlir::irdl::loadDialects(ModuleOp op) {
+  // First, check that all any_of constraints are in a correct form.
+  // This is to ensure we can do the verification correctly.
+  WalkResult anyOfCorrects =
+      op.walk([](AnyOf anyOf) { return (WalkResult)checkCorrectAnyOf(anyOf); });
+  if (anyOfCorrects.wasInterrupted()) {
+    op.emitError("any_of constraints are not in the correct form");
+    return failure();
+  }
+
   // Preallocate all dialects, and type and attribute definitions.
   // In particular, this allocates TypeIDs so type and attributes can have
   // verifiers that refer to each other.
@@ -108,9 +438,33 @@ LogicalResult mlir::irdl::loadDialects(ModuleOp op) {
   DenseMap<AttributeOp, std::unique_ptr<DynamicAttrDefinition>> attrs =
       preallocateAttrDefs(op, dialects);
 
+  // Set the verifier for types.
+  WalkResult res = op.walk([&](TypeOp typeOp) {
+    DynamicAttrDefinition::VerifierFn verifier = getAttrOrTypeVerifier(
+        typeOp, dialects[typeOp.getParentOp()], types, attrs);
+    if (!verifier)
+      return WalkResult::interrupt();
+    types[typeOp]->setVerifyFn(std::move(verifier));
+    return WalkResult::advance();
+  });
+  if (res.wasInterrupted())
+    return failure();
+
+  // Set the verifier for attributes.
+  res = op.walk([&](AttributeOp attrOp) {
+    DynamicAttrDefinition::VerifierFn verifier = getAttrOrTypeVerifier(
+        attrOp, dialects[attrOp.getParentOp()], types, attrs);
+    if (!verifier)
+      return WalkResult::interrupt();
+    attrs[attrOp]->setVerifyFn(std::move(verifier));
+    return WalkResult::advance();
+  });
+  if (res.wasInterrupted())
+    return failure();
+
   // Define and load all operations.
-  WalkResult res = op.walk([&](OperationOp opOp) {
-    return loadOperation(opOp, dialects[opOp.getParentOp()]);
+  res = op.walk([&](OperationOp opOp) {
+    return loadOperation(opOp, dialects[opOp.getParentOp()], types, attrs);
   });
   if (res.wasInterrupted())
     return failure();
diff --git a/mlir/lib/Dialect/IRDL/IRDLVerifiers.cpp b/mlir/lib/Dialect/IRDL/IRDLVerifiers.cpp
new file mode 100644
index 0000000000000..6478d7f6b7ede
--- /dev/null
+++ b/mlir/lib/Dialect/IRDL/IRDLVerifiers.cpp
@@ -0,0 +1,207 @@
+//===- IRDLVerifiers.cpp - IRDL verifiers ------------------------- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Verifiers for objects declared by IRDL.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/IRDL/IRDLVerifiers.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/ExtensibleDialect.h"
+#include "mlir/Support/LogicalResult.h"
+
+using namespace mlir;
+using namespace mlir::irdl;
+
+ConstraintVerifier::ConstraintVerifier(
+    ArrayRef<std::unique_ptr<Constraint>> constraints)
+    : constraints(constraints), assigned() {
+  assigned.resize(this->constraints.size());
+}
+
+LogicalResult
+ConstraintVerifier::verify(function_ref<InFlightDiagnostic()> emitError,
+                           Attribute attr, unsigned variable) {
+
+  assert(variable < constraints.size() && "invalid constraint variable");
+
+  // If the variable is already assigned, check that the attribute is the same.
+  if (assigned[variable].has_value()) {
+    if (attr == assigned[variable].value()) {
+      return success();
+    } else {
+      if (emitError)
+        return emitError() << "expected '" << assigned[variable].value()
+                           << "' but got '" << attr << "'";
+      return failure();
+    }
+  }
+
+  // Otherwise, check the constraint and assign the attribute to the variable.
+  LogicalResult result = constraints[variable]->verify(emitError, attr, *this);
+  if (succeeded(result))
+    assigned[variable] = attr;
+
+  return result;
+}
+
+LogicalResult IsConstraint::verify(function_ref<InFlightDiagnostic()> emitError,
+                                   Attribute attr,
+                                   ConstraintVerifier &context) const {
+  if (attr == expectedAttribute)
+    return success();
+
+  if (emitError)
+    return emitError() << "expected '" << expectedAttribute << "' but got '"
+                       << attr << "'";
+  return failure();
+}
+
+LogicalResult
+ParametricConstraint::verify(function_ref<InFlightDiagnostic()> emitError,
+                             Attribute attr,
+                             ConstraintVerifier &context) const {
+  // Check that the base is the expected one.
+  if (!expectedAttribute->isCorrectAttribute(attr)) {
+    if (emitError)
+      return emitError() << "expected base '" << expectedAttribute->getName()
+                         << "' but got '" << attr << "'";
+    return failure();
+  }
+
+  // Check that the parameters satisfy the constraints.
+  SmallVector<Attribute> params =
+      expectedAttribute->getAttributeParameters(attr);
+  if (params.size() != constraints.size()) {
+    if (emitError)
+      emitError() << "'" << expectedAttribute->getName() << "' expects "
+                  << params.size() << " parameters, but got "
+                  << constraints.size() << " parameters";
+    return failure();
+  }
+
+  for (size_t i = 0, s = params.size(); i < s; i++)
+    if (failed(context.verify(emitError, params[i], constraints[i])))
+      return failure();
+
+  return success();
+}
+
+LogicalResult DynParametricAttrConstraint::verify(
+    function_ref<InFlightDiagnostic()> emitError, Attribute attr,
+    ConstraintVerifier &context) const {
+
+  // Check that the base is the expected one.
+  auto dynAttr = attr.dyn_cast<DynamicAttr>();
+  if (!dynAttr || dynAttr.getAttrDef() != attrDef) {
+    if (emitError) {
+      StringRef dialectName = attrDef->getDialect()->getNamespace();
+      StringRef attrName = attrDef->getName();
+      return emitError() << "expected base attribute '" << attrName << '.'
+                         << dialectName << "' but got '" << attr << "'";
+    }
+    return failure();
+  }
+
+  // Check that the parameters satisfy the constraints.
+  ArrayRef<Attribute> params = dynAttr.getParams();
+  if (params.size() != constraints.size()) {
+    if (emitError) {
+      StringRef dialectName = attrDef->getDialect()->getNamespace();
+      StringRef attrName = attrDef->getName();
+      emitError() << "attribute '" << dialectName << "." << attrName
+                  << "' expects " << params.size() << " parameters but got "
+                  << constraints.size();
+    }
+    return failure();
+  }
+
+  for (size_t i = 0, s = params.size(); i < s; i++)
+    if (failed(context.verify(emitError, params[i], constraints[i])))
+      return failure();
+
+  return success();
+}
+
+LogicalResult DynParametricTypeConstraint::verify(
+    function_ref<InFlightDiagnostic()> emitError, Attribute attr,
+    ConstraintVerifier &context) const {
+  // Check that the base is a TypeAttr.
+  auto typeAttr = attr.dyn_cast<TypeAttr>();
+  if (!typeAttr) {
+    if (emitError)
+      return emitError() << "expected type, got attribute '" << attr;
+    return failure();
+  }
+
+  // Check that the type base is the expected one.
+  auto dynType = typeAttr.getValue().dyn_cast<DynamicType>();
+  if (!dynType || dynType.getTypeDef() != typeDef) {
+    if (emitError) {
+      StringRef dialectName = typeDef->getDialect()->getNamespace();
+      StringRef attrName = typeDef->getName();
+      return emitError() << "expected base type '" << dialectName << '.'
+                         << attrName << "' but got '" << attr << "'";
+    }
+    return failure();
+  }
+
+  // Check that the parameters satisfy the constraints.
+  ArrayRef<Attribute> params = dynType.getParams();
+  if (params.size() != constraints.size()) {
+    if (emitError) {
+      StringRef dialectName = typeDef->getDialect()->getNamespace();
+      StringRef attrName = typeDef->getName();
+      emitError() << "attribute '" << dialectName << "." << attrName
+                  << "' expects " << params.size() << " parameters but got "
+                  << constraints.size();
+    }
+    return failure();
+  }
+
+  for (size_t i = 0, s = params.size(); i < s; i++)
+    if (failed(context.verify(emitError, params[i], constraints[i])))
+      return failure();
+
+  return success();
+}
+
+LogicalResult
+AnyOfConstraint::verify(function_ref<InFlightDiagnostic()> emitError,
+                        Attribute attr, ConstraintVerifier &context) const {
+  for (unsigned constr : constraints) {
+    // We do not pass the `emitError` here, since we want to emit an error
+    // only if none of the constraints are satisfied.
+    if (succeeded(context.verify({}, attr, constr))) {
+      return success();
+    }
+  }
+
+  if (emitError)
+    return emitError() << "'" << attr << "' does not satisfy the constraint";
+  return failure();
+}
+
+LogicalResult
+AllOfConstraint::verify(function_ref<InFlightDiagnostic()> emitError,
+                        Attribute attr, ConstraintVerifier &context) const {
+  for (unsigned constr : constraints) {
+    if (failed(context.verify(emitError, attr, constr))) {
+      return failure();
+    }
+  }
+
+  return success();
+}
+
+LogicalResult
+AnyAttributeConstraint::verify(function_ref<InFlightDiagnostic()> emitError,
+                               Attribute attr,
+                               ConstraintVerifier &context) const {
+  return success();
+}
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index 7e631e7ddc802..9ba03153536c6 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -6,6 +6,7 @@ add_mlir_dialect_library(MLIRLLVMDialect
   IR/LLVMDialect.cpp
   IR/LLVMInlining.cpp
   IR/LLVMInterfaces.cpp
+  IR/LLVMMem2Reg.cpp
   IR/LLVMTypes.cpp
   IR/LLVMTypeSyntax.cpp
 
@@ -33,6 +34,7 @@ add_mlir_dialect_library(MLIRLLVMDialect
   MLIRDataLayoutInterfaces
   MLIRInferTypeOpInterface
   MLIRIR
+  MLIRMem2RegInterfaces
   MLIRSideEffectInterfaces
   MLIRSupport
   )
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMem2Reg.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMem2Reg.cpp
new file mode 100644
index 0000000000000..7fa8ebceed5fb
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMem2Reg.cpp
@@ -0,0 +1,210 @@
+//===- LLVMMem2Reg.cpp - Mem2Reg Interfaces ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements Mem2Reg-related interfaces for LLVM dialect operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Interfaces/Mem2RegInterfaces.h"
+
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+// Interfaces for AllocaOp
+//===----------------------------------------------------------------------===//
+
+llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
+  if (!getOperation()->getBlock()->isEntryBlock())
+    return {};
+
+  Type elemType =
+      getElemType() ? *getElemType() : getResult().getType().getElementType();
+  return {MemorySlot{getResult(), elemType}};
+}
+
+Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
+                                      OpBuilder &builder) {
+  return builder.create<LLVM::UndefOp>(getLoc(), slot.elemType);
+}
+
+void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
+                                         BlockArgument argument,
+                                         OpBuilder &builder) {
+  for (Operation *user : getOperation()->getUsers())
+    if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
+      builder.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
+                                       declareOp.getVarInfo());
+}
+
+void LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
+                                             Value defaultValue) {
+  if (defaultValue && defaultValue.use_empty())
+    defaultValue.getDefiningOp()->erase();
+  erase();
+}
+
+//===----------------------------------------------------------------------===//
+// Interfaces for LoadOp/StoreOp
+//===----------------------------------------------------------------------===//
+
+bool LLVM::LoadOp::loadsFrom(const MemorySlot &slot) {
+  return getAddr() == slot.ptr;
+}
+
+Value LLVM::LoadOp::getStored(const MemorySlot &slot) { return {}; }
+
+bool LLVM::StoreOp::loadsFrom(const MemorySlot &slot) { return false; }
+
+Value LLVM::StoreOp::getStored(const MemorySlot &slot) {
+  return getAddr() == slot.ptr ? getValue() : Value();
+}
+
+bool LLVM::LoadOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  if (blockingUses.size() != 1)
+    return false;
+  Value blockingUse = (*blockingUses.begin())->get();
+  // If the blocking use is the slot ptr itself, there will be enough
+  // context to reconstruct the result of the load at removal time, so it can
+  // be removed (provided it loads the exact stored value and is not
+  // volatile).
+  return blockingUse == slot.ptr && getAddr() == slot.ptr &&
+         getResult().getType() == slot.elemType && !getVolatile_();
+}
+
+DeletionKind LLVM::LoadOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder, Value reachingDefinition) {
+  // `canUsesBeRemoved` checked this blocking use must be the loaded slot
+  // pointer.
+  getResult().replaceAllUsesWith(reachingDefinition);
+  return DeletionKind::Delete;
+}
+
+bool LLVM::StoreOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  if (blockingUses.size() != 1)
+    return false;
+  Value blockingUse = (*blockingUses.begin())->get();
+  // If the blocking use is the slot ptr itself, dropping the store is
+  // fine, provided we are currently promoting its target value. Don't allow a
+  // store OF the slot pointer, only INTO the slot pointer.
+  return blockingUse == slot.ptr && getAddr() == slot.ptr &&
+         getValue() != slot.ptr && getValue().getType() == slot.elemType &&
+         !getVolatile_();
+}
+
+DeletionKind LLVM::StoreOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder, Value reachingDefinition) {
+  // `canUsesBeRemoved` checked this blocking use must be the stored slot
+  // pointer.
+  for (Operation *user : slot.ptr.getUsers())
+    if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
+      builder.create<LLVM::DbgValueOp>(declareOp->getLoc(), getValue(),
+                                       declareOp.getVarInfo());
+  return DeletionKind::Delete;
+}
+
+//===----------------------------------------------------------------------===//
+// Interfaces for discardable OPs
+//===----------------------------------------------------------------------===//
+
+/// Conditions the deletion of the operation to the removal of all its uses.
+static bool forwardToUsers(Operation *op,
+                           SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  for (Value result : op->getResults())
+    for (OpOperand &use : result.getUses())
+      newBlockingUses.push_back(&use);
+  return true;
+}
+
+bool LLVM::BitcastOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return forwardToUsers(*this, newBlockingUses);
+}
+
+DeletionKind LLVM::BitcastOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder) {
+  return DeletionKind::Delete;
+}
+
+bool LLVM::AddrSpaceCastOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return forwardToUsers(*this, newBlockingUses);
+}
+
+DeletionKind LLVM::AddrSpaceCastOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder) {
+  return DeletionKind::Delete;
+}
+
+bool LLVM::LifetimeStartOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return true;
+}
+
+DeletionKind LLVM::LifetimeStartOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder) {
+  return DeletionKind::Delete;
+}
+
+bool LLVM::LifetimeEndOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return true;
+}
+
+DeletionKind LLVM::LifetimeEndOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder) {
+  return DeletionKind::Delete;
+}
+
+bool LLVM::DbgDeclareOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return true;
+}
+
+DeletionKind LLVM::DbgDeclareOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder) {
+  return DeletionKind::Delete;
+}
+
+static bool hasAllZeroIndices(LLVM::GEPOp gepOp) {
+  return llvm::all_of(gepOp.getIndices(), [](auto index) {
+    auto indexAttr = index.template dyn_cast<IntegerAttr>();
+    return indexAttr && indexAttr.getValue() == 0;
+  });
+}
+
+bool LLVM::GEPOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  // GEP can be removed as long as it is a no-op and its users can be removed.
+  if (!hasAllZeroIndices(*this))
+    return false;
+  return forwardToUsers(*this, newBlockingUses);
+}
+
+DeletionKind LLVM::GEPOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder) {
+  return DeletionKind::Delete;
+}
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/CMakeLists.txt b/mlir/lib/Dialect/Linalg/TransformOps/CMakeLists.txt
index 079d585d4ea37..01038ed297f17 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/TransformOps/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_mlir_dialect_library(MLIRLinalgTransformOps
+  DialectExtension.cpp
   LinalgMatchOps.cpp
   LinalgTransformOps.cpp
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp b/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
new file mode 100644
index 0000000000000..6cc296105be71
--- /dev/null
+++ b/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
@@ -0,0 +1,59 @@
+//===- DialectExtension.cpp - Linalg transform dialect extension ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Linalg/TransformOps/DialectExtension.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h"
+#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
+#include "mlir/Dialect/PDL/IR/PDL.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+
+using namespace mlir;
+
+namespace {
+/// Registers new ops and declares PDL as dependent dialect since the
+/// additional ops are using PDL types for operands and results.
+class LinalgTransformDialectExtension
+    : public transform::TransformDialectExtension<
+          LinalgTransformDialectExtension> {
+public:
+  using Base::Base;
+
+  void init() {
+    declareDependentDialect<pdl::PDLDialect>();
+    declareDependentDialect<linalg::LinalgDialect>();
+
+    declareGeneratedDialect<affine::AffineDialect>();
+    declareGeneratedDialect<arith::ArithDialect>();
+    declareGeneratedDialect<scf::SCFDialect>();
+    declareGeneratedDialect<vector::VectorDialect>();
+    declareGeneratedDialect<gpu::GPUDialect>();
+    declareGeneratedDialect<tensor::TensorDialect>();
+
+    registerTransformOps<
+#define GET_OP_LIST
+#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp.inc"
+        >();
+    registerTransformOps<
+#define GET_OP_LIST
+#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp.inc"
+        >();
+  }
+};
+} // namespace
+
+void mlir::linalg::registerTransformDialectExtension(
+    DialectRegistry &registry) {
+  registry.addExtensions<LinalgTransformDialectExtension>();
+}
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
index e2d895f12f536..1936a53201f2d 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h"
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
 #include "mlir/Dialect/Transform/IR/MatchInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/FunctionImplementation.h"
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index f113d3af7b44a..9c7236e21842a 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
 
 #include "mlir/AsmParser/AsmParser.h"
+
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
@@ -3113,46 +3114,7 @@ DiagnosedSilenceableFailure transform::InsertSliceToCopyOp::applyToOne(
   return diag;
 }
 
-//===----------------------------------------------------------------------===//
-// Transform op registration
-//===----------------------------------------------------------------------===//
-
-namespace {
-/// Registers new ops and declares PDL as dependent dialect since the
-/// additional ops are using PDL types for operands and results.
-class LinalgTransformDialectExtension
-    : public transform::TransformDialectExtension<
-          LinalgTransformDialectExtension> {
-public:
-  using Base::Base;
-
-  void init() {
-    declareDependentDialect<pdl::PDLDialect>();
-    declareDependentDialect<LinalgDialect>();
-    declareGeneratedDialect<affine::AffineDialect>();
-    declareGeneratedDialect<arith::ArithDialect>();
-    declareGeneratedDialect<scf::SCFDialect>();
-    declareGeneratedDialect<vector::VectorDialect>();
-    declareGeneratedDialect<gpu::GPUDialect>();
-
-    registerTransformOps<
-#define GET_OP_LIST
-#include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp.inc"
-        >();
-    registerTransformOps<
-#define GET_OP_LIST
-#include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp.inc"
-        >();
-  }
-};
-} // namespace
-
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOpsEnums.cpp.inc"
 
 #define GET_OP_CLASSES
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp.inc"
-
-void mlir::linalg::registerTransformDialectExtension(
-    DialectRegistry &registry) {
-  registry.addExtensions<LinalgTransformDialectExtension>();
-}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 62a00c047775e..b1a3bfff239f2 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -434,14 +434,14 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, tensor::PackOp packOp,
       getOrCreatePackedViewOfOperand(rewriter, genericOp.getLoc(), *packInfo,
                                      genericOp, opOperand);
 
-  // We'll replace the init operand with the destination of pack op if the init
-  // operand has not users in the body of the linalg.generic (pure elementwise).
-  // If it has users we need to pack the init operand too and replace the init
-  // with the packing result.
-  Value dest = (genericOp.getRegionOutputArgs()[0].use_empty())
-                   ? packOpDest
-                   : packedOutOperand;
-
+  // If the dps init operand of the generic is a tensor.empty forward the pack
+  // op destination.
+  Value dest = packedOutOperand;
+  if (auto initTensor = genericOp.getDpsInitOperand(0)
+                            ->get()
+                            .getDefiningOp<tensor::EmptyOp>()) {
+    dest = packOpDest;
+  }
   return packElementWiseOp(rewriter, genericOp, dest, packedOutIndexingMap,
                            *packInfo);
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index 5289ed6c7f519..5fd48853875ce 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -60,7 +60,7 @@ bool shouldBeDetensored(Operation *op, TypeConverter typeConverter) {
          });
 }
 
-/// A conversion patttern for detensoring `linalg.generic` ops.
+/// A conversion pattern for detensoring `linalg.generic` ops.
 class DetensorizeGenericOp : public OpConversionPattern<GenericOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
@@ -69,7 +69,7 @@ class DetensorizeGenericOp : public OpConversionPattern<GenericOp> {
                   ConversionPatternRewriter &rewriter) const override {
     Block *originalBlock = op->getBlock();
 
-    // Gather some information about the op before inling its region.
+    // Gather some information about the op before inlining its region.
     Block *opEntryBlock = &*op.getRegion().begin();
     YieldOp yieldOp = dyn_cast<YieldOp>(op.getRegion().back().getTerminator());
 
@@ -476,6 +476,21 @@ struct LinalgDetensorize
     DenseSet<BlockArgument> blockArgsToDetensor;
     FunctionOpInterface funcOp = getOperation();
 
+    if (funcOp.getFunctionBody().empty())
+      return;
+
+    // Make sure the entry block of the function doesn't contain any Linalg ops.
+    // Otherwise, it may lead to the signature of the block being changed by the
+    // dialect conversion below, which would make the function op invalid
+    // because its type shouldn't change.
+    IRRewriter rewriter(funcOp->getContext());
+    Block *entryBlock = &funcOp.getFunctionBody().front();
+    Block *postEntryBlock =
+        rewriter.splitBlock(entryBlock, entryBlock->begin());
+    rewriter.setInsertionPointToStart(entryBlock);
+    auto branch =
+        rewriter.create<cf::BranchOp>(rewriter.getUnknownLoc(), postEntryBlock);
+
     if (aggressiveMode.getValue()) {
       AggressiveDetensoringModel costModel;
       costModel.compute(funcOp, typeConverter, opsToDetensor,
@@ -553,6 +568,11 @@ struct LinalgDetensorize
     if (failed(applyPatternsAndFoldGreedily(getOperation(),
                                             std::move(canonPatterns))))
       signalPassFailure();
+
+    // Get rid of the dummy entry block we created in the beginning to work
+    // around dialect conversion signature rewriting.
+    rewriter.eraseOp(branch);
+    rewriter.mergeBlocks(postEntryBlock, entryBlock);
   }
 };
 } // namespace
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 4d5ef0edc9f8a..4a5c69c5bc061 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -480,9 +480,6 @@ struct PackedOperandsDimList {
 FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
                                              tensor::PackOp packOp) {
   // 1. Filter out NYI cases.
-  if (!packOp.getOuterDimsPerm().empty())
-    return rewriter.notifyMatchFailure(packOp, "outer dims perm NYI");
-
   auto packedTensorType =
       packOp->getResultTypes().front().cast<RankedTensorType>();
   if (!packedTensorType.hasStaticShape()) {
@@ -495,21 +492,37 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   OpBuilder::InsertionGuard g(rewriter);
   rewriter.setInsertionPoint(packOp);
 
-  // 2. Compute the permutation vector to move the last `numPackedDims` into the
-  // `innerPosDims` of a shape of rank `packedRank`.
+  // 2. Compute the permutation vector to shuffle packed shape into the shape
+  // before any outer or inner permutations have been applied. The permutation
+  // can be obtained from two permutations:
+  //   a) Compute the permutation vector to move the last `numPackedDims` into
+  //      the `innerPosDims` of a shape of rank `packedRank`.
+  //   b) Compute the permutation vector to move outer dims if the pack op
+  //      has outer_dims_perm.
+  // Apply (b) permutation on (a) permutation to get the final permutation.
   int64_t numPackedDims = packOp.getInnerDimsPos().size();
   int64_t packedRank = packedTensorType.getRank();
   auto lastDims = llvm::to_vector(
       llvm::seq<int64_t>(packedRank - numPackedDims, packedRank));
   PackingMetadata packingMetadata = computePackingMetadata(
       packedTensorType.getRank(), packOp.getInnerDimsPos());
-  SmallVector<int64_t> lastDimsToInsertPositionsPerm = computePermutationVector(
+  SmallVector<int64_t> innerPositionsPerm = computePermutationVector(
       packedRank, lastDims, packingMetadata.insertPositions);
 
+  SmallVector<int64_t> outerPos = packingMetadata.outerPositions;
+  ArrayRef<int64_t> outerPerm = packOp.getOuterDimsPerm();
+  if (!outerPerm.empty())
+    applyPermutationToVector(outerPos, outerPerm);
+  SmallVector<int64_t> outerPositionPerm = computePermutationVector(
+      packedRank, packingMetadata.outerPositions, outerPos);
+
+  SmallVector<int64_t> packedToStripMinedShapePerm = innerPositionsPerm;
+  applyPermutationToVector(packedToStripMinedShapePerm, outerPositionPerm);
+
   // 3. Compute the stripMinedShape: this is the packed shape before any outer
   // or inner permutations have been applied.
   SmallVector<int64_t> stripMinedShape(packedTensorType.getShape());
-  applyPermutationToVector(stripMinedShape, lastDimsToInsertPositionsPerm);
+  applyPermutationToVector(stripMinedShape, packedToStripMinedShapePerm);
 
   // 4. Pad the source of packOp to a shape we can expand into stripMinedShape.
   RankedTensorType collapsed = tensor::CollapseShapeOp::inferCollapsedType(
@@ -527,11 +540,17 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   LLVM_DEBUG(
       DBGSNL(); DBGSNL(); llvm::interleaveComma(packingMetadata.insertPositions,
                                                 DBGS() << "insertPositions: ");
+      DBGSNL(); llvm::interleaveComma(packingMetadata.outerPositions,
+                                      DBGS() << "outerPositions: ");
       DBGSNL(); llvm::interleaveComma(packedTensorType.getShape(),
                                       DBGS() << "packedShape: ");
       DBGSNL();
-      llvm::interleaveComma(lastDimsToInsertPositionsPerm,
-                            DBGS() << "lastDimsToInsertPositionsPerm: ");
+      llvm::interleaveComma(outerPositionPerm, DBGS() << "outerPositionPerm: ");
+      DBGSNL(); llvm::interleaveComma(innerPositionsPerm,
+                                      DBGS() << "innerPositionsPerm: ");
+      DBGSNL();
+      llvm::interleaveComma(packedToStripMinedShapePerm,
+                            DBGS() << "packedToStripMinedShapePerm: ");
       DBGSNL(); llvm::interleaveComma(
           packingMetadata.reassociations, DBGS() << "reassociations: ",
           [&](ReassociationIndices ri) {
@@ -572,16 +591,14 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       padOp.getResult(), packingMetadata.reassociations);
 
   // 6. Transpose stripMinedShape to packedShape.
-  SmallVector<int64_t> insertPositionsToLastDimsPerm = computePermutationVector(
-      packedRank, packingMetadata.insertPositions, lastDims);
+  SmallVector<int64_t> transpPerm =
+      invertPermutationVector(packedToStripMinedShapePerm);
   auto transposeOp = rewriter.create<linalg::TransposeOp>(
-      loc, reshapeOp.getResult(), packOp.getDest(),
-      insertPositionsToLastDimsPerm);
+      loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
   LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
              DBGS() << "reshape op: " << reshapeOp; DBGSNL();
-             llvm::interleaveComma(insertPositionsToLastDimsPerm,
-                                   DBGS() << "insertPositionsToLastDimsPerm: ");
+             llvm::interleaveComma(transpPerm, DBGS() << "transpPerm: ");
              DBGSNL(); DBGS() << "transpose op: " << transposeOp; DBGSNL(););
 
   // 7. Replace packOp by transposeOp.
@@ -1224,66 +1241,124 @@ static Value getPackOpSourceOrPaddedSource(OpBuilder &builder,
                                  /*nofold=*/false, loc, builder);
 }
 
+// Normalizes a permutation on a higher rank space to its actual size, e.g.
+//   perm = [1, 4, 2]
+// becomes
+//   norm = [0, 2, 1]
 static SmallVector<int64_t>
-getPackUnpackNormalizedInnerPerm(int rank, ArrayRef<int64_t> innerDimsPos) {
+getPackUnpackNormalizedPerm(int rank, ArrayRef<int64_t> perm) {
   constexpr int64_t kNonTiledMarker = -1;
   SmallVector<int64_t> vec(rank, kNonTiledMarker);
-  for (auto [index, value] : llvm::enumerate(innerDimsPos))
+  for (auto [index, value] : llvm::enumerate(perm))
     vec[value] = index;
-  SmallVector<int64_t> perm = llvm::to_vector(llvm::make_filter_range(
+  SmallVector<int64_t> normalizedPerm = llvm::to_vector(llvm::make_filter_range(
       vec, [&](int64_t v) { return v != kNonTiledMarker; }));
+  // This inverts the permutation in addition to normalizing so invert back.
+  return invertPermutationVector(normalizedPerm);
+}
+
+// Gets the normalized permutation implied by innerDimsPos and outerDimsPerm
+// assuming rank reduction of unit outer dims.
+static SmallVector<int64_t>
+getPackUnpackRankReducedPerm(ArrayRef<int64_t> shape,
+                             ArrayRef<int64_t> innerDimsPos,
+                             ArrayRef<int64_t> outerDimsPerm) {
+  SmallVector<int64_t> rankReducedOuterDimsPerm;
+  SmallVector<int64_t> outerDims;
+  SmallVector<int64_t> innerDims;
+  int64_t dim = 0;
+  int64_t unpackedRank = shape.size();
+  for (auto i : llvm::seq<unsigned>(0, unpackedRank)) {
+    if (llvm::is_contained(innerDimsPos, i)) {
+      innerDims.push_back(dim++);
+      continue;
+    }
+    if (shape[i] == 1)
+      continue;
+    outerDims.push_back(dim++);
+    if (!outerDimsPerm.empty())
+      rankReducedOuterDimsPerm.push_back(outerDimsPerm[i]);
+  }
+
+  // Get the position of the inner dims after permutation.
+  SmallVector<int64_t> innerPerm =
+      getPackUnpackNormalizedPerm(unpackedRank, innerDimsPos);
+  applyPermutationToVector<int64_t>(innerDims, innerPerm);
+
+  // Ditto for the outer dims.
+  SmallVector<int64_t> perm = outerDims;
+
+  rankReducedOuterDimsPerm =
+      getPackUnpackNormalizedPerm(unpackedRank, rankReducedOuterDimsPerm);
+  if (!rankReducedOuterDimsPerm.empty())
+    applyPermutationToVector<int64_t>(perm, rankReducedOuterDimsPerm);
+
+  // The tile always ends up as the inner most dims after packing.
+  perm.append(innerDims);
+
   return perm;
 }
 
 LogicalResult GeneralizeOuterUnitDimsPackOpPattern::matchAndRewrite(
     tensor::PackOp packOp, PatternRewriter &rewriter) const {
-  // TODO: support the case that outer dimensions are not all 1s A
-  // tensor.expand_shape will be generated in this case.
-  int64_t srcRank = packOp.getSourceRank();
-  if (llvm::any_of(packOp.getDestType().getShape().take_front(srcRank),
-                   [](int64_t val) { return val != 1; })) {
-    return rewriter.notifyMatchFailure(
-        packOp, "require the outer dimension of the result are all 1s");
-  }
-
   if (llvm::any_of(packOp.getMixedTiles(),
                    [](OpFoldResult tile) { return tile.is<Value>(); })) {
     return rewriter.notifyMatchFailure(packOp,
                                        "require inner tile sizes being static");
   }
 
-  // 1. Use rank-reduced tensor.extract_slice op to extract the tile.
+  // TODO: support the case that outer dimensions are not all 1s. A
+  // tensor.expand_shape will be generated in this case.
+  auto innerDimsPos = packOp.getInnerDimsPos();
+  int64_t srcRank = packOp.getSourceRank();
+  auto destShape = packOp.getDestType().getShape();
+  if (llvm::any_of(innerDimsPos, [destShape](int64_t index) {
+        return destShape[index] != 1;
+      })) {
+    return rewriter.notifyMatchFailure(
+        packOp, "require the tiled outer dimensions of the result are all 1s");
+  }
+
+  // 1. Use rank-reduced tensor.extract_slice op to extract the tile and untiled
+  // outer dims.
   Location loc = packOp.getLoc();
+  Value input = getPackOpSourceOrPaddedSource(rewriter, packOp);
+  auto inputShape = packOp.getSourceType().getShape();
+  DenseMap<int64_t, OpFoldResult> dimAndTileMapping =
+      packOp.getDimAndTileMapping();
   Attribute zeroIdxAttr = rewriter.getIndexAttr(0);
   Attribute oneIdxAttr = rewriter.getIndexAttr(1);
   SmallVector<OpFoldResult> readOffsets(srcRank, zeroIdxAttr);
   SmallVector<OpFoldResult> readStrides(srcRank, oneIdxAttr);
   SmallVector<OpFoldResult> readSizes;
   SmallVector<int64_t> readShape;
-  DenseMap<int64_t, OpFoldResult> dimAndTileMapping =
-      packOp.getDimAndTileMapping();
   for (auto i : llvm::seq<unsigned>(0, srcRank)) {
-    if (!dimAndTileMapping.count(i)) {
-      readSizes.push_back(oneIdxAttr);
+    if (dimAndTileMapping.count(i)) {
+      readShape.push_back(getConstantIntValue(dimAndTileMapping[i])
+                              .value_or(ShapedType::kDynamic));
+      readSizes.push_back(dimAndTileMapping[i]);
       continue;
     }
-    readSizes.push_back(dimAndTileMapping[i]);
-    readShape.push_back(getConstantIntValue(dimAndTileMapping[i])
-                            .value_or(ShapedType::kDynamic));
+    if (ShapedType::isDynamic(inputShape[i])) {
+      readSizes.push_back(
+          rewriter.create<tensor::DimOp>(loc, input, i).getResult());
+    } else {
+      readSizes.push_back(rewriter.getIndexAttr(inputShape[i]));
+    }
+    if (inputShape[i] != 1)
+      readShape.push_back(inputShape[i]);
   }
+
   Type elemType = packOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShape, elemType);
 
-  Value input = getPackOpSourceOrPaddedSource(rewriter, packOp);
   Value tile = rewriter.create<tensor::ExtractSliceOp>(
       loc, readType, input, readOffsets, readSizes, readStrides);
 
   // 2. Transpose the tile to match the inner tile order.
-  SmallVector<int64_t> perm =
-      getPackUnpackNormalizedInnerPerm(srcRank, packOp.getInnerDimsPos());
-  // The permutation is inverted when normalizing so invert back to match the
-  // ordering in the pack op.
-  perm = invertPermutationVector(perm);
+
+  SmallVector<int64_t> perm = getPackUnpackRankReducedPerm(
+      inputShape, innerDimsPos, packOp.getOuterDimsPerm());
 
   LLVM_DEBUG(DBGS() << "Pack permutation: " << packOp << "\n";
              llvm::interleaveComma(perm, DBGS() << "perm: "); DBGSNL(););
@@ -1299,9 +1374,8 @@ LogicalResult GeneralizeOuterUnitDimsPackOpPattern::matchAndRewrite(
   int64_t destRank = packOp.getDestRank();
   SmallVector<OpFoldResult> writeStrides(destRank, oneIdxAttr);
   SmallVector<OpFoldResult> writeOffsets(destRank, zeroIdxAttr);
-  SmallVector<OpFoldResult> writeSizes(srcRank, oneIdxAttr);
-  for (auto size : transpShape)
-    writeSizes.push_back(rewriter.getIndexAttr(size));
+  SmallVector<OpFoldResult> writeSizes =
+      tensor::getMixedSizes(rewriter, loc, packOp.getDest());
 
   auto insert = rewriter.create<tensor::InsertSliceOp>(
       loc, transposedOp.getResult()[0], packOp.getDest(), writeOffsets,
@@ -1316,35 +1390,59 @@ LogicalResult GeneralizeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   int64_t srcRank = unpackOp.getSourceRank();
   int64_t destRank = unpackOp.getDestRank();
   ArrayRef<int64_t> srcShape = unpackOp.getSourceType().getShape();
-  if (llvm::any_of(srcShape.take_front(destRank),
-                   [](int64_t val) { return val != 1; })) {
+  ArrayRef<int64_t> innerDimsPos = unpackOp.getInnerDimsPos();
+  if (llvm::any_of(innerDimsPos, [srcShape](int64_t index) {
+        return srcShape[index] != 1;
+      })) {
     return rewriter.notifyMatchFailure(
-        unpackOp, "require the outer dimension of the result are all 1s");
+        unpackOp,
+        "require the tiled outer dimensions of the result are all 1s");
   }
 
   // 1. Use rank-reduced tensor.extract_slice op to extract the tile.
   Location loc = unpackOp.getLoc();
+  Value source = unpackOp.getSource();
+  DenseMap<int64_t, OpFoldResult> dimAndTileMapping =
+      unpackOp.getDimAndTileMapping();
   Attribute zeroIdxAttr = rewriter.getIndexAttr(0);
   Attribute oneIdxAttr = rewriter.getIndexAttr(1);
   SmallVector<OpFoldResult> readOffsets(srcRank, zeroIdxAttr);
   SmallVector<OpFoldResult> readStrides(srcRank, oneIdxAttr);
+  SmallVector<OpFoldResult> readSizes;
+  SmallVector<int64_t> readShape;
+  for (auto i : llvm::seq<unsigned>(0, destRank)) {
+    if (dimAndTileMapping.count(i)) {
+      readSizes.push_back(oneIdxAttr);
+      continue;
+    }
 
+    if (ShapedType::isDynamic(srcShape[i])) {
+      readSizes.push_back(
+          rewriter.create<tensor::DimOp>(loc, source, i).getResult());
+    } else {
+      readSizes.push_back(rewriter.getIndexAttr(srcShape[i]));
+    }
+    if (srcShape[i] != 1)
+      readShape.push_back(srcShape[i]);
+  }
   auto mixedTiles = unpackOp.getMixedTiles();
-  SmallVector<OpFoldResult> readSizes(destRank, oneIdxAttr);
   readSizes.append(mixedTiles.begin(), mixedTiles.end());
 
   // Explicitly create the type for extract_slice op because the inner tile
   // size could be 1. We want to represent the whole inner tile in this case.
-  ArrayRef<int64_t> readShape = srcShape.drop_front(destRank);
+  auto tileShape = srcShape.drop_front(destRank);
+  // Append the inner tile shape to the permuted and rank-reduced outer shape.
+  readShape.append(tileShape.begin(), tileShape.end());
   Type elemType = unpackOp.getSourceType().getElementType();
   auto readType = RankedTensorType::get(readShape, elemType);
   Value innerTile = rewriter.create<tensor::ExtractSliceOp>(
       loc, readType, unpackOp.getSource(), readOffsets, readSizes, readStrides);
 
   // 2. Transpose the tile to match the outer corresponding tile order.
-  ArrayRef<int64_t> innerDimsPos = unpackOp.getInnerDimsPos();
-  SmallVector<int64_t> perm =
-      getPackUnpackNormalizedInnerPerm(srcRank, innerDimsPos);
+  SmallVector<int64_t> perm = getPackUnpackRankReducedPerm(
+      srcShape.take_front(destRank), innerDimsPos, unpackOp.getOuterDimsPerm());
+  // Unpack is a transition out of packed space so we invert the permutation.
+  perm = invertPermutationVector(perm);
   SmallVector<int64_t> transpShape(readShape);
   applyPermutationToVector<int64_t>(transpShape, perm);
 
@@ -1358,11 +1456,13 @@ LogicalResult GeneralizeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   SmallVector<OpFoldResult> tileStrides(numLoops, oneIdxAttr);
   SmallVector<OpFoldResult> tileOffsets(numLoops, zeroIdxAttr);
   SmallVector<OpFoldResult> tileSizes;
-  for (int dim : innerDimsPos)
-    tileSizes.push_back(getAsOpFoldResult(
-        rewriter.createOrFold<tensor::DimOp>(loc, unpackOp.getDest(), dim)));
+  ArrayRef<int64_t> destShape = unpackOp.getDestType().getShape();
+  for (auto i : llvm::seq<unsigned>(0, destRank)) {
+    if (dimAndTileMapping.count(i) || destShape[i] != 1)
+      tileSizes.push_back(getAsOpFoldResult(
+          rewriter.createOrFold<tensor::DimOp>(loc, unpackOp.getDest(), i)));
+  }
 
-  applyPermutationToVector<OpFoldResult>(tileSizes, perm);
   auto partialTile = rewriter.create<tensor::ExtractSliceOp>(
       loc, transposedOp.getResult()[0], tileOffsets, tileSizes, tileStrides);
 
@@ -1370,10 +1470,8 @@ LogicalResult GeneralizeOuterUnitDimsUnPackOpPattern::matchAndRewrite(
   SmallVector<OpFoldResult> writeSizes;
   SmallVector<OpFoldResult> writeStrides(destRank, oneIdxAttr);
   SmallVector<OpFoldResult> writeOffsets(destRank, zeroIdxAttr);
-  DenseMap<int64_t, OpFoldResult> dimAndTileMapping =
-      unpackOp.getDimAndTileMapping();
   for (int i = 0, idx = 0; i < destRank; ++i) {
-    if (dimAndTileMapping.count(i))
+    if (dimAndTileMapping.count(i) || destShape[i] != 1)
       writeSizes.push_back(tileSizes[idx++]);
     else
       writeSizes.push_back(oneIdxAttr);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 55beb2f87cd78..56b4516452a11 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -1423,8 +1423,10 @@ mlir::linalg::maskedVectorize(RewriterBase &rewriter, tensor::PadOp padOp,
     return rewriter.notifyMatchFailure(
         padOp, "result tensor shape must match input vector sizes");
   }
-  if (llvm::any_of(padOp.getStaticLow(),
-                   [](int64_t val) { return val != 0; })) {
+  if (llvm::any_of(padOp.getLow(), [](Value v) {
+        std::optional<int64_t> res = getConstantIntValue(v);
+        return !res.has_value() || res.value() != 0;
+      })) {
     LDBG("low pad must all be zero: " << padOp << "\n");
     return rewriter.notifyMatchFailure(padOp, "low pad must all be zero");
   }
diff --git a/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
index f0c57158714b5..c23c2f1a9b650 100644
--- a/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
@@ -8,12 +8,12 @@ add_mlir_dialect_library(MLIRMathTransforms
 
   LINK_LIBS PUBLIC
   MLIRArithDialect
-  MLIRDialectUtils  
+  MLIRDialectUtils
   MLIRIR
   MLIRMathDialect
+  MLIRSCFDialect
   MLIRPass
   MLIRTransforms
   MLIRX86VectorDialect
   MLIRVectorDialect
-  MLIRVectorUtils
   )
diff --git a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
index 3aedd3783fa8f..1b01f783a2224 100644
--- a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_mlir_dialect_library(MLIRMemRefDialect
   MemRefDialect.cpp
+  MemRefMem2Reg.cpp
   MemRefOps.cpp
   ValueBoundsOpInterfaceImpl.cpp
 
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMem2Reg.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMem2Reg.cpp
new file mode 100644
index 0000000000000..acc38b57c7fdb
--- /dev/null
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefMem2Reg.cpp
@@ -0,0 +1,120 @@
+//===- MemRefMem2Reg.cpp - Mem2Reg Interfaces -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements Mem2Reg-related interfaces for MemRef dialect
+// operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Interfaces/Mem2RegInterfaces.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+using namespace mlir;
+
+//===----------------------------------------------------------------------===//
+//  AllocaOp interfaces
+//===----------------------------------------------------------------------===//
+
+static bool isSupportedElementType(Type type) {
+  return type.isa<MemRefType>() ||
+         OpBuilder(type.getContext()).getZeroAttr(type);
+}
+
+SmallVector<MemorySlot> memref::AllocaOp::getPromotableSlots() {
+  MemRefType type = getType();
+  if (!isSupportedElementType(type.getElementType()))
+    return {};
+  if (!type.hasStaticShape())
+    return {};
+  // Make sure the memref contains only a single element.
+  if (any_of(type.getShape(), [](uint64_t dim) { return dim != 1; }))
+    return {};
+
+  return {MemorySlot{getResult(), type.getElementType()}};
+}
+
+Value memref::AllocaOp::getDefaultValue(const MemorySlot &slot,
+                                        OpBuilder &builder) {
+  assert(isSupportedElementType(slot.elemType));
+  // TODO: support more types.
+  return TypeSwitch<Type, Value>(slot.elemType)
+      .Case([&](MemRefType t) {
+        return builder.create<memref::AllocaOp>(getLoc(), t);
+      })
+      .Default([&](Type t) {
+        return builder.create<arith::ConstantOp>(getLoc(), t,
+                                                 builder.getZeroAttr(t));
+      });
+}
+
+void memref::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
+                                               Value defaultValue) {
+  if (defaultValue.use_empty())
+    defaultValue.getDefiningOp()->erase();
+  erase();
+}
+
+void memref::AllocaOp::handleBlockArgument(const MemorySlot &slot,
+                                           BlockArgument argument,
+                                           OpBuilder &builder) {}
+
+//===----------------------------------------------------------------------===//
+//  LoadOp/StoreOp interfaces
+//===----------------------------------------------------------------------===//
+
+bool memref::LoadOp::loadsFrom(const MemorySlot &slot) {
+  return getMemRef() == slot.ptr;
+}
+
+Value memref::LoadOp::getStored(const MemorySlot &slot) { return {}; }
+
+bool memref::LoadOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  if (blockingUses.size() != 1)
+    return false;
+  Value blockingUse = (*blockingUses.begin())->get();
+  return blockingUse == slot.ptr && getMemRef() == slot.ptr &&
+         getResult().getType() == slot.elemType;
+}
+
+DeletionKind memref::LoadOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder, Value reachingDefinition) {
+  // `canUsesBeRemoved` checked this blocking use must be the loaded slot
+  // pointer.
+  getResult().replaceAllUsesWith(reachingDefinition);
+  return DeletionKind::Delete;
+}
+
+bool memref::StoreOp::loadsFrom(const MemorySlot &slot) { return false; }
+
+Value memref::StoreOp::getStored(const MemorySlot &slot) {
+  if (getMemRef() != slot.ptr)
+    return {};
+  return getValue();
+}
+
+bool memref::StoreOp::canUsesBeRemoved(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  if (blockingUses.size() != 1)
+    return false;
+  Value blockingUse = (*blockingUses.begin())->get();
+  return blockingUse == slot.ptr && getMemRef() == slot.ptr &&
+         getValue() != slot.ptr && getValue().getType() == slot.elemType;
+}
+
+DeletionKind memref::StoreOp::removeBlockingUses(
+    const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    OpBuilder &builder, Value reachingDefinition) {
+  return DeletionKind::Delete;
+}
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index a828fb6a7a679..6b0d2b5d014d5 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
+#include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
@@ -1354,9 +1355,10 @@ void ExtractAlignedPointerAsIndexOp::getAsmResultNames(
 /// shape of the source.
 LogicalResult ExtractStridedMetadataOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
-  ExtractStridedMetadataOpAdaptor extractAdaptor(operands, attributes, regions);
+  ExtractStridedMetadataOpAdaptor extractAdaptor(
+      operands, attributes, *properties.as<EmptyProperties *>(), regions);
   auto sourceType = extractAdaptor.getSource().getType().dyn_cast<MemRefType>();
   if (!sourceType)
     return failure();
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 65d7be0991fcf..7f702e1978540 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/MemRef/Transforms/Transforms.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
 #include "mlir/Dialect/PDL/IR/PDL.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformInterfaces.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
@@ -97,6 +98,49 @@ transform::MemRefExtractAddressComputationsOp::applyToOne(
   return DiagnosedSilenceableFailure::success();
 }
 
+//===----------------------------------------------------------------------===//
+// MemRefMakeLoopIndependentOp
+//===----------------------------------------------------------------------===//
+
+DiagnosedSilenceableFailure transform::MemRefMakeLoopIndependentOp::applyToOne(
+    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformState &state) {
+  // Gather IVs.
+  SmallVector<Value> ivs;
+  Operation *nextOp = target;
+  for (uint64_t i = 0, e = getNumLoops(); i < e; ++i) {
+    nextOp = nextOp->getParentOfType<scf::ForOp>();
+    if (!nextOp) {
+      DiagnosedSilenceableFailure diag = emitSilenceableError()
+                                         << "could not find " << i
+                                         << "-th enclosing loop";
+      diag.attachNote(target->getLoc()) << "target op";
+      return diag;
+    }
+    ivs.push_back(cast<scf::ForOp>(nextOp).getInductionVar());
+  }
+
+  // Rewrite IR.
+  IRRewriter rewriter(target->getContext());
+  FailureOr<Value> replacement = failure();
+  if (auto allocaOp = dyn_cast<memref::AllocaOp>(target)) {
+    replacement = memref::replaceWithIndependentOp(rewriter, allocaOp, ivs);
+  } else {
+    DiagnosedSilenceableFailure diag = emitSilenceableError()
+                                       << "unsupported target op";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+  if (failed(replacement)) {
+    DiagnosedSilenceableFailure diag =
+        emitSilenceableError() << "could not make target op loop-independent";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+  results.push_back(replacement->getDefiningOp());
+  return DiagnosedSilenceableFailure::success();
+}
+
 //===----------------------------------------------------------------------===//
 // Transform op registration
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
index a60cd8017e574..203db14885c22 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
@@ -6,6 +6,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms
   EmulateWideInt.cpp
   ExtractAddressComputations.cpp
   FoldMemRefAliasOps.cpp
+  IndependenceTransforms.cpp
   MultiBuffer.cpp
   NormalizeMemRefs.cpp
   ResolveShapedTypeResultDims.cpp
@@ -19,6 +20,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms
 
   LINK_LIBS PUBLIC
   MLIRAffineDialect
+  MLIRAffineTransforms
   MLIRAffineUtils
   MLIRArithDialect
   MLIRArithTransforms
@@ -33,6 +35,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms
   MLIRPass
   MLIRTensorDialect
   MLIRTransforms
+  MLIRValueBoundsOpInterface
   MLIRVectorDialect
 )
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index fcddb1ea14120..ea372bffbc0b7 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -833,7 +833,8 @@ class ExtractStridedMetadataOpReinterpretCastFolder
     SmallVector<Type> inferredReturnTypes;
     if (failed(extractStridedMetadataOp.inferReturnTypes(
             rewriter.getContext(), loc, {reinterpretCastOp.getSource()},
-            /*attributes=*/{}, /*regions=*/{}, inferredReturnTypes)))
+            /*attributes=*/{}, /*properties=*/nullptr, /*regions=*/{},
+            inferredReturnTypes)))
       return rewriter.notifyMatchFailure(
           reinterpretCastOp, "reinterpret_cast source's type is incompatible");
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
new file mode 100644
index 0000000000000..aa1d27dc863e6
--- /dev/null
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -0,0 +1,182 @@
+//===- IndependenceTransforms.cpp - Make ops independent of values --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
+
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/Transforms/Transforms.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/Interfaces/ValueBoundsOpInterface.h"
+
+using namespace mlir;
+using namespace mlir::memref;
+
+/// Make the given OpFoldResult independent of all independencies.
+static FailureOr<OpFoldResult> makeIndependent(OpBuilder &b, Location loc,
+                                               OpFoldResult ofr,
+                                               ValueRange independencies) {
+  if (ofr.is<Attribute>())
+    return ofr;
+  Value value = ofr.get<Value>();
+  AffineMap boundMap;
+  ValueDimList mapOperands;
+  if (failed(ValueBoundsConstraintSet::computeIndependentBound(
+          boundMap, mapOperands, presburger::BoundType::UB, value,
+          /*dim=*/std::nullopt, independencies, /*closedUB=*/true)))
+    return failure();
+  return affine::materializeComputedBound(b, loc, boundMap, mapOperands);
+}
+
+FailureOr<Value> memref::buildIndependentOp(OpBuilder &b,
+                                            memref::AllocaOp allocaOp,
+                                            ValueRange independencies) {
+  OpBuilder::InsertionGuard g(b);
+  b.setInsertionPoint(allocaOp);
+  Location loc = allocaOp.getLoc();
+
+  SmallVector<OpFoldResult> newSizes;
+  for (OpFoldResult ofr : allocaOp.getMixedSizes()) {
+    auto ub = makeIndependent(b, loc, ofr, independencies);
+    if (failed(ub))
+      return failure();
+    newSizes.push_back(*ub);
+  }
+
+  // Return existing memref::AllocaOp if nothing has changed.
+  if (llvm::equal(allocaOp.getMixedSizes(), newSizes))
+    return allocaOp.getResult();
+
+  // Create a new memref::AllocaOp.
+  Value newAllocaOp =
+      b.create<AllocaOp>(loc, newSizes, allocaOp.getType().getElementType());
+
+  // Create a memref::SubViewOp.
+  SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
+  SmallVector<OpFoldResult> strides(newSizes.size(), b.getIndexAttr(1));
+  return b
+      .create<SubViewOp>(loc, newAllocaOp, offsets, allocaOp.getMixedSizes(),
+                         strides)
+      .getResult();
+}
+
+/// Push down an UnrealizedConversionCastOp past a SubViewOp.
+static UnrealizedConversionCastOp
+propagateSubViewOp(RewriterBase &rewriter,
+                   UnrealizedConversionCastOp conversionOp, SubViewOp op) {
+  OpBuilder::InsertionGuard g(rewriter);
+  rewriter.setInsertionPoint(op);
+  auto newResultType =
+      SubViewOp::inferRankReducedResultType(
+          op.getType().getShape(), op.getSourceType(), op.getMixedOffsets(),
+          op.getMixedSizes(), op.getMixedStrides())
+          .cast<MemRefType>();
+  Value newSubview = rewriter.create<SubViewOp>(
+      op.getLoc(), newResultType, conversionOp.getOperand(0),
+      op.getMixedOffsets(), op.getMixedSizes(), op.getMixedStrides());
+  auto newConversionOp = rewriter.create<UnrealizedConversionCastOp>(
+      op.getLoc(), op.getType(), newSubview);
+  rewriter.replaceAllUsesWith(op.getResult(), newConversionOp->getResult(0));
+  return newConversionOp;
+}
+
+/// Given an original op and a new, modified op with the same number of results,
+/// whose memref return types may differ, replace all uses of the original op
+/// with the new op and propagate the new memref types through the IR.
+///
+/// Example:
+/// %from = memref.alloca(%sz) : memref<?xf32>
+/// %to = memref.subview ... : ... to memref<?xf32, strided<[1], offset: ?>>
+/// memref.store %cst, %from[%c0] : memref<?xf32>
+///
+/// In the above example, all uses of %from are replaced with %to. This can be
+/// done directly for ops such as memref.store. For ops that have memref results
+/// (e.g., memref.subview), the result type may depend on the operand type, so
+/// we cannot just replace all uses. There is special handling for common memref
+/// ops. For all other ops, unrealized_conversion_cast is inserted.
+static void replaceAndPropagateMemRefType(RewriterBase &rewriter,
+                                          Operation *from, Operation *to) {
+  assert(from->getNumResults() == to->getNumResults() &&
+         "expected same number of results");
+  OpBuilder::InsertionGuard g(rewriter);
+  rewriter.setInsertionPointAfter(to);
+
+  // Wrap new results in unrealized_conversion_cast and replace all uses of the
+  // original op.
+  SmallVector<UnrealizedConversionCastOp> unrealizedConversions;
+  for (const auto &it :
+       llvm::enumerate(llvm::zip(from->getResults(), to->getResults()))) {
+    unrealizedConversions.push_back(rewriter.create<UnrealizedConversionCastOp>(
+        to->getLoc(), std::get<0>(it.value()).getType(),
+        std::get<1>(it.value())));
+    rewriter.replaceAllUsesWith(from->getResult(it.index()),
+                                unrealizedConversions.back()->getResult(0));
+  }
+
+  // Push unrealized_conversion_cast ops further down in the IR. I.e., try to
+  // wrap results instead of operands in a cast.
+  for (int i = 0; i < static_cast<int>(unrealizedConversions.size()); ++i) {
+    UnrealizedConversionCastOp conversion = unrealizedConversions[i];
+    assert(conversion->getNumOperands() == 1 &&
+           conversion->getNumResults() == 1 &&
+           "expected single operand and single result");
+    SmallVector<Operation *> users = llvm::to_vector(conversion->getUsers());
+    for (Operation *user : users) {
+      // Handle common memref dialect ops that produce new memrefs and must
+      // be recreated with the new result type.
+      if (auto subviewOp = dyn_cast<SubViewOp>(user)) {
+        unrealizedConversions.push_back(
+            propagateSubViewOp(rewriter, conversion, subviewOp));
+        continue;
+      }
+
+      // TODO: Other memref ops such as memref.collapse_shape/expand_shape
+      // should also be handled here.
+
+      // Skip any ops that produce MemRef result or have MemRef region block
+      // arguments. These may need special handling (e.g., scf.for).
+      if (llvm::any_of(user->getResultTypes(),
+                       [](Type t) { return isa<MemRefType>(t); }))
+        continue;
+      if (llvm::any_of(user->getRegions(), [](Region &r) {
+            return llvm::any_of(r.getArguments(), [](BlockArgument bbArg) {
+              return isa<MemRefType>(bbArg.getType());
+            });
+          }))
+        continue;
+
+      // For all other ops, we assume that we can directly replace the operand.
+      // This may have to be revised in the future; e.g., there may be ops that
+      // do not support non-identity layout maps.
+      for (OpOperand &operand : user->getOpOperands()) {
+        if (auto castOp =
+                operand.get().getDefiningOp<UnrealizedConversionCastOp>()) {
+          rewriter.updateRootInPlace(
+              user, [&]() { operand.set(conversion->getOperand(0)); });
+        }
+      }
+    }
+  }
+
+  // Erase all unrealized_conversion_cast ops without uses.
+  for (auto op : unrealizedConversions)
+    if (op->getUses().empty())
+      rewriter.eraseOp(op);
+}
+
+FailureOr<Value> memref::replaceWithIndependentOp(RewriterBase &rewriter,
+                                                  memref::AllocaOp allocaOp,
+                                                  ValueRange independencies) {
+  auto replacement =
+      memref::buildIndependentOp(rewriter, allocaOp, independencies);
+  if (failed(replacement))
+    return failure();
+  replaceAndPropagateMemRefType(rewriter, allocaOp,
+                                replacement->getDefiningOp());
+  return replacement;
+}
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index ff431bf76f20e..2649e1bf0d433 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -56,9 +56,8 @@ void OpenACCDialect::initialize() {
 LogicalResult acc::DataBoundsOp::verify() {
   auto extent = getExtent();
   auto upperbound = getUpperbound();
-  if (!extent && !upperbound) {
+  if (!extent && !upperbound)
     return emitError("expected extent or upperbound.");
-  }
   return success();
 }
 
@@ -66,10 +65,9 @@ LogicalResult acc::DataBoundsOp::verify() {
 // DevicePtrOp
 //===----------------------------------------------------------------------===//
 LogicalResult acc::DevicePtrOp::verify() {
-  if (getDataClause() != acc::DataClause::acc_deviceptr) {
+  if (getDataClause() != acc::DataClause::acc_deviceptr)
     return emitError("data clause associated with deviceptr operation must "
                      "match its intent");
-  }
   return success();
 }
 
@@ -77,10 +75,9 @@ LogicalResult acc::DevicePtrOp::verify() {
 // PresentOp
 //===----------------------------------------------------------------------===//
 LogicalResult acc::PresentOp::verify() {
-  if (getDataClause() != acc::DataClause::acc_present) {
+  if (getDataClause() != acc::DataClause::acc_present)
     return emitError(
         "data clause associated with present operation must match its intent");
-  }
   return success();
 }
 
@@ -91,11 +88,10 @@ LogicalResult acc::CopyinOp::verify() {
   // Test for all clauses this operation can be decomposed from:
   if (getDataClause() != acc::DataClause::acc_copyin &&
       getDataClause() != acc::DataClause::acc_copyin_readonly &&
-      getDataClause() != acc::DataClause::acc_copy) {
+      getDataClause() != acc::DataClause::acc_copy)
     return emitError(
         "data clause associated with copyin operation must match its intent"
         " or specify original clause this operation was decomposed from");
-  }
   return success();
 }
 
@@ -111,11 +107,10 @@ LogicalResult acc::CreateOp::verify() {
   if (getDataClause() != acc::DataClause::acc_create &&
       getDataClause() != acc::DataClause::acc_create_zero &&
       getDataClause() != acc::DataClause::acc_copyout &&
-      getDataClause() != acc::DataClause::acc_copyout_zero) {
+      getDataClause() != acc::DataClause::acc_copyout_zero)
     return emitError(
         "data clause associated with create operation must match its intent"
         " or specify original clause this operation was decomposed from");
-  }
   return success();
 }
 
@@ -129,10 +124,9 @@ bool acc::CreateOp::isCreateZero() {
 // NoCreateOp
 //===----------------------------------------------------------------------===//
 LogicalResult acc::NoCreateOp::verify() {
-  if (getDataClause() != acc::DataClause::acc_no_create) {
+  if (getDataClause() != acc::DataClause::acc_no_create)
     return emitError("data clause associated with no_create operation must "
                      "match its intent");
-  }
   return success();
 }
 
@@ -140,10 +134,9 @@ LogicalResult acc::NoCreateOp::verify() {
 // AttachOp
 //===----------------------------------------------------------------------===//
 LogicalResult acc::AttachOp::verify() {
-  if (getDataClause() != acc::DataClause::acc_attach) {
+  if (getDataClause() != acc::DataClause::acc_attach)
     return emitError(
         "data clause associated with attach operation must match its intent");
-  }
   return success();
 }
 
@@ -157,9 +150,8 @@ LogicalResult acc::GetDevicePtrOp::verify() {
   if (getDataClause() != acc::DataClause::acc_getdeviceptr &&
       getDataClause() != acc::DataClause::acc_copyout &&
       getDataClause() != acc::DataClause::acc_delete &&
-      getDataClause() != acc::DataClause::acc_detach) {
+      getDataClause() != acc::DataClause::acc_detach)
     return emitError("getDevicePtr mismatch");
-  }
   return success();
 }
 
@@ -170,14 +162,12 @@ LogicalResult acc::CopyoutOp::verify() {
   // Test for all clauses this operation can be decomposed from:
   if (getDataClause() != acc::DataClause::acc_copyout &&
       getDataClause() != acc::DataClause::acc_copyout_zero &&
-      getDataClause() != acc::DataClause::acc_copy) {
+      getDataClause() != acc::DataClause::acc_copy)
     return emitError(
         "data clause associated with copyout operation must match its intent"
         " or specify original clause this operation was decomposed from");
-  }
-  if (!getVarPtr() || !getAccPtr()) {
+  if (!getVarPtr() || !getAccPtr())
     return emitError("must have both host and device pointers");
-  }
   return success();
 }
 
@@ -192,14 +182,12 @@ LogicalResult acc::DeleteOp::verify() {
   // Test for all clauses this operation can be decomposed from:
   if (getDataClause() != acc::DataClause::acc_delete &&
       getDataClause() != acc::DataClause::acc_create &&
-      getDataClause() != acc::DataClause::acc_create_zero) {
+      getDataClause() != acc::DataClause::acc_create_zero)
     return emitError(
         "data clause associated with delete operation must match its intent"
         " or specify original clause this operation was decomposed from");
-  }
-  if (!getVarPtr() && !getAccPtr()) {
+  if (!getVarPtr() && !getAccPtr())
     return emitError("must have either host or device pointer");
-  }
   return success();
 }
 
@@ -209,14 +197,12 @@ LogicalResult acc::DeleteOp::verify() {
 LogicalResult acc::DetachOp::verify() {
   // Test for all clauses this operation can be decomposed from:
   if (getDataClause() != acc::DataClause::acc_detach &&
-      getDataClause() != acc::DataClause::acc_attach) {
+      getDataClause() != acc::DataClause::acc_attach)
     return emitError(
         "data clause associated with detach operation must match its intent"
         " or specify original clause this operation was decomposed from");
-  }
-  if (!getVarPtr() && !getAccPtr()) {
+  if (!getVarPtr() && !getAccPtr())
     return emitError("must have either host or device pointer");
-  }
   return success();
 }
 
@@ -228,10 +214,9 @@ static ParseResult parseRegions(OpAsmParser &parser, OperationState &state,
   for (unsigned i = 0; i < nRegions; ++i)
     regions.push_back(state.addRegion());
 
-  for (Region *region : regions) {
+  for (Region *region : regions)
     if (parser.parseRegion(*region, /*arguments=*/{}, /*argTypes=*/{}))
       return failure();
-  }
 
   return success();
 }
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 76b3589b15f49..cbd98f39b4068 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -1779,7 +1779,7 @@ bool mlir::scf::insideMutuallyExclusiveBranches(Operation *a, Operation *b) {
 LogicalResult
 IfOp::inferReturnTypes(MLIRContext *ctx, std::optional<Location> loc,
                        ValueRange operands, DictionaryAttr attrs,
-                       RegionRange regions,
+                       OpaqueProperties properties, RegionRange regions,
                        SmallVectorImpl<Type> &inferredReturnTypes) {
   if (regions.empty())
     return failure();
@@ -1872,7 +1872,8 @@ void IfOp::build(OpBuilder &builder, OperationState &result, Value cond,
   MLIRContext *ctx = builder.getContext();
   auto attrDict = DictionaryAttr::get(ctx, result.attributes);
   if (succeeded(inferReturnTypes(ctx, std::nullopt, result.operands, attrDict,
-                                 result.regions, inferredReturnTypes))) {
+                                 /*properties=*/nullptr, result.regions,
+                                 inferredReturnTypes))) {
     result.addTypes(inferredReturnTypes);
   }
 }
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 3083bb8d8ce69..1f708c2ae37ad 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -28,7 +28,7 @@ using namespace mlir;
 
 static constexpr unsigned maxFreeID = 1 << 20;
 
-/// Returns an unsed symbol in `module` for `oldSymbolName` by trying numeric
+/// Returns an unused symbol in `module` for `oldSymbolName` by trying numeric
 /// suffix in `lastUsedID`.
 static StringAttr renameSymbol(StringRef oldSymName, unsigned &lastUsedID,
                                spirv::ModuleOp module) {
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
index 1713c4490f960..97f16d1b1b95f 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UnifyAliasedResourcePass.cpp
@@ -220,6 +220,9 @@ ResourceAliasAnalysis::ResourceAliasAnalysis(Operation *root) {
 }
 
 bool ResourceAliasAnalysis::shouldUnify(Operation *op) const {
+  if (!op)
+    return false;
+
   if (auto varOp = dyn_cast<spirv::GlobalVariableOp>(op)) {
     auto canonicalOp = getCanonicalResource(varOp);
     return canonicalOp && varOp != canonicalOp;
@@ -566,16 +569,15 @@ class UnifyAliasedResourcePass final
 private:
   spirv::GetTargetEnvFn getTargetEnvFn;
 };
-} // namespace
 
 void UnifyAliasedResourcePass::runOnOperation() {
   spirv::ModuleOp moduleOp = getOperation();
   MLIRContext *context = &getContext();
 
   if (getTargetEnvFn) {
-    // This pass is only needed for targeting WebGPU, Metal, or layering Vulkan
-    // on Metal via MoltenVK, where we need to translate SPIR-V into WGSL or
-    // MSL. The translation has limitations.
+    // This pass is only needed for targeting WebGPU, Metal, or layering
+    // Vulkan on Metal via MoltenVK, where we need to translate SPIR-V into
+    // WGSL or MSL. The translation has limitations.
     spirv::TargetEnvAttr targetEnv = getTargetEnvFn(moduleOp);
     spirv::ClientAPI clientAPI = targetEnv.getClientAPI();
     bool isVulkanOnAppleDevices =
@@ -614,6 +616,7 @@ void UnifyAliasedResourcePass::runOnOperation() {
       resources.front()->removeAttr("aliased");
   }
 }
+} // namespace
 
 std::unique_ptr<mlir::OperationPass<spirv::ModuleOp>>
 spirv::createUnifyAliasedResourcePass(spirv::GetTargetEnvFn getTargetEnv) {
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 3417388d0bb92..d198b00b9caba 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -393,7 +393,7 @@ void AssumingOp::build(
 
 LogicalResult mlir::shape::AddOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType().isa<SizeType>() ||
       operands[1].getType().isa<SizeType>())
@@ -911,7 +911,7 @@ void ConstShapeOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
 
 LogicalResult mlir::shape::ConstShapeOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   Builder b(context);
   auto shape = attributes.getAs<DenseIntElementsAttr>("shape");
@@ -1092,7 +1092,7 @@ OpFoldResult DimOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::DimOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   DimOpAdaptor dimOp(operands);
   inferredReturnTypes.assign({dimOp.getIndex().getType()});
@@ -1140,7 +1140,7 @@ OpFoldResult DivOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::DivOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType().isa<SizeType>() ||
       operands[1].getType().isa<SizeType>())
@@ -1361,7 +1361,7 @@ void GetExtentOp::build(OpBuilder &builder, OperationState &result, Value shape,
 
 LogicalResult mlir::shape::GetExtentOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   inferredReturnTypes.assign({IndexType::get(context)});
   return success();
@@ -1399,7 +1399,7 @@ OpFoldResult IsBroadcastableOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::MeetOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands.empty())
     return failure();
@@ -1535,7 +1535,7 @@ void shape::RankOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
 
 LogicalResult mlir::shape::RankOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType().isa<ShapeType>())
     inferredReturnTypes.assign({SizeType::get(context)});
@@ -1571,7 +1571,7 @@ OpFoldResult NumElementsOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::NumElementsOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType().isa<ShapeType>())
     inferredReturnTypes.assign({SizeType::get(context)});
@@ -1603,7 +1603,7 @@ OpFoldResult MaxOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::MaxOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType() == operands[1].getType())
     inferredReturnTypes.assign({operands[0].getType()});
@@ -1635,7 +1635,7 @@ OpFoldResult MinOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::MinOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType() == operands[1].getType())
     inferredReturnTypes.assign({operands[0].getType()});
@@ -1672,7 +1672,7 @@ OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
 
 LogicalResult mlir::shape::MulOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType().isa<SizeType>() ||
       operands[1].getType().isa<SizeType>())
@@ -1759,7 +1759,7 @@ void ShapeOfOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
 
 LogicalResult mlir::shape::ShapeOfOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType().isa<ValueShapeType>())
     inferredReturnTypes.assign({ShapeType::get(context)});
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index b2353016079b7..4b98030a9dbfa 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -449,8 +449,6 @@ mlir::sparse_tensor::getSparseTensorEncoding(Type type) {
   return nullptr;
 }
 
-/// Returns true iff the given sparse tensor encoding attribute has a trailing
-/// COO region starting at the given level.
 bool mlir::sparse_tensor::isCOOType(SparseTensorEncodingAttr enc,
                                     Level startLvl, bool isUnique) {
   if (!enc ||
@@ -719,7 +717,11 @@ LogicalResult UnpackOp::verify() {
   const auto coordinatesTp = getRankedTensorType(getCoordinates());
   const auto srcTp = getSparseTensorType(getTensor());
   return verifyPackUnPack(*this, false, srcTp, valuesTp, coordinatesTp,
-                          nullptr);
+                          getBatchedLvlsAttr());
+}
+
+unsigned UnpackOp::getNumBatchedLvls() {
+  return getBatchedLvls().has_value() ? getBatchedLvls()->getZExtValue() : 0;
 }
 
 LogicalResult ConvertOp::verify() {
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 8a8b2eda5175d..f17c001308bf0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -153,9 +153,12 @@ struct UnpackOpInterface
     : public BufferizableOpInterface::ExternalModel<UnpackOpInterface,
                                                     sparse_tensor::UnpackOp> {
   bool bufferizesToAllocation(Operation *op, OpResult opResult) const {
-    // Similar to InsertOp, reallocation is not considered to allocate a new
-    // piece of memory.
-    return false;
+    // We allocate and return unpacked memory if this is a batched unpack.
+    // When the number of batched levels equals to zero, we reuse the
+    // coordinates/values memref (and reallocation if the requested output size
+    // is larger than the actual size). Similar to InsertOp, reallocation is
+    // not considered to allocate a new piece of memory.
+    return llvm::cast<UnpackOp>(op).getNumBatchedLvls() != 0;
   }
 
   bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h
index 1ee5c19b284fe..d9ef20220cae5 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h
@@ -85,6 +85,25 @@ class CodegenEnv {
     return latticeMerger.getDimLevelType(b);
   }
 
+  //
+  // LoopEmitter delegates.
+  //
+
+  TensorLevel makeTensorLevel(TensorId t, Level l) const {
+    // Make sure LoopEmitter, GenericOp, and Merger agree on the number of
+    // tensors. Merger has one more synthetic tensor for loop invariants.
+    assert(loopEmitter.getNumTensors() == linalgOp->getNumOperands() &&
+           loopEmitter.getNumTensors() == latticeMerger.getNumTensors() - 1);
+    return loopEmitter.makeTensorLevel(t, l);
+  }
+  std::pair<TensorId, Level> unpackTensorLevel(TensorLevel tl) const {
+    return loopEmitter.unpackTensorLevel(tl);
+  }
+  template <class ContainerTy>
+  auto unpackTensorLevelRange(ContainerTy &&c) const {
+    return loopEmitter.unpackTensorLevelRange(std::forward<ContainerTy>(c));
+  }
+
   //
   // Code generation environment verify functions.
   //
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
index 3a488b311b95a..54a6786231e90 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
@@ -213,6 +213,18 @@ Value sparse_tensor::genCast(OpBuilder &builder, Location loc, Value value,
   return mlir::convertScalarToDtype(builder, loc, value, dstTp, isUnsignedCast);
 }
 
+Value sparse_tensor::genIndexLoad(OpBuilder &builder, Location loc, Value mem,
+                                  Value s) {
+  Value load = builder.create<memref::LoadOp>(loc, mem, s);
+  if (!load.getType().isa<IndexType>()) {
+    if (load.getType().getIntOrFloatBitWidth() < 64)
+      load = builder.create<arith::ExtUIOp>(loc, builder.getI64Type(), load);
+    load =
+        builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), load);
+  }
+  return load;
+}
+
 mlir::TypedAttr mlir::sparse_tensor::getOneAttr(Builder &builder, Type tp) {
   if (tp.isa<FloatType>())
     return builder.getFloatAttr(tp, 1.0);
@@ -376,8 +388,12 @@ func::CallOp mlir::sparse_tensor::createFuncCall(
   return builder.create<func::CallOp>(loc, resultType, fn, operands);
 }
 
-Type mlir::sparse_tensor::getOpaquePointerType(OpBuilder &builder) {
-  return LLVM::LLVMPointerType::get(builder.getI8Type());
+Type mlir::sparse_tensor::getOpaquePointerType(MLIRContext *ctx) {
+  return LLVM::LLVMPointerType::get(IntegerType::get(ctx, 8));
+}
+
+Type mlir::sparse_tensor::getOpaquePointerType(Builder &builder) {
+  return getOpaquePointerType(builder.getContext());
 }
 
 Value mlir::sparse_tensor::genAlloca(OpBuilder &builder, Location loc,
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
index b6e6def4e5860..28acf1aed7de2 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
@@ -75,6 +75,11 @@ StringRef primaryTypeFunctionSuffix(Type elemTp);
 /// Add type casting between arith and index types when needed.
 Value genCast(OpBuilder &builder, Location loc, Value value, Type dstTy);
 
+/// Generates a pointer/index load from the sparse storage scheme. Narrower
+/// data types need to be zero extended before casting the value into the
+/// index type used for looping and indexing.
+Value genIndexLoad(OpBuilder &builder, Location loc, Value mem, Value s);
+
 /// Generates a 1-valued attribute of the given type.  This supports
 /// all the same types as `getZeroAttr`; however, unlike `getZeroAttr`,
 /// for unsupported types we raise `llvm_unreachable` rather than
@@ -116,7 +121,8 @@ func::CallOp createFuncCall(OpBuilder &builder, Location loc, StringRef name,
 
 /// Returns the equivalent of `void*` for opaque arguments to the
 /// execution engine.
-Type getOpaquePointerType(OpBuilder &builder);
+Type getOpaquePointerType(MLIRContext *ctx);
+Type getOpaquePointerType(Builder &builder);
 
 /// Generates an uninitialized temporary buffer of the given size and
 /// type, but returns it as type `memref<? x $tp>` (rather than as type
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
index afa4828bf170a..731a1a9e460e5 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
@@ -41,25 +41,6 @@ using namespace mlir::sparse_tensor;
 // File local helper functions.
 //===----------------------------------------------------------------------===//
 
-/// Generates a pointer/index load from the sparse storage scheme. Narrower
-/// data types need to be zero extended before casting the value into the
-/// index type used for looping and indexing.
-static Value genIndexLoad(OpBuilder &builder, Location loc, Value mem,
-                          Value s) {
-  // For the scalar case, we simply zero extend narrower indices into 64-bit
-  // values before casting to index without a performance penalty. Here too,
-  // however, indices that already are 64-bit, in theory, cannot express the
-  // full range as explained above.
-  Value load = builder.create<memref::LoadOp>(loc, mem, s);
-  if (!load.getType().isa<IndexType>()) {
-    if (load.getType().getIntOrFloatBitWidth() < 64)
-      load = builder.create<arith::ExtUIOp>(loc, builder.getI64Type(), load);
-    load =
-        builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), load);
-  }
-  return load;
-}
-
 static Value genSliceOffset(OpBuilder &builder, Location loc, Value tensor,
                             Level lvl) {
   auto enc = getSparseTensorEncoding(tensor.getType());
@@ -475,13 +456,12 @@ void LoopEmitter::initializeLoopEmit(OpBuilder &builder, Location loc,
 }
 
 void LoopEmitter::enterNewLoopSeq(OpBuilder &builder, Location loc,
-                                  ArrayRef<TensorId> tids,
-                                  ArrayRef<Level> lvls) {
+                                  ArrayRef<TensorLevel> tidLvls) {
   // TODO: sort
   assert(loopSeqStack.size() == loopStack.size());
   // Prepares for all the tensors used in the current loop sequence.
   std::vector<std::tuple<TensorId, Level, bool>> slicedTids;
-  for (auto [tid, lvl] : llvm::zip(tids, lvls)) {
+  for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
     if (!dependentLvlMap[tid][lvl].empty()) {
       bool fullyRed = genSliceBegin(builder, loc, tid, lvl);
       slicedTids.emplace_back(tid, lvl, fullyRed);
@@ -679,17 +659,19 @@ Operation *LoopEmitter::emitWhileLoopOverSliceAtSparseLvl(
   return loop;
 }
 
-Operation *LoopEmitter::enterLoopOverTensorAtLvl(
-    OpBuilder &builder, Location loc, ArrayRef<TensorId> tids,
-    ArrayRef<Level> lvls, MutableArrayRef<Value> reduc, bool isParallel) {
+Operation *LoopEmitter::enterLoopOverTensorAtLvl(OpBuilder &builder,
+                                                 Location loc,
+                                                 ArrayRef<TensorLevel> tidLvls,
+                                                 MutableArrayRef<Value> reduc,
+                                                 bool isParallel) {
   // TODO: support multiple return on parallel for?
   assert(!isParallel || reduc.size() <= 1);
   bool isSparseCond = false, isSparseSliceCond = false;
-  size_t tid = tids.front(), lvl = lvls.front();
+  auto [tid, lvl] = unpackTensorLevel(tidLvls.front());
 
   // Finds out the tensor level that we should use to generate loops. Amongs all
   // the tensor levels, there is at most one sparse tensor level.
-  for (auto [t, l] : llvm::zip(tids, lvls)) {
+  for (auto [t, l] : unpackTensorLevelRange(tidLvls)) {
     assert(lvlTypes[t].size() > l);         // Must be a valid tid, dim pair
     assert(!coords[t][l] ||                 // We cannot re-enter the same level
            !dependentLvlMap[t][l].empty()); // unless it is a slice-driver loop
@@ -707,7 +689,8 @@ Operation *LoopEmitter::enterLoopOverTensorAtLvl(
       continue;
     }
 
-    bool isSparse = isCompressedDLT(lvlType) || isSingletonDLT(lvlType);
+    bool isSparse = isCompressedDLT(lvlType) || isSingletonDLT(lvlType) ||
+                    isCompressedWithHiDLT(lvlType);
     // We can at most have one sparse input, otherwise, a while loop is
     // required to co-iterate multiple sparse tensors.
     assert(!isSparseCond || !isSparse);
@@ -730,12 +713,9 @@ Operation *LoopEmitter::enterLoopOverTensorAtLvl(
   Operation *l = nullptr;
 
   // At most one tensor used as condition in for loop;
-  SmallVector<TensorId, 1> condTid;
-  SmallVector<Level, 1> condLvl;
-  // There Might be multiple dense slice driven tensor.
-  SmallVector<TensorId> sliceTids;
-  SmallVector<Level> sliceLvls;
-  SmallVector<bool> sliceReduc;
+  SmallVector<TensorLevel, 1> condTidLvl;
+  // There might be multiple dense slice driven tensor.
+  SmallVector<SliceLoopInfo> sliceDrivenInfo;
 
   // Generates loops differently depending on whether we need a slice-driven
   // loop or a simple level traversal loop.
@@ -752,9 +732,7 @@ Operation *LoopEmitter::enterLoopOverTensorAtLvl(
           lvl, reduc);
     }
     levelReducedDep[tid][lvl]++;
-    sliceTids.push_back(tid);
-    sliceLvls.push_back(lvl);
-    sliceReduc.push_back(fullyReduced);
+    sliceDrivenInfo.emplace_back(tid, lvl, fullyReduced);
   } else {
     Value lo = isSparseCond ? posits[tid][lvl]           // current offset
                             : loopSeqStack.back().first; // universal index
@@ -765,21 +743,19 @@ Operation *LoopEmitter::enterLoopOverTensorAtLvl(
       // Adjust for loop hi for dense slice-driven loop.
       if (fullyReduced) {
         hi = sliceSz;
-        condTid.push_back(tid);
-        condLvl.push_back(lvl);
+        condTidLvl.push_back(makeTensorLevel(tid, lvl));
       } else {
         hi = SUBI(lvlSizes[tid][lvl], sliceSz);
         hi = ADDI(hi, C_IDX(1));
       }
     } else {
-      condTid.push_back(tid);
-      condLvl.push_back(lvl);
+      condTidLvl.push_back(makeTensorLevel(tid, lvl));
     }
     l = emitForLoopOverTensorAtLvl(builder, loc, tid, lvl, lo, hi, reduc,
                                    isParallel);
   }
   Value iv = coords[tid][lvl];
-  for (auto [t, l] : llvm::zip(tids, lvls)) {
+  for (auto [t, l] : unpackTensorLevelRange(tidLvls)) {
     // We only need to handle slice-driven loops on dense level here.
     // If it is a slice-driven loop on sparse level, it needs a while loop to
     // insert break statements, and it must have been handled correctly in L692.
@@ -792,9 +768,7 @@ Operation *LoopEmitter::enterLoopOverTensorAtLvl(
       } else {
         // Puts sliced dense loop into LoopInfo so that LoopEmitter knows how to
         // exit it.
-        sliceTids.push_back(t);
-        sliceLvls.push_back(l);
-        sliceReduc.push_back(fullyReduc);
+        sliceDrivenInfo.emplace_back(t, l, fullyReduc);
         // Update the slice information as we enter the new loop.
         assert(*info.slicedOnLvl == l);
         info.minCrd = info.offset = iv;
@@ -805,10 +779,10 @@ Operation *LoopEmitter::enterLoopOverTensorAtLvl(
   }
   // NOTE: we can also prepare for next dim here in advance
   // Pushes the loop into stack.
-  loopStack.emplace_back(condTid, condLvl, sliceTids, sliceLvls, sliceReduc, l,
+  loopStack.emplace_back(condTidLvl, sliceDrivenInfo, l,
                          builder.getInsertionBlock(), iv, loopTag);
   // Emit extra locals.
-  emitExtraLocalsForTensorsAtDenseLvls(builder, loc, tids, lvls);
+  emitExtraLocalsForTensorsAtDenseLvls(builder, loc, tidLvls);
   return l;
 }
 
@@ -872,16 +846,17 @@ Operation *LoopEmitter::enterFilterLoopOverTensorAtLvl(
 
   // NOTE: we can also prepare for next lvl here in advance
   // Push the loop into stack
-  loopStack.emplace_back(ArrayRef<TensorId>(tid), ArrayRef<Level>(lvl),
-                         ArrayRef<TensorId>(), ArrayRef<Level>(),
-                         ArrayRef<bool>(), forOp, builder.getInsertionBlock(),
-                         coords[tid][lvl], nullptr);
+  loopStack.emplace_back(ArrayRef<TensorLevel>(makeTensorLevel(tid, lvl)),
+                         ArrayRef<SliceLoopInfo>(), forOp,
+                         builder.getInsertionBlock(), coords[tid][lvl],
+                         nullptr);
   return forOp;
 }
 
 void LoopEmitter::genDenseAffineAddress(OpBuilder &builder, Location loc,
-                                        TensorId tid, Level lvl,
+                                        TensorLevel tidLvl,
                                         AffineExpr lvlExpr) {
+  auto [tid, lvl] = unpackTensorLevel(tidLvl);
   assert(isDenseDLT(lvlTypes[tid][lvl]));
   // For dense levels, the level-coordinate also serves as the position.
   Value lvlCrd = genAffine(builder, loc, lvlExpr);
@@ -889,16 +864,15 @@ void LoopEmitter::genDenseAffineAddress(OpBuilder &builder, Location loc,
 }
 
 Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
-    OpBuilder &builder, Location loc, ArrayRef<TensorId> tids,
-    ArrayRef<Level> lvls, bool needsUniv, MutableArrayRef<Value> reduc) {
+    OpBuilder &builder, Location loc, ArrayRef<TensorLevel> tidLvls,
+    bool needsUniv, MutableArrayRef<Value> reduc) {
   // NOTE: the slice driven tensor-related reduction variable must
   // appear before normal tensors.
-  assert(tids.size() == lvls.size());
   SmallVector<Type> types;
   SmallVector<Value> operands;
   // Construct the while-loop with a parameter for each coordinate.
   const Type indexType = builder.getIndexType();
-  for (auto [tid, lvl] : llvm::zip(tids, lvls)) {
+  for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
     // TODO: support coiteration with slice driven tensors.
     const auto lvlTp = lvlTypes[tid][lvl];
     assert(dependentLvlMap[tid][lvl].empty() && "TODO: not yet implemented");
@@ -940,7 +914,7 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
   builder.setInsertionPointToStart(&whileOp.getBefore().front());
   Value cond;
   unsigned o = 0;
-  for (auto [t, lvl] : llvm::zip(tids, lvls)) {
+  for (auto [t, lvl] : unpackTensorLevelRange(tidLvls)) {
     const TensorId tid = t; // Why `t` can not be captured by lambda?
     const auto lvlTp = lvlTypes[tid][lvl];
     if (isCompressedDLT(lvlTp) || isSingletonDLT(lvlTp) ||
@@ -974,7 +948,7 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
 
   SmallVector<std::pair<Value, unsigned>> slicesPreds;
   unsigned i = 0;
-  for (auto [tid, lvl] : llvm::zip(tids, lvls)) {
+  for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
     // Prepares for next level.
     const auto lvlTp = lvlTypes[tid][lvl];
     if (isCompressedDLT(lvlTp) || isSingletonDLT(lvlTp) ||
@@ -1025,7 +999,7 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
   Value min;
   // Finds the minimum coordinate
   if (!needsUniv) {
-    for (auto [tid, lvl] : llvm::zip(tids, lvls)) {
+    for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
       const auto lvlTp = lvlTypes[tid][lvl];
       if (isCompressedDLT(lvlTp) || isSingletonDLT(lvlTp) ||
           isCompressedWithHiDLT(lvlTp)) {
@@ -1045,12 +1019,11 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
   }
 
   // Sets up the loop stack.
-  loopStack.emplace_back(tids, lvls, ArrayRef<TensorId>(), ArrayRef<Level>(),
-                         ArrayRef<bool>(), whileOp, builder.getInsertionBlock(),
-                         min, loopTag);
+  loopStack.emplace_back(tidLvls, ArrayRef<SliceLoopInfo>(), whileOp,
+                         builder.getInsertionBlock(), min, loopTag);
   assert(loopStack.size() == loopSeqStack.size());
 
-  for (auto [tid, dstLvl] : llvm::zip(tids, lvls)) {
+  for (auto [tid, dstLvl] : unpackTensorLevelRange(tidLvls)) {
     const auto reassoc = getCollapseReassociation(tid, dstLvl);
     assert(reassoc.size() == 1 || isUniqueCOOType(tensors[tid].getType()));
     // TODO: Refactors this into smaller functions.
@@ -1097,7 +1070,7 @@ Operation *LoopEmitter::enterCoIterationOverTensorsAtLvls(
   }
 
   // Emits extra locals
-  emitExtraLocalsForTensorsAtDenseLvls(builder, loc, tids, lvls);
+  emitExtraLocalsForTensorsAtDenseLvls(builder, loc, tidLvls);
 
   // Updates reduction variables
   assert(after->getNumArguments() == o + reduc.size() + (needsUniv ? 1 : 0));
@@ -1158,15 +1131,12 @@ void LoopEmitter::prepareLoopOverTensorAtLvl(OpBuilder &builder, Location loc,
   llvm_unreachable("Unrecognized level-type!");
 }
 
-void LoopEmitter::emitExtraLocalsForTensorsAtDenseLvls(OpBuilder &builder,
-                                                       Location loc,
-                                                       ArrayRef<TensorId> tids,
-                                                       ArrayRef<Level> lvls) {
+void LoopEmitter::emitExtraLocalsForTensorsAtDenseLvls(
+    OpBuilder &builder, Location loc, ArrayRef<TensorLevel> tidLvls) {
   // Initialize dense positions. Note that we generate dense coordinates of the
   // output tensor unconditionally, since they may not appear in the lattice,
   // but may be needed for linearized codegen.
-  assert(tids.size() == lvls.size());
-  for (auto [tid, lvl] : llvm::zip(tids, lvls)) {
+  for (auto [tid, lvl] : unpackTensorLevelRange(tidLvls)) {
     if (isDenseDLT(lvlTypes[tid][lvl])) {
       // Slice-driven dense level should have be handled already.
       if (!dependentLvlMap[tid][lvl].empty())
@@ -1193,8 +1163,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
                               MutableArrayRef<Value> reduc) {
   const LoopInfo &loopInfo = loopStack.back();
   rewriter.setInsertionPointToEnd(loopInfo.userCodeBlock);
-  for (auto [tid, lvl, reduced] : llvm::zip(
-           loopInfo.slicedTids, loopInfo.slicedLvls, loopInfo.sliceReduced)) {
+  for (auto [tid, lvl, reduced] : loopInfo.sliceDrivenInfo) {
     SliceInfo &info = sliceStack[tid].back();
     assert(isDenseDLT(lvlTypes[tid][lvl]));
     assert(*info.slicedOnLvl == lvl && !reduced);
@@ -1271,7 +1240,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc,
   // Finished iterating a tensor, clean up
   // We only do the clean up on for loop as while loops do not necessarily
   // finish the iteration on a sparse tensor
-  for (auto [tid, lvl] : llvm::zip(loopInfo.tids, loopInfo.lvls)) {
+  for (auto [tid, lvl] : unpackTensorLevelRange(loopInfo.tidLvls)) {
     // Reset to null.
     coords[tid][lvl] = Value();
     posits[tid][lvl] = Value();
@@ -1296,8 +1265,7 @@ void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc,
   unsigned o = 0;
   SmallVector<Value> operands;
   unsigned delta = 0;
-  for (auto [tid, lvl, resolved] : llvm::zip(
-           loopInfo.slicedTids, loopInfo.slicedLvls, loopInfo.sliceReduced)) {
+  for (auto [tid, lvl, resolved] : loopInfo.sliceDrivenInfo) {
     // TODO: handle dense.
     assert(isCompressedDLT(lvlTypes[tid][lvl]));
     levelReducedDep[tid][lvl]--;
@@ -1309,7 +1277,7 @@ void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc,
     // fully reduced while op for iterating one slices.
     // FIXME: since we didn't implement coiteration, this must be iteration
     // just on fully resolved slice.
-    assert(loopInfo.slicedTids.size() == 1 && loopInfo.tids.empty());
+    assert(loopInfo.sliceDrivenInfo.size() == 1 && loopInfo.tidLvls.empty());
     // The if guard to filter out out-range coordinates.
     assert(llvm::isa<scf::IfOp>(builder.getInsertionBlock()->getParentOp()));
     posits[tid][lvl] = whileOp->getResult(o++);
@@ -1326,7 +1294,7 @@ void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc,
   };
 
   Value one = C_IDX(1);
-  for (auto [tid, dstLvl] : llvm::zip(loopInfo.tids, loopInfo.lvls)) {
+  for (auto [tid, dstLvl] : unpackTensorLevelRange(loopInfo.tidLvls)) {
     const auto lvlTp = lvlTypes[tid][dstLvl];
     if (isCompressedDLT(lvlTp) || isSingletonDLT(lvlTp) ||
         isCompressedWithHiDLT(lvlTp)) {
@@ -1394,7 +1362,6 @@ void LoopEmitter::exitCurrentLoop(RewriterBase &rewriter, Location loc,
   // Clean up the values, it would help use to discover potential bug at a
   // earlier stage (instead of silently using a wrong value).
   const LoopInfo &loopInfo = loopStack.back();
-  assert(loopInfo.tids.size() == loopInfo.lvls.size());
   SmallVector<Value> red;
   if (llvm::isa<scf::WhileOp>(loopInfo.loop)) {
     exitWhileLoop(rewriter, loc, reduc);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h
index d069633836e6e..67a3f3d038a1d 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h
@@ -42,6 +42,8 @@ namespace sparse_tensor {
 // typecheck this to avoid mixups in the code.
 using LoopOrd = unsigned;
 
+// A compressed <tensor id, level> pair.
+using TensorLevel = unsigned;
 //===----------------------------------------------------------------------===//
 // SparseTensorLoopEmiter class, manages sparse tensors and helps to
 // generate loop structure to (co)-iterate sparse tensors.
@@ -134,7 +136,7 @@ class LoopEmitter {
   ///   // loop sequence end.
   /// }
   void enterNewLoopSeq(OpBuilder &builder, Location loc,
-                       ArrayRef<TensorId> tids, ArrayRef<Level> lvls);
+                       ArrayRef<TensorLevel> tidLvls);
 
   /// Exits the current loop sequence, this will reset universal index to 0.
   void exitCurrentLoopSeq(OpBuilder &builder, Location loc);
@@ -149,8 +151,7 @@ class LoopEmitter {
   /// The function will also perform in-place update on the `reduc` vector to
   /// return the reduction variable used inside the generated loop.
   Operation *enterLoopOverTensorAtLvl(OpBuilder &builder, Location loc,
-                                      ArrayRef<TensorId> tids,
-                                      ArrayRef<Level> lvls,
+                                      ArrayRef<TensorLevel> tidLvls,
                                       MutableArrayRef<Value> reduc = {},
                                       bool isParallel = false);
 
@@ -159,13 +160,13 @@ class LoopEmitter {
                                             AffineExpr affine,
                                             MutableArrayRef<Value> reduc = {});
 
-  void genDenseAffineAddress(OpBuilder &builder, Location loc, TensorId tid,
-                             Level lvl, AffineExpr lvlExpr);
+  void genDenseAffineAddress(OpBuilder &builder, Location loc,
+                             TensorLevel tidLvl, AffineExpr lvlExpr);
 
   /// Emits a co-iteration loop over a set of tensors.
   Operation *enterCoIterationOverTensorsAtLvls(
-      OpBuilder &builder, Location loc, ArrayRef<TensorId> tids,
-      ArrayRef<Level> lvls, bool needsUniv, MutableArrayRef<Value> reduc = {});
+      OpBuilder &builder, Location loc, ArrayRef<TensorLevel> tidLvls,
+      bool needsUniv, MutableArrayRef<Value> reduc = {});
 
   void exitCurrentLoop(RewriterBase &rewriter, Location loc,
                        MutableArrayRef<Value> reduc = {});
@@ -190,6 +191,31 @@ class LoopEmitter {
     return n < getCurrentDepth() ? loopStack[n].iv : Value();
   }
 
+  /// Gets the total number of tensors that loopEmitter is operating on.
+  unsigned getNumTensors() const { return tensors.size(); }
+
+  /// Compresses a TensorId and Level into a TensorLevel.
+  TensorLevel makeTensorLevel(TensorId t, Level l) const {
+    return l * getNumTensors() + t;
+  }
+
+  /// De-compresses a TensorLevel back to a pair of TensorId and Level.
+  std::pair<TensorId, Level> unpackTensorLevel(TensorLevel tidLvl) const {
+    unsigned nt = getNumTensors();
+    return std::make_pair(tidLvl % nt, tidLvl / nt);
+  }
+
+  /// Converts a range of TensorLevel to a range of std::pair<TensorId, Level>
+  template <class ContainerTy>
+  auto unpackTensorLevelRange(ContainerTy &&c) const {
+    using EltTy = decltype(*c.begin());
+    static_assert(std::is_same_v<llvm::remove_cvref_t<EltTy>, TensorLevel>,
+                  "Must be unpacking a TensorLevel range");
+    return llvm::map_range(std::forward<ContainerTy>(c), [this](EltTy tl) {
+      return this->unpackTensorLevel(tl);
+    });
+  }
+
   ///
   /// Getters.
   ///
@@ -209,32 +235,30 @@ class LoopEmitter {
   }
 
 private:
+  // A tuple that stored the slice-driven loop information.
+  struct SliceLoopInfo final {
+    SliceLoopInfo(TensorId tid, Level lvl, bool reduced)
+        : tid(tid), lvl(lvl), reduced(reduced) {}
+    TensorId tid;
+    Level lvl;
+    bool reduced;
+  };
   // LoopInfo stores information of a loop generated by LoopEmitter. E.g.,
   // the set of tensors levels that the loop is iterating over.
   struct LoopInfo final {
-    LoopInfo(ArrayRef<TensorId> tids, ArrayRef<Level> lvls,
-             ArrayRef<TensorId> slicedTids, ArrayRef<Level> slicedLvls,
-             ArrayRef<bool> sliceReduced, Operation *loop, Block *userBlock,
-             Value iv, StringAttr loopTag)
-        : tids(tids), lvls(lvls), slicedTids(slicedTids),
-          slicedLvls(slicedLvls), sliceReduced(sliceReduced), loop(loop),
+    LoopInfo(ArrayRef<TensorLevel> tidLvls,
+             ArrayRef<SliceLoopInfo> sliceDrivenInfo, Operation *loop,
+             Block *userBlock, Value iv, StringAttr loopTag)
+        : tidLvls(tidLvls), sliceDrivenInfo(sliceDrivenInfo), loop(loop),
           userCodeBlock(userBlock), iv(iv) {
       // Attached a special tag to loop emitter generated loop.
       if (loopTag)
         loop->setAttr(LoopEmitter::getLoopEmitterLoopAttrName(), loopTag);
     }
-    // TODO: maybe use a vector<pair> for tid and lvl?
-    //       (Or compress them together with a `TensorLoopId`.)
-    // The set of tensors that the loop is operating on
-    const llvm::SmallVector<TensorId> tids;
-    // The corresponding levels for the tensors
-    const llvm::SmallVector<Level> lvls;
-    // The set of tensors for slice-driven loop conditions.
-    const llvm::SmallVector<TensorId> slicedTids;
-    // The corresponding level for slice-driven tensors.
-    const llvm::SmallVector<Level> slicedLvls;
-    // Whether the tensor is fully reduced (e.g., i + j => j).
-    const llvm::SmallVector<bool> sliceReduced;
+    // The set of <tensor, lvl> that the loop is operating on
+    const llvm::SmallVector<TensorLevel> tidLvls;
+    // Slice-driven loop conditions.
+    const llvm::SmallVector<SliceLoopInfo> sliceDrivenInfo;
     const Operation *loop;      // the loop operation
     Block *const userCodeBlock; // the block holding users' generated code.
     const Value iv;             // the induction variable for the loop
@@ -295,8 +319,6 @@ class LoopEmitter {
                                                  Location loc, Value crd,
                                                  TensorId tid, Level lvl);
 
-  unsigned getNumTensors() const { return tensors.size(); }
-
   bool isOutputTensor(TensorId tid) const {
     return hasOutput && tid == getNumTensors() - 1;
   }
@@ -318,8 +340,7 @@ class LoopEmitter {
   /// point used to generate the loops, but are still required to generate
   /// expressions.
   void emitExtraLocalsForTensorsAtDenseLvls(OpBuilder &builder, Location loc,
-                                            ArrayRef<TensorId> tids,
-                                            ArrayRef<Level> lvls);
+                                            ArrayRef<TensorLevel> tidLvls);
 
   /// Emits a for loop to iterate over a tensor level with the provided lower
   /// bound `lo` and upper bound `hi`.
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index c1cb0926622f6..aebf0542b3332 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -602,6 +602,25 @@ static Value reallocOrSubView(OpBuilder &builder, Location loc, int64_t len,
   return ifOp.getResult(0);
 }
 
+static Value linearize(OpBuilder &builder, Location loc, ValueRange ivs,
+                       ValueRange bounds) {
+  assert(ivs.size() == bounds.size());
+  Value crd = constantIndex(builder, loc, 0);
+  for (unsigned i = 0, e = ivs.size(); i < e; i++) {
+    crd = builder.create<arith::AddIOp>(loc, crd, ivs[i]);
+    if (i != ivs.size() - 1)
+      crd = builder.create<arith::MulIOp>(loc, crd, bounds[i + 1]);
+  }
+  return crd;
+}
+
+ReassociationIndices getReassociationForFlattening(ShapedType srcTp) {
+  ReassociationIndices reassociation;
+  for (int i = 0, e = srcTp.getRank(); i < e; i++)
+    reassociation.push_back(i);
+  return reassociation;
+}
+
 //===----------------------------------------------------------------------===//
 // Codegen rules.
 //===----------------------------------------------------------------------===//
@@ -1252,12 +1271,7 @@ static void populateCompressedWithHiPosArray(OpBuilder &builder, Location loc,
       [&ubs, c0, c1, c2, nse, batV, posMemRef](OpBuilder &builder, Location loc,
                                                ValueRange ivs) {
         // Linearize index variables
-        Value crd = constantIndex(builder, loc, 0);
-        for (unsigned i = 0, e = ivs.size(); i < e; i++) {
-          crd = builder.create<arith::AddIOp>(loc, crd, ivs[i]);
-          if (i != ivs.size() - 1)
-            crd = builder.create<arith::MulIOp>(loc, crd, ubs[i + 1]);
-        }
+        Value crd = linearize(builder, loc, ivs, ubs);
         Value len = constantIndex(builder, loc, nse);
         Value pLo = builder.create<arith::MulIOp>(loc, crd, len);
         SmallVector<Value> indices(ivs.begin(), ivs.end());
@@ -1420,6 +1434,166 @@ struct SparsePackOpConverter : public OpConversionPattern<PackOp> {
   }
 };
 
+static LogicalResult genUnBatchedUnpackOp(UnpackOp op,
+                                          SparseTensorDescriptor desc,
+                                          ConversionPatternRewriter &rewriter) {
+  Location loc = op.getLoc();
+  const auto srcTp = getSparseTensorType(op.getTensor());
+  const Level lvlRank = srcTp.getLvlRank();
+  Value flatBuf = lvlRank == 1 ? desc.getCrdMemRefOrView(rewriter, loc, 0)
+                               : desc.getAOSMemRef();
+  Value valuesBuf = desc.getValMemRef();
+
+  // If frontend requests a static buffer, we reallocate the
+  // values/coordinates to ensure that we meet their need.
+  const auto valuesTp = getRankedTensorType(op.getValues());
+  if (valuesTp.hasStaticShape()) {
+    // FIXME: Reallocation is not always safe! E.g., if we are unpacking a
+    // tensor that is packed from constants.
+    valuesBuf =
+        reallocOrSubView(rewriter, loc, valuesTp.getShape()[0], valuesBuf);
+  }
+
+  const auto coordinatesTp = getRankedTensorType(op.getCoordinates());
+  if (coordinatesTp.hasStaticShape()) {
+    // FIXME: Reallocation is not always safe! E.g., if we are unpacking a
+    // tensor that is packed from constants.
+    auto len = coordinatesTp.getShape()[0] * coordinatesTp.getShape()[1];
+    flatBuf = reallocOrSubView(rewriter, loc, len, flatBuf);
+  }
+
+  Value coordinatesBuf = rewriter.create<memref::ExpandShapeOp>(
+      loc,
+      MemRefType::get(coordinatesTp.getShape(), coordinatesTp.getElementType()),
+      flatBuf, ArrayRef{ReassociationIndices{0, 1}});
+
+  // Converts MemRefs back to Tensors.
+  Value values = rewriter.create<bufferization::ToTensorOp>(loc, valuesBuf);
+  Value coordinates =
+      rewriter.create<bufferization::ToTensorOp>(loc, coordinatesBuf);
+  Value nse = genCast(rewriter, loc, desc.getValMemSize(rewriter, loc),
+                      op.getNse().getType());
+
+  rewriter.replaceOp(op, {values, coordinates, nse});
+  return success();
+}
+
+static LogicalResult genBatchedUnpackOp(UnpackOp op, unsigned nBatched,
+                                        SparseTensorDescriptor desc,
+                                        ConversionPatternRewriter &rewriter) {
+  assert(nBatched != 0);
+  Location loc = op.getLoc();
+  Value c0 = constantIndex(rewriter, loc, 0);
+  Value c1 = constantIndex(rewriter, loc, 1);
+  Value c2 = constantIndex(rewriter, loc, 2);
+
+  auto genZeroedAlloc = [loc,
+                         &rewriter](TensorType tt) -> TypedValue<MemRefType> {
+    auto mem = rewriter
+                   .create<memref::AllocOp>(
+                       loc, MemRefType::get(tt.getShape(), tt.getElementType()))
+                   .getMemref();
+    // TODO: Instead of filling the entire buffer, we can only fill the
+    // trailing zeros.
+    rewriter.create<linalg::FillOp>(
+        loc, ValueRange{constantZero(rewriter, loc, tt.getElementType())}, mem);
+    return mem;
+  };
+  SparseTensorType stt = getSparseTensorType(op.getTensor());
+  TensorType valTensorTp = op.getValues().getType();
+  TensorType crdTensorTp = op.getCoordinates().getType();
+  TypedValue<MemRefType> valMemref = genZeroedAlloc(valTensorTp);
+  TypedValue<MemRefType> crdMemref = genZeroedAlloc(crdTensorTp);
+  assert(valTensorTp.hasStaticShape() && crdTensorTp.hasStaticShape());
+
+  SmallVector<Value> lbs(nBatched, c0), steps(nBatched, c1);
+  SmallVector<Value> ubs;
+  for (unsigned i = 0; i < nBatched; i++) {
+    assert(!ShapedType::isDynamic(stt.getDimShape()[i]));
+    ubs.push_back(constantIndex(rewriter, loc, stt.getDimShape()[i]));
+  }
+
+  DimLevelType dlt = stt.getLvlType(nBatched);
+  assert(isCompressedDLT(dlt) || isCompressedWithHiDLT(dlt));
+  Value posStep = isCompressedDLT(dlt) ? c1  // forward position index by 1
+                                       : c2; // forward position index by 2
+  auto loopNest = scf::buildLoopNest(
+      rewriter, loc, lbs, ubs, steps, {c0 /*maximum nse*/},
+      [&ubs, c0, c1, posStep, desc, nBatched, &valMemref,
+       &crdMemref](OpBuilder &builder, Location loc, ValueRange ivs,
+                   ValueRange args) -> scf::ValueVector {
+        // crdMemref has shape: <... x nse x rank>
+        unsigned unBatchedRank = crdMemref.getType().getShape().back();
+        Value values = desc.getValMemRef();
+        Value flatCrds = unBatchedRank == 1
+                             ? desc.getCrdMemRefOrView(builder, loc, 0)
+                             : desc.getAOSMemRef();
+
+        Value positions = desc.getPosMemRef(nBatched);
+        Value positLo = builder.create<arith::MulIOp>(
+            loc, linearize(builder, loc, ivs, ubs), posStep);
+        Value positHi = builder.create<arith::AddIOp>(loc, positLo, c1);
+
+        Value pLo = genIndexLoad(builder, loc, positions, positLo);
+        Value pHi = genIndexLoad(builder, loc, positions, positHi);
+        Value nse = builder.create<arith::SubIOp>(loc, pHi, pLo);
+
+        Value crdLo = builder.create<arith::MulIOp>(
+            loc, pLo, constantIndex(builder, loc, unBatchedRank));
+        Value nCrd = builder.create<arith::MulIOp>(
+            loc, nse, constantIndex(builder, loc, unBatchedRank));
+
+        SmallVector<Value> offsets, sizes, strides;
+        for (unsigned i = 0; i < nBatched; i++) {
+          offsets.push_back(ivs[i]);
+          sizes.push_back(c1);
+          strides.push_back(c1);
+        }
+        // [0, nse, 1].
+        offsets.push_back(c0);
+        sizes.push_back(nse);
+        strides.push_back(c1);
+
+        auto valView = builder.create<memref::SubViewOp>(
+            loc, valMemref, offsets, sizes, strides);
+        auto valReass = getReassociationForFlattening(valView.getType());
+        Value valDst =
+            builder.create<memref::CollapseShapeOp>(loc, valView, valReass);
+        Value valSrc =
+            builder.create<memref::SubViewOp>(loc, values, pLo, nse, c1);
+        builder.create<memref::CopyOp>(loc, valSrc, valDst);
+
+        // [0, rank, 1].
+        offsets.push_back(c0);
+        sizes.push_back(constantIndex(builder, loc, unBatchedRank));
+        strides.push_back(c1);
+
+        auto crdView = builder.create<memref::SubViewOp>(
+            loc, crdMemref, offsets, sizes, strides);
+        auto crdReass = getReassociationForFlattening(crdView.getType());
+        Value crdDst =
+            builder.create<memref::CollapseShapeOp>(loc, crdView, crdReass);
+        Value crdSrc =
+            builder.create<memref::SubViewOp>(loc, flatCrds, crdLo, nCrd, c1);
+        builder.create<memref::CopyOp>(loc, crdSrc, crdDst);
+
+        Value pred = builder.create<arith::CmpIOp>(
+            loc, arith::CmpIPredicate::ugt, nse, args[0]);
+        // Choose the larger NSE
+        return {builder.create<arith::SelectOp>(loc, pred, nse, args[0])};
+      });
+
+  // Converts MemRefs back to Tensors.
+  Value values = rewriter.create<bufferization::ToTensorOp>(loc, valMemref);
+  Value coordinates =
+      rewriter.create<bufferization::ToTensorOp>(loc, crdMemref);
+  Value nse =
+      genCast(rewriter, loc, loopNest.results.front(), op.getNse().getType());
+
+  rewriter.replaceOp(op, {values, coordinates, nse});
+  return success();
+}
+
 struct SparseUnpackOpConverter : public OpConversionPattern<UnpackOp> {
   using OpConversionPattern::OpConversionPattern;
   SparseUnpackOpConverter(TypeConverter &typeConverter, MLIRContext *context,
@@ -1431,52 +1605,27 @@ struct SparseUnpackOpConverter : public OpConversionPattern<UnpackOp> {
   matchAndRewrite(UnpackOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     auto desc = getDescriptorFromTensorTuple(adaptor.getTensor());
-    Location loc = op.getLoc();
     const auto srcTp = getSparseTensorType(op.getTensor());
-    const Level lvlRank = srcTp.getLvlRank();
+    const unsigned nBatched = op.getNumBatchedLvls();
+    assert(isCOOType(srcTp.getEncoding(), nBatched, true) &&
+           desc.getFields().size() == 4); // specifier + pos + crds + values
+    (void)srcTp;
+    auto logicRes = nBatched == 0
+                        ? genUnBatchedUnpackOp(op, desc, rewriter)
+                        : genBatchedUnpackOp(op, nBatched, desc, rewriter);
+    Value posBuf = desc.getPosMemRef(nBatched);
 
-    assert(isUniqueCOOType(srcTp) && desc.getFields().size() == 4);
-
-    Value flatBuf = lvlRank == 1 ? desc.getCrdMemRefOrView(rewriter, loc, 0)
-                                 : desc.getAOSMemRef();
-    Value valuesBuf = desc.getValMemRef();
-    Value posBuf = desc.getPosMemRef(0);
     if (createDeallocs) {
       // Unpack ends the lifetime of the sparse tensor. While the value array
       // and coordinate array are unpacked and returned, the position array
       // becomes useless and need to be freed (if user requests).
-      rewriter.create<memref::DeallocOp>(loc, posBuf);
-    }
-
-    // If frontend requests a static buffer, we reallocate the
-    // values/coordinates to ensure that we meet their need.
-    const auto valuesTp = getRankedTensorType(op.getValues());
-    if (valuesTp.hasStaticShape()) {
-      valuesBuf =
-          reallocOrSubView(rewriter, loc, valuesTp.getShape()[0], valuesBuf);
-    }
-
-    const auto coordinatesTp = getRankedTensorType(op.getCoordinates());
-    if (coordinatesTp.hasStaticShape()) {
-      auto len = coordinatesTp.getShape()[0] * coordinatesTp.getShape()[1];
-      flatBuf = reallocOrSubView(rewriter, loc, len, flatBuf);
+      // FIXME: Depending on whether the tensor being unpacked is created by
+      // PackOp or not, we may or may not need to free other memref fields of
+      // the sparse tensor too (PackOp borrows value/coordinate buffer).
+      rewriter.create<memref::DeallocOp>(op.getLoc(), posBuf);
     }
 
-    Value coordinatesBuf = rewriter.create<memref::ExpandShapeOp>(
-        loc,
-        MemRefType::get(coordinatesTp.getShape(),
-                        coordinatesTp.getElementType()),
-        flatBuf, ArrayRef{ReassociationIndices{0, 1}});
-
-    // Converts MemRefs back to Tensors.
-    Value values = rewriter.create<bufferization::ToTensorOp>(loc, valuesBuf);
-    Value coordinates =
-        rewriter.create<bufferization::ToTensorOp>(loc, coordinatesBuf);
-    Value nse = genCast(rewriter, loc, desc.getValMemSize(rewriter, loc),
-                        op.getNse().getType());
-
-    rewriter.replaceOp(op, {values, coordinates, nse});
-    return success();
+    return logicRes;
   }
 
 private:
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index d22ea43fbf5ac..4aba829d7dbdd 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -346,6 +346,45 @@ struct FuseSparseMultiplyOverAdd : public OpRewritePattern<GenericOp> {
   }
 };
 
+// Fuse a tensor cast into producing operation. Note that a tensor.cast
+// should really not be used to convert between sparse encodings. Since
+// the pattern currently appears as a result of some prior rewriting
+// we make an attempt to repair very obvious cases.
+// TODO: audit the pure tensor dialect rewriting rules
+struct FuseTensorCast : public OpRewritePattern<tensor::CastOp> {
+public:
+  using OpRewritePattern<tensor::CastOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tensor::CastOp op,
+                                PatternRewriter &rewriter) const override {
+    Type srcType = op.getSource().getType();
+    Type dstType = op.getDest().getType();
+    // A nop cast simply folds away.
+    if (srcType == dstType) {
+      rewriter.replaceOp(op, op->getResults());
+      return success();
+    }
+    // See if a sparsity changing cast can be fused into producer.
+    if (tensor::isSameTypeWithoutEncoding(srcType, dstType)) {
+      if (Operation *def = op.getSource().getDefiningOp()) {
+        if (def->hasOneUse() && isa<tensor::ExtractSliceOp>(def)) {
+          def->getResult(0).setType(op->getResultTypes()[0]);
+          rewriter.replaceOp(op, def->getResult(0));
+          return success();
+        }
+      }
+    }
+    // Repair tensor casts with at least one sparse operand into the
+    // the properly supported sparse_tensor.convert.
+    if (getSparseTensorEncoding(srcType) || getSparseTensorEncoding(dstType)) {
+      rewriter.replaceOpWithNewOp<ConvertOp>(op, dstType, op.getSource());
+      return success();
+    }
+    // Fail otherwise.
+    return failure();
+  }
+};
+
 /// Sparse rewriting rule for sparse-to-sparse reshape operator.
 template <typename ReshapeOp>
 struct Sparse2SparseReshapeRewriter : public OpRewritePattern<ReshapeOp> {
@@ -951,14 +990,12 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     for (Level l = 0; l < lvlRank; l++) {
       // TODO: provide utility function for loop sequences that only contains
       // one for loop?
-      // FIXME(wrengr): what is this "ld" supposed to be really?
-      const Level ld = op.getOrder() ? op.getOrder()->getDimPosition(l) : l;
-      const SmallVector<TensorId, 1> tids{0};
-      loopEmitter.enterNewLoopSeq(rewriter, loc, tids, ld);
+      const SmallVector<TensorLevel, 1> tidLvls{
+          loopEmitter.makeTensorLevel(0, l)};
+      loopEmitter.enterNewLoopSeq(rewriter, loc, tidLvls);
       // Note that reduc will be taken care of by loop emitter and get updated
       // in place.
-
-      loopEmitter.enterLoopOverTensorAtLvl(rewriter, loc, tids, l, reduc);
+      loopEmitter.enterLoopOverTensorAtLvl(rewriter, loc, tidLvls, reduc);
     }
 
     SmallVector<Value> lcvs;
@@ -1125,7 +1162,7 @@ struct OutRewriter : public OpRewritePattern<OutOp> {
 //===---------------------------------------------------------------------===//
 
 void mlir::populatePreSparsificationRewriting(RewritePatternSet &patterns) {
-  patterns.add<FoldInvariantYield, FuseSparseMultiplyOverAdd>(
+  patterns.add<FoldInvariantYield, FuseSparseMultiplyOverAdd, FuseTensorCast>(
       patterns.getContext());
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index 4ed879121e98c..afeabb33fcd78 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -1296,13 +1296,16 @@ static bool isParallelFor(CodegenEnv &env, bool isOuter, bool isSparse) {
 
 /// Generates a for-loop on a single index.
 static Operation *genFor(CodegenEnv &env, OpBuilder &builder, bool isOuter,
-                         bool isInner, LoopId ldx, ArrayRef<TensorId> tids,
-                         ArrayRef<Level> lvls) {
+                         bool isInner, LoopId ldx,
+                         ArrayRef<TensorLevel> tidLvls) {
   linalg::GenericOp op = env.op();
   Location loc = op.getLoc();
   auto iteratorTypes = op.getIteratorTypesArray();
-  bool isSparse = llvm::any_of(tids, [ldx, &env](TensorId tid) {
-    const auto dlt = env.dlt(tid, ldx);
+  bool isSparse = llvm::any_of(tidLvls, [ldx, &env](TensorLevel tidLvl) {
+    // Queries the DLT based on the tensor id and loop idx, as requested by
+    // `CodegenEnv::dlt(TensorId, LoopIdx)`. The returned DLT from CodegenEnv
+    // should be consistent with the DLT indexed by <TensorId, Level>.
+    const auto dlt = env.dlt(env.unpackTensorLevel(tidLvl).first, ldx);
     return isCompressedDLT(dlt) || isSingletonDLT(dlt);
   });
 
@@ -1310,11 +1313,10 @@ static Operation *genFor(CodegenEnv &env, OpBuilder &builder, bool isOuter,
 
   Operation *loop = *env.genLoopBoundary([&](MutableArrayRef<Value> reduc) {
     if (env.merger().isFilterLoop(ldx)) {
-      const TensorId tid = tids.front();
-      const Level lvl = lvls.front();
+      const auto [tid, lvl] = env.unpackTensorLevel(tidLvls.front());
       // tids/lvls must only have one value because filter loops only
       // corresponding to the one and only sparse tensor level.
-      assert(isSparse && tids.size() == 1 && lvls.size() == 1);
+      assert(isSparse && tidLvls.size() == 1);
       OpOperand *t = &op->getOpOperand(tid);
       auto enc = getSparseTensorEncoding(t->get().getType());
       // Retrieves the affine expression for the filter loop.
@@ -1324,8 +1326,8 @@ static Operation *genFor(CodegenEnv &env, OpBuilder &builder, bool isOuter,
       return env.emitter().enterFilterLoopOverTensorAtLvl(builder, loc, tid,
                                                           lvl, a, reduc);
     }
-    return env.emitter().enterLoopOverTensorAtLvl(builder, loc, tids, lvls,
-                                                  reduc, isParallel);
+    return env.emitter().enterLoopOverTensorAtLvl(builder, loc, tidLvls, reduc,
+                                                  isParallel);
   });
   assert(loop);
   return loop;
@@ -1333,13 +1335,12 @@ static Operation *genFor(CodegenEnv &env, OpBuilder &builder, bool isOuter,
 
 /// Emit a while-loop for co-iteration over multiple indices.
 static Operation *genWhile(CodegenEnv &env, OpBuilder &builder, LoopId idx,
-                           bool needsUniv, ArrayRef<TensorId> tids,
-                           ArrayRef<Level> lvls) {
+                           bool needsUniv, ArrayRef<TensorLevel> tidLvls) {
   Operation *loop = *env.genLoopBoundary([&](MutableArrayRef<Value> reduc) {
     // Construct the while-loop with a parameter for each
     // index.
     return env.emitter().enterCoIterationOverTensorsAtLvls(
-        builder, env.op().getLoc(), tids, lvls, needsUniv, reduc);
+        builder, env.op().getLoc(), tidLvls, needsUniv, reduc);
   });
   assert(loop);
   return loop;
@@ -1348,16 +1349,15 @@ static Operation *genWhile(CodegenEnv &env, OpBuilder &builder, LoopId idx,
 /// Generates a for-loop or a while-loop, depending on whether it implements
 /// singleton iteration or co-iteration over the given conjunction.
 static Operation *genLoop(CodegenEnv &env, OpBuilder &builder, LoopOrd at,
-                          bool needsUniv, ArrayRef<TensorId> tids,
-                          ArrayRef<Level> lvls, bool isFor) {
-  assert(tids.size() == lvls.size());
+                          bool needsUniv, ArrayRef<TensorLevel> tidLvls,
+                          bool isFor) {
   const LoopId idx = env.topSortAt(at);
   if (isFor) {
     bool isOuter = at == 0;
     bool isInner = at == env.topSortSize() - 1;
-    return genFor(env, builder, isOuter, isInner, idx, tids, lvls);
+    return genFor(env, builder, isOuter, isInner, idx, tidLvls);
   }
-  return genWhile(env, builder, idx, needsUniv, tids, lvls);
+  return genWhile(env, builder, idx, needsUniv, tidLvls);
 }
 
 /// Generates the induction structure for a while-loop.
@@ -1480,8 +1480,7 @@ static bool startLoopSeq(CodegenEnv &env, OpBuilder &builder, ExprId exp,
   const LatPointId l0 = env.set(lts)[0];
   bool needsUniv = false;
 
-  SmallVector<TensorId> tids;
-  SmallVector<Level> lvls;
+  SmallVector<TensorLevel> tidLvls;
   env.merger().foreachTensorLoopId(l0, [&](TensorLoopId b, TensorId tid,
                                            std::optional<Level> lvl,
                                            DimLevelType dlt, bool isIdxReduc) {
@@ -1493,12 +1492,11 @@ static bool startLoopSeq(CodegenEnv &env, OpBuilder &builder, ExprId exp,
       // Only when this is a index reduction loop, can the dlt be undefined.
       assert(!isUndefDLT(dlt) || isIdxReduc);
       // sparse/singleton levels, or a dense/sparse index reduction loop.
-      tids.push_back(tid);
-      lvls.push_back(*lvl);
+      tidLvls.push_back(env.makeTensorLevel(tid, *lvl));
     }
   });
 
-  env.emitter().enterNewLoopSeq(builder, env.op().getLoc(), tids, lvls);
+  env.emitter().enterNewLoopSeq(builder, env.op().getLoc(), tidLvls);
 
   // Maintain the universal index only if it is actually
   // consumed by a subsequent lattice point.
@@ -1529,7 +1527,8 @@ static void genConstantDenseAddressFromLevel(CodegenEnv &env,
       // FIXME: `toOrigDim` is deprecated.
       AffineExpr lvlExpr = lvlExprs[toOrigDim(enc, l)];
       if (enc.isDenseLvl(l) && lvlExpr.isa<AffineConstantExpr>())
-        env.emitter().genDenseAffineAddress(builder, loc, tid, l, lvlExpr);
+        env.emitter().genDenseAffineAddress(
+            builder, loc, env.makeTensorLevel(tid, l), lvlExpr);
       else
         return; // break on first non-dense non-constant level
     }
@@ -1548,23 +1547,21 @@ static void genInitConstantDenseAddress(CodegenEnv &env,
 
 /// Return true if the lattices bit can be iterated by a for loop.
 static bool translateBitsToTidLvlPairs(
-    CodegenEnv &env, LatPointId li, LoopId ldx, SmallVectorImpl<TensorId> &tids,
-    SmallVectorImpl<Level> &lvls, SmallVectorImpl<TensorId> &affineTids,
-    SmallVectorImpl<Level> &affineLvls, SmallVectorImpl<AffineExpr> &exps) {
+    CodegenEnv &env, LatPointId li, LoopId ldx,
+    SmallVectorImpl<TensorLevel> &tidLvls,
+    SmallVectorImpl<std::pair<TensorLevel, AffineExpr>> &affineTidLvls) {
   const BitVector &simple = env.lat(li).simple;
   const TensorId outTid = env.merger().getOutTensorID();
   const std::optional<Level> outLvl = env.merger().getLvl(outTid, ldx);
 
   unsigned numloopCond = 0;
   bool hasNonUnique = false;
-
   env.merger().foreachTensorLoopId(
       li, [&, ldx](TensorLoopId b, TensorId tid, std::optional<Level> lvl,
                    DimLevelType dlt, bool isIdxReduc) {
         if (simple[b]) {
           if (isIdxReduc) {
-            tids.push_back(tid);
-            lvls.push_back(*lvl);
+            tidLvls.push_back(env.makeTensorLevel(tid, *lvl));
             numloopCond++;
             return;
           }
@@ -1582,12 +1579,10 @@ static bool translateBitsToTidLvlPairs(
               return;
           }
           hasNonUnique = !isUniqueDLT(dlt) || hasNonUnique;
-          tids.push_back(tid);
-          lvls.push_back(*lvl);
+          tidLvls.push_back(env.makeTensorLevel(tid, *lvl));
           numloopCond++;
         } else if (isDenseDLT(dlt) || isIdxReduc) {
-          tids.push_back(tid);
-          lvls.push_back(*lvl);
+          tidLvls.push_back(env.makeTensorLevel(tid, *lvl));
         } else {
           assert(isUndefDLT(dlt));
           linalg::GenericOp op = env.op();
@@ -1625,9 +1620,7 @@ static bool translateBitsToTidLvlPairs(
                 // computeIterationGraph), another more admissible approach
                 // might be accepting out-of-order access between consecutive
                 // dense levels.
-                affineTids.push_back(tid);
-                affineLvls.push_back(l);
-                exps.push_back(exp);
+                affineTidLvls.emplace_back(env.makeTensorLevel(tid, l), exp);
               }
             }
           }
@@ -1638,8 +1631,7 @@ static bool translateBitsToTidLvlPairs(
     // Note that we generate dense indices of the output tensor
     // unconditionally, since they may not appear in the lattice, but may be
     // needed for linearized env.
-    tids.push_back(outTid);
-    lvls.push_back(*outLvl);
+    tidLvls.push_back(env.makeTensorLevel(outTid, *outLvl));
   }
 
   assert(numloopCond > 0);
@@ -1653,29 +1645,27 @@ static std::pair<Operation *, bool> startLoop(CodegenEnv &env,
                                               OpBuilder &builder, LoopOrd at,
                                               LatPointId li, bool needsUniv) {
   // The set of tensors + lvls to generate loops on
-  SmallVector<TensorId> tids, affineTids;
-  SmallVector<Level> lvls, affineLvls;
+  SmallVector<TensorLevel> tidLvls;
   // The set of dense tensors with non-trivial affine expression that just
   // becomes invariant and the address shall now be generated at the current
   // level.
-  SmallVector<AffineExpr> affines;
-  bool isSingleCond = translateBitsToTidLvlPairs(
-      env, li, env.topSortAt(at), tids, lvls, affineTids, affineLvls, affines);
+  SmallVector<std::pair<TensorLevel, AffineExpr>> affineTidLvls;
+  bool isSingleCond = translateBitsToTidLvlPairs(env, li, env.topSortAt(at),
+                                                 tidLvls, affineTidLvls);
 
   // Emit the for/while-loop control.
-  Operation *loop =
-      genLoop(env, builder, at, needsUniv, tids, lvls, isSingleCond);
+  Operation *loop = genLoop(env, builder, at, needsUniv, tidLvls, isSingleCond);
   Location loc = env.op().getLoc();
-  for (auto [tid, lvl, exp] : llvm::zip(affineTids, affineLvls, affines)) {
-    env.emitter().genDenseAffineAddress(builder, loc, tid, lvl, exp);
+  for (auto [tidLvl, exp] : affineTidLvls) {
+    env.emitter().genDenseAffineAddress(builder, loc, tidLvl, exp);
   }
 
   // Until now, we have entered every <tid, lvl> pair in {cond, extra,
   // affine}Tids/Lvls. The addresses of the upcoming levels which are dependent
   // on constant affines expression may now be determined.
-  auto allTids = llvm::concat<TensorId>(tids, affineTids);
-  auto allLvls = llvm::concat<Level>(lvls, affineLvls);
-  for (auto [tid, lvl] : llvm::zip(allTids, allLvls)) {
+  auto allTidLvls =
+      llvm::concat<TensorLevel>(tidLvls, llvm::make_first_range(affineTidLvls));
+  for (auto [tid, lvl] : env.unpackTensorLevelRange(allTidLvls)) {
     if (tid != env.merger().getOutTensorID())
       genConstantDenseAddressFromLevel(env, builder, tid, lvl + 1);
   }
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 99382a375c1f4..74a21771d034e 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -110,6 +110,16 @@ LogicalResult tensor::getOrCreateDestinations(OpBuilder &b, Location loc,
   return success();
 }
 
+bool tensor::isSameTypeWithoutEncoding(Type tp1, Type tp2) {
+  if (auto rtp1 = tp1.dyn_cast<RankedTensorType>()) {
+    if (auto rtp2 = tp2.dyn_cast<RankedTensorType>())
+      return rtp1.getShape() == rtp2.getShape() &&
+             rtp1.getElementType() == rtp2.getElementType();
+    return false;
+  }
+  return tp1 == tp2; // default implementation
+}
+
 /// Compute the dropped dimensions of a rank-reducing tensor.extract_slice op or
 /// rank-extending tensor.insert_slice op.
 static llvm::SmallBitVector getDroppedDims(ArrayRef<int64_t> reducedShape,
@@ -152,6 +162,53 @@ static llvm::SmallBitVector getDroppedDims(ArrayRef<int64_t> reducedShape,
   return droppedDims;
 }
 
+//===----------------------------------------------------------------------===//
+// BitcastOp
+//===----------------------------------------------------------------------===//
+
+bool BitcastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
+  if (inputs.size() != 1 || outputs.size() != 1)
+    return false;
+  Type a = inputs.front(), b = outputs.front();
+  auto aT = dyn_cast<TensorType>(a);
+  auto bT = dyn_cast<TensorType>(b);
+  if (!aT || !bT)
+    return false;
+
+  if (aT.getElementTypeBitWidth() != bT.getElementTypeBitWidth())
+    return false;
+
+  return succeeded(verifyCompatibleShape(aT, bT));
+}
+
+namespace {
+
+/// Replaces chains of two tensor.bitcast operations by a single tensor.bitcast
+/// operation.
+struct ChainedTensorBitcast : public OpRewritePattern<BitcastOp> {
+  using OpRewritePattern<BitcastOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(BitcastOp tensorBitcast,
+                                PatternRewriter &rewriter) const final {
+    auto tensorBitcastOperand =
+        tensorBitcast.getOperand().getDefiningOp<BitcastOp>();
+    if (!tensorBitcastOperand)
+      return failure();
+
+    auto resultType = cast<TensorType>(tensorBitcast.getType());
+    rewriter.replaceOpWithNewOp<BitcastOp>(tensorBitcast, resultType,
+                                           tensorBitcastOperand.getOperand());
+    return success();
+  }
+};
+
+} // namespace
+
+void BitcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
+                                            MLIRContext *context) {
+  results.add<ChainedTensorBitcast>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // CastOp
 //===----------------------------------------------------------------------===//
@@ -852,12 +909,6 @@ void FromElementsOp::getAsmResultNames(
   setNameFn(getResult(), "from_elements");
 }
 
-void FromElementsOp::build(OpBuilder &builder, OperationState &result,
-                           Type resultType, ValueRange elements) {
-  result.addOperands(elements);
-  result.addTypes(resultType);
-}
-
 void FromElementsOp::build(OpBuilder &builder, OperationState &result,
                            ValueRange elements) {
   assert(!elements.empty() && "expected at least one element");
@@ -1349,18 +1400,6 @@ void CollapseShapeOp::build(OpBuilder &b, OperationState &result, Value src,
                       getReassociationIndicesAttribute(b, reassociation));
 }
 
-// Checks if types are the same, but ignoring encoding on ranked tensors.
-static bool isSameTypesWithoutEncoding(Type tp1, Type tp2) {
-  if (auto rtp1 = tp1.dyn_cast<RankedTensorType>()) {
-    if (auto rtp2 = tp2.dyn_cast<RankedTensorType>())
-      return rtp1.getShape() == rtp2.getShape() &&
-             rtp1.getElementType() == rtp2.getElementType();
-    return false;
-  }
-  // Default implementation.
-  return tp1 == tp2;
-}
-
 template <typename TensorReshapeOp, bool isExpansion = std::is_same<
                                         TensorReshapeOp, ExpandShapeOp>::value>
 static LogicalResult verifyTensorReshapeOp(TensorReshapeOp op,
@@ -1373,7 +1412,7 @@ static LogicalResult verifyTensorReshapeOp(TensorReshapeOp op,
   auto maps = op.getReassociationMaps();
   RankedTensorType expectedType =
       CollapseShapeOp::inferCollapsedType(expandedType, maps);
-  if (!isSameTypesWithoutEncoding(collapsedType, expectedType))
+  if (!isSameTypeWithoutEncoding(collapsedType, expectedType))
     return op.emitOpError("expected collapsed type to be ")
            << expectedType << ", but got " << collapsedType;
   return success();
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 0ba705293ce5b..c17afc77ad4cc 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -617,8 +617,10 @@ FailureOr<TilingResult> tensor::bubbleUpPadSlice(OpBuilder &b,
     // Create pad(extract_slice(x)).
     Value newSliceOp = b.create<tensor::ExtractSliceOp>(
         loc, padOp.getSource(), newOffsets, newLengths, newStrides);
-    auto newPadOp = b.create<PadOp>(loc, Type(), newSliceOp, newLows, newHighs,
-                                    /*nofold=*/padOp.getNofold());
+    auto newPadOp = b.create<PadOp>(
+        loc, Type(), newSliceOp, newLows, newHighs,
+        /*nofold=*/padOp.getNofold(),
+        getPrunedAttributeList(padOp, PadOp::getAttributeNames()));
 
     // Copy region to new PadOp.
     IRMapping bvm;
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/CMakeLists.txt b/mlir/lib/Dialect/Tensor/TransformOps/CMakeLists.txt
index e8be901fe8648..be1a5ddf7e7a0 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tensor/TransformOps/CMakeLists.txt
@@ -4,9 +4,15 @@ add_mlir_dialect_library(MLIRTensorTransformOps
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Tensor/TransformOps
 
+  DEPENDS
+  MLIRTensorTransformOpsIncGen
+
   LINK_LIBS PUBLIC
+  MLIRAffineDialect
   MLIRIR
   MLIRPDLDialect
+  MLIRSCFDialect
   MLIRTensorDialect
+  MLIRTensorTransforms
   MLIRTransformDialect
 )
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index 01e1a35e76bce..4394465d4bdd0 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -8,8 +8,12 @@
 
 #include "mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h"
 
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Transform/IR/TransformInterfaces.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
@@ -49,3 +53,80 @@ tensor::TrackingListener::findReplacementOp(Operation *op,
 
   return nullptr;
 }
+
+//===----------------------------------------------------------------------===//
+// MakeLoopIndependentOp
+//===----------------------------------------------------------------------===//
+
+DiagnosedSilenceableFailure transform::MakeLoopIndependentOp::applyToOne(
+    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformState &state) {
+  // Gather IVs.
+  SmallVector<Value> ivs;
+  Operation *nextOp = target;
+  for (uint64_t i = 0, e = getNumLoops(); i < e; ++i) {
+    nextOp = nextOp->getParentOfType<scf::ForOp>();
+    if (!nextOp) {
+      DiagnosedSilenceableFailure diag = emitSilenceableError()
+                                         << "could not find " << i
+                                         << "-th enclosing loop";
+      diag.attachNote(target->getLoc()) << "target op";
+      return diag;
+    }
+    ivs.push_back(cast<scf::ForOp>(nextOp).getInductionVar());
+  }
+
+  // Rewrite IR.
+  IRRewriter rewriter(target->getContext());
+  FailureOr<Value> replacement = failure();
+  if (auto padOp = dyn_cast<tensor::PadOp>(target)) {
+    replacement = tensor::buildIndependentOp(rewriter, padOp, ivs);
+  } else if (auto emptyOp = dyn_cast<tensor::EmptyOp>(target)) {
+    replacement = tensor::buildIndependentOp(rewriter, emptyOp, ivs);
+  } else {
+    DiagnosedSilenceableFailure diag = emitSilenceableError()
+                                       << "unsupported target op";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+  if (failed(replacement)) {
+    DiagnosedSilenceableFailure diag =
+        emitSilenceableError() << "could not make target op loop-independent";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+  rewriter.replaceOp(target, *replacement);
+  results.push_back(replacement->getDefiningOp());
+  return DiagnosedSilenceableFailure::success();
+}
+
+//===----------------------------------------------------------------------===//
+// Transform op registration
+//===----------------------------------------------------------------------===//
+
+namespace {
+class TensorTransformDialectExtension
+    : public transform::TransformDialectExtension<
+          TensorTransformDialectExtension> {
+public:
+  using Base::Base;
+
+  void init() {
+    declareGeneratedDialect<affine::AffineDialect>();
+    declareGeneratedDialect<tensor::TensorDialect>();
+
+    registerTransformOps<
+#define GET_OP_LIST
+#include "mlir/Dialect/Tensor/TransformOps/TensorTransformOps.cpp.inc"
+        >();
+  }
+};
+} // namespace
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Tensor/TransformOps/TensorTransformOps.cpp.inc"
+
+void mlir::tensor::registerTransformDialectExtension(
+    DialectRegistry &registry) {
+  registry.addExtensions<TensorTransformDialectExtension>();
+}
diff --git a/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
index 44579546f7ea5..c41e9e9ce6839 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
@@ -5,6 +5,7 @@ add_mlir_dialect_library(MLIRTensorTransforms
   ExtractSliceFromReshapeUtils.cpp
   FoldIntoPackAndUnpackPatterns.cpp
   FoldTensorSubsetOps.cpp
+  IndependenceTransforms.cpp
   MergeConsecutiveInsertExtractSlicePatterns.cpp
   ReshapePatterns.cpp
   SwapExtractSliceWithProducerPatterns.cpp
@@ -17,6 +18,7 @@ add_mlir_dialect_library(MLIRTensorTransforms
 
   LINK_LIBS PUBLIC
   MLIRAffineDialect
+  MLIRAffineTransforms
   MLIRAffineUtils
   MLIRArithDialect
   MLIRBufferizationDialect
@@ -30,4 +32,5 @@ add_mlir_dialect_library(MLIRTensorTransforms
   MLIRTilingInterface
   MLIRTransforms
   MLIRVectorDialect
+  MLIRValueBoundsOpInterface
 )
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
new file mode 100644
index 0000000000000..721730862d49b
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -0,0 +1,136 @@
+//===- IndependenceTransforms.cpp - Make ops independent of values --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
+
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/Transforms/Transforms.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/Interfaces/ValueBoundsOpInterface.h"
+
+using namespace mlir;
+using namespace mlir::tensor;
+
+/// Make the given OpFoldResult independent of all independencies.
+static FailureOr<OpFoldResult> makeIndependent(OpBuilder &b, Location loc,
+                                               OpFoldResult ofr,
+                                               ValueRange independencies) {
+  if (ofr.is<Attribute>())
+    return ofr;
+  Value value = ofr.get<Value>();
+  AffineMap boundMap;
+  ValueDimList mapOperands;
+  if (failed(ValueBoundsConstraintSet::computeIndependentBound(
+          boundMap, mapOperands, presburger::BoundType::UB, value,
+          /*dim=*/std::nullopt, independencies, /*closedUB=*/true)))
+    return failure();
+  return mlir::affine::materializeComputedBound(b, loc, boundMap, mapOperands);
+}
+
+FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
+                                            ValueRange independencies) {
+  OpBuilder::InsertionGuard g(b);
+  b.setInsertionPoint(padOp);
+  Location loc = padOp.getLoc();
+
+  // Non-constant padding not supported.
+  Value constantPadding = padOp.getConstantPaddingValue();
+  if (!constantPadding)
+    return failure();
+
+  SmallVector<OpFoldResult> newMixedLow, newMixedHigh;
+  for (OpFoldResult ofr : padOp.getMixedLowPad()) {
+    auto ub = makeIndependent(b, loc, ofr, independencies);
+    if (failed(ub))
+      return failure();
+    newMixedLow.push_back(*ub);
+  }
+  for (OpFoldResult ofr : padOp.getMixedHighPad()) {
+    auto ub = makeIndependent(b, loc, ofr, independencies);
+    if (failed(ub))
+      return failure();
+    newMixedHigh.push_back(*ub);
+  }
+
+  // Return existing tensor::PadOp if nothing has changed.
+  if (llvm::equal(padOp.getMixedLowPad(), newMixedLow) &&
+      llvm::equal(padOp.getMixedHighPad(), newMixedHigh))
+    return padOp.getResult();
+
+  // Create a new tensor::PadOp.
+  auto newPadOp = b.create<PadOp>(
+      loc, padOp.getResultType(), padOp.getSource(), newMixedLow, newMixedHigh,
+      constantPadding, padOp.getNofold(), /*attrs=*/ArrayRef<NamedAttribute>{});
+
+  // Create a tensor::ExtractSliceOp.
+  // Reify the result sizes of the old tensor::PadOp.
+  ReifiedRankedShapedTypeDims reifiedSizes;
+  ReifyRankedShapedTypeOpInterface reifyShapedTypeInterface =
+      dyn_cast<ReifyRankedShapedTypeOpInterface>(padOp.getOperation());
+  if (failed(reifyShapedTypeInterface.reifyResultShapes(b, reifiedSizes)))
+    return failure();
+  SmallVector<OpFoldResult> offsets, sizes, strides;
+  for (int64_t i = 0, e = padOp.getResultType().getRank(); i < e; ++i) {
+    // offset = ub(low_padding) - low_padding
+    OpFoldResult prevLow = padOp.getMixedLowPad()[i];
+    if (prevLow.is<Attribute>()) {
+      offsets.push_back(b.getIndexAttr(0));
+    } else {
+      offsets.push_back(
+          b.create<affine::AffineApplyOp>(
+               loc, b.getAffineDimExpr(0) - b.getAffineDimExpr(1),
+               std::initializer_list<Value>{newMixedLow[i].get<Value>(),
+                                            prevLow.get<Value>()})
+              .getResult());
+    }
+    // size = reified result size
+    if (!padOp.getResultType().isDynamicDim(i)) {
+      sizes.push_back(b.getIndexAttr(padOp.getResultType().getDimSize(i)));
+    } else {
+      sizes.push_back(reifiedSizes[0][i]);
+    }
+    // stride = 1
+    strides.push_back(b.getIndexAttr(1));
+  }
+
+  return b.create<ExtractSliceOp>(loc, newPadOp, offsets, sizes, strides)
+      .getResult();
+}
+
+FailureOr<Value> tensor::buildIndependentOp(OpBuilder &b,
+                                            tensor::EmptyOp emptyOp,
+                                            ValueRange independencies) {
+  OpBuilder::InsertionGuard g(b);
+  b.setInsertionPoint(emptyOp);
+  Location loc = emptyOp.getLoc();
+
+  SmallVector<OpFoldResult> newSizes;
+  for (OpFoldResult ofr : emptyOp.getMixedSizes()) {
+    auto ub = makeIndependent(b, loc, ofr, independencies);
+    if (failed(ub))
+      return failure();
+    newSizes.push_back(*ub);
+  }
+
+  // Return existing tensor::EmptyOp if nothing has changed.
+  if (llvm::equal(emptyOp.getMixedSizes(), newSizes))
+    return emptyOp.getResult();
+
+  // Create a new tensor::EmptyOp.
+  Value newEmptyOp =
+      b.create<EmptyOp>(loc, newSizes, emptyOp.getType().getElementType());
+
+  // Create a tensor::ExtractSliceOp.
+  SmallVector<OpFoldResult> offsets(newSizes.size(), b.getIndexAttr(0));
+  SmallVector<OpFoldResult> strides(newSizes.size(), b.getIndexAttr(1));
+  return b
+      .create<ExtractSliceOp>(loc, newEmptyOp, offsets, emptyOp.getMixedSizes(),
+                              strides)
+      .getResult();
+}
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 19a80c783c475..7e161b1298211 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -105,11 +105,9 @@ struct ReshapeConstOptimization : public OpRewritePattern<tosa::ReshapeOp> {
                                          "Used more than once or not-splat");
 
     // Build new const op with correct output shape
-    ShapedType inputShape = input.getType().cast<ShapedType>();
-    DenseElementsAttr outputAttr =
-        inputAttr.reshape(inputShape.clone(op.getNewShape()));
-    rewriter.replaceOpWithNewOp<tosa::ConstOp>(op, outputAttr.getType(),
-                                               outputAttr);
+    DenseElementsAttr outputAttr = inputAttr.reshape(
+        inputAttr.getType().cast<ShapedType>().clone(op.getNewShape()));
+    rewriter.replaceOpWithNewOp<tosa::ConstOp>(op, resultTy, outputAttr);
     return success();
   }
 };
@@ -227,7 +225,7 @@ struct TransposeIsReshape : public OpRewritePattern<tosa::TransposeOp> {
     for (int i = 1, s = nonZeroPerms.size(); i < s; ++i)
       if (nonZeroPerms[i - 1] > nonZeroPerms[i])
         return rewriter.notifyMatchFailure(op,
-                                           "Transpose changes memeory layout.");
+                                           "Transpose changes memory layout.");
 
     SmallVector<int64_t> newShape;
     newShape.reserve(inputTy.getRank());
@@ -998,3 +996,13 @@ OpFoldResult TransposeOp::fold(FoldAdaptor adaptor) {
 
   return getInput1();
 }
+
+OpFoldResult tosa::LogOp::fold(FoldAdaptor adaptor) {
+  auto input = getInput1();
+  // Element-wise log(exp(x)) = x
+  if (auto op = input.getDefiningOp<tosa::ExpOp>()) {
+    return op.getInput1();
+  }
+
+  return {};
+}
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 2da687b81663c..5c17d281c2ec7 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -364,7 +364,8 @@ static LogicalResult resolveBroadcastShape(const ValueShapeRange &operands,
 
 LogicalResult tosa::ArgMaxOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape = operands.getShape(0);
   IntegerAttr axis = attributes.get("axis").cast<IntegerAttr>();
@@ -389,7 +390,8 @@ LogicalResult tosa::ArgMaxOp::inferReturnTypeComponents(
 
 LogicalResult tosa::RFFT2dOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape = operands.getShape(0);
 
@@ -415,7 +417,8 @@ LogicalResult tosa::RFFT2dOp::inferReturnTypeComponents(
 
 LogicalResult tosa::FFT2dOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   inferredReturnShapes.push_back(ShapedTypeComponents(operands.getShape(0)));
   inferredReturnShapes.push_back(ShapedTypeComponents(operands.getShape(1)));
@@ -424,7 +427,8 @@ LogicalResult tosa::FFT2dOp::inferReturnTypeComponents(
 
 LogicalResult tosa::ConcatOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   // Infer all dimension sizes by reducing based on inputs.
   int32_t axis =
@@ -484,16 +488,18 @@ LogicalResult tosa::ConcatOp::inferReturnTypeComponents(
 
 LogicalResult tosa::EqualOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
+  auto elementType = IntegerType::get(context, /*width=*/1);
+
   llvm::SmallVector<int64_t> outShape;
   if (resolveBroadcastShape(operands, outShape).failed()) {
-    inferredReturnShapes.push_back(ShapedTypeComponents());
+    inferredReturnShapes.push_back(ShapedTypeComponents(elementType));
     return success();
   }
 
-  inferredReturnShapes.push_back(
-      ShapedTypeComponents(outShape, IntegerType::get(context, /*width=*/1)));
+  inferredReturnShapes.push_back(ShapedTypeComponents(outShape, elementType));
   return success();
 }
 
@@ -505,7 +511,8 @@ bool tosa::EqualOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 
 LogicalResult tosa::FullyConnectedOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape = operands.getShape(0);
   ShapeAdaptor weightShape = operands.getShape(1);
@@ -536,7 +543,8 @@ LogicalResult FullyConnectedOp::verify() { return verifyConvOp(*this); }
 
 LogicalResult tosa::MatMulOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor lhsShape = operands.getShape(0);
   ShapeAdaptor rhsShape = operands.getShape(1);
@@ -562,7 +570,8 @@ LogicalResult tosa::MatMulOp::inferReturnTypeComponents(
 
 LogicalResult tosa::PadOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape = operands.getShape(0);
   ShapeAdaptor paddingShape = operands.getShape(1);
@@ -624,16 +633,20 @@ static SmallVector<int64_t> convertToMlirShape(ArrayRef<int64_t> shape) {
 
 LogicalResult tosa::SliceOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   inferredReturnShapes.push_back(ShapedTypeComponents(
-      convertToMlirShape(SliceOpAdaptor(operands, attributes).getSize())));
+      convertToMlirShape(SliceOpAdaptor(operands, attributes,
+                                        *properties.as<Properties *>(), regions)
+                             .getSize())));
   return success();
 }
 
 LogicalResult tosa::TableOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape = operands.getShape(0);
 
@@ -649,9 +662,11 @@ LogicalResult tosa::TableOp::inferReturnTypeComponents(
 
 LogicalResult tosa::TileOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  TileOpAdaptor adaptor(operands, attributes);
+  TileOpAdaptor adaptor(operands, attributes, *properties.as<Properties *>(),
+                        regions);
   ArrayRef<int64_t> multiples = adaptor.getMultiples();
   ShapeAdaptor inputShape = operands.getShape(0);
   SmallVector<int64_t> outputShape;
@@ -682,9 +697,11 @@ bool tosa::ReshapeOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 
 LogicalResult tosa::ReshapeOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  ReshapeOpAdaptor adaptor(operands, attributes);
+  ReshapeOpAdaptor adaptor(operands, attributes, *properties.as<Properties *>(),
+                           regions);
   ShapeAdaptor inputShape = operands.getShape(0);
   Type inputType = getElementTypeOrSelf(operands.getType()[0]);
   llvm::SmallVector<int64_t> newShapeValue =
@@ -751,7 +768,8 @@ LogicalResult tosa::TransposeOp::getConstantPerms(SmallVector<int64_t> &perms) {
 
 LogicalResult tosa::TransposeOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   ShapeAdaptor inputShape = operands.getShape(0);
   ShapeAdaptor permsShape = operands.getShape(1);
@@ -818,7 +836,8 @@ LogicalResult tosa::TransposeOp::inferReturnTypeComponents(
 
 LogicalResult tosa::GatherOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<int64_t> outputShape;
   outputShape.resize(3, ShapedType::kDynamic);
@@ -843,9 +862,11 @@ LogicalResult tosa::GatherOp::inferReturnTypeComponents(
 
 LogicalResult tosa::ResizeOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  ResizeOpAdaptor adaptor(operands, attributes);
+  ResizeOpAdaptor adaptor(operands, attributes, *properties.as<Properties *>(),
+                          regions);
   llvm::SmallVector<int64_t, 4> outputShape;
   outputShape.resize(4, ShapedType::kDynamic);
 
@@ -883,7 +904,8 @@ LogicalResult tosa::ResizeOp::inferReturnTypeComponents(
 
 LogicalResult tosa::ScatterOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<int64_t> outputShape;
   outputShape.resize(3, ShapedType::kDynamic);
@@ -942,7 +964,7 @@ static LogicalResult ReduceInferReturnTypes(
   LogicalResult OP::inferReturnTypeComponents(                                 \
       MLIRContext *context, ::std::optional<Location> location,                \
       ValueShapeRange operands, DictionaryAttr attributes,                     \
-      RegionRange regions,                                                     \
+      OpaqueProperties properties, RegionRange regions,                        \
       SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {           \
     Type inputType =                                                           \
         operands.getType()[0].cast<TensorType>().getElementType();             \
@@ -978,7 +1000,7 @@ static LogicalResult NAryInferReturnTypes(
   LogicalResult OP::inferReturnTypeComponents(                                 \
       MLIRContext *context, ::std::optional<Location> location,                \
       ValueShapeRange operands, DictionaryAttr attributes,                     \
-      RegionRange regions,                                                     \
+      OpaqueProperties properties, RegionRange regions,                        \
       SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {           \
     return NAryInferReturnTypes(operands, inferredReturnShapes);               \
   }
@@ -1062,10 +1084,12 @@ static LogicalResult poolingInferReturnTypes(
 
 LogicalResult Conv2DOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<int64_t> outputShape(4, ShapedType::kDynamic);
-  Conv2DOp::Adaptor adaptor(operands.getValues(), attributes);
+  Conv2DOp::Adaptor adaptor(operands, attributes,
+                            *properties.as<Properties *>(), regions);
 
   int64_t inputWidth = ShapedType::kDynamic;
   int64_t inputHeight = ShapedType::kDynamic;
@@ -1125,10 +1149,12 @@ LogicalResult Conv2DOp::verify() { return verifyConvOp(*this); }
 
 LogicalResult Conv3DOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<int64_t> outputShape(5, ShapedType::kDynamic);
-  Conv3DOp::Adaptor adaptor(operands.getValues(), attributes);
+  Conv3DOp::Adaptor adaptor(operands, attributes,
+                            *properties.as<Properties *>(), regions);
 
   int64_t inputWidth = ShapedType::kDynamic;
   int64_t inputHeight = ShapedType::kDynamic;
@@ -1198,24 +1224,28 @@ LogicalResult Conv3DOp::verify() { return verifyConvOp(*this); }
 
 LogicalResult AvgPool2dOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   return poolingInferReturnTypes(operands, attributes, inferredReturnShapes);
 }
 
 LogicalResult MaxPool2dOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   return poolingInferReturnTypes(operands, attributes, inferredReturnShapes);
 }
 
 LogicalResult DepthwiseConv2DOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<int64_t> outputShape(4, ShapedType::kDynamic);
-  DepthwiseConv2DOp::Adaptor adaptor(operands.getValues(), attributes);
+  DepthwiseConv2DOp::Adaptor adaptor(operands, attributes,
+                                     *properties.as<Properties *>(), regions);
 
   int64_t inputWidth = ShapedType::kDynamic;
   int64_t inputHeight = ShapedType::kDynamic;
@@ -1288,9 +1318,11 @@ LogicalResult DepthwiseConv2DOp::verify() { return verifyConvOp(*this); }
 
 LogicalResult TransposeConv2DOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  TransposeConv2DOp::Adaptor adaptor(operands.getValues(), attributes);
+  TransposeConv2DOp::Adaptor adaptor(operands, attributes,
+                                     *properties.as<Properties *>(), regions);
   // outputShape is mutable.
   llvm::SmallVector<int64_t> outputShape =
       convertToMlirShape(adaptor.getOutShape());
@@ -1353,7 +1385,8 @@ LogicalResult TransposeConv2DOp::inferReturnTypeComponents(
 
 LogicalResult IfOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<tosa::YieldOp> yieldOps;
   for (Region *region : regions) {
@@ -1397,7 +1430,8 @@ LogicalResult IfOp::inferReturnTypeComponents(
 
 LogicalResult WhileOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   llvm::SmallVector<tosa::YieldOp> yieldOps;
   for (auto &block : *regions[1])
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
index 0cebdd960a8a6..74533defd055f 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaDecomposeTransposeConv.cpp
@@ -39,6 +39,7 @@ TosaOp createOpAndInfer(PatternRewriter &rewriter, Location loc, Type resultTy,
   if (shapeInterface
           .inferReturnTypeComponents(op.getContext(), op.getLoc(),
                                      op->getOperands(), op->getAttrDictionary(),
+                                     op->getPropertiesStorage(),
                                      op->getRegions(), returnedShapes)
           .failed())
     return op;
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index bd6069468d7f7..0c03cecf61bc4 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -218,9 +218,9 @@ void propagateShapesInRegion(Region &region) {
 
       ValueShapeRange range(op.getOperands(), operandShape);
       if (shapeInterface
-              .inferReturnTypeComponents(op.getContext(), op.getLoc(), range,
-                                         op.getAttrDictionary(),
-                                         op.getRegions(), returnedShapes)
+              .inferReturnTypeComponents(
+                  op.getContext(), op.getLoc(), range, op.getAttrDictionary(),
+                  op.getPropertiesStorage(), op.getRegions(), returnedShapes)
               .succeeded()) {
         for (auto it : llvm::zip(op.getResults(), returnedShapes)) {
           Value result = std::get<0>(it);
diff --git a/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp b/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
index 383c77f3b7340..18646f598bcee 100644
--- a/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
+++ b/mlir/lib/Dialect/Utils/ReshapeOpsUtils.cpp
@@ -480,6 +480,7 @@ PackingMetadata mlir::computePackingMetadata(int64_t packedRank,
                            res.insertPositions.end());
   res.reassociations.reserve(packedRank);
   for (int64_t i = 1; i <= packedRank; ++i) {
+    res.outerPositions.push_back(i - 1);
     if (!posSet.contains(i)) {
       res.reassociations.push_back(ReassociationIndices{i - 1});
       continue;
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index c8cf85e695749..99eed540afb0e 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -1152,7 +1152,7 @@ void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
 LogicalResult
 ExtractOp::inferReturnTypes(MLIRContext *, std::optional<Location>,
                             ValueRange operands, DictionaryAttr attributes,
-                            RegionRange,
+                            OpaqueProperties properties, RegionRange,
                             SmallVectorImpl<Type> &inferredReturnTypes) {
   ExtractOp::Adaptor op(operands, attributes);
   auto vectorType = op.getVector().getType().cast<VectorType>();
@@ -2084,7 +2084,7 @@ LogicalResult ShuffleOp::verify() {
 LogicalResult
 ShuffleOp::inferReturnTypes(MLIRContext *, std::optional<Location>,
                             ValueRange operands, DictionaryAttr attributes,
-                            RegionRange,
+                            OpaqueProperties properties, RegionRange,
                             SmallVectorImpl<Type> &inferredReturnTypes) {
   ShuffleOp::Adaptor op(operands, attributes);
   auto v1Type = op.getV1().getType().cast<VectorType>();
@@ -4630,6 +4630,10 @@ Type GatherOp::getExpectedMaskType() {
                          IntegerType::get(vecType.getContext(), /*width=*/1));
 }
 
+std::optional<SmallVector<int64_t, 4>> GatherOp::getShapeForUnroll() {
+  return llvm::to_vector<4>(getVectorType().getShape());
+}
+
 namespace {
 class GatherFolder final : public OpRewritePattern<GatherOp> {
 public:
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index d412b157e2849..44f3a10c4da5c 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -800,6 +800,91 @@ struct BubbleUpBitCastForStridedSliceInsert
   }
 };
 
+// Breaks down vector.bitcast op
+//
+// This transforms IR like:
+//   %1 = vector.bitcast %0: vector<8xf16> to vector<4xf32>
+// Into:
+//   %cst = vector.splat %c0_f32 : vector<4xf32>
+//   %1 = vector.extract_strided_slice %0 {
+//          offsets = [0], sizes = [4], strides = [1]
+//        } : vector<8xf16> to vector<4xf16>
+//   %2 = vector.bitcast %1 : vector<4xf16> to vector<2xf32>
+//   %4 = vector.insert_strided_slice %2, %cst {
+//          offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
+//   %5 = vector.extract_strided_slice %0 {
+//          offsets = [4], sizes = [4], strides = [1]
+//        } : vector<8xf16> to vector<4xf16>
+//   %6 = vector.bitcast %5 : vector<4xf16> to vector<2xf32>
+//   %7 = vector.insert_strided_slice %6, %cst {
+//          offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
+struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+public:
+  BreakDownVectorBitCast(MLIRContext *context,
+                         std::function<bool(vector::BitCastOp)> controlFn,
+                         PatternBenefit benefit)
+      : OpRewritePattern(context, benefit), controlFn(std::move(controlFn)) {}
+
+  LogicalResult matchAndRewrite(vector::BitCastOp bitcastOp,
+                                PatternRewriter &rewriter) const override {
+
+    if (controlFn && !controlFn(bitcastOp))
+      return failure();
+
+    VectorType castSrcType = bitcastOp.getSourceVectorType();
+    VectorType castDstType = bitcastOp.getResultVectorType();
+    assert(castSrcType.getRank() == castDstType.getRank());
+
+    // Only support rank 1 case for now.
+    if (castSrcType.getRank() != 1)
+      return failure();
+
+    int64_t castSrcLastDim = castSrcType.getShape().back();
+    int64_t castDstLastDim = castDstType.getShape().back();
+    // Require casting to less elements for now; other cases to be implemented.
+    if (castSrcLastDim < castDstLastDim)
+      return failure();
+
+    assert(castSrcLastDim % castDstLastDim == 0);
+    int64_t shrinkRatio = castSrcLastDim / castDstLastDim;
+    // Nothing to do if it is already bitcasting to a single element.
+    if (castSrcLastDim == shrinkRatio)
+      return failure();
+
+    Location loc = bitcastOp.getLoc();
+    Type elemType = castDstType.getElementType();
+    assert(elemType.isSignlessIntOrIndexOrFloat());
+
+    Value zero = rewriter.create<arith::ConstantOp>(
+        loc, elemType, rewriter.getZeroAttr(elemType));
+    Value res = rewriter.create<SplatOp>(loc, castDstType, zero);
+
+    SmallVector<int64_t> sliceShape{castDstLastDim};
+    SmallVector<int64_t> strides{1};
+    VectorType newCastDstType =
+        VectorType::get(SmallVector<int64_t>{castDstLastDim / shrinkRatio},
+                        castDstType.getElementType());
+
+    for (int i = 0, e = shrinkRatio; i < e; ++i) {
+      Value extracted = rewriter.create<ExtractStridedSliceOp>(
+          loc, bitcastOp.getSource(), ArrayRef<int64_t>{i * castDstLastDim},
+          sliceShape, strides);
+      Value bitcast =
+          rewriter.create<BitCastOp>(loc, newCastDstType, extracted);
+      res = rewriter.create<InsertStridedSliceOp>(
+          loc, bitcast, res,
+          ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
+    }
+    rewriter.replaceOp(bitcastOp, res);
+    return success();
+  }
+
+private:
+  std::function<bool(BitCastOp)> controlFn;
+};
+
 // Helper that returns a vector comparison that constructs a mask:
 //     mask = [0,1,..,n-1] + [o,o,..,o] < [b,b,..,b]
 //
@@ -1151,6 +1236,13 @@ void mlir::vector::populateBubbleVectorBitCastOpPatterns(
                                                      benefit);
 }
 
+void mlir::vector::populateBreakDownVectorBitCastOpPatterns(
+    RewritePatternSet &patterns,
+    std::function<bool(vector::BitCastOp)> controlFn, PatternBenefit benefit) {
+  patterns.add<BreakDownVectorBitCast>(patterns.getContext(),
+                                       std::move(controlFn), benefit);
+}
+
 void mlir::vector::populateVectorContractCanonicalizeMatmulToMMT(
     RewritePatternSet &patterns,
     std::function<LogicalResult(vector::ContractionOp)> constraint,
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index 086ae5c5030b3..f56e7cf256033 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -641,6 +641,61 @@ struct UnrollTransposePattern : public OpRewritePattern<vector::TransposeOp> {
   vector::UnrollVectorOptions options;
 };
 
+struct UnrollGatherPattern : public OpRewritePattern<vector::GatherOp> {
+  UnrollGatherPattern(MLIRContext *context,
+                      const vector::UnrollVectorOptions &options,
+                      PatternBenefit benefit = 1)
+      : OpRewritePattern<vector::GatherOp>(context, benefit), options(options) {
+  }
+
+  LogicalResult matchAndRewrite(vector::GatherOp gatherOp,
+                                PatternRewriter &rewriter) const override {
+    VectorType sourceVectorType = gatherOp.getVectorType();
+    if (sourceVectorType.getRank() == 0)
+      return failure();
+    auto targetShape = getTargetShape(options, gatherOp);
+    if (!targetShape)
+      return failure();
+    SmallVector<int64_t> strides(targetShape->size(), 1);
+    Location loc = gatherOp.getLoc();
+    ArrayRef<int64_t> originalSize = gatherOp.getVectorType().getShape();
+
+    // Prepare the result vector;
+    Value result = rewriter.create<arith::ConstantOp>(
+        loc, sourceVectorType, rewriter.getZeroAttr(sourceVectorType));
+    auto targetType =
+        VectorType::get(*targetShape, sourceVectorType.getElementType());
+
+    SmallVector<int64_t> loopOrder =
+        getUnrollOrder(originalSize.size(), gatherOp, options);
+    DecomposeShapeIterator indexToOffsets(originalSize, *targetShape,
+                                          loopOrder);
+    for (int64_t i = 0, e = indexToOffsets.maxIndex(); i < e; ++i) {
+      // To get the unrolled gather, extract the same slice based on the
+      // decomposed shape from each of the index, mask, and pass-through
+      // vectors.
+      SmallVector<int64_t> elementOffsets = indexToOffsets.getVectorOffset(i);
+      Value indexSubVec = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, gatherOp.getIndexVec(), elementOffsets, *targetShape, strides);
+      Value maskSubVec = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, gatherOp.getMask(), elementOffsets, *targetShape, strides);
+      Value passThruSubVec = rewriter.create<vector::ExtractStridedSliceOp>(
+          loc, gatherOp.getPassThru(), elementOffsets, *targetShape, strides);
+      auto slicedGather = rewriter.create<vector::GatherOp>(
+          loc, targetType, gatherOp.getBase(), gatherOp.getIndices(),
+          indexSubVec, maskSubVec, passThruSubVec);
+
+      result = rewriter.create<vector::InsertStridedSliceOp>(
+          loc, slicedGather, result, elementOffsets, strides);
+    }
+    rewriter.replaceOp(gatherOp, result);
+    return success();
+  }
+
+private:
+  vector::UnrollVectorOptions options;
+};
+
 } // namespace
 
 void mlir::vector::populateVectorUnrollPatterns(
@@ -649,5 +704,6 @@ void mlir::vector::populateVectorUnrollPatterns(
   patterns.add<UnrollTransferReadPattern, UnrollTransferWritePattern,
                UnrollContractionPattern, UnrollElementwisePattern,
                UnrollReductionPattern, UnrollMultiReductionPattern,
-               UnrollTransposePattern>(patterns.getContext(), options, benefit);
+               UnrollTransposePattern, UnrollGatherPattern>(
+      patterns.getContext(), options, benefit);
 }
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index 1820803f24258..3c525a6e76351 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -225,8 +225,8 @@ OpPrintingFlags &OpPrintingFlags::printGenericOpForm() {
 }
 
 /// Always skip Regions.
-OpPrintingFlags &OpPrintingFlags::skipRegions() {
-  skipRegionsFlag = true;
+OpPrintingFlags &OpPrintingFlags::skipRegions(bool skip) {
+  skipRegionsFlag = skip;
   return *this;
 }
 
@@ -2575,7 +2575,6 @@ void AsmPrinter::Impl::printOptionalAttrDict(ArrayRef<NamedAttribute> attrs,
   if (!filteredAttrs.empty())
     printFilteredAttributesFn(filteredAttrs);
 }
-
 void AsmPrinter::Impl::printNamedAttribute(NamedAttribute attr) {
   // Print the name without quotes if possible.
   ::printKeywordOrString(attr.getName().strref(), os);
@@ -3355,6 +3354,13 @@ void OperationPrinter::printGenericOp(Operation *op, bool printOpName) {
     os << ']';
   }
 
+  // Print the properties.
+  if (Attribute prop = op->getPropertiesAsAttribute()) {
+    os << " <";
+    Impl::printAttribute(prop);
+    os << '>';
+  }
+
   // Print regions.
   if (op->getNumRegions() != 0) {
     os << " (";
@@ -3365,7 +3371,7 @@ void OperationPrinter::printGenericOp(Operation *op, bool printOpName) {
     os << ')';
   }
 
-  auto attrs = op->getAttrs();
+  auto attrs = op->getDiscardableAttrs();
   printOptionalAttrDict(attrs);
 
   // Print the type signature of the operation.
@@ -3509,6 +3515,10 @@ void OperationPrinter::printRegion(Region &region, bool printEntryBlockArgs,
 
 void OperationPrinter::printAffineMapOfSSAIds(AffineMapAttr mapAttr,
                                               ValueRange operands) {
+  if (!mapAttr) {
+    os << "<<NULL AFFINE MAP>>";
+    return;
+  }
   AffineMap map = mapAttr.getValue();
   unsigned numDims = map.getNumDims();
   auto printValueName = [&](unsigned pos, bool isSymbol) {
diff --git a/mlir/lib/IR/BuiltinDialectBytecode.cpp b/mlir/lib/IR/BuiltinDialectBytecode.cpp
index 22a563dd7b2a2..40af5f3b17448 100644
--- a/mlir/lib/IR/BuiltinDialectBytecode.cpp
+++ b/mlir/lib/IR/BuiltinDialectBytecode.cpp
@@ -8,6 +8,7 @@
 
 #include "BuiltinDialectBytecode.h"
 #include "mlir/Bytecode/BytecodeImplementation.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
@@ -16,547 +17,62 @@
 
 using namespace mlir;
 
-//===----------------------------------------------------------------------===//
-// Encoding
-//===----------------------------------------------------------------------===//
-
-namespace {
-namespace builtin_encoding {
-/// This enum contains marker codes used to indicate which attribute is
-/// currently being decoded, and how it should be decoded. The order of these
-/// codes should generally be unchanged, as any changes will inevitably break
-/// compatibility with older bytecode.
-enum AttributeCode {
-  ///   ArrayAttr {
-  ///     elements: Attribute[]
-  ///   }
-  ///
-  kArrayAttr = 0,
-
-  ///   DictionaryAttr {
-  ///     attrs: <StringAttr, Attribute>[]
-  ///   }
-  kDictionaryAttr = 1,
-
-  ///   StringAttr {
-  ///     value: string
-  ///   }
-  kStringAttr = 2,
-
-  ///   StringAttrWithType {
-  ///     value: string,
-  ///     type: Type
-  ///   }
-  /// A variant of StringAttr with a type.
-  kStringAttrWithType = 3,
-
-  ///   FlatSymbolRefAttr {
-  ///     rootReference: StringAttr
-  ///   }
-  /// A variant of SymbolRefAttr with no leaf references.
-  kFlatSymbolRefAttr = 4,
-
-  ///   SymbolRefAttr {
-  ///     rootReference: StringAttr,
-  ///     leafReferences: FlatSymbolRefAttr[]
-  ///   }
-  kSymbolRefAttr = 5,
-
-  ///   TypeAttr {
-  ///     value: Type
-  ///   }
-  kTypeAttr = 6,
-
-  ///   UnitAttr {
-  ///   }
-  kUnitAttr = 7,
-
-  ///   IntegerAttr {
-  ///     type: Type
-  ///     value: APInt,
-  ///   }
-  kIntegerAttr = 8,
-
-  ///   FloatAttr {
-  ///     type: FloatType
-  ///     value: APFloat
-  ///   }
-  kFloatAttr = 9,
-
-  ///   CallSiteLoc {
-  ///    callee: LocationAttr,
-  ///    caller: LocationAttr
-  ///   }
-  kCallSiteLoc = 10,
-
-  ///   FileLineColLoc {
-  ///     file: StringAttr,
-  ///     line: varint,
-  ///     column: varint
-  ///   }
-  kFileLineColLoc = 11,
-
-  ///   FusedLoc {
-  ///     locations: LocationAttr[]
-  ///   }
-  kFusedLoc = 12,
-
-  ///   FusedLocWithMetadata {
-  ///     locations: LocationAttr[],
-  ///     metadata: Attribute
-  ///   }
-  /// A variant of FusedLoc with metadata.
-  kFusedLocWithMetadata = 13,
-
-  ///   NameLoc {
-  ///     name: StringAttr,
-  ///     childLoc: LocationAttr
-  ///   }
-  kNameLoc = 14,
-
-  ///   UnknownLoc {
-  ///   }
-  kUnknownLoc = 15,
-
-  ///   DenseResourceElementsAttr {
-  ///     type: Type,
-  ///     handle: ResourceHandle
-  ///   }
-  kDenseResourceElementsAttr = 16,
-
-  ///   DenseArrayAttr {
-  ///     type: RankedTensorType,
-  ///     data: blob
-  ///   }
-  kDenseArrayAttr = 17,
-
-  ///   DenseIntOrFPElementsAttr {
-  ///     type: ShapedType,
-  ///     data: blob
-  ///   }
-  kDenseIntOrFPElementsAttr = 18,
-
-  ///   DenseStringElementsAttr {
-  ///     type: ShapedType,
-  ///     isSplat: varint,
-  ///     data: string[]
-  ///   }
-  kDenseStringElementsAttr = 19,
-
-  ///   SparseElementsAttr {
-  ///     type: ShapedType,
-  ///     indices: DenseIntElementsAttr,
-  ///     values: DenseElementsAttr
-  ///   }
-  kSparseElementsAttr = 20,
-};
-
-/// This enum contains marker codes used to indicate which type is currently
-/// being decoded, and how it should be decoded. The order of these codes should
-/// generally be unchanged, as any changes will inevitably break compatibility
-/// with older bytecode.
-enum TypeCode {
-  ///   IntegerType {
-  ///     widthAndSignedness: varint // (width << 2) | (signedness)
-  ///   }
-  ///
-  kIntegerType = 0,
-
-  ///   IndexType {
-  ///   }
-  ///
-  kIndexType = 1,
-
-  ///   FunctionType {
-  ///     inputs: Type[],
-  ///     results: Type[]
-  ///   }
-  ///
-  kFunctionType = 2,
-
-  ///   BFloat16Type {
-  ///   }
-  ///
-  kBFloat16Type = 3,
-
-  ///   Float16Type {
-  ///   }
-  ///
-  kFloat16Type = 4,
-
-  ///   Float32Type {
-  ///   }
-  ///
-  kFloat32Type = 5,
-
-  ///   Float64Type {
-  ///   }
-  ///
-  kFloat64Type = 6,
-
-  ///   Float80Type {
-  ///   }
-  ///
-  kFloat80Type = 7,
-
-  ///   Float128Type {
-  ///   }
-  ///
-  kFloat128Type = 8,
-
-  ///   ComplexType {
-  ///     elementType: Type
-  ///   }
-  ///
-  kComplexType = 9,
-
-  ///   MemRefType {
-  ///     shape: svarint[],
-  ///     elementType: Type,
-  ///     layout: Attribute
-  ///   }
-  ///
-  kMemRefType = 10,
-
-  ///   MemRefTypeWithMemSpace {
-  ///     memorySpace: Attribute,
-  ///     shape: svarint[],
-  ///     elementType: Type,
-  ///     layout: Attribute
-  ///   }
-  /// Variant of MemRefType with non-default memory space.
-  kMemRefTypeWithMemSpace = 11,
-
-  ///   NoneType {
-  ///   }
-  ///
-  kNoneType = 12,
-
-  ///   RankedTensorType {
-  ///     shape: svarint[],
-  ///     elementType: Type,
-  ///   }
-  ///
-  kRankedTensorType = 13,
-
-  ///   RankedTensorTypeWithEncoding {
-  ///     encoding: Attribute,
-  ///     shape: svarint[],
-  ///     elementType: Type
-  ///   }
-  /// Variant of RankedTensorType with an encoding.
-  kRankedTensorTypeWithEncoding = 14,
-
-  ///   TupleType {
-  ///     elementTypes: Type[]
-  ///   }
-  kTupleType = 15,
-
-  ///   UnrankedMemRefType {
-  ///     shape: svarint[]
-  ///   }
-  ///
-  kUnrankedMemRefType = 16,
-
-  ///   UnrankedMemRefTypeWithMemSpace {
-  ///     memorySpace: Attribute,
-  ///     shape: svarint[]
-  ///   }
-  /// Variant of UnrankedMemRefType with non-default memory space.
-  kUnrankedMemRefTypeWithMemSpace = 17,
-
-  ///   UnrankedTensorType {
-  ///     elementType: Type
-  ///   }
-  ///
-  kUnrankedTensorType = 18,
-
-  ///   VectorType {
-  ///     shape: svarint[],
-  ///     elementType: Type
-  ///   }
-  ///
-  kVectorType = 19,
-
-  ///   VectorTypeWithScalableDims {
-  ///     numScalableDims: varint,
-  ///     shape: svarint[],
-  ///     elementType: Type
-  ///   }
-  /// Variant of VectorType with scalable dimensions.
-  kVectorTypeWithScalableDims = 20,
-};
-
-} // namespace builtin_encoding
-} // namespace
-
 //===----------------------------------------------------------------------===//
 // BuiltinDialectBytecodeInterface
 //===----------------------------------------------------------------------===//
 
 namespace {
-/// This class implements the bytecode interface for the builtin dialect.
-struct BuiltinDialectBytecodeInterface : public BytecodeDialectInterface {
-  BuiltinDialectBytecodeInterface(Dialect *dialect)
-      : BytecodeDialectInterface(dialect) {}
-
-  //===--------------------------------------------------------------------===//
-  // Attributes
-
-  Attribute readAttribute(DialectBytecodeReader &reader) const override;
-  ArrayAttr readArrayAttr(DialectBytecodeReader &reader) const;
-  DenseArrayAttr readDenseArrayAttr(DialectBytecodeReader &reader) const;
-  DenseElementsAttr
-  readDenseIntOrFPElementsAttr(DialectBytecodeReader &reader) const;
-  DenseStringElementsAttr
-  readDenseStringElementsAttr(DialectBytecodeReader &reader) const;
-  DenseResourceElementsAttr
-  readDenseResourceElementsAttr(DialectBytecodeReader &reader) const;
-  DictionaryAttr readDictionaryAttr(DialectBytecodeReader &reader) const;
-  FloatAttr readFloatAttr(DialectBytecodeReader &reader) const;
-  IntegerAttr readIntegerAttr(DialectBytecodeReader &reader) const;
-  SparseElementsAttr
-  readSparseElementsAttr(DialectBytecodeReader &reader) const;
-  StringAttr readStringAttr(DialectBytecodeReader &reader, bool hasType) const;
-  SymbolRefAttr readSymbolRefAttr(DialectBytecodeReader &reader,
-                                  bool hasNestedRefs) const;
-  TypeAttr readTypeAttr(DialectBytecodeReader &reader) const;
-
-  LocationAttr readCallSiteLoc(DialectBytecodeReader &reader) const;
-  LocationAttr readFileLineColLoc(DialectBytecodeReader &reader) const;
-  LocationAttr readFusedLoc(DialectBytecodeReader &reader,
-                            bool hasMetadata) const;
-  LocationAttr readNameLoc(DialectBytecodeReader &reader) const;
-
-  LogicalResult writeAttribute(Attribute attr,
-                               DialectBytecodeWriter &writer) const override;
-  void write(ArrayAttr attr, DialectBytecodeWriter &writer) const;
-  void write(DenseArrayAttr attr, DialectBytecodeWriter &writer) const;
-  void write(DenseIntOrFPElementsAttr attr,
-             DialectBytecodeWriter &writer) const;
-  void write(DenseStringElementsAttr attr, DialectBytecodeWriter &writer) const;
-  void write(DenseResourceElementsAttr attr,
-             DialectBytecodeWriter &writer) const;
-  void write(DictionaryAttr attr, DialectBytecodeWriter &writer) const;
-  void write(IntegerAttr attr, DialectBytecodeWriter &writer) const;
-  void write(FloatAttr attr, DialectBytecodeWriter &writer) const;
-  void write(SparseElementsAttr attr, DialectBytecodeWriter &writer) const;
-  void write(StringAttr attr, DialectBytecodeWriter &writer) const;
-  void write(SymbolRefAttr attr, DialectBytecodeWriter &writer) const;
-  void write(TypeAttr attr, DialectBytecodeWriter &writer) const;
-
-  void write(CallSiteLoc attr, DialectBytecodeWriter &writer) const;
-  void write(FileLineColLoc attr, DialectBytecodeWriter &writer) const;
-  void write(FusedLoc attr, DialectBytecodeWriter &writer) const;
-  void write(NameLoc attr, DialectBytecodeWriter &writer) const;
-  LogicalResult write(OpaqueLoc attr, DialectBytecodeWriter &writer) const;
-
-  //===--------------------------------------------------------------------===//
-  // Types
-
-  Type readType(DialectBytecodeReader &reader) const override;
-  ComplexType readComplexType(DialectBytecodeReader &reader) const;
-  IntegerType readIntegerType(DialectBytecodeReader &reader) const;
-  FunctionType readFunctionType(DialectBytecodeReader &reader) const;
-  MemRefType readMemRefType(DialectBytecodeReader &reader,
-                            bool hasMemSpace) const;
-  RankedTensorType readRankedTensorType(DialectBytecodeReader &reader,
-                                        bool hasEncoding) const;
-  TupleType readTupleType(DialectBytecodeReader &reader) const;
-  UnrankedMemRefType readUnrankedMemRefType(DialectBytecodeReader &reader,
-                                            bool hasMemSpace) const;
-  UnrankedTensorType
-  readUnrankedTensorType(DialectBytecodeReader &reader) const;
-  VectorType readVectorType(DialectBytecodeReader &reader,
-                            bool hasScalableDims) const;
-
-  LogicalResult writeType(Type type,
-                          DialectBytecodeWriter &writer) const override;
-  void write(ComplexType type, DialectBytecodeWriter &writer) const;
-  void write(IntegerType type, DialectBytecodeWriter &writer) const;
-  void write(FunctionType type, DialectBytecodeWriter &writer) const;
-  void write(MemRefType type, DialectBytecodeWriter &writer) const;
-  void write(RankedTensorType type, DialectBytecodeWriter &writer) const;
-  void write(TupleType type, DialectBytecodeWriter &writer) const;
-  void write(UnrankedMemRefType type, DialectBytecodeWriter &writer) const;
-  void write(UnrankedTensorType type, DialectBytecodeWriter &writer) const;
-  void write(VectorType type, DialectBytecodeWriter &writer) const;
-};
-} // namespace
-
-void builtin_dialect_detail::addBytecodeInterface(BuiltinDialect *dialect) {
-  dialect->addInterfaces<BuiltinDialectBytecodeInterface>();
-}
-
-//===----------------------------------------------------------------------===//
-// Attributes
-//===----------------------------------------------------------------------===//
-
-Attribute BuiltinDialectBytecodeInterface::readAttribute(
-    DialectBytecodeReader &reader) const {
-  uint64_t code;
-  if (failed(reader.readVarInt(code)))
-    return Attribute();
-  switch (code) {
-  case builtin_encoding::kArrayAttr:
-    return readArrayAttr(reader);
-  case builtin_encoding::kDictionaryAttr:
-    return readDictionaryAttr(reader);
-  case builtin_encoding::kStringAttr:
-    return readStringAttr(reader, /*hasType=*/false);
-  case builtin_encoding::kStringAttrWithType:
-    return readStringAttr(reader, /*hasType=*/true);
-  case builtin_encoding::kFlatSymbolRefAttr:
-    return readSymbolRefAttr(reader, /*hasNestedRefs=*/false);
-  case builtin_encoding::kSymbolRefAttr:
-    return readSymbolRefAttr(reader, /*hasNestedRefs=*/true);
-  case builtin_encoding::kTypeAttr:
-    return readTypeAttr(reader);
-  case builtin_encoding::kUnitAttr:
-    return UnitAttr::get(getContext());
-  case builtin_encoding::kIntegerAttr:
-    return readIntegerAttr(reader);
-  case builtin_encoding::kFloatAttr:
-    return readFloatAttr(reader);
-  case builtin_encoding::kCallSiteLoc:
-    return readCallSiteLoc(reader);
-  case builtin_encoding::kFileLineColLoc:
-    return readFileLineColLoc(reader);
-  case builtin_encoding::kFusedLoc:
-    return readFusedLoc(reader, /*hasMetadata=*/false);
-  case builtin_encoding::kFusedLocWithMetadata:
-    return readFusedLoc(reader, /*hasMetadata=*/true);
-  case builtin_encoding::kNameLoc:
-    return readNameLoc(reader);
-  case builtin_encoding::kUnknownLoc:
-    return UnknownLoc::get(getContext());
-  case builtin_encoding::kDenseResourceElementsAttr:
-    return readDenseResourceElementsAttr(reader);
-  case builtin_encoding::kDenseArrayAttr:
-    return readDenseArrayAttr(reader);
-  case builtin_encoding::kDenseIntOrFPElementsAttr:
-    return readDenseIntOrFPElementsAttr(reader);
-  case builtin_encoding::kDenseStringElementsAttr:
-    return readDenseStringElementsAttr(reader);
-  case builtin_encoding::kSparseElementsAttr:
-    return readSparseElementsAttr(reader);
-  default:
-    reader.emitError() << "unknown builtin attribute code: " << code;
-    return Attribute();
-  }
-}
-
-LogicalResult BuiltinDialectBytecodeInterface::writeAttribute(
-    Attribute attr, DialectBytecodeWriter &writer) const {
-  return TypeSwitch<Attribute, LogicalResult>(attr)
-      .Case<ArrayAttr, DenseArrayAttr, DenseIntOrFPElementsAttr,
-            DenseStringElementsAttr, DenseResourceElementsAttr, DictionaryAttr,
-            FloatAttr, IntegerAttr, SparseElementsAttr, StringAttr,
-            SymbolRefAttr, TypeAttr>([&](auto attr) {
-        write(attr, writer);
-        return success();
-      })
-      .Case<CallSiteLoc, FileLineColLoc, FusedLoc, NameLoc>([&](auto attr) {
-        write(attr, writer);
-        return success();
-      })
-      .Case([&](OpaqueLoc attr) { return write(attr, writer); })
-      .Case([&](UnitAttr) {
-        writer.writeVarInt(builtin_encoding::kUnitAttr);
-        return success();
-      })
-      .Case([&](UnknownLoc) {
-        writer.writeVarInt(builtin_encoding::kUnknownLoc);
-        return success();
-      })
-      .Default([&](Attribute) { return failure(); });
-}
-
-//===----------------------------------------------------------------------===//
-// ArrayAttr
-
-ArrayAttr BuiltinDialectBytecodeInterface::readArrayAttr(
-    DialectBytecodeReader &reader) const {
-  SmallVector<Attribute> elements;
-  if (failed(reader.readAttributes(elements)))
-    return ArrayAttr();
-  return ArrayAttr::get(getContext(), elements);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    ArrayAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kArrayAttr);
-  writer.writeAttributes(attr.getValue());
-}
 
 //===----------------------------------------------------------------------===//
-// DenseArrayAttr
+// Utility functions
 
-DenseArrayAttr BuiltinDialectBytecodeInterface::readDenseArrayAttr(
-    DialectBytecodeReader &reader) const {
-  Type elementType;
-  uint64_t size;
-  ArrayRef<char> blob;
-  if (failed(reader.readType(elementType)) || failed(reader.readVarInt(size)) ||
-      failed(reader.readBlob(blob)))
-    return DenseArrayAttr();
-  return DenseArrayAttr::get(elementType, size, blob);
-}
+// TODO: Move these to separate file.
 
-void BuiltinDialectBytecodeInterface::write(
-    DenseArrayAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kDenseArrayAttr);
-  writer.writeType(attr.getElementType());
-  writer.writeVarInt(attr.getSize());
-  writer.writeOwnedBlob(attr.getRawData());
+// Returns the bitwidth if known, else return 0.
+static unsigned getIntegerBitWidth(DialectBytecodeReader &reader, Type type) {
+  if (auto intType = dyn_cast<IntegerType>(type)) {
+    return intType.getWidth();
+  } else if (type.isa<IndexType>()) {
+    return IndexType::kInternalStorageBitWidth;
+  }
+  reader.emitError()
+      << "expected integer or index type for IntegerAttr, but got: " << type;
+  return 0;
 }
 
-//===----------------------------------------------------------------------===//
-// DenseIntOrFPElementsAttr
-
-DenseElementsAttr BuiltinDialectBytecodeInterface::readDenseIntOrFPElementsAttr(
-    DialectBytecodeReader &reader) const {
-  ShapedType type;
-  ArrayRef<char> blob;
-  if (failed(reader.readType(type)) || failed(reader.readBlob(blob)))
-    return DenseIntOrFPElementsAttr();
-  return DenseIntOrFPElementsAttr::getFromRawBuffer(type, blob);
+static LogicalResult readAPIntWithKnownWidth(DialectBytecodeReader &reader,
+                                             Type type, FailureOr<APInt> &val) {
+  unsigned bitWidth = getIntegerBitWidth(reader, type);
+  if (bitWidth == 0)
+    return failure();
+  val = reader.readAPIntWithKnownWidth(bitWidth);
+  return val;
 }
 
-void BuiltinDialectBytecodeInterface::write(
-    DenseIntOrFPElementsAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kDenseIntOrFPElementsAttr);
-  writer.writeType(attr.getType());
-  writer.writeOwnedBlob(attr.getRawData());
+static LogicalResult
+readAPFloatWithKnownSemantics(DialectBytecodeReader &reader, Type type,
+                              FailureOr<APFloat> &val) {
+  auto ftype = dyn_cast<FloatType>(type);
+  if (!ftype)
+    return failure();
+  val = reader.readAPFloatWithKnownSemantics(ftype.getFloatSemantics());
+  return success();
 }
 
-//===----------------------------------------------------------------------===//
-// DenseStringElementsAttr
-
-DenseStringElementsAttr
-BuiltinDialectBytecodeInterface::readDenseStringElementsAttr(
-    DialectBytecodeReader &reader) const {
-  ShapedType type;
-  uint64_t isSplat;
-  if (failed(reader.readType(type)) || failed(reader.readVarInt(isSplat)))
-    return DenseStringElementsAttr();
-
-  SmallVector<StringRef> values(isSplat ? 1 : type.getNumElements());
-  for (StringRef &value : values)
+LogicalResult
+readPotentiallySplatString(DialectBytecodeReader &reader, ShapedType type,
+                           bool isSplat,
+                           SmallVectorImpl<StringRef> &rawStringData) {
+  rawStringData.resize(isSplat ? 1 : type.getNumElements());
+  for (StringRef &value : rawStringData)
     if (failed(reader.readString(value)))
-      return DenseStringElementsAttr();
-  return DenseStringElementsAttr::get(type, values);
+      return failure();
+  return success();
 }
 
-void BuiltinDialectBytecodeInterface::write(
-    DenseStringElementsAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kDenseStringElementsAttr);
-  writer.writeType(attr.getType());
-
+void writePotentiallySplatString(DialectBytecodeWriter &writer,
+                                 DenseStringElementsAttr attr) {
   bool isSplat = attr.isSplat();
-  writer.writeVarInt(isSplat);
-
-  // If the attribute is a splat, only write out the single value.
   if (isSplat)
     return writer.writeOwnedString(attr.getRawStringData().front());
 
@@ -564,614 +80,39 @@ void BuiltinDialectBytecodeInterface::write(
     writer.writeOwnedString(str);
 }
 
-//===----------------------------------------------------------------------===//
-// DenseResourceElementsAttr
-
-DenseResourceElementsAttr
-BuiltinDialectBytecodeInterface::readDenseResourceElementsAttr(
-    DialectBytecodeReader &reader) const {
-  ShapedType type;
-  if (failed(reader.readType(type)))
-    return DenseResourceElementsAttr();
-
-  FailureOr<DenseResourceElementsHandle> handle =
-      reader.readResourceHandle<DenseResourceElementsHandle>();
-  if (failed(handle))
-    return DenseResourceElementsAttr();
-
-  return DenseResourceElementsAttr::get(type, *handle);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    DenseResourceElementsAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kDenseResourceElementsAttr);
-  writer.writeType(attr.getType());
-  writer.writeResourceHandle(attr.getRawHandle());
-}
-
-//===----------------------------------------------------------------------===//
-// DictionaryAttr
-
-DictionaryAttr BuiltinDialectBytecodeInterface::readDictionaryAttr(
-    DialectBytecodeReader &reader) const {
-  auto readNamedAttr = [&]() -> FailureOr<NamedAttribute> {
-    StringAttr name;
-    Attribute value;
-    if (failed(reader.readAttribute(name)) ||
-        failed(reader.readAttribute(value)))
-      return failure();
-    return NamedAttribute(name, value);
-  };
-  SmallVector<NamedAttribute> attrs;
-  if (failed(reader.readList(attrs, readNamedAttr)))
-    return DictionaryAttr();
-  return DictionaryAttr::get(getContext(), attrs);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    DictionaryAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kDictionaryAttr);
-  writer.writeList(attr.getValue(), [&](NamedAttribute attr) {
-    writer.writeAttribute(attr.getName());
-    writer.writeAttribute(attr.getValue());
-  });
-}
-
-//===----------------------------------------------------------------------===//
-// FloatAttr
-
-FloatAttr BuiltinDialectBytecodeInterface::readFloatAttr(
-    DialectBytecodeReader &reader) const {
-  FloatType type;
-  if (failed(reader.readType(type)))
-    return FloatAttr();
-  FailureOr<APFloat> value =
-      reader.readAPFloatWithKnownSemantics(type.getFloatSemantics());
-  if (failed(value))
-    return FloatAttr();
-  return FloatAttr::get(type, *value);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    FloatAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kFloatAttr);
-  writer.writeType(attr.getType());
-  writer.writeAPFloatWithKnownSemantics(attr.getValue());
-}
-
-//===----------------------------------------------------------------------===//
-// IntegerAttr
-
-IntegerAttr BuiltinDialectBytecodeInterface::readIntegerAttr(
-    DialectBytecodeReader &reader) const {
-  Type type;
-  if (failed(reader.readType(type)))
-    return IntegerAttr();
-
-  // Extract the value storage width from the type.
-  unsigned bitWidth;
-  if (auto intType = type.dyn_cast<IntegerType>()) {
-    bitWidth = intType.getWidth();
-  } else if (type.isa<IndexType>()) {
-    bitWidth = IndexType::kInternalStorageBitWidth;
-  } else {
-    reader.emitError()
-        << "expected integer or index type for IntegerAttr, but got: " << type;
-    return IntegerAttr();
-  }
-
-  FailureOr<APInt> value = reader.readAPIntWithKnownWidth(bitWidth);
-  if (failed(value))
-    return IntegerAttr();
-  return IntegerAttr::get(type, *value);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    IntegerAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kIntegerAttr);
-  writer.writeType(attr.getType());
-  writer.writeAPIntWithKnownWidth(attr.getValue());
-}
-
-//===----------------------------------------------------------------------===//
-// SparseElementsAttr
-
-SparseElementsAttr BuiltinDialectBytecodeInterface::readSparseElementsAttr(
-    DialectBytecodeReader &reader) const {
-  ShapedType type;
-  DenseIntElementsAttr indices;
-  DenseElementsAttr values;
-  if (failed(reader.readType(type)) || failed(reader.readAttribute(indices)) ||
-      failed(reader.readAttribute(values)))
-    return SparseElementsAttr();
-  return SparseElementsAttr::get(type, indices, values);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    SparseElementsAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kSparseElementsAttr);
-  writer.writeType(attr.getType());
-  writer.writeAttribute(attr.getIndices());
-  writer.writeAttribute(attr.getValues());
-}
-
-//===----------------------------------------------------------------------===//
-// StringAttr
-
-StringAttr
-BuiltinDialectBytecodeInterface::readStringAttr(DialectBytecodeReader &reader,
-                                                bool hasType) const {
-  StringRef string;
-  if (failed(reader.readString(string)))
-    return StringAttr();
-
-  // Read the type if present.
-  Type type;
-  if (!hasType)
-    type = NoneType::get(getContext());
-  else if (failed(reader.readType(type)))
-    return StringAttr();
-  return StringAttr::get(string, type);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    StringAttr attr, DialectBytecodeWriter &writer) const {
-  // We only encode the type if it isn't NoneType, which is significantly less
-  // common.
-  Type type = attr.getType();
-  if (!type.isa<NoneType>()) {
-    writer.writeVarInt(builtin_encoding::kStringAttrWithType);
-    writer.writeOwnedString(attr.getValue());
-    writer.writeType(type);
-    return;
-  }
-  writer.writeVarInt(builtin_encoding::kStringAttr);
-  writer.writeOwnedString(attr.getValue());
-}
-
-//===----------------------------------------------------------------------===//
-// SymbolRefAttr
+#include "mlir/IR/BuiltinDialectBytecode.cpp.inc"
 
-SymbolRefAttr BuiltinDialectBytecodeInterface::readSymbolRefAttr(
-    DialectBytecodeReader &reader, bool hasNestedRefs) const {
-  StringAttr rootReference;
-  if (failed(reader.readAttribute(rootReference)))
-    return SymbolRefAttr();
-  SmallVector<FlatSymbolRefAttr> nestedReferences;
-  if (hasNestedRefs && failed(reader.readAttributes(nestedReferences)))
-    return SymbolRefAttr();
-  return SymbolRefAttr::get(rootReference, nestedReferences);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    SymbolRefAttr attr, DialectBytecodeWriter &writer) const {
-  ArrayRef<FlatSymbolRefAttr> nestedRefs = attr.getNestedReferences();
-  writer.writeVarInt(nestedRefs.empty() ? builtin_encoding::kFlatSymbolRefAttr
-                                        : builtin_encoding::kSymbolRefAttr);
-
-  writer.writeAttribute(attr.getRootReference());
-  if (!nestedRefs.empty())
-    writer.writeAttributes(nestedRefs);
-}
-
-//===----------------------------------------------------------------------===//
-// TypeAttr
-
-TypeAttr BuiltinDialectBytecodeInterface::readTypeAttr(
-    DialectBytecodeReader &reader) const {
-  Type type;
-  if (failed(reader.readType(type)))
-    return TypeAttr();
-  return TypeAttr::get(type);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    TypeAttr attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kTypeAttr);
-  writer.writeType(attr.getValue());
-}
-
-//===----------------------------------------------------------------------===//
-// CallSiteLoc
-
-LocationAttr BuiltinDialectBytecodeInterface::readCallSiteLoc(
-    DialectBytecodeReader &reader) const {
-  LocationAttr callee, caller;
-  if (failed(reader.readAttribute(callee)) ||
-      failed(reader.readAttribute(caller)))
-    return LocationAttr();
-  return CallSiteLoc::get(callee, caller);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    CallSiteLoc attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kCallSiteLoc);
-  writer.writeAttribute(attr.getCallee());
-  writer.writeAttribute(attr.getCaller());
-}
-
-//===----------------------------------------------------------------------===//
-// FileLineColLoc
-
-LocationAttr BuiltinDialectBytecodeInterface::readFileLineColLoc(
-    DialectBytecodeReader &reader) const {
-  StringAttr filename;
-  uint64_t line, column;
-  if (failed(reader.readAttribute(filename)) ||
-      failed(reader.readVarInt(line)) || failed(reader.readVarInt(column)))
-    return LocationAttr();
-  return FileLineColLoc::get(filename, line, column);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    FileLineColLoc attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kFileLineColLoc);
-  writer.writeAttribute(attr.getFilename());
-  writer.writeVarInt(attr.getLine());
-  writer.writeVarInt(attr.getColumn());
-}
-
-//===----------------------------------------------------------------------===//
-// FusedLoc
-
-LocationAttr
-BuiltinDialectBytecodeInterface::readFusedLoc(DialectBytecodeReader &reader,
-                                              bool hasMetadata) const {
-  // Parse the child locations.
-  auto readLoc = [&]() -> FailureOr<Location> {
-    LocationAttr locAttr;
-    if (failed(reader.readAttribute(locAttr)))
-      return failure();
-    return Location(locAttr);
-  };
-  SmallVector<Location> locations;
-  if (failed(reader.readList(locations, readLoc)))
-    return LocationAttr();
-
-  // Parse the metadata if present.
-  Attribute metadata;
-  if (hasMetadata && failed(reader.readAttribute(metadata)))
-    return LocationAttr();
-
-  return FusedLoc::get(locations, metadata, getContext());
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    FusedLoc attr, DialectBytecodeWriter &writer) const {
-  if (Attribute metadata = attr.getMetadata()) {
-    writer.writeVarInt(builtin_encoding::kFusedLocWithMetadata);
-    writer.writeAttributes(attr.getLocations());
-    writer.writeAttribute(metadata);
-  } else {
-    writer.writeVarInt(builtin_encoding::kFusedLoc);
-    writer.writeAttributes(attr.getLocations());
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// NameLoc
-
-LocationAttr BuiltinDialectBytecodeInterface::readNameLoc(
-    DialectBytecodeReader &reader) const {
-  StringAttr name;
-  LocationAttr childLoc;
-  if (failed(reader.readAttribute(name)) ||
-      failed(reader.readAttribute(childLoc)))
-    return LocationAttr();
-  return NameLoc::get(name, childLoc);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    NameLoc attr, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kNameLoc);
-  writer.writeAttribute(attr.getName());
-  writer.writeAttribute(attr.getChildLoc());
-}
-
-//===----------------------------------------------------------------------===//
-// OpaqueLoc
-
-LogicalResult
-BuiltinDialectBytecodeInterface::write(OpaqueLoc attr,
-                                       DialectBytecodeWriter &writer) const {
-  // We can't encode an OpaqueLoc directly given that it is in-memory only, so
-  // encode the fallback instead.
-  return writeAttribute(attr.getFallbackLocation(), writer);
-}
-
-//===----------------------------------------------------------------------===//
-// Types
-//===----------------------------------------------------------------------===//
+/// This class implements the bytecode interface for the builtin dialect.
+struct BuiltinDialectBytecodeInterface : public BytecodeDialectInterface {
+  BuiltinDialectBytecodeInterface(Dialect *dialect)
+      : BytecodeDialectInterface(dialect) {}
 
-Type BuiltinDialectBytecodeInterface::readType(
-    DialectBytecodeReader &reader) const {
-  uint64_t code;
-  if (failed(reader.readVarInt(code)))
-    return Type();
-  switch (code) {
-  case builtin_encoding::kIntegerType:
-    return readIntegerType(reader);
-  case builtin_encoding::kIndexType:
-    return IndexType::get(getContext());
-  case builtin_encoding::kFunctionType:
-    return readFunctionType(reader);
-  case builtin_encoding::kBFloat16Type:
-    return BFloat16Type::get(getContext());
-  case builtin_encoding::kFloat16Type:
-    return Float16Type::get(getContext());
-  case builtin_encoding::kFloat32Type:
-    return Float32Type::get(getContext());
-  case builtin_encoding::kFloat64Type:
-    return Float64Type::get(getContext());
-  case builtin_encoding::kFloat80Type:
-    return Float80Type::get(getContext());
-  case builtin_encoding::kFloat128Type:
-    return Float128Type::get(getContext());
-  case builtin_encoding::kComplexType:
-    return readComplexType(reader);
-  case builtin_encoding::kMemRefType:
-    return readMemRefType(reader, /*hasMemSpace=*/false);
-  case builtin_encoding::kMemRefTypeWithMemSpace:
-    return readMemRefType(reader, /*hasMemSpace=*/true);
-  case builtin_encoding::kNoneType:
-    return NoneType::get(getContext());
-  case builtin_encoding::kRankedTensorType:
-    return readRankedTensorType(reader, /*hasEncoding=*/false);
-  case builtin_encoding::kRankedTensorTypeWithEncoding:
-    return readRankedTensorType(reader, /*hasEncoding=*/true);
-  case builtin_encoding::kTupleType:
-    return readTupleType(reader);
-  case builtin_encoding::kUnrankedMemRefType:
-    return readUnrankedMemRefType(reader, /*hasMemSpace=*/false);
-  case builtin_encoding::kUnrankedMemRefTypeWithMemSpace:
-    return readUnrankedMemRefType(reader, /*hasMemSpace=*/true);
-  case builtin_encoding::kUnrankedTensorType:
-    return readUnrankedTensorType(reader);
-  case builtin_encoding::kVectorType:
-    return readVectorType(reader, /*hasScalableDims=*/false);
-  case builtin_encoding::kVectorTypeWithScalableDims:
-    return readVectorType(reader, /*hasScalableDims=*/true);
+  //===--------------------------------------------------------------------===//
+  // Attributes
 
-  default:
-    reader.emitError() << "unknown builtin type code: " << code;
-    return Type();
+  Attribute readAttribute(DialectBytecodeReader &reader) const override {
+    return ::readAttribute(getContext(), reader);
   }
-}
-
-LogicalResult BuiltinDialectBytecodeInterface::writeType(
-    Type type, DialectBytecodeWriter &writer) const {
-  return TypeSwitch<Type, LogicalResult>(type)
-      .Case<ComplexType, IntegerType, FunctionType, MemRefType,
-            RankedTensorType, TupleType, UnrankedMemRefType, UnrankedTensorType,
-            VectorType>([&](auto type) {
-        write(type, writer);
-        return success();
-      })
-      .Case([&](IndexType) {
-        return writer.writeVarInt(builtin_encoding::kIndexType), success();
-      })
-      .Case([&](BFloat16Type) {
-        return writer.writeVarInt(builtin_encoding::kBFloat16Type), success();
-      })
-      .Case([&](Float16Type) {
-        return writer.writeVarInt(builtin_encoding::kFloat16Type), success();
-      })
-      .Case([&](Float32Type) {
-        return writer.writeVarInt(builtin_encoding::kFloat32Type), success();
-      })
-      .Case([&](Float64Type) {
-        return writer.writeVarInt(builtin_encoding::kFloat64Type), success();
-      })
-      .Case([&](Float80Type) {
-        return writer.writeVarInt(builtin_encoding::kFloat80Type), success();
-      })
-      .Case([&](Float128Type) {
-        return writer.writeVarInt(builtin_encoding::kFloat128Type), success();
-      })
-      .Case([&](NoneType) {
-        return writer.writeVarInt(builtin_encoding::kNoneType), success();
-      })
-      .Default([&](Type) { return failure(); });
-}
-
-//===----------------------------------------------------------------------===//
-// ComplexType
-
-ComplexType BuiltinDialectBytecodeInterface::readComplexType(
-    DialectBytecodeReader &reader) const {
-  Type elementType;
-  if (failed(reader.readType(elementType)))
-    return ComplexType();
-  return ComplexType::get(elementType);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    ComplexType type, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kComplexType);
-  writer.writeType(type.getElementType());
-}
-
-//===----------------------------------------------------------------------===//
-// IntegerType
-
-IntegerType BuiltinDialectBytecodeInterface::readIntegerType(
-    DialectBytecodeReader &reader) const {
-  uint64_t encoding;
-  if (failed(reader.readVarInt(encoding)))
-    return IntegerType();
-  return IntegerType::get(
-      getContext(), encoding >> 2,
-      static_cast<IntegerType::SignednessSemantics>(encoding & 0x3));
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    IntegerType type, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kIntegerType);
-  writer.writeVarInt((type.getWidth() << 2) | type.getSignedness());
-}
-
-//===----------------------------------------------------------------------===//
-// FunctionType
-
-FunctionType BuiltinDialectBytecodeInterface::readFunctionType(
-    DialectBytecodeReader &reader) const {
-  SmallVector<Type> inputs, results;
-  if (failed(reader.readTypes(inputs)) || failed(reader.readTypes(results)))
-    return FunctionType();
-  return FunctionType::get(getContext(), inputs, results);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    FunctionType type, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kFunctionType);
-  writer.writeTypes(type.getInputs());
-  writer.writeTypes(type.getResults());
-}
-
-//===----------------------------------------------------------------------===//
-// MemRefType
-
-MemRefType
-BuiltinDialectBytecodeInterface::readMemRefType(DialectBytecodeReader &reader,
-                                                bool hasMemSpace) const {
-  Attribute memorySpace;
-  if (hasMemSpace && failed(reader.readAttribute(memorySpace)))
-    return MemRefType();
-  SmallVector<int64_t> shape;
-  Type elementType;
-  MemRefLayoutAttrInterface layout;
-  if (failed(reader.readSignedVarInts(shape)) ||
-      failed(reader.readType(elementType)) ||
-      failed(reader.readAttribute(layout)))
-    return MemRefType();
-  return MemRefType::get(shape, elementType, layout, memorySpace);
-}
 
-void BuiltinDialectBytecodeInterface::write(
-    MemRefType type, DialectBytecodeWriter &writer) const {
-  if (Attribute memSpace = type.getMemorySpace()) {
-    writer.writeVarInt(builtin_encoding::kMemRefTypeWithMemSpace);
-    writer.writeAttribute(memSpace);
-  } else {
-    writer.writeVarInt(builtin_encoding::kMemRefType);
+  LogicalResult writeAttribute(Attribute attr,
+                               DialectBytecodeWriter &writer) const override {
+    return ::writeAttribute(attr, writer);
   }
-  writer.writeSignedVarInts(type.getShape());
-  writer.writeType(type.getElementType());
-  writer.writeAttribute(type.getLayout());
-}
-
-//===----------------------------------------------------------------------===//
-// RankedTensorType
 
-RankedTensorType BuiltinDialectBytecodeInterface::readRankedTensorType(
-    DialectBytecodeReader &reader, bool hasEncoding) const {
-  Attribute encoding;
-  if (hasEncoding && failed(reader.readAttribute(encoding)))
-    return RankedTensorType();
-  SmallVector<int64_t> shape;
-  Type elementType;
-  if (failed(reader.readSignedVarInts(shape)) ||
-      failed(reader.readType(elementType)))
-    return RankedTensorType();
-  return RankedTensorType::get(shape, elementType, encoding);
-}
+  //===--------------------------------------------------------------------===//
+  // Types
 
-void BuiltinDialectBytecodeInterface::write(
-    RankedTensorType type, DialectBytecodeWriter &writer) const {
-  if (Attribute encoding = type.getEncoding()) {
-    writer.writeVarInt(builtin_encoding::kRankedTensorTypeWithEncoding);
-    writer.writeAttribute(encoding);
-  } else {
-    writer.writeVarInt(builtin_encoding::kRankedTensorType);
+  Type readType(DialectBytecodeReader &reader) const override {
+    return ::readType(getContext(), reader);
   }
-  writer.writeSignedVarInts(type.getShape());
-  writer.writeType(type.getElementType());
-}
-
-//===----------------------------------------------------------------------===//
-// TupleType
-
-TupleType BuiltinDialectBytecodeInterface::readTupleType(
-    DialectBytecodeReader &reader) const {
-  SmallVector<Type> elements;
-  if (failed(reader.readTypes(elements)))
-    return TupleType();
-  return TupleType::get(getContext(), elements);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    TupleType type, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kTupleType);
-  writer.writeTypes(type.getTypes());
-}
-
-//===----------------------------------------------------------------------===//
-// UnrankedMemRefType
-
-UnrankedMemRefType BuiltinDialectBytecodeInterface::readUnrankedMemRefType(
-    DialectBytecodeReader &reader, bool hasMemSpace) const {
-  Attribute memorySpace;
-  if (hasMemSpace && failed(reader.readAttribute(memorySpace)))
-    return UnrankedMemRefType();
-  Type elementType;
-  if (failed(reader.readType(elementType)))
-    return UnrankedMemRefType();
-  return UnrankedMemRefType::get(elementType, memorySpace);
-}
 
-void BuiltinDialectBytecodeInterface::write(
-    UnrankedMemRefType type, DialectBytecodeWriter &writer) const {
-  if (Attribute memSpace = type.getMemorySpace()) {
-    writer.writeVarInt(builtin_encoding::kUnrankedMemRefTypeWithMemSpace);
-    writer.writeAttribute(memSpace);
-  } else {
-    writer.writeVarInt(builtin_encoding::kUnrankedMemRefType);
+  LogicalResult writeType(Type type,
+                          DialectBytecodeWriter &writer) const override {
+    return ::writeType(type, writer);
   }
-  writer.writeType(type.getElementType());
-}
-
-//===----------------------------------------------------------------------===//
-// UnrankedTensorType
-
-UnrankedTensorType BuiltinDialectBytecodeInterface::readUnrankedTensorType(
-    DialectBytecodeReader &reader) const {
-  Type elementType;
-  if (failed(reader.readType(elementType)))
-    return UnrankedTensorType();
-  return UnrankedTensorType::get(elementType);
-}
-
-void BuiltinDialectBytecodeInterface::write(
-    UnrankedTensorType type, DialectBytecodeWriter &writer) const {
-  writer.writeVarInt(builtin_encoding::kUnrankedTensorType);
-  writer.writeType(type.getElementType());
-}
-
-//===----------------------------------------------------------------------===//
-// VectorType
-
-VectorType
-BuiltinDialectBytecodeInterface::readVectorType(DialectBytecodeReader &reader,
-                                                bool hasScalableDims) const {
-  uint64_t numScalableDims = 0;
-  if (hasScalableDims && failed(reader.readVarInt(numScalableDims)))
-    return VectorType();
-  SmallVector<int64_t> shape;
-  Type elementType;
-  if (failed(reader.readSignedVarInts(shape)) ||
-      failed(reader.readType(elementType)))
-    return VectorType();
-  return VectorType::get(shape, elementType, numScalableDims);
-}
+};
+} // namespace
 
-void BuiltinDialectBytecodeInterface::write(
-    VectorType type, DialectBytecodeWriter &writer) const {
-  if (unsigned numScalableDims = type.getNumScalableDims()) {
-    writer.writeVarInt(builtin_encoding::kVectorTypeWithScalableDims);
-    writer.writeVarInt(numScalableDims);
-  } else {
-    writer.writeVarInt(builtin_encoding::kVectorType);
-  }
-  writer.writeSignedVarInts(type.getShape());
-  writer.writeType(type.getElementType());
+void builtin_dialect_detail::addBytecodeInterface(BuiltinDialect *dialect) {
+  dialect->addInterfaces<BuiltinDialectBytecodeInterface>();
 }
diff --git a/mlir/lib/IR/CMakeLists.txt b/mlir/lib/IR/CMakeLists.txt
index 4377ebe160554..c093fa78a9540 100644
--- a/mlir/lib/IR/CMakeLists.txt
+++ b/mlir/lib/IR/CMakeLists.txt
@@ -22,6 +22,7 @@ add_mlir_library(MLIRIR
   IntegerSet.cpp
   Location.cpp
   MLIRContext.cpp
+  ODSSupport.cpp
   Operation.cpp
   OperationSupport.cpp
   PatternMatch.cpp
@@ -44,6 +45,7 @@ add_mlir_library(MLIRIR
   DEPENDS
   MLIRBuiltinAttributesIncGen
   MLIRBuiltinAttributeInterfacesIncGen
+  MLIRBuiltinDialectBytecodeIncGen
   MLIRBuiltinDialectIncGen
   MLIRBuiltinLocationAttributesIncGen
   MLIRBuiltinOpsIncGen
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index e64babf35dac0..e47221f79f1b7 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
@@ -801,6 +802,64 @@ OperationName::UnregisteredOpModel::verifyRegionInvariants(Operation *) {
   return success();
 }
 
+std::optional<Attribute>
+OperationName::UnregisteredOpModel::getInherentAttr(Operation *op,
+                                                    StringRef name) {
+  auto dict = dyn_cast_or_null<DictionaryAttr>(getPropertiesAsAttr(op));
+  if (!dict)
+    return std::nullopt;
+  if (Attribute attr = dict.get(name))
+    return attr;
+  return std::nullopt;
+}
+void OperationName::UnregisteredOpModel::setInherentAttr(Operation *op,
+                                                         StringAttr name,
+                                                         Attribute value) {
+  auto dict = dyn_cast_or_null<DictionaryAttr>(getPropertiesAsAttr(op));
+  assert(dict);
+  NamedAttrList attrs(dict);
+  attrs.set(name, value);
+  *op->getPropertiesStorage().as<Attribute *>() =
+      attrs.getDictionary(op->getContext());
+}
+void OperationName::UnregisteredOpModel::populateInherentAttrs(
+    Operation *op, NamedAttrList &attrs) {}
+LogicalResult OperationName::UnregisteredOpModel::verifyInherentAttrs(
+    OperationName opName, NamedAttrList &attributes,
+    function_ref<InFlightDiagnostic()> getDiag) {
+  return success();
+}
+int OperationName::UnregisteredOpModel::getOpPropertyByteSize() {
+  return sizeof(Attribute);
+}
+void OperationName::UnregisteredOpModel::initProperties(
+    OperationName opName, OpaqueProperties storage, OpaqueProperties init) {
+  new (storage.as<Attribute *>()) Attribute();
+}
+void OperationName::UnregisteredOpModel::deleteProperties(
+    OpaqueProperties prop) {
+  prop.as<Attribute *>()->~Attribute();
+}
+void OperationName::UnregisteredOpModel::populateDefaultProperties(
+    OperationName opName, OpaqueProperties properties) {}
+LogicalResult OperationName::UnregisteredOpModel::setPropertiesFromAttr(
+    Operation *op, Attribute attr, InFlightDiagnostic *diag) {
+  *op->getPropertiesStorage().as<Attribute *>() = attr;
+  return success();
+}
+Attribute
+OperationName::UnregisteredOpModel::getPropertiesAsAttr(Operation *op) {
+  return *op->getPropertiesStorage().as<Attribute *>();
+}
+void OperationName::UnregisteredOpModel::copyProperties(OpaqueProperties lhs,
+                                                        OpaqueProperties rhs) {
+  *lhs.as<Attribute *>() = *rhs.as<Attribute *>();
+}
+llvm::hash_code
+OperationName::UnregisteredOpModel::hashProperties(OpaqueProperties prop) {
+  return llvm::hash_combine(*prop.as<Attribute *>());
+}
+
 //===----------------------------------------------------------------------===//
 // RegisteredOperationName
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/ODSSupport.cpp b/mlir/lib/IR/ODSSupport.cpp
new file mode 100644
index 0000000000000..ffb84f0f718d3
--- /dev/null
+++ b/mlir/lib/IR/ODSSupport.cpp
@@ -0,0 +1,57 @@
+//===- ODSSupport.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains out-of-line implementations of the support types that
+// Operation and related classes build on top of.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/ODSSupport.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Diagnostics.h"
+
+using namespace mlir;
+
+LogicalResult mlir::convertFromAttribute(int64_t &storage,
+                                         ::mlir::Attribute attr,
+                                         ::mlir::InFlightDiagnostic *diag) {
+  auto valueAttr = dyn_cast<IntegerAttr>(attr);
+  if (!valueAttr) {
+    if (diag)
+      *diag << "expected IntegerAttr for key `value`";
+    return failure();
+  }
+  storage = valueAttr.getValue().getSExtValue();
+  return success();
+}
+Attribute mlir::convertToAttribute(MLIRContext *ctx, int64_t storage) {
+  return IntegerAttr::get(IntegerType::get(ctx, 64), storage);
+}
+LogicalResult mlir::convertFromAttribute(MutableArrayRef<int64_t> storage,
+                                         ::mlir::Attribute attr,
+                                         ::mlir::InFlightDiagnostic *diag) {
+  auto valueAttr = dyn_cast<DenseI64ArrayAttr>(attr);
+  if (!valueAttr) {
+    if (diag)
+      *diag << "expected DenseI64ArrayAttr for key `value`";
+    return failure();
+  }
+  if (valueAttr.size() != static_cast<int64_t>(storage.size())) {
+    if (diag)
+      *diag << "Size mismatch in attribute conversion: " << valueAttr.size()
+            << " vs " << storage.size();
+    return failure();
+  }
+  llvm::copy(valueAttr.asArrayRef(), storage.begin());
+  return success();
+}
+Attribute mlir::convertToAttribute(MLIRContext *ctx,
+                                   ArrayRef<int64_t> storage) {
+  return DenseI64ArrayAttr::get(ctx, storage);
+}
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index 5a7594e0f88b7..bc7a9d852fd0f 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -7,15 +7,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/Operation.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/FoldInterfaces.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include <numeric>
+#include <optional>
 
 using namespace mlir;
 
@@ -25,19 +30,31 @@ using namespace mlir;
 
 /// Create a new Operation from operation state.
 Operation *Operation::create(const OperationState &state) {
-  return create(state.location, state.name, state.types, state.operands,
-                state.attributes.getDictionary(state.getContext()),
-                state.successors, state.regions);
+  Operation *op =
+      create(state.location, state.name, state.types, state.operands,
+             state.attributes.getDictionary(state.getContext()),
+             state.properties, state.successors, state.regions);
+  if (LLVM_UNLIKELY(state.propertiesAttr)) {
+    assert(!state.properties);
+    LogicalResult result =
+        op->setPropertiesFromAttribute(state.propertiesAttr,
+                                       /*diagnostic=*/nullptr);
+    assert(result.succeeded() && "invalid properties in op creation");
+    (void)result;
+  }
+  return op;
 }
 
 /// Create a new Operation with the specific fields.
 Operation *Operation::create(Location location, OperationName name,
                              TypeRange resultTypes, ValueRange operands,
-                             NamedAttrList &&attributes, BlockRange successors,
+                             NamedAttrList &&attributes,
+                             OpaqueProperties properties, BlockRange successors,
                              RegionRange regions) {
   unsigned numRegions = regions.size();
-  Operation *op = create(location, name, resultTypes, operands,
-                         std::move(attributes), successors, numRegions);
+  Operation *op =
+      create(location, name, resultTypes, operands, std::move(attributes),
+             properties, successors, numRegions);
   for (unsigned i = 0; i < numRegions; ++i)
     if (regions[i])
       op->getRegion(i).takeBody(*regions[i]);
@@ -47,21 +64,23 @@ Operation *Operation::create(Location location, OperationName name,
 /// Create a new Operation with the specific fields.
 Operation *Operation::create(Location location, OperationName name,
                              TypeRange resultTypes, ValueRange operands,
-                             NamedAttrList &&attributes, BlockRange successors,
+                             NamedAttrList &&attributes,
+                             OpaqueProperties properties, BlockRange successors,
                              unsigned numRegions) {
   // Populate default attributes.
   name.populateDefaultAttrs(attributes);
 
   return create(location, name, resultTypes, operands,
-                attributes.getDictionary(location.getContext()), successors,
-                numRegions);
+                attributes.getDictionary(location.getContext()), properties,
+                successors, numRegions);
 }
 
 /// Overload of create that takes an existing DictionaryAttr to avoid
 /// unnecessarily uniquing a list of attributes.
 Operation *Operation::create(Location location, OperationName name,
                              TypeRange resultTypes, ValueRange operands,
-                             DictionaryAttr attributes, BlockRange successors,
+                             DictionaryAttr attributes,
+                             OpaqueProperties properties, BlockRange successors,
                              unsigned numRegions) {
   assert(llvm::all_of(resultTypes, [](Type t) { return t; }) &&
          "unexpected null result type");
@@ -72,6 +91,7 @@ Operation *Operation::create(Location location, OperationName name,
   unsigned numSuccessors = successors.size();
   unsigned numOperands = operands.size();
   unsigned numResults = resultTypes.size();
+  int opPropertiesAllocSize = name.getOpPropertyByteSize();
 
   // If the operation is known to have no operands, don't allocate an operand
   // storage.
@@ -82,8 +102,10 @@ Operation *Operation::create(Location location, OperationName name,
   // into account the size of the operation, its trailing objects, and its
   // prefixed objects.
   size_t byteSize =
-      totalSizeToAlloc<detail::OperandStorage, BlockOperand, Region, OpOperand>(
-          needsOperandStorage ? 1 : 0, numSuccessors, numRegions, numOperands);
+      totalSizeToAlloc<detail::OperandStorage, detail::OpProperties,
+                       BlockOperand, Region, OpOperand>(
+          needsOperandStorage ? 1 : 0, opPropertiesAllocSize, numSuccessors,
+          numRegions, numOperands);
   size_t prefixByteSize = llvm::alignTo(
       Operation::prefixAllocSize(numTrailingResults, numInlineResults),
       alignof(Operation));
@@ -91,9 +113,9 @@ Operation *Operation::create(Location location, OperationName name,
   void *rawMem = mallocMem + prefixByteSize;
 
   // Create the new Operation.
-  Operation *op =
-      ::new (rawMem) Operation(location, name, numResults, numSuccessors,
-                               numRegions, attributes, needsOperandStorage);
+  Operation *op = ::new (rawMem) Operation(
+      location, name, numResults, numSuccessors, numRegions,
+      opPropertiesAllocSize, attributes, properties, needsOperandStorage);
 
   assert((numSuccessors == 0 || op->mightHaveTrait<OpTrait::IsTerminator>()) &&
          "unexpected successors in a non-terminator operation");
@@ -122,16 +144,22 @@ Operation *Operation::create(Location location, OperationName name,
   for (unsigned i = 0; i != numSuccessors; ++i)
     new (&blockOperands[i]) BlockOperand(op, successors[i]);
 
+  // This must be done after properties are initalized.
+  op->setAttrs(attributes);
+
   return op;
 }
 
 Operation::Operation(Location location, OperationName name, unsigned numResults,
                      unsigned numSuccessors, unsigned numRegions,
-                     DictionaryAttr attributes, bool hasOperandStorage)
+                     int fullPropertiesStorageSize, DictionaryAttr attributes,
+                     OpaqueProperties properties, bool hasOperandStorage)
     : location(location), numResults(numResults), numSuccs(numSuccessors),
-      numRegions(numRegions), hasOperandStorage(hasOperandStorage), name(name),
-      attrs(attributes) {
+      numRegions(numRegions), hasOperandStorage(hasOperandStorage),
+      propertiesStorageSize((fullPropertiesStorageSize + 7) / 8), name(name) {
   assert(attributes && "unexpected null attribute dictionary");
+  assert(fullPropertiesStorageSize <= propertiesCapacity &&
+         "Properties size overflow");
 #ifndef NDEBUG
   if (!getDialect() && !getContext()->allowsUnregisteredDialects())
     llvm::report_fatal_error(
@@ -140,6 +168,8 @@ Operation::Operation(Location location, OperationName name, unsigned numResults,
         "allowUnregisteredDialects() on the MLIRContext, or use "
         "-allow-unregistered-dialect with the MLIR tool used.");
 #endif
+  if (fullPropertiesStorageSize)
+    name.initOpProperties(getPropertiesStorage(), properties);
 }
 
 // Operations are deleted through the destroy() member because they are
@@ -168,6 +198,8 @@ Operation::~Operation() {
   // Explicitly destroy the regions.
   for (auto &region : getRegions())
     region.~Region();
+  if (propertiesStorageSize)
+    name.destroyOpProperties(getPropertiesStorage());
 }
 
 /// Destroy this operation or one of its subclasses.
@@ -259,6 +291,68 @@ InFlightDiagnostic Operation::emitRemark(const Twine &message) {
   return diag;
 }
 
+DictionaryAttr Operation::getAttrDictionary() {
+  if (getPropertiesStorageSize()) {
+    NamedAttrList attrsList = attrs;
+    getName().populateInherentAttrs(this, attrsList);
+    return attrsList.getDictionary(getContext());
+  }
+  return attrs;
+}
+
+void Operation::setAttrs(DictionaryAttr newAttrs) {
+  assert(newAttrs && "expected valid attribute dictionary");
+  if (getPropertiesStorageSize()) {
+    attrs = DictionaryAttr::get(getContext(), {});
+    for (const NamedAttribute &attr : newAttrs)
+      setAttr(attr.getName(), attr.getValue());
+    return;
+  }
+  attrs = newAttrs;
+}
+void Operation::setAttrs(ArrayRef<NamedAttribute> newAttrs) {
+  if (getPropertiesStorageSize()) {
+    setAttrs(DictionaryAttr::get(getContext(), {}));
+    for (const NamedAttribute &attr : newAttrs)
+      setAttr(attr.getName(), attr.getValue());
+    return;
+  }
+  attrs = DictionaryAttr::get(getContext(), newAttrs);
+}
+
+std::optional<Attribute> Operation::getInherentAttr(StringRef name) {
+  return getName().getInherentAttr(this, name);
+}
+
+void Operation::setInherentAttr(StringAttr name, Attribute value) {
+  getName().setInherentAttr(this, name, value);
+}
+
+Attribute Operation::getPropertiesAsAttribute() {
+  std::optional<RegisteredOperationName> info = getRegisteredInfo();
+  if (LLVM_UNLIKELY(!info))
+    return *getPropertiesStorage().as<Attribute *>();
+  return info->getOpPropertiesAsAttribute(this);
+}
+LogicalResult
+Operation::setPropertiesFromAttribute(Attribute attr,
+                                      InFlightDiagnostic *diagnostic) {
+  std::optional<RegisteredOperationName> info = getRegisteredInfo();
+  if (LLVM_UNLIKELY(!info)) {
+    *getPropertiesStorage().as<Attribute *>() = attr;
+    return success();
+  }
+  return info->setOpPropertiesFromAttribute(this, attr, diagnostic);
+}
+
+void Operation::copyProperties(OpaqueProperties rhs) {
+  name.copyOpProperties(getPropertiesStorage(), rhs);
+}
+
+llvm::hash_code Operation::hashProperties() {
+  return name.hashOpProperties(getPropertiesStorage());
+}
+
 //===----------------------------------------------------------------------===//
 // Operation Ordering
 //===----------------------------------------------------------------------===//
@@ -581,7 +675,7 @@ Operation *Operation::clone(IRMapping &mapper, CloneOptions options) {
 
   // Create the new operation.
   auto *newOp = create(getLoc(), getName(), getResultTypes(), operands, attrs,
-                       successors, getNumRegions());
+                       getPropertiesStorage(), successors, getNumRegions());
   mapper.map(this, newOp);
 
   // Clone the regions.
@@ -636,6 +730,20 @@ void OpState::printOpName(Operation *op, OpAsmPrinter &p,
   p.getStream() << name;
 }
 
+/// Parse properties as a Attribute.
+ParseResult OpState::genericParseProperties(OpAsmParser &parser,
+                                            Attribute &result) {
+  if (parser.parseLess() || parser.parseAttribute(result) ||
+      parser.parseGreater())
+    return failure();
+  return success();
+}
+
+/// Print the properties as a Attribute.
+void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties) {
+  p << "<" << properties << ">";
+}
+
 /// Emit an error about fatal conditions with this operation, reporting up to
 /// any diagnostic handlers that may be listening.
 InFlightDiagnostic OpState::emitError(const Twine &message) {
diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp
index f2a2fc2400444..a0a86e776ce37 100644
--- a/mlir/lib/IR/OperationSupport.cpp
+++ b/mlir/lib/IR/OperationSupport.cpp
@@ -193,6 +193,23 @@ OperationState::OperationState(Location location, StringRef name,
     : OperationState(location, OperationName(name, location.getContext()),
                      operands, types, attributes, successors, regions) {}
 
+OperationState::~OperationState() {
+  if (properties)
+    propertiesDeleter(properties);
+}
+
+LogicalResult
+OperationState::setProperties(Operation *op,
+                              InFlightDiagnostic *diagnostic) const {
+  if (LLVM_UNLIKELY(propertiesAttr)) {
+    assert(!properties);
+    return op->setPropertiesFromAttribute(propertiesAttr, diagnostic);
+  }
+  if (properties)
+    propertiesSetter(op->getPropertiesStorage(), properties);
+  return success();
+}
+
 void OperationState::addOperands(ValueRange newOperands) {
   operands.append(newOperands.begin(), newOperands.end());
 }
@@ -633,8 +650,9 @@ llvm::hash_code OperationEquivalence::computeHash(
   //   - Operation Name
   //   - Attributes
   //   - Result Types
-  llvm::hash_code hash = llvm::hash_combine(
-      op->getName(), op->getAttrDictionary(), op->getResultTypes());
+  llvm::hash_code hash =
+      llvm::hash_combine(op->getName(), op->getAttrDictionary(),
+                         op->getResultTypes(), op->hashProperties());
 
   //   - Operands
   ValueRange operands = op->getOperands();
diff --git a/mlir/lib/Interfaces/CMakeLists.txt b/mlir/lib/Interfaces/CMakeLists.txt
index 20073e7030557..dbf6e69a45255 100644
--- a/mlir/lib/Interfaces/CMakeLists.txt
+++ b/mlir/lib/Interfaces/CMakeLists.txt
@@ -9,6 +9,7 @@ set(LLVM_OPTIONAL_SOURCES
   InferIntRangeInterface.cpp
   InferTypeOpInterface.cpp
   LoopLikeInterface.cpp
+  Mem2RegInterfaces.cpp
   ParallelCombiningOpInterface.cpp
   RuntimeVerifiableOpInterface.cpp
   ShapedOpInterfaces.cpp
@@ -45,6 +46,7 @@ add_mlir_interface_library(DestinationStyleOpInterface)
 add_mlir_interface_library(InferIntRangeInterface)
 add_mlir_interface_library(InferTypeOpInterface)
 add_mlir_interface_library(LoopLikeInterface)
+add_mlir_interface_library(Mem2RegInterfaces)
 add_mlir_interface_library(ParallelCombiningOpInterface)
 add_mlir_interface_library(RuntimeVerifiableOpInterface)
 add_mlir_interface_library(ShapedOpInterfaces)
diff --git a/mlir/lib/Interfaces/InferTypeOpInterface.cpp b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
index 7d464af78023e..ebb10e07cebd2 100644
--- a/mlir/lib/Interfaces/InferTypeOpInterface.cpp
+++ b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
@@ -220,15 +220,15 @@ LogicalResult mlir::detail::inferReturnTensorTypes(
     function_ref<
         LogicalResult(MLIRContext *, std::optional<Location> location,
                       ValueShapeRange operands, DictionaryAttr attributes,
-                      RegionRange regions,
+                      OpaqueProperties properties, RegionRange regions,
                       SmallVectorImpl<ShapedTypeComponents> &retComponents)>
         componentTypeFn,
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   SmallVector<ShapedTypeComponents, 2> retComponents;
-  if (failed(componentTypeFn(context, location, operands, attributes, regions,
-                             retComponents)))
+  if (failed(componentTypeFn(context, location, operands, attributes,
+                             properties, regions, retComponents)))
     return failure();
   for (const auto &shapeAndType : retComponents) {
     Type elementTy = shapeAndType.getElementType();
@@ -249,7 +249,12 @@ LogicalResult mlir::detail::inferReturnTensorTypes(
 LogicalResult mlir::detail::verifyInferredResultTypes(Operation *op) {
   SmallVector<Type, 4> inferredReturnTypes(op->getResultTypes());
   auto retTypeFn = cast<InferTypeOpInterface>(op);
-  return retTypeFn.refineReturnTypes(op->getContext(), op->getLoc(),
-                                     op->getOperands(), op->getAttrDictionary(),
-                                     op->getRegions(), inferredReturnTypes);
+  auto result = retTypeFn.refineReturnTypes(
+      op->getContext(), op->getLoc(), op->getOperands(),
+      op->getAttrDictionary(), op->getPropertiesStorage(), op->getRegions(),
+      inferredReturnTypes);
+  if (failed(result))
+    op->emitOpError() << "failed to infer returned types";
+
+  return result;
 }
diff --git a/clang/include/clang/Basic/DiagnosticAnalysisKinds.td b/mlir/lib/Interfaces/Mem2RegInterfaces.cpp
similarity index 61%
rename from clang/include/clang/Basic/DiagnosticAnalysisKinds.td
rename to mlir/lib/Interfaces/Mem2RegInterfaces.cpp
index 20efd96b85fd9..aadd76b44df53 100644
--- a/clang/include/clang/Basic/DiagnosticAnalysisKinds.td
+++ b/mlir/lib/Interfaces/Mem2RegInterfaces.cpp
@@ -1,4 +1,4 @@
-//==--- DiagnosticAnalysisKinds.td - libanalysis diagnostics --------------===//
+//===-- Mem2RegInterfaces.cpp - Mem2Reg interfaces --------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,6 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-let Component = "Analysis" in {
+#include "mlir/Interfaces/Mem2RegInterfaces.h"
 
-}
+#include "mlir/Interfaces/Mem2RegInterfaces.cpp.inc"
diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
index 597bc7d991722..95fb785defdd9 100644
--- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
+++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
@@ -356,7 +356,7 @@ LogicalResult ValueBoundsConstraintSet::computeBound(
   return success();
 }
 
-LogicalResult ValueBoundsConstraintSet::computeBound(
+LogicalResult ValueBoundsConstraintSet::computeDependentBound(
     AffineMap &resultMap, ValueDimList &mapOperands, presburger::BoundType type,
     Value value, std::optional<int64_t> dim, ValueDimList dependencies,
     bool closedUB) {
@@ -368,6 +368,40 @@ LogicalResult ValueBoundsConstraintSet::computeBound(
       closedUB);
 }
 
+LogicalResult ValueBoundsConstraintSet::computeIndependentBound(
+    AffineMap &resultMap, ValueDimList &mapOperands, presburger::BoundType type,
+    Value value, std::optional<int64_t> dim, ValueRange independencies,
+    bool closedUB) {
+  // Return "true" if the given value is independent of all values in
+  // `independencies`. I.e., neither the value itself nor any value in the
+  // backward slice (reverse use-def chain) is contained in `independencies`.
+  auto isIndependent = [&](Value v) {
+    SmallVector<Value> worklist;
+    DenseSet<Value> visited;
+    worklist.push_back(v);
+    while (!worklist.empty()) {
+      Value next = worklist.pop_back_val();
+      if (visited.contains(next))
+        continue;
+      visited.insert(next);
+      if (llvm::is_contained(independencies, next))
+        return false;
+      // TODO: DominanceInfo could be used to stop the traversal early.
+      Operation *op = next.getDefiningOp();
+      if (!op)
+        continue;
+      worklist.append(op->getOperands().begin(), op->getOperands().end());
+    }
+    return true;
+  };
+
+  // Reify bounds in terms of any independent values.
+  return computeBound(
+      resultMap, mapOperands, type, value, dim,
+      [&](Value v, std::optional<int64_t> d) { return isIndependent(v); },
+      closedUB);
+}
+
 FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound(
     presburger::BoundType type, Value value, std::optional<int64_t> dim,
     StopConditionFn stopCondition, bool closedUB) {
diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp
index 1ea4bef5402a7..eca0297733e7d 100644
--- a/mlir/lib/Rewrite/ByteCode.cpp
+++ b/mlir/lib/Rewrite/ByteCode.cpp
@@ -1613,8 +1613,8 @@ void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter,
     // TODO: Handle failure.
     if (failed(inferInterface->inferReturnTypes(
             state.getContext(), state.location, state.operands,
-            state.attributes.getDictionary(state.getContext()), state.regions,
-            state.types)))
+            state.attributes.getDictionary(state.getContext()),
+            state.getRawProperties(), state.regions, state.types)))
       return;
   } else {
     // Otherwise, this is a fixed number of results.
diff --git a/mlir/lib/TableGen/CMakeLists.txt b/mlir/lib/TableGen/CMakeLists.txt
index 74c6cdc834abd..2736d95cc9742 100644
--- a/mlir/lib/TableGen/CMakeLists.txt
+++ b/mlir/lib/TableGen/CMakeLists.txt
@@ -24,6 +24,7 @@ llvm_add_library(MLIRTableGen STATIC
   Pass.cpp
   Pattern.cpp
   Predicate.cpp
+  Property.cpp
   Region.cpp
   SideEffects.cpp
   Successor.cpp
diff --git a/mlir/lib/TableGen/CodeGenHelpers.cpp b/mlir/lib/TableGen/CodeGenHelpers.cpp
index 193e8c1ce374b..afd02b1a64ac9 100644
--- a/mlir/lib/TableGen/CodeGenHelpers.cpp
+++ b/mlir/lib/TableGen/CodeGenHelpers.cpp
@@ -133,13 +133,18 @@ static ::mlir::LogicalResult {0}(
 /// functions are stripped anyways.
 static const char *const attrConstraintCode = R"(
 static ::mlir::LogicalResult {0}(
-    ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {
-  if (attr && !({1})) {
-    return op->emitOpError("attribute '") << attrName
+    ::mlir::Attribute attr, ::llvm::StringRef attrName, llvm::function_ref<::mlir::InFlightDiagnostic()> getDiag) {{
+  if (attr && !({1}))
+    return getDiag() << "attribute '" << attrName
         << "' failed to satisfy constraint: {2}";
-  }
   return ::mlir::success();
 }
+static ::mlir::LogicalResult {0}(
+    ::mlir::Operation *op, ::mlir::Attribute attr, ::llvm::StringRef attrName) {{
+  return {0}(attr, attrName, [op]() {{
+    return op->emitOpError();
+  });
+}
 )";
 
 /// Code for a successor constraint.
diff --git a/mlir/lib/TableGen/Dialect.cpp b/mlir/lib/TableGen/Dialect.cpp
index ec2c31dab440c..74f3981d91947 100644
--- a/mlir/lib/TableGen/Dialect.cpp
+++ b/mlir/lib/TableGen/Dialect.cpp
@@ -103,6 +103,10 @@ bool Dialect::isExtensible() const {
   return def->getValueAsBit("isExtensible");
 }
 
+bool Dialect::usePropertiesForAttributes() const {
+  return def->getValueAsBit("usePropertiesForAttributes");
+}
+
 bool Dialect::operator==(const Dialect &other) const {
   return def == other.def;
 }
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 291777198eafa..03557d9d283a3 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Operator.h"
+#include "mlir/TableGen/Argument.h"
 #include "mlir/TableGen/Predicate.h"
 #include "mlir/TableGen/Trait.h"
 #include "mlir/TableGen/Type.h"
@@ -322,14 +323,23 @@ auto Operator::getTraits() const -> llvm::iterator_range<const_trait_iterator> {
   return {trait_begin(), trait_end()};
 }
 
-auto Operator::attribute_begin() const -> attribute_iterator {
+auto Operator::attribute_begin() const -> const_attribute_iterator {
   return attributes.begin();
 }
-auto Operator::attribute_end() const -> attribute_iterator {
+auto Operator::attribute_end() const -> const_attribute_iterator {
   return attributes.end();
 }
 auto Operator::getAttributes() const
-    -> llvm::iterator_range<attribute_iterator> {
+    -> llvm::iterator_range<const_attribute_iterator> {
+  return {attribute_begin(), attribute_end()};
+}
+auto Operator::attribute_begin() -> attribute_iterator {
+  return attributes.begin();
+}
+auto Operator::attribute_end() -> attribute_iterator {
+  return attributes.end();
+}
+auto Operator::getAttributes() -> llvm::iterator_range<attribute_iterator> {
   return {attribute_begin(), attribute_end()};
 }
 
@@ -542,6 +552,7 @@ void Operator::populateOpStructure() {
   auto &recordKeeper = def.getRecords();
   auto *typeConstraintClass = recordKeeper.getClass("TypeConstraint");
   auto *attrClass = recordKeeper.getClass("Attr");
+  auto *propertyClass = recordKeeper.getClass("Property");
   auto *derivedAttrClass = recordKeeper.getClass("DerivedAttr");
   auto *opVarClass = recordKeeper.getClass("OpVariable");
   numNativeAttributes = 0;
@@ -576,9 +587,14 @@ void Operator::populateOpStructure() {
                         "derived attributes not allowed in argument list");
       attributes.push_back({givenName, Attribute(argDef)});
       ++numNativeAttributes;
+    } else if (argDef->isSubClassOf(propertyClass)) {
+      if (givenName.empty())
+        PrintFatalError(argDef->getLoc(), "properties must be named");
+      properties.push_back({givenName, Property(argDef)});
     } else {
-      PrintFatalError(def.getLoc(), "unexpected def type; only defs deriving "
-                                    "from TypeConstraint or Attr are allowed");
+      PrintFatalError(def.getLoc(),
+                      "unexpected def type; only defs deriving "
+                      "from TypeConstraint or Attr or Property are allowed");
     }
     if (!givenName.empty())
       argumentsAndResultsIndex[givenName] = i;
@@ -608,7 +624,7 @@ void Operator::populateOpStructure() {
   // `attributes` because we will put their elements' pointers in `arguments`.
   // SmallVector may perform re-allocation under the hood when adding new
   // elements.
-  int operandIndex = 0, attrIndex = 0;
+  int operandIndex = 0, attrIndex = 0, propIndex = 0;
   for (unsigned i = 0; i != numArgs; ++i) {
     Record *argDef = dyn_cast<DefInit>(argumentValues->getArg(i))->getDef();
     if (argDef->isSubClassOf(opVarClass))
@@ -618,11 +634,13 @@ void Operator::populateOpStructure() {
       attrOrOperandMapping.push_back(
           {OperandOrAttribute::Kind::Operand, operandIndex});
       arguments.emplace_back(&operands[operandIndex++]);
-    } else {
-      assert(argDef->isSubClassOf(attrClass));
+    } else if (argDef->isSubClassOf(attrClass)) {
       attrOrOperandMapping.push_back(
           {OperandOrAttribute::Kind::Attribute, attrIndex});
       arguments.emplace_back(&attributes[attrIndex++]);
+    } else {
+      assert(argDef->isSubClassOf(propertyClass));
+      arguments.emplace_back(&properties[propIndex++]);
     }
   }
 
diff --git a/mlir/lib/TableGen/Pattern.cpp b/mlir/lib/TableGen/Pattern.cpp
index 1faea137a301b..e8625b2e6b710 100644
--- a/mlir/lib/TableGen/Pattern.cpp
+++ b/mlir/lib/TableGen/Pattern.cpp
@@ -807,15 +807,16 @@ void Pattern::collectBoundSymbols(DagNode tree, SymbolInfoMap &infoMap,
     // The operand in `either` DAG should be bound to the operation in the
     // parent DagNode.
     auto collectSymbolInEither = [&](DagNode parent, DagNode tree,
-                                     int &opArgIdx) {
+                                     int opArgIdx) {
       for (int i = 0; i < tree.getNumArgs(); ++i, ++opArgIdx) {
         if (DagNode subTree = tree.getArgAsNestedDag(i)) {
           collectBoundSymbols(subTree, infoMap, isSrcPattern);
         } else {
           auto argName = tree.getArgName(i);
-          if (!argName.empty() && argName != "_")
+          if (!argName.empty() && argName != "_") {
             verifyBind(infoMap.bindOpArgument(parent, argName, op, opArgIdx),
                        argName);
+          }
         }
       }
     };
@@ -824,6 +825,14 @@ void Pattern::collectBoundSymbols(DagNode tree, SymbolInfoMap &infoMap,
       if (auto treeArg = tree.getArgAsNestedDag(i)) {
         if (treeArg.isEither()) {
           collectSymbolInEither(tree, treeArg, opArgIdx);
+          // `either` DAG is *flattened*. For example,
+          //
+          //  (FooOp (either arg0, arg1), arg2)
+          //
+          //  can be viewed as:
+          //
+          //  (FooOp arg0, arg1, arg2)
+          ++opArgIdx;
         } else {
           // This DAG node argument is a DAG node itself. Go inside recursively.
           collectBoundSymbols(treeArg, infoMap, isSrcPattern);
diff --git a/mlir/lib/TableGen/Property.cpp b/mlir/lib/TableGen/Property.cpp
new file mode 100644
index 0000000000000..966c65da16788
--- /dev/null
+++ b/mlir/lib/TableGen/Property.cpp
@@ -0,0 +1,86 @@
+//===- Property.cpp - Property wrapper class ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Property wrapper to simplify using TableGen Record defining a MLIR
+// Property.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/TableGen/Property.h"
+#include "mlir/TableGen/Format.h"
+#include "mlir/TableGen/Operator.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace mlir;
+using namespace mlir::tblgen;
+
+using llvm::DefInit;
+using llvm::Init;
+using llvm::Record;
+using llvm::StringInit;
+
+// Returns the initializer's value as string if the given TableGen initializer
+// is a code or string initializer. Returns the empty StringRef otherwise.
+static StringRef getValueAsString(const Init *init) {
+  if (const auto *str = dyn_cast<StringInit>(init))
+    return str->getValue().trim();
+  return {};
+}
+
+Property::Property(const Record *record) : def(record) {
+  assert((record->isSubClassOf("Property") || record->isSubClassOf("Attr")) &&
+         "must be subclass of TableGen 'Property' class");
+}
+
+Property::Property(const DefInit *init) : Property(init->getDef()) {}
+
+StringRef Property::getStorageType() const {
+  const auto *init = def->getValueInit("storageType");
+  auto type = getValueAsString(init);
+  if (type.empty())
+    return "Property";
+  return type;
+}
+
+StringRef Property::getInterfaceType() const {
+  const auto *init = def->getValueInit("interfaceType");
+  return getValueAsString(init);
+}
+
+StringRef Property::getConvertFromStorageCall() const {
+  const auto *init = def->getValueInit("convertFromStorage");
+  return getValueAsString(init);
+}
+
+StringRef Property::getAssignToStorageCall() const {
+  const auto *init = def->getValueInit("assignToStorage");
+  return getValueAsString(init);
+}
+
+StringRef Property::getConvertToAttributeCall() const {
+  const auto *init = def->getValueInit("convertToAttribute");
+  return getValueAsString(init);
+}
+
+StringRef Property::getConvertFromAttributeCall() const {
+  const auto *init = def->getValueInit("convertFromAttribute");
+  return getValueAsString(init);
+}
+
+StringRef Property::getHashPropertyCall() const {
+  return getValueAsString(def->getValueInit("hashProperty"));
+}
+
+bool Property::hasDefaultValue() const { return !getDefaultValue().empty(); }
+
+StringRef Property::getDefaultValue() const {
+  const auto *init = def->getValueInit("defaultValue");
+  return getValueAsString(init);
+}
+
+const llvm::Record &Property::getDef() const { return *def; }
diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt
index f2d95949a9740..147b1fdd30d27 100644
--- a/mlir/lib/Target/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt
@@ -39,6 +39,7 @@ add_mlir_translation_library(MLIRTargetLLVMIRExport
   MLIRLLVMDialect
   MLIRLLVMIRTransforms
   MLIRTranslateLib
+  MLIROpenMPDialect
   )
 
 add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 80cc92f97772f..84c39c07816af 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -12,11 +12,13 @@
 //===----------------------------------------------------------------------===//
 #include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
+#include "mlir/Transforms/RegionUtils.h"
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -24,6 +26,7 @@
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/Support/FileSystem.h"
 
 using namespace mlir;
 
@@ -1421,7 +1424,6 @@ static LogicalResult
 convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                      LLVM::ModuleTranslation &moduleTranslation) {
   unsigned numMapOperands;
-  bool mapperFunc = false;
   llvm::Value *ifCond = nullptr;
   int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF;
   SmallVector<Value> mapOperands;
@@ -1468,7 +1470,6 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
             numMapOperands = enterDataOp.getMapOperands().size();
             mapOperands = enterDataOp.getMapOperands();
             mapTypes = enterDataOp.getMapTypes();
-            mapperFunc = true;
             return success();
           })
           .Case([&](omp::ExitDataOp exitDataOp) {
@@ -1522,18 +1523,18 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
     // DataOp has only one region associated with it.
     auto &region = cast<omp::DataOp>(op).getRegion();
     builder.restoreIP(codeGenIP);
-    convertOmpOpRegions(region, "omp.data.region", builder, moduleTranslation,
-                        bodyGenStatus);
+    bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region", builder,
+                                            moduleTranslation);
   };
 
   if (isa<omp::DataOp>(op)) {
     builder.restoreIP(ompBuilder->createTargetData(
         ompLoc, builder.saveIP(), mapTypeFlags, mapNames, mapperAllocas,
-        mapperFunc, deviceID, ifCond, processMapOpCB, bodyCB));
+        /*IsBegin=*/false, deviceID, ifCond, processMapOpCB, bodyCB));
   } else {
     builder.restoreIP(ompBuilder->createTargetData(
         ompLoc, builder.saveIP(), mapTypeFlags, mapNames, mapperAllocas,
-        mapperFunc, deviceID, ifCond, processMapOpCB));
+        isa<omp::EnterDataOp>(op), deviceID, ifCond, processMapOpCB));
   }
 
   if (failed(processMapOpStatus))
@@ -1546,6 +1547,9 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
 /// be passed as flags to the frontend, otherwise they are set to default
 LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute,
                                LLVM::ModuleTranslation &moduleTranslation) {
+  if (!cast<mlir::ModuleOp>(op))
+    return failure();
+  
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
 
   ompBuilder->createGlobalFlag(
@@ -1573,6 +1577,107 @@ LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute,
   return success();
 }
 
+static bool getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo,
+                                     omp::TargetOp targetOp,
+                                     llvm::StringRef parentName = "") {
+  auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    targetOp.emitError("Unable to get unique ID for file");
+    return false;
+  }
+
+  uint64_t line = fileLoc.getLine();
+  targetInfo = llvm::TargetRegionEntryInfo(parentName, id.getDevice(),
+                                           id.getFile(), line);
+  return true;
+}
+
+static bool targetOpSupported(Operation &opInst) {
+  auto targetOp = cast<omp::TargetOp>(opInst);
+  if (targetOp.getIfExpr()) {
+    opInst.emitError("If clause not yet supported");
+    return false;
+  }
+
+  if (targetOp.getDevice()) {
+    opInst.emitError("Device clause not yet supported");
+    return false;
+  }
+
+  if (targetOp.getThreadLimit()) {
+    opInst.emitError("Thread limit clause not yet supported");
+    return false;
+  }
+
+  if (targetOp.getNowait()) {
+    opInst.emitError("Nowait clause not yet supported");
+    return false;
+  }
+
+  return true;
+}
+
+static LogicalResult
+convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
+                 LLVM::ModuleTranslation &moduleTranslation) {
+
+  if (!targetOpSupported(opInst))
+    return failure();
+
+  bool isDevice = false;
+  if (auto offloadMod = dyn_cast<mlir::omp::OffloadModuleInterface>(
+          opInst.getParentOfType<mlir::ModuleOp>().getOperation())) {
+    isDevice = offloadMod.getIsDevice();
+  }
+
+  if (isDevice) // TODO: Implement device codegen.
+    return success();
+
+  auto targetOp = cast<omp::TargetOp>(opInst);
+  auto &targetRegion = targetOp.getRegion();
+
+  llvm::SetVector<Value> operandSet;
+  getUsedValuesDefinedAbove(targetRegion, operandSet);
+
+  // Collect the input arguments.
+  llvm::SmallVector<llvm::Value *> inputs;
+  for (Value operand : operandSet)
+    inputs.push_back(moduleTranslation.lookupValue(operand));
+
+  LogicalResult bodyGenStatus = success();
+
+  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
+  auto bodyCB = [&](InsertPointTy allocaIP,
+                    InsertPointTy codeGenIP) -> InsertPointTy {
+    builder.restoreIP(codeGenIP);
+    llvm::BasicBlock *exitBlock = convertOmpOpRegions(
+        targetRegion, "omp.target", builder, moduleTranslation, bodyGenStatus);
+    builder.SetInsertPoint(exitBlock);
+    return builder.saveIP();
+  };
+
+  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+  StringRef parentName = opInst.getParentOfType<LLVM::LLVMFuncOp>().getName();
+  llvm::TargetRegionEntryInfo entryInfo;
+
+  if (!getTargetEntryUniqueInfo(entryInfo, targetOp, parentName))
+    return failure();
+
+  int32_t defaultValTeams = -1;
+  int32_t defaultValThreads = -1;
+
+  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTarget(
+      ompLoc, builder.saveIP(), entryInfo, defaultValTeams, defaultValThreads,
+      inputs, bodyCB));
+
+  return bodyGenStatus;
+}
+
 namespace {
 
 /// Implementation of the dialect interface that converts operations belonging
@@ -1713,6 +1818,9 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
       .Case<omp::DataOp, omp::EnterDataOp, omp::ExitDataOp>([&](auto op) {
         return convertOmpTargetData(op, builder, moduleTranslation);
       })
+      .Case([&](omp::TargetOp) {
+        return convertOmpTarget(*op, builder, moduleTranslation);
+      })
       .Default([&](Operation *inst) {
         return inst->emitError("unsupported OpenMP operation: ")
                << inst->getName();
diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp
index 000349091afce..a9f080ef0044f 100644
--- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp
@@ -23,7 +23,8 @@ struct LoopMetadataConversion {
   /// Converts this structs loop metadata node into a LoopAnnotationAttr.
   LoopAnnotationAttr convert();
 
-  LogicalResult initPropertyMap();
+  /// Initializes the shared state for the conversion member functions.
+  LogicalResult initConversionState();
 
   /// Helper function to get and erase a property.
   const llvm::MDNode *lookupAndEraseProperty(StringRef name);
@@ -53,7 +54,10 @@ struct LoopMetadataConversion {
   FailureOr<LoopPeeledAttr> convertPeeledAttr();
   FailureOr<LoopUnswitchAttr> convertUnswitchAttr();
   FailureOr<SmallVector<SymbolRefAttr>> convertParallelAccesses();
+  FusedLoc convertStartLoc();
+  FailureOr<FusedLoc> convertEndLoc();
 
+  llvm::SmallVector<llvm::DILocation *, 2> locations;
   llvm::StringMap<const llvm::MDNode *> propertyMap;
   const llvm::MDNode *node;
   Location loc;
@@ -62,16 +66,17 @@ struct LoopMetadataConversion {
 };
 } // namespace
 
-LogicalResult LoopMetadataConversion::initPropertyMap() {
+LogicalResult LoopMetadataConversion::initConversionState() {
   // Check if it's a valid node.
   if (node->getNumOperands() == 0 ||
       dyn_cast<llvm::MDNode>(node->getOperand(0)) != node)
     return emitWarning(loc) << "invalid loop node";
 
   for (const llvm::MDOperand &operand : llvm::drop_begin(node->operands())) {
-    // Skip over DILocations.
-    if (isa<llvm::DILocation>(operand))
+    if (auto *diLoc = dyn_cast<llvm::DILocation>(operand)) {
+      locations.push_back(diLoc);
       continue;
+    }
 
     auto *property = dyn_cast<llvm::MDNode>(operand);
     if (!property)
@@ -405,8 +410,25 @@ LoopMetadataConversion::convertParallelAccesses() {
   return refs;
 }
 
+FusedLoc LoopMetadataConversion::convertStartLoc() {
+  if (locations.empty())
+    return {};
+  return dyn_cast<FusedLoc>(
+      loopAnnotationImporter.moduleImport.translateLoc(locations[0]));
+}
+
+FailureOr<FusedLoc> LoopMetadataConversion::convertEndLoc() {
+  if (locations.size() < 2)
+    return FusedLoc();
+  if (locations.size() > 2)
+    return emitError(loc)
+           << "expected loop metadata to have at most two DILocations";
+  return dyn_cast<FusedLoc>(
+      loopAnnotationImporter.moduleImport.translateLoc(locations[1]));
+}
+
 LoopAnnotationAttr LoopMetadataConversion::convert() {
-  if (failed(initPropertyMap()))
+  if (failed(initConversionState()))
     return {};
 
   FailureOr<BoolAttr> disableNonForced =
@@ -433,10 +455,14 @@ LoopAnnotationAttr LoopMetadataConversion::convert() {
     return {};
   }
 
+  FailureOr<FusedLoc> startLoc = convertStartLoc();
+  FailureOr<FusedLoc> endLoc = convertEndLoc();
+
   return createIfNonNull<LoopAnnotationAttr>(
       ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr,
       unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, peeledAttr,
-      unswitchAttr, mustProgress, isVectorized, parallelAccesses);
+      unswitchAttr, mustProgress, isVectorized, parallelAccesses, startLoc,
+      endLoc);
 }
 
 LoopAnnotationAttr
diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h
index 5d69a63a21502..7dcfc4c750d98 100644
--- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h
+++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.h
@@ -26,7 +26,8 @@ namespace detail {
 /// AccessGroupMetadataOps.
 class LoopAnnotationImporter {
 public:
-  explicit LoopAnnotationImporter(OpBuilder &builder) : builder(builder) {}
+  LoopAnnotationImporter(ModuleImport &moduleImport, OpBuilder &builder)
+      : moduleImport(moduleImport), builder(builder) {}
   LoopAnnotationAttr translateLoopAnnotation(const llvm::MDNode *node,
                                              Location loc);
 
@@ -44,6 +45,9 @@ class LoopAnnotationImporter {
   FailureOr<SmallVector<SymbolRefAttr>>
   lookupAccessGroupAttrs(const llvm::MDNode *node) const;
 
+  /// The ModuleImport owning this instance.
+  ModuleImport &moduleImport;
+
 private:
   /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute.
   LoopAnnotationAttr lookupLoopMetadata(const llvm::MDNode *node) const {
diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
index 6abcf6964abed..8e6906a215325 100644
--- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LoopAnnotationTranslation.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
@@ -32,6 +33,7 @@ struct LoopAnnotationConversion {
   void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false);
   void convertI32Node(StringRef name, IntegerAttr attr);
   void convertFollowupNode(StringRef name, LoopAnnotationAttr attr);
+  void convertLocation(FusedLoc attr);
 
   /// Conversion functions for each for each loop annotation sub-attribute.
   void convertLoopOptions(LoopVectorizeAttr options);
@@ -186,12 +188,33 @@ void LoopAnnotationConversion::convertLoopOptions(LoopUnswitchAttr options) {
               options.getPartialDisable());
 }
 
-llvm::MDNode *LoopAnnotationConversion::convert() {
+void LoopAnnotationConversion::convertLocation(FusedLoc location) {
+  auto localScopeAttr =
+      location.getMetadata().dyn_cast_or_null<DILocalScopeAttr>();
+  if (!localScopeAttr)
+    return;
+  auto *localScope = dyn_cast<llvm::DILocalScope>(
+      loopAnnotationTranslation.moduleTranslation.translateDebugInfo(
+          localScopeAttr));
+  if (!localScope)
+    return;
+  const llvm::Metadata *loc =
+      loopAnnotationTranslation.moduleTranslation.translateLoc(location,
+                                                               localScope);
+  metadataNodes.push_back(const_cast<llvm::Metadata *>(loc));
+}
 
+llvm::MDNode *LoopAnnotationConversion::convert() {
   // Reserve operand 0 for loop id self reference.
   auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
   metadataNodes.push_back(dummy.get());
 
+  if (FusedLoc startLoc = attr.getStartLoc())
+    convertLocation(startLoc);
+
+  if (FusedLoc endLoc = attr.getEndLoc())
+    convertLocation(endLoc);
+
   addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced());
   addUnitNode("llvm.loop.mustprogress", attr.getMustProgress());
   // "isvectorized" is encoded as an i32 value.
diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h
index e663f64b061fc..0e016d9cbeef0 100644
--- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h
+++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.h
@@ -25,8 +25,10 @@ namespace detail {
 /// into a corresponding llvm::MDNodes.
 class LoopAnnotationTranslation {
 public:
-  LoopAnnotationTranslation(Operation *mlirModule, llvm::Module &llvmModule)
-      : mlirModule(mlirModule), llvmModule(llvmModule) {}
+  LoopAnnotationTranslation(ModuleTranslation &moduleTranslation,
+                            Operation *mlirModule, llvm::Module &llvmModule)
+      : moduleTranslation(moduleTranslation), mlirModule(mlirModule),
+        llvmModule(llvmModule) {}
 
   llvm::MDNode *translateLoopAnnotation(LoopAnnotationAttr attr, Operation *op);
 
@@ -43,6 +45,9 @@ class LoopAnnotationTranslation {
   /// referenced by the AccessGroupOpInterface or null if there are none.
   llvm::MDNode *getAccessGroups(AccessGroupOpInterface op) const;
 
+  /// The ModuleTranslation owning this instance.
+  ModuleTranslation &moduleTranslation;
+
 private:
   /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute.
   llvm::MDNode *lookupLoopMetadata(Attribute options) const {
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 4338f0314cdd8..8c5fe35a6f0dc 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -153,7 +153,7 @@ ModuleImport::ModuleImport(ModuleOp mlirModule,
       typeTranslator(*mlirModule->getContext()),
       debugImporter(std::make_unique<DebugImporter>(mlirModule)),
       loopAnnotationImporter(
-          std::make_unique<LoopAnnotationImporter>(builder)) {
+          std::make_unique<LoopAnnotationImporter>(*this, builder)) {
   builder.setInsertionPointToStart(mlirModule.getBody());
 }
 
@@ -1024,10 +1024,15 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
 }
 
 FailureOr<Value> ModuleImport::convertConstantExpr(llvm::Constant *constant) {
+  // Only call the function for constants that have not been translated before
+  // since it updates the constant insertion point assuming the converted
+  // constant has been introduced at the end of the constant section.
+  assert(!valueMapping.contains(constant) &&
+         "expected constant has not been converted before");
   assert(constantInsertionBlock &&
          "expected the constant insertion block to be non-null");
 
-  // Insert the constant after the last one or at the start or the entry block.
+  // Insert the constant after the last one or at the start of the entry block.
   OpBuilder::InsertionGuard guard(builder);
   if (!constantInsertionOp)
     builder.setInsertionPointToStart(constantInsertionBlock);
@@ -1273,7 +1278,7 @@ LogicalResult ModuleImport::convertInstruction(llvm::Instruction *inst) {
     SmallVector<Value> operands;
     operands.reserve(lpInst->getNumClauses());
     for (auto i : llvm::seq<unsigned>(0, lpInst->getNumClauses())) {
-      FailureOr<Value> operand = convertConstantExpr(lpInst->getClause(i));
+      FailureOr<Value> operand = convertValue(lpInst->getClause(i));
       if (failed(operand))
         return failure();
       operands.push_back(*operand);
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 66891afaf5d3b..f8854d75f972b 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -21,6 +21,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
 #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
@@ -443,12 +444,13 @@ ModuleTranslation::ModuleTranslation(Operation *module,
       debugTranslation(
           std::make_unique<DebugTranslation>(module, *this->llvmModule)),
       loopAnnotationTranslation(std::make_unique<LoopAnnotationTranslation>(
-          module, *this->llvmModule)),
+          *this, module, *this->llvmModule)),
       typeTranslator(this->llvmModule->getContext()),
       iface(module->getContext()) {
   assert(satisfiesLLVMModule(mlirModule) &&
          "mlirModule should honor LLVM's module semantics.");
 }
+
 ModuleTranslation::~ModuleTranslation() {
   if (ompBuilder)
     ompBuilder->finalize();
@@ -769,6 +771,10 @@ LogicalResult ModuleTranslation::convertGlobals() {
     }
   }
 
+  for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>())
+    if (failed(convertDialectAttributes(op)))
+      return failure();
+
   return success();
 }
 
@@ -1250,6 +1256,26 @@ SmallVector<llvm::Value *> ModuleTranslation::lookupValues(ValueRange values) {
   return remapped;
 }
 
+llvm::OpenMPIRBuilder *ModuleTranslation::getOpenMPBuilder() {
+  if (!ompBuilder) {
+    ompBuilder = std::make_unique<llvm::OpenMPIRBuilder>(*llvmModule);
+    ompBuilder->initialize();
+
+    bool isDevice = false;
+    if (auto offloadMod =
+            dyn_cast<mlir::omp::OffloadModuleInterface>(mlirModule))
+      isDevice = offloadMod.getIsDevice();
+
+    // TODO: set the flags when available
+    llvm::OpenMPIRBuilderConfig Config(
+        isDevice, /* IsTargetCodegen */ false,
+        /* HasRequiresUnifiedSharedMemory */ false,
+        /* OpenMPOffloadMandatory */ false);
+    ompBuilder->setConfig(Config);
+  }
+  return ompBuilder.get();
+}
+
 const llvm::DILocation *
 ModuleTranslation::translateLoc(Location loc, llvm::DILocalScope *scope) {
   return debugTranslation->translateLoc(loc, scope);
diff --git a/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp b/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
index 8597423946c79..5e4dc07c4c3f1 100644
--- a/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
+++ b/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
@@ -885,7 +885,8 @@ MLIRDocument::convertToBytecode() {
 
     std::string rawBytecodeBuffer;
     llvm::raw_string_ostream os(rawBytecodeBuffer);
-    writeBytecodeToFile(&parsedIR.front(), os, writerConfig);
+    // No desired bytecode version set, so no need to check for error.
+    (void)writeBytecodeToFile(&parsedIR.front(), os, writerConfig);
     result.output = llvm::encodeBase64(rawBytecodeBuffer);
   }
   return result;
diff --git a/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp b/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp
index b9e65b1b8e622..2aad687e41957 100644
--- a/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp
+++ b/mlir/lib/Tools/mlir-opt/MlirOptMain.cpp
@@ -13,7 +13,9 @@
 
 #include "mlir/Tools/mlir-opt/MlirOptMain.h"
 #include "mlir/Bytecode/BytecodeWriter.h"
+#include "mlir/Debug/CLOptionsSetup.h"
 #include "mlir/Debug/Counter.h"
+#include "mlir/Debug/DebuggerExecutionContextHook.h"
 #include "mlir/Debug/ExecutionContext.h"
 #include "mlir/Debug/Observers/ActionLogging.h"
 #include "mlir/Dialect/IRDL/IR/IRDL.h"
@@ -50,6 +52,22 @@ using namespace mlir;
 using namespace llvm;
 
 namespace {
+class BytecodeVersionParser : public cl::parser<std::optional<int64_t>> {
+public:
+  BytecodeVersionParser(cl::Option &O)
+      : cl::parser<std::optional<int64_t>>(O) {}
+
+  bool parse(cl::Option &O, StringRef /*argName*/, StringRef arg,
+             std::optional<int64_t> &v) {
+    long long w;
+    if (getAsSignedInteger(arg, 10, w))
+      return O.error("Invalid argument '" + arg +
+                     "', only integer is supported.");
+    v = w;
+    return false;
+  }
+};
+
 /// This class is intended to manage the handling of command line options for
 /// creating a *-opt config. This is a singleton.
 struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
@@ -72,49 +90,32 @@ struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
         "emit-bytecode", cl::desc("Emit bytecode when generating output"),
         cl::location(emitBytecodeFlag), cl::init(false));
 
+    static cl::opt<std::optional<int64_t>, /*ExternalStorage=*/true,
+                   BytecodeVersionParser>
+        bytecodeVersion(
+            "emit-bytecode-version",
+            cl::desc("Use specified bytecode when generating output"),
+            cl::location(emitBytecodeVersion), cl::init(std::nullopt));
+
     static cl::opt<std::string, /*ExternalStorage=*/true> irdlFile(
         "irdl-file",
         cl::desc("IRDL file to register before processing the input"),
         cl::location(irdlFileFlag), cl::init(""), cl::value_desc("filename"));
 
+    static cl::opt<bool, /*ExternalStorage=*/true> enableDebuggerHook(
+        "mlir-enable-debugger-hook",
+        cl::desc("Enable Debugger hook for debugging MLIR Actions"),
+        cl::location(enableDebuggerActionHookFlag), cl::init(false));
+
     static cl::opt<bool, /*ExternalStorage=*/true> explicitModule(
         "no-implicit-module",
         cl::desc("Disable implicit addition of a top-level module op during "
                  "parsing"),
         cl::location(useExplicitModuleFlag), cl::init(false));
 
-    static cl::opt<std::string, /*ExternalStorage=*/true> logActionsTo{
-        "log-actions-to",
-        cl::desc("Log action execution to a file, or stderr if "
-                 " '-' is passed"),
-        cl::location(logActionsToFlag)};
-
-    static cl::list<std::string> logActionLocationFilter(
-        "log-mlir-actions-filter",
-        cl::desc(
-            "Comma separated list of locations to filter actions from logging"),
-        cl::CommaSeparated,
-        cl::cb<void, std::string>([&](const std::string &location) {
-          static bool register_once = [&] {
-            addLogActionLocFilter(&locBreakpointManager);
-            return true;
-          }();
-          (void)register_once;
-          static std::vector<std::string> locations;
-          locations.push_back(location);
-          StringRef locStr = locations.back();
-
-          // Parse the individual location filters and set the breakpoints.
-          auto diag = [](Twine msg) { llvm::errs() << msg << "\n"; };
-          auto locBreakpoint =
-              tracing::FileLineColLocBreakpoint::parseFromString(locStr, diag);
-          if (failed(locBreakpoint)) {
-            llvm::errs() << "Invalid location filter: " << locStr << "\n";
-            exit(1);
-          }
-          auto [file, line, col] = *locBreakpoint;
-          locBreakpointManager.addBreakpoint(file, line, col);
-        }));
+    static cl::opt<bool, /*ExternalStorage=*/true> runReproducer(
+        "run-reproducer", cl::desc("Run the pipeline stored in the reproducer"),
+        cl::location(runReproducerFlag), cl::init(false));
 
     static cl::opt<bool, /*ExternalStorage=*/true> showDialects(
         "show-dialects",
@@ -165,9 +166,6 @@ struct MlirOptMainConfigCLOptions : public MlirOptMainConfig {
   /// Pointer to static dialectPlugins variable in constructor, needed by
   /// setDialectPluginsCallback(DialectRegistry&).
   cl::list<std::string> *dialectPlugins = nullptr;
-
-  /// The breakpoint manager for the log action location filter.
-  tracing::FileLineColLocBreakpointManager locBreakpointManager;
 };
 } // namespace
 
@@ -175,9 +173,11 @@ ManagedStatic<MlirOptMainConfigCLOptions> clOptionsConfig;
 
 void MlirOptMainConfig::registerCLOptions(DialectRegistry &registry) {
   clOptionsConfig->setDialectPluginsCallback(registry);
+  tracing::DebugConfig::registerCLOptions();
 }
 
 MlirOptMainConfig MlirOptMainConfig::createFromCLOptions() {
+  clOptionsConfig->setDebugConfig(tracing::DebugConfig::createFromCLOptions());
   return *clOptionsConfig;
 }
 
@@ -213,45 +213,6 @@ void MlirOptMainConfigCLOptions::setDialectPluginsCallback(
   });
 }
 
-/// Set the ExecutionContext on the context and handle the observers.
-class InstallDebugHandler {
-public:
-  InstallDebugHandler(MLIRContext &context, const MlirOptMainConfig &config) {
-    if (config.getLogActionsTo().empty()) {
-      if (tracing::DebugCounter::isActivated())
-        context.registerActionHandler(tracing::DebugCounter());
-      return;
-    }
-    if (tracing::DebugCounter::isActivated())
-      emitError(UnknownLoc::get(&context),
-                "Debug counters are incompatible with --log-actions-to option "
-                "and are disabled");
-    std::string errorMessage;
-    logActionsFile = openOutputFile(config.getLogActionsTo(), &errorMessage);
-    if (!logActionsFile) {
-      emitError(UnknownLoc::get(&context),
-                "Opening file for --log-actions-to failed: ")
-          << errorMessage << "\n";
-      return;
-    }
-    logActionsFile->keep();
-    raw_fd_ostream &logActionsStream = logActionsFile->os();
-    actionLogger = std::make_unique<tracing::ActionLogger>(logActionsStream);
-    for (const auto *locationBreakpoint : config.getLogActionsLocFilters())
-      actionLogger->addBreakpointManager(locationBreakpoint);
-
-    executionContext.registerObserver(actionLogger.get());
-    context.registerActionHandler(executionContext);
-  }
-
-private:
-  std::unique_ptr<llvm::ToolOutputFile> logActionsFile;
-  std::unique_ptr<tracing::ActionLogger> actionLogger;
-  std::vector<std::unique_ptr<tracing::FileLineColLocBreakpoint>>
-      locationBreakpoints;
-  tracing::ExecutionContext executionContext;
-};
-
 /// Perform the actions on the input file indicated by the command line flags
 /// within the specified context.
 ///
@@ -279,7 +240,8 @@ performActions(raw_ostream &os,
   FallbackAsmResourceMap fallbackResourceMap;
   ParserConfig parseConfig(context, /*verifyAfterParse=*/true,
                            &fallbackResourceMap);
-  reproOptions.attachResourceParser(parseConfig);
+  if (config.shouldRunReproducer())
+    reproOptions.attachResourceParser(parseConfig);
 
   // Parse the input file and reset the context threading state.
   TimingScope parserTiming = timing.nest("Parser");
@@ -296,7 +258,9 @@ performActions(raw_ostream &os,
   if (failed(applyPassManagerCLOptions(pm)))
     return failure();
   pm.enableTiming(timing);
-  if (failed(reproOptions.apply(pm)) || failed(config.setupPassPipeline(pm)))
+  if (config.shouldRunReproducer() && failed(reproOptions.apply(pm)))
+    return failure();
+  if (failed(config.setupPassPipeline(pm)))
     return failure();
 
   // Run the pipeline.
@@ -307,13 +271,18 @@ performActions(raw_ostream &os,
   TimingScope outputTiming = timing.nest("Output");
   if (config.shouldEmitBytecode()) {
     BytecodeWriterConfig writerConfig(fallbackResourceMap);
-    writeBytecodeToFile(op.get(), os, writerConfig);
-  } else {
-    AsmState asmState(op.get(), OpPrintingFlags(), /*locationMap=*/nullptr,
-                      &fallbackResourceMap);
-    op.get()->print(os, asmState);
-    os << '\n';
+    if (auto v = config.bytecodeVersionToEmit())
+      writerConfig.setDesiredBytecodeVersion(*v);
+    return writeBytecodeToFile(op.get(), os, writerConfig);
   }
+
+  if (config.bytecodeVersionToEmit().has_value())
+    return emitError(UnknownLoc::get(pm.getContext()))
+           << "bytecode version while not emitting bytecode";
+  AsmState asmState(op.get(), OpPrintingFlags(), /*locationMap=*/nullptr,
+                    &fallbackResourceMap);
+  op.get()->print(os, asmState);
+  os << '\n';
   return success();
 }
 
@@ -366,13 +335,12 @@ static LogicalResult processBuffer(raw_ostream &os,
     return failure();
 
   // Parse the input file.
-  if (config.shouldPreloadDialectsInContext())
-    context.loadAllAvailableDialects();
   context.allowUnregisteredDialects(config.shouldAllowUnregisteredDialects());
   if (config.shouldVerifyDiagnostics())
     context.printOpOnDiagnostic(false);
 
-  InstallDebugHandler installDebugHandler(context, config);
+  tracing::InstallDebugHandler installDebugHandler(context,
+                                                   config.getDebugConfig());
 
   // If we are in verify diagnostics mode then we have a lot of work to do,
   // otherwise just perform the actions without worrying about it.
@@ -427,48 +395,8 @@ LogicalResult mlir::MlirOptMain(llvm::raw_ostream &outputStream,
                                /*insertMarkerInOutput=*/true);
 }
 
-LogicalResult mlir::MlirOptMain(raw_ostream &outputStream,
-                                std::unique_ptr<MemoryBuffer> buffer,
-                                PassPipelineFn passManagerSetupFn,
-                                DialectRegistry &registry, bool splitInputFile,
-                                bool verifyDiagnostics, bool verifyPasses,
-                                bool allowUnregisteredDialects,
-                                bool preloadDialectsInContext,
-                                bool emitBytecode, bool explicitModule) {
-  return MlirOptMain(outputStream, std::move(buffer), registry,
-                     MlirOptMainConfig{}
-                         .splitInputFile(splitInputFile)
-                         .verifyDiagnostics(verifyDiagnostics)
-                         .verifyPasses(verifyPasses)
-                         .allowUnregisteredDialects(allowUnregisteredDialects)
-                         .preloadDialectsInContext(preloadDialectsInContext)
-                         .emitBytecode(emitBytecode)
-                         .useExplicitModule(explicitModule)
-                         .setPassPipelineSetupFn(passManagerSetupFn));
-}
-
-LogicalResult mlir::MlirOptMain(
-    raw_ostream &outputStream, std::unique_ptr<MemoryBuffer> buffer,
-    const PassPipelineCLParser &passPipeline, DialectRegistry &registry,
-    bool splitInputFile, bool verifyDiagnostics, bool verifyPasses,
-    bool allowUnregisteredDialects, bool preloadDialectsInContext,
-    bool emitBytecode, bool explicitModule, bool dumpPassPipeline) {
-  return MlirOptMain(outputStream, std::move(buffer), registry,
-                     MlirOptMainConfig{}
-                         .splitInputFile(splitInputFile)
-                         .verifyDiagnostics(verifyDiagnostics)
-                         .verifyPasses(verifyPasses)
-                         .allowUnregisteredDialects(allowUnregisteredDialects)
-                         .preloadDialectsInContext(preloadDialectsInContext)
-                         .emitBytecode(emitBytecode)
-                         .useExplicitModule(explicitModule)
-                         .dumpPassPipeline(dumpPassPipeline)
-                         .setPassPipelineParser(passPipeline));
-}
-
 LogicalResult mlir::MlirOptMain(int argc, char **argv, llvm::StringRef toolName,
-                                DialectRegistry &registry,
-                                bool preloadDialectsInContext) {
+                                DialectRegistry &registry) {
   static cl::opt<std::string> inputFilename(
       cl::Positional, cl::desc("<input file>"), cl::init("-"));
 
@@ -496,7 +424,6 @@ LogicalResult mlir::MlirOptMain(int argc, char **argv, llvm::StringRef toolName,
   // Parse pass names in main to ensure static initialization completed.
   cl::ParseCommandLineOptions(argc, argv, helpHeader);
   MlirOptMainConfig config = MlirOptMainConfig::createFromCLOptions();
-  config.preloadDialectsInContext(preloadDialectsInContext);
 
   // When reading from stdin and the input is a tty, it is often a user mistake
   // and the process "appears to be stuck". Print a message to let the user know
diff --git a/mlir/lib/Transforms/CMakeLists.txt b/mlir/lib/Transforms/CMakeLists.txt
index 4929a20847a78..7b4fb4d6df881 100644
--- a/mlir/lib/Transforms/CMakeLists.txt
+++ b/mlir/lib/Transforms/CMakeLists.txt
@@ -8,6 +8,7 @@ add_mlir_library(MLIRTransforms
   Inliner.cpp
   LocationSnapshot.cpp
   LoopInvariantCodeMotion.cpp
+  Mem2Reg.cpp
   OpStats.cpp
   PrintIR.cpp
   SCCP.cpp
@@ -27,6 +28,7 @@ add_mlir_library(MLIRTransforms
   MLIRAnalysis
   MLIRCopyOpInterface
   MLIRLoopLikeInterface
+  MLIRMem2RegInterfaces
   MLIRPass
   MLIRRuntimeVerifiableOpInterface
   MLIRSideEffectInterfaces
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
new file mode 100644
index 0000000000000..5b5728385a160
--- /dev/null
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -0,0 +1,564 @@
+//===- Mem2Reg.cpp - Promotes memory slots into values ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Transforms/Mem2Reg.h"
+#include "mlir/Analysis/SliceAnalysis.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Transforms/Passes.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/GenericIteratedDominanceFrontier.h"
+
+namespace mlir {
+#define GEN_PASS_DEF_MEM2REG
+#include "mlir/Transforms/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+/// mem2reg
+///
+/// This pass turns unnecessary uses of automatically allocated memory slots
+/// into direct Value-based operations. For example, it will simplify storing a
+/// constant in a memory slot to immediately load it to a direct use of that
+/// constant. In other words, given a memory slot addressed by a non-aliased
+/// "pointer" Value, mem2reg removes all the uses of that pointer.
+///
+/// Within a block, this is done by following the chain of stores and loads of
+/// the slot and replacing the results of loads with the values previously
+/// stored. If a load happens before any other store, a poison value is used
+/// instead.
+///
+/// Control flow can create situations where a load could be replaced by
+/// multiple possible stores depending on the control flow path taken. As a
+/// result, this pass must introduce new block arguments in some blocks to
+/// accomodate for the multiple possible definitions. Each predecessor will
+/// populate the block argument with the definition reached at its end. With
+/// this, the value stored can be well defined at block boundaries, allowing
+/// the propagation of replacement through blocks.
+///
+/// This pass computes this transformation in four main steps:
+/// - A first step computes the list of operations that transitively use the
+/// memory slot we would like to promote. The purpose of this phase is to
+/// identify which uses must be removed to promote the slot, either by rewiring
+/// the user or deleting it. Naturally, direct uses of the slot must be removed.
+/// Sometimes additional uses must also be removed: this is notably the case
+/// when a direct user of the slot cannot rewire its use and must delete itself,
+/// and thus must make its users no longer use it. If any of those uses cannot
+/// be removed by their users in any way, promotion cannot continue: this is
+/// decided at this step.
+/// - A second step computes the list of blocks where a block argument will be
+/// needed ("merge points") without mutating the IR. These blocks are the blocks
+/// leading to a definition clash between two predecessors. Such blocks happen
+/// to be the Iterated Dominance Frontier (IDF) of the set of blocks containing
+/// a store, as they represent the point where a clear defining dominator stops
+/// existing. Computing this information in advance allows making sure the
+/// terminators that will forward values are capable of doing so (inability to
+/// do so aborts promotion at this step).
+/// - A third step computes the reaching definition of the memory slot at each
+/// blocking user. This is the core of the mem2reg algorithm, also known as
+/// load-store forwarding. This analyses loads and stores and propagates which
+/// value must be stored in the slot at each blocking user.  This is achieved by
+/// doing a depth-first walk of the dominator tree of the function. This is
+/// sufficient because the reaching definition at the beginning of a block is
+/// either its new block argument if it is a merge block, or the definition
+/// reaching the end of its immediate dominator (parent in the dominator tree).
+/// We can therefore propagate this information down the dominator tree to
+/// proceed with renaming within blocks.
+/// - The final fourth step uses the reaching definition to remove blocking uses
+/// in topological order.
+///
+/// The two first steps do not mutate IR because promotion can still be aborted
+/// at this point. Once the two last steps are reached, promotion is guaranteed
+/// to succeed, allowing to start mutating IR.
+///
+/// For further reading, chapter three of SSA-based Compiler Design [1]
+/// showcases SSA construction, where mem2reg is an adaptation of the same
+/// process.
+///
+/// [1]: Rastello F. & Bouchez Tichadou F., SSA-based Compiler Design (2022),
+///      Springer.
+
+namespace {
+
+/// The SlotPromoter handles the state of promoting a memory slot. It wraps a
+/// slot and its associated allocator, along with analysis results related to
+/// the slot.
+class SlotPromoter {
+public:
+  SlotPromoter(MemorySlot slot, PromotableAllocationOpInterface allocator,
+               OpBuilder &builder, DominanceInfo &dominance);
+
+  /// Prepare data for the promotion of the slot while checking if it can be
+  /// promoted. Succeeds if the slot can be promoted. This method does not
+  /// mutate IR.
+  LogicalResult prepareSlotPromotion();
+
+  /// Actually promotes the slot by mutating IR. This method must only be
+  /// called after a successful call to `SlotPromoter::prepareSlotPromotion`.
+  /// Promoting a slot does not invalidate the preparation of other slots.
+  void promoteSlot();
+
+private:
+  /// This is the first step of the promotion algorithm.
+  /// Computes the transitive uses of the slot that block promotion. This finds
+  /// uses that would block the promotion, checks that the operation has a
+  /// solution to remove the blocking use, and potentially forwards the analysis
+  /// if the operation needs further blocking uses resolved to resolve its own
+  /// uses (typically, removing its users because it will delete itself to
+  /// resolve its own blocking uses). This will fail if one of the transitive
+  /// users cannot remove a requested use, and should prevent promotion.
+  LogicalResult computeBlockingUses();
+
+  /// Computes in which blocks the value stored in the slot is actually used,
+  /// meaning blocks leading to a load. This method uses `definingBlocks`, the
+  /// set of blocks containing a store to the slot (defining the value of the
+  /// slot).
+  SmallPtrSet<Block *, 16>
+  computeSlotLiveIn(SmallPtrSetImpl<Block *> &definingBlocks);
+
+  /// This is the second step of the promotion algorithm.
+  /// Computes the points in which multiple re-definitions of the slot's value
+  /// (stores) may conflict.
+  void computeMergePoints();
+
+  /// Ensures predecessors of merge points can properly provide their current
+  /// definition of the value stored in the slot to the merge point. This can
+  /// notably be an issue if the terminator used does not have the ability to
+  /// forward values through block operands.
+  bool areMergePointsUsable();
+
+  /// Computes the reaching definition for all the operations that require
+  /// promotion. `reachingDef` is the value the slot should contain at the
+  /// beginning of the block. This method returns the reached definition at the
+  /// end of the block.
+  Value computeReachingDefInBlock(Block *block, Value reachingDef);
+
+  /// This is the third step of the promotion algorithm.
+  /// Computes the reaching definition for all the operations that require
+  /// promotion. `reachingDef` corresponds to the initial value the
+  /// slot will contain before any write, typically a poison value.
+  void computeReachingDefInRegion(Region *region, Value reachingDef);
+
+  /// This is the fourth step of the promotion algorithm.
+  /// Removes the blocking uses of the slot, in topological order.
+  void removeBlockingUses();
+
+  /// Lazily-constructed default value representing the content of the slot when
+  /// no store has been executed. This function may mutate IR.
+  Value getLazyDefaultValue();
+
+  MemorySlot slot;
+  PromotableAllocationOpInterface allocator;
+  OpBuilder &builder;
+  /// Potentially non-initialized default value. Use `lazyDefaultValue` to
+  /// initialize it on demand.
+  Value defaultValue;
+  /// Blocks where multiple definitions of the slot value clash.
+  SmallPtrSet<Block *, 8> mergePoints;
+  /// Contains, for each operation, which uses must be eliminated by promotion.
+  /// This is a DAG structure because an operation that must eliminate some of
+  /// its uses always comes from a request from an operation that must
+  /// eliminate some of its own uses.
+  DenseMap<Operation *, SmallPtrSet<OpOperand *, 4>> userToBlockingUses;
+  /// Contains the reaching definition at this operation. Reaching definitions
+  /// are only computed for promotable memory operations with blocking uses.
+  DenseMap<PromotableMemOpInterface, Value> reachingDefs;
+  DominanceInfo &dominance;
+};
+
+} // namespace
+
+SlotPromoter::SlotPromoter(MemorySlot slot,
+                           PromotableAllocationOpInterface allocator,
+                           OpBuilder &builder, DominanceInfo &dominance)
+    : slot(slot), allocator(allocator), builder(builder), dominance(dominance) {
+#ifndef NDEBUG
+  auto isResultOrNewBlockArgument = [&]() {
+    if (BlockArgument arg = slot.ptr.dyn_cast<BlockArgument>())
+      return arg.getOwner()->getParentOp() == allocator;
+    return slot.ptr.getDefiningOp() == allocator;
+  };
+
+  assert(isResultOrNewBlockArgument() &&
+         "a slot must be a result of the allocator or an argument of the child "
+         "regions of the allocator");
+#endif // NDEBUG
+}
+
+Value SlotPromoter::getLazyDefaultValue() {
+  if (defaultValue)
+    return defaultValue;
+
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToStart(slot.ptr.getParentBlock());
+  return defaultValue = allocator.getDefaultValue(slot, builder);
+}
+
+LogicalResult SlotPromoter::computeBlockingUses() {
+  // The promotion of an operation may require the promotion of further
+  // operations (typically, removing operations that use an operation that must
+  // delete itself). We thus need to start from the use of the slot pointer and
+  // propagate further requests through the forward slice.
+
+  // First insert that all immediate users of the slot pointer must no longer
+  // use it.
+  for (OpOperand &use : slot.ptr.getUses()) {
+    SmallPtrSet<OpOperand *, 4> &blockingUses =
+        userToBlockingUses.getOrInsertDefault(use.getOwner());
+    blockingUses.insert(&use);
+  }
+
+  // Then, propagate the requirements for the removal of uses. The
+  // topologically-sorted forward slice allows for all blocking uses of an
+  // operation to have been computed before we reach it. Operations are
+  // traversed in topological order of their uses, starting from the slot
+  // pointer.
+  SetVector<Operation *> forwardSlice;
+  mlir::getForwardSlice(slot.ptr, &forwardSlice);
+  for (Operation *user : forwardSlice) {
+    // If the next operation has no blocking uses, everything is fine.
+    if (!userToBlockingUses.contains(user))
+      continue;
+
+    SmallPtrSet<OpOperand *, 4> &blockingUses = userToBlockingUses[user];
+
+    SmallVector<OpOperand *> newBlockingUses;
+    // If the operation decides it cannot deal with removing the blocking uses,
+    // promotion must fail.
+    if (auto promotable = dyn_cast<PromotableOpInterface>(user)) {
+      if (!promotable.canUsesBeRemoved(slot, blockingUses, newBlockingUses))
+        return failure();
+    } else if (auto promotable = dyn_cast<PromotableMemOpInterface>(user)) {
+      if (!promotable.canUsesBeRemoved(slot, blockingUses, newBlockingUses))
+        return failure();
+    } else {
+      // An operation that has blocking uses must be promoted. If it is not
+      // promotable, promotion must fail.
+      return failure();
+    }
+
+    // Then, register any new blocking uses for coming operations.
+    for (OpOperand *blockingUse : newBlockingUses) {
+      assert(llvm::is_contained(user->getResults(), blockingUse->get()));
+
+      SmallPtrSetImpl<OpOperand *> &newUserBlockingUseSet =
+          userToBlockingUses.getOrInsertDefault(blockingUse->getOwner());
+      newUserBlockingUseSet.insert(blockingUse);
+    }
+  }
+
+  // Because this pass currently only supports analysing the parent region of
+  // the slot pointer, if a promotable memory op that needs promotion is
+  // outside of this region, promotion must fail because it will be impossible
+  // to provide a valid `reachingDef` for it.
+  for (auto &[toPromote, _] : userToBlockingUses)
+    if (isa<PromotableMemOpInterface>(toPromote) &&
+        toPromote->getParentRegion() != slot.ptr.getParentRegion())
+      return failure();
+
+  return success();
+}
+
+SmallPtrSet<Block *, 16>
+SlotPromoter::computeSlotLiveIn(SmallPtrSetImpl<Block *> &definingBlocks) {
+  SmallPtrSet<Block *, 16> liveIn;
+
+  // The worklist contains blocks in which it is known that the slot value is
+  // live-in. The further blocks where this value is live-in will be inferred
+  // from these.
+  SmallVector<Block *> liveInWorkList;
+
+  // Blocks with a load before any other store to the slot are the starting
+  // points of the analysis. The slot value is definitely live-in in those
+  // blocks.
+  SmallPtrSet<Block *, 16> visited;
+  for (Operation *user : slot.ptr.getUsers()) {
+    if (visited.contains(user->getBlock()))
+      continue;
+    visited.insert(user->getBlock());
+
+    for (Operation &op : user->getBlock()->getOperations()) {
+      if (auto memOp = dyn_cast<PromotableMemOpInterface>(op)) {
+        // If this operation loads the slot, it is loading from it before
+        // ever writing to it, so the value is live-in in this block.
+        if (memOp.loadsFrom(slot)) {
+          liveInWorkList.push_back(user->getBlock());
+          break;
+        }
+
+        // If we store to the slot, further loads will see that value.
+        // Because we did not meet any load before, the value is not live-in.
+        if (memOp.getStored(slot))
+          break;
+      }
+    }
+  }
+
+  // The information is then propagated to the predecessors until a def site
+  // (store) is found.
+  while (!liveInWorkList.empty()) {
+    Block *liveInBlock = liveInWorkList.pop_back_val();
+
+    if (!liveIn.insert(liveInBlock).second)
+      continue;
+
+    // If a predecessor is a defining block, either:
+    // - It has a load before its first store, in which case it is live-in but
+    // has already been processed in the initialisation step.
+    // - It has a store before any load, in which case it is not live-in.
+    // We can thus at this stage insert to the worklist only predecessors that
+    // are not defining blocks.
+    for (Block *pred : liveInBlock->getPredecessors())
+      if (!definingBlocks.contains(pred))
+        liveInWorkList.push_back(pred);
+  }
+
+  return liveIn;
+}
+
+using IDFCalculator = llvm::IDFCalculatorBase<Block, false>;
+void SlotPromoter::computeMergePoints() {
+  if (slot.ptr.getParentRegion()->hasOneBlock())
+    return;
+
+  IDFCalculator idfCalculator(dominance.getDomTree(slot.ptr.getParentRegion()));
+
+  SmallPtrSet<Block *, 16> definingBlocks;
+  for (Operation *user : slot.ptr.getUsers())
+    if (auto storeOp = dyn_cast<PromotableMemOpInterface>(user))
+      if (storeOp.getStored(slot))
+        definingBlocks.insert(user->getBlock());
+
+  idfCalculator.setDefiningBlocks(definingBlocks);
+
+  SmallPtrSet<Block *, 16> liveIn = computeSlotLiveIn(definingBlocks);
+  idfCalculator.setLiveInBlocks(liveIn);
+
+  SmallVector<Block *> mergePointsVec;
+  idfCalculator.calculate(mergePointsVec);
+
+  mergePoints.insert(mergePointsVec.begin(), mergePointsVec.end());
+}
+
+bool SlotPromoter::areMergePointsUsable() {
+  for (Block *mergePoint : mergePoints)
+    for (Block *pred : mergePoint->getPredecessors())
+      if (!isa<BranchOpInterface>(pred->getTerminator()))
+        return false;
+
+  return true;
+}
+
+Value SlotPromoter::computeReachingDefInBlock(Block *block, Value reachingDef) {
+  for (Operation &op : block->getOperations()) {
+    if (auto memOp = dyn_cast<PromotableMemOpInterface>(op)) {
+      if (userToBlockingUses.contains(memOp))
+        reachingDefs.insert({memOp, reachingDef});
+
+      if (Value stored = memOp.getStored(slot))
+        reachingDef = stored;
+    }
+  }
+
+  return reachingDef;
+}
+
+void SlotPromoter::computeReachingDefInRegion(Region *region,
+                                              Value reachingDef) {
+  if (region->hasOneBlock()) {
+    computeReachingDefInBlock(&region->front(), reachingDef);
+    return;
+  }
+
+  struct DfsJob {
+    llvm::DomTreeNodeBase<Block> *block;
+    Value reachingDef;
+  };
+
+  SmallVector<DfsJob> dfsStack;
+
+  auto &domTree = dominance.getDomTree(slot.ptr.getParentRegion());
+
+  dfsStack.emplace_back<DfsJob>(
+      {domTree.getNode(&region->front()), reachingDef});
+
+  while (!dfsStack.empty()) {
+    DfsJob job = dfsStack.pop_back_val();
+    Block *block = job.block->getBlock();
+
+    if (mergePoints.contains(block)) {
+      BlockArgument blockArgument =
+          block->addArgument(slot.elemType, slot.ptr.getLoc());
+      builder.setInsertionPointToStart(block);
+      allocator.handleBlockArgument(slot, blockArgument, builder);
+      job.reachingDef = blockArgument;
+    }
+
+    job.reachingDef = computeReachingDefInBlock(block, job.reachingDef);
+
+    if (auto terminator = dyn_cast<BranchOpInterface>(block->getTerminator())) {
+      for (BlockOperand &blockOperand : terminator->getBlockOperands()) {
+        if (mergePoints.contains(blockOperand.get())) {
+          if (!job.reachingDef)
+            job.reachingDef = getLazyDefaultValue();
+          terminator.getSuccessorOperands(blockOperand.getOperandNumber())
+              .append(job.reachingDef);
+        }
+      }
+    }
+
+    for (auto *child : job.block->children())
+      dfsStack.emplace_back<DfsJob>({child, job.reachingDef});
+  }
+}
+
+void SlotPromoter::removeBlockingUses() {
+  llvm::SetVector<Operation *> usersToRemoveUses;
+  for (auto &user : llvm::make_first_range(userToBlockingUses))
+    usersToRemoveUses.insert(user);
+  SetVector<Operation *> sortedUsersToRemoveUses =
+      mlir::topologicalSort(usersToRemoveUses);
+
+  llvm::SmallVector<Operation *> toErase;
+  for (Operation *toPromote : llvm::reverse(sortedUsersToRemoveUses)) {
+    if (auto toPromoteMemOp = dyn_cast<PromotableMemOpInterface>(toPromote)) {
+      Value reachingDef = reachingDefs.lookup(toPromoteMemOp);
+      // If no reaching definition is known, this use is outside the reach of
+      // the slot. The default value should thus be used.
+      if (!reachingDef)
+        reachingDef = getLazyDefaultValue();
+
+      builder.setInsertionPointAfter(toPromote);
+      if (toPromoteMemOp.removeBlockingUses(slot, userToBlockingUses[toPromote],
+                                            builder, reachingDef) ==
+          DeletionKind::Delete)
+        toErase.push_back(toPromote);
+
+      continue;
+    }
+
+    auto toPromoteBasic = cast<PromotableOpInterface>(toPromote);
+    builder.setInsertionPointAfter(toPromote);
+    if (toPromoteBasic.removeBlockingUses(slot, userToBlockingUses[toPromote],
+                                          builder) == DeletionKind::Delete)
+      toErase.push_back(toPromote);
+  }
+
+  for (Operation *toEraseOp : toErase)
+    toEraseOp->erase();
+
+  assert(slot.ptr.use_empty() &&
+         "after promotion, the slot pointer should not be used anymore");
+}
+
+void SlotPromoter::promoteSlot() {
+  computeReachingDefInRegion(slot.ptr.getParentRegion(), {});
+
+  // Now that reaching definitions are known, remove all users.
+  removeBlockingUses();
+
+  // Update terminators in dead branches to forward default if they are
+  // succeeded by a merge points.
+  for (Block *mergePoint : mergePoints) {
+    for (BlockOperand &use : mergePoint->getUses()) {
+      auto user = cast<BranchOpInterface>(use.getOwner());
+      SuccessorOperands succOperands =
+          user.getSuccessorOperands(use.getOperandNumber());
+      assert(succOperands.size() == mergePoint->getNumArguments() ||
+             succOperands.size() + 1 == mergePoint->getNumArguments());
+      if (succOperands.size() + 1 == mergePoint->getNumArguments())
+        succOperands.append(getLazyDefaultValue());
+    }
+  }
+
+  allocator.handlePromotionComplete(slot, defaultValue);
+}
+
+LogicalResult SlotPromoter::prepareSlotPromotion() {
+  // First, find the set of operations that will need to be changed for the
+  // promotion to happen. These operations need to resolve some of their uses,
+  // either by rewiring them or simply deleting themselves. If any of them
+  // cannot find a way to resolve their blocking uses, we abort the promotion.
+  if (failed(computeBlockingUses()))
+    return failure();
+
+  // Then, compute blocks in which two or more definitions of the allocated
+  // variable may conflict. These blocks will need a new block argument to
+  // accomodate this.
+  computeMergePoints();
+
+  // The slot can be promoted if the block arguments to be created can
+  // actually be populated with values, which may not be possible depending
+  // on their predecessors.
+  return success(areMergePointsUsable());
+}
+
+LogicalResult mlir::tryToPromoteMemorySlots(
+    ArrayRef<PromotableAllocationOpInterface> allocators, OpBuilder &builder,
+    DominanceInfo &dominance) {
+  // Actual promotion may invalidate the dominance analysis, so slot promotion
+  // is prepated in batches.
+  SmallVector<SlotPromoter> toPromote;
+  for (PromotableAllocationOpInterface allocator : allocators) {
+    for (MemorySlot slot : allocator.getPromotableSlots()) {
+      if (slot.ptr.use_empty())
+        continue;
+
+      SlotPromoter promoter(slot, allocator, builder, dominance);
+      if (succeeded(promoter.prepareSlotPromotion()))
+        toPromote.emplace_back(std::move(promoter));
+    }
+  }
+
+  for (SlotPromoter &promoter : toPromote)
+    promoter.promoteSlot();
+
+  return success(!toPromote.empty());
+}
+
+namespace {
+
+struct Mem2Reg : impl::Mem2RegBase<Mem2Reg> {
+  void runOnOperation() override {
+    Operation *scopeOp = getOperation();
+    bool changed = false;
+
+    for (Region &region : scopeOp->getRegions()) {
+      if (region.getBlocks().empty())
+        continue;
+
+      OpBuilder builder(&region.front(), region.front().begin());
+
+      // Promoting a slot can allow for further promotion of other slots,
+      // promotion is tried until no promotion succeeds.
+      while (true) {
+        DominanceInfo &dominance = getAnalysis<DominanceInfo>();
+
+        SmallVector<PromotableAllocationOpInterface> allocators;
+        // Build a list of allocators to attempt to promote the slots of.
+        for (Block &block : region)
+          for (Operation &op : block.getOperations())
+            if (auto allocator = dyn_cast<PromotableAllocationOpInterface>(op))
+              allocators.emplace_back(allocator);
+
+        // Attempt promoting until no promotion succeeds.
+        if (failed(tryToPromoteMemorySlots(allocators, builder, dominance)))
+          break;
+
+        changed = true;
+        getAnalysisManager().invalidate({});
+      }
+    }
+
+    if (!changed)
+      markAllAnalysesPreserved();
+  }
+};
+
+} // namespace
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 47d8a6c40b6c2..3bdd99e5bab53 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/IR/Action.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Rewrite/PatternApplicator.h"
@@ -410,6 +411,24 @@ RegionPatternRewriteDriver::RegionPatternRewriteDriver(
   }
 }
 
+namespace {
+class GreedyPatternRewriteIteration
+    : public tracing::ActionImpl<GreedyPatternRewriteIteration> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(GreedyPatternRewriteIteration)
+  GreedyPatternRewriteIteration(ArrayRef<IRUnit> units, int64_t iteration)
+      : tracing::ActionImpl<GreedyPatternRewriteIteration>(units),
+        iteration(iteration) {}
+  static constexpr StringLiteral tag = "GreedyPatternRewriteIteration";
+  void print(raw_ostream &os) const override {
+    os << "GreedyPatternRewriteIteration(" << iteration << ")";
+  }
+
+private:
+  int64_t iteration = 0;
+};
+} // namespace
+
 LogicalResult RegionPatternRewriteDriver::simplify() && {
   auto insertKnownConstant = [&](Operation *op) {
     // Check for existing constants when populating the worklist. This avoids
@@ -423,6 +442,7 @@ LogicalResult RegionPatternRewriteDriver::simplify() && {
 
   bool changed = false;
   int64_t iteration = 0;
+  MLIRContext *ctx = getContext();
   do {
     // Check if the iteration limit was reached.
     if (iteration++ >= config.maxIterations &&
@@ -455,12 +475,16 @@ LogicalResult RegionPatternRewriteDriver::simplify() && {
         worklistMap[worklist[i]] = i;
     }
 
-    changed = processWorklist();
+    ctx->executeAction<GreedyPatternRewriteIteration>(
+        [&] {
+          changed = processWorklist();
 
-    // After applying patterns, make sure that the CFG of each of the regions
-    // is kept up to date.
-    if (config.enableRegionSimplification)
-      changed |= succeeded(simplifyRegions(*this, region));
+          // After applying patterns, make sure that the CFG of each of the
+          // regions is kept up to date.
+          if (config.enableRegionSimplification)
+            changed |= succeeded(simplifyRegions(*this, region));
+        },
+        {&region}, iteration);
   } while (changed);
 
   // Whether the rewrite converges, i.e. wasn't changed in the last iteration.
diff --git a/mlir/python/mlir/dialects/_loop_transform_ops_ext.py b/mlir/python/mlir/dialects/_loop_transform_ops_ext.py
index 0dc8fc07431a2..a275ea615378e 100644
--- a/mlir/python/mlir/dialects/_loop_transform_ops_ext.py
+++ b/mlir/python/mlir/dialects/_loop_transform_ops_ext.py
@@ -11,65 +11,63 @@
 from typing import Optional, Union
 
 
-def _get_int64_attr(arg: Optional[Union[int, IntegerAttr]],
-                    default_value: int = None):
-  if isinstance(arg, IntegerAttr):
-    return arg
-
-  if arg is None:
-    assert default_value is not None, "must provide default value"
-    arg = default_value
-
-  return IntegerAttr.get(IntegerType.get_signless(64), arg)
-
-
 class GetParentForOp:
   """Extension for GetParentForOp."""
 
-  def __init__(self,
-               result_type: Type,
-               target: Union[Operation, Value],
-               *,
-               num_loops: int = 1,
-               ip=None,
-               loc=None):
+  def __init__(
+      self,
+      result_type: Type,
+      target: Union[Operation, Value],
+      *,
+      num_loops: Optional[int] = None,
+      ip=None,
+      loc=None,
+  ):
+    if num_loops is None:
+      num_loops = 1
     super().__init__(
         result_type,
         _get_op_result_or_value(target),
-        num_loops=_get_int64_attr(num_loops, default_value=1),
+        num_loops=num_loops,
         ip=ip,
-        loc=loc)
+        loc=loc,
+    )
 
 
 class LoopOutlineOp:
   """Extension for LoopOutlineOp."""
 
-  def __init__(self,
-               result_type: Type,
-               target: Union[Operation, Value],
-               *,
-               func_name: Union[str, StringAttr],
-               ip=None,
-               loc=None):
+  def __init__(
+      self,
+      result_type: Type,
+      target: Union[Operation, Value],
+      *,
+      func_name: Union[str, StringAttr],
+      ip=None,
+      loc=None,
+  ):
     super().__init__(
         result_type,
         _get_op_result_or_value(target),
         func_name=(func_name if isinstance(func_name, StringAttr) else
                    StringAttr.get(func_name)),
         ip=ip,
-        loc=loc)
+        loc=loc,
+    )
 
 
 class LoopPeelOp:
   """Extension for LoopPeelOp."""
 
-  def __init__(self,
-               result_type: Type,
-               target: Union[Operation, Value],
-               *,
-               fail_if_already_divisible: Union[bool, BoolAttr] = False,
-               ip=None,
-               loc=None):
+  def __init__(
+      self,
+      result_type: Type,
+      target: Union[Operation, Value],
+      *,
+      fail_if_already_divisible: Union[bool, BoolAttr] = False,
+      ip=None,
+      loc=None,
+  ):
     super().__init__(
         result_type,
         _get_op_result_or_value(target),
@@ -77,40 +75,51 @@ def __init__(self,
             fail_if_already_divisible, BoolAttr) else
                                    BoolAttr.get(fail_if_already_divisible)),
         ip=ip,
-        loc=loc)
+        loc=loc,
+    )
 
 
 class LoopPipelineOp:
   """Extension for LoopPipelineOp."""
 
-  def __init__(self,
-               result_type: Type,
-               target: Union[Operation, Value],
-               *,
-               iteration_interval: Optional[Union[int, IntegerAttr]] = None,
-               read_latency: Optional[Union[int, IntegerAttr]] = None,
-               ip=None,
-               loc=None):
+  def __init__(
+      self,
+      result_type: Type,
+      target: Union[Operation, Value],
+      *,
+      iteration_interval: Optional[Union[int, IntegerAttr]] = None,
+      read_latency: Optional[Union[int, IntegerAttr]] = None,
+      ip=None,
+      loc=None,
+  ):
+    if iteration_interval is None:
+      iteration_interval = 1
+    if read_latency is None:
+      read_latency = 10
     super().__init__(
         result_type,
         _get_op_result_or_value(target),
-        iteration_interval=_get_int64_attr(iteration_interval, default_value=1),
-        read_latency=_get_int64_attr(read_latency, default_value=10),
+        iteration_interval=iteration_interval,
+        read_latency=read_latency,
         ip=ip,
-        loc=loc)
+        loc=loc,
+    )
 
 
 class LoopUnrollOp:
   """Extension for LoopUnrollOp."""
 
-  def __init__(self,
-               target: Union[Operation, Value],
-               *,
-               factor: Union[int, IntegerAttr],
-               ip=None,
-               loc=None):
+  def __init__(
+      self,
+      target: Union[Operation, Value],
+      *,
+      factor: Union[int, IntegerAttr],
+      ip=None,
+      loc=None,
+  ):
     super().__init__(
         _get_op_result_or_value(target),
-        factor=_get_int64_attr(factor),
+        factor=factor,
         ip=ip,
-        loc=loc)
+        loc=loc,
+    )
diff --git a/mlir/python/mlir/dialects/_pdl_ops_ext.py b/mlir/python/mlir/dialects/_pdl_ops_ext.py
index 428301b18f208..40ccbef6351dc 100644
--- a/mlir/python/mlir/dialects/_pdl_ops_ext.py
+++ b/mlir/python/mlir/dialects/_pdl_ops_ext.py
@@ -8,61 +8,26 @@
 except ImportError as e:
   raise RuntimeError("Error loading imports from extension module") from e
 
-from typing import Union, Optional, Sequence, List, Mapping
-from ._ods_common import get_op_result_or_value as _get_value, get_op_results_or_values as _get_values
-
-
-def _get_int_attr(bits: int, value: Union[IntegerAttr, int]) -> IntegerAttr:
-  """Converts the given value to signless integer attribute of given bit width."""
-  if isinstance(value, int):
-    ty = IntegerType.get_signless(bits)
-    return IntegerAttr.get(ty, value)
-  else:
-    return value
-
-
-def _get_array_attr(attrs: Union[ArrayAttr, Sequence[Attribute]]) -> ArrayAttr:
-  """Converts the given value to array attribute."""
-  if isinstance(attrs, ArrayAttr):
-    return attrs
-  else:
-    return ArrayAttr.get(list(attrs))
-
-
-def _get_str_array_attr(attrs: Union[ArrayAttr, Sequence[str]]) -> ArrayAttr:
-  """Converts the given value to string array attribute."""
-  if isinstance(attrs, ArrayAttr):
-    return attrs
-  else:
-    return ArrayAttr.get([StringAttr.get(s) for s in attrs])
-
-
-def _get_str_attr(name: Union[StringAttr, str]) -> Optional[StringAttr]:
-  """Converts the given value to string attribute."""
-  if isinstance(name, str):
-    return StringAttr.get(name)
-  else:
-    return name
-
-
-def _get_type_attr(type: Union[TypeAttr, Type]) -> TypeAttr:
-  """Converts the given value to type attribute."""
-  if isinstance(type, Type):
-    return TypeAttr.get(type)
-  else:
-    return type
+from typing import Union, Optional, Sequence, Mapping
+from ._ods_common import (
+    get_op_result_or_value as _get_value,
+    get_op_results_or_values as _get_values,
+)
 
 
 class ApplyNativeConstraintOp:
   """Specialization for PDL apply native constraint op class."""
 
-  def __init__(self,
-               name: Union[str, StringAttr],
-               args: Sequence[Union[OpView, Operation, Value]] = [],
-               *,
-               loc=None,
-               ip=None):
-    name = _get_str_attr(name)
+  def __init__(
+      self,
+      name: Union[str, StringAttr],
+      args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
+    if args is None:
+      args = []
     args = _get_values(args)
     super().__init__(name, args, loc=loc, ip=ip)
 
@@ -70,14 +35,17 @@ def __init__(self,
 class ApplyNativeRewriteOp:
   """Specialization for PDL apply native rewrite op class."""
 
-  def __init__(self,
-               results: Sequence[Type],
-               name: Union[str, StringAttr],
-               args: Sequence[Union[OpView, Operation, Value]] = [],
-               *,
-               loc=None,
-               ip=None):
-    name = _get_str_attr(name)
+  def __init__(
+      self,
+      results: Sequence[Type],
+      name: Union[str, StringAttr],
+      args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
+    if args is None:
+      args = []
     args = _get_values(args)
     super().__init__(results, name, args, loc=loc, ip=ip)
 
@@ -85,12 +53,14 @@ def __init__(self,
 class AttributeOp:
   """Specialization for PDL attribute op class."""
 
-  def __init__(self,
-               valueType: Optional[Union[OpView, Operation, Value]] = None,
-               value: Optional[Attribute] = None,
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      valueType: Optional[Union[OpView, Operation, Value]] = None,
+      value: Optional[Attribute] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
     valueType = valueType if valueType is None else _get_value(valueType)
     result = pdl.AttributeType.get()
     super().__init__(result, valueType=valueType, value=value, loc=loc, ip=ip)
@@ -99,11 +69,13 @@ def __init__(self,
 class EraseOp:
   """Specialization for PDL erase op class."""
 
-  def __init__(self,
-               operation: Optional[Union[OpView, Operation, Value]] = None,
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      operation: Optional[Union[OpView, Operation, Value]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
     operation = _get_value(operation)
     super().__init__(operation, loc=loc, ip=ip)
 
@@ -111,11 +83,13 @@ def __init__(self,
 class OperandOp:
   """Specialization for PDL operand op class."""
 
-  def __init__(self,
-               type: Optional[Union[OpView, Operation, Value]] = None,
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      type: Optional[Union[OpView, Operation, Value]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
     type = type if type is None else _get_value(type)
     result = pdl.ValueType.get()
     super().__init__(result, valueType=type, loc=loc, ip=ip)
@@ -124,11 +98,13 @@ def __init__(self,
 class OperandsOp:
   """Specialization for PDL operands op class."""
 
-  def __init__(self,
-               types: Optional[Union[OpView, Operation, Value]] = None,
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      types: Optional[Union[OpView, Operation, Value]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
     types = types if types is None else _get_value(types)
     result = pdl.RangeType.get(pdl.ValueType.get())
     super().__init__(result, valueType=types, loc=loc, ip=ip)
@@ -137,15 +113,23 @@ def __init__(self,
 class OperationOp:
   """Specialization for PDL operand op class."""
 
-  def __init__(self,
-               name: Optional[Union[str, StringAttr]] = None,
-               args: Sequence[Union[OpView, Operation, Value]] = [],
-               attributes: Mapping[str, Union[OpView, Operation, Value]] = {},
-               types: Sequence[Union[OpView, Operation, Value]] = [],
-               *,
-               loc=None,
-               ip=None):
-    name = name if name is None else _get_str_attr(name)
+  def __init__(
+      self,
+      name: Optional[Union[str, StringAttr]] = None,
+      args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+      attributes: Optional[Mapping[str, Union[OpView, Operation,
+                                              Value]]] = None,
+      types: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
+    if types is None:
+      types = []
+    if attributes is None:
+      attributes = {}
+    if args is None:
+      args = []
     args = _get_values(args)
     attrNames = []
     attrValues = []
@@ -155,22 +139,29 @@ def __init__(self,
     attrNames = ArrayAttr.get(attrNames)
     types = _get_values(types)
     result = pdl.OperationType.get()
-    super().__init__(result, args, attrValues, attrNames, types, opName=name, loc=loc, ip=ip)
+    super().__init__(result,
+                     args,
+                     attrValues,
+                     attrNames,
+                     types,
+                     opName=name,
+                     loc=loc,
+                     ip=ip)
 
 
 class PatternOp:
   """Specialization for PDL pattern op class."""
 
-  def __init__(self,
-               benefit: Union[IntegerAttr, int],
-               name: Optional[Union[StringAttr, str]] = None,
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      benefit: Union[IntegerAttr, int],
+      name: Optional[Union[StringAttr, str]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
     """Creates an PDL `pattern` operation."""
-    name_attr = None if name is None else _get_str_attr(name)
-    benefit_attr = _get_int_attr(16, benefit)
-    super().__init__(benefit_attr, sym_name=name_attr, loc=loc, ip=ip)
+    super().__init__(benefit, sym_name=name, loc=loc, ip=ip)
     self.regions[0].blocks.append()
 
   @property
@@ -182,13 +173,17 @@ def body(self):
 class ReplaceOp:
   """Specialization for PDL replace op class."""
 
-  def __init__(self,
-               op: Union[OpView, Operation, Value],
-               *,
-               with_op: Optional[Union[OpView, Operation, Value]] = None,
-               with_values: Sequence[Union[OpView, Operation, Value]] = [],
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      op: Union[OpView, Operation, Value],
+      *,
+      with_op: Optional[Union[OpView, Operation, Value]] = None,
+      with_values: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+      loc=None,
+      ip=None,
+  ):
+    if with_values is None:
+      with_values = []
     op = _get_value(op)
     with_op = with_op if with_op is None else _get_value(with_op)
     with_values = _get_values(with_values)
@@ -198,13 +193,14 @@ def __init__(self,
 class ResultOp:
   """Specialization for PDL result op class."""
 
-  def __init__(self,
-               parent: Union[OpView, Operation, Value],
-               index: Union[IntegerAttr, int],
-               *,
-               loc=None,
-               ip=None):
-    index = _get_int_attr(32, index)
+  def __init__(
+      self,
+      parent: Union[OpView, Operation, Value],
+      index: Union[IntegerAttr, int],
+      *,
+      loc=None,
+      ip=None,
+  ):
     parent = _get_value(parent)
     result = pdl.ValueType.get()
     super().__init__(result, parent, index, loc=loc, ip=ip)
@@ -213,32 +209,36 @@ def __init__(self,
 class ResultsOp:
   """Specialization for PDL results op class."""
 
-  def __init__(self,
-               result: Type,
-               parent: Union[OpView, Operation, Value],
-               index: Optional[Union[IntegerAttr, int]] = None,
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      result: Type,
+      parent: Union[OpView, Operation, Value],
+      index: Optional[Union[IntegerAttr, int]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
     parent = _get_value(parent)
-    index = index if index is None else _get_int_attr(32, index)
     super().__init__(result, parent, index=index, loc=loc, ip=ip)
 
 
 class RewriteOp:
   """Specialization for PDL rewrite op class."""
 
-  def __init__(self,
-               root: Optional[Union[OpView, Operation, Value]] = None,
-               name: Optional[Union[StringAttr, str]] = None,
-               args: Sequence[Union[OpView, Operation, Value]] = [],
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      root: Optional[Union[OpView, Operation, Value]] = None,
+      name: Optional[Union[StringAttr, str]] = None,
+      args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
+    if args is None:
+      args = []
     root = root if root is None else _get_value(root)
-    name = name if name is None else _get_str_attr(name)
     args = _get_values(args)
-    super().__init__(args, root=root,name=name, loc=loc, ip=ip)
+    super().__init__(args, root=root, name=name, loc=loc, ip=ip)
 
   def add_body(self):
     """Add body (block) to the rewrite."""
@@ -259,8 +259,6 @@ def __init__(self,
                *,
                loc=None,
                ip=None):
-    constantType = constantType if constantType is None else _get_type_attr(
-        constantType)
     result = pdl.TypeType.get()
     super().__init__(result, constantType=constantType, loc=loc, ip=ip)
 
@@ -268,13 +266,14 @@ def __init__(self,
 class TypesOp:
   """Specialization for PDL types op class."""
 
-  def __init__(self,
-               constantTypes: Sequence[Union[TypeAttr, Type]] = [],
-               *,
-               loc=None,
-               ip=None):
-    constantTypes = _get_array_attr(
-        [_get_type_attr(ty) for ty in constantTypes])
-    constantTypes = None if not constantTypes else constantTypes
+  def __init__(
+      self,
+      constantTypes: Optional[Sequence[Union[TypeAttr, Type]]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
+    if constantTypes is None:
+      constantTypes = []
     result = pdl.RangeType.get(pdl.TypeType.get())
     super().__init__(result, constantTypes=constantTypes, loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/_structured_transform_ops_ext.py b/mlir/python/mlir/dialects/_structured_transform_ops_ext.py
index e2c262ca50201..9c051cd3d146d 100644
--- a/mlir/python/mlir/dialects/_structured_transform_ops_ext.py
+++ b/mlir/python/mlir/dialects/_structured_transform_ops_ext.py
@@ -15,180 +15,159 @@
 OptionalIntList = Optional[Union[ArrayAttr, IntOrAttrList]]
 
 
-def _get_int64_attr(value: Union[int, Attribute]) -> IntegerAttr:
-  if isinstance(value, int):
-    return IntegerAttr.get(IntegerType.get_signless(64), value)
-  return value
-
-
-def _get_array_attr(
-    values: Optional[Union[ArrayAttr, Sequence[Attribute]]]) -> ArrayAttr:
-  """Creates an array attribute from its operand."""
-  if values is None:
-    return ArrayAttr.get([])
-  if isinstance(values, ArrayAttr):
-    return values
-
-  return ArrayAttr.get(values)
-
-
-def _get_int_array_attr(
-    values: Optional[Union[ArrayAttr, Sequence[Union[IntegerAttr, int]]]]
-) -> ArrayAttr:
-  """Creates an integer array attribute from its operand.
-
-  If the operand is already an array attribute, forwards it. Otherwise treats
-  the operand as a list of attributes or integers, possibly intersperced, to
-  create a new array attribute containing integer attributes. Expects the
-  thread-local MLIR context to have been set by the context manager.
-  """
-  if values is None:
-    return ArrayAttr.get([])
-  if isinstance(values, ArrayAttr):
-    return values
-
-  return ArrayAttr.get([_get_int64_attr(v) for v in values])
-
-def _get_dense_int64_array_attr(
-        values: Sequence[int]) -> DenseI64ArrayAttr:
-  """Creates a dense integer array from a sequence of integers.
-    Expects the thread-local MLIR context to have been set by the context 
-    manager.
-    """
-  if values is None:
-    return DenseI64ArrayAttr.get([])
-  return DenseI64ArrayAttr.get(values)
-
 def _get_int_int_array_attr(
     values: Optional[Union[ArrayAttr, Sequence[Union[ArrayAttr,
                                                      IntOrAttrList]]]]
 ) -> ArrayAttr:
   """Creates an array attribute containing array attributes of integers.
 
-  If the operand is already an array attribute, forwards it. Otherwise treats
-  the operand as a list of attributes or integers, potentially interpserced, to
-  create a new array-of-array attribute. Expects the thread-local MLIR context
-  to have been set by the context manager.
-  """
+    If the operand is already an array attribute, forwards it. Otherwise treats
+    the operand as a list of attributes or integers, potentially interpserced, to
+    create a new array-of-array attribute. Expects the thread-local MLIR context
+    to have been set by the context manager.
+    """
   if values is None:
     return ArrayAttr.get([])
   if isinstance(values, ArrayAttr):
     return values
+  if isinstance(values, list):
+    values = [
+        ArrayAttr.get(
+            [IntegerAttr.get(IntegerType.get_signless(64), v)
+             for v in value])
+        for value in values
+    ]
 
-  return ArrayAttr.get([_get_int_array_attr(value) for value in values])
+  return ArrayAttr.get(values)
 
 
 class DecomposeOp:
   """Specialization for DecomposeOp class."""
 
   def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
-    super().__init__(
-        pdl.OperationType.get(),
-        _get_op_result_or_value(target),
-        loc=loc,
-        ip=ip)
+    super().__init__(pdl.OperationType.get(),
+                     _get_op_result_or_value(target),
+                     loc=loc,
+                     ip=ip)
 
 
 class GeneralizeOp:
   """Specialization for GeneralizeOp class."""
 
   def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
-    super().__init__(
-        pdl.OperationType.get(),
-        _get_op_result_or_value(target),
-        loc=loc,
-        ip=ip)
+    super().__init__(pdl.OperationType.get(),
+                     _get_op_result_or_value(target),
+                     loc=loc,
+                     ip=ip)
 
 
 class InterchangeOp:
   """Specialization for InterchangeOp class."""
 
-  def __init__(self,
-               target: Union[Operation, Value],
-               *,
-               iterator_interchange: OptionalIntList = None,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      target: Union[Operation, Value],
+      *,
+      iterator_interchange: OptionalIntList = None,
+      loc=None,
+      ip=None,
+  ):
     pdl_operation_type = pdl.OperationType.get()
-    interchange_attr = _get_dense_int64_array_attr(iterator_interchange)
     super().__init__(
         pdl_operation_type,
         _get_op_result_or_value(target),
-        iterator_interchange=interchange_attr,
+        iterator_interchange=iterator_interchange,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class MatchOp:
   """Specialization for MatchOp class."""
 
   @classmethod
-  def match_op_names(MatchOp,
-                     target: Union[Operation, Value],
-                     names: Sequence[str],
-                     loc=None,
-                     ip=None):
+  def match_op_names(
+      MatchOp,
+      target: Union[Operation, Value],
+      names: Sequence[str],
+      loc=None,
+      ip=None,
+  ):
     pdl_operation_type = pdl.OperationType.get()
     return MatchOp(
         pdl_operation_type,
         _get_op_result_or_value(target),
         ops=ArrayAttr.get(list(map(lambda s: StringAttr.get(s), names))),
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class MultiTileSizesOp:
   """Specialization for MultitileSizesOp class."""
 
-  def __init__(self,
-               result_type: Type,
-               target: Union[Operation, Value],
-               *,
-               dimension: Union[int, IntegerAttr],
-               target_size: Union[int, IntegerAttr],
-               divisor: Optional[Union[int, IntegerAttr]] = None,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      result_type: Type,
+      target: Union[Operation, Value],
+      *,
+      dimension: Union[int, IntegerAttr],
+      target_size: Union[int, IntegerAttr],
+      divisor: Optional[Optional[Union[int, IntegerAttr]]] = None,
+      loc=None,
+      ip=None,
+  ):
+    if divisor is None:
+      divisor = 1
     super().__init__(
         result_type,
         result_type,
         result_type,
         _get_op_result_or_value(target),
-        dimension=_get_int64_attr(dimension),
-        target_size=_get_int64_attr(target_size),
-        divisor=_get_int64_attr(divisor if divisor else 1),
+        dimension=dimension,
+        target_size=target_size,
+        divisor=divisor,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class PadOp:
   """Specialization for PadOp class."""
 
-  def __init__(self,
-               target: Union[Operation, Value],
-               *,
-               padding_values: Optional[Union[ArrayAttr,
-                                              Sequence[Attribute]]] = None,
-               padding_dimensions: OptionalIntList = None,
-               pack_paddings: OptionalIntList = None,
-               transpose_paddings: Optional[Union[ArrayAttr, Sequence[Union[
-                   ArrayAttr, IntOrAttrList]]]] = None,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      target: Union[Operation, Value],
+      *,
+      padding_values: Optional[Optional[Union[ArrayAttr,
+                                              Sequence[Attribute]]]] = None,
+      padding_dimensions: OptionalIntList = None,
+      pack_paddings: OptionalIntList = None,
+      transpose_paddings: Optional[Union[ArrayAttr, Sequence[Union[
+          ArrayAttr, IntOrAttrList]]]] = None,
+      loc=None,
+      ip=None,
+  ):
+    if transpose_paddings is None:
+      transpose_paddings = []
+    if pack_paddings is None:
+      pack_paddings = []
+    if padding_dimensions is None:
+      padding_dimensions = []
+    if padding_values is None:
+      padding_values = []
     pdl_operation_type = pdl.OperationType.get()
-    padding_values_attr = _get_array_attr(padding_values)
-    padding_dimensions_attr = _get_int_array_attr(padding_dimensions)
-    pack_paddings_attr = _get_int_array_attr(pack_paddings)
     transpose_paddings_attr = _get_int_int_array_attr(transpose_paddings)
     super().__init__(
         pdl_operation_type,
         _get_op_result_or_value(target),
-        padding_values=padding_values_attr,
-        padding_dimensions=padding_dimensions_attr,
-        pack_paddings=pack_paddings_attr,
+        padding_values=padding_values,
+        padding_dimensions=padding_dimensions,
+        pack_paddings=pack_paddings,
         transpose_paddings=transpose_paddings_attr,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class ScalarizeOp:
@@ -196,29 +175,29 @@ class ScalarizeOp:
 
   def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
     pdl_operation_type = pdl.OperationType.get()
-    super().__init__(
-        pdl_operation_type, _get_op_result_or_value(target), loc=loc, ip=ip)
+    super().__init__(pdl_operation_type,
+                     _get_op_result_or_value(target),
+                     loc=loc,
+                     ip=ip)
 
 
 class SplitOp:
   """Specialization for SplitOp class."""
 
-  def __init__(self,
-               target: Union[Operation, Value],
-               dimension: Union[int, Attribute],
-               split_point: Union[int, Operation, Value, Attribute],
-               *,
-               loc=None,
-               ip=None):
-    dimension = _get_int64_attr(dimension)
+  def __init__(
+      self,
+      target: Union[Operation, Value],
+      dimension: Union[int, Attribute],
+      split_point: Union[int, Operation, Value, Attribute],
+      *,
+      loc=None,
+      ip=None,
+  ):
     if isinstance(split_point, int):
-      split_point = _get_int64_attr(split_point)
-
-    if isinstance(split_point, Attribute):
       static_split_point = split_point
       dynamic_split_point = None
     else:
-      static_split_point = _get_int64_attr(ShapedType.get_dynamic_size())
+      static_split_point = ShapedType.get_dynamic_size()
       dynamic_split_point = _get_op_result_or_value(split_point)
 
     target = _get_op_result_or_value(target)
@@ -231,44 +210,53 @@ def __init__(self,
         static_split_point=static_split_point,
         dynamic_split_point=dynamic_split_point,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class TileOp:
   """Specialization for TileOp class."""
 
   @overload
-  def __init__(self,
-               loop_types: Union[Type, List[Type]],
-               target: Union[Operation, Value],
-               *,
-               sizes: Optional[Union[Sequence[Union[int, IntegerAttr, Operation,
-                                                    Value]], ArrayAttr]] = None,
-               interchange: OptionalIntList = None,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      loop_types: Union[Type, List[Type]],
+      target: Union[Operation, Value],
+      *,
+      sizes: Optional[Union[Sequence[Union[int, IntegerAttr, Operation, Value]],
+                            ArrayAttr]] = None,
+      interchange: OptionalIntList = None,
+      loc=None,
+      ip=None,
+  ):
     ...
 
   @overload
-  def __init__(self,
-               target: Union[Operation, Value, OpView],
-               *,
-               sizes: Optional[Union[Sequence[Union[int, IntegerAttr, Operation,
-                                                    Value]], ArrayAttr]] = None,
-               interchange: OptionalIntList = None,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      target: Union[Operation, Value, OpView],
+      *,
+      sizes: Optional[Union[Sequence[Union[int, IntegerAttr, Operation, Value]],
+                            ArrayAttr]] = None,
+      interchange: OptionalIntList = None,
+      loc=None,
+      ip=None,
+  ):
     ...
 
-  def __init__(self,
-               loop_types_or_target: Union[Type, List[Type], Operation, Value],
-               target_or_none: Optional[Union[Operation, Value, OpView]] = None,
-               *,
-               sizes: Optional[Union[Sequence[Union[int, IntegerAttr, Operation,
-                                                    Value]], ArrayAttr]] = None,
-               interchange: OptionalIntList = None,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      loop_types_or_target: Union[Type, List[Type], Operation, Value],
+      target_or_none: Optional[Union[Operation, Value, OpView]] = None,
+      *,
+      sizes: Optional[Union[Sequence[Union[int, IntegerAttr, Operation, Value]],
+                            ArrayAttr]] = None,
+      interchange: OptionalIntList = None,
+      loc=None,
+      ip=None,
+  ):
+    if interchange is None:
+      interchange = []
     if sizes is None:
       sizes = []
 
@@ -293,8 +281,8 @@ def __init__(self,
       target = loop_types_or_target
       assert target_or_none is None, "Cannot construct TileOp with two targets."
     else:
-      loop_types = ([loop_types_or_target] * num_loops) if isinstance(
-          loop_types_or_target, Type) else loop_types_or_target
+      loop_types = (([loop_types_or_target] * num_loops) if isinstance(
+          loop_types_or_target, Type) else loop_types_or_target)
       target = target_or_none
 
     target = _get_op_result_or_value(target)
@@ -305,10 +293,10 @@ def __init__(self,
         target,
         dynamic_sizes=dynamic_sizes,
         static_sizes=sizes_attr,
-        interchange=_get_dense_int64_array_attr(interchange)
-        if interchange else None,
+        interchange=interchange,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
   def __extract_values(self, attr: Optional[DenseI64ArrayAttr]) -> List[int]:
     if not attr:
@@ -319,12 +307,14 @@ def __extract_values(self, attr: Optional[DenseI64ArrayAttr]) -> List[int]:
 class VectorizeOp:
   """Specialization for VectorizeOp class."""
 
-  def __init__(self,
-               target: Union[Operation, Value],
-               *,
-               vectorize_padding: Union[bool, BoolAttr] = False,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      target: Union[Operation, Value],
+      *,
+      vectorize_padding: Union[bool, BoolAttr] = False,
+      loc=None,
+      ip=None,
+  ):
     pdl_operation_type = pdl.OperationType.get()
     if isinstance(vectorize_padding, bool):
       vectorize_padding = UnitAttr.get()
@@ -333,4 +323,5 @@ def __init__(self,
         _get_op_result_or_value(target),
         vectorize_padding=vectorize_padding,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
diff --git a/mlir/python/mlir/dialects/_transform_ops_ext.py b/mlir/python/mlir/dialects/_transform_ops_ext.py
index 593b8855c935f..8651c76ea7dfc 100644
--- a/mlir/python/mlir/dialects/_transform_ops_ext.py
+++ b/mlir/python/mlir/dialects/_transform_ops_ext.py
@@ -4,102 +4,119 @@
 
 try:
   from ..ir import *
-  from ._ods_common import get_op_result_or_value as _get_op_result_or_value, get_op_results_or_values as _get_op_results_or_values
+  from ._ods_common import (
+      get_op_result_or_value as _get_op_result_or_value,
+      get_op_results_or_values as _get_op_results_or_values,
+  )
 except ImportError as e:
   raise RuntimeError("Error loading imports from extension module") from e
 
-from argparse import SUPPRESS
-from typing import Optional, overload, Sequence, Union
-
-
-def _get_symbol_ref_attr(value: Union[Attribute, str]):
-  if isinstance(value, Attribute):
-    return value
-  return FlatSymbolRefAttr.get(value)
+from typing import Optional, Sequence, Union
 
 
 class CastOp:
 
-  def __init__(self, result_type: Type, target: Union[Operation, Value], *, loc=None, ip=None):
-    super().__init__(
-      result_type,
-      _get_op_result_or_value(target),
-      loc=loc,
-      ip=ip)
+  def __init__(self,
+               result_type: Type,
+               target: Union[Operation, Value],
+               *,
+               loc=None,
+               ip=None):
+    super().__init__(result_type,
+                     _get_op_result_or_value(target),
+                     loc=loc,
+                     ip=ip)
 
 
 class GetClosestIsolatedParentOp:
 
-  def __init__(self, result_type: Type, target: Union[Operation, Value], *, loc=None, ip=None):
-    super().__init__(
-        result_type,
-        _get_op_result_or_value(target),
-        loc=loc,
-        ip=ip)
-
-
-class MergeHandlesOp:
-
   def __init__(self,
-               handles: Sequence[Union[Operation, Value]],
+               result_type: Type,
+               target: Union[Operation, Value],
                *,
-               deduplicate: bool = False,
                loc=None,
                ip=None):
+    super().__init__(result_type,
+                     _get_op_result_or_value(target),
+                     loc=loc,
+                     ip=ip)
+
+
+class MergeHandlesOp:
+
+  def __init__(
+      self,
+      handles: Sequence[Union[Operation, Value]],
+      *,
+      deduplicate: bool = False,
+      loc=None,
+      ip=None,
+  ):
     super().__init__(
         [_get_op_result_or_value(h) for h in handles],
         deduplicate=deduplicate,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class PDLMatchOp:
 
-  def __init__(self,
-               result_type: Type,
-               target: Union[Operation, Value],
-               pattern_name: Union[Attribute, str],
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      result_type: Type,
+      target: Union[Operation, Value],
+      pattern_name: Union[Attribute, str],
+      *,
+      loc=None,
+      ip=None,
+  ):
     super().__init__(
         result_type,
         _get_op_result_or_value(target),
-        _get_symbol_ref_attr(pattern_name),
+        pattern_name,
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class ReplicateOp:
 
-  def __init__(self,
-               pattern: Union[Operation, Value],
-               handles: Sequence[Union[Operation, Value]],
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      pattern: Union[Operation, Value],
+      handles: Sequence[Union[Operation, Value]],
+      *,
+      loc=None,
+      ip=None,
+  ):
     super().__init__(
         [_get_op_result_or_value(h).type for h in handles],
         _get_op_result_or_value(pattern),
         [_get_op_result_or_value(h) for h in handles],
         loc=loc,
-        ip=ip)
+        ip=ip,
+    )
 
 
 class SequenceOp:
 
-  def __init__(self, failure_propagation_mode, results: Sequence[Type],
-               target: Union[Operation, Value, Type],
-               extra_bindings: Optional[Union[Sequence[Value], Sequence[Type],
-                                              Operation, OpView]] = None):
-    root = _get_op_result_or_value(target) if isinstance(
-        target, (Operation, Value)) else None
+  def __init__(
+      self,
+      failure_propagation_mode,
+      results: Sequence[Type],
+      target: Union[Operation, Value, Type],
+      extra_bindings: Optional[Union[Sequence[Value], Sequence[Type], Operation,
+                                     OpView]] = None,
+  ):
+    root = (_get_op_result_or_value(target) if isinstance(
+        target, (Operation, Value)) else None)
     root_type = root.type if not isinstance(target, Type) else target
     if not isinstance(failure_propagation_mode, Attribute):
       failure_propagation_mode_attr = IntegerAttr.get(
           IntegerType.get_signless(32), failure_propagation_mode._as_int())
     else:
-      failure_propagation_mode = failure_propagation_mode
+      failure_propagation_mode_attr = failure_propagation_mode
 
     if extra_bindings is None:
       extra_bindings = []
@@ -114,10 +131,12 @@ def __init__(self, failure_propagation_mode, results: Sequence[Type],
       else:
         extra_binding_types = [v.type for v in extra_bindings]
 
-    super().__init__(results_=results,
-                     failure_propagation_mode=failure_propagation_mode_attr,
-                     root=root,
-                     extra_bindings=extra_bindings)
+    super().__init__(
+        results_=results,
+        failure_propagation_mode=failure_propagation_mode_attr,
+        root=root,
+        extra_bindings=extra_bindings,
+    )
     self.regions[0].blocks.append(*tuple([root_type] + extra_binding_types))
 
   @property
@@ -143,10 +162,7 @@ def __init__(self,
     root = _get_op_result_or_value(target) if not isinstance(target,
                                                              Type) else None
     root_type = target if isinstance(target, Type) else root.type
-    super().__init__(
-        root=root,
-        loc=loc,
-        ip=ip)
+    super().__init__(root=root, loc=loc, ip=ip)
     self.regions[0].blocks.append(root_type)
 
   @property
@@ -160,9 +176,13 @@ def bodyTarget(self) -> Value:
 
 class YieldOp:
 
-  def __init__(self,
-               operands: Union[Operation, Sequence[Value]] = [],
-               *,
-               loc=None,
-               ip=None):
+  def __init__(
+      self,
+      operands: Optional[Union[Operation, Sequence[Value]]] = None,
+      *,
+      loc=None,
+      ip=None,
+  ):
+    if operands is None:
+      operands = []
     super().__init__(_get_op_results_or_values(operands), loc=loc, ip=ip)
diff --git a/mlir/python/mlir/ir.py b/mlir/python/mlir/ir.py
index 1e24fcbf99e40..714253426b025 100644
--- a/mlir/python/mlir/ir.py
+++ b/mlir/python/mlir/ir.py
@@ -8,9 +8,11 @@
 
 # Convenience decorator for registering user-friendly Attribute builders.
 def register_attribute_builder(kind):
+
   def decorator_builder(func):
     AttrBuilder.insert(kind, func)
     return func
+
   return decorator_builder
 
 
@@ -18,34 +20,77 @@ def decorator_builder(func):
 def _boolAttr(x, context):
   return BoolAttr.get(x, context=context)
 
+
 @register_attribute_builder("IndexAttr")
 def _indexAttr(x, context):
   return IntegerAttr.get(IndexType.get(context=context), x)
 
+
+@register_attribute_builder("I16Attr")
+def _i32Attr(x, context):
+  return IntegerAttr.get(IntegerType.get_signless(16, context=context), x)
+
+
 @register_attribute_builder("I32Attr")
 def _i32Attr(x, context):
-  return IntegerAttr.get(
-      IntegerType.get_signless(32, context=context), x)
+  return IntegerAttr.get(IntegerType.get_signless(32, context=context), x)
+
 
 @register_attribute_builder("I64Attr")
 def _i64Attr(x, context):
-  return IntegerAttr.get(
-      IntegerType.get_signless(64, context=context), x)
+  return IntegerAttr.get(IntegerType.get_signless(64, context=context), x)
+
 
 @register_attribute_builder("StrAttr")
 def _stringAttr(x, context):
   return StringAttr.get(x, context=context)
 
+
 @register_attribute_builder("SymbolNameAttr")
 def _symbolNameAttr(x, context):
   return StringAttr.get(x, context=context)
 
+
+@register_attribute_builder("SymbolRefAttr")
+def _symbolRefAttr(x, context):
+  return FlatSymbolRefAttr.get(x, context=context)
+
+
+@register_attribute_builder("ArrayAttr")
+def _arrayAttr(x, context):
+  return ArrayAttr.get(x, context=context)
+
+
+@register_attribute_builder("I64ArrayAttr")
+def _i64ArrayAttr(x, context):
+  return ArrayAttr.get([_i64Attr(v, context) for v in x])
+
+
+@register_attribute_builder("DenseI64ArrayAttr")
+def _denseI64ArrayAttr(x, context):
+  return DenseI64ArrayAttr.get(x, context=context)
+
+
+@register_attribute_builder("TypeAttr")
+def _typeAttr(x, context):
+  return TypeAttr.get(x, context=context)
+
+
+@register_attribute_builder("TypeArrayAttr")
+def _typeArrayAttr(x, context):
+  return _arrayAttr([TypeAttr.get(t, context=context) for t in x], context)
+
+
 try:
   import numpy as np
+
   @register_attribute_builder("IndexElementsAttr")
   def _indexElementsAttr(x, context):
     return DenseElementsAttr.get(
-        np.array(x, dtype=np.int64), type=IndexType.get(context=context),
-        context=context)
+        np.array(x, dtype=np.int64),
+        type=IndexType.get(context=context),
+        context=context,
+    )
+
 except ImportError:
   pass
diff --git a/mlir/test/Bytecode/versioning/versioned_attr.mlir b/mlir/test/Bytecode/versioning/versioned_attr.mlir
index 98756cc66a99c..0fd6c3c2dfe9e 100644
--- a/mlir/test/Bytecode/versioning/versioned_attr.mlir
+++ b/mlir/test/Bytecode/versioning/versioned_attr.mlir
@@ -11,10 +11,10 @@
 // COM: bytecode contains
 // COM: module {
 // COM:   version: 1.12
-// COM:   "test.versionedB"() {attribute = #test.attr_params<24, 42>} : () -> ()
+// COM:   "test.versionedB"() <{attribute = #test.attr_params<24, 42>}> : () -> ()
 // COM: }
 // RUN: mlir-opt %S/versioned-attr-1.12.mlirbc 2>&1 | FileCheck %s --check-prefix=CHECK1
-// CHECK1: "test.versionedB"() {attribute = #test.attr_params<42, 24>} : () -> ()
+// CHECK1: "test.versionedB"() <{attribute = #test.attr_params<42, 24>}> : () -> ()
 
 //===--------------------------------------------------------------------===//
 // Test attribute upgrade
@@ -23,7 +23,7 @@
 // COM: bytecode contains
 // COM: module {
 // COM:   version: 2.0
-// COM:   "test.versionedB"() {attribute = #test.attr_params<42, 24>} : () -> ()
+// COM:   "test.versionedB"() <{attribute = #test.attr_params<42, 24>}> : () -> ()
 // COM: }
 // RUN: mlir-opt %S/versioned-attr-2.0.mlirbc 2>&1 | FileCheck %s --check-prefix=CHECK2
-// CHECK2: "test.versionedB"() {attribute = #test.attr_params<42, 24>} : () -> ()
+// CHECK2: "test.versionedB"() <{attribute = #test.attr_params<42, 24>}> : () -> ()
diff --git a/mlir/test/Bytecode/versioning/versioned_bytecode.mlir b/mlir/test/Bytecode/versioning/versioned_bytecode.mlir
new file mode 100644
index 0000000000000..bf08a23c03ae0
--- /dev/null
+++ b/mlir/test/Bytecode/versioning/versioned_bytecode.mlir
@@ -0,0 +1,14 @@
+// This file contains test cases related to roundtripping.
+
+// Bytecode currently does not support big-endian platforms
+// UNSUPPORTED: target=s390x-{{.*}}
+
+//===--------------------------------------------------------------------===//
+// Test roundtrip
+//===--------------------------------------------------------------------===//
+
+// RUN: mlir-opt %S/versioned-op-1.12.mlirbc -emit-bytecode \
+// RUN:   -emit-bytecode-version=0 | mlir-opt -o %t.1 && \
+// RUN: mlir-opt %S/versioned-op-1.12.mlirbc -o %t.2 && \
+// RUN: diff %t.1 %t.2
+
diff --git a/mlir/test/Bytecode/versioning/versioned_op.mlir b/mlir/test/Bytecode/versioning/versioned_op.mlir
index b3141de67823f..5fa170bc24904 100644
--- a/mlir/test/Bytecode/versioning/versioned_op.mlir
+++ b/mlir/test/Bytecode/versioning/versioned_op.mlir
@@ -11,10 +11,10 @@
 // COM: bytecode contains
 // COM: module {
 // COM:   version: 2.0
-// COM:   "test.versionedA"() {dims = 123 : i64, modifier = false} : () -> ()
+// COM:   "test.versionedA"() <{dims = 123 : i64, modifier = false}> : () -> ()
 // COM: }
 // RUN: mlir-opt %S/versioned-op-2.0.mlirbc 2>&1 | FileCheck %s --check-prefix=CHECK1
-// CHECK1: "test.versionedA"() {dims = 123 : i64, modifier = false} : () -> ()
+// CHECK1: "test.versionedA"() <{dims = 123 : i64, modifier = false}> : () -> ()
 
 //===--------------------------------------------------------------------===//
 // Test upgrade
@@ -23,10 +23,10 @@
 // COM: bytecode contains
 // COM: module {
 // COM:   version: 1.12
-// COM:   "test.versionedA"() {dimensions = 123 : i64} : () -> ()
+// COM:   "test.versionedA"() <{dimensions = 123 : i64}> : () -> ()
 // COM: }
 // RUN: mlir-opt %S/versioned-op-1.12.mlirbc 2>&1 | FileCheck %s --check-prefix=CHECK2
-// CHECK2: "test.versionedA"() {dims = 123 : i64, modifier = false} : () -> ()
+// CHECK2: "test.versionedA"() <{dims = 123 : i64, modifier = false}> : () -> ()
 
 //===--------------------------------------------------------------------===//
 // Test forbidden downgrade
@@ -35,7 +35,7 @@
 // COM: bytecode contains
 // COM: module {
 // COM:   version: 2.2
-// COM:   "test.versionedA"() {dims = 123 : i64, modifier = false} : () -> ()
+// COM:   "test.versionedA"() <{dims = 123 : i64, modifier = false}> : () -> ()
 // COM: }
 // RUN: not mlir-opt %S/versioned-op-2.2.mlirbc 2>&1 | FileCheck %s --check-prefix=ERR_NEW_VERSION
 // ERR_NEW_VERSION: current test dialect version is 2.0, can't parse version: 2.2
diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c
index 5f205c4ff5e2b..5d4d63c9cda26 100644
--- a/mlir/test/CAPI/ir.c
+++ b/mlir/test/CAPI/ir.c
@@ -473,7 +473,7 @@ static void printFirstOfEach(MlirContext ctx, MlirOperation operation) {
   mlirOperationPrintWithFlags(operation, flags, printToStderr, NULL);
   fprintf(stderr, "\n");
   // clang-format off
-  // CHECK: Op print with all flags: %{{.*}} = "arith.constant"() {elts = dense_resource<__elided__> : tensor<4xi32>, value = 0 : index} : () -> index loc(unknown)
+  // CHECK: Op print with all flags: %{{.*}} = "arith.constant"() <{value = 0 : index}> {elts = dense_resource<__elided__> : tensor<4xi32>} : () -> index loc(unknown)
   // clang-format on
 
   mlirOpPrintingFlagsDestroy(flags);
@@ -1873,9 +1873,61 @@ int testOperands(void) {
     return 3;
   }
 
+  MlirOperationState op2State =
+      mlirOperationStateGet(mlirStringRefCreateFromCString("dummy.op2"), loc);
+  MlirValue initialOperands2[] = {constOneValue};
+  mlirOperationStateAddOperands(&op2State, 1, initialOperands2);
+  MlirOperation op2 = mlirOperationCreate(&op2State);
+
+  MlirOpOperand use3 = mlirValueGetFirstUse(constOneValue);
+  fprintf(stderr, "First use owner: ");
+  mlirOperationPrint(mlirOpOperandGetOwner(use3), printToStderr, NULL);
+  fprintf(stderr, "\n");
+  // CHECK: First use owner: "dummy.op2"
+
+  use3 = mlirOpOperandGetNextUse(mlirValueGetFirstUse(constOneValue));
+  fprintf(stderr, "Second use owner: ");
+  mlirOperationPrint(mlirOpOperandGetOwner(use3), printToStderr, NULL);
+  fprintf(stderr, "\n");
+  // CHECK: Second use owner: "dummy.op"
+
+  MlirAttribute indexTwoLiteral =
+      mlirAttributeParseGet(ctx, mlirStringRefCreateFromCString("2 : index"));
+  MlirNamedAttribute indexTwoValueAttr = mlirNamedAttributeGet(
+      mlirIdentifierGet(ctx, mlirStringRefCreateFromCString("value")),
+      indexTwoLiteral);
+  MlirOperationState constTwoState = mlirOperationStateGet(
+      mlirStringRefCreateFromCString("arith.constant"), loc);
+  mlirOperationStateAddResults(&constTwoState, 1, &indexType);
+  mlirOperationStateAddAttributes(&constTwoState, 1, &indexTwoValueAttr);
+  MlirOperation constTwo = mlirOperationCreate(&constTwoState);
+  MlirValue constTwoValue = mlirOperationGetResult(constTwo, 0);
+
+  mlirValueReplaceAllUsesOfWith(constOneValue, constTwoValue);
+
+  use3 = mlirValueGetFirstUse(constOneValue);
+  if (!mlirOpOperandIsNull(use3)) {
+    fprintf(stderr, "ERROR: Use should be null\n");
+    return 4;
+  }
+
+  MlirOpOperand use4 = mlirValueGetFirstUse(constTwoValue);
+  fprintf(stderr, "First replacement use owner: ");
+  mlirOperationPrint(mlirOpOperandGetOwner(use4), printToStderr, NULL);
+  fprintf(stderr, "\n");
+  // CHECK: First replacement use owner: "dummy.op"
+
+  use4 = mlirOpOperandGetNextUse(mlirValueGetFirstUse(constTwoValue));
+  fprintf(stderr, "Second replacement use owner: ");
+  mlirOperationPrint(mlirOpOperandGetOwner(use4), printToStderr, NULL);
+  fprintf(stderr, "\n");
+  // CHECK: Second replacement use owner: "dummy.op2"
+
   mlirOperationDestroy(op);
+  mlirOperationDestroy(op2);
   mlirOperationDestroy(constZero);
   mlirOperationDestroy(constOne);
+  mlirOperationDestroy(constTwo);
   mlirContextDestroy(ctx);
 
   return 0;
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index f0fc972a4a257..048291bd5b8d8 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -29,6 +29,7 @@ if (MLIR_INCLUDE_INTEGRATION_TESTS)
   option(MLIR_RUN_CUDA_TENSOR_CORE_TESTS "Run CUDA Tensor core WMMA tests.")
   option(MLIR_RUN_CUDA_SM80_TESTS "Run CUDA A100 tests.")
   option(MLIR_RUN_ARM_SVE_TESTS "Run Arm SVE tests.")
+  option(MLIR_RUN_ARM_SME_TESTS "Run Arm SME tests.")
 
 
   # The native target may not be enabled when cross compiling, raise an error.
@@ -52,6 +53,7 @@ llvm_canonicalize_cmake_booleans(
   MLIR_RUN_CUDA_TENSOR_CORE_TESTS
   MLIR_RUN_X86VECTOR_TESTS
   MLIR_RUN_ARM_SVE_TESTS
+  MLIR_RUN_ARM_SME_TESTS
   MLIR_RUN_CUDA_SM80_TESTS
   )
 
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
index a26add21cefb6..1dbb70b015337 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/amdgpu-to-rocdl.mlir
@@ -197,6 +197,35 @@ func.func @gpu_gcn_raw_buffer_atomic_umin_i32(%value: i32, %buf: memref<64xi32>,
   func.return
 }
 
+// CHECK-LABEL: func @amdgpu_raw_buffer_atomic_cmpswap_f32
+// CHECK-SAME: (%[[src:.*]]: f32, %[[cmp:.*]]: f32, {{.*}})
+func.func @amdgpu_raw_buffer_atomic_cmpswap_f32(%src : f32, %cmp : f32, %buf : memref<64xf32>, %idx: i32) -> f32 {
+  // CHECK: %[[srcCast:.*]] = llvm.bitcast %[[src]] : f32 to i32
+  // CHECK: %[[cmpCast:.*]] = llvm.bitcast %[[cmp]] : f32 to i32
+  // CHECK: %[[numRecords:.*]] = llvm.mlir.constant(256 : i32)
+  // CHECK: llvm.insertelement{{.*}}%[[numRecords]]
+  // CHECK: %[[word3:.*]] = llvm.mlir.constant(159744 : i32)
+  // CHECK: %[[resource:.*]] = llvm.insertelement{{.*}}%[[word3]]
+  // CHECK: %[[dst:.*]] = rocdl.raw.buffer.atomic.cmpswap(%[[srcCast]], %[[cmpCast]], %[[resource]], %{{.*}}, %{{.*}}, %{{.*}}) : i32, vector<4xi32>
+  // CHECK: %[[dstCast:.*]] = llvm.bitcast %[[dst]] : i32 to f32
+  // CHECK: return %[[dstCast]]
+  %dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : f32 -> memref<64xf32>, i32
+  func.return %dst : f32
+}
+
+// CHECK-LABEL: func @amdgpu_raw_buffer_atomic_cmpswap_i64
+// CHECK-SAME: (%[[src:.*]]: i64, %[[cmp:.*]]: i64, {{.*}})
+func.func @amdgpu_raw_buffer_atomic_cmpswap_i64(%src : i64, %cmp : i64, %buf : memref<64xi64>, %idx: i32) -> i64 {
+  // CHECK: %[[numRecords:.*]] = llvm.mlir.constant(512 : i32)
+  // CHECK: llvm.insertelement{{.*}}%[[numRecords]]
+  // CHECK: %[[word3:.*]] = llvm.mlir.constant(159744 : i32)
+  // CHECK: %[[resource:.*]] = llvm.insertelement{{.*}}%[[word3]]
+  // CHECK: %[[dst:.*]] = rocdl.raw.buffer.atomic.cmpswap(%[[src]], %[[cmp]], %[[resource]], %{{.*}}, %{{.*}}, %{{.*}}) : i64, vector<4xi32>
+  // CHECK: return %[[dst]]
+  %dst = amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true} %src, %cmp -> %buf[%idx] : i64 -> memref<64xi64>, i32
+  func.return %dst : i64
+}
+
 // CHECK-LABEL: func @lds_barrier
 func.func @lds_barrier() {
   // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "s_waitcnt lgkmcnt(0)\0As_barrier"
diff --git a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
index 2762019794d1a..2457cbcf3251c 100644
--- a/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
+++ b/mlir/test/Conversion/GPUCommon/memory-attrbution.mlir
@@ -225,3 +225,34 @@ gpu.module @kernel {
     "terminator"() : () -> ()
   }
 }
+
+// -----
+
+gpu.module @kernel {
+  // Check that alignment attributes are set correctly
+  // NVVM: llvm.mlir.global internal @[[$buffer:.*]]()
+  // NVVM-SAME:  addr_space = 3
+  // NVVM-SAME:  alignment = 8
+  // NVVM-SAME:  !llvm.array<48 x f32>
+
+  // ROCDL: llvm.mlir.global internal @[[$buffer:.*]]()
+  // ROCDL-SAME:  addr_space = 3
+  // ROCDL-SAME:  alignment = 8
+  // ROCDL-SAME:  !llvm.array<48 x f32>
+
+  // NVVM-LABEL: llvm.func @explicitAlign
+  // ROCDL-LABEL: llvm.func @explicitAlign
+  gpu.func @explicitAlign(%arg0 : index)
+    workgroup(%arg1: memref<48xf32, #gpu.address_space<workgroup>> {llvm.align = 8 : i64})
+    private(%arg2: memref<48xf32, #gpu.address_space<private>> {llvm.align = 4 : i64}) {
+    // NVVM: %[[size:.*]] = llvm.mlir.constant(48 : i64) : i64
+    // NVVM: %[[raw:.*]] = llvm.alloca %[[size]] x f32 {alignment = 4 : i64} : (i64) -> !llvm.ptr
+
+    // ROCDL: %[[size:.*]] = llvm.mlir.constant(48 : i64) : i64
+    // ROCDL: %[[raw:.*]] = llvm.alloca %[[size]] x f32 {alignment = 4 : i64} : (i64) -> !llvm.ptr<5>
+
+    %val = memref.load %arg1[%arg0] : memref<48xf32, #gpu.address_space<workgroup>>
+    memref.store %val, %arg2[%arg0] : memref<48xf32, #gpu.address_space<private>>
+    "terminator"() : () -> ()
+  }
+}
diff --git a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
index 7a21bf36b8537..0e655b5464d96 100644
--- a/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/expand-then-convert-to-llvm.mlir
@@ -668,3 +668,32 @@ func.func @collapse_static_shape_with_non_identity_layout(%arg: memref<1x1x8x8xf
   %1 = memref.collapse_shape %arg [[0, 1, 2, 3]] : memref<1x1x8x8xf32, strided<[64, 64, 8, 1], offset: ?>> into memref<64xf32, strided<[1], offset: ?>>
   return %1 : memref<64xf32, strided<[1], offset: ?>>
 }
+
+// -----
+
+// Check that the address of %arg0 appears with the same value
+// in both the llvm.assume and as base of the load.
+// This is to make sure that later CSEs and alignment propagation
+// will be able to do their job easily.
+
+// CHECK-LABEL: func @load_and_assume(
+// CHECK-SAME: %[[ARG0:.*]]: memref<?x?xf32, strided<[?, ?], offset: ?>>,
+// CHECK: %[[DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG0]] : memref<?x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[ALIGNED_PTR:.*]] = llvm.extractvalue %[[DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[DESC]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[BUFF_ADDR:.*]] = llvm.getelementptr %[[ALIGNED_PTR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[INT_TO_PTR:.*]] = llvm.ptrtoint %[[BUFF_ADDR]] : !llvm.ptr to i64
+// CHECK: %[[AND:.*]] = llvm.and %[[INT_TO_PTR]], {{.*}}  : i64
+// CHECK: %[[CMP:.*]] = llvm.icmp "eq" %[[AND]], {{.*}} : i64
+// CHECK: "llvm.intr.assume"(%[[CMP]]) : (i1) -> ()
+// CHECK: %[[LD_ADDR:.*]] = llvm.getelementptr %[[BUFF_ADDR]][%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+// CHECK: %[[VAL:.*]] = llvm.load %[[LD_ADDR]] : !llvm.ptr -> f32
+// CHECK: return %[[VAL]] : f32
+func.func @load_and_assume(
+    %arg0: memref<?x?xf32, strided<[?, ?], offset: ?>>,
+    %i0: index, %i1: index)
+    -> f32 {
+  memref.assume_alignment %arg0, 16 : memref<?x?xf32, strided<[?, ?], offset: ?>>
+  %2 = memref.load %arg0[%i0, %i1] : memref<?x?xf32, strided<[?, ?], offset: ?>>
+  func.return %2 : f32
+}
diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
index 4b4b3836a0075..5b58198bcfd82 100644
--- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
@@ -130,7 +130,7 @@ func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : in
 
 // -----
 
-// CHECK-LABEL: func @assume_alignment
+// CHECK-LABEL: func @assume_alignment(
 func.func @assume_alignment(%0 : memref<4x4xf16>) {
   // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
   // CHECK-NEXT: %[[ZERO:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -145,6 +145,22 @@ func.func @assume_alignment(%0 : memref<4x4xf16>) {
 
 // -----
 
+// CHECK-LABEL: func @assume_alignment_w_offset
+func.func @assume_alignment_w_offset(%0 : memref<4x4xf16, strided<[?, ?], offset: ?>>) {
+  // CHECK-DAG: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-DAG: %[[OFFSET:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-DAG: %[[BUFF_ADDR:.*]] =  llvm.getelementptr %[[PTR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, f16
+  // CHECK-DAG: %[[ZERO:.*]] = llvm.mlir.constant(0 : index) : i64
+  // CHECK-DAG: %[[MASK:.*]] = llvm.mlir.constant(15 : index) : i64
+  // CHECK-NEXT: %[[INT:.*]] = llvm.ptrtoint %[[BUFF_ADDR]] : !llvm.ptr to i64
+  // CHECK-NEXT: %[[MASKED_PTR:.*]] = llvm.and %[[INT]], %[[MASK:.*]] : i64
+  // CHECK-NEXT: %[[CONDITION:.*]] = llvm.icmp "eq" %[[MASKED_PTR]], %[[ZERO]] : i64
+  // CHECK-NEXT: "llvm.intr.assume"(%[[CONDITION]]) : (i1) -> ()
+  memref.assume_alignment %0, 16 : memref<4x4xf16, strided<[?, ?], offset: ?>>
+  return
+}
+// -----
+
 // CHECK-LABEL: func @dim_of_unranked
 // CHECK32-LABEL: func @dim_of_unranked
 func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index {
diff --git a/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir b/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir
index be2365215dce7..611ec0265cd37 100644
--- a/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir
+++ b/mlir/test/Conversion/OneToNTypeConversion/one-to-n-type-conversion.mlir
@@ -41,16 +41,16 @@ func.func @pack_unpack(%arg0: i1, %arg1: i2) -> (i1, i2) {
 //
 // CHECK-TUP-LABEL: func.func @materializations_tuple_args(
 // CHECK-TUP-SAME:                                         %[[ARG0:.*]]: tuple<tuple<>, i1, tuple<tuple<i2>>>) -> (i1, i2) {
-// CHECK-TUP-DAG:     %[[V0:.*]] = "test.get_tuple_element"(%[[ARG0]]) {index = 0 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-TUP-DAG:     %[[V1:.*]] = "test.get_tuple_element"(%[[ARG0]]) {index = 1 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-TUP-DAG:     %[[V2:.*]] = "test.get_tuple_element"(%[[ARG0]]) {index = 2 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-TUP-DAG:     %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-TUP-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 0 : i32} : (tuple<i2>) -> i2
-// CHECK-TUP-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[ARG0]]) {index = 0 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-TUP-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[ARG0]]) {index = 1 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-TUP-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[ARG0]]) {index = 2 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-TUP-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-TUP-DAG:     %[[V9:.*]] = "test.get_tuple_element"(%[[V8]]) {index = 0 : i32} : (tuple<i2>) -> i2
+// CHECK-TUP-DAG:     %[[V0:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
+// CHECK-TUP-DAG:     %[[V1:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
+// CHECK-TUP-DAG:     %[[V2:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
+// CHECK-TUP-DAG:     %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
+// CHECK-TUP-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
+// CHECK-TUP-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
+// CHECK-TUP-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
+// CHECK-TUP-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[ARG0]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
+// CHECK-TUP-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
+// CHECK-TUP-DAG:     %[[V9:.*]] = "test.get_tuple_element"(%[[V8]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
 // CHECK-TUP-DAG:     return %[[V1]], %[[V9]] : i1, i2
 
 // If we only convert the func ops, argument materializations are created from
@@ -64,11 +64,11 @@ func.func @pack_unpack(%arg0: i1, %arg1: i2) -> (i1, i2) {
 // CHECK-FUNC-DAG:     %[[V1:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
 // CHECK-FUNC-DAG:     %[[V2:.*]] = "test.make_tuple"(%[[V1]]) : (tuple<i2>) -> tuple<tuple<i2>>
 // CHECK-FUNC-DAG:     %[[V3:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]], %[[V2]]) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-// CHECK-FUNC-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 0 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-FUNC-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 1 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-FUNC-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 2 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-FUNC-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-FUNC-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) {index = 0 : i32} : (tuple<i2>) -> i2
+// CHECK-FUNC-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
+// CHECK-FUNC-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
+// CHECK-FUNC-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
+// CHECK-FUNC-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
+// CHECK-FUNC-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
 // CHECK-FUNC-DAG:     return %[[V5]], %[[V8]] : i1, i2
 
 // If we convert both tuple and func ops, basically everything disappears.
@@ -117,11 +117,11 @@ func.func @materializations_tuple_args(%arg0: tuple<tuple<>, i1, tuple<tuple<i2>
 // CHECK-FUNC-DAG:     %[[V1:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
 // CHECK-FUNC-DAG:     %[[V2:.*]] = "test.make_tuple"(%[[V1]]) : (tuple<i2>) -> tuple<tuple<i2>>
 // CHECK-FUNC-DAG:     %[[V3:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]], %[[V2]]) : (tuple<>, i1, tuple<tuple<i2>>) -> tuple<tuple<>, i1, tuple<tuple<i2>>>
-// CHECK-FUNC-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 0 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
-// CHECK-FUNC-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 1 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
-// CHECK-FUNC-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 2 : i32} : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK-FUNC-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK-FUNC-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) {index = 0 : i32} : (tuple<i2>) -> i2
+// CHECK-FUNC-DAG:     %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<>
+// CHECK-FUNC-DAG:     %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> i1
+// CHECK-FUNC-DAG:     %[[V6:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 2 : i32}> : (tuple<tuple<>, i1, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
+// CHECK-FUNC-DAG:     %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
+// CHECK-FUNC-DAG:     %[[V8:.*]] = "test.get_tuple_element"(%[[V7]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
 // CHECK-FUNC-DAG:     return %[[V5]], %[[V8]] : i1, i2
 
 // If we convert both tuple and func ops, basically everything disappears.
diff --git a/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir b/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir
index dd2013c9a7368..263711674a6ec 100644
--- a/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir
+++ b/mlir/test/Conversion/OneToNTypeConversion/scf-structural-one-to-n-type-conversion.mlir
@@ -36,13 +36,13 @@ func.func @if_result(%arg0: tuple<tuple<>, i1, tuple<i2>>, %arg1: i1) -> tuple<t
 // CHECK-NEXT:    %[[V1:.*]] = "test.make_tuple"(%[[V0]], %[[ARG0]]) : (tuple<>, i1) -> tuple<tuple<>, i1>
 // CHECK-NEXT:    %[[V2:.*]] = scf.if %[[ARG1]] -> (i1) {
 // CHECK-NEXT:      %[[V3:.*]] = "test.op"(%[[V1]]) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 0 : i32} : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 1 : i32} : (tuple<tuple<>, i1>) -> i1
+// CHECK-NEXT:      %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
+// CHECK-NEXT:      %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
 // CHECK-NEXT:      scf.yield %[[V5]] : i1
 // CHECK-NEXT:    } else {
 // CHECK-NEXT:      %[[V6:.*]] = "test.source"() : () -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 0 : i32} : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V8:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 1 : i32} : (tuple<tuple<>, i1>) -> i1
+// CHECK-NEXT:      %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
+// CHECK-NEXT:      %[[V8:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
 // CHECK-NEXT:      scf.yield %[[V8]] : i1
 // CHECK-NEXT:    }
 // CHECK-NEXT:    return %[[V2]] : i1
@@ -94,14 +94,14 @@ func.func @while_operands_results(%arg0: tuple<tuple<>, i1, tuple<i2>>, %arg1: i
 // CHECK-NEXT:      %[[V1:.*]] = "test.make_tuple"() : () -> tuple<>
 // CHECK-NEXT:      %[[V2:.*]] = "test.make_tuple"(%[[V1]], %[[ARG2]]) : (tuple<>, i1) -> tuple<tuple<>, i1>
 // CHECK-NEXT:      %[[V3:.*]] = "test.op"(%[[V2]]) : (tuple<tuple<>, i1>) -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 0 : i32} : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) {index = 1 : i32} : (tuple<tuple<>, i1>) -> i1
+// CHECK-NEXT:      %[[V4:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
+// CHECK-NEXT:      %[[V5:.*]] = "test.get_tuple_element"(%[[V3]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
 // CHECK-NEXT:      scf.condition(%[[ARG1]]) %[[V5]] : i1
 // CHECK-NEXT:    } do {
 // CHECK-NEXT:    ^bb0(%[[ARG3:.*]]: i1):
 // CHECK-NEXT:      %[[V6:.*]] = "test.source"() : () -> tuple<tuple<>, i1>
-// CHECK-NEXT:      %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 0 : i32} : (tuple<tuple<>, i1>) -> tuple<>
-// CHECK-NEXT:      %[[V8:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 1 : i32} : (tuple<tuple<>, i1>) -> i1
+// CHECK-NEXT:      %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<tuple<>, i1>) -> tuple<>
+// CHECK-NEXT:      %[[V8:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 1 : i32}> : (tuple<tuple<>, i1>) -> i1
 // CHECK-NEXT:      scf.yield %[[V8]] : i1
 // CHECK-NEXT:    }
 // CHECK-NEXT:    return %[[V0]] : i1
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
index 5ef44d27be61d..838e957ae16b9 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
@@ -318,7 +318,7 @@ func.func @avg_pool_i8(%arg0: tensor<1x6x34x62xi8>) -> (tensor<1x5x33x62xi8>) {
   // CHECK: %[[TRUNC_SHIFT:.+]] = arith.trunci %[[SUB]]
   // CHECK: %[[C30:.+]] = arith.constant 30
   // CHECK: %[[SHIFT:.+]] = arith.addi %[[TRUNC_SHIFT]], %[[C30]] : i8
-  // CHECK: %[[SCALED:.+]] = "tosa.apply_scale"(%[[IN]], %[[TRUNC_MUL]], %[[SHIFT]]) {double_round = false}
+  // CHECK: %[[SCALED:.+]] = "tosa.apply_scale"(%[[IN]], %[[TRUNC_MUL]], %[[SHIFT]]) <{double_round = false}>
 
   // Perform the normalization.
   // CHECK: %[[CMIN:.+]] = arith.constant -128
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index e9e9037ebcdb5..3e654ab9c56b0 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -137,8 +137,8 @@ func.func @test_broadcast_swapped_args(%arg0: tensor<2xf32>, %arg1: tensor<1xf32
 // CHECK-SAME:  %[[ARG1:[0-9a-zA-Z_]*]]
 func.func @test_multibroadcast(%arg0: tensor<1x3xf32>, %arg1: tensor<2x1xf32>) -> tensor<2x3xf32> {
   // CHECK: [[INIT:%.+]] = tensor.empty() : tensor<2x3xf32>
-  // CHECK: [[RESHAPE1:%.+]] = "tosa.reshape"(%[[ARG0]]) {new_shape = array<i64: 3>}
-  // CHECK: [[RESHAPE2:%.+]] = "tosa.reshape"(%[[ARG1]]) {new_shape = array<i64: 2>}
+  // CHECK: [[RESHAPE1:%.+]] = "tosa.reshape"(%[[ARG0]]) <{new_shape = array<i64: 3>}
+  // CHECK: [[RESHAPE2:%.+]] = "tosa.reshape"(%[[ARG1]]) <{new_shape = array<i64: 2>}
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]], #[[$MAP0]]], iterator_types = ["parallel", "parallel"]} ins([[RESHAPE1]], [[RESHAPE2]] : tensor<3xf32>, tensor<2xf32>) outs([[INIT]] : tensor<2x3xf32>) {
   // CHECK: ^bb0(%[[ARG2:.*]]: f32, %[[ARG3:.*]]: f32, %[[ARG4:.*]]: f32):
   // CHECK:   [[ELEMENT:%.+]] = arith.addf %[[ARG2]], %[[ARG3]] : f32
@@ -833,7 +833,7 @@ func.func @rescale_i8(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C22:%.+]] = arith.constant 22
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
-  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[C0]], [[C1]]) {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[C0]], [[C1]]) <{double_round = false}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant -128
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 127
@@ -854,7 +854,7 @@ func.func @rescale_i8(%arg0 : tensor<2xi8>) -> () {
   // CHECK: [[C22:%.+]] = arith.constant 22
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
-  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[C0]], [[C1]]) {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[C0]], [[C1]]) <{double_round = false}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant 0
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 255
@@ -927,7 +927,7 @@ func.func @rescale_ui8(%arg0 : tensor<2xui8>) -> () {
   // CHECK-DAG: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[IN]] : ui8 to i8
   // CHECK-DAG: [[IN32:%.+]] = arith.extui [[CAST]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C17]]
-  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[C0]], [[C1]]) {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[C0]], [[C1]]) <{double_round = false}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C22]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant -128
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 127
@@ -959,7 +959,7 @@ func.func @rescale_per_channel(%arg0 : tensor<3xi8>) -> (tensor<3xi8>) {
 
   // CHECK-DAG: [[IN32:%.+]] = arith.extsi [[IN]]
   // CHECK-DAG: [[IN_ZEROED:%.+]] = arith.subi [[IN32]], [[C243]]
-  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[MULTIPLIER]], [[SHIFT]]) {double_round = false}
+  // CHECK-DAG: [[SCALED:%.+]] = "tosa.apply_scale"([[IN_ZEROED]], [[MULTIPLIER]], [[SHIFT]]) <{double_round = false}
   // CHECK-DAG: [[SCALED_ZEROED:%.+]] = arith.addi [[SCALED]], [[C252]]
   // CHECK-DAG: [[CMIN:%.+]] = arith.constant -128
   // CHECK-DAG: [[CMAX:%.+]] = arith.constant 127
@@ -1065,21 +1065,21 @@ func.func @tile(%arg0 : tensor<2x3xi8>) -> () {
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs([[INIT]] : tensor<2x2x1x3xi8>)
   // CHECK: ^bb0(%[[ARG1:[0-9a-zA-Z_]+]]: i8
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: "tosa.reshape"([[GENERIC]]) {new_shape = array<i64: 4, 3>}
+  // CHECK: "tosa.reshape"([[GENERIC]]) <{new_shape = array<i64: 4, 3>}
   %0 = "tosa.tile"(%arg0) {multiples = array<i64: 2, 1>} : (tensor<2x3xi8>)  -> (tensor<4x3xi8>)
 
   // CHECK: [[INIT:%.+]] = tensor.empty()
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs([[INIT]] : tensor<1x2x2x3xi8>)
   // CHECK: ^bb0(%[[ARG1:[0-9a-zA-Z_]+]]: i8
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: "tosa.reshape"([[GENERIC]]) {new_shape = array<i64: 2, 6>}
+  // CHECK: "tosa.reshape"([[GENERIC]]) <{new_shape = array<i64: 2, 6>}
   %1 = "tosa.tile"(%arg0) {multiples = array<i64: 1, 2>} : (tensor<2x3xi8>)  -> (tensor<2x6xi8>)
 
   // CHECK: [[INIT:%.+]] = tensor.empty()
   // CHECK: [[GENERIC:%.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs([[INIT]] : tensor<5x2x7x3xi8>)
   // CHECK: ^bb0(%[[ARG1:[0-9a-zA-Z_]+]]: i8
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: "tosa.reshape"([[GENERIC]]) {new_shape = array<i64: 10, 21>}
+  // CHECK: "tosa.reshape"([[GENERIC]]) <{new_shape = array<i64: 10, 21>}
   %2 = "tosa.tile"(%arg0) {multiples = array<i64: 5, 7>} : (tensor<2x3xi8>)  -> (tensor<10x21xi8>)
 
   return
@@ -1099,7 +1099,7 @@ func.func @tile_dyn_input(%arg0 : tensor<?x3xi8>) -> () {
   // CHECK: %[[GENERIC:.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x3xi8>) outs(%[[INIT]] : tensor<2x?x1x3xi8>)
   // CHECK: ^bb0(%[[ARG1:.+]]: i8,
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: "tosa.reshape"(%[[GENERIC]]) {new_shape = array<i64: -9223372036854775808, 3>}
+  // CHECK: "tosa.reshape"(%[[GENERIC]]) <{new_shape = array<i64: -9223372036854775808, 3>}
   %0 = "tosa.tile"(%arg0) {multiples = array<i64: 2, 1>} : (tensor<?x3xi8>)  -> (tensor<?x3xi8>)
 
   return
@@ -1119,7 +1119,7 @@ func.func @tile_dyn_multiples(%arg0 : tensor<2x3xi8>) -> () {
   // CHECK: %[[GENERIC:.+]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG0]] : tensor<2x3xi8>) outs(%[[INIT]] : tensor<2x2x?x3xi8>)
   // CHECK: ^bb0(%[[ARG1:.+]]: i8,
   // CHECK:   linalg.yield %[[ARG1]] : i8
-  // CHECK: "tosa.reshape"(%[[GENERIC]]) {new_shape = array<i64: 2, -9223372036854775808>}
+  // CHECK: "tosa.reshape"(%[[GENERIC]]) <{new_shape = array<i64: 2, -9223372036854775808>}
   %0 = "tosa.tile"(%arg0) {multiples = array<i64: 2, -1>} : (tensor<2x3xi8>)  -> (tensor<2x?xi8>)
 
   return
diff --git a/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir b/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir
new file mode 100644
index 0000000000000..95355e13b8ecb
--- /dev/null
+++ b/mlir/test/Dialect/AMDGPU/amdgpu-emulate-atomics.mlir
@@ -0,0 +1,52 @@
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx90a %s | FileCheck %s --check-prefixes=CHECK,GFX9
+// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1030 %s | FileCheck %s --check-prefixes=CHECK,GFX10
+
+// -----
+
+func.func @atomic_fmax(%val: f32, %buffer: memref<?xf32>, %idx: i32) {
+// CHECK: func @atomic_fmax
+// CHECK-SAME: ([[val:%.+]]: f32, [[buffer:%.+]]: memref<?xf32>, [[idx:%.+]]: i32)
+// CHECK: gpu.printf "Begin\0A"
+// GFX10: amdgpu.raw_buffer_atomic_fmax {foo, indexOffset = 4 : i32} [[val]] -> [[buffer]][[[idx]]]
+// GFX9:  [[ld:%.+]] = amdgpu.raw_buffer_load {foo, indexOffset = 4 : i32} [[buffer]][[[idx]]]
+// GFX9:  cf.br [[loop:\^.+]]([[ld]] : f32)
+// GFX9:  [[loop]]([[arg:%.+]]: f32):
+// GFX9:  [[operated:%.+]] = arith.maxf [[val]], [[arg]]
+// GFX9:  [[atomicRes:%.+]] = amdgpu.raw_buffer_atomic_cmpswap {foo, indexOffset = 4 : i32} [[operated]], [[arg]] -> [[buffer]][[[idx]]]
+// GFX9:  [[argCast:%.+]] = arith.bitcast [[arg]] : f32 to i32
+// GFX9:  [[resCast:%.+]] = arith.bitcast [[atomicRes]] : f32 to i32
+// GFX9:  [[test:%.+]] = arith.cmpi eq, [[resCast]], [[argCast]]
+// GFX9:  cf.cond_br [[test]], [[post:\^.+]], [[loop]]([[atomicRes]] : f32)
+// GFX9:  [[post]]:
+// CHECK-NEXT: gpu.printf "End\0A"
+  gpu.printf "Begin\n"
+  amdgpu.raw_buffer_atomic_fmax {foo, indexOffset = 4 : i32} %val -> %buffer[%idx] : f32 -> memref<?xf32>, i32
+  gpu.printf "End\n"
+  func.return
+}
+
+// -----
+
+func.func @atomic_fmax_f64(%val: f64, %buffer: memref<?xf64>, %idx: i32) {
+// CHECK: func @atomic_fmax_f64
+// CHECK-SAME: ([[val:%.+]]: f64, [[buffer:%.+]]: memref<?xf64>, [[idx:%.+]]: i32)
+// CHECK: gpu.printf "Begin\0A"
+// GFX9:  amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]
+// GFX10: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]
+// CHECK-NEXT: gpu.printf "End\0A"
+  gpu.printf "Begin\n"
+  amdgpu.raw_buffer_atomic_fmax %val -> %buffer[%idx] : f64 -> memref<?xf64>, i32
+  gpu.printf "End\n"
+  func.return
+}
+
+// -----
+
+func.func @atomic_fadd(%val: f32, %buffer: memref<?xf32>, %idx: i32) {
+// CHECK: func @atomic_fadd
+// GFX9:  amdgpu.raw_buffer_atomic_fadd
+// GFX10: amdgpu.raw_buffer_load
+// GFX10: amdgpu.raw_buffer_atomic_cmpswap
+  amdgpu.raw_buffer_atomic_fadd %val -> %buffer[%idx] : f32 -> memref<?xf32>, i32
+  func.return
+}
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index 80bd7d4655e8b..a612fba56a763 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -74,6 +74,13 @@ func.func @raw_buffer_atomic_fadd_f32_to_rank_4(%value : f32, %dst : memref<128x
   func.return
 }
 
+// CHECK-LABEL: func @raw_buffer_atomic_cmpswap_f32
+func.func @raw_buffer_atomic_cmpswap_f32(%src : f32, %cmp : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {
+  // CHECK: amdgpu.raw_buffer_atomic_cmpswap {indexOffset = 1 : i32} %{{.*}}, %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
+  amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true, indexOffset = 1 : i32} %src, %cmp -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32
+  func.return
+}
+
 // CHECK-LABEL: func @lds_barrier
 func.func @lds_barrier() {
   // CHECK: amdgpu.lds_barrier
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index d6ca5033d6fc5..ace9798c368dc 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -1264,6 +1264,141 @@ func.func @simplify_div_mod_with_operands(%N: index, %A: memref<64xf32>, %unknow
 
 // -----
 
+#map0 = affine_map<(d0) -> (32, d0 * -32 + 32)>
+#map1 = affine_map<(d0) -> (32, d0 * -32 + 64)>
+#map3 = affine_map<(d0) -> (16, d0 * -16 + 32)>
+
+// CHECK-DAG: #[[$SIMPLE_MAP:.*]] = affine_map<()[s0] -> (3, s0)>
+// CHECK-DAG: #[[$SIMPLE_MAP_MAX:.*]] = affine_map<()[s0] -> (5, s0)>
+// CHECK-DAG: #[[$SIMPLIFIED_MAP:.*]] = affine_map<(d0, d1) -> (-9, d0 * 4 - d1 * 4)>
+// CHECK-DAG: #[[$FLOORDIV:.*]] = affine_map<(d0) -> (d0 floordiv 2)>
+
+// CHECK-LABEL: func @simplify_min_max_bounds_simple
+func.func @simplify_min_max_bounds_simple(%M: index) {
+
+  // CHECK-NEXT: affine.for %{{.*}} = 0 to min #[[$SIMPLE_MAP]]
+  affine.for %i = 0 to min affine_map<(d0) -> (3, 5, d0)>(%M) {
+    "test.foo"() : () -> ()
+  }
+
+  // CHECK: affine.for %{{.*}} = 0 to min #[[$SIMPLE_MAP]]
+  affine.for %i = 0 to min affine_map<(d0) -> (3, 3, d0)>(%M) {
+    "test.foo"() : () -> ()
+  }
+
+  // CHECK: affine.for %{{.*}} = max #[[$SIMPLE_MAP_MAX]]
+  affine.for %i = max affine_map<(d0) -> (3, 5, d0)>(%M) to 10 {
+    "test.foo"() : () -> ()
+  }
+
+  // CHECK: affine.for %{{.*}} = max #[[$SIMPLE_MAP_MAX]]
+  affine.for %i = max affine_map<(d0) -> (5, 5, d0)>(%M) to 10 {
+    "test.foo"() : () -> ()
+  }
+
+  return
+}
+
+// CHECK-LABEL: func @simplify_bounds_tiled
+func.func @simplify_bounds_tiled() {
+  affine.for %arg5 = 0 to 1 {
+    affine.for %arg6 = 0 to 2 {
+      affine.for %arg8 = 0 to min #map0(%arg5) step 16 {
+        affine.for %arg9 = 0 to min #map1(%arg6) step 16 {
+          affine.for %arg10 = 0 to 2 {
+            affine.for %arg12 = 0 to min #map3(%arg10) step 16 {
+              "test.foo"() : () -> ()
+            }
+          }
+        }
+      }
+    }
+  }
+  // CHECK:      affine.for
+  // CHECK-NEXT:   affine.for
+  // CHECK-NEXT:     affine.for %{{.*}} = 0 to 32 step 16
+  // CHECK-NEXT:       affine.for %{{.*}} = 0 to 32 step 16
+  // CHECK-NEXT:         affine.for %{{.*}} = 0 to 2
+  // CHECK-NEXT:           affine.for %{{.*}} = 0 to 16 step 16
+
+  return
+}
+
+// CHECK-LABEL: func @simplify_min_max_multi_expr
+func.func @simplify_min_max_multi_expr() {
+  // Lower bound max.
+  // CHECK: affine.for
+  affine.for %i = 0 to 2 {
+    // CHECK: affine.for %{{.*}} = 5 to
+    affine.for %j = max affine_map<(d0) -> (5, 4 * d0)> (%i) to affine_map<(d0) -> (4 * d0 + 3)>(%i) {
+      "test.foo"() : () -> ()
+    }
+  }
+
+  // Expressions with multiple operands.
+  // CHECK: affine.for
+  affine.for %i = 0 to 2 {
+    // CHECK: affine.for
+    affine.for %j = 0 to 4 {
+      // The first upper bound expression will not be lower than -9. So, it's redundant.
+      // CHECK-NEXT: affine.for %{{.*}} = -10 to -9
+      affine.for %k = -10 to min affine_map<(d0, d1) -> (4 * d0 - 3 * d1, -9)>(%i, %j) {
+        "test.foo"() : () -> ()
+      }
+    }
+  }
+
+  // One expression is redundant but not the others.
+  // CHECK: affine.for
+  affine.for %i = 0 to 2 {
+    // CHECK: affine.for
+    affine.for %j = 0 to 4 {
+      // The first upper bound expression will not be lower than -9. So, it's redundant.
+      // CHECK-NEXT: affine.for %{{.*}} = -10 to min #[[$SIMPLIFIED_MAP]]
+      affine.for %k = -10 to min affine_map<(d0, d1) -> (4 * d0 - 3 * d1, -9, 4 * d0 - 4 * d1)>(%i, %j) {
+        "test.foo"() : () -> ()
+      }
+    }
+  }
+
+  // CHECK: affine.for %{{.*}} = 0 to 1
+  affine.for %i = 0 to 2 {
+    affine.for %j = max affine_map<(d0) -> (d0 floordiv 2, 0)>(%i) to 1 {
+      "test.foo"() : () -> ()
+    }
+  }
+
+  // The constant bound is redundant here.
+  // CHECK: affine.for %{{.*}} = #[[$FLOORDIV]](%{{.*}} to 10
+  affine.for %i = 0 to 8 {
+    affine.for %j = max affine_map<(d0) -> (d0 floordiv 2, 0)>(%i) to 10 {
+      "test.foo"() : () -> ()
+    }
+  }
+
+  return
+}
+
+// CHECK-LABEL: func @no_simplify_min_max
+func.func @no_simplify_min_max(%M: index) {
+  // Negative test cases.
+  // CHECK: affine.for
+  affine.for %i = 0 to 4 {
+    // CHECK-NEXT: affine.for %{{.*}} = 0 to min
+    affine.for %j = 0 to min affine_map<(d0) -> (2 * d0, 2)>(%i) {
+      "test.foo"() : () -> ()
+    }
+    // CHECK:      affine.for %{{.*}} = 0 to min {{.*}}(%{{.*}})[%{{.*}}]
+    affine.for %j = 0 to min affine_map<(d0)[s0] -> (d0, s0)>(%i)[%M] {
+      "test.foo"() : () -> ()
+    }
+  }
+
+  return
+}
+
+// -----
+
 //           CHECK: #[[$map:.*]] = affine_map<()[s0] -> (s0 * ((-s0 + 40961) ceildiv 512))>
 // CHECK-BOTTOM-UP: #[[$map:.*]] = affine_map<()[s0] -> (s0 * ((-s0 + 40961) ceildiv 512))>
 //           CHECK-LABEL: func @regression_do_not_perform_invalid_replacements
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index 1f96876e2cf2c..14589b2915e94 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -629,6 +629,16 @@ func.func @truncExtui2(%arg0: i32) -> i16 {
   return %trunci : i16
 }
 
+// CHECK-LABEL: @truncExtui3
+//       CHECK:  %[[ARG0:.+]]: i8
+//       CHECK:  %[[CST:.*]] = arith.extui %[[ARG0:.+]] : i8 to i16
+//       CHECK:   return  %[[CST:.*]] : i16
+func.func @truncExtui3(%arg0: i8) -> i16 {
+  %extui = arith.extui %arg0 : i8 to i32
+  %trunci = arith.trunci %extui : i32 to i16
+  return %trunci : i16
+}
+
 // CHECK-LABEL: @truncExtuiVector
 //       CHECK:  %[[ARG0:.+]]: vector<2xi32>
 //       CHECK:  %[[CST:.*]] = arith.trunci %[[ARG0:.+]] : vector<2xi32> to vector<2xi16>
@@ -658,6 +668,16 @@ func.func @truncExtsi2(%arg0: i32) -> i16 {
   return %trunci : i16
 }
 
+// CHECK-LABEL: @truncExtsi3
+//       CHECK:  %[[ARG0:.+]]: i8
+//       CHECK:  %[[CST:.*]] = arith.extsi %[[ARG0:.+]] : i8 to i16
+//       CHECK:   return  %[[CST:.*]] : i16
+func.func @truncExtsi3(%arg0: i8) -> i16 {
+  %extsi = arith.extsi %arg0 : i8 to i32
+  %trunci = arith.trunci %extsi : i32 to i16
+  return %trunci : i16
+}
+
 // CHECK-LABEL: @truncExtsiVector
 //       CHECK:  %[[ARG0:.+]]: vector<2xi32>
 //       CHECK:  %[[CST:.*]] = arith.trunci %[[ARG0:.+]] : vector<2xi32> to vector<2xi16>
diff --git a/mlir/test/Dialect/Arith/int-narrowing.mlir b/mlir/test/Dialect/Arith/int-narrowing.mlir
new file mode 100644
index 0000000000000..484f6011e49ea
--- /dev/null
+++ b/mlir/test/Dialect/Arith/int-narrowing.mlir
@@ -0,0 +1,997 @@
+// RUN: mlir-opt --arith-int-narrowing="int-bitwidths-supported=1,8,16,24,32" \
+// RUN:          --verify-diagnostics %s | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// arith.addi
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @addi_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i16
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i16
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.addi %[[LHS]], %[[RHS]] : i16
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[ADD]] : i16 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @addi_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.addi %a, %b : i32
+  return %r : i32
+}
+
+// CHECK-LABEL: func.func @addi_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extui %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i16
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i16
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.addi %[[LHS]], %[[RHS]] : i16
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[ADD]] : i16 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @addi_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.addi %a, %b : i32
+  return %r : i32
+}
+
+// arith.addi produces one more bit of result than the operand bitwidth.
+//
+// CHECK-LABEL: func.func @addi_extsi_i24
+// CHECK-SAME:    (%[[ARG0:.+]]: i16, %[[ARG1:.+]]: i16)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i16 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i16 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i24
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i24
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.addi %[[LHS]], %[[RHS]] : i24
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[ADD]] : i24 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @addi_extsi_i24(%lhs: i16, %rhs: i16) -> i32 {
+  %a = arith.extsi %lhs : i16 to i32
+  %b = arith.extsi %rhs : i16 to i32
+  %r = arith.addi %a, %b : i32
+  return %r : i32
+}
+
+// This case should not get optimized because of mixed extensions.
+//
+// CHECK-LABEL: func.func @addi_mixed_ext_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.addi %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[ADD]] : i32
+func.func @addi_mixed_ext_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.addi %a, %b : i32
+  return %r : i32
+}
+
+// This case should not get optimized because we cannot reduce the bitwidth
+// below i16, given the pass options set.
+//
+// CHECK-LABEL: func.func @addi_extsi_i16
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i16
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i16
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.addi %[[EXT0]], %[[EXT1]] : i16
+// CHECK-NEXT:    return %[[ADD]] : i16
+func.func @addi_extsi_i16(%lhs: i8, %rhs: i8) -> i16 {
+  %a = arith.extsi %lhs : i8 to i16
+  %b = arith.extsi %rhs : i8 to i16
+  %r = arith.addi %a, %b : i16
+  return %r : i16
+}
+
+// CHECK-LABEL: func.func @addi_extsi_3xi8_cst
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi8>)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant dense<[-1, 127, 42]> : vector<3xi16>
+// CHECK-NEXT:    %[[EXT:.+]]  = arith.extsi %[[ARG0]] : vector<3xi8> to vector<3xi32>
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT]] : vector<3xi32> to vector<3xi16>
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.addi %[[LHS]], %[[CST]] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[ADD]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @addi_extsi_3xi8_cst(%lhs: vector<3xi8>) -> vector<3xi32> {
+  %cst = arith.constant dense<[-1, 127, 42]> : vector<3xi32>
+  %a = arith.extsi %lhs : vector<3xi8> to vector<3xi32>
+  %r = arith.addi %a, %cst : vector<3xi32>
+  return %r : vector<3xi32>
+}
+
+//===----------------------------------------------------------------------===//
+// arith.subi
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @subi_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i16
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i16
+// CHECK-NEXT:    %[[SUB:.+]]  = arith.subi %[[LHS]], %[[RHS]] : i16
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[SUB]] : i16 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @subi_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.subi %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.subi` ops with sign-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @subi_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extui %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[SUB:.+]]  = arith.subi %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[SUB]] : i32
+func.func @subi_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.subi %a, %b : i32
+  return %r : i32
+}
+
+// This case should not get optimized because of mixed extensions.
+//
+// CHECK-LABEL: func.func @subi_mixed_ext_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.subi %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[ADD]] : i32
+func.func @subi_mixed_ext_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.subi %a, %b : i32
+  return %r : i32
+}
+
+// arith.subi produces one more bit of result than the operand bitwidth.
+//
+// CHECK-LABEL: func.func @subi_extsi_i24
+// CHECK-SAME:    (%[[ARG0:.+]]: i16, %[[ARG1:.+]]: i16)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i16 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i16 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i24
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i24
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.subi %[[LHS]], %[[RHS]] : i24
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[ADD]] : i24 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @subi_extsi_i24(%lhs: i16, %rhs: i16) -> i32 {
+  %a = arith.extsi %lhs : i16 to i32
+  %b = arith.extsi %rhs : i16 to i32
+  %r = arith.subi %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// arith.muli
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @muli_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i16
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i16
+// CHECK-NEXT:    %[[MUL:.+]]  = arith.muli %[[LHS]], %[[RHS]] : i16
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[MUL]] : i16 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @muli_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.muli %a, %b : i32
+  return %r : i32
+}
+
+// CHECK-LABEL: func.func @muli_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extui %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i16
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i16
+// CHECK-NEXT:    %[[MUL:.+]]  = arith.muli %[[LHS]], %[[RHS]] : i16
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[MUL]] : i16 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @muli_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.muli %a, %b : i32
+  return %r : i32
+}
+
+// We do not expect this case to be optimized because given n-bit operands,
+// arith.muli produces 2n bits of result.
+//
+// CHECK-LABEL: func.func @muli_extsi_i32
+// CHECK-SAME:    (%[[ARG0:.+]]: i16, %[[ARG1:.+]]: i16)
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.extsi %[[ARG0]] : i16 to i32
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.extsi %[[ARG1]] : i16 to i32
+// CHECK-NEXT:    %[[RET:.+]]  = arith.muli %[[LHS]], %[[RHS]] : i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @muli_extsi_i32(%lhs: i16, %rhs: i16) -> i32 {
+  %a = arith.extsi %lhs : i16 to i32
+  %b = arith.extsi %rhs : i16 to i32
+  %r = arith.muli %a, %b : i32
+  return %r : i32
+}
+
+// This case should not get optimized because of mixed extensions.
+//
+// CHECK-LABEL: func.func @muli_mixed_ext_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[MUL:.+]]  = arith.muli %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[MUL]] : i32
+func.func @muli_mixed_ext_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.muli %a, %b : i32
+  return %r : i32
+}
+
+// CHECK-LABEL: func.func @muli_extsi_3xi8_cst
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi8>)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant dense<[-1, 127, 42]> : vector<3xi16>
+// CHECK-NEXT:    %[[EXT:.+]]  = arith.extsi %[[ARG0]] : vector<3xi8> to vector<3xi32>
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT]] : vector<3xi32> to vector<3xi16>
+// CHECK-NEXT:    %[[MUL:.+]]  = arith.muli %[[LHS]], %[[CST]] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[MUL]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @muli_extsi_3xi8_cst(%lhs: vector<3xi8>) -> vector<3xi32> {
+  %cst = arith.constant dense<[-1, 127, 42]> : vector<3xi32>
+  %a = arith.extsi %lhs : vector<3xi8> to vector<3xi32>
+  %r = arith.muli %a, %cst : vector<3xi32>
+  return %r : vector<3xi32>
+}
+
+//===----------------------------------------------------------------------===//
+// arith.divsi
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @divsi_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i16
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i16
+// CHECK-NEXT:    %[[SUB:.+]]  = arith.divsi %[[LHS]], %[[RHS]] : i16
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[SUB]] : i16 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @divsi_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.divsi %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.divsi` ops with sign-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @divsi_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extui %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[SUB:.+]]  = arith.divsi %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[SUB]] : i32
+func.func @divsi_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.divsi %a, %b : i32
+  return %r : i32
+}
+
+// arith.divsi produces one more bit of result than the operand bitwidth.
+//
+// CHECK-LABEL: func.func @divsi_extsi_i24
+// CHECK-SAME:    (%[[ARG0:.+]]: i16, %[[ARG1:.+]]: i16)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i16 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i16 to i32
+// CHECK-NEXT:    %[[LHS:.+]]  = arith.trunci %[[EXT0]] : i32 to i24
+// CHECK-NEXT:    %[[RHS:.+]]  = arith.trunci %[[EXT1]] : i32 to i24
+// CHECK-NEXT:    %[[ADD:.+]]  = arith.divsi %[[LHS]], %[[RHS]] : i24
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[ADD]] : i24 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @divsi_extsi_i24(%lhs: i16, %rhs: i16) -> i32 {
+  %a = arith.extsi %lhs : i16 to i32
+  %b = arith.extsi %rhs : i16 to i32
+  %r = arith.divsi %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// arith.divui
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @divui_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[SUB:.+]]  = arith.divui %[[ARG0]], %[[ARG1]] : i8
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[SUB]] : i8 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @divui_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.divui %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.divui` ops with zero-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @divui_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[SUB:.+]]  = arith.divui %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[SUB]] : i32
+func.func @divui_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.divui %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// arith.*itofp
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @sitofp_extsi_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[RET:.+]] = arith.sitofp %[[ARG]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @sitofp_extsi_i16(%a: i16) -> f16 {
+  %b = arith.extsi %a : i16 to i32
+  %f = arith.sitofp %b : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @sitofp_extsi_vector_i16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>)
+// CHECK-NEXT:    %[[RET:.+]] = arith.sitofp %[[ARG]] : vector<3xi16> to vector<3xf16>
+// CHECK-NEXT:    return %[[RET]] : vector<3xf16>
+func.func @sitofp_extsi_vector_i16(%a: vector<3xi16>) -> vector<3xf16> {
+  %b = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %f = arith.sitofp %b : vector<3xi32> to vector<3xf16>
+  return %f : vector<3xf16>
+}
+
+// CHECK-LABEL: func.func @sitofp_extsi_tensor_i16
+// CHECK-SAME:    (%[[ARG:.+]]: tensor<3x?xi16>)
+// CHECK-NEXT:    %[[RET:.+]] = arith.sitofp %[[ARG]] : tensor<3x?xi16> to tensor<3x?xf16>
+// CHECK-NEXT:    return %[[RET]] : tensor<3x?xf16>
+func.func @sitofp_extsi_tensor_i16(%a: tensor<3x?xi16>) -> tensor<3x?xf16> {
+  %b = arith.extsi %a : tensor<3x?xi16> to tensor<3x?xi32>
+  %f = arith.sitofp %b : tensor<3x?xi32> to tensor<3x?xf16>
+  return %f : tensor<3x?xf16>
+}
+
+// Narrowing to i64 is not enabled in pass options.
+//
+// CHECK-LABEL: func.func @sitofp_extsi_i64
+// CHECK-SAME:    (%[[ARG:.+]]: i64)
+// CHECK-NEXT:    %[[EXT:.+]] = arith.extsi %[[ARG]] : i64 to i128
+// CHECK-NEXT:    %[[RET:.+]] = arith.sitofp %[[EXT]] : i128 to f32
+// CHECK-NEXT:    return %[[RET]] : f32
+func.func @sitofp_extsi_i64(%a: i64) -> f32 {
+  %b = arith.extsi %a : i64 to i128
+  %f = arith.sitofp %b : i128 to f32
+  return %f : f32
+}
+
+// CHECK-LABEL: func.func @uitofp_extui_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[RET:.+]] = arith.uitofp %[[ARG]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @uitofp_extui_i16(%a: i16) -> f16 {
+  %b = arith.extui %a : i16 to i32
+  %f = arith.uitofp %b : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @sitofp_extsi_extsi_i8
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[RET:.+]] = arith.sitofp %[[ARG]] : i8 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @sitofp_extsi_extsi_i8(%a: i8) -> f16 {
+  %b = arith.extsi %a : i8 to i16
+  %c = arith.extsi %b : i16 to i32
+  %f = arith.sitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @uitofp_extui_extui_i8
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[RET:.+]] = arith.uitofp %[[ARG]] : i8 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @uitofp_extui_extui_i8(%a: i8) -> f16 {
+  %b = arith.extui %a : i8 to i16
+  %c = arith.extui %b : i16 to i32
+  %f = arith.uitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @uitofp_extsi_extui_i8
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[EXT:.+]] = arith.extsi %[[ARG]] : i8 to i16
+// CHECK-NEXT:    %[[RET:.+]] = arith.uitofp %[[EXT]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @uitofp_extsi_extui_i8(%a: i8) -> f16 {
+  %b = arith.extsi %a : i8 to i16
+  %c = arith.extui %b : i16 to i32
+  %f = arith.uitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @uitofp_trunci_extui_i8
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[TR:.+]]  = arith.trunci %[[ARG]] : i16 to i8
+// CHECK-NEXT:    %[[RET:.+]] = arith.uitofp %[[TR]] : i8 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @uitofp_trunci_extui_i8(%a: i16) -> f16 {
+  %b = arith.trunci %a : i16 to i8
+  %c = arith.extui %b : i8 to i32
+  %f = arith.uitofp %c : i32 to f16
+  return %f : f16
+}
+
+// This should not be folded because arith.extui changes the signed
+// range of the number. For example:
+//  extsi -1 : i16 to i32 ==> -1
+//  extui -1 : i16 to i32 ==> U16_MAX
+//
+/// CHECK-LABEL: func.func @sitofp_extui_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[EXT:.+]] = arith.extui %[[ARG]] : i16 to i32
+// CHECK-NEXT:    %[[RET:.+]] = arith.sitofp %[[EXT]] : i32 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @sitofp_extui_i16(%a: i16) -> f16 {
+  %b = arith.extui %a : i16 to i32
+  %f = arith.sitofp %b : i32 to f16
+  return %f : f16
+}
+
+// This should not be folded because arith.extsi changes the unsigned
+// range of the number. For example:
+//  extsi -1 : i16 to i32 ==> U32_MAX
+//  extui -1 : i16 to i32 ==> U16_MAX
+//
+// CHECK-LABEL: func.func @uitofp_extsi_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[EXT:.+]] = arith.extsi %[[ARG]] : i16 to i32
+// CHECK-NEXT:    %[[RET:.+]] = arith.uitofp %[[EXT]] : i32 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @uitofp_extsi_i16(%a: i16) -> f16 {
+  %b = arith.extsi %a : i16 to i32
+  %f = arith.uitofp %b : i32 to f16
+  return %f : f16
+}
+
+//===----------------------------------------------------------------------===//
+// arith.maxsi
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @maxsi_extsi_i8
+// CHECK-SAME:    (%[[LHS:.+]]: i8, %[[RHS:.+]]: i8)
+// CHECK-NEXT:    %[[MAX:.+]]  = arith.maxsi %[[LHS]], %[[RHS]] : i8
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[MAX]] : i8 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @maxsi_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.maxsi %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.maxsi` ops with sign-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @maxsi_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extui %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[MAX:.+]]  = arith.maxsi %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[MAX]] : i32
+func.func @maxsi_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.maxsi %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// arith.maxui
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @maxui_extui_i8
+// CHECK-SAME:    (%[[LHS:.+]]: i8, %[[RHS:.+]]: i8)
+// CHECK-NEXT:    %[[MAX:.+]]  = arith.maxui %[[LHS]], %[[RHS]] : i8
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[MAX]] : i8 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @maxui_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.maxui %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.maxsi` ops with zero-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @maxui_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[MAX:.+]]  = arith.maxui %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[MAX]] : i32
+func.func @maxui_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.maxui %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// arith.minsi
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @minsi_extsi_i8
+// CHECK-SAME:    (%[[LHS:.+]]: i8, %[[RHS:.+]]: i8)
+// CHECK-NEXT:    %[[min:.+]]  = arith.minsi %[[LHS]], %[[RHS]] : i8
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[min]] : i8 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @minsi_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.minsi %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.minsi` ops with sign-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @minsi_extui_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extui %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extui %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[min:.+]]  = arith.minsi %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[min]] : i32
+func.func @minsi_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.minsi %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// arith.minui
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @minui_extui_i8
+// CHECK-SAME:    (%[[LHS:.+]]: i8, %[[RHS:.+]]: i8)
+// CHECK-NEXT:    %[[min:.+]]  = arith.minui %[[LHS]], %[[RHS]] : i8
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[min]] : i8 to i32
+// CHECK-NEXT:    return %[[RET]] : i32
+func.func @minui_extui_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extui %lhs : i8 to i32
+  %b = arith.extui %rhs : i8 to i32
+  %r = arith.minui %a, %b : i32
+  return %r : i32
+}
+
+// This patterns should only apply to `arith.minsi` ops with zero-extended
+// arguments.
+//
+// CHECK-LABEL: func.func @minui_extsi_i8
+// CHECK-SAME:    (%[[ARG0:.+]]: i8, %[[ARG1:.+]]: i8)
+// CHECK-NEXT:    %[[EXT0:.+]] = arith.extsi %[[ARG0]] : i8 to i32
+// CHECK-NEXT:    %[[EXT1:.+]] = arith.extsi %[[ARG1]] : i8 to i32
+// CHECK-NEXT:    %[[min:.+]]  = arith.minui %[[EXT0]], %[[EXT1]] : i32
+// CHECK-NEXT:    return %[[min]] : i32
+func.func @minui_extsi_i8(%lhs: i8, %rhs: i8) -> i32 {
+  %a = arith.extsi %lhs : i8 to i32
+  %b = arith.extsi %rhs : i8 to i32
+  %r = arith.minui %a, %b : i32
+  return %r : i32
+}
+
+//===----------------------------------------------------------------------===//
+// Commute Extension over Vector Ops
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @extsi_over_extract_3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extract %[[ARG]][1] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.sitofp %[[EXTR]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @extsi_over_extract_3xi16(%a: vector<3xi16>) -> f16 {
+  %b = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %c = vector.extract %b[1] : vector<3xi32>
+  %f = arith.sitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @extui_over_extract_3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extract %[[ARG]][1] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.uitofp %[[EXTR]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @extui_over_extract_3xi16(%a: vector<3xi16>) -> f16 {
+  %b = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %c = vector.extract %b[1] : vector<3xi32>
+  %f = arith.uitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @extsi_over_extractelement_3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>, %[[POS:.+]]: i32)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extractelement %[[ARG]][%[[POS]] : i32] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.sitofp %[[EXTR]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @extsi_over_extractelement_3xi16(%a: vector<3xi16>, %pos: i32) -> f16 {
+  %b = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %c = vector.extractelement %b[%pos : i32] : vector<3xi32>
+  %f = arith.sitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @extui_over_extractelement_3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>, %[[POS:.+]]: i32)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extractelement %[[ARG]][%[[POS]] : i32] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.uitofp %[[EXTR]] : i16 to f16
+// CHECK-NEXT:    return %[[RET]] : f16
+func.func @extui_over_extractelement_3xi16(%a: vector<3xi16>, %pos: i32) -> f16 {
+  %b = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %c = vector.extractelement %b[%pos : i32] : vector<3xi32>
+  %f = arith.uitofp %c : i32 to f16
+  return %f : f16
+}
+
+// CHECK-LABEL: func.func @extsi_over_extract_strided_slice_1d
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extract_strided_slice %[[ARG]] {offsets = [1], sizes = [2], strides = [1]} : vector<3xi16> to vector<2xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[EXTR]] : vector<2xi16> to vector<2xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2xi32>
+func.func @extsi_over_extract_strided_slice_1d(%a: vector<3xi16>) -> vector<2xi32> {
+  %b = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %c = vector.extract_strided_slice %b
+   {offsets = [1], sizes = [2], strides = [1]} : vector<3xi32> to vector<2xi32>
+  return %c : vector<2xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_extract_strided_slice_1d
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extract_strided_slice %[[ARG]] {offsets = [1], sizes = [2], strides = [1]} : vector<3xi16> to vector<2xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[EXTR]] : vector<2xi16> to vector<2xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2xi32>
+func.func @extui_over_extract_strided_slice_1d(%a: vector<3xi16>) -> vector<2xi32> {
+  %b = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %c = vector.extract_strided_slice %b
+   {offsets = [1], sizes = [2], strides = [1]} : vector<3xi32> to vector<2xi32>
+  return %c : vector<2xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_extract_strided_slice_2d
+// CHECK-SAME:    (%[[ARG:.+]]: vector<2x3xi16>)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 2], strides = [1, 1]} : vector<2x3xi16> to vector<1x2xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[EXTR]] : vector<1x2xi16> to vector<1x2xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<1x2xi32>
+func.func @extsi_over_extract_strided_slice_2d(%a: vector<2x3xi16>) -> vector<1x2xi32> {
+  %b = arith.extsi %a : vector<2x3xi16> to vector<2x3xi32>
+  %c = vector.extract_strided_slice %b
+   {offsets = [1, 1], sizes = [1, 2], strides = [1, 1]} : vector<2x3xi32> to vector<1x2xi32>
+  return %c : vector<1x2xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_extract_strided_slice_2d
+// CHECK-SAME:    (%[[ARG:.+]]: vector<2x3xi16>)
+// CHECK-NEXT:    %[[EXTR:.+]] = vector.extract_strided_slice %arg0 {offsets = [1, 1], sizes = [1, 2], strides = [1, 1]} : vector<2x3xi16> to vector<1x2xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[EXTR]] : vector<1x2xi16> to vector<1x2xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<1x2xi32>
+func.func @extui_over_extract_strided_slice_2d(%a: vector<2x3xi16>) -> vector<1x2xi32> {
+  %b = arith.extui %a : vector<2x3xi16> to vector<2x3xi32>
+  %c = vector.extract_strided_slice %b
+   {offsets = [1, 1], sizes = [1, 2], strides = [1, 1]} : vector<2x3xi32> to vector<1x2xi32>
+  return %c : vector<1x2xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insert_3xi16
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi16>, %[[ARG1:.+]]: i16)
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert %[[ARG1]], %[[ARG0]] [1] : i16 into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extsi %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insert_3xi16(%a: vector<3xi16>, %b: i16) -> vector<3xi32> {
+  %c = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %d = arith.extsi %b : i16 to i32
+  %e = vector.insert %d, %c [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insert_3xi16
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi16>, %[[ARG1:.+]]: i16)
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert %[[ARG1]], %[[ARG0]] [1] : i16 into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extui %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extui_over_insert_3xi16(%a: vector<3xi16>, %b: i16) -> vector<3xi32> {
+  %c = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %d = arith.extui %b : i16 to i32
+  %e = vector.insert %d, %c [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insert_3xi16_cst_0
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[CST:.+]] = arith.constant dense<0> : vector<3xi16>
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert %[[ARG]], %[[CST]] [1] : i16 into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extsi %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insert_3xi16_cst_0(%a: i16) -> vector<3xi32> {
+  %cst = arith.constant dense<0> : vector<3xi32>
+  %d = arith.extsi %a : i16 to i32
+  %e = vector.insert %d, %cst [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insert_3xi8_cst
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[CST:.+]] = arith.constant dense<[-1, 127, -128]> : vector<3xi8>
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert %[[ARG]], %[[CST]] [1] : i8 into vector<3xi8>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extsi %[[INS]] : vector<3xi8> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insert_3xi8_cst(%a: i8) -> vector<3xi32> {
+  %cst = arith.constant dense<[-1, 127, -128]> : vector<3xi32>
+  %d = arith.extsi %a : i8 to i32
+  %e = vector.insert %d, %cst [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insert_3xi8_cst
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[CST:.+]] = arith.constant dense<[1, 127, -1]> : vector<3xi8>
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert %[[ARG]], %[[CST]] [1] : i8 into vector<3xi8>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extui %[[INS]] : vector<3xi8> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extui_over_insert_3xi8_cst(%a: i8) -> vector<3xi32> {
+  %cst = arith.constant dense<[1, 127, 255]> : vector<3xi32>
+  %d = arith.extui %a : i8 to i32
+  %e = vector.insert %d, %cst [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insert_3xi16_cst_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant dense<[-1, 128, 0]> : vector<3xi16>
+// CHECK-NEXT:    %[[SRCE:.+]] = arith.extsi %[[ARG]] : i8 to i32
+// CHECK-NEXT:    %[[SRCT:.+]] = arith.trunci %[[SRCE]] : i32 to i16
+// CHECK-NEXT:    %[[INS:.+]]  = vector.insert %[[SRCT]], %[[CST]] [1] : i16 into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insert_3xi16_cst_i16(%a: i8) -> vector<3xi32> {
+  %cst = arith.constant dense<[-1, 128, 0]> : vector<3xi32>
+  %d = arith.extsi %a : i8 to i32
+  %e = vector.insert %d, %cst [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insert_3xi16_cst_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i8)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant dense<[1, 256, 0]> : vector<3xi16>
+// CHECK-NEXT:    %[[SRCE:.+]] = arith.extui %[[ARG]] : i8 to i32
+// CHECK-NEXT:    %[[SRCT:.+]] = arith.trunci %[[SRCE]] : i32 to i16
+// CHECK-NEXT:    %[[INS:.+]]  = vector.insert %[[SRCT]], %[[CST]] [1] : i16 into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extui_over_insert_3xi16_cst_i16(%a: i8) -> vector<3xi32> {
+  %cst = arith.constant dense<[1, 256, 0]> : vector<3xi32>
+  %d = arith.extui %a : i8 to i32
+  %e = vector.insert %d, %cst [1] : i32 into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insertelement_3xi16
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi16>, %[[ARG1:.+]]: i16, %[[POS:.+]]: i32)
+// CHECK-NEXT:    %[[INS:.+]] = vector.insertelement %[[ARG1]], %[[ARG0]][%[[POS]] : i32] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extsi %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insertelement_3xi16(%a: vector<3xi16>, %b: i16, %pos: i32) -> vector<3xi32> {
+  %c = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %d = arith.extsi %b : i16 to i32
+  %e = vector.insertelement %d, %c[%pos : i32] : vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insertelement_3xi16
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi16>, %[[ARG1:.+]]: i16, %[[POS:.+]]: i32)
+// CHECK-NEXT:    %[[INS:.+]] = vector.insertelement %[[ARG1]], %[[ARG0]][%[[POS]] : i32] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extui %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extui_over_insertelement_3xi16(%a: vector<3xi16>, %b: i16, %pos: i32) -> vector<3xi32> {
+  %c = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %d = arith.extui %b : i16 to i32
+  %e = vector.insertelement %d, %c[%pos : i32] : vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insertelement_3xi16_cst_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i8, %[[POS:.+]]: i32)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant dense<[-1, 128, 0]> : vector<3xi16>
+// CHECK-NEXT:    %[[SRCE:.+]] = arith.extsi %[[ARG]] : i8 to i32
+// CHECK-NEXT:    %[[SRCT:.+]] = arith.trunci %[[SRCE]] : i32 to i16
+// CHECK-NEXT:    %[[INS:.+]] = vector.insertelement %[[SRCT]], %[[CST]][%[[POS]] : i32] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insertelement_3xi16_cst_i16(%a: i8, %pos: i32) -> vector<3xi32> {
+  %cst = arith.constant dense<[-1, 128, 0]> : vector<3xi32>
+  %d = arith.extsi %a : i8 to i32
+  %e = vector.insertelement %d, %cst[%pos : i32] : vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insertelement_3xi16_cst_i16
+// CHECK-SAME:    (%[[ARG:.+]]: i8, %[[POS:.+]]: i32)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant dense<[1, 256, 0]> : vector<3xi16>
+// CHECK-NEXT:    %[[SRCE:.+]] = arith.extui %[[ARG]] : i8 to i32
+// CHECK-NEXT:    %[[SRCT:.+]] = arith.trunci %[[SRCE]] : i32 to i16
+// CHECK-NEXT:    %[[INS:.+]] = vector.insertelement %[[SRCT]], %[[CST]][%[[POS]] : i32] : vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extui_over_insertelement_3xi16_cst_i16(%a: i8, %pos: i32) -> vector<3xi32> {
+  %cst = arith.constant dense<[1, 256, 0]> : vector<3xi32>
+  %d = arith.extui %a : i8 to i32
+  %e = vector.insertelement %d, %cst[%pos : i32] : vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insert_strided_slice_1d
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi16>, %[[ARG1:.+]]: vector<2xi16>)
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert_strided_slice %[[ARG1]], %[[ARG0]]
+// CHECK-SAME:                    {offsets = [1], strides = [1]} : vector<2xi16> into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extsi %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_insert_strided_slice_1d(%a: vector<3xi16>, %b: vector<2xi16>) -> vector<3xi32> {
+  %c = arith.extsi %a : vector<3xi16> to vector<3xi32>
+  %d = arith.extsi %b : vector<2xi16> to vector<2xi32>
+  %e = vector.insert_strided_slice %d, %c {offsets = [1], strides = [1]} : vector<2xi32> into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insert_strided_slice_1d
+// CHECK-SAME:    (%[[ARG0:.+]]: vector<3xi16>, %[[ARG1:.+]]: vector<2xi16>)
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert_strided_slice %[[ARG1]], %[[ARG0]]
+// CHECK-SAME:                    {offsets = [1], strides = [1]} : vector<2xi16> into vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]] = arith.extui %[[INS]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extui_over_insert_strided_slice_1d(%a: vector<3xi16>, %b: vector<2xi16>) -> vector<3xi32> {
+  %c = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %d = arith.extui %b : vector<2xi16> to vector<2xi32>
+  %e = vector.insert_strided_slice %d, %c {offsets = [1], strides = [1]} : vector<2xi32> into vector<3xi32>
+  return %e : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_insert_strided_slice_cst_2d
+// CHECK-SAME:    (%[[ARG:.+]]: vector<1x2xi8>)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant
+// CHECK-SAME{LITERAL}:            dense<[[-1, 128, 0], [-129, 42, 1337]]> : vector<2x3xi16>
+// CHECK-NEXT:    %[[SRCE:.+]] = arith.extsi %[[ARG]] : vector<1x2xi8> to vector<1x2xi32>
+// CHECK-NEXT:    %[[SRCT:.+]] = arith.trunci %[[SRCE]] : vector<1x2xi32> to vector<1x2xi16>
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert_strided_slice %[[SRCT]], %[[CST]]
+// CHECK-SAME:                    {offsets = [0, 1], strides = [1, 1]} : vector<1x2xi16> into vector<2x3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[INS]] : vector<2x3xi16> to vector<2x3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2x3xi32>
+func.func @extsi_over_insert_strided_slice_cst_2d(%a: vector<1x2xi8>) -> vector<2x3xi32> {
+  %cst = arith.constant dense<[[-1, 128, 0], [-129, 42, 1337]]> : vector<2x3xi32>
+  %d = arith.extsi %a : vector<1x2xi8> to vector<1x2xi32>
+  %e = vector.insert_strided_slice %d, %cst {offsets = [0, 1], strides = [1, 1]} : vector<1x2xi32> into vector<2x3xi32>
+  return %e : vector<2x3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_insert_strided_slice_cst_2d
+// CHECK-SAME:    (%[[ARG:.+]]: vector<1x2xi8>)
+// CHECK-NEXT:    %[[CST:.+]]  = arith.constant
+// CHECK-SAME{LITERAL}:            dense<[[1, 128, 0], [256, 42, 1337]]> : vector<2x3xi16>
+// CHECK-NEXT:    %[[SRCE:.+]] = arith.extui %[[ARG]] : vector<1x2xi8> to vector<1x2xi32>
+// CHECK-NEXT:    %[[SRCT:.+]] = arith.trunci %[[SRCE]] : vector<1x2xi32> to vector<1x2xi16>
+// CHECK-NEXT:    %[[INS:.+]] = vector.insert_strided_slice %[[SRCT]], %[[CST]]
+// CHECK-SAME:                    {offsets = [0, 1], strides = [1, 1]} : vector<1x2xi16> into vector<2x3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[INS]] : vector<2x3xi16> to vector<2x3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2x3xi32>
+func.func @extui_over_insert_strided_slice_cst_2d(%a: vector<1x2xi8>) -> vector<2x3xi32> {
+  %cst = arith.constant dense<[[1, 128, 0], [256, 42, 1337]]> : vector<2x3xi32>
+  %d = arith.extui %a : vector<1x2xi8> to vector<1x2xi32>
+  %e = vector.insert_strided_slice %d, %cst {offsets = [0, 1], strides = [1, 1]} : vector<1x2xi32> into vector<2x3xi32>
+  return %e : vector<2x3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_broadcast_3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: i16)
+// CHECK-NEXT:    %[[BCST:.+]] = vector.broadcast %[[ARG]] : i16 to vector<3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[BCST]] : vector<3xi16> to vector<3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3xi32>
+func.func @extsi_over_broadcast_3xi16(%a: i16) -> vector<3xi32> {
+  %b = arith.extsi %a : i16 to i32
+  %r = vector.broadcast %b : i32 to vector<3xi32>
+  return %r : vector<3xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_broadcast_2x3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<3xi16>)
+// CHECK-NEXT:    %[[BCST:.+]] = vector.broadcast %[[ARG]] : vector<3xi16> to vector<2x3xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[BCST]] : vector<2x3xi16> to vector<2x3xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2x3xi32>
+func.func @extui_over_broadcast_2x3xi16(%a: vector<3xi16>) -> vector<2x3xi32> {
+  %b = arith.extui %a : vector<3xi16> to vector<3xi32>
+  %r = vector.broadcast %b : vector<3xi32> to vector<2x3xi32>
+  return %r : vector<2x3xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_shape_cast_2x3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<2x3xi16>)
+// CHECK-NEXT:    %[[CAST:.+]] = vector.shape_cast %[[ARG]] : vector<2x3xi16> to vector<3x2xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[CAST]] : vector<3x2xi16> to vector<3x2xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3x2xi32>
+func.func @extsi_over_shape_cast_2x3xi16(%a: vector<2x3xi16>) -> vector<3x2xi32> {
+  %b = arith.extsi %a : vector<2x3xi16> to vector<2x3xi32>
+  %r = vector.shape_cast %b : vector<2x3xi32> to vector<3x2xi32>
+  return %r : vector<3x2xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_shape_cast_5x2x3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<5x2x3xi16>)
+// CHECK-NEXT:    %[[CAST:.+]] = vector.shape_cast %[[ARG]] : vector<5x2x3xi16> to vector<2x3x5xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[CAST]] : vector<2x3x5xi16> to vector<2x3x5xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2x3x5xi32>
+func.func @extui_over_shape_cast_5x2x3xi16(%a: vector<5x2x3xi16>) -> vector<2x3x5xi32> {
+  %b = arith.extui %a : vector<5x2x3xi16> to vector<5x2x3xi32>
+  %r = vector.shape_cast %b : vector<5x2x3xi32> to vector<2x3x5xi32>
+  return %r : vector<2x3x5xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_transpose_2x3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<2x3xi16>)
+// CHECK-NEXT:    %[[TRAN:.+]] = vector.transpose %[[ARG]], [1, 0] : vector<2x3xi16> to vector<3x2xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[TRAN]] : vector<3x2xi16> to vector<3x2xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<3x2xi32>
+func.func @extsi_over_transpose_2x3xi16(%a: vector<2x3xi16>) -> vector<3x2xi32> {
+  %b = arith.extsi %a : vector<2x3xi16> to vector<2x3xi32>
+  %r = vector.transpose %b, [1, 0] : vector<2x3xi32> to vector<3x2xi32>
+  return %r : vector<3x2xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_transpose_5x2x3xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<5x2x3xi16>)
+// CHECK-NEXT:    %[[TRAN:.+]] = vector.transpose %[[ARG]], [1, 2, 0] : vector<5x2x3xi16> to vector<2x3x5xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[TRAN]] : vector<2x3x5xi16> to vector<2x3x5xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<2x3x5xi32>
+func.func @extui_over_transpose_5x2x3xi16(%a: vector<5x2x3xi16>) -> vector<2x3x5xi32> {
+  %b = arith.extui %a : vector<5x2x3xi16> to vector<5x2x3xi32>
+  %r = vector.transpose %b, [1, 2, 0] : vector<5x2x3xi32> to vector<2x3x5xi32>
+  return %r : vector<2x3x5xi32>
+}
+
+// CHECK-LABEL: func.func @extsi_over_flat_transpose_16xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<16xi16>)
+// CHECK-NEXT:    %[[TRAN:.+]] = vector.flat_transpose %[[ARG]] {columns = 4 : i32, rows = 4 : i32} : vector<16xi16> -> vector<16xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extsi %[[TRAN]] : vector<16xi16> to vector<16xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<16xi32>
+func.func @extsi_over_flat_transpose_16xi16(%a: vector<16xi16>) -> vector<16xi32> {
+  %b = arith.extsi %a : vector<16xi16> to vector<16xi32>
+  %r = vector.flat_transpose %b {columns = 4 : i32, rows = 4 : i32} : vector<16xi32> -> vector<16xi32>
+  return %r : vector<16xi32>
+}
+
+// CHECK-LABEL: func.func @extui_over_flat_transpose_16xi16
+// CHECK-SAME:    (%[[ARG:.+]]: vector<16xi16>)
+// CHECK-NEXT:    %[[TRAN:.+]] = vector.flat_transpose %[[ARG]] {columns = 8 : i32, rows = 2 : i32} : vector<16xi16> -> vector<16xi16>
+// CHECK-NEXT:    %[[RET:.+]]  = arith.extui %[[TRAN]] : vector<16xi16> to vector<16xi32>
+// CHECK-NEXT:    return %[[RET]] : vector<16xi32>
+func.func @extui_over_flat_transpose_16xi16(%a: vector<16xi16>) -> vector<16xi32> {
+  %b = arith.extui %a : vector<16xi16> to vector<16xi32>
+  %r = vector.flat_transpose %b {columns = 8 : i32, rows = 2 : i32} : vector<16xi32> -> vector<16xi32>
+  return %r : vector<16xi32>
+}
diff --git a/mlir/test/Dialect/EmitC/invalid_ops.mlir b/mlir/test/Dialect/EmitC/invalid_ops.mlir
index 64753b8096fe0..377c84ef90dbd 100644
--- a/mlir/test/Dialect/EmitC/invalid_ops.mlir
+++ b/mlir/test/Dialect/EmitC/invalid_ops.mlir
@@ -16,6 +16,14 @@ func.func @const_attribute_return_type_2() {
 
 // -----
 
+func.func @empty_constant() {
+    // expected-error @+1 {{'emitc.constant' op value must not be empty}}
+    %c0 = "emitc.constant"(){value = ""} : () -> i32
+    return
+}
+
+// -----
+
 func.func @index_args_out_of_range_1() {
     // expected-error @+1 {{'emitc.call' op index argument is out of range}}
     emitc.call "test" () {args = [0 : index]} : () -> ()
@@ -80,6 +88,15 @@ func.func @illegal_operator(%arg : i32) {
 
 // -----
 
+func.func @illegal_operand() {
+    %1 = "emitc.constant"(){value = 42: i32} : () -> i32
+    // expected-error @+1 {{'emitc.apply' op cannot apply to constant}}
+    %2 = emitc.apply "&"(%1) : (i32) -> !emitc.ptr<i32>
+    return
+}
+
+// -----
+
 func.func @var_attribute_return_type_1() {
     // expected-error @+1 {{'emitc.variable' op requires attribute's type ('i64') to match op's return type ('i32')}}
     %c0 = "emitc.variable"(){value = 42: i64} : () -> i32
diff --git a/mlir/test/Dialect/GPU/outlining.mlir b/mlir/test/Dialect/GPU/outlining.mlir
index 422e0c154dd47..ca776968d998b 100644
--- a/mlir/test/Dialect/GPU/outlining.mlir
+++ b/mlir/test/Dialect/GPU/outlining.mlir
@@ -310,3 +310,65 @@ func.func @non_constant_launches(%arg0 : index) {
 }
 
 // CHECK-DL-LABEL: gpu.module @non_constant_launches_kernel attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<index, 32 : i32>>}
+
+// CHECK: module attributes {gpu.container_module}
+
+// -----
+
+// This test checks memory attributions for gpu.launch, using both workgroup and private attributions.
+// CHECK-LABEL: func @launch_memory_attributions_0()
+func.func @launch_memory_attributions_0() {
+  %1 = "op"() : () -> (memref<?xf32, 1>)
+  %128 = arith.constant 128 : index
+
+  // CHECK: gpu.launch_func @launch_memory_attributions_0_kernel::@launch_memory_attributions_0_kernel
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %128, %grid_y = %128,
+                                       %grid_z = %128)
+             threads(%tx, %ty, %tz) in (%block_x = %128, %block_y = %128,
+                                        %block_z = %128)
+             workgroup(%shared: memref<42xf32, 3>)
+             private(%priv0: memref<2xf32, 5>, %priv1: memref<1xf32, 5>) {
+    "some_op"(%bx, %block_x) : (index, index) -> ()
+    %42 = memref.load %1[%tx] : memref<?xf32, 1>
+    %43 = memref.load %shared[%tx] : memref<42xf32, 3>
+    %44 = memref.load %priv1[%tx] : memref<1xf32, 5>
+    gpu.terminator
+  }
+  return
+}
+
+// CHECK-DL-LABEL: gpu.module @launch_memory_attributions_0_kernel attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<index, 32 : i32>>}
+
+// CHECK-LABEL: gpu.module @launch_memory_attributions_0_kernel
+// CHECK-NEXT: gpu.func @launch_memory_attributions_0_kernel
+// CHECK-SAME: workgroup(%[[KERNEL_ARG1:.*]] : memref<42xf32, 3>)
+// CHECK-SAME: private(%[[KERNEL_ARG2:.*]] : memref<2xf32, 5>, %[[KERNEL_ARG3:.*]] : memref<1xf32, 5>)
+// CHECK: %[[TID:.*]] = gpu.thread_id x
+// CHECK: = memref.load %[[KERNEL_ARG1]][%[[TID]]] : memref<42xf32, 3>
+// CHECK-NEXT: = memref.load %[[KERNEL_ARG3]][%[[TID]]] : memref<1xf32, 5>
+
+// -----
+
+// This test checks correctness of private attributions in the absence of workgroup attributions.
+// CHECK-LABEL: @launch_memory_attributions_1
+func.func @launch_memory_attributions_1(%arg0 : memref<*xf32>) {
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  %d = memref.dim %arg0, %c2 : memref<*xf32>
+  // CHECK: gpu.func {{.*}}  private(%[[KERNEL_ARG:.*]] : memref<3xf32, 5>) {{.*}} {
+  // CHECK:   %[[C2:.*]] = arith.constant 2 : index
+  // CHECK: = memref.load %[[KERNEL_ARG]][%[[C2]]] : memref<3xf32, 5>
+  // CHECK:   gpu.return
+  // CHECK: }
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %c1, %grid_y = %c1,
+                                       %grid_z = %c1)
+             threads(%tx, %ty, %tz) in (%block_x = %c1, %block_y = %c1,
+                                        %block_z = %c1)
+             private(%priv0: memref<3xf32, 5>) {
+    %42 = memref.load %priv0[%c2] : memref<3xf32, 5>
+    gpu.terminator
+  }
+  return
+}
+
+// CHECK-DL-LABEL: gpu.module @launch_memory_attributions_1_kernel attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<index, 32 : i32>>}
diff --git a/mlir/test/Dialect/IRDL/cmath.irdl.mlir b/mlir/test/Dialect/IRDL/cmath.irdl.mlir
index aaa51501791c0..997af08d24733 100644
--- a/mlir/test/Dialect/IRDL/cmath.irdl.mlir
+++ b/mlir/test/Dialect/IRDL/cmath.irdl.mlir
@@ -6,11 +6,15 @@ module {
 
     // CHECK: irdl.type @complex {
     // CHECK:   %[[v0:[^ ]*]] = irdl.is f32
-    // CHECK:   irdl.parameters(%[[v0]])
+    // CHECK:   %[[v1:[^ ]*]] = irdl.is f64
+    // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+    // CHECK:   irdl.parameters(%[[v2]])
     // CHECK: }
     irdl.type @complex {
       %0 = irdl.is f32
-      irdl.parameters(%0)
+      %1 = irdl.is f64
+      %2 = irdl.any_of(%0, %1)
+      irdl.parameters(%2)
     }
 
     // CHECK: irdl.operation @norm {
@@ -28,13 +32,17 @@ module {
 
     // CHECK: irdl.operation @mul {
     // CHECK:   %[[v0:[^ ]*]] = irdl.is f32
-    // CHECK:   %[[v3:[^ ]*]] = irdl.parametric @complex<%[[v0]]>
+    // CHECK:   %[[v1:[^ ]*]] = irdl.is f64
+    // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+    // CHECK:   %[[v3:[^ ]*]] = irdl.parametric @complex<%[[v2]]>
     // CHECK:   irdl.operands(%[[v3]], %[[v3]])
     // CHECK:   irdl.results(%[[v3]])
     // CHECK: }
     irdl.operation @mul {
       %0 = irdl.is f32
-      %3 = irdl.parametric @complex<%0>
+      %1 = irdl.is f64
+      %2 = irdl.any_of(%0, %1)
+      %3 = irdl.parametric @complex<%2>
       irdl.operands(%3, %3)
       irdl.results(%3)
     }
diff --git a/mlir/test/Dialect/IRDL/cyclic-types.irdl.mlir b/mlir/test/Dialect/IRDL/cyclic-types.irdl.mlir
new file mode 100644
index 0000000000000..b8c04c9e2fb10
--- /dev/null
+++ b/mlir/test/Dialect/IRDL/cyclic-types.irdl.mlir
@@ -0,0 +1,50 @@
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+// Types that have cyclic references.
+
+// CHECK: irdl.dialect @testd {
+irdl.dialect @testd {
+  // CHECK:   irdl.type @self_referencing {
+  // CHECK:   %[[v0:[^ ]*]] = irdl.any
+  // CHECK:   %[[v1:[^ ]*]] = irdl.parametric @self_referencing<%[[v0]]>
+  // CHECK:   %[[v2:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v3:[^ ]*]] = irdl.any_of(%[[v1]], %[[v2]])
+  // CHECK:   irdl.parameters(%[[v3]])
+  // CHECK: }
+  irdl.type @self_referencing {
+    %0 = irdl.any
+    %1 = irdl.parametric @self_referencing<%0>
+    %2 = irdl.is i32
+    %3 = irdl.any_of(%1, %2)
+    irdl.parameters(%3)
+  }
+
+
+  // CHECK:   irdl.type @type1 {
+  // CHECK:   %[[v0:[^ ]*]] = irdl.any
+  // CHECK:   %[[v1:[^ ]*]] = irdl.parametric @type2<%[[v0]]>
+  // CHECK:   %[[v2:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v3:[^ ]*]] = irdl.any_of(%[[v1]], %[[v2]])
+  // CHECK:   irdl.parameters(%[[v3]])
+  irdl.type @type1 {
+    %0 = irdl.any
+    %1 = irdl.parametric @type2<%0>
+    %2 = irdl.is i32
+    %3 = irdl.any_of(%1, %2)
+    irdl.parameters(%3)
+  }
+
+  // CHECK:   irdl.type @type2 {
+  // CHECK:   %[[v0:[^ ]*]] = irdl.any
+  // CHECK:   %[[v1:[^ ]*]] = irdl.parametric @type1<%[[v0]]>
+  // CHECK:   %[[v2:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v3:[^ ]*]] = irdl.any_of(%[[v1]], %[[v2]])
+  // CHECK:   irdl.parameters(%[[v3]])
+  irdl.type @type2 {
+      %0 = irdl.any
+      %1 = irdl.parametric @type1<%0>
+      %2 = irdl.is i32
+      %3 = irdl.any_of(%1, %2)
+      irdl.parameters(%3)
+  }
+}
\ No newline at end of file
diff --git a/mlir/test/Dialect/IRDL/cyclic-types.mlir b/mlir/test/Dialect/IRDL/cyclic-types.mlir
new file mode 100644
index 0000000000000..56dc2d61787b6
--- /dev/null
+++ b/mlir/test/Dialect/IRDL/cyclic-types.mlir
@@ -0,0 +1,57 @@
+// RUN: mlir-opt %s --irdl-file=%S/cyclic-types.irdl.mlir -split-input-file -verify-diagnostics | FileCheck %s
+
+// Types that have cyclic references.
+
+// CHECK: !testd.self_referencing<i32>
+func.func @no_references(%v: !testd.self_referencing<i32>) {
+  return
+}
+
+// -----
+
+// CHECK: !testd.self_referencing<!testd.self_referencing<i32>>
+func.func @one_reference(%v: !testd.self_referencing<!testd.self_referencing<i32>>) {
+  return
+}
+
+// -----
+
+// expected-error@+1 {{'i64' does not satisfy the constraint}}
+func.func @wrong_parameter(%v: !testd.self_referencing<i64>) {
+  return
+}
+
+// -----
+
+// CHECK: !testd.type1<i32>
+func.func @type1_no_references(%v: !testd.type1<i32>) {
+  return
+}
+
+// -----
+
+// CHECK: !testd.type1<!testd.type2<i32>>
+func.func @type1_one_references(%v: !testd.type1<!testd.type2<i32>>) {
+  return
+}
+
+// -----
+
+// CHECK: !testd.type1<!testd.type2<!testd.type1<i32>>>
+func.func @type1_two_references(%v: !testd.type1<!testd.type2<!testd.type1<i32>>>) {
+  return
+}
+
+// -----
+
+// expected-error@+1 {{'i64' does not satisfy the constraint}}
+func.func @wrong_parameter_type1(%v: !testd.type1<i64>) {
+  return
+}
+
+// -----
+
+// expected-error@+1 {{'i64' does not satisfy the constraint}}
+func.func @wrong_parameter_type2(%v: !testd.type2<i64>) {
+  return
+}
diff --git a/mlir/test/Dialect/IRDL/test-type.irdl.mlir b/mlir/test/Dialect/IRDL/test-type.irdl.mlir
new file mode 100644
index 0000000000000..1bcfb0b8e20be
--- /dev/null
+++ b/mlir/test/Dialect/IRDL/test-type.irdl.mlir
@@ -0,0 +1,33 @@
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+module {
+  // CHECK-LABEL: irdl.dialect @testd {
+  irdl.dialect @testd {
+    // CHECK: irdl.type @singleton
+    irdl.type @singleton
+
+    // CHECK: irdl.type @parametrized {
+    // CHECK:   %[[v0:[^ ]*]] = irdl.any
+    // CHECK:   %[[v1:[^ ]*]] = irdl.is i32
+    // CHECK:   %[[v2:[^ ]*]] = irdl.is i64
+    // CHECK:   %[[v3:[^ ]*]] = irdl.any_of(%[[v1]], %[[v2]])
+    // CHECK:   irdl.parameters(%[[v0]], %[[v3]])
+    // CHECK: }
+    irdl.type @parametrized {
+      %0 = irdl.any
+      %1 = irdl.is i32
+      %2 = irdl.is i64
+      %3 = irdl.any_of(%1, %2)
+      irdl.parameters(%0, %3)
+    }
+
+    // CHECK: irdl.operation @any {
+    // CHECK:   %[[v0:[^ ]*]] = irdl.any
+    // CHECK:   irdl.results(%[[v0]])
+    // CHECK: }
+    irdl.operation @any {
+      %0 = irdl.any
+      irdl.results(%0)
+    }
+  }
+}
diff --git a/mlir/test/Dialect/IRDL/test-type.mlir b/mlir/test/Dialect/IRDL/test-type.mlir
new file mode 100644
index 0000000000000..9f79ebe4ba038
--- /dev/null
+++ b/mlir/test/Dialect/IRDL/test-type.mlir
@@ -0,0 +1,35 @@
+// RUN: mlir-opt %s --irdl-file=%S/test-type.irdl.mlir -split-input-file -verify-diagnostics | FileCheck %s
+
+func.func @succeededTypeVerifier() {
+    // CHECK: "testd.any"() : () -> !testd.singleton
+    "testd.any"() : () -> !testd.singleton
+
+    // CHECK-NEXT: "testd.any"() : () -> !testd.parametrized<f32, i32>
+    "testd.any"() : () -> !testd.parametrized<f32, i32>
+
+    // CHECK: "testd.any"() : () -> !testd.parametrized<i1, i64>
+    "testd.any"() : () -> !testd.parametrized<i1, i64>
+
+    return
+}
+
+// -----
+
+func.func @failedSingletonVerifier() {
+     // expected-error@+1 {{expected 0 type arguments, but had 1}}
+     "testd.any"() : () -> !testd.singleton<i32>
+}
+
+// -----
+
+func.func @failedParametrizedVerifierWrongNumOfArgs() {
+     // expected-error@+1 {{expected 2 type arguments, but had 1}}
+     "testd.any"() : () -> !testd.parametrized<i32>
+}
+
+// -----
+
+func.func @failedParametrizedVerifierWrongArgument() {
+     // expected-error@+1 {{'i1' does not satisfy the constraint}}
+     "testd.any"() : () -> !testd.parametrized<i32, i1>
+}
diff --git a/mlir/test/Dialect/IRDL/testd.irdl.mlir b/mlir/test/Dialect/IRDL/testd.irdl.mlir
index d4a33ca38a199..c1cdae4e0bb76 100644
--- a/mlir/test/Dialect/IRDL/testd.irdl.mlir
+++ b/mlir/test/Dialect/IRDL/testd.irdl.mlir
@@ -29,6 +29,34 @@ irdl.dialect @testd {
     irdl.results(%0)
   }
 
+  // CHECK: irdl.operation @anyof {
+  // CHECK:   %[[v0:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v1:[^ ]*]] = irdl.is i64
+  // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+  // CHECK:   irdl.results(%[[v2]])
+  // CHECK: }
+  irdl.operation @anyof {
+    %0 = irdl.is i32
+    %1 = irdl.is i64
+    %2 = irdl.any_of(%0, %1)
+    irdl.results(%2)
+  }
+
+  // CHECK: irdl.operation @all_of {
+  // CHECK:   %[[v0:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v1:[^ ]*]] = irdl.is i64
+  // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+  // CHECK:   %[[v3:[^ ]*]] = irdl.all_of(%[[v2]], %[[v1]])
+  // CHECK:   irdl.results(%[[v3]])
+  // CHECK: }
+  irdl.operation @all_of {
+    %0 = irdl.is i32
+    %1 = irdl.is i64
+    %2 = irdl.any_of(%0, %1)
+    %3 = irdl.all_of(%2, %1)
+    irdl.results(%3)
+  }
+
   // CHECK: irdl.operation @any {
   // CHECK:   %[[v0:[^ ]*]] = irdl.any
   // CHECK:   irdl.results(%[[v0]])
@@ -51,21 +79,44 @@ irdl.dialect @testd {
 
   // CHECK: irdl.operation @dynparams {
   // CHECK:   %[[v0:[^ ]*]] = irdl.is i32
-  // CHECK:   %[[v3:[^ ]*]] = irdl.parametric @parametric<%[[v0]]>
+  // CHECK:   %[[v1:[^ ]*]] = irdl.is i64
+  // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+  // CHECK:   %[[v3:[^ ]*]] = irdl.parametric @parametric<%[[v2]]>
   // CHECK:   irdl.results(%[[v3]])
   // CHECK: }
   irdl.operation @dynparams {
     %0 = irdl.is i32
-    %3 = irdl.parametric @parametric<%0>
+    %1 = irdl.is i64
+    %2 = irdl.any_of(%0, %1)
+    %3 = irdl.parametric @parametric<%2>
+    irdl.results(%3)
+  }
+
+  // CHECK: irdl.operation @params {
+  // CHECK:   %[[v0:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v1:[^ ]*]] = irdl.is i64
+  // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+  // CHECK:   %[[v3:[^ ]*]] = irdl.parametric "!builtin.complex"<%[[v2]]>
+  // CHECK:   irdl.results(%[[v3]])
+  // CHECK: }
+  irdl.operation @params {
+    %0 = irdl.is i32
+    %1 = irdl.is i64
+    %2 = irdl.any_of(%0, %1)
+    %3 = irdl.parametric "!builtin.complex"<%2>
     irdl.results(%3)
   }
 
   // CHECK: irdl.operation @constraint_vars {
-  // CHECK:   %[[v0:[^ ]*]] = irdl.any
-  // CHECK:   irdl.results(%[[v0]], %[[v0]])
+  // CHECK:   %[[v0:[^ ]*]] = irdl.is i32
+  // CHECK:   %[[v1:[^ ]*]] = irdl.is i64
+  // CHECK:   %[[v2:[^ ]*]] = irdl.any_of(%[[v0]], %[[v1]])
+  // CHECK:   irdl.results(%[[v2]], %[[v2]])
   // CHECK: }
   irdl.operation @constraint_vars {
-    %0 = irdl.any
-    irdl.results(%0, %0)
+    %0 = irdl.is i32
+    %1 = irdl.is i64
+    %2 = irdl.any_of(%0, %1)
+    irdl.results(%2, %2)
   }
 }
diff --git a/mlir/test/Dialect/IRDL/testd.mlir b/mlir/test/Dialect/IRDL/testd.mlir
index f6d1bcb0e396f..9a568633ada4c 100644
--- a/mlir/test/Dialect/IRDL/testd.mlir
+++ b/mlir/test/Dialect/IRDL/testd.mlir
@@ -45,6 +45,63 @@ func.func @succeededEqConstraint() {
   return
 }
 
+// -----
+
+func.func @failedEqConstraint() {
+  // expected-error@+1 {{expected 'i32' but got 'i64'}}
+  "testd.eq"() : () -> i64
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// AnyOf constraint
+//===----------------------------------------------------------------------===//
+
+func.func @succeededAnyOfConstraint() {
+  // CHECK: "testd.anyof"() : () -> i32
+  "testd.anyof"() : () -> i32
+  // CHECK: "testd.anyof"() : () -> i64
+  "testd.anyof"() : () -> i64
+  return
+}
+
+// -----
+
+func.func @failedAnyOfConstraint() {
+  // expected-error@+1 {{'i1' does not satisfy the constraint}}
+  "testd.anyof"() : () -> i1
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// AllOf constraint
+//===----------------------------------------------------------------------===//
+
+func.func @succeededAllOfConstraint() {
+  // CHECK: "testd.all_of"() : () -> i64
+  "testd.all_of"() : () -> i64
+  return
+}
+
+// -----
+
+func.func @failedAllOfConstraint1() {
+  // expected-error@+1 {{'i1' does not satisfy the constraint}}
+  "testd.all_of"() : () -> i1
+  return
+}
+
+// -----
+
+func.func @failedAllOfConstraint2() {
+  // expected-error@+1 {{expected 'i64' but got 'i32'}}
+  "testd.all_of"() : () -> i32
+  return
+}
 
 // -----
 
@@ -62,6 +119,36 @@ func.func @succeededAnyConstraint() {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// Non-dynamic parameters constraint
+//===----------------------------------------------------------------------===//
+
+func.func @succeededParamsConstraint() {
+  // CHECK: "testd.params"() : () -> complex<i32>
+  "testd.params"() : () -> complex<i32>
+  // CHECK: "testd.params"() : () -> complex<i64>
+  "testd.params"() : () -> complex<i64>
+  return
+}
+
+// -----
+
+func.func @failedDynParamsConstraintBase() {
+  // expected-error@+1 {{expected base 'builtin.complex' but got 'i32'}}
+  "testd.params"() : () -> i32
+  return
+}
+
+// -----
+
+func.func @failedDynParamsConstraintParam() {
+  // expected-error@+1 {{'i1' does not satisfy the constraint}}
+  "testd.params"() : () -> complex<i1>
+  return
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // Dynamic base constraint
 //===----------------------------------------------------------------------===//
@@ -69,11 +156,20 @@ func.func @succeededAnyConstraint() {
 func.func @succeededDynBaseConstraint() {
   // CHECK: "testd.dynbase"() : () -> !testd.parametric<i32>
   "testd.dynbase"() : () -> !testd.parametric<i32>
-  // CHECK: "testd.dynbase"() : () -> !testd.parametric<!testd.parametric<i32>>
-  "testd.dynbase"() : () -> !testd.parametric<!testd.parametric<i32>>
+  // CHECK: "testd.dynbase"() : () -> !testd.parametric<i64>
+  "testd.dynbase"() : () -> !testd.parametric<i64>
+  // CHECK: "testd.dynbase"() : () -> !testd.parametric<!testd.parametric<i64>>
+  "testd.dynbase"() : () -> !testd.parametric<!testd.parametric<i64>>
   return
 }
 
+// -----
+
+func.func @failedDynBaseConstraint() {
+  // expected-error@+1 {{expected base type 'testd.parametric' but got 'i32'}}
+  "testd.dynbase"() : () -> i32
+  return
+}
 
 // -----
 
@@ -84,6 +180,24 @@ func.func @succeededDynBaseConstraint() {
 func.func @succeededDynParamsConstraint() {
   // CHECK: "testd.dynparams"() : () -> !testd.parametric<i32>
   "testd.dynparams"() : () -> !testd.parametric<i32>
+  // CHECK: "testd.dynparams"() : () -> !testd.parametric<i64>
+  "testd.dynparams"() : () -> !testd.parametric<i64>
+  return
+}
+
+// -----
+
+func.func @failedDynParamsConstraintBase() {
+  // expected-error@+1 {{expected base type 'testd.parametric' but got 'i32'}}
+  "testd.dynparams"() : () -> i32
+  return
+}
+
+// -----
+
+func.func @failedDynParamsConstraintParam() {
+  // expected-error@+1 {{'i1' does not satisfy the constraint}}
+  "testd.dynparams"() : () -> !testd.parametric<i1>
   return
 }
 
@@ -106,3 +220,11 @@ func.func @succeededConstraintVars2() {
   "testd.constraint_vars"() : () -> (i64, i64)
   return
 }
+
+// -----
+
+func.func @failedConstraintVars() {
+  // expected-error@+1 {{expected 'i64' but got 'i32'}}
+  "testd.constraint_vars"() : () -> (i64, i32)
+  return
+}
diff --git a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir
index 8841a1fad9186..631f27139ba9f 100644
--- a/mlir/test/Dialect/LLVMIR/loop-metadata.mlir
+++ b/mlir/test/Dialect/LLVMIR/loop-metadata.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt %s --split-input-file | mlir-opt --split-input-file | FileCheck %s
 
 // CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
 #followup = #llvm.loop_annotation<disableNonforced = true>
@@ -80,3 +80,38 @@ llvm.metadata @metadata {
   llvm.access_group @group1
   llvm.access_group @group2
 }
+
+// -----
+
+#di_file = #llvm.di_file<"metadata-loop.ll" in "/">
+
+// CHECK: #[[START_LOC:.*]] = loc("loop-metadata.mlir":42:4)
+#loc1 = loc("loop-metadata.mlir":42:4)
+// CHECK: #[[END_LOC:.*]] = loc("loop-metadata.mlir":52:4)
+#loc2 = loc("loop-metadata.mlir":52:4)
+
+#di_compile_unit = #llvm.di_compile_unit<sourceLanguage = DW_LANG_C, file = #di_file, isOptimized = false, emissionKind = None>
+// CHECK: #[[SUBPROGRAM:.*]] = #llvm.di_subprogram<
+#di_subprogram = #llvm.di_subprogram<compileUnit = #di_compile_unit, scope = #di_file, name = "loop_locs", file = #di_file, subprogramFlags = Definition>
+
+// CHECK: #[[START_LOC_FUSED:.*]] = loc(fused<#[[SUBPROGRAM]]>[#[[START_LOC]]]
+#start_loc_fused = loc(fused<#di_subprogram>[#loc1])
+// CHECK: #[[END_LOC_FUSED:.*]] = loc(fused<#[[SUBPROGRAM]]>[#[[END_LOC]]]
+#end_loc_fused= loc(fused<#di_subprogram>[#loc2])
+
+// CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation<
+// CHECK-DAG: disableNonforced = false
+// CHECK-DAG: startLoc = #[[START_LOC_FUSED]]
+// CHECK-DAG: endLoc = #[[END_LOC_FUSED]]
+#loopMD = #llvm.loop_annotation<disableNonforced = false,
+        mustProgress = true,
+        startLoc = #start_loc_fused,
+        endLoc = #end_loc_fused>
+
+// CHECK: llvm.func @loop_annotation_with_locs
+llvm.func @loop_annotation_with_locs() {
+  // CHECK: llvm.br ^bb1 {loop_annotation = #[[LOOP_ANNOT]]
+  llvm.br ^bb1 {loop_annotation = #loopMD}
+^bb1:
+  llvm.return
+}
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir b/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir
new file mode 100644
index 0000000000000..d8d04dfcfec51
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir
@@ -0,0 +1,104 @@
+// RUN: mlir-opt %s --pass-pipeline='builtin.module(llvm.func(mem2reg))' | FileCheck %s
+
+llvm.func @use(i64)
+llvm.func @use_ptr(!llvm.ptr)
+
+#di_basic_type = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "ptr sized type", sizeInBits = 64>
+#di_file = #llvm.di_file<"test.ll" in "">
+#di_compile_unit = #llvm.di_compile_unit<sourceLanguage = DW_LANG_C_plus_plus_14, file = #di_file, producer = "clang", isOptimized = false, emissionKind = Full>
+#di_subprogram = #llvm.di_subprogram<compileUnit = #di_compile_unit, scope = #di_file, name = "blah", linkageName = "blah", file = #di_file, line = 7, subprogramFlags = Definition>
+// CHECK: #[[$VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "ptr sized var"{{.*}}>
+#di_local_variable = #llvm.di_local_variable<scope = #di_subprogram, name = "ptr sized var", file = #di_file, line = 7, arg = 1, type = #di_basic_type>
+#di_local_variable_2 = #llvm.di_local_variable<scope = #di_subprogram, name = "ptr sized var 2", file = #di_file, line = 7, arg = 1, type = #di_basic_type>
+
+// CHECK-LABEL: llvm.func @basic_store_load
+llvm.func @basic_store_load(%arg0: i64) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-NOT: = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  // CHECK-NOT: llvm.store
+  llvm.store %arg0, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  // CHECK-NOT: llvm.intr.dbg.declare
+  llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr
+  // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[LOADED:.*]] : i64
+  // CHECK-NOT: llvm.intr.dbg.value
+  // CHECK-NOT: llvm.intr.dbg.declare
+  // CHECK-NOT: llvm.store
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  // CHECK: llvm.return %[[LOADED]] : i64
+  llvm.return %2 : i64
+}
+
+// CHECK-LABEL: llvm.func @block_argument_value
+// CHECK-SAME: (%[[ARG0:.*]]: i64, {{.*}})
+llvm.func @block_argument_value(%arg0: i64, %arg1: i1) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-NOT: = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  // CHECK-NOT: llvm.intr.dbg.declare
+  llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr
+  llvm.cond_br %arg1, ^bb1, ^bb2
+// CHECK: ^{{.*}}:
+^bb1:
+  // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[ARG0]]
+  // CHECK-NOT: llvm.intr.dbg.value
+  llvm.store %arg0, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  llvm.br ^bb2
+// CHECK: ^{{.*}}(%[[BLOCKARG:.*]]: i64):
+^bb2:
+  // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[BLOCKARG]]
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  llvm.return %2 : i64
+}
+
+// CHECK-LABEL: llvm.func @double_block_argument_value
+// CHECK-SAME: (%[[ARG0:.*]]: i64, {{.*}})
+llvm.func @double_block_argument_value(%arg0: i64, %arg1: i1) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-NOT: = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  // CHECK-NOT: llvm.intr.dbg.declare
+  llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr
+  llvm.cond_br %arg1, ^bb1, ^bb2
+// CHECK: ^{{.*}}(%[[BLOCKARG1:.*]]: i64):
+^bb1:
+  // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[BLOCKARG1]]
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  llvm.call @use(%2) : (i64) -> ()
+  // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[ARG0]]
+  llvm.store %arg0, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  llvm.br ^bb2
+  // CHECK-NOT: llvm.intr.dbg.value
+// CHECK: ^{{.*}}(%[[BLOCKARG2:.*]]: i64):
+^bb2:
+  // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[BLOCKARG2]]
+  llvm.br ^bb1
+}
+
+// CHECK-LABEL: llvm.func @always_drop_promoted_declare
+// CHECK-NOT: = llvm.alloca
+// CHECK-NOT: llvm.intr.dbg.
+llvm.func @always_drop_promoted_declare() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr
+  llvm.return
+}
+
+// CHECK-LABEL: llvm.func @keep_dbg_if_not_promoted
+llvm.func @keep_dbg_if_not_promoted() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  // CHECK-NOT: = llvm.alloca
+  // CHECK-NOT: llvm.intr.dbg.declare
+  // CHECK: llvm.intr.dbg.declare #[[$VAR]] = %[[ALLOCA]]
+  // CHECK-NOT: = llvm.alloca
+  // CHECK-NOT: llvm.intr.dbg.declare
+  // CHECK: llvm.call @use_ptr(%[[ALLOCA]])
+  llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr
+  %2 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  llvm.intr.dbg.declare #di_local_variable_2 = %2 : !llvm.ptr
+  llvm.call @use_ptr(%1) : (!llvm.ptr) -> ()
+  llvm.return
+}
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg.mlir b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
new file mode 100644
index 0000000000000..090f9133f7a96
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
@@ -0,0 +1,685 @@
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(llvm.func(mem2reg))" --split-input-file | FileCheck %s
+
+// CHECK-LABEL: llvm.func @default_value
+llvm.func @default_value() -> i32 {
+  // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : i32
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: llvm.return %[[UNDEF]] : i32
+  llvm.return %2 : i32
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @store_of_ptr
+llvm.func @store_of_ptr() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(4 : i32) : i32
+  %2 = llvm.mlir.null : !llvm.ptr
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  // CHECK: llvm.store %{{.*}}, %[[ALLOCA]]
+  llvm.store %1, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  // CHECK: llvm.store %[[ALLOCA]], %{{.*}}
+  llvm.store %3, %2 {alignment = 8 : i64} : !llvm.ptr, !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @unreachable
+llvm.func @unreachable() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  // CHECK-NOT: = llvm.alloca
+  %2 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.return
+
+// CHECK: ^{{.*}}:
+// CHECK-NEXT: llvm.return
+^bb1:  // no predecessors
+  llvm.store %1, %2 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @unreachable_in_loop
+// CHECK-NOT: = llvm.alloca
+llvm.func @unreachable_in_loop() -> i32 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(6 : i32) : i32
+  %2 = llvm.mlir.constant(5 : i32) : i32
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.store %1, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  // CHECK: llvm.br ^[[LOOP:.*]]
+  llvm.br ^bb1
+  
+// CHECK: ^[[LOOP]]:
+^bb1:  // 2 preds: ^bb0, ^bb3
+  // CHECK-NEXT: llvm.br ^[[ENDOFLOOP:.*]]
+  llvm.store %2, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb3
+
+// CHECK: ^[[UNREACHABLE:.*]]:
+^bb2:  // no predecessors
+  // CHECK-NEXT: llvm.br ^[[ENDOFLOOP]]
+  llvm.br ^bb3
+  
+// CHECK: ^[[ENDOFLOOP]]:
+^bb3:  // 2 preds: ^bb1, ^bb2
+  // CHECK-NEXT: llvm.br ^[[LOOP]]
+  llvm.br ^bb1
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @branching
+// CHECK-NOT: = llvm.alloca
+llvm.func @branching(%arg0: i1, %arg1: i1) -> i32 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i32) : i32
+  %2 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  // CHECK: llvm.cond_br %{{.*}}, ^[[BB2:.*]](%{{.*}} : i32), ^{{.*}}
+  llvm.cond_br %arg0, ^bb2, ^bb1
+^bb1:  // pred: ^bb0
+  llvm.store %1, %2 {alignment = 4 : i64} : i32, !llvm.ptr
+  // CHECK: llvm.cond_br %{{.*}}, ^[[BB2]](%{{.*}} : i32), ^[[BB2]](%{{.*}} : i32)
+  llvm.cond_br %arg1, ^bb2, ^bb2
+// CHECK: ^[[BB2]](%[[V3:.*]]: i32):
+^bb2:  // 3 preds: ^bb0, ^bb1, ^bb1
+  %3 = llvm.load %2 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: llvm.return %[[V3]] : i32
+  llvm.return %3 : i32
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @recursive_alloca
+// CHECK-NOT: = llvm.alloca
+llvm.func @recursive_alloca() -> i32 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  %2 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %4 = llvm.alloca %0 x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  llvm.store %1, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.store %3, %4 {alignment = 8 : i64} : !llvm.ptr, !llvm.ptr
+  %5 = llvm.load %4 {alignment = 8 : i64} : !llvm.ptr -> !llvm.ptr
+  %6 = llvm.load %5 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.store %6, %2 {alignment = 4 : i64} : i32, !llvm.ptr
+  %7 = llvm.load %2 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.return %7 : i32
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @reset_in_branch
+// CHECK-NOT: = llvm.alloca
+// CHECK-NOT: ^{{.*}}({{.*}}):
+llvm.func @reset_in_branch(%arg0: i32, %arg1: i1) {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(true) : i1
+  %2 = llvm.mlir.constant(false) : i1
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.store %arg0, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.cond_br %arg1, ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  llvm.store %arg0, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  %4 = llvm.load %3 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.call @reset_in_branch(%4, %2) : (i32, i1) -> ()
+  llvm.br ^bb3
+^bb2:  // pred: ^bb0
+  %5 = llvm.load %3 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.call @reset_in_branch(%5, %1) : (i32, i1) -> ()
+  llvm.br ^bb3
+^bb3:  // 2 preds: ^bb1, ^bb2
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @intertwined_alloca
+// CHECK-NOT: = llvm.alloca
+llvm.func @intertwined_alloca(%arg0: !llvm.ptr, %arg1: i32) {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  %2 = llvm.alloca %0 x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %4 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %6 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.store %arg0, %2 {alignment = 8 : i64} : !llvm.ptr, !llvm.ptr
+  llvm.store %arg1, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.store %1, %4 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb1
+^bb1:  // 2 preds: ^bb0, ^bb4
+  %7 = llvm.load %3 {alignment = 4 : i64} : !llvm.ptr -> i32
+  %8 = llvm.add %7, %0  : i32
+  %9 = llvm.load %4 {alignment = 4 : i64} : !llvm.ptr -> i32
+  %10 = llvm.icmp "sgt" %8, %9 : i32
+  %11 = llvm.zext %10 : i1 to i32
+  llvm.cond_br %10, ^bb2, ^bb5
+^bb2:  // pred: ^bb1
+  %12 = llvm.load %6 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.store %12, %5 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.store %1, %6 {alignment = 4 : i64} : i32, !llvm.ptr
+  %13 = llvm.load %4 {alignment = 4 : i64} : !llvm.ptr -> i32
+  %14 = llvm.icmp "sgt" %13, %1 : i32
+  %15 = llvm.zext %14 : i1 to i32
+  llvm.cond_br %14, ^bb3, ^bb4
+^bb3:  // pred: ^bb2
+  %16 = llvm.load %2 {alignment = 8 : i64} : !llvm.ptr -> !llvm.ptr
+  %17 = llvm.load %4 {alignment = 4 : i64} : !llvm.ptr -> i32
+  %18 = llvm.sub %17, %0  : i32
+  %19 = llvm.getelementptr %16[%18] : (!llvm.ptr, i32) -> !llvm.ptr, i8
+  %20 = llvm.load %5 {alignment = 4 : i64} : !llvm.ptr -> i32
+  %21 = llvm.trunc %20 : i32 to i8
+  llvm.store %21, %19 {alignment = 1 : i64} : i8, !llvm.ptr
+  llvm.br ^bb4
+^bb4:  // 2 preds: ^bb2, ^bb3
+  %22 = llvm.load %4 {alignment = 4 : i64} : !llvm.ptr -> i32
+  %23 = llvm.add %22, %0  : i32
+  llvm.store %23, %4 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb1
+^bb5:  // pred: ^bb1
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @complex_cf
+// CHECK-NOT: = llvm.alloca
+llvm.func @complex_cf(%arg0: i32, ...) {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(false) : i1
+  %2 = llvm.mlir.constant(0 : i32) : i32
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.cond_br %1, ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  llvm.br ^bb2
+^bb2:  // 2 preds: ^bb0, ^bb1
+  llvm.store %2, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb3
+^bb3:  // 2 preds: ^bb2, ^bb16
+  llvm.cond_br %1, ^bb4, ^bb17
+^bb4:  // pred: ^bb3
+  llvm.cond_br %1, ^bb5, ^bb14
+^bb5:  // pred: ^bb4
+  llvm.cond_br %1, ^bb7, ^bb6
+^bb6:  // pred: ^bb5
+  llvm.br ^bb7
+^bb7:  // 2 preds: ^bb5, ^bb6
+  llvm.cond_br %1, ^bb9, ^bb8
+^bb8:  // pred: ^bb7
+  llvm.br ^bb9
+^bb9:  // 2 preds: ^bb7, ^bb8
+  llvm.cond_br %1, ^bb11, ^bb10
+^bb10:  // pred: ^bb9
+  llvm.br ^bb11
+^bb11:  // 2 preds: ^bb9, ^bb10
+  llvm.cond_br %1, ^bb12, ^bb13
+^bb12:  // pred: ^bb11
+  llvm.br ^bb13
+^bb13:  // 2 preds: ^bb11, ^bb12
+  llvm.br ^bb14
+^bb14:  // 2 preds: ^bb4, ^bb13
+  llvm.cond_br %1, ^bb15, ^bb16
+^bb15:  // pred: ^bb14
+  llvm.br ^bb16
+^bb16:  // 2 preds: ^bb14, ^bb15
+  llvm.br ^bb3
+^bb17:  // pred: ^bb3
+  llvm.br ^bb20
+^bb18:  // no predecessors
+  %4 = llvm.load %3 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.br ^bb24
+^bb19:  // no predecessors
+  llvm.br ^bb20
+^bb20:  // 2 preds: ^bb17, ^bb19
+  llvm.cond_br %1, ^bb21, ^bb22
+^bb21:  // pred: ^bb20
+  llvm.br ^bb23
+^bb22:  // pred: ^bb20
+  llvm.br ^bb23
+^bb23:  // 2 preds: ^bb21, ^bb22
+  llvm.br ^bb24
+^bb24:  // 2 preds: ^bb18, ^bb23
+  llvm.br ^bb26
+^bb25:  // no predecessors
+  llvm.br ^bb26
+^bb26:  // 2 preds: ^bb24, ^bb25
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @llvm_crash
+llvm.func @llvm_crash() -> i32 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  %2 = llvm.mlir.addressof @j : !llvm.ptr
+  %3 = llvm.mlir.constant(0 : i8) : i8
+  // CHECK-NOT: = llvm.alloca
+  // CHECK: %[[VOLATILE_ALLOCA:.*]] = llvm.alloca
+  // CHECK-NOT: = llvm.alloca
+  %4 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %6 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  %7 = llvm.bitcast %1 : i32 to i32
+  // CHECK: llvm.store volatile %{{.*}}, %[[VOLATILE_ALLOCA]]
+  llvm.store volatile %1, %5 {alignment = 4 : i64} : i32, !llvm.ptr
+  %8 = llvm.call @_setjmp(%2) : (!llvm.ptr) -> i32
+  %9 = llvm.icmp "ne" %8, %1 : i32
+  %10 = llvm.zext %9 : i1 to i8
+  %11 = llvm.icmp "ne" %10, %3 : i8
+  llvm.cond_br %11, ^bb1, ^bb2
+^bb1:  // pred: ^bb0
+  // CHECK: = llvm.load volatile %[[VOLATILE_ALLOCA]]
+  %12 = llvm.load volatile %5 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.store %12, %6 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb3
+^bb2:  // pred: ^bb0
+  // CHECK: llvm.store volatile %{{.*}}, %[[VOLATILE_ALLOCA]]
+  llvm.store volatile %0, %5 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.call @g() : () -> ()
+  llvm.store %1, %6 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb3
+^bb3:  // 2 preds: ^bb1, ^bb2
+  %13 = llvm.load %6 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.store %13, %4 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb4
+^bb4:  // pred: ^bb3
+  %14 = llvm.load %4 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.return %14 : i32
+}
+llvm.mlir.global external @j() {addr_space = 0 : i32} : !llvm.array<1 x struct<"struct.__jmp_buf_tag", (array<6 x i32>, i32, struct<"struct.__sigset_t", (array<32 x i32>)>)>>
+llvm.func @_setjmp(!llvm.ptr) -> i32 attributes {passthrough = ["returns_twice"]}
+llvm.func @g()
+
+// -----
+
+// CHECK-LABEL: llvm.func amdgpu_kernelcc @addrspace_discard
+// CHECK-NOT: = llvm.alloca
+llvm.func amdgpu_kernelcc @addrspace_discard() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i64) : i64
+  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr<5>
+  %3 = llvm.addrspacecast %2 : !llvm.ptr<5> to !llvm.ptr
+  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @ignore_atomic
+// CHECK-SAME: (%[[ARG0:.*]]: i32) -> i32
+llvm.func @ignore_atomic(%arg0: i32) -> i32 {
+  // CHECK-NOT: = llvm.alloca
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.store %arg0, %1 atomic seq_cst {alignment = 4 : i64} : i32, !llvm.ptr
+  %2 = llvm.load %1 atomic seq_cst {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: llvm.return %[[ARG0]] : i32
+  llvm.return %2 : i32
+}
+
+// -----
+
+// CHECK: llvm.func @landing_pad
+// CHECK-NOT: = llvm.alloca
+llvm.func @landing_pad() -> i32 attributes {personality = @__gxx_personality_v0} {
+  // CHECK-NOT: = llvm.alloca
+  // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : i32
+  // CHECK-NOT: = llvm.alloca
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  // CHECK: %[[V2:.*]] = llvm.invoke
+  %2 = llvm.invoke @landing_padf() to ^bb1 unwind ^bb3 : () -> i32
+// CHECK: ^{{.*}}:
+^bb1:// pred: ^bb0
+  llvm.store %2, %1 {alignment = 4 : i64} : i32, !llvm.ptr
+  // CHECK: llvm.br ^[[BB2:.*]](%[[V2]] : i32)
+  llvm.br ^bb2
+// CHECK: ^[[BB2]]([[V3:.*]]: i32):
+^bb2:// 2 preds: ^bb1, ^bb3
+  %3 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: llvm.return [[V3]] : i32
+  llvm.return %3 : i32
+// CHECK: ^{{.*}}:
+^bb3:// pred: ^bb0
+  %4 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  // CHECK: llvm.br ^[[BB2:.*]](%[[UNDEF]] : i32)
+  llvm.br ^bb2
+}
+llvm.func @landing_padf() -> i32
+llvm.func @__gxx_personality_v0(...) -> i32
+
+// -----
+
+// CHECK-LABEL: llvm.func @unreachable_defines
+llvm.func @unreachable_defines() -> i32 {
+  // CHECK-NOT: = llvm.alloca
+  // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : i32
+  // CHECK-NOT: = llvm.alloca
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.br ^bb1
+^bb1:  // 2 preds: ^bb0, ^bb2
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: llvm.return %[[UNDEF]] : i32
+  llvm.return %2 : i32
+^bb2:  // no predecessors
+  %3 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.store %3, %1 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb1
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @unreachable_jumps_to_merge_point
+// CHECK-NOT: = llvm.alloca
+llvm.func @unreachable_jumps_to_merge_point(%arg0: i1) -> i32 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(6 : i32) : i32
+  %2 = llvm.mlir.constant(5 : i32) : i32
+  %3 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.cond_br %arg0, ^bb1, ^bb2
+^bb1:  // 2 preds: ^bb0, ^bb4
+  llvm.store %1, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb4
+^bb2:  // pred: ^bb0
+  llvm.store %2, %3 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.br ^bb4
+^bb3:  // no predecessors
+  llvm.br ^bb4
+^bb4:  // 3 preds: ^bb1, ^bb2, ^bb3
+  %4 = llvm.load %3 {alignment = 4 : i64} : !llvm.ptr -> i32
+  llvm.return %4 : i32
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @ignore_lifetime
+// CHECK-NOT: = llvm.alloca
+llvm.func @ignore_lifetime() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.intr.lifetime.start 2, %1 : !llvm.ptr
+  llvm.store %0, %1 {alignment = 4 : i64} : i32, !llvm.ptr
+  llvm.intr.lifetime.end 2, %1 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @ignore_discardable_tree
+// CHECK-NOT: = llvm.alloca
+llvm.func @ignore_discardable_tree() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i16) : i16
+  %2 = llvm.mlir.constant(0 : i8) : i8
+  %3 = llvm.mlir.undef : !llvm.struct<(i8, i16)>
+  %4 = llvm.insertvalue %2, %3[0] : !llvm.struct<(i8, i16)> 
+  %5 = llvm.insertvalue %1, %4[1] : !llvm.struct<(i8, i16)> 
+  %6 = llvm.alloca %0 x !llvm.struct<(i8, i16)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %7 = llvm.getelementptr %6[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i8, i16)>
+  llvm.intr.lifetime.start 2, %7 : !llvm.ptr
+  llvm.store %5, %6 {alignment = 2 : i64} : !llvm.struct<(i8, i16)>, !llvm.ptr
+  llvm.intr.lifetime.end 2, %7 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @store_load_forward
+llvm.func @store_load_forward() -> i32 {
+  // CHECK-NOT: = llvm.alloca
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK: %[[RES:.*]] = llvm.mlir.constant(0 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  %2 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.store %1, %2 {alignment = 4 : i64} : i32, !llvm.ptr
+  %3 = llvm.load %2 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: llvm.return %[[RES]] : i32
+  llvm.return %3 : i32
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @store_load_wrong_type
+llvm.func @store_load_wrong_type() -> i16 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(0 : i32) : i32
+  // CHECK: = llvm.alloca
+  %2 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  llvm.store %1, %2 {alignment = 4 : i64} : i32, !llvm.ptr
+  %3 = llvm.load %2 {alignment = 2 : i64} : !llvm.ptr -> i16
+  llvm.return %3 : i16
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @merge_point_cycle
+llvm.func @merge_point_cycle() {
+  // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : i32
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(7 : i32) : i32
+  %2 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  // CHECK: llvm.br ^[[BB1:.*]](%[[UNDEF]] : i32)
+  llvm.br ^bb1
+// CHECK: ^[[BB1]](%[[BARG:.*]]: i32):
+^bb1:  // 2 preds: ^bb0, ^bb1
+  %3 = llvm.load %2 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: = llvm.call @use(%[[BARG]])
+  %4 = llvm.call @use(%3) : (i32) -> i1
+  // CHECK: %[[DEF:.*]] = llvm.call @def
+  %5 = llvm.call @def(%1) : (i32) -> i32
+  llvm.store %5, %2 {alignment = 4 : i64} : i32, !llvm.ptr
+  // CHECK: llvm.cond_br %{{.*}}, ^[[BB1]](%[[DEF]] : i32), ^{{.*}}
+  llvm.cond_br %4, ^bb1, ^bb2
+^bb2:  // pred: ^bb1
+  llvm.return
+}
+
+llvm.func @def(i32) -> i32
+llvm.func @use(i32) -> i1
+
+// -----
+
+// CHECK-LABEL: llvm.func @no_unnecessary_arguments
+llvm.func @no_unnecessary_arguments() {
+  // CHECK: %[[UNDEF:.*]] = llvm.mlir.undef : i32
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
+  // CHECK: llvm.br ^[[BB1:.*]]
+  llvm.br ^bb1
+// CHECK: ^[[BB1]]:
+^bb1:  // 2 preds: ^bb0, ^bb1
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i32
+  // CHECK: = llvm.call @use(%[[UNDEF]])
+  %3 = llvm.call @use(%2) : (i32) -> i1
+  // CHECK: llvm.cond_br %{{.*}}, ^[[BB1]], ^{{.*}}
+  llvm.cond_br %3, ^bb1, ^bb2
+^bb2:  // pred: ^bb1
+  llvm.return
+}
+
+llvm.func @use(i32) -> i1
+
+// -----
+
+// CHECK-LABEL: llvm.func @discardable_use_tree
+// CHECK-NOT: = llvm.alloca
+llvm.func @discardable_use_tree() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i64) : i64
+  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
+  %4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
+  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @non_discardable_use_tree
+llvm.func @non_discardable_use_tree() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i64) : i64
+  // CHECK: = llvm.alloca
+  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
+  %4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
+  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.call @use(%4) : (!llvm.ptr) -> i1
+  llvm.return
+}
+llvm.func @use(!llvm.ptr) -> i1
+
+// -----
+
+// CHECK-LABEL: llvm.func @trivial_get_element_ptr
+// CHECK-NOT: = llvm.alloca
+llvm.func @trivial_get_element_ptr() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i64) : i64
+  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
+  %4 = llvm.getelementptr %3[0, 0, 0] : (!llvm.ptr) -> !llvm.ptr, i8
+  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @nontrivial_get_element_ptr
+llvm.func @nontrivial_get_element_ptr() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i64) : i64
+  // CHECK: = llvm.alloca
+  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
+  %4 = llvm.getelementptr %3[0, 1, 0] : (!llvm.ptr) -> !llvm.ptr, i8
+  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @dynamic_get_element_ptr
+llvm.func @dynamic_get_element_ptr() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i64) : i64
+  // CHECK: = llvm.alloca
+  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
+  %4 = llvm.getelementptr %3[0, %0] : (!llvm.ptr, i32) -> !llvm.ptr, i8
+  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @live_cycle
+// CHECK-SAME: (%[[ARG0:.*]]: i64, %{{.*}}: i1, %[[ARG2:.*]]: i64) -> i64
+llvm.func @live_cycle(%arg0: i64, %arg1: i1, %arg2: i64) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-NOT: = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  llvm.store %arg2, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  // CHECK: llvm.cond_br %{{.*}}, ^[[BB1:.*]](%[[ARG2]] : i64), ^[[BB2:.*]](%[[ARG2]] : i64)
+  llvm.cond_br %arg1, ^bb1, ^bb2
+// CHECK: ^[[BB1]](%[[V1:.*]]: i64):
+^bb1:
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  // CHECK: llvm.call @use(%[[V1]])
+  llvm.call @use(%2) : (i64) -> ()
+  llvm.store %arg0, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  // CHECK: llvm.br ^[[BB2]](%[[ARG0]] : i64)
+  llvm.br ^bb2
+// CHECK: ^[[BB2]](%[[V2:.*]]: i64):
+^bb2:
+  // CHECK: llvm.br ^[[BB1]](%[[V2]] : i64)
+  llvm.br ^bb1
+}
+
+llvm.func @use(i64)
+
+// -----
+
+// This test should no longer be an issue once promotion within subregions
+// is supported.
+// CHECK-LABEL: llvm.func @subregion_block_promotion
+// CHECK-SAME: (%[[ARG0:.*]]: i64, %[[ARG1:.*]]: i64) -> i64
+llvm.func @subregion_block_promotion(%arg0: i64, %arg1: i64) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  // CHECK: llvm.store %[[ARG1]], %[[ALLOCA]]
+  llvm.store %arg1, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  // CHECK: scf.execute_region {
+  scf.execute_region {
+    // CHECK: llvm.store %[[ARG0]], %[[ALLOCA]]
+    llvm.store %arg0, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+    scf.yield
+  }
+  // CHECK: }
+  // CHECK: %[[RES:.*]] = llvm.load %[[ALLOCA]]
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  // CHECK: llvm.return %[[RES]] : i64
+  llvm.return %2 : i64
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @subregion_simple_transitive_promotion
+// CHECK-SAME: (%[[ARG0:.*]]: i64, %[[ARG1:.*]]: i64) -> i64
+llvm.func @subregion_simple_transitive_promotion(%arg0: i64, %arg1: i64) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-NOT: = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  llvm.store %arg1, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  // CHECK: scf.execute_region {
+  scf.execute_region {
+    // CHECK: llvm.call @use(%[[ARG1]])
+    llvm.call @use(%2) : (i64) -> ()
+    scf.yield
+  }
+  // CHECK: }
+  // CHECK: llvm.return %[[ARG1]] : i64
+  llvm.return %2 : i64
+}
+
+llvm.func @use(i64)
+
+// -----
+
+// This behavior is specific to the LLVM dialect, because LLVM semantics are
+// that reaching an alloca multiple times allocates on the stack multiple
+// times. Promoting an alloca that is reached multiple times could lead to
+// changes in observable behavior. Thus only allocas in the entry block are
+// promoted.
+
+// CHECK-LABEL: llvm.func @no_inner_alloca_promotion
+// CHECK-SAME: (%[[ARG:.*]]: i64) -> i64
+llvm.func @no_inner_alloca_promotion(%arg: i64) -> i64 {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  llvm.br ^bb1
+^bb1:
+  // CHECK: %[[ALLOCA:.*]] = llvm.alloca
+  %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  // CHECK: llvm.store %[[ARG]], %[[ALLOCA]]
+  llvm.store %arg, %1 {alignment = 4 : i64} : i64, !llvm.ptr
+  // CHECK: %[[RES:.*]] = llvm.load %[[ALLOCA]]
+  %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64
+  // CHECK: llvm.return %[[RES]] : i64
+  llvm.return %2 : i64
+}
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index 200d3e9f7d757..11605afc7d16a 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -262,9 +262,11 @@ llvm.func @rocdl.raw.buffer.i32(%rsrc : vector<4xi32>,
   // CHECK-LABEL: rocdl.raw.buffer.i32
   // CHECK: rocdl.raw.buffer.atomic.smax %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : i32
   // CHECK: rocdl.raw.buffer.atomic.umin %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : i32
+  // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.cmpswap(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : i32, vector<4xi32>
 
   rocdl.raw.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32
   rocdl.raw.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32
+  %val = rocdl.raw.buffer.atomic.cmpswap(%vdata1, %vdata1, %rsrc, %offset, %soffset, %aux) : i32, vector<4xi32>
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
index 266fe2dd29a45..f8844b7271589 100644
--- a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
+++ b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
@@ -726,9 +726,6 @@ func.func @reduction_pack_transpose_inner_dims(%arg0: tensor<128x256x32xi32>, %d
 // CHECK:      func.func @reduction_pack_transpose_inner_dims
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9]+]]
 // CHECK-SAME:   %[[DEST:[a-zA-Z0-9]+]]
-// CHECK:        %[[ORIG_INIT:.+]] = tensor.empty() : tensor<128x256xi32>
-// CHECK:        %[[INIT_EMPTY:.+]] = tensor.empty() : tensor<4x16x16x32xi32>
-// CHECK:        %[[PACK_INIT:.+]] = tensor.pack %[[ORIG_INIT]]
 // CHECK:        %[[ARG0_EMPTY:.+]] = tensor.empty() : tensor<4x16x32x16x32xi32>
 // CHECK:        %[[PACK_ARG0:.+]] = tensor.pack %[[ARG0]]
 // CHECK-SAME:     inner_dims_pos = [1, 0] inner_tiles = [16, 32]
@@ -737,7 +734,7 @@ func.func @reduction_pack_transpose_inner_dims(%arg0: tensor<128x256x32xi32>, %d
 // CHECK-SAME:     indexing_maps = [#[[MAP0]], #[[MAP1]]]
 // CHECK-SAME:     iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel"]
 // CHECK-SAME:     ins(%[[PACK_ARG0]]
-// CHECK-SAME:     outs(%[[PACK_INIT]]
+// CHECK-SAME:     outs(%[[DEST]]
 // CHECK:        return %[[RED]] : tensor<4x16x16x32xi32>
 
 // -----
@@ -776,11 +773,7 @@ func.func @reduction_pack_with_outer_dims(%arg0: tensor<100x128x200x256xi32>, %a
 // CHECK-SAME:   %[[ARG0:[a-zA-Z0-9]+]]
 // CHECK-SAME:   %[[ARG1:[a-zA-Z0-9]+]]
 // CHECK-SAME:   %[[ARG2:[a-zA-Z0-9]+]]
-// CHECK: %[[INIT:.+]] = tensor.empty() : tensor<100x128x256xi32>
 // CHECK: %[[INIT_EMPTY:.+]] = tensor.empty() : tensor<4x16x100x16x32xi32>
-// CHECK: %[[PACKED_INIT:.+]] = tensor.pack %[[INIT]]
-// CHECK-SAME:  outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [16, 32]
-// CHECK-SAME:  into %[[INIT_EMPTY]]
 // CHECK: %[[ARG0_EMPTY:.+]] = tensor.empty() : tensor<4x16x200x100x16x32xi32>
 // CHECK: %[[PACKED_ARG0:.+]] = tensor.pack %[[ARG0]]
 // CHECK-SAME:  outer_dims_perm = [1, 3, 2, 0] inner_dims_pos = [3, 1] inner_tiles = [16, 32]
@@ -792,7 +785,7 @@ func.func @reduction_pack_with_outer_dims(%arg0: tensor<100x128x200x256xi32>, %a
 // CHECK: %[[RES:.+]] = linalg.generic
 // CHECK-SAME:  indexing_maps = [#[[MAP]], #[[MAP1]], #[[MAP2]], #[[MAP3]]]
 // CHECK-SAME:  ins(%[[PACKED_ARG0]], %[[ARG1]], %[[PACKED_ARG2]]
-// CHECK-SAME:  outs(%[[PACKED_INIT]]
+// CHECK-SAME:  outs(%[[INIT_EMPTY]]
 
 // -----
 
diff --git a/mlir/test/Dialect/Linalg/detensorize_entry_block.mlir b/mlir/test/Dialect/Linalg/detensorize_entry_block.mlir
new file mode 100644
index 0000000000000..d1a89226fdb58
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/detensorize_entry_block.mlir
@@ -0,0 +1,21 @@
+// RUN: mlir-opt %s -allow-unregistered-dialect -pass-pipeline="builtin.module(func.func(linalg-detensorize))" | FileCheck %s
+
+#map = affine_map<() -> ()>
+func.func @main(%arg0: tensor<f32>) -> tensor<f32> {
+  %0 = tensor.empty() : tensor<f32>
+  %1 = linalg.generic {indexing_maps = [#map, #map], iterator_types = []} ins(%arg0 : tensor<f32>) outs(%0 : tensor<f32>) {
+  ^bb0(%in: f32, %out: f32):
+    linalg.yield %in : f32
+  } -> tensor<f32>
+  cf.br ^bb1(%1 : tensor<f32>)
+^bb1(%2: tensor<f32>):  // pred: ^bb0
+  return %2 : tensor<f32>
+}
+
+// CHECK-LABEL: @main
+// CHECK-SAME:       (%[[ARG0:.+]]: tensor<f32>) -> tensor<f32>
+// CHECK:   %[[EXTRACTED:.+]] = tensor.extract %[[ARG0]][] : tensor<f32>
+// CHECK: cf.br ^{{.*}}(%[[EXTRACTED]] : f32)
+// CHECK: ^{{.*}}(%[[ARG1:.+]]: f32):
+// CHECK:   %[[ELEMENTS:.+]] = tensor.from_elements %[[ARG1]] : tensor<f32>
+// CHECK:   return %[[ELEMENTS]] : tensor<f32>
diff --git a/mlir/test/Dialect/Linalg/detensorize_while_pure_cf.mlir b/mlir/test/Dialect/Linalg/detensorize_while_pure_cf.mlir
index 455fcfe7b498e..6d8d5fe71fca5 100644
--- a/mlir/test/Dialect/Linalg/detensorize_while_pure_cf.mlir
+++ b/mlir/test/Dialect/Linalg/detensorize_while_pure_cf.mlir
@@ -44,8 +44,8 @@ func.func @main() -> () attributes {} {
 }
 
 // CHECK-LABEL: func @main
-// CHECK-NEXT:    arith.constant 0 : i32
-// CHECK-NEXT:    arith.constant 10
+// CHECK-DAG:    arith.constant 0 : i32
+// CHECK-DAG:    arith.constant 10
 // CHECK-NEXT:    cf.br ^[[bb1:.*]](%{{.*}} : i32)
 // CHECK-NEXT:  ^[[bb1]](%{{.*}}: i32)
 // CHECK-NEXT:    %{{.*}} = arith.cmpi slt, %{{.*}}, %{{.*}}
diff --git a/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir b/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir
index 8e9b77ed6f679..283cb43e2997b 100644
--- a/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-tensor-pack.mlir
@@ -76,3 +76,22 @@ func.func @simple_CHW_to_CHWhwc(%arg0: tensor<3x5x7xf32>, %arg1: tensor<1x1x1x5x
 // CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[TRANSP]] into %[[DEST]]
 // CHECK-SAME:      [0, 0, 0, 0, 0, 0] [1, 1, 1, 5, 7, 3] [1, 1, 1, 1, 1, 1]
 // CHECK:         return %[[INSERT]]
+
+// -----
+
+func.func @simple_KCRS_to_KRSCsr(%arg0: tensor<3x1x32x8xf32>, %arg1: tensor<3x1x1x1x8x32xf32>) -> tensor<3x1x1x1x8x32xf32> {
+  %0 = tensor.pack %arg0 outer_dims_perm = [0, 2, 3, 1] inner_dims_pos = [3, 2] inner_tiles = [8, 32] into %arg1 : tensor<3x1x32x8xf32> -> tensor<3x1x1x1x8x32xf32>
+  return %0 : tensor<3x1x1x1x8x32xf32>
+}
+// CHECK-LABEL: func.func @simple_KCRS_to_KRSCsr
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:         %[[TILE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0] [3, 1, 32, 8] [1, 1, 1, 1]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<3x8x32xf32>
+// CHECK:         %[[TRANSP:.+]] =  linalg.transpose
+// CHECK-SAME:      ins(%[[TILE]] : tensor<3x32x8xf32>)
+// CHECK-SAME:      outs(%[[EMPTY]] : tensor<3x8x32xf32>)
+// CHECK-SAME:      permutation = [0, 2, 1]
+// CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[TRANSP]] into %[[DEST]]
+// CHECK-SAME:      [0, 0, 0, 0, 0, 0] [3, 1, 1, 1, 8, 32] [1, 1, 1, 1, 1, 1]
+// CHECK:         return %[[INSERT]]
diff --git a/mlir/test/Dialect/Linalg/generalize-tensor-unpack.mlir b/mlir/test/Dialect/Linalg/generalize-tensor-unpack.mlir
index cc734c24d4f56..a596690c2e4fd 100644
--- a/mlir/test/Dialect/Linalg/generalize-tensor-unpack.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-tensor-unpack.mlir
@@ -55,3 +55,42 @@ func.func @simple_CNnc_to_NC(%arg0: tensor<1x1x32x8xf32>, %arg1: tensor<32x8xf32
 //                They have the same type, so the insert_slice op is folded
 //                away.
 // CHECK:         return %[[TRANSP]]
+
+// -----
+
+func.func @simple_NCHWc_to_NCHW(%arg0: tensor<2x1x16x8x32xf32>, %arg1: tensor<2x32x16x8xf32>) -> tensor<2x32x16x8xf32> {
+  %0 = tensor.unpack %arg0 inner_dims_pos = [1] inner_tiles = [32] into %arg1 : tensor<2x1x16x8x32xf32> -> tensor<2x32x16x8xf32>
+  return %0 : tensor<2x32x16x8xf32>
+}
+// CHECK-LABEL: func.func @simple_NCHWc_to_NCHW
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:         %[[TILE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0, 0] [2, 1, 16, 8, 32] [1, 1, 1, 1, 1]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<2x32x16x8xf32>
+// CHECK:         %[[TRANSP:.+]] =  linalg.transpose
+// CHECK-SAME:      ins(%[[TILE]] : tensor<2x16x8x32xf32>)
+// CHECK-SAME:      outs(%[[EMPTY]] : tensor<2x32x16x8xf32>)
+// CHECK-SAME:      permutation = [0, 3, 1, 2]
+//                They have the same type, so the insert_slice op is folded
+//                away.
+// CHECK:         return %[[TRANSP]]
+
+
+// -----
+
+func.func @simple_NHWC_to_NCHW(%arg0: tensor<1x16x8x32xf32>, %arg1: tensor<1x32x16x8xf32>) -> tensor<1x32x16x8xf32> {
+  %0 = tensor.unpack %arg0 outer_dims_perm = [0, 2, 3, 1] inner_dims_pos = [] inner_tiles = [] into %arg1 : tensor<1x16x8x32xf32> -> tensor<1x32x16x8xf32>
+  return %0 : tensor<1x32x16x8xf32>
+}
+// CHECK-LABEL: func.func @simple_NHWC_to_NCHW
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:         %[[TILE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0] [1, 16, 8, 32] [1, 1, 1, 1]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<32x16x8xf32>
+// CHECK:         %[[TRANSP:.+]] =  linalg.transpose
+// CHECK-SAME:      ins(%[[TILE]] : tensor<16x8x32xf32>)
+// CHECK-SAME:      outs(%[[EMPTY]] : tensor<32x16x8xf32>)
+// CHECK-SAME:      permutation = [2, 0, 1]
+// CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[TRANSP]] into %[[DEST]]
+// CHECK-SAME:      [0, 0, 0, 0] [1, 32, 16, 8] [1, 1, 1, 1]
+// CHECK:         return %[[INSERT]]
diff --git a/mlir/test/Dialect/Linalg/int-narrowing.mlir b/mlir/test/Dialect/Linalg/int-narrowing.mlir
new file mode 100644
index 0000000000000..8063d504597a3
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/int-narrowing.mlir
@@ -0,0 +1,147 @@
+// RUN: mlir-opt --arith-int-narrowing="int-bitwidths-supported=1,8,16,32" \
+// RUN:          --verify-diagnostics %s | FileCheck %s
+
+// Check that we can calculate `linalg.index` value bounds and use them to
+// optimize index casts.
+
+//===----------------------------------------------------------------------===//
+// arith.index_cast
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func @linalg_indexcast_dim_0_i8
+// CHECK:         %[[IDX:.+]] = linalg.index 0 : index
+// CHECK-NEXT:    %[[INT:.+]] = arith.index_cast %[[IDX]] : index to i8
+// CHECK-NEXT:    %[[FP:.+]]  = arith.sitofp %[[INT]] : i8 to f16
+// CHECK-NEXT:    linalg.yield %[[FP]] : f16
+func.func @linalg_indexcast_dim_0_i8(%arg0: tensor<f16>) -> tensor<128xf16> {
+  %init = tensor.empty() : tensor<128xf16>
+  %res = linalg.generic {
+      indexing_maps = [affine_map<(d0) -> ()>, affine_map<(d0) -> (d0)>],
+      iterator_types = ["parallel"]
+    }
+    ins(%arg0 : tensor<f16>)
+    outs(%init : tensor<128xf16>) {
+  ^bb0(%in: f16, %out: f16):
+    %idx = linalg.index 0 : index
+    %int = arith.index_cast %idx : index to i64
+    %fp = arith.sitofp %int : i64 to f16
+    linalg.yield %fp : f16
+  } -> tensor<128xf16>
+
+  return %res : tensor<128xf16>
+}
+
+// CHECK-LABEL: func @linalg_indexcast_dim_1_i16
+// CHECK:         %[[IDX:.+]] = linalg.index 1 : index
+// CHECK-NEXT:    %[[INT:.+]] = arith.index_cast %[[IDX]] : index to i16
+// CHECK-NEXT:    %[[FP:.+]]  = arith.sitofp %[[INT]] : i16 to f16
+// CHECK-NEXT:    linalg.yield %[[FP]] : f16
+func.func @linalg_indexcast_dim_1_i16(%arg0: tensor<f16>, %arg1: tensor<?x129xf16>) -> tensor<?x129xf16> {
+  %res = linalg.generic {
+      indexing_maps = [affine_map<(d0, d1) -> ()>, affine_map<(d0, d1) -> (d0, d1)>],
+      iterator_types = ["parallel", "parallel"]
+    }
+    ins(%arg0 : tensor<f16>)
+    outs(%arg1 : tensor<?x129xf16>) {
+  ^bb0(%in: f16, %out: f16):
+    %idx = linalg.index 1 : index
+    %int = arith.index_cast %idx : index to i64
+    %fp = arith.sitofp %int : i64 to f16
+    linalg.yield %fp : f16
+  } -> tensor<?x129xf16>
+
+  return %res : tensor<?x129xf16>
+}
+
+// CHECK-LABEL: func @linalg_indexcast_dynamic_dim_i64
+// CHECK:         %[[IDX:.+]] = linalg.index 0 : index
+// CHECK-NEXT:    %[[INT:.+]] = arith.index_cast %[[IDX]] : index to i64
+// CHECK-NEXT:    %[[FP:.+]]  = arith.sitofp %[[INT]] : i64 to f16
+// CHECK-NEXT:    linalg.yield %[[FP]] : f16
+func.func @linalg_indexcast_dynamic_dim_i64(%arg0: tensor<f16>, %arg1: tensor<?xf16>) -> tensor<?xf16> {
+  %res = linalg.generic {
+      indexing_maps = [affine_map<(d0) -> ()>, affine_map<(d0) -> (d0)>],
+      iterator_types = ["parallel"]
+    }
+    ins(%arg0 : tensor<f16>)
+    outs(%arg1 : tensor<?xf16>) {
+  ^bb0(%in: f16, %out: f16):
+    %idx = linalg.index 0 : index
+    %int = arith.index_cast %idx : index to i64
+    %fp = arith.sitofp %int : i64 to f16
+    linalg.yield %fp : f16
+  } -> tensor<?xf16>
+
+  return %res : tensor<?xf16>
+}
+
+//===----------------------------------------------------------------------===//
+// arith.index_castui
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func @linalg_indexcastui_dim_0_i8
+// CHECK:         %[[IDX:.+]] = linalg.index 0 : index
+// CHECK-NEXT:    %[[INT:.+]] = arith.index_castui %[[IDX]] : index to i8
+// CHECK-NEXT:    %[[FP:.+]]  = arith.uitofp %[[INT]] : i8 to f16
+// CHECK-NEXT:    linalg.yield %[[FP]] : f16
+func.func @linalg_indexcastui_dim_0_i8(%arg0: tensor<f16>) -> tensor<256xf16> {
+  %init = tensor.empty() : tensor<256xf16>
+  %res = linalg.generic {
+      indexing_maps = [affine_map<(d0) -> ()>, affine_map<(d0) -> (d0)>],
+      iterator_types = ["parallel"]
+    }
+    ins(%arg0 : tensor<f16>)
+    outs(%init : tensor<256xf16>) {
+  ^bb0(%in: f16, %out: f16):
+    %idx = linalg.index 0 : index
+    %int = arith.index_castui %idx : index to i64
+    %fp = arith.uitofp %int : i64 to f16
+    linalg.yield %fp : f16
+  } -> tensor<256xf16>
+
+  return %res : tensor<256xf16>
+}
+
+// CHECK-LABEL: func @linalg_indexcastui_dim_1_i16
+// CHECK:         %[[IDX:.+]] = linalg.index 1 : index
+// CHECK-NEXT:    %[[INT:.+]] = arith.index_castui %[[IDX]] : index to i16
+// CHECK-NEXT:    %[[FP:.+]]  = arith.uitofp %[[INT]] : i16 to f16
+// CHECK-NEXT:    linalg.yield %[[FP]] : f16
+func.func @linalg_indexcastui_dim_1_i16(%arg0: tensor<f16>, %arg1: tensor<?x257xf16>) -> tensor<?x257xf16> {
+  %res = linalg.generic {
+      indexing_maps = [affine_map<(d0, d1) -> ()>, affine_map<(d0, d1) -> (d0, d1)>],
+      iterator_types = ["parallel", "parallel"]
+    }
+    ins(%arg0 : tensor<f16>)
+    outs(%arg1 : tensor<?x257xf16>) {
+  ^bb0(%in: f16, %out: f16):
+    %idx = linalg.index 1 : index
+    %int = arith.index_castui %idx : index to i64
+    %fp = arith.uitofp %int : i64 to f16
+    linalg.yield %fp : f16
+  } -> tensor<?x257xf16>
+
+  return %res : tensor<?x257xf16>
+}
+
+// CHECK-LABEL: func @linalg_indexcastui_dynamic_dim_i64
+// CHECK:         %[[IDX:.+]] = linalg.index 0 : index
+// CHECK-NEXT:    %[[INT:.+]] = arith.index_castui %[[IDX]] : index to i64
+// CHECK-NEXT:    %[[FP:.+]]  = arith.uitofp %[[INT]] : i64 to f16
+// CHECK-NEXT:    linalg.yield %[[FP]] : f16
+func.func @linalg_indexcastui_dynamic_dim_i64(%arg0: tensor<f16>, %arg1: tensor<?xf16>) -> tensor<?xf16> {
+  %res = linalg.generic {
+      indexing_maps = [affine_map<(d0) -> ()>, affine_map<(d0) -> (d0)>],
+      iterator_types = ["parallel"]
+    }
+    ins(%arg0 : tensor<f16>)
+    outs(%arg1 : tensor<?xf16>) {
+  ^bb0(%in: f16, %out: f16):
+    %idx = linalg.index 0 : index
+    %int = arith.index_castui %idx : index to i64
+    %fp = arith.uitofp %int : i64 to f16
+    linalg.yield %fp : f16
+  } -> tensor<?xf16>
+
+  return %res : tensor<?xf16>
+}
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index af3dc662e181e..475348f3cb176 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -270,7 +270,7 @@ func.func @generic_result_tensor_type(%arg0: memref<?xf32, affine_map<(i)[off]->
 // -----
 
 func.func @generic(%arg0: memref<?x?xf32>) {
-  // expected-error @+6 {{block with no terminator, has %0 = "arith.addf"(%arg1, %arg1) {fastmath = #arith.fastmath<none>} : (f32, f32) -> f32}}
+  // expected-error @+6 {{block with no terminator, has %0 = "arith.addf"(%arg1, %arg1) <{fastmath = #arith.fastmath<none>}> : (f32, f32) -> f32}}
   linalg.generic  {
     indexing_maps = [ affine_map<(i, j) -> (i, j)> ],
     iterator_types = ["parallel", "parallel"]}
diff --git a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
index 83141ec75aba1..40f9f3e0761e7 100644
--- a/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
+++ b/mlir/test/Dialect/Linalg/transform-lower-pack.mlir
@@ -183,3 +183,104 @@ transform.sequence failures(propagate) {
         !transform.op<"tensor.collapse_shape">,
         !transform.op<"tensor.extract_slice">)
 }
+
+// -----
+
+// CHECK-LABEL: func.func @pack_with_outer_dims_perm(
+func.func @pack_with_outer_dims_perm(%src: tensor<100x200x128x256xi32>,
+                                     %dest: tensor<200x4x16x100x16x32xi32>)
+    -> tensor<200x4x16x100x16x32xi32> {
+  //      CHECK: %[[C0:.*]] = arith.constant 0 : index
+  //      CHECK: tensor.pad {{.*}} low[%[[C0]], %[[C0]], %[[C0]], %[[C0]]]
+  //      CHECK:   : tensor<100x200x128x256xi32> to tensor<100x200x128x256xi32>
+  //      CHECK: tensor.expand_shape %{{.*}} [{{.*}}[0], [1], [2, 3], [4, 5]]
+  // CHECK-SAME:   : tensor<100x200x128x256xi32> into tensor<100x200x4x32x16x16xi32>
+  //      CHECK: linalg.transpose
+  // CHECK-SAME:   ins(%{{.*}} : tensor<100x200x4x32x16x16xi32>)
+  // CHECK-SAME:   outs(%{{.*}} : tensor<200x4x16x100x16x32xi32>)
+  // CHECK-SAME:   permutation = [1, 2, 4, 0, 5, 3]
+  %0 = tensor.pack %src
+    outer_dims_perm = [1, 2, 3, 0]
+    inner_dims_pos = [3, 2]
+    inner_tiles = [16, 32]
+    into %dest : tensor<100x200x128x256xi32> -> tensor<200x4x16x100x16x32xi32>
+  return %0 : tensor<200x4x16x100x16x32xi32>
+}
+
+transform.sequence failures(propagate) {
+^bb1(%module_op: !pdl.operation):
+  %pack = transform.structured.match ops{["tensor.pack"]} in %module_op
+    : (!pdl.operation) -> !transform.op<"tensor.pack">
+  transform.structured.lower_pack %pack : (!transform.op<"tensor.pack">)
+    -> (!transform.op<"tensor.pad">, !transform.op<"tensor.expand_shape">, !transform.op<"linalg.transpose">)
+}
+
+// -----
+
+// CHECK-LABEL: func.func @pack_with_pad_and_outer_dims_perm(
+func.func @pack_with_pad_and_outer_dims_perm(%src: tensor<100x200x127x255xi32>,
+                                             %dest: tensor<200x4x16x100x16x32xi32>)
+    -> tensor<200x4x16x100x16x32xi32> {
+  //      CHECK: %[[C0:.*]] = arith.constant 0 : index
+  //      CHECK: tensor.pad {{.*}} low[%[[C0]], %[[C0]], %[[C0]], %[[C0]]]
+  //      CHECK:   : tensor<100x200x127x255xi32> to tensor<100x200x128x256xi32>
+  //      CHECK: tensor.expand_shape %{{.*}} [{{.*}}[0], [1], [2, 3], [4, 5]]
+  // CHECK-SAME:   : tensor<100x200x128x256xi32> into tensor<100x200x4x32x16x16xi32>
+  //      CHECK: linalg.transpose
+  // CHECK-SAME:   ins(%{{.*}} : tensor<100x200x4x32x16x16xi32>)
+  // CHECK-SAME:   outs(%{{.*}} : tensor<200x4x16x100x16x32xi32>)
+  // CHECK-SAME:   permutation = [1, 2, 4, 0, 5, 3]
+  %cst_0 = arith.constant 0 : i32
+  %0 = tensor.pack %src
+    padding_value(%cst_0 : i32)
+    outer_dims_perm = [1, 2, 3, 0]
+    inner_dims_pos = [3, 2]
+    inner_tiles = [16, 32]
+    into %dest : tensor<100x200x127x255xi32> -> tensor<200x4x16x100x16x32xi32>
+  return %0 : tensor<200x4x16x100x16x32xi32>
+}
+
+transform.sequence failures(propagate) {
+^bb1(%module_op: !pdl.operation):
+  %pack = transform.structured.match ops{["tensor.pack"]} in %module_op
+    : (!pdl.operation) -> !transform.op<"tensor.pack">
+  transform.structured.lower_pack %pack : (!transform.op<"tensor.pack">)
+    -> (!transform.op<"tensor.pad">, !transform.op<"tensor.expand_shape">, !transform.op<"linalg.transpose">)
+}
+
+// -----
+
+// CHECK-LABEL: func.func @pack_as_pad_with_outer_dims_perm(
+func.func @pack_as_pad_with_outer_dims_perm(%arg0: tensor<129x47x16x16xf32>, %arg1: tensor<1x1x1x1x136x64x16x16xf32>) -> tensor<1x1x1x1x136x64x16x16xf32> {
+  %cst_0 = arith.constant 0.0 : f32
+
+  // tensor.pack is lowered to tensor.pad + tensor.insert_slice
+  //      CHECK: %[[C0:.*]] = arith.constant 0 : index
+  //      CHECK: %[[PAD:.*]] = tensor.pad {{.*}} low[%[[C0]], %[[C0]], %[[C0]], %[[C0]]]
+  //      CHECK:   : tensor<129x47x16x16xf32> to tensor<136x64x16x16xf32>
+  //      CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<1x1x1x1x136x64x16x16xf32>
+  //      CHECK: %[[RES:.*]] = tensor.insert_slice %[[PAD]] into %[[EMPTY]]
+  // offsets.
+  // CHECK-SAME:   [0, 0, 0, 0, 0, 0, 0, 0]
+  // sizes.
+  // CHECK-SAME:   [1, 1, 1, 1, 136, 64, 16, 16]
+  // strides multipliers.
+  // CHECK-SAME:   [1, 1, 1, 1, 1, 1, 1, 1]
+  // CHECK-SAME:   : tensor<136x64x16x16xf32> into tensor<1x1x1x1x136x64x16x16xf32>
+  //      CHECK: return %[[RES]]
+  %pack = tensor.pack %arg0
+    padding_value(%cst_0 : f32)
+    outer_dims_perm = [1, 2, 3, 0]
+    inner_dims_pos = [0, 1, 2, 3]
+    inner_tiles = [136, 64, 16, 16]
+    into %arg1 : tensor<129x47x16x16xf32> -> tensor<1x1x1x1x136x64x16x16xf32>
+  return %pack :  tensor<1x1x1x1x136x64x16x16xf32>
+}
+
+transform.sequence failures(propagate) {
+^bb1(%module_op: !pdl.operation):
+  %pack = transform.structured.match ops{["tensor.pack"]} in %module_op
+    : (!pdl.operation) -> !transform.op<"tensor.pack">
+  transform.structured.lower_pack %pack : (!transform.op<"tensor.pack">)
+    -> (!transform.op<"tensor.pad">, !transform.op<"tensor.expand_shape">, !transform.op<"linalg.transpose">)
+}
diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir
index f2f6fedabd492..4712093a81e44 100644
--- a/mlir/test/Dialect/Linalg/vectorization.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization.mlir
@@ -2836,19 +2836,21 @@ func.func @test_masked_vectorize_pad(
     -> tensor<2x4xf32>
 {
   //  CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
+  //  CHECK-DAG: %[[c0_2:.*]] = arith.constant 0 : index
   //  CHECK-DAG: %[[c42:.*]] = arith.constant 4.243000e+01 : f32
   //  CHECK-DAG: %[[empty:.*]] = tensor.empty() : tensor<2x4xf32>
   //      CHECK: %[[d0:.*]] = tensor.dim {{.*}} : tensor<?x?xf32>
   //      CHECK: %[[d1:.*]] = tensor.dim {{.*}} : tensor<?x?xf32>
   //      CHECK: %[[mask:.*]] = vector.create_mask %[[d0]], %[[d1]] : vector<2x4xi1>
   //      CHECK: %[[masked_read:.*]] = vector.mask %[[mask]] {
-  // CHECK-SAME:   vector.transfer_read %{{.*}}[%[[c0]], %[[c0]]], %[[c42]]
+  // CHECK-SAME:   vector.transfer_read %{{.*}}[%[[c0_2]], %[[c0_2]]], %[[c42]]
   // CHECK-SAME:   {in_bounds = [true, true]} : tensor<?x?xf32>, vector<2x4xf32>
   // CHECK-SAME: } : vector<2x4xi1> -> vector<2x4xf32>
-  //      CHECK: vector.transfer_write %[[masked_read]], %[[empty]][%[[c0]], %[[c0]]]
+  //      CHECK: vector.transfer_write %[[masked_read]], %[[empty]][%[[c0_2]], %[[c0_2]]]
   // CHECK-SAME:   {in_bounds = [true, true]} : vector<2x4xf32>, tensor<2x4xf32>
   %cst = arith.constant 42.43 : f32
-  %1 = tensor.pad %0 low[0, 0] high[%h0, %h1]  {
+  %c0 = arith.constant 0 : index
+  %1 = tensor.pad %0 low[0, %c0] high[%h0, %h1]  {
     ^bb0(%hh1: index, %hh2: index):
       tensor.yield %cst : f32
     } : tensor<?x?xf32> to tensor<2x4xf32>
@@ -2864,6 +2866,27 @@ transform.sequence failures(propagate) {
 
 // -----
 
+// CHECK-LABEL: func @test_masked_pad_static_dynamic
+func.func @test_masked_pad_static_dynamic(%arg0: tensor<1x2x2x?xf32>, %low: index, %high: index,
+                  %pad_value: f32) -> tensor<6x?x?x?xf32> {
+  // CHECK: tensor.pad
+  %0 = tensor.pad %arg0 low[2, %low, 3, 3] high[3, 3, %high, 2] {
+    ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: index):
+      tensor.yield %pad_value : f32
+    } : tensor<1x2x2x?xf32> to tensor<6x?x?x?xf32>
+  return %0 : tensor<6x?x?x?xf32>
+}
+
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = get_closest_isolated_parent %0 : (!pdl.operation) -> !pdl.operation
+  %2 = transform.structured.vectorize %1  { vectorize_padding }
+}
+
+// -----
+
 func.func @vectorize_dynamic_matmul(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
   linalg.matmul ins(%A, %B: memref<?x?xf32>, memref<?x?xf32>)
             outs(%C: memref<?x?xf32>)
diff --git a/mlir/test/Dialect/MemRef/make-loop-independent.mlir b/mlir/test/Dialect/MemRef/make-loop-independent.mlir
new file mode 100644
index 0000000000000..86ce9f4df4001
--- /dev/null
+++ b/mlir/test/Dialect/MemRef/make-loop-independent.mlir
@@ -0,0 +1,74 @@
+// RUN: mlir-opt %s -allow-unregistered-dialect \
+// RUN:     -test-transform-dialect-interpreter -canonicalize \
+// RUN:     -split-input-file -verify-diagnostics | FileCheck %s
+
+// CHECK: #[[$map:.*]] = affine_map<()[s0] -> (s0 - 1)>
+// CHECK-LABEL: func @make_alloca_loop_independent(
+//  CHECK-SAME:     %[[lb:.*]]: index, %[[ub:.*]]: index, %[[step:.*]]: index)
+func.func @make_alloca_loop_independent(%lb: index, %ub: index, %step: index) {
+  %cst = arith.constant 5.5 : f32
+  %c0 = arith.constant 0 : index
+  // CHECK: scf.for %[[iv:.*]] = %[[lb]] to %[[ub]]
+  scf.for %i = %lb to %ub step %step {
+    // CHECK: %[[sz:.*]] = affine.apply #[[$map]]()[%[[ub]]]
+    // CHECK: %[[alloca:.*]] = memref.alloca(%[[sz]])
+    // CHECK: %[[subview:.*]] = memref.subview %[[alloca]][0] [%[[iv]]] [1] : memref<?xf32> to memref<?xf32, strided<[1]>>
+    // CHECK: %[[cast:.*]] = builtin.unrealized_conversion_cast %[[subview]] : memref<?xf32, strided<[1]>> to memref<?xf32>
+    %alloc = memref.alloca(%i) : memref<?xf32>
+
+    // memref.subview has special handling.
+    // CHECK: %[[subview2:.*]] = memref.subview %[[subview]][1] [5] [1] : memref<?xf32, strided<[1]>> to memref<5xf32, strided<[1], offset: 1>>
+    %view = memref.subview %alloc[1][5][1] : memref<?xf32> to memref<5xf32, strided<[1], offset: 1>>
+
+    // This op takes a memref but does not produce one. The new alloc is used
+    // directly.
+    // CHECK: "test.some_use"(%[[subview2]])
+    "test.some_use"(%view) : (memref<5xf32, strided<[1], offset: 1>>) -> ()
+
+    // This op produces a memref, so the new alloc cannot be used directly.
+    // It is wrapped in a unrealized_conversion_cast.
+    // CHECK: "test.another_use"(%[[cast]]) : (memref<?xf32>) -> memref<?xf32>
+    "test.another_use"(%alloc) : (memref<?xf32>) -> (memref<?xf32>)
+
+    // CHECK: memref.store %{{.*}}, %[[subview]]
+    memref.store %cst, %alloc[%c0] : memref<?xf32>
+  }
+  return
+}
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["memref.alloca"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = transform.memref.make_loop_independent %0 {num_loops = 1}
+}
+
+// -----
+
+// CHECK: #[[$map:.*]] = affine_map<(d0) -> (-d0 + 128)>
+// CHECK-LABEL: func @make_alloca_loop_independent_static(
+func.func @make_alloca_loop_independent_static(%step: index) {
+  %cst = arith.constant 5.5 : f32
+  %c0 = arith.constant 0 : index
+  %ub = arith.constant 128 : index
+  // CHECK: scf.for %[[iv:.*]] =
+  scf.for %i = %c0 to %ub step %step {
+    // CHECK: %[[sz:.*]] = affine.apply #[[$map]](%[[iv]])
+    %sz = affine.apply affine_map<(d0)[s0] -> (-d0 + s0)>(%i)[%ub]
+
+    // CHECK: %[[alloca:.*]] = memref.alloca() : memref<128xf32>
+    // CHECK: %[[subview:.*]] = memref.subview %[[alloca]][0] [%[[sz]]] [1] : memref<128xf32> to memref<?xf32, strided<[1]>>
+    %alloc = memref.alloca(%sz) : memref<?xf32>
+
+    // CHECK: memref.store %{{.*}}, %[[subview]]
+    memref.store %cst, %alloc[%c0] : memref<?xf32>
+
+    // CHECK: vector.print %[[sz]]
+    %dim = memref.dim %alloc, %c0 : memref<?xf32>
+    vector.print %dim : index
+  }
+  return
+}
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["memref.alloca"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = transform.memref.make_loop_independent %0 {num_loops = 1}
+}
diff --git a/mlir/test/Dialect/MemRef/mem2reg.mlir b/mlir/test/Dialect/MemRef/mem2reg.mlir
new file mode 100644
index 0000000000000..86707ac0b4971
--- /dev/null
+++ b/mlir/test/Dialect/MemRef/mem2reg.mlir
@@ -0,0 +1,167 @@
+// RUN: mlir-opt %s --pass-pipeline='builtin.module(func.func(mem2reg))' --split-input-file | FileCheck %s
+
+// CHECK-LABEL: func.func @basic
+func.func @basic() -> i32 {
+  // CHECK-NOT: = memref.alloca
+  // CHECK: %[[RES:.*]] = arith.constant 5 : i32
+  // CHECK-NOT: = memref.alloca
+  %0 = arith.constant 5 : i32
+  %1 = memref.alloca() : memref<i32>
+  memref.store %0, %1[] : memref<i32>
+  %2 = memref.load %1[] : memref<i32>
+  // CHECK: return %[[RES]] : i32
+  return %2 : i32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @basic_default
+func.func @basic_default() -> i32 {
+  // CHECK-NOT: = memref.alloca
+  // CHECK: %[[RES:.*]] = arith.constant 0 : i32
+  // CHECK-NOT: = memref.alloca
+  %0 = arith.constant 5 : i32
+  %1 = memref.alloca() : memref<i32>
+  %2 = memref.load %1[] : memref<i32>
+  // CHECK-NOT: memref.store
+  memref.store %0, %1[] : memref<i32>
+  // CHECK: return %[[RES]] : i32
+  return %2 : i32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @basic_float
+func.func @basic_float() -> f32 {
+  // CHECK-NOT: = memref.alloca
+  // CHECK: %[[RES:.*]] = arith.constant {{.*}} : f32
+  %0 = arith.constant 5.2 : f32
+  // CHECK-NOT: = memref.alloca
+  %1 = memref.alloca() : memref<f32>
+  memref.store %0, %1[] : memref<f32>
+  %2 = memref.load %1[] : memref<f32>
+  // CHECK: return %[[RES]] : f32
+  return %2 : f32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @basic_ranked
+func.func @basic_ranked() -> i32 {
+  // CHECK-NOT: = memref.alloca
+  // CHECK: %[[RES:.*]] = arith.constant 5 : i32
+  // CHECK-NOT: = memref.alloca
+  %0 = arith.constant 0 : index
+  %1 = arith.constant 5 : i32
+  %2 = memref.alloca() : memref<1x1xi32>
+  memref.store %1, %2[%0, %0] : memref<1x1xi32>
+  %3 = memref.load %2[%0, %0] : memref<1x1xi32>
+  // CHECK: return %[[RES]] : i32
+  return %3 : i32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @reject_multiple_elements
+func.func @reject_multiple_elements() -> i32 {
+  // CHECK: %[[INDEX:.*]] = arith.constant 0 : index
+  %0 = arith.constant 0 : index
+  // CHECK: %[[STORED:.*]] = arith.constant 5 : i32
+  %1 = arith.constant 5 : i32
+  // CHECK: %[[ALLOCA:.*]] = memref.alloca()
+  %2 = memref.alloca() : memref<1x2xi32>
+  // CHECK: memref.store %[[STORED]], %[[ALLOCA]][%[[INDEX]], %[[INDEX]]]
+  memref.store %1, %2[%0, %0] : memref<1x2xi32>
+  // CHECK: %[[RES:.*]] = memref.load %[[ALLOCA]][%[[INDEX]], %[[INDEX]]]
+  %3 = memref.load %2[%0, %0] : memref<1x2xi32>
+  // CHECK: return %[[RES]] : i32
+  return %3 : i32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @cycle
+// CHECK-SAME: (%[[ARG0:.*]]: i64, %[[ARG1:.*]]: i1, %[[ARG2:.*]]: i64)
+func.func @cycle(%arg0: i64, %arg1: i1, %arg2: i64) {
+  // CHECK-NOT: = memref.alloca
+  %alloca = memref.alloca() : memref<i64>
+  memref.store %arg2, %alloca[] : memref<i64>
+  // CHECK: cf.cond_br %[[ARG1:.*]], ^[[BB1:.*]](%[[ARG2]] : i64), ^[[BB2:.*]](%[[ARG2]] : i64)
+  cf.cond_br %arg1, ^bb1, ^bb2
+// CHECK: ^[[BB1]](%[[USE:.*]]: i64):
+^bb1:
+  %use = memref.load %alloca[] : memref<i64>
+  // CHECK: call @use(%[[USE]])
+  func.call @use(%use) : (i64) -> ()
+  memref.store %arg0, %alloca[] : memref<i64>
+  // CHECK: cf.br ^[[BB2]](%[[ARG0]] : i64)
+  cf.br ^bb2
+// CHECK: ^[[BB2]](%[[FWD:.*]]: i64):
+^bb2:
+  // CHECK: cf.br ^[[BB1]](%[[FWD]] : i64)
+  cf.br ^bb1
+}
+
+func.func @use(%arg: i64) { return }
+
+// -----
+
+// CHECK-LABEL: func.func @recursive
+// CHECK-SAME: (%[[ARG:.*]]: i64)
+func.func @recursive(%arg: i64) -> i64 {
+  // CHECK-NOT: = memref.alloca()
+  %alloca0 = memref.alloca() : memref<memref<memref<i64>>>
+  %alloca1 = memref.alloca() : memref<memref<i64>>
+  %alloca2 = memref.alloca() : memref<i64>
+  memref.store %arg, %alloca2[] : memref<i64>
+  memref.store %alloca2, %alloca1[] : memref<memref<i64>>
+  memref.store %alloca1, %alloca0[] : memref<memref<memref<i64>>>
+  %load0 = memref.load %alloca0[] : memref<memref<memref<i64>>>
+  %load1 = memref.load %load0[] : memref<memref<i64>>
+  %load2 = memref.load %load1[] : memref<i64>
+  // CHECK: return %[[ARG]] : i64
+  return %load2 : i64
+}
+
+// -----
+
+// CHECK-LABEL: func.func @deny_store_of_alloca
+// CHECK-SAME: (%[[ARG:.*]]: memref<memref<i32>>)
+func.func @deny_store_of_alloca(%arg: memref<memref<i32>>) -> i32 {
+  // CHECK: %[[VALUE:.*]] = arith.constant 5 : i32
+  %0 = arith.constant 5 : i32
+  // CHECK: %[[ALLOCA:.*]] = memref.alloca
+  %1 = memref.alloca() : memref<i32>
+  // Storing into the memref is allowed.
+  // CHECK: memref.store %[[VALUE]], %[[ALLOCA]][]
+  memref.store %0, %1[] : memref<i32>
+  // Storing the memref itself is NOT allowed.
+  // CHECK: memref.store %[[ALLOCA]], %[[ARG]][]
+  memref.store %1, %arg[] : memref<memref<i32>>
+  // CHECK: %[[RES:.*]] = memref.load %[[ALLOCA]][]
+  %2 = memref.load %1[] : memref<i32>
+  // CHECK: return %[[RES]] : i32
+  return %2 : i32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @promotable_nonpromotable_intertwined
+func.func @promotable_nonpromotable_intertwined() -> i32 {
+  // CHECK: %[[VAL:.*]] = arith.constant 5 : i32
+  %0 = arith.constant 5 : i32
+  // CHECK: %[[NON_PROMOTED:.*]] = memref.alloca() : memref<i32>
+  %1 = memref.alloca() : memref<i32>
+  // CHECK-NOT: = memref.alloca() : memref<memref<i32>>
+  %2 = memref.alloca() : memref<memref<i32>>
+  memref.store %1, %2[] : memref<memref<i32>>
+  %3 = memref.load %2[] : memref<memref<i32>>
+  // CHECK: call @use(%[[NON_PROMOTED]])
+  call @use(%1) : (memref<i32>) -> ()
+  // CHECK: %[[RES:.*]] = memref.load %[[NON_PROMOTED]][]
+  %4 = memref.load %1[] : memref<i32>
+  // CHECK: return %[[RES]] : i32
+  return %4 : i32
+}
+
+func.func @use(%arg: memref<i32>) { return }
diff --git a/mlir/test/Dialect/SPIRV/Linking/ModuleCombiner/basic.mlir b/mlir/test/Dialect/SPIRV/Linking/ModuleCombiner/basic.mlir
index f9d46656358c0..4aed260420ae7 100644
--- a/mlir/test/Dialect/SPIRV/Linking/ModuleCombiner/basic.mlir
+++ b/mlir/test/Dialect/SPIRV/Linking/ModuleCombiner/basic.mlir
@@ -77,3 +77,11 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], [SPV_KHR_storag
 }
 }
 
+// -----
+
+// No spir-v modules to combine.
+module {
+  func.func @foo(%arg0: i32) -> i32 {
+    return %arg0 : i32
+  }
+}
diff --git a/mlir/test/Dialect/SPIRV/Transforms/unify-aliased-resource.mlir b/mlir/test/Dialect/SPIRV/Transforms/unify-aliased-resource.mlir
index 8801fdb0e1653..4f3df8639364d 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/unify-aliased-resource.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/unify-aliased-resource.mlir
@@ -506,3 +506,19 @@ spirv.module Logical GLSL450 {
 // CHECK:   %[[CC:.+]] = spirv.CompositeConstruct %[[BC0]], %[[BC1]] : (vector<2xf32>, vector<2xf32>) -> vector<4xf32>
 // CHECK:   spirv.ReturnValue %[[CC]]
 
+// -----
+
+// Make sure we do not crash on function arguments.
+
+spirv.module Logical GLSL450 {
+  spirv.func @main(%arg0: !spirv.ptr<!spirv.struct<(!spirv.rtarray<f32, stride=4> [0])>, StorageBuffer>) "None" {
+    %cst0_i32 = spirv.Constant 0 : i32
+    %0 = spirv.AccessChain %arg0[%cst0_i32, %cst0_i32] : !spirv.ptr<!spirv.struct<(!spirv.rtarray<f32, stride=4> [0])>, StorageBuffer>, i32, i32
+    spirv.Return
+  }
+}
+
+// CHECK-LABEL: spirv.module
+// CHECK-LABEL: spirv.func @main
+// CHECK-SAME:  (%{{.+}}: !spirv.ptr<!spirv.struct<(!spirv.rtarray<f32, stride=4> [0])>, StorageBuffer>) "None"
+// CHECK:       spirv.Return
diff --git a/mlir/test/Dialect/Shape/invalid.mlir b/mlir/test/Dialect/Shape/invalid.mlir
index daed6a49a0e82..ce2a438d70311 100644
--- a/mlir/test/Dialect/Shape/invalid.mlir
+++ b/mlir/test/Dialect/Shape/invalid.mlir
@@ -98,6 +98,7 @@ func.func @shape_of(%value_arg : !shape.value_shape,
 // -----
 
 func.func @shape_of_incompatible_return_types(%value_arg : tensor<1x2xindex>) {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'shape.shape_of' op inferred type(s) 'tensor<2xindex>' are incompatible with return type(s) of operation 'tensor<3xindex>'}}
   %0 = shape.shape_of %value_arg : tensor<1x2xindex> -> tensor<3xindex>
   return
@@ -268,6 +269,7 @@ func.func @fn(%arg: !shape.shape) -> !shape.witness {
 // Test that type inference flags the wrong return type.
 
 func.func @const_shape() {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tensor<3xindex>' are incompatible with return type(s) of operation 'tensor<2xindex>'}}
   %0 = shape.const_shape [4, 5, 6] : tensor<2xindex>
   return
@@ -276,6 +278,7 @@ func.func @const_shape() {
 // -----
 
 func.func @invalid_meet(%arg0 : !shape.shape, %arg1 : index) -> index {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{requires all sizes or shapes}}
   %result = shape.meet %arg0, %arg1 : !shape.shape, index -> index
   return %result : index
@@ -284,6 +287,7 @@ func.func @invalid_meet(%arg0 : !shape.shape, %arg1 : index) -> index {
 // -----
 
 func.func @invalid_meet(%arg0 : tensor<2xindex>, %arg1 : tensor<3xindex>) -> tensor<?xindex> {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{unequal shape cardinality}}
   %result = shape.meet %arg0, %arg1 : tensor<2xindex>, tensor<3xindex> -> tensor<?xindex>
   return %result : tensor<?xindex>
diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir
index b6f43adaf399c..0766e906c7216 100644
--- a/mlir/test/Dialect/SparseTensor/invalid.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid.mlir
@@ -128,6 +128,18 @@ func.func @invalid_unpack_type(%sp: tensor<100xf32, #SparseVector>)
 
 // -----
 
+#BCOO = #sparse_tensor.encoding<{dimLevelType = ["dense", "compressed-hi"], crdWidth=32}>
+
+func.func @invalid_unpack_type(%sp: tensor<2x100xf32, #BCOO>)
+                            -> (tensor<2x6xf32>, tensor<3x6x2xi32>, i32) {
+  // expected-error@+1 {{values/coordinates batched level sizes don't match statically}}
+  %values, %coordinates, %nse = sparse_tensor.unpack %sp batched_lvls=1
+     : tensor<2x100xf32, #BCOO> to tensor<2x6xf32>, tensor<3x6x2xi32>, i32
+  return %values, %coordinates, %nse : tensor<2x6xf32>, tensor<3x6x2xi32>, i32
+}
+
+// -----
+
 func.func @invalid_positions_dense(%arg0: tensor<128xf64>) -> memref<?xindex> {
   // expected-error@+1 {{'sparse_tensor.positions' op operand #0 must be sparse tensor of any type values, but got 'tensor<128xf64>'}}
   %0 = sparse_tensor.positions %arg0 { level = 0 : index } : tensor<128xf64> to memref<?xindex>
diff --git a/mlir/test/Dialect/SparseTensor/rewriting.mlir b/mlir/test/Dialect/SparseTensor/post_rewriting.mlir
old mode 100755
new mode 100644
similarity index 100%
rename from mlir/test/Dialect/SparseTensor/rewriting.mlir
rename to mlir/test/Dialect/SparseTensor/post_rewriting.mlir
diff --git a/mlir/test/Dialect/SparseTensor/pre_rewriting.mlir b/mlir/test/Dialect/SparseTensor/pre_rewriting.mlir
new file mode 100644
index 0000000000000..bbe1d6a10ee76
--- /dev/null
+++ b/mlir/test/Dialect/SparseTensor/pre_rewriting.mlir
@@ -0,0 +1,45 @@
+// RUN: mlir-opt %s -pre-sparsification-rewrite | FileCheck %s
+
+#SparseVector = #sparse_tensor.encoding<{
+  dimLevelType = ["compressed"]
+}>
+
+#SortedCOO = #sparse_tensor.encoding<{
+  dimLevelType = [ "compressed-nu", "singleton" ]
+}>
+
+#Slice = #sparse_tensor.encoding<{
+  dimLevelType = [ "compressed-nu", "singleton" ],
+  slice = [ (?, 1, 1), (?, 3, 1) ]
+}>
+
+// CHECK-LABEL: func @sparse_nop_cast(
+//  CHECK-SAME: %[[A:.*]]: tensor<?xf32, #sparse_tensor.encoding<{{{.*}}}>>)
+//       CHECK: return %[[A]] : tensor<?xf32, #sparse_tensor.encoding<{{{.*}}}>>
+func.func @sparse_nop_cast(%a : tensor<?xf32, #SparseVector>) -> tensor<?xf32, #SparseVector> {
+  %0 = tensor.cast %a : tensor<?xf32, #SparseVector> to tensor<?xf32, #SparseVector>
+  %1 = tensor.cast %0 : tensor<?xf32, #SparseVector> to tensor<?xf32, #SparseVector>
+  %2 = tensor.cast %1 : tensor<?xf32, #SparseVector> to tensor<?xf32, #SparseVector>
+  return %2 : tensor<?xf32, #SparseVector>
+}
+
+// CHECK-LABEL: func @sparse_repair_cast(
+//  CHECK-SAME: %[[A:.*]]: tensor<?xf32>)
+//       CHECK: %[[C:.*]] = sparse_tensor.convert %[[A]] : tensor<?xf32> to tensor<?xf32, #sparse_tensor.encoding<{{{.*}}}>
+//       CHECK: return %[[C]] : tensor<?xf32, #sparse_tensor.encoding<{{{.*}}}>>
+func.func @sparse_repair_cast(%a : tensor<?xf32>) -> tensor<?xf32, #SparseVector> {
+  %0 = tensor.cast %a : tensor<?xf32> to tensor<?xf32, #SparseVector>
+  return %0 : tensor<?xf32, #SparseVector>
+}
+
+// CHECK-LABEL: func @sparse_fuse_slice(
+//  CHECK-SAME: %[[A:.*]]: tensor<2x3xi64, #sparse_tensor.encoding<{{{.*}}}>>)
+//       CHECK: %[[E:.*]] = tensor.extract_slice %[[A]][1, 0] [1, 3] [1, 1] : tensor<2x3xi64, #sparse_tensor.encoding<{{{.*}}}>> to tensor<1x3xi64, #sparse_tensor.encoding<{{{.*}}}>>
+//       CHECK: %[[C:.*]] = sparse_tensor.convert %[[E]] : tensor<1x3xi64, #sparse_tensor.encoding<{{{.*}}}>> to tensor<1x3xi64, #sparse_tensor.encoding<{{{.*}}}>>
+//       CHECK: return %[[C]] : tensor<1x3xi64, #sparse_tensor.encoding<{{{.*}}}>>
+func.func @sparse_fuse_slice(%a : tensor<2x3xi64, #SortedCOO>) -> tensor<1x3xi64, #SortedCOO> {
+  %extracted_slice = tensor.extract_slice %a[1, 0] [1, 3] [1, 1] : tensor<2x3xi64, #SortedCOO> to tensor<1x3xi64>
+  %cast = tensor.cast %extracted_slice : tensor<1x3xi64> to tensor<1x3xi64, #Slice>
+  %0 = sparse_tensor.convert %cast : tensor<1x3xi64, #Slice> to tensor<1x3xi64, #SortedCOO>
+  return %0 : tensor<1x3xi64, #SortedCOO>
+}
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
index e3e548c993714..3bfa7c2164494 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
@@ -59,6 +59,21 @@ func.func @sparse_unpack(%sp : tensor<100xf64, #SparseVector>)
 
 // -----
 
+#BatchedSparseVector = #sparse_tensor.encoding<{dimLevelType = ["dense", "compressed-hi"], crdWidth=32}>
+
+// CHECK-LABEL: func @sparse_unpack(
+//  CHECK-SAME: %[[T:.*]]: tensor<2x100xf64, #
+//       CHECK: %[[D:.*]], %[[I:.*]], %[[N:.*]] = sparse_tensor.unpack %[[T]] batched_lvls = 1
+//       CHECK: return %[[D]], %[[I]], %[[N]]
+func.func @sparse_unpack(%sp : tensor<2x100xf64, #BatchedSparseVector>)
+                           -> (tensor<2x6xf64>, tensor<2x6x1xi32>, i32) {
+  %data, %indices, %nnz = sparse_tensor.unpack %sp batched_lvls=1
+       : tensor<2x100xf64, #BatchedSparseVector> to tensor<2x6xf64>, tensor<2x6x1xi32>, i32
+  return %data, %indices, %nnz : tensor<2x6xf64>, tensor<2x6x1xi32>, i32
+}
+
+// -----
+
 #SparseVector = #sparse_tensor.encoding<{dimLevelType = ["compressed"]}>
 
 // CHECK-LABEL: func @sparse_dealloc(
diff --git a/mlir/test/Dialect/SparseTensor/sparse_2d.mlir b/mlir/test/Dialect/SparseTensor/sparse_2d.mlir
index 42f2f1c35c5b4..58dc1e49dcf98 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_2d.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_2d.mlir
@@ -603,19 +603,19 @@ func.func @add_ss_ss(%arga: tensor<32x16xf32, #Tss>, %argb: tensor<32x16xf32, #T
   dimLevelType = [ "dense", "compressed-hi" ],
 }>
 // CHECK-LABEL:   func.func @sub_ss_batched(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>,
-// CHECK-SAME:      %[[VAL_1:.*]]: tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>) -> tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> {
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<2x3xf64, #{{.*}}>>,
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<2x3xf64, #{{.*}}>>) -> tensor<2x3xf64, #{{.*}}>> {
 // CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 2 : index
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_5:.*]] = bufferization.alloc_tensor() : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK-DAG:       %[[VAL_6:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xindex>
-// CHECK-DAG:       %[[VAL_7:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xindex>
-// CHECK-DAG:       %[[VAL_8:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xf64>
-// CHECK-DAG:       %[[VAL_9:.*]] = sparse_tensor.positions %[[VAL_1]] {level = 1 : index} : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xindex>
-// CHECK-DAG:       %[[VAL_10:.*]] = sparse_tensor.coordinates %[[VAL_1]] {level = 1 : index} : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xindex>
-// CHECK-DAG:       %[[VAL_11:.*]] = sparse_tensor.values %[[VAL_1]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xf64>
-// CHECK:           %[[VAL_12:.*]] = scf.for %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_2]] step %[[VAL_4]] iter_args(%[[VAL_14:.*]] = %[[VAL_5]]) -> (tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>) {
+// CHECK-DAG:       %[[VAL_5:.*]] = bufferization.alloc_tensor() : tensor<2x3xf64, #{{.*}}>>
+// CHECK-DAG:       %[[VAL_6:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<2x3xf64, #{{.*}}>> to memref<?xindex>
+// CHECK-DAG:       %[[VAL_7:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<2x3xf64, #{{.*}}>> to memref<?xindex>
+// CHECK-DAG:       %[[VAL_8:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<2x3xf64, #{{.*}}>> to memref<?xf64>
+// CHECK-DAG:       %[[VAL_9:.*]] = sparse_tensor.positions %[[VAL_1]] {level = 1 : index} : tensor<2x3xf64, #{{.*}}>> to memref<?xindex>
+// CHECK-DAG:       %[[VAL_10:.*]] = sparse_tensor.coordinates %[[VAL_1]] {level = 1 : index} : tensor<2x3xf64, #{{.*}}>> to memref<?xindex>
+// CHECK-DAG:       %[[VAL_11:.*]] = sparse_tensor.values %[[VAL_1]] : tensor<2x3xf64, #{{.*}}>> to memref<?xf64>
+// CHECK:           %[[VAL_12:.*]] = scf.for %[[VAL_13:.*]] = %[[VAL_3]] to %[[VAL_2]] step %[[VAL_4]] iter_args(%[[VAL_14:.*]] = %[[VAL_5]]) -> (tensor<2x3xf64, #{{.*}}>>) {
 // CHECK:             %[[VAL_15:.*]] = arith.muli %[[VAL_13]], %[[VAL_2]] : index
 // CHECK:             %[[VAL_16:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_15]]] : memref<?xindex>
 // CHECK:             %[[VAL_17:.*]] = arith.addi %[[VAL_15]], %[[VAL_4]] : index
@@ -628,9 +628,9 @@ func.func @add_ss_ss(%arga: tensor<32x16xf32, #Tss>, %argb: tensor<32x16xf32, #T
 // CHECK:               %[[VAL_27:.*]] = arith.cmpi ult, %[[VAL_24]], %[[VAL_18]] : index
 // CHECK:               %[[VAL_28:.*]] = arith.cmpi ult, %[[VAL_25]], %[[VAL_22]] : index
 // CHECK:               %[[VAL_29:.*]] = arith.andi %[[VAL_27]], %[[VAL_28]] : i1
-// CHECK:               scf.condition(%[[VAL_29]]) %[[VAL_24]], %[[VAL_25]], %[[VAL_26]] : index, index, tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:               scf.condition(%[[VAL_29]]) %[[VAL_24]], %[[VAL_25]], %[[VAL_26]] : index, index, tensor<2x3xf64, #{{.*}}>>
 // CHECK:             } do {
-// CHECK:             ^bb0(%[[VAL_30:.*]]: index, %[[VAL_31:.*]]: index, %[[VAL_32:.*]]: tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>):
+// CHECK:             ^bb0(%[[VAL_30:.*]]: index, %[[VAL_31:.*]]: index, %[[VAL_32:.*]]: tensor<2x3xf64, #{{.*}}>>):
 // CHECK:               %[[VAL_33:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_30]]] : memref<?xindex>
 // CHECK:               %[[VAL_34:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_31]]] : memref<?xindex>
 // CHECK:               %[[VAL_35:.*]] = arith.cmpi ult, %[[VAL_34]], %[[VAL_33]] : index
@@ -638,31 +638,31 @@ func.func @add_ss_ss(%arga: tensor<32x16xf32, #Tss>, %argb: tensor<32x16xf32, #T
 // CHECK:               %[[VAL_37:.*]] = arith.cmpi eq, %[[VAL_33]], %[[VAL_36]] : index
 // CHECK:               %[[VAL_38:.*]] = arith.cmpi eq, %[[VAL_34]], %[[VAL_36]] : index
 // CHECK:               %[[VAL_39:.*]] = arith.andi %[[VAL_37]], %[[VAL_38]] : i1
-// CHECK:               %[[VAL_40:.*]] = scf.if %[[VAL_39]] -> (tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>) {
+// CHECK:               %[[VAL_40:.*]] = scf.if %[[VAL_39]] -> (tensor<2x3xf64, #{{.*}}>>) {
 // CHECK:                 %[[VAL_41:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_30]]] : memref<?xf64>
 // CHECK:                 %[[VAL_42:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_31]]] : memref<?xf64>
 // CHECK:                 %[[VAL_43:.*]] = arith.subf %[[VAL_41]], %[[VAL_42]] : f64
-// CHECK:                 %[[VAL_44:.*]] = sparse_tensor.insert %[[VAL_43]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK:                 scf.yield %[[VAL_44]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:                 %[[VAL_44:.*]] = sparse_tensor.insert %[[VAL_43]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #{{.*}}>>
+// CHECK:                 scf.yield %[[VAL_44]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:               } else {
 // CHECK:                 %[[VAL_45:.*]] = arith.cmpi eq, %[[VAL_33]], %[[VAL_36]] : index
-// CHECK:                 %[[VAL_46:.*]] = scf.if %[[VAL_45]] -> (tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>) {
+// CHECK:                 %[[VAL_46:.*]] = scf.if %[[VAL_45]] -> (tensor<2x3xf64, #{{.*}}>>) {
 // CHECK:                   %[[VAL_47:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_30]]] : memref<?xf64>
-// CHECK:                   %[[VAL_48:.*]] = sparse_tensor.insert %[[VAL_47]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK:                   scf.yield %[[VAL_48]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:                   %[[VAL_48:.*]] = sparse_tensor.insert %[[VAL_47]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #{{.*}}>>
+// CHECK:                   scf.yield %[[VAL_48]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:                 } else {
 // CHECK:                   %[[VAL_49:.*]] = arith.cmpi eq, %[[VAL_34]], %[[VAL_36]] : index
-// CHECK:                   %[[VAL_50:.*]] = scf.if %[[VAL_49]] -> (tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>) {
+// CHECK:                   %[[VAL_50:.*]] = scf.if %[[VAL_49]] -> (tensor<2x3xf64, #{{.*}}>>) {
 // CHECK:                     %[[VAL_51:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_31]]] : memref<?xf64>
 // CHECK:                     %[[VAL_52:.*]] = arith.negf %[[VAL_51]] : f64
-// CHECK:                     %[[VAL_53:.*]] = sparse_tensor.insert %[[VAL_52]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK:                     scf.yield %[[VAL_53]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:                     %[[VAL_53:.*]] = sparse_tensor.insert %[[VAL_52]] into %[[VAL_32]]{{\[}}%[[VAL_13]], %[[VAL_36]]] : tensor<2x3xf64, #{{.*}}>>
+// CHECK:                     scf.yield %[[VAL_53]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:                   } else {
-// CHECK:                     scf.yield %[[VAL_32]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:                     scf.yield %[[VAL_32]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:                   }
-// CHECK:                   scf.yield %[[VAL_54:.*]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:                   scf.yield %[[VAL_54:.*]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:                 }
-// CHECK:                 scf.yield %[[VAL_55:.*]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:                 scf.yield %[[VAL_55:.*]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:               }
 // CHECK:               %[[VAL_56:.*]] = arith.cmpi eq, %[[VAL_33]], %[[VAL_36]] : index
 // CHECK:               %[[VAL_57:.*]] = arith.addi %[[VAL_30]], %[[VAL_4]] : index
@@ -670,25 +670,25 @@ func.func @add_ss_ss(%arga: tensor<32x16xf32, #Tss>, %argb: tensor<32x16xf32, #T
 // CHECK:               %[[VAL_59:.*]] = arith.cmpi eq, %[[VAL_34]], %[[VAL_36]] : index
 // CHECK:               %[[VAL_60:.*]] = arith.addi %[[VAL_31]], %[[VAL_4]] : index
 // CHECK:               %[[VAL_61:.*]] = arith.select %[[VAL_59]], %[[VAL_60]], %[[VAL_31]] : index
-// CHECK:               scf.yield %[[VAL_58]], %[[VAL_61]], %[[VAL_62:.*]] : index, index, tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:               scf.yield %[[VAL_58]], %[[VAL_61]], %[[VAL_62:.*]] : index, index, tensor<2x3xf64, #{{.*}}>>
 // CHECK:             } attributes {"Emitted from" = "linalg.generic"}
-// CHECK:             %[[VAL_63:.*]] = scf.for %[[VAL_64:.*]] = %[[VAL_3]] to %[[VAL_18]] step %[[VAL_4]] iter_args(%[[VAL_65:.*]] = %[[VAL_66:.*]]#2)
+// CHECK:             %[[VAL_63:.*]] = scf.for %[[VAL_64:.*]] = %[[VAL_65:.*]]#0 to %[[VAL_18]] step %[[VAL_4]] iter_args(%[[VAL_66:.*]] = %[[VAL_65]]#2)
 // CHECK:               %[[VAL_67:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_64]]] : memref<?xindex>
 // CHECK:               %[[VAL_68:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_64]]] : memref<?xf64>
-// CHECK:               %[[VAL_69:.*]] = sparse_tensor.insert %[[VAL_68]] into %[[VAL_65]]{{\[}}%[[VAL_13]], %[[VAL_67]]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK:               scf.yield %[[VAL_69]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:               %[[VAL_69:.*]] = sparse_tensor.insert %[[VAL_68]] into %[[VAL_66]]{{\[}}%[[VAL_13]], %[[VAL_67]]] : tensor<2x3xf64, #{{.*}}>>
+// CHECK:               scf.yield %[[VAL_69]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:             } {"Emitted from" = "linalg.generic"}
-// CHECK:             %[[VAL_70:.*]] = scf.for %[[VAL_71:.*]] = %[[VAL_3]] to %[[VAL_22]] step %[[VAL_4]] iter_args(%[[VAL_72:.*]] = %[[VAL_73:.*]])
-// CHECK:               %[[VAL_74:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_71]]] : memref<?xindex>
-// CHECK:               %[[VAL_75:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_71]]] : memref<?xf64>
-// CHECK:               %[[VAL_76:.*]] = arith.negf %[[VAL_75]] : f64
-// CHECK:               %[[VAL_77:.*]] = sparse_tensor.insert %[[VAL_76]] into %[[VAL_72]]{{\[}}%[[VAL_13]], %[[VAL_74]]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK:               scf.yield %[[VAL_77]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:             %[[VAL_70:.*]] = scf.for %[[VAL_71:.*]] = %[[VAL_72:.*]]#1 to %[[VAL_22]] step %[[VAL_4]] iter_args(%[[VAL_73:.*]] = %[[VAL_74:.*]])
+// CHECK:               %[[VAL_75:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_71]]] : memref<?xindex>
+// CHECK:               %[[VAL_76:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_71]]] : memref<?xf64>
+// CHECK:               %[[VAL_77:.*]] = arith.negf %[[VAL_76]] : f64
+// CHECK:               %[[VAL_78:.*]] = sparse_tensor.insert %[[VAL_77]] into %[[VAL_73]]{{\[}}%[[VAL_13]], %[[VAL_75]]] : tensor<2x3xf64, #{{.*}}>>
+// CHECK:               scf.yield %[[VAL_78]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:             } {"Emitted from" = "linalg.generic"}
-// CHECK:             scf.yield %[[VAL_78:.*]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:             scf.yield %[[VAL_79:.*]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:           } {"Emitted from" = "linalg.generic"}
-// CHECK:           %[[VAL_79:.*]] = sparse_tensor.load %[[VAL_80:.*]] hasInserts : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
-// CHECK:           return %[[VAL_79]] : tensor<2x3xf64, #sparse_tensor.encoding<{{.*}}>>
+// CHECK:           %[[VAL_80:.*]] = sparse_tensor.load %[[VAL_81:.*]] hasInserts : tensor<2x3xf64, #{{.*}}>>
+// CHECK:           return %[[VAL_80]] : tensor<2x3xf64, #{{.*}}>>
 // CHECK:         }
 func.func @sub_ss_batched(%0: tensor<2x3xf64, #BatchedVector>, %1: tensor<2x3xf64, #BatchedVector>)
                            -> tensor<2x3xf64, #BatchedVector> {
diff --git a/mlir/test/Dialect/SparseTensor/sparse_foreach.mlir b/mlir/test/Dialect/SparseTensor/sparse_foreach.mlir
index 57013e7715c43..3d95c86f4aa12 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_foreach.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_foreach.mlir
@@ -145,23 +145,25 @@ func.func @foreach_print_slice(%A: tensor<4x4xf64, #CSR_SLICE>) {
 }>
 
 // CHECK-LABEL:   func.func @foreach_bcoo(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x4x4xf64, #sparse_tensor.encoding<{{.*}}>>) {
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x4x4xf64, #{{.*}}>>) {
 // CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 4 : index
 // CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_5:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<4x4x4xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xindex>
-// CHECK-DAG:       %[[VAL_6:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<4x4x4xf64, #sparse_tensor.encoding<{{.*}}>> to memref<?xf64>
+// CHECK-DAG:       %[[VAL_5:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<4x4x4xf64, #{{.*}}>> to memref<?xindex>
+// CHECK-DAG:       %[[VAL_6:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<4x4x4xf64, #{{.*}}>> to memref<?xf64>
 // CHECK:           scf.for %[[VAL_7:.*]] = %[[VAL_2]] to %[[VAL_1]] step %[[VAL_3]] {
 // CHECK:             %[[VAL_8:.*]] = arith.muli %[[VAL_7]], %[[VAL_4]] : index
-// CHECK:             %[[VAL_9:.*]] = arith.addi %[[VAL_8]], %[[VAL_3]] : index
-// CHECK:             %[[VAL_10:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_9]]] : memref<?xindex>
-// CHECK:             scf.for %[[VAL_11:.*]] = %[[VAL_2]] to %[[VAL_10]] step %[[VAL_3]] {
-// CHECK:               %[[VAL_12:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_11]]] : memref<?xf64>
-// CHECK:               "test.use"(%[[VAL_12]]) : (f64) -> ()
-// CHECK:             }
-// CHECK:           }
+// CHECK:             %[[VAL_9:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_8]]] : memref<?xindex>
+// CHECK:             %[[VAL_10:.*]] = arith.addi %[[VAL_8]], %[[VAL_3]] : index
+// CHECK:             %[[VAL_11:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_10]]] : memref<?xindex>
+// CHECK:             scf.for %[[VAL_12:.*]] = %[[VAL_9]] to %[[VAL_11]] step %[[VAL_3]] {
+// CHECK:               %[[VAL_13:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_12]]] : memref<?xf64>
+// CHECK:               "test.use"(%[[VAL_13]]) : (f64) -> ()
+// CHECK:             } {"Emitted from" = "sparse_tensor.foreach"}
+// CHECK:           } {"Emitted from" = "sparse_tensor.foreach"}
 // CHECK:           return
+// CHECK:         }
 func.func @foreach_bcoo(%A: tensor<4x4x4xf64, #BCOO>) {
   sparse_tensor.foreach in %A : tensor<4x4x4xf64, #BCOO> do {
   ^bb0(%1: index, %2: index, %3: index,  %v: f64) :
diff --git a/mlir/test/Dialect/SparseTensor/sparse_pack.mlir b/mlir/test/Dialect/SparseTensor/sparse_pack.mlir
index 4648cb3bf2983..fb0d4a73068d9 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_pack.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_pack.mlir
@@ -45,7 +45,6 @@ func.func @sparse_pack(%values: tensor<6xf64>, %coordinates: tensor<6x2xi32>)
 // CHECK-SAME:      %[[VAL_3:.*]]
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 6 : index
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       memref.dealloc %[[VAL_0]] : memref<?xindex>
 // CHECK:           %[[VAL_6:.*]] = memref.dim %[[VAL_2]], %[[VAL_5]] : memref<?xf64>
 // CHECK:           %[[VAL_7:.*]] = arith.cmpi ugt, %[[VAL_4]], %[[VAL_6]] : index
 // CHECK:           %[[VAL_8:.*]] = scf.if %[[VAL_7]] -> (memref<6xf64>) {
@@ -69,6 +68,7 @@ func.func @sparse_pack(%values: tensor<6xf64>, %coordinates: tensor<6x2xi32>)
 // CHECK:           %[[VAL_19:.*]] = bufferization.to_tensor %[[VAL_20:.*]] : memref<6xf64>
 // CHECK:           %[[VAL_21:.*]] = bufferization.to_tensor %[[VAL_17]] : memref<6x2xi32>
 // CHECK:           %[[VAL_22:.*]] = sparse_tensor.storage_specifier
+// CHECK:       memref.dealloc %[[VAL_0]] : memref<?xindex>
 // CHECK:           return %[[VAL_19]], %[[VAL_21]], %[[VAL_22]] : tensor<6xf64>, tensor<6x2xi32>, index
 // CHECK:         }
 func.func @sparse_unpack(%sp: tensor<100x100xf64, #COO>) -> (tensor<6xf64>, tensor<6x2xi32>, index) {
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index 0a42e2bb3a5c9..4a757500920d5 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -1,5 +1,28 @@
 // RUN: mlir-opt %s -split-input-file -canonicalize="test-convergence" | FileCheck %s
 
+// CHECK-LABEL: @tensor_bitcast_chain_ok
+// CHECK-SAME: %[[IN:.*]]: tensor<2xi32>
+func.func @tensor_bitcast_chain_ok(%input: tensor<2xi32>) -> tensor<2xf32> {
+  // CHECK-NEXT: %[[RES:.*]] = tensor.bitcast %[[IN]] : tensor<2xi32> to tensor<2xf32>
+  %0 = tensor.bitcast %input : tensor<2xi32> to tensor<2xui32>
+  %1 = tensor.bitcast %0 : tensor<2xui32> to tensor<2xf32>
+  // CHECK-NEXT: return %[[RES]]
+  return %1 : tensor<2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @tensor_bitcast_chain_nop
+// CHECK-SAME: %[[IN:.*]]: tensor<4xi32>
+func.func @tensor_bitcast_chain_nop(%input: tensor<4xi32>) -> tensor<4xi32> {
+  %0 = tensor.bitcast %input : tensor<4xi32> to tensor<4xui32>
+  %1 = tensor.bitcast %0 : tensor<4xui32> to tensor<4xi32>
+  // CHECK-NEXT: return %[[IN]]
+  return %1 : tensor<4xi32>
+}
+
+// -----
+
 // Checks that NOP casts are removed.
 // CHECK-LABEL: cast_values
 func.func @cast_values(%arg0: tensor<*xi32>) -> tensor<2xi32> {
diff --git a/mlir/test/Dialect/Tensor/transform-op-make-loop-independent.mlir b/mlir/test/Dialect/Tensor/transform-op-make-loop-independent.mlir
new file mode 100644
index 0000000000000..18a99c5c437b1
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/transform-op-make-loop-independent.mlir
@@ -0,0 +1,151 @@
+// RUN: mlir-opt %s -allow-unregistered-dialect \
+// RUN:     -test-transform-dialect-interpreter -canonicalize \
+// RUN:     -split-input-file -verify-diagnostics | FileCheck %s
+
+// This is a test case where "high" padding depends on the IV.
+
+//       CHECK: #[[$map:.*]] = affine_map<()[s0, s1] -> (s0 - s1)>
+//       CHECK: #[[$map1:.*]] = affine_map<(d0)[s0, s1] -> (-d0 + s0 + s1 + 5)>
+// CHECK-LABEL: func @make_pad_loop_independent_1(
+//  CHECK-SAME:     %[[lb:.*]]: index, %[[ub:.*]]: index, %[[step:.*]]: index,
+//  CHECK-SAME:     %[[t:.*]]: tensor<?xf32>
+func.func @make_pad_loop_independent_1(%lb: index, %ub: index, %step: index,
+                                       %t: tensor<?xf32>, %f: f32) {
+  // CHECK: scf.for %[[iv:.*]] = %[[lb]] to %[[ub]]
+  scf.for %i = %lb to %ub step %step {
+    // CHECK: %[[high:.*]] = affine.apply #[[$map]]()[%[[ub]], %[[lb]]]
+    // CHECK: %[[padded:.*]] = tensor.pad %[[t]] low[5] high[%[[high]]]
+    // CHECK: %[[dim:.*]] = tensor.dim %[[t]]
+    // CHECK: %[[size:.*]] = affine.apply #[[$map1]](%[[iv]])[%[[ub]], %[[dim]]]
+    // CHECK: %[[replacement:.*]] = tensor.extract_slice %[[padded]][0] [%[[size]]] [1]
+    %high = affine.apply affine_map<(d0)[s0] -> (s0 - d0)> (%i)[%ub]
+    %p = tensor.pad %t low[5] high[%high] {
+    ^bb0(%arg1: index):
+      tensor.yield %f : f32
+    } : tensor<?xf32> to tensor<?xf32>
+    // CHECK: "dummy.some_use"(%[[replacement]])
+    "dummy.some_use"(%p) : (tensor<?xf32>) -> ()
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = transform.tensor.make_loop_independent %0 {num_loops = 1}
+}
+
+// -----
+
+// This is a test case where "low" padding depends on the IV.
+
+//       CHECK: #[[$map:.*]] = affine_map<()[s0, s1] -> (s0 - s1)>
+//       CHECK: #[[$map1:.*]] = affine_map<(d0)[s0, s1] -> (-d0 + s0 + s1 + 5)>
+//       CHECK: #[[$map2:.*]] = affine_map<(d0)[s0] -> (d0 - s0)>
+// CHECK-LABEL: func @make_pad_loop_independent_1(
+//  CHECK-SAME:     %[[lb:.*]]: index, %[[ub:.*]]: index, %[[step:.*]]: index,
+//  CHECK-SAME:     %[[t:.*]]: tensor<?xf32>
+func.func @make_pad_loop_independent_1(%lb: index, %ub: index, %step: index,
+                                       %t: tensor<?xf32>, %f: f32) {
+  // CHECK: scf.for %[[iv:.*]] = %[[lb]] to %[[ub]]
+  scf.for %i = %lb to %ub step %step {
+    // CHECK: %[[low:.*]] = affine.apply #[[$map]]()[%[[ub]], %[[lb]]]
+    // CHECK: %[[padded:.*]] = tensor.pad %[[t]] low[%[[low]]] high[5]
+    // CHECK: %[[dim:.*]] = tensor.dim %[[t]]
+    // CHECK: %[[size:.*]] = affine.apply #[[$map1]](%[[iv]])[%[[ub]], %[[dim]]]
+    // CHECK: %[[offset:.*]] = affine.apply #[[$map2]](%[[iv]])[%[[lb]]]
+    // CHECK: %[[replacement:.*]] = tensor.extract_slice %[[padded]][%[[offset]]] [%[[size]]] [1]
+    %low = affine.apply affine_map<(d0)[s0] -> (s0 - d0)> (%i)[%ub]
+    %p = tensor.pad %t low[%low] high[5] {
+    ^bb0(%arg1: index):
+      tensor.yield %f : f32
+    } : tensor<?xf32> to tensor<?xf32>
+    // CHECK: "dummy.some_use"(%[[replacement]])
+    "dummy.some_use"(%p) : (tensor<?xf32>) -> ()
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = transform.tensor.make_loop_independent %0 {num_loops = 1}
+}
+
+// -----
+
+//       CHECK: #[[$map:.*]] = affine_map<()[s0] -> (s0 * 2 - 2)>
+// CHECK-LABEL: func @two_loops(
+func.func @two_loops(%lb: index, %ub: index, %step: index,
+                     %t: tensor<?xf32>, %f: f32) {
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      // CHECK: affine.apply #map()[%{{.*}}]
+      %low = affine.apply affine_map<(d0, d1)[] -> (d0 + d1)> (%i, %j)[]
+      %p = tensor.pad %t low[%low] high[5] {
+      ^bb0(%arg1: index):
+        tensor.yield %f : f32
+      } : tensor<?xf32> to tensor<?xf32>
+      "dummy.some_use"(%p) : (tensor<?xf32>) -> ()
+    }
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = transform.tensor.make_loop_independent %0 {num_loops = 2}
+}
+
+// -----
+
+func.func @not_enough_loops(%lb: index, %ub: index, %step: index,
+                            %t: tensor<?xf32>, %f: f32) {
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %low = affine.apply affine_map<(d0, d1)[] -> (d0 + d1)> (%i, %j)[]
+      // expected-note@below {{target op}}
+      %p = tensor.pad %t low[%low] high[5] {
+      ^bb0(%arg1: index):
+        tensor.yield %f : f32
+      } : tensor<?xf32> to tensor<?xf32>
+      "dummy.some_use"(%p) : (tensor<?xf32>) -> ()
+    }
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["tensor.pad"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  // expected-error@below {{could not find 2-th enclosing loop}}
+  %1 = transform.tensor.make_loop_independent %0 {num_loops = 3}
+}
+
+// -----
+
+// CHECK: #[[$map:.*]] = affine_map<(d0)[s0] -> (-d0 + s0)>
+// CHECK: #[[$map1:.*]] = affine_map<()[s0, s1] -> (s0 - s1)>
+// CHECK-LABEL: func @make_empty_loop_independent(
+//  CHECK-SAME:     %[[lb:.*]]: index, %[[ub:.*]]: index, %[[step:.*]]: index)
+func.func @make_empty_loop_independent(%lb: index, %ub: index, %step: index) {
+  // CHECK: scf.for %[[iv:.*]] = %[[lb]] to %[[ub]]
+  scf.for %i = %lb to %ub step %step {
+    // CHECK: %[[slice_sz:.*]] = affine.apply #[[$map]](%[[iv]])[%[[ub]]]
+    // CHECK: %[[empty_sz:.*]] = affine.apply #[[$map1]]()[%[[ub]], %[[lb]]]
+    // CHECK: %[[empty:.*]] = tensor.empty(%[[empty_sz]]) : tensor<?xf32>
+    // CHECK: %[[replacement:.*]] = tensor.extract_slice %[[empty]][0] [%[[slice_sz]]] [1]
+    %sz = affine.apply affine_map<(d0)[s0] -> (s0 - d0)> (%i)[%ub]
+    %empty = tensor.empty(%sz) : tensor<?xf32>
+    // CHECK: "dummy.some_use"(%[[replacement]])
+    "dummy.some_use"(%empty) : (tensor<?xf32>) -> ()
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !pdl.operation):
+  %0 = transform.structured.match ops{["tensor.empty"]} in %arg1 : (!pdl.operation) -> !pdl.operation
+  %1 = transform.tensor.make_loop_independent %0 {num_loops = 1}
+}
diff --git a/mlir/test/Dialect/Tosa/broadcast.mlir b/mlir/test/Dialect/Tosa/broadcast.mlir
index ed1cd1e17b24d..282c386294027 100644
--- a/mlir/test/Dialect/Tosa/broadcast.mlir
+++ b/mlir/test/Dialect/Tosa/broadcast.mlir
@@ -11,7 +11,7 @@ func.func @test_broadcast0(%arg0: tensor<1xf32>, %arg1: tensor<1xf32>) -> tensor
 // -----
 // CHECK-LABEL: broadcast1
 func.func @test_broadcast1(%arg0: tensor<1xf32>, %arg1: tensor<2x1xf32>) -> tensor<2x1xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<1xf32>, tensor<2x1xf32>) -> tensor<2x1xf32>
   return %0 : tensor<2x1xf32>
@@ -20,7 +20,7 @@ func.func @test_broadcast1(%arg0: tensor<1xf32>, %arg1: tensor<2x1xf32>) -> tens
 // -----
 // CHECK-LABEL: broadcast2
 func.func @test_broadcast2(%arg0: tensor<2x1xf32>, %arg1: tensor<1xf32>) -> tensor<2x1xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<2x1xf32>, tensor<1xf32>) -> tensor<2x1xf32>
   return %0 : tensor<2x1xf32>
@@ -29,7 +29,7 @@ func.func @test_broadcast2(%arg0: tensor<2x1xf32>, %arg1: tensor<1xf32>) -> tens
 // -----
 // CHECK-LABEL: broadcast3
 func.func @test_broadcast3(%arg0: tensor<2x1x1x1xf32>, %arg1: tensor<1xf32>) -> tensor<2x1x1x1xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<2x1x1x1xf32>, tensor<1xf32>) -> tensor<2x1x1x1xf32>
   return %0 : tensor<2x1x1x1xf32>
@@ -38,7 +38,7 @@ func.func @test_broadcast3(%arg0: tensor<2x1x1x1xf32>, %arg1: tensor<1xf32>) ->
 // -----
 // CHECK-LABEL: broadcast4
 func.func @test_broadcast4(%arg0: tensor<1x1x1x2xf32>, %arg1: tensor<1xf32>) -> tensor<1x1x1x2xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<1x1x1x2xf32>, tensor<1xf32>) -> tensor<1x1x1x2xf32>
   return %0 : tensor<1x1x1x2xf32>
@@ -47,7 +47,7 @@ func.func @test_broadcast4(%arg0: tensor<1x1x1x2xf32>, %arg1: tensor<1xf32>) ->
 // -----
 // CHECK-LABEL: broadcast5
 func.func @test_broadcast5(%arg0: tensor<1x1x2x1xf32>, %arg1: tensor<1xf32>) -> tensor<1x1x2x1xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<1x1x2x1xf32>, tensor<1xf32>) -> tensor<1x1x2x1xf32>
   return %0 : tensor<1x1x2x1xf32>
@@ -56,7 +56,7 @@ func.func @test_broadcast5(%arg0: tensor<1x1x2x1xf32>, %arg1: tensor<1xf32>) ->
 // -----
 // CHECK-LABEL: broadcast6
 func.func @test_broadcast6(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<1xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<17x16x15x14xf32>, tensor<1xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -65,7 +65,7 @@ func.func @test_broadcast6(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<1xf32>)
 // -----
 // CHECK-LABEL: broadcast7
 func.func @test_broadcast7(%arg0: tensor<17x16x1x14xf32>, %arg1: tensor<1x1xf32>) -> tensor<17x16x1x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<17x16x1x14xf32>, tensor<1x1xf32>) -> tensor<17x16x1x14xf32>
   return %0 : tensor<17x16x1x14xf32>
@@ -74,7 +74,7 @@ func.func @test_broadcast7(%arg0: tensor<17x16x1x14xf32>, %arg1: tensor<1x1xf32>
 // -----
 // CHECK-LABEL: broadcast8
 func.func @test_broadcast8(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<1x1xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<17x16x15x14xf32>, tensor<1x1xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -83,7 +83,7 @@ func.func @test_broadcast8(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<1x1xf32
 // -----
 // CHECK-LABEL: broadcast9
 func.func @test_broadcast9(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<15x1xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 15, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 15, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<17x16x15x14xf32>, tensor<15x1xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -92,7 +92,7 @@ func.func @test_broadcast9(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<15x1xf3
 // -----
 // CHECK-LABEL: broadcast10
 func.func @test_broadcast10(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<15x14xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 15, 14>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 15, 14>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<17x16x15x14xf32>, tensor<15x14xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -101,7 +101,7 @@ func.func @test_broadcast10(%arg0: tensor<17x16x15x14xf32>, %arg1: tensor<15x14x
 // -----
 // CHECK-LABEL: broadcast13
 func.func @test_broadcast13(%arg0: tensor<1xf32>, %arg1: tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<1xf32>, tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -110,7 +110,7 @@ func.func @test_broadcast13(%arg0: tensor<1xf32>, %arg1: tensor<17x16x15x14xf32>
 // -----
 // CHECK-LABEL: broadcast14
 func.func @test_broadcast14(%arg0: tensor<1x1xf32>, %arg1: tensor<17x16x1x14xf32>) -> tensor<17x16x1x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<1x1xf32>, tensor<17x16x1x14xf32>) -> tensor<17x16x1x14xf32>
   return %0 : tensor<17x16x1x14xf32>
@@ -119,7 +119,7 @@ func.func @test_broadcast14(%arg0: tensor<1x1xf32>, %arg1: tensor<17x16x1x14xf32
 // -----
 // CHECK-LABEL: broadcast15
 func.func @test_broadcast15(%arg0: tensor<1x1xf32>, %arg1: tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<1x1xf32>, tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -128,7 +128,7 @@ func.func @test_broadcast15(%arg0: tensor<1x1xf32>, %arg1: tensor<17x16x15x14xf3
 // -----
 // CHECK-LABEL: broadcast16
 func.func @test_broadcast16(%arg0: tensor<15x1xf32>, %arg1: tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 15, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 15, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<15x1xf32>, tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -137,7 +137,7 @@ func.func @test_broadcast16(%arg0: tensor<15x1xf32>, %arg1: tensor<17x16x15x14xf
 // -----
 // CHECK-LABEL: broadcast17
 func.func @test_broadcast17(%arg0: tensor<15x14xf32>, %arg1: tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 15, 14>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 15, 14>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<15x14xf32>, tensor<17x16x15x14xf32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -154,7 +154,7 @@ func.func @test_broadcast18(%arg0: tensor<14x1xf32>, %arg1: tensor<1x15xf32>) ->
 // -----
 // CHECK-LABEL: broadcast19
 func.func @test_broadcast19(%arg0: tensor<64x64x1xf32>, %arg1: tensor<1x17xf32>) -> (tensor<64x64x17xf32> ) {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 17>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 17>}
   // CHECK: %[[VAR1:.*]] = "tosa.sub"(%arg0, %[[VAR0]])
   %0 = "tosa.sub"(%arg0, %arg1) : (tensor<64x64x1xf32>, tensor<1x17xf32>) -> tensor<64x64x17xf32>
   return %0 : tensor<64x64x17xf32>
@@ -163,7 +163,7 @@ func.func @test_broadcast19(%arg0: tensor<64x64x1xf32>, %arg1: tensor<1x17xf32>)
 // -----
 // CHECK-LABEL: broadcast20
 func.func @test_broadcast20(%arg0: tensor<3x3x4x1xf32>, %arg1: tensor<4x5xf32>) -> (tensor<3x3x4x5xf32> ) {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 4, 5>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 4, 5>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%arg0, %[[VAR0]])
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<3x3x4x1xf32>, tensor<4x5xf32>) -> tensor<3x3x4x5xf32>
   return %0 : tensor<3x3x4x5xf32>
@@ -172,25 +172,25 @@ func.func @test_broadcast20(%arg0: tensor<3x3x4x1xf32>, %arg1: tensor<4x5xf32>)
 // -----
 // CHECK-LABEL: broadcast_mul
 func.func @test_broadcast_mul(%arg0: tensor<15x14xi32>, %arg1: tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 15, 14>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 15, 14>}
   // CHECK: %[[VAR1:.*]] = "tosa.mul"(%[[VAR0]], %arg1)
-  %0 = "tosa.mul"(%arg0, %arg1) {shift = 1 : i32 } : (tensor<15x14xi32>, tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32>
+  %0 = "tosa.mul"(%arg0, %arg1) <{shift = 1 : i32 }> : (tensor<15x14xi32>, tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32>
   return %0 : tensor<17x16x15x14xi32>
 }
 
 // -----
 // CHECK-LABEL: broadcast_arithmetic_right_shift
 func.func @test_broadcast_arithmetic_right_shift(%arg0: tensor<15x14xi32>, %arg1: tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 15, 14>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 15, 14>}
   // CHECK: %[[VAR1:.*]] = "tosa.arithmetic_right_shift"(%[[VAR0]], %arg1)
-  %0 = "tosa.arithmetic_right_shift"(%arg0, %arg1) { round = true } : (tensor<15x14xi32>, tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32>
+  %0 = "tosa.arithmetic_right_shift"(%arg0, %arg1) <{ round = true }> : (tensor<15x14xi32>, tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32>
   return %0 : tensor<17x16x15x14xi32>
 }
 
 // -----
 // CHECK-LABEL: broadcast_scalar
 func.func @test_broadcast_scalar(%arg0: tensor<i32>, %arg1: tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32> {
-  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAR1:.*]] = "tosa.add"(%[[VAR0]], %arg1)
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<i32>, tensor<17x16x15x14xi32>) -> tensor<17x16x15x14xi32>
   return %0 : tensor<17x16x15x14xi32>
@@ -199,8 +199,8 @@ func.func @test_broadcast_scalar(%arg0: tensor<i32>, %arg1: tensor<17x16x15x14xi
 // -----
 // CHECK-LABEL: broadcast_select_both_input
 func.func @test_broadcast_select_both_input(%arg0: tensor<1x16x16xi1>, %arg1: tensor<f32>, %arg2: tensor<f32>) -> tensor<1x16x16xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) {new_shape = array<i64: 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) <{new_shape = array<i64: 1, 1, 1>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%arg0, %[[VAL_0]], %[[VAL_1]])
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<1x16x16xi1>, tensor<f32>, tensor<f32>) -> tensor<1x16x16xf32>
   return %0 : tensor<1x16x16xf32>
@@ -209,7 +209,7 @@ func.func @test_broadcast_select_both_input(%arg0: tensor<1x16x16xi1>, %arg1: te
 // -----
 // CHECK-LABEL: broadcast_select_one_input
 func.func @test_broadcast_select_one_input(%arg0: tensor<17x16x15x14xi1>, %arg1: tensor<17x16x15x14xf32>, %arg2: tensor<f32>) -> tensor<17x16x15x14xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg2) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg2) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAL_1:.*]] = "tosa.select"(%arg0, %arg1, %[[VAL_0]])
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<17x16x15x14xi1>, tensor<17x16x15x14xf32>, tensor<f32>) -> tensor<17x16x15x14xf32>
   return %0 : tensor<17x16x15x14xf32>
@@ -218,7 +218,7 @@ func.func @test_broadcast_select_one_input(%arg0: tensor<17x16x15x14xi1>, %arg1:
 // -----
 // CHECK-LABEL: broadcast_select_predicate
 func.func @test_broadcast_select_predicate(%arg0: tensor<i1>, %arg1: tensor<1x32x32x8xf32>, %arg2: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAL_1:.*]] = "tosa.select"(%[[VAL_0]], %arg1, %arg2)
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<i1>, tensor<1x32x32x8xf32>, tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
@@ -227,8 +227,8 @@ func.func @test_broadcast_select_predicate(%arg0: tensor<i1>, %arg1: tensor<1x32
 // -----
 // CHECK-LABEL: broadcast_select_abc
 func.func @test_broadcast_select_abc(%arg0: tensor<i1>, %arg1: tensor<32x8xf32>, %arg2: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 32, 8>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 32, 8>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%[[VAL_0]], %[[VAL_1]], %arg2)
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<i1>, tensor<32x8xf32>, tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
@@ -237,8 +237,8 @@ func.func @test_broadcast_select_abc(%arg0: tensor<i1>, %arg1: tensor<32x8xf32>,
 // -----
 // CHECK-LABEL: broadcast_select_acb
 func.func @test_broadcast_select_acb(%arg0: tensor<i1>, %arg1: tensor<1x32x32x8xf32>, %arg2: tensor<32x8xf32>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 1, 1>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) {new_shape = array<i64: 1, 1, 32, 8>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) <{new_shape = array<i64: 1, 1, 32, 8>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%[[VAL_0]], %arg1, %[[VAL_1]])
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<i1>, tensor<1x32x32x8xf32>, tensor<32x8xf32>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
@@ -247,8 +247,8 @@ func.func @test_broadcast_select_acb(%arg0: tensor<i1>, %arg1: tensor<1x32x32x8x
 // -----
 // CHECK-LABEL: broadcast_select_bac
 func.func @test_broadcast_select_bac(%arg0: tensor<32x8xi1>, %arg1: tensor<f32>, %arg2: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 32, 8>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 32, 8>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%[[VAL_0]], %[[VAL_1]], %arg2)
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<32x8xi1>, tensor<f32>, tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
@@ -257,8 +257,8 @@ func.func @test_broadcast_select_bac(%arg0: tensor<32x8xi1>, %arg1: tensor<f32>,
 // -----
 // CHECK-LABEL: broadcast_select_bca
 func.func @test_broadcast_select_bca(%arg0: tensor<32x8xi1>, %arg1: tensor<1x32x32x8xf32>, %arg2: tensor<i1>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 1, 32, 8>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 1, 32, 8>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%[[VAL_0]], %arg1, %[[VAL_1]])
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<32x8xi1>, tensor<1x32x32x8xf32>, tensor<i1>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
@@ -267,8 +267,8 @@ func.func @test_broadcast_select_bca(%arg0: tensor<32x8xi1>, %arg1: tensor<1x32x
 // -----
 // CHECK-LABEL: broadcast_select_cab
 func.func @test_broadcast_select_cab(%arg0: tensor<1x32x32x8xi1>, %arg1: tensor<f32>, %arg2: tensor<32x8xf32>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 1, 1>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) {new_shape = array<i64: 1, 1, 32, 8>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) <{new_shape = array<i64: 1, 1, 32, 8>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%arg0, %[[VAL_0]], %[[VAL_1]])
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<1x32x32x8xi1>, tensor<f32>, tensor<32x8xf32>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
@@ -277,8 +277,8 @@ func.func @test_broadcast_select_cab(%arg0: tensor<1x32x32x8xi1>, %arg1: tensor<
 // -----
 // CHECK-LABEL: broadcast_select_cba
 func.func @test_broadcast_select_cba(%arg0: tensor<1x32x32x8xi1>, %arg1: tensor<32x8xf32>, %arg2: tensor<i1>) -> tensor<1x32x32x8xf32> {
-  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 1, 1, 32, 8>}
-  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) {new_shape = array<i64: 1, 1, 1, 1>}
+  // CHECK-DAG: %[[VAL_0:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 1, 1, 32, 8>}
+  // CHECK-DAG: %[[VAL_1:.*]] = "tosa.reshape"(%arg2) <{new_shape = array<i64: 1, 1, 1, 1>}
   // CHECK: %[[VAL_2:.*]] = "tosa.select"(%arg0, %[[VAL_0]], %[[VAL_1]])
   %0 = "tosa.select"(%arg0, %arg1, %arg2) : (tensor<1x32x32x8xi1>, tensor<32x8xf32>, tensor<i1>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
diff --git a/mlir/test/Dialect/Tosa/canonicalize.mlir b/mlir/test/Dialect/Tosa/canonicalize.mlir
index bdd4021cb39a1..9633a9405514b 100644
--- a/mlir/test/Dialect/Tosa/canonicalize.mlir
+++ b/mlir/test/Dialect/Tosa/canonicalize.mlir
@@ -80,7 +80,7 @@ func.func @clamp_uint8_is_noop(%arg0: tensor<4xui8>) -> tensor<4xui8> {
 
 // CHECK-LABEL: @clamp_twice_is_single_clamp
 func.func @clamp_twice_is_single_clamp(%arg0: tensor<4xi8>) -> tensor<4xi8> {
-  // CHECK: "tosa.clamp"(%arg0) {max_fp = 3.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -2 : i64}
+  // CHECK: "tosa.clamp"(%arg0) <{max_fp = 3.000000e+00 : f32, max_int = 2 : i64, min_fp = -3.000000e+00 : f32, min_int = -2 : i64}
   %0 = "tosa.clamp"(%arg0) {max_fp = 3.0 : f32, max_int = 4 : i64, min_fp = -5.0 : f32, min_int = -2 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
   %1 = "tosa.clamp"(%0) {max_fp = 5.0 : f32, max_int = 2 : i64, min_fp = -3.0 : f32, min_int = -4 : i64} :  (tensor<4xi8>) -> tensor<4xi8>
   return %1 : tensor<4xi8>
@@ -151,7 +151,7 @@ func.func @pad_noop(%arg0: tensor<?x?xf32>) -> tensor<?x?xf32> {
 
 // CHECK-LABEL: @pad_determine_val_i32
 func.func @pad_determine_val_i32(%arg0: tensor<?x?xi32>, %arg1 : tensor<2x2xi32>) -> tensor<?x?xi32> {
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0> : tensor<i32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0> : tensor<i32>}
   // CHECK: "tosa.pad"(%arg0, %arg1, %[[ZERO]])
   %0 = "tosa.const"() { value = dense<[[1, 0], [0, 1]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32>
   %1 = "tosa.pad"(%arg0, %arg1) : (tensor<?x?xi32>, tensor<2x2xi32>) -> tensor<?x?xi32>
@@ -160,7 +160,7 @@ func.func @pad_determine_val_i32(%arg0: tensor<?x?xi32>, %arg1 : tensor<2x2xi32>
 
 // CHECK-LABEL: @pad_determine_val_f32
 func.func @pad_determine_val_f32(%arg0: tensor<?x?xf32>, %arg1 : tensor<2x2xi32>) -> tensor<?x?xf32> {
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0.000000e+00> : tensor<f32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0.000000e+00> : tensor<f32>}
   // CHECK: "tosa.pad"(%arg0, %arg1, %[[ZERO]])
   %0 = "tosa.const"() { value = dense<[[1, 0], [0, 1]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32>
   %1 = "tosa.pad"(%arg0, %arg1) : (tensor<?x?xf32>, tensor<2x2xi32>) -> tensor<?x?xf32>
@@ -169,7 +169,7 @@ func.func @pad_determine_val_f32(%arg0: tensor<?x?xf32>, %arg1 : tensor<2x2xi32>
 
 // CHECK-LABEL: @pad_determine_val_quant
 func.func @pad_determine_val_quant(%arg0: tensor<?x?xi32>, %arg1 : tensor<2x2xi32>) -> tensor<?x?xi32> {
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<42> : tensor<i32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<42> : tensor<i32>}
   // CHECK: "tosa.pad"(%arg0, %arg1, %[[ZERO]])
   %0 = "tosa.const"() { value = dense<[[1, 0], [0, 1]]> : tensor<2x2xi32>} : () -> tensor<2x2xi32>
   %1 = "tosa.pad"(%arg0, %arg1) {quantization_info = #tosa.pad_quant<input_zp = 42>} : (tensor<?x?xi32>, tensor<2x2xi32>) -> tensor<?x?xi32>
@@ -330,7 +330,7 @@ func.func @reshape_canonicalize(%arg0: tensor<?x10xf32>) -> tensor<?x10xf32> {
 
 // CHECK-LABEL: @reshape_canonicalize_double
 func.func @reshape_canonicalize_double(%arg0: tensor<?x10xf32>) -> tensor<?x5xf32> {
-  // CHECK: %[[VAR0:.+]] = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 5>}
+  // CHECK: %[[VAR0:.+]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: -1, 5>}
   // CHECK: return %[[VAR0]]
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 5, -1>}: (tensor<?x10xf32>) -> tensor<5x?xf32>
   %1 = "tosa.reshape"(%0) {new_shape = array<i64: -1, 5>}: (tensor<5x?xf32>) -> tensor<?x5xf32>
@@ -339,7 +339,7 @@ func.func @reshape_canonicalize_double(%arg0: tensor<?x10xf32>) -> tensor<?x5xf3
 
 // CHECK-LABEL: @reshape_canonicalize_const
 func.func @reshape_canonicalize_const() -> tensor<1x10xi32> {
-  // CHECK: %[[VAR0:.+]] = "tosa.const"() {value = dense<0> : tensor<1x10xi32>}
+  // CHECK: %[[VAR0:.+]] = "tosa.const"() <{value = dense<0> : tensor<1x10xi32>}
   // CHECK: return %[[VAR0]]
   %0 = "tosa.const"() {value = dense<0> : tensor<10xi32>} : () -> tensor<10xi32>
   %1 = "tosa.reshape"(%0) {new_shape = array<i64: 1, 10>} : (tensor<10xi32>) -> tensor<1x10xi32>
@@ -348,8 +348,8 @@ func.func @reshape_canonicalize_const() -> tensor<1x10xi32> {
 
 // CHECK-LABEL: @reshape_canonicalize_const_spat
 func.func @reshape_canonicalize_const_spat() -> (tensor<10xi32>, tensor<1x10xi32>) {
-  // CHECK-DAG: %[[VAR0:.+]] = "tosa.const"() {value = dense<0> : tensor<10xi32>}
-  // CHECK-DAG: %[[VAR1:.+]] = "tosa.const"() {value = dense<0> : tensor<1x10xi32>}
+  // CHECK-DAG: %[[VAR0:.+]] = "tosa.const"() <{value = dense<0> : tensor<10xi32>}
+  // CHECK-DAG: %[[VAR1:.+]] = "tosa.const"() <{value = dense<0> : tensor<1x10xi32>}
   // CHECK: return %[[VAR0]], %[[VAR1]]
   %0 = "tosa.const"() {value = dense<0> : tensor<10xi32>} : () -> tensor<10xi32>
   %1 = "tosa.reshape"(%0) {new_shape = array<i64: 1, 10>} : (tensor<10xi32>) -> tensor<1x10xi32>
@@ -358,12 +358,29 @@ func.func @reshape_canonicalize_const_spat() -> (tensor<10xi32>, tensor<1x10xi32
 
 // CHECK-LABEL: @reshape_canonicalize_const_sparse
 func.func @reshape_canonicalize_const_sparse() -> (tensor<3xi32>, tensor<1x3xi32>) {
-  //CHECK: "tosa.reshape"
+  // CHECK: "tosa.reshape"
   %0 = "tosa.const"() {value = dense<[1, 2, 3]> : tensor<3xi32>} : ()-> tensor<3xi32>
   %1 = "tosa.reshape"(%0) {new_shape = array<i64: 1, 3>} : (tensor<3xi32>) -> tensor<1x3xi32>
   return %0 , %1 : tensor<3xi32>, tensor<1x3xi32>
 }
 
+// CHECK-LABEL: @reshape_canonicalize_quant
+func.func @reshape_canonicalize_quant() -> (tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>) {
+  // CHECK{LITERAL}: "tosa.const"() <{value = dense<[[1, 2, 3]]> : tensor<1x3xi8>}> : () -> tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  %0 = "tosa.const"() {value = dense<[1, 2, 3]> : tensor<3xi8>} : ()-> tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>
+  %1 = "tosa.reshape"(%0) {new_shape = array<i64: 1, 3>} : (tensor<3x!quant.uniform<i8:f32, 1.000000e+00>>) -> tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  return %1 :  tensor<1x3x!quant.uniform<i8:f32, 1.000000e+00>>
+}
+
+// CHECK-LABEL: @transpose_canonicalize_strip_quant
+func.func @transpose_canonicalize_strip_quant() -> (tensor<2x1x3xi8>) {
+  // CHECK: "tosa.const"() <{value = dense<0> : tensor<2x1x3xi8>}> : () -> tensor<2x1x3xi8>
+  %perms = "tosa.const"() {value = dense<[1, 0, 2]> : tensor<3xi32>} : () -> tensor<3xi32>
+  %0 = "tosa.const"() {value = dense<0> : tensor<1x2x3xi8>} : ()-> tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>
+  %1 = "tosa.transpose"(%0, %perms) : (tensor<1x2x3x!quant.uniform<i8:f32, 1.000000e+00>>, tensor<3xi32>) -> tensor<2x1x3xi8>
+  return %1 :  tensor<2x1x3xi8>
+}
+
 // CHECK-LABEL: @slice_fold
 func.func @slice_fold(%arg0: tensor<3x4xf32>) -> tensor<3x4xf32> {
   // CHECK: return %arg0
@@ -403,8 +420,8 @@ func.func @transpose_no_op(%arg0: tensor<3x4x5x6xf32>) -> tensor<3x4x5x6xf32> {
 
 // CHECK-LABEL: @transpose_is_reshape
 func.func @transpose_is_reshape(%arg0: tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf32> {
-  // CHECK: "tosa.reshape"(%arg0) {new_shape = array<i64: 1, 4, 1, 5>} : (tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf32>
-  %perms = "tosa.const"() {value = dense<[3, 1, 0, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
+  // CHECK: "tosa.reshape"(%arg0) <{new_shape = array<i64: 1, 4, 1, 5>}> : (tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf32>
+  %perms = "tosa.const"() <{value = dense<[3, 1, 0, 2]> : tensor<4xi32>}> : () -> tensor<4xi32>
   %0 = "tosa.transpose"(%arg0, %perms) : (tensor<1x4x5x1xf32>, tensor<4xi32>) -> tensor<1x4x1x5xf32>
   return %0 : tensor<1x4x1x5xf32>
 }
@@ -412,9 +429,9 @@ func.func @transpose_is_reshape(%arg0: tensor<1x4x5x1xf32>) -> tensor<1x4x1x5xf3
 // CHECK-LABEL: @single_bit_reshape
 // https://github.com/llvm/llvm-project/issues/55440
 func.func @single_bit_reshape() -> tensor<1xi1> {
-  // CHECK: "tosa.const"() {value = dense<true> : tensor<1xi1>}
+  // CHECK: "tosa.const"() <{value = dense<true> : tensor<1xi1>}
   %0 = arith.constant dense<true> : tensor<1x1xi1>
-  %1 = "tosa.reshape"(%0) {new_shape = array<i64: 1>} : (tensor<1x1xi1>) -> tensor<1xi1>
+  %1 = "tosa.reshape"(%0) <{new_shape = array<i64: 1>}> : (tensor<1x1xi1>) -> tensor<1xi1>
   return %1 : tensor<1xi1>
 }
 
@@ -423,7 +440,7 @@ func.func @single_bit_reshape() -> tensor<1xi1> {
 // CHECK-LABEL: @fold_resize_nearest
 func.func @fold_resize_nearest(%arg0 : tensor<1x15x13x1xi8>) -> tensor<1x15x13x1xi8> {
   // CHECK: return %arg0
-  %resize = "tosa.resize"(%arg0) {mode = "NEAREST_NEIGHBOR", scale = array<i64: 2, 2, 1, 1>, offset = array<i64: 0, 0>, border = array<i64: 0, 0>} : (tensor<1x15x13x1xi8>) -> tensor<1x15x13x1xi8>
+  %resize = "tosa.resize"(%arg0) <{mode = "NEAREST_NEIGHBOR", scale = array<i64: 2, 2, 1, 1>, offset = array<i64: 0, 0>, border = array<i64: 0, 0>}> : (tensor<1x15x13x1xi8>) -> tensor<1x15x13x1xi8>
   return %resize : tensor<1x15x13x1xi8>
 }
 
@@ -464,9 +481,9 @@ func.func @canonicalize_concat_slice_middle_axis(%arg0 : tensor<1x12x12xf32>, %a
 
 // CHECK-LABEL: @canonicalize_cross_concat_inputs
 // CHECK-SAME: %[[VAL_0:.*]]: tensor<1x12x12xf32>, %[[VAL_1:.*]]: tensor<1x12x12xf32>
-// CHECK: %[[VAL_2:.*]] = "tosa.concat"(%[[VAL_0]], %[[VAL_1]]) {axis = 2 : i64} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
-// CHECK: %[[VAL_3:.*]] = "tosa.slice"(%[[VAL_2]]) {size = array<i64: 1, 12, 15>, start = array<i64: 0, 0, 0>} : (tensor<1x12x24xf32>) -> tensor<1x12x15xf32>
-// CHECK: %[[VAL_4:.*]] = "tosa.slice"(%[[VAL_2]]) {size = array<i64: 1, 12, 20>, start = array<i64: 0, 0, 4>} : (tensor<1x12x24xf32>) -> tensor<1x12x20xf32>
+// CHECK: %[[VAL_2:.*]] = "tosa.concat"(%[[VAL_0]], %[[VAL_1]]) <{axis = 2 : i64}> : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
+// CHECK: %[[VAL_3:.*]] = "tosa.slice"(%[[VAL_2]]) <{size = array<i64: 1, 12, 15>, start = array<i64: 0, 0, 0>}> : (tensor<1x12x24xf32>) -> tensor<1x12x15xf32>
+// CHECK: %[[VAL_4:.*]] = "tosa.slice"(%[[VAL_2]]) <{size = array<i64: 1, 12, 20>, start = array<i64: 0, 0, 4>}> : (tensor<1x12x24xf32>) -> tensor<1x12x20xf32>
 // CHECK: return %[[VAL_3]], %[[VAL_4]] : tensor<1x12x15xf32>, tensor<1x12x20xf32>
 func.func @canonicalize_cross_concat_inputs(%arg0 : tensor<1x12x12xf32>, %arg1 : tensor<1x12x12xf32>) -> (tensor<1x12x15xf32>, tensor<1x12x20xf32>) {
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 2 : i64} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
@@ -479,8 +496,8 @@ func.func @canonicalize_cross_concat_inputs(%arg0 : tensor<1x12x12xf32>, %arg1 :
 
 // CHECK-LABEL: @canonicalize_concat_slice_on_non_concat_axis
 // CHECK-SAME: %[[VAL_0:.*]]: tensor<1x12x12xf32>, %[[VAL_1:.*]]: tensor<1x12x12xf32>
-// CHECK: %[[VAL_2:.*]] = "tosa.slice"(%[[VAL_0]]) {size = array<i64: 1, 6, 12>, start = array<i64: 0, 0, 0>} : (tensor<1x12x12xf32>) -> tensor<1x6x12xf32>
-// CHECK: %[[VAL_3:.*]] = "tosa.slice"(%[[VAL_1]]) {size = array<i64: 1, 3, 12>, start = array<i64: 1, 3, 12>} : (tensor<1x12x12xf32>) -> tensor<1x3x12xf32>
+// CHECK: %[[VAL_2:.*]] = "tosa.slice"(%[[VAL_0]]) <{size = array<i64: 1, 6, 12>, start = array<i64: 0, 0, 0>}> : (tensor<1x12x12xf32>) -> tensor<1x6x12xf32>
+// CHECK: %[[VAL_3:.*]] = "tosa.slice"(%[[VAL_1]]) <{size = array<i64: 1, 3, 12>, start = array<i64: 1, 3, 12>}> : (tensor<1x12x12xf32>) -> tensor<1x3x12xf32>
 // CHECK: return %[[VAL_2]], %[[VAL_3]] : tensor<1x6x12xf32>, tensor<1x3x12xf32>
 func.func @canonicalize_concat_slice_on_non_concat_axis(%arg0 : tensor<1x12x12xf32>, %arg1 : tensor<1x12x12xf32>) -> (tensor<1x6x12xf32>, tensor<1x3x12xf32>) {
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 2 : i64} : (tensor<1x12x12xf32>, tensor<1x12x12xf32>) -> tensor<1x12x24xf32>
@@ -488,3 +505,13 @@ func.func @canonicalize_concat_slice_on_non_concat_axis(%arg0 : tensor<1x12x12xf
   %2 = "tosa.slice"(%0) {size = array<i64: 1, 3, 12>, start = array<i64: 1, 3, 12>} : (tensor<1x12x24xf32>) -> tensor<1x3x12xf32>
   return %1, %2 : tensor<1x6x12xf32>, tensor<1x3x12xf32>
 }
+
+// -----
+
+// CHECK-LABEL
+func.func @fold_log_exp(%arg0: tensor<?x1xf32>) -> tensor<?x1xf32> {
+  // CHECK: return %arg{{.*}} : tensor<?x1xf32>
+  %0 = "tosa.exp"(%arg0) : (tensor<?x1xf32>) -> tensor<?x1xf32>
+  %1 = "tosa.log"(%0) : (tensor<?x1xf32>) -> tensor<?x1xf32>
+  return %1 : tensor<?x1xf32>
+}
diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
index 6c8f5935b4cb0..ec4d8bd74a5e8 100644
--- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir
+++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
@@ -156,7 +156,7 @@ func.func @fold_add_splat_i32() -> tensor<10xi32> {
   %one = "tosa.const"() {value = dense<1> : tensor<10xi32>} : () -> tensor<10xi32>
   %two = "tosa.const"() {value = dense<2> : tensor<10xi32>} : () -> tensor<10xi32>
   %add = "tosa.add"(%one, %two) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32>
-  // CHECK: %[[THREE:.+]] = "tosa.const"() {value = dense<3> : tensor<10xi32>}
+  // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<3> : tensor<10xi32>}
   // CHECK: return %[[THREE]]
   return %add : tensor<10xi32>
 }
@@ -168,7 +168,7 @@ func.func @fold_add_splat_f32() -> tensor<10xf32> {
   %one = "tosa.const"() {value = dense<1.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %two = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %add = "tosa.add"(%one, %two) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32>
-  // CHECK: %[[THREE:.+]] = "tosa.const"() {value = dense<3.000000e+00> : tensor<10xf32>}
+  // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<3.000000e+00>
   // CHECK: return %[[THREE]]
   return %add : tensor<10xf32>
 }
@@ -178,7 +178,7 @@ func.func @fold_add_splat_f32() -> tensor<10xf32> {
 // CHECK-LABEL: @fold_div_zero_lhs_i32
 func.func @fold_div_zero_lhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %zero = "tosa.const"() {value = dense<0> : tensor<i32>} : () -> tensor<i32>
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0> : tensor<i32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0>
   %div = "tosa.div"(%zero, %arg0) : (tensor<i32>, tensor<i32>) -> tensor<i32>
   // CHECK: return %[[ZERO]]
   return %div : tensor<i32>
@@ -200,7 +200,7 @@ func.func @fold_div_one_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
 func.func @fold_div_splat_i32() -> tensor<i32> {
   %lhs = "tosa.const"() {value = dense<10> : tensor<i32>} : () -> tensor<i32>
   %rhs = "tosa.const"() {value = dense<-3> : tensor<i32>} : () -> tensor<i32>
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<-3> : tensor<i32>}
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<-3>
   %div = "tosa.div"(%lhs, %rhs) : (tensor<i32>, tensor<i32>) -> tensor<i32>
   // CHECK: return %[[SPLAT]]
   return %div : tensor<i32>
@@ -212,7 +212,7 @@ func.func @fold_div_splat_i32() -> tensor<i32> {
 // CHECK-LABEL: @fold_mul_zero_rhs_f32
 func.func @fold_mul_zero_rhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
   %zero = "tosa.const"() {value = dense<0.0> : tensor<f32>} : () -> tensor<f32>
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0.000000e+00> : tensor<f32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0.000000e+00>
   %mul = "tosa.mul"(%arg0, %zero) {shift = 0 : i32} : (tensor<f32>, tensor<f32>) -> tensor<f32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<f32>
@@ -223,7 +223,7 @@ func.func @fold_mul_zero_rhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
 // CHECK-LABEL: @fold_mul_zero_lhs_f32
 func.func @fold_mul_zero_lhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
   %zero = "tosa.const"() {value = dense<0.0> : tensor<f32>} : () -> tensor<f32>
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0.000000e+00> : tensor<f32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0.000000e+00>
   %mul = "tosa.mul"(%zero, %arg0) {shift = 0 : i32} : (tensor<f32>, tensor<f32>) -> tensor<f32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<f32>
@@ -234,7 +234,7 @@ func.func @fold_mul_zero_lhs_f32(%arg0: tensor<f32>) -> tensor<f32> {
 // CHECK-LABEL: @fold_mul_zero_rhs_i32
 func.func @fold_mul_zero_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %zero = "tosa.const"() {value = dense<0> : tensor<i32>} : () -> tensor<i32>
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0> : tensor<i32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0>
   %mul = "tosa.mul"(%arg0, %zero) {shift = 0 : i32} : (tensor<i32>, tensor<i32>) -> tensor<i32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<i32>
@@ -245,7 +245,7 @@ func.func @fold_mul_zero_rhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
 // CHECK-LABEL: @fold_mul_zero_lhs_i32
 func.func @fold_mul_zero_lhs_i32(%arg0: tensor<i32>) -> tensor<i32> {
   %zero = "tosa.const"() {value = dense<0> : tensor<i32>} : () -> tensor<i32>
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0> : tensor<i32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0>
   %mul = "tosa.mul"(%zero, %arg0) {shift = 0 : i32} : (tensor<i32>, tensor<i32>) -> tensor<i32>
   // CHECK: return %[[ZERO]]
   return %mul : tensor<i32>
@@ -298,7 +298,7 @@ func.func @fold_mul_splat_i8() -> tensor<10xi8> {
   %one = "tosa.const"() {value = dense<17> : tensor<10xi8>} : () -> tensor<10xi8>
   %two = "tosa.const"() {value = dense<32> : tensor<10xi8>} : () -> tensor<10xi8>
   %mul = "tosa.mul"(%one, %two) {shift = 3 : i32} : (tensor<10xi8>, tensor<10xi8>) -> tensor<10xi8>
-  // CHECK: %[[THREE:.+]] = "tosa.const"() {value = dense<68> : tensor<10xi8>}
+  // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<68> : tensor<10xi8>}
   // CHECK: return %[[THREE]]
   return %mul : tensor<10xi8>
 }
@@ -310,7 +310,7 @@ func.func @fold_mul_splat_f32() -> tensor<10xf32> {
   %one = "tosa.const"() {value = dense<3.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %two = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %mul = "tosa.mul"(%one, %two) {shift = 0 : i32} : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32>
-  // CHECK: %[[THREE:.+]] = "tosa.const"() {value = dense<6.000000e+00> : tensor<10xf32>}
+  // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<6.000000e+00> : tensor<10xf32>}
   // CHECK: return %[[THREE]]
   return %mul : tensor<10xf32>
 }
@@ -342,7 +342,7 @@ func.func @fold_sub_splat_i32() -> tensor<10xi32> {
   %one = "tosa.const"() {value = dense<1> : tensor<10xi32>} : () -> tensor<10xi32>
   %two = "tosa.const"() {value = dense<2> : tensor<10xi32>} : () -> tensor<10xi32>
   %sub = "tosa.sub"(%one, %two) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi32>
-  // CHECK: %[[THREE:.+]] = "tosa.const"() {value = dense<-1> : tensor<10xi32>}
+  // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<-1> : tensor<10xi32>}
   // CHECK: return %[[THREE]]
   return %sub : tensor<10xi32>
 }
@@ -354,7 +354,7 @@ func.func @fold_sub_splat_f32() -> tensor<10xf32> {
   %one = "tosa.const"() {value = dense<1.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %two = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %sub = "tosa.sub"(%one, %two) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xf32>
-  // CHECK: %[[THREE:.+]] = "tosa.const"() {value = dense<-1.000000e+00> : tensor<10xf32>}
+  // CHECK: %[[THREE:.+]] = "tosa.const"() <{value = dense<-1.000000e+00> : tensor<10xf32>}
   // CHECK: return %[[THREE]]
   return %sub : tensor<10xf32>
 }
@@ -369,8 +369,8 @@ func.func @fold_greater_splat_f32() -> (tensor<10xi1>, tensor<10xi1>) {
   %3 = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %true = "tosa.greater"(%0, %1) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xi1>
   %false = "tosa.greater"(%2, %3) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xi1>
-  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
-  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() {value = dense<false> : tensor<10xi1>}
+  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
+  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() <{value = dense<false> : tensor<10xi1>}
   // CHECK: return %[[TRUE]], %[[FALSE]]
   return %true, %false : tensor<10xi1>, tensor<10xi1>
 }
@@ -385,8 +385,8 @@ func.func @fold_greater_splat_i32() -> (tensor<10xi1>, tensor<10xi1>) {
   %3 = "tosa.const"() {value = dense<-12> : tensor<10xi32>} : () -> tensor<10xi32>
   %false = "tosa.greater"(%0, %1) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
   %true = "tosa.greater"(%2, %3) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
-  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() {value = dense<false> : tensor<10xi1>}
-  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
+  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() <{value = dense<false> : tensor<10xi1>}
+  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
   // CHECK: return %[[FALSE]], %[[TRUE]]
   return %false, %true : tensor<10xi1>, tensor<10xi1>
 }
@@ -401,8 +401,8 @@ func.func @fold_greater_eq_splat_f32() -> (tensor<10xi1>, tensor<10xi1>) {
   %3 = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %true = "tosa.greater_equal"(%0, %1) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xi1>
   %false = "tosa.greater_equal"(%2, %3) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xi1>
-  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
-  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() {value = dense<false> : tensor<10xi1>}
+  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
+  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() <{value = dense<false> : tensor<10xi1>}
   // CHECK: return %[[TRUE]], %[[FALSE]]
   return %true, %false : tensor<10xi1>, tensor<10xi1>
 }
@@ -417,8 +417,8 @@ func.func @fold_greater_eq_splat_i32() -> (tensor<10xi1>, tensor<10xi1>) {
   %3 = "tosa.const"() {value = dense<-10> : tensor<10xi32>} : () -> tensor<10xi32>
   %true = "tosa.greater_equal"(%2, %3) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
   %false = "tosa.greater_equal"(%0, %1) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
-  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
-  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() {value = dense<false> : tensor<10xi1>}
+  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
+  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() <{value = dense<false> : tensor<10xi1>}
   // CHECK: return %[[TRUE]], %[[FALSE]]
   return %true, %false : tensor<10xi1>, tensor<10xi1>
 }
@@ -433,8 +433,8 @@ func.func @fold_eq_splat_f32() -> (tensor<10xi1>, tensor<10xi1>) {
   %3 = "tosa.const"() {value = dense<2.0> : tensor<10xf32>} : () -> tensor<10xf32>
   %true = "tosa.equal"(%0, %1) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xi1>
   %false = "tosa.equal"(%2, %3) : (tensor<10xf32>, tensor<10xf32>) -> tensor<10xi1>
-  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
-  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() {value = dense<false> : tensor<10xi1>}
+  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
+  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() <{value = dense<false> : tensor<10xi1>}
   // CHECK: return %[[TRUE]], %[[FALSE]]
   return %true, %false : tensor<10xi1>, tensor<10xi1>
 }
@@ -449,8 +449,8 @@ func.func @fold_eq_splat_i32() -> (tensor<10xi1>, tensor<10xi1>) {
   %3 = "tosa.const"() {value = dense<-10> : tensor<10xi32>} : () -> tensor<10xi32>
   %true = "tosa.equal"(%2, %3) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
   %false = "tosa.equal"(%0, %1) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
-  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
-  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() {value = dense<false> : tensor<10xi1>}
+  // CHECK-DAG: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
+  // CHECK-DAG: %[[FALSE:.+]] = "tosa.const"() <{value = dense<false> : tensor<10xi1>}
   // CHECK: return %[[TRUE]], %[[FALSE]]
   return %true, %false : tensor<10xi1>, tensor<10xi1>
 }
@@ -459,7 +459,7 @@ func.func @fold_eq_splat_i32() -> (tensor<10xi1>, tensor<10xi1>) {
 
 // CHECK-LABEL: @fold_eq_i32
 func.func @fold_eq_i32(%arg0 : tensor<10xi32>) -> (tensor<10xi1>) {
-  // CHECK: %[[TRUE:.+]] = "tosa.const"() {value = dense<true> : tensor<10xi1>}
+  // CHECK: %[[TRUE:.+]] = "tosa.const"() <{value = dense<true> : tensor<10xi1>}
   %0 = "tosa.equal"(%arg0, %arg0) : (tensor<10xi32>, tensor<10xi32>) -> tensor<10xi1>
   // CHECK: return %[[TRUE]]
   return %0 : tensor<10xi1>
@@ -468,7 +468,7 @@ func.func @fold_eq_i32(%arg0 : tensor<10xi32>) -> (tensor<10xi1>) {
 // -----
 
 func.func @reshape_splat() -> tensor<6x5x4xi32> {
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<42> : tensor<6x5x4xi32>}
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<42> : tensor<6x5x4xi32>}
   %splat = "tosa.const"() {value = dense<42> : tensor<4x5x6xi32>} : () -> tensor<4x5x6xi32>
   %reshape = "tosa.reshape"(%splat) { new_shape = array<i64: 6, 5, 4> } : (tensor<4x5x6xi32>) -> tensor<6x5x4xi32>
   // CHECK: return %[[SPLAT]]
@@ -479,7 +479,7 @@ func.func @reshape_splat() -> tensor<6x5x4xi32> {
 
 // CHECK-LABEL: @slice_splat
 func.func @slice_splat() -> tensor<1x1x1xi32> {
-  // CHECK: %[[SLICE:.+]] = "tosa.const"() {value = dense<42> : tensor<1x1x1xi32>}
+  // CHECK: %[[SLICE:.+]] = "tosa.const"() <{value = dense<42> : tensor<1x1x1xi32>}
   %splat = "tosa.const"() {value = dense<42> : tensor<4x5x6xi32>} : () -> tensor<4x5x6xi32>
   %slice = "tosa.slice"(%splat) { size = array<i64: 1, 1, 1>, start = array<i64: 1, 2, 3> } : (tensor<4x5x6xi32>) -> tensor<1x1x1xi32>
   // CHECK: return %[[SLICE]]
@@ -491,7 +491,7 @@ func.func @slice_splat() -> tensor<1x1x1xi32> {
 // CHECK-LABEL: @slice_singleton
 func.func @slice_singleton() -> tensor<1x1xi32> {
   %splat = "tosa.const"() {value = dense<[[0, 1, 2], [3, 4, 5], [6, 7 ,8]]> : tensor<3x3xi32>} : () -> tensor<3x3xi32>
-  // CHECK: %[[SLICE:.+]] = "tosa.const"() {value = dense<4> : tensor<1x1xi32>}
+  // CHECK: %[[SLICE:.+]] = "tosa.const"() <{value = dense<4> : tensor<1x1xi32>}
   %slice = "tosa.slice"(%splat) { size = array<i64: 1, 1>, start = array<i64: 1, 1> } : (tensor<3x3xi32>) -> tensor<1x1xi32>
   // CHECK: return %[[SLICE]]
   return %slice : tensor<1x1xi32>
@@ -502,7 +502,7 @@ func.func @slice_singleton() -> tensor<1x1xi32> {
 // CHECK: func.func @cast_float_to_float
 func.func @cast_float_to_float() -> tensor<f16> {
   %splat = "tosa.const"() {value = dense<42.0> : tensor<f32>} : () -> tensor<f32>
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<4.200000e+01> : tensor<f16>} : () -> tensor<f16>
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<4.200000e+01> : tensor<f16>}
   %cast = "tosa.cast"(%splat) : (tensor<f32>) -> tensor<f16>
   // CHECK: return %[[SPLAT]]
   return %cast : tensor<f16>
@@ -513,7 +513,7 @@ func.func @cast_float_to_float() -> tensor<f16> {
 // CHECK: func.func @cast_int_to_float
 func.func @cast_int_to_float() -> tensor<f16> {
   %splat = "tosa.const"() {value = dense<4> : tensor<i32>} : () -> tensor<i32>
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<4.000000e+00> : tensor<f16>} : () -> tensor<f16>
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<4.000000e+00> : tensor<f16>}
   %cast = "tosa.cast"(%splat) : (tensor<i32>) -> tensor<f16>
   // CHECK: return %[[SPLAT]]
   return %cast : tensor<f16>
@@ -524,7 +524,7 @@ func.func @cast_int_to_float() -> tensor<f16> {
 // CHECK: func.func @cast_float_to_int
 func.func @cast_float_to_int() -> tensor<i16> {
   %splat = "tosa.const"() {value = dense<-4.0> : tensor<f32>} : () -> tensor<f32>
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<-4> : tensor<i16>} : () -> tensor<i16>
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<-4> : tensor<i16>}
   %cast = "tosa.cast"(%splat) : (tensor<f32>) -> tensor<i16>
   // CHECK: return %[[SPLAT]]
   return %cast : tensor<i16>
@@ -535,7 +535,7 @@ func.func @cast_float_to_int() -> tensor<i16> {
 // CHECK: func.func @cast_int_to_int_trunc
 func.func @cast_int_to_int_trunc() -> tensor<i16> {
   %splat = "tosa.const"() {value = dense<-1> : tensor<i32>} : () -> tensor<i32>
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<-1> : tensor<i16>} : () -> tensor<i16>
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<-1> : tensor<i16>}
   %cast = "tosa.cast"(%splat) : (tensor<i32>) -> tensor<i16>
   // CHECK: return %[[SPLAT]]
   return %cast : tensor<i16>
@@ -546,7 +546,7 @@ func.func @cast_int_to_int_trunc() -> tensor<i16> {
 // CHECK: func.func @cast_int_to_int_sign
 func.func @cast_int_to_int_sign() -> tensor<i32> {
   %splat = "tosa.const"() {value = dense<-1> : tensor<i16>} : () -> tensor<i16>
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<-1> : tensor<i32>} : () -> tensor<i32>
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<-1> : tensor<i32>}
   %cast = "tosa.cast"(%splat) : (tensor<i16>) -> tensor<i32>
   // CHECK: return %[[SPLAT]]
   return %cast : tensor<i32>
@@ -556,7 +556,7 @@ func.func @cast_int_to_int_sign() -> tensor<i32> {
 
 // CHECK-LABEL: @reverse_splat
 func.func @reverse_splat() -> tensor<10xi32> {
-  // CHECK: %[[SPLAT:.+]] = "tosa.const"() {value = dense<42> : tensor<10xi32>}
+  // CHECK: %[[SPLAT:.+]] = "tosa.const"() <{value = dense<42> : tensor<10xi32>}
   %splat = "tosa.const"() {value = dense<42> : tensor<10xi32>} : () -> tensor<10xi32>
   %reverse = "tosa.reverse"(%splat) { axis = 0 : i64 } : (tensor<10xi32>) -> tensor<10xi32>
   // CHECK: return %[[SPLAT]]
@@ -569,7 +569,7 @@ func.func @reverse_splat() -> tensor<10xi32> {
 func.func @reverse_length_one(%arg0 : tensor<10x1xi32>) -> (tensor<10x1xi32>, tensor<10x1xi32>) {
   %nofold = "tosa.reverse"(%arg0) { axis = 0 : i64 } : (tensor<10x1xi32>) -> tensor<10x1xi32>
   %fold = "tosa.reverse"(%arg0) { axis = 1 : i64 } : (tensor<10x1xi32>) -> tensor<10x1xi32>
-  // CHECK: %[[NOFOLD:.+]] = "tosa.reverse"(%arg0) {axis = 0 : i64}
+  // CHECK: %[[NOFOLD:.+]] = "tosa.reverse"(%arg0) <{axis = 0 : i64}
   // CHECK: return %[[NOFOLD]], %arg0
   return %nofold, %fold : tensor<10x1xi32>, tensor<10x1xi32>
 }
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 27661f4c57847..edb4bb0a873ec 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -39,6 +39,7 @@ func.func @test_conv2d(%arg0: tensor<1x29x29x4xi8>, %arg1: tensor<16x3x3x4xi8>,
 // -----
 
 func.func @test_concat(%arg0 : tensor<2x1xf32>, %arg1 : tensor<2x2xf32>) -> tensor<?x?xf32> {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{Cannot concat tensors with different sizes on the non-axis dimension 1}}
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<2x1xf32>, tensor<2x2xf32>) -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
@@ -47,6 +48,7 @@ func.func @test_concat(%arg0 : tensor<2x1xf32>, %arg1 : tensor<2x2xf32>) -> tens
 // -----
 
 func.func @test_concat_element_type_mismatch(%arg0 : tensor<1x2xf32>, %arg1 : tensor<2x2xf32>) -> tensor<?x?xi8> {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.concat' op inferred type(s) 'tensor<3x2xf32>' are incompatible with return type(s) of operation 'tensor<?x?xi8>}}
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<1x2xf32>, tensor<2x2xf32>) -> tensor<?x?xi8>
   return %0 : tensor<?x?xi8>
@@ -100,6 +102,7 @@ func.func @test_fully_connected_non_const(%arg0: tensor<13x21x3xf32>, %arg1: ten
 // -----
 
 func.func @test_reduce_sum_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reduce_sum' op inferred type(s) 'tensor<1x3x4x5xf32>' are incompatible with return type(s) of operation 'tensor<1x3x4x5xi32>'}}
   %0 = "tosa.reduce_sum"(%arg0) {axis = 0 : i64} : (tensor<2x3x4x5xf32>) -> tensor<1x3x4x5xi32>
   return
@@ -108,6 +111,7 @@ func.func @test_reduce_sum_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
 // -----
 
 func.func @test_reduce_max_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reduce_max' op inferred type(s) 'tensor<2x3x4x1xf32>' are incompatible with return type(s) of operation 'tensor<2x3x4x1xi32>'}}
   %0 = "tosa.reduce_max"(%arg0) {axis = 3 : i64} : (tensor<2x3x4x5xf32>) -> tensor<2x3x4x1xi32>
   return
@@ -116,6 +120,7 @@ func.func @test_reduce_max_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
 // -----
 
 func.func @test_reduce_min_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reduce_min' op inferred type(s) 'tensor<2x1x4x5xf32>' are incompatible with return type(s) of operation 'tensor<2x1x4x5xi32>'}}
   %0 = "tosa.reduce_min"(%arg0) {axis = 1 : i64} : (tensor<2x3x4x5xf32>) -> tensor<2x1x4x5xi32>
   return
@@ -124,6 +129,7 @@ func.func @test_reduce_min_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
 // -----
 
 func.func @test_reduce_prod_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reduce_prod' op inferred type(s) 'tensor<2x1x4x5xf32>' are incompatible with return type(s) of operation 'tensor<2x3x4x5xf32>'}}
   %0 = "tosa.reduce_prod"(%arg0) {axis = 1 : i64} : (tensor<2x3x4x5xf32>) -> tensor<2x3x4x5xf32>
   return
@@ -132,6 +138,7 @@ func.func @test_reduce_prod_type_mismatch(%arg0 : tensor<2x3x4x5xf32>) -> () {
 // -----
 
 func.func @test_reshape_type_mismatch(%arg0 : tensor<13x21x3xf32>) -> () {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{'tosa.reshape' op inferred type(s) 'tensor<13x21x3x1xf32>' are incompatible with return type(s) of operation 'tensor<13x21x3x1xi32>'}}
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3, 1>} : (tensor<13x21x3xf32>) -> tensor<13x21x3x1xi32>
   return
diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir
index 77a7b7e884048..b8f0177646fc8 100644
--- a/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-decompose-conv2d.mlir
@@ -5,13 +5,13 @@
 // CHECK-LABEL: @conv2d_as_fully_connected
 func.func @conv2d_as_fully_connected(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<3x1x1x2xf32>, %arg2: tensor<3xf32>) -> tensor<4x10x10x3xf32> {
   // CHECK-NOT: "tosa.conv2d"
-  // CHECK: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 400, 2>}
+  // CHECK: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 400, 2>}
   // CHECK-SAME: -> tensor<400x2xf32>
-  // CHECK: %[[VAR1:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 3, 2>}
+  // CHECK: %[[VAR1:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 3, 2>}
   // CHECK-SAME: -> tensor<3x2xf32>
   // CHECK: %[[VAR2:.*]] = "tosa.fully_connected"(%[[VAR0]], %[[VAR1]], %arg2)
   // CHECK-SAME: -> tensor<400x3xf32>
-  // CHECK: %[[VAR3:.*]] = "tosa.reshape"(%[[VAR2]]) {new_shape = array<i64: 4, 10, 10, 3>}
+  // CHECK: %[[VAR3:.*]] = "tosa.reshape"(%[[VAR2]]) <{new_shape = array<i64: 4, 10, 10, 3>}
   // CHECK-SAME: -> tensor<4x10x10x3xf32>
   // CHECK: return %[[VAR3]]
   %0 = "tosa.conv2d"(%arg0, %arg1, %arg2) {pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>} : (tensor<4x10x10x2xf32>, tensor<3x1x1x2xf32>, tensor<3xf32>) -> tensor<4x10x10x3xf32>
@@ -23,14 +23,14 @@ func.func @conv2d_as_fully_connected(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor
 // CHECK-LABEL: @conv2d_as_fully_connected_quant
 func.func @conv2d_as_fully_connected_quant(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<3x1x1x2xi8>, %arg2: tensor<3xi32>) -> tensor<4x10x10x3xi32> {
   // CHECK-NOT: "tosa.conv2d"
-  // CHECK: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 400, 2>}
+  // CHECK: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 400, 2>}
   // CHECK-SAME: -> tensor<400x2xi8>
-  // CHECK: %[[VAR1:.*]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 3, 2>}
+  // CHECK: %[[VAR1:.*]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 3, 2>}
   // CHECK-SAME: -> tensor<3x2xi8>
   // CHECK: %[[VAR2:.*]] = "tosa.fully_connected"(%[[VAR0]], %[[VAR1]], %arg2)
   // CHECK-SAME: quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>
   // CHECK-SAME: -> tensor<400x3xi32>
-  // CHECK: %[[VAR3:.*]] = "tosa.reshape"(%[[VAR2]]) {new_shape = array<i64: 4, 10, 10, 3>}
+  // CHECK: %[[VAR3:.*]] = "tosa.reshape"(%[[VAR2]]) <{new_shape = array<i64: 4, 10, 10, 3>}
   // CHECK-SAME: -> tensor<4x10x10x3xi32>
   // CHECK: return %[[VAR3]]
   %0 = "tosa.conv2d"(%arg0, %arg1, %arg2) {pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>} : (tensor<4x10x10x2xi8>, tensor<3x1x1x2xi8>, tensor<3xi32>) -> tensor<4x10x10x3xi32>
@@ -44,10 +44,10 @@ func.func @conv2d_as_fully_connected_quant(%arg0: tensor<4x10x10x2xi8>, %arg1: t
 // CHECK-SAME:                                     %[[VAL_1:.*]]: tensor<384x1x1x64xi8>,
 // CHECK-SAME:                                     %[[VAL_2:.*]]: tensor<384xi32>) -> tensor<?x14x14x384xi32> {
 func.func @conv_with_dynamic_dim(%arg0: tensor<?x14x14x64xi8>, %arg1: tensor<384x1x1x64xi8>, %arg2: tensor<384xi32>) -> tensor<?x14x14x384xi32> {
-// CHECK:           %[[VAL_3:.*]] = "tosa.reshape"(%[[VAL_0]]) {new_shape = array<i64: -1, 64>} : (tensor<?x14x14x64xi8>) -> tensor<?x64xi8>
-// CHECK:           %[[VAL_4:.*]] = "tosa.reshape"(%[[VAL_1]]) {new_shape = array<i64: 384, 64>} : (tensor<384x1x1x64xi8>) -> tensor<384x64xi8>
-// CHECK:           %[[VAL_5:.*]] = "tosa.fully_connected"(%[[VAL_3]], %[[VAL_4]], %[[VAL_2]]) {quantization_info = #tosa.conv_quant<input_zp = -6, weight_zp = 11>} : (tensor<?x64xi8>, tensor<384x64xi8>, tensor<384xi32>) -> tensor<?x384xi32>
-// CHECK:           %[[VAL_6:.*]] = "tosa.reshape"(%[[VAL_5]]) {new_shape = array<i64: -1, 14, 14, 384>} : (tensor<?x384xi32>) -> tensor<?x14x14x384xi32>
+// CHECK:           %[[VAL_3:.*]] = "tosa.reshape"(%[[VAL_0]]) <{new_shape = array<i64: -1, 64>}> : (tensor<?x14x14x64xi8>) -> tensor<?x64xi8>
+// CHECK:           %[[VAL_4:.*]] = "tosa.reshape"(%[[VAL_1]]) <{new_shape = array<i64: 384, 64>}> : (tensor<384x1x1x64xi8>) -> tensor<384x64xi8>
+// CHECK:           %[[VAL_5:.*]] = "tosa.fully_connected"(%[[VAL_3]], %[[VAL_4]], %[[VAL_2]]) <{quantization_info = #tosa.conv_quant<input_zp = -6, weight_zp = 11>}> : (tensor<?x64xi8>, tensor<384x64xi8>, tensor<384xi32>) -> tensor<?x384xi32>
+// CHECK:           %[[VAL_6:.*]] = "tosa.reshape"(%[[VAL_5]]) <{new_shape = array<i64: -1, 14, 14, 384>}> : (tensor<?x384xi32>) -> tensor<?x14x14x384xi32>
 // CHECK:           return %[[VAL_6]] : tensor<?x14x14x384xi32>
 // CHECK:         }
   %0 = "tosa.conv2d"(%arg0, %arg1, %arg2) {dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -6, weight_zp = 11>, stride = array<i64: 1, 1>} : (tensor<?x14x14x64xi8>, tensor<384x1x1x64xi8>, tensor<384xi32>) -> tensor<?x14x14x384xi32>
@@ -58,13 +58,13 @@ func.func @conv_with_dynamic_dim(%arg0: tensor<?x14x14x64xi8>, %arg1: tensor<384
 
 // CHECK-LABEL: @conv2d_as_fully_connected_padded
 func.func @conv2d_as_fully_connected_padded(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<3x1x1x2xi8>, %arg2: tensor<3xi32>) -> tensor<4x12x12x3xi32> {
-  // CHECK-DAG: %[[PAD_SHAPE:.+]] = "tosa.const"() {value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0]]> : tensor<4x2xi64>}
-  // CHECK-DAG: %[[PAD_VAL:.+]] = "tosa.const"() {value = dense<42> : tensor<i8>}
+  // CHECK-DAG: %[[PAD_SHAPE:.+]] = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0]]> : tensor<4x2xi64>}
+  // CHECK-DAG: %[[PAD_VAL:.+]] = "tosa.const"() <{value = dense<42> : tensor<i8>}
   // CHECK-DAG: %[[PAD:.+]] = "tosa.pad"(%arg0, %[[PAD_SHAPE]], %[[PAD_VAL]]) : (tensor<4x10x10x2xi8>, tensor<4x2xi64>, tensor<i8>) -> tensor<4x12x12x2xi8>
-  // CHECK-DAG: %[[RESHAPE_INPUT:.+]] = "tosa.reshape"(%[[PAD]]) {new_shape = array<i64: 576, 2>}
-  // CHECK-DAG: %[[RESHAPE_FILTER:.+]] = "tosa.reshape"(%arg1) {new_shape = array<i64: 3, 2>}
-  // CHECK-DAG: %[[FULLY:.+]] = "tosa.fully_connected"(%[[RESHAPE_INPUT]], %[[RESHAPE_FILTER]], %arg2) {quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>}
-  // CHECK: %[[RESHAPE:.+]] = "tosa.reshape"(%[[FULLY]]) {new_shape = array<i64: 4, 12, 12, 3>}
+  // CHECK-DAG: %[[RESHAPE_INPUT:.+]] = "tosa.reshape"(%[[PAD]]) <{new_shape = array<i64: 576, 2>}
+  // CHECK-DAG: %[[RESHAPE_FILTER:.+]] = "tosa.reshape"(%arg1) <{new_shape = array<i64: 3, 2>}
+  // CHECK-DAG: %[[FULLY:.+]] = "tosa.fully_connected"(%[[RESHAPE_INPUT]], %[[RESHAPE_FILTER]], %arg2) <{quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>}
+  // CHECK: %[[RESHAPE:.+]] = "tosa.reshape"(%[[FULLY]]) <{new_shape = array<i64: 4, 12, 12, 3>}
   %0 = "tosa.conv2d"(%arg0, %arg1, %arg2) {pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 42, weight_zp = 24>} : (tensor<4x10x10x2xi8>, tensor<3x1x1x2xi8>, tensor<3xi32>) -> tensor<4x12x12x3xi32>
   return %0 : tensor<4x12x12x3xi32>
 }
diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir
index f9458c78300c5..e835991273ec5 100644
--- a/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-decompose-depthwise.mlir
@@ -5,11 +5,11 @@
 // CHECK-LABEL: @depthwise_conv2d_as_mul
 func.func @depthwise_conv2d_as_mul(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1x1x2x3xf32>, %arg2: tensor<6xf32>) -> tensor<4x10x10x6xf32> {
   // CHECK-NOT: "tosa.depthwise_conv2d"
-  // CHECK: %[[VAR0:.*]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 4, 10, 10, 2, 1>}
+  // CHECK: %[[VAR0:.*]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 4, 10, 10, 2, 1>}
   // CHECK-SAME: -> tensor<4x10x10x2x1xf32>
   // CHECK: %[[VAR2:.*]] = "tosa.mul"(%[[VAR0]], %arg1)
   // CHECK-SAME: -> tensor<4x10x10x2x3xf32>
-  // CHECK: %[[VAR3:.*]] = "tosa.reshape"(%[[VAR2]]) {new_shape = array<i64: 4, 10, 10, 6>}
+  // CHECK: %[[VAR3:.*]] = "tosa.reshape"(%[[VAR2]]) <{new_shape = array<i64: 4, 10, 10, 6>}
   // CHECK-SAME: -> tensor<4x10x10x6xf32>
   // CHECK: %[[VAR4:.*]] = "tosa.add"(%[[VAR3]], %arg2)
   // CHECK-SAME: -> tensor<4x10x10x6xf32>
@@ -22,15 +22,15 @@ func.func @depthwise_conv2d_as_mul(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1
 
 // CHECK-LABEL: @depthwise_conv2d_as_mul_q
 func.func @depthwise_conv2d_as_mul_q(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<1x1x2x3xi8>, %arg2: tensor<6xi32>) -> tensor<4x10x10x6xi32> {
-  // CHECK: %[[iZp:.+]] = "tosa.const"() {value = dense<7> : tensor<i32>}
-  // CHECK: %[[wZp:.+]] = "tosa.const"() {value = dense<11> : tensor<i32>}
-  // CHECK: %[[rIn:.+]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 4, 10, 10, 2, 1>}
+  // CHECK: %[[iZp:.+]] = "tosa.const"() <{value = dense<7> : tensor<i32>}
+  // CHECK: %[[wZp:.+]] = "tosa.const"() <{value = dense<11> : tensor<i32>}
+  // CHECK: %[[rIn:.+]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 4, 10, 10, 2, 1>}
   // CHECK: %[[cIn:.+]] = "tosa.cast"(%[[rIn]]) : (tensor<4x10x10x2x1xi8>) -> tensor<4x10x10x2x1xi32>
   // CHECK: %[[cWe:.+]] = "tosa.cast"(%arg1) : (tensor<1x1x2x3xi8>) -> tensor<1x1x2x3xi32>
   // CHECK: %[[sIn:.+]] = "tosa.sub"(%[[cIn]], %[[iZp]])
   // CHECK: %[[sWe:.+]] = "tosa.sub"(%[[cWe]], %[[wZp]])
-  // CHECK: %[[mul:.+]] = "tosa.mul"(%[[sIn]], %[[sWe]]) {shift = 0 : i32}
-  // CHECK: %[[reO:.+]] = "tosa.reshape"(%[[mul]]) {new_shape = array<i64: 4, 10, 10, 6>}
+  // CHECK: %[[mul:.+]] = "tosa.mul"(%[[sIn]], %[[sWe]]) <{shift = 0 : i32}
+  // CHECK: %[[reO:.+]] = "tosa.reshape"(%[[mul]]) <{new_shape = array<i64: 4, 10, 10, 6>}
   // CHECK: %[[add:.+]] = "tosa.add"(%[[reO]], %arg2)
   %0 = "tosa.depthwise_conv2d"(%arg0, %arg1, %arg2) {pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>, quantization_info = #tosa.conv_quant<input_zp = 7, weight_zp = 11>} : (tensor<4x10x10x2xi8>, tensor<1x1x2x3xi8>, tensor<6xi32>) -> tensor<4x10x10x6xi32>
   return %0 : tensor<4x10x10x6xi32>
@@ -40,12 +40,12 @@ func.func @depthwise_conv2d_as_mul_q(%arg0: tensor<4x10x10x2xi8>, %arg1: tensor<
 
 // CHECK-LABEL: @depthwise_conv2d_as_mul_padded
 func.func @depthwise_conv2d_as_mul_padded(%arg0: tensor<4x10x10x2xf32>, %arg1: tensor<1x1x2x3xf32>, %arg2: tensor<6xf32>) -> tensor<4x12x12x6xf32> {
-  // CHECK: %[[pad:.+]] = "tosa.const"() {value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0], [0, 0]]> : tensor<5x2xi64>}
-  // CHECK: %[[zero:.+]] = "tosa.const"() {value = dense<0.000000e+00> : tensor<f32>}
-  // CHECK: %[[reIn:.+]] = "tosa.reshape"(%arg0) {new_shape = array<i64: 4, 10, 10, 2, 1>}
+  // CHECK: %[[pad:.+]] = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0], [0, 0]]> : tensor<5x2xi64>}
+  // CHECK: %[[zero:.+]] = "tosa.const"() <{value = dense<0.000000e+00> : tensor<f32>}
+  // CHECK: %[[reIn:.+]] = "tosa.reshape"(%arg0) <{new_shape = array<i64: 4, 10, 10, 2, 1>}
   // CHECK: %[[padded:.+]] = "tosa.pad"(%[[reIn]], %[[pad]], %[[zero]]) : (tensor<4x10x10x2x1xf32>, tensor<5x2xi64>, tensor<f32>) -> tensor<4x12x12x2x1xf32>
-  // CHECK: %[[mul:.+]] = "tosa.mul"(%3, %arg1) {shift = 0 : i32}
-  // CHECK: %[[reOut:.+]] = "tosa.reshape"(%[[mul]]) {new_shape = array<i64: 4, 12, 12, 6>}
+  // CHECK: %[[mul:.+]] = "tosa.mul"(%3, %arg1) <{shift = 0 : i32}
+  // CHECK: %[[reOut:.+]] = "tosa.reshape"(%[[mul]]) <{new_shape = array<i64: 4, 12, 12, 6>}
   // CHECK: %[[add:.+]] = "tosa.add"(%[[reOut]], %arg2)
   %0 = "tosa.depthwise_conv2d"(%arg0, %arg1, %arg2) {pad = array<i64: 1, 1, 1, 1>, stride = array<i64: 1, 1>, dilation = array<i64: 1, 1>} : (tensor<4x10x10x2xf32>, tensor<1x1x2x3xf32>, tensor<6xf32>) -> tensor<4x12x12x6xf32>
   return %0 : tensor<4x12x12x6xf32>
diff --git a/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir b/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir
index 81b046e042002..6ccf510804d99 100644
--- a/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-decompose-transpose-conv.mlir
@@ -2,8 +2,8 @@
 
 // CHECK-LABEL: @transpose_conv2d
 func.func @transpose_conv2d(%arg0: tensor<2x16x14x3xf32>, %arg1: tensor<5x3x6x3xf32>, %arg2: tensor<5xf32>) -> tensor<2x18x19x5xf32> {
-  // CHECK: %[[REV1:.+]] = "tosa.reverse"(%arg1) {axis = 1 : i64}
-  // CHECK: %[[REV2:.+]] = "tosa.reverse"(%[[REV1]]) {axis = 2 : i64}
+  // CHECK: %[[REV1:.+]] = "tosa.reverse"(%arg1) <{axis = 1 : i64}
+  // CHECK: %[[REV2:.+]] = "tosa.reverse"(%[[REV1]]) <{axis = 2 : i64}
   // CHECK: "tosa.conv2d"(%arg0, %[[REV2]], %arg2)
   // CHECK-SAME: dilation = array<i64: 1, 1>, pad = array<i64: 2, 2, 5, 5>, stride = array<i64: 1, 1>
   %0 = "tosa.transpose_conv2d"(%arg0, %arg1, %arg2) {out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 1, 1>} : (tensor<2x16x14x3xf32>, tensor<5x3x6x3xf32>, tensor<5xf32>) -> tensor<2x18x19x5xf32>
@@ -15,9 +15,9 @@ func.func @transpose_conv2d(%arg0: tensor<2x16x14x3xf32>, %arg1: tensor<5x3x6x3x
 // CHECK-LABEL: @transpose_conv2d_quantized
 
 func.func @transpose_conv2d_quantized(%arg0: tensor<2x16x14x3xi8>, %arg1: tensor<5x3x6x3xi8>, %arg2: tensor<5xi32>) -> (tensor<2x18x19x5xi32>) {
-  // CHECK: %[[REV1:.+]] = "tosa.reverse"(%arg1) {axis = 1 : i64}
-  // CHECK: %[[REV2:.+]] = "tosa.reverse"(%[[REV1]]) {axis = 2 : i64}
-  // CHECK: "tosa.conv2d"(%arg0, %[[REV2]], %arg2) {dilation = array<i64: 1, 1>, pad = array<i64: 2, 2, 5, 5>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
+  // CHECK: %[[REV1:.+]] = "tosa.reverse"(%arg1) <{axis = 1 : i64}
+  // CHECK: %[[REV2:.+]] = "tosa.reverse"(%[[REV1]]) <{axis = 2 : i64}
+  // CHECK: "tosa.conv2d"(%arg0, %[[REV2]], %arg2) <{dilation = array<i64: 1, 1>, pad = array<i64: 2, 2, 5, 5>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
   %0 = "tosa.transpose_conv2d"(%arg0, %arg1, %arg2) {out_pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 1, 1>} : (tensor<2x16x14x3xi8>, tensor<5x3x6x3xi8>, tensor<5xi32>) -> tensor<2x18x19x5xi32>
   return %0 : tensor<2x18x19x5xi32>
 }
@@ -26,8 +26,8 @@ func.func @transpose_conv2d_quantized(%arg0: tensor<2x16x14x3xi8>, %arg1: tensor
 
 // CHECK-LABEL: @transpose_conv2d_quantized_padded
 func.func @transpose_conv2d_quantized_padded(%arg0: tensor<2x16x14x3xi8>, %arg1: tensor<5x3x6x3xi8>, %arg2: tensor<5xi32>) -> (tensor<2x21x26x5xi32>) {
-  // CHECK-DAG: %[[REV0:.+]] = "tosa.reverse"(%0) {axis = 2 : i64}
-  // CHECK-DAG: %[[REV1:.+]] = "tosa.reverse"(%arg1) {axis = 1 : i64}
+  // CHECK-DAG: %[[REV0:.+]] = "tosa.reverse"(%0) <{axis = 2 : i64}
+  // CHECK-DAG: %[[REV1:.+]] = "tosa.reverse"(%arg1) <{axis = 1 : i64}
   // CHECK: "tosa.conv2d"(%arg0, %1, %arg2) 
   // CHECK-SAME: dilation = array<i64: 1, 1>, pad = array<i64: 3, 4, 8, 9>,
   // CHECK-SAME: quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
@@ -44,27 +44,27 @@ func.func @transpose_conv2d_quantized_padded(%arg0: tensor<2x16x14x3xi8>, %arg1:
 // CHECK-LABEL: @transpose_conv2d_strided
 func.func @transpose_conv2d_strided(%arg0: tensor<2x17x15x3xf32>, %arg1: tensor<5x3x5x3xf32>, %arg2: tensor<5xf32>) -> tensor<2x?x?x5xf32> {
   // Manipulate the weight matrix to handle striding.
-  // CHECK-DAG: %[[PADV:.+]]  = "tosa.const"() {value = dense<{{\[\[}}0, 0], [0, 1], [0, 1], [0, 0]]> : tensor<4x2xi32>}
-  // CHECK-DAG: %[[TRANSV:.+]]  = "tosa.const"() {value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
+  // CHECK-DAG: %[[PADV:.+]]  = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [0, 1], [0, 1], [0, 0]]> : tensor<4x2xi32>}
+  // CHECK-DAG: %[[TRANSV:.+]]  = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
   // CHECK-DAG: %[[PADW:.+]]  = "tosa.pad"(%arg1, %[[PADV]])
-  // CHECK-DAG: %[[RESW1:.+]]  = "tosa.reshape"(%[[PADW]]) {new_shape = array<i64: 5, 2, 2, 2, 3, 3>}
+  // CHECK-DAG: %[[RESW1:.+]]  = "tosa.reshape"(%[[PADW]]) <{new_shape = array<i64: 5, 2, 2, 2, 3, 3>}
   // CHECK-DAG: %[[TRANS:.+]]  = "tosa.transpose"(%[[RESW1]], %[[TRANSV]])
-  // CHECK-DAG: %[[RESW2:.+]]  = "tosa.reshape"(%[[TRANS]]) {new_shape = array<i64: 30, 2, 2, 3>}
-  // CHECK-DAG: %[[REV1:.+]]  = "tosa.reverse"(%[[RESW2]]) {axis = 1 : i64}
-  // CHECK-DAG: %[[NEWWEIGHT:.+]] = "tosa.reverse"(%[[REV1]]) {axis = 2 : i64}
+  // CHECK-DAG: %[[RESW2:.+]]  = "tosa.reshape"(%[[TRANS]]) <{new_shape = array<i64: 30, 2, 2, 3>}
+  // CHECK-DAG: %[[REV1:.+]]  = "tosa.reverse"(%[[RESW2]]) <{axis = 1 : i64}
+  // CHECK-DAG: %[[NEWWEIGHT:.+]] = "tosa.reverse"(%[[REV1]]) <{axis = 2 : i64}
 
   // Pad out the input matrix to handle the transpose conv.
-  // CHECK-DAG: %[[PAD:.+]]  = "tosa.const"() {value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0]]> : tensor<4x2xi32>}
-  // CHECK-DAG: %[[TRANS2:.+]]  = "tosa.const"() {value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
+  // CHECK-DAG: %[[PAD:.+]]  = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0]]> : tensor<4x2xi32>}
+  // CHECK-DAG: %[[TRANS2:.+]]  = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
   // CHECK-DAG: %[[NEWINPUT:.+]] = "tosa.pad"(%arg0, %[[PAD]])
 
   // Manipulate the final shape.
-  // CHECK-DAG: %[[BIAS:.+]]  = "tosa.const"() {value = dense<0.000000e+00> : tensor<30xf32>}
-  // CHECK-DAG: %[[CONV:.+]] = "tosa.conv2d"(%[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]]) {dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
-  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = "tosa.reshape"(%[[CONV]]) {new_shape = array<i64: 2, 18, 16, 2, 3, 5>}
+  // CHECK-DAG: %[[BIAS:.+]]  = "tosa.const"() <{value = dense<0.000000e+00> : tensor<30xf32>}
+  // CHECK-DAG: %[[CONV:.+]] = "tosa.conv2d"(%[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]]) <{dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>}
+  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = "tosa.reshape"(%[[CONV]]) <{new_shape = array<i64: 2, 18, 16, 2, 3, 5>}
   // CHECK-DAG: %[[TRANS_OUT:.+]] = "tosa.transpose"(%[[RESHAPE_OUT_1]], %[[TRANS2]])
-  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = "tosa.reshape"(%[[TRANS_OUT]]) {new_shape = array<i64: 2, 36, 48, 5>}
-  // CHECK-DAG: %[[SLICE:.+]] = "tosa.slice"(%[[RESHAPE_OUT_2]]) {size = array<i64: 2, 35, 47, 5>, start = array<i64: 0, 0, 0, 0>}
+  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = "tosa.reshape"(%[[TRANS_OUT]]) <{new_shape = array<i64: 2, 36, 48, 5>}
+  // CHECK-DAG: %[[SLICE:.+]] = "tosa.slice"(%[[RESHAPE_OUT_2]]) <{size = array<i64: 2, 35, 47, 5>, start = array<i64: 0, 0, 0, 0>}
   // CHECK: %[[ADD:.+]] = "tosa.add"(%[[SLICE]], %arg2)
   %0 = "tosa.transpose_conv2d"(%arg0, %arg1, %arg2) {out_pad = array<i64: 0, 0, 0, 0>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>} : (tensor<2x17x15x3xf32>, tensor<5x3x5x3xf32>, tensor<5xf32>) -> tensor<2x35x47x5xf32>
   %1 = tensor.cast %0 : tensor<2x35x47x5xf32> to tensor<2x?x?x5xf32>
@@ -76,29 +76,29 @@ func.func @transpose_conv2d_strided(%arg0: tensor<2x17x15x3xf32>, %arg1: tensor<
 // CHECK-LABEL: @transpose_conv2d_strided_quantized
 func.func @transpose_conv2d_strided_quantized(%arg0: tensor<2x17x15x3xi8>, %arg1: tensor<5x3x5x3xi8>, %arg2: tensor<5xi32>) -> (tensor<2x35x47x5xi32>) {
   // Manipulate the weight matrix to handle striding.
-  // CHECK-DAG: %[[PADV:.+]]  = "tosa.const"() {value = dense<{{\[\[}}0, 0], [0, 1], [0, 1], [0, 0]]> : tensor<4x2xi32>}
-  // CHECK-DAG: %[[TRANSV:.+]]  = "tosa.const"() {value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
-  // CHECK-DAG: %[[PADW:.+]]  = "tosa.pad"(%arg1, %[[PADV]]) {quantization_info = #tosa.pad_quant<input_zp = 42>}
-  // CHECK-DAG: %[[RESW1:.+]]  = "tosa.reshape"(%[[PADW]]) {new_shape = array<i64: 5, 2, 2, 2, 3, 3>}
+  // CHECK-DAG: %[[PADV:.+]]  = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [0, 1], [0, 1], [0, 0]]> : tensor<4x2xi32>}
+  // CHECK-DAG: %[[TRANSV:.+]]  = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
+  // CHECK-DAG: %[[PADW:.+]]  = "tosa.pad"(%arg1, %[[PADV]]) <{quantization_info = #tosa.pad_quant<input_zp = 42>}
+  // CHECK-DAG: %[[RESW1:.+]]  = "tosa.reshape"(%[[PADW]]) <{new_shape = array<i64: 5, 2, 2, 2, 3, 3>}
   // CHECK-DAG: %[[TRANS:.+]]  = "tosa.transpose"(%[[RESW1]], %[[TRANSV]])
-  // CHECK-DAG: %[[RESW2:.+]]  = "tosa.reshape"(%[[TRANS]]) {new_shape = array<i64: 30, 2, 2, 3>}
-  // CHECK-DAG: %[[REV1:.+]]  = "tosa.reverse"(%[[RESW2]]) {axis = 1 : i64}
-  // CHECK-DAG: %[[NEWWEIGHT:.+]] = "tosa.reverse"(%[[REV1]]) {axis = 2 : i64}
+  // CHECK-DAG: %[[RESW2:.+]]  = "tosa.reshape"(%[[TRANS]]) <{new_shape = array<i64: 30, 2, 2, 3>}
+  // CHECK-DAG: %[[REV1:.+]]  = "tosa.reverse"(%[[RESW2]]) <{axis = 1 : i64}
+  // CHECK-DAG: %[[NEWWEIGHT:.+]] = "tosa.reverse"(%[[REV1]]) <{axis = 2 : i64}
 
   // Pad out the input matrix to handle the transpose conv.
-  // CHECK-DAG: %[[PAD:.+]]  = "tosa.const"() {value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0]]> : tensor<4x2xi32>}
-  // CHECK-DAG: %[[TRANS2:.+]]  = "tosa.const"() {value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
-  // CHECK-DAG: %[[NEWINPUT:.+]] = "tosa.pad"(%arg0, %[[PAD]]) {quantization_info = #tosa.pad_quant<input_zp = -22>}
+  // CHECK-DAG: %[[PAD:.+]]  = "tosa.const"() <{value = dense<{{\[\[}}0, 0], [1, 1], [1, 1], [0, 0]]> : tensor<4x2xi32>}
+  // CHECK-DAG: %[[TRANS2:.+]]  = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
+  // CHECK-DAG: %[[NEWINPUT:.+]] = "tosa.pad"(%arg0, %[[PAD]]) <{quantization_info = #tosa.pad_quant<input_zp = -22>}
 
   // Manipulate the final shape.
-  // CHECK-DAG: %[[BIAS:.+]]  = "tosa.const"() {value = dense<0> : tensor<30xi32>}
-  // CHECK-DAG: %[[CONV:.+]] = "tosa.conv2d"(%[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]]) {dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
-  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = "tosa.reshape"(%[[CONV]]) {new_shape = array<i64: 2, 18, 16, 2, 3, 5>}
+  // CHECK-DAG: %[[BIAS:.+]]  = "tosa.const"() <{value = dense<0> : tensor<30xi32>}
+  // CHECK-DAG: %[[CONV:.+]] = "tosa.conv2d"(%[[NEWINPUT]], %[[NEWWEIGHT]], %[[BIAS]]) <{dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, stride = array<i64: 1, 1>}
+  // CHECK-DAG: %[[RESHAPE_OUT_1:.+]] = "tosa.reshape"(%[[CONV]]) <{new_shape = array<i64: 2, 18, 16, 2, 3, 5>}
   // CHECK-DAG: %[[TRANS_OUT:.+]] = "tosa.transpose"(%[[RESHAPE_OUT_1]], %[[TRANS2]])
-  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = "tosa.reshape"(%[[TRANS_OUT]]) {new_shape = array<i64: 2, 36, 48, 5>}
-  // CHECK-DAG: %[[SLICE:.+]] = "tosa.slice"(%[[RESHAPE_OUT_2]]) {size = array<i64: 2, 35, 47, 5>, start = array<i64: 0, 0, 0, 0>}
+  // CHECK-DAG: %[[RESHAPE_OUT_2:.+]] = "tosa.reshape"(%[[TRANS_OUT]]) <{new_shape = array<i64: 2, 36, 48, 5>}
+  // CHECK-DAG: %[[SLICE:.+]] = "tosa.slice"(%[[RESHAPE_OUT_2]]) <{size = array<i64: 2, 35, 47, 5>, start = array<i64: 0, 0, 0, 0>}
   // CHECK: %[[ADD:.+]] = "tosa.add"(%[[SLICE]], %arg2)
-  %0 = "tosa.transpose_conv2d"(%arg0, %arg1, %arg2) {out_pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>} : (tensor<2x17x15x3xi8>, tensor<5x3x5x3xi8>, tensor<5xi32>) -> tensor<2x35x47x5xi32>
+  %0 = "tosa.transpose_conv2d"(%arg0, %arg1, %arg2) <{out_pad = array<i64: 0, 0, 0, 0>, quantization_info = #tosa.conv_quant<input_zp = -22, weight_zp = 42>, out_shape = array<i64: -1, -1, -1, -1>, stride = array<i64: 2, 3>}> : (tensor<2x17x15x3xi8>, tensor<5x3x5x3xi8>, tensor<5xi32>) -> tensor<2x35x47x5xi32>
   return %0 : tensor<2x35x47x5xi32>
 }
 
@@ -108,24 +108,24 @@ func.func @transpose_conv2d_strided_quantized(%arg0: tensor<2x17x15x3xi8>, %arg1
 func.func @transpose_conv2d_strided_overpad(%arg0 : tensor<1x16x1x1xi8>, %arg1 : tensor<1x2x1x1xi8>, %arg2 : tensor<1xi32>) -> (tensor<1x19x2x1xi32>) {
   // CHECK: %[[WEIGHT_PAD:.+]] = "tosa.const"() 
   // CHECK-SAME{literal}: value = dense<[[0, 0], [0, 0], [0, 1], [0, 0]]> : tensor<4x2xi32>
-  // CHECK: %[[WEIGHT_PERMS:.+]] = "tosa.const"() {value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>} : () -> tensor<6xi32>
+  // CHECK: %[[WEIGHT_PERMS:.+]] = "tosa.const"() <{value = dense<[2, 4, 0, 1, 3, 5]> : tensor<6xi32>}
   // CHECK: %[[INPUT_PAD:.+]] = "tosa.const"() 
   // CHECK-SAME{literal}: value = dense<[[0, 0], [1, 1], [0, 0], [0, 0]]> : tensor<4x2xi32>}
-  // CHECK: %[[ZERO:.+]] = "tosa.const"() {value = dense<0> : tensor<2xi32>} : () -> tensor<2xi32>
-  // CHECK: %[[RESULT_PERMS:.+]] = "tosa.const"() {value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
+  // CHECK: %[[ZERO:.+]] = "tosa.const"() <{value = dense<0> : tensor<2xi32>}
+  // CHECK: %[[RESULT_PERMS:.+]] = "tosa.const"() <{value = dense<[0, 1, 3, 2, 4, 5]> : tensor<6xi32>}
   // CHECK: %[[RESULT_PAD:.+]] = "tosa.const"() 
   // CHECK-SAME{literal}: value = dense<[[0, 0], [2, 0], [0, 0], [0, 0]]> : tensor<4x2xi32>}
-  // CHECK: %[[PAD_WEIGHT:.+]] = "tosa.pad"(%arg1, %[[WEIGHT_PAD]]) {quantization_info = #tosa.pad_quant<input_zp = 93>}
-  // CHECK: %[[RESHAPE_WEIGHT_0:.+]] = "tosa.reshape"(%[[PAD_WEIGHT]]) {new_shape = array<i64: 1, 2, 1, 1, 2, 1>}
+  // CHECK: %[[PAD_WEIGHT:.+]] = "tosa.pad"(%arg1, %[[WEIGHT_PAD]]) <{quantization_info = #tosa.pad_quant<input_zp = 93>}
+  // CHECK: %[[RESHAPE_WEIGHT_0:.+]] = "tosa.reshape"(%[[PAD_WEIGHT]]) <{new_shape = array<i64: 1, 2, 1, 1, 2, 1>}
   // CHECK: %[[TRANSPOSE_WEIGHT:.+]] = "tosa.transpose"(%[[RESHAPE_WEIGHT_0]], %[[WEIGHT_PERMS]])
-  // CHECK: %[[RESHAPE_WEIGHT_1:.+]] = "tosa.reshape"(%[[TRANSPOSE_WEIGHT]]) {new_shape = array<i64: 2, 2, 1, 1>}
-  // CHECK: %[[REVERSE:.+]] = "tosa.reverse"(%[[RESHAPE_WEIGHT_1]]) {axis = 1 : i64}
-  // CHECK: %[[PAD_INPUT:.+]] = "tosa.pad"(%arg0, %[[INPUT_PAD]]) {quantization_info = #tosa.pad_quant<input_zp = -103>}
+  // CHECK: %[[RESHAPE_WEIGHT_1:.+]] = "tosa.reshape"(%[[TRANSPOSE_WEIGHT]]) <{new_shape = array<i64: 2, 2, 1, 1>}
+  // CHECK: %[[REVERSE:.+]] = "tosa.reverse"(%[[RESHAPE_WEIGHT_1]]) <{axis = 1 : i64}
+  // CHECK: %[[PAD_INPUT:.+]] = "tosa.pad"(%arg0, %[[INPUT_PAD]]) <{quantization_info = #tosa.pad_quant<input_zp = -103>}
   // CHECK: %[[CONV:.+]] = "tosa.conv2d"(%[[PAD_INPUT]], %[[REVERSE]], %[[ZERO]]) 
   // CHECK-SAME{literal}: dilation = [1, 1], pad = [0, 0, 0, 0], quantization_info = #tosa.conv_quant<input_zp = -103, weight_zp = 93>, stride = [1, 1]}
-  // CHECK: %[[RESHAPE_RESULT_0:.+]] = "tosa.reshape"(%[[CONV]]) {new_shape = array<i64: 1, 17, 1, 1, 2, 1>}
+  // CHECK: %[[RESHAPE_RESULT_0:.+]] = "tosa.reshape"(%[[CONV]]) <{new_shape = array<i64: 1, 17, 1, 1, 2, 1>}
   // CHECK: %[[TRANSPOSE_RESULT:.+]] = "tosa.transpose"(%[[RESHAPE_RESULT_0]], %[[RESULT_PERMS]])
-  // CHECK: %[[RESHAPE_RESULT_1:.+]] = "tosa.reshape"(%[[TRANSPOSE_RESULT]]) {new_shape = array<i64: 1, 17, 2, 1>}
+  // CHECK: %[[RESHAPE_RESULT_1:.+]] = "tosa.reshape"(%[[TRANSPOSE_RESULT]]) <{new_shape = array<i64: 1, 17, 2, 1>}
   // CHECK: %[[PAD_RESULT:.+]] = "tosa.pad"(%[[RESHAPE_RESULT_1]], %[[RESULT_PAD]])
   // CHECK: %[[ADD:.+]] = "tosa.add"(%[[PAD_RESULT]], %arg2)
   %2 =  "tosa.transpose_conv2d"(%arg0, %arg1, %arg2)  {
diff --git a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
index 505350786d08d..ea2bd42106a24 100644
--- a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
@@ -51,7 +51,7 @@ func.func @test_unary_f32(%arg0 : tensor<4xf32>) -> () {
   // CHECK: "tosa.reciprocal"(%arg0) : (tensor<4xf32>) -> tensor<4xf32>
   %7 = "tosa.reciprocal"(%arg0) : (tensor<4xf32>) -> tensor<*xf32>
 
-  // CHECK: "tosa.reverse"(%arg0) {axis = 0 : i64} : (tensor<4xf32>) -> tensor<4xf32>
+  // CHECK: "tosa.reverse"(%arg0) <{axis = 0 : i64}> : (tensor<4xf32>) -> tensor<4xf32>
   %8 = "tosa.reverse"(%arg0) { axis = 0 : i64 } : (tensor<4xf32>) -> tensor<?xf32>
 
   // CHECK: "tosa.rsqrt"(%arg0) : (tensor<4xf32>) -> tensor<4xf32>
@@ -87,7 +87,7 @@ func.func @test_unary_i32(%arg0 : tensor<4xi32>) -> () {
   // CHECK: "tosa.negate"(%arg0) : (tensor<4xi32>) -> tensor<4xi32>
   %4 = "tosa.negate"(%arg0) : (tensor<4xi32>) -> tensor<*xi32>
 
-  // CHECK: "tosa.reverse"(%arg0) {axis = 0 : i64} : (tensor<4xi32>) -> tensor<4xi32>
+  // CHECK: "tosa.reverse"(%arg0) <{axis = 0 : i64}> : (tensor<4xi32>) -> tensor<4xi32>
   %5 = "tosa.reverse"(%arg0) { axis = 0 : i64 } : (tensor<4xi32>) -> tensor<?xi32>
 
   // CHECK: "tosa.rescale"(%arg0) {{.+}} : (tensor<4xi32>) -> tensor<4xi16>
@@ -111,7 +111,7 @@ func.func @test_binary_scalar_f32(%arg0 : tensor<4xf32>, %arg1 : tensor<f32>) ->
   // CHECK: "tosa.minimum"(%arg0, %arg1) : (tensor<4xf32>, tensor<f32>) -> tensor<4xf32>
   %2 = "tosa.minimum"(%arg0, %arg1) : (tensor<4xf32>, tensor<f32>) -> tensor<*xf32>
 
-  // CHECK: "tosa.mul"(%arg0, %arg1) {shift = 0 : i32} : (tensor<4xf32>, tensor<f32>) -> tensor<4xf32>
+  // CHECK: "tosa.mul"(%arg0, %arg1) <{shift = 0 : i32}> : (tensor<4xf32>, tensor<f32>) -> tensor<4xf32>
   %3 = "tosa.mul"(%arg0, %arg1) { shift = 0 : i32 }: (tensor<4xf32>, tensor<f32>) -> tensor<*xf32>
 
   // CHECK: "tosa.pow"(%arg0, %arg1) : (tensor<4xf32>, tensor<f32>) -> tensor<4xf32>
@@ -145,7 +145,7 @@ func.func @test_binary_broadcast_f32(%arg0 : tensor<4xf32>, %arg1 : tensor<1xf32
   // CHECK: "tosa.minimum"(%arg0, %arg1) : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
   %2 = "tosa.minimum"(%arg0, %arg1) : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
-  // CHECK: "tosa.mul"(%arg0, %arg1) {shift = 0 : i32} : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
+  // CHECK: "tosa.mul"(%arg0, %arg1) <{shift = 0 : i32}> : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
   %3 = "tosa.mul"(%arg0, %arg1) { shift = 0 : i32 } : (tensor<4xf32>, tensor<1xf32>) -> tensor<*xf32>
 
   // CHECK: "tosa.pow"(%arg0, %arg1) : (tensor<4xf32>, tensor<1xf32>) -> tensor<4xf32>
@@ -173,14 +173,14 @@ func.func @test_binary_i32(%arg0 : tensor<4xi32>, %arg1 : tensor<i32>) -> () {
   // CHECK: "tosa.add"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
   %0 = "tosa.add"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
 
-  // CHECK: "tosa.bitwise_and"(%arg0, %arg1) {shift = 0 : i32} : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
-  %1 = "tosa.bitwise_and"(%arg0, %arg1) { shift = 0 : i32 }: (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
+  // CHECK: "tosa.bitwise_and"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
+  %1 = "tosa.bitwise_and"(%arg0, %arg1): (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
 
-  // CHECK: "tosa.bitwise_or"(%arg0, %arg1) {shift = 0 : i32} : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
-  %2 = "tosa.bitwise_or"(%arg0, %arg1) { shift = 0 : i32 }: (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
+  // CHECK: "tosa.bitwise_or"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
+  %2 = "tosa.bitwise_or"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
 
-  // CHECK: "tosa.bitwise_xor"(%arg0, %arg1) {shift = 0 : i32} : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
-  %3 = "tosa.bitwise_xor"(%arg0, %arg1) { shift = 0 : i32 }: (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
+  // CHECK: "tosa.bitwise_xor"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
+  %3 = "tosa.bitwise_xor"(%arg0, %arg1): (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
 
   // CHECK: "tosa.equal"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi1>
   %4 = "tosa.equal"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<*xi1>
@@ -203,7 +203,7 @@ func.func @test_binary_i32(%arg0 : tensor<4xi32>, %arg1 : tensor<i32>) -> () {
   // CHECK: "tosa.minimum"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
   %10 = "tosa.minimum"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
 
-  // CHECK: "tosa.mul"(%arg0, %arg1) {shift = 0 : i32} : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
+  // CHECK: "tosa.mul"(%arg0, %arg1) <{shift = 0 : i32}> : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
   %11 = "tosa.mul"(%arg0, %arg1) { shift = 0 : i32 }: (tensor<4xi32>, tensor<i32>) -> tensor<*xi32>
 
   // CHECK: "tosa.pow"(%arg0, %arg1) : (tensor<4xi32>, tensor<i32>) -> tensor<4xi32>
@@ -245,10 +245,10 @@ func.func @test_select_i32(%arg0 : tensor<4xi1>, %arg1 : tensor<i32>, %arg2 : te
 
 // CHECK-LABEL: @test_static_argmax
 func.func @test_static_argmax(%arg0 : tensor<2x3xi32>) -> () {
-  // CHECK: "tosa.argmax"(%arg0) {axis = 0 : i64} : (tensor<2x3xi32>) -> tensor<3xi32>
+  // CHECK: "tosa.argmax"(%arg0) <{axis = 0 : i64}> : (tensor<2x3xi32>) -> tensor<3xi32>
   %0 = "tosa.argmax"(%arg0) {axis = 0 : i64} : (tensor<2x3xi32>) -> tensor<?xi32>
 
-  // CHECK: "tosa.argmax"(%arg0) {axis = 1 : i64} : (tensor<2x3xi32>) -> tensor<2xi32>
+  // CHECK: "tosa.argmax"(%arg0) <{axis = 1 : i64}> : (tensor<2x3xi32>) -> tensor<2xi32>
   %1 = "tosa.argmax"(%arg0) {axis = 1 : i64} : (tensor<2x3xi32>) -> tensor<?xi32>
   return
 }
@@ -257,10 +257,10 @@ func.func @test_static_argmax(%arg0 : tensor<2x3xi32>) -> () {
 
 // CHECK-LABEL: @test_dynamic_argmax
 func.func @test_dynamic_argmax(%arg0 : tensor<2x?xi32>) -> () {
-  // CHECK: "tosa.argmax"(%arg0) {axis = 0 : i64} : (tensor<2x?xi32>) -> tensor<?xi32>
+  // CHECK: "tosa.argmax"(%arg0) <{axis = 0 : i64}> : (tensor<2x?xi32>) -> tensor<?xi32>
   %0 = "tosa.argmax"(%arg0) {axis = 0 : i64} : (tensor<2x?xi32>) -> tensor<?xi32>
 
-  // CHECK: "tosa.argmax"(%arg0) {axis = 1 : i64} : (tensor<2x?xi32>) -> tensor<2xi32>
+  // CHECK: "tosa.argmax"(%arg0) <{axis = 1 : i64}> : (tensor<2x?xi32>) -> tensor<2xi32>
   %1 = "tosa.argmax"(%arg0) {axis = 1 : i64} : (tensor<2x?xi32>) -> tensor<?xi32>
   return
 }
@@ -372,13 +372,13 @@ func.func @test_table_dynamic(%arg0 : tensor<4x?xi16>, %arg1 : tensor<513xi16>)
 
 // CHECK-LABEL: @test_static_reshape
 func.func @test_static_reshape(%arg0 : tensor<4x4xi32>) -> () {
-  // CHECK: "tosa.reshape"(%arg0) {new_shape = array<i64: 16>} : (tensor<4x4xi32>) -> tensor<16xi32>
+  // CHECK: "tosa.reshape"(%arg0) <{new_shape = array<i64: 16>}> : (tensor<4x4xi32>) -> tensor<16xi32>
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 16>} : (tensor<4x4xi32>)  -> tensor<?xi32>
 
-  // CHECK: "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<4x4xi32>) -> tensor<16xi32>
+  // CHECK: "tosa.reshape"(%arg0) <{new_shape = array<i64: -1>}> : (tensor<4x4xi32>) -> tensor<16xi32>
   %1 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<4x4xi32>)  -> tensor<?xi32>
 
-  // CHECK: "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<4x4xi32>) -> tensor<2x8xi32>
+  // CHECK: "tosa.reshape"(%arg0) <{new_shape = array<i64: 2, -1>}> : (tensor<4x4xi32>) -> tensor<2x8xi32>
   %2 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<4x4xi32>)  -> tensor<?x?xi32>
 
   return
@@ -387,13 +387,13 @@ func.func @test_static_reshape(%arg0 : tensor<4x4xi32>) -> () {
 
 // CHECK-LABEL: @test_dynamic_reshape
 func.func @test_dynamic_reshape(%arg0 : tensor<4x?xi32>) -> () {
-  // CHECK: %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 16>} : (tensor<4x?xi32>) -> tensor<16xi32>
+  // CHECK: %0 = "tosa.reshape"(%arg0) <{new_shape = array<i64: 16>}> : (tensor<4x?xi32>) -> tensor<16xi32>
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 16>} : (tensor<4x?xi32>)  -> tensor<?xi32>
 
-  // CHECK: %1 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<4x?xi32>) -> tensor<?xi32>
+  // CHECK: %1 = "tosa.reshape"(%arg0) <{new_shape = array<i64: -1>}> : (tensor<4x?xi32>) -> tensor<?xi32>
   %1 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<4x?xi32>)  -> tensor<?xi32>
 
-  // CHECK: %2 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<4x?xi32>) -> tensor<2x?xi32>
+  // CHECK: %2 = "tosa.reshape"(%arg0) <{new_shape = array<i64: 2, -1>}> : (tensor<4x?xi32>) -> tensor<2x?xi32>
   %2 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<4x?xi32>)  -> tensor<?x?xi32>
 
   return
@@ -403,19 +403,19 @@ func.func @test_dynamic_reshape(%arg0 : tensor<4x?xi32>) -> () {
 
 // CHECK: @test_reduce_binary
 func.func @test_reduce_binary(%arg0 : tensor<2x3x?x?xi1>) -> () {
-  // CHECK: "tosa.reduce_all"(%arg0) {axis = 0 : i64} : (tensor<2x3x?x?xi1>) -> tensor<1x3x?x?xi1>
+  // CHECK: "tosa.reduce_all"(%arg0) <{axis = 0 : i64}> : (tensor<2x3x?x?xi1>) -> tensor<1x3x?x?xi1>
   %0 = "tosa.reduce_all"(%arg0) {axis = 0 : i64} : (tensor<2x3x?x?xi1>) -> tensor<?x?x?x?xi1>
 
-  // CHECK: "tosa.reduce_all"(%arg0) {axis = 1 : i64} : (tensor<2x3x?x?xi1>) -> tensor<2x1x?x?xi1>
+  // CHECK: "tosa.reduce_all"(%arg0) <{axis = 1 : i64}> : (tensor<2x3x?x?xi1>) -> tensor<2x1x?x?xi1>
   %1 = "tosa.reduce_all"(%arg0) {axis = 1 : i64} : (tensor<2x3x?x?xi1>) -> tensor<?x?x?x?xi1>
 
-  // CHECK: "tosa.reduce_all"(%arg0) {axis = 2 : i64} : (tensor<2x3x?x?xi1>) -> tensor<2x3x1x?xi1>
+  // CHECK: "tosa.reduce_all"(%arg0) <{axis = 2 : i64}> : (tensor<2x3x?x?xi1>) -> tensor<2x3x1x?xi1>
   %2 = "tosa.reduce_all"(%arg0) {axis = 2 : i64} : (tensor<2x3x?x?xi1>) -> tensor<?x?x?x?xi1>
 
-  // CHECK: "tosa.reduce_all"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xi1>) -> tensor<2x3x?x1xi1>
+  // CHECK: "tosa.reduce_all"(%arg0) <{axis = 3 : i64}> : (tensor<2x3x?x?xi1>) -> tensor<2x3x?x1xi1>
   %3 = "tosa.reduce_all"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xi1>) -> tensor<?x?x?x?xi1>
 
-  // CHECK: "tosa.reduce_any"(%arg0) {axis = 0 : i64} : (tensor<2x3x?x?xi1>) -> tensor<1x3x?x?xi1>
+  // CHECK: "tosa.reduce_any"(%arg0) <{axis = 0 : i64}> : (tensor<2x3x?x?xi1>) -> tensor<1x3x?x?xi1>
   %4 = "tosa.reduce_any"(%arg0) {axis = 0 : i64} : (tensor<2x3x?x?xi1>) -> tensor<?x?x?x?xi1>
 
   return
@@ -425,25 +425,25 @@ func.func @test_reduce_binary(%arg0 : tensor<2x3x?x?xi1>) -> () {
 
 // CHECK: @test_reduce_float
 func.func @test_reduce_float(%arg0 : tensor<2x3x?x?xf32>) -> () {
-  // CHECK: "tosa.reduce_sum"(%arg0) {axis = 0 : i64} : (tensor<2x3x?x?xf32>) -> tensor<1x3x?x?xf32>
+  // CHECK: "tosa.reduce_sum"(%arg0) <{axis = 0 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<1x3x?x?xf32>
   %0 = "tosa.reduce_sum"(%arg0) {axis = 0 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
-  // CHECK: "tosa.reduce_sum"(%arg0) {axis = 1 : i64} : (tensor<2x3x?x?xf32>) -> tensor<2x1x?x?xf32>
+  // CHECK: "tosa.reduce_sum"(%arg0) <{axis = 1 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<2x1x?x?xf32>
   %1 = "tosa.reduce_sum"(%arg0) {axis = 1 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
-  // CHECK: "tosa.reduce_sum"(%arg0) {axis = 2 : i64} : (tensor<2x3x?x?xf32>) -> tensor<2x3x1x?xf32>
+  // CHECK: "tosa.reduce_sum"(%arg0) <{axis = 2 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<2x3x1x?xf32>
   %2 = "tosa.reduce_sum"(%arg0) {axis = 2 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
-  // CHECK: "tosa.reduce_sum"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
+  // CHECK: "tosa.reduce_sum"(%arg0) <{axis = 3 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
   %3 = "tosa.reduce_sum"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
-  // CHECK: "tosa.reduce_max"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
+  // CHECK: "tosa.reduce_max"(%arg0) <{axis = 3 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
   %4 = "tosa.reduce_max"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
-  // CHECK: "tosa.reduce_min"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
+  // CHECK: "tosa.reduce_min"(%arg0) <{axis = 3 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
   %5 = "tosa.reduce_min"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
-  // CHECK: "tosa.reduce_prod"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
+  // CHECK: "tosa.reduce_prod"(%arg0) <{axis = 3 : i64}> : (tensor<2x3x?x?xf32>) -> tensor<2x3x?x1xf32>
   %6 = "tosa.reduce_prod"(%arg0) {axis = 3 : i64} : (tensor<2x3x?x?xf32>) -> tensor<?x?x?x?xf32>
 
   return
@@ -453,7 +453,7 @@ func.func @test_reduce_float(%arg0 : tensor<2x3x?x?xf32>) -> () {
 
 // CHECK-LABEL: @test_concat
 func.func @test_concat(%arg0 : tensor<1x2xf32>, %arg1 : tensor<2x2xf32>) -> () {
-  // CHECK: "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<1x2xf32>, tensor<2x2xf32>) -> tensor<3x2xf32>
+  // CHECK: "tosa.concat"(%arg0, %arg1) <{axis = 0 : i64}> : (tensor<1x2xf32>, tensor<2x2xf32>) -> tensor<3x2xf32>
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<1x2xf32>, tensor<2x2xf32>) -> tensor<?x?xf32>
 
   return
@@ -463,7 +463,7 @@ func.func @test_concat(%arg0 : tensor<1x2xf32>, %arg1 : tensor<2x2xf32>) -> () {
 
 // CHECK-LABEL: @test_concat_dynamic
 func.func @test_concat_dynamic(%arg0 : tensor<1x2xf32>, %arg1 : tensor<2x?xf32>) -> () {
-  // CHECK: "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<1x2xf32>, tensor<2x?xf32>) -> tensor<3x2xf32>
+  // CHECK: "tosa.concat"(%arg0, %arg1) <{axis = 0 : i64}> : (tensor<1x2xf32>, tensor<2x?xf32>) -> tensor<3x2xf32>
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<1x2xf32>, tensor<2x?xf32>) -> tensor<?x?xf32>
 
   return
@@ -473,7 +473,7 @@ func.func @test_concat_dynamic(%arg0 : tensor<1x2xf32>, %arg1 : tensor<2x?xf32>)
 
 // CHECK-LABEL: @test_concat_dynamic_axis
 func.func @test_concat_dynamic_axis(%arg0 : tensor<?x2xf32>, %arg1 : tensor<2x2xf32>) -> () {
-  // CHECK: "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<?x2xf32>, tensor<2x2xf32>) -> tensor<?x2xf32>
+  // CHECK: "tosa.concat"(%arg0, %arg1) <{axis = 0 : i64}> : (tensor<?x2xf32>, tensor<2x2xf32>) -> tensor<?x2xf32>
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 0 : i64} : (tensor<?x2xf32>, tensor<2x2xf32>) -> tensor<?x?xf32>
 
   return
@@ -483,7 +483,7 @@ func.func @test_concat_dynamic_axis(%arg0 : tensor<?x2xf32>, %arg1 : tensor<2x2x
 
 // CHECK-LABEL: @test_concat_axis_1
 func.func @test_concat_axis_1(%arg0 : tensor<2x1xf32>, %arg1 : tensor<2x2xf32>) -> () {
-  // CHECK: "tosa.concat"(%arg0, %arg1) {axis = 1 : i64} : (tensor<2x1xf32>, tensor<2x2xf32>) -> tensor<2x3xf32>
+  // CHECK: "tosa.concat"(%arg0, %arg1) <{axis = 1 : i64}> : (tensor<2x1xf32>, tensor<2x2xf32>) -> tensor<2x3xf32>
   %0 = "tosa.concat"(%arg0, %arg1) {axis = 1 : i64} : (tensor<2x1xf32>, tensor<2x2xf32>) -> tensor<?x?xf32>
 
   return
@@ -522,7 +522,7 @@ func.func @test_padding_simple(%arg0 : tensor<1x2xf32>) -> () {
 
 // CHECK-LABEL: @test_slice
 func.func @test_slice(%arg0 : tensor<?xi32>) -> () {
-  // CHECK: "tosa.slice"(%arg0) {size = array<i64: 2>, start = array<i64: 1>} : (tensor<?xi32>) -> tensor<2xi32>
+  // CHECK: "tosa.slice"(%arg0) <{size = array<i64: 2>, start = array<i64: 1>}> : (tensor<?xi32>) -> tensor<2xi32>
   %0 = "tosa.slice"(%arg0) { size = array<i64: 2>, start = array<i64: 1> } : (tensor<?xi32>) -> tensor<?xi32>
   return
 }
@@ -531,7 +531,7 @@ func.func @test_slice(%arg0 : tensor<?xi32>) -> () {
 
 // CHECK-LABEL: @test_slice_dynamic
 func.func @test_slice_dynamic(%arg0 : tensor<10x?x2xf32>) -> () {
-  // CHECK: "tosa.slice"(%arg0) {size = array<i64: 7, -1, 1>, start = array<i64: 1, 0, 0>} : (tensor<10x?x2xf32>) -> tensor<7x?x1xf32>
+  // CHECK: "tosa.slice"(%arg0) <{size = array<i64: 7, -1, 1>, start = array<i64: 1, 0, 0>}> : (tensor<10x?x2xf32>) -> tensor<7x?x1xf32>
   %0 = "tosa.slice"(%arg0) {size = array<i64: 7, -1, 1>, start = array<i64: 1, 0, 0>} : (tensor<10x?x2xf32>) -> tensor<?x?x?xf32>
   return
 }
@@ -540,8 +540,8 @@ func.func @test_slice_dynamic(%arg0 : tensor<10x?x2xf32>) -> () {
 
 // CHECK-LABEL: @test_tile
 func.func @test_tile(%arg0 : tensor<2x3x?xi32>) -> () {
-  // CHECK: "tosa.tile"(%arg0) {multiples = array<i64: 2, 1, 5>} : (tensor<2x3x?xi32>) -> tensor<4x3x?xi32>
-  %0 = "tosa.tile"(%arg0) {multiples = array<i64: 2, 1, 5>} : (tensor<2x3x?xi32>)  -> (tensor<?x?x?xi32>)
+  // CHECK: "tosa.tile"(%arg0) <{multiples = array<i64: 2, 1, 5>}> : (tensor<2x3x?xi32>) -> tensor<4x3x?xi32>
+  %0 = "tosa.tile"(%arg0) <{multiples = array<i64: 2, 1, 5>}> : (tensor<2x3x?xi32>)  -> (tensor<?x?x?xi32>)
   return
 }
 
@@ -1224,3 +1224,13 @@ func.func @test_dynamic_batch_fft2d(%arg0: tensor<?x4x8xf32>, %arg1: tensor<?x4x
   %output_real, %output_imag = "tosa.fft2d"(%arg0, %arg1) {inverse = false} : (tensor<?x4x8xf32>, tensor<?x4x8xf32>) -> (tensor<?x4x8xf32>, tensor<?x4x8xf32>)
   return %output_real, %output_imag : tensor<?x4x8xf32>, tensor<?x4x8xf32>
 }
+
+// -----
+
+// CHECK-LABEL: @test_unranked_equal
+func.func @test_unranked_equal(%arg0 : tensor<*xf32>, %arg1 : tensor<f32>) -> () {
+  // CHECK: "tosa.equal"(%arg0, %arg1) : (tensor<*xf32>, tensor<f32>) -> tensor<*xi1>
+  %0 = "tosa.equal"(%arg0, %arg1) : (tensor<*xf32>, tensor<f32>) -> tensor<*xi1>
+
+  return
+}
diff --git a/mlir/test/Dialect/Vector/vector-break-down-bitcast.mlir b/mlir/test/Dialect/Vector/vector-break-down-bitcast.mlir
new file mode 100644
index 0000000000000..fbb2f7605e649
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-break-down-bitcast.mlir
@@ -0,0 +1,41 @@
+// RUN: mlir-opt -split-input-file -test-vector-break-down-bitcast %s | FileCheck %s
+
+// CHECK-LABEL: func.func @bitcast_f16_to_f32
+//  CHECK-SAME: (%[[INPUT:.+]]: vector<8xf16>)
+func.func @bitcast_f16_to_f32(%input: vector<8xf16>) -> vector<4xf32> {
+  %0 = vector.bitcast %input : vector<8xf16> to vector<4xf32>
+  return %0: vector<4xf32>
+}
+
+// CHECK: %[[INIT:.+]] = arith.constant dense<0.000000e+00> : vector<4xf32>
+// CHECK: %[[EXTRACT0:.+]] = vector.extract_strided_slice %[[INPUT]] {offsets = [0], sizes = [4], strides = [1]} : vector<8xf16> to vector<4xf16>
+// CHECK: %[[CAST0:.+]] = vector.bitcast %[[EXTRACT0]] : vector<4xf16> to vector<2xf32>
+// CHECK: %[[INSERT0:.+]] = vector.insert_strided_slice %[[CAST0]], %[[INIT]] {offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
+// CHECK: %[[EXTRACT1:.+]] = vector.extract_strided_slice %[[INPUT]] {offsets = [4], sizes = [4], strides = [1]} : vector<8xf16> to vector<4xf16>
+// CHECK: %[[CAST1:.+]] = vector.bitcast %[[EXTRACT1]] : vector<4xf16> to vector<2xf32>
+// CHECK: %[[INSERT1:.+]] = vector.insert_strided_slice %[[CAST1]], %[[INSERT0]] {offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
+// CHECK: return %[[INSERT1]]
+
+// -----
+
+// CHECK-LABEL: func.func @bitcast_i8_to_i32
+//  CHECK-SAME: (%[[INPUT:.+]]: vector<16xi8>)
+func.func @bitcast_i8_to_i32(%input: vector<16xi8>) -> vector<4xi32> {
+  %0 = vector.bitcast %input : vector<16xi8> to vector<4xi32>
+  return %0: vector<4xi32>
+}
+
+// CHECK: %[[INIT:.+]] = arith.constant dense<0> : vector<4xi32>
+// CHECK: %[[EXTRACT0:.+]] = vector.extract_strided_slice %[[INPUT]] {offsets = [0], sizes = [4], strides = [1]} : vector<16xi8> to vector<4xi8>
+// CHECK: %[[CAST0:.+]] = vector.bitcast %[[EXTRACT0]] : vector<4xi8> to vector<1xi32>
+// CHECK: %[[INSERT0:.+]] = vector.insert_strided_slice %[[CAST0]], %[[INIT]] {offsets = [0], strides = [1]} : vector<1xi32> into vector<4xi32>
+// CHECK: %[[EXTRACT1:.+]] = vector.extract_strided_slice %[[INPUT]] {offsets = [4], sizes = [4], strides = [1]} : vector<16xi8> to vector<4xi8>
+// CHECK: %[[CAST1:.+]] = vector.bitcast %[[EXTRACT1]] : vector<4xi8> to vector<1xi32>
+// CHECK: %[[INSERT1:.+]] = vector.insert_strided_slice %[[CAST1]], %[[INSERT0]] {offsets = [1], strides = [1]} : vector<1xi32> into vector<4xi32>
+// CHECK: %[[EXTRACT2:.+]] = vector.extract_strided_slice %[[INPUT]] {offsets = [8], sizes = [4], strides = [1]} : vector<16xi8> to vector<4xi8>
+// CHECK: %[[CAST2:.+]] = vector.bitcast %[[EXTRACT2]] : vector<4xi8> to vector<1xi32>
+// CHECK: %[[INSERT2:.+]] = vector.insert_strided_slice %[[CAST2]], %[[INSERT1]] {offsets = [2], strides = [1]} : vector<1xi32> into vector<4xi32>
+// CHECK: %[[EXTRACT3:.+]] = vector.extract_strided_slice %[[INPUT]] {offsets = [12], sizes = [4], strides = [1]} : vector<16xi8> to vector<4xi8>
+// CHECK: %[[CAST3:.+]] = vector.bitcast %[[EXTRACT3]] : vector<4xi8> to vector<1xi32>
+// CHECK: %[[INSERT3:.+]] = vector.insert_strided_slice %[[CAST3]], %[[INSERT2]] {offsets = [3], strides = [1]} : vector<1xi32> into vector<4xi32>
+// CHECK: return %[[INSERT3]]
diff --git a/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir b/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
index f6f218b6e39eb..578d845a27ad4 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-unroll.mlir
@@ -275,3 +275,90 @@ func.func @transfer_read_unroll_different_rank(%arg0 : memref<?x?x?xf32>) -> vec
   %0 = vector.transfer_read %arg0[%c0, %c0, %c0], %cf0 {permutation_map = #map0} : memref<?x?x?xf32>, vector<6x4xf32>
   return %0 : vector<6x4xf32>
 }
+
+// -----
+
+// CHECK-LABEL: func @vector_gather_unroll
+//  CHECK-SAME:           %[[ARG0:.*]]: memref<?x?x?xf32>
+//  CHECK-SAME:           %[[ARG1:.*]]: vector<6x4xindex>
+//  CHECK-SAME:           %[[ARG2:.*]]: vector<6x4xi1>
+//  CHECK-SAME:           %[[ARG3:.*]]: vector<6x4xf32>
+//  CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//       CHECK:   %[[IDX0:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  CHECK-NEXT:   %[[MASK0:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  CHECK-NEXT:   %[[PASS0:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  CHECK-NEXT:   %[[VGT0:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX0]]], %[[MASK0]], %[[PASS0]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  CHECK-NEXT:   %[[VEC0:.*]] = vector.insert_strided_slice %[[VGT0]], %{{.*}} {offsets = [0, 0], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  CHECK-NEXT:   %[[IDX1:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  CHECK-NEXT:   %[[MASK1:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  CHECK-NEXT:   %[[PASS1:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  CHECK-NEXT:   %[[VGT1:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX1]]], %[[MASK1]], %[[PASS1]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  CHECK-NEXT:   %[[VEC1:.*]] = vector.insert_strided_slice %[[VGT1]], %[[VEC0]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  CHECK-NEXT:   %[[IDX2:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  CHECK-NEXT:   %[[MASK2:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  CHECK-NEXT:   %[[PASS2:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  CHECK-NEXT:   %[[VGT2:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX2]]], %[[MASK2]], %[[PASS2]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  CHECK-NEXT:   %[[VEC2:.*]] = vector.insert_strided_slice %[[VGT2]], %[[VEC1]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  CHECK-NEXT:   %[[IDX3:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  CHECK-NEXT:   %[[MASK3:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  CHECK-NEXT:   %[[PASS3:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  CHECK-NEXT:   %[[VGT3:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX3]]], %[[MASK3]], %[[PASS3]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  CHECK-NEXT:   %[[VEC3:.*]] = vector.insert_strided_slice %[[VGT3]], %[[VEC2]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  CHECK-NEXT:   %[[IDX4:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [4, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  CHECK-NEXT:   %[[MASK4:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [4, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  CHECK-NEXT:   %[[PASS4:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [4, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  CHECK-NEXT:   %[[VGT4:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX4]]], %[[MASK4]], %[[PASS4]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  CHECK-NEXT:   %[[VEC4:.*]] = vector.insert_strided_slice %[[VGT4]], %[[VEC3]] {offsets = [4, 0], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  CHECK-NEXT:   %[[IDX5:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [4, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  CHECK-NEXT:   %[[MASK5:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [4, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  CHECK-NEXT:   %[[PASS5:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [4, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  CHECK-NEXT:   %[[VGT5:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX5]]], %[[MASK5]], %[[PASS5]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  CHECK-NEXT:   %[[VEC5:.*]] = vector.insert_strided_slice %[[VGT5]], %[[VEC4]] {offsets = [4, 2], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  CHECK-NEXT:   return %[[VEC5]] : vector<6x4xf32>
+
+// ORDER-LABEL: func @vector_gather_unroll
+//  ORDER-SAME:           %[[ARG0:.*]]: memref<?x?x?xf32>
+//  ORDER-SAME:           %[[ARG1:.*]]: vector<6x4xindex>
+//  ORDER-SAME:           %[[ARG2:.*]]: vector<6x4xi1>
+//  ORDER-SAME:           %[[ARG3:.*]]: vector<6x4xf32>
+//  ORDER-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//       ORDER:   %[[IDX0:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  ORDER-NEXT:   %[[MASK0:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  ORDER-NEXT:   %[[PASS0:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  ORDER-NEXT:   %[[VGT0:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX0]]], %[[MASK0]], %[[PASS0]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  ORDER-NEXT:   %[[VEC0:.*]] = vector.insert_strided_slice %[[VGT0]], %{{.*}} {offsets = [0, 0], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  ORDER-NEXT:   %[[IDX1:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  ORDER-NEXT:   %[[MASK1:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  ORDER-NEXT:   %[[PASS1:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  ORDER-NEXT:   %[[VGT1:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX1]]], %[[MASK1]], %[[PASS1]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  ORDER-NEXT:   %[[VEC1:.*]] = vector.insert_strided_slice %[[VGT1]], %[[VEC0]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  ORDER-NEXT:   %[[IDX2:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [4, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  ORDER-NEXT:   %[[MASK2:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [4, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  ORDER-NEXT:   %[[PASS2:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [4, 0], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  ORDER-NEXT:   %[[VGT2:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX2]]], %[[MASK2]], %[[PASS2]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  ORDER-NEXT:   %[[VEC2:.*]] = vector.insert_strided_slice %[[VGT2]], %[[VEC1]] {offsets = [4, 0], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  ORDER-NEXT:   %[[IDX3:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  ORDER-NEXT:   %[[MASK3:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  ORDER-NEXT:   %[[PASS3:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  ORDER-NEXT:   %[[VGT3:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX3]]], %[[MASK3]], %[[PASS3]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  ORDER-NEXT:   %[[VEC3:.*]] = vector.insert_strided_slice %[[VGT3]], %[[VEC2]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  ORDER-NEXT:   %[[IDX4:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  ORDER-NEXT:   %[[MASK4:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  ORDER-NEXT:   %[[PASS4:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  ORDER-NEXT:   %[[VGT4:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX4]]], %[[MASK4]], %[[PASS4]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  ORDER-NEXT:   %[[VEC4:.*]] = vector.insert_strided_slice %[[VGT4]], %[[VEC3]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  ORDER-NEXT:   %[[IDX5:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [4, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xindex> to vector<2x2xindex>
+//  ORDER-NEXT:   %[[MASK5:.*]] = vector.extract_strided_slice %[[ARG2]] {offsets = [4, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xi1> to vector<2x2xi1>
+//  ORDER-NEXT:   %[[PASS5:.*]] = vector.extract_strided_slice %[[ARG3]] {offsets = [4, 2], sizes = [2, 2], strides = [1, 1]} : vector<6x4xf32> to vector<2x2xf32>
+//  ORDER-NEXT:   %[[VGT5:.*]] = vector.gather {{.*}}[%[[C0]], %[[C0]], %[[C0]]] [%[[IDX5]]], %[[MASK5]], %[[PASS5]] : memref<?x?x?xf32>, vector<2x2xindex>, vector<2x2xi1>, vector<2x2xf32> into vector<2x2xf32>
+//  ORDER-NEXT:   %[[VEC5:.*]] = vector.insert_strided_slice %[[VGT5]], %[[VEC4]] {offsets = [4, 2], strides = [1, 1]} : vector<2x2xf32> into vector<6x4xf32>
+//  ORDER-NEXT:   return %[[VEC5]] : vector<6x4xf32>
+
+func.func @vector_gather_unroll(%arg0 : memref<?x?x?xf32>,
+                                %indices : vector<6x4xindex>,
+                                %mask : vector<6x4xi1>,
+                                %pass_thru : vector<6x4xf32>) -> vector<6x4xf32> {
+  %c0 = arith.constant 0 : index
+  %0 = vector.gather %arg0[%c0, %c0, %c0] [%indices], %mask, %pass_thru : memref<?x?x?xf32>, vector<6x4xindex>, vector<6x4xi1>, vector<6x4xf32> into vector<6x4xf32>
+  return %0 : vector<6x4xf32>
+}
diff --git a/mlir/test/IR/greedy-pattern-rewriter-driver.mlir b/mlir/test/IR/greedy-pattern-rewriter-driver.mlir
index 6f4923a9f4f75..a4fb56d584a34 100644
--- a/mlir/test/IR/greedy-pattern-rewriter-driver.mlir
+++ b/mlir/test/IR/greedy-pattern-rewriter-driver.mlir
@@ -7,7 +7,7 @@ func.func @add_to_worklist_after_inplace_update() {
   // worklist of the GreedyPatternRewriteDriver (regardless of the value of
   // config.max_iterations).
 
-  // CHECK: "test.any_attr_of_i32_str"() {attr = 3 : i32} : () -> ()
+  // CHECK: "test.any_attr_of_i32_str"() <{attr = 3 : i32}> : () -> ()
   "test.any_attr_of_i32_str"() {attr = 0 : i32} : () -> ()
   return
 }
diff --git a/mlir/test/IR/invalid.mlir b/mlir/test/IR/invalid.mlir
index 9bf7efd8ae5d8..861f4ef6c020d 100644
--- a/mlir/test/IR/invalid.mlir
+++ b/mlir/test/IR/invalid.mlir
@@ -587,11 +587,6 @@ func.func @bad_arrow(%arg : !unreg.ptr<(i32)->)
 
 // -----
 
-// expected-error @+1 {{attribute 'attr' occurs more than once in the attribute list}}
-test.format_symbol_name_attr_op @name { attr = "xx" }
-
-// -----
-
 func.func @forward_reference_type_check() -> (i8) {
   cf.br ^bb2
 
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index 75ebc1a7536c3..2041ca90e8116 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -1437,3 +1437,4 @@ test.dialect_custom_format_fallback custom_format_fallback
 // Check that an op with an optional result parses f80 as type.
 // CHECK: test.format_optional_result_d_op : f80
 test.format_optional_result_d_op : f80
+
diff --git a/mlir/test/IR/print-ir-invalid.mlir b/mlir/test/IR/print-ir-invalid.mlir
index 70a0b90d53683..46b34e5f8b63e 100644
--- a/mlir/test/IR/print-ir-invalid.mlir
+++ b/mlir/test/IR/print-ir-invalid.mlir
@@ -8,10 +8,9 @@ module {}
 // The operation is invalid because the body does not have a terminator, print
 // the generic form.
 // CHECK:      Invalid operation:
-// CHECK-NEXT: "func.func"() ({
+// CHECK-NEXT: "func.func"() <{function_type = () -> (), sym_name = "test"}> ({
 // CHECK-NEXT: ^bb0:
 // CHECK-NEXT: })
-// CHECK-SAME: sym_name = "test"
 
 // The operation is valid because the body has a terminator, print the custom
 // form.
diff --git a/mlir/test/IR/properties.mlir b/mlir/test/IR/properties.mlir
new file mode 100644
index 0000000000000..b073698b40aff
--- /dev/null
+++ b/mlir/test/IR/properties.mlir
@@ -0,0 +1,20 @@
+// # RUN: mlir-opt %s -split-input-file | mlir-opt  |FileCheck %s
+// # RUN: mlir-opt %s -mlir-print-op-generic -split-input-file  | mlir-opt -mlir-print-op-generic | FileCheck %s --check-prefix=GENERIC
+
+// CHECK:   test.with_properties
+// CHECK-SAME: <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, b = "foo"}>
+// GENERIC:   "test.with_properties"()
+// GENERIC-SAME: <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, b = "foo"}> : () -> ()
+test.with_properties <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, b = "foo"}>
+
+// CHECK:   test.with_nice_properties
+// CHECK-SAME:    "foo bar" is -3
+// GENERIC: "test.with_nice_properties"()
+// GENERIC-SAME:  <{prop = {label = "foo bar", value = -3 : i32}}> : () -> ()
+test.with_nice_properties "foo bar" is -3
+
+// CHECK:   test.with_wrapped_properties
+// CHECK-SAME:    "content for properties"
+// GENERIC: "test.with_wrapped_properties"()
+// GENERIC-SAME:  <{prop = "content for properties"}> : () -> ()
+test.with_wrapped_properties <{prop = "content for properties"}>
diff --git a/mlir/test/IR/test-fold-adaptor.mlir b/mlir/test/IR/test-fold-adaptor.mlir
index 7815e729f55ad..a7dec9bb871a9 100644
--- a/mlir/test/IR/test-fold-adaptor.mlir
+++ b/mlir/test/IR/test-fold-adaptor.mlir
@@ -12,5 +12,5 @@ func.func @test() -> i32 {
 }
 
 // CHECK-LABEL: func.func @test
-// CHECK-NEXT: %[[C:.*]] = "test.constant"() {value = 33 : i32}
+// CHECK-NEXT: %[[C:.*]] = "test.constant"() <{value = 33 : i32}>
 // CHECK-NEXT: return %[[C]]
diff --git a/mlir/test/IR/test-manual-cpp-fold.mlir b/mlir/test/IR/test-manual-cpp-fold.mlir
index 592b949f0a139..7364b7ccb5762 100644
--- a/mlir/test/IR/test-manual-cpp-fold.mlir
+++ b/mlir/test/IR/test-manual-cpp-fold.mlir
@@ -7,5 +7,5 @@ func.func @test() -> i32 {
 }
 
 // CHECK-LABEL: func.func @test
-// CHECK-NEXT: %[[C:.*]] = "test.constant"() {value = 5 : i32}
+// CHECK-NEXT: %[[C:.*]] = "test.constant"() <{value = 5 : i32}>
 // CHECK-NEXT: return %[[C]]
diff --git a/mlir/test/IR/traits.mlir b/mlir/test/IR/traits.mlir
index ddba1171649c9..307918defab0e 100644
--- a/mlir/test/IR/traits.mlir
+++ b/mlir/test/IR/traits.mlir
@@ -345,8 +345,8 @@ func.func @failedSingleBlockImplicitTerminator_missing_terminator() {
 
 // -----
 
-// expected-error@+1 {{op attribute 'sym_visibility' failed to satisfy constraint: string attribute}}
-"test.symbol"() {sym_name = "foo_2", sym_visibility} : () -> ()
+// expected-error@+1 {{invalid properties {sym_name = "foo_2", sym_visibility} for op test.symbol: Invalid attribute `sym_visibility` in property conversion: unit}}
+"test.symbol"() <{sym_name = "foo_2", sym_visibility}> : () -> ()
 
 // -----
 
@@ -390,14 +390,14 @@ func.func @failedMissingOperandSizeAttr(%arg: i32) {
 // -----
 
 func.func @failedOperandSizeAttrWrongType(%arg: i32) {
-  // expected-error @+1 {{requires dense i32 array attribute 'operand_segment_sizes'}}
+  // expected-error @+1 {{attribute 'operand_segment_sizes' failed to satisfy constraint: i32 dense array attribute}}
   "test.attr_sized_operands"(%arg, %arg, %arg, %arg) {operand_segment_sizes = 10} : (i32, i32, i32, i32) -> ()
 }
 
 // -----
 
 func.func @failedOperandSizeAttrWrongElementType(%arg: i32) {
-  // expected-error @+1 {{requires dense i32 array attribute 'operand_segment_sizes'}}
+  // expected-error @+1 {{attribute 'operand_segment_sizes' failed to satisfy constraint: i32 dense array attribute}}
   "test.attr_sized_operands"(%arg, %arg, %arg, %arg) {operand_segment_sizes = array<i64: 1, 1, 1, 1>} : (i32, i32, i32, i32) -> ()
 }
 
@@ -655,7 +655,7 @@ func.func @failed_type_traits() {
 
 // Check that we can query traits in attributes
 func.func @succeeded_attr_traits() {
-  // CHECK: "test.attr_with_trait"() {attr = #test.attr_with_trait} : () -> ()
+  // CHECK: "test.attr_with_trait"() <{attr = #test.attr_with_trait}> : () -> ()
   "test.attr_with_trait"() {attr = #test.attr_with_trait} : () -> ()
   return
 }
diff --git a/mlir/test/IR/wrapping_op.mlir b/mlir/test/IR/wrapping_op.mlir
index 1c3992917d6e7..3f7a350ee654e 100644
--- a/mlir/test/IR/wrapping_op.mlir
+++ b/mlir/test/IR/wrapping_op.mlir
@@ -3,6 +3,7 @@
 
 // CHECK-LABEL: func @wrapping_op
 // CHECK-GENERIC: "func.func"
+// CHECK-GENERIC-SAME: sym_name = "wrapping_op"
 func.func @wrapping_op(%arg0 : i32, %arg1 : f32) -> (i3, i2, i1) {
 // CHECK: %0:3 = test.wrapping_region wraps "some.op"(%arg1, %arg0) {test.attr = "attr"} : (f32, i32) -> (i1, i2, i3)
 // CHECK-GENERIC: "test.wrapping_region"() ({
@@ -12,4 +13,3 @@ func.func @wrapping_op(%arg0 : i32, %arg1 : f32) -> (i3, i2, i1) {
   %res:3 = test.wrapping_region wraps "some.op"(%arg1, %arg0) { test.attr = "attr" } : (f32, i32) -> (i1, i2, i3) loc("some_NameLoc")
   return %res#2, %res#1, %res#0 : i3, i2, i1
 }
-// CHECK-GENERIC: sym_name = "wrapping_op"
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
index 8e6fe16ccb909..f0408a56c6d66 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
index 30fd003e26739..30427d9c2594a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
index 8607ac4539c12..9af88ba40f309 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
@@ -12,10 +12,8 @@
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true"
 // RUN: %{compile} | %{run}
 //
-// Do the same run, but now with direct IR generation and vectorization. Enable
-// Arm SVE if supported.
-// REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA reassociate-fp-reductions=true enable-index-optimizations=true"
-// REDEFINE: %{run_option} = %VLA_ARCH_ATTR_OPTIONS
+// Do the same run, but now with direct IR generation and vectorization.
+// REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 reassociate-fp-reductions=true enable-index-optimizations=true"
 // RUN: %{compile} | %{run}
 
 #MAT_C_C = #sparse_tensor.encoding<{dimLevelType = ["compressed", "compressed"]}>
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
index d2fe8df722710..92ffb4c10a657 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
index 423070a323841..0f45ea8a3fcea 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test.mtx" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
index 66a20be2f2986..03b05b156418f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg b/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
index b0a8a6c732441..9bf49cc8246ec 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
@@ -1,4 +1,5 @@
 import sys
+from lit.llvm import llvm_config
 
 # FIXME: %mlir_native_utils_lib_dir is set incorrectly on Windows
 if sys.platform == 'win32':
@@ -8,26 +9,7 @@ if sys.platform == 'win32':
 if config.mlir_run_arm_sve_tests:
     config.substitutions.append(('%ENABLE_VLA', 'true'))
     config.substitutions.append(('%VLA_ARCH_ATTR_OPTIONS', '--march=aarch64 --mattr="+sve"'))
-    lli_cmd = 'lli'
-    if config.arm_emulator_lli_executable:
-       lli_cmd = config.arm_emulator_lli_executable
-
-    if config.arm_emulator_utils_lib_dir:
-        config.substitutions.append(('%mlir_native_utils_lib_dir', config.arm_emulator_utils_lib_dir))
-    else:
-        config.substitutions.append(('%mlir_native_utils_lib_dir', config.mlir_lib_dir))
-
-    if config.arm_emulator_executable:
-        # Run test in emulator (qemu or armie).
-        emulation_cmd = config.arm_emulator_executable
-        if config.arm_emulator_options:
-            emulation_cmd = emulation_cmd + ' ' + config.arm_emulator_options
-        emulation_cmd = emulation_cmd + ' ' + lli_cmd
-        config.substitutions.append(('%lli', emulation_cmd))
-    else:
-        config.substitutions.append(('%lli', lli_cmd))
 else:
     config.substitutions.append(('%ENABLE_VLA', 'false'))
     config.substitutions.append(('%VLA_ARCH_ATTR_OPTIONS', ''))
-    config.substitutions.append(('%lli', 'lli'))
     config.substitutions.append(('%mlir_native_utils_lib_dir', config.mlir_lib_dir))
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
index 34a863e73095b..69336f1964d78 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
index 6eec742f47c1b..8a1d92fe99731 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
index cd054f7208f55..0571104e78709 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
index 909aea71177c3..cdfd87e339f86 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
index dec8e2cd9adb8..1b2492aecc2f1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
index 2591aefe120e5..16a0e790c129c 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
index 4a57bd79e8bdb..c4f00b1e6a3a5 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
index 39997c170e9ca..f87af2aa53437 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
index c48d4aaf2f140..2a83410cb34da 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
index 4bf7735a8a989..f49cd630d98aa 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
index d7a4f2317f880..c91466b71e1c3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA enable-index-reduction=true"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
index 555d1bb232035..65aa6f894df0e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-index-reduction=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
index 933272c1d36a0..902496e6eac16 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA  enable-index-reduction=true"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
index f9602ab93d259..8e2f162ec9591 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA enable-index-reduction=true"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
index 5553f27ce4563..ff792d33268f2 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA enable-index-reduction=true"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
index 6053bb17307fc..05f9a560bd5f8 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
index 9e630a0f54eec..a829d73e15514 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 reassociate-fp-reductions=true enable-index-optimizations=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
index 3ffa323659ee7..564ee40c1a2f1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
index ef0ba27a36b8e..6f5fadaa20357 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 reassociate-fp-reductions=true enable-index-optimizations=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
index b9937d353f1e5..64e17e961be3c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
index fbcd04338c317..393dee01b07b7 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
@@ -19,7 +19,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
index 8681d33ce0f80..bffd171bee4d6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
index 6a60d3e6c89c0..5e52415c34ea1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
index cebc43c9e223f..f22e04685edc7 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
@@ -17,7 +17,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
index 37ec3de2d49be..8b4863e069b26 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_file_io.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_file_io.mlir
index a868280abaede..3cd69d7709cce 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_file_io.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_file_io.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/wide.mtx" TENSOR1="" \
-// REDEFINE:   %lli \
+// REDEFINE:   %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
index e2d691d32211d..69fa2cc57eacf 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
index 371c8f20938bf..bdb167366158c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test.tns" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
index 6508dee90fabf..ecfa5682aa560 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
index 91b02d73c33af..31fd43be936e4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
index c5a66fa73b8e3..027be9d4f7dc0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
index 5ae72e028c0c1..d07712b28326c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
index 0d2aa4990655e..1c804b622eaa1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
index 361ea7afff7ee..e768389b8afbf 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
@@ -26,7 +26,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
index 9da202b1de4f0..729f8212ce177 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
index 23bc5acbe7b01..91381ecd9efe0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
@@ -27,7 +27,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/wide.mtx" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
index 460a201ee9f9c..646f328a8b3a8 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4  enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/mttkrp_b.tns" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
index 29113060162b1..29e0b134cfa3f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
index 191871ccad85d..ad18bd18cf5dd 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
index a495a6df7f403..9fb98913a781a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4  enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test.mtx" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
index b3ba3529f1d0a..34f0188a92720 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
@@ -11,7 +11,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
@@ -31,6 +31,10 @@
   crdWidth = 32
 }>
 
+#BCOO = #sparse_tensor.encoding<{
+  dimLevelType = [ "dense", "compressed-hi-nu", "singleton" ]
+}>
+
 module {
   //
   // Main driver.
@@ -60,6 +64,25 @@ module {
 
     %s4 = sparse_tensor.pack %data, %index : tensor<3xf64>, tensor<3x2xindex>
                                           to tensor<10x10xf64, #SortedCOO>
+    %s5= sparse_tensor.pack %data, %index32 : tensor<3xf64>, tensor<3x2xi32>
+                                           to tensor<10x10xf64, #SortedCOOI32>
+
+    %bdata = arith.constant dense<
+       [[  1.0,  2.0,  3.0],
+        [  4.0,  5.0,  0.0]]
+    > : tensor<2x3xf64>
+
+    %bindex = arith.constant dense<
+      [[[  1,  2],
+        [  5,  6],
+        [  7,  8]],
+       [[  2,  3],
+        [  4,  2],
+        [ 10, 10]]]
+    > : tensor<2x3x2xindex>
+    %bs = sparse_tensor.pack %bdata, %bindex batched_lvls = 1 :
+          tensor<2x3xf64>, tensor<2x3x2xindex> to tensor<2x10x10xf64, #BCOO>
+
     // CHECK:1
     // CHECK-NEXT:2
     // CHECK-NEXT:1
@@ -78,8 +101,6 @@ module {
         vector.print %v: f64
      }
 
-    %s5= sparse_tensor.pack %data, %index32 : tensor<3xf64>, tensor<3x2xi32>
-                                          to tensor<10x10xf64, #SortedCOOI32>
     // CHECK-NEXT:1
     // CHECK-NEXT:2
     // CHECK-NEXT:1
@@ -98,11 +119,23 @@ module {
         vector.print %v: f64
      }
 
+    // CHECK-NEXT:1
+    // CHECK-NEXT:2
+    // CHECK-NEXT:3
+    //
+    // CHECK-NEXT:4
+    // CHECK-NEXT:5
+    //
+    // Make sure the trailing zeros are not traversed.
+    // CHECK-NOT: 0
+    sparse_tensor.foreach in %bs : tensor<2x10x10xf64, #BCOO> do {
+      ^bb0(%0: index, %1: index, %2: index, %v: f64) :
+        vector.print %v: f64
+     }
+
     %d, %i, %n = sparse_tensor.unpack %s5 : tensor<10x10xf64, #SortedCOOI32>
                                          to tensor<3xf64>, tensor<3x2xi32>, i32
 
-
-
     // CHECK-NEXT: ( 1, 2, 3 )
     %vd = vector.transfer_read %d[%c0], %f0 : tensor<3xf64>, vector<3xf64>
     vector.print %vd : vector<3xf64>
@@ -114,8 +147,26 @@ module {
     // CHECK-NEXT: 3
     vector.print %n : i32
 
+
+    %bd, %bi, %bn = sparse_tensor.unpack %bs batched_lvls=1 :
+       tensor<2x10x10xf64, #BCOO> to tensor<2x3xf64>, tensor<2x3x2xindex>, i32
+
+    // CHECK-NEXT: ( ( 1, 2, 3 ), ( 4, 5, 0 ) )
+    %vbd = vector.transfer_read %bd[%c0, %c0], %f0 : tensor<2x3xf64>, vector<2x3xf64>
+    vector.print %vbd : vector<2x3xf64>
+
+    // CHECK-NEXT: ( ( ( 1, 2 ), ( 5, 6 ), ( 7, 8 ) ), ( ( 2, 3 ), ( 4, 2 ), ( 0, 0 ) ) )
+    %vbi = vector.transfer_read %bi[%c0, %c0, %c0], %c0 : tensor<2x3x2xindex>, vector<2x3x2xindex>
+    vector.print %vbi : vector<2x3x2xindex>
+
+    // CHECK-NEXT: 3
+    vector.print %bn : i32
+
     %d1, %i1, %n1 = sparse_tensor.unpack %s4 : tensor<10x10xf64, #SortedCOO>
                                          to tensor<3xf64>, tensor<3x2xindex>, index
+    // FIXME: This should be freed by one-shot-bufferization.
+    bufferization.dealloc_tensor %bd : tensor<2x3xf64>
+    bufferization.dealloc_tensor %bi : tensor<2x3x2xindex>
     return
   }
 }
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
index 59730242257a8..e6f79cfbc9cb1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
index 34e76e0281843..76f2bb0415f26 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
index eada3126e2968..cc28defe78ab2 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 reassociate-fp-reductions=true enable-index-optimizations=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
index a4f5adae21c96..6e7163494a238 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
index 0bf27cd12f3f1..84cac1a72580e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort.mlir
index 5ded72c5c343b..f5ae6e7f72a33 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort.mlir
@@ -10,7 +10,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
index a8678eb11b11e..5bc02fc08e22d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
index 05fc7f65aad3e..f678cdd3f07f4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4  enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test.mtx" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
index ed2ee71470883..8b373e7038e6f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
index 4d54f8ceab427..b429fc87bba7f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
index 479cd80a342ec..1be70b56d7999 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
index 42a7d4d758c25..cf103a7239e82 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
index 3d1b0c40fc95a..11db2c2ff6e82 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
@@ -14,7 +14,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
index 3ebd9dff9111e..224dedf48a707 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
@@ -18,7 +18,7 @@
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/wide.mtx" \
 // REDEFINE:    TENSOR1="%mlir_src_dir/test/Integration/data/mttkrp_b.tns" \
-// REDEFINE:    lli \
+// REDEFINE:    %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
index 190ba707993f0..c32ae69720bc0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4  enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/wide.mtx" \
-// REDEFINE: lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
index 9454c06067659..56c9511f63d09 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
index e7196980cedbb..f9b1ad72ed544 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4  enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test_symmetric.mtx" \
-// REDEFINE: %lli \
+// REDEFINE: %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
index 0e66ec86f39fe..cdbe550ea4adf 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
@@ -20,7 +20,7 @@
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
 // REDEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test_symmetric_complex.mtx" \
-// REDEFINE:   %lli \
+// REDEFINE:   %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
index 77304ee3b1bbd..1769455b2a8d0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
index 73cb2e77832c9..91981bd028999 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
@@ -21,7 +21,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 reassociate-fp-reductions=true enable-index-optimizations=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
index b12ec32dd6207..4a3cc7597fa81 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
index 3a75a50ec75e7..b269a486f29c8 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
index f48dbd5678906..c7e1e09dad8d1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
index 27964f47913c2..ebf1638c32564 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
@@ -19,7 +19,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
index 4723e8df823f1..4af5647230946 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false vl=4 enable-buffer-initialization=true reassociate-fp-reductions=true enable-index-optimizations=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
index 6dfde8e5e696f..b8200851b1003 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
@@ -18,7 +18,7 @@
 // Do the same run, but now with direct IR generation and, if available, VLA
 // vectorization.
 // REDEFINE: %{option} = "enable-runtime-library=false enable-buffer-initialization=true vl=4 reassociate-fp-reductions=true enable-index-optimizations=true enable-arm-sve=%ENABLE_VLA"
-// REDEFINE: %{run} = %lli \
+// REDEFINE: %{run} = %lli_host_or_aarch64_cmd \
 // REDEFINE:   --entry-function=entry_lli \
 // REDEFINE:   --extra-module=%S/Inputs/main_for_lli.ll \
 // REDEFINE:   %VLA_ARCH_ATTR_OPTIONS \
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/lit.local.cfg b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/lit.local.cfg
new file mode 100644
index 0000000000000..0423fc03da5f9
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/lit.local.cfg
@@ -0,0 +1,9 @@
+import sys
+
+# ArmSME tests must be enabled via build flag.
+if not config.mlir_run_arm_sme_tests:
+    config.unsupported = True
+
+# No JIT on win32.
+if sys.platform == 'win32':
+    config.unsupported = True
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-ssve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-ssve.mlir
new file mode 100644
index 0000000000000..fce890fd93a25
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-ssve.mlir
@@ -0,0 +1,65 @@
+// RUN: mlir-opt %s -test-lower-to-llvm | \
+// RUN: mlir-translate -mlir-to-llvmir | \
+// RUN: %lli_aarch64_cmd --march=aarch64 --mattr="+sve,+sme" \
+// RUN:      -force-streaming-compatible-sve \
+// RUN:      --entry-function=entry \
+// RUN:      --dlopen=%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: FileCheck %s
+
+// NOTE: To run this test, your CPU must support SME.
+
+// VLA memcopy in streaming mode.
+func.func @streaming_kernel_copy(%src : memref<?xi64>, %dst : memref<?xi64>, %size : index) attributes {passthrough = ["aarch64_pstate_sm_enabled"]} {
+  %c0 = arith.constant 0 : index
+  %c2 = arith.constant 2 : index
+  %vscale = vector.vscale
+  %step = arith.muli %c2, %vscale : index
+  scf.for %i = %c0 to %size step %step {
+    %0 = vector.load %src[%i] : memref<?xi64>, vector<[2]xi64>
+    vector.store %0, %dst[%i] : memref<?xi64>, vector<[2]xi64>
+  }
+  return
+}
+
+func.func @entry() -> i32 {
+  %i0 = arith.constant 0: i64
+  %r0 = arith.constant 0: i32
+  %c0 = arith.constant 0: index
+  %c4 = arith.constant 4: index
+  %c32 = arith.constant 32: index
+
+  // Set up memory.
+  %a = memref.alloc()      : memref<32xi64>
+  %a_copy = memref.alloc() : memref<32xi64>
+  %a_data = arith.constant dense<[1 , 2,  3 , 4 , 5,  6,  7,  8,
+                                  9, 10, 11, 12, 13, 14, 15, 16,
+                                  17, 18, 19, 20, 21, 22, 23, 24,
+                                  25, 26, 27, 28, 29, 30, 31, 32]> : vector<32xi64>
+  vector.transfer_write %a_data, %a[%c0] : vector<32xi64>, memref<32xi64>
+
+  // Call kernel.
+  %0 = memref.cast %a : memref<32xi64> to memref<?xi64>
+  %1 = memref.cast %a_copy : memref<32xi64> to memref<?xi64>
+  call @streaming_kernel_copy(%0, %1, %c32) : (memref<?xi64>, memref<?xi64>, index) -> ()
+
+  // Print and verify.
+  //
+  // CHECK:      ( 1, 2, 3, 4 )
+  // CHECK-NEXT: ( 5, 6, 7, 8 )
+  // CHECK-NEXT: ( 9, 10, 11, 12 )
+  // CHECK-NEXT: ( 13, 14, 15, 16 )
+  // CHECK-NEXT: ( 17, 18, 19, 20 )
+  // CHECK-NEXT: ( 21, 22, 23, 24 )
+  // CHECK-NEXT: ( 25, 26, 27, 28 )
+  // CHECK-NEXT: ( 29, 30, 31, 32 )
+  scf.for %i = %c0 to %c32 step %c4 {
+    %cv = vector.transfer_read %a_copy[%i], %i0 : memref<32xi64>, vector<4xi64>
+    vector.print %cv : vector<4xi64>
+  }
+
+  // Release resources.
+  memref.dealloc %a      : memref<32xi64>
+  memref.dealloc %a_copy : memref<32xi64>
+
+  return %r0 : i32
+}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/lit.local.cfg b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/lit.local.cfg
index 077904cc44df0..8a0d884509c80 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/lit.local.cfg
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/lit.local.cfg
@@ -7,20 +7,3 @@ if not config.mlir_run_arm_sve_tests:
 # No JIT on win32.
 if sys.platform == 'win32':
     config.unsupported = True
-
-lli_cmd = 'lli'
-if config.arm_emulator_lli_executable:
-    lli_cmd = config.arm_emulator_lli_executable
-
-config.substitutions.append(('%mlir_native_utils_lib_dir',
-    config.arm_emulator_utils_lib_dir or config.mlir_lib_dir))
-
-if config.arm_emulator_executable:
-    # Run test in emulator (qemu or armie)
-    emulation_cmd = config.arm_emulator_executable
-    if config.arm_emulator_options:
-        emulation_cmd = emulation_cmd + ' ' + config.arm_emulator_options
-    emulation_cmd = emulation_cmd + ' ' + lli_cmd
-    config.substitutions.append(('%lli', emulation_cmd))
-else:
-    config.substitutions.append(('%lli', lli_cmd))
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir
index 794ed761e65b0..0570231442fec 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-sve.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s -lower-affine -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -finalize-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \
 // RUN: mlir-translate -mlir-to-llvmir | \
-// RUN: %lli --entry-function=entry --march=aarch64 --mattr="+sve" --dlopen=%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: %lli_aarch64_cmd --entry-function=entry --march=aarch64 --mattr="+sve" --dlopen=%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext | \
 // RUN: FileCheck %s
 
 // Note: To run this test, your CPU must support SVE
diff --git a/mlir/test/Integration/lit.local.cfg b/mlir/test/Integration/lit.local.cfg
index 837d1b7c898d6..80a862a1ce664 100644
--- a/mlir/test/Integration/lit.local.cfg
+++ b/mlir/test/Integration/lit.local.cfg
@@ -1,2 +1,56 @@
+from lit.llvm import llvm_config
+
 if not config.mlir_include_integration_tests:
     config.unsupported = True
+
+def configure_aarch64_lli_cmd():
+    lli_cmd = 'lli'
+
+    # NOTE: If the SVE tests are disabled and the SME tests are enabled to run
+    # under emulation, the SVE specific RUN lines in the SparseTensor tests
+    # will run under emulation.
+    if not (config.mlir_run_arm_sve_tests or config.mlir_run_arm_sme_tests):
+        return lli_cmd
+
+    config.substitutions.append(('%mlir_native_utils_lib_dir',
+        config.arm_emulator_utils_lib_dir or config.mlir_lib_dir))
+
+    if config.arm_emulator_executable:
+        if config.arm_emulator_lli_executable:
+            lli_cmd = config.arm_emulator_lli_executable
+        else:
+            # Top-level lit config adds llvm_tools_dir to PATH but this is lost
+            # when running under an emulator. If the user didn't specify an lli
+            # executable, use absolute path %llvm_tools_dir/lli.
+            lli_cmd = llvm_config.use_llvm_tool(
+                'lli', search_env='LLI', required=True,
+                search_paths=[config.llvm_tools_dir], use_installed=False
+            )
+
+        # Run test in emulator (qemu or armie)
+        emulation_cmd = f'{config.arm_emulator_executable} {config.arm_emulator_options}'
+        lli_cmd = f'{emulation_cmd} {lli_cmd}'
+
+    return lli_cmd
+
+aarch64_lli_cmd = configure_aarch64_lli_cmd()
+
+# Configure the following AArch64 substitutions:
+#
+# * %lli_aarch64_cmd         - Invokes lli. For tests that will run on AArch64 (ArmSVE, ArmSME).
+# * %lli_host_or_aarch64_cmd - Invokes lli. For tests that may run on AArch64 (SparseTensor).
+#
+# AArch64 tests will run under emulation if configured at build time by the
+# following CMake options:
+#
+# * ARM_EMULATOR_EXECUTABLE     - emulator to use.
+# * ARM_EMULATOR_OPTIONS        - options for emulator.
+# * ARM_EMULATOR_LLI_EXECUTABLE - AArch64 native lli to support cross-compilation.
+# * ARM_EMULATOR_UTILS_LIB_DIR  - AArch64 native utilites library to support cross-compilation.
+#
+# Functionally the two substitutions are equivalent, i.e. %lli_aarch64_cmd
+# could be used in the SparseTensor tests where necessary, but the meaning
+# conveyed by the substitution name would be a misnomer if the host target
+# is not AArch64 and MLIR_RUN_ARM_SVE_TESTS=OFF.
+config.substitutions.append(('%lli_aarch64_cmd', aarch64_lli_cmd))
+config.substitutions.append(('%lli_host_or_aarch64_cmd', aarch64_lli_cmd))
diff --git a/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir b/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir
index 81c1531484d43..c74af447d1b1f 100644
--- a/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir
+++ b/mlir/test/Interfaces/InferIntRangeInterface/infer-int-range-test-ops.mlir
@@ -1,7 +1,7 @@
 // RUN: mlir-opt -test-int-range-inference %s | FileCheck %s
 
 // CHECK-LABEL: func @constant
-// CHECK: %[[cst:.*]] = "test.constant"() {value = 3 : index}
+// CHECK: %[[cst:.*]] = "test.constant"() <{value = 3 : index}
 // CHECK: return %[[cst]]
 func.func @constant() -> index {
   %0 = test.with_bounds { umin = 3 : index, umax = 3 : index,
@@ -10,7 +10,7 @@ func.func @constant() -> index {
 }
 
 // CHECK-LABEL: func @increment
-// CHECK: %[[cst:.*]] = "test.constant"() {value = 4 : index}
+// CHECK: %[[cst:.*]] = "test.constant"() <{value = 4 : index}
 // CHECK: return %[[cst]]
 func.func @increment() -> index {
   %0 = test.with_bounds { umin = 3 : index, umax = 3 : index, smin = 0 : index, smax = 0x7fffffffffffffff : index }
@@ -103,8 +103,8 @@ func.func @func_args_unbound(%arg0 : index) -> index {
 
 // CHECK-LABEL: func @propagate_across_while_loop_false()
 func.func @propagate_across_while_loop_false() -> index {
-  // CHECK-DAG: %[[C0:.*]] = "test.constant"() {value = 0
-  // CHECK-DAG: %[[C1:.*]] = "test.constant"() {value = 1
+  // CHECK-DAG: %[[C0:.*]] = "test.constant"() <{value = 0
+  // CHECK-DAG: %[[C1:.*]] = "test.constant"() <{value = 1
   %0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
                           smin = 0 : index, smax = 0 : index }
   %1 = scf.while : () -> index {
@@ -122,8 +122,8 @@ func.func @propagate_across_while_loop_false() -> index {
 
 // CHECK-LABEL: func @propagate_across_while_loop
 func.func @propagate_across_while_loop(%arg0 : i1) -> index {
-  // CHECK-DAG: %[[C0:.*]] = "test.constant"() {value = 0
-  // CHECK-DAG: %[[C1:.*]] = "test.constant"() {value = 1
+  // CHECK-DAG: %[[C0:.*]] = "test.constant"() <{value = 0
+  // CHECK-DAG: %[[C1:.*]] = "test.constant"() <{value = 1
   %0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
                           smin = 0 : index, smax = 0 : index }
   %1 = scf.while : () -> index {
@@ -140,7 +140,7 @@ func.func @propagate_across_while_loop(%arg0 : i1) -> index {
 
 // CHECK-LABEL: func @dont_propagate_across_infinite_loop()
 func.func @dont_propagate_across_infinite_loop() -> index {
-  // CHECK: %[[C0:.*]] = "test.constant"() {value = 0
+  // CHECK: %[[C0:.*]] = "test.constant"() <{value = 0
   %0 = test.with_bounds { umin = 0 : index, umax = 0 : index,
                           smin = 0 : index, smax = 0 : index }
   // CHECK: %[[loopRes:.*]] = scf.while
diff --git a/mlir/test/Pass/action-logging-filter.mlir b/mlir/test/Pass/action-logging-filter.mlir
index c853a08b57a4b..174616391d45b 100644
--- a/mlir/test/Pass/action-logging-filter.mlir
+++ b/mlir/test/Pass/action-logging-filter.mlir
@@ -16,24 +16,24 @@ func.func @c() {
 ////////////////////////////////////
 /// 1. All actions should be logged.
 
-// RUN: mlir-opt %s --log-actions-to=- -pass-pipeline="builtin.module(func.func(canonicalize))" -o %t --mlir-disable-threading | FileCheck %s
+// RUN: mlir-opt %s --log-actions-to=- -pass-pipeline="builtin.module(func.func(test-stats-pass))" -o %t --mlir-disable-threading | FileCheck %s
 // Specify the current file as filter, expect to see all actions.
-// RUN: mlir-opt %s --log-mlir-actions-filter=%s --log-actions-to=- -pass-pipeline="builtin.module(func.func(canonicalize))" -o %t --mlir-disable-threading | FileCheck %s
+// RUN: mlir-opt %s --log-mlir-actions-filter=%s --log-actions-to=- -pass-pipeline="builtin.module(func.func(test-stats-pass))" -o %t --mlir-disable-threading | FileCheck %s
 
-// CHECK: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `Canonicalizer` on Operation `func.func` (func.func @a() {...}
+// CHECK: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestStatisticPass` on Operation `func.func` (func.func @a() {...}
 // CHECK-NEXT: [thread {{.*}}] completed `pass-execution`
-// CHECK-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `Canonicalizer` on Operation `func.func` (func.func @b() {...}
+// CHECK-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestStatisticPass` on Operation `func.func` (func.func @b() {...}
 // CHECK-NEXT: [thread {{.*}}] completed `pass-execution`
-// CHECK-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `Canonicalizer` on Operation `func.func` (func.func @c() {...}
+// CHECK-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestStatisticPass` on Operation `func.func` (func.func @c() {...}
 // CHECK-NEXT: [thread {{.*}}] completed `pass-execution`
 
 ////////////////////////////////////
 /// 2. No match
 
 // Specify a non-existing file as filter, expect to see no actions.
-// RUN: mlir-opt %s --log-mlir-actions-filter=foo.mlir --log-actions-to=- -pass-pipeline="builtin.module(func.func(canonicalize))" -o %t --mlir-disable-threading | FileCheck %s --check-prefix=CHECK-NONE --allow-empty
+// RUN: mlir-opt %s --log-mlir-actions-filter=foo.mlir --log-actions-to=- -pass-pipeline="builtin.module(func.func(test-stats-pass))" -o %t --mlir-disable-threading | FileCheck %s --check-prefix=CHECK-NONE --allow-empty
 // Filter on a non-matching line, expect to see no actions.
-// RUN: mlir-opt %s --log-mlir-actions-filter=%s:1 --log-actions-to=- -pass-pipeline="builtin.module(func.func(canonicalize))" -o %t --mlir-disable-threading | FileCheck %s --check-prefix=CHECK-NONE --allow-empty
+// RUN: mlir-opt %s --log-mlir-actions-filter=%s:1 --log-actions-to=- -pass-pipeline="builtin.module(func.func(test-stats-pass))" -o %t --mlir-disable-threading | FileCheck %s --check-prefix=CHECK-NONE --allow-empty
 
 // Invalid Filter
 // CHECK-NONE-NOT: Canonicalizer
@@ -42,19 +42,19 @@ func.func @c() {
 /// 3. Matching filters
 
 // Filter the second function only
-// RUN: mlir-opt %s --log-mlir-actions-filter=%s:8 --log-actions-to=- -pass-pipeline="builtin.module(func.func(canonicalize))" -o %t --mlir-disable-threading | FileCheck %s --check-prefix=CHECK-SECOND
+// RUN: mlir-opt %s --log-mlir-actions-filter=%s:8 --log-actions-to=- -pass-pipeline="builtin.module(func.func(test-stats-pass))" -o %t --mlir-disable-threading | FileCheck %s --check-prefix=CHECK-SECOND
 
 // CHECK-SECOND-NOT: @a
 // CHECK-SECOND-NOT: @c
-// CHECK-SECOND: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `Canonicalizer` on Operation `func.func` (func.func @b() {...}
+// CHECK-SECOND: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestStatisticPass` on Operation `func.func` (func.func @b() {...}
 // CHECK-SECOND-NEXT: [thread {{.*}}] completed `pass-execution`
 
 // Filter the first and third functions
-// RUN: mlir-opt %s --log-mlir-actions-filter=%s:4,%s:12 --log-actions-to=- -pass-pipeline="builtin.module(func.func(canonicalize))" -o %t --mlir-disable-threading | FileCheck %s  --check-prefix=CHECK-FIRST-THIRD
+// RUN: mlir-opt %s --log-mlir-actions-filter=%s:4,%s:12 --log-actions-to=- -pass-pipeline="builtin.module(func.func(test-stats-pass))" -o %t --mlir-disable-threading | FileCheck %s  --check-prefix=CHECK-FIRST-THIRD
 
 // CHECK-FIRST-THIRD-NOT: Canonicalizer
-// CHECK-FIRST-THIRD: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `Canonicalizer` on Operation `func.func` (func.func @a() {...}
+// CHECK-FIRST-THIRD: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestStatisticPass` on Operation `func.func` (func.func @a() {...}
 // CHECK-FIRST-THIRD-NEXT: [thread {{.*}}] completed `pass-execution`
-// CHECK-FIRST-THIRD-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `Canonicalizer` on Operation `func.func` (func.func @c() {...}
+// CHECK-FIRST-THIRD-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestStatisticPass` on Operation `func.func` (func.func @c() {...}
 // CHECK-FIRST-THIRD-NEXT: [thread {{.*}}] completed `pass-execution`
 // CHECK-FIRST-THIRD-NOT: Canonicalizer
diff --git a/mlir/test/Pass/action-logging.mlir b/mlir/test/Pass/action-logging.mlir
index 5ceb7ebae96bc..158edd7c4862c 100644
--- a/mlir/test/Pass/action-logging.mlir
+++ b/mlir/test/Pass/action-logging.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s --log-actions-to=- -canonicalize -test-module-pass | FileCheck %s
+// RUN: mlir-opt %s --log-actions-to=- -test-stats-pass -test-module-pass | FileCheck %s
 
-// CHECK: [thread {{.*}}] begins (no breakpoint) Action `pass-execution` running `Canonicalizer` on Operation `builtin.module` (module {...}
+// CHECK: [thread {{.*}}] begins (no breakpoint) Action `pass-execution` running `{{.*}}TestStatisticPass` on Operation `builtin.module` (module {...})`
 // CHECK-NEXT: [thread {{.*}}] completed `pass-execution`
 // CHECK-NEXT: [thread {{.*}}] begins (no breakpoint) Action `pass-execution`  running `{{.*}}TestModulePass` on Operation `builtin.module` (module {...}
 // CHECK-NEXT: [thread {{.*}}] completed `pass-execution`
diff --git a/mlir/test/Pass/run-reproducer.mlir b/mlir/test/Pass/run-reproducer.mlir
index 903fd69458efd..57a58dbaa5b96 100644
--- a/mlir/test/Pass/run-reproducer.mlir
+++ b/mlir/test/Pass/run-reproducer.mlir
@@ -1,5 +1,5 @@
-// RUN: mlir-opt %s -dump-pass-pipeline 2>&1 | FileCheck %s
-// RUN: mlir-opt %s -mlir-print-ir-before=cse 2>&1 | FileCheck -check-prefix=BEFORE %s
+// RUN: mlir-opt %s --run-reproducer -dump-pass-pipeline 2>&1 | FileCheck %s
+// RUN: mlir-opt %s --run-reproducer -mlir-print-ir-before=cse 2>&1 | FileCheck -check-prefix=BEFORE %s
 
 func.func @foo() {
   %0 = arith.constant 0 : i32
diff --git a/mlir/test/Target/LLVMIR/Import/basic.ll b/mlir/test/Target/LLVMIR/Import/basic.ll
index d3eebc0feab8f..19e4543f8010d 100644
--- a/mlir/test/Target/LLVMIR/Import/basic.ll
+++ b/mlir/test/Target/LLVMIR/Import/basic.ll
@@ -53,111 +53,6 @@ if.end:
 }
 ; CHECK-DBG: } loc(#[[MODULELOC]])
 
-
-@_ZTIi = external dso_local constant ptr
-@_ZTIii= external dso_local constant ptr
-declare void @foo(ptr)
-declare ptr @bar(ptr)
-declare i32 @__gxx_personality_v0(...)
-
-; CHECK-LABEL: @invokeLandingpad
-define i32 @invokeLandingpad() personality ptr @__gxx_personality_v0 {
-  ; CHECK: %[[a1:[0-9]+]] = llvm.mlir.addressof @_ZTIii : !llvm.ptr
-  ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x i8 {alignment = 1 : i64} : (i32) -> !llvm.ptr
-  %1 = alloca i8
-  ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> ()
-  invoke void @foo(ptr %1) to label %4 unwind label %2
-
-; CHECK: ^bb1:
-  ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i1>) : !llvm.struct<(ptr, i32)>
-  %3 = landingpad { ptr, i32 } catch ptr @_ZTIi catch ptr @_ZTIii
-          filter [1 x i1] [i1 1]
-  resume { ptr, i32 } %3
-
-; CHECK: ^bb2:
-  ; CHECK: llvm.return %{{[0-9]+}} : i32
-  ret i32 1
-
-; CHECK: ^bb3:
-  ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> !llvm.ptr
-  %6 = invoke ptr @bar(ptr %1) to label %4 unwind label %2
-
-; CHECK: ^bb4:
-  ; CHECK: llvm.return %{{[0-9]+}} : i32
-  ret i32 0
-}
-
-declare i32 @foo2()
-
-; CHECK-LABEL: @invokePhi
-; CHECK-SAME:            (%[[cond:.*]]: i1) -> i32
-define i32 @invokePhi(i1 %cond) personality ptr @__gxx_personality_v0 {
-entry:
-  ; CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : i32
-  ; CHECK: llvm.cond_br %[[cond]], ^[[bb1:.*]], ^[[bb2:.*]]
-  br i1 %cond, label %call, label %nocall
-; CHECK: ^[[bb1]]:
-call:
-  ; CHECK: %[[invoke:.*]] = llvm.invoke @foo2() to ^[[bb3:.*]] unwind ^[[bb5:.*]] : () -> i32
-  %invoke = invoke i32 @foo2() to label %bb0 unwind label %bb1
-; CHECK: ^[[bb2]]:
-nocall:
-  ; CHECK: llvm.br ^[[bb4:.*]](%[[c0]] : i32)
-  br label %bb0
-; CHECK: ^[[bb3]]:
-  ; CHECK: llvm.br ^[[bb4]](%[[invoke]] : i32)
-; CHECK: ^[[bb4]](%[[barg:.*]]: i32):
-bb0:
-  %ret = phi i32 [ 0, %nocall ], [ %invoke, %call ]
-  ; CHECK: llvm.return %[[barg]] : i32
-  ret i32 %ret
-; CHECK: ^[[bb5]]:
-bb1:
-  ; CHECK: %[[lp:.*]] = llvm.landingpad cleanup : i32
-  %resume = landingpad i32 cleanup
-  ; CHECK: llvm.resume %[[lp]] : i32
-  resume i32 %resume
-}
-
-; CHECK-LABEL: @invokePhiComplex
-; CHECK-SAME:                   (%[[cond:.*]]: i1) -> i32
-define i32 @invokePhiComplex(i1 %cond) personality ptr @__gxx_personality_v0 {
-entry:
-  ; CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : i32
-  ; CHECK: %[[c1:.*]] = llvm.mlir.constant(1 : i32) : i32
-  ; CHECK: %[[c2:.*]] = llvm.mlir.constant(2 : i32) : i32
-  ; CHECK: %[[c20:.*]] = llvm.mlir.constant(20 : i32) : i32  
-  ; CHECK: llvm.cond_br %[[cond]], ^[[bb1:.*]], ^[[bb2:.*]]
-  br i1 %cond, label %call, label %nocall
-; CHECK: ^[[bb1]]:
-call:
-  ; CHECK: %[[invoke:.*]] = llvm.invoke @foo2() to ^[[bb3:.*]] unwind ^[[bb5:.*]] : () -> i32
-  %invoke = invoke i32 @foo2() to label %bb0 unwind label %bb1
-; CHECK: ^[[bb2]]:
-nocall:
-  ; CHECK: llvm.br ^[[bb4:.*]](%[[c0]], %[[c1]], %[[c2]] : i32, i32, i32)
-  br label %bb0
-; CHECK: ^[[bb3]]:
-  ; CHECK: llvm.br ^[[bb4]](%[[invoke]], %[[c20]], %[[invoke]] : i32, i32, i32)
-; CHECK: ^[[bb4]](%[[barg0:.*]]: i32, %[[barg1:.*]]: i32, %[[barg2:.*]]: i32):
-bb0:
-  %a = phi i32 [ 0, %nocall ], [ %invoke, %call ]
-  %b = phi i32 [ 1, %nocall ], [ 20, %call ]
-  %c = phi i32 [ 2, %nocall ], [ %invoke, %call ]
-  ; CHECK: %[[add0:.*]] = llvm.add %[[barg0]], %[[barg1]] : i32
-  ; CHECK: %[[add1:.*]] = llvm.add %[[barg2]], %[[add0]] : i32
-  %d = add i32 %a, %b
-  %e = add i32 %c, %d
-  ; CHECK: llvm.return %[[add1]] : i32
-  ret i32 %e
-; CHECK: ^[[bb5]]:
-bb1:
-  ; CHECK: %[[lp:.*]] = llvm.landingpad cleanup : i32
-  %resume = landingpad i32 cleanup
-  ; CHECK: llvm.resume %[[lp]] : i32
-  resume i32 %resume
-}
-
 ; CHECK-LABEL: @hasGCFunction
 ; CHECK-SAME: garbageCollector = "statepoint-example"
 define void @hasGCFunction() gc "statepoint-example" {
diff --git a/mlir/test/Target/LLVMIR/Import/exception.ll b/mlir/test/Target/LLVMIR/Import/exception.ll
new file mode 100644
index 0000000000000..944e5de6badd9
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/exception.ll
@@ -0,0 +1,140 @@
+; RUN: mlir-translate -import-llvm %s | FileCheck %s
+
+@_ZTIi = external dso_local constant ptr
+@_ZTIii= external dso_local constant ptr
+declare void @foo(ptr)
+declare ptr @bar(ptr)
+declare i32 @__gxx_personality_v0(...)
+
+; CHECK-LABEL: @invokeLandingpad
+define i32 @invokeLandingpad() personality ptr @__gxx_personality_v0 {
+  ; CHECK: %[[a1:[0-9]+]] = llvm.mlir.addressof @_ZTIii : !llvm.ptr
+  ; CHECK: %[[a3:[0-9]+]] = llvm.alloca %{{[0-9]+}} x i8 {alignment = 1 : i64} : (i32) -> !llvm.ptr
+  %1 = alloca i8
+  ; CHECK: llvm.invoke @foo(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> ()
+  invoke void @foo(ptr %1) to label %4 unwind label %2
+
+; CHECK: ^bb1:
+  ; CHECK: %{{[0-9]+}} = llvm.landingpad (catch %{{[0-9]+}} : !llvm.ptr) (catch %[[a1]] : !llvm.ptr) (filter %{{[0-9]+}} : !llvm.array<1 x i1>) : !llvm.struct<(ptr, i32)>
+  %3 = landingpad { ptr, i32 } catch ptr @_ZTIi catch ptr @_ZTIii
+          filter [1 x i1] [i1 1]
+  resume { ptr, i32 } %3
+
+; CHECK: ^bb2:
+  ; CHECK: llvm.return %{{[0-9]+}} : i32
+  ret i32 1
+
+; CHECK: ^bb3:
+  ; CHECK: %{{[0-9]+}} = llvm.invoke @bar(%[[a3]]) to ^bb2 unwind ^bb1 : (!llvm.ptr) -> !llvm.ptr
+  %6 = invoke ptr @bar(ptr %1) to label %4 unwind label %2
+
+; CHECK: ^bb4:
+  ; CHECK: llvm.return %{{[0-9]+}} : i32
+  ret i32 0
+}
+
+declare i32 @foo2()
+
+; CHECK-LABEL: @invokePhi
+; CHECK-SAME:            (%[[cond:.*]]: i1) -> i32
+define i32 @invokePhi(i1 %cond) personality ptr @__gxx_personality_v0 {
+entry:
+  ; CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : i32
+  ; CHECK: llvm.cond_br %[[cond]], ^[[bb1:.*]], ^[[bb2:.*]]
+  br i1 %cond, label %call, label %nocall
+; CHECK: ^[[bb1]]:
+call:
+  ; CHECK: %[[invoke:.*]] = llvm.invoke @foo2() to ^[[bb3:.*]] unwind ^[[bb5:.*]] : () -> i32
+  %invoke = invoke i32 @foo2() to label %bb0 unwind label %bb1
+; CHECK: ^[[bb2]]:
+nocall:
+  ; CHECK: llvm.br ^[[bb4:.*]](%[[c0]] : i32)
+  br label %bb0
+; CHECK: ^[[bb3]]:
+  ; CHECK: llvm.br ^[[bb4]](%[[invoke]] : i32)
+; CHECK: ^[[bb4]](%[[barg:.*]]: i32):
+bb0:
+  %ret = phi i32 [ 0, %nocall ], [ %invoke, %call ]
+  ; CHECK: llvm.return %[[barg]] : i32
+  ret i32 %ret
+; CHECK: ^[[bb5]]:
+bb1:
+  ; CHECK: %[[lp:.*]] = llvm.landingpad cleanup : i32
+  %resume = landingpad i32 cleanup
+  ; CHECK: llvm.resume %[[lp]] : i32
+  resume i32 %resume
+}
+
+; CHECK-LABEL: @invokePhiComplex
+; CHECK-SAME:                   (%[[cond:.*]]: i1) -> i32
+define i32 @invokePhiComplex(i1 %cond) personality ptr @__gxx_personality_v0 {
+entry:
+  ; CHECK: %[[c0:.*]] = llvm.mlir.constant(0 : i32) : i32
+  ; CHECK: %[[c1:.*]] = llvm.mlir.constant(1 : i32) : i32
+  ; CHECK: %[[c2:.*]] = llvm.mlir.constant(2 : i32) : i32
+  ; CHECK: %[[c20:.*]] = llvm.mlir.constant(20 : i32) : i32  
+  ; CHECK: llvm.cond_br %[[cond]], ^[[bb1:.*]], ^[[bb2:.*]]
+  br i1 %cond, label %call, label %nocall
+; CHECK: ^[[bb1]]:
+call:
+  ; CHECK: %[[invoke:.*]] = llvm.invoke @foo2() to ^[[bb3:.*]] unwind ^[[bb5:.*]] : () -> i32
+  %invoke = invoke i32 @foo2() to label %bb0 unwind label %bb1
+; CHECK: ^[[bb2]]:
+nocall:
+  ; CHECK: llvm.br ^[[bb4:.*]](%[[c0]], %[[c1]], %[[c2]] : i32, i32, i32)
+  br label %bb0
+; CHECK: ^[[bb3]]:
+  ; CHECK: llvm.br ^[[bb4]](%[[invoke]], %[[c20]], %[[invoke]] : i32, i32, i32)
+; CHECK: ^[[bb4]](%[[barg0:.*]]: i32, %[[barg1:.*]]: i32, %[[barg2:.*]]: i32):
+bb0:
+  %a = phi i32 [ 0, %nocall ], [ %invoke, %call ]
+  %b = phi i32 [ 1, %nocall ], [ 20, %call ]
+  %c = phi i32 [ 2, %nocall ], [ %invoke, %call ]
+  ; CHECK: %[[add0:.*]] = llvm.add %[[barg0]], %[[barg1]] : i32
+  ; CHECK: %[[add1:.*]] = llvm.add %[[barg2]], %[[add0]] : i32
+  %d = add i32 %a, %b
+  %e = add i32 %c, %d
+  ; CHECK: llvm.return %[[add1]] : i32
+  ret i32 %e
+; CHECK: ^[[bb5]]:
+bb1:
+  ; CHECK: %[[lp:.*]] = llvm.landingpad cleanup : i32
+  %resume = landingpad i32 cleanup
+  ; CHECK: llvm.resume %[[lp]] : i32
+  resume i32 %resume
+}
+
+declare void @f0(ptr)
+declare void @f1(i32)
+declare void @f2({ptr, i32})
+
+; CHECK-LABEL: @landingpad_dominance
+define void @landingpad_dominance() personality ptr @__gxx_personality_v0 {
+entry:
+  ; CHECK:    %[[null:.*]] = llvm.mlir.null : !llvm.ptr
+  ; CHECK:    %[[c1:.*]] = llvm.mlir.constant(0 : i32) : i32
+  ; CHECK:    %[[undef:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, i32)>
+  ; CHECK:    %[[tmpstruct:.*]] = llvm.insertvalue %[[null]], %[[undef]][0] : !llvm.struct<(ptr, i32)>
+  ; CHECK:    %[[struct:.*]] = llvm.insertvalue %[[c1]], %[[tmpstruct]][1] : !llvm.struct<(ptr, i32)>
+  ; CHECK:    llvm.call @f0(%[[null]]) : (!llvm.ptr) -> ()
+  call void @f0(ptr null)
+  ; CHECK:    llvm.call @f1(%[[c1]]) : (i32) -> ()
+  call void @f1(i32 0)
+  ; CHECK:    llvm.invoke @f0(%[[null]]) to ^[[block2:.*]] unwind ^[[block1:.*]] : (!llvm.ptr) -> ()
+  invoke void @f0(ptr null)
+      to label %exit unwind label %catch
+
+; CHECK:  ^[[block1]]:
+catch:
+  ; CHECK:    %[[lp:.*]] = llvm.landingpad (catch %[[null]] : !llvm.ptr) : !llvm.struct<(ptr, i32)>
+  %lp = landingpad { ptr, i32 } catch ptr null
+  ; CHECK:    llvm.call @f2(%[[struct]]) : (!llvm.struct<(ptr, i32)>) -> ()
+  call void @f2({ptr, i32} {ptr null, i32 0})
+  ; CHECK:    llvm.resume %[[lp]] : !llvm.struct<(ptr, i32)>
+  resume {ptr, i32} %lp
+
+; CHECK:  ^[[block2]]:
+exit:
+  ; CHECK:    llvm.return
+  ret void
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 7fc1f39148e79..3c22a603da574 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -355,6 +355,17 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float>
   ret void
 }
 
+; CHECK-LABEL:  llvm.func @trap_intrinsics
+define void @trap_intrinsics() {
+  ; CHECK: "llvm.intr.trap"() : () -> ()
+  call void @llvm.trap()
+  ; CHECK: "llvm.intr.debugtrap"() : () -> ()
+  call void @llvm.debugtrap()
+  ; CHECK: "llvm.intr.ubsantrap"() {failureKind = 1 : i8} : () -> ()
+  call void @llvm.ubsantrap(i8 1)
+  ret void
+}
+
 ; CHECK-LABEL:  llvm.func @memcpy_test
 define void @memcpy_test(i32 %0, ptr %1, ptr %2) {
   ; CHECK: %[[FALSE:.+]] = llvm.mlir.constant(false) : i1
@@ -743,6 +754,9 @@ declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i
 declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>)
 declare <7 x float> @llvm.masked.expandload.v7f32(ptr, <7 x i1>, <7 x float>)
 declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr, <7 x i1>)
+declare void @llvm.trap()
+declare void @llvm.debugtrap()
+declare void @llvm.ubsantrap(i8 immarg)
 declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
 declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg)
 declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg)
diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll
index 859f7377a3eb1..dcd2004040abb 100644
--- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll
+++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll
@@ -343,3 +343,36 @@ end:
 !1 = distinct !{!1, !2}
 !2 = !{!"llvm.loop.parallel_accesses", !0, !3}
 !3 = distinct !{}
+
+; // -----
+
+; CHECK: #[[start_loc:.*]] = loc("metadata-loop.ll":1:2)
+; CHECK: #[[end_loc:.*]] = loc("metadata-loop.ll":2:2)
+; CHECK: #[[SUBPROGRAM:.*]] = #llvm.di_subprogram<
+; CHECK: #[[start_loc_fused:.*]] = loc(fused<#[[SUBPROGRAM]]>[#[[start_loc]]])
+; CHECK: #[[end_loc_fused:.*]] = loc(fused<#[[SUBPROGRAM]]>[#[[end_loc]]])
+; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<
+; CHECK-SAME: mustProgress = true
+; CHECK-SAME: startLoc = #[[start_loc_fused]]
+; CHECK-SAME: endLoc = #[[end_loc_fused]]
+
+; CHECK-LABEL: @loop_locs
+define void @loop_locs(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
+  br label %end, !llvm.loop !6
+end:
+  ret void
+}
+
+!llvm.dbg.cu = !{!1}
+!llvm.module.flags = !{!0}
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2)
+!2 = !DIFile(filename: "metadata-loop.ll", directory: "/")
+!3 = distinct !DISubprogram(name: "loop_locs", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1)
+!4 = !DILocation(line: 1, column: 2, scope: !3)
+!5 = !DILocation(line: 2, column: 2, scope: !3)
+
+!6 = distinct !{!6, !4, !5, !7}
+!7 = !{!"llvm.loop.mustprogress"}
diff --git a/mlir/test/Target/LLVMIR/arm-ssve.mlir b/mlir/test/Target/LLVMIR/arm-ssve.mlir
new file mode 100644
index 0000000000000..91bf3e6daf517
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/arm-ssve.mlir
@@ -0,0 +1,11 @@
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+// Attribute to enable streaming-mode.
+
+// CHECK-LABEL: @streaming_callee
+// CHECK: #[[ATTR:[0-9]*]]
+llvm.func @streaming_callee() attributes {passthrough = ["aarch64_pstate_sm_enabled"]} {
+  llvm.return
+}
+
+// CHECK: attributes #[[ATTR]] = { "aarch64_pstate_sm_enabled" }
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 7f3c26ab33b82..80a52224a8397 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -374,6 +374,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr<f32>, %mask: vector
   llvm.return
 }
 
+// CHECK-LABEL: @trap_intrinsics
+llvm.func @trap_intrinsics() {
+  // CHECK: call void @llvm.trap()
+  "llvm.intr.trap"() : () -> ()
+  // CHECK: call void @llvm.debugtrap()
+  "llvm.intr.debugtrap"() : () -> ()
+  // CHECK: call void @llvm.ubsantrap(i8 1)
+  "llvm.intr.ubsantrap"() {failureKind = 1 : i8} : () -> ()
+  llvm.return
+}
+
 // CHECK-LABEL: @memcpy_test
 llvm.func @memcpy_test(%arg0: i32, %arg2: !llvm.ptr<i8>, %arg3: !llvm.ptr<i8>) {
   %i1 = llvm.mlir.constant(false) : i1
@@ -788,6 +799,9 @@ llvm.func @lifetime(%p: !llvm.ptr) {
 // CHECK-DAG: declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>)
 // CHECK-DAG: declare <7 x float> @llvm.masked.expandload.v7f32(ptr nocapture, <7 x i1>, <7 x float>)
 // CHECK-DAG: declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr nocapture, <7 x i1>)
+// CHECK-DAG: declare void @llvm.trap()
+// CHECK-DAG: declare void @llvm.debugtrap()
+// CHECK-DAG: declare void @llvm.ubsantrap(i8 immarg)
 // CHECK-DAG: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
 // CHECK-DAG: declare void @llvm.memcpy.inline.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64 immarg, i1 immarg)
 // CHECK-DAG: declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32)
diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir
index 1d668dd56a991..f17cc003aa0ae 100644
--- a/mlir/test/Target/LLVMIR/loop-metadata.mlir
+++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir
@@ -288,3 +288,32 @@ llvm.metadata @metadata {
 // CHECK-DAG: ![[PIPELINE_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 true}
 // CHECK-DAG: ![[II_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2}
 // CHECK-DAG: ![[ACCESS_GROUPS_NODE:[0-9]+]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]}
+
+// -----
+
+#di_file = #llvm.di_file<"metadata-loop.ll" in "/">
+
+#loc1 = loc("loop-metadata.mlir":42:4)
+#loc2 = loc("loop-metadata.mlir":52:4)
+
+#di_compile_unit = #llvm.di_compile_unit<sourceLanguage = DW_LANG_C, file = #di_file, isOptimized = false, emissionKind = None>
+#di_subprogram = #llvm.di_subprogram<compileUnit = #di_compile_unit, scope = #di_file, name = "loop_locs", file = #di_file, subprogramFlags = Definition>
+
+#start_loc_fused = loc(fused<#di_subprogram>[#loc1])
+#end_loc_fused= loc(fused<#di_subprogram>[#loc2])
+
+#loopMD = #llvm.loop_annotation<disableNonforced = false,
+        startLoc = #start_loc_fused,
+        endLoc = #end_loc_fused>
+
+// CHECK-LABEL: @loop_annotation_with_locs
+llvm.func @loop_annotation_with_locs() {
+// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+  llvm.br ^bb1 {loop_annotation = #loopMD}
+^bb1:
+  llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[START_LOC:.*]], ![[END_LOC:.*]]}
+// CHECK: ![[START_LOC]] = !DILocation(line: 42, column: 4, scope:
+// CHECK: ![[END_LOC]] = !DILocation(line: 52, column: 4, scope:
diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
index 7d29ccb1e0da7..6d6aa11875fb5 100644
--- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
@@ -18,26 +18,22 @@ llvm.func @_QPopenmp_target_data() {
 // CHECK:         %[[VAL_2:.*]] = alloca [1 x i64], align 8
 // CHECK:         %[[VAL_3:.*]] = alloca i32, i64 1, align 4
 // CHECK:         br label %[[VAL_4:.*]]
-// CHECK:       entry:                                            ; preds = %[[VAL_5:.*]]
-// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:       [[VAL_4]]:
+// CHECK:         %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         store ptr %[[VAL_3]], ptr %[[VAL_5]], align 8
+// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
 // CHECK:         store ptr %[[VAL_3]], ptr %[[VAL_6]], align 8
-// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_3]], ptr %[[VAL_7]], align 8
-// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_8]], align 4
-// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_11:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr %[[VAL_11]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
-// CHECK:         br label %[[VAL_12:.*]]
-// CHECK:       omp.data.region:                                  ; preds = %[[VAL_4]]
+// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
+// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_7]], align 4
+// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_8]], ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
 // CHECK:         store i32 99, ptr %[[VAL_3]], align 4
-// CHECK:         br label %[[VAL_13:.*]]
-// CHECK:       omp.region.cont:                                  ; preds = %[[VAL_12]]
-// CHECK:         %[[VAL_14:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_15:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_16:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_14]], ptr %[[VAL_15]], ptr %[[VAL_16]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         %[[VAL_11:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         %[[VAL_13:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_11]], ptr %[[VAL_12]], ptr %[[VAL_13]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
 // CHECK:         ret void
 
 // -----
@@ -62,27 +58,23 @@ llvm.func @_QPopenmp_target_data_region(%1 : !llvm.ptr<array<1024 x i32>>) {
 // CHECK:         %[[VAL_1:.*]] = alloca [1 x ptr], align 8
 // CHECK:         %[[VAL_2:.*]] = alloca [1 x i64], align 8
 // CHECK:         br label %[[VAL_3:.*]]
-// CHECK:       entry:                                            ; preds = %[[VAL_4:.*]]
-// CHECK:         %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_6:.*]], ptr %[[VAL_5]], align 8
-// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_6]], ptr %[[VAL_7]], align 8
-// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_8]], align 4
-// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_11:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr %[[VAL_11]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
-// CHECK:         br label %[[VAL_12:.*]]
-// CHECK:       omp.data.region:                                  ; preds = %[[VAL_3]]
-// CHECK:         %[[VAL_13:.*]] = getelementptr [1024 x i32], ptr %[[VAL_6]], i32 0, i64 0
-// CHECK:         store i32 99, ptr %[[VAL_13]], align 4
-// CHECK:         br label %[[VAL_14:.*]]
-// CHECK:       omp.region.cont:                                  ; preds = %[[VAL_12]]
-// CHECK:         %[[VAL_15:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_16:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_17:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_15]], ptr %[[VAL_16]], ptr %[[VAL_17]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:       [[VAL_3]]:
+// CHECK:         %[[VAL_4:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         store ptr %[[VAL_5:.*]], ptr %[[VAL_4]], align 8
+// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         store ptr %[[VAL_5]], ptr %[[VAL_6]], align 8
+// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
+// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_7]], align 4
+// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_8]], ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[VAL_5]], i32 0, i64 0
+// CHECK:         store i32 99, ptr %[[VAL_11]], align 4
+// CHECK:         %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_13:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         %[[VAL_14:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_12]], ptr %[[VAL_13]], ptr %[[VAL_14]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
 // CHECK:         ret void
 
 // -----
@@ -174,3 +166,4 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr<array<1024 x i32>>, %3 : !llv
 // CHECK:         ret void
 
 // -----
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-region-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-region-llvm.mlir
new file mode 100644
index 0000000000000..68b2f4994eff7
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-region-llvm.mlir
@@ -0,0 +1,32 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_device = #omp.isdevice<is_device = false>} {
+  llvm.func @omp_target_region_() {
+    %0 = llvm.mlir.constant(20 : i32) : i32
+    %1 = llvm.mlir.constant(10 : i32) : i32
+    %2 = llvm.mlir.constant(1 : i64) : i64
+    %3 = llvm.alloca %2 x i32 {bindc_name = "a", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_regionEa"} : (i64) -> !llvm.ptr<i32>
+    %4 = llvm.mlir.constant(1 : i64) : i64
+    %5 = llvm.alloca %4 x i32 {bindc_name = "b", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_regionEb"} : (i64) -> !llvm.ptr<i32>
+    %6 = llvm.mlir.constant(1 : i64) : i64
+    %7 = llvm.alloca %6 x i32 {bindc_name = "c", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_regionEc"} : (i64) -> !llvm.ptr<i32>
+    llvm.store %1, %3 : !llvm.ptr<i32>
+    llvm.store %0, %5 : !llvm.ptr<i32>
+    omp.target   {
+      %8 = llvm.load %3 : !llvm.ptr<i32>
+      %9 = llvm.load %5 : !llvm.ptr<i32>
+      %10 = llvm.add %8, %9  : i32
+      llvm.store %10, %7 : !llvm.ptr<i32>
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: call void @__omp_offloading_[[DEV:.*]]_[[FIL:.*]]_omp_target_region__l[[LINE:.*]](ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}})
+// CHECK: define internal void @__omp_offloading_[[DEV]]_[[FIL]]_omp_target_region__l[[LINE]](ptr %[[ADDR_A:.*]], ptr %[[ADDR_B:.*]], ptr %[[ADDR_C:.*]])
+// CHECK: %[[VAL_A:.*]] = load i32, ptr %[[ADDR_A]], align 4
+// CHECK: %[[VAL_B:.*]] = load i32, ptr %[[ADDR_B]], align 4
+// CHECK: %[[SUM:.*]] = add i32 %[[VAL_A]], %[[VAL_B]]
+// CHECK: store i32 %[[SUM]], ptr %[[ADDR_C]], align 4
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-region-parallel-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-region-parallel-llvm.mlir
new file mode 100644
index 0000000000000..455edda301c20
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-region-parallel-llvm.mlir
@@ -0,0 +1,52 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_device = #omp.isdevice<is_device = false>} {
+  llvm.func @omp_target_region_() {
+    %0 = llvm.mlir.constant(20 : i32) : i32
+    %1 = llvm.mlir.constant(10 : i32) : i32
+    %2 = llvm.mlir.constant(1 : i64) : i64
+    %3 = llvm.alloca %2 x i32 {bindc_name = "a", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_regionEa"} : (i64) -> !llvm.ptr<i32>
+    %4 = llvm.mlir.constant(1 : i64) : i64
+    %5 = llvm.alloca %4 x i32 {bindc_name = "b", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_regionEb"} : (i64) -> !llvm.ptr<i32>
+    %6 = llvm.mlir.constant(1 : i64) : i64
+    %7 = llvm.alloca %6 x i32 {bindc_name = "c", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_regionEc"} : (i64) -> !llvm.ptr<i32>
+    llvm.store %1, %3 : !llvm.ptr<i32>
+    llvm.store %0, %5 : !llvm.ptr<i32>
+    omp.target {
+      omp.parallel {
+        %8 = llvm.load %3 : !llvm.ptr<i32>
+        %9 = llvm.load %5 : !llvm.ptr<i32>
+        %10 = llvm.add %8, %9  : i32
+        llvm.store %10, %7 : !llvm.ptr<i32>
+        omp.terminator
+        }
+      omp.terminator
+    }
+    llvm.return
+  }
+}
+
+// CHECK: call void @__omp_offloading_[[DEV:.*]]_[[FIL:.*]]_omp_target_region__l[[LINE:.*]](ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}})
+// CHECK: define internal void @__omp_offloading_[[DEV]]_[[FIL]]_omp_target_region__l[[LINE]](ptr %[[ADDR_A:.*]], ptr %[[ADDR_B:.*]], ptr %[[ADDR_C:.*]])
+// CHECK:  %[[STRUCTARG:.*]] = alloca { ptr, ptr, ptr }, align 8
+// CHECK: %[[GEP1:.*]] = getelementptr { ptr, ptr, ptr }, ptr %[[STRUCTARG]], i32 0, i32 0
+// CHECK: store ptr %[[ADDR_A]], ptr %[[GEP1]], align 8
+// CHECK: %[[GEP2:.*]] = getelementptr { ptr, ptr, ptr }, ptr %[[STRUCTARG]], i32 0, i32 1
+// CHECK: store ptr %[[ADDR_B]], ptr %[[GEP2]], align 8
+// CHECK: %[[GEP3:.*]] = getelementptr { ptr, ptr, ptr }, ptr %[[STRUCTARG]], i32 0, i32 2
+// CHECK: store ptr %[[ADDR_C]], ptr %[[GEP3]], align 8
+// CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @1, i32 1, ptr @__omp_offloading_[[DEV]]_[[FIL]]_omp_target_region__l[[LINE]]..omp_par, ptr %[[STRUCTARG]])
+
+
+// CHECK: define internal void @__omp_offloading_[[DEV]]_[[FIL]]_omp_target_region__l[[LINE]]..omp_par(ptr noalias %tid.addr, ptr noalias %zero.addr, ptr %[[STRUCTARG2:.*]]) #0 {
+// CHECK: %[[GEP4:.*]] = getelementptr { ptr, ptr, ptr }, ptr %[[STRUCTARG2]], i32 0, i32 0
+// CHECK: %[[LOADGEP1:.*]] = load ptr, ptr %[[GEP4]], align 8
+// CHECK: %[[GEP5:.*]] = getelementptr { ptr, ptr, ptr }, ptr %0, i32 0, i32 1
+// CHECK: %[[LOADGEP2:.*]] = load ptr, ptr %[[GEP5]], align 8
+// CHECK: %[[GEP6:.*]] = getelementptr { ptr, ptr, ptr }, ptr %0, i32 0, i32 2
+// CHECK: %[[LOADGEP3:.*]] = load ptr, ptr %[[GEP6]], align 8
+
+// CHECK: %[[VAL_A:.*]] = load i32, ptr %[[LOADGEP1]], align 4
+// CHECK: %[[VAL_B:.*]] = load i32, ptr %[[LOADGEP2]], align 4
+// CHECK: %[[SUM:.*]] = add i32 %[[VAL_A]], %[[VAL_B]]
+// CHECK: store i32 %[[SUM]], ptr %[[LOADGEP3]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
index 4be992376eb76..0e11d677aa590 100644
--- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir
@@ -2457,17 +2457,6 @@ llvm.func @omp_opaque_pointers(%arg0 : !llvm.ptr, %arg1: !llvm.ptr, %expr: i32)
 
 // -----
 
-// CHECK-NOT: @__omp_rtl_debug_kind = weak_odr hidden constant i32 1
-// CHECK-NOT: @__omp_rtl_assume_teams_oversubscription = weak_odr hidden constant i32 1
-// CHECK-NOT: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 1
-// CHECK-NOT: @__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 1
-// CHECK-NOT: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 1
-module attributes {omp.flags = #omp.flags<debug_kind = 1, assume_teams_oversubscription = true, 
-                                          assume_threads_oversubscription = true, assume_no_thread_state = true, 
-                                          assume_no_nested_parallelism = true>} {}
-
-// -----
-
 // CHECK: @__omp_rtl_debug_kind = weak_odr hidden constant i32 1
 // CHECK: @__omp_rtl_assume_teams_oversubscription = weak_odr hidden constant i32 1
 // CHECK: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 1
@@ -2475,18 +2464,7 @@ module attributes {omp.flags = #omp.flags<debug_kind = 1, assume_teams_oversubsc
 // CHECK: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 1
 module attributes {omp.flags = #omp.flags<debug_kind = 1, assume_teams_oversubscription = true, 
                                           assume_threads_oversubscription = true, assume_no_thread_state = true, 
-                                          assume_no_nested_parallelism = true>, 
-                   omp.is_device = #omp.isdevice<is_device = true>} {}
-
-// -----
-
-// CHECK-NOT: @__omp_rtl_debug_kind = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_teams_oversubscription = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
-module attributes {omp.flags = #omp.flags<>} {}
-
+                                          assume_no_nested_parallelism = true>} {}
 // -----
 
 // CHECK: @__omp_rtl_debug_kind = weak_odr hidden constant i32 0
@@ -2494,18 +2472,7 @@ module attributes {omp.flags = #omp.flags<>} {}
 // CHECK: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 0
 // CHECK: @__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 0
 // CHECK: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
-module attributes {omp.flags = #omp.flags<>, omp.is_device = #omp.isdevice<is_device = true>} {}
-
-// -----
-
-// CHECK-NOT: @__omp_rtl_debug_kind = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_teams_oversubscription = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
-module attributes {omp.flags = #omp.flags<debug_kind = 0, assume_teams_oversubscription = false, 
-                                          assume_threads_oversubscription = false, assume_no_thread_state = false, 
-                                          assume_no_nested_parallelism = false>} {}
+module attributes {omp.flags = #omp.flags<>} {}
 
 // -----
 
@@ -2516,17 +2483,7 @@ module attributes {omp.flags = #omp.flags<debug_kind = 0, assume_teams_oversubsc
 // CHECK: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
 module attributes {omp.flags = #omp.flags<debug_kind = 0, assume_teams_oversubscription = false, 
                                           assume_threads_oversubscription = false, assume_no_thread_state = false, 
-                                          assume_no_nested_parallelism = false>, 
-                   omp.is_device = #omp.isdevice<is_device = true>} {}
-
-// -----
-
-// CHECK-NOT: @__omp_rtl_debug_kind = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_teams_oversubscription = weak_odr hidden constant i32 1
-// CHECK-NOT: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 0
-// CHECK-NOT: @__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 1
-// CHECK-NOT: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
-module attributes {omp.flags = #omp.flags<assume_teams_oversubscription = true, assume_no_thread_state = true>} {}
+                                          assume_no_nested_parallelism = false>} {}
 
 // -----
 
@@ -2535,5 +2492,4 @@ module attributes {omp.flags = #omp.flags<assume_teams_oversubscription = true,
 // CHECK: @__omp_rtl_assume_threads_oversubscription = weak_odr hidden constant i32 0
 // CHECK: @__omp_rtl_assume_no_thread_state = weak_odr hidden constant i32 1
 // CHECK: @__omp_rtl_assume_no_nested_parallelism = weak_odr hidden constant i32 0
-module attributes {omp.flags = #omp.flags<assume_teams_oversubscription = true, assume_no_thread_state = true>, 
-                   omp.is_device = #omp.isdevice<is_device = true>} {}
+module attributes {omp.flags = #omp.flags<assume_teams_oversubscription = true, assume_no_thread_state = true>} {}
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 1a65967974e87..e3f942e8bd785 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -294,6 +294,18 @@ llvm.func @rocdl.raw.buffer.atomic.i32(%rsrc : vector<4xi32>,
   llvm.return
 }
 
+llvm.func @rocdl.raw.buffer.atomic.cmpswap(%rsrc : vector<4xi32>,
+                        %offset : i32, %soffset : i32,
+                        %src : i32, %cmp : i32) -> i32 {
+  %aux = llvm.mlir.constant(0 : i32) : i32
+  // CHECK-LABEL: rocdl.raw.buffer.atomic.cmpswap
+  // CHECK: [[val:%.+]] = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %{{.*}}, i32 %{{.*}}, <4 x i32> %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 {{.*}}
+  // CHECK: ret i32 [[val]]
+
+  %val = rocdl.raw.buffer.atomic.cmpswap(%src, %cmp, %rsrc, %offset, %soffset, %aux) : i32, vector<4xi32>
+  llvm.return %val : i32
+}
+
 // CHECK-DAG: attributes #[[$KERNEL_ATTRS]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-implicitarg-num-bytes"="56" }
 // CHECK-DAG: attributes #[[$KERNEL_WORKGROUP_ATTRS]] = { "amdgpu-flat-work-group-size"="1,1024"
 // CHECK-DAG: attributes #[[$KNOWN_BLOCK_SIZE_ATTRS]] = { "amdgpu-flat-work-group-size"="128,128"
diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir
index 5cc0eb539ecf5..47a19bb598c25 100644
--- a/mlir/test/Transforms/canonicalize.mlir
+++ b/mlir/test/Transforms/canonicalize.mlir
@@ -1107,14 +1107,10 @@ func.func @fold_trunci_vector(%arg0: vector<4xi1>) -> vector<4xi1> attributes {}
 
 // -----
 
-// TODO Canonicalize this into:
-//   arith.extui %arg0 : i1 to i2
-
-// CHECK-LABEL: func @do_not_fold_trunci
+// CHECK-LABEL: func @fold_trunci
 // CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: i1)
-func.func @do_not_fold_trunci(%arg0: i1) -> i2 attributes {} {
-  // CHECK-NEXT: arith.extui %[[ARG0]] : i1 to i8
-  // CHECK-NEXT: %[[RES:[0-9a-z]*]] = arith.trunci %{{.*}} : i8 to i2
+func.func @fold_trunci(%arg0: i1) -> i2 attributes {} {
+  // CHECK-NEXT: %[[RES:[0-9a-z]*]] = arith.extui %[[ARG0]] : i1 to i2
   // CHECK-NEXT: return %[[RES]] : i2
   %0 = arith.extui %arg0 : i1 to i8
   %1 = arith.trunci %0 : i8 to i2
@@ -1123,11 +1119,10 @@ func.func @do_not_fold_trunci(%arg0: i1) -> i2 attributes {} {
 
 // -----
 
-// CHECK-LABEL: func @do_not_fold_trunci_vector
+// CHECK-LABEL: func @fold_trunci_vector
 // CHECK-SAME:    (%[[ARG0:[0-9a-z]*]]: vector<4xi1>)
-func.func @do_not_fold_trunci_vector(%arg0: vector<4xi1>) -> vector<4xi2> attributes {} {
-  // CHECK-NEXT: arith.extui %[[ARG0]] : vector<4xi1> to vector<4xi8>
-  // CHECK-NEXT: %[[RES:[0-9a-z]*]] = arith.trunci %{{.*}} : vector<4xi8> to vector<4xi2>
+func.func @fold_trunci_vector(%arg0: vector<4xi1>) -> vector<4xi2> attributes {} {
+  // CHECK-NEXT: %[[RES:[0-9a-z]*]] = arith.extui %[[ARG0]] : vector<4xi1> to vector<4xi2>
   // CHECK-NEXT: return %[[RES]] : vector<4xi2>
   %0 = arith.extui %arg0 : vector<4xi1> to vector<4xi8>
   %1 = arith.trunci %0 : vector<4xi8> to vector<4xi2>
diff --git a/mlir/test/Transforms/decompose-call-graph-types.mlir b/mlir/test/Transforms/decompose-call-graph-types.mlir
index 51b63ba4c0ad9..b8fad63eb4de6 100644
--- a/mlir/test/Transforms/decompose-call-graph-types.mlir
+++ b/mlir/test/Transforms/decompose-call-graph-types.mlir
@@ -10,8 +10,8 @@
 // CHECK-SAME:                   %[[ARG0:.*]]: i1,
 // CHECK-SAME:                   %[[ARG1:.*]]: i32) -> (i1, i32) {
 // CHECK:           %[[ARG_MATERIALIZED:.*]] = "test.make_tuple"(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> tuple<i1, i32>
-// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) {index = 0 : i32} : (tuple<i1, i32>) -> i1
-// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) {index = 1 : i32} : (tuple<i1, i32>) -> i32
+// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
+// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
 // CHECK:           return %[[RET0]], %[[RET1]] : i1, i32
 // CHECK-12N-LABEL:   func @identity(
 // CHECK-12N-SAME:                   %[[ARG0:.*]]: i1,
@@ -61,12 +61,12 @@ func.func @recursive_decomposition(%arg0: tuple<tuple<tuple<i1>>>) -> tuple<tupl
 // CHECK:           %[[V2:.*]] = "test.make_tuple"(%[[ARG1]]) : (i2) -> tuple<i2>
 // CHECK:           %[[V3:.*]] = "test.make_tuple"(%[[V2]]) : (tuple<i2>) -> tuple<tuple<i2>>
 // CHECK:           %[[V4:.*]] = "test.make_tuple"(%[[V0]], %[[V1]], %[[V3]]) : (tuple<>, tuple<i1>, tuple<tuple<i2>>) -> tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>
-// CHECK:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) {index = 0 : i32} : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<>
-// CHECK:           %[[V6:.*]] = "test.get_tuple_element"(%[[V4]]) {index = 1 : i32} : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<i1>
-// CHECK:           %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) {index = 0 : i32} : (tuple<i1>) -> i1
-// CHECK:           %[[V8:.*]] = "test.get_tuple_element"(%[[V4]]) {index = 2 : i32} : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
-// CHECK:           %[[V9:.*]] = "test.get_tuple_element"(%[[V8]]) {index = 0 : i32} : (tuple<tuple<i2>>) -> tuple<i2>
-// CHECK:           %[[V10:.*]] = "test.get_tuple_element"(%[[V9]]) {index = 0 : i32} : (tuple<i2>) -> i2
+// CHECK:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 0 : i32}> : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<>
+// CHECK:           %[[V6:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 1 : i32}> : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<i1>
+// CHECK:           %[[V7:.*]] = "test.get_tuple_element"(%[[V6]]) <{index = 0 : i32}> : (tuple<i1>) -> i1
+// CHECK:           %[[V8:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 2 : i32}> : (tuple<tuple<>, tuple<i1>, tuple<tuple<i2>>>) -> tuple<tuple<i2>>
+// CHECK:           %[[V9:.*]] = "test.get_tuple_element"(%[[V8]]) <{index = 0 : i32}> : (tuple<tuple<i2>>) -> tuple<i2>
+// CHECK:           %[[V10:.*]] = "test.get_tuple_element"(%[[V9]]) <{index = 0 : i32}> : (tuple<i2>) -> i2
 // CHECK:           return %[[V7]], %[[V10]] : i1, i2
 // CHECK-12N-LABEL:   func @mixed_recursive_decomposition(
 // CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
@@ -88,12 +88,12 @@ func.func private @callee(tuple<i1, i32>) -> tuple<i1, i32>
 // CHECK-SAME:                 %[[ARG0:.*]]: i1,
 // CHECK-SAME:                 %[[ARG1:.*]]: i32) -> (i1, i32) {
 // CHECK:           %[[ARG_MATERIALIZED:.*]] = "test.make_tuple"(%[[ARG0]], %[[ARG1]]) : (i1, i32) -> tuple<i1, i32>
-// CHECK:           %[[CALL_ARG0:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) {index = 0 : i32} : (tuple<i1, i32>) -> i1
-// CHECK:           %[[CALL_ARG1:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) {index = 1 : i32} : (tuple<i1, i32>) -> i32
+// CHECK:           %[[CALL_ARG0:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
+// CHECK:           %[[CALL_ARG1:.*]] = "test.get_tuple_element"(%[[ARG_MATERIALIZED]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
 // CHECK:           %[[DECOMPOSED:.*]]:2 = call @callee(%[[CALL_ARG0]], %[[CALL_ARG1]]) : (i1, i32) -> (i1, i32)
 // CHECK:           %[[CALL_RESULT_RECOMPOSED:.*]] = "test.make_tuple"(%[[DECOMPOSED]]#0, %[[DECOMPOSED]]#1) : (i1, i32) -> tuple<i1, i32>
-// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[CALL_RESULT_RECOMPOSED]]) {index = 0 : i32} : (tuple<i1, i32>) -> i1
-// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[CALL_RESULT_RECOMPOSED]]) {index = 1 : i32} : (tuple<i1, i32>) -> i32
+// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[CALL_RESULT_RECOMPOSED]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
+// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[CALL_RESULT_RECOMPOSED]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
 // CHECK:           return %[[RET0]], %[[RET1]] : i1, i32
 // CHECK-12N-LABEL:   func @caller(
 // CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
@@ -131,13 +131,13 @@ func.func @caller(%arg0: tuple<>) -> tuple<> {
 
 // CHECK-LABEL:   func @unconverted_op_result() -> (i1, i32) {
 // CHECK:           %[[UNCONVERTED_VALUE:.*]] = "test.source"() : () -> tuple<i1, i32>
-// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) {index = 0 : i32} : (tuple<i1, i32>) -> i1
-// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) {index = 1 : i32} : (tuple<i1, i32>) -> i32
+// CHECK:           %[[RET0:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
+// CHECK:           %[[RET1:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
 // CHECK:           return %[[RET0]], %[[RET1]] : i1, i32
 // CHECK-12N-LABEL:   func @unconverted_op_result() -> (i1, i32) {
 // CHECK-12N:           %[[UNCONVERTED_VALUE:.*]] = "test.source"() : () -> tuple<i1, i32>
-// CHECK-12N:           %[[RET0:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) {index = 0 : i32} : (tuple<i1, i32>) -> i1
-// CHECK-12N:           %[[RET1:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) {index = 1 : i32} : (tuple<i1, i32>) -> i32
+// CHECK-12N:           %[[RET0:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 0 : i32}> : (tuple<i1, i32>) -> i1
+// CHECK-12N:           %[[RET1:.*]] = "test.get_tuple_element"(%[[UNCONVERTED_VALUE]]) <{index = 1 : i32}> : (tuple<i1, i32>) -> i32
 // CHECK-12N:           return %[[RET0]], %[[RET1]] : i1, i32
 func.func @unconverted_op_result() -> tuple<i1, i32> {
   %0 = "test.source"() : () -> (tuple<i1, i32>)
@@ -155,9 +155,9 @@ func.func @unconverted_op_result() -> tuple<i1, i32> {
 // CHECK:           %[[V0:.*]] = "test.make_tuple"(%[[ARG1]]) : (i32) -> tuple<i32>
 // CHECK:           %[[V1:.*]] = "test.make_tuple"(%[[ARG0]], %[[V0]]) : (i1, tuple<i32>) -> tuple<i1, tuple<i32>>
 // CHECK:           %[[V2:.*]] = "test.op"(%[[V1]]) : (tuple<i1, tuple<i32>>) -> tuple<i1, tuple<i32>>
-// CHECK:           %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) {index = 0 : i32} : (tuple<i1, tuple<i32>>) -> i1
-// CHECK:           %[[V4:.*]] = "test.get_tuple_element"(%[[V2]]) {index = 1 : i32} : (tuple<i1, tuple<i32>>) -> tuple<i32>
-// CHECK:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) {index = 0 : i32} : (tuple<i32>) -> i32
+// CHECK:           %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 0 : i32}> : (tuple<i1, tuple<i32>>) -> i1
+// CHECK:           %[[V4:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 1 : i32}> : (tuple<i1, tuple<i32>>) -> tuple<i32>
+// CHECK:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 0 : i32}> : (tuple<i32>) -> i32
 // CHECK:           return %[[V3]], %[[V5]] : i1, i32
 // CHECK-12N-LABEL:   func @nested_unconverted_op_result(
 // CHECK-12N-SAME:                 %[[ARG0:.*]]: i1,
@@ -165,9 +165,9 @@ func.func @unconverted_op_result() -> tuple<i1, i32> {
 // CHECK-12N:           %[[V0:.*]] = "test.make_tuple"(%[[ARG1]]) : (i32) -> tuple<i32>
 // CHECK-12N:           %[[V1:.*]] = "test.make_tuple"(%[[ARG0]], %[[V0]]) : (i1, tuple<i32>) -> tuple<i1, tuple<i32>>
 // CHECK-12N:           %[[V2:.*]] = "test.op"(%[[V1]]) : (tuple<i1, tuple<i32>>) -> tuple<i1, tuple<i32>>
-// CHECK-12N:           %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) {index = 0 : i32} : (tuple<i1, tuple<i32>>) -> i1
-// CHECK-12N:           %[[V4:.*]] = "test.get_tuple_element"(%[[V2]]) {index = 1 : i32} : (tuple<i1, tuple<i32>>) -> tuple<i32>
-// CHECK-12N:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) {index = 0 : i32} : (tuple<i32>) -> i32
+// CHECK-12N:           %[[V3:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 0 : i32}> : (tuple<i1, tuple<i32>>) -> i1
+// CHECK-12N:           %[[V4:.*]] = "test.get_tuple_element"(%[[V2]]) <{index = 1 : i32}> : (tuple<i1, tuple<i32>>) -> tuple<i32>
+// CHECK-12N:           %[[V5:.*]] = "test.get_tuple_element"(%[[V4]]) <{index = 0 : i32}> : (tuple<i32>) -> i32
 // CHECK-12N:           return %[[V3]], %[[V5]] : i1, i32
 func.func @nested_unconverted_op_result(%arg: tuple<i1, tuple<i32>>) -> tuple<i1, tuple<i32>> {
   %0 = "test.op"(%arg) : (tuple<i1, tuple<i32>>) -> (tuple<i1, tuple<i32>>)
@@ -191,12 +191,12 @@ func.func private @callee(tuple<>, i1, tuple<i2>, i3, tuple<i4, i5>, i6) -> (tup
 // CHECK-SAME:                 %[[I5:.*]]: i5,
 // CHECK-SAME:                 %[[I6:.*]]: i6) -> (i1, i2, i3, i4, i5, i6) {
 // CHECK:           %[[ARG_TUPLE:.*]] = "test.make_tuple"(%[[I4]], %[[I5]]) : (i4, i5) -> tuple<i4, i5>
-// CHECK:           %[[ARG_TUPLE_0:.*]] = "test.get_tuple_element"(%[[ARG_TUPLE]]) {index = 0 : i32} : (tuple<i4, i5>) -> i4
-// CHECK:           %[[ARG_TUPLE_1:.*]] = "test.get_tuple_element"(%[[ARG_TUPLE]]) {index = 1 : i32} : (tuple<i4, i5>) -> i5
+// CHECK:           %[[ARG_TUPLE_0:.*]] = "test.get_tuple_element"(%[[ARG_TUPLE]]) <{index = 0 : i32}> : (tuple<i4, i5>) -> i4
+// CHECK:           %[[ARG_TUPLE_1:.*]] = "test.get_tuple_element"(%[[ARG_TUPLE]]) <{index = 1 : i32}> : (tuple<i4, i5>) -> i5
 // CHECK:           %[[CALL:.*]]:6 = call @callee(%[[I1]], %[[I2]], %[[I3]], %[[ARG_TUPLE_0]], %[[ARG_TUPLE_1]], %[[I6]]) : (i1, i2, i3, i4, i5, i6) -> (i1, i2, i3, i4, i5, i6)
 // CHECK:           %[[RET_TUPLE:.*]] = "test.make_tuple"(%[[CALL]]#3, %[[CALL]]#4) : (i4, i5) -> tuple<i4, i5>
-// CHECK:           %[[RET_TUPLE_0:.*]] = "test.get_tuple_element"(%[[RET_TUPLE]]) {index = 0 : i32} : (tuple<i4, i5>) -> i4
-// CHECK:           %[[RET_TUPLE_1:.*]] = "test.get_tuple_element"(%[[RET_TUPLE]]) {index = 1 : i32} : (tuple<i4, i5>) -> i5
+// CHECK:           %[[RET_TUPLE_0:.*]] = "test.get_tuple_element"(%[[RET_TUPLE]]) <{index = 0 : i32}> : (tuple<i4, i5>) -> i4
+// CHECK:           %[[RET_TUPLE_1:.*]] = "test.get_tuple_element"(%[[RET_TUPLE]]) <{index = 1 : i32}> : (tuple<i4, i5>) -> i5
 // CHECK:           return %[[CALL]]#0, %[[CALL]]#1, %[[CALL]]#2, %[[RET_TUPLE_0]], %[[RET_TUPLE_1]], %[[CALL]]#5 : i1, i2, i3, i4, i5, i6
 // CHECK-12N-LABEL:   func @caller(
 // CHECK-12N-SAME:                 %[[I1:.*]]: i1,
diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir
index 06bd5b6ae4984..6897b6f95f0d0 100644
--- a/mlir/test/Transforms/test-legalizer.mlir
+++ b/mlir/test/Transforms/test-legalizer.mlir
@@ -2,7 +2,7 @@
 
 // CHECK-LABEL: verifyDirectPattern
 func.func @verifyDirectPattern() -> i32 {
-  // CHECK-NEXT:  "test.legal_op_a"() {status = "Success"}
+  // CHECK-NEXT:  "test.legal_op_a"() <{status = "Success"}
   %result = "test.illegal_op_a"() : () -> (i32)
   // expected-remark@+1 {{op 'func.return' is not legalizable}}
   return %result : i32
@@ -10,7 +10,7 @@ func.func @verifyDirectPattern() -> i32 {
 
 // CHECK-LABEL: verifyLargerBenefit
 func.func @verifyLargerBenefit() -> i32 {
-  // CHECK-NEXT:  "test.legal_op_a"() {status = "Success"}
+  // CHECK-NEXT:  "test.legal_op_a"() <{status = "Success"}
   %result = "test.illegal_op_c"() : () -> (i32)
   // expected-remark@+1 {{op 'func.return' is not legalizable}}
   return %result : i32
diff --git a/mlir/test/Transforms/test-operation-folder.mlir b/mlir/test/Transforms/test-operation-folder.mlir
index 670ec232a3922..46ee07af993cc 100644
--- a/mlir/test/Transforms/test-operation-folder.mlir
+++ b/mlir/test/Transforms/test-operation-folder.mlir
@@ -7,7 +7,7 @@ func.func @foo() -> i32 {
   // The new operation should be present in the output and contain an attribute
   // with value "42" that results from folding.
 
-  // CHECK: "test.op_in_place_fold"(%{{.*}}) {attr = 42 : i32}
+  // CHECK: "test.op_in_place_fold"(%{{.*}}) <{attr = 42 : i32}
   %0 = "test.op_in_place_fold_anchor"(%c42) : (i32) -> (i32)
   return %0 : i32
 }
diff --git a/mlir/test/lib/Dialect/SPIRV/TestModuleCombiner.cpp b/mlir/test/lib/Dialect/SPIRV/TestModuleCombiner.cpp
index 56a2d1d3fdc41..3f5f094ed3afc 100644
--- a/mlir/test/lib/Dialect/SPIRV/TestModuleCombiner.cpp
+++ b/mlir/test/lib/Dialect/SPIRV/TestModuleCombiner.cpp
@@ -34,6 +34,8 @@ class TestModuleCombinerPass
 
 void TestModuleCombinerPass::runOnOperation() {
   auto modules = llvm::to_vector<4>(getOperation().getOps<spirv::ModuleOp>());
+  if (modules.empty())
+    return;
 
   OpBuilder combinedModuleBuilder(modules[0]);
 
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 40de4374c6a35..83226a5f3c245 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -21,6 +21,7 @@
 #include "mlir/IR/ExtensibleDialect.h"
 #include "mlir/IR/FunctionImplementation.h"
 #include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/ODSSupport.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -43,6 +44,36 @@
 using namespace mlir;
 using namespace test;
 
+Attribute MyPropStruct::asAttribute(MLIRContext *ctx) const {
+  return StringAttr::get(ctx, content);
+}
+LogicalResult MyPropStruct::setFromAttr(MyPropStruct &prop, Attribute attr,
+                                        InFlightDiagnostic *diag) {
+  StringAttr strAttr = attr.dyn_cast<StringAttr>();
+  if (!strAttr) {
+    if (diag)
+      *diag << "Expect StringAttr but got " << attr;
+    return failure();
+  }
+  prop.content = strAttr.getValue();
+  return success();
+}
+llvm::hash_code MyPropStruct::hash() const {
+  return hash_value(StringRef(content));
+}
+
+static LogicalResult setPropertiesFromAttribute(PropertiesWithCustomPrint &prop,
+                                                Attribute attr,
+                                                InFlightDiagnostic *diagnostic);
+static DictionaryAttr
+getPropertiesAsAttribute(MLIRContext *ctx,
+                         const PropertiesWithCustomPrint &prop);
+static llvm::hash_code computeHash(const PropertiesWithCustomPrint &prop);
+static void customPrintProperties(OpAsmPrinter &p,
+                                  const PropertiesWithCustomPrint &prop);
+static ParseResult customParseProperties(OpAsmParser &parser,
+                                         PropertiesWithCustomPrint &prop);
+
 void test::registerTestDialect(DialectRegistry &registry) {
   registry.insert<TestDialect>();
 }
@@ -514,7 +545,7 @@ Operation *TestDialect::materializeConstant(OpBuilder &builder, Attribute value,
 ::mlir::LogicalResult FormatInferType2Op::inferReturnTypes(
     ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location,
     ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes,
-    ::mlir::RegionRange regions,
+    OpaqueProperties properties, ::mlir::RegionRange regions,
     ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
   inferredReturnTypes.assign({::mlir::IntegerType::get(context, 16)});
   return ::mlir::success();
@@ -1264,7 +1295,7 @@ LogicalResult TestOpWithVariadicResultsAndFolder::fold(
 }
 
 OpFoldResult TestOpInPlaceFold::fold(FoldAdaptor adaptor) {
-  if (adaptor.getOp() && !(*this)->hasAttr("attr")) {
+  if (adaptor.getOp() && !(*this)->getAttr("attr")) {
     // The folder adds "attr" if not present.
     (*this)->setAttr("attr", adaptor.getOp());
     return getResult();
@@ -1297,7 +1328,7 @@ OpFoldResult TestOpFoldWithFoldAdaptor::fold(FoldAdaptor adaptor) {
 
 LogicalResult OpWithInferTypeInterfaceOp::inferReturnTypes(
     MLIRContext *, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &inferredReturnTypes) {
   if (operands[0].getType() != operands[1].getType()) {
     return emitOptionalError(location, "operand type mismatch ",
@@ -1312,16 +1343,17 @@ LogicalResult OpWithInferTypeInterfaceOp::inferReturnTypes(
 // refineReturnType, currently only refineReturnType can be omitted.
 LogicalResult OpWithRefineTypeInterfaceOp::inferReturnTypes(
     MLIRContext *context, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &returnTypes) {
   returnTypes.clear();
   return OpWithRefineTypeInterfaceOp::refineReturnTypes(
-      context, location, operands, attributes, regions, returnTypes);
+      context, location, operands, attributes, properties, regions,
+      returnTypes);
 }
 
 LogicalResult OpWithRefineTypeInterfaceOp::refineReturnTypes(
     MLIRContext *, std::optional<Location> location, ValueRange operands,
-    DictionaryAttr attributes, RegionRange regions,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<Type> &returnTypes) {
   if (operands[0].getType() != operands[1].getType()) {
     return emitOptionalError(location, "operand type mismatch ",
@@ -1340,7 +1372,8 @@ LogicalResult OpWithRefineTypeInterfaceOp::refineReturnTypes(
 
 LogicalResult OpWithShapedTypeInferTypeInterfaceOp::inferReturnTypeComponents(
     MLIRContext *context, std::optional<Location> location,
-    ValueShapeRange operands, DictionaryAttr attributes, RegionRange regions,
+    ValueShapeRange operands, DictionaryAttr attributes,
+    OpaqueProperties properties, RegionRange regions,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
   // Create return type consisting of the last element of the first operand.
   auto operandType = operands.front().getType();
@@ -1797,6 +1830,59 @@ OpFoldResult ManualCppOpWithFold::fold(ArrayRef<Attribute> attributes) {
   return nullptr;
 }
 
+static LogicalResult
+setPropertiesFromAttribute(PropertiesWithCustomPrint &prop, Attribute attr,
+                           InFlightDiagnostic *diagnostic) {
+  DictionaryAttr dict = dyn_cast<DictionaryAttr>(attr);
+  if (!dict) {
+    if (diagnostic)
+      *diagnostic << "expected DictionaryAttr to set TestProperties";
+    return failure();
+  }
+  auto label = dict.getAs<mlir::StringAttr>("label");
+  if (!label) {
+    if (diagnostic)
+      *diagnostic << "expected StringAttr for key `label`";
+    return failure();
+  }
+  auto valueAttr = dict.getAs<IntegerAttr>("value");
+  if (!valueAttr) {
+    if (diagnostic)
+      *diagnostic << "expected IntegerAttr for key `value`";
+    return failure();
+  }
+
+  prop.label = std::make_shared<std::string>(label.getValue());
+  prop.value = valueAttr.getValue().getSExtValue();
+  return success();
+}
+static DictionaryAttr
+getPropertiesAsAttribute(MLIRContext *ctx,
+                         const PropertiesWithCustomPrint &prop) {
+  SmallVector<NamedAttribute> attrs;
+  Builder b{ctx};
+  attrs.push_back(b.getNamedAttr("label", b.getStringAttr(*prop.label)));
+  attrs.push_back(b.getNamedAttr("value", b.getI32IntegerAttr(prop.value)));
+  return b.getDictionaryAttr(attrs);
+}
+static llvm::hash_code computeHash(const PropertiesWithCustomPrint &prop) {
+  return llvm::hash_combine(prop.value, StringRef(*prop.label));
+}
+static void customPrintProperties(OpAsmPrinter &p,
+                                  const PropertiesWithCustomPrint &prop) {
+  p.printKeywordOrString(*prop.label);
+  p << " is " << prop.value;
+}
+static ParseResult customParseProperties(OpAsmParser &parser,
+                                         PropertiesWithCustomPrint &prop) {
+  std::string label;
+  if (parser.parseKeywordOrString(&label) || parser.parseKeyword("is") ||
+      parser.parseInteger(prop.value))
+    return failure();
+  prop.label = std::make_shared<std::string>(std::move(label));
+  return success();
+}
+
 #include "TestOpEnums.cpp.inc"
 #include "TestOpInterfaces.cpp.inc"
 #include "TestTypeInterfaces.cpp.inc"
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.h b/mlir/test/lib/Dialect/Test/TestDialect.h
index ad3ef2a9f1cdd..dcca76d7dc388 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.h
+++ b/mlir/test/lib/Dialect/Test/TestDialect.h
@@ -42,6 +42,8 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 
+#include <memory>
+
 namespace mlir {
 class DLTIDialect;
 class RewritePatternSet;
@@ -54,6 +56,30 @@ class RewritePatternSet;
 #include "TestOpInterfaces.h.inc"
 #include "TestOpsDialect.h.inc"
 
+namespace test {
+// Define some classes to exercises the Properties feature.
+
+struct PropertiesWithCustomPrint {
+  /// A shared_ptr to a const object is safe: it is equivalent to a value-based
+  /// member. Here the label will be deallocated when the last operation
+  /// refering to it is destroyed. However there is no pool-allocation: this is
+  /// offloaded to the client.
+  std::shared_ptr<const std::string> label;
+  int value;
+};
+class MyPropStruct {
+public:
+  std::string content;
+  // These three methods are invoked through the  `MyStructProperty` wrapper
+  // defined in TestOps.td
+  mlir::Attribute asAttribute(mlir::MLIRContext *ctx) const;
+  static mlir::LogicalResult setFromAttr(MyPropStruct &prop,
+                                         mlir::Attribute attr,
+                                         mlir::InFlightDiagnostic *diag);
+  llvm::hash_code hash() const;
+};
+} // namespace test
+
 #define GET_OP_CLASSES
 #include "TestOps.h.inc"
 
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.td b/mlir/test/lib/Dialect/Test/TestDialect.td
index 9ec12749bfcc3..01116a29367b2 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.td
+++ b/mlir/test/lib/Dialect/Test/TestDialect.td
@@ -24,6 +24,7 @@ def Test_Dialect : Dialect {
   let useDefaultTypePrinterParser = 0;
   let useDefaultAttributePrinterParser = 1;
   let isExtensible = 1;
+  let usePropertiesForAttributes = 1;
   let dependentDialects = ["::mlir::DLTIDialect"];
 
   let extraClassDeclaration = [{
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 887031e6ff434..60faf6dfe0e89 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -430,7 +430,7 @@ def VariadicRegionInferredTypesOp : TEST_Op<"variadic_region_inferred",
   let extraClassDeclaration = [{
     static mlir::LogicalResult inferReturnTypes(mlir::MLIRContext *context,
           std::optional<::mlir::Location> location, mlir::ValueRange operands,
-          mlir::DictionaryAttr attributes, mlir::RegionRange regions,
+          mlir::DictionaryAttr attributes, mlir::OpaqueProperties properties, mlir::RegionRange regions,
           llvm::SmallVectorImpl<mlir::Type> &inferredReturnTypes) {
       inferredReturnTypes.assign({mlir::IntegerType::get(context, 16)});
       return mlir::success();
@@ -1691,20 +1691,20 @@ def TestEitherOpA : TEST_Op<"either_op_a"> {
 }
 
 def TestEitherOpB : TEST_Op<"either_op_b"> {
-  let arguments = (ins AnyInteger:$arg0);
+  let arguments = (ins AnyInteger:$arg0, AnyInteger:$arg1);
   let results = (outs I32:$output);
 }
 
-def : Pat<(TestEitherOpA (either I32:$arg1, I16:$arg2), $_),
-          (TestEitherOpB $arg2)>;
+def : Pat<(TestEitherOpA (either I32:$arg1, I16:$arg2), $x),
+          (TestEitherOpB $arg2, $x)>;
 
-def : Pat<(TestEitherOpA (either (TestEitherOpB I32:$arg1), I16:$arg2), $_),
-          (TestEitherOpB $arg2)>;
+def : Pat<(TestEitherOpA (either (TestEitherOpB I32:$arg1, $_), I16:$arg2), $x),
+          (TestEitherOpB $arg2, $x)>;
 
-def : Pat<(TestEitherOpA (either (TestEitherOpB I32:$arg1),
-                                 (TestEitherOpB I16:$arg2)),
-                          $_),
-          (TestEitherOpB $arg2)>;
+def : Pat<(TestEitherOpA (either (TestEitherOpB I32:$arg1, $_),
+                                 (TestEitherOpB I16:$arg2, $_)),
+                          $x),
+          (TestEitherOpB $arg2, $x)>;
 
 //===----------------------------------------------------------------------===//
 // Test Patterns (Location)
@@ -2524,7 +2524,7 @@ def FormatInferTypeOp : TEST_Op<"format_infer_type", [InferTypeOpInterface]> {
   let extraClassDeclaration = [{
     static ::mlir::LogicalResult inferReturnTypes(::mlir::MLIRContext *context,
           ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands,
-          ::mlir::DictionaryAttr attributes, ::mlir::RegionRange regions,
+          ::mlir::DictionaryAttr attributes, mlir::OpaqueProperties properties, ::mlir::RegionRange regions,
           ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
       inferredReturnTypes.assign({::mlir::IntegerType::get(context, 16)});
       return ::mlir::success();
@@ -2547,7 +2547,7 @@ class FormatInferAllTypesBaseOp<string mnemonic, list<Trait> traits = []>
   let extraClassDeclaration = [{
     static ::mlir::LogicalResult inferReturnTypes(::mlir::MLIRContext *context,
           ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands,
-          ::mlir::DictionaryAttr attributes, ::mlir::RegionRange regions,
+          ::mlir::DictionaryAttr attributes, mlir::OpaqueProperties properties, ::mlir::RegionRange regions,
           ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
       ::mlir::TypeRange operandTypes = operands.getTypes();
       inferredReturnTypes.assign(operandTypes.begin(), operandTypes.end());
@@ -2594,7 +2594,7 @@ def FormatInferTypeRegionsOp
   let extraClassDeclaration = [{
     static ::mlir::LogicalResult inferReturnTypes(::mlir::MLIRContext *context,
           ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands,
-          ::mlir::DictionaryAttr attributes, ::mlir::RegionRange regions,
+          ::mlir::DictionaryAttr attributes, mlir::OpaqueProperties properties, ::mlir::RegionRange regions,
           ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
       if (regions.empty())
         return ::mlir::failure();
@@ -2615,9 +2615,10 @@ def FormatInferTypeVariadicOperandsOp
   let extraClassDeclaration = [{
     static ::mlir::LogicalResult inferReturnTypes(::mlir::MLIRContext *context,
           ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands,
-          ::mlir::DictionaryAttr attributes, ::mlir::RegionRange regions,
+          ::mlir::DictionaryAttr attributes, mlir::OpaqueProperties properties, ::mlir::RegionRange regions,
           ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
-      FormatInferTypeVariadicOperandsOpAdaptor adaptor(operands, attributes);
+      FormatInferTypeVariadicOperandsOpAdaptor adaptor(
+          operands, attributes, *properties.as<Properties *>(), {});
       auto aTypes = adaptor.getA().getTypes();
       auto bTypes = adaptor.getB().getTypes();
       inferredReturnTypes.append(aTypes.begin(), aTypes.end());
@@ -2823,7 +2824,7 @@ class TableGenBuildInferReturnTypeBaseOp<string mnemonic,
   let extraClassDeclaration = [{
     static ::mlir::LogicalResult inferReturnTypes(::mlir::MLIRContext *,
           ::std::optional<::mlir::Location> location, ::mlir::ValueRange operands,
-          ::mlir::DictionaryAttr attributes, ::mlir::RegionRange regions,
+          ::mlir::DictionaryAttr attributes, mlir::OpaqueProperties properties, ::mlir::RegionRange regions,
           ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
       inferredReturnTypes.assign({operands[0].getType()});
       return ::mlir::success();
@@ -3280,4 +3281,77 @@ def TestVersionedOpB : TEST_Op<"versionedB"> {
   );
 }
 
+//===----------------------------------------------------------------------===//
+// Test Properties
+//===----------------------------------------------------------------------===//
+
+
+// Op with a properties struct defined inline.
+def TestOpWithProperties : TEST_Op<"with_properties"> {
+  let assemblyFormat = "prop-dict attr-dict";
+  let arguments = (ins
+    Property<"int64_t">:$a,
+    StrAttr:$b, // Attributes can directly be used here.
+    ArrayProperty<"int64_t", 4>:$array // example of an array
+  );
+}
+
+// Demonstrate how to wrap an existing C++ class named MyPropStruct.
+def MyStructProperty : Property<"MyPropStruct"> {
+  let convertToAttribute = "$_storage.asAttribute($_ctxt)";
+  let convertFromAttribute = "return MyPropStruct::setFromAttr($_storage, $_attr, $_diag);";
+  let hashProperty = "$_storage.hash();";
+}
+
+def TestOpWithWrappedProperties : TEST_Op<"with_wrapped_properties"> {
+  let assemblyFormat = "prop-dict attr-dict";
+  let arguments = (ins
+    MyStructProperty:$prop
+  );
+}
+
+// Op with a properties struct defined out-of-line. The struct has custom
+// printer/parser.
+
+def PropertiesWithCustomPrint : Property<"PropertiesWithCustomPrint"> {
+  let convertToAttribute = [{
+    getPropertiesAsAttribute($_ctxt, $_storage)
+  }];
+  let convertFromAttribute = [{
+    return setPropertiesFromAttribute($_storage, $_attr, $_diag);
+  }];
+  let hashProperty = [{
+    computeHash($_storage);
+  }];
+}
+
+def TestOpWithNiceProperties : TEST_Op<"with_nice_properties"> {
+  let assemblyFormat = "prop-dict attr-dict";
+  let arguments = (ins
+     PropertiesWithCustomPrint:$prop
+  );
+  let extraClassDeclaration = [{
+    void printProperties(::mlir::MLIRContext *ctx, ::mlir::OpAsmPrinter &p,
+                         const Properties &prop);
+    static ::mlir::ParseResult  parseProperties(::mlir::OpAsmParser &parser,
+                                     ::mlir::OperationState &result);
+  }];
+  let extraClassDefinition = [{
+    void TestOpWithNiceProperties::printProperties(::mlir::MLIRContext *ctx,
+            ::mlir::OpAsmPrinter &p, const Properties &prop) {
+      customPrintProperties(p, prop.prop);
+    }
+    ::mlir::ParseResult TestOpWithNiceProperties::parseProperties(
+        ::mlir::OpAsmParser &parser,
+        ::mlir::OperationState &result) {
+      Properties &prop = result.getOrAddProperties<Properties>();
+      if (customParseProperties(parser, prop.prop))
+        return failure();
+      return success();
+    }
+  }];
+}
+
+
+
 #endif // TEST_OPS
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 20d3d486c921f..adaa6e1558999 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -434,7 +434,8 @@ static void invokeCreateWithInferredReturnType(Operation *op) {
       SmallVector<Type, 2> inferredReturnTypes;
       if (succeeded(OpTy::inferReturnTypes(
               context, std::nullopt, values, op->getAttrDictionary(),
-              op->getRegions(), inferredReturnTypes))) {
+              op->getPropertiesStorage(), op->getRegions(),
+              inferredReturnTypes))) {
         OperationState state(location, OpTy::getOperationName());
         // TODO: Expand to regions.
         OpTy::build(b, state, values, op->getAttrs());
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index a03f1d5e76d68..dd853aa1dc3cc 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -261,8 +261,8 @@ struct TestVectorTransferUnrollingPatterns
     UnrollVectorOptions opts;
     opts.setNativeShape(ArrayRef<int64_t>{2, 2})
         .setFilterConstraint([](Operation *op) {
-          return success(
-              isa<vector::TransferReadOp, vector::TransferWriteOp>(op));
+          return success(isa<vector::TransferReadOp, vector::TransferWriteOp,
+                             vector::GatherOp>(op));
         });
     if (reverseUnrollOrder.getValue()) {
       opts.setUnrollTraversalOrderFn(
@@ -272,6 +272,8 @@ struct TestVectorTransferUnrollingPatterns
               numLoops = readOp.getVectorType().getRank();
             else if (auto writeOp = dyn_cast<vector::TransferWriteOp>(op))
               numLoops = writeOp.getVectorType().getRank();
+            else if (auto gatherOp = dyn_cast<vector::GatherOp>(op))
+              numLoops = gatherOp.getVectorType().getRank();
             else
               return std::nullopt;
             auto order = llvm::reverse(llvm::seq<int64_t>(0, numLoops));
@@ -602,6 +604,26 @@ struct TestVectorExtractStridedSliceLowering
   }
 };
 
+struct TestVectorBreakDownBitCast
+    : public PassWrapper<TestVectorBreakDownBitCast,
+                         OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorBreakDownBitCast)
+
+  StringRef getArgument() const final {
+    return "test-vector-break-down-bitcast";
+  }
+  StringRef getDescription() const final {
+    return "Test pattern that breaks down vector.bitcast ops ";
+  }
+  void runOnOperation() override {
+    RewritePatternSet patterns(&getContext());
+    populateBreakDownVectorBitCastOpPatterns(patterns, [](BitCastOp op) {
+      return op.getSourceVectorType().getShape().back() > 4;
+    });
+    (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
+  }
+};
+
 struct TestCreateVectorBroadcast
     : public PassWrapper<TestCreateVectorBroadcast,
                          OperationPass<func::FuncOp>> {
@@ -686,6 +708,8 @@ void registerTestVectorLowerings() {
 
   PassRegistration<TestVectorExtractStridedSliceLowering>();
 
+  PassRegistration<TestVectorBreakDownBitCast>();
+
   PassRegistration<TestCreateVectorBroadcast>();
 
   PassRegistration<TestVectorGatherLowering>();
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 1013dc8464adc..bf3f59b4c755e 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -36,6 +36,7 @@ config.enable_bindings_python = @MLIR_ENABLE_BINDINGS_PYTHON@
 config.intel_sde_executable = "@INTEL_SDE_EXECUTABLE@"
 config.mlir_run_amx_tests = @MLIR_RUN_AMX_TESTS@
 config.mlir_run_arm_sve_tests = @MLIR_RUN_ARM_SVE_TESTS@
+config.mlir_run_arm_sme_tests = @MLIR_RUN_ARM_SME_TESTS@
 config.mlir_run_x86vector_tests = @MLIR_RUN_X86VECTOR_TESTS@
 config.mlir_run_riscv_vector_tests = "@MLIR_RUN_RISCV_VECTOR_TESTS@"
 config.mlir_run_cuda_tensor_core_tests = @MLIR_RUN_CUDA_TENSOR_CORE_TESTS@
diff --git a/mlir/test/mlir-lsp-server/hover.test b/mlir/test/mlir-lsp-server/hover.test
index 3a8a4437bc533..edca346b8c733 100644
--- a/mlir/test/mlir-lsp-server/hover.test
+++ b/mlir/test/mlir-lsp-server/hover.test
@@ -18,7 +18,7 @@
 // CHECK-NEXT:  "result": {
 // CHECK-NEXT:    "contents": {
 // CHECK-NEXT:      "kind": "markdown",
-// CHECK-NEXT:      "value": "\"arith.constant\"\n\nGeneric Form:\n\n```mlir\n%0 = \"arith.constant\"() {value = true} : () -> i1\n```\n"
+// CHECK-NEXT:      "value": "\"arith.constant\"\n\nGeneric Form:\n\n```mlir\n%0 = \"arith.constant\"() <{value = true}> : () -> i1\n```\n"
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:      "end": {
@@ -114,7 +114,7 @@
 // CHECK-NEXT:  "result": {
 // CHECK-NEXT:    "contents": {
 // CHECK-NEXT:      "kind": "markdown",
-// CHECK-NEXT:      "value": "\"func.func\" : public @foo\n\nGeneric Form:\n\n```mlir\n\"func.func\"() ({...}) {function_type = (i1) -> (), sym_name = \"foo\"} : () -> ()\n```\n"
+// CHECK-NEXT:      "value": "\"func.func\" : public @foo\n\nGeneric Form:\n\n```mlir\n\"func.func\"() <{function_type = (i1) -> (), sym_name = \"foo\"}> ({...}) : () -> ()\n```\n"
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:      "end": {
@@ -138,7 +138,7 @@
 // CHECK-NEXT:  "result": {
 // CHECK-NEXT:    "contents": {
 // CHECK-NEXT:      "kind": "markdown",
-// CHECK-NEXT:      "value": "\"func.func\" : public @foo\n\nGeneric Form:\n\n```mlir\n\"func.func\"() ({...}) {function_type = (i1) -> (), sym_name = \"foo\"} : () -> ()\n```\n"
+// CHECK-NEXT:      "value": "\"func.func\" : public @foo\n\nGeneric Form:\n\n```mlir\n\"func.func\"() <{function_type = (i1) -> (), sym_name = \"foo\"}> ({...}) : () -> ()\n```\n"
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:      "end": {
diff --git a/mlir/test/mlir-opt/debugcounter.mlir b/mlir/test/mlir-opt/debugcounter.mlir
new file mode 100644
index 0000000000000..594a2183558b5
--- /dev/null
+++ b/mlir/test/mlir-opt/debugcounter.mlir
@@ -0,0 +1,15 @@
+// This test exercises the example in docs/ActionTracing.md ; changes here
+// should probably be reflected there.
+
+// RUN: mlir-opt %s -mlir-debug-counter=unique-tag-for-my-action-skip=-1 -mlir-print-debug-counter --pass-pipeline="builtin.module(func.func(canonicalize))" --mlir-disable-threading 2>&1 | FileCheck %s --check-prefix=CHECK-UNKNOWN-TAG
+// RUN: mlir-opt %s -mlir-debug-counter=pass-execution-skip=1 -mlir-print-debug-counter --pass-pipeline="builtin.module(func.func(canonicalize))" --mlir-disable-threading 2>&1 | FileCheck %s --check-prefix=CHECK-PASS
+
+func.func @foo() {
+    return
+}
+
+// CHECK-UNKNOWN-TAG:  DebugCounter counters:
+// CHECK-UNKNOWN-TAG: unique-tag-for-my-action        : {0,-1,-1}
+
+// CHECK-PASS: DebugCounter counters:
+// CHECK-PASS: pass-execution                  : {1,1,-1}
diff --git a/mlir/test/mlir-tblgen/constraint-unique.td b/mlir/test/mlir-tblgen/constraint-unique.td
index 0d377a454d6f5..ab191cbe7e13b 100644
--- a/mlir/test/mlir-tblgen/constraint-unique.td
+++ b/mlir/test/mlir-tblgen/constraint-unique.td
@@ -69,8 +69,8 @@ def OpC : NS_Op<"op_c"> {
 
 /// Test that an attribute contraint was generated.
 // CHECK:    static ::mlir::LogicalResult [[$A_ATTR_CONSTRAINT:__mlir_ods_local_attr_constraint.*]](
-// CHECK:      if (attr && !((attrPred(attr, *op)))) {
-// CHECK-NEXT:   return op->emitOpError("attribute '") << attrName
+// CHECK:      if (attr && !((attrPred(attr, *op))))
+// CHECK-NEXT:   return getDiag() << "attribute '" << attrName
 // CHECK-NEXT:       << "' failed to satisfy constraint: an attribute";
 
 /// Test that duplicate attribute constraint was not generated.
@@ -78,8 +78,9 @@ def OpC : NS_Op<"op_c"> {
 
 /// Test that a attribute constraint with a different description was generated.
 // CHECK:    static ::mlir::LogicalResult [[$O_ATTR_CONSTRAINT:__mlir_ods_local_attr_constraint.*]](
-// CHECK:      if (attr && !((attrPred(attr, *op)))) {
-// CHECK-NEXT:   return op->emitOpError("attribute '") << attrName
+// CHECK:    static ::mlir::LogicalResult [[$O_ATTR_CONSTRAINT:__mlir_ods_local_attr_constraint.*]](
+// CHECK:      if (attr && !((attrPred(attr, *op))))
+// CHECK-NEXT:   return getDiag() << "attribute '" << attrName
 // CHECK-NEXT:       << "' failed to satisfy constraint: another attribute";
 
 /// Test that a successor contraint was generated.
diff --git a/mlir/test/mlir-tblgen/interfaces-as-constraints.td b/mlir/test/mlir-tblgen/interfaces-as-constraints.td
index 093868000ff92..6468e60c70d29 100644
--- a/mlir/test/mlir-tblgen/interfaces-as-constraints.td
+++ b/mlir/test/mlir-tblgen/interfaces-as-constraints.td
@@ -34,13 +34,13 @@ def OpUsingAllOfThose : Op<Test_Dialect, "OpUsingAllOfThose"> {
 // CHECK-NEXT:        << " must be TypeInterfaceInNamespace instance, but got " << type;
 
 // CHECK: static ::mlir::LogicalResult {{__mlir_ods_local_attr_constraint.*}}(
-// CHECK:   if (attr && !((attr.isa<TopLevelAttrInterface>()))) {
-// CHECK-NEXT:    return op->emitOpError("attribute '") << attrName
+// CHECK:   if (attr && !((attr.isa<TopLevelAttrInterface>())))
+// CHECK-NEXT:     return getDiag() << "attribute '" << attrName
 // CHECK-NEXT:        << "' failed to satisfy constraint: TopLevelAttrInterface instance";
 
 // CHECK: static ::mlir::LogicalResult {{__mlir_ods_local_attr_constraint.*}}(
-// CHECK:   if (attr && !((attr.isa<test::AttrInterfaceInNamespace>()))) {
-// CHECK-NEXT:    return op->emitOpError("attribute '") << attrName
+// CHECK:   if (attr && !((attr.isa<test::AttrInterfaceInNamespace>())))
+// CHECK-NEXT:    return getDiag() << "attribute '" << attrName
 // CHECK-NEXT:        << "' failed to satisfy constraint: AttrInterfaceInNamespace instance";
 
 // CHECK: TopLevelAttrInterface OpUsingAllOfThose::getAttr1()
diff --git a/mlir/test/mlir-tblgen/op-attribute.td b/mlir/test/mlir-tblgen/op-attribute.td
index af1f62221fc07..a10cb0536ebc8 100644
--- a/mlir/test/mlir-tblgen/op-attribute.td
+++ b/mlir/test/mlir-tblgen/op-attribute.td
@@ -68,7 +68,7 @@ def AOp : NS_Op<"a_op", []> {
 
 // DEF:      ::mlir::LogicalResult AOpAdaptor::verify
 // DEF:      ::mlir::Attribute tblgen_aAttr;
-// DEF-NEXT: while (true) {
+// DEF: while (true) {
 // DEF-NEXT:   if (namedAttrIt == namedAttrRange.end())
 // DEF-NEXT:     return emitError(loc, "'test.a_op' op ""requires attribute 'aAttr'");
 // DEF-NEXT:   if (namedAttrIt->getName() == AOp::getAAttrAttrName(*odsOpName)) {
@@ -217,10 +217,10 @@ def AgetOp : Op<Test2_Dialect, "a_get_op", []> {
 
 // DEF:      ::mlir::LogicalResult AgetOpAdaptor::verify
 // DEF:      ::mlir::Attribute tblgen_aAttr;
-// DEF-NEXT: while (true)
+// DEF: while (true)
 // DEF:      ::mlir::Attribute tblgen_bAttr;
 // DEF-NEXT: ::mlir::Attribute tblgen_cAttr;
-// DEF-NEXT: while (true)
+// DEF: while (true)
 // DEF:      if (tblgen_aAttr && !((some-condition)))
 // DEF-NEXT:   return emitError(loc, "'test2.a_get_op' op ""attribute 'aAttr' failed to satisfy constraint: some attribute kind");
 // DEF:      if (tblgen_bAttr && !((some-condition)))
diff --git a/mlir/test/mlir-tblgen/op-decl-and-defs.td b/mlir/test/mlir-tblgen/op-decl-and-defs.td
index ed54ef2238019..618197c76c2a0 100644
--- a/mlir/test/mlir-tblgen/op-decl-and-defs.td
+++ b/mlir/test/mlir-tblgen/op-decl-and-defs.td
@@ -126,7 +126,7 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 
 // DEFS-LABEL: NS::AOp definitions
 
-// DEFS: AOpGenericAdaptorBase::AOpGenericAdaptorBase(::mlir::DictionaryAttr attrs, ::mlir::RegionRange regions) : odsAttrs(attrs), odsRegions(regions)
+// DEFS: AOpGenericAdaptorBase::AOpGenericAdaptorBase(::mlir::DictionaryAttr attrs, ::mlir::EmptyProperties properties, ::mlir::RegionRange regions) : odsAttrs(attrs), odsRegions(regions)
 // DEFS: ::mlir::RegionRange AOpGenericAdaptorBase::getSomeRegions()
 // DEFS-NEXT: return odsRegions.drop_front(1);
 // DEFS: ::mlir::RegionRange AOpGenericAdaptorBase::getRegions()
diff --git a/mlir/test/mlir-tblgen/op-format.td b/mlir/test/mlir-tblgen/op-format.td
index 91c0011ae660c..7083d70be45de 100644
--- a/mlir/test/mlir-tblgen/op-format.td
+++ b/mlir/test/mlir-tblgen/op-format.td
@@ -64,7 +64,7 @@ def OptionalGroupA : TestFormat_Op<[{
 // CHECK-NEXT: result.addAttribute("a", parser.getBuilder().getUnitAttr())
 // CHECK: parser.parseKeyword("bar")
 // CHECK-LABEL: OptionalGroupB::print
-// CHECK: if (!(*this)->getAttr("a"))
+// CHECK: if (!getAAttr())
 // CHECK-NEXT: odsPrinter << ' ' << "foo"
 // CHECK-NEXT: else
 // CHECK-NEXT: odsPrinter << ' ' << "bar"
@@ -74,7 +74,8 @@ def OptionalGroupB : TestFormat_Op<[{
 
 // Optional group anchored on a default-valued attribute:
 // CHECK-LABEL: OptionalGroupC::parse
-//       CHECK: if ((*this)->getAttr("a") != ::mlir::OpBuilder((*this)->getContext()).getStringAttr("default")) {
+
+//       CHECK: if (getAAttr() && getAAttr() != ::mlir::OpBuilder((*this)->getContext()).getStringAttr("default")) {
 //  CHECK-NEXT:   odsPrinter << ' ';
 //  CHECK-NEXT:   odsPrinter.printAttributeWithoutType(getAAttr());
 //  CHECK-NEXT: }
diff --git a/mlir/test/mlir-tblgen/op-result.td b/mlir/test/mlir-tblgen/op-result.td
index d49bffa9cb441..606d4c250d88b 100644
--- a/mlir/test/mlir-tblgen/op-result.td
+++ b/mlir/test/mlir-tblgen/op-result.td
@@ -152,7 +152,7 @@ def OpL3 : NS_Op<"op_with_all_types_constraint",
 
 // CHECK-LABEL: LogicalResult OpL3::inferReturnTypes
 // CHECK-NOT: }
-// CHECK: ::mlir::Type odsInferredType0 = attributes.get("a").cast<::mlir::TypedAttr>().getType();
+// CHECK: ::mlir::Type odsInferredType0 = odsInferredTypeAttr0.getType();
 // CHECK: inferredReturnTypes[0] = odsInferredType0;
 
 def OpL4 : NS_Op<"two_inference_edges", [
diff --git a/mlir/test/mlir-tblgen/pattern.mlir b/mlir/test/mlir-tblgen/pattern.mlir
index 4aedb5e83f669..4c1182fa9eb72 100644
--- a/mlir/test/mlir-tblgen/pattern.mlir
+++ b/mlir/test/mlir-tblgen/pattern.mlir
@@ -5,8 +5,8 @@ func.func @verifyFusedLocs(%arg0 : i32) -> i32 {
   %0 = "test.op_a"(%arg0) {attr = 10 : i32} : (i32) -> i32 loc("a")
   %result = "test.op_a"(%0) {attr = 20 : i32} : (i32) -> i32 loc("b")
 
-  // CHECK: "test.op_b"(%arg0) {attr = 10 : i32} : (i32) -> i32 loc("a")
-  // CHECK: "test.op_b"(%arg0) {attr = 20 : i32} : (i32) -> i32 loc(fused["b", "a"])
+  // CHECK: "test.op_b"(%arg0) <{attr = 10 : i32}> : (i32) -> i32 loc("a")
+  // CHECK: "test.op_b"(%arg0) <{attr = 20 : i32}> : (i32) -> i32 loc(fused["b", "a"])
   return %result : i32
 }
 
@@ -41,7 +41,7 @@ func.func @verifyZeroArg() -> i32 {
 // CHECK-LABEL: testIgnoreArgMatch
 // CHECK-SAME: (%{{[a-z0-9]*}}: i32 loc({{[^)]*}}), %[[ARG1:[a-z0-9]*]]: i32 loc({{[^)]*}}),
 func.func @testIgnoreArgMatch(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: f32) {
-  // CHECK: "test.ignore_arg_match_dst"(%[[ARG1]]) {f = 15 : i64}
+  // CHECK: "test.ignore_arg_match_dst"(%[[ARG1]]) <{f = 15 : i64}>
   "test.ignore_arg_match_src"(%arg0, %arg1, %arg2) {d = 42, e = 24, f = 15} : (i32, i32, i32) -> ()
 
   // CHECK: test.ignore_arg_match_src
@@ -57,7 +57,7 @@ func.func @testIgnoreArgMatch(%arg0: i32, %arg1: i32, %arg2: i32, %arg3: f32) {
 // CHECK-LABEL: verifyInterleavedOperandAttribute
 // CHECK-SAME:    %[[ARG0:.*]]: i32 loc({{[^)]*}}), %[[ARG1:.*]]: i32 loc({{[^)]*}})
 func.func @verifyInterleavedOperandAttribute(%arg0: i32, %arg1: i32) {
-  // CHECK: "test.interleaved_operand_attr2"(%[[ARG0]], %[[ARG1]]) {attr1 = 15 : i64, attr2 = 42 : i64}
+  // CHECK: "test.interleaved_operand_attr2"(%[[ARG0]], %[[ARG1]]) <{attr1 = 15 : i64, attr2 = 42 : i64}>
   "test.interleaved_operand_attr1"(%arg0, %arg1) {attr1 = 15, attr2 = 42} : (i32, i32) -> ()
   return
 }
@@ -69,13 +69,13 @@ func.func @verifyBenefit(%arg0 : i32) -> i32 {
   %2 = "test.op_g"(%1) : (i32) -> i32
 
   // CHECK: "test.op_f"(%arg0)
-  // CHECK: "test.op_b"(%arg0) {attr = 34 : i32}
+  // CHECK: "test.op_b"(%arg0) <{attr = 34 : i32}>
   return %0 : i32
 }
 
 // CHECK-LABEL: verifyNativeCodeCall
 func.func @verifyNativeCodeCall(%arg0: i32, %arg1: i32) -> (i32, i32) {
-  // CHECK: %0 = "test.native_code_call2"(%arg0) {attr = [42, 24]} : (i32) -> i32
+  // CHECK: %0 = "test.native_code_call2"(%arg0) <{attr = [42, 24]}> : (i32) -> i32
   // CHECK:  return %0, %arg1
   %0 = "test.native_code_call1"(%arg0, %arg1) {choice = true, attr1 = 42, attr2 = 24} : (i32, i32) -> (i32)
   %1 = "test.native_code_call1"(%arg0, %arg1) {choice = false, attr1 = 42, attr2 = 24} : (i32, i32) -> (i32)
@@ -215,7 +215,7 @@ func.func @symbolBinding(%arg0: i32) -> i32 {
   // An op with one use is matched.
   // CHECK: %0 = "test.symbol_binding_b"(%arg0)
   // CHECK: %1 = "test.symbol_binding_c"(%0)
-  // CHECK: %2 = "test.symbol_binding_d"(%0, %1) {attr = 42 : i64}
+  // CHECK: %2 = "test.symbol_binding_d"(%0, %1) <{attr = 42 : i64}>
   %0 = "test.symbol_binding_a"(%arg0) {attr = 42} : (i32) -> (i32)
 
   // An op without any use is not matched.
@@ -239,21 +239,21 @@ func.func @symbolBindingNoResult(%arg0: i32) {
 
 // CHECK-LABEL: succeedMatchOpAttr
 func.func @succeedMatchOpAttr() -> i32 {
-  // CHECK: "test.match_op_attribute2"() {default_valued_attr = 3 : i32, more_attr = 4 : i32, optional_attr = 2 : i32, required_attr = 1 : i32}
+  // CHECK: "test.match_op_attribute2"() <{default_valued_attr = 3 : i32, more_attr = 4 : i32, optional_attr = 2 : i32, required_attr = 1 : i32}>
   %0 = "test.match_op_attribute1"() {required_attr = 1: i32, optional_attr = 2: i32, default_valued_attr = 3: i32, more_attr = 4: i32} : () -> (i32)
   return %0: i32
 }
 
 // CHECK-LABEL: succeedMatchMissingOptionalAttr
 func.func @succeedMatchMissingOptionalAttr() -> i32 {
-  // CHECK: "test.match_op_attribute2"() {default_valued_attr = 3 : i32, more_attr = 4 : i32, required_attr = 1 : i32}
+  // CHECK: "test.match_op_attribute2"() <{default_valued_attr = 3 : i32, more_attr = 4 : i32, required_attr = 1 : i32}>
   %0 = "test.match_op_attribute1"() {required_attr = 1: i32, default_valued_attr = 3: i32, more_attr = 4: i32} : () -> (i32)
   return %0: i32
 }
 
 // CHECK-LABEL: succeedMatchMissingDefaultValuedAttr
 func.func @succeedMatchMissingDefaultValuedAttr() -> i32 {
-  // CHECK: "test.match_op_attribute2"() {default_valued_attr = 42 : i32, more_attr = 4 : i32, optional_attr = 2 : i32, required_attr = 1 : i32}
+  // CHECK: "test.match_op_attribute2"() <{default_valued_attr = 42 : i32, more_attr = 4 : i32, optional_attr = 2 : i32, required_attr = 1 : i32}>
   %0 = "test.match_op_attribute1"() {required_attr = 1: i32, optional_attr = 2: i32, more_attr = 4: i32} : () -> (i32)
   return %0: i32
 }
@@ -267,7 +267,7 @@ func.func @failedMatchAdditionalConstraintNotSatisfied() -> i32 {
 
 // CHECK-LABEL: verifyConstantAttr
 func.func @verifyConstantAttr(%arg0 : i32) -> i32 {
-  // CHECK: "test.op_b"(%arg0) {attr = 17 : i32} : (i32) -> i32 loc("a")
+  // CHECK: "test.op_b"(%arg0) <{attr = 17 : i32}> : (i32) -> i32 loc("a")
   %0 = "test.op_c"(%arg0) : (i32) -> i32 loc("a")
   return %0 : i32
 }
@@ -275,12 +275,12 @@ func.func @verifyConstantAttr(%arg0 : i32) -> i32 {
 // CHECK-LABEL: verifyUnitAttr
 func.func @verifyUnitAttr() -> (i32, i32) {
   // Unit attribute present in the matched op is propagated as attr2.
-  // CHECK: "test.match_op_attribute4"() {attr1, attr2} : () -> i32
+  // CHECK: "test.match_op_attribute4"() <{attr1, attr2}> : () -> i32
   %0 = "test.match_op_attribute3"() {attr} : () -> i32
 
   // Since the original op doesn't have the unit attribute, the new op
   // only has the constant-constructed unit attribute attr1.
-  // CHECK: "test.match_op_attribute4"() {attr1} : () -> i32
+  // CHECK: "test.match_op_attribute4"() <{attr1}> : () -> i32
   %1 = "test.match_op_attribute3"() : () -> i32
   return %0, %1 : i32, i32
 }
@@ -291,7 +291,7 @@ func.func @verifyUnitAttr() -> (i32, i32) {
 
 // CHECK-LABEL: testConstOp
 func.func @testConstOp() -> (i32) {
-  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() {value = 1
+  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() <{value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
 
   // CHECK-NEXT: return [[C0]]
@@ -300,7 +300,7 @@ func.func @testConstOp() -> (i32) {
 
 // CHECK-LABEL: testConstOpUsed
 func.func @testConstOpUsed() -> (i32) {
-  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() {value = 1
+  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() <{value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
 
   // CHECK-NEXT: [[V0:%.+]] = "test.op_s"([[C0]])
@@ -312,11 +312,11 @@ func.func @testConstOpUsed() -> (i32) {
 
 // CHECK-LABEL: testConstOpReplaced
 func.func @testConstOpReplaced() -> (i32) {
-  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() {value = 1
+  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() <{value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
   %1 = "test.constant"() {value = 2 : i32} : () -> i32
 
-  // CHECK: [[V0:%.+]] = "test.op_s"([[C0]]) {value = 2 : i32}
+  // CHECK: [[V0:%.+]] = "test.op_s"([[C0]]) <{value = 2 : i32}
   %2 = "test.op_r"(%0, %1) : (i32, i32) -> i32
 
   // CHECK: [[V0]]
@@ -325,10 +325,10 @@ func.func @testConstOpReplaced() -> (i32) {
 
 // CHECK-LABEL: testConstOpMatchFailure
 func.func @testConstOpMatchFailure() -> (i64) {
-  // CHECK-DAG: [[C0:%.+]] = "test.constant"() {value = 1
+  // CHECK-DAG: [[C0:%.+]] = "test.constant"() <{value = 1
   %0 = "test.constant"() {value = 1 : i64} : () -> i64
 
-  // CHECK-DAG: [[C1:%.+]] = "test.constant"() {value = 2
+  // CHECK-DAG: [[C1:%.+]] = "test.constant"() <{value = 2
   %1 = "test.constant"() {value = 2 : i64} : () -> i64
 
   // CHECK: [[V0:%.+]] = "test.op_r"([[C0]], [[C1]])
@@ -340,7 +340,7 @@ func.func @testConstOpMatchFailure() -> (i64) {
 
 // CHECK-LABEL: testConstOpMatchNonConst
 func.func @testConstOpMatchNonConst(%arg0 : i32) -> (i32) {
-  // CHECK-DAG: [[C0:%.+]] = "test.constant"() {value = 1
+  // CHECK-DAG: [[C0:%.+]] = "test.constant"() <{value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
 
   // CHECK: [[V0:%.+]] = "test.op_r"([[C0]], %arg0)
@@ -358,14 +358,14 @@ func.func @testConstOpMatchNonConst(%arg0 : i32) -> (i32) {
 
 // CHECK-LABEL: verifyI32EnumAttr
 func.func @verifyI32EnumAttr() -> i32 {
-  // CHECK: "test.i32_enum_attr"() {attr = 10 : i32}
+  // CHECK: "test.i32_enum_attr"() <{attr = 10 : i32}
   %0 = "test.i32_enum_attr"() {attr = 5: i32} : () -> i32
   return %0 : i32
 }
 
 // CHECK-LABEL: verifyI64EnumAttr
 func.func @verifyI64EnumAttr() -> i32 {
-  // CHECK: "test.i64_enum_attr"() {attr = 10 : i64}
+  // CHECK: "test.i64_enum_attr"() <{attr = 10 : i64}
   %0 = "test.i64_enum_attr"() {attr = 5: i64} : () -> i32
   return %0 : i32
 }
@@ -522,7 +522,7 @@ func.func @generateVariadicOutputOpInNestedPattern() -> (i32) {
 // CHECK-LABEL: redundantTest
 func.func @redundantTest(%arg0: i32) -> i32 {
   %0 = "test.op_m"(%arg0) : (i32) -> i32
-  // CHECK: "test.op_m"(%arg0) {optional_attr = 314159265 : i32} : (i32) -> i32
+  // CHECK: "test.op_m"(%arg0) <{optional_attr = 314159265 : i32}> : (i32) -> i32
   return %0 : i32
 }
 
@@ -532,30 +532,30 @@ func.func @redundantTest(%arg0: i32) -> i32 {
 
 // CHECK: @either_dag_leaf_only
 func.func @either_dag_leaf_only_1(%arg0 : i32, %arg1 : i16, %arg2 : i8) -> () {
-  // CHECK: "test.either_op_b"(%arg1) : (i16) -> i32
+  // CHECK: "test.either_op_b"(%arg1, %arg2) : (i16, i8) -> i32
   %0 = "test.either_op_a"(%arg0, %arg1, %arg2) : (i32, i16, i8) -> i32
-  // CHECK: "test.either_op_b"(%arg1) : (i16) -> i32
+  // CHECK: "test.either_op_b"(%arg1, %arg2) : (i16, i8) -> i32
   %1 = "test.either_op_a"(%arg1, %arg0, %arg2) : (i16, i32, i8) -> i32
   return
 }
 
 // CHECK: @either_dag_leaf_dag_node
 func.func @either_dag_leaf_dag_node(%arg0 : i32, %arg1 : i16, %arg2 : i8) -> () {
-  %0 = "test.either_op_b"(%arg0) : (i32) -> i32
-  // CHECK: "test.either_op_b"(%arg1) : (i16) -> i32
+  %0 = "test.either_op_b"(%arg0, %arg0) : (i32, i32) -> i32
+  // CHECK: "test.either_op_b"(%arg1, %arg2) : (i16, i8) -> i32
   %1 = "test.either_op_a"(%0, %arg1, %arg2) : (i32, i16, i8) -> i32
-  // CHECK: "test.either_op_b"(%arg1) : (i16) -> i32
+  // CHECK: "test.either_op_b"(%arg1, %arg2) : (i16, i8) -> i32
   %2 = "test.either_op_a"(%arg1, %0, %arg2) : (i16, i32, i8) -> i32
   return
 }
 
 // CHECK: @either_dag_node_dag_node
 func.func @either_dag_node_dag_node(%arg0 : i32, %arg1 : i16, %arg2 : i8) -> () {
-  %0 = "test.either_op_b"(%arg0) : (i32) -> i32
-  %1 = "test.either_op_b"(%arg1) : (i16) -> i32
-  // CHECK: "test.either_op_b"(%arg1) : (i16) -> i32
+  %0 = "test.either_op_b"(%arg0, %arg0) : (i32, i32) -> i32
+  %1 = "test.either_op_b"(%arg1, %arg1) : (i16, i16) -> i32
+  // CHECK: "test.either_op_b"(%arg1, %arg2) : (i16, i8) -> i32
   %2 = "test.either_op_a"(%0, %1, %arg2) : (i32, i32, i8) -> i32
-  // CHECK: "test.either_op_b"(%arg1) : (i16) -> i32
+  // CHECK: "test.either_op_b"(%arg1, %arg2) : (i16, i8) -> i32
   %3 = "test.either_op_a"(%1, %0, %arg2) : (i32, i32, i8) -> i32
   return
 }
diff --git a/mlir/test/mlir-tblgen/return-types.mlir b/mlir/test/mlir-tblgen/return-types.mlir
index 39fb44f27695d..24555034d2677 100644
--- a/mlir/test/mlir-tblgen/return-types.mlir
+++ b/mlir/test/mlir-tblgen/return-types.mlir
@@ -24,6 +24,7 @@ func.func @testCreateFunctions(%arg0 : tensor<10xf32, !test.smpla>, %arg1 : tens
 // -----
 
 func.func @testReturnTypeOpInterface(%arg0 : tensor<10xf32>) {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{incompatible with return type}}
   %bad = "test.op_with_infer_type_if"(%arg0, %arg0) : (tensor<10xf32>, tensor<10xf32>) -> tensor<*xf32>
   return
@@ -32,6 +33,7 @@ func.func @testReturnTypeOpInterface(%arg0 : tensor<10xf32>) {
 // -----
 
 func.func @testReturnTypeOpInterfaceMismatch(%arg0 : tensor<10xf32>, %arg1 : tensor<20xf32>) {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{operand type mismatch}}
   %bad = "test.op_with_infer_type_if"(%arg0, %arg1) : (tensor<10xf32>, tensor<20xf32>) -> tensor<*xf32>
   return
@@ -40,6 +42,7 @@ func.func @testReturnTypeOpInterfaceMismatch(%arg0 : tensor<10xf32>, %arg1 : ten
 // -----
 
 func.func @testReturnTypeOpInterface(%arg0 : tensor<10xf32>) {
+  // expected-error@+2 {{failed to infer returned types}}
   // expected-error@+1 {{required first operand and result to match}}
   %bad = "test.op_with_refine_type_if"(%arg0, %arg0) : (tensor<10xf32>, tensor<10xf32>) -> tensor<*xf32>
   return
diff --git a/mlir/test/python/dialects/tensor.py b/mlir/test/python/dialects/tensor.py
index d8ea426beadf2..b0ad4b4ad4d65 100644
--- a/mlir/test/python/dialects/tensor.py
+++ b/mlir/test/python/dialects/tensor.py
@@ -82,7 +82,6 @@ def testInferTypesInsertSlice():
   with Context() as ctx, Location.unknown():
     module = Module.create()
     f32Type = F32Type.get()
-    indexType = IndexType.get()
     with InsertionPoint(module.body):
 
       @func.FuncOp.from_py_func(
@@ -92,8 +91,6 @@ def testInferTypesInsertSlice():
       # CHECK:      tensor.insert_slice %arg0 into %arg1[0, 0] [1, 1] [0, 0] :
       # CHECK-SAME:   tensor<1x1xf32> into tensor<1x1xf32>
       def f(source, dest):
-        c0 = arith.ConstantOp(indexType, 0)
-        c1 = arith.ConstantOp(indexType, 1)
         d0 = tensor.InsertSliceOp(source, dest, [], [], [],
                                   DenseI64ArrayAttr.get([0, 0]),
                                   DenseI64ArrayAttr.get([1, 1]),
@@ -101,3 +98,34 @@ def f(source, dest):
         return [d0.result]
 
   print(module)
+
+
+# CHECK-LABEL: TEST: testFromElementsOp
+@run
+def testFromElementsOp():
+  with Context() as ctx, Location.unknown():
+    module = Module.create()
+    f32 = F32Type.get()
+    with InsertionPoint(module.body):
+      @func.FuncOp.from_py_func()
+      def default_builder():
+        c0 = arith.ConstantOp(f32, 0.0)
+        # CHECK: %[[C0:.*]] = "arith.constant
+        # CHECK-SAME: value = 0.000000e+00 : f32
+        print(c0)
+        c1 = arith.ConstantOp(f32, 1.0)
+        # CHECK: %[[C1:.*]] = "arith.constant
+        # CHECK-SAME: value = 1.000000e+00 : f32
+        print(c1)
+
+        t = tensor.FromElementsOp(RankedTensorType.get((2,), f32), [c0, c1])
+        # CHECK: %{{.*}} = "tensor.from_elements"(%[[C0]], %[[C1]]) : (f32, f32) -> tensor<2xf32>
+        print(t)
+
+        t = tensor.FromElementsOp(RankedTensorType.get((2, 1), f32), [c0, c1])
+        # CHECK: %{{.*}} = "tensor.from_elements"(%[[C0]], %[[C1]]) : (f32, f32) -> tensor<2x1xf32>
+        print(t)
+
+        t = tensor.FromElementsOp(RankedTensorType.get((1, 2), f32), [c0, c1])
+        # CHECK: %{{.*}} = "tensor.from_elements"(%[[C0]], %[[C1]]) : (f32, f32) -> tensor<1x2xf32>
+        print(t)
diff --git a/mlir/test/python/ir/array_attributes.py b/mlir/test/python/ir/array_attributes.py
index 36b0769b20653..3de4edb884157 100644
--- a/mlir/test/python/ir/array_attributes.py
+++ b/mlir/test/python/ir/array_attributes.py
@@ -365,3 +365,20 @@ def testGetDenseElementsUI64():
     # CHECK: {{\[}}4 5 6]]
     print(np.array(attr))
 
+
+# CHECK-LABEL: TEST: testGetDenseElementsIndex
+@run
+def testGetDenseElementsIndex():
+  with Context(), Location.unknown():
+    idx_type = IndexType.get()
+    array = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64)
+    attr = DenseElementsAttr.get(array, type=idx_type)
+    # CHECK: dense<{{\[}}[1, 2, 3], [4, 5, 6]]> : tensor<2x3xindex>
+    print(attr)
+    arr = np.array(attr)
+    # CHECK: {{\[}}[1 2 3]
+    # CHECK: {{\[}}4 5 6]]
+    print(arr)
+    # CHECK: True
+    print(arr.dtype == np.int64)
+
diff --git a/mlir/test/python/ir/operation.py b/mlir/test/python/ir/operation.py
index 2088e1633c66c..ea84d113a9488 100644
--- a/mlir/test/python/ir/operation.py
+++ b/mlir/test/python/ir/operation.py
@@ -571,7 +571,7 @@ def testOperationPrint():
 
   # Test roundtrip to bytecode.
   bytecode_stream = io.BytesIO()
-  module.operation.write_bytecode(bytecode_stream)
+  module.operation.write_bytecode(bytecode_stream, desired_version=1)
   bytecode = bytecode_stream.getvalue()
   assert bytecode.startswith(b'ML\xefR'), "Expected bytecode to start with MLïR"
   module_roundtrip = Module.parse(bytecode, ctx)
diff --git a/mlir/test/python/ir/value.py b/mlir/test/python/ir/value.py
index 98f55de41e150..90fe64ac1762a 100644
--- a/mlir/test/python/ir/value.py
+++ b/mlir/test/python/ir/value.py
@@ -111,3 +111,29 @@ def testValueUses():
     assert use.owner in [op1, op2]
     print(f"Use owner: {use.owner}")
     print(f"Use operand_number: {use.operand_number}")
+
+# CHECK-LABEL: TEST: testValueReplaceAllUsesWith
+@run
+def testValueReplaceAllUsesWith():
+  ctx = Context()
+  ctx.allow_unregistered_dialects = True
+  with Location.unknown(ctx):
+    i32 = IntegerType.get_signless(32)
+    module = Module.create()
+    with InsertionPoint(module.body):
+      value = Operation.create("custom.op1", results=[i32]).results[0]
+      op1 = Operation.create("custom.op2", operands=[value])
+      op2 = Operation.create("custom.op2", operands=[value])
+      value2 = Operation.create("custom.op3", results=[i32]).results[0]
+      value.replace_all_uses_with(value2)
+
+  assert len(list(value.uses)) == 0
+
+  # CHECK: Use owner: "custom.op2"
+  # CHECK: Use operand_number: 0
+  # CHECK: Use owner: "custom.op2"
+  # CHECK: Use operand_number: 0
+  for use in value2.uses:
+    assert use.owner in [op1, op2]
+    print(f"Use owner: {use.owner}")
+    print(f"Use operand_number: {use.operand_number}")
diff --git a/mlir/test/python/python_test_ops.td b/mlir/test/python/python_test_ops.td
index 0c63620251c03..692fcb938961e 100644
--- a/mlir/test/python/python_test_ops.td
+++ b/mlir/test/python/python_test_ops.td
@@ -73,6 +73,7 @@ def InferResultsOp : TestOp<"infer_results_op", [InferTypeOpInterface]> {
     static ::mlir::LogicalResult inferReturnTypes(
       ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location,
       ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes,
+      ::mlir::OpaqueProperties,
       ::mlir::RegionRange regions,
       ::llvm::SmallVectorImpl<::mlir::Type> &inferredReturnTypes) {
       ::mlir::Builder b(context);
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index ce9008b58ba2b..13a525f0bcff8 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -261,6 +261,5 @@ int main(int argc, char **argv) {
   ::test::registerTestDynDialect(registry);
 #endif
   return mlir::asMainReturnCode(
-      mlir::MlirOptMain(argc, argv, "MLIR modular optimizer driver\n", registry,
-                        /*preloadDialectsInContext=*/false));
+      mlir::MlirOptMain(argc, argv, "MLIR modular optimizer driver\n", registry));
 }
diff --git a/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp b/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
new file mode 100644
index 0000000000000..f13bdd49413b0
--- /dev/null
+++ b/mlir/tools/mlir-tblgen/BytecodeDialectGen.cpp
@@ -0,0 +1,467 @@
+//===- BytecodeDialectGen.cpp - Dialect bytecode read/writer gen  ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Support/IndentedOstream.h"
+#include "mlir/TableGen/GenInfo.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include <regex>
+
+using namespace llvm;
+
+static llvm::cl::OptionCategory dialectGenCat("Options for -gen-bytecode");
+static llvm::cl::opt<std::string>
+    selectedBcDialect("bytecode-dialect",
+                      llvm::cl::desc("The dialect to gen for"),
+                      llvm::cl::cat(dialectGenCat), llvm::cl::CommaSeparated);
+
+namespace {
+
+/// Helper class to generate C++ bytecode parser helpers.
+class Generator {
+public:
+  Generator(raw_ostream &output) : output(output) {}
+
+  /// Returns whether successfully emitted attribute/type parsers.
+  void emitParse(StringRef kind, Record &x);
+
+  /// Returns whether successfully emitted attribute/type printers.
+  void emitPrint(StringRef kind, StringRef type,
+                 ArrayRef<std::pair<int64_t, Record *>> vec);
+
+  /// Emits parse dispatch table.
+  void emitParseDispatch(StringRef kind, ArrayRef<Record *> vec);
+
+  /// Emits print dispatch table.
+  void emitPrintDispatch(StringRef kind, ArrayRef<std::string> vec);
+
+private:
+  /// Emits parse calls to construct given kind.
+  void emitParseHelper(StringRef kind, StringRef returnType, StringRef builder,
+                       ArrayRef<Init *> args, ArrayRef<std::string> argNames,
+                       StringRef failure, mlir::raw_indented_ostream &ios);
+
+  /// Emits print instructions.
+  void emitPrintHelper(Record *memberRec, StringRef kind, StringRef parent,
+                       StringRef name, mlir::raw_indented_ostream &ios);
+
+  raw_ostream &output;
+};
+} // namespace
+
+/// Helper to replace set of from strings to target in `s`.
+/// Assumed: non-overlapping replacements.
+static std::string format(StringRef templ,
+                          std::map<std::string, std::string> &&map) {
+  std::string s = templ.str();
+  for (const auto &[from, to] : map)
+    // All replacements start with $, don't treat as anchor.
+    s = std::regex_replace(s, std::regex("\\" + from), to);
+  return s;
+}
+
+/// Return string with first character capitalized.
+static std::string capitalize(StringRef str) {
+  return ((Twine)toUpper(str[0]) + str.drop_front()).str();
+}
+
+/// Return the C++ type for the given record.
+static std::string getCType(Record *def) {
+  std::string format = "{0}";
+  if (def->isSubClassOf("Array")) {
+    def = def->getValueAsDef("elemT");
+    format = "SmallVector<{0}>";
+  }
+
+  StringRef cType = def->getValueAsString("cType");
+  if (cType.empty()) {
+    if (def->isAnonymous())
+      PrintFatalError(def->getLoc(), "Unable to determine cType");
+
+    return formatv(format.c_str(), def->getName().str());
+  }
+  return formatv(format.c_str(), cType.str());
+}
+
+void Generator::emitParseDispatch(StringRef kind, ArrayRef<Record *> vec) {
+  mlir::raw_indented_ostream os(output);
+  char const *head =
+      R"(static {0} read{0}(MLIRContext* context, DialectBytecodeReader &reader))";
+  os << formatv(head, capitalize(kind));
+  auto funScope = os.scope(" {\n", "}\n\n");
+
+  os << "uint64_t kind;\n";
+  os << "if (failed(reader.readVarInt(kind)))\n"
+     << "  return " << capitalize(kind) << "();\n";
+  os << "switch (kind) ";
+  {
+    auto switchScope = os.scope("{\n", "}\n");
+    for (const auto &it : llvm::enumerate(vec)) {
+      os << formatv("case {1}:\n  return read{0}(context, reader);\n",
+                    it.value()->getName(), it.index());
+    }
+    os << "default:\n"
+       << "  reader.emitError() << \"unknown attribute code: \" "
+       << "<< kind;\n"
+       << "  return " << capitalize(kind) << "();\n";
+  }
+  os << "return " << capitalize(kind) << "();\n";
+}
+
+void Generator::emitParse(StringRef kind, Record &x) {
+  char const *head =
+      R"(static {0} read{1}(MLIRContext* context, DialectBytecodeReader &reader) )";
+  mlir::raw_indented_ostream os(output);
+  std::string returnType = getCType(&x);
+  os << formatv(head, returnType, x.getName());
+  DagInit *members = x.getValueAsDag("members");
+  SmallVector<std::string> argNames =
+      llvm::to_vector(map_range(members->getArgNames(), [](StringInit *init) {
+        return init->getAsUnquotedString();
+      }));
+  StringRef builder = x.getValueAsString("cBuilder");
+  emitParseHelper(kind, returnType, builder, members->getArgs(), argNames,
+                  returnType + "()", os);
+  os << "\n\n";
+}
+
+void printParseConditional(mlir::raw_indented_ostream &ios,
+                           ArrayRef<Init *> args,
+                           ArrayRef<std::string> argNames) {
+  ios << "if ";
+  auto parenScope = ios.scope("(", ") {");
+  ios.indent();
+
+  auto listHelperName = [](StringRef name) {
+    return formatv("read{0}", capitalize(name));
+  };
+
+  auto parsedArgs =
+      llvm::to_vector(make_filter_range(args, [](Init *const attr) {
+        Record *def = cast<DefInit>(attr)->getDef();
+        if (def->isSubClassOf("Array"))
+          return true;
+        return !def->getValueAsString("cParser").empty();
+      }));
+
+  interleave(
+      zip(parsedArgs, argNames),
+      [&](std::tuple<llvm::Init *&, const std::string &> it) {
+        Record *attr = cast<DefInit>(std::get<0>(it))->getDef();
+        std::string parser;
+        if (auto optParser = attr->getValueAsOptionalString("cParser")) {
+          parser = *optParser;
+        } else if (attr->isSubClassOf("Array")) {
+          Record *def = attr->getValueAsDef("elemT");
+          bool composite = def->isSubClassOf("CompositeBytecode");
+          if (!composite && def->isSubClassOf("AttributeKind"))
+            parser = "succeeded($_reader.readAttributes($_var))";
+          else if (!composite && def->isSubClassOf("TypeKind"))
+            parser = "succeeded($_reader.readTypes($_var))";
+          else
+            parser = ("succeeded($_reader.readList($_var, " +
+                      listHelperName(std::get<1>(it)) + "))")
+                         .str();
+        } else {
+          PrintFatalError(attr->getLoc(), "No parser specified");
+        }
+        std::string type = getCType(attr);
+        ios << format(parser, {{"$_reader", "reader"},
+                               {"$_resultType", type},
+                               {"$_var", std::get<1>(it)}});
+      },
+      [&]() { ios << " &&\n"; });
+}
+
+void Generator::emitParseHelper(StringRef kind, StringRef returnType,
+                                StringRef builder, ArrayRef<Init *> args,
+                                ArrayRef<std::string> argNames,
+                                StringRef failure,
+                                mlir::raw_indented_ostream &ios) {
+  auto funScope = ios.scope("{\n", "}");
+
+  if (args.empty()) {
+    ios << formatv("return get<{0}>(context);\n", returnType);
+    return;
+  }
+
+  // Print decls.
+  std::string lastCType = "";
+  for (auto [arg, name] : zip(args, argNames)) {
+    DefInit *first = dyn_cast<DefInit>(arg);
+    if (!first)
+      PrintFatalError("Unexpected type for " + name);
+    Record *def = first->getDef();
+
+    // Create variable decls, if there are a block of same type then create
+    // comma separated list of them.
+    std::string cType = getCType(def);
+    if (lastCType == cType) {
+      ios << ", ";
+    } else {
+      if (!lastCType.empty())
+        ios << ";\n";
+      ios << cType << " ";
+    }
+    ios << name;
+    lastCType = cType;
+  }
+  ios << ";\n";
+
+  // Returns the name of the helper used in list parsing. E.g., the name of the
+  // lambda passed to array parsing.
+  auto listHelperName = [](StringRef name) {
+    return formatv("read{0}", capitalize(name));
+  };
+
+  // Emit list helper functions.
+  for (auto [arg, name] : zip(args, argNames)) {
+    Record *attr = cast<DefInit>(arg)->getDef();
+    if (!attr->isSubClassOf("Array"))
+      continue;
+
+    // TODO: Dedupe readers.
+    Record *def = attr->getValueAsDef("elemT");
+    if (!def->isSubClassOf("CompositeBytecode") &&
+        (def->isSubClassOf("AttributeKind") || def->isSubClassOf("TypeKind")))
+      continue;
+
+    std::string returnType = getCType(def);
+    ios << "auto " << listHelperName(name) << " = [&]() -> FailureOr<"
+        << returnType << "> ";
+    SmallVector<Init *> args;
+    SmallVector<std::string> argNames;
+    if (def->isSubClassOf("CompositeBytecode")) {
+      DagInit *members = def->getValueAsDag("members");
+      args = llvm::to_vector(members->getArgs());
+      argNames = llvm::to_vector(
+          map_range(members->getArgNames(), [](StringInit *init) {
+            return init->getAsUnquotedString();
+          }));
+    } else {
+      args = {def->getDefInit()};
+      argNames = {"temp"};
+    }
+    StringRef builder = def->getValueAsString("cBuilder");
+    emitParseHelper(kind, returnType, builder, args, argNames, "failure()",
+                    ios);
+    ios << ";\n";
+  }
+
+  // Print parse conditional.
+  printParseConditional(ios, args, argNames);
+
+  // Compute args to pass to create method.
+  auto passedArgs = llvm::to_vector(make_filter_range(
+      argNames, [](StringRef str) { return !str.starts_with("_"); }));
+  std::string argStr;
+  raw_string_ostream argStream(argStr);
+  interleaveComma(passedArgs, argStream,
+                  [&](const std::string &str) { argStream << str; });
+  // Return the invoked constructor.
+  ios << "\nreturn "
+      << format(builder, {{"$_resultType", returnType.str()},
+                          {"$_args", argStream.str()}})
+      << ";\n";
+  ios.unindent();
+
+  // TODO: Emit error in debug.
+  // This assumes the result types in error case can always be empty
+  // constructed.
+  ios << "}\nreturn " << failure << ";\n";
+}
+
+void Generator::emitPrint(StringRef kind, StringRef type,
+                          ArrayRef<std::pair<int64_t, Record *>> vec) {
+  char const *head =
+      R"(static void write({0} {1}, DialectBytecodeWriter &writer) )";
+  mlir::raw_indented_ostream os(output);
+  os << formatv(head, type, kind);
+  auto funScope = os.scope("{\n", "}\n\n");
+
+  // Check that predicates specified if multiple bytecode instances.
+  for (llvm::Record *rec : make_second_range(vec)) {
+    StringRef pred = rec->getValueAsString("printerPredicate");
+    if (vec.size() > 1 && pred.empty()) {
+      for (auto [index, rec] : vec) {
+        (void)index;
+        StringRef pred = rec->getValueAsString("printerPredicate");
+        if (vec.size() > 1 && pred.empty())
+          PrintError(rec->getLoc(),
+                     "Requires parsing predicate given common cType");
+      }
+      PrintFatalError("Unspecified for shared cType " + type);
+    }
+  }
+
+  for (auto [index, rec] : vec) {
+    StringRef pred = rec->getValueAsString("printerPredicate");
+    if (!pred.empty()) {
+      os << "if (" << format(pred, {{"$_val", kind.str()}}) << ") {\n";
+      os.indent();
+    }
+
+    os << "writer.writeVarInt(/* " << rec->getName() << " */ " << index
+       << ");\n";
+
+    auto *members = rec->getValueAsDag("members");
+    for (auto [arg, name] :
+         llvm::zip(members->getArgs(), members->getArgNames())) {
+      DefInit *def = dyn_cast<DefInit>(arg);
+      assert(def);
+      Record *memberRec = def->getDef();
+      emitPrintHelper(memberRec, kind, kind, name->getAsUnquotedString(), os);
+    }
+
+    if (!pred.empty()) {
+      os.unindent();
+      os << "}\n";
+    }
+  }
+}
+
+void Generator::emitPrintHelper(Record *memberRec, StringRef kind,
+                                StringRef parent, StringRef name,
+                                mlir::raw_indented_ostream &ios) {
+  std::string getter;
+  if (auto cGetter = memberRec->getValueAsOptionalString("cGetter");
+      cGetter && !cGetter->empty()) {
+    getter = format(
+        *cGetter,
+        {{"$_attrType", parent.str()},
+         {"$_member", name.str()},
+         {"$_getMember", "get" + convertToCamelFromSnakeCase(name, true)}});
+  } else {
+    getter =
+        formatv("{0}.get{1}()", parent, convertToCamelFromSnakeCase(name, true))
+            .str();
+  }
+
+  if (memberRec->isSubClassOf("Array")) {
+    Record *def = memberRec->getValueAsDef("elemT");
+    if (!def->isSubClassOf("CompositeBytecode")) {
+      if (def->isSubClassOf("AttributeKind")) {
+        ios << "writer.writeAttributes(" << getter << ");\n";
+        return;
+      }
+      if (def->isSubClassOf("TypeKind")) {
+        ios << "writer.writeTypes(" << getter << ");\n";
+        return;
+      }
+    }
+    std::string returnType = getCType(def);
+    ios << "writer.writeList(" << getter << ", [&](" << returnType << " "
+        << kind << ") ";
+    auto lambdaScope = ios.scope("{\n", "});\n");
+    return emitPrintHelper(def, kind, kind, kind, ios);
+  }
+  if (memberRec->isSubClassOf("CompositeBytecode")) {
+    auto *members = memberRec->getValueAsDag("members");
+    for (auto [arg, argName] :
+         zip(members->getArgs(), members->getArgNames())) {
+      DefInit *def = dyn_cast<DefInit>(arg);
+      assert(def);
+      emitPrintHelper(def->getDef(), kind, parent,
+                      argName->getAsUnquotedString(), ios);
+    }
+  }
+
+  if (std::string printer = memberRec->getValueAsString("cPrinter").str();
+      !printer.empty())
+    ios << format(printer, {{"$_writer", "writer"},
+                            {"$_name", kind.str()},
+                            {"$_getter", getter}})
+        << ";\n";
+}
+
+void Generator::emitPrintDispatch(StringRef kind, ArrayRef<std::string> vec) {
+  mlir::raw_indented_ostream os(output);
+  char const *head = R"(static LogicalResult write{0}({0} {1},
+                                DialectBytecodeWriter &writer))";
+  os << formatv(head, capitalize(kind), kind);
+  auto funScope = os.scope(" {\n", "}\n\n");
+
+  os << "return TypeSwitch<" << capitalize(kind) << ", LogicalResult>(" << kind
+     << ")";
+  auto switchScope = os.scope("", "");
+  for (StringRef type : vec) {
+    os << "\n.Case([&](" << type << " t)";
+    auto caseScope = os.scope(" {\n", "})");
+    os << "return write(t, writer), success();\n";
+  }
+  os << "\n.Default([&](" << capitalize(kind) << ") { return failure(); });\n";
+}
+
+namespace {
+/// Container of Attribute or Type for Dialect.
+struct AttrOrType {
+  std::vector<Record *> attr, type;
+};
+} // namespace
+
+static bool emitBCRW(const RecordKeeper &records, raw_ostream &os) {
+  MapVector<StringRef, AttrOrType> dialectAttrOrType;
+  for (auto &it : records.getAllDerivedDefinitions("DialectAttributes")) {
+    if (!selectedBcDialect.empty() &&
+        it->getValueAsString("dialect") != selectedBcDialect)
+      continue;
+    dialectAttrOrType[it->getValueAsString("dialect")].attr =
+        it->getValueAsListOfDefs("elems");
+  }
+  for (auto &it : records.getAllDerivedDefinitions("DialectTypes")) {
+    if (!selectedBcDialect.empty() &&
+        it->getValueAsString("dialect") != selectedBcDialect)
+      continue;
+    dialectAttrOrType[it->getValueAsString("dialect")].type =
+        it->getValueAsListOfDefs("elems");
+  }
+
+  if (dialectAttrOrType.size() != 1)
+    PrintFatalError("Single dialect per invocation required (either only "
+                    "one in input file or specified via dialect option)");
+
+  auto it = dialectAttrOrType.front();
+  Generator gen(os);
+
+  SmallVector<std::vector<Record *> *, 2> vecs;
+  SmallVector<std::string, 2> kinds;
+  vecs.push_back(&it.second.attr);
+  kinds.push_back("attribute");
+  vecs.push_back(&it.second.type);
+  kinds.push_back("type");
+  for (auto [vec, kind] : zip(vecs, kinds)) {
+    // Handle Attribute/Type emission.
+    std::map<std::string, std::vector<std::pair<int64_t, Record *>>> perType;
+    for (auto kt : llvm::enumerate(*vec))
+      perType[getCType(kt.value())].emplace_back(kt.index(), kt.value());
+    for (const auto &jt : perType) {
+      for (auto kt : jt.second)
+        gen.emitParse(kind, *std::get<1>(kt));
+      gen.emitPrint(kind, jt.first, jt.second);
+    }
+    gen.emitParseDispatch(kind, *vec);
+
+    SmallVector<std::string> types;
+    for (const auto &it : perType) {
+      types.push_back(it.first);
+    }
+    gen.emitPrintDispatch(kind, types);
+  }
+
+  return false;
+}
+
+static mlir::GenRegistration
+    genBCRW("gen-bytecode", "Generate dialect bytecode readers/writers",
+            [](const RecordKeeper &records, raw_ostream &os) {
+              return emitBCRW(records, os);
+            });
diff --git a/mlir/tools/mlir-tblgen/CMakeLists.txt b/mlir/tools/mlir-tblgen/CMakeLists.txt
index fe217450d6fb6..0d9ece2f061e7 100644
--- a/mlir/tools/mlir-tblgen/CMakeLists.txt
+++ b/mlir/tools/mlir-tblgen/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  CodeGenTypes
   Demangle
   Support
   TableGen
@@ -9,6 +10,7 @@ add_tablegen(mlir-tblgen MLIR
   EXPORT MLIR
   AttrOrTypeDefGen.cpp
   AttrOrTypeFormatGen.cpp
+  BytecodeDialectGen.cpp
   DialectGen.cpp
   DirectiveCommonGen.cpp
   EnumsGen.cpp
diff --git a/mlir/tools/mlir-tblgen/FormatGen.cpp b/mlir/tools/mlir-tblgen/FormatGen.cpp
index 7d2e03ecfe278..b4f71fb45b376 100644
--- a/mlir/tools/mlir-tblgen/FormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/FormatGen.cpp
@@ -177,6 +177,7 @@ FormatToken FormatLexer::lexIdentifier(const char *tokStart) {
       StringSwitch<FormatToken::Kind>(str)
           .Case("attr-dict", FormatToken::kw_attr_dict)
           .Case("attr-dict-with-keyword", FormatToken::kw_attr_dict_w_keyword)
+          .Case("prop-dict", FormatToken::kw_prop_dict)
           .Case("custom", FormatToken::kw_custom)
           .Case("functional-type", FormatToken::kw_functional_type)
           .Case("oilist", FormatToken::kw_oilist)
diff --git a/mlir/tools/mlir-tblgen/FormatGen.h b/mlir/tools/mlir-tblgen/FormatGen.h
index e5fd04a24b2f5..da30e35f13532 100644
--- a/mlir/tools/mlir-tblgen/FormatGen.h
+++ b/mlir/tools/mlir-tblgen/FormatGen.h
@@ -60,6 +60,7 @@ class FormatToken {
     keyword_start,
     kw_attr_dict,
     kw_attr_dict_w_keyword,
+    kw_prop_dict,
     kw_custom,
     kw_functional_type,
     kw_oilist,
@@ -287,6 +288,7 @@ class DirectiveElement : public FormatElementBase<FormatElement::Directive> {
   /// These are the kinds of directives.
   enum Kind {
     AttrDict,
+    PropDict,
     Custom,
     FunctionalType,
     OIList,
diff --git a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
index 71bd51394c6eb..4fa037aafde78 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
@@ -14,8 +14,8 @@
 #include "mlir/TableGen/GenInfo.h"
 
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/Signals.h"
diff --git a/mlir/tools/mlir-tblgen/OpClass.cpp b/mlir/tools/mlir-tblgen/OpClass.cpp
index 40b688f2b96ca..698569c790e93 100644
--- a/mlir/tools/mlir-tblgen/OpClass.cpp
+++ b/mlir/tools/mlir-tblgen/OpClass.cpp
@@ -37,5 +37,6 @@ OpClass::OpClass(StringRef name, StringRef extraClassDeclaration,
 void OpClass::finalize() {
   Class::finalize();
   declare<VisibilityDeclaration>(Visibility::Public);
-  declare<ExtraClassDeclaration>(extraClassDeclaration, extraClassDefinition);
+  declare<ExtraClassDeclaration>(extraClassDeclaration.str(),
+                                 extraClassDefinition);
 }
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index e3e7ae087085a..dc257fd97b0e0 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -14,20 +14,25 @@
 #include "OpClass.h"
 #include "OpFormatGen.h"
 #include "OpGenHelpers.h"
+#include "mlir/TableGen/Argument.h"
+#include "mlir/TableGen/Attribute.h"
 #include "mlir/TableGen/Class.h"
 #include "mlir/TableGen/CodeGenHelpers.h"
 #include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/GenInfo.h"
 #include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/Operator.h"
+#include "mlir/TableGen/Property.h"
 #include "mlir/TableGen/SideEffects.h"
 #include "mlir/TableGen/Trait.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -43,6 +48,10 @@ static const char *const tblgenNamePrefix = "tblgen_";
 static const char *const generatedArgName = "odsArg";
 static const char *const odsBuilder = "odsBuilder";
 static const char *const builderOpState = "odsState";
+static const char *const propertyStorage = "propStorage";
+static const char *const propertyValue = "propValue";
+static const char *const propertyAttr = "propAttr";
+static const char *const propertyDiag = "propDiag";
 
 /// The names of the implicit attributes that contain variadic operand and
 /// result segment sizes.
@@ -103,7 +112,7 @@ static const char *const attrSizedSegmentValueRangeCalcCode = R"(
 ///
 /// {0}: The code to get the attribute.
 static const char *const adapterSegmentSizeAttrInitCode = R"(
-  assert(odsAttrs && "missing segment size attribute for op");
+  assert({0} && "missing segment size attribute for op");
   auto sizeAttr = {0}.cast<::mlir::DenseI32ArrayAttr>();
 )";
 /// The code snippet to initialize the sizes for the value range calculation.
@@ -260,6 +269,10 @@ class OpOrAdaptorHelper {
     assert(attrMetadata.count(attrName) && "expected attribute metadata");
     return [this, attrName, isNamed](raw_ostream &os) -> raw_ostream & {
       const AttributeMetadata &attr = attrMetadata.find(attrName)->second;
+      if (hasProperties()) {
+        assert(!isNamed);
+        return os << "getProperties()." << attrName;
+      }
       return os << formatv(subrangeGetAttr, getAttrName(attrName),
                            attr.lowerBound, attr.upperBound, getAttrRange(),
                            isNamed ? "Named" : "");
@@ -324,6 +337,19 @@ class OpOrAdaptorHelper {
     return attrMetadata;
   }
 
+  /// Returns whether to emit a `Properties` struct for this operation or not.
+  bool hasProperties() const {
+    if (!op.getProperties().empty())
+      return true;
+    if (!op.getDialect().usePropertiesForAttributes())
+      return false;
+    return llvm::any_of(getAttrMetadata(),
+                        [](const std::pair<StringRef, AttributeMetadata> &it) {
+                          return !it.second.constraint ||
+                                 !it.second.constraint->isDerivedAttr();
+                        });
+  }
+
 private:
   // Compute the attribute metadata.
   void computeAttrMetadata();
@@ -418,6 +444,9 @@ class OpEmitter {
   // Generates the `getOperationName` method for this op.
   void genOpNameGetter();
 
+  // Generates code to manage the properties, if any!
+  void genPropertiesSupport();
+
   // Generates getters for the attributes.
   void genAttrGetters();
 
@@ -642,6 +671,20 @@ static void genNativeTraitAttrVerifier(MethodBody &body,
   }
 }
 
+// Return true if a verifier can be emitted for the attribute: it is not a
+// derived attribute, it has a predicate, its condition is not empty, and, for
+// adaptors, the condition does not reference the op.
+static bool canEmitAttrVerifier(Attribute attr, bool isEmittingForOp) {
+  if (attr.isDerivedAttr())
+    return false;
+  Pred pred = attr.getPredicate();
+  if (pred.isNull())
+    return false;
+  std::string condition = pred.getCondition();
+  return !condition.empty() &&
+         (!StringRef(condition).contains("$_op") || isEmittingForOp);
+}
+
 // Generate attribute verification. If an op instance is not available, then
 // attribute checks that require one will not be emitted.
 //
@@ -654,9 +697,11 @@ static void genNativeTraitAttrVerifier(MethodBody &body,
 // that depend on the validity of these attributes, e.g. segment size attributes
 // and operand or result getters.
 // 3. Verify the constraints on all present attributes.
-static void genAttributeVerifier(
-    const OpOrAdaptorHelper &emitHelper, FmtContext &ctx, MethodBody &body,
-    const StaticVerifierFunctionEmitter &staticVerifierEmitter) {
+static void
+genAttributeVerifier(const OpOrAdaptorHelper &emitHelper, FmtContext &ctx,
+                     MethodBody &body,
+                     const StaticVerifierFunctionEmitter &staticVerifierEmitter,
+                     bool useProperties) {
   if (emitHelper.getAttrMetadata().empty())
     return;
 
@@ -691,7 +736,8 @@ static void genAttributeVerifier(
   // {0}: Code to get the name of the attribute.
   // {1}: The emit error prefix.
   // {2}: The name of the attribute.
-  const char *const findRequiredAttr = R"(while (true) {{
+  const char *const findRequiredAttr = R"(
+while (true) {{
   if (namedAttrIt == namedAttrRange.end())
     return {1}"requires attribute '{2}'");
   if (namedAttrIt->getName() == {0}) {{
@@ -714,20 +760,6 @@ static void genAttributeVerifier(
     break;
   })";
 
-  // Return true if a verifier can be emitted for the attribute: it is not a
-  // derived attribute, it has a predicate, its condition is not empty, and, for
-  // adaptors, the condition does not reference the op.
-  const auto canEmitVerifier = [&](Attribute attr) {
-    if (attr.isDerivedAttr())
-      return false;
-    Pred pred = attr.getPredicate();
-    if (pred.isNull())
-      return false;
-    std::string condition = pred.getCondition();
-    return !condition.empty() && (!StringRef(condition).contains("$_op") ||
-                                  emitHelper.isEmittingForOp());
-  };
-
   // Emit the verifier for the attribute.
   const auto emitVerifier = [&](Attribute attr, StringRef attrName,
                                 StringRef varName) {
@@ -750,58 +782,74 @@ static void genAttributeVerifier(
     return (tblgenNamePrefix + attrName).str();
   };
 
-  body.indent() << formatv("auto namedAttrRange = {0};\n",
-                           emitHelper.getAttrRange());
-  body << "auto namedAttrIt = namedAttrRange.begin();\n";
-
-  // Iterate over the attributes in sorted order. Keep track of the optional
-  // attributes that may be encountered along the way.
-  SmallVector<const AttributeMetadata *> optionalAttrs;
-  for (const std::pair<StringRef, AttributeMetadata> &it :
-       emitHelper.getAttrMetadata()) {
-    const AttributeMetadata &metadata = it.second;
-    if (!metadata.isRequired) {
-      optionalAttrs.push_back(&metadata);
-      continue;
+  body.indent();
+  if (useProperties) {
+    for (const std::pair<StringRef, AttributeMetadata> &it :
+         emitHelper.getAttrMetadata()) {
+      body << formatv(
+          "auto tblgen_{0} = getProperties().{0}; (void)tblgen_{0};\n",
+          it.first);
+      const AttributeMetadata &metadata = it.second;
+      if (metadata.isRequired)
+        body << formatv(
+            "if (!tblgen_{0}) return {1}\"requires attribute '{0}'\");\n",
+            it.first, emitHelper.emitErrorPrefix());
     }
+  } else {
+    body << formatv("auto namedAttrRange = {0};\n", emitHelper.getAttrRange());
+    body << "auto namedAttrIt = namedAttrRange.begin();\n";
+
+    // Iterate over the attributes in sorted order. Keep track of the optional
+    // attributes that may be encountered along the way.
+    SmallVector<const AttributeMetadata *> optionalAttrs;
+
+    for (const std::pair<StringRef, AttributeMetadata> &it :
+         emitHelper.getAttrMetadata()) {
+      const AttributeMetadata &metadata = it.second;
+      if (!metadata.isRequired) {
+        optionalAttrs.push_back(&metadata);
+        continue;
+      }
 
-    body << formatv("::mlir::Attribute {0};\n", getVarName(it.first));
-    for (const AttributeMetadata *optional : optionalAttrs) {
-      body << formatv("::mlir::Attribute {0};\n",
-                      getVarName(optional->attrName));
-    }
-    body << formatv(findRequiredAttr, emitHelper.getAttrName(it.first),
-                    emitHelper.emitErrorPrefix(), it.first);
-    for (const AttributeMetadata *optional : optionalAttrs) {
-      body << formatv(checkOptionalAttr,
-                      emitHelper.getAttrName(optional->attrName),
-                      optional->attrName);
-    }
-    body << "\n  ++namedAttrIt;\n}\n";
-    optionalAttrs.clear();
-  }
-  // Get trailing optional attributes.
-  if (!optionalAttrs.empty()) {
-    for (const AttributeMetadata *optional : optionalAttrs) {
-      body << formatv("::mlir::Attribute {0};\n",
-                      getVarName(optional->attrName));
+      body << formatv("::mlir::Attribute {0};\n", getVarName(it.first));
+      for (const AttributeMetadata *optional : optionalAttrs) {
+        body << formatv("::mlir::Attribute {0};\n",
+                        getVarName(optional->attrName));
+      }
+      body << formatv(findRequiredAttr, emitHelper.getAttrName(it.first),
+                      emitHelper.emitErrorPrefix(), it.first);
+      for (const AttributeMetadata *optional : optionalAttrs) {
+        body << formatv(checkOptionalAttr,
+                        emitHelper.getAttrName(optional->attrName),
+                        optional->attrName);
+      }
+      body << "\n  ++namedAttrIt;\n}\n";
+      optionalAttrs.clear();
     }
-    body << checkTrailingAttrs;
-    for (const AttributeMetadata *optional : optionalAttrs) {
-      body << formatv(checkOptionalAttr,
-                      emitHelper.getAttrName(optional->attrName),
-                      optional->attrName);
+    // Get trailing optional attributes.
+    if (!optionalAttrs.empty()) {
+      for (const AttributeMetadata *optional : optionalAttrs) {
+        body << formatv("::mlir::Attribute {0};\n",
+                        getVarName(optional->attrName));
+      }
+      body << checkTrailingAttrs;
+      for (const AttributeMetadata *optional : optionalAttrs) {
+        body << formatv(checkOptionalAttr,
+                        emitHelper.getAttrName(optional->attrName),
+                        optional->attrName);
+      }
+      body << "\n  ++namedAttrIt;\n}\n";
     }
-    body << "\n  ++namedAttrIt;\n}\n";
   }
   body.unindent();
 
-  // Emit the checks for segment attributes first so that the other constraints
-  // can call operand and result getters.
+  // Emit the checks for segment attributes first so that the other
+  // constraints can call operand and result getters.
   genNativeTraitAttrVerifier(body, emitHelper);
 
+  bool isEmittingForOp = emitHelper.isEmittingForOp();
   for (const auto &namedAttr : emitHelper.getOp().getAttributes())
-    if (canEmitVerifier(namedAttr.attr))
+    if (canEmitAttrVerifier(namedAttr.attr, isEmittingForOp))
       emitVerifier(namedAttr.attr, namedAttr.name, getVarName(namedAttr.name));
 }
 
@@ -834,6 +882,7 @@ OpEmitter::OpEmitter(const Operator &op,
   genNamedResultGetters();
   genNamedRegionGetters();
   genNamedSuccessorGetters();
+  genPropertiesSupport();
   genAttrGetters();
   genAttrSetters();
   genOptionalAttrRemovers();
@@ -989,6 +1038,274 @@ static void emitAttrGetterWithReturnType(FmtContext &fctx,
        << ";\n";
 }
 
+void OpEmitter::genPropertiesSupport() {
+  if (!emitHelper.hasProperties())
+    return;
+  using ConstArgument =
+      llvm::PointerUnion<const AttributeMetadata *, const NamedProperty *>;
+
+  SmallVector<ConstArgument> attrOrProperties;
+  for (const std::pair<StringRef, AttributeMetadata> &it :
+       emitHelper.getAttrMetadata()) {
+    if (!it.second.constraint || !it.second.constraint->isDerivedAttr())
+      attrOrProperties.push_back(&it.second);
+  }
+  for (const NamedProperty &prop : op.getProperties())
+    attrOrProperties.push_back(&prop);
+  if (attrOrProperties.empty())
+    return;
+  auto &setPropMethod =
+      opClass
+          .addStaticMethod(
+              "::mlir::LogicalResult", "setPropertiesFromAttr",
+              MethodParameter("Properties &", "prop"),
+              MethodParameter("::mlir::Attribute", "attr"),
+              MethodParameter("::mlir::InFlightDiagnostic *", "diag"))
+          ->body();
+  auto &getPropMethod =
+      opClass
+          .addStaticMethod("::mlir::Attribute", "getPropertiesAsAttr",
+                           MethodParameter("::mlir::MLIRContext *", "ctx"),
+                           MethodParameter("const Properties &", "prop"))
+          ->body();
+  auto &hashMethod =
+      opClass
+          .addStaticMethod("llvm::hash_code", "computePropertiesHash",
+                           MethodParameter("const Properties &", "prop"))
+          ->body();
+  auto &getInherentAttrMethod =
+      opClass
+          .addStaticMethod("std::optional<mlir::Attribute>", "getInherentAttr",
+                           MethodParameter("const Properties &", "prop"),
+                           MethodParameter("llvm::StringRef", "name"))
+          ->body();
+  auto &setInherentAttrMethod =
+      opClass
+          .addStaticMethod("void", "setInherentAttr",
+                           MethodParameter("Properties &", "prop"),
+                           MethodParameter("llvm::StringRef", "name"),
+                           MethodParameter("mlir::Attribute", "value"))
+          ->body();
+  auto &populateInherentAttrsMethod =
+      opClass
+          .addStaticMethod("void", "populateInherentAttrs",
+                           MethodParameter("const Properties &", "prop"),
+                           MethodParameter("::mlir::NamedAttrList &", "attrs"))
+          ->body();
+  auto &verifyInherentAttrsMethod =
+      opClass
+          .addStaticMethod(
+              "::mlir::LogicalResult", "verifyInherentAttrs",
+              MethodParameter("::mlir::OperationName", "opName"),
+              MethodParameter("::mlir::NamedAttrList &", "attrs"),
+              MethodParameter(
+                  "llvm::function_ref<::mlir::InFlightDiagnostic()>",
+                  "getDiag"))
+          ->body();
+
+  opClass.declare<UsingDeclaration>("Properties", "FoldAdaptor::Properties");
+
+  // Convert the property to the attribute form.
+
+  setPropMethod << R"decl(
+  ::mlir::DictionaryAttr dict = dyn_cast<::mlir::DictionaryAttr>(attr);
+  if (!dict) {
+    if (diag)
+      *diag << "expected DictionaryAttr to set properties";
+    return failure();
+  }
+    )decl";
+  // TODO: properties might be optional as well.
+  const char *propFromAttrFmt = R"decl(;
+    {{
+      auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr, 
+                             ::mlir::InFlightDiagnostic *propDiag) {{
+        {0};
+      };
+      auto attr = dict.get("{1}");
+      if (!attr) {{
+        if (diag)
+          *diag << "expected key entry for {1} in DictionaryAttr to set "
+                   "Properties.";
+        return failure();
+      }
+      if (failed(setFromAttr(prop.{1}, attr, diag))) return ::mlir::failure();
+    }
+)decl";
+  for (const auto &attrOrProp : attrOrProperties) {
+    if (const auto *namedProperty =
+            attrOrProp.dyn_cast<const NamedProperty *>()) {
+      StringRef name = namedProperty->name;
+      auto &prop = namedProperty->prop;
+      FmtContext fctx;
+      setPropMethod << formatv(propFromAttrFmt,
+                               tgfmt(prop.getConvertFromAttributeCall(),
+                                     &fctx.addSubst("_attr", propertyAttr)
+                                          .addSubst("_storage", propertyStorage)
+                                          .addSubst("_diag", propertyDiag)),
+                               name);
+    } else {
+      const auto *namedAttr = attrOrProp.dyn_cast<const AttributeMetadata *>();
+      StringRef name = namedAttr->attrName;
+      setPropMethod << formatv(R"decl(
+  {{
+    auto &propStorage = prop.{0};
+    auto attr = dict.get("{0}");
+    if (attr || /*isRequired=*/{1}) {{
+      if (!attr) {{
+        if (diag)
+          *diag << "expected key entry for {0} in DictionaryAttr to set "
+                   "Properties.";
+        return failure();
+      }
+      auto convertedAttr = dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
+      if (convertedAttr) {{
+        propStorage = convertedAttr;
+      } else {{
+        if (diag)
+          *diag << "Invalid attribute `{0}` in property conversion: " << attr;
+        return failure();
+      }
+    }
+  }
+)decl",
+                               name, namedAttr->isRequired);
+    }
+  }
+  setPropMethod << "  return ::mlir::success();\n";
+
+  // Convert the attribute form to the property.
+
+  getPropMethod << "    ::mlir::SmallVector<::mlir::NamedAttribute> attrs;\n"
+                << "    ::mlir::Builder odsBuilder{ctx};\n";
+  const char *propToAttrFmt = R"decl(
+    {
+      const auto &propStorage = prop.{0};
+      attrs.push_back(odsBuilder.getNamedAttr("{0}",
+                                              {1}));
+    }
+)decl";
+  for (const auto &attrOrProp : attrOrProperties) {
+    if (const auto *namedProperty =
+            attrOrProp.dyn_cast<const NamedProperty *>()) {
+      StringRef name = namedProperty->name;
+      auto &prop = namedProperty->prop;
+      FmtContext fctx;
+      getPropMethod << formatv(
+          propToAttrFmt, name,
+          tgfmt(prop.getConvertToAttributeCall(),
+                &fctx.addSubst("_ctxt", "ctx")
+                     .addSubst("_storage", propertyStorage)));
+      continue;
+    }
+    const auto *namedAttr = attrOrProp.dyn_cast<const AttributeMetadata *>();
+    StringRef name = namedAttr->attrName;
+    getPropMethod << formatv(R"decl(
+    {{
+      const auto &propStorage = prop.{0};
+      if (propStorage)
+        attrs.push_back(odsBuilder.getNamedAttr("{0}",
+                                       propStorage));
+    }
+)decl",
+                             name);
+  }
+  getPropMethod << R"decl(
+  if (!attrs.empty())
+    return odsBuilder.getDictionaryAttr(attrs);
+  return {};    
+)decl";
+
+  // Hashing for the property
+
+  const char *propHashFmt = R"decl(
+  auto hash_{0} = [] (const auto &propStorage) -> llvm::hash_code {
+    return {1};
+  };
+)decl";
+  for (const auto &attrOrProp : attrOrProperties) {
+    if (const auto *namedProperty =
+            attrOrProp.dyn_cast<const NamedProperty *>()) {
+      StringRef name = namedProperty->name;
+      auto &prop = namedProperty->prop;
+      FmtContext fctx;
+      hashMethod << formatv(propHashFmt, name,
+                            tgfmt(prop.getHashPropertyCall(),
+                                  &fctx.addSubst("_storage", propertyStorage)));
+    }
+  }
+  hashMethod << "  return llvm::hash_combine(";
+  llvm::interleaveComma(
+      attrOrProperties, hashMethod, [&](const ConstArgument &attrOrProp) {
+        if (const auto *namedProperty =
+                attrOrProp.dyn_cast<const NamedProperty *>()) {
+          hashMethod << "\n    hash_" << namedProperty->name << "(prop."
+                     << namedProperty->name << ")";
+          return;
+        }
+        const auto *namedAttr =
+            attrOrProp.dyn_cast<const AttributeMetadata *>();
+        StringRef name = namedAttr->attrName;
+        hashMethod << "\n    llvm::hash_value(prop." << name
+                   << ".getAsOpaquePointer())";
+      });
+  hashMethod << ");\n";
+
+  const char *getInherentAttrMethodFmt = R"decl(
+    if (name == "{0}")
+      return prop.{0};
+)decl";
+  const char *setInherentAttrMethodFmt = R"decl(
+    if (name == "{0}") {{
+       prop.{0} = dyn_cast_or_null<std::remove_reference_t<decltype(prop.{0})>>(value);
+       return;
+    }
+)decl";
+  const char *populateInherentAttrsMethodFmt = R"decl(
+    if (prop.{0}) attrs.append("{0}", prop.{0});
+)decl";
+  for (const auto &attrOrProp : attrOrProperties) {
+    if (const auto *namedAttr =
+            attrOrProp.dyn_cast<const AttributeMetadata *>()) {
+      StringRef name = namedAttr->attrName;
+      getInherentAttrMethod << formatv(getInherentAttrMethodFmt, name);
+      setInherentAttrMethod << formatv(setInherentAttrMethodFmt, name);
+      populateInherentAttrsMethod
+          << formatv(populateInherentAttrsMethodFmt, name);
+      continue;
+    }
+  }
+  getInherentAttrMethod << "  return std::nullopt;\n";
+
+  // Emit the verifiers method for backward compatibility with the generic
+  // syntax. This method verifies the constraint on the properties attributes
+  // before they are set, since dyn_cast<> will silently omit failures.
+  for (const auto &attrOrProp : attrOrProperties) {
+    const auto *namedAttr = attrOrProp.dyn_cast<const AttributeMetadata *>();
+    if (!namedAttr || !namedAttr->constraint)
+      continue;
+    Attribute attr = *namedAttr->constraint;
+    std::optional<StringRef> constraintFn =
+        staticVerifierEmitter.getAttrConstraintFn(attr);
+    if (!constraintFn)
+      continue;
+    if (canEmitAttrVerifier(attr,
+                            /*isEmittingForOp=*/false)) {
+      std::string name = op.getGetterName(namedAttr->attrName);
+      verifyInherentAttrsMethod
+          << formatv(R"(
+    {{
+      ::mlir::Attribute attr = attrs.get({0}AttrName(opName));
+      if (attr && ::mlir::failed({1}(attr, "{2}", getDiag)))
+        return ::mlir::failure();
+    }
+)",
+                     name, constraintFn, namedAttr->attrName);
+    }
+  }
+  verifyInherentAttrsMethod << "    return ::mlir::success();";
+}
+
 void OpEmitter::genAttrGetters() {
   FmtContext fctx;
   fctx.withBuilder("::mlir::Builder((*this)->getContext())");
@@ -999,9 +1316,9 @@ void OpEmitter::genAttrGetters() {
       method->body() << "  " << attr.getDerivedCodeBody() << "\n";
   };
 
-  // Generate named accessor with Attribute return type. This is a wrapper class
-  // that allows referring to the attributes via accessors instead of having to
-  // use the string interface for better compile time verification.
+  // Generate named accessor with Attribute return type. This is a wrapper
+  // class that allows referring to the attributes via accessors instead of
+  // having to use the string interface for better compile time verification.
   auto emitAttrWithStorageType = [&](StringRef name, StringRef attrName,
                                      Attribute attr) {
     auto *method = opClass.addMethod(attr.getStorageType(), name + "Attr");
@@ -1086,7 +1403,8 @@ void OpEmitter::genAttrGetters() {
           body << "    {" << name << "AttrName(),\n"
                << tgfmt(tmpl, &fctx.withSelf(name + "()")
                                    .withBuilder("odsBuilder")
-                                   .addSubst("_ctxt", "ctx"))
+                                   .addSubst("_ctxt", "ctx")
+                                   .addSubst("_storage", "ctx"))
                << "}";
         },
         ",\n");
@@ -1175,19 +1493,29 @@ void OpEmitter::genAttrSetters() {
 void OpEmitter::genOptionalAttrRemovers() {
   // Generate methods for removing optional attributes, instead of having to
   // use the string interface. Enables better compile time verification.
-  auto emitRemoveAttr = [&](StringRef name) {
+  auto emitRemoveAttr = [&](StringRef name, bool useProperties) {
     auto upperInitial = name.take_front().upper();
     auto *method = opClass.addMethod("::mlir::Attribute",
                                      op.getRemoverName(name) + "Attr");
     if (!method)
       return;
-    method->body() << formatv("  return (*this)->removeAttr({0}AttrName());",
+    if (useProperties) {
+      method->body() << formatv(R"(
+    auto &attr = getProperties().{0};
+    attr = {{};
+    return attr;
+)",
+                                name);
+      return;
+    }
+    method->body() << formatv("return (*this)->removeAttr({0}AttrName());",
                               op.getGetterName(name));
   };
 
   for (const NamedAttribute &namedAttr : op.getAttributes())
     if (namedAttr.attr.isOptional())
-      emitRemoveAttr(namedAttr.name);
+      emitRemoveAttr(namedAttr.name,
+                     op.getDialect().usePropertiesForAttributes());
 }
 
 // Generates the code to compute the start and end index of an operand or result
@@ -1417,9 +1745,15 @@ void OpEmitter::genNamedOperandSetters() {
             "::mlir::MutableOperandRange(getOperation(), "
             "range.first, range.second";
     if (attrSizedOperands) {
-      body << formatv(
-          ", ::mlir::MutableOperandRange::OperandSegment({0}u, *{1})", i,
-          emitHelper.getAttr(operandSegmentAttrName, /*isNamed=*/true));
+      if (emitHelper.hasProperties())
+        body << formatv(
+            ", ::mlir::MutableOperandRange::OperandSegment({0}u, "
+            "{getOperandSegmentSizesAttrName(), getProperties().{1}})",
+            i, operandSegmentAttrName);
+      else
+        body << formatv(
+            ", ::mlir::MutableOperandRange::OperandSegment({0}u, *{1})", i,
+            emitHelper.getAttr(operandSegmentAttrName, /*isNamed=*/true));
     }
     body << ");\n";
 
@@ -1623,6 +1957,7 @@ void OpEmitter::genSeparateArgParamBuilder() {
         if (::mlir::succeeded({0}::inferReturnTypes(odsBuilder.getContext(),
                       {1}.location, {1}.operands,
                       {1}.attributes.getDictionary({1}.getContext()),
+                      {1}.getRawProperties(),
                       {1}.regions, inferredReturnTypes)))
           {1}.addTypes(inferredReturnTypes);
         else
@@ -1645,11 +1980,17 @@ void OpEmitter::genSeparateArgParamBuilder() {
       // Automatically create the 'result_segment_sizes' attribute using
       // the length of the type ranges.
       if (op.getTrait("::mlir::OpTrait::AttrSizedResultSegments")) {
-        std::string getterName = op.getGetterName(resultSegmentAttrName);
-        body << " " << builderOpState << ".addAttribute(" << getterName
-             << "AttrName(" << builderOpState << ".name), "
-             << "odsBuilder.getDenseI32ArrayAttr({";
-
+        if (op.getDialect().usePropertiesForAttributes()) {
+          body << "  (" << builderOpState
+               << ".getOrAddProperties<Properties>()." << resultSegmentAttrName
+               << " = \n"
+                  "        odsBuilder.getDenseI32ArrayAttr({";
+        } else {
+          std::string getterName = op.getGetterName(resultSegmentAttrName);
+          body << " " << builderOpState << ".addAttribute(" << getterName
+               << "AttrName(" << builderOpState << ".name), "
+               << "odsBuilder.getDenseI32ArrayAttr({";
+        }
         interleaveComma(
             llvm::seq<int>(0, op.getNumResults()), body, [&](int i) {
               const NamedTypeConstraint &result = op.getResult(i);
@@ -1748,6 +2089,32 @@ void OpEmitter::genPopulateDefaultAttributes() {
       }))
     return;
 
+  if (op.getDialect().usePropertiesForAttributes()) {
+    SmallVector<MethodParameter> paramList;
+    paramList.emplace_back("::mlir::OperationName", "opName");
+    paramList.emplace_back("Properties &", "properties");
+    auto *m =
+        opClass.addStaticMethod("void", "populateDefaultProperties", paramList);
+    ERROR_IF_PRUNED(m, "populateDefaultProperties", op);
+    auto &body = m->body();
+    body.indent();
+    body << "::mlir::Builder " << odsBuilder << "(opName.getContext());\n";
+    for (const NamedAttribute &namedAttr : op.getAttributes()) {
+      auto &attr = namedAttr.attr;
+      if (!attr.hasDefaultValue() || attr.isOptional())
+        continue;
+      StringRef name = namedAttr.name;
+      FmtContext fctx;
+      fctx.withBuilder(odsBuilder);
+      body << "if (!properties." << name << ")\n"
+           << "  properties." << name << " = "
+           << std::string(tgfmt(attr.getConstBuilderTemplate(), &fctx,
+                                tgfmt(attr.getDefaultValue(), &fctx)))
+           << ";\n";
+    }
+    return;
+  }
+
   SmallVector<MethodParameter> paramList;
   paramList.emplace_back("const ::mlir::OperationName &", "opName");
   paramList.emplace_back("::mlir::NamedAttrList &", "attributes");
@@ -1830,6 +2197,7 @@ void OpEmitter::genInferredTypeCollectiveParamBuilder() {
   if (::mlir::succeeded({0}::inferReturnTypes(odsBuilder.getContext(),
           {1}.location, operands,
           {1}.attributes.getDictionary({1}.getContext()),
+          {1}.getRawProperties(),
           {1}.regions, inferredReturnTypes))) {{)",
                   opClass.getClassName(), builderOpState);
   if (numVariadicResults == 0 || numNonVariadicResults != 0)
@@ -2147,6 +2515,10 @@ void OpEmitter::buildParamList(SmallVectorImpl<MethodParameter> &paramList,
                              operand->isOptional());
       continue;
     }
+    if (const auto *operand = arg.dyn_cast<NamedProperty *>()) {
+      // TODO
+      continue;
+    }
     const NamedAttribute &namedAttr = *arg.get<NamedAttribute *>();
     const Attribute &attr = namedAttr.attr;
 
@@ -2207,12 +2579,19 @@ void OpEmitter::genCodeForAddingArgAndRegionForBuilder(
            << "    ::llvm::SmallVector<int32_t> rangeSegments;\n"
            << "    for (::mlir::ValueRange range : " << argName << ")\n"
            << "      rangeSegments.push_back(range.size());\n"
-           << "    " << builderOpState << ".addAttribute("
-           << op.getGetterName(
-                  operand.constraint.getVariadicOfVariadicSegmentSizeAttr())
-           << "AttrName(" << builderOpState << ".name), " << odsBuilder
-           << ".getDenseI32ArrayAttr(rangeSegments));"
-           << "  }\n";
+           << "    auto rangeAttr = " << odsBuilder
+           << ".getDenseI32ArrayAttr(rangeSegments);\n";
+      if (op.getDialect().usePropertiesForAttributes()) {
+        body << "    " << builderOpState << ".getOrAddProperties<Properties>()."
+             << operand.constraint.getVariadicOfVariadicSegmentSizeAttr()
+             << " = rangeAttr;";
+      } else {
+        body << "    " << builderOpState << ".addAttribute("
+             << op.getGetterName(
+                    operand.constraint.getVariadicOfVariadicSegmentSizeAttr())
+             << "AttrName(" << builderOpState << ".name), rangeAttr);";
+      }
+      body << "  }\n";
       continue;
     }
 
@@ -2224,9 +2603,15 @@ void OpEmitter::genCodeForAddingArgAndRegionForBuilder(
   // If the operation has the operand segment size attribute, add it here.
   if (op.getTrait("::mlir::OpTrait::AttrSizedOperandSegments")) {
     std::string sizes = op.getGetterName(operandSegmentAttrName);
-    body << "  " << builderOpState << ".addAttribute(" << sizes << "AttrName("
-         << builderOpState << ".name), "
-         << "odsBuilder.getDenseI32ArrayAttr({";
+    if (op.getDialect().usePropertiesForAttributes()) {
+      body << "   (" << builderOpState << ".getOrAddProperties<Properties>()."
+           << operandSegmentAttrName << "= "
+           << "odsBuilder.getDenseI32ArrayAttr({";
+    } else {
+      body << "  " << builderOpState << ".addAttribute(" << sizes << "AttrName("
+           << builderOpState << ".name), "
+           << "odsBuilder.getDenseI32ArrayAttr({";
+    }
     interleaveComma(llvm::seq<int>(0, op.getNumOperands()), body, [&](int i) {
       const NamedTypeConstraint &operand = op.getOperand(i);
       if (!operand.isVariableLength()) {
@@ -2272,13 +2657,24 @@ void OpEmitter::genCodeForAddingArgAndRegionForBuilder(
       // instance.
       FmtContext fctx;
       fctx.withBuilder("odsBuilder");
-      body << formatv("  {0}.addAttribute({1}AttrName({0}.name), {2});\n",
-                      builderOpState, op.getGetterName(namedAttr.name),
-                      constBuildAttrFromParam(attr, fctx, namedAttr.name));
+      if (op.getDialect().usePropertiesForAttributes()) {
+        body << formatv("  {0}.getOrAddProperties<Properties>().{1} = {2};\n",
+                        builderOpState, namedAttr.name,
+                        constBuildAttrFromParam(attr, fctx, namedAttr.name));
+      } else {
+        body << formatv("  {0}.addAttribute({1}AttrName({0}.name), {2});\n",
+                        builderOpState, op.getGetterName(namedAttr.name),
+                        constBuildAttrFromParam(attr, fctx, namedAttr.name));
+      }
     } else {
-      body << formatv("  {0}.addAttribute({1}AttrName({0}.name), {2});\n",
-                      builderOpState, op.getGetterName(namedAttr.name),
-                      namedAttr.name);
+      if (op.getDialect().usePropertiesForAttributes()) {
+        body << formatv("  {0}.getOrAddProperties<Properties>().{1} = {1};\n",
+                        builderOpState, namedAttr.name);
+      } else {
+        body << formatv("  {0}.addAttribute({1}AttrName({0}.name), {2});\n",
+                        builderOpState, op.getGetterName(namedAttr.name),
+                        namedAttr.name);
+      }
     }
     if (emitNotNullCheck)
       body.unindent() << "  }\n";
@@ -2448,6 +2844,8 @@ void OpEmitter::genSideEffectInterfaceMethods() {
       ++operandIt;
       continue;
     }
+    if (arg.is<NamedProperty *>())
+      continue;
     const NamedAttribute *attr = arg.get<NamedAttribute *>();
     if (attr->attr.getBaseAttr().isSymbolRefAttr())
       resolveDecorators(op.getArgDecorators(i), i, EffectKind::Symbol);
@@ -2544,7 +2942,6 @@ void OpEmitter::genTypeInterfaceMethods() {
         continue;
       const InferredResultType &infer = op.getInferredResultType(i);
       std::string typeStr;
-      body << "  ::mlir::Type odsInferredType" << inferredTypeIdx++ << " = ";
       if (infer.isArg()) {
         // If this is an operand, just index into operand list to access the
         // type.
@@ -2558,9 +2955,22 @@ void OpEmitter::genTypeInterfaceMethods() {
         } else {
           auto *attr =
               op.getArg(arg.operandOrAttributeIndex()).get<NamedAttribute *>();
-          typeStr = ("attributes.get(\"" + attr->name +
-                     "\").cast<::mlir::TypedAttr>().getType()")
-                        .str();
+          body << "  ::mlir::TypedAttr odsInferredTypeAttr" << inferredTypeIdx
+               << " = ";
+          if (op.getDialect().usePropertiesForAttributes()) {
+            body << "(properties ? properties.as<Properties *>()->"
+                 << attr->name
+                 << " : attributes.get(\"" + attr->name +
+                        "\").dyn_cast_or_null<::mlir::TypedAttr>());\n";
+          } else {
+            body << "attributes.get(\"" + attr->name +
+                        "\").dyn_cast_or_null<::mlir::TypedAttr>();\n";
+          }
+          body << "  if (!odsInferredTypeAttr" << inferredTypeIdx
+               << ") return ::mlir::failure();\n";
+          typeStr =
+              ("odsInferredTypeAttr" + Twine(inferredTypeIdx) + ".getType()")
+                  .str();
         }
       } else if (std::optional<StringRef> builder =
                      op.getResult(infer.getResultIndex())
@@ -2572,7 +2982,8 @@ void OpEmitter::genTypeInterfaceMethods() {
       } else {
         continue;
       }
-      body << tgfmt(infer.getTransformer(), &fctx.withSelf(typeStr)) << ";\n";
+      body << "  ::mlir::Type odsInferredType" << inferredTypeIdx++ << " = "
+           << tgfmt(infer.getTransformer(), &fctx.withSelf(typeStr)) << ";\n";
       constructedIndices[i] = inferredTypeIdx - 1;
     }
   }
@@ -2615,9 +3026,11 @@ void OpEmitter::genVerifier() {
       opClass.addMethod("::mlir::LogicalResult", "verifyInvariantsImpl");
   ERROR_IF_PRUNED(implMethod, "verifyInvariantsImpl", op);
   auto &implBody = implMethod->body();
+  bool useProperties = emitHelper.hasProperties();
 
   populateSubstitutions(emitHelper, verifyCtx);
-  genAttributeVerifier(emitHelper, verifyCtx, implBody, staticVerifierEmitter);
+  genAttributeVerifier(emitHelper, verifyCtx, implBody, staticVerifierEmitter,
+                       useProperties);
   genOperandResultVerifier(implBody, op.getOperands(), "operand");
   genOperandResultVerifier(implBody, op.getResults(), "result");
 
@@ -3003,11 +3416,110 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
       staticVerifierEmitter(staticVerifierEmitter),
       emitHelper(op, /*emitForOp=*/false) {
 
+  genericAdaptorBase.declare<VisibilityDeclaration>(Visibility::Public);
+  bool useProperties = emitHelper.hasProperties();
+  if (useProperties) {
+    // Define the properties struct with multiple members.
+    using ConstArgument =
+        llvm::PointerUnion<const AttributeMetadata *, const NamedProperty *>;
+    SmallVector<ConstArgument> attrOrProperties;
+    for (const std::pair<StringRef, AttributeMetadata> &it :
+         emitHelper.getAttrMetadata()) {
+      if (!it.second.constraint || !it.second.constraint->isDerivedAttr())
+        attrOrProperties.push_back(&it.second);
+    }
+    for (const NamedProperty &prop : op.getProperties())
+      attrOrProperties.push_back(&prop);
+    assert(!attrOrProperties.empty());
+    std::string declarations = "  struct Properties {\n";
+    llvm::raw_string_ostream os(declarations);
+    for (const auto &attrOrProp : attrOrProperties) {
+      if (const auto *namedProperty =
+              attrOrProp.dyn_cast<const NamedProperty *>()) {
+        StringRef name = namedProperty->name;
+        if (name.empty())
+          report_fatal_error("missing name for property");
+        std::string camelName =
+            convertToCamelFromSnakeCase(name, /*capitalizeFirst=*/true);
+        auto &prop = namedProperty->prop;
+        // Generate the data member using the storage type.
+        os << "    using " << name << "Ty = " << prop.getStorageType() << ";\n"
+           << "    " << name << "Ty " << name;
+        if (prop.hasDefaultValue())
+          os << " = " << prop.getDefaultValue();
+
+        // Emit accessors using the interface type.
+        const char *accessorFmt = R"decl(;
+    {0} get{1}() {
+      auto &propStorage = this->{2};
+      return {3};
+    }
+    void set{1}(const {0} &propValue) {
+      auto &propStorage = this->{2};
+      {4};
+    }
+)decl";
+        FmtContext fctx;
+        os << formatv(accessorFmt, prop.getInterfaceType(), camelName, name,
+                      tgfmt(prop.getConvertFromStorageCall(),
+                            &fctx.addSubst("_storage", propertyStorage)),
+                      tgfmt(prop.getAssignToStorageCall(),
+                            &fctx.addSubst("_value", propertyValue)
+                                 .addSubst("_storage", propertyStorage)));
+        continue;
+      }
+      const auto *namedAttr = attrOrProp.dyn_cast<const AttributeMetadata *>();
+      const Attribute *attr = nullptr;
+      if (namedAttr->constraint)
+        attr = &*namedAttr->constraint;
+      StringRef name = namedAttr->attrName;
+      if (name.empty())
+        report_fatal_error("missing name for property attr");
+      std::string camelName =
+          convertToCamelFromSnakeCase(name, /*capitalizeFirst=*/true);
+      // Generate the data member using the storage type.
+      StringRef storageType;
+      if (attr) {
+        storageType = attr->getStorageType();
+      } else {
+        if (name != operandSegmentAttrName && name != resultSegmentAttrName) {
+          report_fatal_error("unexpected AttributeMetadata");
+        }
+        // TODO: update to use native integers.
+        storageType = "::mlir::DenseI32ArrayAttr";
+      }
+      os << "    using " << name << "Ty = " << storageType << ";\n"
+         << "    " << name << "Ty " << name << ";\n";
+
+      // Emit accessors using the interface type.
+      if (attr) {
+        const char *accessorFmt = R"decl(
+    auto get{0}() {
+      auto &propStorage = this->{1};
+      return propStorage.{2}<{3}>();
+    }
+    void set{0}(const {3} &propValue) {
+      this->{1} = propValue;
+    }
+)decl";
+        os << formatv(accessorFmt, camelName, name,
+                      attr->isOptional() || attr->hasDefaultValue()
+                          ? "dyn_cast_or_null"
+                          : "cast",
+                      storageType);
+      }
+    }
+    os << "  };\n";
+    os.flush();
+    genericAdaptorBase.declare<ExtraClassDeclaration>(std::move(declarations));
+  }
   genericAdaptorBase.declare<VisibilityDeclaration>(Visibility::Protected);
   genericAdaptorBase.declare<Field>("::mlir::DictionaryAttr", "odsAttrs");
-  genericAdaptorBase.declare<Field>("::mlir::RegionRange", "odsRegions");
   genericAdaptorBase.declare<Field>("::std::optional<::mlir::OperationName>",
                                     "odsOpName");
+  if (useProperties)
+    genericAdaptorBase.declare<Field>("Properties", "properties");
+  genericAdaptorBase.declare<Field>("::mlir::RegionRange", "odsRegions");
 
   genericAdaptor.addTemplateParam("RangeT");
   genericAdaptor.addField("RangeT", "odsOperands");
@@ -3024,9 +3536,15 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
     SmallVector<MethodParameter> paramList;
     paramList.emplace_back("::mlir::DictionaryAttr", "attrs",
                            attrSizedOperands ? "" : "nullptr");
+    if (useProperties)
+      paramList.emplace_back("const Properties &", "properties", "{}");
+    else
+      paramList.emplace_back("::mlir::EmptyProperties", "properties", "{}");
     paramList.emplace_back("::mlir::RegionRange", "regions", "{}");
     auto *baseConstructor = genericAdaptorBase.addConstructor(paramList);
     baseConstructor->addMemberInitializer("odsAttrs", "attrs");
+    if (useProperties)
+      baseConstructor->addMemberInitializer("properties", "properties");
     baseConstructor->addMemberInitializer("odsRegions", "regions");
 
     MethodBody &body = baseConstructor->body();
@@ -3037,7 +3555,7 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
 
     paramList.insert(paramList.begin(), MethodParameter("RangeT", "values"));
     auto *constructor = genericAdaptor.addConstructor(std::move(paramList));
-    constructor->addMemberInitializer("Base", "attrs, regions");
+    constructor->addMemberInitializer("Base", "attrs, properties, regions");
     constructor->addMemberInitializer("odsOperands", "values");
   }
 
@@ -3055,8 +3573,8 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
                               /*rangeSizeCall=*/"odsOperands.size()",
                               /*getOperandCallPattern=*/"odsOperands[{0}]");
 
-  // Any invalid overlap for `getOperands` will have been diagnosed before here
-  // already.
+  // Any invalid overlap for `getOperands` will have been diagnosed before
+  // here already.
   if (auto *m = genericAdaptor.addMethod("RangeT", "getOperands"))
     m->body() << "  return odsOperands;";
 
@@ -3070,8 +3588,10 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
         genericAdaptorBase.addMethod(attr.getStorageType(), emitName + "Attr");
     ERROR_IF_PRUNED(method, "Adaptor::" + emitName + "Attr", op);
     auto &body = method->body().indent();
-    body << "assert(odsAttrs && \"no attributes when constructing adapter\");\n"
-         << formatv("auto attr = {0}.{1}<{2}>();\n", emitHelper.getAttr(name),
+    if (!useProperties)
+      body << "assert(odsAttrs && \"no attributes when constructing "
+              "adapter\");\n";
+    body << formatv("auto attr = {0}.{1}<{2}>();\n", emitHelper.getAttr(name),
                     attr.hasDefaultValue() || attr.isOptional()
                         ? "dyn_cast_or_null"
                         : "cast",
@@ -3088,6 +3608,12 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
     body << "return attr;\n";
   };
 
+  if (useProperties) {
+    auto *m = genericAdaptorBase.addInlineMethod("const Properties &",
+                                                 "getProperties");
+    ERROR_IF_PRUNED(m, "Adaptor::getProperties", op);
+    m->body() << "  return properties;";
+  }
   {
     auto *m =
         genericAdaptorBase.addMethod("::mlir::DictionaryAttr", "getAttributes");
@@ -3124,8 +3650,8 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
     m->body() << formatv("  return *odsRegions[{0}];", i);
   }
   if (numRegions > 0) {
-    // Any invalid overlap for `getRegions` will have been diagnosed before here
-    // already.
+    // Any invalid overlap for `getRegions` will have been diagnosed before
+    // here already.
     if (auto *m =
             genericAdaptorBase.addMethod("::mlir::RegionRange", "getRegions"))
       m->body() << "  return odsRegions;";
@@ -3142,8 +3668,8 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(
     auto *constructor =
         adaptor.addConstructor(MethodParameter(op.getCppClassName(), "op"));
     constructor->addMemberInitializer(
-        adaptor.getClassName(),
-        "op->getOperands(), op->getAttrDictionary(), op->getRegions()");
+        adaptor.getClassName(), "op->getOperands(), op->getAttrDictionary(), "
+                                "op.getProperties(), op->getRegions()");
   }
 
   // Add verification function.
@@ -3159,10 +3685,12 @@ void OpOperandAdaptorEmitter::addVerification() {
                                    MethodParameter("::mlir::Location", "loc"));
   ERROR_IF_PRUNED(method, "verify", op);
   auto &body = method->body();
+  bool useProperties = emitHelper.hasProperties();
 
   FmtContext verifyCtx;
   populateSubstitutions(emitHelper, verifyCtx);
-  genAttributeVerifier(emitHelper, verifyCtx, body, staticVerifierEmitter);
+  genAttributeVerifier(emitHelper, verifyCtx, body, staticVerifierEmitter,
+                       useProperties);
 
   body << "  return ::mlir::success();";
 }
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index ed84bcc049a65..e0472926078d4 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -132,6 +132,14 @@ class AttrDictDirective
   bool withKeyword;
 };
 
+/// This class represents the `prop-dict` directive. This directive represents
+/// the properties of the operation, expressed as a directionary.
+class PropDictDirective
+    : public DirectiveElementBase<DirectiveElement::PropDict> {
+public:
+  explicit PropDictDirective() = default;
+};
+
 /// This class represents the `functional-type` directive. This directive takes
 /// two arguments and formats them, respectively, as the inputs and results of a
 /// FunctionType.
@@ -294,8 +302,9 @@ struct OperationFormat {
   };
 
   OperationFormat(const Operator &op)
-
-  {
+      : useProperties(op.getDialect().usePropertiesForAttributes() &&
+                      !op.getAttributes().empty()),
+        opCppClassName(op.getCppClassName()) {
     operandTypes.resize(op.getNumOperands(), TypeResolution());
     resultTypes.resize(op.getNumResults(), TypeResolution());
 
@@ -351,6 +360,12 @@ struct OperationFormat {
   /// A flag indicating if this operation has the SingleBlock trait.
   bool hasSingleBlockTrait;
 
+  /// Indicate whether attribute are stored in properties.
+  bool useProperties;
+
+  /// The Operation class name
+  StringRef opCppClassName;
+
   /// A map of buildable types to indices.
   llvm::MapVector<StringRef, int, llvm::StringMap<int>> buildableTypes;
 
@@ -389,8 +404,7 @@ static bool shouldFormatSymbolNameAttr(const NamedAttribute *attr) {
 /// {0}: The name of the attribute.
 /// {1}: The type for the attribute.
 const char *const attrParserCode = R"(
-  if (parser.parseCustomAttributeWithFallback({0}Attr, {1}, "{0}",
-          result.attributes)) {{
+  if (parser.parseCustomAttributeWithFallback({0}Attr, {1})) {{
     return ::mlir::failure();
   }
 )";
@@ -400,30 +414,29 @@ const char *const attrParserCode = R"(
 /// {0}: The name of the attribute.
 /// {1}: The type for the attribute.
 const char *const genericAttrParserCode = R"(
-  if (parser.parseAttribute({0}Attr, {1}, "{0}", result.attributes))
+  if (parser.parseAttribute({0}Attr, {1}))
     return ::mlir::failure();
 )";
 
 const char *const optionalAttrParserCode = R"(
-  {
-    ::mlir::OptionalParseResult parseResult =
-      parser.parseOptionalAttribute({0}Attr, {1}, "{0}", result.attributes);
-    if (parseResult.has_value() && failed(*parseResult))
-      return ::mlir::failure();
-  }
+  ::mlir::OptionalParseResult parseResult{0}Attr =
+    parser.parseOptionalAttribute({0}Attr, {1});
+  if (parseResult{0}Attr.has_value() && failed(*parseResult{0}Attr))
+    return ::mlir::failure();
+  if (parseResult{0}Attr.has_value() && succeeded(*parseResult{0}Attr))
 )";
 
 /// The code snippet used to generate a parser call for a symbol name attribute.
 ///
 /// {0}: The name of the attribute.
 const char *const symbolNameAttrParserCode = R"(
-  if (parser.parseSymbolName({0}Attr, "{0}", result.attributes))
+  if (parser.parseSymbolName({0}Attr))
     return ::mlir::failure();
 )";
 const char *const optionalSymbolNameAttrParserCode = R"(
   // Parsing an optional symbol name doesn't fail, so no need to check the
   // result.
-  (void)parser.parseOptionalSymbolName({0}Attr, "{0}", result.attributes);
+  (void)parser.parseOptionalSymbolName({0}Attr);
 )";
 
 /// The code snippet used to generate a parser call for an enum attribute.
@@ -434,6 +447,7 @@ const char *const optionalSymbolNameAttrParserCode = R"(
 /// {3}: The constant builder call to create an attribute of the enum type.
 /// {4}: The set of allowed enum keywords.
 /// {5}: The error message on failure when the enum isn't present.
+/// {6}: The attribute assignment expression
 const char *const enumAttrParserCode = R"(
   {
     ::llvm::StringRef attrStr;
@@ -460,7 +474,7 @@ const char *const enumAttrParserCode = R"(
                << "{0} attribute specification: \"" << attrStr << '"';;
 
       {0}Attr = {3};
-      result.addAttribute("{0}", {0}Attr);
+      {6}
     }
   }
 )";
@@ -572,6 +586,7 @@ const char *const inferReturnTypesParserCode = R"(
   if (::mlir::failed({0}::inferReturnTypes(parser.getContext(),
       result.location, result.operands,
       result.attributes.getDictionary(parser.getContext()),
+      result.getRawProperties(),
       result.regions, inferredReturnTypes)))
     return ::mlir::failure();
   result.addTypes(inferredReturnTypes);
@@ -930,7 +945,9 @@ static void genCustomParameterParser(FormatElement *param, MethodBody &body) {
 }
 
 /// Generate the parser for a custom directive.
-static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body) {
+static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body,
+                                     bool useProperties,
+                                     StringRef opCppClassName) {
   body << "  {\n";
 
   // Preprocess the directive variables.
@@ -1003,9 +1020,15 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body) {
       const NamedAttribute *var = attr->getVar();
       if (var->attr.isOptional() || var->attr.hasDefaultValue())
         body << llvm::formatv("    if ({0}Attr)\n  ", var->name);
+      if (useProperties) {
+        body << formatv(
+            "    result.getOrAddProperties<{1}::Properties>().{0} = {0}Attr;\n",
+            var->name, opCppClassName);
+      } else {
+        body << llvm::formatv("    result.addAttribute(\"{0}\", {0}Attr);\n",
+                              var->name);
+      }
 
-      body << llvm::formatv("    result.addAttribute(\"{0}\", {0}Attr);\n",
-                            var->name);
     } else if (auto *operand = dyn_cast<OperandVariable>(param)) {
       const NamedTypeConstraint *var = operand->getVar();
       if (var->isOptional()) {
@@ -1041,7 +1064,8 @@ static void genCustomDirectiveParser(CustomDirective *dir, MethodBody &body) {
 
 /// Generate the parser for a enum attribute.
 static void genEnumAttrParser(const NamedAttribute *var, MethodBody &body,
-                              FmtContext &attrTypeCtx, bool parseAsOptional) {
+                              FmtContext &attrTypeCtx, bool parseAsOptional,
+                              bool useProperties, StringRef opCppClassName) {
   Attribute baseAttr = var->attr.getBaseAttr();
   const EnumAttr &enumAttr = cast<EnumAttr>(baseAttr);
   std::vector<EnumAttrCase> cases = enumAttr.getAllCases();
@@ -1076,46 +1100,68 @@ static void genEnumAttrParser(const NamedAttribute *var, MethodBody &body,
     });
     errorMessageOS << "]\");";
   }
+  std::string attrAssignment;
+  if (useProperties) {
+    attrAssignment =
+        formatv("  "
+                "result.getOrAddProperties<{1}::Properties>().{0} = {0}Attr;",
+                var->name, opCppClassName);
+  } else {
+    attrAssignment =
+        formatv("result.addAttribute(\"{0}\", {0}Attr);", var->name);
+  }
 
   body << formatv(enumAttrParserCode, var->name, enumAttr.getCppNamespace(),
                   enumAttr.getStringToSymbolFnName(), attrBuilderStr,
-                  validCaseKeywordsStr, errorMessage);
+                  validCaseKeywordsStr, errorMessage, attrAssignment);
 }
 
 // Generate the parser for an attribute.
 static void genAttrParser(AttributeVariable *attr, MethodBody &body,
-                          FmtContext &attrTypeCtx, bool parseAsOptional) {
+                          FmtContext &attrTypeCtx, bool parseAsOptional,
+                          bool useProperties, StringRef opCppClassName) {
   const NamedAttribute *var = attr->getVar();
 
   // Check to see if we can parse this as an enum attribute.
   if (canFormatEnumAttr(var))
-    return genEnumAttrParser(var, body, attrTypeCtx, parseAsOptional);
+    return genEnumAttrParser(var, body, attrTypeCtx, parseAsOptional,
+                             useProperties, opCppClassName);
 
   // Check to see if we should parse this as a symbol name attribute.
   if (shouldFormatSymbolNameAttr(var)) {
     body << formatv(parseAsOptional ? optionalSymbolNameAttrParserCode
                                     : symbolNameAttrParserCode,
                     var->name);
-    return;
-  }
-
-  // If this attribute has a buildable type, use that when parsing the
-  // attribute.
-  std::string attrTypeStr;
-  if (std::optional<StringRef> typeBuilder = attr->getTypeBuilder()) {
-    llvm::raw_string_ostream os(attrTypeStr);
-    os << tgfmt(*typeBuilder, &attrTypeCtx);
   } else {
-    attrTypeStr = "::mlir::Type{}";
+
+    // If this attribute has a buildable type, use that when parsing the
+    // attribute.
+    std::string attrTypeStr;
+    if (std::optional<StringRef> typeBuilder = attr->getTypeBuilder()) {
+      llvm::raw_string_ostream os(attrTypeStr);
+      os << tgfmt(*typeBuilder, &attrTypeCtx);
+    } else {
+      attrTypeStr = "::mlir::Type{}";
+    }
+    if (parseAsOptional) {
+      body << formatv(optionalAttrParserCode, var->name, attrTypeStr);
+    } else {
+      if (attr->shouldBeQualified() ||
+          var->attr.getStorageType() == "::mlir::Attribute")
+        body << formatv(genericAttrParserCode, var->name, attrTypeStr);
+      else
+        body << formatv(attrParserCode, var->name, attrTypeStr);
+    }
   }
-  if (parseAsOptional) {
-    body << formatv(optionalAttrParserCode, var->name, attrTypeStr);
+  if (useProperties) {
+    body << formatv(
+        "  if ({0}Attr) result.getOrAddProperties<{1}::Properties>().{0} = "
+        "{0}Attr;\n",
+        var->name, opCppClassName);
   } else {
-    if (attr->shouldBeQualified() ||
-        var->attr.getStorageType() == "::mlir::Attribute")
-      body << formatv(genericAttrParserCode, var->name, attrTypeStr);
-    else
-      body << formatv(attrParserCode, var->name, attrTypeStr);
+    body << formatv(
+        "  if ({0}Attr) result.attributes.append(\"{0}\", {0}Attr);\n",
+        var->name);
   }
 }
 
@@ -1170,8 +1216,15 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
 
         if (!thenGroup == optional->isInverted()) {
           // Add the anchor unit attribute to the operation state.
-          body << "    result.addAttribute(\"" << anchorAttr->getVar()->name
-               << "\", parser.getBuilder().getUnitAttr());\n";
+          if (useProperties) {
+            body << formatv(
+                "    result.getOrAddProperties<{1}::Properties>().{0} = "
+                "parser.getBuilder().getUnitAttr();",
+                anchorAttr->getVar()->name, opCppClassName);
+          } else {
+            body << "    result.addAttribute(\"" << anchorAttr->getVar()->name
+                 << "\", parser.getBuilder().getUnitAttr());\n";
+          }
         }
       }
 
@@ -1190,7 +1243,8 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
     // parsing of the rest of the elements.
     FormatElement *firstElement = thenElements.front();
     if (auto *attrVar = dyn_cast<AttributeVariable>(firstElement)) {
-      genAttrParser(attrVar, body, attrTypeCtx, /*parseAsOptional=*/true);
+      genAttrParser(attrVar, body, attrTypeCtx, /*parseAsOptional=*/true,
+                    useProperties, opCppClassName);
       body << "  if (" << attrVar->getVar()->name << "Attr) {\n";
     } else if (auto *literal = dyn_cast<LiteralElement>(firstElement)) {
       body << "  if (::mlir::succeeded(parser.parseOptional";
@@ -1248,8 +1302,15 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
       body << formatv(oilistParserCode, lelementName);
       if (AttributeVariable *unitAttrElem =
               oilist->getUnitAttrParsingElement(pelement)) {
-        body << "  result.addAttribute(\"" << unitAttrElem->getVar()->name
-             << "\", UnitAttr::get(parser.getContext()));\n";
+        if (useProperties) {
+          body << formatv(
+              "    result.getOrAddProperties<{1}::Properties>().{0} = "
+              "parser.getBuilder().getUnitAttr();",
+              unitAttrElem->getVar()->name, opCppClassName);
+        } else {
+          body << "  result.addAttribute(\"" << unitAttrElem->getVar()->name
+               << "\", UnitAttr::get(parser.getContext()));\n";
+        }
       } else {
         for (FormatElement *el : pelement)
           genElementParser(el, body, attrTypeCtx);
@@ -1275,7 +1336,8 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
   } else if (auto *attr = dyn_cast<AttributeVariable>(element)) {
     bool parseAsOptional =
         (genCtx == GenContext::Normal && attr->getVar()->attr.isOptional());
-    genAttrParser(attr, body, attrTypeCtx, parseAsOptional);
+    genAttrParser(attr, body, attrTypeCtx, parseAsOptional, useProperties,
+                  opCppClassName);
 
   } else if (auto *operand = dyn_cast<OperandVariable>(element)) {
     ArgumentLengthKind lengthKind = getArgumentLengthKind(operand->getVar());
@@ -1311,13 +1373,27 @@ void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
 
     /// Directives.
   } else if (auto *attrDict = dyn_cast<AttrDictDirective>(element)) {
-    body << "  if (parser.parseOptionalAttrDict"
-         << (attrDict->isWithKeyword() ? "WithKeyword" : "")
-         << "(result.attributes))\n"
+    body.indent() << "{\n";
+    body.indent() << "auto loc = parser.getCurrentLocation();(void)loc;\n"
+                  << "if (parser.parseOptionalAttrDict"
+                  << (attrDict->isWithKeyword() ? "WithKeyword" : "")
+                  << "(result.attributes))\n"
+                  << "  return ::mlir::failure();\n";
+    if (useProperties) {
+      body << "if (failed(verifyInherentAttrs(result.name, result.attributes, "
+              "[&]() {\n"
+           << "    return parser.emitError(loc) << \"'\" << "
+              "result.name.getStringRef() << \"' op \";\n"
+           << "  })))\n"
+           << "  return ::mlir::failure();\n";
+    }
+    body.unindent() << "}\n";
+    body.unindent();
+  } else if (auto *attrDict = dyn_cast<PropDictDirective>(element)) {
+    body << "  if (parseProperties(parser, result))\n"
          << "    return ::mlir::failure();\n";
   } else if (auto *customDir = dyn_cast<CustomDirective>(element)) {
-    genCustomDirectiveParser(customDir, body);
-
+    genCustomDirectiveParser(customDir, body, useProperties, opCppClassName);
   } else if (isa<OperandsDirective>(element)) {
     body << "  ::llvm::SMLoc allOperandLoc = parser.getCurrentLocation();\n"
          << "  if (parser.parseOperandList(allOperands))\n"
@@ -1571,8 +1647,16 @@ void OperationFormat::genParserVariadicSegmentResolution(Operator &op,
                                                          MethodBody &body) {
   if (!allOperands) {
     if (op.getTrait("::mlir::OpTrait::AttrSizedOperandSegments")) {
-      body << "  result.addAttribute(\"operand_segment_sizes\", "
-           << "parser.getBuilder().getDenseI32ArrayAttr({";
+      if (op.getDialect().usePropertiesForAttributes()) {
+        body << formatv("  "
+                        "result.getOrAddProperties<{0}::Properties>().operand_"
+                        "segment_sizes = "
+                        "(parser.getBuilder().getDenseI32ArrayAttr({{",
+                        op.getCppClassName());
+      } else {
+        body << "  result.addAttribute(\"operand_segment_sizes\", "
+             << "parser.getBuilder().getDenseI32ArrayAttr({";
+      }
       auto interleaveFn = [&](const NamedTypeConstraint &operand) {
         // If the operand is variadic emit the parsed size.
         if (operand.isVariableLength())
@@ -1586,18 +1670,36 @@ void OperationFormat::genParserVariadicSegmentResolution(Operator &op,
     for (const NamedTypeConstraint &operand : op.getOperands()) {
       if (!operand.isVariadicOfVariadic())
         continue;
-      body << llvm::formatv(
-          "  result.addAttribute(\"{0}\", "
-          "parser.getBuilder().getDenseI32ArrayAttr({1}OperandGroupSizes));\n",
-          operand.constraint.getVariadicOfVariadicSegmentSizeAttr(),
-          operand.name);
+      if (op.getDialect().usePropertiesForAttributes()) {
+        body << llvm::formatv(
+            "  result.getOrAddProperties<{0}::Properties>().{1} = "
+            "parser.getBuilder().getDenseI32ArrayAttr({2}OperandGroupSizes);\n",
+            op.getCppClassName(),
+            operand.constraint.getVariadicOfVariadicSegmentSizeAttr(),
+            operand.name);
+      } else {
+        body << llvm::formatv(
+            "  result.addAttribute(\"{0}\", "
+            "parser.getBuilder().getDenseI32ArrayAttr({1}OperandGroupSizes));"
+            "\n",
+            operand.constraint.getVariadicOfVariadicSegmentSizeAttr(),
+            operand.name);
+      }
     }
   }
 
   if (!allResultTypes &&
       op.getTrait("::mlir::OpTrait::AttrSizedResultSegments")) {
-    body << "  result.addAttribute(\"result_segment_sizes\", "
-         << "parser.getBuilder().getDenseI32ArrayAttr({";
+    if (op.getDialect().usePropertiesForAttributes()) {
+      body << formatv(
+          "  "
+          "result.getOrAddProperties<{0}::Properties>().result_segment_sizes = "
+          "(parser.getBuilder().getDenseI32ArrayAttr({{",
+          op.getCppClassName());
+    } else {
+      body << "  result.addAttribute(\"result_segment_sizes\", "
+           << "parser.getBuilder().getDenseI32ArrayAttr({";
+    }
     auto interleaveFn = [&](const NamedTypeConstraint &result) {
       // If the result is variadic emit the parsed size.
       if (result.isVariableLength())
@@ -1641,6 +1743,14 @@ const char *enumAttrBeginPrinterCode = R"(
     auto caseValueStr = {1}(caseValue);
 )";
 
+/// Generate the printer for the 'prop-dict' directive.
+static void genPropDictPrinter(OperationFormat &fmt, Operator &op,
+                               MethodBody &body) {
+  body << "  _odsPrinter << \" \";\n"
+       << "  printProperties(this->getContext(), _odsPrinter, "
+          "getProperties());\n";
+}
+
 /// Generate the printer for the 'attr-dict' directive.
 static void genAttrDictPrinter(OperationFormat &fmt, Operator &op,
                                MethodBody &body, bool withKeyword) {
@@ -1898,7 +2008,7 @@ static void genOptionalGroupPrinterAnchor(FormatElement *anchor,
       })
       .Case<AttributeVariable>([&](AttributeVariable *element) {
         Attribute attr = element->getVar()->attr;
-        body << "(*this)->getAttr(\"" << element->getVar()->name << "\")";
+        body << op.getGetterName(element->getVar()->name) << "Attr()";
         if (attr.isOptional())
           return; // done
         if (attr.hasDefaultValue()) {
@@ -1906,7 +2016,8 @@ static void genOptionalGroupPrinterAnchor(FormatElement *anchor,
           // default value.
           FmtContext fctx;
           fctx.withBuilder("::mlir::OpBuilder((*this)->getContext())");
-          body << " != "
+          body << " && " << op.getGetterName(element->getVar()->name)
+               << "Attr() != "
                << tgfmt(attr.getConstBuilderTemplate(), &fctx,
                         attr.getDefaultValue());
           return;
@@ -2063,6 +2174,13 @@ void OperationFormat::genElementPrinter(FormatElement *element,
     return;
   }
 
+  // Emit the attribute dictionary.
+  if (auto *propDict = dyn_cast<PropDictDirective>(element)) {
+    genPropDictPrinter(*this, op, body);
+    lastWasPunctuation = false;
+    return;
+  }
+
   // Optionally insert a space before the next element. The AttrDict printer
   // already adds a space as necessary.
   if (shouldEmitSpace || !lastWasPunctuation)
@@ -2300,6 +2418,7 @@ class OpFormatParser : public FormatParser {
   ConstArgument findSeenArg(StringRef name);
 
   /// Parse the various different directives.
+  FailureOr<FormatElement *> parsePropDictDirective(SMLoc loc, Context context);
   FailureOr<FormatElement *> parseAttrDictDirective(SMLoc loc, Context context,
                                                     bool withKeyword);
   FailureOr<FormatElement *> parseFunctionalTypeDirective(SMLoc loc,
@@ -2329,6 +2448,7 @@ class OpFormatParser : public FormatParser {
   // The following are various bits of format state used for verification
   // during parsing.
   bool hasAttrDict = false;
+  bool hasPropDict = false;
   bool hasAllRegions = false, hasAllSuccessors = false;
   bool canInferResultTypes = false;
   llvm::SmallBitVector seenOperandTypes, seenResultTypes;
@@ -2873,6 +2993,8 @@ FailureOr<FormatElement *>
 OpFormatParser::parseDirectiveImpl(SMLoc loc, FormatToken::Kind kind,
                                    Context ctx) {
   switch (kind) {
+  case FormatToken::kw_prop_dict:
+    return parsePropDictDirective(loc, ctx);
   case FormatToken::kw_attr_dict:
     return parseAttrDictDirective(loc, ctx,
                                   /*withKeyword=*/false);
@@ -2925,6 +3047,23 @@ OpFormatParser::parseAttrDictDirective(SMLoc loc, Context context,
   return create<AttrDictDirective>(withKeyword);
 }
 
+FailureOr<FormatElement *>
+OpFormatParser::parsePropDictDirective(SMLoc loc, Context context) {
+  if (context == TypeDirectiveContext)
+    return emitError(loc, "'prop-dict' directive can only be used as a "
+                          "top-level directive");
+
+  if (context == RefDirectiveContext)
+    llvm::report_fatal_error("'ref' of 'prop-dict' unsupported");
+  // Otherwise, this is a top-level context.
+
+  if (hasPropDict)
+    return emitError(loc, "'prop-dict' directive has already been seen");
+  hasPropDict = true;
+
+  return create<PropDictDirective>();
+}
+
 LogicalResult OpFormatParser::verifyCustomDirectiveArguments(
     SMLoc loc, ArrayRef<FormatElement *> arguments) {
   for (FormatElement *argument : arguments) {
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index abbe8400b5fe0..8a04cc9f5be11 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -582,22 +582,24 @@ void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
   if (!name.empty())
     os << formatv("{0} = {1};\n", name, castedName);
 
-  for (int i = 0, e = tree.getNumArgs(), nextOperand = 0; i != e; ++i) {
-    auto opArg = op.getArg(i);
+  for (int i = 0, opArgIdx = 0, e = tree.getNumArgs(), nextOperand = 0; i != e;
+       ++i, ++opArgIdx) {
+    auto opArg = op.getArg(opArgIdx);
     std::string argName = formatv("op{0}", depth + 1);
 
     // Handle nested DAG construct first
     if (DagNode argTree = tree.getArgAsNestedDag(i)) {
       if (argTree.isEither()) {
-        emitEitherOperandMatch(tree, argTree, castedName, i, nextOperand,
+        emitEitherOperandMatch(tree, argTree, castedName, opArgIdx, nextOperand,
                                depth);
+        ++opArgIdx;
         continue;
       }
       if (auto *operand = opArg.dyn_cast<NamedTypeConstraint *>()) {
         if (operand->isVariableLength()) {
           auto error = formatv("use nested DAG construct to match op {0}'s "
                                "variadic operand #{1} unsupported now",
-                               op.getOperationName(), i);
+                               op.getOperationName(), opArgIdx);
           PrintFatalError(loc, error);
         }
       }
@@ -627,11 +629,10 @@ void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
           formatv("{0}.getODSOperands({1})", castedName, nextOperand);
       emitOperandMatch(tree, castedName, operandName.str(),
                        /*operandMatcher=*/tree.getArgAsLeaf(i),
-                       /*argName=*/tree.getArgName(i),
-                       /*argIndex=*/i);
+                       /*argName=*/tree.getArgName(i), opArgIdx);
       ++nextOperand;
     } else if (opArg.is<NamedAttribute *>()) {
-      emitAttributeMatch(tree, opName, i, depth);
+      emitAttributeMatch(tree, opName, opArgIdx, depth);
     } else {
       PrintFatalError(loc, "unhandled case when matching op");
     }
diff --git a/mlir/unittests/Analysis/Presburger/CMakeLists.txt b/mlir/unittests/Analysis/Presburger/CMakeLists.txt
index 5af423295c0b6..4bfaf95f1a89b 100644
--- a/mlir/unittests/Analysis/Presburger/CMakeLists.txt
+++ b/mlir/unittests/Analysis/Presburger/CMakeLists.txt
@@ -10,6 +10,7 @@ add_mlir_unittest(MLIRPresburgerTests
   PresburgerSpaceTest.cpp
   PWMAFunctionTest.cpp
   SimplexTest.cpp
+  UtilsTest.cpp
 )
 
 target_link_libraries(MLIRPresburgerTests
diff --git a/mlir/unittests/Analysis/Presburger/UtilsTest.cpp b/mlir/unittests/Analysis/Presburger/UtilsTest.cpp
new file mode 100644
index 0000000000000..7c1646aa84116
--- /dev/null
+++ b/mlir/unittests/Analysis/Presburger/UtilsTest.cpp
@@ -0,0 +1,68 @@
+//===- Utils.cpp - Tests for Utils file ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/Presburger/Utils.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+using namespace mlir;
+using namespace presburger;
+
+static DivisionRepr parseDivisionRepr(unsigned numVars, unsigned numDivs,
+                                      ArrayRef<ArrayRef<MPInt>> dividends,
+                                      ArrayRef<MPInt> divisors) {
+  DivisionRepr repr(numVars, numDivs);
+  for (unsigned i = 0, rows = dividends.size(); i < rows; ++i)
+    repr.setDiv(i, dividends[i], divisors[i]);
+  return repr;
+}
+
+static void checkEqual(DivisionRepr &a, DivisionRepr &b) {
+  EXPECT_EQ(a.getNumVars(), b.getNumVars());
+  EXPECT_EQ(a.getNumDivs(), b.getNumDivs());
+  for (unsigned i = 0, rows = a.getNumDivs(); i < rows; ++i) {
+    EXPECT_EQ(a.hasRepr(i), b.hasRepr(i));
+    if (!a.hasRepr(i))
+      continue;
+    EXPECT_TRUE(a.getDenom(i) == b.getDenom(i));
+    EXPECT_TRUE(a.getDividend(i).equals(b.getDividend(i)));
+  }
+}
+
+TEST(UtilsTest, ParseAndCompareDivisionReprTest) {
+  auto merge = [](unsigned i, unsigned j) -> bool { return true; };
+  DivisionRepr a = parseDivisionRepr(1, 1, {{MPInt(1), MPInt(2)}}, {MPInt(2)}),
+               b = parseDivisionRepr(1, 1, {{MPInt(1), MPInt(2)}}, {MPInt(2)}),
+               c = parseDivisionRepr(2, 2,
+                                     {{MPInt(0), MPInt(1), MPInt(2)},
+                                      {MPInt(0), MPInt(1), MPInt(2)}},
+                                     {MPInt(2), MPInt(2)});
+  c.removeDuplicateDivs(merge);
+  checkEqual(a, b);
+  checkEqual(a, c);
+}
+
+TEST(UtilsTest, DivisionReprNormalizeTest) {
+  auto merge = [](unsigned i, unsigned j) -> bool { return true; };
+  DivisionRepr a = parseDivisionRepr(2, 1, {{MPInt(1), MPInt(2), MPInt(-1)}},
+                                     {MPInt(2)}),
+               b = parseDivisionRepr(2, 1, {{MPInt(16), MPInt(32), MPInt(-16)}},
+                                     {MPInt(32)}),
+               c = parseDivisionRepr(1, 1, {{MPInt(12), MPInt(-4)}},
+                                     {MPInt(8)}),
+               d = parseDivisionRepr(2, 2,
+                                     {{MPInt(1), MPInt(2), MPInt(-1)},
+                                      {MPInt(4), MPInt(8), MPInt(-4)}},
+                                     {MPInt(2), MPInt(8)});
+  b.removeDuplicateDivs(merge);
+  c.removeDuplicateDivs(merge);
+  d.removeDuplicateDivs(merge);
+  checkEqual(a, b);
+  checkEqual(c, d);
+}
diff --git a/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp b/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp
index e9cac3949ddbd..48c62ad20a04a 100644
--- a/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp
+++ b/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp
@@ -25,7 +25,7 @@ static Operation *createOp(MLIRContext *context, Location loc,
   context->allowUnregisteredDialects();
   return Operation::create(loc, OperationName(operationName, context),
                            std::nullopt, std::nullopt, std::nullopt,
-                           std::nullopt, numRegions);
+                           OpaqueProperties(nullptr), std::nullopt, numRegions);
 }
 
 namespace {
diff --git a/mlir/unittests/IR/AdaptorTest.cpp b/mlir/unittests/IR/AdaptorTest.cpp
index a3efb34889f41..fcd19addd8e3a 100644
--- a/mlir/unittests/IR/AdaptorTest.cpp
+++ b/mlir/unittests/IR/AdaptorTest.cpp
@@ -38,10 +38,10 @@ TEST(Adaptor, GenericAdaptorsOperandAccess) {
     // Using optional instead of plain int here to differentiate absence of
     // value from the value 0.
     SmallVector<std::optional<int>> v = {0, 4};
-    OIListSimple::GenericAdaptor<ArrayRef<std::optional<int>>> d(
-        v, builder.getDictionaryAttr({builder.getNamedAttr(
-               "operand_segment_sizes",
-               builder.getDenseI32ArrayAttr({1, 0, 1}))}));
+    OIListSimple::Properties prop;
+    prop.operand_segment_sizes = builder.getDenseI32ArrayAttr({1, 0, 1});
+    OIListSimple::GenericAdaptor<ArrayRef<std::optional<int>>> d(v, {}, prop,
+                                                                 {});
     EXPECT_EQ(d.getArg0(), 0);
     EXPECT_EQ(d.getArg1(), std::nullopt);
     EXPECT_EQ(d.getArg2(), 4);
@@ -51,9 +51,10 @@ TEST(Adaptor, GenericAdaptorsOperandAccess) {
   FormatVariadicOfVariadicOperand::FoldAdaptor e({});
   {
     SmallVector<int> v = {0, 1, 2, 3, 4};
-    FormatVariadicOfVariadicOperand::GenericAdaptor<ArrayRef<int>> f(
-        v, builder.getDictionaryAttr({builder.getNamedAttr(
-               "operand_segments", builder.getDenseI32ArrayAttr({3, 2, 0}))}));
+    FormatVariadicOfVariadicOperand::Properties prop;
+    prop.operand_segments = builder.getDenseI32ArrayAttr({3, 2, 0});
+    FormatVariadicOfVariadicOperand::GenericAdaptor<ArrayRef<int>> f(v, {},
+                                                                     prop, {});
     SmallVector<ArrayRef<int>> operand = f.getOperand();
     ASSERT_EQ(operand.size(), (std::size_t)3);
     EXPECT_THAT(operand[0], ElementsAre(0, 1, 2));
diff --git a/mlir/unittests/IR/CMakeLists.txt b/mlir/unittests/IR/CMakeLists.txt
index 7d49283c59c3b..8a74a59096289 100644
--- a/mlir/unittests/IR/CMakeLists.txt
+++ b/mlir/unittests/IR/CMakeLists.txt
@@ -9,6 +9,7 @@ add_mlir_unittest(MLIRIRTests
   PatternMatchTest.cpp
   ShapedTypeTest.cpp
   TypeTest.cpp
+  OpPropertiesTest.cpp
 
   DEPENDS
   MLIRTestInterfaceIncGen
diff --git a/mlir/unittests/IR/OpPropertiesTest.cpp b/mlir/unittests/IR/OpPropertiesTest.cpp
new file mode 100644
index 0000000000000..eda84a0e0b075
--- /dev/null
+++ b/mlir/unittests/IR/OpPropertiesTest.cpp
@@ -0,0 +1,358 @@
+//===- TestOpProperties.cpp - Test all properties-related APIs ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Parser/Parser.h"
+#include "gtest/gtest.h"
+#include <optional>
+
+using namespace mlir;
+
+namespace {
+/// Simple structure definining a struct to define "properties" for a given
+/// operation. Default values are honored when creating an operation.
+struct TestProperties {
+  int a = -1;
+  float b = -1.;
+  std::vector<int64_t> array = {-33};
+  /// A shared_ptr to a const object is safe: it is equivalent to a value-based
+  /// member. Here the label will be deallocated when the last operation
+  /// referring to it is destroyed. However there is no pool-allocation: this is
+  /// offloaded to the client.
+  std::shared_ptr<const std::string> label;
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestProperties)
+};
+
+/// Convert a DictionaryAttr to a TestProperties struct, optionally emit errors
+/// through the provided diagnostic if any. This is used for example during
+/// parsing with the generic format.
+static LogicalResult
+setPropertiesFromAttribute(TestProperties &prop, Attribute attr,
+                           InFlightDiagnostic *diagnostic) {
+  DictionaryAttr dict = dyn_cast<DictionaryAttr>(attr);
+  if (!dict) {
+    if (diagnostic)
+      *diagnostic << "expected DictionaryAttr to set TestProperties";
+    return failure();
+  }
+  auto aAttr = dict.getAs<IntegerAttr>("a");
+  if (!aAttr) {
+    if (diagnostic)
+      *diagnostic << "expected IntegerAttr for key `a`";
+    return failure();
+  }
+  auto bAttr = dict.getAs<FloatAttr>("b");
+  if (!bAttr ||
+      &bAttr.getValue().getSemantics() != &llvm::APFloatBase::IEEEsingle()) {
+    if (diagnostic)
+      *diagnostic << "expected FloatAttr for key `b`";
+    return failure();
+  }
+
+  auto arrayAttr = dict.getAs<DenseI64ArrayAttr>("array");
+  if (!arrayAttr) {
+    if (diagnostic)
+      *diagnostic << "expected DenseI64ArrayAttr for key `array`";
+    return failure();
+  }
+
+  auto label = dict.getAs<mlir::StringAttr>("label");
+  if (!label) {
+    if (diagnostic)
+      *diagnostic << "expected StringAttr for key `label`";
+    return failure();
+  }
+
+  prop.a = aAttr.getValue().getSExtValue();
+  prop.b = bAttr.getValue().convertToFloat();
+  prop.array.assign(arrayAttr.asArrayRef().begin(),
+                    arrayAttr.asArrayRef().end());
+  prop.label = std::make_shared<std::string>(label.getValue());
+  return success();
+}
+
+/// Convert a TestProperties struct to a DictionaryAttr, this is used for
+/// example during printing with the generic format.
+static Attribute getPropertiesAsAttribute(MLIRContext *ctx,
+                                          const TestProperties &prop) {
+  SmallVector<NamedAttribute> attrs;
+  Builder b{ctx};
+  attrs.push_back(b.getNamedAttr("a", b.getI32IntegerAttr(prop.a)));
+  attrs.push_back(b.getNamedAttr("b", b.getF32FloatAttr(prop.b)));
+  attrs.push_back(b.getNamedAttr("array", b.getDenseI64ArrayAttr(prop.array)));
+  attrs.push_back(b.getNamedAttr(
+      "label", b.getStringAttr(prop.label ? *prop.label : "<nullptr>")));
+  return b.getDictionaryAttr(attrs);
+}
+
+inline llvm::hash_code computeHash(const TestProperties &prop) {
+  // We hash `b` which is a float using its underlying array of char:
+  unsigned char const *p = reinterpret_cast<unsigned char const *>(&prop.b);
+  ArrayRef<unsigned char> bBytes{p, sizeof(prop.b)};
+  return llvm::hash_combine(
+      prop.a, llvm::hash_combine_range(bBytes.begin(), bBytes.end()),
+      llvm::hash_combine_range(prop.array.begin(), prop.array.end()),
+      StringRef(*prop.label));
+}
+
+/// A custom operation for the purpose of showcasing how to use "properties".
+class OpWithProperties : public Op<OpWithProperties> {
+public:
+  // Begin boilerplate
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpWithProperties)
+  using Op::Op;
+  static ArrayRef<StringRef> getAttributeNames() { return {}; }
+  static StringRef getOperationName() {
+    return "test_op_properties.op_with_properties";
+  }
+  // End boilerplate
+
+  // This alias is the only definition needed for enabling "properties" for this
+  // operation.
+  using Properties = TestProperties;
+  static std::optional<mlir::Attribute> getInherentAttr(const Properties &prop,
+                                                        StringRef name) {
+    return std::nullopt;
+  }
+  static void setInherentAttr(Properties &prop, StringRef name,
+                              mlir::Attribute value) {}
+  static void populateInherentAttrs(const Properties &prop,
+                                    NamedAttrList &attrs) {}
+  static LogicalResult
+  verifyInherentAttrs(OperationName opName, NamedAttrList &attrs,
+                      function_ref<InFlightDiagnostic()> getDiag) {
+    return success();
+  }
+};
+
+// A trivial supporting dialect to register the above operation.
+class TestOpPropertiesDialect : public Dialect {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestOpPropertiesDialect)
+  static constexpr StringLiteral getDialectNamespace() {
+    return StringLiteral("test_op_properties");
+  }
+  explicit TestOpPropertiesDialect(MLIRContext *context)
+      : Dialect(getDialectNamespace(), context,
+                TypeID::get<TestOpPropertiesDialect>()) {
+    addOperations<OpWithProperties>();
+  }
+};
+
+constexpr StringLiteral mlirSrc = R"mlir(
+    "test_op_properties.op_with_properties"()
+      <{a = -42 : i32,
+        b = -4.200000e+01 : f32,
+        array = array<i64: 40, 41>,
+        label = "bar foo"}> : () -> ()
+)mlir";
+
+TEST(OpPropertiesTest, Properties) {
+  MLIRContext context;
+  context.getOrLoadDialect<TestOpPropertiesDialect>();
+  ParserConfig config(&context);
+  // Parse the operation with some properties.
+  OwningOpRef<Operation *> op = parseSourceString(mlirSrc, config);
+  ASSERT_TRUE(op.get() != nullptr);
+  auto opWithProp = dyn_cast<OpWithProperties>(op.get());
+  ASSERT_TRUE(opWithProp);
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    opWithProp.print(os);
+    ASSERT_STREQ("\"test_op_properties.op_with_properties\"() "
+                 "<{a = -42 : i32, "
+                 "array = array<i64: 40, 41>, "
+                 "b = -4.200000e+01 : f32, "
+                 "label = \"bar foo\"}> : () -> ()\n",
+                 os.str().c_str());
+  }
+  // Get a mutable reference to the properties for this operation and modify it
+  // in place one member at a time.
+  TestProperties &prop = opWithProp.getProperties();
+  prop.a = 42;
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    opWithProp.print(os);
+    EXPECT_TRUE(StringRef(os.str()).contains("a = 42"));
+    EXPECT_TRUE(StringRef(os.str()).contains("b = -4.200000e+01"));
+    EXPECT_TRUE(StringRef(os.str()).contains("array = array<i64: 40, 41>"));
+    EXPECT_TRUE(StringRef(os.str()).contains("label = \"bar foo\""));
+  }
+  prop.b = 42.;
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    opWithProp.print(os);
+    EXPECT_TRUE(StringRef(os.str()).contains("a = 42"));
+    EXPECT_TRUE(StringRef(os.str()).contains("b = 4.200000e+01"));
+    EXPECT_TRUE(StringRef(os.str()).contains("array = array<i64: 40, 41>"));
+    EXPECT_TRUE(StringRef(os.str()).contains("label = \"bar foo\""));
+  }
+  prop.array.push_back(42);
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    opWithProp.print(os);
+    EXPECT_TRUE(StringRef(os.str()).contains("a = 42"));
+    EXPECT_TRUE(StringRef(os.str()).contains("b = 4.200000e+01"));
+    EXPECT_TRUE(StringRef(os.str()).contains("array = array<i64: 40, 41, 42>"));
+    EXPECT_TRUE(StringRef(os.str()).contains("label = \"bar foo\""));
+  }
+  prop.label = std::make_shared<std::string>("foo bar");
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    opWithProp.print(os);
+    EXPECT_TRUE(StringRef(os.str()).contains("a = 42"));
+    EXPECT_TRUE(StringRef(os.str()).contains("b = 4.200000e+01"));
+    EXPECT_TRUE(StringRef(os.str()).contains("array = array<i64: 40, 41, 42>"));
+    EXPECT_TRUE(StringRef(os.str()).contains("label = \"foo bar\""));
+  }
+}
+
+// Test diagnostic emission when using invalid dictionary.
+TEST(OpPropertiesTest, FailedProperties) {
+  MLIRContext context;
+  context.getOrLoadDialect<TestOpPropertiesDialect>();
+  std::string diagnosticStr;
+  context.getDiagEngine().registerHandler([&](Diagnostic &diag) {
+    diagnosticStr += diag.str();
+    return success();
+  });
+
+  // Parse the operation with some properties.
+  ParserConfig config(&context);
+
+  // Parse an operation with invalid (incomplete) properties.
+  OwningOpRef<Operation *> owningOp =
+      parseSourceString("\"test_op_properties.op_with_properties\"() "
+                        "<{a = -42 : i32}> : () -> ()\n",
+                        config);
+  ASSERT_EQ(owningOp.get(), nullptr);
+  EXPECT_STREQ(
+      "invalid properties {a = -42 : i32} for op "
+      "test_op_properties.op_with_properties: expected FloatAttr for key `b`",
+      diagnosticStr.c_str());
+  diagnosticStr.clear();
+
+  owningOp = parseSourceString(mlirSrc, config);
+  Operation *op = owningOp.get();
+  ASSERT_TRUE(op != nullptr);
+  Location loc = op->getLoc();
+  auto opWithProp = dyn_cast<OpWithProperties>(op);
+  ASSERT_TRUE(opWithProp);
+
+  OperationState state(loc, op->getName());
+  Builder b{&context};
+  NamedAttrList attrs;
+  attrs.push_back(b.getNamedAttr("a", b.getStringAttr("foo")));
+  state.propertiesAttr = attrs.getDictionary(&context);
+  {
+    auto diag = op->emitError("setting properties failed: ");
+    auto result = state.setProperties(op, &diag);
+    EXPECT_TRUE(result.failed());
+  }
+  EXPECT_STREQ("setting properties failed: expected IntegerAttr for key `a`",
+               diagnosticStr.c_str());
+}
+
+TEST(OpPropertiesTest, DefaultValues) {
+  MLIRContext context;
+  context.getOrLoadDialect<TestOpPropertiesDialect>();
+  OperationState state(UnknownLoc::get(&context),
+                       "test_op_properties.op_with_properties");
+  Operation *op = Operation::create(state);
+  ASSERT_TRUE(op != nullptr);
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    op->print(os);
+    EXPECT_TRUE(StringRef(os.str()).contains("a = -1"));
+    EXPECT_TRUE(StringRef(os.str()).contains("b = -1"));
+    EXPECT_TRUE(StringRef(os.str()).contains("array = array<i64: -33>"));
+  }
+  op->erase();
+}
+
+TEST(OpPropertiesTest, Cloning) {
+  MLIRContext context;
+  context.getOrLoadDialect<TestOpPropertiesDialect>();
+  ParserConfig config(&context);
+  // Parse the operation with some properties.
+  OwningOpRef<Operation *> op = parseSourceString(mlirSrc, config);
+  ASSERT_TRUE(op.get() != nullptr);
+  auto opWithProp = dyn_cast<OpWithProperties>(op.get());
+  ASSERT_TRUE(opWithProp);
+  Operation *clone = opWithProp->clone();
+
+  // Check that op and its clone prints equally
+  std::string opStr;
+  std::string cloneStr;
+  {
+    llvm::raw_string_ostream os(opStr);
+    op.get()->print(os);
+  }
+  {
+    llvm::raw_string_ostream os(cloneStr);
+    clone->print(os);
+  }
+  clone->erase();
+  EXPECT_STREQ(opStr.c_str(), cloneStr.c_str());
+}
+
+TEST(OpPropertiesTest, Equivalence) {
+  MLIRContext context;
+  context.getOrLoadDialect<TestOpPropertiesDialect>();
+  ParserConfig config(&context);
+  // Parse the operation with some properties.
+  OwningOpRef<Operation *> op = parseSourceString(mlirSrc, config);
+  ASSERT_TRUE(op.get() != nullptr);
+  auto opWithProp = dyn_cast<OpWithProperties>(op.get());
+  ASSERT_TRUE(opWithProp);
+  llvm::hash_code reference = OperationEquivalence::computeHash(opWithProp);
+  TestProperties &prop = opWithProp.getProperties();
+  prop.a = 42;
+  EXPECT_NE(reference, OperationEquivalence::computeHash(opWithProp));
+  prop.a = -42;
+  EXPECT_EQ(reference, OperationEquivalence::computeHash(opWithProp));
+  prop.b = 42.;
+  EXPECT_NE(reference, OperationEquivalence::computeHash(opWithProp));
+  prop.b = -42.;
+  EXPECT_EQ(reference, OperationEquivalence::computeHash(opWithProp));
+  prop.array.push_back(42);
+  EXPECT_NE(reference, OperationEquivalence::computeHash(opWithProp));
+  prop.array.pop_back();
+  EXPECT_EQ(reference, OperationEquivalence::computeHash(opWithProp));
+}
+
+TEST(OpPropertiesTest, getOrAddProperties) {
+  MLIRContext context;
+  context.getOrLoadDialect<TestOpPropertiesDialect>();
+  OperationState state(UnknownLoc::get(&context),
+                       "test_op_properties.op_with_properties");
+  // Test `getOrAddProperties` API on OperationState.
+  TestProperties &prop = state.getOrAddProperties<TestProperties>();
+  prop.a = 1;
+  prop.b = 2;
+  prop.array = {3, 4, 5};
+  Operation *op = Operation::create(state);
+  ASSERT_TRUE(op != nullptr);
+  {
+    std::string output;
+    llvm::raw_string_ostream os(output);
+    op->print(os);
+    EXPECT_TRUE(StringRef(os.str()).contains("a = 1"));
+    EXPECT_TRUE(StringRef(os.str()).contains("b = 2"));
+    EXPECT_TRUE(StringRef(os.str()).contains("array = array<i64: 3, 4, 5>"));
+  }
+  op->erase();
+}
+
+} // namespace
diff --git a/mlir/unittests/IR/OperationSupportTest.cpp b/mlir/unittests/IR/OperationSupportTest.cpp
index 03cdb4b3776f8..81340d3bfdf70 100644
--- a/mlir/unittests/IR/OperationSupportTest.cpp
+++ b/mlir/unittests/IR/OperationSupportTest.cpp
@@ -22,9 +22,9 @@ static Operation *createOp(MLIRContext *context,
                            ArrayRef<Type> resultTypes = std::nullopt,
                            unsigned int numRegions = 0) {
   context->allowUnregisteredDialects();
-  return Operation::create(UnknownLoc::get(context),
-                           OperationName("foo.bar", context), resultTypes,
-                           operands, std::nullopt, std::nullopt, numRegions);
+  return Operation::create(
+      UnknownLoc::get(context), OperationName("foo.bar", context), resultTypes,
+      operands, std::nullopt, nullptr, std::nullopt, numRegions);
 }
 
 namespace {
diff --git a/mlir/unittests/Transforms/DialectConversion.cpp b/mlir/unittests/Transforms/DialectConversion.cpp
index f4a60af82d126..10d7fb041278d 100644
--- a/mlir/unittests/Transforms/DialectConversion.cpp
+++ b/mlir/unittests/Transforms/DialectConversion.cpp
@@ -13,9 +13,9 @@ using namespace mlir;
 
 static Operation *createOp(MLIRContext *context) {
   context->allowUnregisteredDialects();
-  return Operation::create(UnknownLoc::get(context),
-                           OperationName("foo.bar", context), std::nullopt,
-                           std::nullopt, std::nullopt, std::nullopt, 0);
+  return Operation::create(
+      UnknownLoc::get(context), OperationName("foo.bar", context), std::nullopt,
+      std::nullopt, std::nullopt, /*properties=*/nullptr, std::nullopt, 0);
 }
 
 namespace {
diff --git a/mlir/utils/lldb-scripts/action_debugging.py b/mlir/utils/lldb-scripts/action_debugging.py
new file mode 100644
index 0000000000000..2371967bf72a4
--- /dev/null
+++ b/mlir/utils/lldb-scripts/action_debugging.py
@@ -0,0 +1,568 @@
+#!/usr/bin/env python
+
+# ---------------------------------------------------------------------
+# Be sure to add the python path that points to the LLDB shared library.
+#
+# # To use this in the embedded python interpreter using "lldb" just
+# import it with the full path using the "command script import"
+# command
+#   (lldb) command script import /path/to/cmdtemplate.py
+# ---------------------------------------------------------------------
+
+import inspect
+import lldb
+import argparse
+import shlex
+import sys
+
+# Each new breakpoint gets a unique ID starting from 1.
+nextid = 1
+# List of breakpoint set from python, the key is the ID and the value the
+# actual breakpoint. These are NOT LLDB SBBreakpoint objects.
+breakpoints = dict()
+
+exprOptions = lldb.SBExpressionOptions()
+exprOptions.SetIgnoreBreakpoints()
+exprOptions.SetLanguage(lldb.eLanguageTypeC)
+
+
+class MlirDebug:
+    """MLIR debugger commands
+    This is the class that hooks into LLDB and registers the `mlir` command.
+    Other providers can register subcommands below this one.
+    """
+
+    lldb_command = "mlir"
+    parser = None
+
+    def __init__(self, debugger, unused):
+        super().__init__()
+        self.create_options()
+        self.help_string = MlirDebug.parser.format_help()
+
+    @classmethod
+    def create_options(cls):
+        if MlirDebug.parser:
+            return MlirDebug.parser
+        usage = "usage: %s [options]" % (cls.lldb_command)
+        description = "TODO."
+
+        # Pass add_help_option = False, since this keeps the command in line
+        # with lldb commands, and we wire up "help command" to work by
+        # providing the long & short help methods below.
+        MlirDebug.parser = argparse.ArgumentParser(
+            prog=cls.lldb_command, usage=usage, description=description, add_help=False
+        )
+        MlirDebug.subparsers = MlirDebug.parser.add_subparsers(dest="command")
+        return MlirDebug.parser
+
+    def get_short_help(self):
+        return "MLIR debugger commands"
+
+    def get_long_help(self):
+        return self.help_string
+
+    def __call__(self, debugger, command, exe_ctx, result):
+        # Use the Shell Lexer to properly parse up command options just like a
+        # shell would
+        command_args = shlex.split(command)
+
+        try:
+            args = MlirDebug.parser.parse_args(command_args)
+        except:
+            result.SetError("option parsing failed")
+            raise
+        args.func(args, debugger, command, exe_ctx, result)
+
+    @classmethod
+    def on_process_start(frame, bp_loc, dict):
+        print("Process started")
+
+
+class SetControl:
+    # Define the subcommands that controls what to do when a breakpoint is hit.
+    # The key is the subcommand name, the value is a tuple of the command ID to
+    # pass to MLIR and the help string.
+    commands = {
+        "apply": (1, "Apply the current action and continue the execution"),
+        "skip": (2, "Skip the current action and continue the execution"),
+        "step": (3, "Step into the current action"),
+        "next": (4, "Step over the current action"),
+        "finish": (5, "Step out of the current action"),
+    }
+
+    @classmethod
+    def register_mlir_subparser(cls):
+        for cmd, (cmdInt, help) in cls.commands.items():
+            parser = MlirDebug.subparsers.add_parser(
+                cmd,
+                help=help,
+            )
+            parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError("No valid frame (program not running?)")
+            return
+        cmdInt = cls.commands.get(options.command, None)
+        if not cmdInt:
+            result.SetError("Invalid command: %s" % (options.command))
+            return
+
+        result = frame.EvaluateExpression(
+            "((bool (*)(int))mlirDebuggerSetControl)(%d)" % (cmdInt[0]),
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error setting up command: %s" % (result.error))
+            return
+        debugger.SetAsync(True)
+        result = exe_ctx.GetProcess().Continue()
+        debugger.SetAsync(False)
+
+
+class PrintContext:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "context", help="Print the current context"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError("Can't print context without a valid frame")
+            return
+        result = frame.EvaluateExpression(
+            "((bool (*)())&mlirDebuggerPrintContext)()", exprOptions
+        )
+        if not result.error.Success():
+            print("Error printing context: %s" % (result.error))
+            return
+
+
+class Backtrace:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "backtrace", aliases=["bt"], help="Print the current backtrace"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+        cls.parser.add_argument("--context", default=False, action="store_true")
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't backtrace without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)(bool))mlirDebuggerPrintActionBacktrace)(%d)" % (options.context),
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error printing breakpoints: %s" % (result.error))
+            return
+
+
+###############################################################################
+# Cursor manipulation
+###############################################################################
+
+
+class PrintCursor:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "cursor-print", aliases=["cursor-p"], help="Print the current cursor"
+        )
+        cls.parser.add_argument(
+            "--print-region", "--regions", "-r", default=False, action="store_true"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't print cursor without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)(bool))mlirDebuggerCursorPrint)(%d)" % (options.print_region),
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error printing cursor: %s" % (result.error))
+            return
+
+
+class SelectCursorFromContext:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "cursor-select-from-context",
+            aliases=["cursor-s"],
+            help="Select the cursor from the current context",
+        )
+        cls.parser.add_argument("index", type=int, help="Index in the context")
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't manipulate cursor without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)(int))mlirDebuggerCursorSelectIRUnitFromContext)(%d)"
+            % options.index,
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error manipulating cursor: %s" % (result.error))
+            return
+
+
+class CursorSelectParent:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "cursor-parent", aliases=["cursor-up"], help="Select the cursor parent"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't manipulate cursor without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)())mlirDebuggerCursorSelectParentIRUnit)()",
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error manipulating cursor: %s" % (result.error))
+            return
+
+
+class SelectCursorChild:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "cursor-child", aliases=["cursor-c"], help="Select the nth child"
+        )
+        cls.parser.add_argument("index", type=int, help="Index of the child to select")
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't manipulate cursor without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)(int))mlirDebuggerCursorSelectChildIRUnit)(%d)" % options.index,
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error manipulating cursor: %s" % (result.error))
+            return
+
+
+class CursorSelecPrevious:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "cursor-previous",
+            aliases=["cursor-prev"],
+            help="Select the cursor previous element",
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't manipulate cursor without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)())mlirDebuggerCursorSelectPreviousIRUnit)()",
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error manipulating cursor: %s" % (result.error))
+            return
+
+
+class CursorSelecNext:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "cursor-next", aliases=["cursor-n"], help="Select the cursor next element"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        frame = exe_ctx.GetFrame()
+        if not frame.IsValid():
+            result.SetError(
+                "Can't manipulate cursor without a valid frame (program not running?)"
+            )
+        result = frame.EvaluateExpression(
+            "((bool(*)())mlirDebuggerCursorSelectNextIRUnit)()",
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error manipulating cursor: %s" % (result.error))
+            return
+
+
+###############################################################################
+# Breakpoints
+###############################################################################
+
+
+class EnableBreakpoint:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "enable", help="Enable a single breakpoint (given its ID)"
+        )
+        cls.parser.add_argument("id", help="ID of the breakpoint to enable")
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        bp = breakpoints.get(int(options.id), None)
+        if not bp:
+            result.SetError("No breakpoint with ID %d" % int(options.id))
+            return
+        bp.enable(exe_ctx.GetFrame())
+
+
+class DisableBreakpoint:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "disable", help="Disable a single breakpoint (given its ID)"
+        )
+        cls.parser.add_argument("id", help="ID of the breakpoint to disable")
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        bp = breakpoints.get(int(options.id), None)
+        if not bp:
+            result.SetError("No breakpoint with ID %s" % options.id)
+            return
+        bp.disable(exe_ctx.GetFrame())
+
+
+class ListBreakpoints:
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            "list", help="List all current breakpoints"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        for id, bp in sorted(breakpoints.items()):
+            print(id, type(id), str(bp), "enabled" if bp.isEnabled else "disabled")
+
+
+class Breakpoint:
+    def __init__(self):
+        global nextid
+        self.id = nextid
+        nextid += 1
+        breakpoints[self.id] = self
+        self.isEnabled = True
+
+    def enable(self, frame=None):
+        self.isEnabled = True
+        if not frame or not frame.IsValid():
+            return
+        # use a C cast to force the type of the breakpoint handle to be void * so
+        # that we don't rely on DWARF. Also add a fake bool return value otherwise
+        # LLDB can't signal any error with the expression evaluation (at least I don't know how).
+        cmd = (
+            "((bool (*)(void *))mlirDebuggerEnableBreakpoint)((void *)%s)" % self.handle
+        )
+        result = frame.EvaluateExpression(cmd, exprOptions)
+        if not result.error.Success():
+            print("Error enabling breakpoint: %s" % (result.error))
+            return
+
+    def disable(self, frame=None):
+        self.isEnabled = False
+        if not frame or not frame.IsValid():
+            return
+        # use a C cast to force the type of the breakpoint handle to be void * so
+        # that we don't rely on DWARF. Also add a fake bool return value otherwise
+        # LLDB can't signal any error with the expression evaluation (at least I don't know how).
+        cmd = (
+            "((bool (*)(void *)) mlirDebuggerDisableBreakpoint)((void *)%s)"
+            % self.handle
+        )
+        result = frame.EvaluateExpression(cmd, exprOptions)
+        if not result.error.Success():
+            print("Error disabling breakpoint: %s" % (result.error))
+            return
+
+
+class TagBreakpoint(Breakpoint):
+    mlir_subcommand = "break-on-tag"
+
+    def __init__(self, tag):
+        super().__init__()
+        self.tag = tag
+
+    def __str__(self):
+        return "[%d] TagBreakpoint(%s)" % (self.id, self.tag)
+
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            cls.mlir_subcommand, help="add a breakpoint on actions' tag matching"
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+        cls.parser.add_argument("tag", help="tag to match")
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        breakpoint = TagBreakpoint(options.tag)
+        print("Added breakpoint %s" % str(breakpoint))
+
+        frame = exe_ctx.GetFrame()
+        if frame.IsValid():
+            breakpoint.install(frame)
+
+    def install(self, frame):
+        result = frame.EvaluateExpression(
+            '((void *(*)(const char *))mlirDebuggerAddTagBreakpoint)("%s")'
+            % (self.tag),
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error installing breakpoint: %s" % (result.error))
+            return
+        # Save the handle, this is necessary to implement enable/disable.
+        self.handle = result.GetValue()
+
+
+class FileLineBreakpoint(Breakpoint):
+    mlir_subcommand = "break-on-file"
+
+    def __init__(self, file, line, col):
+        super().__init__()
+        self.file = file
+        self.line = line
+        self.col = col
+
+    def __str__(self):
+        return "[%d] FileLineBreakpoint(%s, %d, %d)" % (
+            self.id,
+            self.file,
+            self.line,
+            self.col,
+        )
+
+    @classmethod
+    def register_mlir_subparser(cls):
+        cls.parser = MlirDebug.subparsers.add_parser(
+            cls.mlir_subcommand,
+            help="add a breakpoint that filters on location of the IR affected by an action. The syntax is file:line:col where file and col are optional",
+        )
+        cls.parser.set_defaults(func=cls.process_options)
+        cls.parser.add_argument("location", type=str)
+
+    @classmethod
+    def process_options(cls, options, debugger, command, exe_ctx, result):
+        split_loc = options.location.split(":")
+        file = split_loc[0]
+        line = int(split_loc[1]) if len(split_loc) > 1 else -1
+        col = int(split_loc[2]) if len(split_loc) > 2 else -1
+        breakpoint = FileLineBreakpoint(file, line, col)
+        print("Added breakpoint %s" % str(breakpoint))
+
+        frame = exe_ctx.GetFrame()
+        if frame.IsValid():
+            breakpoint.install(frame)
+
+    def install(self, frame):
+        result = frame.EvaluateExpression(
+            '((void *(*)(const char *, int, int))mlirDebuggerAddFileLineColLocBreakpoint)("%s", %d, %d)'
+            % (self.file, self.line, self.col),
+            exprOptions,
+        )
+        if not result.error.Success():
+            print("Error installing breakpoint: %s" % (result.error))
+            return
+        # Save the handle, this is necessary to implement enable/disable.
+        self.handle = result.GetValue()
+
+
+def on_start(frame, bpno, err):
+    print("MLIR debugger attaching...")
+    for _, bp in sorted(breakpoints.items()):
+        if bp.isEnabled:
+            print("Installing breakpoint %s" % (str(bp)))
+            bp.install(frame)
+        else:
+            print("Skipping disabled breakpoint %s" % (str(bp)))
+
+    return True
+
+
+def __lldb_init_module(debugger, dict):
+    target = debugger.GetTargetAtIndex(0)
+    debugger.SetAsync(False)
+    if not target:
+        print("No target is loaded, please load a target before loading this script.")
+        return
+    if debugger.GetNumTargets() > 1:
+        print(
+            "Multiple targets (%s) loaded, attaching MLIR debugging to %s"
+            % (debugger.GetNumTargets(), target)
+        )
+
+    # Register all classes that have a register_lldb_command method
+    module_name = __name__
+    parser = MlirDebug.create_options()
+    MlirDebug.__doc__ = parser.format_help()
+
+    # Add the MLIR entry point to LLDB as a command.
+    command = "command script add -o -c %s.%s %s" % (
+        module_name,
+        MlirDebug.__name__,
+        MlirDebug.lldb_command,
+    )
+    debugger.HandleCommand(command)
+
+    main_bp = target.BreakpointCreateByName("main")
+    main_bp.SetScriptCallbackFunction("action_debugging.on_start")
+    main_bp.SetAutoContinue(auto_continue=True)
+
+    on_breackpoint = target.BreakpointCreateByName("mlirDebuggerBreakpointHook")
+
+    print(
+        'The "{0}" command has been installed for target `{1}`, type "help {0}" or "{0} '
+        '--help" for detailed help.'.format(MlirDebug.lldb_command, target)
+    )
+    for _name, cls in inspect.getmembers(sys.modules[module_name]):
+        if inspect.isclass(cls) and getattr(cls, "register_mlir_subparser", None):
+            cls.register_mlir_subparser()
diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 98f47bc1c6326..1402192581d3e 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -1160,6 +1160,7 @@ There are several environment variables to change the behavior of the plugins:
 * ``LIBOMPTARGET_AMDGPU_TEAMS_PER_CU``
 * ``LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES``
 * ``LIBOMPTARGET_AMDGPU_NUM_INITIAL_HSA_SIGNALS``
+* ``LIBOMPTARGET_AMDGPU_STREAM_BUSYWAIT``
 
 The environment variables ``LIBOMPTARGET_SHARED_MEMORY_SIZE``,
 ``LIBOMPTARGET_STACK_SIZE`` and ``LIBOMPTARGET_HEAP_SIZE`` are described in
@@ -1238,6 +1239,14 @@ managing several pre-created signals. These signals are mainly used by AMDGPU
 streams. More HSA signals will be created dynamically throughout the execution
 if needed. The default value is ``64``.
 
+LIBOMPTARGET_AMDGPU_STREAM_BUSYWAIT
+"""""""""""""""""""""""""""""""""""
+
+This environment variable controls the timeout hint in microseconds for the
+HSA wait state within the AMDGPU plugin. For the duration of this value
+the HSA runtime may busy wait. This can reduce overall latency.
+The default value is ``2000000``.
+
 .. _remote_offloading_plugin:
 
 Remote Offloading Plugin:
diff --git a/openmp/docs/doxygen.cfg.in b/openmp/docs/doxygen.cfg.in
index f52c234da05de..6292a453f749e 100644
--- a/openmp/docs/doxygen.cfg.in
+++ b/openmp/docs/doxygen.cfg.in
@@ -2091,7 +2091,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2136,7 +2136,7 @@ TEMPLATE_RELATIONS     = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2145,7 +2145,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/openmp/libomptarget/include/Debug.h b/openmp/libomptarget/include/Debug.h
index 91c12688789af..17aa8746afe7c 100644
--- a/openmp/libomptarget/include/Debug.h
+++ b/openmp/libomptarget/include/Debug.h
@@ -55,6 +55,8 @@ enum OpenMPInfoType : uint32_t {
   OMP_INFOTYPE_PLUGIN_KERNEL = 0x0010,
   // Print whenever data is transferred to the device
   OMP_INFOTYPE_DATA_TRANSFER = 0x0020,
+  // Print whenever data does not have a viable device counterpart.
+  OMP_INFOTYPE_EMPTY_MAPPING = 0x0040,
   // Enable every flag.
   OMP_INFOTYPE_ALL = 0xffffffff,
 };
diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h
index 5f147f9c77e64..7bc4629c6220e 100644
--- a/openmp/libomptarget/include/device.h
+++ b/openmp/libomptarget/include/device.h
@@ -312,7 +312,9 @@ struct TargetPointerResultTy {
     /// Flag indicating that this was the last user of the entry and the ref
     /// count is now 0.
     unsigned IsLast : 1;
-  } Flags = {0, 0, 0, 0};
+    /// If the pointer is contained.
+    unsigned IsContained : 1;
+  } Flags = {0, 0, 0, 0, 0};
 
   TargetPointerResultTy(const TargetPointerResultTy &) = delete;
   TargetPointerResultTy &operator=(const TargetPointerResultTy &TPR) = delete;
@@ -348,6 +350,8 @@ struct TargetPointerResultTy {
 
   bool isHostPointer() const { return Flags.IsHostPointer; }
 
+  bool isContained() const { return Flags.IsContained; }
+
   /// The corresponding target pointer
   void *TargetPointer = nullptr;
 
diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
index 95e359cdbf2ec..af02f050f467f 100644
--- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
@@ -52,11 +52,15 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$")
         PluginInterface
         ${LIBOMPTARGET_DEP_LIBFFI_LIBRARIES}
         ${OPENMP_PTHREAD_LIB}
-        "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
 
       NO_INSTALL_RPATH
     )
 
+    if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+      target_link_libraries("omptarget.rtl.${tmachine_libname}.nextgen" PRIVATE
+        "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
+    endif()
+
     # Install plugin under the lib destination folder.
     install(TARGETS "omptarget.rtl.${tmachine_libname}.nextgen"
       LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt
index b813497ab4d87..b689ff5a38d5f 100644
--- a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -85,12 +85,16 @@ add_llvm_library(omptarget.rtl.amdgpu.nextgen SHARED
   PluginInterface
   ${LIBOMPTARGET_DEP_LIBRARIES}
   ${OPENMP_PTHREAD_LIB}
-  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports"
   ${LDFLAGS_UNDEFINED}
 
   NO_INSTALL_RPATH
 )
 
+if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+  target_link_libraries(omptarget.rtl.amdgpu.nextgen PRIVATE
+    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
+endif()
+
 target_include_directories(
   omptarget.rtl.amdgpu.nextgen
   PRIVATE
diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
index f9b0371f903a8..0d2d8fae149ea 100644
--- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -511,8 +511,14 @@ struct AMDGPUSignalTy {
   }
 
   /// Wait until the signal gets a zero value.
-  Error wait() const {
-    // TODO: Is it better to use busy waiting or blocking the thread?
+  Error wait(const uint64_t ActiveTimeout = 0) const {
+    if (ActiveTimeout) {
+      hsa_signal_value_t Got = 1;
+      Got = hsa_signal_wait_scacquire(Signal, HSA_SIGNAL_CONDITION_EQ, 0,
+                                      ActiveTimeout, HSA_WAIT_STATE_ACTIVE);
+      if (Got == 0)
+        return Plugin::success();
+    }
     while (hsa_signal_wait_scacquire(Signal, HSA_SIGNAL_CONDITION_EQ, 0,
                                      UINT64_MAX, HSA_WAIT_STATE_BLOCKED) != 0)
       ;
@@ -884,6 +890,9 @@ struct AMDGPUStreamTy {
   /// Mutex to protect stream's management.
   mutable std::mutex Mutex;
 
+  /// Timeout hint for HSA actively waiting for signal value to change
+  const uint64_t StreamBusyWaitMicroseconds;
+
   /// Return the current number of asychronous operations on the stream.
   uint32_t size() const { return NextSlot; }
 
@@ -1247,7 +1256,7 @@ struct AMDGPUStreamTy {
       return Plugin::success();
 
     // Wait until all previous operations on the stream have completed.
-    if (auto Err = Slots[last()].Signal->wait())
+    if (auto Err = Slots[last()].Signal->wait(StreamBusyWaitMicroseconds))
       return Err;
 
     // Reset the stream and perform all pending post actions.
@@ -1555,6 +1564,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                                1 * 1024 * 1024), // 1MB
         OMPX_InitialNumSignals("LIBOMPTARGET_AMDGPU_NUM_INITIAL_HSA_SIGNALS",
                                64),
+        OMPX_StreamBusyWait("LIBOMPTARGET_AMDGPU_STREAM_BUSYWAIT", 2000000),
         AMDGPUStreamManager(*this), AMDGPUEventManager(*this),
         AMDGPUSignalManager(*this), Agent(Agent), HostDevice(HostDevice),
         Queues() {}
@@ -1679,6 +1689,10 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  const uint64_t getStreamBusyWaitMicroseconds() const {
+    return OMPX_StreamBusyWait;
+  }
+
   Expected<std::unique_ptr<MemoryBuffer>>
   doJITPostProcessing(std::unique_ptr<MemoryBuffer> MB) const override {
 
@@ -1941,7 +1955,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
               Plugin::check(Status, "Error in hsa_amd_memory_async_copy: %s"))
         return Err;
 
-      if (auto Err = Signal.wait())
+      if (auto Err = Signal.wait(getStreamBusyWaitMicroseconds()))
         return Err;
 
       if (auto Err = Signal.deinit())
@@ -1998,7 +2012,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
               Plugin::check(Status, "Error in hsa_amd_memory_async_copy: %s"))
         return Err;
 
-      if (auto Err = Signal.wait())
+      if (auto Err = Signal.wait(getStreamBusyWaitMicroseconds()))
         return Err;
 
       if (auto Err = Signal.deinit())
@@ -2173,6 +2187,12 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// will be created.
   UInt32Envar OMPX_InitialNumSignals;
 
+  /// Environment variables to set the time to wait in active state before
+  /// switching to blocked state. The default 2000000 busywaits for 2 seconds
+  /// before going into a blocking HSA wait state. The unit for these variables
+  /// are microseconds.
+  UInt32Envar OMPX_StreamBusyWait;
+
   /// Stream manager for AMDGPU streams.
   AMDGPUStreamManagerTy AMDGPUStreamManager;
 
@@ -2267,7 +2287,8 @@ AMDGPUStreamTy::AMDGPUStreamTy(AMDGPUDeviceTy &Device)
     : Agent(Device.getAgent()), Queue(Device.getNextQueue()),
       SignalManager(Device.getSignalManager()),
       // Initialize the std::deque with some empty positions.
-      Slots(32), NextSlot(0), SyncCycle(0) {}
+      Slots(32), NextSlot(0), SyncCycle(0),
+      StreamBusyWaitMicroseconds(Device.getStreamBusyWaitMicroseconds()) {}
 
 /// Class implementing the AMDGPU-specific functionalities of the global
 /// handler.
diff --git a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt
index 6b7d2d743c73f..397b06b676923 100644
--- a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt
@@ -38,11 +38,16 @@ add_llvm_library(omptarget.rtl.cuda.nextgen SHARED
   MemoryManager
   PluginInterface
   ${OPENMP_PTHREAD_LIB}
-  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports,-z,defs"
 
   NO_INSTALL_RPATH
 )
 
+if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+  target_link_libraries(omptarget.rtl.cuda.nextgen PRIVATE
+  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports,-z,defs")
+endif()
+
+
 if(LIBOMPTARGET_DEP_CUDA_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA)
   libomptarget_say("Building CUDA plugin linked against libcuda")
   target_link_libraries(omptarget.rtl.cuda.nextgen PRIVATE CUDA::cuda_driver)
diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt
index 2a6cd93c6ec98..ef3a6270b78a5 100644
--- a/openmp/libomptarget/src/CMakeLists.txt
+++ b/openmp/libomptarget/src/CMakeLists.txt
@@ -31,13 +31,15 @@ add_llvm_library(omptarget
   Support
   Object
 
-  LINK_LIBS
-  PRIVATE
-  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports"
   NO_INSTALL_RPATH
 )
 target_include_directories(omptarget PRIVATE ${LIBOMPTARGET_INCLUDE_DIR})
 
+if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+  target_link_libraries(omptarget PRIVATE
+    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+endif()
+
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
 set_target_properties(omptarget PROPERTIES INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 0459648a4802f..04201e8d7a601 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -108,8 +108,9 @@ static const int64_t MaxAlignment = 16;
 /// Return the alignment requirement of partially mapped structs, see
 /// MaxAlignment above.
 static uint64_t getPartialStructRequiredAlignment(void *HstPtrBase) {
-  auto BaseAlignment = reinterpret_cast<uintptr_t>(HstPtrBase) % MaxAlignment;
-  return BaseAlignment == 0 ? MaxAlignment : BaseAlignment;
+  int LowestOneBit = __builtin_ffsl(reinterpret_cast<uintptr_t>(HstPtrBase));
+  uint64_t BaseAlignment = 1 << (LowestOneBit - 1);
+  return MaxAlignment < BaseAlignment ? MaxAlignment : BaseAlignment;
 }
 
 /// Map global data and execute pending ctors
@@ -732,6 +733,15 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
           return OFFLOAD_FAIL;
       }
     }
+
+    // Check if variable can be used on the device:
+    bool IsStructMember = ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF;
+    if (getInfoLevel() & OMP_INFOTYPE_EMPTY_MAPPING && ArgTypes[I] != 0 &&
+        !IsStructMember && !IsImplicit && !TPR.isPresent() &&
+        !TPR.isContained() && !TPR.isHostPointer())
+      INFO(OMP_INFOTYPE_EMPTY_MAPPING, Device.DeviceID,
+           "variable %s does not have a valid device counterpart\n",
+           (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
   }
 
   return OFFLOAD_SUCCESS;
diff --git a/openmp/libomptarget/test/lit.cfg b/openmp/libomptarget/test/lit.cfg
index 33138e910d5fc..489d6109649ff 100644
--- a/openmp/libomptarget/test/lit.cfg
+++ b/openmp/libomptarget/test/lit.cfg
@@ -64,7 +64,8 @@ config.test_format = lit.formats.ShTest()
 # compiler flags
 config.test_flags = " -I " + config.test_source_root + \
     " -I " + config.omp_header_directory + \
-    " -L " + config.library_dir;
+    " -L " + config.library_dir + \
+    " -L " + config.llvm_lib_directory
 
 if config.omp_host_rtl_directory:
     config.test_flags = config.test_flags + " -L " + \
diff --git a/openmp/libomptarget/test/mapping/ompx_hold/struct.c b/openmp/libomptarget/test/mapping/ompx_hold/struct.c
index f61ba44dc2a3d..da2b38028762c 100644
--- a/openmp/libomptarget/test/mapping/ompx_hold/struct.c
+++ b/openmp/libomptarget/test/mapping/ompx_hold/struct.c
@@ -5,9 +5,16 @@
 #include <stdio.h>
 
 #pragma omp begin declare target
-char *N1, *N2;
+#define MAX_NAME_SIZE 100
+char N1[MAX_NAME_SIZE], N2[MAX_NAME_SIZE];
 int V1, V2;
-#pragma omp declare target
+void copy_name(char *dst, char *src) {
+  int i;
+  for (i = 0; i < MAX_NAME_SIZE - 1 && src[i]; ++i)
+    dst[i] = src[i];
+  dst[i] = 0;
+}
+#pragma omp end declare target
 
 #define CHECK_PRESENCE(Var1, Var2, Var3)                                       \
   printf("    presence of %s, %s, %s: %d, %d, %d\n", #Var1, #Var2, #Var3,      \
@@ -19,12 +26,12 @@ int V1, V2;
   printf("    values of %s, %s: %d, %d\n", N1, N2, (Var1), (Var2))
 
 #define CHECK_VALUES_DELAYED(Var1, Var2)                                       \
-  N1 = #Var1;                                                                  \
-  N2 = #Var2;                                                                  \
+  copy_name(N1, #Var1);                                                        \
+  copy_name(N2, #Var2);                                                        \
   V1 = (Var1);                                                                 \
   V2 = (Var2);
 
-#define CHECK_DELAYED_VALUS()                                                  \
+#define CHECK_DELAYED_VALUES()                                                 \
   _Pragma("omp target update from(N1, N2, V1, V2)")                            \
       CHECK_VALUES_HELPER(N1, N2, V1, V2)
 
@@ -149,7 +156,7 @@ int main() {
       // CHECK-NEXT: values of s.i, s.j: 21, 31
       CHECK_VALUES_DELAYED(s.i, s.j);
     }
-    CHECK_DELAYED_VALUS();
+    CHECK_DELAYED_VALUES();
   }
   // CHECK-NEXT: presence of s, s.i, s.j: 0, 0, 0
   // CHECK-NEXT: values of s.i, s.j: 21, 31
@@ -180,7 +187,7 @@ int main() {
       // CHECK-NEXT: values of s.i, s.j: 21, 31
       CHECK_VALUES_DELAYED(s.i, s.j);
     }
-    CHECK_DELAYED_VALUS();
+    CHECK_DELAYED_VALUES();
   }
   // CHECK-NEXT: presence of s, s.i, s.j: 0, 0, 0
   // CHECK-NEXT: values of s.i, s.j: 21, 31
diff --git a/openmp/libomptarget/test/mapping/power_of_two_alignment.c b/openmp/libomptarget/test/mapping/power_of_two_alignment.c
new file mode 100644
index 0000000000000..06b0e457983fb
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/power_of_two_alignment.c
@@ -0,0 +1,87 @@
+// RUN: %libomptarget-compilexx-run-and-check-generic
+
+// Assuming the stack is allocated on the host starting at high addresses, the
+// host memory layout for the following program looks like this:
+//
+//   low addr <----------------------------------------------------- high addr
+//              |   16 bytes  | 16 bytes  |  16 bytes  | ? bytes  |
+//              | collidePost |     s     | collidePre | stackPad |
+//              |             | x | y | z |            |          |
+//              `-------------'
+//                    ^  `--------'
+//                    |      ^
+//                    |      |
+//                    |      `-- too much padding (< 16 bytes) for s maps here
+//                    |
+//                    `------------------array extension error maps here
+//
+// libomptarget used to add too much padding to the device allocation of s and
+// map it back to the host at the location indicated above when all of the
+// following conditions were true:
+// - Multiple members (s.y and s.z below) were mapped.  In this case, initial
+//   padding might be needed to ensure later mapped members (s.z) are aligned
+//   properly on the device.  (If the first member in the struct, s.x, were also
+//   mapped, then the correct initial padding would always be zero.)
+// - mod16 = &s % 16 was not a power of 2 (e.g., 0x7ffcce2b584e % 16 = 14).
+//   libomptarget then incorrectly assumed mod16 was the existing host memory
+//   alignment of s.  (The fix was to only look for alignments that are powers
+//   of 2.)
+// - &s.y % mod16 was > 1 (e.g., 0x7ffcce2b584f % 14 = 11).  libomptarget added
+//   padding of that size for s, but at most 1 byte is ever actually needed.
+//
+// Below, we try many sizes of stackPad to try to produce those conditions.
+//
+// When collidePost was then mapped to the same host memory as the unnecessary
+// padding for s, libomptarget reported an array extension error.  collidePost
+// is never fully contained within that padding (which would avoid the extension
+// error) because collidePost is 16 bytes while the padding is always less than
+// 16 bytes due to the modulo operations.
+
+#include <stdint.h>
+#include <stdio.h>
+
+template <typename StackPad>
+void test() {
+  StackPad stackPad;
+  struct S { char x; char y[7]; char z[8]; };
+  struct S collidePre, s, collidePost;
+  uintptr_t mod16 = (uintptr_t)&s % 16;
+  fprintf(stderr, "&s = %p\n", &s);
+  fprintf(stderr, "&s %% 16 = %lu\n", mod16);
+  if (mod16) {
+    fprintf(stderr, "&s.y = %p\n", &s.y);
+    fprintf(stderr, "&s.y %% %lu = %lu\n", mod16, (uintptr_t)&s.y % mod16);
+  }
+  fprintf(stderr, "&collidePre = %p\n", &collidePre);
+  fprintf(stderr, "&collidePost = %p\n", &collidePost);
+  #pragma omp target data map(to:s.y, s.z)
+  #pragma omp target data map(to:collidePre, collidePost)
+  ;
+}
+
+#define TEST(StackPad)                                                         \
+  fprintf(stderr, "-------------------------------------\n");                  \
+  fprintf(stderr, "StackPad=%s\n", #StackPad);                                 \
+  test<StackPad>()
+
+int main() {
+  TEST(char[1]);
+  TEST(char[2]);
+  TEST(char[3]);
+  TEST(char[4]);
+  TEST(char[5]);
+  TEST(char[6]);
+  TEST(char[7]);
+  TEST(char[8]);
+  TEST(char[9]);
+  TEST(char[10]);
+  TEST(char[11]);
+  TEST(char[12]);
+  TEST(char[13]);
+  TEST(char[14]);
+  TEST(char[15]);
+  TEST(char[16]);
+  // CHECK: pass
+  printf("pass\n");
+  return 0;
+}
diff --git a/openmp/libomptarget/test/mapping/private_mapping.c b/openmp/libomptarget/test/mapping/private_mapping.c
index cd4a2e5111f67..1329a66a5da3a 100644
--- a/openmp/libomptarget/test/mapping/private_mapping.c
+++ b/openmp/libomptarget/test/mapping/private_mapping.c
@@ -1,4 +1,5 @@
 // RUN: %libomptarget-compile-run-and-check-generic
+// UNSUPPORTED: amdgcn-amd-amdhsa
 
 #include <assert.h>
 #include <stdio.h>
diff --git a/openmp/libomptarget/test/mapping/target_wrong_use_device_addr.c b/openmp/libomptarget/test/mapping/target_wrong_use_device_addr.c
new file mode 100644
index 0000000000000..bb5a1d44031a3
--- /dev/null
+++ b/openmp/libomptarget/test/mapping/target_wrong_use_device_addr.c
@@ -0,0 +1,27 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=51 -g
+// RUN: env LIBOMPTARGET_INFO=64 %libomptarget-run-fail-generic 2>&1 \
+// RUN: | %fcheck-generic
+
+#include <stdio.h>
+
+int main() {
+  float arr[10];
+  float *x = &arr[0];
+
+  // CHECK: host addr=0x[[#%x,HOST_ADDR:]]
+  fprintf(stderr, "host addr=%p\n", x);
+
+#pragma omp target data map(to : x [0:10])
+  {
+// CHECK: Libomptarget device 0 info: variable x does not have a valid device
+// counterpart
+#pragma omp target data use_device_addr(x)
+    {
+      // CHECK-NOT: device addr=0x[[#%x,HOST_ADDR:]]
+      fprintf(stderr, "device addr=%p\n", x);
+    }
+  }
+
+  return 0;
+}
+
diff --git a/openmp/runtime/doc/doxygen/config b/openmp/runtime/doc/doxygen/config
index 85968dd113436..04c966766ba6e 100644
--- a/openmp/runtime/doc/doxygen/config
+++ b/openmp/runtime/doc/doxygen/config
@@ -1671,7 +1671,7 @@ CLASS_GRAPH            = YES
 # indirect implementation dependencies (inheritance, containment, and
 # class references variables) of the class with other documented classes.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
 # will generate a graph for groups, showing the direct groups dependencies
@@ -1703,14 +1703,14 @@ TEMPLATE_RELATIONS     = YES
 # file showing the direct and indirect include dependencies of the file with
 # other documented files.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
 # HAVE_DOT tags are set to YES then doxygen will generate a graph for each
 # documented header file showing the documented files that directly or
 # indirectly include this file.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH and HAVE_DOT options are set to YES then
 # doxygen will generate a call dependency graph for every global function
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 43e723fa7387b..bd50d58f0fdcb 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -1100,6 +1100,12 @@ extern void __kmp_init_target_mem();
 
 /* ------------------------------------------------------------------------ */
 
+#if ENABLE_LIBOMPTARGET
+extern void __kmp_init_target_task();
+#endif
+
+/* ------------------------------------------------------------------------ */
+
 #define KMP_UINT64_MAX                                                         \
   (~((kmp_uint64)1 << ((sizeof(kmp_uint64) * (1 << 3)) - 1)))
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 776160289981b..a0a72f163e7c5 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -553,6 +553,14 @@ static void __kmp_fini_allocator() { __kmp_fini_memkind(); }
 
 /* ------------------------------------------------------------------------ */
 
+#if ENABLE_LIBOMPTARGET
+static void __kmp_init_omptarget() {
+  __kmp_init_target_task();
+}
+#endif
+
+/* ------------------------------------------------------------------------ */
+
 #if KMP_DYNAMIC_LIB
 #if KMP_OS_WINDOWS
 
@@ -7041,6 +7049,11 @@ static void __kmp_do_serial_initialize(void) {
 
   __kmp_validate_locks();
 
+#if ENABLE_LIBOMPTARGET
+  /* Initialize functions from libomptarget */
+  __kmp_init_omptarget();
+#endif
+
   /* Initialize internal memory allocator */
   __kmp_init_allocator();
 
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 71922ed708001..7e9147eeeebf2 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -22,8 +22,11 @@
 #endif
 
 #if ENABLE_LIBOMPTARGET
-// Declaration of synchronization function from libomptarget.
-extern "C" void __tgt_target_nowait_query(void **) KMP_WEAK_ATTRIBUTE_INTERNAL;
+static void (*tgt_target_nowait_query)(void **);
+
+void __kmp_init_target_task() {
+  *(void **)(&tgt_target_nowait_query) = KMP_DLSYM("__tgt_target_nowait_query");
+}
 #endif
 
 /* forward declaration */
@@ -1805,7 +1808,8 @@ static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
       // If we have a valid target async handle, that means that we have already
       // executed the task routine once. We must query for the handle completion
       // instead of re-executing the routine.
-      __tgt_target_nowait_query(&taskdata->td_target_data.async_handle);
+      KMP_ASSERT(tgt_target_nowait_query);
+      tgt_target_nowait_query(&taskdata->td_target_data.async_handle);
     } else
 #endif
     if (task->routine != NULL) {
diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg
index d008844a0cc57..b1a5b7a351098 100644
--- a/openmp/runtime/test/lit.cfg
+++ b/openmp/runtime/test/lit.cfg
@@ -120,14 +120,6 @@ if target_arch:
   if target_arch in ['x86_64', 'i386']:
     config.available_features.add('x86-target-arch')
 
-# Avoid Windows heuristics which try to detect potential installer programs
-# (which may need to run with elevated privileges) and ask if the user wants
-# to run them in that way. This heuristic may match for executables containing
-# the word "patch" which is a substring of "dispatch". Set an environment
-# variable indicating that we want to execute them with the current user.
-if config.operating_system == 'Windows':
-    config.environment['__COMPAT_LAYER'] = 'RunAsInvoker'
-
 import multiprocessing
 try:
     if multiprocessing.cpu_count() > 1:
diff --git a/openmp/runtime/test/tasking/omp_task_depend_iterator.cpp b/openmp/runtime/test/tasking/omp_task_depend_iterator.cpp
new file mode 100644
index 0000000000000..27eee0a45fc30
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_task_depend_iterator.cpp
@@ -0,0 +1,86 @@
+// RUN: %libomp-cxx-compile-and-run
+
+/*
+
+This test is imported from SOLLVE: 5.0/task/test_task_depend_iterator.cpp
+SOLLVE page: https://github.com/SOLLVE/sollve_vv
+
+OpenMP API Version 5.0 Nov 2020
+
+This test is for the iterator modifier when used with the task depend
+clause. This modifier should create an iterator that expands to multiple values
+inside the clause they appear. In this particular test case the iterator expands into
+several values creating several dependencies at the same time.
+
+*/
+
+#include <omp.h>
+#include <algorithm>
+#include <cstdlib>
+#include <iostream>
+#include <thread>
+#include <vector>
+#include "omp_testsuite.h"
+
+#define N 1024
+#define FROM 64
+#define LENGTH 128
+
+int test_omp_task_depend_iterator() {
+  int ptr[] = {0, 4, 5, 6, 7, 8, 9, 10, 11};
+  int cols[] = {1, 2, 3, 4, 5, 5, 6, 6, 7, 7, 8};
+  std::vector<int> threadOrder;
+  bool threadOrderError = false;
+#pragma omp parallel num_threads(8)
+  {
+#pragma omp single
+    {
+      for (int i = 0; i < 8; ++i) {
+        int pos = ptr[i], size = ptr[i + 1] - ptr[i];
+#pragma omp task depend(iterator(it = 0 : size), in : ptr[cols[pos + it]]) depend(out : ptr[i])
+        {
+#pragma omp critical
+          {
+            threadOrder.push_back(i);
+          } // end critical section
+        } // end task depend
+      }
+    } // end single
+  } // end parallel
+
+  // store the indices of the execution order of generated tasks in idx[]
+  std::vector<int>::iterator idx[8];
+  for (int i = 0; i < 8; ++i)
+    idx[i] = std::find (threadOrder.begin(), threadOrder.end(), i);
+
+  // verify that dependencies are met in the order
+  if (idx[0] != threadOrder.begin())
+    threadOrderError |= true;
+  if (idx[1] > idx[5] || idx[2] > idx[5])
+    threadOrderError |= true;
+  if (idx[3] > idx[6] || idx[4] > idx[6])
+    threadOrderError |= true;
+  if (idx[5] > idx[7] || idx[6] > idx[7])
+    threadOrderError |= true;
+
+  std::sort(threadOrder.begin(), threadOrder.end());
+  for(int i = 0; i < 8; ++i)
+    threadOrderError = (threadOrder[i] != i) || threadOrderError;
+
+  // FALSE If dependencies between tasks were not enforced in the correct order.
+  return !threadOrderError; 
+}
+
+
+
+int main() {
+  int i;
+  int num_failed=0;
+
+  for(i = 0; i < REPETITIONS; i++) {
+    if(!test_omp_task_depend_iterator()) {
+      num_failed++;
+    }
+  }
+  return num_failed;
+}
diff --git a/polly/docs/doxygen.cfg.in b/polly/docs/doxygen.cfg.in
index c2f9f7ff3fa79..befe6e138ec27 100644
--- a/polly/docs/doxygen.cfg.in
+++ b/polly/docs/doxygen.cfg.in
@@ -2103,7 +2103,7 @@ CLASS_GRAPH            = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
@@ -2148,7 +2148,7 @@ TEMPLATE_RELATIONS     = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDE_GRAPH          = YES
+INCLUDE_GRAPH          = NO
 
 # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
 # set to YES then doxygen will generate a graph for each documented file showing
@@ -2157,7 +2157,7 @@ INCLUDE_GRAPH          = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-INCLUDED_BY_GRAPH      = YES
+INCLUDED_BY_GRAPH      = NO
 
 # If the CALL_GRAPH tag is set to YES then doxygen will generate a call
 # dependency graph for every global function or class method.
diff --git a/polly/test/CodeGen/MemAccess/codegen_address_space.ll b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
index 76bcc40f0639a..7c9b12d64f9ca 100644
--- a/polly/test/CodeGen/MemAccess/codegen_address_space.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_address_space.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -instnamer < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll b/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
index ff12fe77f4698..e008a789fe7d1 100644
--- a/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_constant_offset.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -instnamer < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple.ll b/polly/test/CodeGen/MemAccess/codegen_simple.ll
index 0e64a6b6fbb8e..5ba6f3269fb97 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -instnamer < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 
 ;int A[100];
 ;
diff --git a/polly/test/CodeGen/MemAccess/codegen_simple_float.ll b/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
index 65a2d3b9067bb..cf8913fc51975 100644
--- a/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
+++ b/polly/test/CodeGen/MemAccess/codegen_simple_float.ll
@@ -1,4 +1,4 @@
-;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen -instnamer < %s -S | FileCheck %s
+;RUN: opt %loadPolly -polly-import-jscop -polly-import-jscop-postfix=transformed -polly-codegen < %s -S | FileCheck %s
 ;
 ;float A[100];
 ;
diff --git a/polly/test/CodeGen/only_non_affine_error_region.ll b/polly/test/CodeGen/only_non_affine_error_region.ll
index 64f53bcda2c23..b2ad1c1fe3fd9 100644
--- a/polly/test/CodeGen/only_non_affine_error_region.ll
+++ b/polly/test/CodeGen/only_non_affine_error_region.ll
@@ -13,7 +13,7 @@ for.body:                                         ; preds = %entry
   br label %for.end
 
 for.end:                                          ; preds = %for.body, %entry
-  %i.2 = phi i32 [ 1, %entry ], [ 1, %for.body ]
+  %i.2 = phi i32 [ 1, %entry ], [ 2, %for.body ]
   %cmp170 = icmp eq i32 %i.2, %argc
   br i1 %cmp170, label %if.then172, label %if.end174
 
diff --git a/polly/test/Support/dumpfunction.ll b/polly/test/Support/dumpfunction.ll
index de7e802d98e5b..863212b2ef7de 100644
--- a/polly/test/Support/dumpfunction.ll
+++ b/polly/test/Support/dumpfunction.ll
@@ -20,8 +20,6 @@
 
 %unrelated_type = type { i32 }
 
-@callee_alias = dso_local unnamed_addr alias void(i32, ptr, i32), ptr @callee
-
 define internal void @callee(i32 %n, ptr noalias nonnull %A, i32 %i) #0 {
 entry:
   br label %for
diff --git a/pstl/include/pstl/internal/unseq_backend_simd.h b/pstl/include/pstl/internal/unseq_backend_simd.h
index af2a143bc58ab..9521a420609a2 100644
--- a/pstl/include/pstl/internal/unseq_backend_simd.h
+++ b/pstl/include/pstl/internal/unseq_backend_simd.h
@@ -532,17 +532,17 @@ __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryO
 template <typename _Tp, typename _BinaryOp>
 struct _Combiner
 {
-    _Tp __value;
+    _Tp __value_;
     _BinaryOp* __bin_op; // Here is a pointer to function because of default ctor
 
-    _Combiner() : __value{}, __bin_op(nullptr) {}
-    _Combiner(const _Tp& value, const _BinaryOp* bin_op) : __value(value), __bin_op(const_cast<_BinaryOp*>(bin_op)) {}
-    _Combiner(const _Combiner& __obj) : __value{}, __bin_op(__obj.__bin_op) {}
+    _Combiner() : __value_{}, __bin_op(nullptr) {}
+    _Combiner(const _Tp& value, const _BinaryOp* bin_op) : __value_(value), __bin_op(const_cast<_BinaryOp*>(bin_op)) {}
+    _Combiner(const _Combiner& __obj) : __value_{}, __bin_op(__obj.__bin_op) {}
 
     void
     operator()(const _Combiner& __obj)
     {
-        __value = (*__bin_op)(__value, __obj.__value);
+        __value_ = (*__bin_op)(__value_, __obj.__value);
     }
 };
 
@@ -561,12 +561,12 @@ __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryO
     _PSTL_PRAGMA_SIMD_SCAN(__bin_op : __init_)
     for (_Size __i = 0; __i < __n; ++__i)
     {
-        __result[__i] = __init_.__value;
+        __result[__i] = __init_.__value_;
         _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(__init_)
         _PSTL_PRAGMA_FORCEINLINE
-        __init_.__value = __binary_op(__init_.__value, __unary_op(__first[__i]));
+        __init_.__value_ = __binary_op(__init_.__value_, __unary_op(__first[__i]));
     }
-    return std::make_pair(__result + __n, __init_.__value);
+    return std::make_pair(__result + __n, __init_.__value_);
 }
 
 // Inclusive scan for "+" and arithmetic types
@@ -602,11 +602,11 @@ __simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryO
     for (_Size __i = 0; __i < __n; ++__i)
     {
         _PSTL_PRAGMA_FORCEINLINE
-        __init_.__value = __binary_op(__init_.__value, __unary_op(__first[__i]));
+        __init_.__value_ = __binary_op(__init_.__value_, __unary_op(__first[__i]));
         _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(__init_)
-        __result[__i] = __init_.__value;
+        __result[__i] = __init_.__value_;
     }
-    return std::make_pair(__result + __n, __init_.__value);
+    return std::make_pair(__result + __n, __init_.__value_);
 }
 
 // [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 945daaa7ad9b1..777e2d03beefa 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -228,6 +228,7 @@ gentbl(
     td_file = "include/clang/Basic/riscv_vector.td",
     td_srcs = [
         "include/clang/Basic/riscv_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
     ],
 )
 
@@ -241,6 +242,7 @@ gentbl(
     td_file = "include/clang/Basic/riscv_vector.td",
     td_srcs = [
         "include/clang/Basic/riscv_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
     ],
 )
 
@@ -254,6 +256,49 @@ gentbl(
     td_file = "include/clang/Basic/riscv_vector.td",
     td_srcs = [
         "include/clang/Basic/riscv_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
+    ],
+)
+
+gentbl(
+    name = "basic_riscv_sifive_vector_builtins_gen",
+    tbl_outs = [(
+        "-gen-riscv-sifive-vector-builtins",
+        "include/clang/Basic/riscv_sifive_vector_builtins.inc",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/riscv_sifive_vector.td",
+    td_srcs = [
+        "include/clang/Basic/riscv_sifive_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
+    ],
+)
+
+gentbl(
+    name = "basic_riscv_sifive_vector_builtin_cg_gen",
+    tbl_outs = [(
+        "-gen-riscv-sifive-vector-builtin-codegen",
+        "include/clang/Basic/riscv_sifive_vector_builtin_cg.inc",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/riscv_sifive_vector.td",
+    td_srcs = [
+        "include/clang/Basic/riscv_sifive_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
+    ],
+)
+
+gentbl(
+    name = "basic_riscv_sifive_vector_builtin_sema_gen",
+    tbl_outs = [(
+        "-gen-riscv-sifive-vector-builtin-sema",
+        "include/clang/Basic/riscv_sifive_vector_builtin_sema.inc",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/riscv_sifive_vector.td",
+    td_srcs = [
+        "include/clang/Basic/riscv_sifive_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
     ],
 )
 
@@ -486,6 +531,7 @@ cc_library(
         ":basic_arm_sve_typeflags_gen",
         ":basic_attr_gen",
         ":basic_internal_headers",
+        ":basic_riscv_sifive_vector_builtins_gen",
         ":basic_riscv_vector_builtin_cg_gen",
         ":basic_riscv_vector_builtins_gen",
         ":config",
@@ -929,6 +975,7 @@ cc_library(
         ":basic_arm_cde_sema_gen",
         ":basic_arm_sve_builtins_gen",
         ":basic_arm_sve_sema_rangechecks_gen",
+        ":basic_riscv_sifive_vector_builtin_sema_gen",
         ":basic_riscv_vector_builtin_sema_gen",
         ":edit",
         ":lex",
@@ -1530,6 +1577,7 @@ gentbl(
     td_file = "include/clang/Basic/riscv_vector.td",
     td_srcs = [
         "include/clang/Basic/riscv_vector.td",
+        "include/clang/Basic/riscv_vector_common.td",
     ],
 )
 
@@ -1694,6 +1742,7 @@ cc_library(
         ":basic",
         ":basic_arm_cde_cg_gen",
         ":basic_arm_sve_builtin_cg_gen",
+        ":basic_riscv_sifive_vector_builtin_cg_gen",
         ":basic_riscv_vector_builtin_cg_gen",
         ":driver",
         ":frontend",
@@ -1707,6 +1756,7 @@ cc_library(
         "//llvm:BitWriter",
         "//llvm:BitstreamReader",
         "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Core",
         "//llvm:Coroutines",
         "//llvm:Coverage",
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 424986ae1e687..6e6953ef94fc4 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -20,6 +20,12 @@ package(
 
 licenses(["notice"])
 
+PRINTF_COPTS = [
+    "LIBC_COPT_PRINTF_USE_SYSTEM_FILE",
+    "LIBC_COPT_PRINTF_DISABLE_INDEX_MODE",
+    "LIBC_COPT_PRINTF_DISABLE_WRITE_INT",
+]
+
 # A flag to pick which `mpfr` to use for math tests.
 # Usage: `--@llvm-project//libc:mpfr=<disable|external|system>`.
 # Flag documentation: https://bazel.build/extending/config
@@ -153,10 +159,23 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_cpp_expected",
+    hdrs = ["src/__support/CPP/expected.h"],
+    deps = [
+        ":libc_root",
+    ],
+)
+
 libc_support_library(
     name = "__support_cpp_functional",
     hdrs = ["src/__support/CPP/functional.h"],
-    deps = [":libc_root"],
+    deps = [
+        "__support_cpp_type_traits",
+        "__support_cpp_utility",
+        "__support_macros_attributes",
+        ":libc_root",
+    ],
 )
 
 libc_support_library(
@@ -165,6 +184,16 @@ libc_support_library(
     deps = [":libc_root"],
 )
 
+libc_support_library(
+    name = "__support_cpp_new",
+    srcs = ["src/__support/CPP/new.cpp"],
+    hdrs = ["src/__support/CPP/new.h"],
+    deps = [
+        ":__support_common",
+        ":libc_root",
+    ],
+)
+
 libc_support_library(
     name = "__support_cpp_optional",
     hdrs = ["src/__support/CPP/optional.h"],
@@ -214,6 +243,7 @@ libc_support_library(
     hdrs = ["src/__support/CPP/utility.h"],
     deps = [
         ":__support_cpp_type_traits",
+        ":__support_macros_attributes",
         ":libc_root",
     ],
 )
@@ -238,6 +268,25 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_c_string",
+    hdrs = ["src/__support/c_string.h"],
+    deps = [
+        ":__support_cpp_string",
+        ":libc_root",
+    ],
+)
+
+libc_support_library(
+    name = "__support_error_or",
+    hdrs = ["src/__support/error_or.h"],
+    deps = [
+        ":__support_common",
+        ":__support_cpp_expected",
+        ":libc_root",
+    ],
+)
+
 libc_support_library(
     name = "__support_float_to_string",
     hdrs = [
@@ -392,6 +441,20 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_file_file",
+    srcs = ["src/__support/File/file.cpp"],
+    hdrs = ["src/__support/File/file.h"],
+    deps = [
+        ":__support_cpp_new",
+        ":__support_cpp_span",
+        ":__support_error_or",
+        ":__support_threads_mutex",
+        ":errno",
+        ":libc_root",
+    ],
+)
+
 libc_support_library(
     name = "__support_named_pair",
     hdrs = ["src/__support/named_pair.h"],
@@ -695,6 +758,23 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_threads_mutex",
+    hdrs = [
+        "src/__support/threads/mutex.h",
+        "src/__support/threads/mutex_common.h",
+    ],
+    textual_hdrs = [
+        "src/__support/threads/linux/mutex.h",
+        "src/__support/threads/linux/futex_word.h",
+    ],
+    deps = [
+        ":__support_cpp_atomic",
+        ":__support_osutil_syscall",
+        ":libc_root",
+    ],
+)
+
 ############################### errno targets ################################
 
 libc_function(
@@ -703,6 +783,7 @@ libc_function(
     hdrs = ["src/errno/libc_errno.h"],
     deps = [
         ":__support_common",
+        ":__support_macros_properties_architectures",
     ],
 )
 
@@ -2073,6 +2154,7 @@ libc_function(
 libc_support_library(
     name = "printf_core_structs",
     hdrs = ["src/stdio/printf_core/core_structs.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_cpp_string_view",
         ":__support_fputil_fp_bits",
@@ -2083,6 +2165,7 @@ libc_support_library(
 libc_support_library(
     name = "printf_config",
     hdrs = ["src/stdio/printf_core/printf_config.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":libc_root",
     ],
@@ -2092,6 +2175,7 @@ libc_support_library(
     name = "printf_parser",
     srcs = ["src/stdio/printf_core/parser.cpp"],
     hdrs = ["src/stdio/printf_core/parser.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_arg_list",
         ":__support_common",
@@ -2113,7 +2197,7 @@ libc_support_library(
     name = "printf_mock_parser",
     srcs = ["src/stdio/printf_core/parser.cpp"],
     hdrs = ["src/stdio/printf_core/parser.h"],
-    copts = ["-DLIBC_COPT_MOCK_ARG_LIST"],
+    defines = PRINTF_COPTS + ["LIBC_COPT_MOCK_ARG_LIST"],
     deps = [
         ":__support_arg_list",
         ":__support_common",
@@ -2134,6 +2218,7 @@ libc_support_library(
     name = "printf_string_writer",
     srcs = ["src/stdio/printf_core/string_writer.cpp"],
     hdrs = ["src/stdio/printf_core/string_writer.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_cpp_string_view",
         ":libc_root",
@@ -2142,10 +2227,23 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "printf_file_writer",
+    hdrs = ["src/stdio/printf_core/file_writer.h"],
+    defines = PRINTF_COPTS,
+    deps = [
+        ":__support_cpp_string_view",
+        ":__support_file_file",
+        ":libc_root",
+        ":printf_core_structs",
+    ],
+)
+
 libc_support_library(
     name = "printf_writer",
     srcs = ["src/stdio/printf_core/writer.cpp"],
     hdrs = ["src/stdio/printf_core/writer.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_cpp_string_view",
         ":libc_root",
@@ -2168,6 +2266,7 @@ libc_support_library(
         "src/stdio/printf_core/string_converter.h",
         "src/stdio/printf_core/write_int_converter.h",
     ],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_common",
         ":__support_cpp_limits",
@@ -2190,6 +2289,7 @@ libc_support_library(
     name = "printf_main",
     srcs = ["src/stdio/printf_core/printf_main.cpp"],
     hdrs = ["src/stdio/printf_core/printf_main.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_arg_list",
         ":libc_root",
@@ -2204,6 +2304,7 @@ libc_function(
     name = "sprintf",
     srcs = ["src/stdio/sprintf.cpp"],
     hdrs = ["src/stdio/sprintf.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_arg_list",
         ":errno",
@@ -2217,6 +2318,7 @@ libc_function(
     name = "snprintf",
     srcs = ["src/stdio/snprintf.cpp"],
     hdrs = ["src/stdio/snprintf.h"],
+    defines = PRINTF_COPTS,
     deps = [
         ":__support_arg_list",
         ":errno",
@@ -2225,3 +2327,41 @@ libc_function(
         ":printf_writer",
     ],
 )
+
+libc_support_library(
+    name = "vfprintf_internal",
+    hdrs = ["src/stdio/printf_core/vfprintf_internal.h"],
+    defines = PRINTF_COPTS,
+    deps = [
+        ":__support_arg_list",
+        ":__support_file_file",
+        ":__support_macros_attributes",
+        ":printf_file_writer",
+        ":printf_main",
+        ":printf_writer",
+    ],
+)
+
+libc_function(
+    name = "printf",
+    srcs = ["src/stdio/printf.cpp"],
+    hdrs = ["src/stdio/printf.h"],
+    defines = PRINTF_COPTS,
+    deps = [
+        ":__support_arg_list",
+        ":errno",
+        ":vfprintf_internal",
+    ],
+)
+
+libc_function(
+    name = "fprintf",
+    srcs = ["src/stdio/fprintf.cpp"],
+    hdrs = ["src/stdio/fprintf.h"],
+    defines = PRINTF_COPTS,
+    deps = [
+        ":__support_arg_list",
+        ":errno",
+        ":vfprintf_internal",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index b4506c33603f1..2007d1219b7c1 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -23,7 +23,7 @@ def _libc_library(name, copts = None, **kwargs):
     # We want all libc sources to be compiled with "hidden" visibility.
     # The public symbols will be given "default" visibility explicitly.
     # See src/__support/common.h for more information.
-    copts.append("-fvisibility=hidden")
+    copts = copts + ["-fvisibility=hidden"]
     native.cc_library(
         name = name,
         copts = copts,
@@ -65,11 +65,12 @@ def libc_function(
       **kwargs: Other attributes relevant for a cc_library. For example, deps.
     """
     deps = deps or []
-    deps.append(LIBC_ROOT_TARGET)
+
+    # We use the explicit equals pattern here because append and += mutate the
+    # original list, where this creates a new list and stores it in deps.
+    deps = deps + [LIBC_ROOT_TARGET]
     copts = copts or []
-    copts.append("-O3")
-    copts.append("-fno-builtin")
-    copts.append("-fno-lax-vector-conversions")
+    copts = copts + ["-O3", "-fno-builtin", "-fno-lax-vector-conversions"]
 
     # We compile the code twice, the first target is suffixed with ".__internal__" and contains the
     # C++ functions in the "__llvm_libc" namespace. This allows us to test the function in the
@@ -87,9 +88,9 @@ def libc_function(
 
     func_attrs = ["__attribute__((visibility(\"default\")))"]
     if weak:
-        func_attrs.append("__attribute__((weak))")
+        func_attrs = func_attrs + ["__attribute__((weak))"]
     local_defines = local_defines or ["LIBC_COPT_PUBLIC_PACKAGING"]
-    local_defines.append("LLVM_LIBC_FUNCTION_ATTR='%s'" % " ".join(func_attrs))
+    local_defines = local_defines + ["LLVM_LIBC_FUNCTION_ATTR='%s'" % " ".join(func_attrs)]
     _libc_library(
         name = name,
         srcs = srcs,
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 319f60b34f622..7ab3fe1b6b548 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -23,6 +23,7 @@ cc_library(
 cc_library(
     name = "LibcUnitTest",
     srcs = [
+        "BazelFilePath.cpp",
         "ExecuteFunctionUnix.cpp",
         "LibcTest.cpp",
         "LibcTestMain.cpp",
@@ -35,6 +36,7 @@ cc_library(
     ],
     deps = [
         ":test_logger",
+        "//libc:__support_c_string",
         "//libc:__support_cpp_bit",
         "//libc:__support_cpp_bitset",
         "//libc:__support_cpp_span",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index b4e473d932fdd..693545a8109ff 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -33,5 +33,6 @@ def libc_test(name, srcs, libc_function_deps, deps = [], **kwargs):
             "//libc/test/UnitTest:LibcUnitTest",
         ] + deps,
         features = ["-link_llvmlibc"],  # Do not link libllvmlibc.a
+        linkstatic = 1,
         **kwargs
     )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
index f646079429735..07ffd21dc6737 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/stdio/BUILD.bazel
@@ -31,11 +31,11 @@ libc_test(
     libc_function_deps = [
     ],
     deps = [
-      "//libc:printf_string_writer",
-      "//libc:printf_writer",
-      "//libc:printf_core_structs",
-      "//libc:__support_cpp_string_view",
-      "//libc:__support_arg_list",
+        "//libc:__support_arg_list",
+        "//libc:__support_cpp_string_view",
+        "//libc:printf_core_structs",
+        "//libc:printf_string_writer",
+        "//libc:printf_writer",
     ],
 )
 
@@ -45,12 +45,12 @@ libc_test(
     libc_function_deps = [
     ],
     deps = [
-      "//libc:printf_converter",
-      "//libc:printf_string_writer",
-      "//libc:printf_writer",
-      "//libc:printf_core_structs",
-      "//libc:__support_cpp_string_view",
-      "//libc:__support_arg_list",
+        "//libc:__support_arg_list",
+        "//libc:__support_cpp_string_view",
+        "//libc:printf_converter",
+        "//libc:printf_core_structs",
+        "//libc:printf_string_writer",
+        "//libc:printf_writer",
     ],
 )
 
@@ -74,3 +74,19 @@ libc_test(
         "//libc:snprintf",
     ],
 )
+
+libc_test(
+    name = "printf_test",
+    srcs = ["printf_test.cpp"],
+    libc_function_deps = [
+        "//libc:printf",
+    ],
+)
+
+libc_test(
+    name = "fprintf_test",
+    srcs = ["fprintf_test.cpp"],
+    libc_function_deps = [
+        "//libc:fprintf",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index be421ca0c09dc..2122336fdf27b 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -572,6 +572,31 @@ cc_library(
     ],
 )
 
+cc_binary(
+    name = "llvm-min-tblgen",
+    srcs = [
+        "utils/TableGen/Attributes.cpp",
+        "utils/TableGen/CodeGenIntrinsics.cpp",
+        "utils/TableGen/DirectiveEmitter.cpp",
+        "utils/TableGen/IntrinsicEmitter.cpp",
+        "utils/TableGen/RISCVTargetDefEmitter.cpp",
+        "utils/TableGen/SDNodeProperties.cpp",
+        "utils/TableGen/TableGen.cpp",
+        "utils/TableGen/VTEmitter.cpp",
+
+        "utils/TableGen/CodeGenIntrinsics.h",
+        "utils/TableGen/SDNodeProperties.h",
+        "utils/TableGen/SequenceToOffsetTable.h",
+    ],
+    copts = llvm_copts,
+    stamp = 0,
+    deps = [
+        ":Support",
+        ":TableGen",
+        ":config",
+    ],
+)
+
 cc_library(
     name = "TableGenGlobalISel",
     srcs = glob([
@@ -591,6 +616,7 @@ cc_library(
     features = ["-header_modules"],
     strip_include_prefix = "utils/TableGen",
     deps = [
+        ":CodeGenTypes",
         ":Support",
         ":TableGen",
         ":config",
@@ -617,6 +643,7 @@ cc_binary(
     copts = llvm_copts,
     stamp = 0,
     deps = [
+        ":CodeGenTypes",
         ":Support",
         ":TableGen",
         ":TableGenGlobalISel",
@@ -628,7 +655,7 @@ cc_binary(
 gentbl(
     name = "intrinsic_enums_gen",
     tbl_outs = [("-gen-intrinsic-enums", "include/llvm/IR/IntrinsicEnums.inc")],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/Intrinsics.td",
     td_srcs = glob([
         "include/llvm/CodeGen/*.td",
@@ -639,7 +666,7 @@ gentbl(
 gentbl(
     name = "intrinsics_impl_gen",
     tbl_outs = [("-gen-intrinsic-impl", "include/llvm/IR/IntrinsicImpl.inc")],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/Intrinsics.td",
     td_srcs = glob([
         "include/llvm/CodeGen/*.td",
@@ -647,6 +674,16 @@ gentbl(
     ]),
 )
 
+gentbl(
+    name = "vt_gen",
+    tbl_outs = [("-gen-vt", "include/llvm/CodeGen/GenVT.inc")],
+    tblgen = ":llvm-min-tblgen",
+    td_file = "include/llvm/CodeGen/ValueTypes.td",
+    td_srcs = [
+        "include/llvm/CodeGen/ValueTypes.td",
+    ],
+)
+
 # Note that the intrinsics are not currently set up so they can be pruned for
 # disabled targets.
 llvm_target_intrinsics_list = [
@@ -727,7 +764,7 @@ llvm_target_intrinsics_list = [
             "-gen-intrinsic-enums -intrinsic-prefix=" + target["intrinsic_prefix"],
             "include/llvm/IR/Intrinsics" + target["name"] + ".h",
         )],
-        tblgen = ":llvm-tblgen",
+        tblgen = ":llvm-min-tblgen",
         td_file = "include/llvm/IR/Intrinsics.td",
         td_srcs = glob([
             "include/llvm/CodeGen/*.td",
@@ -739,7 +776,7 @@ llvm_target_intrinsics_list = [
 gentbl(
     name = "attributes_gen",
     tbl_outs = [("-gen-attrs", "include/llvm/IR/Attributes.inc")],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/IR/Attributes.td",
     td_srcs = ["include/llvm/IR/Attributes.td"],
 )
@@ -1108,7 +1145,7 @@ cc_library(
 gentbl(
     name = "RISCVTargetParserDefGen",
     tbl_outs = [("-gen-riscv-target-def", "include/llvm/TargetParser/RISCVTargetParserDef.inc")],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "lib/Target/RISCV/RISCV.td",
     td_srcs = [
         ":common_target_td_sources",
@@ -1375,7 +1412,7 @@ gentbl(
     tbl_outs = [
         ("--gen-directive-decl", "include/llvm/Frontend/OpenMP/OMP.h.inc"),
     ],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenMP/OMP.td",
     td_srcs = [":omp_td_files"],
 )
@@ -1386,7 +1423,7 @@ gentbl(
     tbl_outs = [
         ("--gen-directive-impl", "include/llvm/Frontend/OpenMP/OMP.inc"),
     ],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenMP/OMP.td",
     td_srcs = [":omp_td_files"],
 )
@@ -1434,7 +1471,7 @@ gentbl(
     tbl_outs = [
         ("--gen-directive-decl", "include/llvm/Frontend/OpenACC/ACC.h.inc"),
     ],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenACC/ACC.td",
     td_srcs = [":acc_td_files"],
 )
@@ -1445,7 +1482,7 @@ gentbl(
     tbl_outs = [
         ("--gen-directive-impl", "include/llvm/Frontend/OpenACC/ACC.inc"),
     ],
-    tblgen = ":llvm-tblgen",
+    tblgen = ":llvm-min-tblgen",
     td_file = "include/llvm/Frontend/OpenACC/ACC.td",
     td_srcs = [":acc_td_files"],
 )
@@ -1640,6 +1677,22 @@ cc_library(
     textual_hdrs = glob(["lib/CodeGen/AsmPrinter/*.def"]),
 )
 
+cc_library(
+    name = "CodeGenTypes",
+    srcs = [
+        "lib/CodeGen/LowLevelType.cpp",
+    ],
+    hdrs = [
+        "include/llvm/CodeGen/LowLevelType.h",
+        "include/llvm/CodeGen/MachineValueType.h",
+    ],
+    copts = llvm_copts,
+    deps = [
+        ":Support",
+        ":vt_gen",
+    ],
+)
+
 cc_library(
     name = "CodeGen",
     srcs = glob(
@@ -1649,6 +1702,9 @@ cc_library(
             "lib/CodeGen/SelectionDAG/*.cpp",
             "lib/CodeGen/SelectionDAG/*.h",
         ],
+        exclude = [
+            "lib/CodeGen/LowLevelType.cpp",
+        ],
     ),
     hdrs = [
         "include/llvm/LinkAllPasses.h",
@@ -1669,6 +1725,7 @@ cc_library(
         ":BinaryFormat",
         ":BitReader",
         ":BitWriter",
+        ":CodeGenTypes",
         ":Core",
         ":DebugInfoCodeView",
         ":DebugInfoDWARF",
@@ -2186,6 +2243,7 @@ gentbl(
             # Depending on `:CodeGen` headers in this library is almost
             # certainly a layering problem in numerous targets.
             ":CodeGen",
+            ":CodeGenTypes",
             ":DebugInfoCodeView",
             ":MC",
             ":MCDisassembler",
@@ -2218,6 +2276,7 @@ gentbl(
             ":BinaryFormat",
             ":CFGuard",
             ":CodeGen",
+            ":CodeGenTypes",
             ":Core",
             ":IPO",
             ":MC",
@@ -2243,6 +2302,7 @@ gentbl(
         copts = llvm_copts,
         deps = [
             ":BinaryFormat",
+            ":CodeGenTypes",
             ":MC",
             ":MCParser",
             ":Support",
@@ -2288,7 +2348,7 @@ gentbl(
         copts = llvm_copts,
         features = ["-layering_check"],
         deps = [
-            ":CodeGen",
+            ":CodeGenTypes",
             ":Core",
             ":MC",
             ":MCDisassembler",
@@ -2310,6 +2370,7 @@ gentbl(
         copts = llvm_copts,
         features = ["-layering_check"],
         deps = [
+            ":CodeGenTypes",
             ":MC",
             ":MCA",
             ":MCParser",
@@ -2429,6 +2490,7 @@ cc_library(
         ":BitReader",
         ":BitWriter",
         ":CodeGen",
+        ":CodeGenTypes",
         ":Core",
         ":IPO",
         ":IRPrinter",
@@ -2681,6 +2743,7 @@ cc_library(
     deps = [
         ":BinaryFormat",
         ":CodeGen",
+        ":CodeGenTypes",
         ":DebugInfoDWARF",
         ":MC",
         ":Support",
@@ -2907,6 +2970,7 @@ cc_library(
         ":AllTargetsAsmParsers",
         ":AllTargetsCodeGens",
         ":CodeGen",
+        ":CodeGenTypes",
         ":Core",
         ":ExecutionEngine",
         ":MC",
@@ -2958,6 +3022,7 @@ cc_binary(
         ":AllTargetsCodeGens",
         ":BinaryFormat",
         ":CodeGen",
+        ":CodeGenTypes",
         ":DWARFLinker",
         ":DebugInfo",
         ":DebugInfoDWARF",
@@ -2989,6 +3054,7 @@ cc_binary(
         ":AsmParser",
         ":BitReader",
         ":CodeGen",
+        ":CodeGenTypes",
         ":Core",
         ":IRPrinter",
         ":IRReader",
@@ -3378,6 +3444,7 @@ cc_binary(
     deps = [
         ":AllTargetsAsmParsers",
         ":AllTargetsCodeGens",
+        ":CodeGenTypes",
         ":DWARFLinker",
         ":DebugInfoDWARF",
         ":DwarfutilOptionsTableGen",
@@ -3418,6 +3485,7 @@ cc_binary(
         ":AllTargetsAsmParsers",
         ":AllTargetsCodeGens",
         ":AllTargetsDisassemblers",
+        ":CodeGenTypes",
         ":Exegesis",
         ":MC",
         ":MCParser",
@@ -3449,12 +3517,33 @@ cc_binary(
     ],
 )
 
+gentbl(
+    name = "GSYMUtilOptionsTableGen",
+    strip_include_prefix = "tools/llvm-gsymutil",
+    tbl_outs = [(
+        "-gen-opt-parser-defs",
+        "tools/llvm-gsymutil/Opts.inc",
+    )],
+    tblgen = ":llvm-tblgen",
+    td_file = "tools/llvm-gsymutil/Opts.td",
+    td_srcs = ["include/llvm/Option/OptParser.td"],
+)
+
+expand_template(
+    name = "gsymutil_main",
+    out = "llvm-gsymutil-driver.cpp",
+    substitutions = {
+        "@TOOL_NAME@": "llvm_gsymutil",
+    },
+    template = "cmake/modules/llvm-driver-template.cpp.in",
+)
+
 cc_binary(
     name = "llvm-gsymutil",
     srcs = glob([
         "tools/llvm-gsymutil/*.cpp",
         "tools/llvm-gsymutil/*.h",
-    ]),
+    ]) + ["llvm-gsymutil-driver.cpp"],
     copts = llvm_copts,
     stamp = 0,
     deps = [
@@ -3462,8 +3551,10 @@ cc_binary(
         ":DebugInfo",
         ":DebugInfoDWARF",
         ":DebugInfoGSYM",
+        ":GSYMUtilOptionsTableGen",
         ":MC",
         ":Object",
+        ":Option",
         ":Support",
         ":Target",
         ":TargetParser",
@@ -4258,6 +4349,7 @@ cc_binary(
         ":BitReader",
         ":BitWriter",
         ":CodeGen",
+        ":CodeGenTypes",
         ":Core",
         ":IPO",
         ":IRReader",
diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index cfe9a18f4a03c..e5d9736f8a22b 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -141,6 +141,7 @@ cc_test(
         "//llvm:AsmParser",
         "//llvm:BinaryFormat",
         "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Core",
         "//llvm:MC",
         "//llvm:Passes",
@@ -169,6 +170,7 @@ cc_test(
         "//llvm:AllTargetsAsmParsers",
         "//llvm:AllTargetsCodeGens",
         "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Core",
         "//llvm:FileCheckLib",
         "//llvm:MC",
@@ -202,7 +204,7 @@ cc_test(
         "//llvm:AllTargetsAsmParsers",
         "//llvm:AllTargetsCodeGens",
         "//llvm:BinaryFormat",
-        "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Core",
         "//llvm:DebugInfo",
         "//llvm:DebugInfoDWARF",
@@ -448,6 +450,7 @@ cc_test(
         "//llvm:AllTargetsAsmParsers",
         "//llvm:AllTargetsCodeGens",
         "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Core",
         "//llvm:MC",
         "//llvm:Support",
@@ -554,17 +557,6 @@ gentbl(
     ],
 )
 
-gentbl(
-    name = "vt_gen",
-    strip_include_prefix = "Support",
-    tbl_outs = [("-gen-vt", "Support/GenVT.inc")],
-    tblgen = "//llvm:llvm-tblgen",
-    td_file = "//llvm:include/llvm/CodeGen/ValueTypes.td",
-    td_srcs = [
-        "//llvm:include/llvm/CodeGen/ValueTypes.td",
-    ],
-)
-
 cc_test(
     name = "option_tests",
     size = "small",
@@ -650,7 +642,6 @@ cc_test(
         "//llvm:TargetParser",
         "//llvm:TestingSupport",
         "//llvm:config",
-        ":vt_gen",
         "//third-party/unittest:gmock",
         "//third-party/unittest:gtest",
         "//third-party/unittest:gtest_main",
@@ -710,6 +701,7 @@ cc_test(
         "//llvm:AArch64UtilsAndDesc",
         "//llvm:AsmParser",
         "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Core",
         "//llvm:MC",
         "//llvm:Support",
@@ -779,6 +771,7 @@ cc_test(
         "//llvm:AllTargetsCodeGens",
         "//llvm:AllTargetsDisassemblers",
         "//llvm:CodeGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Exegesis",
         "//llvm:MC",
         "//llvm:MCDisassembler",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 881a12426baf5..daa5b894d7b98 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -91,6 +91,7 @@ td_library(
     ],
     includes = ["include"],
     deps = [
+        ":BytecodeTdFiles",
         ":CallInterfacesTdFiles",
         ":CastInterfacesTdFiles",
         ":DataLayoutInterfacesTdFiles",
@@ -117,6 +118,23 @@ gentbl_cc_library(
     deps = [":BuiltinDialectTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "BuiltinDialectBytecodeGen",
+    strip_include_prefix = "include",
+    tbl_outs = [
+        (
+            [
+                "-gen-bytecode",
+                "-bytecode-dialect=Builtin",
+            ],
+            "include/mlir/IR/BuiltinDialectBytecode.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/IR/BuiltinDialectBytecode.td",
+    deps = [":BuiltinDialectTdFiles"],
+)
+
 gentbl_cc_library(
     name = "BuiltinAttributesIncGen",
     strip_include_prefix = "include",
@@ -277,6 +295,7 @@ cc_library(
     deps = [
         ":BuiltinAttributeInterfacesIncGen",
         ":BuiltinAttributesIncGen",
+        ":BuiltinDialectBytecodeGen",
         ":BuiltinDialectIncGen",
         ":BuiltinLocationAttributesIncGen",
         ":BuiltinOpsIncGen",
@@ -356,6 +375,7 @@ mlir_c_api_cc_library(
         "include/mlir/CAPI/Wrap.h",
     ],
     header_deps = [
+        ":BytecodeWriter",
         ":IR",
         ":Pass",
         ":Support",
@@ -364,7 +384,6 @@ mlir_c_api_cc_library(
     includes = ["include"],
     deps = [
         ":AsmParser",
-        ":BytecodeWriter",
         ":ConversionPassIncGen",
         ":FuncDialect",
         ":InferTypeOpInterface",
@@ -929,6 +948,12 @@ td_library(
     ],
 )
 
+td_library(
+    name = "BytecodeTdFiles",
+    srcs = ["include/mlir/IR/BytecodeBase.td"],
+    includes = ["include"],
+)
+
 td_library(
     name = "CallInterfacesTdFiles",
     srcs = ["include/mlir/Interfaces/CallInterfaces.td"],
@@ -992,6 +1017,13 @@ td_library(
     deps = [":OpBaseTdFiles"],
 )
 
+td_library(
+    name = "Mem2RegInterfacesTdFiles",
+    srcs = ["include/mlir/Interfaces/Mem2RegInterfaces.td"],
+    includes = ["include"],
+    deps = [":OpBaseTdFiles"],
+)
+
 td_library(
     name = "ShapedOpInterfacesTdFiles",
     srcs = ["include/mlir/Interfaces/ShapedOpInterfaces.td"],
@@ -1190,7 +1222,7 @@ cc_library(
 
 td_library(
     name = "AMDGPUTdFiles",
-    srcs = ["include/mlir/Dialect/AMDGPU/AMDGPU.td"],
+    srcs = ["include/mlir/Dialect/AMDGPU/IR/AMDGPU.td"],
     includes = ["include"],
     deps = [
         ":SideEffectInterfacesTdFiles",
@@ -1206,59 +1238,59 @@ gentbl_cc_library(
                 "-gen-attrdef-decls",
                 "-dialect=amdgpu",
             ],
-            "include/mlir/Dialect/AMDGPU/AMDGPUAttributes.h.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPUAttributes.h.inc",
         ),
         (
             [
                 "-gen-attrdef-defs",
                 "-dialect=amdgpu",
             ],
-            "include/mlir/Dialect/AMDGPU/AMDGPUAttributes.cpp.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPUAttributes.cpp.inc",
         ),
         (
             [
                 "-gen-dialect-decls",
                 "-dialect=amdgpu",
             ],
-            "include/mlir/Dialect/AMDGPU/AMDGPUDialect.h.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h.inc",
         ),
         (
             [
                 "-gen-dialect-defs",
                 "-dialect=amdgpu",
             ],
-            "include/mlir/Dialect/AMDGPU/AMDGPUDialect.cpp.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.cpp.inc",
         ),
         (
             ["-gen-enum-decls"],
-            "include/mlir/Dialect/AMDGPU/AMDGPUEnums.h.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.h.inc",
         ),
         (
             ["-gen-enum-defs"],
-            "include/mlir/Dialect/AMDGPU/AMDGPUEnums.cpp.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPUEnums.cpp.inc",
         ),
         (
             ["-gen-op-decls"],
-            "include/mlir/Dialect/AMDGPU/AMDGPU.h.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPU.h.inc",
         ),
         (
             ["-gen-op-defs"],
-            "include/mlir/Dialect/AMDGPU/AMDGPU.cpp.inc",
+            "include/mlir/Dialect/AMDGPU/IR/AMDGPU.cpp.inc",
         ),
         (
             ["-gen-op-doc"],
-            "g3doc/Dialects/AMDGPU/AMDGPU.md",
+            "g3doc/Dialects/AMDGPU/IR/AMDGPU.md",
         ),
     ],
     tblgen = ":mlir-tblgen",
-    td_file = "include/mlir/Dialect/AMDGPU/AMDGPU.td",
+    td_file = "include/mlir/Dialect/AMDGPU/IR/AMDGPU.td",
     deps = [":AMDGPUTdFiles"],
 )
 
 cc_library(
     name = "AMDGPUDialect",
     srcs = ["lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp"],
-    hdrs = ["include/mlir/Dialect/AMDGPU/AMDGPUDialect.h"],
+    hdrs = ["include/mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"],
     includes = ["include"],
     deps = [
         ":AMDGPUIncGen",
@@ -1271,6 +1303,58 @@ cc_library(
     ],
 )
 
+gentbl_cc_library(
+    name = "AMDGPUPassIncGen",
+    strip_include_prefix = "include",
+    tbl_outs = [
+        (
+            [
+                "-gen-pass-decls",
+                "-name=AMDGPU",
+            ],
+            "include/mlir/Dialect/AMDGPU/Transforms/Passes.h.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/AMDGPU/Transforms/Passes.td",
+    deps = [":PassBaseTdFiles"],
+)
+
+cc_library(
+    name = "AMDGPUTransforms",
+    srcs = glob(
+        [
+            "lib/Dialect/AMDGPU/Transforms/*.cpp",
+            "lib/Dialect/AMDGPU/Transforms/*.h",
+        ],
+    ),
+    hdrs = glob(["include/mlir/Dialect/AMDGPU/Transforms/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":AMDGPUDialect",
+        ":AMDGPUPassIncGen",
+        ":AMDGPUUtils",
+        ":ArithDialect",
+        ":ControlFlowDialect",
+        ":IR",
+        ":Pass",
+        ":TransformUtils",
+        ":Transforms",
+    ],
+)
+
+cc_library(
+    name = "AMDGPUUtils",
+    srcs = glob(["lib/Dialect/AMDGPU/Utils/*.cpp"]),
+    hdrs = glob(["include/mlir/Dialect/AMDGPU/Utils/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":AMDGPUDialect",
+        ":Support",
+        "//llvm:Support",
+    ],
+)
+
 ##---------------------------------------------------------------------------##
 # EmitC dialect.
 ##---------------------------------------------------------------------------##
@@ -1872,7 +1956,7 @@ td_library(
         ":InferTypeOpInterfaceTdFiles",
         ":OpBaseTdFiles",
         ":SideEffectInterfacesTdFiles",
-    ]
+    ],
 )
 
 gentbl_cc_library(
@@ -1959,11 +2043,11 @@ cc_library(
     includes = ["include"],
     deps = [
         ":Dialect",
-        ":InferTypeOpInterface",
         ":IR",
         ":IRDLIncGen",
         ":IRDLOpsIncGen",
         ":IRDLTypesIncGen",
+        ":InferTypeOpInterface",
         ":Support",
         "//llvm:Core",
         "//llvm:Support",
@@ -3117,6 +3201,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "Mem2RegInterfaces",
+    srcs = ["lib/Interfaces/Mem2RegInterfaces.cpp"],
+    hdrs = ["include/mlir/Interfaces/Mem2RegInterfaces.h"],
+    includes = ["include"],
+    deps = [
+        ":IR",
+        ":Mem2RegInterfacesIncGen",
+        "//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "ShapedOpInterfaces",
     srcs = ["lib/Interfaces/ShapedOpInterfaces.cpp"],
@@ -3696,44 +3792,20 @@ cc_library(
         "lib/Debug/*.cpp",
         "lib/Debug/*.h",
         "lib/Debug/BreakpointManagers/*.cpp",
+        "lib/Debug/BreakpointManagers/*.h",
+        "lib/Debug/Observers/*.cpp",
+        "lib/Debug/Observers/*.h",
     ]),
     hdrs = glob([
         "include/mlir/Debug/*.h",
-        "include/mlir/Debug/BreakpointManagers/*.h"
-]),
-    includes = ["include"],
-    deps = [
-        ":IR",
-        ":Support",
-        "//llvm:Support",
-    ],
-)
-
-cc_library(
-    name = "Observers",
-    srcs = glob([
-        "lib/Debug/Observers/*.cpp",
-        "lib/Debug/Observers/*.h",
+        "include/mlir/Debug/BreakpointManagers/*.h",
+        "include/mlir/Debug/Observers/*.h",
     ]),
-    hdrs = glob(["include/mlir/Debug/Observers/*.h"]),
     includes = ["include"],
     deps = [
-        ":Debug",
-        ":IR",
-        "//llvm:Support",
-    ],
-)
-
-cc_library(
-    name = "BreakpointManagers",
-    srcs = glob([
-        "lib/Debug/BreakpointManagers/*.cpp",
-        "lib/Debug/BreakpointManagers/*.h",
-    ]),
-    hdrs = glob(["include/mlir/Debug/BreakpointManagers/*.h"]),
-    deps = [
-        ":Debug",
+        ":CAPIIR",
         ":IR",
+        ":Support",
         "//llvm:Support",
     ],
 )
@@ -3944,7 +4016,10 @@ cc_library(
             "include/mlir/Dialect/LLVMIR/ROCDL*.h",
             "include/mlir/Dialect/LLVMIR/*X86Vector*.h",
         ],
-    ) + ["include/mlir/Transforms/InliningUtils.h"],
+    ) + [
+        "include/mlir/Transforms/InliningUtils.h",
+        "include/mlir/Transforms/Mem2Reg.h",
+    ],
     includes = ["include"],
     deps = [
         ":CallOpInterfaces",
@@ -3956,6 +4031,8 @@ cc_library(
         ":LLVMIntrinsicOpsIncGen",
         ":LLVMOpsIncGen",
         ":LLVMTypesIncGen",
+        ":Mem2RegInterfaces",
+        ":Mem2RegInterfacesIncGen",
         ":SideEffectInterfaces",
         ":Support",
         "//llvm:AsmParser",
@@ -4357,6 +4434,7 @@ td_library(
         ":DataLayoutInterfacesTdFiles",
         ":FunctionInterfacesTdFiles",
         ":InferTypeOpInterfaceTdFiles",
+        ":Mem2RegInterfacesTdFiles",
         ":OpBaseTdFiles",
         ":SideEffectInterfacesTdFiles",
     ],
@@ -4446,6 +4524,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":AMDGPUDialect",
+        ":AMDGPUUtils",
         ":ConversionPassIncGen",
         ":IR",
         ":LLVMCommonConversion",
@@ -5785,6 +5864,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":AffineDialect",
+        ":AffineTransforms",
         ":AffineUtils",
         ":ArithDialect",
         ":ArithUtils",
@@ -5801,20 +5881,57 @@ cc_library(
         ":TensorPassIncGen",
         ":TilingInterface",
         ":Transforms",
+        ":ValueBoundsOpInterface",
         ":VectorDialect",
         "//llvm:Support",
     ],
 )
 
+td_library(
+    name = "TensorTransformOpsTdFiles",
+    srcs = [
+        "include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td",
+    ],
+    includes = ["include"],
+    deps = [
+        ":PDLDialect",
+        ":TransformDialectTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "TensorTransformOpsIncGen",
+    strip_include_prefix = "include",
+    tbl_outs = [
+        (
+            ["-gen-op-decls"],
+            "include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h.inc",
+        ),
+        (
+            ["-gen-op-defs"],
+            "include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td",
+    deps = [
+        ":TensorTransformOpsTdFiles",
+    ],
+)
+
 cc_library(
     name = "TensorTransformOps",
     srcs = glob(["lib/Dialect/Tensor/TransformOps/*.cpp"]),
     hdrs = glob(["include/mlir/Dialect/Tensor/TransformOps/*.h"]),
     includes = ["include"],
     deps = [
+        ":AffineDialect",
         ":IR",
         ":PDLDialect",
+        ":SCFDialect",
         ":TensorDialect",
+        ":TensorTransformOpsIncGen",
+        ":TensorTransforms",
         ":TransformDialect",
         "//llvm:Support",
     ],
@@ -5857,6 +5974,7 @@ cc_library(
         ":ControlFlowInterfaces",
         ":IR",
         ":LoopLikeInterface",
+        ":Mem2RegInterfaces",
         ":Rewrite",
         ":SideEffectInterfaces",
         ":Support",
@@ -6012,6 +6130,24 @@ gentbl_cc_library(
     deps = [":LoopLikeInterfaceTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "Mem2RegInterfacesIncGen",
+    strip_include_prefix = "include",
+    tbl_outs = [
+        (
+            ["-gen-op-interface-decls"],
+            "include/mlir/Interfaces/Mem2RegInterfaces.h.inc",
+        ),
+        (
+            ["-gen-op-interface-defs"],
+            "include/mlir/Interfaces/Mem2RegInterfaces.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Interfaces/Mem2RegInterfaces.td",
+    deps = [":Mem2RegInterfacesTdFiles"],
+)
+
 gentbl_cc_library(
     name = "ShapedOpInterfacesIncGen",
     strip_include_prefix = "include",
@@ -6167,6 +6303,7 @@ cc_library(
         ":ControlFlowInterfaces",
         ":IR",
         ":LoopLikeInterface",
+        ":Mem2RegInterfaces",
         ":Pass",
         ":Rewrite",
         ":RuntimeVerifiableOpInterface",
@@ -6997,6 +7134,7 @@ cc_library(
         ":OpenMPDialect",
         ":Support",
         ":ToLLVMIRTranslation",
+        ":Transforms",
         "//llvm:Core",
         "//llvm:FrontendOpenMP",
         "//llvm:Support",
@@ -7173,7 +7311,6 @@ cc_library(
         ":Debug",
         ":IR",
         ":IRDLDialect",
-        ":Observers",
         ":Parser",
         ":Pass",
         ":PluginsLib",
@@ -7219,6 +7356,7 @@ cc_library(
     deps = [
         ":AMDGPUDialect",
         ":AMDGPUToROCDL",
+        ":AMDGPUTransforms",
         ":AMXDialect",
         ":AMXTransforms",
         ":AffineDialect",
@@ -7774,6 +7912,7 @@ cc_binary(
         ":MlirTableGenMain",
         ":Support",
         ":TableGen",
+        "//llvm:CodeGenTypes",
         "//llvm:Support",
         "//llvm:TableGen",
         "//llvm:TargetParser",
@@ -8790,9 +8929,9 @@ cc_library(
     srcs = glob([
         "lib/Dialect/Linalg/TransformOps/*.cpp",
     ]),
-    hdrs = [
-        "include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h",
-    ],
+    hdrs = glob([
+        "include/mlir/Dialect/Linalg/TransformOps/*.h",
+    ]),
     includes = ["include"],
     deps = [
         ":AffineDialect",
@@ -8810,6 +8949,7 @@ cc_library(
         ":LinalgTransforms",
         ":LinalgUtils",
         ":PDLDialect",
+        ":SCFDialect",
         ":SCFTransforms",
         ":Support",
         ":TensorDialect",
@@ -8818,6 +8958,7 @@ cc_library(
         ":TransformDialect",
         ":TransformDialectUtils",
         ":TransformUtils",
+        ":VectorDialect",
         ":VectorTransforms",
         "//llvm:Support",
     ],
@@ -10360,6 +10501,7 @@ td_library(
     ],
     includes = ["include"],
     deps = [
+        ":Mem2RegInterfacesTdFiles",
         ":ArithOpsTdFiles",
         ":CastInterfacesTdFiles",
         ":ControlFlowInterfacesTdFiles",
@@ -10433,11 +10575,13 @@ cc_library(
     deps = [
         ":ArithDialect",
         ":ArithUtils",
+        ":ComplexDialect",
         ":ControlFlowInterfaces",
         ":CopyOpInterface",
         ":DialectUtils",
         ":IR",
         ":InferTypeOpInterface",
+        ":Mem2RegInterfaces",
         ":MemRefBaseIncGen",
         ":MemRefOpsIncGen",
         ":ShapedOpInterfaces",
@@ -10477,6 +10621,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":AffineDialect",
+        ":AffineTransforms",
         ":AffineUtils",
         ":ArithDialect",
         ":ArithTransforms",
@@ -10497,6 +10642,7 @@ cc_library(
         ":Support",
         ":TensorDialect",
         ":Transforms",
+        ":ValueBoundsOpInterface",
         ":VectorDialect",
         "//llvm:Support",
     ],
@@ -10549,6 +10695,7 @@ cc_library(
         ":MemRefTransforms",
         ":NVGPUDialect",
         ":PDLDialect",
+        ":SCFDialect",
         ":TransformDialect",
         ":TransformUtils",
         ":VectorDialect",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 10403f57cbefa..8cc7b64ed8323 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -45,6 +45,7 @@ expand_template(
         "@MLIR_ENABLE_BINDINGS_PYTHON@": "0",
         "@MLIR_RUN_AMX_TESTS@": "0",
         "@MLIR_RUN_ARM_SVE_TESTS@": "0",
+        "@MLIR_RUN_ARM_SME_TESTS@": "0",
         "@MLIR_RUN_X86VECTOR_TESTS@": "0",
         "@MLIR_RUN_CUDA_TENSOR_CORE_TESTS@": "0",
         "@MLIR_RUN_CUDA_SM80_TESTS@": "0",
diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
index f5b95e2e30364..2ed41ad5282c9 100644
--- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
@@ -21,7 +21,6 @@ cc_test(
     deps = [
         "//llvm:Support",
         "//llvm:TestingSupport",
-        "//mlir:BreakpointManagers",
         "//mlir:Debug",
         "//mlir:IR",
         "//mlir:Support",
@@ -41,6 +40,7 @@ cc_test(
     deps = [
         "//llvm:Support",
         "//mlir:IR",
+        "//mlir:Parser",
         "//mlir:Support",
         "//mlir/test:TestDialect",
         "//third-party/unittest:gmock",